diff --git a/.github/workflows/release-binaries.yml b/.github/workflows/release-binaries.yml
index 75d4f419ab1fdc1e4f1089951824afa20ca876c0..4e3eaff97a87832065710048a269100f5a922bd9 100644
--- a/.github/workflows/release-binaries.yml
+++ b/.github/workflows/release-binaries.yml
@@ -15,6 +15,9 @@ on:
         description: 'Tag to build'
         required: true
         type: string
+  schedule:
+    # * is a special character in YAML so you have to quote this string
+    - cron:  '0 8 1 * *'
 
 permissions:
   contents: read # Default everything to read-only
@@ -26,7 +29,7 @@ jobs:
     if: github.repository == 'llvm/llvm-project'
     outputs:
       release-version: ${{ steps.validate-tag.outputs.release-version }}
-      release: ${{ steps.validate-tag.outputs.release }}
+      flags: ${{ steps.validate-tag.outputs.flags }}
       build-dir: ${{ steps.validate-tag.outputs.build-dir }}
       rc-flags: ${{ steps.validate-tag.outputs.rc-flags }}
       ref: ${{ steps.validate-tag.outputs.ref }}
@@ -50,6 +53,11 @@ jobs:
         tag="${{ github.ref_name }}"
         trimmed=$(echo ${{ inputs.tag }} | xargs)
         [[ "$trimmed" != "" ]] && tag="$trimmed"
+        if [ "$tag" = "main" ]; then
+          # If tag is main, then we've been triggered by a scheduled so pass so
+          # use the head commit as the tag.
+          tag=`git rev-parse HEAD`
+        fi
         if [ -n "${{ inputs.upload }}" ]; then
           upload="${{ inputs.upload }}"
         else
@@ -71,7 +79,7 @@ jobs:
     - name: Checkout LLVM
       uses: actions/checkout@v4
       with:
-        ref: ${{ inputs.tag || github.ref_name }}
+        ref: ${{ needs.prepare.outputs.ref }}
 
     - name: Install Ninja
       uses: llvm/actions/install-ninja@main
@@ -140,8 +148,7 @@ jobs:
     - name: Build and test release
       run: |
         ${{ needs.prepare.outputs.build-dir }}/llvm-project/llvm/utils/release/test-release.sh \
-        -release ${{ needs.prepare.outputs.release }} \
-        ${{ needs.prepare.outputs.rc-flags }} \
+        ${{ needs.prepare.outputs.flags }} \
         -triple ${{ matrix.target.triple }} \
         -use-ninja \
         -no-checkout \
diff --git a/.github/workflows/set-release-binary-outputs.sh b/.github/workflows/set-release-binary-outputs.sh
index 8a7944e7e55fa06a4ddb5443df00021d00ae324a..9bc459a24e80194d4f5fa076e5d3a8939204f668 100644
--- a/.github/workflows/set-release-binary-outputs.sh
+++ b/.github/workflows/set-release-binary-outputs.sh
@@ -16,19 +16,32 @@ if [[ "$github_user" != "tstellar" && "$github_user" != "tru" ]]; then
   echo "ERROR: User not allowed: $github_user"
   exit 1
 fi
-pattern='^llvmorg-[0-9]\+\.[0-9]\+\.[0-9]\+\(-rc[0-9]\+\)\?$'
-echo "$tag" | grep -e $pattern
-if [ $? != 0 ]; then
-  echo "ERROR: Tag '$tag' doesn't match pattern: $pattern"
-  exit 1
+
+if echo $tag | grep -e '^[0-9a-f]\+$'; then
+  # This is a plain commit.
+  # TODO: Don't hardcode this.
+  release_version="18"
+  build_dir="$tag"
+  upload='false'
+  ref="$tag"
+  flags="-git-ref $tag -test-asserts"
+
+else
+
+  pattern='^llvmorg-[0-9]\+\.[0-9]\+\.[0-9]\+\(-rc[0-9]\+\)\?$'
+  echo "$tag" | grep -e $pattern
+  if [ $? != 0 ]; then
+    echo "ERROR: Tag '$tag' doesn't match pattern: $pattern"
+    exit 1
+  fi
+  release_version=`echo "$tag" | sed 's/llvmorg-//g'`
+  release=`echo "$release_version" | sed 's/-.*//g'`
+  build_dir=`echo "$release_version" | sed 's,^[^-]\+,final,' | sed 's,[^-]\+-rc\(.\+\),rc\1,'`
+  rc_flags=`echo "$release_version" | sed 's,^[^-]\+,-final,' | sed 's,[^-]\+-rc\(.\+\),-rc \1 -test-asserts,' | sed 's,--,-,'`
+  flags="-release $release $rc_flags"
 fi
-release_version=`echo "$tag" | sed 's/llvmorg-//g'`
-release=`echo "$release_version" | sed 's/-.*//g'`
-build_dir=`echo "$release_version" | sed 's,^[^-]\+,final,' | sed 's,[^-]\+-rc\(.\+\),rc\1,'`
-rc_flags=`echo "$release_version" | sed 's,^[^-]\+,-final,' | sed 's,[^-]\+-rc\(.\+\),-rc \1 -test-asserts,' | sed 's,--,-,'`
 echo "release-version=$release_version" >> $GITHUB_OUTPUT
-echo "release=$release" >> $GITHUB_OUTPUT
 echo "build-dir=$build_dir" >> $GITHUB_OUTPUT
-echo "rc-flags=$rc_flags" >> $GITHUB_OUTPUT
+echo "flags=$flags" >> $GITHUB_OUTPUT
 echo "upload=$upload" >> $GITHUB_OUTPUT
 echo "ref=$tag" >> $GITHUB_OUTPUT
diff --git a/bolt/include/bolt/Core/BinaryContext.h b/bolt/include/bolt/Core/BinaryContext.h
index c7672285b06c09f8f99123e34e8e35dd8793c21a..f0e7a8272ad0ee491706ce8d24cb2dfcecd89d21 100644
--- a/bolt/include/bolt/Core/BinaryContext.h
+++ b/bolt/include/bolt/Core/BinaryContext.h
@@ -900,8 +900,8 @@ public:
   /// Return true if \p SymbolName was generated internally and was not present
   /// in the input binary.
   bool isInternalSymbolName(const StringRef Name) {
-    return Name.startswith("SYMBOLat") || Name.startswith("DATAat") ||
-           Name.startswith("HOLEat");
+    return Name.starts_with("SYMBOLat") || Name.starts_with("DATAat") ||
+           Name.starts_with("HOLEat");
   }
 
   MCSymbol *getHotTextStartSymbol() const {
diff --git a/bolt/lib/Core/BinaryContext.cpp b/bolt/lib/Core/BinaryContext.cpp
index 7b78c9ba30a3032c554bc0784f8aafebd78294bc..3f96ea265e425f72380585aa6f8a08d2ac4deaa9 100644
--- a/bolt/lib/Core/BinaryContext.cpp
+++ b/bolt/lib/Core/BinaryContext.cpp
@@ -1086,7 +1086,7 @@ void BinaryContext::generateSymbolHashes() {
   auto isPadding = [](const BinaryData &BD) {
     StringRef Contents = BD.getSection().getContents();
     StringRef SymData = Contents.substr(BD.getOffset(), BD.getSize());
-    return (BD.getName().startswith("HOLEat") ||
+    return (BD.getName().starts_with("HOLEat") ||
             SymData.find_first_not_of(0) == StringRef::npos);
   };
 
@@ -1326,8 +1326,8 @@ void BinaryContext::postProcessSymbolTable() {
   bool Valid = true;
   for (auto &Entry : BinaryDataMap) {
     BinaryData *BD = Entry.second;
-    if ((BD->getName().startswith("SYMBOLat") ||
-         BD->getName().startswith("DATAat")) &&
+    if ((BD->getName().starts_with("SYMBOLat") ||
+         BD->getName().starts_with("DATAat")) &&
         !BD->getParent() && !BD->getSize() && !BD->isAbsolute() &&
         BD->getSection()) {
       errs() << "BOLT-WARNING: zero-sized top level symbol: " << *BD << "\n";
@@ -1410,9 +1410,9 @@ void BinaryContext::fixBinaryDataHoles() {
     auto isNotHole = [&Section](const binary_data_iterator &Itr) {
       BinaryData *BD = Itr->second;
       bool isHole = (!BD->getParent() && !BD->getSize() && BD->isObject() &&
-                     (BD->getName().startswith("SYMBOLat0x") ||
-                      BD->getName().startswith("DATAat0x") ||
-                      BD->getName().startswith("ANONYMOUS")));
+                     (BD->getName().starts_with("SYMBOLat0x") ||
+                      BD->getName().starts_with("DATAat0x") ||
+                      BD->getName().starts_with("ANONYMOUS")));
       return !isHole && BD->getSection() == Section && !BD->getParent();
     };
 
@@ -1818,14 +1818,14 @@ MarkerSymType BinaryContext::getMarkerType(const SymbolRef &Symbol) const {
   if (*TypeOrError != SymbolRef::ST_Unknown)
     return MarkerSymType::NONE;
 
-  if (*NameOrError == "$x" || NameOrError->startswith("$x."))
+  if (*NameOrError == "$x" || NameOrError->starts_with("$x."))
     return MarkerSymType::CODE;
 
   // $x<ISA>
-  if (isRISCV() && NameOrError->startswith("$x"))
+  if (isRISCV() && NameOrError->starts_with("$x"))
     return MarkerSymType::CODE;
 
-  if (*NameOrError == "$d" || NameOrError->startswith("$d."))
+  if (*NameOrError == "$d" || NameOrError->starts_with("$d."))
     return MarkerSymType::DATA;
 
   return MarkerSymType::NONE;
diff --git a/bolt/lib/Core/BinaryData.cpp b/bolt/lib/Core/BinaryData.cpp
index f963406c17d58cedf4000e30fcdcea4d9b4dc4d0..0068a935800429f6b9f4cf0c56cef813f0af0486 100644
--- a/bolt/lib/Core/BinaryData.cpp
+++ b/bolt/lib/Core/BinaryData.cpp
@@ -65,7 +65,7 @@ bool BinaryData::hasNameRegex(StringRef NameRegex) const {
 
 bool BinaryData::nameStartsWith(StringRef Prefix) const {
   for (const MCSymbol *Symbol : Symbols)
-    if (Symbol->getName().startswith(Prefix))
+    if (Symbol->getName().starts_with(Prefix))
       return true;
   return false;
 }
diff --git a/bolt/lib/Passes/IndirectCallPromotion.cpp b/bolt/lib/Passes/IndirectCallPromotion.cpp
index f40f5e7acbf3b8bf0b5aae459011f7f60efeb591..451758161ef5e69ed69608b0b86bf8a60fcc9706 100644
--- a/bolt/lib/Passes/IndirectCallPromotion.cpp
+++ b/bolt/lib/Passes/IndirectCallPromotion.cpp
@@ -460,7 +460,7 @@ IndirectCallPromotion::maybeGetHotJumpTableTargets(BinaryBasicBlock &BB,
 
     if (AccessInfo.MemoryObject) {
       // Deal with bad/stale data
-      if (!AccessInfo.MemoryObject->getName().startswith(
+      if (!AccessInfo.MemoryObject->getName().starts_with(
               "JUMP_TABLE/" + Function.getOneName().str()))
         return JumpTableInfoType();
       Index =
diff --git a/bolt/lib/Passes/ReorderData.cpp b/bolt/lib/Passes/ReorderData.cpp
index dc89b383552aba7224cd145993d73c5928d6bdc9..3a6654cf1e0b560e0ed6063811a329d96d693a92 100644
--- a/bolt/lib/Passes/ReorderData.cpp
+++ b/bolt/lib/Passes/ReorderData.cpp
@@ -408,14 +408,14 @@ bool ReorderData::markUnmoveableSymbols(BinaryContext &BC,
   // suffix in another.
   auto isPrivate = [&](const BinaryData *BD) {
     auto Prefix = std::string("PG") + BC.AsmInfo->getPrivateGlobalPrefix();
-    return BD->getName().startswith(Prefix.str());
+    return BD->getName().starts_with(Prefix.str());
   };
   auto Range = BC.getBinaryDataForSection(Section);
   bool FoundUnmoveable = false;
   for (auto Itr = Range.begin(); Itr != Range.end(); ++Itr) {
     BinaryData *Next =
         std::next(Itr) != Range.end() ? std::next(Itr)->second : nullptr;
-    if (Itr->second->getName().startswith("PG.")) {
+    if (Itr->second->getName().starts_with("PG.")) {
       BinaryData *Prev =
           Itr != Range.begin() ? std::prev(Itr)->second : nullptr;
       bool PrevIsPrivate = Prev && isPrivate(Prev);
diff --git a/bolt/lib/Passes/ShrinkWrapping.cpp b/bolt/lib/Passes/ShrinkWrapping.cpp
index 2dd57f3bbf3d25b6939c31ae3d1e10826abafdb4..d7b25c1279dc8b25b6aeebc97ebf34df69a0e496 100644
--- a/bolt/lib/Passes/ShrinkWrapping.cpp
+++ b/bolt/lib/Passes/ShrinkWrapping.cpp
@@ -1416,12 +1416,12 @@ bool ShrinkWrapping::foldIdenticalSplitEdges() {
   bool Changed = false;
   for (auto Iter = BF.begin(); Iter != BF.end(); ++Iter) {
     BinaryBasicBlock &BB = *Iter;
-    if (!BB.getName().startswith(".LSplitEdge"))
+    if (!BB.getName().starts_with(".LSplitEdge"))
       continue;
     for (BinaryBasicBlock &RBB : llvm::reverse(BF)) {
       if (&RBB == &BB)
         break;
-      if (!RBB.getName().startswith(".LSplitEdge") || !RBB.isValid() ||
+      if (!RBB.getName().starts_with(".LSplitEdge") || !RBB.isValid() ||
           !isIdenticalSplitEdgeBB(BC, *Iter, RBB))
         continue;
       assert(RBB.pred_size() == 1 && "Invalid split edge BB");
diff --git a/bolt/lib/Profile/DataAggregator.cpp b/bolt/lib/Profile/DataAggregator.cpp
index 0b306b9965cc3dfb50bd3ddc3cf8109c3b6b006f..be1e348b338f0f025675bd0b844a68f3c412a4b9 100644
--- a/bolt/lib/Profile/DataAggregator.cpp
+++ b/bolt/lib/Profile/DataAggregator.cpp
@@ -1915,7 +1915,7 @@ DataAggregator::parseMMapEvent() {
   //   PERF_RECORD_MMAP2 <pid>/<tid>: [<hexbase>(<hexsize>) .*]: .* <file_name>
 
   StringRef FileName = Line.rsplit(FieldSeparator).second;
-  if (FileName.startswith("//") || FileName.startswith("[")) {
+  if (FileName.starts_with("//") || FileName.starts_with("[")) {
     consumeRestOfLine();
     return std::make_pair(StringRef(), ParsedInfo);
   }
@@ -2168,7 +2168,7 @@ DataAggregator::getFileNameForBuildID(StringRef FileBuildID) {
       continue;
     }
 
-    if (IDPair->second.startswith(FileBuildID)) {
+    if (IDPair->second.starts_with(FileBuildID)) {
       FileName = sys::path::filename(IDPair->first);
       break;
     }
diff --git a/bolt/lib/Profile/DataReader.cpp b/bolt/lib/Profile/DataReader.cpp
index 3d4db6feb51a72eb53ca287fe1c9a5fd0229b45d..aa21eb121ad65256695dfff0318058879c0b203c 100644
--- a/bolt/lib/Profile/DataReader.cpp
+++ b/bolt/lib/Profile/DataReader.cpp
@@ -55,7 +55,7 @@ bool hasVolatileName(const BinaryFunction &BF) {
 /// Return standard escaped name of the function possibly renamed by BOLT.
 std::string normalizeName(StringRef NameRef) {
   // Strip "PG." prefix used for globalized locals.
-  NameRef = NameRef.startswith("PG.") ? NameRef.substr(2) : NameRef;
+  NameRef = NameRef.starts_with("PG.") ? NameRef.substr(2) : NameRef;
   return getEscapedName(NameRef);
 }
 
diff --git a/bolt/lib/Profile/YAMLProfileReader.cpp b/bolt/lib/Profile/YAMLProfileReader.cpp
index ade562ef6fb11162cc52eda683ab3c8179dcfa5c..a4a401fd3cabf40278975d74b5713cd6d3670824 100644
--- a/bolt/lib/Profile/YAMLProfileReader.cpp
+++ b/bolt/lib/Profile/YAMLProfileReader.cpp
@@ -39,7 +39,7 @@ namespace bolt {
 bool YAMLProfileReader::isYAML(const StringRef Filename) {
   if (auto MB = MemoryBuffer::getFileOrSTDIN(Filename)) {
     StringRef Buffer = (*MB)->getBuffer();
-    return Buffer.startswith("---\n");
+    return Buffer.starts_with("---\n");
   } else {
     report_error(Filename, MB.getError());
   }
diff --git a/bolt/lib/Rewrite/ExecutableFileMemoryManager.cpp b/bolt/lib/Rewrite/ExecutableFileMemoryManager.cpp
index aa5c8344aa6f5428202050e558ad49dc6886ad91..041d0d8c2b274fecff51d7f20b4348ffac58422c 100644
--- a/bolt/lib/Rewrite/ExecutableFileMemoryManager.cpp
+++ b/bolt/lib/Rewrite/ExecutableFileMemoryManager.cpp
@@ -120,7 +120,7 @@ void ExecutableFileMemoryManager::updateSection(
   }
 
   if (!IsCode && (SectionName == ".strtab" || SectionName == ".symtab" ||
-                  SectionName == "" || SectionName.startswith(".rela.")))
+                  SectionName == "" || SectionName.starts_with(".rela.")))
     return;
 
   SmallVector<char, 256> Buf;
@@ -139,7 +139,7 @@ void ExecutableFileMemoryManager::updateSection(
   }
 
   BinarySection *Section = nullptr;
-  if (!OrgSecPrefix.empty() && SectionName.startswith(OrgSecPrefix)) {
+  if (!OrgSecPrefix.empty() && SectionName.starts_with(OrgSecPrefix)) {
     // Update the original section contents.
     ErrorOr<BinarySection &> OrgSection =
         BC.getUniqueSectionByName(SectionName.substr(OrgSecPrefix.length()));
diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp
index 1e8ca569682f712cd4a2ad3cbabda406d8b4f277..a95b1650753cfdc73584080dff0a687ecce0d175 100644
--- a/bolt/lib/Rewrite/RewriteInstance.cpp
+++ b/bolt/lib/Rewrite/RewriteInstance.cpp
@@ -538,7 +538,7 @@ Error RewriteInstance::discoverStorage() {
 
     if (!opts::HeatmapMode &&
         !(opts::AggregateOnly && BAT->enabledFor(InputFile)) &&
-        (SectionName.startswith(getOrgSecPrefix()) ||
+        (SectionName.starts_with(getOrgSecPrefix()) ||
          SectionName == getBOLTTextSectionName()))
       return createStringError(
           errc::function_not_supported,
@@ -778,12 +778,12 @@ void RewriteInstance::discoverFileObjects() {
   std::unordered_map<SymbolRef, StringRef, SymbolRefHash> SymbolToFileName;
   for (const ELFSymbolRef &Symbol : InputFile->symbols()) {
     Expected<StringRef> NameOrError = Symbol.getName();
-    if (NameOrError && NameOrError->startswith("__asan_init")) {
+    if (NameOrError && NameOrError->starts_with("__asan_init")) {
       errs() << "BOLT-ERROR: input file was compiled or linked with sanitizer "
                 "support. Cannot optimize.\n";
       exit(1);
     }
-    if (NameOrError && NameOrError->startswith("__llvm_coverage_mapping")) {
+    if (NameOrError && NameOrError->starts_with("__llvm_coverage_mapping")) {
       errs() << "BOLT-ERROR: input file was compiled or linked with coverage "
                 "support. Cannot optimize.\n";
       exit(1);
@@ -938,9 +938,10 @@ void RewriteInstance::discoverFileObjects() {
     /// It is possible we are seeing a globalized local. LLVM might treat it as
     /// a local if it has a "private global" prefix, e.g. ".L". Thus we have to
     /// change the prefix to enforce global scope of the symbol.
-    std::string Name = SymName.startswith(BC->AsmInfo->getPrivateGlobalPrefix())
-                           ? "PG" + std::string(SymName)
-                           : std::string(SymName);
+    std::string Name =
+        SymName.starts_with(BC->AsmInfo->getPrivateGlobalPrefix())
+            ? "PG" + std::string(SymName)
+            : std::string(SymName);
 
     // Disambiguate all local symbols before adding to symbol table.
     // Since we don't know if we will see a global with the same name,
@@ -2723,8 +2724,8 @@ void RewriteInstance::handleRelocation(const SectionRef &RelocatedSection,
               BD->nameStartsWith(SymbolName) ||
               BD->nameStartsWith("PG" + SymbolName) ||
               (BD->nameStartsWith("ANONYMOUS") &&
-               (BD->getSectionName().startswith(".plt") ||
-                BD->getSectionName().endswith(".plt")))) &&
+               (BD->getSectionName().starts_with(".plt") ||
+                BD->getSectionName().ends_with(".plt")))) &&
              "BOLT symbol names of all non-section relocations must match up "
              "with symbol names referenced in the relocation");
 
@@ -2740,7 +2741,7 @@ void RewriteInstance::handleRelocation(const SectionRef &RelocatedSection,
       // in relocation sections can get through here too, from .plt.
       assert(
           (IsAArch64 || BC->isRISCV() || IsSectionRelocation ||
-           BC->getSectionNameForAddress(SymbolAddress)->startswith(".plt")) &&
+           BC->getSectionNameForAddress(SymbolAddress)->starts_with(".plt")) &&
           "known symbols should not resolve to anonymous locals");
 
       if (IsSectionRelocation) {
@@ -2757,7 +2758,7 @@ void RewriteInstance::handleRelocation(const SectionRef &RelocatedSection,
           Name = SymbolName;
         } else {
           if (StringRef(SymbolName)
-                  .startswith(BC->AsmInfo->getPrivateGlobalPrefix()))
+                  .starts_with(BC->AsmInfo->getPrivateGlobalPrefix()))
             Name = NR.uniquify("PG" + SymbolName);
           else
             Name = NR.uniquify(SymbolName);
@@ -3464,8 +3465,8 @@ std::vector<BinarySection *> RewriteInstance::getCodeSections() {
     //   ".text.cold.T", ".text.cold.T-1", ... ".text.cold.1", ".text.cold"
     // - if opts::HotFunctionsAtEnd is false, we want order
     //   ".text.cold", ".text.cold.1", ... ".text.cold.T-1", ".text.cold.T"
-    if (A->getName().startswith(BC->getColdCodeSectionName()) &&
-        B->getName().startswith(BC->getColdCodeSectionName())) {
+    if (A->getName().starts_with(BC->getColdCodeSectionName()) &&
+        B->getName().starts_with(BC->getColdCodeSectionName())) {
       if (A->getName().size() != B->getName().size())
         return (opts::HotFunctionsAtEnd)
                    ? (A->getName().size() > B->getName().size())
@@ -5653,16 +5654,16 @@ bool RewriteInstance::willOverwriteSection(StringRef SectionName) {
 }
 
 bool RewriteInstance::isDebugSection(StringRef SectionName) {
-  if (SectionName.startswith(".debug_") || SectionName.startswith(".zdebug_") ||
-      SectionName == ".gdb_index" || SectionName == ".stab" ||
-      SectionName == ".stabstr")
+  if (SectionName.starts_with(".debug_") ||
+      SectionName.starts_with(".zdebug_") || SectionName == ".gdb_index" ||
+      SectionName == ".stab" || SectionName == ".stabstr")
     return true;
 
   return false;
 }
 
 bool RewriteInstance::isKSymtabSection(StringRef SectionName) {
-  if (SectionName.startswith("__ksymtab"))
+  if (SectionName.starts_with("__ksymtab"))
     return true;
 
   return false;
diff --git a/bolt/test/X86/dwarf-test-df-logging.test b/bolt/test/X86/dwarf-test-df-logging.test
index ca5e578a0845fd76c525d345851ac54233fdc408..6126e9628a31a99112b98466156ec5005e8d72dd 100644
--- a/bolt/test/X86/dwarf-test-df-logging.test
+++ b/bolt/test/X86/dwarf-test-df-logging.test
@@ -6,7 +6,7 @@
 ; RUN:  -split-dwarf-file=main.dwo -o main.o
 ; RUN: llvm-mc -dwarf-version=4 -filetype=obj -triple x86_64-unknown-linux %p/Inputs/dwarf4-df-dualcu-helper.s \
 ; RUN:  -split-dwarf-file=helper.dwo -o helper.o
-; RUN: %clang %cflags -gdwarf-5 -gsplit-dwarf=split main.o helper.o -o main.exe
+; RUN: %clang %cflags -gdwarf-5 -gsplit-dwarf=split main.o helper.o -o main.exe -fno-pic -no-pie
 ; RUN: llvm-bolt main.exe -o main.exe.bolt --update-debug-sections | FileCheck -check-prefix=BOLT %s
 
 ; BOLT: BOLT-INFO: processing split DWARF
diff --git a/bolt/test/X86/dwarf4-df-dualcu.test b/bolt/test/X86/dwarf4-df-dualcu.test
index 5564d9ff26450472f672b16fee69951b38e53bf7..91b3e9e4cf0926c69eb894ae7c651571b111bdd0 100644
--- a/bolt/test/X86/dwarf4-df-dualcu.test
+++ b/bolt/test/X86/dwarf4-df-dualcu.test
@@ -5,7 +5,7 @@
 ; RUN: -split-dwarf-file=main.dwo -o main.o
 ; RUN: llvm-mc -dwarf-version=4 -filetype=obj -triple x86_64-unknown-linux %p/Inputs/dwarf4-df-dualcu-helper.s \
 ; RUN: -split-dwarf-file=helper.dwo -o helper.o
-; RUN: %clang %cflags -gdwarf-5 -gsplit-dwarf=split main.o helper.o -o main.exe
+; RUN: %clang %cflags -gdwarf-5 -gsplit-dwarf=split main.o helper.o -o main.exe -fno-pic -no-pie
 ; RUN: llvm-bolt main.exe -o main.exe.bolt --update-debug-sections --always-convert-to-ranges
 ; RUN: llvm-dwarfdump --show-form --verbose --debug-info main.exe | FileCheck -check-prefix=PRE-BOLT %s
 ; RUN: llvm-dwarfdump --show-form --verbose --debug-ranges main.exe.bolt &> %t/foo.txt
diff --git a/bolt/test/X86/dwarf4-split-dwarf-no-address.test b/bolt/test/X86/dwarf4-split-dwarf-no-address.test
index 5baef9238e9047dd37cdccaa9e42cf38c2676ae4..753fad06eb069fd477aaebe8ea4e77f5f331a64e 100644
--- a/bolt/test/X86/dwarf4-split-dwarf-no-address.test
+++ b/bolt/test/X86/dwarf4-split-dwarf-no-address.test
@@ -5,7 +5,7 @@
 ; RUN: --filetype=obj %p/Inputs/dwarf4-split-dwarf-no-address-main.s -o=main.o
 ; RUN: llvm-mc --split-dwarf-file=helper.dwo --triple=x86_64-unknown-linux-gnu \
 ; RUN: --filetype=obj %p/Inputs/dwarf4-split-dwarf-no-address-helper.s -o=helper.o
-; RUN: %clang %cflags -gdwarf-4 -gsplit-dwarf=split main.o helper.o -o main.exe
+; RUN: %clang %cflags -gdwarf-4 -gsplit-dwarf=split main.o helper.o -o main.exe -fno-pic -no-pie
 ; RUN: llvm-bolt main.exe -o main.exe.bolt --update-debug-sections
 ; RUN: llvm-dwarfdump --show-form --verbose --debug-info main.exe.bolt | FileCheck -check-prefix=BOLT %s
 
diff --git a/bolt/test/X86/dwarf5-df-dualcu.test b/bolt/test/X86/dwarf5-df-dualcu.test
index 15d458d1c0251fd21e5cdc48dee0a5f9171c8cf6..deaeea03669081f670e52d529428470a3f87af23 100644
--- a/bolt/test/X86/dwarf5-df-dualcu.test
+++ b/bolt/test/X86/dwarf5-df-dualcu.test
@@ -5,7 +5,7 @@
 ; RUN: -split-dwarf-file=main.dwo -o main.o
 ; RUN: llvm-mc -dwarf-version=5 -filetype=obj -triple x86_64-unknown-linux %p/Inputs/dwarf5-df-dualcu-helper.s \
 ; RUN: -split-dwarf-file=helper.dwo -o helper.o
-; RUN: %clang %cflags -gdwarf-5 -gsplit-dwarf=split main.o helper.o -o main.exe
+; RUN: %clang %cflags -gdwarf-5 -gsplit-dwarf=split main.o helper.o -o main.exe -fno-pic -no-pie
 ; RUN: llvm-bolt main.exe -o main.exe.bolt --update-debug-sections --always-convert-to-ranges
 ; RUN: llvm-dwarfdump --show-form --verbose --debug-info main.exe | FileCheck -check-prefix=PRE-BOLT %s
 ; RUN: llvm-dwarfdump --show-form --verbose --debug-addr main.exe.bolt &> %t/foo.txt
diff --git a/bolt/test/X86/dwarf5-df-mono-dualcu.test b/bolt/test/X86/dwarf5-df-mono-dualcu.test
index a6024997ba72d9bedd96433c970e28688a48a50d..12269287ef132f18fe66b8460cd2aa246674d49d 100644
--- a/bolt/test/X86/dwarf5-df-mono-dualcu.test
+++ b/bolt/test/X86/dwarf5-df-mono-dualcu.test
@@ -4,7 +4,7 @@
 ; RUN: llvm-mc -dwarf-version=5 -filetype=obj -triple x86_64-unknown-linux %p/Inputs/dwarf5-df-mono-main.s \
 ; RUN: -split-dwarf-file=main.dwo -o main.o
 ; RUN: llvm-mc -filetype=obj -triple x86_64-unknown-linux-gnu %p/Inputs/dwarf5-df-mono-helper.s -o=helper.o
-; RUN: %clang %cflags -gdwarf-5 main.o helper.o -o main.exe
+; RUN: %clang %cflags -gdwarf-5 main.o helper.o -o main.exe -fno-pic -no-pie
 ; RUN: llvm-bolt main.exe -o main.exe.bolt --update-debug-sections --always-convert-to-ranges
 ; RUN: llvm-dwarfdump --show-form --verbose --debug-info main.exe | FileCheck -check-prefix=PRE-BOLT %s
 ; RUN: llvm-dwarfdump --show-form --verbose --debug-addr main.exe.bolt &> %t/foo.txt
diff --git a/bolt/test/X86/dwarf5-locaddrx.test b/bolt/test/X86/dwarf5-locaddrx.test
index a1d66bb359d27ae72b52e3b16d6a3be1eafe305f..00e15101f85311235f9c27c66e38a48533b118a6 100644
--- a/bolt/test/X86/dwarf5-locaddrx.test
+++ b/bolt/test/X86/dwarf5-locaddrx.test
@@ -3,7 +3,7 @@
 ; RUN: cd %t
 ; RUN: llvm-mc -dwarf-version=5 -filetype=obj -triple x86_64-unknown-linux %p/Inputs/dwarf5-locaddrx.s \
 ; RUN: -split-dwarf-file=mainlocadddrx.dwo -o mainlocadddrx.o
-; RUN: %clang %cflags -gdwarf-5 -gsplit-dwarf=split mainlocadddrx.o -o mainlocadddrx.exe
+; RUN: %clang %cflags -gdwarf-5 -gsplit-dwarf=split mainlocadddrx.o -o mainlocadddrx.exe -fno-pic -no-pie
 ; RUN: llvm-bolt mainlocadddrx.exe -o mainlocadddrx.exe.bolt --update-debug-sections --always-convert-to-ranges
 ; RUN: llvm-dwarfdump --show-form --verbose --debug-info mainlocadddrx.exe | FileCheck -check-prefix=PRE-BOLT %s
 ; RUN: llvm-dwarfdump --show-form --verbose --debug-addr mainlocadddrx.exe.bolt &> %t/foo.txt
diff --git a/bolt/tools/merge-fdata/merge-fdata.cpp b/bolt/tools/merge-fdata/merge-fdata.cpp
index 757f0536616bec25812e517f39637e33fd4454ce..e21aeba73aca29a2154b209e53789114b7299e4a 100644
--- a/bolt/tools/merge-fdata/merge-fdata.cpp
+++ b/bolt/tools/merge-fdata/merge-fdata.cpp
@@ -253,7 +253,7 @@ bool isYAML(const StringRef Filename) {
   if (std::error_code EC = MB.getError())
     report_error(Filename, EC);
   StringRef Buffer = MB.get()->getBuffer();
-  if (Buffer.startswith("---\n"))
+  if (Buffer.starts_with("---\n"))
     return true;
   return false;
 }
@@ -279,7 +279,7 @@ void mergeLegacyProfiles(const SmallVectorImpl<std::string> &Filenames) {
     {
       std::lock_guard<std::mutex> Lock(BoltedCollectionMutex);
       // Check if the string "boltedcollection" is in the first line
-      if (Buf.startswith("boltedcollection\n")) {
+      if (Buf.starts_with("boltedcollection\n")) {
         if (!BoltedCollection.value_or(true))
           report_error(
               Filename,
diff --git a/clang-tools-extra/clang-change-namespace/ChangeNamespace.cpp b/clang-tools-extra/clang-change-namespace/ChangeNamespace.cpp
index 7f7af7069c418877bbbe54438fd26bae44c38046..879c0d26d472a8dc83fc7129bcd7158f16e2b62e 100644
--- a/clang-tools-extra/clang-change-namespace/ChangeNamespace.cpp
+++ b/clang-tools-extra/clang-change-namespace/ChangeNamespace.cpp
@@ -827,10 +827,10 @@ void ChangeNamespaceTool::replaceQualifiedSymbolInDeclContext(
       // "IsVisibleInNewNs" matcher.
       if (AliasQualifiedName != AliasName) {
         // The alias is defined in some namespace.
-        assert(StringRef(AliasQualifiedName).endswith("::" + AliasName));
+        assert(StringRef(AliasQualifiedName).ends_with("::" + AliasName));
         llvm::StringRef AliasNs =
             StringRef(AliasQualifiedName).drop_back(AliasName.size() + 2);
-        if (!llvm::StringRef(OldNs).startswith(AliasNs))
+        if (!llvm::StringRef(OldNs).starts_with(AliasNs))
           continue;
       }
       std::string NameWithAliasNamespace =
@@ -862,7 +862,7 @@ void ChangeNamespaceTool::replaceQualifiedSymbolInDeclContext(
   // If the new nested name in the new namespace is the same as it was in the
   // old namespace, we don't create replacement unless there can be ambiguity.
   if ((NestedName == ReplaceName && !Conflict) ||
-      (NestedName.startswith("::") && NestedName.drop_front(2) == ReplaceName))
+      (NestedName.starts_with("::") && NestedName.drop_front(2) == ReplaceName))
     return;
   // If the reference need to be fully-qualified, add a leading "::" unless
   // NewNamespace is the global namespace.
@@ -891,7 +891,7 @@ void ChangeNamespaceTool::fixTypeLoc(
   // a typedef type, we need to use the typedef type instead.
   auto IsInMovedNs = [&](const NamedDecl *D) {
     if (!llvm::StringRef(D->getQualifiedNameAsString())
-             .startswith(OldNamespace + "::"))
+             .starts_with(OldNamespace + "::"))
       return false;
     auto ExpansionLoc = Result.SourceManager->getExpansionLoc(D->getBeginLoc());
     if (ExpansionLoc.isInvalid())
diff --git a/clang-tools-extra/clang-include-fixer/IncludeFixerContext.cpp b/clang-tools-extra/clang-include-fixer/IncludeFixerContext.cpp
index f6f8404204ba06405ecb9dda2d80a8658c417b20..d7369b162dc10d8d87fc35f313212f819bc42101 100644
--- a/clang-tools-extra/clang-include-fixer/IncludeFixerContext.cpp
+++ b/clang-tools-extra/clang-include-fixer/IncludeFixerContext.cpp
@@ -28,7 +28,7 @@ std::string createQualifiedNameForReplacement(
     const find_all_symbols::SymbolInfo &MatchedSymbol) {
   // No need to add missing qualifiers if SymbolIdentifier has a global scope
   // operator "::".
-  if (RawSymbolName.startswith("::"))
+  if (RawSymbolName.starts_with("::"))
     return std::string(RawSymbolName);
 
   std::string QualifiedName = MatchedSymbol.getQualifiedName();
@@ -42,7 +42,7 @@ std::string createQualifiedNameForReplacement(
   auto SymbolQualifiers = SplitQualifiers(RawSymbolName);
   std::string StrippedQualifiers;
   while (!SymbolQualifiers.empty() &&
-         !llvm::StringRef(QualifiedName).endswith(SymbolQualifiers.back())) {
+         !llvm::StringRef(QualifiedName).ends_with(SymbolQualifiers.back())) {
     StrippedQualifiers =
         "::" + SymbolQualifiers.back().str() + StrippedQualifiers;
     SymbolQualifiers.pop_back();
diff --git a/clang-tools-extra/clang-include-fixer/SymbolIndexManager.cpp b/clang-tools-extra/clang-include-fixer/SymbolIndexManager.cpp
index 952c5fdedc291e51b3118c0e6f5c7cbfa6302727..027df3cfb2cc26760cc4289de99823d541da4d63 100644
--- a/clang-tools-extra/clang-include-fixer/SymbolIndexManager.cpp
+++ b/clang-tools-extra/clang-include-fixer/SymbolIndexManager.cpp
@@ -82,7 +82,7 @@ SymbolIndexManager::search(llvm::StringRef Identifier,
   Identifier.split(Names, "::");
 
   bool IsFullyQualified = false;
-  if (Identifier.startswith("::")) {
+  if (Identifier.starts_with("::")) {
     Names.erase(Names.begin()); // Drop first (empty) element.
     IsFullyQualified = true;
   }
diff --git a/clang-tools-extra/clang-include-fixer/find-all-symbols/PathConfig.cpp b/clang-tools-extra/clang-include-fixer/find-all-symbols/PathConfig.cpp
index 503fd60cb4c1b86ffb3b9e3584ad842a37e1c1d5..6d52963370ddd7ce89928b3742611ca017478339 100644
--- a/clang-tools-extra/clang-include-fixer/find-all-symbols/PathConfig.cpp
+++ b/clang-tools-extra/clang-include-fixer/find-all-symbols/PathConfig.cpp
@@ -24,7 +24,7 @@ std::string getIncludePath(const SourceManager &SM, SourceLocation Loc,
     FilePath = SM.getFilename(Loc);
     if (FilePath.empty())
       return "";
-    if (!FilePath.endswith(".inc"))
+    if (!FilePath.ends_with(".inc"))
       break;
     FileID ID = SM.getFileID(Loc);
     Loc = SM.getIncludeLoc(ID);
diff --git a/clang-tools-extra/clang-include-fixer/plugin/IncludeFixerPlugin.cpp b/clang-tools-extra/clang-include-fixer/plugin/IncludeFixerPlugin.cpp
index 7908a890fcb63c659804d7a058dc93940601b1c1..ce431d443be4b46aaaabdd8af343adc0f0d6279f 100644
--- a/clang-tools-extra/clang-include-fixer/plugin/IncludeFixerPlugin.cpp
+++ b/clang-tools-extra/clang-include-fixer/plugin/IncludeFixerPlugin.cpp
@@ -54,9 +54,9 @@ public:
     // Parse the extra command line args.
     // FIXME: This is very limited at the moment.
     for (StringRef Arg : Args) {
-      if (Arg.startswith("-db="))
+      if (Arg.starts_with("-db="))
         DB = Arg.substr(strlen("-db="));
-      else if (Arg.startswith("-input="))
+      else if (Arg.starts_with("-input="))
         Input = Arg.substr(strlen("-input="));
     }
 
diff --git a/clang-tools-extra/clang-tidy/ClangTidy.cpp b/clang-tools-extra/clang-tidy/ClangTidy.cpp
index 565f044778c946397261584fde73454c9549715a..40ac6918faf40786b1add8fb16e1c5933d76ca2e 100644
--- a/clang-tools-extra/clang-tidy/ClangTidy.cpp
+++ b/clang-tools-extra/clang-tidy/ClangTidy.cpp
@@ -390,7 +390,7 @@ static CheckersList getAnalyzerCheckersAndPackages(ClangTidyContext &Context,
   for (StringRef CheckName : RegisteredCheckers) {
     std::string ClangTidyCheckName((AnalyzerCheckNamePrefix + CheckName).str());
 
-    if (CheckName.startswith("core") ||
+    if (CheckName.starts_with("core") ||
         Context.isCheckEnabled(ClangTidyCheckName)) {
       List.emplace_back(std::string(CheckName), true);
     }
@@ -541,7 +541,7 @@ runClangTidy(clang::tidy::ClangTidyContext &Context,
         CommandLineArguments AdjustedArgs = Args;
         if (Opts.ExtraArgsBefore) {
           auto I = AdjustedArgs.begin();
-          if (I != AdjustedArgs.end() && !StringRef(*I).startswith("-"))
+          if (I != AdjustedArgs.end() && !StringRef(*I).starts_with("-"))
             ++I; // Skip compiler binary name, if it is there.
           AdjustedArgs.insert(I, Opts.ExtraArgsBefore->begin(),
                               Opts.ExtraArgsBefore->end());
diff --git a/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.cpp b/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.cpp
index b19a84f5dc215776f95920a6fb84315e78b2d965..0a80c996aaaade5fdb7e523ea231354d0cbb0e2f 100644
--- a/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.cpp
+++ b/clang-tools-extra/clang-tidy/ClangTidyDiagnosticConsumer.cpp
@@ -62,7 +62,7 @@ protected:
     // appending the check name to the message in ClangTidyContext::diag and
     // using getCustomDiagID.
     std::string CheckNameInMessage = " [" + Error.DiagnosticName + "]";
-    if (Message.endswith(CheckNameInMessage))
+    if (Message.ends_with(CheckNameInMessage))
       Message = Message.substr(0, Message.size() - CheckNameInMessage.size());
 
     auto TidyMessage =
@@ -457,7 +457,7 @@ bool ClangTidyDiagnosticConsumer::passesLineFilter(StringRef FileName,
   if (Context.getGlobalOptions().LineFilter.empty())
     return true;
   for (const FileFilter &Filter : Context.getGlobalOptions().LineFilter) {
-    if (FileName.endswith(Filter.Name)) {
+    if (FileName.ends_with(Filter.Name)) {
       if (Filter.LineRanges.empty())
         return true;
       for (const FileFilter::LineRange &Range : Filter.LineRanges) {
diff --git a/clang-tools-extra/clang-tidy/ExpandModularHeadersPPCallbacks.cpp b/clang-tools-extra/clang-tidy/ExpandModularHeadersPPCallbacks.cpp
index 0b1e9f59e1a70c7c27edbd005f6747ceb441bea6..e414ac8c770508fed662084f39a6e469eb42edeb 100644
--- a/clang-tools-extra/clang-tidy/ExpandModularHeadersPPCallbacks.cpp
+++ b/clang-tools-extra/clang-tidy/ExpandModularHeadersPPCallbacks.cpp
@@ -23,10 +23,10 @@ public:
   /// Records that a given file entry is needed for replaying callbacks.
   void addNecessaryFile(FileEntryRef File) {
     // Don't record modulemap files because it breaks same file detection.
-    if (!(File.getName().endswith("module.modulemap") ||
-          File.getName().endswith("module.private.modulemap") ||
-          File.getName().endswith("module.map") ||
-          File.getName().endswith("module_private.map")))
+    if (!(File.getName().ends_with("module.modulemap") ||
+          File.getName().ends_with("module.private.modulemap") ||
+          File.getName().ends_with("module.map") ||
+          File.getName().ends_with("module_private.map")))
       FilesToRecord.insert(File);
   }
 
diff --git a/clang-tools-extra/clang-tidy/GlobList.cpp b/clang-tools-extra/clang-tidy/GlobList.cpp
index 4ff9d951110630f4417e3b69b8c4cd54ee06db03..694db35106fd6363b20ab5624cb920c4eda9c108 100644
--- a/clang-tools-extra/clang-tidy/GlobList.cpp
+++ b/clang-tools-extra/clang-tidy/GlobList.cpp
@@ -16,7 +16,7 @@ namespace clang::tidy {
 // from the GlobList.
 static bool consumeNegativeIndicator(StringRef &GlobList) {
   GlobList = GlobList.trim();
-  if (GlobList.startswith("-")) {
+  if (GlobList.starts_with("-")) {
     GlobList = GlobList.substr(1);
     return true;
   }
diff --git a/clang-tools-extra/clang-tidy/abseil/AbseilMatcher.h b/clang-tools-extra/clang-tidy/abseil/AbseilMatcher.h
index 1827f54a2bc064819fbfb6d1752af796621097d8..1eef86ddc00b955e0984cf23b2a371036578de77 100644
--- a/clang-tools-extra/clang-tidy/abseil/AbseilMatcher.h
+++ b/clang-tools-extra/clang-tidy/abseil/AbseilMatcher.h
@@ -52,7 +52,7 @@ AST_POLYMORPHIC_MATCHER(
       "profiling", "random",   "status",    "strings",   "synchronization",
       "time",      "types",    "utility"};
   return llvm::any_of(AbseilLibraries, [&](const char *Library) {
-    return Path.startswith(Library);
+    return Path.starts_with(Library);
   });
 }
 
diff --git a/clang-tools-extra/clang-tidy/abseil/FasterStrsplitDelimiterCheck.cpp b/clang-tools-extra/clang-tidy/abseil/FasterStrsplitDelimiterCheck.cpp
index 74ef6e67f00f34213c070253c2feba7869b7ff76..4a6f17ed5f86891088e24a71382e1745cba71f25 100644
--- a/clang-tools-extra/clang-tidy/abseil/FasterStrsplitDelimiterCheck.cpp
+++ b/clang-tools-extra/clang-tidy/abseil/FasterStrsplitDelimiterCheck.cpp
@@ -27,7 +27,7 @@ std::optional<std::string> makeCharacterLiteral(const StringLiteral *Literal,
   assert(Literal->getCharByteWidth() == 1 &&
          "StrSplit doesn't support wide char");
   std::string Result = clang::tooling::fixit::getText(*Literal, Context).str();
-  bool IsRawStringLiteral = StringRef(Result).startswith(R"(R")");
+  bool IsRawStringLiteral = StringRef(Result).starts_with(R"(R")");
   // Since raw string literal might contain unescaped non-printable characters,
   // we normalize them using `StringLiteral::outputString`.
   if (IsRawStringLiteral) {
diff --git a/clang-tools-extra/clang-tidy/bugprone/ArgumentCommentCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/ArgumentCommentCheck.cpp
index 2a367d737742dca9756c0ab0a7545258a2c960bf..8cdd5d0a564675aa7264f72cdf8984b02ef8d41d 100644
--- a/clang-tools-extra/clang-tidy/bugprone/ArgumentCommentCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/ArgumentCommentCheck.cpp
@@ -185,7 +185,7 @@ static bool sameName(StringRef InComment, StringRef InDecl, bool StrictMode) {
 static bool looksLikeExpectMethod(const CXXMethodDecl *Expect) {
   return Expect != nullptr && Expect->getLocation().isMacroID() &&
          Expect->getNameInfo().getName().isIdentifier() &&
-         Expect->getName().startswith("gmock_");
+         Expect->getName().starts_with("gmock_");
 }
 static bool areMockAndExpectMethods(const CXXMethodDecl *Mock,
                                     const CXXMethodDecl *Expect) {
diff --git a/clang-tools-extra/clang-tidy/bugprone/EasilySwappableParametersCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/EasilySwappableParametersCheck.cpp
index 7b02576a6093cb0eae04b1f6a717d0ce17005258..84e99c7fafc74b2930bea63155fa3a1506152713 100644
--- a/clang-tools-extra/clang-tidy/bugprone/EasilySwappableParametersCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/EasilySwappableParametersCheck.cpp
@@ -1550,7 +1550,7 @@ static bool isIgnoredParameter(const TheCheck &Check, const ParmVarDecl *Node) {
   if (!NodeTypeName.empty()) {
     if (llvm::any_of(Check.IgnoredParameterTypeSuffixes,
                      [NodeTypeName](StringRef E) {
-                       return !E.empty() && NodeTypeName.endswith(E);
+                       return !E.empty() && NodeTypeName.ends_with(E);
                      })) {
       LLVM_DEBUG(llvm::dbgs() << "\tType suffix ignored.\n");
       return true;
diff --git a/clang-tools-extra/clang-tidy/bugprone/NotNullTerminatedResultCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/NotNullTerminatedResultCheck.cpp
index 44db5029fd993629d43e11834668b30729b1ae05..977241e91b9a93b00a1572bbdd8e9df6c0581c67 100644
--- a/clang-tools-extra/clang-tidy/bugprone/NotNullTerminatedResultCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/NotNullTerminatedResultCheck.cpp
@@ -385,7 +385,7 @@ static bool isDestExprFix(const MatchFinder::MatchResult &Result,
 
   std::string TempTyStr = Dest->getType().getAsString();
   StringRef TyStr = TempTyStr;
-  if (TyStr.startswith("char") || TyStr.startswith("wchar_t"))
+  if (TyStr.starts_with("char") || TyStr.starts_with("wchar_t"))
     return false;
 
   Diag << FixItHint::CreateInsertion(Dest->getBeginLoc(), "(char *)");
@@ -721,8 +721,8 @@ void NotNullTerminatedResultCheck::registerMatchers(MatchFinder *Finder) {
 
     // Try to match with 'wchar_t' based function calls.
     std::string WcharHandlerFuncName =
-        "::" + (CC.Name.startswith("mem") ? "w" + CC.Name.str()
-                                          : "wcs" + CC.Name.substr(3).str());
+        "::" + (CC.Name.starts_with("mem") ? "w" + CC.Name.str()
+                                           : "wcs" + CC.Name.substr(3).str());
 
     return allOf(callee(functionDecl(
                      hasAnyName(CharHandlerFuncName, WcharHandlerFuncName))),
@@ -820,13 +820,13 @@ void NotNullTerminatedResultCheck::check(
   }
 
   StringRef Name = FunctionExpr->getDirectCallee()->getName();
-  if (Name.startswith("mem") || Name.startswith("wmem"))
+  if (Name.starts_with("mem") || Name.starts_with("wmem"))
     memoryHandlerFunctionFix(Name, Result);
   else if (Name == "strerror_s")
     strerror_sFix(Result);
-  else if (Name.endswith("ncmp"))
+  else if (Name.ends_with("ncmp"))
     ncmpFix(Name, Result);
-  else if (Name.endswith("xfrm"))
+  else if (Name.ends_with("xfrm"))
     xfrmFix(Name, Result);
 }
 
@@ -835,7 +835,7 @@ void NotNullTerminatedResultCheck::memoryHandlerFunctionFix(
   if (isCorrectGivenLength(Result))
     return;
 
-  if (Name.endswith("chr")) {
+  if (Name.ends_with("chr")) {
     memchrFix(Name, Result);
     return;
   }
@@ -849,13 +849,13 @@ void NotNullTerminatedResultCheck::memoryHandlerFunctionFix(
            "the result from calling '%0' is not null-terminated")
       << Name;
 
-  if (Name.endswith("cpy")) {
+  if (Name.ends_with("cpy")) {
     memcpyFix(Name, Result, Diag);
-  } else if (Name.endswith("cpy_s")) {
+  } else if (Name.ends_with("cpy_s")) {
     memcpy_sFix(Name, Result, Diag);
-  } else if (Name.endswith("move")) {
+  } else if (Name.ends_with("move")) {
     memmoveFix(Name, Result, Diag);
-  } else if (Name.endswith("move_s")) {
+  } else if (Name.ends_with("move_s")) {
     isDestCapacityFix(Result, Diag);
     lengthArgHandle(LengthHandleKind::Increase, Result, Diag);
   }
diff --git a/clang-tools-extra/clang-tidy/bugprone/ReservedIdentifierCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/ReservedIdentifierCheck.cpp
index 6eefde369c59e52ebd3810e26b9644002af9a153..7a06df454be99871dc170f2c0e412873d04e7a29 100644
--- a/clang-tools-extra/clang-tidy/bugprone/ReservedIdentifierCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/ReservedIdentifierCheck.cpp
@@ -81,7 +81,7 @@ static bool hasReservedDoubleUnderscore(StringRef Name,
                                         const LangOptions &LangOpts) {
   if (LangOpts.CPlusPlus)
     return Name.contains("__");
-  return Name.startswith("__");
+  return Name.starts_with("__");
 }
 
 static std::optional<std::string>
diff --git a/clang-tools-extra/clang-tidy/bugprone/SignalHandlerCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/SignalHandlerCheck.cpp
index fd3ca76e68a61bc9d41bfed9541d07a3a2429be6..902490f4d33c13bd3eb1f7addae6995937d0089c 100644
--- a/clang-tools-extra/clang-tidy/bugprone/SignalHandlerCheck.cpp
+++ b/clang-tools-extra/clang-tidy/bugprone/SignalHandlerCheck.cpp
@@ -282,7 +282,7 @@ bool isStandardFunction(const FunctionDecl *FD) {
 /// and every other statement that is declared in file ExprCXX.h.
 bool isCXXOnlyStmt(const Stmt *S) {
   StringRef Name = S->getStmtClassName();
-  if (Name.startswith("CXX"))
+  if (Name.starts_with("CXX"))
     return true;
   // Check for all other class names in ExprCXX.h that have no 'CXX' prefix.
   return isa<ArrayTypeTraitExpr, BuiltinBitCastExpr, CUDAKernelCallExpr,
diff --git a/clang-tools-extra/clang-tidy/google/AvoidCStyleCastsCheck.cpp b/clang-tools-extra/clang-tidy/google/AvoidCStyleCastsCheck.cpp
index 714ac0ee54dafb35ac7d7c567a7ecf15b54b167d..995961b0774802e34c05095a8985594a158a9fb0 100644
--- a/clang-tools-extra/clang-tidy/google/AvoidCStyleCastsCheck.cpp
+++ b/clang-tools-extra/clang-tidy/google/AvoidCStyleCastsCheck.cpp
@@ -148,14 +148,15 @@ void AvoidCStyleCastsCheck::check(const MatchFinder::MatchResult &Result) {
     return;
   // Ignore code in .c files and headers included from them, even if they are
   // compiled as C++.
-  if (getCurrentMainFile().endswith(".c"))
+  if (getCurrentMainFile().ends_with(".c"))
     return;
 
   SourceManager &SM = *Result.SourceManager;
 
   // Ignore code in .c files #included in other files (which shouldn't be done,
   // but people still do this for test and other purposes).
-  if (SM.getFilename(SM.getSpellingLoc(CastExpr->getBeginLoc())).endswith(".c"))
+  if (SM.getFilename(SM.getSpellingLoc(CastExpr->getBeginLoc()))
+          .ends_with(".c"))
     return;
 
   // Leave type spelling exactly as it was (unlike
diff --git a/clang-tools-extra/clang-tidy/google/UpgradeGoogletestCaseCheck.cpp b/clang-tools-extra/clang-tidy/google/UpgradeGoogletestCaseCheck.cpp
index ce57b9fc3cac10a5af6a7095d44487253f918978..805dcaf3ce4025274368993b004fdc48c322e69b 100644
--- a/clang-tools-extra/clang-tidy/google/UpgradeGoogletestCaseCheck.cpp
+++ b/clang-tools-extra/clang-tidy/google/UpgradeGoogletestCaseCheck.cpp
@@ -66,7 +66,7 @@ public:
       // recent enough version of Google Test.
       llvm::StringRef FileName = PP->getSourceManager().getFilename(
           MD->getMacroInfo()->getDefinitionLoc());
-      ReplacementFound = FileName.endswith("gtest/gtest-typed-test.h") &&
+      ReplacementFound = FileName.ends_with("gtest/gtest-typed-test.h") &&
                          PP->getSpelling(MacroNameTok) == "TYPED_TEST_SUITE";
     }
   }
@@ -102,7 +102,7 @@ private:
 
     llvm::StringRef FileName = PP->getSourceManager().getFilename(
         MD.getMacroInfo()->getDefinitionLoc());
-    if (!FileName.endswith("gtest/gtest-typed-test.h"))
+    if (!FileName.ends_with("gtest/gtest-typed-test.h"))
       return;
 
     DiagnosticBuilder Diag = Check->diag(Loc, RenameCaseToSuiteMessage);
diff --git a/clang-tools-extra/clang-tidy/google/UsingNamespaceDirectiveCheck.cpp b/clang-tools-extra/clang-tidy/google/UsingNamespaceDirectiveCheck.cpp
index 2f4913b2d435ac2bfa114399e94b66f6ccd83b66..c97bd48e6c3e3aff904db2d715255e1506fe465e 100644
--- a/clang-tools-extra/clang-tidy/google/UsingNamespaceDirectiveCheck.cpp
+++ b/clang-tools-extra/clang-tidy/google/UsingNamespaceDirectiveCheck.cpp
@@ -40,7 +40,7 @@ void UsingNamespaceDirectiveCheck::check(
 
 bool UsingNamespaceDirectiveCheck::isStdLiteralsNamespace(
     const NamespaceDecl *NS) {
-  if (!NS->getName().endswith("literals"))
+  if (!NS->getName().ends_with("literals"))
     return false;
 
   const auto *Parent = dyn_cast_or_null<NamespaceDecl>(NS->getParent());
diff --git a/clang-tools-extra/clang-tidy/llvm/HeaderGuardCheck.cpp b/clang-tools-extra/clang-tidy/llvm/HeaderGuardCheck.cpp
index 8d35ff56de781dff19be30b9590cbe448ebe9431..42d358a15083ab038f6b9eed73d7a6c4d52fe7ac 100644
--- a/clang-tools-extra/clang-tidy/llvm/HeaderGuardCheck.cpp
+++ b/clang-tools-extra/clang-tidy/llvm/HeaderGuardCheck.cpp
@@ -54,11 +54,11 @@ std::string LLVMHeaderGuardCheck::getHeaderGuard(StringRef Filename,
   std::replace(Guard.begin(), Guard.end(), '-', '_');
 
   // The prevalent style in clang is LLVM_CLANG_FOO_BAR_H
-  if (StringRef(Guard).startswith("clang"))
+  if (StringRef(Guard).starts_with("clang"))
     Guard = "LLVM_" + Guard;
 
   // The prevalent style in flang is FORTRAN_FOO_BAR_H
-  if (StringRef(Guard).startswith("flang"))
+  if (StringRef(Guard).starts_with("flang"))
     Guard = "FORTRAN" + Guard.substr(sizeof("flang") - 1);
 
   return StringRef(Guard).upper();
diff --git a/clang-tools-extra/clang-tidy/llvm/IncludeOrderCheck.cpp b/clang-tools-extra/clang-tidy/llvm/IncludeOrderCheck.cpp
index 3fb2e8daaebed9c9bb1dea517517eea1a8bd6d88..bdd72f85e2a27c66a611ddbf879b88cc230acd3b 100644
--- a/clang-tools-extra/clang-tidy/llvm/IncludeOrderCheck.cpp
+++ b/clang-tools-extra/clang-tidy/llvm/IncludeOrderCheck.cpp
@@ -61,13 +61,13 @@ static int getPriority(StringRef Filename, bool IsAngled, bool IsMainModule) {
     return 0;
 
   // LLVM and clang headers are in the penultimate position.
-  if (Filename.startswith("llvm/") || Filename.startswith("llvm-c/") ||
-      Filename.startswith("clang/") || Filename.startswith("clang-c/"))
+  if (Filename.starts_with("llvm/") || Filename.starts_with("llvm-c/") ||
+      Filename.starts_with("clang/") || Filename.starts_with("clang-c/"))
     return 2;
 
   // Put these between system and llvm headers to be consistent with LLVM
   // clang-format style.
-  if (Filename.startswith("gtest/") || Filename.startswith("gmock/"))
+  if (Filename.starts_with("gtest/") || Filename.starts_with("gmock/"))
     return 3;
 
   // System headers are sorted to the end.
diff --git a/clang-tools-extra/clang-tidy/misc/ConfusableTable/BuildConfusableTable.cpp b/clang-tools-extra/clang-tidy/misc/ConfusableTable/BuildConfusableTable.cpp
index 79ebb2387179ea4f5e433e1897616f793a01522f..e269ab3983f36cfe53e3806d95547d647aec2aa5 100644
--- a/clang-tools-extra/clang-tidy/misc/ConfusableTable/BuildConfusableTable.cpp
+++ b/clang-tools-extra/clang-tidy/misc/ConfusableTable/BuildConfusableTable.cpp
@@ -27,7 +27,7 @@ int main(int argc, char *argv[]) {
   std::vector<std::pair<llvm::UTF32, SmallVector<llvm::UTF32>>> Entries;
   SmallVector<StringRef> Values;
   for (StringRef Line : Lines) {
-    if (Line.startswith("#"))
+    if (Line.starts_with("#"))
       continue;
 
     Values.clear();
diff --git a/clang-tools-extra/clang-tidy/misc/IncludeCleanerCheck.cpp b/clang-tools-extra/clang-tidy/misc/IncludeCleanerCheck.cpp
index 5ae6caedb7f4c0368c755ae9d9dc89d1b6f8400a..5e7a0e65690b7aaa704cf5c033eb92c2a11127bc 100644
--- a/clang-tools-extra/clang-tidy/misc/IncludeCleanerCheck.cpp
+++ b/clang-tools-extra/clang-tidy/misc/IncludeCleanerCheck.cpp
@@ -180,7 +180,7 @@ void IncludeCleanerCheck::check(const MatchFinder::MatchResult &Result) {
       // Since most private -> public mappings happen in a verbatim way, we
       // check textually here. This might go wrong in presence of symlinks or
       // header mappings. But that's not different than rest of the places.
-      if (getCurrentMainFile().endswith(PHeader))
+      if (getCurrentMainFile().ends_with(PHeader))
         continue;
     }
     auto StdHeader = tooling::stdlib::Header::named(
diff --git a/clang-tools-extra/clang-tidy/misc/RedundantExpressionCheck.cpp b/clang-tools-extra/clang-tidy/misc/RedundantExpressionCheck.cpp
index fa8343220dd05a399a8358d80f81014793fefa69..6bb9a349d69b13f22d587cd4fa8120c4f8f6f2ae 100644
--- a/clang-tools-extra/clang-tidy/misc/RedundantExpressionCheck.cpp
+++ b/clang-tools-extra/clang-tidy/misc/RedundantExpressionCheck.cpp
@@ -1285,7 +1285,7 @@ void RedundantExpressionCheck::check(const MatchFinder::MatchResult &Result) {
 
     const auto Diag = diag(Op->getExprLoc(), Message);
     for (const auto &KeyValue : Result.Nodes.getMap()) {
-      if (StringRef(KeyValue.first).startswith("duplicate"))
+      if (StringRef(KeyValue.first).starts_with("duplicate"))
         Diag << KeyValue.second.getSourceRange();
     }
   }
diff --git a/clang-tools-extra/clang-tidy/modernize/LoopConvertUtils.cpp b/clang-tools-extra/clang-tidy/modernize/LoopConvertUtils.cpp
index e7e01bab0e9d5233c1a2ac1fa337833f938940a7..c0bf4903ec3911af762abf99f71d89947b09b732 100644
--- a/clang-tools-extra/clang-tidy/modernize/LoopConvertUtils.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/LoopConvertUtils.cpp
@@ -849,14 +849,14 @@ std::string VariableNamer::createIndexName() {
     ContainerName = TheContainer->getName();
 
   size_t Len = ContainerName.size();
-  if (Len > 1 && ContainerName.endswith(Style == NS_UpperCase ? "S" : "s")) {
+  if (Len > 1 && ContainerName.ends_with(Style == NS_UpperCase ? "S" : "s")) {
     IteratorName = std::string(ContainerName.substr(0, Len - 1));
     // E.g.: (auto thing : things)
     if (!declarationExists(IteratorName) || IteratorName == OldIndex->getName())
       return IteratorName;
   }
 
-  if (Len > 2 && ContainerName.endswith(Style == NS_UpperCase ? "S_" : "s_")) {
+  if (Len > 2 && ContainerName.ends_with(Style == NS_UpperCase ? "S_" : "s_")) {
     IteratorName = std::string(ContainerName.substr(0, Len - 2));
     // E.g.: (auto thing : things_)
     if (!declarationExists(IteratorName) || IteratorName == OldIndex->getName())
diff --git a/clang-tools-extra/clang-tidy/modernize/ReplaceRandomShuffleCheck.cpp b/clang-tools-extra/clang-tidy/modernize/ReplaceRandomShuffleCheck.cpp
index f5d13f31d7a644ddc6e55936910ba2ecd9df4764..6a295dbfd05820cc1f5a14b804ca174280fc26f7 100644
--- a/clang-tools-extra/clang-tidy/modernize/ReplaceRandomShuffleCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/ReplaceRandomShuffleCheck.cpp
@@ -83,7 +83,7 @@ void ReplaceRandomShuffleCheck::check(const MatchFinder::MatchResult &Result) {
   StringRef ContainerText = Lexer::getSourceText(
       CharSourceRange::getTokenRange(MatchedDecl->getSourceRange()),
       *Result.SourceManager, getLangOpts());
-  if (ContainerText.startswith("std::"))
+  if (ContainerText.starts_with("std::"))
     NewName = "std::" + NewName;
 
   Diag << FixItHint::CreateRemoval(MatchedDecl->getSourceRange());
diff --git a/clang-tools-extra/clang-tidy/modernize/UseEmplaceCheck.cpp b/clang-tools-extra/clang-tidy/modernize/UseEmplaceCheck.cpp
index b85dde5644d313fcb6d4e7c3c97610a9fd2491b2..4438f0b22063f5c7650769bee9b25b3819147448 100644
--- a/clang-tools-extra/clang-tidy/modernize/UseEmplaceCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/UseEmplaceCheck.cpp
@@ -41,11 +41,11 @@ AST_MATCHER_P(NamedDecl, hasAnyNameIgnoringTemplates, std::vector<StringRef>,
   // FullNameTrimmed matches any of the given Names.
   const StringRef FullNameTrimmedRef = FullNameTrimmed;
   for (const StringRef Pattern : Names) {
-    if (Pattern.startswith("::")) {
+    if (Pattern.starts_with("::")) {
       if (FullNameTrimmed == Pattern)
         return true;
-    } else if (FullNameTrimmedRef.endswith(Pattern) &&
-               FullNameTrimmedRef.drop_back(Pattern.size()).endswith("::")) {
+    } else if (FullNameTrimmedRef.ends_with(Pattern) &&
+               FullNameTrimmedRef.drop_back(Pattern.size()).ends_with("::")) {
       return true;
     }
   }
diff --git a/clang-tools-extra/clang-tidy/modernize/UseNodiscardCheck.cpp b/clang-tools-extra/clang-tidy/modernize/UseNodiscardCheck.cpp
index 299d4539a6ddcb01045f671b3e82891d0a2611af..6de80dcb99c60d374cc1362675f9a64c45ccd270 100644
--- a/clang-tools-extra/clang-tidy/modernize/UseNodiscardCheck.cpp
+++ b/clang-tools-extra/clang-tidy/modernize/UseNodiscardCheck.cpp
@@ -20,7 +20,7 @@ static bool doesNoDiscardMacroExist(ASTContext &Context,
                                     const llvm::StringRef &MacroId) {
   // Don't check for the Macro existence if we are using an attribute
   // either a C++17 standard attribute or pre C++17 syntax
-  if (MacroId.startswith("[[") || MacroId.startswith("__attribute__"))
+  if (MacroId.starts_with("[[") || MacroId.starts_with("__attribute__"))
     return true;
 
   // Otherwise look up the macro name in the context to see if its defined.
diff --git a/clang-tools-extra/clang-tidy/plugin/ClangTidyPlugin.cpp b/clang-tools-extra/clang-tidy/plugin/ClangTidyPlugin.cpp
index b47ce2b066c65298ccd98afc8779bee6a87b9b43..7911583db30e450bbadc2225c521396ef7849204 100644
--- a/clang-tools-extra/clang-tidy/plugin/ClangTidyPlugin.cpp
+++ b/clang-tools-extra/clang-tidy/plugin/ClangTidyPlugin.cpp
@@ -62,7 +62,7 @@ public:
     // Parse the extra command line args.
     // FIXME: This is very limited at the moment.
     for (StringRef Arg : Args)
-      if (Arg.startswith("-checks="))
+      if (Arg.starts_with("-checks="))
         OverrideOptions.Checks = std::string(Arg.substr(strlen("-checks=")));
 
     auto Options = std::make_unique<FileOptionsProvider>(
diff --git a/clang-tools-extra/clang-tidy/portability/SIMDIntrinsicsCheck.cpp b/clang-tools-extra/clang-tidy/portability/SIMDIntrinsicsCheck.cpp
index 4a3e43bda9172f44a812df0a767e4d89e7f585de..3e77a204d753eb2e3906ef2e93833fd2fbcdcab7 100644
--- a/clang-tools-extra/clang-tidy/portability/SIMDIntrinsicsCheck.cpp
+++ b/clang-tools-extra/clang-tidy/portability/SIMDIntrinsicsCheck.cpp
@@ -59,17 +59,17 @@ static StringRef trySuggestX86(StringRef Name) {
     return {};
 
   // [simd.alg]
-  if (Name.startswith("max_"))
+  if (Name.starts_with("max_"))
     return "$simd::max";
-  if (Name.startswith("min_"))
+  if (Name.starts_with("min_"))
     return "$simd::min";
 
   // [simd.binary]
-  if (Name.startswith("add_"))
+  if (Name.starts_with("add_"))
     return "operator+ on $simd objects";
-  if (Name.startswith("sub_"))
+  if (Name.starts_with("sub_"))
     return "operator- on $simd objects";
-  if (Name.startswith("mul_"))
+  if (Name.starts_with("mul_"))
     return "operator* on $simd objects";
 
   return {};
diff --git a/clang-tools-extra/clang-tidy/readability/BracesAroundStatementsCheck.cpp b/clang-tools-extra/clang-tidy/readability/BracesAroundStatementsCheck.cpp
index d8dbaa7c9d73c419f5641a4dc3f02549a7f82b72..81ca33cbbdfb4b5461b54b8a0b5903c0d1186942 100644
--- a/clang-tools-extra/clang-tidy/readability/BracesAroundStatementsCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/BracesAroundStatementsCheck.cpp
@@ -79,7 +79,7 @@ static SourceLocation findEndLocation(const Stmt &S, const SourceManager &SM,
     SourceRange TokRange(Loc, TokEndLoc);
     StringRef Comment = Lexer::getSourceText(
         CharSourceRange::getTokenRange(TokRange), SM, Context->getLangOpts());
-    if (Comment.startswith("/*") && Comment.contains('\n')) {
+    if (Comment.starts_with("/*") && Comment.contains('\n')) {
       // Multi-line block comment, insert brace before.
       break;
     }
diff --git a/clang-tools-extra/clang-tidy/readability/IdentifierNamingCheck.cpp b/clang-tools-extra/clang-tidy/readability/IdentifierNamingCheck.cpp
index 18c5e144e46fe77a00e06752b008a09d4bee5e34..03dcfa5f811095b0b2045a84c658c2d6828c62da 100644
--- a/clang-tools-extra/clang-tidy/readability/IdentifierNamingCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/IdentifierNamingCheck.cpp
@@ -889,7 +889,7 @@ bool IdentifierNamingCheck::matchesStyle(
 
   // Ensure the name doesn't have any extra underscores beyond those specified
   // in the prefix and suffix.
-  if (Name.startswith("_") || Name.endswith("_"))
+  if (Name.starts_with("_") || Name.ends_with("_"))
     return false;
 
   if (Style.Case && !Matchers[static_cast<size_t>(*Style.Case)].match(Name))
diff --git a/clang-tools-extra/clang-tidy/readability/IsolateDeclarationCheck.cpp b/clang-tools-extra/clang-tidy/readability/IsolateDeclarationCheck.cpp
index 9140b1f51631bb3fe26a7c66bcb14da19d39aca7..ca6503753f6b45a32ae4b9fa8f95124ead2e22e9 100644
--- a/clang-tools-extra/clang-tidy/readability/IsolateDeclarationCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/IsolateDeclarationCheck.cpp
@@ -235,7 +235,7 @@ createIsolatedDecls(llvm::ArrayRef<StringRef> Snippets) {
 
   for (std::size_t I = 1; I < Snippets.size(); ++I)
     Decls[I - 1] = Twine(Snippets[0])
-                       .concat(Snippets[0].endswith(" ") ? "" : " ")
+                       .concat(Snippets[0].ends_with(" ") ? "" : " ")
                        .concat(Snippets[I].ltrim())
                        .concat(";")
                        .str();
diff --git a/clang-tools-extra/clang-tidy/readability/NamespaceCommentCheck.cpp b/clang-tools-extra/clang-tidy/readability/NamespaceCommentCheck.cpp
index 68e9dd4998473b5bb4ec928685350e627a33c911..120ec02e9ad7dc8caa7544e5757f1068fea1a4ab 100644
--- a/clang-tools-extra/clang-tidy/readability/NamespaceCommentCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/NamespaceCommentCheck.cpp
@@ -160,7 +160,7 @@ void NamespaceCommentCheck::check(const MatchFinder::MatchResult &Result) {
       }
 
       // Otherwise we need to fix the comment.
-      NeedLineBreak = Comment.startswith("/*");
+      NeedLineBreak = Comment.starts_with("/*");
       OldCommentRange =
           SourceRange(AfterRBrace, Loc.getLocWithOffset(Tok.getLength()));
       Message =
@@ -168,7 +168,7 @@ void NamespaceCommentCheck::check(const MatchFinder::MatchResult &Result) {
                "%0 ends with a comment that refers to a wrong namespace '") +
            NamespaceNameInComment + "'")
               .str();
-    } else if (Comment.startswith("//")) {
+    } else if (Comment.starts_with("//")) {
       // Assume that this is an unrecognized form of a namespace closing line
       // comment. Replace it.
       NeedLineBreak = false;
diff --git a/clang-tools-extra/clang-tidy/readability/StaticAccessedThroughInstanceCheck.cpp b/clang-tools-extra/clang-tidy/readability/StaticAccessedThroughInstanceCheck.cpp
index 93c31d80c11381efeb6f8b6b42a9265afa825fc5..65356cc3929c54e7a28d2c63e2244d60a53d4dc3 100644
--- a/clang-tools-extra/clang-tidy/readability/StaticAccessedThroughInstanceCheck.cpp
+++ b/clang-tools-extra/clang-tidy/readability/StaticAccessedThroughInstanceCheck.cpp
@@ -85,7 +85,7 @@ void StaticAccessedThroughInstanceCheck::check(
     return;
 
   // Do not warn for CUDA built-in variables.
-  if (StringRef(BaseTypeName).startswith("__cuda_builtin_"))
+  if (StringRef(BaseTypeName).starts_with("__cuda_builtin_"))
     return;
 
   SourceLocation MemberExprStartLoc = MemberExpression->getBeginLoc();
diff --git a/clang-tools-extra/clang-tidy/tool/ClangTidyMain.cpp b/clang-tools-extra/clang-tidy/tool/ClangTidyMain.cpp
index 0d2593e74f052b9a3259d40f7e85387ec6524dcc..9f3d6b6db6cbca1b0cc9aa3b7c71b4746f34d712 100644
--- a/clang-tools-extra/clang-tidy/tool/ClangTidyMain.cpp
+++ b/clang-tools-extra/clang-tidy/tool/ClangTidyMain.cpp
@@ -462,7 +462,7 @@ static bool verifyChecks(const StringSet<> &AllChecks, StringRef CheckGlob,
     if (Cur.empty())
       continue;
     Cur.consume_front("-");
-    if (Cur.startswith("clang-diagnostic"))
+    if (Cur.starts_with("clang-diagnostic"))
       continue;
     if (Cur.contains('*')) {
       SmallString<128> RegexText("^");
diff --git a/clang-tools-extra/clang-tidy/utils/IncludeSorter.cpp b/clang-tools-extra/clang-tidy/utils/IncludeSorter.cpp
index eb21827bdeba3ccca719501e27361bd3a0ea7636..b6d9c50d0b109c1c24323e304500165a607287d3 100644
--- a/clang-tools-extra/clang-tidy/utils/IncludeSorter.cpp
+++ b/clang-tools-extra/clang-tidy/utils/IncludeSorter.cpp
@@ -19,7 +19,7 @@ namespace {
 
 StringRef removeFirstSuffix(StringRef Str, ArrayRef<const char *> Suffixes) {
   for (StringRef Suffix : Suffixes) {
-    if (Str.endswith(Suffix)) {
+    if (Str.ends_with(Suffix)) {
       return Str.substr(0, Str.size() - Suffix.size());
     }
   }
@@ -73,12 +73,12 @@ determineIncludeKind(StringRef CanonicalFile, StringRef IncludeFile,
   if (IsAngled) {
     // If the system include (<foo>) ends with ".h", then it is a normal C-style
     // include. Otherwise assume it is a C++-style extensionless include.
-    return IncludeFile.endswith(".h") ? IncludeSorter::IK_CSystemInclude
-                                      : IncludeSorter::IK_CXXSystemInclude;
+    return IncludeFile.ends_with(".h") ? IncludeSorter::IK_CSystemInclude
+                                       : IncludeSorter::IK_CXXSystemInclude;
   }
   StringRef CanonicalInclude = makeCanonicalName(IncludeFile, Style);
-  if (CanonicalFile.endswith(CanonicalInclude)
-      || CanonicalInclude.endswith(CanonicalFile)) {
+  if (CanonicalFile.ends_with(CanonicalInclude) ||
+      CanonicalInclude.ends_with(CanonicalFile)) {
     return IncludeSorter::IK_MainTUInclude;
   }
   if ((Style == IncludeSorter::IS_Google) ||
@@ -95,8 +95,9 @@ determineIncludeKind(StringRef CanonicalFile, StringRef IncludeFile,
     }
   }
   if (Style == IncludeSorter::IS_Google_ObjC) {
-    if (IncludeFile.endswith(".generated.h") ||
-        IncludeFile.endswith(".proto.h") || IncludeFile.endswith(".pbobjc.h")) {
+    if (IncludeFile.ends_with(".generated.h") ||
+        IncludeFile.ends_with(".proto.h") ||
+        IncludeFile.ends_with(".pbobjc.h")) {
       return IncludeSorter::IK_GeneratedInclude;
     }
   }
diff --git a/clang-tools-extra/clang-tidy/utils/Matchers.h b/clang-tools-extra/clang-tidy/utils/Matchers.h
index 386ea738fbba5088ba5ec57e988f4e4acf48139c..045e3ffbb6a8b45e0b24f877501fbc2d05493465 100644
--- a/clang-tools-extra/clang-tidy/utils/Matchers.h
+++ b/clang-tools-extra/clang-tidy/utils/Matchers.h
@@ -120,7 +120,7 @@ private:
 
   private:
     MatchMode determineMatchMode(llvm::StringRef Regex) {
-      if (Regex.startswith(":") || Regex.startswith("^:")) {
+      if (Regex.starts_with(":") || Regex.starts_with("^:")) {
         return MatchMode::MatchFullyQualified;
       }
       return Regex.contains(":") ? MatchMode::MatchQualified
diff --git a/clang-tools-extra/clangd/AST.cpp b/clang-tools-extra/clangd/AST.cpp
index 5b81ec213ff9844688d6b2db86b774ab5b60b3c5..ae79eb21de9470fa3a82aa24c0b147a9625b34b5 100644
--- a/clang-tools-extra/clangd/AST.cpp
+++ b/clang-tools-extra/clangd/AST.cpp
@@ -193,7 +193,7 @@ std::string printQualifiedName(const NamedDecl &ND) {
   Policy.AnonymousTagLocations = false;
   ND.printQualifiedName(OS, Policy);
   OS.flush();
-  assert(!StringRef(QName).startswith("::"));
+  assert(!StringRef(QName).starts_with("::"));
   return QName;
 }
 
@@ -696,7 +696,7 @@ std::string getQualification(ASTContext &Context,
                              const NamedDecl *ND,
                              llvm::ArrayRef<std::string> VisibleNamespaces) {
   for (llvm::StringRef NS : VisibleNamespaces) {
-    assert(NS.endswith("::"));
+    assert(NS.ends_with("::"));
     (void)NS;
   }
   return getQualification(
diff --git a/clang-tools-extra/clangd/ClangdServer.cpp b/clang-tools-extra/clangd/ClangdServer.cpp
index 13d788162817fb4c1833831eb634a4a988afec03..6fb2641e8793db183bd1e7f4714eb31d6c2bf32c 100644
--- a/clang-tools-extra/clangd/ClangdServer.cpp
+++ b/clang-tools-extra/clangd/ClangdServer.cpp
@@ -437,7 +437,7 @@ void ClangdServer::codeComplete(PathRef File, Position Pos,
     ParseInputs ParseInput{IP->Command, &getHeaderFS(), IP->Contents.str()};
     // FIXME: Add traling new line if there is none at eof, workaround a crash,
     // see https://github.com/clangd/clangd/issues/332
-    if (!IP->Contents.endswith("\n"))
+    if (!IP->Contents.ends_with("\n"))
       ParseInput.Contents.append("\n");
     ParseInput.Index = Index;
 
@@ -488,7 +488,7 @@ void ClangdServer::signatureHelp(PathRef File, Position Pos,
     ParseInputs ParseInput{IP->Command, &getHeaderFS(), IP->Contents.str()};
     // FIXME: Add traling new line if there is none at eof, workaround a crash,
     // see https://github.com/clangd/clangd/issues/332
-    if (!IP->Contents.endswith("\n"))
+    if (!IP->Contents.ends_with("\n"))
       ParseInput.Contents.append("\n");
     ParseInput.Index = Index;
     CB(clangd::signatureHelp(File, Pos, *PreambleData, ParseInput,
@@ -661,7 +661,7 @@ void ClangdServer::codeAction(const CodeActionInputs &Params,
             return true;
           return llvm::any_of(Only, [&](llvm::StringRef Base) {
             return Kind.consume_front(Base) &&
-                   (Kind.empty() || Kind.startswith("."));
+                   (Kind.empty() || Kind.starts_with("."));
           });
         };
 
diff --git a/clang-tools-extra/clangd/CodeComplete.cpp b/clang-tools-extra/clangd/CodeComplete.cpp
index 5eef43e93cb519f9944c9fff0432f8ca7b1f8792..0e5f08cec440cefc0a7169ba00d66c88726d0ab6 100644
--- a/clang-tools-extra/clangd/CodeComplete.cpp
+++ b/clang-tools-extra/clangd/CodeComplete.cpp
@@ -610,7 +610,7 @@ private:
       //   foo<${1:class}>(${2:int p1}).
       //   We transform this pattern to '<$1>()$0' or '<$0>()'.
 
-      bool EmptyArgs = llvm::StringRef(*Snippet).endswith("()");
+      bool EmptyArgs = llvm::StringRef(*Snippet).ends_with("()");
       if (Snippet->front() == '<')
         return EmptyArgs ? "<$1>()$0" : "<$1>($0)";
       if (Snippet->front() == '(')
@@ -625,7 +625,7 @@ private:
 
       // Classes and template using aliases can only have template arguments,
       // e.g. Foo<${1:class}>.
-      if (llvm::StringRef(*Snippet).endswith("<>"))
+      if (llvm::StringRef(*Snippet).ends_with("<>"))
         return "<>"; // can happen with defaulted template arguments.
       return "<$0>";
     }
@@ -1748,7 +1748,7 @@ public:
         S.append("::"); // visibleNamespaces doesn't include trailing ::.
     if (HeuristicPrefix.Qualifier.empty())
       AllScopes = Opts.AllScopes;
-    else if (HeuristicPrefix.Qualifier.startswith("::")) {
+    else if (HeuristicPrefix.Qualifier.starts_with("::")) {
       Scopes.QueryScopes = {""};
       Scopes.UnresolvedQualifier =
           std::string(HeuristicPrefix.Qualifier.drop_front(2));
@@ -2130,7 +2130,7 @@ CompletionPrefix guessCompletionPrefix(llvm::StringRef Content,
   Result.Name = Content.slice(Rest.size(), Offset);
 
   // Consume qualifiers.
-  while (Rest.consume_back("::") && !Rest.endswith(":")) // reject ::::
+  while (Rest.consume_back("::") && !Rest.ends_with(":")) // reject ::::
     while (!Rest.empty() && isAsciiIdentifierContinue(Rest.back()))
       Rest = Rest.drop_back();
   Result.Qualifier =
@@ -2175,7 +2175,7 @@ CodeCompleteResult codeCompleteComment(PathRef FileName, unsigned Offset,
   Result.CompletionRange = CompletionRange;
   Result.Context = CodeCompletionContext::CCC_NaturalLanguage;
   for (llvm::StringRef Name : ParamNames) {
-    if (!Name.startswith(Prefix))
+    if (!Name.starts_with(Prefix))
       continue;
     CodeCompletion Item;
     Item.Name = Name.str() + "=*/";
@@ -2197,7 +2197,7 @@ maybeFunctionArgumentCommentStart(llvm::StringRef Content) {
   while (!Content.empty() && isAsciiIdentifierContinue(Content.back()))
     Content = Content.drop_back();
   Content = Content.rtrim();
-  if (Content.endswith("/*"))
+  if (Content.ends_with("/*"))
     return Content.size() - 2;
   return std::nullopt;
 }
@@ -2408,12 +2408,12 @@ bool allowImplicitCompletion(llvm::StringRef Content, unsigned Offset) {
     Content = Content.substr(Pos + 1);
 
   // Complete after scope operators.
-  if (Content.endswith(".") || Content.endswith("->") ||
-      Content.endswith("::") || Content.endswith("/*"))
+  if (Content.ends_with(".") || Content.ends_with("->") ||
+      Content.ends_with("::") || Content.ends_with("/*"))
     return true;
   // Complete after `#include <` and #include `<foo/`.
-  if ((Content.endswith("<") || Content.endswith("\"") ||
-       Content.endswith("/")) &&
+  if ((Content.ends_with("<") || Content.ends_with("\"") ||
+       Content.ends_with("/")) &&
       isIncludeFile(Content))
     return true;
 
diff --git a/clang-tools-extra/clangd/CodeCompletionStrings.cpp b/clang-tools-extra/clangd/CodeCompletionStrings.cpp
index 8dffed0f9c366269d005f43ec922e25fc52a9fbd..2075e5965f181ede2a7c2b737b63afbc91ba8285 100644
--- a/clang-tools-extra/clangd/CodeCompletionStrings.cpp
+++ b/clang-tools-extra/clangd/CodeCompletionStrings.cpp
@@ -23,7 +23,7 @@ namespace {
 
 bool isInformativeQualifierChunk(CodeCompletionString::Chunk const &Chunk) {
   return Chunk.Kind == CodeCompletionString::CK_Informative &&
-         llvm::StringRef(Chunk.Text).endswith("::");
+         llvm::StringRef(Chunk.Text).ends_with("::");
 }
 
 void appendEscapeSnippet(const llvm::StringRef Text, std::string *Out) {
@@ -165,7 +165,7 @@ void getSignature(const CodeCompletionString &CCS, std::string *Signature,
       //   Completing a method declaration itself (not a method expression) is
       //   similar except that we use the `RequiredQualifiers` to store the
       //   text before the selector, e.g. `- (void)`.
-      if (!llvm::StringRef(Chunk.Text).endswith(":")) { // Treat as C++.
+      if (!llvm::StringRef(Chunk.Text).ends_with(":")) { // Treat as C++.
         if (RequiredQualifiers)
           *RequiredQualifiers = std::move(*Signature);
         Signature->clear();
diff --git a/clang-tools-extra/clangd/CompileCommands.cpp b/clang-tools-extra/clangd/CompileCommands.cpp
index e116a739774b85e09dcdef391519c476ab4daad8..f43ce928463b90cd2e863fff34be2ad418215218 100644
--- a/clang-tools-extra/clangd/CompileCommands.cpp
+++ b/clang-tools-extra/clangd/CompileCommands.cpp
@@ -338,7 +338,7 @@ void CommandMangler::operator()(tooling::CompileCommand &Command,
   };
 
   llvm::erase_if(Cmd, [](llvm::StringRef Elem) {
-    return Elem.startswith("--save-temps") || Elem.startswith("-save-temps");
+    return Elem.starts_with("--save-temps") || Elem.starts_with("-save-temps");
   });
 
   std::vector<std::string> ToAppend;
@@ -587,7 +587,7 @@ const ArgStripper::Rule *ArgStripper::matchingRule(llvm::StringRef Arg,
       continue; // not applicable to current driver mode
     if (BestRule && BestRule->Priority < R.Priority)
       continue; // lower-priority than best candidate.
-    if (!Arg.startswith(R.Text))
+    if (!Arg.starts_with(R.Text))
       continue; // current arg doesn't match the prefix string
     bool PrefixMatch = Arg.size() > R.Text.size();
     // Can rule apply as an exact/prefix match?
diff --git a/clang-tools-extra/clangd/ConfigCompile.cpp b/clang-tools-extra/clangd/ConfigCompile.cpp
index 0c9fc27643be87a7e4883a9d5a1d3797b10d31ad..5bb2eb4a9f803fb805244f87c139ac4cb4cdb2c5 100644
--- a/clang-tools-extra/clangd/ConfigCompile.cpp
+++ b/clang-tools-extra/clangd/ConfigCompile.cpp
@@ -490,7 +490,7 @@ struct FragmentCompiler {
     StringRef Str = StringRef(*Arg).trim();
     // Don't support negating here, its handled if the item is in the Add or
     // Remove list.
-    if (Str.startswith("-") || Str.contains(',')) {
+    if (Str.starts_with("-") || Str.contains(',')) {
       diag(Error, "Invalid clang-tidy check name", Arg.Range);
       return;
     }
diff --git a/clang-tools-extra/clangd/DumpAST.cpp b/clang-tools-extra/clangd/DumpAST.cpp
index 85f2592445f2a52449cb7f595dec7ab01649ce0d..b0cec65c39fa31dc641b395d43f7d3c4935af395 100644
--- a/clang-tools-extra/clangd/DumpAST.cpp
+++ b/clang-tools-extra/clangd/DumpAST.cpp
@@ -118,8 +118,8 @@ class DumpVisitor : public RecursiveASTVisitor<DumpVisitor> {
   std::string getKind(const Decl *D) { return D->getDeclKindName(); }
   std::string getKind(const Stmt *S) {
     std::string Result = S->getStmtClassName();
-    if (llvm::StringRef(Result).endswith("Stmt") ||
-        llvm::StringRef(Result).endswith("Expr"))
+    if (llvm::StringRef(Result).ends_with("Stmt") ||
+        llvm::StringRef(Result).ends_with("Expr"))
       Result.resize(Result.size() - 4);
     return Result;
   }
diff --git a/clang-tools-extra/clangd/FileDistance.cpp b/clang-tools-extra/clangd/FileDistance.cpp
index 09a9e80b7847a6a442d966192f74394f9d2ffad6..06c1a8bc92a862a6870df95039acc33334be9d5b 100644
--- a/clang-tools-extra/clangd/FileDistance.cpp
+++ b/clang-tools-extra/clangd/FileDistance.cpp
@@ -201,7 +201,7 @@ createScopeFileDistance(llvm::ArrayRef<std::string> QueryScopes) {
     // place of enclosing namespaces (e.g. in implementation files).
     if (S == Preferred)
       Param.Cost = S == "" ? 4 : 0;
-    else if (Preferred.startswith(S) && !S.empty())
+    else if (Preferred.starts_with(S) && !S.empty())
       continue; // just rely on up-traversals.
     else
       Param.Cost = S == "" ? 6 : 2;
diff --git a/clang-tools-extra/clangd/FindSymbols.cpp b/clang-tools-extra/clangd/FindSymbols.cpp
index 790ee9af8f4acfc8dbd12f764b9c5f51cc12bfd8..5b3e46a7b4dc167604a620540dae04f052487b78 100644
--- a/clang-tools-extra/clangd/FindSymbols.cpp
+++ b/clang-tools-extra/clangd/FindSymbols.cpp
@@ -41,8 +41,8 @@ struct ScoredSymbolGreater {
 
 // Returns true if \p Query can be found as a sub-sequence inside \p Scope.
 bool approximateScopeMatch(llvm::StringRef Scope, llvm::StringRef Query) {
-  assert(Scope.empty() || Scope.endswith("::"));
-  assert(Query.empty() || Query.endswith("::"));
+  assert(Scope.empty() || Scope.ends_with("::"));
+  assert(Query.empty() || Query.ends_with("::"));
   while (!Scope.empty() && !Query.empty()) {
     auto Colons = Scope.find("::");
     assert(Colons != llvm::StringRef::npos);
diff --git a/clang-tools-extra/clangd/Format.cpp b/clang-tools-extra/clangd/Format.cpp
index c3e92636d1957285f00c2e6b82d43ea5b225e154..272a34d4ed79724fda8a8a4c99597ecfff99fc00 100644
--- a/clang-tools-extra/clangd/Format.cpp
+++ b/clang-tools-extra/clangd/Format.cpp
@@ -180,7 +180,7 @@ IncrementalChanges getIncrementalChangesAfterNewline(llvm::StringRef Code,
   bool NewLineIsComment = !commentMarker(Indentation).empty();
   if (!CommentMarker.empty() &&
       (NewLineIsComment || !commentMarker(NextLine).empty() ||
-       (!TrailingTrim.empty() && !TrailingTrim.startswith("//")))) {
+       (!TrailingTrim.empty() && !TrailingTrim.starts_with("//")))) {
     // We indent the new comment to match the previous one.
     StringRef PreComment =
         Leading.take_front(CommentMarker.data() - Leading.data());
@@ -197,8 +197,8 @@ IncrementalChanges getIncrementalChangesAfterNewline(llvm::StringRef Code,
   }
 
   // If we put a the newline inside a {} pair, put } on its own line...
-  if (CommentMarker.empty() && Leading.endswith("{") &&
-      Trailing.startswith("}")) {
+  if (CommentMarker.empty() && Leading.ends_with("{") &&
+      Trailing.starts_with("}")) {
     cantFail(
         Result.Changes.add(replacement(Code, Trailing.take_front(1), "\n}")));
     // ...and format it.
diff --git a/clang-tools-extra/clangd/Headers.cpp b/clang-tools-extra/clangd/Headers.cpp
index 6005069be01160df40bcd2e24c4c83995e915113..076e636e0e2819a05ac7cb08909de62c74dc9a31 100644
--- a/clang-tools-extra/clangd/Headers.cpp
+++ b/clang-tools-extra/clangd/Headers.cpp
@@ -82,7 +82,7 @@ public:
     if (File) {
       auto IncludingFileEntry = SM.getFileEntryRefForID(SM.getFileID(HashLoc));
       if (!IncludingFileEntry) {
-        assert(SM.getBufferName(HashLoc).startswith("<") &&
+        assert(SM.getBufferName(HashLoc).starts_with("<") &&
                "Expected #include location to be a file or <built-in>");
         // Treat as if included from the main file.
         IncludingFileEntry = SM.getFileEntryRefForID(MainFID);
@@ -131,7 +131,7 @@ private:
 };
 
 bool isLiteralInclude(llvm::StringRef Include) {
-  return Include.startswith("<") || Include.startswith("\"");
+  return Include.starts_with("<") || Include.starts_with("\"");
 }
 
 bool HeaderFile::valid() const {
@@ -316,7 +316,7 @@ IncludeInserter::insert(llvm::StringRef VerbatimHeader,
   std::optional<TextEdit> Edit;
   if (auto Insertion =
           Inserter.insert(VerbatimHeader.trim("\"<>"),
-                          VerbatimHeader.startswith("<"), Directive))
+                          VerbatimHeader.starts_with("<"), Directive))
     Edit = replacementToEdit(Code, *Insertion);
   return Edit;
 }
diff --git a/clang-tools-extra/clangd/Hover.cpp b/clang-tools-extra/clangd/Hover.cpp
index a868d3bb4e3fa1d0a26ab02611efdc11e37bd699..82323fe16c82b6e4e72905e6fbd4b32946f9dcf6 100644
--- a/clang-tools-extra/clangd/Hover.cpp
+++ b/clang-tools-extra/clangd/Hover.cpp
@@ -960,7 +960,7 @@ std::optional<HoverInfo> getHoverContents(const Attr *A, ParsedAST &AST) {
 }
 
 bool isParagraphBreak(llvm::StringRef Rest) {
-  return Rest.ltrim(" \t").startswith("\n");
+  return Rest.ltrim(" \t").starts_with("\n");
 }
 
 bool punctuationIndicatesLineBreak(llvm::StringRef Line) {
@@ -984,7 +984,7 @@ bool isHardLineBreakIndicator(llvm::StringRef Rest) {
 
   if (llvm::isDigit(Rest.front())) {
     llvm::StringRef AfterDigit = Rest.drop_while(llvm::isDigit);
-    if (AfterDigit.startswith(".") || AfterDigit.startswith(")"))
+    if (AfterDigit.starts_with(".") || AfterDigit.starts_with(")"))
       return true;
   }
   return false;
diff --git a/clang-tools-extra/clangd/IncludeCleaner.cpp b/clang-tools-extra/clangd/IncludeCleaner.cpp
index dda7c9f581f69c7b03517416b2ce2f08cb279d79..2f34c949349200337329b4456aff03dc891c4c57 100644
--- a/clang-tools-extra/clangd/IncludeCleaner.cpp
+++ b/clang-tools-extra/clangd/IncludeCleaner.cpp
@@ -95,7 +95,7 @@ bool mayConsiderUnused(const Inclusion &Inc, ParsedAST &AST,
       // Since most private -> public mappings happen in a verbatim way, we
       // check textually here. This might go wrong in presence of symlinks or
       // header mappings. But that's not different than rest of the places.
-      if (AST.tuPath().endswith(PHeader))
+      if (AST.tuPath().ends_with(PHeader))
         return false;
     }
   }
diff --git a/clang-tools-extra/clangd/IncludeFixer.cpp b/clang-tools-extra/clangd/IncludeFixer.cpp
index 0c4d011b80372ba95dfdda7b8b2f299b90bc943f..fadd1105691fc01e183ee19e8d66cbb482eaf45d 100644
--- a/clang-tools-extra/clangd/IncludeFixer.cpp
+++ b/clang-tools-extra/clangd/IncludeFixer.cpp
@@ -416,7 +416,7 @@ std::optional<CheapUnresolvedName> extractUnresolvedNameCheaply(
         //     namespace clang { clangd::X; }
         // In this case, we use the "typo" specifier as extra scope instead
         // of using the scope assumed by sema.
-        if (!Spelling || llvm::StringRef(SpecifiedNS).endswith(*Spelling)) {
+        if (!Spelling || llvm::StringRef(SpecifiedNS).ends_with(*Spelling)) {
           Result.ResolvedScope = std::move(SpecifiedNS);
         } else {
           Result.UnresolvedScope = std::move(*Spelling);
diff --git a/clang-tools-extra/clangd/InlayHints.cpp b/clang-tools-extra/clangd/InlayHints.cpp
index b540c273cbd59672b07c8eede970e2824f17ca53..6fbb310b660a178540707cb7a2cb27d6599418d5 100644
--- a/clang-tools-extra/clangd/InlayHints.cpp
+++ b/clang-tools-extra/clangd/InlayHints.cpp
@@ -1040,7 +1040,7 @@ private:
     if (!SourcePrefix.consume_back(ParamName))
       return false;
     SourcePrefix = SourcePrefix.rtrim(IgnoreChars);
-    return SourcePrefix.endswith("/*");
+    return SourcePrefix.ends_with("/*");
   }
 
   // If "E" spells a single unqualified identifier, return that name.
diff --git a/clang-tools-extra/clangd/JSONTransport.cpp b/clang-tools-extra/clangd/JSONTransport.cpp
index 9dc0df807aa3463116093cdee3610bd759538529..346c7dfb66a1dbb01709e31ed6386470f42749b4 100644
--- a/clang-tools-extra/clangd/JSONTransport.cpp
+++ b/clang-tools-extra/clangd/JSONTransport.cpp
@@ -240,7 +240,7 @@ bool JSONTransport::readStandardMessage(std::string &JSON) {
     // We allow comments in headers. Technically this isn't part
 
     // of the LSP specification, but makes writing tests easier.
-    if (LineRef.startswith("#"))
+    if (LineRef.starts_with("#"))
       continue;
 
     // Content-Length is a mandatory header, and the only one we handle.
@@ -304,7 +304,7 @@ bool JSONTransport::readDelimitedMessage(std::string &JSON) {
   while (readLine(In, Line)) {
     InMirror << Line;
     auto LineRef = Line.str().trim();
-    if (LineRef.startswith("#")) // comment
+    if (LineRef.starts_with("#")) // comment
       continue;
 
     // found a delimiter
diff --git a/clang-tools-extra/clangd/ParsedAST.cpp b/clang-tools-extra/clangd/ParsedAST.cpp
index edd0f77b1031ef03e4da50a87948883ff671e74f..d91ce7283ecee44d5d3895c3431cc9f581b913f2 100644
--- a/clang-tools-extra/clangd/ParsedAST.cpp
+++ b/clang-tools-extra/clangd/ParsedAST.cpp
@@ -288,7 +288,7 @@ public:
       if (Glob) {
         // Is this clang-diagnostic-*, or *, or so?
         // (We ignore all other types of globs).
-        if (CDPrefix.startswith(Check)) {
+        if (CDPrefix.starts_with(Check)) {
           Default = Enable;
           Exceptions.clear();
         }
diff --git a/clang-tools-extra/clangd/PathMapping.cpp b/clang-tools-extra/clangd/PathMapping.cpp
index 2554d34d96b9dfdb323f567a4465f9d9454c2169..4b93ff2c60c5c6035920c4cae17f7d77a9485b9d 100644
--- a/clang-tools-extra/clangd/PathMapping.cpp
+++ b/clang-tools-extra/clangd/PathMapping.cpp
@@ -21,7 +21,7 @@ std::optional<std::string> doPathMapping(llvm::StringRef S,
                                          PathMapping::Direction Dir,
                                          const PathMappings &Mappings) {
   // Return early to optimize for the common case, wherein S is not a file URI
-  if (!S.startswith("file://"))
+  if (!S.starts_with("file://"))
     return std::nullopt;
   auto Uri = URI::parse(S);
   if (!Uri) {
diff --git a/clang-tools-extra/clangd/Protocol.cpp b/clang-tools-extra/clangd/Protocol.cpp
index e44aee2d47819468636501f62dbc68be226d225a..a6370649f5ad1cf3a1810c3499cbac0890b8363c 100644
--- a/clang-tools-extra/clangd/Protocol.cpp
+++ b/clang-tools-extra/clangd/Protocol.cpp
@@ -844,7 +844,7 @@ llvm::raw_ostream &operator<<(llvm::raw_ostream &O, const SymbolDetails &S) {
   if (!S.containerName.empty()) {
     O << S.containerName;
     llvm::StringRef ContNameRef;
-    if (!ContNameRef.endswith("::")) {
+    if (!ContNameRef.ends_with("::")) {
       O << " ";
     }
   }
diff --git a/clang-tools-extra/clangd/SourceCode.cpp b/clang-tools-extra/clangd/SourceCode.cpp
index 31323c08cf1ddf9086a1b583fcce7e5ad911f95a..835038423fdf3729ebff38e3d784262bb2dce855 100644
--- a/clang-tools-extra/clangd/SourceCode.cpp
+++ b/clang-tools-extra/clangd/SourceCode.cpp
@@ -891,10 +891,10 @@ llvm::StringSet<> collectWords(llvm::StringRef Content) {
 static bool isLikelyIdentifier(llvm::StringRef Word, llvm::StringRef Before,
                                llvm::StringRef After) {
   // `foo` is an identifier.
-  if (Before.endswith("`") && After.startswith("`"))
+  if (Before.ends_with("`") && After.starts_with("`"))
     return true;
   // In foo::bar, both foo and bar are identifiers.
-  if (Before.endswith("::") || After.startswith("::"))
+  if (Before.ends_with("::") || After.starts_with("::"))
     return true;
   // Doxygen tags like \c foo indicate identifiers.
   // Don't search too far back.
@@ -1180,7 +1180,7 @@ EligibleRegion getEligiblePoints(llvm::StringRef Code,
     }
 
     // Ignore namespaces that are not a prefix of the target.
-    if (!FullyQualifiedName.startswith(CurrentNamespace))
+    if (!FullyQualifiedName.starts_with(CurrentNamespace))
       return;
 
     // Prefer the namespace that shares the longest prefix with target.
@@ -1213,14 +1213,14 @@ bool isHeaderFile(llvm::StringRef FileName,
 
 bool isProtoFile(SourceLocation Loc, const SourceManager &SM) {
   auto FileName = SM.getFilename(Loc);
-  if (!FileName.endswith(".proto.h") && !FileName.endswith(".pb.h"))
+  if (!FileName.ends_with(".proto.h") && !FileName.ends_with(".pb.h"))
     return false;
   auto FID = SM.getFileID(Loc);
   // All proto generated headers should start with this line.
   static const char *ProtoHeaderComment =
       "// Generated by the protocol buffer compiler.  DO NOT EDIT!";
   // Double check that this is an actual protobuf header.
-  return SM.getBufferData(FID).startswith(ProtoHeaderComment);
+  return SM.getBufferData(FID).starts_with(ProtoHeaderComment);
 }
 
 SourceLocation translatePreamblePatchLocation(SourceLocation Loc,
@@ -1230,7 +1230,7 @@ SourceLocation translatePreamblePatchLocation(SourceLocation Loc,
     auto IncludeLoc = SM.getIncludeLoc(DefFile);
     // Preamble patch is included inside the builtin file.
     if (IncludeLoc.isValid() && SM.isWrittenInBuiltinFile(IncludeLoc) &&
-        FE->getName().endswith(PreamblePatch::HeaderName)) {
+        FE->getName().ends_with(PreamblePatch::HeaderName)) {
       auto Presumed = SM.getPresumedLoc(Loc);
       // Check that line directive is pointing at main file.
       if (Presumed.isValid() && Presumed.getFileID().isInvalid() &&
diff --git a/clang-tools-extra/clangd/SystemIncludeExtractor.cpp b/clang-tools-extra/clangd/SystemIncludeExtractor.cpp
index ea98c7d948a2f6de1976fc5924fff7fa79370f40..d4b9b173d149da06946bd16bc6b88fb0074a3b4d 100644
--- a/clang-tools-extra/clangd/SystemIncludeExtractor.cpp
+++ b/clang-tools-extra/clangd/SystemIncludeExtractor.cpp
@@ -146,13 +146,13 @@ struct DriverArgs {
           Stdlib = Cmd.CommandLine[I + 1];
       } else if (Arg.consume_front("-stdlib=")) {
         Stdlib = Arg.str();
-      } else if (Arg.startswith("-specs=")) {
+      } else if (Arg.starts_with("-specs=")) {
         // clang requires a single token like `-specs=file` or `--specs=file`,
         // but gcc will accept two tokens like `--specs file`. Since the
         // compilation database is presumably correct, we just forward the flags
         // as-is.
         Specs.push_back(Arg.str());
-      } else if (Arg.startswith("--specs=")) {
+      } else if (Arg.starts_with("--specs=")) {
         Specs.push_back(Arg.str());
       } else if (Arg == "--specs" && I + 1 < E) {
         Specs.push_back(Arg.str());
@@ -282,7 +282,7 @@ std::optional<DriverInfo> parseDriverOutput(llvm::StringRef Output) {
       if (!SeenIncludes && Line.trim() == SIS) {
         SeenIncludes = true;
         State = IncludesExtracting;
-      } else if (!SeenTarget && Line.trim().startswith(TS)) {
+      } else if (!SeenTarget && Line.trim().starts_with(TS)) {
         SeenTarget = true;
         llvm::StringRef TargetLine = Line.trim();
         TargetLine.consume_front(TS);
@@ -448,7 +448,7 @@ tooling::CompileCommand &setTarget(tooling::CompileCommand &Cmd,
   if (!Target.empty()) {
     // We do not want to override existing target with extracted one.
     for (llvm::StringRef Arg : Cmd.CommandLine) {
-      if (Arg == "-target" || Arg.startswith("--target="))
+      if (Arg == "-target" || Arg.starts_with("--target="))
         return Cmd;
     }
     // Just append when `--` isn't present.
diff --git a/clang-tools-extra/clangd/URI.cpp b/clang-tools-extra/clangd/URI.cpp
index ca65df329aeebf3c3d709cd9c07c9b1c52002c9f..11d70dc917f56bf9f1c191caf3850d7ed3a4f1a8 100644
--- a/clang-tools-extra/clangd/URI.cpp
+++ b/clang-tools-extra/clangd/URI.cpp
@@ -38,7 +38,7 @@ public:
   llvm::Expected<std::string>
   getAbsolutePath(llvm::StringRef Authority, llvm::StringRef Body,
                   llvm::StringRef /*HintPath*/) const override {
-    if (!Body.startswith("/"))
+    if (!Body.starts_with("/"))
       return error("File scheme: expect body to be an absolute path starting "
                    "with '/': {0}",
                    Body);
@@ -153,7 +153,7 @@ URI::URI(llvm::StringRef Scheme, llvm::StringRef Authority,
          llvm::StringRef Body)
     : Scheme(Scheme), Authority(Authority), Body(Body) {
   assert(!Scheme.empty());
-  assert((Authority.empty() || Body.startswith("/")) &&
+  assert((Authority.empty() || Body.starts_with("/")) &&
          "URI body must start with '/' when authority is present.");
 }
 
@@ -165,8 +165,7 @@ std::string URI::toString() const {
     return Result;
   // If authority if empty, we only print body if it starts with "/"; otherwise,
   // the URI is invalid.
-  if (!Authority.empty() || llvm::StringRef(Body).startswith("/"))
-  {
+  if (!Authority.empty() || llvm::StringRef(Body).starts_with("/")) {
     Result.append("//");
     percentEncode(Authority, Result);
   }
diff --git a/clang-tools-extra/clangd/index/Merge.cpp b/clang-tools-extra/clangd/index/Merge.cpp
index 9687b36252e12ceeb73df70e43a791ffb402a63f..8221d4b1f44405abab5bb6fdf0a0a73783cdfee3 100644
--- a/clang-tools-extra/clangd/index/Merge.cpp
+++ b/clang-tools-extra/clangd/index/Merge.cpp
@@ -197,7 +197,7 @@ static bool prefer(const SymbolLocation &L, const SymbolLocation &R) {
   auto HasCodeGenSuffix = [](const SymbolLocation &Loc) {
     constexpr static const char *CodegenSuffixes[] = {".proto"};
     return llvm::any_of(CodegenSuffixes, [&](llvm::StringRef Suffix) {
-      return llvm::StringRef(Loc.FileURI).endswith(Suffix);
+      return llvm::StringRef(Loc.FileURI).ends_with(Suffix);
     });
   };
   return HasCodeGenSuffix(L) && !HasCodeGenSuffix(R);
diff --git a/clang-tools-extra/clangd/index/Serialization.cpp b/clang-tools-extra/clangd/index/Serialization.cpp
index b905f580c281c9f750601b2df420696732601357..72a4e8b007668f4a6629afdefecc89b764690a7b 100644
--- a/clang-tools-extra/clangd/index/Serialization.cpp
+++ b/clang-tools-extra/clangd/index/Serialization.cpp
@@ -692,7 +692,7 @@ llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const IndexFileOut &O) {
 
 llvm::Expected<IndexFileIn> readIndexFile(llvm::StringRef Data,
                                           SymbolOrigin Origin) {
-  if (Data.startswith("RIFF")) {
+  if (Data.starts_with("RIFF")) {
     return readRIFF(Data, Origin);
   }
   if (auto YAMLContents = readYAML(Data, Origin)) {
diff --git a/clang-tools-extra/clangd/index/StdLib.cpp b/clang-tools-extra/clangd/index/StdLib.cpp
index 390c2e41f6c0f97d95f0feeaae7f1e411d5cefb5..921ab5d1c96d5cf0f5b36c7779667aca6a962883 100644
--- a/clang-tools-extra/clangd/index/StdLib.cpp
+++ b/clang-tools-extra/clangd/index/StdLib.cpp
@@ -167,7 +167,7 @@ SymbolSlab filter(SymbolSlab Slab, const StdLibLocation &Loc) {
         R.first->second = llvm::any_of(
             StdLibURIPrefixes,
             [&, URIStr(llvm::StringRef(URI))](const std::string &Prefix) {
-              return URIStr.startswith(Prefix);
+              return URIStr.starts_with(Prefix);
             });
       }
     }
diff --git a/clang-tools-extra/clangd/index/SymbolCollector.cpp b/clang-tools-extra/clangd/index/SymbolCollector.cpp
index aac6676a995fedfd28fb12aabecf30fe76a132c7..cf6102db8dd317c4587e6b31a75a02d4406bb692 100644
--- a/clang-tools-extra/clangd/index/SymbolCollector.cpp
+++ b/clang-tools-extra/clangd/index/SymbolCollector.cpp
@@ -262,7 +262,7 @@ public:
     if (Canonical.empty())
       return "";
     // If we had a mapping, always use it.
-    assert(Canonical.startswith("<") || Canonical.startswith("\""));
+    assert(Canonical.starts_with("<") || Canonical.starts_with("\""));
     return Canonical;
   }
 
@@ -414,7 +414,7 @@ private:
                                         PP->getHeaderSearchInfo())) {
       // A .inc or .def file is often included into a real header to define
       // symbols (e.g. LLVM tablegen files).
-      if (Filename.endswith(".inc") || Filename.endswith(".def"))
+      if (Filename.ends_with(".inc") || Filename.ends_with(".def"))
         // Don't use cache reentrantly due to iterator invalidation.
         return getIncludeHeaderUncached(SM.getFileID(SM.getIncludeLoc(FID)));
       // Conservatively refuse to insert #includes to files without guards.
diff --git a/clang-tools-extra/clangd/index/dex/Dex.cpp b/clang-tools-extra/clangd/index/dex/Dex.cpp
index 8f504fb9b7ea304ef2f11d1bb1567dcdf1d7956a..19dc3080f9f897b8c85c4f501b0789378ee5ae82 100644
--- a/clang-tools-extra/clangd/index/dex/Dex.cpp
+++ b/clang-tools-extra/clangd/index/dex/Dex.cpp
@@ -395,7 +395,7 @@ generateProximityURIs(llvm::StringRef URI) {
       return Result;
   }
   // The root foo://bar/ is a proximity URI.
-  if (Path.startswith("/"))
+  if (Path.starts_with("/"))
     Result.push_back(URI.substr(0, Path.begin() + 1 - URI.data()));
   return Result;
 }
diff --git a/clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp b/clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp
index 392960d6d6660d130b030b0f936ff385d2d804db..cea59ae409914c048d542a0dec254ee144e343fd 100644
--- a/clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp
+++ b/clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp
@@ -372,7 +372,7 @@ struct {
 };
 
 std::unique_ptr<SymbolIndex> openIndex(llvm::StringRef Index) {
-  return Index.startswith("remote:")
+  return Index.starts_with("remote:")
              ? remote::getClient(Index.drop_front(strlen("remote:")),
                                  ProjectRoot)
              : loadIndex(Index, SymbolOrigin::Static, /*UseDex=*/true);
@@ -424,7 +424,7 @@ int main(int argc, const char *argv[]) {
   llvm::cl::ResetCommandLineParser(); // We reuse it for REPL commands.
   llvm::sys::PrintStackTraceOnErrorSignal(argv[0]);
 
-  bool RemoteMode = llvm::StringRef(IndexLocation).startswith("remote:");
+  bool RemoteMode = llvm::StringRef(IndexLocation).starts_with("remote:");
   if (RemoteMode && ProjectRoot.empty()) {
     llvm::errs() << "--project-root is required in remote mode\n";
     return -1;
diff --git a/clang-tools-extra/clangd/refactor/tweaks/AddUsing.cpp b/clang-tools-extra/clangd/refactor/tweaks/AddUsing.cpp
index ca96da34e092011433717bc7769ba6ae46bbb783..00c05ebdb521663393d65ff2c0ce835fe54c036c 100644
--- a/clang-tools-extra/clangd/refactor/tweaks/AddUsing.cpp
+++ b/clang-tools-extra/clangd/refactor/tweaks/AddUsing.cpp
@@ -352,7 +352,7 @@ bool AddUsing::prepare(const Selection &Inputs) {
       splitQualifiedName(SpelledRange.text(SM));
   QualifierToSpell = getNNSLAsString(
       QualifierToRemove, Inputs.AST->getASTContext().getPrintingPolicy());
-  if (!llvm::StringRef(QualifierToSpell).endswith(SpelledQualifier) ||
+  if (!llvm::StringRef(QualifierToSpell).ends_with(SpelledQualifier) ||
       SpelledName.empty())
     return false; // What's spelled doesn't match the qualifier.
   return true;
diff --git a/clang-tools-extra/clangd/refactor/tweaks/DefineOutline.cpp b/clang-tools-extra/clangd/refactor/tweaks/DefineOutline.cpp
index b84ae04072f2c19b2afd9098cda80b9a57ef164b..fef827a801c33974ef1118d31ccc402af0bb9c73 100644
--- a/clang-tools-extra/clangd/refactor/tweaks/DefineOutline.cpp
+++ b/clang-tools-extra/clangd/refactor/tweaks/DefineOutline.cpp
@@ -71,7 +71,7 @@ std::optional<Path> getSourceFile(llvm::StringRef FileName,
 // Returns std::nullopt if TargetNS is not a prefix of CurContext.
 std::optional<const DeclContext *>
 findContextForNS(llvm::StringRef TargetNS, const DeclContext *CurContext) {
-  assert(TargetNS.empty() || TargetNS.endswith("::"));
+  assert(TargetNS.empty() || TargetNS.ends_with("::"));
   // Skip any non-namespace contexts, e.g. TagDecls, functions/methods.
   CurContext = CurContext->getEnclosingNamespaceContext();
   // If TargetNS is empty, it means global ns, which is translation unit.
@@ -91,7 +91,7 @@ findContextForNS(llvm::StringRef TargetNS, const DeclContext *CurContext) {
   llvm::StringRef CurrentContextNS(TargetContextNS);
   // If TargetNS is not a prefix of CurrentContext, there's no way to reach
   // it.
-  if (!CurrentContextNS.startswith(TargetNS))
+  if (!CurrentContextNS.starts_with(TargetNS))
     return std::nullopt;
 
   while (CurrentContextNS != TargetNS) {
diff --git a/clang-tools-extra/clangd/support/Markup.cpp b/clang-tools-extra/clangd/support/Markup.cpp
index 4d17a2bf2b2b869358e8c58e2ba6215957ed286b..63aff96b02056d3c43360c1df2536f5c0dd24eeb 100644
--- a/clang-tools-extra/clangd/support/Markup.cpp
+++ b/clang-tools-extra/clangd/support/Markup.cpp
@@ -47,7 +47,7 @@ bool looksLikeTag(llvm::StringRef Contents) {
   for (; !Contents.empty(); Contents = Contents.drop_front()) {
     if (llvm::isAlnum(Contents.front()) || llvm::isSpace(Contents.front()))
       continue;
-    if (Contents.front() == '>' || Contents.startswith("/>"))
+    if (Contents.front() == '>' || Contents.starts_with("/>"))
       return true; // May close the tag.
     if (Contents.front() == '=')
       return true; // Don't try to parse attribute values.
@@ -75,7 +75,7 @@ bool needsLeadingEscape(char C, llvm::StringRef Before, llvm::StringRef After,
   };
   auto IsBullet = [&]() {
     return StartsLine && Before.empty() &&
-           (After.empty() || After.startswith(" "));
+           (After.empty() || After.starts_with(" "));
   };
   auto SpaceSurrounds = [&]() {
     return (After.empty() || llvm::isSpace(After.front())) &&
@@ -94,12 +94,12 @@ bool needsLeadingEscape(char C, llvm::StringRef Before, llvm::StringRef After,
     // anywhere (including on another line). We must escape them all.
     return true;
   case '~': // Code block
-    return StartsLine && Before.empty() && After.startswith("~~");
+    return StartsLine && Before.empty() && After.starts_with("~~");
   case '#': { // ATX heading.
     if (!StartsLine || !Before.empty())
       return false;
     llvm::StringRef Rest = After.ltrim(C);
-    return Rest.empty() || Rest.startswith(" ");
+    return Rest.empty() || Rest.starts_with(" ");
   }
   case ']': // Link or link reference.
     // We escape ] rather than [ here, because it's more constrained:
@@ -109,7 +109,7 @@ bool needsLeadingEscape(char C, llvm::StringRef Before, llvm::StringRef After,
     //   ] by itself is a shortcut link
     //   ][...] is an out-of-line link
     // Because we never emit link references, we don't need to handle these.
-    return After.startswith(":") || After.startswith("(");
+    return After.starts_with(":") || After.starts_with("(");
   case '=': // Setex heading.
     return RulerLength() > 0;
   case '_': // Horizontal ruler or matched delimiter.
@@ -145,7 +145,7 @@ bool needsLeadingEscape(char C, llvm::StringRef Before, llvm::StringRef After,
   case '.': // Numbered list indicator. Escape 12. -> 12\. at start of line.
   case ')':
     return StartsLine && !Before.empty() &&
-           llvm::all_of(Before, llvm::isDigit) && After.startswith(" ");
+           llvm::all_of(Before, llvm::isDigit) && After.starts_with(" ");
   default:
     return false;
   }
@@ -180,12 +180,12 @@ std::string renderInlineBlock(llvm::StringRef Input) {
   }
   // If results starts with a backtick, add spaces on both sides. The spaces
   // are ignored by markdown renderers.
-  if (llvm::StringRef(R).startswith("`") || llvm::StringRef(R).endswith("`"))
+  if (llvm::StringRef(R).starts_with("`") || llvm::StringRef(R).ends_with("`"))
     return "` " + std::move(R) + " `";
   // Markdown render should ignore first and last space if both are there. We
   // add an extra pair of spaces in that case to make sure we render what the
   // user intended.
-  if (llvm::StringRef(R).startswith(" ") && llvm::StringRef(R).endswith(" "))
+  if (llvm::StringRef(R).starts_with(" ") && llvm::StringRef(R).ends_with(" "))
     return "` " + std::move(R) + " `";
   return "`" + std::move(R) + "`";
 }
@@ -250,7 +250,7 @@ std::string renderBlocks(llvm::ArrayRef<std::unique_ptr<Block>> Children,
                   return !llvm::StringRef(TrimmedText.data(),
                                           &C - TrimmedText.data() + 1)
                               // We allow at most two newlines.
-                              .endswith("\n\n\n");
+                              .ends_with("\n\n\n");
                 });
 
   return AdjustedResult;
@@ -301,7 +301,7 @@ private:
 // Inserts two spaces after each `\n` to indent each line. First line is not
 // indented.
 std::string indentLines(llvm::StringRef Input) {
-  assert(!Input.endswith("\n") && "Input should've been trimmed.");
+  assert(!Input.ends_with("\n") && "Input should've been trimmed.");
   std::string IndentedR;
   // We'll add 2 spaces after each new line.
   IndentedR.reserve(Input.size() + Input.count('\n') * 2);
diff --git a/clang-tools-extra/clangd/support/ThreadsafeFS.cpp b/clang-tools-extra/clangd/support/ThreadsafeFS.cpp
index 87babef4ee8c88aaaa50461e4fe124a4cac37276..0e249d07d2fd9141bc2224d8e702aee49a59182c 100644
--- a/clang-tools-extra/clangd/support/ThreadsafeFS.cpp
+++ b/clang-tools-extra/clangd/support/ThreadsafeFS.cpp
@@ -39,7 +39,7 @@ public:
     // Try to guess preamble files, they can be memory-mapped even on Windows as
     // clangd has exclusive access to those and nothing else should touch them.
     llvm::StringRef FileName = llvm::sys::path::filename(Path);
-    if (FileName.startswith("preamble-") && FileName.endswith(".pch"))
+    if (FileName.starts_with("preamble-") && FileName.ends_with(".pch"))
       return File;
     return std::unique_ptr<VolatileFile>(new VolatileFile(std::move(*File)));
   }
diff --git a/clang-tools-extra/clangd/tool/ClangdMain.cpp b/clang-tools-extra/clangd/tool/ClangdMain.cpp
index 9fd002d0eebba5f314d0bd953210bb9275ab58b9..c3ba655ee2dc6a7b0cb8bff5f3fd36773801f712 100644
--- a/clang-tools-extra/clangd/tool/ClangdMain.cpp
+++ b/clang-tools-extra/clangd/tool/ClangdMain.cpp
@@ -563,7 +563,7 @@ public:
     using namespace llvm::sys;
     // Still require "/" in body to mimic file scheme, as we want lengths of an
     // equivalent URI in both schemes to be the same.
-    if (!Body.startswith("/"))
+    if (!Body.starts_with("/"))
       return error(
           "Expect URI body to be an absolute path starting with '/': {0}",
           Body);
diff --git a/clang-tools-extra/clangd/unittests/BackgroundIndexTests.cpp b/clang-tools-extra/clangd/unittests/BackgroundIndexTests.cpp
index e5c9fc540880689cce1653583087bdbc2f2bc000..e51942462fbdf87a32aaf3d59280c482dbd96c65 100644
--- a/clang-tools-extra/clangd/unittests/BackgroundIndexTests.cpp
+++ b/clang-tools-extra/clangd/unittests/BackgroundIndexTests.cpp
@@ -132,11 +132,11 @@ TEST_F(BackgroundIndexTest, Config) {
   BackgroundIndex::Options Opts;
   Opts.ContextProvider = [](PathRef P) {
     Config C;
-    if (P.endswith("foo.cpp"))
+    if (P.ends_with("foo.cpp"))
       C.CompileFlags.Edits.push_back([](std::vector<std::string> &Argv) {
         Argv = tooling::getInsertArgumentAdjuster("-Done=two")(Argv, "");
       });
-    if (P.endswith("baz.cpp"))
+    if (P.ends_with("baz.cpp"))
       C.Index.Background = Config::BackgroundPolicy::Skip;
     return Context::current().derive(Config::Key, std::move(C));
   };
diff --git a/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp b/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp
index 692c6db3c51bedf5c6bc9e86a2a50ce232c6dbf1..6d387fec9b3851d10bda8b4164223b453242315e 100644
--- a/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp
+++ b/clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp
@@ -59,7 +59,7 @@ MATCHER_P(named, Name, "") { return arg.Name == Name; }
 MATCHER_P(mainFileRefs, Refs, "") { return arg.MainFileRefs == Refs; }
 MATCHER_P(scopeRefs, Refs, "") { return arg.ScopeRefsInFile == Refs; }
 MATCHER_P(nameStartsWith, Prefix, "") {
-  return llvm::StringRef(arg.Name).startswith(Prefix);
+  return llvm::StringRef(arg.Name).starts_with(Prefix);
 }
 MATCHER_P(filterText, F, "") { return arg.FilterText == F; }
 MATCHER_P(scope, S, "") { return arg.Scope == S; }
diff --git a/clang-tools-extra/clangd/unittests/DiagnosticsTests.cpp b/clang-tools-extra/clangd/unittests/DiagnosticsTests.cpp
index a52b647b0029b4facb59f2eac7587bdadc69ea4a..37643e5afa2304d2aa7673fc5a4aa8a1ccf40f97 100644
--- a/clang-tools-extra/clangd/unittests/DiagnosticsTests.cpp
+++ b/clang-tools-extra/clangd/unittests/DiagnosticsTests.cpp
@@ -1935,7 +1935,7 @@ $fix[[  $diag[[#include "unused.h"]]
   Cfg.Diagnostics.UnusedIncludes = Config::IncludesPolicy::Strict;
   // Set filtering.
   Cfg.Diagnostics.Includes.IgnoreHeader.emplace_back(
-      [](llvm::StringRef Header) { return Header.endswith("ignore.h"); });
+      [](llvm::StringRef Header) { return Header.ends_with("ignore.h"); });
   WithContextValue WithCfg(Config::Key, std::move(Cfg));
   auto AST = TU.build();
   EXPECT_THAT(
diff --git a/clang-tools-extra/clangd/unittests/GlobalCompilationDatabaseTests.cpp b/clang-tools-extra/clangd/unittests/GlobalCompilationDatabaseTests.cpp
index 38bf66a1e25eb57025d765eddd0d518c593e0e0b..2a6ae9c325b736c4dc6156fcb71c3a8652901579 100644
--- a/clang-tools-extra/clangd/unittests/GlobalCompilationDatabaseTests.cpp
+++ b/clang-tools-extra/clangd/unittests/GlobalCompilationDatabaseTests.cpp
@@ -228,14 +228,14 @@ TEST(GlobalCompilationDatabaseTest, DiscoveryWithNestedCDBs) {
     DirectoryBasedGlobalCompilationDatabase::Options Opts(FS);
     Opts.ContextProvider = [&](llvm::StringRef Path) {
       Config Cfg;
-      if (Path.endswith("a.cc")) {
+      if (Path.ends_with("a.cc")) {
         // a.cc uses another directory's CDB, so it won't be discovered.
         Cfg.CompileFlags.CDBSearch.Policy = Config::CDBSearchSpec::FixedDir;
         Cfg.CompileFlags.CDBSearch.FixedCDBPath = testPath("foo");
-      } else if (Path.endswith("gen.cc")) {
+      } else if (Path.ends_with("gen.cc")) {
         // gen.cc has CDB search disabled, so it won't be discovered.
         Cfg.CompileFlags.CDBSearch.Policy = Config::CDBSearchSpec::NoCDBSearch;
-      } else if (Path.endswith("gen2.cc")) {
+      } else if (Path.ends_with("gen2.cc")) {
         // gen2.cc explicitly lists this directory, so it will be discovered.
         Cfg.CompileFlags.CDBSearch.Policy = Config::CDBSearchSpec::FixedDir;
         Cfg.CompileFlags.CDBSearch.FixedCDBPath = testRoot();
diff --git a/clang-tools-extra/clangd/unittests/IndexActionTests.cpp b/clang-tools-extra/clangd/unittests/IndexActionTests.cpp
index fad751bd0f7dc220d8ffb4434b7eefec6b1d2762..fa3d9c3212f9ca7ee1cf9de66c4f1d12b08d468e 100644
--- a/clang-tools-extra/clangd/unittests/IndexActionTests.cpp
+++ b/clang-tools-extra/clangd/unittests/IndexActionTests.cpp
@@ -280,7 +280,7 @@ TEST_F(IndexActionTest, SkipFiles) {
     auto unskippable2() { return S(); }
   )cpp");
   Opts.FileFilter = [](const SourceManager &SM, FileID F) {
-    return !SM.getFileEntryRefForID(F)->getName().endswith("bad.h");
+    return !SM.getFileEntryRefForID(F)->getName().ends_with("bad.h");
   };
   IndexFileIn IndexFile = runIndexingAction(MainFilePath, {"-std=c++14"});
   EXPECT_THAT(*IndexFile.Symbols,
@@ -333,7 +333,7 @@ TEST_F(IndexActionTest, SymbolFromCC) {
  void foo();
  )cpp");
   Opts.FileFilter = [](const SourceManager &SM, FileID F) {
-    return !SM.getFileEntryRefForID(F)->getName().endswith("main.h");
+    return !SM.getFileEntryRefForID(F)->getName().ends_with("main.h");
   };
   IndexFileIn IndexFile = runIndexingAction(MainFilePath, {"-std=c++14"});
   EXPECT_THAT(*IndexFile.Symbols,
diff --git a/clang-tools-extra/clangd/unittests/InlayHintTests.cpp b/clang-tools-extra/clangd/unittests/InlayHintTests.cpp
index 6e91053632e00bfac1800f2771c2cbee097c53ae..0ca95b5fed5d31d4c30d2ce0bff2e0de468b9c12 100644
--- a/clang-tools-extra/clangd/unittests/InlayHintTests.cpp
+++ b/clang-tools-extra/clangd/unittests/InlayHintTests.cpp
@@ -58,8 +58,8 @@ struct ExpectedHint {
 MATCHER_P2(HintMatcher, Expected, Code, llvm::to_string(Expected)) {
   llvm::StringRef ExpectedView(Expected.Label);
   if (arg.label != ExpectedView.trim(" ") ||
-      arg.paddingLeft != ExpectedView.startswith(" ") ||
-      arg.paddingRight != ExpectedView.endswith(" ")) {
+      arg.paddingLeft != ExpectedView.starts_with(" ") ||
+      arg.paddingRight != ExpectedView.ends_with(" ")) {
     *result_listener << "label is '" << arg.label << "'";
     return false;
   }
diff --git a/clang-tools-extra/clangd/unittests/InsertionPointTests.cpp b/clang-tools-extra/clangd/unittests/InsertionPointTests.cpp
index 62c06bb863772f6a5fcb7f98582b6020019d05ef..3d5365a099f0ad586dd6be94fd62781caefafac4 100644
--- a/clang-tools-extra/clangd/unittests/InsertionPointTests.cpp
+++ b/clang-tools-extra/clangd/unittests/InsertionPointTests.cpp
@@ -38,7 +38,7 @@ TEST(InsertionPointTests, Generic) {
       [&](llvm::StringLiteral S) -> std::function<bool(const Decl *)> {
     return [S](const Decl *D) {
       if (const auto *ND = llvm::dyn_cast<NamedDecl>(D))
-        return llvm::StringRef(ND->getNameAsString()).startswith(S);
+        return llvm::StringRef(ND->getNameAsString()).starts_with(S);
       return false;
     };
   };
diff --git a/clang-tools-extra/clangd/unittests/StdLibTests.cpp b/clang-tools-extra/clangd/unittests/StdLibTests.cpp
index ef47141bade153f8ffb6d574a5bfac87fe7038e3..a39d34ea33811ae4171576b7c2ca823de53f6f87 100644
--- a/clang-tools-extra/clangd/unittests/StdLibTests.cpp
+++ b/clang-tools-extra/clangd/unittests/StdLibTests.cpp
@@ -126,7 +126,7 @@ TEST(StdLibTests, StdLibSet) {
 
 MATCHER_P(StdlibSymbol, Name, "") {
   return arg.Name == Name && arg.Includes.size() == 1 &&
-         llvm::StringRef(arg.Includes.front().Header).startswith("<");
+         llvm::StringRef(arg.Includes.front().Header).starts_with("<");
 }
 
 TEST(StdLibTests, EndToEnd) {
diff --git a/clang-tools-extra/clangd/unittests/tweaks/TweakTesting.cpp b/clang-tools-extra/clangd/unittests/tweaks/TweakTesting.cpp
index 51071d89a66e51bbb5a1469db9e5370ca33c43fc..81e65ede00781acb51584f3568f82ffa08602b06 100644
--- a/clang-tools-extra/clangd/unittests/tweaks/TweakTesting.cpp
+++ b/clang-tools-extra/clangd/unittests/tweaks/TweakTesting.cpp
@@ -43,7 +43,7 @@ llvm::StringRef unwrap(Context Ctx, llvm::StringRef Outer) {
   auto Wrapping = wrapping(Ctx);
   // Unwrap only if the code matches the expected wrapping.
   // Don't allow the begin/end wrapping to overlap!
-  if (Outer.startswith(Wrapping.first) && Outer.endswith(Wrapping.second) &&
+  if (Outer.starts_with(Wrapping.first) && Outer.ends_with(Wrapping.second) &&
       Outer.size() >= Wrapping.first.size() + Wrapping.second.size())
     return Outer.drop_front(Wrapping.first.size())
         .drop_back(Wrapping.second.size());
diff --git a/clang-tools-extra/include-cleaner/lib/Analysis.cpp b/clang-tools-extra/include-cleaner/lib/Analysis.cpp
index 450c4c796c141567f8620fe03b631e7d0160b91d..f1cd72f877ca21bfa7eeab6b7278383350350369 100644
--- a/clang-tools-extra/include-cleaner/lib/Analysis.cpp
+++ b/clang-tools-extra/include-cleaner/lib/Analysis.cpp
@@ -126,7 +126,7 @@ analyze(llvm::ArrayRef<Decl *> ASTRoots,
         // Since most private -> public mappings happen in a verbatim way, we
         // check textually here. This might go wrong in presence of symlinks or
         // header mappings. But that's not different than rest of the places.
-        if (MainFile->tryGetRealPathName().endswith(PHeader))
+        if (MainFile->tryGetRealPathName().ends_with(PHeader))
           continue;
       }
     }
diff --git a/clang-tools-extra/include-cleaner/lib/Record.cpp b/clang-tools-extra/include-cleaner/lib/Record.cpp
index 6e00ff93a7fe2fae384026e9db8d05bcad73a7c3..bd726cff12a97d915da4b8026fd27acfccf332ea 100644
--- a/clang-tools-extra/include-cleaner/lib/Record.cpp
+++ b/clang-tools-extra/include-cleaner/lib/Record.cpp
@@ -277,7 +277,7 @@ public:
     int CommentLine = SM.getLineNumber(CommentFID, CommentOffset);
 
     if (InMainFile) {
-      if (Pragma->startswith("keep")) {
+      if (Pragma->starts_with("keep")) {
         KeepStack.push_back({CommentLine, false});
       } else if (Pragma->starts_with("begin_keep")) {
         KeepStack.push_back({CommentLine, true});
@@ -300,9 +300,10 @@ public:
       StringRef PublicHeader;
       if (Pragma->consume_front(", include ")) {
         // We always insert using the spelling from the pragma.
-        PublicHeader = save(Pragma->startswith("<") || Pragma->startswith("\"")
-                                ? (*Pragma)
-                                : ("\"" + *Pragma + "\"").str());
+        PublicHeader =
+            save(Pragma->starts_with("<") || Pragma->starts_with("\"")
+                     ? (*Pragma)
+                     : ("\"" + *Pragma + "\"").str());
       }
       Out->IWYUPublic.insert({CommentUID, PublicHeader});
       return false;
@@ -313,11 +314,11 @@ public:
     }
     auto Filename = FE->getName();
     // Record export pragma.
-    if (Pragma->startswith("export")) {
+    if (Pragma->starts_with("export")) {
       ExportStack.push_back({CommentLine, CommentFID, save(Filename), false});
-    } else if (Pragma->startswith("begin_exports")) {
+    } else if (Pragma->starts_with("begin_exports")) {
       ExportStack.push_back({CommentLine, CommentFID, save(Filename), true});
-    } else if (Pragma->startswith("end_exports")) {
+    } else if (Pragma->starts_with("end_exports")) {
       // FIXME: be robust on unmatching cases. We should only pop the stack if
       // the begin_exports and end_exports is in the same file.
       if (!ExportStack.empty()) {
diff --git a/clang-tools-extra/modularize/CoverageChecker.cpp b/clang-tools-extra/modularize/CoverageChecker.cpp
index d8445053872bf22fbfb0fe73f7b205ce58fd81d7..1e8b0aa37ca309931c9d2c70433ffb22bc6018aa 100644
--- a/clang-tools-extra/modularize/CoverageChecker.cpp
+++ b/clang-tools-extra/modularize/CoverageChecker.cpp
@@ -302,7 +302,7 @@ void CoverageChecker::collectUmbrellaHeaderHeader(StringRef HeaderName) {
     sys::fs::current_path(PathBuf);
   // HeaderName will have an absolute path, so if it's the module map
   // directory, we remove it, also skipping trailing separator.
-  if (HeaderName.startswith(PathBuf))
+  if (HeaderName.starts_with(PathBuf))
     HeaderName = HeaderName.substr(PathBuf.size() + 1);
   // Save header name.
   ModuleMapHeadersSet.insert(ModularizeUtilities::getCanonicalPath(HeaderName));
@@ -356,8 +356,8 @@ bool CoverageChecker::collectFileSystemHeaders(StringRef IncludePath) {
     sys::path::append(Directory, IncludePath);
   if (Directory.size() == 0)
     Directory = ".";
-  if (IncludePath.startswith("/") || IncludePath.startswith("\\") ||
-    ((IncludePath.size() >= 2) && (IncludePath[1] == ':'))) {
+  if (IncludePath.starts_with("/") || IncludePath.starts_with("\\") ||
+      ((IncludePath.size() >= 2) && (IncludePath[1] == ':'))) {
     llvm::errs() << "error: Include path \"" << IncludePath
       << "\" is not relative to the module map file.\n";
     return false;
diff --git a/clang-tools-extra/modularize/ModularizeUtilities.cpp b/clang-tools-extra/modularize/ModularizeUtilities.cpp
index 089f52f52ec4d341826294df72da43eafae0637a..dfca3eefe473203f5b82c82ed5b909acc0c79bd8 100644
--- a/clang-tools-extra/modularize/ModularizeUtilities.cpp
+++ b/clang-tools-extra/modularize/ModularizeUtilities.cpp
@@ -75,12 +75,11 @@ std::error_code ModularizeUtilities::loadAllHeaderListsAndDependencies() {
   for (auto I = InputFilePaths.begin(), E = InputFilePaths.end(); I != E; ++I) {
     llvm::StringRef InputPath = *I;
     // If it's a module map.
-    if (InputPath.endswith(".modulemap")) {
+    if (InputPath.ends_with(".modulemap")) {
       // Load the module map.
       if (std::error_code EC = loadModuleMap(InputPath))
         return EC;
-    }
-    else {
+    } else {
       // Else we assume it's a header list and load it.
       if (std::error_code EC = loadSingleHeaderListsAndDependencies(InputPath)) {
         errs() << "modularize: error: Unable to get header list '" << InputPath
@@ -276,7 +275,7 @@ std::error_code ModularizeUtilities::loadModuleMap(
   StringRef DirName(Dir.getName());
   if (llvm::sys::path::filename(DirName) == "Modules") {
     DirName = llvm::sys::path::parent_path(DirName);
-    if (DirName.endswith(".framework")) {
+    if (DirName.ends_with(".framework")) {
       auto FrameworkDirOrErr = FileMgr->getDirectoryRef(DirName);
       if (!FrameworkDirOrErr) {
         // This can happen if there's a race between the above check and the
@@ -444,7 +443,7 @@ static std::string replaceDotDot(StringRef Path) {
       llvm::sys::path::append(Buffer, *B);
     ++B;
   }
-  if (Path.endswith("/") || Path.endswith("\\"))
+  if (Path.ends_with("/") || Path.ends_with("\\"))
     Buffer.append(1, Path.back());
   return Buffer.c_str();
 }
@@ -457,7 +456,7 @@ std::string ModularizeUtilities::getCanonicalPath(StringRef FilePath) {
   std::string Tmp(replaceDotDot(FilePath));
   std::replace(Tmp.begin(), Tmp.end(), '\\', '/');
   StringRef Tmp2(Tmp);
-  if (Tmp2.startswith("./"))
+  if (Tmp2.starts_with("./"))
     Tmp = std::string(Tmp2.substr(2));
   return Tmp;
 }
diff --git a/clang-tools-extra/modularize/PreprocessorTracker.cpp b/clang-tools-extra/modularize/PreprocessorTracker.cpp
index 335195c6b199e7690ed1fa8c061507631b6d9b83..7557fb177ceb48a24b8bc10960daba160c19e31e 100644
--- a/clang-tools-extra/modularize/PreprocessorTracker.cpp
+++ b/clang-tools-extra/modularize/PreprocessorTracker.cpp
@@ -883,7 +883,7 @@ public:
   // Handle entering a header source file.
   void handleHeaderEntry(clang::Preprocessor &PP, llvm::StringRef HeaderPath) {
     // Ignore <built-in> and <command-line> to reduce message clutter.
-    if (HeaderPath.startswith("<"))
+    if (HeaderPath.starts_with("<"))
       return;
     HeaderHandle H = addHeader(HeaderPath);
     if (H != getCurrentHeaderHandle())
@@ -896,7 +896,7 @@ public:
   // Handle exiting a header source file.
   void handleHeaderExit(llvm::StringRef HeaderPath) {
     // Ignore <built-in> and <command-line> to reduce message clutter.
-    if (HeaderPath.startswith("<"))
+    if (HeaderPath.starts_with("<"))
       return;
     HeaderHandle H = findHeaderHandle(HeaderPath);
     HeaderHandle TH;
diff --git a/clang-tools-extra/pseudo/lib/cxx/CXX.cpp b/clang-tools-extra/pseudo/lib/cxx/CXX.cpp
index 46d837aec44ad3eac8806e5ae661d58f3a9607b7..4188dab31d3a91284e7dba646fab065612617ff2 100644
--- a/clang-tools-extra/pseudo/lib/cxx/CXX.cpp
+++ b/clang-tools-extra/pseudo/lib/cxx/CXX.cpp
@@ -28,9 +28,9 @@ static const char *CXXBNF =
 
 // User-defined string literals look like `""suffix`.
 bool isStringUserDefined(const Token &Tok) {
-  return !Tok.text().endswith("\"");
+  return !Tok.text().ends_with("\"");
 }
-bool isCharUserDefined(const Token &Tok) { return !Tok.text().endswith("'"); }
+bool isCharUserDefined(const Token &Tok) { return !Tok.text().ends_with("'"); }
 
 // Combinable flags describing numbers.
 // Clang has just one numeric_token kind, the grammar has 4.
diff --git a/clang-tools-extra/pseudo/lib/grammar/GrammarBNF.cpp b/clang-tools-extra/pseudo/lib/grammar/GrammarBNF.cpp
index 9706f17eab848b9b449df359a952e358d1cf0539..f1b8e06e22432c63c872c97a3991493671a6a3ae 100644
--- a/clang-tools-extra/pseudo/lib/grammar/GrammarBNF.cpp
+++ b/clang-tools-extra/pseudo/lib/grammar/GrammarBNF.cpp
@@ -33,11 +33,11 @@ public:
 
     assert(llvm::all_of(Specs,
                         [](const RuleSpec &R) {
-                          if (R.Target.endswith(OptSuffix))
+                          if (R.Target.ends_with(OptSuffix))
                             return false;
                           return llvm::all_of(
                               R.Sequence, [](const RuleSpec::Element &E) {
-                                return !E.Symbol.endswith(OptSuffix);
+                                return !E.Symbol.ends_with(OptSuffix);
                               });
                         }) &&
            "Optional symbols should be eliminated!");
@@ -225,7 +225,7 @@ private:
       Chunk = Chunk.trim();
       if (Chunk.empty())
         continue; // skip empty
-      if (Chunk.startswith("[") && Chunk.endswith("]")) {
+      if (Chunk.starts_with("[") && Chunk.ends_with("]")) {
         if (Out.Sequence.empty())
           continue;
 
@@ -241,7 +241,7 @@ private:
   bool parseAttributes(
       llvm::StringRef Content,
       std::vector<std::pair<llvm::StringRef, llvm::StringRef>> &Out) {
-    assert(Content.startswith("[") && Content.endswith("]"));
+    assert(Content.starts_with("[") && Content.ends_with("]"));
     auto KV = Content.drop_front().drop_back().split('=');
     Out.push_back({KV.first, KV.second.trim()});
 
@@ -299,7 +299,7 @@ private:
     if (Elements.empty())
       return CB();
     auto Front = Elements.front();
-    if (!Front.Symbol.endswith(OptSuffix)) {
+    if (!Front.Symbol.ends_with(OptSuffix)) {
       Result.push_back(std::move(Front));
       eliminateOptionalTail(Elements.drop_front(1), Result, CB);
       Result.pop_back();
diff --git a/clang-tools-extra/unittests/clang-tidy/GoogleModuleTest.cpp b/clang-tools-extra/unittests/clang-tidy/GoogleModuleTest.cpp
index feb7ae913f9968487ebfbe82d3e12d44132e6cb1..e9ab987e493c41a2fcb701a1195ab0ae0f7560e6 100644
--- a/clang-tools-extra/unittests/clang-tidy/GoogleModuleTest.cpp
+++ b/clang-tools-extra/unittests/clang-tidy/GoogleModuleTest.cpp
@@ -66,7 +66,7 @@ protected:
                                       "#define SOME_MACRO(x) using x\n";
     std::vector<ClangTidyError> Errors;
     std::vector<std::string> Args;
-    if (!StringRef(Filename).endswith(".cpp")) {
+    if (!StringRef(Filename).ends_with(".cpp")) {
       Args.emplace_back("-xc++-header");
     }
     test::runCheckOnCode<readability::GlobalNamesInHeadersCheck>(
diff --git a/clang/cmake/caches/Fuchsia-stage2.cmake b/clang/cmake/caches/Fuchsia-stage2.cmake
index 4b9085d99378c6fb28fb11d509bcf4df4b777d5b..c4673c8a54c5ef531f2e77502d4338ce29c7a52d 100644
--- a/clang/cmake/caches/Fuchsia-stage2.cmake
+++ b/clang/cmake/caches/Fuchsia-stage2.cmake
@@ -336,6 +336,7 @@ set(LLVM_TOOLCHAIN_TOOLS
   llvm-symbolizer
   llvm-undname
   llvm-xray
+  opt-viewer
   sancov
   scan-build-py
   CACHE STRING "")
diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst
index 066e4ac5b9e54b83965d0cbc64740f12a06aaa0f..05d59d0da264f30a00a9df6f8f359a052e398fb1 100644
--- a/clang/docs/ReleaseNotes.rst
+++ b/clang/docs/ReleaseNotes.rst
@@ -685,6 +685,9 @@ Bug Fixes in This Version
   (`#62157 <https://github.com/llvm/llvm-project/issues/62157>`_) and
   (`#64885 <https://github.com/llvm/llvm-project/issues/64885>`_) and
   (`#65568 <https://github.com/llvm/llvm-project/issues/65568>`_)
+- Fixed false positive error emitted when templated alias inside a class
+  used private members of the same class.
+  Fixes (`#41693 <https://github.com/llvm/llvm-project/issues/41693>`_)
 
 Bug Fixes to Compiler Builtins
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td
index 26c318dd511cbb6bea98d7c8c446ca452eeca1ed..2b57058d3f1c75fc17e31a616e9c5328ed8a00c7 100644
--- a/clang/include/clang/Basic/Attr.td
+++ b/clang/include/clang/Basic/Attr.td
@@ -2215,7 +2215,8 @@ def NotTailCalled : InheritableAttr {
 def : MutualExclusions<[AlwaysInline, NotTailCalled]>;
 
 def NoStackProtector : InheritableAttr {
-  let Spellings = [Clang<"no_stack_protector">, Declspec<"safebuffers">];
+  let Spellings = [Clang<"no_stack_protector">, CXX11<"gnu", "no_stack_protector">,
+                   C23<"gnu", "no_stack_protector">, Declspec<"safebuffers">];
   let Subjects = SubjectList<[Function]>;
   let Documentation = [NoStackProtectorDocs];
   let SimpleHandler = 1;
@@ -2795,9 +2796,10 @@ def SwiftAsyncError : InheritableAttr {
   let Documentation = [SwiftAsyncErrorDocs];
 }
 
-def Suppress : StmtAttr {
-  let Spellings = [CXX11<"gsl", "suppress">];
+def Suppress : DeclOrStmtAttr {
+  let Spellings = [CXX11<"gsl", "suppress">, Clang<"suppress">];
   let Args = [VariadicStringArgument<"DiagnosticIdentifiers">];
+  let Accessors = [Accessor<"isGSL", [CXX11<"gsl", "suppress">]>];
   let Documentation = [SuppressDocs];
 }
 
@@ -2881,7 +2883,7 @@ def Target : InheritableAttr {
 
       for (auto &Feature : AttrFeatures) {
         Feature = Feature.trim();
-        if (Feature.startswith("arch="))
+        if (Feature.starts_with("arch="))
           return Feature.drop_front(sizeof("arch=") - 1);
       }
       return "";
@@ -2899,8 +2901,8 @@ def Target : InheritableAttr {
       for (auto &Feature : AttrFeatures) {
         Feature = Feature.trim();
 
-        if (!Feature.startswith("no-") && !Feature.startswith("arch=") &&
-            !Feature.startswith("fpmath=") && !Feature.startswith("tune="))
+        if (!Feature.starts_with("no-") && !Feature.starts_with("arch=") &&
+            !Feature.starts_with("fpmath=") && !Feature.starts_with("tune="))
           Out.push_back(Feature);
       }
     }
diff --git a/clang/include/clang/Basic/AttrDocs.td b/clang/include/clang/Basic/AttrDocs.td
index a8de566db1a7d301edacf1352361c646ada95a3b..90041fa8dbb30b9b7d7be0f91eb4e03437152a29 100644
--- a/clang/include/clang/Basic/AttrDocs.td
+++ b/clang/include/clang/Basic/AttrDocs.td
@@ -5267,7 +5267,74 @@ the ``int`` parameter is the one that represents the error.
 def SuppressDocs : Documentation {
   let Category = DocCatStmt;
   let Content = [{
-The ``[[gsl::suppress]]`` attribute suppresses specific
+The ``suppress`` attribute suppresses unwanted warnings coming from static
+analysis tools such as the Clang Static Analyzer. The tool will not report
+any issues in source code annotated with the attribute.
+
+The attribute cannot be used to suppress traditional Clang warnings, because
+many such warnings are emitted before the attribute is fully parsed.
+Consider using ``#pragma clang diagnostic`` to control such diagnostics,
+as described in `Controlling Diagnostics via Pragmas
+<https://clang.llvm.org/docs/UsersManual.html#controlling-diagnostics-via-pragmas>`_.
+
+The ``suppress`` attribute can be placed on an individual statement in order to
+suppress warnings about undesirable behavior occurring at that statement:
+
+.. code-block:: c++
+
+  int foo() {
+    int *x = nullptr;
+    ...
+    [[clang::suppress]]
+    return *x;  // null pointer dereference warning suppressed here
+  }
+
+Putting the attribute on a compound statement suppresses all warnings in scope:
+
+.. code-block:: c++
+
+  int foo() {
+    [[clang::suppress]] {
+      int *x = nullptr;
+      ...
+      return *x;  // warnings suppressed in the entire scope
+    }
+  }
+
+Some static analysis warnings are accompanied by one or more notes, and the
+line of code against which the warning is emitted isn't necessarily the best
+for suppression purposes. In such cases the tools are allowed to implement
+additional ways to suppress specific warnings based on the attribute attached
+to a note location.
+
+For example, the Clang Static Analyzer suppresses memory leak warnings when
+the suppression attribute is placed at the allocation site (highlited by
+a "note: memory is allocated"), which may be different from the line of code
+at which the program "loses track" of the pointer (where the warning
+is ultimately emitted):
+
+.. code-block:: c
+
+  int bar1(bool coin_flip) {
+    __attribute__((suppress))
+    int *result = (int *)malloc(sizeof(int));
+    if (coin_flip)
+      return 1;  // warning about this leak path is suppressed
+
+    return *result;  // warning about this leak path is also suppressed
+  }
+
+  int bar2(bool coin_flip) {
+    int *result = (int *)malloc(sizeof(int));
+    if (coin_flip)
+      return 1;  // leak warning on this path NOT suppressed
+
+    __attribute__((suppress))
+    return *result;  // leak warning is suppressed only on this path
+  }
+
+
+When written as ``[[gsl::suppress]]``, this attribute suppresses specific
 clang-tidy diagnostics for rules of the `C++ Core Guidelines`_ in a portable
 way. The attribute can be attached to declarations, statements, and at
 namespace scope.
diff --git a/clang/include/clang/Basic/BuiltinsAMDGPU.def b/clang/include/clang/Basic/BuiltinsAMDGPU.def
index 8b59b3790d7bc661361cf6f2b8f154877425658a..7465f13d552d6e41390fe1acc25f9ce55ac4451c 100644
--- a/clang/include/clang/Basic/BuiltinsAMDGPU.def
+++ b/clang/include/clang/Basic/BuiltinsAMDGPU.def
@@ -406,5 +406,21 @@ TARGET_BUILTIN(__builtin_amdgcn_cvt_pk_fp8_f32, "iffiIb", "nc", "fp8-insts")
 TARGET_BUILTIN(__builtin_amdgcn_cvt_sr_bf8_f32, "ifiiIi", "nc", "fp8-insts")
 TARGET_BUILTIN(__builtin_amdgcn_cvt_sr_fp8_f32, "ifiiIi", "nc", "fp8-insts")
 
+//===----------------------------------------------------------------------===//
+// GFX12+ only builtins.
+//===----------------------------------------------------------------------===//
+
+TARGET_BUILTIN(__builtin_amdgcn_s_barrier_signal, "vIi", "n", "gfx12-insts")
+TARGET_BUILTIN(__builtin_amdgcn_s_barrier_signal_var, "vi", "n", "gfx12-insts")
+TARGET_BUILTIN(__builtin_amdgcn_s_barrier_wait, "vIs", "n", "gfx12-insts")
+TARGET_BUILTIN(__builtin_amdgcn_s_barrier_signal_isfirst, "bIi", "n", "gfx12-insts")
+TARGET_BUILTIN(__builtin_amdgcn_s_barrier_signal_isfirst_var, "bi", "n", "gfx12-insts")
+TARGET_BUILTIN(__builtin_amdgcn_s_barrier_init, "vii", "n", "gfx12-insts")
+TARGET_BUILTIN(__builtin_amdgcn_s_barrier_join, "vi", "n", "gfx12-insts")
+TARGET_BUILTIN(__builtin_amdgcn_s_wakeup_barrier, "vi", "n", "gfx12-insts")
+TARGET_BUILTIN(__builtin_amdgcn_s_barrier_leave, "b", "n", "gfx12-insts")
+TARGET_BUILTIN(__builtin_amdgcn_s_get_barrier_state, "Uii", "n", "gfx12-insts")
+
+
 #undef BUILTIN
 #undef TARGET_BUILTIN
diff --git a/clang/include/clang/Basic/Features.def b/clang/include/clang/Basic/Features.def
index 7473e00a7bd86bae6f92d4882070f2337a169c65..06efac0cf1abd7e7a529a9b7f3616b97eebb7010 100644
--- a/clang/include/clang/Basic/Features.def
+++ b/clang/include/clang/Basic/Features.def
@@ -282,7 +282,6 @@ EXTENSION(matrix_types_scalar_division, true)
 EXTENSION(cxx_attributes_on_using_declarations, LangOpts.CPlusPlus11)
 EXTENSION(datasizeof, LangOpts.CPlusPlus)
 
-FEATURE(builtin_headers_in_system_modules, LangOpts.BuiltinHeadersInSystemModules)
 FEATURE(cxx_abi_relative_vtable, LangOpts.CPlusPlus && LangOpts.RelativeCXXABIVTables)
 
 // CUDA/HIP Features
diff --git a/clang/include/clang/Basic/IdentifierTable.h b/clang/include/clang/Basic/IdentifierTable.h
index 0898e7d39dd7dee20301aa9926e66cefdfae7e0f..1ac182d4fce26f65f2179aaee96b4706bbb696ea 100644
--- a/clang/include/clang/Basic/IdentifierTable.h
+++ b/clang/include/clang/Basic/IdentifierTable.h
@@ -511,7 +511,7 @@ public:
   ///   function(<#int x#>);
   /// \endcode
   bool isEditorPlaceholder() const {
-    return getName().startswith("<#") && getName().endswith("#>");
+    return getName().starts_with("<#") && getName().ends_with("#>");
   }
 
   /// Determine whether \p this is a name reserved for the implementation (C99
diff --git a/clang/include/clang/Basic/TargetBuiltins.h b/clang/include/clang/Basic/TargetBuiltins.h
index 8f7881abf26f7f49f5c71cec97248c166e6b5af5..c9f9cbec7493bfc91d21cf5f423ddb703252d86a 100644
--- a/clang/include/clang/Basic/TargetBuiltins.h
+++ b/clang/include/clang/Basic/TargetBuiltins.h
@@ -309,7 +309,7 @@ namespace clang {
     bool isTupleSet() const { return Flags & IsTupleSet; }
     bool isReadZA() const { return Flags & IsReadZA; }
     bool isWriteZA() const { return Flags & IsWriteZA; }
-
+    bool isReductionQV() const { return Flags & IsReductionQV; }
     uint64_t getBits() const { return Flags; }
     bool isFlagSet(uint64_t Flag) const { return Flags & Flag; }
   };
diff --git a/clang/include/clang/Basic/arm_sve.td b/clang/include/clang/Basic/arm_sve.td
index aa9b105364a51a1004873c95e4dda5ac811268df..278a791ff760dc9131f0f3d2ff796c807dc9c457 100644
--- a/clang/include/clang/Basic/arm_sve.td
+++ b/clang/include/clang/Basic/arm_sve.td
@@ -1946,6 +1946,23 @@ def SVPSEL_COUNT_ALIAS_S : SInst<"svpsel_lane_c32", "}}Pm", "Pi", MergeNone, "",
 def SVPSEL_COUNT_ALIAS_D : SInst<"svpsel_lane_c64", "}}Pm", "Pl", MergeNone, "", [IsStreamingCompatible], []>;
 }
 
+// Standalone sve2.1 builtins
+let TargetGuard = "sve2p1" in {
+def SVORQV   : SInst<"svorqv[_{d}]", "{Pd", "csilUcUsUiUl", MergeNone, "aarch64_sve_orqv", [IsReductionQV]>;
+def SVEORQV  : SInst<"sveorqv[_{d}]", "{Pd", "csilUcUsUiUl", MergeNone, "aarch64_sve_eorqv", [IsReductionQV]>;
+def SVADDQV  : SInst<"svaddqv[_{d}]", "{Pd", "hfdcsilUcUsUiUl", MergeNone, "aarch64_sve_addqv", [IsReductionQV]>;
+def SVANDQV  : SInst<"svandqv[_{d}]", "{Pd", "csilUcUsUiUl", MergeNone, "aarch64_sve_andqv", [IsReductionQV]>;
+def SVSMAXQV : SInst<"svmaxqv[_{d}]", "{Pd", "csil", MergeNone, "aarch64_sve_smaxqv", [IsReductionQV]>;
+def SVUMAXQV : SInst<"svmaxqv[_{d}]", "{Pd", "UcUsUiUl", MergeNone, "aarch64_sve_umaxqv", [IsReductionQV]>;
+def SVSMINQV : SInst<"svminqv[_{d}]", "{Pd", "csil", MergeNone, "aarch64_sve_sminqv", [IsReductionQV]>;
+def SVUMINQV : SInst<"svminqv[_{d}]", "{Pd", "UcUsUiUl", MergeNone, "aarch64_sve_uminqv", [IsReductionQV]>;
+
+def SVFMAXNMQV: SInst<"svmaxnmqv[_{d}]", "{Pd", "hfd", MergeNone, "aarch64_sve_fmaxnmqv", [IsReductionQV]>;
+def SVFMINNMQV: SInst<"svminnmqv[_{d}]", "{Pd", "hfd", MergeNone, "aarch64_sve_fminnmqv", [IsReductionQV]>;
+def SVFMAXQV: SInst<"svmaxqv[_{d}]", "{Pd", "hfd", MergeNone, "aarch64_sve_fmaxqv", [IsReductionQV]>;
+def SVFMINQV: SInst<"svminqv[_{d}]", "{Pd", "hfd", MergeNone, "aarch64_sve_fminqv", [IsReductionQV]>;
+}
+
 let TargetGuard = "sve2p1|sme2" in {
 //FIXME: Replace IsStreamingCompatible with IsStreamingOrHasSVE2p1 when available
 def SVPEXT_SINGLE : SInst<"svpext_lane_{d}", "P}i", "QcQsQiQl", MergeNone, "aarch64_sve_pext", [IsStreamingCompatible], [ImmCheck<1, ImmCheck0_3>]>;
@@ -2168,6 +2185,21 @@ let TargetGuard = "sme2" in {
 
   def REINTERPRET_SVBOOL_TO_SVCOUNT : Inst<"svreinterpret[_c]", "}P", "Pc", MergeNone, "", [IsStreamingCompatible], []>;
   def REINTERPRET_SVCOUNT_TO_SVBOOL : Inst<"svreinterpret[_b]", "P}", "Pc", MergeNone, "", [IsStreamingCompatible], []>;
+
+  // SQDMULH
+  def SVSQDMULH_SINGLE_X2 : SInst<"svqdmulh[_single_{d}_x2]", "22d", "csil", MergeNone, "aarch64_sve_sqdmulh_single_vgx2", [IsStreaming], []>;
+  def SVSQDMULH_SINGLE_X4 : SInst<"svqdmulh[_single_{d}_x4]", "44d", "csil", MergeNone, "aarch64_sve_sqdmulh_single_vgx4", [IsStreaming], []>;
+  def SVSQDMULH_X2        : SInst<"svqdmulh[_{d}_x2]",        "222", "csil", MergeNone, "aarch64_sve_sqdmulh_vgx2",        [IsStreaming], []>;
+  def SVSQDMULH_X4        : SInst<"svqdmulh[_{d}_x4]",        "444", "csil", MergeNone, "aarch64_sve_sqdmulh_vgx4",        [IsStreaming], []>;
+}
+
+let TargetGuard = "sve2p1|sme2" in {
+  // SQRSHRN / UQRSHRN
+  def SVQRSHRN_X2   : SInst<"svqrshrn[_n]_{0}[_{d}_x2]", "h2i", "i",    MergeNone, "aarch64_sve_sqrshrn_x2", [IsStreamingCompatible], [ImmCheck<1, ImmCheck1_16>]>;
+  def SVUQRSHRN_X2  : SInst<"svqrshrn[_n]_{0}[_{d}_x2]", "e2i", "Ui",   MergeNone, "aarch64_sve_uqrshrn_x2", [IsStreamingCompatible], [ImmCheck<1, ImmCheck1_16>]>;
+
+  // SQRSHRUN
+  def SVSQRSHRUN_X2 : SInst<"svqrshrun[_n]_{0}[_{d}_x2]", "e2i", "i",  MergeNone, "aarch64_sve_sqrshrun_x2", [IsStreamingCompatible], [ImmCheck<1, ImmCheck1_16>]>;
 }
 
 let TargetGuard = "sve2p1" in {
diff --git a/clang/include/clang/Basic/arm_sve_sme_incl.td b/clang/include/clang/Basic/arm_sve_sme_incl.td
index 040ce95a57de3dc43385269897de2e0816c5da40..0dba8493bad2d6a48e70cb3382c8161dc47debdb 100644
--- a/clang/include/clang/Basic/arm_sve_sme_incl.td
+++ b/clang/include/clang/Basic/arm_sve_sme_incl.td
@@ -129,6 +129,7 @@
 // Z: const pointer to uint64_t
 
 // Prototype modifiers added for SVE2p1
+// {: 128b vector
 // }: svcount_t
 
 class MergeType<int val, string suffix=""> {
@@ -225,6 +226,7 @@ def IsSharedZA                : FlagType<0x8000000000>;
 def IsPreservesZA             : FlagType<0x10000000000>;
 def IsReadZA                  : FlagType<0x20000000000>;
 def IsWriteZA                 : FlagType<0x40000000000>;
+def IsReductionQV             : FlagType<0x80000000000>;
 
 // These must be kept in sync with the flags in include/clang/Basic/TargetBuiltins.h
 class ImmCheckType<int val> {
diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td
index 25c76cf2ad2c84ad15c2ac6a39390043477afd91..1b02087425b751516424103857846a665b76b53c 100644
--- a/clang/include/clang/Driver/Options.td
+++ b/clang/include/clang/Driver/Options.td
@@ -2896,9 +2896,11 @@ defm asm_blocks : BoolFOption<"asm-blocks",
   LangOpts<"AsmBlocks">, Default<fms_extensions.KeyPath>,
   PosFlag<SetTrue, [], [ClangOption, CC1Option]>,
   NegFlag<SetFalse>>;
-def fms_volatile : Flag<["-"], "fms-volatile">, Group<f_Group>,
-  Visibility<[ClangOption, CC1Option]>,
-  MarshallingInfoFlag<LangOpts<"MSVolatile">>;
+defm ms_volatile : BoolFOption<"ms-volatile",
+  LangOpts<"MSVolatile">, DefaultFalse,
+  PosFlag<SetTrue, [], [ClangOption, CC1Option],
+          "Volatile loads and stores have acquire and release semantics">,
+  NegFlag<SetFalse>>;
 def fmsc_version : Joined<["-"], "fmsc-version=">, Group<f_Group>,
   Visibility<[ClangOption, CLOption]>,
   HelpText<"Microsoft compiler version number to report in _MSC_VER (0 = don't define it (default))">;
@@ -8217,7 +8219,7 @@ def _SLASH_winsysroot : CLJoinedOrSeparate<"winsysroot">,
   HelpText<"Same as \"/diasdkdir <dir>/DIA SDK\" /vctoolsdir <dir>/VC/Tools/MSVC/<vctoolsversion> \"/winsdkdir <dir>/Windows Kits/10\"">,
   MetaVarName<"<dir>">;
 def _SLASH_volatile_iso : Option<["/", "-"], "volatile:iso", KIND_FLAG>,
-  Group<_SLASH_volatile_Group>, Flags<[NoXarchOption]>, Visibility<[CLOption]>,
+  Visibility<[CLOption]>, Alias<fno_ms_volatile>,
   HelpText<"Volatile loads and stores have standard semantics">;
 def _SLASH_vmb : CLFlag<"vmb">,
   HelpText<"Use a best-case representation method for member pointers">;
@@ -8232,7 +8234,7 @@ def _SLASH_vmv : CLFlag<"vmv">,
   HelpText<"Set the default most-general representation to "
            "virtual inheritance">;
 def _SLASH_volatile_ms  : Option<["/", "-"], "volatile:ms", KIND_FLAG>,
-  Group<_SLASH_volatile_Group>, Flags<[NoXarchOption]>, Visibility<[CLOption]>,
+  Visibility<[CLOption]>, Alias<fms_volatile>,
   HelpText<"Volatile loads and stores have acquire and release semantics">;
 def _SLASH_clang : CLJoined<"clang:">,
   HelpText<"Pass <arg> to the clang driver">, MetaVarName<"<arg>">;
diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h
index 1d7b4c729ce84e0968f78d5250c70438e86e927a..7e89e74733a090350a9e9b2d7932d9353c06580b 100644
--- a/clang/include/clang/Sema/Sema.h
+++ b/clang/include/clang/Sema/Sema.h
@@ -8737,7 +8737,7 @@ public:
                              SourceLocation IILoc,
                              bool DeducedTSTContext = true);
 
-
+  bool RebuildingTypesInCurrentInstantiation = false;
   TypeSourceInfo *RebuildTypeInCurrentInstantiation(TypeSourceInfo *T,
                                                     SourceLocation Loc,
                                                     DeclarationName Name);
diff --git a/clang/include/clang/StaticAnalyzer/Core/AnalyzerOptions.h b/clang/include/clang/StaticAnalyzer/Core/AnalyzerOptions.h
index a947bd0867025c2087a6278d3c13b2cbda33bba1..276d11e80a5b21c78acd293650fadeddcf4bd1c3 100644
--- a/clang/include/clang/StaticAnalyzer/Core/AnalyzerOptions.h
+++ b/clang/include/clang/StaticAnalyzer/Core/AnalyzerOptions.h
@@ -409,8 +409,8 @@ AnalyzerOptions::getRegisteredCheckers(bool IncludeExperimental) {
   };
   std::vector<StringRef> Checkers;
   for (StringRef CheckerName : StaticAnalyzerCheckerNames) {
-    if (!CheckerName.startswith("debug.") &&
-        (IncludeExperimental || !CheckerName.startswith("alpha.")))
+    if (!CheckerName.starts_with("debug.") &&
+        (IncludeExperimental || !CheckerName.starts_with("alpha.")))
       Checkers.push_back(CheckerName);
   }
   return Checkers;
diff --git a/clang/include/clang/StaticAnalyzer/Core/BugReporter/BugReporter.h b/clang/include/clang/StaticAnalyzer/Core/BugReporter/BugReporter.h
index 8956552e7bfc21e4c0f46ecbb7e0285f0bb6f032..e762f7548e0b54186883f140bcc793378913a67b 100644
--- a/clang/include/clang/StaticAnalyzer/Core/BugReporter/BugReporter.h
+++ b/clang/include/clang/StaticAnalyzer/Core/BugReporter/BugReporter.h
@@ -19,6 +19,7 @@
 #include "clang/Basic/SourceLocation.h"
 #include "clang/Lex/Preprocessor.h"
 #include "clang/StaticAnalyzer/Core/BugReporter/BugReporterVisitors.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugSuppression.h"
 #include "clang/StaticAnalyzer/Core/BugReporter/BugType.h"
 #include "clang/StaticAnalyzer/Core/CheckerManager.h"
 #include "clang/StaticAnalyzer/Core/PathSensitive/ExplodedGraph.h"
@@ -594,6 +595,9 @@ private:
   /// A vector of BugReports for tracking the allocated pointers and cleanup.
   std::vector<BugReportEquivClass *> EQClassesVector;
 
+  /// User-provided in-code suppressions.
+  BugSuppression UserSuppressions;
+
 public:
   BugReporter(BugReporterData &d);
   virtual ~BugReporter();
diff --git a/clang/include/clang/StaticAnalyzer/Core/BugReporter/BugSuppression.h b/clang/include/clang/StaticAnalyzer/Core/BugReporter/BugSuppression.h
new file mode 100644
index 0000000000000000000000000000000000000000..4fd81b6275197446f3cf40acc1ef36ed9cc22dff
--- /dev/null
+++ b/clang/include/clang/StaticAnalyzer/Core/BugReporter/BugSuppression.h
@@ -0,0 +1,53 @@
+//===- BugSuppression.h - Suppression interface -----------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+//  This file defines BugSuppression, a simple interface class encapsulating
+//  all user provided in-code suppressions.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_STATICANALYZER_CORE_BUGREPORTER_SUPPRESSION_H
+#define LLVM_CLANG_STATICANALYZER_CORE_BUGREPORTER_SUPPRESSION_H
+
+#include "clang/Basic/SourceLocation.h"
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/SmallVector.h"
+
+namespace clang {
+class Decl;
+
+namespace ento {
+class BugReport;
+class PathDiagnosticLocation;
+
+class BugSuppression {
+public:
+  using DiagnosticIdentifierList = llvm::ArrayRef<llvm::StringRef>;
+
+  /// Return true if the given bug report was explicitly suppressed by the user.
+  bool isSuppressed(const BugReport &);
+
+  /// Return true if the bug reported at the given location was explicitly
+  /// suppressed by the user.
+  bool isSuppressed(const PathDiagnosticLocation &Location,
+                    const Decl *DeclWithIssue,
+                    DiagnosticIdentifierList DiagnosticIdentification);
+
+private:
+  // Overly pessimistic number, to be honest.
+  static constexpr unsigned EXPECTED_NUMBER_OF_SUPPRESSIONS = 8;
+  using CachedRanges =
+      llvm::SmallVector<SourceRange, EXPECTED_NUMBER_OF_SUPPRESSIONS>;
+
+  llvm::DenseMap<const Decl *, CachedRanges> CachedSuppressionLocations;
+};
+
+} // end namespace ento
+} // end namespace clang
+
+#endif // LLVM_CLANG_STATICANALYZER_CORE_BUGREPORTER_SUPPRESSION_H
diff --git a/clang/lib/APINotes/APINotesManager.cpp b/clang/lib/APINotes/APINotesManager.cpp
index ec1fb3ffa961c95cbfba635388ff19f8751401dc..a921c8b9fce3e0e0923f4bb621234dbaa189cac3 100644
--- a/clang/lib/APINotes/APINotesManager.cpp
+++ b/clang/lib/APINotes/APINotesManager.cpp
@@ -198,7 +198,7 @@ static void checkPrivateAPINotesName(DiagnosticsEngine &Diags,
   StringRef RealFileName =
       llvm::sys::path::filename(File->tryGetRealPathName());
   StringRef RealStem = llvm::sys::path::stem(RealFileName);
-  if (RealStem.endswith("_private"))
+  if (RealStem.ends_with("_private"))
     return;
 
   unsigned DiagID = diag::warn_apinotes_private_case;
diff --git a/clang/lib/APINotes/APINotesYAMLCompiler.cpp b/clang/lib/APINotes/APINotesYAMLCompiler.cpp
index 4dfd01dae05f2ca8944b57d778de967e4f3b6533..57d6da7a1775960beec3ea1af7e26c5b905602c4 100644
--- a/clang/lib/APINotes/APINotesYAMLCompiler.cpp
+++ b/clang/lib/APINotes/APINotesYAMLCompiler.cpp
@@ -745,7 +745,7 @@ public:
     convertCommonEntity(M, MI, M.Selector);
 
     // Check if the selector ends with ':' to determine if it takes arguments.
-    bool takesArguments = M.Selector.endswith(":");
+    bool takesArguments = M.Selector.ends_with(":");
 
     // Split the selector into pieces.
     llvm::SmallVector<StringRef, 4> Args;
diff --git a/clang/lib/ARCMigrate/ARCMT.cpp b/clang/lib/ARCMigrate/ARCMT.cpp
index 8e398977dcd65d6ba25a17678961345f6fa842b5..b410d5f3b42a7eae63a838ee842637c376df02ba 100644
--- a/clang/lib/ARCMigrate/ARCMT.cpp
+++ b/clang/lib/ARCMigrate/ARCMT.cpp
@@ -201,7 +201,7 @@ createInvocationForMigration(CompilerInvocation &origCI,
   for (std::vector<std::string>::iterator
          I = CInvok->getDiagnosticOpts().Warnings.begin(),
          E = CInvok->getDiagnosticOpts().Warnings.end(); I != E; ++I) {
-    if (!StringRef(*I).startswith("error"))
+    if (!StringRef(*I).starts_with("error"))
       WarnOpts.push_back(*I);
   }
   WarnOpts.push_back("error=arc-unsafe-retained-assign");
diff --git a/clang/lib/ARCMigrate/ObjCMT.cpp b/clang/lib/ARCMigrate/ObjCMT.cpp
index 5a25c88c65f64b4a1468dd6e681ea054b6b00ad3..ed363a46a2004439911a046271b10a34e04d3d02 100644
--- a/clang/lib/ARCMigrate/ObjCMT.cpp
+++ b/clang/lib/ARCMigrate/ObjCMT.cpp
@@ -562,7 +562,7 @@ static void rewriteToObjCProperty(const ObjCMethodDecl *Getter,
 static bool IsCategoryNameWithDeprecatedSuffix(ObjCContainerDecl *D) {
   if (ObjCCategoryDecl *CatDecl = dyn_cast<ObjCCategoryDecl>(D)) {
     StringRef Name = CatDecl->getName();
-    return Name.endswith("Deprecated");
+    return Name.ends_with("Deprecated");
   }
   return false;
 }
@@ -1176,12 +1176,12 @@ bool ObjCMigrateASTConsumer::migrateProperty(ASTContext &Ctx,
   if (!SetterMethod) {
     // try a different naming convention for getter: isXxxxx
     StringRef getterNameString = getterName->getName();
-    bool IsPrefix = getterNameString.startswith("is");
+    bool IsPrefix = getterNameString.starts_with("is");
     // Note that we don't want to change an isXXX method of retainable object
     // type to property (readonly or otherwise).
     if (IsPrefix && GRT->isObjCRetainableType())
       return false;
-    if (IsPrefix || getterNameString.startswith("get")) {
+    if (IsPrefix || getterNameString.starts_with("get")) {
       LengthOfPrefix = (IsPrefix ? 2 : 3);
       const char *CGetterName = getterNameString.data() + LengthOfPrefix;
       // Make sure that first character after "is" or "get" prefix can
@@ -1320,11 +1320,11 @@ void ObjCMigrateASTConsumer::migrateFactoryMethod(ASTContext &Ctx,
   if (OIT_Family == OIT_Singleton || OIT_Family == OIT_ReturnsSelf) {
     StringRef STRefMethodName(MethodName);
     size_t len = 0;
-    if (STRefMethodName.startswith("standard"))
+    if (STRefMethodName.starts_with("standard"))
       len = strlen("standard");
-    else if (STRefMethodName.startswith("shared"))
+    else if (STRefMethodName.starts_with("shared"))
       len = strlen("shared");
-    else if (STRefMethodName.startswith("default"))
+    else if (STRefMethodName.starts_with("default"))
       len = strlen("default");
     else
       return;
@@ -1341,7 +1341,7 @@ void ObjCMigrateASTConsumer::migrateFactoryMethod(ASTContext &Ctx,
   StringRef LoweredMethodName(MethodName);
   std::string StringLoweredMethodName = LoweredMethodName.lower();
   LoweredMethodName = StringLoweredMethodName;
-  if (!LoweredMethodName.startswith(ClassNamePostfix))
+  if (!LoweredMethodName.starts_with(ClassNamePostfix))
     return;
   if (OIT_Family == OIT_ReturnsSelf)
     ReplaceWithClasstype(*this, OM);
diff --git a/clang/lib/ARCMigrate/TransUnbridgedCasts.cpp b/clang/lib/ARCMigrate/TransUnbridgedCasts.cpp
index 40220a2eef4910be7630a481444ea71b3eb57b5d..1e6354f71e294a90cb7164255f67786bcbafd3c8 100644
--- a/clang/lib/ARCMigrate/TransUnbridgedCasts.cpp
+++ b/clang/lib/ARCMigrate/TransUnbridgedCasts.cpp
@@ -146,7 +146,7 @@ private:
             ento::cocoa::isRefType(E->getSubExpr()->getType(), "CF",
                                    FD->getIdentifier()->getName())) {
           StringRef fname = FD->getIdentifier()->getName();
-          if (fname.endswith("Retain") || fname.contains("Create") ||
+          if (fname.ends_with("Retain") || fname.contains("Create") ||
               fname.contains("Copy")) {
             // Do not migrate to couple of bridge transfer casts which
             // cancel each other out. Leave it unchanged so error gets user
diff --git a/clang/lib/ARCMigrate/TransformActions.cpp b/clang/lib/ARCMigrate/TransformActions.cpp
index bd5c793568671d7a4ad814d9ac6d7f57285ffcda..6bc6fed1a9032013a14f5faa54a74d7dddf7e9a5 100644
--- a/clang/lib/ARCMigrate/TransformActions.cpp
+++ b/clang/lib/ARCMigrate/TransformActions.cpp
@@ -431,7 +431,7 @@ bool TransformActionsImpl::canReplaceText(SourceLocation loc, StringRef text) {
   if (invalidTemp)
     return false;
 
-  return file.substr(locInfo.second).startswith(text);
+  return file.substr(locInfo.second).starts_with(text);
 }
 
 void TransformActionsImpl::commitInsert(SourceLocation loc, StringRef text) {
diff --git a/clang/lib/ARCMigrate/Transforms.cpp b/clang/lib/ARCMigrate/Transforms.cpp
index 90b2b32b6b1be24865602a533677ab5afad27466..2808e35135dc358c832f87715a8d699e5c0e6242 100644
--- a/clang/lib/ARCMigrate/Transforms.cpp
+++ b/clang/lib/ARCMigrate/Transforms.cpp
@@ -95,7 +95,7 @@ bool trans::isPlusOne(const Expr *E) {
           ento::cocoa::isRefType(callE->getType(), "CF",
                                  FD->getIdentifier()->getName())) {
         StringRef fname = FD->getIdentifier()->getName();
-        if (fname.endswith("Retain") || fname.contains("Create") ||
+        if (fname.ends_with("Retain") || fname.contains("Create") ||
             fname.contains("Copy"))
           return true;
       }
diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp
index e877f903b34c6bbbb1089231aec07494e1acf0c2..0395b3e47ab6f8f4b4aab79cb14231709c1572c1 100644
--- a/clang/lib/AST/ASTContext.cpp
+++ b/clang/lib/AST/ASTContext.cpp
@@ -8223,7 +8223,7 @@ void ASTContext::getObjCEncodingForTypeImpl(QualType T, std::string &S,
       // Another legacy compatibility encoding. Some ObjC qualifier and type
       // combinations need to be rearranged.
       // Rewrite "in const" from "nr" to "rn"
-      if (StringRef(S).endswith("nr"))
+      if (StringRef(S).ends_with("nr"))
         S.replace(S.end()-2, S.end(), "rn");
     }
 
@@ -13519,7 +13519,7 @@ void ASTContext::getFunctionFeatureMap(llvm::StringMap<bool> &FeatureMap,
                       Target->getTargetOpts().FeaturesAsWritten.begin(),
                       Target->getTargetOpts().FeaturesAsWritten.end());
     } else {
-      if (VersionStr.startswith("arch="))
+      if (VersionStr.starts_with("arch="))
         TargetCPU = VersionStr.drop_front(sizeof("arch=") - 1);
       else if (VersionStr != "default")
         Features.push_back((StringRef{"+"} + VersionStr).str());
diff --git a/clang/lib/AST/DeclPrinter.cpp b/clang/lib/AST/DeclPrinter.cpp
index 30a26d518386c52363645bd8ddb986c6696cbf9b..24da6f2ef32b4fc3c17d90e75b5bd4315c48d4e2 100644
--- a/clang/lib/AST/DeclPrinter.cpp
+++ b/clang/lib/AST/DeclPrinter.cpp
@@ -1728,7 +1728,7 @@ void DeclPrinter::VisitObjCPropertyDecl(ObjCPropertyDecl *PDecl) {
   std::string TypeStr = PDecl->getASTContext().getUnqualifiedObjCPointerType(T).
       getAsString(Policy);
   Out << ' ' << TypeStr;
-  if (!StringRef(TypeStr).endswith("*"))
+  if (!StringRef(TypeStr).ends_with("*"))
     Out << ' ';
   Out << *PDecl;
   if (Policy.PolishForDeclaration)
diff --git a/clang/lib/AST/Interp/ByteCodeExprGen.cpp b/clang/lib/AST/Interp/ByteCodeExprGen.cpp
index f7f8e6c73d84e21369343ff5eac81602a725480c..d0980882f402b9b7a6cd77ae20c3df211786dd43 100644
--- a/clang/lib/AST/Interp/ByteCodeExprGen.cpp
+++ b/clang/lib/AST/Interp/ByteCodeExprGen.cpp
@@ -222,6 +222,64 @@ bool ByteCodeExprGen<Emitter>::VisitCastExpr(const CastExpr *CE) {
     return this->emitNE(PtrT, CE);
   }
 
+  case CK_IntegralComplexToBoolean:
+  case CK_FloatingComplexToBoolean: {
+    std::optional<PrimType> ElemT =
+        classifyComplexElementType(SubExpr->getType());
+    if (!ElemT)
+      return false;
+    // We emit the expression (__real(E) != 0 || __imag(E) != 0)
+    // for us, that means (bool)E[0] || (bool)E[1]
+    if (!this->visit(SubExpr))
+      return false;
+    if (!this->emitConstUint8(0, CE))
+      return false;
+    if (!this->emitArrayElemPtrUint8(CE))
+      return false;
+    if (!this->emitLoadPop(*ElemT, CE))
+      return false;
+    if (*ElemT == PT_Float) {
+      if (!this->emitCastFloatingIntegral(PT_Bool, CE))
+        return false;
+    } else {
+      if (!this->emitCast(*ElemT, PT_Bool, CE))
+        return false;
+    }
+
+    // We now have the bool value of E[0] on the stack.
+    LabelTy LabelTrue = this->getLabel();
+    if (!this->jumpTrue(LabelTrue))
+      return false;
+
+    if (!this->emitConstUint8(1, CE))
+      return false;
+    if (!this->emitArrayElemPtrPopUint8(CE))
+      return false;
+    if (!this->emitLoadPop(*ElemT, CE))
+      return false;
+    if (*ElemT == PT_Float) {
+      if (!this->emitCastFloatingIntegral(PT_Bool, CE))
+        return false;
+    } else {
+      if (!this->emitCast(*ElemT, PT_Bool, CE))
+        return false;
+    }
+    // Leave the boolean value of E[1] on the stack.
+    LabelTy EndLabel = this->getLabel();
+    this->jump(EndLabel);
+
+    this->emitLabel(LabelTrue);
+    if (!this->emitPopPtr(CE))
+      return false;
+    if (!this->emitConstBool(true, CE))
+      return false;
+
+    this->fallthrough(EndLabel);
+    this->emitLabel(EndLabel);
+
+    return true;
+  }
+
   case CK_ToVoid:
     return discard(SubExpr);
 
@@ -258,6 +316,9 @@ bool ByteCodeExprGen<Emitter>::VisitBinaryOperator(const BinaryOperator *BO) {
   if (BO->isLogicalOp())
     return this->VisitLogicalBinOp(BO);
 
+  if (BO->getType()->isAnyComplexType())
+    return this->VisitComplexBinOp(BO);
+
   const Expr *LHS = BO->getLHS();
   const Expr *RHS = BO->getRHS();
 
@@ -500,6 +561,107 @@ bool ByteCodeExprGen<Emitter>::VisitLogicalBinOp(const BinaryOperator *E) {
   return true;
 }
 
+template <class Emitter>
+bool ByteCodeExprGen<Emitter>::VisitComplexBinOp(const BinaryOperator *E) {
+  // FIXME: We expect a pointer on the stack here.
+  //   we should not do that, but that's part of a bigger rework.
+  const Expr *LHS = E->getLHS();
+  const Expr *RHS = E->getRHS();
+  PrimType LHSElemT = *this->classifyComplexElementType(LHS->getType());
+  PrimType RHSElemT = *this->classifyComplexElementType(RHS->getType());
+
+  unsigned LHSOffset = this->allocateLocalPrimitive(LHS, PT_Ptr, true, false);
+  unsigned RHSOffset = this->allocateLocalPrimitive(RHS, PT_Ptr, true, false);
+  unsigned ResultOffset = ~0u;
+  if (!this->DiscardResult)
+    ResultOffset = this->allocateLocalPrimitive(E, PT_Ptr, true, false);
+
+  assert(LHSElemT == RHSElemT);
+
+  // Save result pointer in ResultOffset
+  if (!this->DiscardResult) {
+    if (!this->emitDupPtr(E))
+      return false;
+    if (!this->emitSetLocal(PT_Ptr, ResultOffset, E))
+      return false;
+  }
+
+  // Evaluate LHS and save value to LHSOffset.
+  if (!this->visit(LHS))
+    return false;
+  if (!this->emitSetLocal(PT_Ptr, LHSOffset, E))
+    return false;
+
+  // Same with RHS.
+  if (!this->visit(RHS))
+    return false;
+  if (!this->emitSetLocal(PT_Ptr, RHSOffset, E))
+    return false;
+
+  // Now we can get pointers to the LHS and RHS from the offsets above.
+  BinaryOperatorKind Op = E->getOpcode();
+  for (unsigned ElemIndex = 0; ElemIndex != 2; ++ElemIndex) {
+    // Result pointer for the store later.
+    if (!this->DiscardResult) {
+      if (!this->emitGetLocal(PT_Ptr, ResultOffset, E))
+        return false;
+    }
+
+    if (!this->emitGetLocal(PT_Ptr, LHSOffset, E))
+      return false;
+    if (!this->emitConstUint8(ElemIndex, E))
+      return false;
+    if (!this->emitArrayElemPtrPopUint8(E))
+      return false;
+    if (!this->emitLoadPop(LHSElemT, E))
+      return false;
+
+    if (!this->emitGetLocal(PT_Ptr, RHSOffset, E))
+      return false;
+    if (!this->emitConstUint8(ElemIndex, E))
+      return false;
+    if (!this->emitArrayElemPtrPopUint8(E))
+      return false;
+    if (!this->emitLoadPop(RHSElemT, E))
+      return false;
+
+    // The actual operation.
+    switch (Op) {
+    case BO_Add:
+      if (LHSElemT == PT_Float) {
+        if (!this->emitAddf(getRoundingMode(E), E))
+          return false;
+      } else {
+        if (!this->emitAdd(LHSElemT, E))
+          return false;
+      }
+      break;
+    case BO_Sub:
+      if (LHSElemT == PT_Float) {
+        if (!this->emitSubf(getRoundingMode(E), E))
+          return false;
+      } else {
+        if (!this->emitSub(LHSElemT, E))
+          return false;
+      }
+      break;
+
+    default:
+      return false;
+    }
+
+    if (!this->DiscardResult) {
+      // Initialize array element with the value we just computed.
+      if (!this->emitInitElemPop(LHSElemT, ElemIndex, E))
+        return false;
+    } else {
+      if (!this->emitPop(LHSElemT, E))
+        return false;
+    }
+  }
+  return true;
+}
+
 template <class Emitter>
 bool ByteCodeExprGen<Emitter>::VisitImplicitValueInitExpr(const ImplicitValueInitExpr *E) {
   QualType QT = E->getType();
@@ -671,6 +833,32 @@ bool ByteCodeExprGen<Emitter>::VisitInitListExpr(const InitListExpr *E) {
     return true;
   }
 
+  if (T->isAnyComplexType()) {
+    unsigned NumInits = E->getNumInits();
+    QualType ElemQT = E->getType()->getAs<ComplexType>()->getElementType();
+    PrimType ElemT = classifyPrim(ElemQT);
+    if (NumInits == 0) {
+      // Zero-initialize both elements.
+      for (unsigned I = 0; I < 2; ++I) {
+        if (!this->visitZeroInitializer(ElemT, ElemQT, E))
+          return false;
+        if (!this->emitInitElem(ElemT, I, E))
+          return false;
+      }
+    } else if (NumInits == 2) {
+      unsigned InitIndex = 0;
+      for (const Expr *Init : E->inits()) {
+        if (!this->visit(Init))
+          return false;
+
+        if (!this->emitInitElem(ElemT, InitIndex, E))
+          return false;
+        ++InitIndex;
+      }
+    }
+    return true;
+  }
+
   return false;
 }
 
@@ -1647,7 +1835,8 @@ template <class Emitter> bool ByteCodeExprGen<Emitter>::visit(const Expr *E) {
     return this->discard(E);
 
   // Create local variable to hold the return value.
-  if (!E->isGLValue() && !classify(E->getType())) {
+  if (!E->isGLValue() && !E->getType()->isAnyComplexType() &&
+      !classify(E->getType())) {
     std::optional<unsigned> LocalIndex = allocateLocal(E, /*IsExtended=*/true);
     if (!LocalIndex)
       return false;
@@ -1833,6 +2022,9 @@ bool ByteCodeExprGen<Emitter>::dereference(
     return Indirect(*T);
   }
 
+  if (LV->getType()->isAnyComplexType())
+    return visit(LV);
+
   return false;
 }
 
@@ -2092,15 +2284,33 @@ const Function *ByteCodeExprGen<Emitter>::getFunction(const FunctionDecl *FD) {
 template <class Emitter>
 bool ByteCodeExprGen<Emitter>::visitExpr(const Expr *E) {
   ExprScope<Emitter> RootScope(this);
-  if (!visit(E))
-    return false;
-
-  if (E->getType()->isVoidType())
+  // Void expressions.
+  if (E->getType()->isVoidType()) {
+    if (!visit(E))
+      return false;
     return this->emitRetVoid(E);
+  }
 
-  if (std::optional<PrimType> T = classify(E))
+  // Expressions with a primitive return type.
+  if (std::optional<PrimType> T = classify(E)) {
+    if (!visit(E))
+      return false;
     return this->emitRet(*T, E);
-  return this->emitRetValue(E);
+  }
+
+  // Expressions with a composite return type.
+  // For us, that means everything we don't
+  // have a PrimType for.
+  if (std::optional<unsigned> LocalOffset = this->allocateLocal(E)) {
+    if (!this->visitLocalInitializer(E, *LocalOffset))
+      return false;
+
+    if (!this->emitGetPtrLocal(*LocalOffset, E))
+      return false;
+    return this->emitRetValue(E);
+  }
+
+  return false;
 }
 
 /// Toplevel visitDecl().
@@ -2550,8 +2760,36 @@ bool ByteCodeExprGen<Emitter>::VisitUnaryOperator(const UnaryOperator *E) {
     if (!this->visit(SubExpr))
       return false;
     return DiscardResult ? this->emitPop(*T, E) : this->emitComp(*T, E);
-  case UO_Real:   // __real x
-  case UO_Imag:   // __imag x
+  case UO_Real: { // __real x
+    assert(!T);
+    if (!this->visit(SubExpr))
+      return false;
+    if (!this->emitConstUint8(0, E))
+      return false;
+    if (!this->emitArrayElemPtrPopUint8(E))
+      return false;
+
+    // Since our _Complex implementation does not map to a primitive type,
+    // we sometimes have to do the lvalue-to-rvalue conversion here manually.
+    if (!SubExpr->isLValue())
+      return this->emitLoadPop(classifyPrim(E->getType()), E);
+    return true;
+  }
+  case UO_Imag: { // __imag x
+    assert(!T);
+    if (!this->visit(SubExpr))
+      return false;
+    if (!this->emitConstUint8(1, E))
+      return false;
+    if (!this->emitArrayElemPtrPopUint8(E))
+      return false;
+
+    // Since our _Complex implementation does not map to a primitive type,
+    // we sometimes have to do the lvalue-to-rvalue conversion here manually.
+    if (!SubExpr->isLValue())
+      return this->emitLoadPop(classifyPrim(E->getType()), E);
+    return true;
+  }
   case UO_Extension:
     return this->delegate(SubExpr);
   case UO_Coawait:
diff --git a/clang/lib/AST/Interp/ByteCodeExprGen.h b/clang/lib/AST/Interp/ByteCodeExprGen.h
index bc1d5d11a1151356e75620d448d0627ff60ba9a0..bbb13e97e725692237f5388936012e9a24a28cbd 100644
--- a/clang/lib/AST/Interp/ByteCodeExprGen.h
+++ b/clang/lib/AST/Interp/ByteCodeExprGen.h
@@ -65,6 +65,7 @@ public:
   bool VisitBinaryOperator(const BinaryOperator *E);
   bool VisitLogicalBinOp(const BinaryOperator *E);
   bool VisitPointerArithBinOp(const BinaryOperator *E);
+  bool VisitComplexBinOp(const BinaryOperator *E);
   bool VisitCXXDefaultArgExpr(const CXXDefaultArgExpr *E);
   bool VisitCallExpr(const CallExpr *E);
   bool VisitBuiltinCallExpr(const CallExpr *E);
@@ -285,6 +286,14 @@ private:
   }
 
   bool emitPrimCast(PrimType FromT, PrimType ToT, QualType ToQT, const Expr *E);
+  std::optional<PrimType> classifyComplexElementType(QualType T) const {
+    assert(T->isAnyComplexType());
+
+    QualType ElemType = T->getAs<ComplexType>()->getElementType();
+
+    return this->classify(ElemType);
+  }
+
   bool emitRecordDestruction(const Descriptor *Desc);
   unsigned collectBaseOffset(const RecordType *BaseType,
                              const RecordType *DerivedType);
diff --git a/clang/lib/AST/Interp/Context.cpp b/clang/lib/AST/Interp/Context.cpp
index 4fe6d1173f427e7923a91bb5e944c2cf45e37ed5..17abb71635839c794b2511a6e2bde2dfa7899f80 100644
--- a/clang/lib/AST/Interp/Context.cpp
+++ b/clang/lib/AST/Interp/Context.cpp
@@ -92,6 +92,9 @@ std::optional<PrimType> Context::classify(QualType T) const {
   if (T->isBooleanType())
     return PT_Bool;
 
+  if (T->isAnyComplexType())
+    return std::nullopt;
+
   if (T->isSignedIntegerOrEnumerationType()) {
     switch (Ctx.getIntWidth(T)) {
     case 64:
diff --git a/clang/lib/AST/Interp/EvalEmitter.cpp b/clang/lib/AST/Interp/EvalEmitter.cpp
index 9bc42057c5f57822ed6cf980f7aaa1b14adbf2c9..0ff0bde8fd17e85b14babdc06995e034a99e8c96 100644
--- a/clang/lib/AST/Interp/EvalEmitter.cpp
+++ b/clang/lib/AST/Interp/EvalEmitter.cpp
@@ -208,6 +208,27 @@ bool EvalEmitter::emitRetValue(const SourceInfo &Info) {
       }
       return Ok;
     }
+
+    // Complex types.
+    if (const auto *CT = Ty->getAs<ComplexType>()) {
+      QualType ElemTy = CT->getElementType();
+      std::optional<PrimType> ElemT = Ctx.classify(ElemTy);
+      assert(ElemT);
+
+      if (ElemTy->isIntegerType()) {
+        INT_TYPE_SWITCH(*ElemT, {
+          auto V1 = Ptr.atIndex(0).deref<T>();
+          auto V2 = Ptr.atIndex(1).deref<T>();
+          Result = APValue(V1.toAPSInt(), V2.toAPSInt());
+          return true;
+        });
+      } else if (ElemTy->isFloatingType()) {
+        Result = APValue(Ptr.atIndex(0).deref<Floating>().getAPFloat(),
+                         Ptr.atIndex(1).deref<Floating>().getAPFloat());
+        return true;
+      }
+      return false;
+    }
     llvm_unreachable("invalid value to return");
   };
 
diff --git a/clang/lib/AST/Interp/Interp.cpp b/clang/lib/AST/Interp/Interp.cpp
index 13b77e9a87725c73efd19346cafde1f43728bd9c..a82d1c3c7c622a3db101443ed6fc8aa5d144f51d 100644
--- a/clang/lib/AST/Interp/Interp.cpp
+++ b/clang/lib/AST/Interp/Interp.cpp
@@ -350,11 +350,6 @@ bool CheckCallable(InterpState &S, CodePtr OpPC, const Function *F) {
   }
 
   if (!F->isConstexpr()) {
-    // Don't emit anything if we're checking for a potential constant
-    // expression. That will happen later when actually executing.
-    if (S.checkingPotentialConstantExpression())
-      return false;
-
     const SourceLocation &Loc = S.Current->getLocation(OpPC);
     if (S.getLangOpts().CPlusPlus11) {
       const FunctionDecl *DiagDecl = F->getDecl();
@@ -371,13 +366,21 @@ bool CheckCallable(InterpState &S, CodePtr OpPC, const Function *F) {
       // FIXME: If DiagDecl is an implicitly-declared special member function
       // or an inheriting constructor, we should be much more explicit about why
       // it's not constexpr.
-      if (CD && CD->isInheritingConstructor())
+      if (CD && CD->isInheritingConstructor()) {
         S.FFDiag(Loc, diag::note_constexpr_invalid_inhctor, 1)
           << CD->getInheritedConstructor().getConstructor()->getParent();
-      else
+        S.Note(DiagDecl->getLocation(), diag::note_declared_at);
+      } else {
+        // Don't emit anything if the function isn't defined and we're checking
+        // for a constant expression. It might be defined at the point we're
+        // actually calling it.
+        if (!DiagDecl->isDefined() && S.checkingPotentialConstantExpression())
+          return false;
+
         S.FFDiag(Loc, diag::note_constexpr_invalid_function, 1)
           << DiagDecl->isConstexpr() << (bool)CD << DiagDecl;
-      S.Note(DiagDecl->getLocation(), diag::note_declared_at);
+        S.Note(DiagDecl->getLocation(), diag::note_declared_at);
+      }
     } else {
       S.FFDiag(Loc, diag::note_invalid_subexpr_in_const_expr);
     }
diff --git a/clang/lib/AST/Mangle.cpp b/clang/lib/AST/Mangle.cpp
index 64c971912a91d073328c835523430be6b36e1ac2..d3a6b61fd2bec9c659b24c4ab995f4459254d54f 100644
--- a/clang/lib/AST/Mangle.cpp
+++ b/clang/lib/AST/Mangle.cpp
@@ -147,7 +147,7 @@ void MangleContext::mangleName(GlobalDecl GD, raw_ostream &Out) {
 
     // If the label isn't literal, or if this is an alias for an LLVM intrinsic,
     // do not add a "\01" prefix.
-    if (!ALA->getIsLiteralLabel() || ALA->getLabel().startswith("llvm.")) {
+    if (!ALA->getIsLiteralLabel() || ALA->getLabel().starts_with("llvm.")) {
       Out << ALA->getLabel();
       return;
     }
diff --git a/clang/lib/AST/MicrosoftMangle.cpp b/clang/lib/AST/MicrosoftMangle.cpp
index c59a66e103a6e3b929fa950f93414bd58c185de0..8346ad87b409b6196126164086468673b0ca48c4 100644
--- a/clang/lib/AST/MicrosoftMangle.cpp
+++ b/clang/lib/AST/MicrosoftMangle.cpp
@@ -63,7 +63,7 @@ struct msvc_hashing_ostream : public llvm::raw_svector_ostream {
       : llvm::raw_svector_ostream(Buffer), OS(OS) {}
   ~msvc_hashing_ostream() override {
     StringRef MangledName = str();
-    bool StartsWithEscape = MangledName.startswith("\01");
+    bool StartsWithEscape = MangledName.starts_with("\01");
     if (StartsWithEscape)
       MangledName = MangledName.drop_front(1);
     if (MangledName.size() < 4096) {
diff --git a/clang/lib/AST/PrintfFormatString.cpp b/clang/lib/AST/PrintfFormatString.cpp
index f0b9d0ecaf23461ac1b846a45cb39dab0257cf1b..3b09ca40bd2a53e99f50ea8bb805e4e56bb6c4f9 100644
--- a/clang/lib/AST/PrintfFormatString.cpp
+++ b/clang/lib/AST/PrintfFormatString.cpp
@@ -140,7 +140,7 @@ static PrintfSpecifierResult ParsePrintfSpecifier(FormatStringHandler &H,
         // Set the privacy flag if the privacy annotation in the
         // comma-delimited segment is at least as strict as the privacy
         // annotations in previous comma-delimited segments.
-        if (MatchedStr.startswith("mask")) {
+        if (MatchedStr.starts_with("mask")) {
           StringRef MaskType = MatchedStr.substr(sizeof("mask.") - 1);
           unsigned Size = MaskType.size();
           if (Warn && (Size == 0 || Size > 8))
diff --git a/clang/lib/AST/RawCommentList.cpp b/clang/lib/AST/RawCommentList.cpp
index c3beb23228887a8865a17268494ec46f7becb051..dffa007b6588bc05895f2789532d0f23c4586b5b 100644
--- a/clang/lib/AST/RawCommentList.cpp
+++ b/clang/lib/AST/RawCommentList.cpp
@@ -141,8 +141,8 @@ RawComment::RawComment(const SourceManager &SourceMgr, SourceRange SR,
     Kind = K.first;
     IsTrailingComment |= K.second;
 
-    IsAlmostTrailingComment = RawText.startswith("//<") ||
-                                 RawText.startswith("/*<");
+    IsAlmostTrailingComment =
+        RawText.starts_with("//<") || RawText.starts_with("/*<");
   } else {
     Kind = RCK_Merged;
     IsTrailingComment =
diff --git a/clang/lib/AST/Stmt.cpp b/clang/lib/AST/Stmt.cpp
index c31fb48a2addfa3a627e8344f2d8920c970056a5..afd05881cb162107501d372bbfbaeb5a6e1d1a8a 100644
--- a/clang/lib/AST/Stmt.cpp
+++ b/clang/lib/AST/Stmt.cpp
@@ -811,11 +811,12 @@ std::string MSAsmStmt::generateAsmString(const ASTContext &C) const {
     StringRef Instruction = Pieces[I];
     // For vex/vex2/vex3/evex masm style prefix, convert it to att style
     // since we don't support masm style prefix in backend.
-    if (Instruction.startswith("vex "))
+    if (Instruction.starts_with("vex "))
       MSAsmString += '{' + Instruction.substr(0, 3).str() + '}' +
                      Instruction.substr(3).str();
-    else if (Instruction.startswith("vex2 ") ||
-             Instruction.startswith("vex3 ") || Instruction.startswith("evex "))
+    else if (Instruction.starts_with("vex2 ") ||
+             Instruction.starts_with("vex3 ") ||
+             Instruction.starts_with("evex "))
       MSAsmString += '{' + Instruction.substr(0, 4).str() + '}' +
                      Instruction.substr(4).str();
     else
diff --git a/clang/lib/ASTMatchers/ASTMatchersInternal.cpp b/clang/lib/ASTMatchers/ASTMatchersInternal.cpp
index 435bbdeda22066e8cd42255178af66e135b639ca..8ed213ca2ce0965bf9f9c22e6a5bef8287beb4c3 100644
--- a/clang/lib/ASTMatchers/ASTMatchersInternal.cpp
+++ b/clang/lib/ASTMatchers/ASTMatchersInternal.cpp
@@ -480,11 +480,11 @@ HasNameMatcher::HasNameMatcher(std::vector<std::string> N)
 
 static bool consumeNameSuffix(StringRef &FullName, StringRef Suffix) {
   StringRef Name = FullName;
-  if (!Name.endswith(Suffix))
+  if (!Name.ends_with(Suffix))
     return false;
   Name = Name.drop_back(Suffix.size());
   if (!Name.empty()) {
-    if (!Name.endswith("::"))
+    if (!Name.ends_with("::"))
       return false;
     Name = Name.drop_back(2);
   }
@@ -530,7 +530,7 @@ public:
   PatternSet(ArrayRef<std::string> Names) {
     Patterns.reserve(Names.size());
     for (StringRef Name : Names)
-      Patterns.push_back({Name, Name.startswith("::")});
+      Patterns.push_back({Name, Name.starts_with("::")});
   }
 
   /// Consumes the name suffix from each pattern in the set and removes the ones
@@ -652,11 +652,11 @@ bool HasNameMatcher::matchesNodeFullSlow(const NamedDecl &Node) const {
     const StringRef FullName = OS.str();
 
     for (const StringRef Pattern : Names) {
-      if (Pattern.startswith("::")) {
+      if (Pattern.starts_with("::")) {
         if (FullName == Pattern)
           return true;
-      } else if (FullName.endswith(Pattern) &&
-                 FullName.drop_back(Pattern.size()).endswith("::")) {
+      } else if (FullName.ends_with(Pattern) &&
+                 FullName.drop_back(Pattern.size()).ends_with("::")) {
         return true;
       }
     }
diff --git a/clang/lib/ASTMatchers/Dynamic/Parser.cpp b/clang/lib/ASTMatchers/Dynamic/Parser.cpp
index 33a10fe838a6aa6f27fe0dc2b77c4543760a1133..27096a83b8dd603eff2dd9d42b542eb6061d3795 100644
--- a/clang/lib/ASTMatchers/Dynamic/Parser.cpp
+++ b/clang/lib/ASTMatchers/Dynamic/Parser.cpp
@@ -187,10 +187,10 @@ private:
             break;
           ++TokenLength;
         }
-        if (TokenLength == 4 && Code.startswith("true")) {
+        if (TokenLength == 4 && Code.starts_with("true")) {
           Result.Kind = TokenInfo::TK_Literal;
           Result.Value = true;
-        } else if (TokenLength == 5 && Code.startswith("false")) {
+        } else if (TokenLength == 5 && Code.starts_with("false")) {
           Result.Kind = TokenInfo::TK_Literal;
           Result.Value = false;
         } else {
@@ -737,7 +737,7 @@ bool Parser::parseMatcherExpressionImpl(const TokenInfo &NameToken,
 // Completions minus the prefix.
 void Parser::addCompletion(const TokenInfo &CompToken,
                            const MatcherCompletion& Completion) {
-  if (StringRef(Completion.TypedText).startswith(CompToken.Text) &&
+  if (StringRef(Completion.TypedText).starts_with(CompToken.Text) &&
       Completion.Specificity > 0) {
     Completions.emplace_back(Completion.TypedText.substr(CompToken.Text.size()),
                              Completion.MatcherDecl, Completion.Specificity);
diff --git a/clang/lib/Analysis/BodyFarm.cpp b/clang/lib/Analysis/BodyFarm.cpp
index 13ec9b65c9f0b2ef7cd71eacceb94220a1afcb75..127e843d4ead21637f6a29c6dcd9811403df2ec6 100644
--- a/clang/lib/Analysis/BodyFarm.cpp
+++ b/clang/lib/Analysis/BodyFarm.cpp
@@ -726,8 +726,8 @@ Stmt *BodyFarm::getBody(const FunctionDecl *D) {
       FF = nullptr;
       break;
     }
-  } else if (Name.startswith("OSAtomicCompareAndSwap") ||
-             Name.startswith("objc_atomicCompareAndSwap")) {
+  } else if (Name.starts_with("OSAtomicCompareAndSwap") ||
+             Name.starts_with("objc_atomicCompareAndSwap")) {
     FF = create_OSAtomicCompareAndSwap;
   } else if (Name == "call_once" && D->getDeclContext()->isStdNamespace()) {
     FF = create_call_once;
diff --git a/clang/lib/Analysis/CallGraph.cpp b/clang/lib/Analysis/CallGraph.cpp
index 59cc939b6fd15e991c4584b98c7310c379b9259f..f892980ed31386b1c6e27b085c7716b6fd856250 100644
--- a/clang/lib/Analysis/CallGraph.cpp
+++ b/clang/lib/Analysis/CallGraph.cpp
@@ -168,7 +168,7 @@ bool CallGraph::includeCalleeInGraph(const Decl *D) {
       return false;
 
     IdentifierInfo *II = FD->getIdentifier();
-    if (II && II->getName().startswith("__inline"))
+    if (II && II->getName().starts_with("__inline"))
       return false;
   }
 
diff --git a/clang/lib/Analysis/CalledOnceCheck.cpp b/clang/lib/Analysis/CalledOnceCheck.cpp
index 5b4fc24b6f0e2a557066dee417abd68d5aa3863f..04c5f6aa9c7450cdaf1b9a76c94f7dd3ddda3f59 100644
--- a/clang/lib/Analysis/CalledOnceCheck.cpp
+++ b/clang/lib/Analysis/CalledOnceCheck.cpp
@@ -973,7 +973,7 @@ private:
   /// Return true if the given name has conventional suffixes.
   static bool hasConventionalSuffix(llvm::StringRef Name) {
     return llvm::any_of(CONVENTIONAL_SUFFIXES, [Name](llvm::StringRef Suffix) {
-      return Name.endswith(Suffix);
+      return Name.ends_with(Suffix);
     });
   }
 
diff --git a/clang/lib/Analysis/CocoaConventions.cpp b/clang/lib/Analysis/CocoaConventions.cpp
index 571d72e1a841656e681858332a336e539d4d9042..836859c2234585beed0dc70b65c6a443c9cc0428 100644
--- a/clang/lib/Analysis/CocoaConventions.cpp
+++ b/clang/lib/Analysis/CocoaConventions.cpp
@@ -26,10 +26,10 @@ bool cocoa::isRefType(QualType RetTy, StringRef Prefix,
   // Recursively walk the typedef stack, allowing typedefs of reference types.
   while (const TypedefType *TD = RetTy->getAs<TypedefType>()) {
     StringRef TDName = TD->getDecl()->getIdentifier()->getName();
-    if (TDName.startswith(Prefix) && TDName.endswith("Ref"))
+    if (TDName.starts_with(Prefix) && TDName.ends_with("Ref"))
       return true;
     // XPC unfortunately uses CF-style function names, but aren't CF types.
-    if (TDName.startswith("xpc_"))
+    if (TDName.starts_with("xpc_"))
       return false;
     RetTy = TD->getDecl()->getUnderlyingType();
   }
@@ -43,7 +43,7 @@ bool cocoa::isRefType(QualType RetTy, StringRef Prefix,
     return false;
 
   // Does the name start with the prefix?
-  return Name.startswith(Prefix);
+  return Name.starts_with(Prefix);
 }
 
 /// Returns true when the passed-in type is a CF-style reference-counted
@@ -127,10 +127,9 @@ bool coreFoundation::followsCreateRule(const FunctionDecl *fn) {
     // Scan for *lowercase* 'reate' or 'opy', followed by no lowercase
     // character.
     StringRef suffix = functionName.substr(it - start);
-    if (suffix.startswith("reate")) {
+    if (suffix.starts_with("reate")) {
       it += 5;
-    }
-    else if (suffix.startswith("opy")) {
+    } else if (suffix.starts_with("opy")) {
       it += 3;
     } else {
       // Keep scanning.
diff --git a/clang/lib/Analysis/FlowSensitive/Models/ChromiumCheckModel.cpp b/clang/lib/Analysis/FlowSensitive/Models/ChromiumCheckModel.cpp
index f49087ababc44ce325c8eb08efde418da28a977f..5ac71e1d6bf64d5c8543cf01a71e44b82a606535 100644
--- a/clang/lib/Analysis/FlowSensitive/Models/ChromiumCheckModel.cpp
+++ b/clang/lib/Analysis/FlowSensitive/Models/ChromiumCheckModel.cpp
@@ -43,7 +43,7 @@ bool isCheckLikeMethod(llvm::SmallDenseSet<const CXXMethodDecl *> &CheckDecls,
       return false;
 
     for (const CXXMethodDecl *M : ParentClass->methods())
-      if (M->getDeclName().isIdentifier() && M->getName().endswith("Check"))
+      if (M->getDeclName().isIdentifier() && M->getName().ends_with("Check"))
         CheckDecls.insert(M);
   }
 
diff --git a/clang/lib/Analysis/RetainSummaryManager.cpp b/clang/lib/Analysis/RetainSummaryManager.cpp
index 4cbeb0c35b6f6642d9ecec63ba823fe34155454e..6f50d95b179f412342d76f19a243d3bcda7e693f 100644
--- a/clang/lib/Analysis/RetainSummaryManager.cpp
+++ b/clang/lib/Analysis/RetainSummaryManager.cpp
@@ -174,7 +174,7 @@ static bool isOSObjectPtr(QualType QT) {
 }
 
 static bool isISLObjectRef(QualType Ty) {
-  return StringRef(Ty.getAsString()).startswith("isl_");
+  return StringRef(Ty.getAsString()).starts_with("isl_");
 }
 
 static bool isOSIteratorSubclass(const Decl *D) {
@@ -255,13 +255,13 @@ RetainSummaryManager::getSummaryForOSObject(const FunctionDecl *FD,
       // TODO: Add support for the slightly common *Matching(table) idiom.
       // Cf. IOService::nameMatching() etc. - these function have an unusual
       // contract of returning at +0 or +1 depending on their last argument.
-      if (FName.endswith("Matching")) {
+      if (FName.ends_with("Matching")) {
         return getPersistentStopSummary();
       }
 
       // All objects returned with functions *not* starting with 'get',
       // or iterators, are returned at +1.
-      if ((!FName.startswith("get") && !FName.startswith("Get")) ||
+      if ((!FName.starts_with("get") && !FName.starts_with("Get")) ||
           isOSIteratorSubclass(PD)) {
         return getOSSummaryCreateRule(FD);
       } else {
@@ -392,9 +392,9 @@ const RetainSummary *RetainSummaryManager::getSummaryForObjCOrCFObject(
     return getPersistentSummary(RetEffect::MakeNoRet(),
                                 ScratchArgs,
                                 ArgEffect(DoNothing), ArgEffect(DoNothing));
-  } else if (FName.startswith("NSLog")) {
+  } else if (FName.starts_with("NSLog")) {
     return getDoNothingSummary();
-  } else if (FName.startswith("NS") && FName.contains("Insert")) {
+  } else if (FName.starts_with("NS") && FName.contains("Insert")) {
     // Allowlist NSXXInsertXX, for example NSMapInsertIfAbsent, since they can
     // be deallocated by NSMapRemove.
     ScratchArgs = AF.add(ScratchArgs, 1, ArgEffect(StopTracking));
@@ -453,9 +453,9 @@ const RetainSummary *RetainSummaryManager::getSummaryForObjCOrCFObject(
 
   // Check for release functions, the only kind of functions that we care
   // about that don't return a pointer type.
-  if (FName.startswith("CG") || FName.startswith("CF")) {
+  if (FName.starts_with("CG") || FName.starts_with("CF")) {
     // Test for 'CGCF'.
-    FName = FName.substr(FName.startswith("CGCF") ? 4 : 2);
+    FName = FName.substr(FName.starts_with("CGCF") ? 4 : 2);
 
     if (isRelease(FD, FName))
       return getUnarySummary(FT, DecRef);
diff --git a/clang/lib/Basic/Attributes.cpp b/clang/lib/Basic/Attributes.cpp
index bb495216ca93ca587d776e1775ff12982a5ebbc0..44a4f1890d39e11bc70678b7cbc1cbe514b367f9 100644
--- a/clang/lib/Basic/Attributes.cpp
+++ b/clang/lib/Basic/Attributes.cpp
@@ -33,7 +33,7 @@ int clang::hasAttribute(AttributeCommonInfo::Syntax Syntax,
                         const TargetInfo &Target, const LangOptions &LangOpts) {
   StringRef Name = Attr->getName();
   // Normalize the attribute name, __foo__ becomes foo.
-  if (Name.size() >= 4 && Name.startswith("__") && Name.endswith("__"))
+  if (Name.size() >= 4 && Name.starts_with("__") && Name.ends_with("__"))
     Name = Name.substr(2, Name.size() - 4);
 
   // Normalize the scope name, but only for gnu and clang attributes.
@@ -103,8 +103,8 @@ static StringRef normalizeAttrName(const IdentifierInfo *Name,
        (NormalizedScopeName.empty() || NormalizedScopeName == "gnu" ||
         NormalizedScopeName == "clang"));
   StringRef AttrName = Name->getName();
-  if (ShouldNormalize && AttrName.size() >= 4 && AttrName.startswith("__") &&
-      AttrName.endswith("__"))
+  if (ShouldNormalize && AttrName.size() >= 4 && AttrName.starts_with("__") &&
+      AttrName.ends_with("__"))
     AttrName = AttrName.slice(2, AttrName.size() - 2);
 
   return AttrName;
diff --git a/clang/lib/Basic/DiagnosticIDs.cpp b/clang/lib/Basic/DiagnosticIDs.cpp
index e5667d57f8cff114c316a40526e4b116ce9e2924..6c7bd50eefb7ef618ce8df9b175aa2f13d6ade1c 100644
--- a/clang/lib/Basic/DiagnosticIDs.cpp
+++ b/clang/lib/Basic/DiagnosticIDs.cpp
@@ -853,5 +853,5 @@ bool DiagnosticIDs::isUnrecoverable(unsigned DiagID) const {
 
 bool DiagnosticIDs::isARCDiagnostic(unsigned DiagID) {
   unsigned cat = getCategoryNumberForDiag(DiagID);
-  return DiagnosticIDs::getCategoryNameFromID(cat).startswith("ARC ");
+  return DiagnosticIDs::getCategoryNameFromID(cat).starts_with("ARC ");
 }
diff --git a/clang/lib/Basic/IdentifierTable.cpp b/clang/lib/Basic/IdentifierTable.cpp
index 38150d1640d709d96583feab5639dbb6d0d64108..5902c6dc3ce0b48073bac9663a50e6cc82106fa0 100644
--- a/clang/lib/Basic/IdentifierTable.cpp
+++ b/clang/lib/Basic/IdentifierTable.cpp
@@ -604,7 +604,7 @@ LLVM_DUMP_METHOD void Selector::dump() const { print(llvm::errs()); }
 static bool startsWithWord(StringRef name, StringRef word) {
   if (name.size() < word.size()) return false;
   return ((name.size() == word.size() || !isLowercase(name[word.size()])) &&
-          name.startswith(word));
+          name.starts_with(word));
 }
 
 ObjCMethodFamily Selector::getMethodFamilyImpl(Selector sel) {
@@ -742,7 +742,7 @@ SelectorTable::constructSetterSelector(IdentifierTable &Idents,
 
 std::string SelectorTable::getPropertyNameFromSetterSelector(Selector Sel) {
   StringRef Name = Sel.getNameForSlot(0);
-  assert(Name.startswith("set") && "invalid setter name");
+  assert(Name.starts_with("set") && "invalid setter name");
   return (Twine(toLowercase(Name[3])) + Name.drop_front(4)).str();
 }
 
diff --git a/clang/lib/Basic/Module.cpp b/clang/lib/Basic/Module.cpp
index 7523e509a47108c207559585d0e988371c1532ef..925217431d4d021a73154202dc50cfaf2adc5567 100644
--- a/clang/lib/Basic/Module.cpp
+++ b/clang/lib/Basic/Module.cpp
@@ -166,7 +166,8 @@ bool Module::isForBuilding(const LangOptions &LangOpts) const {
   // for either.
   if (!LangOpts.isCompilingModule() && getTopLevelModule()->IsFramework &&
       CurrentModule == LangOpts.ModuleName &&
-      !CurrentModule.endswith("_Private") && TopLevelName.endswith("_Private"))
+      !CurrentModule.ends_with("_Private") &&
+      TopLevelName.ends_with("_Private"))
     TopLevelName = TopLevelName.drop_back(8);
 
   return TopLevelName == CurrentModule;
diff --git a/clang/lib/Basic/Sarif.cpp b/clang/lib/Basic/Sarif.cpp
index 3476103cc39d40890c6b5e37a2a045f91167089f..1cae7b937bc6ee2b676e68c5a8b9575b6dd7e9cf 100644
--- a/clang/lib/Basic/Sarif.cpp
+++ b/clang/lib/Basic/Sarif.cpp
@@ -74,7 +74,7 @@ static std::string fileNameToURI(StringRef Filename) {
 
   // Get the root name to see if it has a URI authority.
   StringRef Root = sys::path::root_name(Filename);
-  if (Root.startswith("//")) {
+  if (Root.starts_with("//")) {
     // There is an authority, so add it to the URI.
     Ret += Root.drop_front(2).str();
   } else if (!Root.empty()) {
diff --git a/clang/lib/Basic/TargetInfo.cpp b/clang/lib/Basic/TargetInfo.cpp
index 6cd5d618a4acaa59e26aa1c9aa74a0a8abac11a3..96b3ad9ba2f2731b1bd3a5443a0b6d1a5ceae6f5 100644
--- a/clang/lib/Basic/TargetInfo.cpp
+++ b/clang/lib/Basic/TargetInfo.cpp
@@ -551,26 +551,26 @@ ParsedTargetAttr TargetInfo::parseTargetAttr(StringRef Features) const {
     // TODO: Support the fpmath option. It will require checking
     // overall feature validity for the function with the rest of the
     // attributes on the function.
-    if (Feature.startswith("fpmath="))
+    if (Feature.starts_with("fpmath="))
       continue;
 
-    if (Feature.startswith("branch-protection=")) {
+    if (Feature.starts_with("branch-protection=")) {
       Ret.BranchProtection = Feature.split('=').second.trim();
       continue;
     }
 
     // While we're here iterating check for a different target cpu.
-    if (Feature.startswith("arch=")) {
+    if (Feature.starts_with("arch=")) {
       if (!Ret.CPU.empty())
         Ret.Duplicate = "arch=";
       else
         Ret.CPU = Feature.split("=").second.trim();
-    } else if (Feature.startswith("tune=")) {
+    } else if (Feature.starts_with("tune=")) {
       if (!Ret.Tune.empty())
         Ret.Duplicate = "tune=";
       else
         Ret.Tune = Feature.split("=").second.trim();
-    } else if (Feature.startswith("no-"))
+    } else if (Feature.starts_with("no-"))
       Ret.Features.push_back("-" + Feature.split("-").second.str());
     else
       Ret.Features.push_back("+" + Feature.str());
diff --git a/clang/lib/Basic/Targets/AArch64.cpp b/clang/lib/Basic/Targets/AArch64.cpp
index e3e08b571667cce2df44ebc0658b2f4f68bcd2d8..def16c032c869e59c3acb51bd4e8dfd19880fe2b 100644
--- a/clang/lib/Basic/Targets/AArch64.cpp
+++ b/clang/lib/Basic/Targets/AArch64.cpp
@@ -1062,7 +1062,7 @@ ParsedTargetAttr AArch64TargetInfo::parseTargetAttr(StringRef Features) const {
       else
         // Pushing the original feature string to give a sema error later on
         // when they get checked.
-        if (Feature.startswith("no"))
+        if (Feature.starts_with("no"))
           Features.push_back("-" + Feature.drop_front(2).str());
         else
           Features.push_back("+" + Feature.str());
@@ -1071,15 +1071,15 @@ ParsedTargetAttr AArch64TargetInfo::parseTargetAttr(StringRef Features) const {
 
   for (auto &Feature : AttrFeatures) {
     Feature = Feature.trim();
-    if (Feature.startswith("fpmath="))
+    if (Feature.starts_with("fpmath="))
       continue;
 
-    if (Feature.startswith("branch-protection=")) {
+    if (Feature.starts_with("branch-protection=")) {
       Ret.BranchProtection = Feature.split('=').second.trim();
       continue;
     }
 
-    if (Feature.startswith("arch=")) {
+    if (Feature.starts_with("arch=")) {
       if (FoundArch)
         Ret.Duplicate = "arch=";
       FoundArch = true;
@@ -1095,7 +1095,7 @@ ParsedTargetAttr AArch64TargetInfo::parseTargetAttr(StringRef Features) const {
       Ret.Features.push_back(AI->ArchFeature.str());
       // Add any extra features, after the +
       SplitAndAddFeatures(Split.second, Ret.Features);
-    } else if (Feature.startswith("cpu=")) {
+    } else if (Feature.starts_with("cpu=")) {
       if (!Ret.CPU.empty())
         Ret.Duplicate = "cpu=";
       else {
@@ -1106,14 +1106,14 @@ ParsedTargetAttr AArch64TargetInfo::parseTargetAttr(StringRef Features) const {
         Ret.CPU = Split.first;
         SplitAndAddFeatures(Split.second, Ret.Features);
       }
-    } else if (Feature.startswith("tune=")) {
+    } else if (Feature.starts_with("tune=")) {
       if (!Ret.Tune.empty())
         Ret.Duplicate = "tune=";
       else
         Ret.Tune = Feature.split("=").second.trim();
-    } else if (Feature.startswith("+")) {
+    } else if (Feature.starts_with("+")) {
       SplitAndAddFeatures(Feature, Ret.Features);
-    } else if (Feature.startswith("no-")) {
+    } else if (Feature.starts_with("no-")) {
       StringRef FeatureName =
           llvm::AArch64::getArchExtFeature(Feature.split("-").second);
       if (!FeatureName.empty())
diff --git a/clang/lib/Basic/Targets/AMDGPU.cpp b/clang/lib/Basic/Targets/AMDGPU.cpp
index 719fc51bfc286fb57247ea71b2c7626ce49e7e83..b064ec2b3c9a6e6639074f2f4961eb43c1c68948 100644
--- a/clang/lib/Basic/Targets/AMDGPU.cpp
+++ b/clang/lib/Basic/Targets/AMDGPU.cpp
@@ -279,7 +279,7 @@ void AMDGPUTargetInfo::getTargetDefines(const LangOptions &Opts,
     Builder.defineMacro(Twine("__") + Twine(CanonName) + Twine("__"));
     // Emit macros for gfx family e.g. gfx906 -> __GFX9__, gfx1030 -> __GFX10___
     if (isAMDGCN(getTriple())) {
-      assert(CanonName.startswith("gfx") && "Invalid amdgcn canonical name");
+      assert(CanonName.starts_with("gfx") && "Invalid amdgcn canonical name");
       Builder.defineMacro(Twine("__") + Twine(CanonName.drop_back(2).upper()) +
                           Twine("__"));
     }
diff --git a/clang/lib/Basic/Targets/Mips.cpp b/clang/lib/Basic/Targets/Mips.cpp
index bc90d1b93d53f6daf14e10497d0df35d4fcfe508..3a65f53c524851b1188b9d7afb8597b2364fb9e0 100644
--- a/clang/lib/Basic/Targets/Mips.cpp
+++ b/clang/lib/Basic/Targets/Mips.cpp
@@ -196,7 +196,7 @@ void MipsTargetInfo::getTargetDefines(const LangOptions &Opts,
   else
     Builder.defineMacro("_MIPS_ARCH_" + StringRef(CPU).upper());
 
-  if (StringRef(CPU).startswith("octeon"))
+  if (StringRef(CPU).starts_with("octeon"))
     Builder.defineMacro("__OCTEON__");
 
   if (CPU != "mips1") {
diff --git a/clang/lib/Basic/Targets/NVPTX.cpp b/clang/lib/Basic/Targets/NVPTX.cpp
index 5c601812f617596526cc6d2a0e22cbc23bb5e05d..c0b5db795e2708a84a05934585271e1b6009b2fe 100644
--- a/clang/lib/Basic/Targets/NVPTX.cpp
+++ b/clang/lib/Basic/Targets/NVPTX.cpp
@@ -42,7 +42,7 @@ NVPTXTargetInfo::NVPTXTargetInfo(const llvm::Triple &Triple,
   PTXVersion = 32;
   for (const StringRef Feature : Opts.FeaturesAsWritten) {
     int PTXV;
-    if (!Feature.startswith("+ptx") ||
+    if (!Feature.starts_with("+ptx") ||
         Feature.drop_front(4).getAsInteger(10, PTXV))
       continue;
     PTXVersion = PTXV; // TODO: should it be max(PTXVersion, PTXV)?
diff --git a/clang/lib/Basic/Targets/RISCV.cpp b/clang/lib/Basic/Targets/RISCV.cpp
index 45d23022b5306b3c5e5ec41d637520dccc0cde72..60a4e0ed69c34dcc5b9106cbc4630665f64ca2d6 100644
--- a/clang/lib/Basic/Targets/RISCV.cpp
+++ b/clang/lib/Basic/Targets/RISCV.cpp
@@ -434,14 +434,14 @@ ParsedTargetAttr RISCVTargetInfo::parseTargetAttr(StringRef Features) const {
     Feature = Feature.trim();
     StringRef AttrString = Feature.split("=").second.trim();
 
-    if (Feature.startswith("arch=")) {
+    if (Feature.starts_with("arch=")) {
       // Override last features
       Ret.Features.clear();
       if (FoundArch)
         Ret.Duplicate = "arch=";
       FoundArch = true;
 
-      if (AttrString.startswith("+")) {
+      if (AttrString.starts_with("+")) {
         // EXTENSION like arch=+v,+zbb
         SmallVector<StringRef, 1> Exts;
         AttrString.split(Exts, ",");
@@ -461,7 +461,7 @@ ParsedTargetAttr RISCVTargetInfo::parseTargetAttr(StringRef Features) const {
         // full-arch-string like arch=rv64gcv
         handleFullArchString(AttrString, Ret.Features);
       }
-    } else if (Feature.startswith("cpu=")) {
+    } else if (Feature.starts_with("cpu=")) {
       if (!Ret.CPU.empty())
         Ret.Duplicate = "cpu=";
 
@@ -475,7 +475,7 @@ ParsedTargetAttr RISCVTargetInfo::parseTargetAttr(StringRef Features) const {
           handleFullArchString(MarchFromCPU, Ret.Features);
         }
       }
-    } else if (Feature.startswith("tune=")) {
+    } else if (Feature.starts_with("tune=")) {
       if (!Ret.Tune.empty())
         Ret.Duplicate = "tune=";
 
diff --git a/clang/lib/Basic/Warnings.cpp b/clang/lib/Basic/Warnings.cpp
index cc8c138233ca1d9fcc4ef04840b87fb2efb94655..cb23d844ef8f6f24538e920f80f7f483c9d50732 100644
--- a/clang/lib/Basic/Warnings.cpp
+++ b/clang/lib/Basic/Warnings.cpp
@@ -97,7 +97,7 @@ void clang::ProcessWarningOptions(DiagnosticsEngine &Diags,
       // Check to see if this warning starts with "no-", if so, this is a
       // negative form of the option.
       bool isPositive = true;
-      if (Opt.startswith("no-")) {
+      if (Opt.starts_with("no-")) {
         isPositive = false;
         Opt = Opt.substr(3);
       }
@@ -133,7 +133,7 @@ void clang::ProcessWarningOptions(DiagnosticsEngine &Diags,
       // table. It also has the "specifier" form of -Werror=foo. GCC supports
       // the deprecated -Werror-implicit-function-declaration which is used by
       // a few projects.
-      if (Opt.startswith("error")) {
+      if (Opt.starts_with("error")) {
         StringRef Specifier;
         if (Opt.size() > 5) {  // Specifier must be present.
           if (Opt[5] != '=' &&
@@ -162,7 +162,7 @@ void clang::ProcessWarningOptions(DiagnosticsEngine &Diags,
       }
 
       // -Wfatal-errors is yet another special case.
-      if (Opt.startswith("fatal-errors")) {
+      if (Opt.starts_with("fatal-errors")) {
         StringRef Specifier;
         if (Opt.size() != 12) {
           if ((Opt[12] != '=' && Opt[12] != '-') || Opt.size() == 13) {
@@ -204,7 +204,7 @@ void clang::ProcessWarningOptions(DiagnosticsEngine &Diags,
 
       // Check to see if this warning starts with "no-", if so, this is a
       // negative form of the option.
-      bool IsPositive = !Opt.startswith("no-");
+      bool IsPositive = !Opt.starts_with("no-");
       if (!IsPositive) Opt = Opt.substr(3);
 
       auto Severity = IsPositive ? diag::Severity::Remark
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index 83d0a72aac5495f55d960f066bfff6bc4a201324..353b7930b3c1ea42a9680110462e8ea5609ecbd9 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -9985,6 +9985,10 @@ CodeGenFunction::getSVEOverloadTypes(const SVETypeFlags &TypeFlags,
   if (TypeFlags.isOverloadCvt())
     return {Ops[0]->getType(), Ops.back()->getType()};
 
+  if (TypeFlags.isReductionQV() && !ResultType->isScalableTy() &&
+      ResultType->isVectorTy())
+    return {ResultType, Ops[1]->getType()};
+
   assert(TypeFlags.isOverloadDefault() && "Unexpected value for overloads");
   return {DefaultType};
 }
diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp
index a24aeea7ae32bfc9b58a8fe5c2b024fb3d46f2e7..51a43b5f85b3cc40cb135ab04593f181f1e29369 100644
--- a/clang/lib/CodeGen/CGCall.cpp
+++ b/clang/lib/CodeGen/CGCall.cpp
@@ -5609,7 +5609,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo &CallInfo,
     EmitBlock(Cont);
   }
   if (CI->getCalledFunction() && CI->getCalledFunction()->hasName() &&
-      CI->getCalledFunction()->getName().startswith("_Z4sqrt")) {
+      CI->getCalledFunction()->getName().starts_with("_Z4sqrt")) {
     SetSqrtFPAccuracy(CI);
   }
   if (callOrInvoke)
diff --git a/clang/lib/CodeGen/CGDebugInfo.cpp b/clang/lib/CodeGen/CGDebugInfo.cpp
index 37d7a6755d3908c051440edb25f38a4f1c02d216..236d53bee4e8f1888782c0bd8ae46b2122b1ee75 100644
--- a/clang/lib/CodeGen/CGDebugInfo.cpp
+++ b/clang/lib/CodeGen/CGDebugInfo.cpp
@@ -637,7 +637,8 @@ void CGDebugInfo::CreateCompileUnit() {
     Sysroot = CGM.getHeaderSearchOpts().Sysroot;
     auto B = llvm::sys::path::rbegin(Sysroot);
     auto E = llvm::sys::path::rend(Sysroot);
-    auto It = std::find_if(B, E, [](auto SDK) { return SDK.endswith(".sdk"); });
+    auto It =
+        std::find_if(B, E, [](auto SDK) { return SDK.ends_with(".sdk"); });
     if (It != E)
       SDK = *It;
   }
@@ -2885,7 +2886,7 @@ llvm::DIModule *CGDebugInfo::getOrCreateModuleRef(ASTSourceDescriptor Mod,
   // clang::Module object, but it won't actually be built or imported; it will
   // be textual.
   if (CreateSkeletonCU && IsRootModule && Mod.getASTFile().empty() && M)
-    assert(StringRef(M->Name).startswith(CGM.getLangOpts().ModuleName) &&
+    assert(StringRef(M->Name).starts_with(CGM.getLangOpts().ModuleName) &&
            "clang module without ASTFile must be specified by -fmodule-name");
 
   // Return a StringRef to the remapped Path.
@@ -4249,7 +4250,7 @@ void CGDebugInfo::emitFunctionStart(GlobalDecl GD, SourceLocation Loc,
 
     Flags |= llvm::DINode::FlagPrototyped;
   }
-  if (Name.startswith("\01"))
+  if (Name.starts_with("\01"))
     Name = Name.substr(1);
 
   assert((!D || !isa<VarDecl>(D) ||
diff --git a/clang/lib/CodeGen/CGException.cpp b/clang/lib/CodeGen/CGException.cpp
index bae8babb8efe4a8cfdbcad74c0877e3f0064ad15..0d507da5c1ba92f78ac5df4f0840c9e7edf605ac 100644
--- a/clang/lib/CodeGen/CGException.cpp
+++ b/clang/lib/CodeGen/CGException.cpp
@@ -277,7 +277,7 @@ static bool LandingPadHasOnlyCXXUses(llvm::LandingPadInst *LPI) {
       if (llvm::GlobalVariable *GV = dyn_cast<llvm::GlobalVariable>(Val))
         // ObjC EH selector entries are always global variables with
         // names starting like this.
-        if (GV->getName().startswith("OBJC_EHTYPE"))
+        if (GV->getName().starts_with("OBJC_EHTYPE"))
           return false;
     } else {
       // Check if any of the filter values have the ObjC prefix.
@@ -288,7 +288,7 @@ static bool LandingPadHasOnlyCXXUses(llvm::LandingPadInst *LPI) {
             cast<llvm::GlobalVariable>((*II)->stripPointerCasts()))
           // ObjC EH selector entries are always global variables with
           // names starting like this.
-          if (GV->getName().startswith("OBJC_EHTYPE"))
+          if (GV->getName().starts_with("OBJC_EHTYPE"))
             return false;
       }
     }
@@ -1917,7 +1917,7 @@ void CodeGenFunction::EmitCapturedLocals(CodeGenFunction &ParentCGF,
         const VarDecl *D = cast<VarDecl>(I.first);
         if (isa<ImplicitParamDecl>(D) &&
             D->getType() == getContext().VoidPtrTy) {
-          assert(D->getName().startswith("frame_pointer"));
+          assert(D->getName().starts_with("frame_pointer"));
           FramePtrAddrAlloca = cast<llvm::AllocaInst>(I.second.getPointer());
           break;
         }
diff --git a/clang/lib/CodeGen/CGExpr.cpp b/clang/lib/CodeGen/CGExpr.cpp
index 69cf7f76be9a7091742e3498c1a904ec838eafe5..ed9aaa28c257337c321592b88e3af54103c2c968 100644
--- a/clang/lib/CodeGen/CGExpr.cpp
+++ b/clang/lib/CodeGen/CGExpr.cpp
@@ -431,7 +431,7 @@ static Address createReferenceTemporary(CodeGenFunction &CGF,
 
 /// Helper method to check if the underlying ABI is AAPCS
 static bool isAAPCS(const TargetInfo &TargetInfo) {
-  return TargetInfo.getABI().startswith("aapcs");
+  return TargetInfo.getABI().starts_with("aapcs");
 }
 
 LValue CodeGenFunction::
@@ -3156,7 +3156,7 @@ LValue CodeGenFunction::EmitPredefinedLValue(const PredefinedExpr *E) {
   auto SL = E->getFunctionName();
   assert(SL != nullptr && "No StringLiteral name in PredefinedExpr");
   StringRef FnName = CurFn->getName();
-  if (FnName.startswith("\01"))
+  if (FnName.starts_with("\01"))
     FnName = FnName.substr(1);
   StringRef NameItems[] = {
       PredefinedExpr::getIdentKindName(E->getIdentKind()), FnName};
diff --git a/clang/lib/CodeGen/CGObjCMac.cpp b/clang/lib/CodeGen/CGObjCMac.cpp
index ba52b23be018b85cda05422c1bb397788d32b07e..517f7cddebc1a2b58ff52858e9a65e34222d087a 100644
--- a/clang/lib/CodeGen/CGObjCMac.cpp
+++ b/clang/lib/CodeGen/CGObjCMac.cpp
@@ -1850,7 +1850,7 @@ static bool hasObjCExceptionAttribute(ASTContext &Context,
 static llvm::GlobalValue::LinkageTypes
 getLinkageTypeForObjCMetadata(CodeGenModule &CGM, StringRef Section) {
   if (CGM.getTriple().isOSBinFormatMachO() &&
-      (Section.empty() || Section.startswith("__DATA")))
+      (Section.empty() || Section.starts_with("__DATA")))
     return llvm::GlobalValue::InternalLinkage;
   return llvm::GlobalValue::PrivateLinkage;
 }
@@ -6162,7 +6162,7 @@ void CGObjCNonFragileABIMac::AddModuleClassList(
   // Section name is obtained by calling GetSectionName, which returns
   // sections in the __DATA segment on MachO.
   assert((!CGM.getTriple().isOSBinFormatMachO() ||
-          SectionName.startswith("__DATA")) &&
+          SectionName.starts_with("__DATA")) &&
          "SectionName expected to start with __DATA on MachO");
   llvm::GlobalVariable *GV = new llvm::GlobalVariable(
       CGM.getModule(), Init->getType(), false,
diff --git a/clang/lib/CodeGen/CGRecordLayoutBuilder.cpp b/clang/lib/CodeGen/CGRecordLayoutBuilder.cpp
index cbfa79e10bfefcc95eaeafdde2ea35a52e112081..868ef810f3c4e8c5d84c5b8b90d7d2f5738278f3 100644
--- a/clang/lib/CodeGen/CGRecordLayoutBuilder.cpp
+++ b/clang/lib/CodeGen/CGRecordLayoutBuilder.cpp
@@ -111,7 +111,7 @@ struct CGRecordLowering {
 
   /// Helper function to check if we are targeting AAPCS.
   bool isAAPCS() const {
-    return Context.getTargetInfo().getABI().startswith("aapcs");
+    return Context.getTargetInfo().getABI().starts_with("aapcs");
   }
 
   /// Helper function to check if the target machine is BigEndian.
diff --git a/clang/lib/CodeGen/CGStmt.cpp b/clang/lib/CodeGen/CGStmt.cpp
index a5cb80640641bb21cf81fa1cfab95dede66d981c..0f79a2e861d220474883cba554bc0779805c9677 100644
--- a/clang/lib/CodeGen/CGStmt.cpp
+++ b/clang/lib/CodeGen/CGStmt.cpp
@@ -2548,7 +2548,7 @@ void CodeGenFunction::EmitAsmStmt(const AsmStmt &S) {
       ResultRegQualTys.push_back(QTy);
       ResultRegDests.push_back(Dest);
 
-      bool IsFlagReg = llvm::StringRef(OutputConstraint).startswith("{@cc");
+      bool IsFlagReg = llvm::StringRef(OutputConstraint).starts_with("{@cc");
       ResultRegIsFlagReg.push_back(IsFlagReg);
 
       llvm::Type *Ty = ConvertTypeForMem(QTy);
diff --git a/clang/lib/CodeGen/CodeGenAction.cpp b/clang/lib/CodeGen/CodeGenAction.cpp
index bb6b1a3bc228cf9889bfc3655c48884c20410acb..753a8fd74fa696045eb0eb13920f42a4a2dac472 100644
--- a/clang/lib/CodeGen/CodeGenAction.cpp
+++ b/clang/lib/CodeGen/CodeGenAction.cpp
@@ -1139,7 +1139,7 @@ CodeGenAction::loadModule(MemoryBufferRef MBRef) {
 
   // Strip off a leading diagnostic code if there is one.
   StringRef Msg = Err.getMessage();
-  if (Msg.startswith("error: "))
+  if (Msg.starts_with("error: "))
     Msg = Msg.substr(7);
 
   unsigned DiagID =
diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp
index b931a81bc00871b6d1de5fc0919e64467324b6cb..7ad26ace328ab2e324c3470f6265104a3a16e4f1 100644
--- a/clang/lib/CodeGen/CodeGenModule.cpp
+++ b/clang/lib/CodeGen/CodeGenModule.cpp
@@ -225,9 +225,9 @@ createTargetCodeGenInfo(CodeGenModule &CGM) {
     StringRef ABIStr = Target.getABI();
     unsigned XLen = Target.getPointerWidth(LangAS::Default);
     unsigned ABIFLen = 0;
-    if (ABIStr.endswith("f"))
+    if (ABIStr.ends_with("f"))
       ABIFLen = 32;
-    else if (ABIStr.endswith("d"))
+    else if (ABIStr.ends_with("d"))
       ABIFLen = 64;
     return createRISCVTargetCodeGenInfo(CGM, XLen, ABIFLen);
   }
@@ -308,9 +308,9 @@ createTargetCodeGenInfo(CodeGenModule &CGM) {
   case llvm::Triple::loongarch64: {
     StringRef ABIStr = Target.getABI();
     unsigned ABIFRLen = 0;
-    if (ABIStr.endswith("f"))
+    if (ABIStr.ends_with("f"))
       ABIFRLen = 32;
-    else if (ABIStr.endswith("d"))
+    else if (ABIStr.ends_with("d"))
       ABIFRLen = 64;
     return createLoongArchTargetCodeGenInfo(
         CGM, Target.getPointerWidth(LangAS::Default), ABIFRLen);
@@ -1715,7 +1715,7 @@ static void AppendTargetMangling(const CodeGenModule &CGM,
   llvm::sort(Info.Features, [&Target](StringRef LHS, StringRef RHS) {
     // Multiversioning doesn't allow "no-${feature}", so we can
     // only have "+" prefixes here.
-    assert(LHS.startswith("+") && RHS.startswith("+") &&
+    assert(LHS.starts_with("+") && RHS.starts_with("+") &&
            "Features should always have a prefix.");
     return Target.multiVersionSortPriority(LHS.substr(1)) >
            Target.multiVersionSortPriority(RHS.substr(1));
@@ -1769,7 +1769,7 @@ static void AppendTargetClonesMangling(const CodeGenModule &CGM,
   } else {
     Out << '.';
     StringRef FeatureStr = Attr->getFeatureStr(VersionIndex);
-    if (FeatureStr.startswith("arch="))
+    if (FeatureStr.starts_with("arch="))
       Out << "arch_" << FeatureStr.substr(sizeof("arch=") - 1);
     else
       Out << FeatureStr;
@@ -3828,7 +3828,7 @@ namespace {
       if (!BuiltinID || !BI.isLibFunction(BuiltinID))
         return false;
       StringRef BuiltinName = BI.getName(BuiltinID);
-      if (BuiltinName.startswith("__builtin_") &&
+      if (BuiltinName.starts_with("__builtin_") &&
           Name == BuiltinName.slice(strlen("__builtin_"), StringRef::npos)) {
         return true;
       }
@@ -4164,7 +4164,7 @@ void CodeGenModule::emitMultiVersionFunctions() {
               Feature.push_back(CurFeat.trim());
           }
         } else {
-          if (Version.startswith("arch="))
+          if (Version.starts_with("arch="))
             Architecture = Version.drop_front(sizeof("arch=") - 1);
           else if (Version != "default")
             Feature.push_back(Version);
diff --git a/clang/lib/CodeGen/CoverageMappingGen.cpp b/clang/lib/CodeGen/CoverageMappingGen.cpp
index b16358ee117ae20707f767dcff0c58af3a89a437..56411e2240e505ea617587199fe28de761f60509 100644
--- a/clang/lib/CodeGen/CoverageMappingGen.cpp
+++ b/clang/lib/CodeGen/CoverageMappingGen.cpp
@@ -1623,8 +1623,12 @@ static void dump(llvm::raw_ostream &OS, StringRef FunctionName,
       OS << "Gap,";
       break;
     case CounterMappingRegion::BranchRegion:
+    case CounterMappingRegion::MCDCBranchRegion:
       OS << "Branch,";
       break;
+    case CounterMappingRegion::MCDCDecisionRegion:
+      OS << "Decision,";
+      break;
     }
 
     OS << "File " << R.FileID << ", " << R.LineStart << ":" << R.ColumnStart
diff --git a/clang/lib/CodeGen/Targets/SPIR.cpp b/clang/lib/CodeGen/Targets/SPIR.cpp
index 8bacba65617e431c1e2911a87284875de2e0920d..cf068cbc4fcd36806f5f2d452a216aac56b8ed2a 100644
--- a/clang/lib/CodeGen/Targets/SPIR.cpp
+++ b/clang/lib/CodeGen/Targets/SPIR.cpp
@@ -146,14 +146,14 @@ static llvm::Type *getSPIRVImageType(llvm::LLVMContext &Ctx, StringRef BaseType,
 
   // Choose the dimension of the image--this corresponds to the Dim enum in
   // SPIR-V (first integer parameter of OpTypeImage).
-  if (OpenCLName.startswith("image2d"))
+  if (OpenCLName.starts_with("image2d"))
     IntParams[0] = 1; // 1D
-  else if (OpenCLName.startswith("image3d"))
+  else if (OpenCLName.starts_with("image3d"))
     IntParams[0] = 2; // 2D
   else if (OpenCLName == "image1d_buffer")
     IntParams[0] = 5; // Buffer
   else
-    assert(OpenCLName.startswith("image1d") && "Unknown image type");
+    assert(OpenCLName.starts_with("image1d") && "Unknown image type");
 
   // Set the other integer parameters of OpTypeImage if necessary. Note that the
   // OpenCL image types don't provide any information for the Sampled or
diff --git a/clang/lib/Driver/Distro.cpp b/clang/lib/Driver/Distro.cpp
index 36f828f8cae26d31193f925ed8b07f15b321923f..a7e7f169dc1419e3d64c7acbe047d56753952afe 100644
--- a/clang/lib/Driver/Distro.cpp
+++ b/clang/lib/Driver/Distro.cpp
@@ -34,7 +34,7 @@ static Distro::DistroType DetectOsRelease(llvm::vfs::FileSystem &VFS) {
 
   // Obviously this can be improved a lot.
   for (StringRef Line : Lines)
-    if (Version == Distro::UnknownDistro && Line.startswith("ID="))
+    if (Version == Distro::UnknownDistro && Line.starts_with("ID="))
       Version = llvm::StringSwitch<Distro::DistroType>(Line.substr(3))
                     .Case("alpine", Distro::AlpineLinux)
                     .Case("fedora", Distro::Fedora)
@@ -60,7 +60,7 @@ static Distro::DistroType DetectLsbRelease(llvm::vfs::FileSystem &VFS) {
 
   for (StringRef Line : Lines)
     if (Version == Distro::UnknownDistro &&
-        Line.startswith("DISTRIB_CODENAME="))
+        Line.starts_with("DISTRIB_CODENAME="))
       Version = llvm::StringSwitch<Distro::DistroType>(Line.substr(17))
                     .Case("hardy", Distro::UbuntuHardy)
                     .Case("intrepid", Distro::UbuntuIntrepid)
@@ -119,10 +119,10 @@ static Distro::DistroType DetectDistro(llvm::vfs::FileSystem &VFS) {
 
   if (File) {
     StringRef Data = File.get()->getBuffer();
-    if (Data.startswith("Fedora release"))
+    if (Data.starts_with("Fedora release"))
       return Distro::Fedora;
-    if (Data.startswith("Red Hat Enterprise Linux") ||
-        Data.startswith("CentOS") || Data.startswith("Scientific Linux")) {
+    if (Data.starts_with("Red Hat Enterprise Linux") ||
+        Data.starts_with("CentOS") || Data.starts_with("Scientific Linux")) {
       if (Data.contains("release 7"))
         return Distro::RHEL7;
       else if (Data.contains("release 6"))
@@ -182,7 +182,7 @@ static Distro::DistroType DetectDistro(llvm::vfs::FileSystem &VFS) {
     SmallVector<StringRef, 8> Lines;
     Data.split(Lines, "\n");
     for (const StringRef &Line : Lines) {
-      if (!Line.trim().startswith("VERSION"))
+      if (!Line.trim().starts_with("VERSION"))
         continue;
       std::pair<StringRef, StringRef> SplitLine = Line.split('=');
       // Old versions have split VERSION and PATCHLEVEL
diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp
index f392f6794f857e62df637602027a56e99ad2fb9e..ff95c899c5f3d4e91770f8203001a62eafddb199 100644
--- a/clang/lib/Driver/Driver.cpp
+++ b/clang/lib/Driver/Driver.cpp
@@ -1451,7 +1451,7 @@ Compilation *Driver::BuildCompilation(ArrayRef<const char *> ArgList) {
     case llvm::Triple::aarch64:
     case llvm::Triple::aarch64_be:
     case llvm::Triple::aarch64_32:
-      if (TC.getTriple().getEnvironmentName().startswith("eabi")) {
+      if (TC.getTriple().getEnvironmentName().starts_with("eabi")) {
         Diag(diag::warn_target_unrecognized_env)
             << TargetTriple
             << (TC.getTriple().getArchName().str() + "-none-elf");
@@ -1540,7 +1540,7 @@ bool Driver::getCrashDiagnosticFile(StringRef ReproCrashFilename,
   for (fs::directory_iterator File(CrashDiagDir, EC), FileEnd;
        File != FileEnd && !EC; File.increment(EC)) {
     StringRef FileName = path::filename(File->path());
-    if (!FileName.startswith(Name))
+    if (!FileName.starts_with(Name))
       continue;
     if (fs::status(File->path(), FileStatus))
       continue;
@@ -1551,7 +1551,7 @@ bool Driver::getCrashDiagnosticFile(StringRef ReproCrashFilename,
     // The first line should start with "Process:", otherwise this isn't a real
     // .crash file.
     StringRef Data = CrashFile.get()->getBuffer();
-    if (!Data.startswith("Process:"))
+    if (!Data.starts_with("Process:"))
       continue;
     // Parse parent process pid line, e.g: "Parent Process: clang-4.0 [79141]"
     size_t ParentProcPos = Data.find("Parent Process:");
@@ -1780,7 +1780,7 @@ void Driver::generateCompilationDiagnostics(
       ReproCrashFilename = TempFile;
       llvm::sys::path::replace_extension(ReproCrashFilename, ".crash");
     }
-    if (StringRef(TempFile).endswith(".cache")) {
+    if (StringRef(TempFile).ends_with(".cache")) {
       // In some cases (modules) we'll dump extra data to help with reproducing
       // the crash into a directory next to the output.
       VFS = llvm::sys::path::filename(TempFile);
@@ -2001,7 +2001,7 @@ void Driver::HandleAutocompletions(StringRef PassedFlags) const {
   // Distinguish "--autocomplete=-someflag" and "--autocomplete=-someflag,"
   // because the latter indicates that the user put space before pushing tab
   // which should end up in a file completion.
-  const bool HasSpace = PassedFlags.endswith(",");
+  const bool HasSpace = PassedFlags.ends_with(",");
 
   // Parse PassedFlags by "," as all the command-line flags are passed to this
   // function separated by ","
@@ -2041,7 +2041,7 @@ void Driver::HandleAutocompletions(StringRef PassedFlags) const {
 
   // When flag ends with '=' and there was no value completion, return empty
   // string and fall back to the file autocompletion.
-  if (SuggestedCompletions.empty() && !Cur.endswith("=")) {
+  if (SuggestedCompletions.empty() && !Cur.ends_with("=")) {
     // If the flag is in the form of "--autocomplete=-foo",
     // we were requested to print out all option names that start with "-foo".
     // For example, "--autocomplete=-fsyn" is expanded to "-fsyntax-only".
@@ -2053,7 +2053,7 @@ void Driver::HandleAutocompletions(StringRef PassedFlags) const {
     // TODO: Find a good way to add them to OptTable instead and them remove
     // this code.
     for (StringRef S : DiagnosticIDs::getDiagnosticFlags())
-      if (S.startswith(Cur))
+      if (S.starts_with(Cur))
         SuggestedCompletions.push_back(std::string(S));
   }
 
@@ -2535,7 +2535,7 @@ bool Driver::DiagnoseInputExistence(const DerivedArgList &Args, StringRef Value,
   // so we can't downgrade diagnostics for `/GR-` from an error to a warning
   // in cc mode. (We can in cl mode because cl.exe itself only warns on
   // unknown flags.)
-  if (IsCLMode() && Ty == types::TY_Object && !Value.startswith("/"))
+  if (IsCLMode() && Ty == types::TY_Object && !Value.starts_with("/"))
     return true;
 
   Diag(clang::diag::err_drv_no_such_file) << Value;
@@ -6565,7 +6565,7 @@ llvm::StringRef clang::driver::getDriverMode(StringRef ProgName,
       getDriverOptTable().getOption(options::OPT_driver_mode).getPrefixedName();
   llvm::StringRef Opt;
   for (StringRef Arg : Args) {
-    if (!Arg.startswith(OptName))
+    if (!Arg.starts_with(OptName))
       continue;
     Opt = Arg;
   }
@@ -6606,7 +6606,7 @@ llvm::Error driver::expandResponseFiles(SmallVectorImpl<const char *> &Args,
   else
     Tokenizer = &llvm::cl::TokenizeGNUCommandLine;
 
-  if (MarkEOLs && Args.size() > 1 && StringRef(Args[1]).startswith("-cc1"))
+  if (MarkEOLs && Args.size() > 1 && StringRef(Args[1]).starts_with("-cc1"))
     MarkEOLs = false;
 
   llvm::cl::ExpansionContext ECtx(Alloc, Tokenizer);
@@ -6620,7 +6620,7 @@ llvm::Error driver::expandResponseFiles(SmallVectorImpl<const char *> &Args,
   // If -cc1 came from a response file, remove the EOL sentinels.
   auto FirstArg = llvm::find_if(llvm::drop_begin(Args),
                                 [](const char *A) { return A != nullptr; });
-  if (FirstArg != Args.end() && StringRef(*FirstArg).startswith("-cc1")) {
+  if (FirstArg != Args.end() && StringRef(*FirstArg).starts_with("-cc1")) {
     // If -cc1 came from a response file, remove the EOL sentinels.
     if (MarkEOLs) {
       auto newEnd = std::remove(Args.begin(), Args.end(), nullptr);
diff --git a/clang/lib/Driver/Job.cpp b/clang/lib/Driver/Job.cpp
index 203400440f9f61f13fe60023cb93382654fed0cf..a6c1581be796261ebc1530dd3460b44d6e215507 100644
--- a/clang/lib/Driver/Job.cpp
+++ b/clang/lib/Driver/Job.cpp
@@ -95,10 +95,10 @@ static bool skipArgs(const char *Flag, bool HaveCrashVFS, int &SkipNum,
 
   // These flags are treated as a single argument (e.g., -F<Dir>).
   StringRef FlagRef(Flag);
-  IsInclude = FlagRef.startswith("-F") || FlagRef.startswith("-I");
+  IsInclude = FlagRef.starts_with("-F") || FlagRef.starts_with("-I");
   if (IsInclude)
     return !HaveCrashVFS;
-  if (FlagRef.startswith("-fmodules-cache-path="))
+  if (FlagRef.starts_with("-fmodules-cache-path="))
     return true;
 
   SkipNum = 0;
@@ -185,8 +185,8 @@ rewriteIncludes(const llvm::ArrayRef<const char *> &Args, size_t Idx,
   SmallString<128> NewInc;
   if (NumArgs == 1) {
     StringRef FlagRef(Args[Idx + NumArgs - 1]);
-    assert((FlagRef.startswith("-F") || FlagRef.startswith("-I")) &&
-            "Expecting -I or -F");
+    assert((FlagRef.starts_with("-F") || FlagRef.starts_with("-I")) &&
+           "Expecting -I or -F");
     StringRef Inc = FlagRef.slice(2, StringRef::npos);
     if (getAbsPath(Inc, NewInc)) {
       SmallString<128> NewArg(FlagRef.slice(0, 2));
diff --git a/clang/lib/Driver/ToolChain.cpp b/clang/lib/Driver/ToolChain.cpp
index ab19166f18c2dcf96187eb36b62b02d8edcc677b..96a57927339a970d344d1a7eafdf74bca742eea9 100644
--- a/clang/lib/Driver/ToolChain.cpp
+++ b/clang/lib/Driver/ToolChain.cpp
@@ -315,7 +315,7 @@ static const DriverSuffix *FindDriverSuffix(StringRef ProgName, size_t &Pos) {
 
   for (const auto &DS : DriverSuffixes) {
     StringRef Suffix(DS.Suffix);
-    if (ProgName.endswith(Suffix)) {
+    if (ProgName.ends_with(Suffix)) {
       Pos = ProgName.size() - Suffix.size();
       return &DS;
     }
@@ -345,7 +345,7 @@ static const DriverSuffix *parseDriverSuffix(StringRef ProgName, size_t &Pos) {
   // added via -target as implicit first argument.
   const DriverSuffix *DS = FindDriverSuffix(ProgName, Pos);
 
-  if (!DS && ProgName.endswith(".exe")) {
+  if (!DS && ProgName.ends_with(".exe")) {
     // Try again after stripping the executable suffix:
     // clang++.exe -> clang++
     ProgName = ProgName.drop_back(StringRef(".exe").size());
diff --git a/clang/lib/Driver/ToolChains/AIX.cpp b/clang/lib/Driver/ToolChains/AIX.cpp
index aed8734b2bab0ffe77caa3807bdba74849d9f25f..f9670ea6f251bd2ed7e8b4334dbd0c3cb99fd512 100644
--- a/clang/lib/Driver/ToolChains/AIX.cpp
+++ b/clang/lib/Driver/ToolChains/AIX.cpp
@@ -88,7 +88,7 @@ static bool hasExportListLinkerOpts(const ArgStringList &CmdArgs) {
   for (size_t i = 0, Size = CmdArgs.size(); i < Size; ++i) {
     llvm::StringRef ArgString(CmdArgs[i]);
 
-    if (ArgString.startswith("-bE:") || ArgString.startswith("-bexport:") ||
+    if (ArgString.starts_with("-bE:") || ArgString.starts_with("-bexport:") ||
         ArgString == "-bexpall" || ArgString == "-bexpfull")
       return true;
 
@@ -96,8 +96,8 @@ static bool hasExportListLinkerOpts(const ArgStringList &CmdArgs) {
     if (ArgString == "-b" && i + 1 < Size) {
       ++i;
       llvm::StringRef ArgNextString(CmdArgs[i]);
-      if (ArgNextString.startswith("E:") ||
-          ArgNextString.startswith("export:") || ArgNextString == "expall" ||
+      if (ArgNextString.starts_with("E:") ||
+          ArgNextString.starts_with("export:") || ArgNextString == "expall" ||
           ArgNextString == "expfull")
         return true;
     }
diff --git a/clang/lib/Driver/ToolChains/AMDGPU.cpp b/clang/lib/Driver/ToolChains/AMDGPU.cpp
index cad206ea4df1bc5c4c0e032dd40d90a57f1a2dd1..56f06fc5fccb7eb75ffec08dc4fce6fe4c61f0cc 100644
--- a/clang/lib/Driver/ToolChains/AMDGPU.cpp
+++ b/clang/lib/Driver/ToolChains/AMDGPU.cpp
@@ -49,7 +49,7 @@ RocmInstallationDetector::findSPACKPackage(const Candidate &Cand,
                                      FileEnd;
        File != FileEnd && !EC; File.increment(EC)) {
     llvm::StringRef FileName = llvm::sys::path::filename(File->path());
-    if (FileName.startswith(Prefix)) {
+    if (FileName.starts_with(Prefix)) {
       SubDirs.push_back(FileName);
       if (SubDirs.size() > 1)
         break;
@@ -84,13 +84,13 @@ void RocmInstallationDetector::scanLibDevicePath(llvm::StringRef Path) {
        !EC && LI != LE; LI = LI.increment(EC)) {
     StringRef FilePath = LI->path();
     StringRef FileName = llvm::sys::path::filename(FilePath);
-    if (!FileName.endswith(Suffix))
+    if (!FileName.ends_with(Suffix))
       continue;
 
     StringRef BaseName;
-    if (FileName.endswith(Suffix2))
+    if (FileName.ends_with(Suffix2))
       BaseName = FileName.drop_back(Suffix2.size());
-    else if (FileName.endswith(Suffix))
+    else if (FileName.ends_with(Suffix))
       BaseName = FileName.drop_back(Suffix.size());
 
     const StringRef ABIVersionPrefix = "oclc_abi_version_";
@@ -124,7 +124,7 @@ void RocmInstallationDetector::scanLibDevicePath(llvm::StringRef Path) {
       WavefrontSize64.On = FilePath;
     } else if (BaseName == "oclc_wavefrontsize64_off") {
       WavefrontSize64.Off = FilePath;
-    } else if (BaseName.startswith(ABIVersionPrefix)) {
+    } else if (BaseName.starts_with(ABIVersionPrefix)) {
       unsigned ABIVersionNumber;
       if (BaseName.drop_front(ABIVersionPrefix.size())
               .getAsInteger(/*Redex=*/0, ABIVersionNumber))
@@ -134,7 +134,7 @@ void RocmInstallationDetector::scanLibDevicePath(llvm::StringRef Path) {
       // Process all bitcode filenames that look like
       // ocl_isa_version_XXX.amdgcn.bc
       const StringRef DeviceLibPrefix = "oclc_isa_version_";
-      if (!BaseName.startswith(DeviceLibPrefix))
+      if (!BaseName.starts_with(DeviceLibPrefix))
         continue;
 
       StringRef IsaVersionNumber =
@@ -230,7 +230,7 @@ RocmInstallationDetector::getInstallationPathCandidates() {
     // <rocm_root>/llvm-amdgpu-<rocm_release_string>-<hash>/bin directory.
     // We only consider the parent directory of llvm-amdgpu package as ROCm
     // installation candidate for SPACK.
-    if (ParentName.startswith("llvm-amdgpu-")) {
+    if (ParentName.starts_with("llvm-amdgpu-")) {
       auto SPACKPostfix =
           ParentName.drop_front(strlen("llvm-amdgpu-")).split('-');
       auto SPACKReleaseStr = SPACKPostfix.first;
@@ -243,7 +243,7 @@ RocmInstallationDetector::getInstallationPathCandidates() {
 
     // Some versions of the rocm llvm package install to /opt/rocm/llvm/bin
     // Some versions of the aomp package install to /opt/rocm/aomp/bin
-    if (ParentName == "llvm" || ParentName.startswith("aomp"))
+    if (ParentName == "llvm" || ParentName.starts_with("aomp"))
       ParentDir = llvm::sys::path::parent_path(ParentDir);
 
     return Candidate(ParentDir.str(), /*StrictChecking=*/true);
@@ -292,7 +292,7 @@ RocmInstallationDetector::getInstallationPathCandidates() {
            FileEnd;
        File != FileEnd && !EC; File.increment(EC)) {
     llvm::StringRef FileName = llvm::sys::path::filename(File->path());
-    if (!FileName.startswith("rocm-"))
+    if (!FileName.starts_with("rocm-"))
       continue;
     if (LatestROCm.empty()) {
       LatestROCm = FileName.str();
diff --git a/clang/lib/Driver/ToolChains/Arch/AArch64.cpp b/clang/lib/Driver/ToolChains/Arch/AArch64.cpp
index 097258b16924442abf7b067343eea81d9c8cb0ec..912df79417ae21e4892251960d7b05640d547a93 100644
--- a/clang/lib/Driver/ToolChains/Arch/AArch64.cpp
+++ b/clang/lib/Driver/ToolChains/Arch/AArch64.cpp
@@ -228,7 +228,7 @@ getAArch64MicroArchFeaturesFromMtune(const Driver &D, StringRef Mtune,
   if (MtuneLowerCase == "native")
     MtuneLowerCase = std::string(llvm::sys::getHostCPUName());
   if (MtuneLowerCase == "cyclone" ||
-      StringRef(MtuneLowerCase).startswith("apple")) {
+      StringRef(MtuneLowerCase).starts_with("apple")) {
     Features.push_back("+zcm");
     Features.push_back("+zcz");
   }
@@ -262,7 +262,7 @@ void aarch64::getAArch64TargetFeatures(const Driver &D,
     for (const auto *A :
          Args.filtered(options::OPT_Wa_COMMA, options::OPT_Xassembler))
       for (StringRef Value : A->getValues())
-        if (Value.startswith("-march="))
+        if (Value.starts_with("-march="))
           WaMArch = Value.substr(7);
   // Call getAArch64ArchFeaturesFromMarch only if "-Wa,-march=" or
   // "-Xassembler -march" is detected. Otherwise it may return false
diff --git a/clang/lib/Driver/ToolChains/Arch/ARM.cpp b/clang/lib/Driver/ToolChains/Arch/ARM.cpp
index f1d7aeb555f8bd0974cc5790cd1391d27fa9c5fe..25470db2b6cebd708bb7803299b38aa43a600cf1 100644
--- a/clang/lib/Driver/ToolChains/Arch/ARM.cpp
+++ b/clang/lib/Driver/ToolChains/Arch/ARM.cpp
@@ -67,9 +67,9 @@ void arm::getARMArchCPUFromArgs(const ArgList &Args, llvm::StringRef &Arch,
     // Use getValues because -Wa can have multiple arguments
     // e.g. -Wa,-mcpu=foo,-mcpu=bar
     for (StringRef Value : A->getValues()) {
-      if (Value.startswith("-mcpu="))
+      if (Value.starts_with("-mcpu="))
         CPU = Value.substr(6);
-      if (Value.startswith("-march="))
+      if (Value.starts_with("-march="))
         Arch = Value.substr(7);
     }
   }
@@ -285,9 +285,9 @@ void arm::setArchNameInTriple(const Driver &D, const ArgList &Args,
         // There is no assembler equivalent of -mno-thumb, -marm, or -mno-arm.
         if (Value == "-mthumb")
           IsThumb = true;
-        else if (Value.startswith("-march="))
+        else if (Value.starts_with("-march="))
           WaMArch = Value.substr(7);
-        else if (Value.startswith("-mcpu="))
+        else if (Value.starts_with("-mcpu="))
           WaMCPU = Value.substr(6);
       }
     }
@@ -528,13 +528,13 @@ llvm::ARM::FPUKind arm::getARMTargetFeatures(const Driver &D,
       // We use getValues here because you can have many options per -Wa
       // We will keep the last one we find for each of these
       for (StringRef Value : A->getValues()) {
-        if (Value.startswith("-mfpu=")) {
+        if (Value.starts_with("-mfpu=")) {
           WaFPU = std::make_pair(A, Value.substr(6));
-        } else if (Value.startswith("-mcpu=")) {
+        } else if (Value.starts_with("-mcpu=")) {
           WaCPU = std::make_pair(A, Value.substr(6));
-        } else if (Value.startswith("-mhwdiv=")) {
+        } else if (Value.starts_with("-mhwdiv=")) {
           WaHDiv = std::make_pair(A, Value.substr(8));
-        } else if (Value.startswith("-march=")) {
+        } else if (Value.starts_with("-march=")) {
           WaArch = std::make_pair(A, Value.substr(7));
         }
       }
@@ -796,7 +796,7 @@ fp16_fml_fallthrough:
   // Propagate frame-chain model selection
   if (Arg *A = Args.getLastArg(options::OPT_mframe_chain)) {
     StringRef FrameChainOption = A->getValue();
-    if (FrameChainOption.startswith("aapcs"))
+    if (FrameChainOption.starts_with("aapcs"))
       Features.push_back("+aapcs-frame-chain");
     if (FrameChainOption == "aapcs+leaf")
       Features.push_back("+aapcs-frame-chain-leaf");
diff --git a/clang/lib/Driver/ToolChains/Arch/X86.cpp b/clang/lib/Driver/ToolChains/Arch/X86.cpp
index 3e51b2b5ce8642e0ac08afc98617e96b630e2816..fef0522aaf45b8a3d34b12782bf6c4ca27ab5645 100644
--- a/clang/lib/Driver/ToolChains/Arch/X86.cpp
+++ b/clang/lib/Driver/ToolChains/Arch/X86.cpp
@@ -234,15 +234,15 @@ void x86::getX86TargetFeatures(const Driver &D, const llvm::Triple &Triple,
     A->claim();
 
     // Skip over "-m".
-    assert(Name.startswith("m") && "Invalid feature name.");
+    assert(Name.starts_with("m") && "Invalid feature name.");
     Name = Name.substr(1);
 
-    bool IsNegative = Name.startswith("no-");
+    bool IsNegative = Name.starts_with("no-");
     if (IsNegative)
       Name = Name.substr(3);
 
 #ifndef NDEBUG
-    assert(Name.startswith("avx10.") && "Invalid AVX10 feature name.");
+    assert(Name.starts_with("avx10.") && "Invalid AVX10 feature name.");
     StringRef Version, Width;
     std::tie(Version, Width) = Name.substr(6).split('-');
     assert(Version == "1" && "Invalid AVX10 feature name.");
@@ -260,7 +260,7 @@ void x86::getX86TargetFeatures(const Driver &D, const llvm::Triple &Triple,
     A->claim();
 
     // Skip over "-m".
-    assert(Name.startswith("m") && "Invalid feature name.");
+    assert(Name.starts_with("m") && "Invalid feature name.");
     Name = Name.substr(1);
 
     // Replace -mgeneral-regs-only with -x87, -mmx, -sse
@@ -269,7 +269,7 @@ void x86::getX86TargetFeatures(const Driver &D, const llvm::Triple &Triple,
       continue;
     }
 
-    bool IsNegative = Name.startswith("no-");
+    bool IsNegative = Name.starts_with("no-");
     if (A->getOption().matches(options::OPT_mapx_features_EQ) ||
         A->getOption().matches(options::OPT_mno_apx_features_EQ)) {
 
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index f95f3227aba7d03bb4956b62f39569ce6ee49768..de9fd5eaa1e020f6f6d3d43ffccb0dd4f82e9840 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -223,7 +223,7 @@ static void ParseMRecip(const Driver &D, const ArgList &Args,
   for (unsigned i = 0; i != NumOptions; ++i) {
     StringRef Val = A->getValue(i);
 
-    bool IsDisabled = Val.startswith(DisabledPrefixIn);
+    bool IsDisabled = Val.starts_with(DisabledPrefixIn);
     // Ignore the disablement token for string matching.
     if (IsDisabled)
       Val = Val.substr(1);
@@ -433,7 +433,7 @@ static void addDebugObjectName(const ArgList &Args, ArgStringList &CmdArgs,
                                const char *OutputFileName) {
   // No need to generate a value for -object-file-name if it was provided.
   for (auto *Arg : Args.filtered(options::OPT_Xclang))
-    if (StringRef(Arg->getValue()).startswith("-object-file-name"))
+    if (StringRef(Arg->getValue()).starts_with("-object-file-name"))
       return;
 
   if (Args.hasArg(options::OPT_object_file_name_EQ))
@@ -940,7 +940,7 @@ static void handleAMDGPUCodeObjectVersionOptions(const Driver &D,
 static bool hasClangPchSignature(const Driver &D, StringRef Path) {
   if (llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> MemBuf =
           D.getVFS().getBufferForFile(Path))
-    return (*MemBuf)->getBuffer().startswith("CPCH");
+    return (*MemBuf)->getBuffer().starts_with("CPCH");
   return false;
 }
 
@@ -1715,7 +1715,7 @@ void Clang::AddAArch64TargetArgs(const ArgList &Args,
         Val.equals("256+") || Val.equals("512+") || Val.equals("1024+") ||
         Val.equals("2048+")) {
       unsigned Bits = 0;
-      if (Val.endswith("+"))
+      if (Val.ends_with("+"))
         Val = Val.substr(0, Val.size() - 1);
       else {
         bool Invalid = Val.getAsInteger(10, Bits); (void)Invalid;
@@ -2503,7 +2503,7 @@ static void CollectArgsForIntegratedAssembler(Compilation &C,
       case llvm::Triple::thumbeb:
       case llvm::Triple::arm:
       case llvm::Triple::armeb:
-        if (Value.startswith("-mimplicit-it=")) {
+        if (Value.starts_with("-mimplicit-it=")) {
           // Only store the value; the last value set takes effect.
           ImplicitIt = Value.split("=").second;
           if (CheckARMImplicitITArg(ImplicitIt))
@@ -2528,12 +2528,12 @@ static void CollectArgsForIntegratedAssembler(Compilation &C,
           CmdArgs.push_back("-use-tcc-in-div");
           continue;
         }
-        if (Value.startswith("-msoft-float")) {
+        if (Value.starts_with("-msoft-float")) {
           CmdArgs.push_back("-target-feature");
           CmdArgs.push_back("+soft-float");
           continue;
         }
-        if (Value.startswith("-mhard-float")) {
+        if (Value.starts_with("-mhard-float")) {
           CmdArgs.push_back("-target-feature");
           CmdArgs.push_back("-soft-float");
           continue;
@@ -2570,8 +2570,8 @@ static void CollectArgsForIntegratedAssembler(Compilation &C,
         CmdArgs.push_back("-massembler-no-warn");
       } else if (Value == "--noexecstack") {
         UseNoExecStack = true;
-      } else if (Value.startswith("-compress-debug-sections") ||
-                 Value.startswith("--compress-debug-sections") ||
+      } else if (Value.starts_with("-compress-debug-sections") ||
+                 Value.starts_with("--compress-debug-sections") ||
                  Value == "-nocompress-debug-sections" ||
                  Value == "--nocompress-debug-sections") {
         CmdArgs.push_back(Value.data());
@@ -2581,13 +2581,13 @@ static void CollectArgsForIntegratedAssembler(Compilation &C,
       } else if (Value == "-mrelax-relocations=no" ||
                  Value == "--mrelax-relocations=no") {
         UseRelaxRelocations = false;
-      } else if (Value.startswith("-I")) {
+      } else if (Value.starts_with("-I")) {
         CmdArgs.push_back(Value.data());
         // We need to consume the next argument if the current arg is a plain
         // -I. The next arg will be the include directory.
         if (Value == "-I")
           TakeNextArg = true;
-      } else if (Value.startswith("-gdwarf-")) {
+      } else if (Value.starts_with("-gdwarf-")) {
         // "-gdwarf-N" options are not cc1as options.
         unsigned DwarfVersion = DwarfVersionNum(Value);
         if (DwarfVersion == 0) { // Send it onward, and let cc1as complain.
@@ -2597,30 +2597,30 @@ static void CollectArgsForIntegratedAssembler(Compilation &C,
                                   llvm::codegenoptions::DebugInfoConstructor,
                                   DwarfVersion, llvm::DebuggerKind::Default);
         }
-      } else if (Value.startswith("-mcpu") || Value.startswith("-mfpu") ||
-                 Value.startswith("-mhwdiv") || Value.startswith("-march")) {
+      } else if (Value.starts_with("-mcpu") || Value.starts_with("-mfpu") ||
+                 Value.starts_with("-mhwdiv") || Value.starts_with("-march")) {
         // Do nothing, we'll validate it later.
       } else if (Value == "-defsym") {
-          if (A->getNumValues() != 2) {
-            D.Diag(diag::err_drv_defsym_invalid_format) << Value;
-            break;
-          }
-          const char *S = A->getValue(1);
-          auto Pair = StringRef(S).split('=');
-          auto Sym = Pair.first;
-          auto SVal = Pair.second;
-
-          if (Sym.empty() || SVal.empty()) {
-            D.Diag(diag::err_drv_defsym_invalid_format) << S;
-            break;
-          }
-          int64_t IVal;
-          if (SVal.getAsInteger(0, IVal)) {
-            D.Diag(diag::err_drv_defsym_invalid_symval) << SVal;
-            break;
-          }
-          CmdArgs.push_back(Value.data());
-          TakeNextArg = true;
+        if (A->getNumValues() != 2) {
+          D.Diag(diag::err_drv_defsym_invalid_format) << Value;
+          break;
+        }
+        const char *S = A->getValue(1);
+        auto Pair = StringRef(S).split('=');
+        auto Sym = Pair.first;
+        auto SVal = Pair.second;
+
+        if (Sym.empty() || SVal.empty()) {
+          D.Diag(diag::err_drv_defsym_invalid_format) << S;
+          break;
+        }
+        int64_t IVal;
+        if (SVal.getAsInteger(0, IVal)) {
+          D.Diag(diag::err_drv_defsym_invalid_symval) << SVal;
+          break;
+        }
+        CmdArgs.push_back(Value.data());
+        TakeNextArg = true;
       } else if (Value == "-fdebug-compilation-dir") {
         CmdArgs.push_back("-fdebug-compilation-dir");
         TakeNextArg = true;
@@ -3331,7 +3331,7 @@ static void RenderSSPOptions(const Driver &D, const ToolChain &TC,
   // --param ssp-buffer-size=
   for (const Arg *A : Args.filtered(options::OPT__param)) {
     StringRef Str(A->getValue());
-    if (Str.startswith("ssp-buffer-size=")) {
+    if (Str.starts_with("ssp-buffer-size=")) {
       if (StackProtectorLevel) {
         CmdArgs.push_back("-stack-protector-buffer-size");
         // FIXME: Verify the argument is a valid integer.
@@ -5615,6 +5615,10 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
                        options::OPT_fno_auto_import);
   }
 
+  if (Args.hasFlag(options::OPT_fms_volatile, options::OPT_fno_ms_volatile,
+                   Triple.isX86() && D.IsCLMode()))
+    CmdArgs.push_back("-fms-volatile");
+
   // Non-PIC code defaults to -fdirect-access-external-data while PIC code
   // defaults to -fno-direct-access-external-data. Pass the option if different
   // from the default.
@@ -5882,7 +5886,7 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
     StringRef Val = A->getValue();
     if (Triple.isX86() && Triple.isOSBinFormatELF()) {
       if (Val != "all" && Val != "labels" && Val != "none" &&
-          !Val.startswith("list="))
+          !Val.starts_with("list="))
         D.Diag(diag::err_drv_invalid_value)
             << A->getAsString(Args) << A->getValue();
       else
@@ -7947,18 +7951,6 @@ void Clang::AddClangCLArgs(const ArgList &Args, types::ID InputType,
     CmdArgs.push_back("-P");
   }
 
-  unsigned VolatileOptionID;
-  if (getToolChain().getTriple().isX86())
-    VolatileOptionID = options::OPT__SLASH_volatile_ms;
-  else
-    VolatileOptionID = options::OPT__SLASH_volatile_iso;
-
-  if (Arg *A = Args.getLastArg(options::OPT__SLASH_volatile_Group))
-    VolatileOptionID = A->getOption().getID();
-
-  if (VolatileOptionID == options::OPT__SLASH_volatile_ms)
-    CmdArgs.push_back("-fms-volatile");
-
  if (Args.hasFlag(options::OPT__SLASH_Zc_dllexportInlines_,
                   options::OPT__SLASH_Zc_dllexportInlines,
                   false)) {
@@ -8390,14 +8382,14 @@ void ClangAs::ConstructJob(Compilation &C, const JobAction &JA,
       continue;
     auto &JArgs = J.getArguments();
     for (unsigned I = 0; I < JArgs.size(); ++I) {
-      if (StringRef(JArgs[I]).startswith("-object-file-name=") &&
+      if (StringRef(JArgs[I]).starts_with("-object-file-name=") &&
           Output.isFilename()) {
-        ArgStringList NewArgs(JArgs.begin(), JArgs.begin() + I);
-        addDebugObjectName(Args, NewArgs, DebugCompilationDir,
-                           Output.getFilename());
-        NewArgs.append(JArgs.begin() + I + 1, JArgs.end());
-        J.replaceArguments(NewArgs);
-        break;
+       ArgStringList NewArgs(JArgs.begin(), JArgs.begin() + I);
+       addDebugObjectName(Args, NewArgs, DebugCompilationDir,
+                          Output.getFilename());
+       NewArgs.append(JArgs.begin() + I + 1, JArgs.end());
+       J.replaceArguments(NewArgs);
+       break;
       }
     }
   }
@@ -8661,7 +8653,7 @@ void OffloadPackager::ConstructJob(Compilation &C, const JobAction &JA,
     getTargetFeatures(TC->getDriver(), TC->getTriple(), TCArgs, Features,
                       false);
     llvm::copy_if(Features, std::back_inserter(FeatureArgs),
-                  [](StringRef Arg) { return !Arg.startswith("-target"); });
+                  [](StringRef Arg) { return !Arg.starts_with("-target"); });
 
     if (TC->getTriple().isAMDGPU()) {
       for (StringRef Feature : llvm::split(Arch.split(':').second, ':')) {
diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp
index 31e7d68161ff1b224dfa4b49433d82242b02794d..01fb0718b4079d6a8d83dc4d38321ee3c78acc84 100644
--- a/clang/lib/Driver/ToolChains/CommonArgs.cpp
+++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp
@@ -303,7 +303,7 @@ void tools::handleTargetFeaturesGroup(const Driver &D,
     A->claim();
 
     // Skip over "-m".
-    assert(Name.startswith("m") && "Invalid feature name.");
+    assert(Name.starts_with("m") && "Invalid feature name.");
     Name = Name.substr(1);
 
     auto Proc = getCPUName(D, Args, Triple);
@@ -317,7 +317,7 @@ void tools::handleTargetFeaturesGroup(const Driver &D,
       continue;
     }
 
-    bool IsNegative = Name.startswith("no-");
+    bool IsNegative = Name.starts_with("no-");
     if (IsNegative)
       Name = Name.substr(3);
 
@@ -2336,20 +2336,20 @@ static void GetSDLFromOffloadArchive(
   llvm::Triple Triple(D.getTargetTriple());
   bool IsMSVC = Triple.isWindowsMSVCEnvironment();
   auto Ext = IsMSVC ? ".lib" : ".a";
-  if (!Lib.startswith(":") && !Lib.startswith("-l")) {
+  if (!Lib.starts_with(":") && !Lib.starts_with("-l")) {
     if (llvm::sys::fs::exists(Lib)) {
       ArchiveOfBundles = Lib;
       FoundAOB = true;
     }
   } else {
-    if (Lib.startswith("-l"))
+    if (Lib.starts_with("-l"))
       Lib = Lib.drop_front(2);
     for (auto LPath : LibraryPaths) {
       ArchiveOfBundles.clear();
-      auto LibFile =
-          (Lib.startswith(":") ? Lib.drop_front()
-                               : IsMSVC ? Lib + Ext : "lib" + Lib + Ext)
-              .str();
+      auto LibFile = (Lib.starts_with(":") ? Lib.drop_front()
+                      : IsMSVC             ? Lib + Ext
+                                           : "lib" + Lib + Ext)
+                         .str();
       for (auto Prefix : {"/libdevice/", "/"}) {
         auto AOB = Twine(LPath + Prefix + LibFile).str();
         if (llvm::sys::fs::exists(AOB)) {
diff --git a/clang/lib/Driver/ToolChains/Cuda.cpp b/clang/lib/Driver/ToolChains/Cuda.cpp
index ef1e77974c1eaaf79f707bfc22c565024f3b5862..f7a208575cb0cf8d617ab3da73d917285af5462e 100644
--- a/clang/lib/Driver/ToolChains/Cuda.cpp
+++ b/clang/lib/Driver/ToolChains/Cuda.cpp
@@ -238,7 +238,7 @@ CudaInstallationDetector::CudaInstallationDetector(
         // Process all bitcode filenames that look like
         // libdevice.compute_XX.YY.bc
         const StringRef LibDeviceName = "libdevice.";
-        if (!(FileName.startswith(LibDeviceName) && FileName.endswith(".bc")))
+        if (!(FileName.starts_with(LibDeviceName) && FileName.ends_with(".bc")))
           continue;
         StringRef GpuArch = FileName.slice(
             LibDeviceName.size(), FileName.find('.', LibDeviceName.size()));
diff --git a/clang/lib/Driver/ToolChains/Darwin.cpp b/clang/lib/Driver/ToolChains/Darwin.cpp
index 692b3a3f285d744afc868a622ea7dc0aa1559de6..65846cace461e3eedf6a8c18ed53fd6f516f366d 100644
--- a/clang/lib/Driver/ToolChains/Darwin.cpp
+++ b/clang/lib/Driver/ToolChains/Darwin.cpp
@@ -1010,13 +1010,13 @@ static const char *ArmMachOArchNameCPU(StringRef CPU) {
 
   // FIXME: Make sure this MachO triple mangling is really necessary.
   // ARMv5* normalises to ARMv5.
-  if (Arch.startswith("armv5"))
+  if (Arch.starts_with("armv5"))
     Arch = Arch.substr(0, 5);
   // ARMv6*, except ARMv6M, normalises to ARMv6.
-  else if (Arch.startswith("armv6") && !Arch.endswith("6m"))
+  else if (Arch.starts_with("armv6") && !Arch.ends_with("6m"))
     Arch = Arch.substr(0, 5);
   // ARMv7A normalises to ARMv7.
-  else if (Arch.endswith("v7a"))
+  else if (Arch.ends_with("v7a"))
     Arch = Arch.substr(0, 5);
   return Arch.data();
 }
@@ -1319,8 +1319,8 @@ StringRef Darwin::getSDKName(StringRef isysroot) {
   auto EndSDK = llvm::sys::path::rend(isysroot);
   for (auto IT = BeginSDK; IT != EndSDK; ++IT) {
     StringRef SDK = *IT;
-    if (SDK.endswith(".sdk"))
-      return SDK.slice(0, SDK.size() - 4);
+    if (SDK.ends_with(".sdk"))
+        return SDK.slice(0, SDK.size() - 4);
   }
   return "";
 }
@@ -1959,22 +1959,23 @@ inferDeploymentTargetFromSDK(DerivedArgList &Args,
 
   auto CreatePlatformFromSDKName =
       [&](StringRef SDK) -> std::optional<DarwinPlatform> {
-    if (SDK.startswith("iPhoneOS") || SDK.startswith("iPhoneSimulator"))
+    if (SDK.starts_with("iPhoneOS") || SDK.starts_with("iPhoneSimulator"))
       return DarwinPlatform::createFromSDK(
           Darwin::IPhoneOS, Version,
-          /*IsSimulator=*/SDK.startswith("iPhoneSimulator"));
-    else if (SDK.startswith("MacOSX"))
+          /*IsSimulator=*/SDK.starts_with("iPhoneSimulator"));
+    else if (SDK.starts_with("MacOSX"))
       return DarwinPlatform::createFromSDK(Darwin::MacOS,
                                            getSystemOrSDKMacOSVersion(Version));
-    else if (SDK.startswith("WatchOS") || SDK.startswith("WatchSimulator"))
+    else if (SDK.starts_with("WatchOS") || SDK.starts_with("WatchSimulator"))
       return DarwinPlatform::createFromSDK(
           Darwin::WatchOS, Version,
-          /*IsSimulator=*/SDK.startswith("WatchSimulator"));
-    else if (SDK.startswith("AppleTVOS") || SDK.startswith("AppleTVSimulator"))
+          /*IsSimulator=*/SDK.starts_with("WatchSimulator"));
+    else if (SDK.starts_with("AppleTVOS") ||
+             SDK.starts_with("AppleTVSimulator"))
       return DarwinPlatform::createFromSDK(
           Darwin::TvOS, Version,
-          /*IsSimulator=*/SDK.startswith("AppleTVSimulator"));
-    else if (SDK.startswith("DriverKit"))
+          /*IsSimulator=*/SDK.starts_with("AppleTVSimulator"));
+    else if (SDK.starts_with("DriverKit"))
       return DarwinPlatform::createFromSDK(Darwin::DriverKit, Version);
     return std::nullopt;
   };
@@ -2339,8 +2340,8 @@ void Darwin::AddDeploymentTarget(DerivedArgList &Args) const {
     if (SDK.size() > 0) {
       size_t StartVer = SDK.find_first_of("0123456789");
       StringRef SDKName = SDK.slice(0, StartVer);
-      if (!SDKName.startswith(getPlatformFamily()) &&
-          !dropSDKNamePrefix(SDKName).startswith(getPlatformFamily()))
+      if (!SDKName.starts_with(getPlatformFamily()) &&
+          !dropSDKNamePrefix(SDKName).starts_with(getPlatformFamily()))
         getDriver().Diag(diag::warn_incompatible_sysroot)
             << SDKName << getPlatformFamily();
     }
diff --git a/clang/lib/Driver/ToolChains/Flang.cpp b/clang/lib/Driver/ToolChains/Flang.cpp
index 502b9f17a06c52ff06434f206489f817f81dfcc4..41eaad3bbad0a37f1d191fc71facbc183286314a 100644
--- a/clang/lib/Driver/ToolChains/Flang.cpp
+++ b/clang/lib/Driver/ToolChains/Flang.cpp
@@ -182,7 +182,7 @@ void Flang::AddAArch64TargetArgs(const ArgList &Args,
         Val.equals("256+") || Val.equals("512+") || Val.equals("1024+") ||
         Val.equals("2048+")) {
       unsigned Bits = 0;
-      if (Val.endswith("+"))
+      if (Val.ends_with("+"))
         Val = Val.substr(0, Val.size() - 1);
       else {
         [[maybe_unused]] bool Invalid = Val.getAsInteger(10, Bits);
diff --git a/clang/lib/Driver/ToolChains/Gnu.cpp b/clang/lib/Driver/ToolChains/Gnu.cpp
index b875991844ffff5240cdb2195a77bf74945dca5e..835215a83c4037eee2e5351993fff8983432f0ff 100644
--- a/clang/lib/Driver/ToolChains/Gnu.cpp
+++ b/clang/lib/Driver/ToolChains/Gnu.cpp
@@ -1084,7 +1084,7 @@ static bool findMipsCsMultilibs(const Multilib::flags_list &Flags,
             .FilterOut(NonExistent)
             .setIncludeDirsCallback([](const Multilib &M) {
               std::vector<std::string> Dirs({"/include"});
-              if (StringRef(M.includeSuffix()).startswith("/uclibc"))
+              if (StringRef(M.includeSuffix()).starts_with("/uclibc"))
                 Dirs.push_back(
                     "/../../../../mips-linux-gnu/libc/uclibc/usr/include");
               else
@@ -1288,7 +1288,7 @@ static bool findMipsMtiMultilibs(const Multilib::flags_list &Flags,
             .FilterOut(NonExistent)
             .setIncludeDirsCallback([](const Multilib &M) {
               std::vector<std::string> Dirs({"/include"});
-              if (StringRef(M.includeSuffix()).startswith("/uclibc"))
+              if (StringRef(M.includeSuffix()).starts_with("/uclibc"))
                 Dirs.push_back("/../../../../sysroot/uclibc/usr/include");
               else
                 Dirs.push_back("/../../../../sysroot/usr/include");
@@ -3055,7 +3055,7 @@ void Generic_GCC::AddMultilibPaths(const Driver &D,
     // the cross. Note that GCC does include some of these directories in some
     // configurations but this seems somewhere between questionable and simply
     // a bug.
-    if (StringRef(LibPath).startswith(SysRoot))
+    if (StringRef(LibPath).starts_with(SysRoot))
       addPathIfExists(D, LibPath + "/../" + OSLibDir, Paths);
   }
 }
diff --git a/clang/lib/Driver/ToolChains/Hexagon.cpp b/clang/lib/Driver/ToolChains/Hexagon.cpp
index d0ff7d0c1310da565c02b95097d706264e73fb16..6a2a105bee32ef3e183559c0a9b4ae7771e8483e 100644
--- a/clang/lib/Driver/ToolChains/Hexagon.cpp
+++ b/clang/lib/Driver/ToolChains/Hexagon.cpp
@@ -54,11 +54,11 @@ static void handleHVXTargetFeatures(const Driver &D, const ArgList &Args,
   auto makeFeature = [&Args](Twine T, bool Enable) -> StringRef {
     const std::string &S = T.str();
     StringRef Opt(S);
-    if (Opt.endswith("="))
+    if (Opt.ends_with("="))
       Opt = Opt.drop_back(1);
-    if (Opt.startswith("mno-"))
+    if (Opt.starts_with("mno-"))
       Opt = Opt.drop_front(4);
-    else if (Opt.startswith("m"))
+    else if (Opt.starts_with("m"))
       Opt = Opt.drop_front(1);
     return Args.MakeArgString(Twine(Enable ? "+" : "-") + Twine(Opt));
   };
@@ -801,7 +801,7 @@ StringRef HexagonToolChain::GetTargetCPUVersion(const ArgList &Args) {
     CpuArg = A;
 
   StringRef CPU = CpuArg ? CpuArg->getValue() : GetDefaultCPU();
-  if (CPU.startswith("hexagon"))
+  if (CPU.starts_with("hexagon"))
     return CPU.substr(sizeof("hexagon") - 1);
   return CPU;
 }
diff --git a/clang/lib/Driver/ToolChains/Hurd.cpp b/clang/lib/Driver/ToolChains/Hurd.cpp
index 2dfc90ef37f75d0e93e1ad77ffa514eac39e9732..7a4c2bb7ede1d5f1a9ba25ba93e7c6becdc599c5 100644
--- a/clang/lib/Driver/ToolChains/Hurd.cpp
+++ b/clang/lib/Driver/ToolChains/Hurd.cpp
@@ -92,7 +92,7 @@ Hurd::Hurd(const Driver &D, const llvm::Triple &Triple, const ArgList &Args)
   // those searched.
   // FIXME: It's not clear whether we should use the driver's installed
   // directory ('Dir' below) or the ResourceDir.
-  if (StringRef(D.Dir).startswith(SysRoot)) {
+  if (StringRef(D.Dir).starts_with(SysRoot)) {
     addPathIfExists(D, D.Dir + "/../lib/" + MultiarchTriple, Paths);
     addPathIfExists(D, D.Dir + "/../" + OSLibDir, Paths);
   }
@@ -110,7 +110,7 @@ Hurd::Hurd(const Driver &D, const llvm::Triple &Triple, const ArgList &Args)
   // searched.
   // FIXME: It's not clear whether we should use the driver's installed
   // directory ('Dir' below) or the ResourceDir.
-  if (StringRef(D.Dir).startswith(SysRoot))
+  if (StringRef(D.Dir).starts_with(SysRoot))
     addPathIfExists(D, D.Dir + "/../lib", Paths);
 
   addPathIfExists(D, SysRoot + "/lib", Paths);
diff --git a/clang/lib/Driver/ToolChains/MSP430.cpp b/clang/lib/Driver/ToolChains/MSP430.cpp
index b28d5e45706c9d6d7a382e951634e91f8ff7acec..8dc23521f400a09fb6dd288ff364ed332a4ef8d0 100644
--- a/clang/lib/Driver/ToolChains/MSP430.cpp
+++ b/clang/lib/Driver/ToolChains/MSP430.cpp
@@ -166,7 +166,7 @@ void MSP430ToolChain::addClangTargetOptions(const ArgList &DriverArgs,
     return;
 
   const StringRef MCU = MCUArg->getValue();
-  if (MCU.startswith("msp430i")) {
+  if (MCU.starts_with("msp430i")) {
     // 'i' should be in lower case as it's defined in TI MSP430-GCC headers
     CC1Args.push_back(DriverArgs.MakeArgString(
         "-D__MSP430i" + MCU.drop_front(7).upper() + "__"));
diff --git a/clang/lib/Driver/ToolChains/MSVC.cpp b/clang/lib/Driver/ToolChains/MSVC.cpp
index 6d925555b7bb4b2d462ef91ab5eb0f0974ab1a0c..8e1e95173836ffe2c3c97fc0b916c7cfa163d584 100644
--- a/clang/lib/Driver/ToolChains/MSVC.cpp
+++ b/clang/lib/Driver/ToolChains/MSVC.cpp
@@ -302,7 +302,7 @@ void visualstudio::Linker::ConstructJob(Compilation &C, const JobAction &JA,
     if (A.getOption().matches(options::OPT_l)) {
       StringRef Lib = A.getValue();
       const char *LinkLibArg;
-      if (Lib.endswith(".lib"))
+      if (Lib.ends_with(".lib"))
         LinkLibArg = Args.MakeArgString(Lib);
       else
         LinkLibArg = Args.MakeArgString(Lib + ".lib");
diff --git a/clang/lib/Driver/ToolChains/MinGW.cpp b/clang/lib/Driver/ToolChains/MinGW.cpp
index 5d7f8675daf8d28c35d62bfd3d9091ba8e0beaf6..65512f16357d044d4cbfcda328983bf24b50fe68 100644
--- a/clang/lib/Driver/ToolChains/MinGW.cpp
+++ b/clang/lib/Driver/ToolChains/MinGW.cpp
@@ -86,9 +86,9 @@ void tools::MinGW::Linker::AddLibGCC(const ArgList &Args,
   CmdArgs.push_back("-lmoldname");
   CmdArgs.push_back("-lmingwex");
   for (auto Lib : Args.getAllArgValues(options::OPT_l))
-    if (StringRef(Lib).startswith("msvcr") ||
-        StringRef(Lib).startswith("ucrt") ||
-        StringRef(Lib).startswith("crtdll"))
+    if (StringRef(Lib).starts_with("msvcr") ||
+        StringRef(Lib).starts_with("ucrt") ||
+        StringRef(Lib).starts_with("crtdll"))
       return;
   CmdArgs.push_back("-lmsvcrt");
 }
diff --git a/clang/lib/Driver/ToolChains/PPCLinux.cpp b/clang/lib/Driver/ToolChains/PPCLinux.cpp
index bdbecaef60405c24369f7f143cbf6b8776928583..0ed0f91ad166c0b1af2d7272a5ac515ce3e0d668 100644
--- a/clang/lib/Driver/ToolChains/PPCLinux.cpp
+++ b/clang/lib/Driver/ToolChains/PPCLinux.cpp
@@ -31,10 +31,10 @@ static bool GlibcSupportsFloat128(const std::string &Linker) {
 
   // Since glibc 2.34, the installed .so file is not symlink anymore. But we can
   // still safely assume it's newer than 2.32.
-  if (LinkerName.startswith("ld64.so"))
+  if (LinkerName.starts_with("ld64.so"))
     return true;
 
-  if (!LinkerName.startswith("ld-2."))
+  if (!LinkerName.starts_with("ld-2."))
     return false;
   unsigned Minor = (LinkerName[5] - '0') * 10 + (LinkerName[6] - '0');
   if (Minor < 32)
diff --git a/clang/lib/Driver/ToolChains/Solaris.cpp b/clang/lib/Driver/ToolChains/Solaris.cpp
index 485730da7df1f8cd4c8016a66edb061cc65a449c..9a9792d019d5edb2fef62359e9cd07d4ed3a9fd1 100644
--- a/clang/lib/Driver/ToolChains/Solaris.cpp
+++ b/clang/lib/Driver/ToolChains/Solaris.cpp
@@ -328,7 +328,7 @@ Solaris::Solaris(const Driver &D, const llvm::Triple &Triple,
 
   // If we are currently running Clang inside of the requested system root,
   // add its parent library path to those searched.
-  if (StringRef(D.Dir).startswith(D.SysRoot))
+  if (StringRef(D.Dir).starts_with(D.SysRoot))
     addPathIfExists(D, D.Dir + "/../lib", Paths);
 
   addPathIfExists(D, D.SysRoot + "/usr/lib" + LibSuffix, Paths);
diff --git a/clang/lib/Driver/ToolChains/WebAssembly.cpp b/clang/lib/Driver/ToolChains/WebAssembly.cpp
index f131b6cf3baff9d7ccfc2782d3ddfe6ea9b17b57..57f4600727ec89db91c8938bbdc19f3da845fdbc 100644
--- a/clang/lib/Driver/ToolChains/WebAssembly.cpp
+++ b/clang/lib/Driver/ToolChains/WebAssembly.cpp
@@ -328,7 +328,7 @@ void WebAssembly::addClangTargetOptions(const ArgList &DriverArgs,
 
   for (const Arg *A : DriverArgs.filtered(options::OPT_mllvm)) {
     StringRef Opt = A->getValue(0);
-    if (Opt.startswith("-emscripten-cxx-exceptions-allowed")) {
+    if (Opt.starts_with("-emscripten-cxx-exceptions-allowed")) {
       // '-mllvm -emscripten-cxx-exceptions-allowed' should be used with
       // '-mllvm -enable-emscripten-cxx-exceptions'
       bool EmEHArgExists = false;
@@ -355,7 +355,7 @@ void WebAssembly::addClangTargetOptions(const ArgList &DriverArgs,
       }
     }
 
-    if (Opt.startswith("-wasm-enable-sjlj")) {
+    if (Opt.starts_with("-wasm-enable-sjlj")) {
       // '-mllvm -wasm-enable-sjlj' is not compatible with
       // '-mno-exception-handling'
       if (DriverArgs.hasFlag(options::OPT_mno_exception_handing,
diff --git a/clang/lib/Edit/Commit.cpp b/clang/lib/Edit/Commit.cpp
index 7c5aea6e5069a504b5c9c0f30ef9ae866932b530..6e785e866666f161b4a2a656cbfec1564147d5a8 100644
--- a/clang/lib/Edit/Commit.cpp
+++ b/clang/lib/Edit/Commit.cpp
@@ -334,7 +334,7 @@ bool Commit::canReplaceText(SourceLocation loc, StringRef text,
     return false;
 
   Len = text.size();
-  return file.substr(Offs.getOffset()).startswith(text);
+  return file.substr(Offs.getOffset()).starts_with(text);
 }
 
 bool Commit::isAtStartOfMacroExpansion(SourceLocation loc,
diff --git a/clang/lib/Edit/RewriteObjCFoundationAPI.cpp b/clang/lib/Edit/RewriteObjCFoundationAPI.cpp
index adb34eba49703064b2748f3c8f444ecd71b04c1e..d5bf553e241240daf72277bb374b6af7cecd1381 100644
--- a/clang/lib/Edit/RewriteObjCFoundationAPI.cpp
+++ b/clang/lib/Edit/RewriteObjCFoundationAPI.cpp
@@ -697,7 +697,7 @@ static bool getLiteralInfo(SourceRange literalRange,
 
   struct Suff {
     static bool has(StringRef suff, StringRef &text) {
-      if (text.endswith(suff)) {
+      if (text.ends_with(suff)) {
         text = text.substr(0, text.size()-suff.size());
         return true;
       }
@@ -739,9 +739,9 @@ static bool getLiteralInfo(SourceRange literalRange,
   Info.F = UpperF ? "F" : "f";
 
   Info.Hex = Info.Octal = false;
-  if (text.startswith("0x"))
+  if (text.starts_with("0x"))
     Info.Hex = true;
-  else if (!isFloat && !isIntZero && text.startswith("0"))
+  else if (!isFloat && !isIntZero && text.starts_with("0"))
     Info.Octal = true;
 
   SourceLocation B = literalRange.getBegin();
diff --git a/clang/lib/ExtractAPI/ExtractAPIConsumer.cpp b/clang/lib/ExtractAPI/ExtractAPIConsumer.cpp
index fe282dfb19e8aa7fa9bf6ff43ee2c18b309864a7..fd62d841197d9f08e02ab3c4d906e703e19ec289 100644
--- a/clang/lib/ExtractAPI/ExtractAPIConsumer.cpp
+++ b/clang/lib/ExtractAPI/ExtractAPIConsumer.cpp
@@ -105,10 +105,10 @@ std::optional<std::string> getRelativeIncludeName(const CompilerInstance &CI,
       // Special case Apple .sdk folders since the search path is typically a
       // symlink like `iPhoneSimulator14.5.sdk` while the file is instead
       // located in `iPhoneSimulator.sdk` (the real folder).
-      if (NI->endswith(".sdk") && DI->endswith(".sdk")) {
+      if (NI->ends_with(".sdk") && DI->ends_with(".sdk")) {
         StringRef NBasename = path::stem(*NI);
         StringRef DBasename = path::stem(*DI);
-        if (DBasename.startswith(NBasename))
+        if (DBasename.starts_with(NBasename))
           continue;
       }
 
diff --git a/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp b/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp
index d9675b0c94de3962fa185b87f9819bb2d844b268..53b22297ee0ea19e4e23b504baa42ea899b6bf8c 100644
--- a/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp
+++ b/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp
@@ -743,7 +743,7 @@ bool SymbolGraphSerializer::shouldSkip(const APIRecord &Record) const {
 
   // Filter out symbols prefixed with an underscored as they are understood to
   // be symbols clients should not use.
-  if (Record.Name.startswith("_"))
+  if (Record.Name.starts_with("_"))
     return true;
 
   return false;
diff --git a/clang/lib/Format/BreakableToken.cpp b/clang/lib/Format/BreakableToken.cpp
index 954eeb9a6f24fc4a73a98d4d3290a274fc379cbb..473908e8fee3b3da6557e77d3c2dc6be8931e209 100644
--- a/clang/lib/Format/BreakableToken.cpp
+++ b/clang/lib/Format/BreakableToken.cpp
@@ -55,7 +55,7 @@ static StringRef getLineCommentIndentPrefix(StringRef Comment,
       }));
 
   for (StringRef KnownPrefix : KnownPrefixes) {
-    if (Comment.startswith(KnownPrefix)) {
+    if (Comment.starts_with(KnownPrefix)) {
       const auto PrefixLength =
           Comment.find_first_not_of(' ', KnownPrefix.size());
       return Comment.substr(0, PrefixLength);
@@ -220,8 +220,8 @@ bool switchesFormatting(const FormatToken &Token) {
   assert((Token.is(TT_BlockComment) || Token.is(TT_LineComment)) &&
          "formatting regions are switched by comment tokens");
   StringRef Content = Token.TokenText.substr(2).ltrim();
-  return Content.startswith("clang-format on") ||
-         Content.startswith("clang-format off");
+  return Content.starts_with("clang-format on") ||
+         Content.starts_with("clang-format off");
 }
 
 unsigned
@@ -271,7 +271,7 @@ BreakableStringLiteral::BreakableStringLiteral(
     : BreakableToken(Tok, InPPDirective, Encoding, Style),
       StartColumn(StartColumn), Prefix(Prefix), Postfix(Postfix),
       UnbreakableTailLength(UnbreakableTailLength) {
-  assert(Tok.TokenText.startswith(Prefix) && Tok.TokenText.endswith(Postfix));
+  assert(Tok.TokenText.starts_with(Prefix) && Tok.TokenText.ends_with(Postfix));
   Line = Tok.TokenText.substr(
       Prefix.size(), Tok.TokenText.size() - Prefix.size() - Postfix.size());
 }
@@ -454,7 +454,7 @@ static bool mayReflowContent(StringRef Content) {
   bool hasSpecialMeaningPrefix = false;
   for (StringRef Prefix :
        {"@", "TODO", "FIXME", "XXX", "-# ", "- ", "+ ", "* "}) {
-    if (Content.startswith(Prefix)) {
+    if (Content.starts_with(Prefix)) {
       hasSpecialMeaningPrefix = true;
       break;
     }
@@ -471,7 +471,7 @@ static bool mayReflowContent(StringRef Content) {
   // characters and either the first or second character must be
   // non-punctuation.
   return Content.size() >= 2 && !hasSpecialMeaningPrefix &&
-         !Content.endswith("\\") &&
+         !Content.ends_with("\\") &&
          // Note that this is UTF-8 safe, since if isPunctuation(Content[0]) is
          // true, then the first code point must be 1 byte long.
          (!isPunctuation(Content[0]) || !isPunctuation(Content[1]));
@@ -488,7 +488,7 @@ BreakableBlockComment::BreakableBlockComment(
          "block comment section must start with a block comment");
 
   StringRef TokenText(Tok.TokenText);
-  assert(TokenText.startswith("/*") && TokenText.endswith("*/"));
+  assert(TokenText.starts_with("/*") && TokenText.ends_with("*/"));
   TokenText.substr(2, TokenText.size() - 4)
       .split(Lines, UseCRLF ? "\r\n" : "\n");
 
@@ -511,7 +511,7 @@ BreakableBlockComment::BreakableBlockComment(
   // /*
   // ** blah blah blah
   // */
-  if (Lines.size() >= 2 && Content[1].startswith("**") &&
+  if (Lines.size() >= 2 && Content[1].starts_with("**") &&
       static_cast<unsigned>(ContentColumn[1]) == StartColumn) {
     DecorationColumn = StartColumn;
   }
@@ -531,10 +531,10 @@ BreakableBlockComment::BreakableBlockComment(
       // If the last line is empty, the closing "*/" will have a star.
       if (Text.empty())
         break;
-    } else if (!Text.empty() && Decoration.startswith(Text)) {
+    } else if (!Text.empty() && Decoration.starts_with(Text)) {
       continue;
     }
-    while (!Text.startswith(Decoration))
+    while (!Text.starts_with(Decoration))
       Decoration = Decoration.drop_back(1);
   }
 
@@ -562,13 +562,13 @@ BreakableBlockComment::BreakableBlockComment(
     // The last line excludes the star if LastLineNeedsDecoration is false.
     // For all other lines, adjust the line to exclude the star and
     // (optionally) the first whitespace.
-    unsigned DecorationSize = Decoration.startswith(Content[i])
+    unsigned DecorationSize = Decoration.starts_with(Content[i])
                                   ? Content[i].size()
                                   : Decoration.size();
     if (DecorationSize)
       ContentColumn[i] = DecorationColumn + DecorationSize;
     Content[i] = Content[i].substr(DecorationSize);
-    if (!Decoration.startswith(Content[i])) {
+    if (!Decoration.starts_with(Content[i])) {
       IndentAtLineBreak =
           std::min<int>(IndentAtLineBreak, std::max(0, ContentColumn[i]));
     }
@@ -577,10 +577,10 @@ BreakableBlockComment::BreakableBlockComment(
 
   // Detect a multiline jsdoc comment and set DelimitersOnNewline in that case.
   if (Style.isJavaScript() || Style.Language == FormatStyle::LK_Java) {
-    if ((Lines[0] == "*" || Lines[0].startswith("* ")) && Lines.size() > 1) {
+    if ((Lines[0] == "*" || Lines[0].starts_with("* ")) && Lines.size() > 1) {
       // This is a multiline jsdoc comment.
       DelimitersOnNewline = true;
-    } else if (Lines[0].startswith("* ") && Lines.size() == 1) {
+    } else if (Lines[0].starts_with("* ") && Lines.size() == 1) {
       // Detect a long single-line comment, like:
       // /** long long long */
       // Below, '2' is the width of '*/'.
@@ -612,7 +612,7 @@ BreakableToken::Split BreakableBlockComment::getSplit(
     return Split(StringRef::npos, 0);
   return getCommentSplit(Content[LineIndex].substr(TailOffset),
                          ContentStartColumn, ColumnLimit, Style.TabWidth,
-                         Encoding, Style, Decoration.endswith("*"));
+                         Encoding, Style, Decoration.ends_with("*"));
 }
 
 void BreakableBlockComment::adjustWhitespace(unsigned LineIndex,
@@ -623,7 +623,7 @@ void BreakableBlockComment::adjustWhitespace(unsigned LineIndex,
   // trimming the trailing whitespace. The backslash will be re-added later when
   // inserting a line break.
   size_t EndOfPreviousLine = Lines[LineIndex - 1].size();
-  if (InPPDirective && Lines[LineIndex - 1].endswith("\\"))
+  if (InPPDirective && Lines[LineIndex - 1].ends_with("\\"))
     --EndOfPreviousLine;
 
   // Calculate the end of the non-whitespace text in the previous line.
@@ -672,7 +672,7 @@ unsigned BreakableBlockComment::getRemainingLength(unsigned LineIndex,
     // We never need a decoration when breaking just the trailing "*/" postfix.
     bool HasRemainingText = Offset < Content[LineIndex].size();
     if (!HasRemainingText) {
-      bool HasDecoration = Lines[LineIndex].ltrim().startswith(Decoration);
+      bool HasDecoration = Lines[LineIndex].ltrim().starts_with(Decoration);
       if (HasDecoration)
         LineLength -= Decoration.size();
     }
@@ -700,7 +700,7 @@ unsigned BreakableBlockComment::getContentIndent(unsigned LineIndex) const {
   // /** line 0 */
   // is "* line 0", so we need to skip over the decoration in that case.
   StringRef ContentWithNoDecoration = Content[LineIndex];
-  if (LineIndex == 0 && ContentWithNoDecoration.startswith("*"))
+  if (LineIndex == 0 && ContentWithNoDecoration.starts_with("*"))
     ContentWithNoDecoration = ContentWithNoDecoration.substr(1).ltrim(Blanks);
   StringRef FirstWord = ContentWithNoDecoration.substr(
       0, ContentWithNoDecoration.find_first_of(Blanks));
@@ -853,7 +853,7 @@ bool BreakableBlockComment::mayReflow(
   // Content[LineIndex] may exclude the indent after the '*' decoration. In that
   // case, we compute the start of the comment pragma manually.
   StringRef IndentContent = Content[LineIndex];
-  if (Lines[LineIndex].ltrim(Blanks).startswith("*"))
+  if (Lines[LineIndex].ltrim(Blanks).starts_with("*"))
     IndentContent = Lines[LineIndex].ltrim(Blanks).substr(1);
   return LineIndex > 0 && !CommentPragmasRegex.match(IndentContent) &&
          mayReflowContent(Content[LineIndex]) && !Tok.Finalized &&
@@ -876,7 +876,7 @@ BreakableLineCommentSection::BreakableLineCommentSection(
        CurrentTok = CurrentTok->Next) {
     LastLineTok = LineTok;
     StringRef TokenText(CurrentTok->TokenText);
-    assert((TokenText.startswith("//") || TokenText.startswith("#")) &&
+    assert((TokenText.starts_with("//") || TokenText.starts_with("#")) &&
            "unsupported line comment prefix, '//' and '#' are supported");
     size_t FirstLineIndex = Lines.size();
     TokenText.split(Lines, "\n");
@@ -913,7 +913,7 @@ BreakableLineCommentSection::BreakableLineCommentSection(
         // #########
         // # section
         // #########
-        if (FirstCommentChar == '#' && !TokenText.startswith("#"))
+        if (FirstCommentChar == '#' && !TokenText.starts_with("#"))
           return false;
         return FirstCommentChar == '\\' || isPunctuation(FirstCommentChar) ||
                isHorizontalWhitespace(FirstCommentChar);
@@ -1152,7 +1152,7 @@ bool BreakableLineCommentSection::mayReflow(
   // Line comments have the indent as part of the prefix, so we need to
   // recompute the start of the line.
   StringRef IndentContent = Content[LineIndex];
-  if (Lines[LineIndex].startswith("//"))
+  if (Lines[LineIndex].starts_with("//"))
     IndentContent = Lines[LineIndex].substr(2);
   // FIXME: Decide whether we want to reflow non-regular indents:
   // Currently, we only reflow when the OriginalPrefix[LineIndex] matches the
diff --git a/clang/lib/Format/ContinuationIndenter.cpp b/clang/lib/Format/ContinuationIndenter.cpp
index 9e4e939503dfe4a0b5dd501e14353cdb4133c664..bd319f21b05f86415bd4677cc81d2fb5c42a8523 100644
--- a/clang/lib/Format/ContinuationIndenter.cpp
+++ b/clang/lib/Format/ContinuationIndenter.cpp
@@ -161,7 +161,7 @@ static bool opensProtoMessageField(const FormatToken &LessTok,
 // string.  For example, the delimiter of R"deli(cont)deli" is deli.
 static std::optional<StringRef> getRawStringDelimiter(StringRef TokenText) {
   if (TokenText.size() < 5 // The smallest raw string possible is 'R"()"'.
-      || !TokenText.startswith("R\"") || !TokenText.endswith("\"")) {
+      || !TokenText.starts_with("R\"") || !TokenText.ends_with("\"")) {
     return std::nullopt;
   }
 
@@ -177,7 +177,7 @@ static std::optional<StringRef> getRawStringDelimiter(StringRef TokenText) {
   size_t RParenPos = TokenText.size() - Delimiter.size() - 2;
   if (TokenText[RParenPos] != ')')
     return std::nullopt;
-  if (!TokenText.substr(RParenPos + 1).startswith(Delimiter))
+  if (!TokenText.substr(RParenPos + 1).starts_with(Delimiter))
     return std::nullopt;
   return Delimiter;
 }
@@ -608,7 +608,7 @@ bool ContinuationIndenter::mustBreak(const LineState &State) {
 
   if (Current.is(tok::lessless) &&
       ((Previous.is(tok::identifier) && Previous.TokenText == "endl") ||
-       (Previous.Tok.isLiteral() && (Previous.TokenText.endswith("\\n\"") ||
+       (Previous.Tok.isLiteral() && (Previous.TokenText.ends_with("\\n\"") ||
                                      Previous.TokenText == "\'\\n\'")))) {
     return true;
   }
@@ -2293,12 +2293,13 @@ ContinuationIndenter::createBreakableToken(const FormatToken &Current,
     if (Style.isVerilog() || Style.Language == FormatStyle::LK_Java ||
         Style.isJavaScript() || Style.isCSharp()) {
       BreakableStringLiteralUsingOperators::QuoteStyleType QuoteStyle;
-      if (Style.isJavaScript() && Text.startswith("'") && Text.endswith("'")) {
+      if (Style.isJavaScript() && Text.starts_with("'") &&
+          Text.ends_with("'")) {
         QuoteStyle = BreakableStringLiteralUsingOperators::SingleQuotes;
-      } else if (Style.isCSharp() && Text.startswith("@\"") &&
-                 Text.endswith("\"")) {
+      } else if (Style.isCSharp() && Text.starts_with("@\"") &&
+                 Text.ends_with("\"")) {
         QuoteStyle = BreakableStringLiteralUsingOperators::AtDoubleQuotes;
-      } else if (Text.startswith("\"") && Text.endswith("\"")) {
+      } else if (Text.starts_with("\"") && Text.ends_with("\"")) {
         QuoteStyle = BreakableStringLiteralUsingOperators::DoubleQuotes;
       } else {
         return nullptr;
@@ -2315,12 +2316,14 @@ ContinuationIndenter::createBreakableToken(const FormatToken &Current,
     // FIXME: Store Prefix and Suffix (or PrefixLength and SuffixLength to
     // reduce the overhead) for each FormatToken, which is a string, so that we
     // don't run multiple checks here on the hot path.
-    if ((Text.endswith(Postfix = "\"") &&
-         (Text.startswith(Prefix = "@\"") || Text.startswith(Prefix = "\"") ||
-          Text.startswith(Prefix = "u\"") || Text.startswith(Prefix = "U\"") ||
-          Text.startswith(Prefix = "u8\"") ||
-          Text.startswith(Prefix = "L\""))) ||
-        (Text.startswith(Prefix = "_T(\"") && Text.endswith(Postfix = "\")"))) {
+    if ((Text.ends_with(Postfix = "\"") &&
+         (Text.starts_with(Prefix = "@\"") || Text.starts_with(Prefix = "\"") ||
+          Text.starts_with(Prefix = "u\"") ||
+          Text.starts_with(Prefix = "U\"") ||
+          Text.starts_with(Prefix = "u8\"") ||
+          Text.starts_with(Prefix = "L\""))) ||
+        (Text.starts_with(Prefix = "_T(\"") &&
+         Text.ends_with(Postfix = "\")"))) {
       return std::make_unique<BreakableStringLiteral>(
           Current, StartColumn, Prefix, Postfix, UnbreakableTailLength,
           State.Line->InPPDirective, Encoding, Style);
@@ -2342,7 +2345,7 @@ ContinuationIndenter::createBreakableToken(const FormatToken &Current,
     bool RegularComments = [&]() {
       for (const FormatToken *T = &Current; T && T->is(TT_LineComment);
            T = T->Next) {
-        if (!(T->TokenText.startswith("//") || T->TokenText.startswith("#")))
+        if (!(T->TokenText.starts_with("//") || T->TokenText.starts_with("#")))
           return false;
       }
       return true;
@@ -2754,7 +2757,7 @@ bool ContinuationIndenter::nextIsMultilineString(const LineState &State) {
   // We never consider raw string literals "multiline" for the purpose of
   // AlwaysBreakBeforeMultilineStrings implementation as they are special-cased
   // (see TokenAnnotator::mustBreakBefore().
-  if (Current.TokenText.startswith("R\""))
+  if (Current.TokenText.starts_with("R\""))
     return false;
   if (Current.IsMultiline)
     return true;
diff --git a/clang/lib/Format/Format.cpp b/clang/lib/Format/Format.cpp
index 8feee7457fc31b76a09f9c9ebd9a9104db281404..668e959a9416bad1274f262ba726c389f8635449 100644
--- a/clang/lib/Format/Format.cpp
+++ b/clang/lib/Format/Format.cpp
@@ -2346,9 +2346,9 @@ private:
             // NB: testing for not starting with a double quote to avoid
             // breaking `template strings`.
             (Style.JavaScriptQuotes == FormatStyle::JSQS_Single &&
-             !Input.startswith("\"")) ||
+             !Input.starts_with("\"")) ||
             (Style.JavaScriptQuotes == FormatStyle::JSQS_Double &&
-             !Input.startswith("\'"))) {
+             !Input.starts_with("\'"))) {
           continue;
         }
 
@@ -2932,7 +2932,7 @@ private:
     };
 
     for (auto *Line : AnnotatedLines) {
-      if (Line->First && (Line->First->TokenText.startswith("#") ||
+      if (Line->First && (Line->First->TokenText.starts_with("#") ||
                           Line->First->TokenText == "__pragma" ||
                           Line->First->TokenText == "_Pragma")) {
         continue;
@@ -3217,7 +3217,7 @@ tooling::Replacements sortCppIncludes(const FormatStyle &Style, StringRef Code,
          Style.IncludeStyle.IncludeBlocks ==
              tooling::IncludeStyle::IBS_Regroup);
 
-    bool MergeWithNextLine = Trimmed.endswith("\\");
+    bool MergeWithNextLine = Trimmed.ends_with("\\");
     if (!FormattingOff && !MergeWithNextLine) {
       if (tooling::HeaderIncludes::IncludeRegex.match(Line, &Matches)) {
         StringRef IncludeName = Matches[2];
@@ -3243,7 +3243,7 @@ tooling::Replacements sortCppIncludes(const FormatStyle &Style, StringRef Code,
         sortCppIncludes(Style, IncludesInBlock, Ranges, FileName, Code,
                         Replaces, Cursor);
         IncludesInBlock.clear();
-        if (Trimmed.startswith("#pragma hdrstop")) // Precompiled headers.
+        if (Trimmed.starts_with("#pragma hdrstop")) // Precompiled headers.
           FirstIncludeBlock = true;
         else
           FirstIncludeBlock = false;
@@ -3271,7 +3271,7 @@ static unsigned findJavaImportGroup(const FormatStyle &Style,
   unsigned LongestMatchLength = 0;
   for (unsigned I = 0; I < Style.JavaImportGroups.size(); I++) {
     const std::string &GroupPrefix = Style.JavaImportGroups[I];
-    if (ImportIdentifier.startswith(GroupPrefix) &&
+    if (ImportIdentifier.starts_with(GroupPrefix) &&
         GroupPrefix.length() > LongestMatchLength) {
       LongestMatchIndex = I;
       LongestMatchLength = GroupPrefix.length();
@@ -3426,7 +3426,7 @@ bool isMpegTS(StringRef Code) {
   return Code.size() > 188 && Code[0] == 0x47 && Code[188] == 0x47;
 }
 
-bool isLikelyXml(StringRef Code) { return Code.ltrim().startswith("<"); }
+bool isLikelyXml(StringRef Code) { return Code.ltrim().starts_with("<"); }
 
 tooling::Replacements sortIncludes(const FormatStyle &Style, StringRef Code,
                                    ArrayRef<tooling::Range> Ranges,
@@ -3538,7 +3538,7 @@ fixCppIncludeInsertions(StringRef Code, const tooling::Replacements &Replaces,
 
   for (const auto &Header : HeadersToDelete) {
     tooling::Replacements Replaces =
-        Includes.remove(Header.trim("\"<>"), Header.startswith("<"));
+        Includes.remove(Header.trim("\"<>"), Header.starts_with("<"));
     for (const auto &R : Replaces) {
       auto Err = Result.add(R);
       if (Err) {
@@ -3560,7 +3560,7 @@ fixCppIncludeInsertions(StringRef Code, const tooling::Replacements &Replaces,
     (void)Matched;
     auto IncludeName = Matches[2];
     auto Replace =
-        Includes.insert(IncludeName.trim("\"<>"), IncludeName.startswith("<"),
+        Includes.insert(IncludeName.trim("\"<>"), IncludeName.starts_with("<"),
                         tooling::IncludeDirective::Include);
     if (Replace) {
       auto Err = Result.add(*Replace);
@@ -3882,14 +3882,14 @@ const char *StyleOptionHelpDescription =
     "   --style=\"{BasedOnStyle: llvm, IndentWidth: 8}\"";
 
 static FormatStyle::LanguageKind getLanguageByFileName(StringRef FileName) {
-  if (FileName.endswith(".java"))
+  if (FileName.ends_with(".java"))
     return FormatStyle::LK_Java;
   if (FileName.ends_with_insensitive(".js") ||
       FileName.ends_with_insensitive(".mjs") ||
       FileName.ends_with_insensitive(".ts")) {
     return FormatStyle::LK_JavaScript; // (module) JavaScript or TypeScript.
   }
-  if (FileName.endswith(".m") || FileName.endswith(".mm"))
+  if (FileName.ends_with(".m") || FileName.ends_with(".mm"))
     return FormatStyle::LK_ObjC;
   if (FileName.ends_with_insensitive(".proto") ||
       FileName.ends_with_insensitive(".protodevel")) {
@@ -3963,7 +3963,7 @@ llvm::Expected<FormatStyle> getStyle(StringRef StyleName, StringRef FileName,
   llvm::SmallVector<std::unique_ptr<llvm::MemoryBuffer>, 1>
       ChildFormatTextToApply;
 
-  if (StyleName.startswith("{")) {
+  if (StyleName.starts_with("{")) {
     // Parse YAML/JSON style from the command line.
     StringRef Source = "<command-line>";
     if (std::error_code ec =
@@ -4123,7 +4123,7 @@ static bool isClangFormatOnOff(StringRef Comment, bool On) {
   static const char ClangFormatOff[] = "// clang-format off";
   const unsigned Size = (On ? sizeof ClangFormatOn : sizeof ClangFormatOff) - 1;
 
-  return Comment.startswith(On ? ClangFormatOn : ClangFormatOff) &&
+  return Comment.starts_with(On ? ClangFormatOn : ClangFormatOff) &&
          (Comment.size() == Size || Comment[Size] == ':');
 }
 
diff --git a/clang/lib/Format/FormatToken.h b/clang/lib/Format/FormatToken.h
index 14a3c21ba44eaee9c77da15e37cd914a32518c43..3f9664f8f78a3e4d3329b04cb53ff5d776bc1cd8 100644
--- a/clang/lib/Format/FormatToken.h
+++ b/clang/lib/Format/FormatToken.h
@@ -667,7 +667,7 @@ public:
   /// Returns whether \p Tok is ([{ or an opening < of a template or in
   /// protos.
   bool opensScope() const {
-    if (is(TT_TemplateString) && TokenText.endswith("${"))
+    if (is(TT_TemplateString) && TokenText.ends_with("${"))
       return true;
     if (is(TT_DictLiteral) && is(tok::less))
       return true;
@@ -677,7 +677,7 @@ public:
   /// Returns whether \p Tok is )]} or a closing > of a template or in
   /// protos.
   bool closesScope() const {
-    if (is(TT_TemplateString) && TokenText.startswith("}"))
+    if (is(TT_TemplateString) && TokenText.starts_with("}"))
       return true;
     if (is(TT_DictLiteral) && is(tok::greater))
       return true;
@@ -743,9 +743,9 @@ public:
     if (isNot(tok::string_literal))
       return false;
     StringRef Content = TokenText;
-    if (Content.startswith("\"") || Content.startswith("'"))
+    if (Content.starts_with("\"") || Content.starts_with("'"))
       Content = Content.drop_front(1);
-    if (Content.endswith("\"") || Content.endswith("'"))
+    if (Content.ends_with("\"") || Content.ends_with("'"))
       Content = Content.drop_back(1);
     Content = Content.trim();
     return Content.size() > 1 &&
@@ -1823,7 +1823,7 @@ private:
 };
 
 inline bool isLineComment(const FormatToken &FormatTok) {
-  return FormatTok.is(tok::comment) && !FormatTok.TokenText.startswith("/*");
+  return FormatTok.is(tok::comment) && !FormatTok.TokenText.starts_with("/*");
 }
 
 // Checks if \p FormatTok is a line comment that continues the line comment
diff --git a/clang/lib/Format/FormatTokenLexer.cpp b/clang/lib/Format/FormatTokenLexer.cpp
index e4e32e2671df57d925ab5d18af39419f7b0660eb..61430282c6f88ca09e7e2b14c25103d66a9469c2 100644
--- a/clang/lib/Format/FormatTokenLexer.cpp
+++ b/clang/lib/Format/FormatTokenLexer.cpp
@@ -711,12 +711,12 @@ void FormatTokenLexer::handleCSharpVerbatimAndInterpolatedStrings() {
 
   bool Verbatim = false;
   bool Interpolated = false;
-  if (TokenText.startswith(R"($@")") || TokenText.startswith(R"(@$")")) {
+  if (TokenText.starts_with(R"($@")") || TokenText.starts_with(R"(@$")")) {
     Verbatim = true;
     Interpolated = true;
-  } else if (TokenText.startswith(R"(@")")) {
+  } else if (TokenText.starts_with(R"(@")")) {
     Verbatim = true;
-  } else if (TokenText.startswith(R"($")")) {
+  } else if (TokenText.starts_with(R"($")")) {
     Interpolated = true;
   }
 
@@ -1110,7 +1110,7 @@ FormatToken *FormatTokenLexer::getNextToken() {
   // the comment token at the backslash, and resets the lexer to restart behind
   // the backslash.
   if ((Style.isJavaScript() || Style.Language == FormatStyle::LK_Java) &&
-      FormatTok->is(tok::comment) && FormatTok->TokenText.startswith("//")) {
+      FormatTok->is(tok::comment) && FormatTok->TokenText.starts_with("//")) {
     size_t BackslashPos = FormatTok->TokenText.find('\\');
     while (BackslashPos != StringRef::npos) {
       if (BackslashPos + 1 < FormatTok->TokenText.size() &&
diff --git a/clang/lib/Format/SortJavaScriptImports.cpp b/clang/lib/Format/SortJavaScriptImports.cpp
index 8c6722e915344fb233b78bbde6f85ee2ca4c074d..1a6a1b19e7022a080460b1dfd0a0cd152673a4bf 100644
--- a/clang/lib/Format/SortJavaScriptImports.cpp
+++ b/clang/lib/Format/SortJavaScriptImports.cpp
@@ -468,10 +468,10 @@ private:
       // URL = TokenText without the quotes.
       Reference.URL =
           Current->TokenText.substr(1, Current->TokenText.size() - 2);
-      if (Reference.URL.startswith("..")) {
+      if (Reference.URL.starts_with("..")) {
         Reference.Category =
             JsModuleReference::ReferenceCategory::RELATIVE_PARENT;
-      } else if (Reference.URL.startswith(".")) {
+      } else if (Reference.URL.starts_with(".")) {
         Reference.Category = JsModuleReference::ReferenceCategory::RELATIVE;
       } else {
         Reference.Category = JsModuleReference::ReferenceCategory::ABSOLUTE;
diff --git a/clang/lib/Format/TokenAnnotator.cpp b/clang/lib/Format/TokenAnnotator.cpp
index eaccb5881ca30f5e381f20b1e17d36894041985d..f3551af3424396967c2ec5ce1250d7d4911fd63a 100644
--- a/clang/lib/Format/TokenAnnotator.cpp
+++ b/clang/lib/Format/TokenAnnotator.cpp
@@ -164,6 +164,10 @@ private:
                TT_OverloadedOperatorLParen))) {
         return false;
       }
+      if (Previous.Previous->is(tok::kw_operator) &&
+          CurrentToken->is(tok::l_paren)) {
+        return false;
+      }
     }
 
     FormatToken *Left = CurrentToken->Previous;
@@ -1307,7 +1311,7 @@ private:
         if (Previous->isOneOf(TT_BinaryOperator, TT_UnaryOperator, tok::comma,
                               tok::star, tok::arrow, tok::amp, tok::ampamp) ||
             // User defined literal.
-            Previous->TokenText.startswith("\"\"")) {
+            Previous->TokenText.starts_with("\"\"")) {
           Previous->setType(TT_OverloadedOperator);
           if (CurrentToken->isOneOf(tok::less, tok::greater))
             break;
@@ -1466,7 +1470,7 @@ private:
         // Mark tokens up to the trailing line comments as implicit string
         // literals.
         if (CurrentToken->isNot(tok::comment) &&
-            !CurrentToken->TokenText.startswith("//")) {
+            !CurrentToken->TokenText.starts_with("//")) {
           CurrentToken->setType(TT_ImplicitStringLiteral);
         }
         next();
@@ -2077,8 +2081,8 @@ private:
       }
       Current.setType(TT_BinaryOperator);
     } else if (Current.is(tok::comment)) {
-      if (Current.TokenText.startswith("/*")) {
-        if (Current.TokenText.endswith("*/")) {
+      if (Current.TokenText.starts_with("/*")) {
+        if (Current.TokenText.ends_with("*/")) {
           Current.setType(TT_BlockComment);
         } else {
           // The lexer has for some reason determined a comment here. But we
@@ -3724,8 +3728,8 @@ unsigned TokenAnnotator::splitPenalty(const AnnotatedLine &Line,
       return 100;
     if (Left.is(TT_JsTypeColon))
       return 35;
-    if ((Left.is(TT_TemplateString) && Left.TokenText.endswith("${")) ||
-        (Right.is(TT_TemplateString) && Right.TokenText.startswith("}"))) {
+    if ((Left.is(TT_TemplateString) && Left.TokenText.ends_with("${")) ||
+        (Right.is(TT_TemplateString) && Right.TokenText.starts_with("}"))) {
       return 100;
     }
     // Prefer breaking call chains (".foo") over empty "{}", "[]" or "()".
@@ -4224,7 +4228,7 @@ bool TokenAnnotator::spaceRequiredBetween(const AnnotatedLine &Line,
   }
   if (Left.is(TT_BlockComment)) {
     // No whitespace in x(/*foo=*/1), except for JavaScript.
-    return Style.isJavaScript() || !Left.TokenText.endswith("=*/");
+    return Style.isJavaScript() || !Left.TokenText.ends_with("=*/");
   }
 
   // Space between template and attribute.
@@ -4572,8 +4576,8 @@ bool TokenAnnotator::spaceRequiredBefore(const AnnotatedLine &Line,
       if (Next && Next->is(TT_FatArrow))
         return true;
     }
-    if ((Left.is(TT_TemplateString) && Left.TokenText.endswith("${")) ||
-        (Right.is(TT_TemplateString) && Right.TokenText.startswith("}"))) {
+    if ((Left.is(TT_TemplateString) && Left.TokenText.ends_with("${")) ||
+        (Right.is(TT_TemplateString) && Right.TokenText.starts_with("}"))) {
       return false;
     }
     // In tagged template literals ("html`bar baz`"), there is no space between
@@ -5212,7 +5216,7 @@ bool TokenAnnotator::mustBreakBefore(const AnnotatedLine &Line,
       Left.is(TT_InheritanceComma)) {
     return true;
   }
-  if (Right.is(tok::string_literal) && Right.TokenText.startswith("R\"")) {
+  if (Right.is(tok::string_literal) && Right.TokenText.starts_with("R\"")) {
     // Multiline raw string literals are special wrt. line breaks. The author
     // has made a deliberate choice and might have aligned the contents of the
     // string literal accordingly. Thus, we try keep existing line breaks.
diff --git a/clang/lib/Format/UnwrappedLineParser.cpp b/clang/lib/Format/UnwrappedLineParser.cpp
index 57126b8dfeac5de62da5b519c1ad0e0b3b5cdc62..c38b4c884070bb74ccad00dcf8a2539a3653c30e 100644
--- a/clang/lib/Format/UnwrappedLineParser.cpp
+++ b/clang/lib/Format/UnwrappedLineParser.cpp
@@ -1333,7 +1333,7 @@ bool UnwrappedLineParser::parseModuleImport() {
         // Mark tokens up to the trailing line comments as implicit string
         // literals.
         if (FormatTok->isNot(tok::comment) &&
-            !FormatTok->TokenText.startswith("//")) {
+            !FormatTok->TokenText.starts_with("//")) {
           FormatTok->setFinalizedType(TT_ImplicitStringLiteral);
         }
         nextToken();
@@ -1371,7 +1371,7 @@ void UnwrappedLineParser::readTokenWithJavaScriptASI() {
 
   bool PreviousMustBeValue = mustBeJSIdentOrValue(Keywords, Previous);
   bool PreviousStartsTemplateExpr =
-      Previous->is(TT_TemplateString) && Previous->TokenText.endswith("${");
+      Previous->is(TT_TemplateString) && Previous->TokenText.ends_with("${");
   if (PreviousMustBeValue || Previous->is(tok::r_paren)) {
     // If the line contains an '@' sign, the previous token might be an
     // annotation, which can precede another identifier/value.
@@ -1385,7 +1385,7 @@ void UnwrappedLineParser::readTokenWithJavaScriptASI() {
     return addUnwrappedLine();
   bool NextMustBeValue = mustBeJSIdentOrValue(Keywords, Next);
   bool NextEndsTemplateExpr =
-      Next->is(TT_TemplateString) && Next->TokenText.startswith("}");
+      Next->is(TT_TemplateString) && Next->TokenText.starts_with("}");
   if (NextMustBeValue && !NextEndsTemplateExpr && !PreviousStartsTemplateExpr &&
       (PreviousMustBeValue ||
        Previous->isOneOf(tok::r_square, tok::r_paren, tok::plusplus,
@@ -4459,8 +4459,8 @@ continuesLineCommentSection(const FormatToken &FormatTok,
     return false;
 
   StringRef IndentContent = FormatTok.TokenText;
-  if (FormatTok.TokenText.startswith("//") ||
-      FormatTok.TokenText.startswith("/*")) {
+  if (FormatTok.TokenText.starts_with("//") ||
+      FormatTok.TokenText.starts_with("/*")) {
     IndentContent = FormatTok.TokenText.substr(2);
   }
   if (CommentPragmasRegex.match(IndentContent))
diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp
index b33bdad2ad81bad041c9ca791066ac347706d1e7..11f3f2c2d6425cce7f51ca5c79b28b6ac79646fd 100644
--- a/clang/lib/Frontend/CompilerInvocation.cpp
+++ b/clang/lib/Frontend/CompilerInvocation.cpp
@@ -2829,7 +2829,7 @@ static bool ParseFrontendArgs(FrontendOptions &Opts, ArgList &Args,
                "-interface-stub-version=ifs-v1"
             << ErrorMessage;
         ProgramAction = frontend::ParseSyntaxOnly;
-      } else if (!ArgStr.startswith("ifs-")) {
+      } else if (!ArgStr.starts_with("ifs-")) {
         std::string ErrorMessage =
             "Invalid interface stub format: " + ArgStr.str() + ".";
         Diags.Report(diag::err_drv_invalid_value)
@@ -4106,13 +4106,13 @@ bool CompilerInvocation::ParseLangArgs(LangOptions &Opts, ArgList &Args,
 
     // Check the version number is valid: either 3.x (0 <= x <= 9) or
     // y or y.0 (4 <= y <= current version).
-    if (!VerParts.first.startswith("0") &&
-        !VerParts.first.getAsInteger(10, Major) &&
-        3 <= Major && Major <= CLANG_VERSION_MAJOR &&
-        (Major == 3 ? VerParts.second.size() == 1 &&
-                      !VerParts.second.getAsInteger(10, Minor)
-                    : VerParts.first.size() == Ver.size() ||
-                      VerParts.second == "0")) {
+    if (!VerParts.first.starts_with("0") &&
+        !VerParts.first.getAsInteger(10, Major) && 3 <= Major &&
+        Major <= CLANG_VERSION_MAJOR &&
+        (Major == 3
+             ? VerParts.second.size() == 1 &&
+                   !VerParts.second.getAsInteger(10, Minor)
+             : VerParts.first.size() == Ver.size() || VerParts.second == "0")) {
       // Got a valid version number.
       if (Major == 3 && Minor <= 8)
         Opts.setClangABICompat(LangOptions::ClangABI::Ver3_8);
diff --git a/clang/lib/Frontend/DependencyGraph.cpp b/clang/lib/Frontend/DependencyGraph.cpp
index 6aad04370f6e7ad27efff42769186eebf8a4be31..e96669f856bb18536aa91a087c258149adad4748 100644
--- a/clang/lib/Frontend/DependencyGraph.cpp
+++ b/clang/lib/Frontend/DependencyGraph.cpp
@@ -110,7 +110,7 @@ void DependencyGraphCallback::OutputGraphFile() {
     writeNodeReference(OS, AllFiles[I]);
     OS << " [ shape=\"box\", label=\"";
     StringRef FileName = AllFiles[I].getName();
-    if (FileName.startswith(SysRoot))
+    if (FileName.starts_with(SysRoot))
       FileName = FileName.substr(SysRoot.size());
 
     OS << DOT::EscapeString(std::string(FileName)) << "\"];\n";
diff --git a/clang/lib/Frontend/Rewrite/InclusionRewriter.cpp b/clang/lib/Frontend/Rewrite/InclusionRewriter.cpp
index 2c3a253a67d5c930ed823f447af511573bb51919..b6b37461089e4860e5f46cf8ec2b78121f08a96d 100644
--- a/clang/lib/Frontend/Rewrite/InclusionRewriter.cpp
+++ b/clang/lib/Frontend/Rewrite/InclusionRewriter.cpp
@@ -307,7 +307,7 @@ void InclusionRewriter::OutputContentUpTo(const MemoryBufferRef &FromFile,
       Rest = Rest.substr(Idx);
     }
   }
-  if (EnsureNewline && !TextToWrite.endswith(LocalEOL))
+  if (EnsureNewline && !TextToWrite.ends_with(LocalEOL))
     OS << MainEOL;
 
   WriteFrom = WriteTo;
diff --git a/clang/lib/Frontend/VerifyDiagnosticConsumer.cpp b/clang/lib/Frontend/VerifyDiagnosticConsumer.cpp
index ab8174f4f4db9216a21c912bec1346dfaee4af09..09c1460d54e1d7701542de30e34e4f738e7b1129 100644
--- a/clang/lib/Frontend/VerifyDiagnosticConsumer.cpp
+++ b/clang/lib/Frontend/VerifyDiagnosticConsumer.cpp
@@ -226,10 +226,10 @@ public:
     P = C;
     while (P < End) {
       StringRef S(P, End - P);
-      if (S.startswith(OpenBrace)) {
+      if (S.starts_with(OpenBrace)) {
         ++Depth;
         P += OpenBrace.size();
-      } else if (S.startswith(CloseBrace)) {
+      } else if (S.starts_with(CloseBrace)) {
         --Depth;
         if (Depth == 0) {
           PEnd = P + CloseBrace.size();
@@ -445,7 +445,7 @@ static bool ParseDirective(StringRef S, ExpectedData *ED, SourceManager &SM,
     // others.
 
     // Regex in initial directive token: -re
-    if (DToken.endswith("-re")) {
+    if (DToken.ends_with("-re")) {
       D.RegexKind = true;
       KindStr = "regex";
       DToken = DToken.substr(0, DToken.size()-3);
@@ -454,20 +454,19 @@ static bool ParseDirective(StringRef S, ExpectedData *ED, SourceManager &SM,
     // Type in initial directive token: -{error|warning|note|no-diagnostics}
     bool NoDiag = false;
     StringRef DType;
-    if (DToken.endswith(DType="-error"))
+    if (DToken.ends_with(DType = "-error"))
       D.DL = ED ? &ED->Errors : nullptr;
-    else if (DToken.endswith(DType="-warning"))
+    else if (DToken.ends_with(DType = "-warning"))
       D.DL = ED ? &ED->Warnings : nullptr;
-    else if (DToken.endswith(DType="-remark"))
+    else if (DToken.ends_with(DType = "-remark"))
       D.DL = ED ? &ED->Remarks : nullptr;
-    else if (DToken.endswith(DType="-note"))
+    else if (DToken.ends_with(DType = "-note"))
       D.DL = ED ? &ED->Notes : nullptr;
-    else if (DToken.endswith(DType="-no-diagnostics")) {
+    else if (DToken.ends_with(DType = "-no-diagnostics")) {
       NoDiag = true;
       if (D.RegexKind)
         continue;
-    }
-    else
+    } else
       continue;
     DToken = DToken.substr(0, DToken.size()-DType.size());
 
@@ -1145,7 +1144,7 @@ std::unique_ptr<Directive> Directive::create(bool RegexKind,
   std::string RegexStr;
   StringRef S = Text;
   while (!S.empty()) {
-    if (S.startswith("{{")) {
+    if (S.starts_with("{{")) {
       S = S.drop_front(2);
       size_t RegexMatchLength = S.find("}}");
       assert(RegexMatchLength != StringRef::npos);
diff --git a/clang/lib/Headers/CMakeLists.txt b/clang/lib/Headers/CMakeLists.txt
index fdd54c05eedf8250436b1bff46c66178846f5f91..f8fdd402777e484695b92a87225f45f5b33bba21 100644
--- a/clang/lib/Headers/CMakeLists.txt
+++ b/clang/lib/Headers/CMakeLists.txt
@@ -387,6 +387,8 @@ if(ARM IN_LIST LLVM_TARGETS_TO_BUILD OR AArch64 IN_LIST LLVM_TARGETS_TO_BUILD)
   clang_generate_header(-gen-arm-mve-header arm_mve.td arm_mve.h)
   # Generate arm_cde.h
   clang_generate_header(-gen-arm-cde-header arm_cde.td arm_cde.h)
+  # Generate arm_vector_types.h
+  clang_generate_header(-gen-arm-vector-type arm_neon.td arm_vector_types.h)
 
   # Add headers to target specific lists
   list(APPEND arm_common_generated_files
@@ -403,6 +405,7 @@ if(ARM IN_LIST LLVM_TARGETS_TO_BUILD OR AArch64 IN_LIST LLVM_TARGETS_TO_BUILD)
     "${CMAKE_CURRENT_BINARY_DIR}/arm_sve.h"
     "${CMAKE_CURRENT_BINARY_DIR}/arm_sme_draft_spec_subject_to_change.h"
     "${CMAKE_CURRENT_BINARY_DIR}/arm_bf16.h"
+    "${CMAKE_CURRENT_BINARY_DIR}/arm_vector_types.h"
     )
 endif()
 if(RISCV IN_LIST LLVM_TARGETS_TO_BUILD)
diff --git a/clang/lib/Index/IndexSymbol.cpp b/clang/lib/Index/IndexSymbol.cpp
index c67810ad126b6e2ddab9dafb65e1bc960ab3c518..0f79694d1faac7b9073574bece56f51de9c87526 100644
--- a/clang/lib/Index/IndexSymbol.cpp
+++ b/clang/lib/Index/IndexSymbol.cpp
@@ -36,7 +36,7 @@ static bool isUnitTest(const ObjCMethodDecl *D) {
     return false;
   if (!D->getReturnType()->isVoidType())
     return false;
-  if (!D->getSelector().getNameForSlot(0).startswith("test"))
+  if (!D->getSelector().getNameForSlot(0).starts_with("test"))
     return false;
   return isUnitTestCase(D->getClassInterface());
 }
diff --git a/clang/lib/IndexSerialization/SerializablePathCollection.cpp b/clang/lib/IndexSerialization/SerializablePathCollection.cpp
index bd5f861bf482e511168ca428bdf7046aacb4751b..74ed18a4f612def61128db427752da370a6f238a 100644
--- a/clang/lib/IndexSerialization/SerializablePathCollection.cpp
+++ b/clang/lib/IndexSerialization/SerializablePathCollection.cpp
@@ -70,11 +70,11 @@ PathPool::DirPath SerializablePathCollection::tryStoreDirPath(StringRef Dir) {
   const std::string OrigDir = Dir.str();
 
   PathPool::RootDirKind Root = PathPool::RootDirKind::Regular;
-  if (!SysRoot.empty() && Dir.startswith(SysRoot) &&
+  if (!SysRoot.empty() && Dir.starts_with(SysRoot) &&
       llvm::sys::path::is_separator(Dir[SysRoot.size()])) {
     Root = PathPool::RootDirKind::SysRoot;
     Dir = Dir.drop_front(SysRoot.size());
-  } else if (!WorkDir.empty() && Dir.startswith(WorkDir) &&
+  } else if (!WorkDir.empty() && Dir.starts_with(WorkDir) &&
              llvm::sys::path::is_separator(Dir[WorkDir.size()])) {
     Root = PathPool::RootDirKind::CurrentWorkDir;
     Dir = Dir.drop_front(WorkDir.size());
diff --git a/clang/lib/Lex/HeaderMap.cpp b/clang/lib/Lex/HeaderMap.cpp
index 22a1532c2d93838bf66f3dd27a1ea69d985d2d1f..00bf880726ee3e0120495e8ee71ab9973d5419e2 100644
--- a/clang/lib/Lex/HeaderMap.cpp
+++ b/clang/lib/Lex/HeaderMap.cpp
@@ -11,16 +11,17 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/Lex/HeaderMap.h"
-#include "clang/Lex/HeaderMapTypes.h"
 #include "clang/Basic/CharInfo.h"
 #include "clang/Basic/FileManager.h"
+#include "clang/Lex/HeaderMapTypes.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/DataTypes.h"
+#include "llvm/Support/Debug.h"
 #include "llvm/Support/MathExtras.h"
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/SwapByteOrder.h"
-#include "llvm/Support/Debug.h"
+#include "llvm/Support/SystemZ/zOSSupport.h"
 #include <cstring>
 #include <memory>
 #include <optional>
diff --git a/clang/lib/Lex/HeaderSearch.cpp b/clang/lib/Lex/HeaderSearch.cpp
index c03cf19688d961adb9a02e33b4390b6cbf2b4748..f24013d68795aeee090d5174c1b7fedb3b0cf744 100644
--- a/clang/lib/Lex/HeaderSearch.cpp
+++ b/clang/lib/Lex/HeaderSearch.cpp
@@ -796,7 +796,7 @@ static bool isFrameworkStylePath(StringRef Path, bool &IsPrivateHeader,
     } else if (*I == "PrivateHeaders") {
       ++FoundComp;
       IsPrivateHeader = true;
-    } else if (I->endswith(".framework")) {
+    } else if (I->ends_with(".framework")) {
       StringRef Name = I->drop_back(10); // Drop .framework
       // Need to reset the strings and counter to support nested frameworks.
       FrameworkName.clear();
@@ -1085,7 +1085,7 @@ OptionalFileEntryRef HeaderSearch::LookupFile(
     // If the filename matches a known system header prefix, override
     // whether the file is a system header.
     for (unsigned j = SystemHeaderPrefixes.size(); j; --j) {
-      if (Filename.startswith(SystemHeaderPrefixes[j-1].first)) {
+      if (Filename.starts_with(SystemHeaderPrefixes[j - 1].first)) {
         HFI.DirInfo = SystemHeaderPrefixes[j-1].second ? SrcMgr::C_System
                                                        : SrcMgr::C_User;
         break;
@@ -1694,7 +1694,7 @@ bool HeaderSearch::loadModuleMapFile(FileEntryRef File, bool IsSystem,
     StringRef DirName(Dir->getName());
     if (llvm::sys::path::filename(DirName) == "Modules") {
       DirName = llvm::sys::path::parent_path(DirName);
-      if (DirName.endswith(".framework"))
+      if (DirName.ends_with(".framework"))
         if (auto MaybeDir = FileMgr.getOptionalDirectoryRef(DirName))
           Dir = *MaybeDir;
       // FIXME: This assert can fail if there's a race between the above check
@@ -1965,10 +1965,10 @@ std::string HeaderSearch::suggestPathToFileForDiagnostics(
       // Special case Apple .sdk folders since the search path is typically a
       // symlink like `iPhoneSimulator14.5.sdk` while the file is instead
       // located in `iPhoneSimulator.sdk` (the real folder).
-      if (NI->endswith(".sdk") && DI->endswith(".sdk")) {
+      if (NI->ends_with(".sdk") && DI->ends_with(".sdk")) {
         StringRef NBasename = path::stem(*NI);
         StringRef DBasename = path::stem(*DI);
-        if (DBasename.startswith(NBasename))
+        if (DBasename.starts_with(NBasename))
           continue;
       }
 
diff --git a/clang/lib/Lex/InitHeaderSearch.cpp b/clang/lib/Lex/InitHeaderSearch.cpp
index 5b1b7c859c85d9346858ee7472f6cfd3d9838f8e..2218db15013d926506c576f8a533596ac595ee72 100644
--- a/clang/lib/Lex/InitHeaderSearch.cpp
+++ b/clang/lib/Lex/InitHeaderSearch.cpp
@@ -141,8 +141,8 @@ bool InitHeaderSearch::AddUnmappedPath(const Twine &Path, IncludeDirGroup Group,
   StringRef MappedPathStr = Path.toStringRef(MappedPathStorage);
 
   // If use system headers while cross-compiling, emit the warning.
-  if (HasSysroot && (MappedPathStr.startswith("/usr/include") ||
-                     MappedPathStr.startswith("/usr/local/include"))) {
+  if (HasSysroot && (MappedPathStr.starts_with("/usr/include") ||
+                     MappedPathStr.starts_with("/usr/local/include"))) {
     Headers.getDiags().Report(diag::warn_poison_system_directories)
         << MappedPathStr;
   }
diff --git a/clang/lib/Lex/Lexer.cpp b/clang/lib/Lex/Lexer.cpp
index f4f1daab857f043696585083e3c99a0839abd1ea..50b56265f6e164d9f5f24546197db33d141d137f 100644
--- a/clang/lib/Lex/Lexer.cpp
+++ b/clang/lib/Lex/Lexer.cpp
@@ -3212,8 +3212,8 @@ bool Lexer::IsStartOfConflictMarker(const char *CurPtr) {
     return false;
 
   // Check to see if we have <<<<<<< or >>>>.
-  if (!StringRef(CurPtr, BufferEnd - CurPtr).startswith("<<<<<<<") &&
-      !StringRef(CurPtr, BufferEnd - CurPtr).startswith(">>>> "))
+  if (!StringRef(CurPtr, BufferEnd - CurPtr).starts_with("<<<<<<<") &&
+      !StringRef(CurPtr, BufferEnd - CurPtr).starts_with(">>>> "))
     return false;
 
   // If we have a situation where we don't care about conflict markers, ignore
diff --git a/clang/lib/Lex/ModuleMap.cpp b/clang/lib/Lex/ModuleMap.cpp
index d35c282543c564d6ac94f8db51d50d38f5a4f7db..ea5d13deb11470325a4e54974a1366e528566daf 100644
--- a/clang/lib/Lex/ModuleMap.cpp
+++ b/clang/lib/Lex/ModuleMap.cpp
@@ -235,7 +235,7 @@ OptionalFileEntryRef ModuleMap::findHeader(
   llvm::sys::path::append(FullPathName, RelativePathName);
   auto NormalHdrFile = GetFile(FullPathName);
 
-  if (!NormalHdrFile && Directory->getName().endswith(".framework")) {
+  if (!NormalHdrFile && Directory->getName().ends_with(".framework")) {
     // The lack of 'framework' keyword in a module declaration it's a simple
     // mistake we can diagnose when the header exists within the proper
     // framework style path.
@@ -1034,7 +1034,7 @@ Module *ModuleMap::inferFrameworkModule(DirectoryEntryRef FrameworkDir,
         if (inferred == InferredDirectories.end()) {
           // We haven't looked here before. Load a module map, if there is
           // one.
-          bool IsFrameworkDir = Parent.endswith(".framework");
+          bool IsFrameworkDir = Parent.ends_with(".framework");
           if (OptionalFileEntryRef ModMapFile =
                   HeaderInfo.lookupModuleMapFile(*ParentDir, IsFrameworkDir)) {
             parseModuleMapFile(*ModMapFile, Attrs.IsSystem, *ParentDir);
@@ -1125,7 +1125,7 @@ Module *ModuleMap::inferFrameworkModule(DirectoryEntryRef FrameworkDir,
            Dir = FS.dir_begin(SubframeworksDirName, EC),
            DirEnd;
        Dir != DirEnd && !EC; Dir.increment(EC)) {
-    if (!StringRef(Dir->path()).endswith(".framework"))
+    if (!StringRef(Dir->path()).ends_with(".framework"))
       continue;
 
     if (auto SubframeworkDir = FileMgr.getOptionalDirectoryRef(Dir->path())) {
@@ -1337,7 +1337,7 @@ ModuleMap::canonicalizeModuleMapPath(SmallVectorImpl<char> &Path) {
   // Modules/ not Versions/A/Modules.
   if (llvm::sys::path::filename(Dir) == "Modules") {
     StringRef Parent = llvm::sys::path::parent_path(Dir);
-    if (Parent.endswith(".framework"))
+    if (Parent.ends_with(".framework"))
       Dir = Parent;
   }
 
@@ -2119,8 +2119,8 @@ void ModuleMapParser::parseModuleDecl() {
   ActiveModule->Directory = Directory;
 
   StringRef MapFileName(ModuleMapFile.getName());
-  if (MapFileName.endswith("module.private.modulemap") ||
-      MapFileName.endswith("module_private.map")) {
+  if (MapFileName.ends_with("module.private.modulemap") ||
+      MapFileName.ends_with("module_private.map")) {
     ActiveModule->ModuleMapIsPrivate = true;
   }
 
diff --git a/clang/lib/Lex/PPDirectives.cpp b/clang/lib/Lex/PPDirectives.cpp
index 14003480d7fa2e91f38a363a7932879fec2f2d9e..112bc8dc572c9271f3d4dbab133fe12c14c9d805 100644
--- a/clang/lib/Lex/PPDirectives.cpp
+++ b/clang/lib/Lex/PPDirectives.cpp
@@ -164,13 +164,13 @@ static bool isLanguageDefinedBuiltin(const SourceManager &SourceMgr,
     return false;
   // C defines macros starting with __STDC, and C++ defines macros starting with
   // __STDCPP
-  if (MacroName.startswith("__STDC"))
+  if (MacroName.starts_with("__STDC"))
     return true;
   // C++ defines the __cplusplus macro
   if (MacroName == "__cplusplus")
     return true;
   // C++ defines various feature-test macros starting with __cpp
-  if (MacroName.startswith("__cpp"))
+  if (MacroName.starts_with("__cpp"))
     return true;
   // Anything else isn't language-defined
   return false;
@@ -646,7 +646,7 @@ void Preprocessor::SkipExcludedConditionalBlock(SourceLocation HashTokenLoc,
       Directive = StringRef(DirectiveBuf, IdLen);
     }
 
-    if (Directive.startswith("if")) {
+    if (Directive.starts_with("if")) {
       StringRef Sub = Directive.substr(2);
       if (Sub.empty() ||   // "if"
           Sub == "def" ||   // "ifdef"
@@ -2788,14 +2788,14 @@ static bool isConfigurationPattern(Token &MacroName, MacroInfo *MI,
         return false;
       StringRef ValueText = II->getName();
       StringRef TrimmedValue = ValueText;
-      if (!ValueText.startswith("__")) {
-        if (ValueText.startswith("_"))
+      if (!ValueText.starts_with("__")) {
+        if (ValueText.starts_with("_"))
           TrimmedValue = TrimmedValue.drop_front(1);
         else
           return false;
       } else {
         TrimmedValue = TrimmedValue.drop_front(2);
-        if (TrimmedValue.endswith("__"))
+        if (TrimmedValue.ends_with("__"))
           TrimmedValue = TrimmedValue.drop_back(2);
       }
       return TrimmedValue.equals(MacroText);
diff --git a/clang/lib/Lex/PPExpressions.cpp b/clang/lib/Lex/PPExpressions.cpp
index 269984aae07bf288a283fbe37080aec13493c467..1feb0eb18d71e6eaa18c0acd6ba596c526b39d15 100644
--- a/clang/lib/Lex/PPExpressions.cpp
+++ b/clang/lib/Lex/PPExpressions.cpp
@@ -267,7 +267,7 @@ static bool EvaluateValue(PPValue &Result, Token &PeekTok, DefinedTracker &DT,
             const StringRef IdentifierName = II->getName();
             if (llvm::any_of(UndefPrefixes,
                              [&IdentifierName](const std::string &Prefix) {
-                               return IdentifierName.startswith(Prefix);
+                               return IdentifierName.starts_with(Prefix);
                              }))
               PP.Diag(PeekTok, diag::warn_pp_undef_prefix)
                   << AddFlagValue{llvm::join(UndefPrefixes, ",")} << II;
diff --git a/clang/lib/Lex/PPMacroExpansion.cpp b/clang/lib/Lex/PPMacroExpansion.cpp
index 30c4abdbad8aa4462552d1639651f9068d1dbc17..ad02f31209b0b7804447e2430013c0933745a416 100644
--- a/clang/lib/Lex/PPMacroExpansion.cpp
+++ b/clang/lib/Lex/PPMacroExpansion.cpp
@@ -1136,7 +1136,8 @@ static bool HasFeature(const Preprocessor &PP, StringRef Feature) {
   const LangOptions &LangOpts = PP.getLangOpts();
 
   // Normalize the feature name, __foo__ becomes foo.
-  if (Feature.startswith("__") && Feature.endswith("__") && Feature.size() >= 4)
+  if (Feature.starts_with("__") && Feature.ends_with("__") &&
+      Feature.size() >= 4)
     Feature = Feature.substr(2, Feature.size() - 4);
 
 #define FEATURE(Name, Predicate) .Case(#Name, Predicate)
@@ -1162,7 +1163,7 @@ static bool HasExtension(const Preprocessor &PP, StringRef Extension) {
   const LangOptions &LangOpts = PP.getLangOpts();
 
   // Normalize the extension name, __foo__ becomes foo.
-  if (Extension.startswith("__") && Extension.endswith("__") &&
+  if (Extension.starts_with("__") && Extension.ends_with("__") &&
       Extension.size() >= 4)
     Extension = Extension.substr(2, Extension.size() - 4);
 
@@ -1691,9 +1692,9 @@ void Preprocessor::ExpandBuiltinMacro(Token &Tok) {
           // as being "builtin functions", even if the syntax isn't a valid
           // function call (for example, because the builtin takes a type
           // argument).
-          if (II->getName().startswith("__builtin_") ||
-              II->getName().startswith("__is_") ||
-              II->getName().startswith("__has_"))
+          if (II->getName().starts_with("__builtin_") ||
+              II->getName().starts_with("__is_") ||
+              II->getName().starts_with("__has_"))
             return true;
           return llvm::StringSwitch<bool>(II->getName())
               .Case("__array_rank", true)
diff --git a/clang/lib/Parse/ParseDecl.cpp b/clang/lib/Parse/ParseDecl.cpp
index ece3698967e2f67e0c7044f3a179a3742985b2d4..ed006f9d67de454c43e85c43007aef8da6f13ae1 100644
--- a/clang/lib/Parse/ParseDecl.cpp
+++ b/clang/lib/Parse/ParseDecl.cpp
@@ -84,7 +84,7 @@ TypeResult Parser::ParseTypeName(SourceRange *Range, DeclaratorContext Context,
 
 /// Normalizes an attribute name by dropping prefixed and suffixed __.
 static StringRef normalizeAttrName(StringRef Name) {
-  if (Name.size() >= 4 && Name.startswith("__") && Name.endswith("__"))
+  if (Name.size() >= 4 && Name.starts_with("__") && Name.ends_with("__"))
     return Name.drop_front(2).drop_back(2);
   return Name;
 }
@@ -7854,7 +7854,7 @@ void Parser::ParseTypeofSpecifier(DeclSpec &DS) {
 
   bool IsUnqual = Tok.is(tok::kw_typeof_unqual);
   const IdentifierInfo *II = Tok.getIdentifierInfo();
-  if (getLangOpts().C23 && !II->getName().startswith("__"))
+  if (getLangOpts().C23 && !II->getName().starts_with("__"))
     Diag(Tok.getLocation(), diag::warn_c23_compat_keyword) << Tok.getName();
 
   Token OpTok = Tok;
diff --git a/clang/lib/Parse/ParseExprCXX.cpp b/clang/lib/Parse/ParseExprCXX.cpp
index 2fc364fc811b3245f9a37fac80920ad81872e3f2..ef9ea6575205cdf31f79462d456b54d14d416e8b 100644
--- a/clang/lib/Parse/ParseExprCXX.cpp
+++ b/clang/lib/Parse/ParseExprCXX.cpp
@@ -1311,18 +1311,6 @@ ExprResult Parser::ParseLambdaExpressionAfterIntroducer(
     D.takeAttributes(Attributes);
   }
 
-  // Helper to emit a warning if we see a CUDA host/device/global attribute
-  // after '(...)'. nvcc doesn't accept this.
-  auto WarnIfHasCUDATargetAttr = [&] {
-    if (getLangOpts().CUDA)
-      for (const ParsedAttr &A : Attributes)
-        if (A.getKind() == ParsedAttr::AT_CUDADevice ||
-            A.getKind() == ParsedAttr::AT_CUDAHost ||
-            A.getKind() == ParsedAttr::AT_CUDAGlobal)
-          Diag(A.getLoc(), diag::warn_cuda_attr_lambda_position)
-              << A.getAttrName()->getName();
-  };
-
   MultiParseScope TemplateParamScope(*this);
   if (Tok.is(tok::less)) {
     Diag(Tok, getLangOpts().CPlusPlus20
@@ -1377,91 +1365,6 @@ ExprResult Parser::ParseLambdaExpressionAfterIntroducer(
   bool HasSpecifiers = false;
   SourceLocation MutableLoc;
 
-  auto ParseConstexprAndMutableSpecifiers = [&] {
-    // GNU-style attributes must be parsed before the mutable specifier to
-    // be compatible with GCC. MSVC-style attributes must be parsed before
-    // the mutable specifier to be compatible with MSVC.
-    MaybeParseAttributes(PAKM_GNU | PAKM_Declspec, Attributes);
-    // Parse mutable-opt and/or constexpr-opt or consteval-opt, and update
-    // the DeclEndLoc.
-    SourceLocation ConstexprLoc;
-    SourceLocation ConstevalLoc;
-    SourceLocation StaticLoc;
-
-    tryConsumeLambdaSpecifierToken(*this, MutableLoc, StaticLoc, ConstexprLoc,
-                                   ConstevalLoc, DeclEndLoc);
-
-    DiagnoseStaticSpecifierRestrictions(*this, StaticLoc, MutableLoc, Intro);
-
-    addStaticToLambdaDeclSpecifier(*this, StaticLoc, DS);
-    addConstexprToLambdaDeclSpecifier(*this, ConstexprLoc, DS);
-    addConstevalToLambdaDeclSpecifier(*this, ConstevalLoc, DS);
-  };
-
-  auto ParseLambdaSpecifiers =
-      [&](MutableArrayRef<DeclaratorChunk::ParamInfo> ParamInfo,
-          SourceLocation EllipsisLoc) {
-        // Parse exception-specification[opt].
-        ExceptionSpecificationType ESpecType = EST_None;
-        SourceRange ESpecRange;
-        SmallVector<ParsedType, 2> DynamicExceptions;
-        SmallVector<SourceRange, 2> DynamicExceptionRanges;
-        ExprResult NoexceptExpr;
-        CachedTokens *ExceptionSpecTokens;
-
-        ESpecType = tryParseExceptionSpecification(
-            /*Delayed=*/false, ESpecRange, DynamicExceptions,
-            DynamicExceptionRanges, NoexceptExpr, ExceptionSpecTokens);
-
-        if (ESpecType != EST_None)
-          DeclEndLoc = ESpecRange.getEnd();
-
-        // Parse attribute-specifier[opt].
-        if (MaybeParseCXX11Attributes(Attributes))
-          DeclEndLoc = Attributes.Range.getEnd();
-
-        // Parse OpenCL addr space attribute.
-        if (Tok.isOneOf(tok::kw___private, tok::kw___global, tok::kw___local,
-                        tok::kw___constant, tok::kw___generic)) {
-          ParseOpenCLQualifiers(DS.getAttributes());
-          ConsumeToken();
-        }
-
-        SourceLocation FunLocalRangeEnd = DeclEndLoc;
-
-        // Parse trailing-return-type[opt].
-        if (Tok.is(tok::arrow)) {
-          FunLocalRangeEnd = Tok.getLocation();
-          SourceRange Range;
-          TrailingReturnType = ParseTrailingReturnType(
-              Range, /*MayBeFollowedByDirectInit*/ false);
-          TrailingReturnTypeLoc = Range.getBegin();
-          if (Range.getEnd().isValid())
-            DeclEndLoc = Range.getEnd();
-        }
-
-        SourceLocation NoLoc;
-        D.AddTypeInfo(
-            DeclaratorChunk::getFunction(
-                /*HasProto=*/true,
-                /*IsAmbiguous=*/false, LParenLoc, ParamInfo.data(),
-                ParamInfo.size(), EllipsisLoc, RParenLoc,
-                /*RefQualifierIsLvalueRef=*/true,
-                /*RefQualifierLoc=*/NoLoc, MutableLoc, ESpecType, ESpecRange,
-                DynamicExceptions.data(), DynamicExceptionRanges.data(),
-                DynamicExceptions.size(),
-                NoexceptExpr.isUsable() ? NoexceptExpr.get() : nullptr,
-                /*ExceptionSpecTokens*/ nullptr,
-                /*DeclsInPrototype=*/std::nullopt, LParenLoc, FunLocalRangeEnd,
-                D, TrailingReturnType, TrailingReturnTypeLoc, &DS),
-            std::move(Attributes), DeclEndLoc);
-
-        Actions.ActOnLambdaClosureQualifiers(Intro, MutableLoc);
-
-        if (HasParentheses && Tok.is(tok::kw_requires))
-          ParseTrailingRequiresClause(D);
-      };
-
   ParseScope Prototype(this, Scope::FunctionPrototypeScope |
                                  Scope::FunctionDeclarationScope |
                                  Scope::DeclScope);
@@ -1511,18 +1414,104 @@ ExprResult Parser::ParseLambdaExpressionAfterIntroducer(
         << FixItHint::CreateInsertion(Tok.getLocation(), "() ");
   }
 
-  if (HasParentheses || HasSpecifiers)
-    ParseConstexprAndMutableSpecifiers();
+  if (HasParentheses || HasSpecifiers) {
+    // GNU-style attributes must be parsed before the mutable specifier to
+    // be compatible with GCC. MSVC-style attributes must be parsed before
+    // the mutable specifier to be compatible with MSVC.
+    MaybeParseAttributes(PAKM_GNU | PAKM_Declspec, Attributes);
+    // Parse mutable-opt and/or constexpr-opt or consteval-opt, and update
+    // the DeclEndLoc.
+    SourceLocation ConstexprLoc;
+    SourceLocation ConstevalLoc;
+    SourceLocation StaticLoc;
+
+    tryConsumeLambdaSpecifierToken(*this, MutableLoc, StaticLoc, ConstexprLoc,
+                                   ConstevalLoc, DeclEndLoc);
+
+    DiagnoseStaticSpecifierRestrictions(*this, StaticLoc, MutableLoc, Intro);
+
+    addStaticToLambdaDeclSpecifier(*this, StaticLoc, DS);
+    addConstexprToLambdaDeclSpecifier(*this, ConstexprLoc, DS);
+    addConstevalToLambdaDeclSpecifier(*this, ConstevalLoc, DS);
+  }
 
   Actions.ActOnLambdaClosureParameters(getCurScope(), ParamInfo);
 
   if (!HasParentheses)
     Actions.ActOnLambdaClosureQualifiers(Intro, MutableLoc);
 
-  if (HasSpecifiers || HasParentheses)
-    ParseLambdaSpecifiers(ParamInfo, EllipsisLoc);
+  if (HasSpecifiers || HasParentheses) {
+    // Parse exception-specification[opt].
+    ExceptionSpecificationType ESpecType = EST_None;
+    SourceRange ESpecRange;
+    SmallVector<ParsedType, 2> DynamicExceptions;
+    SmallVector<SourceRange, 2> DynamicExceptionRanges;
+    ExprResult NoexceptExpr;
+    CachedTokens *ExceptionSpecTokens;
+
+    ESpecType = tryParseExceptionSpecification(
+        /*Delayed=*/false, ESpecRange, DynamicExceptions,
+        DynamicExceptionRanges, NoexceptExpr, ExceptionSpecTokens);
+
+    if (ESpecType != EST_None)
+      DeclEndLoc = ESpecRange.getEnd();
+
+    // Parse attribute-specifier[opt].
+    if (MaybeParseCXX11Attributes(Attributes))
+      DeclEndLoc = Attributes.Range.getEnd();
+
+    // Parse OpenCL addr space attribute.
+    if (Tok.isOneOf(tok::kw___private, tok::kw___global, tok::kw___local,
+                    tok::kw___constant, tok::kw___generic)) {
+      ParseOpenCLQualifiers(DS.getAttributes());
+      ConsumeToken();
+    }
+
+    SourceLocation FunLocalRangeEnd = DeclEndLoc;
+
+    // Parse trailing-return-type[opt].
+    if (Tok.is(tok::arrow)) {
+      FunLocalRangeEnd = Tok.getLocation();
+      SourceRange Range;
+      TrailingReturnType =
+          ParseTrailingReturnType(Range, /*MayBeFollowedByDirectInit=*/false);
+      TrailingReturnTypeLoc = Range.getBegin();
+      if (Range.getEnd().isValid())
+        DeclEndLoc = Range.getEnd();
+    }
+
+    SourceLocation NoLoc;
+    D.AddTypeInfo(DeclaratorChunk::getFunction(
+                      /*HasProto=*/true,
+                      /*IsAmbiguous=*/false, LParenLoc, ParamInfo.data(),
+                      ParamInfo.size(), EllipsisLoc, RParenLoc,
+                      /*RefQualifierIsLvalueRef=*/true,
+                      /*RefQualifierLoc=*/NoLoc, MutableLoc, ESpecType,
+                      ESpecRange, DynamicExceptions.data(),
+                      DynamicExceptionRanges.data(), DynamicExceptions.size(),
+                      NoexceptExpr.isUsable() ? NoexceptExpr.get() : nullptr,
+                      /*ExceptionSpecTokens*/ nullptr,
+                      /*DeclsInPrototype=*/std::nullopt, LParenLoc,
+                      FunLocalRangeEnd, D, TrailingReturnType,
+                      TrailingReturnTypeLoc, &DS),
+                  std::move(Attributes), DeclEndLoc);
 
-  WarnIfHasCUDATargetAttr();
+    Actions.ActOnLambdaClosureQualifiers(Intro, MutableLoc);
+
+    if (HasParentheses && Tok.is(tok::kw_requires))
+      ParseTrailingRequiresClause(D);
+  }
+
+  // Emit a warning if we see a CUDA host/device/global attribute
+  // after '(...)'. nvcc doesn't accept this.
+  if (getLangOpts().CUDA) {
+    for (const ParsedAttr &A : Attributes)
+      if (A.getKind() == ParsedAttr::AT_CUDADevice ||
+          A.getKind() == ParsedAttr::AT_CUDAHost ||
+          A.getKind() == ParsedAttr::AT_CUDAGlobal)
+        Diag(A.getLoc(), diag::warn_cuda_attr_lambda_position)
+            << A.getAttrName()->getName();
+  }
 
   Prototype.Exit();
 
diff --git a/clang/lib/Parse/Parser.cpp b/clang/lib/Parse/Parser.cpp
index ec67faf7dcaf863985a2b595e517d3007ff3fa68..b703c2d9b8e04d831c1908dc26b689332879866d 100644
--- a/clang/lib/Parse/Parser.cpp
+++ b/clang/lib/Parse/Parser.cpp
@@ -2645,7 +2645,7 @@ Decl *Parser::ParseModuleImport(SourceLocation AtLoc,
     auto &SrcMgr = PP.getSourceManager();
     auto FE = SrcMgr.getFileEntryRefForID(SrcMgr.getFileID(AtLoc));
     if (FE && llvm::sys::path::parent_path(FE->getDir().getName())
-                  .endswith(".framework"))
+                  .ends_with(".framework"))
       Diags.Report(AtLoc, diag::warn_atimport_in_framework_header);
   }
 
diff --git a/clang/lib/Rewrite/Rewriter.cpp b/clang/lib/Rewrite/Rewriter.cpp
index 0896221dd0bdeb5672b89ea2d450f6faa78b8d3c..0e6ae365064463bf6a6866a8a21c6a5c4a1dcb7a 100644
--- a/clang/lib/Rewrite/Rewriter.cpp
+++ b/clang/lib/Rewrite/Rewriter.cpp
@@ -386,7 +386,7 @@ bool Rewriter::IncreaseIndentation(CharSourceRange range,
   }
   if (parentSpace.size() >= startSpace.size())
     return true;
-  if (!startSpace.startswith(parentSpace))
+  if (!startSpace.starts_with(parentSpace))
     return true;
 
   StringRef indent = startSpace.substr(parentSpace.size());
@@ -399,7 +399,7 @@ bool Rewriter::IncreaseIndentation(CharSourceRange range,
     while (isWhitespaceExceptNL(MB[i]))
       ++i;
     StringRef origIndent = MB.substr(offs, i-offs);
-    if (origIndent.startswith(startSpace))
+    if (origIndent.starts_with(startSpace))
       RB.InsertText(offs, indent, /*InsertAfter=*/false);
   }
 
diff --git a/clang/lib/Sema/CodeCompleteConsumer.cpp b/clang/lib/Sema/CodeCompleteConsumer.cpp
index 9caa1a8431e9021cf01cf45930e211acc4a467fe..350bd78b57107bbfaa0dde13af5ec7c18af9b503 100644
--- a/clang/lib/Sema/CodeCompleteConsumer.cpp
+++ b/clang/lib/Sema/CodeCompleteConsumer.cpp
@@ -630,15 +630,16 @@ bool PrintingCodeCompleteConsumer::isResultFilteredOut(
     StringRef Filter, CodeCompletionResult Result) {
   switch (Result.Kind) {
   case CodeCompletionResult::RK_Declaration:
-    return !(Result.Declaration->getIdentifier() &&
-             Result.Declaration->getIdentifier()->getName().startswith(Filter));
+    return !(
+        Result.Declaration->getIdentifier() &&
+        Result.Declaration->getIdentifier()->getName().starts_with(Filter));
   case CodeCompletionResult::RK_Keyword:
-    return !StringRef(Result.Keyword).startswith(Filter);
+    return !StringRef(Result.Keyword).starts_with(Filter);
   case CodeCompletionResult::RK_Macro:
-    return !Result.Macro->getName().startswith(Filter);
+    return !Result.Macro->getName().starts_with(Filter);
   case CodeCompletionResult::RK_Pattern:
     return !(Result.Pattern->getTypedText() &&
-             StringRef(Result.Pattern->getTypedText()).startswith(Filter));
+             StringRef(Result.Pattern->getTypedText()).starts_with(Filter));
   }
   llvm_unreachable("Unknown code completion result Kind.");
 }
diff --git a/clang/lib/Sema/SemaCXXScopeSpec.cpp b/clang/lib/Sema/SemaCXXScopeSpec.cpp
index 44a40215b90dfb4c3b5ca3e75de7aabde4b4169e..b3b19b7ed7ffffde3ef5fbedc9819ce70dfcdfa3 100644
--- a/clang/lib/Sema/SemaCXXScopeSpec.cpp
+++ b/clang/lib/Sema/SemaCXXScopeSpec.cpp
@@ -30,6 +30,20 @@ static CXXRecordDecl *getCurrentInstantiationOf(QualType T,
     return nullptr;
 
   const Type *Ty = T->getCanonicalTypeInternal().getTypePtr();
+  if (isa<TemplateSpecializationType>(Ty)) {
+    if (auto *Record = dyn_cast<CXXRecordDecl>(CurContext)) {
+      if (isa<ClassTemplatePartialSpecializationDecl>(Record) ||
+          Record->getDescribedClassTemplate()) {
+        const Type *ICNT = Record->getTypeForDecl();
+        QualType Injected =
+            cast<InjectedClassNameType>(ICNT)->getInjectedSpecializationType();
+
+        if (Ty == Injected->getCanonicalTypeInternal().getTypePtr())
+          return Record;
+      }
+    }
+  }
+
   if (const RecordType *RecordTy = dyn_cast<RecordType>(Ty)) {
     CXXRecordDecl *Record = cast<CXXRecordDecl>(RecordTy->getDecl());
     if (!Record->isDependentContext() ||
@@ -37,10 +51,12 @@ static CXXRecordDecl *getCurrentInstantiationOf(QualType T,
       return Record;
 
     return nullptr;
-  } else if (isa<InjectedClassNameType>(Ty))
-    return cast<InjectedClassNameType>(Ty)->getDecl();
-  else
-    return nullptr;
+  }
+
+  if (auto *ICNT = dyn_cast<InjectedClassNameType>(Ty))
+    return ICNT->getDecl();
+
+  return nullptr;
 }
 
 /// Compute the DeclContext that is associated with the given type.
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index cdb6e9584e9554104e3b4921f15d4752440661fe..254c272b8093d0cf68fc00a6041b3dd115d10e6e 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -1219,7 +1219,7 @@ void Sema::checkFortifiedBuiltinMemoryFunction(FunctionDecl *FD,
     if (IsChkVariant) {
       FunctionName = FunctionName.drop_front(std::strlen("__builtin___"));
       FunctionName = FunctionName.drop_back(std::strlen("_chk"));
-    } else if (FunctionName.startswith("__builtin_")) {
+    } else if (FunctionName.starts_with("__builtin_")) {
       FunctionName = FunctionName.drop_front(std::strlen("__builtin_"));
     }
     return FunctionName;
@@ -18270,15 +18270,14 @@ static bool isSetterLikeSelector(Selector sel) {
 
   StringRef str = sel.getNameForSlot(0);
   while (!str.empty() && str.front() == '_') str = str.substr(1);
-  if (str.startswith("set"))
+  if (str.starts_with("set"))
     str = str.substr(3);
-  else if (str.startswith("add")) {
+  else if (str.starts_with("add")) {
     // Specially allow 'addOperationWithBlock:'.
-    if (sel.getNumArgs() == 1 && str.startswith("addOperationWithBlock"))
+    if (sel.getNumArgs() == 1 && str.starts_with("addOperationWithBlock"))
       return false;
     str = str.substr(3);
-  }
-  else
+  } else
     return false;
 
   if (str.empty()) return true;
diff --git a/clang/lib/Sema/SemaCodeComplete.cpp b/clang/lib/Sema/SemaCodeComplete.cpp
index 143968b4ab0442fe017705f21dccbef066c164ab..c44be0df9b0a853f2ee2fcb88e59691a5398f27c 100644
--- a/clang/lib/Sema/SemaCodeComplete.cpp
+++ b/clang/lib/Sema/SemaCodeComplete.cpp
@@ -9798,7 +9798,7 @@ void Sema::CodeCompleteObjCMethodDeclSelector(
   Results.ExitScope();
 
   if (!AtParameterName && !SelIdents.empty() &&
-      SelIdents.front()->getName().startswith("init")) {
+      SelIdents.front()->getName().starts_with("init")) {
     for (const auto &M : PP.macros()) {
       if (M.first->getName() != "NS_DESIGNATED_INITIALIZER")
         continue;
@@ -10110,7 +10110,7 @@ void Sema::CodeCompleteIncludedFile(llvm::StringRef Dir, bool Angled) {
     }
 
     const StringRef &Dirname = llvm::sys::path::filename(Dir);
-    const bool isQt = Dirname.startswith("Qt") || Dirname == "ActiveQt";
+    const bool isQt = Dirname.starts_with("Qt") || Dirname == "ActiveQt";
     const bool ExtensionlessHeaders =
         IsSystem || isQt || Dir.ends_with(".framework/Headers");
     std::error_code EC;
diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp
index 19d972ed8ab2d830ec9b099d3a820ff00561bc03..be6a136ef37bc4e9e290aac066d833395d5ae525 100644
--- a/clang/lib/Sema/SemaDecl.cpp
+++ b/clang/lib/Sema/SemaDecl.cpp
@@ -15800,7 +15800,7 @@ Decl *Sema::ActOnSkippedFunctionBody(Decl *Decl) {
 }
 
 Decl *Sema::ActOnFinishFunctionBody(Decl *D, Stmt *BodyArg) {
-  return ActOnFinishFunctionBody(D, BodyArg, false);
+  return ActOnFinishFunctionBody(D, BodyArg, /*IsInstantiation=*/false);
 }
 
 /// RAII object that pops an ExpressionEvaluationContext when exiting a function
@@ -16005,7 +16005,7 @@ Decl *Sema::ActOnFinishFunctionBody(Decl *dcl, Stmt *Body,
 
             return StartTok.consume_front("const") &&
                    (StartTok.empty() || isWhitespace(StartTok[0]) ||
-                    StartTok.startswith("/*") || StartTok.startswith("//"));
+                    StartTok.starts_with("/*") || StartTok.starts_with("//"));
           };
 
           auto findBeginLoc = [&]() {
@@ -16359,7 +16359,7 @@ NamedDecl *Sema::ImplicitlyDefineFunction(SourceLocation Loc,
 
   // Extension in C99 (defaults to error). Legal in C89, but warn about it.
   unsigned diag_id;
-  if (II.getName().startswith("__builtin_"))
+  if (II.getName().starts_with("__builtin_"))
     diag_id = diag::warn_builtin_unknown;
   // OpenCL v2.0 s6.9.u - Implicit function declaration is not supported.
   else if (getLangOpts().C99)
diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp
index 59e456fd9f729837de00d86e5511db839d960c6d..5b29b05dee54b3e95032df22b729594fda8ea7dd 100644
--- a/clang/lib/Sema/SemaDeclAttr.cpp
+++ b/clang/lib/Sema/SemaDeclAttr.cpp
@@ -1809,8 +1809,8 @@ static void handleAssumumptionAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
 /// Normalize the attribute, __foo__ becomes foo.
 /// Returns true if normalization was applied.
 static bool normalizeName(StringRef &AttrName) {
-  if (AttrName.size() > 4 && AttrName.startswith("__") &&
-      AttrName.endswith("__")) {
+  if (AttrName.size() > 4 && AttrName.starts_with("__") &&
+      AttrName.ends_with("__")) {
     AttrName = AttrName.drop_front(2).drop_back(2);
     return true;
   }
@@ -3605,7 +3605,7 @@ bool Sema::checkTargetClonesAttrString(
       }
     } else {
       // Other targets ( currently X86 )
-      if (Cur.startswith("arch=")) {
+      if (Cur.starts_with("arch=")) {
         if (!Context.getTargetInfo().isValidCPUName(
                 Cur.drop_front(sizeof("arch=") - 1)))
           return Diag(CurLoc, diag::warn_unsupported_target_attribute)
@@ -3623,7 +3623,7 @@ bool Sema::checkTargetClonesAttrString(
       StringsBuffer.push_back(Cur);
     }
   }
-  if (Str.rtrim().endswith(","))
+  if (Str.rtrim().ends_with(","))
     return Diag(LiteralLoc, diag::warn_unsupported_target_attribute)
            << Unsupported << None << "" << TargetClones;
   return false;
@@ -5225,8 +5225,16 @@ static void handleCallConvAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
 }
 
 static void handleSuppressAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
-  if (!AL.checkAtLeastNumArgs(S, 1))
+  if (AL.getAttributeSpellingListIndex() == SuppressAttr::CXX11_gsl_suppress) {
+    // Suppression attribute with GSL spelling requires at least 1 argument.
+    if (!AL.checkAtLeastNumArgs(S, 1))
+      return;
+  } else if (!isa<VarDecl>(D)) {
+    // Analyzer suppression applies only to variables and statements.
+    S.Diag(AL.getLoc(), diag::err_attribute_wrong_decl_type_str)
+        << AL << 0 << "variables and statements";
     return;
+  }
 
   std::vector<StringRef> DiagnosticIdentifiers;
   for (unsigned I = 0, E = AL.getNumArgs(); I != E; ++I) {
@@ -5235,8 +5243,6 @@ static void handleSuppressAttr(Sema &S, Decl *D, const ParsedAttr &AL) {
     if (!S.checkStringLiteralArgumentAttr(AL, I, RuleName, nullptr))
       return;
 
-    // FIXME: Warn if the rule name is unknown. This is tricky because only
-    // clang-tidy knows about available rules.
     DiagnosticIdentifiers.push_back(RuleName);
   }
   D->addAttr(::new (S.Context)
@@ -5819,7 +5825,7 @@ struct IntrinToName {
 static bool ArmBuiltinAliasValid(unsigned BuiltinID, StringRef AliasName,
                                  ArrayRef<IntrinToName> Map,
                                  const char *IntrinNames) {
-  if (AliasName.startswith("__arm_"))
+  if (AliasName.starts_with("__arm_"))
     AliasName = AliasName.substr(6);
   const IntrinToName *It =
       llvm::lower_bound(Map, BuiltinID, [](const IntrinToName &L, unsigned Id) {
@@ -6663,10 +6669,10 @@ validateSwiftFunctionName(Sema &S, const ParsedAttr &AL, SourceLocation Loc,
 
   // Check whether this will be mapped to a getter or setter of a property.
   bool IsGetter = false, IsSetter = false;
-  if (Name.startswith("getter:")) {
+  if (Name.starts_with("getter:")) {
     IsGetter = true;
     Name = Name.substr(7);
-  } else if (Name.startswith("setter:")) {
+  } else if (Name.starts_with("setter:")) {
     IsSetter = true;
     Name = Name.substr(7);
   }
@@ -7292,7 +7298,7 @@ static void handleHLSLResourceBindingAttr(Sema &S, Decl *D,
     }
   }
 
-  if (!Space.startswith("space")) {
+  if (!Space.starts_with("space")) {
     S.Diag(SpaceArgLoc, diag::err_hlsl_expected_space) << Space;
     return;
   }
diff --git a/clang/lib/Sema/SemaDeclObjC.cpp b/clang/lib/Sema/SemaDeclObjC.cpp
index cdfa6ad3f281a4359bf4b0da41233a50f1eefa9f..c3b95e168a605166c9f4a3e96f1297cd43ca64e4 100644
--- a/clang/lib/Sema/SemaDeclObjC.cpp
+++ b/clang/lib/Sema/SemaDeclObjC.cpp
@@ -296,7 +296,7 @@ static void DiagnoseObjCImplementedDeprecations(Sema &S, const NamedDecl *ND,
         RealizedPlatform = S.Context.getTargetInfo().getPlatformName();
       // Warn about implementing unavailable methods, unless the unavailable
       // is for an app extension.
-      if (RealizedPlatform.endswith("_app_extension"))
+      if (RealizedPlatform.ends_with("_app_extension"))
         return;
       S.Diag(ImplLoc, diag::warn_unavailable_def);
       S.Diag(ND->getLocation(), diag::note_method_declared_at)
diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp
index d629be083d8c38c2da587dd40210f19ee23deb86..c7185d56cc99739882d5f7a75aed0d2f81ba0a24 100644
--- a/clang/lib/Sema/SemaExpr.cpp
+++ b/clang/lib/Sema/SemaExpr.cpp
@@ -13808,12 +13808,12 @@ static void diagnoseXorMisusedAsPow(Sema &S, const ExprResult &XorLHS,
   StringRef RHSStrRef = RHSStr;
   // Do not diagnose literals with digit separators, binary, hexadecimal, octal
   // literals.
-  if (LHSStrRef.startswith("0b") || LHSStrRef.startswith("0B") ||
-      RHSStrRef.startswith("0b") || RHSStrRef.startswith("0B") ||
-      LHSStrRef.startswith("0x") || LHSStrRef.startswith("0X") ||
-      RHSStrRef.startswith("0x") || RHSStrRef.startswith("0X") ||
-      (LHSStrRef.size() > 1 && LHSStrRef.startswith("0")) ||
-      (RHSStrRef.size() > 1 && RHSStrRef.startswith("0")) ||
+  if (LHSStrRef.starts_with("0b") || LHSStrRef.starts_with("0B") ||
+      RHSStrRef.starts_with("0b") || RHSStrRef.starts_with("0B") ||
+      LHSStrRef.starts_with("0x") || LHSStrRef.starts_with("0X") ||
+      RHSStrRef.starts_with("0x") || RHSStrRef.starts_with("0X") ||
+      (LHSStrRef.size() > 1 && LHSStrRef.starts_with("0")) ||
+      (RHSStrRef.size() > 1 && RHSStrRef.starts_with("0")) ||
       LHSStrRef.contains('\'') || RHSStrRef.contains('\''))
     return;
 
@@ -15508,7 +15508,7 @@ static void checkObjCPointerIntrospection(Sema &S, ExprResult &L, ExprResult &R,
     if (const ObjCMessageExpr *ME = dyn_cast<ObjCMessageExpr>(Ex)) {
       Selector S = ME->getSelector();
       StringRef SelArg0 = S.getNameForSlot(0);
-      if (SelArg0.startswith("performSelector"))
+      if (SelArg0.starts_with("performSelector"))
         Diag = diag::warn_objc_pointer_masking_performSelector;
     }
 
diff --git a/clang/lib/Sema/SemaInit.cpp b/clang/lib/Sema/SemaInit.cpp
index 5ca6b232df66a560b97751c73f3c6ff87cb978d5..4028b2d642b21239c028fb3d2235994a23e06970 100644
--- a/clang/lib/Sema/SemaInit.cpp
+++ b/clang/lib/Sema/SemaInit.cpp
@@ -465,7 +465,8 @@ class InitListChecker {
   void FillInEmptyInitForField(unsigned Init, FieldDecl *Field,
                                const InitializedEntity &ParentEntity,
                                InitListExpr *ILE, bool &RequiresSecondPass,
-                               bool FillWithNoInit = false);
+                               bool FillWithNoInit = false,
+                               bool WarnIfMissing = false);
   void FillInEmptyInitializations(const InitializedEntity &Entity,
                                   InitListExpr *ILE, bool &RequiresSecondPass,
                                   InitListExpr *OuterILE, unsigned OuterIndex,
@@ -654,11 +655,16 @@ void InitListChecker::FillInEmptyInitForBase(
   }
 }
 
-void InitListChecker::FillInEmptyInitForField(unsigned Init, FieldDecl *Field,
-                                        const InitializedEntity &ParentEntity,
-                                              InitListExpr *ILE,
-                                              bool &RequiresSecondPass,
-                                              bool FillWithNoInit) {
+static bool hasAnyDesignatedInits(const InitListExpr *IL) {
+  return llvm::any_of(*IL, [=](const Stmt *Init) {
+    return isa_and_nonnull<DesignatedInitExpr>(Init);
+  });
+}
+
+void InitListChecker::FillInEmptyInitForField(
+    unsigned Init, FieldDecl *Field, const InitializedEntity &ParentEntity,
+    InitListExpr *ILE, bool &RequiresSecondPass, bool FillWithNoInit,
+    bool WarnIfMissing) {
   SourceLocation Loc = ILE->getEndLoc();
   unsigned NumInits = ILE->getNumInits();
   InitializedEntity MemberEntity
@@ -726,15 +732,52 @@ void InitListChecker::FillInEmptyInitForField(unsigned Init, FieldDecl *Field,
 
     if (hadError || VerifyOnly) {
       // Do nothing
-    } else if (Init < NumInits) {
-      ILE->setInit(Init, MemberInit.getAs<Expr>());
-    } else if (!isa<ImplicitValueInitExpr>(MemberInit.get())) {
-      // Empty initialization requires a constructor call, so
-      // extend the initializer list to include the constructor
-      // call and make a note that we'll need to take another pass
-      // through the initializer list.
-      ILE->updateInit(SemaRef.Context, Init, MemberInit.getAs<Expr>());
-      RequiresSecondPass = true;
+    } else {
+      if (WarnIfMissing) {
+        auto CheckAnonMember = [&](const FieldDecl *FD,
+                                   auto &&CheckAnonMember) -> FieldDecl * {
+          FieldDecl *Uninitialized = nullptr;
+          RecordDecl *RD = FD->getType()->getAsRecordDecl();
+          assert(RD && "Not anonymous member checked?");
+          for (auto *F : RD->fields()) {
+            FieldDecl *UninitializedFieldInF = nullptr;
+            if (F->isAnonymousStructOrUnion())
+              UninitializedFieldInF = CheckAnonMember(F, CheckAnonMember);
+            else if (!F->isUnnamedBitfield() &&
+                     !F->getType()->isIncompleteArrayType() &&
+                     !F->hasInClassInitializer())
+              UninitializedFieldInF = F;
+
+            if (RD->isUnion() && !UninitializedFieldInF)
+              return nullptr;
+            if (!Uninitialized)
+              Uninitialized = UninitializedFieldInF;
+          }
+          return Uninitialized;
+        };
+
+        FieldDecl *FieldToDiagnose = nullptr;
+        if (Field->isAnonymousStructOrUnion())
+          FieldToDiagnose = CheckAnonMember(Field, CheckAnonMember);
+        else if (!Field->isUnnamedBitfield() &&
+                 !Field->getType()->isIncompleteArrayType())
+          FieldToDiagnose = Field;
+
+        if (FieldToDiagnose)
+          SemaRef.Diag(Loc, diag::warn_missing_field_initializers)
+              << FieldToDiagnose;
+      }
+
+      if (Init < NumInits) {
+        ILE->setInit(Init, MemberInit.getAs<Expr>());
+      } else if (!isa<ImplicitValueInitExpr>(MemberInit.get())) {
+        // Empty initialization requires a constructor call, so
+        // extend the initializer list to include the constructor
+        // call and make a note that we'll need to take another pass
+        // through the initializer list.
+        ILE->updateInit(SemaRef.Context, Init, MemberInit.getAs<Expr>());
+        RequiresSecondPass = true;
+      }
     }
   } else if (InitListExpr *InnerILE
                = dyn_cast<InitListExpr>(ILE->getInit(Init))) {
@@ -802,9 +845,25 @@ InitListChecker::FillInEmptyInitializations(const InitializedEntity &Entity,
         }
       }
     } else {
+      InitListExpr *SForm =
+          ILE->isSyntacticForm() ? ILE : ILE->getSyntacticForm();
       // The fields beyond ILE->getNumInits() are default initialized, so in
       // order to leave them uninitialized, the ILE is expanded and the extra
       // fields are then filled with NoInitExpr.
+
+      // Some checks that are required for missing fields warning are bound to
+      // how many elements the initializer list originally was provided; perform
+      // them before the list is expanded.
+      bool WarnIfMissingField =
+          !SForm->isIdiomaticZeroInitializer(SemaRef.getLangOpts()) &&
+          ILE->getNumInits();
+
+      // Disable check for missing fields when designators are used in C to
+      // match gcc behaviour.
+      // FIXME: Should we emulate possible gcc warning bug?
+      WarnIfMissingField &=
+          SemaRef.getLangOpts().CPlusPlus || !hasAnyDesignatedInits(SForm);
+
       unsigned NumElems = numStructUnionElements(ILE->getType());
       if (!RDecl->isUnion() && RDecl->hasFlexibleArrayMember())
         ++NumElems;
@@ -832,7 +891,7 @@ InitListChecker::FillInEmptyInitializations(const InitializedEntity &Entity,
           return;
 
         FillInEmptyInitForField(Init, Field, Entity, ILE, RequiresSecondPass,
-                                FillWithNoInit);
+                                FillWithNoInit, WarnIfMissingField);
         if (hadError)
           return;
 
@@ -947,13 +1006,6 @@ InitListChecker::FillInEmptyInitializations(const InitializedEntity &Entity,
   }
 }
 
-static bool hasAnyDesignatedInits(const InitListExpr *IL) {
-  for (const Stmt *Init : *IL)
-    if (isa_and_nonnull<DesignatedInitExpr>(Init))
-      return true;
-  return false;
-}
-
 InitListChecker::InitListChecker(
     Sema &S, const InitializedEntity &Entity, InitListExpr *IL, QualType &T,
     bool VerifyOnly, bool TreatUnavailableAsInvalid, bool InOverloadResolution,
@@ -2225,12 +2277,8 @@ void InitListChecker::CheckStructUnionTypes(
   size_t NumRecordDecls = llvm::count_if(RD->decls(), [&](const Decl *D) {
     return isa<FieldDecl>(D) || isa<RecordDecl>(D);
   });
-  bool CheckForMissingFields =
-    !IList->isIdiomaticZeroInitializer(SemaRef.getLangOpts());
   bool HasDesignatedInit = false;
 
-  llvm::SmallPtrSet<FieldDecl *, 4> InitializedFields;
-
   while (Index < IList->getNumInits()) {
     Expr *Init = IList->getInit(Index);
     SourceLocation InitLoc = Init->getBeginLoc();
@@ -2254,24 +2302,17 @@ void InitListChecker::CheckStructUnionTypes(
 
       // Find the field named by the designated initializer.
       DesignatedInitExpr::Designator *D = DIE->getDesignator(0);
-      if (!VerifyOnly && D->isFieldDesignator()) {
+      if (!VerifyOnly && D->isFieldDesignator() && !DesignatedInitFailed) {
         FieldDecl *F = D->getFieldDecl();
-        InitializedFields.insert(F);
-        if (!DesignatedInitFailed) {
-          QualType ET = SemaRef.Context.getBaseElementType(F->getType());
-          if (checkDestructorReference(ET, InitLoc, SemaRef)) {
-            hadError = true;
-            return;
-          }
+        QualType ET = SemaRef.Context.getBaseElementType(F->getType());
+        if (checkDestructorReference(ET, InitLoc, SemaRef)) {
+          hadError = true;
+          return;
         }
       }
 
       InitializedSomething = true;
 
-      // Disable check for missing fields when designators are used.
-      // This matches gcc behaviour.
-      if (!SemaRef.getLangOpts().CPlusPlus)
-        CheckForMissingFields = false;
       continue;
     }
 
@@ -2350,7 +2391,6 @@ void InitListChecker::CheckStructUnionTypes(
     CheckSubElementType(MemberEntity, IList, Field->getType(), Index,
                         StructuredList, StructuredIndex);
     InitializedSomething = true;
-    InitializedFields.insert(*Field);
 
     if (RD->isUnion() && StructuredList) {
       // Initialize the first field within the union.
@@ -2360,28 +2400,6 @@ void InitListChecker::CheckStructUnionTypes(
     ++Field;
   }
 
-  // Emit warnings for missing struct field initializers.
-  if (!VerifyOnly && InitializedSomething && CheckForMissingFields &&
-      !RD->isUnion()) {
-    // It is possible we have one or more unnamed bitfields remaining.
-    // Find first (if any) named field and emit warning.
-    for (RecordDecl::field_iterator it = HasDesignatedInit ? RD->field_begin()
-                                                           : Field,
-                                    end = RD->field_end();
-         it != end; ++it) {
-      if (HasDesignatedInit && InitializedFields.count(*it))
-        continue;
-
-      if (!it->isUnnamedBitfield() && !it->hasInClassInitializer() &&
-          !it->getType()->isIncompleteArrayType()) {
-        SemaRef.Diag(IList->getSourceRange().getEnd(),
-                     diag::warn_missing_field_initializers)
-            << *it;
-        break;
-      }
-    }
-  }
-
   // Check that any remaining fields can be value-initialized if we're not
   // building a structured list. (If we are, we'll check this later.)
   if (!StructuredList && Field != FieldEnd && !RD->isUnion() &&
diff --git a/clang/lib/Sema/SemaModule.cpp b/clang/lib/Sema/SemaModule.cpp
index 9282ceb8dee03de050315306ade7710f0f40f97d..db0cbd5ec6d6ca6c9efe0ebe8b7943bf21844dec 100644
--- a/clang/lib/Sema/SemaModule.cpp
+++ b/clang/lib/Sema/SemaModule.cpp
@@ -268,7 +268,7 @@ Sema::ActOnModuleDecl(SourceLocation StartLoc, SourceLocation ModuleLoc,
   StringRef FirstComponentName = Path[0].first->getName();
   if (!getSourceManager().isInSystemHeader(Path[0].second) &&
       (FirstComponentName == "std" ||
-       (FirstComponentName.startswith("std") &&
+       (FirstComponentName.starts_with("std") &&
         llvm::all_of(FirstComponentName.drop_front(3), &llvm::isDigit))))
     Diag(Path[0].second, diag::warn_reserved_module_name) << Path[0].first;
 
diff --git a/clang/lib/Sema/SemaRISCVVectorLookup.cpp b/clang/lib/Sema/SemaRISCVVectorLookup.cpp
index 0d411fca0f9c8275489fb476c1850714f49d5b44..e4642e4da016a44d3c770a178f48c56784ebd2d6 100644
--- a/clang/lib/Sema/SemaRISCVVectorLookup.cpp
+++ b/clang/lib/Sema/SemaRISCVVectorLookup.cpp
@@ -43,7 +43,7 @@ struct RVVIntrinsicDef {
 
 struct RVVOverloadIntrinsicDef {
   // Indexes of RISCVIntrinsicManagerImpl::IntrinsicList.
-  SmallVector<size_t, 8> Indexes;
+  SmallVector<uint32_t, 8> Indexes;
 };
 
 } // namespace
@@ -162,7 +162,7 @@ private:
   // List of all RVV intrinsic.
   std::vector<RVVIntrinsicDef> IntrinsicList;
   // Mapping function name to index of IntrinsicList.
-  StringMap<size_t> Intrinsics;
+  StringMap<uint32_t> Intrinsics;
   // Mapping function name to RVVOverloadIntrinsicDef.
   StringMap<RVVOverloadIntrinsicDef> OverloadIntrinsics;
 
@@ -174,7 +174,7 @@ private:
 
   // Create FunctionDecl for a vector intrinsic.
   void CreateRVVIntrinsicDecl(LookupResult &LR, IdentifierInfo *II,
-                              Preprocessor &PP, unsigned Index,
+                              Preprocessor &PP, uint32_t Index,
                               bool IsOverload);
 
   void ConstructRVVIntrinsics(ArrayRef<RVVIntrinsicRecord> Recs,
@@ -386,7 +386,7 @@ void RISCVIntrinsicManagerImpl::InitRVVIntrinsic(
                                      Record.HasFRMRoundModeOp);
 
   // Put into IntrinsicList.
-  size_t Index = IntrinsicList.size();
+  uint32_t Index = IntrinsicList.size();
   IntrinsicList.push_back({BuiltinName, Signature});
 
   // Creating mapping to Intrinsics.
@@ -403,7 +403,7 @@ void RISCVIntrinsicManagerImpl::InitRVVIntrinsic(
 void RISCVIntrinsicManagerImpl::CreateRVVIntrinsicDecl(LookupResult &LR,
                                                        IdentifierInfo *II,
                                                        Preprocessor &PP,
-                                                       unsigned Index,
+                                                       uint32_t Index,
                                                        bool IsOverload) {
   ASTContext &Context = S.Context;
   RVVIntrinsicDef &IDef = IntrinsicList[Index];
diff --git a/clang/lib/Sema/SemaStmtAttr.cpp b/clang/lib/Sema/SemaStmtAttr.cpp
index 725d8efe3828d659dbbeb58b91571a1453db50c3..0d0a7bcebab4e89279a67008378bf740c5665ddb 100644
--- a/clang/lib/Sema/SemaStmtAttr.cpp
+++ b/clang/lib/Sema/SemaStmtAttr.cpp
@@ -53,6 +53,13 @@ static Attr *handleFallThroughAttr(Sema &S, Stmt *St, const ParsedAttr &A,
 
 static Attr *handleSuppressAttr(Sema &S, Stmt *St, const ParsedAttr &A,
                                 SourceRange Range) {
+  if (A.getAttributeSpellingListIndex() == SuppressAttr::CXX11_gsl_suppress &&
+      A.getNumArgs() < 1) {
+    // Suppression attribute with GSL spelling requires at least 1 argument.
+    S.Diag(A.getLoc(), diag::err_attribute_too_few_arguments) << A << 1;
+    return nullptr;
+  }
+
   std::vector<StringRef> DiagnosticIdentifiers;
   for (unsigned I = 0, E = A.getNumArgs(); I != E; ++I) {
     StringRef RuleName;
@@ -60,8 +67,6 @@ static Attr *handleSuppressAttr(Sema &S, Stmt *St, const ParsedAttr &A,
     if (!S.checkStringLiteralArgumentAttr(A, I, RuleName, nullptr))
       return nullptr;
 
-    // FIXME: Warn if the rule name is unknown. This is tricky because only
-    // clang-tidy knows about available rules.
     DiagnosticIdentifiers.push_back(RuleName);
   }
 
diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp
index f10abeaba0d4517ed5df60d4fbaae7919b7d829a..cca7d61306156455b5a6092754f55139e714e1dd 100644
--- a/clang/lib/Sema/SemaTemplate.cpp
+++ b/clang/lib/Sema/SemaTemplate.cpp
@@ -39,6 +39,7 @@
 #include "llvm/ADT/SmallBitVector.h"
 #include "llvm/ADT/SmallString.h"
 #include "llvm/ADT/StringExtras.h"
+#include "llvm/Support/SaveAndRestore.h"
 
 #include <iterator>
 #include <optional>
@@ -3990,9 +3991,14 @@ QualType Sema::CheckTemplateIdType(TemplateName Name,
     if (Inst.isInvalid())
       return QualType();
 
-    CanonType = SubstType(Pattern->getUnderlyingType(),
-                          TemplateArgLists, AliasTemplate->getLocation(),
-                          AliasTemplate->getDeclName());
+    {
+      Sema::ContextRAII SavedContext(*this, Pattern->getDeclContext());
+      if (RebuildingTypesInCurrentInstantiation)
+        SavedContext.pop();
+      CanonType =
+          SubstType(Pattern->getUnderlyingType(), TemplateArgLists,
+                    AliasTemplate->getLocation(), AliasTemplate->getDeclName());
+    }
     if (CanonType.isNull()) {
       // If this was enable_if and we failed to find the nested type
       // within enable_if in a SFINAE context, dig out the specific
@@ -11392,6 +11398,8 @@ TypeSourceInfo *Sema::RebuildTypeInCurrentInstantiation(TypeSourceInfo *T,
   if (!T || !T->getType()->isInstantiationDependentType())
     return T;
 
+  llvm::SaveAndRestore DisableContextSwitchForTypeAliases(
+      RebuildingTypesInCurrentInstantiation, true);
   CurrentInstantiationRebuilder Rebuilder(*this, Loc, Name);
   return Rebuilder.TransformType(T);
 }
diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp
index 83610503ed9b16b12089c54dc5e8766ef5272bbc..a376f20fa4f4e08540b3a86ac5a9ac1e96f88b6d 100644
--- a/clang/lib/Sema/SemaType.cpp
+++ b/clang/lib/Sema/SemaType.cpp
@@ -5371,7 +5371,7 @@ static TypeSourceInfo *GetFullTypeForDeclarator(TypeProcessingState &state,
             !(D.getIdentifier() &&
               ((D.getIdentifier()->getName() == "printf" &&
                 LangOpts.getOpenCLCompatibleVersion() >= 120) ||
-               D.getIdentifier()->getName().startswith("__")))) {
+               D.getIdentifier()->getName().starts_with("__")))) {
           S.Diag(D.getIdentifierLoc(), diag::err_opencl_variadic_function);
           D.setInvalidType(true);
         }
@@ -8360,12 +8360,25 @@ static void HandleNeonVectorTypeAttr(QualType &CurType, const ParsedAttr &Attr,
   // not to need a separate attribute)
   if (!(S.Context.getTargetInfo().hasFeature("neon") ||
         S.Context.getTargetInfo().hasFeature("mve") ||
-        IsTargetCUDAAndHostARM)) {
+        S.Context.getTargetInfo().hasFeature("sve") ||
+        S.Context.getTargetInfo().hasFeature("sme") ||
+        IsTargetCUDAAndHostARM) &&
+      VecKind == VectorKind::Neon) {
+    S.Diag(Attr.getLoc(), diag::err_attribute_unsupported)
+        << Attr << "'neon', 'mve', 'sve' or 'sme'";
+    Attr.setInvalid();
+    return;
+  }
+  if (!(S.Context.getTargetInfo().hasFeature("neon") ||
+        S.Context.getTargetInfo().hasFeature("mve") ||
+        IsTargetCUDAAndHostARM) &&
+      VecKind == VectorKind::NeonPoly) {
     S.Diag(Attr.getLoc(), diag::err_attribute_unsupported)
         << Attr << "'neon' or 'mve'";
     Attr.setInvalid();
     return;
   }
+
   // Check the attribute arguments.
   if (Attr.getNumArgs() != 1) {
     S.Diag(Attr.getLoc(), diag::err_attribute_wrong_number_arguments)
diff --git a/clang/lib/StaticAnalyzer/Checkers/CheckObjCDealloc.cpp b/clang/lib/StaticAnalyzer/Checkers/CheckObjCDealloc.cpp
index bd6655cc1e3fa352daf2e3a2145f5a8357cd36b2..fedc6db3723aac815aba0f6c1e60376868a53b7c 100644
--- a/clang/lib/StaticAnalyzer/Checkers/CheckObjCDealloc.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/CheckObjCDealloc.cpp
@@ -1045,8 +1045,8 @@ bool ObjCDeallocChecker::isReleasedByCIFilterDealloc(
   StringRef IvarName = PropImpl->getPropertyIvarDecl()->getName();
 
   const char *ReleasePrefix = "input";
-  if (!(PropName.startswith(ReleasePrefix) ||
-        IvarName.startswith(ReleasePrefix))) {
+  if (!(PropName.starts_with(ReleasePrefix) ||
+        IvarName.starts_with(ReleasePrefix))) {
     return false;
   }
 
diff --git a/clang/lib/StaticAnalyzer/Checkers/CheckSecuritySyntaxOnly.cpp b/clang/lib/StaticAnalyzer/Checkers/CheckSecuritySyntaxOnly.cpp
index dbba12bb4355c3a48b20a523d75f44b232e4deea..afc5e6b48008d869d0d1077df33f04abcc1c8f24 100644
--- a/clang/lib/StaticAnalyzer/Checkers/CheckSecuritySyntaxOnly.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/CheckSecuritySyntaxOnly.cpp
@@ -140,7 +140,7 @@ void WalkAST::VisitCallExpr(CallExpr *CE) {
   if (!II)   // if no identifier, not a simple C function
     return;
   StringRef Name = II->getName();
-  if (Name.startswith("__builtin_"))
+  if (Name.starts_with("__builtin_"))
     Name = Name.substr(10);
 
   // Set the evaluation function by switching on the callee name.
@@ -763,7 +763,7 @@ void WalkAST::checkDeprecatedOrUnsafeBufferHandling(const CallExpr *CE,
   enum { DEPR_ONLY = -1, UNKNOWN_CALL = -2 };
 
   StringRef Name = FD->getIdentifier()->getName();
-  if (Name.startswith("__builtin_"))
+  if (Name.starts_with("__builtin_"))
     Name = Name.substr(10);
 
   int ArgIndex =
diff --git a/clang/lib/StaticAnalyzer/Checkers/DeadStoresChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/DeadStoresChecker.cpp
index 5f44c9476928dd827ccff91267835437f165be85..86f446fc411ca2bdccd1e044f60f7525790ad133 100644
--- a/clang/lib/StaticAnalyzer/Checkers/DeadStoresChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/DeadStoresChecker.cpp
@@ -183,7 +183,7 @@ public:
 
     // Files autogenerated by DriverKit IIG contain some dead stores that
     // we don't want to report.
-    if (Data.startswith("/* iig"))
+    if (Data.starts_with("/* iig"))
       return true;
 
     return false;
diff --git a/clang/lib/StaticAnalyzer/Checkers/GCDAntipatternChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/GCDAntipatternChecker.cpp
index 8e02ef74c66863ef9ebe2a42ade8946681070eed..5637941a58f0260053fc4331563fab1a3ef61f7e 100644
--- a/clang/lib/StaticAnalyzer/Checkers/GCDAntipatternChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/GCDAntipatternChecker.cpp
@@ -73,7 +73,7 @@ decltype(auto) bindAssignmentToDecl(const char *DeclName) {
 static bool isTest(const Decl *D) {
   if (const auto* ND = dyn_cast<NamedDecl>(D)) {
     std::string DeclName = ND->getNameAsString();
-    if (StringRef(DeclName).startswith("test"))
+    if (StringRef(DeclName).starts_with("test"))
       return true;
   }
   if (const auto *OD = dyn_cast<ObjCMethodDecl>(D)) {
diff --git a/clang/lib/StaticAnalyzer/Checkers/LocalizationChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/LocalizationChecker.cpp
index b77e9bf09a33204c57dd52275b3f129bb1299a39..70f911fc66abca8625a384f97e28a7568d80bf32 100644
--- a/clang/lib/StaticAnalyzer/Checkers/LocalizationChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/LocalizationChecker.cpp
@@ -817,9 +817,9 @@ void NonLocalizedStringChecker::checkPreObjCMessage(const ObjCMethodCall &msg,
     // Handle the case where the receiver is an NSString
     // These special NSString methods draw to the screen
 
-    if (!(SelectorName.startswith("drawAtPoint") ||
-          SelectorName.startswith("drawInRect") ||
-          SelectorName.startswith("drawWithRect")))
+    if (!(SelectorName.starts_with("drawAtPoint") ||
+          SelectorName.starts_with("drawInRect") ||
+          SelectorName.starts_with("drawWithRect")))
       return;
 
     SVal svTitle = msg.getReceiverSVal();
diff --git a/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp
index c5e4add501886a4d4b654d89d972c210ccba092e..79ab05f2c7866aa2a45e81f2b7b4295f3b01cc4e 100644
--- a/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/MallocChecker.cpp
@@ -3150,16 +3150,16 @@ bool MallocChecker::mayFreeAnyEscapedMemoryOrIsModeledExplicitly(
     // transferred. Again, though, we can't be sure that the object will use
     // free() to deallocate the memory, so we can't model it explicitly.
     StringRef FirstSlot = Msg->getSelector().getNameForSlot(0);
-    if (FirstSlot.endswith("NoCopy"))
+    if (FirstSlot.ends_with("NoCopy"))
       return true;
 
     // If the first selector starts with addPointer, insertPointer,
     // or replacePointer, assume we are dealing with NSPointerArray or similar.
     // This is similar to C++ containers (vector); we still might want to check
     // that the pointers get freed by following the container itself.
-    if (FirstSlot.startswith("addPointer") ||
-        FirstSlot.startswith("insertPointer") ||
-        FirstSlot.startswith("replacePointer") ||
+    if (FirstSlot.starts_with("addPointer") ||
+        FirstSlot.starts_with("insertPointer") ||
+        FirstSlot.starts_with("replacePointer") ||
         FirstSlot.equals("valueWithPointer")) {
       return true;
     }
@@ -3199,7 +3199,7 @@ bool MallocChecker::mayFreeAnyEscapedMemoryOrIsModeledExplicitly(
 
   // White list the 'XXXNoCopy' CoreFoundation functions.
   // We specifically check these before
-  if (FName.endswith("NoCopy")) {
+  if (FName.ends_with("NoCopy")) {
     // Look for the deallocator argument. We know that the memory ownership
     // is not transferred only if the deallocator argument is
     // 'kCFAllocatorNull'.
diff --git a/clang/lib/StaticAnalyzer/Checkers/NullabilityChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/NullabilityChecker.cpp
index 627b51af6bd44af45a43c8e4e89e41cf3af75f14..06f1ad00eaf20ddd6e3fbb9533db127c7ffe2c16 100644
--- a/clang/lib/StaticAnalyzer/Checkers/NullabilityChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/NullabilityChecker.cpp
@@ -890,7 +890,7 @@ void NullabilityChecker::checkPostCall(const CallEvent &Call,
   // of CG calls.
   const SourceManager &SM = C.getSourceManager();
   StringRef FilePath = SM.getFilename(SM.getSpellingLoc(Decl->getBeginLoc()));
-  if (llvm::sys::path::filename(FilePath).startswith("CG")) {
+  if (llvm::sys::path::filename(FilePath).starts_with("CG")) {
     State = State->set<NullabilityMap>(Region, Nullability::Contradicted);
     C.addTransition(State);
     return;
@@ -992,7 +992,7 @@ void NullabilityChecker::checkPostObjCMessage(const ObjCMethodCall &M,
   // In order to reduce the noise in the diagnostics generated by this checker,
   // some framework and programming style based heuristics are used. These
   // heuristics are for Cocoa APIs which have NS prefix.
-  if (Name.startswith("NS")) {
+  if (Name.starts_with("NS")) {
     // Developers rely on dynamic invariants such as an item should be available
     // in a collection, or a collection is not empty often. Those invariants can
     // not be inferred by any static analysis tool. To not to bother the users
diff --git a/clang/lib/StaticAnalyzer/Checkers/ObjCPropertyChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/ObjCPropertyChecker.cpp
index 4636fd1605118f776c5afb51f82b26e9287360ba..08ad6877cbe6b6523417caf8f36f8c78f11c1ebf 100644
--- a/clang/lib/StaticAnalyzer/Checkers/ObjCPropertyChecker.cpp
+++ b/clang/lib/StaticAnalyzer/Checkers/ObjCPropertyChecker.cpp
@@ -50,7 +50,7 @@ void ObjCPropertyChecker::checkCopyMutable(const ObjCPropertyDecl *D,
   const std::string &PropTypeName(T->getPointeeType().getCanonicalType()
                                                      .getUnqualifiedType()
                                                      .getAsString());
-  if (!StringRef(PropTypeName).startswith("NSMutable"))
+  if (!StringRef(PropTypeName).starts_with("NSMutable"))
     return;
 
   const ObjCImplDecl *ImplD = nullptr;
diff --git a/clang/lib/StaticAnalyzer/Core/BugReporter.cpp b/clang/lib/StaticAnalyzer/Core/BugReporter.cpp
index 9532254e3c459681bef893d7f4905fd5040d778a..f3e0a5f9f314aded8b40e30a8a919d1ab6d400dd 100644
--- a/clang/lib/StaticAnalyzer/Core/BugReporter.cpp
+++ b/clang/lib/StaticAnalyzer/Core/BugReporter.cpp
@@ -12,12 +12,15 @@
 //===----------------------------------------------------------------------===//
 
 #include "clang/StaticAnalyzer/Core/BugReporter/BugReporter.h"
+#include "clang/AST/ASTTypeTraits.h"
+#include "clang/AST/Attr.h"
 #include "clang/AST/Decl.h"
 #include "clang/AST/DeclBase.h"
 #include "clang/AST/DeclObjC.h"
 #include "clang/AST/Expr.h"
 #include "clang/AST/ExprCXX.h"
 #include "clang/AST/ParentMap.h"
+#include "clang/AST/ParentMapContext.h"
 #include "clang/AST/Stmt.h"
 #include "clang/AST/StmtCXX.h"
 #include "clang/AST/StmtObjC.h"
@@ -2139,15 +2142,14 @@ PathSensitiveBugReport::PathSensitiveBugReport(
          "checkers to emit warnings, because checkers should depend on "
          "*modeling*, not *diagnostics*.");
 
-  assert(
-      (bt.getCheckerName().startswith("debug") ||
-       !isHidden(ErrorNode->getState()
-                     ->getAnalysisManager()
-                     .getCheckerManager()
-                     ->getCheckerRegistryData(),
-                 bt.getCheckerName())) &&
-          "Hidden checkers musn't emit diagnostics as they are by definition "
-          "non-user facing!");
+  assert((bt.getCheckerName().starts_with("debug") ||
+          !isHidden(ErrorNode->getState()
+                        ->getAnalysisManager()
+                        .getCheckerManager()
+                        ->getCheckerRegistryData(),
+                    bt.getCheckerName())) &&
+         "Hidden checkers musn't emit diagnostics as they are by definition "
+         "non-user facing!");
 }
 
 void PathSensitiveBugReport::addVisitor(
@@ -2425,6 +2427,12 @@ PathSensitiveBugReport::getLocation() const {
   }
 
   if (S) {
+    // Attributed statements usually have corrupted begin locations,
+    // it's OK to ignore attributes for our purposes and deal with
+    // the actual annotated statement.
+    if (const auto *AS = dyn_cast<AttributedStmt>(S))
+      S = AS->getSubStmt();
+
     // For member expressions, return the location of the '.' or '->'.
     if (const auto *ME = dyn_cast<MemberExpr>(S))
       return PathDiagnosticLocation::createMemberLoc(ME, SM);
@@ -2897,6 +2905,10 @@ void BugReporter::emitReport(std::unique_ptr<BugReport> R) {
   if (!ValidSourceLoc)
     return;
 
+  // If the user asked to suppress this report, we should skip it.
+  if (UserSuppressions.isSuppressed(*R))
+    return;
+
   // Compute the bug report's hash to determine its equivalence class.
   llvm::FoldingSetNodeID ID;
   R->Profile(ID);
@@ -3064,8 +3076,7 @@ void BugReporter::FlushReport(BugReportEquivClass& EQ) {
   // See whether we need to silence the checker/package.
   for (const std::string &CheckerOrPackage :
        getAnalyzerOptions().SilencedCheckersAndPackages) {
-    if (report->getBugType().getCheckerName().startswith(
-            CheckerOrPackage))
+    if (report->getBugType().getCheckerName().starts_with(CheckerOrPackage))
       return;
   }
 
diff --git a/clang/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp b/clang/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp
index 4a9d130c240aec307dce9fc4189a257b9f3645df..2f9965036b9ef91fe9181c50fad3c88269f5e739 100644
--- a/clang/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp
+++ b/clang/lib/StaticAnalyzer/Core/BugReporterVisitors.cpp
@@ -3372,7 +3372,7 @@ void LikelyFalsePositiveSuppressionBRVisitor::finalizeVisitor(
   FullSourceLoc Loc = BR.getLocation().asLocation();
   while (Loc.isMacroID()) {
     Loc = Loc.getSpellingLoc();
-    if (SM.getFilename(Loc).endswith("sys/queue.h")) {
+    if (SM.getFilename(Loc).ends_with("sys/queue.h")) {
       BR.markInvalid(getTag(), nullptr);
       return;
     }
diff --git a/clang/lib/StaticAnalyzer/Core/BugSuppression.cpp b/clang/lib/StaticAnalyzer/Core/BugSuppression.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..b5991e47a5388752c4a6318fd89d6e1610d8c3df
--- /dev/null
+++ b/clang/lib/StaticAnalyzer/Core/BugSuppression.cpp
@@ -0,0 +1,169 @@
+//===- BugSuppression.cpp - Suppression interface -------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/StaticAnalyzer/Core/BugReporter/BugSuppression.h"
+#include "clang/AST/RecursiveASTVisitor.h"
+#include "clang/StaticAnalyzer/Core/BugReporter/BugReporter.h"
+
+using namespace clang;
+using namespace ento;
+
+namespace {
+
+using Ranges = llvm::SmallVectorImpl<SourceRange>;
+
+inline bool hasSuppression(const Decl *D) {
+  // FIXME: Implement diagnostic identifier arguments
+  // (checker names, "hashtags").
+  if (const auto *Suppression = D->getAttr<SuppressAttr>())
+    return !Suppression->isGSL() &&
+           (Suppression->diagnosticIdentifiers().empty());
+  return false;
+}
+inline bool hasSuppression(const AttributedStmt *S) {
+  // FIXME: Implement diagnostic identifier arguments
+  // (checker names, "hashtags").
+  return llvm::any_of(S->getAttrs(), [](const Attr *A) {
+    const auto *Suppression = dyn_cast<SuppressAttr>(A);
+    return Suppression && !Suppression->isGSL() &&
+           (Suppression->diagnosticIdentifiers().empty());
+  });
+}
+
+template <class NodeType> inline SourceRange getRange(const NodeType *Node) {
+  return Node->getSourceRange();
+}
+template <> inline SourceRange getRange(const AttributedStmt *S) {
+  // Begin location for attributed statement node seems to be ALWAYS invalid.
+  //
+  // It is unlikely that we ever report any warnings on suppression
+  // attribute itself, but even if we do, we wouldn't want that warning
+  // to be suppressed by that same attribute.
+  //
+  // Long story short, we can use inner statement and it's not going to break
+  // anything.
+  return getRange(S->getSubStmt());
+}
+
+inline bool isLessOrEqual(SourceLocation LHS, SourceLocation RHS,
+                          const SourceManager &SM) {
+  // SourceManager::isBeforeInTranslationUnit tests for strict
+  // inequality, when we need a non-strict comparison (bug
+  // can be reported directly on the annotated note).
+  // For this reason, we use the following equivalence:
+  //
+  //   A <= B <==> !(B < A)
+  //
+  return !SM.isBeforeInTranslationUnit(RHS, LHS);
+}
+
+inline bool fullyContains(SourceRange Larger, SourceRange Smaller,
+                          const SourceManager &SM) {
+  // Essentially this means:
+  //
+  //   Larger.fullyContains(Smaller)
+  //
+  // However, that method has a very trivial implementation and couldn't
+  // compare regular locations and locations from macro expansions.
+  // We could've converted everything into regular locations as a solution,
+  // but the following solution seems to be the most bulletproof.
+  return isLessOrEqual(Larger.getBegin(), Smaller.getBegin(), SM) &&
+         isLessOrEqual(Smaller.getEnd(), Larger.getEnd(), SM);
+}
+
+class CacheInitializer : public RecursiveASTVisitor<CacheInitializer> {
+public:
+  static void initialize(const Decl *D, Ranges &ToInit) {
+    CacheInitializer(ToInit).TraverseDecl(const_cast<Decl *>(D));
+  }
+
+  bool VisitVarDecl(VarDecl *VD) {
+    // Bug location could be somewhere in the init value of
+    // a freshly declared variable.  Even though it looks like the
+    // user applied attribute to a statement, it will apply to a
+    // variable declaration, and this is where we check for it.
+    return VisitAttributedNode(VD);
+  }
+
+  bool VisitAttributedStmt(AttributedStmt *AS) {
+    // When we apply attributes to statements, it actually creates
+    // a wrapper statement that only contains attributes and the wrapped
+    // statement.
+    return VisitAttributedNode(AS);
+  }
+
+private:
+  template <class NodeType> bool VisitAttributedNode(NodeType *Node) {
+    if (hasSuppression(Node)) {
+      // TODO: In the future, when we come up with good stable IDs for checkers
+      //       we can return a list of kinds to ignore, or all if no arguments
+      //       were provided.
+      addRange(getRange(Node));
+    }
+    // We should keep traversing AST.
+    return true;
+  }
+
+  void addRange(SourceRange R) {
+    if (R.isValid()) {
+      Result.push_back(R);
+    }
+  }
+
+  CacheInitializer(Ranges &R) : Result(R) {}
+  Ranges &Result;
+};
+
+} // end anonymous namespace
+
+// TODO: Introduce stable IDs for checkers and check for those here
+//       to be more specific.  Attribute without arguments should still
+//       be considered as "suppress all".
+//       It is already much finer granularity than what we have now
+//       (i.e. removing the whole function from the analysis).
+bool BugSuppression::isSuppressed(const BugReport &R) {
+  PathDiagnosticLocation Location = R.getLocation();
+  PathDiagnosticLocation UniqueingLocation = R.getUniqueingLocation();
+  const Decl *DeclWithIssue = R.getDeclWithIssue();
+
+  return isSuppressed(Location, DeclWithIssue, {}) ||
+         isSuppressed(UniqueingLocation, DeclWithIssue, {});
+}
+
+bool BugSuppression::isSuppressed(const PathDiagnosticLocation &Location,
+                                  const Decl *DeclWithIssue,
+                                  DiagnosticIdentifierList Hashtags) {
+  if (!Location.isValid() || DeclWithIssue == nullptr)
+    return false;
+
+  // While some warnings are attached to AST nodes (mostly path-sensitive
+  // checks), others are simply associated with a plain source location
+  // or range.  Figuring out the node based on locations can be tricky,
+  // so instead, we traverse the whole body of the declaration and gather
+  // information on ALL suppressions.  After that we can simply check if
+  // any of those suppressions affect the warning in question.
+  //
+  // Traversing AST of a function is not a heavy operation, but for
+  // large functions with a lot of bugs it can make a dent in performance.
+  // In order to avoid this scenario, we cache traversal results.
+  auto InsertionResult = CachedSuppressionLocations.insert(
+      std::make_pair(DeclWithIssue, CachedRanges{}));
+  Ranges &SuppressionRanges = InsertionResult.first->second;
+  if (InsertionResult.second) {
+    // We haven't checked this declaration for suppressions yet!
+    CacheInitializer::initialize(DeclWithIssue, SuppressionRanges);
+  }
+
+  SourceRange BugRange = Location.asRange();
+  const SourceManager &SM = Location.getManager();
+
+  return llvm::any_of(SuppressionRanges,
+                      [BugRange, &SM](SourceRange Suppression) {
+                        return fullyContains(Suppression, BugRange, SM);
+                      });
+}
diff --git a/clang/lib/StaticAnalyzer/Core/CMakeLists.txt b/clang/lib/StaticAnalyzer/Core/CMakeLists.txt
index 3df93374dada753bf1edaf821f532bace6c63de2..8672876c0608d0de06d81785081e309ea2765c2a 100644
--- a/clang/lib/StaticAnalyzer/Core/CMakeLists.txt
+++ b/clang/lib/StaticAnalyzer/Core/CMakeLists.txt
@@ -11,6 +11,7 @@ add_clang_library(clangStaticAnalyzerCore
   BlockCounter.cpp
   BugReporter.cpp
   BugReporterVisitors.cpp
+  BugSuppression.cpp
   CallDescription.cpp
   CallEvent.cpp
   Checker.cpp
diff --git a/clang/lib/StaticAnalyzer/Core/CallEvent.cpp b/clang/lib/StaticAnalyzer/Core/CallEvent.cpp
index d004c12bf2c1e8b36d372badf6041ea6c78a5e68..0ac1d91b79beb5ac2d00838a45b2097f15b8fe32 100644
--- a/clang/lib/StaticAnalyzer/Core/CallEvent.cpp
+++ b/clang/lib/StaticAnalyzer/Core/CallEvent.cpp
@@ -659,17 +659,17 @@ bool AnyFunctionCall::argumentsMayEscape() const {
 
   // - CoreFoundation functions that end with "NoCopy" can free a passed-in
   //   buffer even if it is const.
-  if (FName.endswith("NoCopy"))
+  if (FName.ends_with("NoCopy"))
     return true;
 
   // - NSXXInsertXX, for example NSMapInsertIfAbsent, since they can
   //   be deallocated by NSMapRemove.
-  if (FName.startswith("NS") && FName.contains("Insert"))
+  if (FName.starts_with("NS") && FName.contains("Insert"))
     return true;
 
   // - Many CF containers allow objects to escape through custom
   //   allocators/deallocators upon container construction. (PR12101)
-  if (FName.startswith("CF") || FName.startswith("CG")) {
+  if (FName.starts_with("CF") || FName.starts_with("CG")) {
     return StrInStrNoCase(FName, "InsertValue")  != StringRef::npos ||
            StrInStrNoCase(FName, "AddValue")     != StringRef::npos ||
            StrInStrNoCase(FName, "SetValue")     != StringRef::npos ||
diff --git a/clang/lib/StaticAnalyzer/Core/CheckerContext.cpp b/clang/lib/StaticAnalyzer/Core/CheckerContext.cpp
index c25165cce1287650a9b64f0e35b37edc231f948c..d6d4cec9dd3d4d01dd5ab2ffb9fbbeab337e4ddb 100644
--- a/clang/lib/StaticAnalyzer/Core/CheckerContext.cpp
+++ b/clang/lib/StaticAnalyzer/Core/CheckerContext.cpp
@@ -105,10 +105,11 @@ bool CheckerContext::isCLibraryFunction(const FunctionDecl *FD,
   if (FName.equals(Name))
     return true;
 
-  if (FName.startswith("__inline") && FName.contains(Name))
+  if (FName.starts_with("__inline") && FName.contains(Name))
     return true;
 
-  if (FName.startswith("__") && FName.endswith("_chk") && FName.contains(Name))
+  if (FName.starts_with("__") && FName.ends_with("_chk") &&
+      FName.contains(Name))
     return true;
 
   return false;
diff --git a/clang/lib/StaticAnalyzer/Core/CheckerRegistryData.cpp b/clang/lib/StaticAnalyzer/Core/CheckerRegistryData.cpp
index 1b3e8b11549dd9d5b029649e6b94adcd52d8b971..b9c6278991f43f6a8092390a7e2b83106fb9bd89 100644
--- a/clang/lib/StaticAnalyzer/Core/CheckerRegistryData.cpp
+++ b/clang/lib/StaticAnalyzer/Core/CheckerRegistryData.cpp
@@ -82,7 +82,7 @@ static constexpr char PackageSeparator = '.';
 
 static bool isInPackage(const CheckerInfo &Checker, StringRef PackageName) {
   // Does the checker's full name have the package as a prefix?
-  if (!Checker.FullName.startswith(PackageName))
+  if (!Checker.FullName.starts_with(PackageName))
     return false;
 
   // Is the package actually just the name of a specific checker?
@@ -158,7 +158,7 @@ void CheckerRegistryData::printCheckerWithDescList(
       continue;
     }
 
-    if (Checker.FullName.startswith("alpha")) {
+    if (Checker.FullName.starts_with("alpha")) {
       if (AnOpts.ShowCheckerHelpAlpha)
         Print(Out, Checker,
               ("(Enable only for development!) " + Checker.Desc).str());
@@ -228,7 +228,7 @@ void CheckerRegistryData::printCheckerOptionList(const AnalyzerOptions &AnOpts,
     }
 
     if (Option.DevelopmentStatus == "alpha" ||
-        Entry.first.startswith("alpha")) {
+        Entry.first.starts_with("alpha")) {
       if (AnOpts.ShowCheckerOptionAlphaList)
         Print(Out, FullOption,
               llvm::Twine("(Enable only for development!) " + Desc).str());
diff --git a/clang/lib/StaticAnalyzer/Frontend/AnalysisConsumer.cpp b/clang/lib/StaticAnalyzer/Frontend/AnalysisConsumer.cpp
index 142acab7cd081d4ed884d7aaa3623a319eec6443..b6ef40595e3c97a5e218516f84d5d325263a7540 100644
--- a/clang/lib/StaticAnalyzer/Frontend/AnalysisConsumer.cpp
+++ b/clang/lib/StaticAnalyzer/Frontend/AnalysisConsumer.cpp
@@ -308,7 +308,7 @@ public:
 
   bool VisitFunctionDecl(FunctionDecl *FD) {
     IdentifierInfo *II = FD->getIdentifier();
-    if (II && II->getName().startswith("__inline"))
+    if (II && II->getName().starts_with("__inline"))
       return true;
 
     // We skip function template definitions, as their semantics is
diff --git a/clang/lib/StaticAnalyzer/Frontend/CheckerRegistry.cpp b/clang/lib/StaticAnalyzer/Frontend/CheckerRegistry.cpp
index f0d3f43c414c6e715b47045dcf7261adc235a8bf..317df90a7781e243f178897019173bfc9a8c1983 100644
--- a/clang/lib/StaticAnalyzer/Frontend/CheckerRegistry.cpp
+++ b/clang/lib/StaticAnalyzer/Frontend/CheckerRegistry.cpp
@@ -310,8 +310,8 @@ template <bool IsWeak> void CheckerRegistry::resolveDependencies() {
            "Failed to find the dependency of a checker!");
 
     // We do allow diagnostics from unit test/example dependency checkers.
-    assert((DependencyIt->FullName.startswith("test") ||
-            DependencyIt->FullName.startswith("example") || IsWeak ||
+    assert((DependencyIt->FullName.starts_with("test") ||
+            DependencyIt->FullName.starts_with("example") || IsWeak ||
             DependencyIt->IsHidden) &&
            "Strong dependencies are modeling checkers, and as such "
            "non-user facing! Mark them hidden in Checkers.td!");
diff --git a/clang/lib/Support/RISCVVIntrinsicUtils.cpp b/clang/lib/Support/RISCVVIntrinsicUtils.cpp
index a04694e628de4041be05fdaf2121db9409f13a69..bb9f7dc7e7e3d439f97608d50eee874bdec57180 100644
--- a/clang/lib/Support/RISCVVIntrinsicUtils.cpp
+++ b/clang/lib/Support/RISCVVIntrinsicUtils.cpp
@@ -464,7 +464,7 @@ PrototypeDescriptor::parsePrototypeDescriptor(
   PrototypeDescriptorStr = PrototypeDescriptorStr.drop_back();
 
   // Compute the vector type transformers, it can only appear one time.
-  if (PrototypeDescriptorStr.startswith("(")) {
+  if (PrototypeDescriptorStr.starts_with("(")) {
     assert(VTM == VectorTypeModifier::NoModifier &&
            "VectorTypeModifier should only have one modifier");
     size_t Idx = PrototypeDescriptorStr.find(')');
diff --git a/clang/lib/Tooling/ASTDiff/ASTDiff.cpp b/clang/lib/Tooling/ASTDiff/ASTDiff.cpp
index 52e57976ac09117422d21e999dcaa83bc1c188a0..356b4bd5a1b85cbef27c0cc3e8436cf19afd9e22 100644
--- a/clang/lib/Tooling/ASTDiff/ASTDiff.cpp
+++ b/clang/lib/Tooling/ASTDiff/ASTDiff.cpp
@@ -371,7 +371,7 @@ SyntaxTree::Impl::getRelativeName(const NamedDecl *ND,
   // Strip the qualifier, if Val refers to something in the current scope.
   // But leave one leading ':' in place, so that we know that this is a
   // relative path.
-  if (!ContextPrefix.empty() && StringRef(Val).startswith(ContextPrefix))
+  if (!ContextPrefix.empty() && StringRef(Val).starts_with(ContextPrefix))
     Val = Val.substr(ContextPrefix.size() + 1);
   return Val;
 }
diff --git a/clang/lib/Tooling/ArgumentsAdjusters.cpp b/clang/lib/Tooling/ArgumentsAdjusters.cpp
index e40df625737841e67c2e06a855cf43c335626f09..df4c74205b0874d7ed5e40cb54f6e633974c1c79 100644
--- a/clang/lib/Tooling/ArgumentsAdjusters.cpp
+++ b/clang/lib/Tooling/ArgumentsAdjusters.cpp
@@ -45,12 +45,12 @@ ArgumentsAdjuster getClangSyntaxOnlyAdjuster() {
       StringRef Arg = Args[i];
       // Skip output commands.
       if (llvm::any_of(OutputCommands, [&Arg](llvm::StringRef OutputCommand) {
-            return Arg.startswith(OutputCommand);
+            return Arg.starts_with(OutputCommand);
           }))
         continue;
 
-      if (!Arg.startswith("-fcolor-diagnostics") &&
-          !Arg.startswith("-fdiagnostics-color"))
+      if (!Arg.starts_with("-fcolor-diagnostics") &&
+          !Arg.starts_with("-fdiagnostics-color"))
         AdjustedArgs.push_back(Args[i]);
       // If we strip a color option, make sure we strip any preceeding `-Xclang`
       // option as well.
@@ -73,7 +73,7 @@ ArgumentsAdjuster getClangStripOutputAdjuster() {
     CommandLineArguments AdjustedArgs;
     for (size_t i = 0, e = Args.size(); i < e; ++i) {
       StringRef Arg = Args[i];
-      if (!Arg.startswith("-o"))
+      if (!Arg.starts_with("-o"))
         AdjustedArgs.push_back(Args[i]);
 
       if (Arg == "-o") {
@@ -102,11 +102,11 @@ ArgumentsAdjuster getClangStripDependencyFileAdjuster() {
       // When not using the cl driver mode, dependency file generation options
       // begin with -M. These include -MM, -MF, -MG, -MP, -MT, -MQ, -MD, and
       // -MMD.
-      if (!UsingClDriver && Arg.startswith("-M"))
+      if (!UsingClDriver && Arg.starts_with("-M"))
         continue;
       // Under MSVC's cl driver mode, dependency file generation is controlled
       // using /showIncludes
-      if (Arg.startswith("/showIncludes") || Arg.startswith("-showIncludes"))
+      if (Arg.starts_with("/showIncludes") || Arg.starts_with("-showIncludes"))
         continue;
 
       AdjustedArgs.push_back(Args[i]);
@@ -159,7 +159,7 @@ ArgumentsAdjuster getStripPluginsAdjuster() {
       // -Xclang <arbitrary-argument>
       if (I + 4 < E && Args[I] == "-Xclang" &&
           (Args[I + 1] == "-load" || Args[I + 1] == "-plugin" ||
-           llvm::StringRef(Args[I + 1]).startswith("-plugin-arg-") ||
+           llvm::StringRef(Args[I + 1]).starts_with("-plugin-arg-") ||
            Args[I + 1] == "-add-plugin") &&
           Args[I + 2] == "-Xclang") {
         I += 3;
diff --git a/clang/lib/Tooling/CompilationDatabase.cpp b/clang/lib/Tooling/CompilationDatabase.cpp
index fdf6015508d94ba3686413a925797c65c30a2975..87ad8f25a1ab4ed0094e4e98352cd06e112d6806 100644
--- a/clang/lib/Tooling/CompilationDatabase.cpp
+++ b/clang/lib/Tooling/CompilationDatabase.cpp
@@ -204,7 +204,7 @@ public:
 // which don't support these options.
 struct FilterUnusedFlags {
   bool operator() (StringRef S) {
-    return (S == "-no-integrated-as") || S.startswith("-Wa,");
+    return (S == "-no-integrated-as") || S.starts_with("-Wa,");
   }
 };
 
diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
index 3e53c8fc57408754544d6da182bfa3d3ae95e342..6f71650a3982c08448e2aaea676257e99a619314 100644
--- a/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
+++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningFilesystem.cpp
@@ -290,7 +290,7 @@ DependencyScanningWorkerFilesystem::status(const Twine &Path) {
   SmallString<256> OwnedFilename;
   StringRef Filename = Path.toStringRef(OwnedFilename);
 
-  if (Filename.endswith(".pcm"))
+  if (Filename.ends_with(".pcm"))
     return getUnderlyingFS().status(Path);
 
   llvm::ErrorOr<EntryRef> Result = getOrCreateFileSystemEntry(Filename);
@@ -350,7 +350,7 @@ DependencyScanningWorkerFilesystem::openFileForRead(const Twine &Path) {
   SmallString<256> OwnedFilename;
   StringRef Filename = Path.toStringRef(OwnedFilename);
 
-  if (Filename.endswith(".pcm"))
+  if (Filename.ends_with(".pcm"))
     return getUnderlyingFS().openFileForRead(Path);
 
   llvm::ErrorOr<EntryRef> Result = getOrCreateFileSystemEntry(Filename);
diff --git a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp
index f65da413bb87c35d46ceb76d5daf1d085d7d27d7..bfaa897851041db83e56818b357246df76b8441e 100644
--- a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp
+++ b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp
@@ -119,6 +119,8 @@ makeCommonInvocationForModuleBuild(CompilerInvocation CI) {
   // units.
   CI.getFrontendOpts().Inputs.clear();
   CI.getFrontendOpts().OutputFile.clear();
+  // LLVM options are not going to affect the AST
+  CI.getFrontendOpts().LLVMArgs.clear();
 
   // TODO: Figure out better way to set options to their default value.
   CI.getCodeGenOpts().MainFileName.clear();
@@ -530,7 +532,7 @@ ModuleDepCollectorPP::handleTopLevelModule(const Module *M) {
         // this file in the proper directory and relies on the rest of Clang to
         // handle it like normal. With explicitly built modules we don't need
         // to play VFS tricks, so replace it with the correct module map.
-        if (StringRef(IFI.Filename).endswith("__inferred_module.map")) {
+        if (StringRef(IFI.Filename).ends_with("__inferred_module.map")) {
           MDC.addFileDep(MD, ModuleMap->getName());
           return;
         }
@@ -548,7 +550,7 @@ ModuleDepCollectorPP::handleTopLevelModule(const Module *M) {
         if (!(IFI.TopLevel && IFI.ModuleMap))
           return;
         if (StringRef(IFI.FilenameAsRequested)
-                .endswith("__inferred_module.map"))
+                .ends_with("__inferred_module.map"))
           return;
         MD.ModuleMapFileDeps.emplace_back(IFI.FilenameAsRequested);
       });
diff --git a/clang/lib/Tooling/Inclusions/HeaderAnalysis.cpp b/clang/lib/Tooling/Inclusions/HeaderAnalysis.cpp
index 0b3c4de08ab8593e8206c43ae4272d85335693f4..52b634e2e1af8a5a283e40e51b5d7bb7ba270d43 100644
--- a/clang/lib/Tooling/Inclusions/HeaderAnalysis.cpp
+++ b/clang/lib/Tooling/Inclusions/HeaderAnalysis.cpp
@@ -21,7 +21,7 @@ bool isIf(llvm::StringRef Line) {
   if (!Line.consume_front("#"))
     return false;
   Line = Line.ltrim();
-  return Line.startswith("if");
+  return Line.starts_with("if");
 }
 
 // Is Line an #error directive mentioning includes?
@@ -30,7 +30,7 @@ bool isErrorAboutInclude(llvm::StringRef Line) {
   if (!Line.consume_front("#"))
     return false;
   Line = Line.ltrim();
-  if (!Line.startswith("error"))
+  if (!Line.starts_with("error"))
     return false;
   return Line.contains_insensitive(
       "includ"); // Matches "include" or "including".
@@ -54,7 +54,7 @@ bool isImportLine(llvm::StringRef Line) {
   if (!Line.consume_front("#"))
     return false;
   Line = Line.ltrim();
-  return Line.startswith("import");
+  return Line.starts_with("import");
 }
 
 llvm::StringRef getFileContents(FileEntryRef FE, const SourceManager &SM) {
diff --git a/clang/lib/Tooling/Inclusions/HeaderIncludes.cpp b/clang/lib/Tooling/Inclusions/HeaderIncludes.cpp
index 15a2024c4788870d83cbae1a9e15d7d7ad381e1b..d275222ac6b5874bf2db91eb4f93244154b11274 100644
--- a/clang/lib/Tooling/Inclusions/HeaderIncludes.cpp
+++ b/clang/lib/Tooling/Inclusions/HeaderIncludes.cpp
@@ -196,10 +196,10 @@ IncludeCategoryManager::IncludeCategoryManager(const IncludeStyle &Style,
                                                     ? llvm::Regex::NoFlags
                                                     : llvm::Regex::IgnoreCase);
   }
-  IsMainFile = FileName.endswith(".c") || FileName.endswith(".cc") ||
-               FileName.endswith(".cpp") || FileName.endswith(".c++") ||
-               FileName.endswith(".cxx") || FileName.endswith(".m") ||
-               FileName.endswith(".mm");
+  IsMainFile = FileName.ends_with(".c") || FileName.ends_with(".cc") ||
+               FileName.ends_with(".cpp") || FileName.ends_with(".c++") ||
+               FileName.ends_with(".cxx") || FileName.ends_with(".m") ||
+               FileName.ends_with(".mm");
   if (!Style.IncludeIsMainSourceRegex.empty()) {
     llvm::Regex MainFileRegex(Style.IncludeIsMainSourceRegex);
     IsMainFile |= MainFileRegex.match(FileName);
@@ -234,7 +234,7 @@ int IncludeCategoryManager::getSortIncludePriority(StringRef IncludeName,
   return Ret;
 }
 bool IncludeCategoryManager::isMainHeader(StringRef IncludeName) const {
-  if (!IncludeName.startswith("\""))
+  if (!IncludeName.starts_with("\""))
     return false;
 
   IncludeName =
@@ -357,8 +357,8 @@ HeaderIncludes::insert(llvm::StringRef IncludeName, bool IsAngled,
   if (It != ExistingIncludes.end()) {
     for (const auto &Inc : It->second)
       if (Inc.Directive == Directive &&
-          ((IsAngled && StringRef(Inc.Name).startswith("<")) ||
-           (!IsAngled && StringRef(Inc.Name).startswith("\""))))
+          ((IsAngled && StringRef(Inc.Name).starts_with("<")) ||
+           (!IsAngled && StringRef(Inc.Name).starts_with("\""))))
         return std::nullopt;
   }
   std::string Quoted =
@@ -400,8 +400,8 @@ tooling::Replacements HeaderIncludes::remove(llvm::StringRef IncludeName,
   if (Iter == ExistingIncludes.end())
     return Result;
   for (const auto &Inc : Iter->second) {
-    if ((IsAngled && StringRef(Inc.Name).startswith("\"")) ||
-        (!IsAngled && StringRef(Inc.Name).startswith("<")))
+    if ((IsAngled && StringRef(Inc.Name).starts_with("\"")) ||
+        (!IsAngled && StringRef(Inc.Name).starts_with("<")))
       continue;
     llvm::Error Err = Result.add(tooling::Replacement(
         FileName, Inc.R.getOffset(), Inc.R.getLength(), ""));
diff --git a/clang/lib/Tooling/Refactoring/AtomicChange.cpp b/clang/lib/Tooling/Refactoring/AtomicChange.cpp
index 7237393f00e57e239e025d164fd317a3ee3f2a2b..3d5ae2fed014c441aeba7271c9f15967089d2009 100644
--- a/clang/lib/Tooling/Refactoring/AtomicChange.cpp
+++ b/clang/lib/Tooling/Refactoring/AtomicChange.cpp
@@ -150,7 +150,7 @@ createReplacementsForHeaders(llvm::StringRef FilePath, llvm::StringRef Code,
   for (const auto &Change : Changes) {
     for (llvm::StringRef Header : Change.getInsertedHeaders()) {
       std::string EscapedHeader =
-          Header.startswith("<") || Header.startswith("\"")
+          Header.starts_with("<") || Header.starts_with("\"")
               ? Header.str()
               : ("\"" + Header + "\"").str();
       std::string ReplacementText = "#include " + EscapedHeader;
diff --git a/clang/lib/Tooling/Refactoring/Lookup.cpp b/clang/lib/Tooling/Refactoring/Lookup.cpp
index 9468d4d032a7945ba3d653240522408627459ce0..52799f16fab2a99b1dc28ad1fd57e602416f080e 100644
--- a/clang/lib/Tooling/Refactoring/Lookup.cpp
+++ b/clang/lib/Tooling/Refactoring/Lookup.cpp
@@ -98,7 +98,7 @@ static StringRef getBestNamespaceSubstr(const DeclContext *DeclA,
     // from NewName if it has an identical prefix.
     std::string NS =
         "::" + cast<NamespaceDecl>(DeclA)->getQualifiedNameAsString() + "::";
-    if (NewName.startswith(NS))
+    if (NewName.starts_with(NS))
       return NewName.substr(NS.size());
 
     // No match yet. Strip of a namespace from the end of the chain and try
@@ -128,9 +128,9 @@ static std::string disambiguateSpellingInScope(StringRef Spelling,
                                                StringRef QName,
                                                const DeclContext &UseContext,
                                                SourceLocation UseLoc) {
-  assert(QName.startswith("::"));
-  assert(QName.endswith(Spelling));
-  if (Spelling.startswith("::"))
+  assert(QName.starts_with("::"));
+  assert(QName.ends_with(Spelling));
+  if (Spelling.starts_with("::"))
     return std::string(Spelling);
 
   auto UnspelledSpecifier = QName.drop_back(Spelling.size());
@@ -146,7 +146,7 @@ static std::string disambiguateSpellingInScope(StringRef Spelling,
   UseLoc = SM.getSpellingLoc(UseLoc);
 
   auto IsAmbiguousSpelling = [&](const llvm::StringRef CurSpelling) {
-    if (CurSpelling.startswith("::"))
+    if (CurSpelling.starts_with("::"))
       return false;
     // Lookup the first component of Spelling in all enclosing namespaces
     // and check if there is any existing symbols with the same name but in
@@ -160,7 +160,7 @@ static std::string disambiguateSpellingInScope(StringRef Spelling,
           // ambiguous. For example, a reference in a header file should not be
           // affected by a potentially ambiguous name in some file that includes
           // the header.
-          if (!TrimmedQName.startswith(Res->getQualifiedNameAsString()) &&
+          if (!TrimmedQName.starts_with(Res->getQualifiedNameAsString()) &&
               SM.isBeforeInTranslationUnit(
                   SM.getSpellingLoc(Res->getLocation()), UseLoc))
             return true;
@@ -187,7 +187,7 @@ std::string tooling::replaceNestedName(const NestedNameSpecifier *Use,
                                        const DeclContext *UseContext,
                                        const NamedDecl *FromDecl,
                                        StringRef ReplacementString) {
-  assert(ReplacementString.startswith("::") &&
+  assert(ReplacementString.starts_with("::") &&
          "Expected fully-qualified name!");
 
   // We can do a raw name replacement when we are not inside the namespace for
diff --git a/clang/lib/Tooling/Refactoring/Rename/USRLocFinder.cpp b/clang/lib/Tooling/Refactoring/Rename/USRLocFinder.cpp
index 9cdeeec0574b4d0749d38983c81a9c6613ccfa9a..c18f9290471fe4262db39a126b7bd8d9a8dc643f 100644
--- a/clang/lib/Tooling/Refactoring/Rename/USRLocFinder.cpp
+++ b/clang/lib/Tooling/Refactoring/Rename/USRLocFinder.cpp
@@ -562,8 +562,8 @@ createRenameAtomicChanges(llvm::ArrayRef<std::string> USRs,
           ReplacedName = tooling::replaceNestedName(
               RenameInfo.Specifier, RenameInfo.Begin,
               RenameInfo.Context->getDeclContext(), RenameInfo.FromDecl,
-              NewName.startswith("::") ? NewName.str()
-                                       : ("::" + NewName).str());
+              NewName.starts_with("::") ? NewName.str()
+                                        : ("::" + NewName).str());
         } else {
           // This fixes the case where type `T` is a parameter inside a function
           // type (e.g. `std::function<void(T)>`) and the DeclContext of `T`
@@ -578,13 +578,13 @@ createRenameAtomicChanges(llvm::ArrayRef<std::string> USRs,
               SM, TranslationUnitDecl->getASTContext().getLangOpts());
           // Add the leading "::" back if the name written in the code contains
           // it.
-          if (ActualName.startswith("::") && !NewName.startswith("::")) {
+          if (ActualName.starts_with("::") && !NewName.starts_with("::")) {
             ReplacedName = "::" + NewName.str();
           }
         }
       }
       // If the NewName contains leading "::", add it back.
-      if (NewName.startswith("::") && NewName.substr(2) == ReplacedName)
+      if (NewName.starts_with("::") && NewName.substr(2) == ReplacedName)
         ReplacedName = NewName.str();
     }
     Replace(RenameInfo.Begin, RenameInfo.End, ReplacedName);
diff --git a/clang/lib/Tooling/Tooling.cpp b/clang/lib/Tooling/Tooling.cpp
index e292fa724d2bb712122e69042d8e09ee55d08edf..33bfa8d3d81f1e3da6f127259c305ac1f98e1312 100644
--- a/clang/lib/Tooling/Tooling.cpp
+++ b/clang/lib/Tooling/Tooling.cpp
@@ -255,7 +255,7 @@ llvm::Expected<std::string> getAbsolutePath(llvm::vfs::FileSystem &FS,
                                             StringRef File) {
   StringRef RelativePath(File);
   // FIXME: Should '.\\' be accepted on Win32?
-  if (RelativePath.startswith("./")) {
+  if (RelativePath.starts_with("./")) {
     RelativePath = RelativePath.substr(strlen("./"));
   }
 
@@ -294,9 +294,9 @@ void addTargetAndModeForProgramName(std::vector<std::string> &CommandLine,
   for (auto Token = ++CommandLine.begin(); Token != CommandLine.end();
        ++Token) {
     StringRef TokenRef(*Token);
-    ShouldAddTarget = ShouldAddTarget && !TokenRef.startswith(TargetOPT) &&
+    ShouldAddTarget = ShouldAddTarget && !TokenRef.starts_with(TargetOPT) &&
                       !TokenRef.equals(TargetOPTLegacy);
-    ShouldAddMode = ShouldAddMode && !TokenRef.startswith(DriverModeOPT);
+    ShouldAddMode = ShouldAddMode && !TokenRef.starts_with(DriverModeOPT);
   }
   if (ShouldAddMode) {
     CommandLine.insert(++CommandLine.begin(), TargetMode.DriverMode);
@@ -507,7 +507,7 @@ static void injectResourceDir(CommandLineArguments &Args, const char *Argv0,
                               void *MainAddr) {
   // Allow users to override the resource dir.
   for (StringRef Arg : Args)
-    if (Arg.startswith("-resource-dir"))
+    if (Arg.starts_with("-resource-dir"))
       return;
 
   // If there's no override in place add our resource dir.
diff --git a/clang/lib/Tooling/Transformer/SourceCode.cpp b/clang/lib/Tooling/Transformer/SourceCode.cpp
index 30009537b5923ced0ea42ff86d91492fae7799a9..6aae834b0db563ea336dfdfabf431c28b8e3ac94 100644
--- a/clang/lib/Tooling/Transformer/SourceCode.cpp
+++ b/clang/lib/Tooling/Transformer/SourceCode.cpp
@@ -425,7 +425,7 @@ CharSourceRange tooling::getAssociatedRange(const Decl &Decl,
 
     for (llvm::StringRef Prefix : {"[[", "__attribute__(("}) {
       // Handle whitespace between attribute prefix and attribute value.
-      if (BeforeAttrStripped.endswith(Prefix)) {
+      if (BeforeAttrStripped.ends_with(Prefix)) {
         // Move start to start position of prefix, which is
         // length(BeforeAttr) - length(BeforeAttrStripped) + length(Prefix)
         // positions to the left.
diff --git a/clang/test/AST/Interp/complex.cpp b/clang/test/AST/Interp/complex.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..66490e973988bb5409bf55f029e44a9b8823578a
--- /dev/null
+++ b/clang/test/AST/Interp/complex.cpp
@@ -0,0 +1,126 @@
+// RUN: %clang_cc1 -fexperimental-new-constant-interpreter -verify %s
+// RUN: %clang_cc1 -verify=ref %s
+
+// expected-no-diagnostics
+// ref-no-diagnostics
+
+constexpr _Complex double z1 = {1.0, 2.0};
+static_assert(__real(z1) == 1.0, "");
+static_assert(__imag(z1) == 2.0, "");
+
+constexpr double setter() {
+  _Complex float d = {1.0, 2.0};
+
+  __imag(d) = 4.0;
+  return __imag(d);
+}
+static_assert(setter() == 4, "");
+
+constexpr _Complex double getter() {
+  return {1.0, 3.0};
+}
+constexpr _Complex double D = getter();
+static_assert(__real(D) == 1.0, "");
+static_assert(__imag(D) == 3.0, "");
+
+
+constexpr _Complex int I1 = {1, 2};
+static_assert(__real(I1) == 1, "");
+static_assert(__imag(I1) == 2, "");
+
+
+constexpr _Complex double D1 = {};
+static_assert(__real(D1) == 0, "");
+static_assert(__imag(D1) == 0, "");
+
+constexpr _Complex int I2 = {};
+static_assert(__real(I2) == 0, "");
+static_assert(__imag(I2) == 0, "");
+
+
+/// Standalone complex expressions.
+static_assert(__real((_Complex float){1.0, 3.0}) == 1.0, "");
+
+
+#if 0
+/// FIXME: This should work in the new interpreter.
+constexpr _Complex double D2 = {12};
+static_assert(__real(D2) == 12, "");
+static_assert(__imag(D2) == 12, "");
+
+constexpr _Complex int I3 = {15};
+static_assert(__real(I3) == 15, "");
+static_assert(__imag(I3) == 15, "");
+#endif
+
+/// FIXME: This should work in the new interpreter as well.
+// constexpr _Complex _BitInt(8) A = 0;// = {4};
+
+namespace CastToBool {
+  constexpr _Complex int F = {0, 1};
+  static_assert(F, "");
+  constexpr _Complex int F2 = {1, 0};
+  static_assert(F2, "");
+  constexpr _Complex int F3 = {0, 0};
+  static_assert(!F3, "");
+
+  constexpr _Complex unsigned char F4 = {0, 1};
+  static_assert(F4, "");
+  constexpr _Complex unsigned char F5 = {1, 0};
+  static_assert(F5, "");
+  constexpr _Complex unsigned char F6 = {0, 0};
+  static_assert(!F6, "");
+
+  constexpr _Complex float F7 = {0, 1};
+  static_assert(F7, "");
+  constexpr _Complex float F8 = {1, 0};
+  static_assert(F8, "");
+  constexpr _Complex double F9 = {0, 0};
+  static_assert(!F9, "");
+}
+
+namespace BinOps {
+namespace Add {
+  constexpr _Complex float A = { 13.0, 2.0 };
+  constexpr _Complex float B = { 2.0, 1.0  };
+  constexpr _Complex float C = A + B;
+  static_assert(__real(C) == 15.0, "");
+  static_assert(__imag(C) == 3.0, "");
+
+  constexpr _Complex float D = B + A;
+  static_assert(__real(D) == 15.0, "");
+  static_assert(__imag(D) == 3.0, "");
+
+  constexpr _Complex unsigned int I1 = { 5,  10 };
+  constexpr _Complex unsigned int I2 = { 40, 2  };
+  constexpr _Complex unsigned int I3 = I1 + I2;
+  static_assert(__real(I3) == 45, "");
+  static_assert(__imag(I3) == 12, "");
+}
+
+namespace Sub {
+  constexpr _Complex float A = { 13.0, 2.0 };
+  constexpr _Complex float B = { 2.0, 1.0  };
+  constexpr _Complex float C = A - B;
+  static_assert(__real(C) == 11.0, "");
+  static_assert(__imag(C) == 1.0, "");
+
+  constexpr _Complex float D = B - A;
+  static_assert(__real(D) == -11.0, "");
+  static_assert(__imag(D) == -1.0, "");
+
+  constexpr _Complex unsigned int I1 = { 5,  10 };
+  constexpr _Complex unsigned int I2 = { 40, 2  };
+  constexpr _Complex unsigned int I3 = I1 - I2;
+  static_assert(__real(I3) == -35, "");
+  static_assert(__imag(I3) == 8, "");
+
+  using Bobble = _Complex float;
+  constexpr _Complex float A_ = { 13.0, 2.0 };
+  constexpr Bobble B_ = { 2.0, 1.0  };
+  constexpr _Complex float D_ = A_ - B_;
+  static_assert(__real(D_) == 11.0, "");
+  static_assert(__imag(D_) == 1.0, "");
+}
+
+}
diff --git a/clang/test/AST/Interp/functions.cpp b/clang/test/AST/Interp/functions.cpp
index ab562e70606b6721c95d66cbdb9ad7f7afd27a47..179a195098b1327879b16d80791e876be90d3c43 100644
--- a/clang/test/AST/Interp/functions.cpp
+++ b/clang/test/AST/Interp/functions.cpp
@@ -267,6 +267,17 @@ namespace InvalidCall {
                    // ref-error {{must be initialized by a constant expression}} \
                    // ref-note {{in call to 'SS()'}}
 
+
+  /// This should not emit a diagnostic.
+  constexpr int f();
+  constexpr int a() {
+    return f();
+  }
+  constexpr int f() {
+    return 5;
+  }
+  static_assert(a() == 5, "");
+
 }
 
 namespace CallWithArgs {
diff --git a/clang/test/Analysis/suppression-attr-doc.cpp b/clang/test/Analysis/suppression-attr-doc.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..1208842799ed9a1d635019b63518d6e354cf6221
--- /dev/null
+++ b/clang/test/Analysis/suppression-attr-doc.cpp
@@ -0,0 +1,54 @@
+// RUN: %clang_analyze_cc1 -analyzer-checker=core,unix \
+// RUN:                    -analyzer-disable-checker=core.uninitialized \
+// RUN:                    -verify %s
+
+// NOTE: These tests correspond to examples provided in documentation
+// of [[clang::suppress]]. If you break them intentionally, it's likely that
+// you need to update the documentation!
+
+typedef __typeof(sizeof(int)) size_t;
+void *malloc(size_t);
+
+int foo_initial() {
+  int *x = nullptr;
+  return *x; // expected-warning{{Dereference of null pointer (loaded from variable 'x')}}
+}
+
+int foo1() {
+  int *x = nullptr;
+  [[clang::suppress]]
+  return *x;  // null pointer dereference warning suppressed here
+}
+
+int foo2() {
+  [[clang::suppress]] {
+    int *x = nullptr;
+    return *x;  // null pointer dereference warning suppressed here
+  }
+}
+
+int bar_initial(bool coin_flip) {
+  int *result = (int *)malloc(sizeof(int));
+  if (coin_flip)
+    return 1; // There's no warning here YET, but it will show up if the other one is suppressed.
+
+  return *result;  // expected-warning{{Potential leak of memory pointed to by 'result'}}
+}
+
+int bar1(bool coin_flip) {
+  __attribute__((suppress))
+  int *result = (int *)malloc(sizeof(int));
+  if (coin_flip)
+    return 1;  // warning about this leak path is suppressed
+
+  return *result;  // warning about this leak path also suppressed
+}
+
+int bar2(bool coin_flip) {
+  int *result = (int *)malloc(sizeof(int));
+  if (coin_flip)
+    return 1;  // expected-warning{{Potential leak of memory pointed to by 'result'}}
+
+  __attribute__((suppress))
+  return *result;  // leak warning is suppressed only on this path
+}
diff --git a/clang/test/Analysis/suppression-attr.m b/clang/test/Analysis/suppression-attr.m
new file mode 100644
index 0000000000000000000000000000000000000000..8ba8dda722721b2b1257fc7302d5cb18795b5eb4
--- /dev/null
+++ b/clang/test/Analysis/suppression-attr.m
@@ -0,0 +1,271 @@
+// RUN: %clang_analyze_cc1 -fblocks \
+// RUN:   -analyzer-checker=core \
+// RUN:   -analyzer-checker=osx.cocoa.MissingSuperCall \
+// RUN:   -analyzer-checker=osx.cocoa.NSError \
+// RUN:   -analyzer-checker=osx.ObjCProperty \
+// RUN:   -analyzer-checker=osx.cocoa.RetainCount \
+// RUN:   -analyzer-checker=unix.Malloc \
+// RUN:   -analyzer-checker=alpha.core.CastToStruct \
+// RUN:   -Wno-unused-value -Wno-objc-root-class -verify %s
+
+#define SUPPRESS __attribute__((suppress))
+#define SUPPRESS_SPECIFIC(...) __attribute__((suppress(__VA_ARGS__)))
+
+@protocol NSObject
+- (id)retain;
+- (oneway void)release;
+@end
+@interface NSObject <NSObject> {
+}
+- (id)init;
++ (id)alloc;
+@end
+typedef int NSInteger;
+typedef char BOOL;
+typedef struct _NSZone NSZone;
+@class NSInvocation, NSMethodSignature, NSCoder, NSString, NSEnumerator;
+@protocol NSCopying
+- (id)copyWithZone:(NSZone *)zone;
+@end
+@protocol NSCoding
+- (void)encodeWithCoder:(NSCoder *)aCoder;
+@end
+@class NSDictionary;
+@interface NSError : NSObject <NSCopying, NSCoding> {
+}
++ (id)errorWithDomain:(NSString *)domain code:(NSInteger)code userInfo:(NSDictionary *)dict;
+@end
+
+@interface NSMutableString : NSObject
+@end
+
+typedef __typeof__(sizeof(int)) size_t;
+void *malloc(size_t);
+void free(void *);
+
+void dereference_1() {
+  int *x = 0;
+  *x; // expected-warning{{Dereference of null pointer (loaded from variable 'x')}}
+}
+
+void dereference_suppression_1() {
+  int *x = 0;
+  SUPPRESS { *x; } // no-warning
+}
+
+void dereference_2() {
+  int *x = 0;
+  if (*x) { // expected-warning{{Dereference of null pointer (loaded from variable 'x')}}
+  }
+}
+
+void dereference_suppression_2() {
+  int *x = 0;
+  SUPPRESS if (*x) { // no-warning
+  }
+}
+
+void dereference_suppression_2a() {
+  int *x = 0;
+  // FIXME: Implement suppressing individual checkers.
+  SUPPRESS_SPECIFIC("core.NullDereference") if (*x) { // expected-warning{{Dereference of null pointer (loaded from variable 'x')}}
+  }
+}
+
+void dereference_suppression_2b() {
+  int *x = 0;
+  // This is not a MallocChecker issue so it shouldn't be suppressed. (Though the attribute
+  // doesn't really understand any of those arguments yet.)
+  SUPPRESS_SPECIFIC("unix.Malloc") if (*x) { // expected-warning{{Dereference of null pointer (loaded from variable 'x')}}
+  }
+}
+
+void dereference_3(int cond) {
+  int *x = 0;
+  if (cond) {
+    (*x)++; // expected-warning{{Dereference of null pointer (loaded from variable 'x')}}
+  }
+}
+
+void dereference_suppression_3(int cond) {
+  int *x = 0;
+  SUPPRESS if (cond) {
+    (*x)++; // no-warning
+  }
+}
+
+void dereference_4() {
+  int *x = 0;
+  int y = *x; // expected-warning{{Dereference of null pointer (loaded from variable 'x')}}
+}
+
+void dereference_suppression_4() {
+  int *x = 0;
+  SUPPRESS int y = *x; // no-warning
+}
+
+void dereference_5() {
+  int *x = 0;
+  int y = *x; // expected-warning{{Dereference of null pointer (loaded from variable 'x')}}
+  int z = *x; // no-warning (duplicate)
+}
+
+void dereference_suppression_5_1() {
+  int *x = 0;
+  SUPPRESS int y = *x; // no-warning
+  int z = *x;          // no-warning (duplicate)
+}
+
+void dereference_suppression_5_2() {
+  int *x = 0;
+  int y = *x;          // expected-warning{{Dereference of null pointer (loaded from variable 'x')}}
+  SUPPRESS int z = *x; // no-warning
+}
+
+void do_deref(int *y) {
+  *y = 1; // expected-warning{{Dereference of null pointer (loaded from variable 'y')}}
+}
+
+void dereference_interprocedural() {
+  int *x = 0;
+  do_deref(x);
+}
+
+void do_deref_suppressed(int *y) {
+  SUPPRESS *y = 1; // no-warning
+}
+
+void dereference_interprocedural_suppressed() {
+  int *x = 0;
+  do_deref_suppressed(x);
+}
+
+int malloc_leak_1() {
+  int *x = (int *)malloc(sizeof(int));
+  *x = 42;
+  return *x; // expected-warning{{Potential leak of memory pointed to by 'x'}}
+}
+
+int malloc_leak_suppression_1_1() {
+  SUPPRESS int *x = (int *)malloc(sizeof(int));
+  *x = 42;
+  return *x;
+}
+
+int malloc_leak_suppression_1_2() {
+  int *x = (int *)malloc(sizeof(int));
+  *x = 42;
+  SUPPRESS return *x;
+}
+
+void malloc_leak_2() {
+  int *x = (int *)malloc(sizeof(int));
+  *x = 42;
+} // expected-warning{{Potential leak of memory pointed to by 'x'}}
+
+void malloc_leak_suppression_2_1() {
+  SUPPRESS int *x = (int *)malloc(sizeof(int));
+  *x = 42;
+}
+
+// TODO: reassess when we decide what to do with declaration annotations
+void malloc_leak_suppression_2_2() /* SUPPRESS */ {
+  int *x = (int *)malloc(sizeof(int));
+  *x = 42;
+} // expected-warning{{Potential leak of memory pointed to by 'x'}}
+
+// TODO: reassess when we decide what to do with declaration annotations
+/* SUPPRESS */ void malloc_leak_suppression_2_3() {
+  int *x = (int *)malloc(sizeof(int));
+  *x = 42;
+} // expected-warning{{Potential leak of memory pointed to by 'x'}}
+
+void malloc_leak_suppression_2_4(int cond) {
+  int *x = (int *)malloc(sizeof(int));
+  *x = 42;
+  SUPPRESS;
+  // FIXME: The warning should be suppressed but dead symbol elimination
+  // happens too late.
+} // expected-warning{{Potential leak of memory pointed to by 'x'}}
+
+void retain_release_leak_1() {
+  [[NSMutableString alloc] init]; // expected-warning{{Potential leak of an object of type 'NSMutableString *'}}
+}
+
+void retain_release_leak_suppression_1() {
+  SUPPRESS { [[NSMutableString alloc] init]; }
+}
+
+void retain_release_leak_2(int cond) {
+  id obj = [[NSMutableString alloc] init]; // expected-warning{{Potential leak of an object stored into 'obj'}}
+  if (cond) {
+    [obj release];
+  }
+}
+
+void retain_release_leak__suppression_2(int cond) {
+  SUPPRESS id obj = [[NSMutableString alloc] init];
+  if (cond) {
+    [obj release];
+  }
+}
+
+@interface UIResponder : NSObject {
+}
+- (char)resignFirstResponder;
+@end
+
+@interface Test : UIResponder {
+}
+@property(copy) NSMutableString *mutableStr;
+// expected-warning@-1 {{Property of mutable type 'NSMutableString' has 'copy' attribute; an immutable object will be stored instead}}
+@end
+@implementation Test
+
+- (BOOL)resignFirstResponder {
+  return 0;
+} // expected-warning {{The 'resignFirstResponder' instance method in UIResponder subclass 'Test' is missing a [super resignFirstResponder] call}}
+
+- (void)methodWhichMayFail:(NSError **)error {
+  // expected-warning@-1 {{Method accepting NSError** should have a non-void return value to indicate whether or not an error occurred}}
+}
+@end
+
+@interface TestSuppress : UIResponder {
+}
+// TODO: reassess when we decide what to do with declaration annotations
+@property(copy) /* SUPPRESS */ NSMutableString *mutableStr;
+// expected-warning@-1 {{Property of mutable type 'NSMutableString' has 'copy' attribute; an immutable object will be stored instead}}
+@end
+@implementation TestSuppress
+
+// TODO: reassess when we decide what to do with declaration annotations
+- (BOOL)resignFirstResponder /* SUPPRESS */ {
+  return 0;
+} // expected-warning {{The 'resignFirstResponder' instance method in UIResponder subclass 'TestSuppress' is missing a [super resignFirstResponder] call}}
+
+// TODO: reassess when we decide what to do with declaration annotations
+- (void)methodWhichMayFail:(NSError **)error /* SUPPRESS */ {
+  // expected-warning@-1 {{Method accepting NSError** should have a non-void return value to indicate whether or not an error occurred}}
+}
+@end
+
+struct AB {
+  int A, B;
+};
+
+struct ABC {
+  int A, B, C;
+};
+
+void ast_checker_1() {
+  struct AB Ab;
+  struct ABC *Abc;
+  Abc = (struct ABC *)&Ab; // expected-warning {{Casting data to a larger structure type and accessing a field can lead to memory access errors or data corruption}}
+}
+
+void ast_checker_suppress_1() {
+  struct AB Ab;
+  struct ABC *Abc;
+  SUPPRESS { Abc = (struct ABC *)&Ab; }
+}
diff --git a/clang/test/CXX/temp/temp.decls/temp.alias/p3.cpp b/clang/test/CXX/temp/temp.decls/temp.alias/p3.cpp
index 2d46502e1d9b352d3463d56fac513789432ed23b..2b33a4ef566dadff328bd65300d74ee6e1892d26 100644
--- a/clang/test/CXX/temp/temp.decls/temp.alias/p3.cpp
+++ b/clang/test/CXX/temp/temp.decls/temp.alias/p3.cpp
@@ -2,11 +2,12 @@
 
 // The example given in the standard (this is rejected for other reasons anyway).
 template<class T> struct A;
-template<class T> using B = typename A<T>::U; // expected-error {{no type named 'U' in 'A<T>'}}
+template<class T> using B = typename A<T>::U; // expected-error {{no type named 'U' in 'A<short>'}}
+                                              // expected-note@-1 {{in instantiation of template class 'A<short>' requested here}}
 template<class T> struct A {
   typedef B<T> U; // expected-note {{in instantiation of template type alias 'B' requested here}}
 };
-B<short> b;
+B<short> b; // expected-note {{in instantiation of template type alias 'B' requested here}}
 
 template<typename T> using U = int;
 
diff --git a/clang/test/ClangScanDeps/strip-llvm-args.m b/clang/test/ClangScanDeps/strip-llvm-args.m
new file mode 100644
index 0000000000000000000000000000000000000000..ca8eab7729232b16d08fdcc8b3631155943bd638
--- /dev/null
+++ b/clang/test/ClangScanDeps/strip-llvm-args.m
@@ -0,0 +1,46 @@
+// RUN: rm -rf %t
+// RUN: split-file %s %t
+// RUN: sed -e "s|DIR|%/t|g" %t/cdb1.json.template > %t/cdb1.json
+
+// RUN: clang-scan-deps -compilation-database %t/cdb1.json -format experimental-full > %t/result1.txt
+// RUN: FileCheck %s -input-file %t/result1.txt
+
+// CHECK:        "modules": [
+// CHECK-NEXT:     {
+// CHECK:            "command-line": [
+// CHECK-NOT:          "-mllvm"
+// CHECK:            ]
+// CHECK:            "name": "A"
+// CHECK:          }
+// CHECK-NOT:        "name": "A"
+// CHECK:        "translation-units"
+
+//--- cdb1.json.template
+[
+  {
+    "directory": "DIR",
+    "command": "clang -Imodules/A -fmodules -fmodules-cache-path=DIR/module-cache -fimplicit-modules -fimplicit-module-maps -fsyntax-only DIR/t1.m",
+    "file": "DIR/t1.m"
+  },
+  {
+    "directory": "DIR",
+    "command": "clang -Imodules/A -fmodules -fmodules-cache-path=DIR/module-cache -fimplicit-modules -fimplicit-module-maps -mllvm -stackmap-version=2 -fsyntax-only DIR/t2.m",
+    "file": "DIR/t2.m"
+  }
+]
+
+//--- modules/A/module.modulemap
+
+module A {
+  umbrella header "A.h"
+}
+
+//--- modules/A/A.h
+
+typedef int A_t;
+
+//--- t1.m
+@import A;
+
+//--- t2.m
+@import A;
diff --git a/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_sqdmulh.c b/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_sqdmulh.c
new file mode 100644
index 0000000000000000000000000000000000000000..6bbd23ccd32a521b36f5a86cde6a0af61baa1024
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_sqdmulh.c
@@ -0,0 +1,584 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// REQUIRES: aarch64-registered-target
+
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s
+#include <arm_sme_draft_spec_subject_to_change.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED,A5) A1##A3##A5
+#else
+#define SVE_ACLE_FUNC(A1,A2,A3,A4,A5) A1##A2##A3##A4##A5
+#endif
+
+// Single, x2
+
+// CHECK-LABEL: @test_svqdmulh_single_s8_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZDN:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZDN]], i64 16)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.sqdmulh.single.vgx2.nxv16i8(<vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[ZM:%.*]])
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP2]], 0
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 32 x i8> @llvm.vector.insert.nxv32i8.nxv16i8(<vscale x 32 x i8> poison, <vscale x 16 x i8> [[TMP3]], i64 0)
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP2]], 1
+// CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 32 x i8> @llvm.vector.insert.nxv32i8.nxv16i8(<vscale x 32 x i8> [[TMP4]], <vscale x 16 x i8> [[TMP5]], i64 16)
+// CHECK-NEXT:    ret <vscale x 32 x i8> [[TMP6]]
+//
+// CPP-CHECK-LABEL: @_Z26test_svqdmulh_single_s8_x210svint8x2_tu10__SVInt8_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZDN:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZDN]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.sqdmulh.single.vgx2.nxv16i8(<vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[ZM:%.*]])
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP2]], 0
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 32 x i8> @llvm.vector.insert.nxv32i8.nxv16i8(<vscale x 32 x i8> poison, <vscale x 16 x i8> [[TMP3]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP2]], 1
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 32 x i8> @llvm.vector.insert.nxv32i8.nxv16i8(<vscale x 32 x i8> [[TMP4]], <vscale x 16 x i8> [[TMP5]], i64 16)
+// CPP-CHECK-NEXT:    ret <vscale x 32 x i8> [[TMP6]]
+//
+svint8x2_t test_svqdmulh_single_s8_x2(svint8x2_t zdn, svint8_t zm) __arm_streaming {
+  return SVE_ACLE_FUNC(svqdmulh,_single_s8_x2,,,)(zdn, zm);
+}
+
+// CHECK-LABEL: @test_svqdmulh_single_s16_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZDN:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZDN]], i64 8)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.sqdmulh.single.vgx2.nxv8i16(<vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[ZM:%.*]])
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP2]], 0
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 16 x i16> @llvm.vector.insert.nxv16i16.nxv8i16(<vscale x 16 x i16> poison, <vscale x 8 x i16> [[TMP3]], i64 0)
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP2]], 1
+// CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 16 x i16> @llvm.vector.insert.nxv16i16.nxv8i16(<vscale x 16 x i16> [[TMP4]], <vscale x 8 x i16> [[TMP5]], i64 8)
+// CHECK-NEXT:    ret <vscale x 16 x i16> [[TMP6]]
+//
+// CPP-CHECK-LABEL: @_Z27test_svqdmulh_single_s16_x211svint16x2_tu11__SVInt16_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZDN:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZDN]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.sqdmulh.single.vgx2.nxv8i16(<vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[ZM:%.*]])
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP2]], 0
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 16 x i16> @llvm.vector.insert.nxv16i16.nxv8i16(<vscale x 16 x i16> poison, <vscale x 8 x i16> [[TMP3]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP2]], 1
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 16 x i16> @llvm.vector.insert.nxv16i16.nxv8i16(<vscale x 16 x i16> [[TMP4]], <vscale x 8 x i16> [[TMP5]], i64 8)
+// CPP-CHECK-NEXT:    ret <vscale x 16 x i16> [[TMP6]]
+//
+svint16x2_t test_svqdmulh_single_s16_x2(svint16x2_t zdn, svint16_t zm) __arm_streaming {
+  return SVE_ACLE_FUNC(svqdmulh,_single_s16_x2,,,)(zdn, zm);
+}
+
+// CHECK-LABEL: @test_svqdmulh_single_s32_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[ZDN:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[ZDN]], i64 4)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.sqdmulh.single.vgx2.nxv4i32(<vscale x 4 x i32> [[TMP0]], <vscale x 4 x i32> [[TMP1]], <vscale x 4 x i32> [[ZM:%.*]])
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP2]], 0
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> poison, <vscale x 4 x i32> [[TMP3]], i64 0)
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP2]], 1
+// CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> [[TMP4]], <vscale x 4 x i32> [[TMP5]], i64 4)
+// CHECK-NEXT:    ret <vscale x 8 x i32> [[TMP6]]
+//
+// CPP-CHECK-LABEL: @_Z27test_svqdmulh_single_s32_x211svint32x2_tu11__SVInt32_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[ZDN:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[ZDN]], i64 4)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.sqdmulh.single.vgx2.nxv4i32(<vscale x 4 x i32> [[TMP0]], <vscale x 4 x i32> [[TMP1]], <vscale x 4 x i32> [[ZM:%.*]])
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP2]], 0
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> poison, <vscale x 4 x i32> [[TMP3]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP2]], 1
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> [[TMP4]], <vscale x 4 x i32> [[TMP5]], i64 4)
+// CPP-CHECK-NEXT:    ret <vscale x 8 x i32> [[TMP6]]
+//
+svint32x2_t test_svqdmulh_single_s32_x2(svint32x2_t zdn, svint32_t zm) __arm_streaming {
+  return SVE_ACLE_FUNC(svqdmulh,_single_s32_x2,,,)(zdn, zm);
+}
+
+// CHECK-LABEL: @test_svqdmulh_single_s64_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv4i64(<vscale x 4 x i64> [[ZDN:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv4i64(<vscale x 4 x i64> [[ZDN]], i64 2)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.sqdmulh.single.vgx2.nxv2i64(<vscale x 2 x i64> [[TMP0]], <vscale x 2 x i64> [[TMP1]], <vscale x 2 x i64> [[ZM:%.*]])
+// CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP2]], 0
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 4 x i64> @llvm.vector.insert.nxv4i64.nxv2i64(<vscale x 4 x i64> poison, <vscale x 2 x i64> [[TMP3]], i64 0)
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP2]], 1
+// CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 4 x i64> @llvm.vector.insert.nxv4i64.nxv2i64(<vscale x 4 x i64> [[TMP4]], <vscale x 2 x i64> [[TMP5]], i64 2)
+// CHECK-NEXT:    ret <vscale x 4 x i64> [[TMP6]]
+//
+// CPP-CHECK-LABEL: @_Z27test_svqdmulh_single_s64_x211svint64x2_tu11__SVInt64_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv4i64(<vscale x 4 x i64> [[ZDN:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv4i64(<vscale x 4 x i64> [[ZDN]], i64 2)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.sqdmulh.single.vgx2.nxv2i64(<vscale x 2 x i64> [[TMP0]], <vscale x 2 x i64> [[TMP1]], <vscale x 2 x i64> [[ZM:%.*]])
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP2]], 0
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 4 x i64> @llvm.vector.insert.nxv4i64.nxv2i64(<vscale x 4 x i64> poison, <vscale x 2 x i64> [[TMP3]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP2]], 1
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 4 x i64> @llvm.vector.insert.nxv4i64.nxv2i64(<vscale x 4 x i64> [[TMP4]], <vscale x 2 x i64> [[TMP5]], i64 2)
+// CPP-CHECK-NEXT:    ret <vscale x 4 x i64> [[TMP6]]
+//
+svint64x2_t test_svqdmulh_single_s64_x2(svint64x2_t zdn, svint64_t zm) __arm_streaming {
+  return SVE_ACLE_FUNC(svqdmulh,_single_s64_x2,,,)(zdn, zm);
+}
+
+// Single, x4
+
+// CHECK-LABEL: @test_svqdmulh_single_s8_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZDN:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZDN]], i64 16)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZDN]], i64 32)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZDN]], i64 48)
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.sqdmulh.single.vgx4.nxv16i8(<vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[ZM:%.*]])
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP4]], 0
+// CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> poison, <vscale x 16 x i8> [[TMP5]], i64 0)
+// CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP4]], 1
+// CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP6]], <vscale x 16 x i8> [[TMP7]], i64 16)
+// CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP4]], 2
+// CHECK-NEXT:    [[TMP10:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP8]], <vscale x 16 x i8> [[TMP9]], i64 32)
+// CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP4]], 3
+// CHECK-NEXT:    [[TMP12:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP10]], <vscale x 16 x i8> [[TMP11]], i64 48)
+// CHECK-NEXT:    ret <vscale x 64 x i8> [[TMP12]]
+//
+// CPP-CHECK-LABEL: @_Z26test_svqdmulh_single_s8_x410svint8x4_tu10__SVInt8_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZDN:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZDN]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZDN]], i64 32)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZDN]], i64 48)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.sqdmulh.single.vgx4.nxv16i8(<vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[ZM:%.*]])
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP4]], 0
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> poison, <vscale x 16 x i8> [[TMP5]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP4]], 1
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP6]], <vscale x 16 x i8> [[TMP7]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP4]], 2
+// CPP-CHECK-NEXT:    [[TMP10:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP8]], <vscale x 16 x i8> [[TMP9]], i64 32)
+// CPP-CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP4]], 3
+// CPP-CHECK-NEXT:    [[TMP12:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP10]], <vscale x 16 x i8> [[TMP11]], i64 48)
+// CPP-CHECK-NEXT:    ret <vscale x 64 x i8> [[TMP12]]
+//
+svint8x4_t test_svqdmulh_single_s8_x4(svint8x4_t zdn, svint8_t zm) __arm_streaming {
+  return SVE_ACLE_FUNC(svqdmulh,_single_s8_x4,,,)(zdn, zm);
+}
+
+// CHECK-LABEL: @test_svqdmulh_single_s16_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZDN:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZDN]], i64 8)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZDN]], i64 16)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZDN]], i64 24)
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.sqdmulh.single.vgx4.nxv8i16(<vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[ZM:%.*]])
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP4]], 0
+// CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> poison, <vscale x 8 x i16> [[TMP5]], i64 0)
+// CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP4]], 1
+// CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP6]], <vscale x 8 x i16> [[TMP7]], i64 8)
+// CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP4]], 2
+// CHECK-NEXT:    [[TMP10:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP8]], <vscale x 8 x i16> [[TMP9]], i64 16)
+// CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP4]], 3
+// CHECK-NEXT:    [[TMP12:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP10]], <vscale x 8 x i16> [[TMP11]], i64 24)
+// CHECK-NEXT:    ret <vscale x 32 x i16> [[TMP12]]
+//
+// CPP-CHECK-LABEL: @_Z27test_svqdmulh_single_s16_x411svint16x4_tu11__SVInt16_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZDN:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZDN]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZDN]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZDN]], i64 24)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.sqdmulh.single.vgx4.nxv8i16(<vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[ZM:%.*]])
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP4]], 0
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> poison, <vscale x 8 x i16> [[TMP5]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP4]], 1
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP6]], <vscale x 8 x i16> [[TMP7]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP4]], 2
+// CPP-CHECK-NEXT:    [[TMP10:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP8]], <vscale x 8 x i16> [[TMP9]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP4]], 3
+// CPP-CHECK-NEXT:    [[TMP12:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP10]], <vscale x 8 x i16> [[TMP11]], i64 24)
+// CPP-CHECK-NEXT:    ret <vscale x 32 x i16> [[TMP12]]
+//
+svint16x4_t test_svqdmulh_single_s16_x4(svint16x4_t zdn, svint16_t zm) __arm_streaming {
+  return SVE_ACLE_FUNC(svqdmulh,_single_s16_x4,,,)(zdn, zm);
+}
+
+// CHECK-LABEL: @test_svqdmulh_single_s32_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[ZDN:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[ZDN]], i64 4)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[ZDN]], i64 8)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[ZDN]], i64 12)
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.sqdmulh.single.vgx4.nxv4i32(<vscale x 4 x i32> [[TMP0]], <vscale x 4 x i32> [[TMP1]], <vscale x 4 x i32> [[TMP2]], <vscale x 4 x i32> [[TMP3]], <vscale x 4 x i32> [[ZM:%.*]])
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP4]], 0
+// CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> poison, <vscale x 4 x i32> [[TMP5]], i64 0)
+// CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP4]], 1
+// CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP6]], <vscale x 4 x i32> [[TMP7]], i64 4)
+// CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP4]], 2
+// CHECK-NEXT:    [[TMP10:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP8]], <vscale x 4 x i32> [[TMP9]], i64 8)
+// CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP4]], 3
+// CHECK-NEXT:    [[TMP12:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP10]], <vscale x 4 x i32> [[TMP11]], i64 12)
+// CHECK-NEXT:    ret <vscale x 16 x i32> [[TMP12]]
+//
+// CPP-CHECK-LABEL: @_Z27test_svqdmulh_single_s32_x411svint32x4_tu11__SVInt32_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[ZDN:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[ZDN]], i64 4)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[ZDN]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[ZDN]], i64 12)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.sqdmulh.single.vgx4.nxv4i32(<vscale x 4 x i32> [[TMP0]], <vscale x 4 x i32> [[TMP1]], <vscale x 4 x i32> [[TMP2]], <vscale x 4 x i32> [[TMP3]], <vscale x 4 x i32> [[ZM:%.*]])
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP4]], 0
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> poison, <vscale x 4 x i32> [[TMP5]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP4]], 1
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP6]], <vscale x 4 x i32> [[TMP7]], i64 4)
+// CPP-CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP4]], 2
+// CPP-CHECK-NEXT:    [[TMP10:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP8]], <vscale x 4 x i32> [[TMP9]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP4]], 3
+// CPP-CHECK-NEXT:    [[TMP12:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP10]], <vscale x 4 x i32> [[TMP11]], i64 12)
+// CPP-CHECK-NEXT:    ret <vscale x 16 x i32> [[TMP12]]
+//
+svint32x4_t test_svqdmulh_single_s32_x4(svint32x4_t zdn, svint32_t zm) __arm_streaming {
+  return SVE_ACLE_FUNC(svqdmulh,_single_s32_x4,,,)(zdn, zm);
+}
+
+// CHECK-LABEL: @test_svqdmulh_single_s64_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[ZDN:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[ZDN]], i64 2)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[ZDN]], i64 4)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[ZDN]], i64 6)
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.sqdmulh.single.vgx4.nxv2i64(<vscale x 2 x i64> [[TMP0]], <vscale x 2 x i64> [[TMP1]], <vscale x 2 x i64> [[TMP2]], <vscale x 2 x i64> [[TMP3]], <vscale x 2 x i64> [[ZM:%.*]])
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP4]], 0
+// CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> poison, <vscale x 2 x i64> [[TMP5]], i64 0)
+// CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP4]], 1
+// CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP6]], <vscale x 2 x i64> [[TMP7]], i64 2)
+// CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP4]], 2
+// CHECK-NEXT:    [[TMP10:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP8]], <vscale x 2 x i64> [[TMP9]], i64 4)
+// CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP4]], 3
+// CHECK-NEXT:    [[TMP12:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP10]], <vscale x 2 x i64> [[TMP11]], i64 6)
+// CHECK-NEXT:    ret <vscale x 8 x i64> [[TMP12]]
+//
+// CPP-CHECK-LABEL: @_Z27test_svqdmulh_single_s64_x411svint64x4_tu11__SVInt64_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[ZDN:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[ZDN]], i64 2)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[ZDN]], i64 4)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[ZDN]], i64 6)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.sqdmulh.single.vgx4.nxv2i64(<vscale x 2 x i64> [[TMP0]], <vscale x 2 x i64> [[TMP1]], <vscale x 2 x i64> [[TMP2]], <vscale x 2 x i64> [[TMP3]], <vscale x 2 x i64> [[ZM:%.*]])
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP4]], 0
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> poison, <vscale x 2 x i64> [[TMP5]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP4]], 1
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP6]], <vscale x 2 x i64> [[TMP7]], i64 2)
+// CPP-CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP4]], 2
+// CPP-CHECK-NEXT:    [[TMP10:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP8]], <vscale x 2 x i64> [[TMP9]], i64 4)
+// CPP-CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP4]], 3
+// CPP-CHECK-NEXT:    [[TMP12:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP10]], <vscale x 2 x i64> [[TMP11]], i64 6)
+// CPP-CHECK-NEXT:    ret <vscale x 8 x i64> [[TMP12]]
+//
+svint64x4_t test_svqdmulh_single_s64_x4(svint64x4_t zdn, svint64_t zm) __arm_streaming {
+  return SVE_ACLE_FUNC(svqdmulh,_single_s64_x4,,,)(zdn, zm);
+}
+
+// Multi, x2
+
+// CHECK-LABEL: @test_svqdmulh_s8_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZDN:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZDN]], i64 16)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZM:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZM]], i64 16)
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.sqdmulh.vgx2.nxv16i8(<vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]])
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP4]], 0
+// CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 32 x i8> @llvm.vector.insert.nxv32i8.nxv16i8(<vscale x 32 x i8> poison, <vscale x 16 x i8> [[TMP5]], i64 0)
+// CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP4]], 1
+// CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 32 x i8> @llvm.vector.insert.nxv32i8.nxv16i8(<vscale x 32 x i8> [[TMP6]], <vscale x 16 x i8> [[TMP7]], i64 16)
+// CHECK-NEXT:    ret <vscale x 32 x i8> [[TMP8]]
+//
+// CPP-CHECK-LABEL: @_Z19test_svqdmulh_s8_x210svint8x2_tS_(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZDN:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZDN]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZM:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv32i8(<vscale x 32 x i8> [[ZM]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.sqdmulh.vgx2.nxv16i8(<vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]])
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP4]], 0
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 32 x i8> @llvm.vector.insert.nxv32i8.nxv16i8(<vscale x 32 x i8> poison, <vscale x 16 x i8> [[TMP5]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP4]], 1
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 32 x i8> @llvm.vector.insert.nxv32i8.nxv16i8(<vscale x 32 x i8> [[TMP6]], <vscale x 16 x i8> [[TMP7]], i64 16)
+// CPP-CHECK-NEXT:    ret <vscale x 32 x i8> [[TMP8]]
+//
+svint8x2_t test_svqdmulh_s8_x2(svint8x2_t zdn, svint8x2_t zm) __arm_streaming {
+  return SVE_ACLE_FUNC(svqdmulh,_s8_x2,,,)(zdn, zm);
+}
+
+// CHECK-LABEL: @test_svqdmulh_s16_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZDN:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZDN]], i64 8)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZM:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZM]], i64 8)
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.sqdmulh.vgx2.nxv8i16(<vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]])
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP4]], 0
+// CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 16 x i16> @llvm.vector.insert.nxv16i16.nxv8i16(<vscale x 16 x i16> poison, <vscale x 8 x i16> [[TMP5]], i64 0)
+// CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP4]], 1
+// CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 16 x i16> @llvm.vector.insert.nxv16i16.nxv8i16(<vscale x 16 x i16> [[TMP6]], <vscale x 8 x i16> [[TMP7]], i64 8)
+// CHECK-NEXT:    ret <vscale x 16 x i16> [[TMP8]]
+//
+// CPP-CHECK-LABEL: @_Z20test_svqdmulh_s16_x211svint16x2_tS_(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZDN:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZDN]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZM:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv16i16(<vscale x 16 x i16> [[ZM]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.sqdmulh.vgx2.nxv8i16(<vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]])
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP4]], 0
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 16 x i16> @llvm.vector.insert.nxv16i16.nxv8i16(<vscale x 16 x i16> poison, <vscale x 8 x i16> [[TMP5]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP4]], 1
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 16 x i16> @llvm.vector.insert.nxv16i16.nxv8i16(<vscale x 16 x i16> [[TMP6]], <vscale x 8 x i16> [[TMP7]], i64 8)
+// CPP-CHECK-NEXT:    ret <vscale x 16 x i16> [[TMP8]]
+//
+svint16x2_t test_svqdmulh_s16_x2(svint16x2_t zdn, svint16x2_t zm) __arm_streaming {
+  return SVE_ACLE_FUNC(svqdmulh,_s16_x2,,,)(zdn, zm);
+}
+
+// CHECK-LABEL: @test_svqdmulh_s32_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[ZDN:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[ZDN]], i64 4)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[ZM:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[ZM]], i64 4)
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.sqdmulh.vgx2.nxv4i32(<vscale x 4 x i32> [[TMP0]], <vscale x 4 x i32> [[TMP1]], <vscale x 4 x i32> [[TMP2]], <vscale x 4 x i32> [[TMP3]])
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP4]], 0
+// CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> poison, <vscale x 4 x i32> [[TMP5]], i64 0)
+// CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP4]], 1
+// CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> [[TMP6]], <vscale x 4 x i32> [[TMP7]], i64 4)
+// CHECK-NEXT:    ret <vscale x 8 x i32> [[TMP8]]
+//
+// CPP-CHECK-LABEL: @_Z20test_svqdmulh_s32_x211svint32x2_tS_(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[ZDN:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[ZDN]], i64 4)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[ZM:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[ZM]], i64 4)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call { <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.sqdmulh.vgx2.nxv4i32(<vscale x 4 x i32> [[TMP0]], <vscale x 4 x i32> [[TMP1]], <vscale x 4 x i32> [[TMP2]], <vscale x 4 x i32> [[TMP3]])
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP4]], 0
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> poison, <vscale x 4 x i32> [[TMP5]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP4]], 1
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 8 x i32> @llvm.vector.insert.nxv8i32.nxv4i32(<vscale x 8 x i32> [[TMP6]], <vscale x 4 x i32> [[TMP7]], i64 4)
+// CPP-CHECK-NEXT:    ret <vscale x 8 x i32> [[TMP8]]
+//
+svint32x2_t test_svqdmulh_s32_x2(svint32x2_t zdn, svint32x2_t zm) __arm_streaming {
+  return SVE_ACLE_FUNC(svqdmulh,_s32_x2,,,)(zdn, zm);
+}
+
+// CHECK-LABEL: @test_svqdmulh_s64_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv4i64(<vscale x 4 x i64> [[ZDN:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv4i64(<vscale x 4 x i64> [[ZDN]], i64 2)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv4i64(<vscale x 4 x i64> [[ZM:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv4i64(<vscale x 4 x i64> [[ZM]], i64 2)
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.sqdmulh.vgx2.nxv2i64(<vscale x 2 x i64> [[TMP0]], <vscale x 2 x i64> [[TMP1]], <vscale x 2 x i64> [[TMP2]], <vscale x 2 x i64> [[TMP3]])
+// CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP4]], 0
+// CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 4 x i64> @llvm.vector.insert.nxv4i64.nxv2i64(<vscale x 4 x i64> poison, <vscale x 2 x i64> [[TMP5]], i64 0)
+// CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP4]], 1
+// CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 4 x i64> @llvm.vector.insert.nxv4i64.nxv2i64(<vscale x 4 x i64> [[TMP6]], <vscale x 2 x i64> [[TMP7]], i64 2)
+// CHECK-NEXT:    ret <vscale x 4 x i64> [[TMP8]]
+//
+// CPP-CHECK-LABEL: @_Z20test_svqdmulh_s64_x211svint64x2_tS_(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv4i64(<vscale x 4 x i64> [[ZDN:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv4i64(<vscale x 4 x i64> [[ZDN]], i64 2)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv4i64(<vscale x 4 x i64> [[ZM:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv4i64(<vscale x 4 x i64> [[ZM]], i64 2)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call { <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.sqdmulh.vgx2.nxv2i64(<vscale x 2 x i64> [[TMP0]], <vscale x 2 x i64> [[TMP1]], <vscale x 2 x i64> [[TMP2]], <vscale x 2 x i64> [[TMP3]])
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP4]], 0
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 4 x i64> @llvm.vector.insert.nxv4i64.nxv2i64(<vscale x 4 x i64> poison, <vscale x 2 x i64> [[TMP5]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP4]], 1
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = tail call <vscale x 4 x i64> @llvm.vector.insert.nxv4i64.nxv2i64(<vscale x 4 x i64> [[TMP6]], <vscale x 2 x i64> [[TMP7]], i64 2)
+// CPP-CHECK-NEXT:    ret <vscale x 4 x i64> [[TMP8]]
+//
+svint64x2_t test_svqdmulh_s64_x2(svint64x2_t zdn, svint64x2_t zm) __arm_streaming {
+  return SVE_ACLE_FUNC(svqdmulh,_s64_x2,,,)(zdn, zm);
+}
+
+// Multi, x4
+
+// CHECK-LABEL: @test_svqdmulh_s8_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZDN:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZDN]], i64 16)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZDN]], i64 32)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZDN]], i64 48)
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 16)
+// CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 32)
+// CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 48)
+// CHECK-NEXT:    [[TMP8:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.sqdmulh.vgx4.nxv16i8(<vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[TMP4]], <vscale x 16 x i8> [[TMP5]], <vscale x 16 x i8> [[TMP6]], <vscale x 16 x i8> [[TMP7]])
+// CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP8]], 0
+// CHECK-NEXT:    [[TMP10:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> poison, <vscale x 16 x i8> [[TMP9]], i64 0)
+// CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP8]], 1
+// CHECK-NEXT:    [[TMP12:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP10]], <vscale x 16 x i8> [[TMP11]], i64 16)
+// CHECK-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP8]], 2
+// CHECK-NEXT:    [[TMP14:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP12]], <vscale x 16 x i8> [[TMP13]], i64 32)
+// CHECK-NEXT:    [[TMP15:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP8]], 3
+// CHECK-NEXT:    [[TMP16:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP14]], <vscale x 16 x i8> [[TMP15]], i64 48)
+// CHECK-NEXT:    ret <vscale x 64 x i8> [[TMP16]]
+//
+// CPP-CHECK-LABEL: @_Z19test_svqdmulh_s8_x410svint8x4_tS_(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZDN:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZDN]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZDN]], i64 32)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZDN]], i64 48)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 32)
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 16 x i8> @llvm.vector.extract.nxv16i8.nxv64i8(<vscale x 64 x i8> [[ZM]], i64 48)
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = tail call { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } @llvm.aarch64.sve.sqdmulh.vgx4.nxv16i8(<vscale x 16 x i8> [[TMP0]], <vscale x 16 x i8> [[TMP1]], <vscale x 16 x i8> [[TMP2]], <vscale x 16 x i8> [[TMP3]], <vscale x 16 x i8> [[TMP4]], <vscale x 16 x i8> [[TMP5]], <vscale x 16 x i8> [[TMP6]], <vscale x 16 x i8> [[TMP7]])
+// CPP-CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP8]], 0
+// CPP-CHECK-NEXT:    [[TMP10:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> poison, <vscale x 16 x i8> [[TMP9]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP8]], 1
+// CPP-CHECK-NEXT:    [[TMP12:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP10]], <vscale x 16 x i8> [[TMP11]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP8]], 2
+// CPP-CHECK-NEXT:    [[TMP14:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP12]], <vscale x 16 x i8> [[TMP13]], i64 32)
+// CPP-CHECK-NEXT:    [[TMP15:%.*]] = extractvalue { <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8>, <vscale x 16 x i8> } [[TMP8]], 3
+// CPP-CHECK-NEXT:    [[TMP16:%.*]] = tail call <vscale x 64 x i8> @llvm.vector.insert.nxv64i8.nxv16i8(<vscale x 64 x i8> [[TMP14]], <vscale x 16 x i8> [[TMP15]], i64 48)
+// CPP-CHECK-NEXT:    ret <vscale x 64 x i8> [[TMP16]]
+//
+svint8x4_t test_svqdmulh_s8_x4(svint8x4_t zdn, svint8x4_t zm) __arm_streaming {
+  return SVE_ACLE_FUNC(svqdmulh,_s8_x4,,,)(zdn, zm);
+}
+
+// CHECK-LABEL: @test_svqdmulh_s16_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZDN:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZDN]], i64 8)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZDN]], i64 16)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZDN]], i64 24)
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 8)
+// CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 16)
+// CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 24)
+// CHECK-NEXT:    [[TMP8:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.sqdmulh.vgx4.nxv8i16(<vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP4]], <vscale x 8 x i16> [[TMP5]], <vscale x 8 x i16> [[TMP6]], <vscale x 8 x i16> [[TMP7]])
+// CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP8]], 0
+// CHECK-NEXT:    [[TMP10:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> poison, <vscale x 8 x i16> [[TMP9]], i64 0)
+// CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP8]], 1
+// CHECK-NEXT:    [[TMP12:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP10]], <vscale x 8 x i16> [[TMP11]], i64 8)
+// CHECK-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP8]], 2
+// CHECK-NEXT:    [[TMP14:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP12]], <vscale x 8 x i16> [[TMP13]], i64 16)
+// CHECK-NEXT:    [[TMP15:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP8]], 3
+// CHECK-NEXT:    [[TMP16:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP14]], <vscale x 8 x i16> [[TMP15]], i64 24)
+// CHECK-NEXT:    ret <vscale x 32 x i16> [[TMP16]]
+//
+// CPP-CHECK-LABEL: @_Z20test_svqdmulh_s16_x411svint16x4_tS_(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZDN:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZDN]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZDN]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZDN]], i64 24)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 8 x i16> @llvm.vector.extract.nxv8i16.nxv32i16(<vscale x 32 x i16> [[ZM]], i64 24)
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = tail call { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } @llvm.aarch64.sve.sqdmulh.vgx4.nxv8i16(<vscale x 8 x i16> [[TMP0]], <vscale x 8 x i16> [[TMP1]], <vscale x 8 x i16> [[TMP2]], <vscale x 8 x i16> [[TMP3]], <vscale x 8 x i16> [[TMP4]], <vscale x 8 x i16> [[TMP5]], <vscale x 8 x i16> [[TMP6]], <vscale x 8 x i16> [[TMP7]])
+// CPP-CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP8]], 0
+// CPP-CHECK-NEXT:    [[TMP10:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> poison, <vscale x 8 x i16> [[TMP9]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP8]], 1
+// CPP-CHECK-NEXT:    [[TMP12:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP10]], <vscale x 8 x i16> [[TMP11]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP8]], 2
+// CPP-CHECK-NEXT:    [[TMP14:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP12]], <vscale x 8 x i16> [[TMP13]], i64 16)
+// CPP-CHECK-NEXT:    [[TMP15:%.*]] = extractvalue { <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16>, <vscale x 8 x i16> } [[TMP8]], 3
+// CPP-CHECK-NEXT:    [[TMP16:%.*]] = tail call <vscale x 32 x i16> @llvm.vector.insert.nxv32i16.nxv8i16(<vscale x 32 x i16> [[TMP14]], <vscale x 8 x i16> [[TMP15]], i64 24)
+// CPP-CHECK-NEXT:    ret <vscale x 32 x i16> [[TMP16]]
+//
+svint16x4_t test_svqdmulh_s16_x4(svint16x4_t zdn, svint16x4_t zm) __arm_streaming {
+  return SVE_ACLE_FUNC(svqdmulh,_s16_x4,,,)(zdn, zm);
+}
+
+// CHECK-LABEL: @test_svqdmulh_s32_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[ZDN:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[ZDN]], i64 4)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[ZDN]], i64 8)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[ZDN]], i64 12)
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[ZM:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[ZM]], i64 4)
+// CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[ZM]], i64 8)
+// CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[ZM]], i64 12)
+// CHECK-NEXT:    [[TMP8:%.*]] = tail call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.sqdmulh.vgx4.nxv4i32(<vscale x 4 x i32> [[TMP0]], <vscale x 4 x i32> [[TMP1]], <vscale x 4 x i32> [[TMP2]], <vscale x 4 x i32> [[TMP3]], <vscale x 4 x i32> [[TMP4]], <vscale x 4 x i32> [[TMP5]], <vscale x 4 x i32> [[TMP6]], <vscale x 4 x i32> [[TMP7]])
+// CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP8]], 0
+// CHECK-NEXT:    [[TMP10:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> poison, <vscale x 4 x i32> [[TMP9]], i64 0)
+// CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP8]], 1
+// CHECK-NEXT:    [[TMP12:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP10]], <vscale x 4 x i32> [[TMP11]], i64 4)
+// CHECK-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP8]], 2
+// CHECK-NEXT:    [[TMP14:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP12]], <vscale x 4 x i32> [[TMP13]], i64 8)
+// CHECK-NEXT:    [[TMP15:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP8]], 3
+// CHECK-NEXT:    [[TMP16:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP14]], <vscale x 4 x i32> [[TMP15]], i64 12)
+// CHECK-NEXT:    ret <vscale x 16 x i32> [[TMP16]]
+//
+// CPP-CHECK-LABEL: @_Z20test_svqdmulh_s32_x411svint32x4_tS_(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[ZDN:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[ZDN]], i64 4)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[ZDN]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[ZDN]], i64 12)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[ZM:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[ZM]], i64 4)
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[ZM]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv16i32(<vscale x 16 x i32> [[ZM]], i64 12)
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = tail call { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } @llvm.aarch64.sve.sqdmulh.vgx4.nxv4i32(<vscale x 4 x i32> [[TMP0]], <vscale x 4 x i32> [[TMP1]], <vscale x 4 x i32> [[TMP2]], <vscale x 4 x i32> [[TMP3]], <vscale x 4 x i32> [[TMP4]], <vscale x 4 x i32> [[TMP5]], <vscale x 4 x i32> [[TMP6]], <vscale x 4 x i32> [[TMP7]])
+// CPP-CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP8]], 0
+// CPP-CHECK-NEXT:    [[TMP10:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> poison, <vscale x 4 x i32> [[TMP9]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP8]], 1
+// CPP-CHECK-NEXT:    [[TMP12:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP10]], <vscale x 4 x i32> [[TMP11]], i64 4)
+// CPP-CHECK-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP8]], 2
+// CPP-CHECK-NEXT:    [[TMP14:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP12]], <vscale x 4 x i32> [[TMP13]], i64 8)
+// CPP-CHECK-NEXT:    [[TMP15:%.*]] = extractvalue { <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32>, <vscale x 4 x i32> } [[TMP8]], 3
+// CPP-CHECK-NEXT:    [[TMP16:%.*]] = tail call <vscale x 16 x i32> @llvm.vector.insert.nxv16i32.nxv4i32(<vscale x 16 x i32> [[TMP14]], <vscale x 4 x i32> [[TMP15]], i64 12)
+// CPP-CHECK-NEXT:    ret <vscale x 16 x i32> [[TMP16]]
+//
+svint32x4_t test_svqdmulh_s32_x4(svint32x4_t zdn, svint32x4_t zm) __arm_streaming {
+  return SVE_ACLE_FUNC(svqdmulh,_s32_x4,,,)(zdn, zm);
+}
+
+// CHECK-LABEL: @test_svqdmulh_s64_x4(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[ZDN:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[ZDN]], i64 2)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[ZDN]], i64 4)
+// CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[ZDN]], i64 6)
+// CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[ZM:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[ZM]], i64 2)
+// CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[ZM]], i64 4)
+// CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[ZM]], i64 6)
+// CHECK-NEXT:    [[TMP8:%.*]] = tail call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.sqdmulh.vgx4.nxv2i64(<vscale x 2 x i64> [[TMP0]], <vscale x 2 x i64> [[TMP1]], <vscale x 2 x i64> [[TMP2]], <vscale x 2 x i64> [[TMP3]], <vscale x 2 x i64> [[TMP4]], <vscale x 2 x i64> [[TMP5]], <vscale x 2 x i64> [[TMP6]], <vscale x 2 x i64> [[TMP7]])
+// CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP8]], 0
+// CHECK-NEXT:    [[TMP10:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> poison, <vscale x 2 x i64> [[TMP9]], i64 0)
+// CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP8]], 1
+// CHECK-NEXT:    [[TMP12:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP10]], <vscale x 2 x i64> [[TMP11]], i64 2)
+// CHECK-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP8]], 2
+// CHECK-NEXT:    [[TMP14:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP12]], <vscale x 2 x i64> [[TMP13]], i64 4)
+// CHECK-NEXT:    [[TMP15:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP8]], 3
+// CHECK-NEXT:    [[TMP16:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP14]], <vscale x 2 x i64> [[TMP15]], i64 6)
+// CHECK-NEXT:    ret <vscale x 8 x i64> [[TMP16]]
+//
+// CPP-CHECK-LABEL: @_Z20test_svqdmulh_s64_x411svint64x4_tS_(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[ZDN:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[ZDN]], i64 2)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[ZDN]], i64 4)
+// CPP-CHECK-NEXT:    [[TMP3:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[ZDN]], i64 6)
+// CPP-CHECK-NEXT:    [[TMP4:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[ZM:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP5:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[ZM]], i64 2)
+// CPP-CHECK-NEXT:    [[TMP6:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[ZM]], i64 4)
+// CPP-CHECK-NEXT:    [[TMP7:%.*]] = tail call <vscale x 2 x i64> @llvm.vector.extract.nxv2i64.nxv8i64(<vscale x 8 x i64> [[ZM]], i64 6)
+// CPP-CHECK-NEXT:    [[TMP8:%.*]] = tail call { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } @llvm.aarch64.sve.sqdmulh.vgx4.nxv2i64(<vscale x 2 x i64> [[TMP0]], <vscale x 2 x i64> [[TMP1]], <vscale x 2 x i64> [[TMP2]], <vscale x 2 x i64> [[TMP3]], <vscale x 2 x i64> [[TMP4]], <vscale x 2 x i64> [[TMP5]], <vscale x 2 x i64> [[TMP6]], <vscale x 2 x i64> [[TMP7]])
+// CPP-CHECK-NEXT:    [[TMP9:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP8]], 0
+// CPP-CHECK-NEXT:    [[TMP10:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> poison, <vscale x 2 x i64> [[TMP9]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP11:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP8]], 1
+// CPP-CHECK-NEXT:    [[TMP12:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP10]], <vscale x 2 x i64> [[TMP11]], i64 2)
+// CPP-CHECK-NEXT:    [[TMP13:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP8]], 2
+// CPP-CHECK-NEXT:    [[TMP14:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP12]], <vscale x 2 x i64> [[TMP13]], i64 4)
+// CPP-CHECK-NEXT:    [[TMP15:%.*]] = extractvalue { <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64>, <vscale x 2 x i64> } [[TMP8]], 3
+// CPP-CHECK-NEXT:    [[TMP16:%.*]] = tail call <vscale x 8 x i64> @llvm.vector.insert.nxv8i64.nxv2i64(<vscale x 8 x i64> [[TMP14]], <vscale x 2 x i64> [[TMP15]], i64 6)
+// CPP-CHECK-NEXT:    ret <vscale x 8 x i64> [[TMP16]]
+//
+svint64x4_t test_svqdmulh_s64_x4(svint64x4_t zdn, svint64x4_t zm) __arm_streaming {
+  return SVE_ACLE_FUNC(svqdmulh,_s64_x4,,,)(zdn, zm);
+}
diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_fp_reduce.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_fp_reduce.c
new file mode 100644
index 0000000000000000000000000000000000000000..e58cf4e49a37f92c8ca2db3d19db54beb5349430
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_fp_reduce.c
@@ -0,0 +1,285 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s
+#include <arm_neon.h>
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
+#endif
+
+// FADDQV
+
+// CHECK-LABEL: @test_svaddqv_f16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x half> @llvm.aarch64.sve.addqv.v8f16.nxv8f16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x half> [[OP:%.*]])
+// CHECK-NEXT:    ret <8 x half> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z16test_svaddqv_f16u10__SVBool_tu13__SVFloat16_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x half> @llvm.aarch64.sve.addqv.v8f16.nxv8f16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x half> [[OP:%.*]])
+// CPP-CHECK-NEXT:    ret <8 x half> [[TMP1]]
+//
+float16x8_t test_svaddqv_f16(svbool_t pg, svfloat16_t op)
+{
+  return SVE_ACLE_FUNC(svaddqv,,_f16,)(pg, op);
+}
+
+// CHECK-LABEL: @test_svaddqv_f32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x float> @llvm.aarch64.sve.addqv.v4f32.nxv4f32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x float> [[OP:%.*]])
+// CHECK-NEXT:    ret <4 x float> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z16test_svaddqv_f32u10__SVBool_tu13__SVFloat32_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x float> @llvm.aarch64.sve.addqv.v4f32.nxv4f32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x float> [[OP:%.*]])
+// CPP-CHECK-NEXT:    ret <4 x float> [[TMP1]]
+//
+float32x4_t test_svaddqv_f32(svbool_t pg, svfloat32_t op)
+{
+  return SVE_ACLE_FUNC(svaddqv,,_f32,)(pg, op);
+}
+
+// CHECK-LABEL: @test_svaddqv_f64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x double> @llvm.aarch64.sve.addqv.v2f64.nxv2f64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x double> [[OP:%.*]])
+// CHECK-NEXT:    ret <2 x double> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z16test_svaddqv_f64u10__SVBool_tu13__SVFloat64_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x double> @llvm.aarch64.sve.addqv.v2f64.nxv2f64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x double> [[OP:%.*]])
+// CPP-CHECK-NEXT:    ret <2 x double> [[TMP1]]
+//
+float64x2_t test_svaddqv_f64(svbool_t pg, svfloat64_t op)
+{
+  return SVE_ACLE_FUNC(svaddqv,,_f64,)(pg, op);
+}
+
+
+// FMAXQV
+
+// CHECK-LABEL: @test_svmaxqv_f16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x half> @llvm.aarch64.sve.fmaxqv.v8f16.nxv8f16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x half> [[OP:%.*]])
+// CHECK-NEXT:    ret <8 x half> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z16test_svmaxqv_f16u10__SVBool_tu13__SVFloat16_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x half> @llvm.aarch64.sve.fmaxqv.v8f16.nxv8f16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x half> [[OP:%.*]])
+// CPP-CHECK-NEXT:    ret <8 x half> [[TMP1]]
+//
+float16x8_t test_svmaxqv_f16(svbool_t pg, svfloat16_t op)
+{
+  return SVE_ACLE_FUNC(svmaxqv,,_f16,)(pg, op);
+}
+
+// CHECK-LABEL: @test_svmaxqv_f32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x float> @llvm.aarch64.sve.fmaxqv.v4f32.nxv4f32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x float> [[OP:%.*]])
+// CHECK-NEXT:    ret <4 x float> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z16test_svmaxqv_f32u10__SVBool_tu13__SVFloat32_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x float> @llvm.aarch64.sve.fmaxqv.v4f32.nxv4f32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x float> [[OP:%.*]])
+// CPP-CHECK-NEXT:    ret <4 x float> [[TMP1]]
+//
+float32x4_t test_svmaxqv_f32(svbool_t pg, svfloat32_t op)
+{
+  return SVE_ACLE_FUNC(svmaxqv,,_f32,)(pg, op);
+}
+
+// CHECK-LABEL: @test_svmaxqv_f64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x double> @llvm.aarch64.sve.fmaxqv.v2f64.nxv2f64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x double> [[OP:%.*]])
+// CHECK-NEXT:    ret <2 x double> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z16test_svmaxqv_f64u10__SVBool_tu13__SVFloat64_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x double> @llvm.aarch64.sve.fmaxqv.v2f64.nxv2f64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x double> [[OP:%.*]])
+// CPP-CHECK-NEXT:    ret <2 x double> [[TMP1]]
+//
+float64x2_t test_svmaxqv_f64(svbool_t pg, svfloat64_t op)
+{
+  return SVE_ACLE_FUNC(svmaxqv,,_f64,)(pg, op);
+}
+
+
+// FMINQV
+
+// CHECK-LABEL: @test_svminqv_f16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x half> @llvm.aarch64.sve.fminqv.v8f16.nxv8f16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x half> [[OP:%.*]])
+// CHECK-NEXT:    ret <8 x half> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z16test_svminqv_f16u10__SVBool_tu13__SVFloat16_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x half> @llvm.aarch64.sve.fminqv.v8f16.nxv8f16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x half> [[OP:%.*]])
+// CPP-CHECK-NEXT:    ret <8 x half> [[TMP1]]
+//
+float16x8_t test_svminqv_f16(svbool_t pg, svfloat16_t op)
+{
+  return SVE_ACLE_FUNC(svminqv,,_f16,)(pg, op);
+}
+
+// CHECK-LABEL: @test_svminqv_f32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x float> @llvm.aarch64.sve.fminqv.v4f32.nxv4f32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x float> [[OP:%.*]])
+// CHECK-NEXT:    ret <4 x float> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z16test_svminqv_f32u10__SVBool_tu13__SVFloat32_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x float> @llvm.aarch64.sve.fminqv.v4f32.nxv4f32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x float> [[OP:%.*]])
+// CPP-CHECK-NEXT:    ret <4 x float> [[TMP1]]
+//
+float32x4_t test_svminqv_f32(svbool_t pg, svfloat32_t op)
+{
+  return SVE_ACLE_FUNC(svminqv,,_f32,)(pg, op);
+}
+
+// CHECK-LABEL: @test_svminqv_f64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x double> @llvm.aarch64.sve.fminqv.v2f64.nxv2f64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x double> [[OP:%.*]])
+// CHECK-NEXT:    ret <2 x double> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z16test_svminqv_f64u10__SVBool_tu13__SVFloat64_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x double> @llvm.aarch64.sve.fminqv.v2f64.nxv2f64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x double> [[OP:%.*]])
+// CPP-CHECK-NEXT:    ret <2 x double> [[TMP1]]
+//
+float64x2_t test_svminqv_f64(svbool_t pg, svfloat64_t op)
+{
+  return SVE_ACLE_FUNC(svminqv,,_f64,)(pg, op);
+}
+
+
+// FMAXNMQV
+
+// CHECK-LABEL: @test_svmaxnmqv_f16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x half> @llvm.aarch64.sve.fmaxnmqv.v8f16.nxv8f16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x half> [[OP:%.*]])
+// CHECK-NEXT:    ret <8 x half> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z18test_svmaxnmqv_f16u10__SVBool_tu13__SVFloat16_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x half> @llvm.aarch64.sve.fmaxnmqv.v8f16.nxv8f16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x half> [[OP:%.*]])
+// CPP-CHECK-NEXT:    ret <8 x half> [[TMP1]]
+//
+float16x8_t test_svmaxnmqv_f16(svbool_t pg, svfloat16_t op)
+{
+  return SVE_ACLE_FUNC(svmaxnmqv,,_f16,)(pg, op);
+}
+
+// CHECK-LABEL: @test_svmaxnmqv_f32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x float> @llvm.aarch64.sve.fmaxnmqv.v4f32.nxv4f32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x float> [[OP:%.*]])
+// CHECK-NEXT:    ret <4 x float> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z18test_svmaxnmqv_f32u10__SVBool_tu13__SVFloat32_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x float> @llvm.aarch64.sve.fmaxnmqv.v4f32.nxv4f32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x float> [[OP:%.*]])
+// CPP-CHECK-NEXT:    ret <4 x float> [[TMP1]]
+//
+float32x4_t test_svmaxnmqv_f32(svbool_t pg, svfloat32_t op)
+{
+  return SVE_ACLE_FUNC(svmaxnmqv,,_f32,)(pg, op);
+}
+
+// CHECK-LABEL: @test_svmaxnmqv_f64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x double> @llvm.aarch64.sve.fmaxnmqv.v2f64.nxv2f64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x double> [[OP:%.*]])
+// CHECK-NEXT:    ret <2 x double> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z18test_svmaxnmqv_f64u10__SVBool_tu13__SVFloat64_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x double> @llvm.aarch64.sve.fmaxnmqv.v2f64.nxv2f64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x double> [[OP:%.*]])
+// CPP-CHECK-NEXT:    ret <2 x double> [[TMP1]]
+//
+float64x2_t test_svmaxnmqv_f64(svbool_t pg, svfloat64_t op)
+{
+  return SVE_ACLE_FUNC(svmaxnmqv,,_f64,)(pg, op);
+}
+
+
+// FMINNMQV
+
+// CHECK-LABEL: @test_svminnmqv_f16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x half> @llvm.aarch64.sve.fminnmqv.v8f16.nxv8f16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x half> [[OP:%.*]])
+// CHECK-NEXT:    ret <8 x half> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z18test_svminnmqv_f16u10__SVBool_tu13__SVFloat16_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x half> @llvm.aarch64.sve.fminnmqv.v8f16.nxv8f16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x half> [[OP:%.*]])
+// CPP-CHECK-NEXT:    ret <8 x half> [[TMP1]]
+//
+float16x8_t test_svminnmqv_f16(svbool_t pg, svfloat16_t op)
+{
+  return SVE_ACLE_FUNC(svminnmqv,,_f16,)(pg, op);
+}
+
+// CHECK-LABEL: @test_svminnmqv_f32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x float> @llvm.aarch64.sve.fminnmqv.v4f32.nxv4f32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x float> [[OP:%.*]])
+// CHECK-NEXT:    ret <4 x float> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z18test_svminnmqv_f32u10__SVBool_tu13__SVFloat32_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x float> @llvm.aarch64.sve.fminnmqv.v4f32.nxv4f32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x float> [[OP:%.*]])
+// CPP-CHECK-NEXT:    ret <4 x float> [[TMP1]]
+//
+float32x4_t test_svminnmqv_f32(svbool_t pg, svfloat32_t op)
+{
+  return SVE_ACLE_FUNC(svminnmqv,,_f32,)(pg, op);
+}
+
+// CHECK-LABEL: @test_svminnmqv_f64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x double> @llvm.aarch64.sve.fminnmqv.v2f64.nxv2f64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x double> [[OP:%.*]])
+// CHECK-NEXT:    ret <2 x double> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z18test_svminnmqv_f64u10__SVBool_tu13__SVFloat64_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x double> @llvm.aarch64.sve.fminnmqv.v2f64.nxv2f64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x double> [[OP:%.*]])
+// CPP-CHECK-NEXT:    ret <2 x double> [[TMP1]]
+//
+float64x2_t test_svminnmqv_f64(svbool_t pg, svfloat64_t op)
+{
+  return SVE_ACLE_FUNC(svminnmqv,,_f64,)(pg, op);
+}
diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_int_reduce.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_int_reduce.c
new file mode 100644
index 0000000000000000000000000000000000000000..b395b4d1323ed5ebf685cf62b1c8c38b273e1b32
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_int_reduce.c
@@ -0,0 +1,783 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - %s | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -Wall -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4
+#endif
+
+
+// ADDQV
+
+// CHECK-LABEL: @test_svaddqv_s8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <16 x i8> @llvm.aarch64.sve.addqv.v16i8.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[OP1:%.*]])
+// CHECK-NEXT:    ret <16 x i8> [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z15test_svaddqv_s8u10__SVBool_tu10__SVInt8_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <16 x i8> @llvm.aarch64.sve.addqv.v16i8.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[OP1:%.*]])
+// CPP-CHECK-NEXT:    ret <16 x i8> [[TMP0]]
+//
+int8x16_t test_svaddqv_s8(svbool_t pg, svint8_t op1) {
+  return SVE_ACLE_FUNC(svaddqv,_s8,,)(pg, op1);
+}
+
+// CHECK-LABEL: @test_svaddqv_s16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i16> @llvm.aarch64.sve.addqv.v8i16.nxv8i16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[OP1:%.*]])
+// CHECK-NEXT:    ret <8 x i16> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z16test_svaddqv_s16u10__SVBool_tu11__SVInt16_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i16> @llvm.aarch64.sve.addqv.v8i16.nxv8i16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[OP1:%.*]])
+// CPP-CHECK-NEXT:    ret <8 x i16> [[TMP1]]
+//
+int16x8_t test_svaddqv_s16(svbool_t pg, svint16_t op1) {
+  return SVE_ACLE_FUNC(svaddqv,_s16,,)(pg, op1);
+}
+
+// CHECK-LABEL: @test_svaddqv_s32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x i32> @llvm.aarch64.sve.addqv.v4i32.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[OP1:%.*]])
+// CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z16test_svaddqv_s32u10__SVBool_tu11__SVInt32_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x i32> @llvm.aarch64.sve.addqv.v4i32.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[OP1:%.*]])
+// CPP-CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+//
+int32x4_t test_svaddqv_s32(svbool_t pg, svint32_t op1) {
+  return SVE_ACLE_FUNC(svaddqv,_s32,,)(pg, op1);
+}
+
+// CHECK-LABEL: @test_svaddqv_s64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x i64> @llvm.aarch64.sve.addqv.v2i64.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[OP1:%.*]])
+// CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z16test_svaddqv_s64u10__SVBool_tu11__SVInt64_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x i64> @llvm.aarch64.sve.addqv.v2i64.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[OP1:%.*]])
+// CPP-CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+//
+int64x2_t test_svaddqv_s64(svbool_t pg, svint64_t op1) {
+  return SVE_ACLE_FUNC(svaddqv,_s64,,)(pg, op1);
+}
+
+// CHECK-LABEL: @test_svaddqv_u8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <16 x i8> @llvm.aarch64.sve.addqv.v16i8.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[OP1:%.*]])
+// CHECK-NEXT:    ret <16 x i8> [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z15test_svaddqv_u8u10__SVBool_tu11__SVUint8_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <16 x i8> @llvm.aarch64.sve.addqv.v16i8.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[OP1:%.*]])
+// CPP-CHECK-NEXT:    ret <16 x i8> [[TMP0]]
+//
+uint8x16_t test_svaddqv_u8(svbool_t pg, svuint8_t op1) {
+  return SVE_ACLE_FUNC(svaddqv,_u8,,)(pg, op1);
+}
+
+// CHECK-LABEL: @test_svaddqv_u16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i16> @llvm.aarch64.sve.addqv.v8i16.nxv8i16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[OP1:%.*]])
+// CHECK-NEXT:    ret <8 x i16> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z16test_svaddqv_u16u10__SVBool_tu12__SVUint16_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i16> @llvm.aarch64.sve.addqv.v8i16.nxv8i16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[OP1:%.*]])
+// CPP-CHECK-NEXT:    ret <8 x i16> [[TMP1]]
+//
+uint16x8_t test_svaddqv_u16(svbool_t pg, svuint16_t op1) {
+  return SVE_ACLE_FUNC(svaddqv,_u16,,)(pg, op1);
+}
+
+// CHECK-LABEL: @test_svaddqv_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x i32> @llvm.aarch64.sve.addqv.v4i32.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[OP1:%.*]])
+// CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z16test_svaddqv_u32u10__SVBool_tu12__SVUint32_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x i32> @llvm.aarch64.sve.addqv.v4i32.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[OP1:%.*]])
+// CPP-CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+//
+uint32x4_t test_svaddqv_u32(svbool_t pg, svuint32_t op1) {
+  return SVE_ACLE_FUNC(svaddqv,_u32,,)(pg, op1);
+}
+
+// CHECK-LABEL: @test_svaddqv_u64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x i64> @llvm.aarch64.sve.addqv.v2i64.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[OP1:%.*]])
+// CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z16test_svaddqv_u64u10__SVBool_tu12__SVUint64_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x i64> @llvm.aarch64.sve.addqv.v2i64.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[OP1:%.*]])
+// CPP-CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+//
+uint64x2_t test_svaddqv_u64(svbool_t pg, svuint64_t op1) {
+  return SVE_ACLE_FUNC(svaddqv,_u64,,)(pg, op1);
+}
+
+
+// ANDQV
+
+// CHECK-LABEL: @test_svandqv_s8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <16 x i8> @llvm.aarch64.sve.andqv.v16i8.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[OP1:%.*]])
+// CHECK-NEXT:    ret <16 x i8> [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z15test_svandqv_s8u10__SVBool_tu10__SVInt8_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <16 x i8> @llvm.aarch64.sve.andqv.v16i8.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[OP1:%.*]])
+// CPP-CHECK-NEXT:    ret <16 x i8> [[TMP0]]
+//
+int8x16_t test_svandqv_s8(svbool_t pg, svint8_t op1) {
+  return SVE_ACLE_FUNC(svandqv,_s8,,)(pg, op1);
+}
+
+// CHECK-LABEL: @test_svandqv_s16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i16> @llvm.aarch64.sve.andqv.v8i16.nxv8i16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[OP1:%.*]])
+// CHECK-NEXT:    ret <8 x i16> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z16test_svandqv_s16u10__SVBool_tu11__SVInt16_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i16> @llvm.aarch64.sve.andqv.v8i16.nxv8i16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[OP1:%.*]])
+// CPP-CHECK-NEXT:    ret <8 x i16> [[TMP1]]
+//
+int16x8_t test_svandqv_s16(svbool_t pg, svint16_t op1) {
+  return SVE_ACLE_FUNC(svandqv,_s16,,)(pg, op1);
+}
+
+// CHECK-LABEL: @test_svandqv_s32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x i32> @llvm.aarch64.sve.andqv.v4i32.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[OP1:%.*]])
+// CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z16test_svandqv_s32u10__SVBool_tu11__SVInt32_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x i32> @llvm.aarch64.sve.andqv.v4i32.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[OP1:%.*]])
+// CPP-CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+//
+int32x4_t test_svandqv_s32(svbool_t pg, svint32_t op1) {
+  return SVE_ACLE_FUNC(svandqv,_s32,,)(pg, op1);
+}
+
+// CHECK-LABEL: @test_svandqv_s64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x i64> @llvm.aarch64.sve.andqv.v2i64.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[OP1:%.*]])
+// CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z16test_svandqv_s64u10__SVBool_tu11__SVInt64_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x i64> @llvm.aarch64.sve.andqv.v2i64.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[OP1:%.*]])
+// CPP-CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+//
+int64x2_t test_svandqv_s64(svbool_t pg, svint64_t op1) {
+  return SVE_ACLE_FUNC(svandqv,_s64,,)(pg, op1);
+}
+
+// CHECK-LABEL: @test_svandqv_u8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <16 x i8> @llvm.aarch64.sve.andqv.v16i8.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[OP1:%.*]])
+// CHECK-NEXT:    ret <16 x i8> [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z15test_svandqv_u8u10__SVBool_tu11__SVUint8_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <16 x i8> @llvm.aarch64.sve.andqv.v16i8.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[OP1:%.*]])
+// CPP-CHECK-NEXT:    ret <16 x i8> [[TMP0]]
+//
+uint8x16_t test_svandqv_u8(svbool_t pg, svuint8_t op1) {
+  return SVE_ACLE_FUNC(svandqv,_u8,,)(pg, op1);
+}
+
+// CHECK-LABEL: @test_svandqv_u16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i16> @llvm.aarch64.sve.andqv.v8i16.nxv8i16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[OP1:%.*]])
+// CHECK-NEXT:    ret <8 x i16> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z16test_svandqv_u16u10__SVBool_tu12__SVUint16_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i16> @llvm.aarch64.sve.andqv.v8i16.nxv8i16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[OP1:%.*]])
+// CPP-CHECK-NEXT:    ret <8 x i16> [[TMP1]]
+//
+uint16x8_t test_svandqv_u16(svbool_t pg, svuint16_t op1) {
+  return SVE_ACLE_FUNC(svandqv,_u16,,)(pg, op1);
+}
+
+// CHECK-LABEL: @test_svandqv_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x i32> @llvm.aarch64.sve.andqv.v4i32.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[OP1:%.*]])
+// CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z16test_svandqv_u32u10__SVBool_tu12__SVUint32_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x i32> @llvm.aarch64.sve.andqv.v4i32.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[OP1:%.*]])
+// CPP-CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+//
+uint32x4_t test_svandqv_u32(svbool_t pg, svuint32_t op1) {
+  return SVE_ACLE_FUNC(svandqv,_u32,,)(pg, op1);
+}
+
+// CHECK-LABEL: @test_svandqv_u64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x i64> @llvm.aarch64.sve.andqv.v2i64.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[OP1:%.*]])
+// CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z16test_svandqv_u64u10__SVBool_tu12__SVUint64_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x i64> @llvm.aarch64.sve.andqv.v2i64.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[OP1:%.*]])
+// CPP-CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+//
+uint64x2_t test_svandqv_u64(svbool_t pg, svuint64_t op1) {
+  return SVE_ACLE_FUNC(svandqv,_u64,,)(pg, op1);
+}
+
+
+// EORQV
+
+// CHECK-LABEL: @test_sveorqv_s8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <16 x i8> @llvm.aarch64.sve.eorqv.v16i8.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[OP1:%.*]])
+// CHECK-NEXT:    ret <16 x i8> [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z15test_sveorqv_s8u10__SVBool_tu10__SVInt8_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <16 x i8> @llvm.aarch64.sve.eorqv.v16i8.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[OP1:%.*]])
+// CPP-CHECK-NEXT:    ret <16 x i8> [[TMP0]]
+//
+int8x16_t test_sveorqv_s8(svbool_t pg, svint8_t op1) {
+  return SVE_ACLE_FUNC(sveorqv,_s8,,)(pg, op1);
+}
+
+// CHECK-LABEL: @test_sveorqv_s16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i16> @llvm.aarch64.sve.eorqv.v8i16.nxv8i16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[OP1:%.*]])
+// CHECK-NEXT:    ret <8 x i16> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z16test_sveorqv_s16u10__SVBool_tu11__SVInt16_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i16> @llvm.aarch64.sve.eorqv.v8i16.nxv8i16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[OP1:%.*]])
+// CPP-CHECK-NEXT:    ret <8 x i16> [[TMP1]]
+//
+int16x8_t test_sveorqv_s16(svbool_t pg, svint16_t op1) {
+  return SVE_ACLE_FUNC(sveorqv,_s16,,)(pg, op1);
+}
+
+// CHECK-LABEL: @test_sveorqv_s32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x i32> @llvm.aarch64.sve.eorqv.v4i32.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[OP1:%.*]])
+// CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z16test_sveorqv_s32u10__SVBool_tu11__SVInt32_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x i32> @llvm.aarch64.sve.eorqv.v4i32.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[OP1:%.*]])
+// CPP-CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+//
+int32x4_t test_sveorqv_s32(svbool_t pg, svint32_t op1) {
+  return SVE_ACLE_FUNC(sveorqv,_s32,,)(pg, op1);
+}
+
+// CHECK-LABEL: @test_sveorqv_s64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x i64> @llvm.aarch64.sve.eorqv.v2i64.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[OP1:%.*]])
+// CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z16test_sveorqv_s64u10__SVBool_tu11__SVInt64_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x i64> @llvm.aarch64.sve.eorqv.v2i64.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[OP1:%.*]])
+// CPP-CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+//
+int64x2_t test_sveorqv_s64(svbool_t pg, svint64_t op1) {
+  return SVE_ACLE_FUNC(sveorqv,_s64,,)(pg, op1);
+}
+
+// CHECK-LABEL: @test_sveorqv_u8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <16 x i8> @llvm.aarch64.sve.eorqv.v16i8.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[OP1:%.*]])
+// CHECK-NEXT:    ret <16 x i8> [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z15test_sveorqv_u8u10__SVBool_tu11__SVUint8_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <16 x i8> @llvm.aarch64.sve.eorqv.v16i8.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[OP1:%.*]])
+// CPP-CHECK-NEXT:    ret <16 x i8> [[TMP0]]
+//
+uint8x16_t test_sveorqv_u8(svbool_t pg, svuint8_t op1) {
+  return SVE_ACLE_FUNC(sveorqv,_u8,,)(pg, op1);
+}
+
+// CHECK-LABEL: @test_sveorqv_u16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i16> @llvm.aarch64.sve.eorqv.v8i16.nxv8i16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[OP1:%.*]])
+// CHECK-NEXT:    ret <8 x i16> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z16test_sveorqv_u16u10__SVBool_tu12__SVUint16_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i16> @llvm.aarch64.sve.eorqv.v8i16.nxv8i16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[OP1:%.*]])
+// CPP-CHECK-NEXT:    ret <8 x i16> [[TMP1]]
+//
+uint16x8_t test_sveorqv_u16(svbool_t pg, svuint16_t op1) {
+  return SVE_ACLE_FUNC(sveorqv,_u16,,)(pg, op1);
+}
+
+// CHECK-LABEL: @test_sveorqv_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x i32> @llvm.aarch64.sve.eorqv.v4i32.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[OP1:%.*]])
+// CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z16test_sveorqv_u32u10__SVBool_tu12__SVUint32_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x i32> @llvm.aarch64.sve.eorqv.v4i32.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[OP1:%.*]])
+// CPP-CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+//
+uint32x4_t test_sveorqv_u32(svbool_t pg, svuint32_t op1) {
+  return SVE_ACLE_FUNC(sveorqv,_u32,,)(pg, op1);
+}
+
+// CHECK-LABEL: @test_sveorqv_u64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x i64> @llvm.aarch64.sve.eorqv.v2i64.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[OP1:%.*]])
+// CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z16test_sveorqv_u64u10__SVBool_tu12__SVUint64_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x i64> @llvm.aarch64.sve.eorqv.v2i64.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[OP1:%.*]])
+// CPP-CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+//
+uint64x2_t test_sveorqv_u64(svbool_t pg, svuint64_t op1) {
+  return SVE_ACLE_FUNC(sveorqv,_u64,,)(pg, op1);
+}
+
+
+// ORQV
+
+// CHECK-LABEL: @test_svorqv_s8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <16 x i8> @llvm.aarch64.sve.orqv.v16i8.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[OP1:%.*]])
+// CHECK-NEXT:    ret <16 x i8> [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z14test_svorqv_s8u10__SVBool_tu10__SVInt8_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <16 x i8> @llvm.aarch64.sve.orqv.v16i8.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[OP1:%.*]])
+// CPP-CHECK-NEXT:    ret <16 x i8> [[TMP0]]
+//
+int8x16_t test_svorqv_s8(svbool_t pg, svint8_t op1) {
+  return SVE_ACLE_FUNC(svorqv,_s8,,)(pg, op1);
+}
+
+// CHECK-LABEL: @test_svorqv_s16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i16> @llvm.aarch64.sve.orqv.v8i16.nxv8i16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[OP1:%.*]])
+// CHECK-NEXT:    ret <8 x i16> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z15test_svorqv_s16u10__SVBool_tu11__SVInt16_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i16> @llvm.aarch64.sve.orqv.v8i16.nxv8i16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[OP1:%.*]])
+// CPP-CHECK-NEXT:    ret <8 x i16> [[TMP1]]
+//
+int16x8_t test_svorqv_s16(svbool_t pg, svint16_t op1) {
+  return SVE_ACLE_FUNC(svorqv,_s16,,)(pg, op1);
+}
+
+// CHECK-LABEL: @test_svorqv_s32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x i32> @llvm.aarch64.sve.orqv.v4i32.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[OP1:%.*]])
+// CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z15test_svorqv_s32u10__SVBool_tu11__SVInt32_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x i32> @llvm.aarch64.sve.orqv.v4i32.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[OP1:%.*]])
+// CPP-CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+//
+int32x4_t test_svorqv_s32(svbool_t pg, svint32_t op1) {
+  return SVE_ACLE_FUNC(svorqv,_s32,,)(pg, op1);
+}
+
+// CHECK-LABEL: @test_svorqv_s64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x i64> @llvm.aarch64.sve.orqv.v2i64.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[OP1:%.*]])
+// CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z15test_svorqv_s64u10__SVBool_tu11__SVInt64_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x i64> @llvm.aarch64.sve.orqv.v2i64.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[OP1:%.*]])
+// CPP-CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+//
+int64x2_t test_svorqv_s64(svbool_t pg, svint64_t op1) {
+  return SVE_ACLE_FUNC(svorqv,_s64,,)(pg, op1);
+}
+
+// CHECK-LABEL: @test_svorqv_u8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <16 x i8> @llvm.aarch64.sve.orqv.v16i8.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[OP1:%.*]])
+// CHECK-NEXT:    ret <16 x i8> [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z14test_svorqv_u8u10__SVBool_tu11__SVUint8_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <16 x i8> @llvm.aarch64.sve.orqv.v16i8.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[OP1:%.*]])
+// CPP-CHECK-NEXT:    ret <16 x i8> [[TMP0]]
+//
+uint8x16_t test_svorqv_u8(svbool_t pg, svuint8_t op1) {
+  return SVE_ACLE_FUNC(svorqv,_u8,,)(pg, op1);
+}
+
+// CHECK-LABEL: @test_svorqv_u16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i16> @llvm.aarch64.sve.orqv.v8i16.nxv8i16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[OP1:%.*]])
+// CHECK-NEXT:    ret <8 x i16> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z15test_svorqv_u16u10__SVBool_tu12__SVUint16_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i16> @llvm.aarch64.sve.orqv.v8i16.nxv8i16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[OP1:%.*]])
+// CPP-CHECK-NEXT:    ret <8 x i16> [[TMP1]]
+//
+uint16x8_t test_svorqv_u16(svbool_t pg, svuint16_t op1) {
+  return SVE_ACLE_FUNC(svorqv,_u16,,)(pg, op1);
+}
+
+// CHECK-LABEL: @test_svorqv_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x i32> @llvm.aarch64.sve.orqv.v4i32.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[OP1:%.*]])
+// CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z15test_svorqv_u32u10__SVBool_tu12__SVUint32_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x i32> @llvm.aarch64.sve.orqv.v4i32.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[OP1:%.*]])
+// CPP-CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+//
+uint32x4_t test_svorqv_u32(svbool_t pg, svuint32_t op1) {
+  return SVE_ACLE_FUNC(svorqv,_u32,,)(pg, op1);
+}
+
+// CHECK-LABEL: @test_svorqv_u64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x i64> @llvm.aarch64.sve.orqv.v2i64.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[OP1:%.*]])
+// CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z15test_svorqv_u64u10__SVBool_tu12__SVUint64_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x i64> @llvm.aarch64.sve.orqv.v2i64.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[OP1:%.*]])
+// CPP-CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+//
+uint64x2_t test_svorqv_u64(svbool_t pg, svuint64_t op1) {
+  return SVE_ACLE_FUNC(svorqv,_u64,,)(pg, op1);
+}
+
+
+// SMAXQV
+
+// CHECK-LABEL: @test_svmaxqv_s8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <16 x i8> @llvm.aarch64.sve.smaxqv.v16i8.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[OP1:%.*]])
+// CHECK-NEXT:    ret <16 x i8> [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z15test_svmaxqv_s8u10__SVBool_tu10__SVInt8_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <16 x i8> @llvm.aarch64.sve.smaxqv.v16i8.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[OP1:%.*]])
+// CPP-CHECK-NEXT:    ret <16 x i8> [[TMP0]]
+//
+int8x16_t test_svmaxqv_s8(svbool_t pg, svint8_t op1) {
+  return SVE_ACLE_FUNC(svmaxqv,_s8,,)(pg, op1);
+}
+
+// CHECK-LABEL: @test_svmaxqv_s16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i16> @llvm.aarch64.sve.smaxqv.v8i16.nxv8i16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[OP1:%.*]])
+// CHECK-NEXT:    ret <8 x i16> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z16test_svmaxqv_s16u10__SVBool_tu11__SVInt16_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i16> @llvm.aarch64.sve.smaxqv.v8i16.nxv8i16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[OP1:%.*]])
+// CPP-CHECK-NEXT:    ret <8 x i16> [[TMP1]]
+//
+int16x8_t test_svmaxqv_s16(svbool_t pg, svint16_t op1) {
+  return SVE_ACLE_FUNC(svmaxqv,_s16,,)(pg, op1);
+}
+
+// CHECK-LABEL: @test_svmaxqv_s32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x i32> @llvm.aarch64.sve.smaxqv.v4i32.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[OP1:%.*]])
+// CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z16test_svmaxqv_s32u10__SVBool_tu11__SVInt32_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x i32> @llvm.aarch64.sve.smaxqv.v4i32.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[OP1:%.*]])
+// CPP-CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+//
+int32x4_t test_svmaxqv_s32(svbool_t pg, svint32_t op1) {
+  return SVE_ACLE_FUNC(svmaxqv,_s32,,)(pg, op1);
+}
+
+// CHECK-LABEL: @test_svmaxqv_s64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x i64> @llvm.aarch64.sve.smaxqv.v2i64.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[OP1:%.*]])
+// CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z16test_svmaxqv_s64u10__SVBool_tu11__SVInt64_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x i64> @llvm.aarch64.sve.smaxqv.v2i64.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[OP1:%.*]])
+// CPP-CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+//
+int64x2_t test_svmaxqv_s64(svbool_t pg, svint64_t op1) {
+  return SVE_ACLE_FUNC(svmaxqv,_s64,,)(pg, op1);
+}
+
+
+// UMAXQV
+
+// CHECK-LABEL: @test_svmaxqv_u8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <16 x i8> @llvm.aarch64.sve.umaxqv.v16i8.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[OP1:%.*]])
+// CHECK-NEXT:    ret <16 x i8> [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z15test_svmaxqv_u8u10__SVBool_tu11__SVUint8_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <16 x i8> @llvm.aarch64.sve.umaxqv.v16i8.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[OP1:%.*]])
+// CPP-CHECK-NEXT:    ret <16 x i8> [[TMP0]]
+//
+uint8x16_t test_svmaxqv_u8(svbool_t pg, svuint8_t op1) {
+  return SVE_ACLE_FUNC(svmaxqv,_u8,,)(pg, op1);
+}
+
+// CHECK-LABEL: @test_svmaxqv_u16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i16> @llvm.aarch64.sve.umaxqv.v8i16.nxv8i16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[OP1:%.*]])
+// CHECK-NEXT:    ret <8 x i16> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z16test_svmaxqv_u16u10__SVBool_tu12__SVUint16_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i16> @llvm.aarch64.sve.umaxqv.v8i16.nxv8i16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[OP1:%.*]])
+// CPP-CHECK-NEXT:    ret <8 x i16> [[TMP1]]
+//
+uint16x8_t test_svmaxqv_u16(svbool_t pg, svuint16_t op1) {
+  return SVE_ACLE_FUNC(svmaxqv,_u16,,)(pg, op1);
+}
+
+// CHECK-LABEL: @test_svmaxqv_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x i32> @llvm.aarch64.sve.umaxqv.v4i32.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[OP1:%.*]])
+// CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z16test_svmaxqv_u32u10__SVBool_tu12__SVUint32_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x i32> @llvm.aarch64.sve.umaxqv.v4i32.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[OP1:%.*]])
+// CPP-CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+//
+uint32x4_t test_svmaxqv_u32(svbool_t pg, svuint32_t op1) {
+  return SVE_ACLE_FUNC(svmaxqv,_u32,,)(pg, op1);
+}
+
+// CHECK-LABEL: @test_svmaxqv_u64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x i64> @llvm.aarch64.sve.umaxqv.v2i64.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[OP1:%.*]])
+// CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z16test_svmaxqv_u64u10__SVBool_tu12__SVUint64_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x i64> @llvm.aarch64.sve.umaxqv.v2i64.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[OP1:%.*]])
+// CPP-CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+//
+uint64x2_t test_svmaxqv_u64(svbool_t pg, svuint64_t op1) {
+  return SVE_ACLE_FUNC(svmaxqv,_u64,,)(pg, op1);
+}
+
+
+// SMINQV
+
+// CHECK-LABEL: @test_svminqv_s8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <16 x i8> @llvm.aarch64.sve.sminqv.v16i8.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[OP1:%.*]])
+// CHECK-NEXT:    ret <16 x i8> [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z15test_svminqv_s8u10__SVBool_tu10__SVInt8_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <16 x i8> @llvm.aarch64.sve.sminqv.v16i8.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[OP1:%.*]])
+// CPP-CHECK-NEXT:    ret <16 x i8> [[TMP0]]
+//
+int8x16_t test_svminqv_s8(svbool_t pg, svint8_t op1) {
+  return SVE_ACLE_FUNC(svminqv,_s8,,)(pg, op1);
+}
+
+// CHECK-LABEL: @test_svminqv_s16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i16> @llvm.aarch64.sve.sminqv.v8i16.nxv8i16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[OP1:%.*]])
+// CHECK-NEXT:    ret <8 x i16> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z16test_svminqv_s16u10__SVBool_tu11__SVInt16_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i16> @llvm.aarch64.sve.sminqv.v8i16.nxv8i16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[OP1:%.*]])
+// CPP-CHECK-NEXT:    ret <8 x i16> [[TMP1]]
+//
+int16x8_t test_svminqv_s16(svbool_t pg, svint16_t op1) {
+  return SVE_ACLE_FUNC(svminqv,_s16,,)(pg, op1);
+}
+
+// CHECK-LABEL: @test_svminqv_s32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x i32> @llvm.aarch64.sve.sminqv.v4i32.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[OP1:%.*]])
+// CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z16test_svminqv_s32u10__SVBool_tu11__SVInt32_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x i32> @llvm.aarch64.sve.sminqv.v4i32.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[OP1:%.*]])
+// CPP-CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+//
+int32x4_t test_svminqv_s32(svbool_t pg, svint32_t op1) {
+  return SVE_ACLE_FUNC(svminqv,_s32,,)(pg, op1);
+}
+
+// CHECK-LABEL: @test_svminqv_s64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x i64> @llvm.aarch64.sve.sminqv.v2i64.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[OP1:%.*]])
+// CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z16test_svminqv_s64u10__SVBool_tu11__SVInt64_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x i64> @llvm.aarch64.sve.sminqv.v2i64.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[OP1:%.*]])
+// CPP-CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+//
+int64x2_t test_svminqv_s64(svbool_t pg, svint64_t op1) {
+  return SVE_ACLE_FUNC(svminqv,_s64,,)(pg, op1);
+}
+
+
+// UMINQV
+
+// CHECK-LABEL: @test_svminqv_u8(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <16 x i8> @llvm.aarch64.sve.uminqv.v16i8.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[OP1:%.*]])
+// CHECK-NEXT:    ret <16 x i8> [[TMP0]]
+//
+// CPP-CHECK-LABEL: @_Z15test_svminqv_u8u10__SVBool_tu11__SVUint8_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <16 x i8> @llvm.aarch64.sve.uminqv.v16i8.nxv16i8(<vscale x 16 x i1> [[PG:%.*]], <vscale x 16 x i8> [[OP1:%.*]])
+// CPP-CHECK-NEXT:    ret <16 x i8> [[TMP0]]
+//
+uint8x16_t test_svminqv_u8(svbool_t pg, svuint8_t op1) {
+  return SVE_ACLE_FUNC(svminqv,_u8,,)(pg, op1);
+}
+
+// CHECK-LABEL: @test_svminqv_u16(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i16> @llvm.aarch64.sve.uminqv.v8i16.nxv8i16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[OP1:%.*]])
+// CHECK-NEXT:    ret <8 x i16> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z16test_svminqv_u16u10__SVBool_tu12__SVUint16_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 8 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv8i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <8 x i16> @llvm.aarch64.sve.uminqv.v8i16.nxv8i16(<vscale x 8 x i1> [[TMP0]], <vscale x 8 x i16> [[OP1:%.*]])
+// CPP-CHECK-NEXT:    ret <8 x i16> [[TMP1]]
+//
+uint16x8_t test_svminqv_u16(svbool_t pg, svuint16_t op1) {
+  return SVE_ACLE_FUNC(svminqv,_u16,,)(pg, op1);
+}
+
+// CHECK-LABEL: @test_svminqv_u32(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x i32> @llvm.aarch64.sve.uminqv.v4i32.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[OP1:%.*]])
+// CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z16test_svminqv_u32u10__SVBool_tu12__SVUint32_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv4i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <4 x i32> @llvm.aarch64.sve.uminqv.v4i32.nxv4i32(<vscale x 4 x i1> [[TMP0]], <vscale x 4 x i32> [[OP1:%.*]])
+// CPP-CHECK-NEXT:    ret <4 x i32> [[TMP1]]
+//
+uint32x4_t test_svminqv_u32(svbool_t pg, svuint32_t op1) {
+  return SVE_ACLE_FUNC(svminqv,_u32,,)(pg, op1);
+}
+
+// CHECK-LABEL: @test_svminqv_u64(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x i64> @llvm.aarch64.sve.uminqv.v2i64.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[OP1:%.*]])
+// CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+//
+// CPP-CHECK-LABEL: @_Z16test_svminqv_u64u10__SVBool_tu12__SVUint64_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 2 x i1> @llvm.aarch64.sve.convert.from.svbool.nxv2i1(<vscale x 16 x i1> [[PG:%.*]])
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <2 x i64> @llvm.aarch64.sve.uminqv.v2i64.nxv2i64(<vscale x 2 x i1> [[TMP0]], <vscale x 2 x i64> [[OP1:%.*]])
+// CPP-CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+//
+uint64x2_t test_svminqv_u64(svbool_t pg, svuint64_t op1) {
+  return SVE_ACLE_FUNC(svminqv,_u64,,)(pg, op1);
+}
diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_qrshr.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_qrshr.c
new file mode 100644
index 0000000000000000000000000000000000000000..6ebf224db92377cb7c398ece8b54c04876d167fe
--- /dev/null
+++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_qrshr.c
@@ -0,0 +1,79 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S  -passes=mem2reg,instcombine,tailcallelim | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S  -passes=mem2reg,instcombine,tailcallelim | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S  -passes=mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S  -passes=mem2reg,instcombine,tailcallelim | FileCheck %s
+// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S  -passes=mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme2 -target-feature +sme-f64f64 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s
+
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+// A simple used,unused... macro, long enough to represent any SVE builtin.
+#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED,A5) A1##A3##A5
+#else
+#define SVE_ACLE_FUNC(A1,A2,A3,A4,A5) A1##A2##A3##A4##A5
+#endif
+
+
+// SQRSHRN x 2
+
+// CHECK-LABEL: @test_svqrshrn_s16_s32_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[ZN:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[ZN]], i64 4)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sve.sqrshrn.x2.nxv4i32(<vscale x 4 x i32> [[TMP0]], <vscale x 4 x i32> [[TMP1]], i32 16)
+// CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP2]]
+//
+// CPP-CHECK-LABEL: @_Z24test_svqrshrn_s16_s32_x211svint32x2_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[ZN:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[ZN]], i64 4)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sve.sqrshrn.x2.nxv4i32(<vscale x 4 x i32> [[TMP0]], <vscale x 4 x i32> [[TMP1]], i32 16)
+// CPP-CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP2]]
+//
+svint16_t test_svqrshrn_s16_s32_x2(svint32x2_t zn) __arm_streaming_compatible {
+  return SVE_ACLE_FUNC(svqrshrn,_n,_s16,_s32_x2,)(zn, 16);
+}
+
+// UQRSHRN x 2
+
+// CHECK-LABEL: @test_svqrshrn_u16_u32_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[ZN:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[ZN]], i64 4)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sve.uqrshrn.x2.nxv4i32(<vscale x 4 x i32> [[TMP0]], <vscale x 4 x i32> [[TMP1]], i32 16)
+// CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP2]]
+//
+// CPP-CHECK-LABEL: @_Z24test_svqrshrn_u16_u32_x212svuint32x2_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[ZN:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[ZN]], i64 4)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sve.uqrshrn.x2.nxv4i32(<vscale x 4 x i32> [[TMP0]], <vscale x 4 x i32> [[TMP1]], i32 16)
+// CPP-CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP2]]
+//
+svuint16_t test_svqrshrn_u16_u32_x2(svuint32x2_t zn) __arm_streaming_compatible {
+  return SVE_ACLE_FUNC(svqrshrn,_n,_u16,_u32_x2,)(zn, 16);
+}
+
+// SQRSHRUN x 2
+
+// CHECK-LABEL: @test_svqrshrun_u16_s32_x2(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[ZN:%.*]], i64 0)
+// CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[ZN]], i64 4)
+// CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sve.sqrshrun.x2.nxv4i32(<vscale x 4 x i32> [[TMP0]], <vscale x 4 x i32> [[TMP1]], i32 16)
+// CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP2]]
+//
+// CPP-CHECK-LABEL: @_Z25test_svqrshrun_u16_s32_x211svint32x2_t(
+// CPP-CHECK-NEXT:  entry:
+// CPP-CHECK-NEXT:    [[TMP0:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[ZN:%.*]], i64 0)
+// CPP-CHECK-NEXT:    [[TMP1:%.*]] = tail call <vscale x 4 x i32> @llvm.vector.extract.nxv4i32.nxv8i32(<vscale x 8 x i32> [[ZN]], i64 4)
+// CPP-CHECK-NEXT:    [[TMP2:%.*]] = tail call <vscale x 8 x i16> @llvm.aarch64.sve.sqrshrun.x2.nxv4i32(<vscale x 4 x i32> [[TMP0]], <vscale x 4 x i32> [[TMP1]], i32 16)
+// CPP-CHECK-NEXT:    ret <vscale x 8 x i16> [[TMP2]]
+//
+svuint16_t test_svqrshrun_u16_s32_x2(svint32x2_t zn) __arm_streaming_compatible {
+  return SVE_ACLE_FUNC(svqrshrun,_n,_u16,_s32_x2,)(zn, 16);
+}
diff --git a/clang/test/CodeGen/arm-vector_type-params-returns.c b/clang/test/CodeGen/arm-vector_type-params-returns.c
new file mode 100644
index 0000000000000000000000000000000000000000..14c3512ab81a9fa6069ff6fce339b7e4a45e09e2
--- /dev/null
+++ b/clang/test/CodeGen/arm-vector_type-params-returns.c
@@ -0,0 +1,136 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 3
+
+// RUN: %clang_cc1 -DSVE_HEADER -triple aarch64 -target-feature +sve -emit-llvm -O2 -o - %s | opt -S -passes=mem2reg,sroa | FileCheck %s
+// RUN: %clang_cc1 -DSVE_HEADER -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -o - /dev/null %s
+
+// RUN: %clang_cc1 -DNEON_HEADER -triple aarch64 -target-feature +sve -emit-llvm -O2 -o - %s | opt -S -passes=mem2reg,sroa | FileCheck %s
+// RUN: %clang_cc1 -DNEON_HEADER -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -o - /dev/null %s
+
+// RUN: %clang_cc1 -DSVE_HEADER -DNEON_HEADER -triple aarch64 -target-feature +sve -emit-llvm -O2 -o - %s | opt -S -passes=mem2reg,sroa | FileCheck %s
+// RUN: %clang_cc1 -DSVE_HEADER -DNEON_HEADER -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -o - /dev/null %s
+
+// RUN: %clang_cc1 -DNEON_HEADER -DSVE_HEADER2  -triple aarch64 -target-feature +sve -emit-llvm -O2 -o - %s | opt -S -passes=mem2reg,sroa | FileCheck %s
+// RUN: %clang_cc1 -DNEON_HEADER -DSVE_HEADER2 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -o - /dev/null %s
+
+// REQUIRES: aarch64-registered-target
+
+#ifdef SVE_HEADER
+  #include <arm_sve.h>
+#endif
+
+#ifdef NEON_HEADER
+  #include <arm_neon.h>
+#endif
+
+#ifdef SVE_HEADER_2
+  #include <arm_sve.h>
+#endif
+
+// function return types
+// CHECK-LABEL: define dso_local <8 x half> @test_ret_v8f16(
+// CHECK-SAME: <8 x half> noundef returned [[V:%.*]]) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret <8 x half> [[V]]
+//
+float16x8_t test_ret_v8f16(float16x8_t v) {
+  return v;
+}
+
+// CHECK-LABEL: define dso_local <4 x float> @test_ret_v4f32(
+// CHECK-SAME: <4 x float> noundef returned [[V:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret <4 x float> [[V]]
+//
+float32x4_t test_ret_v4f32(float32x4_t v) {
+  return v;
+}
+
+// CHECK-LABEL: define dso_local <2 x double> @test_ret_v2f64(
+// CHECK-SAME: <2 x double> noundef returned [[V:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret <2 x double> [[V]]
+//
+float64x2_t test_ret_v2f64(float64x2_t v) {
+  return v;
+}
+
+// CHECK-LABEL: define dso_local <8 x bfloat> @test_ret_v8bf16(
+// CHECK-SAME: <8 x bfloat> noundef returned [[V:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret <8 x bfloat> [[V]]
+//
+bfloat16x8_t test_ret_v8bf16(bfloat16x8_t v) {
+  return v;
+}
+
+// CHECK-LABEL: define dso_local <16 x i8> @test_ret_v16s8(
+// CHECK-SAME: <16 x i8> noundef returned [[V:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret <16 x i8> [[V]]
+//
+int8x16_t test_ret_v16s8(int8x16_t v) {
+  return v;
+}
+
+// CHECK-LABEL: define dso_local <8 x i16> @test_ret_v8s16(
+// CHECK-SAME: <8 x i16> noundef returned [[V:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret <8 x i16> [[V]]
+//
+int16x8_t test_ret_v8s16(int16x8_t v) {
+  return v;
+}
+
+// CHECK-LABEL: define dso_local <4 x i32> @test_ret_v32s4(
+// CHECK-SAME: <4 x i32> noundef returned [[V:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret <4 x i32> [[V]]
+//
+int32x4_t test_ret_v32s4(int32x4_t v) {
+  return v;
+}
+
+// CHECK-LABEL: define dso_local <2 x i64> @test_ret_v64s2(
+// CHECK-SAME: <2 x i64> noundef returned [[V:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret <2 x i64> [[V]]
+//
+int64x2_t test_ret_v64s2(int64x2_t v) {
+  return v;
+}
+
+// CHECK-LABEL: define dso_local <16 x i8> @test_ret_v16u8(
+// CHECK-SAME: <16 x i8> noundef returned [[V:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret <16 x i8> [[V]]
+//
+uint8x16_t test_ret_v16u8(uint8x16_t v) {
+  return v;
+}
+
+// CHECK-LABEL: define dso_local <8 x i16> @test_ret_v8u16(
+// CHECK-SAME: <8 x i16> noundef returned [[V:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret <8 x i16> [[V]]
+//
+uint16x8_t test_ret_v8u16(uint16x8_t v) {
+  return v;
+}
+
+// CHECK-LABEL: define dso_local <4 x i32> @test_ret_v32u4(
+// CHECK-SAME: <4 x i32> noundef returned [[V:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret <4 x i32> [[V]]
+//
+uint32x4_t test_ret_v32u4(uint32x4_t v) {
+  return v;
+}
+
+// CHECK-LABEL: define dso_local <2 x i64> @test_ret_v64u2(
+// CHECK-SAME: <2 x i64> noundef returned [[V:%.*]]) local_unnamed_addr #[[ATTR0]] {
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    ret <2 x i64> [[V]]
+//
+uint64x2_t test_ret_v64u2(uint64x2_t v) {
+  return v;
+}
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx12-err.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx12-err.cl
new file mode 100644
index 0000000000000000000000000000000000000000..5e0153c42825e35decbc4ddf6458923aae107c88
--- /dev/null
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx12-err.cl
@@ -0,0 +1,24 @@
+// REQUIRES: amdgpu-registered-target
+
+// RUN: %clang_cc1 -triple amdgcn-unknown-unknown -target-cpu gfx1200 -verify -S -emit-llvm -o - %s
+
+kernel void builtins_amdgcn_s_barrier_signal_err(global int* in, global int* out, int barrier) {
+
+  __builtin_amdgcn_s_barrier_signal(barrier); // expected-error {{'__builtin_amdgcn_s_barrier_signal' must be a constant integer}}
+  __builtin_amdgcn_s_barrier_wait(-1);
+  *out = *in;
+}
+
+kernel void builtins_amdgcn_s_barrier_wait_err(global int* in, global int* out, int barrier) {
+
+  __builtin_amdgcn_s_barrier_signal(-1);
+  __builtin_amdgcn_s_barrier_wait(barrier); // expected-error {{'__builtin_amdgcn_s_barrier_wait' must be a constant integer}}
+  *out = *in;
+}
+
+kernel void builtins_amdgcn_s_barrier_signal_isfirst_err(global int* in, global int* out, int barrier) {
+
+  __builtin_amdgcn_s_barrier_signal_isfirst(barrier); // expected-error {{'__builtin_amdgcn_s_barrier_signal_isfirst' must be a constant integer}}
+  __builtin_amdgcn_s_barrier_wait(-1);
+  *out = *in;
+}
diff --git a/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx12.cl b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx12.cl
new file mode 100644
index 0000000000000000000000000000000000000000..b8d281531e218e6116519fb82cef4787a6e85104
--- /dev/null
+++ b/clang/test/CodeGenOpenCL/builtins-amdgcn-gfx12.cl
@@ -0,0 +1,174 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+// RUN: %clang_cc1 -cl-std=CL2.0 -O0 -triple amdgcn-unknown-unknown -target-cpu gfx1200 -S -emit-llvm -o - %s | FileCheck %s
+
+// CHECK-LABEL: @test_s_barrier_signal(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    call void @llvm.amdgcn.s.barrier.signal(i32 -1)
+// CHECK-NEXT:    call void @llvm.amdgcn.s.barrier.wait(i16 -1)
+// CHECK-NEXT:    ret void
+//
+void test_s_barrier_signal()
+{
+  __builtin_amdgcn_s_barrier_signal(-1);
+  __builtin_amdgcn_s_barrier_wait(-1);
+}
+
+// CHECK-LABEL: @test_s_barrier_signal_var(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4, addrspace(5)
+// CHECK-NEXT:    store i32 [[A:%.*]], ptr addrspace(5) [[A_ADDR]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr addrspace(5) [[A_ADDR]], align 4
+// CHECK-NEXT:    call void @llvm.amdgcn.s.barrier.signal.var(i32 [[TMP0]])
+// CHECK-NEXT:    ret void
+//
+void test_s_barrier_signal_var(int a)
+{
+  __builtin_amdgcn_s_barrier_signal_var(a);
+}
+
+// CHECK-LABEL: @test_s_barrier_signal_isfirst(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca ptr, align 8, addrspace(5)
+// CHECK-NEXT:    [[B_ADDR:%.*]] = alloca ptr, align 8, addrspace(5)
+// CHECK-NEXT:    [[C_ADDR:%.*]] = alloca ptr, align 8, addrspace(5)
+// CHECK-NEXT:    store ptr [[A:%.*]], ptr addrspace(5) [[A_ADDR]], align 8
+// CHECK-NEXT:    store ptr [[B:%.*]], ptr addrspace(5) [[B_ADDR]], align 8
+// CHECK-NEXT:    store ptr [[C:%.*]], ptr addrspace(5) [[C_ADDR]], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = call i1 @llvm.amdgcn.s.barrier.signal.isfirst(i32 1)
+// CHECK-NEXT:    br i1 [[TMP0]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+// CHECK:       if.then:
+// CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr addrspace(5) [[B_ADDR]], align 8
+// CHECK-NEXT:    store ptr [[TMP1]], ptr addrspace(5) [[A_ADDR]], align 8
+// CHECK-NEXT:    br label [[IF_END:%.*]]
+// CHECK:       if.else:
+// CHECK-NEXT:    [[TMP2:%.*]] = load ptr, ptr addrspace(5) [[C_ADDR]], align 8
+// CHECK-NEXT:    store ptr [[TMP2]], ptr addrspace(5) [[A_ADDR]], align 8
+// CHECK-NEXT:    br label [[IF_END]]
+// CHECK:       if.end:
+// CHECK-NEXT:    call void @llvm.amdgcn.s.barrier.wait(i16 1)
+// CHECK-NEXT:    ret void
+//
+void test_s_barrier_signal_isfirst(int* a, int* b, int *c)
+{
+  if(__builtin_amdgcn_s_barrier_signal_isfirst(1))
+    a = b;
+  else
+    a = c;
+
+  __builtin_amdgcn_s_barrier_wait(1);
+}
+
+// CHECK-LABEL: @test_s_barrier_isfirst_var(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca ptr, align 8, addrspace(5)
+// CHECK-NEXT:    [[B_ADDR:%.*]] = alloca ptr, align 8, addrspace(5)
+// CHECK-NEXT:    [[C_ADDR:%.*]] = alloca ptr, align 8, addrspace(5)
+// CHECK-NEXT:    [[D_ADDR:%.*]] = alloca i32, align 4, addrspace(5)
+// CHECK-NEXT:    store ptr [[A:%.*]], ptr addrspace(5) [[A_ADDR]], align 8
+// CHECK-NEXT:    store ptr [[B:%.*]], ptr addrspace(5) [[B_ADDR]], align 8
+// CHECK-NEXT:    store ptr [[C:%.*]], ptr addrspace(5) [[C_ADDR]], align 8
+// CHECK-NEXT:    store i32 [[D:%.*]], ptr addrspace(5) [[D_ADDR]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr addrspace(5) [[D_ADDR]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = call i1 @llvm.amdgcn.s.barrier.signal.isfirst.var(i32 [[TMP0]])
+// CHECK-NEXT:    br i1 [[TMP1]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+// CHECK:       if.then:
+// CHECK-NEXT:    [[TMP2:%.*]] = load ptr, ptr addrspace(5) [[B_ADDR]], align 8
+// CHECK-NEXT:    store ptr [[TMP2]], ptr addrspace(5) [[A_ADDR]], align 8
+// CHECK-NEXT:    br label [[IF_END:%.*]]
+// CHECK:       if.else:
+// CHECK-NEXT:    [[TMP3:%.*]] = load ptr, ptr addrspace(5) [[C_ADDR]], align 8
+// CHECK-NEXT:    store ptr [[TMP3]], ptr addrspace(5) [[A_ADDR]], align 8
+// CHECK-NEXT:    br label [[IF_END]]
+// CHECK:       if.end:
+// CHECK-NEXT:    call void @llvm.amdgcn.s.barrier.wait(i16 1)
+// CHECK-NEXT:    ret void
+//
+void test_s_barrier_isfirst_var(int* a, int* b, int *c, int d)
+{
+ if ( __builtin_amdgcn_s_barrier_signal_isfirst_var(d))
+    a = b;
+  else
+    a = c;
+
+  __builtin_amdgcn_s_barrier_wait(1);
+
+}
+
+// CHECK-LABEL: @test_s_barrier_init(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4, addrspace(5)
+// CHECK-NEXT:    store i32 [[A:%.*]], ptr addrspace(5) [[A_ADDR]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr addrspace(5) [[A_ADDR]], align 4
+// CHECK-NEXT:    call void @llvm.amdgcn.s.barrier.init(i32 1, i32 [[TMP0]])
+// CHECK-NEXT:    ret void
+//
+void test_s_barrier_init(int a)
+{
+  __builtin_amdgcn_s_barrier_init(1, a);
+}
+
+// CHECK-LABEL: @test_s_barrier_join(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    call void @llvm.amdgcn.s.barrier.join(i32 1)
+// CHECK-NEXT:    ret void
+//
+void test_s_barrier_join()
+{
+  __builtin_amdgcn_s_barrier_join(1);
+}
+
+// CHECK-LABEL: @test_s_wakeup_barrier(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    call void @llvm.amdgcn.s.barrier.join(i32 1)
+// CHECK-NEXT:    ret void
+//
+void test_s_wakeup_barrier()
+{
+  __builtin_amdgcn_s_barrier_join(1);
+}
+
+// CHECK-LABEL: @test_s_barrier_leave(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca ptr, align 8, addrspace(5)
+// CHECK-NEXT:    [[B_ADDR:%.*]] = alloca ptr, align 8, addrspace(5)
+// CHECK-NEXT:    [[C_ADDR:%.*]] = alloca ptr, align 8, addrspace(5)
+// CHECK-NEXT:    store ptr [[A:%.*]], ptr addrspace(5) [[A_ADDR]], align 8
+// CHECK-NEXT:    store ptr [[B:%.*]], ptr addrspace(5) [[B_ADDR]], align 8
+// CHECK-NEXT:    store ptr [[C:%.*]], ptr addrspace(5) [[C_ADDR]], align 8
+// CHECK-NEXT:    [[TMP0:%.*]] = call i1 @llvm.amdgcn.s.barrier.leave()
+// CHECK-NEXT:    br i1 [[TMP0]], label [[IF_THEN:%.*]], label [[IF_ELSE:%.*]]
+// CHECK:       if.then:
+// CHECK-NEXT:    [[TMP1:%.*]] = load ptr, ptr addrspace(5) [[B_ADDR]], align 8
+// CHECK-NEXT:    store ptr [[TMP1]], ptr addrspace(5) [[A_ADDR]], align 8
+// CHECK-NEXT:    br label [[IF_END:%.*]]
+// CHECK:       if.else:
+// CHECK-NEXT:    [[TMP2:%.*]] = load ptr, ptr addrspace(5) [[C_ADDR]], align 8
+// CHECK-NEXT:    store ptr [[TMP2]], ptr addrspace(5) [[A_ADDR]], align 8
+// CHECK-NEXT:    br label [[IF_END]]
+// CHECK:       if.end:
+// CHECK-NEXT:    ret void
+//
+void test_s_barrier_leave(int* a, int* b, int *c)
+{
+  if (__builtin_amdgcn_s_barrier_leave())
+    a = b;
+  else
+    a = c;
+}
+
+// CHECK-LABEL: @test_s_get_barrier_state(
+// CHECK-NEXT:  entry:
+// CHECK-NEXT:    [[A_ADDR:%.*]] = alloca i32, align 4, addrspace(5)
+// CHECK-NEXT:    [[STATE:%.*]] = alloca i32, align 4, addrspace(5)
+// CHECK-NEXT:    store i32 [[A:%.*]], ptr addrspace(5) [[A_ADDR]], align 4
+// CHECK-NEXT:    [[TMP0:%.*]] = load i32, ptr addrspace(5) [[A_ADDR]], align 4
+// CHECK-NEXT:    [[TMP1:%.*]] = call i32 @llvm.amdgcn.s.get.barrier.state(i32 [[TMP0]])
+// CHECK-NEXT:    store i32 [[TMP1]], ptr addrspace(5) [[STATE]], align 4
+// CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr addrspace(5) [[STATE]], align 4
+// CHECK-NEXT:    ret i32 [[TMP2]]
+//
+unsigned test_s_get_barrier_state(int a)
+{
+  unsigned State = __builtin_amdgcn_s_get_barrier_state(a);
+  return State;
+}
diff --git a/clang/test/Driver/cl-options.c b/clang/test/Driver/cl-options.c
index 81d1b907eced188c5c7096aca6e9c751f9973eb3..5b6dfe308a76eae703bc216db2410369ae59e6d4 100644
--- a/clang/test/Driver/cl-options.c
+++ b/clang/test/Driver/cl-options.c
@@ -272,10 +272,12 @@
 // RUN: not %clang_cl /vmg /vmm /vms -### -- %s 2>&1 | FileCheck -check-prefix=VMX %s
 // VMX: '/vms' not allowed with '/vmm'
 
-// RUN: %clang_cl /volatile:iso -### -- %s 2>&1 | FileCheck -check-prefix=VOLATILE-ISO %s
+// RUN: %clang_cl --target=i686-pc-win32 /volatile:iso -### -- %s 2>&1 | FileCheck -check-prefix=VOLATILE-ISO %s
+// RUN: %clang_cl --target=aarch64-pc-win32 -### -- %s 2>&1 | FileCheck -check-prefix=VOLATILE-ISO %s
 // VOLATILE-ISO-NOT: "-fms-volatile"
 
-// RUN: %clang_cl /volatile:ms -### -- %s 2>&1 | FileCheck -check-prefix=VOLATILE-MS %s
+// RUN: %clang_cl --target=aarch64-pc-win32 /volatile:ms -### -- %s 2>&1 | FileCheck -check-prefix=VOLATILE-MS %s
+// RUN: %clang_cl --target=i686-pc-win32 -### -- %s 2>&1 | FileCheck -check-prefix=VOLATILE-MS %s
 // VOLATILE-MS: "-fms-volatile"
 
 // RUN: %clang_cl /W0 -### -- %s 2>&1 | FileCheck -check-prefix=W0 %s
diff --git a/clang/test/Driver/clang_f_opts.c b/clang/test/Driver/clang_f_opts.c
index ebe8a0520bf0fca399484e2391896a9d74161683..c8b44e056e58f76fe433fcc3176fc49c1831c4b6 100644
--- a/clang/test/Driver/clang_f_opts.c
+++ b/clang/test/Driver/clang_f_opts.c
@@ -611,3 +611,9 @@
 // CHECK-INT-OBJEMITTER-NOT: unsupported option '-fintegrated-objemitter' for target
 // RUN: not %clang -### -fno-integrated-objemitter --target=x86_64 %s 2>&1 | FileCheck -check-prefix=CHECK-NOINT-OBJEMITTER %s
 // CHECK-NOINT-OBJEMITTER: unsupported option '-fno-integrated-objemitter' for target
+
+// RUN: %clang -### --target=aarch64-windows-msvc %s 2>&1 | FileCheck -check-prefix=CHECK-NO-MS-VOLATILE %s
+// RUN: %clang -### --target=aarch64-windows-msvc -fms-volatile %s 2>&1 | FileCheck -check-prefix=CHECK-MS-VOLATILE %s
+// RUN: %clang -### --target=aarch64-windows-msvc -fno-ms-volatile %s 2>&1 | FileCheck -check-prefix=CHECK-NO-MS-VOLATILE %s
+// CHECK-MS-VOLATILE: -fms-volatile
+// CHECK-NO-MS-VOLATILE-NOT: -fms-volatile
diff --git a/clang/test/Driver/darwin-builtin-modules.c b/clang/test/Driver/darwin-builtin-modules.c
index 215f2b8d2c142ac66c34a42bffef7cdf2b8bfd91..1c56e13bfb9293b438b601e791b82180b4210059 100644
--- a/clang/test/Driver/darwin-builtin-modules.c
+++ b/clang/test/Driver/darwin-builtin-modules.c
@@ -9,19 +9,3 @@
 // RUN: %clang -isysroot %S/Inputs/MacOSX99.0.sdk -target x86_64-apple-macos98.0 -### %s 2>&1 | FileCheck --check-prefix=CHECK_FUTURE %s
 // RUN: %clang -isysroot %S/Inputs/MacOSX99.0.sdk -target x86_64-apple-macos99.0 -### %s 2>&1 | FileCheck --check-prefix=CHECK_FUTURE %s
 // CHECK_FUTURE-NOT: -fbuiltin-headers-in-system-modules
-
-
-// Check that builtin_headers_in_system_modules is only set if -fbuiltin-headers-in-system-modules and -fmodules are both set.
-
-// RUN: %clang -isysroot %S/Inputs/iPhoneOS13.0.sdk -target arm64-apple-ios13.0 -fsyntax-only %s -Xclang -verify=no-feature
-// RUN: %clang -isysroot %S/Inputs/iPhoneOS13.0.sdk -target arm64-apple-ios13.0 -fsyntax-only %s -fmodules -Xclang -verify=yes-feature
-// RUN: %clang -isysroot %S/Inputs/MacOSX99.0.sdk -target x86_64-apple-macos99.0 -fsyntax-only %s -Xclang -verify=no-feature
-// RUN: %clang -isysroot %S/Inputs/MacOSX99.0.sdk -target x86_64-apple-macos99.0 -fsyntax-only %s -fmodules -Xclang -verify=no-feature
-
-#if __has_feature(builtin_headers_in_system_modules)
-#error "has builtin_headers_in_system_modules"
-// yes-feature-error@-1 {{}}
-#else
-#error "no builtin_headers_in_system_modules"
-// no-feature-error@-1 {{}}
-#endif
diff --git a/clang/test/Preprocessor/riscv-target-features.c b/clang/test/Preprocessor/riscv-target-features.c
index 6fc921a8c6ee155d65d9a2d762487f6392da8194..35208b2eae8fbd329e46983149179eca73f98bd1 100644
--- a/clang/test/Preprocessor/riscv-target-features.c
+++ b/clang/test/Preprocessor/riscv-target-features.c
@@ -1056,12 +1056,12 @@
 // CHECK-ZFBFMIN-EXT: __riscv_zfbfmin 8000{{$}}
 
 // RUN: %clang --target=riscv32 -menable-experimental-extensions \
-// RUN: -march=rv32i_zicfilp0p2 -x c -E -dM %s \
+// RUN: -march=rv32i_zicfilp0p4 -x c -E -dM %s \
 // RUN: -o - | FileCheck --check-prefix=CHECK-ZICFILP-EXT %s
 // RUN: %clang --target=riscv64 -menable-experimental-extensions \
-// RUN: -march=rv64i_zicfilp0p2 -x c -E -dM %s \
+// RUN: -march=rv64i_zicfilp0p4 -x c -E -dM %s \
 // RUN: -o - | FileCheck --check-prefix=CHECK-ZICFILP-EXT %s
-// CHECK-ZICFILP-EXT: __riscv_zicfilp 2000{{$}}
+// CHECK-ZICFILP-EXT: __riscv_zicfilp 4000{{$}}
 
 // RUN: %clang --target=riscv32 -menable-experimental-extensions \
 // RUN: -march=rv32i_zicond1p0 -x c -E -dM %s \
diff --git a/clang/test/Sema/aarch64-sve-intrinsics/acle_sve_target.cpp b/clang/test/Sema/aarch64-sve-intrinsics/acle_sve_target.cpp
index f41030c18e932bc68037046a87685fa9bdff0747..2f771ca170e76aab1b3fd8e5a42f299bf485567c 100644
--- a/clang/test/Sema/aarch64-sve-intrinsics/acle_sve_target.cpp
+++ b/clang/test/Sema/aarch64-sve-intrinsics/acle_sve_target.cpp
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -fsyntax-only -verify -emit-llvm -o - -ferror-limit 100 %s
+// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +neon -fsyntax-only -verify -emit-llvm -o - -ferror-limit 100 %s
 // REQUIRES: aarch64-registered-target
 
 // Test that functions with the correct target attributes can use the correct SVE intrinsics.
@@ -29,4 +29,5 @@ void __attribute__((target("sve2-sha3"))) test_sve2_sha3()
 void __attribute__((target("sve2"))) test_f16(svbool_t pg)
 {
   svlogb_f16_z(pg, svundef_f16());
-}
\ No newline at end of file
+}
+
diff --git a/clang/test/Sema/arm-vector-types-support.c b/clang/test/Sema/arm-vector-types-support.c
index fa101afa3122adee457589f8165b0192b3bbd827..83a83ddfe780170cde85da81058b22d425ced113 100644
--- a/clang/test/Sema/arm-vector-types-support.c
+++ b/clang/test/Sema/arm-vector-types-support.c
@@ -1,5 +1,5 @@
 // RUN: %clang_cc1 %s -triple armv7 -fsyntax-only -verify
 
-typedef __attribute__((neon_vector_type(2))) int int32x2_t; // expected-error{{'neon_vector_type' attribute is not supported on targets missing 'neon' or 'mve'; specify an appropriate -march= or -mcpu=}}
+typedef __attribute__((neon_vector_type(2))) int int32x2_t; // expected-error{{'neon_vector_type' attribute is not supported on targets missing 'neon', 'mve', 'sve' or 'sme'; specify an appropriate -march= or -mcpu=}}
 typedef __attribute__((neon_polyvector_type(16))) short poly8x16_t; // expected-error{{'neon_polyvector_type' attribute is not supported on targets missing 'neon' or 'mve'; specify an appropriate -march= or -mcpu=}}
 typedef __attribute__((arm_sve_vector_bits(256))) void nosveflag; // expected-error{{'arm_sve_vector_bits' attribute is not supported on targets missing 'sve'; specify an appropriate -march= or -mcpu=}}
diff --git a/clang/test/Sema/missing-field-initializers.c b/clang/test/Sema/missing-field-initializers.c
index 1e65b2d62e1ab84c0ca01db8a7cdece6f8a1d8b8..8653591ff1187a6f78aa706b696a6af77b0f4e00 100644
--- a/clang/test/Sema/missing-field-initializers.c
+++ b/clang/test/Sema/missing-field-initializers.c
@@ -18,7 +18,7 @@ struct Foo bar1[] = {
   1, 2,
   1, 2,
   1
-}; // expected-warning {{missing field 'b' initializer}}
+}; // expected-warning@-1 {{missing field 'b' initializer}}
 
 struct Foo bar2[] = { {}, {}, {} };
 
diff --git a/clang/test/Sema/no_stack_protector.c b/clang/test/Sema/no_stack_protector.c
index 0007435901e8405689ba2a6262e67ba00b942885..1ecd46bc624ceb14dbda75549ef28d0dcbe88d4d 100644
--- a/clang/test/Sema/no_stack_protector.c
+++ b/clang/test/Sema/no_stack_protector.c
@@ -1,4 +1,7 @@
-// RUN: %clang_cc1 -fsyntax-only -verify %s
+// RUN: %clang_cc1 -fsyntax-only -verify -std=c23 %s
+
+[[gnu::no_stack_protector]] void test1(void) {}
+[[clang::no_stack_protector]] void test2(void) {}
 
 void __attribute__((no_stack_protector)) foo(void) {}
 int __attribute__((no_stack_protector)) var; // expected-warning {{'no_stack_protector' attribute only applies to functions}}
diff --git a/clang/test/Sema/no_stack_protector.cpp b/clang/test/Sema/no_stack_protector.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..160e3d32a9389a782da8c7a0f9b769ba44211306
--- /dev/null
+++ b/clang/test/Sema/no_stack_protector.cpp
@@ -0,0 +1,5 @@
+// RUN: %clang_cc1 -fsyntax-only -verify -std=c++11 %s
+// expected-no-diagnostics
+
+[[gnu::no_stack_protector]] void test1() {}
+[[clang::no_stack_protector]] void test2() {}
diff --git a/clang/test/SemaCUDA/neon-attrs.cu b/clang/test/SemaCUDA/neon-attrs.cu
index a72b03f3bbbd7e31dc9777c78db46d92546ba625..129056741ac9a46ae7811768369a45f0c7166829 100644
--- a/clang/test/SemaCUDA/neon-attrs.cu
+++ b/clang/test/SemaCUDA/neon-attrs.cu
@@ -15,7 +15,8 @@
 
 // quiet-no-diagnostics
 typedef __attribute__((neon_vector_type(4))) float float32x4_t;
-// expected-error@-1 {{'neon_vector_type' attribute is not supported on targets missing 'neon' or 'mve'}}
+// expected-error@-1 {{'neon_vector_type' attribute is not supported on targets missing 'neon', 'mve', 'sve' or 'sme'}}
+// expect
 typedef unsigned char poly8_t;
 typedef __attribute__((neon_polyvector_type(8))) poly8_t poly8x8_t;
 // expected-error@-1 {{'neon_polyvector_type' attribute is not supported on targets missing 'neon' or 'mve'}}
diff --git a/clang/test/SemaCXX/alias-template.cpp b/clang/test/SemaCXX/alias-template.cpp
index 5189405e23db567a3f877033e40f4092e65d1b84..dca63e15f5bb8a283935986a5948873af5a222a3 100644
--- a/clang/test/SemaCXX/alias-template.cpp
+++ b/clang/test/SemaCXX/alias-template.cpp
@@ -192,3 +192,68 @@ int g = sfinae_me<int>(); // expected-error{{no matching function for call to 's
 namespace NullExceptionDecl {
 template<int... I> auto get = []() { try { } catch(...) {}; return I; }; // expected-error{{initializer contains unexpanded parameter pack 'I'}}
 }
+
+namespace GH41693 {
+// No errors when a type alias defined in a class or a friend of a class
+// accesses private members of the same class.
+struct S {
+private:
+  template <typename> static constexpr void Impl() {}
+
+public:
+  template <typename X> using U = decltype(Impl<X>());
+};
+
+using X = S::U<void>;
+struct Y {
+private:
+  static constexpr int x=0;
+
+  template <typename>
+  static constexpr int y=0;
+
+  template <typename>
+  static constexpr int foo();
+
+public:
+  template <typename U>
+  using bar1 = decltype(foo<U>());
+  using bar2 = decltype(x);
+  template <typename U>
+  using bar3 = decltype(y<U>);
+};
+
+
+using type1 = Y::bar1<float>;
+using type2 = Y::bar2;
+using type3 = Y::bar3<float>;
+
+struct theFriend{
+  template<class T>
+  using theAlias = decltype(&T::i);
+};
+
+class theC{
+  int i;
+  public:
+  friend struct theFriend;
+};
+
+int foo(){
+  (void)sizeof(theFriend::theAlias<theC>);
+}
+
+// Test case that regressed with the first iteration of the fix for GH41693.
+template <typename T> class SP {
+    T* data;
+};
+
+template <typename T> class A {
+    static SP<A> foo();
+};
+
+template<typename T> using TRet = SP<A<T>>;
+
+template<typename T> TRet<T> A<T>::foo() { return TRet<T>{};};
+
+}
diff --git a/clang/test/SemaCXX/attr-suppress.cpp b/clang/test/SemaCXX/attr-suppress.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..fb5e2ac7ce20667aa7aa57176f49d7ef9c70c4d5
--- /dev/null
+++ b/clang/test/SemaCXX/attr-suppress.cpp
@@ -0,0 +1,62 @@
+// RUN: %clang_cc1 -std=c++11 -fsyntax-only %s -verify
+
+[[gsl::suppress("globally")]];
+
+namespace N {
+[[gsl::suppress("in-a-namespace")]];
+}
+
+[[gsl::suppress("readability-identifier-naming")]] void f_() {
+  int *p;
+  [[gsl::suppress("type", "bounds")]] {
+    p = reinterpret_cast<int *>(7);
+  }
+
+  [[gsl::suppress]] int x;       // expected-error {{'suppress' attribute takes at least 1 argument}}
+  [[gsl::suppress()]] int y;     // expected-error {{'suppress' attribute takes at least 1 argument}}
+  int [[gsl::suppress("r")]] z;  // expected-error {{'suppress' attribute cannot be applied to types}}
+  [[gsl::suppress(f_)]] float f; // expected-error {{expected string literal as argument of 'suppress' attribute}}
+}
+
+union [[gsl::suppress("type.1")]] U {
+  int i;
+  float f;
+};
+
+[[clang::suppress]];
+// expected-error@-1 {{'suppress' attribute only applies to variables and statements}}
+
+namespace N {
+[[clang::suppress("in-a-namespace")]];
+// expected-error@-1 {{'suppress' attribute only applies to variables and statements}}
+} // namespace N
+
+[[clang::suppress]] int global = 42;
+
+[[clang::suppress]] void foo() {
+  // expected-error@-1 {{'suppress' attribute only applies to variables and statements}}
+  [[clang::suppress]] int *p;
+
+  [[clang::suppress]] int a = 0;           // no-warning
+  [[clang::suppress()]] int b = 1;         // no-warning
+  [[clang::suppress("a")]] int c = a + b;  // no-warning
+  [[clang::suppress("a", "b")]] b = c - a; // no-warning
+
+  [[clang::suppress("a", "b")]] if (b == 10) a += 4; // no-warning
+  [[clang::suppress]] while (true) {}                // no-warning
+  [[clang::suppress]] switch (a) {                   // no-warning
+  default:
+    c -= 10;
+  }
+
+  int [[clang::suppress("r")]] z;
+  // expected-error@-1 {{'suppress' attribute cannot be applied to types}}
+  [[clang::suppress(foo)]] float f;
+  // expected-error@-1 {{expected string literal as argument of 'suppress' attribute}}
+}
+
+class [[clang::suppress("type.1")]] V {
+  // expected-error@-1 {{'suppress' attribute only applies to variables and statements}}
+  int i;
+  float f;
+};
diff --git a/clang/test/SemaCXX/cxx2a-initializer-aggregates.cpp b/clang/test/SemaCXX/cxx2a-initializer-aggregates.cpp
index 510ace58c35a6aaaf5a308013f121d532f495156..0d977e07ed034e0f423cb0ed2f37d52d943ea357 100644
--- a/clang/test/SemaCXX/cxx2a-initializer-aggregates.cpp
+++ b/clang/test/SemaCXX/cxx2a-initializer-aggregates.cpp
@@ -1,6 +1,6 @@
 // RUN: %clang_cc1 -std=c++20 %s -verify=cxx20,expected,pedantic,override,reorder -pedantic-errors
 // RUN: %clang_cc1 -std=c++17 %s -verify=expected,pedantic,override,reorder -Wno-c++20-designator -pedantic-errors
-// RUN: %clang_cc1 -std=c++20 %s -verify=cxx20,expected,pedantic -Werror=c99-designator -Wno-reorder-init-list -Wno-initializer-overrides
+// RUN: %clang_cc1 -std=c++20 %s -verify=cxx20,expected,pedantic -Werror=c99-designator -Wno-reorder-init-list -Wno-initializer-overrides -Werror=nested-anon-types -Werror=gnu-anonymous-struct
 // RUN: %clang_cc1 -std=c++20 %s -verify=cxx20,expected,reorder -Wno-c99-designator -Werror=reorder-init-list -Wno-initializer-overrides
 // RUN: %clang_cc1 -std=c++20 %s -verify=cxx20,expected,override -Wno-c99-designator -Wno-reorder-init-list -Werror=initializer-overrides
 // RUN: %clang_cc1 -std=c++20 %s -verify=cxx20,expected -Wno-c99-designator -Wno-reorder-init-list -Wno-initializer-overrides
@@ -39,6 +39,7 @@ A a1 = {
 };
 int arr[3] = {[1] = 5}; // pedantic-error {{array designators are a C99 extension}}
 B b = {.a.x = 0}; // pedantic-error {{nested designators are a C99 extension}}
+                  // wmissing-warning@-1 {{missing field 'y' initializer}}
 A a2 = {
   .x = 1, // pedantic-error {{mixture of designated and non-designated initializers in the same initializer list is a C99 extension}}
   2 // pedantic-note {{first non-designated initializer is here}}
@@ -60,7 +61,6 @@ B b2 = {.a = 1}; // pedantic-error {{brace elision for designated initializer is
 B b3 = {.a = 1, 2}; // pedantic-error {{mixture of designated and non-designated}} pedantic-note {{first non-designated}} pedantic-error {{brace elision}}
 B b4 = {.a = 1, 2, 3}; // pedantic-error {{mixture of designated and non-designated}} pedantic-note {{first non-designated}} pedantic-error {{brace elision}} expected-error {{excess elements}}
 B b5 = {.a = nullptr}; // expected-error {{cannot initialize}}
-                       // wmissing-warning@-1 {{missing field 'y' initializer}}
 struct C { int :0, x, :0, y, :0; };
 C c = {
   .x = 1, // override-note {{previous}}
@@ -247,3 +247,87 @@ void foo() {
                            //
 }
 }
+
+namespace GH70384 {
+
+struct A {
+  int m;
+  union { int a; float n = 0; };
+};
+
+struct B {
+  int m;
+  int b;
+  union { int a ; };
+};
+
+union CU {
+  int a = 1;
+  double b;
+};
+
+struct C {
+  int a;
+  union { int b; CU c;};
+};
+
+struct CC {
+  int a;
+  CU c;
+};
+
+void foo() {
+  A a = A{.m = 0};
+  A aa = {0};
+  A aaa = {.a = 7}; // wmissing-warning {{missing field 'm' initializer}}
+  B b = {.m = 1, .b = 3 }; //wmissing-warning {{missing field 'a' initializer}}
+  B bb = {1}; // wmissing-warning {{missing field 'b' initializer}}
+              // wmissing-warning@-1 {{missing field 'a' initializer}}
+  C c = {.a = 1}; // wmissing-warning {{missing field 'b' initializer}}
+  CC cc = {.a = 1}; // wmissing-warning {{missing field 'c' initializer}}
+}
+
+struct C1 {
+  int m;
+  union { float b; union {int n = 1; }; };
+  // pedantic-error@-1 {{anonymous types declared in an anonymous union are an extension}}
+};
+
+struct C2 {
+  int m;
+  struct { float b; int n = 1; }; // pedantic-error {{anonymous structs are a GNU extension}}
+};
+
+struct C3 {
+  int m;
+  struct { float b = 1; union {int a;}; int n = 1; };
+  // pedantic-error@-1 {{anonymous structs are a GNU extension}}
+  // pedantic-error@-2 {{anonymous types declared in an anonymous struct are an extension}}
+};
+
+C1 c = C1{.m = 1};
+C1 cc = C1{.b = 1}; // wmissing-warning {{missing field 'm' initializer}}
+C2 c1 = C2{.m = 1}; // wmissing-warning {{missing field 'b' initializer}}
+C2 c22 = C2{.m = 1, .b = 1};
+C3 c2 = C3{.b = 1}; // wmissing-warning {{missing field 'a' initializer}}
+                    // wmissing-warning@-1 {{missing field 'm' initializer}}
+
+struct C4 {
+  union {
+    struct { int n; }; // pedantic-error {{anonymous structs are a GNU extension}}
+    // pedantic-error@-1 {{anonymous types declared in an anonymous union are an extension}}
+    int m = 0; };
+  int z;
+};
+C4 a = {.z = 1};
+
+struct C5 {
+  int a;
+  struct { // pedantic-error {{anonymous structs are a GNU extension}}
+    int x;
+    struct { int y = 0; };  // pedantic-error {{anonymous types declared in an anonymous struct are an extension}}
+                            // pedantic-error@-1 {{anonymous structs are a GNU extension}}
+  };
+};
+C5 c5 = C5{.a = 0}; //wmissing-warning {{missing field 'x' initializer}}
+}
diff --git a/clang/test/SemaCXX/suppress.cpp b/clang/test/SemaCXX/suppress.cpp
deleted file mode 100644
index 29544b3c573ccc8695589605675ad1f54d33a75b..0000000000000000000000000000000000000000
--- a/clang/test/SemaCXX/suppress.cpp
+++ /dev/null
@@ -1,25 +0,0 @@
-// RUN: %clang_cc1 -std=c++11 -fsyntax-only %s -verify
-
-[[gsl::suppress("globally")]];
-
-namespace N {
-  [[gsl::suppress("in-a-namespace")]];
-}
-
-[[gsl::suppress("readability-identifier-naming")]]
-void f_() {
-  int *p;
-  [[gsl::suppress("type", "bounds")]] {
-    p = reinterpret_cast<int *>(7);
-  }
-
-  [[gsl::suppress]] int x; // expected-error {{'suppress' attribute takes at least 1 argument}}
-  [[gsl::suppress()]] int y; // expected-error {{'suppress' attribute takes at least 1 argument}}
-  int [[gsl::suppress("r")]] z; // expected-error {{'suppress' attribute cannot be applied to types}}
-  [[gsl::suppress(f_)]] float f; // expected-error {{expected string literal as argument of 'suppress' attribut}}
-}
-
-union [[gsl::suppress("type.1")]] U {
-  int i;
-  float f;
-};
diff --git a/clang/test/SemaObjC/attr-suppress.m b/clang/test/SemaObjC/attr-suppress.m
new file mode 100644
index 0000000000000000000000000000000000000000..ade8f94ec5895ef6d4f7a86780408be98527737b
--- /dev/null
+++ b/clang/test/SemaObjC/attr-suppress.m
@@ -0,0 +1,50 @@
+// RUN: %clang_cc1 -fsyntax-only -fblocks %s -verify
+
+#define SUPPRESS1 __attribute__((suppress))
+#define SUPPRESS2(...) __attribute__((suppress(__VA_ARGS__)))
+
+SUPPRESS1 int global = 42;
+
+SUPPRESS1 void foo() {
+  // expected-error@-1 {{'suppress' attribute only applies to variables and statements}}
+  SUPPRESS1 int *p;
+
+  SUPPRESS1 int a = 0; // no-warning
+  SUPPRESS2()
+  int b = 1; // no-warning
+  SUPPRESS2("a")
+  int c = a + b;                     // no-warning
+  SUPPRESS2("a", "b") { b = c - a; } // no-warning
+
+  SUPPRESS2("a", "b")
+  if (b == 10)
+    a += 4;              // no-warning
+  SUPPRESS1 while (1) {} // no-warning
+  SUPPRESS1 switch (a) { // no-warning
+  default:
+    c -= 10;
+  }
+
+  // GNU-style attributes and C++11 attributes apply to different things when
+  // written like this.  GNU  attribute gets attached to the declaration, while
+  // C++11 attribute ends up on the type.
+  int SUPPRESS2("r") z;
+  SUPPRESS2(foo)
+  float f;
+  // expected-error@-2 {{expected string literal as argument of 'suppress' attribute}}
+}
+
+union SUPPRESS2("type.1") U {
+  // expected-error@-1 {{'suppress' attribute only applies to variables and statements}}
+  int i;
+  float f;
+};
+
+SUPPRESS1 @interface Test {
+  // expected-error@-1 {{'suppress' attribute only applies to variables and statements}}
+}
+@property SUPPRESS2("prop") int *prop;
+// expected-error@-1 {{'suppress' attribute only applies to variables and statements}}
+- (void)bar:(int)x SUPPRESS1;
+// expected-error@-1 {{'suppress' attribute only applies to variables and statements}}
+@end
diff --git a/clang/tools/arcmt-test/arcmt-test.cpp b/clang/tools/arcmt-test/arcmt-test.cpp
index 53229ac570bc188fd40680d4be5f6d5c39e979d5..b61f38e9905dba7cfe8ac07229a7cb47e1feebb4 100644
--- a/clang/tools/arcmt-test/arcmt-test.cpp
+++ b/clang/tools/arcmt-test/arcmt-test.cpp
@@ -230,7 +230,7 @@ static bool verifyTransformedFiles(ArrayRef<std::string> resultFiles) {
   for (ArrayRef<std::string>::iterator
          I = resultFiles.begin(), E = resultFiles.end(); I != E; ++I) {
     StringRef fname(*I);
-    if (!fname.endswith(".result")) {
+    if (!fname.ends_with(".result")) {
       errs() << "error: filename '" << fname
                    << "' does not have '.result' extension\n";
       return true;
diff --git a/clang/tools/c-arcmt-test/c-arcmt-test.c b/clang/tools/c-arcmt-test/c-arcmt-test.c
index 3bbb2d5d6a8564ab1223cc4bd40e516a54da3bcf..00999f188c7dcedc1540b824394cb0e77c402bcb 100644
--- a/clang/tools/c-arcmt-test/c-arcmt-test.c
+++ b/clang/tools/c-arcmt-test/c-arcmt-test.c
@@ -1,8 +1,9 @@
 /* c-arcmt-test.c */
 
 #include "clang-c/Index.h"
-#include <stdlib.h>
+#include "llvm/Support/AutoConvert.h"
 #include <stdio.h>
+#include <stdlib.h>
 #include <string.h>
 #if defined(_WIN32)
 #include <io.h>
@@ -107,6 +108,14 @@ static void flush_atexit(void) {
 }
 
 int main(int argc, const char **argv) {
+#ifdef __MVS__
+  if (enableAutoConversion(fileno(stdout)) == -1)
+    fprintf(stderr, "Setting conversion on stdout failed\n");
+
+  if (enableAutoConversion(fileno(stderr)) == -1)
+    fprintf(stderr, "Setting conversion on stderr failed\n");
+#endif
+
   thread_info client_data;
 
   atexit(flush_atexit);
diff --git a/clang/tools/c-index-test/c-index-test.c b/clang/tools/c-index-test/c-index-test.c
index 2c0c9cb8eb5e42fd701ad8a13d96a35f28dc7cd0..6fa400a0675b7a84e43712b6e3969d239e71117d 100644
--- a/clang/tools/c-index-test/c-index-test.c
+++ b/clang/tools/c-index-test/c-index-test.c
@@ -8,6 +8,7 @@
 #include "clang-c/Documentation.h"
 #include "clang-c/Index.h"
 #include "clang/Config/config.h"
+#include "llvm/Support/AutoConvert.h"
 #include <assert.h>
 #include <ctype.h>
 #include <stdio.h>
@@ -5150,6 +5151,14 @@ static void flush_atexit(void) {
 int main(int argc, const char **argv) {
   thread_info client_data;
 
+#ifdef __MVS__
+  if (enableAutoConversion(fileno(stdout)) == -1)
+    fprintf(stderr, "Setting conversion on stdout failed\n");
+
+  if (enableAutoConversion(fileno(stderr)) == -1)
+    fprintf(stderr, "Setting conversion on stderr failed\n");
+#endif
+
   atexit(flush_atexit);
 
 #ifdef CLANG_HAVE_LIBXML
diff --git a/clang/tools/clang-extdef-mapping/ClangExtDefMapGen.cpp b/clang/tools/clang-extdef-mapping/ClangExtDefMapGen.cpp
index 769727eedec721612cdd81958321548db5a10b88..c048f335f91ca1e7c43fa5bf7e8813c5eb89bd55 100644
--- a/clang/tools/clang-extdef-mapping/ClangExtDefMapGen.cpp
+++ b/clang/tools/clang-extdef-mapping/ClangExtDefMapGen.cpp
@@ -181,7 +181,7 @@ static int HandleFiles(ArrayRef<std::string> SourceFiles,
   // process them directly in HandleAST, otherwise put them
   // on a list for ClangTool to handle.
   for (StringRef Src : SourceFiles) {
-    if (Src.endswith(".ast")) {
+    if (Src.ends_with(".ast")) {
       if (!HandleAST(Src)) {
         return 1;
       }
diff --git a/clang/tools/clang-format/git-clang-format.bat b/clang/tools/clang-format/git-clang-format.bat
index d4bc5172989cb098e2b1290b5c3906d590562911..9965cd4312fe39eec0d3395050f3d8664d373566 100644
--- a/clang/tools/clang-format/git-clang-format.bat
+++ b/clang/tools/clang-format/git-clang-format.bat
@@ -1 +1 @@
-py -3 git-clang-format %*
+py -3 %~pn0 %*
diff --git a/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp b/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
index 5d2fe98fe56011588a4fa21748cd2c9bc10565a8..bd2fd02c6a3e8f93b519e54050d178da3d263add 100644
--- a/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
+++ b/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp
@@ -1151,7 +1151,7 @@ linkAndWrapDeviceFiles(SmallVectorImpl<OffloadFile> &LinkerInputFiles,
 std::optional<std::string> findFile(StringRef Dir, StringRef Root,
                                     const Twine &Name) {
   SmallString<128> Path;
-  if (Dir.startswith("="))
+  if (Dir.starts_with("="))
     sys::path::append(Path, Root, Dir.substr(1), Name);
   else
     sys::path::append(Path, Dir, Name);
@@ -1188,7 +1188,7 @@ searchLibraryBaseName(StringRef Name, StringRef Root,
 /// `-lfoo` or `-l:libfoo.a`.
 std::optional<std::string> searchLibrary(StringRef Input, StringRef Root,
                                          ArrayRef<StringRef> SearchPaths) {
-  if (Input.startswith(":") || Input.ends_with(".lib"))
+  if (Input.starts_with(":") || Input.ends_with(".lib"))
     return findFromSearchPaths(Input.drop_front(), Root, SearchPaths);
   return searchLibraryBaseName(Input, Root, SearchPaths);
 }
diff --git a/clang/tools/clang-refactor/ClangRefactor.cpp b/clang/tools/clang-refactor/ClangRefactor.cpp
index d362eecf06d8a5a62193ff9826f41b20478f32be..175a2b8234e9acd9c8efb1f1661c14c6381af409 100644
--- a/clang/tools/clang-refactor/ClangRefactor.cpp
+++ b/clang/tools/clang-refactor/ClangRefactor.cpp
@@ -146,7 +146,7 @@ private:
 
 std::unique_ptr<SourceSelectionArgument>
 SourceSelectionArgument::fromString(StringRef Value) {
-  if (Value.startswith("test:")) {
+  if (Value.starts_with("test:")) {
     StringRef Filename = Value.drop_front(strlen("test:"));
     std::optional<TestSelectionRangesInFile> ParsedTestSelection =
         findTestSelectionRanges(Filename);
diff --git a/clang/tools/clang-repl/ClangRepl.cpp b/clang/tools/clang-repl/ClangRepl.cpp
index 5663c2c5a6c9285450cbc850741e4e45afc11b49..b9b287127015fd920d66e7097b331a843cb6166a 100644
--- a/clang/tools/clang-repl/ClangRepl.cpp
+++ b/clang/tools/clang-repl/ClangRepl.cpp
@@ -234,7 +234,7 @@ int main(int argc, const char **argv) {
     while (std::optional<std::string> Line = LE.readLine()) {
       llvm::StringRef L = *Line;
       L = L.trim();
-      if (L.endswith("\\")) {
+      if (L.ends_with("\\")) {
         // FIXME: Support #ifdef X \ ...
         Input += L.drop_back(1);
         LE.setPrompt("clang-repl...   ");
diff --git a/clang/tools/clang-scan-deps/ClangScanDeps.cpp b/clang/tools/clang-scan-deps/ClangScanDeps.cpp
index f11c933d9576565a91a936121988f78ce2ecb149..75aa4ae97c618c4efc13104eec2ff267a0b18405 100644
--- a/clang/tools/clang-scan-deps/ClangScanDeps.cpp
+++ b/clang/tools/clang-scan-deps/ClangScanDeps.cpp
@@ -830,9 +830,9 @@ int clang_scan_deps_main(int argc, char **argv, const llvm::ToolContext &) {
                 // Also, clang-cl adds ".obj" extension if none is found.
                 if ((Arg == "-o" || Arg == "/o") && I != R)
                   LastO = I[-1]; // Next argument (reverse iterator)
-                else if (Arg.startswith("/Fo") || Arg.startswith("-Fo"))
+                else if (Arg.starts_with("/Fo") || Arg.starts_with("-Fo"))
                   LastO = Arg.drop_front(3).str();
-                else if (Arg.startswith("/o") || Arg.startswith("-o"))
+                else if (Arg.starts_with("/o") || Arg.starts_with("-o"))
                   LastO = Arg.drop_front(2).str();
 
                 if (!LastO.empty() && !llvm::sys::path::has_extension(LastO))
diff --git a/clang/tools/diagtool/TreeView.cpp b/clang/tools/diagtool/TreeView.cpp
index 4f5d3fd3ef0a89c8869fc83e097acc94efedbcef..eae16243d3d59cb42f296b8807c4e42ff01d9dee 100644
--- a/clang/tools/diagtool/TreeView.cpp
+++ b/clang/tools/diagtool/TreeView.cpp
@@ -160,7 +160,7 @@ int TreeView::run(unsigned int argc, char **argv, llvm::raw_ostream &out) {
     break;
   case 1:
     RootGroup = argv[0];
-    if (RootGroup.startswith("-W"))
+    if (RootGroup.starts_with("-W"))
       RootGroup = RootGroup.substr(2);
     if (RootGroup == "everything")
       ShowAll = true;
diff --git a/clang/tools/driver/driver.cpp b/clang/tools/driver/driver.cpp
index 531b5b4a61c1804452918c3caab702440a3571e6..4adc7f7ad0dac397c26fe126030ea2a3728569c5 100644
--- a/clang/tools/driver/driver.cpp
+++ b/clang/tools/driver/driver.cpp
@@ -122,7 +122,7 @@ static void ApplyOneQAOverride(raw_ostream &OS,
       GetStableCStr(SavedStrings, Edit.substr(1));
     OS << "### Adding argument " << Str << " at end\n";
     Args.push_back(Str);
-  } else if (Edit[0] == 's' && Edit[1] == '/' && Edit.endswith("/") &&
+  } else if (Edit[0] == 's' && Edit[1] == '/' && Edit.ends_with("/") &&
              Edit.slice(2, Edit.size() - 1).contains('/')) {
     StringRef MatchPattern = Edit.substr(2).split('/').first;
     StringRef ReplPattern = Edit.substr(2).split('/').second;
@@ -403,7 +403,7 @@ int clang_main(int Argc, char **Argv, const llvm::ToolContext &ToolContext) {
   }
 
   // Handle -cc1 integrated tools.
-  if (Args.size() >= 2 && StringRef(Args[1]).startswith("-cc1"))
+  if (Args.size() >= 2 && StringRef(Args[1]).starts_with("-cc1"))
     return ExecuteCC1Tool(Args, ToolContext);
 
   // Handle options that need handling before the real command line parsing in
diff --git a/clang/tools/libclang/CIndexUSRs.cpp b/clang/tools/libclang/CIndexUSRs.cpp
index 75bb3b01299f10b16f8f9290db1b2d6c13aac333..be7c670cca0113617ef07b874ff875f96bc9dc4e 100644
--- a/clang/tools/libclang/CIndexUSRs.cpp
+++ b/clang/tools/libclang/CIndexUSRs.cpp
@@ -28,7 +28,7 @@ using namespace clang::index;
 //===----------------------------------------------------------------------===//
 
 static inline StringRef extractUSRSuffix(StringRef s) {
-  return s.startswith("c:") ? s.substr(2) : "";
+  return s.starts_with("c:") ? s.substr(2) : "";
 }
 
 bool cxcursor::getDeclCursorUSR(const Decl *D, SmallVectorImpl<char> &Buf) {
diff --git a/clang/unittests/Analysis/CloneDetectionTest.cpp b/clang/unittests/Analysis/CloneDetectionTest.cpp
index fe65fab98c5ed47a3b900958e19976b20b84f2c4..738f6efd2018d77733520848fe987b0a9a5b107c 100644
--- a/clang/unittests/Analysis/CloneDetectionTest.cpp
+++ b/clang/unittests/Analysis/CloneDetectionTest.cpp
@@ -42,7 +42,7 @@ public:
           for (const StmtSequence &Arg : {A, B}) {
             if (const auto *D =
                     dyn_cast<const FunctionDecl>(Arg.getContainingDecl())) {
-              if (D->getName().startswith("bar"))
+              if (D->getName().starts_with("bar"))
                 return false;
             }
           }
diff --git a/clang/unittests/Driver/ModuleCacheTest.cpp b/clang/unittests/Driver/ModuleCacheTest.cpp
index 6a0f68f26a6767a96688cc362756d20fd3c6bf2d..48744415647e68471f53bbb8827b55a5669a42b9 100644
--- a/clang/unittests/Driver/ModuleCacheTest.cpp
+++ b/clang/unittests/Driver/ModuleCacheTest.cpp
@@ -22,6 +22,6 @@ TEST(ModuleCacheTest, GetTargetAndMode) {
   Driver::getDefaultModuleCachePath(Buf);
   StringRef Path = Buf;
   EXPECT_TRUE(Path.find("clang") != Path.npos);
-  EXPECT_TRUE(Path.endswith("ModuleCache"));  
+  EXPECT_TRUE(Path.ends_with("ModuleCache"));
 }
 } // end anonymous namespace.
diff --git a/clang/unittests/Driver/MultilibBuilderTest.cpp b/clang/unittests/Driver/MultilibBuilderTest.cpp
index 60fe10ac3ba556e9851f474635517727a5226b01..e23fe7e2441db534148ca3b54191e99f88e45b97 100644
--- a/clang/unittests/Driver/MultilibBuilderTest.cpp
+++ b/clang/unittests/Driver/MultilibBuilderTest.cpp
@@ -144,7 +144,7 @@ TEST(MultilibBuilderTest, SetFilterObject) {
       << "Size before filter was incorrect. Contents:\n"
       << MS;
   MS.FilterOut([](const Multilib &M) {
-    return StringRef(M.gccSuffix()).startswith("/p");
+    return StringRef(M.gccSuffix()).starts_with("/p");
   });
   ASSERT_EQ((int)MS.size(), 1 /* Default */ + 1 /* orange */ +
                                 1 /* orange/pear */ + 1 /* orange/plum */ +
@@ -152,7 +152,7 @@ TEST(MultilibBuilderTest, SetFilterObject) {
       << "Size after filter was incorrect. Contents:\n"
       << MS;
   for (MultilibSet::const_iterator I = MS.begin(), E = MS.end(); I != E; ++I) {
-    ASSERT_FALSE(StringRef(I->gccSuffix()).startswith("/p"))
+    ASSERT_FALSE(StringRef(I->gccSuffix()).starts_with("/p"))
         << "The filter should have removed " << *I;
   }
 }
diff --git a/clang/unittests/Driver/ToolChainTest.cpp b/clang/unittests/Driver/ToolChainTest.cpp
index acbbb87390d5e9ae3a26586ac290f34059bd671c..a9b5f3c700315c49d0234587edfa2f61a7cedab8 100644
--- a/clang/unittests/Driver/ToolChainTest.cpp
+++ b/clang/unittests/Driver/ToolChainTest.cpp
@@ -531,7 +531,7 @@ TEST(ToolChainTest, CommandOutput) {
   const auto &InFile = CmdCompile->getInputInfos().front().getFilename();
   EXPECT_STREQ(InFile, "foo.cpp");
   auto ObjFile = CmdCompile->getOutputFilenames().front();
-  EXPECT_TRUE(StringRef(ObjFile).endswith(".o"));
+  EXPECT_TRUE(StringRef(ObjFile).ends_with(".o"));
 
   const auto &CmdLink = Jobs.getJobs().back();
   const auto LinkInFile = CmdLink->getInputInfos().front().getFilename();
diff --git a/clang/unittests/Format/TokenAnnotatorTest.cpp b/clang/unittests/Format/TokenAnnotatorTest.cpp
index 65b1f0f4b57659870048ca86e2e00d863402afae..8e6935319b2f3d6da5b40a0d7b2d3bfaddc71b0d 100644
--- a/clang/unittests/Format/TokenAnnotatorTest.cpp
+++ b/clang/unittests/Format/TokenAnnotatorTest.cpp
@@ -298,6 +298,17 @@ TEST_F(TokenAnnotatorTest, UnderstandsUsesOfStarAndAmp) {
   ASSERT_EQ(Tokens.size(), 12u) << Tokens;
   EXPECT_TOKEN(Tokens[2], tok::identifier, TT_TypeName);
   EXPECT_TOKEN(Tokens[3], tok::star, TT_PointerOrReference);
+
+  Tokens = annotate("class Foo {\n"
+                    "  void operator<() {}\n"
+                    "  Foo &f;\n"
+                    "};");
+  ASSERT_EQ(Tokens.size(), 17u) << Tokens;
+  EXPECT_TOKEN(Tokens[4], tok::kw_operator, TT_FunctionDeclarationName);
+  EXPECT_TOKEN(Tokens[5], tok::less, TT_OverloadedOperator);
+  EXPECT_TOKEN(Tokens[6], tok::l_paren, TT_OverloadedOperatorLParen);
+  EXPECT_TOKEN(Tokens[8], tok::l_brace, TT_FunctionLBrace);
+  EXPECT_TOKEN(Tokens[11], tok::amp, TT_PointerOrReference);
 }
 
 TEST_F(TokenAnnotatorTest, UnderstandsUsesOfPlusAndMinus) {
diff --git a/clang/unittests/Frontend/OutputStreamTest.cpp b/clang/unittests/Frontend/OutputStreamTest.cpp
index 9cb101ecff8aafe025dd306f4f484396dfc9d609..7d360f661daa30f05a9c9db790dea8ac537ab5cc 100644
--- a/clang/unittests/Frontend/OutputStreamTest.cpp
+++ b/clang/unittests/Frontend/OutputStreamTest.cpp
@@ -42,7 +42,7 @@ TEST(FrontendOutputTests, TestOutputStream) {
   bool Success = ExecuteCompilerInvocation(&Compiler);
   EXPECT_TRUE(Success);
   EXPECT_TRUE(!IRBuffer.empty());
-  EXPECT_TRUE(StringRef(IRBuffer.data()).startswith("BC"));
+  EXPECT_TRUE(StringRef(IRBuffer.data()).starts_with("BC"));
 }
 
 TEST(FrontendOutputTests, TestVerboseOutputStreamShared) {
diff --git a/clang/unittests/Interpreter/IncrementalProcessingTest.cpp b/clang/unittests/Interpreter/IncrementalProcessingTest.cpp
index f43b3ddac68f941ee5e7f1c37fdde94527a6d5c7..accdf682896340b4f570f0ef665d664231024202 100644
--- a/clang/unittests/Interpreter/IncrementalProcessingTest.cpp
+++ b/clang/unittests/Interpreter/IncrementalProcessingTest.cpp
@@ -44,7 +44,7 @@ const char TestProgram2[] = "extern \"C\" int funcForProg2() { return 42; }\n"
 
 const Function *getGlobalInit(llvm::Module *M) {
   for (const auto &Func : *M)
-    if (Func.hasName() && Func.getName().startswith("_GLOBAL__sub_I_"))
+    if (Func.hasName() && Func.getName().starts_with("_GLOBAL__sub_I_"))
       return &Func;
 
   return nullptr;
diff --git a/clang/unittests/StaticAnalyzer/AnalyzerOptionsTest.cpp b/clang/unittests/StaticAnalyzer/AnalyzerOptionsTest.cpp
index cd78014eae9d61ce53f0c43d76192398c9601f46..aace4b991b5ecabac79f7493d839b005cfc7a8bc 100644
--- a/clang/unittests/StaticAnalyzer/AnalyzerOptionsTest.cpp
+++ b/clang/unittests/StaticAnalyzer/AnalyzerOptionsTest.cpp
@@ -15,10 +15,10 @@ namespace ento {
 
 TEST(StaticAnalyzerOptions, getRegisteredCheckers) {
   auto IsDebugChecker = [](StringRef CheckerName) {
-    return CheckerName.startswith("debug");
+    return CheckerName.starts_with("debug");
   };
   auto IsAlphaChecker = [](StringRef CheckerName) {
-    return CheckerName.startswith("alpha");
+    return CheckerName.starts_with("alpha");
   };
   const auto &AllCheckers =
       AnalyzerOptions::getRegisteredCheckers(/*IncludeExperimental=*/true);
diff --git a/clang/unittests/Tooling/HeaderIncludesTest.cpp b/clang/unittests/Tooling/HeaderIncludesTest.cpp
index 256aa825554c5073ba9f0d8d4c83d13297a9cee8..929156a11d0d9a23b5cbe3547d79d3f1fc826e17 100644
--- a/clang/unittests/Tooling/HeaderIncludesTest.cpp
+++ b/clang/unittests/Tooling/HeaderIncludesTest.cpp
@@ -23,9 +23,9 @@ protected:
   std::string insert(llvm::StringRef Code, llvm::StringRef Header,
                      IncludeDirective Directive = IncludeDirective::Include) {
     HeaderIncludes Includes(FileName, Code, Style);
-    assert(Header.startswith("\"") || Header.startswith("<"));
-    auto R =
-        Includes.insert(Header.trim("\"<>"), Header.startswith("<"), Directive);
+    assert(Header.starts_with("\"") || Header.starts_with("<"));
+    auto R = Includes.insert(Header.trim("\"<>"), Header.starts_with("<"),
+                             Directive);
     if (!R)
       return std::string(Code);
     auto Result = applyAllReplacements(Code, Replacements(*R));
@@ -35,8 +35,9 @@ protected:
 
   std::string remove(llvm::StringRef Code, llvm::StringRef Header) {
     HeaderIncludes Includes(FileName, Code, Style);
-    assert(Header.startswith("\"") || Header.startswith("<"));
-    auto Replaces = Includes.remove(Header.trim("\"<>"), Header.startswith("<"));
+    assert(Header.starts_with("\"") || Header.starts_with("<"));
+    auto Replaces =
+        Includes.remove(Header.trim("\"<>"), Header.starts_with("<"));
     auto Result = applyAllReplacements(Code, Replaces);
     EXPECT_TRUE(static_cast<bool>(Result));
     return *Result;
diff --git a/clang/unittests/libclang/LibclangTest.cpp b/clang/unittests/libclang/LibclangTest.cpp
index 60904bc3baf89e1aae7970deed12c06dac69335b..87075a46d75187b0d8a1e2c3fa4be877a93f00ae 100644
--- a/clang/unittests/libclang/LibclangTest.cpp
+++ b/clang/unittests/libclang/LibclangTest.cpp
@@ -451,8 +451,8 @@ public:
 
       const auto Filename = llvm::sys::path::filename(File->path());
       EXPECT_EQ(Filename.size(), std::strlen("preamble-%%%%%%.pch"));
-      EXPECT_TRUE(Filename.startswith("preamble-"));
-      EXPECT_TRUE(Filename.endswith(".pch"));
+      EXPECT_TRUE(Filename.starts_with("preamble-"));
+      EXPECT_TRUE(Filename.ends_with(".pch"));
 
       const auto Status = File->status();
       ASSERT_TRUE(Status);
@@ -659,7 +659,7 @@ TEST_F(LibclangReparseTest, FileName) {
   clang_disposeString(cxname);
 
   cxname = clang_File_tryGetRealPathName(cxf);
-  ASSERT_TRUE(llvm::StringRef(clang_getCString(cxname)).endswith("main.cpp"));
+  ASSERT_TRUE(llvm::StringRef(clang_getCString(cxname)).ends_with("main.cpp"));
   clang_disposeString(cxname);
 }
 
diff --git a/clang/utils/TableGen/ASTTableGen.cpp b/clang/utils/TableGen/ASTTableGen.cpp
index 60f563d9a1ffc8b8eed2ff1d81251805a1604e19..54288ff6a03be3b7203bec275d167d35d95f7f39 100644
--- a/clang/utils/TableGen/ASTTableGen.cpp
+++ b/clang/utils/TableGen/ASTTableGen.cpp
@@ -33,7 +33,7 @@ llvm::StringRef clang::tblgen::HasProperties::getName() const {
 
 static StringRef removeExpectedNodeNameSuffix(Record *node, StringRef suffix) {
   StringRef nodeName = node->getName();
-  if (!nodeName.endswith(suffix)) {
+  if (!nodeName.ends_with(suffix)) {
     PrintFatalError(node->getLoc(),
                     Twine("name of node doesn't end in ") + suffix);
   }
diff --git a/clang/utils/TableGen/MveEmitter.cpp b/clang/utils/TableGen/MveEmitter.cpp
index fae889d68346c33c8ad22c8ad67eb63843722562..f0bd0865c1c8b56fac081ea70f4e4a433c90583e 100644
--- a/clang/utils/TableGen/MveEmitter.cpp
+++ b/clang/utils/TableGen/MveEmitter.cpp
@@ -882,7 +882,7 @@ public:
       } else if (V->varnameUsed()) {
         std::string Type = V->typeName();
         OS << V->typeName();
-        if (!StringRef(Type).endswith("*"))
+        if (!StringRef(Type).ends_with("*"))
           OS << " ";
         OS << V->varname() << " = ";
       }
@@ -1680,7 +1680,7 @@ void EmitterBase::EmitBuiltinCG(raw_ostream &OS) {
       for (size_t i = 0, e = MG.ParamTypes.size(); i < e; ++i) {
         StringRef Type = MG.ParamTypes[i];
         OS << "  " << Type;
-        if (!Type.endswith("*"))
+        if (!Type.ends_with("*"))
           OS << " ";
         OS << " Param" << utostr(i) << ";\n";
       }
@@ -1833,7 +1833,7 @@ void MveEmitter::EmitHeader(raw_ostream &OS) {
         // prototype.
 
         std::string RetTypeName = Int.returnType()->cName();
-        if (!StringRef(RetTypeName).endswith("*"))
+        if (!StringRef(RetTypeName).ends_with("*"))
           RetTypeName += " ";
 
         std::vector<std::string> ArgTypeNames;
@@ -2078,7 +2078,7 @@ void CdeEmitter::EmitHeader(raw_ostream &OS) {
       // Make strings for the types involved in the function's
       // prototype.
       std::string RetTypeName = Int.returnType()->cName();
-      if (!StringRef(RetTypeName).endswith("*"))
+      if (!StringRef(RetTypeName).ends_with("*"))
         RetTypeName += " ";
 
       std::vector<std::string> ArgTypeNames;
diff --git a/clang/utils/TableGen/NeonEmitter.cpp b/clang/utils/TableGen/NeonEmitter.cpp
index 4b112972a1ec98195a91983a2040a8e4196a17a9..e5f79ba99c5c81c5ec1c121c1241992679448758 100644
--- a/clang/utils/TableGen/NeonEmitter.cpp
+++ b/clang/utils/TableGen/NeonEmitter.cpp
@@ -593,6 +593,8 @@ public:
   // Emit arm_bf16.h.inc
   void runBF16(raw_ostream &o);
 
+  void runVectorTypes(raw_ostream &o);
+
   // Emit all the __builtin prototypes used in arm_neon.h, arm_fp16.h and
   // arm_bf16.h
   void runHeader(raw_ostream &o);
@@ -2355,13 +2357,7 @@ void NeonEmitter::run(raw_ostream &OS) {
 
   OS << "#include <arm_bf16.h>\n";
 
-  // Emit NEON-specific scalar typedefs.
-  OS << "typedef float float32_t;\n";
-  OS << "typedef __fp16 float16_t;\n";
-
-  OS << "#ifdef __aarch64__\n";
-  OS << "typedef double float64_t;\n";
-  OS << "#endif\n\n";
+  OS << "#include <arm_vector_types.h>\n";
 
   // For now, signedness of polynomial types depends on target
   OS << "#ifdef __aarch64__\n";
@@ -2374,10 +2370,7 @@ void NeonEmitter::run(raw_ostream &OS) {
   OS << "typedef int16_t poly16_t;\n";
   OS << "typedef int64_t poly64_t;\n";
   OS << "#endif\n";
-
-  emitNeonTypeDefs("cQcsQsiQilQlUcQUcUsQUsUiQUiUlQUlhQhfQfdQdPcQPcPsQPsPlQPl", OS);
-
-  emitNeonTypeDefs("bQb", OS);
+  emitNeonTypeDefs("PcQPcPsQPsPlQPl", OS);
 
   OS << "#define __ai static __inline__ __attribute__((__always_inline__, "
         "__nodebug__))\n\n";
@@ -2546,6 +2539,38 @@ void NeonEmitter::runFP16(raw_ostream &OS) {
   OS << "#endif /* __ARM_FP16_H */\n";
 }
 
+void NeonEmitter::runVectorTypes(raw_ostream &OS) {
+  OS << "/*===---- arm_vector_types - ARM vector type "
+        "------===\n"
+        " *\n"
+        " *\n"
+        " * Part of the LLVM Project, under the Apache License v2.0 with LLVM "
+        "Exceptions.\n"
+        " * See https://llvm.org/LICENSE.txt for license information.\n"
+        " * SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception\n"
+        " *\n"
+        " *===-----------------------------------------------------------------"
+        "------===\n"
+        " */\n\n";
+  OS << "#if !defined(__ARM_NEON_H) && !defined(__ARM_SVE_H)\n";
+  OS << "#error \"This file should not be used standalone. Please include"
+        " arm_neon.h or arm_sve.h instead\"\n\n";
+  OS << "#endif\n";
+  OS << "#ifndef __ARM_NEON_TYPES_H\n";
+  OS << "#define __ARM_NEON_TYPES_H\n";
+  OS << "typedef float float32_t;\n";
+  OS << "typedef __fp16 float16_t;\n";
+
+  OS << "#ifdef __aarch64__\n";
+  OS << "typedef double float64_t;\n";
+  OS << "#endif\n\n";
+
+  emitNeonTypeDefs("cQcsQsiQilQlUcQUcUsQUsUiQUiUlQUlhQhfQfdQd", OS);
+
+  emitNeonTypeDefs("bQb", OS);
+  OS << "#endif // __ARM_NEON_TYPES_H\n";
+}
+
 void NeonEmitter::runBF16(raw_ostream &OS) {
   OS << "/*===---- arm_bf16.h - ARM BF16 intrinsics "
         "-----------------------------------===\n"
@@ -2640,6 +2665,10 @@ void clang::EmitNeonSema(RecordKeeper &Records, raw_ostream &OS) {
   NeonEmitter(Records).runHeader(OS);
 }
 
+void clang::EmitVectorTypes(RecordKeeper &Records, raw_ostream &OS) {
+  NeonEmitter(Records).runVectorTypes(OS);
+}
+
 void clang::EmitNeonTest(RecordKeeper &Records, raw_ostream &OS) {
   llvm_unreachable("Neon test generation no longer implemented!");
 }
diff --git a/clang/utils/TableGen/SveEmitter.cpp b/clang/utils/TableGen/SveEmitter.cpp
index b8a1fb4bab0f499d74cc1e0c2cf9128c32d8754d..9361b99506377d126da4e19ae8c071a98c9d07f7 100644
--- a/clang/utils/TableGen/SveEmitter.cpp
+++ b/clang/utils/TableGen/SveEmitter.cpp
@@ -97,6 +97,7 @@ public:
   bool isScalar() const { return NumVectors == 0; }
   bool isVector() const { return NumVectors > 0; }
   bool isScalableVector() const { return isVector() && IsScalable; }
+  bool isFixedLengthVector() const { return isVector() && !IsScalable; }
   bool isChar() const { return ElementBitwidth == 8; }
   bool isVoid() const { return Void & !Pointer; }
   bool isDefault() const { return DefaultType; }
@@ -466,7 +467,8 @@ std::string SVEType::builtin_str() const {
     return S;
   }
 
-  assert(isScalableVector() && "Unsupported type");
+  if (isFixedLengthVector())
+    return "V" + utostr(getNumElements() * NumVectors) + S;
   return "q" + utostr(getNumElements() * NumVectors) + S;
 }
 
@@ -499,7 +501,7 @@ std::string SVEType::str() const {
 
     if (!isScalarPredicate() && !isPredicateVector() && !isSvcount())
       S += utostr(ElementBitwidth);
-    if (!isScalableVector() && isVector())
+    if (isFixedLengthVector())
       S += "x" + utostr(getNumElements());
     if (NumVectors > 1)
       S += "x" + utostr(NumVectors);
@@ -610,6 +612,11 @@ void SVEType::applyModifier(char Mod) {
     Bitwidth = 16;
     ElementBitwidth = 1;
     break;
+  case '{':
+    IsScalable = false;
+    Bitwidth = 128;
+    NumVectors = 1;
+    break;
   case 's':
   case 'a':
     Bitwidth = ElementBitwidth;
@@ -1286,6 +1293,7 @@ void SVEEmitter::createHeader(raw_ostream &OS) {
   OS << "typedef __SVBfloat16_t svbfloat16_t;\n";
 
   OS << "#include <arm_bf16.h>\n";
+  OS << "#include <arm_vector_types.h>\n";
 
   OS << "typedef __SVFloat32_t svfloat32_t;\n";
   OS << "typedef __SVFloat64_t svfloat64_t;\n";
@@ -1730,4 +1738,5 @@ void EmitSmeBuiltinCG(RecordKeeper &Records, raw_ostream &OS) {
 void EmitSmeRangeChecks(RecordKeeper &Records, raw_ostream &OS) {
   SVEEmitter(Records).createSMERangeChecks(OS);
 }
+
 } // End namespace clang
diff --git a/clang/utils/TableGen/TableGen.cpp b/clang/utils/TableGen/TableGen.cpp
index 7efb6c731d3e5ee2c1f202f556d1c6eee62e4f31..3ad46b95984ec752eb033b78fb8262ad109ba5d2 100644
--- a/clang/utils/TableGen/TableGen.cpp
+++ b/clang/utils/TableGen/TableGen.cpp
@@ -73,6 +73,7 @@ enum ActionType {
   GenArmNeon,
   GenArmFP16,
   GenArmBF16,
+  GenArmVectorType,
   GenArmNeonSema,
   GenArmNeonTest,
   GenArmMveHeader,
@@ -229,6 +230,8 @@ cl::opt<ActionType> Action(
         clEnumValN(GenArmNeon, "gen-arm-neon", "Generate arm_neon.h for clang"),
         clEnumValN(GenArmFP16, "gen-arm-fp16", "Generate arm_fp16.h for clang"),
         clEnumValN(GenArmBF16, "gen-arm-bf16", "Generate arm_bf16.h for clang"),
+        clEnumValN(GenArmVectorType, "gen-arm-vector-type",
+                   "Generate arm_vector_types.h for clang"),
         clEnumValN(GenArmNeonSema, "gen-arm-neon-sema",
                    "Generate ARM NEON sema support for clang"),
         clEnumValN(GenArmNeonTest, "gen-arm-neon-test",
@@ -449,6 +452,9 @@ bool ClangTableGenMain(raw_ostream &OS, RecordKeeper &Records) {
   case GenArmFP16:
     EmitFP16(Records, OS);
     break;
+  case GenArmVectorType:
+    EmitVectorTypes(Records, OS);
+    break;
   case GenArmBF16:
     EmitBF16(Records, OS);
     break;
diff --git a/clang/utils/TableGen/TableGenBackends.h b/clang/utils/TableGen/TableGenBackends.h
index d8f447069376bca352b74dbd55df337573078562..ef255612f4b8b8537142e56cdaf70d89bddcf8aa 100644
--- a/clang/utils/TableGen/TableGenBackends.h
+++ b/clang/utils/TableGen/TableGenBackends.h
@@ -97,6 +97,7 @@ void EmitNeon(llvm::RecordKeeper &Records, llvm::raw_ostream &OS);
 void EmitFP16(llvm::RecordKeeper &Records, llvm::raw_ostream &OS);
 void EmitBF16(llvm::RecordKeeper &Records, llvm::raw_ostream &OS);
 void EmitNeonSema(llvm::RecordKeeper &Records, llvm::raw_ostream &OS);
+void EmitVectorTypes(llvm::RecordKeeper &Records, llvm::raw_ostream &OS);
 void EmitNeonTest(llvm::RecordKeeper &Records, llvm::raw_ostream &OS);
 
 void EmitSveHeader(llvm::RecordKeeper &Records, llvm::raw_ostream &OS);
diff --git a/clang/www/analyzer/faq.html b/clang/www/analyzer/faq.html
index 72ca27eb8c36b907e3306d4d3040634bf3a9d551..156d383db2f2343d26d8190b1348eff9f614dba7 100644
--- a/clang/www/analyzer/faq.html
+++ b/clang/www/analyzer/faq.html
@@ -195,12 +195,45 @@ int foo(int length) {
 
 <h4 id="suppress_issue" class="faq">Q: How can I suppress a specific analyzer warning?</h4>
 
-<p>There is currently no solid mechanism for suppressing an analyzer warning,
-although this is currently being investigated. When you encounter an analyzer
-bug/false positive, check if it's one of the issues discussed above or if the
-analyzer <a href = "annotations.html#custom_assertions" >annotations</a> can
-resolve the issue. Second, please <a href = "filing_bugs.html">report it</a> to
-help us improve user experience. As the last resort, consider using <tt>__clang_analyzer__</tt> macro
+<p>When you encounter an analyzer bug/false positive, check if it's one of the
+issues discussed above or if the analyzer
+<a href = "annotations.html#custom_assertions" >annotations</a> can
+resolve the issue by helping the static analyzer understand the code better.
+Second, please <a href = "filing_bugs.html">report it</a> to help us improve
+user experience.</p>
+
+<p>Sometimes there's really no "good" way to eliminate the issue. In such cases
+you can "silence" it directly by annotating the problematic line of code with
+the help of Clang attribute '<a href="https://clang.llvm.org/docs/AttributeReference.html#suppress">suppress</a>':
+
+<pre class="code_example">
+int foo() {
+  int *x = nullptr;
+  ...
+  <span class="code_highlight">[[clang::suppress]]</span> {
+    // all warnings in this scope are suppressed
+    int y = *x;
+  }
+
+  // null pointer dereference warning suppressed on the next line
+  <span class="code_highlight">[[clang::suppress]]</span>
+  return *x
+}
+
+int bar(bool coin_flip) {
+  // suppress all memory leak warnings about this allocation
+  <span class="code_highlight">[[clang::suppress]]</span>
+  int *result = (int *)malloc(sizeof(int));
+
+  if (coin_flip)
+    return 0;      // including this leak path
+
+  return *result;  // as well as this leak path
+}
+</pre>
+
+
+<p>You can also consider using <tt>__clang_analyzer__</tt> macro
 <a href = "faq.html#exclude_code" >described below</a>.</p>
 
 <h4 id="exclude_code" class="faq">Q: How can I selectively exclude code the analyzer examines?</h4>
diff --git a/compiler-rt/include/sanitizer/hwasan_interface.h b/compiler-rt/include/sanitizer/hwasan_interface.h
index abe310c0666948300c2c4a8f352ec60d89456870..407f488a24a617f67822b57b1e1b623f0c53426b 100644
--- a/compiler-rt/include/sanitizer/hwasan_interface.h
+++ b/compiler-rt/include/sanitizer/hwasan_interface.h
@@ -44,6 +44,10 @@ void SANITIZER_CDECL __hwasan_tag_memory(const volatile void *p,
 void *SANITIZER_CDECL __hwasan_tag_pointer(const volatile void *p,
                                            unsigned char tag);
 
+/// Get tag from the pointer.
+unsigned char SANITIZER_CDECL
+__hwasan_get_tag_from_pointer(const volatile void *p);
+
 // Set memory tag from the current SP address to the given address to zero.
 // This is meant to annotate longjmp and other non-local jumps.
 // This function needs to know the (almost) exact destination frame address;
diff --git a/compiler-rt/lib/asan/asan_fuchsia.cpp b/compiler-rt/lib/asan/asan_fuchsia.cpp
index 2b15504123bee7c95908ffb7dc0f99290d469327..12625e9d75833de9537c29cb366aabb5be30e773 100644
--- a/compiler-rt/lib/asan/asan_fuchsia.cpp
+++ b/compiler-rt/lib/asan/asan_fuchsia.cpp
@@ -240,6 +240,8 @@ void FlushUnneededASanShadowMemory(uptr p, uptr size) {
 // So this doesn't install any atexit hook like on other platforms.
 void InstallAtExitCheckLeaks() {}
 
+void InstallAtForkHandler() {}
+
 }  // namespace __asan
 
 namespace __lsan {
diff --git a/compiler-rt/lib/asan/asan_internal.h b/compiler-rt/lib/asan/asan_internal.h
index 5b97e77882cd67482e16947545f0290b81490d02..2944ebe213b5d56991a2d954c7871e6242bf1209 100644
--- a/compiler-rt/lib/asan/asan_internal.h
+++ b/compiler-rt/lib/asan/asan_internal.h
@@ -126,6 +126,7 @@ void *AsanDlSymNext(const char *sym);
 bool HandleDlopenInit();
 
 void InstallAtExitCheckLeaks();
+void InstallAtForkHandler();
 
 #define ASAN_ON_ERROR() \
   if (&__asan_on_error) \
diff --git a/compiler-rt/lib/asan/asan_posix.cpp b/compiler-rt/lib/asan/asan_posix.cpp
index e1f66641617cc14b4336ccc25d012ff1194d5772..206551b6ef910e5a806dfa33dd06af90adcaa152 100644
--- a/compiler-rt/lib/asan/asan_posix.cpp
+++ b/compiler-rt/lib/asan/asan_posix.cpp
@@ -148,6 +148,30 @@ void PlatformTSDDtor(void *tsd) {
 }
 #endif
 
+void InstallAtForkHandler() {
+  auto before = []() {
+    if (CAN_SANITIZE_LEAKS) {
+      __lsan::LockGlobal();
+    }
+    // `_lsan` functions defined regardless of `CAN_SANITIZE_LEAKS` and lock the
+    // stuff we need.
+    __lsan::LockThreads();
+    __lsan::LockAllocator();
+    StackDepotLockAll();
+  };
+  auto after = []() {
+    StackDepotUnlockAll();
+    // `_lsan` functions defined regardless of `CAN_SANITIZE_LEAKS` and unlock
+    // the stuff we need.
+    __lsan::UnlockAllocator();
+    __lsan::UnlockThreads();
+    if (CAN_SANITIZE_LEAKS) {
+      __lsan::UnlockGlobal();
+    }
+  };
+  pthread_atfork(before, after, after);
+}
+
 void InstallAtExitCheckLeaks() {
   if (CAN_SANITIZE_LEAKS) {
     if (common_flags()->detect_leaks && common_flags()->leak_check_at_exit) {
diff --git a/compiler-rt/lib/asan/asan_rtl.cpp b/compiler-rt/lib/asan/asan_rtl.cpp
index 04ecd20821fa6deea8a38fcceb85bf8e49fa0435..a61deed7382b0291c212ef4c3a0c7fe5875d8cfd 100644
--- a/compiler-rt/lib/asan/asan_rtl.cpp
+++ b/compiler-rt/lib/asan/asan_rtl.cpp
@@ -71,16 +71,16 @@ static void CheckUnwind() {
 }
 
 // -------------------------- Globals --------------------- {{{1
-static int asan_inited = 0;
-static int asan_init_is_running = 0;
+static StaticSpinMutex asan_inited_mutex;
+static atomic_uint8_t asan_inited = {0};
 
-static void SetAsanInited() { asan_inited = 1; }
-
-static void SetAsanInitIsRunning(u32 val) { asan_init_is_running = val; }
-
-bool AsanInited() { return asan_inited == 1; }
+static void SetAsanInited() {
+  atomic_store(&asan_inited, 1, memory_order_release);
+}
 
-static bool AsanInitIsRunning() { return asan_init_is_running == 1; }
+bool AsanInited() {
+  return atomic_load(&asan_inited, memory_order_acquire) == 1;
+}
 
 bool replace_intrin_cached;
 
@@ -390,12 +390,10 @@ void PrintAddressSpaceLayout() {
           kHighShadowBeg > kMidMemEnd);
 }
 
-static void AsanInitInternal() {
+static bool AsanInitInternal() {
   if (LIKELY(AsanInited()))
-    return;
+    return true;
   SanitizerToolName = "AddressSanitizer";
-  CHECK(!AsanInitIsRunning() && "ASan init calls itself!");
-  SetAsanInitIsRunning(1);
 
   CacheBinaryName();
 
@@ -408,9 +406,8 @@ static void AsanInitInternal() {
   // Stop performing init at this point if we are being loaded via
   // dlopen() and the platform supports it.
   if (SANITIZER_SUPPORTS_INIT_FOR_DLOPEN && UNLIKELY(HandleDlopenInit())) {
-    SetAsanInitIsRunning(0);
     VReport(1, "AddressSanitizer init is being performed for dlopen().\n");
-    return;
+    return false;
   }
 
   AsanCheckIncompatibleRT();
@@ -471,7 +468,6 @@ static void AsanInitInternal() {
   // should be set to 1 prior to initializing the threads.
   replace_intrin_cached = flags()->replace_intrin;
   SetAsanInited();
-  SetAsanInitIsRunning(0);
 
   if (flags()->atexit)
     Atexit(asan_atexit);
@@ -497,6 +493,8 @@ static void AsanInitInternal() {
     InstallAtExitCheckLeaks();
   }
 
+  InstallAtForkHandler();
+
 #if CAN_SANITIZE_UB
   __ubsan::InitAsPlugin();
 #endif
@@ -515,22 +513,27 @@ static void AsanInitInternal() {
   VReport(1, "AddressSanitizer Init done\n");
 
   WaitForDebugger(flags()->sleep_after_init, "after init");
+
+  return true;
 }
 
 // Initialize as requested from some part of ASan runtime library (interceptors,
 // allocator, etc).
 void AsanInitFromRtl() {
-  CHECK(!AsanInitIsRunning());
-  if (UNLIKELY(!AsanInited()))
-    AsanInitInternal();
+  if (LIKELY(AsanInited()))
+    return;
+  SpinMutexLock lock(&asan_inited_mutex);
+  AsanInitInternal();
 }
 
 bool TryAsanInitFromRtl() {
-  if (UNLIKELY(AsanInitIsRunning()))
+  if (LIKELY(AsanInited()))
+    return true;
+  if (!asan_inited_mutex.TryLock())
     return false;
-  if (UNLIKELY(!AsanInited()))
-    AsanInitInternal();
-  return true;
+  bool result = AsanInitInternal();
+  asan_inited_mutex.Unlock();
+  return result;
 }
 
 #if ASAN_DYNAMIC
@@ -603,7 +606,7 @@ static void UnpoisonFakeStack() {
 using namespace __asan;
 
 void NOINLINE __asan_handle_no_return() {
-  if (AsanInitIsRunning())
+  if (UNLIKELY(!AsanInited()))
     return;
 
   if (!PlatformUnpoisonStacks())
@@ -633,7 +636,7 @@ void NOINLINE __asan_set_death_callback(void (*callback)(void)) {
 // We use this call as a trigger to wake up ASan from deactivated state.
 void __asan_init() {
   AsanActivate();
-  AsanInitInternal();
+  AsanInitFromRtl();
 }
 
 void __asan_version_mismatch_check() {
diff --git a/compiler-rt/lib/asan/asan_win.cpp b/compiler-rt/lib/asan/asan_win.cpp
index d5a30f471e2b0d45de18820b983a15d662dfbb56..f16ce677618e4fe58d895f94fc8fb2a1eeeb93f0 100644
--- a/compiler-rt/lib/asan/asan_win.cpp
+++ b/compiler-rt/lib/asan/asan_win.cpp
@@ -203,6 +203,8 @@ void InitializePlatformInterceptors() {
 
 void InstallAtExitCheckLeaks() {}
 
+void InstallAtForkHandler() {}
+
 void AsanApplyToGlobals(globals_op_fptr op, const void *needle) {
   UNIMPLEMENTED();
 }
diff --git a/compiler-rt/lib/hwasan/hwasan.cpp b/compiler-rt/lib/hwasan/hwasan.cpp
index 2f6cb10caf1be602b8aad0fb985a056e7bd6b067..52780becbdb264394b6bbc04a607b93b361bb752 100644
--- a/compiler-rt/lib/hwasan/hwasan.cpp
+++ b/compiler-rt/lib/hwasan/hwasan.cpp
@@ -678,6 +678,8 @@ uptr __hwasan_tag_pointer(uptr p, u8 tag) {
   return AddTagToPointer(p, tag);
 }
 
+u8 __hwasan_get_tag_from_pointer(uptr p) { return GetTagFromPointer(p); }
+
 void __hwasan_handle_longjmp(const void *sp_dst) {
   uptr dst = (uptr)sp_dst;
   // HWASan does not support tagged SP.
diff --git a/compiler-rt/lib/hwasan/hwasan.h b/compiler-rt/lib/hwasan/hwasan.h
index 37ef482228511064d1e16ea8bd6055e2f0f5d582..df21375e81671fb13165f7334da1e428db9cc284 100644
--- a/compiler-rt/lib/hwasan/hwasan.h
+++ b/compiler-rt/lib/hwasan/hwasan.h
@@ -104,9 +104,9 @@ static inline void *UntagPtr(const void *tagged_ptr) {
 }
 
 static inline uptr AddTagToPointer(uptr p, tag_t tag) {
-  return InTaggableRegion(p)
-             ? ((p & ~kAddressTagMask) | ((uptr)tag << kAddressTagShift))
-             : p;
+  return InTaggableRegion(p) ? ((p & ~kAddressTagMask) |
+                                ((uptr)(tag & kTagMask) << kAddressTagShift))
+                             : p;
 }
 
 namespace __hwasan {
diff --git a/compiler-rt/lib/hwasan/hwasan_interface_internal.h b/compiler-rt/lib/hwasan/hwasan_interface_internal.h
index e7804cc4903343238511e993d4234fa6ffbfa2bc..8f2f77dad917d27fdf77559ac9a4b25e51a42932 100644
--- a/compiler-rt/lib/hwasan/hwasan_interface_internal.h
+++ b/compiler-rt/lib/hwasan/hwasan_interface_internal.h
@@ -160,6 +160,9 @@ void __hwasan_tag_memory(uptr p, u8 tag, uptr sz);
 SANITIZER_INTERFACE_ATTRIBUTE
 uptr __hwasan_tag_pointer(uptr p, u8 tag);
 
+SANITIZER_INTERFACE_ATTRIBUTE
+u8 __hwasan_get_tag_from_pointer(uptr p);
+
 SANITIZER_INTERFACE_ATTRIBUTE
 void __hwasan_tag_mismatch(uptr addr, u8 ts);
 
diff --git a/compiler-rt/lib/hwasan/hwasan_linux.cpp b/compiler-rt/lib/hwasan/hwasan_linux.cpp
index f01fa427641347b8a44de7da8260ecaf3b12a80e..3271a955e7ed102de886e750c631a62597f9f555 100644
--- a/compiler-rt/lib/hwasan/hwasan_linux.cpp
+++ b/compiler-rt/lib/hwasan/hwasan_linux.cpp
@@ -523,12 +523,24 @@ uptr TagMemoryAligned(uptr p, uptr size, tag_t tag) {
 
 void HwasanInstallAtForkHandler() {
   auto before = []() {
-    HwasanAllocatorLock();
+    if (CAN_SANITIZE_LEAKS) {
+      __lsan::LockGlobal();
+    }
+    // `_lsan` functions defined regardless of `CAN_SANITIZE_LEAKS` and lock the
+    // stuff we need.
+    __lsan::LockThreads();
+    __lsan::LockAllocator();
     StackDepotLockAll();
   };
   auto after = []() {
     StackDepotUnlockAll();
-    HwasanAllocatorUnlock();
+    // `_lsan` functions defined regardless of `CAN_SANITIZE_LEAKS` and unlock
+    // the stuff we need.
+    __lsan::UnlockAllocator();
+    __lsan::UnlockThreads();
+    if (CAN_SANITIZE_LEAKS) {
+      __lsan::UnlockGlobal();
+    }
   };
   pthread_atfork(before, after, after);
 }
diff --git a/compiler-rt/lib/hwasan/hwasan_thread.cpp b/compiler-rt/lib/hwasan/hwasan_thread.cpp
index ce36547580e6e60ccdaaa6bfb697d50c47ebac22..3e14a718513d7f3d1c26cb2a59bfd6168a16930c 100644
--- a/compiler-rt/lib/hwasan/hwasan_thread.cpp
+++ b/compiler-rt/lib/hwasan/hwasan_thread.cpp
@@ -68,6 +68,7 @@ void Thread::Init(uptr stack_buffer_start, uptr stack_buffer_size,
     }
     Print("Creating  : ");
   }
+  ClearShadowForThreadStackAndTLS();
 }
 
 void Thread::InitStackRingBuffer(uptr stack_buffer_start,
diff --git a/compiler-rt/lib/lsan/lsan.cpp b/compiler-rt/lib/lsan/lsan.cpp
index 6b223603c6a79c4f5c1645386f3ad98edadc6946..7a27b600f203f7f9e397cfebd7b69c302956c8cf 100644
--- a/compiler-rt/lib/lsan/lsan.cpp
+++ b/compiler-rt/lib/lsan/lsan.cpp
@@ -101,6 +101,7 @@ extern "C" void __lsan_init() {
   InstallDeadlySignalHandlers(LsanOnDeadlySignal);
   InitializeMainThread();
   InstallAtExitCheckLeaks();
+  InstallAtForkHandler();
 
   InitializeCoverage(common_flags()->coverage, common_flags()->coverage_dir);
 
diff --git a/compiler-rt/lib/lsan/lsan.h b/compiler-rt/lib/lsan/lsan.h
index 757edec8e104f906206520ce6207d99b6a02cd06..0074ad5308785cf875f2dde66c5269f1e0612dac 100644
--- a/compiler-rt/lib/lsan/lsan.h
+++ b/compiler-rt/lib/lsan/lsan.h
@@ -40,6 +40,7 @@ void InitializeInterceptors();
 void ReplaceSystemMalloc();
 void LsanOnDeadlySignal(int signo, void *siginfo, void *context);
 void InstallAtExitCheckLeaks();
+void InstallAtForkHandler();
 
 #define ENSURE_LSAN_INITED        \
   do {                            \
diff --git a/compiler-rt/lib/lsan/lsan_common.cpp b/compiler-rt/lib/lsan/lsan_common.cpp
index 8b1af5b629fbce10edbaaf7e23092b4d324a1757..e24839c984b346e087a511d4c1311acfa5a3e20d 100644
--- a/compiler-rt/lib/lsan/lsan_common.cpp
+++ b/compiler-rt/lib/lsan/lsan_common.cpp
@@ -42,6 +42,9 @@ namespace __lsan {
 // also to protect the global list of root regions.
 static Mutex global_mutex;
 
+void LockGlobal() SANITIZER_ACQUIRE(global_mutex) { global_mutex.Lock(); }
+void UnlockGlobal() SANITIZER_RELEASE(global_mutex) { global_mutex.Unlock(); }
+
 Flags lsan_flags;
 
 void DisableCounterUnderflow() {
diff --git a/compiler-rt/lib/lsan/lsan_common.h b/compiler-rt/lib/lsan/lsan_common.h
index d3e768363e93b96e9162ea2ea165752cd231471e..c598b62105873e36bde865f71f2c510cbbff431b 100644
--- a/compiler-rt/lib/lsan/lsan_common.h
+++ b/compiler-rt/lib/lsan/lsan_common.h
@@ -120,6 +120,10 @@ void GetRunningThreadsLocked(InternalMmapVector<tid_t> *threads);
 void LockAllocator();
 void UnlockAllocator();
 
+// Lock/unlock global mutext.
+void LockGlobal();
+void UnlockGlobal();
+
 // Returns the address range occupied by the global allocator object.
 void GetAllocatorGlobalRange(uptr *begin, uptr *end);
 // If p points into a chunk that has been allocated to the user, returns its
diff --git a/compiler-rt/lib/lsan/lsan_fuchsia.cpp b/compiler-rt/lib/lsan/lsan_fuchsia.cpp
index 4edac9757a9c4927649acd1e15ecf8f664566aa0..ba59bc9b71e3325d28802ef262b55ae1fb14a1cd 100644
--- a/compiler-rt/lib/lsan/lsan_fuchsia.cpp
+++ b/compiler-rt/lib/lsan/lsan_fuchsia.cpp
@@ -80,6 +80,7 @@ void GetAllThreadAllocatorCachesLocked(InternalMmapVector<uptr> *caches) {
 // On Fuchsia, leak detection is done by a special hook after atexit hooks.
 // So this doesn't install any atexit hook like on other platforms.
 void InstallAtExitCheckLeaks() {}
+void InstallAtForkHandler() {}
 
 // ASan defines this to check its `halt_on_error` flag.
 bool UseExitcodeOnLeak() { return true; }
diff --git a/compiler-rt/lib/lsan/lsan_posix.cpp b/compiler-rt/lib/lsan/lsan_posix.cpp
index d99e1cc0105ef768487d44e3ef6c954be358452a..3677f0141a2f02f3a2048185bac2e371adfbb2e0 100644
--- a/compiler-rt/lib/lsan/lsan_posix.cpp
+++ b/compiler-rt/lib/lsan/lsan_posix.cpp
@@ -14,11 +14,13 @@
 #include "sanitizer_common/sanitizer_platform.h"
 
 #if SANITIZER_POSIX
-#include "lsan.h"
-#include "lsan_allocator.h"
-#include "lsan_thread.h"
-#include "sanitizer_common/sanitizer_stacktrace.h"
-#include "sanitizer_common/sanitizer_tls_get_addr.h"
+#  include <pthread.h>
+
+#  include "lsan.h"
+#  include "lsan_allocator.h"
+#  include "lsan_thread.h"
+#  include "sanitizer_common/sanitizer_stacktrace.h"
+#  include "sanitizer_common/sanitizer_tls_get_addr.h"
 
 namespace __lsan {
 
@@ -98,6 +100,22 @@ void InstallAtExitCheckLeaks() {
     Atexit(DoLeakCheck);
 }
 
+void InstallAtForkHandler() {
+  auto before = []() {
+    LockGlobal();
+    LockThreads();
+    LockAllocator();
+    StackDepotLockAll();
+  };
+  auto after = []() {
+    StackDepotUnlockAll();
+    UnlockAllocator();
+    UnlockThreads();
+    UnlockGlobal();
+  };
+  pthread_atfork(before, after, after);
+}
+
 }  // namespace __lsan
 
 #endif  // SANITIZER_POSIX
diff --git a/compiler-rt/lib/msan/msan.cpp b/compiler-rt/lib/msan/msan.cpp
index c4f47dea1104321198fab79f4e6d5a47d5bd0732..3cdf10c149902c276282ff684fb1e864290dd3bd 100644
--- a/compiler-rt/lib/msan/msan.cpp
+++ b/compiler-rt/lib/msan/msan.cpp
@@ -449,6 +449,7 @@ void __msan_init() {
   __sanitizer_set_report_path(common_flags()->log_path);
 
   InitializeInterceptors();
+  InstallAtForkHandler();
   CheckASLR();
   InitTlsSize();
   InstallDeadlySignalHandlers(MsanOnDeadlySignal);
diff --git a/compiler-rt/lib/msan/msan.h b/compiler-rt/lib/msan/msan.h
index b3a9c641b4fb298c7ed60488081b46ddbe560657..25fa2212bdadd31769a697ea1d5d8b55e2ed9287 100644
--- a/compiler-rt/lib/msan/msan.h
+++ b/compiler-rt/lib/msan/msan.h
@@ -336,6 +336,8 @@ void *MsanTSDGet();
 void MsanTSDSet(void *tsd);
 void MsanTSDDtor(void *tsd);
 
+void InstallAtForkHandler();
+
 }  // namespace __msan
 
 #endif  // MSAN_H
diff --git a/compiler-rt/lib/msan/msan_allocator.cpp b/compiler-rt/lib/msan/msan_allocator.cpp
index c3b0f8512e82d83755ddff431980a9e7db520a0d..72a7f980d39fb03c6a5148adcccfab1855621d72 100644
--- a/compiler-rt/lib/msan/msan_allocator.cpp
+++ b/compiler-rt/lib/msan/msan_allocator.cpp
@@ -159,6 +159,10 @@ void MsanAllocatorInit() {
     max_malloc_size = kMaxAllowedMallocSize;
 }
 
+void LockAllocator() { allocator.ForceLock(); }
+
+void UnlockAllocator() { allocator.ForceUnlock(); }
+
 AllocatorCache *GetAllocatorCache(MsanThreadLocalMallocStorage *ms) {
   CHECK(ms);
   CHECK_LE(sizeof(AllocatorCache), sizeof(ms->allocator_cache));
diff --git a/compiler-rt/lib/msan/msan_allocator.h b/compiler-rt/lib/msan/msan_allocator.h
index 364331d964068e562379a141833955eb2915fe16..c2a38a401f3b6b02383db3da5045899b3557418c 100644
--- a/compiler-rt/lib/msan/msan_allocator.h
+++ b/compiler-rt/lib/msan/msan_allocator.h
@@ -28,5 +28,8 @@ struct MsanThreadLocalMallocStorage {
   MsanThreadLocalMallocStorage() {}
 };
 
+void LockAllocator();
+void UnlockAllocator();
+
 } // namespace __msan
 #endif // MSAN_ALLOCATOR_H
diff --git a/compiler-rt/lib/msan/msan_interceptors.cpp b/compiler-rt/lib/msan/msan_interceptors.cpp
index c2d740e7762b4bfdb0e012e340384667f606e229..2c9f2c01e14b061be12e4effabfeb266d91bc875 100644
--- a/compiler-rt/lib/msan/msan_interceptors.cpp
+++ b/compiler-rt/lib/msan/msan_interceptors.cpp
@@ -1326,24 +1326,6 @@ static int setup_at_exit_wrapper(void(*f)(), void *arg, void *dso) {
   return res;
 }
 
-static void BeforeFork() {
-  StackDepotLockAll();
-  ChainedOriginDepotLockAll();
-}
-
-static void AfterFork() {
-  ChainedOriginDepotUnlockAll();
-  StackDepotUnlockAll();
-}
-
-INTERCEPTOR(int, fork, void) {
-  ENSURE_MSAN_INITED();
-  BeforeFork();
-  int pid = REAL(fork)();
-  AfterFork();
-  return pid;
-}
-
 // NetBSD ships with openpty(3) in -lutil, that needs to be prebuilt explicitly
 // with MSan.
 #if SANITIZER_LINUX
@@ -1933,7 +1915,6 @@ void InitializeInterceptors() {
   INTERCEPT_FUNCTION(atexit);
   INTERCEPT_FUNCTION(__cxa_atexit);
   INTERCEPT_FUNCTION(shmat);
-  INTERCEPT_FUNCTION(fork);
   MSAN_MAYBE_INTERCEPT_OPENPTY;
   MSAN_MAYBE_INTERCEPT_FORKPTY;
 
diff --git a/compiler-rt/lib/msan/msan_linux.cpp b/compiler-rt/lib/msan/msan_linux.cpp
index bced00ba242822414018b39859c92dcbf4406ce6..04af6f4b27ac8921c2fd52a9b6b36a0f25573366 100644
--- a/compiler-rt/lib/msan/msan_linux.cpp
+++ b/compiler-rt/lib/msan/msan_linux.cpp
@@ -14,23 +14,25 @@
 #include "sanitizer_common/sanitizer_platform.h"
 #if SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD
 
-#include "msan.h"
-#include "msan_report.h"
-#include "msan_thread.h"
-
-#include <elf.h>
-#include <link.h>
-#include <pthread.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <signal.h>
-#include <unistd.h>
-#include <unwind.h>
-#include <sys/time.h>
-#include <sys/resource.h>
-
-#include "sanitizer_common/sanitizer_common.h"
-#include "sanitizer_common/sanitizer_procmaps.h"
+#  include <elf.h>
+#  include <link.h>
+#  include <pthread.h>
+#  include <signal.h>
+#  include <stdio.h>
+#  include <stdlib.h>
+#  include <sys/resource.h>
+#  include <sys/time.h>
+#  include <unistd.h>
+#  include <unwind.h>
+
+#  include "msan.h"
+#  include "msan_allocator.h"
+#  include "msan_chained_origin_depot.h"
+#  include "msan_report.h"
+#  include "msan_thread.h"
+#  include "sanitizer_common/sanitizer_common.h"
+#  include "sanitizer_common/sanitizer_procmaps.h"
+#  include "sanitizer_common/sanitizer_stackdepot.h"
 
 namespace __msan {
 
@@ -256,6 +258,22 @@ void MsanTSDDtor(void *tsd) {
 }
 #endif
 
+void InstallAtForkHandler() {
+  auto before = []() {
+    // Usually we lock ThreadRegistry, but msan does not have one.
+    LockAllocator();
+    StackDepotLockAll();
+    ChainedOriginDepotLockAll();
+  };
+  auto after = []() {
+    ChainedOriginDepotUnlockAll();
+    StackDepotUnlockAll();
+    UnlockAllocator();
+    // Usually we unlock ThreadRegistry, but msan does not have one.
+  };
+  pthread_atfork(before, after, after);
+}
+
 } // namespace __msan
 
 #endif // SANITIZER_FREEBSD || SANITIZER_LINUX || SANITIZER_NETBSD
diff --git a/compiler-rt/test/hwasan/TestCases/tag-ptr.cpp b/compiler-rt/test/hwasan/TestCases/tag-ptr.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..2f00e7913bb155d8ad3c835c1004ec4ff154697a
--- /dev/null
+++ b/compiler-rt/test/hwasan/TestCases/tag-ptr.cpp
@@ -0,0 +1,24 @@
+// RUN: %clangxx_hwasan -O0 %s -o %t && %run %t
+
+#include <assert.h>
+#include <memory>
+#include <sanitizer/hwasan_interface.h>
+#include <set>
+#include <stdio.h>
+
+int main() {
+  auto p = std::make_unique<char>();
+  std::set<void *> ptrs;
+  for (unsigned i = 0;; ++i) {
+    void *ptr = __hwasan_tag_pointer(p.get(), i);
+    if (!ptrs.insert(ptr).second)
+      break;
+    fprintf(stderr, "%p, %u, %u\n", ptr, i, __hwasan_get_tag_from_pointer(ptr));
+    assert(__hwasan_get_tag_from_pointer(ptr) == i);
+  }
+#ifdef __x86_64__
+  assert(ptrs.size() == 8);
+#else
+  assert(ptrs.size() == 256);
+#endif
+}
diff --git a/compiler-rt/test/sanitizer_common/TestCases/Posix/fork_threaded.c b/compiler-rt/test/sanitizer_common/TestCases/Posix/fork_threaded.c
new file mode 100644
index 0000000000000000000000000000000000000000..1b4a0ad6140db40f8bf94a3c3e6e51d0e528cfd2
--- /dev/null
+++ b/compiler-rt/test/sanitizer_common/TestCases/Posix/fork_threaded.c
@@ -0,0 +1,93 @@
+// RUN: %clang -O0 %s -o %t && %env_tool_opts=die_after_fork=0 %run %t
+
+// The test uses pthread barriers which are not available on Darwin.
+// UNSUPPORTED: darwin
+
+// Forking in multithread environment is unsupported. However we already have
+// some workarounds, and will add more, so this is the test.
+// The test try to check two things:
+//  1. Internal mutexes used by `inparent` thread do not deadlock `inchild`
+//     thread.
+//  2. Stack poisoned by `inparent` is not poisoned in `inchild` thread.
+
+#include <assert.h>
+#include <pthread.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <unistd.h>
+
+#include "sanitizer_common/sanitizer_specific.h"
+
+static const size_t kBufferSize = 1 << 20;
+
+pthread_barrier_t bar;
+
+// Without appropriate workarounds this code can cause the forked process to
+// start with locked internal mutexes.
+void ShouldNotDeadlock() {
+  // Don't bother with leaks, we try to trigger allocator or lsan deadlock.
+  __lsan_disable();
+  void *volatile p = malloc(10);
+  __lsan_do_recoverable_leak_check();
+  free(p);
+  __lsan_enable();
+}
+
+// Prevent stack buffer cleanup by instrumentation.
+#define NOSAN __attribute__((no_sanitize("address", "hwaddress", "memory")))
+
+NOSAN static void *inparent(void *arg) {
+  char t[kBufferSize];
+  make_mem_bad(t, sizeof(t));
+
+  pthread_barrier_wait(&bar);
+
+  for (;;)
+    ShouldNotDeadlock();
+
+  return 0;
+}
+
+NOSAN static void *inchild(void *arg) {
+  char t[kBufferSize];
+  check_mem_is_good(t, sizeof(t));
+  ShouldNotDeadlock();
+  return 0;
+}
+
+int main(void) {
+#if __has_feature(hwaddress_sanitizer)
+  __hwasan_enable_allocator_tagging();
+#endif
+
+  pid_t pid;
+
+  pthread_barrier_init(&bar, NULL, 2);
+  pthread_t thread_id;
+  while (pthread_create(&thread_id, 0, &inparent, 0) != 0) {
+  }
+  pthread_barrier_wait(&bar);
+
+  pid = fork();
+  switch (pid) {
+  case -1:
+    perror("fork");
+    return -1;
+  case 0:
+    while (pthread_create(&thread_id, 0, &inchild, 0) != 0) {
+    }
+    break;
+  default: {
+    int status;
+    while (waitpid(-1, &status, __WALL) != pid) {
+    }
+    assert(WIFEXITED(status) && WEXITSTATUS(status) == 0);
+    break;
+  }
+  }
+
+  return 0;
+}
diff --git a/compiler-rt/test/sanitizer_common/sanitizer_specific.h b/compiler-rt/test/sanitizer_common/sanitizer_specific.h
index 1a802020cfd66836d7dec9aebd46f9c46ba9885f..99a4dd98c614a1eaed0c979606f5e8d3f69f08f0 100644
--- a/compiler-rt/test/sanitizer_common/sanitizer_specific.h
+++ b/compiler-rt/test/sanitizer_common/sanitizer_specific.h
@@ -1,6 +1,12 @@
 #ifndef __SANITIZER_COMMON_SANITIZER_SPECIFIC_H__
 #define __SANITIZER_COMMON_SANITIZER_SPECIFIC_H__
 
+#include <sanitizer/lsan_interface.h>
+
+__attribute__((weak)) int __lsan_do_recoverable_leak_check() { return 0; }
+__attribute__((weak)) void __lsan_disable(void) {}
+__attribute__((weak)) void __lsan_enable(void) {}
+
 #ifndef __has_feature
 #  define __has_feature(x) 0
 #endif
@@ -10,6 +16,8 @@
 static void check_mem_is_good(void *p, size_t s) {
   __msan_check_mem_is_initialized(p, s);
 }
+static void make_mem_good(void *p, size_t s) { __msan_unpoison(p, s); }
+static void make_mem_bad(void *p, size_t s) { __msan_poison(p, s); }
 #elif __has_feature(address_sanitizer)
 #  include <sanitizer/asan_interface.h>
 #  include <stdlib.h>
@@ -17,8 +25,40 @@ static void check_mem_is_good(void *p, size_t s) {
   if (__asan_region_is_poisoned(p, s))
     abort();
 }
+static void make_mem_good(void *p, size_t s) {
+  __asan_unpoison_memory_region(p, s);
+}
+static void make_mem_bad(void *p, size_t s) {
+  __asan_poison_memory_region(p, s);
+}
+#elif __has_feature(hwaddress_sanitizer)
+#  include <sanitizer/hwasan_interface.h>
+#  include <stdlib.h>
+static void check_mem_is_good(void *p, size_t s) {
+  if (__hwasan_test_shadow(p, s) != -1)
+    abort();
+}
+static void make_mem_good(void *p, size_t s) {
+  __hwasan_tag_memory(p, __hwasan_get_tag_from_pointer(p), s);
+}
+static void make_mem_bad(void *p, size_t s) {
+  uint8_t tag = ~__hwasan_get_tag_from_pointer(p);
+  if (!tag) {
+    // Nothing wrong with tag zero, but non-zero tags help to detect never
+    // tagged memory.
+    tag = 1;
+  }
+  __hwasan_tag_memory(p, tag, s);
+  // With misaligned `p` or short granules we can't guarantee tag mismatch.
+  if (__hwasan_test_shadow(p, s) != 0)
+    abort();
+  if (s > 1 && __hwasan_test_shadow(((char *)p) + s - 1, 1) != 0)
+    abort();
+}
 #else
 static void check_mem_is_good(void *p, size_t s) {}
+static void make_mem_good(void *p, size_t s) {}
+static void make_mem_bad(void *p, size_t s) {}
 #endif
 
-#endif // __SANITIZER_COMMON_SANITIZER_SPECIFIC_H__
\ No newline at end of file
+#endif // __SANITIZER_COMMON_SANITIZER_SPECIFIC_H__
diff --git a/flang/include/flang/Lower/AbstractConverter.h b/flang/include/flang/Lower/AbstractConverter.h
index 980fde881373249d9127c8e965e329ccea964194..b91303387f3d710d6bf4159f0ae433bca7cf261c 100644
--- a/flang/include/flang/Lower/AbstractConverter.h
+++ b/flang/include/flang/Lower/AbstractConverter.h
@@ -280,6 +280,10 @@ public:
   // Miscellaneous
   //===--------------------------------------------------------------------===//
 
+  /// Generate IR for Evaluation \p eval.
+  virtual void genEval(pft::Evaluation &eval,
+                       bool unstructuredContext = true) = 0;
+
   /// Return options controlling lowering behavior.
   const Fortran::lower::LoweringOptions &getLoweringOptions() const {
     return loweringOptions;
diff --git a/flang/include/flang/Optimizer/Dialect/FIRType.h b/flang/include/flang/Optimizer/Dialect/FIRType.h
index 2abcc6547bbabf6343272695b7bd257f9fa38742..a79c67dfe6de862be6bd704d445e1757cd0781f8 100644
--- a/flang/include/flang/Optimizer/Dialect/FIRType.h
+++ b/flang/include/flang/Optimizer/Dialect/FIRType.h
@@ -118,8 +118,8 @@ inline bool isa_derived(mlir::Type t) { return t.isa<fir::RecordType>(); }
 /// Is `t` type(c_ptr) or type(c_funptr)?
 inline bool isa_builtin_cptr_type(mlir::Type t) {
   if (auto recTy = t.dyn_cast_or_null<fir::RecordType>())
-    return recTy.getName().endswith("T__builtin_c_ptr") ||
-           recTy.getName().endswith("T__builtin_c_funptr");
+    return recTy.getName().ends_with("T__builtin_c_ptr") ||
+           recTy.getName().ends_with("T__builtin_c_funptr");
   return false;
 }
 
diff --git a/flang/lib/Frontend/CompilerInstance.cpp b/flang/lib/Frontend/CompilerInstance.cpp
index a6b8f1a9d29ee4746ebf67c39bb3d4f154178b1f..555ac91f6dc7c077cdfec74390b0b4e631a9ef6c 100644
--- a/flang/lib/Frontend/CompilerInstance.cpp
+++ b/flang/lib/Frontend/CompilerInstance.cpp
@@ -257,7 +257,7 @@ getExplicitAndImplicitNVPTXTargetFeatures(clang::DiagnosticsEngine &diags,
     llvm::StringRef userKeyString(llvm::StringRef(userFeature).drop_front(1));
     implicitFeaturesMap[userKeyString.str()] = (userFeature[0] == '+');
     // Check if the user provided a PTX version
-    if (userKeyString.startswith("ptx"))
+    if (userKeyString.starts_with("ptx"))
       ptxVer = true;
   }
 
diff --git a/flang/lib/FrontendTool/ExecuteCompilerInvocation.cpp b/flang/lib/FrontendTool/ExecuteCompilerInvocation.cpp
index bc09dec17b7ae341285d0278dcffc19be1ae8786..4cad640562c619067002a9a6bcff3057b5873613 100644
--- a/flang/lib/FrontendTool/ExecuteCompilerInvocation.cpp
+++ b/flang/lib/FrontendTool/ExecuteCompilerInvocation.cpp
@@ -131,7 +131,7 @@ updateDiagEngineForOptRemarks(clang::DiagnosticsEngine &diagsEng,
 
     // Check to see if this opt starts with "no-", if so, this is a
     // negative form of the option.
-    bool isPositive = !remarkOpt.startswith("no-");
+    bool isPositive = !remarkOpt.starts_with("no-");
     if (!isPositive)
       remarkOpt = remarkOpt.substr(3);
 
diff --git a/flang/lib/Lower/Bridge.cpp b/flang/lib/Lower/Bridge.cpp
index 7e64adc3c144c9647510220c746315b91d93ad32..6ca910d26967421371f492fcb7b2473484dccea8 100644
--- a/flang/lib/Lower/Bridge.cpp
+++ b/flang/lib/Lower/Bridge.cpp
@@ -839,6 +839,11 @@ public:
     }
   }
 
+  void genEval(Fortran::lower::pft::Evaluation &eval,
+               bool unstructuredContext) override final {
+    genFIR(eval, unstructuredContext);
+  }
+
   //===--------------------------------------------------------------------===//
   // Utility methods
   //===--------------------------------------------------------------------===//
diff --git a/flang/lib/Lower/IO.cpp b/flang/lib/Lower/IO.cpp
index 4186d6158fb1d040aa994f32aba22ee4a4ae0277..a60ca92a8733e360071b4a8a4eae1cc69f70ab06 100644
--- a/flang/lib/Lower/IO.cpp
+++ b/flang/lib/Lower/IO.cpp
@@ -641,7 +641,8 @@ static void genNamelistIO(Fortran::lower::AbstractConverter &converter,
   mlir::Location loc = converter.getCurrentLocation();
   makeNextConditionalOn(builder, loc, checkResult, ok);
   mlir::Type argType = funcOp.getFunctionType().getInput(1);
-  mlir::Value groupAddr = getNamelistGroup(converter, symbol, stmtCtx);
+  mlir::Value groupAddr =
+      getNamelistGroup(converter, symbol.GetUltimate(), stmtCtx);
   groupAddr = builder.createConvert(loc, argType, groupAddr);
   llvm::SmallVector<mlir::Value> args = {cookie, groupAddr};
   ok = builder.create<fir::CallOp>(loc, funcOp, args).getResult(0);
diff --git a/flang/lib/Lower/OpenMP.cpp b/flang/lib/Lower/OpenMP.cpp
index eeba87fcd151169ac0f2abdf5ea2b16ffa6dd1b1..12b8ea82884d9d1f16b4808eb98762722bda4314 100644
--- a/flang/lib/Lower/OpenMP.cpp
+++ b/flang/lib/Lower/OpenMP.cpp
@@ -2117,12 +2117,8 @@ static void createBodyOfOp(
     for (const Fortran::semantics::Symbol *arg : args)
       loopVarTypeSize = std::max(loopVarTypeSize, arg->GetUltimate().size());
     mlir::Type loopVarType = getLoopVarType(converter, loopVarTypeSize);
-    llvm::SmallVector<mlir::Type> tiv;
-    llvm::SmallVector<mlir::Location> locs;
-    for (int i = 0; i < (int)args.size(); i++) {
-      tiv.push_back(loopVarType);
-      locs.push_back(loc);
-    }
+    llvm::SmallVector<mlir::Type> tiv(args.size(), loopVarType);
+    llvm::SmallVector<mlir::Location> locs(args.size(), loc);
     firOpBuilder.createBlock(&op.getRegion(), {}, tiv, locs);
     int argIndex = 0;
     // The argument is not currently in memory, so make a temporary for the
diff --git a/flang/lib/Lower/PFTBuilder.cpp b/flang/lib/Lower/PFTBuilder.cpp
index 32ed539c775b827c1bec544720017c3adbe60ab8..8e224c17edad1955d4a059957c46b565601024d4 100644
--- a/flang/lib/Lower/PFTBuilder.cpp
+++ b/flang/lib/Lower/PFTBuilder.cpp
@@ -149,22 +149,22 @@ public:
     // Modules IEEE_FEATURES, IEEE_EXCEPTIONS, and IEEE_ARITHMETIC get common
     // declarations from several __fortran_... support module files.
     llvm::StringRef modName = toStringRef(modSym.name());
-    if (!modName.startswith("ieee_") && !modName.startswith("__fortran_"))
+    if (!modName.starts_with("ieee_") && !modName.starts_with("__fortran_"))
       return;
     llvm::StringRef procName = toStringRef(procSym.name());
-    if (!procName.startswith("ieee_"))
+    if (!procName.starts_with("ieee_"))
       return;
     lower::pft::FunctionLikeUnit *proc =
         evaluationListStack.back()->back().getOwningProcedure();
     proc->hasIeeeAccess = true;
-    if (!procName.startswith("ieee_set_"))
+    if (!procName.starts_with("ieee_set_"))
       return;
-    if (procName.startswith("ieee_set_modes_") ||
-        procName.startswith("ieee_set_status_"))
+    if (procName.starts_with("ieee_set_modes_") ||
+        procName.starts_with("ieee_set_status_"))
       proc->mayModifyHaltingMode = proc->mayModifyRoundingMode = true;
-    else if (procName.startswith("ieee_set_halting_mode_"))
+    else if (procName.starts_with("ieee_set_halting_mode_"))
       proc->mayModifyHaltingMode = true;
-    else if (procName.startswith("ieee_set_rounding_mode_"))
+    else if (procName.starts_with("ieee_set_rounding_mode_"))
       proc->mayModifyRoundingMode = true;
   }
 
diff --git a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
index 45c75fc5dd28bc0534daefff90fd721fde6dee92..ff5dbff04360a034d4ea28eef7eda31c83e5de3d 100644
--- a/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
+++ b/flang/lib/Optimizer/Builder/IntrinsicCall.cpp
@@ -1418,13 +1418,13 @@ mlir::Value toValue(const fir::ExtendedValue &val, fir::FirOpBuilder &builder,
 //===----------------------------------------------------------------------===//
 
 static bool isIntrinsicModuleProcedure(llvm::StringRef name) {
-  return name.startswith("c_") || name.startswith("compiler_") ||
-         name.startswith("ieee_") || name.startswith("__ppc_");
+  return name.starts_with("c_") || name.starts_with("compiler_") ||
+         name.starts_with("ieee_") || name.starts_with("__ppc_");
 }
 
 static bool isCoarrayIntrinsic(llvm::StringRef name) {
-  return name.startswith("atomic_") || name.startswith("co_") ||
-         name.contains("image") || name.endswith("cobound") ||
+  return name.starts_with("atomic_") || name.starts_with("co_") ||
+         name.contains("image") || name.ends_with("cobound") ||
          name.equals("team_number");
 }
 
@@ -1433,7 +1433,7 @@ static bool isCoarrayIntrinsic(llvm::StringRef name) {
 /// {_[ail]?[0-9]+}*, such as _1 or _a4.
 llvm::StringRef genericName(llvm::StringRef specificName) {
   const std::string builtin = "__builtin_";
-  llvm::StringRef name = specificName.startswith(builtin)
+  llvm::StringRef name = specificName.starts_with(builtin)
                              ? specificName.drop_front(builtin.size())
                              : specificName;
   size_t size = name.size();
diff --git a/flang/lib/Optimizer/Dialect/Support/KindMapping.cpp b/flang/lib/Optimizer/Dialect/Support/KindMapping.cpp
index c71e3939dc31d28b0704d230fb66d346e18b7560..bcb112186aeefa7af868dda11e1b0e60e64ad269 100644
--- a/flang/lib/Optimizer/Dialect/Support/KindMapping.cpp
+++ b/flang/lib/Optimizer/Dialect/Support/KindMapping.cpp
@@ -180,7 +180,7 @@ static MatchResult parseInt(unsigned &result, const char *&ptr,
 static mlir::LogicalResult matchString(const char *&ptr, const char *endPtr,
                                        llvm::StringRef literal) {
   llvm::StringRef s(ptr, endPtr - ptr);
-  if (s.startswith(literal)) {
+  if (s.starts_with(literal)) {
     ptr += literal.size();
     return mlir::success();
   }
diff --git a/flang/lib/Optimizer/Support/InternalNames.cpp b/flang/lib/Optimizer/Support/InternalNames.cpp
index 6138c1f425d62cdc4de1558d7cdef8411c4b6827..d99245f0a012e6664b3265a12729dd6a31665418 100644
--- a/flang/lib/Optimizer/Support/InternalNames.cpp
+++ b/flang/lib/Optimizer/Support/InternalNames.cpp
@@ -240,7 +240,7 @@ llvm::StringRef fir::NameUniquer::doProgramEntry() {
 
 std::pair<fir::NameUniquer::NameKind, fir::NameUniquer::DeconstructedName>
 fir::NameUniquer::deconstruct(llvm::StringRef uniq) {
-  if (uniq.startswith("_Q")) {
+  if (uniq.starts_with("_Q")) {
     llvm::SmallVector<std::string> modules;
     llvm::SmallVector<std::string> procs;
     std::int64_t blockId = 0;
diff --git a/flang/lib/Optimizer/Transforms/SimplifyIntrinsics.cpp b/flang/lib/Optimizer/Transforms/SimplifyIntrinsics.cpp
index 3eddb9e61ae3b3df4aad127678795b2eff8bc03c..8ecf7fb44f15d049b9ed975a71cacc143d0eca26 100644
--- a/flang/lib/Optimizer/Transforms/SimplifyIntrinsics.cpp
+++ b/flang/lib/Optimizer/Transforms/SimplifyIntrinsics.cpp
@@ -1279,11 +1279,11 @@ void SimplifyIntrinsicsPass::runOnOperation() {
         // RTNAME(Sum<T>)(const Descriptor &x, const char *source, int line,
         //                int dim, const Descriptor *mask)
         //
-        if (funcName.startswith(RTNAME_STRING(Sum))) {
+        if (funcName.starts_with(RTNAME_STRING(Sum))) {
           simplifyIntOrFloatReduction(call, kindMap, genRuntimeSumBody);
           return;
         }
-        if (funcName.startswith(RTNAME_STRING(DotProduct))) {
+        if (funcName.starts_with(RTNAME_STRING(DotProduct))) {
           LLVM_DEBUG(llvm::dbgs() << "Handling " << funcName << "\n");
           LLVM_DEBUG(llvm::dbgs() << "Call operation:\n"; op->dump();
                      llvm::dbgs() << "\n");
@@ -1350,23 +1350,23 @@ void SimplifyIntrinsicsPass::runOnOperation() {
                      llvm::dbgs() << "\n");
           return;
         }
-        if (funcName.startswith(RTNAME_STRING(Maxval))) {
+        if (funcName.starts_with(RTNAME_STRING(Maxval))) {
           simplifyIntOrFloatReduction(call, kindMap, genRuntimeMaxvalBody);
           return;
         }
-        if (funcName.startswith(RTNAME_STRING(Count))) {
+        if (funcName.starts_with(RTNAME_STRING(Count))) {
           simplifyLogicalDim0Reduction(call, kindMap, genRuntimeCountBody);
           return;
         }
-        if (funcName.startswith(RTNAME_STRING(Any))) {
+        if (funcName.starts_with(RTNAME_STRING(Any))) {
           simplifyLogicalDim1Reduction(call, kindMap, genRuntimeAnyBody);
           return;
         }
-        if (funcName.endswith(RTNAME_STRING(All))) {
+        if (funcName.ends_with(RTNAME_STRING(All))) {
           simplifyLogicalDim1Reduction(call, kindMap, genRuntimeAllBody);
           return;
         }
-        if (funcName.startswith(RTNAME_STRING(Minloc))) {
+        if (funcName.starts_with(RTNAME_STRING(Minloc))) {
           simplifyMinlocReduction(call, kindMap);
           return;
         }
diff --git a/flang/lib/Parser/source.cpp b/flang/lib/Parser/source.cpp
index d0fe399424e1395bbe525631c63b562f196a01e4..4b4fed64a1a40a3bd61edc1480ac301683b7202c 100644
--- a/flang/lib/Parser/source.cpp
+++ b/flang/lib/Parser/source.cpp
@@ -46,7 +46,7 @@ void SourceFile::RecordLineStarts() {
 void SourceFile::IdentifyPayload() {
   llvm::StringRef content{buf_->getBufferStart(), buf_->getBufferSize()};
   constexpr llvm::StringLiteral UTF8_BOM{"\xef\xbb\xbf"};
-  if (content.startswith(UTF8_BOM)) {
+  if (content.starts_with(UTF8_BOM)) {
     bom_end_ = UTF8_BOM.size();
     encoding_ = Encoding::UTF_8;
   }
diff --git a/flang/test/Lower/namelist.f90 b/flang/test/Lower/namelist.f90
index bba7a0ea19774d2f31b017b6ce68913286456c2e..9fdd8a2c8f61314789879f3ff0d8c1cdbd10d28d 100644
--- a/flang/test/Lower/namelist.f90
+++ b/flang/test/Lower/namelist.f90
@@ -1,88 +1,144 @@
-! RUN: bbc -emit-fir -hlfir=false -o - %s | FileCheck %s
+! RUN: bbc -emit-fir -o - %s | FileCheck %s
 
-! CHECK-LABEL: func @_QQmain
+! CHECK-LABEL: c.func @_QQmain
 program p
-  ! CHECK-DAG: [[ccc:%[0-9]+]] = fir.alloca !fir.array<4x!fir.char<1,3>> {bindc_name = "ccc", uniq_name = "_QFEccc"}
-  ! CHECK-DAG: [[jjj:%[0-9]+]] = fir.alloca i32 {bindc_name = "jjj", uniq_name = "_QFEjjj"}
+  ! CHECK:     %[[V_1:[0-9]+]] = fir.alloca !fir.box<!fir.ptr<!fir.array<4x!fir.char<1,3>>>>
+  ! CHECK:     %[[V_2:[0-9]+]] = fir.alloca !fir.box<!fir.ptr<i32>>
+  ! CHECK:     %[[V_3:[0-9]+]] = fir.alloca !fir.box<!fir.ptr<!fir.array<4x!fir.char<1,3>>>>
+  ! CHECK:     %[[V_4:[0-9]+]] = fir.alloca !fir.box<!fir.ptr<i32>>
+  ! CHECK:     %[[V_5:[0-9]+]] = fir.alloca !fir.array<4x!fir.char<1,3>> {bindc_name = "ccc", uniq_name = "_QFEccc"}
+  ! CHECK:     %[[V_6:[0-9]+]] = fir.shape %c4{{.*}} : (index) -> !fir.shape<1>
+  ! CHECK:     %[[V_7:[0-9]+]] = fir.declare %[[V_5]](%[[V_6]]) typeparams %c3{{.*}} {uniq_name = "_QFEccc"} : (!fir.ref<!fir.array<4x!fir.char<1,3>>>, !fir.shape<1>, index) -> !fir.ref<!fir.array<4x!fir.char<1,3>>>
+  ! CHECK:     %[[V_8:[0-9]+]] = fir.alloca i32 {bindc_name = "jjj", uniq_name = "_QFEjjj"}
+  ! CHECK:     %[[V_9:[0-9]+]] = fir.declare %[[V_8]] {uniq_name = "_QFEjjj"} : (!fir.ref<i32>) -> !fir.ref<i32>
+  ! CHECK:     fir.store %c17{{.*}} to %[[V_9]] : !fir.ref<i32>
   character*3 ccc(4)
   namelist /nnn/ jjj, ccc
   jjj = 17
   ccc = ["aa ", "bb ", "cc ", "dd "]
-  ! CHECK: [[cookie:%[0-9]+]] = fir.call @_FortranAioBeginExternalListOutput
-  ! CHECK: fir.alloca !fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>
-  ! CHECK: fir.undefined
-  ! CHECK: fir.address_of
-  ! CHECK: fir.insert_value
-  ! CHECK: fir.embox [[jjj]]
-  ! CHECK: fir.insert_value
-  ! CHECK: fir.address_of
-  ! CHECK: fir.insert_value
-  ! CHECK: fir.embox [[ccc]]
-  ! CHECK: fir.insert_value
-  ! CHECK: fir.alloca tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>
-  ! CHECK: fir.address_of
-  ! CHECK-COUNT-4: fir.insert_value
-  ! CHECK: fir.call @_FortranAioOutputNamelist([[cookie]]
-  ! CHECK: fir.call @_FortranAioEndIoStatement([[cookie]]
+
+  ! CHECK:     %[[V_23:[0-9]+]] = fir.call @_FortranAioBeginExternalListOutput
+  ! CHECK:     %[[V_24:[0-9]+]] = fir.alloca !fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>
+  ! CHECK:     %[[V_25:[0-9]+]] = fir.undefined !fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>
+  ! CHECK:     %[[V_26:[0-9]+]] = fir.address_of(@_QQclX6A6A6A00) : !fir.ref<!fir.char<1,4>>
+  ! CHECK:     %[[V_27:[0-9]+]] = fir.convert %[[V_26]] : (!fir.ref<!fir.char<1,4>>) -> !fir.ref<i8>
+  ! CHECK:     %[[V_28:[0-9]+]] = fir.insert_value %[[V_25]], %[[V_27]], [0 : index, 0 : index] : (!fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>, !fir.ref<i8>) -> !fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>
+  ! CHECK:     %[[V_29:[0-9]+]] = fir.embox %[[V_9]] : (!fir.ref<i32>) -> !fir.box<!fir.ptr<i32>>
+  ! CHECK:     fir.store %[[V_29]] to %[[V_4]] : !fir.ref<!fir.box<!fir.ptr<i32>>>
+  ! CHECK:     %[[V_30:[0-9]+]] = fir.convert %[[V_4]] : (!fir.ref<!fir.box<!fir.ptr<i32>>>) -> !fir.ref<!fir.box<none>>
+  ! CHECK:     %[[V_31:[0-9]+]] = fir.insert_value %[[V_28]], %[[V_30]], [0 : index, 1 : index] : (!fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>, !fir.ref<!fir.box<none>>) -> !fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>
+  ! CHECK:     %[[V_32:[0-9]+]] = fir.address_of(@_QQclX63636300) : !fir.ref<!fir.char<1,4>>
+  ! CHECK:     %[[V_33:[0-9]+]] = fir.convert %[[V_32]] : (!fir.ref<!fir.char<1,4>>) -> !fir.ref<i8>
+  ! CHECK:     %[[V_34:[0-9]+]] = fir.insert_value %[[V_31]], %[[V_33]], [1 : index, 0 : index] : (!fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>, !fir.ref<i8>) -> !fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>
+  ! CHECK:     %[[V_35:[0-9]+]] = fir.embox %[[V_7]](%[[V_6]]) : (!fir.ref<!fir.array<4x!fir.char<1,3>>>, !fir.shape<1>) -> !fir.box<!fir.ptr<!fir.array<4x!fir.char<1,3>>>>
+  ! CHECK:     fir.store %[[V_35]] to %[[V_3]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<4x!fir.char<1,3>>>>>
+  ! CHECK:     %[[V_36:[0-9]+]] = fir.convert %[[V_3]] : (!fir.ref<!fir.box<!fir.ptr<!fir.array<4x!fir.char<1,3>>>>>) -> !fir.ref<!fir.box<none>>
+  ! CHECK:     %[[V_37:[0-9]+]] = fir.insert_value %[[V_34]], %[[V_36]], [1 : index, 1 : index] : (!fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>, !fir.ref<!fir.box<none>>) -> !fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>
+  ! CHECK:     fir.store %[[V_37]] to %[[V_24]] : !fir.ref<!fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>
+  ! CHECK:     %[[V_38:[0-9]+]] = fir.alloca tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>
+  ! CHECK:     %[[V_39:[0-9]+]] = fir.undefined tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>
+  ! CHECK:     %[[V_40:[0-9]+]] = fir.address_of(@_QQclX6E6E6E00) : !fir.ref<!fir.char<1,4>>
+  ! CHECK:     %[[V_41:[0-9]+]] = fir.convert %[[V_40]] : (!fir.ref<!fir.char<1,4>>) -> !fir.ref<i8>
+  ! CHECK:     %[[V_42:[0-9]+]] = fir.insert_value %[[V_39]], %[[V_41]], [0 : index] : (tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>, !fir.ref<i8>) -> tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>
+  ! CHECK:     %[[V_43:[0-9]+]] = fir.insert_value %[[V_42]], %c2{{.*}}, [1 : index] : (tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>, i64) -> tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>
+  ! CHECK:     %[[V_44:[0-9]+]] = fir.insert_value %[[V_43]], %[[V_24]], [2 : index] : (tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>, !fir.ref<!fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>) -> tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>
+  ! CHECK:     %[[V_45:[0-9]+]] = fir.address_of(@default.nonTbpDefinedIoTable) : !fir.ref<tuple<i64, !fir.ref<!fir.array<0xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>>
+  ! CHECK:     %[[V_46:[0-9]+]] = fir.convert %[[V_45]] : (!fir.ref<tuple<i64, !fir.ref<!fir.array<0xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>>) -> !fir.ref<none>
+  ! CHECK:     %[[V_47:[0-9]+]] = fir.insert_value %[[V_44]], %[[V_46]], [3 : index] : (tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>, !fir.ref<none>) -> tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>
+  ! CHECK:     fir.store %[[V_47]] to %[[V_38]] : !fir.ref<tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>>
+  ! CHECK:     %[[V_48:[0-9]+]] = fir.convert %[[V_38]] : (!fir.ref<tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>>) -> !fir.ref<tuple<>>
+  ! CHECK:     %[[V_49:[0-9]+]] = fir.call @_FortranAioOutputNamelist(%[[V_23]], %[[V_48]]) fastmath<contract> : (!fir.ref<i8>, !fir.ref<tuple<>>) -> i1
+  ! CHECK:     %[[V_50:[0-9]+]] = fir.call @_FortranAioEndIoStatement(%[[V_23]]) fastmath<contract> : (!fir.ref<i8>) -> i32
   write(*, nnn)
   jjj = 27
-  ! CHECK: fir.coordinate_of
   ccc(4) = "zz "
-  ! CHECK: [[cookie:%[0-9]+]] = fir.call @_FortranAioBeginExternalListOutput
-  ! CHECK: fir.alloca !fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>
-  ! CHECK: fir.undefined
-  ! CHECK: fir.address_of
-  ! CHECK: fir.insert_value
-  ! CHECK: fir.embox [[jjj]]
-  ! CHECK: fir.insert_value
-  ! CHECK: fir.address_of
-  ! CHECK: fir.insert_value
-  ! CHECK: fir.embox [[ccc]]
-  ! CHECK: fir.insert_value
-  ! CHECK: fir.alloca tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>
-  ! CHECK: fir.address_of
-  ! CHECK-COUNT-4: fir.insert_value
-  ! CHECK: fir.call @_FortranAioOutputNamelist([[cookie]]
-  ! CHECK: fir.call @_FortranAioEndIoStatement([[cookie]]
+  ! CHECK:     %[[V_58:[0-9]+]] = fir.call @_FortranAioBeginExternalListOutput
+  ! CHECK:     %[[V_59:[0-9]+]] = fir.alloca !fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>
+  ! CHECK:     fir.store %[[V_29]] to %[[V_2]] : !fir.ref<!fir.box<!fir.ptr<i32>>>
+  ! CHECK:     %[[V_60:[0-9]+]] = fir.convert %[[V_2]] : (!fir.ref<!fir.box<!fir.ptr<i32>>>) -> !fir.ref<!fir.box<none>>
+  ! CHECK:     %[[V_61:[0-9]+]] = fir.insert_value %[[V_28]], %[[V_60]], [0 : index, 1 : index] : (!fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>, !fir.ref<!fir.box<none>>) -> !fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>
+  ! CHECK:     %[[V_62:[0-9]+]] = fir.insert_value %[[V_61]], %[[V_33]], [1 : index, 0 : index] : (!fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>, !fir.ref<i8>) -> !fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>
+  ! CHECK:     fir.store %[[V_35]] to %[[V_1]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<4x!fir.char<1,3>>>>>
+  ! CHECK:     %[[V_63:[0-9]+]] = fir.convert %[[V_1]] : (!fir.ref<!fir.box<!fir.ptr<!fir.array<4x!fir.char<1,3>>>>>) -> !fir.ref<!fir.box<none>>
+  ! CHECK:     %[[V_64:[0-9]+]] = fir.insert_value %[[V_62]], %[[V_63]], [1 : index, 1 : index] : (!fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>, !fir.ref<!fir.box<none>>) -> !fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>
+  ! CHECK:     fir.store %[[V_64]] to %[[V_59]] : !fir.ref<!fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>
+  ! CHECK:     %[[V_65:[0-9]+]] = fir.alloca tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>
+  ! CHECK:     %[[V_66:[0-9]+]] = fir.insert_value %[[V_43]], %[[V_59]], [2 : index] : (tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>, !fir.ref<!fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>) -> tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>
+  ! CHECK:     %[[V_67:[0-9]+]] = fir.insert_value %[[V_66]], %[[V_46]], [3 : index] : (tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>, !fir.ref<none>) -> tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>
+  ! CHECK:     fir.store %[[V_67]] to %[[V_65]] : !fir.ref<tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>>
+  ! CHECK:     %[[V_68:[0-9]+]] = fir.convert %[[V_65]] : (!fir.ref<tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<2xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>>) -> !fir.ref<tuple<>>
+  ! CHECK:     %[[V_69:[0-9]+]] = fir.call @_FortranAioOutputNamelist(%[[V_58]], %[[V_68]]) fastmath<contract> : (!fir.ref<i8>, !fir.ref<tuple<>>) -> i1
+  ! CHECK:     %[[V_70:[0-9]+]] = fir.call @_FortranAioEndIoStatement(%[[V_58]]) fastmath<contract> : (!fir.ref<i8>) -> i32
   write(*, nnn)
+
+  call rename
 end
 
-! CHECK-LABEL: sss
+! CHECK-LABEL: c.func @_QPsss
 subroutine sss
+  ! CHECK:     %[[V_0:[0-9]+]] = fir.alloca !fir.box<!fir.ptr<!fir.array<3xi32>>>
+  ! CHECK:     %[[V_1:[0-9]+]] = fir.alloca !fir.array<3xi32> {bindc_name = "xxx", uniq_name = "_QFsssExxx"}
+  ! CHECK:     %[[V_2:[0-9]+]] = fir.shape_shift %c11{{.*}}, %c3{{.*}} : (index, index) -> !fir.shapeshift<1>
+  ! CHECK:     %[[V_3:[0-9]+]] = fir.declare %[[V_1]](%[[V_2]]) {uniq_name = "_QFsssExxx"} : (!fir.ref<!fir.array<3xi32>>, !fir.shapeshift<1>) -> !fir.ref<!fir.array<3xi32>>
   integer xxx(11:13)
+
+  ! CHECK:     %[[V_7:[0-9]+]] = fir.call @_FortranAioBeginExternalListInput
+  ! CHECK:     %[[V_8:[0-9]+]] = fir.alloca !fir.array<1xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>
+  ! CHECK:     %[[V_9:[0-9]+]] = fir.undefined !fir.array<1xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>
+  ! CHECK:     %[[V_10:[0-9]+]] = fir.address_of(@_QQclX78787800) : !fir.ref<!fir.char<1,4>>
+  ! CHECK:     %[[V_11:[0-9]+]] = fir.convert %[[V_10]] : (!fir.ref<!fir.char<1,4>>) -> !fir.ref<i8>
+  ! CHECK:     %[[V_12:[0-9]+]] = fir.insert_value %[[V_9]], %[[V_11]], [0 : index, 0 : index] : (!fir.array<1xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>, !fir.ref<i8>) -> !fir.array<1xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>
+  ! CHECK:     %[[V_13:[0-9]+]] = fir.embox %[[V_3]](%[[V_2]]) : (!fir.ref<!fir.array<3xi32>>, !fir.shapeshift<1>) -> !fir.box<!fir.ptr<!fir.array<3xi32>>>
+  ! CHECK:     fir.store %[[V_13]] to %[[V_0]] : !fir.ref<!fir.box<!fir.ptr<!fir.array<3xi32>>>>
+  ! CHECK:     %[[V_14:[0-9]+]] = fir.convert %[[V_0]] : (!fir.ref<!fir.box<!fir.ptr<!fir.array<3xi32>>>>) -> !fir.ref<!fir.box<none>>
+  ! CHECK:     %[[V_15:[0-9]+]] = fir.insert_value %[[V_12]], %[[V_14]], [0 : index, 1 : index] : (!fir.array<1xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>, !fir.ref<!fir.box<none>>) -> !fir.array<1xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>
+  ! CHECK:     fir.store %[[V_15]] to %[[V_8]] : !fir.ref<!fir.array<1xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>
+  ! CHECK:     %[[V_16:[0-9]+]] = fir.alloca tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<1xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>
+  ! CHECK:     %[[V_17:[0-9]+]] = fir.undefined tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<1xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>
+  ! CHECK:     %[[V_18:[0-9]+]] = fir.address_of(@_QQclX72727200) : !fir.ref<!fir.char<1,4>>
+  ! CHECK:     %[[V_19:[0-9]+]] = fir.convert %[[V_18]] : (!fir.ref<!fir.char<1,4>>) -> !fir.ref<i8>
+  ! CHECK:     %[[V_20:[0-9]+]] = fir.insert_value %[[V_17]], %[[V_19]], [0 : index] : (tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<1xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>, !fir.ref<i8>) -> tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<1xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>
+  ! CHECK:     %[[V_21:[0-9]+]] = fir.insert_value %[[V_20]], %c1{{.*}}, [1 : index] : (tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<1xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>, i64) -> tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<1xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>
+  ! CHECK:     %[[V_22:[0-9]+]] = fir.insert_value %[[V_21]], %[[V_8]], [2 : index] : (tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<1xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>, !fir.ref<!fir.array<1xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>) -> tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<1xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>
+  ! CHECK:     %[[V_23:[0-9]+]] = fir.address_of(@default.nonTbpDefinedIoTable) : !fir.ref<tuple<i64, !fir.ref<!fir.array<0xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>>
+  ! CHECK:     %[[V_24:[0-9]+]] = fir.convert %[[V_23]] : (!fir.ref<tuple<i64, !fir.ref<!fir.array<0xtuple<!fir.ref<none>, !fir.ref<none>, i32, i1>>>, i1>>) -> !fir.ref<none>
+  ! CHECK:     %[[V_25:[0-9]+]] = fir.insert_value %[[V_22]], %[[V_24]], [3 : index] : (tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<1xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>, !fir.ref<none>) -> tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<1xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>
+  ! CHECK:     fir.store %[[V_25]] to %[[V_16]] : !fir.ref<tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<1xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>>
+  ! CHECK:     %[[V_26:[0-9]+]] = fir.convert %[[V_16]] : (!fir.ref<tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<1xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>>) -> !fir.ref<tuple<>>
+  ! CHECK:     %[[V_27:[0-9]+]] = fir.call @_FortranAioInputNamelist(%[[V_7]], %[[V_26]]) fastmath<contract> : (!fir.ref<i8>, !fir.ref<tuple<>>) -> i1
+  ! CHECK:     %[[V_28:[0-9]+]] = fir.call @_FortranAioEndIoStatement(%[[V_7]]) fastmath<contract> : (!fir.ref<i8>) -> i32
   namelist /rrr/ xxx
-  ! CHECK: [[xxx:%[0-9]+]] = fir.alloca {{.*}} = "xxx"
-  ! CHECK: [[cookie:%[0-9]+]] = fir.call @_FortranAioBeginExternalListInput
-  ! CHECK: alloca
-  ! CHECK: undefined
-  ! CHECK: fir.address_of{{.*}}787878
-  ! CHECK: fir.insert_value
-  ! CHECK: fir.shape_shift %c11
-  ! CHECK: fir.embox [[xxx]]
-  ! CHECK: fir.insert_value
-  ! CHECK: fir.alloca
-  ! CHECK: fir.undefined
-  ! CHECK: fir.address_of{{.*}}727272
-  ! CHECK-COUNT-3: fir.insert_value
-  ! CHECK: fir.call @_FortranAioInputNamelist([[cookie]]
-  ! CHECK: fir.call @_FortranAioEndIoStatement([[cookie]]
   read(*, rrr)
 end
 
-! CHECK-LABEL: global_pointer
+! CHECK-LABEL: c.func @_QPglobal_pointer
 subroutine global_pointer
   real,pointer,save::ptrarray(:)
-  ! CHECK: %[[a0:.*]] = fir.address_of
+  ! CHECK:     %[[V_4:[0-9]+]] = fir.call @_FortranAioBeginExternalListOutput
+  ! CHECK:     %[[V_5:[0-9]+]] = fir.address_of(@_QFglobal_pointerNmygroup) : !fir.ref<tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<1xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>>
+  ! CHECK:     %[[V_6:[0-9]+]] = fir.convert %[[V_5]] : (!fir.ref<tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<1xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>>) -> !fir.ref<tuple<>>
+  ! CHECK:     %[[V_7:[0-9]+]] = fir.call @_FortranAioOutputNamelist(%[[V_4]], %[[V_6]]) fastmath<contract> : (!fir.ref<i8>, !fir.ref<tuple<>>) -> i1
+  ! CHECK:     %[[V_8:[0-9]+]] = fir.call @_FortranAioEndIoStatement(%[[V_4]]) fastmath<contract> : (!fir.ref<i8>) -> i32
   namelist/mygroup/ptrarray
-  ! CHECK: %[[a1:.*]] = fir.convert %[[a0]]
-  ! CHECK: %[[a2:.*]] = fir.call @_FortranAioBeginExternalListOutput({{.*}}, %[[a1]], {{.*}}) {{.*}}: (i32, !fir.ref<i8>, i32) -> !fir.ref<i8>
-  ! CHECK: %[[a3:.*]] = fir.address_of
-  ! CHECK: %[[a4:.*]] = fir.convert %[[a3]]
-  ! CHECK: %[[a5:.*]] = fir.call @_FortranAioOutputNamelist(%[[a2]], %[[a4]])
-  ! CHECK: %[[a6:.*]] = fir.call @_FortranAioEndIoStatement(%[[a2]])
   write(10, nml=mygroup)
 end
 
-  ! CHECK-DAG: fir.global linkonce @_QQclX6A6A6A00 constant : !fir.char<1,4>
-  ! CHECK-DAG: fir.global linkonce @_QQclX63636300 constant : !fir.char<1,4>
-  ! CHECK-DAG: fir.global linkonce @_QQclX6E6E6E00 constant : !fir.char<1,4>
+module mmm
+  real rrr
+  namelist /aaa/ rrr
+end
+
+! CHECK-LABEL: c.func @_QPrename
+subroutine rename
+  use mmm, bbb => aaa
+  rrr = 3.
+  ! CHECK:     %[[V_4:[0-9]+]] = fir.call @_FortranAioBeginExternalListOutput
+  ! CHECK:     %[[V_5:[0-9]+]] = fir.address_of(@_QMmmmNaaa) : !fir.ref<tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<1xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>>
+  ! CHECK:     %[[V_6:[0-9]+]] = fir.convert %[[V_5]] : (!fir.ref<tuple<!fir.ref<i8>, i64, !fir.ref<!fir.array<1xtuple<!fir.ref<i8>, !fir.ref<!fir.box<none>>>>>, !fir.ref<none>>>) -> !fir.ref<tuple<>>
+  ! CHECK:     %[[V_7:[0-9]+]] = fir.call @_FortranAioOutputNamelist(%[[V_4]], %[[V_6]]) fastmath<contract> : (!fir.ref<i8>, !fir.ref<tuple<>>) -> i1
+  ! CHECK:     %[[V_8:[0-9]+]] = fir.call @_FortranAioEndIoStatement(%[[V_4]]) fastmath<contract> : (!fir.ref<i8>) -> i32
+  write(*,bbb)
+end
+
+! CHECK-NOT:   bbb
+! CHECK:       fir.string_lit "aaa\00"(4) : !fir.char<1,4>
diff --git a/flang/tools/flang-driver/driver.cpp b/flang/tools/flang-driver/driver.cpp
index 99fa66b0dc8e266a84fac7227fbf6b104b65ab24..c4e56a862c861359311a035e8d71342df1d7d5bd 100644
--- a/flang/tools/flang-driver/driver.cpp
+++ b/flang/tools/flang-driver/driver.cpp
@@ -99,13 +99,13 @@ int main(int argc, const char **argv) {
   auto firstArg = std::find_if(args.begin() + 1, args.end(),
                                [](const char *a) { return a != nullptr; });
   if (firstArg != args.end()) {
-    if (llvm::StringRef(args[1]).startswith("-cc1")) {
+    if (llvm::StringRef(args[1]).starts_with("-cc1")) {
       llvm::errs() << "error: unknown integrated tool '" << args[1] << "'. "
                    << "Valid tools include '-fc1'.\n";
       return 1;
     }
     // Call flang-new frontend
-    if (llvm::StringRef(args[1]).startswith("-fc1")) {
+    if (llvm::StringRef(args[1]).starts_with("-fc1")) {
       return executeFC1Tool(args);
     }
   }
diff --git a/flang/unittests/Frontend/CompilerInstanceTest.cpp b/flang/unittests/Frontend/CompilerInstanceTest.cpp
index 6dbbf9b4e1bbd3cbb96117bc456b42352c52560c..35f1ec1748a3f69fd6e67ab613d59650a716805e 100644
--- a/flang/unittests/Frontend/CompilerInstanceTest.cpp
+++ b/flang/unittests/Frontend/CompilerInstanceTest.cpp
@@ -55,7 +55,7 @@ TEST(CompilerInstance, SanityCheckForFileManager) {
   llvm::ArrayRef<char> fileContent = sf->content();
   EXPECT_FALSE(fileContent.size() == 0);
   EXPECT_TRUE(
-      llvm::StringRef(fileContent.data()).startswith("InputSourceFile"));
+      llvm::StringRef(fileContent.data()).starts_with("InputSourceFile"));
 
   // 4. Delete the test file
   ec = llvm::sys::fs::remove(inputFile);
diff --git a/flang/unittests/Frontend/FrontendActionTest.cpp b/flang/unittests/Frontend/FrontendActionTest.cpp
index d57154cb1001c7295e2b45f2283e8929b62d8c2a..6ec15832d96d3cb11aec687c8cc5a30d93ece45e 100644
--- a/flang/unittests/Frontend/FrontendActionTest.cpp
+++ b/flang/unittests/Frontend/FrontendActionTest.cpp
@@ -112,7 +112,7 @@ TEST_F(FrontendActionTest, TestInputOutput) {
   EXPECT_TRUE(success);
   EXPECT_TRUE(!outputFileBuffer.empty());
   EXPECT_TRUE(llvm::StringRef(outputFileBuffer.data())
-                  .startswith("End Program arithmetic"));
+                  .starts_with("End Program arithmetic"));
 }
 
 TEST_F(FrontendActionTest, PrintPreprocessedInput) {
@@ -143,7 +143,7 @@ TEST_F(FrontendActionTest, PrintPreprocessedInput) {
   EXPECT_TRUE(success);
   EXPECT_TRUE(!outputFileBuffer.empty());
   EXPECT_TRUE(
-      llvm::StringRef(outputFileBuffer.data()).startswith("program b\n"));
+      llvm::StringRef(outputFileBuffer.data()).starts_with("program b\n"));
 }
 
 TEST_F(FrontendActionTest, ParseSyntaxOnly) {
diff --git a/libc/config/linux/api.td b/libc/config/linux/api.td
index 726e58f376eaa766e30e7b2ced0f37300c947c75..85f6b59264eb06d29254135dff4571acaf205927 100644
--- a/libc/config/linux/api.td
+++ b/libc/config/linux/api.td
@@ -205,7 +205,12 @@ def SysSelectAPI : PublicAPI<"sys/select.h"> {
 }
 
 def SysSocketAPI : PublicAPI<"sys/socket.h"> {
-  let Types = ["struct sockaddr", "sa_family_t"];
+  let Types = [
+    "sa_family_t",
+    "socklen_t",
+    "struct sockaddr",
+    "struct sockaddr_un",
+  ];
 }
 
 def SysResourceAPI : PublicAPI<"sys/resource.h"> {
diff --git a/libc/config/linux/x86_64/entrypoints.txt b/libc/config/linux/x86_64/entrypoints.txt
index 13b81d3b7ca702c7d4f0b4be523de6403fe32a2c..1c93063e25e90c08c64857824e6333ec43c6016c 100644
--- a/libc/config/linux/x86_64/entrypoints.txt
+++ b/libc/config/linux/x86_64/entrypoints.txt
@@ -153,9 +153,6 @@ set(TARGET_LIBC_ENTRYPOINTS
     # sys/sendfile entrypoints
     libc.src.sys.sendfile.sendfile
 
-    # sys/socket.h entrypoints
-    libc.src.sys.socket.socket
-
     # sys/stat.h entrypoints
     libc.src.sys.stat.chmod
     libc.src.sys.stat.fchmod
@@ -557,6 +554,10 @@ if(LLVM_LIBC_FULL_BUILD)
 
     # sys/select.h entrypoints
     libc.src.sys.select.select
+
+    # sys/socket.h entrypoints
+    libc.src.sys.socket.socket
+    libc.src.sys.socket.bind
   )
 endif()
 
diff --git a/libc/include/CMakeLists.txt b/libc/include/CMakeLists.txt
index 429c0f1f12866a8988f469b32b30dc126cbf2ed3..59c6c4a9bb4200045aff79905a8f6bc9eaf1094a 100644
--- a/libc/include/CMakeLists.txt
+++ b/libc/include/CMakeLists.txt
@@ -417,8 +417,10 @@ add_gen_header(
   DEPENDS
     .llvm_libc_common_h
     .llvm-libc-macros.sys_socket_macros
-    .llvm-libc-types.struct_sockaddr
     .llvm-libc-types.sa_family_t
+    .llvm-libc-types.socklen_t
+    .llvm-libc-types.struct_sockaddr
+    .llvm-libc-types.struct_sockaddr_un
 )
 
 add_gen_header(
diff --git a/libc/include/llvm-libc-macros/linux/fcntl-macros.h b/libc/include/llvm-libc-macros/linux/fcntl-macros.h
index cdd1cf22d7b69cbdc0d128d86651feef4278c7db..495c5ec780edb0ee76bd735b74120011c4e25f3a 100644
--- a/libc/include/llvm-libc-macros/linux/fcntl-macros.h
+++ b/libc/include/llvm-libc-macros/linux/fcntl-macros.h
@@ -46,31 +46,6 @@
 #define O_RDWR 00000002
 #define O_WRONLY 00000001
 
-// File mode flags
-#define S_IRWXU 0700
-#define S_IRUSR 0400
-#define S_IWUSR 0200
-#define S_IXUSR 0100
-#define S_IRWXG 070
-#define S_IRGRP 040
-#define S_IWGRP 020
-#define S_IXGRP 010
-#define S_IRWXO 07
-#define S_IROTH 04
-#define S_IWOTH 02
-#define S_IXOTH 01
-#define S_ISUID 04000
-#define S_ISGID 02000
-
-// File type flags
-#define S_IFMT 0170000
-#define S_IFDIR 0040000
-#define S_IFCHR 0020000
-#define S_IFBLK 0060000
-#define S_IFREG 0100000
-#define S_FIFO 0010000
-#define S_IFLNK 0120000
-
 // Special directory FD to indicate that the path argument to
 // openat is relative to the current directory.
 #define AT_FDCWD -100
diff --git a/libc/include/llvm-libc-macros/linux/sys-stat-macros.h b/libc/include/llvm-libc-macros/linux/sys-stat-macros.h
index 3be743328a26bb71315161e9732f0365c26a0f80..48606cfa08ce935f685eb09177a06a891c1bf887 100644
--- a/libc/include/llvm-libc-macros/linux/sys-stat-macros.h
+++ b/libc/include/llvm-libc-macros/linux/sys-stat-macros.h
@@ -10,7 +10,7 @@
 #define __LLVM_LIBC_MACROS_LINUX_SYS_STAT_MACROS_H
 
 // Definitions from linux/stat.h
-#define S_IFMT  00170000
+#define S_IFMT 0170000
 #define S_IFSOCK 0140000
 #define S_IFLNK  0120000
 #define S_IFREG  0100000
diff --git a/libc/include/llvm-libc-types/CMakeLists.txt b/libc/include/llvm-libc-types/CMakeLists.txt
index 225ad780c4d01f2412b806aca38a5e5e5a5c271f..500900ffa0bbb05ed2a6aa12cd66d861771af09b 100644
--- a/libc/include/llvm-libc-types/CMakeLists.txt
+++ b/libc/include/llvm-libc-types/CMakeLists.txt
@@ -89,6 +89,8 @@ add_header(__getoptargv_t HDR __getoptargv_t.h)
 add_header(wchar_t HDR wchar_t.h)
 add_header(wint_t HDR wint_t.h)
 add_header(sa_family_t HDR sa_family_t.h)
+add_header(socklen_t HDR socklen_t.h)
+add_header(struct_sockaddr_un HDR struct_sockaddr_un.h)
 add_header(struct_sockaddr HDR struct_sockaddr.h)
 add_header(rpc_opcodes_t HDR rpc_opcodes_t.h)
 add_header(ACTION HDR ACTION.h)
diff --git a/libc/include/llvm-libc-types/socklen_t.h b/libc/include/llvm-libc-types/socklen_t.h
new file mode 100644
index 0000000000000000000000000000000000000000..3134a53390e71ed420de55e6cdf87b47c2156cac
--- /dev/null
+++ b/libc/include/llvm-libc-types/socklen_t.h
@@ -0,0 +1,18 @@
+//===-- Definition of socklen_t type ------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __LLVM_LIBC_TYPES_SOCKLEN_T_H__
+#define __LLVM_LIBC_TYPES_SOCKLEN_T_H__
+
+// The posix standard only says of socklen_t that it must be an integer type of
+// width of at least 32 bits. The long type is defined as being at least 32
+// bits, so an unsigned long should be fine.
+
+typedef unsigned long socklen_t;
+
+#endif // __LLVM_LIBC_TYPES_SOCKLEN_T_H__
diff --git a/libc/include/llvm-libc-types/struct_sockaddr.h b/libc/include/llvm-libc-types/struct_sockaddr.h
index 1ef907904ca3ec182ce2cec3bbceccdbae16c18c..9a6214c7d3e6b9b837e3a434d706a339dd38dea9 100644
--- a/libc/include/llvm-libc-types/struct_sockaddr.h
+++ b/libc/include/llvm-libc-types/struct_sockaddr.h
@@ -1,4 +1,4 @@
-//===-- Definition of struct stat -----------------------------------------===//
+//===-- Definition of struct sockaddr -------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef __LLVM_LIBC_TYPES_STRUCT_STAT_H__
-#define __LLVM_LIBC_TYPES_STRUCT_STAT_H__
+#ifndef __LLVM_LIBC_TYPES_STRUCT_SOCKADDR_H__
+#define __LLVM_LIBC_TYPES_STRUCT_SOCKADDR_H__
 
 #include <llvm-libc-types/sa_family_t.h>
 
@@ -15,7 +15,7 @@ struct sockaddr {
   sa_family_t sa_family;
   // sa_data is a variable length array. It is provided with a length of one
   // here as a placeholder.
-  char sa_data[1];
+  char sa_data[];
 };
 
-#endif // __LLVM_LIBC_TYPES_STRUCT_STAT_H__
+#endif // __LLVM_LIBC_TYPES_STRUCT_SOCKADDR_H__
diff --git a/libc/include/llvm-libc-types/struct_sockaddr_un.h b/libc/include/llvm-libc-types/struct_sockaddr_un.h
new file mode 100644
index 0000000000000000000000000000000000000000..9c3efea279256ec2927de18cba579bf94abdf843
--- /dev/null
+++ b/libc/include/llvm-libc-types/struct_sockaddr_un.h
@@ -0,0 +1,22 @@
+//===-- Definition of struct sockaddr_un ----------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef __LLVM_LIBC_TYPES_STRUCT_SOCKADDR_UN_H__
+#define __LLVM_LIBC_TYPES_STRUCT_SOCKADDR_UN_H__
+
+#include <llvm-libc-types/sa_family_t.h>
+
+// This is the sockaddr specialization for AF_UNIX or AF_LOCAL sockets, as
+// defined by posix.
+
+struct sockaddr_un {
+  sa_family_t sun_family; /* AF_UNIX */
+  char sun_path[108];     /* Pathname */
+};
+
+#endif // __LLVM_LIBC_TYPES_STRUCT_SOCKADDR_UN_H__
diff --git a/libc/spec/posix.td b/libc/spec/posix.td
index c7acf6d25a2d8739b8e639d1fcd149497c37990c..7e1cf892135acf99c3f305736310e41c5e149350 100644
--- a/libc/spec/posix.td
+++ b/libc/spec/posix.td
@@ -81,9 +81,14 @@ def RestrictedFdSetPtr : RestrictedPtrType<FdSet>;
 
 def GetoptArgvT : NamedType<"__getoptargv_t">;
 
+def SAFamilyType : NamedType<"sa_family_t">;
+def SocklenType : NamedType<"socklen_t">;
+
 def StructSockAddr : NamedType<"struct sockaddr">;
 def StructSockAddrPtr : PtrType<StructSockAddr>;
-def SAFamilyType : NamedType<"sa_family_t">;
+def ConstStructSockAddrPtr : ConstType<StructSockAddrPtr>;
+
+def StructSockAddrUn : NamedType<"struct sockaddr_un">;
 
 def POSIX : StandardSpec<"POSIX"> {
   PtrType CharPtr = PtrType<CharType>;
@@ -1400,7 +1405,10 @@ def POSIX : StandardSpec<"POSIX"> {
         Macro<"SOCK_PACKET">,
       ], // Macros
       [
-        StructSockAddr, SAFamilyType,
+        SAFamilyType,
+        StructSockAddr,
+        StructSockAddrUn,
+        SocklenType,
       ], // Types
       [], // Enumerations
       [
@@ -1409,6 +1417,11 @@ def POSIX : StandardSpec<"POSIX"> {
           RetValSpec<IntType>,
           [ArgSpec<IntType>, ArgSpec<IntType>, ArgSpec<IntType>]
         >,
+        FunctionSpec<
+          "bind",
+          RetValSpec<IntType>,
+          [ArgSpec<IntType>, ArgSpec<ConstStructSockAddrPtr>, ArgSpec<SocklenType>]
+        >,
       ]  // Functions
   >;
 
diff --git a/libc/src/__support/FPUtil/FPBits.h b/libc/src/__support/FPUtil/FPBits.h
index 65c53921181a73a6a783d01d37faa73916640d4b..d1e26de22ef1308e34f1d0ee4458b35a66c3b3c6 100644
--- a/libc/src/__support/FPUtil/FPBits.h
+++ b/libc/src/__support/FPUtil/FPBits.h
@@ -20,14 +20,6 @@
 namespace LIBC_NAMESPACE {
 namespace fputil {
 
-template <typename T> struct MantissaWidth {
-  static constexpr unsigned VALUE = FloatProperties<T>::MANTISSA_WIDTH;
-};
-
-template <typename T> struct ExponentWidth {
-  static constexpr unsigned VALUE = FloatProperties<T>::EXPONENT_WIDTH;
-};
-
 // A generic class to represent single precision, double precision, and quad
 // precision IEEE 754 floating point formats.
 // On most platforms, the 'float' type corresponds to single precision floating
@@ -36,71 +28,74 @@ template <typename T> struct ExponentWidth {
 // floating numbers. On x86 platforms however, the 'long double' type maps to
 // an x87 floating point format. This format is an IEEE 754 extension format.
 // It is handled as an explicit specialization of this class.
-template <typename T> struct FPBits {
+template <typename T> struct FPBits : private FloatProperties<T> {
   static_assert(cpp::is_floating_point_v<T>,
                 "FPBits instantiated with invalid type.");
+  using typename FloatProperties<T>::UIntType;
+  using FloatProperties<T>::BIT_WIDTH;
+  using FloatProperties<T>::EXP_MANT_MASK;
+  using FloatProperties<T>::EXPONENT_MASK;
+  using FloatProperties<T>::EXPONENT_BIAS;
+  using FloatProperties<T>::EXPONENT_WIDTH;
+  using FloatProperties<T>::MANTISSA_MASK;
+  using FloatProperties<T>::MANTISSA_WIDTH;
+  using FloatProperties<T>::QUIET_NAN_MASK;
+  using FloatProperties<T>::SIGN_MASK;
 
   // Reinterpreting bits as an integer value and interpreting the bits of an
   // integer value as a floating point value is used in tests. So, a convenient
   // type is provided for such reinterpretations.
-  using FloatProp = FloatProperties<T>;
-  using UIntType = typename FloatProp::UIntType;
-
   UIntType bits;
 
   LIBC_INLINE constexpr void set_mantissa(UIntType mantVal) {
-    mantVal &= (FloatProp::MANTISSA_MASK);
-    bits &= ~(FloatProp::MANTISSA_MASK);
+    mantVal &= MANTISSA_MASK;
+    bits &= ~MANTISSA_MASK;
     bits |= mantVal;
   }
 
   LIBC_INLINE constexpr UIntType get_mantissa() const {
-    return bits & FloatProp::MANTISSA_MASK;
+    return bits & MANTISSA_MASK;
   }
 
   LIBC_INLINE constexpr void set_biased_exponent(UIntType expVal) {
-    expVal = (expVal << (FloatProp::MANTISSA_WIDTH)) & FloatProp::EXPONENT_MASK;
-    bits &= ~(FloatProp::EXPONENT_MASK);
+    expVal = (expVal << MANTISSA_WIDTH) & EXPONENT_MASK;
+    bits &= ~EXPONENT_MASK;
     bits |= expVal;
   }
 
   LIBC_INLINE constexpr uint16_t get_biased_exponent() const {
-    return uint16_t((bits & FloatProp::EXPONENT_MASK) >>
-                    (FloatProp::MANTISSA_WIDTH));
+    return uint16_t((bits & EXPONENT_MASK) >> MANTISSA_WIDTH);
   }
 
   // The function return mantissa with the implicit bit set iff the current
   // value is a valid normal number.
   LIBC_INLINE constexpr UIntType get_explicit_mantissa() {
     return ((get_biased_exponent() > 0 && !is_inf_or_nan())
-                ? (FloatProp::MANTISSA_MASK + 1)
+                ? (MANTISSA_MASK + 1)
                 : 0) |
-           (FloatProp::MANTISSA_MASK & bits);
+           (MANTISSA_MASK & bits);
   }
 
   LIBC_INLINE constexpr void set_sign(bool signVal) {
-    bits |= FloatProp::SIGN_MASK;
+    bits |= SIGN_MASK;
     if (!signVal)
-      bits -= FloatProp::SIGN_MASK;
+      bits -= SIGN_MASK;
   }
 
   LIBC_INLINE constexpr bool get_sign() const {
-    return (bits & FloatProp::SIGN_MASK) != 0;
+    return (bits & SIGN_MASK) != 0;
   }
 
   static_assert(sizeof(T) == sizeof(UIntType),
                 "Data type and integral representation have different sizes.");
 
-  static constexpr int EXPONENT_BIAS = (1 << (ExponentWidth<T>::VALUE - 1)) - 1;
-  static constexpr int MAX_EXPONENT = (1 << ExponentWidth<T>::VALUE) - 1;
+  static constexpr int MAX_EXPONENT = (1 << EXPONENT_WIDTH) - 1;
 
   static constexpr UIntType MIN_SUBNORMAL = UIntType(1);
-  static constexpr UIntType MAX_SUBNORMAL =
-      (UIntType(1) << MantissaWidth<T>::VALUE) - 1;
-  static constexpr UIntType MIN_NORMAL =
-      (UIntType(1) << MantissaWidth<T>::VALUE);
+  static constexpr UIntType MAX_SUBNORMAL = (UIntType(1) << MANTISSA_WIDTH) - 1;
+  static constexpr UIntType MIN_NORMAL = (UIntType(1) << MANTISSA_WIDTH);
   static constexpr UIntType MAX_NORMAL =
-      ((UIntType(MAX_EXPONENT) - 1) << MantissaWidth<T>::VALUE) | MAX_SUBNORMAL;
+      ((UIntType(MAX_EXPONENT) - 1) << MANTISSA_WIDTH) | MAX_SUBNORMAL;
 
   // We don't want accidental type promotions/conversions, so we require exact
   // type match.
@@ -151,32 +146,29 @@ template <typename T> struct FPBits {
   }
 
   LIBC_INLINE constexpr bool is_inf() const {
-    return (bits & FloatProp::EXP_MANT_MASK) == FloatProp::EXPONENT_MASK;
+    return (bits & EXP_MANT_MASK) == EXPONENT_MASK;
   }
 
   LIBC_INLINE constexpr bool is_nan() const {
-    return (bits & FloatProp::EXP_MANT_MASK) > FloatProp::EXPONENT_MASK;
+    return (bits & EXP_MANT_MASK) > EXPONENT_MASK;
   }
 
   LIBC_INLINE constexpr bool is_quiet_nan() const {
-    return (bits & FloatProp::EXP_MANT_MASK) ==
-           (FloatProp::EXPONENT_MASK | FloatProp::QUIET_NAN_MASK);
+    return (bits & EXP_MANT_MASK) == (EXPONENT_MASK | QUIET_NAN_MASK);
   }
 
   LIBC_INLINE constexpr bool is_inf_or_nan() const {
-    return (bits & FloatProp::EXPONENT_MASK) == FloatProp::EXPONENT_MASK;
+    return (bits & EXPONENT_MASK) == EXPONENT_MASK;
   }
 
   LIBC_INLINE static constexpr T zero(bool sign = false) {
-    return FPBits(sign ? FloatProp::SIGN_MASK : UIntType(0)).get_val();
+    return FPBits(sign ? SIGN_MASK : UIntType(0)).get_val();
   }
 
   LIBC_INLINE static constexpr T neg_zero() { return zero(true); }
 
   LIBC_INLINE static constexpr T inf(bool sign = false) {
-    return FPBits((sign ? FloatProp::SIGN_MASK : UIntType(0)) |
-                  FloatProp::EXPONENT_MASK)
-        .get_val();
+    return FPBits((sign ? SIGN_MASK : UIntType(0)) | EXPONENT_MASK).get_val();
   }
 
   LIBC_INLINE static constexpr T neg_inf() { return inf(true); }
@@ -204,7 +196,7 @@ template <typename T> struct FPBits {
   }
 
   LIBC_INLINE static constexpr T build_quiet_nan(UIntType v) {
-    return build_nan(FloatProp::QUIET_NAN_MASK | v);
+    return build_nan(QUIET_NAN_MASK | v);
   }
 
   // The function convert integer number and unbiased exponent to proper float
@@ -220,7 +212,7 @@ template <typename T> struct FPBits {
   LIBC_INLINE static constexpr FPBits<T> make_value(UIntType number, int ep) {
     FPBits<T> result;
     // offset: +1 for sign, but -1 for implicit first bit
-    int lz = cpp::countl_zero(number) - FloatProp::EXPONENT_WIDTH;
+    int lz = cpp::countl_zero(number) - EXPONENT_WIDTH;
     number <<= lz;
     ep -= lz;
 
diff --git a/libc/src/__support/FPUtil/Hypot.h b/libc/src/__support/FPUtil/Hypot.h
index 42d9e1b3f8cec5c96458ec1822da1fe3a830ae71..ad6b72db0524fc76015c15b8e2fb3e0061c95243 100644
--- a/libc/src/__support/FPUtil/Hypot.h
+++ b/libc/src/__support/FPUtil/Hypot.h
@@ -124,7 +124,7 @@ LIBC_INLINE T hypot(T x, T y) {
   uint16_t y_exp = y_bits.get_biased_exponent();
   uint16_t exp_diff = (x_exp > y_exp) ? (x_exp - y_exp) : (y_exp - x_exp);
 
-  if ((exp_diff >= MantissaWidth<T>::VALUE + 2) || (x == 0) || (y == 0)) {
+  if ((exp_diff >= FPBits_t::MANTISSA_WIDTH + 2) || (x == 0) || (y == 0)) {
     return abs(x) + abs(y);
   }
 
@@ -148,7 +148,7 @@ LIBC_INLINE T hypot(T x, T y) {
   out_exp = a_exp;
 
   // Add an extra bit to simplify the final rounding bit computation.
-  constexpr UIntType ONE = UIntType(1) << (MantissaWidth<T>::VALUE + 1);
+  constexpr UIntType ONE = UIntType(1) << (FPBits_t::MANTISSA_WIDTH + 1);
 
   a_mant <<= 1;
   b_mant <<= 1;
@@ -158,7 +158,7 @@ LIBC_INLINE T hypot(T x, T y) {
   if (a_exp != 0) {
     leading_one = ONE;
     a_mant |= ONE;
-    y_mant_width = MantissaWidth<T>::VALUE + 1;
+    y_mant_width = FPBits_t::MANTISSA_WIDTH + 1;
   } else {
     leading_one = internal::find_leading_one(a_mant, y_mant_width);
     a_exp = 1;
@@ -258,7 +258,7 @@ LIBC_INLINE T hypot(T x, T y) {
     }
   }
 
-  y_new |= static_cast<UIntType>(out_exp) << MantissaWidth<T>::VALUE;
+  y_new |= static_cast<UIntType>(out_exp) << FPBits_t::MANTISSA_WIDTH;
   return cpp::bit_cast<T>(y_new);
 }
 
diff --git a/libc/src/__support/FPUtil/ManipulationFunctions.h b/libc/src/__support/FPUtil/ManipulationFunctions.h
index 08adb074b121fa1564abb04a0f99b54cc137b5f5..51b58ba29bab89ff6753191e1a99c41addc0b64e 100644
--- a/libc/src/__support/FPUtil/ManipulationFunctions.h
+++ b/libc/src/__support/FPUtil/ManipulationFunctions.h
@@ -130,7 +130,7 @@ LIBC_INLINE T ldexp(T x, int exp) {
   // early. Because the result of the ldexp operation can be a subnormal number,
   // we need to accommodate the (mantissaWidht + 1) worth of shift in
   // calculating the limit.
-  int exp_limit = FPBits<T>::MAX_EXPONENT + MantissaWidth<T>::VALUE + 1;
+  int exp_limit = FPBits<T>::MAX_EXPONENT + FPBits<T>::MANTISSA_WIDTH + 1;
   if (exp > exp_limit)
     return bits.get_sign() ? T(FPBits<T>::neg_inf()) : T(FPBits<T>::inf());
 
diff --git a/libc/src/__support/FPUtil/NearestIntegerOperations.h b/libc/src/__support/FPUtil/NearestIntegerOperations.h
index 8c4b24803bec1d31d61dc5229adb70adf0250008..b0ae8d0040ea19282f7851e46f70f347c47af8d7 100644
--- a/libc/src/__support/FPUtil/NearestIntegerOperations.h
+++ b/libc/src/__support/FPUtil/NearestIntegerOperations.h
@@ -36,7 +36,7 @@ LIBC_INLINE T trunc(T x) {
 
   // If the exponent is greater than the most negative mantissa
   // exponent, then x is already an integer.
-  if (exponent >= static_cast<int>(MantissaWidth<T>::VALUE))
+  if (exponent >= static_cast<int>(FPBits<T>::MANTISSA_WIDTH))
     return x;
 
   // If the exponent is such that abs(x) is less than 1, then return 0.
@@ -47,7 +47,7 @@ LIBC_INLINE T trunc(T x) {
       return T(0.0);
   }
 
-  int trim_size = MantissaWidth<T>::VALUE - exponent;
+  int trim_size = FPBits<T>::MANTISSA_WIDTH - exponent;
   bits.set_mantissa((bits.get_mantissa() >> trim_size) << trim_size);
   return T(bits);
 }
@@ -65,7 +65,7 @@ LIBC_INLINE T ceil(T x) {
 
   // If the exponent is greater than the most negative mantissa
   // exponent, then x is already an integer.
-  if (exponent >= static_cast<int>(MantissaWidth<T>::VALUE))
+  if (exponent >= static_cast<int>(FPBits<T>::MANTISSA_WIDTH))
     return x;
 
   if (exponent <= -1) {
@@ -75,7 +75,7 @@ LIBC_INLINE T ceil(T x) {
       return T(1.0);
   }
 
-  uint32_t trim_size = MantissaWidth<T>::VALUE - exponent;
+  uint32_t trim_size = FPBits<T>::MANTISSA_WIDTH - exponent;
   bits.set_mantissa((bits.get_mantissa() >> trim_size) << trim_size);
   T trunc_value = T(bits);
 
@@ -114,7 +114,7 @@ LIBC_INLINE T round(T x) {
 
   // If the exponent is greater than the most negative mantissa
   // exponent, then x is already an integer.
-  if (exponent >= static_cast<int>(MantissaWidth<T>::VALUE))
+  if (exponent >= static_cast<int>(FPBits<T>::MANTISSA_WIDTH))
     return x;
 
   if (exponent == -1) {
@@ -133,7 +133,7 @@ LIBC_INLINE T round(T x) {
       return T(0.0);
   }
 
-  uint32_t trim_size = MantissaWidth<T>::VALUE - exponent;
+  uint32_t trim_size = FPBits<T>::MANTISSA_WIDTH - exponent;
   bool half_bit_set =
       bool(bits.get_mantissa() & (UIntType(1) << (trim_size - 1)));
   bits.set_mantissa((bits.get_mantissa() >> trim_size) << trim_size);
@@ -167,7 +167,7 @@ LIBC_INLINE T round_using_current_rounding_mode(T x) {
 
   // If the exponent is greater than the most negative mantissa
   // exponent, then x is already an integer.
-  if (exponent >= static_cast<int>(MantissaWidth<T>::VALUE))
+  if (exponent >= static_cast<int>(FPBits<T>::MANTISSA_WIDTH))
     return x;
 
   if (exponent <= -1) {
@@ -188,7 +188,7 @@ LIBC_INLINE T round_using_current_rounding_mode(T x) {
     }
   }
 
-  uint32_t trim_size = MantissaWidth<T>::VALUE - exponent;
+  uint32_t trim_size = FPBits<T>::MANTISSA_WIDTH - exponent;
   FPBits<T> new_bits = bits;
   new_bits.set_mantissa((bits.get_mantissa() >> trim_size) << trim_size);
   T trunc_value = T(new_bits);
diff --git a/libc/src/__support/FPUtil/NormalFloat.h b/libc/src/__support/FPUtil/NormalFloat.h
index d3236316a87995165f6aa2ad1e03e561b5c936bf..397a3bb41673b093a1f8be162c12c8ca9d417dd1 100644
--- a/libc/src/__support/FPUtil/NormalFloat.h
+++ b/libc/src/__support/FPUtil/NormalFloat.h
@@ -32,7 +32,7 @@ template <typename T> struct NormalFloat {
       "NormalFloat template parameter has to be a floating point type.");
 
   using UIntType = typename FPBits<T>::UIntType;
-  static constexpr UIntType ONE = (UIntType(1) << MantissaWidth<T>::VALUE);
+  static constexpr UIntType ONE = (UIntType(1) << FPBits<T>::MANTISSA_WIDTH);
 
   // Unbiased exponent value.
   int32_t exponent;
@@ -40,7 +40,7 @@ template <typename T> struct NormalFloat {
   UIntType mantissa;
   // We want |UIntType| to have atleast one bit more than the actual mantissa
   // bit width to accommodate the implicit 1 value.
-  static_assert(sizeof(UIntType) * 8 >= MantissaWidth<T>::VALUE + 1,
+  static_assert(sizeof(UIntType) * 8 >= FPBits<T>::MANTISSA_WIDTH + 1,
                 "Bad type for mantissa in NormalFloat.");
 
   bool sign;
@@ -92,7 +92,7 @@ template <typename T> struct NormalFloat {
   LIBC_INLINE operator T() const {
     int biased_exponent = exponent + FPBits<T>::EXPONENT_BIAS;
     // Max exponent is of the form 0xFF...E. That is why -2 and not -1.
-    constexpr int MAX_EXPONENT_VALUE = (1 << ExponentWidth<T>::VALUE) - 2;
+    constexpr int MAX_EXPONENT_VALUE = (1 << FPBits<T>::EXPONENT_WIDTH) - 2;
     if (biased_exponent > MAX_EXPONENT_VALUE) {
       return sign ? T(FPBits<T>::neg_inf()) : T(FPBits<T>::inf());
     }
@@ -105,7 +105,7 @@ template <typename T> struct NormalFloat {
       unsigned shift = SUBNORMAL_EXPONENT - exponent;
       // Since exponent > subnormalExponent, shift is strictly greater than
       // zero.
-      if (shift <= MantissaWidth<T>::VALUE + 1) {
+      if (shift <= FPBits<T>::MANTISSA_WIDTH + 1) {
         // Generate a subnormal number. Might lead to loss of precision.
         // We round to nearest and round halfway cases to even.
         const UIntType shift_out_mask = (UIntType(1) << shift) - 1;
@@ -163,7 +163,7 @@ private:
 
   LIBC_INLINE unsigned evaluate_normalization_shift(UIntType m) {
     unsigned shift = 0;
-    for (; (ONE & m) == 0 && (shift < MantissaWidth<T>::VALUE);
+    for (; (ONE & m) == 0 && (shift < FPBits<T>::MANTISSA_WIDTH);
          m <<= 1, ++shift)
       ;
     return shift;
@@ -208,21 +208,21 @@ NormalFloat<long double>::init_from_bits(FPBits<long double> bits) {
 }
 
 template <> LIBC_INLINE NormalFloat<long double>::operator long double() const {
-  int biased_exponent = exponent + FPBits<long double>::EXPONENT_BIAS;
+  using LDBits = FPBits<long double>;
+  int biased_exponent = exponent + LDBits::EXPONENT_BIAS;
   // Max exponent is of the form 0xFF...E. That is why -2 and not -1.
-  constexpr int MAX_EXPONENT_VALUE =
-      (1 << ExponentWidth<long double>::VALUE) - 2;
+  constexpr int MAX_EXPONENT_VALUE = (1 << LDBits::EXPONENT_WIDTH) - 2;
   if (biased_exponent > MAX_EXPONENT_VALUE) {
-    return sign ? FPBits<long double>::neg_inf() : FPBits<long double>::inf();
+    return sign ? LDBits::neg_inf() : LDBits::inf();
   }
 
   FPBits<long double> result(0.0l);
   result.set_sign(sign);
 
-  constexpr int SUBNORMAL_EXPONENT = -FPBits<long double>::EXPONENT_BIAS + 1;
+  constexpr int SUBNORMAL_EXPONENT = -LDBits::EXPONENT_BIAS + 1;
   if (exponent < SUBNORMAL_EXPONENT) {
     unsigned shift = SUBNORMAL_EXPONENT - exponent;
-    if (shift <= MantissaWidth<long double>::VALUE + 1) {
+    if (shift <= LDBits::MANTISSA_WIDTH + 1) {
       // Generate a subnormal number. Might lead to loss of precision.
       // We round to nearest and round halfway cases to even.
       const UIntType shift_out_mask = (UIntType(1) << shift) - 1;
diff --git a/libc/src/__support/FPUtil/generic/FMA.h b/libc/src/__support/FPUtil/generic/FMA.h
index 61a1401c30e827c63882c7f3fdf0ceec2a593124..3c4d943a7c71fb64d0428528e197ad45639a9eda 100644
--- a/libc/src/__support/FPUtil/generic/FMA.h
+++ b/libc/src/__support/FPUtil/generic/FMA.h
@@ -159,11 +159,10 @@ template <> LIBC_INLINE double fma<double>(double x, double y, double z) {
 
   UInt128 prod_mant = x_mant * y_mant << 10;
   int prod_lsb_exp =
-      x_exp + y_exp -
-      (FPBits::EXPONENT_BIAS + 2 * MantissaWidth<double>::VALUE + 10);
+      x_exp + y_exp - (FPBits::EXPONENT_BIAS + 2 * FPBits::MANTISSA_WIDTH + 10);
 
   z_mant <<= 64;
-  int z_lsb_exp = z_exp - (MantissaWidth<double>::VALUE + 64);
+  int z_lsb_exp = z_exp - (FPBits::MANTISSA_WIDTH + 64);
   bool round_bit = false;
   bool sticky_bits = false;
   bool z_shifted = false;
diff --git a/libc/src/__support/FPUtil/generic/FMod.h b/libc/src/__support/FPUtil/generic/FMod.h
index 7502660c88a133da54b71fc870c62318c6f36374..f30586f9d7f341ed226e6f307e1ddee55c69249f 100644
--- a/libc/src/__support/FPUtil/generic/FMod.h
+++ b/libc/src/__support/FPUtil/generic/FMod.h
@@ -167,11 +167,11 @@ template <typename T> struct FModFastMathWrapper {
 
 template <typename T> class FModDivisionSimpleHelper {
 private:
-  using intU_t = typename FPBits<T>::UIntType;
+  using UIntType = typename FPBits<T>::UIntType;
 
 public:
-  LIBC_INLINE constexpr static intU_t
-  execute(int exp_diff, int sides_zeroes_count, intU_t m_x, intU_t m_y) {
+  LIBC_INLINE constexpr static UIntType
+  execute(int exp_diff, int sides_zeroes_count, UIntType m_x, UIntType m_y) {
     while (exp_diff > sides_zeroes_count) {
       exp_diff -= sides_zeroes_count;
       m_x <<= sides_zeroes_count;
@@ -186,23 +186,22 @@ public:
 template <typename T> class FModDivisionInvMultHelper {
 private:
   using FPB = FPBits<T>;
-  using intU_t = typename FPB::UIntType;
+  using UIntType = typename FPB::UIntType;
 
 public:
-  LIBC_INLINE constexpr static intU_t
-  execute(int exp_diff, int sides_zeroes_count, intU_t m_x, intU_t m_y) {
+  LIBC_INLINE constexpr static UIntType
+  execute(int exp_diff, int sides_zeroes_count, UIntType m_x, UIntType m_y) {
     if (exp_diff > sides_zeroes_count) {
-      intU_t inv_hy = (cpp::numeric_limits<intU_t>::max() / m_y);
+      UIntType inv_hy = (cpp::numeric_limits<UIntType>::max() / m_y);
       while (exp_diff > sides_zeroes_count) {
         exp_diff -= sides_zeroes_count;
-        intU_t hd =
-            (m_x * inv_hy) >> (FPB::FloatProp::BIT_WIDTH - sides_zeroes_count);
+        UIntType hd = (m_x * inv_hy) >> (FPB::BIT_WIDTH - sides_zeroes_count);
         m_x <<= sides_zeroes_count;
         m_x -= hd * m_y;
         while (LIBC_UNLIKELY(m_x > m_y))
           m_x -= m_y;
       }
-      intU_t hd = (m_x * inv_hy) >> (FPB::FloatProp::BIT_WIDTH - exp_diff);
+      UIntType hd = (m_x * inv_hy) >> (FPB::BIT_WIDTH - exp_diff);
       m_x <<= exp_diff;
       m_x -= hd * m_y;
       while (LIBC_UNLIKELY(m_x > m_y))
@@ -223,7 +222,7 @@ class FMod {
 
 private:
   using FPB = FPBits<T>;
-  using intU_t = typename FPB::UIntType;
+  using UIntType = typename FPB::UIntType;
 
   LIBC_INLINE static constexpr FPB eval_internal(FPB sx, FPB sy) {
 
@@ -237,11 +236,11 @@ private:
     int e_y = sy.get_biased_exponent();
 
     // Most common case where |y| is "very normal" and |x/y| < 2^EXPONENT_WIDTH
-    if (LIBC_LIKELY(e_y > int(FPB::FloatProp::MANTISSA_WIDTH) &&
-                    e_x - e_y <= int(FPB::FloatProp::EXPONENT_WIDTH))) {
-      intU_t m_x = sx.get_explicit_mantissa();
-      intU_t m_y = sy.get_explicit_mantissa();
-      intU_t d = (e_x == e_y) ? (m_x - m_y) : (m_x << (e_x - e_y)) % m_y;
+    if (LIBC_LIKELY(e_y > int(FPB::MANTISSA_WIDTH) &&
+                    e_x - e_y <= int(FPB::EXPONENT_WIDTH))) {
+      UIntType m_x = sx.get_explicit_mantissa();
+      UIntType m_y = sy.get_explicit_mantissa();
+      UIntType d = (e_x == e_y) ? (m_x - m_y) : (m_x << (e_x - e_y)) % m_y;
       if (d == 0)
         return FPB(FPB::zero());
       // iy - 1 because of "zero power" for number with power 1
@@ -255,11 +254,11 @@ private:
     }
 
     // Note that hx is not subnormal by conditions above.
-    intU_t m_x = sx.get_explicit_mantissa();
+    UIntType m_x = sx.get_explicit_mantissa();
     e_x--;
 
-    intU_t m_y = sy.get_explicit_mantissa();
-    int lead_zeros_m_y = FPB::FloatProp::EXPONENT_WIDTH;
+    UIntType m_y = sy.get_explicit_mantissa();
+    int lead_zeros_m_y = FPB::EXPONENT_WIDTH;
     if (LIBC_LIKELY(e_y > 0)) {
       e_y--;
     } else {
@@ -282,9 +281,8 @@ private:
 
     {
       // Shift hx left until the end or n = 0
-      int left_shift = exp_diff < int(FPB::FloatProp::EXPONENT_WIDTH)
-                           ? exp_diff
-                           : FPB::FloatProp::EXPONENT_WIDTH;
+      int left_shift =
+          exp_diff < int(FPB::EXPONENT_WIDTH) ? exp_diff : FPB::EXPONENT_WIDTH;
       m_x <<= left_shift;
       exp_diff -= left_shift;
     }
diff --git a/libc/src/__support/FPUtil/generic/sqrt.h b/libc/src/__support/FPUtil/generic/sqrt.h
index 5bde9589fdc012c5686656f21d3d7c515f2b5a5a..cd5ec58bcdbd5fe94d78b782908f9cf90e2ededf 100644
--- a/libc/src/__support/FPUtil/generic/sqrt.h
+++ b/libc/src/__support/FPUtil/generic/sqrt.h
@@ -37,7 +37,7 @@ template <typename T>
 LIBC_INLINE void normalize(int &exponent,
                            typename FPBits<T>::UIntType &mantissa) {
   const int shift = cpp::countl_zero(mantissa) -
-                    (8 * sizeof(mantissa) - 1 - MantissaWidth<T>::VALUE);
+                    (8 * sizeof(mantissa) - 1 - FPBits<T>::MANTISSA_WIDTH);
   exponent -= shift;
   mantissa <<= shift;
 }
@@ -72,7 +72,7 @@ LIBC_INLINE cpp::enable_if_t<cpp::is_floating_point_v<T>, T> sqrt(T x) {
   } else {
     // IEEE floating points formats.
     using UIntType = typename FPBits<T>::UIntType;
-    constexpr UIntType ONE = UIntType(1) << MantissaWidth<T>::VALUE;
+    constexpr UIntType ONE = UIntType(1) << FPBits<T>::MANTISSA_WIDTH;
 
     FPBits<T> bits(x);
 
@@ -147,7 +147,8 @@ LIBC_INLINE cpp::enable_if_t<cpp::is_floating_point_v<T>, T> sqrt(T x) {
       // Remove hidden bit and append the exponent field.
       x_exp = ((x_exp >> 1) + FPBits<T>::EXPONENT_BIAS);
 
-      y = (y - ONE) | (static_cast<UIntType>(x_exp) << MantissaWidth<T>::VALUE);
+      y = (y - ONE) |
+          (static_cast<UIntType>(x_exp) << FPBits<T>::MANTISSA_WIDTH);
 
       switch (quick_get_round()) {
       case FE_TONEAREST:
diff --git a/libc/src/__support/FPUtil/generic/sqrt_80_bit_long_double.h b/libc/src/__support/FPUtil/generic/sqrt_80_bit_long_double.h
index 2f25be54e0bc3626085687fac6b3ae50e1942eb5..46ca796aeb4b60e29d8e53471066767f14753482 100644
--- a/libc/src/__support/FPUtil/generic/sqrt_80_bit_long_double.h
+++ b/libc/src/__support/FPUtil/generic/sqrt_80_bit_long_double.h
@@ -23,7 +23,7 @@ namespace x86 {
 LIBC_INLINE void normalize(int &exponent, UInt128 &mantissa) {
   const unsigned int shift = static_cast<unsigned int>(
       cpp::countl_zero(static_cast<uint64_t>(mantissa)) -
-      (8 * sizeof(uint64_t) - 1 - MantissaWidth<long double>::VALUE));
+      (8 * sizeof(uint64_t) - 1 - FPBits<long double>::MANTISSA_WIDTH));
   exponent -= shift;
   mantissa <<= shift;
 }
@@ -36,16 +36,16 @@ LIBC_INLINE long double sqrt(long double x);
 // Shift-and-add algorithm.
 #if defined(LIBC_LONG_DOUBLE_IS_X86_FLOAT80)
 LIBC_INLINE long double sqrt(long double x) {
-  using UIntType = typename FPBits<long double>::UIntType;
-  constexpr UIntType ONE = UIntType(1)
-                           << int(MantissaWidth<long double>::VALUE);
+  using LDBits = FPBits<long double>;
+  using UIntType = typename LDBits::UIntType;
+  constexpr UIntType ONE = UIntType(1) << int(LDBits::MANTISSA_WIDTH);
 
   FPBits<long double> bits(x);
 
   if (bits.is_inf_or_nan()) {
     if (bits.get_sign() && (bits.get_mantissa() == 0)) {
       // sqrt(-Inf) = NaN
-      return FPBits<long double>::build_quiet_nan(ONE >> 1);
+      return LDBits::build_quiet_nan(ONE >> 1);
     } else {
       // sqrt(NaN) = NaN
       // sqrt(+Inf) = +Inf
@@ -57,7 +57,7 @@ LIBC_INLINE long double sqrt(long double x) {
     return x;
   } else if (bits.get_sign()) {
     // sqrt( negative numbers ) = NaN
-    return FPBits<long double>::build_quiet_nan(ONE >> 1);
+    return LDBits::build_quiet_nan(ONE >> 1);
   } else {
     int x_exp = bits.get_explicit_exponent();
     UIntType x_mant = bits.get_mantissa();
@@ -110,9 +110,8 @@ LIBC_INLINE long double sqrt(long double x) {
     }
 
     // Append the exponent field.
-    x_exp = ((x_exp >> 1) + FPBits<long double>::EXPONENT_BIAS);
-    y |= (static_cast<UIntType>(x_exp)
-          << (MantissaWidth<long double>::VALUE + 1));
+    x_exp = ((x_exp >> 1) + LDBits::EXPONENT_BIAS);
+    y |= (static_cast<UIntType>(x_exp) << (LDBits::MANTISSA_WIDTH + 1));
 
     switch (quick_get_round()) {
     case FE_TONEAREST:
diff --git a/libc/src/__support/FPUtil/x86_64/LongDoubleBits.h b/libc/src/__support/FPUtil/x86_64/LongDoubleBits.h
index f1ef928f2308194321eb5913e8e0eba32aeccb72..a31667528be2b0081c94eabcd73c3e810348b32f 100644
--- a/libc/src/__support/FPUtil/x86_64/LongDoubleBits.h
+++ b/libc/src/__support/FPUtil/x86_64/LongDoubleBits.h
@@ -26,74 +26,73 @@
 namespace LIBC_NAMESPACE {
 namespace fputil {
 
-template <> struct FPBits<long double> {
-  using UIntType = UInt128;
+template <> struct FPBits<long double> : private FloatProperties<long double> {
+  using typename FloatProperties<long double>::UIntType;
+  using FloatProperties<long double>::BIT_WIDTH;
+  using FloatProperties<long double>::EXP_MANT_MASK;
+  using FloatProperties<long double>::EXPONENT_MASK;
+  using FloatProperties<long double>::EXPONENT_BIAS;
+  using FloatProperties<long double>::EXPONENT_WIDTH;
+  using FloatProperties<long double>::MANTISSA_MASK;
+  using FloatProperties<long double>::MANTISSA_WIDTH;
+  using FloatProperties<long double>::QUIET_NAN_MASK;
+  using FloatProperties<long double>::SIGN_MASK;
 
-  static constexpr int EXPONENT_BIAS = 0x3FFF;
   static constexpr int MAX_EXPONENT = 0x7FFF;
   static constexpr UIntType MIN_SUBNORMAL = UIntType(1);
   // Subnormal numbers include the implicit bit in x86 long double formats.
-  static constexpr UIntType MAX_SUBNORMAL =
-      (UIntType(1) << (MantissaWidth<long double>::VALUE)) - 1;
-  static constexpr UIntType MIN_NORMAL =
-      (UIntType(3) << MantissaWidth<long double>::VALUE);
+  static constexpr UIntType MAX_SUBNORMAL = (UIntType(1) << MANTISSA_WIDTH) - 1;
+  static constexpr UIntType MIN_NORMAL = (UIntType(3) << MANTISSA_WIDTH);
   static constexpr UIntType MAX_NORMAL =
-      (UIntType(MAX_EXPONENT - 1) << (MantissaWidth<long double>::VALUE + 1)) |
-      (UIntType(1) << MantissaWidth<long double>::VALUE) | MAX_SUBNORMAL;
-
-  using FloatProp = FloatProperties<long double>;
+      (UIntType(MAX_EXPONENT - 1) << (MANTISSA_WIDTH + 1)) |
+      (UIntType(1) << MANTISSA_WIDTH) | MAX_SUBNORMAL;
 
   UIntType bits;
 
   LIBC_INLINE constexpr void set_mantissa(UIntType mantVal) {
-    mantVal &= (FloatProp::MANTISSA_MASK);
-    bits &= ~(FloatProp::MANTISSA_MASK);
+    mantVal &= MANTISSA_MASK;
+    bits &= ~MANTISSA_MASK;
     bits |= mantVal;
   }
 
   LIBC_INLINE constexpr UIntType get_mantissa() const {
-    return bits & FloatProp::MANTISSA_MASK;
+    return bits & MANTISSA_MASK;
   }
 
   LIBC_INLINE constexpr UIntType get_explicit_mantissa() const {
     // The x86 80 bit float represents the leading digit of the mantissa
     // explicitly. This is the mask for that bit.
-    constexpr UIntType EXPLICIT_BIT_MASK =
-        (UIntType(1) << FloatProp::MANTISSA_WIDTH);
-    return bits & (FloatProp::MANTISSA_MASK | EXPLICIT_BIT_MASK);
+    constexpr UIntType EXPLICIT_BIT_MASK = UIntType(1) << MANTISSA_WIDTH;
+    return bits & (MANTISSA_MASK | EXPLICIT_BIT_MASK);
   }
 
   LIBC_INLINE constexpr void set_biased_exponent(UIntType expVal) {
-    expVal =
-        (expVal << (FloatProp::BIT_WIDTH - 1 - FloatProp::EXPONENT_WIDTH)) &
-        FloatProp::EXPONENT_MASK;
-    bits &= ~(FloatProp::EXPONENT_MASK);
+    expVal = (expVal << (BIT_WIDTH - 1 - EXPONENT_WIDTH)) & EXPONENT_MASK;
+    bits &= ~EXPONENT_MASK;
     bits |= expVal;
   }
 
   LIBC_INLINE constexpr uint16_t get_biased_exponent() const {
-    return uint16_t((bits & FloatProp::EXPONENT_MASK) >>
-                    (FloatProp::BIT_WIDTH - 1 - FloatProp::EXPONENT_WIDTH));
+    return uint16_t((bits & EXPONENT_MASK) >> (BIT_WIDTH - 1 - EXPONENT_WIDTH));
   }
 
   LIBC_INLINE constexpr void set_implicit_bit(bool implicitVal) {
-    bits &= ~(UIntType(1) << FloatProp::MANTISSA_WIDTH);
-    bits |= (UIntType(implicitVal) << FloatProp::MANTISSA_WIDTH);
+    bits &= ~(UIntType(1) << MANTISSA_WIDTH);
+    bits |= (UIntType(implicitVal) << MANTISSA_WIDTH);
   }
 
   LIBC_INLINE constexpr bool get_implicit_bit() const {
-    return bool((bits & (UIntType(1) << FloatProp::MANTISSA_WIDTH)) >>
-                FloatProp::MANTISSA_WIDTH);
+    return bool((bits & (UIntType(1) << MANTISSA_WIDTH)) >> MANTISSA_WIDTH);
   }
 
   LIBC_INLINE constexpr void set_sign(bool signVal) {
-    bits &= ~(FloatProp::SIGN_MASK);
-    UIntType sign1 = UIntType(signVal) << (FloatProp::BIT_WIDTH - 1);
+    bits &= ~SIGN_MASK;
+    UIntType sign1 = UIntType(signVal) << (BIT_WIDTH - 1);
     bits |= sign1;
   }
 
   LIBC_INLINE constexpr bool get_sign() const {
-    return bool((bits & FloatProp::SIGN_MASK) >> (FloatProp::BIT_WIDTH - 1));
+    return bool((bits & SIGN_MASK) >> (BIT_WIDTH - 1));
   }
 
   LIBC_INLINE constexpr FPBits() : bits(0) {}
@@ -117,7 +116,7 @@ template <> struct FPBits<long double> {
 
   LIBC_INLINE constexpr UIntType uintval() {
     // We zero the padding bits as they can contain garbage.
-    return bits & FloatProp::FP_MASK;
+    return bits & FP_MASK;
   }
 
   LIBC_INLINE constexpr long double get_val() const {
@@ -196,7 +195,7 @@ template <> struct FPBits<long double> {
   }
 
   LIBC_INLINE static constexpr long double build_quiet_nan(UIntType v) {
-    return build_nan(FloatProp::QUIET_NAN_MASK | v);
+    return build_nan(QUIET_NAN_MASK | v);
   }
 
   LIBC_INLINE static constexpr long double min_normal() {
diff --git a/libc/src/__support/FPUtil/x86_64/NextAfterLongDouble.h b/libc/src/__support/FPUtil/x86_64/NextAfterLongDouble.h
index 5e32f766ad58636f3efa75119462de8380e06458..653b6a48f3cc53b212f96878e6b4b36c0be6517b 100644
--- a/libc/src/__support/FPUtil/x86_64/NextAfterLongDouble.h
+++ b/libc/src/__support/FPUtil/x86_64/NextAfterLongDouble.h
@@ -46,7 +46,7 @@ LIBC_INLINE long double nextafter(long double from, long double to) {
   using UIntType = FPBits::UIntType;
   constexpr UIntType SIGN_VAL = (UIntType(1) << 79);
   constexpr UIntType MANTISSA_MASK =
-      (UIntType(1) << MantissaWidth<long double>::VALUE) - 1;
+      (UIntType(1) << FPBits::MANTISSA_WIDTH) - 1;
   UIntType int_val = from_bits.uintval();
   if (from < 0.0l) {
     if (from > to) {
@@ -117,8 +117,7 @@ LIBC_INLINE long double nextafter(long double from, long double to) {
     }
   }
 
-  UIntType implicit_bit =
-      int_val & (UIntType(1) << MantissaWidth<long double>::VALUE);
+  UIntType implicit_bit = int_val & (UIntType(1) << FPBits::MANTISSA_WIDTH);
   if (implicit_bit == UIntType(0))
     raise_except_if_required(FE_UNDERFLOW | FE_INEXACT);
 
diff --git a/libc/src/__support/File/linux/file.cpp b/libc/src/__support/File/linux/file.cpp
index 2d4cea5b53c5815c8ff9a3f5a1387a27c8b17c2a..b84da64cbe6358957a86b93d33aa5a0044693f20 100644
--- a/libc/src/__support/File/linux/file.cpp
+++ b/libc/src/__support/File/linux/file.cpp
@@ -17,6 +17,7 @@
 
 #include <fcntl.h> // For mode_t and other flags to the open syscall
 #include <stdio.h>
+#include <sys/stat.h>    // For S_IS*, S_IF*, and S_IR* flags.
 #include <sys/syscall.h> // For syscall numbers
 
 namespace LIBC_NAMESPACE {
diff --git a/libc/src/__support/float_to_string.h b/libc/src/__support/float_to_string.h
index be105830a91ac1c25e7b9ed47ed34846c98345b5..d53bb4b4c62b15958fd7a96feac8bdb6d272f524 100644
--- a/libc/src/__support/float_to_string.h
+++ b/libc/src/__support/float_to_string.h
@@ -416,7 +416,7 @@ class FloatToString {
   int exponent;
   FloatProp::UIntType mantissa;
 
-  static constexpr int MANT_WIDTH = fputil::MantissaWidth<T>::VALUE;
+  static constexpr int MANT_WIDTH = fputil::FPBits<T>::MANTISSA_WIDTH;
   static constexpr int EXP_BIAS = fputil::FPBits<T>::EXPONENT_BIAS;
 
 public:
diff --git a/libc/src/__support/str_to_float.h b/libc/src/__support/str_to_float.h
index 2a6f15c018f1ecbd0707d46957fa8ab3df407e99..3807d3ff572162ddc633ead9a31e5a9c35258b16 100644
--- a/libc/src/__support/str_to_float.h
+++ b/libc/src/__support/str_to_float.h
@@ -71,7 +71,7 @@ LIBC_INLINE cpp::optional<ExpandedFloat<T>>
 eisel_lemire(ExpandedFloat<T> init_num,
              RoundDirection round = RoundDirection::Nearest) {
   using FPBits = typename fputil::FPBits<T>;
-  using FloatProp = typename FPBits::FloatProp;
+  using FloatProp = typename fputil::FloatProperties<T>;
   using UIntType = typename FPBits::UIntType;
 
   UIntType mantissa = init_num.mantissa;
@@ -184,7 +184,7 @@ LIBC_INLINE cpp::optional<ExpandedFloat<long double>>
 eisel_lemire<long double>(ExpandedFloat<long double> init_num,
                           RoundDirection round) {
   using FPBits = typename fputil::FPBits<long double>;
-  using FloatProp = typename FPBits::FloatProp;
+  using FloatProp = typename fputil::FloatProperties<long double>;
   using UIntType = typename FPBits::UIntType;
 
   UIntType mantissa = init_num.mantissa;
@@ -322,7 +322,7 @@ LIBC_INLINE FloatConvertReturn<T>
 simple_decimal_conversion(const char *__restrict numStart,
                           RoundDirection round = RoundDirection::Nearest) {
   using FPBits = typename fputil::FPBits<T>;
-  using FloatProp = typename FPBits::FloatProp;
+  using FloatProp = typename fputil::FloatProperties<T>;
   using UIntType = typename FPBits::UIntType;
 
   int32_t exp2 = 0;
@@ -516,7 +516,7 @@ LIBC_INLINE cpp::optional<ExpandedFloat<T>>
 clinger_fast_path(ExpandedFloat<T> init_num,
                   RoundDirection round = RoundDirection::Nearest) {
   using FPBits = typename fputil::FPBits<T>;
-  using FloatProp = typename FPBits::FloatProp;
+  using FloatProp = typename fputil::FloatProperties<T>;
   using UIntType = typename FPBits::UIntType;
 
   UIntType mantissa = init_num.mantissa;
@@ -724,7 +724,7 @@ LIBC_INLINE FloatConvertReturn<T> binary_exp_to_float(ExpandedFloat<T> init_num,
                                                       bool truncated,
                                                       RoundDirection round) {
   using FPBits = typename fputil::FPBits<T>;
-  using FloatProp = typename FPBits::FloatProp;
+  using FloatProp = typename fputil::FloatProperties<T>;
   using UIntType = typename FPBits::UIntType;
 
   UIntType mantissa = init_num.mantissa;
diff --git a/libc/src/math/generic/acoshf.cpp b/libc/src/math/generic/acoshf.cpp
index 9438be1bee74eb4614739319a35f200b17c5f568..142c17795d083a2cbfa359935539fc207cd50956 100644
--- a/libc/src/math/generic/acoshf.cpp
+++ b/libc/src/math/generic/acoshf.cpp
@@ -34,7 +34,7 @@ LLVM_LIBC_FUNCTION(float, acoshf, (float x)) {
 
   if (LIBC_UNLIKELY(x_u >= 0x4f8ffb03)) {
     // Check for exceptional values.
-    uint32_t x_abs = x_u & FPBits_t::FloatProp::EXP_MANT_MASK;
+    uint32_t x_abs = x_u & FPBits_t::EXP_MANT_MASK;
     if (LIBC_UNLIKELY(x_abs >= 0x7f80'0000U)) {
       // x is +inf or NaN.
       return x;
diff --git a/libc/src/math/generic/asinf.cpp b/libc/src/math/generic/asinf.cpp
index f40a08e752ed3989247f3462cc77f79f90591e55..5406e6660d78415370547dc030e8339c629633ec 100644
--- a/libc/src/math/generic/asinf.cpp
+++ b/libc/src/math/generic/asinf.cpp
@@ -108,8 +108,7 @@ LLVM_LIBC_FUNCTION(float, asinf, (float x)) {
       fputil::set_errno_if_required(EDOM);
       fputil::raise_except_if_required(FE_INVALID);
     }
-    return x +
-           FPBits::build_nan(1 << (fputil::MantissaWidth<float>::VALUE - 1));
+    return x + FPBits::build_nan(1 << (FPBits::MANTISSA_WIDTH - 1));
   }
 
   // Check for exceptional values
diff --git a/libc/src/math/generic/asinhf.cpp b/libc/src/math/generic/asinhf.cpp
index 6bde08d42a429cdd3533b0ea389399c8c95709d9..5b2f63d3fe144e9700536d5716eb57ff3e1128b2 100644
--- a/libc/src/math/generic/asinhf.cpp
+++ b/libc/src/math/generic/asinhf.cpp
@@ -21,7 +21,7 @@ LLVM_LIBC_FUNCTION(float, asinhf, (float x)) {
   using FPBits_t = typename fputil::FPBits<float>;
   FPBits_t xbits(x);
   uint32_t x_u = xbits.uintval();
-  uint32_t x_abs = x_u & FPBits_t::FloatProp::EXP_MANT_MASK;
+  uint32_t x_abs = x_u & FPBits_t::EXP_MANT_MASK;
 
   // |x| <= 2^-3
   if (LIBC_UNLIKELY(x_abs <= 0x3e80'0000U)) {
diff --git a/libc/src/math/generic/atanhf.cpp b/libc/src/math/generic/atanhf.cpp
index 839ef5b076ac35576a854c837687e160ecd4de84..dfec28e9a44a7234a27cb941080d2663b9ad4414 100644
--- a/libc/src/math/generic/atanhf.cpp
+++ b/libc/src/math/generic/atanhf.cpp
@@ -17,7 +17,7 @@ LLVM_LIBC_FUNCTION(float, atanhf, (float x)) {
   using FPBits = typename fputil::FPBits<float>;
   FPBits xbits(x);
   bool sign = xbits.get_sign();
-  uint32_t x_abs = xbits.uintval() & FPBits::FloatProp::EXP_MANT_MASK;
+  uint32_t x_abs = xbits.uintval() & FPBits::EXP_MANT_MASK;
 
   // |x| >= 1.0
   if (LIBC_UNLIKELY(x_abs >= 0x3F80'0000U)) {
diff --git a/libc/src/math/generic/erff.cpp b/libc/src/math/generic/erff.cpp
index a7b0897c3b58cb76a900c1b5429fac53e145ad49..d63fb8e31384d2415343b02d2de257d9de83bbf2 100644
--- a/libc/src/math/generic/erff.cpp
+++ b/libc/src/math/generic/erff.cpp
@@ -154,7 +154,7 @@ LLVM_LIBC_FUNCTION(float, erff, (float x)) {
   double xd = static_cast<double>(x);
   double xsq = xd * xd;
 
-  const uint32_t EIGHT = 3 << FPBits::FloatProp::MANTISSA_WIDTH;
+  const uint32_t EIGHT = 3 << FPBits::MANTISSA_WIDTH;
   int idx = static_cast<int>(FPBits(x_abs + EIGHT).get_val());
 
   double x4 = xsq * xsq;
diff --git a/libc/src/math/generic/explogxf.h b/libc/src/math/generic/explogxf.h
index 77ec9cb94e085455355f4a4e776516268e25e618..3dae5af068b4b010c0f49c93cfa5ca3812ef73cf 100644
--- a/libc/src/math/generic/explogxf.h
+++ b/libc/src/math/generic/explogxf.h
@@ -280,12 +280,11 @@ LIBC_INLINE static double log2_eval(double x) {
   double result = 0;
   result += bs.get_exponent();
 
-  int p1 =
-      (bs.get_mantissa() >> (FPB::FloatProp::MANTISSA_WIDTH - LOG_P1_BITS)) &
-      (LOG_P1_SIZE - 1);
+  int p1 = (bs.get_mantissa() >> (FPB::MANTISSA_WIDTH - LOG_P1_BITS)) &
+           (LOG_P1_SIZE - 1);
 
-  bs.bits &= FPB::FloatProp::MANTISSA_MASK >> LOG_P1_BITS;
-  bs.set_biased_exponent(FPB::FloatProp::EXPONENT_BIAS);
+  bs.bits &= FPB::MANTISSA_MASK >> LOG_P1_BITS;
+  bs.set_biased_exponent(FPB::EXPONENT_BIAS);
   double dx = (bs.get_val() - 1.0) * LOG_P1_1_OVER[p1];
 
   // Taylor series for log(2,1+x)
@@ -311,12 +310,11 @@ LIBC_INLINE static double log_eval(double x) {
 
   // p1 is the leading 7 bits of mx, i.e.
   // p1 * 2^(-7) <= m_x < (p1 + 1) * 2^(-7).
-  int p1 = static_cast<int>(bs.get_mantissa() >>
-                            (FPB::FloatProp::MANTISSA_WIDTH - 7));
+  int p1 = static_cast<int>(bs.get_mantissa() >> (FPB::MANTISSA_WIDTH - 7));
 
   // Set bs to (1 + (mx - p1*2^(-7))
-  bs.bits &= FPB::FloatProp::MANTISSA_MASK >> 7;
-  bs.set_biased_exponent(FPB::FloatProp::EXPONENT_BIAS);
+  bs.bits &= FPB::MANTISSA_MASK >> 7;
+  bs.set_biased_exponent(FPB::EXPONENT_BIAS);
   // dx = (mx - p1*2^(-7)) / (1 + p1*2^(-7)).
   double dx = (bs.get_val() - 1.0) * ONE_OVER_F[p1];
 
diff --git a/libc/src/math/generic/hypotf.cpp b/libc/src/math/generic/hypotf.cpp
index 389de3c450299d479ad4c53a1daf8e9568b79ab3..5795291a043ae6eeb944cfedcedf84ea58fea058 100644
--- a/libc/src/math/generic/hypotf.cpp
+++ b/libc/src/math/generic/hypotf.cpp
@@ -23,7 +23,7 @@ LLVM_LIBC_FUNCTION(float, hypotf, (float x, float y)) {
   uint16_t y_exp = y_bits.get_biased_exponent();
   uint16_t exp_diff = (x_exp > y_exp) ? (x_exp - y_exp) : (y_exp - x_exp);
 
-  if (exp_diff >= fputil::MantissaWidth<float>::VALUE + 2) {
+  if (exp_diff >= FPBits::MANTISSA_WIDTH + 2) {
     return fputil::abs(x) + fputil::abs(y);
   }
 
diff --git a/libc/src/math/generic/inv_trigf_utils.h b/libc/src/math/generic/inv_trigf_utils.h
index c88ded20b5bf24f4da93451cd1cad2d22604ba62..2ecd033ecbed6d5816e8404009097f9e6fbb84c9 100644
--- a/libc/src/math/generic/inv_trigf_utils.h
+++ b/libc/src/math/generic/inv_trigf_utils.h
@@ -51,7 +51,7 @@ LIBC_INLINE double atan_eval(double x) {
 
   FPB bs(x);
   bool sign = bs.get_sign();
-  auto x_abs = bs.uintval() & FPB::FloatProp::EXP_MANT_MASK;
+  auto x_abs = bs.uintval() & FPB::EXP_MANT_MASK;
 
   if (x_abs <= umin) {
     double pe = LIBC_NAMESPACE::fputil::polyeval(
@@ -64,7 +64,8 @@ LIBC_INLINE double atan_eval(double x) {
     double one_over_x2 = one_over_x_m * one_over_x_m;
     double pe = LIBC_NAMESPACE::fputil::polyeval(
         one_over_x2, ATAN_K[0], ATAN_K[1], ATAN_K[2], ATAN_K[3]);
-    return fputil::multiply_add(pe, one_over_x_m, sign ? (-M_MATH_PI_2) : (M_MATH_PI_2));
+    return fputil::multiply_add(pe, one_over_x_m,
+                                sign ? (-M_MATH_PI_2) : (M_MATH_PI_2));
   }
 
   double pos_x = FPB(x_abs).get_val();
diff --git a/libc/src/math/generic/log1p.cpp b/libc/src/math/generic/log1p.cpp
index c8b45fd57b42f8803bdd1102e7ff31085d6b89a1..757f2793cb9cf794ac3f574a0c8cae3e5ac416fd 100644
--- a/libc/src/math/generic/log1p.cpp
+++ b/libc/src/math/generic/log1p.cpp
@@ -873,8 +873,8 @@ LIBC_INLINE double log1p_accurate(int e_x, int index,
 LLVM_LIBC_FUNCTION(double, log1p, (double x)) {
   using FPBits_t = typename fputil::FPBits<double>;
   constexpr int EXPONENT_BIAS = FPBits_t::EXPONENT_BIAS;
-  constexpr int MANTISSA_WIDTH = FPBits_t::FloatProp::MANTISSA_WIDTH;
-  constexpr uint64_t MANTISSA_MASK = FPBits_t::FloatProp::MANTISSA_MASK;
+  constexpr int MANTISSA_WIDTH = FPBits_t::MANTISSA_WIDTH;
+  constexpr uint64_t MANTISSA_MASK = FPBits_t::MANTISSA_MASK;
   FPBits_t xbits(x);
   uint64_t x_u = xbits.uintval();
 
@@ -969,7 +969,7 @@ LLVM_LIBC_FUNCTION(double, log1p, (double x)) {
   // Scaling factior = 2^(-xh_bits.get_exponent())
   uint64_t s_u =
       (static_cast<uint64_t>(EXPONENT_BIAS) << (MANTISSA_WIDTH + 1)) -
-      (x_u & FPBits_t::FloatProp::EXPONENT_MASK);
+      (x_u & FPBits_t::EXPONENT_MASK);
   // When the exponent of x is 2^1023, its inverse, 2^(-1023), is subnormal.
   const double EXPONENT_CORRECTION[2] = {0.0, 0x1.0p-1023};
   double scaling = FPBits_t(s_u).get_val() + EXPONENT_CORRECTION[s_u == 0];
diff --git a/libc/src/math/generic/log1pf.cpp b/libc/src/math/generic/log1pf.cpp
index fd3cf4647fd0da93f3c9bb5330b8118eb2f6047e..fabd2bdc31bf3b511ae6812e470fe476f580257b 100644
--- a/libc/src/math/generic/log1pf.cpp
+++ b/libc/src/math/generic/log1pf.cpp
@@ -56,8 +56,8 @@ LIBC_INLINE float log(double x) {
 
   // Get the 8 highest bits, use 7 bits (excluding the implicit hidden bit) for
   // lookup tables.
-  int f_index = static_cast<int>(
-      xbits.get_mantissa() >> 45); // fputil::MantissaWidth<double>::VALUE - 7
+  int f_index = static_cast<int>(xbits.get_mantissa() >>
+                                 (fputil::FPBits<double>::MANTISSA_WIDTH - 7));
 
   // Set bits to 1.m
   xbits.set_biased_exponent(0x3FF);
diff --git a/libc/src/math/generic/sincosf.cpp b/libc/src/math/generic/sincosf.cpp
index 7ed1c4381078b1f2460d9106db3c26829e680203..c5694fd1bea58052043d22e8a0c3843405879e60 100644
--- a/libc/src/math/generic/sincosf.cpp
+++ b/libc/src/math/generic/sincosf.cpp
@@ -148,8 +148,7 @@ LLVM_LIBC_FUNCTION(void, sincosf, (float x, float *sinp, float *cosp)) {
       fputil::set_errno_if_required(EDOM);
       fputil::raise_except_if_required(FE_INVALID);
     }
-    *sinp =
-        x + FPBits::build_nan(1 << (fputil::MantissaWidth<float>::VALUE - 1));
+    *sinp = x + FPBits::build_nan(1 << (FPBits::MANTISSA_WIDTH - 1));
     *cosp = *sinp;
     return;
   }
diff --git a/libc/src/math/generic/sinhf.cpp b/libc/src/math/generic/sinhf.cpp
index 2f48ddbc0f88d3c47531636b343632536f935cc0..db6794620b068c7dede0e529e8c7c13adcd6aa98 100644
--- a/libc/src/math/generic/sinhf.cpp
+++ b/libc/src/math/generic/sinhf.cpp
@@ -17,7 +17,7 @@ namespace LIBC_NAMESPACE {
 LLVM_LIBC_FUNCTION(float, sinhf, (float x)) {
   using FPBits = typename fputil::FPBits<float>;
   FPBits xbits(x);
-  uint32_t x_abs = xbits.uintval() & FPBits::FloatProp::EXP_MANT_MASK;
+  uint32_t x_abs = xbits.uintval() & FPBits::EXP_MANT_MASK;
 
   // When |x| >= 90, or x is inf or nan
   if (LIBC_UNLIKELY(x_abs >= 0x42b4'0000U || x_abs <= 0x3da0'0000U)) {
@@ -57,8 +57,7 @@ LLVM_LIBC_FUNCTION(float, sinhf, (float x)) {
     int rounding = fputil::quick_get_round();
     if (sign) {
       if (LIBC_UNLIKELY(rounding == FE_UPWARD || rounding == FE_TOWARDZERO))
-        return FPBits(FPBits::MAX_NORMAL | FPBits::FloatProp::SIGN_MASK)
-            .get_val();
+        return FPBits(FPBits::MAX_NORMAL | FPBits::SIGN_MASK).get_val();
     } else {
       if (LIBC_UNLIKELY(rounding == FE_DOWNWARD || rounding == FE_TOWARDZERO))
         return FPBits(FPBits::MAX_NORMAL).get_val();
diff --git a/libc/src/math/generic/tanhf.cpp b/libc/src/math/generic/tanhf.cpp
index 7d9f86cf9044b237941896b150f16779aeee882b..a0046d3dabc62d08400f7deac3833177aa910358 100644
--- a/libc/src/math/generic/tanhf.cpp
+++ b/libc/src/math/generic/tanhf.cpp
@@ -24,7 +24,7 @@ LLVM_LIBC_FUNCTION(float, tanhf, (float x)) {
   using FPBits = typename fputil::FPBits<float>;
   FPBits xbits(x);
   uint32_t x_u = xbits.uintval();
-  uint32_t x_abs = x_u & FPBits::FloatProp::EXP_MANT_MASK;
+  uint32_t x_abs = x_u & FPBits::EXP_MANT_MASK;
 
   // When |x| >= 15, or x is inf or nan, or |x| <= 0.078125
   if (LIBC_UNLIKELY((x_abs >= 0x4170'0000U) || (x_abs <= 0x3da0'0000U))) {
diff --git a/libc/src/stdio/printf_core/float_dec_converter.h b/libc/src/stdio/printf_core/float_dec_converter.h
index ca522710040696354de0ddf22f3c04bf6aa573fb..98b573646f7c317197ce9a79a8a1227abc859387 100644
--- a/libc/src/stdio/printf_core/float_dec_converter.h
+++ b/libc/src/stdio/printf_core/float_dec_converter.h
@@ -478,7 +478,7 @@ LIBC_INLINE int convert_float_decimal_typed(Writer *writer,
                                             const FormatSection &to_conv,
                                             fputil::FPBits<T> float_bits) {
   // signed because later we use -MANT_WIDTH
-  constexpr int32_t MANT_WIDTH = fputil::MantissaWidth<T>::VALUE;
+  constexpr int32_t MANT_WIDTH = fputil::FloatProperties<T>::MANTISSA_WIDTH;
   bool is_negative = float_bits.get_sign();
   int exponent = float_bits.get_explicit_exponent();
 
@@ -591,7 +591,7 @@ LIBC_INLINE int convert_float_dec_exp_typed(Writer *writer,
                                             const FormatSection &to_conv,
                                             fputil::FPBits<T> float_bits) {
   // signed because later we use -MANT_WIDTH
-  constexpr int32_t MANT_WIDTH = fputil::MantissaWidth<T>::VALUE;
+  constexpr int32_t MANT_WIDTH = fputil::FloatProperties<T>::MANTISSA_WIDTH;
   bool is_negative = float_bits.get_sign();
   int exponent = float_bits.get_explicit_exponent();
   MantissaInt mantissa = float_bits.get_explicit_mantissa();
@@ -754,7 +754,7 @@ LIBC_INLINE int convert_float_dec_auto_typed(Writer *writer,
                                              const FormatSection &to_conv,
                                              fputil::FPBits<T> float_bits) {
   // signed because later we use -MANT_WIDTH
-  constexpr int32_t MANT_WIDTH = fputil::MantissaWidth<T>::VALUE;
+  constexpr int32_t MANT_WIDTH = fputil::FloatProperties<T>::MANTISSA_WIDTH;
   bool is_negative = float_bits.get_sign();
   int exponent = float_bits.get_explicit_exponent();
   MantissaInt mantissa = float_bits.get_explicit_mantissa();
diff --git a/libc/src/stdio/printf_core/float_hex_converter.h b/libc/src/stdio/printf_core/float_hex_converter.h
index 1f105492e8e5a3944608ce3f3384769b4581a0fd..cb10e219388be8a1a2f1532976cc5750326093f0 100644
--- a/libc/src/stdio/printf_core/float_hex_converter.h
+++ b/libc/src/stdio/printf_core/float_hex_converter.h
@@ -25,10 +25,10 @@
 namespace LIBC_NAMESPACE {
 namespace printf_core {
 
-using MantissaInt = fputil::FPBits<long double>::UIntType;
-
 LIBC_INLINE int convert_float_hex_exp(Writer *writer,
                                       const FormatSection &to_conv) {
+  using LDBits = fputil::FPBits<long double>;
+  using MantissaInt = LDBits::UIntType;
   // All of the letters will be defined relative to variable a, which will be
   // the appropriate case based on the name of the conversion. This converts any
   // conversion name into the letter 'a' with the appropriate case.
@@ -40,18 +40,19 @@ LIBC_INLINE int convert_float_hex_exp(Writer *writer,
   bool is_inf_or_nan;
   uint32_t mantissa_width;
   if (to_conv.length_modifier == LengthModifier::L) {
-    mantissa_width = fputil::MantissaWidth<long double>::VALUE;
-    fputil::FPBits<long double>::UIntType float_raw = to_conv.conv_val_raw;
-    fputil::FPBits<long double> float_bits(float_raw);
+    mantissa_width = LDBits::MANTISSA_WIDTH;
+    LDBits::UIntType float_raw = to_conv.conv_val_raw;
+    LDBits float_bits(float_raw);
     is_negative = float_bits.get_sign();
     exponent = float_bits.get_explicit_exponent();
     mantissa = float_bits.get_explicit_mantissa();
     is_inf_or_nan = float_bits.is_inf_or_nan();
   } else {
-    mantissa_width = fputil::MantissaWidth<double>::VALUE;
-    fputil::FPBits<double>::UIntType float_raw =
-        static_cast<fputil::FPBits<double>::UIntType>(to_conv.conv_val_raw);
-    fputil::FPBits<double> float_bits(float_raw);
+    using LBits = fputil::FPBits<double>;
+    mantissa_width = LBits::MANTISSA_WIDTH;
+    LBits::UIntType float_raw =
+        static_cast<LBits::UIntType>(to_conv.conv_val_raw);
+    LBits float_bits(float_raw);
     is_negative = float_bits.get_sign();
     exponent = float_bits.get_explicit_exponent();
     mantissa = float_bits.get_explicit_mantissa();
@@ -86,7 +87,7 @@ LIBC_INLINE int convert_float_hex_exp(Writer *writer,
   // for the extra implicit bit. We use the larger of the two possible values
   // since the size must be constant.
   constexpr size_t MANT_BUFF_LEN =
-      (fputil::MantissaWidth<long double>::VALUE / BITS_IN_HEX_DIGIT) + 1;
+      (LDBits::MANTISSA_WIDTH / BITS_IN_HEX_DIGIT) + 1;
   char mant_buffer[MANT_BUFF_LEN];
 
   size_t mant_len = (mantissa_width / BITS_IN_HEX_DIGIT) + 1;
@@ -157,8 +158,7 @@ LIBC_INLINE int convert_float_hex_exp(Writer *writer,
   // 15 -> 5
   // 11 -> 4
   // 8  -> 3
-  constexpr size_t EXP_LEN =
-      (((fputil::ExponentWidth<long double>::VALUE * 5) + 15) / 16) + 1;
+  constexpr size_t EXP_LEN = (((LDBits::EXPONENT_WIDTH * 5) + 15) / 16) + 1;
   char exp_buffer[EXP_LEN];
 
   bool exp_is_negative = false;
diff --git a/libc/src/stdlib/CMakeLists.txt b/libc/src/stdlib/CMakeLists.txt
index c54d32eb7afdb1adf07630323f32f2c9a52838f4..a4d51fb9a11eef19d5fe512b552b8ccfd63516e1 100644
--- a/libc/src/stdlib/CMakeLists.txt
+++ b/libc/src/stdlib/CMakeLists.txt
@@ -268,18 +268,27 @@ if(LLVM_LIBC_INCLUDE_SCUDO)
   set(SCUDO_DEPS "")
 
   include(${LIBC_SOURCE_DIR}/../compiler-rt/cmake/Modules/AllSupportedArchDefs.cmake)
-  if(NOT (LIBC_TARGET_ARCHITECTURE IN_LIST ALL_SCUDO_STANDALONE_SUPPORTED_ARCH))
-    message(FATAL_ERROR "Architecture ${LIBC_TARGET_ARCHITECTURE} is not supported by SCUDO. 
+  
+  # scudo distinguishes riscv32 and riscv64, so we need to translate the architecture
+  set(LIBC_TARGET_ARCHITECTURE_FOR_SCUDO ${LIBC_TARGET_ARCHITECTURE})
+  if(LIBC_TARGET_ARCHITECTURE_IS_RISCV64)
+    set(LIBC_TARGET_ARCHITECTURE_FOR_SCUDO riscv64)
+  elseif(LIBC_TARGET_ARCHITECTURE_IS_RISCV32)
+    set(LIBC_TARGET_ARCHITECTURE_FOR_SCUDO riscv32)
+  endif()
+
+  if(NOT (LIBC_TARGET_ARCHITECTURE_FOR_SCUDO IN_LIST ALL_SCUDO_STANDALONE_SUPPORTED_ARCH))
+    message(FATAL_ERROR "Architecture ${LIBC_TARGET_ARCHITECTURE_FOR_SCUDO} is not supported by SCUDO. 
       Either disable LLVM_LIBC_INCLUDE_SCUDO or change your target architecture.")
   endif()
 
-  list(APPEND SCUDO_DEPS RTScudoStandalone.${LIBC_TARGET_ARCHITECTURE}
-      RTScudoStandaloneCWrappers.${LIBC_TARGET_ARCHITECTURE})
+  list(APPEND SCUDO_DEPS RTScudoStandalone.${LIBC_TARGET_ARCHITECTURE_FOR_SCUDO}
+      RTScudoStandaloneCWrappers.${LIBC_TARGET_ARCHITECTURE_FOR_SCUDO})
 
   list(APPEND SCUDO_DEPS
-    RTGwpAsan.${LIBC_TARGET_ARCHITECTURE}
-    RTGwpAsanBacktraceLibc.${LIBC_TARGET_ARCHITECTURE}
-    RTGwpAsanSegvHandler.${LIBC_TARGET_ARCHITECTURE}
+    RTGwpAsan.${LIBC_TARGET_ARCHITECTURE_FOR_SCUDO}
+    RTGwpAsanBacktraceLibc.${LIBC_TARGET_ARCHITECTURE_FOR_SCUDO}
+    RTGwpAsanSegvHandler.${LIBC_TARGET_ARCHITECTURE_FOR_SCUDO}
     )
   
   add_entrypoint_external(
diff --git a/libc/src/sys/socket/CMakeLists.txt b/libc/src/sys/socket/CMakeLists.txt
index 7079d6e4466c68311a088ea70bb8504687a22ce7..e0bc48735a031465093ed511cdc5d4c0cc23e964 100644
--- a/libc/src/sys/socket/CMakeLists.txt
+++ b/libc/src/sys/socket/CMakeLists.txt
@@ -9,3 +9,9 @@ add_entrypoint_object(
     .${LIBC_TARGET_OS}.socket
 )
 
+add_entrypoint_object(
+  bind
+  ALIAS
+  DEPENDS
+    .${LIBC_TARGET_OS}.bind
+)
diff --git a/libc/src/sys/socket/bind.h b/libc/src/sys/socket/bind.h
new file mode 100644
index 0000000000000000000000000000000000000000..62e6221bf1b2d74f6b69fca1b4f95bbab5caeba6
--- /dev/null
+++ b/libc/src/sys/socket/bind.h
@@ -0,0 +1,20 @@
+//===-- Implementation header for bind --------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_SYS_SOCKET_BIND_H
+#define LLVM_LIBC_SRC_SYS_SOCKET_BIND_H
+
+#include <sys/socket.h>
+
+namespace LIBC_NAMESPACE {
+
+int bind(int domain, const struct sockaddr *address, socklen_t address_len);
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_SYS_SOCKET_BIND_H
diff --git a/libc/src/sys/socket/linux/CMakeLists.txt b/libc/src/sys/socket/linux/CMakeLists.txt
index 41bcc9c9055f4769d8a68c3b125ffc80636887f9..fc9febdec2cc3c3b77213ad28dbb5b4dbe266c44 100644
--- a/libc/src/sys/socket/linux/CMakeLists.txt
+++ b/libc/src/sys/socket/linux/CMakeLists.txt
@@ -10,3 +10,16 @@ add_entrypoint_object(
     libc.src.__support.OSUtil.osutil
     libc.src.errno.errno
 )
+
+add_entrypoint_object(
+  bind
+  SRCS
+    bind.cpp
+  HDRS
+    ../bind.h
+  DEPENDS
+    libc.include.sys_syscall
+    libc.include.sys_socket
+    libc.src.__support.OSUtil.osutil
+    libc.src.errno.errno
+)
diff --git a/libc/src/sys/socket/linux/bind.cpp b/libc/src/sys/socket/linux/bind.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..36afc646d29f6d0f10de6074b1073ac4b0e9ed4c
--- /dev/null
+++ b/libc/src/sys/socket/linux/bind.cpp
@@ -0,0 +1,43 @@
+//===-- Linux implementation of bind --------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/sys/socket/bind.h"
+
+#include "src/__support/OSUtil/syscall.h" // For internal syscall function.
+#include "src/__support/common.h"
+
+#include "src/errno/libc_errno.h"
+
+#include <linux/net.h>   // For SYS_SOCKET socketcall number.
+#include <sys/syscall.h> // For syscall numbers.
+
+namespace LIBC_NAMESPACE {
+
+LLVM_LIBC_FUNCTION(int, bind,
+                   (int domain, const struct sockaddr *address,
+                    socklen_t address_len)) {
+#ifdef SYS_socket
+  int ret =
+      LIBC_NAMESPACE::syscall_impl<int>(SYS_bind, domain, address, address_len);
+#elif defined(SYS_socketcall)
+  unsigned long sockcall_args[3] = {static_cast<unsigned long>(domain),
+                                    reinterpret_cast<unsigned long>(address),
+                                    static_cast<unsigned long>(address_len)};
+  int ret = LIBC_NAMESPACE::syscall_impl<int>(SYS_socketcall, SYS_BIND,
+                                              sockcall_args);
+#else
+#error "socket and socketcall syscalls unavailable for this platform."
+#endif
+  if (ret < 0) {
+    libc_errno = -ret;
+    return -1;
+  }
+  return ret;
+}
+
+} // namespace LIBC_NAMESPACE
diff --git a/libc/src/sys/socket/linux/socket.cpp b/libc/src/sys/socket/linux/socket.cpp
index 6429fd12013ea04397ff1f89b8e525e7a70fd9a5..90a7dc632e26960aab6cf24962269900c9096839 100644
--- a/libc/src/sys/socket/linux/socket.cpp
+++ b/libc/src/sys/socket/linux/socket.cpp
@@ -23,7 +23,9 @@ LLVM_LIBC_FUNCTION(int, socket, (int domain, int type, int protocol)) {
   int ret =
       LIBC_NAMESPACE::syscall_impl<int>(SYS_socket, domain, type, protocol);
 #elif defined(SYS_socketcall)
-  unsigned long sockcall_args[3] = {domain, type, protocol};
+  unsigned long sockcall_args[3] = {static_cast<unsigned long>(domain),
+                                    static_cast<unsigned long>(type),
+                                    static_cast<unsigned long>(protocol)};
   int ret = LIBC_NAMESPACE::syscall_impl<int>(SYS_socketcall, SYS_SOCKET,
                                               sockcall_args);
 #else
diff --git a/libc/test/src/fcntl/creat_test.cpp b/libc/test/src/fcntl/creat_test.cpp
index ca926b30e62faf72134abfaf973d372765fafe3b..ef30d8862c45f263bdc93b9d96a10c80f503d9bd 100644
--- a/libc/test/src/fcntl/creat_test.cpp
+++ b/libc/test/src/fcntl/creat_test.cpp
@@ -13,6 +13,8 @@
 #include "test/UnitTest/ErrnoSetterMatcher.h"
 #include "test/UnitTest/Test.h"
 
+#include <sys/stat.h>
+
 TEST(LlvmLibcCreatTest, CreatAndOpen) {
   using LIBC_NAMESPACE::testing::ErrnoSetterMatcher::Succeeds;
   constexpr const char *TEST_FILE = "testdata/creat.test";
diff --git a/libc/test/src/math/FDimTest.h b/libc/test/src/math/FDimTest.h
index 3118c3661013e7cf97101e8cdb28adba09145972..3fb82ed8bca2ced18689ce45edd4e8089289665f 100644
--- a/libc/test/src/math/FDimTest.h
+++ b/libc/test/src/math/FDimTest.h
@@ -74,9 +74,9 @@ public:
 private:
   // constexpr does not work on FPBits yet, so we cannot have these constants as
   // static.
-  const T nan = T(LIBC_NAMESPACE::fputil::FPBits<T>::build_quiet_nan(1));
-  const T inf = T(LIBC_NAMESPACE::fputil::FPBits<T>::inf());
-  const T neg_inf = T(LIBC_NAMESPACE::fputil::FPBits<T>::neg_inf());
-  const T zero = T(LIBC_NAMESPACE::fputil::FPBits<T>::zero());
-  const T neg_zero = T(LIBC_NAMESPACE::fputil::FPBits<T>::neg_zero());
+  const T nan = T(FPBits::build_quiet_nan(1));
+  const T inf = T(FPBits::inf());
+  const T neg_inf = T(FPBits::neg_inf());
+  const T zero = T(FPBits::zero());
+  const T neg_zero = T(FPBits::neg_zero());
 };
diff --git a/libc/test/src/math/FmaTest.h b/libc/test/src/math/FmaTest.h
index c2573d03792692fa9d27481176c2bc96fad46dea..94412b7021df37934d64d488dd9182fe33b67ea5 100644
--- a/libc/test/src/math/FmaTest.h
+++ b/libc/test/src/math/FmaTest.h
@@ -23,11 +23,11 @@ private:
   using Func = T (*)(T, T, T);
   using FPBits = LIBC_NAMESPACE::fputil::FPBits<T>;
   using UIntType = typename FPBits::UIntType;
-  const T nan = T(LIBC_NAMESPACE::fputil::FPBits<T>::build_quiet_nan(1));
-  const T inf = T(LIBC_NAMESPACE::fputil::FPBits<T>::inf());
-  const T neg_inf = T(LIBC_NAMESPACE::fputil::FPBits<T>::neg_inf());
-  const T zero = T(LIBC_NAMESPACE::fputil::FPBits<T>::zero());
-  const T neg_zero = T(LIBC_NAMESPACE::fputil::FPBits<T>::neg_zero());
+  const T nan = T(FPBits::build_quiet_nan(1));
+  const T inf = T(FPBits::inf());
+  const T neg_inf = T(FPBits::neg_inf());
+  const T zero = T(FPBits::zero());
+  const T neg_zero = T(FPBits::neg_zero());
 
   UIntType get_random_bit_pattern() {
     UIntType bits{0};
diff --git a/libc/test/src/math/FrexpTest.h b/libc/test/src/math/FrexpTest.h
index 19c98872e0411671bf445d22f1dee1442ab543c6..2f93987b1ea75f7aff3155e5665ef224cc5f2818 100644
--- a/libc/test/src/math/FrexpTest.h
+++ b/libc/test/src/math/FrexpTest.h
@@ -20,7 +20,7 @@ template <typename T> class FrexpTest : public LIBC_NAMESPACE::testing::Test {
   DECLARE_SPECIAL_CONSTANTS(T)
 
   static constexpr UIntType HIDDEN_BIT =
-      UIntType(1) << LIBC_NAMESPACE::fputil::MantissaWidth<T>::VALUE;
+      UIntType(1) << LIBC_NAMESPACE::fputil::FloatProperties<T>::MANTISSA_WIDTH;
 
 public:
   typedef T (*FrexpFunc)(T, int *);
diff --git a/libc/test/src/math/ILogbTest.h b/libc/test/src/math/ILogbTest.h
index e51a5d7a2544cdc3877c8a93cb7c49d10563ddfd..9e12b4515c28525d66755425f729d108eaf27f78 100644
--- a/libc/test/src/math/ILogbTest.h
+++ b/libc/test/src/math/ILogbTest.h
@@ -24,15 +24,12 @@ public:
 
   template <typename T>
   void test_special_numbers(typename ILogbFunc<T>::Func func) {
-    EXPECT_EQ(FP_ILOGB0, func(T(LIBC_NAMESPACE::fputil::FPBits<T>::zero())));
-    EXPECT_EQ(FP_ILOGB0,
-              func(T(LIBC_NAMESPACE::fputil::FPBits<T>::neg_zero())));
-
-    EXPECT_EQ(FP_ILOGBNAN,
-              func(T(LIBC_NAMESPACE::fputil::FPBits<T>::build_quiet_nan(1))));
-
-    EXPECT_EQ(INT_MAX, func(T(LIBC_NAMESPACE::fputil::FPBits<T>::inf())));
-    EXPECT_EQ(INT_MAX, func(T(LIBC_NAMESPACE::fputil::FPBits<T>::neg_inf())));
+    using FPBits = LIBC_NAMESPACE::fputil::FPBits<T>;
+    EXPECT_EQ(FP_ILOGB0, func(T(FPBits::zero())));
+    EXPECT_EQ(FP_ILOGB0, func(T(FPBits::neg_zero())));
+    EXPECT_EQ(FP_ILOGBNAN, func(T(FPBits::build_quiet_nan(1))));
+    EXPECT_EQ(INT_MAX, func(T(FPBits::inf())));
+    EXPECT_EQ(INT_MAX, func(T(FPBits::neg_inf())));
   }
 
   template <typename T>
diff --git a/libc/test/src/math/LdExpTest.h b/libc/test/src/math/LdExpTest.h
index a75c8ef31a2cf83e9e279008eff5ec7553a0919e..5ebba62a3dbb8acd620a7950860b0c2cb0d80f58 100644
--- a/libc/test/src/math/LdExpTest.h
+++ b/libc/test/src/math/LdExpTest.h
@@ -23,16 +23,15 @@ class LdExpTestTemplate : public LIBC_NAMESPACE::testing::Test {
   using FPBits = LIBC_NAMESPACE::fputil::FPBits<T>;
   using NormalFloat = LIBC_NAMESPACE::fputil::NormalFloat<T>;
   using UIntType = typename FPBits::UIntType;
-  static constexpr UIntType MANTISSA_WIDTH =
-      LIBC_NAMESPACE::fputil::MantissaWidth<T>::VALUE;
+  static constexpr UIntType MANTISSA_WIDTH = FPBits::MANTISSA_WIDTH;
   // A normalized mantissa to be used with tests.
   static constexpr UIntType MANTISSA = NormalFloat::ONE + 0x1234;
 
-  const T zero = T(LIBC_NAMESPACE::fputil::FPBits<T>::zero());
-  const T neg_zero = T(LIBC_NAMESPACE::fputil::FPBits<T>::neg_zero());
-  const T inf = T(LIBC_NAMESPACE::fputil::FPBits<T>::inf());
-  const T neg_inf = T(LIBC_NAMESPACE::fputil::FPBits<T>::neg_inf());
-  const T nan = T(LIBC_NAMESPACE::fputil::FPBits<T>::build_quiet_nan(1));
+  const T zero = T(FPBits::zero());
+  const T neg_zero = T(FPBits::neg_zero());
+  const T inf = T(FPBits::inf());
+  const T neg_inf = T(FPBits::neg_inf());
+  const T nan = T(FPBits::build_quiet_nan(1));
 
 public:
   typedef T (*LdExpFunc)(T, int);
diff --git a/libc/test/src/math/LogbTest.h b/libc/test/src/math/LogbTest.h
index c2cf4f1f1483336cd34106c4529c614868e4276e..3912385623ef19f486f75e2417efcd1d2265cf57 100644
--- a/libc/test/src/math/LogbTest.h
+++ b/libc/test/src/math/LogbTest.h
@@ -20,7 +20,7 @@ template <typename T> class LogbTest : public LIBC_NAMESPACE::testing::Test {
   DECLARE_SPECIAL_CONSTANTS(T)
 
   static constexpr UIntType HIDDEN_BIT =
-      UIntType(1) << LIBC_NAMESPACE::fputil::MantissaWidth<T>::VALUE;
+      UIntType(1) << LIBC_NAMESPACE::fputil::FloatProperties<T>::MANTISSA_WIDTH;
 
 public:
   typedef T (*LogbFunc)(T);
diff --git a/libc/test/src/math/NextAfterTest.h b/libc/test/src/math/NextAfterTest.h
index 57a801dfb28a316ae9f0a2d623217f199f2a3359..ca34753871b9606ba6d483857351887e6eb2c9d5 100644
--- a/libc/test/src/math/NextAfterTest.h
+++ b/libc/test/src/math/NextAfterTest.h
@@ -20,7 +20,6 @@
 template <typename T>
 class NextAfterTestTemplate : public LIBC_NAMESPACE::testing::Test {
   using FPBits = LIBC_NAMESPACE::fputil::FPBits<T>;
-  using MantissaWidth = LIBC_NAMESPACE::fputil::MantissaWidth<T>;
   using UIntType = typename FPBits::UIntType;
 
   static constexpr int BIT_WIDTH_OF_TYPE =
@@ -165,7 +164,7 @@ public:
     ASSERT_EQ(result_bits.get_biased_exponent(),
               uint16_t(x_bits.get_biased_exponent() - 1));
     ASSERT_EQ(result_bits.get_mantissa(),
-              (UIntType(1) << MantissaWidth::VALUE) - 1);
+              (UIntType(1) << FPBits::MANTISSA_WIDTH) - 1);
 
     result = func(x, T(33.0));
     result_bits = FPBits(result);
@@ -179,7 +178,7 @@ public:
     ASSERT_EQ(result_bits.get_biased_exponent(),
               uint16_t(x_bits.get_biased_exponent() - 1));
     ASSERT_EQ(result_bits.get_mantissa(),
-              (UIntType(1) << MantissaWidth::VALUE) - 1);
+              (UIntType(1) << FPBits::MANTISSA_WIDTH) - 1);
 
     result = func(x, T(-33.0));
     result_bits = FPBits(result);
diff --git a/libc/test/src/math/RemQuoTest.h b/libc/test/src/math/RemQuoTest.h
index 6da0756c3a1b3de2859970bbe100a6e666cb9c8e..a345e8b72e78d02f8d0bbca63ad318f34b8ca0e1 100644
--- a/libc/test/src/math/RemQuoTest.h
+++ b/libc/test/src/math/RemQuoTest.h
@@ -23,11 +23,11 @@ class RemQuoTestTemplate : public LIBC_NAMESPACE::testing::Test {
   using FPBits = LIBC_NAMESPACE::fputil::FPBits<T>;
   using UIntType = typename FPBits::UIntType;
 
-  const T zero = T(LIBC_NAMESPACE::fputil::FPBits<T>::zero());
-  const T neg_zero = T(LIBC_NAMESPACE::fputil::FPBits<T>::neg_zero());
-  const T inf = T(LIBC_NAMESPACE::fputil::FPBits<T>::inf());
-  const T neg_inf = T(LIBC_NAMESPACE::fputil::FPBits<T>::neg_inf());
-  const T nan = T(LIBC_NAMESPACE::fputil::FPBits<T>::build_quiet_nan(1));
+  const T zero = T(FPBits::zero());
+  const T neg_zero = T(FPBits::neg_zero());
+  const T inf = T(FPBits::inf());
+  const T neg_inf = T(FPBits::neg_inf());
+  const T nan = T(FPBits::build_quiet_nan(1));
 
 public:
   typedef T (*RemQuoFunc)(T, T, int *);
diff --git a/libc/test/src/math/RoundToIntegerTest.h b/libc/test/src/math/RoundToIntegerTest.h
index 1a976e97359eb76ab108a1a7ca824767c851946c..feee040c72f88dd85694c41ae2c1fcc7c247cda2 100644
--- a/libc/test/src/math/RoundToIntegerTest.h
+++ b/libc/test/src/math/RoundToIntegerTest.h
@@ -32,11 +32,11 @@ private:
   using FPBits = LIBC_NAMESPACE::fputil::FPBits<F>;
   using UIntType = typename FPBits::UIntType;
 
-  const F zero = F(LIBC_NAMESPACE::fputil::FPBits<F>::zero());
-  const F neg_zero = F(LIBC_NAMESPACE::fputil::FPBits<F>::neg_zero());
-  const F inf = F(LIBC_NAMESPACE::fputil::FPBits<F>::inf());
-  const F neg_inf = F(LIBC_NAMESPACE::fputil::FPBits<F>::neg_inf());
-  const F nan = F(LIBC_NAMESPACE::fputil::FPBits<F>::build_quiet_nan(1));
+  const F zero = F(FPBits::zero());
+  const F neg_zero = F(FPBits::neg_zero());
+  const F inf = F(FPBits::inf());
+  const F neg_inf = F(FPBits::neg_inf());
+  const F nan = F(FPBits::build_quiet_nan(1));
   static constexpr I INTEGER_MIN = I(1) << (sizeof(I) * 8 - 1);
   static constexpr I INTEGER_MAX = -(INTEGER_MIN + 1);
 
@@ -192,8 +192,7 @@ public:
     FPBits bits(F(1.0));
     bits.set_biased_exponent(EXPONENT_LIMIT + FPBits::EXPONENT_BIAS);
     bits.set_sign(1);
-    bits.set_mantissa(UIntType(0x1)
-                      << (LIBC_NAMESPACE::fputil::MantissaWidth<F>::VALUE - 1));
+    bits.set_mantissa(UIntType(0x1) << (FPBits::MANTISSA_WIDTH - 1));
 
     F x = F(bits);
     if (TestModes) {
diff --git a/libc/test/src/math/SqrtTest.h b/libc/test/src/math/SqrtTest.h
index 24f14b78d2f0f5d850c3335a61e9e0b2d2b3e1d8..ab14e30d3ded6ff9fb45ed46d9840bac6b3820c8 100644
--- a/libc/test/src/math/SqrtTest.h
+++ b/libc/test/src/math/SqrtTest.h
@@ -20,7 +20,7 @@ template <typename T> class SqrtTest : public LIBC_NAMESPACE::testing::Test {
   DECLARE_SPECIAL_CONSTANTS(T)
 
   static constexpr UIntType HIDDEN_BIT =
-      UIntType(1) << LIBC_NAMESPACE::fputil::MantissaWidth<T>::VALUE;
+      UIntType(1) << LIBC_NAMESPACE::fputil::FloatProperties<T>::MANTISSA_WIDTH;
 
 public:
   typedef T (*SqrtFunc)(T);
diff --git a/libc/test/src/math/smoke/FDimTest.h b/libc/test/src/math/smoke/FDimTest.h
index 3118c3661013e7cf97101e8cdb28adba09145972..3fb82ed8bca2ced18689ce45edd4e8089289665f 100644
--- a/libc/test/src/math/smoke/FDimTest.h
+++ b/libc/test/src/math/smoke/FDimTest.h
@@ -74,9 +74,9 @@ public:
 private:
   // constexpr does not work on FPBits yet, so we cannot have these constants as
   // static.
-  const T nan = T(LIBC_NAMESPACE::fputil::FPBits<T>::build_quiet_nan(1));
-  const T inf = T(LIBC_NAMESPACE::fputil::FPBits<T>::inf());
-  const T neg_inf = T(LIBC_NAMESPACE::fputil::FPBits<T>::neg_inf());
-  const T zero = T(LIBC_NAMESPACE::fputil::FPBits<T>::zero());
-  const T neg_zero = T(LIBC_NAMESPACE::fputil::FPBits<T>::neg_zero());
+  const T nan = T(FPBits::build_quiet_nan(1));
+  const T inf = T(FPBits::inf());
+  const T neg_inf = T(FPBits::neg_inf());
+  const T zero = T(FPBits::zero());
+  const T neg_zero = T(FPBits::neg_zero());
 };
diff --git a/libc/test/src/math/smoke/FmaTest.h b/libc/test/src/math/smoke/FmaTest.h
index 3c1c120d77d450864efdb08040e0524bb0ada331..1da9652bfee155efb743ce1b6900b36930850e2f 100644
--- a/libc/test/src/math/smoke/FmaTest.h
+++ b/libc/test/src/math/smoke/FmaTest.h
@@ -19,11 +19,11 @@ private:
   using Func = T (*)(T, T, T);
   using FPBits = LIBC_NAMESPACE::fputil::FPBits<T>;
   using UIntType = typename FPBits::UIntType;
-  const T nan = T(LIBC_NAMESPACE::fputil::FPBits<T>::build_quiet_nan(1));
-  const T inf = T(LIBC_NAMESPACE::fputil::FPBits<T>::inf());
-  const T neg_inf = T(LIBC_NAMESPACE::fputil::FPBits<T>::neg_inf());
-  const T zero = T(LIBC_NAMESPACE::fputil::FPBits<T>::zero());
-  const T neg_zero = T(LIBC_NAMESPACE::fputil::FPBits<T>::neg_zero());
+  const T nan = T(FPBits::build_quiet_nan(1));
+  const T inf = T(FPBits::inf());
+  const T neg_inf = T(FPBits::neg_inf());
+  const T zero = T(FPBits::zero());
+  const T neg_zero = T(FPBits::neg_zero());
 
 public:
   void test_special_numbers(Func func) {
diff --git a/libc/test/src/math/smoke/FrexpTest.h b/libc/test/src/math/smoke/FrexpTest.h
index 3ff169091cc9a38f1bf48a50f8d965c59e7a4d49..643a151d89c7da81fc30494ae5a52f92061d7d02 100644
--- a/libc/test/src/math/smoke/FrexpTest.h
+++ b/libc/test/src/math/smoke/FrexpTest.h
@@ -17,7 +17,7 @@ template <typename T> class FrexpTest : public LIBC_NAMESPACE::testing::Test {
   DECLARE_SPECIAL_CONSTANTS(T)
 
   static constexpr UIntType HIDDEN_BIT =
-      UIntType(1) << LIBC_NAMESPACE::fputil::MantissaWidth<T>::VALUE;
+      UIntType(1) << LIBC_NAMESPACE::fputil::FloatProperties<T>::MANTISSA_WIDTH;
 
 public:
   typedef T (*FrexpFunc)(T, int *);
diff --git a/libc/test/src/math/smoke/ILogbTest.h b/libc/test/src/math/smoke/ILogbTest.h
index e51a5d7a2544cdc3877c8a93cb7c49d10563ddfd..9e12b4515c28525d66755425f729d108eaf27f78 100644
--- a/libc/test/src/math/smoke/ILogbTest.h
+++ b/libc/test/src/math/smoke/ILogbTest.h
@@ -24,15 +24,12 @@ public:
 
   template <typename T>
   void test_special_numbers(typename ILogbFunc<T>::Func func) {
-    EXPECT_EQ(FP_ILOGB0, func(T(LIBC_NAMESPACE::fputil::FPBits<T>::zero())));
-    EXPECT_EQ(FP_ILOGB0,
-              func(T(LIBC_NAMESPACE::fputil::FPBits<T>::neg_zero())));
-
-    EXPECT_EQ(FP_ILOGBNAN,
-              func(T(LIBC_NAMESPACE::fputil::FPBits<T>::build_quiet_nan(1))));
-
-    EXPECT_EQ(INT_MAX, func(T(LIBC_NAMESPACE::fputil::FPBits<T>::inf())));
-    EXPECT_EQ(INT_MAX, func(T(LIBC_NAMESPACE::fputil::FPBits<T>::neg_inf())));
+    using FPBits = LIBC_NAMESPACE::fputil::FPBits<T>;
+    EXPECT_EQ(FP_ILOGB0, func(T(FPBits::zero())));
+    EXPECT_EQ(FP_ILOGB0, func(T(FPBits::neg_zero())));
+    EXPECT_EQ(FP_ILOGBNAN, func(T(FPBits::build_quiet_nan(1))));
+    EXPECT_EQ(INT_MAX, func(T(FPBits::inf())));
+    EXPECT_EQ(INT_MAX, func(T(FPBits::neg_inf())));
   }
 
   template <typename T>
diff --git a/libc/test/src/math/smoke/LdExpTest.h b/libc/test/src/math/smoke/LdExpTest.h
index a75c8ef31a2cf83e9e279008eff5ec7553a0919e..5ebba62a3dbb8acd620a7950860b0c2cb0d80f58 100644
--- a/libc/test/src/math/smoke/LdExpTest.h
+++ b/libc/test/src/math/smoke/LdExpTest.h
@@ -23,16 +23,15 @@ class LdExpTestTemplate : public LIBC_NAMESPACE::testing::Test {
   using FPBits = LIBC_NAMESPACE::fputil::FPBits<T>;
   using NormalFloat = LIBC_NAMESPACE::fputil::NormalFloat<T>;
   using UIntType = typename FPBits::UIntType;
-  static constexpr UIntType MANTISSA_WIDTH =
-      LIBC_NAMESPACE::fputil::MantissaWidth<T>::VALUE;
+  static constexpr UIntType MANTISSA_WIDTH = FPBits::MANTISSA_WIDTH;
   // A normalized mantissa to be used with tests.
   static constexpr UIntType MANTISSA = NormalFloat::ONE + 0x1234;
 
-  const T zero = T(LIBC_NAMESPACE::fputil::FPBits<T>::zero());
-  const T neg_zero = T(LIBC_NAMESPACE::fputil::FPBits<T>::neg_zero());
-  const T inf = T(LIBC_NAMESPACE::fputil::FPBits<T>::inf());
-  const T neg_inf = T(LIBC_NAMESPACE::fputil::FPBits<T>::neg_inf());
-  const T nan = T(LIBC_NAMESPACE::fputil::FPBits<T>::build_quiet_nan(1));
+  const T zero = T(FPBits::zero());
+  const T neg_zero = T(FPBits::neg_zero());
+  const T inf = T(FPBits::inf());
+  const T neg_inf = T(FPBits::neg_inf());
+  const T nan = T(FPBits::build_quiet_nan(1));
 
 public:
   typedef T (*LdExpFunc)(T, int);
diff --git a/libc/test/src/math/smoke/LogbTest.h b/libc/test/src/math/smoke/LogbTest.h
index 34cf92c19ed8b1b1e7532de7bdd55f00c35cc71a..41b356fe0524e604e7721bcba7af79533332696f 100644
--- a/libc/test/src/math/smoke/LogbTest.h
+++ b/libc/test/src/math/smoke/LogbTest.h
@@ -17,7 +17,7 @@ template <typename T> class LogbTest : public LIBC_NAMESPACE::testing::Test {
   DECLARE_SPECIAL_CONSTANTS(T)
 
   static constexpr UIntType HIDDEN_BIT =
-      UIntType(1) << LIBC_NAMESPACE::fputil::MantissaWidth<T>::VALUE;
+      UIntType(1) << LIBC_NAMESPACE::fputil::FloatProperties<T>::MANTISSA_WIDTH;
 
 public:
   typedef T (*LogbFunc)(T);
diff --git a/libc/test/src/math/smoke/NextAfterTest.h b/libc/test/src/math/smoke/NextAfterTest.h
index 29098e0f49a45755d9000fc12dcddecd3f6b6a1d..90155a8b293f84366459fd434b6285321149ae91 100644
--- a/libc/test/src/math/smoke/NextAfterTest.h
+++ b/libc/test/src/math/smoke/NextAfterTest.h
@@ -31,7 +31,6 @@
 template <typename T>
 class NextAfterTestTemplate : public LIBC_NAMESPACE::testing::Test {
   using FPBits = LIBC_NAMESPACE::fputil::FPBits<T>;
-  using MantissaWidth = LIBC_NAMESPACE::fputil::MantissaWidth<T>;
   using UIntType = typename FPBits::UIntType;
 
   static constexpr int BIT_WIDTH_OF_TYPE =
@@ -176,7 +175,7 @@ public:
     ASSERT_EQ(result_bits.get_biased_exponent(),
               uint16_t(x_bits.get_biased_exponent() - 1));
     ASSERT_EQ(result_bits.get_mantissa(),
-              (UIntType(1) << MantissaWidth::VALUE) - 1);
+              (UIntType(1) << FPBits::MANTISSA_WIDTH) - 1);
 
     result = func(x, T(33.0));
     result_bits = FPBits(result);
@@ -190,7 +189,7 @@ public:
     ASSERT_EQ(result_bits.get_biased_exponent(),
               uint16_t(x_bits.get_biased_exponent() - 1));
     ASSERT_EQ(result_bits.get_mantissa(),
-              (UIntType(1) << MantissaWidth::VALUE) - 1);
+              (UIntType(1) << FPBits::MANTISSA_WIDTH) - 1);
 
     result = func(x, T(-33.0));
     result_bits = FPBits(result);
diff --git a/libc/test/src/math/smoke/NextTowardTest.h b/libc/test/src/math/smoke/NextTowardTest.h
index 111d8017e691d36841de58843b062be46f0e0c4b..bd719275a41552f6fb0afe09e688eb6dc62ea72a 100644
--- a/libc/test/src/math/smoke/NextTowardTest.h
+++ b/libc/test/src/math/smoke/NextTowardTest.h
@@ -33,7 +33,6 @@ template <typename T>
 class NextTowardTestTemplate : public LIBC_NAMESPACE::testing::Test {
   using FPBits = LIBC_NAMESPACE::fputil::FPBits<T>;
   using ToFPBits = LIBC_NAMESPACE::fputil::FPBits<long double>;
-  using MantissaWidth = LIBC_NAMESPACE::fputil::MantissaWidth<T>;
   using UIntType = typename FPBits::UIntType;
 
   static constexpr int BIT_WIDTH_OF_TYPE =
@@ -190,7 +189,7 @@ public:
     ASSERT_EQ(result_bits.get_biased_exponent(),
               uint16_t(x_bits.get_biased_exponent() - 1));
     ASSERT_EQ(result_bits.get_mantissa(),
-              (UIntType(1) << MantissaWidth::VALUE) - 1);
+              (UIntType(1) << FPBits::MANTISSA_WIDTH) - 1);
 
     result = func(x, 33.0);
     result_bits = FPBits(result);
@@ -204,7 +203,7 @@ public:
     ASSERT_EQ(result_bits.get_biased_exponent(),
               uint16_t(x_bits.get_biased_exponent() - 1));
     ASSERT_EQ(result_bits.get_mantissa(),
-              (UIntType(1) << MantissaWidth::VALUE) - 1);
+              (UIntType(1) << FPBits::MANTISSA_WIDTH) - 1);
 
     result = func(x, -33.0);
     result_bits = FPBits(result);
diff --git a/libc/test/src/math/smoke/RemQuoTest.h b/libc/test/src/math/smoke/RemQuoTest.h
index 5a5d143777502351321ae86d7a97e5f5e4e31b5e..514190b6b31caa623aca11a0048a6c09f1d7778a 100644
--- a/libc/test/src/math/smoke/RemQuoTest.h
+++ b/libc/test/src/math/smoke/RemQuoTest.h
@@ -20,11 +20,11 @@ class RemQuoTestTemplate : public LIBC_NAMESPACE::testing::Test {
   using FPBits = LIBC_NAMESPACE::fputil::FPBits<T>;
   using UIntType = typename FPBits::UIntType;
 
-  const T zero = T(LIBC_NAMESPACE::fputil::FPBits<T>::zero());
-  const T neg_zero = T(LIBC_NAMESPACE::fputil::FPBits<T>::neg_zero());
-  const T inf = T(LIBC_NAMESPACE::fputil::FPBits<T>::inf());
-  const T neg_inf = T(LIBC_NAMESPACE::fputil::FPBits<T>::neg_inf());
-  const T nan = T(LIBC_NAMESPACE::fputil::FPBits<T>::build_quiet_nan(1));
+  const T zero = T(FPBits::zero());
+  const T neg_zero = T(FPBits::neg_zero());
+  const T inf = T(FPBits::inf());
+  const T neg_inf = T(FPBits::neg_inf());
+  const T nan = T(FPBits::build_quiet_nan(1));
 
 public:
   typedef T (*RemQuoFunc)(T, T, int *);
diff --git a/libc/test/src/math/smoke/SqrtTest.h b/libc/test/src/math/smoke/SqrtTest.h
index d4b2f9dd2624f1d1e10ad62bdf93c53bcae621b8..dffff520cd69e87a23984fdb29ae882b2591e74e 100644
--- a/libc/test/src/math/smoke/SqrtTest.h
+++ b/libc/test/src/math/smoke/SqrtTest.h
@@ -17,7 +17,7 @@ template <typename T> class SqrtTest : public LIBC_NAMESPACE::testing::Test {
   DECLARE_SPECIAL_CONSTANTS(T)
 
   static constexpr UIntType HIDDEN_BIT =
-      UIntType(1) << LIBC_NAMESPACE::fputil::MantissaWidth<T>::VALUE;
+      UIntType(1) << LIBC_NAMESPACE::fputil::FloatProperties<T>::MANTISSA_WIDTH;
 
 public:
   typedef T (*SqrtFunc)(T);
diff --git a/libc/test/src/sys/resource/getrlimit_setrlimit_test.cpp b/libc/test/src/sys/resource/getrlimit_setrlimit_test.cpp
index 7e6bb0aaca92518f0f465c9df7174b2f28d16909..0870deeeb5d801076a8fbe1af489be46dd999cb7 100644
--- a/libc/test/src/sys/resource/getrlimit_setrlimit_test.cpp
+++ b/libc/test/src/sys/resource/getrlimit_setrlimit_test.cpp
@@ -17,6 +17,7 @@
 #include "test/UnitTest/Test.h"
 
 #include <sys/resource.h>
+#include <sys/stat.h>
 
 TEST(LlvmLibcResourceLimitsTest, SetNoFileLimit) {
   using LIBC_NAMESPACE::testing::ErrnoSetterMatcher::Fails;
diff --git a/libc/test/src/sys/socket/linux/CMakeLists.txt b/libc/test/src/sys/socket/linux/CMakeLists.txt
index 4380597e5515799083bb4d90489c8c12a59a4ab8..666dc28c7e4ee199f955b013c645b6fe0e91bd95 100644
--- a/libc/test/src/sys/socket/linux/CMakeLists.txt
+++ b/libc/test/src/sys/socket/linux/CMakeLists.txt
@@ -12,3 +12,19 @@ add_libc_unittest(
     libc.src.sys.socket.socket
     libc.src.unistd.close
 )
+
+
+add_libc_unittest(
+  bind_test
+  SUITE
+    libc_sys_socket_unittests
+  SRCS
+    bind_test.cpp
+  DEPENDS
+    libc.include.sys_socket
+    libc.src.errno.errno
+    libc.src.sys.socket.socket
+    libc.src.sys.socket.bind
+    libc.src.stdio.remove
+    libc.src.unistd.close
+)
diff --git a/libc/test/src/sys/socket/linux/bind_test.cpp b/libc/test/src/sys/socket/linux/bind_test.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..5a3a1c227c9b58efca1851737dad9f317ea614b1
--- /dev/null
+++ b/libc/test/src/sys/socket/linux/bind_test.cpp
@@ -0,0 +1,55 @@
+//===-- Unittests for bind ------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/sys/socket/bind.h"
+#include "src/sys/socket/socket.h"
+
+#include "src/stdio/remove.h"
+#include "src/unistd/close.h"
+
+#include "src/errno/libc_errno.h"
+#include "test/UnitTest/LibcTest.h"
+#include "test/UnitTest/Test.h"
+
+#include <sys/socket.h> // For AF_UNIX and SOCK_DGRAM
+
+TEST(LlvmLibcSocketTest, BindLocalSocket) {
+
+  const char *FILENAME = "bind_file.test";
+  auto SOCK_PATH = libc_make_test_file_path(FILENAME);
+
+  int sock = LIBC_NAMESPACE::socket(AF_UNIX, SOCK_DGRAM, 0);
+  ASSERT_GE(sock, 0);
+  ASSERT_EQ(libc_errno, 0);
+
+  struct sockaddr_un my_addr;
+
+  my_addr.sun_family = AF_UNIX;
+  unsigned int i = 0;
+  for (;
+       SOCK_PATH[i] != '\0' && (i < sizeof(sockaddr_un) - sizeof(sa_family_t));
+       ++i)
+    my_addr.sun_path[i] = SOCK_PATH[i];
+  my_addr.sun_path[i] = '\0';
+
+  // It's important that the path fits in the struct, if it doesn't then we
+  // can't try to bind to the file.
+  ASSERT_LT(
+      i, static_cast<unsigned int>(sizeof(sockaddr_un) - sizeof(sa_family_t)));
+
+  int result =
+      LIBC_NAMESPACE::bind(sock, reinterpret_cast<struct sockaddr *>(&my_addr),
+                           sizeof(struct sockaddr_un));
+
+  ASSERT_EQ(result, 0);
+  ASSERT_EQ(libc_errno, 0);
+
+  LIBC_NAMESPACE::close(sock);
+
+  LIBC_NAMESPACE::remove(SOCK_PATH);
+}
diff --git a/libc/test/src/sys/socket/linux/socket_test.cpp b/libc/test/src/sys/socket/linux/socket_test.cpp
index 9037888441a357f674db0a40843e1e9d94a2205f..9d5bfacde0a4099344d186fc5502aacdbcd3d740 100644
--- a/libc/test/src/sys/socket/linux/socket_test.cpp
+++ b/libc/test/src/sys/socket/linux/socket_test.cpp
@@ -13,10 +13,10 @@
 #include "src/errno/libc_errno.h"
 #include "test/UnitTest/Test.h"
 
-#include <sys/socket.h> // For AF_LOCAL and SOCK_DGRAM
+#include <sys/socket.h> // For AF_UNIX and SOCK_DGRAM
 
 TEST(LlvmLibcSocketTest, LocalSocket) {
-  int sock = LIBC_NAMESPACE::socket(AF_LOCAL, SOCK_DGRAM, 0);
+  int sock = LIBC_NAMESPACE::socket(AF_UNIX, SOCK_DGRAM, 0);
   ASSERT_GE(sock, 0);
   ASSERT_EQ(libc_errno, 0);
 
diff --git a/libc/test/src/unistd/access_test.cpp b/libc/test/src/unistd/access_test.cpp
index ed1fe96f4f4a1672312168bd678733516f4a4ee9..7d4b3be443fed492146df25d4238e5e0735f8f74 100644
--- a/libc/test/src/unistd/access_test.cpp
+++ b/libc/test/src/unistd/access_test.cpp
@@ -14,6 +14,7 @@
 #include "test/UnitTest/ErrnoSetterMatcher.h"
 #include "test/UnitTest/Test.h"
 
+#include <sys/stat.h>
 #include <unistd.h>
 
 TEST(LlvmLibcAccessTest, CreateAndTest) {
diff --git a/libc/test/src/unistd/dup2_test.cpp b/libc/test/src/unistd/dup2_test.cpp
index ff870db0334c93dca4086650484583a1e5ed5481..d46c4b919ce72cb315ee5bb7b0445d940d2d898c 100644
--- a/libc/test/src/unistd/dup2_test.cpp
+++ b/libc/test/src/unistd/dup2_test.cpp
@@ -16,6 +16,8 @@
 #include "test/UnitTest/ErrnoSetterMatcher.h"
 #include "test/UnitTest/Test.h"
 
+#include <sys/stat.h>
+
 TEST(LlvmLibcdupTest, ReadAndWriteViaDup) {
   constexpr int DUPFD = 0xD0;
   libc_errno = 0;
diff --git a/libc/test/src/unistd/dup3_test.cpp b/libc/test/src/unistd/dup3_test.cpp
index 279cfbfea1b144be6a3b1e0bcd3f8b28d2cc8f02..d2d544d5d9a12f854b308eeed99f8bdbbbe7e2d2 100644
--- a/libc/test/src/unistd/dup3_test.cpp
+++ b/libc/test/src/unistd/dup3_test.cpp
@@ -16,6 +16,8 @@
 #include "test/UnitTest/ErrnoSetterMatcher.h"
 #include "test/UnitTest/Test.h"
 
+#include <sys/stat.h>
+
 // The tests here are exactly the same as those of dup2. We only test the
 // plumbing of the dup3 syscall and not the dup3 functionality itself as it is
 // a simple syscall wrapper. Testing dup3 functionality is beyond the scope of
diff --git a/libc/test/src/unistd/dup_test.cpp b/libc/test/src/unistd/dup_test.cpp
index 38c439125db3d7bb65bf81f2318f2449abe8f0d7..856b004fbe65d58f74464305a63e051262e5db61 100644
--- a/libc/test/src/unistd/dup_test.cpp
+++ b/libc/test/src/unistd/dup_test.cpp
@@ -16,6 +16,8 @@
 #include "test/UnitTest/ErrnoSetterMatcher.h"
 #include "test/UnitTest/Test.h"
 
+#include <sys/stat.h>
+
 TEST(LlvmLibcdupTest, ReadAndWriteViaDup) {
   libc_errno = 0;
   using LIBC_NAMESPACE::testing::ErrnoSetterMatcher::Succeeds;
diff --git a/libc/test/src/unistd/ftruncate_test.cpp b/libc/test/src/unistd/ftruncate_test.cpp
index ae743b385e220dbe611e230c0f737e61185c760b..fc68348e32ec6672987ceb6090bd47d4fa59dbf1 100644
--- a/libc/test/src/unistd/ftruncate_test.cpp
+++ b/libc/test/src/unistd/ftruncate_test.cpp
@@ -17,6 +17,8 @@
 #include "test/UnitTest/ErrnoSetterMatcher.h"
 #include "test/UnitTest/Test.h"
 
+#include <sys/stat.h>
+
 namespace cpp = LIBC_NAMESPACE::cpp;
 
 TEST(LlvmLibcFtruncateTest, CreateAndTruncate) {
diff --git a/libc/test/src/unistd/isatty_test.cpp b/libc/test/src/unistd/isatty_test.cpp
index fce4a3e5f5062aa383a8ff7ece89cf3644804a92..7bf8dd708bfa6f371523e890cc198c5fd9a6ad01 100644
--- a/libc/test/src/unistd/isatty_test.cpp
+++ b/libc/test/src/unistd/isatty_test.cpp
@@ -13,6 +13,8 @@
 #include "test/UnitTest/ErrnoSetterMatcher.h"
 #include "test/UnitTest/Test.h"
 
+#include <sys/stat.h>
+
 using LIBC_NAMESPACE::testing::ErrnoSetterMatcher::Fails;
 using LIBC_NAMESPACE::testing::ErrnoSetterMatcher::Succeeds;
 
diff --git a/libc/test/src/unistd/link_test.cpp b/libc/test/src/unistd/link_test.cpp
index b1b9383cb62ee07f7abcd16f88e51849a498f06b..2d5aa7588b08b902e761755666161f99c98d4ffb 100644
--- a/libc/test/src/unistd/link_test.cpp
+++ b/libc/test/src/unistd/link_test.cpp
@@ -14,6 +14,8 @@
 #include "test/UnitTest/ErrnoSetterMatcher.h"
 #include "test/UnitTest/Test.h"
 
+#include <sys/stat.h>
+
 TEST(LlvmLibcLinkTest, CreateAndUnlink) {
   using LIBC_NAMESPACE::testing::ErrnoSetterMatcher::Succeeds;
   constexpr const char *TEST_FILE = "testdata/link.test";
diff --git a/libc/test/src/unistd/linkat_test.cpp b/libc/test/src/unistd/linkat_test.cpp
index 2ef41cd4e75f6a7840b85ce93a12ed0257518e39..d1ffe37d63b48c71573e57323196ad8eca168fb4 100644
--- a/libc/test/src/unistd/linkat_test.cpp
+++ b/libc/test/src/unistd/linkat_test.cpp
@@ -14,6 +14,8 @@
 #include "test/UnitTest/ErrnoSetterMatcher.h"
 #include "test/UnitTest/Test.h"
 
+#include <sys/stat.h>
+
 TEST(LlvmLibcLinkatTest, CreateAndUnlink) {
   using LIBC_NAMESPACE::testing::ErrnoSetterMatcher::Succeeds;
   constexpr const char *TEST_DIR = "testdata";
diff --git a/libc/test/src/unistd/pread_pwrite_test.cpp b/libc/test/src/unistd/pread_pwrite_test.cpp
index 6819f559424f3e03eea69248552b013ce1e3974b..f90d1341c20108abd955176c9335edd9d6b58e97 100644
--- a/libc/test/src/unistd/pread_pwrite_test.cpp
+++ b/libc/test/src/unistd/pread_pwrite_test.cpp
@@ -17,6 +17,8 @@
 #include "test/UnitTest/ErrnoSetterMatcher.h"
 #include "test/UnitTest/Test.h"
 
+#include <sys/stat.h>
+
 TEST(LlvmLibcUniStd, PWriteAndPReadBackTest) {
   // The strategy here is that we first create a file and write to it. Next,
   // we open that file again and write at an offset. Finally, we open the
diff --git a/libc/test/src/unistd/read_write_test.cpp b/libc/test/src/unistd/read_write_test.cpp
index 20b51bb8082b1209c84912ecaae3b571f32c98d0..3007d4c38711579e1c99896178fd57cdbc0e5b36 100644
--- a/libc/test/src/unistd/read_write_test.cpp
+++ b/libc/test/src/unistd/read_write_test.cpp
@@ -15,6 +15,8 @@
 #include "test/UnitTest/ErrnoSetterMatcher.h"
 #include "test/UnitTest/Test.h"
 
+#include <sys/stat.h>
+
 TEST(LlvmLibcUniStd, WriteAndReadBackTest) {
   using LIBC_NAMESPACE::testing::ErrnoSetterMatcher::Succeeds;
   constexpr const char *TEST_FILE = "__unistd_read_write.test";
diff --git a/libc/test/src/unistd/symlink_test.cpp b/libc/test/src/unistd/symlink_test.cpp
index 84d963b7a207ce8f86d2a68cdb4bcdc26384c9e4..b25cfa4f857699592808c051c3f19ce1e8523e3b 100644
--- a/libc/test/src/unistd/symlink_test.cpp
+++ b/libc/test/src/unistd/symlink_test.cpp
@@ -14,6 +14,8 @@
 #include "test/UnitTest/ErrnoSetterMatcher.h"
 #include "test/UnitTest/Test.h"
 
+#include <sys/stat.h>
+
 TEST(LlvmLibcSymlinkTest, CreateAndUnlink) {
   using LIBC_NAMESPACE::testing::ErrnoSetterMatcher::Succeeds;
   constexpr const char *TEST_FILE_BASE = "symlink.test";
diff --git a/libc/test/src/unistd/symlinkat_test.cpp b/libc/test/src/unistd/symlinkat_test.cpp
index b0308787f2052a2644fd80648c283f140acf1d78..8aba2daee8cf3830431ad06d12c90adfbf5ad2f2 100644
--- a/libc/test/src/unistd/symlinkat_test.cpp
+++ b/libc/test/src/unistd/symlinkat_test.cpp
@@ -14,6 +14,8 @@
 #include "test/UnitTest/ErrnoSetterMatcher.h"
 #include "test/UnitTest/Test.h"
 
+#include <sys/stat.h>
+
 TEST(LlvmLibcSymlinkatTest, CreateAndUnlink) {
   using LIBC_NAMESPACE::testing::ErrnoSetterMatcher::Succeeds;
   constexpr const char *TEST_DIR = "testdata";
diff --git a/libc/test/src/unistd/syscall_test.cpp b/libc/test/src/unistd/syscall_test.cpp
index 211b27c3188c4d89e06fbdecd11bef497177d8fd..6a5ca47f8d494a8fc19ca58f64db206aab0e1570 100644
--- a/libc/test/src/unistd/syscall_test.cpp
+++ b/libc/test/src/unistd/syscall_test.cpp
@@ -12,6 +12,7 @@
 #include "test/UnitTest/Test.h"
 
 #include <fcntl.h>
+#include <sys/stat.h>    // For S_* flags.
 #include <sys/syscall.h> // For syscall numbers.
 #include <unistd.h>
 
diff --git a/libc/test/src/unistd/truncate_test.cpp b/libc/test/src/unistd/truncate_test.cpp
index 15940321e5ad456a7e12054a4cb9ede727bb88cf..6d8a3b8e53f7281e08b474bf6fcd9138d2b10a27 100644
--- a/libc/test/src/unistd/truncate_test.cpp
+++ b/libc/test/src/unistd/truncate_test.cpp
@@ -17,6 +17,8 @@
 #include "test/UnitTest/ErrnoSetterMatcher.h"
 #include "test/UnitTest/Test.h"
 
+#include <sys/stat.h>
+
 namespace cpp = LIBC_NAMESPACE::cpp;
 
 TEST(LlvmLibcTruncateTest, CreateAndTruncate) {
diff --git a/libc/test/src/unistd/unlink_test.cpp b/libc/test/src/unistd/unlink_test.cpp
index 8a64f88ee6f26db0b59f0a965bd691745f6fc4b2..77f65b5ecc6a1a12d9c00c4e587b0d0f9c5c1da0 100644
--- a/libc/test/src/unistd/unlink_test.cpp
+++ b/libc/test/src/unistd/unlink_test.cpp
@@ -13,6 +13,8 @@
 #include "test/UnitTest/ErrnoSetterMatcher.h"
 #include "test/UnitTest/Test.h"
 
+#include <sys/stat.h>
+
 TEST(LlvmLibcUnlinkTest, CreateAndUnlink) {
   using LIBC_NAMESPACE::testing::ErrnoSetterMatcher::Succeeds;
   constexpr const char *TEST_FILE = "testdata/unlink.test";
diff --git a/libc/test/src/unistd/unlinkat_test.cpp b/libc/test/src/unistd/unlinkat_test.cpp
index 5953085bf12b66ca7379223f511a18378f5bf7f7..22a20bc6ad07bce4cf23cdb3dab45da6a2f2e623 100644
--- a/libc/test/src/unistd/unlinkat_test.cpp
+++ b/libc/test/src/unistd/unlinkat_test.cpp
@@ -14,6 +14,8 @@
 #include "test/UnitTest/ErrnoSetterMatcher.h"
 #include "test/UnitTest/Test.h"
 
+#include <sys/stat.h>
+
 TEST(LlvmLibcUnlinkatTest, CreateAndDeleteTest) {
   using LIBC_NAMESPACE::testing::ErrnoSetterMatcher::Succeeds;
   constexpr const char *TEST_DIR = "testdata";
diff --git a/libc/utils/MPFRWrapper/MPFRUtils.cpp b/libc/utils/MPFRWrapper/MPFRUtils.cpp
index e3dffadf4ed9674fae495f71cebf6927d4366dc0..68146dff64c4e3e8eea0bacfb6cd7a59f0f50023 100644
--- a/libc/utils/MPFRWrapper/MPFRUtils.cpp
+++ b/libc/utils/MPFRWrapper/MPFRUtils.cpp
@@ -468,7 +468,7 @@ public:
       mpfr_sub(inputMPFR.value, value, inputMPFR.value, MPFR_RNDN);
       mpfr_abs(inputMPFR.value, inputMPFR.value, MPFR_RNDN);
       mpfr_mul_2si(inputMPFR.value, inputMPFR.value,
-                   -thisExponent + int(fputil::MantissaWidth<T>::VALUE),
+                   -thisExponent + int(fputil::FPBits<T>::MANTISSA_WIDTH),
                    MPFR_RNDN);
       return inputMPFR;
     }
@@ -496,12 +496,12 @@ public:
 
     mpfr_sub(minMPFR.value, pivot.value, minMPFR.value, MPFR_RNDN);
     mpfr_mul_2si(minMPFR.value, minMPFR.value,
-                 -minExponent + int(fputil::MantissaWidth<T>::VALUE),
+                 -minExponent + int(fputil::FPBits<T>::MANTISSA_WIDTH),
                  MPFR_RNDN);
 
     mpfr_sub(maxMPFR.value, maxMPFR.value, pivot.value, MPFR_RNDN);
     mpfr_mul_2si(maxMPFR.value, maxMPFR.value,
-                 -maxExponent + int(fputil::MantissaWidth<T>::VALUE),
+                 -maxExponent + int(fputil::FPBits<T>::MANTISSA_WIDTH),
                  MPFR_RNDN);
 
     mpfr_add(minMPFR.value, minMPFR.value, maxMPFR.value, MPFR_RNDN);
diff --git a/libcxx/CMakeLists.txt b/libcxx/CMakeLists.txt
index 7751bf1efc59d202427f1a9bc48a490b8e9f1034..75cb63222da35c52aaa5f9b8c084216e143809bf 100644
--- a/libcxx/CMakeLists.txt
+++ b/libcxx/CMakeLists.txt
@@ -651,6 +651,19 @@ get_sanitizer_flags(SANITIZER_FLAGS "${LLVM_USE_SANITIZER}")
 add_library(cxx-sanitizer-flags INTERFACE)
 target_compile_options(cxx-sanitizer-flags INTERFACE ${SANITIZER_FLAGS})
 
+# _LIBCPP_INSTRUMENTED_WITH_ASAN informs that library was built with ASan.
+# Defining _LIBCPP_INSTRUMENTED_WITH_ASAN while building the library with ASan is required.
+# Normally, the _LIBCPP_INSTRUMENTED_WITH_ASAN flag is used to keep information whether
+# dylibs are built with AddressSanitizer. However, when building libc++,
+# this flag needs to be defined so that the resulting dylib has all ASan functionalities guarded by this flag.
+# If the _LIBCPP_INSTRUMENTED_WITH_ASAN flag is not defined, then parts of the ASan instrumentation code in libc++
+# will not be compiled into it, resulting in false positives.
+# For context, read: https://github.com/llvm/llvm-project/pull/72677#pullrequestreview-1765402800
+string(FIND "${LLVM_USE_SANITIZER}" "Address" building_with_asan)
+if (NOT "${building_with_asan}" STREQUAL "-1")
+  config_define(ON _LIBCPP_INSTRUMENTED_WITH_ASAN)
+endif()
+
 # Link system libraries =======================================================
 function(cxx_link_system_libraries target)
   if (NOT MSVC)
@@ -664,15 +677,17 @@ function(cxx_link_system_libraries target)
     target_add_link_flags_if_supported(${target} PRIVATE "--unwindlib=none")
   endif()
 
-  if (LIBCXX_USE_COMPILER_RT)
-    find_compiler_rt_library(builtins LIBCXX_BUILTINS_LIBRARY)
-    if (LIBCXX_BUILTINS_LIBRARY)
-      target_link_libraries(${target} PRIVATE "${LIBCXX_BUILTINS_LIBRARY}")
+  if (MSVC)
+    if (LIBCXX_USE_COMPILER_RT)
+      find_compiler_rt_library(builtins LIBCXX_BUILTINS_LIBRARY)
+      if (LIBCXX_BUILTINS_LIBRARY)
+        target_link_libraries(${target} PRIVATE "${LIBCXX_BUILTINS_LIBRARY}")
+      endif()
+    elseif (LIBCXX_HAS_GCC_LIB)
+      target_link_libraries(${target} PRIVATE gcc)
+    elseif (LIBCXX_HAS_GCC_S_LIB)
+      target_link_libraries(${target} PRIVATE gcc_s)
     endif()
-  elseif (LIBCXX_HAS_GCC_LIB)
-    target_link_libraries(${target} PRIVATE gcc)
-  elseif (LIBCXX_HAS_GCC_S_LIB)
-    target_link_libraries(${target} PRIVATE gcc_s)
   endif()
 
   if (LIBCXX_HAS_ATOMIC_LIB)
diff --git a/libcxx/cmake/config-ix.cmake b/libcxx/cmake/config-ix.cmake
index a365517936e7565abb0db90ac5fc55072eb11f8b..1e8c2f5ce463213b572af12c23be28a4811bbfd8 100644
--- a/libcxx/cmake/config-ix.cmake
+++ b/libcxx/cmake/config-ix.cmake
@@ -45,7 +45,9 @@ else()
   endif()
 endif()
 
-if (CXX_SUPPORTS_NOSTDLIBXX_FLAG OR C_SUPPORTS_NODEFAULTLIBS_FLAG)
+# Only link against compiler-rt manually if we use -nodefaultlibs, since
+# otherwise the compiler will do the right thing on its own.
+if (NOT CXX_SUPPORTS_NOSTDLIBXX_FLAG AND C_SUPPORTS_NODEFAULTLIBS_FLAG)
   if (LIBCXX_USE_COMPILER_RT)
     include(HandleCompilerRT)
     find_compiler_rt_library(builtins LIBCXX_BUILTINS_LIBRARY
@@ -73,6 +75,9 @@ if (CXX_SUPPORTS_NOSTDLIBXX_FLAG OR C_SUPPORTS_NODEFAULTLIBS_FLAG)
                         moldname mingwex msvcrt)
     list(APPEND CMAKE_REQUIRED_LIBRARIES ${MINGW_LIBRARIES})
   endif()
+endif()
+
+if (CXX_SUPPORTS_NOSTDLIBXX_FLAG OR C_SUPPORTS_NODEFAULTLIBS_FLAG)
   if (CMAKE_C_FLAGS MATCHES -fsanitize OR CMAKE_CXX_FLAGS MATCHES -fsanitize)
     set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -fno-sanitize=all")
   endif ()
diff --git a/libcxx/docs/index.rst b/libcxx/docs/index.rst
index e8b4a95dbcffae15126aef3c55072041df8180ec..c7769bae6bb17d809c849c9c08fd337630e56a9b 100644
--- a/libcxx/docs/index.rst
+++ b/libcxx/docs/index.rst
@@ -202,6 +202,11 @@ Design Documents
 Build Bots and Test Coverage
 ============================
 
+.. image:: https://github.com/llvm/llvm-project/actions/workflows/libcxx-build-and-test.yaml/badge.svg?branch=main&event=schedule
+   :target: https://github.com/llvm/llvm-project/actions/workflows/libcxx-build-and-test.yaml?query=event%3Aschedule
+   :alt: Build and Test libc++
+
+* `Github Actions CI pipeline <https://github.com/llvm/llvm-project/actions/workflows/libcxx-build-and-test.yaml>`_
 * `Buildkite CI pipeline <https://buildkite.com/llvm-project/libcxx-ci>`_
 * `LLVM Buildbot Builders <https://lab.llvm.org/buildbot>`_
 * :ref:`Adding New CI Jobs <AddingNewCIJobs>`
diff --git a/libcxx/include/__chrono/day.h b/libcxx/include/__chrono/day.h
index c907c036c146ad08e52f7ac461ca32de7be280ce..d908453d5b082265f80dbb61e939ec11ada70595 100644
--- a/libcxx/include/__chrono/day.h
+++ b/libcxx/include/__chrono/day.h
@@ -46,7 +46,7 @@ _LIBCPP_HIDE_FROM_ABI inline constexpr
 bool operator==(const day& __lhs, const day& __rhs) noexcept
 { return static_cast<unsigned>(__lhs) == static_cast<unsigned>(__rhs); }
 
-_LIBCPP_HIDE_FROM_ABI constexpr strong_ordering operator<=>(const day& __lhs, const day& __rhs) noexcept {
+_LIBCPP_HIDE_FROM_ABI inline constexpr strong_ordering operator<=>(const day& __lhs, const day& __rhs) noexcept {
   return static_cast<unsigned>(__lhs) <=> static_cast<unsigned>(__rhs);
 }
 
diff --git a/libcxx/include/__chrono/hh_mm_ss.h b/libcxx/include/__chrono/hh_mm_ss.h
index 5bd452e57fa3c2175a022159c19bd1c362442c11..0adee2d60db8a4a8a9b2cb417554be5392759169 100644
--- a/libcxx/include/__chrono/hh_mm_ss.h
+++ b/libcxx/include/__chrono/hh_mm_ss.h
@@ -87,17 +87,17 @@ private:
 };
 _LIBCPP_CTAD_SUPPORTED_FOR_TYPE(hh_mm_ss);
 
-_LIBCPP_HIDE_FROM_ABI constexpr bool is_am(const hours& __h) noexcept { return __h >= hours( 0) && __h < hours(12); }
-_LIBCPP_HIDE_FROM_ABI constexpr bool is_pm(const hours& __h) noexcept { return __h >= hours(12) && __h < hours(24); }
+_LIBCPP_HIDE_FROM_ABI inline constexpr bool is_am(const hours& __h) noexcept { return __h >= hours( 0) && __h < hours(12); }
+_LIBCPP_HIDE_FROM_ABI inline constexpr bool is_pm(const hours& __h) noexcept { return __h >= hours(12) && __h < hours(24); }
 
-_LIBCPP_HIDE_FROM_ABI constexpr hours make12(const hours& __h) noexcept
+_LIBCPP_HIDE_FROM_ABI inline constexpr hours make12(const hours& __h) noexcept
 {
     if      (__h == hours( 0)) return hours(12);
     else if (__h <= hours(12)) return __h;
     else                       return __h - hours(12);
 }
 
-_LIBCPP_HIDE_FROM_ABI constexpr hours make24(const hours& __h, bool __is_pm) noexcept
+_LIBCPP_HIDE_FROM_ABI inline constexpr hours make24(const hours& __h, bool __is_pm) noexcept
 {
     if (__is_pm)
         return __h == hours(12) ? __h : __h + hours(12);
diff --git a/libcxx/include/__chrono/month.h b/libcxx/include/__chrono/month.h
index 7566e4ed29983f49f2d1a0fc1a02adff06dbfc67..2dee5d8c6c70d4085762c455e55809bff9315db3 100644
--- a/libcxx/include/__chrono/month.h
+++ b/libcxx/include/__chrono/month.h
@@ -46,7 +46,7 @@ _LIBCPP_HIDE_FROM_ABI inline constexpr
 bool operator==(const month& __lhs, const month& __rhs) noexcept
 { return static_cast<unsigned>(__lhs) == static_cast<unsigned>(__rhs); }
 
-_LIBCPP_HIDE_FROM_ABI constexpr strong_ordering operator<=>(const month& __lhs, const month& __rhs) noexcept {
+_LIBCPP_HIDE_FROM_ABI inline constexpr strong_ordering operator<=>(const month& __lhs, const month& __rhs) noexcept {
     return static_cast<unsigned>(__lhs) <=> static_cast<unsigned>(__rhs);
 }
 
diff --git a/libcxx/include/__chrono/monthday.h b/libcxx/include/__chrono/monthday.h
index 03fd7503a6b435ce256b880905a196a2f9ba35be..8403d9ec4eebe25c825f927688de510256709aab 100644
--- a/libcxx/include/__chrono/monthday.h
+++ b/libcxx/include/__chrono/monthday.h
@@ -59,7 +59,7 @@ _LIBCPP_HIDE_FROM_ABI inline constexpr
 bool operator==(const month_day& __lhs, const month_day& __rhs) noexcept
 { return __lhs.month() == __rhs.month() && __lhs.day() == __rhs.day(); }
 
-_LIBCPP_HIDE_FROM_ABI constexpr strong_ordering operator<=>(const month_day& __lhs, const month_day& __rhs) noexcept {
+_LIBCPP_HIDE_FROM_ABI inline constexpr strong_ordering operator<=>(const month_day& __lhs, const month_day& __rhs) noexcept {
     if (auto __c = __lhs.month() <=> __rhs.month(); __c != 0)
         return __c;
     return __lhs.day() <=> __rhs.day();
@@ -69,7 +69,7 @@ _LIBCPP_HIDE_FROM_ABI inline constexpr
 month_day operator/(const month& __lhs, const day& __rhs) noexcept
 { return month_day{__lhs, __rhs}; }
 
-_LIBCPP_HIDE_FROM_ABI constexpr
+_LIBCPP_HIDE_FROM_ABI inline constexpr
 month_day operator/(const day& __lhs, const month& __rhs) noexcept
 { return __rhs / __lhs; }
 
@@ -77,11 +77,11 @@ _LIBCPP_HIDE_FROM_ABI inline constexpr
 month_day operator/(const month& __lhs, int __rhs) noexcept
 { return __lhs / day(__rhs); }
 
-_LIBCPP_HIDE_FROM_ABI constexpr
+_LIBCPP_HIDE_FROM_ABI inline constexpr
 month_day operator/(int __lhs, const day& __rhs) noexcept
 { return month(__lhs) / __rhs; }
 
-_LIBCPP_HIDE_FROM_ABI constexpr
+_LIBCPP_HIDE_FROM_ABI inline constexpr
 month_day operator/(const day& __lhs, int __rhs) noexcept
 { return month(__rhs) / __lhs; }
 
@@ -99,7 +99,7 @@ _LIBCPP_HIDE_FROM_ABI inline constexpr
 bool operator==(const month_day_last& __lhs, const month_day_last& __rhs) noexcept
 { return __lhs.month() == __rhs.month(); }
 
-_LIBCPP_HIDE_FROM_ABI constexpr strong_ordering
+_LIBCPP_HIDE_FROM_ABI inline constexpr strong_ordering
 operator<=>(const month_day_last& __lhs, const month_day_last& __rhs) noexcept {
     return __lhs.month() <=> __rhs.month();
 }
diff --git a/libcxx/include/__chrono/weekday.h b/libcxx/include/__chrono/weekday.h
index 776d8ed3124caa726c5da58cbc38df70ef4c400c..292fcb40dc30613d5ae6547f6c7202d6ca111da1 100644
--- a/libcxx/include/__chrono/weekday.h
+++ b/libcxx/include/__chrono/weekday.h
@@ -85,7 +85,7 @@ _LIBCPP_HIDE_FROM_ABI inline constexpr
 bool operator>=(const weekday& __lhs, const weekday& __rhs) noexcept
 { return !(__lhs < __rhs); }
 
-_LIBCPP_HIDE_FROM_ABI constexpr
+_LIBCPP_HIDE_FROM_ABI inline constexpr
 weekday operator+(const weekday& __lhs, const days& __rhs) noexcept
 {
     auto const __mu = static_cast<long long>(__lhs.c_encoding()) + __rhs.count();
@@ -93,15 +93,15 @@ weekday operator+(const weekday& __lhs, const days& __rhs) noexcept
     return weekday{static_cast<unsigned>(__mu - __yr * 7)};
 }
 
-_LIBCPP_HIDE_FROM_ABI constexpr
+_LIBCPP_HIDE_FROM_ABI inline constexpr
 weekday operator+(const days& __lhs, const weekday& __rhs) noexcept
 { return __rhs + __lhs; }
 
-_LIBCPP_HIDE_FROM_ABI constexpr
+_LIBCPP_HIDE_FROM_ABI inline constexpr
 weekday operator-(const weekday& __lhs, const days& __rhs) noexcept
 { return __lhs + -__rhs; }
 
-_LIBCPP_HIDE_FROM_ABI constexpr
+_LIBCPP_HIDE_FROM_ABI inline constexpr
 days operator-(const weekday& __lhs, const weekday& __rhs) noexcept
 {
     const int __wdu = __lhs.c_encoding() - __rhs.c_encoding();
diff --git a/libcxx/include/__chrono/year_month.h b/libcxx/include/__chrono/year_month.h
index d1657b61015b9fb95a00e81ef7dcdc499f6fa6ba..320cf588ccd30e3943f55bdf74929b293b555324 100644
--- a/libcxx/include/__chrono/year_month.h
+++ b/libcxx/include/__chrono/year_month.h
@@ -53,13 +53,13 @@ _LIBCPP_HIDE_FROM_ABI inline constexpr
 bool operator==(const year_month& __lhs, const year_month& __rhs) noexcept
 { return __lhs.year() == __rhs.year() && __lhs.month() == __rhs.month(); }
 
-_LIBCPP_HIDE_FROM_ABI constexpr strong_ordering operator<=>(const year_month& __lhs, const year_month& __rhs) noexcept {
+_LIBCPP_HIDE_FROM_ABI inline constexpr strong_ordering operator<=>(const year_month& __lhs, const year_month& __rhs) noexcept {
     if (auto __c = __lhs.year() <=> __rhs.year(); __c != 0)
       return __c;
     return __lhs.month() <=> __rhs.month();
 }
 
-_LIBCPP_HIDE_FROM_ABI constexpr
+_LIBCPP_HIDE_FROM_ABI inline constexpr
 year_month operator+(const year_month& __lhs, const months& __rhs) noexcept
 {
     int __dmi = static_cast<int>(static_cast<unsigned>(__lhs.month())) - 1 + __rhs.count();
@@ -68,27 +68,27 @@ year_month operator+(const year_month& __lhs, const months& __rhs) noexcept
     return (__lhs.year() + years(__dy)) / month(static_cast<unsigned>(__dmi));
 }
 
-_LIBCPP_HIDE_FROM_ABI constexpr
+_LIBCPP_HIDE_FROM_ABI inline constexpr
 year_month operator+(const months& __lhs, const year_month& __rhs) noexcept
 { return __rhs + __lhs; }
 
-_LIBCPP_HIDE_FROM_ABI constexpr
+_LIBCPP_HIDE_FROM_ABI inline constexpr
 year_month operator+(const year_month& __lhs, const years& __rhs) noexcept
 { return (__lhs.year() + __rhs) / __lhs.month(); }
 
-_LIBCPP_HIDE_FROM_ABI constexpr
+_LIBCPP_HIDE_FROM_ABI inline constexpr
 year_month operator+(const years& __lhs, const year_month& __rhs) noexcept
 { return __rhs + __lhs; }
 
-_LIBCPP_HIDE_FROM_ABI constexpr
+_LIBCPP_HIDE_FROM_ABI inline constexpr
 months operator-(const year_month& __lhs, const year_month& __rhs) noexcept
 { return (__lhs.year() - __rhs.year()) + months(static_cast<unsigned>(__lhs.month()) - static_cast<unsigned>(__rhs.month())); }
 
-_LIBCPP_HIDE_FROM_ABI constexpr
+_LIBCPP_HIDE_FROM_ABI inline constexpr
 year_month operator-(const year_month& __lhs, const months& __rhs) noexcept
 { return __lhs + -__rhs; }
 
-_LIBCPP_HIDE_FROM_ABI constexpr
+_LIBCPP_HIDE_FROM_ABI inline constexpr
 year_month operator-(const year_month& __lhs, const years& __rhs) noexcept
 { return __lhs + -__rhs; }
 
diff --git a/libcxx/include/__chrono/year_month_day.h b/libcxx/include/__chrono/year_month_day.h
index ed5903f7d3f6956bb22c82fbfb6e4dbdcaaf7b1a..e84d2f8a838b40bc3f0f9fe7242add3c01418bfa 100644
--- a/libcxx/include/__chrono/year_month_day.h
+++ b/libcxx/include/__chrono/year_month_day.h
@@ -110,7 +110,7 @@ _LIBCPP_HIDE_FROM_ABI inline constexpr
 bool operator==(const year_month_day& __lhs, const year_month_day& __rhs) noexcept
 { return __lhs.year() == __rhs.year() && __lhs.month() == __rhs.month() && __lhs.day() == __rhs.day(); }
 
-_LIBCPP_HIDE_FROM_ABI constexpr strong_ordering
+_LIBCPP_HIDE_FROM_ABI inline constexpr strong_ordering
 operator<=>(const year_month_day& __lhs, const year_month_day& __rhs) noexcept {
     if (auto __c = __lhs.year() <=> __rhs.year(); __c != 0)
       return __c;
diff --git a/libcxx/include/__config_site.in b/libcxx/include/__config_site.in
index 6cade6f10d8acc1e9afe2795b1c1fe683d5d988f..7c002c5bfcf8e77f84e0cf14254b358d0896356f 100644
--- a/libcxx/include/__config_site.in
+++ b/libcxx/include/__config_site.in
@@ -29,6 +29,7 @@
 #cmakedefine _LIBCPP_HAS_NO_WIDE_CHARACTERS
 #cmakedefine _LIBCPP_HAS_NO_STD_MODULES
 #cmakedefine _LIBCPP_HAS_NO_TIME_ZONE_DATABASE
+#cmakedefine _LIBCPP_INSTRUMENTED_WITH_ASAN
 
 // PSTL backends
 #cmakedefine _LIBCPP_PSTL_CPU_BACKEND_SERIAL
diff --git a/libcxx/include/__ranges/lazy_split_view.h b/libcxx/include/__ranges/lazy_split_view.h
index 2c654bfd325a63c23305e3f3d8499e52ca9dc5b5..8ed4bcfdeb56d4b3f04d2a9c37340a56259738bf 100644
--- a/libcxx/include/__ranges/lazy_split_view.h
+++ b/libcxx/include/__ranges/lazy_split_view.h
@@ -437,7 +437,7 @@ lazy_split_view(_Range&&, range_value_t<_Range>)
 
 namespace views {
 namespace __lazy_split_view {
-struct __fn : __range_adaptor_closure<__fn> {
+struct __fn {
   template <class _Range, class _Pattern>
   [[nodiscard]] _LIBCPP_HIDE_FROM_ABI
   constexpr auto operator()(_Range&& __range, _Pattern&& __pattern) const
diff --git a/libcxx/include/__ranges/split_view.h b/libcxx/include/__ranges/split_view.h
index a27ac4ef7a1965ac4c3a9d7055d8fd4a8f961a83..7f03be3c346a42ce9882df191b635bc3c0e7e6fa 100644
--- a/libcxx/include/__ranges/split_view.h
+++ b/libcxx/include/__ranges/split_view.h
@@ -194,7 +194,7 @@ public:
 
 namespace views {
 namespace __split_view {
-struct __fn : __range_adaptor_closure<__fn> {
+struct __fn {
   // clang-format off
   template <class _Range, class _Pattern>
   _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI
diff --git a/libcxx/include/__ranges/take_view.h b/libcxx/include/__ranges/take_view.h
index 4204017d9249bcf8ce7cabbc0d594d967e7c7b41..518375d684abdd3cc85fc1c413bcc28c3cf2f36e 100644
--- a/libcxx/include/__ranges/take_view.h
+++ b/libcxx/include/__ranges/take_view.h
@@ -180,10 +180,9 @@ public:
     return __lhs.count() == 0 || __lhs.base() == __rhs.__end_;
   }
 
-  template<bool _OtherConst = !_Const>
+  template <bool _OtherConst = !_Const>
     requires sentinel_for<sentinel_t<_Base>, iterator_t<__maybe_const<_OtherConst, _View>>>
-  _LIBCPP_HIDE_FROM_ABI
-  friend constexpr bool operator==(const _Iter<_Const>& __lhs, const __sentinel& __rhs) {
+  _LIBCPP_HIDE_FROM_ABI friend constexpr bool operator==(const _Iter<_OtherConst>& __lhs, const __sentinel& __rhs) {
     return __lhs.count() == 0 || __lhs.base() == __rhs.__end_;
   }
 };
diff --git a/libcxx/include/__variant/monostate.h b/libcxx/include/__variant/monostate.h
index 8fec34008f2d5d8751008b6afd0748ecd318ec30..2944e41ac70426f6697124ef6d373471b617f6e5 100644
--- a/libcxx/include/__variant/monostate.h
+++ b/libcxx/include/__variant/monostate.h
@@ -25,25 +25,25 @@ _LIBCPP_BEGIN_NAMESPACE_STD
 
 struct _LIBCPP_TEMPLATE_VIS monostate {};
 
-_LIBCPP_HIDE_FROM_ABI constexpr bool operator==(monostate, monostate) noexcept { return true; }
+_LIBCPP_HIDE_FROM_ABI inline constexpr bool operator==(monostate, monostate) noexcept { return true; }
 
 #  if _LIBCPP_STD_VER >= 20
 
-_LIBCPP_HIDE_FROM_ABI constexpr strong_ordering operator<=>(monostate, monostate) noexcept {
+_LIBCPP_HIDE_FROM_ABI inline constexpr strong_ordering operator<=>(monostate, monostate) noexcept {
   return strong_ordering::equal;
 }
 
 #  else // _LIBCPP_STD_VER >= 20
 
-_LIBCPP_HIDE_FROM_ABI constexpr bool operator!=(monostate, monostate) noexcept { return false; }
+_LIBCPP_HIDE_FROM_ABI inline constexpr bool operator!=(monostate, monostate) noexcept { return false; }
 
-_LIBCPP_HIDE_FROM_ABI constexpr bool operator<(monostate, monostate) noexcept { return false; }
+_LIBCPP_HIDE_FROM_ABI inline constexpr bool operator<(monostate, monostate) noexcept { return false; }
 
-_LIBCPP_HIDE_FROM_ABI constexpr bool operator>(monostate, monostate) noexcept { return false; }
+_LIBCPP_HIDE_FROM_ABI inline constexpr bool operator>(monostate, monostate) noexcept { return false; }
 
-_LIBCPP_HIDE_FROM_ABI constexpr bool operator<=(monostate, monostate) noexcept { return true; }
+_LIBCPP_HIDE_FROM_ABI inline constexpr bool operator<=(monostate, monostate) noexcept { return true; }
 
-_LIBCPP_HIDE_FROM_ABI constexpr bool operator>=(monostate, monostate) noexcept { return true; }
+_LIBCPP_HIDE_FROM_ABI inline constexpr bool operator>=(monostate, monostate) noexcept { return true; }
 
 #  endif // _LIBCPP_STD_VER >= 20
 
diff --git a/libcxx/include/cmath b/libcxx/include/cmath
index 37f3c63fcef8a83c0221766244a503948bd1877f..e8a2acf078cd58cc011a47ca21d89d2112380856 100644
--- a/libcxx/include/cmath
+++ b/libcxx/include/cmath
@@ -798,13 +798,13 @@ _Fp __lerp(_Fp __a, _Fp __b, _Fp __t) noexcept {
         return __x < __b ? __x : __b;
 }
 
-_LIBCPP_HIDE_FROM_ABI constexpr float
+_LIBCPP_HIDE_FROM_ABI inline constexpr float
 lerp(float __a, float __b, float __t)                   _NOEXCEPT { return __lerp(__a, __b, __t); }
 
-_LIBCPP_HIDE_FROM_ABI constexpr double
+_LIBCPP_HIDE_FROM_ABI inline constexpr double
 lerp(double __a, double __b, double __t)                _NOEXCEPT { return __lerp(__a, __b, __t); }
 
-_LIBCPP_HIDE_FROM_ABI constexpr long double
+_LIBCPP_HIDE_FROM_ABI inline constexpr long double
 lerp(long double __a, long double __b, long double __t) _NOEXCEPT { return __lerp(__a, __b, __t); }
 
 template <class _A1, class _A2, class _A3>
diff --git a/libcxx/include/complex b/libcxx/include/complex
index 7017f25e6c5e0bfd12a4b2bfb1ee8ee2216f9d10..44579b1ad528548dcdf8daa26a872a732a52ce72 100644
--- a/libcxx/include/complex
+++ b/libcxx/include/complex
@@ -1503,34 +1503,34 @@ inline namespace literals
 {
   inline namespace complex_literals
   {
-    _LIBCPP_HIDE_FROM_ABI constexpr complex<long double> operator""il(long double __im)
+    _LIBCPP_HIDE_FROM_ABI inline constexpr complex<long double> operator""il(long double __im)
     {
         return { 0.0l, __im };
     }
 
-    _LIBCPP_HIDE_FROM_ABI constexpr complex<long double> operator""il(unsigned long long __im)
+    _LIBCPP_HIDE_FROM_ABI inline constexpr complex<long double> operator""il(unsigned long long __im)
     {
         return { 0.0l, static_cast<long double>(__im) };
     }
 
 
-    _LIBCPP_HIDE_FROM_ABI constexpr complex<double> operator""i(long double __im)
+    _LIBCPP_HIDE_FROM_ABI inline constexpr complex<double> operator""i(long double __im)
     {
         return { 0.0, static_cast<double>(__im) };
     }
 
-    _LIBCPP_HIDE_FROM_ABI constexpr complex<double> operator""i(unsigned long long __im)
+    _LIBCPP_HIDE_FROM_ABI inline constexpr complex<double> operator""i(unsigned long long __im)
     {
         return { 0.0, static_cast<double>(__im) };
     }
 
 
-    _LIBCPP_HIDE_FROM_ABI constexpr complex<float> operator""if(long double __im)
+    _LIBCPP_HIDE_FROM_ABI inline constexpr complex<float> operator""if(long double __im)
     {
         return { 0.0f, static_cast<float>(__im) };
     }
 
-    _LIBCPP_HIDE_FROM_ABI constexpr complex<float> operator""if(unsigned long long __im)
+    _LIBCPP_HIDE_FROM_ABI inline constexpr complex<float> operator""if(unsigned long long __im)
     {
         return { 0.0f, static_cast<float>(__im) };
     }
diff --git a/libcxx/include/cstddef b/libcxx/include/cstddef
index 3844d4a373323dbdfc17933ac7212aaa87bd7af4..24be0fe780585b187636e4c0348f68fe7a129124 100644
--- a/libcxx/include/cstddef
+++ b/libcxx/include/cstddef
@@ -71,7 +71,7 @@ namespace std  // purposefully not versioned
 {
 enum class byte : unsigned char {};
 
-_LIBCPP_HIDE_FROM_ABI constexpr byte  operator| (byte  __lhs, byte __rhs) noexcept
+_LIBCPP_HIDE_FROM_ABI inline constexpr byte  operator| (byte  __lhs, byte __rhs) noexcept
 {
     return static_cast<byte>(
       static_cast<unsigned char>(
@@ -79,10 +79,10 @@ _LIBCPP_HIDE_FROM_ABI constexpr byte  operator| (byte  __lhs, byte __rhs) noexce
     ));
 }
 
-_LIBCPP_HIDE_FROM_ABI constexpr byte& operator|=(byte& __lhs, byte __rhs) noexcept
+_LIBCPP_HIDE_FROM_ABI inline constexpr byte& operator|=(byte& __lhs, byte __rhs) noexcept
 { return __lhs = __lhs | __rhs; }
 
-_LIBCPP_HIDE_FROM_ABI constexpr byte  operator& (byte  __lhs, byte __rhs) noexcept
+_LIBCPP_HIDE_FROM_ABI inline constexpr byte  operator& (byte  __lhs, byte __rhs) noexcept
 {
     return static_cast<byte>(
       static_cast<unsigned char>(
@@ -90,10 +90,10 @@ _LIBCPP_HIDE_FROM_ABI constexpr byte  operator& (byte  __lhs, byte __rhs) noexce
     ));
 }
 
-_LIBCPP_HIDE_FROM_ABI constexpr byte& operator&=(byte& __lhs, byte __rhs) noexcept
+_LIBCPP_HIDE_FROM_ABI inline constexpr byte& operator&=(byte& __lhs, byte __rhs) noexcept
 { return __lhs = __lhs & __rhs; }
 
-_LIBCPP_HIDE_FROM_ABI constexpr byte  operator^ (byte  __lhs, byte __rhs) noexcept
+_LIBCPP_HIDE_FROM_ABI inline constexpr byte  operator^ (byte  __lhs, byte __rhs) noexcept
 {
     return static_cast<byte>(
       static_cast<unsigned char>(
@@ -101,10 +101,10 @@ _LIBCPP_HIDE_FROM_ABI constexpr byte  operator^ (byte  __lhs, byte __rhs) noexce
     ));
 }
 
-_LIBCPP_HIDE_FROM_ABI constexpr byte& operator^=(byte& __lhs, byte __rhs) noexcept
+_LIBCPP_HIDE_FROM_ABI inline constexpr byte& operator^=(byte& __lhs, byte __rhs) noexcept
 { return __lhs = __lhs ^ __rhs; }
 
-_LIBCPP_HIDE_FROM_ABI constexpr byte  operator~ (byte __b) noexcept
+_LIBCPP_HIDE_FROM_ABI inline constexpr byte  operator~ (byte __b) noexcept
 {
     return static_cast<byte>(
       static_cast<unsigned char>(
diff --git a/libcxx/include/deque b/libcxx/include/deque
index b5d094dc415ddfd4eaddb9ee6b78876f96ae944a..d45793c502a9a74db6430d290f5442214ccfe242 100644
--- a/libcxx/include/deque
+++ b/libcxx/include/deque
@@ -967,12 +967,18 @@ public:
 // For more details, see the "Using libc++" documentation page or
 // the documentation for __sanitizer_annotate_contiguous_container.
     _LIBCPP_HIDE_FROM_ABI void __annotate_double_ended_contiguous_container(
-       [[__maybe_unused__]] const void* __beg,
-       [[__maybe_unused__]] const void* __end,
-       [[__maybe_unused__]] const void* __old_con_beg,
-       [[__maybe_unused__]] const void* __old_con_end,
-       [[__maybe_unused__]] const void* __new_con_beg,
-       [[__maybe_unused__]] const void* __new_con_end) const {
+        const void* __beg,
+        const void* __end,
+        const void* __old_con_beg,
+        const void* __old_con_end,
+        const void* __new_con_beg,
+        const void* __new_con_end) const {
+        (void)__beg;
+        (void)__end;
+        (void)__old_con_beg;
+        (void)__old_con_end;
+        (void)__new_con_beg;
+        (void)__new_con_end;
 #ifndef _LIBCPP_HAS_NO_ASAN
         if (__beg != nullptr && __asan_annotate_container_with_allocator<_Allocator>::value)
             __sanitizer_annotate_double_ended_contiguous_container(
@@ -982,10 +988,14 @@ public:
 
     _LIBCPP_HIDE_FROM_ABI
     void __annotate_from_to(
-            [[__maybe_unused__]] size_type __beg,
-            [[__maybe_unused__]] size_type __end,
-            [[__maybe_unused__]] __asan_annotation_type __annotation_type,
-            [[__maybe_unused__]] __asan_annotation_place __place) const _NOEXCEPT {
+            size_type __beg,
+            size_type __end,
+            __asan_annotation_type __annotation_type,
+            __asan_annotation_place __place) const _NOEXCEPT {
+        (void)__beg;
+        (void)__end;
+        (void)__annotation_type;
+        (void)__place;
 #ifndef _LIBCPP_HAS_NO_ASAN
         // __beg - index of the first item to annotate
         // __end - index behind the last item to annotate (so last item + 1)
diff --git a/libcxx/include/string b/libcxx/include/string
index 9c97abefcb8d07c607be406acd21e6e28fb313d0..5bb4e941af36f74666bb5a891579dc07d92600f7 100644
--- a/libcxx/include/string
+++ b/libcxx/include/string
@@ -649,6 +649,17 @@ basic_string<char32_t> operator""s( const char32_t *str, size_t len );
 _LIBCPP_PUSH_MACROS
 #include <__undef_macros>
 
+#if !defined(_LIBCPP_HAS_NO_ASAN) && defined(_LIBCPP_INSTRUMENTED_WITH_ASAN)
+#  define _LIBCPP_STRING_INTERNAL_MEMORY_ACCESS __attribute__((__no_sanitize__("address")))
+// This macro disables AddressSanitizer (ASan) instrumentation for a specific function,
+// allowing memory accesses that would normally trigger ASan errors to proceed without crashing.
+// This is useful for accessing parts of objects memory, which should not be accessed,
+// such as unused bytes in short strings, that should never be accessed
+// by other parts of the program.
+#else
+#  define _LIBCPP_STRING_INTERNAL_MEMORY_ACCESS
+#endif
+#define _LIBCPP_SHORT_STRING_ANNOTATIONS_ALLOWED false
 
 _LIBCPP_BEGIN_NAMESPACE_STD
 
@@ -706,6 +717,9 @@ struct __init_with_sentinel_tag {};
 template<class _CharT, class _Traits, class _Allocator>
 class basic_string
 {
+private:
+  using __default_allocator_type = allocator<_CharT>;
+
 public:
     typedef basic_string                                 __self;
     typedef basic_string_view<_CharT, _Traits>           __self_view;
@@ -860,6 +874,7 @@ private:
             __set_long_pointer(__allocation);
             __set_long_size(__size);
         }
+        __annotate_new(__size);
     }
 
     template <class _Iter, class _Sent>
@@ -882,7 +897,9 @@ public:
 
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string()
       _NOEXCEPT_(is_nothrow_default_constructible<allocator_type>::value)
-      : __r_(__value_init_tag(), __default_init_tag()) {}
+      : __r_(__value_init_tag(), __default_init_tag()) {
+            __annotate_new(0);
+  }
 
   _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 explicit basic_string(const allocator_type& __a)
 #if _LIBCPP_STD_VER <= 14
@@ -890,44 +907,65 @@ public:
 #else
       _NOEXCEPT
 #endif
-      : __r_(__value_init_tag(), __a) {}
+      : __r_(__value_init_tag(), __a) {
+            __annotate_new(0);
+  }
 
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string(const basic_string& __str)
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_STRING_INTERNAL_MEMORY_ACCESS basic_string(const basic_string& __str)
       : __r_(__default_init_tag(), __alloc_traits::select_on_container_copy_construction(__str.__alloc())) {
     if (!__str.__is_long())
+    {
       __r_.first() = __str.__r_.first();
+      __annotate_new(__get_short_size());
+    }
     else
       __init_copy_ctor_external(std::__to_address(__str.__get_long_pointer()), __str.__get_long_size());
   }
 
-  _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string(const basic_string& __str, const allocator_type& __a)
+  _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_STRING_INTERNAL_MEMORY_ACCESS basic_string(const basic_string& __str, const allocator_type& __a)
       : __r_(__default_init_tag(), __a) {
     if (!__str.__is_long())
+    {
       __r_.first() = __str.__r_.first();
+      __annotate_new(__get_short_size());
+    }
     else
       __init_copy_ctor_external(std::__to_address(__str.__get_long_pointer()), __str.__get_long_size());
   }
 
 #ifndef _LIBCPP_CXX03_LANG
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string(basic_string&& __str)
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
+  basic_string(basic_string&& __str)
 #  if _LIBCPP_STD_VER <= 14
       _NOEXCEPT_(is_nothrow_move_constructible<allocator_type>::value)
 #  else
       _NOEXCEPT
 #  endif
-      : __r_(std::move(__str.__r_)) {
+      // Turning off ASan instrumentation for variable initialization with _LIBCPP_STRING_INTERNAL_MEMORY_ACCESS
+      // does not work consistently during initialization of __r_, so we instead unpoison __str's memory manually first.
+      // __str's memory needs to be unpoisoned only in the case where it's a short string.
+      : __r_( ( (__str.__is_long() ? 0 : (__str.__annotate_delete(), 0)), std::move(__str.__r_)) ) {
     __str.__r_.first() = __rep();
+    __str.__annotate_new(0);
+    if(!__is_long())
+      __annotate_new(size());
   }
 
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string(basic_string&& __str, const allocator_type& __a)
+  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
+  basic_string(basic_string&& __str, const allocator_type& __a)
       : __r_(__default_init_tag(), __a) {
     if (__str.__is_long() && __a != __str.__alloc()) // copy, not move
       __init(std::__to_address(__str.__get_long_pointer()), __str.__get_long_size());
     else {
       if (__libcpp_is_constant_evaluated())
         __r_.first() = __rep();
+      if (!__str.__is_long())
+        __str.__annotate_delete();
       __r_.first() = __str.__r_.first();
       __str.__r_.first() = __rep();
+      __str.__annotate_new(0);
+      if(!__is_long() && this != &__str)
+        __annotate_new(size());
     }
   }
 #endif // _LIBCPP_CXX03_LANG
@@ -1085,6 +1123,7 @@ public:
 #endif // _LIBCPP_CXX03_LANG
 
   inline _LIBCPP_CONSTEXPR_SINCE_CXX20 ~basic_string() {
+    __annotate_delete();
     if (__is_long())
       __alloc_traits::deallocate(__alloc(), __get_long_pointer(), __get_long_cap());
   }
@@ -1092,7 +1131,7 @@ public:
     _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
     operator __self_view() const _NOEXCEPT { return __self_view(data(), size()); }
 
-    _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& operator=(const basic_string& __str);
+    _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_STRING_INTERNAL_MEMORY_ACCESS basic_string& operator=(const basic_string& __str);
 
     template <class _Tp, __enable_if_t<__can_be_converted_to_string_view<_CharT, _Traits, _Tp>::value &&
                                            !__is_same_uncvref<_Tp, basic_string>::value, int> = 0>
@@ -1102,8 +1141,8 @@ public:
     }
 
 #ifndef _LIBCPP_CXX03_LANG
-  _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& operator=(basic_string&& __str)
-      _NOEXCEPT_((__noexcept_move_assign_container<_Allocator, __alloc_traits>::value)) {
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string&
+    operator=(basic_string&& __str) _NOEXCEPT_((__noexcept_move_assign_container<_Allocator, __alloc_traits>::value)) {
     __move_assign(__str, integral_constant<bool, __alloc_traits::propagate_on_container_move_assignment::value>());
     return *this;
   }
@@ -1116,7 +1155,7 @@ public:
 #if _LIBCPP_STD_VER >= 23
     basic_string& operator=(nullptr_t) = delete;
 #endif
-    _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& operator=(value_type __c);
+    _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_STRING_INTERNAL_MEMORY_ACCESS basic_string& operator=(value_type __c);
 
     _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
     iterator begin() _NOEXCEPT
@@ -1339,12 +1378,22 @@ public:
   void __move_assign(basic_string&& __str, size_type __pos, size_type __len) {
     // Pilfer the allocation from __str.
     _LIBCPP_ASSERT_INTERNAL(__alloc() == __str.__alloc(), "__move_assign called with wrong allocator");
+    size_type __old_sz = __str.size();
+    if (!__str.__is_long())
+      __str.__annotate_delete();
     __r_.first() = __str.__r_.first();
     __str.__r_.first() = __rep();
+    __str.__annotate_new(0);
 
     _Traits::move(data(), data() + __pos, __len);
     __set_size(__len);
     _Traits::assign(data()[__len], value_type());
+
+    if (!__is_long()) {
+      __annotate_new(__len);
+    } else if(__old_sz > __len) {
+      __annotate_shrink(__old_sz);
+    }
   }
 #endif
 
@@ -1742,7 +1791,7 @@ private:
 
     _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __shrink_or_extend(size_type __target_capacity);
 
-    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_STRING_INTERNAL_MEMORY_ACCESS
     bool __is_long() const _NOEXCEPT {
         if (__libcpp_is_constant_evaluated() && __builtin_constant_p(__r_.first().__l.__is_long_)) {
             return __r_.first().__l.__is_long_;
@@ -1782,6 +1831,7 @@ private:
         value_type* __p;
         if (__cap - __sz >= __n)
         {
+          __annotate_increase(__n);
             __p = std::__to_address(__get_pointer());
             size_type __n_move = __sz - __ip;
             if (__n_move != 0)
@@ -1808,7 +1858,7 @@ private:
     _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX14 allocator_type& __alloc() _NOEXCEPT { return __r_.second(); }
     _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR const allocator_type& __alloc() const _NOEXCEPT { return __r_.second(); }
 
-    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_STRING_INTERNAL_MEMORY_ACCESS
     void __set_short_size(size_type __s) _NOEXCEPT {
         _LIBCPP_ASSERT_INTERNAL(
             __s < __min_cap, "__s should never be greater than or equal to the short string capacity");
@@ -1816,7 +1866,7 @@ private:
         __r_.first().__s.__is_long_ = false;
     }
 
-    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_STRING_INTERNAL_MEMORY_ACCESS
     size_type __get_short_size() const _NOEXCEPT {
         _LIBCPP_ASSERT_INTERNAL(
             !__r_.first().__s.__is_long_, "String has to be short when trying to get the short size");
@@ -1866,6 +1916,43 @@ private:
     const_pointer __get_pointer() const _NOEXCEPT
         {return __is_long() ? __get_long_pointer() : __get_short_pointer();}
 
+    // The following functions are no-ops outside of AddressSanitizer mode.
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __annotate_contiguous_container(const void* __old_mid, const void* __new_mid) const {
+        (void)__old_mid;
+        (void)__new_mid;
+#if !defined(_LIBCPP_HAS_NO_ASAN) && defined(_LIBCPP_INSTRUMENTED_WITH_ASAN)
+        const void* __begin = data();
+        const void* __end = data() + capacity() + 1;
+        if (!__libcpp_is_constant_evaluated() && __begin != nullptr && is_same<allocator_type, __default_allocator_type>::value)
+          __sanitizer_annotate_contiguous_container(__begin, __end, __old_mid, __new_mid);
+#endif
+    }
+
+    // ASan: short string is poisoned if and only if this function returns true.
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 bool __asan_short_string_is_annotated() const _NOEXCEPT {
+      return _LIBCPP_SHORT_STRING_ANNOTATIONS_ALLOWED && !__libcpp_is_constant_evaluated();
+    }
+
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __annotate_new(size_type __current_size) const _NOEXCEPT {
+      if (!__libcpp_is_constant_evaluated() && (__asan_short_string_is_annotated() || __is_long()))
+        __annotate_contiguous_container(data() + capacity() + 1, data() + __current_size + 1);
+    }
+
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __annotate_delete() const _NOEXCEPT {
+      if (!__libcpp_is_constant_evaluated() && (__asan_short_string_is_annotated() || __is_long()))
+        __annotate_contiguous_container(data() + size() + 1, data() + capacity() + 1);
+    }
+
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __annotate_increase(size_type __n) const _NOEXCEPT {
+      if (!__libcpp_is_constant_evaluated() && (__asan_short_string_is_annotated() || __is_long()))
+        __annotate_contiguous_container(data() + size() + 1, data() + size() + 1 + __n);
+    }
+
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 void __annotate_shrink(size_type __old_size) const _NOEXCEPT {
+      if (!__libcpp_is_constant_evaluated() && (__asan_short_string_is_annotated() || __is_long()))
+        __annotate_contiguous_container(data() + __old_size + 1, data() + size() + 1);
+    }
+
     template <size_type __a> static
         _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
         size_type __align_it(size_type __s) _NOEXCEPT
@@ -1968,6 +2055,7 @@ private:
                 }
                 else
                 {
+                    __annotate_delete();
                     allocator_type __a = __str.__alloc();
                     auto __allocation = std::__allocate_at_least(__a, __str.__get_long_cap());
                     __begin_lifetime(__allocation.ptr, __allocation.count);
@@ -1977,6 +2065,7 @@ private:
                     __set_long_pointer(__allocation.ptr);
                     __set_long_cap(__allocation.count);
                     __set_long_size(__str.size());
+                    __annotate_new(__get_long_size());
                 }
             }
         }
@@ -1989,7 +2078,7 @@ private:
     _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
     void __move_assign(basic_string& __str, false_type)
         _NOEXCEPT_(__alloc_traits::is_always_equal::value);
-    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
+    _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_STRING_INTERNAL_MEMORY_ACCESS
     void __move_assign(basic_string& __str, true_type)
 #if _LIBCPP_STD_VER >= 17
         _NOEXCEPT;
@@ -2024,18 +2113,28 @@ private:
 
     // Assigns the value in __s, guaranteed to be __n < __min_cap in length.
     inline _LIBCPP_CONSTEXPR_SINCE_CXX20 basic_string& __assign_short(const value_type* __s, size_type __n) {
+      size_type __old_size = size();
+      if (__n > __old_size)
+        __annotate_increase(__n - __old_size);
       pointer __p = __is_long()
                         ? (__set_long_size(__n), __get_long_pointer())
                         : (__set_short_size(__n), __get_short_pointer());
       traits_type::move(std::__to_address(__p), __s, __n);
       traits_type::assign(__p[__n], value_type());
+      if (__old_size > __n)
+        __annotate_shrink(__old_size);
       return *this;
     }
 
     _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
     basic_string& __null_terminate_at(value_type* __p, size_type __newsz) {
+      size_type __old_size = size();
+      if (__newsz > __old_size)
+        __annotate_increase(__newsz - __old_size);
       __set_size(__newsz);
       traits_type::assign(__p[__newsz], value_type());
+      if (__old_size > __newsz)
+        __annotate_shrink(__old_size);
       return *this;
     }
 
@@ -2142,6 +2241,7 @@ void basic_string<_CharT, _Traits, _Allocator>::__init(const value_type* __s,
     }
     traits_type::copy(std::__to_address(__p), __s, __sz);
     traits_type::assign(__p[__sz], value_type());
+    __annotate_new(__sz);
 }
 
 template <class _CharT, class _Traits, class _Allocator>
@@ -2170,6 +2270,7 @@ basic_string<_CharT, _Traits, _Allocator>::__init(const value_type* __s, size_ty
     }
     traits_type::copy(std::__to_address(__p), __s, __sz);
     traits_type::assign(__p[__sz], value_type());
+    __annotate_new(__sz);
 }
 
 template <class _CharT, class _Traits, class _Allocator>
@@ -2194,6 +2295,7 @@ void basic_string<_CharT, _Traits, _Allocator>::__init_copy_ctor_external(
     __set_long_size(__sz);
   }
   traits_type::copy(std::__to_address(__p), __s, __sz + 1);
+  __annotate_new(__sz);
 }
 
 template <class _CharT, class _Traits, class _Allocator>
@@ -2223,6 +2325,7 @@ basic_string<_CharT, _Traits, _Allocator>::__init(size_type __n, value_type __c)
     }
     traits_type::assign(std::__to_address(__p), __n, __c);
     traits_type::assign(__p[__n], value_type());
+    __annotate_new(__n);
 }
 
 template <class _CharT, class _Traits, class _Allocator>
@@ -2238,6 +2341,7 @@ template <class _InputIterator, class _Sentinel>
 _LIBCPP_HIDE_FROM_ABI _LIBCPP_CONSTEXPR_SINCE_CXX20
 void basic_string<_CharT, _Traits, _Allocator>::__init_with_sentinel(_InputIterator __first, _Sentinel __last) {
     __r_.first() = __rep();
+    __annotate_new(0);
 
 #ifndef _LIBCPP_HAS_NO_EXCEPTIONS
     try
@@ -2249,6 +2353,7 @@ void basic_string<_CharT, _Traits, _Allocator>::__init_with_sentinel(_InputItera
     }
     catch (...)
     {
+        __annotate_delete();
         if (__is_long())
             __alloc_traits::deallocate(__alloc(), __get_long_pointer(), __get_long_cap());
         throw;
@@ -2309,6 +2414,7 @@ void basic_string<_CharT, _Traits, _Allocator>::__init_with_size(
         throw;
     }
 #endif  // _LIBCPP_HAS_NO_EXCEPTIONS
+    __annotate_new(__sz);
 }
 
 template <class _CharT, class _Traits, class _Allocator>
@@ -2325,6 +2431,7 @@ basic_string<_CharT, _Traits, _Allocator>::__grow_by_and_replace
     size_type __cap = __old_cap < __ms / 2 - __alignment ?
                           __recommend(std::max(__old_cap + __delta_cap, 2 * __old_cap)) :
                           __ms - 1;
+    __annotate_delete();
     auto __allocation = std::__allocate_at_least(__alloc(), __cap + 1);
     pointer __p = __allocation.ptr;
     __begin_lifetime(__p, __allocation.count);
@@ -2344,6 +2451,7 @@ basic_string<_CharT, _Traits, _Allocator>::__grow_by_and_replace
     __old_sz = __n_copy + __n_add + __sec_cp_sz;
     __set_long_size(__old_sz);
     traits_type::assign(__p[__old_sz], value_type());
+    __annotate_new(__old_cap + __delta_cap);
 }
 
 // __grow_by is deprecated because it does not set the size. It may not update the size when the size is changed, and it
@@ -2366,6 +2474,7 @@ basic_string<_CharT, _Traits, _Allocator>::__grow_by(size_type __old_cap, size_t
     size_type __cap = __old_cap < __ms / 2 - __alignment ?
                           __recommend(std::max(__old_cap + __delta_cap, 2 * __old_cap)) :
                           __ms - 1;
+    __annotate_delete();
     auto __allocation = std::__allocate_at_least(__alloc(), __cap + 1);
     pointer __p = __allocation.ptr;
     __begin_lifetime(__p, __allocation.count);
@@ -2396,6 +2505,7 @@ basic_string<_CharT, _Traits, _Allocator>::__grow_by_without_replace(
     __grow_by(__old_cap, __delta_cap, __old_sz, __n_copy, __n_del, __n_add);
     _LIBCPP_SUPPRESS_DEPRECATED_POP
     __set_long_size(__old_sz - __n_del + __n_add);
+    __annotate_new(__old_sz - __n_del + __n_add);
 }
 
 // assign
@@ -2408,10 +2518,15 @@ basic_string<_CharT, _Traits, _Allocator>::__assign_no_alias(
     const value_type* __s, size_type __n) {
   size_type __cap = __is_short ? static_cast<size_type>(__min_cap) : __get_long_cap();
   if (__n < __cap) {
+    size_type __old_size = __is_short ? __get_short_size() : __get_long_size();
+    if (__n > __old_size)
+      __annotate_increase(__n - __old_size);
     pointer __p = __is_short ? __get_short_pointer() : __get_long_pointer();
     __is_short ? __set_short_size(__n) : __set_long_size(__n);
     traits_type::copy(std::__to_address(__p), __s, __n);
     traits_type::assign(__p[__n], value_type());
+    if (__old_size > __n)
+      __annotate_shrink(__old_size);
   } else {
     size_type __sz = __is_short ? __get_short_size() : __get_long_size();
     __grow_by_and_replace(__cap - 1, __n - __cap + 1, __sz, 0, __sz, __n, __s);
@@ -2426,6 +2541,9 @@ basic_string<_CharT, _Traits, _Allocator>::__assign_external(
     const value_type* __s, size_type __n) {
   size_type __cap = capacity();
   if (__cap >= __n) {
+    size_type __old_size = size();
+    if (__n > __old_size)
+      __annotate_increase(__n - __old_size);
     value_type* __p = std::__to_address(__get_pointer());
     traits_type::move(__p, __s, __n);
     return __null_terminate_at(__p, __n);
@@ -2453,11 +2571,15 @@ basic_string<_CharT, _Traits, _Allocator>&
 basic_string<_CharT, _Traits, _Allocator>::assign(size_type __n, value_type __c)
 {
     size_type __cap = capacity();
+    size_type __old_size = size();
     if (__cap < __n)
     {
         size_type __sz = size();
         __grow_by_without_replace(__cap, __n - __cap, __sz, 0, __sz);
+        __annotate_increase(__n);
     }
+    else if(__n > __old_size)
+        __annotate_increase(__n - __old_size);
     value_type* __p = std::__to_address(__get_pointer());
     traits_type::assign(__p, __n, __c);
     return __null_terminate_at(__p, __n);
@@ -2468,24 +2590,26 @@ _LIBCPP_CONSTEXPR_SINCE_CXX20
 basic_string<_CharT, _Traits, _Allocator>&
 basic_string<_CharT, _Traits, _Allocator>::operator=(value_type __c)
 {
-    pointer __p;
-    if (__is_long())
-    {
-        __p = __get_long_pointer();
-        __set_long_size(1);
-    }
-    else
-    {
-        __p = __get_short_pointer();
-        __set_short_size(1);
-    }
-    traits_type::assign(*__p, __c);
-    traits_type::assign(*++__p, value_type());
-    return *this;
+  pointer __p;
+  size_type __old_size = size();
+  if (__old_size == 0)
+    __annotate_increase(1);
+  if (__is_long()) {
+    __p = __get_long_pointer();
+    __set_long_size(1);
+  } else {
+    __p = __get_short_pointer();
+    __set_short_size(1);
+  }
+  traits_type::assign(*__p, __c);
+  traits_type::assign(*++__p, value_type());
+  if (__old_size > 1)
+    __annotate_shrink(__old_size);
+  return *this;
 }
 
 template <class _CharT, class _Traits, class _Allocator>
-_LIBCPP_CONSTEXPR_SINCE_CXX20
+_LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_STRING_INTERNAL_MEMORY_ACCESS
 basic_string<_CharT, _Traits, _Allocator>&
 basic_string<_CharT, _Traits, _Allocator>::operator=(const basic_string& __str)
 {
@@ -2493,7 +2617,12 @@ basic_string<_CharT, _Traits, _Allocator>::operator=(const basic_string& __str)
     __copy_assign_alloc(__str);
     if (!__is_long()) {
       if (!__str.__is_long()) {
+        size_type __old_size = __get_short_size();
+        if (__get_short_size() < __str.__get_short_size())
+          __annotate_increase(__str.__get_short_size() - __get_short_size());
         __r_.first() = __str.__r_.first();
+        if (__old_size > __get_short_size())
+          __annotate_shrink(__old_size);
       } else {
         return __assign_no_alias<true>(__str.data(), __str.size());
       }
@@ -2519,7 +2648,7 @@ basic_string<_CharT, _Traits, _Allocator>::__move_assign(basic_string& __str, fa
 }
 
 template <class _CharT, class _Traits, class _Allocator>
-inline _LIBCPP_CONSTEXPR_SINCE_CXX20
+inline _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_STRING_INTERNAL_MEMORY_ACCESS
 void
 basic_string<_CharT, _Traits, _Allocator>::__move_assign(basic_string& __str, true_type)
 #if _LIBCPP_STD_VER >= 17
@@ -2528,6 +2657,7 @@ basic_string<_CharT, _Traits, _Allocator>::__move_assign(basic_string& __str, tr
     _NOEXCEPT_(is_nothrow_move_assignable<allocator_type>::value)
 #endif
 {
+  __annotate_delete();
   if (__is_long()) {
     __alloc_traits::deallocate(__alloc(), __get_long_pointer(),
                                __get_long_cap());
@@ -2535,13 +2665,35 @@ basic_string<_CharT, _Traits, _Allocator>::__move_assign(basic_string& __str, tr
     if (!is_nothrow_move_assignable<allocator_type>::value) {
       __set_short_size(0);
       traits_type::assign(__get_short_pointer()[0], value_type());
+      __annotate_new(0);
     }
 #endif
   }
+  size_type __str_old_size = __str.size();
+  bool __str_was_short   = !__str.__is_long();
+
   __move_assign_alloc(__str);
   __r_.first() = __str.__r_.first();
   __str.__set_short_size(0);
   traits_type::assign(__str.__get_short_pointer()[0], value_type());
+
+  if (__str_was_short && this != &__str)
+    __str.__annotate_shrink(__str_old_size);
+  else
+    // ASan annotations: was long, so object memory is unpoisoned as new.
+    // Or is same as *this, and __annotate_delete() was called.
+    __str.__annotate_new(0);
+
+  // ASan annotations: Guard against `std::string s; s = std::move(s);`
+  // You can find more here: https://en.cppreference.com/w/cpp/utility/move
+  // Quote: "Unless otherwise specified, all standard library objects that have been moved
+  // from are placed in a "valid but unspecified state", meaning the object's class
+  // invariants hold (so functions without preconditions, such as the assignment operator,
+  // can be safely used on the object after it was moved from):"
+  // Quote: "v = std::move(v); // the value of v is unspecified"
+  if (!__is_long() && &__str != this)
+    // If it is long string, delete was never called on original __str's buffer.
+    __annotate_new(__get_short_size());
 }
 
 #endif
@@ -2587,6 +2739,7 @@ basic_string<_CharT, _Traits, _Allocator>::__assign_trivial(_Iterator __first, _
   _LIBCPP_ASSERT_INTERNAL(
       __string_is_trivial_iterator<_Iterator>::value, "The iterator type given to `__assign_trivial` must be trivial");
 
+  size_type __old_size = size();
   size_type __cap = capacity();
   if (__cap < __n) {
     // Unlike `append` functions, if the input range points into the string itself, there is no case that the input
@@ -2597,12 +2750,17 @@ basic_string<_CharT, _Traits, _Allocator>::__assign_trivial(_Iterator __first, _
     //    object itself stays valid even if reallocation happens.
     size_type __sz = size();
     __grow_by_without_replace(__cap, __n - __cap, __sz, 0, __sz);
+    __annotate_increase(__n);
     }
+    else if (__n > __old_size)
+        __annotate_increase(__n - __old_size);
     pointer __p = __get_pointer();
     for (; __first != __last; ++__p, (void) ++__first)
         traits_type::assign(*__p, *__first);
     traits_type::assign(*__p, value_type());
     __set_size(__n);
+    if (__n < __old_size)
+        __annotate_shrink(__old_size);
 }
 
 template <class _CharT, class _Traits, class _Allocator>
@@ -2663,6 +2821,7 @@ basic_string<_CharT, _Traits, _Allocator>::append(const value_type* __s, size_ty
     {
         if (__n)
         {
+            __annotate_increase(__n);
             value_type* __p = std::__to_address(__get_pointer());
             traits_type::copy(__p + __sz, __s, __n);
             __sz += __n;
@@ -2686,6 +2845,7 @@ basic_string<_CharT, _Traits, _Allocator>::append(size_type __n, value_type __c)
         size_type __sz = size();
         if (__cap - __sz < __n)
             __grow_by_without_replace(__cap, __sz + __n - __cap, __sz, __sz, 0);
+        __annotate_increase(__n);
         pointer __p = __get_pointer();
         traits_type::assign(std::__to_address(__p) + __sz, __n, __c);
         __sz += __n;
@@ -2705,6 +2865,7 @@ basic_string<_CharT, _Traits, _Allocator>::__append_default_init(size_type __n)
         size_type __sz = size();
         if (__cap - __sz < __n)
             __grow_by_without_replace(__cap, __sz + __n - __cap, __sz, __sz, 0);
+        __annotate_increase(__n);
         pointer __p = __get_pointer();
         __sz += __n;
         __set_size(__sz);
@@ -2733,8 +2894,10 @@ basic_string<_CharT, _Traits, _Allocator>::push_back(value_type __c)
     if (__sz == __cap)
     {
         __grow_by_without_replace(__cap, 1, __sz, __sz, 0);
+        __annotate_increase(1);
         __is_short = false; // the string is always long after __grow_by
-    }
+    } else
+        __annotate_increase(1);
     pointer __p = __get_pointer();
     if (__is_short)
     {
@@ -2766,6 +2929,7 @@ basic_string<_CharT, _Traits, _Allocator>::append(
         {
             if (__cap - __sz < __n)
               __grow_by_without_replace(__cap, __sz + __n - __cap, __sz, __sz, 0);
+            __annotate_increase(__n);
             pointer __p = __get_pointer() + __sz;
             for (; __first != __last; ++__p, (void) ++__first)
                 traits_type::assign(*__p, *__first);
@@ -2831,6 +2995,7 @@ basic_string<_CharT, _Traits, _Allocator>::insert(size_type __pos, const value_t
     {
         if (__n)
         {
+            __annotate_increase(__n);
             value_type* __p = std::__to_address(__get_pointer());
             size_type __n_move = __sz - __pos;
             if (__n_move != 0)
@@ -2864,6 +3029,7 @@ basic_string<_CharT, _Traits, _Allocator>::insert(size_type __pos, size_type __n
         value_type* __p;
         if (__cap - __sz >= __n)
         {
+            __annotate_increase(__n);
             __p = std::__to_address(__get_pointer());
             size_type __n_move = __sz - __pos;
             if (__n_move != 0)
@@ -2972,6 +3138,7 @@ basic_string<_CharT, _Traits, _Allocator>::insert(const_iterator __pos, value_ty
     }
     else
     {
+        __annotate_increase(1);
         __p = std::__to_address(__get_pointer());
         size_type __n_move = __sz - __ip;
         if (__n_move != 0)
@@ -3002,6 +3169,8 @@ basic_string<_CharT, _Traits, _Allocator>::replace(size_type __pos, size_type __
         value_type* __p = std::__to_address(__get_pointer());
         if (__n1 != __n2)
         {
+            if (__n2 > __n1)
+                __annotate_increase(__n2 - __n1);
             size_type __n_move = __sz - __pos - __n1;
             if (__n_move != 0)
             {
@@ -3046,20 +3215,18 @@ basic_string<_CharT, _Traits, _Allocator>::replace(size_type __pos, size_type __
     __n1 = std::min(__n1, __sz - __pos);
     size_type __cap = capacity();
     value_type* __p;
-    if (__cap - __sz + __n1 >= __n2)
-    {
-        __p = std::__to_address(__get_pointer());
-        if (__n1 != __n2)
-        {
-            size_type __n_move = __sz - __pos - __n1;
-            if (__n_move != 0)
-                traits_type::move(__p + __pos + __n2, __p + __pos + __n1, __n_move);
-        }
-    }
-    else
-    {
-        __grow_by_without_replace(__cap, __sz - __n1 + __n2 - __cap, __sz, __pos, __n1, __n2);
-        __p = std::__to_address(__get_long_pointer());
+    if (__cap - __sz + __n1 >= __n2) {
+      __p = std::__to_address(__get_pointer());
+      if (__n1 != __n2) {
+        if (__n2 > __n1)
+          __annotate_increase(__n2 - __n1);
+        size_type __n_move = __sz - __pos - __n1;
+        if (__n_move != 0)
+          traits_type::move(__p + __pos + __n2, __p + __pos + __n1, __n_move);
+      }
+    } else {
+      __grow_by_without_replace(__cap, __sz - __n1 + __n2 - __cap, __sz, __pos, __n1, __n2);
+      __p = std::__to_address(__get_long_pointer());
     }
     traits_type::assign(__p + __pos, __n2, __c);
     return __null_terminate_at(__p, __sz - (__n1 - __n2));
@@ -3187,6 +3354,7 @@ inline _LIBCPP_CONSTEXPR_SINCE_CXX20
 void
 basic_string<_CharT, _Traits, _Allocator>::clear() _NOEXCEPT
 {
+    size_type __old_size = size();
     if (__is_long())
     {
         traits_type::assign(*__get_long_pointer(), value_type());
@@ -3197,6 +3365,7 @@ basic_string<_CharT, _Traits, _Allocator>::clear() _NOEXCEPT
         traits_type::assign(*__get_short_pointer(), value_type());
         __set_short_size(0);
     }
+    __annotate_shrink(__old_size);
 }
 
 template <class _CharT, class _Traits, class _Allocator>
@@ -3259,6 +3428,7 @@ inline _LIBCPP_CONSTEXPR_SINCE_CXX20
 void
 basic_string<_CharT, _Traits, _Allocator>::__shrink_or_extend(size_type __target_capacity)
 {
+    __annotate_delete();
     size_type __cap = capacity();
     size_type __sz = size();
 
@@ -3315,6 +3485,7 @@ basic_string<_CharT, _Traits, _Allocator>::__shrink_or_extend(size_type __target
     }
     else
         __set_short_size(__sz);
+    __annotate_new(__sz);
 }
 
 template <class _CharT, class _Traits, class _Allocator>
@@ -3365,8 +3536,16 @@ basic_string<_CharT, _Traits, _Allocator>::swap(basic_string& __str)
         __alloc_traits::propagate_on_container_swap::value ||
         __alloc_traits::is_always_equal::value ||
         __alloc() == __str.__alloc(), "swapping non-equal allocators");
+    if (!__is_long())
+      __annotate_delete();
+    if (this != &__str && !__str.__is_long())
+      __str.__annotate_delete();
     std::swap(__r_.first(), __str.__r_.first());
     std::__swap_allocator(__alloc(), __str.__alloc());
+    if (!__is_long())
+      __annotate_new(__get_short_size());
+    if (this != &__str && !__str.__is_long())
+      __str.__annotate_new(__str.__get_short_size());
 }
 
 // find
@@ -3854,12 +4033,12 @@ inline _LIBCPP_CONSTEXPR_SINCE_CXX20
 void
 basic_string<_CharT, _Traits, _Allocator>::__clear_and_shrink() _NOEXCEPT
 {
-    clear();
-    if(__is_long())
-    {
-        __alloc_traits::deallocate(__alloc(), __get_long_pointer(), capacity() + 1);
-        __r_.first() = __rep();
-    }
+  clear();
+  if (__is_long()) {
+    __annotate_delete();
+    __alloc_traits::deallocate(__alloc(), __get_long_pointer(), capacity() + 1);
+    __r_.first() = __rep();
+  }
 }
 
 // operator==
diff --git a/libcxx/include/vector b/libcxx/include/vector
index fd2d5e11f0ea4e7557347216a12895a6bda6bfaa..d010a1f6ec9f91f41a343778950a064c0bd30b96 100644
--- a/libcxx/include/vector
+++ b/libcxx/include/vector
@@ -852,11 +852,15 @@ private:
     // the documentation for __sanitizer_annotate_contiguous_container.
 
     _LIBCPP_CONSTEXPR_SINCE_CXX20 _LIBCPP_HIDE_FROM_ABI
-    void __annotate_contiguous_container([[__maybe_unused__]] const void *__beg,
-                                         [[__maybe_unused__]] const void *__end,
-                                         [[__maybe_unused__]] const void *__old_mid,
-                                         [[__maybe_unused__]] const void *__new_mid) const
-    {
+    void __annotate_contiguous_container(const void *__beg,
+                                         const void *__end,
+                                         const void *__old_mid,
+                                         const void *__new_mid) const
+    {
+      (void)__beg;
+      (void)__end;
+      (void)__old_mid;
+      (void)__new_mid;
 #ifndef _LIBCPP_HAS_NO_ASAN
       if (!__libcpp_is_constant_evaluated() && __beg != nullptr && __asan_annotate_container_with_allocator<_Allocator>::value)
         __sanitizer_annotate_contiguous_container(__beg, __end, __old_mid, __new_mid);
diff --git a/libcxx/modules/std/string.inc b/libcxx/modules/std/string.inc
index 8366690fd9d37d078863cee7f80b18c54b56fb1a..c83ee7643f87e9a8acc6b6a1af5030dcf9919064 100644
--- a/libcxx/modules/std/string.inc
+++ b/libcxx/modules/std/string.inc
@@ -67,15 +67,10 @@ export namespace std {
   // [basic.string.hash], hash support
   using std::hash;
 
-  // TODO MODULES is this a bug?
-#if _LIBCPP_STD_VER >= 23
-  using std::operator""s;
-#else
   inline namespace literals {
     inline namespace string_literals {
       // [basic.string.literals], suffix for basic_string literals
       using std::literals::string_literals::operator""s;
     } // namespace string_literals
-  }   // namespace literals
-#endif
+  } // namespace literals
 } // namespace std
diff --git a/libcxx/test/std/localization/locale.categories/category.numeric/locale.num.get/user_defined_char_type.pass.cpp b/libcxx/test/std/localization/locale.categories/category.numeric/locale.num.get/user_defined_char_type.pass.cpp
index d7b4b816d975b9fc37dba095984da1c369642b7c..9a4a2f0d5657e1d8bc980d5f88bf24d79a3bb96c 100644
--- a/libcxx/test/std/localization/locale.categories/category.numeric/locale.num.get/user_defined_char_type.pass.cpp
+++ b/libcxx/test/std/localization/locale.categories/category.numeric/locale.num.get/user_defined_char_type.pass.cpp
@@ -16,8 +16,6 @@
 #include <locale>
 #include <string>
 
-#include "test_macros.h"
-
 struct Char {
   Char() = default;
   Char(char c) : underlying_(c) {}
@@ -73,15 +71,53 @@ struct char_traits<Char> {
   static int_type eof() { return char_traits<char>::eof(); }
 };
 
+// This ctype specialization treats all characters as spaces
 template <>
-class ctype<Char> : public locale::facet {
+class ctype<Char> : public locale::facet, public ctype_base {
 public:
+  using char_type = Char;
   static locale::id id;
-  Char toupper(Char c) const { return Char(std::toupper(c.underlying_)); }
-  const char* widen(const char* first, const char* last, Char* dst) const {
-    for (; first != last;)
-      *dst++ = Char(*first++);
-    return last;
+  explicit ctype(std::size_t refs = 0) : locale::facet(refs) {}
+
+  bool is(mask m, char_type) const { return m & ctype_base::space; }
+  const char_type* is(const char_type* low, const char_type* high, mask* vec) const {
+    for (; low != high; ++low)
+      *vec++ = ctype_base::space;
+    return high;
+  }
+
+  const char_type* scan_is(mask m, const char_type* beg, const char_type* end) const {
+    for (; beg != end; ++beg)
+      if (this->is(m, *beg))
+        return beg;
+    return end;
+  }
+
+  const char_type* scan_not(mask m, const char_type* beg, const char_type* end) const {
+    for (; beg != end; ++beg)
+      if (!this->is(m, *beg))
+        return beg;
+    return end;
+  }
+
+  char_type toupper(char_type c) const { return c; }
+  const char_type* toupper(char_type*, const char_type* end) const { return end; }
+
+  char_type tolower(char_type c) const { return c; }
+  const char_type* tolower(char_type*, const char_type* end) const { return end; }
+
+  char_type widen(char c) const { return char_type(c); }
+  const char* widen(const char* beg, const char* end, char_type* dst) const {
+    for (; beg != end; ++beg, ++dst)
+      *dst = char_type(*beg);
+    return end;
+  }
+
+  char narrow(char_type c, char /*dflt*/) const { return c.underlying_; }
+  const char_type* narrow(const char_type* beg, const char_type* end, char /*dflt*/, char* dst) const {
+    for (; beg != end; ++beg, ++dst)
+      *dst = beg->underlying_;
+    return end;
   }
 };
 
diff --git a/libcxx/test/std/ranges/range.adaptors/range.lazy.split/adaptor.pass.cpp b/libcxx/test/std/ranges/range.adaptors/range.lazy.split/adaptor.pass.cpp
index da4bd9fbbe1794f9df9c3b30be4591a14b102438..6bfa0ab487ba1ba45b77f768edb50c4dd3aed8b6 100644
--- a/libcxx/test/std/ranges/range.adaptors/range.lazy.split/adaptor.pass.cpp
+++ b/libcxx/test/std/ranges/range.adaptors/range.lazy.split/adaptor.pass.cpp
@@ -40,10 +40,16 @@ static_assert(!std::is_invocable_v<decltype(std::views::lazy_split), SomeView, N
 static_assert(!std::is_invocable_v<decltype(std::views::lazy_split), NotAView, SomeView>);
 static_assert( std::is_invocable_v<decltype(std::views::lazy_split), SomeView, SomeView>);
 
-static_assert( CanBePiped<SomeView&,    decltype(std::views::lazy_split)>);
-static_assert( CanBePiped<char(&)[10],  decltype(std::views::lazy_split)>);
-static_assert(!CanBePiped<char(&&)[10], decltype(std::views::lazy_split)>);
-static_assert(!CanBePiped<NotAView,     decltype(std::views::lazy_split)>);
+// Regression test for #75002, views::lazy_split shouldn't be a range adaptor closure
+static_assert(!CanBePiped<SomeView&, decltype(std::views::lazy_split)>);
+static_assert(!CanBePiped<char (&)[10], decltype(std::views::lazy_split)>);
+static_assert(!CanBePiped<char (&&)[10], decltype(std::views::lazy_split)>);
+static_assert(!CanBePiped<NotAView, decltype(std::views::lazy_split)>);
+
+static_assert(CanBePiped<SomeView&, decltype(std::views::lazy_split('x'))>);
+static_assert(CanBePiped<char (&)[10], decltype(std::views::lazy_split('x'))>);
+static_assert(!CanBePiped<char (&&)[10], decltype(std::views::lazy_split('x'))>);
+static_assert(!CanBePiped<NotAView, decltype(std::views::lazy_split('x'))>);
 
 static_assert(std::same_as<decltype(std::views::lazy_split), decltype(std::ranges::views::lazy_split)>);
 
diff --git a/libcxx/test/std/ranges/range.adaptors/range.split/adaptor.pass.cpp b/libcxx/test/std/ranges/range.adaptors/range.split/adaptor.pass.cpp
index cd12011daeab5d5997053a0eb560641e0bab901b..85d13ac5c29dfbc9aab8ec6fabb2633ddf255e23 100644
--- a/libcxx/test/std/ranges/range.adaptors/range.split/adaptor.pass.cpp
+++ b/libcxx/test/std/ranges/range.adaptors/range.split/adaptor.pass.cpp
@@ -39,10 +39,16 @@ static_assert(!std::is_invocable_v<decltype(std::views::split), SomeView, NotAVi
 static_assert(!std::is_invocable_v<decltype(std::views::split), NotAView, SomeView>);
 static_assert( std::is_invocable_v<decltype(std::views::split), SomeView, SomeView>);
 
-static_assert( CanBePiped<SomeView&,    decltype(std::views::split)>);
-static_assert( CanBePiped<char(&)[10],  decltype(std::views::split)>);
-static_assert(!CanBePiped<char(&&)[10], decltype(std::views::split)>);
-static_assert(!CanBePiped<NotAView,     decltype(std::views::split)>);
+// Regression test for #75002, views::split shouldn't be a range adaptor closure
+static_assert(!CanBePiped<SomeView&, decltype(std::views::split)>);
+static_assert(!CanBePiped<char (&)[10], decltype(std::views::split)>);
+static_assert(!CanBePiped<char (&&)[10], decltype(std::views::split)>);
+static_assert(!CanBePiped<NotAView, decltype(std::views::split)>);
+
+static_assert(CanBePiped<SomeView&, decltype(std::views::split('x'))>);
+static_assert(CanBePiped<char (&)[10], decltype(std::views::split('x'))>);
+static_assert(!CanBePiped<char (&&)[10], decltype(std::views::split('x'))>);
+static_assert(!CanBePiped<NotAView, decltype(std::views::split('x'))>);
 
 static_assert(std::same_as<decltype(std::views::split), decltype(std::ranges::views::split)>);
 
diff --git a/libcxx/test/std/ranges/range.adaptors/range.take/sentinel/base.pass.cpp b/libcxx/test/std/ranges/range.adaptors/range.take/range.take.sentinel/base.pass.cpp
similarity index 83%
rename from libcxx/test/std/ranges/range.adaptors/range.take/sentinel/base.pass.cpp
rename to libcxx/test/std/ranges/range.adaptors/range.take/range.take.sentinel/base.pass.cpp
index c949eb7cc08469aa378bedf73a4797cf11d1e993..15b2b5476e86dd742fc750ee4eb29c49745eb733 100644
--- a/libcxx/test/std/ranges/range.adaptors/range.take/sentinel/base.pass.cpp
+++ b/libcxx/test/std/ranges/range.adaptors/range.take/range.take.sentinel/base.pass.cpp
@@ -8,10 +8,7 @@
 
 // UNSUPPORTED: c++03, c++11, c++14, c++17
 
-// sentinel() = default;
-// constexpr explicit sentinel(sentinel_t<Base> end);
-// constexpr sentinel(sentinel<!Const> s)
-//   requires Const && convertible_to<sentinel_t<V>, sentinel_t<Base>>;
+// constexpr sentinel_t<Base> base() const;
 
 #include <ranges>
 #include <cassert>
diff --git a/libcxx/test/std/ranges/range.adaptors/range.take/sentinel/ctor.pass.cpp b/libcxx/test/std/ranges/range.adaptors/range.take/range.take.sentinel/ctor.pass.cpp
similarity index 77%
rename from libcxx/test/std/ranges/range.adaptors/range.take/sentinel/ctor.pass.cpp
rename to libcxx/test/std/ranges/range.adaptors/range.take/range.take.sentinel/ctor.pass.cpp
index 9d1bdaa82d95dc1d7fbc9f377ccf1d3ff394ddbe..8928371939c87251e5a3eeddd1a9323a98678e90 100644
--- a/libcxx/test/std/ranges/range.adaptors/range.take/sentinel/ctor.pass.cpp
+++ b/libcxx/test/std/ranges/range.adaptors/range.take/range.take.sentinel/ctor.pass.cpp
@@ -34,14 +34,14 @@ constexpr bool test() {
 
   {
     // Test the conversion from "sentinel" to "sentinel-to-const".
-    using TakeView = std::ranges::take_view<MoveOnlyView>;
-    using Sentinel = std::ranges::sentinel_t<TakeView>;
+    using TakeView      = std::ranges::take_view<MoveOnlyView>;
+    using Sentinel      = std::ranges::sentinel_t<TakeView>;
     using ConstSentinel = std::ranges::sentinel_t<const TakeView>;
     static_assert(std::is_convertible_v<Sentinel, ConstSentinel>);
-    TakeView tv = TakeView(MoveOnlyView(buffer), 4);
-    Sentinel s = tv.end();
+    TakeView tv      = TakeView(MoveOnlyView(buffer), 4);
+    Sentinel s       = tv.end();
     ConstSentinel cs = s;
-    cs = s;  // test assignment also
+    cs               = s; // test assignment also
     assert(tv.begin() + 4 == s);
     assert(tv.begin() + 4 == cs);
     assert(std::as_const(tv).begin() + 4 == s);
@@ -50,12 +50,12 @@ constexpr bool test() {
 
   {
     // Test the constructor from "base-sentinel" to "sentinel".
-    using TakeView = std::ranges::take_view<MoveOnlyView>;
-    using Sentinel = std::ranges::sentinel_t<TakeView>;
+    using TakeView             = std::ranges::take_view<MoveOnlyView>;
+    using Sentinel             = std::ranges::sentinel_t<TakeView>;
     sentinel_wrapper<int*> sw1 = MoveOnlyView(buffer).end();
-    static_assert( std::is_constructible_v<Sentinel, sentinel_wrapper<int*>>);
+    static_assert(std::is_constructible_v<Sentinel, sentinel_wrapper<int*>>);
     static_assert(!std::is_convertible_v<sentinel_wrapper<int*>, Sentinel>);
-    auto s = Sentinel(sw1);
+    auto s                                        = Sentinel(sw1);
     std::same_as<sentinel_wrapper<int*>> auto sw2 = s.base();
     assert(base(sw2) == base(sw1));
   }
diff --git a/libcxx/test/std/ranges/range.adaptors/range.take/range.take.sentinel/eq.pass.cpp b/libcxx/test/std/ranges/range.adaptors/range.take/range.take.sentinel/eq.pass.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..e6f433e30f60db2128abea66ea75f00564c2b85c
--- /dev/null
+++ b/libcxx/test/std/ranges/range.adaptors/range.take/range.take.sentinel/eq.pass.cpp
@@ -0,0 +1,150 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// UNSUPPORTED: c++03, c++11, c++14, c++17
+
+// friend constexpr bool operator==(const CI<Const>& y, const sentinel& x);
+// template<bool OtherConst = !Const>
+//   requires sentinel_for<sentinel_t<Base>, iterator_t<maybe-const<OtherConst, V>>>
+// friend constexpr bool operator==(const CI<OtherConst>& y, const sentinel& x);
+
+#include <cassert>
+#include <cstddef>
+#include <ranges>
+#include <type_traits>
+#include <utility>
+
+#include "test_comparisons.h"
+#include "test_iterators.h"
+
+template <bool Const>
+using MaybeConstIterator = cpp20_input_iterator<std::conditional_t<Const, const int*, int*>>;
+
+template <bool Const>
+class CrossConstComparableSentinel {
+  using Base = std::conditional_t<Const, const int*, int*>;
+  Base base_;
+
+public:
+  CrossConstComparableSentinel() = default;
+  constexpr explicit CrossConstComparableSentinel(Base base) : base_(base) {}
+
+  friend constexpr bool operator==(const MaybeConstIterator<Const>& it, const CrossConstComparableSentinel& se) {
+    return base(it) == se.base_;
+  }
+
+  friend constexpr bool operator==(const MaybeConstIterator<!Const>& it, const CrossConstComparableSentinel& se) {
+    return base(it) == se.base_;
+  }
+};
+
+static_assert(std::sentinel_for<CrossConstComparableSentinel<true>, MaybeConstIterator<false>>);
+static_assert(std::sentinel_for<CrossConstComparableSentinel<true>, MaybeConstIterator<true>>);
+static_assert(std::sentinel_for<CrossConstComparableSentinel<false>, MaybeConstIterator<false>>);
+static_assert(std::sentinel_for<CrossConstComparableSentinel<false>, MaybeConstIterator<true>>);
+
+struct CrossConstComparableView : std::ranges::view_base {
+  template <std::size_t N>
+  constexpr explicit CrossConstComparableView(int (&arr)[N]) : b_(arr), e_(arr + N) {}
+
+  constexpr MaybeConstIterator<false> begin() { return MaybeConstIterator<false>{b_}; }
+  constexpr CrossConstComparableSentinel<false> end() { return CrossConstComparableSentinel<false>{e_}; }
+
+  constexpr MaybeConstIterator<true> begin() const { return MaybeConstIterator<true>{b_}; }
+  constexpr CrossConstComparableSentinel<true> end() const { return CrossConstComparableSentinel<true>{e_}; }
+
+private:
+  int* b_;
+  int* e_;
+};
+
+static_assert(std::ranges::range<CrossConstComparableView>);
+static_assert(std::ranges::range<const CrossConstComparableView>);
+
+struct NonCrossConstComparableView : std::ranges::view_base {
+  int* begin();
+  sentinel_wrapper<int*> end();
+
+  long* begin() const;
+  sentinel_wrapper<long*> end() const;
+};
+
+static_assert(std::ranges::range<NonCrossConstComparableView>);
+static_assert(std::ranges::range<const NonCrossConstComparableView>);
+
+template <class T, class U>
+concept weakly_equality_comparable_with = requires(const T& t, const U& u) {
+  t == u;
+  t != u;
+  u == t;
+  u != t;
+};
+
+constexpr bool test() {
+  int buffer[8]                      = {1, 2, 3, 4, 5, 6, 7, 8};
+  using CrossConstComparableTakeView = std::ranges::take_view<CrossConstComparableView>;
+
+  {   // Compare CI<Const> with sentinel<Const>
+    { // Const == true
+      AssertEqualityReturnBool<std::ranges::iterator_t<const CrossConstComparableTakeView>,
+                               std::ranges::sentinel_t<const CrossConstComparableTakeView>>();
+      const CrossConstComparableTakeView tv(CrossConstComparableView{buffer}, 4);
+      assert(testEquality(std::ranges::next(tv.begin(), 4), tv.end(), true));
+      assert(testEquality(tv.begin(), tv.end(), false));
+    }
+
+    { // Const == false
+      AssertEqualityReturnBool<std::ranges::iterator_t<CrossConstComparableTakeView>,
+                               std::ranges::sentinel_t<CrossConstComparableTakeView>>();
+      CrossConstComparableTakeView tv(CrossConstComparableView{buffer}, 4);
+      assert(testEquality(std::ranges::next(tv.begin(), 4), tv.end(), true));
+      assert(testEquality(std::ranges::next(tv.begin(), 1), tv.end(), false));
+    }
+  }
+
+  {   // Compare CI<Const> with sentinel<!Const>
+    { // Const == true
+      AssertEqualityReturnBool<std::ranges::iterator_t<const CrossConstComparableTakeView>,
+                               std::ranges::sentinel_t<CrossConstComparableTakeView>>();
+      CrossConstComparableTakeView tv(CrossConstComparableView{buffer}, 4);
+      assert(testEquality(std::ranges::next(std::as_const(tv).begin(), 4), tv.end(), true));
+      assert(testEquality(std::ranges::next(std::as_const(tv).begin(), 2), tv.end(), false));
+    }
+
+    { // Const == false
+      AssertEqualityReturnBool<std::ranges::iterator_t<CrossConstComparableTakeView>,
+                               std::ranges::sentinel_t<const CrossConstComparableTakeView>>();
+      CrossConstComparableTakeView tv(CrossConstComparableView{buffer}, 4);
+      assert(testEquality(std::ranges::next(tv.begin(), 4), std::as_const(tv).end(), true));
+      assert(testEquality(std::ranges::next(tv.begin(), 3), std::as_const(tv).end(), false));
+    }
+  }
+
+  { // Check invalid comparisons between CI<Const> and sentinel<!Const>
+    using TakeView = std::ranges::take_view<NonCrossConstComparableView>;
+    static_assert(
+        !weakly_equality_comparable_with<std::ranges::iterator_t<const TakeView>, std::ranges::sentinel_t<TakeView>>);
+    static_assert(
+        !weakly_equality_comparable_with<std::ranges::iterator_t<TakeView>, std::ranges::sentinel_t<const TakeView>>);
+
+    // Those should be valid
+    static_assert(
+        weakly_equality_comparable_with<std::ranges::iterator_t<TakeView>, std::ranges::sentinel_t<TakeView>>);
+    static_assert(weakly_equality_comparable_with<std::ranges::iterator_t<const TakeView>,
+                                                  std::ranges::sentinel_t<const TakeView>>);
+  }
+
+  return true;
+}
+
+int main(int, char**) {
+  test();
+  static_assert(test());
+
+  return 0;
+}
diff --git a/libcxx/test/std/ranges/range.adaptors/range.take/sentinel/eq.pass.cpp b/libcxx/test/std/ranges/range.adaptors/range.take/sentinel/eq.pass.cpp
deleted file mode 100644
index eb265a7e034817fc436062089c497fca9f878610..0000000000000000000000000000000000000000
--- a/libcxx/test/std/ranges/range.adaptors/range.take/sentinel/eq.pass.cpp
+++ /dev/null
@@ -1,55 +0,0 @@
-//===----------------------------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-// UNSUPPORTED: c++03, c++11, c++14, c++17
-
-// sentinel() = default;
-// constexpr explicit sentinel(sentinel_t<Base> end);
-// constexpr sentinel(sentinel<!Const> s)
-//   requires Const && convertible_to<sentinel_t<V>, sentinel_t<Base>>;
-
-#include <ranges>
-#include <cassert>
-
-#include "test_macros.h"
-#include "test_iterators.h"
-#include "../types.h"
-
-constexpr bool test() {
-  int buffer[8] = {1, 2, 3, 4, 5, 6, 7, 8};
-
-  {
-    {
-      const std::ranges::take_view<MoveOnlyView> tv(MoveOnlyView{buffer}, 4);
-      assert(tv.end() == std::ranges::next(tv.begin(), 4));
-      assert(std::ranges::next(tv.begin(), 4) == tv.end());
-    }
-
-    {
-      std::ranges::take_view<MoveOnlyView> tv(MoveOnlyView{buffer}, 4);
-      assert(tv.end() == std::ranges::next(tv.begin(), 4));
-      assert(std::ranges::next(tv.begin(), 4) == tv.end());
-    }
-  }
-
-  {
-    std::ranges::take_view<MoveOnlyView> tvNonConst(MoveOnlyView{buffer}, 4);
-    const std::ranges::take_view<MoveOnlyView> tvConst(MoveOnlyView{buffer}, 4);
-    assert(tvNonConst.end() == std::ranges::next(tvConst.begin(), 4));
-    assert(std::ranges::next(tvConst.begin(), 4) == tvNonConst.end());
-  }
-
-  return true;
-}
-
-int main(int, char**) {
-  test();
-  static_assert(test());
-
-  return 0;
-}
diff --git a/libcxx/test/std/strings/basic.string/string.capacity/capacity.pass.cpp b/libcxx/test/std/strings/basic.string/string.capacity/capacity.pass.cpp
index e1d20662e41de89083c82aa4c9621e34feb56f7c..61867cfb087b9392fe42497dfd8a6002c768a46d 100644
--- a/libcxx/test/std/strings/basic.string/string.capacity/capacity.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.capacity/capacity.pass.cpp
@@ -15,6 +15,7 @@
 
 #include "test_allocator.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 #include "test_macros.h"
 
@@ -28,6 +29,7 @@ TEST_CONSTEXPR_CXX20 void test_invariant(S s, test_allocator_statistics& alloc_s
     while (s.size() < s.capacity())
       s.push_back(typename S::value_type());
     assert(s.size() == s.capacity());
+    LIBCPP_ASSERT(is_string_asan_correct(s));
   }
 #ifndef TEST_HAS_NO_EXCEPTIONS
   catch (...) {
@@ -43,10 +45,12 @@ TEST_CONSTEXPR_CXX20 void test_string(const Alloc& a) {
   {
     S const s((Alloc(a)));
     assert(s.capacity() >= 0);
+    LIBCPP_ASSERT(is_string_asan_correct(s));
   }
   {
     S const s(3, 'x', Alloc(a));
     assert(s.capacity() >= 3);
+    LIBCPP_ASSERT(is_string_asan_correct(s));
   }
 #if TEST_STD_VER >= 11
   // Check that we perform SSO
@@ -54,6 +58,7 @@ TEST_CONSTEXPR_CXX20 void test_string(const Alloc& a) {
     S const s;
     assert(s.capacity() > 0);
     ASSERT_NOEXCEPT(s.capacity());
+    LIBCPP_ASSERT(is_string_asan_correct(s));
   }
 #endif
 }
@@ -63,18 +68,22 @@ TEST_CONSTEXPR_CXX20 bool test() {
   test_string(test_allocator<char>());
   test_string(test_allocator<char>(3));
   test_string(min_allocator<char>());
+  test_string(safe_allocator<char>());
 
   {
     test_allocator_statistics alloc_stats;
     typedef std::basic_string<char, std::char_traits<char>, test_allocator<char> > S;
     S s((test_allocator<char>(&alloc_stats)));
     test_invariant(s, alloc_stats);
+    LIBCPP_ASSERT(is_string_asan_correct(s));
     s.assign(10, 'a');
     s.erase(5);
     test_invariant(s, alloc_stats);
+    LIBCPP_ASSERT(is_string_asan_correct(s));
     s.assign(100, 'a');
     s.erase(50);
     test_invariant(s, alloc_stats);
+    LIBCPP_ASSERT(is_string_asan_correct(s));
   }
 
   return true;
diff --git a/libcxx/test/std/strings/basic.string/string.capacity/clear.pass.cpp b/libcxx/test/std/strings/basic.string/string.capacity/clear.pass.cpp
index 3a308de9b7569a3fa230081eab0b5c21b68bc1cc..643ea4a3bdad486d3205848b02d18c28026a640b 100644
--- a/libcxx/test/std/strings/basic.string/string.capacity/clear.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.capacity/clear.pass.cpp
@@ -15,31 +15,39 @@
 
 #include "test_macros.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class S>
 TEST_CONSTEXPR_CXX20 void test(S s) {
   s.clear();
   assert(s.size() == 0);
+  LIBCPP_ASSERT(is_string_asan_correct(s));
 }
 
 template <class S>
 TEST_CONSTEXPR_CXX20 void test_string() {
   S s;
   test(s);
+  LIBCPP_ASSERT(is_string_asan_correct(s));
 
   s.assign(10, 'a');
   s.erase(5);
+  LIBCPP_ASSERT(is_string_asan_correct(s));
   test(s);
+  LIBCPP_ASSERT(is_string_asan_correct(s));
 
   s.assign(100, 'a');
   s.erase(50);
+  LIBCPP_ASSERT(is_string_asan_correct(s));
   test(s);
+  LIBCPP_ASSERT(is_string_asan_correct(s));
 }
 
 TEST_CONSTEXPR_CXX20 bool test() {
   test_string<std::string>();
 #if TEST_STD_VER >= 11
   test_string<std::basic_string<char, std::char_traits<char>, min_allocator<char>>>();
+  test_string<std::basic_string<char, std::char_traits<char>, safe_allocator<char>>>();
 #endif
 
   return true;
diff --git a/libcxx/test/std/strings/basic.string/string.capacity/reserve.pass.cpp b/libcxx/test/std/strings/basic.string/string.capacity/reserve.pass.cpp
index b740901be1c4d5b59c03ffb05b2ac0cb1b0c4ce2..43414da3794a5f8e34114eb3be05d9c35abe888a 100644
--- a/libcxx/test/std/strings/basic.string/string.capacity/reserve.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.capacity/reserve.pass.cpp
@@ -18,6 +18,7 @@
 
 #include "test_macros.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class S>
 void test(typename S::size_type min_cap, typename S::size_type erased_index) {
@@ -33,6 +34,7 @@ void test(typename S::size_type min_cap, typename S::size_type erased_index) {
   assert(s == s0);
   assert(s.capacity() <= old_cap);
   assert(s.capacity() >= s.size());
+  LIBCPP_ASSERT(is_string_asan_correct(s));
 }
 
 template <class S>
@@ -47,6 +49,7 @@ bool test() {
   test_string<std::string>();
 #if TEST_STD_VER >= 11
   test_string<std::basic_string<char, std::char_traits<char>, min_allocator<char>>>();
+  test_string<std::basic_string<char, std::char_traits<char>, safe_allocator<char>>>();
 #endif
 
   return true;
diff --git a/libcxx/test/std/strings/basic.string/string.capacity/reserve_size.asan.pass.cpp b/libcxx/test/std/strings/basic.string/string.capacity/reserve_size.asan.pass.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..d35a5bcefc46aece41645cacbd4bb6b6555c9eba
--- /dev/null
+++ b/libcxx/test/std/strings/basic.string/string.capacity/reserve_size.asan.pass.cpp
@@ -0,0 +1,63 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// <string>
+
+// This test verifies that the ASan annotations for basic_string objects remain accurate
+// after invoking basic_string::reserve(size_type __requested_capacity).
+// Different types are used to confirm that ASan works correctly with types of different sizes.
+#include <string>
+#include <cassert>
+
+#include "test_macros.h"
+#include "asan_testing.h"
+
+template <class S>
+void test() {
+  S short_s1(3, 'a'), long_s1(100, 'c');
+  short_s1.reserve(0x1337);
+  long_s1.reserve(0x1337);
+
+  LIBCPP_ASSERT(is_string_asan_correct(short_s1));
+  LIBCPP_ASSERT(is_string_asan_correct(long_s1));
+
+  short_s1.clear();
+  long_s1.clear();
+
+  LIBCPP_ASSERT(is_string_asan_correct(short_s1));
+  LIBCPP_ASSERT(is_string_asan_correct(long_s1));
+
+  short_s1.reserve(0x1);
+  long_s1.reserve(0x1);
+
+  LIBCPP_ASSERT(is_string_asan_correct(short_s1));
+  LIBCPP_ASSERT(is_string_asan_correct(long_s1));
+
+  S short_s2(3, 'a'), long_s2(100, 'c');
+  short_s2.reserve(0x1);
+  long_s2.reserve(0x1);
+
+  LIBCPP_ASSERT(is_string_asan_correct(short_s2));
+  LIBCPP_ASSERT(is_string_asan_correct(long_s2));
+}
+
+int main(int, char**) {
+  test<std::string>();
+#ifndef TEST_HAS_NO_WIDE_CHARACTERS
+  test<std::wstring>();
+#endif
+#if TEST_STD_VER >= 11
+  test<std::u16string>();
+  test<std::u32string>();
+#endif
+#if TEST_STD_VER >= 20
+  test<std::u8string>();
+#endif
+
+  return 0;
+}
diff --git a/libcxx/test/std/strings/basic.string/string.capacity/reserve_size.pass.cpp b/libcxx/test/std/strings/basic.string/string.capacity/reserve_size.pass.cpp
index dfb3b270f750ec708a1f08f7e25b18ed65ad603b..30c171680a23c4ea1bf0f6ffd30d118e0b3a1ffd 100644
--- a/libcxx/test/std/strings/basic.string/string.capacity/reserve_size.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.capacity/reserve_size.pass.cpp
@@ -20,6 +20,7 @@
 
 #include "test_macros.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class S>
 TEST_CONSTEXPR_CXX20 void
@@ -28,6 +29,7 @@ test(typename S::size_type min_cap, typename S::size_type erased_index, typename
   s.erase(erased_index);
   assert(s.size() == erased_index);
   assert(s.capacity() >= min_cap); // Check that we really have at least this capacity.
+  LIBCPP_ASSERT(is_string_asan_correct(s));
 
 #if TEST_STD_VER > 17
   typename S::size_type old_cap = s.capacity();
@@ -39,6 +41,7 @@ test(typename S::size_type min_cap, typename S::size_type erased_index, typename
     assert(s == s0);
     assert(s.capacity() >= res_arg);
     assert(s.capacity() >= s.size());
+    LIBCPP_ASSERT(is_string_asan_correct(s));
 #if TEST_STD_VER > 17
     assert(s.capacity() >= old_cap); // reserve never shrinks as of P0966 (C++20)
 #endif
diff --git a/libcxx/test/std/strings/basic.string/string.capacity/resize_and_overwrite.pass.cpp b/libcxx/test/std/strings/basic.string/string.capacity/resize_and_overwrite.pass.cpp
index bbe6551a0ff1181dc1e99b8b09d476607662f952..edc8b67808b8566d733c6f78b617244c99a9b26d 100644
--- a/libcxx/test/std/strings/basic.string/string.capacity/resize_and_overwrite.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.capacity/resize_and_overwrite.pass.cpp
@@ -19,6 +19,7 @@
 
 #include "make_string.h"
 #include "test_macros.h"
+#include "asan_testing.h"
 
 template <class S>
 constexpr void test_appending(std::size_t k, size_t N, size_t new_capacity) {
@@ -37,6 +38,7 @@ constexpr void test_appending(std::size_t k, size_t N, size_t new_capacity) {
   const S expected = S(k, 'a') + S(N - k, 'b');
   assert(s == expected);
   assert(s.c_str()[N] == '\0');
+  LIBCPP_ASSERT(is_string_asan_correct(s));
 }
 
 template <class S>
@@ -55,6 +57,7 @@ constexpr void test_truncating(std::size_t o, size_t N) {
   const S expected = S(N - 1, 'a') + S(1, 'b');
   assert(s == expected);
   assert(s.c_str()[N] == '\0');
+  LIBCPP_ASSERT(is_string_asan_correct(s));
 }
 
 template <class String>
@@ -76,11 +79,14 @@ constexpr bool test() {
 void test_value_categories() {
   std::string s;
   s.resize_and_overwrite(10, [](char*&&, std::size_t&&) { return 0; });
+  LIBCPP_ASSERT(is_string_asan_correct(s));
   s.resize_and_overwrite(10, [](char* const&, const std::size_t&) { return 0; });
+  LIBCPP_ASSERT(is_string_asan_correct(s));
   struct RefQualified {
     int operator()(char*, std::size_t) && { return 0; }
   };
   s.resize_and_overwrite(10, RefQualified{});
+  LIBCPP_ASSERT(is_string_asan_correct(s));
 }
 
 int main(int, char**) {
diff --git a/libcxx/test/std/strings/basic.string/string.capacity/resize_size.pass.cpp b/libcxx/test/std/strings/basic.string/string.capacity/resize_size.pass.cpp
index 487b12d9df87f338790464511ce8a20bdf8f4657..7cf4b7ca3b6efd65bfd39ef39b2de240b97a987b 100644
--- a/libcxx/test/std/strings/basic.string/string.capacity/resize_size.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.capacity/resize_size.pass.cpp
@@ -16,6 +16,7 @@
 
 #include "test_macros.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class S>
 TEST_CONSTEXPR_CXX20 void test(S s, typename S::size_type n, S expected) {
@@ -23,6 +24,7 @@ TEST_CONSTEXPR_CXX20 void test(S s, typename S::size_type n, S expected) {
     s.resize(n);
     LIBCPP_ASSERT(s.__invariants());
     assert(s == expected);
+    LIBCPP_ASSERT(is_string_asan_correct(s));
   }
 #ifndef TEST_HAS_NO_EXCEPTIONS
   else if (!TEST_IS_CONSTANT_EVALUATED) {
@@ -61,6 +63,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
   test_string<std::string>();
 #if TEST_STD_VER >= 11
   test_string<std::basic_string<char, std::char_traits<char>, min_allocator<char>>>();
+  test_string<std::basic_string<char, std::char_traits<char>, safe_allocator<char>>>();
 #endif
 
   return true;
diff --git a/libcxx/test/std/strings/basic.string/string.capacity/resize_size_char.pass.cpp b/libcxx/test/std/strings/basic.string/string.capacity/resize_size_char.pass.cpp
index 3b6adc0b0afeb191710cb3d2a2f84bb1eb847b6b..e3b925ca8bcdbde5a67ba250a539b213f7e4027d 100644
--- a/libcxx/test/std/strings/basic.string/string.capacity/resize_size_char.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.capacity/resize_size_char.pass.cpp
@@ -16,6 +16,7 @@
 
 #include "test_macros.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class S>
 TEST_CONSTEXPR_CXX20 void test(S s, typename S::size_type n, typename S::value_type c, S expected) {
@@ -23,6 +24,7 @@ TEST_CONSTEXPR_CXX20 void test(S s, typename S::size_type n, typename S::value_t
     s.resize(n, c);
     LIBCPP_ASSERT(s.__invariants());
     assert(s == expected);
+    LIBCPP_ASSERT(is_string_asan_correct(s));
   }
 #ifndef TEST_HAS_NO_EXCEPTIONS
   else if (!TEST_IS_CONSTANT_EVALUATED) {
@@ -57,12 +59,23 @@ TEST_CONSTEXPR_CXX20 void test_string() {
        'a',
        S("12345678901234567890123456789012345678901234567890aaaaaaaaaa"));
   test(S(), S::npos, 'a', S("not going to happen"));
+  //ASan:
+  test(S(), 21, 'a', S("aaaaaaaaaaaaaaaaaaaaa"));
+  test(S(), 22, 'a', S("aaaaaaaaaaaaaaaaaaaaaa"));
+  test(S(), 23, 'a', S("aaaaaaaaaaaaaaaaaaaaaaa"));
+  test(S(), 24, 'a', S("aaaaaaaaaaaaaaaaaaaaaaaa"));
+  test(S(), 29, 'a', S("aaaaaaaaaaaaaaaaaaaaaaaaaaaaa"));
+  test(S(), 30, 'a', S("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"));
+  test(S(), 31, 'a', S("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"));
+  test(S(), 32, 'a', S("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"));
+  test(S(), 33, 'a', S("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"));
 }
 
 TEST_CONSTEXPR_CXX20 bool test() {
   test_string<std::string>();
 #if TEST_STD_VER >= 11
   test_string<std::basic_string<char, std::char_traits<char>, min_allocator<char>>>();
+  test_string<std::basic_string<char, std::char_traits<char>, safe_allocator<char>>>();
 #endif
 
   return true;
diff --git a/libcxx/test/std/strings/basic.string/string.capacity/shrink_to_fit.pass.cpp b/libcxx/test/std/strings/basic.string/string.capacity/shrink_to_fit.pass.cpp
index 66eefdd383dc08ec74dab113292fe3b41b9f6e2e..057050cdcf7fa3a486e5827e3b426431dcb68eef 100644
--- a/libcxx/test/std/strings/basic.string/string.capacity/shrink_to_fit.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.capacity/shrink_to_fit.pass.cpp
@@ -15,6 +15,7 @@
 
 #include "test_macros.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class S>
 TEST_CONSTEXPR_CXX20 void test(S s) {
@@ -25,6 +26,7 @@ TEST_CONSTEXPR_CXX20 void test(S s) {
   assert(s == s0);
   assert(s.capacity() <= old_cap);
   assert(s.capacity() >= s.size());
+  LIBCPP_ASSERT(is_string_asan_correct(s));
 }
 
 template <class S>
@@ -43,12 +45,19 @@ TEST_CONSTEXPR_CXX20 void test_string() {
   s.assign(100, 'a');
   s.erase(50);
   test(s);
+
+  s.assign(100, 'a');
+  for (int i = 0; i <= 9; ++i) {
+    s.erase(90 - 10 * i);
+    test(s);
+  }
 }
 
 TEST_CONSTEXPR_CXX20 bool test() {
   test_string<std::string>();
 #if TEST_STD_VER >= 11
   test_string<std::basic_string<char, std::char_traits<char>, min_allocator<char>>>();
+  test_string<std::basic_string<char, std::char_traits<char>, safe_allocator<char>>>();
 #endif
 
   return true;
diff --git a/libcxx/test/std/strings/basic.string/string.cons/T_size_size.pass.cpp b/libcxx/test/std/strings/basic.string/string.cons/T_size_size.pass.cpp
index a6b625b7b0e8113ef4a0657042e4d43ecf465524..dcf697bed752fa4a6860e53cfde5a6b790984c54 100644
--- a/libcxx/test/std/strings/basic.string/string.cons/T_size_size.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.cons/T_size_size.pass.cpp
@@ -23,6 +23,7 @@
 #include "test_macros.h"
 #include "test_allocator.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class S, class SV>
 TEST_CONSTEXPR_CXX20 void test(SV sv, std::size_t pos, std::size_t n) {
@@ -38,6 +39,7 @@ TEST_CONSTEXPR_CXX20 void test(SV sv, std::size_t pos, std::size_t n) {
     assert(T::compare(s2.data(), sv.data() + pos, rlen) == 0);
     assert(s2.get_allocator() == A());
     assert(s2.capacity() >= s2.size());
+    LIBCPP_ASSERT(is_string_asan_correct(s2));
   }
 #ifndef TEST_HAS_NO_EXCEPTIONS
   else if (!TEST_IS_CONSTANT_EVALUATED) {
@@ -113,6 +115,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
   test_string(test_allocator<char>(8));
 #if TEST_STD_VER >= 11
   test_string(min_allocator<char>());
+  test_string(safe_allocator<char>());
 #endif
 
   {
diff --git a/libcxx/test/std/strings/basic.string/string.cons/alloc.pass.cpp b/libcxx/test/std/strings/basic.string/string.cons/alloc.pass.cpp
index 97a0566ba031b00daf5ed71fd11c67ed884a3428..91beac37764db48b43a6694cb02de473bf20296a 100644
--- a/libcxx/test/std/strings/basic.string/string.cons/alloc.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.cons/alloc.pass.cpp
@@ -16,6 +16,7 @@
 #include "test_macros.h"
 #include "test_allocator.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class S>
 TEST_CONSTEXPR_CXX20 void test() {
@@ -31,6 +32,7 @@ TEST_CONSTEXPR_CXX20 void test() {
     assert(s.size() == 0);
     assert(s.capacity() >= s.size());
     assert(s.get_allocator() == typename S::allocator_type());
+    LIBCPP_ASSERT(is_string_asan_correct(s));
   }
   {
 #if TEST_STD_VER > 14
@@ -46,6 +48,7 @@ TEST_CONSTEXPR_CXX20 void test() {
     assert(s.size() == 0);
     assert(s.capacity() >= s.size());
     assert(s.get_allocator() == typename S::allocator_type(5));
+    LIBCPP_ASSERT(is_string_asan_correct(s));
   }
 }
 
@@ -65,6 +68,7 @@ TEST_CONSTEXPR_CXX20 void test2() {
     assert(s.size() == 0);
     assert(s.capacity() >= s.size());
     assert(s.get_allocator() == typename S::allocator_type());
+    LIBCPP_ASSERT(is_string_asan_correct(s));
   }
   {
 #  if TEST_STD_VER > 14
@@ -80,6 +84,7 @@ TEST_CONSTEXPR_CXX20 void test2() {
     assert(s.size() == 0);
     assert(s.capacity() >= s.size());
     assert(s.get_allocator() == typename S::allocator_type());
+    LIBCPP_ASSERT(is_string_asan_correct(s));
   }
 }
 
@@ -89,6 +94,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
   test<std::basic_string<char, std::char_traits<char>, test_allocator<char> > >();
 #if TEST_STD_VER >= 11
   test2<std::basic_string<char, std::char_traits<char>, min_allocator<char> > >();
+  test2<std::basic_string<char, std::char_traits<char>, safe_allocator<char> > >();
   test2<std::basic_string<char, std::char_traits<char>, explicit_allocator<char> > >();
 #endif
 
diff --git a/libcxx/test/std/strings/basic.string/string.cons/brace_assignment.pass.cpp b/libcxx/test/std/strings/basic.string/string.cons/brace_assignment.pass.cpp
index e7d18b4ca87179d9cdcdd479e0329a91901210ba..49a90872c56faede921759f8d4770e06ddce481b 100644
--- a/libcxx/test/std/strings/basic.string/string.cons/brace_assignment.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.cons/brace_assignment.pass.cpp
@@ -17,6 +17,7 @@
 #include <cassert>
 
 #include "test_macros.h"
+#include "asan_testing.h"
 
 TEST_CONSTEXPR_CXX20 bool test() {
   // Test that assignment from {} and {ptr, len} are allowed and are not
@@ -25,11 +26,37 @@ TEST_CONSTEXPR_CXX20 bool test() {
     std::string s = "hello world";
     s             = {};
     assert(s.empty());
+    LIBCPP_ASSERT(is_string_asan_correct(s));
   }
   {
     std::string s = "hello world";
     s             = {"abc", 2};
     assert(s == "ab");
+    LIBCPP_ASSERT(is_string_asan_correct(s));
+  }
+  {
+    std::string s = "hello world";
+    s             = {"It'sALongString!NoSSO!qwertyuiop", 30};
+    assert(s == "It'sALongString!NoSSO!qwertyui");
+    LIBCPP_ASSERT(is_string_asan_correct(s));
+  }
+  {
+    std::string s = "Hello world! Hello world! Hello world! Hello world! Hello world!";
+    s             = {"It'sALongString!NoSSO!qwertyuiop", 30};
+    assert(s == "It'sALongString!NoSSO!qwertyui");
+    LIBCPP_ASSERT(is_string_asan_correct(s));
+  }
+  {
+    std::string s = "Hello world! Hello world! Hello world! Hello world! Hello world!";
+    s             = {"abc", 2};
+    assert(s == "ab");
+    LIBCPP_ASSERT(is_string_asan_correct(s));
+  }
+  {
+    std::string s = "Hello world! Hello world! Hello world! Hello world! Hello world!";
+    s             = {"abc", 0};
+    assert(s == "");
+    LIBCPP_ASSERT(is_string_asan_correct(s));
   }
 
   return true;
diff --git a/libcxx/test/std/strings/basic.string/string.cons/char_assignment.pass.cpp b/libcxx/test/std/strings/basic.string/string.cons/char_assignment.pass.cpp
index 3cffc82e94835263a94d77b951cbb419c860483b..1019dc8bca5df520cb188818d7a7c421c19132c4 100644
--- a/libcxx/test/std/strings/basic.string/string.cons/char_assignment.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.cons/char_assignment.pass.cpp
@@ -15,6 +15,7 @@
 
 #include "test_macros.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class S>
 TEST_CONSTEXPR_CXX20 void test(S s1, typename S::value_type s2) {
@@ -24,6 +25,7 @@ TEST_CONSTEXPR_CXX20 void test(S s1, typename S::value_type s2) {
   assert(s1.size() == 1);
   assert(T::eq(s1[0], s2));
   assert(s1.capacity() >= s1.size());
+  LIBCPP_ASSERT(is_string_asan_correct(s1));
 }
 
 template <class S>
@@ -38,6 +40,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
   test_string<std::string>();
 #if TEST_STD_VER >= 11
   test_string<std::basic_string<char, std::char_traits<char>, min_allocator<char>>>();
+  test_string<std::basic_string<char, std::char_traits<char>, safe_allocator<char>>>();
 #endif
 
   return true;
diff --git a/libcxx/test/std/strings/basic.string/string.cons/copy.pass.cpp b/libcxx/test/std/strings/basic.string/string.cons/copy.pass.cpp
index 3afe76e88316f99cf9a6ff6d3a98a23f1bcdc7d1..f65f8e97c98249c74053179db722c490e99e58b4 100644
--- a/libcxx/test/std/strings/basic.string/string.cons/copy.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.cons/copy.pass.cpp
@@ -16,6 +16,7 @@
 #include "test_macros.h"
 #include "test_allocator.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class S>
 TEST_CONSTEXPR_CXX20 void test(S s1) {
@@ -24,6 +25,8 @@ TEST_CONSTEXPR_CXX20 void test(S s1) {
   assert(s2 == s1);
   assert(s2.capacity() >= s2.size());
   assert(s2.get_allocator() == s1.get_allocator());
+  LIBCPP_ASSERT(is_string_asan_correct(s1));
+  LIBCPP_ASSERT(is_string_asan_correct(s2));
 }
 
 template <class Alloc>
@@ -40,6 +43,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
   test_string(test_allocator<char>(3));
 #if TEST_STD_VER >= 11
   test_string(min_allocator<char>());
+  test_string(safe_allocator<char>());
 #endif
 
   return true;
diff --git a/libcxx/test/std/strings/basic.string/string.cons/copy_alloc.pass.cpp b/libcxx/test/std/strings/basic.string/string.cons/copy_alloc.pass.cpp
index 6b0040376a424ec1afd709707f82e8d831c2798a..b0045cb4afbba585274006712371632e260aea90 100644
--- a/libcxx/test/std/strings/basic.string/string.cons/copy_alloc.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.cons/copy_alloc.pass.cpp
@@ -16,6 +16,7 @@
 #include "test_macros.h"
 #include "test_allocator.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 #ifndef TEST_HAS_NO_EXCEPTIONS
 struct alloc_imp {
@@ -83,6 +84,8 @@ TEST_CONSTEXPR_CXX20 void test(S s1, const typename S::allocator_type& a) {
   assert(s2 == s1);
   assert(s2.capacity() >= s2.size());
   assert(s2.get_allocator() == a);
+  LIBCPP_ASSERT(is_string_asan_correct(s1));
+  LIBCPP_ASSERT(is_string_asan_correct(s2));
 }
 
 template <class Alloc>
@@ -99,6 +102,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
   test_string(test_allocator<char>(3));
 #if TEST_STD_VER >= 11
   test_string(min_allocator<char>());
+  test_string(safe_allocator<char>());
 #endif
 
 #if TEST_STD_VER >= 11
diff --git a/libcxx/test/std/strings/basic.string/string.cons/copy_assignment.pass.cpp b/libcxx/test/std/strings/basic.string/string.cons/copy_assignment.pass.cpp
index eb522aafa243024ba51ab74f6ac204d4ca3329d4..2e98fccb5394bba6c34b2d9c5b903eb6fad5147c 100644
--- a/libcxx/test/std/strings/basic.string/string.cons/copy_assignment.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.cons/copy_assignment.pass.cpp
@@ -16,6 +16,7 @@
 
 #include "test_macros.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class S>
 TEST_CONSTEXPR_CXX20 void test(S s1, const S& s2) {
@@ -23,6 +24,8 @@ TEST_CONSTEXPR_CXX20 void test(S s1, const S& s2) {
   LIBCPP_ASSERT(s1.__invariants());
   assert(s1 == s2);
   assert(s1.capacity() >= s1.size());
+  LIBCPP_ASSERT(is_string_asan_correct(s1));
+  LIBCPP_ASSERT(is_string_asan_correct(s2));
 }
 
 template <class S>
@@ -47,6 +50,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
   test_string<std::string>();
 #if TEST_STD_VER >= 11
   test_string<std::basic_string<char, std::char_traits<char>, min_allocator<char>>>();
+  test_string<std::basic_string<char, std::char_traits<char>, safe_allocator<char>>>();
 #endif
 
 #if TEST_STD_VER >= 11
diff --git a/libcxx/test/std/strings/basic.string/string.cons/default.pass.cpp b/libcxx/test/std/strings/basic.string/string.cons/default.pass.cpp
index 3993a40dd5a165c24c7cd654571bb34273088991..fc263f9820cb5b79f14d2c201a3d7d38fc5400d7 100644
--- a/libcxx/test/std/strings/basic.string/string.cons/default.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.cons/default.pass.cpp
@@ -15,6 +15,7 @@
 
 #include "test_macros.h"
 #include "test_allocator.h"
+#include "asan_testing.h"
 
 #if TEST_STD_VER >= 11
 // Test the noexcept specification, which is a conforming extension
@@ -30,6 +31,7 @@ LIBCPP_STATIC_ASSERT(!std::is_nothrow_default_constructible<
 TEST_CONSTEXPR_CXX20 bool test() {
   std::string str;
   assert(str.empty());
+  LIBCPP_ASSERT(is_string_asan_correct(str));
 
   return true;
 }
diff --git a/libcxx/test/std/strings/basic.string/string.cons/from_range.pass.cpp b/libcxx/test/std/strings/basic.string/string.cons/from_range.pass.cpp
index 3ae5b74a3504a04120744cda413fc2231f4d3195..7f33237de4631da4c9b92df152460cec5ab15db3 100644
--- a/libcxx/test/std/strings/basic.string/string.cons/from_range.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.cons/from_range.pass.cpp
@@ -19,6 +19,7 @@
 #include "../../../containers/from_range_helpers.h"
 #include "../../../containers/sequences/from_range_sequence_containers.h"
 #include "test_macros.h"
+#include "asan_testing.h"
 
 template <class Container, class Range, class Alloc>
 concept StringHasFromRangeAllocCtr =
@@ -70,6 +71,7 @@ constexpr void test_with_input(std::vector<char> input) {
     LIBCPP_ASSERT(c.__invariants());
     assert(c.size() == static_cast<std::size_t>(std::distance(c.begin(), c.end())));
     assert(std::ranges::equal(in, c));
+    LIBCPP_ASSERT(is_string_asan_correct(c));
   }
 
   { // (range, allocator)
@@ -80,6 +82,7 @@ constexpr void test_with_input(std::vector<char> input) {
     assert(c.get_allocator() == alloc);
     assert(c.size() == static_cast<std::size_t>(std::distance(c.begin(), c.end())));
     assert(std::ranges::equal(in, c));
+    LIBCPP_ASSERT(is_string_asan_correct(c));
   }
 }
 
diff --git a/libcxx/test/std/strings/basic.string/string.cons/from_range_deduction.pass.cpp b/libcxx/test/std/strings/basic.string/string.cons/from_range_deduction.pass.cpp
index b2dab03506f03a46cef6a5929ade9402b0f184d7..83c3dfdfa79dded9958390709d019b338f67e1db 100644
--- a/libcxx/test/std/strings/basic.string/string.cons/from_range_deduction.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.cons/from_range_deduction.pass.cpp
@@ -26,6 +26,7 @@
 
 #include "deduction_guides_sfinae_checks.h"
 #include "test_allocator.h"
+#include "asan_testing.h"
 
 int main(int, char**) {
   using Char = char16_t;
@@ -33,12 +34,14 @@ int main(int, char**) {
   {
     std::basic_string c(std::from_range, std::array<Char, 0>());
     static_assert(std::is_same_v<decltype(c), std::basic_string<Char>>);
+    LIBCPP_ASSERT(is_string_asan_correct(c));
   }
 
   {
     using Alloc = test_allocator<Char>;
     std::basic_string c(std::from_range, std::array<Char, 0>(), Alloc());
     static_assert(std::is_same_v<decltype(c), std::basic_string<Char, std::char_traits<Char>, Alloc>>);
+    LIBCPP_ASSERT(is_string_asan_correct(c));
   }
 
   // Note: defining `value_type` is a workaround because one of the deduction guides will end up instantiating
diff --git a/libcxx/test/std/strings/basic.string/string.cons/initializer_list.pass.cpp b/libcxx/test/std/strings/basic.string/string.cons/initializer_list.pass.cpp
index 5b7e8bde2e6e877ee13b155e7beeda6bd23286e5..ebdcc523f055d4ab40177b94eb74de1f3e024631 100644
--- a/libcxx/test/std/strings/basic.string/string.cons/initializer_list.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.cons/initializer_list.pass.cpp
@@ -18,6 +18,7 @@
 #include "test_macros.h"
 #include "test_allocator.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 // clang-format off
 template <template <class> class Alloc>
@@ -25,11 +26,23 @@ TEST_CONSTEXPR_CXX20 void test_string() {
   {
     std::basic_string<char, std::char_traits<char>, Alloc<char> > s = {'a', 'b', 'c'};
     assert(s == "abc");
+    LIBCPP_ASSERT(is_string_asan_correct(s));
+  }
+  {
+    std::basic_string<char, std::char_traits<char>, Alloc<char> > s = {'a', 'a', 'a', 'a', 'a', 'a', 'a', 'a', 'a', 'a', 'a', 'a', 'a', 'a', 'a', 'a', 'a', 'a', 'a', 'a', 'a', 'a', 'a', 'a', 'a'};
+    assert(s == "aaaaaaaaaaaaaaaaaaaaaaaaa");
+    LIBCPP_ASSERT(is_string_asan_correct(s));
   }
 #ifndef TEST_HAS_NO_WIDE_CHARACTERS
   {
     std::basic_string<wchar_t, std::char_traits<wchar_t>, Alloc<wchar_t> > s = {L'a', L'b', L'c'};
     assert(s == L"abc");
+    LIBCPP_ASSERT(is_string_asan_correct(s));
+  }
+  {
+    std::basic_string<wchar_t, std::char_traits<wchar_t>, Alloc<wchar_t> > s = {L'a', L'a', L'a', L'a', L'a', L'a', L'a', L'a', L'a', L'a', L'a', L'a', L'a', L'a', L'a', L'a', L'a', L'a', L'a', L'a', L'a', L'a', L'a', L'a', L'a'};
+    assert(s == L"aaaaaaaaaaaaaaaaaaaaaaaaa");
+    LIBCPP_ASSERT(is_string_asan_correct(s));
   }
 #endif
 }
@@ -38,6 +51,7 @@ TEST_CONSTEXPR_CXX20 void test_string() {
 TEST_CONSTEXPR_CXX20 bool test() {
   test_string<std::allocator>();
   test_string<min_allocator>();
+  test_string<safe_allocator>();
 
   return true;
 }
diff --git a/libcxx/test/std/strings/basic.string/string.cons/initializer_list_assignment.pass.cpp b/libcxx/test/std/strings/basic.string/string.cons/initializer_list_assignment.pass.cpp
index 88597daba121ac476f6259db18eaf7a8b5969cae..a3491696a4fc8aa1a658e6632a065fc67d7561d8 100644
--- a/libcxx/test/std/strings/basic.string/string.cons/initializer_list_assignment.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.cons/initializer_list_assignment.pass.cpp
@@ -17,6 +17,7 @@
 
 #include "test_macros.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 // clang-format off
 template <template <class> class Alloc>
@@ -27,6 +28,16 @@ TEST_CONSTEXPR_CXX20 void test_string() {
     S& result = (s = {'a', 'b', 'c'});
     assert(s == "abc");
     assert(&result == &s);
+    LIBCPP_ASSERT(is_string_asan_correct(s));
+    LIBCPP_ASSERT(is_string_asan_correct(result));
+  }
+  {
+    typedef std::basic_string<char, std::char_traits<char>, Alloc<char>> S;
+    S s;
+    s = {'a', 'a', 'a', 'a', 'a', 'a', 'a', 'a', 'a', 'a', 'a', 'a',
+         'a', 'a', 'a', 'a', 'a', 'a', 'a', 'a', 'a', 'a', 'a', 'a'};
+    assert(s == "aaaaaaaaaaaaaaaaaaaaaaaa");
+    LIBCPP_ASSERT(is_string_asan_correct(s));
   }
 #ifndef TEST_HAS_NO_WIDE_CHARACTERS
   {
@@ -35,6 +46,17 @@ TEST_CONSTEXPR_CXX20 void test_string() {
     S& result = (s = {L'a', L'b', L'c'});
     assert(s == L"abc");
     assert(&result == &s);
+    LIBCPP_ASSERT(is_string_asan_correct(s));
+    LIBCPP_ASSERT(is_string_asan_correct(result));
+  }
+  {
+    typedef std::basic_string<wchar_t, std::char_traits<wchar_t>, Alloc<wchar_t>> S;
+    S s;
+    S& result = (s = {L'a', L'a', L'a', L'a', L'a', L'a', L'a', L'a', L'a', L'a', L'a', L'a', L'a', L'a', L'a', L'a', L'a', L'a', L'a', L'a', L'a', L'a', L'a', L'a', L'a'});
+    assert(s == L"aaaaaaaaaaaaaaaaaaaaaaaaa");
+    assert(&result == &s);
+    LIBCPP_ASSERT(is_string_asan_correct(s));
+    LIBCPP_ASSERT(is_string_asan_correct(result));
   }
 #endif
 }
@@ -43,6 +65,7 @@ TEST_CONSTEXPR_CXX20 void test_string() {
 TEST_CONSTEXPR_CXX20 bool test() {
   test_string<std::allocator>();
   test_string<min_allocator>();
+  test_string<safe_allocator>();
 
   return true;
 }
diff --git a/libcxx/test/std/strings/basic.string/string.cons/iter_alloc.pass.cpp b/libcxx/test/std/strings/basic.string/string.cons/iter_alloc.pass.cpp
index 2ef73e5ec8c1adeaabfa6695ed37a464c0ba976d..e14227d1a77171ee9bdfd8df42fd68c5f19c59fd 100644
--- a/libcxx/test/std/strings/basic.string/string.cons/iter_alloc.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.cons/iter_alloc.pass.cpp
@@ -21,6 +21,7 @@
 #include "test_allocator.h"
 #include "test_iterators.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class Alloc, class It>
 TEST_CONSTEXPR_CXX20 void test(It first, It last) {
@@ -37,6 +38,7 @@ TEST_CONSTEXPR_CXX20 void test(It first, It last) {
   }
   assert(s2.get_allocator() == Alloc());
   assert(s2.capacity() >= s2.size());
+  LIBCPP_ASSERT(is_string_asan_correct(s2));
 }
 
 template <class Alloc, class It>
@@ -54,6 +56,7 @@ TEST_CONSTEXPR_CXX20 void test(It first, It last, const Alloc& a) {
   }
   assert(s2.get_allocator() == a);
   assert(s2.capacity() >= s2.size());
+  LIBCPP_ASSERT(is_string_asan_correct(s2));
 }
 
 template <class Alloc>
diff --git a/libcxx/test/std/strings/basic.string/string.cons/iter_alloc_deduction.pass.cpp b/libcxx/test/std/strings/basic.string/string.cons/iter_alloc_deduction.pass.cpp
index 65762ccf89d93049b31e2b4bde22c1751cc13890..d9176da63d0dc73a5721a7462004d41a7c1fb612 100644
--- a/libcxx/test/std/strings/basic.string/string.cons/iter_alloc_deduction.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.cons/iter_alloc_deduction.pass.cpp
@@ -29,6 +29,7 @@
 #include "test_macros.h"
 #include "test_allocator.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 class NotAnIterator {};
 using NotAnInputIterator = std::back_insert_iterator<std::basic_string<char16_t>>;
@@ -66,6 +67,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
     static_assert(std::is_same_v<S::allocator_type, std::allocator<char>>, "");
     assert(s1.size() == 10);
     assert(s1.compare(0, s1.size(), s, s1.size()) == 0);
+    LIBCPP_ASSERT(is_string_asan_correct(s1));
   }
   {
     const char* s = "12345678901234";
@@ -76,6 +78,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
     static_assert(std::is_same_v<S::allocator_type, std::allocator<char>>, "");
     assert(s1.size() == 10);
     assert(s1.compare(0, s1.size(), s, s1.size()) == 0);
+    LIBCPP_ASSERT(is_string_asan_correct(s1));
   }
 #ifndef TEST_HAS_NO_WIDE_CHARACTERS
   {
@@ -87,6 +90,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
     static_assert(std::is_same_v<S::allocator_type, test_allocator<wchar_t>>, "");
     assert(s1.size() == 10);
     assert(s1.compare(0, s1.size(), s, s1.size()) == 0);
+    LIBCPP_ASSERT(is_string_asan_correct(s1));
   }
 #endif
   {
@@ -98,6 +102,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
     static_assert(std::is_same_v<S::allocator_type, min_allocator<char16_t>>, "");
     assert(s1.size() == 10);
     assert(s1.compare(0, s1.size(), s, s1.size()) == 0);
+    LIBCPP_ASSERT(is_string_asan_correct(s1));
   }
   {
     const char32_t* s = U"12345678901234";
@@ -108,6 +113,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
     static_assert(std::is_same_v<S::allocator_type, explicit_allocator<char32_t>>, "");
     assert(s1.size() == 10);
     assert(s1.compare(0, s1.size(), s, s1.size()) == 0);
+    LIBCPP_ASSERT(is_string_asan_correct(s1));
   }
 
   return true;
diff --git a/libcxx/test/std/strings/basic.string/string.cons/move.pass.cpp b/libcxx/test/std/strings/basic.string/string.cons/move.pass.cpp
index f5ffebd35d50e92d86ed14645122a3af4a7eb3bb..5d2d774b3606383e1cb587b65fe4ac3c2d10bf21 100644
--- a/libcxx/test/std/strings/basic.string/string.cons/move.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.cons/move.pass.cpp
@@ -18,6 +18,7 @@
 #include "test_macros.h"
 #include "test_allocator.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class S>
 TEST_CONSTEXPR_CXX20 void test(S s0) {
@@ -28,6 +29,9 @@ TEST_CONSTEXPR_CXX20 void test(S s0) {
   assert(s2 == s1);
   assert(s2.capacity() >= s2.size());
   assert(s2.get_allocator() == s1.get_allocator());
+  LIBCPP_ASSERT(is_string_asan_correct(s0));
+  LIBCPP_ASSERT(is_string_asan_correct(s1));
+  LIBCPP_ASSERT(is_string_asan_correct(s2));
 }
 
 template <class Alloc>
diff --git a/libcxx/test/std/strings/basic.string/string.cons/move_alloc.pass.cpp b/libcxx/test/std/strings/basic.string/string.cons/move_alloc.pass.cpp
index 0e5c72192adea64aef4af73daf4ce1b77cb555ea..7210d67b5cb226bcb386457ce9af2f502f6da43b 100644
--- a/libcxx/test/std/strings/basic.string/string.cons/move_alloc.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.cons/move_alloc.pass.cpp
@@ -18,6 +18,7 @@
 #include "test_macros.h"
 #include "test_allocator.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class S>
 TEST_CONSTEXPR_CXX20 void test(S s0, const typename S::allocator_type& a) {
@@ -28,6 +29,9 @@ TEST_CONSTEXPR_CXX20 void test(S s0, const typename S::allocator_type& a) {
   assert(s2 == s1);
   assert(s2.capacity() >= s2.size());
   assert(s2.get_allocator() == a);
+  LIBCPP_ASSERT(is_string_asan_correct(s0));
+  LIBCPP_ASSERT(is_string_asan_correct(s1));
+  LIBCPP_ASSERT(is_string_asan_correct(s2));
 }
 
 TEST_CONSTEXPR_CXX20 bool test() {
diff --git a/libcxx/test/std/strings/basic.string/string.cons/move_assignment.pass.cpp b/libcxx/test/std/strings/basic.string/string.cons/move_assignment.pass.cpp
index cb0fe0396cb7900ca04fa62258647477779be80c..9af9abdfc449cf85fa03c0dd757ec71a4a0a4d80 100644
--- a/libcxx/test/std/strings/basic.string/string.cons/move_assignment.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.cons/move_assignment.pass.cpp
@@ -19,6 +19,7 @@
 #include "test_macros.h"
 #include "test_allocator.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class S>
 TEST_CONSTEXPR_CXX20 void test(S s1, S s2) {
@@ -28,6 +29,9 @@ TEST_CONSTEXPR_CXX20 void test(S s1, S s2) {
   LIBCPP_ASSERT(s2.__invariants());
   assert(s1 == s0);
   assert(s1.capacity() >= s1.size());
+  LIBCPP_ASSERT(is_string_asan_correct(s0));
+  LIBCPP_ASSERT(is_string_asan_correct(s1));
+  LIBCPP_ASSERT(is_string_asan_correct(s2));
 }
 
 template <class S>
@@ -52,6 +56,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
   test_string<std::string>();
 #if TEST_STD_VER >= 11
   test_string<std::basic_string<char, std::char_traits<char>, min_allocator<char>>>();
+  test_string<std::basic_string<char, std::char_traits<char>, safe_allocator<char>>>();
 #endif
 
   return true;
diff --git a/libcxx/test/std/strings/basic.string/string.cons/pointer_alloc.pass.cpp b/libcxx/test/std/strings/basic.string/string.cons/pointer_alloc.pass.cpp
index 84a29df5b57bf1626f5ad5ec19ec323fe9ddab62..d5d5152e11e54e990bd28aeae5b31b21974e8976 100644
--- a/libcxx/test/std/strings/basic.string/string.cons/pointer_alloc.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.cons/pointer_alloc.pass.cpp
@@ -19,6 +19,7 @@
 #include "test_macros.h"
 #include "test_allocator.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class Alloc, class charT>
 TEST_CONSTEXPR_CXX20 void test(const charT* s) {
@@ -31,6 +32,7 @@ TEST_CONSTEXPR_CXX20 void test(const charT* s) {
   assert(T::compare(s2.data(), s, n) == 0);
   assert(s2.get_allocator() == Alloc());
   assert(s2.capacity() >= s2.size());
+  LIBCPP_ASSERT(is_string_asan_correct(s2));
 }
 
 template <class Alloc, class charT>
@@ -44,6 +46,7 @@ TEST_CONSTEXPR_CXX20 void test(const charT* s, const Alloc& a) {
   assert(T::compare(s2.data(), s, n) == 0);
   assert(s2.get_allocator() == a);
   assert(s2.capacity() >= s2.size());
+  LIBCPP_ASSERT(is_string_asan_correct(s2));
 }
 
 template <class Alloc>
@@ -67,6 +70,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
   test(test_allocator<char>(2));
 #if TEST_STD_VER >= 11
   test(min_allocator<char>());
+  test(safe_allocator<char>());
 #endif
 
   return true;
diff --git a/libcxx/test/std/strings/basic.string/string.cons/pointer_assignment.pass.cpp b/libcxx/test/std/strings/basic.string/string.cons/pointer_assignment.pass.cpp
index eb3296d60403b6bc2f07ebea88556c9be7f1de02..98430cdd6ab998ad49a1ba521fdf4ae7bc523c42 100644
--- a/libcxx/test/std/strings/basic.string/string.cons/pointer_assignment.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.cons/pointer_assignment.pass.cpp
@@ -16,6 +16,7 @@
 
 #include "test_macros.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class S>
 TEST_CONSTEXPR_CXX20 void test(S s1, const typename S::value_type* s2) {
@@ -25,6 +26,7 @@ TEST_CONSTEXPR_CXX20 void test(S s1, const typename S::value_type* s2) {
   assert(s1.size() == T::length(s2));
   assert(T::compare(s1.data(), s2, s1.size()) == 0);
   assert(s1.capacity() >= s1.size());
+  LIBCPP_ASSERT(is_string_asan_correct(s1));
 }
 
 template <class S>
@@ -49,6 +51,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
   test_string<std::string>();
 #if TEST_STD_VER >= 11
   test_string<std::basic_string<char, std::char_traits<char>, min_allocator<char>>>();
+  test_string<std::basic_string<char, std::char_traits<char>, safe_allocator<char>>>();
 #endif
 
   return true;
diff --git a/libcxx/test/std/strings/basic.string/string.cons/pointer_size_alloc.pass.cpp b/libcxx/test/std/strings/basic.string/string.cons/pointer_size_alloc.pass.cpp
index e0dab72b5c6327eadd684cc2d2820043c8ed9c8d..b6ba2be46d18bd0b32ffc08ad97b684b7d915c2d 100644
--- a/libcxx/test/std/strings/basic.string/string.cons/pointer_size_alloc.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.cons/pointer_size_alloc.pass.cpp
@@ -18,6 +18,7 @@
 #include "test_macros.h"
 #include "test_allocator.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class Alloc, class CharT>
 TEST_CONSTEXPR_CXX20 void test(const CharT* s, unsigned n) {
@@ -29,6 +30,7 @@ TEST_CONSTEXPR_CXX20 void test(const CharT* s, unsigned n) {
   assert(T::compare(s2.data(), s, n) == 0);
   assert(s2.get_allocator() == Alloc());
   assert(s2.capacity() >= s2.size());
+  LIBCPP_ASSERT(is_string_asan_correct(s2));
 }
 
 template <class Alloc, class CharT>
@@ -41,6 +43,7 @@ TEST_CONSTEXPR_CXX20 void test(const CharT* s, unsigned n, const Alloc& a) {
   assert(T::compare(s2.data(), s, n) == 0);
   assert(s2.get_allocator() == a);
   assert(s2.capacity() >= s2.size());
+  LIBCPP_ASSERT(is_string_asan_correct(s2));
 }
 
 template <class Alloc>
@@ -64,6 +67,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
   test(test_allocator<char>(2));
 #if TEST_STD_VER >= 11
   test(min_allocator<char>());
+  test(safe_allocator<char>());
 #endif
 
 #if TEST_STD_VER >= 11
diff --git a/libcxx/test/std/strings/basic.string/string.cons/size_char_alloc.pass.cpp b/libcxx/test/std/strings/basic.string/string.cons/size_char_alloc.pass.cpp
index 3afbcdffa6bdfe370a5578d042e79d27d3e9882b..a87e01f0fc0881f637e295cfc122988f3ffbaf00 100644
--- a/libcxx/test/std/strings/basic.string/string.cons/size_char_alloc.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.cons/size_char_alloc.pass.cpp
@@ -19,6 +19,7 @@
 #include "test_macros.h"
 #include "test_allocator.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class Alloc, class charT>
 TEST_CONSTEXPR_CXX20 void test(unsigned n, charT c) {
@@ -30,6 +31,7 @@ TEST_CONSTEXPR_CXX20 void test(unsigned n, charT c) {
     assert(s2[i] == c);
   assert(s2.get_allocator() == Alloc());
   assert(s2.capacity() >= s2.size());
+  LIBCPP_ASSERT(is_string_asan_correct(s2));
 }
 
 template <class Alloc, class charT>
@@ -42,6 +44,7 @@ TEST_CONSTEXPR_CXX20 void test(unsigned n, charT c, const Alloc& a) {
     assert(s2[i] == c);
   assert(s2.get_allocator() == a);
   assert(s2.capacity() >= s2.size());
+  LIBCPP_ASSERT(is_string_asan_correct(s2));
 }
 
 template <class Alloc, class Tp>
@@ -94,6 +97,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
   test_string(test_allocator<char>(2));
 #if TEST_STD_VER >= 11
   test_string(min_allocator<char>());
+  test_string(safe_allocator<char>());
 #endif
 
   return true;
diff --git a/libcxx/test/std/strings/basic.string/string.cons/string_view.pass.cpp b/libcxx/test/std/strings/basic.string/string.cons/string_view.pass.cpp
index 74fa76a013a6926c956398741948725cb734163a..ad37b50b83ba7eee740036eb420faf3aa218b4bd 100644
--- a/libcxx/test/std/strings/basic.string/string.cons/string_view.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.cons/string_view.pass.cpp
@@ -20,6 +20,7 @@
 #include "min_allocator.h"
 #include "test_allocator.h"
 #include "test_macros.h"
+#include "asan_testing.h"
 
 static_assert(!std::is_convertible<std::string_view, std::string const&>::value, "");
 static_assert(!std::is_convertible<std::string_view, std::string>::value, "");
@@ -35,6 +36,7 @@ TEST_CONSTEXPR_CXX20 void test(std::basic_string_view<CharT> sv) {
     assert(T::compare(s2.data(), sv.data(), sv.size()) == 0);
     assert(s2.get_allocator() == Alloc());
     assert(s2.capacity() >= s2.size());
+    LIBCPP_ASSERT(is_string_asan_correct(s2));
   }
   {
     S s2;
@@ -44,6 +46,7 @@ TEST_CONSTEXPR_CXX20 void test(std::basic_string_view<CharT> sv) {
     assert(T::compare(s2.data(), sv.data(), sv.size()) == 0);
     assert(s2.get_allocator() == Alloc());
     assert(s2.capacity() >= s2.size());
+    LIBCPP_ASSERT(is_string_asan_correct(s2));
   }
 }
 
@@ -58,6 +61,7 @@ TEST_CONSTEXPR_CXX20 void test(std::basic_string_view<CharT> sv, const Alloc& a)
     assert(T::compare(s2.data(), sv.data(), sv.size()) == 0);
     assert(s2.get_allocator() == a);
     assert(s2.capacity() >= s2.size());
+    LIBCPP_ASSERT(is_string_asan_correct(s2));
   }
   {
     S s2(a);
@@ -67,6 +71,7 @@ TEST_CONSTEXPR_CXX20 void test(std::basic_string_view<CharT> sv, const Alloc& a)
     assert(T::compare(s2.data(), sv.data(), sv.size()) == 0);
     assert(s2.get_allocator() == a);
     assert(s2.capacity() >= s2.size());
+    LIBCPP_ASSERT(is_string_asan_correct(s2));
   }
 }
 
@@ -93,6 +98,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
   test_string(test_allocator<char>(2));
 #if TEST_STD_VER >= 11
   test_string(min_allocator<char>());
+  test_string(safe_allocator<char>());
 #endif
 
   return true;
diff --git a/libcxx/test/std/strings/basic.string/string.cons/string_view_assignment.pass.cpp b/libcxx/test/std/strings/basic.string/string.cons/string_view_assignment.pass.cpp
index b41d1336fd134c928fe22e95dbcbc3741df94c0a..2e6c54a584b6e148d0550270726303d404ec3a49 100644
--- a/libcxx/test/std/strings/basic.string/string.cons/string_view_assignment.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.cons/string_view_assignment.pass.cpp
@@ -15,6 +15,7 @@
 
 #include "test_macros.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class S, class SV>
 TEST_CONSTEXPR_CXX20 void test(S s1, SV sv) {
@@ -24,6 +25,7 @@ TEST_CONSTEXPR_CXX20 void test(S s1, SV sv) {
   assert(s1.size() == sv.size());
   assert(T::compare(s1.data(), sv.data(), s1.size()) == 0);
   assert(s1.capacity() >= s1.size());
+  LIBCPP_ASSERT(is_string_asan_correct(s1));
 }
 
 template <class S>
@@ -49,6 +51,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
   test_string<std::string>();
 #if TEST_STD_VER >= 11
   test_string<std::basic_string<char, std::char_traits<char>, min_allocator<char>>>();
+  test_string<std::basic_string<char, std::char_traits<char>, safe_allocator<char>>>();
 #endif
 
   return true;
diff --git a/libcxx/test/std/strings/basic.string/string.cons/substr.pass.cpp b/libcxx/test/std/strings/basic.string/string.cons/substr.pass.cpp
index 693edab76b17b0ce471fe9501f94b1d8a1585f39..cafd9674f4898d729025fd89e06c3803f662c55d 100644
--- a/libcxx/test/std/strings/basic.string/string.cons/substr.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.cons/substr.pass.cpp
@@ -26,6 +26,7 @@
 #include "test_macros.h"
 #include "test_allocator.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class S>
 TEST_CONSTEXPR_CXX20 void test(S str, unsigned pos) {
@@ -40,6 +41,7 @@ TEST_CONSTEXPR_CXX20 void test(S str, unsigned pos) {
     assert(T::compare(s2.data(), str.data() + pos, rlen) == 0);
     assert(s2.get_allocator() == A());
     assert(s2.capacity() >= s2.size());
+    LIBCPP_ASSERT(is_string_asan_correct(s2));
   }
 #ifndef TEST_HAS_NO_EXCEPTIONS
   else if (!TEST_IS_CONSTANT_EVALUATED) {
@@ -65,6 +67,7 @@ TEST_CONSTEXPR_CXX20 void test(S str, unsigned pos, unsigned n) {
     assert(T::compare(s2.data(), str.data() + pos, rlen) == 0);
     assert(s2.get_allocator() == A());
     assert(s2.capacity() >= s2.size());
+    LIBCPP_ASSERT(is_string_asan_correct(s2));
   }
 #ifndef TEST_HAS_NO_EXCEPTIONS
   else if (!TEST_IS_CONSTANT_EVALUATED) {
@@ -90,6 +93,7 @@ TEST_CONSTEXPR_CXX20 void test(S str, unsigned pos, unsigned n, const typename S
     assert(T::compare(s2.data(), str.data() + pos, rlen) == 0);
     assert(s2.get_allocator() == a);
     assert(s2.capacity() >= s2.size());
+    LIBCPP_ASSERT(is_string_asan_correct(s2));
   }
 #ifndef TEST_HAS_NO_EXCEPTIONS
   else if (!TEST_IS_CONSTANT_EVALUATED) {
@@ -164,6 +168,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
   test_string(test_allocator<char>(3), test_allocator<char>(5));
 #if TEST_STD_VER >= 11
   test_string(min_allocator<char>(), min_allocator<char>());
+  test_string(safe_allocator<char>(), safe_allocator<char>());
 #endif
 
   return true;
diff --git a/libcxx/test/std/strings/basic.string/string.cons/substr_rvalue.pass.cpp b/libcxx/test/std/strings/basic.string/string.cons/substr_rvalue.pass.cpp
index 272ce339f3adc610c6c8a4d104bb30a245dc3f42..a9b07a6eb1aa46280470c56ba852e1d62b5ff968 100644
--- a/libcxx/test/std/strings/basic.string/string.cons/substr_rvalue.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.cons/substr_rvalue.pass.cpp
@@ -23,6 +23,7 @@
 #include "min_allocator.h"
 #include "test_allocator.h"
 #include "test_macros.h"
+#include "asan_testing.h"
 
 #define STR(string) MAKE_CSTRING(typename S::value_type, string)
 
@@ -39,6 +40,8 @@ constexpr void test_string_pos(S orig, typename S::size_type pos, S expected) {
   LIBCPP_ASSERT(orig.empty());
   LIBCPP_ASSERT(substr.__invariants());
   assert(substr == expected);
+  LIBCPP_ASSERT(is_string_asan_correct(orig));
+  LIBCPP_ASSERT(is_string_asan_correct(substr));
 }
 
 template <class S>
@@ -65,6 +68,8 @@ test_string_pos_alloc(S orig, typename S::size_type pos, const typename S::alloc
   LIBCPP_ASSERT(substr.__invariants());
   assert(substr == expected);
   assert(substr.get_allocator() == alloc);
+  LIBCPP_ASSERT(is_string_asan_correct(orig));
+  LIBCPP_ASSERT(is_string_asan_correct(substr));
 }
 
 template <class S>
@@ -95,6 +100,8 @@ constexpr void test_string_pos_n(S orig, typename S::size_type pos, typename S::
   LIBCPP_ASSERT(orig.empty());
   LIBCPP_ASSERT(substr.__invariants());
   assert(substr == expected);
+  LIBCPP_ASSERT(is_string_asan_correct(orig));
+  LIBCPP_ASSERT(is_string_asan_correct(substr));
 }
 
 template <class S>
@@ -122,6 +129,8 @@ constexpr void test_string_pos_n_alloc(
   LIBCPP_ASSERT(substr.__invariants());
   assert(substr == expected);
   assert(substr.get_allocator() == alloc);
+  LIBCPP_ASSERT(is_string_asan_correct(orig));
+  LIBCPP_ASSERT(is_string_asan_correct(substr));
 }
 
 template <class S>
diff --git a/libcxx/test/std/strings/basic.string/string.modifiers/string_append/append_range.pass.cpp b/libcxx/test/std/strings/basic.string/string.modifiers/string_append/append_range.pass.cpp
index d88e9f95736005c724aca8b59ae363614851cf24..9e06aa4aaa790ab926dbd59e1ccae31f2b0a407f 100644
--- a/libcxx/test/std/strings/basic.string/string.modifiers/string_append/append_range.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.modifiers/string_append/append_range.pass.cpp
@@ -15,6 +15,7 @@
 
 #include "../../../../containers/sequences/insert_range_sequence_containers.h"
 #include "test_macros.h"
+#include "asan_testing.h"
 
 // Tested cases:
 // - different kinds of insertions (appending an {empty/one-element/mid-sized/long range} into an
@@ -43,9 +44,13 @@ int main(int, char**) {
     std::string c;
     static_assert(std::is_lvalue_reference_v<decltype(c.append_range(FullContainer_Begin_EmptyRange<char>.input))>);
     assert(&c.append_range(FullContainer_Begin_EmptyRange<char>.input) == &c);
+    LIBCPP_ASSERT(is_string_asan_correct(c));
     assert(&c.append_range(FullContainer_Begin_OneElementRange<char>.input) == &c);
+    LIBCPP_ASSERT(is_string_asan_correct(c));
     assert(&c.append_range(FullContainer_Begin_MidRange<char>.input) == &c);
+    LIBCPP_ASSERT(is_string_asan_correct(c));
     assert(&c.append_range(FullContainer_Begin_LongRange<char>.input) == &c);
+    LIBCPP_ASSERT(is_string_asan_correct(c));
   }
 
   // Note: `test_append_range_exception_safety_throwing_copy` doesn't apply because copying chars cannot throw.
diff --git a/libcxx/test/std/strings/basic.string/string.modifiers/string_append/initializer_list.pass.cpp b/libcxx/test/std/strings/basic.string/string.modifiers/string_append/initializer_list.pass.cpp
index 6c5049f16bc61926e4634b756eba3939bc20c5a2..f1f5828bfe21d7a935b6be25eab573df580e03a7 100644
--- a/libcxx/test/std/strings/basic.string/string.modifiers/string_append/initializer_list.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.modifiers/string_append/initializer_list.pass.cpp
@@ -18,6 +18,7 @@
 #include "test_macros.h"
 #include "min_allocator.h"
 #include "nasty_string.h"
+#include "asan_testing.h"
 
 template <class S>
 TEST_CONSTEXPR_CXX20 void test() {
@@ -26,6 +27,7 @@ TEST_CONSTEXPR_CXX20 void test() {
   S s(CONVERT_TO_CSTRING(CharT, "123"));
   s.append({CharT('a'), CharT('b'), CharT('c')});
   assert(s == CONVERT_TO_CSTRING(CharT, "123abc"));
+  LIBCPP_ASSERT(is_string_asan_correct(s));
 }
 
 TEST_CONSTEXPR_CXX20 bool test() {
@@ -40,6 +42,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
   test<std::u32string>();
 
   test<std::basic_string<char, std::char_traits<char>, min_allocator<char>>>();
+  test<std::basic_string<char, std::char_traits<char>, safe_allocator<char>>>();
 #ifndef TEST_HAS_NO_NASTY_STRING
   test<nasty_string>();
 #endif
diff --git a/libcxx/test/std/strings/basic.string/string.modifiers/string_append/iterator.pass.cpp b/libcxx/test/std/strings/basic.string/string.modifiers/string_append/iterator.pass.cpp
index 2982cfe77042c0d6cbafb7f2adda89bb01f984df..684661aeab8fc3d48043a196806b38469408e240 100644
--- a/libcxx/test/std/strings/basic.string/string.modifiers/string_append/iterator.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.modifiers/string_append/iterator.pass.cpp
@@ -17,12 +17,14 @@
 #include "test_macros.h"
 #include "test_iterators.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class S, class It>
 TEST_CONSTEXPR_CXX20 void test(S s, It first, It last, S expected) {
   s.append(first, last);
   LIBCPP_ASSERT(s.__invariants());
   assert(s == expected);
+  LIBCPP_ASSERT(is_string_asan_correct(s));
 }
 
 #ifndef TEST_HAS_NO_EXCEPTIONS
@@ -225,6 +227,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
   test_string<std::string>();
 #if TEST_STD_VER >= 11
   test_string<std::basic_string<char, std::char_traits<char>, min_allocator<char> > >();
+  test_string<std::basic_string<char, std::char_traits<char>, safe_allocator<char> > >();
 #endif
 
   return true;
diff --git a/libcxx/test/std/strings/basic.string/string.modifiers/string_append/pointer.pass.cpp b/libcxx/test/std/strings/basic.string/string.modifiers/string_append/pointer.pass.cpp
index 68f5838ccf2f02a660232e15f6a4ff6e6fba7f4d..af334eb080c37ec9081ce57cf2324e48c45416ac 100644
--- a/libcxx/test/std/strings/basic.string/string.modifiers/string_append/pointer.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.modifiers/string_append/pointer.pass.cpp
@@ -16,12 +16,14 @@
 
 #include "test_macros.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class S>
 TEST_CONSTEXPR_CXX20 void test(S s, const typename S::value_type* str, S expected) {
   s.append(str);
   LIBCPP_ASSERT(s.__invariants());
   assert(s == expected);
+  LIBCPP_ASSERT(is_string_asan_correct(s));
 }
 
 template <class S>
@@ -43,6 +45,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
   test_string<std::string>();
 #if TEST_STD_VER >= 11
   test_string<std::basic_string<char, std::char_traits<char>, min_allocator<char>>>();
+  test_string<std::basic_string<char, std::char_traits<char>, safe_allocator<char>>>();
 #endif
 
   { // test appending to self
diff --git a/libcxx/test/std/strings/basic.string/string.modifiers/string_append/pointer_size.pass.cpp b/libcxx/test/std/strings/basic.string/string.modifiers/string_append/pointer_size.pass.cpp
index b98c91cf097b1a0b1a8114ed5cab427a9270b0cb..93e7500a11967ce8cdacd52ce5043eec6395f036 100644
--- a/libcxx/test/std/strings/basic.string/string.modifiers/string_append/pointer_size.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.modifiers/string_append/pointer_size.pass.cpp
@@ -17,12 +17,14 @@
 
 #include "test_macros.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class S>
 TEST_CONSTEXPR_CXX20 void test(S s, const typename S::value_type* str, typename S::size_type n, S expected) {
   s.append(str, n);
   LIBCPP_ASSERT(s.__invariants());
   assert(s == expected);
+  LIBCPP_ASSERT(is_string_asan_correct(s));
 }
 
 template <class S>
@@ -48,6 +50,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
   test_string<std::string>();
 #if TEST_STD_VER >= 11
   test_string<std::basic_string<char, std::char_traits<char>, min_allocator<char>>>();
+  test_string<std::basic_string<char, std::char_traits<char>, safe_allocator<char>>>();
 #endif
 
   { // test appending to self
diff --git a/libcxx/test/std/strings/basic.string/string.modifiers/string_append/push_back.pass.cpp b/libcxx/test/std/strings/basic.string/string.modifiers/string_append/push_back.pass.cpp
index 120de4fef7406281cb9386716d6140db5ff2e1c7..a352b4cf9b92e68b78b5f1a3cbb457875a19e652 100644
--- a/libcxx/test/std/strings/basic.string/string.modifiers/string_append/push_back.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.modifiers/string_append/push_back.pass.cpp
@@ -16,6 +16,7 @@
 
 #include "test_macros.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 struct VeryLarge {
   long long a;
@@ -61,6 +62,7 @@ TEST_CONSTEXPR_CXX20 void test(S s, typename S::value_type c, S expected) {
   s.push_back(c);
   LIBCPP_ASSERT(s.__invariants());
   assert(s == expected);
+  LIBCPP_ASSERT(is_string_asan_correct(s));
 }
 
 template <class S>
@@ -78,6 +80,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
   test_string<std::string>();
 #if TEST_STD_VER >= 11
   test_string<std::basic_string<char, std::char_traits<char>, min_allocator<char> > >();
+  test_string<std::basic_string<char, std::char_traits<char>, safe_allocator<char> > >();
 #endif
   {
     // https://llvm.org/PR31454
diff --git a/libcxx/test/std/strings/basic.string/string.modifiers/string_append/size_char.pass.cpp b/libcxx/test/std/strings/basic.string/string.modifiers/string_append/size_char.pass.cpp
index 86072efc09cd4790f4f4b39fbd11716f921ce3ac..a898efcf21855f51229aa3598d5a27b07f84882a 100644
--- a/libcxx/test/std/strings/basic.string/string.modifiers/string_append/size_char.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.modifiers/string_append/size_char.pass.cpp
@@ -16,12 +16,14 @@
 
 #include "test_macros.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class S>
 TEST_CONSTEXPR_CXX20 void test(S s, typename S::size_type n, typename S::value_type c, S expected) {
   s.append(n, c);
   LIBCPP_ASSERT(s.__invariants());
   assert(s == expected);
+  LIBCPP_ASSERT(is_string_asan_correct(s));
 }
 
 template <class S>
@@ -49,6 +51,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
   test_string<std::string>();
 #if TEST_STD_VER >= 11
   test_string<std::basic_string<char, std::char_traits<char>, min_allocator<char>>>();
+  test_string<std::basic_string<char, std::char_traits<char>, safe_allocator<char>>>();
 #endif
 
   return true;
diff --git a/libcxx/test/std/strings/basic.string/string.modifiers/string_append/string.pass.cpp b/libcxx/test/std/strings/basic.string/string.modifiers/string_append/string.pass.cpp
index 9d6237507e8a3c73a78b32fafb3f70e975011d83..66a2dc26341f849bd95a95f73ae244b2efde163c 100644
--- a/libcxx/test/std/strings/basic.string/string.modifiers/string_append/string.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.modifiers/string_append/string.pass.cpp
@@ -16,12 +16,14 @@
 
 #include "test_macros.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class S>
 TEST_CONSTEXPR_CXX20 void test(S s, S str, S expected) {
   s.append(str);
   LIBCPP_ASSERT(s.__invariants());
   assert(s == expected);
+  LIBCPP_ASSERT(is_string_asan_correct(s));
 }
 
 template <class S>
@@ -59,6 +61,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
   test_string<std::string>();
 #if TEST_STD_VER >= 11
   test_string<std::basic_string<char, std::char_traits<char>, min_allocator<char>>>();
+  test_string<std::basic_string<char, std::char_traits<char>, safe_allocator<char>>>();
 #endif
 
 #if TEST_STD_VER >= 11
diff --git a/libcxx/test/std/strings/basic.string/string.modifiers/string_append/string_size_size.pass.cpp b/libcxx/test/std/strings/basic.string/string.modifiers/string_append/string_size_size.pass.cpp
index 5b578a1a0bf8d93507a614d041fc55d5c01618e5..d30f13d96b5b87350ed9402241fce457b1645cf1 100644
--- a/libcxx/test/std/strings/basic.string/string.modifiers/string_append/string_size_size.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.modifiers/string_append/string_size_size.pass.cpp
@@ -18,6 +18,7 @@
 
 #include "test_macros.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class S>
 TEST_CONSTEXPR_CXX20 void test(S s, S str, typename S::size_type pos, typename S::size_type n, S expected) {
@@ -25,6 +26,7 @@ TEST_CONSTEXPR_CXX20 void test(S s, S str, typename S::size_type pos, typename S
     s.append(str, pos, n);
     LIBCPP_ASSERT(s.__invariants());
     assert(s == expected);
+    LIBCPP_ASSERT(is_string_asan_correct(s));
   }
 #ifndef TEST_HAS_NO_EXCEPTIONS
   else if (!TEST_IS_CONSTANT_EVALUATED) {
@@ -94,6 +96,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
   test_string<std::string>();
 #if TEST_STD_VER >= 11
   test_string<std::basic_string<char, std::char_traits<char>, min_allocator<char>>>();
+  test_string<std::basic_string<char, std::char_traits<char>, safe_allocator<char>>>();
 #endif
 
   {
diff --git a/libcxx/test/std/strings/basic.string/string.modifiers/string_assign/T_size_size.pass.cpp b/libcxx/test/std/strings/basic.string/string.modifiers/string_assign/T_size_size.pass.cpp
index a2a954cb94aac6a992830d95593fd1a48f063f59..b35f6b9497d03bc1dba7d04db9bc09e28a77f7b6 100644
--- a/libcxx/test/std/strings/basic.string/string.modifiers/string_assign/T_size_size.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.modifiers/string_assign/T_size_size.pass.cpp
@@ -17,6 +17,7 @@
 
 #include "test_macros.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class S, class SV>
 TEST_CONSTEXPR_CXX20 void test(S s, SV sv, typename S::size_type pos, typename S::size_type n, S expected) {
@@ -24,6 +25,7 @@ TEST_CONSTEXPR_CXX20 void test(S s, SV sv, typename S::size_type pos, typename S
     s.assign(sv, pos, n);
     LIBCPP_ASSERT(s.__invariants());
     assert(s == expected);
+    LIBCPP_ASSERT(is_string_asan_correct(s));
   }
 #ifndef TEST_HAS_NO_EXCEPTIONS
   else if (!TEST_IS_CONSTANT_EVALUATED) {
@@ -85,6 +87,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
   test_string<std::string>();
 #if TEST_STD_VER >= 11
   test_string<std::basic_string<char, std::char_traits<char>, min_allocator<char>>>();
+  test_string<std::basic_string<char, std::char_traits<char>, safe_allocator<char>>>();
 #endif
 
   {
diff --git a/libcxx/test/std/strings/basic.string/string.modifiers/string_assign/assign_range.pass.cpp b/libcxx/test/std/strings/basic.string/string.modifiers/string_assign/assign_range.pass.cpp
index d51326f3833046e784fe468ccc9f64b84dd36661..fd060b351a6cf7461dec82a32a8b3f71df76a518 100644
--- a/libcxx/test/std/strings/basic.string/string.modifiers/string_assign/assign_range.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.modifiers/string_assign/assign_range.pass.cpp
@@ -15,6 +15,7 @@
 
 #include "../../../../containers/sequences/insert_range_sequence_containers.h"
 #include "test_macros.h"
+#include "asan_testing.h"
 
 // Tested cases:
 // - different kinds of assignments (assigning an {empty/one-element/mid-sized/long range} to an
@@ -43,9 +44,13 @@ int main(int, char**) {
     std::string c;
     static_assert(std::is_lvalue_reference_v<decltype(c.assign_range(FullContainer_Begin_EmptyRange<char>.input))>);
     assert(&c.assign_range(FullContainer_Begin_EmptyRange<char>.input) == &c);
+    LIBCPP_ASSERT(is_string_asan_correct(c));
     assert(&c.assign_range(FullContainer_Begin_OneElementRange<char>.input) == &c);
+    LIBCPP_ASSERT(is_string_asan_correct(c));
     assert(&c.assign_range(FullContainer_Begin_MidRange<char>.input) == &c);
+    LIBCPP_ASSERT(is_string_asan_correct(c));
     assert(&c.assign_range(FullContainer_Begin_LongRange<char>.input) == &c);
+    LIBCPP_ASSERT(is_string_asan_correct(c));
   }
 
   // Note: `test_assign_range_exception_safety_throwing_copy` doesn't apply because copying chars cannot throw.
diff --git a/libcxx/test/std/strings/basic.string/string.modifiers/string_assign/initializer_list.pass.cpp b/libcxx/test/std/strings/basic.string/string.modifiers/string_assign/initializer_list.pass.cpp
index ba55f30785e94f46cd1248ef3272ffc9722dfb25..47a59e8997e62eb773ed1c1bb4da468786f9ee11 100644
--- a/libcxx/test/std/strings/basic.string/string.modifiers/string_assign/initializer_list.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.modifiers/string_assign/initializer_list.pass.cpp
@@ -17,18 +17,31 @@
 
 #include "test_macros.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class S>
 TEST_CONSTEXPR_CXX20 void test_string() {
   S s("123");
+
+  s.assign({'a', 'b'});
+  assert(s == "ab");
+  LIBCPP_ASSERT(is_string_asan_correct(s));
+
   s.assign({'a', 'b', 'c'});
   assert(s == "abc");
+  LIBCPP_ASSERT(is_string_asan_correct(s));
+
+  s.assign({'a', 'a', 'a', 'a', 'a', 'a', 'a', 'a', 'a', 'a', 'a', 'a', 'a',
+            'a', 'a', 'a', 'a', 'a', 'a', 'a', 'a', 'a', 'a', 'a', 'a'});
+  assert(s == "aaaaaaaaaaaaaaaaaaaaaaaaa");
+  LIBCPP_ASSERT(is_string_asan_correct(s));
 }
 
 TEST_CONSTEXPR_CXX20 bool test() {
   test_string<std::string>();
 #if TEST_STD_VER >= 11
   test_string<std::basic_string<char, std::char_traits<char>, min_allocator<char> > >();
+  test_string<std::basic_string<char, std::char_traits<char>, safe_allocator<char> > >();
 #endif
 
   return true;
diff --git a/libcxx/test/std/strings/basic.string/string.modifiers/string_assign/iterator.pass.cpp b/libcxx/test/std/strings/basic.string/string.modifiers/string_assign/iterator.pass.cpp
index b2ca4fb5ddd937099d8f2f82d8b568d103126ad3..c183def9ee75fee3f2db1a8b3f48c5923a4a432f 100644
--- a/libcxx/test/std/strings/basic.string/string.modifiers/string_assign/iterator.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.modifiers/string_assign/iterator.pass.cpp
@@ -17,12 +17,14 @@
 #include "test_macros.h"
 #include "test_iterators.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class S, class It>
 TEST_CONSTEXPR_CXX20 void test(S s, It first, It last, S expected) {
   s.assign(first, last);
   LIBCPP_ASSERT(s.__invariants());
   assert(s == expected);
+  LIBCPP_ASSERT(is_string_asan_correct(s));
 }
 
 #ifndef TEST_HAS_NO_EXCEPTIONS
@@ -171,6 +173,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
   test_string<std::string>();
 #if TEST_STD_VER >= 11
   test_string<std::basic_string<char, std::char_traits<char>, min_allocator<char> > >();
+  test_string<std::basic_string<char, std::char_traits<char>, safe_allocator<char> > >();
 #endif
 
   return true;
diff --git a/libcxx/test/std/strings/basic.string/string.modifiers/string_assign/pointer.pass.cpp b/libcxx/test/std/strings/basic.string/string.modifiers/string_assign/pointer.pass.cpp
index 175062c91a8835b616d1709fb92dc7971881d5cc..0d5c269d30df47fea19754a3860b831e6ef18491 100644
--- a/libcxx/test/std/strings/basic.string/string.modifiers/string_assign/pointer.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.modifiers/string_assign/pointer.pass.cpp
@@ -16,12 +16,14 @@
 
 #include "test_macros.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class S>
 TEST_CONSTEXPR_CXX20 void test(S s, const typename S::value_type* str, S expected) {
   s.assign(str);
   LIBCPP_ASSERT(s.__invariants());
   assert(s == expected);
+  LIBCPP_ASSERT(is_string_asan_correct(s));
 }
 
 template <class S>
@@ -48,6 +50,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
   test_string<std::string>();
 #if TEST_STD_VER >= 11
   test_string<std::basic_string<char, std::char_traits<char>, min_allocator<char>>>();
+  test_string<std::basic_string<char, std::char_traits<char>, safe_allocator<char>>>();
 #endif
 
   { // test assignment to self
diff --git a/libcxx/test/std/strings/basic.string/string.modifiers/string_assign/pointer_size.pass.cpp b/libcxx/test/std/strings/basic.string/string.modifiers/string_assign/pointer_size.pass.cpp
index 851f016829c3f16426a4020539d4fd2246f6776d..1f40898b71bfeb7953ed3463e0598415750e78f7 100644
--- a/libcxx/test/std/strings/basic.string/string.modifiers/string_assign/pointer_size.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.modifiers/string_assign/pointer_size.pass.cpp
@@ -17,12 +17,14 @@
 
 #include "test_macros.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class S>
 TEST_CONSTEXPR_CXX20 void test(S s, const typename S::value_type* str, typename S::size_type n, S expected) {
   s.assign(str, n);
   LIBCPP_ASSERT(s.__invariants());
   assert(s == expected);
+  LIBCPP_ASSERT(is_string_asan_correct(s));
 }
 
 template <class S>
@@ -54,6 +56,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
   test_string<std::string>();
 #if TEST_STD_VER >= 11
   test_string<std::basic_string<char, std::char_traits<char>, min_allocator<char>>>();
+  test_string<std::basic_string<char, std::char_traits<char>, safe_allocator<char>>>();
 #endif
 
   { // test assign to self
diff --git a/libcxx/test/std/strings/basic.string/string.modifiers/string_assign/size_char.pass.cpp b/libcxx/test/std/strings/basic.string/string.modifiers/string_assign/size_char.pass.cpp
index b52c66103395f19a84a9b10f8823ff93fe60b425..3c7651135f1aa11881cd2cd00d092332167403c3 100644
--- a/libcxx/test/std/strings/basic.string/string.modifiers/string_assign/size_char.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.modifiers/string_assign/size_char.pass.cpp
@@ -16,12 +16,14 @@
 
 #include "test_macros.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class S>
 TEST_CONSTEXPR_CXX20 void test(S s, typename S::size_type n, typename S::value_type c, S expected) {
   s.assign(n, c);
   LIBCPP_ASSERT(s.__invariants());
   assert(s == expected);
+  LIBCPP_ASSERT(is_string_asan_correct(s));
 }
 
 template <class S>
@@ -44,6 +46,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
   test_string<std::string>();
 #if TEST_STD_VER >= 11
   test_string<std::basic_string<char, std::char_traits<char>, min_allocator<char>>>();
+  test_string<std::basic_string<char, std::char_traits<char>, safe_allocator<char>>>();
 #endif
 
   return true;
diff --git a/libcxx/test/std/strings/basic.string/string.modifiers/string_assign/string.pass.cpp b/libcxx/test/std/strings/basic.string/string.modifiers/string_assign/string.pass.cpp
index f39b7f66c4b187a46ee23781ad7f943964117505..8b310630bf07ab8a6221853d1f134385de75d3f1 100644
--- a/libcxx/test/std/strings/basic.string/string.modifiers/string_assign/string.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.modifiers/string_assign/string.pass.cpp
@@ -18,12 +18,15 @@
 #include "nasty_string.h"
 #include "min_allocator.h"
 #include "test_allocator.h"
+#include "asan_testing.h"
 
 template <class S>
 TEST_CONSTEXPR_CXX20 void test(S dest, S src) {
   dest.assign(src);
   LIBCPP_ASSERT(dest.__invariants());
   assert(dest == src);
+  LIBCPP_ASSERT(is_string_asan_correct(src));
+  LIBCPP_ASSERT(is_string_asan_correct(dest));
 }
 
 template <class S>
diff --git a/libcxx/test/std/strings/basic.string/string.modifiers/string_assign/string_size_size.pass.cpp b/libcxx/test/std/strings/basic.string/string.modifiers/string_assign/string_size_size.pass.cpp
index 9448452bafad9c36fba26925ec6b1a41d0ff3004..1cd2368b2a6b3f2cb4c25f3199956e819f014c9e 100644
--- a/libcxx/test/std/strings/basic.string/string.modifiers/string_assign/string_size_size.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.modifiers/string_assign/string_size_size.pass.cpp
@@ -18,6 +18,7 @@
 
 #include "test_macros.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class S>
 TEST_CONSTEXPR_CXX20 void test(S s, S str, typename S::size_type pos, typename S::size_type n, S expected) {
@@ -25,6 +26,7 @@ TEST_CONSTEXPR_CXX20 void test(S s, S str, typename S::size_type pos, typename S
     s.assign(str, pos, n);
     LIBCPP_ASSERT(s.__invariants());
     assert(s == expected);
+    LIBCPP_ASSERT(is_string_asan_correct(s));
   }
 #ifndef TEST_HAS_NO_EXCEPTIONS
   else if (!TEST_IS_CONSTANT_EVALUATED) {
@@ -85,6 +87,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
   test_string<std::string>();
 #if TEST_STD_VER >= 11
   test_string<std::basic_string<char, std::char_traits<char>, min_allocator<char>>>();
+  test_string<std::basic_string<char, std::char_traits<char>, safe_allocator<char>>>();
 #endif
 
   {
diff --git a/libcxx/test/std/strings/basic.string/string.modifiers/string_copy/copy.pass.cpp b/libcxx/test/std/strings/basic.string/string.modifiers/string_copy/copy.pass.cpp
index 1bd686c5edbf84f758a033e7f3be7c6bd56947da..0d7221ced8034f1cde85351d358c9aa253e586a2 100644
--- a/libcxx/test/std/strings/basic.string/string.modifiers/string_copy/copy.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.modifiers/string_copy/copy.pass.cpp
@@ -17,6 +17,7 @@
 
 #include "test_macros.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class S>
 TEST_CONSTEXPR_CXX20 void test(S str, typename S::value_type* s, typename S::size_type n, typename S::size_type pos) {
@@ -25,6 +26,9 @@ TEST_CONSTEXPR_CXX20 void test(S str, typename S::value_type* s, typename S::siz
     typename S::size_type r    = cs.copy(s, n, pos);
     typename S::size_type rlen = std::min(n, cs.size() - pos);
     assert(r == rlen);
+    LIBCPP_ASSERT(is_string_asan_correct(str));
+    LIBCPP_ASSERT(is_string_asan_correct(cs));
+
     for (r = 0; r < rlen; ++r)
       assert(S::traits_type::eq(cs[pos + r], s[r]));
   }
@@ -110,6 +114,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
   test_string<std::string>();
 #if TEST_STD_VER >= 11
   test_string<std::basic_string<char, std::char_traits<char>, min_allocator<char>>>();
+  test_string<std::basic_string<char, std::char_traits<char>, safe_allocator<char>>>();
 #endif
 
   return true;
diff --git a/libcxx/test/std/strings/basic.string/string.modifiers/string_erase/iter.pass.cpp b/libcxx/test/std/strings/basic.string/string.modifiers/string_erase/iter.pass.cpp
index 03c5b965537c6fe3f5448ff89fcf234587a54758..b724973150296b5bc089fd3b5421eaf045a49c10 100644
--- a/libcxx/test/std/strings/basic.string/string.modifiers/string_erase/iter.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.modifiers/string_erase/iter.pass.cpp
@@ -15,6 +15,7 @@
 
 #include "test_macros.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class S>
 TEST_CONSTEXPR_CXX20 void test(S s, typename S::difference_type pos, S expected) {
@@ -24,6 +25,7 @@ TEST_CONSTEXPR_CXX20 void test(S s, typename S::difference_type pos, S expected)
   assert(s[s.size()] == typename S::value_type());
   assert(s == expected);
   assert(i - s.begin() == pos);
+  LIBCPP_ASSERT(is_string_asan_correct(s));
 }
 
 template <class S>
@@ -46,6 +48,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
   test_string<std::string>();
 #if TEST_STD_VER >= 11
   test_string<std::basic_string<char, std::char_traits<char>, min_allocator<char>>>();
+  test_string<std::basic_string<char, std::char_traits<char>, safe_allocator<char>>>();
 #endif
 
   return true;
diff --git a/libcxx/test/std/strings/basic.string/string.modifiers/string_erase/iter_iter.pass.cpp b/libcxx/test/std/strings/basic.string/string.modifiers/string_erase/iter_iter.pass.cpp
index f092ad718cdaa89f44fa051dbf78ab04c5502120..5270335aa9ff3dd38d1b945beb835b7636b475c2 100644
--- a/libcxx/test/std/strings/basic.string/string.modifiers/string_erase/iter_iter.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.modifiers/string_erase/iter_iter.pass.cpp
@@ -15,6 +15,7 @@
 
 #include "test_macros.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class S>
 TEST_CONSTEXPR_CXX20 void test(S s, typename S::difference_type pos, typename S::difference_type n, S expected) {
@@ -25,6 +26,7 @@ TEST_CONSTEXPR_CXX20 void test(S s, typename S::difference_type pos, typename S:
   assert(s[s.size()] == typename S::value_type());
   assert(s == expected);
   assert(i - s.begin() == pos);
+  LIBCPP_ASSERT(is_string_asan_correct(s));
 }
 
 template <class S>
@@ -89,6 +91,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
   test_string<std::string>();
 #if TEST_STD_VER >= 11
   test_string<std::basic_string<char, std::char_traits<char>, min_allocator<char>>>();
+  test_string<std::basic_string<char, std::char_traits<char>, safe_allocator<char>>>();
 #endif
 
   return true;
diff --git a/libcxx/test/std/strings/basic.string/string.modifiers/string_erase/pop_back.pass.cpp b/libcxx/test/std/strings/basic.string/string.modifiers/string_erase/pop_back.pass.cpp
index 9c63fa47a7e4a7f4155c0fc70644bf6aba20fd13..f52512231e094d542c67a55fd7a286fb78877c14 100644
--- a/libcxx/test/std/strings/basic.string/string.modifiers/string_erase/pop_back.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.modifiers/string_erase/pop_back.pass.cpp
@@ -15,6 +15,7 @@
 
 #include "test_macros.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class S>
 TEST_CONSTEXPR_CXX20 void test(S s, S expected) {
@@ -22,6 +23,7 @@ TEST_CONSTEXPR_CXX20 void test(S s, S expected) {
   LIBCPP_ASSERT(s.__invariants());
   assert(s[s.size()] == typename S::value_type());
   assert(s == expected);
+  LIBCPP_ASSERT(is_string_asan_correct(s));
 }
 
 template <class S>
@@ -29,12 +31,15 @@ TEST_CONSTEXPR_CXX20 void test_string() {
   test(S("abcde"), S("abcd"));
   test(S("abcdefghij"), S("abcdefghi"));
   test(S("abcdefghijklmnopqrst"), S("abcdefghijklmnopqrs"));
+  test(S("abcdefghijklmnopqrstabcdefghijklmnopqrstabcdefghijklmnopqrstabcdefghijklmnopqrst"),
+       S("abcdefghijklmnopqrstabcdefghijklmnopqrstabcdefghijklmnopqrstabcdefghijklmnopqrs"));
 }
 
 TEST_CONSTEXPR_CXX20 bool test() {
   test_string<std::string>();
 #if TEST_STD_VER >= 11
   test_string<std::basic_string<char, std::char_traits<char>, min_allocator<char>>>();
+  test_string<std::basic_string<char, std::char_traits<char>, safe_allocator<char>>>();
 #endif
 
   return true;
diff --git a/libcxx/test/std/strings/basic.string/string.modifiers/string_erase/size_size.pass.cpp b/libcxx/test/std/strings/basic.string/string.modifiers/string_erase/size_size.pass.cpp
index 610256061a5f2cd7c16596888b25b171ae45172b..97acffd484e646c62e39ee6c3f5d9c16b8d9eb1e 100644
--- a/libcxx/test/std/strings/basic.string/string.modifiers/string_erase/size_size.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.modifiers/string_erase/size_size.pass.cpp
@@ -17,6 +17,7 @@
 
 #include "test_macros.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class S>
 TEST_CONSTEXPR_CXX20 void test(S s, typename S::size_type pos, typename S::size_type n, S expected) {
@@ -27,6 +28,7 @@ TEST_CONSTEXPR_CXX20 void test(S s, typename S::size_type pos, typename S::size_
     LIBCPP_ASSERT(s.__invariants());
     assert(s[s.size()] == typename S::value_type());
     assert(s == expected);
+    LIBCPP_ASSERT(is_string_asan_correct(s));
   }
 #ifndef TEST_HAS_NO_EXCEPTIONS
   else if (!TEST_IS_CONSTANT_EVALUATED) {
@@ -180,6 +182,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
   test_string<std::string>();
 #if TEST_STD_VER >= 11
   test_string<std::basic_string<char, std::char_traits<char>, min_allocator<char>>>();
+  test_string<std::basic_string<char, std::char_traits<char>, safe_allocator<char>>>();
 #endif
 
   return true;
diff --git a/libcxx/test/std/strings/basic.string/string.modifiers/string_insert/iter_char.pass.cpp b/libcxx/test/std/strings/basic.string/string.modifiers/string_insert/iter_char.pass.cpp
index eb1c10b070a2cfe5bbf9c2f7131ed5f05f67ca8f..1d8c7e06ed8c363da56a99c580bdc06a96bd04a0 100644
--- a/libcxx/test/std/strings/basic.string/string.modifiers/string_insert/iter_char.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.modifiers/string_insert/iter_char.pass.cpp
@@ -16,9 +16,11 @@
 
 #include "test_macros.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class S>
 TEST_CONSTEXPR_CXX20 void test(S& s, typename S::const_iterator p, typename S::value_type c, S expected) {
+  LIBCPP_ASSERT(is_string_asan_correct(s));
   bool sufficient_cap             = s.size() < s.capacity();
   typename S::difference_type pos = p - s.begin();
   typename S::iterator i          = s.insert(p, c);
@@ -28,6 +30,7 @@ TEST_CONSTEXPR_CXX20 void test(S& s, typename S::const_iterator p, typename S::v
   assert(*i == c);
   if (sufficient_cap)
     assert(i == p);
+  LIBCPP_ASSERT(is_string_asan_correct(s));
 }
 
 template <class S>
@@ -70,6 +73,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
   test_string<std::string>();
 #if TEST_STD_VER >= 11
   test_string<std::basic_string<char, std::char_traits<char>, min_allocator<char>>>();
+  test_string<std::basic_string<char, std::char_traits<char>, safe_allocator<char>>>();
 #endif
 
   return true;
diff --git a/libcxx/test/std/strings/basic.string/string.modifiers/string_insert/iter_initializer_list.pass.cpp b/libcxx/test/std/strings/basic.string/string.modifiers/string_insert/iter_initializer_list.pass.cpp
index 876cc9ecd14c6a8f78dda4eced4d91261ba1a29a..769bd7c4274ff4cd081d055edbe60b093404af01 100644
--- a/libcxx/test/std/strings/basic.string/string.modifiers/string_insert/iter_initializer_list.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.modifiers/string_insert/iter_initializer_list.pass.cpp
@@ -17,6 +17,7 @@
 
 #include "test_macros.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class S>
 TEST_CONSTEXPR_CXX20 void test_string() {
@@ -24,11 +25,18 @@ TEST_CONSTEXPR_CXX20 void test_string() {
   typename S::iterator i = s.insert(s.begin() + 3, {'a', 'b', 'c'});
   assert(i - s.begin() == 3);
   assert(s == "123abc456");
+  LIBCPP_ASSERT(is_string_asan_correct(s));
+  typename S::iterator j = s.insert(s.begin() + 6, {'a', 'a', 'a', 'a', 'a', 'a', 'a', 'a', 'a', 'a', 'a', 'a', 'a',
+                                                    'a', 'a', 'a', 'a', 'a', 'a', 'a', 'a', 'a', 'a', 'a', 'a'});
+  assert(j - s.begin() == 6);
+  assert(s == "123abcaaaaaaaaaaaaaaaaaaaaaaaaa456");
+  LIBCPP_ASSERT(is_string_asan_correct(s));
 }
 
 TEST_CONSTEXPR_CXX20 bool test() {
   test_string<std::string>();
   test_string<std::basic_string<char, std::char_traits<char>, min_allocator<char> > >();
+  test_string<std::basic_string<char, std::char_traits<char>, safe_allocator<char> > >();
 
   return true;
 }
diff --git a/libcxx/test/std/strings/basic.string/string.modifiers/string_insert/iter_iter_iter.infinite_recursion.pass.cpp b/libcxx/test/std/strings/basic.string/string.modifiers/string_insert/iter_iter_iter.infinite_recursion.pass.cpp
index e40528e516531bb8f6d488ddd38f673509d99247..b7b70fa9118c67e65c5e45b703740b48b7facc1c 100644
--- a/libcxx/test/std/strings/basic.string/string.modifiers/string_insert/iter_iter_iter.infinite_recursion.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.modifiers/string_insert/iter_iter_iter.infinite_recursion.pass.cpp
@@ -16,6 +16,7 @@
 #include <string>
 
 #include "test_macros.h"
+#include "asan_testing.h"
 
 struct char_ascii {
   char char_;
@@ -95,5 +96,6 @@ int main(int, char**) {
   str.insert(str.begin(), &ch, &ch + 1);
   assert(str.size() == 1);
   assert(str[0].char_ == 'A');
+  LIBCPP_ASSERT(is_string_asan_correct(str));
   return 0;
 }
diff --git a/libcxx/test/std/strings/basic.string/string.modifiers/string_insert/iter_iter_iter.pass.cpp b/libcxx/test/std/strings/basic.string/string.modifiers/string_insert/iter_iter_iter.pass.cpp
index e3d0f1322e85de2b255ca36c369604f255f62a9e..060216e7a82b306c7bb0936cbf0b7f321fbbe3b1 100644
--- a/libcxx/test/std/strings/basic.string/string.modifiers/string_insert/iter_iter_iter.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.modifiers/string_insert/iter_iter_iter.pass.cpp
@@ -17,6 +17,7 @@
 #include "test_macros.h"
 #include "test_iterators.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class S, class It>
 TEST_CONSTEXPR_CXX20 void test(S s, typename S::difference_type pos, It first, It last, S expected) {
@@ -25,6 +26,7 @@ TEST_CONSTEXPR_CXX20 void test(S s, typename S::difference_type pos, It first, I
   LIBCPP_ASSERT(s.__invariants());
   assert(i - s.begin() == pos);
   assert(s == expected);
+  LIBCPP_ASSERT(is_string_asan_correct(s));
 }
 
 #ifndef TEST_HAS_NO_EXCEPTIONS
@@ -147,6 +149,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
   test_string<std::string>();
 #if TEST_STD_VER >= 11
   test_string<std::basic_string<char, std::char_traits<char>, min_allocator<char>>>();
+  test_string<std::basic_string<char, std::char_traits<char>, min_allocator<char>>>();
 #endif
 
 #ifndef TEST_HAS_NO_EXCEPTIONS
diff --git a/libcxx/test/std/strings/basic.string/string.modifiers/string_insert/iter_size_char.pass.cpp b/libcxx/test/std/strings/basic.string/string.modifiers/string_insert/iter_size_char.pass.cpp
index 1e0d16e4cc2e172e087aa756f04cdb986b580d7f..6937ccc0d540a475bd4b581036f992154b2c8601 100644
--- a/libcxx/test/std/strings/basic.string/string.modifiers/string_insert/iter_size_char.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.modifiers/string_insert/iter_size_char.pass.cpp
@@ -15,6 +15,7 @@
 
 #include "test_macros.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class S>
 TEST_CONSTEXPR_CXX20 void
@@ -24,6 +25,7 @@ test(S s, typename S::difference_type pos, typename S::size_type n, typename S::
   LIBCPP_ASSERT(s.__invariants());
   assert(i - s.begin() == pos);
   assert(s == expected);
+  LIBCPP_ASSERT(is_string_asan_correct(s));
 }
 
 template <class S>
@@ -98,6 +100,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
   test_string<std::string>();
 #if TEST_STD_VER >= 11
   test_string<std::basic_string<char, std::char_traits<char>, min_allocator<char>>>();
+  test_string<std::basic_string<char, std::char_traits<char>, safe_allocator<char>>>();
 #endif
 
   return true;
diff --git a/libcxx/test/std/strings/basic.string/string.modifiers/string_insert/size_pointer.pass.cpp b/libcxx/test/std/strings/basic.string/string.modifiers/string_insert/size_pointer.pass.cpp
index 359fd999598f8890d5a0cd981bbf7d4b1360cbf2..9e2d00279b96d655d7b633654c26c49b40413399 100644
--- a/libcxx/test/std/strings/basic.string/string.modifiers/string_insert/size_pointer.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.modifiers/string_insert/size_pointer.pass.cpp
@@ -17,6 +17,7 @@
 
 #include "test_macros.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class S>
 TEST_CONSTEXPR_CXX20 void test(S s, typename S::size_type pos, const typename S::value_type* str, S expected) {
@@ -26,6 +27,7 @@ TEST_CONSTEXPR_CXX20 void test(S s, typename S::size_type pos, const typename S:
     s.insert(pos, str);
     LIBCPP_ASSERT(s.__invariants());
     assert(s == expected);
+    LIBCPP_ASSERT(is_string_asan_correct(s));
   }
 #ifndef TEST_HAS_NO_EXCEPTIONS
   else if (!TEST_IS_CONSTANT_EVALUATED) {
@@ -128,6 +130,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
   test_string<std::string>();
 #if TEST_STD_VER >= 11
   test_string<std::basic_string<char, std::char_traits<char>, min_allocator<char>>>();
+  test_string<std::basic_string<char, std::char_traits<char>, safe_allocator<char>>>();
 #endif
 
   { // test inserting into self
diff --git a/libcxx/test/std/strings/basic.string/string.modifiers/string_insert/size_pointer_size.pass.cpp b/libcxx/test/std/strings/basic.string/string.modifiers/string_insert/size_pointer_size.pass.cpp
index 44f20ff63cc0ca5c00b384c2d87c31874f96cab1..3e581b56a63901996e2ebc6e3d2fbd6e041d0dcb 100644
--- a/libcxx/test/std/strings/basic.string/string.modifiers/string_insert/size_pointer_size.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.modifiers/string_insert/size_pointer_size.pass.cpp
@@ -17,6 +17,7 @@
 
 #include "test_macros.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class S>
 TEST_CONSTEXPR_CXX20 void
@@ -27,6 +28,7 @@ test(S s, typename S::size_type pos, const typename S::value_type* str, typename
     s.insert(pos, str, n);
     LIBCPP_ASSERT(s.__invariants());
     assert(s == expected);
+    LIBCPP_ASSERT(is_string_asan_correct(s));
   }
 #ifndef TEST_HAS_NO_EXCEPTIONS
   else if (!TEST_IS_CONSTANT_EVALUATED) {
@@ -388,6 +390,7 @@ int main(int, char**) {
   test<std::string>();
 #if TEST_STD_VER >= 11
   test<std::basic_string<char, std::char_traits<char>, min_allocator<char>>>();
+  test<std::basic_string<char, std::char_traits<char>, safe_allocator<char>>>();
 #endif
 
 #if TEST_STD_VER > 17
diff --git a/libcxx/test/std/strings/basic.string/string.modifiers/string_insert/size_size_char.pass.cpp b/libcxx/test/std/strings/basic.string/string.modifiers/string_insert/size_size_char.pass.cpp
index 2396b7e2f54df7324e912497d3bfe4b6372a142c..f18ef484b5a0475bea920536cae913f1bb2e5051 100644
--- a/libcxx/test/std/strings/basic.string/string.modifiers/string_insert/size_size_char.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.modifiers/string_insert/size_size_char.pass.cpp
@@ -17,6 +17,7 @@
 
 #include "test_macros.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class S>
 TEST_CONSTEXPR_CXX20 void
@@ -27,6 +28,7 @@ test(S s, typename S::size_type pos, typename S::size_type n, typename S::value_
     s.insert(pos, n, str);
     LIBCPP_ASSERT(s.__invariants());
     assert(s == expected);
+    LIBCPP_ASSERT(is_string_asan_correct(s));
   }
 #ifndef TEST_HAS_NO_EXCEPTIONS
   else if (!TEST_IS_CONSTANT_EVALUATED) {
@@ -129,6 +131,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
   test_string<std::string>();
 #if TEST_STD_VER >= 11
   test_string<std::basic_string<char, std::char_traits<char>, min_allocator<char>>>();
+  test_string<std::basic_string<char, std::char_traits<char>, safe_allocator<char>>>();
 #endif
 
   return true;
diff --git a/libcxx/test/std/strings/basic.string/string.modifiers/string_insert/size_string.pass.cpp b/libcxx/test/std/strings/basic.string/string.modifiers/string_insert/size_string.pass.cpp
index 1b908ac464c31342bc85c8de5d264cab65473338..49870ba05e35726c9a209ac464ab2ed8000d5114 100644
--- a/libcxx/test/std/strings/basic.string/string.modifiers/string_insert/size_string.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.modifiers/string_insert/size_string.pass.cpp
@@ -17,6 +17,7 @@
 
 #include "test_macros.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class S>
 TEST_CONSTEXPR_CXX20 void test(S s, typename S::size_type pos, S str, S expected) {
@@ -26,6 +27,7 @@ TEST_CONSTEXPR_CXX20 void test(S s, typename S::size_type pos, S str, S expected
     s.insert(pos, str);
     LIBCPP_ASSERT(s.__invariants());
     assert(s == expected);
+    LIBCPP_ASSERT(is_string_asan_correct(s));
   }
 #ifndef TEST_HAS_NO_EXCEPTIONS
   else if (!TEST_IS_CONSTANT_EVALUATED) {
@@ -128,6 +130,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
   test_string<std::string>();
 #if TEST_STD_VER >= 11
   test_string<std::basic_string<char, std::char_traits<char>, min_allocator<char>>>();
+  test_string<std::basic_string<char, std::char_traits<char>, safe_allocator<char>>>();
 #endif
 
 #if TEST_STD_VER >= 11
diff --git a/libcxx/test/std/strings/basic.string/string.modifiers/string_insert/size_string_size_size.pass.cpp b/libcxx/test/std/strings/basic.string/string.modifiers/string_insert/size_string_size_size.pass.cpp
index 1a8f6b35b42c445bfe2a2edd6423e212d5adbbb0..2c0579485d0fb54ffcb75882a2027bf5ce02f514 100644
--- a/libcxx/test/std/strings/basic.string/string.modifiers/string_insert/size_string_size_size.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.modifiers/string_insert/size_string_size_size.pass.cpp
@@ -19,6 +19,7 @@
 
 #include "test_macros.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class S>
 TEST_CONSTEXPR_CXX20 void
@@ -29,6 +30,7 @@ test(S s, typename S::size_type pos1, S str, typename S::size_type pos2, typenam
     s.insert(pos1, str, pos2, n);
     LIBCPP_ASSERT(s.__invariants());
     assert(s == expected);
+    LIBCPP_ASSERT(is_string_asan_correct(s));
   }
 #ifndef TEST_HAS_NO_EXCEPTIONS
   else if (!TEST_IS_CONSTANT_EVALUATED) {
@@ -1816,6 +1818,7 @@ int main(int, char**) {
   test<std::string>();
 #if TEST_STD_VER >= 11
   test<std::basic_string<char, std::char_traits<char>, min_allocator<char>>>();
+  test<std::basic_string<char, std::char_traits<char>, safe_allocator<char>>>();
 #endif
 
   return 0;
diff --git a/libcxx/test/std/strings/basic.string/string.modifiers/string_op_plus_equal/char.pass.cpp b/libcxx/test/std/strings/basic.string/string.modifiers/string_op_plus_equal/char.pass.cpp
index 3d5e2c0cbf6995335fcdcd82ad128748ee418e00..753c38fa232b3a8912dafac19ad9039ad753bd6a 100644
--- a/libcxx/test/std/strings/basic.string/string.modifiers/string_op_plus_equal/char.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.modifiers/string_op_plus_equal/char.pass.cpp
@@ -15,12 +15,14 @@
 
 #include "test_macros.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class S>
 TEST_CONSTEXPR_CXX20 void test(S s, typename S::value_type str, S expected) {
   s += str;
   LIBCPP_ASSERT(s.__invariants());
   assert(s == expected);
+  LIBCPP_ASSERT(is_string_asan_correct(s));
 }
 
 template <class S>
@@ -36,6 +38,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
   test_string<std::string>();
 #if TEST_STD_VER >= 11
   test_string<std::basic_string<char, std::char_traits<char>, min_allocator<char> > >();
+  test_string<std::basic_string<char, std::char_traits<char>, safe_allocator<char> > >();
 #endif
 
   return true;
diff --git a/libcxx/test/std/strings/basic.string/string.modifiers/string_op_plus_equal/initializer_list.pass.cpp b/libcxx/test/std/strings/basic.string/string.modifiers/string_op_plus_equal/initializer_list.pass.cpp
index e7334b96276a5f8261ea101a8bb1dc372a3c8c12..165fb2f96313b33da5376016281976a4afdfe6e8 100644
--- a/libcxx/test/std/strings/basic.string/string.modifiers/string_op_plus_equal/initializer_list.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.modifiers/string_op_plus_equal/initializer_list.pass.cpp
@@ -17,17 +17,34 @@
 
 #include "test_macros.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class S>
 TEST_CONSTEXPR_CXX20 void test_string() {
   S s("123");
   s += {'a', 'b', 'c'};
   assert(s == "123abc");
+  LIBCPP_ASSERT(is_string_asan_correct(s));
+
+  s += {'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'};
+  assert(s == "123abcxxxxxxxxxxxxxxxx");
+  LIBCPP_ASSERT(is_string_asan_correct(s));
+
+  s += {'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'};
+  assert(s == "123abcxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx");
+  LIBCPP_ASSERT(is_string_asan_correct(s));
+
+  s += {'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x',
+        'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x',
+        'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x'};
+  assert(s == "123abcxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx");
+  LIBCPP_ASSERT(is_string_asan_correct(s));
 }
 
 TEST_CONSTEXPR_CXX20 bool test() {
   test_string<std::string>();
   test_string<std::basic_string<char, std::char_traits<char>, min_allocator<char> > >();
+  test_string<std::basic_string<char, std::char_traits<char>, safe_allocator<char> > >();
   return true;
 }
 
diff --git a/libcxx/test/std/strings/basic.string/string.modifiers/string_op_plus_equal/pointer.pass.cpp b/libcxx/test/std/strings/basic.string/string.modifiers/string_op_plus_equal/pointer.pass.cpp
index ba14b3831778f5c2b505ec277679e9b1aa210c54..4231b0c6cd8d625ce3a50362019181df25a584de 100644
--- a/libcxx/test/std/strings/basic.string/string.modifiers/string_op_plus_equal/pointer.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.modifiers/string_op_plus_equal/pointer.pass.cpp
@@ -15,12 +15,14 @@
 
 #include "test_macros.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class S>
 TEST_CONSTEXPR_CXX20 void test(S s, const typename S::value_type* str, S expected) {
   s += str;
   LIBCPP_ASSERT(s.__invariants());
   assert(s == expected);
+  LIBCPP_ASSERT(is_string_asan_correct(s));
 }
 
 template <class S>
@@ -60,6 +62,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
   test_string<std::string>();
 #if TEST_STD_VER >= 11
   test_string<std::basic_string<char, std::char_traits<char>, min_allocator<char>>>();
+  test_string<std::basic_string<char, std::char_traits<char>, safe_allocator<char>>>();
 #endif
 
   return true;
diff --git a/libcxx/test/std/strings/basic.string/string.modifiers/string_op_plus_equal/string.pass.cpp b/libcxx/test/std/strings/basic.string/string.modifiers/string_op_plus_equal/string.pass.cpp
index 59c76ece67c66a50ec4ed32d214c4bf6274f4028..0aee67f27fb0561f9eceee9ffd05072cffe72d9e 100644
--- a/libcxx/test/std/strings/basic.string/string.modifiers/string_op_plus_equal/string.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.modifiers/string_op_plus_equal/string.pass.cpp
@@ -16,12 +16,14 @@
 
 #include "test_macros.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class S>
 TEST_CONSTEXPR_CXX20 void test(S s, S str, S expected) {
   s += str;
   LIBCPP_ASSERT(s.__invariants());
   assert(s == expected);
+  LIBCPP_ASSERT(is_string_asan_correct(s));
 }
 
 template <class S>
@@ -61,11 +63,13 @@ TEST_CONSTEXPR_CXX20 bool test() {
   test_string<std::string>();
 #if TEST_STD_VER >= 11
   test_string<std::basic_string<char, std::char_traits<char>, min_allocator<char> > >();
+  test_string<std::basic_string<char, std::char_traits<char>, safe_allocator<char> > >();
   { // LWG 2946
     std::string s;
     s += {"abc", 1};
     assert(s.size() == 1);
     assert(s == "a");
+    LIBCPP_ASSERT(is_string_asan_correct(s));
   }
 #endif
 
diff --git a/libcxx/test/std/strings/basic.string/string.modifiers/string_replace/iter_iter_string.pass.cpp b/libcxx/test/std/strings/basic.string/string.modifiers/string_replace/iter_iter_string.pass.cpp
index 35db1d6368191693af71cc5f24f08ddf144adfc0..eb0c62a66504ce6631564a83fbdfc507171e7e6b 100644
--- a/libcxx/test/std/strings/basic.string/string.modifiers/string_replace/iter_iter_string.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.modifiers/string_replace/iter_iter_string.pass.cpp
@@ -17,6 +17,7 @@
 
 #include "test_macros.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class S>
 TEST_CONSTEXPR_CXX20 void test(S s, typename S::size_type pos1, typename S::size_type n1, S str, S expected) {
@@ -29,6 +30,7 @@ TEST_CONSTEXPR_CXX20 void test(S s, typename S::size_type pos1, typename S::size
   assert(s == expected);
   typename S::size_type rlen = str.size();
   assert(s.size() == old_size - xlen + rlen);
+  LIBCPP_ASSERT(is_string_asan_correct(s));
 }
 
 template <class S>
@@ -292,6 +294,7 @@ int main(int, char**) {
   test<std::string>();
 #if TEST_STD_VER >= 11
   test<std::basic_string<char, std::char_traits<char>, min_allocator<char>>>();
+  test<std::basic_string<char, std::char_traits<char>, safe_allocator<char>>>();
 #endif
 
   return 0;
diff --git a/libcxx/test/std/strings/basic.string/string.modifiers/string_replace/size_size_T_size_size.pass.cpp b/libcxx/test/std/strings/basic.string/string.modifiers/string_replace/size_size_T_size_size.pass.cpp
index 004a519262853945c28b1161fbfbd74321f1e731..df08ae675a11e51bfc02b82237384a60aa1a57b7 100644
--- a/libcxx/test/std/strings/basic.string/string.modifiers/string_replace/size_size_T_size_size.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.modifiers/string_replace/size_size_T_size_size.pass.cpp
@@ -23,6 +23,7 @@
 
 #include "test_macros.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class S, class SV>
 TEST_CONSTEXPR_CXX20 void
@@ -49,6 +50,7 @@ test(S s,
     SizeT xlen = std::min<SizeT>(n1, old_size - pos1);
     SizeT rlen = std::min<SizeT>(n2, sv.size() - pos2);
     assert(s.size() == old_size - xlen + rlen);
+    LIBCPP_ASSERT(is_string_asan_correct(s));
   }
 #ifndef TEST_HAS_NO_EXCEPTIONS
   else if (!TEST_IS_CONSTANT_EVALUATED) {
@@ -6279,6 +6281,7 @@ int main(int, char**) {
   test<char, std::allocator>();
 #if TEST_STD_VER >= 11
   test<char, min_allocator>();
+  test<char, safe_allocator>();
 #endif
 
   return 0;
diff --git a/libcxx/test/std/strings/basic.string/string.modifiers/string_replace/size_size_pointer.pass.cpp b/libcxx/test/std/strings/basic.string/string.modifiers/string_replace/size_size_pointer.pass.cpp
index a7282615e485541727b16f92def1d029615182dc..aefe3ced9765ef1638d43773037d8f78b5db8f67 100644
--- a/libcxx/test/std/strings/basic.string/string.modifiers/string_replace/size_size_pointer.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.modifiers/string_replace/size_size_pointer.pass.cpp
@@ -18,6 +18,7 @@
 
 #include "test_macros.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class S>
 TEST_CONSTEXPR_CXX20 void
@@ -31,6 +32,7 @@ test(S s, typename S::size_type pos, typename S::size_type n1, const typename S:
     typename S::size_type xlen = std::min(n1, old_size - pos);
     typename S::size_type rlen = S::traits_type::length(str);
     assert(s.size() == old_size - xlen + rlen);
+    LIBCPP_ASSERT(is_string_asan_correct(s));
   }
 #ifndef TEST_HAS_NO_EXCEPTIONS
   else if (!TEST_IS_CONSTANT_EVALUATED) {
@@ -377,6 +379,7 @@ int main(int, char**) {
   test<std::string>();
 #if TEST_STD_VER >= 11
   test<std::basic_string<char, std::char_traits<char>, min_allocator<char>>>();
+  test<std::basic_string<char, std::char_traits<char>, safe_allocator<char>>>();
 #endif
 
   return 0;
diff --git a/libcxx/test/std/strings/basic.string/string.modifiers/string_replace/size_size_pointer_size.pass.cpp b/libcxx/test/std/strings/basic.string/string.modifiers/string_replace/size_size_pointer_size.pass.cpp
index 7ef63bdf313ddafe85fde55479aeca35a305c3f0..cb01c34e43d4aa1f9d217359ed36cecc638d10e1 100644
--- a/libcxx/test/std/strings/basic.string/string.modifiers/string_replace/size_size_pointer_size.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.modifiers/string_replace/size_size_pointer_size.pass.cpp
@@ -18,6 +18,7 @@
 
 #include "test_macros.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class S>
 TEST_CONSTEXPR_CXX20 void
@@ -36,6 +37,7 @@ test(S s,
     typename S::size_type xlen = std::min(n1, old_size - pos);
     typename S::size_type rlen = n2;
     assert(s.size() == old_size - xlen + rlen);
+    LIBCPP_ASSERT(is_string_asan_correct(s));
   }
 #ifndef TEST_HAS_NO_EXCEPTIONS
   else if (!TEST_IS_CONSTANT_EVALUATED) {
@@ -1341,6 +1343,7 @@ int main(int, char**) {
   test<std::string>();
 #if TEST_STD_VER >= 11
   test<std::basic_string<char, std::char_traits<char>, min_allocator<char>>>();
+  test<std::basic_string<char, std::char_traits<char>, safe_allocator<char>>>();
 #endif
 
   return 0;
diff --git a/libcxx/test/std/strings/basic.string/string.modifiers/string_replace/size_size_size_char.pass.cpp b/libcxx/test/std/strings/basic.string/string.modifiers/string_replace/size_size_size_char.pass.cpp
index 8841318c1d4df5b74b13b38c903af9273d2601b6..67e9316794225031424f7f3a054041a350fab8df 100644
--- a/libcxx/test/std/strings/basic.string/string.modifiers/string_replace/size_size_size_char.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.modifiers/string_replace/size_size_size_char.pass.cpp
@@ -18,6 +18,7 @@
 
 #include "test_macros.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class S>
 TEST_CONSTEXPR_CXX20 void
@@ -36,6 +37,7 @@ test(S s,
     typename S::size_type xlen = std::min(n1, old_size - pos);
     typename S::size_type rlen = n2;
     assert(s.size() == old_size - xlen + rlen);
+    LIBCPP_ASSERT(is_string_asan_correct(s));
   }
 #ifndef TEST_HAS_NO_EXCEPTIONS
   else if (!TEST_IS_CONSTANT_EVALUATED) {
@@ -381,6 +383,7 @@ int main(int, char**) {
   test<std::string>();
 #if TEST_STD_VER >= 11
   test<std::basic_string<char, std::char_traits<char>, min_allocator<char>>>();
+  test<std::basic_string<char, std::char_traits<char>, safe_allocator<char>>>();
 #endif
 
   return 0;
diff --git a/libcxx/test/std/strings/basic.string/string.modifiers/string_replace/size_size_string.pass.cpp b/libcxx/test/std/strings/basic.string/string.modifiers/string_replace/size_size_string.pass.cpp
index fab2beeabd5721ad4a47a946763598626a81a19a..47fe7030c83b35e0ff901d112844dc7838aa6ccf 100644
--- a/libcxx/test/std/strings/basic.string/string.modifiers/string_replace/size_size_string.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.modifiers/string_replace/size_size_string.pass.cpp
@@ -18,6 +18,7 @@
 
 #include "test_macros.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class S>
 TEST_CONSTEXPR_CXX20 void test(S s, typename S::size_type pos1, typename S::size_type n1, S str, S expected) {
@@ -30,6 +31,7 @@ TEST_CONSTEXPR_CXX20 void test(S s, typename S::size_type pos1, typename S::size
     typename S::size_type xlen = std::min(n1, old_size - pos1);
     typename S::size_type rlen = str.size();
     assert(s.size() == old_size - xlen + rlen);
+    LIBCPP_ASSERT(is_string_asan_correct(s));
   }
 #ifndef TEST_HAS_NO_EXCEPTIONS
   else if (!TEST_IS_CONSTANT_EVALUATED) {
@@ -385,6 +387,7 @@ int main(int, char**) {
   test<std::string>();
 #if TEST_STD_VER >= 11
   test<std::basic_string<char, std::char_traits<char>, min_allocator<char>>>();
+  test<std::basic_string<char, std::char_traits<char>, safe_allocator<char>>>();
 #endif
   return 0;
 }
diff --git a/libcxx/test/std/strings/basic.string/string.modifiers/string_replace/size_size_string_size_size.pass.cpp b/libcxx/test/std/strings/basic.string/string.modifiers/string_replace/size_size_string_size_size.pass.cpp
index 073de60289857ae71405a07671be8d58b9c34c77..48645f6280990954ff2519cc8f49e7552afa8eb4 100644
--- a/libcxx/test/std/strings/basic.string/string.modifiers/string_replace/size_size_string_size_size.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.modifiers/string_replace/size_size_string_size_size.pass.cpp
@@ -22,6 +22,7 @@
 
 #include "test_macros.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class S>
 TEST_CONSTEXPR_CXX20 void
@@ -41,6 +42,7 @@ test(S s,
     typename S::size_type xlen = std::min(n1, old_size - pos1);
     typename S::size_type rlen = std::min(n2, str.size() - pos2);
     assert(s.size() == old_size - xlen + rlen);
+    LIBCPP_ASSERT(is_string_asan_correct(s));
   }
 #ifndef TEST_HAS_NO_EXCEPTIONS
   else if (!TEST_IS_CONSTANT_EVALUATED) {
@@ -6099,6 +6101,7 @@ int main(int, char**) {
   test<std::string>();
 #if TEST_STD_VER >= 11
   test<std::basic_string<char, std::char_traits<char>, min_allocator<char> > >();
+  test<std::basic_string<char, std::char_traits<char>, safe_allocator<char> > >();
 #endif
 
   return 0;
diff --git a/libcxx/test/std/strings/basic.string/string.modifiers/string_replace/size_size_string_view.pass.cpp b/libcxx/test/std/strings/basic.string/string.modifiers/string_replace/size_size_string_view.pass.cpp
index cec8cc0c52dd9f345d8eca86f4b3ee4ec2148789..7f17160419065cb7f3495f88ca59b65bce74d56d 100644
--- a/libcxx/test/std/strings/basic.string/string.modifiers/string_replace/size_size_string_view.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.modifiers/string_replace/size_size_string_view.pass.cpp
@@ -18,6 +18,7 @@
 
 #include "test_macros.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class S, class SV>
 TEST_CONSTEXPR_CXX20 void test(S s, typename S::size_type pos1, typename S::size_type n1, SV sv, S expected) {
@@ -30,6 +31,7 @@ TEST_CONSTEXPR_CXX20 void test(S s, typename S::size_type pos1, typename S::size
     typename S::size_type xlen = std::min(n1, old_size - pos1);
     typename S::size_type rlen = sv.size();
     assert(s.size() == old_size - xlen + rlen);
+    LIBCPP_ASSERT(is_string_asan_correct(s));
   }
 #ifndef TEST_HAS_NO_EXCEPTIONS
   else if (!TEST_IS_CONSTANT_EVALUATED) {
@@ -376,6 +378,7 @@ int main(int, char**) {
   test<char, std::allocator>();
 #if TEST_STD_VER >= 11
   test<char, min_allocator>();
+  test<char, safe_allocator>();
 #endif
 
   return 0;
diff --git a/libcxx/test/std/strings/basic.string/string.modifiers/string_swap/swap.asan.pass.cpp b/libcxx/test/std/strings/basic.string/string.modifiers/string_swap/swap.asan.pass.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..a3466e76a5f5fee2ba85b9a740bc8b4574712485
--- /dev/null
+++ b/libcxx/test/std/strings/basic.string/string.modifiers/string_swap/swap.asan.pass.cpp
@@ -0,0 +1,91 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+// <string>
+
+// This test validates that ASan annotations are correctly updated after swaps.
+// This test meticulously interchanges objects that store data both within their internal memory (Short
+// String Optimization) and in external buffers (non-SSO).
+
+#include <string>
+#include <cassert>
+
+#include "test_macros.h"
+#include "asan_testing.h"
+
+template <class CharT>
+void test(const CharT val) {
+  using S = std::basic_string<CharT>;
+
+  S empty_s;
+  S short_s(3, val);
+  S long_s(1100, val);
+
+  std::swap(empty_s, empty_s);
+  std::swap(short_s, short_s);
+  std::swap(long_s, long_s);
+  LIBCPP_ASSERT(is_string_asan_correct(empty_s));
+  LIBCPP_ASSERT(is_string_asan_correct(short_s));
+  LIBCPP_ASSERT(is_string_asan_correct(long_s));
+
+  std::swap(empty_s, short_s);
+  LIBCPP_ASSERT(is_string_asan_correct(empty_s));
+  LIBCPP_ASSERT(is_string_asan_correct(short_s));
+
+  std::swap(empty_s, short_s);
+  LIBCPP_ASSERT(is_string_asan_correct(empty_s));
+  LIBCPP_ASSERT(is_string_asan_correct(short_s));
+
+  std::swap(empty_s, long_s);
+  LIBCPP_ASSERT(is_string_asan_correct(empty_s));
+  LIBCPP_ASSERT(is_string_asan_correct(long_s));
+
+  std::swap(empty_s, long_s);
+  LIBCPP_ASSERT(is_string_asan_correct(empty_s));
+  LIBCPP_ASSERT(is_string_asan_correct(long_s));
+
+  std::swap(short_s, long_s);
+  LIBCPP_ASSERT(is_string_asan_correct(short_s));
+  LIBCPP_ASSERT(is_string_asan_correct(long_s));
+
+  std::swap(short_s, long_s);
+  LIBCPP_ASSERT(is_string_asan_correct(short_s));
+  LIBCPP_ASSERT(is_string_asan_correct(long_s));
+
+  S long_s2(11100, val);
+
+  std::swap(long_s, long_s2);
+  LIBCPP_ASSERT(is_string_asan_correct(long_s));
+  LIBCPP_ASSERT(is_string_asan_correct(long_s2));
+
+  std::swap(long_s, long_s2);
+  LIBCPP_ASSERT(is_string_asan_correct(long_s));
+  LIBCPP_ASSERT(is_string_asan_correct(long_s2));
+
+  S long_s3(111, val);
+
+  std::swap(long_s, long_s3);
+  LIBCPP_ASSERT(is_string_asan_correct(long_s));
+  LIBCPP_ASSERT(is_string_asan_correct(long_s2));
+}
+
+int main(int, char**) {
+  test<char>('x');
+#ifndef TEST_HAS_NO_WIDE_CHARACTERS
+  test<wchar_t>(L'x');
+#endif
+#if TEST_STD_VER >= 11
+  test<char16_t>(u'x');
+  test<char32_t>(U'x');
+#endif
+#if TEST_STD_VER >= 20
+  test<char8_t>(u8'x');
+#endif
+
+  return 0;
+}
diff --git a/libcxx/test/std/strings/basic.string/string.modifiers/string_swap/swap.pass.cpp b/libcxx/test/std/strings/basic.string/string.modifiers/string_swap/swap.pass.cpp
index 3d9ec016bcfbd3a24f4ba8258f30f4f36e9a339d..16bdac1c4dcc63ff9f5f02a43945f4558a820b6c 100644
--- a/libcxx/test/std/strings/basic.string/string.modifiers/string_swap/swap.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.modifiers/string_swap/swap.pass.cpp
@@ -17,6 +17,7 @@
 
 #include "test_macros.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class S>
 TEST_CONSTEXPR_CXX20 void test(S s1, S s2) {
@@ -27,6 +28,8 @@ TEST_CONSTEXPR_CXX20 void test(S s1, S s2) {
   LIBCPP_ASSERT(s2.__invariants());
   assert(s1 == s2_);
   assert(s2 == s1_);
+  LIBCPP_ASSERT(is_string_asan_correct(s1));
+  LIBCPP_ASSERT(is_string_asan_correct(s2));
 }
 
 template <class S>
@@ -47,12 +50,18 @@ TEST_CONSTEXPR_CXX20 void test_string() {
   test(S("abcdefghijklmnopqrst"), S("12345"));
   test(S("abcdefghijklmnopqrst"), S("1234567890"));
   test(S("abcdefghijklmnopqrst"), S("12345678901234567890"));
+  test(S("abcdefghijklmnopqrstabcdefghijklmnopqrst"), S(""));
+  test(S("abcdefghijklmnopqrstabcdefghijklmnopqrst"), S("12345"));
+  test(S("abcdefghijklmnopqrstabcdefghijklmnopqrst"), S("1234567890"));
+  test(S("abcdefghijklmnopqrstabcdefghijklmnopqrst"), S("12345678901234567890"));
+  test(S("abcdefghijklmnopqrstabcdefghijklmnopqrst"), S("12345678901234567890abcdefghijklmnopqrst"));
 }
 
 TEST_CONSTEXPR_CXX20 bool test() {
   test_string<std::string>();
 #if TEST_STD_VER >= 11
   test_string<std::basic_string<char, std::char_traits<char>, min_allocator<char>>>();
+  test_string<std::basic_string<char, std::char_traits<char>, safe_allocator<char>>>();
 #endif
 
   return true;
diff --git a/libcxx/test/std/strings/basic.string/string.nonmembers/string.special/swap.pass.cpp b/libcxx/test/std/strings/basic.string/string.nonmembers/string.special/swap.pass.cpp
index 376637e3efc9428e5e9d1d42691f88cf757f3681..c807007fea352851efce9a72424993491926a6dc 100644
--- a/libcxx/test/std/strings/basic.string/string.nonmembers/string.special/swap.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.nonmembers/string.special/swap.pass.cpp
@@ -19,6 +19,7 @@
 
 #include "test_macros.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class S>
 TEST_CONSTEXPR_CXX20 void test(S s1, S s2) {
@@ -29,6 +30,8 @@ TEST_CONSTEXPR_CXX20 void test(S s1, S s2) {
   LIBCPP_ASSERT(s2.__invariants());
   assert(s1 == s2_);
   assert(s2 == s1_);
+  LIBCPP_ASSERT(is_string_asan_correct(s1));
+  LIBCPP_ASSERT(is_string_asan_correct(s2));
 }
 
 template <class S>
@@ -59,6 +62,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
   test_string<std::string>();
 #if TEST_STD_VER >= 11
   test_string<std::basic_string<char, std::char_traits<char>, min_allocator<char> > >();
+  test_string<std::basic_string<char, std::char_traits<char>, safe_allocator<char> > >();
 #endif
 
   return true;
diff --git a/libcxx/test/std/strings/basic.string/string.nonmembers/string_op+/char_string.pass.cpp b/libcxx/test/std/strings/basic.string/string.nonmembers/string_op+/char_string.pass.cpp
index 82a591e0533628d694fef01c838ad362998162a0..2ca5788e6d8a3015729711cacfd53c6df75ca8a7 100644
--- a/libcxx/test/std/strings/basic.string/string.nonmembers/string_op+/char_string.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.nonmembers/string_op+/char_string.pass.cpp
@@ -22,10 +22,12 @@
 
 #include "test_macros.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class S>
 TEST_CONSTEXPR_CXX20 void test0(typename S::value_type lhs, const S& rhs, const S& x) {
   assert(lhs + rhs == x);
+  LIBCPP_ASSERT(is_string_asan_correct(lhs + rhs));
 }
 
 #if TEST_STD_VER >= 11
@@ -53,6 +55,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
   test_string<std::string>();
 #if TEST_STD_VER >= 11
   test_string<std::basic_string<char, std::char_traits<char>, min_allocator<char> > >();
+  test_string<std::basic_string<char, std::char_traits<char>, safe_allocator<char> > >();
 #endif
 
   return true;
diff --git a/libcxx/test/std/strings/basic.string/string.nonmembers/string_op+/string_char.pass.cpp b/libcxx/test/std/strings/basic.string/string.nonmembers/string_op+/string_char.pass.cpp
index 66cbac28f9f2c3ac6598679c021ee561c1903e27..9c0fb56e1f1c995a33a9d4e5cad5ae01b8b6b047 100644
--- a/libcxx/test/std/strings/basic.string/string.nonmembers/string_op+/string_char.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.nonmembers/string_op+/string_char.pass.cpp
@@ -22,10 +22,12 @@
 
 #include "test_macros.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class S>
 TEST_CONSTEXPR_CXX20 void test0(const S& lhs, typename S::value_type rhs, const S& x) {
   assert(lhs + rhs == x);
+  LIBCPP_ASSERT(is_string_asan_correct(lhs + rhs));
 }
 
 #if TEST_STD_VER >= 11
@@ -55,6 +57,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
   test_string<std::string>();
 #if TEST_STD_VER >= 11
   test_string<std::basic_string<char, std::char_traits<char>, min_allocator<char> > >();
+  test_string<std::basic_string<char, std::char_traits<char>, safe_allocator<char> > >();
 #endif
 
   return true;
diff --git a/libcxx/test/std/strings/basic.string/string.nonmembers/string_op+/string_pointer.pass.cpp b/libcxx/test/std/strings/basic.string/string.nonmembers/string_op+/string_pointer.pass.cpp
index 2802189f843c59c00d93f4dd89513e8fd82e739b..3e1aaebf112eb5e96d4b7bd05e0cd11dd38cd704 100644
--- a/libcxx/test/std/strings/basic.string/string.nonmembers/string_op+/string_pointer.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.nonmembers/string_op+/string_pointer.pass.cpp
@@ -22,10 +22,12 @@
 
 #include "test_macros.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class S>
 TEST_CONSTEXPR_CXX20 void test0(const S& lhs, const typename S::value_type* rhs, const S& x) {
   assert(lhs + rhs == x);
+  LIBCPP_ASSERT(is_string_asan_correct(lhs + rhs));
 }
 
 #if TEST_STD_VER >= 11
@@ -77,6 +79,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
   test_string<std::string>();
 #if TEST_STD_VER >= 11
   test_string<std::basic_string<char, std::char_traits<char>, min_allocator<char> > >();
+  test_string<std::basic_string<char, std::char_traits<char>, safe_allocator<char> > >();
 #endif
 
   return true;
diff --git a/libcxx/test/std/strings/basic.string/string.nonmembers/string_op+/string_string.pass.cpp b/libcxx/test/std/strings/basic.string/string.nonmembers/string_op+/string_string.pass.cpp
index f8ebb98767a8375ac6491d835f974eaebf77b659..fcc8ceab870715cfc19d5eda7fcf2e2da0382230 100644
--- a/libcxx/test/std/strings/basic.string/string.nonmembers/string_op+/string_string.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.nonmembers/string_op+/string_string.pass.cpp
@@ -34,10 +34,12 @@
 
 #include "test_macros.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class S>
 TEST_CONSTEXPR_CXX20 void test0(const S& lhs, const S& rhs, const S& x) {
   assert(lhs + rhs == x);
+  LIBCPP_ASSERT(is_string_asan_correct(lhs + rhs));
 }
 
 #if TEST_STD_VER >= 11
@@ -133,6 +135,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
   test_string<std::string>();
 #if TEST_STD_VER >= 11
   test_string<std::basic_string<char, std::char_traits<char>, min_allocator<char> > >();
+  test_string<std::basic_string<char, std::char_traits<char>, safe_allocator<char> > >();
 #endif
 
   return true;
diff --git a/libcxx/test/std/strings/basic.string/string.ops/string_substr/substr.pass.cpp b/libcxx/test/std/strings/basic.string/string.ops/string_substr/substr.pass.cpp
index 9b8b7a8a223d8ce8bbcb5665ccc832bd090f2989..6a66a0067b390b4c3f5cebd07af967b6ae769855 100644
--- a/libcxx/test/std/strings/basic.string/string.ops/string_substr/substr.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.ops/string_substr/substr.pass.cpp
@@ -19,6 +19,7 @@
 #include "test_allocator.h"
 #include "test_macros.h"
 #include "min_allocator.h"
+#include "asan_testing.h"
 
 template <class S>
 TEST_CONSTEXPR_CXX20 void test(const S& s, typename S::size_type pos, typename S::size_type n) {
@@ -29,6 +30,8 @@ TEST_CONSTEXPR_CXX20 void test(const S& s, typename S::size_type pos, typename S
     typename S::size_type rlen = std::min(n, s.size() - pos);
     assert(str.size() == rlen);
     assert(S::traits_type::compare(s.data() + pos, str.data(), rlen) == 0);
+    LIBCPP_ASSERT(is_string_asan_correct(s));
+    LIBCPP_ASSERT(is_string_asan_correct(str));
   }
 #ifndef TEST_HAS_NO_EXCEPTIONS
   else if (!TEST_IS_CONSTANT_EVALUATED) {
@@ -117,6 +120,7 @@ TEST_CONSTEXPR_CXX20 bool test() {
   test_string<std::string>();
 #if TEST_STD_VER >= 11
   test_string<std::basic_string<char, std::char_traits<char>, min_allocator<char>>>();
+  test_string<std::basic_string<char, std::char_traits<char>, safe_allocator<char>>>();
 #endif
 
   return true;
diff --git a/libcxx/test/std/strings/basic.string/string.ops/string_substr/substr_rvalue.pass.cpp b/libcxx/test/std/strings/basic.string/string.ops/string_substr/substr_rvalue.pass.cpp
index c3c6c3f56b91961459335a87c13fb856a77e62f9..596eb923db3100627c9419283297bbbf600f037f 100644
--- a/libcxx/test/std/strings/basic.string/string.ops/string_substr/substr_rvalue.pass.cpp
+++ b/libcxx/test/std/strings/basic.string/string.ops/string_substr/substr_rvalue.pass.cpp
@@ -20,6 +20,7 @@
 #include "make_string.h"
 #include "min_allocator.h"
 #include "test_allocator.h"
+#include "asan_testing.h"
 
 #define STR(string) MAKE_CSTRING(typename S::value_type, string)
 
@@ -32,6 +33,8 @@ constexpr void test(S orig, typename S::size_type pos, typename S::size_type n,
   LIBCPP_ASSERT(orig.__invariants());
   LIBCPP_ASSERT(str.__invariants());
   assert(str == expected);
+  LIBCPP_ASSERT(is_string_asan_correct(orig));
+  LIBCPP_ASSERT(is_string_asan_correct(str));
 }
 
 template <class S>
diff --git a/libcxx/test/support/asan_testing.h b/libcxx/test/support/asan_testing.h
index 5d044374044dea79c91b21265f082c1831cb94c2..2dfec5c42b00b2075534ef6bf71ac894bcbca0fd 100644
--- a/libcxx/test/support/asan_testing.h
+++ b/libcxx/test/support/asan_testing.h
@@ -11,26 +11,25 @@
 
 #include "test_macros.h"
 #include <vector>
+#include <string>
+#include <memory>
+#include <type_traits>
 
 #if TEST_HAS_FEATURE(address_sanitizer)
-extern "C" int __sanitizer_verify_contiguous_container
-     ( const void *beg, const void *mid, const void *end );
+extern "C" int __sanitizer_verify_contiguous_container(const void* beg, const void* mid, const void* end);
 
 template <typename T, typename Alloc>
-TEST_CONSTEXPR bool is_contiguous_container_asan_correct ( const std::vector<T, Alloc> &c )
-{
-    if (TEST_IS_CONSTANT_EVALUATED)
-        return true;
-    if (std::is_same<Alloc, std::allocator<T> >::value && c.data() != NULL)
-        return __sanitizer_verify_contiguous_container(
-            c.data(), c.data() + c.size(), c.data() + c.capacity()) != 0;
+TEST_CONSTEXPR bool is_contiguous_container_asan_correct(const std::vector<T, Alloc>& c) {
+  if (TEST_IS_CONSTANT_EVALUATED)
     return true;
+  if (std::is_same<Alloc, std::allocator<T> >::value && c.data() != NULL)
+    return __sanitizer_verify_contiguous_container(c.data(), c.data() + c.size(), c.data() + c.capacity()) != 0;
+  return true;
 }
 #else
 template <typename T, typename Alloc>
-TEST_CONSTEXPR bool is_contiguous_container_asan_correct ( const std::vector<T, Alloc> &)
-{
-    return true;
+TEST_CONSTEXPR bool is_contiguous_container_asan_correct(const std::vector<T, Alloc>&) {
+  return true;
 }
 #endif // TEST_HAS_FEATURE(address_sanitizer)
 
@@ -56,4 +55,42 @@ TEST_CONSTEXPR bool is_double_ended_contiguous_container_asan_correct(const std:
 }
 #endif
 
+#if TEST_HAS_FEATURE(address_sanitizer)
+template <typename S>
+bool is_string_short(S const& s) {
+  // We do not have access to __is_long(), but we can check if strings
+  // buffer is inside strings memory. If strings memory contains its content,
+  // SSO is in use. To check it, we can just confirm that the beginning is in
+  // the string object memory block.
+  // &s    - beginning of objects memory
+  // &s[0] - beginning of the buffer
+  // (&s+1) - end of objects memory
+  return (void*)std::addressof(s) <= (void*)std::addressof(s[0]) &&
+         (void*)std::addressof(s[0]) < (void*)(std::addressof(s) + 1);
+}
+
+template <typename ChrT, typename TraitsT, typename Alloc>
+TEST_CONSTEXPR bool is_string_asan_correct(const std::basic_string<ChrT, TraitsT, Alloc>& c) {
+  if (TEST_IS_CONSTANT_EVALUATED)
+    return true;
+
+  if (!is_string_short(c) || _LIBCPP_SHORT_STRING_ANNOTATIONS_ALLOWED) {
+    if (std::is_same<Alloc, std::allocator<ChrT>>::value)
+      return __sanitizer_verify_contiguous_container(c.data(), c.data() + c.size() + 1, c.data() + c.capacity() + 1) !=
+             0;
+    else
+      return __sanitizer_verify_contiguous_container(
+                 c.data(), c.data() + c.capacity() + 1, c.data() + c.capacity() + 1) != 0;
+  } else {
+    return __sanitizer_verify_contiguous_container(std::addressof(c), std::addressof(c) + 1, std::addressof(c) + 1) !=
+           0;
+  }
+}
+#else
+#  include <string>
+template <typename ChrT, typename TraitsT, typename Alloc>
+TEST_CONSTEXPR bool is_string_asan_correct(const std::basic_string<ChrT, TraitsT, Alloc>&) {
+  return true;
+}
+#endif // TEST_HAS_FEATURE(address_sanitizer)
 #endif // ASAN_TESTING_H
diff --git a/libcxx/test/support/assert_macros.h b/libcxx/test/support/assert_macros.h
index b7011794025bf03b13b1d2e3c24dabdfc060c010..1059823dcb246509453c13a9e06d48db65b5a100 100644
--- a/libcxx/test/support/assert_macros.h
+++ b/libcxx/test/support/assert_macros.h
@@ -50,7 +50,7 @@ void test_log(const char* condition, const char* file, int line, const F& functo
 }
 
 template <class Arg>
-[[noreturn]] void test_fail(const char* file, int line, const Arg& arg) {
+TEST_NORETURN void test_fail(const char* file, int line, const Arg& arg) {
   test_log("", file, line, arg);
   std::abort();
 }
diff --git a/libcxx/test/support/test_comparisons.h b/libcxx/test/support/test_comparisons.h
index e006f69f8bf67555d8f829c1b32bfb3900ef984a..db6977a96a2fea5b2e018a861fac906857096edc 100644
--- a/libcxx/test/support/test_comparisons.h
+++ b/libcxx/test/support/test_comparisons.h
@@ -213,13 +213,11 @@ void AssertEqualityAreNoexcept()
 }
 
 template <class T, class U = T>
-void AssertEqualityReturnBool()
-{
-    ASSERT_SAME_TYPE(decltype(std::declval<const T&>() == std::declval<const U&>()), bool);
-    ASSERT_SAME_TYPE(decltype(std::declval<const T&>() != std::declval<const U&>()), bool);
+TEST_CONSTEXPR_CXX14 void AssertEqualityReturnBool() {
+  ASSERT_SAME_TYPE(decltype(std::declval<const T&>() == std::declval<const U&>()), bool);
+  ASSERT_SAME_TYPE(decltype(std::declval<const T&>() != std::declval<const U&>()), bool);
 }
 
-
 template <class T, class U = T>
 void AssertEqualityConvertibleToBool()
 {
diff --git a/libcxx/test/tools/clang_tidy_checks/header_exportable_declarations.cpp b/libcxx/test/tools/clang_tidy_checks/header_exportable_declarations.cpp
index fcb5865adf0d4e175a6a8a9310bb08aff896f1ff..35f020da45c4355f467a875b4374a09df17afac8 100644
--- a/libcxx/test/tools/clang_tidy_checks/header_exportable_declarations.cpp
+++ b/libcxx/test/tools/clang_tidy_checks/header_exportable_declarations.cpp
@@ -166,7 +166,7 @@ void header_exportable_declarations::registerMatchers(clang::ast_matchers::Match
   }
 }
 
-/// Returns the qualified name of a declaration.
+/// Returns the qualified name of a public declaration.
 ///
 /// There is a small issue with qualified names. Typically the name returned is
 /// in the namespace \c std instead of the namespace \c std::__1. Except when a
@@ -182,14 +182,37 @@ void header_exportable_declarations::registerMatchers(clang::ast_matchers::Match
 /// * exception has equality operators for the type \c exception_ptr
 /// * initializer_list has the functions \c begin and \c end
 ///
-/// \warning In some cases the returned name can be an empty string.
-/// The cause has not been investigated.
+/// When the named declaration uses a reserved name the result is an
+/// empty string.
 static std::string get_qualified_name(const clang::NamedDecl& decl) {
-  std::string result = decl.getQualifiedNameAsString();
-
-  if (result.starts_with("std::__1::"))
-    result.erase(5, 5);
-
+  std::string result = decl.getNameAsString();
+  // Reject reserved names (ignoring _ in global namespace).
+  if (result.size() >= 2 && result[0] == '_')
+    if (result[1] == '_' || std::isupper(result[1]))
+      if (result != "_Exit")
+        return "";
+
+  for (auto* context = llvm::dyn_cast_or_null<clang::NamespaceDecl>(decl.getDeclContext()); //
+       context;
+       context = llvm::dyn_cast_or_null<clang::NamespaceDecl>(context->getDeclContext())) {
+    std::string ns = std::string(context->getName());
+
+    if (ns.starts_with("__")) {
+      // When the reserved name is an inline namespace the namespace is
+      // not added to the qualified name instead of removed. Libc++ uses
+      // several inline namespace with reserved names. For example,
+      // __1 for every declaration, __cpo in range-based algorithms.
+      //
+      // Note other inline namespaces are expanded. This resolves
+      // ambiguity when two named declarations have the same name but in
+      // different inline namespaces. These typically are the literal
+      // conversion operators like operator""s which can be a
+      // std::string or std::chrono::seconds.
+      if (!context->isInline())
+        return "";
+    } else
+      result = ns + "::" + result;
+  }
   return result;
 }
 
@@ -220,38 +243,6 @@ static bool is_viable_declaration(const clang::NamedDecl* decl) {
   return llvm::isa<clang::EnumDecl, clang::VarDecl, clang::ConceptDecl, clang::TypedefNameDecl, clang::UsingDecl>(decl);
 }
 
-/// Returns the name is a reserved name.
-///
-/// Detected reserved names are names starting with __ or _[A-Z].
-/// These names can be in the global namespace, std namespace or any namespace
-/// inside std. For example, std::ranges contains reserved names to implement
-/// the Niebloids.
-///
-/// This test misses candidates which are not used in libc++
-/// * any identifier with two underscores not at the start
-bool is_reserved_name(std::string_view name) {
-  if (name.starts_with("_")) {
-    // This is a public name declared in cstdlib.
-    if (name == "_Exit")
-      return false;
-
-    return name.size() > 1 && (name[1] == '_' || std::isupper(name[1]));
-  }
-
-  std::size_t pos = name.find("::_");
-  if (pos == std::string::npos)
-    return false;
-
-  if (pos + 3 > name.size())
-    return false;
-
-  // This is a public name declared in cstdlib.
-  if (name == "std::_Exit")
-    return false;
-
-  return name[pos + 3] == '_' || std::isupper(name[pos + 3]);
-}
-
 /// Some declarations in the global namespace are exported from the std module.
 static bool is_global_name_exported_by_std_module(std::string_view name) {
   static const std::set<std::string_view> valid{
@@ -297,9 +288,6 @@ void header_exportable_declarations::check(const clang::ast_matchers::MatchFinde
     if (name.empty())
       return;
 
-    if (is_reserved_name(name))
-      return;
-
     // For modules only take the declarations exported.
     if (is_module(file_type_))
       if (decl->getModuleOwnershipKind() != clang::Decl::ModuleOwnershipKind::VisibleWhenImported)
@@ -336,7 +324,7 @@ void header_exportable_declarations::check(const clang::ast_matchers::MatchFinde
       return;
 
     std::string name = get_qualified_name(*decl);
-    if (is_reserved_name(name))
+    if (name.empty())
       return;
 
     if (global_decls_.contains(name))
diff --git a/libcxx/utils/ci/buildkite-pipeline.yml b/libcxx/utils/ci/buildkite-pipeline.yml
index 51b9999026247fd33d08da1b5eb9944919c72693..c63f6a78b16070c50b462c56ff09c2083ef39174 100644
--- a/libcxx/utils/ci/buildkite-pipeline.yml
+++ b/libcxx/utils/ci/buildkite-pipeline.yml
@@ -41,45 +41,6 @@ definitions:
       - "**/*.abilist"
       - "**/crash_diagnostics/*"
 
-# Define agents using YAML anchors to reduce duplication
-agents_definitions:
-  _windows_agent: &windows_agent
-    agents:
-      queue: windows
-
-  # Mac OS Builders
-  _mac_agent_x86: &mac_agent_x86
-    agents:
-      queue: libcxx-builders
-      os: macos
-      arch: x86_64
-  _mac_agent_arm64: &mac_agent_arm64
-    agents:
-      queue: libcxx-builders
-      os: macos
-      arch: arm64
-  _mac_agent_any_arch: &mac_agent_any_arch
-    agents:
-      queue: libcxx-builders
-      os: macos
-  _arm_agent_aarch64: &arm_agent_aarch64
-    agents:
-      queue: libcxx-builders-linaro-arm
-      arch: aarch64
-  _arm_agent_armv8l: &arm_agent_armv8l
-    agents:
-      queue: libcxx-builders-linaro-arm
-      arch: armv8l
-  _aix_agent: &aix_agent
-    agents:
-      queue: libcxx-builders
-      os: aix
-  _android_agent: &android_agent
-    agents:
-      queue: libcxx-builders
-      os: android
-
-
 environment_definitions:
   _common_env: &common_env
       ENABLE_CLANG_TIDY: "On"
@@ -101,117 +62,166 @@ steps:
   steps:
   - label: Clang-cl (DLL)
     command: bash libcxx/utils/ci/run-buildbot clang-cl-dll
-    <<: *windows_agent
+    agents:
+      queue: windows
     <<: *common
 
   - label: Clang-cl (Static)
     command: bash libcxx/utils/ci/run-buildbot clang-cl-static
-    <<: *windows_agent
+    agents:
+      queue: windows
     <<: *common
 
   - label: Clang-cl (no vcruntime exceptions)
     command: bash libcxx/utils/ci/run-buildbot clang-cl-no-vcruntime
     <<: *common
-    <<: *windows_agent
+    agents:
+      queue: windows
 
   - label: Clang-cl (Debug mode)
     command: bash libcxx/utils/ci/run-buildbot clang-cl-debug
-    <<: *windows_agent
+    agents:
+      queue: windows
     <<: *common
 
   - label: Clang-cl (Static CRT)
     command: bash libcxx/utils/ci/run-buildbot clang-cl-static-crt
-    <<: *windows_agent
+    agents:
+      queue: windows
     <<: *common
 
   - label: MinGW (DLL, x86_64)
     command: bash libcxx/utils/ci/run-buildbot mingw-dll
-    <<: *windows_agent
+    agents:
+      queue: windows
     <<: *common
 
   - label: MinGW (Static, x86_64)
     command: bash libcxx/utils/ci/run-buildbot mingw-static
-    <<: *windows_agent
+    agents:
+      queue: windows
     <<: *common
 
   - label: MinGW (DLL, i686)
     command: bash libcxx/utils/ci/run-buildbot mingw-dll-i686
-    <<: *windows_agent
+    agents:
+      queue: windows
     <<: *common
 
 - group: ':mac: Apple'
   steps:
   - label: MacOS x86_64
     command: libcxx/utils/ci/run-buildbot generic-cxx23
-    <<: *mac_agent_x86
+    agents:
+      queue: libcxx-builders
+      os: macos
+      arch: x86_64
     <<: *common
 
   - label: MacOS arm64
     command: libcxx/utils/ci/run-buildbot generic-cxx23
-    <<: *mac_agent_arm64
+    agents:
+      queue: libcxx-builders
+      os: macos
+      arch: arm64
     <<: *common
 
   - label: MacOS with Modules
     command: libcxx/utils/ci/run-buildbot generic-modules
-    <<: *mac_agent_any_arch
+    agents:
+      queue: libcxx-builders
+      os: macos
+    <<: *common
+
+  - label: MacOS with C++03
+    command: libcxx/utils/ci/run-buildbot generic-cxx03
+    agents:
+      queue: libcxx-builders
+      os: macos
     <<: *common
 
     # Build with the configuration we use to generate libc++.dylib on Apple platforms
   - label: Apple system
     command: libcxx/utils/ci/run-buildbot apple-system
-    <<: *mac_agent_arm64  # This can technically run on any architecture, but we have more resources on arm64 so we pin this job to arm64
+    agents:
+      queue: libcxx-builders
+      os: macos
+      arch: arm64 # This can technically run on any architecture, but we have more resources on arm64 so we pin this job to arm64
     <<: *common
 
   - label: Apple back-deployment macosx10.13
     command: libcxx/utils/ci/run-buildbot apple-system-backdeployment-10.13
-    <<: *mac_agent_x86 # We need to use x86_64 for back-deployment CI on this target since macOS didn't support arm64 back then
+    agents:
+      queue: libcxx-builders
+      os: macos
+      arch: x86_64 # We need to use x86_64 for back-deployment CI on this target since macOS didn't support arm64 back then
     <<: *common
 
   - label: Apple back-deployment macosx10.15
     command: libcxx/utils/ci/run-buildbot apple-system-backdeployment-10.15
-    <<: *mac_agent_x86 # We need to use x86_64 for back-deployment CI on this target since macOS didn't support arm64 back then
+    agents:
+      queue: libcxx-builders
+      os: macos
+      arch: x86_64 # We need to use x86_64 for back-deployment CI on this target since macOS didn't support arm64 back then
     <<: *common
 
   - label: Apple back-deployment with hardening enabled
     command: libcxx/utils/ci/run-buildbot apple-system-backdeployment-hardened-11.0
-    <<: *mac_agent_x86  # TODO: Remove this once we are able to run back-deployment on arm64 again, since this isn't x86_64 specific
+    agents:
+      queue: libcxx-builders
+      os: macos
+      arch: x86_64 # TODO: Remove this once we are able to run back-deployment on arm64 again, since this isn't x86_64 specific
     <<: *common
 
 - group: ARM
   steps:
   - label: AArch64
     command: libcxx/utils/ci/run-buildbot aarch64
-    <<: *arm_agent_aarch64
+    agents:
+      queue: libcxx-builders-linaro-arm
+      arch: aarch64
     <<: *common
 
   - label: AArch64 -fno-exceptions
     command: libcxx/utils/ci/run-buildbot aarch64-no-exceptions
-    <<: *arm_agent_aarch64
+    agents:
+      queue: libcxx-builders-linaro-arm
+      arch: aarch64
     <<: *common
 
   - label: Armv8
     command: libcxx/utils/ci/run-buildbot armv8
-    <<: *arm_agent_armv8l
+    agents:
+      queue: libcxx-builders-linaro-arm
+      arch: armv8l
     <<: *common
 
   - label: Armv8 -fno-exceptions
     command: libcxx/utils/ci/run-buildbot armv8-no-exceptions
-    <<: *arm_agent_armv8l
+    agents:
+      queue: libcxx-builders-linaro-arm
+      arch: armv8l
     <<: *common
 
   - label: Armv7
     command: libcxx/utils/ci/run-buildbot armv7
-    <<: *arm_agent_armv8l
+    agents:
+      queue: libcxx-builders-linaro-arm
+      arch: armv8l
     <<: *common
 
   - label: Armv7 -fno-exceptions
     command: libcxx/utils/ci/run-buildbot armv7-no-exceptions
-    <<: *arm_agent_armv8l
+    agents:
+      queue: libcxx-builders-linaro-arm
+      arch: armv8l
     <<: *common
 
   - label: Armv7-M picolibc
     command: libcxx/utils/ci/run-buildbot armv7m-picolibc
-    <<: *arm_agent_aarch64
+    agents:
+      queue: libcxx-builders-linaro-arm
+      arch: aarch64
     <<: *common
 
 - group: AIX
@@ -222,7 +232,9 @@ steps:
       CC: clang
       CXX: clang++
       OBJECT_MODE: '32'
-    <<: *aix_agent
+    agents:
+      queue: libcxx-builders
+      os: aix
     <<: *common
 
   - label: AIX (64-bit)
@@ -231,7 +243,9 @@ steps:
       CC: clang
       CXX: clang++
       OBJECT_MODE: '64'
-    <<: *aix_agent
+    agents:
+      queue: libcxx-builders
+      os: aix
     <<: *common
 
 - group: ':freebsd: FreeBSD'
@@ -251,12 +265,16 @@ steps:
   steps:
   - label: Android 5.0, x86 NDK
     command: libcxx/utils/ci/run-buildbot android-ndk-21-def-x86
-    <<: *android_agent
+    agents:
+      queue: libcxx-builders
+      os: android
     <<: *common
 
   - label: Android 13, x86_64 NDK
     command: libcxx/utils/ci/run-buildbot android-ndk-33-goog-x86_64
-    <<: *android_agent
+    agents:
+      queue: libcxx-builders
+      os: android
     <<: *common
 
 
diff --git a/libcxx/utils/libcxx/test/modules.py b/libcxx/utils/libcxx/test/modules.py
index deaac450381c38761fce4b752a33e192db0d7a4d..bd19fac314dd9b58dfccb80f687d4c3b896ee89d 100644
--- a/libcxx/utils/libcxx/test/modules.py
+++ b/libcxx/utils/libcxx/test/modules.py
@@ -141,14 +141,6 @@ class module_test_generator:
                 f"#  include <{header}>{nl}"
                 f"#endif{nl}"
             )
-        elif header == "chrono":
-            # When localization is disabled the header string is not included.
-            # When string is included chrono's operator""s is a named declaration
-            #   using std::chrono_literals::operator""s;
-            # else it is a named declaration
-            #   using std::operator""s;
-            # TODO MODULES investigate why
-            include = f"#include <string>{nl}#include <chrono>{nl}"
         else:
             include = f"#include <{header}>{nl}"
 
diff --git a/libcxxabi/cmake/config-ix.cmake b/libcxxabi/cmake/config-ix.cmake
index 39b9284b780ebcf593df8321b5b068e082c6de7e..10f2087c68c5e79f6f360ae9b079a378f0fed89e 100644
--- a/libcxxabi/cmake/config-ix.cmake
+++ b/libcxxabi/cmake/config-ix.cmake
@@ -33,7 +33,9 @@ else()
   endif()
 endif()
 
-if (CXX_SUPPORTS_NOSTDLIBXX_FLAG OR C_SUPPORTS_NODEFAULTLIBS_FLAG)
+# Only link against compiler-rt manually if we use -nodefaultlibs, since
+# otherwise the compiler will do the right thing on its own.
+if (NOT CXX_SUPPORTS_NOSTDLIBXX_FLAG AND C_SUPPORTS_NODEFAULTLIBS_FLAG)
   if (LIBCXXABI_HAS_C_LIB)
     list(APPEND CMAKE_REQUIRED_LIBRARIES c)
   endif ()
@@ -67,6 +69,9 @@ if (CXX_SUPPORTS_NOSTDLIBXX_FLAG OR C_SUPPORTS_NODEFAULTLIBS_FLAG)
                         moldname mingwex msvcrt)
     list(APPEND CMAKE_REQUIRED_LIBRARIES ${MINGW_LIBRARIES})
   endif()
+endif()
+
+if (CXX_SUPPORTS_NOSTDLIBXX_FLAG OR C_SUPPORTS_NODEFAULTLIBS_FLAG)
   if (CMAKE_C_FLAGS MATCHES -fsanitize OR CMAKE_CXX_FLAGS MATCHES -fsanitize)
     set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -fno-sanitize=all")
   endif ()
diff --git a/libcxxabi/src/CMakeLists.txt b/libcxxabi/src/CMakeLists.txt
index 02031b69fd9156ac2ce6e16cbe12ce7c931cc74d..4198827203fc8b314353bef1a53065725375967c 100644
--- a/libcxxabi/src/CMakeLists.txt
+++ b/libcxxabi/src/CMakeLists.txt
@@ -166,7 +166,10 @@ if (LIBCXXABI_USE_LLVM_UNWINDER)
     target_link_libraries(cxxabi_shared_objects PUBLIC unwind_shared)
   endif()
 endif()
-target_link_libraries(cxxabi_shared_objects PRIVATE cxx-headers ${LIBCXXABI_BUILTINS_LIBRARY} ${LIBCXXABI_LIBRARIES})
+target_link_libraries(cxxabi_shared_objects PRIVATE cxx-headers ${LIBCXXABI_LIBRARIES})
+if (NOT CXX_SUPPORTS_NOSTDLIBXX_FLAG)
+  target_link_libraries(cxxabi_shared_objects PRIVATE ${LIBCXXABI_BUILTINS_LIBRARY})
+endif()
 target_link_libraries(cxxabi_shared_objects PUBLIC cxxabi-headers)
 set_target_properties(cxxabi_shared_objects
   PROPERTIES
diff --git a/libcxxabi/test/native/x86_64/lpstart-zero.pass.sh.s b/libcxxabi/test/native/x86_64/lpstart-zero.pass.sh.s
index ea9dd0104c086bddf780fa59113e8363be4c0596..e18134cf886391de1263b39fd7f2ff1fcdc1dc9e 100644
--- a/libcxxabi/test/native/x86_64/lpstart-zero.pass.sh.s
+++ b/libcxxabi/test/native/x86_64/lpstart-zero.pass.sh.s
@@ -1,7 +1,8 @@
 # RUN: %{cxx} %{flags} %s %{link_flags} -no-pie -o %t.exe
-# RUN: %t.exe
+# RUN: %{exec} %t.exe
 
 # REQUIRES: linux && target={{x86_64-.+}}
+# UNSUPPORTED: target={{.+-android.*}}
 # UNSUPPORTED: no-exceptions
 
 ## Check that libc++abi works correctly when LPStart address is explicitly set
@@ -82,7 +83,7 @@ GCC_except_table0:
 	.byte	155                             # @TType Encoding = indirect pcrel sdata4
 	.uleb128 .Lttbase0-.Lttbaseref0
 .Lttbaseref0:
-	.byte	11                              # Call site Encoding = udata4
+	.byte	11                              # Call site Encoding = sdata4
 	.uleb128 .Lcst_end0-.Lcst_begin0
 .Lcst_begin0:
 	.long .Lfunc_begin0-.Lfunc_begin0    # >> Call Site 1 <<
diff --git a/libunwind/cmake/config-ix.cmake b/libunwind/cmake/config-ix.cmake
index 7df8974b0a9254006d0eb9893264a90f894afef4..126c872f0d4895271d818d66587ed55f01f275c3 100644
--- a/libunwind/cmake/config-ix.cmake
+++ b/libunwind/cmake/config-ix.cmake
@@ -45,7 +45,9 @@ else()
   endif()
 endif()
 
-if (CXX_SUPPORTS_NOSTDLIBXX_FLAG OR C_SUPPORTS_NODEFAULTLIBS_FLAG)
+# Only link against compiler-rt manually if we use -nodefaultlibs, since
+# otherwise the compiler will do the right thing on its own.
+if (NOT CXX_SUPPORTS_NOSTDLIBXX_FLAG AND C_SUPPORTS_NODEFAULTLIBS_FLAG)
   if (LIBUNWIND_HAS_C_LIB)
     list(APPEND CMAKE_REQUIRED_LIBRARIES c)
   endif ()
@@ -78,6 +80,9 @@ if (CXX_SUPPORTS_NOSTDLIBXX_FLAG OR C_SUPPORTS_NODEFAULTLIBS_FLAG)
                         moldname mingwex msvcrt)
     list(APPEND CMAKE_REQUIRED_LIBRARIES ${MINGW_LIBRARIES})
   endif()
+endif()
+
+if (CXX_SUPPORTS_NOSTDLIBXX_FLAG OR C_SUPPORTS_NODEFAULTLIBS_FLAG)
   if (CMAKE_C_FLAGS MATCHES -fsanitize OR CMAKE_CXX_FLAGS MATCHES -fsanitize)
     set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -fno-sanitize=all")
   endif ()
diff --git a/lld/COFF/Driver.cpp b/lld/COFF/Driver.cpp
index 99c1a60735adc5affca148847a756ee8c6ff765c..d4a2f5767a2e871e63cb7b5b19660a9a40512d4f 100644
--- a/lld/COFF/Driver.cpp
+++ b/lld/COFF/Driver.cpp
@@ -1940,6 +1940,7 @@ void LinkerDriver::linkerMain(ArrayRef<const char *> argsArr) {
   parseMerge(".didat=.rdata");
   parseMerge(".edata=.rdata");
   parseMerge(".xdata=.rdata");
+  parseMerge(".00cfg=.rdata");
   parseMerge(".bss=.data");
 
   if (config->mingw) {
diff --git a/lld/ELF/Arch/ARM.cpp b/lld/ELF/Arch/ARM.cpp
index c10ffebc166640424095950a4605598f49eea373..d34e74a11c6d8d8bc679af323e8ca610fba80a46 100644
--- a/lld/ELF/Arch/ARM.cpp
+++ b/lld/ELF/Arch/ARM.cpp
@@ -948,15 +948,15 @@ int64_t ARM::getImplicitAddend(const uint8_t *buf, RelType type) const {
 }
 
 static bool isArmMapSymbol(const Symbol *b) {
-  return b->getName() == "$a" || b->getName().startswith("$a.");
+  return b->getName() == "$a" || b->getName().starts_with("$a.");
 }
 
 static bool isThumbMapSymbol(const Symbol *s) {
-  return s->getName() == "$t" || s->getName().startswith("$t.");
+  return s->getName() == "$t" || s->getName().starts_with("$t.");
 }
 
 static bool isDataMapSymbol(const Symbol *b) {
-  return b->getName() == "$d" || b->getName().startswith("$d.");
+  return b->getName() == "$d" || b->getName().starts_with("$d.");
 }
 
 void elf::sortArmMappingSymbols() {
@@ -1189,7 +1189,7 @@ void elf::processArmCmseSymbols() {
   // Only symbols with external linkage end up in symtab, so no need to do
   // linkage checks. Only check symbol type.
   for (Symbol *acleSeSym : symtab.getSymbols()) {
-    if (!acleSeSym->getName().startswith(ACLESESYM_PREFIX))
+    if (!acleSeSym->getName().starts_with(ACLESESYM_PREFIX))
       continue;
     // If input object build attributes do not support CMSE, error and disable
     // further scanning for <sym>, __acle_se_<sym> pairs.
diff --git a/lld/test/COFF/Inputs/loadconfig-arm64ec.s b/lld/test/COFF/Inputs/loadconfig-arm64ec.s
index 8bb5ccfed8ebc8739ebb2c1cd2eade725aed44da..a270d281095dd6b687d305a16b6b334a858af548 100644
--- a/lld/test/COFF/Inputs/loadconfig-arm64ec.s
+++ b/lld/test/COFF/Inputs/loadconfig-arm64ec.s
@@ -1,4 +1,4 @@
-        .section .rdata,"dr"
+        .section .00cfg,"dr"
         .globl _load_config_used
         .p2align 3, 0
 _load_config_used:
diff --git a/lld/test/COFF/Inputs/loadconfig-cfg-x64.s b/lld/test/COFF/Inputs/loadconfig-cfg-x64.s
index 1440b115f46ad40f5ee03113d6915e3e5e37b867..349d8c5a8db36e3152bcf8716b8e7134ba6e1434 100644
--- a/lld/test/COFF/Inputs/loadconfig-cfg-x64.s
+++ b/lld/test/COFF/Inputs/loadconfig-cfg-x64.s
@@ -1,6 +1,6 @@
 # This is the _load_config_used definition needed for /guard:cf tests.
 
-        .section .rdata,"dr"
+        .section .00cfg,"dr"
 .globl _load_config_used
 _load_config_used:
         .long 256
diff --git a/lld/test/COFF/merge-00cfg.s b/lld/test/COFF/merge-00cfg.s
new file mode 100644
index 0000000000000000000000000000000000000000..89a4c8f08342109136411f8e1b2c2952554e4970
--- /dev/null
+++ b/lld/test/COFF/merge-00cfg.s
@@ -0,0 +1,17 @@
+// REQUIRES: x86
+
+// RUN: llvm-mc -filetype=obj -triple=x86_64-windows %s -o %t-x86_64.obj
+// RUN: llvm-mc -filetype=obj -triple=i686-windows %s -o %t-x86.obj
+// RUN: lld-link -machine:amd64 -out:%t-x86_64.dll %t-x86_64.obj -dll -noentry
+// RUN: lld-link -machine:x86 -out:%t-x86.dll %t-x86.obj -dll -noentry -safeseh:no
+
+// RUN: llvm-readobj --hex-dump=.rdata %t-x86_64.dll | FileCheck %s -check-prefix=RDATA
+// RUN: llvm-readobj --hex-dump=.rdata %t-x86.dll | FileCheck %s -check-prefix=RDATA
+// RDATA: 78563412
+
+// RUN: llvm-readobj --sections %t-x86_64.dll | FileCheck %s -check-prefix=SECTIONS
+// RUN: llvm-readobj --sections %t-x86.dll | FileCheck %s -check-prefix=SECTIONS
+// SECTIONS-NOT: .00cfg
+
+        .section ".00cfg", "dr"
+        .long 0x12345678
diff --git a/lld/test/ELF/linkerscript/provide-shared.s b/lld/test/ELF/linkerscript/provide-shared.s
index 31a297f0991bdaf920e3f4c17963197768c035b2..96ad703d7a369485848a975edb9bf20cca4bb58a 100644
--- a/lld/test/ELF/linkerscript/provide-shared.s
+++ b/lld/test/ELF/linkerscript/provide-shared.s
@@ -1,13 +1,33 @@
 # REQUIRES: x86
-# RUN: llvm-mc -filetype=obj -triple=x86_64-pc-linux %s -o %t.o
-# RUN: llvm-mc -filetype=obj -triple=x86_64-pc-linux %p/Inputs/provide-shared.s -o %t2.o
-# RUN: ld.lld %t2.o -o %t2.so -shared
-# RUN: echo "SECTIONS { . = . + SIZEOF_HEADERS; PROVIDE(foo = 42);}" > %t.script
-# RUN: ld.lld -o %t --script %t.script %t.o %t2.so
-# RUN: llvm-readelf -s %t | FileCheck  %s
+# RUN: rm -rf %t && split-file %s %t && cd %t
+# RUN: llvm-mc -filetype=obj -triple=x86_64 a.s -o a.o
+# RUN: llvm-mc -filetype=obj -triple=x86_64 b.s -o b.o
+# RUN: ld.lld b.o -o b.so -shared -soname=b.so
+# RUN: ld.lld -T a.t a.o b.so -o a
+# RUN: llvm-readelf -s a | FileCheck  %s
 
-# CHECK: 000000000000002a 0 NOTYPE GLOBAL DEFAULT ABS foo
+# CHECK: 000000000000002a 0 NOTYPE GLOBAL DEFAULT ABS f1
+# CHECK: 000000000000002a 0 NOTYPE GLOBAL DEFAULT ABS f2
 
+#--- a.s
 .global _start
 _start:
-.quad foo
+.quad f1
+
+#--- b.s
+.global f1
+.size f1, 8
+.type f1, @object
+f1:
+.quad 42
+
+.global f2
+.data
+.dc.a f2
+
+#--- a.t
+SECTIONS {
+  . = . + SIZEOF_HEADERS;
+  PROVIDE(f1 = 42);
+  PROVIDE(f2 = 42);
+}
diff --git a/lld/test/ELF/linkerscript/provide-shared2.s b/lld/test/ELF/linkerscript/provide-shared2.s
deleted file mode 100644
index 3c55d2fa6318c3bb5d8f465f0830bcf6a7ed687f..0000000000000000000000000000000000000000
--- a/lld/test/ELF/linkerscript/provide-shared2.s
+++ /dev/null
@@ -1,13 +0,0 @@
-# REQUIRES: x86
-# RUN: llvm-mc -filetype=obj -triple=x86_64-pc-linux %s -o %t.o
-# RUN: llvm-mc -filetype=obj -triple=x86_64-pc-linux %p/Inputs/provide-shared2.s -o %t2.o
-# RUN: ld.lld %t2.o -o %t2.so -shared
-# RUN: echo "SECTIONS { . = . + SIZEOF_HEADERS; PROVIDE(foo = 42); }" > %t.script
-# RUN: ld.lld -o %t --script %t.script %t.o %t2.so
-# RUN: llvm-readelf --dyn-symbols %t | FileCheck  %s
-
-# CHECK: 1: 000000000000002a 0 NOTYPE GLOBAL DEFAULT ABS foo
-
-.global _start
-_start:
-        nop
diff --git a/lld/test/ELF/linkerscript/symbolreferenced.s b/lld/test/ELF/linkerscript/symbolreferenced.s
index f6d9a1214fdd9d4155db062435bab23ab5487c8c..ba7a7721ea969706725f35256d929c51ebf5751a 100644
--- a/lld/test/ELF/linkerscript/symbolreferenced.s
+++ b/lld/test/ELF/linkerscript/symbolreferenced.s
@@ -1,22 +1,36 @@
 # REQUIRES: x86
-# RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %s -o %t
+# RUN: rm -rf %t && split-file %s %t && cd %t
+# RUN: llvm-mc -filetype=obj -triple=x86_64 a.s -o a.o
 
 # Provide new symbol. The value should be 1, like set in PROVIDE()
-# RUN: echo "SECTIONS { PROVIDE(newsym = 1);}" > %t.script
-# RUN: ld.lld -o %t1 --script %t.script %t
-# RUN: llvm-objdump -t %t1 | FileCheck --check-prefix=PROVIDE1 %s
+# RUN: echo "SECTIONS { PROVIDE(newsym = 1);}" > a1.t
+# RUN: ld.lld -o a1 -T a1.t a.o
+# RUN: llvm-objdump -t a1 | FileCheck --check-prefix=PROVIDE1 %s
 # PROVIDE1: 0000000000000001 g       *ABS*  0000000000000000 newsym
 
 # Provide new symbol (hidden). The value should be 1
-# RUN: echo "SECTIONS { PROVIDE_HIDDEN(newsym = 1);}" > %t.script
-# RUN: ld.lld -o %t1 --script %t.script %t
-# RUN: llvm-objdump -t %t1 | FileCheck --check-prefix=HIDDEN1 %s
+# RUN: echo "SECTIONS { PROVIDE_HIDDEN(newsym = 1);}" > a2.t
+# RUN: ld.lld -o a2 -T a2.t a.o
+# RUN: llvm-objdump -t a2 | FileCheck --check-prefix=HIDDEN1 %s
 # HIDDEN1: 0000000000000001 l       *ABS*  0000000000000000 .hidden newsym
 
-# RUN: echo 'SECTIONS { PROVIDE_HIDDEN("newsym" = 1);}' > %t.script
-# RUN: ld.lld -o %t1 --script %t.script %t
-# RUN: llvm-objdump -t %t1 | FileCheck --check-prefix=HIDDEN1 %s
+# RUN: echo 'SECTIONS { PROVIDE_HIDDEN("newsym" = 1);}' > a2.t
+# RUN: ld.lld -o a2 -T a2.t a.o
+# RUN: llvm-objdump -t a2 | FileCheck --check-prefix=HIDDEN1 %s
 
+# RUN: ld.lld -o chain -T chain.t a.o
+# RUN: llvm-nm chain | FileCheck %s
+
+# CHECK:      0000000000001000 a f1
+# CHECK-NEXT: 0000000000001000 A f2
+# CHECK-NEXT: 0000000000001000 a g1
+# CHECK-NEXT: 0000000000001000 A g2
+# CHECK-NEXT: 0000000000001000 A newsym
+
+# RUN: not ld.lld -T chain2.t a.o 2>&1 | FileCheck %s --check-prefix=ERR --implicit-check-not=error:
+# ERR-COUNT-3: error: chain2.t:1: symbol not found: undef
+
+#--- a.s
 .global _start
 _start:
  nop
@@ -24,3 +38,17 @@ _start:
 .globl patatino
 patatino:
   movl newsym, %eax
+
+#--- chain.t
+PROVIDE(f2 = 0x1000);
+PROVIDE_HIDDEN(f1 = f2);
+PROVIDE(newsym = f1);
+
+PROVIDE(g2 = 0x1000);
+PROVIDE_HIDDEN(g1 = g2);
+PROVIDE(unused = g1);
+
+#--- chain2.t
+PROVIDE(f2 = undef);
+PROVIDE_HIDDEN(f1 = f2);
+PROVIDE(newsym = f1);
diff --git a/lld/test/ELF/ppc32-reloc-addr.s b/lld/test/ELF/ppc32-reloc-addr.s
index 78865b82ccbf1d4b2e9c3674feaa81fd6c749daf..9b40bce77aec485595d61a7893ba5d189f6dd7cd 100644
--- a/lld/test/ELF/ppc32-reloc-addr.s
+++ b/lld/test/ELF/ppc32-reloc-addr.s
@@ -22,7 +22,7 @@
 .section .R_PPC_ADDR24,"ax",@progbits
   ba a
 # CHECK-LABEL: section .R_PPC_ADDR24:
-# CHECK: ba 4660
+# CHECK: ba 0x1234
 
 .section .R_PPC_ADDR32,"a",@progbits
   .long a
diff --git a/lld/wasm/Driver.cpp b/lld/wasm/Driver.cpp
index f8b0fc357bd90c23ddb7aeaec46b517c4578ec78..c68fe33a14e29d6a76a6e613217ee5fe2b2b6052 100644
--- a/lld/wasm/Driver.cpp
+++ b/lld/wasm/Driver.cpp
@@ -606,7 +606,6 @@ static void setConfigs() {
       config->memoryImport =
           std::pair<llvm::StringRef, llvm::StringRef>(defaultModule, memoryName);
     }
-    config->importUndefined = true;
   }
 
   // If neither export-memory nor import-memory is specified, default to
diff --git a/lldb/include/lldb/Core/Module.h b/lldb/include/lldb/Core/Module.h
index 2973ee0e7ec4ac190309ef5b0c00274cf032bcb5..f4973cdda1efcc0de433cb29b4ad3f2c987cd759 100644
--- a/lldb/include/lldb/Core/Module.h
+++ b/lldb/include/lldb/Core/Module.h
@@ -415,70 +415,19 @@ public:
   void FindGlobalVariables(const RegularExpression &regex, size_t max_matches,
                            VariableList &variable_list);
 
-  /// Find types by name.
-  ///
-  /// Type lookups in modules go through the SymbolFile. The SymbolFile needs to
-  /// be able to lookup types by basename and not the fully qualified typename.
-  /// This allows the type accelerator tables to stay small, even with heavily
-  /// templatized C++. The type search will then narrow down the search
-  /// results. If "exact_match" is true, then the type search will only match
-  /// exact type name matches. If "exact_match" is false, the type will match
-  /// as long as the base typename matches and as long as any immediate
-  /// containing namespaces/class scopes that are specified match. So to
-  /// search for a type "d" in "b::c", the name "b::c::d" can be specified and
-  /// it will match any class/namespace "b" which contains a class/namespace
-  /// "c" which contains type "d". We do this to allow users to not always
-  /// have to specify complete scoping on all expressions, but it also allows
-  /// for exact matching when required.
-  ///
-  /// \param[in] type_name
-  ///     The name of the type we are looking for that is a fully
-  ///     or partially qualified type name.
-  ///
-  /// \param[in] exact_match
-  ///     If \b true, \a type_name is fully qualified and must match
-  ///     exactly. If \b false, \a type_name is a partially qualified
-  ///     name where the leading namespaces or classes can be
-  ///     omitted to make finding types that a user may type
-  ///     easier.
-  ///
-  /// \param[out] types
-  ///     A type list gets populated with any matches.
+  /// Find types using a type-matching object that contains all search
+  /// parameters.
   ///
-  void
-  FindTypes(ConstString type_name, bool exact_match, size_t max_matches,
-            llvm::DenseSet<lldb_private::SymbolFile *> &searched_symbol_files,
-            TypeList &types);
-
-  /// Find types by name.
-  ///
-  /// This behaves like the other FindTypes method but allows to
-  /// specify a DeclContext and a language for the type being searched
-  /// for.
-  ///
-  /// \param searched_symbol_files
-  ///     Prevents one file from being visited multiple times.
-  void
-  FindTypes(llvm::ArrayRef<CompilerContext> pattern, LanguageSet languages,
-            llvm::DenseSet<lldb_private::SymbolFile *> &searched_symbol_files,
-            TypeMap &types);
-
-  lldb::TypeSP FindFirstType(const SymbolContext &sc, ConstString type_name,
-                             bool exact_match);
-
-  /// Find types by name that are in a namespace. This function is used by the
-  /// expression parser when searches need to happen in an exact namespace
-  /// scope.
+  /// \see lldb_private::TypeQuery
   ///
-  /// \param[in] type_name
-  ///     The name of a type within a namespace that should not include
-  ///     any qualifying namespaces (just a type basename).
+  /// \param[in] query
+  ///     A type matching object that contains all of the details of the type
+  ///     search.
   ///
-  /// \param[out] type_list
-  ///     A type list gets populated with any matches.
-  void FindTypesInNamespace(ConstString type_name,
-                            const CompilerDeclContext &parent_decl_ctx,
-                            size_t max_matches, TypeList &type_list);
+  /// \param[in] results
+  ///     Any matching types will be populated into the \a results object using
+  ///     TypeMap::InsertUnique(...).
+  void FindTypes(const TypeQuery &query, TypeResults &results);
 
   /// Get const accessor for the module architecture.
   ///
@@ -1122,12 +1071,6 @@ protected:
 private:
   Module(); // Only used internally by CreateJITModule ()
 
-  void FindTypes_Impl(
-      ConstString name, const CompilerDeclContext &parent_decl_ctx,
-      size_t max_matches,
-      llvm::DenseSet<lldb_private::SymbolFile *> &searched_symbol_files,
-      TypeMap &types);
-
   Module(const Module &) = delete;
   const Module &operator=(const Module &) = delete;
 
diff --git a/lldb/include/lldb/Core/ModuleList.h b/lldb/include/lldb/Core/ModuleList.h
index 9826dd09e91d714088505e69477759a250de41f6..d78f7c5ef3f7064c5a1ab2edb1ee9644f2a9ec7f 100644
--- a/lldb/include/lldb/Core/ModuleList.h
+++ b/lldb/include/lldb/Core/ModuleList.h
@@ -340,26 +340,22 @@ public:
                                        lldb::SymbolType symbol_type,
                                        SymbolContextList &sc_list) const;
 
-  /// Find types by name.
+  /// Find types using a type-matching object that contains all search
+  /// parameters.
   ///
   /// \param[in] search_first
   ///     If non-null, this module will be searched before any other
   ///     modules.
   ///
-  /// \param[in] name
-  ///     The name of the type we are looking for.
-  ///
-  /// \param[in] max_matches
-  ///     Allow the number of matches to be limited to \a
-  ///     max_matches. Specify UINT32_MAX to get all possible matches.
-  ///
-  /// \param[out] types
-  ///     A type list gets populated with any matches.
+  /// \param[in] query
+  ///     A type matching object that contains all of the details of the type
+  ///     search.
   ///
-  void FindTypes(Module *search_first, ConstString name,
-                 bool name_is_fully_qualified, size_t max_matches,
-                 llvm::DenseSet<SymbolFile *> &searched_symbol_files,
-                 TypeList &types) const;
+  /// \param[in] results
+  ///     Any matching types will be populated into the \a results object using
+  ///     TypeMap::InsertUnique(...).
+  void FindTypes(Module *search_first, const TypeQuery &query,
+                 lldb_private::TypeResults &results) const;
 
   bool FindSourceFile(const FileSpec &orig_spec, FileSpec &new_spec) const;
 
diff --git a/lldb/include/lldb/Core/ValueObject.h b/lldb/include/lldb/Core/ValueObject.h
index 20b3086138457f712c4fe7895d6e258cdcd254dc..a158199e7fab1a89ae2e81d158b9cc4aacc70cbd 100644
--- a/lldb/include/lldb/Core/ValueObject.h
+++ b/lldb/include/lldb/Core/ValueObject.h
@@ -468,7 +468,7 @@ public:
   virtual lldb::ValueObjectSP GetChildAtIndex(size_t idx,
                                               bool can_create = true);
 
-  // this will always create the children if necessary
+  // The method always creates missing children in the path, if necessary.
   lldb::ValueObjectSP GetChildAtIndexPath(llvm::ArrayRef<size_t> idxs,
                                           size_t *index_of_error = nullptr);
 
@@ -476,7 +476,7 @@ public:
   GetChildAtIndexPath(llvm::ArrayRef<std::pair<size_t, bool>> idxs,
                       size_t *index_of_error = nullptr);
 
-  // this will always create the children if necessary
+  // The method always creates missing children in the path, if necessary.
   lldb::ValueObjectSP GetChildAtNamePath(llvm::ArrayRef<llvm::StringRef> names);
 
   virtual lldb::ValueObjectSP GetChildMemberWithName(llvm::StringRef name,
diff --git a/lldb/include/lldb/Symbol/CompilerDecl.h b/lldb/include/lldb/Symbol/CompilerDecl.h
index 67290b9be0663a3739e9d7bc415f9e0b0fab8268..825a4f15836fce8aa7bca038cc54a67ffe1fd02d 100644
--- a/lldb/include/lldb/Symbol/CompilerDecl.h
+++ b/lldb/include/lldb/Symbol/CompilerDecl.h
@@ -84,6 +84,13 @@ public:
   // based argument index
   CompilerType GetFunctionArgumentType(size_t arg_idx) const;
 
+  /// Populate a valid compiler context from the current declaration.
+  ///
+  /// \returns A valid vector of CompilerContext entries that describes
+  /// this declaration. The first entry in the vector is the parent of
+  /// the subsequent entry, so the topmost entry is the global namespace.
+  std::vector<lldb_private::CompilerContext> GetCompilerContext() const;
+
 private:
   TypeSystem *m_type_system = nullptr;
   void *m_opaque_decl = nullptr;
diff --git a/lldb/include/lldb/Symbol/CompilerDeclContext.h b/lldb/include/lldb/Symbol/CompilerDeclContext.h
index 61a9c9c341bfe13f223e233413901311cb7e8189..89b4a9787688bca1b84634b5926f4ad22a9bbc8c 100644
--- a/lldb/include/lldb/Symbol/CompilerDeclContext.h
+++ b/lldb/include/lldb/Symbol/CompilerDeclContext.h
@@ -11,6 +11,7 @@
 
 #include <vector>
 
+#include "lldb/Symbol/Type.h"
 #include "lldb/Utility/ConstString.h"
 #include "lldb/lldb-private.h"
 
@@ -56,6 +57,13 @@ public:
     return m_type_system != nullptr && m_opaque_decl_ctx != nullptr;
   }
 
+  /// Populate a valid compiler context from the current decl context.
+  ///
+  /// \returns A valid vector of CompilerContext entries that describes
+  /// this declaration context. The first entry in the vector is the parent of
+  /// the subsequent entry, so the topmost entry is the global namespace.
+  std::vector<lldb_private::CompilerContext> GetCompilerContext() const;
+
   std::vector<CompilerDecl> FindDeclByName(ConstString name,
                                            const bool ignore_using_decls);
 
diff --git a/lldb/include/lldb/Symbol/SymbolFile.h b/lldb/include/lldb/Symbol/SymbolFile.h
index a546b05bfd31811a44a09f0acbdb4aebed3d1d92..c9a2a647a039dcd222f7874fdbb4aadbe32995b5 100644
--- a/lldb/include/lldb/Symbol/SymbolFile.h
+++ b/lldb/include/lldb/Symbol/SymbolFile.h
@@ -301,21 +301,20 @@ public:
                              bool include_inlines, SymbolContextList &sc_list);
   virtual void FindFunctions(const RegularExpression &regex,
                              bool include_inlines, SymbolContextList &sc_list);
-  virtual void
-  FindTypes(ConstString name, const CompilerDeclContext &parent_decl_ctx,
-            uint32_t max_matches,
-            llvm::DenseSet<lldb_private::SymbolFile *> &searched_symbol_files,
-            TypeMap &types);
-
-  /// Find types specified by a CompilerContextPattern.
-  /// \param languages
-  ///     Only return results in these languages.
-  /// \param searched_symbol_files
-  ///     Prevents one file from being visited multiple times.
-  virtual void
-  FindTypes(llvm::ArrayRef<CompilerContext> pattern, LanguageSet languages,
-            llvm::DenseSet<lldb_private::SymbolFile *> &searched_symbol_files,
-            TypeMap &types);
+
+  /// Find types using a type-matching object that contains all search
+  /// parameters.
+  ///
+  /// \see lldb_private::TypeQuery
+  ///
+  /// \param[in] query
+  ///     A type matching object that contains all of the details of the type
+  ///     search.
+  ///
+  /// \param[in] results
+  ///     Any matching types will be populated into the \a results object using
+  ///     TypeMap::InsertUnique(...).
+  virtual void FindTypes(const TypeQuery &query, TypeResults &results) {}
 
   virtual void
   GetMangledNamesForFunction(const std::string &scope_qualified_name,
diff --git a/lldb/include/lldb/Symbol/SymbolFileOnDemand.h b/lldb/include/lldb/Symbol/SymbolFileOnDemand.h
index 9cbcef2a111d32001c6865a2aaf616c4f6777609..cde9f3c3b8ce1f1c63de08d404e637a3f575c519 100644
--- a/lldb/include/lldb/Symbol/SymbolFileOnDemand.h
+++ b/lldb/include/lldb/Symbol/SymbolFileOnDemand.h
@@ -152,17 +152,8 @@ public:
       const std::string &scope_qualified_name,
       std::vector<lldb_private::ConstString> &mangled_names) override;
 
-  void
-  FindTypes(lldb_private::ConstString name,
-            const lldb_private::CompilerDeclContext &parent_decl_ctx,
-            uint32_t max_matches,
-            llvm::DenseSet<lldb_private::SymbolFile *> &searched_symbol_files,
-            lldb_private::TypeMap &types) override;
-
-  void FindTypes(llvm::ArrayRef<lldb_private::CompilerContext> pattern,
-                 lldb_private::LanguageSet languages,
-                 llvm::DenseSet<SymbolFile *> &searched_symbol_files,
-                 lldb_private::TypeMap &types) override;
+  void FindTypes(const lldb_private::TypeQuery &query,
+                 lldb_private::TypeResults &results) override;
 
   void GetTypes(lldb_private::SymbolContextScope *sc_scope,
                 lldb::TypeClass type_mask,
diff --git a/lldb/include/lldb/Symbol/Type.h b/lldb/include/lldb/Symbol/Type.h
index 15edbea3cc7ae73be74bb17565b7ea213b1b2caa..307be6c55e01613975e9163f958e685c522cc6e1 100644
--- a/lldb/include/lldb/Symbol/Type.h
+++ b/lldb/include/lldb/Symbol/Type.h
@@ -12,11 +12,15 @@
 #include "lldb/Core/Declaration.h"
 #include "lldb/Symbol/CompilerDecl.h"
 #include "lldb/Symbol/CompilerType.h"
+#include "lldb/Symbol/TypeList.h"
+#include "lldb/Symbol/TypeMap.h"
+#include "lldb/Symbol/TypeSystem.h"
 #include "lldb/Utility/ConstString.h"
 #include "lldb/Utility/UserID.h"
 #include "lldb/lldb-private.h"
 
 #include "llvm/ADT/APSInt.h"
+#include "llvm/ADT/DenseSet.h"
 
 #include <optional>
 #include <set>
@@ -24,6 +28,23 @@
 namespace lldb_private {
 class SymbolFileCommon;
 
+/// A SmallBitVector that represents a set of source languages (\p
+/// lldb::LanguageType).  Each lldb::LanguageType is represented by
+/// the bit with the position of its enumerator. The largest
+/// LanguageType is < 64, so this is space-efficient and on 64-bit
+/// architectures a LanguageSet can be completely stack-allocated.
+struct LanguageSet {
+  llvm::SmallBitVector bitvector;
+  LanguageSet();
+
+  /// If the set contains a single language only, return it.
+  std::optional<lldb::LanguageType> GetSingularLanguage();
+  void Insert(lldb::LanguageType language);
+  bool Empty() const;
+  size_t Size() const;
+  bool operator[](unsigned i) const;
+};
+
 /// CompilerContext allows an array of these items to be passed to perform
 /// detailed lookups in SymbolVendor and SymbolFile functions.
 struct CompilerContext {
@@ -45,6 +66,290 @@ struct CompilerContext {
 bool contextMatches(llvm::ArrayRef<CompilerContext> context_chain,
                     llvm::ArrayRef<CompilerContext> pattern);
 
+FLAGS_ENUM(TypeQueryOptions){
+    e_none = 0u,
+    /// If set, TypeQuery::m_context contains an exact context that must match
+    /// the full context. If not set, TypeQuery::m_context can contain a partial
+    /// type match where the full context isn't fully specified.
+    e_exact_match = (1u << 0),
+    /// If set, TypeQuery::m_context is a clang module compiler context. If not
+    /// set TypeQuery::m_context is normal type lookup context.
+    e_module_search = (1u << 1),
+    /// When true, the find types call should stop the query as soon as a single
+    /// matching type is found. When false, the type query should find all
+    /// matching types.
+    e_find_one = (1u << 2),
+};
+LLDB_MARK_AS_BITMASK_ENUM(TypeQueryOptions)
+
+/// A class that contains all state required for type lookups.
+///
+/// Using a TypeQuery class for matching types simplifies the internal APIs we
+/// need to implement type lookups in LLDB. Type lookups can fully specify the
+/// exact typename by filling out a complete or partial CompilerContext array.
+/// This technique allows for powerful searches and also allows the SymbolFile
+/// classes to use the m_context array to lookup types by basename, then
+/// eliminate potential matches without having to resolve types into each
+/// TypeSystem. This makes type lookups vastly more efficient and allows the
+/// SymbolFile objects to stop looking up types when the type matching is
+/// complete, like if we are looking for only a single type in our search.
+class TypeQuery {
+public:
+  TypeQuery() = delete;
+
+  /// Construct a type match object using a fully- or partially-qualified name.
+  ///
+  /// The specified \a type_name will be chopped up and the m_context will be
+  /// populated by separating the string by looking for "::". We do this because
+  /// symbol files have indexes that contain only the type's basename. This also
+  /// allows symbol files to efficiently not realize types that don't match the
+  /// specified context. Example of \a type_name values that can be specified
+  /// include:
+  ///   "foo": Look for any type whose basename matches "foo".
+  ///     If \a exact_match is true, then the type can't be contained in any
+  ///     declaration context like a namespace, class, or other containing
+  ///     scope.
+  ///     If \a exact match is false, then we will find all matches including
+  ///     ones that are contained in other declaration contexts, including top
+  ///     level types.
+  ///   "foo::bar": Look for any type whose basename matches "bar" but make sure
+  ///     its parent declaration context is any named declaration context
+  ///     (namespace, class, struct, etc) whose name matches "foo".
+  ///     If \a exact_match is true, then the "foo" declaration context must
+  ///     appear at the source file level or inside of a function.
+  ///     If \a exact match is false, then the "foo" declaration context can
+  ///     be contained in any other declaration contexts.
+  ///   "class foo": Only match types that are classes whose basename matches
+  ///     "foo".
+  ///   "struct foo": Only match types that are structures whose basename
+  ///     matches "foo".
+  ///   "class foo::bar": Only match types that are classes whose basename
+  ///     matches "bar" and that are contained in any named declaration context
+  ///     named "foo".
+  ///
+  /// \param[in] type_name
+  ///   A fully- or partially-qualified type name. This name will be parsed and
+  ///   broken up and the m_context will be populated with the various parts of
+  ///   the name. This typename can be prefixed with "struct ", "class ",
+  ///   "union", "enum " or "typedef " before the actual type name to limit the
+  ///   results of the types that match. The declaration context can be
+  ///   specified with the "::" string. For example, "a::b::my_type".
+  ///
+  /// \param[in] options A set of boolean enumeration flags from the
+  ///   TypeQueryOptions enumerations. \see TypeQueryOptions.
+  TypeQuery(llvm::StringRef name, TypeQueryOptions options = e_none);
+
+  /// Construct a type-match object that matches a type basename that exists
+  /// in the specified declaration context.
+  ///
+  /// This allows the m_context to be first populated using a declaration
+  /// context to exactly identify the containing declaration context of a type.
+  /// This can be used when you have a forward declaration to a type and you
+  /// need to search for its complete type.
+  ///
+  /// \param[in] decl_ctx
+  ///   A declaration context object that comes from a TypeSystem plug-in. This
+  ///   object will be asked to populate the array of CompilerContext objects
+  ///   by adding the top most declaration context first into the array and then
+  ///   adding any containing declaration contexts.
+  ///
+  /// \param[in] type_basename
+  ///   The basename of the type to lookup in the specified declaration context.
+  ///
+  /// \param[in] options A set of boolean enumeration flags from the
+  ///   TypeQueryOptions enumerations. \see TypeQueryOptions.
+  TypeQuery(const CompilerDeclContext &decl_ctx, ConstString type_basename,
+            TypeQueryOptions options = e_none);
+  /// Construct a type-match object using a compiler declaration that specifies
+  /// a typename and a declaration context to use when doing exact type lookups.
+  ///
+  /// This allows the m_context to be first populated using a type declaration.
+  /// The type declaration might have a declaration context and each TypeSystem
+  /// plug-in can populate the declaration context needed to perform an exact
+  /// lookup for a type.
+  /// This can be used when you have a forward declaration to a type and you
+  /// need to search for its complete type.
+  ///
+  /// \param[in] decl
+  ///   A type declaration context object that comes from a TypeSystem plug-in.
+  ///   This object will be asked to full the array of CompilerContext objects
+  ///   by adding the top most declaration context first into the array and then
+  ///   adding any containing declaration contexts, and ending with the exact
+  ///   typename and the kind of type it is (class, struct, union, enum, etc).
+  ///
+  /// \param[in] options A set of boolean enumeration flags from the
+  ///   TypeQueryOptions enumerations. \see TypeQueryOptions.
+  TypeQuery(const CompilerDecl &decl, TypeQueryOptions options = e_none);
+
+  /// Construct a type-match object using a CompilerContext array.
+  ///
+  /// Clients can manually create compiler contexts and use these to find
+  /// matches when searching for types. There are two types of contexts that
+  /// are supported when doing type searchs: type contexts and clang module
+  /// contexts. Type contexts have contexts that specify the type and its
+  /// containing declaration context like namespaces and classes. Clang module
+  /// contexts specify contexts more completely to find exact matches within
+  /// clang module debug information. They will include the modules that the
+  /// type is included in and any functions that the type might be defined in.
+  /// This allows very fine-grained type resolution.
+  ///
+  /// \param[in] context The compiler context to use when doing the search.
+  ///
+  /// \param[in] options A set of boolean enumeration flags from the
+  ///   TypeQueryOptions enumerations. \see TypeQueryOptions.
+  TypeQuery(const llvm::ArrayRef<lldb_private::CompilerContext> &context,
+            TypeQueryOptions options = e_none);
+
+  /// Construct a type-match object that duplicates all matching criterea,
+  /// but not any searched symbol files or the type map for matches. This allows
+  /// the m_context to be modified prior to performing another search.
+  TypeQuery(const TypeQuery &rhs) = default;
+  /// Assign a type-match object that duplicates all matching criterea,
+  /// but not any searched symbol files or the type map for matches. This allows
+  /// the m_context to be modified prior to performing another search.
+  TypeQuery &operator=(const TypeQuery &rhs) = default;
+
+  /// Check of a CompilerContext array from matching type from a symbol file
+  /// matches the \a m_context.
+  ///
+  /// \param[in] context
+  ///   A fully qualified CompilerContext array for a potential match that is
+  ///   created by the symbol file prior to trying to actually resolve a type.
+  ///
+  /// \returns
+  ///   True if the context matches, false if it doesn't. If e_exact_match
+  ///   is set in m_options, then \a context must exactly match \a m_context. If
+  ///   e_exact_match is not set, then the bottom m_context.size() objects in
+  ///   \a context must match. This allows SymbolFile objects the fill in a
+  ///   potential type basename match from the index into \a context, and see if
+  ///   it matches prior to having to resolve a lldb_private::Type object for
+  ///   the type from the index. This allows type parsing to be as efficient as
+  ///   possible and only realize the types that match the query.
+  bool
+  ContextMatches(llvm::ArrayRef<lldb_private::CompilerContext> context) const;
+
+  /// Get the type basename to use when searching the type indexes in each
+  /// SymbolFile object.
+  ///
+  /// Debug information indexes often contain indexes that track the basename
+  /// of types only, not a fully qualified path. This allows the indexes to be
+  /// smaller and allows for efficient lookups.
+  ///
+  /// \returns
+  ///   The type basename to use when doing lookups as a constant string.
+  ConstString GetTypeBasename() const;
+
+  /// Returns true if any matching languages have been specified in this type
+  /// matching object.
+  bool HasLanguage() const { return m_languages.has_value(); }
+
+  /// Add a language family to the list of languages that should produce a
+  /// match.
+  void AddLanguage(lldb::LanguageType language);
+
+  /// Check if the language matches any languages that have been added to this
+  /// match object.
+  ///
+  /// \returns
+  ///   True if no language have been specified, or if some language have been
+  ///   added using AddLanguage(...) and they match. False otherwise.
+  bool LanguageMatches(lldb::LanguageType language) const;
+
+  bool GetExactMatch() const { return (m_options & e_exact_match) != 0; }
+  /// The \a m_context can be used in two ways: normal types searching with
+  /// the context containing a stanadard declaration context for a type, or
+  /// with the context being more complete for exact matches in clang modules.
+  /// Set this to true if you wish to search for a type in clang module.
+  bool GetModuleSearch() const { return (m_options & e_module_search) != 0; }
+
+  /// Returns true if the type query is supposed to find only a single matching
+  /// type. Returns false if the type query should find all matches.
+  bool GetFindOne() const { return (m_options & e_find_one) != 0; }
+  void SetFindOne(bool b) {
+    if (b)
+      m_options |= e_find_one;
+    else
+      m_options &= (e_exact_match | e_find_one);
+  }
+
+  /// Access the internal compiler context array.
+  ///
+  /// Clients can use this to populate the context manually.
+  std::vector<lldb_private::CompilerContext> &GetContextRef() {
+    return m_context;
+  }
+
+protected:
+  /// A full or partial compiler context array where the parent declaration
+  /// contexts appear at the top of the array starting at index zero and the
+  /// last entry contains the type and name of the type we are looking for.
+  std::vector<lldb_private::CompilerContext> m_context;
+  /// An options bitmask that contains enabled options for the type query.
+  /// \see TypeQueryOptions.
+  TypeQueryOptions m_options;
+  /// If this variable has a value, then the language family must match at least
+  /// one of the specified languages. If this variable has no value, then the
+  /// language of the type doesn't need to match any types that are searched.
+  std::optional<LanguageSet> m_languages;
+};
+
+/// This class tracks the state and results of a \ref TypeQuery.
+///
+/// Any mutable state required for type lookups and the results are tracked in
+/// this object.
+class TypeResults {
+public:
+  /// Construct a type results object
+  TypeResults() = default;
+
+  /// When types that match a TypeQuery are found, this API is used to insert
+  /// the matching types.
+  ///
+  /// \return
+  ///   True if the type was added, false if the \a type_sp was already in the
+  ///   results.
+  bool InsertUnique(const lldb::TypeSP &type_sp);
+
+  /// Check if the type matching has found all of the matches that it needs.
+  bool Done(const TypeQuery &query) const;
+
+  /// Check if a SymbolFile object has already been searched by this type match
+  /// object.
+  ///
+  /// This function will add \a sym_file to the set of SymbolFile objects if it
+  /// isn't already in the set and return \a false. Returns true if \a sym_file
+  /// was already in the set and doesn't need to be searched.
+  ///
+  /// Any clients that search for types should first check that the symbol file
+  /// has not already been searched. If this function returns true, the type
+  /// search function should early return to avoid duplicating type searchihng
+  /// efforts.
+  ///
+  /// \param[in] sym_file
+  ///   A SymbolFile pointer that will be used to track which symbol files have
+  ///   already been searched.
+  ///
+  /// \returns
+  ///   True if the symbol file has been search already, false otherwise.
+  bool AlreadySearched(lldb_private::SymbolFile *sym_file);
+
+  /// Access the set of searched symbol files.
+  llvm::DenseSet<lldb_private::SymbolFile *> &GetSearchedSymbolFiles() {
+    return m_searched_symbol_files;
+  }
+
+  lldb::TypeSP GetFirstType() const { return m_type_map.FirstType(); }
+  TypeMap &GetTypeMap() { return m_type_map; }
+  const TypeMap &GetTypeMap() const { return m_type_map; }
+
+private:
+  /// Matching types get added to this map as type search continues.
+  TypeMap m_type_map;
+  /// This set is used to track and make sure we only perform lookups in a
+  /// symbol file one time.
+  llvm::DenseSet<lldb_private::SymbolFile *> m_searched_symbol_files;
+};
+
 class SymbolFileType : public std::enable_shared_from_this<SymbolFileType>,
                        public UserID {
 public:
diff --git a/lldb/include/lldb/Symbol/TypeMap.h b/lldb/include/lldb/Symbol/TypeMap.h
index c200ccb9844fdb5d776ab64b902e072eee1ccc58..433711875e55319be8b0a2f153df4f3f8a69b2c4 100644
--- a/lldb/include/lldb/Symbol/TypeMap.h
+++ b/lldb/include/lldb/Symbol/TypeMap.h
@@ -27,7 +27,7 @@ public:
   void Clear();
 
   void Dump(Stream *s, bool show_context,
-            lldb::DescriptionLevel level = lldb::eDescriptionLevelFull);
+            lldb::DescriptionLevel level = lldb::eDescriptionLevelFull) const;
 
   TypeMap FindTypes(ConstString name);
 
@@ -41,10 +41,12 @@ public:
 
   lldb::TypeSP GetTypeAtIndex(uint32_t idx);
 
+  lldb::TypeSP FirstType() const;
+
   typedef std::multimap<lldb::user_id_t, lldb::TypeSP> collection;
   typedef AdaptedIterable<collection, lldb::TypeSP, map_adapter> TypeIterable;
 
-  TypeIterable Types() { return TypeIterable(m_types); }
+  TypeIterable Types() const { return TypeIterable(m_types); }
 
   void ForEach(
       std::function<bool(const lldb::TypeSP &type_sp)> const &callback) const;
diff --git a/lldb/include/lldb/Symbol/TypeSystem.h b/lldb/include/lldb/Symbol/TypeSystem.h
index cd5004a3f34db42331ff6a5f3248880fb874e9e2..63829131556e87f6b9cb08a1b14f644936279a1e 100644
--- a/lldb/include/lldb/Symbol/TypeSystem.h
+++ b/lldb/include/lldb/Symbol/TypeSystem.h
@@ -26,6 +26,7 @@
 #include "lldb/Expression/Expression.h"
 #include "lldb/Symbol/CompilerDecl.h"
 #include "lldb/Symbol/CompilerDeclContext.h"
+#include "lldb/Symbol/Type.h"
 #include "lldb/lldb-private.h"
 
 class PDBASTParser;
@@ -43,23 +44,6 @@ namespace npdb {
   class PdbAstBuilder;
 } // namespace npdb
 
-/// A SmallBitVector that represents a set of source languages (\p
-/// lldb::LanguageType).  Each lldb::LanguageType is represented by
-/// the bit with the position of its enumerator. The largest
-/// LanguageType is < 64, so this is space-efficient and on 64-bit
-/// architectures a LanguageSet can be completely stack-allocated.
-struct LanguageSet {
-  llvm::SmallBitVector bitvector;
-  LanguageSet();
-
-  /// If the set contains a single language only, return it.
-  std::optional<lldb::LanguageType> GetSingularLanguage();
-  void Insert(lldb::LanguageType language);
-  bool Empty() const;
-  size_t Size() const;
-  bool operator[](unsigned i) const;
-};
-
 /// Interface for representing a type system.
 ///
 /// Implemented by language plugins to define the type system for a given
@@ -122,6 +106,9 @@ public:
   virtual CompilerType DeclGetFunctionArgumentType(void *opaque_decl,
                                                    size_t arg_idx);
 
+  virtual std::vector<lldb_private::CompilerContext>
+  DeclGetCompilerContext(void *opaque_decl);
+
   virtual CompilerType GetTypeForDecl(void *opaque_decl) = 0;
 
   // CompilerDeclContext functions
@@ -146,6 +133,9 @@ public:
   virtual CompilerDeclContext
   GetCompilerDeclContextForType(const CompilerType &type);
 
+  virtual std::vector<lldb_private::CompilerContext>
+  DeclContextGetCompilerContext(void *opaque_decl_ctx);
+
   // Tests
 #ifndef NDEBUG
   /// Verify the integrity of the type to catch CompilerTypes that mix
diff --git a/lldb/include/lldb/lldb-forward.h b/lldb/include/lldb/lldb-forward.h
index 068fce4976ed2659dbecdff61eae2bd85bb45e37..4e0c62fa26cae44cf086b620e84bfdc36b8f0973 100644
--- a/lldb/include/lldb/lldb-forward.h
+++ b/lldb/include/lldb/lldb-forward.h
@@ -258,9 +258,11 @@ class TypeImpl;
 class TypeList;
 class TypeListImpl;
 class TypeMap;
+class TypeQuery;
 class TypeMemberFunctionImpl;
 class TypeMemberImpl;
 class TypeNameSpecifierImpl;
+class TypeResults;
 class TypeSummaryImpl;
 class TypeSummaryOptions;
 class TypeSystem;
diff --git a/lldb/include/lldb/lldb-private-enumerations.h b/lldb/include/lldb/lldb-private-enumerations.h
index 7f98220f9f1643a638dfa8afcd4c99791f13649b..5f1597200a83e753607a312626b507b1dbb990c7 100644
--- a/lldb/include/lldb/lldb-private-enumerations.h
+++ b/lldb/include/lldb/lldb-private-enumerations.h
@@ -198,12 +198,15 @@ enum class CompilerContextKind : uint16_t {
   Variable = 1 << 7,
   Enum = 1 << 8,
   Typedef = 1 << 9,
+  Builtin = 1 << 10,
 
   Any = 1 << 15,
   /// Match 0..n nested modules.
   AnyModule = Any | Module,
   /// Match any type.
-  AnyType = Any | Class | Struct | Union | Enum | Typedef
+  AnyType = Any | Class | Struct | Union | Enum | Typedef | Builtin,
+  /// Math any declaration context.
+  AnyDeclContext = Any | Namespace | Class | Struct | Union | Enum | Function
 };
 
 // Enumerations that can be used to specify the kind of metric we're looking at
diff --git a/lldb/source/API/SBModule.cpp b/lldb/source/API/SBModule.cpp
index b865502228e0a15f93adc5088c31d4f628c5d56b..262e26c6bf4318a1351efc9b79bc483aeb3ed150 100644
--- a/lldb/source/API/SBModule.cpp
+++ b/lldb/source/API/SBModule.cpp
@@ -437,26 +437,25 @@ lldb::SBType SBModule::FindFirstType(const char *name_cstr) {
   LLDB_INSTRUMENT_VA(this, name_cstr);
 
   ModuleSP module_sp(GetSP());
-  if (!name_cstr || !module_sp)
-    return {};
-  SymbolContext sc;
-  const bool exact_match = false;
-  ConstString name(name_cstr);
+  if (name_cstr && module_sp) {
+    ConstString name(name_cstr);
+    TypeQuery query(name.GetStringRef(), TypeQueryOptions::e_find_one);
+    TypeResults results;
+    module_sp->FindTypes(query, results);
+    TypeSP type_sp = results.GetFirstType();
+    if (type_sp)
+      return SBType(type_sp);
 
-  SBType sb_type = SBType(module_sp->FindFirstType(sc, name, exact_match));
-
-  if (sb_type.IsValid())
-    return sb_type;
+    auto type_system_or_err =
+        module_sp->GetTypeSystemForLanguage(eLanguageTypeC);
+    if (auto err = type_system_or_err.takeError()) {
+      llvm::consumeError(std::move(err));
+      return {};
+    }
 
-  auto type_system_or_err = module_sp->GetTypeSystemForLanguage(eLanguageTypeC);
-  if (auto err = type_system_or_err.takeError()) {
-    llvm::consumeError(std::move(err));
-    return {};
+    if (auto ts = *type_system_or_err)
+      return SBType(ts->GetBuiltinTypeByName(name));
   }
-
-  if (auto ts = *type_system_or_err)
-    return SBType(ts->GetBuiltinTypeByName(name));
-
   return {};
 }
 
@@ -471,7 +470,7 @@ lldb::SBType SBModule::GetBasicType(lldb::BasicType type) {
       llvm::consumeError(std::move(err));
     } else {
       if (auto ts = *type_system_or_err)
-        return SBType(ts->GetBasicTypeFromAST(type));              
+        return SBType(ts->GetBasicTypeFromAST(type));
     }
   }
   return SBType();
@@ -485,13 +484,11 @@ lldb::SBTypeList SBModule::FindTypes(const char *type) {
   ModuleSP module_sp(GetSP());
   if (type && module_sp) {
     TypeList type_list;
-    const bool exact_match = false;
-    ConstString name(type);
-    llvm::DenseSet<SymbolFile *> searched_symbol_files;
-    module_sp->FindTypes(name, exact_match, UINT32_MAX, searched_symbol_files,
-                         type_list);
-
-    if (type_list.Empty()) {
+    TypeQuery query(type);
+    TypeResults results;
+    module_sp->FindTypes(query, results);
+    if (results.GetTypeMap().Empty()) {
+      ConstString name(type);
       auto type_system_or_err =
           module_sp->GetTypeSystemForLanguage(eLanguageTypeC);
       if (auto err = type_system_or_err.takeError()) {
@@ -502,11 +499,8 @@ lldb::SBTypeList SBModule::FindTypes(const char *type) {
             retval.Append(SBType(compiler_type));
       }
     } else {
-      for (size_t idx = 0; idx < type_list.GetSize(); idx++) {
-        TypeSP type_sp(type_list.GetTypeAtIndex(idx));
-        if (type_sp)
-          retval.Append(SBType(type_sp));
-      }
+      for (const TypeSP &type_sp : results.GetTypeMap().Types())
+        retval.Append(SBType(type_sp));
     }
   }
   return retval;
diff --git a/lldb/source/API/SBTarget.cpp b/lldb/source/API/SBTarget.cpp
index 2d029554492a05cc8419e5a8b4801e215392b0f8..8e616afbcb4e8dfc8fab11297c427e1d1912f282 100644
--- a/lldb/source/API/SBTarget.cpp
+++ b/lldb/source/API/SBTarget.cpp
@@ -1804,21 +1804,13 @@ lldb::SBType SBTarget::FindFirstType(const char *typename_cstr) {
   TargetSP target_sp(GetSP());
   if (typename_cstr && typename_cstr[0] && target_sp) {
     ConstString const_typename(typename_cstr);
-    SymbolContext sc;
-    const bool exact_match = false;
-
-    const ModuleList &module_list = target_sp->GetImages();
-    size_t count = module_list.GetSize();
-    for (size_t idx = 0; idx < count; idx++) {
-      ModuleSP module_sp(module_list.GetModuleAtIndex(idx));
-      if (module_sp) {
-        TypeSP type_sp(
-            module_sp->FindFirstType(sc, const_typename, exact_match));
-        if (type_sp)
-          return SBType(type_sp);
-      }
-    }
-
+    TypeQuery query(const_typename.GetStringRef(),
+                    TypeQueryOptions::e_find_one);
+    TypeResults results;
+    target_sp->GetImages().FindTypes(/*search_first=*/nullptr, query, results);
+    TypeSP type_sp = results.GetFirstType();
+    if (type_sp)
+      return SBType(type_sp);
     // Didn't find the type in the symbols; Try the loaded language runtimes.
     if (auto process_sp = target_sp->GetProcessSP()) {
       for (auto *runtime : process_sp->GetLanguageRuntimes()) {
@@ -1859,17 +1851,11 @@ lldb::SBTypeList SBTarget::FindTypes(const char *typename_cstr) {
   if (typename_cstr && typename_cstr[0] && target_sp) {
     ModuleList &images = target_sp->GetImages();
     ConstString const_typename(typename_cstr);
-    bool exact_match = false;
-    TypeList type_list;
-    llvm::DenseSet<SymbolFile *> searched_symbol_files;
-    images.FindTypes(nullptr, const_typename, exact_match, UINT32_MAX,
-                     searched_symbol_files, type_list);
-
-    for (size_t idx = 0; idx < type_list.GetSize(); idx++) {
-      TypeSP type_sp(type_list.GetTypeAtIndex(idx));
-      if (type_sp)
-        sb_type_list.Append(SBType(type_sp));
-    }
+    TypeQuery query(typename_cstr);
+    TypeResults results;
+    images.FindTypes(nullptr, query, results);
+    for (const TypeSP &type_sp : results.GetTypeMap().Types())
+      sb_type_list.Append(SBType(type_sp));
 
     // Try the loaded language runtimes
     if (auto process_sp = target_sp->GetProcessSP()) {
diff --git a/lldb/source/Commands/CommandObjectMemory.cpp b/lldb/source/Commands/CommandObjectMemory.cpp
index b02b7dee5619f842918f6c69b7338d6acfb9eefc..4ecac732d0dca45dfff1933d97bdd75aae212c29 100644
--- a/lldb/source/Commands/CommandObjectMemory.cpp
+++ b/lldb/source/Commands/CommandObjectMemory.cpp
@@ -372,8 +372,6 @@ protected:
     if (view_as_type_cstr && view_as_type_cstr[0]) {
       // We are viewing memory as a type
 
-      const bool exact_match = false;
-      TypeList type_list;
       uint32_t reference_count = 0;
       uint32_t pointer_count = 0;
       size_t idx;
@@ -452,18 +450,18 @@ protected:
         }
       }
 
-      llvm::DenseSet<lldb_private::SymbolFile *> searched_symbol_files;
       ConstString lookup_type_name(type_str.c_str());
       StackFrame *frame = m_exe_ctx.GetFramePtr();
       ModuleSP search_first;
-      if (frame) {
+      if (frame)
         search_first = frame->GetSymbolContext(eSymbolContextModule).module_sp;
-      }
-      target->GetImages().FindTypes(search_first.get(), lookup_type_name,
-                                    exact_match, 1, searched_symbol_files,
-                                    type_list);
+      TypeQuery query(lookup_type_name.GetStringRef(),
+                      TypeQueryOptions::e_find_one);
+      TypeResults results;
+      target->GetImages().FindTypes(search_first.get(), query, results);
+      TypeSP type_sp = results.GetFirstType();
 
-      if (type_list.GetSize() == 0 && lookup_type_name.GetCString()) {
+      if (!type_sp && lookup_type_name.GetCString()) {
         LanguageType language_for_type =
             m_memory_options.m_language_for_type.GetCurrentValue();
         std::set<LanguageType> languages_to_check;
@@ -499,15 +497,14 @@ protected:
       }
 
       if (!compiler_type.IsValid()) {
-        if (type_list.GetSize() == 0) {
+        if (type_sp) {
+          compiler_type = type_sp->GetFullCompilerType();
+        } else {
           result.AppendErrorWithFormat("unable to find any types that match "
                                        "the raw type '%s' for full type '%s'\n",
                                        lookup_type_name.GetCString(),
                                        view_as_type_cstr);
           return;
-        } else {
-          TypeSP type_sp(type_list.GetTypeAtIndex(0));
-          compiler_type = type_sp->GetFullCompilerType();
         }
       }
 
diff --git a/lldb/source/Commands/CommandObjectTarget.cpp b/lldb/source/Commands/CommandObjectTarget.cpp
index 63232c221ad1db934641338136eccd893272140d..bc8bc51356c8caa3a16646af63e3b6b8bdcc1117 100644
--- a/lldb/source/Commands/CommandObjectTarget.cpp
+++ b/lldb/source/Commands/CommandObjectTarget.cpp
@@ -1706,16 +1706,18 @@ static size_t LookupTypeInModule(Target *target,
                                  CommandInterpreter &interpreter, Stream &strm,
                                  Module *module, const char *name_cstr,
                                  bool name_is_regex) {
-  TypeList type_list;
   if (module && name_cstr && name_cstr[0]) {
-    const uint32_t max_num_matches = UINT32_MAX;
-    bool name_is_fully_qualified = false;
-
-    ConstString name(name_cstr);
-    llvm::DenseSet<lldb_private::SymbolFile *> searched_symbol_files;
-    module->FindTypes(name, name_is_fully_qualified, max_num_matches,
-                      searched_symbol_files, type_list);
+    TypeQuery query(name_cstr);
+    TypeResults results;
+    module->FindTypes(query, results);
 
+    TypeList type_list;
+    SymbolContext sc;
+    if (module)
+      sc.module_sp = module->shared_from_this();
+    // Sort the type results and put the results that matched in \a module
+    // first if \a module was specified.
+    sc.SortTypeList(results.GetTypeMap(), type_list);
     if (type_list.Empty())
       return 0;
 
@@ -1748,22 +1750,21 @@ static size_t LookupTypeInModule(Target *target,
       }
       strm.EOL();
     }
+    return type_list.GetSize();
   }
-  return type_list.GetSize();
+  return 0;
 }
 
 static size_t LookupTypeHere(Target *target, CommandInterpreter &interpreter,
                              Stream &strm, Module &module,
                              const char *name_cstr, bool name_is_regex) {
+  TypeQuery query(name_cstr);
+  TypeResults results;
+  module.FindTypes(query, results);
   TypeList type_list;
-  const uint32_t max_num_matches = UINT32_MAX;
-  bool name_is_fully_qualified = false;
-
-  ConstString name(name_cstr);
-  llvm::DenseSet<SymbolFile *> searched_symbol_files;
-  module.FindTypes(name, name_is_fully_qualified, max_num_matches,
-                   searched_symbol_files, type_list);
-
+  SymbolContext sc;
+  sc.module_sp = module.shared_from_this();
+  sc.SortTypeList(results.GetTypeMap(), type_list);
   if (type_list.Empty())
     return 0;
 
@@ -2082,7 +2083,7 @@ protected:
             result.GetOutputStream().EOL();
             result.GetOutputStream().EOL();
           }
-          if (INTERRUPT_REQUESTED(GetDebugger(), 
+          if (INTERRUPT_REQUESTED(GetDebugger(),
                                   "Interrupted in dump all symtabs with {0} "
                                   "of {1} dumped.", num_dumped, num_modules))
             break;
@@ -2112,8 +2113,8 @@ protected:
                 result.GetOutputStream().EOL();
                 result.GetOutputStream().EOL();
               }
-              if (INTERRUPT_REQUESTED(GetDebugger(), 
-                    "Interrupted in dump symtab list with {0} of {1} dumped.", 
+              if (INTERRUPT_REQUESTED(GetDebugger(),
+                    "Interrupted in dump symtab list with {0} of {1} dumped.",
                     num_dumped, num_matches))
                 break;
 
@@ -2175,7 +2176,7 @@ protected:
       result.GetOutputStream().Format("Dumping sections for {0} modules.\n",
                                       num_modules);
       for (size_t image_idx = 0; image_idx < num_modules; ++image_idx) {
-        if (INTERRUPT_REQUESTED(GetDebugger(), 
+        if (INTERRUPT_REQUESTED(GetDebugger(),
               "Interrupted in dump all sections with {0} of {1} dumped",
               image_idx, num_modules))
           break;
@@ -2196,7 +2197,7 @@ protected:
             FindModulesByName(target, arg_cstr, module_list, true);
         if (num_matches > 0) {
           for (size_t i = 0; i < num_matches; ++i) {
-            if (INTERRUPT_REQUESTED(GetDebugger(), 
+            if (INTERRUPT_REQUESTED(GetDebugger(),
                   "Interrupted in dump section list with {0} of {1} dumped.",
                   i, num_matches))
               break;
@@ -2338,7 +2339,7 @@ protected:
       }
 
       for (size_t i = 0; i < num_matches; ++i) {
-        if (INTERRUPT_REQUESTED(GetDebugger(), 
+        if (INTERRUPT_REQUESTED(GetDebugger(),
               "Interrupted in dump clang ast list with {0} of {1} dumped.",
               i, num_matches))
           break;
@@ -2477,9 +2478,9 @@ protected:
         if (num_modules > 0) {
           uint32_t num_dumped = 0;
           for (ModuleSP module_sp : target_modules.ModulesNoLocking()) {
-            if (INTERRUPT_REQUESTED(GetDebugger(), 
+            if (INTERRUPT_REQUESTED(GetDebugger(),
                                     "Interrupted in dump all line tables with "
-                                    "{0} of {1} dumped", num_dumped, 
+                                    "{0} of {1} dumped", num_dumped,
                                     num_modules))
               break;
 
diff --git a/lldb/source/Core/Module.cpp b/lldb/source/Core/Module.cpp
index e6279a0feda8684dbbe7c946a63aeaf11a7e6ec2..65a65c455efa767850a5dcfa804fa62e32336e5e 100644
--- a/lldb/source/Core/Module.cpp
+++ b/lldb/source/Core/Module.cpp
@@ -949,99 +949,9 @@ void Module::FindAddressesForLine(const lldb::TargetSP target_sp,
   }
 }
 
-void Module::FindTypes_Impl(
-    ConstString name, const CompilerDeclContext &parent_decl_ctx,
-    size_t max_matches,
-    llvm::DenseSet<lldb_private::SymbolFile *> &searched_symbol_files,
-    TypeMap &types) {
+void Module::FindTypes(const TypeQuery &query, TypeResults &results) {
   if (SymbolFile *symbols = GetSymbolFile())
-    symbols->FindTypes(name, parent_decl_ctx, max_matches,
-                       searched_symbol_files, types);
-}
-
-void Module::FindTypesInNamespace(ConstString type_name,
-                                  const CompilerDeclContext &parent_decl_ctx,
-                                  size_t max_matches, TypeList &type_list) {
-  TypeMap types_map;
-  llvm::DenseSet<lldb_private::SymbolFile *> searched_symbol_files;
-  FindTypes_Impl(type_name, parent_decl_ctx, max_matches, searched_symbol_files,
-                 types_map);
-  if (types_map.GetSize()) {
-    SymbolContext sc;
-    sc.module_sp = shared_from_this();
-    sc.SortTypeList(types_map, type_list);
-  }
-}
-
-lldb::TypeSP Module::FindFirstType(const SymbolContext &sc, ConstString name,
-                                   bool exact_match) {
-  TypeList type_list;
-  llvm::DenseSet<lldb_private::SymbolFile *> searched_symbol_files;
-  FindTypes(name, exact_match, 1, searched_symbol_files, type_list);
-  if (type_list.GetSize())
-    return type_list.GetTypeAtIndex(0);
-  return TypeSP();
-}
-
-void Module::FindTypes(
-    ConstString name, bool exact_match, size_t max_matches,
-    llvm::DenseSet<lldb_private::SymbolFile *> &searched_symbol_files,
-    TypeList &types) {
-  const char *type_name_cstr = name.GetCString();
-  llvm::StringRef type_scope;
-  llvm::StringRef type_basename;
-  TypeClass type_class = eTypeClassAny;
-  TypeMap typesmap;
-
-  if (Type::GetTypeScopeAndBasename(type_name_cstr, type_scope, type_basename,
-                                    type_class)) {
-    // Check if "name" starts with "::" which means the qualified type starts
-    // from the root namespace and implies and exact match. The typenames we
-    // get back from clang do not start with "::" so we need to strip this off
-    // in order to get the qualified names to match
-    exact_match = type_scope.consume_front("::");
-
-    ConstString type_basename_const_str(type_basename);
-    FindTypes_Impl(type_basename_const_str, CompilerDeclContext(), max_matches,
-                   searched_symbol_files, typesmap);
-    if (typesmap.GetSize())
-      typesmap.RemoveMismatchedTypes(type_scope, type_basename, type_class,
-                                     exact_match);
-  } else {
-    // The type is not in a namespace/class scope, just search for it by
-    // basename
-    if (type_class != eTypeClassAny && !type_basename.empty()) {
-      // The "type_name_cstr" will have been modified if we have a valid type
-      // class prefix (like "struct", "class", "union", "typedef" etc).
-      FindTypes_Impl(ConstString(type_basename), CompilerDeclContext(),
-                     UINT_MAX, searched_symbol_files, typesmap);
-      typesmap.RemoveMismatchedTypes(type_scope, type_basename, type_class,
-                                     exact_match);
-    } else {
-      FindTypes_Impl(name, CompilerDeclContext(), UINT_MAX,
-                     searched_symbol_files, typesmap);
-      if (exact_match) {
-        typesmap.RemoveMismatchedTypes(type_scope, name, type_class,
-                                       exact_match);
-      }
-    }
-  }
-  if (typesmap.GetSize()) {
-    SymbolContext sc;
-    sc.module_sp = shared_from_this();
-    sc.SortTypeList(typesmap, types);
-  }
-}
-
-void Module::FindTypes(
-    llvm::ArrayRef<CompilerContext> pattern, LanguageSet languages,
-    llvm::DenseSet<lldb_private::SymbolFile *> &searched_symbol_files,
-    TypeMap &types) {
-  // If a scoped timer is needed, place it in a SymbolFile::FindTypes override.
-  // A timer here is too high volume for some cases, for example when calling
-  // FindTypes on each object file.
-  if (SymbolFile *symbols = GetSymbolFile())
-    symbols->FindTypes(pattern, languages, searched_symbol_files, types);
+    symbols->FindTypes(query, results);
 }
 
 static Debugger::DebuggerList 
diff --git a/lldb/source/Core/ModuleList.cpp b/lldb/source/Core/ModuleList.cpp
index 04a9df7dd63ba4a1fc1e20c04c8cd05e6c05967a..aa89c93c8d052113ce8e7b6a9a0e5458b7f7b5e5 100644
--- a/lldb/source/Core/ModuleList.cpp
+++ b/lldb/source/Core/ModuleList.cpp
@@ -557,36 +557,21 @@ ModuleSP ModuleList::FindModule(const UUID &uuid) const {
   return module_sp;
 }
 
-void ModuleList::FindTypes(Module *search_first, ConstString name,
-                           bool name_is_fully_qualified, size_t max_matches,
-                           llvm::DenseSet<SymbolFile *> &searched_symbol_files,
-                           TypeList &types) const {
+void ModuleList::FindTypes(Module *search_first, const TypeQuery &query,
+                           TypeResults &results) const {
   std::lock_guard<std::recursive_mutex> guard(m_modules_mutex);
-
-  collection::const_iterator pos, end = m_modules.end();
   if (search_first) {
-    for (pos = m_modules.begin(); pos != end; ++pos) {
-      if (search_first == pos->get()) {
-        search_first->FindTypes(name, name_is_fully_qualified, max_matches,
-                                searched_symbol_files, types);
-
-        if (types.GetSize() >= max_matches)
-          return;
-      }
-    }
-  }
-
-  for (pos = m_modules.begin(); pos != end; ++pos) {
-    // Search the module if the module is not equal to the one in the symbol
-    // context "sc". If "sc" contains a empty module shared pointer, then the
-    // comparison will always be true (valid_module_ptr != nullptr).
-    if (search_first != pos->get())
-      (*pos)->FindTypes(name, name_is_fully_qualified, max_matches,
-                        searched_symbol_files, types);
-
-    if (types.GetSize() >= max_matches)
+    search_first->FindTypes(query, results);
+    if (results.Done(query))
       return;
   }
+  for (const auto &module_sp : m_modules) {
+    if (search_first != module_sp.get()) {
+      module_sp->FindTypes(query, results);
+      if (results.Done(query))
+        return;
+    }
+  }
 }
 
 bool ModuleList::FindSourceFile(const FileSpec &orig_spec,
diff --git a/lldb/source/Core/ValueObject.cpp b/lldb/source/Core/ValueObject.cpp
index a7f7ee64282d8919e1bf05737d4e2755af476e43..b13bffa0ca809bc7dc6eb07de12f299ff846d71c 100644
--- a/lldb/source/Core/ValueObject.cpp
+++ b/lldb/source/Core/ValueObject.cpp
@@ -398,13 +398,16 @@ ValueObject::GetChildAtIndexPath(llvm::ArrayRef<size_t> idxs,
   if (idxs.size() == 0)
     return GetSP();
   ValueObjectSP root(GetSP());
+
+  size_t current_index = 0;
   for (size_t idx : idxs) {
     root = root->GetChildAtIndex(idx);
     if (!root) {
       if (index_of_error)
-        *index_of_error = idx;
+        *index_of_error = current_index;
       return root;
     }
+    current_index += 1;
   }
   return root;
 }
@@ -414,13 +417,17 @@ lldb::ValueObjectSP ValueObject::GetChildAtIndexPath(
   if (idxs.size() == 0)
     return GetSP();
   ValueObjectSP root(GetSP());
+
+  size_t current_index = 0;
   for (std::pair<size_t, bool> idx : idxs) {
     root = root->GetChildAtIndex(idx.first, idx.second);
     if (!root) {
       if (index_of_error)
-        *index_of_error = idx.first;
+        *index_of_error = current_index;
       return root;
     }
+
+    current_index += 1;
   }
   return root;
 }
diff --git a/lldb/source/DataFormatters/TypeFormat.cpp b/lldb/source/DataFormatters/TypeFormat.cpp
index 126240aeca65e43a596c91e40bf8d09795f2107b..409c452110bddc4411af8cf4ce2260ca5e6f8aa1 100644
--- a/lldb/source/DataFormatters/TypeFormat.cpp
+++ b/lldb/source/DataFormatters/TypeFormat.cpp
@@ -161,13 +161,12 @@ bool TypeFormatImpl_EnumType::FormatObject(ValueObject *valobj,
     if (!target_sp)
       return false;
     const ModuleList &images(target_sp->GetImages());
-    TypeList types;
-    llvm::DenseSet<lldb_private::SymbolFile *> searched_symbol_files;
-    images.FindTypes(nullptr, m_enum_type, false, UINT32_MAX,
-                     searched_symbol_files, types);
-    if (types.Empty())
+    TypeQuery query(m_enum_type.GetStringRef());
+    TypeResults results;
+    images.FindTypes(nullptr, query, results);
+    if (results.GetTypeMap().Empty())
       return false;
-    for (lldb::TypeSP type_sp : types.Types()) {
+    for (lldb::TypeSP type_sp : results.GetTypeMap().Types()) {
       if (!type_sp)
         continue;
       if ((type_sp->GetForwardCompilerType().GetTypeInfo() &
diff --git a/lldb/source/Plugins/ExpressionParser/Clang/ClangASTSource.cpp b/lldb/source/Plugins/ExpressionParser/Clang/ClangASTSource.cpp
index 5d7e1252038da512807d8370c797be0a095e3f18..00ab6a04bd3236ae46147cf4e13986190b4dc5a9 100644
--- a/lldb/source/Plugins/ExpressionParser/Clang/ClangASTSource.cpp
+++ b/lldb/source/Plugins/ExpressionParser/Clang/ClangASTSource.cpp
@@ -201,19 +201,17 @@ TagDecl *ClangASTSource::FindCompleteType(const TagDecl *decl) {
       LLDB_LOG(log, "      CTD Searching namespace {0} in module {1}",
                item.second.GetName(), item.first->GetFileSpec().GetFilename());
 
-      TypeList types;
-
       ConstString name(decl->getName());
 
-      item.first->FindTypesInNamespace(name, item.second, UINT32_MAX, types);
-
-      for (uint32_t ti = 0, te = types.GetSize(); ti != te; ++ti) {
-        lldb::TypeSP type = types.GetTypeAtIndex(ti);
-
-        if (!type)
-          continue;
+      // Create a type matcher using the CompilerDeclContext for the namespace
+      // as the context (item.second) and search for the name inside of this
+      // context.
+      TypeQuery query(item.second, name);
+      TypeResults results;
+      item.first->FindTypes(query, results);
 
-        CompilerType clang_type(type->GetFullCompilerType());
+      for (const lldb::TypeSP &type_sp : results.GetTypeMap().Types()) {
+        CompilerType clang_type(type_sp->GetFullCompilerType());
 
         if (!ClangUtil::IsClangType(clang_type))
           continue;
@@ -233,24 +231,15 @@ TagDecl *ClangASTSource::FindCompleteType(const TagDecl *decl) {
       }
     }
   } else {
-    TypeList types;
-
-    ConstString name(decl->getName());
-
     const ModuleList &module_list = m_target->GetImages();
+    // Create a type matcher using a CompilerDecl. Each TypeSystem class knows
+    // how to fill out a CompilerContext array using a CompilerDecl.
+    TypeQuery query(CompilerDecl(m_clang_ast_context, (void *)decl));
+    TypeResults results;
+    module_list.FindTypes(nullptr, query, results);
+    for (const lldb::TypeSP &type_sp : results.GetTypeMap().Types()) {
 
-    bool exact_match = false;
-    llvm::DenseSet<SymbolFile *> searched_symbol_files;
-    module_list.FindTypes(nullptr, name, exact_match, UINT32_MAX,
-                          searched_symbol_files, types);
-
-    for (uint32_t ti = 0, te = types.GetSize(); ti != te; ++ti) {
-      lldb::TypeSP type = types.GetTypeAtIndex(ti);
-
-      if (!type)
-        continue;
-
-      CompilerType clang_type(type->GetFullCompilerType());
+      CompilerType clang_type(type_sp->GetFullCompilerType());
 
       if (!ClangUtil::IsClangType(clang_type))
         continue;
@@ -263,13 +252,6 @@ TagDecl *ClangASTSource::FindCompleteType(const TagDecl *decl) {
 
       TagDecl *candidate_tag_decl = const_cast<TagDecl *>(tag_type->getDecl());
 
-      // We have found a type by basename and we need to make sure the decl
-      // contexts are the same before we can try to complete this type with
-      // another
-      if (!TypeSystemClang::DeclsAreEquivalent(const_cast<TagDecl *>(decl),
-                                               candidate_tag_decl))
-        continue;
-
       if (TypeSystemClang::GetCompleteDecl(&candidate_tag_decl->getASTContext(),
                                            candidate_tag_decl))
         return candidate_tag_decl;
@@ -614,41 +596,40 @@ void ClangASTSource::FindExternalVisibleDecls(
   if (context.m_found_type)
     return;
 
-  TypeList types;
-  const bool exact_match = true;
-  llvm::DenseSet<lldb_private::SymbolFile *> searched_symbol_files;
-  if (module_sp && namespace_decl)
-    module_sp->FindTypesInNamespace(name, namespace_decl, 1, types);
-  else {
-    m_target->GetImages().FindTypes(module_sp.get(), name, exact_match, 1,
-                                    searched_symbol_files, types);
+  lldb::TypeSP type_sp;
+  TypeResults results;
+  if (module_sp && namespace_decl) {
+    // Match the name in the specified decl context.
+    TypeQuery query(namespace_decl, name, TypeQueryOptions::e_find_one);
+    module_sp->FindTypes(query, results);
+    type_sp = results.GetFirstType();
+  } else {
+    // Match the exact name of the type at the root level.
+    TypeQuery query(name.GetStringRef(), TypeQueryOptions::e_exact_match |
+                                             TypeQueryOptions::e_find_one);
+    m_target->GetImages().FindTypes(nullptr, query, results);
+    type_sp = results.GetFirstType();
   }
 
-  if (size_t num_types = types.GetSize()) {
-    for (size_t ti = 0; ti < num_types; ++ti) {
-      lldb::TypeSP type_sp = types.GetTypeAtIndex(ti);
-
-      if (log) {
-        const char *name_string = type_sp->GetName().GetCString();
-
-        LLDB_LOG(log, "  CAS::FEVD Matching type found for \"{0}\": {1}", name,
-                 (name_string ? name_string : "<anonymous>"));
-      }
+  if (type_sp) {
+    if (log) {
+      const char *name_string = type_sp->GetName().GetCString();
 
-      CompilerType full_type = type_sp->GetFullCompilerType();
+      LLDB_LOG(log, "  CAS::FEVD Matching type found for \"{0}\": {1}", name,
+               (name_string ? name_string : "<anonymous>"));
+    }
 
-      CompilerType copied_clang_type(GuardedCopyType(full_type));
+    CompilerType full_type = type_sp->GetFullCompilerType();
 
-      if (!copied_clang_type) {
-        LLDB_LOG(log, "  CAS::FEVD - Couldn't export a type");
+    CompilerType copied_clang_type(GuardedCopyType(full_type));
 
-        continue;
-      }
+    if (!copied_clang_type) {
+      LLDB_LOG(log, "  CAS::FEVD - Couldn't export a type");
+    } else {
 
       context.AddTypeDecl(copied_clang_type);
 
       context.m_found_type = true;
-      break;
     }
   }
 
diff --git a/lldb/source/Plugins/LanguageRuntime/CPlusPlus/ItaniumABI/ItaniumABILanguageRuntime.cpp b/lldb/source/Plugins/LanguageRuntime/CPlusPlus/ItaniumABI/ItaniumABILanguageRuntime.cpp
index a5c9ead55f4cd720a39d0c2276156144d4dd5526..0ea9201901ab38cd9e3675c9a50c62ae704d88b3 100644
--- a/lldb/source/Plugins/LanguageRuntime/CPlusPlus/ItaniumABI/ItaniumABILanguageRuntime.cpp
+++ b/lldb/source/Plugins/LanguageRuntime/CPlusPlus/ItaniumABI/ItaniumABILanguageRuntime.cpp
@@ -82,24 +82,30 @@ TypeAndOrName ItaniumABILanguageRuntime::GetTypeInfo(
       lookup_name.append(class_name.data(), class_name.size());
 
       type_info.SetName(class_name);
-      const bool exact_match = true;
+      ConstString const_lookup_name(lookup_name);
       TypeList class_types;
-
+      ModuleSP module_sp = vtable_info.symbol->CalculateSymbolContextModule();
       // First look in the module that the vtable symbol came from and
       // look for a single exact match.
-      llvm::DenseSet<SymbolFile *> searched_symbol_files;
-      ModuleSP module_sp = vtable_info.symbol->CalculateSymbolContextModule();
-      if (module_sp)
-        module_sp->FindTypes(ConstString(lookup_name), exact_match, 1,
-                              searched_symbol_files, class_types);
+      TypeResults results;
+      TypeQuery query(const_lookup_name.GetStringRef(),
+                      TypeQueryOptions::e_exact_match |
+                          TypeQueryOptions::e_find_one);
+      if (module_sp) {
+        module_sp->FindTypes(query, results);
+        TypeSP type_sp = results.GetFirstType();
+        if (type_sp)
+          class_types.Insert(type_sp);
+      }
 
       // If we didn't find a symbol, then move on to the entire module
       // list in the target and get as many unique matches as possible
-      Target &target = m_process->GetTarget();
-      if (class_types.Empty())
-        target.GetImages().FindTypes(nullptr, ConstString(lookup_name),
-                                      exact_match, UINT32_MAX,
-                                      searched_symbol_files, class_types);
+      if (class_types.Empty()) {
+        query.SetFindOne(false);
+        m_process->GetTarget().GetImages().FindTypes(nullptr, query, results);
+        for (const auto &type_sp : results.GetTypeMap().Types())
+          class_types.Insert(type_sp);
+      }
 
       lldb::TypeSP type_sp;
       if (class_types.Empty()) {
diff --git a/lldb/source/Plugins/LanguageRuntime/ObjC/ObjCLanguageRuntime.cpp b/lldb/source/Plugins/LanguageRuntime/ObjC/ObjCLanguageRuntime.cpp
index 289288a86245f0c0f5ce7e29ee0cc0b350b39afe..ba52444f0c2fc28782fcdd7b5dd4c9c7a137f2e6 100644
--- a/lldb/source/Plugins/LanguageRuntime/ObjC/ObjCLanguageRuntime.cpp
+++ b/lldb/source/Plugins/LanguageRuntime/ObjC/ObjCLanguageRuntime.cpp
@@ -139,17 +139,10 @@ ObjCLanguageRuntime::LookupInCompleteClassCache(ConstString &name) {
     if (!module_sp)
       return TypeSP();
 
-    const bool exact_match = true;
-    const uint32_t max_matches = UINT32_MAX;
-    TypeList types;
-
-    llvm::DenseSet<SymbolFile *> searched_symbol_files;
-    module_sp->FindTypes(name, exact_match, max_matches, searched_symbol_files,
-                         types);
-
-    for (uint32_t i = 0; i < types.GetSize(); ++i) {
-      TypeSP type_sp(types.GetTypeAtIndex(i));
-
+    TypeQuery query(name.GetStringRef(), TypeQueryOptions::e_exact_match);
+    TypeResults results;
+    module_sp->FindTypes(query, results);
+    for (const TypeSP &type_sp : results.GetTypeMap().Types()) {
       if (TypeSystemClang::IsObjCObjectOrInterfaceType(
               type_sp->GetForwardCompilerType())) {
         if (TypePayloadClang(type_sp->GetPayload()).IsCompleteObjCClass()) {
diff --git a/lldb/source/Plugins/SymbolFile/Breakpad/SymbolFileBreakpad.cpp b/lldb/source/Plugins/SymbolFile/Breakpad/SymbolFileBreakpad.cpp
index cd52233cc8cc70ada06750c98e329337edb34fc5..729d6af02402d8e207a9ac339efb402e6c68f3fc 100644
--- a/lldb/source/Plugins/SymbolFile/Breakpad/SymbolFileBreakpad.cpp
+++ b/lldb/source/Plugins/SymbolFile/Breakpad/SymbolFileBreakpad.cpp
@@ -448,15 +448,6 @@ void SymbolFileBreakpad::FindFunctions(const RegularExpression &regex,
   // TODO
 }
 
-void SymbolFileBreakpad::FindTypes(
-    ConstString name, const CompilerDeclContext &parent_decl_ctx,
-    uint32_t max_matches, llvm::DenseSet<SymbolFile *> &searched_symbol_files,
-    TypeMap &types) {}
-
-void SymbolFileBreakpad::FindTypes(
-    llvm::ArrayRef<CompilerContext> pattern, LanguageSet languages,
-    llvm::DenseSet<SymbolFile *> &searched_symbol_files, TypeMap &types) {}
-
 void SymbolFileBreakpad::AddSymbols(Symtab &symtab) {
   Log *log = GetLog(LLDBLog::Symbols);
   Module &module = *m_objfile_sp->GetModule();
diff --git a/lldb/source/Plugins/SymbolFile/Breakpad/SymbolFileBreakpad.h b/lldb/source/Plugins/SymbolFile/Breakpad/SymbolFileBreakpad.h
index 4a01a64202eedf53ada964d8158eaa5418ec6ee1..214fbdd3ff3aa0619cc7f214f880d29bb28765a8 100644
--- a/lldb/source/Plugins/SymbolFile/Breakpad/SymbolFileBreakpad.h
+++ b/lldb/source/Plugins/SymbolFile/Breakpad/SymbolFileBreakpad.h
@@ -118,15 +118,6 @@ public:
   void FindFunctions(const RegularExpression &regex, bool include_inlines,
                      SymbolContextList &sc_list) override;
 
-  void FindTypes(ConstString name, const CompilerDeclContext &parent_decl_ctx,
-                 uint32_t max_matches,
-                 llvm::DenseSet<SymbolFile *> &searched_symbol_files,
-                 TypeMap &types) override;
-
-  void FindTypes(llvm::ArrayRef<CompilerContext> pattern, LanguageSet languages,
-                 llvm::DenseSet<SymbolFile *> &searched_symbol_files,
-                 TypeMap &types) override;
-
   llvm::Expected<lldb::TypeSystemSP>
   GetTypeSystemForLanguage(lldb::LanguageType language) override {
     return llvm::make_error<llvm::StringError>(
diff --git a/lldb/source/Plugins/SymbolFile/CTF/SymbolFileCTF.cpp b/lldb/source/Plugins/SymbolFile/CTF/SymbolFileCTF.cpp
index 7a2b4c00eedf374ffcd026fb3bbf47a6b57e4dd5..d192944bb9d086048fe81c1e123d5b5c4a3dac1b 100644
--- a/lldb/source/Plugins/SymbolFile/CTF/SymbolFileCTF.cpp
+++ b/lldb/source/Plugins/SymbolFile/CTF/SymbolFileCTF.cpp
@@ -1020,23 +1020,18 @@ lldb_private::Type *SymbolFileCTF::ResolveTypeUID(lldb::user_id_t type_uid) {
   return type_sp.get();
 }
 
-void SymbolFileCTF::FindTypes(
-    lldb_private::ConstString name,
-    const lldb_private::CompilerDeclContext &parent_decl_ctx,
-    uint32_t max_matches,
-    llvm::DenseSet<lldb_private::SymbolFile *> &searched_symbol_files,
-    lldb_private::TypeMap &types) {
-
-  searched_symbol_files.clear();
-  searched_symbol_files.insert(this);
+void SymbolFileCTF::FindTypes(const lldb_private::TypeQuery &match,
+                              lldb_private::TypeResults &results) {
+  // Make sure we haven't already searched this SymbolFile before.
+  if (results.AlreadySearched(this))
+    return;
 
-  size_t matches = 0;
+  ConstString name = match.GetTypeBasename();
   for (TypeSP type_sp : GetTypeList().Types()) {
-    if (matches == max_matches)
-      break;
     if (type_sp && type_sp->GetName() == name) {
-      types.Insert(type_sp);
-      matches++;
+      results.InsertUnique(type_sp);
+      if (results.Done(match))
+        return;
     }
   }
 }
diff --git a/lldb/source/Plugins/SymbolFile/CTF/SymbolFileCTF.h b/lldb/source/Plugins/SymbolFile/CTF/SymbolFileCTF.h
index 787dc1892bb3dd2fb7bc75c6f410ab15364fb4b3..f111937dbd6ef027935e32881fe30725ab4c5712 100644
--- a/lldb/source/Plugins/SymbolFile/CTF/SymbolFileCTF.h
+++ b/lldb/source/Plugins/SymbolFile/CTF/SymbolFileCTF.h
@@ -105,12 +105,8 @@ public:
                 lldb::TypeClass type_mask,
                 lldb_private::TypeList &type_list) override {}
 
-  void
-  FindTypes(lldb_private::ConstString name,
-            const lldb_private::CompilerDeclContext &parent_decl_ctx,
-            uint32_t max_matches,
-            llvm::DenseSet<lldb_private::SymbolFile *> &searched_symbol_files,
-            lldb_private::TypeMap &types) override;
+  void FindTypes(const lldb_private::TypeQuery &match,
+                 lldb_private::TypeResults &results) override;
 
   void FindTypesByRegex(const lldb_private::RegularExpression &regex,
                         uint32_t max_matches, lldb_private::TypeMap &types);
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
index e3c64640c7917649e818205b06945f08628568bb..334876620249fc6d84dd6bc369b7b883884e6031 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFASTParserClang.cpp
@@ -152,16 +152,16 @@ TypeSP DWARFASTParserClang::ParseTypeFromClangModule(const SymbolContext &sc,
   // If this type comes from a Clang module, recursively look in the
   // DWARF section of the .pcm file in the module cache. Clang
   // generates DWO skeleton units as breadcrumbs to find them.
-  std::vector<CompilerContext> decl_context = die.GetDeclContext();
-  TypeMap pcm_types;
+  std::vector<lldb_private::CompilerContext> die_context = die.GetDeclContext();
+  TypeQuery query(die_context, TypeQueryOptions::e_module_search |
+                                   TypeQueryOptions::e_find_one);
+  TypeResults results;
 
   // The type in the Clang module must have the same language as the current CU.
-  LanguageSet languages;
-  languages.Insert(SymbolFileDWARF::GetLanguageFamily(*die.GetCU()));
-  llvm::DenseSet<SymbolFile *> searched_symbol_files;
-  clang_module_sp->GetSymbolFile()->FindTypes(decl_context, languages,
-                                              searched_symbol_files, pcm_types);
-  if (pcm_types.Empty()) {
+  query.AddLanguage(SymbolFileDWARF::GetLanguageFamily(*die.GetCU()));
+  clang_module_sp->FindTypes(query, results);
+  TypeSP pcm_type_sp = results.GetTypeMap().FirstType();
+  if (!pcm_type_sp) {
     // Since this type is defined in one of the Clang modules imported
     // by this symbol file, search all of them. Instead of calling
     // sym_file->FindTypes(), which would return this again, go straight
@@ -170,24 +170,20 @@ TypeSP DWARFASTParserClang::ParseTypeFromClangModule(const SymbolContext &sc,
 
     // Well-formed clang modules never form cycles; guard against corrupted
     // ones by inserting the current file.
-    searched_symbol_files.insert(&sym_file);
+    results.AlreadySearched(&sym_file);
     sym_file.ForEachExternalModule(
-        *sc.comp_unit, searched_symbol_files, [&](Module &module) {
-          module.GetSymbolFile()->FindTypes(decl_context, languages,
-                                            searched_symbol_files, pcm_types);
-          return pcm_types.GetSize();
+        *sc.comp_unit, results.GetSearchedSymbolFiles(), [&](Module &module) {
+          module.FindTypes(query, results);
+          pcm_type_sp = results.GetTypeMap().FirstType();
+          return !pcm_type_sp;
         });
   }
 
-  if (!pcm_types.GetSize())
+  if (!pcm_type_sp)
     return TypeSP();
 
   // We found a real definition for this type in the Clang module, so lets use
   // it and cache the fact that we found a complete type for this die.
-  TypeSP pcm_type_sp = pcm_types.GetTypeAtIndex(0);
-  if (!pcm_type_sp)
-    return TypeSP();
-
   lldb_private::CompilerType pcm_type = pcm_type_sp->GetForwardCompilerType();
   lldb_private::CompilerType type =
       GetClangASTImporter().CopyType(m_ast, pcm_type);
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDIE.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDIE.cpp
index 1f9524f8add982f4c2e4b2bd72ad997801c46f22..bed68f45426f679dfd63ddec4ce0eb1fa4fd2bc3 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDIE.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDIE.cpp
@@ -418,6 +418,54 @@ std::vector<lldb_private::CompilerContext> DWARFDIE::GetDeclContext() const {
   return context;
 }
 
+std::vector<lldb_private::CompilerContext>
+DWARFDIE::GetTypeLookupContext() const {
+  std::vector<lldb_private::CompilerContext> context;
+  // If there is no name, then there is no need to look anything up for this
+  // DIE.
+  const char *name = GetName();
+  if (!name || !name[0])
+    return context;
+  const dw_tag_t tag = Tag();
+  if (tag == DW_TAG_compile_unit || tag == DW_TAG_partial_unit)
+    return context;
+  DWARFDIE parent = GetParent();
+  if (parent)
+    context = parent.GetTypeLookupContext();
+  auto push_ctx = [&](CompilerContextKind kind, llvm::StringRef name) {
+    context.push_back({kind, ConstString(name)});
+  };
+  switch (tag) {
+  case DW_TAG_namespace:
+    push_ctx(CompilerContextKind::Namespace, name);
+    break;
+  case DW_TAG_structure_type:
+    push_ctx(CompilerContextKind::Struct, name);
+    break;
+  case DW_TAG_union_type:
+    push_ctx(CompilerContextKind::Union, name);
+    break;
+  case DW_TAG_class_type:
+    push_ctx(CompilerContextKind::Class, name);
+    break;
+  case DW_TAG_enumeration_type:
+    push_ctx(CompilerContextKind::Enum, name);
+    break;
+  case DW_TAG_variable:
+    push_ctx(CompilerContextKind::Variable, GetPubname());
+    break;
+  case DW_TAG_typedef:
+    push_ctx(CompilerContextKind::Typedef, name);
+    break;
+  case DW_TAG_base_type:
+    push_ctx(CompilerContextKind::Builtin, name);
+    break;
+  default:
+    break;
+  }
+  return context;
+}
+
 DWARFDIE
 DWARFDIE::GetParentDeclContextDIE() const {
   if (IsValid())
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDIE.h b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDIE.h
index a68af62c8b3e20c057c632b6c43ec0de026d263f..511ca62d0197a89f8af7e7d01191e1b4c4af9d22 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDIE.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDIE.h
@@ -73,9 +73,22 @@ public:
   std::vector<DWARFDIE> GetDeclContextDIEs() const;
 
   /// Return this DIE's decl context as it is needed to look up types
-  /// in Clang's -gmodules debug info format.
+  /// in Clang modules. This context will include any modules or functions that
+  /// the type is declared in so an exact module match can be efficiently made.
   std::vector<CompilerContext> GetDeclContext() const;
 
+  /// Get a context to a type so it can be looked up.
+  ///
+  /// This function uses the current DIE to fill in a CompilerContext array
+  /// that is suitable for type lookup for comparison to a TypeQuery's compiler
+  /// context (TypeQuery::GetContextRef()). If this DIE represents a named type,
+  /// it should fill out the compiler context with the type itself as the last
+  /// entry. The declaration context should be above the type and stop at an
+  /// appropriate time, like either the translation unit or at a function
+  /// context. This is designed to allow users to efficiently look for types
+  /// using a full or partial CompilerContext array.
+  std::vector<CompilerContext> GetTypeLookupContext() const;
+
   // Getting attribute values from the DIE.
   //
   // GetAttributeValueAsXXX() functions should only be used if you are
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
index d4c573ecd468c416c1f976f775547b304dca591a..7eddc5074eff12050fa223e7e37e0e022a460ecf 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp
@@ -2596,177 +2596,157 @@ void SymbolFileDWARF::GetMangledNamesForFunction(
   }
 }
 
-void SymbolFileDWARF::FindTypes(
-    ConstString name, const CompilerDeclContext &parent_decl_ctx,
-    uint32_t max_matches,
-    llvm::DenseSet<lldb_private::SymbolFile *> &searched_symbol_files,
-    TypeMap &types) {
-  std::lock_guard<std::recursive_mutex> guard(GetModuleMutex());
-  // Make sure we haven't already searched this SymbolFile before.
-  if (!searched_symbol_files.insert(this).second)
-    return;
-
-  Log *log = GetLog(DWARFLog::Lookups);
+/// Split a name up into a basename and template parameters.
+static bool SplitTemplateParams(llvm::StringRef fullname,
+                                llvm::StringRef &basename,
+                                llvm::StringRef &template_params) {
+  auto it = fullname.find('<');
+  if (it == llvm::StringRef::npos) {
+    basename = fullname;
+    template_params = llvm::StringRef();
+    return false;
+  }
+  basename = fullname.slice(0, it);
+  template_params = fullname.slice(it, fullname.size());
+  return true;
+}
 
-  if (log) {
-    if (parent_decl_ctx)
-      GetObjectFile()->GetModule()->LogMessage(
-          log,
-          "SymbolFileDWARF::FindTypes (sc, name=\"{0}\", parent_decl_ctx = "
-          "{1:p} (\"{2}\"), max_matches={3}, type_list)",
-          name.GetCString(), static_cast<const void *>(&parent_decl_ctx),
-          parent_decl_ctx.GetName().AsCString("<NULL>"), max_matches);
-    else
-      GetObjectFile()->GetModule()->LogMessage(
-          log,
-          "SymbolFileDWARF::FindTypes (sc, name=\"{0}\", parent_decl_ctx = "
-          "NULL, max_matches={1}, type_list)",
-          name.GetCString(), max_matches);
+static bool UpdateCompilerContextForSimpleTemplateNames(TypeQuery &match) {
+  // We need to find any names in the context that have template parameters
+  // and strip them so the context can be matched when -gsimple-template-names
+  // is being used. Returns true if any of the context items were updated.
+  bool any_context_updated = false;
+  for (auto &context : match.GetContextRef()) {
+    llvm::StringRef basename, params;
+    if (SplitTemplateParams(context.name.GetStringRef(), basename, params)) {
+      context.name = ConstString(basename);
+      any_context_updated = true;
+    }
   }
+  return any_context_updated;
+}
+void SymbolFileDWARF::FindTypes(const TypeQuery &query, TypeResults &results) {
 
-  if (!DeclContextMatchesThisSymbolFile(parent_decl_ctx))
+  // Make sure we haven't already searched this SymbolFile before.
+  if (results.AlreadySearched(this))
     return;
 
-  // Unlike FindFunctions(), FindTypes() following cannot produce false
-  // positives.
-
-  const llvm::StringRef name_ref = name.GetStringRef();
-  auto name_bracket_index = name_ref.find('<');
-  m_index->GetTypes(name, [&](DWARFDIE die) {
-    if (!DIEInDeclContext(parent_decl_ctx, die))
-      return true; // The containing decl contexts don't match
+  std::lock_guard<std::recursive_mutex> guard(GetModuleMutex());
 
-    Type *matching_type = ResolveType(die, true, true);
-    if (!matching_type)
-      return true;
+  bool have_index_match = false;
+  m_index->GetTypes(query.GetTypeBasename(), [&](DWARFDIE die) {
+    // Check the language, but only if we have a language filter.
+    if (query.HasLanguage()) {
+      if (!query.LanguageMatches(GetLanguageFamily(*die.GetCU())))
+        return true; // Keep iterating over index types, language mismatch.
+    }
 
-    // With -gsimple-template-names, a templated type's DW_AT_name will not
-    // contain the template parameters. Make sure that if the original query
-    // didn't contain a '<', we filter out entries with template parameters.
-    if (name_bracket_index == llvm::StringRef::npos &&
-        matching_type->IsTemplateType())
-      return true;
+    // Check the context matches
+    std::vector<lldb_private::CompilerContext> die_context;
+    if (query.GetModuleSearch())
+      die_context = die.GetDeclContext();
+    else
+      die_context = die.GetTypeLookupContext();
+    assert(!die_context.empty());
+    if (!query.ContextMatches(die_context))
+      return true; // Keep iterating over index types, context mismatch.
 
-    // We found a type pointer, now find the shared pointer form our type
-    // list
-    types.InsertUnique(matching_type->shared_from_this());
-    return types.GetSize() < max_matches;
+    // Try to resolve the type.
+    if (Type *matching_type = ResolveType(die, true, true)) {
+      if (matching_type->IsTemplateType()) {
+        // We have to watch out for case where we lookup a type by basename and
+        // it matches a template with simple template names. Like looking up
+        // "Foo" and if we have simple template names then we will match
+        // "Foo<int>" and "Foo<double>" because all the DWARF has is "Foo" in
+        // the accelerator tables. The main case we see this in is when the
+        // expression parser is trying to parse "Foo<int>" and it will first do
+        // a lookup on just "Foo". We verify the type basename matches before
+        // inserting the type in the results.
+        auto CompilerTypeBasename =
+            matching_type->GetForwardCompilerType().GetTypeName(true);
+        if (CompilerTypeBasename != query.GetTypeBasename())
+          return true; // Keep iterating over index types, basename mismatch.
+      }
+      have_index_match = true;
+      results.InsertUnique(matching_type->shared_from_this());
+    }
+    return !results.Done(query); // Keep iterating if we aren't done.
   });
 
+  if (results.Done(query))
+    return;
+
   // With -gsimple-template-names, a templated type's DW_AT_name will not
   // contain the template parameters. Try again stripping '<' and anything
   // after, filtering out entries with template parameters that don't match.
-  if (types.GetSize() < max_matches) {
-    if (name_bracket_index != llvm::StringRef::npos) {
-      const llvm::StringRef name_no_template_params =
-          name_ref.slice(0, name_bracket_index);
-      const llvm::StringRef template_params =
-          name_ref.slice(name_bracket_index, name_ref.size());
-      m_index->GetTypes(ConstString(name_no_template_params), [&](DWARFDIE die) {
-        if (!DIEInDeclContext(parent_decl_ctx, die))
-          return true; // The containing decl contexts don't match
-
-        const llvm::StringRef base_name = GetTypeForDIE(die)->GetBaseName().AsCString();
-        auto it = base_name.find('<');
-        // If the candidate qualified name doesn't have '<', it doesn't have
-        // template params to compare.
-        if (it == llvm::StringRef::npos)
-          return true;
-
-        // Filter out non-matching instantiations by comparing template params.
-        const llvm::StringRef base_name_template_params =
-            base_name.slice(it, base_name.size());
-
-        if (template_params != base_name_template_params)
-          return true;
-
-        Type *matching_type = ResolveType(die, true, true);
-        if (!matching_type)
-          return true;
+  if (!have_index_match) {
+    // Create a type matcher with a compiler context that is tuned for
+    // -gsimple-template-names. We will use this for the index lookup and the
+    // context matching, but will use the original "match" to insert matches
+    // into if things match. The "match_simple" has a compiler context with
+    // all template parameters removed to allow the names and context to match.
+    // The UpdateCompilerContextForSimpleTemplateNames(...) will return true if
+    // it trims any context items down by removing template parameter names.
+    TypeQuery query_simple(query);
+    if (UpdateCompilerContextForSimpleTemplateNames(query_simple)) {
+
+      // Copy our match's context and update the basename we are looking for
+      // so we can use this only to compare the context correctly.
+      m_index->GetTypes(query_simple.GetTypeBasename(), [&](DWARFDIE die) {
+        // Check the language, but only if we have a language filter.
+        if (query.HasLanguage()) {
+          if (!query.LanguageMatches(GetLanguageFamily(*die.GetCU())))
+            return true; // Keep iterating over index types, language mismatch.
+        }
 
-        // We found a type pointer, now find the shared pointer form our type
-        // list.
-        types.InsertUnique(matching_type->shared_from_this());
-        return types.GetSize() < max_matches;
+        // Check the context matches
+        std::vector<lldb_private::CompilerContext> die_context;
+        if (query.GetModuleSearch())
+          die_context = die.GetDeclContext();
+        else
+          die_context = die.GetTypeLookupContext();
+        assert(!die_context.empty());
+        if (!query_simple.ContextMatches(die_context))
+          return true; // Keep iterating over index types, context mismatch.
+
+        // Try to resolve the type.
+        if (Type *matching_type = ResolveType(die, true, true)) {
+          ConstString name = matching_type->GetQualifiedName();
+          // We have found a type that still might not match due to template
+          // parameters. If we create a new TypeQuery that uses the new type's
+          // fully qualified name, we can find out if this type matches at all
+          // context levels. We can't use just the "match_simple" context
+          // because all template parameters were stripped off. The fully
+          // qualified name of the type will have the template parameters and
+          // will allow us to make sure it matches correctly.
+          TypeQuery die_query(name.GetStringRef(),
+                              TypeQueryOptions::e_exact_match);
+          if (!query.ContextMatches(die_query.GetContextRef()))
+            return true; // Keep iterating over index types, context mismatch.
+
+          results.InsertUnique(matching_type->shared_from_this());
+        }
+        return !results.Done(query); // Keep iterating if we aren't done.
       });
+      if (results.Done(query))
+        return;
     }
   }
 
   // Next search through the reachable Clang modules. This only applies for
   // DWARF objects compiled with -gmodules that haven't been processed by
   // dsymutil.
-  if (types.GetSize() < max_matches) {
-    UpdateExternalModuleListIfNeeded();
-
-    for (const auto &pair : m_external_type_modules)
-      if (ModuleSP external_module_sp = pair.second)
-        if (SymbolFile *sym_file = external_module_sp->GetSymbolFile())
-          sym_file->FindTypes(name, parent_decl_ctx, max_matches,
-                              searched_symbol_files, types);
-  }
+  UpdateExternalModuleListIfNeeded();
 
-  if (log && types.GetSize()) {
-    if (parent_decl_ctx) {
-      GetObjectFile()->GetModule()->LogMessage(
-          log,
-          "SymbolFileDWARF::FindTypes (sc, name=\"{0}\", parent_decl_ctx "
-          "= {1:p} (\"{2}\"), max_matches={3}, type_list) => {4}",
-          name.GetCString(), static_cast<const void *>(&parent_decl_ctx),
-          parent_decl_ctx.GetName().AsCString("<NULL>"), max_matches,
-          types.GetSize());
-    } else {
-      GetObjectFile()->GetModule()->LogMessage(
-          log,
-          "SymbolFileDWARF::FindTypes (sc, name=\"{0}\", parent_decl_ctx "
-          "= NULL, max_matches={1}, type_list) => {2}",
-          name.GetCString(), max_matches, types.GetSize());
+  for (const auto &pair : m_external_type_modules) {
+    if (ModuleSP external_module_sp = pair.second) {
+      external_module_sp->FindTypes(query, results);
+      if (results.Done(query))
+        return;
     }
   }
 }
 
-void SymbolFileDWARF::FindTypes(
-    llvm::ArrayRef<CompilerContext> pattern, LanguageSet languages,
-    llvm::DenseSet<SymbolFile *> &searched_symbol_files, TypeMap &types) {
-  // Make sure we haven't already searched this SymbolFile before.
-  if (!searched_symbol_files.insert(this).second)
-    return;
-
-  std::lock_guard<std::recursive_mutex> guard(GetModuleMutex());
-  if (pattern.empty())
-    return;
-
-  ConstString name = pattern.back().name;
-
-  if (!name)
-    return;
-
-  m_index->GetTypes(name, [&](DWARFDIE die) {
-    if (!languages[GetLanguageFamily(*die.GetCU())])
-      return true;
-
-    std::vector<CompilerContext> die_context = die.GetDeclContext();
-    if (!contextMatches(die_context, pattern))
-      return true;
-
-    if (Type *matching_type = ResolveType(die, true, true)) {
-      // We found a type pointer, now find the shared pointer form our type
-      // list.
-      types.InsertUnique(matching_type->shared_from_this());
-    }
-    return true;
-  });
-
-  // Next search through the reachable Clang modules. This only applies for
-  // DWARF objects compiled with -gmodules that haven't been processed by
-  // dsymutil.
-  UpdateExternalModuleListIfNeeded();
-
-  for (const auto &pair : m_external_type_modules)
-    if (ModuleSP external_module_sp = pair.second)
-      external_module_sp->FindTypes(pattern, languages, searched_symbol_files,
-                                    types);
-}
-
 CompilerDeclContext
 SymbolFileDWARF::FindNamespace(ConstString name,
                                const CompilerDeclContext &parent_decl_ctx,
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h
index e6efbba7e24990a4772218f94ca25e99705614cd..78819edd0062bffbd8abff86dbb018d976df5ea3 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.h
@@ -186,14 +186,8 @@ public:
   GetMangledNamesForFunction(const std::string &scope_qualified_name,
                              std::vector<ConstString> &mangled_names) override;
 
-  void FindTypes(ConstString name, const CompilerDeclContext &parent_decl_ctx,
-                 uint32_t max_matches,
-                 llvm::DenseSet<SymbolFile *> &searched_symbol_files,
-                 TypeMap &types) override;
-
-  void FindTypes(llvm::ArrayRef<CompilerContext> pattern, LanguageSet languages,
-                 llvm::DenseSet<SymbolFile *> &searched_symbol_files,
-                 TypeMap &types) override;
+  void FindTypes(const lldb_private::TypeQuery &match,
+                 lldb_private::TypeResults &results) override;
 
   void GetTypes(SymbolContextScope *sc_scope, lldb::TypeClass type_mask,
                 TypeList &type_list) override;
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.cpp b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.cpp
index 263ada9cbb87f16bd34bc3c06fc3a106b7a73384..e5b59460cb85d5b186f765af79a5d33d8a35d974 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.cpp
+++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.cpp
@@ -1227,27 +1227,12 @@ TypeSP SymbolFileDWARFDebugMap::FindCompleteObjCDefinitionTypeForDIE(
   return TypeSP();
 }
 
-void SymbolFileDWARFDebugMap::FindTypes(
-    ConstString name, const CompilerDeclContext &parent_decl_ctx,
-    uint32_t max_matches,
-    llvm::DenseSet<lldb_private::SymbolFile *> &searched_symbol_files,
-    TypeMap &types) {
+void SymbolFileDWARFDebugMap::FindTypes(const TypeQuery &query,
+                                        TypeResults &results) {
   std::lock_guard<std::recursive_mutex> guard(GetModuleMutex());
   ForEachSymbolFile([&](SymbolFileDWARF *oso_dwarf) -> bool {
-    oso_dwarf->FindTypes(name, parent_decl_ctx, max_matches,
-                         searched_symbol_files, types);
-    return types.GetSize() >= max_matches;
-  });
-}
-
-void SymbolFileDWARFDebugMap::FindTypes(
-    llvm::ArrayRef<CompilerContext> context, LanguageSet languages,
-    llvm::DenseSet<lldb_private::SymbolFile *> &searched_symbol_files,
-    TypeMap &types) {
-  LLDB_SCOPED_TIMER();
-  ForEachSymbolFile([&](SymbolFileDWARF *oso_dwarf) -> bool {
-    oso_dwarf->FindTypes(context, languages, searched_symbol_files, types);
-    return false;
+    oso_dwarf->FindTypes(query, results);
+    return !results.Done(query); // Keep iterating if we aren't done.
   });
 }
 
diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.h b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.h
index 13f94f6d93e9168e1a5ca5f1969ec21e4b7b75df..cd0a4bb6e41c2f936a99f9820202a8b170912cf4 100644
--- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.h
+++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARFDebugMap.h
@@ -118,13 +118,8 @@ public:
                      bool include_inlines, SymbolContextList &sc_list) override;
   void FindFunctions(const RegularExpression &regex, bool include_inlines,
                      SymbolContextList &sc_list) override;
-  void FindTypes(ConstString name, const CompilerDeclContext &parent_decl_ctx,
-                 uint32_t max_matches,
-                 llvm::DenseSet<SymbolFile *> &searched_symbol_files,
-                 TypeMap &types) override;
-  void FindTypes(llvm::ArrayRef<CompilerContext> context, LanguageSet languages,
-                 llvm::DenseSet<SymbolFile *> &searched_symbol_files,
-                 TypeMap &types) override;
+  void FindTypes(const lldb_private::TypeQuery &match,
+                 lldb_private::TypeResults &results) override;
   CompilerDeclContext FindNamespace(ConstString name,
                                     const CompilerDeclContext &parent_decl_ctx,
                                     bool only_root_namespaces) override;
diff --git a/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.cpp b/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.cpp
index eaca4761a485e9094b5d0b709c068bc921a3ed9d..35c2575028d85e5b2d3608373d9c50778badc93a 100644
--- a/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.cpp
+++ b/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.cpp
@@ -1717,24 +1717,33 @@ void SymbolFileNativePDB::FindFunctions(const RegularExpression &regex,
                                         bool include_inlines,
                                         SymbolContextList &sc_list) {}
 
-void SymbolFileNativePDB::FindTypes(
-    ConstString name, const CompilerDeclContext &parent_decl_ctx,
-    uint32_t max_matches, llvm::DenseSet<SymbolFile *> &searched_symbol_files,
-    TypeMap &types) {
-  std::lock_guard<std::recursive_mutex> guard(GetModuleMutex());
-  if (!name)
+void SymbolFileNativePDB::FindTypes(const lldb_private::TypeQuery &query,
+                                    lldb_private::TypeResults &results) {
+
+  // Make sure we haven't already searched this SymbolFile before.
+  if (results.AlreadySearched(this))
     return;
 
-  searched_symbol_files.clear();
-  searched_symbol_files.insert(this);
+  std::lock_guard<std::recursive_mutex> guard(GetModuleMutex());
 
-  // There is an assumption 'name' is not a regex
-  FindTypesByName(name.GetStringRef(), max_matches, types);
-}
+  std::vector<TypeIndex> matches =
+      m_index->tpi().findRecordsByName(query.GetTypeBasename().GetStringRef());
 
-void SymbolFileNativePDB::FindTypes(
-    llvm::ArrayRef<CompilerContext> pattern, LanguageSet languages,
-    llvm::DenseSet<SymbolFile *> &searched_symbol_files, TypeMap &types) {}
+  for (TypeIndex type_idx : matches) {
+    TypeSP type_sp = GetOrCreateType(type_idx);
+    if (!type_sp)
+      continue;
+
+    // We resolved a type. Get the fully qualified name to ensure it matches.
+    ConstString name = type_sp->GetQualifiedName();
+    TypeQuery type_match(name.GetStringRef(), TypeQueryOptions::e_exact_match);
+    if (query.ContextMatches(type_match.GetContextRef())) {
+      results.InsertUnique(type_sp);
+      if (results.Done(query))
+        return;
+    }
+  }
+}
 
 void SymbolFileNativePDB::FindTypesByName(llvm::StringRef name,
                                           uint32_t max_matches,
diff --git a/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.h b/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.h
index bf64cd330c1f987f047e79617e81eba424762f15..9d0458cf7ebfebf8063786880016f829d553b50e 100644
--- a/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.h
+++ b/lldb/source/Plugins/SymbolFile/NativePDB/SymbolFileNativePDB.h
@@ -140,14 +140,8 @@ public:
 
   std::optional<PdbCompilandSymId> FindSymbolScope(PdbCompilandSymId id);
 
-  void FindTypes(ConstString name, const CompilerDeclContext &parent_decl_ctx,
-                 uint32_t max_matches,
-                 llvm::DenseSet<SymbolFile *> &searched_symbol_files,
-                 TypeMap &types) override;
-
-  void FindTypes(llvm::ArrayRef<CompilerContext> pattern, LanguageSet languages,
-                 llvm::DenseSet<SymbolFile *> &searched_symbol_files,
-                 TypeMap &types) override;
+  void FindTypes(const lldb_private::TypeQuery &match,
+                 lldb_private::TypeResults &results) override;
 
   llvm::Expected<lldb::TypeSystemSP>
   GetTypeSystemForLanguage(lldb::LanguageType language) override;
diff --git a/lldb/source/Plugins/SymbolFile/PDB/SymbolFilePDB.cpp b/lldb/source/Plugins/SymbolFile/PDB/SymbolFilePDB.cpp
index 78eabc35ebf9f16e1c41775390e5f00d23930326..96036de5671d1d441bbb1ff14c9712972a46fb1a 100644
--- a/lldb/source/Plugins/SymbolFile/PDB/SymbolFilePDB.cpp
+++ b/lldb/source/Plugins/SymbolFile/PDB/SymbolFilePDB.cpp
@@ -1446,24 +1446,6 @@ void SymbolFilePDB::AddSymbols(lldb_private::Symtab &symtab) {
   symtab.Finalize();
 }
 
-void SymbolFilePDB::FindTypes(
-    lldb_private::ConstString name, const CompilerDeclContext &parent_decl_ctx,
-    uint32_t max_matches,
-    llvm::DenseSet<lldb_private::SymbolFile *> &searched_symbol_files,
-    lldb_private::TypeMap &types) {
-  std::lock_guard<std::recursive_mutex> guard(GetModuleMutex());
-  if (!name)
-    return;
-  if (!DeclContextMatchesThisSymbolFile(parent_decl_ctx))
-    return;
-
-  searched_symbol_files.clear();
-  searched_symbol_files.insert(this);
-
-  // There is an assumption 'name' is not a regex
-  FindTypesByName(name.GetStringRef(), parent_decl_ctx, max_matches, types);
-}
-
 void SymbolFilePDB::DumpClangAST(Stream &s) {
   auto type_system_or_err =
       GetTypeSystemForLanguage(lldb::eLanguageTypeC_plus_plus);
@@ -1536,25 +1518,27 @@ void SymbolFilePDB::FindTypesByRegex(
   }
 }
 
-void SymbolFilePDB::FindTypesByName(
-    llvm::StringRef name,
-    const lldb_private::CompilerDeclContext &parent_decl_ctx,
-    uint32_t max_matches, lldb_private::TypeMap &types) {
+void SymbolFilePDB::FindTypes(const lldb_private::TypeQuery &query,
+                              lldb_private::TypeResults &type_results) {
+
+  // Make sure we haven't already searched this SymbolFile before.
+  if (type_results.AlreadySearched(this))
+    return;
+
+  std::lock_guard<std::recursive_mutex> guard(GetModuleMutex());
+
   std::unique_ptr<IPDBEnumSymbols> results;
-  if (name.empty())
+  llvm::StringRef basename = query.GetTypeBasename().GetStringRef();
+  if (basename.empty())
     return;
   results = m_global_scope_up->findAllChildren(PDB_SymType::None);
   if (!results)
     return;
 
-  uint32_t matches = 0;
-
   while (auto result = results->getNext()) {
-    if (max_matches > 0 && matches >= max_matches)
-      break;
 
     if (MSVCUndecoratedNameParser::DropScope(
-            result->getRawSymbol().getName()) != name)
+            result->getRawSymbol().getName()) != basename)
       continue;
 
     switch (result->getSymTag()) {
@@ -1573,23 +1557,20 @@ void SymbolFilePDB::FindTypesByName(
     if (!ResolveTypeUID(result->getSymIndexId()))
       continue;
 
-    if (parent_decl_ctx.IsValid() &&
-        GetDeclContextContainingUID(result->getSymIndexId()) != parent_decl_ctx)
-      continue;
-
     auto iter = m_types.find(result->getSymIndexId());
     if (iter == m_types.end())
       continue;
-    types.Insert(iter->second);
-    ++matches;
+    // We resolved a type. Get the fully qualified name to ensure it matches.
+    ConstString name = iter->second->GetQualifiedName();
+    TypeQuery type_match(name.GetStringRef(), TypeQueryOptions::e_exact_match);
+    if (query.ContextMatches(type_match.GetContextRef())) {
+      type_results.InsertUnique(iter->second);
+      if (type_results.Done(query))
+        return;
+    }
   }
 }
 
-void SymbolFilePDB::FindTypes(
-    llvm::ArrayRef<CompilerContext> pattern, LanguageSet languages,
-    llvm::DenseSet<SymbolFile *> &searched_symbol_files,
-    lldb_private::TypeMap &types) {}
-
 void SymbolFilePDB::GetTypesForPDBSymbol(const llvm::pdb::PDBSymbol &pdb_symbol,
                                          uint32_t type_mask,
                                          TypeCollection &type_collection) {
diff --git a/lldb/source/Plugins/SymbolFile/PDB/SymbolFilePDB.h b/lldb/source/Plugins/SymbolFile/PDB/SymbolFilePDB.h
index 5b98c6e8b4860c1b18188d87ce704852b2c51c7f..01851f1418f3a84f3e654e616932a71c763c7ca8 100644
--- a/lldb/source/Plugins/SymbolFile/PDB/SymbolFilePDB.h
+++ b/lldb/source/Plugins/SymbolFile/PDB/SymbolFilePDB.h
@@ -134,19 +134,8 @@ public:
       std::vector<lldb_private::ConstString> &mangled_names) override;
 
   void AddSymbols(lldb_private::Symtab &symtab) override;
-
-  void
-  FindTypes(lldb_private::ConstString name,
-            const lldb_private::CompilerDeclContext &parent_decl_ctx,
-            uint32_t max_matches,
-            llvm::DenseSet<lldb_private::SymbolFile *> &searched_symbol_files,
-            lldb_private::TypeMap &types) override;
-
-  void FindTypes(llvm::ArrayRef<lldb_private::CompilerContext> pattern,
-                 lldb_private::LanguageSet languages,
-                 llvm::DenseSet<SymbolFile *> &searched_symbol_files,
-                 lldb_private::TypeMap &types) override;
-
+  void FindTypes(const lldb_private::TypeQuery &match,
+                 lldb_private::TypeResults &results) override;
   void FindTypesByRegex(const lldb_private::RegularExpression &regex,
                         uint32_t max_matches, lldb_private::TypeMap &types);
 
diff --git a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
index 7c28935f5741c5429a3a6178b380210bb3630a27..47024cd03536a3f0faedbf6442e204187006abf0 100644
--- a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
+++ b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.cpp
@@ -2461,89 +2461,6 @@ void TypeSystemClang::DumpDeclHiearchy(clang::Decl *decl) {
   }
 }
 
-bool TypeSystemClang::DeclsAreEquivalent(clang::Decl *lhs_decl,
-                                         clang::Decl *rhs_decl) {
-  if (lhs_decl && rhs_decl) {
-    // Make sure the decl kinds match first
-    const clang::Decl::Kind lhs_decl_kind = lhs_decl->getKind();
-    const clang::Decl::Kind rhs_decl_kind = rhs_decl->getKind();
-
-    if (lhs_decl_kind == rhs_decl_kind) {
-      // Now check that the decl contexts kinds are all equivalent before we
-      // have to check any names of the decl contexts...
-      clang::DeclContext *lhs_decl_ctx = lhs_decl->getDeclContext();
-      clang::DeclContext *rhs_decl_ctx = rhs_decl->getDeclContext();
-      if (lhs_decl_ctx && rhs_decl_ctx) {
-        while (true) {
-          if (lhs_decl_ctx && rhs_decl_ctx) {
-            const clang::Decl::Kind lhs_decl_ctx_kind =
-                lhs_decl_ctx->getDeclKind();
-            const clang::Decl::Kind rhs_decl_ctx_kind =
-                rhs_decl_ctx->getDeclKind();
-            if (lhs_decl_ctx_kind == rhs_decl_ctx_kind) {
-              lhs_decl_ctx = lhs_decl_ctx->getParent();
-              rhs_decl_ctx = rhs_decl_ctx->getParent();
-
-              if (lhs_decl_ctx == nullptr && rhs_decl_ctx == nullptr)
-                break;
-            } else
-              return false;
-          } else
-            return false;
-        }
-
-        // Now make sure the name of the decls match
-        clang::NamedDecl *lhs_named_decl =
-            llvm::dyn_cast<clang::NamedDecl>(lhs_decl);
-        clang::NamedDecl *rhs_named_decl =
-            llvm::dyn_cast<clang::NamedDecl>(rhs_decl);
-        if (lhs_named_decl && rhs_named_decl) {
-          clang::DeclarationName lhs_decl_name = lhs_named_decl->getDeclName();
-          clang::DeclarationName rhs_decl_name = rhs_named_decl->getDeclName();
-          if (lhs_decl_name.getNameKind() == rhs_decl_name.getNameKind()) {
-            if (lhs_decl_name.getAsString() != rhs_decl_name.getAsString())
-              return false;
-          } else
-            return false;
-        } else
-          return false;
-
-        // We know that the decl context kinds all match, so now we need to
-        // make sure the names match as well
-        lhs_decl_ctx = lhs_decl->getDeclContext();
-        rhs_decl_ctx = rhs_decl->getDeclContext();
-        while (true) {
-          switch (lhs_decl_ctx->getDeclKind()) {
-          case clang::Decl::TranslationUnit:
-            // We don't care about the translation unit names
-            return true;
-          default: {
-            clang::NamedDecl *lhs_named_decl =
-                llvm::dyn_cast<clang::NamedDecl>(lhs_decl_ctx);
-            clang::NamedDecl *rhs_named_decl =
-                llvm::dyn_cast<clang::NamedDecl>(rhs_decl_ctx);
-            if (lhs_named_decl && rhs_named_decl) {
-              clang::DeclarationName lhs_decl_name =
-                  lhs_named_decl->getDeclName();
-              clang::DeclarationName rhs_decl_name =
-                  rhs_named_decl->getDeclName();
-              if (lhs_decl_name.getNameKind() == rhs_decl_name.getNameKind()) {
-                if (lhs_decl_name.getAsString() != rhs_decl_name.getAsString())
-                  return false;
-              } else
-                return false;
-            } else
-              return false;
-          } break;
-          }
-          lhs_decl_ctx = lhs_decl_ctx->getParent();
-          rhs_decl_ctx = rhs_decl_ctx->getParent();
-        }
-      }
-    }
-  }
-  return false;
-}
 bool TypeSystemClang::GetCompleteDecl(clang::ASTContext *ast,
                                       clang::Decl *decl) {
   if (!decl)
@@ -9070,6 +8987,66 @@ size_t TypeSystemClang::DeclGetFunctionNumArguments(void *opaque_decl) {
     return 0;
 }
 
+static CompilerContextKind GetCompilerKind(clang::Decl::Kind clang_kind,
+                                           clang::DeclContext const *decl_ctx) {
+  switch (clang_kind) {
+  case Decl::TranslationUnit:
+    return CompilerContextKind::TranslationUnit;
+  case Decl::Namespace:
+    return CompilerContextKind::Namespace;
+  case Decl::Var:
+    return CompilerContextKind::Variable;
+  case Decl::Enum:
+    return CompilerContextKind::Enum;
+  case Decl::Typedef:
+    return CompilerContextKind::Typedef;
+  default:
+    // Many other kinds have multiple values
+    if (decl_ctx) {
+      if (decl_ctx->isFunctionOrMethod())
+        return CompilerContextKind::Function;
+      else if (decl_ctx->isRecord())
+        return (CompilerContextKind)((uint16_t)CompilerContextKind::Class |
+                                     (uint16_t)CompilerContextKind::Struct |
+                                     (uint16_t)CompilerContextKind::Union);
+    }
+    break;
+  }
+  return CompilerContextKind::Any;
+}
+
+static void
+InsertCompilerContext(TypeSystemClang *ts, clang::DeclContext *decl_ctx,
+                      std::vector<lldb_private::CompilerContext> &context) {
+  if (decl_ctx == nullptr)
+    return;
+  InsertCompilerContext(ts, decl_ctx->getParent(), context);
+  clang::Decl::Kind clang_kind = decl_ctx->getDeclKind();
+  if (clang_kind == Decl::TranslationUnit)
+    return; // Stop at the translation unit.
+  const CompilerContextKind compiler_kind =
+      GetCompilerKind(clang_kind, decl_ctx);
+  ConstString decl_ctx_name = ts->DeclContextGetName(decl_ctx);
+  context.push_back({compiler_kind, decl_ctx_name});
+}
+
+std::vector<lldb_private::CompilerContext>
+TypeSystemClang::DeclGetCompilerContext(void *opaque_decl) {
+  std::vector<lldb_private::CompilerContext> context;
+  ConstString decl_name = DeclGetName(opaque_decl);
+  if (decl_name) {
+    clang::Decl *decl = (clang::Decl *)opaque_decl;
+    // Add the entire decl context first
+    clang::DeclContext *decl_ctx = decl->getDeclContext();
+    InsertCompilerContext(this, decl_ctx, context);
+    // Now add the decl information
+    auto compiler_kind =
+        GetCompilerKind(decl->getKind(), dyn_cast<DeclContext>(decl));
+    context.push_back({compiler_kind, decl_name});
+  }
+  return context;
+}
+
 CompilerType TypeSystemClang::DeclGetFunctionArgumentType(void *opaque_decl,
                                                           size_t idx) {
   if (clang::FunctionDecl *func_decl =
@@ -9308,6 +9285,14 @@ bool TypeSystemClang::DeclContextIsClassMethod(void *opaque_decl_ctx) {
   return false;
 }
 
+std::vector<lldb_private::CompilerContext>
+TypeSystemClang::DeclContextGetCompilerContext(void *opaque_decl_ctx) {
+  auto *decl_ctx = (clang::DeclContext *)opaque_decl_ctx;
+  std::vector<lldb_private::CompilerContext> context;
+  InsertCompilerContext(this, decl_ctx, context);
+  return context;
+}
+
 bool TypeSystemClang::DeclContextIsContainedInLookup(
     void *opaque_decl_ctx, void *other_opaque_decl_ctx) {
   auto *decl_ctx = (clang::DeclContext *)opaque_decl_ctx;
diff --git a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h
index 19f267396e0f0e5a975f57d07d4a23962c55eb19..a73164895baa8050f27091fceddfa410fcfd3cc9 100644
--- a/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h
+++ b/lldb/source/Plugins/TypeSystem/Clang/TypeSystemClang.h
@@ -183,8 +183,6 @@ public:
 
   static void DumpDeclContextHiearchy(clang::DeclContext *decl_ctx);
 
-  static bool DeclsAreEquivalent(clang::Decl *lhs_decl, clang::Decl *rhs_decl);
-
   static bool GetCompleteDecl(clang::ASTContext *ast, clang::Decl *decl);
 
   void SetMetadataAsUserID(const clang::Decl *decl, lldb::user_id_t user_id);
@@ -558,6 +556,9 @@ public:
   CompilerType DeclGetFunctionArgumentType(void *opaque_decl,
                                            size_t arg_idx) override;
 
+  std::vector<lldb_private::CompilerContext>
+  DeclGetCompilerContext(void *opaque_decl) override;
+
   CompilerType GetTypeForDecl(void *opaque_decl) override;
 
   // CompilerDeclContext override functions
@@ -587,6 +588,9 @@ public:
 
   lldb::LanguageType DeclContextGetLanguage(void *opaque_decl_ctx) override;
 
+  std::vector<lldb_private::CompilerContext>
+  DeclContextGetCompilerContext(void *opaque_decl_ctx) override;
+
   // Clang specific clang::DeclContext functions
 
   static clang::DeclContext *
diff --git a/lldb/source/Symbol/CompilerDecl.cpp b/lldb/source/Symbol/CompilerDecl.cpp
index 3cafa9535a72b255f4763e602a82c7a99cfc77d7..0eb630e5b9e1137b189d25eff0e03770615224e2 100644
--- a/lldb/source/Symbol/CompilerDecl.cpp
+++ b/lldb/source/Symbol/CompilerDecl.cpp
@@ -47,3 +47,8 @@ bool lldb_private::operator!=(const lldb_private::CompilerDecl &lhs,
   return lhs.GetTypeSystem() != rhs.GetTypeSystem() ||
          lhs.GetOpaqueDecl() != rhs.GetOpaqueDecl();
 }
+
+std::vector<lldb_private::CompilerContext>
+CompilerDecl::GetCompilerContext() const {
+  return m_type_system->DeclGetCompilerContext(m_opaque_decl);
+}
diff --git a/lldb/source/Symbol/CompilerDeclContext.cpp b/lldb/source/Symbol/CompilerDeclContext.cpp
index a188e60251f7c89509981c434d9654950ca320da..b40a08e9b1953d4be553b9b14ef0d4b0f1366a3d 100644
--- a/lldb/source/Symbol/CompilerDeclContext.cpp
+++ b/lldb/source/Symbol/CompilerDeclContext.cpp
@@ -59,6 +59,13 @@ bool CompilerDeclContext::IsContainedInLookup(CompilerDeclContext other) const {
                                                        other.m_opaque_decl_ctx);
 }
 
+std::vector<lldb_private::CompilerContext>
+CompilerDeclContext::GetCompilerContext() const {
+  if (IsValid())
+    return m_type_system->DeclContextGetCompilerContext(m_opaque_decl_ctx);
+  return {};
+}
+
 bool lldb_private::operator==(const lldb_private::CompilerDeclContext &lhs,
                               const lldb_private::CompilerDeclContext &rhs) {
   return lhs.GetTypeSystem() == rhs.GetTypeSystem() &&
diff --git a/lldb/source/Symbol/SymbolFile.cpp b/lldb/source/Symbol/SymbolFile.cpp
index 4b9c3863e461590c847927700d8a73eb3e710817..e318e2beb6547b4e94591b28b646e833115e13dc 100644
--- a/lldb/source/Symbol/SymbolFile.cpp
+++ b/lldb/source/Symbol/SymbolFile.cpp
@@ -134,17 +134,6 @@ void SymbolFile::GetMangledNamesForFunction(
     const std::string &scope_qualified_name,
     std::vector<ConstString> &mangled_names) {}
 
-void SymbolFile::FindTypes(
-    ConstString name, const CompilerDeclContext &parent_decl_ctx,
-    uint32_t max_matches,
-    llvm::DenseSet<lldb_private::SymbolFile *> &searched_symbol_files,
-    TypeMap &types) {}
-
-void SymbolFile::FindTypes(llvm::ArrayRef<CompilerContext> pattern,
-                           LanguageSet languages,
-                           llvm::DenseSet<SymbolFile *> &searched_symbol_files,
-                           TypeMap &types) {}
-
 void SymbolFile::AssertModuleLock() {
   // The code below is too expensive to leave enabled in release builds. It's
   // enabled in debug builds or when the correct macro is set.
diff --git a/lldb/source/Symbol/SymbolFileOnDemand.cpp b/lldb/source/Symbol/SymbolFileOnDemand.cpp
index 19b519c63a8a0b7a6f6435a9943d905f5a394bd0..33995252bfe2c9ae7f46988980039bbcae088b2e 100644
--- a/lldb/source/Symbol/SymbolFileOnDemand.cpp
+++ b/lldb/source/Symbol/SymbolFileOnDemand.cpp
@@ -431,31 +431,14 @@ void SymbolFileOnDemand::GetMangledNamesForFunction(
                                                      mangled_names);
 }
 
-void SymbolFileOnDemand::FindTypes(
-    ConstString name, const CompilerDeclContext &parent_decl_ctx,
-    uint32_t max_matches,
-    llvm::DenseSet<lldb_private::SymbolFile *> &searched_symbol_files,
-    TypeMap &types) {
-  if (!m_debug_info_enabled) {
-    Log *log = GetLog();
-    LLDB_LOG(log, "[{0}] {1}({2}) is skipped", GetSymbolFileName(),
-             __FUNCTION__, name);
-    return;
-  }
-  return m_sym_file_impl->FindTypes(name, parent_decl_ctx, max_matches,
-                                    searched_symbol_files, types);
-}
-
-void SymbolFileOnDemand::FindTypes(
-    llvm::ArrayRef<CompilerContext> pattern, LanguageSet languages,
-    llvm::DenseSet<SymbolFile *> &searched_symbol_files, TypeMap &types) {
+void SymbolFileOnDemand::FindTypes(const TypeQuery &match,
+                                   TypeResults &results) {
   if (!m_debug_info_enabled) {
     LLDB_LOG(GetLog(), "[{0}] {1} is skipped", GetSymbolFileName(),
              __FUNCTION__);
     return;
   }
-  return m_sym_file_impl->FindTypes(pattern, languages, searched_symbol_files,
-                                    types);
+  return m_sym_file_impl->FindTypes(match, results);
 }
 
 void SymbolFileOnDemand::GetTypes(SymbolContextScope *sc_scope,
diff --git a/lldb/source/Symbol/Type.cpp b/lldb/source/Symbol/Type.cpp
index 54eeace93b964875b4e2d2a7eeaf3e614a6468f8..293fe1b78f4a54bea4df63d1200661cb995d857e 100644
--- a/lldb/source/Symbol/Type.cpp
+++ b/lldb/source/Symbol/Type.cpp
@@ -64,6 +64,127 @@ bool lldb_private::contextMatches(llvm::ArrayRef<CompilerContext> context_chain,
   return true;
 }
 
+static CompilerContextKind ConvertTypeClass(lldb::TypeClass type_class) {
+  if (type_class == eTypeClassAny)
+    return CompilerContextKind::AnyType;
+  uint16_t result = 0;
+  if (type_class & lldb::eTypeClassClass)
+    result |= (uint16_t)CompilerContextKind::Class;
+  if (type_class & lldb::eTypeClassStruct)
+    result |= (uint16_t)CompilerContextKind::Struct;
+  if (type_class & lldb::eTypeClassUnion)
+    result |= (uint16_t)CompilerContextKind::Union;
+  if (type_class & lldb::eTypeClassEnumeration)
+    result |= (uint16_t)CompilerContextKind::Enum;
+  if (type_class & lldb::eTypeClassFunction)
+    result |= (uint16_t)CompilerContextKind::Function;
+  if (type_class & lldb::eTypeClassTypedef)
+    result |= (uint16_t)CompilerContextKind::Typedef;
+  return (CompilerContextKind)result;
+}
+
+TypeQuery::TypeQuery(llvm::StringRef name, TypeQueryOptions options)
+    : m_options(options) {
+  llvm::StringRef scope, basename;
+  lldb::TypeClass type_class = lldb::eTypeClassAny;
+  if (Type::GetTypeScopeAndBasename(name, scope, basename, type_class)) {
+    if (scope.consume_front("::"))
+      m_options |= e_exact_match;
+    if (!scope.empty()) {
+      std::pair<llvm::StringRef, llvm::StringRef> scope_pair =
+          scope.split("::");
+      while (!scope_pair.second.empty()) {
+        m_context.push_back({CompilerContextKind::AnyDeclContext,
+                             ConstString(scope_pair.first.str())});
+        scope_pair = scope_pair.second.split("::");
+      }
+      m_context.push_back({CompilerContextKind::AnyDeclContext,
+                           ConstString(scope_pair.first.str())});
+    }
+    m_context.push_back(
+        {ConvertTypeClass(type_class), ConstString(basename.str())});
+  } else {
+    m_context.push_back(
+        {CompilerContextKind::AnyType, ConstString(name.str())});
+  }
+}
+
+TypeQuery::TypeQuery(const CompilerDeclContext &decl_ctx,
+                     ConstString type_basename, TypeQueryOptions options)
+    : m_options(options) {
+  // Always use an exact match if we are looking for a type in compiler context.
+  m_options |= e_exact_match;
+  m_context = decl_ctx.GetCompilerContext();
+  m_context.push_back({CompilerContextKind::AnyType, type_basename});
+}
+
+TypeQuery::TypeQuery(
+    const llvm::ArrayRef<lldb_private::CompilerContext> &context,
+    TypeQueryOptions options)
+    : m_context(context), m_options(options) {
+  // Always use an exact match if we are looking for a type in compiler context.
+  m_options |= e_exact_match;
+}
+
+TypeQuery::TypeQuery(const CompilerDecl &decl, TypeQueryOptions options)
+    : m_options(options) {
+  // Always for an exact match if we are looking for a type using a declaration.
+  m_options |= e_exact_match;
+  m_context = decl.GetCompilerContext();
+}
+
+ConstString TypeQuery::GetTypeBasename() const {
+  if (m_context.empty())
+    return ConstString();
+  return m_context.back().name;
+}
+
+void TypeQuery::AddLanguage(LanguageType language) {
+  if (!m_languages)
+    m_languages = LanguageSet();
+  m_languages->Insert(language);
+}
+
+bool TypeQuery::ContextMatches(
+    llvm::ArrayRef<CompilerContext> context_chain) const {
+  if (GetExactMatch() || context_chain.size() == m_context.size())
+    return ::contextMatches(context_chain, m_context);
+
+  // We don't have an exact match, we need to bottom m_context.size() items to
+  // match for a successful lookup.
+  if (context_chain.size() < m_context.size())
+    return false; // Not enough items in context_chain to allow for a match.
+
+  size_t compare_count = context_chain.size() - m_context.size();
+  return ::contextMatches(
+      llvm::ArrayRef<CompilerContext>(context_chain.data() + compare_count,
+                                      m_context.size()),
+      m_context);
+}
+
+bool TypeQuery::LanguageMatches(lldb::LanguageType language) const {
+  // If we have no language filterm language always matches.
+  if (!m_languages.has_value())
+    return true;
+  return (*m_languages)[language];
+}
+
+bool TypeResults::AlreadySearched(lldb_private::SymbolFile *sym_file) {
+  return !m_searched_symbol_files.insert(sym_file).second;
+}
+
+bool TypeResults::InsertUnique(const lldb::TypeSP &type_sp) {
+  if (type_sp)
+    return m_type_map.InsertUnique(type_sp);
+  return false;
+}
+
+bool TypeResults::Done(const TypeQuery &query) const {
+  if (query.GetFindOne())
+    return !m_type_map.Empty();
+  return false;
+}
+
 void CompilerContext::Dump(Stream &s) const {
   switch (kind) {
   default:
@@ -641,6 +762,8 @@ bool Type::GetTypeScopeAndBasename(llvm::StringRef name,
   if (name.empty())
     return false;
 
+  // Clear the scope in case we have just a type class and a basename.
+  scope = llvm::StringRef();
   basename = name;
   if (basename.consume_front("struct "))
     type_class = eTypeClassStruct;
@@ -654,8 +777,10 @@ bool Type::GetTypeScopeAndBasename(llvm::StringRef name,
     type_class = eTypeClassTypedef;
 
   size_t namespace_separator = basename.find("::");
-  if (namespace_separator == llvm::StringRef::npos)
-    return false;
+  if (namespace_separator == llvm::StringRef::npos) {
+    // If "name" started a type class we need to return true with no scope.
+    return type_class != eTypeClassAny;
+  }
 
   size_t template_begin = basename.find('<');
   while (namespace_separator != llvm::StringRef::npos) {
@@ -1049,16 +1174,19 @@ CompilerType TypeImpl::FindDirectNestedType(llvm::StringRef name) {
     return CompilerType();
   auto type_system = GetTypeSystem(/*prefer_dynamic*/ false);
   auto *symbol_file = type_system->GetSymbolFile();
+  if (!symbol_file)
+    return CompilerType();
   auto decl_context = type_system->GetCompilerDeclContextForType(m_static_type);
   if (!decl_context.IsValid())
     return CompilerType();
-  llvm::DenseSet<lldb_private::SymbolFile *> searched_symbol_files;
-  TypeMap search_result;
-  symbol_file->FindTypes(ConstString(name), decl_context, /*max_matches*/ 1,
-                         searched_symbol_files, search_result);
-  if (search_result.Empty())
-    return CompilerType();
-  return search_result.GetTypeAtIndex(0)->GetFullCompilerType();
+  TypeQuery query(decl_context, ConstString(name),
+                  TypeQueryOptions::e_find_one);
+  TypeResults results;
+  symbol_file->FindTypes(query, results);
+  TypeSP type_sp = results.GetFirstType();
+  if (type_sp)
+    return type_sp->GetFullCompilerType();
+  return CompilerType();
 }
 
 bool TypeMemberFunctionImpl::IsValid() {
diff --git a/lldb/source/Symbol/TypeMap.cpp b/lldb/source/Symbol/TypeMap.cpp
index 0d5f6d53e5a0481c789227c26e2c62cd0b849f4a..8933de53749cc9923291d67838c2689225f763b9 100644
--- a/lldb/source/Symbol/TypeMap.cpp
+++ b/lldb/source/Symbol/TypeMap.cpp
@@ -91,6 +91,12 @@ TypeSP TypeMap::GetTypeAtIndex(uint32_t idx) {
   return TypeSP();
 }
 
+lldb::TypeSP TypeMap::FirstType() const {
+  if (m_types.empty())
+    return TypeSP();
+  return m_types.begin()->second;
+}
+
 void TypeMap::ForEach(
     std::function<bool(const lldb::TypeSP &type_sp)> const &callback) const {
   for (auto pos = m_types.begin(), end = m_types.end(); pos != end; ++pos) {
@@ -121,10 +127,10 @@ bool TypeMap::Remove(const lldb::TypeSP &type_sp) {
   return false;
 }
 
-void TypeMap::Dump(Stream *s, bool show_context, lldb::DescriptionLevel level) {
-  for (iterator pos = m_types.begin(), end = m_types.end(); pos != end; ++pos) {
-    pos->second->Dump(s, show_context, level);
-  }
+void TypeMap::Dump(Stream *s, bool show_context,
+                   lldb::DescriptionLevel level) const {
+  for (const auto &pair : m_types)
+    pair.second->Dump(s, show_context, level);
 }
 
 void TypeMap::RemoveMismatchedTypes(llvm::StringRef type_scope,
diff --git a/lldb/source/Symbol/TypeSystem.cpp b/lldb/source/Symbol/TypeSystem.cpp
index 874f12573eca3f000efd835a31bca6961844fd66..59b1b39e635ac5d865d6e6f54f1b2ac645cc16fd 100644
--- a/lldb/source/Symbol/TypeSystem.cpp
+++ b/lldb/source/Symbol/TypeSystem.cpp
@@ -171,6 +171,16 @@ CompilerType TypeSystem::DeclGetFunctionArgumentType(void *opaque_decl,
   return CompilerType();
 }
 
+std::vector<lldb_private::CompilerContext>
+TypeSystem::DeclGetCompilerContext(void *opaque_decl) {
+  return {};
+}
+
+std::vector<lldb_private::CompilerContext>
+TypeSystem::DeclContextGetCompilerContext(void *opaque_decl_ctx) {
+  return {};
+}
+
 std::vector<CompilerDecl>
 TypeSystem::DeclContextFindDeclByName(void *opaque_decl_ctx, ConstString name,
                                       bool ignore_imported_decls) {
diff --git a/lldb/source/Target/Language.cpp b/lldb/source/Target/Language.cpp
index 42c3350806d90f53a0356e37e640a5303a0074a4..caf3e6636c1de13dea2586445146c9dafbe2201c 100644
--- a/lldb/source/Target/Language.cpp
+++ b/lldb/source/Target/Language.cpp
@@ -434,12 +434,10 @@ bool Language::ImageListTypeScavenger::Find_Impl(
   Target *target = exe_scope->CalculateTarget().get();
   if (target) {
     const auto &images(target->GetImages());
-    ConstString cs_key(key);
-    llvm::DenseSet<SymbolFile *> searched_sym_files;
-    TypeList matches;
-    images.FindTypes(nullptr, cs_key, false, UINT32_MAX, searched_sym_files,
-                     matches);
-    for (const auto &match : matches.Types()) {
+    TypeQuery query(key);
+    TypeResults type_results;
+    images.FindTypes(nullptr, query, type_results);
+    for (const auto &match : type_results.GetTypeMap().Types()) {
       if (match) {
         CompilerType compiler_type(match->GetFullCompilerType());
         compiler_type = AdjustForInclusion(compiler_type);
diff --git a/lldb/source/Target/Thread.cpp b/lldb/source/Target/Thread.cpp
index 865cee97e6d87832b28b5c38b3183b6aada92bc9..4801e3e58d9ef1010f746173f3129b43d3c1a24f 100644
--- a/lldb/source/Target/Thread.cpp
+++ b/lldb/source/Target/Thread.cpp
@@ -717,6 +717,17 @@ void Thread::DidResume() {
 
 void Thread::DidStop() { SetState(eStateStopped); }
 
+#define CHECK_BEFORE_POP_PLAN                                                  \
+  {                                                                            \
+    uint32_t i = 0;                                                            \
+    ThreadPlanSP p;                                                            \
+    while ((p = GetPlans().GetPlanByIndex(i, false)))                          \
+      i++;                                                                     \
+    (void)i;                                                                   \
+    assert(i != 1 &&                                                           \
+           "Cannot pop plan when there is only one plan (the base plan)");     \
+  }
+
 bool Thread::ShouldStop(Event *event_ptr) {
   ThreadPlan *current_plan = GetCurrentPlan();
 
@@ -831,6 +842,7 @@ bool Thread::ShouldStop(Event *event_ptr) {
             do {
               if (should_stop)
                 current_plan->WillStop();
+              CHECK_BEFORE_POP_PLAN;
               PopPlan();
             } while ((current_plan = GetCurrentPlan()) != prev_plan_ptr);
             // Now, if the responsible plan was not "Okay to discard" then
@@ -883,6 +895,7 @@ bool Thread::ShouldStop(Event *event_ptr) {
           // If a Controlling Plan wants to stop, we let it. Otherwise, see if
           // the plan's parent wants to stop.
 
+          CHECK_BEFORE_POP_PLAN;
           PopPlan();
           if (should_stop && current_plan->IsControllingPlan() &&
               !current_plan->OkayToDiscard()) {
@@ -931,6 +944,7 @@ bool Thread::ShouldStop(Event *event_ptr) {
           // plan is complete but does not explain the stop (example: step to a
           // line with breakpoint), let us move the plan to
           // completed_plan_stack anyway
+          CHECK_BEFORE_POP_PLAN;
           PopPlan();
         } else
           DiscardPlan();
diff --git a/lldb/source/Utility/ConstString.cpp b/lldb/source/Utility/ConstString.cpp
index ea897dc611cc94272859f003bb5bade0d6ebf1d9..4535771adfb73563131157b810dab70617bc8aac 100644
--- a/lldb/source/Utility/ConstString.cpp
+++ b/lldb/source/Utility/ConstString.cpp
@@ -77,10 +77,10 @@ public:
     return 0;
   }
 
-  StringPoolValueType GetMangledCounterpart(const char *ccstr) {
+  StringPoolValueType GetMangledCounterpart(const char *ccstr) const {
     if (ccstr != nullptr) {
-      const PoolEntry &pool = selectPool(llvm::StringRef(ccstr));
-      llvm::sys::SmartScopedReader<false> rlock(pool.m_mutex);
+      const uint8_t h = hash(llvm::StringRef(ccstr));
+      llvm::sys::SmartScopedReader<false> rlock(m_string_pools[h].m_mutex);
       return GetStringMapEntryFromKeyData(ccstr).getValue();
     }
     return nullptr;
@@ -100,20 +100,19 @@ public:
 
   const char *GetConstCStringWithStringRef(llvm::StringRef string_ref) {
     if (string_ref.data()) {
-      const uint32_t string_hash = StringPool::hash(string_ref);
-      PoolEntry &pool = selectPool(string_hash);
+      const uint8_t h = hash(string_ref);
 
       {
-        llvm::sys::SmartScopedReader<false> rlock(pool.m_mutex);
-        auto it = pool.m_string_map.find(string_ref, string_hash);
-        if (it != pool.m_string_map.end())
+        llvm::sys::SmartScopedReader<false> rlock(m_string_pools[h].m_mutex);
+        auto it = m_string_pools[h].m_string_map.find(string_ref);
+        if (it != m_string_pools[h].m_string_map.end())
           return it->getKeyData();
       }
 
-      llvm::sys::SmartScopedWriter<false> wlock(pool.m_mutex);
+      llvm::sys::SmartScopedWriter<false> wlock(m_string_pools[h].m_mutex);
       StringPoolEntryType &entry =
-          *pool.m_string_map
-               .insert(std::make_pair(string_ref, nullptr), string_hash)
+          *m_string_pools[h]
+               .m_string_map.insert(std::make_pair(string_ref, nullptr))
                .first;
       return entry.getKeyData();
     }
@@ -126,14 +125,12 @@ public:
     const char *demangled_ccstr = nullptr;
 
     {
-      const uint32_t demangled_hash = StringPool::hash(demangled);
-      PoolEntry &pool = selectPool(demangled_hash);
-      llvm::sys::SmartScopedWriter<false> wlock(pool.m_mutex);
+      const uint8_t h = hash(demangled);
+      llvm::sys::SmartScopedWriter<false> wlock(m_string_pools[h].m_mutex);
 
       // Make or update string pool entry with the mangled counterpart
-      StringPool &map = pool.m_string_map;
-      StringPoolEntryType &entry =
-          *map.try_emplace_with_hash(demangled, demangled_hash).first;
+      StringPool &map = m_string_pools[h].m_string_map;
+      StringPoolEntryType &entry = *map.try_emplace(demangled).first;
 
       entry.second = mangled_ccstr;
 
@@ -144,8 +141,8 @@ public:
     {
       // Now assign the demangled const string as the counterpart of the
       // mangled const string...
-      PoolEntry &pool = selectPool(llvm::StringRef(mangled_ccstr));
-      llvm::sys::SmartScopedWriter<false> wlock(pool.m_mutex);
+      const uint8_t h = hash(llvm::StringRef(mangled_ccstr));
+      llvm::sys::SmartScopedWriter<false> wlock(m_string_pools[h].m_mutex);
       GetStringMapEntryFromKeyData(mangled_ccstr).setValue(demangled_ccstr);
     }
 
@@ -174,20 +171,17 @@ public:
   }
 
 protected:
+  uint8_t hash(llvm::StringRef s) const {
+    uint32_t h = llvm::djbHash(s);
+    return ((h >> 24) ^ (h >> 16) ^ (h >> 8) ^ h) & 0xff;
+  }
+
   struct PoolEntry {
     mutable llvm::sys::SmartRWMutex<false> m_mutex;
     StringPool m_string_map;
   };
 
   std::array<PoolEntry, 256> m_string_pools;
-
-  PoolEntry &selectPool(const llvm::StringRef &s) {
-    return selectPool(StringPool::hash(s));
-  }
-
-  PoolEntry &selectPool(uint32_t h) {
-    return m_string_pools[((h >> 24) ^ (h >> 16) ^ (h >> 8) ^ h) & 0xff];
-  }
 };
 
 // Frameworks and dylibs aren't supposed to have global C++ initializers so we
@@ -203,7 +197,7 @@ static Pool &StringPool() {
   static Pool *g_string_pool = nullptr;
 
   llvm::call_once(g_pool_initialization_flag,
-                  []() { g_string_pool = new Pool(); });
+                 []() { g_string_pool = new Pool(); });
 
   return *g_string_pool;
 }
diff --git a/lldb/test/API/functionalities/data-formatter/builtin-formats/TestBuiltinFormats.py b/lldb/test/API/functionalities/data-formatter/builtin-formats/TestBuiltinFormats.py
index 4e0f14d039a743fe40b3cd90bd05e860311b3125..8c3bdabeaac1bab968187266a228f58124f0c41c 100644
--- a/lldb/test/API/functionalities/data-formatter/builtin-formats/TestBuiltinFormats.py
+++ b/lldb/test/API/functionalities/data-formatter/builtin-formats/TestBuiltinFormats.py
@@ -19,6 +19,7 @@ class TestCase(TestBase):
         self.assertTrue(result.Succeeded(), result.GetError())
         return result.GetOutput()
 
+    @skipIf(dwarf_version=["<", "3"])
     @no_debug_info_test
     @skipIfWindows
     def testAllPlatforms(self):
diff --git a/lldb/test/API/functionalities/location-list-lookup/TestLocationListLookup.py b/lldb/test/API/functionalities/location-list-lookup/TestLocationListLookup.py
index feea14ff355e829d7858a3698c6d02aa69ba21e3..c5f4a7c6dedc19e9eb9aaa11f80f248e1b1ff1a8 100644
--- a/lldb/test/API/functionalities/location-list-lookup/TestLocationListLookup.py
+++ b/lldb/test/API/functionalities/location-list-lookup/TestLocationListLookup.py
@@ -43,7 +43,8 @@ class LocationListLookupTestCase(TestBase):
         self.build()
         self.check_local_vars(self.launch(), check_expr=False)
 
-    @skipIf(compiler="clang", compiler_version=["<=", "11.0"])
+    @skipIf(dwarf_version=["<", "3"])
+    @skipIf(compiler="clang", compiler_version=["<", "12.0"])
     @skipUnlessDarwin
     def test_loclist_expr(self):
         self.build()
diff --git a/lldb/test/API/functionalities/type_find_first/Makefile b/lldb/test/API/functionalities/type_find_first/Makefile
new file mode 100644
index 0000000000000000000000000000000000000000..3d0b98f13f3d7be79aa6f4f5fd40d1697bb1e1a1
--- /dev/null
+++ b/lldb/test/API/functionalities/type_find_first/Makefile
@@ -0,0 +1,2 @@
+CXX_SOURCES := main.cpp
+include Makefile.rules
diff --git a/lldb/test/API/functionalities/type_find_first/TestFindFirstType.py b/lldb/test/API/functionalities/type_find_first/TestFindFirstType.py
new file mode 100644
index 0000000000000000000000000000000000000000..432708d144f2697c371e235c24dbfec8f812c39f
--- /dev/null
+++ b/lldb/test/API/functionalities/type_find_first/TestFindFirstType.py
@@ -0,0 +1,38 @@
+"""
+Test the SBModule and SBTarget type lookup APIs.
+"""
+
+import lldb
+from lldbsuite.test.lldbtest import *
+from lldbsuite.test import lldbutil
+
+
+class TypeFindFirstTestCase(TestBase):
+
+    NO_DEBUG_INFO_TESTCASE = True
+
+    def test_find_first_type(self):
+        """
+            Test SBTarget::FindFirstType() and SBModule::FindFirstType() APIs.
+
+            This function had regressed after some past modification of the type
+            lookup internal code where if we had multiple types with the same
+            basename, FindFirstType() could end up failing depending on which
+            type was found first in the debug info indexes. This test will
+            ensure this doesn't regress in the future.
+        """
+        self.build()
+        target = self.createTestTarget()
+        # Test the SBTarget APIs for FindFirstType
+        integer_type = target.FindFirstType("Integer::Point")
+        self.assertTrue(integer_type.IsValid())
+        float_type = target.FindFirstType("Float::Point")
+        self.assertTrue(float_type.IsValid())
+
+        # Test the SBModule APIs for FindFirstType
+        exe_module = target.GetModuleAtIndex(0)
+        self.assertTrue(exe_module.IsValid())
+        integer_type = exe_module.FindFirstType("Integer::Point")
+        self.assertTrue(integer_type.IsValid())
+        float_type = exe_module.FindFirstType("Float::Point")
+        self.assertTrue(float_type.IsValid())
diff --git a/lldb/test/API/functionalities/type_find_first/main.cpp b/lldb/test/API/functionalities/type_find_first/main.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..f4e467286004d6fe46324c38ce7b8cd828266173
--- /dev/null
+++ b/lldb/test/API/functionalities/type_find_first/main.cpp
@@ -0,0 +1,17 @@
+namespace Integer {
+struct Point {
+  int x, y;
+};
+} // namespace Integer
+
+namespace Float {
+struct Point {
+  float x, y;
+};
+} // namespace Float
+
+int main(int argc, char const *argv[]) {
+  Integer::Point ip = {2, 3};
+  Float::Point fp = {2.0, 3.0};
+  return 0;
+}
diff --git a/lldb/test/API/lang/cpp/union-static-data-members/TestCppUnionStaticMembers.py b/lldb/test/API/lang/cpp/union-static-data-members/TestCppUnionStaticMembers.py
index 1988e997499b222b00307f023deb4885ca53aa2b..dff23da8662a04ec65dc7543be8d4eff33d4dabe 100644
--- a/lldb/test/API/lang/cpp/union-static-data-members/TestCppUnionStaticMembers.py
+++ b/lldb/test/API/lang/cpp/union-static-data-members/TestCppUnionStaticMembers.py
@@ -42,7 +42,7 @@ class CppUnionStaticMembersTestCase(TestBase):
                 name="val", value="42"
             )])
 
-    @expectedFailureAll
+    @expectedFailureWindows
     def test_union_in_anon_namespace(self):
         """Tests that frame variable and expr work
            for union static data members in anonymous
diff --git a/lldb/test/API/lang/cpp/unique-types4/TestUniqueTypes4.py b/lldb/test/API/lang/cpp/unique-types4/TestUniqueTypes4.py
index d9ac07fd00da5c89cd426fcebc4d300b980c1c3e..3fa694fa159e15e9e73a78fdcdb229f077dd5f72 100644
--- a/lldb/test/API/lang/cpp/unique-types4/TestUniqueTypes4.py
+++ b/lldb/test/API/lang/cpp/unique-types4/TestUniqueTypes4.py
@@ -17,27 +17,20 @@ class UniqueTypesTestCase4(TestBase):
         )
         # FIXME: these should successfully print the values
         self.expect(
-            "expression ns::Foo<double>::value", substrs=["no member named"], error=True
+            "expression ns::Foo<double>::value", substrs=["'Foo' in namespace 'ns'"], error=True
         )
         self.expect(
-            "expression ns::Foo<int>::value", substrs=["no member named"], error=True
+            "expression ns::Foo<int>::value", substrs=["'Foo' in namespace 'ns'"], error=True
         )
         self.expect(
-            "expression ns::Bar<double>::value", substrs=["no member named"], error=True
+            "expression ns::Bar<double>::value", substrs=["'Bar' in namespace 'ns'"], error=True
         )
         self.expect(
-            "expression ns::Bar<int>::value", substrs=["no member named"], error=True
-        )
-        self.expect(
-            "expression ns::FooDouble::value",
-            substrs=["Couldn't look up symbols"],
-            error=True,
-        )
-        self.expect(
-            "expression ns::FooInt::value",
-            substrs=["Couldn't look up symbols"],
-            error=True,
+            "expression ns::Bar<int>::value", substrs=["'Bar' in namespace 'ns'"], error=True
         )
+        self.expect_expr("ns::FooDouble::value", result_type="double", result_value="0")
+        self.expect_expr("ns::FooInt::value", result_type="int", result_value="0")
+
 
     @skipIf(compiler=no_match("clang"))
     @skipIf(compiler_version=["<", "15.0"])
diff --git a/lldb/test/API/lang/cpp/unique-types4/main.cpp b/lldb/test/API/lang/cpp/unique-types4/main.cpp
index 830635202a750a6bff93c1fc2707e6ac3011cbb5..815b4f92c7dbc2a3c74c136d457c6afacc09df63 100644
--- a/lldb/test/API/lang/cpp/unique-types4/main.cpp
+++ b/lldb/test/API/lang/cpp/unique-types4/main.cpp
@@ -4,6 +4,8 @@ template <typename T> struct Foo {
   static T value;
 };
 
+template <class T> T Foo<T>::value = 0;
+
 template <typename T> using Bar = Foo<T>;
 
 using FooInt = Foo<int>;
@@ -20,4 +22,6 @@ ns::FooDouble f;
 
 int main() {
   // Set breakpoint here
+  return (int)a.value + b.value + (int)c.value + d.value + e.value +
+         (int)f.value;
 }
diff --git a/lldb/test/API/python_api/global_module_cache/TestGlobalModuleCache.py b/lldb/test/API/python_api/global_module_cache/TestGlobalModuleCache.py
index 98ebdec0404eb40dbd334d74b9b13b9065057dd9..b8675532e6394f41e641451959e750aade40f4ae 100644
--- a/lldb/test/API/python_api/global_module_cache/TestGlobalModuleCache.py
+++ b/lldb/test/API/python_api/global_module_cache/TestGlobalModuleCache.py
@@ -50,6 +50,11 @@ class GlobalModuleCacheTestCase(TestBase):
         self.do_test(True, False)
 
     def do_test(self, one_target, one_debugger):
+        # Here to debug flakiness on Arm, remove later!
+        log_cmd_result = lldb.SBCommandReturnObject()
+        interp = self.dbg.GetCommandInterpreter()
+        interp.HandleCommand("log enable lldb step", log_cmd_result)
+
         # Make sure that if we have one target, and we run, then
         # change the binary and rerun, the binary (and any .o files
         # if using dwarf in .o file debugging) get removed from the
diff --git a/lldb/tools/lldb-test/lldb-test.cpp b/lldb/tools/lldb-test/lldb-test.cpp
index 45911b9065c2f8d5cb1218cd33a16a770fba2edd..e326a84c1dbd2faa2daa97d6975b0174807bcfeb 100644
--- a/lldb/tools/lldb-test/lldb-test.cpp
+++ b/lldb/tools/lldb-test/lldb-test.cpp
@@ -290,8 +290,8 @@ int lldb_assert(Debugger &Dbg);
 } // namespace assert
 } // namespace opts
 
-std::vector<CompilerContext> parseCompilerContext() {
-  std::vector<CompilerContext> result;
+llvm::SmallVector<CompilerContext, 4> parseCompilerContext() {
+  llvm::SmallVector<CompilerContext, 4> result;
   if (opts::symbols::CompilerContext.empty())
     return result;
 
@@ -577,29 +577,33 @@ Error opts::symbols::findTypes(lldb_private::Module &Module) {
   Expected<CompilerDeclContext> ContextOr = getDeclContext(Symfile);
   if (!ContextOr)
     return ContextOr.takeError();
-  const CompilerDeclContext &ContextPtr =
-      ContextOr->IsValid() ? *ContextOr : CompilerDeclContext();
-
-  LanguageSet languages;
-  if (!Language.empty())
-    languages.Insert(Language::GetLanguageTypeFromString(Language));
-
-  DenseSet<SymbolFile *> SearchedFiles;
-  TypeMap Map;
-  if (!Name.empty())
-    Symfile.FindTypes(ConstString(Name), ContextPtr, UINT32_MAX, SearchedFiles,
-                      Map);
-  else
-    Module.FindTypes(parseCompilerContext(), languages, SearchedFiles, Map);
 
-  outs() << formatv("Found {0} types:\n", Map.GetSize());
+  TypeResults results;
+  if (!Name.empty()) {
+    if (ContextOr->IsValid()) {
+      TypeQuery query(*ContextOr, ConstString(Name),
+                      TypeQueryOptions::e_module_search);
+      if (!Language.empty())
+        query.AddLanguage(Language::GetLanguageTypeFromString(Language));
+      Symfile.FindTypes(query, results);
+    } else {
+      TypeQuery query(Name);
+      if (!Language.empty())
+        query.AddLanguage(Language::GetLanguageTypeFromString(Language));
+      Symfile.FindTypes(query, results);
+    }
+  } else {
+    TypeQuery query(parseCompilerContext(), TypeQueryOptions::e_module_search);
+    if (!Language.empty())
+      query.AddLanguage(Language::GetLanguageTypeFromString(Language));
+    Symfile.FindTypes(query, results);
+  }
+  outs() << formatv("Found {0} types:\n", results.GetTypeMap().GetSize());
   StreamString Stream;
   // Resolve types to force-materialize typedef types.
-  Map.ForEach([&](TypeSP &type) {
-    type->GetFullCompilerType();
-    return false;
-  });
-  Map.Dump(&Stream, false, GetDescriptionLevel());
+  for (const auto &type_sp : results.GetTypeMap().Types())
+    type_sp->GetFullCompilerType();
+  results.GetTypeMap().Dump(&Stream, false, GetDescriptionLevel());
   outs() << Stream.GetData() << "\n";
   return Error::success();
 }
diff --git a/lldb/unittests/SymbolFile/PDB/SymbolFilePDBTests.cpp b/lldb/unittests/SymbolFile/PDB/SymbolFilePDBTests.cpp
index acd381ccad13d21ca4aca7dedd57720a5a05d9a7..cdaa4fafa4c873ea8769bb9f466227d07df3c47f 100644
--- a/lldb/unittests/SymbolFile/PDB/SymbolFilePDBTests.cpp
+++ b/lldb/unittests/SymbolFile/PDB/SymbolFilePDBTests.cpp
@@ -362,10 +362,9 @@ TEST_F(SymbolFilePDBTests, TestSimpleClassTypes) {
   SymbolFilePDB *symfile =
       static_cast<SymbolFilePDB *>(module->GetSymbolFile());
   llvm::pdb::IPDBSession &session = symfile->GetPDBSession();
-  llvm::DenseSet<SymbolFile *> searched_files;
-  TypeMap results;
-  symfile->FindTypes(ConstString("Class"), CompilerDeclContext(), 0,
-                     searched_files, results);
+  TypeResults query_results;
+  symfile->FindTypes(TypeQuery("Class"), query_results);
+  TypeMap &results = query_results.GetTypeMap();
   EXPECT_EQ(1u, results.GetSize());
   lldb::TypeSP udt_type = results.GetTypeAtIndex(0);
   EXPECT_EQ(ConstString("Class"), udt_type->GetName());
@@ -383,8 +382,6 @@ TEST_F(SymbolFilePDBTests, TestNestedClassTypes) {
   SymbolFilePDB *symfile =
       static_cast<SymbolFilePDB *>(module->GetSymbolFile());
   llvm::pdb::IPDBSession &session = symfile->GetPDBSession();
-  llvm::DenseSet<SymbolFile *> searched_files;
-  TypeMap results;
 
   auto clang_ast_ctx_or_err =
       symfile->GetTypeSystemForLanguage(lldb::eLanguageTypeC_plus_plus);
@@ -394,8 +391,10 @@ TEST_F(SymbolFilePDBTests, TestNestedClassTypes) {
       llvm::dyn_cast_or_null<TypeSystemClang>(clang_ast_ctx_or_err->get());
   EXPECT_NE(nullptr, clang_ast_ctx);
 
-  symfile->FindTypes(ConstString("Class"), CompilerDeclContext(), 0,
-                     searched_files, results);
+  TypeResults query_results;
+  symfile->FindTypes(TypeQuery("Class"), query_results);
+  TypeMap &results = query_results.GetTypeMap();
+
   EXPECT_EQ(1u, results.GetSize());
 
   auto Class = results.GetTypeAtIndex(0);
@@ -413,10 +412,12 @@ TEST_F(SymbolFilePDBTests, TestNestedClassTypes) {
   // compiler type for both, but `FindTypes` may return more than one type
   // (with the same compiler type) because the symbols have different IDs.
 
-  TypeMap more_results;
   auto ClassCompilerDeclCtx = CompilerDeclContext(clang_ast_ctx, ClassDeclCtx);
-  symfile->FindTypes(ConstString("NestedClass"), ClassCompilerDeclCtx, 0,
-                     searched_files, more_results);
+  TypeResults query_results_nested;
+  symfile->FindTypes(
+      TypeQuery(ClassCompilerDeclCtx, ConstString("NestedClass")),
+      query_results_nested);
+  TypeMap &more_results = query_results_nested.GetTypeMap();
   EXPECT_LE(1u, more_results.GetSize());
 
   lldb::TypeSP udt_type = more_results.GetTypeAtIndex(0);
@@ -437,9 +438,6 @@ TEST_F(SymbolFilePDBTests, TestClassInNamespace) {
   SymbolFilePDB *symfile =
       static_cast<SymbolFilePDB *>(module->GetSymbolFile());
   llvm::pdb::IPDBSession &session = symfile->GetPDBSession();
-  llvm::DenseSet<SymbolFile *> searched_files;
-  TypeMap results;
-
   auto clang_ast_ctx_or_err =
       symfile->GetTypeSystemForLanguage(lldb::eLanguageTypeC_plus_plus);
   ASSERT_THAT_EXPECTED(clang_ast_ctx_or_err, llvm::Succeeded());
@@ -456,12 +454,14 @@ TEST_F(SymbolFilePDBTests, TestClassInNamespace) {
   symfile->ParseDeclsForContext(CompilerDeclContext(
       clang_ast_ctx, static_cast<clang::DeclContext *>(tu)));
 
-  auto ns_namespace =
+  auto ns_namespace_decl_ctx =
       symfile->FindNamespace(ConstString("NS"), CompilerDeclContext(), true);
-  EXPECT_TRUE(ns_namespace.IsValid());
+  EXPECT_TRUE(ns_namespace_decl_ctx.IsValid());
 
-  symfile->FindTypes(ConstString("NSClass"), ns_namespace, 0, searched_files,
-                     results);
+  TypeResults query_results;
+  symfile->FindTypes(TypeQuery(ns_namespace_decl_ctx, ConstString("NSClass")),
+                     query_results);
+  TypeMap &results = query_results.GetTypeMap();
   EXPECT_EQ(1u, results.GetSize());
 
   lldb::TypeSP udt_type = results.GetTypeAtIndex(0);
@@ -482,12 +482,12 @@ TEST_F(SymbolFilePDBTests, TestEnumTypes) {
   SymbolFilePDB *symfile =
       static_cast<SymbolFilePDB *>(module->GetSymbolFile());
   llvm::pdb::IPDBSession &session = symfile->GetPDBSession();
-  llvm::DenseSet<SymbolFile *> searched_files;
   const char *EnumsToCheck[] = {"Enum", "ShortEnum"};
   for (auto Enum : EnumsToCheck) {
-    TypeMap results;
-    symfile->FindTypes(ConstString(Enum), CompilerDeclContext(), 0,
-                       searched_files, results);
+
+    TypeResults query_results;
+    symfile->FindTypes(TypeQuery(Enum), query_results);
+    TypeMap &results = query_results.GetTypeMap();
     EXPECT_EQ(1u, results.GetSize());
     lldb::TypeSP enum_type = results.GetTypeAtIndex(0);
     EXPECT_EQ(ConstString(Enum), enum_type->GetName());
@@ -527,16 +527,15 @@ TEST_F(SymbolFilePDBTests, TestTypedefs) {
   SymbolFilePDB *symfile =
       static_cast<SymbolFilePDB *>(module->GetSymbolFile());
   llvm::pdb::IPDBSession &session = symfile->GetPDBSession();
-  llvm::DenseSet<SymbolFile *> searched_files;
   TypeMap results;
 
   const char *TypedefsToCheck[] = {"ClassTypedef", "NSClassTypedef",
                                    "FuncPointerTypedef",
                                    "VariadicFuncPointerTypedef"};
   for (auto Typedef : TypedefsToCheck) {
-    TypeMap results;
-    symfile->FindTypes(ConstString(Typedef), CompilerDeclContext(), 0,
-                       searched_files, results);
+    TypeResults query_results;
+    symfile->FindTypes(TypeQuery(Typedef), query_results);
+    TypeMap &results = query_results.GetTypeMap();
     EXPECT_EQ(1u, results.GetSize());
     lldb::TypeSP typedef_type = results.GetTypeAtIndex(0);
     EXPECT_EQ(ConstString(Typedef), typedef_type->GetName());
@@ -578,22 +577,24 @@ TEST_F(SymbolFilePDBTests, TestMaxMatches) {
 
   SymbolFilePDB *symfile =
       static_cast<SymbolFilePDB *>(module->GetSymbolFile());
-  llvm::DenseSet<SymbolFile *> searched_files;
-  TypeMap results;
-  const ConstString name("ClassTypedef");
-  symfile->FindTypes(name, CompilerDeclContext(), 0, searched_files, results);
-  // Try to limit ourselves from 1 to 10 results, otherwise we could
-  // be doing this thousands of times.  The idea is just to make sure
-  // that for a variety of values, the number of limited results
-  // always comes out to the number we are expecting.
-  uint32_t num_results = results.GetSize();
-  uint32_t iterations = std::min(num_results, 10u);
-  for (uint32_t i = 1; i <= iterations; ++i) {
-    TypeMap more_results;
-    symfile->FindTypes(name, CompilerDeclContext(), i, searched_files,
-                       more_results);
-    uint32_t num_limited_results = more_results.GetSize();
-    EXPECT_EQ(i, num_limited_results);
+
+  // Make a type query object we can use for all types and for one type
+  TypeQuery query("ClassTypedef");
+  {
+    // Find all types that match
+    TypeResults query_results;
+    symfile->FindTypes(query, query_results);
+    TypeMap &results = query_results.GetTypeMap();
+    EXPECT_GT(results.GetSize(), 1u);
+  }
+
+  {
+    // Find a single type that matches
+    query.SetFindOne(true);
+    TypeResults query_results;
+    symfile->FindTypes(query, query_results);
+    TypeMap &results = query_results.GetTypeMap();
+    EXPECT_EQ(results.GetSize(), 1u);
   }
 }
 
@@ -604,10 +605,10 @@ TEST_F(SymbolFilePDBTests, TestNullName) {
 
   SymbolFilePDB *symfile =
       static_cast<SymbolFilePDB *>(module->GetSymbolFile());
-  llvm::DenseSet<SymbolFile *> searched_files;
-  TypeMap results;
-  symfile->FindTypes(ConstString(), CompilerDeclContext(), 0, searched_files,
-                     results);
+
+  TypeResults query_results;
+  symfile->FindTypes(TypeQuery(llvm::StringRef()), query_results);
+  TypeMap &results = query_results.GetTypeMap();
   EXPECT_EQ(0u, results.GetSize());
 }
 
diff --git a/llvm/CMakeLists.txt b/llvm/CMakeLists.txt
index 35be9bf16b5e1e634a058e2b634c1f9fa6cc755b..1ecfb36ab41a3c177d463cb0cdab2a572a53134d 100644
--- a/llvm/CMakeLists.txt
+++ b/llvm/CMakeLists.txt
@@ -1259,6 +1259,7 @@ if( LLVM_INCLUDE_TESTS )
   add_custom_target(test-depends
       DEPENDS ${LLVM_ALL_LIT_DEPENDS} ${LLVM_ALL_ADDITIONAL_TEST_DEPENDS})
   set_target_properties(test-depends PROPERTIES FOLDER "Tests")
+  add_dependencies(check-all test-depends)
 endif()
 
 if (LLVM_INCLUDE_DOCS)
diff --git a/llvm/docs/AMDGPUUsage.rst b/llvm/docs/AMDGPUUsage.rst
index 7fb3d70bbeffebb019dbea5e41545c1b85376a9a..5fcf651046943f518dfb4327832c7470bb43e509 100644
--- a/llvm/docs/AMDGPUUsage.rst
+++ b/llvm/docs/AMDGPUUsage.rst
@@ -1211,10 +1211,12 @@ The AMDGPU backend supports the following LLVM IR attributes.
                                              "amdgpu-flat-work-group-size" value, the implied occupancy
                                              bounds by the workgroup size takes precedence.
 
-     "amdgpu-ieee" true/false.               Specify whether the function expects the IEEE field of the
+     "amdgpu-ieee" true/false.               GFX6-GFX11 Only
+                                             Specify whether the function expects the IEEE field of the
                                              mode register to be set on entry. Overrides the default for
                                              the calling convention.
-     "amdgpu-dx10-clamp" true/false.         Specify whether the function expects the DX10_CLAMP field of
+     "amdgpu-dx10-clamp" true/false.         GFX6-GFX11 Only
+                                             Specify whether the function expects the DX10_CLAMP field of
                                              the mode register to be set on entry. Overrides the default
                                              for the calling convention.
 
@@ -4390,21 +4392,21 @@ The fields used by CP for code objects before V3 also match those specified in
                                                        ``COMPUTE_PGM_RSRC3``
                                                        configuration
                                                        register. See
-                                                       :ref:`amdgpu-amdhsa-compute_pgm_rsrc3-gfx10-gfx11-table`.
+                                                       :ref:`amdgpu-amdhsa-compute_pgm_rsrc3-gfx10-gfx12-table`.
      415:384 4 bytes COMPUTE_PGM_RSRC1               Compute Shader (CS)
                                                      program settings used by
                                                      CP to set up
                                                      ``COMPUTE_PGM_RSRC1``
                                                      configuration
                                                      register. See
-                                                     :ref:`amdgpu-amdhsa-compute_pgm_rsrc1-gfx6-gfx11-table`.
+                                                     :ref:`amdgpu-amdhsa-compute_pgm_rsrc1-gfx6-gfx12-table`.
      447:416 4 bytes COMPUTE_PGM_RSRC2               Compute Shader (CS)
                                                      program settings used by
                                                      CP to set up
                                                      ``COMPUTE_PGM_RSRC2``
                                                      configuration
                                                      register. See
-                                                     :ref:`amdgpu-amdhsa-compute_pgm_rsrc2-gfx6-gfx11-table`.
+                                                     :ref:`amdgpu-amdhsa-compute_pgm_rsrc2-gfx6-gfx12-table`.
      458:448 7 bits  *See separate bits below.*      Enable the setup of the
                                                      SGPR user data registers
                                                      (see
@@ -4472,8 +4474,8 @@ The fields used by CP for code objects before V3 also match those specified in
 
 ..
 
-  .. table:: compute_pgm_rsrc1 for GFX6-GFX11
-     :name: amdgpu-amdhsa-compute_pgm_rsrc1-gfx6-gfx11-table
+  .. table:: compute_pgm_rsrc1 for GFX6-GFX12
+     :name: amdgpu-amdhsa-compute_pgm_rsrc1-gfx6-gfx12-table
 
      ======= ======= =============================== ===========================================================================
      Bits    Size    Field Name                      Description
@@ -4642,17 +4644,27 @@ The fields used by CP for code objects before V3 also match those specified in
                                                      CP is responsible for
                                                      filling in
                                                      ``COMPUTE_PGM_RSRC1.PRIV``.
-     21      1 bit   ENABLE_DX10_CLAMP               Wavefront starts execution
-                                                     with DX10 clamp mode
-                                                     enabled. Used by the vector
-                                                     ALU to force DX10 style
-                                                     treatment of NaN's (when
-                                                     set, clamp NaN to zero,
-                                                     otherwise pass NaN
-                                                     through).
+     21      1 bit   ENABLE_DX10_CLAMP               GFX9-GFX11
+                                                       Wavefront starts execution
+                                                       with DX10 clamp mode
+                                                       enabled. Used by the vector
+                                                       ALU to force DX10 style
+                                                       treatment of NaN's (when
+                                                       set, clamp NaN to zero,
+                                                       otherwise pass NaN
+                                                       through).
 
-                                                     Used by CP to set up
-                                                     ``COMPUTE_PGM_RSRC1.DX10_CLAMP``.
+                                                       Used by CP to set up
+                                                       ``COMPUTE_PGM_RSRC1.DX10_CLAMP``.
+                     WG_RR_EN                        GFX12
+                                                       If 1, wavefronts are scheduled
+                                                       in a round-robin fashion with
+                                                       respect to the other wavefronts
+                                                       of the SIMD. Otherwise, wavefronts
+                                                       are scheduled in oldest age order.
+
+                                                       CP is responsible for filling in
+                                                       ``COMPUTE_PGM_RSRC1.WG_RR_EN``.
      22      1 bit   DEBUG_MODE                      Must be 0.
 
                                                      Start executing wavefront
@@ -4661,21 +4673,24 @@ The fields used by CP for code objects before V3 also match those specified in
                                                      CP is responsible for
                                                      filling in
                                                      ``COMPUTE_PGM_RSRC1.DEBUG_MODE``.
-     23      1 bit   ENABLE_IEEE_MODE                Wavefront starts execution
-                                                     with IEEE mode
-                                                     enabled. Floating point
-                                                     opcodes that support
-                                                     exception flag gathering
-                                                     will quiet and propagate
-                                                     signaling-NaN inputs per
-                                                     IEEE 754-2008. Min_dx10 and
-                                                     max_dx10 become IEEE
-                                                     754-2008 compliant due to
-                                                     signaling-NaN propagation
-                                                     and quieting.
+     23      1 bit   ENABLE_IEEE_MODE                GFX9-GFX11
+                                                       Wavefront starts execution
+                                                       with IEEE mode
+                                                       enabled. Floating point
+                                                       opcodes that support
+                                                       exception flag gathering
+                                                       will quiet and propagate
+                                                       signaling-NaN inputs per
+                                                       IEEE 754-2008. Min_dx10 and
+                                                       max_dx10 become IEEE
+                                                       754-2008 compliant due to
+                                                       signaling-NaN propagation
+                                                       and quieting.
 
-                                                     Used by CP to set up
-                                                     ``COMPUTE_PGM_RSRC1.IEEE_MODE``.
+                                                       Used by CP to set up
+                                                       ``COMPUTE_PGM_RSRC1.IEEE_MODE``.
+                     DISABLE_PERF                    GFX12
+                                                       Reserved. Must be 0.
      24      1 bit   BULKY                           Must be 0.
 
                                                      Only one work-group allowed
@@ -4763,8 +4778,8 @@ The fields used by CP for code objects before V3 also match those specified in
 
 ..
 
-  .. table:: compute_pgm_rsrc2 for GFX6-GFX11
-     :name: amdgpu-amdhsa-compute_pgm_rsrc2-gfx6-gfx11-table
+  .. table:: compute_pgm_rsrc2 for GFX6-GFX12
+     :name: amdgpu-amdhsa-compute_pgm_rsrc2-gfx6-gfx12-table
 
      ======= ======= =============================== ===========================================================================
      Bits    Size    Field Name                      Description
@@ -4957,8 +4972,8 @@ The fields used by CP for code objects before V3 also match those specified in
 
 ..
 
-  .. table:: compute_pgm_rsrc3 for GFX10-GFX11
-     :name: amdgpu-amdhsa-compute_pgm_rsrc3-gfx10-gfx11-table
+  .. table:: compute_pgm_rsrc3 for GFX10-GFX12
+     :name: amdgpu-amdhsa-compute_pgm_rsrc3-gfx10-gfx12-table
 
      ======= ======= =============================== ===========================================================================
      Bits    Size    Field Name                      Description
@@ -5437,7 +5452,7 @@ There are different methods used for initializing flat scratch:
   specifies *Architected flat scratch*:
 
   If ENABLE_PRIVATE_SEGMENT is enabled in
-  :ref:`amdgpu-amdhsa-compute_pgm_rsrc2-gfx6-gfx11-table` then the FLAT_SCRATCH
+  :ref:`amdgpu-amdhsa-compute_pgm_rsrc2-gfx6-gfx12-table` then the FLAT_SCRATCH
   register pair will be initialized to the 64-bit address of the base of scratch
   backing memory being managed by SPI for the queue executing the kernel
   dispatch plus the value of the wave's Scratch Wavefront Offset for use as the
@@ -11819,7 +11834,7 @@ Wavefronts are executed in native mode with in-order reporting of loads and
 sample instructions. In this mode vmcnt reports completion of load, atomic with
 return and sample instructions in order, and the vscnt reports the completion of
 store and atomic without return in order. See ``MEM_ORDERED`` field in
-:ref:`amdgpu-amdhsa-compute_pgm_rsrc1-gfx6-gfx11-table`.
+:ref:`amdgpu-amdhsa-compute_pgm_rsrc1-gfx6-gfx12-table`.
 
 Wavefronts can be executed in WGP or CU wavefront execution mode:
 
@@ -11835,7 +11850,7 @@ Wavefronts can be executed in WGP or CU wavefront execution mode:
   work-group synchronization.
 
 See ``WGP_MODE`` field in
-:ref:`amdgpu-amdhsa-compute_pgm_rsrc1-gfx6-gfx11-table` and
+:ref:`amdgpu-amdhsa-compute_pgm_rsrc1-gfx6-gfx12-table` and
 :ref:`amdgpu-target-features`.
 
 The code sequences used to implement the memory model for GFX10-GFX11 are defined in
@@ -15375,123 +15390,125 @@ terminated by an ``.end_amdhsa_kernel`` directive.
      ======================================================== =================== ============ ===================
      Directive                                                Default             Supported On Description
      ======================================================== =================== ============ ===================
-     ``.amdhsa_group_segment_fixed_size``                     0                   GFX6-GFX11   Controls GROUP_SEGMENT_FIXED_SIZE in
+     ``.amdhsa_group_segment_fixed_size``                     0                   GFX6-GFX12   Controls GROUP_SEGMENT_FIXED_SIZE in
                                                                                                :ref:`amdgpu-amdhsa-kernel-descriptor-v3-table`.
-     ``.amdhsa_private_segment_fixed_size``                   0                   GFX6-GFX11   Controls PRIVATE_SEGMENT_FIXED_SIZE in
+     ``.amdhsa_private_segment_fixed_size``                   0                   GFX6-GFX12   Controls PRIVATE_SEGMENT_FIXED_SIZE in
                                                                                                :ref:`amdgpu-amdhsa-kernel-descriptor-v3-table`.
-     ``.amdhsa_kernarg_size``                                 0                   GFX6-GFX11   Controls KERNARG_SIZE in
+     ``.amdhsa_kernarg_size``                                 0                   GFX6-GFX12   Controls KERNARG_SIZE in
                                                                                                :ref:`amdgpu-amdhsa-kernel-descriptor-v3-table`.
-     ``.amdhsa_user_sgpr_count``                              0                   GFX6-GFX11   Controls USER_SGPR_COUNT in COMPUTE_PGM_RSRC2
-                                                                                               :ref:`amdgpu-amdhsa-compute_pgm_rsrc2-gfx6-gfx11-table`
+     ``.amdhsa_user_sgpr_count``                              0                   GFX6-GFX12   Controls USER_SGPR_COUNT in COMPUTE_PGM_RSRC2
+                                                                                               :ref:`amdgpu-amdhsa-compute_pgm_rsrc2-gfx6-gfx12-table`
      ``.amdhsa_user_sgpr_private_segment_buffer``             0                   GFX6-GFX10   Controls ENABLE_SGPR_PRIVATE_SEGMENT_BUFFER in
                                                                                   (except      :ref:`amdgpu-amdhsa-kernel-descriptor-v3-table`.
                                                                                   GFX940)
-     ``.amdhsa_user_sgpr_dispatch_ptr``                       0                   GFX6-GFX11   Controls ENABLE_SGPR_DISPATCH_PTR in
+     ``.amdhsa_user_sgpr_dispatch_ptr``                       0                   GFX6-GFX12   Controls ENABLE_SGPR_DISPATCH_PTR in
                                                                                                :ref:`amdgpu-amdhsa-kernel-descriptor-v3-table`.
-     ``.amdhsa_user_sgpr_queue_ptr``                          0                   GFX6-GFX11   Controls ENABLE_SGPR_QUEUE_PTR in
+     ``.amdhsa_user_sgpr_queue_ptr``                          0                   GFX6-GFX12   Controls ENABLE_SGPR_QUEUE_PTR in
                                                                                                :ref:`amdgpu-amdhsa-kernel-descriptor-v3-table`.
-     ``.amdhsa_user_sgpr_kernarg_segment_ptr``                0                   GFX6-GFX11   Controls ENABLE_SGPR_KERNARG_SEGMENT_PTR in
+     ``.amdhsa_user_sgpr_kernarg_segment_ptr``                0                   GFX6-GFX12   Controls ENABLE_SGPR_KERNARG_SEGMENT_PTR in
                                                                                                :ref:`amdgpu-amdhsa-kernel-descriptor-v3-table`.
-     ``.amdhsa_user_sgpr_dispatch_id``                        0                   GFX6-GFX11   Controls ENABLE_SGPR_DISPATCH_ID in
+     ``.amdhsa_user_sgpr_dispatch_id``                        0                   GFX6-GFX12   Controls ENABLE_SGPR_DISPATCH_ID in
                                                                                                :ref:`amdgpu-amdhsa-kernel-descriptor-v3-table`.
      ``.amdhsa_user_sgpr_flat_scratch_init``                  0                   GFX6-GFX10   Controls ENABLE_SGPR_FLAT_SCRATCH_INIT in
                                                                                   (except      :ref:`amdgpu-amdhsa-kernel-descriptor-v3-table`.
                                                                                   GFX940)
-     ``.amdhsa_user_sgpr_private_segment_size``               0                   GFX6-GFX11   Controls ENABLE_SGPR_PRIVATE_SEGMENT_SIZE in
+     ``.amdhsa_user_sgpr_private_segment_size``               0                   GFX6-GFX12   Controls ENABLE_SGPR_PRIVATE_SEGMENT_SIZE in
                                                                                                :ref:`amdgpu-amdhsa-kernel-descriptor-v3-table`.
-     ``.amdhsa_wavefront_size32``                             Target              GFX10-GFX11  Controls ENABLE_WAVEFRONT_SIZE32 in
+     ``.amdhsa_wavefront_size32``                             Target              GFX10-GFX12  Controls ENABLE_WAVEFRONT_SIZE32 in
                                                               Feature                          :ref:`amdgpu-amdhsa-kernel-descriptor-v3-table`.
                                                               Specific
                                                               (wavefrontsize64)
-     ``.amdhsa_uses_dynamic_stack``                           0                   GFX6-GFX11   Controls USES_DYNAMIC_STACK in
+     ``.amdhsa_uses_dynamic_stack``                           0                   GFX6-GFX12   Controls USES_DYNAMIC_STACK in
                                                                                                :ref:`amdgpu-amdhsa-kernel-descriptor-v3-table`.
      ``.amdhsa_system_sgpr_private_segment_wavefront_offset`` 0                   GFX6-GFX10   Controls ENABLE_PRIVATE_SEGMENT in
-                                                                                  (except      :ref:`amdgpu-amdhsa-compute_pgm_rsrc2-gfx6-gfx11-table`.
+                                                                                  (except      :ref:`amdgpu-amdhsa-compute_pgm_rsrc2-gfx6-gfx12-table`.
                                                                                   GFX940)
      ``.amdhsa_enable_private_segment``                       0                   GFX940,      Controls ENABLE_PRIVATE_SEGMENT in
-                                                                                  GFX11        :ref:`amdgpu-amdhsa-compute_pgm_rsrc2-gfx6-gfx11-table`.
-     ``.amdhsa_system_sgpr_workgroup_id_x``                   1                   GFX6-GFX11   Controls ENABLE_SGPR_WORKGROUP_ID_X in
-                                                                                               :ref:`amdgpu-amdhsa-compute_pgm_rsrc2-gfx6-gfx11-table`.
-     ``.amdhsa_system_sgpr_workgroup_id_y``                   0                   GFX6-GFX11   Controls ENABLE_SGPR_WORKGROUP_ID_Y in
-                                                                                               :ref:`amdgpu-amdhsa-compute_pgm_rsrc2-gfx6-gfx11-table`.
-     ``.amdhsa_system_sgpr_workgroup_id_z``                   0                   GFX6-GFX11   Controls ENABLE_SGPR_WORKGROUP_ID_Z in
-                                                                                               :ref:`amdgpu-amdhsa-compute_pgm_rsrc2-gfx6-gfx11-table`.
-     ``.amdhsa_system_sgpr_workgroup_info``                   0                   GFX6-GFX11   Controls ENABLE_SGPR_WORKGROUP_INFO in
-                                                                                               :ref:`amdgpu-amdhsa-compute_pgm_rsrc2-gfx6-gfx11-table`.
-     ``.amdhsa_system_vgpr_workitem_id``                      0                   GFX6-GFX11   Controls ENABLE_VGPR_WORKITEM_ID in
-                                                                                               :ref:`amdgpu-amdhsa-compute_pgm_rsrc2-gfx6-gfx11-table`.
+                                                                                  GFX11-GFX12  :ref:`amdgpu-amdhsa-compute_pgm_rsrc2-gfx6-gfx12-table`.
+     ``.amdhsa_system_sgpr_workgroup_id_x``                   1                   GFX6-GFX12   Controls ENABLE_SGPR_WORKGROUP_ID_X in
+                                                                                               :ref:`amdgpu-amdhsa-compute_pgm_rsrc2-gfx6-gfx12-table`.
+     ``.amdhsa_system_sgpr_workgroup_id_y``                   0                   GFX6-GFX12   Controls ENABLE_SGPR_WORKGROUP_ID_Y in
+                                                                                               :ref:`amdgpu-amdhsa-compute_pgm_rsrc2-gfx6-gfx12-table`.
+     ``.amdhsa_system_sgpr_workgroup_id_z``                   0                   GFX6-GFX12   Controls ENABLE_SGPR_WORKGROUP_ID_Z in
+                                                                                               :ref:`amdgpu-amdhsa-compute_pgm_rsrc2-gfx6-gfx12-table`.
+     ``.amdhsa_system_sgpr_workgroup_info``                   0                   GFX6-GFX12   Controls ENABLE_SGPR_WORKGROUP_INFO in
+                                                                                               :ref:`amdgpu-amdhsa-compute_pgm_rsrc2-gfx6-gfx12-table`.
+     ``.amdhsa_system_vgpr_workitem_id``                      0                   GFX6-GFX12   Controls ENABLE_VGPR_WORKITEM_ID in
+                                                                                               :ref:`amdgpu-amdhsa-compute_pgm_rsrc2-gfx6-gfx12-table`.
                                                                                                Possible values are defined in
                                                                                                :ref:`amdgpu-amdhsa-system-vgpr-work-item-id-enumeration-values-table`.
-     ``.amdhsa_next_free_vgpr``                               Required            GFX6-GFX11   Maximum VGPR number explicitly referenced, plus one.
+     ``.amdhsa_next_free_vgpr``                               Required            GFX6-GFX12   Maximum VGPR number explicitly referenced, plus one.
                                                                                                Used to calculate GRANULATED_WORKITEM_VGPR_COUNT in
-                                                                                               :ref:`amdgpu-amdhsa-compute_pgm_rsrc1-gfx6-gfx11-table`.
-     ``.amdhsa_next_free_sgpr``                               Required            GFX6-GFX11   Maximum SGPR number explicitly referenced, plus one.
+                                                                                               :ref:`amdgpu-amdhsa-compute_pgm_rsrc1-gfx6-gfx12-table`.
+     ``.amdhsa_next_free_sgpr``                               Required            GFX6-GFX12   Maximum SGPR number explicitly referenced, plus one.
                                                                                                Used to calculate GRANULATED_WAVEFRONT_SGPR_COUNT in
-                                                                                               :ref:`amdgpu-amdhsa-compute_pgm_rsrc1-gfx6-gfx11-table`.
+                                                                                               :ref:`amdgpu-amdhsa-compute_pgm_rsrc1-gfx6-gfx12-table`.
      ``.amdhsa_accum_offset``                                 Required            GFX90A,      Offset of a first AccVGPR in the unified register file.
                                                                                   GFX940       Used to calculate ACCUM_OFFSET in
                                                                                                :ref:`amdgpu-amdhsa-compute_pgm_rsrc3-gfx90a-table`.
-     ``.amdhsa_reserve_vcc``                                  1                   GFX6-GFX11   Whether the kernel may use the special VCC SGPR.
+     ``.amdhsa_reserve_vcc``                                  1                   GFX6-GFX12   Whether the kernel may use the special VCC SGPR.
                                                                                                Used to calculate GRANULATED_WAVEFRONT_SGPR_COUNT in
-                                                                                               :ref:`amdgpu-amdhsa-compute_pgm_rsrc1-gfx6-gfx11-table`.
+                                                                                               :ref:`amdgpu-amdhsa-compute_pgm_rsrc1-gfx6-gfx12-table`.
      ``.amdhsa_reserve_flat_scratch``                         1                   GFX7-GFX10   Whether the kernel may use flat instructions to access
                                                                                   (except      scratch memory. Used to calculate
                                                                                   GFX940)      GRANULATED_WAVEFRONT_SGPR_COUNT in
-                                                                                               :ref:`amdgpu-amdhsa-compute_pgm_rsrc1-gfx6-gfx11-table`.
+                                                                                               :ref:`amdgpu-amdhsa-compute_pgm_rsrc1-gfx6-gfx12-table`.
      ``.amdhsa_reserve_xnack_mask``                           Target              GFX8-GFX10   Whether the kernel may trigger XNACK replay.
                                                               Feature                          Used to calculate GRANULATED_WAVEFRONT_SGPR_COUNT in
-                                                              Specific                         :ref:`amdgpu-amdhsa-compute_pgm_rsrc1-gfx6-gfx11-table`.
+                                                              Specific                         :ref:`amdgpu-amdhsa-compute_pgm_rsrc1-gfx6-gfx12-table`.
                                                               (xnack)
-     ``.amdhsa_float_round_mode_32``                          0                   GFX6-GFX11   Controls FLOAT_ROUND_MODE_32 in
-                                                                                               :ref:`amdgpu-amdhsa-compute_pgm_rsrc1-gfx6-gfx11-table`.
+     ``.amdhsa_float_round_mode_32``                          0                   GFX6-GFX12   Controls FLOAT_ROUND_MODE_32 in
+                                                                                               :ref:`amdgpu-amdhsa-compute_pgm_rsrc1-gfx6-gfx12-table`.
                                                                                                Possible values are defined in
                                                                                                :ref:`amdgpu-amdhsa-floating-point-rounding-mode-enumeration-values-table`.
-     ``.amdhsa_float_round_mode_16_64``                       0                   GFX6-GFX11   Controls FLOAT_ROUND_MODE_16_64 in
-                                                                                               :ref:`amdgpu-amdhsa-compute_pgm_rsrc1-gfx6-gfx11-table`.
+     ``.amdhsa_float_round_mode_16_64``                       0                   GFX6-GFX12   Controls FLOAT_ROUND_MODE_16_64 in
+                                                                                               :ref:`amdgpu-amdhsa-compute_pgm_rsrc1-gfx6-gfx12-table`.
                                                                                                Possible values are defined in
                                                                                                :ref:`amdgpu-amdhsa-floating-point-rounding-mode-enumeration-values-table`.
-     ``.amdhsa_float_denorm_mode_32``                         0                   GFX6-GFX11   Controls FLOAT_DENORM_MODE_32 in
-                                                                                               :ref:`amdgpu-amdhsa-compute_pgm_rsrc1-gfx6-gfx11-table`.
+     ``.amdhsa_float_denorm_mode_32``                         0                   GFX6-GFX12   Controls FLOAT_DENORM_MODE_32 in
+                                                                                               :ref:`amdgpu-amdhsa-compute_pgm_rsrc1-gfx6-gfx12-table`.
                                                                                                Possible values are defined in
                                                                                                :ref:`amdgpu-amdhsa-floating-point-denorm-mode-enumeration-values-table`.
-     ``.amdhsa_float_denorm_mode_16_64``                      3                   GFX6-GFX11   Controls FLOAT_DENORM_MODE_16_64 in
-                                                                                               :ref:`amdgpu-amdhsa-compute_pgm_rsrc1-gfx6-gfx11-table`.
+     ``.amdhsa_float_denorm_mode_16_64``                      3                   GFX6-GFX12   Controls FLOAT_DENORM_MODE_16_64 in
+                                                                                               :ref:`amdgpu-amdhsa-compute_pgm_rsrc1-gfx6-gfx12-table`.
                                                                                                Possible values are defined in
                                                                                                :ref:`amdgpu-amdhsa-floating-point-denorm-mode-enumeration-values-table`.
      ``.amdhsa_dx10_clamp``                                   1                   GFX6-GFX11   Controls ENABLE_DX10_CLAMP in
-                                                                                               :ref:`amdgpu-amdhsa-compute_pgm_rsrc1-gfx6-gfx11-table`.
+                                                                                               :ref:`amdgpu-amdhsa-compute_pgm_rsrc1-gfx6-gfx12-table`.
      ``.amdhsa_ieee_mode``                                    1                   GFX6-GFX11   Controls ENABLE_IEEE_MODE in
-                                                                                               :ref:`amdgpu-amdhsa-compute_pgm_rsrc1-gfx6-gfx11-table`.
-     ``.amdhsa_fp16_overflow``                                0                   GFX9-GFX11   Controls FP16_OVFL in
-                                                                                               :ref:`amdgpu-amdhsa-compute_pgm_rsrc1-gfx6-gfx11-table`.
+                                                                                               :ref:`amdgpu-amdhsa-compute_pgm_rsrc1-gfx6-gfx12-table`.
+     ``.amdhsa_round_robin_scheduling``                       0                   GFX12        Controls ENABLE_WG_RR_EN in
+                                                                                               :ref:`amdgpu-amdhsa-compute_pgm_rsrc1-gfx6-gfx12-table`.
+     ``.amdhsa_fp16_overflow``                                0                   GFX9-GFX12   Controls FP16_OVFL in
+                                                                                               :ref:`amdgpu-amdhsa-compute_pgm_rsrc1-gfx6-gfx12-table`.
      ``.amdhsa_tg_split``                                     Target              GFX90A,      Controls TG_SPLIT in
                                                               Feature             GFX940,      :ref:`amdgpu-amdhsa-compute_pgm_rsrc3-gfx90a-table`.
-                                                              Specific            GFX11
+                                                              Specific            GFX11-GFX12
                                                               (tgsplit)
-     ``.amdhsa_workgroup_processor_mode``                     Target              GFX10-GFX11  Controls ENABLE_WGP_MODE in
+     ``.amdhsa_workgroup_processor_mode``                     Target              GFX10-GFX12  Controls ENABLE_WGP_MODE in
                                                               Feature                          :ref:`amdgpu-amdhsa-kernel-descriptor-v3-table`.
                                                               Specific
                                                               (cumode)
-     ``.amdhsa_memory_ordered``                               1                   GFX10-GFX11  Controls MEM_ORDERED in
-                                                                                               :ref:`amdgpu-amdhsa-compute_pgm_rsrc1-gfx6-gfx11-table`.
-     ``.amdhsa_forward_progress``                             0                   GFX10-GFX11  Controls FWD_PROGRESS in
-                                                                                               :ref:`amdgpu-amdhsa-compute_pgm_rsrc1-gfx6-gfx11-table`.
+     ``.amdhsa_memory_ordered``                               1                   GFX10-GFX12  Controls MEM_ORDERED in
+                                                                                               :ref:`amdgpu-amdhsa-compute_pgm_rsrc1-gfx6-gfx12-table`.
+     ``.amdhsa_forward_progress``                             0                   GFX10-GFX12  Controls FWD_PROGRESS in
+                                                                                               :ref:`amdgpu-amdhsa-compute_pgm_rsrc1-gfx6-gfx12-table`.
      ``.amdhsa_shared_vgpr_count``                            0                   GFX10-GFX11  Controls SHARED_VGPR_COUNT in
-                                                                                               :ref:`amdgpu-amdhsa-compute_pgm_rsrc3-gfx10-gfx11-table`.
-     ``.amdhsa_exception_fp_ieee_invalid_op``                 0                   GFX6-GFX11   Controls ENABLE_EXCEPTION_IEEE_754_FP_INVALID_OPERATION in
-                                                                                               :ref:`amdgpu-amdhsa-compute_pgm_rsrc2-gfx6-gfx11-table`.
-     ``.amdhsa_exception_fp_denorm_src``                      0                   GFX6-GFX11   Controls ENABLE_EXCEPTION_FP_DENORMAL_SOURCE in
-                                                                                               :ref:`amdgpu-amdhsa-compute_pgm_rsrc2-gfx6-gfx11-table`.
-     ``.amdhsa_exception_fp_ieee_div_zero``                   0                   GFX6-GFX11   Controls ENABLE_EXCEPTION_IEEE_754_FP_DIVISION_BY_ZERO in
-                                                                                               :ref:`amdgpu-amdhsa-compute_pgm_rsrc2-gfx6-gfx11-table`.
-     ``.amdhsa_exception_fp_ieee_overflow``                   0                   GFX6-GFX11   Controls ENABLE_EXCEPTION_IEEE_754_FP_OVERFLOW in
-                                                                                               :ref:`amdgpu-amdhsa-compute_pgm_rsrc2-gfx6-gfx11-table`.
-     ``.amdhsa_exception_fp_ieee_underflow``                  0                   GFX6-GFX11   Controls ENABLE_EXCEPTION_IEEE_754_FP_UNDERFLOW in
-                                                                                               :ref:`amdgpu-amdhsa-compute_pgm_rsrc2-gfx6-gfx11-table`.
-     ``.amdhsa_exception_fp_ieee_inexact``                    0                   GFX6-GFX11   Controls ENABLE_EXCEPTION_IEEE_754_FP_INEXACT in
-                                                                                               :ref:`amdgpu-amdhsa-compute_pgm_rsrc2-gfx6-gfx11-table`.
-     ``.amdhsa_exception_int_div_zero``                       0                   GFX6-GFX11   Controls ENABLE_EXCEPTION_INT_DIVIDE_BY_ZERO in
-                                                                                               :ref:`amdgpu-amdhsa-compute_pgm_rsrc2-gfx6-gfx11-table`.
+                                                                                               :ref:`amdgpu-amdhsa-compute_pgm_rsrc3-gfx10-gfx12-table`.
+     ``.amdhsa_exception_fp_ieee_invalid_op``                 0                   GFX6-GFX12   Controls ENABLE_EXCEPTION_IEEE_754_FP_INVALID_OPERATION in
+                                                                                               :ref:`amdgpu-amdhsa-compute_pgm_rsrc2-gfx6-gfx12-table`.
+     ``.amdhsa_exception_fp_denorm_src``                      0                   GFX6-GFX12   Controls ENABLE_EXCEPTION_FP_DENORMAL_SOURCE in
+                                                                                               :ref:`amdgpu-amdhsa-compute_pgm_rsrc2-gfx6-gfx12-table`.
+     ``.amdhsa_exception_fp_ieee_div_zero``                   0                   GFX6-GFX12   Controls ENABLE_EXCEPTION_IEEE_754_FP_DIVISION_BY_ZERO in
+                                                                                               :ref:`amdgpu-amdhsa-compute_pgm_rsrc2-gfx6-gfx12-table`.
+     ``.amdhsa_exception_fp_ieee_overflow``                   0                   GFX6-GFX12   Controls ENABLE_EXCEPTION_IEEE_754_FP_OVERFLOW in
+                                                                                               :ref:`amdgpu-amdhsa-compute_pgm_rsrc2-gfx6-gfx12-table`.
+     ``.amdhsa_exception_fp_ieee_underflow``                  0                   GFX6-GFX12   Controls ENABLE_EXCEPTION_IEEE_754_FP_UNDERFLOW in
+                                                                                               :ref:`amdgpu-amdhsa-compute_pgm_rsrc2-gfx6-gfx12-table`.
+     ``.amdhsa_exception_fp_ieee_inexact``                    0                   GFX6-GFX12   Controls ENABLE_EXCEPTION_IEEE_754_FP_INEXACT in
+                                                                                               :ref:`amdgpu-amdhsa-compute_pgm_rsrc2-gfx6-gfx12-table`.
+     ``.amdhsa_exception_int_div_zero``                       0                   GFX6-GFX12   Controls ENABLE_EXCEPTION_INT_DIVIDE_BY_ZERO in
+                                                                                               :ref:`amdgpu-amdhsa-compute_pgm_rsrc2-gfx6-gfx12-table`.
      ``.amdhsa_user_sgpr_kernarg_preload_length``             0                   GFX90A,      Controls KERNARG_PRELOAD_SPEC_LENGTH in
                                                                                   GFX940       :ref:`amdgpu-amdhsa-kernel-descriptor-v3-table`.
      ``.amdhsa_user_sgpr_kernarg_preload_offset``             0                   GFX90A,      Controls KERNARG_PRELOAD_SPEC_OFFSET in
diff --git a/llvm/docs/CommandGuide/llvm-cov.rst b/llvm/docs/CommandGuide/llvm-cov.rst
index be0c6b3b7e0463a4c3f93d888d2d0c6058514eee..968f3c452f5584337531b20c6385174699820e45 100644
--- a/llvm/docs/CommandGuide/llvm-cov.rst
+++ b/llvm/docs/CommandGuide/llvm-cov.rst
@@ -222,6 +222,11 @@ OPTIONS
  Show coverage for branch conditions in terms of either count or percentage.
  The supported views are: "count", "percent".
 
+.. option:: -show-mcdc
+
+ Show modified condition/decision coverage (MC/DC) for each applicable boolean
+ expression.
+
 .. option:: -show-line-counts
 
  Show the execution counts for each line. Defaults to true, unless another
@@ -426,6 +431,10 @@ OPTIONS
 
  Show statistics for all branch conditions. Defaults to true.
 
+.. option:: -show-mcdc-summary
+
+ Show MC/DC statistics. Defaults to false.
+
 .. option:: -show-functions
 
  Show coverage summaries for each function. Defaults to false.
diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
index 17d0d5176b09f4ec95a3602a9727b79a4f55796d..8f0c45f674ead8ecb420268bc7051d5bc7fdcdaf 100644
--- a/llvm/docs/LangRef.rst
+++ b/llvm/docs/LangRef.rst
@@ -1577,6 +1577,20 @@ Currently, only the following parameter attributes are defined:
     ``readonly`` or a ``memory`` attribute that does not contain
     ``argmem: write``.
 
+``dead_on_unwind``
+    At a high level, this attribute indicates that the pointer argument is dead
+    if the call unwinds, in the sense that the caller will not depend on the
+    contents of the memory. Stores that would only be visible on the unwind
+    path can be elided.
+
+    More precisely, the behavior is as-if any memory written through the
+    pointer during the execution of the function is overwritten with a poison
+    value on unwind. This includes memory written by the implicit write implied
+    by the ``writable`` attribute. The caller is allowed to access the affected
+    memory, but all loads that are not preceded by a store will return poison.
+
+    This attribute cannot be applied to return values.
+
 .. _gc:
 
 Garbage Collector Strategy Names
diff --git a/llvm/docs/ProgrammersManual.rst b/llvm/docs/ProgrammersManual.rst
index 551a23c0570e966a1377ed466e945cea0b7807b9..491e6b1dd2498b124b883a7545b21cf9c58c3483 100644
--- a/llvm/docs/ProgrammersManual.rst
+++ b/llvm/docs/ProgrammersManual.rst
@@ -2747,24 +2747,6 @@ pointer from an iterator is very straight-forward.  Assuming that ``i`` is a
   Instruction* pinst = &*i; // Grab pointer to instruction reference
   const Instruction& inst = *j;
 
-However, the iterators you'll be working with in the LLVM framework are special:
-they will automatically convert to a ptr-to-instance type whenever they need to.
-Instead of dereferencing the iterator and then taking the address of the result,
-you can simply assign the iterator to the proper pointer type and you get the
-dereference and address-of operation as a result of the assignment (behind the
-scenes, this is a result of overloading casting mechanisms).  Thus the second
-line of the last example,
-
-.. code-block:: c++
-
-  Instruction *pinst = &*i;
-
-is semantically equivalent to
-
-.. code-block:: c++
-
-  Instruction *pinst = i;
-
 It's also possible to turn a class pointer into the corresponding iterator, and
 this is a constant time operation (very efficient).  The following code snippet
 illustrates use of the conversion constructors provided by LLVM iterators.  By
@@ -2779,18 +2761,6 @@ obtaining it via iteration over some structure:
     if (it != inst->getParent()->end()) errs() << *it << "\n";
   }
 
-Unfortunately, these implicit conversions come at a cost; they prevent these
-iterators from conforming to standard iterator conventions, and thus from being
-usable with standard algorithms and containers.  For example, they prevent the
-following code, where ``B`` is a ``BasicBlock``, from compiling:
-
-.. code-block:: c++
-
-  llvm::SmallVector<llvm::Instruction *, 16>(B->begin(), B->end());
-
-Because of this, these implicit conversions may be removed some day, and
-``operator*`` changed to return a pointer instead of a reference.
-
 .. _iterate_complex:
 
 Finding call sites: a slightly more complex example
diff --git a/llvm/docs/RISCVUsage.rst b/llvm/docs/RISCVUsage.rst
index 65dd0d83448ed1048ff3b3fb162d23d0e37ad698..842ebf45305952fa07b7d9d10c064232a85cec6f 100644
--- a/llvm/docs/RISCVUsage.rst
+++ b/llvm/docs/RISCVUsage.rst
@@ -197,7 +197,7 @@ The primary goal of experimental support is to assist in the process of ratifica
   LLVM implements assembler support for the `0.8.0 draft specification <https://github.com/riscv/riscv-bfloat16/releases/tag/20230629>`_.
 
 ``experimental-zicfilp``
-  LLVM implements the `0.2 draft specification <https://github.com/riscv/riscv-cfi/releases/tag/v0.2.0>`__.
+  LLVM implements the `0.4 draft specification <https://github.com/riscv/riscv-cfi/releases/tag/v0.4.0>`__.
 
 ``experimental-zicond``
   LLVM implements the `1.0-rc1 draft specification <https://github.com/riscv/riscv-zicond/releases/tag/v1.0-rc1>`__.
diff --git a/llvm/docs/TableGen/index.rst b/llvm/docs/TableGen/index.rst
index 00569274be3c4025ce31a14c73d458d23955714e..e916c152f5a43de92b2d6c5ba175575416dbe5ff 100644
--- a/llvm/docs/TableGen/index.rst
+++ b/llvm/docs/TableGen/index.rst
@@ -283,6 +283,13 @@ See :doc:`TableGen BackEnds <./BackEnds>` for a list of available
 backends, and see the :doc:`TableGen Backend Developer's Guide <./BackGuide>`
 for information on how to write and debug a new backend.
 
+Tools and Resources
+===================
+
+In addition to this documentation, a list of tools and resources for TableGen
+can be found in TableGen's
+`README <https://github.com/llvm/llvm-project/blob/main/llvm/utils/TableGen/README.md>`_.
+
 TableGen Deficiencies
 =====================
 
diff --git a/llvm/include/llvm/ADT/StringMap.h b/llvm/include/llvm/ADT/StringMap.h
index 32e611467ac58ec44b9102cf8ca50343007ef3e4..466f95254d102e98343290b211f317f749d7692b 100644
--- a/llvm/include/llvm/ADT/StringMap.h
+++ b/llvm/include/llvm/ADT/StringMap.h
@@ -17,7 +17,6 @@
 #include "llvm/ADT/StringMapEntry.h"
 #include "llvm/ADT/iterator.h"
 #include "llvm/Support/AllocatorBase.h"
-#include "llvm/Support/DJB.h"
 #include "llvm/Support/PointerLikeTypeTraits.h"
 #include <initializer_list>
 #include <iterator>
@@ -61,20 +60,12 @@ protected:
   /// specified bucket will be non-null.  Otherwise, it will be null.  In either
   /// case, the FullHashValue field of the bucket will be set to the hash value
   /// of the string.
-  unsigned LookupBucketFor(StringRef Key) {
-    return LookupBucketFor(Key, hash(Key));
-  }
-
-  /// Overload that explicitly takes precomputed hash(Key).
-  unsigned LookupBucketFor(StringRef Key, uint32_t FullHashValue);
+  unsigned LookupBucketFor(StringRef Key);
 
   /// FindKey - Look up the bucket that contains the specified key. If it exists
   /// in the map, return the bucket number of the key.  Otherwise return -1.
   /// This does not modify the map.
-  int FindKey(StringRef Key) const { return FindKey(Key, hash(Key)); }
-
-  /// Overload that explicitly takes precomputed hash(Key).
-  int FindKey(StringRef Key, uint32_t FullHashValue) const;
+  int FindKey(StringRef Key) const;
 
   /// RemoveKey - Remove the specified StringMapEntry from the table, but do not
   /// delete it.  This aborts if the value isn't in the table.
@@ -103,13 +94,6 @@ public:
   bool empty() const { return NumItems == 0; }
   unsigned size() const { return NumItems; }
 
-  /// Returns the hash value that will be used for the given string.
-  /// This allows precomputing the value and passing it explicitly
-  /// to some of the functions.
-  /// The implementation of this function is not guaranteed to be stable
-  /// and may change.
-  static uint32_t hash(StringRef Key);
-
   void swap(StringMapImpl &Other) {
     std::swap(TheTable, Other.TheTable);
     std::swap(NumBuckets, Other.NumBuckets);
@@ -231,19 +215,15 @@ public:
                       StringMapKeyIterator<ValueTy>(end()));
   }
 
-  iterator find(StringRef Key) { return find(Key, hash(Key)); }
-
-  iterator find(StringRef Key, uint32_t FullHashValue) {
-    int Bucket = FindKey(Key, FullHashValue);
+  iterator find(StringRef Key) {
+    int Bucket = FindKey(Key);
     if (Bucket == -1)
       return end();
     return iterator(TheTable + Bucket, true);
   }
 
-  const_iterator find(StringRef Key) const { return find(Key, hash(Key)); }
-
-  const_iterator find(StringRef Key, uint32_t FullHashValue) const {
-    int Bucket = FindKey(Key, FullHashValue);
+  const_iterator find(StringRef Key) const {
+    int Bucket = FindKey(Key);
     if (Bucket == -1)
       return end();
     return const_iterator(TheTable + Bucket, true);
@@ -325,13 +305,7 @@ public:
   /// if and only if the insertion takes place, and the iterator component of
   /// the pair points to the element with key equivalent to the key of the pair.
   std::pair<iterator, bool> insert(std::pair<StringRef, ValueTy> KV) {
-    return try_emplace_with_hash(KV.first, hash(KV.first),
-                                 std::move(KV.second));
-  }
-
-  std::pair<iterator, bool> insert(std::pair<StringRef, ValueTy> KV,
-                                   uint32_t FullHashValue) {
-    return try_emplace_with_hash(KV.first, FullHashValue, std::move(KV.second));
+    return try_emplace(KV.first, std::move(KV.second));
   }
 
   /// Inserts elements from range [first, last). If multiple elements in the
@@ -365,14 +339,7 @@ public:
   /// the pair points to the element with key equivalent to the key of the pair.
   template <typename... ArgsTy>
   std::pair<iterator, bool> try_emplace(StringRef Key, ArgsTy &&...Args) {
-    return try_emplace_with_hash(Key, hash(Key), std::forward<ArgsTy>(Args)...);
-  }
-
-  template <typename... ArgsTy>
-  std::pair<iterator, bool> try_emplace_with_hash(StringRef Key,
-                                                  uint32_t FullHashValue,
-                                                  ArgsTy &&...Args) {
-    unsigned BucketNo = LookupBucketFor(Key, FullHashValue);
+    unsigned BucketNo = LookupBucketFor(Key);
     StringMapEntryBase *&Bucket = TheTable[BucketNo];
     if (Bucket && Bucket != getTombstoneVal())
       return std::make_pair(iterator(TheTable + BucketNo, false),
diff --git a/llvm/include/llvm/Analysis/AliasSetTracker.h b/llvm/include/llvm/Analysis/AliasSetTracker.h
index e485e1ff2f4c9875fb4c4c6475b2711ea9e75e58..4a952ccae7a0a105e2ea9dd84b74aae99baa4109 100644
--- a/llvm/include/llvm/Analysis/AliasSetTracker.h
+++ b/llvm/include/llvm/Analysis/AliasSetTracker.h
@@ -330,7 +330,7 @@ public:
   /// These methods return true if inserting the instruction resulted in the
   /// addition of a new alias set (i.e., the pointer did not alias anything).
   ///
-  void add(Value *Ptr, LocationSize Size, const AAMDNodes &AAInfo); // Add a loc
+  void add(const MemoryLocation &Loc);
   void add(LoadInst *LI);
   void add(StoreInst *SI);
   void add(VAArgInst *VAAI);
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index fb6f3287e3d262b59fc32ad4b93f1b8a5ddcb51c..f5114fa40c70ad1e5f3f33717b437d94fa1be9da 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -1002,6 +1002,16 @@ public:
   /// more beneficial constant hoisting is).
   InstructionCost getIntImmCodeSizeCost(unsigned Opc, unsigned Idx,
                                         const APInt &Imm, Type *Ty) const;
+
+  /// It can be advantageous to detach complex constants from their uses to make
+  /// their generation cheaper. This hook allows targets to report when such
+  /// transformations might negatively effect the code generation of the
+  /// underlying operation. The motivating example is divides whereby hoisting
+  /// constants prevents the code generator's ability to transform them into
+  /// combinations of simpler operations.
+  bool preferToKeepConstantsAttached(const Instruction &Inst,
+                                     const Function &Fn) const;
+
   /// @}
 
   /// \name Vector Target Information
@@ -1873,6 +1883,8 @@ public:
   virtual InstructionCost getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx,
                                               const APInt &Imm, Type *Ty,
                                               TargetCostKind CostKind) = 0;
+  virtual bool preferToKeepConstantsAttached(const Instruction &Inst,
+                                             const Function &Fn) const = 0;
   virtual unsigned getNumberOfRegisters(unsigned ClassID) const = 0;
   virtual unsigned getRegisterClassForType(bool Vector,
                                            Type *Ty = nullptr) const = 0;
@@ -2430,6 +2442,10 @@ public:
                                       TargetCostKind CostKind) override {
     return Impl.getIntImmCostIntrin(IID, Idx, Imm, Ty, CostKind);
   }
+  bool preferToKeepConstantsAttached(const Instruction &Inst,
+                                     const Function &Fn) const override {
+    return Impl.preferToKeepConstantsAttached(Inst, Fn);
+  }
   unsigned getNumberOfRegisters(unsigned ClassID) const override {
     return Impl.getNumberOfRegisters(ClassID);
   }
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index 5b612fd813067aac4ae121a499184fe8dd7ef67b..1d8f523e9792ba08f3d37176f991a3e971b35e83 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -427,6 +427,11 @@ public:
     return TTI::TCC_Free;
   }
 
+  bool preferToKeepConstantsAttached(const Instruction &Inst,
+                                     const Function &Fn) const {
+    return false;
+  }
+
   unsigned getNumberOfRegisters(unsigned ClassID) const { return 8; }
 
   unsigned getRegisterClassForType(bool Vector, Type *Ty = nullptr) const {
@@ -709,6 +714,7 @@ public:
     case Intrinsic::coro_subfn_addr:
     case Intrinsic::threadlocal_address:
     case Intrinsic::experimental_widenable_condition:
+    case Intrinsic::ssa_copy:
       // These intrinsics don't actually represent code after lowering.
       return 0;
     }
diff --git a/llvm/include/llvm/BinaryFormat/ELF.h b/llvm/include/llvm/BinaryFormat/ELF.h
index 40c795410f95adac858e6ecd4cd30a1a9482dad7..da38f6ef064f95ad91222c2f4a4fc7f4a5abc447 100644
--- a/llvm/include/llvm/BinaryFormat/ELF.h
+++ b/llvm/include/llvm/BinaryFormat/ELF.h
@@ -1693,6 +1693,7 @@ enum : unsigned {
 enum : unsigned {
   GNU_PROPERTY_AARCH64_FEATURE_1_BTI = 1 << 0,
   GNU_PROPERTY_AARCH64_FEATURE_1_PAC = 1 << 1,
+  GNU_PROPERTY_AARCH64_FEATURE_1_GCS = 1 << 2,
 };
 
 // x86 processor feature bits.
diff --git a/llvm/include/llvm/BinaryFormat/Magic.h b/llvm/include/llvm/BinaryFormat/Magic.h
index a28710dcdfaf2c6810acd3589df9d24cdc284415..c635a269576587b90bd2c86b6c4cba1312b78e37 100644
--- a/llvm/include/llvm/BinaryFormat/Magic.h
+++ b/llvm/include/llvm/BinaryFormat/Magic.h
@@ -57,6 +57,7 @@ struct file_magic {
     dxcontainer_object,        ///< DirectX container file
     offload_bundle,            ///< Clang offload bundle file
     offload_bundle_compressed, ///< Compressed clang offload bundle file
+    spirv_object,              ///< A binary SPIR-V file
   };
 
   bool is_object() const { return V != unknown; }
diff --git a/llvm/include/llvm/Bitcode/LLVMBitCodes.h b/llvm/include/llvm/Bitcode/LLVMBitCodes.h
index 6549f5660cc3b5fa7f2e9dc3177467ecb0cdcecb..c6f0ddf29a6da88dbcd608ea0a2df1424b89ca12 100644
--- a/llvm/include/llvm/Bitcode/LLVMBitCodes.h
+++ b/llvm/include/llvm/Bitcode/LLVMBitCodes.h
@@ -723,6 +723,7 @@ enum AttributeKindCodes {
   ATTR_KIND_OPTIMIZE_FOR_DEBUGGING = 88,
   ATTR_KIND_WRITABLE = 89,
   ATTR_KIND_CORO_ONLY_DESTROY_WHEN_COMPLETE = 90,
+  ATTR_KIND_DEAD_ON_UNWIND = 91,
 };
 
 enum ComdatSelectionKindCodes {
diff --git a/llvm/include/llvm/CodeGen/BasicTTIImpl.h b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
index e05ce2890a08c8c0b39c70febe806734bb5ffa90..5e7bdcdf72a49f8403698b35666007eca21ba795 100644
--- a/llvm/include/llvm/CodeGen/BasicTTIImpl.h
+++ b/llvm/include/llvm/CodeGen/BasicTTIImpl.h
@@ -545,6 +545,25 @@ public:
     return TargetTransformInfo::TCC_Expensive;
   }
 
+  bool preferToKeepConstantsAttached(const Instruction &Inst,
+                                     const Function &Fn) const {
+    switch (Inst.getOpcode()) {
+    default:
+      break;
+    case Instruction::SDiv:
+    case Instruction::SRem:
+    case Instruction::UDiv:
+    case Instruction::URem: {
+      if (!isa<ConstantInt>(Inst.getOperand(1)))
+        return false;
+      EVT VT = getTLI()->getValueType(DL, Inst.getType());
+      return !getTLI()->isIntDivCheap(VT, Fn.getAttributes());
+    }
+    };
+
+    return false;
+  }
+
   unsigned getInliningThresholdMultiplier() const { return 1; }
   unsigned adjustInliningThreshold(const CallBase *CB) { return 0; }
   unsigned getCallerAllocaCost(const CallBase *CB, const AllocaInst *AI) const {
diff --git a/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h b/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h
index 92bfef2b0148b9f5263b9e53a819f2aed5fa81ba..3d8646291bb04cb28fd9b019c82cf8768a3080ef 100644
--- a/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h
+++ b/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h
@@ -25,8 +25,12 @@
 #include "llvm/Analysis/TypeBasedAliasAnalysis.h"
 #include "llvm/CodeGen/CallBrPrepare.h"
 #include "llvm/CodeGen/DwarfEHPrepare.h"
+#include "llvm/CodeGen/ExpandMemCmp.h"
 #include "llvm/CodeGen/ExpandReductions.h"
+#include "llvm/CodeGen/GCMetadata.h"
+#include "llvm/CodeGen/IndirectBrExpand.h"
 #include "llvm/CodeGen/InterleavedAccess.h"
+#include "llvm/CodeGen/InterleavedLoadCombine.h"
 #include "llvm/CodeGen/JMCInstrumenter.h"
 #include "llvm/CodeGen/MachinePassManager.h"
 #include "llvm/CodeGen/PreISelIntrinsicLowering.h"
@@ -48,6 +52,7 @@
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Target/CGPassBuilderOption.h"
 #include "llvm/Target/TargetMachine.h"
+#include "llvm/Transforms/CFGuard.h"
 #include "llvm/Transforms/Scalar/ConstantHoisting.h"
 #include "llvm/Transforms/Scalar/LoopPassManager.h"
 #include "llvm/Transforms/Scalar/LoopStrengthReduce.h"
@@ -624,7 +629,7 @@ void CodeGenPassBuilder<Derived>::addIRPasses(AddIRPass &addPass) const {
     // target lowering hook.
     if (!Opt.DisableMergeICmps)
       addPass(MergeICmpsPass());
-    addPass(ExpandMemCmpPass());
+    addPass(ExpandMemCmpPass(&TM));
   }
 
   // Run GC lowering passes for builtin collectors
diff --git a/llvm/include/llvm/CodeGen/ExpandMemCmp.h b/llvm/include/llvm/CodeGen/ExpandMemCmp.h
new file mode 100644
index 0000000000000000000000000000000000000000..94a877854f327a3b73e5f37f71e8d0b0dc38ca04
--- /dev/null
+++ b/llvm/include/llvm/CodeGen/ExpandMemCmp.h
@@ -0,0 +1,29 @@
+//===--- ExpandMemCmp.h - Expand memcmp() to load/stores --------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGEN_EXPANDMEMCMP_H
+#define LLVM_CODEGEN_EXPANDMEMCMP_H
+
+#include "llvm/IR/PassManager.h"
+
+namespace llvm {
+
+class TargetMachine;
+
+class ExpandMemCmpPass : public PassInfoMixin<ExpandMemCmpPass> {
+  const TargetMachine *TM;
+
+public:
+  explicit ExpandMemCmpPass(const TargetMachine *TM_) : TM(TM_) {}
+
+  PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM);
+};
+
+} // namespace llvm
+
+#endif // LLVM_CODEGEN_EXPANDMEMCMP_H
diff --git a/llvm/include/llvm/CodeGen/GCMetadata.h b/llvm/include/llvm/CodeGen/GCMetadata.h
index 334c5c23b8facc90b3d490a440b98ae427734bbe..9e4e8342ea29eddd4e971a1d4ff1f76450d7d26f 100644
--- a/llvm/include/llvm/CodeGen/GCMetadata.h
+++ b/llvm/include/llvm/CodeGen/GCMetadata.h
@@ -38,6 +38,7 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/IR/DebugLoc.h"
 #include "llvm/IR/GCStrategy.h"
+#include "llvm/IR/PassManager.h"
 #include "llvm/Pass.h"
 #include <algorithm>
 #include <cstddef>
@@ -101,6 +102,10 @@ public:
   GCFunctionInfo(const Function &F, GCStrategy &S);
   ~GCFunctionInfo();
 
+  /// Handle invalidation explicitly.
+  bool invalidate(Function &F, const PreservedAnalyses &PA,
+                  FunctionAnalysisManager::Invalidator &Inv);
+
   /// getFunction - Return the function to which this metadata applies.
   const Function &getFunction() const { return F; }
 
@@ -146,6 +151,41 @@ public:
   size_t live_size(const iterator &p) const { return roots_size(); }
 };
 
+struct GCStrategyMap {
+  StringMap<std::unique_ptr<GCStrategy>> StrategyMap;
+
+  GCStrategyMap() = default;
+  GCStrategyMap(GCStrategyMap &&) = default;
+
+  /// Handle invalidation explicitly.
+  bool invalidate(Module &M, const PreservedAnalyses &PA,
+                  ModuleAnalysisManager::Invalidator &Inv);
+};
+
+/// An analysis pass which caches information about the entire Module.
+/// Records a cache of the 'active' gc strategy objects for the current Module.
+class CollectorMetadataAnalysis
+    : public AnalysisInfoMixin<CollectorMetadataAnalysis> {
+  friend struct AnalysisInfoMixin<CollectorMetadataAnalysis>;
+  static AnalysisKey Key;
+
+public:
+  using Result = GCStrategyMap;
+  Result run(Module &M, ModuleAnalysisManager &MAM);
+};
+
+/// An analysis pass which caches information about the Function.
+/// Records the function level information used by GCRoots.
+/// This pass depends on `CollectorMetadataAnalysis`.
+class GCFunctionAnalysis : public AnalysisInfoMixin<GCFunctionAnalysis> {
+  friend struct AnalysisInfoMixin<GCFunctionAnalysis>;
+  static AnalysisKey Key;
+
+public:
+  using Result = GCFunctionInfo;
+  Result run(Function &F, FunctionAnalysisManager &FAM);
+};
+
 /// An analysis pass which caches information about the entire Module.
 /// Records both the function level information used by GCRoots and a
 /// cache of the 'active' gc strategy objects for the current Module.
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h b/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h
index f5d9f5f40881cb55818539cc4e0b72650a59a2fd..694d3d8004afff18c800cf3f6469694258f0e94f 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h
@@ -54,10 +54,30 @@ enum {
   GICXXCustomAction_Invalid = 0,
 };
 
+/// The MatchTable is encoded as an array of bytes.
+/// Thus, opcodes are expected to be <255.
+///
+/// Operands can be variable-sized, their size is always after their name
+/// in the docs, e.g. "Foo(4)" means that "Foo" takes 4 entries in the table,
+/// so 4 bytes. "Foo()"
+///
+/// As a general rule of thumb:
+///   - Instruction & Operand IDs are ULEB128
+///   - LLT IDs are 1 byte
+///   - Predicates and target opcodes, register and register class IDs are 2
+///     bytes.
+///   - Indexes into the table are 4 bytes.
+///   - Inline constants are 8 bytes
+///
+/// Design notes:
+///   - Inst/Op IDs have to be LEB128 because some targets generate
+///     extremely long patterns which need more than 255 temporaries.
+///     We could just use 2 bytes everytime, but then some targets like
+///     X86/AMDGPU that have no need for it will pay the price all the time.
 enum {
   /// Begin a try-block to attempt a match and jump to OnFail if it is
   /// unsuccessful.
-  /// - OnFail - The MatchTable entry at which to resume if the match fails.
+  /// - OnFail(4) - The MatchTable entry at which to resume if the match fails.
   ///
   /// FIXME: This ought to take an argument indicating the number of try-blocks
   ///        to exit on failure. It's usually one but the last match attempt of
@@ -68,100 +88,103 @@ enum {
   GIM_Try,
 
   /// Switch over the opcode on the specified instruction
-  /// - InsnID - Instruction ID
-  /// - LowerBound - numerically minimum opcode supported
-  /// - UpperBound - numerically maximum + 1 opcode supported
-  /// - Default - failure jump target
-  /// - JumpTable... - (UpperBound - LowerBound) (at least 2) jump targets
+  /// - InsnID(ULEB128) - Instruction ID
+  /// - LowerBound(2) - numerically minimum opcode supported
+  /// - UpperBound(2) - numerically maximum + 1 opcode supported
+  /// - Default(4) - failure jump target
+  /// - JumpTable(4)... - (UpperBound - LowerBound) (at least 2) jump targets
   GIM_SwitchOpcode,
 
   /// Switch over the LLT on the specified instruction operand
-  /// - InsnID - Instruction ID
-  /// - OpIdx - Operand index
-  /// - LowerBound - numerically minimum Type ID supported
-  /// - UpperBound - numerically maximum + 1 Type ID supported
-  /// - Default - failure jump target
-  /// - JumpTable... - (UpperBound - LowerBound) (at least 2) jump targets
+  /// - InsnID(ULEB128) - Instruction ID
+  /// - OpIdx(ULEB128) - Operand index
+  /// - LowerBound(2) - numerically minimum Type ID supported
+  /// - UpperBound(2) - numerically maximum + 1 Type ID supported
+  /// - Default(4) - failure jump target
+  /// - JumpTable(4)... - (UpperBound - LowerBound) (at least 2) jump targets
   GIM_SwitchType,
 
   /// Record the specified instruction.
   /// The IgnoreCopies variant ignores COPY instructions.
-  /// - NewInsnID - Instruction ID to define
-  /// - InsnID - Instruction ID
-  /// - OpIdx - Operand index
+  /// - NewInsnID(ULEB128) - Instruction ID to define
+  /// - InsnID(ULEB128) - Instruction ID
+  /// - OpIdx(ULEB128) - Operand index
   GIM_RecordInsn,
   GIM_RecordInsnIgnoreCopies,
 
   /// Check the feature bits
-  /// - Expected features
+  ///   Feature(2) - Expected features
   GIM_CheckFeatures,
 
   /// Check the opcode on the specified instruction
-  /// - InsnID - Instruction ID
-  /// - Expected opcode
+  /// - InsnID(ULEB128) - Instruction ID
+  /// - Opc(2) - Expected opcode
   GIM_CheckOpcode,
 
   /// Check the opcode on the specified instruction, checking 2 acceptable
   /// alternatives.
-  /// - InsnID - Instruction ID
-  /// - Expected opcode
-  /// - Alternative expected opcode
+  /// - InsnID(ULEB128) - Instruction ID
+  /// - Opc(2) - Expected opcode
+  /// - Opc(2) - Alternative expected opcode
   GIM_CheckOpcodeIsEither,
 
   /// Check the instruction has the right number of operands
-  /// - InsnID - Instruction ID
-  /// - Expected number of operands
+  /// - InsnID(ULEB128) - Instruction ID
+  /// - Ops(ULEB128) - Expected number of operands
   GIM_CheckNumOperands,
+
   /// Check an immediate predicate on the specified instruction
-  /// - InsnID - Instruction ID
-  /// - The predicate to test
+  /// - InsnID(ULEB128) - Instruction ID
+  /// - Pred(2) - The predicate to test
   GIM_CheckI64ImmPredicate,
   /// Check an immediate predicate on the specified instruction via an APInt.
-  /// - InsnID - Instruction ID
-  /// - The predicate to test
+  /// - InsnID(ULEB128) - Instruction ID
+  /// - Pred(2) - The predicate to test
   GIM_CheckAPIntImmPredicate,
   /// Check a floating point immediate predicate on the specified instruction.
-  /// - InsnID - Instruction ID
-  /// - The predicate to test
+  /// - InsnID(ULEB128) - Instruction ID
+  /// - Pred(2) - The predicate to test
   GIM_CheckAPFloatImmPredicate,
   /// Check an immediate predicate on the specified instruction
-  /// - InsnID - Instruction ID
-  /// - OpIdx - Operand index
-  /// - The predicate to test
+  /// - InsnID(ULEB128) - Instruction ID
+  /// - OpIdx(ULEB128) - Operand index
+  /// - Pred(2) - The predicate to test
   GIM_CheckImmOperandPredicate,
+
   /// Check a memory operation has the specified atomic ordering.
-  /// - InsnID - Instruction ID
-  /// - Ordering - The AtomicOrdering value
+  /// - InsnID(ULEB128) - Instruction ID
+  /// - Ordering(ULEB128) - The AtomicOrdering value
   GIM_CheckAtomicOrdering,
   GIM_CheckAtomicOrderingOrStrongerThan,
   GIM_CheckAtomicOrderingWeakerThan,
+
   /// Check the size of the memory access for the given machine memory operand.
-  /// - InsnID - Instruction ID
-  /// - MMOIdx - MMO index
-  /// - Size - The size in bytes of the memory access
+  /// - InsnID(ULEB128) - Instruction ID
+  /// - MMOIdx(ULEB128) - MMO index
+  /// - Size(4) - The size in bytes of the memory access
   GIM_CheckMemorySizeEqualTo,
 
   /// Check the address space of the memory access for the given machine memory
   /// operand.
-  /// - InsnID - Instruction ID
-  /// - MMOIdx - MMO index
-  /// - NumAddrSpace - Number of valid address spaces
-  /// - AddrSpaceN - An allowed space of the memory access
+  /// - InsnID(ULEB128) - Instruction ID
+  /// - MMOIdx(ULEB128) - MMO index
+  /// - NumAddrSpace(ULEB128) - Number of valid address spaces
+  /// - AddrSpaceN(ULEB128) - An allowed space of the memory access
   /// - AddrSpaceN+1 ...
   GIM_CheckMemoryAddressSpace,
 
   /// Check the minimum alignment of the memory access for the given machine
   /// memory operand.
-  /// - InsnID - Instruction ID
-  /// - MMOIdx - MMO index
-  /// - MinAlign - Minimum acceptable alignment
+  /// - InsnID(ULEB128) - Instruction ID
+  /// - MMOIdx(ULEB128) - MMO index
+  /// - MinAlign(ULEB128) - Minimum acceptable alignment
   GIM_CheckMemoryAlignment,
 
   /// Check the size of the memory access for the given machine memory operand
   /// against the size of an operand.
-  /// - InsnID - Instruction ID
-  /// - MMOIdx - MMO index
-  /// - OpIdx - The operand index to compare the MMO against
+  /// - InsnID(ULEB128) - Instruction ID
+  /// - MMOIdx(ULEB128) - MMO index
+  /// - OpIdx(ULEB128) - The operand index to compare the MMO against
   GIM_CheckMemorySizeEqualToLLT,
   GIM_CheckMemorySizeLessThanLLT,
   GIM_CheckMemorySizeGreaterThanLLT,
@@ -170,106 +193,117 @@ enum {
   /// constant. This is valid for both G_BUILD_VECTOR as well as
   /// G_BUILD_VECTOR_TRUNC. For AllOnes refers to individual bits, so a -1
   /// element.
-  /// - InsnID - Instruction ID
+  /// - InsnID(ULEB128) - Instruction ID
   GIM_CheckIsBuildVectorAllOnes,
   GIM_CheckIsBuildVectorAllZeros,
 
   /// Check a trivial predicate which takes no arguments.
   /// This can be used by executors to implement custom flags that don't fit in
   /// target features.
+  /// - Pred(2) - Predicate ID to check.
   GIM_CheckSimplePredicate,
 
   /// Check a generic C++ instruction predicate
-  /// - InsnID - Instruction ID
-  /// - PredicateID - The ID of the predicate function to call
+  /// - InsnID(ULEB128) - Instruction ID
+  /// - PredicateID(2) - The ID of the predicate function to call
   GIM_CheckCxxInsnPredicate,
 
   /// Check if there's no use of the first result.
-  /// - InsnID - Instruction ID
+  /// - InsnID(ULEB128) - Instruction ID
   GIM_CheckHasNoUse,
 
   /// Check the type for the specified operand
-  /// - InsnID - Instruction ID
-  /// - OpIdx - Operand index
-  /// - Expected type
+  /// - InsnID(ULEB128) - Instruction ID
+  /// - OpIdx(ULEB128) - Operand index
+  /// - Ty(1) - Expected type
   GIM_CheckType,
+
   /// Check the type of a pointer to any address space.
-  /// - InsnID - Instruction ID
-  /// - OpIdx - Operand index
-  /// - SizeInBits - The size of the pointer value in bits.
+  /// - InsnID(ULEB128) - Instruction ID
+  /// - OpIdx(ULEB128) - Operand index
+  /// - SizeInBits(ULEB128) - The size of the pointer value in bits.
   GIM_CheckPointerToAny,
+
   /// Check the register bank for the specified operand
-  /// - InsnID - Instruction ID
-  /// - OpIdx - Operand index
-  /// - Expected register bank (specified as a register class)
+  /// - InsnID(ULEB128) - Instruction ID
+  /// - OpIdx(ULEB128) - Operand index
+  /// - RC(2) - Expected register bank (specified as a register class)
   GIM_CheckRegBankForClass,
 
   /// Check the operand matches a complex predicate
-  /// - InsnID - Instruction ID
-  /// - OpIdx - Operand index
-  /// - RendererID - The renderer to hold the result
-  /// - Complex predicate ID
+  /// - InsnID(ULEB128) - Instruction ID
+  /// - OpIdx(ULEB128) - Operand index
+  /// - RendererID(2) - The renderer to hold the result
+  /// - Pred(2) - Complex predicate ID
   GIM_CheckComplexPattern,
 
   /// Check the operand is a specific integer
-  /// - InsnID - Instruction ID
-  /// - OpIdx - Operand index
-  /// - Expected integer
+  /// - InsnID(ULEB128) - Instruction ID
+  /// - OpIdx(ULEB128) - Operand index
+  /// - Val(8) Expected integer
   GIM_CheckConstantInt,
+
+  /// Check the operand is a specific 8-bit signed integer
+  /// - InsnID(ULEB128) - Instruction ID
+  /// - OpIdx(ULEB128) - Operand index
+  /// - Val(1) Expected integer
+  GIM_CheckConstantInt8,
+
   /// Check the operand is a specific literal integer (i.e. MO.isImm() or
   /// MO.isCImm() is true).
-  /// - InsnID - Instruction ID
-  /// - OpIdx - Operand index
-  /// - Expected integer
+  /// - InsnID(ULEB128) - Instruction ID
+  /// - OpIdx(ULEB128) - Operand index
+  /// - Val(8) - Expected integer
   GIM_CheckLiteralInt,
+
   /// Check the operand is a specific intrinsic ID
-  /// - InsnID - Instruction ID
-  /// - OpIdx - Operand index
-  /// - Expected Intrinsic ID
+  /// - InsnID(ULEB128) - Instruction ID
+  /// - OpIdx(ULEB128) - Operand index
+  /// - IID(2) - Expected Intrinsic ID
   GIM_CheckIntrinsicID,
 
   /// Check the operand is a specific predicate
-  /// - InsnID - Instruction ID
-  /// - OpIdx - Operand index
-  /// - Expected predicate
+  /// - InsnID(ULEB128) - Instruction ID
+  /// - OpIdx(ULEB128) - Operand index
+  /// - Pred(2) - Expected predicate
   GIM_CheckCmpPredicate,
 
   /// Check the specified operand is an MBB
-  /// - InsnID - Instruction ID
-  /// - OpIdx - Operand index
+  /// - InsnID(ULEB128) - Instruction ID
+  /// - OpIdx(ULEB128) - Operand index
   GIM_CheckIsMBB,
 
   /// Check the specified operand is an Imm
-  /// - InsnID - Instruction ID
-  /// - OpIdx - Operand index
+  /// - InsnID(ULEB128) - Instruction ID
+  /// - OpIdx(ULEB128) - Operand index
   GIM_CheckIsImm,
 
   /// Check if the specified operand is safe to fold into the current
   /// instruction.
-  /// - InsnID - Instruction ID
+  /// - InsnID(ULEB128) - Instruction ID
   GIM_CheckIsSafeToFold,
 
   /// Check the specified operands are identical.
   /// The IgnoreCopies variant looks through COPY instructions before
   /// comparing the operands.
-  /// - InsnID - Instruction ID
-  /// - OpIdx - Operand index
-  /// - OtherInsnID - Other instruction ID
-  /// - OtherOpIdx - Other operand index
+  /// - InsnID(ULEB128) - Instruction ID
+  /// - OpIdx(ULEB128) - Operand index
+  /// - OtherInsnID(ULEB128) - Other instruction ID
+  /// - OtherOpIdx(ULEB128) - Other operand index
   GIM_CheckIsSameOperand,
   GIM_CheckIsSameOperandIgnoreCopies,
 
   /// Check we can replace all uses of a register with another.
-  /// - OldInsnID
-  /// - OldOpIdx
-  /// - NewInsnID
-  /// - NewOpIdx
+  /// - OldInsnID(ULEB128)
+  /// - OldOpIdx(ULEB128)
+  /// - NewInsnID(ULEB128)
+  /// - NewOpIdx(ULEB128)
   GIM_CheckCanReplaceReg,
 
   /// Check that a matched instruction has, or doesn't have a MIFlag.
   ///
-  /// - InsnID  - Instruction to check.
-  /// - Flag(s) - (can be one or more flags OR'd together)
+  /// - InsnID(ULEB128) - Instruction to check.
+  /// - Flags(4) - (can be one or more flags OR'd together)
   GIM_MIFlags,
   GIM_MIFlagsNot,
 
@@ -277,15 +311,15 @@ enum {
   /// named operands that will be recorded in RecordedOperands. Names of these
   /// operands are referenced in predicate argument list. Emitter determines
   /// StoreIdx(corresponds to the order in which names appear in argument list).
-  /// - InsnID - Instruction ID
-  /// - OpIdx - Operand index
-  /// - StoreIdx - Store location in RecordedOperands.
+  /// - InsnID(ULEB128) - Instruction ID
+  /// - OpIdx(ULEB128) - Operand index
+  /// - StoreIdx(ULEB128) - Store location in RecordedOperands.
   GIM_RecordNamedOperand,
 
   /// Records an operand's register type into the set of temporary types.
-  /// - InsnID - Instruction ID
-  /// - OpIdx - Operand index
-  /// - TempTypeIdx - Temp Type Index, always negative.
+  /// - InsnID(ULEB128) - Instruction ID
+  /// - OpIdx(ULEB128) - Operand index
+  /// - TempTypeIdx(1) - Temp Type Index, always negative.
   GIM_RecordRegType,
 
   /// Fail the current try-block, or completely fail to match if there is no
@@ -295,121 +329,133 @@ enum {
   //=== Renderers ===
 
   /// Mutate an instruction
-  /// - NewInsnID - Instruction ID to define
-  /// - OldInsnID - Instruction ID to mutate
-  /// - NewOpcode - The new opcode to use
+  /// - NewInsnID(ULEB128) - Instruction ID to define
+  /// - OldInsnID(ULEB128) - Instruction ID to mutate
+  /// - NewOpcode(2) - The new opcode to use
   GIR_MutateOpcode,
 
   /// Build a new instruction
-  /// - InsnID - Instruction ID to define
-  /// - Opcode - The new opcode to use
+  /// - InsnID(ULEB128) - Instruction ID to define
+  /// - Opcode(2) - The new opcode to use
   GIR_BuildMI,
 
   /// Builds a constant and stores its result in a TempReg.
-  /// - TempRegID - Temp Register to define.
-  /// - Imm - The immediate to add
+  /// - TempRegID(ULEB128) - Temp Register to define.
+  /// - Imm(8) - The immediate to add
   GIR_BuildConstant,
 
   /// Copy an operand to the specified instruction
-  /// - NewInsnID - Instruction ID to modify
-  /// - OldInsnID - Instruction ID to copy from
-  /// - OpIdx - The operand to copy
+  /// - NewInsnID(ULEB128) - Instruction ID to modify
+  /// - OldInsnID(ULEB128) - Instruction ID to copy from
+  /// - OpIdx(ULEB128) - The operand to copy
   GIR_Copy,
 
   /// Copy an operand to the specified instruction or add a zero register if the
   /// operand is a zero immediate.
-  /// - NewInsnID - Instruction ID to modify
-  /// - OldInsnID - Instruction ID to copy from
-  /// - OpIdx - The operand to copy
-  /// - ZeroReg - The zero register to use
+  /// - NewInsnID(ULEB128) - Instruction ID to modify
+  /// - OldInsnID(ULEB128) - Instruction ID to copy from
+  /// - OpIdx(ULEB128) - The operand to copy
+  /// - ZeroReg(2) - The zero register to use
   GIR_CopyOrAddZeroReg,
   /// Copy an operand to the specified instruction
-  /// - NewInsnID - Instruction ID to modify
-  /// - OldInsnID - Instruction ID to copy from
-  /// - OpIdx - The operand to copy
-  /// - SubRegIdx - The subregister to copy
+  /// - NewInsnID(ULEB128) - Instruction ID to modify
+  /// - OldInsnID(ULEB128) - Instruction ID to copy from
+  /// - OpIdx(ULEB128) - The operand to copy
+  /// - SubRegIdx(2) - The subregister to copy
   GIR_CopySubReg,
 
   /// Add an implicit register def to the specified instruction
-  /// - InsnID - Instruction ID to modify
-  /// - RegNum - The register to add
-  /// - Flags - Register Flags
+  /// - InsnID(ULEB128) - Instruction ID to modify
+  /// - RegNum(2) - The register to add
+  /// - Flags(2) - Register Flags
   GIR_AddImplicitDef,
   /// Add an implicit register use to the specified instruction
-  /// - InsnID - Instruction ID to modify
-  /// - RegNum - The register to add
+  /// - InsnID(ULEB128) - Instruction ID to modify
+  /// - RegNum(2) - The register to add
   GIR_AddImplicitUse,
   /// Add an register to the specified instruction
-  /// - InsnID - Instruction ID to modify
-  /// - RegNum - The register to add
+  /// - InsnID(ULEB128) - Instruction ID to modify
+  /// - RegNum(2) - The register to add
+  /// - Flags(2) - Register Flags
   GIR_AddRegister,
 
   /// Marks the implicit def of a register as dead.
-  /// - InsnID - Instruction ID to modify
-  /// - OpIdx - The implicit def operand index
+  /// - InsnID(ULEB128) - Instruction ID to modify
+  /// - OpIdx(ULEB128) - The implicit def operand index
   ///
   /// OpIdx starts at 0 for the first implicit def.
   GIR_SetImplicitDefDead,
 
   /// Set or unset a MIFlag on an instruction.
   ///
-  /// - InsnID  - Instruction to modify.
-  /// - Flag(s) - (can be one or more flags OR'd together)
+  /// - InsnID(ULEB128)  - Instruction to modify.
+  /// - Flags(4) - (can be one or more flags OR'd together)
   GIR_SetMIFlags,
   GIR_UnsetMIFlags,
 
   /// Copy the MIFlags of a matched instruction into an
   /// output instruction. The flags are OR'd together.
   ///
-  /// - InsnID     - Instruction to modify.
-  /// - OldInsnID  - Matched instruction to copy flags from.
+  /// - InsnID(ULEB128)     - Instruction to modify.
+  /// - OldInsnID(ULEB128)  - Matched instruction to copy flags from.
   GIR_CopyMIFlags,
 
   /// Add a temporary register to the specified instruction
-  /// - InsnID - Instruction ID to modify
-  /// - TempRegID - The temporary register ID to add
-  /// - TempRegFlags - The register flags to set
+  /// - InsnID(ULEB128) - Instruction ID to modify
+  /// - TempRegID(ULEB128) - The temporary register ID to add
+  /// - TempRegFlags(2) - The register flags to set
   GIR_AddTempRegister,
 
+  /// Add a temporary register to the specified instruction without
+  /// setting any flags.
+  /// - InsnID(ULEB128) - Instruction ID to modify
+  /// - TempRegID(ULEB128) - The temporary register ID to add
+  GIR_AddSimpleTempRegister,
+
   /// Add a temporary register to the specified instruction
-  /// - InsnID - Instruction ID to modify
-  /// - TempRegID - The temporary register ID to add
-  /// - TempRegFlags - The register flags to set
-  /// - SubRegIndex - The subregister index to set
+  /// - InsnID(ULEB128) - Instruction ID to modify
+  /// - TempRegID(ULEB128) - The temporary register ID to add
+  /// - TempRegFlags(2) - The register flags to set
+  /// - SubRegIndex(2) - The subregister index to set
   GIR_AddTempSubRegister,
 
   /// Add an immediate to the specified instruction
-  /// - InsnID - Instruction ID to modify
-  /// - Imm - The immediate to add
+  /// - InsnID(ULEB128) - Instruction ID to modify
+  /// - Imm(8) - The immediate to add
   GIR_AddImm,
 
+  /// Add signed 8 bit immediate to the specified instruction
+  /// - InsnID(ULEB128) - Instruction ID to modify
+  /// - Imm(1) - The immediate to add
+  GIR_AddImm8,
+
   /// Add an CImm to the specified instruction
-  /// - InsnID - Instruction ID to modify
-  /// - Ty - Type of the constant immediate.
-  /// - Imm - The immediate to add
+  /// - InsnID(ULEB128) - Instruction ID to modify
+  /// - Ty(1) - Type of the constant immediate.
+  /// - Imm(8) - The immediate to add
   GIR_AddCImm,
 
   /// Render complex operands to the specified instruction
-  /// - InsnID - Instruction ID to modify
-  /// - RendererID - The renderer to call
+  /// - InsnID(ULEB128) - Instruction ID to modify
+  /// - RendererID(2) - The renderer to call
   GIR_ComplexRenderer,
   /// Render sub-operands of complex operands to the specified instruction
-  /// - InsnID - Instruction ID to modify
-  /// - RendererID - The renderer to call
-  /// - RenderOpID - The suboperand to render.
+  /// - InsnID(ULEB128) - Instruction ID to modify
+  /// - RendererID(2) - The renderer to call
+  /// - RenderOpID(ULEB128) - The suboperand to render.
   GIR_ComplexSubOperandRenderer,
   /// Render subregisters of suboperands of complex operands to the
   /// specified instruction
-  /// - InsnID - Instruction ID to modify
-  /// - RendererID - The renderer to call
-  /// - RenderOpID - The suboperand to render
-  /// - SubRegIdx - The subregister to extract
+  /// - InsnID(ULEB128) - Instruction ID to modify
+  /// - RendererID(2) - The renderer to call
+  /// - RenderOpID(ULEB128) - The suboperand to render
+  /// - SubRegIdx(2) - The subregister to extract
   GIR_ComplexSubOperandSubRegRenderer,
 
   /// Render operands to the specified instruction using a custom function
-  /// - InsnID - Instruction ID to modify
-  /// - OldInsnID - Instruction ID to get the matched operand from
-  /// - RendererFnID - Custom renderer function to call
+  /// - InsnID(ULEB128) - Instruction ID to modify
+  /// - OldInsnID(ULEB128) - Instruction ID to get the matched operand from
+  /// - RendererFnID(2) - Custom renderer function to call
   GIR_CustomRenderer,
 
   /// Calls a C++ function to perform an action when a match is complete.
@@ -418,90 +464,85 @@ enum {
   /// This is less constrained than a custom renderer and can update
   /// instructions
   /// in the state.
-  /// - FnID - The function to call.
+  /// - FnID(2) - The function to call.
   /// TODO: Remove this at some point when combiners aren't reliant on it. It's
   /// a bit of a hack.
   GIR_CustomAction,
 
   /// Render operands to the specified instruction using a custom function,
   /// reading from a specific operand.
-  /// - InsnID - Instruction ID to modify
-  /// - OldInsnID - Instruction ID to get the matched operand from
-  /// - OpIdx - Operand index in OldInsnID the render function should read
+  /// - InsnID(ULEB128) - Instruction ID to modify
+  /// - OldInsnID(ULEB128) - Instruction ID to get the matched operand from
+  /// - OpIdx(ULEB128) - Operand index in OldInsnID the render function should
+  /// read
   /// from..
-  /// - RendererFnID - Custom renderer function to call
+  /// - RendererFnID(2) - Custom renderer function to call
   GIR_CustomOperandRenderer,
 
   /// Render a G_CONSTANT operator as a sign-extended immediate.
-  /// - NewInsnID - Instruction ID to modify
-  /// - OldInsnID - Instruction ID to copy from
+  /// - NewInsnID(ULEB128) - Instruction ID to modify
+  /// - OldInsnID(ULEB128) - Instruction ID to copy from
   /// The operand index is implicitly 1.
   GIR_CopyConstantAsSImm,
 
   /// Render a G_FCONSTANT operator as a sign-extended immediate.
-  /// - NewInsnID - Instruction ID to modify
-  /// - OldInsnID - Instruction ID to copy from
+  /// - NewInsnID(ULEB128) - Instruction ID to modify
+  /// - OldInsnID(ULEB128) - Instruction ID to copy from
   /// The operand index is implicitly 1.
   GIR_CopyFConstantAsFPImm,
 
   /// Constrain an instruction operand to a register class.
-  /// - InsnID - Instruction ID to modify
-  /// - OpIdx - Operand index
-  /// - RCEnum - Register class enumeration value
+  /// - InsnID(ULEB128) - Instruction ID to modify
+  /// - OpIdx(ULEB128) - Operand index
+  /// - RCEnum(2) - Register class enumeration value
   GIR_ConstrainOperandRC,
 
   /// Constrain an instructions operands according to the instruction
   /// description.
-  /// - InsnID - Instruction ID to modify
+  /// - InsnID(ULEB128) - Instruction ID to modify
   GIR_ConstrainSelectedInstOperands,
 
   /// Merge all memory operands into instruction.
-  /// - InsnID - Instruction ID to modify
-  /// - MergeInsnID... - One or more Instruction ID to merge into the result.
-  /// - GIU_MergeMemOperands_EndOfList - Terminates the list of instructions to
-  ///                                    merge.
+  /// - InsnID(ULEB128) - Instruction ID to modify
+  /// - NumInsnID(1) - Number of instruction IDs following this argument
+  /// - MergeInsnID(ULEB128)... - One or more Instruction ID to merge into the
+  /// result.
   GIR_MergeMemOperands,
 
   /// Erase from parent.
-  /// - InsnID - Instruction ID to erase
+  /// - InsnID(ULEB128) - Instruction ID to erase
   GIR_EraseFromParent,
 
   /// Create a new temporary register that's not constrained.
-  /// - TempRegID - The temporary register ID to initialize.
-  /// - Expected type
+  /// - TempRegID(ULEB128) - The temporary register ID to initialize.
+  /// - Ty(1) - Expected type
   GIR_MakeTempReg,
 
   /// Replaces all references to a register from an instruction
   /// with another register from another instruction.
-  /// - OldInsnID
-  /// - OldOpIdx
-  /// - NewInsnID
-  /// - NewOpIdx
+  /// - OldInsnID(ULEB128)
+  /// - OldOpIdx(ULEB128)
+  /// - NewInsnID(ULEB128)
+  /// - NewOpIdx(ULEB128)
   GIR_ReplaceReg,
 
   /// Replaces all references to a register with a temporary register.
-  /// - OldInsnID
-  /// - OldOpIdx
-  /// - TempRegIdx
+  /// - OldInsnID(ULEB128)
+  /// - OldOpIdx(ULEB128)
+  /// - TempRegIdx(ULEB128)
   GIR_ReplaceRegWithTempReg,
 
   /// A successful emission
   GIR_Done,
 
   /// Increment the rule coverage counter.
-  /// - RuleID - The ID of the rule that was covered.
+  /// - RuleID(4) - The ID of the rule that was covered.
   GIR_Coverage,
 
   /// Keeping track of the number of the GI opcodes. Must be the last entry.
   GIU_NumOpcodes,
 };
 
-enum {
-  /// Indicates the end of the variable-length MergeInsnID list in a
-  /// GIR_MergeMemOperands opcode.
-  GIU_MergeMemOperands_EndOfList = -1,
-};
-
 /// Provides the logic to execute GlobalISel match tables, which are used by the
 /// instruction selector and instruction combiners as their engine to match and
 /// apply MIR patterns.
@@ -595,14 +636,14 @@ protected:
   bool executeMatchTable(TgtExecutor &Exec, MatcherState &State,
                          const ExecInfoTy<PredicateBitset, ComplexMatcherMemFn,
                                           CustomRendererFn> &ExecInfo,
-                         MachineIRBuilder &Builder, const int64_t *MatchTable,
+                         MachineIRBuilder &Builder, const uint8_t *MatchTable,
                          const TargetInstrInfo &TII, MachineRegisterInfo &MRI,
                          const TargetRegisterInfo &TRI,
                          const RegisterBankInfo &RBI,
                          const PredicateBitset &AvailableFeatures,
                          CodeGenCoverage *CoverageInfo) const;
 
-  virtual const int64_t *getMatchTable() const {
+  virtual const uint8_t *getMatchTable() const {
     llvm_unreachable("Should have been overridden by tablegen if used");
   }
 
@@ -647,6 +688,12 @@ protected:
   /// MI and IntoMI do not need to be in the same basic blocks, but MI must
   /// preceed IntoMI.
   bool isObviouslySafeToFold(MachineInstr &MI, MachineInstr &IntoMI) const;
+
+  template <typename Ty> static Ty readBytesAs(const uint8_t *MatchTable) {
+    Ty Ret;
+    memcpy(&Ret, MatchTable, sizeof(Ret));
+    return Ret;
+  }
 };
 
 } // end namespace llvm
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h b/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h
index f0ee76c097bcab5faf7f6551774bedcf2e318501..0a2709ef216be440a7152745c1ce1d66875721b3 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h
@@ -33,6 +33,7 @@
 #include "llvm/Support/CodeGenCoverage.h"
 #include "llvm/Support/Debug.h"
 #include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/LEB128.h"
 #include "llvm/Support/raw_ostream.h"
 #include <cassert>
 #include <cstddef>
@@ -46,7 +47,7 @@ bool GIMatchTableExecutor::executeMatchTable(
     TgtExecutor &Exec, MatcherState &State,
     const ExecInfoTy<PredicateBitset, ComplexMatcherMemFn, CustomRendererFn>
         &ExecInfo,
-    MachineIRBuilder &Builder, const int64_t *MatchTable,
+    MachineIRBuilder &Builder, const uint8_t *MatchTable,
     const TargetInstrInfo &TII, MachineRegisterInfo &MRI,
     const TargetRegisterInfo &TRI, const RegisterBankInfo &RBI,
     const PredicateBitset &AvailableFeatures,
@@ -92,28 +93,60 @@ bool GIMatchTableExecutor::executeMatchTable(
 
   // If the index is >= 0, it's an index in the type objects generated by
   // TableGen. If the index is <0, it's an index in the recorded types object.
-  auto getTypeFromIdx = [&](int64_t Idx) -> LLT {
+  const auto getTypeFromIdx = [&](int64_t Idx) -> LLT {
     if (Idx >= 0)
       return ExecInfo.TypeObjects[Idx];
     return State.RecordedTypes[1 - Idx];
   };
 
+  const auto readULEB = [&]() {
+    unsigned N = 0;
+    uint64_t Val = decodeULEB128(MatchTable + CurrentIdx, &N);
+    CurrentIdx += N;
+    return Val;
+  };
+
+  // Convenience function to return a signed value. This avoids
+  // us forgetting to first cast to int8_t before casting to a
+  // wider signed int type.
+  // if we casted uint8 directly to a wider type we'd lose
+  // negative values.
+  const auto readS8 = [&]() { return (int8_t)MatchTable[CurrentIdx++]; };
+
+  const auto readU16 = [&]() {
+    auto V = readBytesAs<uint16_t>(MatchTable + CurrentIdx);
+    CurrentIdx += 2;
+    return V;
+  };
+
+  const auto readU32 = [&]() {
+    auto V = readBytesAs<uint32_t>(MatchTable + CurrentIdx);
+    CurrentIdx += 4;
+    return V;
+  };
+
+  const auto readU64 = [&]() {
+    auto V = readBytesAs<uint64_t>(MatchTable + CurrentIdx);
+    CurrentIdx += 8;
+    return V;
+  };
+
   while (true) {
     assert(CurrentIdx != ~0u && "Invalid MatchTable index");
-    int64_t MatcherOpcode = MatchTable[CurrentIdx++];
+    uint8_t MatcherOpcode = MatchTable[CurrentIdx++];
     switch (MatcherOpcode) {
     case GIM_Try: {
       DEBUG_WITH_TYPE(TgtExecutor::getName(),
                       dbgs() << CurrentIdx << ": Begin try-block\n");
-      OnFailResumeAt.push_back(MatchTable[CurrentIdx++]);
+      OnFailResumeAt.push_back(readU32());
       break;
     }
 
     case GIM_RecordInsn:
     case GIM_RecordInsnIgnoreCopies: {
-      int64_t NewInsnID = MatchTable[CurrentIdx++];
-      int64_t InsnID = MatchTable[CurrentIdx++];
-      int64_t OpIdx = MatchTable[CurrentIdx++];
+      uint64_t NewInsnID = readULEB();
+      uint64_t InsnID = readULEB();
+      uint64_t OpIdx = readULEB();
 
       // As an optimisation we require that MIs[0] is always the root. Refuse
       // any attempt to modify it.
@@ -156,7 +189,7 @@ bool GIMatchTableExecutor::executeMatchTable(
     }
 
     case GIM_CheckFeatures: {
-      int64_t ExpectedBitsetID = MatchTable[CurrentIdx++];
+      uint16_t ExpectedBitsetID = readU16();
       DEBUG_WITH_TYPE(TgtExecutor::getName(),
                       dbgs() << CurrentIdx
                              << ": GIM_CheckFeatures(ExpectedBitsetID="
@@ -170,11 +203,11 @@ bool GIMatchTableExecutor::executeMatchTable(
     }
     case GIM_CheckOpcode:
     case GIM_CheckOpcodeIsEither: {
-      int64_t InsnID = MatchTable[CurrentIdx++];
-      int64_t Expected0 = MatchTable[CurrentIdx++];
-      int64_t Expected1 = -1;
+      uint64_t InsnID = readULEB();
+      uint16_t Expected0 = readU16();
+      uint16_t Expected1 = -1;
       if (MatcherOpcode == GIM_CheckOpcodeIsEither)
-        Expected1 = MatchTable[CurrentIdx++];
+        Expected1 = readU16();
 
       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
       unsigned Opcode = State.MIs[InsnID]->getOpcode();
@@ -193,10 +226,10 @@ bool GIMatchTableExecutor::executeMatchTable(
       break;
     }
     case GIM_SwitchOpcode: {
-      int64_t InsnID = MatchTable[CurrentIdx++];
-      int64_t LowerBound = MatchTable[CurrentIdx++];
-      int64_t UpperBound = MatchTable[CurrentIdx++];
-      int64_t Default = MatchTable[CurrentIdx++];
+      uint64_t InsnID = readULEB();
+      uint16_t LowerBound = readU16();
+      uint16_t UpperBound = readU16();
+      uint32_t Default = readU32();
 
       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
       const int64_t Opcode = State.MIs[InsnID]->getOpcode();
@@ -210,7 +243,10 @@ bool GIMatchTableExecutor::executeMatchTable(
         CurrentIdx = Default;
         break;
       }
-      CurrentIdx = MatchTable[CurrentIdx + (Opcode - LowerBound)];
+      const auto EntryIdx = (Opcode - LowerBound);
+      // Each entry is 4 bytes
+      CurrentIdx =
+          readBytesAs<uint32_t>(MatchTable + CurrentIdx + (EntryIdx * 4));
       if (!CurrentIdx) {
         CurrentIdx = Default;
         break;
@@ -220,11 +256,11 @@ bool GIMatchTableExecutor::executeMatchTable(
     }
 
     case GIM_SwitchType: {
-      int64_t InsnID = MatchTable[CurrentIdx++];
-      int64_t OpIdx = MatchTable[CurrentIdx++];
-      int64_t LowerBound = MatchTable[CurrentIdx++];
-      int64_t UpperBound = MatchTable[CurrentIdx++];
-      int64_t Default = MatchTable[CurrentIdx++];
+      uint64_t InsnID = readULEB();
+      uint64_t OpIdx = readULEB();
+      uint16_t LowerBound = readU16();
+      uint16_t UpperBound = readU16();
+      int64_t Default = readU32();
 
       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
       MachineOperand &MO = State.MIs[InsnID]->getOperand(OpIdx);
@@ -254,7 +290,10 @@ bool GIMatchTableExecutor::executeMatchTable(
         CurrentIdx = Default;
         break;
       }
-      CurrentIdx = MatchTable[CurrentIdx + (TypeID - LowerBound)];
+      const auto NumEntry = (TypeID - LowerBound);
+      // Each entry is 4 bytes
+      CurrentIdx =
+          readBytesAs<uint32_t>(MatchTable + CurrentIdx + (NumEntry * 4));
       if (!CurrentIdx) {
         CurrentIdx = Default;
         break;
@@ -264,8 +303,8 @@ bool GIMatchTableExecutor::executeMatchTable(
     }
 
     case GIM_CheckNumOperands: {
-      int64_t InsnID = MatchTable[CurrentIdx++];
-      int64_t Expected = MatchTable[CurrentIdx++];
+      uint64_t InsnID = readULEB();
+      uint64_t Expected = readULEB();
       DEBUG_WITH_TYPE(TgtExecutor::getName(),
                       dbgs() << CurrentIdx << ": GIM_CheckNumOperands(MIs["
                              << InsnID << "], Expected=" << Expected << ")\n");
@@ -278,11 +317,10 @@ bool GIMatchTableExecutor::executeMatchTable(
     }
     case GIM_CheckI64ImmPredicate:
     case GIM_CheckImmOperandPredicate: {
-      int64_t InsnID = MatchTable[CurrentIdx++];
-      int64_t OpIdx = MatcherOpcode == GIM_CheckImmOperandPredicate
-                          ? MatchTable[CurrentIdx++]
-                          : 1;
-      int64_t Predicate = MatchTable[CurrentIdx++];
+      uint64_t InsnID = readULEB();
+      unsigned OpIdx =
+          MatcherOpcode == GIM_CheckImmOperandPredicate ? readULEB() : 1;
+      uint16_t Predicate = readU16();
       DEBUG_WITH_TYPE(TgtExecutor::getName(),
                       dbgs() << CurrentIdx << ": GIM_CheckImmPredicate(MIs["
                              << InsnID << "]->getOperand(" << OpIdx
@@ -306,8 +344,8 @@ bool GIMatchTableExecutor::executeMatchTable(
       break;
     }
     case GIM_CheckAPIntImmPredicate: {
-      int64_t InsnID = MatchTable[CurrentIdx++];
-      int64_t Predicate = MatchTable[CurrentIdx++];
+      uint64_t InsnID = readULEB();
+      uint16_t Predicate = readU16();
       DEBUG_WITH_TYPE(TgtExecutor::getName(),
                       dbgs()
                           << CurrentIdx << ": GIM_CheckAPIntImmPredicate(MIs["
@@ -327,8 +365,8 @@ bool GIMatchTableExecutor::executeMatchTable(
       break;
     }
     case GIM_CheckAPFloatImmPredicate: {
-      int64_t InsnID = MatchTable[CurrentIdx++];
-      int64_t Predicate = MatchTable[CurrentIdx++];
+      uint64_t InsnID = readULEB();
+      uint16_t Predicate = readU16();
       DEBUG_WITH_TYPE(TgtExecutor::getName(),
                       dbgs()
                           << CurrentIdx << ": GIM_CheckAPFloatImmPredicate(MIs["
@@ -349,7 +387,7 @@ bool GIMatchTableExecutor::executeMatchTable(
     }
     case GIM_CheckIsBuildVectorAllOnes:
     case GIM_CheckIsBuildVectorAllZeros: {
-      int64_t InsnID = MatchTable[CurrentIdx++];
+      uint64_t InsnID = readULEB();
 
       DEBUG_WITH_TYPE(TgtExecutor::getName(),
                       dbgs() << CurrentIdx
@@ -380,7 +418,7 @@ bool GIMatchTableExecutor::executeMatchTable(
       // Note: we don't check for invalid here because this is purely a hook to
       // allow some executors (such as the combiner) to check arbitrary,
       // contextless predicates, such as whether a rule is enabled or not.
-      int64_t Predicate = MatchTable[CurrentIdx++];
+      uint16_t Predicate = readU16();
       DEBUG_WITH_TYPE(TgtExecutor::getName(),
                       dbgs() << CurrentIdx
                              << ": GIM_CheckSimplePredicate(Predicate="
@@ -393,8 +431,8 @@ bool GIMatchTableExecutor::executeMatchTable(
       break;
     }
     case GIM_CheckCxxInsnPredicate: {
-      int64_t InsnID = MatchTable[CurrentIdx++];
-      int64_t Predicate = MatchTable[CurrentIdx++];
+      uint64_t InsnID = readULEB();
+      uint16_t Predicate = readU16();
       DEBUG_WITH_TYPE(TgtExecutor::getName(),
                       dbgs()
                           << CurrentIdx << ": GIM_CheckCxxPredicate(MIs["
@@ -408,7 +446,7 @@ bool GIMatchTableExecutor::executeMatchTable(
       break;
     }
     case GIM_CheckHasNoUse: {
-      int64_t InsnID = MatchTable[CurrentIdx++];
+      uint64_t InsnID = readULEB();
 
       DEBUG_WITH_TYPE(TgtExecutor::getName(),
                       dbgs() << CurrentIdx << ": GIM_CheckHasNoUse(MIs["
@@ -427,8 +465,8 @@ bool GIMatchTableExecutor::executeMatchTable(
       break;
     }
     case GIM_CheckAtomicOrdering: {
-      int64_t InsnID = MatchTable[CurrentIdx++];
-      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
+      uint64_t InsnID = readULEB();
+      auto Ordering = (AtomicOrdering)readULEB();
       DEBUG_WITH_TYPE(TgtExecutor::getName(),
                       dbgs() << CurrentIdx << ": GIM_CheckAtomicOrdering(MIs["
                              << InsnID << "], " << (uint64_t)Ordering << ")\n");
@@ -444,8 +482,8 @@ bool GIMatchTableExecutor::executeMatchTable(
       break;
     }
     case GIM_CheckAtomicOrderingOrStrongerThan: {
-      int64_t InsnID = MatchTable[CurrentIdx++];
-      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
+      uint64_t InsnID = readULEB();
+      auto Ordering = (AtomicOrdering)readULEB();
       DEBUG_WITH_TYPE(TgtExecutor::getName(),
                       dbgs() << CurrentIdx
                              << ": GIM_CheckAtomicOrderingOrStrongerThan(MIs["
@@ -462,8 +500,8 @@ bool GIMatchTableExecutor::executeMatchTable(
       break;
     }
     case GIM_CheckAtomicOrderingWeakerThan: {
-      int64_t InsnID = MatchTable[CurrentIdx++];
-      AtomicOrdering Ordering = (AtomicOrdering)MatchTable[CurrentIdx++];
+      uint64_t InsnID = readULEB();
+      auto Ordering = (AtomicOrdering)readULEB();
       DEBUG_WITH_TYPE(TgtExecutor::getName(),
                       dbgs() << CurrentIdx
                              << ": GIM_CheckAtomicOrderingWeakerThan(MIs["
@@ -480,10 +518,10 @@ bool GIMatchTableExecutor::executeMatchTable(
       break;
     }
     case GIM_CheckMemoryAddressSpace: {
-      int64_t InsnID = MatchTable[CurrentIdx++];
-      int64_t MMOIdx = MatchTable[CurrentIdx++];
+      uint64_t InsnID = readULEB();
+      uint64_t MMOIdx = readULEB();
       // This accepts a list of possible address spaces.
-      const int NumAddrSpace = MatchTable[CurrentIdx++];
+      const uint64_t NumAddrSpace = readULEB();
 
       if (State.MIs[InsnID]->getNumMemOperands() <= MMOIdx) {
         if (handleReject() == RejectAndGiveUp)
@@ -500,8 +538,8 @@ bool GIMatchTableExecutor::executeMatchTable(
       const unsigned MMOAddrSpace = MMO->getAddrSpace();
 
       bool Success = false;
-      for (int I = 0; I != NumAddrSpace; ++I) {
-        unsigned AddrSpace = MatchTable[CurrentIdx++];
+      for (unsigned I = 0; I != NumAddrSpace; ++I) {
+        uint64_t AddrSpace = readULEB();
         DEBUG_WITH_TYPE(TgtExecutor::getName(),
                         dbgs() << "addrspace(" << MMOAddrSpace << ") vs "
                                << AddrSpace << '\n');
@@ -518,9 +556,9 @@ bool GIMatchTableExecutor::executeMatchTable(
       break;
     }
     case GIM_CheckMemoryAlignment: {
-      int64_t InsnID = MatchTable[CurrentIdx++];
-      int64_t MMOIdx = MatchTable[CurrentIdx++];
-      unsigned MinAlign = MatchTable[CurrentIdx++];
+      uint64_t InsnID = readULEB();
+      uint64_t MMOIdx = readULEB();
+      uint64_t MinAlign = readULEB();
 
       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
 
@@ -543,9 +581,9 @@ bool GIMatchTableExecutor::executeMatchTable(
       break;
     }
     case GIM_CheckMemorySizeEqualTo: {
-      int64_t InsnID = MatchTable[CurrentIdx++];
-      int64_t MMOIdx = MatchTable[CurrentIdx++];
-      uint64_t Size = MatchTable[CurrentIdx++];
+      uint64_t InsnID = readULEB();
+      uint64_t MMOIdx = readULEB();
+      uint32_t Size = readU32();
 
       DEBUG_WITH_TYPE(TgtExecutor::getName(),
                       dbgs() << CurrentIdx << ": GIM_CheckMemorySizeEqual(MIs["
@@ -574,9 +612,9 @@ bool GIMatchTableExecutor::executeMatchTable(
     case GIM_CheckMemorySizeEqualToLLT:
     case GIM_CheckMemorySizeLessThanLLT:
     case GIM_CheckMemorySizeGreaterThanLLT: {
-      int64_t InsnID = MatchTable[CurrentIdx++];
-      int64_t MMOIdx = MatchTable[CurrentIdx++];
-      int64_t OpIdx = MatchTable[CurrentIdx++];
+      uint64_t InsnID = readULEB();
+      uint64_t MMOIdx = readULEB();
+      uint64_t OpIdx = readULEB();
 
       DEBUG_WITH_TYPE(
           TgtExecutor::getName(),
@@ -624,9 +662,9 @@ bool GIMatchTableExecutor::executeMatchTable(
       break;
     }
     case GIM_CheckType: {
-      int64_t InsnID = MatchTable[CurrentIdx++];
-      int64_t OpIdx = MatchTable[CurrentIdx++];
-      int64_t TypeID = MatchTable[CurrentIdx++];
+      uint64_t InsnID = readULEB();
+      uint64_t OpIdx = readULEB();
+      int TypeID = readS8();
       DEBUG_WITH_TYPE(TgtExecutor::getName(),
                       dbgs() << CurrentIdx << ": GIM_CheckType(MIs[" << InsnID
                              << "]->getOperand(" << OpIdx
@@ -640,9 +678,9 @@ bool GIMatchTableExecutor::executeMatchTable(
       break;
     }
     case GIM_CheckPointerToAny: {
-      int64_t InsnID = MatchTable[CurrentIdx++];
-      int64_t OpIdx = MatchTable[CurrentIdx++];
-      uint64_t SizeInBits = MatchTable[CurrentIdx++];
+      uint64_t InsnID = readULEB();
+      uint64_t OpIdx = readULEB();
+      uint64_t SizeInBits = readULEB();
 
       DEBUG_WITH_TYPE(TgtExecutor::getName(),
                       dbgs() << CurrentIdx << ": GIM_CheckPointerToAny(MIs["
@@ -671,9 +709,9 @@ bool GIMatchTableExecutor::executeMatchTable(
       break;
     }
     case GIM_RecordNamedOperand: {
-      int64_t InsnID = MatchTable[CurrentIdx++];
-      int64_t OpIdx = MatchTable[CurrentIdx++];
-      uint64_t StoreIdx = MatchTable[CurrentIdx++];
+      uint64_t InsnID = readULEB();
+      uint64_t OpIdx = readULEB();
+      uint64_t StoreIdx = readULEB();
 
       DEBUG_WITH_TYPE(TgtExecutor::getName(),
                       dbgs() << CurrentIdx << ": GIM_RecordNamedOperand(MIs["
@@ -685,16 +723,16 @@ bool GIMatchTableExecutor::executeMatchTable(
       break;
     }
     case GIM_RecordRegType: {
-      int64_t InsnID = MatchTable[CurrentIdx++];
-      int64_t OpIdx = MatchTable[CurrentIdx++];
-      int64_t TypeIdx = MatchTable[CurrentIdx++];
+      uint64_t InsnID = readULEB();
+      uint64_t OpIdx = readULEB();
+      int TypeIdx = readS8();
 
       DEBUG_WITH_TYPE(TgtExecutor::getName(),
                       dbgs() << CurrentIdx << ": GIM_RecordRegType(MIs["
                              << InsnID << "]->getOperand(" << OpIdx
                              << "), TypeIdx=" << TypeIdx << ")\n");
       assert(State.MIs[InsnID] != nullptr && "Used insn before defined");
-      assert(TypeIdx <= 0 && "Temp types always have negative indexes!");
+      assert(TypeIdx < 0 && "Temp types always have negative indexes!");
       // Indexes start at -1.
       TypeIdx = 1 - TypeIdx;
       const auto &Op = State.MIs[InsnID]->getOperand(OpIdx);
@@ -704,9 +742,9 @@ bool GIMatchTableExecutor::executeMatchTable(
       break;
     }
     case GIM_CheckRegBankForClass: {
-      int64_t InsnID = MatchTable[CurrentIdx++];
-      int64_t OpIdx = MatchTable[CurrentIdx++];
-      int64_t RCEnum = MatchTable[CurrentIdx++];
+      uint64_t InsnID = readULEB();
+      uint64_t OpIdx = readULEB();
+      uint16_t RCEnum = readU16();
       DEBUG_WITH_TYPE(TgtExecutor::getName(),
                       dbgs() << CurrentIdx << ": GIM_CheckRegBankForClass(MIs["
                              << InsnID << "]->getOperand(" << OpIdx
@@ -724,10 +762,10 @@ bool GIMatchTableExecutor::executeMatchTable(
     }
 
     case GIM_CheckComplexPattern: {
-      int64_t InsnID = MatchTable[CurrentIdx++];
-      int64_t OpIdx = MatchTable[CurrentIdx++];
-      int64_t RendererID = MatchTable[CurrentIdx++];
-      int64_t ComplexPredicateID = MatchTable[CurrentIdx++];
+      uint64_t InsnID = readULEB();
+      uint64_t OpIdx = readULEB();
+      uint16_t RendererID = readU16();
+      uint16_t ComplexPredicateID = readU16();
       DEBUG_WITH_TYPE(TgtExecutor::getName(),
                       dbgs() << CurrentIdx << ": State.Renderers[" << RendererID
                              << "] = GIM_CheckComplexPattern(MIs[" << InsnID
@@ -746,10 +784,13 @@ bool GIMatchTableExecutor::executeMatchTable(
       break;
     }
 
-    case GIM_CheckConstantInt: {
-      int64_t InsnID = MatchTable[CurrentIdx++];
-      int64_t OpIdx = MatchTable[CurrentIdx++];
-      int64_t Value = MatchTable[CurrentIdx++];
+    case GIM_CheckConstantInt:
+    case GIM_CheckConstantInt8: {
+      const bool IsInt8 = (MatcherOpcode == GIM_CheckConstantInt8);
+
+      uint64_t InsnID = readULEB();
+      uint64_t OpIdx = readULEB();
+      uint64_t Value = IsInt8 ? (int64_t)readS8() : readU64();
       DEBUG_WITH_TYPE(TgtExecutor::getName(),
                       dbgs() << CurrentIdx << ": GIM_CheckConstantInt(MIs["
                              << InsnID << "]->getOperand(" << OpIdx
@@ -779,9 +820,9 @@ bool GIMatchTableExecutor::executeMatchTable(
     }
 
     case GIM_CheckLiteralInt: {
-      int64_t InsnID = MatchTable[CurrentIdx++];
-      int64_t OpIdx = MatchTable[CurrentIdx++];
-      int64_t Value = MatchTable[CurrentIdx++];
+      uint64_t InsnID = readULEB();
+      uint64_t OpIdx = readULEB();
+      int64_t Value = readU64();
       DEBUG_WITH_TYPE(TgtExecutor::getName(),
                       dbgs() << CurrentIdx << ": GIM_CheckLiteralInt(MIs["
                              << InsnID << "]->getOperand(" << OpIdx
@@ -801,9 +842,9 @@ bool GIMatchTableExecutor::executeMatchTable(
     }
 
     case GIM_CheckIntrinsicID: {
-      int64_t InsnID = MatchTable[CurrentIdx++];
-      int64_t OpIdx = MatchTable[CurrentIdx++];
-      int64_t Value = MatchTable[CurrentIdx++];
+      uint64_t InsnID = readULEB();
+      uint64_t OpIdx = readULEB();
+      uint16_t Value = readU16();
       DEBUG_WITH_TYPE(TgtExecutor::getName(),
                       dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs["
                              << InsnID << "]->getOperand(" << OpIdx
@@ -816,9 +857,9 @@ bool GIMatchTableExecutor::executeMatchTable(
       break;
     }
     case GIM_CheckCmpPredicate: {
-      int64_t InsnID = MatchTable[CurrentIdx++];
-      int64_t OpIdx = MatchTable[CurrentIdx++];
-      int64_t Value = MatchTable[CurrentIdx++];
+      uint64_t InsnID = readULEB();
+      uint64_t OpIdx = readULEB();
+      uint16_t Value = readU16();
       DEBUG_WITH_TYPE(TgtExecutor::getName(),
                       dbgs() << CurrentIdx << ": GIM_CheckCmpPredicate(MIs["
                              << InsnID << "]->getOperand(" << OpIdx
@@ -831,8 +872,8 @@ bool GIMatchTableExecutor::executeMatchTable(
       break;
     }
     case GIM_CheckIsMBB: {
-      int64_t InsnID = MatchTable[CurrentIdx++];
-      int64_t OpIdx = MatchTable[CurrentIdx++];
+      uint64_t InsnID = readULEB();
+      uint64_t OpIdx = readULEB();
       DEBUG_WITH_TYPE(TgtExecutor::getName(),
                       dbgs() << CurrentIdx << ": GIM_CheckIsMBB(MIs[" << InsnID
                              << "]->getOperand(" << OpIdx << "))\n");
@@ -844,8 +885,8 @@ bool GIMatchTableExecutor::executeMatchTable(
       break;
     }
     case GIM_CheckIsImm: {
-      int64_t InsnID = MatchTable[CurrentIdx++];
-      int64_t OpIdx = MatchTable[CurrentIdx++];
+      uint64_t InsnID = readULEB();
+      uint64_t OpIdx = readULEB();
       DEBUG_WITH_TYPE(TgtExecutor::getName(),
                       dbgs() << CurrentIdx << ": GIM_CheckIsImm(MIs[" << InsnID
                              << "]->getOperand(" << OpIdx << "))\n");
@@ -857,7 +898,7 @@ bool GIMatchTableExecutor::executeMatchTable(
       break;
     }
     case GIM_CheckIsSafeToFold: {
-      int64_t InsnID = MatchTable[CurrentIdx++];
+      uint64_t InsnID = readULEB();
       DEBUG_WITH_TYPE(TgtExecutor::getName(),
                       dbgs() << CurrentIdx << ": GIM_CheckIsSafeToFold(MIs["
                              << InsnID << "])\n");
@@ -870,10 +911,10 @@ bool GIMatchTableExecutor::executeMatchTable(
     }
     case GIM_CheckIsSameOperand:
     case GIM_CheckIsSameOperandIgnoreCopies: {
-      int64_t InsnID = MatchTable[CurrentIdx++];
-      int64_t OpIdx = MatchTable[CurrentIdx++];
-      int64_t OtherInsnID = MatchTable[CurrentIdx++];
-      int64_t OtherOpIdx = MatchTable[CurrentIdx++];
+      uint64_t InsnID = readULEB();
+      uint64_t OpIdx = readULEB();
+      uint64_t OtherInsnID = readULEB();
+      uint64_t OtherOpIdx = readULEB();
       DEBUG_WITH_TYPE(TgtExecutor::getName(),
                       dbgs() << CurrentIdx << ": GIM_CheckIsSameOperand(MIs["
                              << InsnID << "][" << OpIdx << "], MIs["
@@ -899,10 +940,10 @@ bool GIMatchTableExecutor::executeMatchTable(
       break;
     }
     case GIM_CheckCanReplaceReg: {
-      int64_t OldInsnID = MatchTable[CurrentIdx++];
-      int64_t OldOpIdx = MatchTable[CurrentIdx++];
-      int64_t NewInsnID = MatchTable[CurrentIdx++];
-      int64_t NewOpIdx = MatchTable[CurrentIdx++];
+      uint64_t OldInsnID = readULEB();
+      uint64_t OldOpIdx = readULEB();
+      uint64_t NewInsnID = readULEB();
+      uint64_t NewOpIdx = readULEB();
 
       DEBUG_WITH_TYPE(TgtExecutor::getName(),
                       dbgs() << CurrentIdx << ": GIM_CheckCanReplaceReg(MIs["
@@ -918,8 +959,8 @@ bool GIMatchTableExecutor::executeMatchTable(
       break;
     }
     case GIM_MIFlags: {
-      int64_t InsnID = MatchTable[CurrentIdx++];
-      uint32_t Flags = (uint32_t)MatchTable[CurrentIdx++];
+      uint64_t InsnID = readULEB();
+      uint32_t Flags = readU32();
 
       DEBUG_WITH_TYPE(TgtExecutor::getName(),
                       dbgs() << CurrentIdx << ": GIM_MIFlags(MIs[" << InsnID
@@ -931,8 +972,8 @@ bool GIMatchTableExecutor::executeMatchTable(
       break;
     }
     case GIM_MIFlagsNot: {
-      int64_t InsnID = MatchTable[CurrentIdx++];
-      uint32_t Flags = (uint32_t)MatchTable[CurrentIdx++];
+      uint64_t InsnID = readULEB();
+      uint32_t Flags = readU32();
 
       DEBUG_WITH_TYPE(TgtExecutor::getName(),
                       dbgs() << CurrentIdx << ": GIM_MIFlagsNot(MIs[" << InsnID
@@ -950,9 +991,9 @@ bool GIMatchTableExecutor::executeMatchTable(
         return false;
       break;
     case GIR_MutateOpcode: {
-      int64_t OldInsnID = MatchTable[CurrentIdx++];
-      uint64_t NewInsnID = MatchTable[CurrentIdx++];
-      int64_t NewOpcode = MatchTable[CurrentIdx++];
+      uint64_t OldInsnID = readULEB();
+      uint64_t NewInsnID = readULEB();
+      uint16_t NewOpcode = readU16();
       if (NewInsnID >= OutMIs.size())
         OutMIs.resize(NewInsnID + 1);
 
@@ -971,8 +1012,8 @@ bool GIMatchTableExecutor::executeMatchTable(
     }
 
     case GIR_BuildMI: {
-      uint64_t NewInsnID = MatchTable[CurrentIdx++];
-      int64_t Opcode = MatchTable[CurrentIdx++];
+      uint64_t NewInsnID = readULEB();
+      uint16_t Opcode = readU16();
       if (NewInsnID >= OutMIs.size())
         OutMIs.resize(NewInsnID + 1);
 
@@ -984,8 +1025,8 @@ bool GIMatchTableExecutor::executeMatchTable(
     }
 
     case GIR_BuildConstant: {
-      int64_t TempRegID = MatchTable[CurrentIdx++];
-      int64_t Imm = MatchTable[CurrentIdx++];
+      uint64_t TempRegID = readULEB();
+      uint64_t Imm = readU64();
       Builder.buildConstant(State.TempRegisters[TempRegID], Imm);
       DEBUG_WITH_TYPE(TgtExecutor::getName(),
                       dbgs() << CurrentIdx << ": GIR_BuildConstant(TempReg["
@@ -994,9 +1035,9 @@ bool GIMatchTableExecutor::executeMatchTable(
     }
 
     case GIR_Copy: {
-      int64_t NewInsnID = MatchTable[CurrentIdx++];
-      int64_t OldInsnID = MatchTable[CurrentIdx++];
-      int64_t OpIdx = MatchTable[CurrentIdx++];
+      uint64_t NewInsnID = readULEB();
+      uint64_t OldInsnID = readULEB();
+      uint64_t OpIdx = readULEB();
       assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
       OutMIs[NewInsnID].add(State.MIs[OldInsnID]->getOperand(OpIdx));
       DEBUG_WITH_TYPE(TgtExecutor::getName(),
@@ -1007,10 +1048,10 @@ bool GIMatchTableExecutor::executeMatchTable(
     }
 
     case GIR_CopyOrAddZeroReg: {
-      int64_t NewInsnID = MatchTable[CurrentIdx++];
-      int64_t OldInsnID = MatchTable[CurrentIdx++];
-      int64_t OpIdx = MatchTable[CurrentIdx++];
-      int64_t ZeroReg = MatchTable[CurrentIdx++];
+      uint64_t NewInsnID = readULEB();
+      uint64_t OldInsnID = readULEB();
+      uint64_t OpIdx = readULEB();
+      uint16_t ZeroReg = readU16();
       assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
       MachineOperand &MO = State.MIs[OldInsnID]->getOperand(OpIdx);
       if (isOperandImmEqual(MO, 0, MRI))
@@ -1025,10 +1066,10 @@ bool GIMatchTableExecutor::executeMatchTable(
     }
 
     case GIR_CopySubReg: {
-      int64_t NewInsnID = MatchTable[CurrentIdx++];
-      int64_t OldInsnID = MatchTable[CurrentIdx++];
-      int64_t OpIdx = MatchTable[CurrentIdx++];
-      int64_t SubRegIdx = MatchTable[CurrentIdx++];
+      uint64_t NewInsnID = readULEB();
+      uint64_t OldInsnID = readULEB();
+      uint64_t OpIdx = readULEB();
+      uint16_t SubRegIdx = readU16();
       assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
       OutMIs[NewInsnID].addReg(State.MIs[OldInsnID]->getOperand(OpIdx).getReg(),
                                0, SubRegIdx);
@@ -1040,9 +1081,9 @@ bool GIMatchTableExecutor::executeMatchTable(
     }
 
     case GIR_AddImplicitDef: {
-      int64_t InsnID = MatchTable[CurrentIdx++];
-      int64_t RegNum = MatchTable[CurrentIdx++];
-      auto Flags = (uint64_t)MatchTable[CurrentIdx++];
+      uint64_t InsnID = readULEB();
+      uint16_t RegNum = readU16();
+      uint16_t Flags = readU16();
       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
       Flags |= RegState::Implicit;
       OutMIs[InsnID].addDef(RegNum, Flags);
@@ -1053,8 +1094,8 @@ bool GIMatchTableExecutor::executeMatchTable(
     }
 
     case GIR_AddImplicitUse: {
-      int64_t InsnID = MatchTable[CurrentIdx++];
-      int64_t RegNum = MatchTable[CurrentIdx++];
+      uint64_t InsnID = readULEB();
+      uint16_t RegNum = readU16();
       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
       OutMIs[InsnID].addUse(RegNum, RegState::Implicit);
       DEBUG_WITH_TYPE(TgtExecutor::getName(),
@@ -1064,9 +1105,9 @@ bool GIMatchTableExecutor::executeMatchTable(
     }
 
     case GIR_AddRegister: {
-      int64_t InsnID = MatchTable[CurrentIdx++];
-      int64_t RegNum = MatchTable[CurrentIdx++];
-      uint64_t RegFlags = MatchTable[CurrentIdx++];
+      uint64_t InsnID = readULEB();
+      uint16_t RegNum = readU16();
+      uint16_t RegFlags = readU16();
       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
       OutMIs[InsnID].addReg(RegNum, RegFlags);
       DEBUG_WITH_TYPE(TgtExecutor::getName(),
@@ -1076,8 +1117,8 @@ bool GIMatchTableExecutor::executeMatchTable(
       break;
     }
     case GIR_SetImplicitDefDead: {
-      int64_t InsnID = MatchTable[CurrentIdx++];
-      int64_t OpIdx = MatchTable[CurrentIdx++];
+      uint64_t InsnID = readULEB();
+      uint64_t OpIdx = readULEB();
       DEBUG_WITH_TYPE(TgtExecutor::getName(),
                       dbgs() << CurrentIdx << ": GIR_SetImplicitDefDead(OutMIs["
                              << InsnID << "], OpIdx=" << OpIdx << ")\n");
@@ -1087,8 +1128,8 @@ bool GIMatchTableExecutor::executeMatchTable(
       break;
     }
     case GIR_SetMIFlags: {
-      int64_t InsnID = MatchTable[CurrentIdx++];
-      uint32_t Flags = (uint32_t)MatchTable[CurrentIdx++];
+      uint64_t InsnID = readULEB();
+      uint32_t Flags = readU32();
 
       DEBUG_WITH_TYPE(TgtExecutor::getName(),
                       dbgs() << CurrentIdx << ": GIR_SetMIFlags(OutMIs["
@@ -1098,8 +1139,8 @@ bool GIMatchTableExecutor::executeMatchTable(
       break;
     }
     case GIR_UnsetMIFlags: {
-      int64_t InsnID = MatchTable[CurrentIdx++];
-      uint32_t Flags = (uint32_t)MatchTable[CurrentIdx++];
+      uint64_t InsnID = readULEB();
+      uint32_t Flags = readU32();
 
       DEBUG_WITH_TYPE(TgtExecutor::getName(),
                       dbgs() << CurrentIdx << ": GIR_UnsetMIFlags(OutMIs["
@@ -1109,8 +1150,8 @@ bool GIMatchTableExecutor::executeMatchTable(
       break;
     }
     case GIR_CopyMIFlags: {
-      int64_t InsnID = MatchTable[CurrentIdx++];
-      int64_t OldInsnID = MatchTable[CurrentIdx++];
+      uint64_t InsnID = readULEB();
+      uint64_t OldInsnID = readULEB();
 
       DEBUG_WITH_TYPE(TgtExecutor::getName(),
                       dbgs() << CurrentIdx << ": GIR_CopyMIFlags(OutMIs["
@@ -1119,14 +1160,17 @@ bool GIMatchTableExecutor::executeMatchTable(
       MI->setFlags(MI->getFlags() | State.MIs[OldInsnID]->getFlags());
       break;
     }
+    case GIR_AddSimpleTempRegister:
     case GIR_AddTempRegister:
     case GIR_AddTempSubRegister: {
-      int64_t InsnID = MatchTable[CurrentIdx++];
-      int64_t TempRegID = MatchTable[CurrentIdx++];
-      uint64_t TempRegFlags = MatchTable[CurrentIdx++];
-      unsigned SubReg = 0;
+      uint64_t InsnID = readULEB();
+      uint64_t TempRegID = readULEB();
+      uint16_t TempRegFlags = 0;
+      if (MatcherOpcode != GIR_AddSimpleTempRegister)
+        TempRegFlags = readU16();
+      uint16_t SubReg = 0;
       if (MatcherOpcode == GIR_AddTempSubRegister)
-        SubReg = MatchTable[CurrentIdx++];
+        SubReg = readU16();
 
       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
 
@@ -1141,9 +1185,11 @@ bool GIMatchTableExecutor::executeMatchTable(
       break;
     }
 
+    case GIR_AddImm8:
     case GIR_AddImm: {
-      int64_t InsnID = MatchTable[CurrentIdx++];
-      int64_t Imm = MatchTable[CurrentIdx++];
+      const bool IsAdd8 = (MatcherOpcode == GIR_AddImm8);
+      uint64_t InsnID = readULEB();
+      uint64_t Imm = IsAdd8 ? (int64_t)readS8() : readU64();
       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
       OutMIs[InsnID].addImm(Imm);
       DEBUG_WITH_TYPE(TgtExecutor::getName(),
@@ -1153,9 +1199,9 @@ bool GIMatchTableExecutor::executeMatchTable(
     }
 
     case GIR_AddCImm: {
-      int64_t InsnID = MatchTable[CurrentIdx++];
-      int64_t TypeID = MatchTable[CurrentIdx++];
-      int64_t Imm = MatchTable[CurrentIdx++];
+      uint64_t InsnID = readULEB();
+      int TypeID = readS8();
+      uint64_t Imm = readU64();
       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
 
       unsigned Width = ExecInfo.TypeObjects[TypeID].getScalarSizeInBits();
@@ -1170,8 +1216,8 @@ bool GIMatchTableExecutor::executeMatchTable(
     }
 
     case GIR_ComplexRenderer: {
-      int64_t InsnID = MatchTable[CurrentIdx++];
-      int64_t RendererID = MatchTable[CurrentIdx++];
+      uint64_t InsnID = readULEB();
+      uint16_t RendererID = readU16();
       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
       for (const auto &RenderOpFn : State.Renderers[RendererID])
         RenderOpFn(OutMIs[InsnID]);
@@ -1181,9 +1227,9 @@ bool GIMatchTableExecutor::executeMatchTable(
       break;
     }
     case GIR_ComplexSubOperandRenderer: {
-      int64_t InsnID = MatchTable[CurrentIdx++];
-      int64_t RendererID = MatchTable[CurrentIdx++];
-      int64_t RenderOpID = MatchTable[CurrentIdx++];
+      uint64_t InsnID = readULEB();
+      uint16_t RendererID = readU16();
+      uint64_t RenderOpID = readULEB();
       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
       State.Renderers[RendererID][RenderOpID](OutMIs[InsnID]);
       DEBUG_WITH_TYPE(TgtExecutor::getName(),
@@ -1194,10 +1240,10 @@ bool GIMatchTableExecutor::executeMatchTable(
       break;
     }
     case GIR_ComplexSubOperandSubRegRenderer: {
-      int64_t InsnID = MatchTable[CurrentIdx++];
-      int64_t RendererID = MatchTable[CurrentIdx++];
-      int64_t RenderOpID = MatchTable[CurrentIdx++];
-      int64_t SubRegIdx = MatchTable[CurrentIdx++];
+      uint64_t InsnID = readULEB();
+      uint16_t RendererID = readU16();
+      uint64_t RenderOpID = readULEB();
+      uint16_t SubRegIdx = readU16();
       MachineInstrBuilder &MI = OutMIs[InsnID];
       assert(MI && "Attempted to add to undefined instruction");
       State.Renderers[RendererID][RenderOpID](MI);
@@ -1211,8 +1257,8 @@ bool GIMatchTableExecutor::executeMatchTable(
     }
 
     case GIR_CopyConstantAsSImm: {
-      int64_t NewInsnID = MatchTable[CurrentIdx++];
-      int64_t OldInsnID = MatchTable[CurrentIdx++];
+      uint64_t NewInsnID = readULEB();
+      uint64_t OldInsnID = readULEB();
       assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
       assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_CONSTANT &&
              "Expected G_CONSTANT");
@@ -1231,8 +1277,8 @@ bool GIMatchTableExecutor::executeMatchTable(
 
     // TODO: Needs a test case once we have a pattern that uses this.
     case GIR_CopyFConstantAsFPImm: {
-      int64_t NewInsnID = MatchTable[CurrentIdx++];
-      int64_t OldInsnID = MatchTable[CurrentIdx++];
+      uint64_t NewInsnID = readULEB();
+      uint64_t OldInsnID = readULEB();
       assert(OutMIs[NewInsnID] && "Attempted to add to undefined instruction");
       assert(State.MIs[OldInsnID]->getOpcode() == TargetOpcode::G_FCONSTANT &&
              "Expected G_FCONSTANT");
@@ -1249,9 +1295,9 @@ bool GIMatchTableExecutor::executeMatchTable(
     }
 
     case GIR_CustomRenderer: {
-      int64_t InsnID = MatchTable[CurrentIdx++];
-      int64_t OldInsnID = MatchTable[CurrentIdx++];
-      int64_t RendererFnID = MatchTable[CurrentIdx++];
+      uint64_t InsnID = readULEB();
+      uint64_t OldInsnID = readULEB();
+      uint16_t RendererFnID = readU16();
       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
       DEBUG_WITH_TYPE(TgtExecutor::getName(),
                       dbgs() << CurrentIdx << ": GIR_CustomRenderer(OutMIs["
@@ -1263,7 +1309,7 @@ bool GIMatchTableExecutor::executeMatchTable(
       break;
     }
     case GIR_CustomAction: {
-      int64_t FnID = MatchTable[CurrentIdx++];
+      uint16_t FnID = readU16();
       DEBUG_WITH_TYPE(TgtExecutor::getName(),
                       dbgs() << CurrentIdx << ": GIR_CustomAction(FnID=" << FnID
                              << ")\n");
@@ -1272,10 +1318,10 @@ bool GIMatchTableExecutor::executeMatchTable(
       break;
     }
     case GIR_CustomOperandRenderer: {
-      int64_t InsnID = MatchTable[CurrentIdx++];
-      int64_t OldInsnID = MatchTable[CurrentIdx++];
-      int64_t OpIdx = MatchTable[CurrentIdx++];
-      int64_t RendererFnID = MatchTable[CurrentIdx++];
+      uint64_t InsnID = readULEB();
+      uint64_t OldInsnID = readULEB();
+      uint64_t OpIdx = readULEB();
+      uint16_t RendererFnID = readU16();
       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
 
       DEBUG_WITH_TYPE(TgtExecutor::getName(),
@@ -1288,9 +1334,9 @@ bool GIMatchTableExecutor::executeMatchTable(
       break;
     }
     case GIR_ConstrainOperandRC: {
-      int64_t InsnID = MatchTable[CurrentIdx++];
-      int64_t OpIdx = MatchTable[CurrentIdx++];
-      int64_t RCEnum = MatchTable[CurrentIdx++];
+      uint64_t InsnID = readULEB();
+      uint64_t OpIdx = readULEB();
+      uint16_t RCEnum = readU16();
       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
       MachineInstr &I = *OutMIs[InsnID].getInstr();
       MachineFunction &MF = *I.getParent()->getParent();
@@ -1306,7 +1352,7 @@ bool GIMatchTableExecutor::executeMatchTable(
     }
 
     case GIR_ConstrainSelectedInstOperands: {
-      int64_t InsnID = MatchTable[CurrentIdx++];
+      uint64_t InsnID = readULEB();
       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
       constrainSelectedInstRegOperands(*OutMIs[InsnID].getInstr(), TII, TRI,
                                        RBI);
@@ -1318,18 +1364,18 @@ bool GIMatchTableExecutor::executeMatchTable(
     }
 
     case GIR_MergeMemOperands: {
-      int64_t InsnID = MatchTable[CurrentIdx++];
+      uint64_t InsnID = readULEB();
+      uint64_t NumInsn = MatchTable[CurrentIdx++];
       assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
 
       DEBUG_WITH_TYPE(TgtExecutor::getName(),
                       dbgs() << CurrentIdx << ": GIR_MergeMemOperands(OutMIs["
                              << InsnID << "]");
-      int64_t MergeInsnID = GIU_MergeMemOperands_EndOfList;
-      while ((MergeInsnID = MatchTable[CurrentIdx++]) !=
-             GIU_MergeMemOperands_EndOfList) {
+      for (unsigned K = 0; K < NumInsn; ++K) {
+        uint64_t NextID = readULEB();
         DEBUG_WITH_TYPE(TgtExecutor::getName(),
-                        dbgs() << ", MIs[" << MergeInsnID << "]");
-        for (const auto &MMO : State.MIs[MergeInsnID]->memoperands())
+                        dbgs() << ", MIs[" << NextID << "]");
+        for (const auto &MMO : State.MIs[NextID]->memoperands())
           OutMIs[InsnID].addMemOperand(MMO);
       }
       DEBUG_WITH_TYPE(TgtExecutor::getName(), dbgs() << ")\n");
@@ -1337,7 +1383,7 @@ bool GIMatchTableExecutor::executeMatchTable(
     }
 
     case GIR_EraseFromParent: {
-      int64_t InsnID = MatchTable[CurrentIdx++];
+      uint64_t InsnID = readULEB();
       MachineInstr *MI = State.MIs[InsnID];
       assert(MI && "Attempted to erase an undefined instruction");
       DEBUG_WITH_TYPE(TgtExecutor::getName(),
@@ -1354,8 +1400,8 @@ bool GIMatchTableExecutor::executeMatchTable(
     }
 
     case GIR_MakeTempReg: {
-      int64_t TempRegID = MatchTable[CurrentIdx++];
-      int64_t TypeID = MatchTable[CurrentIdx++];
+      uint64_t TempRegID = readULEB();
+      int TypeID = readS8();
 
       State.TempRegisters[TempRegID] =
           MRI.createGenericVirtualRegister(getTypeFromIdx(TypeID));
@@ -1365,10 +1411,10 @@ bool GIMatchTableExecutor::executeMatchTable(
       break;
     }
     case GIR_ReplaceReg: {
-      int64_t OldInsnID = MatchTable[CurrentIdx++];
-      int64_t OldOpIdx = MatchTable[CurrentIdx++];
-      int64_t NewInsnID = MatchTable[CurrentIdx++];
-      int64_t NewOpIdx = MatchTable[CurrentIdx++];
+      uint64_t OldInsnID = readULEB();
+      uint64_t OldOpIdx = readULEB();
+      uint64_t NewInsnID = readULEB();
+      uint64_t NewOpIdx = readULEB();
 
       DEBUG_WITH_TYPE(TgtExecutor::getName(),
                       dbgs() << CurrentIdx << ": GIR_ReplaceReg(MIs["
@@ -1385,9 +1431,9 @@ bool GIMatchTableExecutor::executeMatchTable(
       break;
     }
     case GIR_ReplaceRegWithTempReg: {
-      int64_t OldInsnID = MatchTable[CurrentIdx++];
-      int64_t OldOpIdx = MatchTable[CurrentIdx++];
-      int64_t TempRegID = MatchTable[CurrentIdx++];
+      uint64_t OldInsnID = readULEB();
+      uint64_t OldOpIdx = readULEB();
+      uint64_t TempRegID = readULEB();
 
       DEBUG_WITH_TYPE(TgtExecutor::getName(),
                       dbgs() << CurrentIdx << ": GIR_ReplaceRegWithTempReg(MIs["
@@ -1404,7 +1450,7 @@ bool GIMatchTableExecutor::executeMatchTable(
       break;
     }
     case GIR_Coverage: {
-      int64_t RuleID = MatchTable[CurrentIdx++];
+      uint32_t RuleID = readU32();
       assert(CoverageInfo);
       CoverageInfo->setCovered(RuleID);
 
diff --git a/llvm/include/llvm/CodeGen/IndirectBrExpand.h b/llvm/include/llvm/CodeGen/IndirectBrExpand.h
new file mode 100644
index 0000000000000000000000000000000000000000..f7d9d5df6fe272492b539e00f6992e30b3a15117
--- /dev/null
+++ b/llvm/include/llvm/CodeGen/IndirectBrExpand.h
@@ -0,0 +1,28 @@
+//===- llvm/CodeGen/IndirectBrExpand.h -------------------------*- C++ -*--===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGEN_INDIRECTBREXPAND_H
+#define LLVM_CODEGEN_INDIRECTBREXPAND_H
+
+#include "llvm/IR/PassManager.h"
+
+namespace llvm {
+
+class TargetMachine;
+
+class IndirectBrExpandPass : public PassInfoMixin<IndirectBrExpandPass> {
+  const TargetMachine *TM;
+
+public:
+  IndirectBrExpandPass(const TargetMachine *TM) : TM(TM) {}
+  PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM);
+};
+
+} // namespace llvm
+
+#endif // LLVM_CODEGEN_INDIRECTBREXPAND_H
diff --git a/llvm/include/llvm/CodeGen/InterleavedLoadCombine.h b/llvm/include/llvm/CodeGen/InterleavedLoadCombine.h
new file mode 100644
index 0000000000000000000000000000000000000000..fa99aa316c2a68aac9eeaa91aad8a794c79a55c5
--- /dev/null
+++ b/llvm/include/llvm/CodeGen/InterleavedLoadCombine.h
@@ -0,0 +1,29 @@
+//===- llvm/CodeGen/InterleavedLoadCombine.h --------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CODEGEN_INTERLEAVEDLOADCOMBINE_H
+#define LLVM_CODEGEN_INTERLEAVEDLOADCOMBINE_H
+
+#include "llvm/IR/PassManager.h"
+
+namespace llvm {
+
+class TargetMachine;
+
+class InterleavedLoadCombinePass
+    : public PassInfoMixin<InterleavedLoadCombinePass> {
+  const TargetMachine *TM;
+
+public:
+  explicit InterleavedLoadCombinePass(const TargetMachine *TM) : TM(TM) {}
+  PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM);
+};
+
+} // namespace llvm
+
+#endif // InterleavedLoadCombine
diff --git a/llvm/include/llvm/CodeGen/MachinePassRegistry.def b/llvm/include/llvm/CodeGen/MachinePassRegistry.def
index 9ebf33b2b9a5a3c7c7537c85e820aa46d05f2c60..ba81dbadf4bd601a5fc102921e69d24c2325b808 100644
--- a/llvm/include/llvm/CodeGen/MachinePassRegistry.def
+++ b/llvm/include/llvm/CodeGen/MachinePassRegistry.def
@@ -16,6 +16,7 @@
 #ifndef MODULE_ANALYSIS
 #define MODULE_ANALYSIS(NAME, PASS_NAME, CONSTRUCTOR)
 #endif
+MODULE_ANALYSIS("collector-metadata", CollectorMetadataAnalysis, ())
 MODULE_ANALYSIS("pass-instrumentation", PassInstrumentationAnalysis, (PIC))
 #undef MODULE_ANALYSIS
 
@@ -29,6 +30,7 @@ MODULE_PASS("jmc-instrumenter", JMCInstrumenterPass, ())
 #ifndef FUNCTION_ANALYSIS
 #define FUNCTION_ANALYSIS(NAME, PASS_NAME, CONSTRUCTOR)
 #endif
+FUNCTION_ANALYSIS("gc-function", GCFunctionAnalysis, ())
 FUNCTION_ANALYSIS("pass-instrumentation", PassInstrumentationAnalysis, (PIC))
 FUNCTION_ANALYSIS("targetir", TargetIRAnalysis,
                   (std::move(TM.getTargetIRAnalysis())))
@@ -38,14 +40,18 @@ FUNCTION_ANALYSIS("targetir", TargetIRAnalysis,
 #define FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR)
 #endif
 FUNCTION_PASS("callbrprepare", CallBrPreparePass, ())
+FUNCTION_PASS("cfguard", CFGuardPass, ())
 FUNCTION_PASS("consthoist", ConstantHoistingPass, ())
 FUNCTION_PASS("dwarf-eh-prepare", DwarfEHPreparePass, (TM))
 FUNCTION_PASS("ee-instrument", EntryExitInstrumenterPass, (false))
 FUNCTION_PASS("expand-large-div-rem", ExpandLargeDivRemPass, ())
 FUNCTION_PASS("expand-large-fp-convert", ExpandLargeFpConvertPass, ())
+FUNCTION_PASS("expand-memcmp", ExpandMemCmpPass, (TM))
 FUNCTION_PASS("expand-reductions", ExpandReductionsPass, ())
 FUNCTION_PASS("expandvp", ExpandVectorPredicationPass, ())
+FUNCTION_PASS("indirectbr-expand", IndirectBrExpandPass, (TM))
 FUNCTION_PASS("interleaved-access", InterleavedAccessPass, (TM))
+FUNCTION_PASS("interleaved-load-combine", InterleavedLoadCombinePass, (TM))
 FUNCTION_PASS("lower-constant-intrinsics", LowerConstantIntrinsicsPass, ())
 FUNCTION_PASS("lowerinvoke", LowerInvokePass, ())
 FUNCTION_PASS("mergeicmps", MergeICmpsPass, ())
@@ -124,12 +130,8 @@ MACHINE_FUNCTION_ANALYSIS("pass-instrumentation", PassInstrumentationAnalysis,
 #define DUMMY_FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR)
 #endif
 DUMMY_FUNCTION_PASS("atomic-expand", AtomicExpandPass, ())
-DUMMY_FUNCTION_PASS("cfguard-check", CFGuardCheckPass, ())
-DUMMY_FUNCTION_PASS("cfguard-dispatch", CFGuardDispatchPass, ())
 DUMMY_FUNCTION_PASS("codegenprepare", CodeGenPreparePass, ())
-DUMMY_FUNCTION_PASS("expandmemcmp", ExpandMemCmpPass, ())
 DUMMY_FUNCTION_PASS("gc-lowering", GCLoweringPass, ())
-DUMMY_FUNCTION_PASS("indirectbr-expand", IndirectBrExpandPass, ())
 DUMMY_FUNCTION_PASS("shadow-stack-gc-lowering", ShadowStackGCLoweringPass, ())
 DUMMY_FUNCTION_PASS("stack-protector", StackProtectorPass, ())
 #undef DUMMY_FUNCTION_PASS
diff --git a/llvm/include/llvm/CodeGen/Passes.h b/llvm/include/llvm/CodeGen/Passes.h
index 80def6dfef495123803a0bcdf589c58787f09c11..ca9fbb1def7624d03ef57d262b553f3b58968ffa 100644
--- a/llvm/include/llvm/CodeGen/Passes.h
+++ b/llvm/include/llvm/CodeGen/Passes.h
@@ -520,7 +520,7 @@ namespace llvm {
   FunctionPass *createExpandLargeFpConvertPass();
 
   // This pass expands memcmp() to load/stores.
-  FunctionPass *createExpandMemCmpPass();
+  FunctionPass *createExpandMemCmpLegacyPass();
 
   /// Creates Break False Dependencies pass. \see BreakFalseDeps.cpp
   FunctionPass *createBreakFalseDeps();
diff --git a/llvm/include/llvm/CodeGen/TargetInstrInfo.h b/llvm/include/llvm/CodeGen/TargetInstrInfo.h
index 2bbe430dc68d90f13417549fe3fd4eddaa0e5b71..a113100f04e621101b97bf8e6ff7915b7f84f4d2 100644
--- a/llvm/include/llvm/CodeGen/TargetInstrInfo.h
+++ b/llvm/include/llvm/CodeGen/TargetInstrInfo.h
@@ -1025,6 +1025,11 @@ protected:
     return std::nullopt;
   }
 
+  virtual std::optional<DestSourcePair>
+  isCopyLikeInstrImpl(const MachineInstr &MI) const {
+    return std::nullopt;
+  }
+
   /// Return true if the given terminator MI is not expected to spill. This
   /// sets the live interval as not spillable and adjusts phi node lowering to
   /// not introduce copies after the terminator. Use with care, these are
@@ -1050,6 +1055,14 @@ public:
     return isCopyInstrImpl(MI);
   }
 
+  // Similar to `isCopyInstr`, but adds non-copy semantics on MIR, but
+  // ultimately generates a copy instruction.
+  std::optional<DestSourcePair> isCopyLikeInstr(const MachineInstr &MI) const {
+    if (auto IsCopyInstr = isCopyInstr(MI))
+      return IsCopyInstr;
+    return isCopyLikeInstrImpl(MI);
+  }
+
   bool isFullCopyInstr(const MachineInstr &MI) const {
     auto DestSrc = isCopyInstr(MI);
     if (!DestSrc)
diff --git a/llvm/include/llvm/DebugInfo/Symbolize/MarkupFilter.h b/llvm/include/llvm/DebugInfo/Symbolize/MarkupFilter.h
index a1514d91702b9f12835626e15d74b3d4735434e2..10613ec57586df6e45a933a3107d9a40780e90cd 100644
--- a/llvm/include/llvm/DebugInfo/Symbolize/MarkupFilter.h
+++ b/llvm/include/llvm/DebugInfo/Symbolize/MarkupFilter.h
@@ -39,7 +39,7 @@ public:
   ///
   /// Invalid or unimplemented markup elements are removed. Some output may be
   /// deferred until future filter() or finish() call.
-  void filter(StringRef Line);
+  void filter(std::string &&InputLine);
 
   /// Records that the input stream has ended and writes any deferred output.
   void finish();
@@ -142,7 +142,7 @@ private:
   MarkupParser Parser;
 
   // Current line being filtered.
-  StringRef Line;
+  std::string Line;
 
   // A module info line currently being built. This incorporates as much mmap
   // information as possible before being emitted.
diff --git a/llvm/include/llvm/IR/Attributes.td b/llvm/include/llvm/IR/Attributes.td
index fc38e68ad273b6b1527f48f0d1c698ed1401e7af..864f87f3383891176f73563a75c982b54ccc93dd 100644
--- a/llvm/include/llvm/IR/Attributes.td
+++ b/llvm/include/llvm/IR/Attributes.td
@@ -151,6 +151,9 @@ def NoDuplicate : EnumAttr<"noduplicate", [FnAttr]>;
 /// Function does not deallocate memory.
 def NoFree : EnumAttr<"nofree", [FnAttr, ParamAttr]>;
 
+/// Argument is dead if the call unwinds.
+def DeadOnUnwind : EnumAttr<"dead_on_unwind", [ParamAttr]>;
+
 /// Disable implicit floating point insts.
 def NoImplicitFloat : EnumAttr<"noimplicitfloat", [FnAttr]>;
 
diff --git a/llvm/include/llvm/IR/IntrinsicsAArch64.td b/llvm/include/llvm/IR/IntrinsicsAArch64.td
index 83fc208e7f7eeec0875c94f2ea70984fca86e067..9088168b4c67b1ab733d0f77c43ee0b18f959119 100644
--- a/llvm/include/llvm/IR/IntrinsicsAArch64.td
+++ b/llvm/include/llvm/IR/IntrinsicsAArch64.td
@@ -1401,6 +1401,13 @@ class AdvSIMD_SVE_Reduce_Intrinsic
                llvm_anyvector_ty],
               [IntrNoMem]>;
 
+class AdvSIMD_SVE_V128_Reduce_Intrinsic
+  : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
+              [LLVMScalarOrSameVectorWidth<1, llvm_i1_ty>,
+               llvm_anyvector_ty],
+               [IntrNoMem]>;
+
+
 class AdvSIMD_SVE_SADDV_Reduce_Intrinsic
   : DefaultAttrsIntrinsic<[llvm_i64_ty],
               [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
@@ -1723,6 +1730,15 @@ def int_aarch64_sve_sqsub_x   : AdvSIMD_2VectorArg_Intrinsic;
 def int_aarch64_sve_uqadd_x   : AdvSIMD_2VectorArg_Intrinsic;
 def int_aarch64_sve_uqsub_x   : AdvSIMD_2VectorArg_Intrinsic;
 
+def int_aarch64_sve_orqv      : AdvSIMD_SVE_V128_Reduce_Intrinsic;
+def int_aarch64_sve_eorqv     : AdvSIMD_SVE_V128_Reduce_Intrinsic;
+def int_aarch64_sve_andqv     : AdvSIMD_SVE_V128_Reduce_Intrinsic;
+def int_aarch64_sve_smaxqv    : AdvSIMD_SVE_V128_Reduce_Intrinsic;
+def int_aarch64_sve_umaxqv    : AdvSIMD_SVE_V128_Reduce_Intrinsic;
+def int_aarch64_sve_sminqv    : AdvSIMD_SVE_V128_Reduce_Intrinsic;
+def int_aarch64_sve_uminqv    : AdvSIMD_SVE_V128_Reduce_Intrinsic;
+
+
 // Shifts
 
 def int_aarch64_sve_asr      : AdvSIMD_Pred2VectorArg_Intrinsic;
@@ -2033,6 +2049,11 @@ def int_aarch64_sve_fmaxv   : AdvSIMD_SVE_Reduce_Intrinsic;
 def int_aarch64_sve_fmaxnmv : AdvSIMD_SVE_Reduce_Intrinsic;
 def int_aarch64_sve_fminv   : AdvSIMD_SVE_Reduce_Intrinsic;
 def int_aarch64_sve_fminnmv : AdvSIMD_SVE_Reduce_Intrinsic;
+def int_aarch64_sve_addqv : AdvSIMD_SVE_V128_Reduce_Intrinsic;
+def int_aarch64_sve_fmaxnmqv : AdvSIMD_SVE_V128_Reduce_Intrinsic;
+def int_aarch64_sve_fminnmqv : AdvSIMD_SVE_V128_Reduce_Intrinsic;
+def int_aarch64_sve_fmaxqv : AdvSIMD_SVE_V128_Reduce_Intrinsic;
+def int_aarch64_sve_fminqv : AdvSIMD_SVE_V128_Reduce_Intrinsic;
 
 //
 // Floating-point conversions
diff --git a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
index bc9f99783d98f28d8d2c27b445b6d1b54a7d5b37..09e88152e65d2a41aa095b17b39d24daa586bf7a 100644
--- a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
+++ b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td
@@ -227,6 +227,45 @@ def int_amdgcn_s_sendmsg_rtn : Intrinsic <[llvm_anyint_ty], [llvm_i32_ty],
 def int_amdgcn_s_barrier : ClangBuiltin<"__builtin_amdgcn_s_barrier">,
   Intrinsic<[], [], [IntrNoMem, IntrHasSideEffects, IntrConvergent, IntrWillReturn, IntrNoCallback, IntrNoFree]>;
 
+def int_amdgcn_s_barrier_signal : ClangBuiltin<"__builtin_amdgcn_s_barrier_signal">,
+  Intrinsic<[], [llvm_i32_ty], [ImmArg<ArgIndex<0>>, IntrNoMem, IntrHasSideEffects, IntrConvergent, IntrWillReturn,
+                                IntrNoCallback, IntrNoFree]>;
+
+def int_amdgcn_s_barrier_signal_var : ClangBuiltin<"__builtin_amdgcn_s_barrier_signal_var">,
+  Intrinsic<[], [llvm_i32_ty], [IntrNoMem, IntrHasSideEffects, IntrConvergent, IntrWillReturn,
+                                IntrNoCallback, IntrNoFree]>;
+
+def int_amdgcn_s_barrier_signal_isfirst : ClangBuiltin<"__builtin_amdgcn_s_barrier_signal_isfirst">,
+  Intrinsic<[llvm_i1_ty], [llvm_i32_ty], [ImmArg<ArgIndex<0>>, IntrNoMem, IntrHasSideEffects, IntrConvergent,
+                                IntrWillReturn, IntrNoCallback, IntrNoFree]>;
+
+def int_amdgcn_s_barrier_signal_isfirst_var : ClangBuiltin<"__builtin_amdgcn_s_barrier_signal_isfirst_var">,
+  Intrinsic<[llvm_i1_ty], [llvm_i32_ty], [IntrNoMem, IntrHasSideEffects, IntrConvergent, IntrWillReturn,
+                                IntrNoCallback, IntrNoFree]>;
+
+def int_amdgcn_s_barrier_init : ClangBuiltin<"__builtin_amdgcn_s_barrier_init">,
+  Intrinsic<[], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem, IntrHasSideEffects, IntrConvergent,
+                                IntrWillReturn, IntrNoCallback, IntrNoFree]>;
+
+def int_amdgcn_s_barrier_join : ClangBuiltin<"__builtin_amdgcn_s_barrier_join">,
+  Intrinsic<[], [llvm_i32_ty], [IntrNoMem, IntrHasSideEffects, IntrConvergent, IntrWillReturn,
+                                IntrNoCallback, IntrNoFree]>;
+
+def int_amdgcn_s_wakeup_barrier : ClangBuiltin<"__builtin_amdgcn_s_wakeup_barrier">,
+  Intrinsic<[], [llvm_i32_ty], [IntrNoMem, IntrHasSideEffects, IntrConvergent, IntrWillReturn,
+                                IntrNoCallback, IntrNoFree]>;
+
+def int_amdgcn_s_barrier_wait : ClangBuiltin<"__builtin_amdgcn_s_barrier_wait">,
+  Intrinsic<[], [llvm_i16_ty], [ImmArg<ArgIndex<0>>, IntrNoMem, IntrHasSideEffects, IntrConvergent,
+                                IntrWillReturn, IntrNoCallback, IntrNoFree]>;
+
+def int_amdgcn_s_barrier_leave : ClangBuiltin<"__builtin_amdgcn_s_barrier_leave">,
+  Intrinsic<[llvm_i1_ty], [], [IntrNoMem, IntrHasSideEffects, IntrConvergent, IntrWillReturn, IntrNoCallback, IntrNoFree]>;
+
+def int_amdgcn_s_get_barrier_state : ClangBuiltin<"__builtin_amdgcn_s_get_barrier_state">,
+  Intrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrNoMem, IntrHasSideEffects, IntrConvergent, IntrWillReturn,
+                                IntrNoCallback, IntrNoFree]>;
+
 def int_amdgcn_wave_barrier : ClangBuiltin<"__builtin_amdgcn_wave_barrier">,
   Intrinsic<[], [], [IntrNoMem, IntrHasSideEffects, IntrConvergent, IntrWillReturn, IntrNoCallback, IntrNoFree]>;
 
diff --git a/llvm/include/llvm/InitializePasses.h b/llvm/include/llvm/InitializePasses.h
index 66177f9b9f774f488badde4e43f97e284fe232cc..46b1e95c3c15f366cb72053cbed962fd3dd853d1 100644
--- a/llvm/include/llvm/InitializePasses.h
+++ b/llvm/include/llvm/InitializePasses.h
@@ -103,7 +103,7 @@ void initializeEdgeBundlesPass(PassRegistry&);
 void initializeEHContGuardCatchretPass(PassRegistry &);
 void initializeExpandLargeFpConvertLegacyPassPass(PassRegistry&);
 void initializeExpandLargeDivRemLegacyPassPass(PassRegistry&);
-void initializeExpandMemCmpPassPass(PassRegistry&);
+void initializeExpandMemCmpLegacyPassPass(PassRegistry &);
 void initializeExpandPostRAPass(PassRegistry&);
 void initializeExpandReductionsPass(PassRegistry&);
 void initializeExpandVectorPredicationPass(PassRegistry &);
@@ -129,7 +129,7 @@ void initializeIVUsersWrapperPassPass(PassRegistry&);
 void initializeIfConverterPass(PassRegistry&);
 void initializeImmutableModuleSummaryIndexWrapperPassPass(PassRegistry&);
 void initializeImplicitNullChecksPass(PassRegistry&);
-void initializeIndirectBrExpandPassPass(PassRegistry&);
+void initializeIndirectBrExpandLegacyPassPass(PassRegistry &);
 void initializeInferAddressSpacesPass(PassRegistry&);
 void initializeInstSimplifyLegacyPassPass(PassRegistry &);
 void initializeInstructionCombiningPassPass(PassRegistry&);
diff --git a/llvm/include/llvm/LinkAllPasses.h b/llvm/include/llvm/LinkAllPasses.h
index bf990a1408faba467bedd63eeffc7bffafd222ea..7a21876e565a7c06d9ebfcf4d855bee3fd399e83 100644
--- a/llvm/include/llvm/LinkAllPasses.h
+++ b/llvm/include/llvm/LinkAllPasses.h
@@ -119,7 +119,7 @@ namespace {
       (void) llvm::createPostDomTree();
       (void) llvm::createMergeICmpsLegacyPass();
       (void) llvm::createExpandLargeDivRemPass();
-      (void) llvm::createExpandMemCmpPass();
+      (void)llvm::createExpandMemCmpLegacyPass();
       (void) llvm::createExpandVectorPredicationPass();
       std::string buf;
       llvm::raw_string_ostream os(buf);
@@ -148,8 +148,7 @@ namespace {
       llvm::AliasAnalysis AA(TLI);
       llvm::BatchAAResults BAA(AA);
       llvm::AliasSetTracker X(BAA);
-      X.add(nullptr, llvm::LocationSize::beforeOrAfterPointer(),
-            llvm::AAMDNodes()); // for -print-alias-sets
+      X.add(llvm::MemoryLocation()); // for -print-alias-sets
       (void) llvm::AreStatisticsEnabled();
       (void) llvm::sys::RunningOnValgrind();
     }
diff --git a/llvm/include/llvm/MC/MCFragment.h b/llvm/include/llvm/MC/MCFragment.h
index c314fdd3aa69730dfa70cb277915f926ef3f400a..a9b19dc56f16a5e9dd09b97be5a1f57a49047a51 100644
--- a/llvm/include/llvm/MC/MCFragment.h
+++ b/llvm/include/llvm/MC/MCFragment.h
@@ -428,7 +428,7 @@ public:
   }
 };
 
-class MCLEBFragment final : public MCEncodedFragmentWithFixups<10, 1> {
+class MCLEBFragment final : public MCEncodedFragmentWithFixups<8, 0> {
   /// True if this is a sleb128, false if uleb128.
   bool IsSigned;
 
@@ -437,7 +437,7 @@ class MCLEBFragment final : public MCEncodedFragmentWithFixups<10, 1> {
 
 public:
   MCLEBFragment(const MCExpr &Value, bool IsSigned, MCSection *Sec = nullptr)
-      : MCEncodedFragmentWithFixups<10, 1>(FT_LEB, false, Sec),
+      : MCEncodedFragmentWithFixups<8, 0>(FT_LEB, false, Sec),
         IsSigned(IsSigned), Value(&Value) {
     getContents().push_back(0);
   }
diff --git a/llvm/include/llvm/ObjectYAML/GOFFYAML.h b/llvm/include/llvm/ObjectYAML/GOFFYAML.h
new file mode 100644
index 0000000000000000000000000000000000000000..f9bf45e95bd3a46dcaadea9cee6e45da9b94136b
--- /dev/null
+++ b/llvm/include/llvm/ObjectYAML/GOFFYAML.h
@@ -0,0 +1,49 @@
+//===- GOFFYAML.h - GOFF YAMLIO implementation ------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file declares classes for handling the YAML representation of GOFF.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_OBJECTYAML_GOFFYAML_H
+#define LLVM_OBJECTYAML_GOFFYAML_H
+
+#include "llvm/ADT/StringRef.h"
+#include "llvm/BinaryFormat/GOFF.h"
+#include "llvm/ObjectYAML/YAML.h"
+#include <cstdint>
+#include <vector>
+
+namespace llvm {
+
+// The structure of the yaml files is not an exact 1:1 match to GOFF. In order
+// to use yaml::IO, we use these structures which are closer to the source.
+namespace GOFFYAML {
+
+struct FileHeader {
+  uint32_t TargetEnvironment = 0;
+  uint32_t TargetOperatingSystem = 0;
+  uint16_t CCSID = 0;
+  StringRef CharacterSetName;
+  StringRef LanguageProductIdentifier;
+  uint32_t ArchitectureLevel = 0;
+  std::optional<uint16_t> InternalCCSID;
+  std::optional<uint8_t> TargetSoftwareEnvironment;
+};
+
+struct Object {
+  FileHeader Header;
+  Object();
+};
+} // end namespace GOFFYAML
+} // end namespace llvm
+
+LLVM_YAML_DECLARE_MAPPING_TRAITS(GOFFYAML::FileHeader)
+LLVM_YAML_DECLARE_MAPPING_TRAITS(GOFFYAML::Object)
+
+#endif // LLVM_OBJECTYAML_GOFFYAML_H
diff --git a/llvm/include/llvm/ObjectYAML/ObjectYAML.h b/llvm/include/llvm/ObjectYAML/ObjectYAML.h
index b63607e6796b0b74cb8e06066609dbd9fcd60762..7fd15cf290f4a4b0a78105b99a55fb2c28f7e185 100644
--- a/llvm/include/llvm/ObjectYAML/ObjectYAML.h
+++ b/llvm/include/llvm/ObjectYAML/ObjectYAML.h
@@ -13,6 +13,7 @@
 #include "llvm/ObjectYAML/COFFYAML.h"
 #include "llvm/ObjectYAML/DXContainerYAML.h"
 #include "llvm/ObjectYAML/ELFYAML.h"
+#include "llvm/ObjectYAML/GOFFYAML.h"
 #include "llvm/ObjectYAML/MachOYAML.h"
 #include "llvm/ObjectYAML/MinidumpYAML.h"
 #include "llvm/ObjectYAML/OffloadYAML.h"
@@ -30,6 +31,7 @@ struct YamlObjectFile {
   std::unique_ptr<ArchYAML::Archive> Arch;
   std::unique_ptr<ELFYAML::Object> Elf;
   std::unique_ptr<COFFYAML::Object> Coff;
+  std::unique_ptr<GOFFYAML::Object> Goff;
   std::unique_ptr<MachOYAML::Object> MachO;
   std::unique_ptr<MachOYAML::UniversalBinary> FatMachO;
   std::unique_ptr<MinidumpYAML::Object> Minidump;
diff --git a/llvm/include/llvm/ObjectYAML/yaml2obj.h b/llvm/include/llvm/ObjectYAML/yaml2obj.h
index 000da077bb18c6a6eba13d022deea8705d89e923..3b458c3cd890b121ea68f4ac652bc13fb27d5784 100644
--- a/llvm/include/llvm/ObjectYAML/yaml2obj.h
+++ b/llvm/include/llvm/ObjectYAML/yaml2obj.h
@@ -32,6 +32,10 @@ namespace ELFYAML {
 struct Object;
 }
 
+namespace GOFFYAML {
+struct Object;
+}
+
 namespace MinidumpYAML {
 struct Object;
 }
@@ -64,6 +68,7 @@ using ErrorHandler = llvm::function_ref<void(const Twine &Msg)>;
 
 bool yaml2archive(ArchYAML::Archive &Doc, raw_ostream &Out, ErrorHandler EH);
 bool yaml2coff(COFFYAML::Object &Doc, raw_ostream &Out, ErrorHandler EH);
+bool yaml2goff(GOFFYAML::Object &Doc, raw_ostream &Out, ErrorHandler EH);
 bool yaml2elf(ELFYAML::Object &Doc, raw_ostream &Out, ErrorHandler EH,
               uint64_t MaxSize);
 bool yaml2macho(YamlObjectFile &Doc, raw_ostream &Out, ErrorHandler EH);
diff --git a/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h b/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h
index e9fbc4631dd5fb60579df6f4401711a0dcc3ce45..493689f6a61e7ef372fb4da9ceaffcae0b8d9aea 100644
--- a/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h
+++ b/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h
@@ -15,6 +15,7 @@
 #define LLVM_PROFILEDATA_COVERAGE_COVERAGEMAPPING_H
 
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/BitVector.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/Hashing.h"
@@ -33,6 +34,7 @@
 #include <cstdint>
 #include <iterator>
 #include <memory>
+#include <sstream>
 #include <string>
 #include <system_error>
 #include <tuple>
@@ -237,7 +239,27 @@ struct CounterMappingRegion {
     /// A BranchRegion represents leaf-level boolean expressions and is
     /// associated with two counters, each representing the number of times the
     /// expression evaluates to true or false.
-    BranchRegion
+    BranchRegion,
+
+    /// A DecisionRegion represents a top-level boolean expression and is
+    /// associated with a variable length bitmap index and condition number.
+    MCDCDecisionRegion,
+
+    /// A Branch Region can be extended to include IDs to facilitate MC/DC.
+    MCDCBranchRegion
+  };
+
+  using MCDCConditionID = unsigned int;
+  struct MCDCParameters {
+    /// Byte Index of Bitmap Coverage Object for a Decision Region.
+    unsigned BitmapIdx = 0;
+
+    /// Number of Conditions used for a Decision Region.
+    unsigned NumConditions = 0;
+
+    /// IDs used to represent a branch region and other branch regions
+    /// evaluated based on True and False branches.
+    MCDCConditionID ID = 0, TrueID = 0, FalseID = 0;
   };
 
   /// Primary Counter that is also used for Branch Regions (TrueCount).
@@ -246,8 +268,13 @@ struct CounterMappingRegion {
   /// Secondary Counter used for Branch Regions (FalseCount).
   Counter FalseCount;
 
-  unsigned FileID, ExpandedFileID;
+  /// Parameters used for Modified Condition/Decision Coverage
+  MCDCParameters MCDCParams;
+
+  unsigned FileID = 0;
+  unsigned ExpandedFileID = 0;
   unsigned LineStart, ColumnStart, LineEnd, ColumnEnd;
+
   RegionKind Kind;
 
   CounterMappingRegion(Counter Count, unsigned FileID, unsigned ExpandedFileID,
@@ -257,15 +284,24 @@ struct CounterMappingRegion {
         LineStart(LineStart), ColumnStart(ColumnStart), LineEnd(LineEnd),
         ColumnEnd(ColumnEnd), Kind(Kind) {}
 
-  CounterMappingRegion(Counter Count, Counter FalseCount, unsigned FileID,
+  CounterMappingRegion(Counter Count, Counter FalseCount,
+                       MCDCParameters MCDCParams, unsigned FileID,
                        unsigned ExpandedFileID, unsigned LineStart,
                        unsigned ColumnStart, unsigned LineEnd,
                        unsigned ColumnEnd, RegionKind Kind)
-      : Count(Count), FalseCount(FalseCount), FileID(FileID),
-        ExpandedFileID(ExpandedFileID), LineStart(LineStart),
+      : Count(Count), FalseCount(FalseCount), MCDCParams(MCDCParams),
+        FileID(FileID), ExpandedFileID(ExpandedFileID), LineStart(LineStart),
         ColumnStart(ColumnStart), LineEnd(LineEnd), ColumnEnd(ColumnEnd),
         Kind(Kind) {}
 
+  CounterMappingRegion(MCDCParameters MCDCParams, unsigned FileID,
+                       unsigned ExpandedFileID, unsigned LineStart,
+                       unsigned ColumnStart, unsigned LineEnd,
+                       unsigned ColumnEnd, RegionKind Kind)
+      : MCDCParams(MCDCParams), ExpandedFileID(ExpandedFileID),
+        LineStart(LineStart), ColumnStart(ColumnStart), LineEnd(LineEnd),
+        ColumnEnd(ColumnEnd), Kind(Kind) {}
+
   static CounterMappingRegion
   makeRegion(Counter Count, unsigned FileID, unsigned LineStart,
              unsigned ColumnStart, unsigned LineEnd, unsigned ColumnEnd) {
@@ -299,8 +335,27 @@ struct CounterMappingRegion {
   makeBranchRegion(Counter Count, Counter FalseCount, unsigned FileID,
                    unsigned LineStart, unsigned ColumnStart, unsigned LineEnd,
                    unsigned ColumnEnd) {
-    return CounterMappingRegion(Count, FalseCount, FileID, 0, LineStart,
-                                ColumnStart, LineEnd, ColumnEnd, BranchRegion);
+    return CounterMappingRegion(Count, FalseCount, MCDCParameters(), FileID, 0,
+                                LineStart, ColumnStart, LineEnd, ColumnEnd,
+                                BranchRegion);
+  }
+
+  static CounterMappingRegion
+  makeBranchRegion(Counter Count, Counter FalseCount, MCDCParameters MCDCParams,
+                   unsigned FileID, unsigned LineStart, unsigned ColumnStart,
+                   unsigned LineEnd, unsigned ColumnEnd) {
+    return CounterMappingRegion(Count, FalseCount, MCDCParams, FileID, 0,
+                                LineStart, ColumnStart, LineEnd, ColumnEnd,
+                                MCDCParams.ID == 0 ? BranchRegion
+                                                   : MCDCBranchRegion);
+  }
+
+  static CounterMappingRegion
+  makeDecisionRegion(MCDCParameters MCDCParams, unsigned FileID,
+                     unsigned LineStart, unsigned ColumnStart, unsigned LineEnd,
+                     unsigned ColumnEnd) {
+    return CounterMappingRegion(MCDCParams, FileID, 0, LineStart, ColumnStart,
+                                LineEnd, ColumnEnd, MCDCDecisionRegion);
   }
 
   inline LineColPair startLoc() const {
@@ -326,11 +381,189 @@ struct CountedRegion : public CounterMappingRegion {
         FalseExecutionCount(FalseExecutionCount), Folded(false) {}
 };
 
+/// MCDC Record grouping all information together.
+struct MCDCRecord {
+  /// CondState represents the evaluation of a condition in an executed test
+  /// vector, which can be True or False. A DontCare is used to mask an
+  /// unevaluatable condition resulting from short-circuit behavior of logical
+  /// operators in languages like C/C++. When comparing the evaluation of a
+  /// condition across executed test vectors, comparisons against a DontCare
+  /// are effectively ignored.
+  enum CondState { MCDC_DontCare = -1, MCDC_False = 0, MCDC_True = 1 };
+
+  using TestVector = llvm::SmallVector<CondState>;
+  using TestVectors = llvm::SmallVector<TestVector>;
+  using BoolVector = llvm::SmallVector<bool>;
+  using TVRowPair = std::pair<unsigned, unsigned>;
+  using TVPairMap = llvm::DenseMap<unsigned, TVRowPair>;
+  using CondIDMap = llvm::DenseMap<unsigned, unsigned>;
+  using LineColPairMap = llvm::DenseMap<unsigned, LineColPair>;
+
+private:
+  CounterMappingRegion Region;
+  TestVectors TV;
+  TVPairMap IndependencePairs;
+  BoolVector Folded;
+  CondIDMap PosToID;
+  LineColPairMap CondLoc;
+
+public:
+  MCDCRecord(CounterMappingRegion Region, TestVectors TV,
+             TVPairMap IndependencePairs, BoolVector Folded, CondIDMap PosToID,
+             LineColPairMap CondLoc)
+      : Region(Region), TV(TV), IndependencePairs(IndependencePairs),
+        Folded(Folded), PosToID(PosToID), CondLoc(CondLoc){};
+
+  CounterMappingRegion getDecisionRegion() const { return Region; }
+  unsigned getNumConditions() const {
+    assert(Region.MCDCParams.NumConditions != 0 &&
+           "In MC/DC, NumConditions should never be zero!");
+    return Region.MCDCParams.NumConditions;
+  }
+  unsigned getNumTestVectors() const { return TV.size(); }
+  bool isCondFolded(unsigned Condition) const { return Folded[Condition]; }
+
+  /// Return the evaluation of a condition (indicated by Condition) in an
+  /// executed test vector (indicated by TestVectorIndex), which will be True,
+  /// False, or DontCare if the condition is unevaluatable. Because condition
+  /// IDs are not associated based on their position in the expression,
+  /// accessing conditions in the TestVectors requires a translation from a
+  /// ordinal position to actual condition ID. This is done via PosToID[].
+  CondState getTVCondition(unsigned TestVectorIndex, unsigned Condition) {
+    return TV[TestVectorIndex][PosToID[Condition]];
+  }
+
+  /// Return the Result evaluation for an executed test vector.
+  /// See MCDCRecordProcessor::RecordTestVector().
+  CondState getTVResult(unsigned TestVectorIndex) {
+    return TV[TestVectorIndex][getNumConditions()];
+  }
+
+  /// Determine whether a given condition (indicated by Condition) is covered
+  /// by an Independence Pair. Because condition IDs are not associated based
+  /// on their position in the expression, accessing conditions in the
+  /// TestVectors requires a translation from a ordinal position to actual
+  /// condition ID. This is done via PosToID[].
+  bool isConditionIndependencePairCovered(unsigned Condition) const {
+    auto It = PosToID.find(Condition);
+    if (It != PosToID.end())
+      return (IndependencePairs.find(It->second) != IndependencePairs.end());
+    llvm_unreachable("Condition ID without an Ordinal mapping");
+  }
+
+  /// Return the Independence Pair that covers the given condition. Because
+  /// condition IDs are not associated based on their position in the
+  /// expression, accessing conditions in the TestVectors requires a
+  /// translation from a ordinal position to actual condition ID. This is done
+  /// via PosToID[].
+  TVRowPair getConditionIndependencePair(unsigned Condition) {
+    assert(isConditionIndependencePairCovered(Condition));
+    return IndependencePairs[PosToID[Condition]];
+  }
+
+  float getPercentCovered() const {
+    unsigned Folded = 0;
+    unsigned Covered = 0;
+    for (unsigned C = 0; C < getNumConditions(); C++) {
+      if (isCondFolded(C))
+        Folded++;
+      else if (isConditionIndependencePairCovered(C))
+        Covered++;
+    }
+
+    unsigned Total = getNumConditions() - Folded;
+    if (Total == 0)
+      return 0.0;
+    return (static_cast<double>(Covered) / static_cast<double>(Total)) * 100.0;
+  }
+
+  std::string getConditionHeaderString(unsigned Condition) {
+    std::ostringstream OS;
+    OS << "Condition C" << Condition + 1 << " --> (";
+    OS << CondLoc[Condition].first << ":" << CondLoc[Condition].second;
+    OS << ")\n";
+    return OS.str();
+  }
+
+  std::string getTestVectorHeaderString() const {
+    std::ostringstream OS;
+    if (getNumTestVectors() == 0) {
+      OS << "None.\n";
+      return OS.str();
+    }
+    const auto NumConditions = getNumConditions();
+    for (unsigned I = 0; I < NumConditions; I++) {
+      OS << "C" << I + 1;
+      if (I != NumConditions - 1)
+        OS << ", ";
+    }
+    OS << "    Result\n";
+    return OS.str();
+  }
+
+  std::string getTestVectorString(unsigned TestVectorIndex) {
+    assert(TestVectorIndex < getNumTestVectors() &&
+           "TestVector index out of bounds!");
+    std::ostringstream OS;
+    const auto NumConditions = getNumConditions();
+    // Add individual condition values to the string.
+    OS << "  " << TestVectorIndex + 1 << " { ";
+    for (unsigned Condition = 0; Condition < NumConditions; Condition++) {
+      if (isCondFolded(Condition))
+        OS << "C";
+      else {
+        switch (getTVCondition(TestVectorIndex, Condition)) {
+        case MCDCRecord::MCDC_DontCare:
+          OS << "-";
+          break;
+        case MCDCRecord::MCDC_True:
+          OS << "T";
+          break;
+        case MCDCRecord::MCDC_False:
+          OS << "F";
+          break;
+        }
+      }
+      if (Condition != NumConditions - 1)
+        OS << ",  ";
+    }
+
+    // Add result value to the string.
+    OS << "  = ";
+    if (getTVResult(TestVectorIndex) == MCDC_True)
+      OS << "T";
+    else
+      OS << "F";
+    OS << "      }\n";
+
+    return OS.str();
+  }
+
+  std::string getConditionCoverageString(unsigned Condition) {
+    assert(Condition < getNumConditions() &&
+           "Condition index is out of bounds!");
+    std::ostringstream OS;
+
+    OS << "  C" << Condition + 1 << "-Pair: ";
+    if (isCondFolded(Condition)) {
+      OS << "constant folded\n";
+    } else if (isConditionIndependencePairCovered(Condition)) {
+      TVRowPair rows = getConditionIndependencePair(Condition);
+      OS << "covered: (" << rows.first << ",";
+      OS << rows.second << ")\n";
+    } else
+      OS << "not covered\n";
+
+    return OS.str();
+  }
+};
+
 /// A Counter mapping context is used to connect the counters, expressions
 /// and the obtained counter values.
 class CounterMappingContext {
   ArrayRef<CounterExpression> Expressions;
   ArrayRef<uint64_t> CounterValues;
+  ArrayRef<uint8_t> BitmapBytes;
 
 public:
   CounterMappingContext(ArrayRef<CounterExpression> Expressions,
@@ -338,6 +571,7 @@ public:
       : Expressions(Expressions), CounterValues(CounterValues) {}
 
   void setCounts(ArrayRef<uint64_t> Counts) { CounterValues = Counts; }
+  void setBitmapBytes(ArrayRef<uint8_t> Bytes) { BitmapBytes = Bytes; }
 
   void dump(const Counter &C, raw_ostream &OS) const;
   void dump(const Counter &C) const { dump(C, dbgs()); }
@@ -346,6 +580,17 @@ public:
   /// counter was executed.
   Expected<int64_t> evaluate(const Counter &C) const;
 
+  /// Return the number of times that a region of code associated with this
+  /// counter was executed.
+  Expected<BitVector>
+  evaluateBitmap(const CounterMappingRegion *MCDCDecision) const;
+
+  /// Return an MCDC record that indicates executed test vectors and condition
+  /// pairs.
+  Expected<MCDCRecord>
+  evaluateMCDCRegion(CounterMappingRegion Region, BitVector Bitmap,
+                     ArrayRef<CounterMappingRegion> Branches);
+
   unsigned getMaxCounterID(const Counter &C) const;
 };
 
@@ -364,6 +609,8 @@ struct FunctionRecord {
   std::vector<CountedRegion> CountedRegions;
   /// Branch Regions in the function along with their counts.
   std::vector<CountedRegion> CountedBranchRegions;
+  /// MCDC Records record a DecisionRegion and associated BranchRegions.
+  std::vector<MCDCRecord> MCDCRecords;
   /// The number of times this function was executed.
   uint64_t ExecutionCount = 0;
 
@@ -373,9 +620,12 @@ struct FunctionRecord {
   FunctionRecord(FunctionRecord &&FR) = default;
   FunctionRecord &operator=(FunctionRecord &&) = default;
 
+  void pushMCDCRecord(MCDCRecord Record) { MCDCRecords.push_back(Record); }
+
   void pushRegion(CounterMappingRegion Region, uint64_t Count,
                   uint64_t FalseCount) {
-    if (Region.Kind == CounterMappingRegion::BranchRegion) {
+    if (Region.Kind == CounterMappingRegion::BranchRegion ||
+        Region.Kind == CounterMappingRegion::MCDCBranchRegion) {
       CountedBranchRegions.emplace_back(Region, Count, FalseCount);
       // If both counters are hard-coded to zero, then this region represents a
       // constant-folded branch.
@@ -546,6 +796,7 @@ class CoverageData {
   std::vector<CoverageSegment> Segments;
   std::vector<ExpansionRecord> Expansions;
   std::vector<CountedRegion> BranchRegions;
+  std::vector<MCDCRecord> MCDCRecords;
 
 public:
   CoverageData() = default;
@@ -572,6 +823,9 @@ public:
 
   /// Branches that can be further processed.
   ArrayRef<CountedRegion> getBranches() const { return BranchRegions; }
+
+  /// MCDC Records that can be further processed.
+  ArrayRef<MCDCRecord> getMCDCRecords() const { return MCDCRecords; }
 };
 
 /// The mapping of profile information to coverage data.
diff --git a/llvm/include/llvm/Support/AMDHSAKernelDescriptor.h b/llvm/include/llvm/Support/AMDHSAKernelDescriptor.h
index 0574f96e6e15c4ff622539ef946e22696fa48cd9..2de2cf4185d86e65f764eb9deff5891f54ab92ad 100644
--- a/llvm/include/llvm/Support/AMDHSAKernelDescriptor.h
+++ b/llvm/include/llvm/Support/AMDHSAKernelDescriptor.h
@@ -88,12 +88,18 @@ enum : uint8_t {
 // [GFX6-GFX9].
 #define COMPUTE_PGM_RSRC1_GFX6_GFX9(NAME, SHIFT, WIDTH) \
   AMDHSA_BITS_ENUM_ENTRY(COMPUTE_PGM_RSRC1_GFX6_GFX9_ ## NAME, SHIFT, WIDTH)
+// [GFX6-GFX11].
+#define COMPUTE_PGM_RSRC1_GFX6_GFX11(NAME, SHIFT, WIDTH)                       \
+  AMDHSA_BITS_ENUM_ENTRY(COMPUTE_PGM_RSRC1_GFX6_GFX11_##NAME, SHIFT, WIDTH)
 // GFX9+.
 #define COMPUTE_PGM_RSRC1_GFX9_PLUS(NAME, SHIFT, WIDTH) \
   AMDHSA_BITS_ENUM_ENTRY(COMPUTE_PGM_RSRC1_GFX9_PLUS_ ## NAME, SHIFT, WIDTH)
 // GFX10+.
 #define COMPUTE_PGM_RSRC1_GFX10_PLUS(NAME, SHIFT, WIDTH) \
   AMDHSA_BITS_ENUM_ENTRY(COMPUTE_PGM_RSRC1_GFX10_PLUS_ ## NAME, SHIFT, WIDTH)
+// GFX12+.
+#define COMPUTE_PGM_RSRC1_GFX12_PLUS(NAME, SHIFT, WIDTH)                       \
+  AMDHSA_BITS_ENUM_ENTRY(COMPUTE_PGM_RSRC1_GFX12_PLUS_##NAME, SHIFT, WIDTH)
 enum : int32_t {
   COMPUTE_PGM_RSRC1(GRANULATED_WORKITEM_VGPR_COUNT, 0, 6),
   COMPUTE_PGM_RSRC1(GRANULATED_WAVEFRONT_SGPR_COUNT, 6, 4),
@@ -103,9 +109,11 @@ enum : int32_t {
   COMPUTE_PGM_RSRC1(FLOAT_DENORM_MODE_32, 16, 2),
   COMPUTE_PGM_RSRC1(FLOAT_DENORM_MODE_16_64, 18, 2),
   COMPUTE_PGM_RSRC1(PRIV, 20, 1),
-  COMPUTE_PGM_RSRC1(ENABLE_DX10_CLAMP, 21, 1),
+  COMPUTE_PGM_RSRC1_GFX6_GFX11(ENABLE_DX10_CLAMP, 21, 1),
+  COMPUTE_PGM_RSRC1_GFX12_PLUS(ENABLE_WG_RR_EN, 21, 1),
   COMPUTE_PGM_RSRC1(DEBUG_MODE, 22, 1),
-  COMPUTE_PGM_RSRC1(ENABLE_IEEE_MODE, 23, 1),
+  COMPUTE_PGM_RSRC1_GFX6_GFX11(ENABLE_IEEE_MODE, 23, 1),
+  COMPUTE_PGM_RSRC1_GFX12_PLUS(DISABLE_PERF, 23, 1),
   COMPUTE_PGM_RSRC1(BULKY, 24, 1),
   COMPUTE_PGM_RSRC1(CDBG_USER, 25, 1),
   COMPUTE_PGM_RSRC1_GFX6_GFX8(RESERVED0, 26, 1),
diff --git a/llvm/include/llvm/Support/AutoConvert.h b/llvm/include/llvm/Support/AutoConvert.h
index bcf7473feac8f77e45a1e21cb8bfe14dd5ed6902..79566189d819760abbdf4c2c8e666a31e41abb02 100644
--- a/llvm/include/llvm/Support/AutoConvert.h
+++ b/llvm/include/llvm/Support/AutoConvert.h
@@ -15,10 +15,26 @@
 #define LLVM_SUPPORT_AUTOCONVERT_H
 
 #ifdef __MVS__
-#define CCSID_IBM_1047 1047
-#define CCSID_UTF_8 1208
+#include <_Ccsid.h>
+#ifdef __cplusplus
 #include <system_error>
+#endif // __cplusplus
 
+#define CCSID_IBM_1047 1047
+#define CCSID_UTF_8 1208
+#define CCSID_ISO8859_1 819
+
+#ifdef __cplusplus
+extern "C" {
+#endif // __cplusplus
+int enableAutoConversion(int FD);
+int disableAutoConversion(int FD);
+int restoreStdHandleAutoConversion(int FD);
+#ifdef __cplusplus
+}
+#endif // __cplusplus
+
+#ifdef __cplusplus
 namespace llvm {
 
 /// \brief Disable the z/OS enhanced ASCII auto-conversion for the file
@@ -30,10 +46,14 @@ std::error_code disableAutoConversion(int FD);
 /// codepage.
 std::error_code enableAutoConversion(int FD);
 
+/// Restore the z/OS enhanced ASCII auto-conversion for the std handle.
+std::error_code restoreStdHandleAutoConversion(int FD);
+
 /// \brief Set the tag information for a file descriptor.
 std::error_code setFileTag(int FD, int CCSID, bool Text);
 
 } // namespace llvm
+#endif // __cplusplus
 
 #endif // __MVS__
 
diff --git a/llvm/include/llvm/Support/SystemZ/zOSSupport.h b/llvm/include/llvm/Support/SystemZ/zOSSupport.h
new file mode 100644
index 0000000000000000000000000000000000000000..f9a61f887d5dd13f14eac946e4120d492312b9b1
--- /dev/null
+++ b/llvm/include/llvm/Support/SystemZ/zOSSupport.h
@@ -0,0 +1,47 @@
+//===- zOSSupport.h - Common z/OS Include File ------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines z/OS implementations for common functions.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_SUPPORT_ZOSSUPPORT_H
+#define LLVM_SUPPORT_ZOSSUPPORT_H
+
+#ifdef __MVS__
+#include <sys/resource.h>
+#include <sys/wait.h>
+
+// z/OS Unix System Services does not have strsignal() support, so the
+// strsignal() function is implemented here.
+inline char *strsignal(int sig) {
+  static char msg[256];
+  sprintf(msg, "%d", sig);
+  return msg;
+}
+
+// z/OS Unix System Services does not have wait4() support, so the wait4
+// function is implemented here.
+inline pid_t wait4(pid_t pid, int *wstatus, int options,
+                   struct rusage *rusage) {
+  pid_t Result = waitpid(pid, wstatus, options);
+  int GetrusageRC = getrusage(RUSAGE_CHILDREN, rusage);
+  assert(!GetrusageRC && "Must have valid measure of the resources!");
+  return Result;
+}
+
+// z/OS Unix System Services does not have strnlen() support, so the strnlen()
+// function is implemented here.
+inline std::size_t strnlen(const char *S, std::size_t MaxLen) {
+  const char *PtrToNullChar =
+      static_cast<const char *>(std::memchr(S, '\0', MaxLen));
+  return PtrToNullChar ? PtrToNullChar - S : MaxLen;
+}
+
+#endif
+#endif
diff --git a/llvm/include/llvm/Support/raw_ostream.h b/llvm/include/llvm/Support/raw_ostream.h
index 1e01eb9ea19c4187302a91457b6d34fbe5b67584..7c8d264afeff2fcdda59fe37d7534a88a9a122b1 100644
--- a/llvm/include/llvm/Support/raw_ostream.h
+++ b/llvm/include/llvm/Support/raw_ostream.h
@@ -16,6 +16,7 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/DataTypes.h"
+#include "llvm/Support/Threading.h"
 #include <cassert>
 #include <cstddef>
 #include <cstdint>
@@ -615,6 +616,8 @@ public:
   /// immediately destroyed.
   raw_fd_stream(StringRef Filename, std::error_code &EC);
 
+  raw_fd_stream(int fd, bool shouldClose);
+
   /// This reads the \p Size bytes into a buffer pointed by \p Ptr.
   ///
   /// \param Ptr The start of the buffer to hold data to be read.
@@ -630,6 +633,54 @@ public:
   static bool classof(const raw_ostream *OS);
 };
 
+//===----------------------------------------------------------------------===//
+// Socket Streams
+//===----------------------------------------------------------------------===//
+
+/// A raw stream for sockets reading/writing
+
+class raw_socket_stream;
+
+// Make sure that calls to WSAStartup and WSACleanup are balanced.
+#ifdef _WIN32
+class WSABalancer {
+public:
+  WSABalancer();
+  ~WSABalancer();
+};
+#endif // _WIN32
+
+class ListeningSocket {
+  int FD;
+  std::string SocketPath;
+  ListeningSocket(int SocketFD, StringRef SocketPath);
+#ifdef _WIN32
+  WSABalancer _;
+#endif // _WIN32
+
+public:
+  static Expected<ListeningSocket> createUnix(
+      StringRef SocketPath,
+      int MaxBacklog = llvm::hardware_concurrency().compute_thread_count());
+  Expected<std::unique_ptr<raw_socket_stream>> accept();
+  ListeningSocket(ListeningSocket &&LS);
+  ~ListeningSocket();
+};
+class raw_socket_stream : public raw_fd_stream {
+  uint64_t current_pos() const override { return 0; }
+#ifdef _WIN32
+  WSABalancer _;
+#endif // _WIN32
+
+public:
+  raw_socket_stream(int SocketFD);
+  /// Create a \p raw_socket_stream connected to the Unix domain socket at \p
+  /// SocketPath.
+  static Expected<std::unique_ptr<raw_socket_stream>>
+  createConnectedUnix(StringRef SocketPath);
+  ~raw_socket_stream();
+};
+
 //===----------------------------------------------------------------------===//
 // Output Stream Adaptors
 //===----------------------------------------------------------------------===//
diff --git a/llvm/include/llvm/Target/TargetMachine.h b/llvm/include/llvm/Target/TargetMachine.h
index 4830ecbe1cd6349962d13c9151424a428dc069dc..4c29f25bedf4196e42ec5679b5c3a8a5fc682286 100644
--- a/llvm/include/llvm/Target/TargetMachine.h
+++ b/llvm/include/llvm/Target/TargetMachine.h
@@ -239,7 +239,7 @@ public:
   void setCodeModel(CodeModel::Model CM) { CMModel = CM; }
 
   void setLargeDataThreshold(uint64_t LDT) { LargeDataThreshold = LDT; }
-  bool isLargeGlobalObject(const GlobalObject *GO) const;
+  bool isLargeGlobalValue(const GlobalValue *GV) const;
 
   bool isPositionIndependent() const;
 
diff --git a/llvm/include/llvm/Transforms/CFGuard.h b/llvm/include/llvm/Transforms/CFGuard.h
index 86fcbc3c13e8b71f7dab7c75b53543d4543bd327..caf822a2ec9fb3a2e6f956b7529908a3de691fc0 100644
--- a/llvm/include/llvm/Transforms/CFGuard.h
+++ b/llvm/include/llvm/Transforms/CFGuard.h
@@ -11,10 +11,23 @@
 #ifndef LLVM_TRANSFORMS_CFGUARD_H
 #define LLVM_TRANSFORMS_CFGUARD_H
 
+#include "llvm/IR/PassManager.h"
+
 namespace llvm {
 
 class FunctionPass;
 
+class CFGuardPass : public PassInfoMixin<CFGuardPass> {
+public:
+  enum class Mechanism { Check, Dispatch };
+
+  CFGuardPass(Mechanism M = Mechanism::Check) : GuardMechanism(M) {}
+  PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM);
+
+private:
+  Mechanism GuardMechanism;
+};
+
 /// Insert Control FLow Guard checks on indirect function calls.
 FunctionPass *createCFGuardCheckPass();
 
diff --git a/llvm/lib/Analysis/AliasAnalysis.cpp b/llvm/lib/Analysis/AliasAnalysis.cpp
index b8a22184d9602f47e292d5443858473a8953e511..da18279ae9b93d693eab870f70a89d47eb7ebdca 100644
--- a/llvm/lib/Analysis/AliasAnalysis.cpp
+++ b/llvm/lib/Analysis/AliasAnalysis.cpp
@@ -896,7 +896,7 @@ bool llvm::isNotVisibleOnUnwind(const Value *Object,
 
   // Byval goes out of scope on unwind.
   if (auto *A = dyn_cast<Argument>(Object))
-    return A->hasByValAttr();
+    return A->hasByValAttr() || A->hasAttribute(Attribute::DeadOnUnwind);
 
   // A noalias return is not accessible from any other code. If the pointer
   // does not escape prior to the unwind, then the caller cannot access the
diff --git a/llvm/lib/Analysis/AliasSetTracker.cpp b/llvm/lib/Analysis/AliasSetTracker.cpp
index debdd328ce53fdc8797b2e38737c6b632c6a9d01..99d20c7bef3be9e91e457880374c7107906238ee 100644
--- a/llvm/lib/Analysis/AliasSetTracker.cpp
+++ b/llvm/lib/Analysis/AliasSetTracker.cpp
@@ -375,9 +375,8 @@ AliasSet &AliasSetTracker::getAliasSetFor(const MemoryLocation &MemLoc) {
   return AliasSets.back();
 }
 
-void AliasSetTracker::add(Value *Ptr, LocationSize Size,
-                          const AAMDNodes &AAInfo) {
-  addPointer(MemoryLocation(Ptr, Size, AAInfo), AliasSet::NoAccess);
+void AliasSetTracker::add(const MemoryLocation &Loc) {
+  addPointer(Loc, AliasSet::NoAccess);
 }
 
 void AliasSetTracker::add(LoadInst *LI) {
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 9a9d06a81f49181c42ec9ba2c03deee4295faeee..0894560fd07898eefad8037c7c2da68ca46be3c4 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -666,7 +666,7 @@ public:
   /// Register a load  and whether it is only read from.
   void addLoad(MemoryLocation &Loc, Type *AccessTy, bool IsReadOnly) {
     Value *Ptr = const_cast<Value*>(Loc.Ptr);
-    AST.add(Ptr, LocationSize::beforeOrAfterPointer(), Loc.AATags);
+    AST.add(Loc.getWithNewSize(LocationSize::beforeOrAfterPointer()));
     Accesses[MemAccessInfo(Ptr, false)].insert(AccessTy);
     if (IsReadOnly)
       ReadOnlyPtr.insert(Ptr);
@@ -675,7 +675,7 @@ public:
   /// Register a store.
   void addStore(MemoryLocation &Loc, Type *AccessTy) {
     Value *Ptr = const_cast<Value*>(Loc.Ptr);
-    AST.add(Ptr, LocationSize::beforeOrAfterPointer(), Loc.AATags);
+    AST.add(Loc.getWithNewSize(LocationSize::beforeOrAfterPointer()));
     Accesses[MemAccessInfo(Ptr, true)].insert(AccessTy);
   }
 
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index 63b1b7567c8e9f4c0800dedbb44baaf487d0bc4d..3f76dfdaac317c2a066fbe0b6bcb245bf305ccf1 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -682,6 +682,11 @@ TargetTransformInfo::getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx,
   return Cost;
 }
 
+bool TargetTransformInfo::preferToKeepConstantsAttached(
+    const Instruction &Inst, const Function &Fn) const {
+  return TTIImpl->preferToKeepConstantsAttached(Inst, Fn);
+}
+
 unsigned TargetTransformInfo::getNumberOfRegisters(unsigned ClassID) const {
   return TTIImpl->getNumberOfRegisters(ClassID);
 }
diff --git a/llvm/lib/AsmParser/LLLexer.cpp b/llvm/lib/AsmParser/LLLexer.cpp
index fcb25f5f1d49dfbaef6686bd68468c25d094a1ee..c8da3efbb68aff71d7727dac5985255e5de7768d 100644
--- a/llvm/lib/AsmParser/LLLexer.cpp
+++ b/llvm/lib/AsmParser/LLLexer.cpp
@@ -906,7 +906,7 @@ lltok::Kind LLLexer::LexIdentifier() {
 
 #define DWKEYWORD(TYPE, TOKEN)                                                 \
   do {                                                                         \
-    if (Keyword.startswith("DW_" #TYPE "_")) {                                 \
+    if (Keyword.starts_with("DW_" #TYPE "_")) {                                \
       StrVal.assign(Keyword.begin(), Keyword.end());                           \
       return lltok::TOKEN;                                                     \
     }                                                                          \
diff --git a/llvm/lib/BinaryFormat/Magic.cpp b/llvm/lib/BinaryFormat/Magic.cpp
index 255937a5bdd04ac06b42ed8fa2497d7f7462aa8a..45a0b7e11452b4382d370b28d9890e38bbc6e1dd 100644
--- a/llvm/lib/BinaryFormat/Magic.cpp
+++ b/llvm/lib/BinaryFormat/Magic.cpp
@@ -72,6 +72,14 @@ file_magic llvm::identify_magic(StringRef Magic) {
   case 0x03:
     if (startswith(Magic, "\x03\xF0\x00"))
       return file_magic::goff_object;
+    // SPIR-V format in little-endian mode.
+    if (startswith(Magic, "\x03\x02\x23\x07"))
+      return file_magic::spirv_object;
+    break;
+
+  case 0x07: // SPIR-V format in big-endian mode.
+    if (startswith(Magic, "\x07\x23\x02\x03"))
+      return file_magic::spirv_object;
     break;
 
   case 0x10:
diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index a11690a19a0e698e8a185869381e0f3aab3dcf1d..8907f6fa4ff3fd21e0511ba4f46cf5536e57448e 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -2098,6 +2098,8 @@ static Attribute::AttrKind getAttrFromCode(uint64_t Code) {
     return Attribute::Writable;
   case bitc::ATTR_KIND_CORO_ONLY_DESTROY_WHEN_COMPLETE:
     return Attribute::CoroDestroyOnlyWhenComplete;
+  case bitc::ATTR_KIND_DEAD_ON_UNWIND:
+    return Attribute::DeadOnUnwind;
   }
 }
 
diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
index 4e0379f5f4075c8d18de1f5f22f702779d89f2e8..8fca569a391bafec2df6c5045bbb6574b28e00f5 100644
--- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -827,6 +827,8 @@ static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) {
     return bitc::ATTR_KIND_WRITABLE;
   case Attribute::CoroDestroyOnlyWhenComplete:
     return bitc::ATTR_KIND_CORO_ONLY_DESTROY_WHEN_COMPLETE;
+  case Attribute::DeadOnUnwind:
+    return bitc::ATTR_KIND_DEAD_ON_UNWIND;
   case Attribute::EndAttrKinds:
     llvm_unreachable("Can not encode end-attribute kinds marker.");
   case Attribute::None:
diff --git a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
index a52a1293672c5c51e1c89b6bd3459f75c7b0846f..d0d390b020356e4debdbd59028b7ace0348531a0 100644
--- a/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
+++ b/llvm/lib/CodeGen/AsmPrinter/AsmPrinter.cpp
@@ -2173,61 +2173,58 @@ void AsmPrinter::emitGlobalIFunc(Module &M, const GlobalIFunc &GI) {
     return;
   }
 
-  if (TM.getTargetTriple().isOSBinFormatMachO() && getIFuncMCSubtargetInfo()) {
-    // On Darwin platforms, emit a manually-constructed .symbol_resolver that
-    // implements the symbol resolution duties of the IFunc.
-    //
-    // Normally, this would be handled by linker magic, but unfortunately there
-    // are a few limitations in ld64 and ld-prime's implementation of
-    // .symbol_resolver that mean we can't always use them:
-    //
-    //    *  resolvers cannot be the target of an alias
-    //    *  resolvers cannot have private linkage
-    //    *  resolvers cannot have linkonce linkage
-    //    *  resolvers cannot appear in executables
-    //    *  resolvers cannot appear in bundles
-    //
-    // This works around that by emitting a close approximation of what the
-    // linker would have done.
-
-    MCSymbol *LazyPointer =
-        GetExternalSymbolSymbol(GI.getName() + ".lazy_pointer");
-    MCSymbol *StubHelper =
-        GetExternalSymbolSymbol(GI.getName() + ".stub_helper");
-
-    OutStreamer->switchSection(
-        OutContext.getObjectFileInfo()->getDataSection());
-
-    const DataLayout &DL = M.getDataLayout();
-    emitAlignment(Align(DL.getPointerSize()));
-    OutStreamer->emitLabel(LazyPointer);
-    emitVisibility(LazyPointer, GI.getVisibility());
-    OutStreamer->emitValue(MCSymbolRefExpr::create(StubHelper, OutContext), 8);
-
-    OutStreamer->switchSection(
-        OutContext.getObjectFileInfo()->getTextSection());
+  if (!TM.getTargetTriple().isOSBinFormatMachO() || !getIFuncMCSubtargetInfo())
+    llvm::report_fatal_error("IFuncs are not supported on this platform");
 
-    const TargetSubtargetInfo *STI =
-        TM.getSubtargetImpl(*GI.getResolverFunction());
-    const TargetLowering *TLI = STI->getTargetLowering();
-    Align TextAlign(TLI->getMinFunctionAlignment());
-
-    MCSymbol *Stub = getSymbol(&GI);
-    EmitLinkage(Stub);
-    OutStreamer->emitCodeAlignment(TextAlign, getIFuncMCSubtargetInfo());
-    OutStreamer->emitLabel(Stub);
-    emitVisibility(Stub, GI.getVisibility());
-    emitMachOIFuncStubBody(M, GI, LazyPointer);
-
-    OutStreamer->emitCodeAlignment(TextAlign, getIFuncMCSubtargetInfo());
-    OutStreamer->emitLabel(StubHelper);
-    emitVisibility(StubHelper, GI.getVisibility());
-    emitMachOIFuncStubHelperBody(M, GI, LazyPointer);
-
-    return;
-  }
-
-  llvm::report_fatal_error("IFuncs are not supported on this platform");
+  // On Darwin platforms, emit a manually-constructed .symbol_resolver that
+  // implements the symbol resolution duties of the IFunc.
+  //
+  // Normally, this would be handled by linker magic, but unfortunately there
+  // are a few limitations in ld64 and ld-prime's implementation of
+  // .symbol_resolver that mean we can't always use them:
+  //
+  //    *  resolvers cannot be the target of an alias
+  //    *  resolvers cannot have private linkage
+  //    *  resolvers cannot have linkonce linkage
+  //    *  resolvers cannot appear in executables
+  //    *  resolvers cannot appear in bundles
+  //
+  // This works around that by emitting a close approximation of what the
+  // linker would have done.
+
+  MCSymbol *LazyPointer =
+      GetExternalSymbolSymbol(GI.getName() + ".lazy_pointer");
+  MCSymbol *StubHelper =
+      GetExternalSymbolSymbol(GI.getName() + ".stub_helper");
+
+  OutStreamer->switchSection(
+      OutContext.getObjectFileInfo()->getDataSection());
+
+  const DataLayout &DL = M.getDataLayout();
+  emitAlignment(Align(DL.getPointerSize()));
+  OutStreamer->emitLabel(LazyPointer);
+  emitVisibility(LazyPointer, GI.getVisibility());
+  OutStreamer->emitValue(MCSymbolRefExpr::create(StubHelper, OutContext), 8);
+
+  OutStreamer->switchSection(
+      OutContext.getObjectFileInfo()->getTextSection());
+
+  const TargetSubtargetInfo *STI =
+      TM.getSubtargetImpl(*GI.getResolverFunction());
+  const TargetLowering *TLI = STI->getTargetLowering();
+  Align TextAlign(TLI->getMinFunctionAlignment());
+
+  MCSymbol *Stub = getSymbol(&GI);
+  EmitLinkage(Stub);
+  OutStreamer->emitCodeAlignment(TextAlign, getIFuncMCSubtargetInfo());
+  OutStreamer->emitLabel(Stub);
+  emitVisibility(Stub, GI.getVisibility());
+  emitMachOIFuncStubBody(M, GI, LazyPointer);
+
+  OutStreamer->emitCodeAlignment(TextAlign, getIFuncMCSubtargetInfo());
+  OutStreamer->emitLabel(StubHelper);
+  emitVisibility(StubHelper, GI.getVisibility());
+  emitMachOIFuncStubHelperBody(M, GI, LazyPointer);
 }
 
 void AsmPrinter::emitRemarksSection(remarks::RemarkStreamer &RS) {
diff --git a/llvm/lib/CodeGen/CodeGen.cpp b/llvm/lib/CodeGen/CodeGen.cpp
index 79a95ee0d747a1c857b33eecfd76b01b8a2485c1..7b73a7b11ddf1c9e98815646e941b20e0eca378c 100644
--- a/llvm/lib/CodeGen/CodeGen.cpp
+++ b/llvm/lib/CodeGen/CodeGen.cpp
@@ -41,7 +41,7 @@ void llvm::initializeCodeGen(PassRegistry &Registry) {
   initializeEarlyTailDuplicatePass(Registry);
   initializeExpandLargeDivRemLegacyPassPass(Registry);
   initializeExpandLargeFpConvertLegacyPassPass(Registry);
-  initializeExpandMemCmpPassPass(Registry);
+  initializeExpandMemCmpLegacyPassPass(Registry);
   initializeExpandPostRAPass(Registry);
   initializeFEntryInserterPass(Registry);
   initializeFinalizeISelPass(Registry);
@@ -53,7 +53,7 @@ void llvm::initializeCodeGen(PassRegistry &Registry) {
   initializeHardwareLoopsLegacyPass(Registry);
   initializeIfConverterPass(Registry);
   initializeImplicitNullChecksPass(Registry);
-  initializeIndirectBrExpandPassPass(Registry);
+  initializeIndirectBrExpandLegacyPassPass(Registry);
   initializeInterleavedLoadCombinePass(Registry);
   initializeInterleavedAccessPass(Registry);
   initializeJMCInstrumenterPass(Registry);
diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp
index aa5cdd2e04a99482d54f6d49c5762dfaf93c4e5d..f9e791c733483844514d283dcc6095cba7c94cdd 100644
--- a/llvm/lib/CodeGen/CodeGenPrepare.cpp
+++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp
@@ -1384,7 +1384,8 @@ static bool SinkCast(CastInst *CI) {
       BasicBlock::iterator InsertPt = UserBB->getFirstInsertionPt();
       assert(InsertPt != UserBB->end());
       InsertedCast = CastInst::Create(CI->getOpcode(), CI->getOperand(0),
-                                      CI->getType(), "", &*InsertPt);
+                                      CI->getType(), "");
+      InsertedCast->insertBefore(*UserBB, InsertPt);
       InsertedCast->setDebugLoc(CI->getDebugLoc());
     }
 
@@ -2058,12 +2059,13 @@ SinkShiftAndTruncate(BinaryOperator *ShiftI, Instruction *User, ConstantInt *CI,
       assert(InsertPt != TruncUserBB->end());
       // Sink the shift
       if (ShiftI->getOpcode() == Instruction::AShr)
-        InsertedShift = BinaryOperator::CreateAShr(ShiftI->getOperand(0), CI,
-                                                   "", &*InsertPt);
+        InsertedShift =
+            BinaryOperator::CreateAShr(ShiftI->getOperand(0), CI, "");
       else
-        InsertedShift = BinaryOperator::CreateLShr(ShiftI->getOperand(0), CI,
-                                                   "", &*InsertPt);
+        InsertedShift =
+            BinaryOperator::CreateLShr(ShiftI->getOperand(0), CI, "");
       InsertedShift->setDebugLoc(ShiftI->getDebugLoc());
+      InsertedShift->insertBefore(*TruncUserBB, InsertPt);
 
       // Sink the trunc
       BasicBlock::iterator TruncInsertPt = TruncUserBB->getFirstInsertionPt();
@@ -2162,11 +2164,12 @@ static bool OptimizeExtractBits(BinaryOperator *ShiftI, ConstantInt *CI,
       assert(InsertPt != UserBB->end());
 
       if (ShiftI->getOpcode() == Instruction::AShr)
-        InsertedShift = BinaryOperator::CreateAShr(ShiftI->getOperand(0), CI,
-                                                   "", &*InsertPt);
+        InsertedShift =
+            BinaryOperator::CreateAShr(ShiftI->getOperand(0), CI, "");
       else
-        InsertedShift = BinaryOperator::CreateLShr(ShiftI->getOperand(0), CI,
-                                                   "", &*InsertPt);
+        InsertedShift =
+            BinaryOperator::CreateLShr(ShiftI->getOperand(0), CI, "");
+      InsertedShift->insertBefore(*UserBB, InsertPt);
       InsertedShift->setDebugLoc(ShiftI->getDebugLoc());
 
       MadeChange = true;
@@ -6628,7 +6631,8 @@ bool CodeGenPrepare::optimizeExtUses(Instruction *I) {
     if (!InsertedTrunc) {
       BasicBlock::iterator InsertPt = UserBB->getFirstInsertionPt();
       assert(InsertPt != UserBB->end());
-      InsertedTrunc = new TruncInst(I, Src->getType(), "", &*InsertPt);
+      InsertedTrunc = new TruncInst(I, Src->getType(), "");
+      InsertedTrunc->insertBefore(*UserBB, InsertPt);
       InsertedInsts.insert(InsertedTrunc);
     }
 
diff --git a/llvm/lib/CodeGen/ExpandMemCmp.cpp b/llvm/lib/CodeGen/ExpandMemCmp.cpp
index e6ca1409624905a068909086ff423f98b274ebae..bb84813569f4d5be1699a15398567a3158b288cd 100644
--- a/llvm/lib/CodeGen/ExpandMemCmp.cpp
+++ b/llvm/lib/CodeGen/ExpandMemCmp.cpp
@@ -11,6 +11,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "llvm/CodeGen/ExpandMemCmp.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Analysis/ConstantFolding.h"
 #include "llvm/Analysis/DomTreeUpdater.h"
@@ -38,7 +39,7 @@ namespace llvm {
 class TargetLowering;
 }
 
-#define DEBUG_TYPE "expandmemcmp"
+#define DEBUG_TYPE "expand-memcmp"
 
 STATISTIC(NumMemCmpCalls, "Number of memcmp calls");
 STATISTIC(NumMemCmpNotConstant, "Number of memcmp calls without constant size");
@@ -886,12 +887,24 @@ static bool expandMemCmp(CallInst *CI, const TargetTransformInfo *TTI,
   return true;
 }
 
-class ExpandMemCmpPass : public FunctionPass {
+// Returns true if a change was made.
+static bool runOnBlock(BasicBlock &BB, const TargetLibraryInfo *TLI,
+                       const TargetTransformInfo *TTI, const TargetLowering *TL,
+                       const DataLayout &DL, ProfileSummaryInfo *PSI,
+                       BlockFrequencyInfo *BFI, DomTreeUpdater *DTU);
+
+static PreservedAnalyses runImpl(Function &F, const TargetLibraryInfo *TLI,
+                                 const TargetTransformInfo *TTI,
+                                 const TargetLowering *TL,
+                                 ProfileSummaryInfo *PSI,
+                                 BlockFrequencyInfo *BFI, DominatorTree *DT);
+
+class ExpandMemCmpLegacyPass : public FunctionPass {
 public:
   static char ID;
 
-  ExpandMemCmpPass() : FunctionPass(ID) {
-    initializeExpandMemCmpPassPass(*PassRegistry::getPassRegistry());
+  ExpandMemCmpLegacyPass() : FunctionPass(ID) {
+    initializeExpandMemCmpLegacyPassPass(*PassRegistry::getPassRegistry());
   }
 
   bool runOnFunction(Function &F) override {
@@ -928,25 +941,13 @@ private:
     LazyBlockFrequencyInfoPass::getLazyBFIAnalysisUsage(AU);
     FunctionPass::getAnalysisUsage(AU);
   }
-
-  PreservedAnalyses runImpl(Function &F, const TargetLibraryInfo *TLI,
-                            const TargetTransformInfo *TTI,
-                            const TargetLowering *TL, ProfileSummaryInfo *PSI,
-                            BlockFrequencyInfo *BFI, DominatorTree *DT);
-  // Returns true if a change was made.
-  bool runOnBlock(BasicBlock &BB, const TargetLibraryInfo *TLI,
-                  const TargetTransformInfo *TTI, const TargetLowering *TL,
-                  const DataLayout &DL, ProfileSummaryInfo *PSI,
-                  BlockFrequencyInfo *BFI, DomTreeUpdater *DTU);
 };
 
-bool ExpandMemCmpPass::runOnBlock(BasicBlock &BB, const TargetLibraryInfo *TLI,
-                                  const TargetTransformInfo *TTI,
-                                  const TargetLowering *TL,
-                                  const DataLayout &DL, ProfileSummaryInfo *PSI,
-                                  BlockFrequencyInfo *BFI,
-                                  DomTreeUpdater *DTU) {
-  for (Instruction& I : BB) {
+bool runOnBlock(BasicBlock &BB, const TargetLibraryInfo *TLI,
+                const TargetTransformInfo *TTI, const TargetLowering *TL,
+                const DataLayout &DL, ProfileSummaryInfo *PSI,
+                BlockFrequencyInfo *BFI, DomTreeUpdater *DTU) {
+  for (Instruction &I : BB) {
     CallInst *CI = dyn_cast<CallInst>(&I);
     if (!CI) {
       continue;
@@ -961,8 +962,7 @@ bool ExpandMemCmpPass::runOnBlock(BasicBlock &BB, const TargetLibraryInfo *TLI,
   return false;
 }
 
-PreservedAnalyses
-ExpandMemCmpPass::runImpl(Function &F, const TargetLibraryInfo *TLI,
+PreservedAnalyses runImpl(Function &F, const TargetLibraryInfo *TLI,
                           const TargetTransformInfo *TTI,
                           const TargetLowering *TL, ProfileSummaryInfo *PSI,
                           BlockFrequencyInfo *BFI, DominatorTree *DT) {
@@ -994,17 +994,32 @@ ExpandMemCmpPass::runImpl(Function &F, const TargetLibraryInfo *TLI,
 
 } // namespace
 
-char ExpandMemCmpPass::ID = 0;
-INITIALIZE_PASS_BEGIN(ExpandMemCmpPass, "expandmemcmp",
+PreservedAnalyses ExpandMemCmpPass::run(Function &F,
+                                        FunctionAnalysisManager &FAM) {
+  const auto *TL = TM->getSubtargetImpl(F)->getTargetLowering();
+  const auto &TLI = FAM.getResult<TargetLibraryAnalysis>(F);
+  const auto &TTI = FAM.getResult<TargetIRAnalysis>(F);
+  auto *PSI = FAM.getResult<ModuleAnalysisManagerFunctionProxy>(F)
+                  .getCachedResult<ProfileSummaryAnalysis>(*F.getParent());
+  BlockFrequencyInfo *BFI = (PSI && PSI->hasProfileSummary())
+                                ? &FAM.getResult<BlockFrequencyAnalysis>(F)
+                                : nullptr;
+  auto *DT = FAM.getCachedResult<DominatorTreeAnalysis>(F);
+
+  return runImpl(F, &TLI, &TTI, TL, PSI, BFI, DT);
+}
+
+char ExpandMemCmpLegacyPass::ID = 0;
+INITIALIZE_PASS_BEGIN(ExpandMemCmpLegacyPass, DEBUG_TYPE,
                       "Expand memcmp() to load/stores", false, false)
 INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(LazyBlockFrequencyInfoPass)
 INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass)
 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
-INITIALIZE_PASS_END(ExpandMemCmpPass, "expandmemcmp",
+INITIALIZE_PASS_END(ExpandMemCmpLegacyPass, DEBUG_TYPE,
                     "Expand memcmp() to load/stores", false, false)
 
-FunctionPass *llvm::createExpandMemCmpPass() {
-  return new ExpandMemCmpPass();
+FunctionPass *llvm::createExpandMemCmpLegacyPass() {
+  return new ExpandMemCmpLegacyPass();
 }
diff --git a/llvm/lib/CodeGen/GCMetadata.cpp b/llvm/lib/CodeGen/GCMetadata.cpp
index 49e03b4b132c6e5c6eec3b1e12dbd15efa1ab08b..cad7d1f1137bba4db82276687ce544715a0d26f9 100644
--- a/llvm/lib/CodeGen/GCMetadata.cpp
+++ b/llvm/lib/CodeGen/GCMetadata.cpp
@@ -24,6 +24,50 @@
 
 using namespace llvm;
 
+bool GCStrategyMap::invalidate(Module &M, const PreservedAnalyses &PA,
+                               ModuleAnalysisManager::Invalidator &) {
+  for (const auto &F : M) {
+    if (F.isDeclaration() || !F.hasGC())
+      continue;
+    if (!StrategyMap.contains(F.getGC()))
+      return true;
+  }
+  return false;
+}
+
+AnalysisKey CollectorMetadataAnalysis::Key;
+
+CollectorMetadataAnalysis::Result
+CollectorMetadataAnalysis::run(Module &M, ModuleAnalysisManager &MAM) {
+  Result R;
+  auto &Map = R.StrategyMap;
+  for (auto &F : M) {
+    if (F.isDeclaration() || !F.hasGC())
+      continue;
+    if (auto GCName = F.getGC(); !Map.contains(GCName))
+      Map[GCName] = getGCStrategy(GCName);
+  }
+  return R;
+}
+
+AnalysisKey GCFunctionAnalysis::Key;
+
+GCFunctionAnalysis::Result
+GCFunctionAnalysis::run(Function &F, FunctionAnalysisManager &FAM) {
+  assert(!F.isDeclaration() && "Can only get GCFunctionInfo for a definition!");
+  assert(F.hasGC() && "Function doesn't have GC!");
+
+  auto &MAMProxy = FAM.getResult<ModuleAnalysisManagerFunctionProxy>(F);
+  assert(
+      MAMProxy.cachedResultExists<CollectorMetadataAnalysis>(*F.getParent()) &&
+      "This pass need module analysis `collector-metadata`!");
+  auto &Map =
+      MAMProxy.getCachedResult<CollectorMetadataAnalysis>(*F.getParent())
+          ->StrategyMap;
+  GCFunctionInfo Info(F, *Map[F.getGC()]);
+  return Info;
+}
+
 INITIALIZE_PASS(GCModuleInfo, "collector-metadata",
                 "Create Garbage Collector Module Metadata", false, false)
 
@@ -34,6 +78,12 @@ GCFunctionInfo::GCFunctionInfo(const Function &F, GCStrategy &S)
 
 GCFunctionInfo::~GCFunctionInfo() = default;
 
+bool GCFunctionInfo::invalidate(Function &F, const PreservedAnalyses &PA,
+                                FunctionAnalysisManager::Invalidator &) {
+  auto PAC = PA.getChecker<GCFunctionAnalysis>();
+  return !PAC.preserved() && !PAC.preservedSet<AllAnalysesOn<Function>>();
+}
+
 // -----------------------------------------------------------------------------
 
 char GCModuleInfo::ID = 0;
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index 27a53e55f32fa30e192e269213e9c389f883f58c..bea29642cd0031f1ff0255595f399d2d7c95f911 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -1550,8 +1550,10 @@ bool IRTranslator::translateGetElementPtr(const User &U,
       // If this is a scalar constant or a splat vector of constants,
       // handle it quickly.
       if (const auto *CI = dyn_cast<ConstantInt>(Idx)) {
-        Offset += ElementSize * CI->getSExtValue();
-        continue;
+        if (std::optional<int64_t> Val = CI->getValue().trySExtValue()) {
+          Offset += ElementSize * *Val;
+          continue;
+        }
       }
 
       if (Offset != 0) {
diff --git a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
index 6e1c0e907e8e4155eaa7a4356b0c0b8a5bec475b..a5827c26c04f48b245817e220591228783aa415e 100644
--- a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp
@@ -314,7 +314,10 @@ MachineInstrBuilder MachineIRBuilder::buildConstant(const DstOp &Res,
   assert(EltTy.getScalarSizeInBits() == Val.getBitWidth() &&
          "creating constant with the wrong size");
 
-  if (Ty.isVector()) {
+  assert(!Ty.isScalableVector() &&
+         "unexpected scalable vector in buildConstant");
+
+  if (Ty.isFixedVector()) {
     auto Const = buildInstr(TargetOpcode::G_CONSTANT)
     .addDef(getMRI()->createGenericVirtualRegister(EltTy))
     .addCImm(&Val);
@@ -347,7 +350,10 @@ MachineInstrBuilder MachineIRBuilder::buildFConstant(const DstOp &Res,
 
   assert(!Ty.isPointer() && "invalid operand type");
 
-  if (Ty.isVector()) {
+  assert(!Ty.isScalableVector() &&
+         "unexpected scalable vector in buildFConstant");
+
+  if (Ty.isFixedVector()) {
     auto Const = buildInstr(TargetOpcode::G_FCONSTANT)
     .addDef(getMRI()->createGenericVirtualRegister(EltTy))
     .addFPImm(&Val);
diff --git a/llvm/lib/CodeGen/IndirectBrExpandPass.cpp b/llvm/lib/CodeGen/IndirectBrExpandPass.cpp
index 012892166ae790568e58df8b8f587443bfa273b1..f7b931a3bdac2dd4e02a79b8995ba4345964a387 100644
--- a/llvm/lib/CodeGen/IndirectBrExpandPass.cpp
+++ b/llvm/lib/CodeGen/IndirectBrExpandPass.cpp
@@ -29,6 +29,7 @@
 #include "llvm/ADT/Sequence.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Analysis/DomTreeUpdater.h"
+#include "llvm/CodeGen/IndirectBrExpand.h"
 #include "llvm/CodeGen/TargetPassConfig.h"
 #include "llvm/CodeGen/TargetSubtargetInfo.h"
 #include "llvm/IR/BasicBlock.h"
@@ -48,14 +49,12 @@ using namespace llvm;
 
 namespace {
 
-class IndirectBrExpandPass : public FunctionPass {
-  const TargetLowering *TLI = nullptr;
-
+class IndirectBrExpandLegacyPass : public FunctionPass {
 public:
   static char ID; // Pass identification, replacement for typeid
 
-  IndirectBrExpandPass() : FunctionPass(ID) {
-    initializeIndirectBrExpandPassPass(*PassRegistry::getPassRegistry());
+  IndirectBrExpandLegacyPass() : FunctionPass(ID) {
+    initializeIndirectBrExpandLegacyPassPass(*PassRegistry::getPassRegistry());
   }
 
   void getAnalysisUsage(AnalysisUsage &AU) const override {
@@ -67,33 +66,41 @@ public:
 
 } // end anonymous namespace
 
-char IndirectBrExpandPass::ID = 0;
+static bool runImpl(Function &F, const TargetLowering *TLI,
+                    DomTreeUpdater *DTU);
+
+PreservedAnalyses IndirectBrExpandPass::run(Function &F,
+                                            FunctionAnalysisManager &FAM) {
+  auto *STI = TM->getSubtargetImpl(F);
+  if (!STI->enableIndirectBrExpand())
+    return PreservedAnalyses::all();
+
+  auto *TLI = STI->getTargetLowering();
+  auto *DT = FAM.getCachedResult<DominatorTreeAnalysis>(F);
+  DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);
+
+  bool Changed = runImpl(F, TLI, DT ? &DTU : nullptr);
+  if (!Changed)
+    return PreservedAnalyses::all();
+  PreservedAnalyses PA;
+  PA.preserve<DominatorTreeAnalysis>();
+  return PA;
+}
+
+char IndirectBrExpandLegacyPass::ID = 0;
 
-INITIALIZE_PASS_BEGIN(IndirectBrExpandPass, DEBUG_TYPE,
+INITIALIZE_PASS_BEGIN(IndirectBrExpandLegacyPass, DEBUG_TYPE,
                       "Expand indirectbr instructions", false, false)
 INITIALIZE_PASS_DEPENDENCY(DominatorTreeWrapperPass)
-INITIALIZE_PASS_END(IndirectBrExpandPass, DEBUG_TYPE,
+INITIALIZE_PASS_END(IndirectBrExpandLegacyPass, DEBUG_TYPE,
                     "Expand indirectbr instructions", false, false)
 
 FunctionPass *llvm::createIndirectBrExpandPass() {
-  return new IndirectBrExpandPass();
+  return new IndirectBrExpandLegacyPass();
 }
 
-bool IndirectBrExpandPass::runOnFunction(Function &F) {
+bool runImpl(Function &F, const TargetLowering *TLI, DomTreeUpdater *DTU) {
   auto &DL = F.getParent()->getDataLayout();
-  auto *TPC = getAnalysisIfAvailable<TargetPassConfig>();
-  if (!TPC)
-    return false;
-
-  auto &TM = TPC->getTM<TargetMachine>();
-  auto &STI = *TM.getSubtargetImpl(F);
-  if (!STI.enableIndirectBrExpand())
-    return false;
-  TLI = STI.getTargetLowering();
-
-  std::optional<DomTreeUpdater> DTU;
-  if (auto *DTWP = getAnalysisIfAvailable<DominatorTreeWrapperPass>())
-    DTU.emplace(DTWP->getDomTree(), DomTreeUpdater::UpdateStrategy::Lazy);
 
   SmallVector<IndirectBrInst *, 1> IndirectBrs;
 
@@ -268,3 +275,21 @@ bool IndirectBrExpandPass::runOnFunction(Function &F) {
 
   return true;
 }
+
+bool IndirectBrExpandLegacyPass::runOnFunction(Function &F) {
+  auto *TPC = getAnalysisIfAvailable<TargetPassConfig>();
+  if (!TPC)
+    return false;
+
+  auto &TM = TPC->getTM<TargetMachine>();
+  auto &STI = *TM.getSubtargetImpl(F);
+  if (!STI.enableIndirectBrExpand())
+    return false;
+  auto *TLI = STI.getTargetLowering();
+
+  std::optional<DomTreeUpdater> DTU;
+  if (auto *DTWP = getAnalysisIfAvailable<DominatorTreeWrapperPass>())
+    DTU.emplace(DTWP->getDomTree(), DomTreeUpdater::UpdateStrategy::Lazy);
+
+  return runImpl(F, TLI, DTU ? &*DTU : nullptr);
+}
diff --git a/llvm/lib/CodeGen/InterleavedLoadCombinePass.cpp b/llvm/lib/CodeGen/InterleavedLoadCombinePass.cpp
index 3b1d26cfed79fc6e4bda7dd4a186a71cd52b6c7a..f2d5c3c867c2dda6f0322c6d9efd19961045ea41 100644
--- a/llvm/lib/CodeGen/InterleavedLoadCombinePass.cpp
+++ b/llvm/lib/CodeGen/InterleavedLoadCombinePass.cpp
@@ -23,6 +23,7 @@
 #include "llvm/Analysis/MemorySSAUpdater.h"
 #include "llvm/Analysis/OptimizationRemarkEmitter.h"
 #include "llvm/Analysis/TargetTransformInfo.h"
+#include "llvm/CodeGen/InterleavedLoadCombine.h"
 #include "llvm/CodeGen/Passes.h"
 #include "llvm/CodeGen/TargetLowering.h"
 #include "llvm/CodeGen/TargetPassConfig.h"
@@ -63,7 +64,7 @@ struct VectorInfo;
 struct InterleavedLoadCombineImpl {
 public:
   InterleavedLoadCombineImpl(Function &F, DominatorTree &DT, MemorySSA &MSSA,
-                             TargetMachine &TM)
+                             const TargetMachine &TM)
       : F(F), DT(DT), MSSA(MSSA),
         TLI(*TM.getSubtargetImpl(F)->getTargetLowering()),
         TTI(TM.getTargetTransformInfo(F)) {}
@@ -1339,6 +1340,15 @@ private:
 };
 } // anonymous namespace
 
+PreservedAnalyses
+InterleavedLoadCombinePass::run(Function &F, FunctionAnalysisManager &FAM) {
+
+  auto &DT = FAM.getResult<DominatorTreeAnalysis>(F);
+  auto &MemSSA = FAM.getResult<MemorySSAAnalysis>(F).getMSSA();
+  bool Changed = InterleavedLoadCombineImpl(F, DT, MemSSA, *TM).run();
+  return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all();
+}
+
 char InterleavedLoadCombine::ID = 0;
 
 INITIALIZE_PASS_BEGIN(
diff --git a/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.cpp b/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.cpp
index b2c2b40139eda6e127c5ae086d192b00f709c37a..87a0ba58b14c01a58b31cbd9d591603ec195270f 100644
--- a/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.cpp
+++ b/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.cpp
@@ -1429,7 +1429,7 @@ bool InstrRefBasedLDV::transferDebugValue(const MachineInstr &MI) {
 
 std::optional<ValueIDNum> InstrRefBasedLDV::getValueForInstrRef(
     unsigned InstNo, unsigned OpNo, MachineInstr &MI,
-    const ValueTable *MLiveOuts, const ValueTable *MLiveIns) {
+    const FuncValueTable *MLiveOuts, const FuncValueTable *MLiveIns) {
   // Various optimizations may have happened to the value during codegen,
   // recorded in the value substitution table. Apply any substitutions to
   // the instruction / operand number in this DBG_INSTR_REF, and collect
@@ -1495,7 +1495,8 @@ std::optional<ValueIDNum> InstrRefBasedLDV::getValueForInstrRef(
   } else if (PHIIt != DebugPHINumToValue.end() && PHIIt->InstrNum == InstNo) {
     // It's actually a PHI value. Which value it is might not be obvious, use
     // the resolver helper to find out.
-    NewID = resolveDbgPHIs(*MI.getParent()->getParent(), MLiveOuts, MLiveIns,
+    assert(MLiveOuts && MLiveIns);
+    NewID = resolveDbgPHIs(*MI.getParent()->getParent(), *MLiveOuts, *MLiveIns,
                            MI, InstNo);
   }
 
@@ -1574,8 +1575,8 @@ std::optional<ValueIDNum> InstrRefBasedLDV::getValueForInstrRef(
 }
 
 bool InstrRefBasedLDV::transferDebugInstrRef(MachineInstr &MI,
-                                             const ValueTable *MLiveOuts,
-                                             const ValueTable *MLiveIns) {
+                                             const FuncValueTable *MLiveOuts,
+                                             const FuncValueTable *MLiveIns) {
   if (!MI.isDebugRef())
     return false;
 
@@ -2116,7 +2117,7 @@ bool InstrRefBasedLDV::transferSpillOrRestoreInst(MachineInstr &MI) {
 }
 
 bool InstrRefBasedLDV::transferRegisterCopy(MachineInstr &MI) {
-  auto DestSrc = TII->isCopyInstr(MI);
+  auto DestSrc = TII->isCopyLikeInstr(MI);
   if (!DestSrc)
     return false;
 
@@ -2245,8 +2246,9 @@ void InstrRefBasedLDV::accumulateFragmentMap(MachineInstr &MI) {
   AllSeenFragments.insert(ThisFragment);
 }
 
-void InstrRefBasedLDV::process(MachineInstr &MI, const ValueTable *MLiveOuts,
-                               const ValueTable *MLiveIns) {
+void InstrRefBasedLDV::process(MachineInstr &MI,
+                               const FuncValueTable *MLiveOuts,
+                               const FuncValueTable *MLiveIns) {
   // Try to interpret an MI as a debug or transfer instruction. Only if it's
   // none of these should we interpret it's register defs as new value
   // definitions.
@@ -3503,7 +3505,10 @@ bool InstrRefBasedLDV::depthFirstVLocAndEmit(
   // Helper lambda for ejecting a block -- if nothing is going to use the block,
   // we can translate the variable location information into DBG_VALUEs and then
   // free all of InstrRefBasedLDV's data structures.
+  SmallPtrSet<const MachineBasicBlock *, 8> EjectedBBs;
   auto EjectBlock = [&](MachineBasicBlock &MBB) -> void {
+    if (EjectedBBs.insert(&MBB).second == false)
+      return;
     unsigned BBNum = MBB.getNumber();
     AllTheVLocs[BBNum].clear();
 
@@ -3517,14 +3522,14 @@ bool InstrRefBasedLDV::depthFirstVLocAndEmit(
     CurBB = BBNum;
     CurInst = 1;
     for (auto &MI : MBB) {
-      process(MI, MOutLocs.get(), MInLocs.get());
+      process(MI, &MOutLocs, &MInLocs);
       TTracker->checkInstForNewValues(CurInst, MI.getIterator());
       ++CurInst;
     }
 
     // Free machine-location tables for this block.
-    MInLocs[BBNum].reset();
-    MOutLocs[BBNum].reset();
+    MInLocs[BBNum] = ValueTable();
+    MOutLocs[BBNum] = ValueTable();
     // We don't need live-in variable values for this block either.
     Output[BBNum].clear();
     AllTheVLocs[BBNum].clear();
@@ -3589,8 +3594,7 @@ bool InstrRefBasedLDV::depthFirstVLocAndEmit(
   // anything for such out-of-scope blocks, but for the sake of being similar
   // to VarLocBasedLDV, eject these too.
   for (auto *MBB : ArtificialBlocks)
-    if (MOutLocs[MBB->getNumber()])
-      EjectBlock(*MBB);
+    EjectBlock(*MBB);
 
   return emitTransfers(AllVarsNumbering);
 }
@@ -3688,14 +3692,9 @@ bool InstrRefBasedLDV::ExtendRanges(MachineFunction &MF,
   // Allocate and initialize two array-of-arrays for the live-in and live-out
   // machine values. The outer dimension is the block number; while the inner
   // dimension is a LocIdx from MLocTracker.
-  FuncValueTable MOutLocs = std::make_unique<ValueTable[]>(MaxNumBlocks);
-  FuncValueTable MInLocs = std::make_unique<ValueTable[]>(MaxNumBlocks);
   unsigned NumLocs = MTracker->getNumLocs();
-  for (int i = 0; i < MaxNumBlocks; ++i) {
-    // These all auto-initialize to ValueIDNum::EmptyValue
-    MOutLocs[i] = std::make_unique<ValueIDNum[]>(NumLocs);
-    MInLocs[i] = std::make_unique<ValueIDNum[]>(NumLocs);
-  }
+  FuncValueTable MOutLocs(MaxNumBlocks, ValueTable(NumLocs));
+  FuncValueTable MInLocs(MaxNumBlocks, ValueTable(NumLocs));
 
   // Solve the machine value dataflow problem using the MLocTransfer function,
   // storing the computed live-ins / live-outs into the array-of-arrays. We use
@@ -3736,7 +3735,7 @@ bool InstrRefBasedLDV::ExtendRanges(MachineFunction &MF,
     MTracker->loadFromArray(MInLocs[CurBB], CurBB);
     CurInst = 1;
     for (auto &MI : MBB) {
-      process(MI, MOutLocs.get(), MInLocs.get());
+      process(MI, &MOutLocs, &MInLocs);
       ++CurInst;
     }
     MTracker->reset();
@@ -3917,9 +3916,9 @@ public:
   /// Machine location where any PHI must occur.
   LocIdx Loc;
   /// Table of live-in machine value numbers for blocks / locations.
-  const ValueTable *MLiveIns;
+  const FuncValueTable &MLiveIns;
 
-  LDVSSAUpdater(LocIdx L, const ValueTable *MLiveIns)
+  LDVSSAUpdater(LocIdx L, const FuncValueTable &MLiveIns)
       : Loc(L), MLiveIns(MLiveIns) {}
 
   void reset() {
@@ -4075,12 +4074,8 @@ public:
 } // end namespace llvm
 
 std::optional<ValueIDNum> InstrRefBasedLDV::resolveDbgPHIs(
-    MachineFunction &MF, const ValueTable *MLiveOuts,
-    const ValueTable *MLiveIns, MachineInstr &Here, uint64_t InstrNum) {
-  assert(MLiveOuts && MLiveIns &&
-         "Tried to resolve DBG_PHI before location "
-         "tables allocated?");
-
+    MachineFunction &MF, const FuncValueTable &MLiveOuts,
+    const FuncValueTable &MLiveIns, MachineInstr &Here, uint64_t InstrNum) {
   // This function will be called twice per DBG_INSTR_REF, and might end up
   // computing lots of SSA information: memoize it.
   auto SeenDbgPHIIt = SeenDbgPHIs.find(std::make_pair(&Here, InstrNum));
@@ -4094,8 +4089,8 @@ std::optional<ValueIDNum> InstrRefBasedLDV::resolveDbgPHIs(
 }
 
 std::optional<ValueIDNum> InstrRefBasedLDV::resolveDbgPHIsImpl(
-    MachineFunction &MF, const ValueTable *MLiveOuts,
-    const ValueTable *MLiveIns, MachineInstr &Here, uint64_t InstrNum) {
+    MachineFunction &MF, const FuncValueTable &MLiveOuts,
+    const FuncValueTable &MLiveIns, MachineInstr &Here, uint64_t InstrNum) {
   // Pick out records of DBG_PHI instructions that have been observed. If there
   // are none, then we cannot compute a value number.
   auto RangePair = std::equal_range(DebugPHINumToValue.begin(),
diff --git a/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.h b/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.h
index 5e94962f9d7e6485eadde52c991cb42075724e6e..d6dbb1feda3e8e6827f50003e3173edbea72163d 100644
--- a/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.h
+++ b/llvm/lib/CodeGen/LiveDebugValues/InstrRefBasedImpl.h
@@ -205,11 +205,11 @@ namespace LiveDebugValues {
 using namespace llvm;
 
 /// Type for a table of values in a block.
-using ValueTable = std::unique_ptr<ValueIDNum[]>;
+using ValueTable = SmallVector<ValueIDNum, 0>;
 
 /// Type for a table-of-table-of-values, i.e., the collection of either
 /// live-in or live-out values for each block in the function.
-using FuncValueTable = std::unique_ptr<ValueTable[]>;
+using FuncValueTable = SmallVector<ValueTable, 0>;
 
 /// Thin wrapper around an integer -- designed to give more type safety to
 /// spill location numbers.
@@ -1200,12 +1200,12 @@ private:
   /// exists, otherwise returns std::nullopt.
   std::optional<ValueIDNum> getValueForInstrRef(unsigned InstNo, unsigned OpNo,
                                                 MachineInstr &MI,
-                                                const ValueTable *MLiveOuts,
-                                                const ValueTable *MLiveIns);
+                                                const FuncValueTable *MLiveOuts,
+                                                const FuncValueTable *MLiveIns);
 
   /// Observe a single instruction while stepping through a block.
-  void process(MachineInstr &MI, const ValueTable *MLiveOuts,
-               const ValueTable *MLiveIns);
+  void process(MachineInstr &MI, const FuncValueTable *MLiveOuts,
+               const FuncValueTable *MLiveIns);
 
   /// Examines whether \p MI is a DBG_VALUE and notifies trackers.
   /// \returns true if MI was recognized and processed.
@@ -1213,8 +1213,8 @@ private:
 
   /// Examines whether \p MI is a DBG_INSTR_REF and notifies trackers.
   /// \returns true if MI was recognized and processed.
-  bool transferDebugInstrRef(MachineInstr &MI, const ValueTable *MLiveOuts,
-                             const ValueTable *MLiveIns);
+  bool transferDebugInstrRef(MachineInstr &MI, const FuncValueTable *MLiveOuts,
+                             const FuncValueTable *MLiveIns);
 
   /// Stores value-information about where this PHI occurred, and what
   /// instruction number is associated with it.
@@ -1246,14 +1246,14 @@ private:
   /// \p InstrNum Debug instruction number defined by DBG_PHI instructions.
   /// \returns The machine value number at position Here, or std::nullopt.
   std::optional<ValueIDNum> resolveDbgPHIs(MachineFunction &MF,
-                                           const ValueTable *MLiveOuts,
-                                           const ValueTable *MLiveIns,
+                                           const FuncValueTable &MLiveOuts,
+                                           const FuncValueTable &MLiveIns,
                                            MachineInstr &Here,
                                            uint64_t InstrNum);
 
   std::optional<ValueIDNum> resolveDbgPHIsImpl(MachineFunction &MF,
-                                               const ValueTable *MLiveOuts,
-                                               const ValueTable *MLiveIns,
+                                               const FuncValueTable &MLiveOuts,
+                                               const FuncValueTable &MLiveIns,
                                                MachineInstr &Here,
                                                uint64_t InstrNum);
 
diff --git a/llvm/lib/CodeGen/LiveDebugValues/VarLocBasedImpl.cpp b/llvm/lib/CodeGen/LiveDebugValues/VarLocBasedImpl.cpp
index 116c6b7e2d19efad22b780f7cda3e3c816b27938..bf730be00a9a92c93e1b4e749e1085f7529a6694 100644
--- a/llvm/lib/CodeGen/LiveDebugValues/VarLocBasedImpl.cpp
+++ b/llvm/lib/CodeGen/LiveDebugValues/VarLocBasedImpl.cpp
@@ -1364,7 +1364,7 @@ void VarLocBasedLDV::removeEntryValue(const MachineInstr &MI,
     // TODO: Try to keep tracking of an entry value if we encounter a propagated
     // DBG_VALUE describing the copy of the entry value. (Propagated entry value
     // does not indicate the parameter modification.)
-    auto DestSrc = TII->isCopyInstr(*TransferInst);
+    auto DestSrc = TII->isCopyLikeInstr(*TransferInst);
     if (DestSrc) {
       const MachineOperand *SrcRegOp, *DestRegOp;
       SrcRegOp = DestSrc->Source;
@@ -1840,7 +1840,7 @@ void VarLocBasedLDV::transferRegisterCopy(MachineInstr &MI,
                                            OpenRangesSet &OpenRanges,
                                            VarLocMap &VarLocIDs,
                                            TransferMap &Transfers) {
-  auto DestSrc = TII->isCopyInstr(MI);
+  auto DestSrc = TII->isCopyLikeInstr(MI);
   if (!DestSrc)
     return;
 
diff --git a/llvm/lib/CodeGen/MachinePassManager.cpp b/llvm/lib/CodeGen/MachinePassManager.cpp
index a13f820e457766f3e51870db2bf56d65de500969..914e6b19fde9ac8485003c73e7ccae1b8112181a 100644
--- a/llvm/lib/CodeGen/MachinePassManager.cpp
+++ b/llvm/lib/CodeGen/MachinePassManager.cpp
@@ -33,10 +33,11 @@ Error MachineFunctionPassManager::run(Module &M,
   (void)RequireCodeGenSCCOrder;
   assert(!RequireCodeGenSCCOrder && "not implemented");
 
+  // M is unused here
+  PassInstrumentation PI = MFAM.getResult<PassInstrumentationAnalysis>(M);
+
   // Add a PIC to verify machine functions.
   if (VerifyMachineFunction) {
-    PassInstrumentation PI = MFAM.getResult<PassInstrumentationAnalysis>(M);
-
     // No need to pop this callback later since MIR pipeline is flat which means
     // current pipeline is the top-level pipeline. Callbacks are not used after
     // current pipeline.
@@ -59,8 +60,11 @@ Error MachineFunctionPassManager::run(Module &M,
   do {
     // Run machine module passes
     for (; MachineModulePasses.count(Idx) && Idx != Size; ++Idx) {
+      if (!PI.runBeforePass<Module>(*Passes[Idx], M))
+        continue;
       if (auto Err = MachineModulePasses.at(Idx)(M, MFAM))
         return Err;
+      PI.runAfterPass(*Passes[Idx], M, PreservedAnalyses::all());
     }
 
     // Finish running all passes.
@@ -81,7 +85,6 @@ Error MachineFunctionPassManager::run(Module &M,
         continue;
 
       MachineFunction &MF = MMI.getOrCreateMachineFunction(F);
-      PassInstrumentation PI = MFAM.getResult<PassInstrumentationAnalysis>(MF);
 
       for (unsigned I = Begin, E = Idx; I != E; ++I) {
         auto *P = Passes[I].get();
diff --git a/llvm/lib/CodeGen/MachineSink.cpp b/llvm/lib/CodeGen/MachineSink.cpp
index 83d775055dfd7334ae769c92fcadfccf31422511..e7e8f6026834800e2d53dedbeb5d8e2cf7bdad0e 100644
--- a/llvm/lib/CodeGen/MachineSink.cpp
+++ b/llvm/lib/CodeGen/MachineSink.cpp
@@ -500,11 +500,6 @@ bool MachineSinking::PerformSinkAndFold(MachineInstr &MI,
     return false;
 
   // Now we know we can fold the instruction in all its users.
-  if (UsedRegA)
-    MRI->clearKillFlags(UsedRegA);
-  if (UsedRegB)
-    MRI->clearKillFlags(UsedRegB);
-
   for (auto &[SinkDst, MaybeAM] : SinkInto) {
     MachineInstr *New = nullptr;
     LLVM_DEBUG(dbgs() << "Sinking copy of"; MI.dump(); dbgs() << "into";
@@ -527,9 +522,25 @@ bool MachineSinking::PerformSinkAndFold(MachineInstr &MI,
       New = &*std::prev(InsertPt);
       if (!New->getDebugLoc())
         New->setDebugLoc(SinkDst->getDebugLoc());
+
+      // The operand registers of the "sunk" instruction have their live range
+      // extended and their kill flags may no longer be correct. Conservatively
+      // clear the kill flags.
+      if (UsedRegA)
+        MRI->clearKillFlags(UsedRegA);
+      if (UsedRegB)
+        MRI->clearKillFlags(UsedRegB);
     } else {
       // Fold instruction into the addressing mode of a memory instruction.
       New = TII->emitLdStWithAddr(*SinkDst, MaybeAM);
+
+      // The registers of the addressing mode may have their live range extended
+      // and their kill flags may no longer be correct. Conservatively clear the
+      // kill flags.
+      if (Register R = MaybeAM.BaseReg; R.isValid() && R.isVirtual())
+        MRI->clearKillFlags(R);
+      if (Register R = MaybeAM.ScaledReg; R.isValid() && R.isVirtual())
+        MRI->clearKillFlags(R);
     }
     LLVM_DEBUG(dbgs() << "yielding"; New->dump());
     // Clear the StoreInstrCache, since we may invalidate it by erasing.
diff --git a/llvm/lib/CodeGen/RegisterCoalescer.cpp b/llvm/lib/CodeGen/RegisterCoalescer.cpp
index c067d87a9fd810827716b2e878f353460054eef2..c1af37c8510ff54b40e1d0e36bfd6dd9aa28818f 100644
--- a/llvm/lib/CodeGen/RegisterCoalescer.cpp
+++ b/llvm/lib/CodeGen/RegisterCoalescer.cpp
@@ -1201,6 +1201,8 @@ bool RegisterCoalescer::removePartialRedundancy(const CoalescerPair &CP,
                       << printMBBReference(MBB) << '\t' << CopyMI);
   }
 
+  const bool IsUndefCopy = CopyMI.getOperand(1).isUndef();
+
   // Remove CopyMI.
   // Note: This is fine to remove the copy before updating the live-ranges.
   // While updating the live-ranges, we only look at slot indices and
@@ -1214,6 +1216,19 @@ bool RegisterCoalescer::removePartialRedundancy(const CoalescerPair &CP,
   LIS->pruneValue(*static_cast<LiveRange *>(&IntB), CopyIdx.getRegSlot(),
                   &EndPoints);
   BValNo->markUnused();
+
+  if (IsUndefCopy) {
+    // We're introducing an undef phi def, and need to set undef on any users of
+    // the previously local def to avoid artifically extending the lifetime
+    // through the block.
+    for (MachineOperand &MO : MRI->use_nodbg_operands(IntB.reg())) {
+      const MachineInstr &MI = *MO.getParent();
+      SlotIndex UseIdx = LIS->getInstructionIndex(MI);
+      if (!IntB.liveAt(UseIdx))
+        MO.setIsUndef(true);
+    }
+  }
+
   // Extend IntB to the EndPoints of its original live interval.
   LIS->extendToIndices(IntB, EndPoints);
 
@@ -1596,8 +1611,7 @@ bool RegisterCoalescer::reMaterializeTrivialDef(const CoalescerPair &CP,
         LR->createDeadDef(NewMIIdx.getRegSlot(), LIS->getVNInfoAllocator());
   }
 
-  if (NewMI.getOperand(0).getSubReg())
-    NewMI.getOperand(0).setIsUndef();
+  NewMI.setRegisterDefReadUndef(NewMI.getOperand(0).getReg());
 
   // Transfer over implicit operands to the rematerialized instruction.
   for (MachineOperand &MO : ImplicitOps)
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index f35f663d6ba1b43e899843a22fb0c316cd05dfcd..ba8591529399b70e8ed7e027d26e8cc153977e55 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -14821,12 +14821,11 @@ SDValue DAGCombiner::visitTRUNCATE(SDNode *N) {
     // Handle the case where the load remains an extending load even
     // after truncation.
     if (N0.hasOneUse() && ISD::isUNINDEXEDLoad(N0.getNode())) {
-      LoadSDNode *LN0 = cast<LoadSDNode>(N0);
+      auto *LN0 = cast<LoadSDNode>(N0);
       if (LN0->isSimple() && LN0->getMemoryVT().bitsLT(VT)) {
-        SDValue NewLoad = DAG.getExtLoad(LN0->getExtensionType(), SDLoc(LN0),
-                                         VT, LN0->getChain(), LN0->getBasePtr(),
-                                         LN0->getMemoryVT(),
-                                         LN0->getMemOperand());
+        SDValue NewLoad = DAG.getExtLoad(
+            LN0->getExtensionType(), SDLoc(LN0), VT, LN0->getChain(),
+            LN0->getBasePtr(), LN0->getMemoryVT(), LN0->getMemOperand());
         DAG.ReplaceAllUsesOfValueWith(N0.getValue(1), NewLoad.getValue(1));
         return NewLoad;
       }
diff --git a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
index 27d47aab4b6838d587ab26307bfc3248594bb563..18f4d546aa82d714892fd64575219920b474e721 100644
--- a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp
@@ -645,7 +645,7 @@ getELFSectionNameForGlobal(const GlobalObject *GO, SectionKind Kind,
     Name = ".rodata.cst";
     Name += utostr(EntrySize);
   } else {
-    Name = getSectionPrefixForGlobal(Kind, TM.isLargeGlobalObject(GO));
+    Name = getSectionPrefixForGlobal(Kind, TM.isLargeGlobalValue(GO));
   }
 
   bool HasPrefix = false;
@@ -765,7 +765,7 @@ getGlobalObjectInfo(const GlobalObject *GO, const TargetMachine &TM) {
     Group = C->getName();
     IsComdat = C->getSelectionKind() == Comdat::Any;
   }
-  if (TM.isLargeGlobalObject(GO))
+  if (TM.isLargeGlobalValue(GO))
     Flags |= ELF::SHF_X86_64_LARGE;
   return {Group, IsComdat, Flags};
 }
diff --git a/llvm/lib/CodeGen/TargetPassConfig.cpp b/llvm/lib/CodeGen/TargetPassConfig.cpp
index b82fe5fb99f98da52eca7723e88ee9af51c22711..f15d59d4e35b498205c534e6e8e6fac69e9b1247 100644
--- a/llvm/lib/CodeGen/TargetPassConfig.cpp
+++ b/llvm/lib/CodeGen/TargetPassConfig.cpp
@@ -868,7 +868,7 @@ void TargetPassConfig::addIRPasses() {
     // target lowering hook.
     if (!DisableMergeICmps)
       addPass(createMergeICmpsLegacyPass());
-    addPass(createExpandMemCmpPass());
+    addPass(createExpandMemCmpLegacyPass());
   }
 
   // Run GC lowering passes for builtin collectors
diff --git a/llvm/lib/DebugInfo/Symbolize/MarkupFilter.cpp b/llvm/lib/DebugInfo/Symbolize/MarkupFilter.cpp
index f7503ef49693d8dfd3a1ac5b67a80ace0a89beec..01d49709f9b29c42184958a5d31b73a99a306209 100644
--- a/llvm/lib/DebugInfo/Symbolize/MarkupFilter.cpp
+++ b/llvm/lib/DebugInfo/Symbolize/MarkupFilter.cpp
@@ -41,8 +41,8 @@ MarkupFilter::MarkupFilter(raw_ostream &OS, LLVMSymbolizer &Symbolizer,
       ColorsEnabled(
           ColorsEnabled.value_or(WithColor::defaultAutoDetectFunction()(OS))) {}
 
-void MarkupFilter::filter(StringRef Line) {
-  this->Line = Line;
+void MarkupFilter::filter(std::string &&InputLine) {
+  Line = std::move(InputLine);
   resetColor();
 
   Parser.parseLine(Line);
@@ -695,7 +695,9 @@ void MarkupFilter::reportTypeError(StringRef Str, StringRef TypeName) const {
 // passed to beginLine().
 void MarkupFilter::reportLocation(StringRef::iterator Loc) const {
   errs() << Line;
-  WithColor(errs().indent(Loc - Line.begin()), HighlightColor::String) << '^';
+  WithColor(errs().indent(Loc - StringRef(Line).begin()),
+            HighlightColor::String)
+      << '^';
   errs() << '\n';
 }
 
@@ -741,7 +743,7 @@ uint64_t MarkupFilter::adjustAddr(uint64_t Addr, PCType Type) const {
 }
 
 StringRef MarkupFilter::lineEnding() const {
-  return Line.ends_with("\r\n") ? "\r\n" : "\n";
+  return StringRef(Line).ends_with("\r\n") ? "\r\n" : "\n";
 }
 
 bool MarkupFilter::MMap::contains(uint64_t Addr) const {
diff --git a/llvm/lib/IR/Attributes.cpp b/llvm/lib/IR/Attributes.cpp
index 10a74ed68a392c607e69e81705bbade953707e11..fd5160209506f23c4360debbde71a72bf54a4328 100644
--- a/llvm/lib/IR/Attributes.cpp
+++ b/llvm/lib/IR/Attributes.cpp
@@ -1962,7 +1962,8 @@ AttributeMask AttributeFuncs::typeIncompatible(Type *Ty,
           .addAttribute(Attribute::ReadOnly)
           .addAttribute(Attribute::Dereferenceable)
           .addAttribute(Attribute::DereferenceableOrNull)
-          .addAttribute(Attribute::Writable);
+          .addAttribute(Attribute::Writable)
+          .addAttribute(Attribute::DeadOnUnwind);
     if (ASK & ASK_UNSAFE_TO_DROP)
       Incompatible.addAttribute(Attribute::Nest)
           .addAttribute(Attribute::SwiftError)
diff --git a/llvm/lib/IR/BasicBlock.cpp b/llvm/lib/IR/BasicBlock.cpp
index f364c56a42c5281a737611f219f311c3d828da53..03b74b0480f0713b16ca5bd6256e0da2c7c67b35 100644
--- a/llvm/lib/IR/BasicBlock.cpp
+++ b/llvm/lib/IR/BasicBlock.cpp
@@ -73,7 +73,10 @@ void BasicBlock::convertToNewDbgValues() {
   SmallVector<DPValue *, 4> DPVals;
   for (Instruction &I : make_early_inc_range(InstList)) {
     assert(!I.DbgMarker && "DbgMarker already set on old-format instrs?");
-    if (DbgValueInst *DVI = dyn_cast<DbgValueInst>(&I)) {
+    if (DbgVariableIntrinsic *DVI = dyn_cast<DbgVariableIntrinsic>(&I)) {
+      if (isa<DbgAssignIntrinsic>(DVI))
+        continue;
+
       // Convert this dbg.value to a DPValue.
       DPValue *Value = new DPValue(DVI);
       DPVals.push_back(Value);
@@ -1088,6 +1091,8 @@ void BasicBlock::insertDPValueBefore(DPValue *DPV,
   // shouldn't be generated at times when there's no terminator.
   assert(Where != end());
   assert(Where->getParent() == this);
+  if (!Where->DbgMarker)
+    createMarker(Where);
   bool InsertAtHead = Where.getHeadBit();
   Where->DbgMarker->insertDPValue(DPV, InsertAtHead);
 }
diff --git a/llvm/lib/IR/Operator.cpp b/llvm/lib/IR/Operator.cpp
index 017641a8dba19d4a8f14ea4be3c1d31f395170c7..cd982c7da102af47b0e7809f9c4e9ed7fd962fff 100644
--- a/llvm/lib/IR/Operator.cpp
+++ b/llvm/lib/IR/Operator.cpp
@@ -229,8 +229,8 @@ bool GEPOperator::collectOffset(
     // Insert an initial offset of 0 for V iff none exists already, then
     // increment the offset by IndexedSize.
     if (!IndexedSize.isZero()) {
-      VariableOffsets.insert({V, APInt(BitWidth, 0)});
-      VariableOffsets[V] += IndexedSize;
+      auto *It = VariableOffsets.insert({V, APInt(BitWidth, 0)}).first;
+      It->second += IndexedSize;
     }
   }
   return true;
diff --git a/llvm/lib/ObjCopy/MachO/MachOLayoutBuilder.cpp b/llvm/lib/ObjCopy/MachO/MachOLayoutBuilder.cpp
index 067ef39d90522fe74c2f015672c714179f98c648..a3d4ba3a94f7ac58f8548d4350cc5dcc2da37c44 100644
--- a/llvm/lib/ObjCopy/MachO/MachOLayoutBuilder.cpp
+++ b/llvm/lib/ObjCopy/MachO/MachOLayoutBuilder.cpp
@@ -10,6 +10,7 @@
 #include "llvm/Support/Alignment.h"
 #include "llvm/Support/Errc.h"
 #include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/SystemZ/zOSSupport.h"
 
 using namespace llvm;
 using namespace llvm::objcopy::macho;
diff --git a/llvm/lib/ObjCopy/MachO/MachOObject.cpp b/llvm/lib/ObjCopy/MachO/MachOObject.cpp
index 9a4abadc8710a4d42b822b62e927b4aaf155c379..d593d6788e112fa5086ef22df76c11d4519fcba2 100644
--- a/llvm/lib/ObjCopy/MachO/MachOObject.cpp
+++ b/llvm/lib/ObjCopy/MachO/MachOObject.cpp
@@ -8,6 +8,7 @@
 
 #include "MachOObject.h"
 #include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/Support/SystemZ/zOSSupport.h"
 #include <unordered_set>
 
 using namespace llvm;
diff --git a/llvm/lib/ObjCopy/MachO/MachOReader.cpp b/llvm/lib/ObjCopy/MachO/MachOReader.cpp
index 25f8c020cde94d519f405169e6c8c0a41ff359f1..4549977c12c3db382d22f69ad1ce16fd5e89c984 100644
--- a/llvm/lib/ObjCopy/MachO/MachOReader.cpp
+++ b/llvm/lib/ObjCopy/MachO/MachOReader.cpp
@@ -11,6 +11,7 @@
 #include "llvm/BinaryFormat/MachO.h"
 #include "llvm/Object/MachO.h"
 #include "llvm/Support/Errc.h"
+#include "llvm/Support/SystemZ/zOSSupport.h"
 #include <memory>
 
 using namespace llvm;
diff --git a/llvm/lib/Object/Binary.cpp b/llvm/lib/Object/Binary.cpp
index 0ee9f7fac448a26a0abe2268b4543b8c81fc8c28..0b9d95485287dc3cca6b2672bf0010845e5adb6f 100644
--- a/llvm/lib/Object/Binary.cpp
+++ b/llvm/lib/Object/Binary.cpp
@@ -89,6 +89,7 @@ Expected<std::unique_ptr<Binary>> object::createBinary(MemoryBufferRef Buffer,
   case file_magic::dxcontainer_object:
   case file_magic::offload_bundle:
   case file_magic::offload_bundle_compressed:
+  case file_magic::spirv_object:
     // Unrecognized object file format.
     return errorCodeToError(object_error::invalid_file_type);
   case file_magic::offload_binary:
diff --git a/llvm/lib/Object/ObjectFile.cpp b/llvm/lib/Object/ObjectFile.cpp
index 428166f58070d0bcbf745696d5c54bf660efaf54..ca921836b7f65a809a9e04c6ed6bf667b7c3aa23 100644
--- a/llvm/lib/Object/ObjectFile.cpp
+++ b/llvm/lib/Object/ObjectFile.cpp
@@ -160,6 +160,7 @@ ObjectFile::createObjectFile(MemoryBufferRef Object, file_magic Type,
   case file_magic::dxcontainer_object:
   case file_magic::offload_bundle:
   case file_magic::offload_bundle_compressed:
+  case file_magic::spirv_object:
     return errorCodeToError(object_error::invalid_file_type);
   case file_magic::tapi_file:
     return errorCodeToError(object_error::invalid_file_type);
diff --git a/llvm/lib/ObjectYAML/CMakeLists.txt b/llvm/lib/ObjectYAML/CMakeLists.txt
index c081009653d4f8c6797f0ba02c1e304328757810..b36974d47d9f8976f17aacd277066973c3b9dd3d 100644
--- a/llvm/lib/ObjectYAML/CMakeLists.txt
+++ b/llvm/lib/ObjectYAML/CMakeLists.txt
@@ -13,6 +13,8 @@ add_llvm_component_library(LLVMObjectYAML
   DXContainerYAML.cpp
   ELFEmitter.cpp
   ELFYAML.cpp
+  GOFFEmitter.cpp
+  GOFFYAML.cpp
   MachOEmitter.cpp
   MachOYAML.cpp
   ObjectYAML.cpp
diff --git a/llvm/lib/ObjectYAML/GOFFEmitter.cpp b/llvm/lib/ObjectYAML/GOFFEmitter.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..345904407e1d24d6e571c28f84d5a1fd66d4094d
--- /dev/null
+++ b/llvm/lib/ObjectYAML/GOFFEmitter.cpp
@@ -0,0 +1,282 @@
+//===- yaml2goff - Convert YAML to a GOFF object file ---------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// The GOFF component of yaml2obj.
+///
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/IndexedMap.h"
+#include "llvm/ObjectYAML/ObjectYAML.h"
+#include "llvm/ObjectYAML/yaml2obj.h"
+#include "llvm/Support/ConvertEBCDIC.h"
+#include "llvm/Support/Endian.h"
+#include "llvm/Support/raw_ostream.h"
+
+using namespace llvm;
+
+namespace {
+
+// Common flag values on records.
+enum {
+  // Flag: This record is continued.
+  Rec_Continued = 1,
+
+  // Flag: This record is a continuation.
+  Rec_Continuation = 1 << (8 - 6 - 1),
+};
+
+template <typename ValueType> struct BinaryBeImpl {
+  ValueType Value;
+  BinaryBeImpl(ValueType V) : Value(V) {}
+};
+
+template <typename ValueType>
+raw_ostream &operator<<(raw_ostream &OS, const BinaryBeImpl<ValueType> &BBE) {
+  char Buffer[sizeof(BBE.Value)];
+  support::endian::write<ValueType, llvm::endianness::big, support::unaligned>(
+      Buffer, BBE.Value);
+  OS.write(Buffer, sizeof(BBE.Value));
+  return OS;
+}
+
+template <typename ValueType> BinaryBeImpl<ValueType> binaryBe(ValueType V) {
+  return BinaryBeImpl<ValueType>(V);
+}
+
+struct ZerosImpl {
+  size_t NumBytes;
+};
+
+raw_ostream &operator<<(raw_ostream &OS, const ZerosImpl &Z) {
+  OS.write_zeros(Z.NumBytes);
+  return OS;
+}
+
+ZerosImpl zeros(const size_t NumBytes) { return ZerosImpl{NumBytes}; }
+
+// The GOFFOstream is responsible to write the data into the fixed physical
+// records of the format. A user of this class announces the start of a new
+// logical record and the size of its payload. While writing the payload, the
+// physical records are created for the data. Possible fill bytes at the end of
+// a physical record are written automatically.
+class GOFFOstream : public raw_ostream {
+public:
+  explicit GOFFOstream(raw_ostream &OS)
+      : OS(OS), LogicalRecords(0), RemainingSize(0), NewLogicalRecord(false) {
+    SetBufferSize(GOFF::PayloadLength);
+  }
+
+  ~GOFFOstream() { finalize(); }
+
+  void makeNewRecord(GOFF::RecordType Type, size_t Size) {
+    fillRecord();
+    CurrentType = Type;
+    RemainingSize = Size;
+    if (size_t Gap = (RemainingSize % GOFF::PayloadLength))
+      RemainingSize += GOFF::PayloadLength - Gap;
+    NewLogicalRecord = true;
+    ++LogicalRecords;
+  }
+
+  void finalize() { fillRecord(); }
+
+  uint32_t logicalRecords() { return LogicalRecords; }
+
+private:
+  // The underlying raw_ostream.
+  raw_ostream &OS;
+
+  // The number of logical records emitted so far.
+  uint32_t LogicalRecords;
+
+  // The remaining size of this logical record, including fill bytes.
+  size_t RemainingSize;
+
+  // The type of the current (logical) record.
+  GOFF::RecordType CurrentType;
+
+  // Signals start of new record.
+  bool NewLogicalRecord;
+
+  // Return the number of bytes left to write until next physical record.
+  // Please note that we maintain the total number of bytes left, not the
+  // written size.
+  size_t bytesToNextPhysicalRecord() {
+    size_t Bytes = RemainingSize % GOFF::PayloadLength;
+    return Bytes ? Bytes : GOFF::PayloadLength;
+  }
+
+  // Write the record prefix of a physical record, using the current record
+  // type.
+  static void writeRecordPrefix(raw_ostream &OS, GOFF::RecordType Type,
+                                size_t RemainingSize,
+                                uint8_t Flags = Rec_Continuation) {
+    uint8_t TypeAndFlags = Flags | (Type << 4);
+    if (RemainingSize > GOFF::RecordLength)
+      TypeAndFlags |= Rec_Continued;
+    OS << binaryBe(static_cast<unsigned char>(GOFF::PTVPrefix))
+       << binaryBe(static_cast<unsigned char>(TypeAndFlags))
+       << binaryBe(static_cast<unsigned char>(0));
+  }
+
+  // Fill the last physical record of a logical record with zero bytes.
+  void fillRecord() {
+    assert((GetNumBytesInBuffer() <= RemainingSize) &&
+           "More bytes in buffer than expected");
+    size_t Remains = RemainingSize - GetNumBytesInBuffer();
+    if (Remains) {
+      assert((Remains < GOFF::RecordLength) &&
+             "Attempting to fill more than one physical record");
+      raw_ostream::write_zeros(Remains);
+    }
+    flush();
+    assert(RemainingSize == 0 && "Not fully flushed");
+    assert(GetNumBytesInBuffer() == 0 && "Buffer not fully empty");
+  }
+
+  // See raw_ostream::write_impl.
+  void write_impl(const char *Ptr, size_t Size) override {
+    assert((RemainingSize >= Size) && "Attempt to write too much data");
+    assert(RemainingSize && "Logical record overflow");
+    if (!(RemainingSize % GOFF::PayloadLength)) {
+      writeRecordPrefix(OS, CurrentType, RemainingSize,
+                        NewLogicalRecord ? 0 : Rec_Continuation);
+      NewLogicalRecord = false;
+    }
+    assert(!NewLogicalRecord &&
+           "New logical record not on physical record boundary");
+
+    size_t Idx = 0;
+    while (Size > 0) {
+      size_t BytesToWrite = bytesToNextPhysicalRecord();
+      if (BytesToWrite > Size)
+        BytesToWrite = Size;
+      OS.write(Ptr + Idx, BytesToWrite);
+      Idx += BytesToWrite;
+      Size -= BytesToWrite;
+      RemainingSize -= BytesToWrite;
+      if (Size) {
+        writeRecordPrefix(OS, CurrentType, RemainingSize);
+      }
+    }
+  }
+
+  // Return the current position within the stream, not counting the bytes
+  // currently in the buffer.
+  uint64_t current_pos() const override { return OS.tell(); }
+};
+
+class GOFFState {
+  void writeHeader(GOFFYAML::FileHeader &FileHdr);
+  void writeEnd();
+
+  void reportError(const Twine &Msg) {
+    ErrHandler(Msg);
+    HasError = true;
+  }
+
+  GOFFState(raw_ostream &OS, GOFFYAML::Object &Doc,
+            yaml::ErrorHandler ErrHandler)
+      : GW(OS), Doc(Doc), ErrHandler(ErrHandler), HasError(false) {}
+
+  ~GOFFState() { GW.finalize(); }
+
+  bool writeObject();
+
+public:
+  static bool writeGOFF(raw_ostream &OS, GOFFYAML::Object &Doc,
+                        yaml::ErrorHandler ErrHandler);
+
+private:
+  GOFFOstream GW;
+  GOFFYAML::Object &Doc;
+  yaml::ErrorHandler ErrHandler;
+  bool HasError;
+};
+
+void GOFFState::writeHeader(GOFFYAML::FileHeader &FileHdr) {
+  SmallString<16> CCSIDName;
+  if (std::error_code EC =
+          ConverterEBCDIC::convertToEBCDIC(FileHdr.CharacterSetName, CCSIDName))
+    reportError("Conversion error on " + FileHdr.CharacterSetName);
+  if (CCSIDName.size() > 16) {
+    reportError("CharacterSetName too long");
+    CCSIDName.resize(16);
+  }
+  SmallString<16> LangProd;
+  if (std::error_code EC = ConverterEBCDIC::convertToEBCDIC(
+          FileHdr.LanguageProductIdentifier, LangProd))
+    reportError("Conversion error on " + FileHdr.LanguageProductIdentifier);
+  if (LangProd.size() > 16) {
+    reportError("LanguageProductIdentifier too long");
+    LangProd.resize(16);
+  }
+
+  GW.makeNewRecord(GOFF::RT_HDR, GOFF::PayloadLength);
+  GW << binaryBe(FileHdr.TargetEnvironment)     // TargetEnvironment
+     << binaryBe(FileHdr.TargetOperatingSystem) // TargetOperatingSystem
+     << zeros(2)                                // Reserved
+     << binaryBe(FileHdr.CCSID)                 // CCSID
+     << CCSIDName                               // CharacterSetName
+     << zeros(16 - CCSIDName.size())            // Fill bytes
+     << LangProd                                // LanguageProductIdentifier
+     << zeros(16 - LangProd.size())             // Fill bytes
+     << binaryBe(FileHdr.ArchitectureLevel);    // ArchitectureLevel
+  // The module propties are optional. Figure out if we need to write them.
+  uint16_t ModPropLen = 0;
+  if (FileHdr.TargetSoftwareEnvironment)
+    ModPropLen = 3;
+  else if (FileHdr.InternalCCSID)
+    ModPropLen = 2;
+  if (ModPropLen) {
+    GW << binaryBe(ModPropLen) << zeros(6);
+    if (ModPropLen >= 2)
+      GW << binaryBe(FileHdr.InternalCCSID ? *FileHdr.InternalCCSID : 0);
+    if (ModPropLen >= 3)
+      GW << binaryBe(FileHdr.TargetSoftwareEnvironment
+                         ? *FileHdr.TargetSoftwareEnvironment
+                         : 0);
+  }
+}
+
+void GOFFState::writeEnd() {
+  GW.makeNewRecord(GOFF::RT_END, GOFF::PayloadLength);
+  GW << binaryBe(uint8_t(0)) // No entry point
+     << binaryBe(uint8_t(0)) // No AMODE
+     << zeros(3)             // Reserved
+     << binaryBe(GW.logicalRecords());
+  // No entry point yet. Automatically fill remaining space with zero bytes.
+  GW.finalize();
+}
+
+bool GOFFState::writeObject() {
+  writeHeader(Doc.Header);
+  if (HasError)
+    return false;
+  writeEnd();
+  return true;
+}
+
+bool GOFFState::writeGOFF(raw_ostream &OS, GOFFYAML::Object &Doc,
+                          yaml::ErrorHandler ErrHandler) {
+  GOFFState State(OS, Doc, ErrHandler);
+  return State.writeObject();
+}
+} // namespace
+
+namespace llvm {
+namespace yaml {
+
+bool yaml2goff(llvm::GOFFYAML::Object &Doc, raw_ostream &Out,
+               ErrorHandler ErrHandler) {
+  return GOFFState::writeGOFF(Out, Doc, ErrHandler);
+}
+
+} // namespace yaml
+} // namespace llvm
diff --git a/llvm/lib/ObjectYAML/GOFFYAML.cpp b/llvm/lib/ObjectYAML/GOFFYAML.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..ae857980a521b00aee2a15b619d1399cfbac2f4d
--- /dev/null
+++ b/llvm/lib/ObjectYAML/GOFFYAML.cpp
@@ -0,0 +1,46 @@
+//===-- GOFFYAML.cpp - GOFF YAMLIO implementation ---------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines classes for handling the YAML representation of GOFF.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ObjectYAML/GOFFYAML.h"
+#include "llvm/BinaryFormat/GOFF.h"
+#include <string.h>
+
+namespace llvm {
+namespace GOFFYAML {
+
+Object::Object() {}
+
+} // namespace GOFFYAML
+
+namespace yaml {
+
+void MappingTraits<GOFFYAML::FileHeader>::mapping(
+    IO &IO, GOFFYAML::FileHeader &FileHdr) {
+  IO.mapOptional("TargetEnvironment", FileHdr.TargetEnvironment, 0);
+  IO.mapOptional("TargetOperatingSystem", FileHdr.TargetOperatingSystem, 0);
+  IO.mapOptional("CCSID", FileHdr.CCSID, 0);
+  IO.mapOptional("CharacterSetName", FileHdr.CharacterSetName, "");
+  IO.mapOptional("LanguageProductIdentifier", FileHdr.LanguageProductIdentifier,
+                 "");
+  IO.mapOptional("ArchitectureLevel", FileHdr.ArchitectureLevel, 1);
+  IO.mapOptional("InternalCCSID", FileHdr.InternalCCSID);
+  IO.mapOptional("TargetSoftwareEnvironment",
+                 FileHdr.TargetSoftwareEnvironment);
+}
+
+void MappingTraits<GOFFYAML::Object>::mapping(IO &IO, GOFFYAML::Object &Obj) {
+  IO.mapTag("!GOFF", true);
+  IO.mapRequired("FileHeader", Obj.Header);
+}
+
+} // namespace yaml
+} // namespace llvm
diff --git a/llvm/lib/ObjectYAML/MachOEmitter.cpp b/llvm/lib/ObjectYAML/MachOEmitter.cpp
index 6bcc2cee27edb6c4383cdf574e2413c8ce0d0498..c08b389daea9c36c93c67ad636e461a93e43286e 100644
--- a/llvm/lib/ObjectYAML/MachOEmitter.cpp
+++ b/llvm/lib/ObjectYAML/MachOEmitter.cpp
@@ -19,6 +19,7 @@
 #include "llvm/Support/Error.h"
 #include "llvm/Support/FormatVariadic.h"
 #include "llvm/Support/LEB128.h"
+#include "llvm/Support/SystemZ/zOSSupport.h"
 #include "llvm/Support/YAMLTraits.h"
 #include "llvm/Support/raw_ostream.h"
 
diff --git a/llvm/lib/ObjectYAML/MachOYAML.cpp b/llvm/lib/ObjectYAML/MachOYAML.cpp
index 86342c5501c708f479918a59d951e0b42e6b0dde..82b2eaecec9be9a51bc55d96e9795d823bacbb14 100644
--- a/llvm/lib/ObjectYAML/MachOYAML.cpp
+++ b/llvm/lib/ObjectYAML/MachOYAML.cpp
@@ -14,6 +14,7 @@
 #include "llvm/ADT/StringRef.h"
 #include "llvm/BinaryFormat/MachO.h"
 #include "llvm/Support/Format.h"
+#include "llvm/Support/SystemZ/zOSSupport.h"
 #include "llvm/Support/YAMLTraits.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/TargetParser/Host.h"
diff --git a/llvm/lib/ObjectYAML/ObjectYAML.cpp b/llvm/lib/ObjectYAML/ObjectYAML.cpp
index d57e5583016b507cf22a47fbf246985571de968e..1815eaff8e36ddeb088d9673b7c5e872e025e8b0 100644
--- a/llvm/lib/ObjectYAML/ObjectYAML.cpp
+++ b/llvm/lib/ObjectYAML/ObjectYAML.cpp
@@ -26,6 +26,8 @@ void MappingTraits<YamlObjectFile>::mapping(IO &IO,
       MappingTraits<ELFYAML::Object>::mapping(IO, *ObjectFile.Elf);
     if (ObjectFile.Coff)
       MappingTraits<COFFYAML::Object>::mapping(IO, *ObjectFile.Coff);
+    if (ObjectFile.Goff)
+      MappingTraits<GOFFYAML::Object>::mapping(IO, *ObjectFile.Goff);
     if (ObjectFile.MachO)
       MappingTraits<MachOYAML::Object>::mapping(IO, *ObjectFile.MachO);
     if (ObjectFile.FatMachO)
@@ -46,6 +48,9 @@ void MappingTraits<YamlObjectFile>::mapping(IO &IO,
     } else if (IO.mapTag("!COFF")) {
       ObjectFile.Coff.reset(new COFFYAML::Object());
       MappingTraits<COFFYAML::Object>::mapping(IO, *ObjectFile.Coff);
+    } else if (IO.mapTag("!GOFF")) {
+      ObjectFile.Goff.reset(new GOFFYAML::Object());
+      MappingTraits<GOFFYAML::Object>::mapping(IO, *ObjectFile.Goff);
     } else if (IO.mapTag("!mach-o")) {
       ObjectFile.MachO.reset(new MachOYAML::Object());
       MappingTraits<MachOYAML::Object>::mapping(IO, *ObjectFile.MachO);
diff --git a/llvm/lib/ObjectYAML/yaml2obj.cpp b/llvm/lib/ObjectYAML/yaml2obj.cpp
index 06050e246fbff32cb51a161f6fdfe532527600f4..b9a9ad6397099566ca5fe1d959399b5539490069 100644
--- a/llvm/lib/ObjectYAML/yaml2obj.cpp
+++ b/llvm/lib/ObjectYAML/yaml2obj.cpp
@@ -38,6 +38,8 @@ bool convertYAML(yaml::Input &YIn, raw_ostream &Out, ErrorHandler ErrHandler,
       return yaml2elf(*Doc.Elf, Out, ErrHandler, MaxSize);
     if (Doc.Coff)
       return yaml2coff(*Doc.Coff, Out, ErrHandler);
+    if (Doc.Goff)
+      return yaml2goff(*Doc.Goff, Out, ErrHandler);
     if (Doc.MachO || Doc.FatMachO)
       return yaml2macho(Doc, Out, ErrHandler);
     if (Doc.Minidump)
diff --git a/llvm/lib/Passes/CMakeLists.txt b/llvm/lib/Passes/CMakeLists.txt
index e42edfe949697455233d05e150d99aa45ea1b66b..98d2de76c0e11473e60e7272b007224a457c3bff 100644
--- a/llvm/lib/Passes/CMakeLists.txt
+++ b/llvm/lib/Passes/CMakeLists.txt
@@ -16,6 +16,7 @@ add_llvm_component_library(LLVMPasses
   LINK_COMPONENTS
   AggressiveInstCombine
   Analysis
+  CFGuard
   CodeGen
   Core
   Coroutines
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index c48e591fc600d5f79001b2edd3782d3b83254ece..95b9fb7ad735004fb77d980b9e6293967df8f6b4 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -76,8 +76,12 @@
 #include "llvm/CodeGen/DwarfEHPrepare.h"
 #include "llvm/CodeGen/ExpandLargeDivRem.h"
 #include "llvm/CodeGen/ExpandLargeFpConvert.h"
+#include "llvm/CodeGen/ExpandMemCmp.h"
+#include "llvm/CodeGen/GCMetadata.h"
 #include "llvm/CodeGen/HardwareLoops.h"
+#include "llvm/CodeGen/IndirectBrExpand.h"
 #include "llvm/CodeGen/InterleavedAccess.h"
+#include "llvm/CodeGen/InterleavedLoadCombine.h"
 #include "llvm/CodeGen/JMCInstrumenter.h"
 #include "llvm/CodeGen/SafeStack.h"
 #include "llvm/CodeGen/SelectOptimize.h"
@@ -100,6 +104,7 @@
 #include "llvm/Support/Regex.h"
 #include "llvm/Target/TargetMachine.h"
 #include "llvm/Transforms/AggressiveInstCombine/AggressiveInstCombine.h"
+#include "llvm/Transforms/CFGuard.h"
 #include "llvm/Transforms/Coroutines/CoroCleanup.h"
 #include "llvm/Transforms/Coroutines/CoroConditionalWrapper.h"
 #include "llvm/Transforms/Coroutines/CoroEarly.h"
@@ -738,6 +743,26 @@ Expected<bool> parsePostOrderFunctionAttrsPassOptions(StringRef Params) {
                                "PostOrderFunctionAttrs");
 }
 
+Expected<CFGuardPass::Mechanism> parseCFGuardPassOptions(StringRef Params) {
+  if (Params.empty())
+    return CFGuardPass::Mechanism::Check;
+
+  auto [Param, RHS] = Params.split(';');
+  if (!RHS.empty())
+    return make_error<StringError>(
+        formatv("too many CFGuardPass parameters '{0}' ", Params).str(),
+        inconvertibleErrorCode());
+
+  if (Param == "check")
+    return CFGuardPass::Mechanism::Check;
+  if (Param == "dispatch")
+    return CFGuardPass::Mechanism::Dispatch;
+
+  return make_error<StringError>(
+      formatv("invalid CFGuardPass mechanism: '{0}' ", Param).str(),
+      inconvertibleErrorCode());
+}
+
 Expected<bool> parseEarlyCSEPassOptions(StringRef Params) {
   return parseSinglePassOption(Params, "memssa", "EarlyCSE");
 }
diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def
index 6afc8b4898fefd2556d9e781349a761cf9dbbe37..d8fc7cd8a231ff4d036f4177bc3aaf61af50536d 100644
--- a/llvm/lib/Passes/PassRegistry.def
+++ b/llvm/lib/Passes/PassRegistry.def
@@ -19,6 +19,7 @@
 #define MODULE_ANALYSIS(NAME, CREATE_PASS)
 #endif
 MODULE_ANALYSIS("callgraph", CallGraphAnalysis())
+MODULE_ANALYSIS("collector-metadata", CollectorMetadataAnalysis())
 MODULE_ANALYSIS("inline-advisor", InlineAdvisorAnalysis())
 MODULE_ANALYSIS("ir-similarity", IRSimilarityAnalysis())
 MODULE_ANALYSIS("lcg", LazyCallGraphAnalysis())
@@ -235,6 +236,7 @@ FUNCTION_ANALYSIS("demanded-bits", DemandedBitsAnalysis())
 FUNCTION_ANALYSIS("domfrontier", DominanceFrontierAnalysis())
 FUNCTION_ANALYSIS("domtree", DominatorTreeAnalysis())
 FUNCTION_ANALYSIS("func-properties", FunctionPropertiesAnalysis())
+FUNCTION_ANALYSIS("gc-function", GCFunctionAnalysis())
 FUNCTION_ANALYSIS("inliner-size-estimator", InlineSizeEstimatorAnalysis())
 FUNCTION_ANALYSIS("lazy-value-info", LazyValueAnalysis())
 FUNCTION_ANALYSIS("loops", LoopAnalysis())
@@ -306,6 +308,7 @@ FUNCTION_PASS("dse", DSEPass())
 FUNCTION_PASS("dwarf-eh-prepare", DwarfEHPreparePass(TM))
 FUNCTION_PASS("expand-large-div-rem", ExpandLargeDivRemPass(TM))
 FUNCTION_PASS("expand-large-fp-convert", ExpandLargeFpConvertPass(TM))
+FUNCTION_PASS("expand-memcmp", ExpandMemCmpPass(TM))
 FUNCTION_PASS("fix-irreducible", FixIrreduciblePass())
 FUNCTION_PASS("flattencfg", FlattenCFGPass())
 FUNCTION_PASS("float2int", Float2IntPass())
@@ -313,6 +316,7 @@ FUNCTION_PASS("guard-widening", GuardWideningPass())
 FUNCTION_PASS("gvn-hoist", GVNHoistPass())
 FUNCTION_PASS("gvn-sink", GVNSinkPass())
 FUNCTION_PASS("helloworld", HelloWorldPass())
+FUNCTION_PASS("indirectbr-expand", IndirectBrExpandPass(TM))
 FUNCTION_PASS("infer-address-spaces", InferAddressSpacesPass())
 FUNCTION_PASS("infer-alignment", InferAlignmentPass())
 FUNCTION_PASS("inject-tli-mappings", InjectTLIMappings())
@@ -320,6 +324,7 @@ FUNCTION_PASS("instcount", InstCountPass())
 FUNCTION_PASS("instnamer", InstructionNamerPass())
 FUNCTION_PASS("instsimplify", InstSimplifyPass())
 FUNCTION_PASS("interleaved-access", InterleavedAccessPass(TM))
+FUNCTION_PASS("interleaved-load-combine", InterleavedLoadCombinePass(TM))
 FUNCTION_PASS("invalidate<all>", InvalidateAllAnalysesPass())
 FUNCTION_PASS("irce", IRCEPass())
 FUNCTION_PASS("jump-threading", JumpThreadingPass())
@@ -433,6 +438,10 @@ FUNCTION_PASS("wasm-eh-prepare", WasmEHPreparePass())
 #ifndef FUNCTION_PASS_WITH_PARAMS
 #define FUNCTION_PASS_WITH_PARAMS(NAME, CLASS, CREATE_PASS, PARSER, PARAMS)
 #endif
+FUNCTION_PASS_WITH_PARAMS(
+    "cfguard", "CFGuardPass",
+    [](CFGuardPass::Mechanism M) { return CFGuardPass(M); },
+    parseCFGuardPassOptions, "check;dispatch")
 FUNCTION_PASS_WITH_PARAMS(
     "early-cse", "EarlyCSEPass",
     [](bool UseMemorySSA) { return EarlyCSEPass(UseMemorySSA); },
diff --git a/llvm/lib/Passes/StandardInstrumentations.cpp b/llvm/lib/Passes/StandardInstrumentations.cpp
index df445c2dd78b770144d8e4fb2db6ca352d638501..fd1317e3eb25670ffb6cb361803e1ae0ac13bd9c 100644
--- a/llvm/lib/Passes/StandardInstrumentations.cpp
+++ b/llvm/lib/Passes/StandardInstrumentations.cpp
@@ -130,6 +130,11 @@ static cl::opt<std::string> IRDumpDirectory(
              "files in this directory rather than written to stderr"),
     cl::Hidden, cl::value_desc("filename"));
 
+template <typename IRUnitT> static const IRUnitT *unwrapIR(Any IR) {
+  const IRUnitT **IRPtr = llvm::any_cast<const IRUnitT *>(&IR);
+  return IRPtr ? *IRPtr : nullptr;
+}
+
 namespace {
 
 // An option for specifying an executable that will be called with the IR
@@ -147,18 +152,18 @@ static cl::opt<std::string>
 /// Extract Module out of \p IR unit. May return nullptr if \p IR does not match
 /// certain global filters. Will never return nullptr if \p Force is true.
 const Module *unwrapModule(Any IR, bool Force = false) {
-  if (const auto **M = llvm::any_cast<const Module *>(&IR))
-    return *M;
+  if (const auto *M = unwrapIR<Module>(IR))
+    return M;
 
-  if (const auto **F = llvm::any_cast<const Function *>(&IR)) {
-    if (!Force && !isFunctionInPrintList((*F)->getName()))
+  if (const auto *F = unwrapIR<Function>(IR)) {
+    if (!Force && !isFunctionInPrintList(F->getName()))
       return nullptr;
 
-    return (*F)->getParent();
+    return F->getParent();
   }
 
-  if (const auto **C = llvm::any_cast<const LazyCallGraph::SCC *>(&IR)) {
-    for (const LazyCallGraph::Node &N : **C) {
+  if (const auto *C = unwrapIR<LazyCallGraph::SCC>(IR)) {
+    for (const LazyCallGraph::Node &N : *C) {
       const Function &F = N.getFunction();
       if (Force || (!F.isDeclaration() && isFunctionInPrintList(F.getName()))) {
         return F.getParent();
@@ -168,8 +173,8 @@ const Module *unwrapModule(Any IR, bool Force = false) {
     return nullptr;
   }
 
-  if (const auto **L = llvm::any_cast<const Loop *>(&IR)) {
-    const Function *F = (*L)->getHeader()->getParent();
+  if (const auto *L = unwrapIR<Loop>(IR)) {
+    const Function *F = L->getHeader()->getParent();
     if (!Force && !isFunctionInPrintList(F->getName()))
       return nullptr;
     return F->getParent();
@@ -211,20 +216,20 @@ void printIR(raw_ostream &OS, const Loop *L) {
 }
 
 std::string getIRName(Any IR) {
-  if (llvm::any_cast<const Module *>(&IR))
+  if (unwrapIR<Module>(IR))
     return "[module]";
 
-  if (const auto **F = llvm::any_cast<const Function *>(&IR))
-    return (*F)->getName().str();
+  if (const auto *F = unwrapIR<Function>(IR))
+    return F->getName().str();
 
-  if (const auto **C = llvm::any_cast<const LazyCallGraph::SCC *>(&IR))
-    return (*C)->getName();
+  if (const auto *C = unwrapIR<LazyCallGraph::SCC>(IR))
+    return C->getName();
 
-  if (const auto **L = llvm::any_cast<const Loop *>(&IR))
-    return (*L)->getName().str();
+  if (const auto *L = unwrapIR<Loop>(IR))
+    return L->getName().str();
 
-  if (const auto **MF = llvm::any_cast<const MachineFunction *>(&IR))
-    return (*MF)->getName().str();
+  if (const auto *MF = unwrapIR<MachineFunction>(IR))
+    return MF->getName().str();
 
   llvm_unreachable("Unknown wrapped IR type");
 }
@@ -246,17 +251,17 @@ bool sccContainsFilterPrintFunc(const LazyCallGraph::SCC &C) {
 }
 
 bool shouldPrintIR(Any IR) {
-  if (const auto **M = llvm::any_cast<const Module *>(&IR))
-    return moduleContainsFilterPrintFunc(**M);
+  if (const auto *M = unwrapIR<Module>(IR))
+    return moduleContainsFilterPrintFunc(*M);
 
-  if (const auto **F = llvm::any_cast<const Function *>(&IR))
-    return isFunctionInPrintList((*F)->getName());
+  if (const auto *F = unwrapIR<Function>(IR))
+    return isFunctionInPrintList(F->getName());
 
-  if (const auto **C = llvm::any_cast<const LazyCallGraph::SCC *>(&IR))
-    return sccContainsFilterPrintFunc(**C);
+  if (const auto *C = unwrapIR<LazyCallGraph::SCC>(IR))
+    return sccContainsFilterPrintFunc(*C);
 
-  if (const auto **L = llvm::any_cast<const Loop *>(&IR))
-    return isFunctionInPrintList((*L)->getHeader()->getParent()->getName());
+  if (const auto *L = unwrapIR<Loop>(IR))
+    return isFunctionInPrintList(L->getHeader()->getParent()->getName());
   llvm_unreachable("Unknown wrapped IR type");
 }
 
@@ -273,23 +278,23 @@ void unwrapAndPrint(raw_ostream &OS, Any IR) {
     return;
   }
 
-  if (const auto **M = llvm::any_cast<const Module *>(&IR)) {
-    printIR(OS, *M);
+  if (const auto *M = unwrapIR<Module>(IR)) {
+    printIR(OS, M);
     return;
   }
 
-  if (const auto **F = llvm::any_cast<const Function *>(&IR)) {
-    printIR(OS, *F);
+  if (const auto *F = unwrapIR<Function>(IR)) {
+    printIR(OS, F);
     return;
   }
 
-  if (const auto **C = llvm::any_cast<const LazyCallGraph::SCC *>(&IR)) {
-    printIR(OS, *C);
+  if (const auto *C = unwrapIR<LazyCallGraph::SCC>(IR)) {
+    printIR(OS, C);
     return;
   }
 
-  if (const auto **L = llvm::any_cast<const Loop *>(&IR)) {
-    printIR(OS, *L);
+  if (const auto *L = unwrapIR<Loop>(IR)) {
+    printIR(OS, L);
     return;
   }
   llvm_unreachable("Unknown wrapped IR type");
@@ -320,13 +325,10 @@ std::string makeHTMLReady(StringRef SR) {
 
 // Return the module when that is the appropriate level of comparison for \p IR.
 const Module *getModuleForComparison(Any IR) {
-  if (const auto **M = llvm::any_cast<const Module *>(&IR))
-    return *M;
-  if (const auto **C = llvm::any_cast<const LazyCallGraph::SCC *>(&IR))
-    return (*C)
-        ->begin()
-        ->getFunction()
-        .getParent();
+  if (const auto *M = unwrapIR<Module>(IR))
+    return M;
+  if (const auto *C = unwrapIR<LazyCallGraph::SCC>(IR))
+    return C->begin()->getFunction().getParent();
   return nullptr;
 }
 
@@ -339,8 +341,8 @@ bool isInterestingFunction(const Function &F) {
 bool isInteresting(Any IR, StringRef PassID, StringRef PassName) {
   if (isIgnored(PassID) || !isPassInPrintList(PassName))
     return false;
-  if (const auto **F = llvm::any_cast<const Function *>(&IR))
-    return isInterestingFunction(**F);
+  if (const auto *F = unwrapIR<Function>(IR))
+    return isInterestingFunction(*F);
   return true;
 }
 
@@ -662,12 +664,11 @@ template <typename T> void IRComparer<T>::analyzeIR(Any IR, IRDataT<T> &Data) {
     return;
   }
 
-  const Function **FPtr = llvm::any_cast<const Function *>(&IR);
-  const Function *F = FPtr ? *FPtr : nullptr;
+  const auto *F = unwrapIR<Function>(IR);
   if (!F) {
-    const Loop **L = llvm::any_cast<const Loop *>(&IR);
+    const auto *L = unwrapIR<Loop>(IR);
     assert(L && "Unknown IR unit.");
-    F = (*L)->getHeader()->getParent();
+    F = L->getHeader()->getParent();
   }
   assert(F && "Unknown IR unit.");
   generateFunctionData(Data, *F);
@@ -706,21 +707,20 @@ static SmallString<32> getIRFileDisplayName(Any IR) {
   stable_hash NameHash = stable_hash_combine_string(M->getName());
   unsigned int MaxHashWidth = sizeof(stable_hash) * 8 / 4;
   write_hex(ResultStream, NameHash, HexPrintStyle::Lower, MaxHashWidth);
-  if (llvm::any_cast<const Module *>(&IR)) {
+  if (unwrapIR<Module>(IR)) {
     ResultStream << "-module";
-  } else if (const Function **F = llvm::any_cast<const Function *>(&IR)) {
+  } else if (const auto *F = unwrapIR<Function>(IR)) {
     ResultStream << "-function-";
-    stable_hash FunctionNameHash = stable_hash_combine_string((*F)->getName());
+    stable_hash FunctionNameHash = stable_hash_combine_string(F->getName());
     write_hex(ResultStream, FunctionNameHash, HexPrintStyle::Lower,
               MaxHashWidth);
-  } else if (const LazyCallGraph::SCC **C =
-                 llvm::any_cast<const LazyCallGraph::SCC *>(&IR)) {
+  } else if (const auto *C = unwrapIR<LazyCallGraph::SCC>(IR)) {
     ResultStream << "-scc-";
-    stable_hash SCCNameHash = stable_hash_combine_string((*C)->getName());
+    stable_hash SCCNameHash = stable_hash_combine_string(C->getName());
     write_hex(ResultStream, SCCNameHash, HexPrintStyle::Lower, MaxHashWidth);
-  } else if (const Loop **L = llvm::any_cast<const Loop *>(&IR)) {
+  } else if (const auto *L = unwrapIR<Loop>(IR)) {
     ResultStream << "-loop-";
-    stable_hash LoopNameHash = stable_hash_combine_string((*L)->getName());
+    stable_hash LoopNameHash = stable_hash_combine_string(L->getName());
     write_hex(ResultStream, LoopNameHash, HexPrintStyle::Lower, MaxHashWidth);
   } else {
     llvm_unreachable("Unknown wrapped IR type");
@@ -975,11 +975,10 @@ void OptNoneInstrumentation::registerCallbacks(
 }
 
 bool OptNoneInstrumentation::shouldRun(StringRef PassID, Any IR) {
-  const Function **FPtr = llvm::any_cast<const Function *>(&IR);
-  const Function *F = FPtr ? *FPtr : nullptr;
+  const auto *F = unwrapIR<Function>(IR);
   if (!F) {
-    if (const auto **L = llvm::any_cast<const Loop *>(&IR))
-      F = (*L)->getHeader()->getParent();
+    if (const auto *L = unwrapIR<Loop>(IR))
+      F = L->getHeader()->getParent();
   }
   bool ShouldRun = !(F && F->hasOptNone());
   if (!ShouldRun && DebugLogging) {
@@ -1054,15 +1053,14 @@ void PrintPassInstrumentation::registerCallbacks(
 
     auto &OS = print();
     OS << "Running pass: " << PassID << " on " << getIRName(IR);
-    if (const auto **F = llvm::any_cast<const Function *>(&IR)) {
-      unsigned Count = (*F)->getInstructionCount();
+    if (const auto *F = unwrapIR<Function>(IR)) {
+      unsigned Count = F->getInstructionCount();
       OS << " (" << Count << " instruction";
       if (Count != 1)
         OS << 's';
       OS << ')';
-    } else if (const auto **C =
-                   llvm::any_cast<const LazyCallGraph::SCC *>(&IR)) {
-      int Count = (*C)->size();
+    } else if (const auto *C = unwrapIR<LazyCallGraph::SCC>(IR)) {
+      int Count = C->size();
       OS << " (" << Count << " node";
       if (Count != 1)
         OS << 's';
@@ -1277,10 +1275,10 @@ bool PreservedCFGCheckerInstrumentation::CFG::invalidate(
 static SmallVector<Function *, 1> GetFunctions(Any IR) {
   SmallVector<Function *, 1> Functions;
 
-  if (const auto **MaybeF = llvm::any_cast<const Function *>(&IR)) {
-    Functions.push_back(*const_cast<Function **>(MaybeF));
-  } else if (const auto **MaybeM = llvm::any_cast<const Module *>(&IR)) {
-    for (Function &F : **const_cast<Module **>(MaybeM))
+  if (const auto *MaybeF = unwrapIR<Function>(IR)) {
+    Functions.push_back(const_cast<Function *>(MaybeF));
+  } else if (const auto *MaybeM = unwrapIR<Module>(IR)) {
+    for (Function &F : *const_cast<Module *>(MaybeM))
       Functions.push_back(&F);
   }
   return Functions;
@@ -1315,8 +1313,8 @@ void PreservedCFGCheckerInstrumentation::registerCallbacks(
       FAM.getResult<PreservedFunctionHashAnalysis>(*F);
     }
 
-    if (auto *MaybeM = llvm::any_cast<const Module *>(&IR)) {
-      Module &M = **const_cast<Module **>(MaybeM);
+    if (const auto *MPtr = unwrapIR<Module>(IR)) {
+      auto &M = *const_cast<Module *>(MPtr);
       MAM.getResult<PreservedModuleHashAnalysis>(M);
     }
   });
@@ -1374,8 +1372,8 @@ void PreservedCFGCheckerInstrumentation::registerCallbacks(
         CheckCFG(P, F->getName(), *GraphBefore,
                  CFG(F, /* TrackBBLifetime */ false));
     }
-    if (auto *MaybeM = llvm::any_cast<const Module *>(&IR)) {
-      Module &M = **const_cast<Module **>(MaybeM);
+    if (const auto *MPtr = unwrapIR<Module>(IR)) {
+      auto &M = *const_cast<Module *>(MPtr);
       if (auto *HashBefore =
               MAM.getCachedResult<PreservedModuleHashAnalysis>(M)) {
         if (HashBefore->Hash != StructuralHash(M)) {
@@ -1393,11 +1391,10 @@ void VerifyInstrumentation::registerCallbacks(
       [this](StringRef P, Any IR, const PreservedAnalyses &PassPA) {
         if (isIgnored(P) || P == "VerifierPass")
           return;
-        const Function **FPtr = llvm::any_cast<const Function *>(&IR);
-        const Function *F = FPtr ? *FPtr : nullptr;
+        const auto *F = unwrapIR<Function>(IR);
         if (!F) {
-          if (const auto **L = llvm::any_cast<const Loop *>(&IR))
-            F = (*L)->getHeader()->getParent();
+          if (const auto *L = unwrapIR<Loop>(IR))
+            F = L->getHeader()->getParent();
         }
 
         if (F) {
@@ -1409,12 +1406,10 @@ void VerifyInstrumentation::registerCallbacks(
                                        "\"{0}\", compilation aborted!",
                                        P));
         } else {
-          const Module **MPtr = llvm::any_cast<const Module *>(&IR);
-          const Module *M = MPtr ? *MPtr : nullptr;
+          const auto *M = unwrapIR<Module>(IR);
           if (!M) {
-            if (const auto **C =
-                    llvm::any_cast<const LazyCallGraph::SCC *>(&IR))
-              M = (*C)->begin()->getFunction().getParent();
+            if (const auto *C = unwrapIR<LazyCallGraph::SCC>(IR))
+              M = C->begin()->getFunction().getParent();
           }
 
           if (M) {
diff --git a/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp b/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp
index 380a3aef9b1462179a3743028362a1b211de8181..80875702b02ab9005ef9531a256bd4d8ed191102 100644
--- a/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp
+++ b/llvm/lib/ProfileData/Coverage/CoverageMapping.cpp
@@ -30,6 +30,7 @@
 #include "llvm/Support/raw_ostream.h"
 #include <algorithm>
 #include <cassert>
+#include <cmath>
 #include <cstdint>
 #include <iterator>
 #include <map>
@@ -221,6 +222,264 @@ Expected<int64_t> CounterMappingContext::evaluate(const Counter &C) const {
   return LastPoppedValue;
 }
 
+Expected<BitVector> CounterMappingContext::evaluateBitmap(
+    const CounterMappingRegion *MCDCDecision) const {
+  unsigned ID = MCDCDecision->MCDCParams.BitmapIdx;
+  unsigned NC = MCDCDecision->MCDCParams.NumConditions;
+  unsigned SizeInBits = llvm::alignTo(uint64_t(1) << NC, CHAR_BIT);
+  unsigned SizeInBytes = SizeInBits / CHAR_BIT;
+
+  ArrayRef<uint8_t> Bytes(&BitmapBytes[ID], SizeInBytes);
+
+  // Mask each bitmap byte into the BitVector. Go in reverse so that the
+  // bitvector can just be shifted over by one byte on each iteration.
+  BitVector Result(SizeInBits, false);
+  for (auto Byte = std::rbegin(Bytes); Byte != std::rend(Bytes); ++Byte) {
+    uint32_t Data = *Byte;
+    Result <<= CHAR_BIT;
+    Result.setBitsInMask(&Data, 1);
+  }
+  return Result;
+}
+
+class MCDCRecordProcessor {
+  /// A bitmap representing the executed test vectors for a boolean expression.
+  /// Each index of the bitmap corresponds to a possible test vector. An index
+  /// with a bit value of '1' indicates that the corresponding Test Vector
+  /// identified by that index was executed.
+  BitVector &ExecutedTestVectorBitmap;
+
+  /// Decision Region to which the ExecutedTestVectorBitmap applies.
+  CounterMappingRegion &Region;
+
+  /// Array of branch regions corresponding each conditions in the boolean
+  /// expression.
+  ArrayRef<CounterMappingRegion> Branches;
+
+  /// Total number of conditions in the boolean expression.
+  unsigned NumConditions;
+
+  /// Mapping of a condition ID to its corresponding branch region.
+  llvm::DenseMap<unsigned, const CounterMappingRegion *> Map;
+
+  /// Vector used to track whether a condition is constant folded.
+  MCDCRecord::BoolVector Folded;
+
+  /// Mapping of calculated MC/DC Independence Pairs for each condition.
+  MCDCRecord::TVPairMap IndependencePairs;
+
+  /// Total number of possible Test Vectors for the boolean expression.
+  MCDCRecord::TestVectors TestVectors;
+
+  /// Actual executed Test Vectors for the boolean expression, based on
+  /// ExecutedTestVectorBitmap.
+  MCDCRecord::TestVectors ExecVectors;
+
+public:
+  MCDCRecordProcessor(BitVector &Bitmap, CounterMappingRegion &Region,
+                      ArrayRef<CounterMappingRegion> Branches)
+      : ExecutedTestVectorBitmap(Bitmap), Region(Region), Branches(Branches),
+        NumConditions(Region.MCDCParams.NumConditions),
+        Folded(NumConditions, false), IndependencePairs(NumConditions),
+        TestVectors(pow(2, NumConditions)) {}
+
+private:
+  void recordTestVector(MCDCRecord::TestVector &TV,
+                        MCDCRecord::CondState Result) {
+    // Calculate an index that is used to identify the test vector in a vector
+    // of test vectors.  This index also corresponds to the index values of an
+    // MCDC Region's bitmap (see findExecutedTestVectors()).
+    unsigned Index = 0;
+    for (auto Cond = std::rbegin(TV); Cond != std::rend(TV); ++Cond) {
+      Index <<= 1;
+      Index |= (*Cond == MCDCRecord::MCDC_True) ? 0x1 : 0x0;
+    }
+
+    // Copy the completed test vector to the vector of testvectors.
+    TestVectors[Index] = TV;
+
+    // The final value (T,F) is equal to the last non-dontcare state on the
+    // path (in a short-circuiting system).
+    TestVectors[Index].push_back(Result);
+  }
+
+  void shouldCopyOffTestVectorForTruePath(MCDCRecord::TestVector &TV,
+                                          unsigned ID) {
+    // Branch regions are hashed based on an ID.
+    const CounterMappingRegion *Branch = Map[ID];
+
+    TV[ID - 1] = MCDCRecord::MCDC_True;
+    if (Branch->MCDCParams.TrueID > 0)
+      buildTestVector(TV, Branch->MCDCParams.TrueID);
+    else
+      recordTestVector(TV, MCDCRecord::MCDC_True);
+  }
+
+  void shouldCopyOffTestVectorForFalsePath(MCDCRecord::TestVector &TV,
+                                           unsigned ID) {
+    // Branch regions are hashed based on an ID.
+    const CounterMappingRegion *Branch = Map[ID];
+
+    TV[ID - 1] = MCDCRecord::MCDC_False;
+    if (Branch->MCDCParams.FalseID > 0)
+      buildTestVector(TV, Branch->MCDCParams.FalseID);
+    else
+      recordTestVector(TV, MCDCRecord::MCDC_False);
+  }
+
+  /// Starting with the base test vector, build a comprehensive list of
+  /// possible test vectors by recursively walking the branch condition IDs
+  /// provided. Once an end node is reached, record the test vector in a vector
+  /// of test vectors that can be matched against during MC/DC analysis, and
+  /// then reset the positions to 'DontCare'.
+  void buildTestVector(MCDCRecord::TestVector &TV, unsigned ID = 1) {
+    shouldCopyOffTestVectorForTruePath(TV, ID);
+    shouldCopyOffTestVectorForFalsePath(TV, ID);
+
+    // Reset back to DontCare.
+    TV[ID - 1] = MCDCRecord::MCDC_DontCare;
+  }
+
+  /// Walk the bits in the bitmap.  A bit set to '1' indicates that the test
+  /// vector at the corresponding index was executed during a test run.
+  void findExecutedTestVectors(BitVector &ExecutedTestVectorBitmap) {
+    for (unsigned Idx = 0; Idx < ExecutedTestVectorBitmap.size(); ++Idx) {
+      if (ExecutedTestVectorBitmap[Idx] == 0)
+        continue;
+      assert(!TestVectors[Idx].empty() && "Test Vector doesn't exist.");
+      ExecVectors.push_back(TestVectors[Idx]);
+    }
+  }
+
+  /// For a given condition and two executed Test Vectors, A and B, see if the
+  /// two test vectors match forming an Independence Pair for the condition.
+  /// For two test vectors to match, the following must be satisfied:
+  /// - The condition's value in each test vector must be opposite.
+  /// - The result's value in each test vector must be opposite.
+  /// - All other conditions' values must be equal or marked as "don't care".
+  bool matchTestVectors(unsigned Aidx, unsigned Bidx, unsigned ConditionIdx) {
+    const MCDCRecord::TestVector &A = ExecVectors[Aidx];
+    const MCDCRecord::TestVector &B = ExecVectors[Bidx];
+
+    // If condition values in both A and B aren't opposites, no match.
+    // Because a value can be 0 (false), 1 (true), or -1 (DontCare), a check
+    // that "XOR != 1" will ensure that the values are opposites and that
+    // neither of them is a DontCare.
+    //  1 XOR  0 ==  1 | 0 XOR  0 ==  0 | -1 XOR  0 == -1
+    //  1 XOR  1 ==  0 | 0 XOR  1 ==  1 | -1 XOR  1 == -2
+    //  1 XOR -1 == -2 | 0 XOR -1 == -1 | -1 XOR -1 ==  0
+    if ((A[ConditionIdx] ^ B[ConditionIdx]) != 1)
+      return false;
+
+    // If the results of both A and B aren't opposites, no match.
+    if ((A[NumConditions] ^ B[NumConditions]) != 1)
+      return false;
+
+    for (unsigned Idx = 0; Idx < NumConditions; ++Idx) {
+      // Look for other conditions that don't match. Skip over the given
+      // Condition as well as any conditions marked as "don't care".
+      const auto ARecordTyForCond = A[Idx];
+      const auto BRecordTyForCond = B[Idx];
+      if (Idx == ConditionIdx ||
+          ARecordTyForCond == MCDCRecord::MCDC_DontCare ||
+          BRecordTyForCond == MCDCRecord::MCDC_DontCare)
+        continue;
+
+      // If there is a condition mismatch with any of the other conditions,
+      // there is no match for the test vectors.
+      if (ARecordTyForCond != BRecordTyForCond)
+        return false;
+    }
+
+    // Otherwise, match.
+    return true;
+  }
+
+  /// Find all possible Independence Pairs for a boolean expression given its
+  /// executed Test Vectors.  This process involves looking at each condition
+  /// and attempting to find two Test Vectors that "match", giving us a pair.
+  void findIndependencePairs() {
+    unsigned NumTVs = ExecVectors.size();
+
+    // For each condition.
+    for (unsigned C = 0; C < NumConditions; ++C) {
+      bool PairFound = false;
+
+      // For each executed test vector.
+      for (unsigned I = 0; !PairFound && I < NumTVs; ++I) {
+        // Compared to every other executed test vector.
+        for (unsigned J = 0; !PairFound && J < NumTVs; ++J) {
+          if (I == J)
+            continue;
+
+          // If a matching pair of vectors is found, record them.
+          if ((PairFound = matchTestVectors(I, J, C)))
+            IndependencePairs[C] = std::make_pair(I + 1, J + 1);
+        }
+      }
+    }
+  }
+
+public:
+  /// Process the MC/DC Record in order to produce a result for a boolean
+  /// expression. This process includes tracking the conditions that comprise
+  /// the decision region, calculating the list of all possible test vectors,
+  /// marking the executed test vectors, and then finding an Independence Pair
+  /// out of the executed test vectors for each condition in the boolean
+  /// expression. A condition is tracked to ensure that its ID can be mapped to
+  /// its ordinal position in the boolean expression. The condition's source
+  /// location is also tracked, as well as whether it is constant folded (in
+  /// which case it is excuded from the metric).
+  MCDCRecord processMCDCRecord() {
+    unsigned I = 0;
+    MCDCRecord::CondIDMap PosToID;
+    MCDCRecord::LineColPairMap CondLoc;
+
+    // Walk the Record's BranchRegions (representing Conditions) in order to:
+    // - Hash the condition based on its corresponding ID. This will be used to
+    //   calculate the test vectors.
+    // - Keep a map of the condition's ordinal position (1, 2, 3, 4) to its
+    //   actual ID.  This will be used to visualize the conditions in the
+    //   correct order.
+    // - Keep track of the condition source location. This will be used to
+    //   visualize where the condition is.
+    // - Record whether the condition is constant folded so that we exclude it
+    //   from being measured.
+    for (const auto &B : Branches) {
+      Map[B.MCDCParams.ID] = &B;
+      PosToID[I] = B.MCDCParams.ID - 1;
+      CondLoc[I] = B.startLoc();
+      Folded[I++] = (B.Count.isZero() && B.FalseCount.isZero());
+    }
+
+    // Initialize a base test vector as 'DontCare'.
+    MCDCRecord::TestVector TV(NumConditions, MCDCRecord::MCDC_DontCare);
+
+    // Use the base test vector to build the list of all possible test vectors.
+    buildTestVector(TV);
+
+    // Using Profile Bitmap from runtime, mark the executed test vectors.
+    findExecutedTestVectors(ExecutedTestVectorBitmap);
+
+    // Compare executed test vectors against each other to find an independence
+    // pairs for each condition.  This processing takes the most time.
+    findIndependencePairs();
+
+    // Record Test vectors, executed vectors, and independence pairs.
+    MCDCRecord Res(Region, ExecVectors, IndependencePairs, Folded, PosToID,
+                   CondLoc);
+    return Res;
+  }
+};
+
+Expected<MCDCRecord> CounterMappingContext::evaluateMCDCRegion(
+    CounterMappingRegion Region, BitVector ExecutedTestVectorBitmap,
+    ArrayRef<CounterMappingRegion> Branches) {
+
+  MCDCRecordProcessor MCDCProcessor(ExecutedTestVectorBitmap, Region, Branches);
+  return MCDCProcessor.processMCDCRecord();
+}
+
 unsigned CounterMappingContext::getMaxCounterID(const Counter &C) const {
   struct StackElem {
     Counter ICounter;
@@ -303,6 +562,24 @@ static unsigned getMaxCounterID(const CounterMappingContext &Ctx,
   return MaxCounterID;
 }
 
+static unsigned getMaxBitmapSize(const CounterMappingContext &Ctx,
+                                 const CoverageMappingRecord &Record) {
+  unsigned MaxBitmapID = 0;
+  unsigned NumConditions = 0;
+  // The last DecisionRegion has the highest bitmap byte index used in the
+  // function, which when combined with its number of conditions, yields the
+  // full bitmap size.
+  for (const auto &Region : reverse(Record.MappingRegions)) {
+    if (Region.Kind == CounterMappingRegion::MCDCDecisionRegion) {
+      MaxBitmapID = Region.MCDCParams.BitmapIdx;
+      NumConditions = Region.MCDCParams.NumConditions;
+      break;
+    }
+  }
+  unsigned SizeInBits = llvm::alignTo(uint64_t(1) << NumConditions, CHAR_BIT);
+  return MaxBitmapID + (SizeInBits / CHAR_BIT);
+}
+
 Error CoverageMapping::loadFunctionRecord(
     const CoverageMappingRecord &Record,
     IndexedInstrProfReader &ProfileReader) {
@@ -326,12 +603,28 @@ Error CoverageMapping::loadFunctionRecord(
       FuncHashMismatches.emplace_back(std::string(Record.FunctionName),
                                       Record.FunctionHash);
       return Error::success();
-    } else if (IPE != instrprof_error::unknown_function)
+    }
+    if (IPE != instrprof_error::unknown_function)
       return make_error<InstrProfError>(IPE);
     Counts.assign(getMaxCounterID(Ctx, Record) + 1, 0);
   }
   Ctx.setCounts(Counts);
 
+  std::vector<uint8_t> BitmapBytes;
+  if (Error E = ProfileReader.getFunctionBitmapBytes(
+          Record.FunctionName, Record.FunctionHash, BitmapBytes)) {
+    instrprof_error IPE = std::get<0>(InstrProfError::take(std::move(E)));
+    if (IPE == instrprof_error::hash_mismatch) {
+      FuncHashMismatches.emplace_back(std::string(Record.FunctionName),
+                                      Record.FunctionHash);
+      return Error::success();
+    }
+    if (IPE != instrprof_error::unknown_function)
+      return make_error<InstrProfError>(IPE);
+    BitmapBytes.assign(getMaxBitmapSize(Ctx, Record) + 1, 0);
+  }
+  Ctx.setBitmapBytes(BitmapBytes);
+
   assert(!Record.MappingRegions.empty() && "Function has no regions");
 
   // This coverage record is a zero region for a function that's unused in
@@ -343,8 +636,20 @@ Error CoverageMapping::loadFunctionRecord(
       Record.MappingRegions[0].Count.isZero() && Counts[0] > 0)
     return Error::success();
 
+  unsigned NumConds = 0;
+  const CounterMappingRegion *MCDCDecision;
+  std::vector<CounterMappingRegion> MCDCBranches;
+
   FunctionRecord Function(OrigFuncName, Record.Filenames);
   for (const auto &Region : Record.MappingRegions) {
+    // If an MCDCDecisionRegion is seen, track the BranchRegions that follow
+    // it according to Region.NumConditions.
+    if (Region.Kind == CounterMappingRegion::MCDCDecisionRegion) {
+      assert(NumConds == 0);
+      MCDCDecision = &Region;
+      NumConds = Region.MCDCParams.NumConditions;
+      continue;
+    }
     Expected<int64_t> ExecutionCount = Ctx.evaluate(Region.Count);
     if (auto E = ExecutionCount.takeError()) {
       consumeError(std::move(E));
@@ -356,6 +661,44 @@ Error CoverageMapping::loadFunctionRecord(
       return Error::success();
     }
     Function.pushRegion(Region, *ExecutionCount, *AltExecutionCount);
+
+    // If a MCDCDecisionRegion was seen, store the BranchRegions that
+    // correspond to it in a vector, according to the number of conditions
+    // recorded for the region (tracked by NumConds).
+    if (NumConds > 0 && Region.Kind == CounterMappingRegion::MCDCBranchRegion) {
+      MCDCBranches.push_back(Region);
+
+      // As we move through all of the MCDCBranchRegions that follow the
+      // MCDCDecisionRegion, decrement NumConds to make sure we account for
+      // them all before we calculate the bitmap of executed test vectors.
+      if (--NumConds == 0) {
+        // Evaluating the test vector bitmap for the decision region entails
+        // calculating precisely what bits are pertinent to this region alone.
+        // This is calculated based on the recorded offset into the global
+        // profile bitmap; the length is calculated based on the recorded
+        // number of conditions.
+        Expected<BitVector> ExecutedTestVectorBitmap =
+            Ctx.evaluateBitmap(MCDCDecision);
+        if (auto E = ExecutedTestVectorBitmap.takeError()) {
+          consumeError(std::move(E));
+          return Error::success();
+        }
+
+        // Since the bitmap identifies the executed test vectors for an MC/DC
+        // DecisionRegion, all of the information is now available to process.
+        // This is where the bulk of the MC/DC progressing takes place.
+        Expected<MCDCRecord> Record = Ctx.evaluateMCDCRegion(
+            *MCDCDecision, *ExecutedTestVectorBitmap, MCDCBranches);
+        if (auto E = Record.takeError()) {
+          consumeError(std::move(E));
+          return Error::success();
+        }
+
+        // Save the MC/DC Record so that it can be visualized later.
+        Function.pushMCDCRecord(*Record);
+        MCDCBranches.clear();
+      }
+    }
   }
 
   // Don't create records for (filenames, function) pairs we've already seen.
@@ -862,6 +1205,10 @@ CoverageData CoverageMapping::getCoverageForFile(StringRef Filename) const {
     for (const auto &CR : Function.CountedBranchRegions)
       if (FileIDs.test(CR.FileID) && (CR.FileID == CR.ExpandedFileID))
         FileCoverage.BranchRegions.push_back(CR);
+    // Capture MCDC records specific to the function.
+    for (const auto &MR : Function.MCDCRecords)
+      if (FileIDs.test(MR.getDecisionRegion().FileID))
+        FileCoverage.MCDCRecords.push_back(MR);
   }
 
   LLVM_DEBUG(dbgs() << "Emitting segments for file: " << Filename << "\n");
@@ -914,6 +1261,11 @@ CoverageMapping::getCoverageForFunction(const FunctionRecord &Function) const {
     if (CR.FileID == *MainFileID)
       FunctionCoverage.BranchRegions.push_back(CR);
 
+  // Capture MCDC records specific to the function.
+  for (const auto &MR : Function.MCDCRecords)
+    if (MR.getDecisionRegion().FileID == *MainFileID)
+      FunctionCoverage.MCDCRecords.push_back(MR);
+
   LLVM_DEBUG(dbgs() << "Emitting segments for function: " << Function.Name
                     << "\n");
   FunctionCoverage.Segments = SegmentBuilder::buildSegments(Regions);
diff --git a/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp b/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
index d6aade6fcd0f8ddac08ea9191b03c9061da1fb2e..56a7ab24a2f08adcbe703eea28a8115bc1cab1b1 100644
--- a/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
+++ b/llvm/lib/ProfileData/Coverage/CoverageMappingReader.cpp
@@ -244,6 +244,7 @@ Error RawCoverageMappingReader::readMappingRegionsSubArray(
   unsigned LineStart = 0;
   for (size_t I = 0; I < NumRegions; ++I) {
     Counter C, C2;
+    uint64_t BIDX = 0, NC = 0, ID = 0, TID = 0, FID = 0;
     CounterMappingRegion::RegionKind Kind = CounterMappingRegion::CodeRegion;
 
     // Read the combined counter + region kind.
@@ -294,6 +295,27 @@ Error RawCoverageMappingReader::readMappingRegionsSubArray(
           if (auto Err = readCounter(C2))
             return Err;
           break;
+        case CounterMappingRegion::MCDCBranchRegion:
+          // For a MCDC Branch Region, read two successive counters and 3 IDs.
+          Kind = CounterMappingRegion::MCDCBranchRegion;
+          if (auto Err = readCounter(C))
+            return Err;
+          if (auto Err = readCounter(C2))
+            return Err;
+          if (auto Err = readIntMax(ID, std::numeric_limits<unsigned>::max()))
+            return Err;
+          if (auto Err = readIntMax(TID, std::numeric_limits<unsigned>::max()))
+            return Err;
+          if (auto Err = readIntMax(FID, std::numeric_limits<unsigned>::max()))
+            return Err;
+          break;
+        case CounterMappingRegion::MCDCDecisionRegion:
+          Kind = CounterMappingRegion::MCDCDecisionRegion;
+          if (auto Err = readIntMax(BIDX, std::numeric_limits<unsigned>::max()))
+            return Err;
+          if (auto Err = readIntMax(NC, std::numeric_limits<unsigned>::max()))
+            return Err;
+          break;
         default:
           return make_error<CoverageMapError>(coveragemap_error::malformed,
                                               "region kind is incorrect");
@@ -347,9 +369,14 @@ Error RawCoverageMappingReader::readMappingRegionsSubArray(
       dbgs() << "\n";
     });
 
-    auto CMR = CounterMappingRegion(C, C2, InferredFileID, ExpandedFileID,
-                                    LineStart, ColumnStart,
-                                    LineStart + NumLines, ColumnEnd, Kind);
+    auto CMR = CounterMappingRegion(
+        C, C2,
+        CounterMappingRegion::MCDCParameters{
+            static_cast<unsigned>(BIDX), static_cast<unsigned>(NC),
+            static_cast<unsigned>(ID), static_cast<unsigned>(TID),
+            static_cast<unsigned>(FID)},
+        InferredFileID, ExpandedFileID, LineStart, ColumnStart,
+        LineStart + NumLines, ColumnEnd, Kind);
     if (CMR.startLoc() > CMR.endLoc())
       return make_error<CoverageMapError>(
           coveragemap_error::malformed,
diff --git a/llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp b/llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
index 2abfbbad16ee92ea929f6aab11715276d6a8fdd5..1c7d8a8909c488e83852fb7a18280bb755bb3df4 100644
--- a/llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
+++ b/llvm/lib/ProfileData/Coverage/CoverageMappingWriter.cpp
@@ -237,6 +237,23 @@ void CoverageMappingWriter::write(raw_ostream &OS) {
       writeCounter(MinExpressions, Count, OS);
       writeCounter(MinExpressions, FalseCount, OS);
       break;
+    case CounterMappingRegion::MCDCBranchRegion:
+      encodeULEB128(unsigned(I->Kind)
+                        << Counter::EncodingCounterTagAndExpansionRegionTagBits,
+                    OS);
+      writeCounter(MinExpressions, Count, OS);
+      writeCounter(MinExpressions, FalseCount, OS);
+      encodeULEB128(unsigned(I->MCDCParams.ID), OS);
+      encodeULEB128(unsigned(I->MCDCParams.TrueID), OS);
+      encodeULEB128(unsigned(I->MCDCParams.FalseID), OS);
+      break;
+    case CounterMappingRegion::MCDCDecisionRegion:
+      encodeULEB128(unsigned(I->Kind)
+                        << Counter::EncodingCounterTagAndExpansionRegionTagBits,
+                    OS);
+      encodeULEB128(unsigned(I->MCDCParams.BitmapIdx), OS);
+      encodeULEB128(unsigned(I->MCDCParams.NumConditions), OS);
+      break;
     }
     assert(I->LineStart >= PrevLineStart);
     encodeULEB128(I->LineStart - PrevLineStart, OS);
diff --git a/llvm/lib/Support/AutoConvert.cpp b/llvm/lib/Support/AutoConvert.cpp
index 4fb7e242c34806382eb3220df847f0d58458db82..8170e553ac6e10d0eaa2cd3f765ac1704b475127 100644
--- a/llvm/lib/Support/AutoConvert.cpp
+++ b/llvm/lib/Support/AutoConvert.cpp
@@ -14,21 +14,36 @@
 #ifdef __MVS__
 
 #include "llvm/Support/AutoConvert.h"
+#include <cassert>
 #include <fcntl.h>
 #include <sys/stat.h>
+#include <unistd.h>
 
-std::error_code llvm::disableAutoConversion(int FD) {
+static int savedStdHandleAutoConversionMode[3] = {-1, -1, -1};
+
+int disableAutoConversion(int FD) {
   static const struct f_cnvrt Convert = {
-      SETCVTOFF,        // cvtcmd
-      0,                // pccsid
-      (short)FT_BINARY, // fccsid
+      SETCVTOFF, // cvtcmd
+      0,         // pccsid
+      0,         // fccsid
   };
-  if (fcntl(FD, F_CONTROL_CVT, &Convert) == -1)
-    return std::error_code(errno, std::generic_category());
-  return std::error_code();
+
+  return fcntl(FD, F_CONTROL_CVT, &Convert);
 }
 
-std::error_code llvm::enableAutoConversion(int FD) {
+int restoreStdHandleAutoConversion(int FD) {
+  assert(FD == STDIN_FILENO || FD == STDOUT_FILENO || FD == STDERR_FILENO);
+  if (savedStdHandleAutoConversionMode[FD] == -1)
+    return 0;
+  struct f_cnvrt Cvt = {
+      savedStdHandleAutoConversionMode[FD], // cvtcmd
+      0,                                    // pccsid
+      0,                                    // fccsid
+  };
+  return (fcntl(FD, F_CONTROL_CVT, &Cvt));
+}
+
+int enableAutoConversion(int FD) {
   struct f_cnvrt Query = {
       QUERYCVT, // cvtcmd
       0,        // pccsid
@@ -36,17 +51,53 @@ std::error_code llvm::enableAutoConversion(int FD) {
   };
 
   if (fcntl(FD, F_CONTROL_CVT, &Query) == -1)
-    return std::error_code(errno, std::generic_category());
+    return -1;
+
+  // We don't need conversion for UTF-8 tagged files.
+  // TODO: Remove the assumption of ISO8859-1 = UTF-8 here when we fully resolve
+  // problems related to UTF-8 tagged source files.
+  // When the pccsid is not ISO8859-1, autoconversion is still needed.
+  if (Query.pccsid == CCSID_ISO8859_1 &&
+      (Query.fccsid == CCSID_UTF_8 || Query.fccsid == CCSID_ISO8859_1))
+    return 0;
+
+  // Save the state of std handles before we make changes to it.
+  if ((FD == STDIN_FILENO || FD == STDOUT_FILENO || FD == STDERR_FILENO) &&
+      savedStdHandleAutoConversionMode[FD] == -1)
+    savedStdHandleAutoConversionMode[FD] = Query.cvtcmd;
+
+  if (FD == STDOUT_FILENO || FD == STDERR_FILENO)
+    Query.cvtcmd = SETCVTON;
+  else
+    Query.cvtcmd = SETCVTALL;
 
-  Query.cvtcmd = SETCVTALL;
   Query.pccsid =
       (FD == STDIN_FILENO || FD == STDOUT_FILENO || FD == STDERR_FILENO)
           ? 0
           : CCSID_UTF_8;
   // Assume untagged files to be IBM-1047 encoded.
   Query.fccsid = (Query.fccsid == FT_UNTAGGED) ? CCSID_IBM_1047 : Query.fccsid;
-  if (fcntl(FD, F_CONTROL_CVT, &Query) == -1)
+  return fcntl(FD, F_CONTROL_CVT, &Query);
+}
+
+std::error_code llvm::disableAutoConversion(int FD) {
+  if (::disableAutoConversion(FD) == -1)
+    return std::error_code(errno, std::generic_category());
+
+  return std::error_code();
+}
+
+std::error_code llvm::enableAutoConversion(int FD) {
+  if (::enableAutoConversion(FD) == -1)
     return std::error_code(errno, std::generic_category());
+
+  return std::error_code();
+}
+
+std::error_code llvm::restoreStdHandleAutoConversion(int FD) {
+  if (::restoreStdHandleAutoConversion(FD) == -1)
+    return std::error_code(errno, std::generic_category());
+
   return std::error_code();
 }
 
diff --git a/llvm/lib/Support/CMakeLists.txt b/llvm/lib/Support/CMakeLists.txt
index b96d62c7a6224d63da261e0db9d6c491559a4d04..494b218fdadcc02bb83fa2823ce7148d938c0790 100644
--- a/llvm/lib/Support/CMakeLists.txt
+++ b/llvm/lib/Support/CMakeLists.txt
@@ -40,7 +40,7 @@ endif()
 if( MSVC OR MINGW )
   # libuuid required for FOLDERID_Profile usage in lib/Support/Windows/Path.inc.
   # advapi32 required for CryptAcquireContextW in lib/Support/Windows/Path.inc.
-  set(system_libs ${system_libs} psapi shell32 ole32 uuid advapi32)
+  set(system_libs ${system_libs} psapi shell32 ole32 uuid advapi32 ws2_32)
 elseif( CMAKE_HOST_UNIX )
   if( HAVE_LIBRT )
     set(system_libs ${system_libs} rt)
diff --git a/llvm/lib/Support/CommandLine.cpp b/llvm/lib/Support/CommandLine.cpp
index 31f79972125da90021efc8131d668a5089be917e..088b4e4d755cbe919264a05e1a861d3c09cee345 100644
--- a/llvm/lib/Support/CommandLine.cpp
+++ b/llvm/lib/Support/CommandLine.cpp
@@ -324,6 +324,13 @@ public:
     return false;
   }
 
+  bool hasNamedSubCommands() const {
+    for (const auto *S : RegisteredSubCommands)
+      if (!S->getName().empty())
+        return true;
+    return false;
+  }
+
   SubCommand *getActiveSubCommand() { return ActiveSubCommand; }
 
   void updateArgStr(Option *O, StringRef NewName, SubCommand *SC) {
@@ -425,7 +432,7 @@ private:
       return nullptr;
     return Opt;
   }
-  SubCommand *LookupSubCommand(StringRef Name);
+  SubCommand *LookupSubCommand(StringRef Name, std::string &NearestString);
 };
 
 } // namespace
@@ -550,9 +557,12 @@ Option *CommandLineParser::LookupOption(SubCommand &Sub, StringRef &Arg,
   return I->second;
 }
 
-SubCommand *CommandLineParser::LookupSubCommand(StringRef Name) {
+SubCommand *CommandLineParser::LookupSubCommand(StringRef Name,
+                                                std::string &NearestString) {
   if (Name.empty())
     return &SubCommand::getTopLevel();
+  // Find a subcommand with the edit distance == 1.
+  SubCommand *NearestMatch = nullptr;
   for (auto *S : RegisteredSubCommands) {
     if (S == &SubCommand::getAll())
       continue;
@@ -561,7 +571,14 @@ SubCommand *CommandLineParser::LookupSubCommand(StringRef Name) {
 
     if (StringRef(S->getName()) == StringRef(Name))
       return S;
+
+    if (!NearestMatch && S->getName().edit_distance(Name) < 2)
+      NearestMatch = S;
   }
+
+  if (NearestMatch)
+    NearestString = NearestMatch->getName();
+
   return &SubCommand::getTopLevel();
 }
 
@@ -1527,10 +1544,14 @@ bool CommandLineParser::ParseCommandLineOptions(int argc,
 
   int FirstArg = 1;
   SubCommand *ChosenSubCommand = &SubCommand::getTopLevel();
-  if (argc >= 2 && argv[FirstArg][0] != '-') {
+  std::string NearestSubCommandString;
+  bool MaybeNamedSubCommand =
+      argc >= 2 && argv[FirstArg][0] != '-' && hasNamedSubCommands();
+  if (MaybeNamedSubCommand) {
     // If the first argument specifies a valid subcommand, start processing
     // options from the second argument.
-    ChosenSubCommand = LookupSubCommand(StringRef(argv[FirstArg]));
+    ChosenSubCommand =
+        LookupSubCommand(StringRef(argv[FirstArg]), NearestSubCommandString);
     if (ChosenSubCommand != &SubCommand::getTopLevel())
       FirstArg = 2;
   }
@@ -1602,7 +1623,6 @@ bool CommandLineParser::ParseCommandLineOptions(int argc,
   bool DashDashFound = false; // Have we read '--'?
   for (int i = FirstArg; i < argc; ++i) {
     Option *Handler = nullptr;
-    Option *NearestHandler = nullptr;
     std::string NearestHandlerString;
     StringRef Value;
     StringRef ArgName = "";
@@ -1682,26 +1702,39 @@ bool CommandLineParser::ParseCommandLineOptions(int argc,
       // Otherwise, look for the closest available option to report to the user
       // in the upcoming error.
       if (!Handler && SinkOpts.empty())
-        NearestHandler =
-            LookupNearestOption(ArgName, OptionsMap, NearestHandlerString);
+        LookupNearestOption(ArgName, OptionsMap, NearestHandlerString);
     }
 
     if (!Handler) {
-      if (SinkOpts.empty()) {
-        *Errs << ProgramName << ": Unknown command line argument '" << argv[i]
-              << "'.  Try: '" << argv[0] << " --help'\n";
-
-        if (NearestHandler) {
-          // If we know a near match, report it as well.
-          *Errs << ProgramName << ": Did you mean '"
-                << PrintArg(NearestHandlerString, 0) << "'?\n";
-        }
-
-        ErrorParsing = true;
-      } else {
+      if (!SinkOpts.empty()) {
         for (Option *SinkOpt : SinkOpts)
           SinkOpt->addOccurrence(i, "", StringRef(argv[i]));
+        continue;
       }
+
+      auto ReportUnknownArgument = [&](bool IsArg,
+                                       StringRef NearestArgumentName) {
+        *Errs << ProgramName << ": Unknown "
+              << (IsArg ? "command line argument" : "subcommand") << " '"
+              << argv[i] << "'.  Try: '" << argv[0] << " --help'\n";
+
+        if (NearestArgumentName.empty())
+          return;
+
+        *Errs << ProgramName << ": Did you mean '";
+        if (IsArg)
+          *Errs << PrintArg(NearestArgumentName, 0);
+        else
+          *Errs << NearestArgumentName;
+        *Errs << "'?\n";
+      };
+
+      if (i > 1 || !MaybeNamedSubCommand)
+        ReportUnknownArgument(/*IsArg=*/true, NearestHandlerString);
+      else
+        ReportUnknownArgument(/*IsArg=*/false, NearestSubCommandString);
+
+      ErrorParsing = true;
       continue;
     }
 
diff --git a/llvm/lib/Support/InitLLVM.cpp b/llvm/lib/Support/InitLLVM.cpp
index 2b7173b28940345209a4b0b7ce6f8279e9e55798..7f475f42f3cb81db36ee31f4dff85702d938db82 100644
--- a/llvm/lib/Support/InitLLVM.cpp
+++ b/llvm/lib/Support/InitLLVM.cpp
@@ -8,6 +8,8 @@
 
 #include "llvm/Support/InitLLVM.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/AutoConvert.h"
+#include "llvm/Support/Error.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/ManagedStatic.h"
 #include "llvm/Support/PrettyStackTrace.h"
@@ -15,15 +17,31 @@
 #include "llvm/Support/SwapByteOrder.h"
 
 #ifdef _WIN32
-#include "llvm/Support/Error.h"
 #include "llvm/Support/Windows/WindowsSupport.h"
 #endif
 
+#ifdef __MVS__
+#include <unistd.h>
+
+void CleanupStdHandles(void *Cookie) {
+  llvm::raw_ostream *Outs = &llvm::outs(), *Errs = &llvm::errs();
+  Outs->flush();
+  Errs->flush();
+  llvm::restoreStdHandleAutoConversion(STDIN_FILENO);
+  llvm::restoreStdHandleAutoConversion(STDOUT_FILENO);
+  llvm::restoreStdHandleAutoConversion(STDERR_FILENO);
+}
+#endif
+
 using namespace llvm;
 using namespace llvm::sys;
 
 InitLLVM::InitLLVM(int &Argc, const char **&Argv,
                    bool InstallPipeSignalExitHandler) {
+#ifdef __MVS__
+  // Bring stdin/stdout/stderr into a known state.
+  sys::AddSignalHandler(CleanupStdHandles, nullptr);
+#endif
   if (InstallPipeSignalExitHandler)
     // The pipe signal handler must be installed before any other handlers are
     // registered. This is because the Unix \ref RegisterHandlers function does
@@ -37,6 +55,20 @@ InitLLVM::InitLLVM(int &Argc, const char **&Argv,
   sys::PrintStackTraceOnErrorSignal(Argv[0]);
   install_out_of_memory_new_handler();
 
+#ifdef __MVS__
+
+  // We use UTF-8 as the internal character encoding. On z/OS, all external
+  // output is encoded in EBCDIC. In order to be able to read all
+  // error messages, we turn conversion to EBCDIC on for stderr fd.
+  std::string Banner = std::string(Argv[0]) + ": ";
+  ExitOnError ExitOnErr(Banner);
+
+  // If turning on conversion for stderr fails then the error message
+  // may be garbled. There is no solution to this problem.
+  ExitOnErr(errorCodeToError(llvm::enableAutoConversion(STDERR_FILENO)));
+  ExitOnErr(errorCodeToError(llvm::enableAutoConversion(STDOUT_FILENO)));
+#endif
+
 #ifdef _WIN32
   // We use UTF-8 as the internal character encoding. On Windows,
   // arguments passed to main() may not be encoded in UTF-8. In order
@@ -61,4 +93,9 @@ InitLLVM::InitLLVM(int &Argc, const char **&Argv,
 #endif
 }
 
-InitLLVM::~InitLLVM() { llvm_shutdown(); }
+InitLLVM::~InitLLVM() {
+#ifdef __MVS__
+  CleanupStdHandles(nullptr);
+#endif
+  llvm_shutdown();
+}
diff --git a/llvm/lib/Support/RISCVISAInfo.cpp b/llvm/lib/Support/RISCVISAInfo.cpp
index 6322748430063cea312db27b0ed91cbb3197bdc9..85c34dd6206307e0084e0e6a4e732a08158c5b23 100644
--- a/llvm/lib/Support/RISCVISAInfo.cpp
+++ b/llvm/lib/Support/RISCVISAInfo.cpp
@@ -174,7 +174,7 @@ static const RISCVSupportedExtension SupportedExperimentalExtensions[] = {
 
     {"zfbfmin", RISCVExtensionVersion{0, 8}},
 
-    {"zicfilp", RISCVExtensionVersion{0, 2}},
+    {"zicfilp", RISCVExtensionVersion{0, 4}},
     {"zicond", RISCVExtensionVersion{1, 0}},
 
     {"ztso", RISCVExtensionVersion{0, 1}},
diff --git a/llvm/lib/Support/StringMap.cpp b/llvm/lib/Support/StringMap.cpp
index 432e1fc343f1f03a46b77d72a8e510da8e1fa328..67c05a87959cf0c243d17646ae2f28f6c9f0d708 100644
--- a/llvm/lib/Support/StringMap.cpp
+++ b/llvm/lib/Support/StringMap.cpp
@@ -43,8 +43,6 @@ static inline unsigned *getHashTable(StringMapEntryBase **TheTable,
   return reinterpret_cast<unsigned *>(TheTable + NumBuckets + 1);
 }
 
-uint32_t StringMapImpl::hash(StringRef Key) { return xxh3_64bits(Key); }
-
 StringMapImpl::StringMapImpl(unsigned InitSize, unsigned itemSize) {
   ItemSize = itemSize;
 
@@ -83,14 +81,11 @@ void StringMapImpl::init(unsigned InitSize) {
 /// specified bucket will be non-null.  Otherwise, it will be null.  In either
 /// case, the FullHashValue field of the bucket will be set to the hash value
 /// of the string.
-unsigned StringMapImpl::LookupBucketFor(StringRef Name,
-                                        uint32_t FullHashValue) {
-#ifdef EXPENSIVE_CHECKS
-  assert(FullHashValue == hash(Name));
-#endif
+unsigned StringMapImpl::LookupBucketFor(StringRef Name) {
   // Hash table unallocated so far?
   if (NumBuckets == 0)
     init(16);
+  unsigned FullHashValue = xxh3_64bits(Name);
   if (shouldReverseIterate())
     FullHashValue = ~FullHashValue;
   unsigned BucketNo = FullHashValue & (NumBuckets - 1);
@@ -144,12 +139,10 @@ unsigned StringMapImpl::LookupBucketFor(StringRef Name,
 /// FindKey - Look up the bucket that contains the specified key. If it exists
 /// in the map, return the bucket number of the key.  Otherwise return -1.
 /// This does not modify the map.
-int StringMapImpl::FindKey(StringRef Key, uint32_t FullHashValue) const {
+int StringMapImpl::FindKey(StringRef Key) const {
   if (NumBuckets == 0)
     return -1; // Really empty table?
-#ifdef EXPENSIVE_CHECKS
-  assert(FullHashValue == hash(Key));
-#endif
+  unsigned FullHashValue = xxh3_64bits(Key);
   if (shouldReverseIterate())
     FullHashValue = ~FullHashValue;
   unsigned BucketNo = FullHashValue & (NumBuckets - 1);
diff --git a/llvm/lib/Support/Unix/Process.inc b/llvm/lib/Support/Unix/Process.inc
index 2babf07944bf7b8ba4c71f8115c516647e7c68f6..551f0d7f0f029dcf6b7526e9b2a5462538c4d606 100644
--- a/llvm/lib/Support/Unix/Process.inc
+++ b/llvm/lib/Support/Unix/Process.inc
@@ -62,7 +62,9 @@ getRUsageTimes() {
   ::getrusage(RUSAGE_SELF, &RU);
   return {toDuration(RU.ru_utime), toDuration(RU.ru_stime)};
 #else
+#ifndef __MVS__ // Exclude for MVS in case -pedantic is used
 #warning Cannot get usage times on this platform
+#endif
   return {std::chrono::microseconds::zero(), std::chrono::microseconds::zero()};
 #endif
 }
@@ -117,7 +119,9 @@ size_t Process::GetMallocUsage() {
     return EndOfMemory - StartOfMemory;
   return 0;
 #else
+#ifndef __MVS__ // Exclude for MVS in case -pedantic is used
 #warning Cannot get malloc info on this platform
+#endif
   return 0;
 #endif
 }
diff --git a/llvm/lib/Support/Unix/Program.inc b/llvm/lib/Support/Unix/Program.inc
index 9466d0f0ba859ee7f4e824ae5da846708639265c..260719b2b58d92a8a98e10f1313ef7b1729040fa 100644
--- a/llvm/lib/Support/Unix/Program.inc
+++ b/llvm/lib/Support/Unix/Program.inc
@@ -20,11 +20,13 @@
 #include "Unix.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Config/config.h"
+#include "llvm/Support/AutoConvert.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/Errc.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Path.h"
 #include "llvm/Support/StringSaver.h"
+#include "llvm/Support/SystemZ/zOSSupport.h"
 #include "llvm/Support/raw_ostream.h"
 #if HAVE_SYS_STAT_H
 #include <sys/stat.h>
@@ -466,7 +468,7 @@ ProcessInfo llvm::sys::Wait(const ProcessInfo &PI,
     std::chrono::microseconds UserT = toDuration(Info.ru_utime);
     std::chrono::microseconds KernelT = toDuration(Info.ru_stime);
     uint64_t PeakMemory = 0;
-#ifndef __HAIKU__
+#if !defined(__HAIKU__) && !defined(__MVS__)
     PeakMemory = static_cast<uint64_t>(Info.ru_maxrss);
 #endif
     *ProcStat = ProcessStatistics{UserT + KernelT, UserT, PeakMemory};
@@ -520,8 +522,12 @@ std::error_code llvm::sys::ChangeStdoutMode(fs::OpenFlags Flags) {
 }
 
 std::error_code llvm::sys::ChangeStdinToBinary() {
+#ifdef __MVS__
+  return disableAutoConversion(STDIN_FILENO);
+#else
   // Do nothing, as Unix doesn't differentiate between text and binary.
   return std::error_code();
+#endif
 }
 
 std::error_code llvm::sys::ChangeStdoutToBinary() {
diff --git a/llvm/lib/Support/raw_ostream.cpp b/llvm/lib/Support/raw_ostream.cpp
index 8908e7b6a150cab6a580c7e8a8a7a4719a3f4f44..28ab85d4344c30c72883e370ced62ac988012ba3 100644
--- a/llvm/lib/Support/raw_ostream.cpp
+++ b/llvm/lib/Support/raw_ostream.cpp
@@ -13,8 +13,10 @@
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/Config/config.h"
+#include "llvm/Support/AutoConvert.h"
 #include "llvm/Support/Compiler.h"
 #include "llvm/Support/Duration.h"
+#include "llvm/Support/Error.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/Format.h"
@@ -23,11 +25,17 @@
 #include "llvm/Support/NativeFormatting.h"
 #include "llvm/Support/Process.h"
 #include "llvm/Support/Program.h"
+#include "llvm/Support/Threading.h"
 #include <algorithm>
 #include <cerrno>
 #include <cstdio>
 #include <sys/stat.h>
 
+#ifndef _WIN32
+#include <sys/socket.h>
+#include <sys/un.h>
+#endif // _WIN32
+
 // <fcntl.h> may provide O_BINARY.
 #if defined(HAVE_FCNTL_H)
 # include <fcntl.h>
@@ -58,6 +66,13 @@
 #include "llvm/Support/ConvertUTF.h"
 #include "llvm/Support/Signals.h"
 #include "llvm/Support/Windows/WindowsSupport.h"
+// winsock2.h must be included before afunix.h. Briefly turn off clang-format to
+// avoid error.
+// clang-format off
+#include <winsock2.h>
+#include <afunix.h>
+// clang-format on
+#include <io.h>
 #endif
 
 using namespace llvm;
@@ -644,7 +659,7 @@ raw_fd_ostream::raw_fd_ostream(int fd, bool shouldClose, bool unbuffered,
   // Check if this is a console device. This is not equivalent to isatty.
   IsWindowsConsole =
       ::GetFileType((HANDLE)::_get_osfhandle(fd)) == FILE_TYPE_CHAR;
-#endif
+#endif // _WIN32
 
   // Get the starting position.
   off_t loc = ::lseek(FD, 0, SEEK_CUR);
@@ -895,6 +910,10 @@ void raw_fd_ostream::anchor() {}
 raw_fd_ostream &llvm::outs() {
   // Set buffer settings to model stdout behavior.
   std::error_code EC;
+#ifdef __MVS__
+  EC = enableAutoConversion(STDOUT_FILENO);
+  assert(!EC);
+#endif
   static raw_fd_ostream S("-", EC, sys::fs::OF_None);
   assert(!EC);
   return S;
@@ -902,6 +921,10 @@ raw_fd_ostream &llvm::outs() {
 
 raw_fd_ostream &llvm::errs() {
   // Set standard error to be unbuffered and tied to outs() by default.
+#ifdef __MVS__
+  std::error_code EC = enableAutoConversion(STDERR_FILENO);
+  assert(!EC);
+#endif
   static raw_fd_ostream S(STDERR_FILENO, false, true);
   return S;
 }
@@ -928,6 +951,9 @@ raw_fd_stream::raw_fd_stream(StringRef Filename, std::error_code &EC)
     EC = std::make_error_code(std::errc::invalid_argument);
 }
 
+raw_fd_stream::raw_fd_stream(int fd, bool shouldClose)
+    : raw_fd_ostream(fd, shouldClose, false, OStreamKind::OK_FDStream) {}
+
 ssize_t raw_fd_stream::read(char *Ptr, size_t Size) {
   assert(get_fd() >= 0 && "File already closed.");
   ssize_t Ret = ::read(get_fd(), (void *)Ptr, Size);
@@ -942,6 +968,145 @@ bool raw_fd_stream::classof(const raw_ostream *OS) {
   return OS->get_kind() == OStreamKind::OK_FDStream;
 }
 
+//===----------------------------------------------------------------------===//
+//  raw_socket_stream
+//===----------------------------------------------------------------------===//
+
+#ifdef _WIN32
+WSABalancer::WSABalancer() {
+  WSADATA WsaData;
+  ::memset(&WsaData, 0, sizeof(WsaData));
+  if (WSAStartup(MAKEWORD(2, 2), &WsaData) != 0) {
+    llvm::report_fatal_error("WSAStartup failed");
+  }
+}
+
+WSABalancer::~WSABalancer() { WSACleanup(); }
+
+#endif // _WIN32
+
+static std::error_code getLastSocketErrorCode() {
+#ifdef _WIN32
+  return std::error_code(::WSAGetLastError(), std::system_category());
+#else
+  return std::error_code(errno, std::system_category());
+#endif
+}
+
+ListeningSocket::ListeningSocket(int SocketFD, StringRef SocketPath)
+    : FD(SocketFD), SocketPath(SocketPath) {}
+
+ListeningSocket::ListeningSocket(ListeningSocket &&LS)
+    : FD(LS.FD), SocketPath(LS.SocketPath) {
+  LS.FD = -1;
+}
+
+Expected<ListeningSocket> ListeningSocket::createUnix(StringRef SocketPath,
+                                                      int MaxBacklog) {
+
+#ifdef _WIN32
+  WSABalancer _;
+  SOCKET MaybeWinsocket = socket(AF_UNIX, SOCK_STREAM, 0);
+  if (MaybeWinsocket == INVALID_SOCKET) {
+#else
+  int MaybeWinsocket = socket(AF_UNIX, SOCK_STREAM, 0);
+  if (MaybeWinsocket == -1) {
+#endif
+    return llvm::make_error<StringError>(getLastSocketErrorCode(),
+                                         "socket create failed");
+  }
+
+  struct sockaddr_un Addr;
+  memset(&Addr, 0, sizeof(Addr));
+  Addr.sun_family = AF_UNIX;
+  strncpy(Addr.sun_path, SocketPath.str().c_str(), sizeof(Addr.sun_path) - 1);
+
+  if (bind(MaybeWinsocket, (struct sockaddr *)&Addr, sizeof(Addr)) == -1) {
+    std::error_code Err = getLastSocketErrorCode();
+    if (Err == std::errc::address_in_use)
+      ::close(MaybeWinsocket);
+    return llvm::make_error<StringError>(Err, "Bind error");
+  }
+  if (listen(MaybeWinsocket, MaxBacklog) == -1) {
+    return llvm::make_error<StringError>(getLastSocketErrorCode(),
+                                         "Listen error");
+  }
+  int UnixSocket;
+#ifdef _WIN32
+  UnixSocket = _open_osfhandle(MaybeWinsocket, 0);
+#else
+  UnixSocket = MaybeWinsocket;
+#endif // _WIN32
+  return ListeningSocket{UnixSocket, SocketPath};
+}
+
+Expected<std::unique_ptr<raw_socket_stream>> ListeningSocket::accept() {
+  int AcceptFD;
+#ifdef _WIN32
+  SOCKET WinServerSock = _get_osfhandle(FD);
+  SOCKET WinAcceptSock = ::accept(WinServerSock, NULL, NULL);
+  AcceptFD = _open_osfhandle(WinAcceptSock, 0);
+#else
+  AcceptFD = ::accept(FD, NULL, NULL);
+#endif //_WIN32
+  if (AcceptFD == -1)
+    return llvm::make_error<StringError>(getLastSocketErrorCode(),
+                                         "Accept failed");
+  return std::make_unique<raw_socket_stream>(AcceptFD);
+}
+
+ListeningSocket::~ListeningSocket() {
+  if (FD == -1)
+    return;
+  ::close(FD);
+  unlink(SocketPath.c_str());
+}
+
+static Expected<int> GetSocketFD(StringRef SocketPath) {
+#ifdef _WIN32
+  SOCKET MaybeWinsocket = socket(AF_UNIX, SOCK_STREAM, 0);
+  if (MaybeWinsocket == INVALID_SOCKET) {
+#else
+  int MaybeWinsocket = socket(AF_UNIX, SOCK_STREAM, 0);
+  if (MaybeWinsocket == -1) {
+#endif // _WIN32
+    return llvm::make_error<StringError>(getLastSocketErrorCode(),
+                                         "Create socket failed");
+  }
+
+  struct sockaddr_un Addr;
+  memset(&Addr, 0, sizeof(Addr));
+  Addr.sun_family = AF_UNIX;
+  strncpy(Addr.sun_path, SocketPath.str().c_str(), sizeof(Addr.sun_path) - 1);
+
+  int status = connect(MaybeWinsocket, (struct sockaddr *)&Addr, sizeof(Addr));
+  if (status == -1) {
+    return llvm::make_error<StringError>(getLastSocketErrorCode(),
+                                         "Connect socket failed");
+  }
+#ifdef _WIN32
+  return _open_osfhandle(MaybeWinsocket, 0);
+#else
+  return MaybeWinsocket;
+#endif // _WIN32
+}
+
+raw_socket_stream::raw_socket_stream(int SocketFD)
+    : raw_fd_stream(SocketFD, true) {}
+
+Expected<std::unique_ptr<raw_socket_stream>>
+raw_socket_stream::createConnectedUnix(StringRef SocketPath) {
+#ifdef _WIN32
+  WSABalancer _;
+#endif // _WIN32
+  Expected<int> FD = GetSocketFD(SocketPath);
+  if (!FD)
+    return FD.takeError();
+  return std::make_unique<raw_socket_stream>(*FD);
+}
+
+raw_socket_stream::~raw_socket_stream() {}
+
 //===----------------------------------------------------------------------===//
 //  raw_string_ostream
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index 50cbd3672fbd0d749ae1718ae6babd10d0d57b09..7d71c316bcb0a2265e2ff7388faa0130053dd3ae 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -9185,19 +9185,32 @@ AArch64InstrInfo::isCopyInstrImpl(const MachineInstr &MI) const {
   // and zero immediate operands used as an alias for mov instruction.
   if (MI.getOpcode() == AArch64::ORRWrs &&
       MI.getOperand(1).getReg() == AArch64::WZR &&
-      MI.getOperand(3).getImm() == 0x0) {
+      MI.getOperand(3).getImm() == 0x0 &&
+      // Check that the w->w move is not a zero-extending w->x mov.
+      (!MI.getOperand(0).getReg().isVirtual() ||
+       MI.getOperand(0).getSubReg() == 0) &&
+      (!MI.getOperand(0).getReg().isPhysical() ||
+       MI.findRegisterDefOperandIdx(MI.getOperand(0).getReg() - AArch64::W0 +
+                                    AArch64::X0) == -1))
     return DestSourcePair{MI.getOperand(0), MI.getOperand(2)};
-  }
 
   if (MI.getOpcode() == AArch64::ORRXrs &&
       MI.getOperand(1).getReg() == AArch64::XZR &&
-      MI.getOperand(3).getImm() == 0x0) {
+      MI.getOperand(3).getImm() == 0x0)
     return DestSourcePair{MI.getOperand(0), MI.getOperand(2)};
-  }
 
   return std::nullopt;
 }
 
+std::optional<DestSourcePair>
+AArch64InstrInfo::isCopyLikeInstrImpl(const MachineInstr &MI) const {
+  if (MI.getOpcode() == AArch64::ORRWrs &&
+      MI.getOperand(1).getReg() == AArch64::WZR &&
+      MI.getOperand(3).getImm() == 0x0)
+    return DestSourcePair{MI.getOperand(0), MI.getOperand(2)};
+  return std::nullopt;
+}
+
 std::optional<RegImmPair>
 AArch64InstrInfo::isAddImmediate(const MachineInstr &MI, Register Reg) const {
   int Sign = 1;
@@ -9241,7 +9254,7 @@ static std::optional<ParamLoadedValue>
 describeORRLoadedValue(const MachineInstr &MI, Register DescribedReg,
                        const TargetInstrInfo *TII,
                        const TargetRegisterInfo *TRI) {
-  auto DestSrc = TII->isCopyInstr(MI);
+  auto DestSrc = TII->isCopyLikeInstr(MI);
   if (!DestSrc)
     return std::nullopt;
 
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.h b/llvm/lib/Target/AArch64/AArch64InstrInfo.h
index e97ff0a9758d69226a6802bea4bea46030daa73c..6526f6740747abb362ebb8a1d2584a032e78d82f 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.h
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.h
@@ -401,6 +401,8 @@ protected:
   /// registers as machine operands.
   std::optional<DestSourcePair>
   isCopyInstrImpl(const MachineInstr &MI) const override;
+  std::optional<DestSourcePair>
+  isCopyLikeInstrImpl(const MachineInstr &MI) const override;
 
 private:
   unsigned getInstBundleLength(const MachineInstr &MI) const;
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 21cafe9b6c4453a912917b3cdc9ffcf106f377eb..50527e08a06165b9c0c36f640f709e349dafba56 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -4063,11 +4063,11 @@ defm BFCLAMP_ZZZ : sve2p1_bfclamp<"bfclamp", int_aarch64_sve_fclamp>;
 // SME2.1 or SVE2.1 instructions
 //===----------------------------------------------------------------------===//
 let Predicates = [HasSVE2p1_or_HasSME2p1] in {
-defm FADDQV   : sve2p1_fp_reduction_q<0b000, "faddqv">;
-defm FMAXNMQV : sve2p1_fp_reduction_q<0b100, "fmaxnmqv">;
-defm FMINNMQV : sve2p1_fp_reduction_q<0b101, "fminnmqv">;
-defm FMAXQV   : sve2p1_fp_reduction_q<0b110, "fmaxqv">;
-defm FMINQV   : sve2p1_fp_reduction_q<0b111, "fminqv">;
+defm FADDQV   : sve2p1_fp_reduction_q<0b000, "faddqv", int_aarch64_sve_addqv>;
+defm FMAXNMQV : sve2p1_fp_reduction_q<0b100, "fmaxnmqv", int_aarch64_sve_fmaxnmqv>;
+defm FMINNMQV : sve2p1_fp_reduction_q<0b101, "fminnmqv", int_aarch64_sve_fminnmqv>;
+defm FMAXQV   : sve2p1_fp_reduction_q<0b110, "fmaxqv", int_aarch64_sve_fmaxqv>;
+defm FMINQV   : sve2p1_fp_reduction_q<0b111, "fminqv", int_aarch64_sve_fminqv>;
 
 defm DUPQ_ZZI : sve2p1_dupq<"dupq">;
 defm EXTQ_ZZI : sve2p1_extq<"extq", int_aarch64_sve_extq_lane>;
@@ -4075,14 +4075,14 @@ defm EXTQ_ZZI : sve2p1_extq<"extq", int_aarch64_sve_extq_lane>;
 defm PMOV_PZI : sve2p1_vector_to_pred<"pmov", int_aarch64_sve_pmov_to_pred_lane, int_aarch64_sve_pmov_to_pred_lane_zero>;
 defm PMOV_ZIP : sve2p1_pred_to_vector<"pmov", int_aarch64_sve_pmov_to_vector_lane_merging, int_aarch64_sve_pmov_to_vector_lane_zeroing>;
 
-defm ORQV_VPZ   : sve2p1_int_reduce_q<0b1100, "orqv">;
-defm EORQV_VPZ  : sve2p1_int_reduce_q<0b1101, "eorqv">;
-defm ANDQV_VPZ  : sve2p1_int_reduce_q<0b1110, "andqv">;
-defm ADDQV_VPZ  : sve2p1_int_reduce_q<0b0001, "addqv">;
-defm SMAXQV_VPZ : sve2p1_int_reduce_q<0b0100, "smaxqv">;
-defm UMAXQV_VPZ : sve2p1_int_reduce_q<0b0101, "umaxqv">;
-defm SMINQV_VPZ : sve2p1_int_reduce_q<0b0110, "sminqv">;
-defm UMINQV_VPZ : sve2p1_int_reduce_q<0b0111, "uminqv">;
+defm ORQV_VPZ   : sve2p1_int_reduce_q<0b1100, "orqv", int_aarch64_sve_orqv>;
+defm EORQV_VPZ  : sve2p1_int_reduce_q<0b1101, "eorqv", int_aarch64_sve_eorqv>;
+defm ANDQV_VPZ  : sve2p1_int_reduce_q<0b1110, "andqv", int_aarch64_sve_andqv>;
+defm ADDQV_VPZ  : sve2p1_int_reduce_q<0b0001, "addqv", int_aarch64_sve_addqv>;
+defm SMAXQV_VPZ : sve2p1_int_reduce_q<0b0100, "smaxqv", int_aarch64_sve_smaxqv>;
+defm UMAXQV_VPZ : sve2p1_int_reduce_q<0b0101, "umaxqv", int_aarch64_sve_umaxqv>;
+defm SMINQV_VPZ : sve2p1_int_reduce_q<0b0110, "sminqv", int_aarch64_sve_sminqv>;
+defm UMINQV_VPZ : sve2p1_int_reduce_q<0b0111, "uminqv", int_aarch64_sve_uminqv>;
 
 defm ZIPQ1_ZZZ : sve2p1_permute_vec_elems_q<0b000, "zipq1", int_aarch64_sve_zipq1>;
 defm ZIPQ2_ZZZ : sve2p1_permute_vec_elems_q<0b001, "zipq2", int_aarch64_sve_zipq2>;
diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td
index c0894e9c70680a441420d230d41d993e332bd515..9edf26052247aed3237d181e70df43901ec38667 100644
--- a/llvm/lib/Target/AArch64/SVEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td
@@ -9963,10 +9963,14 @@ class sve2p1_fp_reduction_q<bits<2> sz, bits<3> opc, string mnemonic,
   let mayRaiseFPException = 1;
 }
 
-multiclass sve2p1_fp_reduction_q<bits<3> opc, string mnemonic> {
+multiclass sve2p1_fp_reduction_q<bits<3> opc, string mnemonic, SDPatternOperator op> {
   def _H : sve2p1_fp_reduction_q<0b01, opc, mnemonic, ZPR16, "8h">;
   def _S : sve2p1_fp_reduction_q<0b10, opc, mnemonic, ZPR32, "4s">;
   def _D : sve2p1_fp_reduction_q<0b11, opc, mnemonic, ZPR64, "2d">;
+
+  def : SVE_2_Op_Pat<v8f16, op, nxv8i1, nxv8f16, !cast<Instruction>(NAME # _H)>;
+  def : SVE_2_Op_Pat<v4f32, op, nxv4i1, nxv4f32, !cast<Instruction>(NAME # _S)>;
+  def : SVE_2_Op_Pat<v2f64, op, nxv2i1, nxv2f64, !cast<Instruction>(NAME # _D)>;
 }
 
 
@@ -10183,11 +10187,16 @@ class sve2p1_int_reduce_q<bits<2> sz, bits<4> opc, string mnemonic,
   let hasSideEffects = 0;
 }
 
-multiclass sve2p1_int_reduce_q<bits<4> opc, string mnemonic> {
+multiclass sve2p1_int_reduce_q<bits<4> opc, string mnemonic, SDPatternOperator op> {
   def _B : sve2p1_int_reduce_q<0b00, opc, mnemonic, ZPR8,  "16b">;
   def _H : sve2p1_int_reduce_q<0b01, opc, mnemonic, ZPR16, "8h">;
   def _S : sve2p1_int_reduce_q<0b10, opc, mnemonic, ZPR32, "4s">;
   def _D : sve2p1_int_reduce_q<0b11, opc, mnemonic, ZPR64, "2d">;
+
+  def : SVE_2_Op_Pat<v16i8, op, nxv16i1, nxv16i8, !cast<Instruction>(NAME # _B)>;
+  def : SVE_2_Op_Pat<v8i16, op, nxv8i1, nxv8i16, !cast<Instruction>(NAME # _H)>;
+  def : SVE_2_Op_Pat<v4i32, op, nxv4i1, nxv4i32, !cast<Instruction>(NAME # _S)>;
+  def : SVE_2_Op_Pat<v2i64, op, nxv2i1, nxv2i64, !cast<Instruction>(NAME # _D)>;
 }
 
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h
index 1b75607e1dc32b15cb97d57a1dff5ffe9bd79954..89319527c410a6e170fd371691fbf89bfc7a6e03 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.h
@@ -36,6 +36,7 @@ FunctionPass *createSIAnnotateControlFlowPass();
 FunctionPass *createSIFoldOperandsPass();
 FunctionPass *createSIPeepholeSDWAPass();
 FunctionPass *createSILowerI1CopiesPass();
+FunctionPass *createAMDGPUGlobalISelDivergenceLoweringPass();
 FunctionPass *createSIShrinkInstructionsPass();
 FunctionPass *createSILoadStoreOptimizerPass();
 FunctionPass *createSIWholeQuadModePass();
@@ -162,6 +163,9 @@ extern char &SILowerWWMCopiesID;
 void initializeSILowerI1CopiesPass(PassRegistry &);
 extern char &SILowerI1CopiesID;
 
+void initializeAMDGPUGlobalISelDivergenceLoweringPass(PassRegistry &);
+extern char &AMDGPUGlobalISelDivergenceLoweringID;
+
 void initializeSILowerSGPRSpillsPass(PassRegistry &);
 extern char &SILowerSGPRSpillsID;
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
index 4bf1f1357b694eebf09672927eed03ae3d4481e2..d317a733d4331c1437d9588bbda2f986302ba083 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUAsmPrinter.cpp
@@ -426,7 +426,7 @@ amdhsa::kernel_descriptor_t AMDGPUAsmPrinter::getAmdhsaKernelDescriptor(
   memset(&KernelDescriptor, 0x0, sizeof(KernelDescriptor));
 
   assert(isUInt<32>(PI.ScratchSize));
-  assert(isUInt<32>(PI.getComputePGMRSrc1()));
+  assert(isUInt<32>(PI.getComputePGMRSrc1(STM)));
   assert(isUInt<32>(PI.getComputePGMRSrc2()));
 
   KernelDescriptor.group_segment_fixed_size = PI.LDSSize;
@@ -435,7 +435,7 @@ amdhsa::kernel_descriptor_t AMDGPUAsmPrinter::getAmdhsaKernelDescriptor(
   Align MaxKernArgAlign;
   KernelDescriptor.kernarg_size = STM.getKernArgSegmentSize(F, MaxKernArgAlign);
 
-  KernelDescriptor.compute_pgm_rsrc1 = PI.getComputePGMRSrc1();
+  KernelDescriptor.compute_pgm_rsrc1 = PI.getComputePGMRSrc1(STM);
   KernelDescriptor.compute_pgm_rsrc2 = PI.getComputePGMRSrc2();
   KernelDescriptor.kernel_code_properties = getAmdhsaKernelCodeProperties(MF);
 
@@ -974,7 +974,7 @@ void AMDGPUAsmPrinter::EmitProgramInfoSI(const MachineFunction &MF,
   if (AMDGPU::isCompute(MF.getFunction().getCallingConv())) {
     OutStreamer->emitInt32(R_00B848_COMPUTE_PGM_RSRC1);
 
-    OutStreamer->emitInt32(CurrentProgramInfo.getComputePGMRSrc1());
+    OutStreamer->emitInt32(CurrentProgramInfo.getComputePGMRSrc1(STM));
 
     OutStreamer->emitInt32(R_00B84C_COMPUTE_PGM_RSRC2);
     OutStreamer->emitInt32(CurrentProgramInfo.getComputePGMRSrc2());
@@ -1038,7 +1038,7 @@ void AMDGPUAsmPrinter::EmitPALMetadata(const MachineFunction &MF,
 
   MD->setNumUsedSgprs(CC, CurrentProgramInfo.NumSGPRsForWavesPerEU);
   if (MD->getPALMajorVersion() < 3) {
-    MD->setRsrc1(CC, CurrentProgramInfo.getPGMRSrc1(CC));
+    MD->setRsrc1(CC, CurrentProgramInfo.getPGMRSrc1(CC, STM));
     if (AMDGPU::isCompute(CC)) {
       MD->setRsrc2(CC, CurrentProgramInfo.getComputePGMRSrc2());
     } else {
@@ -1116,10 +1116,11 @@ void AMDGPUAsmPrinter::emitPALFunctionMetadata(const MachineFunction &MF) {
   const MachineFrameInfo &MFI = MF.getFrameInfo();
   StringRef FnName = MF.getFunction().getName();
   MD->setFunctionScratchSize(FnName, MFI.getStackSize());
+  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
 
   // Set compute registers
   MD->setRsrc1(CallingConv::AMDGPU_CS,
-               CurrentProgramInfo.getPGMRSrc1(CallingConv::AMDGPU_CS));
+               CurrentProgramInfo.getPGMRSrc1(CallingConv::AMDGPU_CS, ST));
   MD->setRsrc2(CallingConv::AMDGPU_CS, CurrentProgramInfo.getComputePGMRSrc2());
 
   // Set optional info
@@ -1155,7 +1156,7 @@ void AMDGPUAsmPrinter::getAmdKernelCode(amd_kernel_code_t &Out,
   AMDGPU::initDefaultAMDKernelCodeT(Out, &STM);
 
   Out.compute_pgm_resource_registers =
-      CurrentProgramInfo.getComputePGMRSrc1() |
+      CurrentProgramInfo.getComputePGMRSrc1(STM) |
       (CurrentProgramInfo.getComputePGMRSrc2() << 32);
   Out.code_properties |= AMD_CODE_PROPERTY_IS_PTR64;
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
index 4caa9cd9225b6900f86783956dfe7991b73ce0f1..87b1957c799e2cddec818967e8287dd7bb553136 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
@@ -2199,7 +2199,7 @@ bool AMDGPUCodeGenPrepare::runOnFunction(Function &F) {
   auto *DTWP = getAnalysisIfAvailable<DominatorTreeWrapperPass>();
   Impl.DT = DTWP ? &DTWP->getDomTree() : nullptr;
   Impl.HasUnsafeFPMath = hasUnsafeFPMath(F);
-  SIModeRegisterDefaults Mode(F);
+  SIModeRegisterDefaults Mode(F, *Impl.ST);
   Impl.HasFP32DenormalFlush =
       Mode.FP32Denormals == DenormalMode::getPreserveSign();
   return Impl.run(F);
@@ -2216,7 +2216,7 @@ PreservedAnalyses AMDGPUCodeGenPreparePass::run(Function &F,
   Impl.UA = &FAM.getResult<UniformityInfoAnalysis>(F);
   Impl.DT = FAM.getCachedResult<DominatorTreeAnalysis>(F);
   Impl.HasUnsafeFPMath = hasUnsafeFPMath(F);
-  SIModeRegisterDefaults Mode(F);
+  SIModeRegisterDefaults Mode(F, *Impl.ST);
   Impl.HasFP32DenormalFlush =
       Mode.FP32Denormals == DenormalMode::getPreserveSign();
   PreservedAnalyses PA = PreservedAnalyses::none();
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCombinerHelper.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCombinerHelper.cpp
index d48916670112c0fc9259539da1fd01f2430305dc..69dc78d33c838cb93a2d4a58a9afd74065247fb1 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCombinerHelper.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCombinerHelper.cpp
@@ -29,6 +29,8 @@ static bool fnegFoldsIntoMI(const MachineInstr &MI) {
   case AMDGPU::G_FMAXNUM:
   case AMDGPU::G_FMINNUM_IEEE:
   case AMDGPU::G_FMAXNUM_IEEE:
+  case AMDGPU::G_FMINIMUM:
+  case AMDGPU::G_FMAXIMUM:
   case AMDGPU::G_FSIN:
   case AMDGPU::G_FPEXT:
   case AMDGPU::G_INTRINSIC_TRUNC:
@@ -174,6 +176,10 @@ static unsigned inverseMinMax(unsigned Opc) {
     return AMDGPU::G_FMINNUM_IEEE;
   case AMDGPU::G_FMINNUM_IEEE:
     return AMDGPU::G_FMAXNUM_IEEE;
+  case AMDGPU::G_FMAXIMUM:
+    return AMDGPU::G_FMINIMUM;
+  case AMDGPU::G_FMINIMUM:
+    return AMDGPU::G_FMAXIMUM;
   case AMDGPU::G_AMDGPU_FMAX_LEGACY:
     return AMDGPU::G_AMDGPU_FMIN_LEGACY;
   case AMDGPU::G_AMDGPU_FMIN_LEGACY:
@@ -207,6 +213,8 @@ bool AMDGPUCombinerHelper::matchFoldableFneg(MachineInstr &MI,
   case AMDGPU::G_FMAXNUM:
   case AMDGPU::G_FMINNUM_IEEE:
   case AMDGPU::G_FMAXNUM_IEEE:
+  case AMDGPU::G_FMINIMUM:
+  case AMDGPU::G_FMAXIMUM:
   case AMDGPU::G_AMDGPU_FMIN_LEGACY:
   case AMDGPU::G_AMDGPU_FMAX_LEGACY:
     // 0 doesn't have a negated inline immediate.
@@ -304,6 +312,8 @@ void AMDGPUCombinerHelper::applyFoldableFneg(MachineInstr &MI,
   case AMDGPU::G_FMAXNUM:
   case AMDGPU::G_FMINNUM_IEEE:
   case AMDGPU::G_FMAXNUM_IEEE:
+  case AMDGPU::G_FMINIMUM:
+  case AMDGPU::G_FMAXIMUM:
   case AMDGPU::G_AMDGPU_FMIN_LEGACY:
   case AMDGPU::G_AMDGPU_FMAX_LEGACY: {
     NegateOperand(MatchInfo->getOperand(1));
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelDivergenceLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelDivergenceLowering.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..4cd8b1ec1051f4b928e5f4a42d189fdc94c191ef
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelDivergenceLowering.cpp
@@ -0,0 +1,68 @@
+//===-- AMDGPUGlobalISelDivergenceLowering.cpp ----------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file
+/// GlobalISel pass that selects divergent i1 phis as lane mask phis.
+/// Lane mask merging uses same algorithm as SDAG in SILowerI1Copies.
+/// Handles all cases of temporal divergence.
+/// For divergent non-phi i1 and uniform i1 uses outside of the cycle this pass
+/// currently depends on LCSSA to insert phis with one incoming.
+//
+//===----------------------------------------------------------------------===//
+
+#include "AMDGPU.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+
+#define DEBUG_TYPE "amdgpu-global-isel-divergence-lowering"
+
+using namespace llvm;
+
+namespace {
+
+class AMDGPUGlobalISelDivergenceLowering : public MachineFunctionPass {
+public:
+  static char ID;
+
+public:
+  AMDGPUGlobalISelDivergenceLowering() : MachineFunctionPass(ID) {
+    initializeAMDGPUGlobalISelDivergenceLoweringPass(
+        *PassRegistry::getPassRegistry());
+  }
+
+  bool runOnMachineFunction(MachineFunction &MF) override;
+
+  StringRef getPassName() const override {
+    return "AMDGPU GlobalISel divergence lowering";
+  }
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.setPreservesCFG();
+    MachineFunctionPass::getAnalysisUsage(AU);
+  }
+};
+
+} // End anonymous namespace.
+
+INITIALIZE_PASS_BEGIN(AMDGPUGlobalISelDivergenceLowering, DEBUG_TYPE,
+                      "AMDGPU GlobalISel divergence lowering", false, false)
+INITIALIZE_PASS_END(AMDGPUGlobalISelDivergenceLowering, DEBUG_TYPE,
+                    "AMDGPU GlobalISel divergence lowering", false, false)
+
+char AMDGPUGlobalISelDivergenceLowering::ID = 0;
+
+char &llvm::AMDGPUGlobalISelDivergenceLoweringID =
+    AMDGPUGlobalISelDivergenceLowering::ID;
+
+FunctionPass *llvm::createAMDGPUGlobalISelDivergenceLoweringPass() {
+  return new AMDGPUGlobalISelDivergenceLowering();
+}
+
+bool AMDGPUGlobalISelDivergenceLowering::runOnMachineFunction(
+    MachineFunction &MF) {
+  return false;
+}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
index a6d1da94b890780a128b9d45d353d51ada293f46..66ba08ef0dc12abe69b29906310b8130d47fb9fb 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
@@ -132,7 +132,7 @@ bool AMDGPUDAGToDAGISel::runOnMachineFunction(MachineFunction &MF) {
   }
 #endif
   Subtarget = &MF.getSubtarget<GCNSubtarget>();
-  Mode = SIModeRegisterDefaults(MF.getFunction());
+  Mode = SIModeRegisterDefaults(MF.getFunction(), *Subtarget);
   return SelectionDAGISel::runOnMachineFunction(MF);
 }
 
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
index fcbdf51b03c1fc8a1bcb9f76c6395f67d706cfae..9d7443012e3da33ef7510a0437f08f0e46d7bfdd 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -585,6 +585,8 @@ static bool fnegFoldsIntoOpcode(unsigned Opc) {
   case ISD::FMAXNUM:
   case ISD::FMINNUM_IEEE:
   case ISD::FMAXNUM_IEEE:
+  case ISD::FMINIMUM:
+  case ISD::FMAXIMUM:
   case ISD::SELECT:
   case ISD::FSIN:
   case ISD::FTRUNC:
@@ -4572,6 +4574,10 @@ static unsigned inverseMinMax(unsigned Opc) {
     return ISD::FMINNUM_IEEE;
   case ISD::FMINNUM_IEEE:
     return ISD::FMAXNUM_IEEE;
+  case ISD::FMAXIMUM:
+    return ISD::FMINIMUM;
+  case ISD::FMINIMUM:
+    return ISD::FMAXIMUM;
   case AMDGPUISD::FMAX_LEGACY:
     return AMDGPUISD::FMIN_LEGACY;
   case AMDGPUISD::FMIN_LEGACY:
@@ -4695,6 +4701,8 @@ SDValue AMDGPUTargetLowering::performFNegCombine(SDNode *N,
   case ISD::FMINNUM:
   case ISD::FMAXNUM_IEEE:
   case ISD::FMINNUM_IEEE:
+  case ISD::FMINIMUM:
+  case ISD::FMAXIMUM:
   case AMDGPUISD::FMAX_LEGACY:
   case AMDGPUISD::FMIN_LEGACY: {
     // fneg (fmaxnum x, y) -> fminnum (fneg x), (fneg y)
@@ -5305,6 +5313,8 @@ const char* AMDGPUTargetLowering::getTargetNodeName(unsigned Opcode) const {
   NODE_NAME_CASE(FMED3)
   NODE_NAME_CASE(SMED3)
   NODE_NAME_CASE(UMED3)
+  NODE_NAME_CASE(FMAXIMUM3)
+  NODE_NAME_CASE(FMINIMUM3)
   NODE_NAME_CASE(FDOT2)
   NODE_NAME_CASE(URECIP)
   NODE_NAME_CASE(DIV_SCALE)
@@ -5759,6 +5769,8 @@ bool AMDGPUTargetLowering::isKnownNeverNaNForTargetNode(SDValue Op,
   case AMDGPUISD::FMED3:
   case AMDGPUISD::FMIN3:
   case AMDGPUISD::FMAX3:
+  case AMDGPUISD::FMINIMUM3:
+  case AMDGPUISD::FMAXIMUM3:
   case AMDGPUISD::FMAD_FTZ: {
     if (SNaN)
       return true;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
index 6841067e31b3b4ef1352e5024e46341ec1d83f83..827fb106b551994ea7a0d3b116f9e512595c1c31 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h
@@ -449,6 +449,8 @@ enum NodeType : unsigned {
   FMED3,
   SMED3,
   UMED3,
+  FMAXIMUM3,
+  FMINIMUM3,
   FDOT2,
   URECIP,
   DIV_SCALE,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstrInfo.td b/llvm/lib/Target/AMDGPU/AMDGPUInstrInfo.td
index fd38739876c4d57f9e91535d7453297a3d790310..82f58ea38fd0a7221815b003795f6ed1af25b32b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstrInfo.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstrInfo.td
@@ -170,6 +170,11 @@ def AMDGPUfmax3 : SDNode<"AMDGPUISD::FMAX3", SDTFPTernaryOp,
   [/*SDNPCommutative, SDNPAssociative*/]
 >;
 
+// out = max(a, b, c) a, b and c are floats. Operation is IEEE2019 compliant.
+def AMDGPUfmaximum3 : SDNode<"AMDGPUISD::FMAXIMUM3", SDTFPTernaryOp,
+  [/*SDNPCommutative, SDNPAssociative*/]
+>;
+
 // out = max(a, b, c) a, b, and c are signed ints
 def AMDGPUsmax3 : SDNode<"AMDGPUISD::SMAX3", AMDGPUDTIntTernaryOp,
   [/*SDNPCommutative, SDNPAssociative*/]
@@ -185,6 +190,11 @@ def AMDGPUfmin3 : SDNode<"AMDGPUISD::FMIN3", SDTFPTernaryOp,
   [/*SDNPCommutative, SDNPAssociative*/]
 >;
 
+// out = min(a, b, c) a, b and c are floats. Operation is IEEE2019 compliant.
+def AMDGPUfminimum3 : SDNode<"AMDGPUISD::FMINIMUM3", SDTFPTernaryOp,
+  [/*SDNPCommutative, SDNPAssociative*/]
+>;
+
 // out = min(a, b, c) a, b and c are signed ints
 def AMDGPUsmin3 : SDNode<"AMDGPUISD::SMIN3", AMDGPUDTIntTernaryOp,
   [/*SDNPCommutative, SDNPAssociative*/]
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
index d24c7da964ce85057921459469f133513959a030..75fac09d0b99fadd582485c12edee1d77a72af54 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.cpp
@@ -1791,6 +1791,19 @@ bool AMDGPUInstructionSelector::selectSBarrier(MachineInstr &MI) const {
       return true;
     }
   }
+
+  // On GFX12 lower s_barrier into s_barrier_signal_imm and s_barrier_wait
+  if (STI.hasSplitBarriers()) {
+    MachineBasicBlock *MBB = MI.getParent();
+    const DebugLoc &DL = MI.getDebugLoc();
+    BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::S_BARRIER_SIGNAL_IMM))
+        .addImm(AMDGPU::Barrier::WORKGROUP);
+    BuildMI(*MBB, &MI, DL, TII.get(AMDGPU::S_BARRIER_WAIT))
+        .addImm(AMDGPU::Barrier::WORKGROUP);
+    MI.eraseFromParent();
+    return true;
+  }
+
   return selectImpl(MI, *CoverageInfo);
 }
 
@@ -2137,6 +2150,16 @@ bool AMDGPUInstructionSelector::selectG_INTRINSIC_W_SIDE_EFFECTS(
     break;
   case Intrinsic::amdgcn_ds_bvh_stack_rtn:
     return selectDSBvhStackIntrinsic(I);
+  case Intrinsic::amdgcn_s_barrier_init:
+  case Intrinsic::amdgcn_s_barrier_join:
+  case Intrinsic::amdgcn_s_wakeup_barrier:
+  case Intrinsic::amdgcn_s_get_barrier_state:
+    return selectNamedBarrierInst(I, IntrinsicID);
+  case Intrinsic::amdgcn_s_barrier_signal_isfirst:
+  case Intrinsic::amdgcn_s_barrier_signal_isfirst_var:
+    return selectSBarrierSignalIsfirst(I, IntrinsicID);
+  case Intrinsic::amdgcn_s_barrier_leave:
+    return selectSBarrierLeave(I);
   }
   return selectImpl(I, *CoverageInfo);
 }
@@ -5239,6 +5262,135 @@ AMDGPUInstructionSelector::selectVOP3PMadMixMods(MachineOperand &Root) const {
   }};
 }
 
+bool AMDGPUInstructionSelector::selectSBarrierSignalIsfirst(
+    MachineInstr &I, Intrinsic::ID IntrID) const {
+  MachineBasicBlock *MBB = I.getParent();
+  const DebugLoc &DL = I.getDebugLoc();
+  Register CCReg = I.getOperand(0).getReg();
+
+  bool HasM0 = IntrID == Intrinsic::amdgcn_s_barrier_signal_isfirst_var;
+
+  if (HasM0) {
+    auto CopyMIB = BuildMI(*MBB, &I, DL, TII.get(AMDGPU::COPY), AMDGPU::M0)
+                       .addReg(I.getOperand(2).getReg());
+    BuildMI(*MBB, &I, DL, TII.get(AMDGPU::S_BARRIER_SIGNAL_ISFIRST_M0));
+    if (!constrainSelectedInstRegOperands(*CopyMIB, TII, TRI, RBI))
+      return false;
+  } else {
+    BuildMI(*MBB, &I, DL, TII.get(AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM))
+        .addImm(I.getOperand(2).getImm());
+  }
+
+  BuildMI(*MBB, &I, DL, TII.get(AMDGPU::COPY), CCReg).addReg(AMDGPU::SCC);
+
+  I.eraseFromParent();
+  return RBI.constrainGenericRegister(CCReg, AMDGPU::SReg_32_XM0_XEXECRegClass,
+                                      *MRI);
+}
+
+unsigned getNamedBarrierOp(bool HasInlineConst, Intrinsic::ID IntrID) {
+  if (HasInlineConst) {
+    switch (IntrID) {
+    default:
+      llvm_unreachable("not a named barrier op");
+    case Intrinsic::amdgcn_s_barrier_init:
+      return AMDGPU::S_BARRIER_INIT_IMM;
+    case Intrinsic::amdgcn_s_barrier_join:
+      return AMDGPU::S_BARRIER_JOIN_IMM;
+    case Intrinsic::amdgcn_s_wakeup_barrier:
+      return AMDGPU::S_WAKEUP_BARRIER_IMM;
+    case Intrinsic::amdgcn_s_get_barrier_state:
+      return AMDGPU::S_GET_BARRIER_STATE_IMM;
+    };
+  } else {
+    switch (IntrID) {
+    default:
+      llvm_unreachable("not a named barrier op");
+    case Intrinsic::amdgcn_s_barrier_init:
+      return AMDGPU::S_BARRIER_INIT_M0;
+    case Intrinsic::amdgcn_s_barrier_join:
+      return AMDGPU::S_BARRIER_JOIN_M0;
+    case Intrinsic::amdgcn_s_wakeup_barrier:
+      return AMDGPU::S_WAKEUP_BARRIER_M0;
+    case Intrinsic::amdgcn_s_get_barrier_state:
+      return AMDGPU::S_GET_BARRIER_STATE_M0;
+    };
+  }
+}
+
+bool AMDGPUInstructionSelector::selectNamedBarrierInst(
+    MachineInstr &I, Intrinsic::ID IntrID) const {
+  MachineBasicBlock *MBB = I.getParent();
+  const DebugLoc &DL = I.getDebugLoc();
+  MachineOperand BarOp = IntrID == Intrinsic::amdgcn_s_get_barrier_state
+                             ? I.getOperand(2)
+                             : I.getOperand(1);
+  std::optional<int64_t> BarValImm =
+      getIConstantVRegSExtVal(BarOp.getReg(), *MRI);
+  Register M0Val;
+  Register TmpReg0;
+
+  // For S_BARRIER_INIT, member count will always be read from M0[16:22]
+  if (IntrID == Intrinsic::amdgcn_s_barrier_init) {
+    Register MemberCount = I.getOperand(2).getReg();
+    TmpReg0 = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
+    // TODO: This should be expanded during legalization so that the the S_LSHL
+    // and S_OR can be constant-folded
+    BuildMI(*MBB, &I, DL, TII.get(AMDGPU::S_LSHL_B32), TmpReg0)
+        .addImm(16)
+        .addReg(MemberCount);
+    M0Val = TmpReg0;
+  }
+
+  // If not inlinable, get reference to barrier depending on the instruction
+  if (!BarValImm) {
+    if (IntrID == Intrinsic::amdgcn_s_barrier_init) {
+      // If reference to barrier id is not an inlinable constant then it must be
+      // referenced with M0[4:0]. Perform an OR with the member count to include
+      // it in M0 for S_BARRIER_INIT.
+      Register TmpReg1 = MRI->createVirtualRegister(&AMDGPU::SReg_32RegClass);
+      BuildMI(*MBB, &I, DL, TII.get(AMDGPU::S_OR_B32), TmpReg1)
+          .addReg(BarOp.getReg())
+          .addReg(TmpReg0);
+      M0Val = TmpReg1;
+    } else {
+      M0Val = BarOp.getReg();
+    }
+  }
+
+  // Build copy to M0 if needed. For S_BARRIER_INIT, M0 is always required.
+  if (M0Val) {
+    auto CopyMIB =
+        BuildMI(*MBB, &I, DL, TII.get(AMDGPU::COPY), AMDGPU::M0).addReg(M0Val);
+    constrainSelectedInstRegOperands(*CopyMIB, TII, TRI, RBI);
+  }
+
+  MachineInstrBuilder MIB;
+  unsigned Opc = getNamedBarrierOp(BarValImm.has_value(), IntrID);
+  MIB = BuildMI(*MBB, &I, DL, TII.get(Opc));
+
+  if (IntrID == Intrinsic::amdgcn_s_get_barrier_state)
+    MIB.addDef(I.getOperand(0).getReg());
+
+  if (BarValImm)
+    MIB.addImm(*BarValImm);
+
+  I.eraseFromParent();
+  return true;
+}
+bool AMDGPUInstructionSelector::selectSBarrierLeave(MachineInstr &I) const {
+  MachineBasicBlock *BB = I.getParent();
+  const DebugLoc &DL = I.getDebugLoc();
+  Register CCReg = I.getOperand(0).getReg();
+
+  BuildMI(*BB, &I, DL, TII.get(AMDGPU::S_BARRIER_LEAVE));
+  BuildMI(*BB, &I, DL, TII.get(AMDGPU::COPY), CCReg).addReg(AMDGPU::SCC);
+
+  I.eraseFromParent();
+  return RBI.constrainGenericRegister(CCReg, AMDGPU::SReg_32_XM0_XEXECRegClass,
+                                      *MRI);
+}
+
 void AMDGPUInstructionSelector::renderTruncImm32(MachineInstrBuilder &MIB,
                                                  const MachineInstr &MI,
                                                  int OpIdx) const {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
index c93e3de66d40550204404194f06df32c4f2c0b9c..00ff1747ce57a33dcf8e4a13fc30762e78ecd183 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructionSelector.h
@@ -149,6 +149,9 @@ private:
   bool selectSMFMACIntrin(MachineInstr &I) const;
   bool selectWaveAddress(MachineInstr &I) const;
   bool selectStackRestore(MachineInstr &MI) const;
+  bool selectNamedBarrierInst(MachineInstr &I, Intrinsic::ID IID) const;
+  bool selectSBarrierSignalIsfirst(MachineInstr &I, Intrinsic::ID IID) const;
+  bool selectSBarrierLeave(MachineInstr &I) const;
 
   std::pair<Register, unsigned> selectVOP3ModsImpl(MachineOperand &Root,
                                                    bool IsCanonicalizing = true,
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td b/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td
index d2d09a0b1fc54859d80c1e065a33b2fcdce6f186..121026aca603559521584cf622e93aa9399e3bb7 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td
@@ -252,6 +252,8 @@ def umin_oneuse : HasOneUseBinOp<umin>;
 
 def fminnum_oneuse : HasOneUseBinOp<fminnum>;
 def fmaxnum_oneuse : HasOneUseBinOp<fmaxnum>;
+def fminimum_oneuse : HasOneUseBinOp<fminimum>;
+def fmaximum_oneuse : HasOneUseBinOp<fmaximum>;
 
 def fminnum_ieee_oneuse : HasOneUseBinOp<fminnum_ieee>;
 def fmaxnum_ieee_oneuse : HasOneUseBinOp<fmaxnum_ieee>;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 2cf804e3348e80dccb4ffbe6b503923131a0fccf..489b4f5a8d86a5e8e28c957193f1ee324485c863 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -1959,20 +1959,25 @@ AMDGPULegalizerInfo::AMDGPULegalizerInfo(const GCNSubtarget &ST_,
       .widenScalarToNextPow2(0)
       .scalarize(0);
 
-  getActionDefinitionsBuilder({
-      // TODO: Verify V_BFI_B32 is generated from expanded bit ops
-      G_FCOPYSIGN,
+  getActionDefinitionsBuilder(
+      {// TODO: Verify V_BFI_B32 is generated from expanded bit ops
+       G_FCOPYSIGN,
 
-      G_ATOMIC_CMPXCHG_WITH_SUCCESS,
-      G_ATOMICRMW_NAND,
-      G_ATOMICRMW_FSUB,
-      G_READ_REGISTER,
-      G_WRITE_REGISTER,
+       G_ATOMIC_CMPXCHG_WITH_SUCCESS, G_ATOMICRMW_NAND, G_ATOMICRMW_FSUB,
+       G_READ_REGISTER, G_WRITE_REGISTER,
 
-      G_SADDO, G_SSUBO,
+       G_SADDO, G_SSUBO})
+      .lower();
 
-       // TODO: Implement
-      G_FMINIMUM, G_FMAXIMUM}).lower();
+  if (ST.hasIEEEMinMax()) {
+    getActionDefinitionsBuilder({G_FMINIMUM, G_FMAXIMUM})
+        .legalFor(FPTypesPK16)
+        .clampMaxNumElements(0, S16, 2)
+        .scalarize(0);
+  } else {
+    // TODO: Implement
+    getActionDefinitionsBuilder({G_FMINIMUM, G_FMAXIMUM}).lower();
+  }
 
   getActionDefinitionsBuilder({G_MEMCPY, G_MEMCPY_INLINE, G_MEMMOVE, G_MEMSET})
       .lower();
diff --git a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
index 62996a3b3fb79fb66b4b6e38c66d90ee4cca4531..03b6d19b2b3c060a4b1beb5d6eaa440f4dee4fc2 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPURegisterBankInfo.cpp
@@ -3101,6 +3101,22 @@ void AMDGPURegisterBankInfo::applyMappingImpl(
       applyDefaultMapping(OpdMapper);
       constrainOpWithReadfirstlane(B, MI, 8); // M0
       return;
+    case Intrinsic::amdgcn_s_barrier_signal_var:
+    case Intrinsic::amdgcn_s_barrier_join:
+    case Intrinsic::amdgcn_s_wakeup_barrier:
+      constrainOpWithReadfirstlane(B, MI, 1);
+      return;
+    case Intrinsic::amdgcn_s_barrier_signal_isfirst_var:
+      constrainOpWithReadfirstlane(B, MI, 2);
+      return;
+    case Intrinsic::amdgcn_s_barrier_init:
+      constrainOpWithReadfirstlane(B, MI, 1);
+      constrainOpWithReadfirstlane(B, MI, 2);
+      return;
+    case Intrinsic::amdgcn_s_get_barrier_state: {
+      constrainOpWithReadfirstlane(B, MI, 2);
+      return;
+    }
     default: {
       if (const AMDGPU::RsrcIntrinsic *RSrcIntrin =
               AMDGPU::lookupRsrcIntrinsic(IntrID)) {
@@ -3727,14 +3743,17 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
   case AMDGPU::G_INTRINSIC_ROUNDEVEN:
   case AMDGPU::G_FMINNUM:
   case AMDGPU::G_FMAXNUM:
+  case AMDGPU::G_FMINIMUM:
+  case AMDGPU::G_FMAXIMUM:
   case AMDGPU::G_INTRINSIC_TRUNC:
   case AMDGPU::G_STRICT_FADD:
   case AMDGPU::G_STRICT_FSUB:
   case AMDGPU::G_STRICT_FMUL:
   case AMDGPU::G_STRICT_FMA: {
-    unsigned Size = MRI.getType(MI.getOperand(0).getReg()).getSizeInBits();
-    if (Subtarget.hasSALUFloatInsts() && (Size == 32 || Size == 16) &&
-        isSALUMapping(MI))
+    LLT Ty = MRI.getType(MI.getOperand(0).getReg());
+    unsigned Size = Ty.getSizeInBits();
+    if (Subtarget.hasSALUFloatInsts() && Ty.isScalar() &&
+        (Size == 32 || Size == 16) && isSALUMapping(MI))
       return getDefaultMappingSOP(MI);
     return getDefaultMappingVOP(MI);
   }
@@ -4830,7 +4849,34 @@ AMDGPURegisterBankInfo::getInstrMapping(const MachineInstr &MI) const {
           getVGPROpMapping(MI.getOperand(5).getReg(), MRI, *TRI); // %data1
       break;
     }
-
+    case Intrinsic::amdgcn_s_barrier_signal_var:
+    case Intrinsic::amdgcn_s_barrier_join:
+    case Intrinsic::amdgcn_s_wakeup_barrier:
+      OpdsMapping[1] = getSGPROpMapping(MI.getOperand(1).getReg(), MRI, *TRI);
+      break;
+    case Intrinsic::amdgcn_s_barrier_init:
+      OpdsMapping[1] = getSGPROpMapping(MI.getOperand(1).getReg(), MRI, *TRI);
+      OpdsMapping[2] = getSGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
+      break;
+    case Intrinsic::amdgcn_s_barrier_signal_isfirst_var: {
+      const unsigned ResultSize = 1;
+      OpdsMapping[0] =
+          AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, ResultSize);
+      OpdsMapping[2] = getSGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
+      break;
+    }
+    case Intrinsic::amdgcn_s_barrier_signal_isfirst:
+    case Intrinsic::amdgcn_s_barrier_leave: {
+      const unsigned ResultSize = 1;
+      OpdsMapping[0] =
+          AMDGPU::getValueMapping(AMDGPU::SGPRRegBankID, ResultSize);
+      break;
+    }
+    case Intrinsic::amdgcn_s_get_barrier_state: {
+      OpdsMapping[0] = getSGPROpMapping(MI.getOperand(0).getReg(), MRI, *TRI);
+      OpdsMapping[2] = getSGPROpMapping(MI.getOperand(2).getReg(), MRI, *TRI);
+      break;
+    }
     default:
       return getInvalidInstructionMapping();
     }
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index 97db19064f6fd194ef47b02f2682236d26a7c6de..0e0094cb9cd6ef4d532e63f14445f789fd3b6edb 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -352,7 +352,7 @@ static cl::opt<bool> EnableMaxIlpSchedStrategy(
 
 static cl::opt<bool> EnableRewritePartialRegUses(
     "amdgpu-enable-rewrite-partial-reg-uses",
-    cl::desc("Enable rewrite partial reg uses pass"), cl::init(false),
+    cl::desc("Enable rewrite partial reg uses pass"), cl::init(true),
     cl::Hidden);
 
 static cl::opt<bool> EnableHipStdPar(
@@ -375,6 +375,7 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
   initializeAMDGPUDAGToDAGISelPass(*PR);
   initializeGCNDPPCombinePass(*PR);
   initializeSILowerI1CopiesPass(*PR);
+  initializeAMDGPUGlobalISelDivergenceLoweringPass(*PR);
   initializeSILowerWWMCopiesPass(*PR);
   initializeSILowerSGPRSpillsPass(*PR);
   initializeSIFixSGPRCopiesPass(*PR);
@@ -1255,6 +1256,7 @@ bool GCNPassConfig::addLegalizeMachineIR() {
 void GCNPassConfig::addPreRegBankSelect() {
   bool IsOptNone = getOptLevel() == CodeGenOptLevel::None;
   addPass(createAMDGPUPostLegalizeCombiner(IsOptNone));
+  addPass(createAMDGPUGlobalISelDivergenceLoweringPass());
 }
 
 bool GCNPassConfig::addRegBankSelect() {
@@ -1499,13 +1501,13 @@ bool GCNTargetMachine::parseMachineFunctionInfo(
       static_cast<const yaml::SIMachineFunctionInfo &>(MFI_);
   MachineFunction &MF = PFS.MF;
   SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
+  const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
 
   if (MFI->initializeBaseYamlFields(YamlMFI, MF, PFS, Error, SourceRange))
     return true;
 
   if (MFI->Occupancy == 0) {
     // Fixup the subtarget dependent default value.
-    const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
     MFI->Occupancy = ST.computeOccupancy(MF.getFunction(), MFI->getLDSSize());
   }
 
@@ -1659,8 +1661,10 @@ bool GCNTargetMachine::parseMachineFunctionInfo(
                              MFI->ArgInfo.WorkItemIDZ, 0, 0)))
     return true;
 
-  MFI->Mode.IEEE = YamlMFI.Mode.IEEE;
-  MFI->Mode.DX10Clamp = YamlMFI.Mode.DX10Clamp;
+  if (ST.hasIEEEMode())
+    MFI->Mode.IEEE = YamlMFI.Mode.IEEE;
+  if (ST.hasDX10ClampMode())
+    MFI->Mode.DX10Clamp = YamlMFI.Mode.DX10Clamp;
 
   // FIXME: Move proper support for denormal-fp-math into base MachineFunction
   MFI->Mode.FP32Denormals.Input = YamlMFI.Mode.FP32InputDenormals
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index 3316659671bd137e0f346eb6dd9505f925e69b46..d754058d4f0645207e45589b9c601c36df06cc27 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -296,7 +296,7 @@ GCNTTIImpl::GCNTTIImpl(const AMDGPUTargetMachine *TM, const Function &F)
       ST(static_cast<const GCNSubtarget *>(TM->getSubtargetImpl(F))),
       TLI(ST->getTargetLowering()), CommonTTI(TM, F),
       IsGraphics(AMDGPU::isGraphics(F.getCallingConv())) {
-  SIModeRegisterDefaults Mode(F);
+  SIModeRegisterDefaults Mode(F, *ST);
   HasFP32Denormals = Mode.FP32Denormals != DenormalMode::getPreserveSign();
   HasFP64FP16Denormals =
       Mode.FP64FP16Denormals != DenormalMode::getPreserveSign();
@@ -1163,8 +1163,8 @@ bool GCNTTIImpl::areInlineCompatible(const Function *Caller,
 
   // FIXME: dx10_clamp can just take the caller setting, but there seems to be
   // no way to support merge for backend defined attributes.
-  SIModeRegisterDefaults CallerMode(*Caller);
-  SIModeRegisterDefaults CalleeMode(*Callee);
+  SIModeRegisterDefaults CallerMode(*Caller, *CallerST);
+  SIModeRegisterDefaults CalleeMode(*Callee, *CalleeST);
   if (!CallerMode.isInlineCompatible(CalleeMode))
     return false;
 
diff --git a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
index 92427335c0ad2f7a7e7052b7533238edee2117cb..2f1e1809b6cd73d454d349e19d9ac8b3a7a70336 100644
--- a/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
+++ b/llvm/lib/Target/AMDGPU/AsmParser/AMDGPUAsmParser.cpp
@@ -373,7 +373,7 @@ public:
   bool isOffen() const { return isImmTy(ImmTyOffen); }
   bool isIdxen() const { return isImmTy(ImmTyIdxen); }
   bool isAddr64() const { return isImmTy(ImmTyAddr64); }
-  bool isOffset() const { return isImmTy(ImmTyOffset) && isUInt<16>(getImm()); }
+  bool isOffset() const { return isImmTy(ImmTyOffset); }
   bool isOffset0() const { return isImmTy(ImmTyOffset0) && isUInt<8>(getImm()); }
   bool isOffset1() const { return isImmTy(ImmTyOffset1) && isUInt<8>(getImm()); }
   bool isSMEMOffsetMod() const { return isImmTy(ImmTySMEMOffsetMod); }
@@ -893,6 +893,7 @@ public:
   bool isSDelayALU() const;
   bool isHwreg() const;
   bool isSendMsg() const;
+  bool isSplitBarrier() const;
   bool isSwizzle() const;
   bool isSMRDOffset8() const;
   bool isSMEMOffset() const;
@@ -1665,6 +1666,7 @@ private:
   SMLoc getInstLoc(const OperandVector &Operands) const;
 
   bool validateInstruction(const MCInst &Inst, const SMLoc &IDLoc, const OperandVector &Operands);
+  bool validateOffset(const MCInst &Inst, const OperandVector &Operands);
   bool validateFlatOffset(const MCInst &Inst, const OperandVector &Operands);
   bool validateSMEMOffset(const MCInst &Inst, const OperandVector &Operands);
   bool validateSOPLiteral(const MCInst &Inst) const;
@@ -1856,6 +1858,7 @@ static const fltSemantics *getOpFltSemantics(uint8_t OperandType) {
   case AMDGPU::OPERAND_REG_INLINE_C_V2INT32:
   case AMDGPU::OPERAND_REG_IMM_V2INT32:
   case AMDGPU::OPERAND_KIMM32:
+  case AMDGPU::OPERAND_INLINE_SPLIT_BARRIER_INT32:
     return &APFloat::IEEEsingle();
   case AMDGPU::OPERAND_REG_IMM_INT64:
   case AMDGPU::OPERAND_REG_IMM_FP64:
@@ -2185,7 +2188,8 @@ void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val, bool ApplyMo
     case AMDGPU::OPERAND_REG_INLINE_C_V2INT32:
     case AMDGPU::OPERAND_REG_IMM_V2INT32:
     case AMDGPU::OPERAND_KIMM32:
-    case AMDGPU::OPERAND_KIMM16: {
+    case AMDGPU::OPERAND_KIMM16:
+    case AMDGPU::OPERAND_INLINE_SPLIT_BARRIER_INT32: {
       bool lost;
       APFloat FPLiteral(APFloat::IEEEdouble(), Literal);
       // Convert literal to single precision
@@ -2226,6 +2230,7 @@ void AMDGPUOperand::addLiteralImmOperand(MCInst &Inst, int64_t Val, bool ApplyMo
   case AMDGPU::OPERAND_REG_INLINE_C_V2FP32:
   case AMDGPU::OPERAND_REG_IMM_V2INT32:
   case AMDGPU::OPERAND_REG_INLINE_C_V2INT32:
+  case AMDGPU::OPERAND_INLINE_SPLIT_BARRIER_INT32:
     if (isSafeTruncation(Val, 32) &&
         AMDGPU::isInlinableLiteral32(static_cast<int32_t>(Val),
                                      AsmParser->hasInv2PiInlineImm())) {
@@ -4139,6 +4144,40 @@ SMLoc AMDGPUAsmParser::getFlatOffsetLoc(const OperandVector &Operands) const {
   return getLoc();
 }
 
+bool AMDGPUAsmParser::validateOffset(const MCInst &Inst,
+                                     const OperandVector &Operands) {
+  auto Opcode = Inst.getOpcode();
+  auto OpNum = AMDGPU::getNamedOperandIdx(Opcode, AMDGPU::OpName::offset);
+  if (OpNum == -1)
+    return true;
+
+  uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
+  if ((TSFlags & SIInstrFlags::FLAT))
+    return validateFlatOffset(Inst, Operands);
+
+  if ((TSFlags & SIInstrFlags::SMRD))
+    return validateSMEMOffset(Inst, Operands);
+
+  const auto &Op = Inst.getOperand(OpNum);
+  if (isGFX12Plus() &&
+      (TSFlags & (SIInstrFlags::MUBUF | SIInstrFlags::MTBUF))) {
+    const unsigned OffsetSize = 24;
+    if (!isIntN(OffsetSize, Op.getImm())) {
+      Error(getFlatOffsetLoc(Operands),
+            Twine("expected a ") + Twine(OffsetSize) + "-bit signed offset");
+      return false;
+    }
+  } else {
+    const unsigned OffsetSize = 16;
+    if (!isUIntN(OffsetSize, Op.getImm())) {
+      Error(getFlatOffsetLoc(Operands),
+            Twine("expected a ") + Twine(OffsetSize) + "-bit unsigned offset");
+      return false;
+    }
+  }
+  return true;
+}
+
 bool AMDGPUAsmParser::validateFlatOffset(const MCInst &Inst,
                                          const OperandVector &Operands) {
   uint64_t TSFlags = MII.get(Inst.getOpcode()).TSFlags;
@@ -4156,11 +4195,12 @@ bool AMDGPUAsmParser::validateFlatOffset(const MCInst &Inst,
     return false;
   }
 
-  // For FLAT segment the offset must be positive;
+  // For pre-GFX12 FLAT instructions the offset must be positive;
   // MSB is ignored and forced to zero.
   unsigned OffsetSize = AMDGPU::getNumFlatOffsetBits(getSTI());
   bool AllowNegative =
-      TSFlags & (SIInstrFlags::FlatGlobal | SIInstrFlags::FlatScratch);
+      (TSFlags & (SIInstrFlags::FlatGlobal | SIInstrFlags::FlatScratch)) ||
+      isGFX12Plus();
   if (!isIntN(OffsetSize, Op.getImm()) || (!AllowNegative && Op.getImm() < 0)) {
     Error(getFlatOffsetLoc(Operands),
           Twine("expected a ") +
@@ -4796,10 +4836,7 @@ bool AMDGPUAsmParser::validateInstruction(const MCInst &Inst,
   if (!validateMovrels(Inst, Operands)) {
     return false;
   }
-  if (!validateFlatOffset(Inst, Operands)) {
-    return false;
-  }
-  if (!validateSMEMOffset(Inst, Operands)) {
+  if (!validateOffset(Inst, Operands)) {
     return false;
   }
   if (!validateMAIAccWrite(Inst, Operands)) {
@@ -5342,11 +5379,17 @@ bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() {
                        COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64, Val,
                        ValRange);
     } else if (ID == ".amdhsa_dx10_clamp") {
+      if (IVersion.Major >= 12)
+        return Error(IDRange.Start, "directive unsupported on gfx12+", IDRange);
       PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1,
-                       COMPUTE_PGM_RSRC1_ENABLE_DX10_CLAMP, Val, ValRange);
+                       COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_DX10_CLAMP, Val,
+                       ValRange);
     } else if (ID == ".amdhsa_ieee_mode") {
-      PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1, COMPUTE_PGM_RSRC1_ENABLE_IEEE_MODE,
-                       Val, ValRange);
+      if (IVersion.Major >= 12)
+        return Error(IDRange.Start, "directive unsupported on gfx12+", IDRange);
+      PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1,
+                       COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_IEEE_MODE, Val,
+                       ValRange);
     } else if (ID == ".amdhsa_fp16_overflow") {
       if (IVersion.Major < 9)
         return Error(IDRange.Start, "directive requires gfx9+", IDRange);
@@ -5409,6 +5452,12 @@ bool AMDGPUAsmParser::ParseDirectiveAMDHSAKernel() {
       PARSE_BITS_ENTRY(KD.compute_pgm_rsrc2,
                        COMPUTE_PGM_RSRC2_ENABLE_EXCEPTION_INT_DIVIDE_BY_ZERO,
                        Val, ValRange);
+    } else if (ID == ".amdhsa_round_robin_scheduling") {
+      if (IVersion.Major < 12)
+        return Error(IDRange.Start, "directive requires gfx12+", IDRange);
+      PARSE_BITS_ENTRY(KD.compute_pgm_rsrc1,
+                       COMPUTE_PGM_RSRC1_GFX12_PLUS_ENABLE_WG_RR_EN, Val,
+                       ValRange);
     } else {
       return Error(IDRange.Start, "unknown .amdhsa_kernel directive", IDRange);
     }
@@ -5562,6 +5611,18 @@ bool AMDGPUAsmParser::ParseAMDKernelCodeTValue(StringRef ID,
   }
   Lex();
 
+  if (ID == "enable_dx10_clamp") {
+    if (G_00B848_DX10_CLAMP(Header.compute_pgm_resource_registers) &&
+        isGFX12Plus())
+      return TokError("enable_dx10_clamp=1 is not allowed on GFX12+");
+  }
+
+  if (ID == "enable_ieee_mode") {
+    if (G_00B848_IEEE_MODE(Header.compute_pgm_resource_registers) &&
+        isGFX12Plus())
+      return TokError("enable_ieee_mode=1 is not allowed on GFX12+");
+  }
+
   if (ID == "enable_wavefront_size32") {
     if (Header.code_properties & AMD_CODE_PROPERTY_ENABLE_WAVEFRONT_SIZE32) {
       if (!isGFX10Plus())
@@ -9137,3 +9198,9 @@ bool AMDGPUOperand::isWaitVDST() const {
 bool AMDGPUOperand::isWaitEXP() const {
   return isImmTy(ImmTyWaitEXP) && isUInt<3>(getImm());
 }
+
+//===----------------------------------------------------------------------===//
+// Split Barrier
+//===----------------------------------------------------------------------===//
+
+bool AMDGPUOperand::isSplitBarrier() const { return isInlinableImm(MVT::i32); }
diff --git a/llvm/lib/Target/AMDGPU/BUFInstructions.td b/llvm/lib/Target/AMDGPU/BUFInstructions.td
index 44fd4ef864127037f099941b5e363c87a7d8f91f..9832d89c6ac6f7239a2c0e6fa89c003610333299 100644
--- a/llvm/lib/Target/AMDGPU/BUFInstructions.td
+++ b/llvm/lib/Target/AMDGPU/BUFInstructions.td
@@ -152,24 +152,32 @@ class MTBUF_Real <MTBUF_Pseudo ps, string real_name = ps.Mnemonic> :
 }
 
 class getMTBUFInsDA<list<RegisterClass> vdataList,
-                    list<RegisterClass> vaddrList=[]> {
+                    list<RegisterClass> vaddrList=[], bit hasGFX12Enc> {
   RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList));
   RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList));
   RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret;
 
-  dag NonVaddrInputs = (ins SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, FORMAT:$format, CPol:$cpol, i1imm:$swz);
-  dag Inputs = !if(!empty(vaddrList), NonVaddrInputs, !con((ins vaddrClass:$vaddr), NonVaddrInputs));
-  dag ret = !if(!empty(vdataList), Inputs, !con((ins vdata_op:$vdata), Inputs));
-}
+  dag SOffset = !if(hasGFX12Enc, (ins SReg_32:$soffset),
+                                 (ins SCSrc_b32:$soffset));
 
+  dag NonVaddrInputs = !con((ins SReg_128:$srsrc), SOffset,
+                            (ins offset:$offset, FORMAT:$format, CPol_0:$cpol, i1imm_0:$swz));
 
-class getMTBUFIns<int addrKind, list<RegisterClass> vdataList=[]> {
+  dag Inputs = !if(!empty(vaddrList),
+                   NonVaddrInputs,
+                   !con((ins vaddrClass:$vaddr), NonVaddrInputs));
+  dag ret = !if(!empty(vdataList),
+                Inputs,
+                !con((ins vdata_op:$vdata), Inputs));
+}
+
+class getMTBUFIns<int addrKind, list<RegisterClass> vdataList=[], bit hasGFX12Enc> {
   dag ret =
-    !if(!eq(addrKind, BUFAddrKind.Offset), getMTBUFInsDA<vdataList>.ret,
-    !if(!eq(addrKind, BUFAddrKind.OffEn),  getMTBUFInsDA<vdataList, [VGPR_32]>.ret,
-    !if(!eq(addrKind, BUFAddrKind.IdxEn),  getMTBUFInsDA<vdataList, [VGPR_32]>.ret,
-    !if(!eq(addrKind, BUFAddrKind.BothEn), getMTBUFInsDA<vdataList, [VReg_64]>.ret,
-    !if(!eq(addrKind, BUFAddrKind.Addr64), getMTBUFInsDA<vdataList, [VReg_64]>.ret,
+    !if(!eq(addrKind, BUFAddrKind.Offset), getMTBUFInsDA<vdataList, [], hasGFX12Enc>.ret,
+    !if(!eq(addrKind, BUFAddrKind.OffEn),  getMTBUFInsDA<vdataList, [VGPR_32], hasGFX12Enc>.ret,
+    !if(!eq(addrKind, BUFAddrKind.IdxEn),  getMTBUFInsDA<vdataList, [VGPR_32], hasGFX12Enc>.ret,
+    !if(!eq(addrKind, BUFAddrKind.BothEn), getMTBUFInsDA<vdataList, [VReg_64], hasGFX12Enc>.ret,
+    !if(!eq(addrKind, BUFAddrKind.Addr64), getMTBUFInsDA<vdataList, [VReg_64], hasGFX12Enc>.ret,
     (ins))))));
 }
 
@@ -204,12 +212,13 @@ class MTBUF_Load_Pseudo <string opName,
                          int addrKind,
                          RegisterClass vdataClass,
                          int elems,
+                         bit hasGFX12Enc = 0,
                          list<dag> pattern=[],
                          // Workaround bug bz30254
                          int addrKindCopy = addrKind>
   : MTBUF_Pseudo<opName,
                  (outs getLdStRegisterOperand<vdataClass>.ret:$vdata),
-                 getMTBUFIns<addrKindCopy>.ret,
+                 getMTBUFIns<addrKindCopy, [], hasGFX12Enc>.ret,
                  getMTBUFAsmOps<addrKindCopy>.ret,
                  pattern>,
     MTBUF_SetupAddr<addrKindCopy> {
@@ -219,38 +228,45 @@ class MTBUF_Load_Pseudo <string opName,
   let elements = elems;
 }
 
-multiclass MTBUF_Pseudo_Loads<string opName, RegisterClass vdataClass,
-                              int elems> {
+multiclass MTBUF_Pseudo_Loads_Helper<string opName, RegisterClass vdataClass,
+                              int elems, bit hasGFX12Enc> {
 
-  def _OFFSET : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>,
+  def _OFFSET : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems, hasGFX12Enc>,
                 MTBUFAddr64Table<0, NAME>;
 
-  def _ADDR64 : MTBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, elems>,
+  def _ADDR64 : MTBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, elems, hasGFX12Enc>,
                 MTBUFAddr64Table<1, NAME>;
 
-  def _OFFEN  : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>;
-  def _IDXEN  : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>;
-  def _BOTHEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>;
+  def _OFFEN  : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems, hasGFX12Enc>;
+  def _IDXEN  : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems, hasGFX12Enc>;
+  def _BOTHEN : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems, hasGFX12Enc>;
 
   let DisableWQM = 1 in {
-    def _OFFSET_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>;
-    def _OFFEN_exact  : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>;
-    def _IDXEN_exact  : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>;
-    def _BOTHEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>;
+    def _OFFSET_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems, hasGFX12Enc>;
+    def _OFFEN_exact  : MTBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems, hasGFX12Enc>;
+    def _IDXEN_exact  : MTBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems, hasGFX12Enc>;
+    def _BOTHEN_exact : MTBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems, hasGFX12Enc>;
   }
 }
 
+multiclass MTBUF_Pseudo_Loads<string opName, RegisterClass vdataClass,
+                              int elems> {
+  defm NAME : MTBUF_Pseudo_Loads_Helper<opName, vdataClass, elems, 0>;
+  defm _VBUFFER : MTBUF_Pseudo_Loads_Helper<opName, vdataClass, elems, 1>;
+}
+
 class MTBUF_Store_Pseudo <string opName,
                           int addrKind,
                           RegisterClass vdataClass,
                           int elems,
+                          bit hasGFX12Enc = 0,
                           list<dag> pattern=[],
                           // Workaround bug bz30254
                           int addrKindCopy = addrKind,
                           RegisterClass vdataClassCopy = vdataClass>
   : MTBUF_Pseudo<opName,
                  (outs),
-                 getMTBUFIns<addrKindCopy, [vdataClassCopy]>.ret,
+                 getMTBUFIns<addrKindCopy, [vdataClassCopy], hasGFX12Enc>.ret,
                  getMTBUFAsmOps<addrKindCopy>.ret,
                  pattern>,
     MTBUF_SetupAddr<addrKindCopy> {
@@ -260,27 +276,32 @@ class MTBUF_Store_Pseudo <string opName,
   let elements = elems;
 }
 
-multiclass MTBUF_Pseudo_Stores<string opName, RegisterClass vdataClass,
-                               int elems> {
+multiclass MTBUF_Pseudo_Stores_Helper<string opName, RegisterClass vdataClass,
+                               int elems, bit hasGFX12Enc> {
 
-  def _OFFSET : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>,
+  def _OFFSET : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems, hasGFX12Enc>,
     MTBUFAddr64Table<0, NAME>;
 
-  def _ADDR64 : MTBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, elems>,
+  def _ADDR64 : MTBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, elems, hasGFX12Enc>,
     MTBUFAddr64Table<1, NAME>;
 
-  def _OFFEN  : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>;
-  def _IDXEN  : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>;
-  def _BOTHEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>;
+  def _OFFEN  : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems, hasGFX12Enc>;
+  def _IDXEN  : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems, hasGFX12Enc>;
+  def _BOTHEN : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems, hasGFX12Enc>;
 
   let DisableWQM = 1 in {
-    def _OFFSET_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems>;
-    def _OFFEN_exact  : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems>;
-    def _IDXEN_exact  : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems>;
-    def _BOTHEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems>;
+    def _OFFSET_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.Offset, vdataClass, elems, hasGFX12Enc>;
+    def _OFFEN_exact  : MTBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, vdataClass, elems, hasGFX12Enc>;
+    def _IDXEN_exact  : MTBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, vdataClass, elems, hasGFX12Enc>;
+    def _BOTHEN_exact : MTBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, elems, hasGFX12Enc>;
   }
 }
 
+multiclass MTBUF_Pseudo_Stores<string opName, RegisterClass vdataClass,
+                               int elems> {
+  defm NAME : MTBUF_Pseudo_Stores_Helper<opName, vdataClass, elems, 0>;
+  defm _VBUFFER : MTBUF_Pseudo_Stores_Helper<opName, vdataClass, elems, 1>;
+}
 
 //===----------------------------------------------------------------------===//
 // MUBUF classes
@@ -381,12 +402,14 @@ class getLdStVDataRegisterOperand<RegisterClass RC, bit isTFE> {
 }
 
 class getMUBUFInsDA<list<RegisterClass> vdataList,
-                    list<RegisterClass> vaddrList, bit isTFE> {
+                    list<RegisterClass> vaddrList, bit isTFE, bit hasGFX12Enc> {
   RegisterClass vdataClass = !if(!empty(vdataList), ?, !head(vdataList));
   RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList));
   RegisterOperand vdata_op = getLdStVDataRegisterOperand<vdataClass, isTFE>.ret;
 
-  dag NonVaddrInputs = (ins SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset, CPol_0:$cpol, i1imm_0:$swz);
+  dag SOffset = !if(hasGFX12Enc, (ins SReg_32:$soffset), (ins SCSrc_b32:$soffset));
+  dag NonVaddrInputs = !con((ins SReg_128:$srsrc), SOffset, (ins offset:$offset, CPol_0:$cpol, i1imm_0:$swz));
+
   dag Inputs = !if(!empty(vaddrList), NonVaddrInputs, !con((ins vaddrClass:$vaddr), NonVaddrInputs));
   dag ret = !if(!empty(vdataList), Inputs, !con((ins vdata_op:$vdata), Inputs));
 }
@@ -410,13 +433,13 @@ class getMUBUFElements<ValueType vt> {
     );
 }
 
-class getMUBUFIns<int addrKind, list<RegisterClass> vdataList, bit isTFE> {
+class getMUBUFIns<int addrKind, list<RegisterClass> vdataList, bit isTFE, bit hasGFX12Enc> {
   dag ret =
-    !if(!eq(addrKind, BUFAddrKind.Offset), getMUBUFInsDA<vdataList, [], isTFE>.ret,
-    !if(!eq(addrKind, BUFAddrKind.OffEn),  getMUBUFInsDA<vdataList, [VGPR_32], isTFE>.ret,
-    !if(!eq(addrKind, BUFAddrKind.IdxEn),  getMUBUFInsDA<vdataList, [VGPR_32], isTFE>.ret,
-    !if(!eq(addrKind, BUFAddrKind.BothEn), getMUBUFInsDA<vdataList, [VReg_64], isTFE>.ret,
-    !if(!eq(addrKind, BUFAddrKind.Addr64), getMUBUFInsDA<vdataList, [VReg_64], isTFE>.ret,
+    !if(!eq(addrKind, BUFAddrKind.Offset), getMUBUFInsDA<vdataList, [], isTFE, hasGFX12Enc>.ret,
+    !if(!eq(addrKind, BUFAddrKind.OffEn),  getMUBUFInsDA<vdataList, [VGPR_32], isTFE, hasGFX12Enc>.ret,
+    !if(!eq(addrKind, BUFAddrKind.IdxEn),  getMUBUFInsDA<vdataList, [VGPR_32], isTFE, hasGFX12Enc>.ret,
+    !if(!eq(addrKind, BUFAddrKind.BothEn), getMUBUFInsDA<vdataList, [VReg_64], isTFE, hasGFX12Enc>.ret,
+    !if(!eq(addrKind, BUFAddrKind.Addr64), getMUBUFInsDA<vdataList, [VReg_64], isTFE, hasGFX12Enc>.ret,
     (ins))))));
 }
 
@@ -456,6 +479,7 @@ class MUBUF_Load_Pseudo <string opName,
                          bit isLds = 0,
                          bit isLdsOpc = 0,
                          bit isTFE = 0,
+                         bit hasGFX12Enc = 0,
                          list<dag> pattern=[],
                          // Workaround bug bz30254
                          int addrKindCopy = addrKind,
@@ -463,7 +487,7 @@ class MUBUF_Load_Pseudo <string opName,
                          RegisterOperand vdata_op = getLdStVDataRegisterOperand<vdata_rc, isTFE>.ret>
   : MUBUF_Pseudo<opName,
                  !if(!or(isLds, isLdsOpc), (outs), (outs vdata_op:$vdata)),
-                 !con(getMUBUFIns<addrKindCopy, [], isTFE>.ret,
+                 !con(getMUBUFIns<addrKindCopy, [], isTFE, hasGFX12Enc>.ret,
                       !if(HasTiedDest, (ins vdata_op:$vdata_in), (ins))),
                  getMUBUFAsmOps<addrKindCopy, !or(isLds, isLdsOpc), isLds, isTFE>.ret,
                  pattern>,
@@ -503,32 +527,36 @@ multiclass MUBUF_Pseudo_Load_Pats<string BaseInst, ValueType load_vt = i32, SDPa
 }
 
 multiclass MUBUF_Pseudo_Loads_Helper<string opName, ValueType load_vt,
-                                     bit TiedDest, bit isLds, bit isTFE> {
+                                     bit TiedDest, bit isLds, bit isTFE, bit hasGFX12Enc> {
   defvar legal_load_vt = !if(!eq(load_vt, v3f16), v4f16, load_vt);
 
-  def _OFFSET : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, legal_load_vt, TiedDest, isLds, 0, isTFE>,
+  def _OFFSET : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, legal_load_vt, TiedDest, isLds, 0, isTFE, hasGFX12Enc>,
     MUBUFAddr64Table<0, NAME # !if(isLds, "_LDS", "")>;
 
-  def _ADDR64 : MUBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, legal_load_vt, TiedDest, isLds, 0, isTFE>,
+  def _ADDR64 : MUBUF_Load_Pseudo <opName, BUFAddrKind.Addr64, legal_load_vt, TiedDest, isLds, 0, isTFE, hasGFX12Enc>,
     MUBUFAddr64Table<1, NAME # !if(isLds, "_LDS", "")>;
 
-  def _OFFEN  : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, legal_load_vt, TiedDest, isLds, 0, isTFE>;
-  def _IDXEN  : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, legal_load_vt, TiedDest, isLds, 0, isTFE>;
-  def _BOTHEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, legal_load_vt, TiedDest, isLds, 0, isTFE>;
+  def _OFFEN  : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, legal_load_vt, TiedDest, isLds, 0, isTFE, hasGFX12Enc>;
+  def _IDXEN  : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, legal_load_vt, TiedDest, isLds, 0, isTFE, hasGFX12Enc>;
+  def _BOTHEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, legal_load_vt, TiedDest, isLds, 0, isTFE, hasGFX12Enc>;
 
   let DisableWQM = 1 in {
-    def _OFFSET_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, legal_load_vt, TiedDest, isLds, 0, isTFE>;
-    def _OFFEN_exact  : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, legal_load_vt, TiedDest, isLds, 0, isTFE>;
-    def _IDXEN_exact  : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, legal_load_vt, TiedDest, isLds, 0, isTFE>;
-    def _BOTHEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, legal_load_vt, TiedDest, isLds, 0, isTFE>;
+    def _OFFSET_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, legal_load_vt, TiedDest, isLds, 0, isTFE, hasGFX12Enc>;
+    def _OFFEN_exact  : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, legal_load_vt, TiedDest, isLds, 0, isTFE, hasGFX12Enc>;
+    def _IDXEN_exact  : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, legal_load_vt, TiedDest, isLds, 0, isTFE, hasGFX12Enc>;
+    def _BOTHEN_exact : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, legal_load_vt, TiedDest, isLds, 0, isTFE, hasGFX12Enc>;
   }
 }
 
 multiclass MUBUF_Pseudo_Loads<string opName, ValueType load_vt = i32,
                               bit TiedDest = 0, bit isLds = 0> {
-  defm NAME : MUBUF_Pseudo_Loads_Helper<opName, load_vt, TiedDest, isLds, 0>;
-  if !not(isLds) then
-    defm _TFE : MUBUF_Pseudo_Loads_Helper<opName, load_vt, TiedDest, isLds, 1>;
+  defm NAME : MUBUF_Pseudo_Loads_Helper<opName, load_vt, TiedDest, isLds, 0, 0>;
+  defm _VBUFFER : MUBUF_Pseudo_Loads_Helper<opName, load_vt, TiedDest, isLds, 0, 1>;
+
+  if !not(isLds) then {
+    defm _TFE : MUBUF_Pseudo_Loads_Helper<opName, load_vt, TiedDest, isLds, 1, 0>;
+    defm _TFE_VBUFFER : MUBUF_Pseudo_Loads_Helper<opName, load_vt, TiedDest, isLds, 1, 1>;
+  }
 }
 
 multiclass MUBUF_Pseudo_Loads_Lds<string opName, ValueType load_vt = i32> {
@@ -548,18 +576,24 @@ multiclass MUBUF_Pseudo_Loads_LDSOpc<string opName,
   def _OFFEN  : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, legal_load_vt, TiedDest, isLds, isLdsOpc>;
   def _IDXEN  : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, legal_load_vt, TiedDest, isLds, isLdsOpc>;
   def _BOTHEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, legal_load_vt, TiedDest, isLds, isLdsOpc>;
+
+  def _VBUFFER_OFFSET : MUBUF_Load_Pseudo <opName, BUFAddrKind.Offset, legal_load_vt, TiedDest, isLds, isLdsOpc, 0, 1>;
+  def _VBUFFER_OFFEN  : MUBUF_Load_Pseudo <opName, BUFAddrKind.OffEn, legal_load_vt, TiedDest, isLds, isLdsOpc, 0, 1>;
+  def _VBUFFER_IDXEN  : MUBUF_Load_Pseudo <opName, BUFAddrKind.IdxEn, legal_load_vt, TiedDest, isLds, isLdsOpc, 0, 1>;
+  def _VBUFFER_BOTHEN : MUBUF_Load_Pseudo <opName, BUFAddrKind.BothEn, legal_load_vt, TiedDest, isLds, isLdsOpc, 0, 1>;
 }
 
 class MUBUF_Store_Pseudo <string opName,
                           int addrKind,
                           ValueType store_vt,
                           bit isTFE = 0,
+                          bit hasGFX12Enc = 0,
                           list<dag> pattern=[],
                           // Workaround bug bz30254
                           int addrKindCopy = addrKind>
   : MUBUF_Pseudo<opName,
                  (outs),
-                 getMUBUFIns<addrKindCopy, [getVregSrcForVT<store_vt>.ret], isTFE>.ret,
+                 getMUBUFIns<addrKindCopy, [getVregSrcForVT<store_vt>.ret], isTFE, hasGFX12Enc>.ret,
                  getMUBUFAsmOps<addrKindCopy, 0, 0, isTFE>.ret,
                  pattern>,
     MUBUF_SetupAddr<addrKindCopy> {
@@ -584,30 +618,33 @@ multiclass MUBUF_Pseudo_Store_Pats<string BaseInst, ValueType store_vt = i32, SD
 }
 
 multiclass MUBUF_Pseudo_Stores_Helper<string opName, ValueType store_vt,
-                                      bit isTFE> {
+                                      bit isTFE, bit hasGFX12Enc> {
   defvar legal_store_vt = !if(!eq(store_vt, v3f16), v4f16, store_vt);
 
-  def _OFFSET : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, legal_store_vt, isTFE>,
+  def _OFFSET : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, legal_store_vt, isTFE, hasGFX12Enc>,
     MUBUFAddr64Table<0, NAME>;
 
-  def _ADDR64 : MUBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, legal_store_vt, isTFE>,
+  def _ADDR64 : MUBUF_Store_Pseudo <opName, BUFAddrKind.Addr64, legal_store_vt, isTFE, hasGFX12Enc>,
     MUBUFAddr64Table<1, NAME>;
 
-  def _OFFEN  : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, legal_store_vt, isTFE>;
-  def _IDXEN  : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, legal_store_vt, isTFE>;
-  def _BOTHEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, legal_store_vt, isTFE>;
+  def _OFFEN  : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, legal_store_vt, isTFE, hasGFX12Enc>;
+  def _IDXEN  : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, legal_store_vt, isTFE, hasGFX12Enc>;
+  def _BOTHEN : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, legal_store_vt, isTFE, hasGFX12Enc>;
 
   let DisableWQM = 1 in {
-    def _OFFSET_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, legal_store_vt, isTFE>;
-    def _OFFEN_exact  : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, legal_store_vt, isTFE>;
-    def _IDXEN_exact  : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, legal_store_vt, isTFE>;
-    def _BOTHEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, legal_store_vt, isTFE>;
+    def _OFFSET_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.Offset, legal_store_vt, isTFE, hasGFX12Enc>;
+    def _OFFEN_exact  : MUBUF_Store_Pseudo <opName, BUFAddrKind.OffEn, legal_store_vt, isTFE, hasGFX12Enc>;
+    def _IDXEN_exact  : MUBUF_Store_Pseudo <opName, BUFAddrKind.IdxEn, legal_store_vt, isTFE, hasGFX12Enc>;
+    def _BOTHEN_exact : MUBUF_Store_Pseudo <opName, BUFAddrKind.BothEn, legal_store_vt, isTFE, hasGFX12Enc>;
   }
 }
 
 multiclass MUBUF_Pseudo_Stores<string opName, ValueType store_vt = i32> {
-  defm NAME : MUBUF_Pseudo_Stores_Helper<opName, store_vt, 0>;
-  defm _TFE : MUBUF_Pseudo_Stores_Helper<opName, store_vt, 1>;
+  defm NAME : MUBUF_Pseudo_Stores_Helper<opName, store_vt, 0, 0>;
+  defm _TFE : MUBUF_Pseudo_Stores_Helper<opName, store_vt, 1, 0>;
+
+  defm _VBUFFER : MUBUF_Pseudo_Stores_Helper<opName, store_vt, 0, 1>;
+  defm _TFE_VBUFFER : MUBUF_Pseudo_Stores_Helper<opName, store_vt, 1, 1>;
 }
 
 class MUBUF_Pseudo_Store_Lds<string opName>
@@ -629,14 +666,15 @@ class MUBUF_Pseudo_Store_Lds<string opName>
   let AsmMatchConverter = "cvtMubuf";
 }
 
-class getMUBUFAtomicInsDA<RegisterClass vdataClass, bit vdata_in,
+class getMUBUFAtomicInsDA<RegisterClass vdataClass, bit vdata_in, bit hasGFX12Enc,
                           list<RegisterClass> vaddrList=[]> {
   RegisterClass vaddrClass = !if(!empty(vaddrList), ?, !head(vaddrList));
   RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret;
 
   dag VData = !if(vdata_in, (ins vdata_op:$vdata_in), (ins vdata_op:$vdata));
   dag Data = !if(!empty(vaddrList), VData, !con(VData, (ins vaddrClass:$vaddr)));
-  dag MainInputs = (ins SReg_128:$srsrc, SCSrc_b32:$soffset, offset:$offset);
+  dag SOffset = !if(hasGFX12Enc, (ins SReg_32:$soffset), (ins SCSrc_b32:$soffset));
+  dag MainInputs = !con((ins SReg_128:$srsrc), SOffset, (ins offset:$offset));
   dag CPol = !if(vdata_in, (ins CPol_GLC_WithDefault:$cpol),
                            (ins CPol_NonGLC_WithDefault:$cpol));
 
@@ -646,19 +684,20 @@ class getMUBUFAtomicInsDA<RegisterClass vdataClass, bit vdata_in,
 class getMUBUFAtomicIns<int addrKind,
                         RegisterClass vdataClass,
                         bit vdata_in,
+                        bit hasGFX12Enc,
                         // Workaround bug bz30254
                         RegisterClass vdataClassCopy=vdataClass> {
   dag ret =
     !if(!eq(addrKind, BUFAddrKind.Offset),
-            getMUBUFAtomicInsDA<vdataClassCopy, vdata_in>.ret,
+            getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, hasGFX12Enc>.ret,
     !if(!eq(addrKind, BUFAddrKind.OffEn),
-            getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret,
+            getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, hasGFX12Enc, [VGPR_32]>.ret,
     !if(!eq(addrKind, BUFAddrKind.IdxEn),
-            getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VGPR_32]>.ret,
+            getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, hasGFX12Enc, [VGPR_32]>.ret,
     !if(!eq(addrKind, BUFAddrKind.BothEn),
-            getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret,
+            getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, hasGFX12Enc, [VReg_64]>.ret,
     !if(!eq(addrKind, BUFAddrKind.Addr64),
-            getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, [VReg_64]>.ret,
+            getMUBUFAtomicInsDA<vdataClassCopy, vdata_in, hasGFX12Enc, [VReg_64]>.ret,
     (ins))))));
 }
 
@@ -686,13 +725,14 @@ class MUBUF_Atomic_Pseudo<string opName,
 
 class MUBUF_AtomicNoRet_Pseudo<string opName, int addrKind,
                                RegisterClass vdataClass,
+                               bit hasGFX12Enc = 0,
                                list<dag> pattern=[],
                                // Workaround bug bz30254
                                int addrKindCopy = addrKind,
                                RegisterClass vdataClassCopy = vdataClass>
   : MUBUF_Atomic_Pseudo<opName, addrKindCopy,
                         (outs),
-                        getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 0>.ret,
+                        getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 0, hasGFX12Enc>.ret,
                         getMUBUFAsmOps<addrKindCopy>.ret,
                         pattern>,
     AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 0> {
@@ -705,6 +745,7 @@ class MUBUF_AtomicNoRet_Pseudo<string opName, int addrKind,
 
 class MUBUF_AtomicRet_Pseudo<string opName, int addrKind,
                              RegisterClass vdataClass,
+                             bit hasGFX12Enc = 0,
                              list<dag> pattern=[],
                              // Workaround bug bz30254
                              int addrKindCopy = addrKind,
@@ -712,7 +753,7 @@ class MUBUF_AtomicRet_Pseudo<string opName, int addrKind,
                              RegisterOperand vdata_op = getLdStRegisterOperand<vdataClass>.ret>
   : MUBUF_Atomic_Pseudo<opName, addrKindCopy,
                         (outs vdata_op:$vdata),
-                        getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 1>.ret,
+                        getMUBUFAtomicIns<addrKindCopy, vdataClassCopy, 1, hasGFX12Enc>.ret,
                         getMUBUFAsmOps<addrKindCopy>.ret,
                         pattern>,
     AtomicNoRet<opName # "_" # getAddrName<addrKindCopy>.ret, 1> {
@@ -730,13 +771,21 @@ multiclass MUBUF_Pseudo_Atomics_NO_RTN <string opName,
                                         ValueType vdataType,
                                         bit isFP = isFloatType<vdataType>.ret> {
   let FPAtomic = isFP in {
-    def _OFFSET : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass>,
+    def _OFFSET : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 0>,
                   MUBUFAddr64Table <0, NAME>;
-    def _ADDR64 : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass>,
+    def _ADDR64 : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 0>,
                   MUBUFAddr64Table <1, NAME>;
-    def _OFFEN  : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.OffEn,  vdataClass>;
-    def _IDXEN  : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.IdxEn,  vdataClass>;
-    def _BOTHEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
+    def _OFFEN  : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.OffEn,  vdataClass, 0>;
+    def _IDXEN  : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.IdxEn,  vdataClass, 0>;
+    def _BOTHEN : MUBUF_AtomicNoRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, 0>;
+
+    def _VBUFFER_OFFSET : MUBUF_AtomicNoRet_Pseudo <opName #_vbuffer, BUFAddrKind.Offset, vdataClass, 1>,
+                  MUBUFAddr64Table <0, NAME # "_VBUFFER">;
+    def _VBUFFER_ADDR64 : MUBUF_AtomicNoRet_Pseudo <opName #_vbuffer, BUFAddrKind.Addr64, vdataClass, 1>,
+                  MUBUFAddr64Table <1, NAME # "_VBUFFER">;
+    def _VBUFFER_OFFEN  : MUBUF_AtomicNoRet_Pseudo <opName #_vbuffer, BUFAddrKind.OffEn,  vdataClass, 1>;
+    def _VBUFFER_IDXEN  : MUBUF_AtomicNoRet_Pseudo <opName #_vbuffer, BUFAddrKind.IdxEn,  vdataClass, 1>;
+    def _VBUFFER_BOTHEN : MUBUF_AtomicNoRet_Pseudo <opName #_vbuffer, BUFAddrKind.BothEn, vdataClass, 1>;
   }
 }
 
@@ -746,21 +795,37 @@ multiclass MUBUF_Pseudo_Atomics_RTN <string opName,
                                      SDPatternOperator atomic,
                                      bit isFP = isFloatType<vdataType>.ret> {
   let FPAtomic = isFP in {
-    def _OFFSET_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass,
+    def _OFFSET_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Offset, vdataClass, 0,
       [(set vdataType:$vdata,
        (atomic (MUBUFOffset v4i32:$srsrc, i32:$soffset, i32:$offset),
                vdataType:$vdata_in))]>,
       MUBUFAddr64Table <0, NAME # "_RTN">;
 
-    def _ADDR64_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass,
+    def _ADDR64_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.Addr64, vdataClass, 0,
       [(set vdataType:$vdata,
        (atomic (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset),
                 vdataType:$vdata_in))]>,
       MUBUFAddr64Table <1, NAME # "_RTN">;
 
-    def _OFFEN_RTN  : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.OffEn,  vdataClass>;
-    def _IDXEN_RTN  : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.IdxEn,  vdataClass>;
-    def _BOTHEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass>;
+    def _OFFEN_RTN  : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.OffEn,  vdataClass, 0>;
+    def _IDXEN_RTN  : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.IdxEn,  vdataClass, 0>;
+    def _BOTHEN_RTN : MUBUF_AtomicRet_Pseudo <opName, BUFAddrKind.BothEn, vdataClass, 0>;
+
+    def _VBUFFER_OFFSET_RTN : MUBUF_AtomicRet_Pseudo <opName #_vbuffer, BUFAddrKind.Offset, vdataClass, 1,
+      [(set vdataType:$vdata,
+       (atomic (MUBUFOffset v4i32:$srsrc, i32:$soffset, i32:$offset),
+               vdataType:$vdata_in))]>,
+      MUBUFAddr64Table <0, NAME # "_VBUFFER_RTN">;
+
+    def _VBUFFER_ADDR64_RTN : MUBUF_AtomicRet_Pseudo <opName #_vbuffer, BUFAddrKind.Addr64, vdataClass, 1,
+      [(set vdataType:$vdata,
+       (atomic (MUBUFAddr64 v4i32:$srsrc, i64:$vaddr, i32:$soffset, i32:$offset),
+                vdataType:$vdata_in))]>,
+      MUBUFAddr64Table <1, NAME # "_VBUFFER_RTN">;
+
+    def _VBUFFER_OFFEN_RTN  : MUBUF_AtomicRet_Pseudo <opName #_vbuffer, BUFAddrKind.OffEn,  vdataClass, 1>;
+    def _VBUFFER_IDXEN_RTN  : MUBUF_AtomicRet_Pseudo <opName #_vbuffer, BUFAddrKind.IdxEn,  vdataClass, 1>;
+    def _VBUFFER_BOTHEN_RTN : MUBUF_AtomicRet_Pseudo <opName #_vbuffer, BUFAddrKind.BothEn, vdataClass, 1>;
   }
 }
 
@@ -2122,7 +2187,88 @@ class MUBUF_Real_gfx6_gfx7<bits<8> op, MUBUF_Pseudo ps> :
 }
 
 //===----------------------------------------------------------------------===//
-// MUBUF - GFX11.
+// Base ENC_VBUFFER for GFX12.
+//===----------------------------------------------------------------------===//
+
+class VBUFFER_Real <BUF_Pseudo ps, string real_name = ps.Mnemonic> :
+  InstSI <ps.OutOperandList, ps.InOperandList, real_name # ps.AsmOperands, []>, Enc96 {
+
+  let isPseudo = 0;
+  let isCodeGenOnly = 0;
+
+  let VM_CNT = 1;
+  let EXP_CNT = 1;
+
+  // copy relevant pseudo op flags
+  let SubtargetPredicate = ps.SubtargetPredicate;
+  let AsmMatchConverter  = ps.AsmMatchConverter;
+  let OtherPredicates    = ps.OtherPredicates;
+  let Constraints        = ps.Constraints;
+  let DisableEncoding    = ps.DisableEncoding;
+  let TSFlags            = ps.TSFlags;
+  let UseNamedOperandTable = ps.UseNamedOperandTable;
+  let SchedRW            = ps.SchedRW;
+  let mayLoad            = ps.mayLoad;
+  let mayStore           = ps.mayStore;
+  let IsAtomicRet        = ps.IsAtomicRet;
+  let IsAtomicNoRet      = ps.IsAtomicNoRet;
+  let VALU               = ps.VALU;
+  let LGKM_CNT           = ps.LGKM_CNT;
+
+  bits<24> offset;
+  bits<8>  vaddr;
+  bits<10> vdata;
+
+  bits<7>  srsrc;
+  bits<7>  soffset;
+  bits<6>  cpol;
+
+  let Inst{95-72} = !if(ps.has_offset, offset, ?);
+  let Inst{71-64} = !if(ps.has_vaddr, vaddr, ?);
+  let Inst{39-32} = !if(ps.has_vdata, vdata{7-0}, ?);
+
+  let Inst{47-41} = !if(ps.has_srsrc, srsrc, ?);
+  let Inst{49-48} = 0b00;
+  let Inst{6-0}   = !if(ps.has_soffset, soffset, ?);
+  let Inst{22}    = ps.tfe;
+  let Inst{62}    = ps.offen;
+  let Inst{63}    = ps.idxen;
+
+  let Inst{54-53} = cpol{2-1}; // th{2-1}
+  let Inst{52}    = !if(ps.IsAtomicRet, 1, cpol{0}); // th{0}
+  let Inst{51-50} = cpol{4-3}; // scope
+
+  let Inst{31-26} = 0b110001;
+}
+
+class VBUFFER_MUBUF_Real_gfx12<bits<8> op, MUBUF_Pseudo ps,
+                               string real_name = ps.Mnemonic> :
+  VBUFFER_Real<ps, real_name>, SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX12> {
+
+  let MUBUF = 1;
+
+  // Set the last bit of format to 1 to avoid round-trip issues, as some tools
+  // print BUF_FMT_INVALID for format 0.
+  let Inst{55} = 0b1;
+  let Inst{21-14} = op;
+  let SubtargetPredicate = isGFX12Only;
+}
+
+class VBUFFER_MTBUF_Real_gfx12<bits<4> op, MTBUF_Pseudo ps,
+                               string real_name = ps.Mnemonic> :
+  VBUFFER_Real<ps, real_name>, SIMCInstr<ps.PseudoInstr, SIEncodingFamily.GFX12> {
+
+  let MTBUF = 1;
+
+  bits<7>  format;
+
+  let Inst{17-14} = op;
+  let Inst{21-18} = 0b1000;
+  let Inst{61-55} = format;
+}
+
+//===----------------------------------------------------------------------===//
+// MUBUF - GFX11, GFX12.
 //===----------------------------------------------------------------------===//
 
 // Shortcut to default Mnemonic from MUBUF_Pseudo. Hides the cast to the
@@ -2132,18 +2278,42 @@ class get_MUBUF_ps<string name> {
 }
 
 // gfx11 instruction that accept both old and new assembler name.
-class Pre_gfx11_MUBUF_Name <string mnemonic, string real_name> :
+class Mnem_gfx11_gfx12 <string mnemonic, string real_name> :
   MnemonicAlias<mnemonic, real_name>, Requires<[isGFX11Plus]>;
 
+class Mnem_gfx11 <string mnemonic, string real_name> :
+  MnemonicAlias<mnemonic, real_name>, Requires<[isGFX11Only]>;
+
+class Mnem_gfx12 <string mnemonic, string real_name> :
+  MnemonicAlias<mnemonic, real_name>, Requires<[isGFX12Plus]>;
+
 class MUBUF_Real_gfx11_impl<bits<8> op, string ps_name, string real_name> :
   MUBUF_Real_gfx11<op, !cast<MUBUF_Pseudo>(ps_name), real_name>;
-let DecoderNamespace = "GFX11" in
+
+class VBUFFER_MUBUF_Real_gfx12_impl<bits<8> op, string ps_name, string real_name> :
+  VBUFFER_MUBUF_Real_gfx12<op, !cast<MUBUF_Pseudo>(ps_name), real_name>;
+
 multiclass MUBUF_Real_AllAddr_gfx11_Renamed_Impl2<bits<8> op, string real_name> {
+  let DecoderNamespace = "GFX11" in {
   def _BOTHEN_gfx11 : MUBUF_Real_gfx11_impl<op, NAME # "_BOTHEN", real_name>;
   def _IDXEN_gfx11 : MUBUF_Real_gfx11_impl<op, NAME # "_IDXEN", real_name>;
   def _OFFEN_gfx11 : MUBUF_Real_gfx11_impl<op, NAME # "_OFFEN", real_name>;
   def _OFFSET_gfx11 : MUBUF_Real_gfx11_impl<op, NAME # "_OFFSET", real_name>;
 }
+}
+
+multiclass MUBUF_Real_AllAddr_gfx12_Renamed_Impl2<bits<8> op, string real_name> {
+  let DecoderNamespace = "GFX12" in {
+    def _BOTHEN_gfx12 : VBUFFER_MUBUF_Real_gfx12_impl<op, NAME # "_VBUFFER_BOTHEN", real_name>;
+    def _IDXEN_gfx12 : VBUFFER_MUBUF_Real_gfx12_impl<op, NAME # "_VBUFFER_IDXEN", real_name>;
+    def _OFFEN_gfx12 : VBUFFER_MUBUF_Real_gfx12_impl<op, NAME # "_VBUFFER_OFFEN", real_name>;
+    def _OFFSET_gfx12 : VBUFFER_MUBUF_Real_gfx12_impl<op, NAME # "_VBUFFER_OFFSET", real_name>;
+  }
+}
+
+multiclass MUBUF_Real_AllAddr_gfx11_gfx12_Renamed_Impl2<bits<8> op, string real_name> :
+  MUBUF_Real_AllAddr_gfx11_Renamed_Impl2<op, real_name>,
+  MUBUF_Real_AllAddr_gfx12_Renamed_Impl2<op, real_name>;
 
 multiclass MUBUF_Real_AllAddr_gfx11_Renamed_Impl<bits<8> op, string real_name,
                                                  bit hasTFE = 1> {
@@ -2152,21 +2322,36 @@ multiclass MUBUF_Real_AllAddr_gfx11_Renamed_Impl<bits<8> op, string real_name,
     defm _TFE : MUBUF_Real_AllAddr_gfx11_Renamed_Impl2<op, real_name>;
 }
 
-// Non-renamed, non-atomic gfx11 mubuf instructions.
+multiclass MUBUF_Real_AllAddr_gfx11_gfx12_Renamed_Impl<bits<8> op, string real_name,
+                                                 bit hasTFE = 1> {
+  defm NAME : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed_Impl2<op, real_name>;
+  if hasTFE then
+    defm _TFE : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed_Impl2<op, real_name>;
+}
+
+// Non-renamed, non-atomic gfx11/gfx12 mubuf instructions.
 multiclass MUBUF_Real_AllAddr_gfx11<bits<8> op, bit hasTFE = 1> :
   MUBUF_Real_AllAddr_gfx11_Renamed_Impl<op, get_MUBUF_ps<NAME>.Mnemonic, hasTFE>;
 
-multiclass MUBUF_Real_AllAddr_gfx11_Renamed<bits<8> op, string real_name> :
-  MUBUF_Real_AllAddr_gfx11_Renamed_Impl<op, real_name> {
-  def : Pre_gfx11_MUBUF_Name<get_MUBUF_ps<NAME>.Mnemonic, real_name>;
+multiclass MUBUF_Real_AllAddr_gfx11_gfx12<bits<8> op, bit hasTFE = 1> :
+  MUBUF_Real_AllAddr_gfx11_gfx12_Renamed_Impl<op, get_MUBUF_ps<NAME>.Mnemonic, hasTFE>;
+
+multiclass MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<bits<8> op, string real_name> :
+  MUBUF_Real_AllAddr_gfx11_gfx12_Renamed_Impl<op, real_name> {
+  def : Mnem_gfx11_gfx12<get_MUBUF_ps<NAME>.Mnemonic, real_name>;
 }
 
 class MUBUF_Real_Atomic_gfx11_impl<bits<8> op, string ps_name,
                                    string real_name> :
   Base_MUBUF_Real_Atomic_gfx11<op, !cast<MUBUF_Pseudo>(ps_name), real_name>;
-let DecoderNamespace = "GFX11" in
+
+class MUBUF_Real_Atomic_gfx12_impl<bits<8> op, string ps_name,
+                                   string real_name> :
+  VBUFFER_MUBUF_Real_gfx12<op, !cast<MUBUF_Pseudo>(ps_name), real_name>;
+
 multiclass MUBUF_Real_Atomic_gfx11_Renamed_impl<bits<8> op, bit is_return,
                                                 string real_name> {
+  let DecoderNamespace = "GFX11" in {
   defvar Rtn = !if(!eq(is_return, 1), "_RTN", "");
   def _BOTHEN#Rtn#_gfx11 :
     MUBUF_Real_Atomic_gfx11_impl<op, NAME # "_BOTHEN" # Rtn, real_name>,
@@ -2181,16 +2366,61 @@ multiclass MUBUF_Real_Atomic_gfx11_Renamed_impl<bits<8> op, bit is_return,
     MUBUF_Real_Atomic_gfx11_impl<op, NAME # "_OFFSET" # Rtn, real_name>,
     AtomicNoRet<NAME # "_OFFSET_gfx11", is_return>;
 }
+}
 
-// Non-renamed gfx11 mubuf atomic.
-multiclass MUBUF_Real_Atomic_gfx11<bits<8> op> :
-  MUBUF_Real_Atomic_gfx11_Renamed_impl<op, 0, get_MUBUF_ps<NAME>.Mnemonic>,
-  MUBUF_Real_Atomic_gfx11_Renamed_impl<op, 1, get_MUBUF_ps<NAME>.Mnemonic>;
+multiclass MUBUF_Real_Atomic_gfx12_Renamed_impl<bits<8> op, bit is_return,
+                                                string real_name> {
+  let DecoderNamespace = "GFX12" in {
+    defvar Rtn = !if(!eq(is_return, 1), "_RTN", "");
+    def _BOTHEN#Rtn#_gfx12 :
+      MUBUF_Real_Atomic_gfx12_impl<op, NAME # "_VBUFFER_BOTHEN" # Rtn, real_name>,
+      AtomicNoRet<NAME # "_BOTHEN_gfx12", is_return>;
+    def _IDXEN#Rtn#_gfx12 :
+      MUBUF_Real_Atomic_gfx12_impl<op, NAME # "_VBUFFER_IDXEN" # Rtn, real_name>,
+      AtomicNoRet<NAME # "_IDXEN_gfx12", is_return>;
+    def _OFFEN#Rtn#_gfx12 :
+      MUBUF_Real_Atomic_gfx12_impl<op, NAME # "_VBUFFER_OFFEN" # Rtn, real_name>,
+      AtomicNoRet<NAME # "_OFFEN_gfx12", is_return>;
+    def _OFFSET#Rtn#_gfx12 :
+      MUBUF_Real_Atomic_gfx12_impl<op, NAME # "_VBUFFER_OFFSET" # Rtn, real_name>,
+      AtomicNoRet<NAME # "_OFFSET_gfx12", is_return>;
+  }
+}
+
+multiclass MUBUF_Real_Atomic_gfx11_gfx12_Renamed_impl<bits<8> op, bit is_return,
+                                                string real_name> :
+  MUBUF_Real_Atomic_gfx11_Renamed_impl<op, is_return, real_name>,
+  MUBUF_Real_Atomic_gfx12_Renamed_impl<op, is_return, real_name>;
+
+// Non-renamed gfx11/gfx12 mubuf atomic.
+multiclass MUBUF_Real_Atomic_gfx11_gfx12<bits<8> op> :
+  MUBUF_Real_Atomic_gfx11_gfx12_Renamed_impl<op, 0, get_MUBUF_ps<NAME>.Mnemonic>,
+  MUBUF_Real_Atomic_gfx11_gfx12_Renamed_impl<op, 1, get_MUBUF_ps<NAME>.Mnemonic>;
+
+multiclass MUBUF_Real_Atomic_gfx12<bits<8> op> :
+  MUBUF_Real_Atomic_gfx12_Renamed_impl<op, 0, get_MUBUF_ps<NAME>.Mnemonic>,
+  MUBUF_Real_Atomic_gfx12_Renamed_impl<op, 1, get_MUBUF_ps<NAME>.Mnemonic>;
 
 multiclass MUBUF_Real_Atomic_gfx11_Renamed<bits<8> op, string real_name> :
   MUBUF_Real_Atomic_gfx11_Renamed_impl<op, 0, real_name>,
-  MUBUF_Real_Atomic_gfx11_Renamed_impl<op, 1, real_name>  {
-  def : Pre_gfx11_MUBUF_Name<get_MUBUF_ps<NAME>.Mnemonic, real_name>;
+  MUBUF_Real_Atomic_gfx11_Renamed_impl<op, 1, real_name> {
+  def : Mnem_gfx11_gfx12<get_MUBUF_ps<NAME>.Mnemonic, real_name>;
+}
+
+multiclass MUBUF_Real_Atomic_gfx11_gfx12_Renamed<bits<8> op, string real_name> :
+  MUBUF_Real_Atomic_gfx11_gfx12_Renamed_impl<op, 0, real_name>,
+  MUBUF_Real_Atomic_gfx11_gfx12_Renamed_impl<op, 1, real_name>  {
+  def : Mnem_gfx11_gfx12<get_MUBUF_ps<NAME>.Mnemonic, real_name>;
+}
+
+multiclass MUBUF_Real_Atomic_gfx11_gfx12_Renamed_gfx12_Renamed<bits<8> op, string gfx12_name, string gfx11_name> :
+  MUBUF_Real_Atomic_gfx11_Renamed_impl<op, 0, gfx11_name>,
+  MUBUF_Real_Atomic_gfx11_Renamed_impl<op, 1, gfx11_name>,
+  MUBUF_Real_Atomic_gfx12_Renamed_impl<op, 0, gfx12_name>,
+  MUBUF_Real_Atomic_gfx12_Renamed_impl<op, 1, gfx12_name> {
+  def : Mnem_gfx11<get_MUBUF_ps<NAME>.Mnemonic, gfx11_name>;
+  def : Mnem_gfx12<get_MUBUF_ps<NAME>.Mnemonic, gfx12_name>;
+  def : Mnem_gfx12<gfx11_name, gfx12_name>;
 }
 
 let DecoderNamespace = "GFX11" in {
@@ -2198,84 +2428,84 @@ def BUFFER_GL0_INV_gfx11          : MUBUF_Real_gfx11<0x02B, BUFFER_GL0_INV>;
 def BUFFER_GL1_INV_gfx11          : MUBUF_Real_gfx11<0x02C, BUFFER_GL1_INV>;
 }
 
-defm BUFFER_LOAD_DWORD            : MUBUF_Real_AllAddr_gfx11_Renamed<0x014, "buffer_load_b32">;
-defm BUFFER_LOAD_DWORDX2          : MUBUF_Real_AllAddr_gfx11_Renamed<0x015, "buffer_load_b64">;
-defm BUFFER_LOAD_DWORDX3          : MUBUF_Real_AllAddr_gfx11_Renamed<0x016, "buffer_load_b96">;
-defm BUFFER_LOAD_DWORDX4          : MUBUF_Real_AllAddr_gfx11_Renamed<0x017, "buffer_load_b128">;
-defm BUFFER_LOAD_SHORT_D16        : MUBUF_Real_AllAddr_gfx11_Renamed<0x020, "buffer_load_d16_b16">;
-defm BUFFER_LOAD_FORMAT_D16_X     : MUBUF_Real_AllAddr_gfx11_Renamed<0x008, "buffer_load_d16_format_x">;
-defm BUFFER_LOAD_FORMAT_D16_XY    : MUBUF_Real_AllAddr_gfx11_Renamed<0x009, "buffer_load_d16_format_xy">;
-defm BUFFER_LOAD_FORMAT_D16_XYZ   : MUBUF_Real_AllAddr_gfx11_Renamed<0x00a, "buffer_load_d16_format_xyz">;
-defm BUFFER_LOAD_FORMAT_D16_XYZW  : MUBUF_Real_AllAddr_gfx11_Renamed<0x00b, "buffer_load_d16_format_xyzw">;
-defm BUFFER_LOAD_SHORT_D16_HI     : MUBUF_Real_AllAddr_gfx11_Renamed<0x023, "buffer_load_d16_hi_b16">;
-defm BUFFER_LOAD_FORMAT_D16_HI_X  : MUBUF_Real_AllAddr_gfx11_Renamed<0x026, "buffer_load_d16_hi_format_x">;
-defm BUFFER_LOAD_SBYTE_D16_HI     : MUBUF_Real_AllAddr_gfx11_Renamed<0x022, "buffer_load_d16_hi_i8">;
-defm BUFFER_LOAD_UBYTE_D16_HI     : MUBUF_Real_AllAddr_gfx11_Renamed<0x021, "buffer_load_d16_hi_u8">;
-defm BUFFER_LOAD_SBYTE_D16        : MUBUF_Real_AllAddr_gfx11_Renamed<0x01f, "buffer_load_d16_i8">;
-defm BUFFER_LOAD_UBYTE_D16        : MUBUF_Real_AllAddr_gfx11_Renamed<0x01e, "buffer_load_d16_u8">;
-defm BUFFER_LOAD_FORMAT_X         : MUBUF_Real_AllAddr_gfx11<0x000>;
-defm BUFFER_LOAD_FORMAT_XY        : MUBUF_Real_AllAddr_gfx11<0x001>;
-defm BUFFER_LOAD_FORMAT_XYZ       : MUBUF_Real_AllAddr_gfx11<0x002>;
-defm BUFFER_LOAD_FORMAT_XYZW      : MUBUF_Real_AllAddr_gfx11<0x003>;
-defm BUFFER_LOAD_SBYTE            : MUBUF_Real_AllAddr_gfx11_Renamed<0x011, "buffer_load_i8">;
-defm BUFFER_LOAD_SSHORT           : MUBUF_Real_AllAddr_gfx11_Renamed<0x013, "buffer_load_i16">;
-defm BUFFER_LOAD_UBYTE            : MUBUF_Real_AllAddr_gfx11_Renamed<0x010, "buffer_load_u8">;
-defm BUFFER_LOAD_USHORT           : MUBUF_Real_AllAddr_gfx11_Renamed<0x012, "buffer_load_u16">;
+defm BUFFER_LOAD_DWORD            : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x014, "buffer_load_b32">;
+defm BUFFER_LOAD_DWORDX2          : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x015, "buffer_load_b64">;
+defm BUFFER_LOAD_DWORDX3          : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x016, "buffer_load_b96">;
+defm BUFFER_LOAD_DWORDX4          : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x017, "buffer_load_b128">;
+defm BUFFER_LOAD_SHORT_D16        : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x020, "buffer_load_d16_b16">;
+defm BUFFER_LOAD_FORMAT_D16_X     : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x008, "buffer_load_d16_format_x">;
+defm BUFFER_LOAD_FORMAT_D16_XY    : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x009, "buffer_load_d16_format_xy">;
+defm BUFFER_LOAD_FORMAT_D16_XYZ   : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x00a, "buffer_load_d16_format_xyz">;
+defm BUFFER_LOAD_FORMAT_D16_XYZW  : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x00b, "buffer_load_d16_format_xyzw">;
+defm BUFFER_LOAD_SHORT_D16_HI     : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x023, "buffer_load_d16_hi_b16">;
+defm BUFFER_LOAD_FORMAT_D16_HI_X  : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x026, "buffer_load_d16_hi_format_x">;
+defm BUFFER_LOAD_SBYTE_D16_HI     : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x022, "buffer_load_d16_hi_i8">;
+defm BUFFER_LOAD_UBYTE_D16_HI     : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x021, "buffer_load_d16_hi_u8">;
+defm BUFFER_LOAD_SBYTE_D16        : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x01f, "buffer_load_d16_i8">;
+defm BUFFER_LOAD_UBYTE_D16        : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x01e, "buffer_load_d16_u8">;
+defm BUFFER_LOAD_FORMAT_X         : MUBUF_Real_AllAddr_gfx11_gfx12<0x000>;
+defm BUFFER_LOAD_FORMAT_XY        : MUBUF_Real_AllAddr_gfx11_gfx12<0x001>;
+defm BUFFER_LOAD_FORMAT_XYZ       : MUBUF_Real_AllAddr_gfx11_gfx12<0x002>;
+defm BUFFER_LOAD_FORMAT_XYZW      : MUBUF_Real_AllAddr_gfx11_gfx12<0x003>;
+defm BUFFER_LOAD_SBYTE            : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x011, "buffer_load_i8">;
+defm BUFFER_LOAD_SSHORT           : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x013, "buffer_load_i16">;
+defm BUFFER_LOAD_UBYTE            : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x010, "buffer_load_u8">;
+defm BUFFER_LOAD_USHORT           : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x012, "buffer_load_u16">;
 defm BUFFER_LOAD_LDS_B32          : MUBUF_Real_AllAddr_gfx11<0x031, 0>;
 defm BUFFER_LOAD_LDS_FORMAT_X     : MUBUF_Real_AllAddr_gfx11<0x032, 0>;
 defm BUFFER_LOAD_LDS_I8           : MUBUF_Real_AllAddr_gfx11<0x02e, 0>;
 defm BUFFER_LOAD_LDS_I16          : MUBUF_Real_AllAddr_gfx11<0x030, 0>;
 defm BUFFER_LOAD_LDS_U8           : MUBUF_Real_AllAddr_gfx11<0x02d, 0>;
 defm BUFFER_LOAD_LDS_U16          : MUBUF_Real_AllAddr_gfx11<0x02f, 0>;
-defm BUFFER_STORE_BYTE            : MUBUF_Real_AllAddr_gfx11_Renamed<0x018, "buffer_store_b8">;
-defm BUFFER_STORE_SHORT           : MUBUF_Real_AllAddr_gfx11_Renamed<0x019, "buffer_store_b16">;
-defm BUFFER_STORE_DWORD           : MUBUF_Real_AllAddr_gfx11_Renamed<0x01A, "buffer_store_b32">;
-defm BUFFER_STORE_DWORDX2         : MUBUF_Real_AllAddr_gfx11_Renamed<0x01B, "buffer_store_b64">;
-defm BUFFER_STORE_DWORDX3         : MUBUF_Real_AllAddr_gfx11_Renamed<0x01C, "buffer_store_b96">;
-defm BUFFER_STORE_DWORDX4         : MUBUF_Real_AllAddr_gfx11_Renamed<0x01D, "buffer_store_b128">;
-defm BUFFER_STORE_FORMAT_D16_X    : MUBUF_Real_AllAddr_gfx11_Renamed<0x00C, "buffer_store_d16_format_x">;
-defm BUFFER_STORE_FORMAT_D16_XY   : MUBUF_Real_AllAddr_gfx11_Renamed<0x00D, "buffer_store_d16_format_xy">;
-defm BUFFER_STORE_FORMAT_D16_XYZ  : MUBUF_Real_AllAddr_gfx11_Renamed<0x00E, "buffer_store_d16_format_xyz">;
-defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx11_Renamed<0x00F, "buffer_store_d16_format_xyzw">;
-defm BUFFER_STORE_BYTE_D16_HI     : MUBUF_Real_AllAddr_gfx11_Renamed<0x024, "buffer_store_d16_hi_b8">;
-defm BUFFER_STORE_SHORT_D16_HI    : MUBUF_Real_AllAddr_gfx11_Renamed<0x025, "buffer_store_d16_hi_b16">;
-defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx11_Renamed<0x027, "buffer_store_d16_hi_format_x">;
-defm BUFFER_STORE_FORMAT_X        : MUBUF_Real_AllAddr_gfx11<0x004>;
-defm BUFFER_STORE_FORMAT_XY       : MUBUF_Real_AllAddr_gfx11<0x005>;
-defm BUFFER_STORE_FORMAT_XYZ      : MUBUF_Real_AllAddr_gfx11<0x006>;
-defm BUFFER_STORE_FORMAT_XYZW     : MUBUF_Real_AllAddr_gfx11<0x007>;
-defm BUFFER_ATOMIC_ADD_F32        : MUBUF_Real_Atomic_gfx11<0x056>;
-defm BUFFER_ATOMIC_ADD            : MUBUF_Real_Atomic_gfx11_Renamed<0x035, "buffer_atomic_add_u32">;
-defm BUFFER_ATOMIC_ADD_X2         : MUBUF_Real_Atomic_gfx11_Renamed<0x043, "buffer_atomic_add_u64">;
-defm BUFFER_ATOMIC_AND            : MUBUF_Real_Atomic_gfx11_Renamed<0x03C, "buffer_atomic_and_b32">;
-defm BUFFER_ATOMIC_AND_X2         : MUBUF_Real_Atomic_gfx11_Renamed<0x049, "buffer_atomic_and_b64">;
-defm BUFFER_ATOMIC_CMPSWAP        : MUBUF_Real_Atomic_gfx11_Renamed<0x034, "buffer_atomic_cmpswap_b32">;
-defm BUFFER_ATOMIC_CMPSWAP_X2     : MUBUF_Real_Atomic_gfx11_Renamed<0x042, "buffer_atomic_cmpswap_b64">;
+defm BUFFER_STORE_BYTE            : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x018, "buffer_store_b8">;
+defm BUFFER_STORE_SHORT           : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x019, "buffer_store_b16">;
+defm BUFFER_STORE_DWORD           : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x01A, "buffer_store_b32">;
+defm BUFFER_STORE_DWORDX2         : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x01B, "buffer_store_b64">;
+defm BUFFER_STORE_DWORDX3         : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x01C, "buffer_store_b96">;
+defm BUFFER_STORE_DWORDX4         : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x01D, "buffer_store_b128">;
+defm BUFFER_STORE_FORMAT_D16_X    : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x00C, "buffer_store_d16_format_x">;
+defm BUFFER_STORE_FORMAT_D16_XY   : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x00D, "buffer_store_d16_format_xy">;
+defm BUFFER_STORE_FORMAT_D16_XYZ  : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x00E, "buffer_store_d16_format_xyz">;
+defm BUFFER_STORE_FORMAT_D16_XYZW : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x00F, "buffer_store_d16_format_xyzw">;
+defm BUFFER_STORE_BYTE_D16_HI     : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x024, "buffer_store_d16_hi_b8">;
+defm BUFFER_STORE_SHORT_D16_HI    : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x025, "buffer_store_d16_hi_b16">;
+defm BUFFER_STORE_FORMAT_D16_HI_X : MUBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x027, "buffer_store_d16_hi_format_x">;
+defm BUFFER_STORE_FORMAT_X        : MUBUF_Real_AllAddr_gfx11_gfx12<0x004>;
+defm BUFFER_STORE_FORMAT_XY       : MUBUF_Real_AllAddr_gfx11_gfx12<0x005>;
+defm BUFFER_STORE_FORMAT_XYZ      : MUBUF_Real_AllAddr_gfx11_gfx12<0x006>;
+defm BUFFER_STORE_FORMAT_XYZW     : MUBUF_Real_AllAddr_gfx11_gfx12<0x007>;
+defm BUFFER_ATOMIC_ADD_F32        : MUBUF_Real_Atomic_gfx11_gfx12<0x056>;
+defm BUFFER_ATOMIC_ADD            : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x035, "buffer_atomic_add_u32">;
+defm BUFFER_ATOMIC_ADD_X2         : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x043, "buffer_atomic_add_u64">;
+defm BUFFER_ATOMIC_AND            : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x03C, "buffer_atomic_and_b32">;
+defm BUFFER_ATOMIC_AND_X2         : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x049, "buffer_atomic_and_b64">;
+defm BUFFER_ATOMIC_CMPSWAP        : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x034, "buffer_atomic_cmpswap_b32">;
+defm BUFFER_ATOMIC_CMPSWAP_X2     : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x042, "buffer_atomic_cmpswap_b64">;
 defm BUFFER_ATOMIC_FCMPSWAP       : MUBUF_Real_Atomic_gfx11_Renamed<0x050, "buffer_atomic_cmpswap_f32">;
-defm BUFFER_ATOMIC_CSUB           : MUBUF_Real_Atomic_gfx11_Renamed<0x037, "buffer_atomic_csub_u32">;
-def : Pre_gfx11_MUBUF_Name<"buffer_atomic_csub", "buffer_atomic_csub_u32">;
-defm BUFFER_ATOMIC_DEC            : MUBUF_Real_Atomic_gfx11_Renamed<0x040, "buffer_atomic_dec_u32">;
-defm BUFFER_ATOMIC_DEC_X2         : MUBUF_Real_Atomic_gfx11_Renamed<0x04D, "buffer_atomic_dec_u64">;
-defm BUFFER_ATOMIC_INC            : MUBUF_Real_Atomic_gfx11_Renamed<0x03F, "buffer_atomic_inc_u32">;
-defm BUFFER_ATOMIC_INC_X2         : MUBUF_Real_Atomic_gfx11_Renamed<0x04C, "buffer_atomic_inc_u64">;
-defm BUFFER_ATOMIC_FMAX           : MUBUF_Real_Atomic_gfx11_Renamed<0x052, "buffer_atomic_max_f32">;
-defm BUFFER_ATOMIC_SMAX           : MUBUF_Real_Atomic_gfx11_Renamed<0x03A, "buffer_atomic_max_i32">;
-defm BUFFER_ATOMIC_SMAX_X2        : MUBUF_Real_Atomic_gfx11_Renamed<0x047, "buffer_atomic_max_i64">;
-defm BUFFER_ATOMIC_UMAX           : MUBUF_Real_Atomic_gfx11_Renamed<0x03B, "buffer_atomic_max_u32">;
-defm BUFFER_ATOMIC_UMAX_X2        : MUBUF_Real_Atomic_gfx11_Renamed<0x048, "buffer_atomic_max_u64">;
-defm BUFFER_ATOMIC_FMIN           : MUBUF_Real_Atomic_gfx11_Renamed<0x051, "buffer_atomic_min_f32">;
-defm BUFFER_ATOMIC_SMIN           : MUBUF_Real_Atomic_gfx11_Renamed<0x038, "buffer_atomic_min_i32">;
-defm BUFFER_ATOMIC_SMIN_X2        : MUBUF_Real_Atomic_gfx11_Renamed<0x045, "buffer_atomic_min_i64">;
-defm BUFFER_ATOMIC_UMIN           : MUBUF_Real_Atomic_gfx11_Renamed<0x039, "buffer_atomic_min_u32">;
-defm BUFFER_ATOMIC_UMIN_X2        : MUBUF_Real_Atomic_gfx11_Renamed<0x046, "buffer_atomic_min_u64">;
-defm BUFFER_ATOMIC_OR             : MUBUF_Real_Atomic_gfx11_Renamed<0x03D, "buffer_atomic_or_b32">;
-defm BUFFER_ATOMIC_OR_X2          : MUBUF_Real_Atomic_gfx11_Renamed<0x04A, "buffer_atomic_or_b64">;
-defm BUFFER_ATOMIC_SUB            : MUBUF_Real_Atomic_gfx11_Renamed<0x036, "buffer_atomic_sub_u32">;
-defm BUFFER_ATOMIC_SUB_X2         : MUBUF_Real_Atomic_gfx11_Renamed<0x044, "buffer_atomic_sub_u64">;
-defm BUFFER_ATOMIC_SWAP           : MUBUF_Real_Atomic_gfx11_Renamed<0x033, "buffer_atomic_swap_b32">;
-defm BUFFER_ATOMIC_SWAP_X2        : MUBUF_Real_Atomic_gfx11_Renamed<0x041, "buffer_atomic_swap_b64">;
-defm BUFFER_ATOMIC_XOR            : MUBUF_Real_Atomic_gfx11_Renamed<0x03E, "buffer_atomic_xor_b32">;
-defm BUFFER_ATOMIC_XOR_X2         : MUBUF_Real_Atomic_gfx11_Renamed<0x04B, "buffer_atomic_xor_b64">;
+defm BUFFER_ATOMIC_CSUB           : MUBUF_Real_Atomic_gfx11_gfx12_Renamed_gfx12_Renamed<0x037, "buffer_atomic_sub_clamp_u32", "buffer_atomic_csub_u32">;
+def : Mnem_gfx11_gfx12<"buffer_atomic_csub", "buffer_atomic_csub_u32">;
+defm BUFFER_ATOMIC_DEC            : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x040, "buffer_atomic_dec_u32">;
+defm BUFFER_ATOMIC_DEC_X2         : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x04D, "buffer_atomic_dec_u64">;
+defm BUFFER_ATOMIC_INC            : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x03F, "buffer_atomic_inc_u32">;
+defm BUFFER_ATOMIC_INC_X2         : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x04C, "buffer_atomic_inc_u64">;
+defm BUFFER_ATOMIC_FMAX           : MUBUF_Real_Atomic_gfx11_gfx12_Renamed_gfx12_Renamed<0x052, "buffer_atomic_max_num_f32", "buffer_atomic_max_f32">;
+defm BUFFER_ATOMIC_SMAX           : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x03A, "buffer_atomic_max_i32">;
+defm BUFFER_ATOMIC_SMAX_X2        : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x047, "buffer_atomic_max_i64">;
+defm BUFFER_ATOMIC_UMAX           : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x03B, "buffer_atomic_max_u32">;
+defm BUFFER_ATOMIC_UMAX_X2        : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x048, "buffer_atomic_max_u64">;
+defm BUFFER_ATOMIC_FMIN           : MUBUF_Real_Atomic_gfx11_gfx12_Renamed_gfx12_Renamed<0x051, "buffer_atomic_min_num_f32", "buffer_atomic_min_f32">;
+defm BUFFER_ATOMIC_SMIN           : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x038, "buffer_atomic_min_i32">;
+defm BUFFER_ATOMIC_SMIN_X2        : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x045, "buffer_atomic_min_i64">;
+defm BUFFER_ATOMIC_UMIN           : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x039, "buffer_atomic_min_u32">;
+defm BUFFER_ATOMIC_UMIN_X2        : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x046, "buffer_atomic_min_u64">;
+defm BUFFER_ATOMIC_OR             : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x03D, "buffer_atomic_or_b32">;
+defm BUFFER_ATOMIC_OR_X2          : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x04A, "buffer_atomic_or_b64">;
+defm BUFFER_ATOMIC_SUB            : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x036, "buffer_atomic_sub_u32">;
+defm BUFFER_ATOMIC_SUB_X2         : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x044, "buffer_atomic_sub_u64">;
+defm BUFFER_ATOMIC_SWAP           : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x033, "buffer_atomic_swap_b32">;
+defm BUFFER_ATOMIC_SWAP_X2        : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x041, "buffer_atomic_swap_b64">;
+defm BUFFER_ATOMIC_XOR            : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x03E, "buffer_atomic_xor_b32">;
+defm BUFFER_ATOMIC_XOR_X2         : MUBUF_Real_Atomic_gfx11_gfx12_Renamed<0x04B, "buffer_atomic_xor_b64">;
 
 //===----------------------------------------------------------------------===//
 // MUBUF - GFX10.
@@ -2571,8 +2801,8 @@ class Base_MTBUF_Real_gfx6_gfx7_gfx10<bits<3> op, MTBUF_Pseudo ps, int ef> :
 // MTBUF - GFX11.
 //===----------------------------------------------------------------------===//
 
-let AssemblerPredicate = isGFX11Only, DecoderNamespace = "GFX11" in
-multiclass MTBUF_Real_AllAddr_gfx11_Renamed_Impl<bits<4> op, string real_name> {
+multiclass MTBUF_Real_AllAddr_gfx11_gfx12_Renamed_Impl<bits<4> op, string real_name> {
+  let AssemblerPredicate = isGFX11Only, DecoderNamespace = "GFX11" in {
   def _BOTHEN_gfx11 :
     Base_MTBUF_Real_gfx11<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN"), real_name>;
   def _IDXEN_gfx11 :
@@ -2583,35 +2813,47 @@ multiclass MTBUF_Real_AllAddr_gfx11_Renamed_Impl<bits<4> op, string real_name> {
     Base_MTBUF_Real_gfx11<op, !cast<MTBUF_Pseudo>(NAME#"_OFFSET"), real_name>;
 }
 
-multiclass MTBUF_Real_AllAddr_gfx11_Impl<bits<4> op, MTBUF_Pseudo ps>
- : MTBUF_Real_AllAddr_gfx11_Renamed_Impl<op, ps.Mnemonic>;
-multiclass MTBUF_Real_AllAddr_gfx11<bits<4> op>
- : MTBUF_Real_AllAddr_gfx11_Impl<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>;
+  let AssemblerPredicate = isGFX12Plus, DecoderNamespace = "GFX12" in {
+    def _BOTHEN_gfx12 :
+      VBUFFER_MTBUF_Real_gfx12<op, !cast<MTBUF_Pseudo>(NAME#"_VBUFFER_BOTHEN"), real_name>;
+    def _IDXEN_gfx12 :
+      VBUFFER_MTBUF_Real_gfx12<op, !cast<MTBUF_Pseudo>(NAME#"_VBUFFER_IDXEN"), real_name>;
+    def _OFFEN_gfx12 :
+      VBUFFER_MTBUF_Real_gfx12<op, !cast<MTBUF_Pseudo>(NAME#"_VBUFFER_OFFEN"), real_name>;
+    def _OFFSET_gfx12 :
+      VBUFFER_MTBUF_Real_gfx12<op, !cast<MTBUF_Pseudo>(NAME#"_VBUFFER_OFFSET"), real_name>;
+  }
+}
+
+multiclass MTBUF_Real_AllAddr_gfx11_gfx12_Impl<bits<4> op, MTBUF_Pseudo ps>
+ : MTBUF_Real_AllAddr_gfx11_gfx12_Renamed_Impl<op, ps.Mnemonic>;
+multiclass MTBUF_Real_AllAddr_gfx11_gfx12<bits<4> op>
+ : MTBUF_Real_AllAddr_gfx11_gfx12_Impl<op, !cast<MTBUF_Pseudo>(NAME#"_BOTHEN")>;
 
 
 class Pre_gfx11_MTBUF_Name <MTBUF_Pseudo ps, string real_name>
   : MnemonicAlias<ps.Mnemonic, real_name>, Requires<[isGFX11Plus]>;
-multiclass MTBUF_Real_AllAddr_gfx11_Renamed<bits<4> op, string real_name>
-  : MTBUF_Real_AllAddr_gfx11_Renamed_Impl<op, real_name> {
+multiclass MTBUF_Real_AllAddr_gfx11_gfx12_Renamed<bits<4> op, string real_name>
+  : MTBUF_Real_AllAddr_gfx11_gfx12_Renamed_Impl<op, real_name> {
   def : Pre_gfx11_MTBUF_Name<!cast<MTBUF_Pseudo>(NAME#"_BOTHEN"), real_name>;
 }
 
-defm TBUFFER_LOAD_FORMAT_D16_X     : MTBUF_Real_AllAddr_gfx11_Renamed<0x008, "tbuffer_load_d16_format_x">;
-defm TBUFFER_LOAD_FORMAT_D16_XY    : MTBUF_Real_AllAddr_gfx11_Renamed<0x009, "tbuffer_load_d16_format_xy">;
-defm TBUFFER_LOAD_FORMAT_D16_XYZ   : MTBUF_Real_AllAddr_gfx11_Renamed<0x00a, "tbuffer_load_d16_format_xyz">;
-defm TBUFFER_LOAD_FORMAT_D16_XYZW  : MTBUF_Real_AllAddr_gfx11_Renamed<0x00b, "tbuffer_load_d16_format_xyzw">;
-defm TBUFFER_LOAD_FORMAT_X         : MTBUF_Real_AllAddr_gfx11<0x000>;
-defm TBUFFER_LOAD_FORMAT_XY        : MTBUF_Real_AllAddr_gfx11<0x001>;
-defm TBUFFER_LOAD_FORMAT_XYZ       : MTBUF_Real_AllAddr_gfx11<0x002>;
-defm TBUFFER_LOAD_FORMAT_XYZW      : MTBUF_Real_AllAddr_gfx11<0x003>;
-defm TBUFFER_STORE_FORMAT_D16_X    : MTBUF_Real_AllAddr_gfx11_Renamed<0x00c, "tbuffer_store_d16_format_x">;
-defm TBUFFER_STORE_FORMAT_D16_XY   : MTBUF_Real_AllAddr_gfx11_Renamed<0x00d, "tbuffer_store_d16_format_xy">;
-defm TBUFFER_STORE_FORMAT_D16_XYZ  : MTBUF_Real_AllAddr_gfx11_Renamed<0x00e, "tbuffer_store_d16_format_xyz">;
-defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx11_Renamed<0x00f, "tbuffer_store_d16_format_xyzw">;
-defm TBUFFER_STORE_FORMAT_X        : MTBUF_Real_AllAddr_gfx11<0x004>;
-defm TBUFFER_STORE_FORMAT_XY       : MTBUF_Real_AllAddr_gfx11<0x005>;
-defm TBUFFER_STORE_FORMAT_XYZ      : MTBUF_Real_AllAddr_gfx11<0x006>;
-defm TBUFFER_STORE_FORMAT_XYZW     : MTBUF_Real_AllAddr_gfx11<0x007>;
+defm TBUFFER_LOAD_FORMAT_D16_X     : MTBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x008, "tbuffer_load_d16_format_x">;
+defm TBUFFER_LOAD_FORMAT_D16_XY    : MTBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x009, "tbuffer_load_d16_format_xy">;
+defm TBUFFER_LOAD_FORMAT_D16_XYZ   : MTBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x00a, "tbuffer_load_d16_format_xyz">;
+defm TBUFFER_LOAD_FORMAT_D16_XYZW  : MTBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x00b, "tbuffer_load_d16_format_xyzw">;
+defm TBUFFER_LOAD_FORMAT_X         : MTBUF_Real_AllAddr_gfx11_gfx12<0x000>;
+defm TBUFFER_LOAD_FORMAT_XY        : MTBUF_Real_AllAddr_gfx11_gfx12<0x001>;
+defm TBUFFER_LOAD_FORMAT_XYZ       : MTBUF_Real_AllAddr_gfx11_gfx12<0x002>;
+defm TBUFFER_LOAD_FORMAT_XYZW      : MTBUF_Real_AllAddr_gfx11_gfx12<0x003>;
+defm TBUFFER_STORE_FORMAT_D16_X    : MTBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x00c, "tbuffer_store_d16_format_x">;
+defm TBUFFER_STORE_FORMAT_D16_XY   : MTBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x00d, "tbuffer_store_d16_format_xy">;
+defm TBUFFER_STORE_FORMAT_D16_XYZ  : MTBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x00e, "tbuffer_store_d16_format_xyz">;
+defm TBUFFER_STORE_FORMAT_D16_XYZW : MTBUF_Real_AllAddr_gfx11_gfx12_Renamed<0x00f, "tbuffer_store_d16_format_xyzw">;
+defm TBUFFER_STORE_FORMAT_X        : MTBUF_Real_AllAddr_gfx11_gfx12<0x004>;
+defm TBUFFER_STORE_FORMAT_XY       : MTBUF_Real_AllAddr_gfx11_gfx12<0x005>;
+defm TBUFFER_STORE_FORMAT_XYZ      : MTBUF_Real_AllAddr_gfx11_gfx12<0x006>;
+defm TBUFFER_STORE_FORMAT_XYZW     : MTBUF_Real_AllAddr_gfx11_gfx12<0x007>;
 
 //===----------------------------------------------------------------------===//
 // MTBUF - GFX10.
diff --git a/llvm/lib/Target/AMDGPU/CMakeLists.txt b/llvm/lib/Target/AMDGPU/CMakeLists.txt
index 53a33f8210d2a842f4782911dd52520b029b0dd0..2c92e7a07388553a1ca60bcb05bcaa300e0c4a20 100644
--- a/llvm/lib/Target/AMDGPU/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/CMakeLists.txt
@@ -55,6 +55,7 @@ add_llvm_target(AMDGPUCodeGen
   AMDGPUCtorDtorLowering.cpp
   AMDGPUExportClustering.cpp
   AMDGPUFrameLowering.cpp
+  AMDGPUGlobalISelDivergenceLowering.cpp
   AMDGPUGlobalISelUtils.cpp
   AMDGPUHSAMetadataStreamer.cpp
   AMDGPUInsertDelayAlu.cpp
diff --git a/llvm/lib/Target/AMDGPU/DSInstructions.td b/llvm/lib/Target/AMDGPU/DSInstructions.td
index 1a10a8fcaadca176c114ead0c30de85a6bec3b07..3a895923fa4b9871dc6105e3845471015f78a8a8 100644
--- a/llvm/lib/Target/AMDGPU/DSInstructions.td
+++ b/llvm/lib/Target/AMDGPU/DSInstructions.td
@@ -711,17 +711,33 @@ def DS_ADD_SRC2_F32 : DS_1A<"ds_add_src2_f32">;
 
 
 //===----------------------------------------------------------------------===//
-// Instruction definitions for GFX11 and newer.
+// Instruction definitions for GFX11.
 //===----------------------------------------------------------------------===//
 
-let SubtargetPredicate = isGFX11Plus in {
+let SubtargetPredicate = isGFX11Only in {
 
 def DS_ADD_GS_REG_RTN : DS_0A1D_RET_GDS<"ds_add_gs_reg_rtn", VReg_64, VGPR_32>;
 def DS_SUB_GS_REG_RTN : DS_0A1D_RET_GDS<"ds_sub_gs_reg_rtn", VReg_64, VGPR_32>;
+
+} // let SubtargetPredicate = isGFX11Only
+
+let SubtargetPredicate = isGFX11Plus in {
+
 def DS_BVH_STACK_RTN_B32 : DS_BVH_STACK<"ds_bvh_stack_rtn_b32">;
 
 } // let SubtargetPredicate = isGFX11Plus
 
+//===----------------------------------------------------------------------===//
+// Instruction definitions for GFX12 and newer.
+//===----------------------------------------------------------------------===//
+
+let SubtargetPredicate = isGFX12Plus in {
+
+defm DS_SUB_CLAMP_U32     : DS_1A1D_NORET_mc<"ds_sub_clamp_u32">;
+defm DS_SUB_CLAMP_RTN_U32 : DS_1A1D_RET_mc<"ds_sub_clamp_rtn_u32", VGPR_32, "ds_sub_clamp_u32">;
+
+} // let SubtargetPredicate = isGFX12Plus
+
 //===----------------------------------------------------------------------===//
 // DS Patterns
 //===----------------------------------------------------------------------===//
@@ -1169,11 +1185,12 @@ def : GCNPat <
 //===----------------------------------------------------------------------===//
 
 //===----------------------------------------------------------------------===//
-// Base ENC_DS for GFX6, GFX7, GFX10, GFX11.
+// Base ENC_DS for GFX6, GFX7, GFX10, GFX11, GFX12.
 //===----------------------------------------------------------------------===//
 
-class Base_DS_Real_gfx6_gfx7_gfx10_gfx11<bits<8> op, DS_Pseudo ps, int ef, string opName = ps.Mnemonic> :
-    DS_Real<ps, opName>, SIMCInstr <ps.Mnemonic, ef> {
+class Base_DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<bits<8> op, DS_Pseudo ps, int ef,
+                                               string opName = ps.Mnemonic>
+    : DS_Real<ps, opName>, SIMCInstr <ps.Mnemonic, ef> {
 
   let Inst{7-0}   = !if(ps.has_offset0, offset0, 0);
   let Inst{15-8}  = !if(ps.has_offset1, offset1, 0);
@@ -1186,75 +1203,118 @@ class Base_DS_Real_gfx6_gfx7_gfx10_gfx11<bits<8> op, DS_Pseudo ps, int ef, strin
   let Inst{63-56} = !if(ps.has_vdst, vdst{7-0}, 0);
 }
 
+//===----------------------------------------------------------------------===//
+// GFX12.
+//===----------------------------------------------------------------------===//
+
+let AssemblerPredicate = isGFX12Plus, DecoderNamespace = "GFX12" in {
+  multiclass DS_Real_gfx12<bits<8> op> {
+    defvar ps = !cast<DS_Pseudo>(NAME);
+    def _gfx12 :
+      Base_DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<op, ps, SIEncodingFamily.GFX12,
+                                               ps.Mnemonic>;
+  }
+
+  multiclass DS_Real_Renamed_gfx12<bits<8> op, DS_Pseudo backing_pseudo,
+                                   string real_name> {
+    def _gfx12 :
+      Base_DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<op, backing_pseudo,
+                                               SIEncodingFamily.GFX12,
+                                               real_name>,
+      MnemonicAlias<backing_pseudo.Mnemonic, real_name>,
+      Requires<[isGFX12Plus]>;
+  }
+} // End AssemblerPredicate = isGFX12Plus, DecoderNamespace = "GFX12"
+
+defm DS_MIN_NUM_F32       : DS_Real_Renamed_gfx12<0x012, DS_MIN_F32, "ds_min_num_f32">;
+defm DS_MAX_NUM_F32       : DS_Real_Renamed_gfx12<0x013, DS_MAX_F32, "ds_max_num_f32">;
+defm DS_MIN_NUM_RTN_F32   : DS_Real_Renamed_gfx12<0x032, DS_MIN_RTN_F32, "ds_min_num_rtn_f32">;
+defm DS_MAX_NUM_RTN_F32   : DS_Real_Renamed_gfx12<0x033, DS_MAX_RTN_F32, "ds_max_num_rtn_f32">;
+defm DS_MIN_NUM_F64       : DS_Real_Renamed_gfx12<0x052, DS_MIN_F64, "ds_min_num_f64">;
+defm DS_MAX_NUM_F64       : DS_Real_Renamed_gfx12<0x053, DS_MAX_F64, "ds_max_num_f64">;
+defm DS_MIN_NUM_RTN_F64   : DS_Real_Renamed_gfx12<0x072, DS_MIN_RTN_F64, "ds_min_num_rtn_f64">;
+defm DS_MAX_NUM_RTN_F64   : DS_Real_Renamed_gfx12<0x073, DS_MAX_RTN_F64, "ds_max_num_rtn_f64">;
+defm DS_SUB_CLAMP_U32     : DS_Real_gfx12<0x099>;
+defm DS_SUB_CLAMP_RTN_U32 : DS_Real_gfx12<0x0a9>;
+
 //===----------------------------------------------------------------------===//
 // GFX11.
 //===----------------------------------------------------------------------===//
 
-let AssemblerPredicate = isGFX11Plus, DecoderNamespace = "GFX11" in {
+let AssemblerPredicate = isGFX11Only, DecoderNamespace = "GFX11" in {
   multiclass DS_Real_gfx11<bits<8> op>  {
-    def _gfx11 : Base_DS_Real_gfx6_gfx7_gfx10_gfx11<op, !cast<DS_Pseudo>(NAME),
+    def _gfx11 :
+      Base_DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<op, !cast<DS_Pseudo>(NAME),
                                               SIEncodingFamily.GFX11>;
   }
 
   multiclass DS_Real_Renamed_gfx11<bits<8> op, DS_Pseudo backing_pseudo, string real_name> {
-     def _gfx11 : Base_DS_Real_gfx6_gfx7_gfx10_gfx11<op, backing_pseudo, SIEncodingFamily.GFX11, real_name>,
-               MnemonicAlias<backing_pseudo.Mnemonic, real_name>, Requires<[isGFX11Plus]>;
+     def _gfx11 : Base_DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<op, backing_pseudo, SIEncodingFamily.GFX11, real_name>,
+               MnemonicAlias<backing_pseudo.Mnemonic, real_name>, Requires<[isGFX11Only]>;
   }
-} // End AssemblerPredicate = isGFX11Plus, DecoderNamespace = "GFX11"
-
-defm DS_STORE_B32                        : DS_Real_Renamed_gfx11<0x00d, DS_WRITE_B32, "ds_store_b32">;
-defm DS_STORE_2ADDR_B32                  : DS_Real_Renamed_gfx11<0x00e, DS_WRITE2_B32, "ds_store_2addr_b32">;
-defm DS_STORE_2ADDR_STRIDE64_B32         : DS_Real_Renamed_gfx11<0x00f, DS_WRITE2ST64_B32, "ds_store_2addr_stride64_b32">;
-defm DS_STORE_B8                         : DS_Real_Renamed_gfx11<0x01e, DS_WRITE_B8, "ds_store_b8">;
-defm DS_STORE_B16                        : DS_Real_Renamed_gfx11<0x01f, DS_WRITE_B16, "ds_store_b16">;
-defm DS_STOREXCHG_RTN_B32                : DS_Real_Renamed_gfx11<0x02d, DS_WRXCHG_RTN_B32, "ds_storexchg_rtn_b32">;
-defm DS_STOREXCHG_2ADDR_RTN_B32          : DS_Real_Renamed_gfx11<0x02e, DS_WRXCHG2_RTN_B32, "ds_storexchg_2addr_rtn_b32">;
-defm DS_STOREXCHG_2ADDR_STRIDE64_RTN_B32 : DS_Real_Renamed_gfx11<0x02f, DS_WRXCHG2ST64_RTN_B32, "ds_storexchg_2addr_stride64_rtn_b32">;
-defm DS_LOAD_B32                         : DS_Real_Renamed_gfx11<0x036, DS_READ_B32, "ds_load_b32">;
-defm DS_LOAD_2ADDR_B32                   : DS_Real_Renamed_gfx11<0x037, DS_READ2_B32, "ds_load_2addr_b32">;
-defm DS_LOAD_2ADDR_STRIDE64_B32          : DS_Real_Renamed_gfx11<0x038, DS_READ2ST64_B32, "ds_load_2addr_stride64_b32">;
-defm DS_LOAD_I8                          : DS_Real_Renamed_gfx11<0x039, DS_READ_I8, "ds_load_i8">;
-defm DS_LOAD_U8                          : DS_Real_Renamed_gfx11<0x03a, DS_READ_U8, "ds_load_u8">;
-defm DS_LOAD_I16                         : DS_Real_Renamed_gfx11<0x03b, DS_READ_I16, "ds_load_i16">;
-defm DS_LOAD_U16                         : DS_Real_Renamed_gfx11<0x03c, DS_READ_U16, "ds_load_u16">;
-defm DS_STORE_B64                        : DS_Real_Renamed_gfx11<0x04d, DS_WRITE_B64, "ds_store_b64">;
-defm DS_STORE_2ADDR_B64                  : DS_Real_Renamed_gfx11<0x04e, DS_WRITE2_B64, "ds_store_2addr_b64">;
-defm DS_STORE_2ADDR_STRIDE64_B64         : DS_Real_Renamed_gfx11<0x04f, DS_WRITE2ST64_B64, "ds_store_2addr_stride64_b64">;
-defm DS_STOREXCHG_RTN_B64                : DS_Real_Renamed_gfx11<0x06d, DS_WRXCHG_RTN_B64, "ds_storexchg_rtn_b64">;
-defm DS_STOREXCHG_2ADDR_RTN_B64          : DS_Real_Renamed_gfx11<0x06e, DS_WRXCHG2_RTN_B64, "ds_storexchg_2addr_rtn_b64">;
-defm DS_STOREXCHG_2ADDR_STRIDE64_RTN_B64 : DS_Real_Renamed_gfx11<0x06f, DS_WRXCHG2ST64_RTN_B64, "ds_storexchg_2addr_stride64_rtn_b64">;
-defm DS_LOAD_B64                         : DS_Real_Renamed_gfx11<0x076, DS_READ_B64, "ds_load_b64">;
-defm DS_LOAD_2ADDR_B64                   : DS_Real_Renamed_gfx11<0x077, DS_READ2_B64, "ds_load_2addr_b64">;
-defm DS_LOAD_2ADDR_STRIDE64_B64          : DS_Real_Renamed_gfx11<0x078, DS_READ2ST64_B64, "ds_load_2addr_stride64_b64">;
-defm DS_STORE_B8_D16_HI                  : DS_Real_Renamed_gfx11<0x0a0, DS_WRITE_B8_D16_HI, "ds_store_b8_d16_hi">;
-defm DS_STORE_B16_D16_HI                 : DS_Real_Renamed_gfx11<0x0a1, DS_WRITE_B16_D16_HI, "ds_store_b16_d16_hi">;
-defm DS_LOAD_U8_D16                      : DS_Real_Renamed_gfx11<0x0a2, DS_READ_U8_D16, "ds_load_u8_d16">;
-defm DS_LOAD_U8_D16_HI                   : DS_Real_Renamed_gfx11<0x0a3, DS_READ_U8_D16_HI, "ds_load_u8_d16_hi">;
-defm DS_LOAD_I8_D16                      : DS_Real_Renamed_gfx11<0x0a4, DS_READ_I8_D16, "ds_load_i8_d16">;
-defm DS_LOAD_I8_D16_HI                   : DS_Real_Renamed_gfx11<0x0a5, DS_READ_I8_D16_HI, "ds_load_i8_d16_hi">;
-defm DS_LOAD_U16_D16                     : DS_Real_Renamed_gfx11<0x0a6, DS_READ_U16_D16, "ds_load_u16_d16">;
-defm DS_LOAD_U16_D16_HI                  : DS_Real_Renamed_gfx11<0x0a7, DS_READ_U16_D16_HI, "ds_load_u16_d16_hi">;
-defm DS_STORE_ADDTID_B32                 : DS_Real_Renamed_gfx11<0x0b0, DS_WRITE_ADDTID_B32, "ds_store_addtid_b32">;
-defm DS_LOAD_ADDTID_B32                  : DS_Real_Renamed_gfx11<0x0b1, DS_READ_ADDTID_B32, "ds_load_addtid_b32">;
-defm DS_STORE_B96                        : DS_Real_Renamed_gfx11<0x0de, DS_WRITE_B96, "ds_store_b96">;
-defm DS_STORE_B128                       : DS_Real_Renamed_gfx11<0x0df, DS_WRITE_B128, "ds_store_b128">;
-defm DS_LOAD_B96                         : DS_Real_Renamed_gfx11<0x0fe, DS_READ_B96, "ds_load_b96">;
-defm DS_LOAD_B128                        : DS_Real_Renamed_gfx11<0x0ff, DS_READ_B128, "ds_load_b128">;
+} // End AssemblerPredicate = isGFX11Only, DecoderNamespace = "GFX11"
+
+multiclass DS_Real_gfx11_gfx12<bits<8> op>
+  : DS_Real_gfx11<op>, DS_Real_gfx12<op>;
+
+multiclass DS_Real_Renamed_gfx11_gfx12<bits<8> op, DS_Pseudo backing_pseudo,
+                                       string real_name>
+  : DS_Real_Renamed_gfx11<op, backing_pseudo, real_name>,
+    DS_Real_Renamed_gfx12<op, backing_pseudo, real_name>;
+
+defm DS_STORE_B32                        : DS_Real_Renamed_gfx11_gfx12<0x00d, DS_WRITE_B32, "ds_store_b32">;
+defm DS_STORE_2ADDR_B32                  : DS_Real_Renamed_gfx11_gfx12<0x00e, DS_WRITE2_B32, "ds_store_2addr_b32">;
+defm DS_STORE_2ADDR_STRIDE64_B32         : DS_Real_Renamed_gfx11_gfx12<0x00f, DS_WRITE2ST64_B32, "ds_store_2addr_stride64_b32">;
+defm DS_STORE_B8                         : DS_Real_Renamed_gfx11_gfx12<0x01e, DS_WRITE_B8, "ds_store_b8">;
+defm DS_STORE_B16                        : DS_Real_Renamed_gfx11_gfx12<0x01f, DS_WRITE_B16, "ds_store_b16">;
+defm DS_STOREXCHG_RTN_B32                : DS_Real_Renamed_gfx11_gfx12<0x02d, DS_WRXCHG_RTN_B32, "ds_storexchg_rtn_b32">;
+defm DS_STOREXCHG_2ADDR_RTN_B32          : DS_Real_Renamed_gfx11_gfx12<0x02e, DS_WRXCHG2_RTN_B32, "ds_storexchg_2addr_rtn_b32">;
+defm DS_STOREXCHG_2ADDR_STRIDE64_RTN_B32 : DS_Real_Renamed_gfx11_gfx12<0x02f, DS_WRXCHG2ST64_RTN_B32, "ds_storexchg_2addr_stride64_rtn_b32">;
+defm DS_LOAD_B32                         : DS_Real_Renamed_gfx11_gfx12<0x036, DS_READ_B32, "ds_load_b32">;
+defm DS_LOAD_2ADDR_B32                   : DS_Real_Renamed_gfx11_gfx12<0x037, DS_READ2_B32, "ds_load_2addr_b32">;
+defm DS_LOAD_2ADDR_STRIDE64_B32          : DS_Real_Renamed_gfx11_gfx12<0x038, DS_READ2ST64_B32, "ds_load_2addr_stride64_b32">;
+defm DS_LOAD_I8                          : DS_Real_Renamed_gfx11_gfx12<0x039, DS_READ_I8, "ds_load_i8">;
+defm DS_LOAD_U8                          : DS_Real_Renamed_gfx11_gfx12<0x03a, DS_READ_U8, "ds_load_u8">;
+defm DS_LOAD_I16                         : DS_Real_Renamed_gfx11_gfx12<0x03b, DS_READ_I16, "ds_load_i16">;
+defm DS_LOAD_U16                         : DS_Real_Renamed_gfx11_gfx12<0x03c, DS_READ_U16, "ds_load_u16">;
+defm DS_STORE_B64                        : DS_Real_Renamed_gfx11_gfx12<0x04d, DS_WRITE_B64, "ds_store_b64">;
+defm DS_STORE_2ADDR_B64                  : DS_Real_Renamed_gfx11_gfx12<0x04e, DS_WRITE2_B64, "ds_store_2addr_b64">;
+defm DS_STORE_2ADDR_STRIDE64_B64         : DS_Real_Renamed_gfx11_gfx12<0x04f, DS_WRITE2ST64_B64, "ds_store_2addr_stride64_b64">;
+defm DS_STOREXCHG_RTN_B64                : DS_Real_Renamed_gfx11_gfx12<0x06d, DS_WRXCHG_RTN_B64, "ds_storexchg_rtn_b64">;
+defm DS_STOREXCHG_2ADDR_RTN_B64          : DS_Real_Renamed_gfx11_gfx12<0x06e, DS_WRXCHG2_RTN_B64, "ds_storexchg_2addr_rtn_b64">;
+defm DS_STOREXCHG_2ADDR_STRIDE64_RTN_B64 : DS_Real_Renamed_gfx11_gfx12<0x06f, DS_WRXCHG2ST64_RTN_B64, "ds_storexchg_2addr_stride64_rtn_b64">;
+defm DS_LOAD_B64                         : DS_Real_Renamed_gfx11_gfx12<0x076, DS_READ_B64, "ds_load_b64">;
+defm DS_LOAD_2ADDR_B64                   : DS_Real_Renamed_gfx11_gfx12<0x077, DS_READ2_B64, "ds_load_2addr_b64">;
+defm DS_LOAD_2ADDR_STRIDE64_B64          : DS_Real_Renamed_gfx11_gfx12<0x078, DS_READ2ST64_B64, "ds_load_2addr_stride64_b64">;
+defm DS_STORE_B8_D16_HI                  : DS_Real_Renamed_gfx11_gfx12<0x0a0, DS_WRITE_B8_D16_HI, "ds_store_b8_d16_hi">;
+defm DS_STORE_B16_D16_HI                 : DS_Real_Renamed_gfx11_gfx12<0x0a1, DS_WRITE_B16_D16_HI, "ds_store_b16_d16_hi">;
+defm DS_LOAD_U8_D16                      : DS_Real_Renamed_gfx11_gfx12<0x0a2, DS_READ_U8_D16, "ds_load_u8_d16">;
+defm DS_LOAD_U8_D16_HI                   : DS_Real_Renamed_gfx11_gfx12<0x0a3, DS_READ_U8_D16_HI, "ds_load_u8_d16_hi">;
+defm DS_LOAD_I8_D16                      : DS_Real_Renamed_gfx11_gfx12<0x0a4, DS_READ_I8_D16, "ds_load_i8_d16">;
+defm DS_LOAD_I8_D16_HI                   : DS_Real_Renamed_gfx11_gfx12<0x0a5, DS_READ_I8_D16_HI, "ds_load_i8_d16_hi">;
+defm DS_LOAD_U16_D16                     : DS_Real_Renamed_gfx11_gfx12<0x0a6, DS_READ_U16_D16, "ds_load_u16_d16">;
+defm DS_LOAD_U16_D16_HI                  : DS_Real_Renamed_gfx11_gfx12<0x0a7, DS_READ_U16_D16_HI, "ds_load_u16_d16_hi">;
+defm DS_STORE_ADDTID_B32                 : DS_Real_Renamed_gfx11_gfx12<0x0b0, DS_WRITE_ADDTID_B32, "ds_store_addtid_b32">;
+defm DS_LOAD_ADDTID_B32                  : DS_Real_Renamed_gfx11_gfx12<0x0b1, DS_READ_ADDTID_B32, "ds_load_addtid_b32">;
+defm DS_STORE_B96                        : DS_Real_Renamed_gfx11_gfx12<0x0de, DS_WRITE_B96, "ds_store_b96">;
+defm DS_STORE_B128                       : DS_Real_Renamed_gfx11_gfx12<0x0df, DS_WRITE_B128, "ds_store_b128">;
+defm DS_LOAD_B96                         : DS_Real_Renamed_gfx11_gfx12<0x0fe, DS_READ_B96, "ds_load_b96">;
+defm DS_LOAD_B128                        : DS_Real_Renamed_gfx11_gfx12<0x0ff, DS_READ_B128, "ds_load_b128">;
 
 // DS_CMPST_* are renamed to DS_CMPSTORE_* in GFX11, but also the data operands (src and cmp) are swapped
 // comparing to pre-GFX11.
 // Note: the mnemonic alias is not generated to avoid a potential ambiguity due to the semantics change.
 
-defm DS_CMPSTORE_B32                     : DS_Real_gfx11<0x010>;
+defm DS_CMPSTORE_B32                     : DS_Real_gfx11_gfx12<0x010>;
 defm DS_CMPSTORE_F32                     : DS_Real_gfx11<0x011>;
-defm DS_CMPSTORE_RTN_B32                 : DS_Real_gfx11<0x030>;
+defm DS_CMPSTORE_RTN_B32                 : DS_Real_gfx11_gfx12<0x030>;
 defm DS_CMPSTORE_RTN_F32                 : DS_Real_gfx11<0x031>;
-defm DS_CMPSTORE_B64                     : DS_Real_gfx11<0x050>;
+defm DS_CMPSTORE_B64                     : DS_Real_gfx11_gfx12<0x050>;
 defm DS_CMPSTORE_F64                     : DS_Real_gfx11<0x051>;
-defm DS_CMPSTORE_RTN_B64                 : DS_Real_gfx11<0x070>;
+defm DS_CMPSTORE_RTN_B64                 : DS_Real_gfx11_gfx12<0x070>;
 defm DS_CMPSTORE_RTN_F64                 : DS_Real_gfx11<0x071>;
 
-defm DS_ADD_RTN_F32                      : DS_Real_gfx11<0x079>;
+defm DS_ADD_RTN_F32                      : DS_Real_gfx11_gfx12<0x079>;
 defm DS_ADD_GS_REG_RTN                   : DS_Real_gfx11<0x07a>;
 defm DS_SUB_GS_REG_RTN                   : DS_Real_gfx11<0x07b>;
 defm DS_BVH_STACK_RTN_B32                : DS_Real_gfx11<0x0ad>;
@@ -1265,8 +1325,8 @@ defm DS_BVH_STACK_RTN_B32                : DS_Real_gfx11<0x0ad>;
 
 let AssemblerPredicate = isGFX10Only, DecoderNamespace = "GFX10" in {
   multiclass DS_Real_gfx10<bits<8> op>  {
-    def _gfx10 : Base_DS_Real_gfx6_gfx7_gfx10_gfx11<op, !cast<DS_Pseudo>(NAME),
-                                              SIEncodingFamily.GFX10>;
+    def _gfx10 : Base_DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<op,
+      !cast<DS_Pseudo>(NAME), SIEncodingFamily.GFX10>;
   }
 } // End AssemblerPredicate = isGFX10Only, DecoderNamespace = "GFX10"
 
@@ -1283,28 +1343,34 @@ defm DS_WRITE_ADDTID_B32 : DS_Real_gfx10<0x0b0>;
 defm DS_READ_ADDTID_B32  : DS_Real_gfx10<0x0b1>;
 
 //===----------------------------------------------------------------------===//
-// GFX10, GFX11.
+// GFX10, GFX11, GFX12.
 //===----------------------------------------------------------------------===//
 
+multiclass DS_Real_gfx10_gfx11_gfx12<bits<8> op> :
+  DS_Real_gfx10<op>, DS_Real_gfx11<op>, DS_Real_gfx12<op>;
+
 multiclass DS_Real_gfx10_gfx11<bits<8> op> :
   DS_Real_gfx10<op>, DS_Real_gfx11<op>;
 
-defm DS_ADD_F32          : DS_Real_gfx10_gfx11<0x015>;
+defm DS_ADD_F32          : DS_Real_gfx10_gfx11_gfx12<0x015>;
 defm DS_ADD_SRC2_F32     : DS_Real_gfx10<0x095>;
-defm DS_PERMUTE_B32      : DS_Real_gfx10_gfx11<0x0b2>;
-defm DS_BPERMUTE_B32     : DS_Real_gfx10_gfx11<0x0b3>;
+defm DS_PERMUTE_B32      : DS_Real_gfx10_gfx11_gfx12<0x0b2>;
+defm DS_BPERMUTE_B32     : DS_Real_gfx10_gfx11_gfx12<0x0b3>;
 
 //===----------------------------------------------------------------------===//
-// GFX7, GFX10, GFX11.
+// GFX7, GFX10, GFX11, GFX12.
 //===----------------------------------------------------------------------===//
 
 let AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7" in {
   multiclass DS_Real_gfx7<bits<8> op> {
-    def _gfx7 : Base_DS_Real_gfx6_gfx7_gfx10_gfx11<op, !cast<DS_Pseudo>(NAME),
-                                             SIEncodingFamily.SI>;
+    def _gfx7 : Base_DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<op,
+      !cast<DS_Pseudo>(NAME), SIEncodingFamily.SI>;
   }
 } // End AssemblerPredicate = isGFX7Only, DecoderNamespace = "GFX7"
 
+multiclass DS_Real_gfx7_gfx10_gfx11_gfx12<bits<8> op> :
+  DS_Real_gfx7<op>, DS_Real_gfx10_gfx11_gfx12<op>;
+
 multiclass DS_Real_gfx7_gfx10_gfx11<bits<8> op> :
   DS_Real_gfx7<op>, DS_Real_gfx10_gfx11<op>;
 
@@ -1314,7 +1380,7 @@ multiclass DS_Real_gfx7_gfx10<bits<8> op> :
 // FIXME-GFX7: Add tests when upstreaming this part.
 defm DS_GWS_SEMA_RELEASE_ALL : DS_Real_gfx7_gfx10_gfx11<0x018>;
 defm DS_WRAP_RTN_B32         : DS_Real_gfx7_gfx10_gfx11<0x034>;
-defm DS_CONDXCHG32_RTN_B64   : DS_Real_gfx7_gfx10_gfx11<0x07e>;
+defm DS_CONDXCHG32_RTN_B64   : DS_Real_gfx7_gfx10_gfx11_gfx12<0x07e>;
 defm DS_WRITE_B96            : DS_Real_gfx7_gfx10<0x0de>;
 defm DS_WRITE_B128           : DS_Real_gfx7_gfx10<0x0df>;
 defm DS_READ_B96             : DS_Real_gfx7_gfx10<0x0fe>;
@@ -1326,30 +1392,33 @@ defm DS_READ_B128            : DS_Real_gfx7_gfx10<0x0ff>;
 
 let AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7" in {
   multiclass DS_Real_gfx6_gfx7<bits<8> op> {
-    def _gfx6_gfx7 : Base_DS_Real_gfx6_gfx7_gfx10_gfx11<op, !cast<DS_Pseudo>(NAME),
-                                                  SIEncodingFamily.SI>;
+    def _gfx6_gfx7 : Base_DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<op,
+      !cast<DS_Pseudo>(NAME), SIEncodingFamily.SI>;
   }
 } // End AssemblerPredicate = isGFX6GFX7, DecoderNamespace = "GFX6GFX7"
 
+multiclass DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<bits<8> op> :
+  DS_Real_gfx6_gfx7<op>, DS_Real_gfx10_gfx11_gfx12<op>;
+
 multiclass DS_Real_gfx6_gfx7_gfx10_gfx11<bits<8> op> :
   DS_Real_gfx6_gfx7<op>, DS_Real_gfx10_gfx11<op>;
 
 multiclass DS_Real_gfx6_gfx7_gfx10<bits<8> op> :
   DS_Real_gfx6_gfx7<op>, DS_Real_gfx10<op>;
 
-defm DS_ADD_U32             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x000>;
-defm DS_SUB_U32             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x001>;
-defm DS_RSUB_U32            : DS_Real_gfx6_gfx7_gfx10_gfx11<0x002>;
-defm DS_INC_U32             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x003>;
-defm DS_DEC_U32             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x004>;
-defm DS_MIN_I32             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x005>;
-defm DS_MAX_I32             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x006>;
-defm DS_MIN_U32             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x007>;
-defm DS_MAX_U32             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x008>;
-defm DS_AND_B32             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x009>;
-defm DS_OR_B32              : DS_Real_gfx6_gfx7_gfx10_gfx11<0x00a>;
-defm DS_XOR_B32             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x00b>;
-defm DS_MSKOR_B32           : DS_Real_gfx6_gfx7_gfx10_gfx11<0x00c>;
+defm DS_ADD_U32             : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x000>;
+defm DS_SUB_U32             : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x001>;
+defm DS_RSUB_U32            : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x002>;
+defm DS_INC_U32             : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x003>;
+defm DS_DEC_U32             : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x004>;
+defm DS_MIN_I32             : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x005>;
+defm DS_MAX_I32             : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x006>;
+defm DS_MIN_U32             : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x007>;
+defm DS_MAX_U32             : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x008>;
+defm DS_AND_B32             : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x009>;
+defm DS_OR_B32              : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x00a>;
+defm DS_XOR_B32             : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x00b>;
+defm DS_MSKOR_B32           : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x00c>;
 
 defm DS_WRITE_B32           : DS_Real_gfx6_gfx7_gfx10<0x00d>;
 defm DS_WRITE2_B32          : DS_Real_gfx6_gfx7_gfx10<0x00e>;
@@ -1359,7 +1428,7 @@ defm DS_CMPST_F32           : DS_Real_gfx6_gfx7_gfx10<0x011>;
 
 defm DS_MIN_F32             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x012>;
 defm DS_MAX_F32             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x013>;
-defm DS_NOP                 : DS_Real_gfx6_gfx7_gfx10_gfx11<0x014>;
+defm DS_NOP                 : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x014>;
 defm DS_GWS_INIT            : DS_Real_gfx6_gfx7_gfx10_gfx11<0x019>;
 defm DS_GWS_SEMA_V          : DS_Real_gfx6_gfx7_gfx10_gfx11<0x01a>;
 defm DS_GWS_SEMA_BR         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x01b>;
@@ -1369,19 +1438,19 @@ defm DS_GWS_BARRIER         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x01d>;
 defm DS_WRITE_B8            : DS_Real_gfx6_gfx7_gfx10<0x01e>;
 defm DS_WRITE_B16           : DS_Real_gfx6_gfx7_gfx10<0x01f>;
 
-defm DS_ADD_RTN_U32         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x020>;
-defm DS_SUB_RTN_U32         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x021>;
-defm DS_RSUB_RTN_U32        : DS_Real_gfx6_gfx7_gfx10_gfx11<0x022>;
-defm DS_INC_RTN_U32         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x023>;
-defm DS_DEC_RTN_U32         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x024>;
-defm DS_MIN_RTN_I32         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x025>;
-defm DS_MAX_RTN_I32         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x026>;
-defm DS_MIN_RTN_U32         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x027>;
-defm DS_MAX_RTN_U32         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x028>;
-defm DS_AND_RTN_B32         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x029>;
-defm DS_OR_RTN_B32          : DS_Real_gfx6_gfx7_gfx10_gfx11<0x02a>;
-defm DS_XOR_RTN_B32         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x02b>;
-defm DS_MSKOR_RTN_B32       : DS_Real_gfx6_gfx7_gfx10_gfx11<0x02c>;
+defm DS_ADD_RTN_U32         : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x020>;
+defm DS_SUB_RTN_U32         : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x021>;
+defm DS_RSUB_RTN_U32        : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x022>;
+defm DS_INC_RTN_U32         : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x023>;
+defm DS_DEC_RTN_U32         : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x024>;
+defm DS_MIN_RTN_I32         : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x025>;
+defm DS_MAX_RTN_I32         : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x026>;
+defm DS_MIN_RTN_U32         : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x027>;
+defm DS_MAX_RTN_U32         : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x028>;
+defm DS_AND_RTN_B32         : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x029>;
+defm DS_OR_RTN_B32          : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x02a>;
+defm DS_XOR_RTN_B32         : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x02b>;
+defm DS_MSKOR_RTN_B32       : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x02c>;
 
 defm DS_WRXCHG_RTN_B32      : DS_Real_gfx6_gfx7_gfx10<0x02d>;
 defm DS_WRXCHG2_RTN_B32     : DS_Real_gfx6_gfx7_gfx10<0x02e>;
@@ -1391,7 +1460,7 @@ defm DS_CMPST_RTN_F32       : DS_Real_gfx6_gfx7_gfx10<0x031>;
 
 defm DS_MIN_RTN_F32         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x032>;
 defm DS_MAX_RTN_F32         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x033>;
-defm DS_SWIZZLE_B32         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x035>;
+defm DS_SWIZZLE_B32         : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x035>;
 
 defm DS_READ_B32            : DS_Real_gfx6_gfx7_gfx10<0x036>;
 defm DS_READ2_B32           : DS_Real_gfx6_gfx7_gfx10<0x037>;
@@ -1401,22 +1470,22 @@ defm DS_READ_U8             : DS_Real_gfx6_gfx7_gfx10<0x03a>;
 defm DS_READ_I16            : DS_Real_gfx6_gfx7_gfx10<0x03b>;
 defm DS_READ_U16            : DS_Real_gfx6_gfx7_gfx10<0x03c>;
 
-defm DS_CONSUME             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x03d>;
-defm DS_APPEND              : DS_Real_gfx6_gfx7_gfx10_gfx11<0x03e>;
+defm DS_CONSUME             : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x03d>;
+defm DS_APPEND              : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x03e>;
 defm DS_ORDERED_COUNT       : DS_Real_gfx6_gfx7_gfx10_gfx11<0x03f>;
-defm DS_ADD_U64             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x040>;
-defm DS_SUB_U64             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x041>;
-defm DS_RSUB_U64            : DS_Real_gfx6_gfx7_gfx10_gfx11<0x042>;
-defm DS_INC_U64             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x043>;
-defm DS_DEC_U64             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x044>;
-defm DS_MIN_I64             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x045>;
-defm DS_MAX_I64             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x046>;
-defm DS_MIN_U64             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x047>;
-defm DS_MAX_U64             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x048>;
-defm DS_AND_B64             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x049>;
-defm DS_OR_B64              : DS_Real_gfx6_gfx7_gfx10_gfx11<0x04a>;
-defm DS_XOR_B64             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x04b>;
-defm DS_MSKOR_B64           : DS_Real_gfx6_gfx7_gfx10_gfx11<0x04c>;
+defm DS_ADD_U64             : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x040>;
+defm DS_SUB_U64             : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x041>;
+defm DS_RSUB_U64            : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x042>;
+defm DS_INC_U64             : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x043>;
+defm DS_DEC_U64             : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x044>;
+defm DS_MIN_I64             : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x045>;
+defm DS_MAX_I64             : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x046>;
+defm DS_MIN_U64             : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x047>;
+defm DS_MAX_U64             : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x048>;
+defm DS_AND_B64             : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x049>;
+defm DS_OR_B64              : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x04a>;
+defm DS_XOR_B64             : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x04b>;
+defm DS_MSKOR_B64           : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x04c>;
 
 defm DS_WRITE_B64           : DS_Real_gfx6_gfx7_gfx10<0x04d>;
 defm DS_WRITE2_B64          : DS_Real_gfx6_gfx7_gfx10<0x04e>;
@@ -1426,19 +1495,19 @@ defm DS_CMPST_F64           : DS_Real_gfx6_gfx7_gfx10<0x051>;
 
 defm DS_MIN_F64             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x052>;
 defm DS_MAX_F64             : DS_Real_gfx6_gfx7_gfx10_gfx11<0x053>;
-defm DS_ADD_RTN_U64         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x060>;
-defm DS_SUB_RTN_U64         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x061>;
-defm DS_RSUB_RTN_U64        : DS_Real_gfx6_gfx7_gfx10_gfx11<0x062>;
-defm DS_INC_RTN_U64         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x063>;
-defm DS_DEC_RTN_U64         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x064>;
-defm DS_MIN_RTN_I64         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x065>;
-defm DS_MAX_RTN_I64         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x066>;
-defm DS_MIN_RTN_U64         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x067>;
-defm DS_MAX_RTN_U64         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x068>;
-defm DS_AND_RTN_B64         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x069>;
-defm DS_OR_RTN_B64          : DS_Real_gfx6_gfx7_gfx10_gfx11<0x06a>;
-defm DS_XOR_RTN_B64         : DS_Real_gfx6_gfx7_gfx10_gfx11<0x06b>;
-defm DS_MSKOR_RTN_B64       : DS_Real_gfx6_gfx7_gfx10_gfx11<0x06c>;
+defm DS_ADD_RTN_U64         : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x060>;
+defm DS_SUB_RTN_U64         : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x061>;
+defm DS_RSUB_RTN_U64        : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x062>;
+defm DS_INC_RTN_U64         : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x063>;
+defm DS_DEC_RTN_U64         : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x064>;
+defm DS_MIN_RTN_I64         : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x065>;
+defm DS_MAX_RTN_I64         : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x066>;
+defm DS_MIN_RTN_U64         : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x067>;
+defm DS_MAX_RTN_U64         : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x068>;
+defm DS_AND_RTN_B64         : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x069>;
+defm DS_OR_RTN_B64          : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x06a>;
+defm DS_XOR_RTN_B64         : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x06b>;
+defm DS_MSKOR_RTN_B64       : DS_Real_gfx6_gfx7_gfx10_gfx11_gfx12<0x06c>;
 
 defm DS_WRXCHG_RTN_B64      : DS_Real_gfx6_gfx7_gfx10<0x06d>;
 defm DS_WRXCHG2_RTN_B64     : DS_Real_gfx6_gfx7_gfx10<0x06e>;
diff --git a/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp b/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp
index ed019d26c1dfd81162414aa1993e45d70b80f71a..1cfc5af571c1f9d9c7fe2dd8ce83ec14e4c31aca 100644
--- a/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp
+++ b/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp
@@ -107,6 +107,13 @@ static DecodeStatus decodeBoolReg(MCInst &Inst, unsigned Val, uint64_t Addr,
   return addOperand(Inst, DAsm->decodeBoolReg(Val));
 }
 
+static DecodeStatus decodeSplitBarrier(MCInst &Inst, unsigned Val,
+                                       uint64_t Addr,
+                                       const MCDisassembler *Decoder) {
+  auto DAsm = static_cast<const AMDGPUDisassembler *>(Decoder);
+  return addOperand(Inst, DAsm->decodeSplitBarrier(Val));
+}
+
 #define DECODE_OPERAND(StaticDecoderName, DecoderName)                         \
   static DecodeStatus StaticDecoderName(MCInst &Inst, unsigned Imm,            \
                                         uint64_t /*Addr*/,                     \
@@ -1747,6 +1754,10 @@ MCOperand AMDGPUDisassembler::decodeBoolReg(unsigned Val) const {
              : decodeSrcOp(OPW32, Val);
 }
 
+MCOperand AMDGPUDisassembler::decodeSplitBarrier(unsigned Val) const {
+  return decodeSrcOp(OPW32, Val);
+}
+
 bool AMDGPUDisassembler::isVI() const {
   return STI.hasFeature(AMDGPU::FeatureVolcanicIslands);
 }
@@ -1868,12 +1879,16 @@ MCDisassembler::DecodeStatus AMDGPUDisassembler::decodeCOMPUTE_PGM_RSRC1(
   if (FourByteBuffer & COMPUTE_PGM_RSRC1_PRIV)
     return MCDisassembler::Fail;
 
-  PRINT_DIRECTIVE(".amdhsa_dx10_clamp", COMPUTE_PGM_RSRC1_ENABLE_DX10_CLAMP);
+  if (!isGFX12Plus())
+    PRINT_DIRECTIVE(".amdhsa_dx10_clamp",
+                    COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_DX10_CLAMP);
 
   if (FourByteBuffer & COMPUTE_PGM_RSRC1_DEBUG_MODE)
     return MCDisassembler::Fail;
 
-  PRINT_DIRECTIVE(".amdhsa_ieee_mode", COMPUTE_PGM_RSRC1_ENABLE_IEEE_MODE);
+  if (!isGFX12Plus())
+    PRINT_DIRECTIVE(".amdhsa_ieee_mode",
+                    COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_IEEE_MODE);
 
   if (FourByteBuffer & COMPUTE_PGM_RSRC1_BULKY)
     return MCDisassembler::Fail;
@@ -1899,6 +1914,11 @@ MCDisassembler::DecodeStatus AMDGPUDisassembler::decodeCOMPUTE_PGM_RSRC1(
     PRINT_DIRECTIVE(".amdhsa_memory_ordered", COMPUTE_PGM_RSRC1_GFX10_PLUS_MEM_ORDERED);
     PRINT_DIRECTIVE(".amdhsa_forward_progress", COMPUTE_PGM_RSRC1_GFX10_PLUS_FWD_PROGRESS);
   }
+
+  if (isGFX12Plus())
+    PRINT_DIRECTIVE(".amdhsa_round_robin_scheduling",
+                    COMPUTE_PGM_RSRC1_GFX12_PLUS_ENABLE_WG_RR_EN);
+
   return MCDisassembler::Success;
 }
 
diff --git a/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.h b/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.h
index 7e233dcb54ea160e0049ec775e5a7b7885f18b53..233581949d712415c2ceda9340b3e74f7a5912a6 100644
--- a/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.h
+++ b/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.h
@@ -251,6 +251,7 @@ public:
   MCOperand decodeSDWAVopcDst(unsigned Val) const;
 
   MCOperand decodeBoolReg(unsigned Val) const;
+  MCOperand decodeSplitBarrier(unsigned Val) const;
 
   int getTTmpIdx(unsigned Val) const;
 
diff --git a/llvm/lib/Target/AMDGPU/FLATInstructions.td b/llvm/lib/Target/AMDGPU/FLATInstructions.td
index c0251164faee8bddc48441580ac99f8ea1a82ce2..f3771bff247fd8c40f0f93d2d0c37ef5b49ea345 100644
--- a/llvm/lib/Target/AMDGPU/FLATInstructions.td
+++ b/llvm/lib/Target/AMDGPU/FLATInstructions.td
@@ -144,6 +144,47 @@ class FLAT_Real <bits<7> op, FLAT_Pseudo ps, string opName = ps.Mnemonic> :
   let Inst{63-56} = !if(ps.has_vdst, vdst{7-0}, ?);
 }
 
+class VFLAT_Real <bits<8> op, FLAT_Pseudo ps, string opName = ps.Mnemonic> :
+  InstSI <ps.OutOperandList, ps.InOperandList, opName # ps.AsmOperands, []>,
+  Enc96 {
+
+  let FLAT = 1;
+
+  // copy relevant pseudo op flags
+  let SubtargetPredicate   = ps.SubtargetPredicate;
+  let AsmMatchConverter    = ps.AsmMatchConverter;
+  let OtherPredicates      = ps.OtherPredicates;
+  let TSFlags              = ps.TSFlags;
+  let UseNamedOperandTable = ps.UseNamedOperandTable;
+  let SchedRW              = ps.SchedRW;
+  let mayLoad              = ps.mayLoad;
+  let mayStore             = ps.mayStore;
+  let IsAtomicRet          = ps.IsAtomicRet;
+  let IsAtomicNoRet        = ps.IsAtomicNoRet;
+  let VM_CNT               = ps.VM_CNT;
+  let LGKM_CNT             = ps.LGKM_CNT;
+  let VALU                 = ps.VALU;
+
+  bits<7> saddr;
+  bits<8> vdst;
+  bits<6> cpol;
+  bits<8> vdata; // vsrc
+  bits<8> vaddr;
+  bits<24> offset;
+
+  let Inst{6-0} = !if(ps.has_saddr, !if(ps.enabled_saddr, saddr, 0x7f), 0);
+  let Inst{21-14} = op;
+  let Inst{31-26} = 0x3b;
+  let Inst{39-32} = !if(ps.has_vdst, vdst, ?);
+  let Inst{49} = ps.sve;
+  let Inst{54-53} = cpol{2-1}; // th{2-1}
+  let Inst{52} = !if(ps.IsAtomicRet, 1, cpol{0}); // th{0}
+  let Inst{51-50} = cpol{4-3}; // scope
+  let Inst{62-55} = !if(ps.has_data, vdata{7-0}, ?);
+  let Inst{71-64} = !if(ps.has_vaddr, vaddr, ?);
+  let Inst{95-72} = offset;
+}
+
 class GlobalSaddrTable <bit is_saddr, string Name = ""> {
   bit IsSaddr = is_saddr;
   string SaddrOp = Name;
@@ -758,6 +799,10 @@ let SubtargetPredicate = HasFlatAtomicFaddF32Inst in {
   defm FLAT_ATOMIC_ADD_F32       : FLAT_Atomic_Pseudo<"flat_atomic_add_f32",     VGPR_32, f32>;
 } // End SubtargetPredicate = HasFlatAtomicFaddF32Inst
 
+let SubtargetPredicate = isGFX12Plus in {
+  defm FLAT_ATOMIC_CSUB_U32 : FLAT_Atomic_Pseudo <"flat_atomic_csub_u32", VGPR_32, i32>;
+} // End SubtargetPredicate = isGFX12Plus
+
 defm GLOBAL_LOAD_UBYTE    : FLAT_Global_Load_Pseudo <"global_load_ubyte", VGPR_32>;
 defm GLOBAL_LOAD_SBYTE    : FLAT_Global_Load_Pseudo <"global_load_sbyte", VGPR_32>;
 defm GLOBAL_LOAD_USHORT   : FLAT_Global_Load_Pseudo <"global_load_ushort", VGPR_32>;
@@ -2181,7 +2226,7 @@ defm SCRATCH_LOAD_LDS_DWORD     : FLAT_Real_ScratchAllAddr_LDS_gfx10 <0x00c>;
 class FLAT_Real_gfx11 <bits<7> op, FLAT_Pseudo ps, string opName = ps.Mnemonic> :
   FLAT_Real <op, ps, opName>,
   SIMCInstr <ps.PseudoInstr, SIEncodingFamily.GFX11> {
-  let AssemblerPredicate = isGFX11Plus;
+  let AssemblerPredicate = isGFX11Only;
   let DecoderNamespace = "GFX11";
 
   let Inst{13}    = !if(ps.has_dlc, cpol{CPolBit.DLC}, ps.dlcValue);
@@ -2193,7 +2238,7 @@ class FLAT_Real_gfx11 <bits<7> op, FLAT_Pseudo ps, string opName = ps.Mnemonic>
 
 multiclass FLAT_Aliases_gfx11<string ps, string opName, int renamed> {
   if renamed then
-    def _renamed_gfx11 : MnemonicAlias<!cast<FLAT_Pseudo>(ps).Mnemonic, opName>, Requires<[isGFX11Plus]>;
+    def _renamed_gfx11 : MnemonicAlias<!cast<FLAT_Pseudo>(ps).Mnemonic, opName>, Requires<[isGFX11Only]>;
 }
 
 multiclass FLAT_Real_Base_gfx11<bits<7> op, string ps, string opName, int renamed = false> :
@@ -2388,3 +2433,213 @@ defm SCRATCH_LOAD_D16_HI_I8     : FLAT_Real_ScratchAllAddr_gfx11<0x22, "SCRATCH_
 defm SCRATCH_LOAD_D16_HI_B16    : FLAT_Real_ScratchAllAddr_gfx11<0x23, "SCRATCH_LOAD_SHORT_D16_HI", "scratch_load_d16_hi_b16">;
 defm SCRATCH_STORE_D16_HI_B8    : FLAT_Real_ScratchAllAddr_gfx11<0x24, "SCRATCH_STORE_BYTE_D16_HI", "scratch_store_d16_hi_b8">;
 defm SCRATCH_STORE_D16_HI_B16   : FLAT_Real_ScratchAllAddr_gfx11<0x25, "SCRATCH_STORE_SHORT_D16_HI", "scratch_store_d16_hi_b16">;
+
+//===----------------------------------------------------------------------===//
+// GFX12
+//===----------------------------------------------------------------------===//
+
+class VFLAT_Real_gfx12 <bits<8> op, FLAT_Pseudo ps,
+                        string opName = ps.Mnemonic> :
+  VFLAT_Real <op, ps, opName>,
+  SIMCInstr <ps.PseudoInstr, SIEncodingFamily.GFX12> {
+  let AssemblerPredicate = isGFX12Plus;
+  let DecoderNamespace = "GFX12";
+
+  let Inst{25-24} = !if(ps.is_flat_scratch, 0b01,
+                        !if(ps.is_flat_global, 0b10, 0b00));
+}
+
+multiclass VFLAT_Aliases_gfx12<string ps, string opName, int renamed, string alias> {
+  if renamed then
+    def _renamed_gfx12 : MnemonicAlias<!cast<FLAT_Pseudo>(ps).Mnemonic, opName>, Requires<[isGFX12Plus]>;
+  if !not(!empty(alias)) then
+    def _alias_gfx12 : MnemonicAlias<alias, opName>, Requires<[isGFX12Plus]>;
+}
+
+multiclass VFLAT_Real_Base_gfx12<bits<8> op, string ps, string opName, int renamed = false, string alias = ""> :
+  VFLAT_Aliases_gfx12<ps, opName, renamed, alias> {
+  def _gfx12 : VFLAT_Real_gfx12<op, !cast<FLAT_Pseudo>(ps), opName> {
+    let Inst{6-0} = !cast<int>(SGPR_NULL_gfx11plus.HWEncoding);
+  }
+}
+
+multiclass VFLAT_Real_RTN_gfx12<bits<8> op, string ps, string opName> {
+  def _RTN_gfx12 : VFLAT_Real_gfx12<op, !cast<FLAT_Pseudo>(ps#"_RTN"), opName> {
+    let Inst{6-0} = !cast<int>(SGPR_NULL_gfx11plus.HWEncoding);
+  }
+}
+
+multiclass VFLAT_Real_SADDR_gfx12<bits<8> op, string ps, string opName> {
+  def _SADDR_gfx12 : VFLAT_Real_gfx12<op, !cast<FLAT_Pseudo>(ps#"_SADDR"), opName>;
+}
+
+multiclass VFLAT_Real_SADDR_RTN_gfx12<bits<8> op, string ps, string opName> {
+  def _SADDR_RTN_gfx12 : VFLAT_Real_gfx12<op, !cast<FLAT_Pseudo>(ps#"_SADDR_RTN"), opName>;
+}
+
+multiclass VFLAT_Real_ST_gfx12<bits<8> op, string ps, string opName> {
+  def _ST_gfx12 : VFLAT_Real_gfx12<op, !cast<FLAT_Pseudo>(ps#"_ST"), opName> {
+    let Inst{6-0} = !cast<int>(SGPR_NULL_gfx11plus.HWEncoding);
+    let OtherPredicates = [HasFlatScratchSTMode];
+  }
+}
+
+multiclass VFLAT_Real_SVS_gfx12<bits<8> op, string ps, string opName> {
+  def _SVS_gfx12 : VFLAT_Real_gfx12<op, !cast<FLAT_Pseudo>(ps#"_SVS"), opName> {
+    let OtherPredicates = [HasFlatScratchSVSMode];
+  }
+}
+
+multiclass VFLAT_Real_Atomics_gfx12<bits<8> op, string ps, string opName, int renamed = false, string alias = ""> :
+  VFLAT_Real_Base_gfx12<op, ps, opName, renamed, alias>,
+  VFLAT_Real_RTN_gfx12<op, ps, opName>;
+
+multiclass VGLOBAL_Real_AllAddr_gfx12<bits<8> op, string ps, string opName, int renamed = false, string alias = ""> :
+  VFLAT_Real_Base_gfx12<op, ps, opName, renamed, alias>,
+  VFLAT_Real_SADDR_gfx12<op, ps, opName>;
+
+multiclass VGLOBAL_Real_Atomics_gfx12<bits<8> op, string ps, string opName, int renamed = false, string alias = ""> :
+  VGLOBAL_Real_AllAddr_gfx12<op, ps, opName, renamed, alias>,
+  VFLAT_Real_RTN_gfx12<op, ps, opName>,
+  VFLAT_Real_SADDR_RTN_gfx12<op, ps, opName>;
+
+multiclass VSCRATCH_Real_AllAddr_gfx12<bits<8> op, string ps, string opName, int renamed = false> :
+  VFLAT_Real_Base_gfx12<op, ps, opName, renamed>,
+  VFLAT_Real_SADDR_gfx12<op, ps, opName>,
+  VFLAT_Real_ST_gfx12<op, ps, opName>,
+  VFLAT_Real_SVS_gfx12<op, ps, opName>;
+
+// ENC_VFLAT.
+defm FLAT_LOAD_U8                  : VFLAT_Real_Base_gfx12<0x010, "FLAT_LOAD_UBYTE", "flat_load_u8", true>;
+defm FLAT_LOAD_I8                  : VFLAT_Real_Base_gfx12<0x011, "FLAT_LOAD_SBYTE", "flat_load_i8", true>;
+defm FLAT_LOAD_U16                 : VFLAT_Real_Base_gfx12<0x012, "FLAT_LOAD_USHORT", "flat_load_u16", true>;
+defm FLAT_LOAD_I16                 : VFLAT_Real_Base_gfx12<0x013, "FLAT_LOAD_SSHORT", "flat_load_i16", true>;
+defm FLAT_LOAD_B32                 : VFLAT_Real_Base_gfx12<0x014, "FLAT_LOAD_DWORD", "flat_load_b32", true>;
+defm FLAT_LOAD_B64                 : VFLAT_Real_Base_gfx12<0x015, "FLAT_LOAD_DWORDX2", "flat_load_b64", true>;
+defm FLAT_LOAD_B96                 : VFLAT_Real_Base_gfx12<0x016, "FLAT_LOAD_DWORDX3", "flat_load_b96", true>;
+defm FLAT_LOAD_B128                : VFLAT_Real_Base_gfx12<0x017, "FLAT_LOAD_DWORDX4", "flat_load_b128", true>;
+defm FLAT_STORE_B8                 : VFLAT_Real_Base_gfx12<0x018, "FLAT_STORE_BYTE", "flat_store_b8", true>;
+defm FLAT_STORE_B16                : VFLAT_Real_Base_gfx12<0x019, "FLAT_STORE_SHORT", "flat_store_b16", true>;
+defm FLAT_STORE_B32                : VFLAT_Real_Base_gfx12<0x01a, "FLAT_STORE_DWORD", "flat_store_b32", true>;
+defm FLAT_STORE_B64                : VFLAT_Real_Base_gfx12<0x01b, "FLAT_STORE_DWORDX2", "flat_store_b64", true>;
+defm FLAT_STORE_B96                : VFLAT_Real_Base_gfx12<0x01c, "FLAT_STORE_DWORDX3", "flat_store_b96", true>;
+defm FLAT_STORE_B128               : VFLAT_Real_Base_gfx12<0x01d, "FLAT_STORE_DWORDX4", "flat_store_b128", true>;
+defm FLAT_LOAD_D16_U8              : VFLAT_Real_Base_gfx12<0x01e, "FLAT_LOAD_UBYTE_D16", "flat_load_d16_u8">;
+defm FLAT_LOAD_D16_I8              : VFLAT_Real_Base_gfx12<0x01f, "FLAT_LOAD_SBYTE_D16", "flat_load_d16_i8">;
+defm FLAT_LOAD_D16_B16             : VFLAT_Real_Base_gfx12<0x020, "FLAT_LOAD_SHORT_D16", "flat_load_d16_b16">;
+defm FLAT_LOAD_D16_HI_U8           : VFLAT_Real_Base_gfx12<0x021, "FLAT_LOAD_UBYTE_D16_HI", "flat_load_d16_hi_u8">;
+defm FLAT_LOAD_D16_HI_I8           : VFLAT_Real_Base_gfx12<0x022, "FLAT_LOAD_SBYTE_D16_HI", "flat_load_d16_hi_i8">;
+defm FLAT_LOAD_D16_HI_B16          : VFLAT_Real_Base_gfx12<0x023, "FLAT_LOAD_SHORT_D16_HI", "flat_load_d16_hi_b16">;
+defm FLAT_STORE_D16_HI_B8          : VFLAT_Real_Base_gfx12<0x024, "FLAT_STORE_BYTE_D16_HI", "flat_store_d16_hi_b8">;
+defm FLAT_STORE_D16_HI_B16         : VFLAT_Real_Base_gfx12<0x025, "FLAT_STORE_SHORT_D16_HI", "flat_store_d16_hi_b16">;
+defm FLAT_ATOMIC_SWAP_B32          : VFLAT_Real_Atomics_gfx12<0x033, "FLAT_ATOMIC_SWAP", "flat_atomic_swap_b32", true>;
+defm FLAT_ATOMIC_CMPSWAP_B32       : VFLAT_Real_Atomics_gfx12<0x034, "FLAT_ATOMIC_CMPSWAP", "flat_atomic_cmpswap_b32", true>;
+defm FLAT_ATOMIC_ADD_U32           : VFLAT_Real_Atomics_gfx12<0x035, "FLAT_ATOMIC_ADD", "flat_atomic_add_u32", true>;
+defm FLAT_ATOMIC_SUB_U32           : VFLAT_Real_Atomics_gfx12<0x036, "FLAT_ATOMIC_SUB", "flat_atomic_sub_u32", true>;
+defm FLAT_ATOMIC_SUB_CLAMP_U32     : VFLAT_Real_Atomics_gfx12<0x037, "FLAT_ATOMIC_CSUB_U32", "flat_atomic_sub_clamp_u32", true>;
+defm FLAT_ATOMIC_MIN_I32           : VFLAT_Real_Atomics_gfx12<0x038, "FLAT_ATOMIC_SMIN", "flat_atomic_min_i32", true>;
+defm FLAT_ATOMIC_MIN_U32           : VFLAT_Real_Atomics_gfx12<0x039, "FLAT_ATOMIC_UMIN", "flat_atomic_min_u32", true>;
+defm FLAT_ATOMIC_MAX_I32           : VFLAT_Real_Atomics_gfx12<0x03a, "FLAT_ATOMIC_SMAX", "flat_atomic_max_i32", true>;
+defm FLAT_ATOMIC_MAX_U32           : VFLAT_Real_Atomics_gfx12<0x03b, "FLAT_ATOMIC_UMAX", "flat_atomic_max_u32", true>;
+defm FLAT_ATOMIC_AND_B32           : VFLAT_Real_Atomics_gfx12<0x03c, "FLAT_ATOMIC_AND", "flat_atomic_and_b32", true>;
+defm FLAT_ATOMIC_OR_B32            : VFLAT_Real_Atomics_gfx12<0x03d, "FLAT_ATOMIC_OR", "flat_atomic_or_b32", true>;
+defm FLAT_ATOMIC_XOR_B32           : VFLAT_Real_Atomics_gfx12<0x03e, "FLAT_ATOMIC_XOR", "flat_atomic_xor_b32", true>;
+defm FLAT_ATOMIC_INC_U32           : VFLAT_Real_Atomics_gfx12<0x03f, "FLAT_ATOMIC_INC", "flat_atomic_inc_u32", true>;
+defm FLAT_ATOMIC_DEC_U32           : VFLAT_Real_Atomics_gfx12<0x040, "FLAT_ATOMIC_DEC", "flat_atomic_dec_u32", true>;
+defm FLAT_ATOMIC_SWAP_B64          : VFLAT_Real_Atomics_gfx12<0x041, "FLAT_ATOMIC_SWAP_X2", "flat_atomic_swap_b64", true>;
+defm FLAT_ATOMIC_CMPSWAP_B64       : VFLAT_Real_Atomics_gfx12<0x042, "FLAT_ATOMIC_CMPSWAP_X2", "flat_atomic_cmpswap_b64", true>;
+defm FLAT_ATOMIC_ADD_U64           : VFLAT_Real_Atomics_gfx12<0x043, "FLAT_ATOMIC_ADD_X2", "flat_atomic_add_u64", true>;
+defm FLAT_ATOMIC_SUB_U64           : VFLAT_Real_Atomics_gfx12<0x044, "FLAT_ATOMIC_SUB_X2", "flat_atomic_sub_u64", true>;
+defm FLAT_ATOMIC_MIN_I64           : VFLAT_Real_Atomics_gfx12<0x045, "FLAT_ATOMIC_SMIN_X2", "flat_atomic_min_i64", true>;
+defm FLAT_ATOMIC_MIN_U64           : VFLAT_Real_Atomics_gfx12<0x046, "FLAT_ATOMIC_UMIN_X2", "flat_atomic_min_u64", true>;
+defm FLAT_ATOMIC_MAX_I64           : VFLAT_Real_Atomics_gfx12<0x047, "FLAT_ATOMIC_SMAX_X2", "flat_atomic_max_i64", true>;
+defm FLAT_ATOMIC_MAX_U64           : VFLAT_Real_Atomics_gfx12<0x048, "FLAT_ATOMIC_UMAX_X2", "flat_atomic_max_u64", true>;
+defm FLAT_ATOMIC_AND_B64           : VFLAT_Real_Atomics_gfx12<0x049, "FLAT_ATOMIC_AND_X2", "flat_atomic_and_b64", true>;
+defm FLAT_ATOMIC_OR_B64            : VFLAT_Real_Atomics_gfx12<0x04a, "FLAT_ATOMIC_OR_X2", "flat_atomic_or_b64", true>;
+defm FLAT_ATOMIC_XOR_B64           : VFLAT_Real_Atomics_gfx12<0x04b, "FLAT_ATOMIC_XOR_X2", "flat_atomic_xor_b64", true>;
+defm FLAT_ATOMIC_INC_U64           : VFLAT_Real_Atomics_gfx12<0x04c, "FLAT_ATOMIC_INC_X2", "flat_atomic_inc_u64", true>;
+defm FLAT_ATOMIC_DEC_U64           : VFLAT_Real_Atomics_gfx12<0x04d, "FLAT_ATOMIC_DEC_X2", "flat_atomic_dec_u64", true>;
+defm FLAT_ATOMIC_MIN_NUM_F32       : VFLAT_Real_Atomics_gfx12<0x051, "FLAT_ATOMIC_FMIN", "flat_atomic_min_num_f32", true, "flat_atomic_min_f32">;
+defm FLAT_ATOMIC_MAX_NUM_F32       : VFLAT_Real_Atomics_gfx12<0x052, "FLAT_ATOMIC_FMAX", "flat_atomic_max_num_f32", true, "flat_atomic_max_f32">;
+defm FLAT_ATOMIC_ADD_F32           : VFLAT_Real_Atomics_gfx12<0x056, "FLAT_ATOMIC_ADD_F32", "flat_atomic_add_f32">;
+
+// ENC_VGLOBAL.
+defm GLOBAL_LOAD_U8                : VGLOBAL_Real_AllAddr_gfx12<0x010, "GLOBAL_LOAD_UBYTE", "global_load_u8", true>;
+defm GLOBAL_LOAD_I8                : VGLOBAL_Real_AllAddr_gfx12<0x011, "GLOBAL_LOAD_SBYTE", "global_load_i8", true>;
+defm GLOBAL_LOAD_U16               : VGLOBAL_Real_AllAddr_gfx12<0x012, "GLOBAL_LOAD_USHORT", "global_load_u16", true>;
+defm GLOBAL_LOAD_I16               : VGLOBAL_Real_AllAddr_gfx12<0x013, "GLOBAL_LOAD_SSHORT", "global_load_i16", true>;
+defm GLOBAL_LOAD_B32               : VGLOBAL_Real_AllAddr_gfx12<0x014, "GLOBAL_LOAD_DWORD", "global_load_b32", true>;
+defm GLOBAL_LOAD_B64               : VGLOBAL_Real_AllAddr_gfx12<0x015, "GLOBAL_LOAD_DWORDX2", "global_load_b64", true>;
+defm GLOBAL_LOAD_B96               : VGLOBAL_Real_AllAddr_gfx12<0x016, "GLOBAL_LOAD_DWORDX3", "global_load_b96", true>;
+defm GLOBAL_LOAD_B128              : VGLOBAL_Real_AllAddr_gfx12<0x017, "GLOBAL_LOAD_DWORDX4", "global_load_b128", true>;
+defm GLOBAL_STORE_B8               : VGLOBAL_Real_AllAddr_gfx12<0x018, "GLOBAL_STORE_BYTE", "global_store_b8", true>;
+defm GLOBAL_STORE_B16              : VGLOBAL_Real_AllAddr_gfx12<0x019, "GLOBAL_STORE_SHORT", "global_store_b16", true>;
+defm GLOBAL_STORE_B32              : VGLOBAL_Real_AllAddr_gfx12<0x01a, "GLOBAL_STORE_DWORD", "global_store_b32", true>;
+defm GLOBAL_STORE_B64              : VGLOBAL_Real_AllAddr_gfx12<0x01b, "GLOBAL_STORE_DWORDX2", "global_store_b64", true>;
+defm GLOBAL_STORE_B96              : VGLOBAL_Real_AllAddr_gfx12<0x01c, "GLOBAL_STORE_DWORDX3", "global_store_b96", true>;
+defm GLOBAL_STORE_B128             : VGLOBAL_Real_AllAddr_gfx12<0x01d, "GLOBAL_STORE_DWORDX4", "global_store_b128", true>;
+defm GLOBAL_LOAD_D16_U8            : VGLOBAL_Real_AllAddr_gfx12<0x01e, "GLOBAL_LOAD_UBYTE_D16", "global_load_d16_u8">;
+defm GLOBAL_LOAD_D16_I8            : VGLOBAL_Real_AllAddr_gfx12<0x01f, "GLOBAL_LOAD_SBYTE_D16", "global_load_d16_i8">;
+defm GLOBAL_LOAD_D16_B16           : VGLOBAL_Real_AllAddr_gfx12<0x020, "GLOBAL_LOAD_SHORT_D16", "global_load_d16_b16">;
+defm GLOBAL_LOAD_D16_HI_U8         : VGLOBAL_Real_AllAddr_gfx12<0x021, "GLOBAL_LOAD_UBYTE_D16_HI", "global_load_d16_hi_u8">;
+defm GLOBAL_LOAD_D16_HI_I8         : VGLOBAL_Real_AllAddr_gfx12<0x022, "GLOBAL_LOAD_SBYTE_D16_HI", "global_load_d16_hi_i8">;
+defm GLOBAL_LOAD_D16_HI_B16        : VGLOBAL_Real_AllAddr_gfx12<0x023, "GLOBAL_LOAD_SHORT_D16_HI", "global_load_d16_hi_b16">;
+defm GLOBAL_STORE_D16_HI_B8        : VGLOBAL_Real_AllAddr_gfx12<0x024, "GLOBAL_STORE_BYTE_D16_HI", "global_store_d16_hi_b8">;
+defm GLOBAL_STORE_D16_HI_B16       : VGLOBAL_Real_AllAddr_gfx12<0x025, "GLOBAL_STORE_SHORT_D16_HI", "global_store_d16_hi_b16">;
+defm GLOBAL_LOAD_ADDTID_B32        : VGLOBAL_Real_AllAddr_gfx12<0x028, "GLOBAL_LOAD_DWORD_ADDTID", "global_load_addtid_b32">;
+defm GLOBAL_STORE_ADDTID_B32       : VGLOBAL_Real_AllAddr_gfx12<0x029, "GLOBAL_STORE_DWORD_ADDTID", "global_store_addtid_b32">;
+
+defm GLOBAL_ATOMIC_SWAP_B32        : VGLOBAL_Real_Atomics_gfx12<0x033, "GLOBAL_ATOMIC_SWAP", "global_atomic_swap_b32", true>;
+defm GLOBAL_ATOMIC_CMPSWAP_B32     : VGLOBAL_Real_Atomics_gfx12<0x034, "GLOBAL_ATOMIC_CMPSWAP", "global_atomic_cmpswap_b32", true>;
+defm GLOBAL_ATOMIC_ADD_U32         : VGLOBAL_Real_Atomics_gfx12<0x035, "GLOBAL_ATOMIC_ADD", "global_atomic_add_u32", true>;
+defm GLOBAL_ATOMIC_SUB_U32         : VGLOBAL_Real_Atomics_gfx12<0x036, "GLOBAL_ATOMIC_SUB", "global_atomic_sub_u32", true>;
+defm GLOBAL_ATOMIC_SUB_CLAMP_U32   : VGLOBAL_Real_Atomics_gfx12<0x037, "GLOBAL_ATOMIC_CSUB", "global_atomic_sub_clamp_u32", true, "global_atomic_csub_u32">;
+defm GLOBAL_ATOMIC_MIN_I32         : VGLOBAL_Real_Atomics_gfx12<0x038, "GLOBAL_ATOMIC_SMIN", "global_atomic_min_i32", true>;
+defm GLOBAL_ATOMIC_MIN_U32         : VGLOBAL_Real_Atomics_gfx12<0x039, "GLOBAL_ATOMIC_UMIN", "global_atomic_min_u32", true>;
+defm GLOBAL_ATOMIC_MAX_I32         : VGLOBAL_Real_Atomics_gfx12<0x03a, "GLOBAL_ATOMIC_SMAX", "global_atomic_max_i32", true>;
+defm GLOBAL_ATOMIC_MAX_U32         : VGLOBAL_Real_Atomics_gfx12<0x03b, "GLOBAL_ATOMIC_UMAX", "global_atomic_max_u32", true>;
+defm GLOBAL_ATOMIC_AND_B32         : VGLOBAL_Real_Atomics_gfx12<0x03c, "GLOBAL_ATOMIC_AND", "global_atomic_and_b32", true>;
+defm GLOBAL_ATOMIC_OR_B32          : VGLOBAL_Real_Atomics_gfx12<0x03d, "GLOBAL_ATOMIC_OR", "global_atomic_or_b32", true>;
+defm GLOBAL_ATOMIC_XOR_B32         : VGLOBAL_Real_Atomics_gfx12<0x03e, "GLOBAL_ATOMIC_XOR", "global_atomic_xor_b32", true>;
+defm GLOBAL_ATOMIC_INC_U32         : VGLOBAL_Real_Atomics_gfx12<0x03f, "GLOBAL_ATOMIC_INC", "global_atomic_inc_u32", true>;
+defm GLOBAL_ATOMIC_DEC_U32         : VGLOBAL_Real_Atomics_gfx12<0x040, "GLOBAL_ATOMIC_DEC", "global_atomic_dec_u32", true>;
+defm GLOBAL_ATOMIC_SWAP_B64        : VGLOBAL_Real_Atomics_gfx12<0x041, "GLOBAL_ATOMIC_SWAP_X2", "global_atomic_swap_b64", true>;
+defm GLOBAL_ATOMIC_CMPSWAP_B64     : VGLOBAL_Real_Atomics_gfx12<0x042, "GLOBAL_ATOMIC_CMPSWAP_X2", "global_atomic_cmpswap_b64", true>;
+defm GLOBAL_ATOMIC_ADD_U64         : VGLOBAL_Real_Atomics_gfx12<0x043, "GLOBAL_ATOMIC_ADD_X2", "global_atomic_add_u64", true>;
+defm GLOBAL_ATOMIC_SUB_U64         : VGLOBAL_Real_Atomics_gfx12<0x044, "GLOBAL_ATOMIC_SUB_X2", "global_atomic_sub_u64", true>;
+defm GLOBAL_ATOMIC_MIN_I64         : VGLOBAL_Real_Atomics_gfx12<0x045, "GLOBAL_ATOMIC_SMIN_X2", "global_atomic_min_i64", true>;
+defm GLOBAL_ATOMIC_MIN_U64         : VGLOBAL_Real_Atomics_gfx12<0x046, "GLOBAL_ATOMIC_UMIN_X2", "global_atomic_min_u64", true>;
+defm GLOBAL_ATOMIC_MAX_I64         : VGLOBAL_Real_Atomics_gfx12<0x047, "GLOBAL_ATOMIC_SMAX_X2", "global_atomic_max_i64", true>;
+defm GLOBAL_ATOMIC_MAX_U64         : VGLOBAL_Real_Atomics_gfx12<0x048, "GLOBAL_ATOMIC_UMAX_X2", "global_atomic_max_u64", true>;
+defm GLOBAL_ATOMIC_AND_B64         : VGLOBAL_Real_Atomics_gfx12<0x049, "GLOBAL_ATOMIC_AND_X2", "global_atomic_and_b64", true>;
+defm GLOBAL_ATOMIC_OR_B64          : VGLOBAL_Real_Atomics_gfx12<0x04a, "GLOBAL_ATOMIC_OR_X2", "global_atomic_or_b64", true>;
+defm GLOBAL_ATOMIC_XOR_B64         : VGLOBAL_Real_Atomics_gfx12<0x04b, "GLOBAL_ATOMIC_XOR_X2", "global_atomic_xor_b64", true>;
+defm GLOBAL_ATOMIC_INC_U64         : VGLOBAL_Real_Atomics_gfx12<0x04c, "GLOBAL_ATOMIC_INC_X2", "global_atomic_inc_u64", true>;
+defm GLOBAL_ATOMIC_DEC_U64         : VGLOBAL_Real_Atomics_gfx12<0x04d, "GLOBAL_ATOMIC_DEC_X2", "global_atomic_dec_u64", true>;
+defm GLOBAL_ATOMIC_MIN_NUM_F32     : VGLOBAL_Real_Atomics_gfx12<0x051, "GLOBAL_ATOMIC_FMIN", "global_atomic_min_num_f32", true, "global_atomic_min_f32">;
+defm GLOBAL_ATOMIC_MAX_NUM_F32     : VGLOBAL_Real_Atomics_gfx12<0x052, "GLOBAL_ATOMIC_FMAX", "global_atomic_max_num_f32", true, "global_atomic_max_f32">;
+defm GLOBAL_ATOMIC_ADD_F32         : VGLOBAL_Real_Atomics_gfx12<0x056, "GLOBAL_ATOMIC_ADD_F32", "global_atomic_add_f32">;
+
+// ENC_VSCRATCH.
+defm SCRATCH_LOAD_U8               : VSCRATCH_Real_AllAddr_gfx12<0x10, "SCRATCH_LOAD_UBYTE", "scratch_load_u8", true>;
+defm SCRATCH_LOAD_I8               : VSCRATCH_Real_AllAddr_gfx12<0x11, "SCRATCH_LOAD_SBYTE", "scratch_load_i8", true>;
+defm SCRATCH_LOAD_U16              : VSCRATCH_Real_AllAddr_gfx12<0x12, "SCRATCH_LOAD_USHORT", "scratch_load_u16", true>;
+defm SCRATCH_LOAD_I16              : VSCRATCH_Real_AllAddr_gfx12<0x13, "SCRATCH_LOAD_SSHORT", "scratch_load_i16", true>;
+defm SCRATCH_LOAD_B32              : VSCRATCH_Real_AllAddr_gfx12<0x14, "SCRATCH_LOAD_DWORD", "scratch_load_b32", true>;
+defm SCRATCH_LOAD_B64              : VSCRATCH_Real_AllAddr_gfx12<0x15, "SCRATCH_LOAD_DWORDX2", "scratch_load_b64", true>;
+defm SCRATCH_LOAD_B96              : VSCRATCH_Real_AllAddr_gfx12<0x16, "SCRATCH_LOAD_DWORDX3", "scratch_load_b96", true>;
+defm SCRATCH_LOAD_B128             : VSCRATCH_Real_AllAddr_gfx12<0x17, "SCRATCH_LOAD_DWORDX4", "scratch_load_b128", true>;
+defm SCRATCH_STORE_B8              : VSCRATCH_Real_AllAddr_gfx12<0x18, "SCRATCH_STORE_BYTE", "scratch_store_b8", true>;
+defm SCRATCH_STORE_B16             : VSCRATCH_Real_AllAddr_gfx12<0x19, "SCRATCH_STORE_SHORT", "scratch_store_b16", true>;
+defm SCRATCH_STORE_B32             : VSCRATCH_Real_AllAddr_gfx12<0x1a, "SCRATCH_STORE_DWORD", "scratch_store_b32", true>;
+defm SCRATCH_STORE_B64             : VSCRATCH_Real_AllAddr_gfx12<0x1b, "SCRATCH_STORE_DWORDX2", "scratch_store_b64", true>;
+defm SCRATCH_STORE_B96             : VSCRATCH_Real_AllAddr_gfx12<0x1c, "SCRATCH_STORE_DWORDX3", "scratch_store_b96", true>;
+defm SCRATCH_STORE_B128            : VSCRATCH_Real_AllAddr_gfx12<0x1d, "SCRATCH_STORE_DWORDX4", "scratch_store_b128", true>;
+defm SCRATCH_LOAD_D16_U8           : VSCRATCH_Real_AllAddr_gfx12<0x1e, "SCRATCH_LOAD_UBYTE_D16", "scratch_load_d16_u8">;
+defm SCRATCH_LOAD_D16_I8           : VSCRATCH_Real_AllAddr_gfx12<0x1f, "SCRATCH_LOAD_SBYTE_D16", "scratch_load_d16_i8">;
+defm SCRATCH_LOAD_D16_B16          : VSCRATCH_Real_AllAddr_gfx12<0x20, "SCRATCH_LOAD_SHORT_D16", "scratch_load_d16_b16">;
+defm SCRATCH_LOAD_D16_HI_U8        : VSCRATCH_Real_AllAddr_gfx12<0x21, "SCRATCH_LOAD_UBYTE_D16_HI", "scratch_load_d16_hi_u8">;
+defm SCRATCH_LOAD_D16_HI_I8        : VSCRATCH_Real_AllAddr_gfx12<0x22, "SCRATCH_LOAD_SBYTE_D16_HI", "scratch_load_d16_hi_i8">;
+defm SCRATCH_LOAD_D16_HI_B16       : VSCRATCH_Real_AllAddr_gfx12<0x23, "SCRATCH_LOAD_SHORT_D16_HI", "scratch_load_d16_hi_b16">;
+defm SCRATCH_STORE_D16_HI_B8       : VSCRATCH_Real_AllAddr_gfx12<0x24, "SCRATCH_STORE_BYTE_D16_HI", "scratch_store_d16_hi_b8">;
+defm SCRATCH_STORE_D16_HI_B16      : VSCRATCH_Real_AllAddr_gfx12<0x25, "SCRATCH_STORE_SHORT_D16_HI", "scratch_store_d16_hi_b16">;
diff --git a/llvm/lib/Target/AMDGPU/GCNSubtarget.h b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
index 2eed024af60ae69645ca251044d1adbcdaa44832..8e1350be8b45fc0bb14f3e71d43eea8d709d2fde 100644
--- a/llvm/lib/Target/AMDGPU/GCNSubtarget.h
+++ b/llvm/lib/Target/AMDGPU/GCNSubtarget.h
@@ -1206,12 +1206,28 @@ public:
     return hasKernargPreload() && !hasGFX940Insts();
   }
 
+  // \returns true if the target has split barriers feature
+  bool hasSplitBarriers() const { return getGeneration() >= GFX12; }
+
   // \returns true if FP8/BF8 VOP1 form of conversion to F32 is unreliable.
   bool hasCvtFP8VOP1Bug() const { return true; }
 
-  // \returns true is CSUB atomics support a no-return form.
+  // \returns true if CSUB (a.k.a. SUB_CLAMP on GFX12) atomics support a
+  // no-return form.
   bool hasAtomicCSubNoRtnInsts() const { return HasAtomicCSubNoRtnInsts; }
 
+  // \returns true if the target has DX10_CLAMP kernel descriptor mode bit
+  bool hasDX10ClampMode() const { return getGeneration() < GFX12; }
+
+  // \returns true if the target has IEEE kernel descriptor mode bit
+  bool hasIEEEMode() const { return getGeneration() < GFX12; }
+
+  // \returns true if the target has IEEE fminimum/fmaximum instructions
+  bool hasIEEEMinMax() const { return getGeneration() >= GFX12; }
+
+  // \returns true if the target has WG_RR_MODE kernel descriptor mode bit
+  bool hasRrWGMode() const { return getGeneration() >= GFX12; }
+
   /// \returns SGPR allocation granularity supported by the subtarget.
   unsigned getSGPRAllocGranule() const {
     return AMDGPU::IsaInfo::getSGPRAllocGranule(this);
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp
index 57f74ae08b35c40b49d490e29f8770779751f148..edc244db613d865d243821f9a659fcaab029506c 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUInstPrinter.cpp
@@ -97,28 +97,36 @@ void AMDGPUInstPrinter::printNamedBit(const MCInst *MI, unsigned OpNo,
 void AMDGPUInstPrinter::printOffset(const MCInst *MI, unsigned OpNo,
                                     const MCSubtargetInfo &STI,
                                     raw_ostream &O) {
-  uint16_t Imm = MI->getOperand(OpNo).getImm();
+  uint32_t Imm = MI->getOperand(OpNo).getImm();
   if (Imm != 0) {
     O << " offset:";
-    printU16ImmDecOperand(MI, OpNo, O);
+
+    // GFX12 uses a 24-bit signed offset for VBUFFER.
+    const MCInstrDesc &Desc = MII.get(MI->getOpcode());
+    bool IsVBuffer = Desc.TSFlags & (SIInstrFlags::MUBUF | SIInstrFlags::MTBUF);
+    if (AMDGPU::isGFX12(STI) && IsVBuffer)
+      O << formatDec(SignExtend32<24>(Imm));
+    else
+      printU16ImmDecOperand(MI, OpNo, O);
   }
 }
 
 void AMDGPUInstPrinter::printFlatOffset(const MCInst *MI, unsigned OpNo,
                                         const MCSubtargetInfo &STI,
                                         raw_ostream &O) {
-  uint16_t Imm = MI->getOperand(OpNo).getImm();
+  uint32_t Imm = MI->getOperand(OpNo).getImm();
   if (Imm != 0) {
     O << " offset:";
 
     const MCInstrDesc &Desc = MII.get(MI->getOpcode());
-    bool IsFlatSeg = !(Desc.TSFlags &
-                       (SIInstrFlags::FlatGlobal | SIInstrFlags::FlatScratch));
+    bool AllowNegative = (Desc.TSFlags & (SIInstrFlags::FlatGlobal |
+                                          SIInstrFlags::FlatScratch)) ||
+                         AMDGPU::isGFX12(STI);
 
-    if (IsFlatSeg) // Unsigned offset
-      printU16ImmDecOperand(MI, OpNo, O);
-    else // Signed offset
+    if (AllowNegative) // Signed offset
       O << formatDec(SignExtend32(Imm, AMDGPU::getNumFlatOffsetBits(STI)));
+    else // Unsigned offset
+      printU16ImmDecOperand(MI, OpNo, O);
   }
 }
 
@@ -708,6 +716,7 @@ void AMDGPUInstPrinter::printRegularOperand(const MCInst *MI, unsigned OpNo,
     case AMDGPU::OPERAND_REG_INLINE_C_V2INT32:
     case AMDGPU::OPERAND_REG_INLINE_C_V2FP32:
     case MCOI::OPERAND_IMMEDIATE:
+    case AMDGPU::OPERAND_INLINE_SPLIT_BARRIER_INT32:
       printImmediate32(Op.getImm(), STI, O);
       break;
     case AMDGPU::OPERAND_REG_IMM_INT64:
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCCodeEmitter.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCCodeEmitter.cpp
index 80e7ca2b39d1b5a70f3afffb50e1ea5561fb0bc4..b403d69d9ff13715a7a709e878eb8e38e980f6fd 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCCodeEmitter.cpp
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUMCCodeEmitter.cpp
@@ -262,6 +262,7 @@ AMDGPUMCCodeEmitter::getLitEncoding(const MCOperand &MO,
   case AMDGPU::OPERAND_REG_IMM_V2FP32:
   case AMDGPU::OPERAND_REG_INLINE_C_V2INT32:
   case AMDGPU::OPERAND_REG_INLINE_C_V2FP32:
+  case AMDGPU::OPERAND_INLINE_SPLIT_BARRIER_INT32:
     return getLit32Encoding(static_cast<uint32_t>(Imm), STI);
 
   case AMDGPU::OPERAND_REG_IMM_INT64:
diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp
index eba8e49a46f82f76e2a3b41f1024153a08904d9d..a855cf585205bcfde31a1fd9bbc3e542fd9e6a7f 100644
--- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp
+++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUTargetStreamer.cpp
@@ -451,12 +451,12 @@ void AMDGPUTargetAsmStreamer::EmitAmdhsaKernelDescriptor(
   PRINT_FIELD(OS, ".amdhsa_float_denorm_mode_16_64", KD,
               compute_pgm_rsrc1,
               amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64);
-  PRINT_FIELD(OS, ".amdhsa_dx10_clamp", KD,
-              compute_pgm_rsrc1,
-              amdhsa::COMPUTE_PGM_RSRC1_ENABLE_DX10_CLAMP);
-  PRINT_FIELD(OS, ".amdhsa_ieee_mode", KD,
-              compute_pgm_rsrc1,
-              amdhsa::COMPUTE_PGM_RSRC1_ENABLE_IEEE_MODE);
+  if (IVersion.Major < 12) {
+    PRINT_FIELD(OS, ".amdhsa_dx10_clamp", KD, compute_pgm_rsrc1,
+                amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_DX10_CLAMP);
+    PRINT_FIELD(OS, ".amdhsa_ieee_mode", KD, compute_pgm_rsrc1,
+                amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_IEEE_MODE);
+  }
   if (IVersion.Major >= 9)
     PRINT_FIELD(OS, ".amdhsa_fp16_overflow", KD,
                 compute_pgm_rsrc1,
@@ -478,6 +478,9 @@ void AMDGPUTargetAsmStreamer::EmitAmdhsaKernelDescriptor(
     PRINT_FIELD(OS, ".amdhsa_shared_vgpr_count", KD, compute_pgm_rsrc3,
                 amdhsa::COMPUTE_PGM_RSRC3_GFX10_PLUS_SHARED_VGPR_COUNT);
   }
+  if (IVersion.Major >= 12)
+    PRINT_FIELD(OS, ".amdhsa_round_robin_scheduling", KD, compute_pgm_rsrc1,
+                amdhsa::COMPUTE_PGM_RSRC1_GFX12_PLUS_ENABLE_WG_RR_EN);
   PRINT_FIELD(
       OS, ".amdhsa_exception_fp_ieee_invalid_op", KD,
       compute_pgm_rsrc2,
diff --git a/llvm/lib/Target/AMDGPU/SIDefines.h b/llvm/lib/Target/AMDGPU/SIDefines.h
index 9b2f52c1286e032d5d86d1ca7f077c1f99c97da4..b291400a947c7a1141ceeac59f10f331c2baef0b 100644
--- a/llvm/lib/Target/AMDGPU/SIDefines.h
+++ b/llvm/lib/Target/AMDGPU/SIDefines.h
@@ -213,6 +213,9 @@ enum OperandType : unsigned {
   OPERAND_REG_INLINE_C_V2INT32,
   OPERAND_REG_INLINE_C_V2FP32,
 
+  // Operand for split barrier inline constant
+  OPERAND_INLINE_SPLIT_BARRIER_INT32,
+
   /// Operand with 32-bit immediate that uses the constant bus.
   OPERAND_KIMM32,
   OPERAND_KIMM16,
@@ -1026,6 +1029,12 @@ enum Register_Flag : uint8_t {
 
 } // namespace AMDGPU
 
+namespace AMDGPU {
+namespace Barrier {
+enum Type { TRAP = -2, WORKGROUP = -1 };
+} // namespace Barrier
+} // namespace AMDGPU
+
 // clang-format off
 
 #define R_00B028_SPI_SHADER_PGM_RSRC1_PS                                0x00B028
@@ -1120,6 +1129,9 @@ enum Register_Flag : uint8_t {
 #define   S_00B848_DX10_CLAMP(x)                                      (((x) & 0x1) << 21)
 #define   G_00B848_DX10_CLAMP(x)                                      (((x) >> 21) & 0x1)
 #define   C_00B848_DX10_CLAMP                                         0xFFDFFFFF
+#define   S_00B848_RR_WG_MODE(x)                                      (((x) & 0x1) << 21)
+#define   G_00B848_RR_WG_MODE(x)                                      (((x) >> 21) & 0x1)
+#define   C_00B848_RR_WG_MODE                                         0xFFDFFFFF
 #define   S_00B848_DEBUG_MODE(x)                                      (((x) & 0x1) << 22)
 #define   G_00B848_DEBUG_MODE(x)                                      (((x) >> 22) & 0x1)
 #define   C_00B848_DEBUG_MODE                                         0xFFBFFFFF
@@ -1136,8 +1148,6 @@ enum Register_Flag : uint8_t {
 #define   G_00B848_FWD_PROGRESS(x)                                    (((x) >> 31) & 0x1)
 #define   C_00B848_FWD_PROGRESS                                       0x7FFFFFFF
 
-// clang-format on
-
 // Helpers for setting FLOAT_MODE
 #define FP_ROUND_ROUND_TO_NEAREST 0
 #define FP_ROUND_ROUND_TO_INF 1
@@ -1179,6 +1189,9 @@ enum Register_Flag : uint8_t {
 
 #define R_SPILLED_SGPRS         0x4
 #define R_SPILLED_VGPRS         0x8
+
+// clang-format on
+
 } // End namespace llvm
 
 #endif
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 85cc3cfec19cd39ef7e86538d3628cd508f2a339..708f212e204acff321afdc967e73369fccf8b4c4 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -763,6 +763,10 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
   if (Subtarget->hasMad64_32())
     setOperationAction({ISD::SMUL_LOHI, ISD::UMUL_LOHI}, MVT::i32, Custom);
 
+  if (Subtarget->hasIEEEMinMax())
+    setOperationAction({ISD::FMAXIMUM, ISD::FMINIMUM},
+                       {MVT::f16, MVT::f32, MVT::f64, MVT::v2f16}, Legal);
+
   setOperationAction(ISD::INTRINSIC_WO_CHAIN,
                      {MVT::Other, MVT::f32, MVT::v4f32, MVT::i16, MVT::f16,
                       MVT::v2i16, MVT::v2f16, MVT::i128},
@@ -800,6 +804,8 @@ SITargetLowering::SITargetLowering(const TargetMachine &TM,
                        ISD::FMAXNUM,
                        ISD::FMINNUM_IEEE,
                        ISD::FMAXNUM_IEEE,
+                       ISD::FMINIMUM,
+                       ISD::FMAXIMUM,
                        ISD::FMA,
                        ISD::SMIN,
                        ISD::SMAX,
@@ -8623,6 +8629,31 @@ SDValue SITargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
                                    M->getVTList(), Ops, M->getMemoryVT(),
                                    M->getMemOperand());
   }
+  case Intrinsic::amdgcn_s_get_barrier_state: {
+    SDValue Chain = Op->getOperand(0);
+    SmallVector<SDValue, 2> Ops;
+    unsigned Opc;
+    bool IsInlinableBarID = false;
+    int64_t BarID;
+
+    if (isa<ConstantSDNode>(Op->getOperand(2))) {
+      BarID = cast<ConstantSDNode>(Op->getOperand(2))->getSExtValue();
+      IsInlinableBarID = AMDGPU::isInlinableIntLiteral(BarID);
+    }
+
+    if (IsInlinableBarID) {
+      Opc = AMDGPU::S_GET_BARRIER_STATE_IMM;
+      SDValue K = DAG.getTargetConstant(BarID, DL, MVT::i32);
+      Ops.push_back(K);
+    } else {
+      Opc = AMDGPU::S_GET_BARRIER_STATE_M0;
+      SDValue M0Val = copyToM0(DAG, Chain, DL, Op.getOperand(2));
+      Ops.push_back(M0Val.getValue(0));
+    }
+
+    auto NewMI = DAG.getMachineNode(Opc, DL, Op->getVTList(), Ops);
+    return SDValue(NewMI, 0);
+  }
   default:
 
     if (const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr =
@@ -8800,13 +8831,29 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
     return SDValue(DAG.getMachineNode(Opc, DL, Op->getVTList(), Ops), 0);
   }
   case Intrinsic::amdgcn_s_barrier: {
+    const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
     if (getTargetMachine().getOptLevel() > CodeGenOptLevel::None) {
-      const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
       unsigned WGSize = ST.getFlatWorkGroupSizes(MF.getFunction()).second;
       if (WGSize <= ST.getWavefrontSize())
         return SDValue(DAG.getMachineNode(AMDGPU::WAVE_BARRIER, DL, MVT::Other,
                                           Op.getOperand(0)), 0);
     }
+
+    // On GFX12 lower s_barrier into s_barrier_signal_imm and s_barrier_wait
+    if (ST.hasSplitBarriers()) {
+      SDValue K =
+          DAG.getTargetConstant(AMDGPU::Barrier::WORKGROUP, DL, MVT::i32);
+      SDValue BarSignal =
+          SDValue(DAG.getMachineNode(AMDGPU::S_BARRIER_SIGNAL_IMM, DL,
+                                     MVT::Other, K, Op.getOperand(0)),
+                  0);
+      SDValue BarWait =
+          SDValue(DAG.getMachineNode(AMDGPU::S_BARRIER_WAIT, DL, MVT::Other, K,
+                                     BarSignal.getValue(0)),
+                  0);
+      return BarWait;
+    }
+
     return SDValue();
   };
   case Intrinsic::amdgcn_tbuffer_store: {
@@ -9192,7 +9239,76 @@ SDValue SITargetLowering::LowerINTRINSIC_VOID(SDValue Op,
   case Intrinsic::amdgcn_end_cf:
     return SDValue(DAG.getMachineNode(AMDGPU::SI_END_CF, DL, MVT::Other,
                                       Op->getOperand(2), Chain), 0);
+  case Intrinsic::amdgcn_s_barrier_init:
+  case Intrinsic::amdgcn_s_barrier_join:
+  case Intrinsic::amdgcn_s_wakeup_barrier: {
+    SDValue Chain = Op->getOperand(0);
+    SmallVector<SDValue, 2> Ops;
+    SDValue BarOp = Op->getOperand(2);
+    unsigned Opc;
+    bool IsInlinableBarID = false;
+    int64_t BarVal;
+
+    if (isa<ConstantSDNode>(BarOp)) {
+      BarVal = cast<ConstantSDNode>(BarOp)->getSExtValue();
+      IsInlinableBarID = AMDGPU::isInlinableIntLiteral(BarVal);
+    }
+
+    if (IsInlinableBarID) {
+      switch (IntrinsicID) {
+      default:
+        return SDValue();
+      case Intrinsic::amdgcn_s_barrier_init:
+        Opc = AMDGPU::S_BARRIER_INIT_IMM;
+        break;
+      case Intrinsic::amdgcn_s_barrier_join:
+        Opc = AMDGPU::S_BARRIER_JOIN_IMM;
+        break;
+      case Intrinsic::amdgcn_s_wakeup_barrier:
+        Opc = AMDGPU::S_WAKEUP_BARRIER_IMM;
+        break;
+      }
+
+      SDValue K = DAG.getTargetConstant(BarVal, DL, MVT::i32);
+      Ops.push_back(K);
+    } else {
+      switch (IntrinsicID) {
+      default:
+        return SDValue();
+      case Intrinsic::amdgcn_s_barrier_init:
+        Opc = AMDGPU::S_BARRIER_INIT_M0;
+        break;
+      case Intrinsic::amdgcn_s_barrier_join:
+        Opc = AMDGPU::S_BARRIER_JOIN_M0;
+        break;
+      case Intrinsic::amdgcn_s_wakeup_barrier:
+        Opc = AMDGPU::S_WAKEUP_BARRIER_M0;
+        break;
+      }
+    }
+
+    if (IntrinsicID == Intrinsic::amdgcn_s_barrier_init) {
+      SDValue M0Val;
+      // Member count will be read from M0[16:22]
+      M0Val = DAG.getNode(ISD::SHL, DL, MVT::i32, Op.getOperand(3),
+                          DAG.getShiftAmountConstant(16, MVT::i32, DL));
 
+      if (!IsInlinableBarID) {
+        // If reference to barrier id is not an inline constant then it must be
+        // referenced with M0[4:0]. Perform an OR with the member count to
+        // include it in M0.
+        M0Val = SDValue(DAG.getMachineNode(AMDGPU::S_OR_B32, DL, MVT::i32,
+                                           Op.getOperand(2), M0Val),
+                        0);
+      }
+      Ops.push_back(copyToM0(DAG, Chain, DL, M0Val).getValue(0));
+    } else if (!IsInlinableBarID) {
+      Ops.push_back(copyToM0(DAG, Chain, DL, BarOp).getValue(0));
+    }
+
+    auto NewMI = DAG.getMachineNode(Opc, DL, Op->getVTList(), Ops);
+    return SDValue(NewMI, 0);
+  }
   default: {
     if (const AMDGPU::ImageDimIntrinsicInfo *ImageDimIntr =
             AMDGPU::getImageDimIntrinsicInfo(IntrinsicID))
@@ -11786,10 +11902,14 @@ bool SITargetLowering::isCanonicalized(SelectionDAG &DAG, SDValue Op,
   case ISD::FMAXNUM:
   case ISD::FMINNUM_IEEE:
   case ISD::FMAXNUM_IEEE:
+  case ISD::FMINIMUM:
+  case ISD::FMAXIMUM:
   case AMDGPUISD::CLAMP:
   case AMDGPUISD::FMED3:
   case AMDGPUISD::FMAX3:
-  case AMDGPUISD::FMIN3: {
+  case AMDGPUISD::FMIN3:
+  case AMDGPUISD::FMAXIMUM3:
+  case AMDGPUISD::FMINIMUM3: {
     // FIXME: Shouldn't treat the generic operations different based these.
     // However, we aren't really required to flush the result from
     // minnum/maxnum..
@@ -11943,7 +12063,9 @@ bool SITargetLowering::isCanonicalized(Register Reg, MachineFunction &MF,
   case AMDGPU::G_FMINNUM:
   case AMDGPU::G_FMAXNUM:
   case AMDGPU::G_FMINNUM_IEEE:
-  case AMDGPU::G_FMAXNUM_IEEE: {
+  case AMDGPU::G_FMAXNUM_IEEE:
+  case AMDGPU::G_FMINIMUM:
+  case AMDGPU::G_FMAXIMUM: {
     if (Subtarget->supportsMinMaxDenormModes() ||
         // FIXME: denormalsEnabledForType is broken for dynamic
         denormalsEnabledForType(MRI.getType(Reg), MF))
@@ -12131,6 +12253,8 @@ static unsigned minMaxOpcToMin3Max3Opc(unsigned Opc) {
   case ISD::FMAXNUM:
   case ISD::FMAXNUM_IEEE:
     return AMDGPUISD::FMAX3;
+  case ISD::FMAXIMUM:
+    return AMDGPUISD::FMAXIMUM3;
   case ISD::SMAX:
     return AMDGPUISD::SMAX3;
   case ISD::UMAX:
@@ -12138,6 +12262,8 @@ static unsigned minMaxOpcToMin3Max3Opc(unsigned Opc) {
   case ISD::FMINNUM:
   case ISD::FMINNUM_IEEE:
     return AMDGPUISD::FMIN3;
+  case ISD::FMINIMUM:
+    return AMDGPUISD::FMINIMUM3;
   case ISD::SMIN:
     return AMDGPUISD::SMIN3;
   case ISD::UMIN:
@@ -12497,7 +12623,9 @@ SDValue SITargetLowering::performExtractVectorEltCombine(
     case ISD::FMAXNUM:
     case ISD::FMINNUM:
     case ISD::FMAXNUM_IEEE:
-    case ISD::FMINNUM_IEEE: {
+    case ISD::FMINNUM_IEEE:
+    case ISD::FMAXIMUM:
+    case ISD::FMINIMUM: {
       SDValue Elt0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, ResVT,
                                  Vec.getOperand(0), Idx);
       SDValue Elt1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SL, ResVT,
@@ -13759,6 +13887,8 @@ SDValue SITargetLowering::PerformDAGCombine(SDNode *N,
   case ISD::FMINNUM:
   case ISD::FMAXNUM_IEEE:
   case ISD::FMINNUM_IEEE:
+  case ISD::FMAXIMUM:
+  case ISD::FMINIMUM:
   case ISD::SMAX:
   case ISD::SMIN:
   case ISD::UMAX:
diff --git a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
index ede4841b8a5fd7db280f9522aed0cc913bfd94e8..c16583f6a7f9ac28a99d36d018a707d32d78ffcb 100644
--- a/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInsertWaitcnts.cpp
@@ -452,7 +452,9 @@ public:
   // FLAT instruction.
   WaitEventType getVmemWaitEventType(const MachineInstr &Inst) const {
     assert(SIInstrInfo::isVMEM(Inst) || SIInstrInfo::isFLAT(Inst));
-    if (!ST->hasVscnt())
+    // LDS DMA loads are also stores, but on the LDS side. On the VMEM side
+    // these should use VM_CNT.
+    if (!ST->hasVscnt() || SIInstrInfo::mayWriteLDSThroughDMA(Inst))
       return VMEM_ACCESS;
     if (Inst.mayStore() && !SIInstrInfo::isAtomicRet(Inst)) {
       // FLAT and SCRATCH instructions may access scratch. Other VMEM
@@ -544,14 +546,6 @@ void WaitcntBrackets::setExpScore(const MachineInstr *MI,
   }
 }
 
-// MUBUF and FLAT LDS DMA operations need a wait on vmcnt before LDS written
-// can be accessed. A load from LDS to VMEM does not need a wait.
-static bool mayWriteLDSThroughDMA(const MachineInstr &MI) {
-  return SIInstrInfo::isVALU(MI) &&
-         (SIInstrInfo::isMUBUF(MI) || SIInstrInfo::isFLAT(MI)) &&
-         MI.getOpcode() != AMDGPU::BUFFER_STORE_LDS_DWORD;
-}
-
 void WaitcntBrackets::updateByEvent(const SIInstrInfo *TII,
                                     const SIRegisterInfo *TRI,
                                     const MachineRegisterInfo *MRI,
@@ -703,7 +697,10 @@ void WaitcntBrackets::updateByEvent(const SIInstrInfo *TII,
         setRegScore(RegNo, T, CurrScore);
       }
     }
-    if (Inst.mayStore() && (TII->isDS(Inst) || mayWriteLDSThroughDMA(Inst))) {
+    if (Inst.mayStore() &&
+        (TII->isDS(Inst) || TII->mayWriteLDSThroughDMA(Inst))) {
+      // MUBUF and FLAT LDS DMA operations need a wait on vmcnt before LDS
+      // written can be accessed. A load from LDS to VMEM does not need a wait.
       setRegScore(SQ_MAX_PGM_VGPRS + EXTRA_VGPR_LDS, T, CurrScore);
     }
   }
@@ -1178,7 +1175,7 @@ bool SIInsertWaitcnts::generateWaitcntInstBefore(MachineInstr &MI,
         if (AS != AMDGPUAS::LOCAL_ADDRESS && AS != AMDGPUAS::FLAT_ADDRESS)
           continue;
         // No need to wait before load from VMEM to LDS.
-        if (mayWriteLDSThroughDMA(MI))
+        if (TII->mayWriteLDSThroughDMA(MI))
           continue;
         unsigned RegNo = SQ_MAX_PGM_VGPRS + EXTRA_VGPR_LDS;
         // VM_CNT is only relevant to vgpr or LDS.
@@ -1504,6 +1501,11 @@ void SIInsertWaitcnts::updateEventWaitcntAfter(MachineInstr &Inst,
       break;
     case AMDGPU::S_MEMTIME:
     case AMDGPU::S_MEMREALTIME:
+    case AMDGPU::S_BARRIER_SIGNAL_ISFIRST_M0:
+    case AMDGPU::S_BARRIER_SIGNAL_ISFIRST_IMM:
+    case AMDGPU::S_BARRIER_LEAVE:
+    case AMDGPU::S_GET_BARRIER_STATE_M0:
+    case AMDGPU::S_GET_BARRIER_STATE_IMM:
       ScoreBrackets->updateByEvent(TII, TRI, MRI, SMEM_ACCESS, Inst);
       break;
     }
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index d4e4526795f3b36ce35f47df04a88015b6018a33..d4746b559d925667b7776f8a0add3a360dcdf19c 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -4118,7 +4118,8 @@ bool SIInstrInfo::isInlineConstant(const MachineOperand &MO,
   case AMDGPU::OPERAND_REG_IMM_V2INT32:
   case AMDGPU::OPERAND_REG_INLINE_C_V2INT32:
   case AMDGPU::OPERAND_REG_INLINE_AC_INT32:
-  case AMDGPU::OPERAND_REG_INLINE_AC_FP32: {
+  case AMDGPU::OPERAND_REG_INLINE_AC_FP32:
+  case AMDGPU::OPERAND_INLINE_SPLIT_BARRIER_INT32: {
     int32_t Trunc = static_cast<int32_t>(Imm);
     return AMDGPU::isInlinableLiteral32(Trunc, ST.hasInv2PiInlineImm());
   }
@@ -4559,6 +4560,12 @@ bool SIInstrInfo::verifyInstruction(const MachineInstr &MI,
       }
       break;
     }
+    case AMDGPU::OPERAND_INLINE_SPLIT_BARRIER_INT32:
+      if (!MI.getOperand(i).isImm() || !isInlineConstant(MI, i)) {
+        ErrInfo = "Expected inline constant for operand.";
+        return false;
+      }
+      break;
     case MCOI::OPERAND_IMMEDIATE:
     case AMDGPU::OPERAND_KIMM32:
       // Check if this operand is an immediate.
@@ -5255,11 +5262,15 @@ unsigned SIInstrInfo::getVALUOp(const MachineInstr &MI) const {
   case AMDGPU::S_SUB_F32: return AMDGPU::V_SUB_F32_e64;
   case AMDGPU::S_MIN_F32: return AMDGPU::V_MIN_F32_e64;
   case AMDGPU::S_MAX_F32: return AMDGPU::V_MAX_F32_e64;
+  case AMDGPU::S_MINIMUM_F32: return AMDGPU::V_MINIMUM_F32_e64;
+  case AMDGPU::S_MAXIMUM_F32: return AMDGPU::V_MAXIMUM_F32_e64;
   case AMDGPU::S_MUL_F32: return AMDGPU::V_MUL_F32_e64;
   case AMDGPU::S_ADD_F16: return AMDGPU::V_ADD_F16_fake16_e64;
   case AMDGPU::S_SUB_F16: return AMDGPU::V_SUB_F16_fake16_e64;
   case AMDGPU::S_MIN_F16: return AMDGPU::V_MIN_F16_fake16_e64;
   case AMDGPU::S_MAX_F16: return AMDGPU::V_MAX_F16_fake16_e64;
+  case AMDGPU::S_MINIMUM_F16: return AMDGPU::V_MINIMUM_F16_e64;
+  case AMDGPU::S_MAXIMUM_F16: return AMDGPU::V_MAXIMUM_F16_e64;
   case AMDGPU::S_MUL_F16: return AMDGPU::V_MUL_F16_fake16_e64;
   case AMDGPU::S_CVT_PK_RTZ_F16_F32: return AMDGPU::V_CVT_PKRTZ_F16_F32_e64;
   case AMDGPU::S_FMAC_F32: return AMDGPU::V_FMAC_F32_e64;
@@ -7101,6 +7112,26 @@ void SIInstrInfo::moveToVALUImpl(SIInstrWorklist &Worklist,
     Inst.eraseFromParent();
     return;
   }
+  case AMDGPU::S_MINIMUM_F32:
+  case AMDGPU::S_MAXIMUM_F32:
+  case AMDGPU::S_MINIMUM_F16:
+  case AMDGPU::S_MAXIMUM_F16: {
+    const DebugLoc &DL = Inst.getDebugLoc();
+    Register NewDst = MRI.createVirtualRegister(&AMDGPU::VGPR_32RegClass);
+    MachineInstr *NewInstr = BuildMI(*MBB, Inst, DL, get(NewOpcode), NewDst)
+                                 .addImm(0) // src0_modifiers
+                                 .add(Inst.getOperand(1))
+                                 .addImm(0) // src1_modifiers
+                                 .add(Inst.getOperand(2))
+                                 .addImm(0)  // clamp
+                                 .addImm(0); // omod
+    MRI.replaceRegWith(Inst.getOperand(0).getReg(), NewDst);
+
+    legalizeOperands(*NewInstr, MDT);
+    addUsersToMoveToVALUWorklist(NewDst, MRI, Worklist);
+    Inst.eraseFromParent();
+    return;
+  }
   }
 
   if (NewOpcode == AMDGPU::INSTRUCTION_LIST_END) {
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
index e794d8cf7cc220e6665a36b1485940a9d41e81c7..32a81d67cc246c3577b556cba29fa1395a2272c8 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h
@@ -546,6 +546,14 @@ public:
     return get(Opcode).TSFlags & SIInstrFlags::DS;
   }
 
+  static bool isLDSDMA(const MachineInstr &MI) {
+    return isVALU(MI) && (isMUBUF(MI) || isFLAT(MI));
+  }
+
+  bool isLDSDMA(uint16_t Opcode) {
+    return isVALU(Opcode) && (isMUBUF(Opcode) || isFLAT(Opcode));
+  }
+
   static bool isGWS(const MachineInstr &MI) {
     return MI.getDesc().TSFlags & SIInstrFlags::GWS;
   }
@@ -667,6 +675,10 @@ public:
                                   SIInstrFlags::IsAtomicNoRet);
   }
 
+  static bool mayWriteLDSThroughDMA(const MachineInstr &MI) {
+    return isLDSDMA(MI) && MI.getOpcode() != AMDGPU::BUFFER_STORE_LDS_DWORD;
+  }
+
   static bool isWQM(const MachineInstr &MI) {
     return MI.getDesc().TSFlags & SIInstrFlags::WQM;
   }
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.td b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
index 9e60bdda5ef3af2cedd18f0833ce4a86208bba80..6c106b8b68b5c4330953614342c846584a748b9f 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.td
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.td
@@ -919,6 +919,13 @@ def InterpAttr : CustomOperand<i32>;
 
 def InterpAttrChan : ImmOperand<i32>;
 
+def SplitBarrier : ImmOperand<i32> {
+  let OperandNamespace = "AMDGPU";
+  let OperandType = "OPERAND_INLINE_SPLIT_BARRIER_INT32";
+  let DecoderMethod = "decodeSplitBarrier";
+  let PrintMethod = "printOperand";
+}
+
 def VReg32OrOffClass : AsmOperandClass {
   let Name = "VReg32OrOff";
   let ParserMethod = "parseVReg32OrOff";
diff --git a/llvm/lib/Target/AMDGPU/SIInstructions.td b/llvm/lib/Target/AMDGPU/SIInstructions.td
index 9362fe5d9678b422ea3313519a069c5ce87f324b..f9bc623abcd04b1eee3cf2268a24d032ea6387b2 100644
--- a/llvm/lib/Target/AMDGPU/SIInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SIInstructions.td
@@ -3441,6 +3441,12 @@ defm : Int16Med3Pat<V_MED3_I16_e64, smin, smax>;
 defm : Int16Med3Pat<V_MED3_U16_e64, umin, umax>;
 } // End Predicates = [isGFX9Plus]
 
+let OtherPredicates = [isGFX12Plus] in {
+def : FPMinMaxPat<V_MINIMUMMAXIMUM_F32_e64, f32, DivergentBinFrag<fmaximum>, fminimum_oneuse>;
+def : FPMinMaxPat<V_MAXIMUMMINIMUM_F32_e64, f32, DivergentBinFrag<fminimum>, fmaximum_oneuse>;
+def : FPMinMaxPat<V_MINIMUMMAXIMUM_F16_e64, f16, DivergentBinFrag<fmaximum>, fminimum_oneuse>;
+def : FPMinMaxPat<V_MAXIMUMMINIMUM_F16_e64, f16, DivergentBinFrag<fminimum>, fmaximum_oneuse>;
+}
 
 // Convert a floating-point power of 2 to the integer exponent.
 def FPPow2ToExponentXForm : SDNodeXForm<fpimm, [{
diff --git a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp
index e0c23e40e19d82890298d3dedc1c4ecf42fab4de..48c341917ddec70b6b99c82b76bd1dbb5a7fca25 100644
--- a/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIMachineFunctionInfo.cpp
@@ -37,7 +37,7 @@ const GCNTargetMachine &getTM(const GCNSubtarget *STI) {
 
 SIMachineFunctionInfo::SIMachineFunctionInfo(const Function &F,
                                              const GCNSubtarget *STI)
-    : AMDGPUMachineFunction(F, *STI), Mode(F), GWSResourcePSV(getTM(STI)),
+    : AMDGPUMachineFunction(F, *STI), Mode(F, *STI), GWSResourcePSV(getTM(STI)),
       UserSGPRInfo(F, *STI), WorkGroupIDX(false), WorkGroupIDY(false),
       WorkGroupIDZ(false), WorkGroupInfo(false), LDSKernelId(false),
       PrivateSegmentWaveByteOffset(false), WorkItemIDX(false),
diff --git a/llvm/lib/Target/AMDGPU/SIModeRegisterDefaults.cpp b/llvm/lib/Target/AMDGPU/SIModeRegisterDefaults.cpp
index ffed268244eda02c614e62045abc8bd9dbb25a49..2684a1e3c3358a3bdb0dccd89f6879fde0984b2d 100644
--- a/llvm/lib/Target/AMDGPU/SIModeRegisterDefaults.cpp
+++ b/llvm/lib/Target/AMDGPU/SIModeRegisterDefaults.cpp
@@ -7,20 +7,26 @@
 //===----------------------------------------------------------------------===//
 
 #include "SIModeRegisterDefaults.h"
+#include "GCNSubtarget.h"
 
 using namespace llvm;
 
-SIModeRegisterDefaults::SIModeRegisterDefaults(const Function &F) {
+SIModeRegisterDefaults::SIModeRegisterDefaults(const Function &F,
+                                               const GCNSubtarget &ST) {
   *this = getDefaultForCallingConv(F.getCallingConv());
 
-  StringRef IEEEAttr = F.getFnAttribute("amdgpu-ieee").getValueAsString();
-  if (!IEEEAttr.empty())
-    IEEE = IEEEAttr == "true";
+  if (ST.hasIEEEMode()) {
+    StringRef IEEEAttr = F.getFnAttribute("amdgpu-ieee").getValueAsString();
+    if (!IEEEAttr.empty())
+      IEEE = IEEEAttr == "true";
+  }
 
-  StringRef DX10ClampAttr =
-      F.getFnAttribute("amdgpu-dx10-clamp").getValueAsString();
-  if (!DX10ClampAttr.empty())
-    DX10Clamp = DX10ClampAttr == "true";
+  if (ST.hasDX10ClampMode()) {
+    StringRef DX10ClampAttr =
+        F.getFnAttribute("amdgpu-dx10-clamp").getValueAsString();
+    if (!DX10ClampAttr.empty())
+      DX10Clamp = DX10ClampAttr == "true";
+  }
 
   StringRef DenormF32Attr =
       F.getFnAttribute("denormal-fp-math-f32").getValueAsString();
diff --git a/llvm/lib/Target/AMDGPU/SIModeRegisterDefaults.h b/llvm/lib/Target/AMDGPU/SIModeRegisterDefaults.h
index 58e2c67c248bd73b3a6c3ca5c8e2f2afed3cfe9a..9fbd74c3eede3207b7588dd3b26e95a0164c6984 100644
--- a/llvm/lib/Target/AMDGPU/SIModeRegisterDefaults.h
+++ b/llvm/lib/Target/AMDGPU/SIModeRegisterDefaults.h
@@ -14,6 +14,8 @@
 
 namespace llvm {
 
+class GCNSubtarget;
+
 // Track defaults for fields in the MODE register.
 struct SIModeRegisterDefaults {
   /// Floating point opcodes that support exception flag gathering quiet and
@@ -40,7 +42,7 @@ struct SIModeRegisterDefaults {
     FP32Denormals(DenormalMode::getIEEE()),
     FP64FP16Denormals(DenormalMode::getIEEE()) {}
 
-  SIModeRegisterDefaults(const Function &F);
+  SIModeRegisterDefaults(const Function &F, const GCNSubtarget &ST);
 
   static SIModeRegisterDefaults getDefaultForCallingConv(CallingConv::ID CC) {
     SIModeRegisterDefaults Mode;
diff --git a/llvm/lib/Target/AMDGPU/SIProgramInfo.cpp b/llvm/lib/Target/AMDGPU/SIProgramInfo.cpp
index b6839c8308d8e876652f0ead5d4e62db0695020b..9ed7aacc0538ec155727ed9e24570db5beb08247 100644
--- a/llvm/lib/Target/AMDGPU/SIProgramInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIProgramInfo.cpp
@@ -15,27 +15,48 @@
 //
 
 #include "SIProgramInfo.h"
+#include "GCNSubtarget.h"
 #include "SIDefines.h"
 #include "Utils/AMDGPUBaseInfo.h"
 
 using namespace llvm;
 
-uint64_t SIProgramInfo::getComputePGMRSrc1() const {
-  return S_00B848_VGPRS(VGPRBlocks) | S_00B848_SGPRS(SGPRBlocks) |
-         S_00B848_PRIORITY(Priority) | S_00B848_FLOAT_MODE(FloatMode) |
-         S_00B848_PRIV(Priv) | S_00B848_DX10_CLAMP(DX10Clamp) |
-         S_00B848_DEBUG_MODE(DebugMode) | S_00B848_IEEE_MODE(IEEEMode) |
-         S_00B848_WGP_MODE(WgpMode) | S_00B848_MEM_ORDERED(MemOrdered);
+uint64_t SIProgramInfo::getComputePGMRSrc1(const GCNSubtarget &ST) const {
+  uint64_t Reg = S_00B848_VGPRS(VGPRBlocks) | S_00B848_SGPRS(SGPRBlocks) |
+                 S_00B848_PRIORITY(Priority) | S_00B848_FLOAT_MODE(FloatMode) |
+                 S_00B848_PRIV(Priv) | S_00B848_DEBUG_MODE(DebugMode) |
+                 S_00B848_WGP_MODE(WgpMode) | S_00B848_MEM_ORDERED(MemOrdered);
+
+  if (ST.hasDX10ClampMode())
+    Reg |= S_00B848_DX10_CLAMP(DX10Clamp);
+
+  if (ST.hasIEEEMode())
+    Reg |= S_00B848_IEEE_MODE(IEEEMode);
+
+  if (ST.hasRrWGMode())
+    Reg |= S_00B848_RR_WG_MODE(RrWgMode);
+
+  return Reg;
 }
 
-uint64_t SIProgramInfo::getPGMRSrc1(CallingConv::ID CC) const {
+uint64_t SIProgramInfo::getPGMRSrc1(CallingConv::ID CC,
+                                    const GCNSubtarget &ST) const {
   if (AMDGPU::isCompute(CC)) {
-    return getComputePGMRSrc1();
+    return getComputePGMRSrc1(ST);
   }
   uint64_t Reg = S_00B848_VGPRS(VGPRBlocks) | S_00B848_SGPRS(SGPRBlocks) |
                  S_00B848_PRIORITY(Priority) | S_00B848_FLOAT_MODE(FloatMode) |
-                 S_00B848_PRIV(Priv) | S_00B848_DX10_CLAMP(DX10Clamp) |
-                 S_00B848_DEBUG_MODE(DebugMode) | S_00B848_IEEE_MODE(IEEEMode);
+                 S_00B848_PRIV(Priv) | S_00B848_DEBUG_MODE(DebugMode);
+
+  if (ST.hasDX10ClampMode())
+    Reg |= S_00B848_DX10_CLAMP(DX10Clamp);
+
+  if (ST.hasIEEEMode())
+    Reg |= S_00B848_IEEE_MODE(IEEEMode);
+
+  if (ST.hasRrWGMode())
+    Reg |= S_00B848_RR_WG_MODE(RrWgMode);
+
   switch (CC) {
   case CallingConv::AMDGPU_PS:
     Reg |= S_00B028_MEM_ORDERED(MemOrdered);
diff --git a/llvm/lib/Target/AMDGPU/SIProgramInfo.h b/llvm/lib/Target/AMDGPU/SIProgramInfo.h
index aab127e494630adbb12179bcf279db1ffa84db4a..8c26789f936cff4e11861b7ab414b2d82fcf5761 100644
--- a/llvm/lib/Target/AMDGPU/SIProgramInfo.h
+++ b/llvm/lib/Target/AMDGPU/SIProgramInfo.h
@@ -21,6 +21,8 @@
 
 namespace llvm {
 
+class GCNSubtarget;
+
 /// Track resource usage for kernels / entry functions.
 struct SIProgramInfo {
     // Fields set in PGM_RSRC1 pm4 packet.
@@ -34,6 +36,7 @@ struct SIProgramInfo {
     uint32_t IEEEMode = 0;
     uint32_t WgpMode = 0; // GFX10+
     uint32_t MemOrdered = 0; // GFX10+
+    uint32_t RrWgMode = 0;   // GFX12+
     uint64_t ScratchSize = 0;
 
     // State used to calculate fields set in PGM_RSRC2 pm4 packet.
@@ -85,8 +88,8 @@ struct SIProgramInfo {
     SIProgramInfo() = default;
 
     /// Compute the value of the ComputePGMRsrc1 register.
-    uint64_t getComputePGMRSrc1() const;
-    uint64_t getPGMRSrc1(CallingConv::ID CC) const;
+    uint64_t getComputePGMRSrc1(const GCNSubtarget &ST) const;
+    uint64_t getPGMRSrc1(CallingConv::ID CC, const GCNSubtarget &ST) const;
 
     /// Compute the value of the ComputePGMRsrc2 register.
     uint64_t getComputePGMRSrc2() const;
diff --git a/llvm/lib/Target/AMDGPU/SOPInstructions.td b/llvm/lib/Target/AMDGPU/SOPInstructions.td
index 9ff64968ef01b2c6cb2fb6b8896dbfd97b0c0ed6..50c4d279cfe23ddd168ad6eee68d87b968c9d581 100644
--- a/llvm/lib/Target/AMDGPU/SOPInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SOPInstructions.td
@@ -438,6 +438,89 @@ let SubtargetPredicate = HasSALUFloatInsts, Uses = [MODE],
 } // End SubtargetPredicate = HasSALUFloatInsts, Uses = [MODE]
   // SchedRW = [WriteSFPU], isReMaterializable = 1
 
+let hasSideEffects = 1 in {
+let has_sdst = 0 in {
+let Uses = [M0] in {
+def S_BARRIER_SIGNAL_M0 : SOP1_Pseudo <"s_barrier_signal m0", (outs), (ins),
+  "", [(int_amdgcn_s_barrier_signal_var M0)]>{
+  let SchedRW = [WriteBarrier];
+  let isConvergent = 1;
+}
+
+def S_BARRIER_SIGNAL_ISFIRST_M0 : SOP1_Pseudo <"s_barrier_signal_isfirst m0", (outs), (ins),
+  "", [(set SCC, (int_amdgcn_s_barrier_signal_isfirst_var M0))]>{
+  let Defs = [SCC];
+  let SchedRW = [WriteBarrier];
+  let isConvergent = 1;
+}
+
+def S_BARRIER_INIT_M0 : SOP1_Pseudo <"s_barrier_init m0", (outs), (ins),
+  "", []>{
+  let SchedRW = [WriteBarrier];
+  let isConvergent = 1;
+}
+
+def S_BARRIER_INIT_IMM : SOP1_Pseudo <"s_barrier_init", (outs),
+  (ins SplitBarrier:$src0), "$src0", []>{
+  let SchedRW = [WriteBarrier];
+  let isConvergent = 1;
+}
+
+def S_BARRIER_JOIN_M0 : SOP1_Pseudo <"s_barrier_join m0", (outs), (ins),
+  "", []>{
+  let SchedRW = [WriteBarrier];
+  let isConvergent = 1;
+}
+
+def S_WAKEUP_BARRIER_M0 : SOP1_Pseudo <"s_wakeup_barrier m0", (outs), (ins),
+  "", []>{
+  let SchedRW = [WriteBarrier];
+  let isConvergent = 1;
+}
+} // End Uses = [M0]
+
+def S_BARRIER_SIGNAL_IMM : SOP1_Pseudo <"s_barrier_signal", (outs),
+  (ins SplitBarrier:$src0), "$src0", [(int_amdgcn_s_barrier_signal timm:$src0)]>{
+  let SchedRW = [WriteBarrier];
+  let isConvergent = 1;
+}
+
+def S_BARRIER_SIGNAL_ISFIRST_IMM : SOP1_Pseudo <"s_barrier_signal_isfirst", (outs),
+  (ins SplitBarrier:$src0), "$src0", [(set SCC, (int_amdgcn_s_barrier_signal_isfirst timm:$src0))]>{
+  let Defs = [SCC];
+  let SchedRW = [WriteBarrier];
+  let isConvergent = 1;
+}
+
+def S_BARRIER_JOIN_IMM : SOP1_Pseudo <"s_barrier_join", (outs),
+  (ins SplitBarrier:$src0), "$src0", []>{
+  let SchedRW = [WriteBarrier];
+  let isConvergent = 1;
+}
+
+def S_WAKEUP_BARRIER_IMM : SOP1_Pseudo <"s_wakeup_barrier", (outs),
+  (ins SplitBarrier:$src0), "$src0", []>{
+  let SchedRW = [WriteBarrier];
+  let isConvergent = 1;
+
+
+}
+} // End has_sdst = 0
+
+def S_GET_BARRIER_STATE_IMM : SOP1_Pseudo <"s_get_barrier_state", (outs SSrc_b32:$sdst),
+  (ins SplitBarrier:$src0), "$sdst, $src0", []>{
+  let SchedRW = [WriteBarrier];
+  let isConvergent = 1;
+}
+
+def S_GET_BARRIER_STATE_M0 : SOP1_Pseudo <"s_get_barrier_state $sdst, m0", (outs SSrc_b32:$sdst),
+  (ins), "", []>{
+  let Uses = [M0];
+  let SchedRW = [WriteBarrier];
+  let isConvergent = 1;
+}
+} // End hasSideEffects = 1
+
 //===----------------------------------------------------------------------===//
 // SOP2 Instructions
 //===----------------------------------------------------------------------===//
@@ -847,6 +930,15 @@ let SubtargetPredicate = HasSALUFloatInsts, mayRaiseFPException = 1,
 } // End SubtargetPredicate = HasSALUFloatInsts, mayRaiseFPException = 1,
   // Uses = [MODE], SchedRW = [WriteSFPU]
 
+// On GFX12 MIN/MAX instructions do not read MODE register.
+let SubtargetPredicate = isGFX12Plus, mayRaiseFPException = 1, isCommutable = 1,
+    isReMaterializable = 1, SchedRW = [WriteSFPU] in {
+  def S_MINIMUM_F32 : SOP2_F32_Inst<"s_minimum_f32", fminimum>;
+  def S_MAXIMUM_F32 : SOP2_F32_Inst<"s_maximum_f32", fmaximum>;
+  def S_MINIMUM_F16 : SOP2_F16_Inst<"s_minimum_f16", fminimum>;
+  def S_MAXIMUM_F16 : SOP2_F16_Inst<"s_maximum_f16", fmaximum>;
+}
+
 //===----------------------------------------------------------------------===//
 // SOPK Instructions
 //===----------------------------------------------------------------------===//
@@ -1473,6 +1565,21 @@ def S_BARRIER : SOPP_Pseudo <"s_barrier", (ins), "",
   let isConvergent = 1;
 }
 
+def S_BARRIER_WAIT : SOPP_Pseudo <"s_barrier_wait", (ins i16imm:$simm16), "$simm16",
+  [(int_amdgcn_s_barrier_wait timm:$simm16)]> {
+  let SchedRW = [WriteBarrier];
+  let isConvergent = 1;
+}
+
+def S_BARRIER_LEAVE : SOPP_Pseudo <"s_barrier_leave", (ins), "",
+  [(set SCC, (int_amdgcn_s_barrier_leave))]> {
+  let SchedRW = [WriteBarrier];
+  let simm16 = 0;
+  let fixed_imm = 1;
+  let isConvergent = 1;
+  let Defs = [SCC];
+}
+
 def S_WAKEUP : SOPP_Pseudo <"s_wakeup", (ins) > {
   let SubtargetPredicate = isGFX8Plus;
   let simm16 = 0;
@@ -1878,6 +1985,18 @@ defm S_SWAPPC_B64                 : SOP1_Real_gfx11_gfx12<0x049>;
 defm S_RFE_B64                    : SOP1_Real_gfx11_gfx12<0x04a>;
 defm S_SENDMSG_RTN_B32            : SOP1_Real_gfx11_gfx12<0x04c>;
 defm S_SENDMSG_RTN_B64            : SOP1_Real_gfx11_gfx12<0x04d>;
+defm S_BARRIER_SIGNAL_M0          : SOP1_M0_Real_gfx12<0x04e>;
+defm S_BARRIER_SIGNAL_ISFIRST_M0  : SOP1_M0_Real_gfx12<0x04f>;
+defm S_GET_BARRIER_STATE_M0       : SOP1_M0_Real_gfx12<0x050>;
+defm S_BARRIER_INIT_M0            : SOP1_M0_Real_gfx12<0x051>;
+defm S_BARRIER_JOIN_M0            : SOP1_M0_Real_gfx12<0x052>;
+defm S_WAKEUP_BARRIER_M0          : SOP1_M0_Real_gfx12<0x057>;
+defm S_BARRIER_SIGNAL_IMM         : SOP1_Real_gfx12<0x04e>;
+defm S_BARRIER_SIGNAL_ISFIRST_IMM : SOP1_Real_gfx12<0x04f>;
+defm S_GET_BARRIER_STATE_IMM      : SOP1_Real_gfx12<0x050>;
+defm S_BARRIER_INIT_IMM           : SOP1_Real_gfx12<0x051>;
+defm S_BARRIER_JOIN_IMM           : SOP1_Real_gfx12<0x052>;
+defm S_WAKEUP_BARRIER_IMM         : SOP1_Real_gfx12<0x057>;
 
 //===----------------------------------------------------------------------===//
 // SOP1 - GFX1150, GFX12
@@ -2017,6 +2136,10 @@ defm S_MIN_NUM_F32 : SOP2_Real_Renamed_gfx12<0x042, S_MIN_F32, "s_min_num_f32">;
 defm S_MAX_NUM_F32 : SOP2_Real_Renamed_gfx12<0x043, S_MAX_F32, "s_max_num_f32">;
 defm S_MIN_NUM_F16 : SOP2_Real_Renamed_gfx12<0x04b, S_MIN_F16, "s_min_num_f16">;
 defm S_MAX_NUM_F16 : SOP2_Real_Renamed_gfx12<0x04c, S_MAX_F16, "s_max_num_f16">;
+defm S_MINIMUM_F32 : SOP2_Real_gfx12<0x04f>;
+defm S_MAXIMUM_F32 : SOP2_Real_gfx12<0x050>;
+defm S_MINIMUM_F16 : SOP2_Real_gfx12<0x051>;
+defm S_MAXIMUM_F16 : SOP2_Real_gfx12<0x052>;
 
 defm S_ADD_CO_U32    : SOP2_Real_Renamed_gfx12<0x000, S_ADD_U32, "s_add_co_u32">;
 defm S_SUB_CO_U32    : SOP2_Real_Renamed_gfx12<0x001, S_SUB_U32, "s_sub_co_u32">;
@@ -2365,6 +2488,8 @@ multiclass SOPP_Real_32_Renamed_gfx12<bits<7> op, SOPP_Pseudo backing_pseudo, st
 }
 
 defm S_WAIT_ALU             : SOPP_Real_32_Renamed_gfx12<0x008, S_WAITCNT_DEPCTR, "s_wait_alu">;
+defm S_BARRIER_WAIT         : SOPP_Real_32_gfx12<0x014>;
+defm S_BARRIER_LEAVE        : SOPP_Real_32_gfx12<0x015>;
 
 //===----------------------------------------------------------------------===//
 // SOPP - GFX11, GFX12.
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
index ede70c3f3404813a811afd9ef63ac747c75bf47c..4edd7960bd8c4069e3786e50669e701459911383 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.cpp
@@ -1147,10 +1147,17 @@ amdhsa::kernel_descriptor_t getDefaultAmdhsaKernelDescriptor(
   AMDHSA_BITS_SET(KD.compute_pgm_rsrc1,
                   amdhsa::COMPUTE_PGM_RSRC1_FLOAT_DENORM_MODE_16_64,
                   amdhsa::FLOAT_DENORM_MODE_FLUSH_NONE);
-  AMDHSA_BITS_SET(KD.compute_pgm_rsrc1,
-                  amdhsa::COMPUTE_PGM_RSRC1_ENABLE_DX10_CLAMP, 1);
-  AMDHSA_BITS_SET(KD.compute_pgm_rsrc1,
-                  amdhsa::COMPUTE_PGM_RSRC1_ENABLE_IEEE_MODE, 1);
+  if (Version.Major >= 12) {
+    AMDHSA_BITS_SET(KD.compute_pgm_rsrc1,
+                    amdhsa::COMPUTE_PGM_RSRC1_GFX12_PLUS_ENABLE_WG_RR_EN, 0);
+    AMDHSA_BITS_SET(KD.compute_pgm_rsrc1,
+                    amdhsa::COMPUTE_PGM_RSRC1_GFX12_PLUS_DISABLE_PERF, 0);
+  } else {
+    AMDHSA_BITS_SET(KD.compute_pgm_rsrc1,
+                    amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_DX10_CLAMP, 1);
+    AMDHSA_BITS_SET(KD.compute_pgm_rsrc1,
+                    amdhsa::COMPUTE_PGM_RSRC1_GFX6_GFX11_ENABLE_IEEE_MODE, 1);
+  }
   AMDHSA_BITS_SET(KD.compute_pgm_rsrc2,
                   amdhsa::COMPUTE_PGM_RSRC2_ENABLE_SGPR_WORKGROUP_ID_X, 1);
   if (Version.Major >= 10) {
@@ -2684,10 +2691,11 @@ std::optional<int64_t> getSMRDEncodedLiteralOffset32(const MCSubtargetInfo &ST,
 }
 
 unsigned getNumFlatOffsetBits(const MCSubtargetInfo &ST) {
-  // Address offset is 12-bit signed for GFX10, 13-bit for GFX9 and GFX11+.
   if (AMDGPU::isGFX10(ST))
     return 12;
 
+  if (AMDGPU::isGFX12(ST))
+    return 24;
   return 13;
 }
 
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
index bd333ed2cca71c3527a86c92fd0194252eef0e1a..3c9f330cbcded9b3b5165f9b5b5d5da385397e3a 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h
@@ -1238,6 +1238,7 @@ inline unsigned getOperandSize(const MCOperandInfo &OpInfo) {
   case AMDGPU::OPERAND_REG_INLINE_C_V2FP32:
   case AMDGPU::OPERAND_KIMM32:
   case AMDGPU::OPERAND_KIMM16: // mandatory literal is always size 4
+  case AMDGPU::OPERAND_INLINE_SPLIT_BARRIER_INT32:
     return 4;
 
   case AMDGPU::OPERAND_REG_IMM_INT64:
@@ -1333,7 +1334,7 @@ std::optional<int64_t> getSMRDEncodedOffset(const MCSubtargetInfo &ST,
 std::optional<int64_t> getSMRDEncodedLiteralOffset32(const MCSubtargetInfo &ST,
                                                      int64_t ByteOffset);
 
-/// For FLAT segment the offset must be positive;
+/// For pre-GFX12 FLAT instructions the offset must be positive;
 /// MSB is ignored and forced to zero.
 ///
 /// \return The number of bits available for the signed offset field in flat
diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUMemoryUtils.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUMemoryUtils.cpp
index cbdbf1c16f9f0af3c76c543c48f0dd94617a48c5..25e628e5cbc558b60d352d9a2a218bcd787ccfb9 100644
--- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUMemoryUtils.cpp
+++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUMemoryUtils.cpp
@@ -74,6 +74,16 @@ bool isReallyAClobber(const Value *Ptr, MemoryDef *Def, AAResults *AA) {
   if (const IntrinsicInst *II = dyn_cast<IntrinsicInst>(DefInst)) {
     switch (II->getIntrinsicID()) {
     case Intrinsic::amdgcn_s_barrier:
+    case Intrinsic::amdgcn_s_barrier_signal:
+    case Intrinsic::amdgcn_s_barrier_signal_var:
+    case Intrinsic::amdgcn_s_barrier_signal_isfirst:
+    case Intrinsic::amdgcn_s_barrier_signal_isfirst_var:
+    case Intrinsic::amdgcn_s_barrier_init:
+    case Intrinsic::amdgcn_s_barrier_join:
+    case Intrinsic::amdgcn_s_barrier_wait:
+    case Intrinsic::amdgcn_s_barrier_leave:
+    case Intrinsic::amdgcn_s_get_barrier_state:
+    case Intrinsic::amdgcn_s_wakeup_barrier:
     case Intrinsic::amdgcn_wave_barrier:
     case Intrinsic::amdgcn_sched_barrier:
     case Intrinsic::amdgcn_sched_group_barrier:
diff --git a/llvm/lib/Target/AMDGPU/VOP3Instructions.td b/llvm/lib/Target/AMDGPU/VOP3Instructions.td
index 617773b34ae98cc23c28717e619abb52bd5dfb7b..685c9ac6a2be40ccd45318575facbd6d33809c8c 100644
--- a/llvm/lib/Target/AMDGPU/VOP3Instructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP3Instructions.td
@@ -161,6 +161,19 @@ defm V_MUL_HI_U32 : VOP3Inst <"v_mul_hi_u32", V_MUL_PROF<VOP_I32_I32_I32>, mulhu
 defm V_MUL_LO_I32 : VOP3Inst <"v_mul_lo_i32", V_MUL_PROF<VOP_I32_I32_I32>>;
 defm V_MUL_HI_I32 : VOP3Inst <"v_mul_hi_i32", V_MUL_PROF<VOP_I32_I32_I32>, mulhs>;
 } // End SchedRW = [WriteIntMul]
+
+let SubtargetPredicate = isGFX12Plus, ReadsModeReg = 0 in {
+defm V_MINIMUM_F32 : VOP3Inst <"v_minimum_f32", VOP3_Profile<VOP_F32_F32_F32>, DivergentBinFrag<fminimum>>;
+defm V_MAXIMUM_F32 : VOP3Inst <"v_maximum_f32", VOP3_Profile<VOP_F32_F32_F32>, DivergentBinFrag<fmaximum>>;
+defm V_MINIMUM_F16 : VOP3Inst <"v_minimum_f16", VOP3_Profile<VOP_F16_F16_F16>, DivergentBinFrag<fminimum>>;
+defm V_MAXIMUM_F16 : VOP3Inst <"v_maximum_f16", VOP3_Profile<VOP_F16_F16_F16>, DivergentBinFrag<fmaximum>>;
+
+let SchedRW = [WriteDoubleAdd] in {
+defm V_MINIMUM_F64 : VOP3Inst <"v_minimum_f64", VOP3_Profile<VOP_F64_F64_F64>, fminimum>;
+defm V_MAXIMUM_F64 : VOP3Inst <"v_maximum_f64", VOP3_Profile<VOP_F64_F64_F64>, fmaximum>;
+} // End SchedRW = [WriteDoubleAdd]
+} // End SubtargetPredicate = isGFX12Plus, ReadsModeReg = 0
+
 } // End isReMaterializable = 1
 
 let Uses = [MODE, VCC, EXEC] in {
@@ -211,6 +224,11 @@ let mayRaiseFPException = 0 in {
   defm V_MED3_F32 : VOP3Inst <"v_med3_f32", VOP3_Profile<VOP_F32_F32_F32_F32>, AMDGPUfmed3>;
 } // End mayRaiseFPException = 0
 
+let SubtargetPredicate = isGFX12Plus, ReadsModeReg = 0 in {
+  defm V_MINIMUM3_F32 : VOP3Inst <"v_minimum3_f32", VOP3_Profile<VOP_F32_F32_F32_F32>, AMDGPUfminimum3>;
+  defm V_MAXIMUM3_F32 : VOP3Inst <"v_maximum3_f32", VOP3_Profile<VOP_F32_F32_F32_F32>, AMDGPUfmaximum3>;
+} // End SubtargetPredicate = isGFX12Plus, ReadsModeReg = 0
+
 let isCommutable = 1 in {
   defm V_SAD_U8 : VOP3Inst <"v_sad_u8", VOP3_Profile<VOP_I32_I32_I32_I32, VOP3_CLAMP>>;
   defm V_SAD_HI_U8 : VOP3Inst <"v_sad_hi_u8", VOP3_Profile<VOP_I32_I32_I32_I32, VOP3_CLAMP>>;
@@ -555,6 +573,11 @@ defm V_MAX3_F16 : VOP3Inst <"v_max3_f16", VOP3_Profile<VOP_F16_F16_F16_F16, VOP3
 defm V_MAX3_I16 : VOP3Inst <"v_max3_i16", VOP3_Profile<VOP_I16_I16_I16_I16, VOP3_OPSEL>, AMDGPUsmax3>;
 defm V_MAX3_U16 : VOP3Inst <"v_max3_u16", VOP3_Profile<VOP_I16_I16_I16_I16, VOP3_OPSEL>, AMDGPUumax3>;
 
+let SubtargetPredicate = isGFX12Plus, ReadsModeReg = 0 in {
+  defm V_MINIMUM3_F16 : VOP3Inst <"v_minimum3_f16", VOP3_Profile<VOP_F16_F16_F16_F16, VOP3_OPSEL>, AMDGPUfminimum3>;
+  defm V_MAXIMUM3_F16 : VOP3Inst <"v_maximum3_f16", VOP3_Profile<VOP_F16_F16_F16_F16, VOP3_OPSEL>, AMDGPUfmaximum3>;
+} // End SubtargetPredicate = isGFX12Plus, ReadsModeReg = 0
+
 defm V_ADD_I16 : VOP3Inst <"v_add_i16", VOP3_Profile<VOP_I16_I16_I16, VOP3_OPSEL>>;
 defm V_SUB_I16 : VOP3Inst <"v_sub_i16", VOP3_Profile<VOP_I16_I16_I16, VOP3_OPSEL>>;
 
@@ -805,6 +828,13 @@ let SubtargetPredicate = isGFX11Plus in {
   defm V_CVT_PK_U16_F32 : VOP3Inst<"v_cvt_pk_u16_f32", VOP3_Profile<VOP_V2I16_F32_F32>>;
 } // End SubtargetPredicate = isGFX11Plus
 
+let SubtargetPredicate = isGFX12Plus, ReadsModeReg = 0 in {
+  defm V_MAXIMUMMINIMUM_F32 : VOP3Inst<"v_maximumminimum_f32", VOP3_Profile<VOP_F32_F32_F32_F32>>;
+  defm V_MINIMUMMAXIMUM_F32 : VOP3Inst<"v_minimummaximum_f32", VOP3_Profile<VOP_F32_F32_F32_F32>>;
+  defm V_MAXIMUMMINIMUM_F16 : VOP3Inst<"v_maximumminimum_f16", VOP3_Profile<VOP_F16_F16_F16_F16, VOP3_OPSEL>>;
+  defm V_MINIMUMMAXIMUM_F16 : VOP3Inst<"v_minimummaximum_f16", VOP3_Profile<VOP_F16_F16_F16_F16, VOP3_OPSEL>>;
+} // End SubtargetPredicate = isGFX12Plus, ReadsModeReg = 0
+
 let SubtargetPredicate = HasDot9Insts, IsDOT=1 in {
   defm V_DOT2_F16_F16 :   VOP3Inst<"v_dot2_f16_f16",   VOP3_DOT_Profile<VOP_F16_V2F16_V2F16_F16>, int_amdgcn_fdot2_f16_f16>;
   defm V_DOT2_BF16_BF16 : VOP3Inst<"v_dot2_bf16_bf16", VOP3_DOT_Profile<VOP_I16_V2I16_V2I16_I16>, int_amdgcn_fdot2_bf16_bf16>;
@@ -862,14 +892,28 @@ defm V_MIN3_NUM_F32       : VOP3_Realtriple_with_name_gfx12<0x229, "V_MIN3_F32",
 defm V_MAX3_NUM_F32       : VOP3_Realtriple_with_name_gfx12<0x22a, "V_MAX3_F32", "v_max3_num_f32">;
 defm V_MIN3_NUM_F16       : VOP3_Realtriple_with_name_gfx12<0x22b, "V_MIN3_F16", "v_min3_num_f16">;
 defm V_MAX3_NUM_F16       : VOP3_Realtriple_with_name_gfx12<0x22c, "V_MAX3_F16", "v_max3_num_f16">;
+defm V_MINIMUM3_F32       : VOP3Only_Realtriple_gfx12<0x22d>;
+defm V_MAXIMUM3_F32       : VOP3Only_Realtriple_gfx12<0x22e>;
+defm V_MINIMUM3_F16       : VOP3Only_Realtriple_t16_gfx12<0x22f>;
+defm V_MAXIMUM3_F16       : VOP3Only_Realtriple_t16_gfx12<0x230>;
 defm V_MED3_NUM_F32       : VOP3_Realtriple_with_name_gfx12<0x231, "V_MED3_F32", "v_med3_num_f32">;
 defm V_MED3_NUM_F16       : VOP3_Realtriple_with_name_gfx12<0x232, "V_MED3_F16", "v_med3_num_f16">;
 defm V_MINMAX_NUM_F32     : VOP3_Realtriple_with_name_gfx12<0x268, "V_MINMAX_F32", "v_minmax_num_f32">;
 defm V_MAXMIN_NUM_F32     : VOP3_Realtriple_with_name_gfx12<0x269, "V_MAXMIN_F32", "v_maxmin_num_f32">;
 defm V_MINMAX_NUM_F16     : VOP3_Realtriple_with_name_gfx12<0x26a, "V_MINMAX_F16", "v_minmax_num_f16">;
 defm V_MAXMIN_NUM_F16     : VOP3_Realtriple_with_name_gfx12<0x26b, "V_MAXMIN_F16", "v_maxmin_num_f16">;
+defm V_MINIMUMMAXIMUM_F32 : VOP3Only_Realtriple_gfx12<0x26c>;
+defm V_MAXIMUMMINIMUM_F32 : VOP3Only_Realtriple_gfx12<0x26d>;
+defm V_MINIMUMMAXIMUM_F16 : VOP3Only_Realtriple_t16_gfx12<0x26e>;
+defm V_MAXIMUMMINIMUM_F16 : VOP3Only_Realtriple_t16_gfx12<0x26f>;
 defm V_MAD_CO_U64_U32     : VOP3be_Real_with_name_gfx12<0x2fe, "V_MAD_U64_U32", "v_mad_co_u64_u32">;
 defm V_MAD_CO_I64_I32     : VOP3be_Real_with_name_gfx12<0x2ff, "V_MAD_I64_I32", "v_mad_co_i64_i32">;
+defm V_MINIMUM_F64        : VOP3Only_Real_Base_gfx12<0x341>;
+defm V_MAXIMUM_F64        : VOP3Only_Real_Base_gfx12<0x342>;
+defm V_MINIMUM_F32        : VOP3Only_Realtriple_gfx12<0x365>;
+defm V_MAXIMUM_F32        : VOP3Only_Realtriple_gfx12<0x366>;
+defm V_MINIMUM_F16        : VOP3Only_Realtriple_t16_gfx12<0x367>;
+defm V_MAXIMUM_F16        : VOP3Only_Realtriple_t16_gfx12<0x368>;
 
 //===----------------------------------------------------------------------===//
 // GFX11, GFX12
diff --git a/llvm/lib/Target/AMDGPU/VOP3PInstructions.td b/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
index eb8990238268d3c713561163d2665b29576859b0..d3cefb339d9e704c05604fa1f0f129a38547de9a 100644
--- a/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
+++ b/llvm/lib/Target/AMDGPU/VOP3PInstructions.td
@@ -108,6 +108,11 @@ defm V_PK_MIN_I16 : VOP3PInst<"v_pk_min_i16", VOP3P_Profile<VOP_V2I16_V2I16_V2I1
 defm V_PK_MIN_U16 : VOP3PInst<"v_pk_min_u16", VOP3P_Profile<VOP_V2I16_V2I16_V2I16>, umin>;
 defm V_PK_MAX_I16 : VOP3PInst<"v_pk_max_i16", VOP3P_Profile<VOP_V2I16_V2I16_V2I16>, smax>;
 defm V_PK_MAX_U16 : VOP3PInst<"v_pk_max_u16", VOP3P_Profile<VOP_V2I16_V2I16_V2I16>, umax>;
+
+let SubtargetPredicate = isGFX12Plus, ReadsModeReg = 0 in {
+defm V_PK_MAXIMUM_F16 : VOP3PInst<"v_pk_maximum_f16", VOP3P_Profile<VOP_V2F16_V2F16_V2F16>, fmaximum>;
+defm V_PK_MINIMUM_F16 : VOP3PInst<"v_pk_minimum_f16", VOP3P_Profile<VOP_V2F16_V2F16_V2F16>, fminimum>;
+} // End SubtargetPredicate = isGFX12Plus, ReadsModeReg = 0
 }
 
 defm V_PK_SUB_U16 : VOP3PInst<"v_pk_sub_u16", VOP3P_Profile<VOP_V2I16_V2I16_V2I16>>;
@@ -1003,6 +1008,9 @@ multiclass VOP3P_Real_with_name_gfx12<bits<7> op,
 defm V_PK_MIN_NUM_F16 : VOP3P_Real_with_name_gfx12<0x1b, "V_PK_MIN_F16", "v_pk_min_num_f16">;
 defm V_PK_MAX_NUM_F16 : VOP3P_Real_with_name_gfx12<0x1c, "V_PK_MAX_F16", "v_pk_max_num_f16">;
 
+defm V_PK_MINIMUM_F16 : VOP3P_Real_gfx12<0x1d>;
+defm V_PK_MAXIMUM_F16 : VOP3P_Real_gfx12<0x1e>;
+
 //===----------------------------------------------------------------------===//
 // GFX11
 //===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp b/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
index b85107ec4719112f801680aeb240a386ff331ccf..a0776296b8ebc79aa890873590f3c0f34ba8979d 100644
--- a/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
@@ -6435,20 +6435,20 @@ void ARMBaseInstrInfo::saveLROnStack(MachineBasicBlock &MBB,
                                      MachineBasicBlock::iterator It, bool CFI,
                                      bool Auth) const {
   int Align = std::max(Subtarget.getStackAlignment().value(), uint64_t(8));
+  unsigned MIFlags = CFI ? MachineInstr::FrameSetup : 0;
   assert(Align >= 8 && Align <= 256);
   if (Auth) {
     assert(Subtarget.isThumb2());
     // Compute PAC in R12. Outlining ensures R12 is dead across the outlined
     // sequence.
-    BuildMI(MBB, It, DebugLoc(), get(ARM::t2PAC))
-        .setMIFlags(MachineInstr::FrameSetup);
+    BuildMI(MBB, It, DebugLoc(), get(ARM::t2PAC)).setMIFlags(MIFlags);
     BuildMI(MBB, It, DebugLoc(), get(ARM::t2STRD_PRE), ARM::SP)
         .addReg(ARM::R12, RegState::Kill)
         .addReg(ARM::LR, RegState::Kill)
         .addReg(ARM::SP)
         .addImm(-Align)
         .add(predOps(ARMCC::AL))
-        .setMIFlags(MachineInstr::FrameSetup);
+        .setMIFlags(MIFlags);
   } else {
     unsigned Opc = Subtarget.isThumb() ? ARM::t2STR_PRE : ARM::STR_PRE_IMM;
     BuildMI(MBB, It, DebugLoc(), get(Opc), ARM::SP)
@@ -6456,7 +6456,7 @@ void ARMBaseInstrInfo::saveLROnStack(MachineBasicBlock &MBB,
         .addReg(ARM::SP)
         .addImm(-Align)
         .add(predOps(ARMCC::AL))
-        .setMIFlags(MachineInstr::FrameSetup);
+        .setMIFlags(MIFlags);
   }
 
   if (!CFI)
@@ -6511,6 +6511,7 @@ void ARMBaseInstrInfo::restoreLRFromStack(MachineBasicBlock &MBB,
                                           MachineBasicBlock::iterator It,
                                           bool CFI, bool Auth) const {
   int Align = Subtarget.getStackAlignment().value();
+  unsigned MIFlags = CFI ? MachineInstr::FrameDestroy : 0;
   if (Auth) {
     assert(Subtarget.isThumb2());
     // Restore return address PAC and LR.
@@ -6521,7 +6522,7 @@ void ARMBaseInstrInfo::restoreLRFromStack(MachineBasicBlock &MBB,
         .addReg(ARM::SP)
         .addImm(Align)
         .add(predOps(ARMCC::AL))
-        .setMIFlags(MachineInstr::FrameDestroy);
+        .setMIFlags(MIFlags);
     // LR authentication is after the CFI instructions, below.
   } else {
     unsigned Opc = Subtarget.isThumb() ? ARM::t2LDR_POST : ARM::LDR_POST_IMM;
@@ -6532,7 +6533,7 @@ void ARMBaseInstrInfo::restoreLRFromStack(MachineBasicBlock &MBB,
       MIB.addReg(0);
     MIB.addImm(Subtarget.getStackAlignment().value())
         .add(predOps(ARMCC::AL))
-        .setMIFlags(MachineInstr::FrameDestroy);
+        .setMIFlags(MIFlags);
   }
 
   if (CFI) {
diff --git a/llvm/lib/Target/ARM/ARMMachineFunctionInfo.cpp b/llvm/lib/Target/ARM/ARMMachineFunctionInfo.cpp
index ec2d8f59ee9b507e7691f4b63714128c29682e19..a364992fab3ed5036d44b22a9112920ead7a9c90 100644
--- a/llvm/lib/Target/ARM/ARMMachineFunctionInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMMachineFunctionInfo.cpp
@@ -13,6 +13,18 @@ using namespace llvm;
 
 void ARMFunctionInfo::anchor() {}
 
+yaml::ARMFunctionInfo::ARMFunctionInfo(const llvm::ARMFunctionInfo &MFI)
+    : LRSpilled(MFI.isLRSpilled()) {}
+
+void yaml::ARMFunctionInfo::mappingImpl(yaml::IO &YamlIO) {
+  MappingTraits<ARMFunctionInfo>::mapping(YamlIO, *this);
+}
+
+void ARMFunctionInfo::initializeBaseYamlFields(
+    const yaml::ARMFunctionInfo &YamlMFI) {
+  LRSpilled = YamlMFI.LRSpilled;
+}
+
 static bool GetBranchTargetEnforcement(const Function &F,
                                        const ARMSubtarget *Subtarget) {
   if (!Subtarget->isMClass() || !Subtarget->hasV7Ops())
diff --git a/llvm/lib/Target/ARM/ARMMachineFunctionInfo.h b/llvm/lib/Target/ARM/ARMMachineFunctionInfo.h
index f7531ce78ccae4efc3b4f1d15e7df572ed716fdb..b9ff3a08f998f673e62eb2845d0be8c35350c27f 100644
--- a/llvm/lib/Target/ARM/ARMMachineFunctionInfo.h
+++ b/llvm/lib/Target/ARM/ARMMachineFunctionInfo.h
@@ -15,6 +15,7 @@
 
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/CodeGen/MIRYamlMapping.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/IR/GlobalVariable.h"
 #include "llvm/Support/ErrorHandling.h"
@@ -22,6 +23,10 @@
 
 namespace llvm {
 
+namespace yaml {
+struct ARMFunctionInfo;
+} // end namespace yaml
+
 class ARMSubtarget;
 
 /// ARMFunctionInfo - This class is derived from MachineFunctionInfo and
@@ -293,8 +298,29 @@ public:
   }
 
   bool branchTargetEnforcement() const { return BranchTargetEnforcement; }
+
+  void initializeBaseYamlFields(const yaml::ARMFunctionInfo &YamlMFI);
 };
 
+namespace yaml {
+struct ARMFunctionInfo final : public yaml::MachineFunctionInfo {
+  bool LRSpilled;
+
+  ARMFunctionInfo() = default;
+  ARMFunctionInfo(const llvm::ARMFunctionInfo &MFI);
+
+  void mappingImpl(yaml::IO &YamlIO) override;
+  ~ARMFunctionInfo() = default;
+};
+
+template <> struct MappingTraits<ARMFunctionInfo> {
+  static void mapping(IO &YamlIO, ARMFunctionInfo &MFI) {
+    YamlIO.mapOptional("isLRSpilled", MFI.LRSpilled);
+  }
+};
+
+} // end namespace yaml
+
 } // end namespace llvm
 
 #endif // LLVM_LIB_TARGET_ARM_ARMMACHINEFUNCTIONINFO_H
diff --git a/llvm/lib/Target/ARM/ARMTargetMachine.cpp b/llvm/lib/Target/ARM/ARMTargetMachine.cpp
index deae3c9df35d0efd6b662f22522e780d4fdaafd4..a99773691df123f946288a9553a7813e0cd7f8ae 100644
--- a/llvm/lib/Target/ARM/ARMTargetMachine.cpp
+++ b/llvm/lib/Target/ARM/ARMTargetMachine.cpp
@@ -30,6 +30,7 @@
 #include "llvm/CodeGen/GlobalISel/Legalizer.h"
 #include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
 #include "llvm/CodeGen/GlobalISel/RegBankSelect.h"
+#include "llvm/CodeGen/MIRParser/MIParser.h"
 #include "llvm/CodeGen/MachineFunction.h"
 #include "llvm/CodeGen/MachineScheduler.h"
 #include "llvm/CodeGen/Passes.h"
@@ -619,3 +620,23 @@ void ARMPassConfig::addPreEmitPass2() {
     addPass(createEHContGuardCatchretPass());
   }
 }
+
+yaml::MachineFunctionInfo *
+ARMBaseTargetMachine::createDefaultFuncInfoYAML() const {
+  return new yaml::ARMFunctionInfo();
+}
+
+yaml::MachineFunctionInfo *
+ARMBaseTargetMachine::convertFuncInfoToYAML(const MachineFunction &MF) const {
+  const auto *MFI = MF.getInfo<ARMFunctionInfo>();
+  return new yaml::ARMFunctionInfo(*MFI);
+}
+
+bool ARMBaseTargetMachine::parseMachineFunctionInfo(
+    const yaml::MachineFunctionInfo &MFI, PerFunctionMIParsingState &PFS,
+    SMDiagnostic &Error, SMRange &SourceRange) const {
+  const auto &YamlMFI = static_cast<const yaml::ARMFunctionInfo &>(MFI);
+  MachineFunction &MF = PFS.MF;
+  MF.getInfo<ARMFunctionInfo>()->initializeBaseYamlFields(YamlMFI);
+  return false;
+}
diff --git a/llvm/lib/Target/ARM/ARMTargetMachine.h b/llvm/lib/Target/ARM/ARMTargetMachine.h
index 1754382692baadbc38f6289fc747926c51731fd5..69d8fa8ada649835ab786b7e18c70752d0084216 100644
--- a/llvm/lib/Target/ARM/ARMTargetMachine.h
+++ b/llvm/lib/Target/ARM/ARMTargetMachine.h
@@ -83,6 +83,14 @@ public:
     // Addrspacecasts are always noops.
     return true;
   }
+
+  yaml::MachineFunctionInfo *createDefaultFuncInfoYAML() const override;
+  yaml::MachineFunctionInfo *
+  convertFuncInfoToYAML(const MachineFunction &MF) const override;
+  bool parseMachineFunctionInfo(const yaml::MachineFunctionInfo &,
+                                PerFunctionMIParsingState &PFS,
+                                SMDiagnostic &Error,
+                                SMRange &SourceRange) const override;
 };
 
 /// ARM/Thumb little endian target machine.
diff --git a/llvm/lib/Target/BPF/BPFISelDAGToDAG.cpp b/llvm/lib/Target/BPF/BPFISelDAGToDAG.cpp
index b0cfca3b7ab66a720bdfe7e7fb2648d7c1dcdeba..909c7c005735b6965e115ccf4b9994da2962afe0 100644
--- a/llvm/lib/Target/BPF/BPFISelDAGToDAG.cpp
+++ b/llvm/lib/Target/BPF/BPFISelDAGToDAG.cpp
@@ -192,20 +192,6 @@ void BPFDAGToDAGISel::Select(SDNode *Node) {
   switch (Opcode) {
   default:
     break;
-  case ISD::SDIV: {
-    if (!Subtarget->hasSdivSmod()) {
-      DebugLoc Empty;
-      const DebugLoc &DL = Node->getDebugLoc();
-      if (DL != Empty)
-        errs() << "Error at line " << DL.getLine() << ": ";
-      else
-        errs() << "Error: ";
-      errs() << "Unsupport signed division for DAG: ";
-      Node->print(errs(), CurDAG);
-      errs() << "Please convert to unsigned div/mod.\n";
-    }
-    break;
-  }
   case ISD::INTRINSIC_W_CHAIN: {
     unsigned IntNo = cast<ConstantSDNode>(Node->getOperand(1))->getZExtValue();
     switch (IntNo) {
diff --git a/llvm/lib/Target/BPF/BPFISelLowering.cpp b/llvm/lib/Target/BPF/BPFISelLowering.cpp
index f3368b8979d6f5ba889556f9088d82b830ed412b..2fe86e75ddae8f7e92e6b165666d8e6918c54a1d 100644
--- a/llvm/lib/Target/BPF/BPFISelLowering.cpp
+++ b/llvm/lib/Target/BPF/BPFISelLowering.cpp
@@ -99,8 +99,10 @@ BPFTargetLowering::BPFTargetLowering(const TargetMachine &TM,
 
     setOperationAction(ISD::SDIVREM, VT, Expand);
     setOperationAction(ISD::UDIVREM, VT, Expand);
-    if (!STI.hasSdivSmod())
-      setOperationAction(ISD::SREM, VT, Expand);
+    if (!STI.hasSdivSmod()) {
+      setOperationAction(ISD::SDIV, VT, Custom);
+      setOperationAction(ISD::SREM, VT, Custom);
+    }
     setOperationAction(ISD::MULHU, VT, Expand);
     setOperationAction(ISD::MULHS, VT, Expand);
     setOperationAction(ISD::UMUL_LOHI, VT, Expand);
@@ -307,8 +309,11 @@ SDValue BPFTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
     return LowerGlobalAddress(Op, DAG);
   case ISD::SELECT_CC:
     return LowerSELECT_CC(Op, DAG);
+  case ISD::SDIV:
+  case ISD::SREM:
+    return LowerSDIVSREM(Op, DAG);
   case ISD::DYNAMIC_STACKALLOC:
-    report_fatal_error("unsupported dynamic stack allocation");
+    return LowerDYNAMIC_STACKALLOC(Op, DAG);
   }
 }
 
@@ -617,6 +622,21 @@ static void NegateCC(SDValue &LHS, SDValue &RHS, ISD::CondCode &CC) {
   }
 }
 
+SDValue BPFTargetLowering::LowerSDIVSREM(SDValue Op, SelectionDAG &DAG) const {
+  SDLoc DL(Op);
+  fail(DL, DAG,
+       "unsupported signed division, please convert to unsigned div/mod.");
+  return DAG.getUNDEF(Op->getValueType(0));
+}
+
+SDValue BPFTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
+                                                   SelectionDAG &DAG) const {
+  SDLoc DL(Op);
+  fail(DL, DAG, "unsupported dynamic stack allocation");
+  auto Ops = {DAG.getConstant(0, SDLoc(), Op.getValueType()), Op.getOperand(0)};
+  return DAG.getMergeValues(Ops, SDLoc());
+}
+
 SDValue BPFTargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
   SDValue Chain = Op.getOperand(0);
   ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(1))->get();
diff --git a/llvm/lib/Target/BPF/BPFISelLowering.h b/llvm/lib/Target/BPF/BPFISelLowering.h
index 3be1c04bca3d6541792911846bfa26cded51e258..819711b650c15f3c9103665ad7eefc548cb7b62a 100644
--- a/llvm/lib/Target/BPF/BPFISelLowering.h
+++ b/llvm/lib/Target/BPF/BPFISelLowering.h
@@ -73,6 +73,8 @@ private:
   bool HasJmpExt;
   bool HasMovsx;
 
+  SDValue LowerSDIVSREM(SDValue Op, SelectionDAG &DAG) const;
+  SDValue LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerBR_CC(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const;
   SDValue LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const;
diff --git a/llvm/lib/Target/M68k/GISel/M68kCallLowering.cpp b/llvm/lib/Target/M68k/GISel/M68kCallLowering.cpp
index bb63516e957fd5e7232fb533ada67822843d9c9a..e7e62951649416ba776b132e717f2235ee3ea156 100644
--- a/llvm/lib/Target/M68k/GISel/M68kCallLowering.cpp
+++ b/llvm/lib/Target/M68k/GISel/M68kCallLowering.cpp
@@ -25,6 +25,27 @@
 
 using namespace llvm;
 
+namespace {
+
+struct M68kFormalArgHandler : public M68kIncomingValueHandler {
+  M68kFormalArgHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI)
+      : M68kIncomingValueHandler(MIRBuilder, MRI) {}
+};
+
+struct CallReturnHandler : public M68kIncomingValueHandler {
+  CallReturnHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
+                    MachineInstrBuilder &MIB)
+      : M68kIncomingValueHandler(MIRBuilder, MRI), MIB(MIB) {}
+
+private:
+  void assignValueToReg(Register ValVReg, Register PhysReg,
+                        const CCValAssign &VA) override;
+
+  MachineInstrBuilder &MIB;
+};
+
+} // end anonymous namespace
+
 M68kCallLowering::M68kCallLowering(const M68kTargetLowering &TLI)
     : CallLowering(&TLI) {}
 
@@ -119,7 +140,7 @@ bool M68kCallLowering::lowerFormalArguments(MachineIRBuilder &MIRBuilder,
   CCAssignFn *AssignFn =
       TLI.getCCAssignFn(F.getCallingConv(), false, F.isVarArg());
   IncomingValueAssigner ArgAssigner(AssignFn);
-  FormalArgHandler ArgHandler(MIRBuilder, MRI);
+  M68kFormalArgHandler ArgHandler(MIRBuilder, MRI);
   return determineAndHandleAssignments(ArgHandler, ArgAssigner, SplitArgs,
                                        MIRBuilder, F.getCallingConv(),
                                        F.isVarArg());
diff --git a/llvm/lib/Target/M68k/GISel/M68kCallLowering.h b/llvm/lib/Target/M68k/GISel/M68kCallLowering.h
index 7644e6cffbb1bc18f2a0ee323f30eb174476760b..53696df21794cc4f92c032db5e0cac5691af2d24 100644
--- a/llvm/lib/Target/M68k/GISel/M68kCallLowering.h
+++ b/llvm/lib/Target/M68k/GISel/M68kCallLowering.h
@@ -63,23 +63,6 @@ private:
                            MachinePointerInfo &MPO,
                            ISD::ArgFlagsTy Flags) override;
 };
-
-struct FormalArgHandler : public M68kIncomingValueHandler {
-  FormalArgHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI)
-      : M68kIncomingValueHandler(MIRBuilder, MRI) {}
-};
-
-struct CallReturnHandler : public M68kIncomingValueHandler {
-  CallReturnHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
-                    MachineInstrBuilder &MIB)
-      : M68kIncomingValueHandler(MIRBuilder, MRI), MIB(MIB) {}
-
-private:
-  void assignValueToReg(Register ValVReg, Register PhysReg,
-                        const CCValAssign &VA) override;
-
-  MachineInstrBuilder &MIB;
-};
 } // end namespace llvm
 
 #endif // LLVM_LIB_TARGET_M68K_GLSEL_M68KCALLLOWERING_H
diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCInstPrinter.cpp b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCInstPrinter.cpp
index ccbb650c65365b4dad753670a2edc60be25422b9..5e2106e9184f84e61315486f0a317cbaa2bbebe9 100644
--- a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCInstPrinter.cpp
+++ b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCInstPrinter.cpp
@@ -484,7 +484,10 @@ void PPCInstPrinter::printAbsBranchOperand(const MCInst *MI, unsigned OpNo,
   if (!MI->getOperand(OpNo).isImm())
     return printOperand(MI, OpNo, STI, O);
 
-  O << SignExtend32<32>((unsigned)MI->getOperand(OpNo).getImm() << 2);
+  uint64_t Imm = MI->getOperand(OpNo).getImm() << 2;
+  if (!TT.isPPC64())
+    Imm = static_cast<uint32_t>(Imm);
+  O << formatHex(Imm);
 }
 
 void PPCInstPrinter::printcrbitm(const MCInst *MI, unsigned OpNo,
diff --git a/llvm/lib/Target/PowerPC/PPCRegisterInfo.td b/llvm/lib/Target/PowerPC/PPCRegisterInfo.td
index 6151faf403aaaf1cdfd3ee7d56bc69f28a17e1e5..375e63654db11847d615fd70e08ff8f041b8a121 100644
--- a/llvm/lib/Target/PowerPC/PPCRegisterInfo.td
+++ b/llvm/lib/Target/PowerPC/PPCRegisterInfo.td
@@ -798,6 +798,7 @@ def directbrtarget : Operand<OtherVT> {
 def absdirectbrtarget : Operand<OtherVT> {
   let PrintMethod = "printAbsBranchOperand";
   let EncoderMethod = "getAbsDirectBrEncoding";
+  let DecoderMethod = "decodeDirectBrTarget";
   let ParserMatchClass = PPCDirectBrAsmOperand;
 }
 def PPCCondBrAsmOperand : AsmOperandClass {
@@ -814,6 +815,7 @@ def condbrtarget : Operand<OtherVT> {
 def abscondbrtarget : Operand<OtherVT> {
   let PrintMethod = "printAbsBranchOperand";
   let EncoderMethod = "getAbsCondBrEncoding";
+  let DecoderMethod = "decodeCondBrTarget";
   let ParserMatchClass = PPCCondBrAsmOperand;
 }
 def calltarget : Operand<iPTR> {
@@ -826,6 +828,7 @@ def calltarget : Operand<iPTR> {
 def abscalltarget : Operand<iPTR> {
   let PrintMethod = "printAbsBranchOperand";
   let EncoderMethod = "getAbsDirectBrEncoding";
+  let DecoderMethod = "decodeDirectBrTarget";
   let ParserMatchClass = PPCDirectBrAsmOperand;
 }
 def PPCCRBitMaskOperand : AsmOperandClass {
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index f2ec422b54a9268282346c7dceecbf86405307e0..91bccc77f93fd0797bdbbbda8608d1bf58f5db8c 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -1363,9 +1363,6 @@ RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
   setPrefFunctionAlignment(Subtarget.getPrefFunctionAlignment());
   setPrefLoopAlignment(Subtarget.getPrefLoopAlignment());
 
-  // Jumps are expensive, compared to logic
-  setJumpIsExpensive();
-
   setTargetDAGCombine({ISD::INTRINSIC_VOID, ISD::INTRINSIC_W_CHAIN,
                        ISD::INTRINSIC_WO_CHAIN, ISD::ADD, ISD::SUB, ISD::AND,
                        ISD::OR, ISD::XOR, ISD::SETCC, ISD::SELECT});
@@ -2583,8 +2580,15 @@ static SDValue getAllOnesMask(MVT VecVT, SDValue VL, const SDLoc &DL,
   return DAG.getNode(RISCVISD::VMSET_VL, DL, MaskVT, VL);
 }
 
-static SDValue getVLOp(uint64_t NumElts, const SDLoc &DL, SelectionDAG &DAG,
-                       const RISCVSubtarget &Subtarget) {
+static SDValue getVLOp(uint64_t NumElts, MVT ContainerVT, const SDLoc &DL,
+                       SelectionDAG &DAG, const RISCVSubtarget &Subtarget) {
+  // If we know the exact VLEN, our VL is exactly equal to VLMAX, and
+  // we can't encode the AVL as an immediate, use the VLMAX encoding.
+  const auto [MinVLMAX, MaxVLMAX] =
+      RISCVTargetLowering::computeVLMAXBounds(ContainerVT, Subtarget);
+  if (MinVLMAX == MaxVLMAX && NumElts == MinVLMAX && NumElts > 31)
+    return DAG.getRegister(RISCV::X0, Subtarget.getXLenVT());
+
   return DAG.getConstant(NumElts, DL, Subtarget.getXLenVT());
 }
 
@@ -2601,7 +2605,7 @@ static std::pair<SDValue, SDValue>
 getDefaultVLOps(uint64_t NumElts, MVT ContainerVT, const SDLoc &DL,
                 SelectionDAG &DAG, const RISCVSubtarget &Subtarget) {
   assert(ContainerVT.isScalableVector() && "Expecting scalable container type");
-  SDValue VL = getVLOp(NumElts, DL, DAG, Subtarget);
+  SDValue VL = getVLOp(NumElts, ContainerVT, DL, DAG, Subtarget);
   SDValue Mask = getAllOnesMask(ContainerVT, VL, DL, DAG);
   return {Mask, VL};
 }
@@ -2627,6 +2631,25 @@ SDValue RISCVTargetLowering::computeVLMax(MVT VecVT, const SDLoc &DL,
                              VecVT.getVectorElementCount());
 }
 
+std::pair<unsigned, unsigned>
+RISCVTargetLowering::computeVLMAXBounds(MVT VecVT,
+                                        const RISCVSubtarget &Subtarget) {
+  assert(VecVT.isScalableVector() && "Expected scalable vector");
+
+  unsigned EltSize = VecVT.getScalarSizeInBits();
+  unsigned MinSize = VecVT.getSizeInBits().getKnownMinValue();
+
+  unsigned VectorBitsMax = Subtarget.getRealMaxVLen();
+  unsigned MaxVLMAX =
+      RISCVTargetLowering::computeVLMAX(VectorBitsMax, EltSize, MinSize);
+
+  unsigned VectorBitsMin = Subtarget.getRealMinVLen();
+  unsigned MinVLMAX =
+      RISCVTargetLowering::computeVLMAX(VectorBitsMin, EltSize, MinSize);
+
+  return std::make_pair(MinVLMAX, MaxVLMAX);
+}
+
 // The state of RVV BUILD_VECTOR and VECTOR_SHUFFLE lowering is that very few
 // of either is (currently) supported. This can get us into an infinite loop
 // where we try to lower a BUILD_VECTOR as a VECTOR_SHUFFLE as a BUILD_VECTOR
@@ -8126,16 +8149,8 @@ static SDValue lowerVectorIntrinsicScalars(SDValue Op, SelectionDAG &DAG,
 
     // Optimize for constant AVL
     if (isa<ConstantSDNode>(AVL)) {
-      unsigned EltSize = VT.getScalarSizeInBits();
-      unsigned MinSize = VT.getSizeInBits().getKnownMinValue();
-
-      unsigned VectorBitsMax = Subtarget.getRealMaxVLen();
-      unsigned MaxVLMAX =
-          RISCVTargetLowering::computeVLMAX(VectorBitsMax, EltSize, MinSize);
-
-      unsigned VectorBitsMin = Subtarget.getRealMinVLen();
-      unsigned MinVLMAX =
-          RISCVTargetLowering::computeVLMAX(VectorBitsMin, EltSize, MinSize);
+      const auto [MinVLMAX, MaxVLMAX] =
+          RISCVTargetLowering::computeVLMAXBounds(VT, Subtarget);
 
       uint64_t AVLInt = cast<ConstantSDNode>(AVL)->getZExtValue();
       if (AVLInt <= MinVLMAX) {
@@ -8642,7 +8657,8 @@ SDValue RISCVTargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
     MVT VT = Op->getSimpleValueType(0);
     MVT ContainerVT = getContainerForFixedLengthVector(VT);
 
-    SDValue VL = getVLOp(VT.getVectorNumElements(), DL, DAG, Subtarget);
+    SDValue VL = getVLOp(VT.getVectorNumElements(), ContainerVT, DL, DAG,
+                         Subtarget);
     SDValue IntID = DAG.getTargetConstant(VlsegInts[NF - 2], DL, XLenVT);
     auto *Load = cast<MemIntrinsicSDNode>(Op);
     SmallVector<EVT, 9> ContainerVTs(NF, ContainerVT);
@@ -8777,7 +8793,8 @@ SDValue RISCVTargetLowering::LowerINTRINSIC_VOID(SDValue Op,
     MVT VT = Op->getOperand(2).getSimpleValueType();
     MVT ContainerVT = getContainerForFixedLengthVector(VT);
 
-    SDValue VL = getVLOp(VT.getVectorNumElements(), DL, DAG, Subtarget);
+    SDValue VL = getVLOp(VT.getVectorNumElements(), ContainerVT, DL, DAG,
+                         Subtarget);
     SDValue IntID = DAG.getTargetConstant(VssegInts[NF - 2], DL, XLenVT);
     SDValue Ptr = Op->getOperand(NF + 2);
 
@@ -9236,7 +9253,7 @@ SDValue RISCVTargetLowering::lowerINSERT_SUBVECTOR(SDValue Op,
     // Set the vector length to only the number of elements we care about. Note
     // that for slideup this includes the offset.
     unsigned EndIndex = OrigIdx + SubVecVT.getVectorNumElements();
-    SDValue VL = getVLOp(EndIndex, DL, DAG, Subtarget);
+    SDValue VL = getVLOp(EndIndex, ContainerVT, DL, DAG, Subtarget);
 
     // Use tail agnostic policy if we're inserting over Vec's tail.
     unsigned Policy = RISCVII::TAIL_UNDISTURBED_MASK_UNDISTURBED;
@@ -9413,7 +9430,8 @@ SDValue RISCVTargetLowering::lowerEXTRACT_SUBVECTOR(SDValue Op,
         getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget).first;
     // Set the vector length to only the number of elements we care about. This
     // avoids sliding down elements we're going to discard straight away.
-    SDValue VL = getVLOp(SubVecVT.getVectorNumElements(), DL, DAG, Subtarget);
+    SDValue VL = getVLOp(SubVecVT.getVectorNumElements(), ContainerVT, DL, DAG,
+                         Subtarget);
     SDValue SlidedownAmt = DAG.getConstant(OrigIdx, DL, XLenVT);
     SDValue Slidedown =
         getVSlidedown(DAG, Subtarget, DL, ContainerVT,
@@ -9820,7 +9838,7 @@ RISCVTargetLowering::lowerFixedLengthVectorLoadToRVV(SDValue Op,
   MVT XLenVT = Subtarget.getXLenVT();
   MVT ContainerVT = getContainerForFixedLengthVector(VT);
 
-  SDValue VL = getVLOp(VT.getVectorNumElements(), DL, DAG, Subtarget);
+  SDValue VL = getVLOp(VT.getVectorNumElements(), ContainerVT, DL, DAG, Subtarget);
 
   bool IsMaskOp = VT.getVectorElementType() == MVT::i1;
   SDValue IntID = DAG.getTargetConstant(
@@ -9864,7 +9882,8 @@ RISCVTargetLowering::lowerFixedLengthVectorStoreToRVV(SDValue Op,
 
   MVT ContainerVT = getContainerForFixedLengthVector(VT);
 
-  SDValue VL = getVLOp(VT.getVectorNumElements(), DL, DAG, Subtarget);
+  SDValue VL = getVLOp(VT.getVectorNumElements(), ContainerVT, DL, DAG,
+                       Subtarget);
 
   SDValue NewValue =
       convertToScalableVector(ContainerVT, StoreVal, DAG, Subtarget);
@@ -16038,6 +16057,7 @@ unsigned RISCVTargetLowering::ComputeNumSignBitsForTargetNode(
       assert(Subtarget.hasStdExtA());
       return 33;
     }
+    break;
   }
   }
 
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.h b/llvm/lib/Target/RISCV/RISCVISelLowering.h
index ae798cc47bf833349b3a9d073e66dee513a26282..41a2dc5771c82d2d612ddedc2fee55770d09570d 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.h
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.h
@@ -744,7 +744,13 @@ public:
     // The following equations have been reordered to prevent loss of precision
     // when calculating fractional LMUL.
     return ((VectorBits / EltSize) * MinSize) / RISCV::RVVBitsPerBlock;
-  };
+  }
+
+  // Return inclusive (low, high) bounds on the value of VLMAX for the
+  // given scalable container type given known bounds on VLEN.
+  static std::pair<unsigned, unsigned>
+  computeVLMAXBounds(MVT ContainerVT, const RISCVSubtarget &Subtarget);
+
   static unsigned getRegClassIDForLMUL(RISCVII::VLMUL LMul);
   static unsigned getSubregIndexByMVT(MVT VT, unsigned Index);
   static unsigned getRegClassIDForVecVT(MVT VT);
diff --git a/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp b/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp
index 096d7ce0cb64dfd6b5fb91c38acb42a2cd14729f..b2d36b362b3a0fa6852314a5b912c2b06aa5f126 100644
--- a/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp
+++ b/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp
@@ -1291,9 +1291,20 @@ void RISCVInsertVSETVLI::emitVSETVLIs(MachineBasicBlock &MBB) {
       if (RISCVII::hasVLOp(TSFlags)) {
         MachineOperand &VLOp = MI.getOperand(getVLOpNum(MI));
         if (VLOp.isReg()) {
+          Register Reg = VLOp.getReg();
+          MachineInstr *VLOpDef = MRI->getVRegDef(Reg);
+
           // Erase the AVL operand from the instruction.
           VLOp.setReg(RISCV::NoRegister);
           VLOp.setIsKill(false);
+
+          // If the AVL was an immediate > 31, then it would have been emitted
+          // as an ADDI. However, the ADDI might not have been used in the
+          // vsetvli, or a vsetvli might not have been emitted, so it may be
+          // dead now.
+          if (VLOpDef && TII->isAddImmediate(*VLOpDef, Reg) &&
+              MRI->use_nodbg_empty(Reg))
+            VLOpDef->eraseFromParent();
         }
         MI.addOperand(MachineOperand::CreateReg(RISCV::VL, /*isDef*/ false,
                                                 /*isImp*/ true));
@@ -1444,9 +1455,19 @@ static void doUnion(DemandedFields &A, DemandedFields B) {
   A.MaskPolicy |= B.MaskPolicy;
 }
 
-static bool isNonZeroAVL(const MachineOperand &MO) {
-  if (MO.isReg())
-    return RISCV::X0 == MO.getReg();
+static bool isNonZeroAVL(const MachineOperand &MO,
+                         const MachineRegisterInfo &MRI) {
+  if (MO.isReg()) {
+    if (MO.getReg() == RISCV::X0)
+      return true;
+    if (MachineInstr *MI = MRI.getVRegDef(MO.getReg());
+        MI && MI->getOpcode() == RISCV::ADDI &&
+        MI->getOperand(1).isReg() && MI->getOperand(2).isImm() &&
+        MI->getOperand(1).getReg() == RISCV::X0 &&
+        MI->getOperand(2).getImm() != 0)
+      return true;
+    return false;
+  }
   assert(MO.isImm());
   return 0 != MO.getImm();
 }
@@ -1455,7 +1476,8 @@ static bool isNonZeroAVL(const MachineOperand &MO) {
 // fields which would be observed.
 static bool canMutatePriorConfig(const MachineInstr &PrevMI,
                                  const MachineInstr &MI,
-                                 const DemandedFields &Used) {
+                                 const DemandedFields &Used,
+                                 const MachineRegisterInfo &MRI) {
   // If the VL values aren't equal, return false if either a) the former is
   // demanded, or b) we can't rewrite the former to be the later for
   // implementation reasons.
@@ -1468,8 +1490,8 @@ static bool canMutatePriorConfig(const MachineInstr &PrevMI,
     if (Used.VLZeroness) {
       if (isVLPreservingConfig(PrevMI))
         return false;
-      if (!isNonZeroAVL(MI.getOperand(1)) ||
-          !isNonZeroAVL(PrevMI.getOperand(1)))
+      if (!isNonZeroAVL(MI.getOperand(1), MRI) ||
+          !isNonZeroAVL(PrevMI.getOperand(1), MRI))
         return false;
     }
 
@@ -1513,14 +1535,23 @@ void RISCVInsertVSETVLI::doLocalPostpass(MachineBasicBlock &MBB) {
         ToDelete.push_back(&MI);
         // Leave NextMI unchanged
         continue;
-      } else if (canMutatePriorConfig(MI, *NextMI, Used)) {
+      } else if (canMutatePriorConfig(MI, *NextMI, Used, *MRI)) {
         if (!isVLPreservingConfig(*NextMI)) {
           MI.getOperand(0).setReg(NextMI->getOperand(0).getReg());
           MI.getOperand(0).setIsDead(false);
+          Register OldVLReg;
+          if (MI.getOperand(1).isReg())
+            OldVLReg = MI.getOperand(1).getReg();
           if (NextMI->getOperand(1).isImm())
             MI.getOperand(1).ChangeToImmediate(NextMI->getOperand(1).getImm());
           else
             MI.getOperand(1).ChangeToRegister(NextMI->getOperand(1).getReg(), false);
+          if (OldVLReg) {
+            MachineInstr *VLOpDef = MRI->getUniqueVRegDef(OldVLReg);
+            if (VLOpDef && TII->isAddImmediate(*VLOpDef, OldVLReg) &&
+                MRI->use_nodbg_empty(OldVLReg))
+              VLOpDef->eraseFromParent();
+          }
           MI.setDesc(NextMI->getDesc());
         }
         MI.getOperand(2).setImm(NextMI->getOperand(2).getImm());
diff --git a/llvm/lib/Target/SystemZ/SystemZRegisterInfo.cpp b/llvm/lib/Target/SystemZ/SystemZRegisterInfo.cpp
index 4d6b94da3a2754bd3b13ff6b85d427cdae9bd382..d5313acd87856c2885b205f11485be2297f00dae 100644
--- a/llvm/lib/Target/SystemZ/SystemZRegisterInfo.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZRegisterInfo.cpp
@@ -377,12 +377,12 @@ SystemZRegisterInfo::eliminateFrameIndex(MachineBasicBlock::iterator MI,
 }
 
 bool SystemZRegisterInfo::shouldCoalesce(MachineInstr *MI,
-                                  const TargetRegisterClass *SrcRC,
-                                  unsigned SubReg,
-                                  const TargetRegisterClass *DstRC,
-                                  unsigned DstSubReg,
-                                  const TargetRegisterClass *NewRC,
-                                  LiveIntervals &LIS) const {
+                                         const TargetRegisterClass *SrcRC,
+                                         unsigned SubReg,
+                                         const TargetRegisterClass *DstRC,
+                                         unsigned DstSubReg,
+                                         const TargetRegisterClass *NewRC,
+                                         LiveIntervals &LIS) const {
   assert (MI->isCopy() && "Only expecting COPY instructions");
 
   // Coalesce anything which is not a COPY involving a subreg to/from GR128.
@@ -390,44 +390,26 @@ bool SystemZRegisterInfo::shouldCoalesce(MachineInstr *MI,
         (getRegSizeInBits(*SrcRC) <= 64 || getRegSizeInBits(*DstRC) <= 64)))
     return true;
 
-  // Allow coalescing of a GR128 subreg COPY only if the live ranges are small
-  // and local to one MBB with not too much interferring registers. Otherwise
+  // Allow coalescing of a GR128 subreg COPY only if the subreg liverange is
+  // local to one MBB with not too many interferring physreg clobbers. Otherwise
   // regalloc may run out of registers.
+  unsigned SubregOpIdx = getRegSizeInBits(*SrcRC) == 128 ? 0 : 1;
+  LiveInterval &LI = LIS.getInterval(MI->getOperand(SubregOpIdx).getReg());
 
-  unsigned WideOpNo = (getRegSizeInBits(*SrcRC) == 128 ? 1 : 0);
-  Register GR128Reg = MI->getOperand(WideOpNo).getReg();
-  Register GRNarReg = MI->getOperand((WideOpNo == 1) ? 0 : 1).getReg();
-  LiveInterval &IntGR128 = LIS.getInterval(GR128Reg);
-  LiveInterval &IntGRNar = LIS.getInterval(GRNarReg);
-
-  // Check that the two virtual registers are local to MBB.
+  // Check that the subreg is local to MBB.
   MachineBasicBlock *MBB = MI->getParent();
-  MachineInstr *FirstMI_GR128 =
-    LIS.getInstructionFromIndex(IntGR128.beginIndex());
-  MachineInstr *FirstMI_GRNar =
-    LIS.getInstructionFromIndex(IntGRNar.beginIndex());
-  MachineInstr *LastMI_GR128 = LIS.getInstructionFromIndex(IntGR128.endIndex());
-  MachineInstr *LastMI_GRNar = LIS.getInstructionFromIndex(IntGRNar.endIndex());
-  if ((!FirstMI_GR128 || FirstMI_GR128->getParent() != MBB) ||
-      (!FirstMI_GRNar || FirstMI_GRNar->getParent() != MBB) ||
-      (!LastMI_GR128 || LastMI_GR128->getParent() != MBB) ||
-      (!LastMI_GRNar || LastMI_GRNar->getParent() != MBB))
+  MachineInstr *FirstMI = LIS.getInstructionFromIndex(LI.beginIndex());
+  MachineInstr *LastMI = LIS.getInstructionFromIndex(LI.endIndex());
+  if (!FirstMI || FirstMI->getParent() != MBB ||
+      !LastMI || LastMI->getParent() != MBB)
     return false;
 
-  MachineBasicBlock::iterator MII = nullptr, MEE = nullptr;
-  if (WideOpNo == 1) {
-    MII = FirstMI_GR128;
-    MEE = LastMI_GRNar;
-  } else {
-    MII = FirstMI_GRNar;
-    MEE = LastMI_GR128;
-  }
-
   // Check if coalescing seems safe by finding the set of clobbered physreg
   // pairs in the region.
   BitVector PhysClobbered(getNumRegs());
-  MEE++;
-  for (; MII != MEE; ++MII) {
+  for (MachineBasicBlock::iterator MII = FirstMI,
+                                   MEE = std::next(LastMI->getIterator());
+       MII != MEE; ++MII)
     for (const MachineOperand &MO : MII->operands())
       if (MO.isReg() && MO.getReg().isPhysical()) {
         for (MCPhysReg SI : superregs_inclusive(MO.getReg()))
@@ -436,7 +418,6 @@ bool SystemZRegisterInfo::shouldCoalesce(MachineInstr *MI,
             break;
           }
       }
-  }
 
   // Demand an arbitrary margin of free regs.
   unsigned const DemandedFreeGR128 = 3;
diff --git a/llvm/lib/Target/TargetMachine.cpp b/llvm/lib/Target/TargetMachine.cpp
index 246f6b4b492c8b6e5fbbd7a77bccb189ae918f41..3f96bd37755e58761fb11a1fbab573f8f646b165 100644
--- a/llvm/lib/Target/TargetMachine.cpp
+++ b/llvm/lib/Target/TargetMachine.cpp
@@ -39,10 +39,16 @@ TargetMachine::TargetMachine(const Target &T, StringRef DataLayoutString,
 
 TargetMachine::~TargetMachine() = default;
 
-bool TargetMachine::isLargeGlobalObject(const GlobalObject *GO) const {
+bool TargetMachine::isLargeGlobalValue(const GlobalValue *GVal) const {
   if (getTargetTriple().getArch() != Triple::x86_64)
     return false;
 
+  auto *GO = GVal->getAliaseeObject();
+
+  // Be conservative if we can't find an underlying GlobalObject.
+  if (!GO)
+    return true;
+
   auto *GV = dyn_cast<GlobalVariable>(GO);
 
   // Functions/GlobalIFuncs are only large under the large code model.
@@ -55,6 +61,8 @@ bool TargetMachine::isLargeGlobalObject(const GlobalObject *GO) const {
   // We should properly mark well-known section name prefixes as small/large,
   // because otherwise the output section may have the wrong section flags and
   // the linker will lay it out in an unexpected way.
+  // TODO: bring back lbss/ldata/lrodata checks after fixing accesses to large
+  // globals in the small code model.
   StringRef Name = GV->getSection();
   if (!Name.empty()) {
     auto IsPrefix = [&](StringRef Prefix) {
@@ -63,8 +71,6 @@ bool TargetMachine::isLargeGlobalObject(const GlobalObject *GO) const {
     };
     if (IsPrefix(".bss") || IsPrefix(".data") || IsPrefix(".rodata"))
       return false;
-    if (IsPrefix(".lbss") || IsPrefix(".ldata") || IsPrefix(".lrodata"))
-      return true;
   }
 
   // For x86-64, we treat an explicit GlobalVariable small code model to mean
diff --git a/llvm/lib/Target/X86/X86FastISel.cpp b/llvm/lib/Target/X86/X86FastISel.cpp
index 068119a6332a67b4618dcf1f31a35e453f518d39..7f134fe1c72bdaab7a609ade7a4f8a960c4be8bc 100644
--- a/llvm/lib/Target/X86/X86FastISel.cpp
+++ b/llvm/lib/Target/X86/X86FastISel.cpp
@@ -711,7 +711,12 @@ bool X86FastISel::handleConstantAddresses(const Value *V, X86AddressMode &AM) {
   // Handle constant address.
   if (const GlobalValue *GV = dyn_cast<GlobalValue>(V)) {
     // Can't handle alternate code models yet.
-    if (TM.getCodeModel() != CodeModel::Small)
+    if (TM.getCodeModel() != CodeModel::Small &&
+        TM.getCodeModel() != CodeModel::Medium)
+      return false;
+
+    // Can't handle large objects yet.
+    if (TM.isLargeGlobalValue(GV))
       return false;
 
     // Can't handle TLS yet.
@@ -1044,7 +1049,8 @@ bool X86FastISel::X86SelectCallAddress(const Value *V, X86AddressMode &AM) {
   // Handle constant address.
   if (const GlobalValue *GV = dyn_cast<GlobalValue>(V)) {
     // Can't handle alternate code models yet.
-    if (TM.getCodeModel() != CodeModel::Small)
+    if (TM.getCodeModel() != CodeModel::Small &&
+        TM.getCodeModel() != CodeModel::Medium)
       return false;
 
     // RIP-relative addresses can't have additional register operands.
@@ -3768,7 +3774,8 @@ unsigned X86FastISel::X86MaterializeFP(const ConstantFP *CFP, MVT VT) {
 
   // Can't handle alternate code models yet.
   CodeModel::Model CM = TM.getCodeModel();
-  if (CM != CodeModel::Small && CM != CodeModel::Large)
+  if (CM != CodeModel::Small && CM != CodeModel::Medium &&
+      CM != CodeModel::Large)
     return 0;
 
   // Get opcode and regclass of the output for the given load instruction.
@@ -3806,7 +3813,7 @@ unsigned X86FastISel::X86MaterializeFP(const ConstantFP *CFP, MVT VT) {
     PICBase = getInstrInfo()->getGlobalBaseReg(FuncInfo.MF);
   else if (OpFlag == X86II::MO_GOTOFF)
     PICBase = getInstrInfo()->getGlobalBaseReg(FuncInfo.MF);
-  else if (Subtarget->is64Bit() && TM.getCodeModel() == CodeModel::Small)
+  else if (Subtarget->is64Bit() && TM.getCodeModel() != CodeModel::Large)
     PICBase = X86::RIP;
 
   // Create the load from the constant pool.
@@ -3836,8 +3843,11 @@ unsigned X86FastISel::X86MaterializeFP(const ConstantFP *CFP, MVT VT) {
 }
 
 unsigned X86FastISel::X86MaterializeGV(const GlobalValue *GV, MVT VT) {
-  // Can't handle alternate code models yet.
-  if (TM.getCodeModel() != CodeModel::Small)
+  // Can't handle large GlobalValues yet.
+  if (TM.getCodeModel() != CodeModel::Small &&
+      TM.getCodeModel() != CodeModel::Medium)
+    return 0;
+  if (TM.isLargeGlobalValue(GV))
     return 0;
 
   // Materialize addresses with LEA/MOV instructions.
diff --git a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
index 545039b79f16163bcfd3f9f76f257787d6b7bbf6..ce4fbfe88739dfcc4353479c50072d84e4472d58 100644
--- a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
@@ -2927,6 +2927,10 @@ bool X86DAGToDAGISel::selectAddr(SDNode *Parent, SDValue N, SDValue &Base,
 }
 
 bool X86DAGToDAGISel::selectMOV64Imm32(SDValue N, SDValue &Imm) {
+  // Cannot use 32 bit constants to reference objects in kernel code model.
+  if (TM.getCodeModel() == CodeModel::Kernel)
+    return false;
+
   // In static codegen with small code model, we can get the address of a label
   // into a register with 'movl'
   if (N->getOpcode() != X86ISD::Wrapper)
@@ -2940,15 +2944,18 @@ bool X86DAGToDAGISel::selectMOV64Imm32(SDValue N, SDValue &Imm) {
     return false;
 
   Imm = N;
-  if (N->getOpcode() != ISD::TargetGlobalAddress)
-    return TM.getCodeModel() == CodeModel::Small;
+  // Small/medium code model can reference non-TargetGlobalAddress objects with
+  // 32 bit constants.
+  if (N->getOpcode() != ISD::TargetGlobalAddress) {
+    return TM.getCodeModel() == CodeModel::Small ||
+           TM.getCodeModel() == CodeModel::Medium;
+  }
 
-  std::optional<ConstantRange> CR =
-      cast<GlobalAddressSDNode>(N)->getGlobal()->getAbsoluteSymbolRange();
-  if (!CR)
-    return TM.getCodeModel() == CodeModel::Small;
+  const GlobalValue *GV = cast<GlobalAddressSDNode>(N)->getGlobal();
+  if (std::optional<ConstantRange> CR = GV->getAbsoluteSymbolRange())
+    return CR->getUnsignedMax().ult(1ull << 32);
 
-  return CR->getUnsignedMax().ult(1ull << 32);
+  return !TM.isLargeGlobalValue(GV);
 }
 
 bool X86DAGToDAGISel::selectLEA64_32Addr(SDValue N, SDValue &Base,
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index e779a3d3dab6c24c43894b38b73c345ef089ec4e..cb117475dbe4cf876ea1dd001dbf179e8d2fdb2d 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -49951,33 +49951,27 @@ static SDValue combineLoad(SDNode *N, SelectionDAG &DAG,
           Extract = DAG.getBitcast(RegVT, Extract);
           return DCI.CombineTo(N, Extract, SDValue(User, 1));
         }
-        if (User->getOpcode() == X86ISD::VBROADCAST_LOAD &&
-            getTargetConstantFromBasePtr(Ptr)) {
-          // See if we are loading a constant that has also been broadcast.
-          APInt Undefs, UserUndefs;
-          SmallVector<APInt> Bits, UserBits;
-          if (getTargetConstantBitsFromNode(SDValue(N, 0), 8, Undefs, Bits) &&
-              getTargetConstantBitsFromNode(SDValue(User, 0), 8, UserUndefs,
-                                            UserBits)) {
-            UserUndefs = UserUndefs.trunc(Undefs.getBitWidth());
-            UserBits.truncate(Bits.size());
-            if (Bits == UserBits && UserUndefs.isSubsetOf(Undefs)) {
-              SDValue Extract = extractSubVector(
-                  SDValue(User, 0), 0, DAG, SDLoc(N), RegVT.getSizeInBits());
-              Extract = DAG.getBitcast(RegVT, Extract);
-              return DCI.CombineTo(N, Extract, SDValue(User, 1));
-            }
+        auto MatchingBits = [](const APInt &Undefs, const APInt &UserUndefs,
+                               ArrayRef<APInt> Bits, ArrayRef<APInt> UserBits) {
+          if (!UserUndefs.isSubsetOf(Undefs))
+            return false;
+          for (unsigned I = 0, E = Undefs.getBitWidth(); I != E; ++I) {
+            if (Undefs[I])
+              continue;
+            if (Bits[I] != UserBits[I])
+              return false;
           }
-        }
-        if (ISD::isNormalLoad(User)) {
-          // See if we are loading a constant that matches in the lower
-          // bits of a longer constant (but from a different constant pool ptr).
-          SDValue UserPtr = cast<MemSDNode>(User)->getBasePtr();
-          const Constant *LdC = getTargetConstantFromBasePtr(Ptr);
-          const Constant *UserC = getTargetConstantFromBasePtr(UserPtr);
-          if (LdC && UserC && UserPtr != Ptr &&
-              LdC->getType()->getPrimitiveSizeInBits() <
-                  UserC->getType()->getPrimitiveSizeInBits()) {
+          return true;
+        };
+        // See if we are loading a constant that matches in the lower
+        // bits of a longer constant (but from a different constant pool ptr).
+        SDValue UserPtr = cast<MemSDNode>(User)->getBasePtr();
+        const Constant *LdC = getTargetConstantFromBasePtr(Ptr);
+        const Constant *UserC = getTargetConstantFromBasePtr(UserPtr);
+        if (LdC && UserC && UserPtr != Ptr) {
+          unsigned LdSize = LdC->getType()->getPrimitiveSizeInBits();
+          unsigned UserSize = UserC->getType()->getPrimitiveSizeInBits();
+          if (LdSize < UserSize || !ISD::isNormalLoad(User)) {
             APInt Undefs, UserUndefs;
             SmallVector<APInt> Bits, UserBits;
             if (getTargetConstantBitsFromNode(SDValue(N, 0), 8, Undefs, Bits) &&
@@ -49985,7 +49979,7 @@ static SDValue combineLoad(SDNode *N, SelectionDAG &DAG,
                                               UserBits)) {
               UserUndefs = UserUndefs.trunc(Undefs.getBitWidth());
               UserBits.truncate(Bits.size());
-              if (Bits == UserBits && UserUndefs.isSubsetOf(Undefs)) {
+              if (MatchingBits(Undefs, UserUndefs, Bits, UserBits)) {
                 SDValue Extract = extractSubVector(
                     SDValue(User, 0), 0, DAG, SDLoc(N), RegVT.getSizeInBits());
                 Extract = DAG.getBitcast(RegVT, Extract);
diff --git a/llvm/lib/Target/X86/X86InstrAVX512.td b/llvm/lib/Target/X86/X86InstrAVX512.td
index 5eb893a82fcc7ef58947548d810699f6d07d6a76..e1fe2b680b96a4ebbdce76e576898e5a7c497341 100644
--- a/llvm/lib/Target/X86/X86InstrAVX512.td
+++ b/llvm/lib/Target/X86/X86InstrAVX512.td
@@ -1624,19 +1624,19 @@ multiclass avx512_perm_i_sizes<bits<8> opc, string OpcodeStr,
                                X86FoldableSchedWrite sched,
                                AVX512VLVectorVTInfo VTInfo,
                                AVX512VLVectorVTInfo ShuffleMask> {
-  defm NAME: avx512_perm_i<opc, OpcodeStr, sched, VTInfo.info512,
-                           ShuffleMask.info512>,
-            avx512_perm_i_mb<opc, OpcodeStr, sched, VTInfo.info512,
-                             ShuffleMask.info512>, EVEX_V512;
+  defm NAME#Z: avx512_perm_i<opc, OpcodeStr, sched, VTInfo.info512,
+                             ShuffleMask.info512>,
+               avx512_perm_i_mb<opc, OpcodeStr, sched, VTInfo.info512,
+                                ShuffleMask.info512>, EVEX_V512;
   let Predicates = [HasVLX] in {
-  defm NAME#128: avx512_perm_i<opc, OpcodeStr, sched, VTInfo.info128,
-                               ShuffleMask.info128>,
-                 avx512_perm_i_mb<opc, OpcodeStr, sched, VTInfo.info128,
-                                  ShuffleMask.info128>, EVEX_V128;
-  defm NAME#256: avx512_perm_i<opc, OpcodeStr, sched, VTInfo.info256,
-                               ShuffleMask.info256>,
-                 avx512_perm_i_mb<opc, OpcodeStr, sched, VTInfo.info256,
-                                  ShuffleMask.info256>, EVEX_V256;
+  defm NAME#Z128: avx512_perm_i<opc, OpcodeStr, sched, VTInfo.info128,
+                                ShuffleMask.info128>,
+                  avx512_perm_i_mb<opc, OpcodeStr, sched, VTInfo.info128,
+                                   ShuffleMask.info128>, EVEX_V128;
+  defm NAME#Z256: avx512_perm_i<opc, OpcodeStr, sched, VTInfo.info256,
+                                ShuffleMask.info256>,
+                  avx512_perm_i_mb<opc, OpcodeStr, sched, VTInfo.info256,
+                                   ShuffleMask.info256>, EVEX_V256;
   }
 }
 
@@ -1646,13 +1646,13 @@ multiclass avx512_perm_i_sizes_bw<bits<8> opc, string OpcodeStr,
                                   AVX512VLVectorVTInfo Idx,
                                   Predicate Prd> {
   let Predicates = [Prd] in
-  defm NAME: avx512_perm_i<opc, OpcodeStr, sched, VTInfo.info512,
-                           Idx.info512>, EVEX_V512;
+  defm NAME#Z: avx512_perm_i<opc, OpcodeStr, sched, VTInfo.info512,
+                             Idx.info512>, EVEX_V512;
   let Predicates = [Prd, HasVLX] in {
-  defm NAME#128: avx512_perm_i<opc, OpcodeStr, sched, VTInfo.info128,
-                               Idx.info128>, EVEX_V128;
-  defm NAME#256: avx512_perm_i<opc, OpcodeStr, sched, VTInfo.info256,
-                               Idx.info256>,  EVEX_V256;
+  defm NAME#Z128: avx512_perm_i<opc, OpcodeStr, sched, VTInfo.info128,
+                                Idx.info128>, EVEX_V128;
+  defm NAME#Z256: avx512_perm_i<opc, OpcodeStr, sched, VTInfo.info256,
+                                Idx.info256>,  EVEX_V256;
   }
 }
 
@@ -1702,9 +1702,9 @@ multiclass avx512_perm_i_lowering<string InstrStr, X86VectorVTInfo _,
 }
 
 // TODO: Should we add more casts? The vXi64 case is common due to ABI.
-defm : avx512_perm_i_lowering<"VPERMI2PS", v16f32_info, v16i32_info, v8i64_info>;
-defm : avx512_perm_i_lowering<"VPERMI2PS256", v8f32x_info, v8i32x_info, v4i64x_info>;
-defm : avx512_perm_i_lowering<"VPERMI2PS128", v4f32x_info, v4i32x_info, v2i64x_info>;
+defm : avx512_perm_i_lowering<"VPERMI2PSZ", v16f32_info, v16i32_info, v8i64_info>;
+defm : avx512_perm_i_lowering<"VPERMI2PSZ256", v8f32x_info, v8i32x_info, v4i64x_info>;
+defm : avx512_perm_i_lowering<"VPERMI2PSZ128", v4f32x_info, v4i32x_info, v2i64x_info>;
 
 // VPERMT2
 multiclass avx512_perm_t<bits<8> opc, string OpcodeStr,
@@ -1743,19 +1743,19 @@ multiclass avx512_perm_t_sizes<bits<8> opc, string OpcodeStr,
                                X86FoldableSchedWrite sched,
                                AVX512VLVectorVTInfo VTInfo,
                                AVX512VLVectorVTInfo ShuffleMask> {
-  defm NAME: avx512_perm_t<opc, OpcodeStr, sched, VTInfo.info512,
-                              ShuffleMask.info512>,
-            avx512_perm_t_mb<opc, OpcodeStr, sched, VTInfo.info512,
-                              ShuffleMask.info512>, EVEX_V512;
+  defm NAME#Z: avx512_perm_t<opc, OpcodeStr, sched, VTInfo.info512,
+                             ShuffleMask.info512>,
+               avx512_perm_t_mb<opc, OpcodeStr, sched, VTInfo.info512,
+                                ShuffleMask.info512>, EVEX_V512;
   let Predicates = [HasVLX] in {
-  defm NAME#128: avx512_perm_t<opc, OpcodeStr, sched, VTInfo.info128,
-                              ShuffleMask.info128>,
-                 avx512_perm_t_mb<opc, OpcodeStr, sched, VTInfo.info128,
-                              ShuffleMask.info128>, EVEX_V128;
-  defm NAME#256: avx512_perm_t<opc, OpcodeStr, sched, VTInfo.info256,
-                              ShuffleMask.info256>,
-                 avx512_perm_t_mb<opc, OpcodeStr, sched, VTInfo.info256,
-                              ShuffleMask.info256>, EVEX_V256;
+  defm NAME#Z128: avx512_perm_t<opc, OpcodeStr, sched, VTInfo.info128,
+                                ShuffleMask.info128>,
+                  avx512_perm_t_mb<opc, OpcodeStr, sched, VTInfo.info128,
+                                   ShuffleMask.info128>, EVEX_V128;
+  defm NAME#Z256: avx512_perm_t<opc, OpcodeStr, sched, VTInfo.info256,
+                                ShuffleMask.info256>,
+                   avx512_perm_t_mb<opc, OpcodeStr, sched, VTInfo.info256,
+                                    ShuffleMask.info256>, EVEX_V256;
   }
 }
 
@@ -1764,13 +1764,13 @@ multiclass avx512_perm_t_sizes_bw<bits<8> opc, string OpcodeStr,
                                   AVX512VLVectorVTInfo VTInfo,
                                   AVX512VLVectorVTInfo Idx, Predicate Prd> {
   let Predicates = [Prd] in
-  defm NAME: avx512_perm_t<opc, OpcodeStr, sched, VTInfo.info512,
-                           Idx.info512>, EVEX_V512;
+  defm NAME#Z: avx512_perm_t<opc, OpcodeStr, sched, VTInfo.info512,
+                             Idx.info512>, EVEX_V512;
   let Predicates = [Prd, HasVLX] in {
-  defm NAME#128: avx512_perm_t<opc, OpcodeStr, sched, VTInfo.info128,
-                               Idx.info128>, EVEX_V128;
-  defm NAME#256: avx512_perm_t<opc, OpcodeStr, sched, VTInfo.info256,
-                               Idx.info256>, EVEX_V256;
+  defm NAME#Z128: avx512_perm_t<opc, OpcodeStr, sched, VTInfo.info128,
+                                Idx.info128>, EVEX_V128;
+  defm NAME#Z256: avx512_perm_t<opc, OpcodeStr, sched, VTInfo.info256,
+                                Idx.info256>, EVEX_V256;
   }
 }
 
diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp
index 7b607c6f198c72aa477ad3dcbd7e5bd8004ce468..bc2d5ed1e17dd8130ead6eb32b30e99d23b79856 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.cpp
+++ b/llvm/lib/Target/X86/X86InstrInfo.cpp
@@ -2138,45 +2138,45 @@ static void commuteVPTERNLOG(MachineInstr &MI, unsigned SrcOpIdx1,
 // commuted.
 static bool isCommutableVPERMV3Instruction(unsigned Opcode) {
 #define VPERM_CASES(Suffix)                                                    \
-  case X86::VPERMI2##Suffix##128rr:                                            \
-  case X86::VPERMT2##Suffix##128rr:                                            \
-  case X86::VPERMI2##Suffix##256rr:                                            \
-  case X86::VPERMT2##Suffix##256rr:                                            \
-  case X86::VPERMI2##Suffix##rr:                                               \
-  case X86::VPERMT2##Suffix##rr:                                               \
-  case X86::VPERMI2##Suffix##128rm:                                            \
-  case X86::VPERMT2##Suffix##128rm:                                            \
-  case X86::VPERMI2##Suffix##256rm:                                            \
-  case X86::VPERMT2##Suffix##256rm:                                            \
-  case X86::VPERMI2##Suffix##rm:                                               \
-  case X86::VPERMT2##Suffix##rm:                                               \
-  case X86::VPERMI2##Suffix##128rrkz:                                          \
-  case X86::VPERMT2##Suffix##128rrkz:                                          \
-  case X86::VPERMI2##Suffix##256rrkz:                                          \
-  case X86::VPERMT2##Suffix##256rrkz:                                          \
-  case X86::VPERMI2##Suffix##rrkz:                                             \
-  case X86::VPERMT2##Suffix##rrkz:                                             \
-  case X86::VPERMI2##Suffix##128rmkz:                                          \
-  case X86::VPERMT2##Suffix##128rmkz:                                          \
-  case X86::VPERMI2##Suffix##256rmkz:                                          \
-  case X86::VPERMT2##Suffix##256rmkz:                                          \
-  case X86::VPERMI2##Suffix##rmkz:                                             \
-  case X86::VPERMT2##Suffix##rmkz:
+  case X86::VPERMI2##Suffix##Z128rr:                                           \
+  case X86::VPERMT2##Suffix##Z128rr:                                           \
+  case X86::VPERMI2##Suffix##Z256rr:                                           \
+  case X86::VPERMT2##Suffix##Z256rr:                                           \
+  case X86::VPERMI2##Suffix##Zrr:                                              \
+  case X86::VPERMT2##Suffix##Zrr:                                              \
+  case X86::VPERMI2##Suffix##Z128rm:                                           \
+  case X86::VPERMT2##Suffix##Z128rm:                                           \
+  case X86::VPERMI2##Suffix##Z256rm:                                           \
+  case X86::VPERMT2##Suffix##Z256rm:                                           \
+  case X86::VPERMI2##Suffix##Zrm:                                              \
+  case X86::VPERMT2##Suffix##Zrm:                                              \
+  case X86::VPERMI2##Suffix##Z128rrkz:                                         \
+  case X86::VPERMT2##Suffix##Z128rrkz:                                         \
+  case X86::VPERMI2##Suffix##Z256rrkz:                                         \
+  case X86::VPERMT2##Suffix##Z256rrkz:                                         \
+  case X86::VPERMI2##Suffix##Zrrkz:                                            \
+  case X86::VPERMT2##Suffix##Zrrkz:                                            \
+  case X86::VPERMI2##Suffix##Z128rmkz:                                         \
+  case X86::VPERMT2##Suffix##Z128rmkz:                                         \
+  case X86::VPERMI2##Suffix##Z256rmkz:                                         \
+  case X86::VPERMT2##Suffix##Z256rmkz:                                         \
+  case X86::VPERMI2##Suffix##Zrmkz:                                            \
+  case X86::VPERMT2##Suffix##Zrmkz:
 
 #define VPERM_CASES_BROADCAST(Suffix)                                          \
   VPERM_CASES(Suffix)                                                          \
-  case X86::VPERMI2##Suffix##128rmb:                                           \
-  case X86::VPERMT2##Suffix##128rmb:                                           \
-  case X86::VPERMI2##Suffix##256rmb:                                           \
-  case X86::VPERMT2##Suffix##256rmb:                                           \
-  case X86::VPERMI2##Suffix##rmb:                                              \
-  case X86::VPERMT2##Suffix##rmb:                                              \
-  case X86::VPERMI2##Suffix##128rmbkz:                                         \
-  case X86::VPERMT2##Suffix##128rmbkz:                                         \
-  case X86::VPERMI2##Suffix##256rmbkz:                                         \
-  case X86::VPERMT2##Suffix##256rmbkz:                                         \
-  case X86::VPERMI2##Suffix##rmbkz:                                            \
-  case X86::VPERMT2##Suffix##rmbkz:
+  case X86::VPERMI2##Suffix##Z128rmb:                                          \
+  case X86::VPERMT2##Suffix##Z128rmb:                                          \
+  case X86::VPERMI2##Suffix##Z256rmb:                                          \
+  case X86::VPERMT2##Suffix##Z256rmb:                                          \
+  case X86::VPERMI2##Suffix##Zrmb:                                             \
+  case X86::VPERMT2##Suffix##Zrmb:                                             \
+  case X86::VPERMI2##Suffix##Z128rmbkz:                                        \
+  case X86::VPERMT2##Suffix##Z128rmbkz:                                        \
+  case X86::VPERMI2##Suffix##Z256rmbkz:                                        \
+  case X86::VPERMT2##Suffix##Z256rmbkz:                                        \
+  case X86::VPERMI2##Suffix##Zrmbkz:                                           \
+  case X86::VPERMT2##Suffix##Zrmbkz:
 
   switch (Opcode) {
   default:
@@ -2197,45 +2197,45 @@ static bool isCommutableVPERMV3Instruction(unsigned Opcode) {
 // from the I opcode to the T opcode and vice versa.
 static unsigned getCommutedVPERMV3Opcode(unsigned Opcode) {
 #define VPERM_CASES(Orig, New)                                                 \
-  case X86::Orig##128rr:                                                       \
-    return X86::New##128rr;                                                    \
-  case X86::Orig##128rrkz:                                                     \
-    return X86::New##128rrkz;                                                  \
-  case X86::Orig##128rm:                                                       \
-    return X86::New##128rm;                                                    \
-  case X86::Orig##128rmkz:                                                     \
-    return X86::New##128rmkz;                                                  \
-  case X86::Orig##256rr:                                                       \
-    return X86::New##256rr;                                                    \
-  case X86::Orig##256rrkz:                                                     \
-    return X86::New##256rrkz;                                                  \
-  case X86::Orig##256rm:                                                       \
-    return X86::New##256rm;                                                    \
-  case X86::Orig##256rmkz:                                                     \
-    return X86::New##256rmkz;                                                  \
-  case X86::Orig##rr:                                                          \
-    return X86::New##rr;                                                       \
-  case X86::Orig##rrkz:                                                        \
-    return X86::New##rrkz;                                                     \
-  case X86::Orig##rm:                                                          \
-    return X86::New##rm;                                                       \
-  case X86::Orig##rmkz:                                                        \
-    return X86::New##rmkz;
+  case X86::Orig##Z128rr:                                                      \
+    return X86::New##Z128rr;                                                   \
+  case X86::Orig##Z128rrkz:                                                    \
+    return X86::New##Z128rrkz;                                                 \
+  case X86::Orig##Z128rm:                                                      \
+    return X86::New##Z128rm;                                                   \
+  case X86::Orig##Z128rmkz:                                                    \
+    return X86::New##Z128rmkz;                                                 \
+  case X86::Orig##Z256rr:                                                      \
+    return X86::New##Z256rr;                                                   \
+  case X86::Orig##Z256rrkz:                                                    \
+    return X86::New##Z256rrkz;                                                 \
+  case X86::Orig##Z256rm:                                                      \
+    return X86::New##Z256rm;                                                   \
+  case X86::Orig##Z256rmkz:                                                    \
+    return X86::New##Z256rmkz;                                                 \
+  case X86::Orig##Zrr:                                                         \
+    return X86::New##Zrr;                                                      \
+  case X86::Orig##Zrrkz:                                                       \
+    return X86::New##Zrrkz;                                                    \
+  case X86::Orig##Zrm:                                                         \
+    return X86::New##Zrm;                                                      \
+  case X86::Orig##Zrmkz:                                                       \
+    return X86::New##Zrmkz;
 
 #define VPERM_CASES_BROADCAST(Orig, New)                                       \
   VPERM_CASES(Orig, New)                                                       \
-  case X86::Orig##128rmb:                                                      \
-    return X86::New##128rmb;                                                   \
-  case X86::Orig##128rmbkz:                                                    \
-    return X86::New##128rmbkz;                                                 \
-  case X86::Orig##256rmb:                                                      \
-    return X86::New##256rmb;                                                   \
-  case X86::Orig##256rmbkz:                                                    \
-    return X86::New##256rmbkz;                                                 \
-  case X86::Orig##rmb:                                                         \
-    return X86::New##rmb;                                                      \
-  case X86::Orig##rmbkz:                                                       \
-    return X86::New##rmbkz;
+  case X86::Orig##Z128rmb:                                                     \
+    return X86::New##Z128rmb;                                                  \
+  case X86::Orig##Z128rmbkz:                                                   \
+    return X86::New##Z128rmbkz;                                                \
+  case X86::Orig##Z256rmb:                                                     \
+    return X86::New##Z256rmb;                                                  \
+  case X86::Orig##Z256rmbkz:                                                   \
+    return X86::New##Z256rmbkz;                                                \
+  case X86::Orig##Zrmb:                                                        \
+    return X86::New##Zrmb;                                                     \
+  case X86::Orig##Zrmbkz:                                                      \
+    return X86::New##Zrmbkz;
 
   switch (Opcode) {
     VPERM_CASES(VPERMI2B, VPERMT2B)
diff --git a/llvm/lib/Target/X86/X86SchedIceLake.td b/llvm/lib/Target/X86/X86SchedIceLake.td
index e27af1433d4559fb302fdc9a0bff0c15e60e273a..2c660fad2ec7db9c2c429b869743634000034cbb 100644
--- a/llvm/lib/Target/X86/X86SchedIceLake.td
+++ b/llvm/lib/Target/X86/X86SchedIceLake.td
@@ -1392,12 +1392,12 @@ def ICXWriteResGroup97 : SchedWriteRes<[ICXPort5,ICXPort015]> {
   let NumMicroOps = 3;
   let ReleaseAtCycles = [2,1];
 }
-def: InstRW<[ICXWriteResGroup97], (instregex "VPERMI2W128rr",
-                                             "VPERMI2W256rr",
-                                             "VPERMI2Wrr",
-                                             "VPERMT2W128rr",
-                                             "VPERMT2W256rr",
-                                             "VPERMT2Wrr")>;
+def: InstRW<[ICXWriteResGroup97], (instregex "VPERMI2WZ128rr",
+                                             "VPERMI2WZ256rr",
+                                             "VPERMI2WZrr",
+                                             "VPERMT2WZ128rr",
+                                             "VPERMT2WZ256rr",
+                                             "VPERMT2WZrr")>;
 
 def ICXWriteResGroup99 : SchedWriteRes<[ICXPort23,ICXPort0156]> {
   let Latency = 7;
@@ -1692,14 +1692,14 @@ def: InstRW<[ICXWriteResGroup136], (instregex "VALIGN(D|Q)Z128rm(b?)i",
                                               "VFPCLASSSDZrm(b?)",
                                               "VFPCLASSSSZrm(b?)",
                                               "(V?)PCMPGTQrm",
-                                              "VPERMI2D128rm(b?)",
-                                              "VPERMI2PD128rm(b?)",
-                                              "VPERMI2PS128rm(b?)",
-                                              "VPERMI2Q128rm(b?)",
-                                              "VPERMT2D128rm(b?)",
-                                              "VPERMT2PD128rm(b?)",
-                                              "VPERMT2PS128rm(b?)",
-                                              "VPERMT2Q128rm(b?)",
+                                              "VPERMI2DZ128rm(b?)",
+                                              "VPERMI2PDZ128rm(b?)",
+                                              "VPERMI2PSZ128rm(b?)",
+                                              "VPERMI2QZ128rm(b?)",
+                                              "VPERMT2DZ128rm(b?)",
+                                              "VPERMT2PDZ128rm(b?)",
+                                              "VPERMT2PSZ128rm(b?)",
+                                              "VPERMT2QZ128rm(b?)",
                                               "VPMAXSQZ128rm(b?)",
                                               "VPMAXUQZ128rm(b?)",
                                               "VPMINSQZ128rm(b?)",
@@ -2002,8 +2002,8 @@ def ICXWriteResGroup183 : SchedWriteRes<[ICXPort5,ICXPort23,ICXPort015]> {
   let NumMicroOps = 4;
   let ReleaseAtCycles = [2,1,1];
 }
-def: InstRW<[ICXWriteResGroup183], (instregex "VPERMI2W128rm(b?)",
-                                              "VPERMT2W128rm(b?)")>;
+def: InstRW<[ICXWriteResGroup183], (instregex "VPERMI2WZ128rm(b?)",
+                                              "VPERMT2WZ128rm(b?)")>;
 
 def ICXWriteResGroup187 : SchedWriteRes<[ICXPort0,ICXPort5,ICXPort23]> {
   let Latency = 14;
@@ -2029,10 +2029,10 @@ def ICXWriteResGroup189 : SchedWriteRes<[ICXPort5,ICXPort23,ICXPort015]> {
   let NumMicroOps = 4;
   let ReleaseAtCycles = [2,1,1];
 }
-def: InstRW<[ICXWriteResGroup189], (instregex "VPERMI2W256rm(b?)",
-                                              "VPERMI2Wrm(b?)",
-                                              "VPERMT2W256rm(b?)",
-                                              "VPERMT2Wrm(b?)")>;
+def: InstRW<[ICXWriteResGroup189], (instregex "VPERMI2WZ256rm(b?)",
+                                              "VPERMI2WZrm(b?)",
+                                              "VPERMT2WZ256rm(b?)",
+                                              "VPERMT2WZrm(b?)")>;
 
 def ICXWriteResGroup190 : SchedWriteRes<[ICXPort1,ICXPort06,ICXPort15,ICXPort0156]> {
   let Latency = 14;
diff --git a/llvm/lib/Target/X86/X86SchedSapphireRapids.td b/llvm/lib/Target/X86/X86SchedSapphireRapids.td
index 4eac53385ae54deeb6d587af2790951426e7d628..bf9e4b7dc6d9ae7591147eaf2b809f626e57c71b 100644
--- a/llvm/lib/Target/X86/X86SchedSapphireRapids.td
+++ b/llvm/lib/Target/X86/X86SchedSapphireRapids.td
@@ -635,10 +635,10 @@ def : InstRW<[SPRWriteResGroup10, ReadAfterVecXLd], (instregex "^(V?)PACK(S|U)S(
                                                                "^VPMULTISHIFTQBZ128rm(b?)$")>;
 def : InstRW<[SPRWriteResGroup10, ReadAfterVecXLd], (instrs VFPCLASSPHZ128rm)>;
 def : InstRW<[SPRWriteResGroup10, ReadAfterVecYLd], (instregex "^VFPCLASSP(D|H|S)Z((256)?)rm$",
-                                                               "^VPERM(I|T)2(D|Q|PS)128rm((b|k|bk|kz)?)$",
-                                                               "^VPERM(I|T)2(D|Q|PS)128rmbkz$",
-                                                               "^VPERM(I|T)2PD128rm((b|k|bk|kz)?)$",
-                                                               "^VPERM(I|T)2PD128rmbkz$")>;
+                                                               "^VPERM(I|T)2(D|Q|PS)Z128rm((b|k|bk|kz)?)$",
+                                                               "^VPERM(I|T)2(D|Q|PS)Z128rmbkz$",
+                                                               "^VPERM(I|T)2PDZ128rm((b|k|bk|kz)?)$",
+                                                               "^VPERM(I|T)2PDZ128rmbkz$")>;
 def : InstRW<[SPRWriteResGroup10, ReadAfterVecYLd], (instrs VPERMBZ128rm)>;
 
 def SPRWriteResGroup11 : SchedWriteRes<[SPRPort02_03_11, SPRPort05]> {
@@ -678,8 +678,8 @@ def : InstRW<[SPRWriteResGroup12], (instregex "^ADD_F(P?)rST0$",
                                               "^VPERM(B|D|Q)Zrr$",
                                               "^VPERM(D|Q)Z256rr((k|kz)?)$",
                                               "^VPERM(D|Q)Zrrk(z?)$",
-                                              "^VPERM(I|T)2(D|Q)(128|256)rr((k|kz)?)$",
-                                              "^VPERM(I|T)2(D|Q)rr((k|kz)?)$",
+                                              "^VPERM(I|T)2(D|Q)Z(128|256)rr((k|kz)?)$",
+                                              "^VPERM(I|T)2(D|Q)Zrr((k|kz)?)$",
                                               "^VPM(AX|IN)(S|U)QZ(128|256)rr((k|kz)?)$",
                                               "^VPMULTISHIFTQBZ(128|256)rr$",
                                               "^VPOPCNT(B|D|Q|W)Z(128|256)rr$",
@@ -4274,7 +4274,7 @@ def SPRWriteResGroup455 : SchedWriteRes<[SPRPort00_01_05, SPRPort02_03_11, SPRPo
 }
 def : InstRW<[SPRWriteResGroup455], (instregex "^VPCONFLICTQZ128rm((b|k|bk|kz)?)$")>;
 def : InstRW<[SPRWriteResGroup455], (instrs VPCONFLICTQZ128rmbkz)>;
-def : InstRW<[SPRWriteResGroup455, ReadAfterVecYLd], (instregex "^VPERM(I|T)2B128rm$")>;
+def : InstRW<[SPRWriteResGroup455, ReadAfterVecYLd], (instregex "^VPERM(I|T)2BZ128rm$")>;
 
 def SPRWriteResGroup456 : SchedWriteRes<[SPRPort00_01_05, SPRPort05]> {
   let ReleaseAtCycles = [1, 2];
@@ -4325,127 +4325,127 @@ def SPRWriteResGroup462 : SchedWriteRes<[SPRPort00_01_05, SPRPort02_03_11, SPRPo
   let Latency = 13;
   let NumMicroOps = 4;
 }
-def : InstRW<[SPRWriteResGroup462, ReadAfterVecYLd], (instregex "^VPERM(I|T)2B128rmk(z?)$")>;
-def : InstRW<[SPRWriteResGroup462, ReadAfterVecYLd], (instrs VPERMT2W128rm)>;
+def : InstRW<[SPRWriteResGroup462, ReadAfterVecYLd], (instregex "^VPERM(I|T)2BZ128rmk(z?)$")>;
+def : InstRW<[SPRWriteResGroup462, ReadAfterVecYLd], (instrs VPERMT2WZ128rm)>;
 
 def SPRWriteResGroup463 : SchedWriteRes<[SPRPort00_01_05, SPRPort05]> {
   let ReleaseAtCycles = [1, 2];
   let Latency = 5;
   let NumMicroOps = 3;
 }
-def : InstRW<[SPRWriteResGroup463], (instregex "^VPERM(I|T)2B(128|256)rr$")>;
+def : InstRW<[SPRWriteResGroup463], (instregex "^VPERM(I|T)2BZ(128|256)rr$")>;
 
 def SPRWriteResGroup464 : SchedWriteRes<[SPRPort00_01_05, SPRPort05]> {
   let ReleaseAtCycles = [1, 2];
   let Latency = 7;
   let NumMicroOps = 3;
 }
-def : InstRW<[SPRWriteResGroup464], (instregex "^VPERM(I|T)2B(128|256)rrk(z?)$",
-                                               "^VPERM(I|T)2W(128|256)rr$")>;
+def : InstRW<[SPRWriteResGroup464], (instregex "^VPERM(I|T)2BZ(128|256)rrk(z?)$",
+                                               "^VPERM(I|T)2WZ(128|256)rr$")>;
 
 def SPRWriteResGroup465 : SchedWriteRes<[SPRPort00_01_05, SPRPort02_03_11, SPRPort05]> {
   let ReleaseAtCycles = [1, 1, 2];
   let Latency = 12;
   let NumMicroOps = 4;
 }
-def : InstRW<[SPRWriteResGroup465, ReadAfterVecYLd], (instregex "^VPERM(I|T)2B256rm$")>;
+def : InstRW<[SPRWriteResGroup465, ReadAfterVecYLd], (instregex "^VPERM(I|T)2BZ256rm$")>;
 
 def SPRWriteResGroup466 : SchedWriteRes<[SPRPort00_01_05, SPRPort02_03_11, SPRPort05]> {
   let ReleaseAtCycles = [1, 1, 2];
   let Latency = 14;
   let NumMicroOps = 4;
 }
-def : InstRW<[SPRWriteResGroup466, ReadAfterVecYLd], (instregex "^VPERM(I|T)2B256rmk(z?)$")>;
-def : InstRW<[SPRWriteResGroup466, ReadAfterVecYLd], (instrs VPERMI2W128rm,
-                                                             VPERMT2W256rm)>;
+def : InstRW<[SPRWriteResGroup466, ReadAfterVecYLd], (instregex "^VPERM(I|T)2BZ256rmk(z?)$")>;
+def : InstRW<[SPRWriteResGroup466, ReadAfterVecYLd], (instrs VPERMI2WZ128rm,
+                                                             VPERMT2WZ256rm)>;
 
 def SPRWriteResGroup467 : SchedWriteRes<[SPRPort00_05, SPRPort02_03_11, SPRPort05]> {
   let ReleaseAtCycles = [1, 1, 2];
   let Latency = 12;
   let NumMicroOps = 4;
 }
-def : InstRW<[SPRWriteResGroup467, ReadAfterVecYLd], (instregex "^VPERM(I|T)2Brm$")>;
+def : InstRW<[SPRWriteResGroup467, ReadAfterVecYLd], (instregex "^VPERM(I|T)2BZrm$")>;
 
 def SPRWriteResGroup468 : SchedWriteRes<[SPRPort00_05, SPRPort02_03_11, SPRPort05]> {
   let ReleaseAtCycles = [1, 1, 2];
   let Latency = 14;
   let NumMicroOps = 4;
 }
-def : InstRW<[SPRWriteResGroup468, ReadAfterVecYLd], (instregex "^VPERM(I|T)2Brmk(z?)$")>;
-def : InstRW<[SPRWriteResGroup468, ReadAfterVecYLd], (instrs VPERMT2Wrm)>;
+def : InstRW<[SPRWriteResGroup468, ReadAfterVecYLd], (instregex "^VPERM(I|T)2BZrmk(z?)$")>;
+def : InstRW<[SPRWriteResGroup468, ReadAfterVecYLd], (instrs VPERMT2WZrm)>;
 
 def SPRWriteResGroup469 : SchedWriteRes<[SPRPort00_05, SPRPort05]> {
   let ReleaseAtCycles = [1, 2];
   let Latency = 5;
   let NumMicroOps = 3;
 }
-def : InstRW<[SPRWriteResGroup469], (instregex "^VPERM(I|T)2Brr$")>;
+def : InstRW<[SPRWriteResGroup469], (instregex "^VPERM(I|T)2BZrr$")>;
 
 def SPRWriteResGroup470 : SchedWriteRes<[SPRPort00_05, SPRPort05]> {
   let ReleaseAtCycles = [1, 2];
   let Latency = 7;
   let NumMicroOps = 3;
 }
-def : InstRW<[SPRWriteResGroup470], (instregex "^VPERM(I|T)2Brrk(z?)$",
-                                               "^VPERM(I|T)2Wrr$")>;
+def : InstRW<[SPRWriteResGroup470], (instregex "^VPERM(I|T)2BZrrk(z?)$",
+                                               "^VPERM(I|T)2WZrr$")>;
 
 def SPRWriteResGroup471 : SchedWriteRes<[SPRPort00_01_05, SPRPort02_03_11, SPRPort05]> {
   let ReleaseAtCycles = [1, 1, 2];
   let Latency = 16;
   let NumMicroOps = 4;
 }
-def : InstRW<[SPRWriteResGroup471, ReadAfterVecYLd], (instregex "^VPERMI2W128rmk(z?)$",
-                                                                "^VPERMT2W256rmk(z?)$")>;
+def : InstRW<[SPRWriteResGroup471, ReadAfterVecYLd], (instregex "^VPERMI2WZ128rmk(z?)$",
+                                                                "^VPERMT2WZ256rmk(z?)$")>;
 
 def SPRWriteResGroup472 : SchedWriteRes<[SPRPort00_01_05, SPRPort05]> {
   let ReleaseAtCycles = [1, 2];
   let Latency = 9;
   let NumMicroOps = 3;
 }
-def : InstRW<[SPRWriteResGroup472], (instregex "^VPERM(I|T)2W(128|256)rrk(z?)$")>;
+def : InstRW<[SPRWriteResGroup472], (instregex "^VPERM(I|T)2WZ(128|256)rrk(z?)$")>;
 
 def SPRWriteResGroup473 : SchedWriteRes<[SPRPort00_01_05, SPRPort02_03_11, SPRPort05]> {
   let ReleaseAtCycles = [1, 1, 2];
   let Latency = 15;
   let NumMicroOps = 4;
 }
-def : InstRW<[SPRWriteResGroup473, ReadAfterVecYLd], (instregex "^VPERMT2W128rmk(z?)$")>;
-def : InstRW<[SPRWriteResGroup473, ReadAfterVecYLd], (instrs VPERMI2W256rm)>;
+def : InstRW<[SPRWriteResGroup473, ReadAfterVecYLd], (instregex "^VPERMT2WZ128rmk(z?)$")>;
+def : InstRW<[SPRWriteResGroup473, ReadAfterVecYLd], (instrs VPERMI2WZ256rm)>;
 
 def SPRWriteResGroup474 : SchedWriteRes<[SPRPort00_01_05, SPRPort02_03_11, SPRPort05]> {
   let ReleaseAtCycles = [1, 1, 2];
   let Latency = 17;
   let NumMicroOps = 4;
 }
-def : InstRW<[SPRWriteResGroup474, ReadAfterVecYLd], (instregex "^VPERMI2W256rmk(z?)$")>;
+def : InstRW<[SPRWriteResGroup474, ReadAfterVecYLd], (instregex "^VPERMI2WZ256rmk(z?)$")>;
 
 def SPRWriteResGroup475 : SchedWriteRes<[SPRPort00_05, SPRPort02_03_11, SPRPort05]> {
   let ReleaseAtCycles = [1, 1, 2];
   let Latency = 15;
   let NumMicroOps = 4;
 }
-def : InstRW<[SPRWriteResGroup475, ReadAfterVecYLd], (instrs VPERMI2Wrm)>;
+def : InstRW<[SPRWriteResGroup475, ReadAfterVecYLd], (instrs VPERMI2WZrm)>;
 
 def SPRWriteResGroup476 : SchedWriteRes<[SPRPort00_05, SPRPort02_03_11, SPRPort05]> {
   let ReleaseAtCycles = [1, 1, 2];
   let Latency = 17;
   let NumMicroOps = 4;
 }
-def : InstRW<[SPRWriteResGroup476, ReadAfterVecYLd], (instregex "^VPERMI2Wrmk(z?)$")>;
+def : InstRW<[SPRWriteResGroup476, ReadAfterVecYLd], (instregex "^VPERMI2WZrmk(z?)$")>;
 
 def SPRWriteResGroup477 : SchedWriteRes<[SPRPort00_05, SPRPort05]> {
   let ReleaseAtCycles = [1, 2];
   let Latency = 9;
   let NumMicroOps = 3;
 }
-def : InstRW<[SPRWriteResGroup477], (instregex "^VPERM(I|T)2Wrrk(z?)$")>;
+def : InstRW<[SPRWriteResGroup477], (instregex "^VPERM(I|T)2WZrrk(z?)$")>;
 
 def SPRWriteResGroup478 : SchedWriteRes<[SPRPort00_05, SPRPort02_03_11, SPRPort05]> {
   let ReleaseAtCycles = [1, 1, 2];
   let Latency = 16;
   let NumMicroOps = 4;
 }
-def : InstRW<[SPRWriteResGroup478, ReadAfterVecYLd], (instregex "^VPERMT2Wrmk(z?)$")>;
+def : InstRW<[SPRWriteResGroup478, ReadAfterVecYLd], (instregex "^VPERMT2WZrmk(z?)$")>;
 
 def SPRWriteResGroup479 : SchedWriteRes<[SPRPort00_01, SPRPort02_03_11, SPRPort05]> {
   let Latency = 10;
diff --git a/llvm/lib/Target/X86/X86SchedSkylakeServer.td b/llvm/lib/Target/X86/X86SchedSkylakeServer.td
index 8194af8a6e1db2bb202f709d98d60b34276b9436..3da688cda2c6c4c4d7da5ff3dd28be38c4828b17 100644
--- a/llvm/lib/Target/X86/X86SchedSkylakeServer.td
+++ b/llvm/lib/Target/X86/X86SchedSkylakeServer.td
@@ -1379,12 +1379,12 @@ def SKXWriteResGroup97 : SchedWriteRes<[SKXPort5,SKXPort015]> {
   let NumMicroOps = 3;
   let ReleaseAtCycles = [2,1];
 }
-def: InstRW<[SKXWriteResGroup97], (instregex "VPERMI2W128rr",
-                                             "VPERMI2W256rr",
-                                             "VPERMI2Wrr",
-                                             "VPERMT2W128rr",
-                                             "VPERMT2W256rr",
-                                             "VPERMT2Wrr")>;
+def: InstRW<[SKXWriteResGroup97], (instregex "VPERMI2WZ128rr",
+                                             "VPERMI2WZ256rr",
+                                             "VPERMI2WZrr",
+                                             "VPERMT2WZ128rr",
+                                             "VPERMT2WZ256rr",
+                                             "VPERMT2WZrr")>;
 
 def SKXWriteResGroup99 : SchedWriteRes<[SKXPort23,SKXPort0156]> {
   let Latency = 7;
@@ -1675,14 +1675,14 @@ def: InstRW<[SKXWriteResGroup136], (instregex "VALIGN(D|Q)Z128rm(b?)i",
                                               "VFPCLASSSDZrm(b?)",
                                               "VFPCLASSSSZrm(b?)",
                                               "(V?)PCMPGTQrm",
-                                              "VPERMI2D128rm(b?)",
-                                              "VPERMI2PD128rm(b?)",
-                                              "VPERMI2PS128rm(b?)",
-                                              "VPERMI2Q128rm(b?)",
-                                              "VPERMT2D128rm(b?)",
-                                              "VPERMT2PD128rm(b?)",
-                                              "VPERMT2PS128rm(b?)",
-                                              "VPERMT2Q128rm(b?)",
+                                              "VPERMI2DZ128rm(b?)",
+                                              "VPERMI2PDZ128rm(b?)",
+                                              "VPERMI2PSZ128rm(b?)",
+                                              "VPERMI2QZ128rm(b?)",
+                                              "VPERMT2DZ128rm(b?)",
+                                              "VPERMT2PDZ128rm(b?)",
+                                              "VPERMT2PSZ128rm(b?)",
+                                              "VPERMT2QZ128rm(b?)",
                                               "VPMAXSQZ128rm(b?)",
                                               "VPMAXUQZ128rm(b?)",
                                               "VPMINSQZ128rm(b?)",
@@ -1983,8 +1983,8 @@ def SKXWriteResGroup183 : SchedWriteRes<[SKXPort5,SKXPort23,SKXPort015]> {
   let NumMicroOps = 4;
   let ReleaseAtCycles = [2,1,1];
 }
-def: InstRW<[SKXWriteResGroup183], (instregex "VPERMI2W128rm(b?)",
-                                              "VPERMT2W128rm(b?)")>;
+def: InstRW<[SKXWriteResGroup183], (instregex "VPERMI2WZ128rm(b?)",
+                                              "VPERMT2WZ128rm(b?)")>;
 
 def SKXWriteResGroup187 : SchedWriteRes<[SKXPort0,SKXPort5,SKXPort23]> {
   let Latency = 14;
@@ -2010,10 +2010,10 @@ def SKXWriteResGroup189 : SchedWriteRes<[SKXPort5,SKXPort23,SKXPort015]> {
   let NumMicroOps = 4;
   let ReleaseAtCycles = [2,1,1];
 }
-def: InstRW<[SKXWriteResGroup189], (instregex "VPERMI2W256rm(b?)",
-                                              "VPERMI2Wrm(b?)",
-                                              "VPERMT2W256rm(b?)",
-                                              "VPERMT2Wrm(b?)")>;
+def: InstRW<[SKXWriteResGroup189], (instregex "VPERMI2WZ256rm(b?)",
+                                              "VPERMI2WZrm(b?)",
+                                              "VPERMT2WZ256rm(b?)",
+                                              "VPERMT2WZrm(b?)")>;
 
 def SKXWriteResGroup190 : SchedWriteRes<[SKXPort1,SKXPort06,SKXPort15,SKXPort0156]> {
   let Latency = 14;
diff --git a/llvm/lib/Target/X86/X86ScheduleZnver4.td b/llvm/lib/Target/X86/X86ScheduleZnver4.td
index 405000f0851e9d3b9c66901eea55fccc04435753..dac4d8422582a46cda07c3466672cf073b1ab306 100644
--- a/llvm/lib/Target/X86/X86ScheduleZnver4.td
+++ b/llvm/lib/Target/X86/X86ScheduleZnver4.td
@@ -1743,8 +1743,8 @@ def Zn4PERMIT2_128: SchedWriteRes<[Zn4FPFMisc12]> {
   let NumMicroOps = 1;
 }
 def : InstRW<[Zn4PERMIT2_128], (instregex
-        "VPERM(I2|T2)(PS|PD|W)128(rr|rrk|rrkz)",
-        "VPERM(I2|T2)(B|D|Q)128(rr|rrk|rrkz)"
+	"VPERM(I2|T2)(PS|PD|W)Z128(rr|rrk|rrkz)",
+	"VPERM(I2|T2)(B|D|Q)Z128(rr|rrk|rrkz)"
 	)>;
 
 def Zn4PERMIT2_128rr:SchedWriteRes<[Zn4FPFMisc12]> {
@@ -1763,11 +1763,11 @@ def Zn4PERMIT2_256: SchedWriteRes<[Zn4FPFMisc12]> {
   let NumMicroOps = 1;
 }
 def : InstRW<[Zn4PERMIT2_256], (instregex
-        "VPERM(I2|T2)(PS|PD|W)256(rr|rrk|rrkz)",
+	"VPERM(I2|T2)(PS|PD|W)Z256(rr|rrk|rrkz)",
 	"VPERMP(S|D)Z256(rr|rrk|rrkz)",
 	"V(P?)COMPRESS(B|W|D|Q|PD|PS|SD|SQ)Z256(rr|rrk|rrkz)",
 	"VPERM(B|D|Q|W)Z256(rr|rrk|rrkz)",
-	"VPERM(I2|Q|T2)(B|D|Q)(Z?)256(rr|rrk|rrkz)",
+	"VPERM(I2|Q|T2)(B|D|Q)Z256(rr|rrk|rrkz)",
 	"VPEXPAND(B|W)Z256(rr|rrk|rrkz)"
 	)>;
 
@@ -1777,12 +1777,12 @@ def Zn4PERMIT2Z: SchedWriteRes<[Zn4FPFMisc12]> {
   let NumMicroOps = 1;
 }
 def : InstRW<[Zn4PERMIT2Z], (instregex
-        "VPERM(I2|T2)(PS|PD|W)(rr|rrk|rrkz)",
+	"VPERM(I2|T2)(PS|PD|W)Z(rr|rrk|rrkz)",
 	"VPERM(B|D|W)Z(rr|rrk|rrkz)",
-	"VPERM(I2|Q|T2)(B|D|Q)(Z?)(rr|rrk|rrkz)",
+	"VPERM(I2|Q|T2)(B|D|Q)Z(rr|rrk|rrkz)",
 	"V(P?)COMPRESS(B|W|D|Q|PD|PS|SD|SQ)Z(rr|rrk|rrkz)",
 	"VPEXPAND(B|W)Z(rr|rrk|rrkz)",
-        "VPERMP(S|D)Z(rr|rrk|rrkz)"
+	"VPERMP(S|D)Z(rr|rrk|rrkz)"
 	)>;
 
 // ALU SLOW Misc Instructions
diff --git a/llvm/lib/Target/X86/X86Subtarget.cpp b/llvm/lib/Target/X86/X86Subtarget.cpp
index 0f76daffd46339f101842b3d45662ca6c044db96..d63f1ca1695b209cbf345684460d0069d1677c40 100644
--- a/llvm/lib/Target/X86/X86Subtarget.cpp
+++ b/llvm/lib/Target/X86/X86Subtarget.cpp
@@ -86,16 +86,16 @@ X86Subtarget::classifyLocalReference(const GlobalValue *GV) const {
       CodeModel::Model CM = TM.getCodeModel();
       assert(CM != CodeModel::Tiny &&
              "Tiny codesize model not supported on X86");
-      // In the large code model, even referencing a global under the large data
-      // threshold which is considered "small", we need to use GOTOFF.
+      // In the large code model, all text is far from any global data, so we
+      // use GOTOFF.
       if (CM == CodeModel::Large)
         return X86II::MO_GOTOFF;
-      // Large objects use GOTOFF, otherwise use RIP-rel access.
-      if (auto *GO = dyn_cast_or_null<GlobalObject>(GV))
-        return TM.isLargeGlobalObject(GO) ? X86II::MO_GOTOFF
-                                          : X86II::MO_NO_FLAG;
-      // For non-GlobalObjects, the small and medium code models treat them as
-      // accessible with a RIP-rel access.
+      // Large GlobalValues use GOTOFF, otherwise use RIP-rel access.
+      if (GV)
+        return TM.isLargeGlobalValue(GV) ? X86II::MO_GOTOFF : X86II::MO_NO_FLAG;
+      // GV == nullptr is for all other non-GlobalValue global data like the
+      // constant pool, jump tables, labels, etc. The small and medium code
+      // models treat these as accessible with a RIP-rel access.
       return X86II::MO_NO_FLAG;
     }
 
diff --git a/llvm/lib/Transforms/CFGuard/CFGuard.cpp b/llvm/lib/Transforms/CFGuard/CFGuard.cpp
index 387734358775b3873e2d1531d64ce629a2fb964e..4d4306576017be5f90ecea1dc3908b0672f49b29 100644
--- a/llvm/lib/Transforms/CFGuard/CFGuard.cpp
+++ b/llvm/lib/Transforms/CFGuard/CFGuard.cpp
@@ -34,25 +34,22 @@ namespace {
 
 /// Adds Control Flow Guard (CFG) checks on indirect function calls/invokes.
 /// These checks ensure that the target address corresponds to the start of an
-/// address-taken function. X86_64 targets use the CF_Dispatch mechanism. X86,
-/// ARM, and AArch64 targets use the CF_Check machanism.
-class CFGuard : public FunctionPass {
+/// address-taken function. X86_64 targets use the Mechanism::Dispatch
+/// mechanism. X86, ARM, and AArch64 targets use the Mechanism::Check machanism.
+class CFGuardImpl {
 public:
-  static char ID;
-
-  enum Mechanism { CF_Check, CF_Dispatch };
-
-  // Default constructor required for the INITIALIZE_PASS macro.
-  CFGuard() : FunctionPass(ID) {
-    initializeCFGuardPass(*PassRegistry::getPassRegistry());
-    // By default, use the guard check mechanism.
-    GuardMechanism = CF_Check;
-  }
-
-  // Recommended constructor used to specify the type of guard mechanism.
-  CFGuard(Mechanism Var) : FunctionPass(ID) {
-    initializeCFGuardPass(*PassRegistry::getPassRegistry());
-    GuardMechanism = Var;
+  using Mechanism = CFGuardPass::Mechanism;
+
+  CFGuardImpl(Mechanism M) : GuardMechanism(M) {
+    // Get or insert the guard check or dispatch global symbols.
+    switch (GuardMechanism) {
+    case Mechanism::Check:
+      GuardFnName = "__guard_check_icall_fptr";
+      break;
+    case Mechanism::Dispatch:
+      GuardFnName = "__guard_dispatch_icall_fptr";
+      break;
+    }
   }
 
   /// Inserts a Control Flow Guard (CFG) check on an indirect call using the CFG
@@ -141,21 +138,37 @@ public:
   /// \param CB indirect call to instrument.
   void insertCFGuardDispatch(CallBase *CB);
 
-  bool doInitialization(Module &M) override;
-  bool runOnFunction(Function &F) override;
+  bool doInitialization(Module &M);
+  bool runOnFunction(Function &F);
 
 private:
   // Only add checks if the module has the cfguard=2 flag.
   int cfguard_module_flag = 0;
-  Mechanism GuardMechanism = CF_Check;
+  StringRef GuardFnName;
+  Mechanism GuardMechanism = Mechanism::Check;
   FunctionType *GuardFnType = nullptr;
   PointerType *GuardFnPtrType = nullptr;
   Constant *GuardFnGlobal = nullptr;
 };
 
+class CFGuard : public FunctionPass {
+  CFGuardImpl Impl;
+
+public:
+  static char ID;
+
+  // Default constructor required for the INITIALIZE_PASS macro.
+  CFGuard(CFGuardImpl::Mechanism M) : FunctionPass(ID), Impl(M) {
+    initializeCFGuardPass(*PassRegistry::getPassRegistry());
+  }
+
+  bool doInitialization(Module &M) override { return Impl.doInitialization(M); }
+  bool runOnFunction(Function &F) override { return Impl.runOnFunction(F); }
+};
+
 } // end anonymous namespace
 
-void CFGuard::insertCFGuardCheck(CallBase *CB) {
+void CFGuardImpl::insertCFGuardCheck(CallBase *CB) {
 
   assert(Triple(CB->getModule()->getTargetTriple()).isOSWindows() &&
          "Only applicable for Windows targets");
@@ -184,7 +197,7 @@ void CFGuard::insertCFGuardCheck(CallBase *CB) {
   GuardCheck->setCallingConv(CallingConv::CFGuard_Check);
 }
 
-void CFGuard::insertCFGuardDispatch(CallBase *CB) {
+void CFGuardImpl::insertCFGuardDispatch(CallBase *CB) {
 
   assert(Triple(CB->getModule()->getTargetTriple()).isOSWindows() &&
          "Only applicable for Windows targets");
@@ -218,7 +231,7 @@ void CFGuard::insertCFGuardDispatch(CallBase *CB) {
   CB->eraseFromParent();
 }
 
-bool CFGuard::doInitialization(Module &M) {
+bool CFGuardImpl::doInitialization(Module &M) {
 
   // Check if this module has the cfguard flag and read its value.
   if (auto *MD =
@@ -235,15 +248,6 @@ bool CFGuard::doInitialization(Module &M) {
                         {PointerType::getUnqual(M.getContext())}, false);
   GuardFnPtrType = PointerType::get(GuardFnType, 0);
 
-  // Get or insert the guard check or dispatch global symbols.
-  llvm::StringRef GuardFnName;
-  if (GuardMechanism == CF_Check) {
-    GuardFnName = "__guard_check_icall_fptr";
-  } else if (GuardMechanism == CF_Dispatch) {
-    GuardFnName = "__guard_dispatch_icall_fptr";
-  } else {
-    assert(false && "Invalid CFGuard mechanism");
-  }
   GuardFnGlobal = M.getOrInsertGlobal(GuardFnName, GuardFnPtrType, [&] {
     auto *Var = new GlobalVariable(M, GuardFnPtrType, false,
                                    GlobalVariable::ExternalLinkage, nullptr,
@@ -255,7 +259,7 @@ bool CFGuard::doInitialization(Module &M) {
   return true;
 }
 
-bool CFGuard::runOnFunction(Function &F) {
+bool CFGuardImpl::runOnFunction(Function &F) {
 
   // Skip modules for which CFGuard checks have been disabled.
   if (cfguard_module_flag != 2)
@@ -283,7 +287,7 @@ bool CFGuard::runOnFunction(Function &F) {
   }
 
   // For each indirect call/invoke, add the appropriate dispatch or check.
-  if (GuardMechanism == CF_Dispatch) {
+  if (GuardMechanism == Mechanism::Dispatch) {
     for (CallBase *CB : IndirectCalls) {
       insertCFGuardDispatch(CB);
     }
@@ -296,13 +300,20 @@ bool CFGuard::runOnFunction(Function &F) {
   return true;
 }
 
+PreservedAnalyses CFGuardPass::run(Function &F, FunctionAnalysisManager &FAM) {
+  CFGuardImpl Impl(GuardMechanism);
+  bool Changed = Impl.doInitialization(*F.getParent());
+  Changed |= Impl.runOnFunction(F);
+  return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all();
+}
+
 char CFGuard::ID = 0;
 INITIALIZE_PASS(CFGuard, "CFGuard", "CFGuard", false, false)
 
 FunctionPass *llvm::createCFGuardCheckPass() {
-  return new CFGuard(CFGuard::CF_Check);
+  return new CFGuard(CFGuardPass::Mechanism::Check);
 }
 
 FunctionPass *llvm::createCFGuardDispatchPass() {
-  return new CFGuard(CFGuard::CF_Dispatch);
+  return new CFGuard(CFGuardPass::Mechanism::Dispatch);
 }
diff --git a/llvm/lib/Transforms/Coroutines/CoroFrame.cpp b/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
index 678d7fb3f5719fcfb4b290cefb69c5e38f012cd2..f37b4dc938d3046dab48d6c9b17ca71ffe23c992 100644
--- a/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroFrame.cpp
@@ -964,12 +964,17 @@ static void cacheDIVar(FrameDataInfo &FrameData,
       continue;
 
     SmallVector<DbgDeclareInst *, 1> DDIs;
-    findDbgDeclares(DDIs, V);
-    auto *I = llvm::find_if(DDIs, [](DbgDeclareInst *DDI) {
-      return DDI->getExpression()->getNumElements() == 0;
-    });
-    if (I != DDIs.end())
-      DIVarCache.insert({V, (*I)->getVariable()});
+    SmallVector<DPValue *, 1> DPVs;
+    findDbgDeclares(DDIs, V, &DPVs);
+    auto CacheIt = [&DIVarCache, V](auto &Container) {
+      auto *I = llvm::find_if(Container, [](auto *DDI) {
+        return DDI->getExpression()->getNumElements() == 0;
+      });
+      if (I != Container.end())
+        DIVarCache.insert({V, (*I)->getVariable()});
+    };
+    CacheIt(DDIs);
+    CacheIt(DPVs);
   }
 }
 
@@ -1121,15 +1126,25 @@ static void buildFrameDebugInfo(Function &F, coro::Shape &Shape,
          "Coroutine with switch ABI should own Promise alloca");
 
   SmallVector<DbgDeclareInst *, 1> DIs;
-  findDbgDeclares(DIs, PromiseAlloca);
-  if (DIs.empty())
+  SmallVector<DPValue *, 1> DPVs;
+  findDbgDeclares(DIs, PromiseAlloca, &DPVs);
+
+  DILocalVariable *PromiseDIVariable = nullptr;
+  DILocation *DILoc = nullptr;
+  if (!DIs.empty()) {
+    DbgDeclareInst *PromiseDDI = DIs.front();
+    PromiseDIVariable = PromiseDDI->getVariable();
+    DILoc = PromiseDDI->getDebugLoc().get();
+  } else if (!DPVs.empty()) {
+    DPValue *PromiseDPV = DPVs.front();
+    PromiseDIVariable = PromiseDPV->getVariable();
+    DILoc = PromiseDPV->getDebugLoc().get();
+  } else {
     return;
+  }
 
-  DbgDeclareInst *PromiseDDI = DIs.front();
-  DILocalVariable *PromiseDIVariable = PromiseDDI->getVariable();
   DILocalScope *PromiseDIScope = PromiseDIVariable->getScope();
   DIFile *DFile = PromiseDIScope->getFile();
-  DILocation *DILoc = PromiseDDI->getDebugLoc().get();
   unsigned LineNum = PromiseDIVariable->getLine();
 
   DICompositeType *FrameDITy = DBuilder.createStructType(
@@ -1243,7 +1258,7 @@ static void buildFrameDebugInfo(Function &F, coro::Shape &Shape,
   auto *FrameDIVar = DBuilder.createAutoVariable(PromiseDIScope, "__coro_frame",
                                                  DFile, LineNum, FrameDITy,
                                                  true, DINode::FlagArtificial);
-  assert(FrameDIVar->isValidLocationForIntrinsic(PromiseDDI->getDebugLoc()));
+  assert(FrameDIVar->isValidLocationForIntrinsic(DILoc));
 
   // Subprogram would have ContainedNodes field which records the debug
   // variables it contained. So we need to add __coro_frame to the
@@ -1261,9 +1276,17 @@ static void buildFrameDebugInfo(Function &F, coro::Shape &Shape,
         7, (MDTuple::get(F.getContext(), RetainedNodesVec)));
   }
 
-  DBuilder.insertDeclare(Shape.FramePtr, FrameDIVar,
-                         DBuilder.createExpression(), DILoc,
-                         Shape.getInsertPtAfterFramePtr());
+  if (UseNewDbgInfoFormat) {
+    DPValue *NewDPV = new DPValue(ValueAsMetadata::get(Shape.FramePtr),
+                                  FrameDIVar, DBuilder.createExpression(),
+                                  DILoc, DPValue::LocationType::Declare);
+    BasicBlock::iterator It = Shape.getInsertPtAfterFramePtr();
+    It->getParent()->insertDPValueBefore(NewDPV, It);
+  } else {
+    DBuilder.insertDeclare(Shape.FramePtr, FrameDIVar,
+                           DBuilder.createExpression(), DILoc,
+                           &*Shape.getInsertPtAfterFramePtr());
+  }
 }
 
 // Build a struct that will keep state for an active coroutine.
@@ -1771,7 +1794,7 @@ static void insertSpills(const FrameDataInfo &FrameData, coro::Shape &Shape) {
     if (auto *Arg = dyn_cast<Argument>(Def)) {
       // For arguments, we will place the store instruction right after
       // the coroutine frame pointer instruction, i.e. coro.begin.
-      InsertPt = Shape.getInsertPtAfterFramePtr()->getIterator();
+      InsertPt = Shape.getInsertPtAfterFramePtr();
 
       // If we're spilling an Argument, make sure we clear 'nocapture'
       // from the coroutine function.
@@ -1788,7 +1811,7 @@ static void insertSpills(const FrameDataInfo &FrameData, coro::Shape &Shape) {
       if (!DT.dominates(CB, I)) {
         // If it is not dominated by CoroBegin, then spill should be
         // inserted immediately after CoroFrame is computed.
-        InsertPt = Shape.getInsertPtAfterFramePtr()->getIterator();
+        InsertPt = Shape.getInsertPtAfterFramePtr();
       } else if (auto *II = dyn_cast<InvokeInst>(I)) {
         // If we are spilling the result of the invoke instruction, split
         // the normal edge and insert the spill in the new block.
@@ -1843,7 +1866,8 @@ static void insertSpills(const FrameDataInfo &FrameData, coro::Shape &Shape) {
               SpillAlignment, E.first->getName() + Twine(".reload"));
 
         SmallVector<DbgDeclareInst *, 1> DIs;
-        findDbgDeclares(DIs, Def);
+        SmallVector<DPValue *, 1> DPVs;
+        findDbgDeclares(DIs, Def, &DPVs);
         // Try best to find dbg.declare. If the spill is a temp, there may not
         // be a direct dbg.declare. Walk up the load chain to find one from an
         // alias.
@@ -1858,24 +1882,36 @@ static void insertSpills(const FrameDataInfo &FrameData, coro::Shape &Shape) {
             if (!isa<AllocaInst, LoadInst>(CurDef))
               break;
             DIs.clear();
-            findDbgDeclares(DIs, CurDef);
+            DPVs.clear();
+            findDbgDeclares(DIs, CurDef, &DPVs);
           }
         }
 
-        for (DbgDeclareInst *DDI : DIs) {
+        auto SalvageOne = [&](auto *DDI) {
           bool AllowUnresolved = false;
           // This dbg.declare is preserved for all coro-split function
           // fragments. It will be unreachable in the main function, and
           // processed by coro::salvageDebugInfo() by CoroCloner.
-          DIBuilder(*CurrentBlock->getParent()->getParent(), AllowUnresolved)
-              .insertDeclare(CurrentReload, DDI->getVariable(),
-                             DDI->getExpression(), DDI->getDebugLoc(),
-                             &*Builder.GetInsertPoint());
+          if (UseNewDbgInfoFormat) {
+            DPValue *NewDPV =
+                new DPValue(ValueAsMetadata::get(CurrentReload),
+                            DDI->getVariable(), DDI->getExpression(),
+                            DDI->getDebugLoc(), DPValue::LocationType::Declare);
+            Builder.GetInsertPoint()->getParent()->insertDPValueBefore(
+                NewDPV, Builder.GetInsertPoint());
+          } else {
+            DIBuilder(*CurrentBlock->getParent()->getParent(), AllowUnresolved)
+                .insertDeclare(CurrentReload, DDI->getVariable(),
+                               DDI->getExpression(), DDI->getDebugLoc(),
+                               &*Builder.GetInsertPoint());
+          }
           // This dbg.declare is for the main function entry point.  It
           // will be deleted in all coro-split functions.
-          coro::salvageDebugInfo(ArgToAllocaMap, DDI, Shape.OptimizeFrame,
+          coro::salvageDebugInfo(ArgToAllocaMap, *DDI, Shape.OptimizeFrame,
                                  false /*UseEntryValue*/);
-        }
+        };
+        for_each(DIs, SalvageOne);
+        for_each(DPVs, SalvageOne);
       }
 
       // If we have a single edge PHINode, remove it and replace it with a
@@ -1893,6 +1929,10 @@ static void insertSpills(const FrameDataInfo &FrameData, coro::Shape &Shape) {
       // Replace all uses of CurrentValue in the current instruction with
       // reload.
       U->replaceUsesOfWith(Def, CurrentReload);
+      // Instructions are added to Def's user list if the attached
+      // debug records use Def. Update those now.
+      for (auto &DPV : U->getDbgValueRange())
+        DPV.replaceVariableLocationOp(Def, CurrentReload, true);
     }
   }
 
@@ -1943,9 +1983,12 @@ static void insertSpills(const FrameDataInfo &FrameData, coro::Shape &Shape) {
     G->setName(Alloca->getName() + Twine(".reload.addr"));
 
     SmallVector<DbgVariableIntrinsic *, 4> DIs;
-    findDbgUsers(DIs, Alloca);
+    SmallVector<DPValue *> DPValues;
+    findDbgUsers(DIs, Alloca, &DPValues);
     for (auto *DVI : DIs)
       DVI->replaceUsesOfWith(Alloca, G);
+    for (auto *DPV : DPValues)
+      DPV->replaceVariableLocationOp(Alloca, G);
 
     for (Instruction *I : UsersToUpdate) {
       // It is meaningless to retain the lifetime intrinsics refer for the
@@ -1959,7 +2002,7 @@ static void insertSpills(const FrameDataInfo &FrameData, coro::Shape &Shape) {
       I->replaceUsesOfWith(Alloca, G);
     }
   }
-  Builder.SetInsertPoint(Shape.getInsertPtAfterFramePtr());
+  Builder.SetInsertPoint(&*Shape.getInsertPtAfterFramePtr());
   for (const auto &A : FrameData.Allocas) {
     AllocaInst *Alloca = A.Alloca;
     if (A.MayWriteBeforeCoroBegin) {
@@ -2020,7 +2063,7 @@ static void insertSpills(const FrameDataInfo &FrameData, coro::Shape &Shape) {
              isa<BitCastInst>(Inst);
     });
     if (HasAccessingPromiseBeforeCB) {
-      Builder.SetInsertPoint(Shape.getInsertPtAfterFramePtr());
+      Builder.SetInsertPoint(&*Shape.getInsertPtAfterFramePtr());
       auto *G = GetFramePointer(PA);
       auto *Value = Builder.CreateLoad(PA->getAllocatedType(), PA);
       Builder.CreateStore(Value, G);
@@ -2802,21 +2845,16 @@ static void collectFrameAlloca(AllocaInst *AI, coro::Shape &Shape,
                        Visitor.getMayWriteBeforeCoroBegin());
 }
 
-void coro::salvageDebugInfo(
-    SmallDenseMap<Argument *, AllocaInst *, 4> &ArgToAllocaMap,
-    DbgVariableIntrinsic *DVI, bool OptimizeFrame, bool UseEntryValue) {
-  Function *F = DVI->getFunction();
+static std::optional<std::pair<Value &, DIExpression &>>
+salvageDebugInfoImpl(SmallDenseMap<Argument *, AllocaInst *, 4> &ArgToAllocaMap,
+                     bool OptimizeFrame, bool UseEntryValue, Function *F,
+                     Value *Storage, DIExpression *Expr,
+                     bool SkipOutermostLoad) {
   IRBuilder<> Builder(F->getContext());
   auto InsertPt = F->getEntryBlock().getFirstInsertionPt();
   while (isa<IntrinsicInst>(InsertPt))
     ++InsertPt;
   Builder.SetInsertPoint(&F->getEntryBlock(), InsertPt);
-  DIExpression *Expr = DVI->getExpression();
-  // Follow the pointer arithmetic all the way to the incoming
-  // function argument and convert into a DIExpression.
-  bool SkipOutermostLoad = !isa<DbgValueInst>(DVI);
-  Value *Storage = DVI->getVariableLocationOp(0);
-  Value *OriginalStorage = Storage;
 
   while (auto *Inst = dyn_cast_or_null<Instruction>(Storage)) {
     if (auto *LdInst = dyn_cast<LoadInst>(Inst)) {
@@ -2848,7 +2886,7 @@ void coro::salvageDebugInfo(
     SkipOutermostLoad = false;
   }
   if (!Storage)
-    return;
+    return std::nullopt;
 
   auto *StorageAsArg = dyn_cast<Argument>(Storage);
   const bool IsSwiftAsyncArg =
@@ -2884,21 +2922,78 @@ void coro::salvageDebugInfo(
     Expr = DIExpression::prepend(Expr, DIExpression::DerefBefore);
   }
 
-  DVI->replaceVariableLocationOp(OriginalStorage, Storage);
-  DVI->setExpression(Expr);
+  return {{*Storage, *Expr}};
+}
+
+void coro::salvageDebugInfo(
+    SmallDenseMap<Argument *, AllocaInst *, 4> &ArgToAllocaMap,
+    DbgVariableIntrinsic &DVI, bool OptimizeFrame, bool UseEntryValue) {
+
+  Function *F = DVI.getFunction();
+  // Follow the pointer arithmetic all the way to the incoming
+  // function argument and convert into a DIExpression.
+  bool SkipOutermostLoad = !isa<DbgValueInst>(DVI);
+  Value *OriginalStorage = DVI.getVariableLocationOp(0);
+
+  auto SalvagedInfo = ::salvageDebugInfoImpl(
+      ArgToAllocaMap, OptimizeFrame, UseEntryValue, F, OriginalStorage,
+      DVI.getExpression(), SkipOutermostLoad);
+  if (!SalvagedInfo)
+    return;
+
+  Value *Storage = &SalvagedInfo->first;
+  DIExpression *Expr = &SalvagedInfo->second;
+
+  DVI.replaceVariableLocationOp(OriginalStorage, Storage);
+  DVI.setExpression(Expr);
   // We only hoist dbg.declare today since it doesn't make sense to hoist
   // dbg.value since it does not have the same function wide guarantees that
   // dbg.declare does.
   if (isa<DbgDeclareInst>(DVI)) {
     std::optional<BasicBlock::iterator> InsertPt;
-    if (auto *I = dyn_cast<Instruction>(Storage)) {
+    if (auto *I = dyn_cast<Instruction>(Storage))
       InsertPt = I->getInsertionPointAfterDef();
-      if (I->getDebugLoc())
-        DVI->setDebugLoc(I->getDebugLoc());
-    } else if (isa<Argument>(Storage))
+    else if (isa<Argument>(Storage))
       InsertPt = F->getEntryBlock().begin();
     if (InsertPt)
-      DVI->moveBefore(*(*InsertPt)->getParent(), *InsertPt);
+      DVI.moveBefore(*(*InsertPt)->getParent(), *InsertPt);
+  }
+}
+
+void coro::salvageDebugInfo(
+    SmallDenseMap<Argument *, AllocaInst *, 4> &ArgToAllocaMap, DPValue &DPV,
+    bool OptimizeFrame, bool UseEntryValue) {
+
+  Function *F = DPV.getFunction();
+  // Follow the pointer arithmetic all the way to the incoming
+  // function argument and convert into a DIExpression.
+  bool SkipOutermostLoad = DPV.getType() == DPValue::LocationType::Declare;
+  Value *OriginalStorage = DPV.getVariableLocationOp(0);
+
+  auto SalvagedInfo = ::salvageDebugInfoImpl(
+      ArgToAllocaMap, OptimizeFrame, UseEntryValue, F, OriginalStorage,
+      DPV.getExpression(), SkipOutermostLoad);
+  if (!SalvagedInfo)
+    return;
+
+  Value *Storage = &SalvagedInfo->first;
+  DIExpression *Expr = &SalvagedInfo->second;
+
+  DPV.replaceVariableLocationOp(OriginalStorage, Storage);
+  DPV.setExpression(Expr);
+  // We only hoist dbg.declare today since it doesn't make sense to hoist
+  // dbg.value since it does not have the same function wide guarantees that
+  // dbg.declare does.
+  if (DPV.getType() == DPValue::LocationType::Declare) {
+    std::optional<BasicBlock::iterator> InsertPt;
+    if (auto *I = dyn_cast<Instruction>(Storage))
+      InsertPt = I->getInsertionPointAfterDef();
+    else if (isa<Argument>(Storage))
+      InsertPt = F->getEntryBlock().begin();
+    if (InsertPt) {
+      DPV.removeFromParent();
+      (*InsertPt)->getParent()->insertDPValueBefore(&DPV, *InsertPt);
+    }
   }
 }
 
@@ -3089,10 +3184,15 @@ void coro::buildCoroutineFrame(
   for (auto &Iter : FrameData.Spills) {
     auto *V = Iter.first;
     SmallVector<DbgValueInst *, 16> DVIs;
-    findDbgValues(DVIs, V);
+    SmallVector<DPValue *, 16> DPVs;
+    findDbgValues(DVIs, V, &DPVs);
     for (DbgValueInst *DVI : DVIs)
       if (Checker.isDefinitionAcrossSuspend(*V, DVI))
         FrameData.Spills[V].push_back(DVI);
+    // Add the instructions which carry debug info that is in the frame.
+    for (DPValue *DPV : DPVs)
+      if (Checker.isDefinitionAcrossSuspend(*V, DPV->Marker->MarkedInstr))
+        FrameData.Spills[V].push_back(DPV->Marker->MarkedInstr);
   }
 
   LLVM_DEBUG(dumpSpills("Spills", FrameData.Spills));
diff --git a/llvm/lib/Transforms/Coroutines/CoroInternal.h b/llvm/lib/Transforms/Coroutines/CoroInternal.h
index 0856c4925cc5d5e857eda70b26cc162c66ed153a..fb16a4090689b438401cfa4eed328c1b0f7040ec 100644
--- a/llvm/lib/Transforms/Coroutines/CoroInternal.h
+++ b/llvm/lib/Transforms/Coroutines/CoroInternal.h
@@ -32,7 +32,10 @@ void replaceCoroFree(CoroIdInst *CoroId, bool Elide);
 /// OptimizeFrame is false.
 void salvageDebugInfo(
     SmallDenseMap<Argument *, AllocaInst *, 4> &ArgToAllocaMap,
-    DbgVariableIntrinsic *DVI, bool OptimizeFrame, bool IsEntryPoint);
+    DbgVariableIntrinsic &DVI, bool OptimizeFrame, bool IsEntryPoint);
+void salvageDebugInfo(
+    SmallDenseMap<Argument *, AllocaInst *, 4> &ArgToAllocaMap, DPValue &DPV,
+    bool OptimizeFrame, bool UseEntryValue);
 
 // Keeps data and helper functions for lowering coroutine intrinsics.
 struct LowererBase {
@@ -240,10 +243,13 @@ struct LLVM_LIBRARY_VISIBILITY Shape {
     return nullptr;
   }
 
-  Instruction *getInsertPtAfterFramePtr() const {
-    if (auto *I = dyn_cast<Instruction>(FramePtr))
-      return I->getNextNode();
-    return &cast<Argument>(FramePtr)->getParent()->getEntryBlock().front();
+  BasicBlock::iterator getInsertPtAfterFramePtr() const {
+    if (auto *I = dyn_cast<Instruction>(FramePtr)) {
+      BasicBlock::iterator It = std::next(I->getIterator());
+      It.setHeadBit(true); // Copy pre-RemoveDIs behaviour.
+      return It;
+    }
+    return cast<Argument>(FramePtr)->getParent()->getEntryBlock().begin();
   }
 
   /// Allocate memory according to the rules of the active lowering.
diff --git a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
index 244580f503d5b423291ca86a85b7917f8b296f41..7758b52abc204667da5d4b9a869f78fd708cdaa6 100644
--- a/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
+++ b/llvm/lib/Transforms/Coroutines/CoroSplit.cpp
@@ -725,13 +725,17 @@ static void replaceSwiftErrorOps(Function &F, coro::Shape &Shape,
 }
 
 /// Returns all DbgVariableIntrinsic in F.
-static SmallVector<DbgVariableIntrinsic *, 8>
+static std::pair<SmallVector<DbgVariableIntrinsic *, 8>, SmallVector<DPValue *>>
 collectDbgVariableIntrinsics(Function &F) {
   SmallVector<DbgVariableIntrinsic *, 8> Intrinsics;
-  for (auto &I : instructions(F))
+  SmallVector<DPValue *> DPValues;
+  for (auto &I : instructions(F)) {
+    for (DPValue &DPV : I.getDbgValueRange())
+      DPValues.push_back(&DPV);
     if (auto *DVI = dyn_cast<DbgVariableIntrinsic>(&I))
       Intrinsics.push_back(DVI);
-  return Intrinsics;
+  }
+  return {Intrinsics, DPValues};
 }
 
 void CoroCloner::replaceSwiftErrorOps() {
@@ -739,15 +743,17 @@ void CoroCloner::replaceSwiftErrorOps() {
 }
 
 void CoroCloner::salvageDebugInfo() {
-  SmallVector<DbgVariableIntrinsic *, 8> Worklist =
-      collectDbgVariableIntrinsics(*NewF);
+  auto [Worklist, DPValues] = collectDbgVariableIntrinsics(*NewF);
   SmallDenseMap<Argument *, AllocaInst *, 4> ArgToAllocaMap;
 
   // Only 64-bit ABIs have a register we can refer to with the entry value.
   bool UseEntryValue =
       llvm::Triple(OrigF.getParent()->getTargetTriple()).isArch64Bit();
   for (DbgVariableIntrinsic *DVI : Worklist)
-    coro::salvageDebugInfo(ArgToAllocaMap, DVI, Shape.OptimizeFrame,
+    coro::salvageDebugInfo(ArgToAllocaMap, *DVI, Shape.OptimizeFrame,
+                           UseEntryValue);
+  for (DPValue *DPV : DPValues)
+    coro::salvageDebugInfo(ArgToAllocaMap, *DPV, Shape.OptimizeFrame,
                            UseEntryValue);
 
   // Remove all salvaged dbg.declare intrinsics that became
@@ -757,7 +763,7 @@ void CoroCloner::salvageDebugInfo() {
     return !isPotentiallyReachable(&NewF->getEntryBlock(), BB, nullptr,
                                    &DomTree);
   };
-  for (DbgVariableIntrinsic *DVI : Worklist) {
+  auto RemoveOne = [&](auto *DVI) {
     if (IsUnreachableBlock(DVI->getParent()))
       DVI->eraseFromParent();
     else if (isa_and_nonnull<AllocaInst>(DVI->getVariableLocationOp(0))) {
@@ -770,7 +776,9 @@ void CoroCloner::salvageDebugInfo() {
       if (!Uses)
         DVI->eraseFromParent();
     }
-  }
+  };
+  for_each(Worklist, RemoveOne);
+  for_each(DPValues, RemoveOne);
 }
 
 void CoroCloner::replaceEntryBlock() {
@@ -1243,7 +1251,7 @@ static void updateCoroFrame(coro::Shape &Shape, Function *ResumeFn,
                             Function *DestroyFn, Function *CleanupFn) {
   assert(Shape.ABI == coro::ABI::Switch);
 
-  IRBuilder<> Builder(Shape.getInsertPtAfterFramePtr());
+  IRBuilder<> Builder(&*Shape.getInsertPtAfterFramePtr());
 
   auto *ResumeAddr = Builder.CreateStructGEP(
       Shape.FrameTy, Shape.FramePtr, coro::Shape::SwitchFieldIndex::Resume,
@@ -2039,10 +2047,13 @@ splitCoroutine(Function &F, SmallVectorImpl<Function *> &Clones,
   // original function. The Cloner has already salvaged debug info in the new
   // coroutine funclets.
   SmallDenseMap<Argument *, AllocaInst *, 4> ArgToAllocaMap;
-  for (auto *DDI : collectDbgVariableIntrinsics(F))
-    coro::salvageDebugInfo(ArgToAllocaMap, DDI, Shape.OptimizeFrame,
+  auto [DbgInsts, DPValues] = collectDbgVariableIntrinsics(F);
+  for (auto *DDI : DbgInsts)
+    coro::salvageDebugInfo(ArgToAllocaMap, *DDI, Shape.OptimizeFrame,
+                           false /*UseEntryValue*/);
+  for (DPValue *DPV : DPValues)
+    coro::salvageDebugInfo(ArgToAllocaMap, *DPV, Shape.OptimizeFrame,
                            false /*UseEntryValue*/);
-
   return Shape;
 }
 
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
index 6002f599ca71ab32ccbdce3d7d77435f98e3f6dc..31db1d3164b772dcd77a22ef78d3ee4cb4b9024f 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
@@ -4280,6 +4280,12 @@ Instruction *InstCombinerImpl::foldNot(BinaryOperator &I) {
     if (match(NotVal, m_AShr(m_Not(m_Value(X)), m_Value(Y))))
       return BinaryOperator::CreateAShr(X, Y);
 
+    // Treat lshr with non-negative operand as ashr.
+    // ~(~X >>u Y) --> (X >>s Y) iff X is known negative
+    if (match(NotVal, m_LShr(m_Not(m_Value(X)), m_Value(Y))) &&
+        isKnownNegative(X, SQ.getWithInstruction(NotVal)))
+      return BinaryOperator::CreateAShr(X, Y);
+
     // Bit-hack form of a signbit test for iN type:
     // ~(X >>s (N - 1)) --> sext i1 (X > -1) to iN
     unsigned FullShift = Ty->getScalarSizeInBits() - 1;
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
index 255ce6973a16fbac643345bd49f9095e7b391bb1..1539fa9a3269e13e9eaa43adfc5676bb7b899258 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp
@@ -1536,6 +1536,9 @@ Instruction *InstCombinerImpl::visitCallInst(CallInst &CI) {
   }
 
   if (II->isCommutative()) {
+    if (Instruction *I = foldCommutativeIntrinsicOverSelects(*II))
+      return I;
+
     if (CallInst *NewCall = canonicalizeConstantArg0ToArg1(CI))
       return NewCall;
   }
@@ -4217,3 +4220,20 @@ InstCombinerImpl::transformCallThroughTrampoline(CallBase &Call,
   Call.setCalledFunction(FTy, NestF);
   return &Call;
 }
+
+// op(select(%v, %x, %y), select(%v, %y, %x)) --> op(%x, %y)
+Instruction *
+InstCombinerImpl::foldCommutativeIntrinsicOverSelects(IntrinsicInst &II) {
+  assert(II.isCommutative());
+
+  Value *A, *B, *C;
+  if (match(II.getOperand(0), m_Select(m_Value(A), m_Value(B), m_Value(C))) &&
+      match(II.getOperand(1),
+            m_Select(m_Specific(A), m_Specific(C), m_Specific(B)))) {
+    replaceOperand(II, 0, B);
+    replaceOperand(II, 1, C);
+    return &II;
+  }
+
+  return nullptr;
+}
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
index 91642e3babceec2bbd2652b66c8aa10d4b8d41ae..289976718e52f30a30afed5d4feb99c677aacbf7 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp
@@ -1323,34 +1323,21 @@ Instruction *InstCombinerImpl::foldICmpWithConstant(ICmpInst &Cmp) {
 
 /// Canonicalize icmp instructions based on dominating conditions.
 Instruction *InstCombinerImpl::foldICmpWithDominatingICmp(ICmpInst &Cmp) {
-  // This is a cheap/incomplete check for dominance - just match a single
-  // predecessor with a conditional branch.
-  BasicBlock *CmpBB = Cmp.getParent();
-  BasicBlock *DomBB = CmpBB->getSinglePredecessor();
-  if (!DomBB)
-    return nullptr;
-
-  Value *DomCond;
-  BasicBlock *TrueBB, *FalseBB;
-  if (!match(DomBB->getTerminator(), m_Br(m_Value(DomCond), TrueBB, FalseBB)))
-    return nullptr;
-
-  assert((TrueBB == CmpBB || FalseBB == CmpBB) &&
-         "Predecessor block does not point to successor?");
-
-  // The branch should get simplified. Don't bother simplifying this condition.
-  if (TrueBB == FalseBB)
-    return nullptr;
-
   // We already checked simple implication in InstSimplify, only handle complex
   // cases here.
-
-  CmpInst::Predicate Pred = Cmp.getPredicate();
   Value *X = Cmp.getOperand(0), *Y = Cmp.getOperand(1);
   ICmpInst::Predicate DomPred;
-  const APInt *C, *DomC;
-  if (match(DomCond, m_ICmp(DomPred, m_Specific(X), m_APInt(DomC))) &&
-      match(Y, m_APInt(C))) {
+  const APInt *C;
+  if (!match(Y, m_APInt(C)))
+    return nullptr;
+
+  CmpInst::Predicate Pred = Cmp.getPredicate();
+  ConstantRange CR = ConstantRange::makeExactICmpRegion(Pred, *C);
+
+  auto handleDomCond = [&](Value *DomCond, bool CondIsTrue) -> Instruction * {
+    const APInt *DomC;
+    if (!match(DomCond, m_ICmp(DomPred, m_Specific(X), m_APInt(DomC))))
+      return nullptr;
     // We have 2 compares of a variable with constants. Calculate the constant
     // ranges of those compares to see if we can transform the 2nd compare:
     // DomBB:
@@ -1358,11 +1345,10 @@ Instruction *InstCombinerImpl::foldICmpWithDominatingICmp(ICmpInst &Cmp) {
     //   br DomCond, CmpBB, FalseBB
     // CmpBB:
     //   Cmp = icmp Pred X, C
-    ConstantRange CR = ConstantRange::makeExactICmpRegion(Pred, *C);
+    if (!CondIsTrue)
+      DomPred = CmpInst::getInversePredicate(DomPred);
     ConstantRange DominatingCR =
-        (CmpBB == TrueBB) ? ConstantRange::makeExactICmpRegion(DomPred, *DomC)
-                          : ConstantRange::makeExactICmpRegion(
-                                CmpInst::getInversePredicate(DomPred), *DomC);
+        ConstantRange::makeExactICmpRegion(DomPred, *DomC);
     ConstantRange Intersection = DominatingCR.intersectWith(CR);
     ConstantRange Difference = DominatingCR.difference(CR);
     if (Intersection.isEmptySet())
@@ -1390,6 +1376,21 @@ Instruction *InstCombinerImpl::foldICmpWithDominatingICmp(ICmpInst &Cmp) {
       return new ICmpInst(ICmpInst::ICMP_EQ, X, Builder.getInt(*EqC));
     if (const APInt *NeC = Difference.getSingleElement())
       return new ICmpInst(ICmpInst::ICMP_NE, X, Builder.getInt(*NeC));
+    return nullptr;
+  };
+
+  for (BranchInst *BI : DC.conditionsFor(X)) {
+    auto *Cond = BI->getCondition();
+    BasicBlockEdge Edge0(BI->getParent(), BI->getSuccessor(0));
+    if (DT.dominates(Edge0, Cmp.getParent())) {
+      if (auto *V = handleDomCond(Cond, true))
+        return V;
+    } else {
+      BasicBlockEdge Edge1(BI->getParent(), BI->getSuccessor(1));
+      if (DT.dominates(Edge1, Cmp.getParent()))
+        if (auto *V = handleDomCond(Cond, false))
+          return V;
+    }
   }
 
   return nullptr;
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
index bb620ad8d41c13c4de91940b8882820d1510d59a..1d50fa9b6bf74b94dfdfddfc240dbc3f2f97ac1e 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
+++ b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h
@@ -276,6 +276,7 @@ private:
   bool transformConstExprCastCall(CallBase &Call);
   Instruction *transformCallThroughTrampoline(CallBase &Call,
                                               IntrinsicInst &Tramp);
+  Instruction *foldCommutativeIntrinsicOverSelects(IntrinsicInst &II);
 
   Value *simplifyMaskedLoad(IntrinsicInst &II);
   Instruction *simplifyMaskedStore(IntrinsicInst &II);
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp b/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
index 8d5866e98a8ef9c7f15f87abd42fa83e03cb9d99..e5566578869ddcae7bd0d2c28e13add51db286c0 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp
@@ -1205,6 +1205,38 @@ Instruction *InstCombinerImpl::commonIDivTransforms(BinaryOperator &I) {
     }
   }
 
+  // (X * Y) / (X * Z) --> Y / Z (and commuted variants)
+  if (match(Op0, m_Mul(m_Value(X), m_Value(Y)))) {
+    auto OB0HasNSW = cast<OverflowingBinaryOperator>(Op0)->hasNoSignedWrap();
+    auto OB0HasNUW = cast<OverflowingBinaryOperator>(Op0)->hasNoUnsignedWrap();
+
+    auto CreateDivOrNull = [&](Value *A, Value *B) -> Instruction * {
+      auto OB1HasNSW = cast<OverflowingBinaryOperator>(Op1)->hasNoSignedWrap();
+      auto OB1HasNUW =
+          cast<OverflowingBinaryOperator>(Op1)->hasNoUnsignedWrap();
+      const APInt *C1, *C2;
+      if (IsSigned && OB0HasNSW) {
+        if (OB1HasNSW && match(B, m_APInt(C1)) && !C1->isAllOnes())
+          return BinaryOperator::CreateSDiv(A, B);
+      }
+      if (!IsSigned && OB0HasNUW) {
+        if (OB1HasNUW)
+          return BinaryOperator::CreateUDiv(A, B);
+        if (match(A, m_APInt(C1)) && match(B, m_APInt(C2)) && C2->ule(*C1))
+          return BinaryOperator::CreateUDiv(A, B);
+      }
+      return nullptr;
+    };
+
+    if (match(Op1, m_c_Mul(m_Specific(X), m_Value(Z)))) {
+      if (auto *Val = CreateDivOrNull(Y, Z))
+        return Val;
+    }
+    if (match(Op1, m_c_Mul(m_Specific(Y), m_Value(Z)))) {
+      if (auto *Val = CreateDivOrNull(X, Z))
+        return Val;
+    }
+  }
   return nullptr;
 }
 
@@ -1375,20 +1407,7 @@ Instruction *InstCombinerImpl::visitUDiv(BinaryOperator &I) {
   if (Instruction *NarrowDiv = narrowUDivURem(I, *this))
     return NarrowDiv;
 
-  // If the udiv operands are non-overflowing multiplies with a common operand,
-  // then eliminate the common factor:
-  // (A * B) / (A * X) --> B / X (and commuted variants)
-  // TODO: The code would be reduced if we had m_c_NUWMul pattern matching.
-  // TODO: If -reassociation handled this generally, we could remove this.
   Value *A, *B;
-  if (match(Op0, m_NUWMul(m_Value(A), m_Value(B)))) {
-    if (match(Op1, m_NUWMul(m_Specific(A), m_Value(X))) ||
-        match(Op1, m_NUWMul(m_Value(X), m_Specific(A))))
-      return BinaryOperator::CreateUDiv(B, X);
-    if (match(Op1, m_NUWMul(m_Specific(B), m_Value(X))) ||
-        match(Op1, m_NUWMul(m_Value(X), m_Specific(B))))
-      return BinaryOperator::CreateUDiv(A, X);
-  }
 
   // Look through a right-shift to find the common factor:
   // ((Op1 *nuw A) >> B) / Op1 --> A >> B
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
index 046ce9d1207e8e2e60af3fec568712af7c16c0a4..846116a929b156f8131153be05a66e4b71776625 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineSimplifyDemanded.cpp
@@ -552,6 +552,17 @@ Value *InstCombinerImpl::SimplifyDemandedUseBits(Value *V, APInt DemandedMask,
     if (DemandedFromOps.isSubsetOf(LHSKnown.Zero))
       return I->getOperand(1);
 
+    // (add X, C) --> (xor X, C) IFF C is equal to the top bit of the DemandMask
+    {
+      const APInt *C;
+      if (match(I->getOperand(1), m_APInt(C)) &&
+          C->isOneBitSet(DemandedMask.getActiveBits() - 1)) {
+        IRBuilderBase::InsertPointGuard Guard(Builder);
+        Builder.SetInsertPoint(I);
+        return Builder.CreateXor(I->getOperand(0), ConstantInt::get(VTy, *C));
+      }
+    }
+
     // Otherwise just compute the known bits of the result.
     bool NSW = cast<OverflowingBinaryOperator>(I)->hasNoSignedWrap();
     Known = KnownBits::computeForAddSub(true, NSW, LHSKnown, RHSKnown);
diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
index f072f5cec3094ad80c30c0a20fc30b725aba052f..6ac1fdb9252bff470cdad7b211a01e76bfdceeca 100644
--- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp
@@ -1132,6 +1132,14 @@ Value *InstCombinerImpl::SimplifySelectsFeedingBinaryOp(BinaryOperator &I,
   };
 
   if (LHSIsSelect && RHSIsSelect && A == D) {
+    // op(select(%v, %x, %y), select(%v, %y, %x)) --> op(%x, %y)
+    if (I.isCommutative() && B == F && C == E) {
+      Value *BI = Builder.CreateBinOp(I.getOpcode(), B, E);
+      if (auto *BO = dyn_cast<BinaryOperator>(BI))
+        BO->copyIRFlags(&I);
+      return BI;
+    }
+
     // (A ? B : C) op (A ? E : F) -> A ? (B op E) : (C op F)
     Cond = A;
     True = simplifyBinOp(Opcode, B, E, FMF, Q);
@@ -2182,16 +2190,6 @@ Value *InstCombiner::getFreelyInvertedImpl(Value *V, bool WillInvertAllUses,
     return nullptr;
   }
 
-  // Treat lshr with non-negative operand as ashr.
-  if (match(V, m_LShr(m_Value(A), m_Value(B))) &&
-      isKnownNonNegative(A, SQ.getWithInstruction(cast<Instruction>(V)),
-                         Depth)) {
-    if (auto *AV = getFreelyInvertedImpl(A, A->hasOneUse(), Builder,
-                                         DoesConsume, Depth))
-      return Builder ? Builder->CreateAShr(AV, B) : NonNull;
-    return nullptr;
-  }
-
   Value *Cond;
   // LogicOps are special in that we canonicalize them at the cost of an
   // instruction.
diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
index 4a5a0b25bebbaf146c71a51ebe894af9527efa8b..57ff1648788f9ac8475990f10d71392240c8f39a 100644
--- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
+++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
@@ -1783,6 +1783,8 @@ static bool skipPGOUse(const Function &F) {
 static bool skipPGOGen(const Function &F) {
   if (skipPGOUse(F))
     return true;
+  if (F.hasFnAttribute(llvm::Attribute::Naked))
+    return true;
   if (F.hasFnAttribute(llvm::Attribute::NoProfile))
     return true;
   if (F.hasFnAttribute(llvm::Attribute::SkipProfile))
diff --git a/llvm/lib/Transforms/Scalar/ConstantHoisting.cpp b/llvm/lib/Transforms/Scalar/ConstantHoisting.cpp
index 3e5d979f11cc50adff3fd53aa95d65712d549ff6..1fb9d7fff32f661cb8f5e525d9b74a57fece3db7 100644
--- a/llvm/lib/Transforms/Scalar/ConstantHoisting.cpp
+++ b/llvm/lib/Transforms/Scalar/ConstantHoisting.cpp
@@ -523,7 +523,8 @@ void ConstantHoistingPass::collectConstantCandidates(Function &Fn) {
     if (!DT->isReachableFromEntry(&BB))
       continue;
     for (Instruction &Inst : BB)
-      collectConstantCandidates(ConstCandMap, &Inst);
+      if (!TTI->preferToKeepConstantsAttached(Inst, Fn))
+        collectConstantCandidates(ConstCandMap, &Inst);
   }
 }
 
diff --git a/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp b/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp
index fafbe17583f5da8cc3d58e1b4cf6f57cc580aaee..18266ba0789897a3d2f612e739163d14f03e71fc 100644
--- a/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp
+++ b/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp
@@ -1109,6 +1109,16 @@ void State::addInfoFor(BasicBlock &BB) {
         CmpI->getOperand(1)));
 }
 
+#ifndef NDEBUG
+static void dumpUnpackedICmp(raw_ostream &OS, ICmpInst::Predicate Pred,
+                             Value *LHS, Value *RHS) {
+  OS << "icmp " << Pred << ' ';
+  LHS->printAsOperand(OS, /*PrintType=*/true);
+  OS << ", ";
+  RHS->printAsOperand(OS, /*PrintType=*/false);
+}
+#endif
+
 namespace {
 /// Helper to keep track of a condition and if it should be treated as negated
 /// for reproducer construction.
@@ -1241,10 +1251,9 @@ static void generateReproducer(CmpInst *Cond, Module *M,
     if (Entry.Pred == ICmpInst::BAD_ICMP_PREDICATE)
       continue;
 
-    LLVM_DEBUG(
-        dbgs() << "  Materializing assumption icmp " << Entry.Pred << ' ';
-        Entry.LHS->printAsOperand(dbgs(), /*PrintType=*/true); dbgs() << ", ";
-        Entry.RHS->printAsOperand(dbgs(), /*PrintType=*/false); dbgs() << "\n");
+    LLVM_DEBUG(dbgs() << "  Materializing assumption ";
+               dumpUnpackedICmp(dbgs(), Entry.Pred, Entry.LHS, Entry.RHS);
+               dbgs() << "\n");
     CloneInstructions({Entry.LHS, Entry.RHS}, CmpInst::isSigned(Entry.Pred));
 
     auto *Cmp = Builder.CreateICmp(Entry.Pred, Entry.LHS, Entry.RHS);
@@ -1260,14 +1269,12 @@ static void generateReproducer(CmpInst *Cond, Module *M,
   assert(!verifyFunction(*F, &dbgs()));
 }
 
-static std::optional<bool> checkCondition(CmpInst *Cmp, ConstraintInfo &Info,
-                                          unsigned NumIn, unsigned NumOut,
+static std::optional<bool> checkCondition(CmpInst::Predicate Pred, Value *A,
+                                          Value *B, Instruction *CheckInst,
+                                          ConstraintInfo &Info, unsigned NumIn,
+                                          unsigned NumOut,
                                           Instruction *ContextInst) {
-  LLVM_DEBUG(dbgs() << "Checking " << *Cmp << "\n");
-
-  CmpInst::Predicate Pred = Cmp->getPredicate();
-  Value *A = Cmp->getOperand(0);
-  Value *B = Cmp->getOperand(1);
+  LLVM_DEBUG(dbgs() << "Checking " << *CheckInst << "\n");
 
   auto R = Info.getConstraintForSolving(Pred, A, B);
   if (R.empty() || !R.isValid(Info)){
@@ -1292,13 +1299,10 @@ static std::optional<bool> checkCondition(CmpInst *Cmp, ConstraintInfo &Info,
       return std::nullopt;
 
     LLVM_DEBUG({
-      if (*ImpliedCondition) {
-        dbgs() << "Condition " << *Cmp;
-      } else {
-        auto InversePred = Cmp->getInversePredicate();
-        dbgs() << "Condition " << CmpInst::getPredicateName(InversePred) << " "
-               << *A << ", " << *B;
-      }
+      dbgs() << "Condition ";
+      dumpUnpackedICmp(
+          dbgs(), *ImpliedCondition ? Pred : CmpInst::getInversePredicate(Pred),
+          A, B);
       dbgs() << " implied by dominating constraints\n";
       CSToUse.dump();
     });
@@ -1338,8 +1342,9 @@ static bool checkAndReplaceCondition(
     return true;
   };
 
-  if (auto ImpliedCondition =
-          checkCondition(Cmp, Info, NumIn, NumOut, ContextInst))
+  if (auto ImpliedCondition = checkCondition(
+          Cmp->getPredicate(), Cmp->getOperand(0), Cmp->getOperand(1), Cmp,
+          Info, NumIn, NumOut, ContextInst))
     return ReplaceCmpWithConstant(Cmp, *ImpliedCondition);
   return false;
 }
@@ -1380,9 +1385,10 @@ static bool checkAndSecondOpImpliedByFirst(
 
   bool Changed = false;
   // Check if the second condition can be simplified now.
-  if (auto ImpliedCondition =
-          checkCondition(cast<ICmpInst>(And->getOperand(1)), Info, CB.NumIn,
-                         CB.NumOut, CB.getContextInst())) {
+  ICmpInst *Cmp = cast<ICmpInst>(And->getOperand(1));
+  if (auto ImpliedCondition = checkCondition(
+          Cmp->getPredicate(), Cmp->getOperand(0), Cmp->getOperand(1), Cmp,
+          Info, CB.NumIn, CB.NumOut, CB.getContextInst())) {
     And->setOperand(1, ConstantInt::getBool(And->getType(), *ImpliedCondition));
     Changed = true;
   }
@@ -1408,9 +1414,8 @@ void ConstraintInfo::addFact(CmpInst::Predicate Pred, Value *A, Value *B,
   if (!R.isValid(*this) || R.isNe())
     return;
 
-  LLVM_DEBUG(dbgs() << "Adding '" << Pred << " ";
-             A->printAsOperand(dbgs(), false); dbgs() << ", ";
-             B->printAsOperand(dbgs(), false); dbgs() << "'\n");
+  LLVM_DEBUG(dbgs() << "Adding '"; dumpUnpackedICmp(dbgs(), Pred, A, B);
+             dbgs() << "'\n");
   bool Added = false;
   auto &CSToUse = getCS(R.IsSigned);
   if (R.Coefficients.empty())
@@ -1616,10 +1621,8 @@ static bool eliminateConstraints(Function &F, DominatorTree &DT, LoopInfo &LI,
     }
 
     auto AddFact = [&](CmpInst::Predicate Pred, Value *A, Value *B) {
-      LLVM_DEBUG(dbgs() << "fact to add to the system: "
-                        << CmpInst::getPredicateName(Pred) << " ";
-                 A->printAsOperand(dbgs()); dbgs() << ", ";
-                 B->printAsOperand(dbgs(), false); dbgs() << "\n");
+      LLVM_DEBUG(dbgs() << "fact to add to the system: ";
+                 dumpUnpackedICmp(dbgs(), Pred, A, B); dbgs() << "\n");
       if (Info.getCS(CmpInst::isSigned(Pred)).size() > MaxRows) {
         LLVM_DEBUG(
             dbgs()
diff --git a/llvm/lib/Transforms/Scalar/LICM.cpp b/llvm/lib/Transforms/Scalar/LICM.cpp
index d0afe09ce41dfbf4dd0f19bd419e10ef8c298c6d..9117378568b7edd0bd527ba1dc6df7485a183c3e 100644
--- a/llvm/lib/Transforms/Scalar/LICM.cpp
+++ b/llvm/lib/Transforms/Scalar/LICM.cpp
@@ -193,7 +193,7 @@ static Instruction *cloneInstructionInExitBlock(
 static void eraseInstruction(Instruction &I, ICFLoopSafetyInfo &SafetyInfo,
                              MemorySSAUpdater &MSSAU);
 
-static void moveInstructionBefore(Instruction &I, Instruction &Dest,
+static void moveInstructionBefore(Instruction &I, BasicBlock::iterator Dest,
                                   ICFLoopSafetyInfo &SafetyInfo,
                                   MemorySSAUpdater &MSSAU, ScalarEvolution *SE);
 
@@ -1011,7 +1011,8 @@ bool llvm::hoistRegion(DomTreeNode *N, AAResults *AA, LoopInfo *LI,
         LLVM_DEBUG(dbgs() << "LICM rehoisting to "
                           << HoistPoint->getParent()->getNameOrAsOperand()
                           << ": " << *I << "\n");
-        moveInstructionBefore(*I, *HoistPoint, *SafetyInfo, MSSAU, SE);
+        moveInstructionBefore(*I, HoistPoint->getIterator(), *SafetyInfo, MSSAU,
+                              SE);
         HoistPoint = I;
         Changed = true;
       }
@@ -1491,16 +1492,17 @@ static void eraseInstruction(Instruction &I, ICFLoopSafetyInfo &SafetyInfo,
   I.eraseFromParent();
 }
 
-static void moveInstructionBefore(Instruction &I, Instruction &Dest,
+static void moveInstructionBefore(Instruction &I, BasicBlock::iterator Dest,
                                   ICFLoopSafetyInfo &SafetyInfo,
                                   MemorySSAUpdater &MSSAU,
                                   ScalarEvolution *SE) {
   SafetyInfo.removeInstruction(&I);
-  SafetyInfo.insertInstructionTo(&I, Dest.getParent());
-  I.moveBefore(&Dest);
+  SafetyInfo.insertInstructionTo(&I, Dest->getParent());
+  I.moveBefore(*Dest->getParent(), Dest);
   if (MemoryUseOrDef *OldMemAcc = cast_or_null<MemoryUseOrDef>(
           MSSAU.getMemorySSA()->getMemoryAccess(&I)))
-    MSSAU.moveToPlace(OldMemAcc, Dest.getParent(), MemorySSA::BeforeTerminator);
+    MSSAU.moveToPlace(OldMemAcc, Dest->getParent(),
+                      MemorySSA::BeforeTerminator);
   if (SE)
     SE->forgetBlockAndLoopDispositions(&I);
 }
@@ -1747,10 +1749,11 @@ static void hoist(Instruction &I, const DominatorTree *DT, const Loop *CurLoop,
 
   if (isa<PHINode>(I))
     // Move the new node to the end of the phi list in the destination block.
-    moveInstructionBefore(I, *Dest->getFirstNonPHI(), *SafetyInfo, MSSAU, SE);
+    moveInstructionBefore(I, Dest->getFirstNonPHIIt(), *SafetyInfo, MSSAU, SE);
   else
     // Move the new node to the destination block, before its terminator.
-    moveInstructionBefore(I, *Dest->getTerminator(), *SafetyInfo, MSSAU, SE);
+    moveInstructionBefore(I, Dest->getTerminator()->getIterator(), *SafetyInfo,
+                          MSSAU, SE);
 
   I.updateLocationAfterHoist();
 
diff --git a/llvm/lib/Transforms/Scalar/LoopVersioningLICM.cpp b/llvm/lib/Transforms/Scalar/LoopVersioningLICM.cpp
index 13e06c79d0d7eb63749906dc6928e6b181a6a1d8..9d5e6693c0e53a60e28a9b77a4825777bc969952 100644
--- a/llvm/lib/Transforms/Scalar/LoopVersioningLICM.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopVersioningLICM.cpp
@@ -266,6 +266,11 @@ bool LoopVersioningLICM::legalLoopMemoryAccesses() {
     for (const auto &A : AS) {
       Value *Ptr = A.getValue();
       // Alias tracker should have pointers of same data type.
+      //
+      // FIXME: check no longer effective since opaque pointers?
+      // If the intent is to check that the memory accesses use the
+      // same data type (such that LICM can promote them), then we
+      // can no longer see this from the pointer value types.
       TypeCheck = (TypeCheck && (SomePtr->getType() == Ptr->getType()));
     }
     // At least one alias tracker should have pointers of same data type.
diff --git a/llvm/lib/Transforms/Utils/CodeExtractor.cpp b/llvm/lib/Transforms/Utils/CodeExtractor.cpp
index 9c1186232e024bb9b0a6364622365f21fa5dce7e..f5abed0dd517865fa982ed67f550ccc6b11ca898 100644
--- a/llvm/lib/Transforms/Utils/CodeExtractor.cpp
+++ b/llvm/lib/Transforms/Utils/CodeExtractor.cpp
@@ -998,6 +998,7 @@ Function *CodeExtractor::constructFunction(const ValueSet &inputs,
       case Attribute::ByRef:
       case Attribute::WriteOnly:
       case Attribute::Writable:
+      case Attribute::DeadOnUnwind:
       //  These are not really attributes.
       case Attribute::None:
       case Attribute::EndAttrKinds:
diff --git a/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp b/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp
index 76280ed492b3db041c0f3f72e8555cf3d42d5bb5..504f4430dc2ca66866e076e3dc7d956be090676d 100644
--- a/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp
+++ b/llvm/lib/Transforms/Utils/LoopRotationUtils.cpp
@@ -708,12 +708,13 @@ bool LoopRotate::rotateLoop(Loop *L, bool SimplifiedLatch) {
       // as U1'' and U1' scopes will not be compatible wrt to the local restrict
 
       // Clone the llvm.experimental.noalias.decl again for the NewHeader.
-      Instruction *NewHeaderInsertionPoint = &(*NewHeader->getFirstNonPHI());
+      BasicBlock::iterator NewHeaderInsertionPoint =
+          NewHeader->getFirstNonPHIIt();
       for (NoAliasScopeDeclInst *NAD : NoAliasDeclInstructions) {
         LLVM_DEBUG(dbgs() << "  Cloning llvm.experimental.noalias.scope.decl:"
                           << *NAD << "\n");
         Instruction *NewNAD = NAD->clone();
-        NewNAD->insertBefore(NewHeaderInsertionPoint);
+        NewNAD->insertBefore(*NewHeader, NewHeaderInsertionPoint);
       }
 
       // Scopes must now be duplicated, once for OrigHeader and once for
diff --git a/llvm/lib/Transforms/Utils/MemoryOpRemark.cpp b/llvm/lib/Transforms/Utils/MemoryOpRemark.cpp
index 5c3776683d5d6fec09a9e9eb5a7df57f34d2d18f..47c6bcbaf26ec2e4ebcf5b893d566a1208216be8 100644
--- a/llvm/lib/Transforms/Utils/MemoryOpRemark.cpp
+++ b/llvm/lib/Transforms/Utils/MemoryOpRemark.cpp
@@ -322,8 +322,9 @@ void MemoryOpRemark::visitVariable(const Value *V,
   // Try to get an llvm.dbg.declare, which has a DILocalVariable giving us the
   // real debug info name and size of the variable.
   SmallVector<DbgDeclareInst *, 1> DbgDeclares;
-  findDbgDeclares(DbgDeclares, const_cast<Value *>(V));
-  for (const DbgVariableIntrinsic *DVI : DbgDeclares) {
+  SmallVector<DPValue *, 1> DPValues;
+  findDbgDeclares(DbgDeclares, const_cast<Value *>(V), &DPValues);
+  auto FindDI = [&](const auto *DVI) {
     if (DILocalVariable *DILV = DVI->getVariable()) {
       std::optional<uint64_t> DISize = getSizeInBytes(DILV->getSizeInBits());
       VariableInfo Var{DILV->getName(), DISize};
@@ -332,7 +333,10 @@ void MemoryOpRemark::visitVariable(const Value *V,
         FoundDI = true;
       }
     }
-  }
+  };
+  for_each(DbgDeclares, FindDI);
+  for_each(DPValues, FindDI);
+
   if (FoundDI) {
     assert(!Result.empty());
     return;
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index fe2aac78e5ab0debb427b4bfbdeec54873e62312..4bc65067473eefded030ab590aea4fc44ea590eb 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -391,8 +391,10 @@ static SmallBitVector isUndefVector(const Value *V,
         if (isa<T>(II->getOperand(1)))
           continue;
         std::optional<unsigned> Idx = getInsertIndex(II);
-        if (!Idx)
-          continue;
+        if (!Idx) {
+          Res.reset();
+          return Res;
+        }
         if (*Idx < UseMask.size() && !UseMask.test(*Idx))
           Res.reset(*Idx);
       }
diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
index ffdeea80fa074b8b14e964c1c52f3b8bea26eb28..02e400d590bed449645580f5ca6c34a649f2f89e 100644
--- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
@@ -44,6 +44,8 @@ extern cl::opt<bool> EnableVPlanNativePath;
 
 bool VPRecipeBase::mayWriteToMemory() const {
   switch (getVPDefID()) {
+  case VPInterleaveSC:
+    return cast<VPInterleaveRecipe>(this)->getNumStoreOperands() > 0;
   case VPWidenMemoryInstructionSC: {
     return cast<VPWidenMemoryInstructionRecipe>(this)->isStore();
   }
@@ -118,7 +120,6 @@ bool VPRecipeBase::mayHaveSideEffects() const {
   case VPInstructionSC:
     switch (cast<VPInstruction>(this)->getOpcode()) {
     case Instruction::ICmp:
-    case Instruction::Select:
     case VPInstruction::Not:
     case VPInstruction::CalculateTripCountMinusVF:
     case VPInstruction::CanonicalIVIncrementForPart:
@@ -147,6 +148,8 @@ bool VPRecipeBase::mayHaveSideEffects() const {
            "underlying instruction has side-effects");
     return false;
   }
+  case VPInterleaveSC:
+    return mayWriteToMemory();
   case VPWidenMemoryInstructionSC:
     assert(cast<VPWidenMemoryInstructionRecipe>(this)
                    ->getIngredient()
diff --git a/llvm/test/Analysis/CostModel/ARM/intrinsic-cost-kinds.ll b/llvm/test/Analysis/CostModel/ARM/intrinsic-cost-kinds.ll
index 1565a138b93d3cc5f44f576852aee14108127f13..d58f3348e41fedc4425a901a1e2a15945b8ed3bf 100644
--- a/llvm/test/Analysis/CostModel/ARM/intrinsic-cost-kinds.ll
+++ b/llvm/test/Analysis/CostModel/ARM/intrinsic-cost-kinds.ll
@@ -42,6 +42,10 @@ declare float @llvm.vector.reduce.fmax.v16f32(<16 x float>)
 
 declare void @llvm.memcpy.p0.p0.i32(ptr, ptr, i32, i1)
 
+declare i32 @llvm.ssa.copy.i32(i32)
+declare float @llvm.ssa.copy.f32(float)
+declare ptr @llvm.ssa.copy.p0(ptr)
+
 define void @smax(i32 %a, i32 %b, <16 x i32> %va, <16 x i32> %vb) {
 ; THRU-LABEL: 'smax'
 ; THRU-NEXT:  Cost Model: Found an estimated cost of 2 for instruction: %s = call i32 @llvm.smax.i32(i32 %a, i32 %b)
@@ -333,3 +337,35 @@ define void @memcpy(ptr %a, ptr %b, i32 %c) {
   call void @llvm.memcpy.p0.p0.i32(ptr align 1 %a, ptr align 1 %b, i32 32, i1 false)
   ret void
 }
+
+define void @ssa_copy() {
+  ; CHECK: %{{.*}} = llvm.intr.ssa.copy %{{.*}} : f32
+; THRU-LABEL: 'ssa_copy'
+; THRU-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i = call i32 @llvm.ssa.copy.i32(i32 undef)
+; THRU-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %f = call float @llvm.ssa.copy.f32(float undef)
+; THRU-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %p = call ptr @llvm.ssa.copy.p0(ptr undef)
+; THRU-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: ret void
+;
+; LATE-LABEL: 'ssa_copy'
+; LATE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i = call i32 @llvm.ssa.copy.i32(i32 undef)
+; LATE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %f = call float @llvm.ssa.copy.f32(float undef)
+; LATE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %p = call ptr @llvm.ssa.copy.p0(ptr undef)
+; LATE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; SIZE-LABEL: 'ssa_copy'
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i = call i32 @llvm.ssa.copy.i32(i32 undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %f = call float @llvm.ssa.copy.f32(float undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %p = call ptr @llvm.ssa.copy.p0(ptr undef)
+; SIZE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+; SIZE_LATE-LABEL: 'ssa_copy'
+; SIZE_LATE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %i = call i32 @llvm.ssa.copy.i32(i32 undef)
+; SIZE_LATE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %f = call float @llvm.ssa.copy.f32(float undef)
+; SIZE_LATE-NEXT:  Cost Model: Found an estimated cost of 0 for instruction: %p = call ptr @llvm.ssa.copy.p0(ptr undef)
+; SIZE_LATE-NEXT:  Cost Model: Found an estimated cost of 1 for instruction: ret void
+;
+  %i = call i32 @llvm.ssa.copy.i32(i32 undef)
+  %f = call float @llvm.ssa.copy.f32(float undef)
+  %p = call ptr @llvm.ssa.copy.p0(ptr undef)
+  ret void
+}
diff --git a/llvm/test/Bitcode/attributes.ll b/llvm/test/Bitcode/attributes.ll
index c4bced5c003bef74ea053ff36b950e44ca8d29d3..6921f11a352ddf9522d5af0bcbe355578a37ecfe 100644
--- a/llvm/test/Bitcode/attributes.ll
+++ b/llvm/test/Bitcode/attributes.ll
@@ -521,6 +521,11 @@ define void @f90(ptr writable %p) {
   ret void
 }
 
+; CHECK: define void @f91(ptr dead_on_unwind %p)
+define void @f91(ptr dead_on_unwind %p) {
+  ret void
+}
+
 ; CHECK: attributes #0 = { noreturn }
 ; CHECK: attributes #1 = { nounwind }
 ; CHECK: attributes #2 = { memory(none) }
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator-gep.ll b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator-gep.ll
index 9307d55f7960c1628305d0fde511767c4639b076..fcc399d41fad89588f5fc4a6b65c1c3b14bb829f 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator-gep.ll
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator-gep.ll
@@ -52,3 +52,46 @@ define i32 @cse_gep(ptr %ptr, i32 %idx) {
   %res = add i32 %v1, %v2
   ret i32 %res
 }
+
+; OSS Fuzz https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=65052
+define void @ossfuzz65052() {
+  ; O0-LABEL: name: ossfuzz65052
+  ; O0: bb.1 (%ir-block.0):
+  ; O0-NEXT:   successors: %bb.2(0x80000000)
+  ; O0-NEXT: {{  $}}
+  ; O0-NEXT:   [[DEF:%[0-9]+]]:_(p0) = G_IMPLICIT_DEF
+  ; O0-NEXT:   [[C:%[0-9]+]]:_(s128) = G_CONSTANT i128 -170141183460469231731687303715884105728
+  ; O0-NEXT:   [[TRUNC:%[0-9]+]]:_(s64) = G_TRUNC [[C]](s128)
+  ; O0-NEXT:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+  ; O0-NEXT:   [[MUL:%[0-9]+]]:_(s64) = G_MUL [[TRUNC]], [[C1]]
+  ; O0-NEXT:   [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[DEF]], [[MUL]](s64)
+  ; O0-NEXT:   [[COPY:%[0-9]+]]:_(p0) = COPY [[PTR_ADD]](p0)
+  ; O0-NEXT:   G_BR %bb.2
+  ; O0-NEXT: {{  $}}
+  ; O0-NEXT: bb.2.BB:
+  ; O0-NEXT:   successors: %bb.2(0x80000000)
+  ; O0-NEXT: {{  $}}
+  ; O0-NEXT:   G_BR %bb.2
+  ;
+  ; O3-LABEL: name: ossfuzz65052
+  ; O3: bb.1 (%ir-block.0):
+  ; O3-NEXT:   successors: %bb.2(0x80000000)
+  ; O3-NEXT: {{  $}}
+  ; O3-NEXT:   [[DEF:%[0-9]+]]:_(p0) = G_IMPLICIT_DEF
+  ; O3-NEXT:   [[C:%[0-9]+]]:_(s128) = G_CONSTANT i128 -170141183460469231731687303715884105728
+  ; O3-NEXT:   [[TRUNC:%[0-9]+]]:_(s64) = G_TRUNC [[C]](s128)
+  ; O3-NEXT:   [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
+  ; O3-NEXT:   [[MUL:%[0-9]+]]:_(s64) = G_MUL [[TRUNC]], [[C1]]
+  ; O3-NEXT:   [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[DEF]], [[MUL]](s64)
+  ; O3-NEXT:   [[COPY:%[0-9]+]]:_(p0) = COPY [[PTR_ADD]](p0)
+  ; O3-NEXT: {{  $}}
+  ; O3-NEXT: bb.2.BB:
+  ; O3-NEXT:   successors: %bb.2(0x80000000)
+  ; O3-NEXT: {{  $}}
+  ; O3-NEXT:   G_BR %bb.2
+  %G15 = getelementptr i128, ptr poison, i128 -170141183460469231731687303715884105728
+  br label %BB
+
+BB:                                               ; preds = %BB, %0
+  br label %BB
+}
diff --git a/llvm/test/CodeGen/AArch64/aarch64-interleaved-ld-combine.ll b/llvm/test/CodeGen/AArch64/aarch64-interleaved-ld-combine.ll
index a5d94c13ef01016c6ad6e2eeaf89b2fbe7f70f34..35a140cb4bd400b34471a7b3cd6d8ec3b0b4663f 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-interleaved-ld-combine.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-interleaved-ld-combine.ll
@@ -1,5 +1,6 @@
 ; RUN: llc < %s | FileCheck --check-prefix AS %s
 ; RUN: opt -S -interleaved-load-combine < %s | FileCheck %s
+; RUN: opt -S -passes=interleaved-load-combine < %s | FileCheck %s
 
 ; ModuleID = 'aarch64_interleaved-ld-combine.bc'
 target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
diff --git a/llvm/test/CodeGen/AArch64/machine-cp-sub-reg.mir b/llvm/test/CodeGen/AArch64/machine-cp-sub-reg.mir
new file mode 100644
index 0000000000000000000000000000000000000000..23cf1dcda839e3072af578a617e5823cb3fbff2d
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/machine-cp-sub-reg.mir
@@ -0,0 +1,33 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 4
+# RUN: llc -o - %s --run-pass=machine-cp -mcp-use-is-copy-instr -mtriple=arm64-apple-macos --verify-machineinstrs | FileCheck %s
+
+---
+name: test
+tracksRegLiveness: true
+body:             |
+  ; CHECK-LABEL: name: test
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT:   liveins: $w0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   $x8 = ORRXrs $xzr, $x0, 0, implicit $w0
+  ; CHECK-NEXT:   $w8 = ORRWrs $wzr, $w0, 0, implicit-def $x8
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   liveins: $x8
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   $x0 = ADDXri $x8, 1, 0
+  ; CHECK-NEXT:   RET undef $lr, implicit $x0
+  bb.0:
+    successors: %bb.1(0x80000000)
+    liveins: $w0
+
+    $x8 = ORRXrs $xzr, $x0, 0, implicit $w0
+    $w8 = ORRWrs $wzr, $w0, 0, implicit-def $x8
+
+  bb.1:
+    liveins: $x8
+    $x0 = ADDXri $x8, 1, 0
+
+    RET undef $lr, implicit $x0
+...
diff --git a/llvm/test/CodeGen/AArch64/new-load-requires-renaming-in-mssa.ll b/llvm/test/CodeGen/AArch64/new-load-requires-renaming-in-mssa.ll
index 6ba29a664be12f4ae5ce970618fe75f14449d7f5..d45c06a6811ce6767e591d198193f5f7aba9a7f7 100644
--- a/llvm/test/CodeGen/AArch64/new-load-requires-renaming-in-mssa.ll
+++ b/llvm/test/CodeGen/AArch64/new-load-requires-renaming-in-mssa.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
 ; RUN: opt -interleaved-load-combine -S -verify-memoryssa %s | FileCheck %s
+; RUN: opt -passes=interleaved-load-combine -S -verify-memoryssa %s | FileCheck %s
 
 target triple = "arm64-apple-darwin"
 
diff --git a/llvm/test/CodeGen/AArch64/sink-and-fold-clear-kill-flags.mir b/llvm/test/CodeGen/AArch64/sink-and-fold-clear-kill-flags.mir
new file mode 100644
index 0000000000000000000000000000000000000000..1303776230f1a397e9b91310f31e2fabf442daa4
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sink-and-fold-clear-kill-flags.mir
@@ -0,0 +1,193 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 4
+# RUN: llc --run-pass=machine-sink %s -o - | FileCheck %s
+
+# Test that the "killed" flags are cleared in the ORRWrs and SUBSWrr  instructions
+# in 'f and @g, respectively
+
+--- |
+  source_filename = "crash.ll"
+  target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
+  target triple = "aarch64-linux"
+
+  define i32 @f(ptr %image, i32 %i) {
+  entry:
+    %add = add i32 %i, 1
+    %idx = zext i32 %add to i64
+    br label %A
+
+  A:                                                ; preds = %B, %A, %entry
+    %sunkaddr = getelementptr i8, ptr %image, i64 %idx
+    %0 = load i8, ptr %sunkaddr, align 1
+    %cmp153 = icmp eq i8 %0, 0
+    br i1 %cmp153, label %B, label %A
+
+  B:                                                ; preds = %A
+    store i32 0, ptr %image, align 1
+    br label %A
+  }
+
+  define i32 @g(i32 %i, i32 %j) {
+  entry:
+    %add = add i32 %i, %j
+    %neg = sub i32 0, %i
+    br label %A
+
+  A:                                                ; preds = %B, %A, %entry
+    %0 = call i8 @h(i32 %add)
+    %c = icmp eq i8 %0, 0
+    br i1 %c, label %B, label %A
+
+  B:                                                ; preds = %A
+    %1 = call i8 @h(i32 %neg)
+    br label %A
+  }
+
+  declare i8 @h(i32)
+
+...
+---
+name:            f
+alignment:       4
+tracksRegLiveness: true
+registers:
+  - { id: 0, class: gpr64, preferred-register: '' }
+  - { id: 1, class: gpr64common, preferred-register: '' }
+  - { id: 2, class: gpr32common, preferred-register: '' }
+  - { id: 3, class: gpr32common, preferred-register: '' }
+  - { id: 4, class: gpr32, preferred-register: '' }
+  - { id: 5, class: gpr32, preferred-register: '' }
+  - { id: 6, class: gpr32, preferred-register: '' }
+liveins:
+  - { reg: '$x0', virtual-reg: '%1' }
+  - { reg: '$w1', virtual-reg: '%2' }
+body:             |
+  ; CHECK-LABEL: name: f
+  ; CHECK: bb.0.entry:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT:   liveins: $x0, $w1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gpr32common = COPY $w1
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gpr64common = COPY $x0
+  ; CHECK-NEXT:   [[ADDWri:%[0-9]+]]:gpr32common = ADDWri [[COPY]], 1, 0
+  ; CHECK-NEXT:   [[ORRWrs:%[0-9]+]]:gpr32 = ORRWrs $wzr, [[ADDWri]], 0
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1.A:
+  ; CHECK-NEXT:   successors: %bb.2(0x30000000), %bb.1(0x50000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[LDRBBroW:%[0-9]+]]:gpr32 = LDRBBroW [[COPY1]], [[ADDWri]], 0, 0 :: (load (s8) from %ir.sunkaddr)
+  ; CHECK-NEXT:   CBNZW killed [[LDRBBroW]], %bb.1
+  ; CHECK-NEXT:   B %bb.2
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2.B:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:gpr32 = COPY $wzr
+  ; CHECK-NEXT:   STRWui [[COPY2]], [[COPY1]], 0 :: (store (s32) into %ir.image, align 1)
+  ; CHECK-NEXT:   B %bb.1
+  bb.0.entry:
+    successors: %bb.1(0x80000000)
+    liveins: $x0, $w1
+
+    %2:gpr32common = COPY $w1
+    %1:gpr64common = COPY $x0
+    %3:gpr32common = ADDWri %2, 1, 0
+    %4:gpr32 = ORRWrs $wzr, killed %3, 0
+    %0:gpr64 = SUBREG_TO_REG 0, killed %4, %subreg.sub_32
+
+  bb.1.A:
+    successors: %bb.2(0x30000000), %bb.1(0x50000000)
+
+    %5:gpr32 = LDRBBroX %1, %0, 0, 0 :: (load (s8) from %ir.sunkaddr)
+    CBNZW killed %5, %bb.1
+    B %bb.2
+
+  bb.2.B:
+    successors: %bb.1(0x80000000)
+
+    %6:gpr32 = COPY $wzr
+    STRWui %6, %1, 0 :: (store (s32) into %ir.image, align 1)
+    B %bb.1
+...
+---
+name:            g
+alignment:       4
+registers:
+  - { id: 0, class: gpr32all, preferred-register: '' }
+  - { id: 1, class: gpr32all, preferred-register: '' }
+  - { id: 2, class: gpr32, preferred-register: '' }
+  - { id: 3, class: gpr32, preferred-register: '' }
+  - { id: 4, class: gpr32, preferred-register: '' }
+  - { id: 5, class: gpr32, preferred-register: '' }
+  - { id: 6, class: gpr32, preferred-register: '' }
+  - { id: 7, class: gpr32, preferred-register: '' }
+  - { id: 8, class: gpr32common, preferred-register: '' }
+  - { id: 9, class: gpr32all, preferred-register: '' }
+liveins:
+  - { reg: '$w0', virtual-reg: '%2' }
+  - { reg: '$w1', virtual-reg: '%3' }
+body:             |
+  ; CHECK-LABEL: name: g
+  ; CHECK: bb.0.entry:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT:   liveins: $w0, $w1
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gpr32 = COPY $w1
+  ; CHECK-NEXT:   [[COPY1:%[0-9]+]]:gpr32 = COPY $w0
+  ; CHECK-NEXT:   [[COPY2:%[0-9]+]]:gpr32 = COPY $wzr
+  ; CHECK-NEXT:   [[SUBSWrr:%[0-9]+]]:gpr32 = SUBSWrr [[COPY2]], [[COPY1]], implicit-def dead $nzcv
+  ; CHECK-NEXT:   [[COPY3:%[0-9]+]]:gpr32all = COPY [[SUBSWrr]]
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1.A:
+  ; CHECK-NEXT:   successors: %bb.2(0x30000000), %bb.1(0x50000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def dead $sp, implicit $sp
+  ; CHECK-NEXT:   $w0 = ADDWrr [[COPY1]], [[COPY]]
+  ; CHECK-NEXT:   BL @h, csr_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit $w0, implicit-def $sp, implicit-def $w0
+  ; CHECK-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def dead $sp, implicit $sp
+  ; CHECK-NEXT:   [[COPY4:%[0-9]+]]:gpr32 = COPY $w0
+  ; CHECK-NEXT:   $wzr = ANDSWri [[COPY4]], 7, implicit-def $nzcv
+  ; CHECK-NEXT:   Bcc 1, %bb.1, implicit $nzcv
+  ; CHECK-NEXT:   B %bb.2
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2.B:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   ADJCALLSTACKDOWN 0, 0, implicit-def dead $sp, implicit $sp
+  ; CHECK-NEXT:   $w0 = COPY [[COPY3]]
+  ; CHECK-NEXT:   BL @h, csr_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit $w0, implicit-def $sp, implicit-def $w0
+  ; CHECK-NEXT:   ADJCALLSTACKUP 0, 0, implicit-def dead $sp, implicit $sp
+  ; CHECK-NEXT:   B %bb.1
+  bb.0.entry:
+    successors: %bb.1(0x80000000)
+    liveins: $w0, $w1
+
+    %3:gpr32 = COPY $w1
+    %2:gpr32 = COPY $w0
+    %4:gpr32 = ADDWrr %2, killed %3
+    %0:gpr32all = COPY %4
+    %5:gpr32 = COPY $wzr
+    %6:gpr32 = SUBSWrr %5, killed %2, implicit-def dead $nzcv
+    %1:gpr32all = COPY %6
+
+  bb.1.A:
+    successors: %bb.2(0x30000000), %bb.1(0x50000000)
+
+    ADJCALLSTACKDOWN 0, 0, implicit-def dead $sp, implicit $sp
+    $w0 = COPY %0
+    BL @h, csr_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit $w0, implicit-def $sp, implicit-def $w0
+    ADJCALLSTACKUP 0, 0, implicit-def dead $sp, implicit $sp
+    %7:gpr32 = COPY $w0
+    $wzr = ANDSWri %7, 7, implicit-def $nzcv
+    Bcc 1, %bb.1, implicit $nzcv
+    B %bb.2
+
+  bb.2.B:
+    successors: %bb.1(0x80000000)
+
+    ADJCALLSTACKDOWN 0, 0, implicit-def dead $sp, implicit $sp
+    $w0 = COPY %1
+    BL @h, csr_aarch64_aapcs, implicit-def dead $lr, implicit $sp, implicit $w0, implicit-def $sp, implicit-def $w0
+    ADJCALLSTACKUP 0, 0, implicit-def dead $sp, implicit $sp
+    B %bb.1
+
+...
diff --git a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-fp-reduce.ll b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-fp-reduce.ll
new file mode 100644
index 0000000000000000000000000000000000000000..7957366fce3b05a578742f2d346d0115e389c883
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-fp-reduce.ll
@@ -0,0 +1,189 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve2p1 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sme2p1 < %s | FileCheck %s
+
+;
+; FMAXNMQV
+;
+
+define <8 x half> @fmaxnmqv_f16(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a) {
+; CHECK-LABEL: fmaxnmqv_f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmaxnmqv v0.8h, p0, z0.h
+; CHECK-NEXT:    ret
+  %res = call <8 x half> @llvm.aarch64.sve.fmaxnmqv.v8f16.nxv8f16(<vscale x 8 x i1> %pg,
+                                                                  <vscale x 8 x half> %a)
+  ret <8 x half> %res
+}
+
+define <4 x float> @fmaxnmqv_f32(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a) {
+; CHECK-LABEL: fmaxnmqv_f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmaxnmqv v0.4s, p0, z0.s
+; CHECK-NEXT:    ret
+  %res = call <4 x float> @llvm.aarch64.sve.fmaxnmqv.v4f32.nxv4f32(<vscale x 4 x i1> %pg,
+                                                                   <vscale x 4 x float> %a)
+  ret <4 x float> %res
+}
+
+define <2 x double> @fmaxnmqv_f64(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a) {
+; CHECK-LABEL: fmaxnmqv_f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmaxnmqv v0.2d, p0, z0.d
+; CHECK-NEXT:    ret
+  %res = call <2 x double> @llvm.aarch64.sve.fmaxnmqv.v2f64.nxv2f64(<vscale x 2 x i1> %pg,
+                                                                    <vscale x 2 x double> %a)
+  ret <2 x double> %res
+}
+
+;
+; FMINNMQV
+;
+
+define <8 x half> @fminnmqv_f16(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a) {
+; CHECK-LABEL: fminnmqv_f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fminnmqv v0.8h, p0, z0.h
+; CHECK-NEXT:    ret
+  %res = call <8 x half> @llvm.aarch64.sve.fminnmqv.v8f16.nxv8f16(<vscale x 8 x i1> %pg,
+                                                                  <vscale x 8 x half> %a)
+  ret <8 x half> %res
+}
+
+define <4 x float> @fminnmqv_f32(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a) {
+; CHECK-LABEL: fminnmqv_f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fminnmqv v0.4s, p0, z0.s
+; CHECK-NEXT:    ret
+  %res = call <4 x float> @llvm.aarch64.sve.fminnmqv.v4f32.nxv4f32(<vscale x 4 x i1> %pg,
+                                                                   <vscale x 4 x float> %a)
+  ret <4 x float> %res
+}
+
+define <2 x double> @fminnmqv_f64(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a) {
+; CHECK-LABEL: fminnmqv_f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fminnmqv v0.2d, p0, z0.d
+; CHECK-NEXT:    ret
+  %res = call <2 x double> @llvm.aarch64.sve.fminnmqv.v2f64.nxv2f64(<vscale x 2 x i1> %pg,
+                                                                    <vscale x 2 x double> %a)
+  ret <2 x double> %res
+}
+
+;
+; FADDQV
+;
+
+define <8 x half> @faddqv_f16(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a) {
+; CHECK-LABEL: faddqv_f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    faddqv v0.8h, p0, z0.h
+; CHECK-NEXT:    ret
+  %res = call <8 x half> @llvm.aarch64.sve.addqv.v8f16.nxv8f16(<vscale x 8 x i1> %pg,
+                                                               <vscale x 8 x half> %a)
+  ret <8 x half> %res
+}
+
+define <4 x float> @faddqv_f32(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a) {
+; CHECK-LABEL: faddqv_f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    faddqv v0.4s, p0, z0.s
+; CHECK-NEXT:    ret
+  %res = call <4 x float> @llvm.aarch64.sve.addqv.v4f32.nxv4f32(<vscale x 4 x i1> %pg,
+                                                                <vscale x 4 x float> %a)
+  ret <4 x float> %res
+}
+
+define <2 x double> @faddqv_f64(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a) {
+; CHECK-LABEL: faddqv_f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    faddqv v0.2d, p0, z0.d
+; CHECK-NEXT:    ret
+  %res = call <2 x double> @llvm.aarch64.sve.addqv.v2f64.nxv2f64(<vscale x 2 x i1> %pg,
+                                                                 <vscale x 2 x double> %a)
+  ret <2 x double> %res
+}
+
+;
+; FMINQV
+;
+
+define <8 x half> @fminqv_f16(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a) {
+; CHECK-LABEL: fminqv_f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fminqv v0.8h, p0, z0.h
+; CHECK-NEXT:    ret
+  %res = call <8 x half> @llvm.aarch64.sve.fminqv.v8f16.nxv8f16(<vscale x 8 x i1> %pg,
+                                                                <vscale x 8 x half> %a)
+  ret <8 x half> %res
+}
+
+define <4 x float> @fminqv_f32(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a) {
+; CHECK-LABEL: fminqv_f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fminqv v0.4s, p0, z0.s
+; CHECK-NEXT:    ret
+  %res = call <4 x float> @llvm.aarch64.sve.fminqv.v4f32.nxv4f32(<vscale x 4 x i1> %pg,
+                                                                 <vscale x 4 x float> %a)
+  ret <4 x float> %res
+}
+
+define <2 x double> @fminqv_f64(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a) {
+; CHECK-LABEL: fminqv_f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fminqv v0.2d, p0, z0.d
+; CHECK-NEXT:    ret
+  %res = call <2 x double> @llvm.aarch64.sve.fminqv.v2f64.nxv2f64(<vscale x 2 x i1> %pg,
+                                                                  <vscale x 2 x double> %a)
+  ret <2 x double> %res
+}
+
+;
+; FMAXQV
+;
+
+define <8 x half> @fmaxqv_f16(<vscale x 8 x i1> %pg, <vscale x 8 x half> %a) {
+; CHECK-LABEL: fmaxqv_f16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmaxqv v0.8h, p0, z0.h
+; CHECK-NEXT:    ret
+  %res = call <8 x half> @llvm.aarch64.sve.fmaxqv.v8f16.nxv8f16(<vscale x 8 x i1> %pg,
+                                                                <vscale x 8 x half> %a)
+  ret <8 x half> %res
+}
+
+define <4 x float> @fmaxqv_f32(<vscale x 4 x i1> %pg, <vscale x 4 x float> %a) {
+; CHECK-LABEL: fmaxqv_f32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmaxqv v0.4s, p0, z0.s
+; CHECK-NEXT:    ret
+  %res = call <4 x float> @llvm.aarch64.sve.fmaxqv.v4f32.nxv4f32(<vscale x 4 x i1> %pg,
+                                                                 <vscale x 4 x float> %a)
+  ret <4 x float> %res
+}
+
+define <2 x double> @fmaxqv_f64(<vscale x 2 x i1> %pg, <vscale x 2 x double> %a) {
+; CHECK-LABEL: fmaxqv_f64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    fmaxqv v0.2d, p0, z0.d
+; CHECK-NEXT:    ret
+  %res = call <2 x double> @llvm.aarch64.sve.fmaxqv.v2f64.nxv2f64(<vscale x 2 x i1> %pg,
+                                                                  <vscale x 2 x double> %a)
+  ret <2 x double> %res
+}
+
+declare <8 x half> @llvm.aarch64.sve.fmaxnmqv.v8f16.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>)
+declare <4 x float> @llvm.aarch64.sve.fmaxnmqv.v4f32.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>)
+declare <2 x double> @llvm.aarch64.sve.fmaxnmqv.v2f64.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>)
+declare <8 x half> @llvm.aarch64.sve.fminnmqv.v8f16.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>)
+declare <4 x float> @llvm.aarch64.sve.fminnmqv.v4f32.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>)
+declare <2 x double> @llvm.aarch64.sve.fminnmqv.v2f64.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>)
+declare <8 x half> @llvm.aarch64.sve.addqv.v8f16.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>)
+declare <4 x float> @llvm.aarch64.sve.addqv.v4f32.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>)
+declare <2 x double> @llvm.aarch64.sve.addqv.v2f64.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>)
+declare <8 x half> @llvm.aarch64.sve.fminqv.v8f16.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>)
+declare <4 x float> @llvm.aarch64.sve.fminqv.v4f32.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>)
+declare <2 x double> @llvm.aarch64.sve.fminqv.v2f64.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>)
+declare <8 x half> @llvm.aarch64.sve.fmaxqv.v8f16.nxv8f16(<vscale x 8 x i1>, <vscale x 8 x half>)
+declare <4 x float> @llvm.aarch64.sve.fmaxqv.v4f32.nxv4f32(<vscale x 4 x i1>, <vscale x 4 x float>)
+declare <2 x double> @llvm.aarch64.sve.fmaxqv.v2f64.nxv2f64(<vscale x 2 x i1>, <vscale x 2 x double>)
diff --git a/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-int-reduce.ll b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-int-reduce.ll
new file mode 100644
index 0000000000000000000000000000000000000000..a730ba9c93209307aa1183186d16f7452a5f7410
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve2p1-intrinsics-int-reduce.ll
@@ -0,0 +1,356 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sve2p1 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64--linux-gnu -mattr=+sme2p1 < %s | FileCheck %s
+
+;
+; ORQV
+;
+
+define <16 x i8> @orqv_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a) {
+; CHECK-LABEL: orqv_i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    orqv v0.16b, p0, z0.b
+; CHECK-NEXT:    ret
+  %res = call <16 x i8> @llvm.aarch64.sve.orqv.v16i8.nxv16i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a);
+  ret <16 x i8> %res
+}
+
+define <8 x i16> @orqv_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a) {
+; CHECK-LABEL: orqv_i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    orqv v0.8h, p0, z0.h
+; CHECK-NEXT:    ret
+  %res = call <8 x i16> @llvm.aarch64.sve.orqv.v8i16.nxv8i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a);
+  ret <8 x i16> %res
+}
+
+define <4 x i32> @orqv_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a) {
+; CHECK-LABEL: orqv_i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    orqv v0.4s, p0, z0.s
+; CHECK-NEXT:    ret
+  %res = call <4 x i32> @llvm.aarch64.sve.orqv.v4i32.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a);
+  ret <4 x i32> %res
+}
+
+define <2 x i64> @orqv_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a) {
+; CHECK-LABEL: orqv_i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    orqv v0.2d, p0, z0.d
+; CHECK-NEXT:    ret
+  %res = call <2 x i64> @llvm.aarch64.sve.orqv.v2i64.nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a);
+  ret <2 x i64> %res
+}
+
+;
+; EORQV
+;
+
+define <16 x i8> @eorqv_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a) {
+; CHECK-LABEL: eorqv_i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    eorqv v0.16b, p0, z0.b
+; CHECK-NEXT:    ret
+  %res = call <16 x i8> @llvm.aarch64.sve.eorqv.v16i8.nxv16i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a);
+  ret <16 x i8> %res
+}
+
+define <8 x i16> @eorqv_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a) {
+; CHECK-LABEL: eorqv_i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    eorqv v0.8h, p0, z0.h
+; CHECK-NEXT:    ret
+  %res = call <8 x i16> @llvm.aarch64.sve.eorqv.v8i16.nxv8i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a);
+  ret <8 x i16> %res
+}
+
+define <4 x i32> @eorqv_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a) {
+; CHECK-LABEL: eorqv_i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    eorqv v0.4s, p0, z0.s
+; CHECK-NEXT:    ret
+  %res = call <4 x i32> @llvm.aarch64.sve.eorqv.v4i32.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a);
+  ret <4 x i32> %res
+}
+
+define <2 x i64> @eorqv_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a) {
+; CHECK-LABEL: eorqv_i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    eorqv v0.2d, p0, z0.d
+; CHECK-NEXT:    ret
+  %res = call <2 x i64> @llvm.aarch64.sve.eorqv.v2i64.nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a);
+  ret <2 x i64> %res
+}
+
+;
+; ANDQV
+;
+
+define <16 x i8> @andqv_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a) {
+; CHECK-LABEL: andqv_i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    andqv v0.16b, p0, z0.b
+; CHECK-NEXT:    ret
+  %res = call <16 x i8> @llvm.aarch64.sve.andqv.v16i8.nxv16i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a);
+  ret <16 x i8> %res
+}
+
+define <8 x i16> @andqv_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a) {
+; CHECK-LABEL: andqv_i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    andqv v0.8h, p0, z0.h
+; CHECK-NEXT:    ret
+  %res = call <8 x i16> @llvm.aarch64.sve.andqv.v8i16.nxv8i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a);
+  ret <8 x i16> %res
+}
+
+define <4 x i32> @andqv_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a) {
+; CHECK-LABEL: andqv_i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    andqv v0.4s, p0, z0.s
+; CHECK-NEXT:    ret
+  %res = call <4 x i32> @llvm.aarch64.sve.andqv.v4i32.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a);
+  ret <4 x i32> %res
+}
+
+define <2 x i64> @andqv_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a) {
+; CHECK-LABEL: andqv_i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    andqv v0.2d, p0, z0.d
+; CHECK-NEXT:    ret
+  %res = call <2 x i64> @llvm.aarch64.sve.andqv.v2i64.nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a);
+  ret <2 x i64> %res
+}
+
+;
+; ADDQV
+;
+
+define <16 x i8> @addqv_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a) {
+; CHECK-LABEL: addqv_i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    addqv v0.16b, p0, z0.b
+; CHECK-NEXT:    ret
+  %res = call <16 x i8> @llvm.aarch64.sve.addqv.v16i8.nxv16i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a);
+  ret <16 x i8> %res
+}
+
+define <8 x i16> @addqv_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a) {
+; CHECK-LABEL: addqv_i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    addqv v0.8h, p0, z0.h
+; CHECK-NEXT:    ret
+  %res = call <8 x i16> @llvm.aarch64.sve.addqv.v8i16.nxv8i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a);
+  ret <8 x i16> %res
+}
+
+define <4 x i32> @addqv_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a) {
+; CHECK-LABEL: addqv_i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    addqv v0.4s, p0, z0.s
+; CHECK-NEXT:    ret
+  %res = call <4 x i32> @llvm.aarch64.sve.addqv.v4i32.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a);
+  ret <4 x i32> %res
+}
+
+define <2 x i64> @addqv_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a) {
+; CHECK-LABEL: addqv_i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    addqv v0.2d, p0, z0.d
+; CHECK-NEXT:    ret
+  %res = call <2 x i64> @llvm.aarch64.sve.addqv.v2i64.nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a);
+  ret <2 x i64> %res
+}
+
+;
+; SMAXQV
+;
+
+define <16 x i8> @smaxqv_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a) {
+; CHECK-LABEL: smaxqv_i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    smaxqv v0.16b, p0, z0.b
+; CHECK-NEXT:    ret
+  %res = call <16 x i8> @llvm.aarch64.sve.smaxqv.v16i8.nxv16i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a);
+  ret <16 x i8> %res
+}
+
+define <8 x i16> @smaxqv_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a) {
+; CHECK-LABEL: smaxqv_i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    smaxqv v0.8h, p0, z0.h
+; CHECK-NEXT:    ret
+  %res = call <8 x i16> @llvm.aarch64.sve.smaxqv.v8i16.nxv8i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a);
+  ret <8 x i16> %res
+}
+
+define <4 x i32> @smaxqv_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a) {
+; CHECK-LABEL: smaxqv_i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    smaxqv v0.4s, p0, z0.s
+; CHECK-NEXT:    ret
+  %res = call <4 x i32> @llvm.aarch64.sve.smaxqv.v4i32.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a);
+  ret <4 x i32> %res
+}
+
+define <2 x i64> @smaxqv_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a) {
+; CHECK-LABEL: smaxqv_i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    smaxqv v0.2d, p0, z0.d
+; CHECK-NEXT:    ret
+  %res = call <2 x i64> @llvm.aarch64.sve.smaxqv.v2i64.nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a);
+  ret <2 x i64> %res
+}
+
+;
+; UMAXQV
+;
+
+define <16 x i8> @umaxqv_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a) {
+; CHECK-LABEL: umaxqv_i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    umaxqv v0.16b, p0, z0.b
+; CHECK-NEXT:    ret
+  %res = call <16 x i8> @llvm.aarch64.sve.umaxqv.v16i8.nxv16i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a);
+  ret <16 x i8> %res
+}
+
+define <8 x i16> @umaxqv_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a) {
+; CHECK-LABEL: umaxqv_i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    umaxqv v0.8h, p0, z0.h
+; CHECK-NEXT:    ret
+  %res = call <8 x i16> @llvm.aarch64.sve.umaxqv.v8i16.nxv8i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a);
+  ret <8 x i16> %res
+}
+
+define <4 x i32> @umaxqv_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a) {
+; CHECK-LABEL: umaxqv_i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    umaxqv v0.4s, p0, z0.s
+; CHECK-NEXT:    ret
+  %res = call <4 x i32> @llvm.aarch64.sve.umaxqv.v4i32.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a);
+  ret <4 x i32> %res
+}
+
+define <2 x i64> @umaxqv_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a) {
+; CHECK-LABEL: umaxqv_i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    umaxqv v0.2d, p0, z0.d
+; CHECK-NEXT:    ret
+  %res = call <2 x i64> @llvm.aarch64.sve.umaxqv.v2i64.nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a);
+  ret <2 x i64> %res
+}
+
+;
+; SMINQV
+;
+
+define <16 x i8> @sminqv_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a) {
+; CHECK-LABEL: sminqv_i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sminqv v0.16b, p0, z0.b
+; CHECK-NEXT:    ret
+  %res = call <16 x i8> @llvm.aarch64.sve.sminqv.v16i8.nxv16i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a);
+  ret <16 x i8> %res
+}
+
+define <8 x i16> @sminqv_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a) {
+; CHECK-LABEL: sminqv_i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sminqv v0.8h, p0, z0.h
+; CHECK-NEXT:    ret
+  %res = call <8 x i16> @llvm.aarch64.sve.sminqv.v8i16.nxv8i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a);
+  ret <8 x i16> %res
+}
+
+define <4 x i32> @sminqv_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a) {
+; CHECK-LABEL: sminqv_i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sminqv v0.4s, p0, z0.s
+; CHECK-NEXT:    ret
+  %res = call <4 x i32> @llvm.aarch64.sve.sminqv.v4i32.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a);
+  ret <4 x i32> %res
+}
+
+define <2 x i64> @sminqv_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a) {
+; CHECK-LABEL: sminqv_i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    sminqv v0.2d, p0, z0.d
+; CHECK-NEXT:    ret
+  %res = call <2 x i64> @llvm.aarch64.sve.sminqv.v2i64.nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a);
+  ret <2 x i64> %res
+}
+
+;
+; UMINQV
+;
+
+define <16 x i8> @uminqv_i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a) {
+; CHECK-LABEL: uminqv_i8:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    uminqv v0.16b, p0, z0.b
+; CHECK-NEXT:    ret
+  %res = call <16 x i8> @llvm.aarch64.sve.uminqv.v16i8.nxv16i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a);
+  ret <16 x i8> %res
+}
+
+define <8 x i16> @uminqv_i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a) {
+; CHECK-LABEL: uminqv_i16:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    uminqv v0.8h, p0, z0.h
+; CHECK-NEXT:    ret
+  %res = call <8 x i16> @llvm.aarch64.sve.uminqv.v8i16.nxv8i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a);
+  ret <8 x i16> %res
+}
+
+define <4 x i32> @uminqv_i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a) {
+; CHECK-LABEL: uminqv_i32:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    uminqv v0.4s, p0, z0.s
+; CHECK-NEXT:    ret
+  %res = call <4 x i32> @llvm.aarch64.sve.uminqv.v4i32.nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a);
+  ret <4 x i32> %res
+}
+
+define <2 x i64> @uminqv_i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a) {
+; CHECK-LABEL: uminqv_i64:
+; CHECK:       // %bb.0:
+; CHECK-NEXT:    uminqv v0.2d, p0, z0.d
+; CHECK-NEXT:    ret
+  %res = call <2 x i64> @llvm.aarch64.sve.uminqv.v2i64.nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a);
+  ret <2 x i64> %res
+}
+
+declare <16 x i8> @llvm.aarch64.sve.orqv.v16i8.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>)
+declare <8 x i16> @llvm.aarch64.sve.orqv.v8i16.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>)
+declare <4 x i32> @llvm.aarch64.sve.orqv.v4i32.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>)
+declare <2 x i64> @llvm.aarch64.sve.orqv.v2i64.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>)
+declare <16 x i8> @llvm.aarch64.sve.eorqv.v16i8.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>)
+declare <8 x i16> @llvm.aarch64.sve.eorqv.v8i16.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>)
+declare <4 x i32> @llvm.aarch64.sve.eorqv.v4i32.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>)
+declare <2 x i64> @llvm.aarch64.sve.eorqv.v2i64.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>)
+declare <16 x i8> @llvm.aarch64.sve.andqv.v16i8.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>)
+declare <8 x i16> @llvm.aarch64.sve.andqv.v8i16.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>)
+declare <4 x i32> @llvm.aarch64.sve.andqv.v4i32.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>)
+declare <2 x i64> @llvm.aarch64.sve.andqv.v2i64.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>)
+declare <16 x i8> @llvm.aarch64.sve.addqv.v16i8.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>)
+declare <8 x i16> @llvm.aarch64.sve.addqv.v8i16.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>)
+declare <4 x i32> @llvm.aarch64.sve.addqv.v4i32.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>)
+declare <2 x i64> @llvm.aarch64.sve.addqv.v2i64.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>)
+declare <16 x i8> @llvm.aarch64.sve.smaxqv.v16i8.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>)
+declare <8 x i16> @llvm.aarch64.sve.smaxqv.v8i16.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>)
+declare <4 x i32> @llvm.aarch64.sve.smaxqv.v4i32.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>)
+declare <2 x i64> @llvm.aarch64.sve.smaxqv.v2i64.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>)
+declare <16 x i8> @llvm.aarch64.sve.umaxqv.v16i8.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>)
+declare <8 x i16> @llvm.aarch64.sve.umaxqv.v8i16.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>)
+declare <4 x i32> @llvm.aarch64.sve.umaxqv.v4i32.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>)
+declare <2 x i64> @llvm.aarch64.sve.umaxqv.v2i64.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>)
+declare <16 x i8> @llvm.aarch64.sve.sminqv.v16i8.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>)
+declare <8 x i16> @llvm.aarch64.sve.sminqv.v8i16.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>)
+declare <4 x i32> @llvm.aarch64.sve.sminqv.v4i32.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>)
+declare <2 x i64> @llvm.aarch64.sve.sminqv.v2i64.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>)
+declare <16 x i8> @llvm.aarch64.sve.uminqv.v16i8.nxv16i8(<vscale x 16 x i1>, <vscale x 16 x i8>)
+declare <8 x i16> @llvm.aarch64.sve.uminqv.v8i16.nxv8i16(<vscale x 8 x i1>, <vscale x 8 x i16>)
+declare <4 x i32> @llvm.aarch64.sve.uminqv.v4i32.nxv4i32(<vscale x 4 x i1>, <vscale x 4 x i32>)
+declare <2 x i64> @llvm.aarch64.sve.uminqv.v2i64.nxv2i64(<vscale x 2 x i1>, <vscale x 2 x i64>)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/clamp-fmed3-const-combine.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/clamp-fmed3-const-combine.ll
index ea46f4d2d419e8ff3741d0b7326147b0a3cf790e..62e5bce23664cda6791a3aa673d450190a638327 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/clamp-fmed3-const-combine.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/clamp-fmed3-const-combine.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -global-isel -mtriple=amdgcn-amd-mesa3d -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX10 %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-mesa3d -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX12 %s
 
 define float @test_fmed3_f32_known_nnan_ieee_true(float %a) #0 {
 ; GFX10-LABEL: test_fmed3_f32_known_nnan_ieee_true:
@@ -7,6 +8,12 @@ define float @test_fmed3_f32_known_nnan_ieee_true(float %a) #0 {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    v_mul_f32_e64 v0, v0, 2.0 clamp
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: test_fmed3_f32_known_nnan_ieee_true:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX12-NEXT:    v_mul_f32_e64 v0, v0, 2.0 clamp
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %fmul = fmul float %a, 2.0
   %fmed = call nnan float @llvm.amdgcn.fmed3.f32(float %fmul, float 0.0, float 1.0)
   ret float %fmed
@@ -18,6 +25,12 @@ define half @test_fmed3_f16_known_nnan_ieee_false(half %a) #1 {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    v_mul_f16_e64 v0, v0, 2.0 clamp
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: test_fmed3_f16_known_nnan_ieee_false:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX12-NEXT:    v_mul_f16_e64 v0, v0, 2.0 clamp
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %fmul = fmul half %a, 2.0
   %fmed = call nnan half @llvm.amdgcn.fmed3.f16(half %fmul, half 0.0, half 1.0)
   ret half %fmed
@@ -31,6 +44,14 @@ define float @test_fmed3_non_SNaN_input_ieee_true_dx10clamp_true(float %a) #2 {
 ; GFX10-NEXT:    v_max_f32_e32 v0, v0, v0
 ; GFX10-NEXT:    v_min_f32_e64 v0, 0x41200000, v0 clamp
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: test_fmed3_non_SNaN_input_ieee_true_dx10clamp_true:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX12-NEXT:    v_max_num_f32_e32 v0, v0, v0
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT:    v_min_num_f32_e64 v0, 0x41200000, v0 clamp
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %fmin = call float @llvm.minnum.f32(float %a, float 10.0)
   %fmed = call float @llvm.amdgcn.fmed3.f32(float %fmin, float 0.0, float 1.0)
   ret float %fmed
@@ -43,6 +64,12 @@ define float @test_fmed3_maybe_SNaN_input_zero_third_operand_ieee_true_dx10clamp
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    v_mul_f32_e64 v0, v0, 2.0 clamp
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: test_fmed3_maybe_SNaN_input_zero_third_operand_ieee_true_dx10clamp_true:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX12-NEXT:    v_mul_f32_e64 v0, v0, 2.0 clamp
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %fmul = fmul float %a, 2.0
   %fmed = call float @llvm.amdgcn.fmed3.f32(float %fmul, float 1.0, float 0.0)
   ret float %fmed
@@ -56,6 +83,12 @@ define float @test_fmed3_global_nnan(float %a) #3 {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    v_mul_f32_e64 v0, v0, 2.0 clamp
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: test_fmed3_global_nnan:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX12-NEXT:    v_mul_f32_e64 v0, v0, 2.0 clamp
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %fmul = fmul float %a, 2.0
   %fmed = call float @llvm.amdgcn.fmed3.f32(float %fmul, float 0.0, float 1.0)
   ret float %fmed
@@ -73,6 +106,12 @@ define float @test_fmed3_f32_maybe_NaN_ieee_false(float %a) #1 {
 ; GFX10-NEXT:    v_mul_f32_e32 v0, 2.0, v0
 ; GFX10-NEXT:    v_med3_f32 v0, v0, 1.0, 0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: test_fmed3_f32_maybe_NaN_ieee_false:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX12-NEXT:    v_mul_f32_e64 v0, v0, 2.0 clamp
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %fmul = fmul float %a, 2.0
   %fmed = call float @llvm.amdgcn.fmed3.f32(float %fmul, float 1.0, float 0.0)
   ret float %fmed
@@ -87,6 +126,14 @@ define float @test_fmed3_non_SNaN_input_ieee_true_dx10clamp_false(float %a) #4 {
 ; GFX10-NEXT:    v_min_f32_e32 v0, 0x41200000, v0
 ; GFX10-NEXT:    v_med3_f32 v0, v0, 0, 1.0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: test_fmed3_non_SNaN_input_ieee_true_dx10clamp_false:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX12-NEXT:    v_max_num_f32_e32 v0, v0, v0
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT:    v_min_num_f32_e64 v0, 0x41200000, v0 clamp
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %fmin = call float @llvm.minnum.f32(float %a, float 10.0)
   %fmed = call float @llvm.amdgcn.fmed3.f32(float %fmin, float 0.0, float 1.0)
   ret float %fmed
@@ -99,6 +146,12 @@ define float @test_fmed3_maybe_SNaN_input_ieee_true_dx10clamp_true(float %a) #2
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    v_mul_f32_e64 v0, v0, 2.0 clamp
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: test_fmed3_maybe_SNaN_input_ieee_true_dx10clamp_true:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX12-NEXT:    v_mul_f32_e64 v0, v0, 2.0 clamp
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %fmul = fmul float %a, 2.0
   %fmed = call float @llvm.amdgcn.fmed3.f32(float %fmul, float 0.0, float 1.0)
   ret float %fmed
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/clamp-minmax-const-combine.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/clamp-minmax-const-combine.ll
index 4f75d205cda35c3f7cf87b75a3bbc34d2bb0a2c7..bba3687dbbc2ccddf75c455d090f7feda7a727a1 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/clamp-minmax-const-combine.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/clamp-minmax-const-combine.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -global-isel -mtriple=amdgcn-amd-mesa3d -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX10 %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-mesa3d -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX12 %s
 
 define float @test_min_max_ValK0_K1_f32(float %a) #0 {
 ; GFX10-LABEL: test_min_max_ValK0_K1_f32:
@@ -7,6 +8,12 @@ define float @test_min_max_ValK0_K1_f32(float %a) #0 {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    v_mul_f32_e64 v0, v0, 2.0 clamp
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: test_min_max_ValK0_K1_f32:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX12-NEXT:    v_mul_f32_e64 v0, v0, 2.0 clamp
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %fmul = fmul float %a, 2.0
   %maxnum = call nnan float @llvm.maxnum.f32(float %fmul, float 0.0)
   %fmed = call nnan float @llvm.minnum.f32(float %maxnum, float 1.0)
@@ -19,6 +26,12 @@ define double @test_min_max_K0Val_K1_f64(double %a) #1 {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    v_mul_f64 v[0:1], v[0:1], 2.0 clamp
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: test_min_max_K0Val_K1_f64:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX12-NEXT:    v_mul_f64_e64 v[0:1], v[0:1], 2.0 clamp
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %fmul = fmul double %a, 2.0
   %maxnum = call nnan double @llvm.maxnum.f64(double 0.0, double %fmul)
   %fmed = call nnan double @llvm.minnum.f64(double %maxnum, double 1.0)
@@ -32,6 +45,12 @@ define half @test_min_K1max_ValK0_f16(half %a) #2 {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    v_mul_f16_e64 v0, v0, 2.0 clamp
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: test_min_K1max_ValK0_f16:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX12-NEXT:    v_mul_f16_e64 v0, v0, 2.0 clamp
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %fmul = fmul half %a, 2.0
   %maxnum = call half @llvm.maxnum.f16(half %fmul, half 0.0)
   %fmed = call half @llvm.minnum.f16(half 1.0, half %maxnum)
@@ -44,6 +63,12 @@ define <2 x half> @test_min_K1max_K0Val_f16(<2 x half> %a) #1 {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    v_pk_mul_f16 v0, v0, 2.0 op_sel_hi:[1,0] clamp
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: test_min_K1max_K0Val_f16:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX12-NEXT:    v_pk_mul_f16 v0, v0, 2.0 op_sel_hi:[1,0] clamp
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %fmul = fmul <2 x half> %a, <half 2.0, half 2.0>
   %maxnum = call nnan <2 x half> @llvm.maxnum.v2f16(<2 x half> <half 0.0, half 0.0>, <2 x half> %fmul)
   %fmed = call nnan <2 x half> @llvm.minnum.v2f16(<2 x half> <half 1.0, half 1.0>, <2 x half> %maxnum)
@@ -56,6 +81,12 @@ define <2 x half> @test_min_max_splat_padded_with_undef(<2 x half> %a) #2 {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    v_pk_mul_f16 v0, v0, 2.0 op_sel_hi:[1,0] clamp
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: test_min_max_splat_padded_with_undef:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX12-NEXT:    v_pk_mul_f16 v0, v0, 2.0 op_sel_hi:[1,0] clamp
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %fmul = fmul <2 x half> %a, <half 2.0, half 2.0>
   %maxnum = call <2 x half> @llvm.maxnum.v2f16(<2 x half> <half 0.0, half undef>, <2 x half> %fmul)
   %fmed = call <2 x half> @llvm.minnum.v2f16(<2 x half> <half 1.0, half undef>, <2 x half> %maxnum)
@@ -70,6 +101,12 @@ define float @test_max_min_ValK1_K0_f32(float %a) #0 {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    v_mul_f32_e64 v0, v0, 2.0 clamp
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: test_max_min_ValK1_K0_f32:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX12-NEXT:    v_mul_f32_e64 v0, v0, 2.0 clamp
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %fmul = fmul float %a, 2.0
   %minnum = call nnan float @llvm.minnum.f32(float %fmul, float 1.0)
   %fmed = call nnan float @llvm.maxnum.f32(float %minnum, float 0.0)
@@ -82,6 +119,12 @@ define double @test_max_min_K1Val_K0_f64(double %a) #1 {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    v_mul_f64 v[0:1], v[0:1], 2.0 clamp
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: test_max_min_K1Val_K0_f64:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX12-NEXT:    v_mul_f64_e64 v[0:1], v[0:1], 2.0 clamp
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %fmul = fmul double %a, 2.0
   %minnum = call nnan double @llvm.minnum.f64(double 1.0, double %fmul)
   %fmed = call nnan double @llvm.maxnum.f64(double %minnum, double 0.0)
@@ -94,6 +137,12 @@ define half @test_max_K0min_ValK1_f16(half %a) #0 {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    v_mul_f16_e64 v0, v0, 2.0 clamp
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: test_max_K0min_ValK1_f16:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX12-NEXT:    v_mul_f16_e64 v0, v0, 2.0 clamp
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %fmul = fmul half %a, 2.0
   %minnum = call nnan half @llvm.minnum.f16(half %fmul, half 1.0)
   %fmed = call nnan half @llvm.maxnum.f16(half 0.0, half %minnum)
@@ -107,6 +156,12 @@ define <2 x half> @test_max_K0min_K1Val_v2f16(<2 x half> %a) #1 {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    v_pk_mul_f16 v0, v0, 2.0 op_sel_hi:[1,0] clamp
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: test_max_K0min_K1Val_v2f16:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX12-NEXT:    v_pk_mul_f16 v0, v0, 2.0 op_sel_hi:[1,0] clamp
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %fmul = fmul <2 x half> %a, <half 2.0, half 2.0>
   %minnum = call nnan <2 x half> @llvm.minnum.v2f16(<2 x half> <half 1.0, half undef>, <2 x half> %fmul)
   %fmed = call nnan <2 x half> @llvm.maxnum.v2f16(<2 x half> <half undef, half 0.0>, <2 x half> %minnum)
@@ -121,6 +176,12 @@ define float @test_min_max_global_nnan(float %a) #3 {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    v_max_f32_e64 v0, v0, v0 clamp
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: test_min_max_global_nnan:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX12-NEXT:    v_max_num_f32_e64 v0, v0, v0 clamp
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %maxnum = call float @llvm.maxnum.f32(float %a, float 0.0)
   %fmed = call float @llvm.minnum.f32(float %maxnum, float 1.0)
   ret float %fmed
@@ -132,6 +193,12 @@ define float @test_max_min_global_nnan(float %a) #3 {
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX10-NEXT:    v_max_f32_e64 v0, v0, v0 clamp
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: test_max_min_global_nnan:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX12-NEXT:    v_max_num_f32_e64 v0, v0, v0 clamp
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %minnum = call float @llvm.minnum.f32(float %a, float 1.0)
   %fmed = call float @llvm.maxnum.f32(float %minnum, float 0.0)
   ret float %fmed
@@ -149,6 +216,12 @@ define float @test_min_max_K0_gt_K1(float %a) #0 {
 ; GFX10-NEXT:    v_max_f32_e32 v0, 1.0, v0
 ; GFX10-NEXT:    v_min_f32_e32 v0, 0, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: test_min_max_K0_gt_K1:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX12-NEXT:    v_maxmin_num_f32 v0, v0, 1.0, 0
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %maxnum = call nnan float @llvm.maxnum.f32(float %a, float 1.0)
   %fmed = call nnan float @llvm.minnum.f32(float %maxnum, float 0.0)
   ret float %fmed
@@ -162,6 +235,12 @@ define float @test_max_min_K0_gt_K1(float %a) #0 {
 ; GFX10-NEXT:    v_min_f32_e32 v0, 0, v0
 ; GFX10-NEXT:    v_max_f32_e32 v0, 1.0, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: test_max_min_K0_gt_K1:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX12-NEXT:    v_minmax_num_f32 v0, v0, 0, 1.0
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %minnum = call nnan float @llvm.minnum.f32(float %a, float 0.0)
   %fmed = call nnan float @llvm.maxnum.f32(float %minnum, float 1.0)
   ret float %fmed
@@ -178,6 +257,12 @@ define float @test_min_max_maybe_NaN_input_ieee_false(float %a) #1 {
 ; GFX10-NEXT:    v_max_f32_e32 v0, 0, v0
 ; GFX10-NEXT:    v_min_f32_e32 v0, 1.0, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: test_min_max_maybe_NaN_input_ieee_false:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX12-NEXT:    v_mul_f32_e64 v0, v0, 2.0 clamp
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %fmul = fmul float %a, 2.0
   %maxnum = call float @llvm.maxnum.f32(float %fmul, float 0.0)
   %fmed = call float @llvm.minnum.f32(float %maxnum, float 1.0)
@@ -192,6 +277,12 @@ define float @test_min_max_maybe_NaN_input_ieee_true_dx10clamp_false(float %a) #
 ; GFX10-NEXT:    v_mul_f32_e32 v0, 2.0, v0
 ; GFX10-NEXT:    v_med3_f32 v0, v0, 0, 1.0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: test_min_max_maybe_NaN_input_ieee_true_dx10clamp_false:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX12-NEXT:    v_mul_f32_e64 v0, v0, 2.0 clamp
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %fmul = fmul float %a, 2.0
   %maxnum = call float @llvm.maxnum.f32(float %fmul, float 0.0)
   %fmed = call float @llvm.minnum.f32(float %maxnum, float 1.0)
@@ -208,6 +299,14 @@ define float @test_max_min_maybe_NaN_input_ieee_true(float %a) #0 {
 ; GFX10-NEXT:    v_min_f32_e32 v0, 1.0, v0
 ; GFX10-NEXT:    v_max_f32_e32 v0, 0, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: test_max_min_maybe_NaN_input_ieee_true:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX12-NEXT:    v_mul_f32_e32 v0, 2.0, v0
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT:    v_minmax_num_f32 v0, v0, 1.0, 0
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %fmul = fmul float %a, 2.0
   %minnum = call float @llvm.minnum.f32(float %fmul, float 1.0)
   %fmed = call float @llvm.maxnum.f32(float %minnum, float 0.0)
@@ -222,6 +321,14 @@ define float @test_max_min_maybe_NaN_input_ieee_false(float %a) #1 {
 ; GFX10-NEXT:    v_min_f32_e32 v0, 1.0, v0
 ; GFX10-NEXT:    v_max_f32_e32 v0, 0, v0
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: test_max_min_maybe_NaN_input_ieee_false:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX12-NEXT:    v_mul_f32_e32 v0, 2.0, v0
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT:    v_minmax_num_f32 v0, v0, 1.0, 0
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %fmul = fmul float %a, 2.0
   %minnum = call float @llvm.minnum.f32(float %fmul, float 1.0)
   %fmed = call float @llvm.maxnum.f32(float %minnum, float 0.0)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-phis-no-lane-mask-merging.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-phis-no-lane-mask-merging.ll
new file mode 100644
index 0000000000000000000000000000000000000000..ccf4e84fbbbd16b9724a958248fe78005400275d
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-phis-no-lane-mask-merging.ll
@@ -0,0 +1,336 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
+; RUN: llc -global-isel -amdgpu-global-isel-risky-select -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 < %s | FileCheck -check-prefix=GFX10 %s
+
+; Divergent phis that don't require lowering using lane mask merging
+
+; - divergent phi that has divergent incoming value (this makes it divergent)
+;   but is reachable through only one path - branch instruction that chooses
+;   path is uniform
+
+; - divergent phi that is used only inside the loop and has incoming from
+;   previous iteration. After phi-elimination (rewrite lane mask in phi def with
+;   lane mask value from previous iteration), phi will hold lane mask valid for
+;   current iteration which is fine since it is not used outside of the loop.
+
+; And one more that is tricky (is branch divergent or not ?)
+; "amdgpu-flat-work-group-size"="1,1" aka single lane execution does not stop
+; shader from activating multiple lanes by using some intrinsic (entering wwm
+; and using dpp instructions)
+; - there are cases with single lane execution where branch instructions are not
+;   lowered to si_if (or other intrinsic branches) - with intention to use
+;   uniform branch after instruction selection?
+;   PhiIncomingAnalysis does not recognize G_BRCOND as divergent branch and does
+;   not perform lane mask merging
+
+
+
+define amdgpu_ps void @divergent_i1_phi_uniform_branch(ptr addrspace(1) %out, i32 %tid, i32 inreg %cond, ptr addrspace(1) %dummyaddr) {
+; GFX10-LABEL: divergent_i1_phi_uniform_branch:
+; GFX10:       ; %bb.0: ; %A
+; GFX10-NEXT:    s_cmp_lg_u32 s0, 0
+; GFX10-NEXT:    s_cbranch_scc0 .LBB0_2
+; GFX10-NEXT:  ; %bb.1:
+; GFX10-NEXT:    v_cmp_le_u32_e64 s0, 6, v2
+; GFX10-NEXT:    s_branch .LBB0_3
+; GFX10-NEXT:  .LBB0_2: ; %dummy
+; GFX10-NEXT:    v_mov_b32_e32 v5, 0x7b
+; GFX10-NEXT:    v_cmp_gt_u32_e64 s0, 1, v2
+; GFX10-NEXT:    global_store_dword v[3:4], v5, off
+; GFX10-NEXT:  .LBB0_3: ; %exit
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, 2, 1, s0
+; GFX10-NEXT:    global_store_dword v[0:1], v2, off
+; GFX10-NEXT:    s_endpgm
+A:
+  %val_A = icmp uge i32 %tid, 6
+  %cmp = icmp eq i32 %cond, 0
+  br i1 %cmp, label %dummy, label %exit
+
+dummy:
+  store i32 123, ptr addrspace(1) %dummyaddr
+  br label %B
+
+B:
+  %val_B = icmp ult i32 %tid, 1
+  br label %exit
+
+exit:
+  %phi = phi i1 [ %val_A, %A ], [ %val_B, %B ]
+  %sel = select i1 %phi, i32 1, i32 2
+  store i32 %sel, ptr addrspace(1) %out
+  ret void
+}
+
+; Fix me - there is no need to merge lane masks here
+define amdgpu_ps void @divergent_i1_phi_uniform_branch_simple(ptr addrspace(1) %out, i32 %tid, i32 inreg %cond) {
+; GFX10-LABEL: divergent_i1_phi_uniform_branch_simple:
+; GFX10:       ; %bb.0: ; %A
+; GFX10-NEXT:    s_cmp_lg_u32 s0, 0
+; GFX10-NEXT:    s_cbranch_scc0 .LBB1_2
+; GFX10-NEXT:  ; %bb.1:
+; GFX10-NEXT:    v_cmp_le_u32_e64 s0, 6, v2
+; GFX10-NEXT:    s_branch .LBB1_3
+; GFX10-NEXT:  .LBB1_2: ; %B
+; GFX10-NEXT:    v_cmp_gt_u32_e64 s0, 1, v2
+; GFX10-NEXT:  .LBB1_3: ; %exit
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, 2, 1, s0
+; GFX10-NEXT:    global_store_dword v[0:1], v2, off
+; GFX10-NEXT:    s_endpgm
+A:
+  %val_A = icmp uge i32 %tid, 6
+  %cmp = icmp eq i32 %cond, 0
+  br i1 %cmp, label %B, label %exit
+
+B:
+  %val_B = icmp ult i32 %tid, 1
+  br label %exit
+
+exit:
+  %phi = phi i1 [ %val_A, %A ], [ %val_B, %B ]
+  %sel = select i1 %phi, i32 1, i32 2
+  store i32 %sel, ptr addrspace(1) %out
+  ret void
+}
+
+
+; Divergent i1 phi that uses value from previous iteration.
+; Used only inside the loop (variable name is bool_counter)
+define void @divergent_i1_phi_used_inside_loop(float %val, ptr %addr) {
+; GFX10-LABEL: divergent_i1_phi_used_inside_loop:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_mov_b32 s4, 0
+; GFX10-NEXT:    v_mov_b32_e32 v3, 1
+; GFX10-NEXT:    v_mov_b32_e32 v4, s4
+; GFX10-NEXT:  .LBB2_1: ; %loop
+; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX10-NEXT:    v_cvt_f32_u32_e32 v5, v4
+; GFX10-NEXT:    v_xor_b32_e32 v3, 1, v3
+; GFX10-NEXT:    v_add_nc_u32_e32 v4, 1, v4
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v5, v0
+; GFX10-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT:    s_cbranch_execnz .LBB2_1
+; GFX10-NEXT:  ; %bb.2: ; %exit
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT:    v_and_b32_e32 v0, 1, v3
+; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v0
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, vcc_lo
+; GFX10-NEXT:    flat_store_dword v[1:2], v0
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+entry:
+  br label %loop
+
+loop:
+  %counter = phi i32 [ 0, %entry ], [ %counterPlus1, %loop ]
+  %bool_counter = phi i1 [ true, %entry ], [ %neg_bool_counter, %loop ]
+  %neg_bool_counter = xor i1 %bool_counter, true
+  %fcounter = uitofp i32 %counter to float
+  %cond = fcmp ogt float %fcounter, %val
+  %counterPlus1 = add i32 %counter, 1
+  br i1 %cond, label %exit, label %loop
+
+exit:
+  %select = select i1 %neg_bool_counter, float 1.000000e+00, float 0.000000e+00
+  store float %select, ptr %addr
+  ret void
+}
+
+define void @divergent_i1_phi_used_inside_loop_bigger_loop_body(float %val, float %pre_cond_val, ptr %addr, ptr %addr_if, ptr %addr_else) {
+; GFX10-LABEL: divergent_i1_phi_used_inside_loop_bigger_loop_body:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, 1.0, v1
+; GFX10-NEXT:    s_mov_b32 s4, 0
+; GFX10-NEXT:    v_mov_b32_e32 v8, 0x3e8
+; GFX10-NEXT:    v_mov_b32_e32 v9, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc_lo
+; GFX10-NEXT:    s_branch .LBB3_2
+; GFX10-NEXT:  .LBB3_1: ; %loop_body
+; GFX10-NEXT:    ; in Loop: Header=BB3_2 Depth=1
+; GFX10-NEXT:    v_cvt_f32_u32_e32 v10, v9
+; GFX10-NEXT:    v_xor_b32_e32 v1, 1, v1
+; GFX10-NEXT:    v_add_nc_u32_e32 v9, 1, v9
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v10, v0
+; GFX10-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT:    s_cbranch_execz .LBB3_6
+; GFX10-NEXT:  .LBB3_2: ; %loop_start
+; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX10-NEXT:    v_cmp_ge_i32_e32 vcc_lo, 0x3e8, v9
+; GFX10-NEXT:    s_mov_b32 s5, 1
+; GFX10-NEXT:    s_cbranch_vccz .LBB3_4
+; GFX10-NEXT:  ; %bb.3: ; %else
+; GFX10-NEXT:    ; in Loop: Header=BB3_2 Depth=1
+; GFX10-NEXT:    s_mov_b32 s5, 0
+; GFX10-NEXT:    flat_store_dword v[6:7], v8
+; GFX10-NEXT:  .LBB3_4: ; %Flow
+; GFX10-NEXT:    ; in Loop: Header=BB3_2 Depth=1
+; GFX10-NEXT:    s_xor_b32 s5, s5, 1
+; GFX10-NEXT:    s_and_b32 s5, s5, 1
+; GFX10-NEXT:    s_cmp_lg_u32 s5, 0
+; GFX10-NEXT:    s_cbranch_scc1 .LBB3_1
+; GFX10-NEXT:  ; %bb.5: ; %if
+; GFX10-NEXT:    ; in Loop: Header=BB3_2 Depth=1
+; GFX10-NEXT:    flat_store_dword v[4:5], v8
+; GFX10-NEXT:    s_branch .LBB3_1
+; GFX10-NEXT:  .LBB3_6: ; %exit
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT:    v_and_b32_e32 v0, 1, v1
+; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v0
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, vcc_lo
+; GFX10-NEXT:    flat_store_dword v[2:3], v0
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+entry:
+  %pre_cond = fcmp ogt float %pre_cond_val, 1.0
+  br label %loop_start
+
+loop_start:
+  %counter = phi i32 [ 0, %entry ], [ %counterPlus1, %loop_body ]
+  %bool_counter = phi i1 [ %pre_cond, %entry ], [ %neg_bool_counter, %loop_body ]
+  %cond_break = icmp sgt i32 %counter, 1000
+  br i1 %cond_break, label %if, label %else
+
+if:
+  store i32 1000, ptr %addr_if
+  br label %loop_body
+
+else:
+  store i32 1000, ptr %addr_else
+  br label %loop_body
+
+loop_body:
+  %neg_bool_counter = xor i1 %bool_counter, true
+  %fcounter = uitofp i32 %counter to float
+  %cond = fcmp ogt float %fcounter, %val
+  %counterPlus1 = add i32 %counter, 1
+  br i1 %cond, label %exit, label %loop_start
+
+exit:
+  %select = select i1 %neg_bool_counter, float 1.000000e+00, float 0.000000e+00
+  store float %select, ptr %addr
+  ret void
+}
+
+; There is a divergent, according to machine uniformity info, g_brcond branch
+; here, not lowered to si_if because of "amdgpu-flat-work-group-size"="1,1".
+define amdgpu_cs void @single_lane_execution_attribute(i32 inreg %.userdata0, <3 x i32> inreg %.WorkgroupId, <3 x i32> %.LocalInvocationId) #0 {
+; GFX10-LABEL: single_lane_execution_attribute:
+; GFX10:       ; %bb.0: ; %.entry
+; GFX10-NEXT:    s_mov_b32 s12, 0
+; GFX10-NEXT:    s_getpc_b64 s[4:5]
+; GFX10-NEXT:    s_mov_b32 s13, -1
+; GFX10-NEXT:    s_mov_b32 s2, s0
+; GFX10-NEXT:    s_and_b64 s[4:5], s[4:5], s[12:13]
+; GFX10-NEXT:    s_mov_b32 s3, s12
+; GFX10-NEXT:    v_mbcnt_lo_u32_b32 v1, -1, 0
+; GFX10-NEXT:    s_or_b64 s[2:3], s[4:5], s[2:3]
+; GFX10-NEXT:    s_load_dwordx8 s[4:11], s[2:3], 0x0
+; GFX10-NEXT:    s_mov_b32 s2, 1
+; GFX10-NEXT:    v_mbcnt_hi_u32_b32 v2, -1, v1
+; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 2, v2
+; GFX10-NEXT:    v_and_b32_e32 v3, 1, v2
+; GFX10-NEXT:    v_xor_b32_e32 v3, 1, v3
+; GFX10-NEXT:    v_and_b32_e32 v3, 1, v3
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    buffer_load_dword v1, v1, s[4:7], 0 offen
+; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v3
+; GFX10-NEXT:    ; implicit-def: $vgpr3
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_cmp_eq_u32_e64 s0, 0, v1
+; GFX10-NEXT:    s_cbranch_vccnz .LBB4_4
+; GFX10-NEXT:  ; %bb.1: ; %.preheader.preheader
+; GFX10-NEXT:    v_mov_b32_e32 v4, s12
+; GFX10-NEXT:    v_mov_b32_e32 v3, s12
+; GFX10-NEXT:  .LBB4_2: ; %.preheader
+; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX10-NEXT:    buffer_load_dword v5, v4, s[4:7], 0 offen
+; GFX10-NEXT:    v_add_nc_u32_e32 v2, -1, v2
+; GFX10-NEXT:    v_add_nc_u32_e32 v4, 4, v4
+; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v2
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_add_nc_u32_e32 v3, v5, v3
+; GFX10-NEXT:    s_cbranch_vccnz .LBB4_2
+; GFX10-NEXT:  ; %bb.3: ; %.preheader._crit_edge
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v3, v1
+; GFX10-NEXT:    s_or_b32 s2, s0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v3, 0, 1, s2
+; GFX10-NEXT:    s_mov_b32 s2, 0
+; GFX10-NEXT:  .LBB4_4: ; %Flow
+; GFX10-NEXT:    s_and_b32 s2, s2, 1
+; GFX10-NEXT:    s_cmp_lg_u32 s2, 0
+; GFX10-NEXT:    s_cbranch_scc0 .LBB4_6
+; GFX10-NEXT:  ; %bb.5: ; %.19
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, 0, 1, s0
+; GFX10-NEXT:    v_or_b32_e32 v3, 2, v1
+; GFX10-NEXT:  .LBB4_6: ; %.22
+; GFX10-NEXT:    v_add_lshl_u32 v0, v0, s1, 2
+; GFX10-NEXT:    buffer_store_dword v3, v0, s[8:11], 0 offen
+; GFX10-NEXT:    s_endpgm
+.entry:
+  %.0 = call i64 @llvm.amdgcn.s.getpc()
+  %.1 = and i64 %.0, -4294967296
+  %.2 = zext i32 %.userdata0 to i64
+  %.3 = or i64 %.1, %.2
+  %.4 = inttoptr i64 %.3 to ptr addrspace(4)
+  %.5 = getelementptr i8, ptr addrspace(4) %.4, i64 16
+  %.6 = load <4 x i32>, ptr addrspace(4) %.5, align 16
+  %.7 = load <4 x i32>, ptr addrspace(4) %.4, align 16
+  %.8 = call i32 @llvm.amdgcn.mbcnt.lo(i32 -1, i32 0)
+  %.9 = call i32 @llvm.amdgcn.mbcnt.hi(i32 -1, i32 %.8)
+  %.fr11 = freeze i32 %.9
+  %.idx = shl i32 %.fr11, 2
+  %.10 = call i32 @llvm.amdgcn.raw.buffer.load.i32(<4 x i32> %.7, i32 %.idx, i32 0, i32 0)
+  %.11 = icmp eq i32 %.10, 0
+  %.12 = and i32 %.fr11, 1
+  %.not = icmp eq i32 %.12, 0
+  br i1 %.not, label %.19, label %.preheader
+
+.preheader:                                       ; preds = %.entry, %.preheader
+  %._96.02 = phi i32 [ %.15, %.preheader ], [ 0, %.entry ]
+  %._50.01 = phi i32 [ %.14, %.preheader ], [ 0, %.entry ]
+  %.idx5 = shl i32 %._96.02, 2
+  %.13 = call i32 @llvm.amdgcn.raw.buffer.load.i32(<4 x i32> %.7, i32 %.idx5, i32 0, i32 0)
+  %.14 = add i32 %.13, %._50.01
+  %.15 = add nuw i32 %._96.02, 1
+  %.exitcond.not = icmp eq i32 %.15, %.fr11
+  br i1 %.exitcond.not, label %.preheader._crit_edge, label %.preheader
+
+.preheader._crit_edge:                            ; preds = %.preheader
+  %.16 = icmp eq i32 %.14, %.10
+  %.17 = or i1 %.11, %.16
+  %.18 = zext i1 %.17 to i32
+  br label %.22
+
+.19:                                               ; preds = %.entry
+  %.20 = zext i1 %.11 to i32
+  %.21 = or i32 %.20, 2
+  br label %.22
+
+.22:                                               ; preds = %.19, %.preheader._crit_edge
+  %._51.0 = phi i32 [ %.18, %.preheader._crit_edge ], [ %.21, %.19 ]
+  %.WorkgroupId.i0 = extractelement <3 x i32> %.WorkgroupId, i64 0
+  %.LocalInvocationId.i0 = extractelement <3 x i32> %.LocalInvocationId, i64 0
+  %.i0 = add i32 %.LocalInvocationId.i0, %.WorkgroupId.i0
+  %.idx6 = shl i32 %.i0, 2
+  call void @llvm.amdgcn.raw.buffer.store.i32(i32 %._51.0, <4 x i32> %.6, i32 %.idx6, i32 0, i32 0)
+  ret void
+}
+
+; Function Attrs: nocallback nofree nosync nounwind willreturn memory(none)
+declare i32 @llvm.amdgcn.mbcnt.lo(i32, i32)
+
+; Function Attrs: nocallback nofree nosync nounwind willreturn memory(none)
+declare i32 @llvm.amdgcn.mbcnt.hi(i32, i32)
+
+; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
+declare i64 @llvm.amdgcn.s.getpc()
+
+; Function Attrs: nocallback nofree nosync nounwind willreturn memory(read)
+declare i32 @llvm.amdgcn.raw.buffer.load.i32(<4 x i32>, i32, i32, i32 immarg)
+
+; Function Attrs: nocallback nofree nosync nounwind willreturn memory(write)
+declare void @llvm.amdgcn.raw.buffer.store.i32(i32, <4 x i32>, i32, i32, i32 immarg)
+
+attributes #0 = { nounwind memory(readwrite) "amdgpu-flat-work-group-size"="1,1" }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-phis-no-lane-mask-merging.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-phis-no-lane-mask-merging.mir
new file mode 100644
index 0000000000000000000000000000000000000000..d314ebe355f51d0c0cf0cb811cd7409de1d0b14a
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-phis-no-lane-mask-merging.mir
@@ -0,0 +1,610 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 4
+# RUN: llc -global-isel -mtriple=amdgcn-mesa-amdpal -mcpu=gfx1010 -run-pass=amdgpu-global-isel-divergence-lowering %s -o - | FileCheck -check-prefix=GFX10 %s
+
+--- |
+  define void @divergent_i1_phi_uniform_branch() {ret void}
+  define void @divergent_i1_phi_uniform_branch_simple() {ret void}
+  define void @divergent_i1_phi_used_inside_loop() {ret void}
+  define void @divergent_i1_phi_used_inside_loop_bigger_loop_body() {ret void}
+  define void @_amdgpu_cs_main() #0 {ret void}
+
+  attributes #0 = {"amdgpu-flat-work-group-size"="1,1"}
+...
+
+---
+name: divergent_i1_phi_uniform_branch
+legalized: true
+tracksRegLiveness: true
+body: |
+  ; GFX10-LABEL: name: divergent_i1_phi_uniform_branch
+  ; GFX10: bb.0:
+  ; GFX10-NEXT:   successors: %bb.1(0x30000000), %bb.2(0x50000000)
+  ; GFX10-NEXT:   liveins: $sgpr0, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GFX10-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GFX10-NEXT:   [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY]](s32), [[COPY1]](s32)
+  ; GFX10-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; GFX10-NEXT:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr0
+  ; GFX10-NEXT:   [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; GFX10-NEXT:   [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; GFX10-NEXT:   [[MV1:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY4]](s32), [[COPY5]](s32)
+  ; GFX10-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 6
+  ; GFX10-NEXT:   [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[COPY2]](s32), [[C]]
+  ; GFX10-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT:   [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[COPY3]](s32), [[C1]]
+  ; GFX10-NEXT:   G_BRCOND [[ICMP1]](s1), %bb.2
+  ; GFX10-NEXT:   G_BR %bb.1
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.1:
+  ; GFX10-NEXT:   successors: %bb.3(0x80000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 123
+  ; GFX10-NEXT:   G_STORE [[C2]](s32), [[MV1]](p1) :: (store (s32), addrspace 1)
+  ; GFX10-NEXT:   G_BR %bb.3
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.2:
+  ; GFX10-NEXT:   successors: %bb.4(0x80000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:_(s1) = G_PHI %14(s1), %bb.3, [[ICMP]](s1), %bb.0
+  ; GFX10-NEXT:   G_BR %bb.4
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.3:
+  ; GFX10-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; GFX10-NEXT:   [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[COPY2]](s32), [[C3]]
+  ; GFX10-NEXT:   G_BR %bb.2
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.4:
+  ; GFX10-NEXT:   [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; GFX10-NEXT:   [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; GFX10-NEXT:   [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[PHI]](s1), [[C5]], [[C4]]
+  ; GFX10-NEXT:   G_STORE [[SELECT]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+  ; GFX10-NEXT:   S_ENDPGM 0
+  bb.0:
+    successors: %bb.1(0x30000000), %bb.2(0x50000000)
+    liveins: $sgpr0, $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(p1) = G_MERGE_VALUES %0(s32), %1(s32)
+    %3:_(s32) = COPY $vgpr2
+    %4:_(s32) = COPY $sgpr0
+    %5:_(s32) = COPY $vgpr3
+    %6:_(s32) = COPY $vgpr4
+    %7:_(p1) = G_MERGE_VALUES %5(s32), %6(s32)
+    %8:_(s32) = G_CONSTANT i32 6
+    %9:_(s1) = G_ICMP intpred(uge), %3(s32), %8
+    %10:_(s32) = G_CONSTANT i32 0
+    %11:_(s1) = G_ICMP intpred(ne), %4(s32), %10
+    G_BRCOND %11(s1), %bb.2
+    G_BR %bb.1
+
+  bb.1:
+    successors: %bb.3(0x80000000)
+
+    %12:_(s32) = G_CONSTANT i32 123
+    G_STORE %12(s32), %7(p1) :: (store (s32), addrspace 1)
+    G_BR %bb.3
+
+  bb.2:
+    successors: %bb.4(0x80000000)
+
+    %13:_(s1) = G_PHI %14(s1), %bb.3, %9(s1), %bb.0
+    G_BR %bb.4
+
+  bb.3:
+    successors: %bb.2(0x80000000)
+
+    %15:_(s32) = G_CONSTANT i32 1
+    %14:_(s1) = G_ICMP intpred(ult), %3(s32), %15
+    G_BR %bb.2
+
+  bb.4:
+    %16:_(s32) = G_CONSTANT i32 2
+    %17:_(s32) = G_CONSTANT i32 1
+    %18:_(s32) = G_SELECT %13(s1), %17, %16
+    G_STORE %18(s32), %2(p1) :: (store (s32), addrspace 1)
+    S_ENDPGM 0
+...
+
+---
+name: divergent_i1_phi_uniform_branch_simple
+legalized: true
+tracksRegLiveness: true
+body: |
+  ; GFX10-LABEL: name: divergent_i1_phi_uniform_branch_simple
+  ; GFX10: bb.0:
+  ; GFX10-NEXT:   successors: %bb.1(0x30000000), %bb.2(0x50000000)
+  ; GFX10-NEXT:   liveins: $sgpr0, $vgpr0, $vgpr1, $vgpr2
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GFX10-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GFX10-NEXT:   [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY]](s32), [[COPY1]](s32)
+  ; GFX10-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; GFX10-NEXT:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr0
+  ; GFX10-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 6
+  ; GFX10-NEXT:   [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[COPY2]](s32), [[C]]
+  ; GFX10-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT:   [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[COPY3]](s32), [[C1]]
+  ; GFX10-NEXT:   G_BRCOND [[ICMP1]](s1), %bb.2
+  ; GFX10-NEXT:   G_BR %bb.1
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.1:
+  ; GFX10-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; GFX10-NEXT:   [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[COPY2]](s32), [[C2]]
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.2:
+  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:_(s1) = G_PHI [[ICMP]](s1), %bb.0, [[ICMP2]](s1), %bb.1
+  ; GFX10-NEXT:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; GFX10-NEXT:   [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; GFX10-NEXT:   [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[PHI]](s1), [[C4]], [[C3]]
+  ; GFX10-NEXT:   G_STORE [[SELECT]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+  ; GFX10-NEXT:   S_ENDPGM 0
+  bb.0:
+    successors: %bb.1(0x30000000), %bb.2(0x50000000)
+    liveins: $sgpr0, $vgpr0, $vgpr1, $vgpr2
+
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(p1) = G_MERGE_VALUES %0(s32), %1(s32)
+    %3:_(s32) = COPY $vgpr2
+    %4:_(s32) = COPY $sgpr0
+    %5:_(s32) = G_CONSTANT i32 6
+    %6:_(s1) = G_ICMP intpred(uge), %3(s32), %5
+    %7:_(s32) = G_CONSTANT i32 0
+    %8:_(s1) = G_ICMP intpred(ne), %4(s32), %7
+    G_BRCOND %8(s1), %bb.2
+    G_BR %bb.1
+
+  bb.1:
+    successors: %bb.2(0x80000000)
+
+    %9:_(s32) = G_CONSTANT i32 1
+    %10:_(s1) = G_ICMP intpred(ult), %3(s32), %9
+
+  bb.2:
+    %11:_(s1) = G_PHI %6(s1), %bb.0, %10(s1), %bb.1
+    %12:_(s32) = G_CONSTANT i32 2
+    %13:_(s32) = G_CONSTANT i32 1
+    %14:_(s32) = G_SELECT %11(s1), %13, %12
+    G_STORE %14(s32), %2(p1) :: (store (s32), addrspace 1)
+    S_ENDPGM 0
+...
+
+---
+name: divergent_i1_phi_used_inside_loop
+legalized: true
+tracksRegLiveness: true
+body: |
+  ; GFX10-LABEL: name: divergent_i1_phi_used_inside_loop
+  ; GFX10: bb.0:
+  ; GFX10-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX10-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GFX10-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GFX10-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; GFX10-NEXT:   [[MV:%[0-9]+]]:_(p0) = G_MERGE_VALUES [[COPY1]](s32), [[COPY2]](s32)
+  ; GFX10-NEXT:   [[C:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
+  ; GFX10-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.1:
+  ; GFX10-NEXT:   successors: %bb.2(0x04000000), %bb.1(0x7c000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:_(s32) = G_PHI %7(s32), %bb.1, [[C1]](s32), %bb.0
+  ; GFX10-NEXT:   [[PHI1:%[0-9]+]]:_(s32) = G_PHI [[C1]](s32), %bb.0, %9(s32), %bb.1
+  ; GFX10-NEXT:   [[PHI2:%[0-9]+]]:_(s1) = G_PHI [[C]](s1), %bb.0, %11(s1), %bb.1
+  ; GFX10-NEXT:   [[C2:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
+  ; GFX10-NEXT:   [[XOR:%[0-9]+]]:_(s1) = G_XOR [[PHI2]], [[C2]]
+  ; GFX10-NEXT:   [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[PHI1]](s32)
+  ; GFX10-NEXT:   [[FCMP:%[0-9]+]]:_(s1) = G_FCMP floatpred(ogt), [[UITOFP]](s32), [[COPY]]
+  ; GFX10-NEXT:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; GFX10-NEXT:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[PHI1]], [[C3]]
+  ; GFX10-NEXT:   [[INTRINSIC_CONVERGENT:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.if.break), [[FCMP]](s1), [[PHI]](s32)
+  ; GFX10-NEXT:   SI_LOOP [[INTRINSIC_CONVERGENT]](s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX10-NEXT:   G_BR %bb.2
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.2:
+  ; GFX10-NEXT:   [[PHI3:%[0-9]+]]:_(s1) = G_PHI [[XOR]](s1), %bb.1
+  ; GFX10-NEXT:   [[PHI4:%[0-9]+]]:_(s32) = G_PHI [[INTRINSIC_CONVERGENT]](s32), %bb.1
+  ; GFX10-NEXT:   G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[PHI4]](s32)
+  ; GFX10-NEXT:   [[C4:%[0-9]+]]:_(s32) = G_FCONSTANT float 0.000000e+00
+  ; GFX10-NEXT:   [[C5:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.000000e+00
+  ; GFX10-NEXT:   [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[PHI3]](s1), [[C5]], [[C4]]
+  ; GFX10-NEXT:   G_STORE [[SELECT]](s32), [[MV]](p0) :: (store (s32))
+  ; GFX10-NEXT:   SI_RETURN
+  bb.0:
+    successors: %bb.1(0x80000000)
+    liveins: $vgpr0, $vgpr1, $vgpr2
+
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $vgpr2
+    %3:_(p0) = G_MERGE_VALUES %1(s32), %2(s32)
+    %4:_(s1) = G_CONSTANT i1 true
+    %5:_(s32) = G_CONSTANT i32 0
+
+  bb.1:
+    successors: %bb.2(0x04000000), %bb.1(0x7c000000)
+
+    %6:_(s32) = G_PHI %7(s32), %bb.1, %5(s32), %bb.0
+    %8:_(s32) = G_PHI %5(s32), %bb.0, %9(s32), %bb.1
+    %10:_(s1) = G_PHI %4(s1), %bb.0, %11(s1), %bb.1
+    %12:_(s1) = G_CONSTANT i1 true
+    %11:_(s1) = G_XOR %10, %12
+    %13:_(s32) = G_UITOFP %8(s32)
+    %14:_(s1) = G_FCMP floatpred(ogt), %13(s32), %0
+    %15:_(s32) = G_CONSTANT i32 1
+    %9:_(s32) = G_ADD %8, %15
+    %7:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.if.break), %14(s1), %6(s32)
+    SI_LOOP %7(s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.2
+
+  bb.2:
+    %16:_(s1) = G_PHI %11(s1), %bb.1
+    %17:_(s32) = G_PHI %7(s32), %bb.1
+    G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %17(s32)
+    %18:_(s32) = G_FCONSTANT float 0.000000e+00
+    %19:_(s32) = G_FCONSTANT float 1.000000e+00
+    %20:_(s32) = G_SELECT %16(s1), %19, %18
+    G_STORE %20(s32), %3(p0) :: (store (s32))
+    SI_RETURN
+...
+
+---
+name: divergent_i1_phi_used_inside_loop_bigger_loop_body
+legalized: true
+tracksRegLiveness: true
+body: |
+  ; GFX10-LABEL: name: divergent_i1_phi_used_inside_loop_bigger_loop_body
+  ; GFX10: bb.0:
+  ; GFX10-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX10-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GFX10-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GFX10-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; GFX10-NEXT:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; GFX10-NEXT:   [[MV:%[0-9]+]]:_(p0) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32)
+  ; GFX10-NEXT:   [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; GFX10-NEXT:   [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
+  ; GFX10-NEXT:   [[MV1:%[0-9]+]]:_(p0) = G_MERGE_VALUES [[COPY4]](s32), [[COPY5]](s32)
+  ; GFX10-NEXT:   [[COPY6:%[0-9]+]]:_(s32) = COPY $vgpr6
+  ; GFX10-NEXT:   [[COPY7:%[0-9]+]]:_(s32) = COPY $vgpr7
+  ; GFX10-NEXT:   [[MV2:%[0-9]+]]:_(p0) = G_MERGE_VALUES [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX10-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.000000e+00
+  ; GFX10-NEXT:   [[FCMP:%[0-9]+]]:_(s1) = G_FCMP floatpred(ogt), [[COPY1]](s32), [[C1]]
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.1:
+  ; GFX10-NEXT:   successors: %bb.4(0x40000000), %bb.2(0x40000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:_(s32) = G_PHI %15(s32), %bb.5, [[C]](s32), %bb.0
+  ; GFX10-NEXT:   [[PHI1:%[0-9]+]]:_(s32) = G_PHI [[C]](s32), %bb.0, %17(s32), %bb.5
+  ; GFX10-NEXT:   [[PHI2:%[0-9]+]]:_(s1) = G_PHI [[FCMP]](s1), %bb.0, %19(s1), %bb.5
+  ; GFX10-NEXT:   [[C2:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
+  ; GFX10-NEXT:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1000
+  ; GFX10-NEXT:   [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(sle), [[PHI1]](s32), [[C3]]
+  ; GFX10-NEXT:   G_BRCOND [[ICMP]](s1), %bb.4
+  ; GFX10-NEXT:   G_BR %bb.2
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.2:
+  ; GFX10-NEXT:   successors: %bb.3(0x40000000), %bb.5(0x40000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[PHI3:%[0-9]+]]:_(s1) = G_PHI %24(s1), %bb.4, [[C2]](s1), %bb.1
+  ; GFX10-NEXT:   [[C4:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
+  ; GFX10-NEXT:   [[XOR:%[0-9]+]]:_(s1) = G_XOR [[PHI3]], [[C4]]
+  ; GFX10-NEXT:   G_BRCOND [[XOR]](s1), %bb.5
+  ; GFX10-NEXT:   G_BR %bb.3
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.3:
+  ; GFX10-NEXT:   successors: %bb.5(0x80000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 1000
+  ; GFX10-NEXT:   G_STORE [[C5]](s32), [[MV1]](p0) :: (store (s32))
+  ; GFX10-NEXT:   G_BR %bb.5
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.4:
+  ; GFX10-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[C6:%[0-9]+]]:_(s1) = G_CONSTANT i1 false
+  ; GFX10-NEXT:   [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 1000
+  ; GFX10-NEXT:   G_STORE [[C7]](s32), [[MV2]](p0) :: (store (s32))
+  ; GFX10-NEXT:   G_BR %bb.2
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.5:
+  ; GFX10-NEXT:   successors: %bb.6(0x04000000), %bb.1(0x7c000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[C8:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
+  ; GFX10-NEXT:   [[XOR1:%[0-9]+]]:_(s1) = G_XOR [[PHI2]], [[C8]]
+  ; GFX10-NEXT:   [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[PHI1]](s32)
+  ; GFX10-NEXT:   [[FCMP1:%[0-9]+]]:_(s1) = G_FCMP floatpred(ogt), [[UITOFP]](s32), [[COPY]]
+  ; GFX10-NEXT:   [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; GFX10-NEXT:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[PHI1]], [[C9]]
+  ; GFX10-NEXT:   [[INTRINSIC_CONVERGENT:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.if.break), [[FCMP1]](s1), [[PHI]](s32)
+  ; GFX10-NEXT:   SI_LOOP [[INTRINSIC_CONVERGENT]](s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX10-NEXT:   G_BR %bb.6
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.6:
+  ; GFX10-NEXT:   [[PHI4:%[0-9]+]]:_(s1) = G_PHI [[XOR1]](s1), %bb.5
+  ; GFX10-NEXT:   [[PHI5:%[0-9]+]]:_(s32) = G_PHI [[INTRINSIC_CONVERGENT]](s32), %bb.5
+  ; GFX10-NEXT:   G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[PHI5]](s32)
+  ; GFX10-NEXT:   [[C10:%[0-9]+]]:_(s32) = G_FCONSTANT float 0.000000e+00
+  ; GFX10-NEXT:   [[C11:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.000000e+00
+  ; GFX10-NEXT:   [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[PHI4]](s1), [[C11]], [[C10]]
+  ; GFX10-NEXT:   G_STORE [[SELECT]](s32), [[MV]](p0) :: (store (s32))
+  ; GFX10-NEXT:   SI_RETURN
+  bb.0:
+    successors: %bb.1(0x80000000)
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7
+
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $vgpr2
+    %3:_(s32) = COPY $vgpr3
+    %4:_(p0) = G_MERGE_VALUES %2(s32), %3(s32)
+    %5:_(s32) = COPY $vgpr4
+    %6:_(s32) = COPY $vgpr5
+    %7:_(p0) = G_MERGE_VALUES %5(s32), %6(s32)
+    %8:_(s32) = COPY $vgpr6
+    %9:_(s32) = COPY $vgpr7
+    %10:_(p0) = G_MERGE_VALUES %8(s32), %9(s32)
+    %11:_(s32) = G_CONSTANT i32 0
+    %12:_(s32) = G_FCONSTANT float 1.000000e+00
+    %13:_(s1) = G_FCMP floatpred(ogt), %1(s32), %12
+
+  bb.1:
+    successors: %bb.4(0x40000000), %bb.2(0x40000000)
+
+    %14:_(s32) = G_PHI %15(s32), %bb.5, %11(s32), %bb.0
+    %16:_(s32) = G_PHI %11(s32), %bb.0, %17(s32), %bb.5
+    %18:_(s1) = G_PHI %13(s1), %bb.0, %19(s1), %bb.5
+    %20:_(s1) = G_CONSTANT i1 true
+    %21:_(s32) = G_CONSTANT i32 1000
+    %22:_(s1) = G_ICMP intpred(sle), %16(s32), %21
+    G_BRCOND %22(s1), %bb.4
+    G_BR %bb.2
+
+  bb.2:
+    successors: %bb.3(0x40000000), %bb.5(0x40000000)
+
+    %23:_(s1) = G_PHI %24(s1), %bb.4, %20(s1), %bb.1
+    %25:_(s1) = G_CONSTANT i1 true
+    %26:_(s1) = G_XOR %23, %25
+    G_BRCOND %26(s1), %bb.5
+    G_BR %bb.3
+
+  bb.3:
+    successors: %bb.5(0x80000000)
+
+    %27:_(s32) = G_CONSTANT i32 1000
+    G_STORE %27(s32), %7(p0) :: (store (s32))
+    G_BR %bb.5
+
+  bb.4:
+    successors: %bb.2(0x80000000)
+
+    %24:_(s1) = G_CONSTANT i1 false
+    %28:_(s32) = G_CONSTANT i32 1000
+    G_STORE %28(s32), %10(p0) :: (store (s32))
+    G_BR %bb.2
+
+  bb.5:
+    successors: %bb.6(0x04000000), %bb.1(0x7c000000)
+
+    %29:_(s1) = G_CONSTANT i1 true
+    %19:_(s1) = G_XOR %18, %29
+    %30:_(s32) = G_UITOFP %16(s32)
+    %31:_(s1) = G_FCMP floatpred(ogt), %30(s32), %0
+    %32:_(s32) = G_CONSTANT i32 1
+    %17:_(s32) = G_ADD %16, %32
+    %15:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.if.break), %31(s1), %14(s32)
+    SI_LOOP %15(s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.6
+
+  bb.6:
+    %33:_(s1) = G_PHI %19(s1), %bb.5
+    %34:_(s32) = G_PHI %15(s32), %bb.5
+    G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %34(s32)
+    %35:_(s32) = G_FCONSTANT float 0.000000e+00
+    %36:_(s32) = G_FCONSTANT float 1.000000e+00
+    %37:_(s32) = G_SELECT %33(s1), %36, %35
+    G_STORE %37(s32), %4(p0) :: (store (s32))
+    SI_RETURN
+...
+
+---
+name: _amdgpu_cs_main
+legalized: true
+tracksRegLiveness: true
+body: |
+  ; GFX10-LABEL: name: _amdgpu_cs_main
+  ; GFX10: bb.0:
+  ; GFX10-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; GFX10-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $sgpr0
+  ; GFX10-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $sgpr1
+  ; GFX10-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GFX10-NEXT:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+  ; GFX10-NEXT:   [[INT:%[0-9]+]]:_(s64) = G_INTRINSIC intrinsic(@llvm.amdgcn.s.getpc)
+  ; GFX10-NEXT:   [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 -4294967296
+  ; GFX10-NEXT:   [[AND:%[0-9]+]]:_(s64) = G_AND [[INT]], [[C]]
+  ; GFX10-NEXT:   [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[COPY]](s32)
+  ; GFX10-NEXT:   [[OR:%[0-9]+]]:_(s64) = G_OR [[AND]], [[ZEXT]]
+  ; GFX10-NEXT:   [[INTTOPTR:%[0-9]+]]:_(p4) = G_INTTOPTR [[OR]](s64)
+  ; GFX10-NEXT:   [[LOAD:%[0-9]+]]:_(<8 x s32>) = G_LOAD [[INTTOPTR]](p4) :: (load (<8 x s32>))
+  ; GFX10-NEXT:   [[BITCAST:%[0-9]+]]:_(s256) = G_BITCAST [[LOAD]](<8 x s32>)
+  ; GFX10-NEXT:   [[TRUNC:%[0-9]+]]:_(s128) = G_TRUNC [[BITCAST]](s256)
+  ; GFX10-NEXT:   [[BITCAST1:%[0-9]+]]:_(<4 x s32>) = G_BITCAST [[TRUNC]](s128)
+  ; GFX10-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+  ; GFX10-NEXT:   [[INT1:%[0-9]+]]:_(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.mbcnt.lo), [[C2]](s32), [[C1]](s32)
+  ; GFX10-NEXT:   [[INT2:%[0-9]+]]:_(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.mbcnt.hi), [[C2]](s32), [[INT1]](s32)
+  ; GFX10-NEXT:   [[FREEZE:%[0-9]+]]:_(s32) = G_FREEZE [[INT2]]
+  ; GFX10-NEXT:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; GFX10-NEXT:   [[SHL:%[0-9]+]]:_(s32) = G_SHL [[FREEZE]], [[C3]](s32)
+  ; GFX10-NEXT:   [[AMDGPU_BUFFER_LOAD:%[0-9]+]]:_(s32) = G_AMDGPU_BUFFER_LOAD [[BITCAST1]](<4 x s32>), [[C1]](s32), [[SHL]], [[C1]], 0, 0, 0 :: (load (s32), align 1, addrspace 8)
+  ; GFX10-NEXT:   [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[AMDGPU_BUFFER_LOAD]](s32), [[C1]]
+  ; GFX10-NEXT:   [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; GFX10-NEXT:   [[AND1:%[0-9]+]]:_(s32) = G_AND [[FREEZE]], [[C4]]
+  ; GFX10-NEXT:   [[TRUNC1:%[0-9]+]]:_(s1) = G_TRUNC [[AND1]](s32)
+  ; GFX10-NEXT:   [[C5:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
+  ; GFX10-NEXT:   [[XOR:%[0-9]+]]:_(s1) = G_XOR [[TRUNC1]], [[C5]]
+  ; GFX10-NEXT:   G_BRCOND [[XOR]](s1), %bb.2
+  ; GFX10-NEXT:   G_BR %bb.1
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.1:
+  ; GFX10-NEXT:   successors: %bb.3(0x80000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT:   G_BR %bb.3
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.2:
+  ; GFX10-NEXT:   successors: %bb.5(0x40000000), %bb.6(0x40000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:_(s32) = G_PHI %30(s32), %bb.4, [[DEF]](s32), %bb.0
+  ; GFX10-NEXT:   [[PHI1:%[0-9]+]]:_(s1) = G_PHI %32(s1), %bb.4, [[C5]](s1), %bb.0
+  ; GFX10-NEXT:   G_BRCOND [[PHI1]](s1), %bb.5
+  ; GFX10-NEXT:   G_BR %bb.6
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.3:
+  ; GFX10-NEXT:   successors: %bb.4(0x04000000), %bb.3(0x7c000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[PHI2:%[0-9]+]]:_(s32) = G_PHI %34(s32), %bb.3, [[C6]](s32), %bb.1
+  ; GFX10-NEXT:   [[PHI3:%[0-9]+]]:_(s32) = G_PHI %36(s32), %bb.3, [[FREEZE]](s32), %bb.1
+  ; GFX10-NEXT:   [[PHI4:%[0-9]+]]:_(s32) = G_PHI %38(s32), %bb.3, [[C6]](s32), %bb.1
+  ; GFX10-NEXT:   [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT:   [[AMDGPU_BUFFER_LOAD1:%[0-9]+]]:_(s32) = G_AMDGPU_BUFFER_LOAD [[BITCAST1]](<4 x s32>), [[C7]](s32), [[PHI2]], [[C7]], 0, 0, 0 :: (load (s32), align 1, addrspace 8)
+  ; GFX10-NEXT:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[AMDGPU_BUFFER_LOAD1]], [[PHI4]]
+  ; GFX10-NEXT:   [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+  ; GFX10-NEXT:   [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[PHI3]], [[C8]]
+  ; GFX10-NEXT:   [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 4
+  ; GFX10-NEXT:   [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[PHI2]], [[C9]]
+  ; GFX10-NEXT:   [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[ADD1]](s32), [[C7]]
+  ; GFX10-NEXT:   G_BRCOND [[ICMP1]](s1), %bb.3
+  ; GFX10-NEXT:   G_BR %bb.4
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.4:
+  ; GFX10-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[PHI5:%[0-9]+]]:_(s32) = G_PHI [[ADD]](s32), %bb.3
+  ; GFX10-NEXT:   [[C10:%[0-9]+]]:_(s1) = G_CONSTANT i1 false
+  ; GFX10-NEXT:   [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[PHI5]](s32), [[AMDGPU_BUFFER_LOAD]]
+  ; GFX10-NEXT:   [[OR1:%[0-9]+]]:_(s1) = G_OR [[ICMP]], [[ICMP2]]
+  ; GFX10-NEXT:   [[ZEXT1:%[0-9]+]]:_(s32) = G_ZEXT [[OR1]](s1)
+  ; GFX10-NEXT:   G_BR %bb.2
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.5:
+  ; GFX10-NEXT:   successors: %bb.6(0x80000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[ZEXT2:%[0-9]+]]:_(s32) = G_ZEXT [[ICMP]](s1)
+  ; GFX10-NEXT:   [[C11:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; GFX10-NEXT:   [[OR2:%[0-9]+]]:_(s32) = G_OR [[ZEXT2]], [[C11]]
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.6:
+  ; GFX10-NEXT:   [[PHI6:%[0-9]+]]:_(s32) = G_PHI [[PHI]](s32), %bb.2, [[OR2]](s32), %bb.5
+  ; GFX10-NEXT:   [[UV:%[0-9]+]]:_(<4 x s32>), [[UV1:%[0-9]+]]:_(<4 x s32>) = G_UNMERGE_VALUES [[LOAD]](<8 x s32>)
+  ; GFX10-NEXT:   [[ADD3:%[0-9]+]]:_(s32) = G_ADD [[COPY2]], [[COPY1]]
+  ; GFX10-NEXT:   [[C12:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; GFX10-NEXT:   [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[ADD3]], [[C12]](s32)
+  ; GFX10-NEXT:   [[C13:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT:   G_AMDGPU_BUFFER_STORE [[PHI6]](s32), [[UV1]](<4 x s32>), [[C13]](s32), [[SHL1]], [[C13]], 0, 0, 0 :: (store (s32), align 1, addrspace 8)
+  ; GFX10-NEXT:   S_ENDPGM 0
+  bb.0:
+    successors: %bb.1(0x40000000), %bb.2(0x40000000)
+    liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2
+
+    %0:_(s32) = COPY $sgpr0
+    %1:_(s32) = COPY $sgpr1
+    %2:_(s32) = COPY $vgpr0
+    %3:_(s32) = G_IMPLICIT_DEF
+    %4:_(s64) = G_INTRINSIC intrinsic(@llvm.amdgcn.s.getpc)
+    %5:_(s64) = G_CONSTANT i64 -4294967296
+    %6:_(s64) = G_AND %4, %5
+    %7:_(s64) = G_ZEXT %0(s32)
+    %8:_(s64) = G_OR %6, %7
+    %9:_(p4) = G_INTTOPTR %8(s64)
+    %10:_(<8 x s32>) = G_LOAD %9(p4) :: (load (<8 x s32>))
+    %11:_(s256) = G_BITCAST %10(<8 x s32>)
+    %12:_(s128) = G_TRUNC %11(s256)
+    %13:_(<4 x s32>) = G_BITCAST %12(s128)
+    %15:_(s32) = G_CONSTANT i32 0
+    %14:_(s32) = G_CONSTANT i32 -1
+    %16:_(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.mbcnt.lo), %14(s32), %15(s32)
+    %17:_(s32) = G_INTRINSIC intrinsic(@llvm.amdgcn.mbcnt.hi), %14(s32), %16(s32)
+    %18:_(s32) = G_FREEZE %17
+    %19:_(s32) = G_CONSTANT i32 2
+    %20:_(s32) = G_SHL %18, %19(s32)
+    %21:_(s32) = G_AMDGPU_BUFFER_LOAD %13(<4 x s32>), %15(s32), %20, %15, 0, 0, 0 :: (load (s32), align 1, addrspace 8)
+    %22:_(s1) = G_ICMP intpred(eq), %21(s32), %15
+    %23:_(s32) = G_CONSTANT i32 1
+    %24:_(s32) = G_AND %18, %23
+    %25:_(s1) = G_TRUNC %24(s32)
+    %26:_(s1) = G_CONSTANT i1 true
+    %27:_(s1) = G_XOR %25, %26
+    G_BRCOND %27(s1), %bb.2
+    G_BR %bb.1
+
+  bb.1:
+    successors: %bb.3(0x80000000)
+
+    %28:_(s32) = G_CONSTANT i32 0
+    G_BR %bb.3
+
+  bb.2:
+    successors: %bb.5(0x40000000), %bb.6(0x40000000)
+
+    %29:_(s32) = G_PHI %30(s32), %bb.4, %3(s32), %bb.0
+    %31:_(s1) = G_PHI %32(s1), %bb.4, %26(s1), %bb.0
+    G_BRCOND %31(s1), %bb.5
+    G_BR %bb.6
+
+  bb.3:
+    successors: %bb.4(0x04000000), %bb.3(0x7c000000)
+
+    %33:_(s32) = G_PHI %34(s32), %bb.3, %28(s32), %bb.1
+    %35:_(s32) = G_PHI %36(s32), %bb.3, %18(s32), %bb.1
+    %37:_(s32) = G_PHI %38(s32), %bb.3, %28(s32), %bb.1
+    %39:_(s32) = G_CONSTANT i32 0
+    %40:_(s32) = G_AMDGPU_BUFFER_LOAD %13(<4 x s32>), %39(s32), %33, %39, 0, 0, 0 :: (load (s32), align 1, addrspace 8)
+    %38:_(s32) = G_ADD %40, %37
+    %41:_(s32) = G_CONSTANT i32 -1
+    %36:_(s32) = G_ADD %35, %41
+    %42:_(s32) = G_CONSTANT i32 4
+    %34:_(s32) = G_ADD %33, %42
+    %43:_(s1) = G_ICMP intpred(ne), %36(s32), %39
+    G_BRCOND %43(s1), %bb.3
+    G_BR %bb.4
+
+  bb.4:
+    successors: %bb.2(0x80000000)
+
+    %44:_(s32) = G_PHI %38(s32), %bb.3
+    %32:_(s1) = G_CONSTANT i1 false
+    %45:_(s1) = G_ICMP intpred(eq), %44(s32), %21
+    %46:_(s1) = G_OR %22, %45
+    %30:_(s32) = G_ZEXT %46(s1)
+    G_BR %bb.2
+
+  bb.5:
+    successors: %bb.6(0x80000000)
+
+    %47:_(s32) = G_ZEXT %22(s1)
+    %48:_(s32) = G_CONSTANT i32 2
+    %49:_(s32) = G_OR %47, %48
+
+  bb.6:
+    %50:_(s32) = G_PHI %29(s32), %bb.2, %49(s32), %bb.5
+    %51:_(<4 x s32>), %52:_(<4 x s32>) = G_UNMERGE_VALUES %10(<8 x s32>)
+    %53:_(s32) = G_ADD %2, %1
+    %54:_(s32) = G_CONSTANT i32 2
+    %55:_(s32) = G_SHL %53, %54(s32)
+    %56:_(s32) = G_CONSTANT i32 0
+    G_AMDGPU_BUFFER_STORE %50(s32), %52(<4 x s32>), %56(s32), %55, %56, 0, 0, 0 :: (store (s32), align 1, addrspace 8)
+    S_ENDPGM 0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-used-outside-loop.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-used-outside-loop.ll
new file mode 100644
index 0000000000000000000000000000000000000000..34dedfe10365f3902aefe3a671caa6cc3e6eed03
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-used-outside-loop.ll
@@ -0,0 +1,512 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -amdgpu-global-isel-risky-select -mcpu=gfx1010 < %s | FileCheck -check-prefix=GFX10 %s
+
+; This file contains various tests that have divergent i1s used outside of
+; the loop. These are lane masks is sgpr and need to have correct value in
+; corresponding bit at the iteration lane exits the loop.
+; Achieved by merging lane mask with same lane mask from previous iteration
+; and using that merged lane mask outside of the loop.
+
+; Phi used outside of the loop directly (loopfinder will figure out that it
+; needs to merge lane mask across all iterations)
+define void @divergent_i1_phi_used_outside_loop(float %val, float %pre.cond.val, ptr %addr) {
+; GFX10-LABEL: divergent_i1_phi_used_outside_loop:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, 1.0, v1
+; GFX10-NEXT:    s_mov_b32 s4, 0
+; GFX10-NEXT:    v_mov_b32_e32 v1, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v4, 0, 1, vcc_lo
+; GFX10-NEXT:  .LBB0_1: ; %loop
+; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX10-NEXT:    v_cvt_f32_u32_e32 v6, v1
+; GFX10-NEXT:    v_mov_b32_e32 v5, v4
+; GFX10-NEXT:    v_add_nc_u32_e32 v1, 1, v1
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v6, v0
+; GFX10-NEXT:    v_xor_b32_e32 v4, 1, v5
+; GFX10-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT:    s_cbranch_execnz .LBB0_1
+; GFX10-NEXT:  ; %bb.2: ; %exit
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT:    v_and_b32_e32 v0, 1, v5
+; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v0
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, vcc_lo
+; GFX10-NEXT:    flat_store_dword v[2:3], v0
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+entry:
+  %pre.cond = fcmp ogt float %pre.cond.val, 1.0
+  br label %loop
+
+loop:
+  %counter = phi i32 [ 0, %entry ], [ %counter.plus.1, %loop ]
+  %bool.counter = phi i1 [ %pre.cond, %entry ], [ %neg.bool.counter, %loop ]
+  %neg.bool.counter = xor i1 %bool.counter, true
+  %f.counter = uitofp i32 %counter to float
+  %cond = fcmp ogt float %f.counter, %val
+  %counter.plus.1 = add i32 %counter, 1
+  br i1 %cond, label %exit, label %loop
+
+exit:
+  %select = select i1 %bool.counter, float 1.000000e+00, float 0.000000e+00
+  store float %select, ptr %addr
+  ret void
+}
+
+define void @divergent_i1_phi_used_outside_loop_larger_loop_body(float %val, ptr addrspace(1) %a, ptr %addr) {
+; GFX10-LABEL: divergent_i1_phi_used_outside_loop_larger_loop_body:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_mov_b32 s4, -1
+; GFX10-NEXT:    v_mov_b32_e32 v5, 1
+; GFX10-NEXT:    v_mov_b32_e32 v0, s4
+; GFX10-NEXT:    s_branch .LBB1_2
+; GFX10-NEXT:  .LBB1_1: ; %loop.cond
+; GFX10-NEXT:    ; in Loop: Header=BB1_2 Depth=1
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT:    v_add_nc_u32_e32 v0, 1, v0
+; GFX10-NEXT:    v_add_co_u32 v1, s4, v1, 4
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v2, s4, 0, v2, s4
+; GFX10-NEXT:    v_cmp_le_i32_e32 vcc_lo, 10, v0
+; GFX10-NEXT:    v_cndmask_b32_e64 v5, 0, 1, s6
+; GFX10-NEXT:    s_cbranch_vccz .LBB1_4
+; GFX10-NEXT:  .LBB1_2: ; %loop.start
+; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX10-NEXT:    v_and_b32_e32 v5, 1, v5
+; GFX10-NEXT:    v_cmp_ne_u32_e64 s5, 0, v5
+; GFX10-NEXT:    s_mov_b32 s6, s5
+; GFX10-NEXT:    s_and_saveexec_b32 s4, s5
+; GFX10-NEXT:    s_cbranch_execz .LBB1_1
+; GFX10-NEXT:  ; %bb.3: ; %is.eq.zero
+; GFX10-NEXT:    ; in Loop: Header=BB1_2 Depth=1
+; GFX10-NEXT:    global_load_dword v5, v[1:2], off
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_cmp_eq_u32_e64 s6, 0, v5
+; GFX10-NEXT:    s_branch .LBB1_1
+; GFX10-NEXT:  .LBB1_4: ; %exit
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s5
+; GFX10-NEXT:    flat_store_dword v[3:4], v0
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+entry:
+  br label %loop.start
+
+loop.start:
+  %i = phi i32 [ 0, %entry ], [ %i.plus.1, %loop.cond ]
+  %all.eq.zero = phi i1 [ true, %entry ], [ %eq.zero, %loop.cond ]
+  br i1 %all.eq.zero, label %is.eq.zero, label %loop.cond
+
+is.eq.zero:
+  %a.plus.i = getelementptr i32, ptr addrspace(1) %a, i32 %i
+  %elt.i = load i32, ptr addrspace(1) %a.plus.i
+  %elt.i.eq.zero = icmp eq i32 %elt.i, 0
+  br label %loop.cond
+
+loop.cond:
+  %eq.zero = phi i1 [ %all.eq.zero, %loop.start ], [ %elt.i.eq.zero, %is.eq.zero ]
+  %cond = icmp slt i32 %i, 10
+  %i.plus.1 = add i32 %i, 1
+  br i1 %cond, label %exit, label %loop.start
+
+exit:
+  %select = select i1 %all.eq.zero, float 1.000000e+00, float 0.000000e+00
+  store float %select, ptr %addr
+  ret void
+}
+
+; Non-phi used outside of the loop
+
+define void @divergent_i1_xor_used_outside_loop(float %val, float %pre.cond.val, ptr %addr) {
+; GFX10-LABEL: divergent_i1_xor_used_outside_loop:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_cmp_lt_f32_e32 vcc_lo, 1.0, v1
+; GFX10-NEXT:    s_mov_b32 s4, 0
+; GFX10-NEXT:    v_mov_b32_e32 v4, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, 0, 1, vcc_lo
+; GFX10-NEXT:  .LBB2_1: ; %loop
+; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX10-NEXT:    v_cvt_f32_u32_e32 v5, v4
+; GFX10-NEXT:    v_xor_b32_e32 v1, 1, v1
+; GFX10-NEXT:    v_add_nc_u32_e32 v4, 1, v4
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v5, v0
+; GFX10-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT:    s_cbranch_execnz .LBB2_1
+; GFX10-NEXT:  ; %bb.2: ; %exit
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT:    v_and_b32_e32 v0, 1, v1
+; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v0
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, vcc_lo
+; GFX10-NEXT:    flat_store_dword v[2:3], v0
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+entry:
+  %pre.cond = fcmp ogt float %pre.cond.val, 1.0
+  br label %loop
+
+loop:
+  %counter = phi i32 [ 0, %entry ], [ %counter.plus.1, %loop ]
+  %bool.counter = phi i1 [ %pre.cond, %entry ], [ %neg.bool.counter, %loop ]
+  %neg.bool.counter = xor i1 %bool.counter, true
+  %f.counter = uitofp i32 %counter to float
+  %cond = fcmp ogt float %f.counter, %val
+  %counter.plus.1 = add i32 %counter, 1
+  br i1 %cond, label %exit, label %loop
+
+exit:
+  %select = select i1 %neg.bool.counter, float 1.000000e+00, float 0.000000e+00
+  store float %select, ptr %addr
+  ret void
+}
+
+;void xor(int num_elts, int* a, int* addr) {
+;for(int i=0; i<num_elts; ++i) {
+;  if(a[i]==0)
+;    return;
+;}
+;addr[0] = 5
+;return;
+;}
+
+define void @divergent_i1_xor_used_outside_loop_larger_loop_body(i32 %num.elts, ptr addrspace(1) %a, ptr %addr) {
+; GFX10-LABEL: divergent_i1_xor_used_outside_loop_larger_loop_body:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v0
+; GFX10-NEXT:    s_mov_b32 s5, 0
+; GFX10-NEXT:    s_mov_b32 s6, -1
+; GFX10-NEXT:    s_and_saveexec_b32 s4, vcc_lo
+; GFX10-NEXT:    s_cbranch_execz .LBB3_6
+; GFX10-NEXT:  ; %bb.1: ; %loop.start.preheader
+; GFX10-NEXT:    v_mov_b32_e32 v5, s5
+; GFX10-NEXT:    s_branch .LBB3_3
+; GFX10-NEXT:  .LBB3_2: ; %Flow
+; GFX10-NEXT:    ; in Loop: Header=BB3_3 Depth=1
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s8
+; GFX10-NEXT:    s_xor_b32 s7, s7, 1
+; GFX10-NEXT:    s_and_b32 s6, exec_lo, s6
+; GFX10-NEXT:    s_or_b32 s5, s6, s5
+; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT:    s_cbranch_execz .LBB3_5
+; GFX10-NEXT:  .LBB3_3: ; %loop.start
+; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX10-NEXT:    v_ashrrev_i32_e32 v6, 31, v5
+; GFX10-NEXT:    s_mov_b32 s6, -1
+; GFX10-NEXT:    s_mov_b32 s7, 1
+; GFX10-NEXT:    v_lshlrev_b64 v[6:7], 2, v[5:6]
+; GFX10-NEXT:    v_add_co_u32 v6, vcc_lo, v1, v6
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, v2, v7, vcc_lo
+; GFX10-NEXT:    global_load_dword v6, v[6:7], off
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v6
+; GFX10-NEXT:    s_and_saveexec_b32 s8, vcc_lo
+; GFX10-NEXT:    s_cbranch_execz .LBB3_2
+; GFX10-NEXT:  ; %bb.4: ; %loop.cond
+; GFX10-NEXT:    ; in Loop: Header=BB3_3 Depth=1
+; GFX10-NEXT:    v_add_nc_u32_e32 v6, 1, v5
+; GFX10-NEXT:    v_cmp_lt_i32_e64 s6, v5, v0
+; GFX10-NEXT:    s_mov_b32 s7, 0
+; GFX10-NEXT:    v_mov_b32_e32 v5, v6
+; GFX10-NEXT:    s_branch .LBB3_2
+; GFX10-NEXT:  .LBB3_5: ; %loop.exit.guard
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT:    s_and_b32 s5, 1, s7
+; GFX10-NEXT:    v_cmp_ne_u32_e64 s6, 0, s5
+; GFX10-NEXT:  .LBB3_6: ; %Flow1
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT:    s_and_saveexec_b32 s4, s6
+; GFX10-NEXT:    s_cbranch_execz .LBB3_8
+; GFX10-NEXT:  ; %bb.7: ; %block.after.loop
+; GFX10-NEXT:    v_mov_b32_e32 v0, 5
+; GFX10-NEXT:    flat_store_dword v[3:4], v0
+; GFX10-NEXT:  .LBB3_8: ; %exit
+; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+entry:
+  %start.cond = icmp eq i32 %num.elts, 0
+  br i1 %start.cond, label %loop.start, label %block.after.loop
+
+loop.start:
+  %i = phi i32 [ 0, %entry ], [ %i.plus.1, %loop.cond ]
+  %a.plus.i = getelementptr i32, ptr addrspace(1) %a, i32 %i
+  %elt.i = load i32, ptr addrspace(1) %a.plus.i
+  %elt.i.eq.zero = icmp eq i32 %elt.i, 0
+  br i1 %elt.i.eq.zero, label %exit, label %loop.cond
+
+loop.cond:
+  %cond = icmp slt i32 %i, %num.elts
+  %i.plus.1 = add i32 %i, 1
+  br i1 %cond, label %block.after.loop, label %loop.start
+
+block.after.loop:
+  store i32 5, ptr %addr
+  br label %exit
+
+exit:
+  ret void
+}
+
+
+;void icmp(int num_elts, int* a, int* addr) {
+;for(;;) {
+;  if(a[i]==0)
+;    return;
+;}
+;addr[0] = 5
+;return;
+;}
+
+define void @divergent_i1_icmp_used_outside_loop(i32 %v0, i32 %v1, ptr addrspace(1) %a, ptr addrspace(1) %b, ptr addrspace(1) %c, ptr %addr) {
+; GFX10-LABEL: divergent_i1_icmp_used_outside_loop:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_mov_b32 s5, 0
+; GFX10-NEXT:    v_mov_b32_e32 v5, s5
+; GFX10-NEXT:    s_branch .LBB4_2
+; GFX10-NEXT:  .LBB4_1: ; %Flow
+; GFX10-NEXT:    ; in Loop: Header=BB4_2 Depth=1
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s7
+; GFX10-NEXT:    s_and_b32 s4, 1, s6
+; GFX10-NEXT:    v_cmp_ne_u32_e64 s4, 0, s4
+; GFX10-NEXT:    s_or_b32 s5, s4, s5
+; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT:    s_cbranch_execz .LBB4_6
+; GFX10-NEXT:  .LBB4_2: ; %cond.block.0
+; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX10-NEXT:    v_mov_b32_e32 v4, v5
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, v0, v4
+; GFX10-NEXT:    s_and_saveexec_b32 s6, vcc_lo
+; GFX10-NEXT:    s_cbranch_execz .LBB4_4
+; GFX10-NEXT:  ; %bb.3: ; %if.block.0
+; GFX10-NEXT:    ; in Loop: Header=BB4_2 Depth=1
+; GFX10-NEXT:    v_ashrrev_i32_e32 v5, 31, v4
+; GFX10-NEXT:    v_lshlrev_b64 v[8:9], 2, v[4:5]
+; GFX10-NEXT:    v_add_co_u32 v8, s4, v2, v8
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v9, s4, v3, v9, s4
+; GFX10-NEXT:    global_store_dword v[8:9], v4, off
+; GFX10-NEXT:  .LBB4_4: ; %loop.break.block
+; GFX10-NEXT:    ; in Loop: Header=BB4_2 Depth=1
+; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s6
+; GFX10-NEXT:    v_cmp_ne_u32_e64 s4, v1, v4
+; GFX10-NEXT:    s_mov_b32 s6, 1
+; GFX10-NEXT:    ; implicit-def: $vgpr5
+; GFX10-NEXT:    s_and_saveexec_b32 s7, s4
+; GFX10-NEXT:    s_cbranch_execz .LBB4_1
+; GFX10-NEXT:  ; %bb.5: ; %loop.cond
+; GFX10-NEXT:    ; in Loop: Header=BB4_2 Depth=1
+; GFX10-NEXT:    v_add_nc_u32_e32 v5, 1, v4
+; GFX10-NEXT:    s_mov_b32 s6, 0
+; GFX10-NEXT:    s_branch .LBB4_1
+; GFX10-NEXT:  .LBB4_6: ; %cond.block.1
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s5
+; GFX10-NEXT:    s_and_saveexec_b32 s4, vcc_lo
+; GFX10-NEXT:    s_cbranch_execz .LBB4_8
+; GFX10-NEXT:  ; %bb.7: ; %if.block.1
+; GFX10-NEXT:    global_store_dword v[6:7], v4, off
+; GFX10-NEXT:  .LBB4_8: ; %exit
+; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+entry:
+  br label %loop.start
+
+loop.start:
+  %i = phi i32 [ 0, %entry ], [ %i.plus.1, %loop.cond ]
+  br label %cond.block.0
+
+cond.block.0:
+  %cond.0 = icmp eq i32 %v0, %i
+  br i1 %cond.0, label %if.block.0, label %loop.break.block
+
+if.block.0:
+  %a.plus.i = getelementptr i32, ptr addrspace(1) %a, i32 %i
+  store i32 %i, ptr addrspace(1) %a.plus.i
+  br label %loop.break.block
+
+loop.break.block:
+  %cond.1 = icmp eq i32 %v1, %i
+  br i1 %cond.1, label %cond.block.1, label %loop.cond
+
+loop.cond:
+  ; no cond, infinite loop with one break
+  %i.plus.1 = add i32 %i, 1
+  br label %loop.start
+
+cond.block.1:
+  %cond.2 = icmp eq i32 %v0, %i
+  br i1 %cond.2, label %if.block.1, label %exit
+
+if.block.1:
+  store i32 %i, ptr addrspace(1) %c
+  br label %exit
+
+exit:
+  ret void
+}
+
+
+; bool all_eq_zero = true;
+; i32 i = 0;
+; do {
+;   if(all_eq_zero)
+;     all_eq_zero = (a[i] == 0);
+;
+;   i += 1;
+; } while ( i < n )
+
+; *addr = all_eq_zero ? 1.0 : 0.0;
+
+; check that all elements in an array of size n are zero, loop has divergent
+; exit condition based on array size, but zero check does not break out of the
+; loop but instead skips zero check in remaining iterations
+; llpc "freezes" zero check since it is (via phi) used in a conditional branch
+define amdgpu_ps void @divergent_i1_freeze_used_outside_loop(i32 %n, ptr addrspace(1) %a, ptr %addr) {
+; GFX10-LABEL: divergent_i1_freeze_used_outside_loop:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:    s_mov_b32 s0, 0
+; GFX10-NEXT:    v_mov_b32_e32 v6, 1
+; GFX10-NEXT:    v_mov_b32_e32 v5, s0
+; GFX10-NEXT:    s_branch .LBB5_2
+; GFX10-NEXT:  .LBB5_1: ; %loop.cond
+; GFX10-NEXT:    ; in Loop: Header=BB5_2 Depth=1
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s2
+; GFX10-NEXT:    v_add_nc_u32_e32 v7, 1, v5
+; GFX10-NEXT:    v_cmp_lt_i32_e32 vcc_lo, v5, v0
+; GFX10-NEXT:    v_cndmask_b32_e64 v6, 0, 1, s1
+; GFX10-NEXT:    v_mov_b32_e32 v5, v7
+; GFX10-NEXT:    s_or_b32 s0, vcc_lo, s0
+; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s0
+; GFX10-NEXT:    s_cbranch_execz .LBB5_4
+; GFX10-NEXT:  .LBB5_2: ; %loop.start
+; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX10-NEXT:    v_and_b32_e32 v6, 1, v6
+; GFX10-NEXT:    v_cmp_ne_u32_e64 s1, 0, v6
+; GFX10-NEXT:    s_and_saveexec_b32 s2, s1
+; GFX10-NEXT:    s_cbranch_execz .LBB5_1
+; GFX10-NEXT:  ; %bb.3: ; %is.eq.zero
+; GFX10-NEXT:    ; in Loop: Header=BB5_2 Depth=1
+; GFX10-NEXT:    v_ashrrev_i32_e32 v6, 31, v5
+; GFX10-NEXT:    v_lshlrev_b64 v[6:7], 2, v[5:6]
+; GFX10-NEXT:    v_add_co_u32 v6, vcc_lo, v1, v6
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, v2, v7, vcc_lo
+; GFX10-NEXT:    global_load_dword v6, v[6:7], off
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_cmp_eq_u32_e64 s1, 0, v6
+; GFX10-NEXT:    s_branch .LBB5_1
+; GFX10-NEXT:  .LBB5_4: ; %exit
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, s1
+; GFX10-NEXT:    flat_store_dword v[3:4], v0
+; GFX10-NEXT:    s_endpgm
+entry:
+  br label %loop.start
+
+loop.start:
+  %i = phi i32 [ 0, %entry ], [ %i.plus.1, %loop.cond ]
+  %all.eq.zero = phi i1 [ true, %entry ], [ %eq.zero.fr, %loop.cond ]
+  br i1 %all.eq.zero, label %is.eq.zero, label %loop.cond
+
+is.eq.zero:
+  %a.plus.i = getelementptr i32, ptr addrspace(1) %a, i32 %i
+  %elt.i = load i32, ptr addrspace(1) %a.plus.i
+  %elt.i.eq.zero = icmp eq i32 %elt.i, 0
+  br label %loop.cond
+
+loop.cond:
+  %eq.zero = phi i1 [ %all.eq.zero, %loop.start ], [ %elt.i.eq.zero, %is.eq.zero ]
+  %eq.zero.fr = freeze i1 %eq.zero
+  %cond = icmp slt i32 %i, %n
+  %i.plus.1 = add i32 %i, 1
+  br i1 %cond, label %exit, label %loop.start
+
+exit:
+  %select = select i1 %eq.zero.fr, float 1.000000e+00, float 0.000000e+00
+  store float %select, ptr %addr
+  ret void
+}
+
+; Divergent i1 phi from structurize-cfg used outside of the loop
+define amdgpu_cs void @loop_with_1break(ptr addrspace(1) %x, ptr addrspace(1) %a, ptr addrspace(1) %a.break) {
+; GFX10-LABEL: loop_with_1break:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:    s_mov_b32 s0, 0
+; GFX10-NEXT:    v_mov_b32_e32 v6, s0
+; GFX10-NEXT:    s_branch .LBB6_2
+; GFX10-NEXT:  .LBB6_1: ; %Flow
+; GFX10-NEXT:    ; in Loop: Header=BB6_2 Depth=1
+; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s1
+; GFX10-NEXT:    s_and_b32 s1, exec_lo, s2
+; GFX10-NEXT:    s_or_b32 s0, s1, s0
+; GFX10-NEXT:    s_and_b32 s1, 1, s3
+; GFX10-NEXT:    v_cmp_ne_u32_e64 s1, 0, s1
+; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s0
+; GFX10-NEXT:    s_cbranch_execz .LBB6_4
+; GFX10-NEXT:  .LBB6_2: ; %A
+; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX10-NEXT:    v_ashrrev_i32_e32 v7, 31, v6
+; GFX10-NEXT:    s_mov_b32 s2, -1
+; GFX10-NEXT:    s_mov_b32 s3, 1
+; GFX10-NEXT:    v_lshlrev_b64 v[7:8], 2, v[6:7]
+; GFX10-NEXT:    v_add_co_u32 v9, vcc_lo, v2, v7
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v10, vcc_lo, v3, v8, vcc_lo
+; GFX10-NEXT:    global_load_dword v9, v[9:10], off
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v9
+; GFX10-NEXT:    s_and_saveexec_b32 s1, vcc_lo
+; GFX10-NEXT:    s_cbranch_execz .LBB6_1
+; GFX10-NEXT:  ; %bb.3: ; %loop.body
+; GFX10-NEXT:    ; in Loop: Header=BB6_2 Depth=1
+; GFX10-NEXT:    v_add_co_u32 v7, vcc_lo, v0, v7
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v8, vcc_lo, v1, v8, vcc_lo
+; GFX10-NEXT:    v_add_nc_u32_e32 v10, 1, v6
+; GFX10-NEXT:    v_cmp_gt_u32_e64 s2, 0x64, v6
+; GFX10-NEXT:    s_mov_b32 s3, 0
+; GFX10-NEXT:    global_load_dword v9, v[7:8], off
+; GFX10-NEXT:    v_mov_b32_e32 v6, v10
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_add_nc_u32_e32 v9, 1, v9
+; GFX10-NEXT:    global_store_dword v[7:8], v9, off
+; GFX10-NEXT:    s_branch .LBB6_1
+; GFX10-NEXT:  .LBB6_4: ; %loop.exit.guard
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX10-NEXT:    s_and_saveexec_b32 s0, s1
+; GFX10-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX10-NEXT:    s_cbranch_execz .LBB6_6
+; GFX10-NEXT:  ; %bb.5: ; %break.body
+; GFX10-NEXT:    v_mov_b32_e32 v0, 10
+; GFX10-NEXT:    global_store_dword v[4:5], v0, off
+; GFX10-NEXT:  .LBB6_6: ; %exit
+; GFX10-NEXT:    s_endpgm
+entry:
+  br label %A
+
+A:
+  %counter = phi i32 [ %counter.plus.1, %loop.body ], [ 0, %entry ]
+  %a.plus.counter = getelementptr inbounds i32, ptr addrspace(1) %a, i32 %counter
+  %a.val = load i32, ptr addrspace(1) %a.plus.counter
+  %a.cond = icmp eq i32 %a.val, 0
+  br i1 %a.cond, label %break.body, label %loop.body
+
+break.body:
+  store i32 10, ptr addrspace(1) %a.break
+  br label %exit
+
+loop.body:
+  %x.plus.counter = getelementptr inbounds i32, ptr addrspace(1) %x, i32 %counter
+  %x.val = load i32, ptr addrspace(1) %x.plus.counter
+  %x.val.plus.1 = add i32 %x.val, 1
+  store i32 %x.val.plus.1, ptr addrspace(1) %x.plus.counter
+  %counter.plus.1 = add i32 %counter, 1
+  %x.cond = icmp ult i32 %counter, 100
+  br i1 %x.cond, label %exit, label %A
+
+exit:
+  ret void
+}
+
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-used-outside-loop.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-used-outside-loop.mir
new file mode 100644
index 0000000000000000000000000000000000000000..92463714ec69491495eaffd1695f9328be5c4d0f
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-divergent-i1-used-outside-loop.mir
@@ -0,0 +1,914 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 4
+# RUN: llc -global-isel -mtriple=amdgcn-mesa-amdpal -mcpu=gfx1010 -run-pass=amdgpu-global-isel-divergence-lowering %s -o - | FileCheck -check-prefix=GFX10 %s
+
+---
+name: divergent_i1_phi_used_outside_loop
+legalized: true
+tracksRegLiveness: true
+body: |
+  ; GFX10-LABEL: name: divergent_i1_phi_used_outside_loop
+  ; GFX10: bb.0:
+  ; GFX10-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX10-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GFX10-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GFX10-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; GFX10-NEXT:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; GFX10-NEXT:   [[MV:%[0-9]+]]:_(p0) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32)
+  ; GFX10-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.000000e+00
+  ; GFX10-NEXT:   [[FCMP:%[0-9]+]]:_(s1) = G_FCMP floatpred(ogt), [[COPY1]](s32), [[C1]]
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.1:
+  ; GFX10-NEXT:   successors: %bb.2(0x04000000), %bb.1(0x7c000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:_(s32) = G_PHI %9(s32), %bb.1, [[C]](s32), %bb.0
+  ; GFX10-NEXT:   [[PHI1:%[0-9]+]]:_(s32) = G_PHI [[C]](s32), %bb.0, %11(s32), %bb.1
+  ; GFX10-NEXT:   [[PHI2:%[0-9]+]]:_(s1) = G_PHI [[FCMP]](s1), %bb.0, %13(s1), %bb.1
+  ; GFX10-NEXT:   [[C2:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
+  ; GFX10-NEXT:   [[XOR:%[0-9]+]]:_(s1) = G_XOR [[PHI2]], [[C2]]
+  ; GFX10-NEXT:   [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[PHI1]](s32)
+  ; GFX10-NEXT:   [[FCMP1:%[0-9]+]]:_(s1) = G_FCMP floatpred(ogt), [[UITOFP]](s32), [[COPY]]
+  ; GFX10-NEXT:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; GFX10-NEXT:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[PHI1]], [[C3]]
+  ; GFX10-NEXT:   [[INTRINSIC_CONVERGENT:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.if.break), [[FCMP1]](s1), [[PHI]](s32)
+  ; GFX10-NEXT:   SI_LOOP [[INTRINSIC_CONVERGENT]](s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX10-NEXT:   G_BR %bb.2
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.2:
+  ; GFX10-NEXT:   [[PHI3:%[0-9]+]]:_(s1) = G_PHI [[PHI2]](s1), %bb.1
+  ; GFX10-NEXT:   [[PHI4:%[0-9]+]]:_(s32) = G_PHI [[INTRINSIC_CONVERGENT]](s32), %bb.1
+  ; GFX10-NEXT:   G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[PHI4]](s32)
+  ; GFX10-NEXT:   [[C4:%[0-9]+]]:_(s32) = G_FCONSTANT float 0.000000e+00
+  ; GFX10-NEXT:   [[C5:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.000000e+00
+  ; GFX10-NEXT:   [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[PHI3]](s1), [[C5]], [[C4]]
+  ; GFX10-NEXT:   G_STORE [[SELECT]](s32), [[MV]](p0) :: (store (s32))
+  ; GFX10-NEXT:   SI_RETURN
+  bb.0:
+    successors: %bb.1(0x80000000)
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $vgpr2
+    %3:_(s32) = COPY $vgpr3
+    %4:_(p0) = G_MERGE_VALUES %2(s32), %3(s32)
+    %5:_(s32) = G_CONSTANT i32 0
+    %6:_(s32) = G_FCONSTANT float 1.000000e+00
+    %7:_(s1) = G_FCMP floatpred(ogt), %1(s32), %6
+
+  bb.1:
+    successors: %bb.2(0x04000000), %bb.1(0x7c000000)
+
+    %8:_(s32) = G_PHI %9(s32), %bb.1, %5(s32), %bb.0
+    %10:_(s32) = G_PHI %5(s32), %bb.0, %11(s32), %bb.1
+    %12:_(s1) = G_PHI %7(s1), %bb.0, %13(s1), %bb.1
+    %14:_(s1) = G_CONSTANT i1 true
+    %13:_(s1) = G_XOR %12, %14
+    %15:_(s32) = G_UITOFP %10(s32)
+    %16:_(s1) = G_FCMP floatpred(ogt), %15(s32), %0
+    %17:_(s32) = G_CONSTANT i32 1
+    %11:_(s32) = G_ADD %10, %17
+    %9:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.if.break), %16(s1), %8(s32)
+    SI_LOOP %9(s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.2
+
+  bb.2:
+    %18:_(s1) = G_PHI %12(s1), %bb.1
+    %19:_(s32) = G_PHI %9(s32), %bb.1
+    G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %19(s32)
+    %20:_(s32) = G_FCONSTANT float 0.000000e+00
+    %21:_(s32) = G_FCONSTANT float 1.000000e+00
+    %22:_(s32) = G_SELECT %18(s1), %21, %20
+    G_STORE %22(s32), %4(p0) :: (store (s32))
+    SI_RETURN
+...
+
+---
+name: divergent_i1_phi_used_outside_loop_larger_loop_body
+legalized: true
+tracksRegLiveness: true
+body: |
+  ; GFX10-LABEL: name: divergent_i1_phi_used_outside_loop_larger_loop_body
+  ; GFX10: bb.0:
+  ; GFX10-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX10-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GFX10-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; GFX10-NEXT:   [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY]](s32), [[COPY1]](s32)
+  ; GFX10-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; GFX10-NEXT:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; GFX10-NEXT:   [[MV1:%[0-9]+]]:_(p0) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32)
+  ; GFX10-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+  ; GFX10-NEXT:   [[C1:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.1:
+  ; GFX10-NEXT:   successors: %bb.2(0x40000000), %bb.3(0x40000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:_(s32) = G_PHI [[C]](s32), %bb.0, %9(s32), %bb.3
+  ; GFX10-NEXT:   [[PHI1:%[0-9]+]]:_(p1) = G_PHI [[MV]](p1), %bb.0, %11(p1), %bb.3
+  ; GFX10-NEXT:   [[PHI2:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_PHI [[C1]](s1), %bb.0, %13(s1), %bb.3
+  ; GFX10-NEXT:   [[SI_IF:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[PHI2]](s1), %bb.3, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX10-NEXT:   G_BR %bb.2
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.2:
+  ; GFX10-NEXT:   successors: %bb.3(0x80000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PHI1]](p1) :: (load (s32), addrspace 1)
+  ; GFX10-NEXT:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT:   [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[LOAD]](s32), [[C2]]
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.3:
+  ; GFX10-NEXT:   successors: %bb.4(0x04000000), %bb.1(0x7c000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[PHI3:%[0-9]+]]:_(s1) = G_PHI [[ICMP]](s1), %bb.2, [[PHI2]](s1), %bb.1
+  ; GFX10-NEXT:   G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[SI_IF]](s32)
+  ; GFX10-NEXT:   [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 4
+  ; GFX10-NEXT:   [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[PHI1]], [[C3]](s64)
+  ; GFX10-NEXT:   [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; GFX10-NEXT:   [[ADD:%[0-9]+]]:_(s32) = nsw G_ADD [[PHI]], [[C4]]
+  ; GFX10-NEXT:   [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; GFX10-NEXT:   [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(sge), [[ADD]](s32), [[C5]]
+  ; GFX10-NEXT:   G_BRCOND [[ICMP1]](s1), %bb.1
+  ; GFX10-NEXT:   G_BR %bb.4
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.4:
+  ; GFX10-NEXT:   [[PHI4:%[0-9]+]]:_(s1) = G_PHI [[PHI2]](s1), %bb.3
+  ; GFX10-NEXT:   [[C6:%[0-9]+]]:_(s32) = G_FCONSTANT float 0.000000e+00
+  ; GFX10-NEXT:   [[C7:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.000000e+00
+  ; GFX10-NEXT:   [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[PHI4]](s1), [[C7]], [[C6]]
+  ; GFX10-NEXT:   G_STORE [[SELECT]](s32), [[MV1]](p0) :: (store (s32))
+  ; GFX10-NEXT:   SI_RETURN
+  bb.0:
+    successors: %bb.1(0x80000000)
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+
+    %0:_(s32) = COPY $vgpr1
+    %1:_(s32) = COPY $vgpr2
+    %2:_(p1) = G_MERGE_VALUES %0(s32), %1(s32)
+    %3:_(s32) = COPY $vgpr3
+    %4:_(s32) = COPY $vgpr4
+    %5:_(p0) = G_MERGE_VALUES %3(s32), %4(s32)
+    %6:_(s32) = G_CONSTANT i32 -1
+    %7:_(s1) = G_CONSTANT i1 true
+
+  bb.1:
+    successors: %bb.2(0x40000000), %bb.3(0x40000000)
+
+    %8:_(s32) = G_PHI %6(s32), %bb.0, %9(s32), %bb.3
+    %10:_(p1) = G_PHI %2(p1), %bb.0, %11(p1), %bb.3
+    %12:sreg_32_xm0_xexec(s1) = G_PHI %7(s1), %bb.0, %13(s1), %bb.3
+    %14:sreg_32_xm0_xexec(s32) = SI_IF %12(s1), %bb.3, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.2
+
+  bb.2:
+    successors: %bb.3(0x80000000)
+
+    %15:_(s32) = G_LOAD %10(p1) :: (load (s32), addrspace 1)
+    %16:_(s32) = G_CONSTANT i32 0
+    %17:_(s1) = G_ICMP intpred(eq), %15(s32), %16
+
+  bb.3:
+    successors: %bb.4(0x04000000), %bb.1(0x7c000000)
+
+    %13:_(s1) = G_PHI %17(s1), %bb.2, %12(s1), %bb.1
+    G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %14(s32)
+    %18:_(s64) = G_CONSTANT i64 4
+    %11:_(p1) = G_PTR_ADD %10, %18(s64)
+    %19:_(s32) = G_CONSTANT i32 1
+    %9:_(s32) = nsw G_ADD %8, %19
+    %20:_(s32) = G_CONSTANT i32 10
+    %21:_(s1) = G_ICMP intpred(sge), %9(s32), %20
+    G_BRCOND %21(s1), %bb.1
+    G_BR %bb.4
+
+  bb.4:
+    %22:_(s1) = G_PHI %12(s1), %bb.3
+    %23:_(s32) = G_FCONSTANT float 0.000000e+00
+    %24:_(s32) = G_FCONSTANT float 1.000000e+00
+    %25:_(s32) = G_SELECT %22(s1), %24, %23
+    G_STORE %25(s32), %5(p0) :: (store (s32))
+    SI_RETURN
+...
+
+---
+name: divergent_i1_xor_used_outside_loop
+legalized: true
+tracksRegLiveness: true
+body: |
+  ; GFX10-LABEL: name: divergent_i1_xor_used_outside_loop
+  ; GFX10: bb.0:
+  ; GFX10-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX10-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GFX10-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GFX10-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; GFX10-NEXT:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; GFX10-NEXT:   [[MV:%[0-9]+]]:_(p0) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32)
+  ; GFX10-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.000000e+00
+  ; GFX10-NEXT:   [[FCMP:%[0-9]+]]:_(s1) = G_FCMP floatpred(ogt), [[COPY1]](s32), [[C1]]
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.1:
+  ; GFX10-NEXT:   successors: %bb.2(0x04000000), %bb.1(0x7c000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:_(s32) = G_PHI %9(s32), %bb.1, [[C]](s32), %bb.0
+  ; GFX10-NEXT:   [[PHI1:%[0-9]+]]:_(s32) = G_PHI [[C]](s32), %bb.0, %11(s32), %bb.1
+  ; GFX10-NEXT:   [[PHI2:%[0-9]+]]:_(s1) = G_PHI [[FCMP]](s1), %bb.0, %13(s1), %bb.1
+  ; GFX10-NEXT:   [[C2:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
+  ; GFX10-NEXT:   [[XOR:%[0-9]+]]:_(s1) = G_XOR [[PHI2]], [[C2]]
+  ; GFX10-NEXT:   [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[PHI1]](s32)
+  ; GFX10-NEXT:   [[FCMP1:%[0-9]+]]:_(s1) = G_FCMP floatpred(ogt), [[UITOFP]](s32), [[COPY]]
+  ; GFX10-NEXT:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; GFX10-NEXT:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[PHI1]], [[C3]]
+  ; GFX10-NEXT:   [[INTRINSIC_CONVERGENT:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.if.break), [[FCMP1]](s1), [[PHI]](s32)
+  ; GFX10-NEXT:   SI_LOOP [[INTRINSIC_CONVERGENT]](s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX10-NEXT:   G_BR %bb.2
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.2:
+  ; GFX10-NEXT:   [[PHI3:%[0-9]+]]:_(s1) = G_PHI [[XOR]](s1), %bb.1
+  ; GFX10-NEXT:   [[PHI4:%[0-9]+]]:_(s32) = G_PHI [[INTRINSIC_CONVERGENT]](s32), %bb.1
+  ; GFX10-NEXT:   G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[PHI4]](s32)
+  ; GFX10-NEXT:   [[C4:%[0-9]+]]:_(s32) = G_FCONSTANT float 0.000000e+00
+  ; GFX10-NEXT:   [[C5:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.000000e+00
+  ; GFX10-NEXT:   [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[PHI3]](s1), [[C5]], [[C4]]
+  ; GFX10-NEXT:   G_STORE [[SELECT]](s32), [[MV]](p0) :: (store (s32))
+  ; GFX10-NEXT:   SI_RETURN
+  bb.0:
+    successors: %bb.1(0x80000000)
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $vgpr2
+    %3:_(s32) = COPY $vgpr3
+    %4:_(p0) = G_MERGE_VALUES %2(s32), %3(s32)
+    %5:_(s32) = G_CONSTANT i32 0
+    %6:_(s32) = G_FCONSTANT float 1.000000e+00
+    %7:_(s1) = G_FCMP floatpred(ogt), %1(s32), %6
+
+  bb.1:
+    successors: %bb.2(0x04000000), %bb.1(0x7c000000)
+
+    %8:_(s32) = G_PHI %9(s32), %bb.1, %5(s32), %bb.0
+    %10:_(s32) = G_PHI %5(s32), %bb.0, %11(s32), %bb.1
+    %12:_(s1) = G_PHI %7(s1), %bb.0, %13(s1), %bb.1
+    %14:_(s1) = G_CONSTANT i1 true
+    %13:_(s1) = G_XOR %12, %14
+    %15:_(s32) = G_UITOFP %10(s32)
+    %16:_(s1) = G_FCMP floatpred(ogt), %15(s32), %0
+    %17:_(s32) = G_CONSTANT i32 1
+    %11:_(s32) = G_ADD %10, %17
+    %9:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.if.break), %16(s1), %8(s32)
+    SI_LOOP %9(s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.2
+
+  bb.2:
+    %18:_(s1) = G_PHI %13(s1), %bb.1
+    %19:_(s32) = G_PHI %9(s32), %bb.1
+    G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %19(s32)
+    %20:_(s32) = G_FCONSTANT float 0.000000e+00
+    %21:_(s32) = G_FCONSTANT float 1.000000e+00
+    %22:_(s32) = G_SELECT %18(s1), %21, %20
+    G_STORE %22(s32), %4(p0) :: (store (s32))
+    SI_RETURN
+...
+
+---
+name: divergent_i1_xor_used_outside_loop_larger_loop_body
+legalized: true
+tracksRegLiveness: true
+body: |
+  ; GFX10-LABEL: name: divergent_i1_xor_used_outside_loop_larger_loop_body
+  ; GFX10: bb.0:
+  ; GFX10-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; GFX10-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GFX10-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GFX10-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; GFX10-NEXT:   [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY1]](s32), [[COPY2]](s32)
+  ; GFX10-NEXT:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; GFX10-NEXT:   [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; GFX10-NEXT:   [[MV1:%[0-9]+]]:_(p0) = G_MERGE_VALUES [[COPY3]](s32), [[COPY4]](s32)
+  ; GFX10-NEXT:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+  ; GFX10-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT:   [[ICMP:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_ICMP intpred(eq), [[COPY]](s32), [[C]]
+  ; GFX10-NEXT:   [[C1:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
+  ; GFX10-NEXT:   [[SI_IF:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[ICMP]](s1), %bb.2, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX10-NEXT:   G_BR %bb.1
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.1:
+  ; GFX10-NEXT:   successors: %bb.3(0x80000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT:   G_BR %bb.3
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.2:
+  ; GFX10-NEXT:   successors: %bb.5(0x40000000), %bb.6(0x40000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_PHI %14(s1), %bb.8, [[C1]](s1), %bb.0
+  ; GFX10-NEXT:   G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[SI_IF]](s32)
+  ; GFX10-NEXT:   [[SI_IF1:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[PHI]](s1), %bb.6, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX10-NEXT:   G_BR %bb.5
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.3:
+  ; GFX10-NEXT:   successors: %bb.4(0x40000000), %bb.7(0x40000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[PHI1:%[0-9]+]]:_(s32) = G_PHI [[C2]](s32), %bb.1, %17(s32), %bb.7
+  ; GFX10-NEXT:   [[PHI2:%[0-9]+]]:_(s32) = G_PHI %19(s32), %bb.7, [[C2]](s32), %bb.1
+  ; GFX10-NEXT:   [[C3:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
+  ; GFX10-NEXT:   [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[PHI2]](s32)
+  ; GFX10-NEXT:   [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; GFX10-NEXT:   [[SHL:%[0-9]+]]:_(s64) = G_SHL [[SEXT]], [[C4]](s32)
+  ; GFX10-NEXT:   [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[MV]], [[SHL]](s64)
+  ; GFX10-NEXT:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p1) :: (load (s32), addrspace 1)
+  ; GFX10-NEXT:   [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT:   [[ICMP1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), [[LOAD]](s32), [[C5]]
+  ; GFX10-NEXT:   [[SI_IF2:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[ICMP1]](s1), %bb.7, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX10-NEXT:   G_BR %bb.4
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.4:
+  ; GFX10-NEXT:   successors: %bb.7(0x80000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[C6:%[0-9]+]]:_(s1) = G_CONSTANT i1 false
+  ; GFX10-NEXT:   [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; GFX10-NEXT:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[PHI2]], [[C7]]
+  ; GFX10-NEXT:   [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[PHI2]](s32), [[COPY]]
+  ; GFX10-NEXT:   G_BR %bb.7
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.5:
+  ; GFX10-NEXT:   successors: %bb.6(0x80000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 5
+  ; GFX10-NEXT:   G_STORE [[C8]](s32), [[MV1]](p0) :: (store (s32))
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.6:
+  ; GFX10-NEXT:   G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[SI_IF1]](s32)
+  ; GFX10-NEXT:   SI_RETURN
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.7:
+  ; GFX10-NEXT:   successors: %bb.8(0x04000000), %bb.3(0x7c000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[PHI3:%[0-9]+]]:_(s32) = G_PHI [[ADD]](s32), %bb.4, [[DEF]](s32), %bb.3
+  ; GFX10-NEXT:   [[PHI4:%[0-9]+]]:_(s1) = G_PHI [[C6]](s1), %bb.4, [[C3]](s1), %bb.3
+  ; GFX10-NEXT:   [[PHI5:%[0-9]+]]:_(s1) = G_PHI [[ICMP2]](s1), %bb.4, [[C3]](s1), %bb.3
+  ; GFX10-NEXT:   G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[SI_IF2]](s32)
+  ; GFX10-NEXT:   [[C9:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
+  ; GFX10-NEXT:   [[XOR:%[0-9]+]]:_(s1) = G_XOR [[PHI4]], [[C9]]
+  ; GFX10-NEXT:   [[INTRINSIC_CONVERGENT:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.if.break), [[PHI5]](s1), [[PHI1]](s32)
+  ; GFX10-NEXT:   SI_LOOP [[INTRINSIC_CONVERGENT]](s32), %bb.3, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX10-NEXT:   G_BR %bb.8
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.8:
+  ; GFX10-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[PHI6:%[0-9]+]]:_(s1) = G_PHI [[XOR]](s1), %bb.7
+  ; GFX10-NEXT:   [[PHI7:%[0-9]+]]:_(s32) = G_PHI [[INTRINSIC_CONVERGENT]](s32), %bb.7
+  ; GFX10-NEXT:   G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[PHI7]](s32)
+  ; GFX10-NEXT:   G_BR %bb.2
+  bb.0:
+    successors: %bb.1(0x40000000), %bb.2(0x40000000)
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $vgpr2
+    %3:_(p1) = G_MERGE_VALUES %1(s32), %2(s32)
+    %4:_(s32) = COPY $vgpr3
+    %5:_(s32) = COPY $vgpr4
+    %6:_(p0) = G_MERGE_VALUES %4(s32), %5(s32)
+    %7:_(s32) = G_IMPLICIT_DEF
+    %8:_(s32) = G_CONSTANT i32 0
+    %9:sreg_32_xm0_xexec(s1) = G_ICMP intpred(eq), %0(s32), %8
+    %10:_(s1) = G_CONSTANT i1 true
+    %11:sreg_32_xm0_xexec(s32) = SI_IF %9(s1), %bb.2, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.1
+
+  bb.1:
+    successors: %bb.3(0x80000000)
+
+    %12:_(s32) = G_CONSTANT i32 0
+    G_BR %bb.3
+
+  bb.2:
+    successors: %bb.5(0x40000000), %bb.6(0x40000000)
+
+    %13:sreg_32_xm0_xexec(s1) = G_PHI %14(s1), %bb.8, %10(s1), %bb.0
+    G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %11(s32)
+    %15:sreg_32_xm0_xexec(s32) = SI_IF %13(s1), %bb.6, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.5
+
+  bb.3:
+    successors: %bb.4(0x40000000), %bb.7(0x40000000)
+
+    %16:_(s32) = G_PHI %12(s32), %bb.1, %17(s32), %bb.7
+    %18:_(s32) = G_PHI %19(s32), %bb.7, %12(s32), %bb.1
+    %20:_(s1) = G_CONSTANT i1 true
+    %21:_(s64) = G_SEXT %18(s32)
+    %22:_(s32) = G_CONSTANT i32 2
+    %23:_(s64) = G_SHL %21, %22(s32)
+    %24:_(p1) = G_PTR_ADD %3, %23(s64)
+    %25:_(s32) = G_LOAD %24(p1) :: (load (s32), addrspace 1)
+    %26:_(s32) = G_CONSTANT i32 0
+    %27:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), %25(s32), %26
+    %28:sreg_32_xm0_xexec(s32) = SI_IF %27(s1), %bb.7, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.4
+
+  bb.4:
+    successors: %bb.7(0x80000000)
+
+    %29:_(s1) = G_CONSTANT i1 false
+    %30:_(s32) = G_CONSTANT i32 1
+    %31:_(s32) = G_ADD %18, %30
+    %32:_(s1) = G_ICMP intpred(slt), %18(s32), %0
+    G_BR %bb.7
+
+  bb.5:
+    successors: %bb.6(0x80000000)
+
+    %33:_(s32) = G_CONSTANT i32 5
+    G_STORE %33(s32), %6(p0) :: (store (s32))
+
+  bb.6:
+    G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %15(s32)
+    SI_RETURN
+
+  bb.7:
+    successors: %bb.8(0x04000000), %bb.3(0x7c000000)
+
+    %19:_(s32) = G_PHI %31(s32), %bb.4, %7(s32), %bb.3
+    %34:_(s1) = G_PHI %29(s1), %bb.4, %20(s1), %bb.3
+    %35:_(s1) = G_PHI %32(s1), %bb.4, %20(s1), %bb.3
+    G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %28(s32)
+    %36:_(s1) = G_CONSTANT i1 true
+    %37:_(s1) = G_XOR %34, %36
+    %17:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.if.break), %35(s1), %16(s32)
+    SI_LOOP %17(s32), %bb.3, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.8
+
+  bb.8:
+    successors: %bb.2(0x80000000)
+
+    %14:_(s1) = G_PHI %37(s1), %bb.7
+    %38:_(s32) = G_PHI %17(s32), %bb.7
+    G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %38(s32)
+    G_BR %bb.2
+...
+
+---
+name: divergent_i1_icmp_used_outside_loop
+legalized: true
+tracksRegLiveness: true
+body: |
+  ; GFX10-LABEL: name: divergent_i1_icmp_used_outside_loop
+  ; GFX10: bb.0:
+  ; GFX10-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX10-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GFX10-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GFX10-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; GFX10-NEXT:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; GFX10-NEXT:   [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32)
+  ; GFX10-NEXT:   [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr6
+  ; GFX10-NEXT:   [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr7
+  ; GFX10-NEXT:   [[MV1:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY4]](s32), [[COPY5]](s32)
+  ; GFX10-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.1:
+  ; GFX10-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:_(s32) = G_PHI %11(s32), %bb.6, [[C]](s32), %bb.0
+  ; GFX10-NEXT:   [[PHI1:%[0-9]+]]:_(s32) = G_PHI [[C]](s32), %bb.0, %13(s32), %bb.6
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.2:
+  ; GFX10-NEXT:   successors: %bb.3(0x40000000), %bb.4(0x40000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[ICMP:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_ICMP intpred(eq), [[COPY]](s32), [[PHI1]]
+  ; GFX10-NEXT:   [[SI_IF:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[ICMP]](s1), %bb.4, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX10-NEXT:   G_BR %bb.3
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.3:
+  ; GFX10-NEXT:   successors: %bb.4(0x80000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[PHI1]](s32)
+  ; GFX10-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; GFX10-NEXT:   [[SHL:%[0-9]+]]:_(s64) = G_SHL [[SEXT]], [[C1]](s32)
+  ; GFX10-NEXT:   [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[MV]], [[SHL]](s64)
+  ; GFX10-NEXT:   G_STORE [[PHI1]](s32), [[PTR_ADD]](p1) :: (store (s32), addrspace 1)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.4:
+  ; GFX10-NEXT:   successors: %bb.5(0x40000000), %bb.6(0x40000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[C2:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
+  ; GFX10-NEXT:   G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[SI_IF]](s32)
+  ; GFX10-NEXT:   [[ICMP1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), [[COPY1]](s32), [[PHI1]]
+  ; GFX10-NEXT:   [[SI_IF1:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[ICMP1]](s1), %bb.6, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX10-NEXT:   G_BR %bb.5
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.5:
+  ; GFX10-NEXT:   successors: %bb.6(0x80000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[C3:%[0-9]+]]:_(s1) = G_CONSTANT i1 false
+  ; GFX10-NEXT:   [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; GFX10-NEXT:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[PHI1]], [[C4]]
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.6:
+  ; GFX10-NEXT:   successors: %bb.7(0x04000000), %bb.1(0x7c000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[PHI2:%[0-9]+]]:_(s32) = G_PHI [[ADD]](s32), %bb.5, [[DEF]](s32), %bb.4
+  ; GFX10-NEXT:   [[PHI3:%[0-9]+]]:_(s1) = G_PHI [[C3]](s1), %bb.5, [[C2]](s1), %bb.4
+  ; GFX10-NEXT:   G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[SI_IF1]](s32)
+  ; GFX10-NEXT:   [[INTRINSIC_CONVERGENT:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.if.break), [[PHI3]](s1), [[PHI]](s32)
+  ; GFX10-NEXT:   SI_LOOP [[INTRINSIC_CONVERGENT]](s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX10-NEXT:   G_BR %bb.7
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.7:
+  ; GFX10-NEXT:   successors: %bb.8(0x40000000), %bb.9(0x40000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[PHI4:%[0-9]+]]:_(s32) = G_PHI [[INTRINSIC_CONVERGENT]](s32), %bb.6
+  ; GFX10-NEXT:   [[PHI5:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_PHI [[ICMP]](s1), %bb.6
+  ; GFX10-NEXT:   [[PHI6:%[0-9]+]]:_(s32) = G_PHI [[PHI1]](s32), %bb.6
+  ; GFX10-NEXT:   G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[PHI4]](s32)
+  ; GFX10-NEXT:   [[SI_IF2:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[PHI5]](s1), %bb.9, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX10-NEXT:   G_BR %bb.8
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.8:
+  ; GFX10-NEXT:   successors: %bb.9(0x80000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   G_STORE [[PHI6]](s32), [[MV1]](p1) :: (store (s32), addrspace 1)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.9:
+  ; GFX10-NEXT:   G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[SI_IF2]](s32)
+  ; GFX10-NEXT:   SI_RETURN
+  bb.0:
+    successors: %bb.1(0x80000000)
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7, $vgpr8, $vgpr9
+
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $vgpr2
+    %3:_(s32) = COPY $vgpr3
+    %4:_(p1) = G_MERGE_VALUES %2(s32), %3(s32)
+    %5:_(s32) = COPY $vgpr6
+    %6:_(s32) = COPY $vgpr7
+    %7:_(p1) = G_MERGE_VALUES %5(s32), %6(s32)
+    %8:_(s32) = G_CONSTANT i32 0
+    %9:_(s32) = G_IMPLICIT_DEF
+
+  bb.1:
+    successors: %bb.2(0x80000000)
+
+    %10:_(s32) = G_PHI %11(s32), %bb.6, %8(s32), %bb.0
+    %12:_(s32) = G_PHI %8(s32), %bb.0, %13(s32), %bb.6
+
+  bb.2:
+    successors: %bb.3(0x40000000), %bb.4(0x40000000)
+
+    %14:sreg_32_xm0_xexec(s1) = G_ICMP intpred(eq), %0(s32), %12
+    %15:sreg_32_xm0_xexec(s32) = SI_IF %14(s1), %bb.4, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.3
+
+  bb.3:
+    successors: %bb.4(0x80000000)
+
+    %16:_(s64) = G_SEXT %12(s32)
+    %17:_(s32) = G_CONSTANT i32 2
+    %18:_(s64) = G_SHL %16, %17(s32)
+    %19:_(p1) = G_PTR_ADD %4, %18(s64)
+    G_STORE %12(s32), %19(p1) :: (store (s32), addrspace 1)
+
+  bb.4:
+    successors: %bb.5(0x40000000), %bb.6(0x40000000)
+
+    %20:_(s1) = G_CONSTANT i1 true
+    G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %15(s32)
+    %21:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), %1(s32), %12
+    %22:sreg_32_xm0_xexec(s32) = SI_IF %21(s1), %bb.6, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.5
+
+  bb.5:
+    successors: %bb.6(0x80000000)
+
+    %23:_(s1) = G_CONSTANT i1 false
+    %24:_(s32) = G_CONSTANT i32 1
+    %25:_(s32) = G_ADD %12, %24
+
+  bb.6:
+    successors: %bb.7(0x04000000), %bb.1(0x7c000000)
+
+    %13:_(s32) = G_PHI %25(s32), %bb.5, %9(s32), %bb.4
+    %26:_(s1) = G_PHI %23(s1), %bb.5, %20(s1), %bb.4
+    G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %22(s32)
+    %11:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.if.break), %26(s1), %10(s32)
+    SI_LOOP %11(s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.7
+
+  bb.7:
+    successors: %bb.8(0x40000000), %bb.9(0x40000000)
+
+    %27:_(s32) = G_PHI %11(s32), %bb.6
+    %28:sreg_32_xm0_xexec(s1) = G_PHI %14(s1), %bb.6
+    %29:_(s32) = G_PHI %12(s32), %bb.6
+    G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %27(s32)
+    %30:sreg_32_xm0_xexec(s32) = SI_IF %28(s1), %bb.9, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.8
+
+  bb.8:
+    successors: %bb.9(0x80000000)
+
+    G_STORE %29(s32), %7(p1) :: (store (s32), addrspace 1)
+
+  bb.9:
+    G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %30(s32)
+    SI_RETURN
+...
+
+---
+name: divergent_i1_freeze_used_outside_loop
+legalized: true
+tracksRegLiveness: true
+body: |
+  ; GFX10-LABEL: name: divergent_i1_freeze_used_outside_loop
+  ; GFX10: bb.0:
+  ; GFX10-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX10-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GFX10-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GFX10-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; GFX10-NEXT:   [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY1]](s32), [[COPY2]](s32)
+  ; GFX10-NEXT:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; GFX10-NEXT:   [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; GFX10-NEXT:   [[MV1:%[0-9]+]]:_(p0) = G_MERGE_VALUES [[COPY3]](s32), [[COPY4]](s32)
+  ; GFX10-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT:   [[C1:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.1:
+  ; GFX10-NEXT:   successors: %bb.2(0x40000000), %bb.3(0x40000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:_(s32) = G_PHI %10(s32), %bb.3, [[C]](s32), %bb.0
+  ; GFX10-NEXT:   [[PHI1:%[0-9]+]]:_(s32) = G_PHI [[C]](s32), %bb.0, %12(s32), %bb.3
+  ; GFX10-NEXT:   [[PHI2:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_PHI [[C1]](s1), %bb.0, %14(s1), %bb.3
+  ; GFX10-NEXT:   [[SI_IF:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[PHI2]](s1), %bb.3, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX10-NEXT:   G_BR %bb.2
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.2:
+  ; GFX10-NEXT:   successors: %bb.3(0x80000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[PHI1]](s32)
+  ; GFX10-NEXT:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; GFX10-NEXT:   [[SHL:%[0-9]+]]:_(s64) = G_SHL [[SEXT]], [[C2]](s32)
+  ; GFX10-NEXT:   [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[MV]], [[SHL]](s64)
+  ; GFX10-NEXT:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p1) :: (load (s32), addrspace 1)
+  ; GFX10-NEXT:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT:   [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(eq), [[LOAD]](s32), [[C3]]
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.3:
+  ; GFX10-NEXT:   successors: %bb.4(0x04000000), %bb.1(0x7c000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[PHI3:%[0-9]+]]:_(s1) = G_PHI [[ICMP]](s1), %bb.2, [[PHI2]](s1), %bb.1
+  ; GFX10-NEXT:   G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[SI_IF]](s32)
+  ; GFX10-NEXT:   [[FREEZE:%[0-9]+]]:_(s1) = G_FREEZE [[PHI3]]
+  ; GFX10-NEXT:   [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; GFX10-NEXT:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[PHI1]], [[C4]]
+  ; GFX10-NEXT:   [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(slt), [[PHI1]](s32), [[COPY]]
+  ; GFX10-NEXT:   [[INTRINSIC_CONVERGENT:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.if.break), [[ICMP1]](s1), [[PHI]](s32)
+  ; GFX10-NEXT:   SI_LOOP [[INTRINSIC_CONVERGENT]](s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX10-NEXT:   G_BR %bb.4
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.4:
+  ; GFX10-NEXT:   [[PHI4:%[0-9]+]]:_(s1) = G_PHI [[FREEZE]](s1), %bb.3
+  ; GFX10-NEXT:   [[PHI5:%[0-9]+]]:_(s32) = G_PHI [[INTRINSIC_CONVERGENT]](s32), %bb.3
+  ; GFX10-NEXT:   G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[PHI5]](s32)
+  ; GFX10-NEXT:   [[C5:%[0-9]+]]:_(s32) = G_FCONSTANT float 0.000000e+00
+  ; GFX10-NEXT:   [[C6:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.000000e+00
+  ; GFX10-NEXT:   [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[PHI4]](s1), [[C6]], [[C5]]
+  ; GFX10-NEXT:   G_STORE [[SELECT]](s32), [[MV1]](p0) :: (store (s32))
+  ; GFX10-NEXT:   S_ENDPGM 0
+  bb.0:
+    successors: %bb.1(0x80000000)
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4
+
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $vgpr2
+    %3:_(p1) = G_MERGE_VALUES %1(s32), %2(s32)
+    %4:_(s32) = COPY $vgpr3
+    %5:_(s32) = COPY $vgpr4
+    %6:_(p0) = G_MERGE_VALUES %4(s32), %5(s32)
+    %7:_(s32) = G_CONSTANT i32 0
+    %8:_(s1) = G_CONSTANT i1 true
+
+  bb.1:
+    successors: %bb.2(0x40000000), %bb.3(0x40000000)
+
+    %9:_(s32) = G_PHI %10(s32), %bb.3, %7(s32), %bb.0
+    %11:_(s32) = G_PHI %7(s32), %bb.0, %12(s32), %bb.3
+    %13:sreg_32_xm0_xexec(s1) = G_PHI %8(s1), %bb.0, %14(s1), %bb.3
+    %15:sreg_32_xm0_xexec(s32) = SI_IF %13(s1), %bb.3, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.2
+
+  bb.2:
+    successors: %bb.3(0x80000000)
+
+    %16:_(s64) = G_SEXT %11(s32)
+    %17:_(s32) = G_CONSTANT i32 2
+    %18:_(s64) = G_SHL %16, %17(s32)
+    %19:_(p1) = G_PTR_ADD %3, %18(s64)
+    %20:_(s32) = G_LOAD %19(p1) :: (load (s32), addrspace 1)
+    %21:_(s32) = G_CONSTANT i32 0
+    %22:_(s1) = G_ICMP intpred(eq), %20(s32), %21
+
+  bb.3:
+    successors: %bb.4(0x04000000), %bb.1(0x7c000000)
+
+    %23:_(s1) = G_PHI %22(s1), %bb.2, %13(s1), %bb.1
+    G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %15(s32)
+    %14:_(s1) = G_FREEZE %23
+    %24:_(s32) = G_CONSTANT i32 1
+    %12:_(s32) = G_ADD %11, %24
+    %25:_(s1) = G_ICMP intpred(slt), %11(s32), %0
+    %10:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.if.break), %25(s1), %9(s32)
+    SI_LOOP %10(s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.4
+
+  bb.4:
+    %26:_(s1) = G_PHI %14(s1), %bb.3
+    %27:_(s32) = G_PHI %10(s32), %bb.3
+    G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %27(s32)
+    %28:_(s32) = G_FCONSTANT float 0.000000e+00
+    %29:_(s32) = G_FCONSTANT float 1.000000e+00
+    %30:_(s32) = G_SELECT %26(s1), %29, %28
+    G_STORE %30(s32), %6(p0) :: (store (s32))
+    S_ENDPGM 0
+...
+
+---
+name: loop_with_1break
+legalized: true
+tracksRegLiveness: true
+body: |
+  ; GFX10-LABEL: name: loop_with_1break
+  ; GFX10: bb.0:
+  ; GFX10-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX10-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GFX10-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GFX10-NEXT:   [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY]](s32), [[COPY1]](s32)
+  ; GFX10-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; GFX10-NEXT:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; GFX10-NEXT:   [[MV1:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32)
+  ; GFX10-NEXT:   [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; GFX10-NEXT:   [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
+  ; GFX10-NEXT:   [[MV2:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY4]](s32), [[COPY5]](s32)
+  ; GFX10-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.1:
+  ; GFX10-NEXT:   successors: %bb.3(0x40000000), %bb.5(0x40000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:_(s32) = G_PHI %12(s32), %bb.5, [[C]](s32), %bb.0
+  ; GFX10-NEXT:   [[PHI1:%[0-9]+]]:_(s32) = G_PHI [[C]](s32), %bb.0, %14(s32), %bb.5
+  ; GFX10-NEXT:   [[C1:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
+  ; GFX10-NEXT:   [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[PHI1]](s32)
+  ; GFX10-NEXT:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; GFX10-NEXT:   [[SHL:%[0-9]+]]:_(s64) = G_SHL [[SEXT]], [[C2]](s32)
+  ; GFX10-NEXT:   [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[MV1]], [[SHL]](s64)
+  ; GFX10-NEXT:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p1) :: (load (s32), addrspace 1)
+  ; GFX10-NEXT:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT:   [[ICMP:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), [[LOAD]](s32), [[C3]]
+  ; GFX10-NEXT:   [[SI_IF:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[ICMP]](s1), %bb.5, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX10-NEXT:   G_BR %bb.3
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.2:
+  ; GFX10-NEXT:   successors: %bb.4(0x80000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; GFX10-NEXT:   G_STORE [[C4]](s32), [[MV2]](p1) :: (store (s32), addrspace 1)
+  ; GFX10-NEXT:   G_BR %bb.4
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.3:
+  ; GFX10-NEXT:   successors: %bb.5(0x80000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[C5:%[0-9]+]]:_(s1) = G_CONSTANT i1 false
+  ; GFX10-NEXT:   [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; GFX10-NEXT:   [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[SEXT]], [[C6]](s32)
+  ; GFX10-NEXT:   [[PTR_ADD1:%[0-9]+]]:_(p1) = G_PTR_ADD [[MV]], [[SHL1]](s64)
+  ; GFX10-NEXT:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p1) :: (load (s32), addrspace 1)
+  ; GFX10-NEXT:   [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; GFX10-NEXT:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[LOAD1]], [[C7]]
+  ; GFX10-NEXT:   G_STORE [[ADD]](s32), [[PTR_ADD1]](p1) :: (store (s32), addrspace 1)
+  ; GFX10-NEXT:   [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[PHI1]], [[C7]]
+  ; GFX10-NEXT:   [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 100
+  ; GFX10-NEXT:   [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[PHI1]](s32), [[C8]]
+  ; GFX10-NEXT:   G_BR %bb.5
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.4:
+  ; GFX10-NEXT:   G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %35(s32)
+  ; GFX10-NEXT:   S_ENDPGM 0
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.5:
+  ; GFX10-NEXT:   successors: %bb.6(0x04000000), %bb.1(0x7c000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[PHI2:%[0-9]+]]:_(s32) = G_PHI [[ADD1]](s32), %bb.3, [[DEF]](s32), %bb.1
+  ; GFX10-NEXT:   [[PHI3:%[0-9]+]]:_(s1) = G_PHI [[C5]](s1), %bb.3, [[C1]](s1), %bb.1
+  ; GFX10-NEXT:   [[PHI4:%[0-9]+]]:_(s1) = G_PHI [[ICMP1]](s1), %bb.3, [[C1]](s1), %bb.1
+  ; GFX10-NEXT:   G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[SI_IF]](s32)
+  ; GFX10-NEXT:   [[INTRINSIC_CONVERGENT:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.if.break), [[PHI4]](s1), [[PHI]](s32)
+  ; GFX10-NEXT:   SI_LOOP [[INTRINSIC_CONVERGENT]](s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX10-NEXT:   G_BR %bb.6
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.6:
+  ; GFX10-NEXT:   successors: %bb.2(0x40000000), %bb.4(0x40000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[PHI5:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_PHI [[PHI3]](s1), %bb.5
+  ; GFX10-NEXT:   [[PHI6:%[0-9]+]]:_(s32) = G_PHI [[INTRINSIC_CONVERGENT]](s32), %bb.5
+  ; GFX10-NEXT:   G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[PHI6]](s32)
+  ; GFX10-NEXT:   [[SI_IF1:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[PHI5]](s1), %bb.4, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX10-NEXT:   G_BR %bb.2
+  bb.0:
+    successors: %bb.1(0x80000000)
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(p1) = G_MERGE_VALUES %0(s32), %1(s32)
+    %3:_(s32) = COPY $vgpr2
+    %4:_(s32) = COPY $vgpr3
+    %5:_(p1) = G_MERGE_VALUES %3(s32), %4(s32)
+    %6:_(s32) = COPY $vgpr4
+    %7:_(s32) = COPY $vgpr5
+    %8:_(p1) = G_MERGE_VALUES %6(s32), %7(s32)
+    %9:_(s32) = G_CONSTANT i32 0
+    %10:_(s32) = G_IMPLICIT_DEF
+
+  bb.1:
+    successors: %bb.3(0x40000000), %bb.5(0x40000000)
+
+    %11:_(s32) = G_PHI %12(s32), %bb.5, %9(s32), %bb.0
+    %13:_(s32) = G_PHI %9(s32), %bb.0, %14(s32), %bb.5
+    %15:_(s1) = G_CONSTANT i1 true
+    %16:_(s64) = G_SEXT %13(s32)
+    %17:_(s32) = G_CONSTANT i32 2
+    %18:_(s64) = G_SHL %16, %17(s32)
+    %19:_(p1) = G_PTR_ADD %5, %18(s64)
+    %20:_(s32) = G_LOAD %19(p1) :: (load (s32), addrspace 1)
+    %21:_(s32) = G_CONSTANT i32 0
+    %22:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), %20(s32), %21
+    %23:sreg_32_xm0_xexec(s32) = SI_IF %22(s1), %bb.5, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.3
+
+  bb.2:
+    successors: %bb.4(0x80000000)
+
+    %24:_(s32) = G_CONSTANT i32 10
+    G_STORE %24(s32), %8(p1) :: (store (s32), addrspace 1)
+    G_BR %bb.4
+
+  bb.3:
+    successors: %bb.5(0x80000000)
+
+    %25:_(s1) = G_CONSTANT i1 false
+    %26:_(s32) = G_CONSTANT i32 2
+    %27:_(s64) = G_SHL %16, %26(s32)
+    %28:_(p1) = G_PTR_ADD %2, %27(s64)
+    %29:_(s32) = G_LOAD %28(p1) :: (load (s32), addrspace 1)
+    %30:_(s32) = G_CONSTANT i32 1
+    %31:_(s32) = G_ADD %29, %30
+    G_STORE %31(s32), %28(p1) :: (store (s32), addrspace 1)
+    %32:_(s32) = G_ADD %13, %30
+    %33:_(s32) = G_CONSTANT i32 100
+    %34:_(s1) = G_ICMP intpred(ult), %13(s32), %33
+    G_BR %bb.5
+
+  bb.4:
+    G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %35(s32)
+    S_ENDPGM 0
+
+  bb.5:
+    successors: %bb.6(0x04000000), %bb.1(0x7c000000)
+
+    %14:_(s32) = G_PHI %32(s32), %bb.3, %10(s32), %bb.1
+    %36:_(s1) = G_PHI %25(s1), %bb.3, %15(s1), %bb.1
+    %37:_(s1) = G_PHI %34(s1), %bb.3, %15(s1), %bb.1
+    G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %23(s32)
+    %12:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.if.break), %37(s1), %11(s32)
+    SI_LOOP %12(s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.6
+
+  bb.6:
+    successors: %bb.2(0x40000000), %bb.4(0x40000000)
+
+    %38:sreg_32_xm0_xexec(s1) = G_PHI %36(s1), %bb.5
+    %39:_(s32) = G_PHI %12(s32), %bb.5
+    G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %39(s32)
+    %35:sreg_32_xm0_xexec(s32) = SI_IF %38(s1), %bb.4, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.2
+...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-structurizer.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-structurizer.ll
new file mode 100644
index 0000000000000000000000000000000000000000..afd271c9957700f2e231de6b210707cc7482eb47
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-structurizer.ll
@@ -0,0 +1,461 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
+; RUN: llc -global-isel -mtriple=amdgcn-amd-amdpal -amdgpu-global-isel-risky-select -mcpu=gfx1010 < %s | FileCheck -check-prefix=GFX10 %s
+
+; Simples case, if - then, that requires lane mask merging,
+; %phi lane mask will hold %val_A at %A. Lanes that are active in %B
+; will overwrite its own lane bit in lane mask with val_B
+define amdgpu_ps void @divergent_i1_phi_if_then(ptr addrspace(1) %out, i32 %tid, i32 %cond) {
+; GFX10-LABEL: divergent_i1_phi_if_then:
+; GFX10:       ; %bb.0: ; %A
+; GFX10-NEXT:    v_cmp_le_u32_e64 s0, 6, v2
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v3
+; GFX10-NEXT:    s_and_saveexec_b32 s1, vcc_lo
+; GFX10-NEXT:  ; %bb.1: ; %B
+; GFX10-NEXT:    v_cmp_gt_u32_e64 s0, 1, v2
+; GFX10-NEXT:  ; %bb.2: ; %exit
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s1
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, 2, 1, s0
+; GFX10-NEXT:    global_store_dword v[0:1], v2, off
+; GFX10-NEXT:    s_endpgm
+A:
+  %val_A = icmp uge i32 %tid, 6
+  %cmp = icmp eq i32 %cond, 0
+  br i1 %cmp, label %B, label %exit
+
+B:
+  %val_B = icmp ult i32 %tid, 1
+  br label %exit
+
+exit:
+  %phi = phi i1 [ %val_A, %A ], [ %val_B, %B ]
+  %sel = select i1 %phi, i32 1, i32 2
+  store i32 %sel, ptr addrspace(1) %out
+  ret void
+}
+
+; if - else
+define amdgpu_ps void @divergent_i1_phi_if_else(ptr addrspace(1) %out, i32 %tid, i32 %cond) {
+; GFX10-LABEL: divergent_i1_phi_if_else:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:    s_and_b32 s0, 1, s0
+; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v3
+; GFX10-NEXT:    v_cmp_ne_u32_e64 s0, 0, s0
+; GFX10-NEXT:    s_and_saveexec_b32 s1, vcc_lo
+; GFX10-NEXT:    s_xor_b32 s1, exec_lo, s1
+; GFX10-NEXT:  ; %bb.1: ; %B
+; GFX10-NEXT:    v_cmp_gt_u32_e64 s0, 2, v2
+; GFX10-NEXT:    ; implicit-def: $vgpr2
+; GFX10-NEXT:  ; %bb.2: ; %Flow
+; GFX10-NEXT:    s_andn2_saveexec_b32 s1, s1
+; GFX10-NEXT:  ; %bb.3: ; %A
+; GFX10-NEXT:    v_cmp_le_u32_e64 s0, 1, v2
+; GFX10-NEXT:  ; %bb.4: ; %exit
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s1
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, 2, 1, s0
+; GFX10-NEXT:    global_store_dword v[0:1], v2, off
+; GFX10-NEXT:    s_endpgm
+entry:
+  %cmp = icmp eq i32 %cond, 0
+  br i1 %cmp, label %A, label %B
+
+A:
+  %val_A = icmp uge i32 %tid, 1
+  br label %exit
+
+B:
+  %val_B = icmp ult i32 %tid, 2
+  br label %exit
+
+exit:
+  %phi = phi i1 [ %val_A, %A ], [ %val_B, %B ]
+  %sel = select i1 %phi, i32 1, i32 2
+  store i32 %sel, ptr addrspace(1) %out
+  ret void
+}
+
+; if - break;
+
+;  counter = 0;
+;  do {
+;    if (a[counter] == 0)
+;      break;
+;    if (b[counter] == 0)
+;      break;
+;    if (c[counter] == 0)
+;      break;
+;    x[counter++]+=1;
+;  } while (counter<100);
+
+; Tests with multiple break conditions. Divergent phis will be used to track
+; if any of the break conditions was reached. We only need to do simple lane
+; mask merging (for current loop iteration only). There is an intrinsic,
+; if_break, that will merge lane masks across all iterations of the loop.
+
+define amdgpu_cs void @loop_with_1break(ptr addrspace(1) %x, ptr addrspace(1) %a) {
+; GFX10-LABEL: loop_with_1break:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:    s_mov_b32 s0, 0
+; GFX10-NEXT:    v_mov_b32_e32 v4, s0
+; GFX10-NEXT:    s_branch .LBB2_2
+; GFX10-NEXT:  .LBB2_1: ; %Flow
+; GFX10-NEXT:    ; in Loop: Header=BB2_2 Depth=1
+; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s1
+; GFX10-NEXT:    s_and_b32 s1, exec_lo, s2
+; GFX10-NEXT:    s_or_b32 s0, s1, s0
+; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s0
+; GFX10-NEXT:    s_cbranch_execz .LBB2_4
+; GFX10-NEXT:  .LBB2_2: ; %A
+; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX10-NEXT:    v_ashrrev_i32_e32 v5, 31, v4
+; GFX10-NEXT:    s_mov_b32 s2, -1
+; GFX10-NEXT:    v_lshlrev_b64 v[5:6], 2, v[4:5]
+; GFX10-NEXT:    v_add_co_u32 v7, vcc_lo, v2, v5
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v8, vcc_lo, v3, v6, vcc_lo
+; GFX10-NEXT:    global_load_dword v7, v[7:8], off
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v7
+; GFX10-NEXT:    s_and_saveexec_b32 s1, vcc_lo
+; GFX10-NEXT:    s_cbranch_execz .LBB2_1
+; GFX10-NEXT:  ; %bb.3: ; %loop.body
+; GFX10-NEXT:    ; in Loop: Header=BB2_2 Depth=1
+; GFX10-NEXT:    v_add_co_u32 v5, vcc_lo, v0, v5
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v6, vcc_lo, v1, v6, vcc_lo
+; GFX10-NEXT:    v_add_nc_u32_e32 v8, 1, v4
+; GFX10-NEXT:    v_cmp_gt_u32_e64 s2, 0x64, v4
+; GFX10-NEXT:    global_load_dword v7, v[5:6], off
+; GFX10-NEXT:    v_mov_b32_e32 v4, v8
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_add_nc_u32_e32 v7, 1, v7
+; GFX10-NEXT:    global_store_dword v[5:6], v7, off
+; GFX10-NEXT:    s_branch .LBB2_1
+; GFX10-NEXT:  .LBB2_4: ; %exit
+; GFX10-NEXT:    s_endpgm
+entry:
+  br label %A
+
+A:
+  %counter = phi i32 [ %counter.plus.1, %loop.body ], [ 0, %entry ]
+  %a.plus.counter = getelementptr inbounds i32, ptr addrspace(1) %a, i32 %counter
+  %a.val = load i32, ptr addrspace(1) %a.plus.counter
+  %a.cond = icmp eq i32 %a.val, 0
+  br i1 %a.cond, label %exit, label %loop.body
+
+loop.body:
+  %x.plus.counter = getelementptr inbounds i32, ptr addrspace(1) %x, i32 %counter
+  %x.val = load i32, ptr addrspace(1) %x.plus.counter
+  %x.val.plus.1 = add i32 %x.val, 1
+  store i32 %x.val.plus.1, ptr addrspace(1) %x.plus.counter
+  %counter.plus.1 = add i32 %counter, 1
+  %x.cond = icmp ult i32 %counter, 100
+  br i1 %x.cond, label %exit, label %A
+
+exit:
+  ret void
+}
+
+define amdgpu_cs void @loop_with_2breaks(ptr addrspace(1) %x, ptr addrspace(1) %a, ptr addrspace(1) %b) {
+; GFX10-LABEL: loop_with_2breaks:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:    s_mov_b32 s0, 0
+; GFX10-NEXT:    v_mov_b32_e32 v6, s0
+; GFX10-NEXT:    s_branch .LBB3_3
+; GFX10-NEXT:  .LBB3_1: ; %Flow3
+; GFX10-NEXT:    ; in Loop: Header=BB3_3 Depth=1
+; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s3
+; GFX10-NEXT:  .LBB3_2: ; %Flow
+; GFX10-NEXT:    ; in Loop: Header=BB3_3 Depth=1
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s1
+; GFX10-NEXT:    s_and_b32 s1, exec_lo, s2
+; GFX10-NEXT:    s_or_b32 s0, s1, s0
+; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s0
+; GFX10-NEXT:    s_cbranch_execz .LBB3_6
+; GFX10-NEXT:  .LBB3_3: ; %A
+; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX10-NEXT:    v_ashrrev_i32_e32 v7, 31, v6
+; GFX10-NEXT:    s_mov_b32 s2, -1
+; GFX10-NEXT:    v_lshlrev_b64 v[7:8], 2, v[6:7]
+; GFX10-NEXT:    v_add_co_u32 v9, vcc_lo, v2, v7
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v10, vcc_lo, v3, v8, vcc_lo
+; GFX10-NEXT:    global_load_dword v9, v[9:10], off
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v9
+; GFX10-NEXT:    s_and_saveexec_b32 s1, vcc_lo
+; GFX10-NEXT:    s_cbranch_execz .LBB3_2
+; GFX10-NEXT:  ; %bb.4: ; %B
+; GFX10-NEXT:    ; in Loop: Header=BB3_3 Depth=1
+; GFX10-NEXT:    v_add_co_u32 v9, vcc_lo, v4, v7
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v10, vcc_lo, v5, v8, vcc_lo
+; GFX10-NEXT:    global_load_dword v9, v[9:10], off
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v9
+; GFX10-NEXT:    s_and_saveexec_b32 s3, vcc_lo
+; GFX10-NEXT:    s_cbranch_execz .LBB3_1
+; GFX10-NEXT:  ; %bb.5: ; %loop.body
+; GFX10-NEXT:    ; in Loop: Header=BB3_3 Depth=1
+; GFX10-NEXT:    v_add_co_u32 v7, vcc_lo, v0, v7
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v8, vcc_lo, v1, v8, vcc_lo
+; GFX10-NEXT:    v_add_nc_u32_e32 v10, 1, v6
+; GFX10-NEXT:    v_cmp_gt_u32_e64 s2, 0x64, v6
+; GFX10-NEXT:    global_load_dword v9, v[7:8], off
+; GFX10-NEXT:    v_mov_b32_e32 v6, v10
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_add_nc_u32_e32 v9, 1, v9
+; GFX10-NEXT:    global_store_dword v[7:8], v9, off
+; GFX10-NEXT:    s_branch .LBB3_1
+; GFX10-NEXT:  .LBB3_6: ; %exit
+; GFX10-NEXT:    s_endpgm
+entry:
+  br label %A
+
+A:
+  %counter = phi i32 [ %counter.plus.1, %loop.body ], [ 0, %entry ]
+  %a.plus.counter = getelementptr inbounds i32, ptr addrspace(1) %a, i32 %counter
+  %a.val = load i32, ptr addrspace(1) %a.plus.counter
+  %a.cond = icmp eq i32 %a.val, 0
+  br i1 %a.cond, label %exit, label %B
+
+B:
+  %b.plus.counter = getelementptr inbounds i32, ptr addrspace(1) %b, i32 %counter
+  %b.val = load i32, ptr addrspace(1) %b.plus.counter
+  %b.cond = icmp eq i32 %b.val, 0
+  br i1 %b.cond, label %exit, label %loop.body
+
+loop.body:
+  %x.plus.counter = getelementptr inbounds i32, ptr addrspace(1) %x, i32 %counter
+  %x.val = load i32, ptr addrspace(1) %x.plus.counter
+  %x.val.plus.1 = add i32 %x.val, 1
+  store i32 %x.val.plus.1, ptr addrspace(1) %x.plus.counter
+  %counter.plus.1 = add i32 %counter, 1
+  %x.cond = icmp ult i32 %counter, 100
+  br i1 %x.cond, label %exit, label %A
+
+exit:
+  ret void
+}
+
+define amdgpu_cs void @loop_with_3breaks(ptr addrspace(1) %x, ptr addrspace(1) %a, ptr addrspace(1) %b, ptr addrspace(1) %c) {
+; GFX10-LABEL: loop_with_3breaks:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:    s_mov_b32 s0, 0
+; GFX10-NEXT:    v_mov_b32_e32 v8, s0
+; GFX10-NEXT:    s_branch .LBB4_4
+; GFX10-NEXT:  .LBB4_1: ; %Flow5
+; GFX10-NEXT:    ; in Loop: Header=BB4_4 Depth=1
+; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT:  .LBB4_2: ; %Flow4
+; GFX10-NEXT:    ; in Loop: Header=BB4_4 Depth=1
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s3
+; GFX10-NEXT:  .LBB4_3: ; %Flow
+; GFX10-NEXT:    ; in Loop: Header=BB4_4 Depth=1
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s1
+; GFX10-NEXT:    s_and_b32 s1, exec_lo, s2
+; GFX10-NEXT:    s_or_b32 s0, s1, s0
+; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s0
+; GFX10-NEXT:    s_cbranch_execz .LBB4_8
+; GFX10-NEXT:  .LBB4_4: ; %A
+; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX10-NEXT:    v_ashrrev_i32_e32 v9, 31, v8
+; GFX10-NEXT:    s_mov_b32 s2, -1
+; GFX10-NEXT:    v_lshlrev_b64 v[9:10], 2, v[8:9]
+; GFX10-NEXT:    v_add_co_u32 v11, vcc_lo, v2, v9
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v12, vcc_lo, v3, v10, vcc_lo
+; GFX10-NEXT:    global_load_dword v11, v[11:12], off
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v11
+; GFX10-NEXT:    s_and_saveexec_b32 s1, vcc_lo
+; GFX10-NEXT:    s_cbranch_execz .LBB4_3
+; GFX10-NEXT:  ; %bb.5: ; %B
+; GFX10-NEXT:    ; in Loop: Header=BB4_4 Depth=1
+; GFX10-NEXT:    v_add_co_u32 v11, vcc_lo, v4, v9
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v12, vcc_lo, v5, v10, vcc_lo
+; GFX10-NEXT:    global_load_dword v11, v[11:12], off
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v11
+; GFX10-NEXT:    s_and_saveexec_b32 s3, vcc_lo
+; GFX10-NEXT:    s_cbranch_execz .LBB4_2
+; GFX10-NEXT:  ; %bb.6: ; %C
+; GFX10-NEXT:    ; in Loop: Header=BB4_4 Depth=1
+; GFX10-NEXT:    v_add_co_u32 v11, vcc_lo, v6, v9
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v12, vcc_lo, v7, v10, vcc_lo
+; GFX10-NEXT:    global_load_dword v11, v[11:12], off
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v11
+; GFX10-NEXT:    s_and_saveexec_b32 s4, vcc_lo
+; GFX10-NEXT:    s_cbranch_execz .LBB4_1
+; GFX10-NEXT:  ; %bb.7: ; %loop.body
+; GFX10-NEXT:    ; in Loop: Header=BB4_4 Depth=1
+; GFX10-NEXT:    v_add_co_u32 v9, vcc_lo, v0, v9
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v10, vcc_lo, v1, v10, vcc_lo
+; GFX10-NEXT:    v_add_nc_u32_e32 v12, 1, v8
+; GFX10-NEXT:    v_cmp_gt_u32_e64 s2, 0x64, v8
+; GFX10-NEXT:    global_load_dword v11, v[9:10], off
+; GFX10-NEXT:    v_mov_b32_e32 v8, v12
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_add_nc_u32_e32 v11, 1, v11
+; GFX10-NEXT:    global_store_dword v[9:10], v11, off
+; GFX10-NEXT:    s_branch .LBB4_1
+; GFX10-NEXT:  .LBB4_8: ; %exit
+; GFX10-NEXT:    s_endpgm
+entry:
+  br label %A
+
+A:
+  %counter = phi i32 [ %counter.plus.1, %loop.body ], [ 0, %entry ]
+  %a.plus.counter = getelementptr inbounds i32, ptr addrspace(1) %a, i32 %counter
+  %a.val = load i32, ptr addrspace(1) %a.plus.counter
+  %a.cond = icmp eq i32 %a.val, 0
+  br i1 %a.cond, label %exit, label %B
+
+B:
+  %b.plus.counter = getelementptr inbounds i32, ptr addrspace(1) %b, i32 %counter
+  %b.val = load i32, ptr addrspace(1) %b.plus.counter
+  %b.cond = icmp eq i32 %b.val, 0
+  br i1 %b.cond, label %exit, label %C
+
+C:
+  %c.plus.counter = getelementptr inbounds i32, ptr addrspace(1) %c, i32 %counter
+  %c.val = load i32, ptr addrspace(1) %c.plus.counter
+  %c.cond = icmp eq i32 %c.val, 0
+  br i1 %c.cond, label %exit, label %loop.body
+
+loop.body:
+  %x.plus.counter = getelementptr inbounds i32, ptr addrspace(1) %x, i32 %counter
+  %x.val = load i32, ptr addrspace(1) %x.plus.counter
+  %x.val.plus.1 = add i32 %x.val, 1
+  store i32 %x.val.plus.1, ptr addrspace(1) %x.plus.counter
+  %counter.plus.1 = add i32 %counter, 1
+  %x.cond = icmp ult i32 %counter, 100
+  br i1 %x.cond, label %exit, label %A
+
+exit:
+  ret void
+}
+
+; Divergent condition if with body, ending with break. This is loop with two
+; exits but structurizer will create phi that will track exit from break
+; and move break.body after the loop. Loop will then have one exit and phi
+; used outside of the loop by condition used to enter the break.body.
+define amdgpu_cs void @loop_with_div_break_with_body(ptr addrspace(1) %x, ptr addrspace(1) %a, ptr addrspace(1) %a.break) {
+; GFX10-LABEL: loop_with_div_break_with_body:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:    s_mov_b32 s0, 0
+; GFX10-NEXT:    v_mov_b32_e32 v6, s0
+; GFX10-NEXT:    s_branch .LBB5_2
+; GFX10-NEXT:  .LBB5_1: ; %Flow
+; GFX10-NEXT:    ; in Loop: Header=BB5_2 Depth=1
+; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s1
+; GFX10-NEXT:    s_and_b32 s1, exec_lo, s2
+; GFX10-NEXT:    s_or_b32 s0, s1, s0
+; GFX10-NEXT:    s_and_b32 s1, 1, s3
+; GFX10-NEXT:    v_cmp_ne_u32_e64 s1, 0, s1
+; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s0
+; GFX10-NEXT:    s_cbranch_execz .LBB5_4
+; GFX10-NEXT:  .LBB5_2: ; %A
+; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX10-NEXT:    v_ashrrev_i32_e32 v7, 31, v6
+; GFX10-NEXT:    s_mov_b32 s2, -1
+; GFX10-NEXT:    s_mov_b32 s3, 1
+; GFX10-NEXT:    v_lshlrev_b64 v[7:8], 2, v[6:7]
+; GFX10-NEXT:    v_add_co_u32 v9, vcc_lo, v2, v7
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v10, vcc_lo, v3, v8, vcc_lo
+; GFX10-NEXT:    global_load_dword v9, v[9:10], off
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v9
+; GFX10-NEXT:    s_and_saveexec_b32 s1, vcc_lo
+; GFX10-NEXT:    s_cbranch_execz .LBB5_1
+; GFX10-NEXT:  ; %bb.3: ; %loop.body
+; GFX10-NEXT:    ; in Loop: Header=BB5_2 Depth=1
+; GFX10-NEXT:    v_add_co_u32 v7, vcc_lo, v0, v7
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v8, vcc_lo, v1, v8, vcc_lo
+; GFX10-NEXT:    v_add_nc_u32_e32 v10, 1, v6
+; GFX10-NEXT:    v_cmp_gt_u32_e64 s2, 0x64, v6
+; GFX10-NEXT:    s_mov_b32 s3, 0
+; GFX10-NEXT:    global_load_dword v9, v[7:8], off
+; GFX10-NEXT:    v_mov_b32_e32 v6, v10
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_add_nc_u32_e32 v9, 1, v9
+; GFX10-NEXT:    global_store_dword v[7:8], v9, off
+; GFX10-NEXT:    s_branch .LBB5_1
+; GFX10-NEXT:  .LBB5_4: ; %loop.exit.guard
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s0
+; GFX10-NEXT:    s_and_saveexec_b32 s0, s1
+; GFX10-NEXT:    s_xor_b32 s0, exec_lo, s0
+; GFX10-NEXT:    s_cbranch_execz .LBB5_6
+; GFX10-NEXT:  ; %bb.5: ; %break.body
+; GFX10-NEXT:    v_mov_b32_e32 v0, 10
+; GFX10-NEXT:    global_store_dword v[4:5], v0, off
+; GFX10-NEXT:  .LBB5_6: ; %exit
+; GFX10-NEXT:    s_endpgm
+entry:
+  br label %A
+
+A:
+  %counter = phi i32 [ %counter.plus.1, %loop.body ], [ 0, %entry ]
+  %a.plus.counter = getelementptr inbounds i32, ptr addrspace(1) %a, i32 %counter
+  %a.val = load i32, ptr addrspace(1) %a.plus.counter
+  %a.cond = icmp eq i32 %a.val, 0
+  br i1 %a.cond, label %break.body, label %loop.body
+
+break.body:
+  store i32 10, ptr addrspace(1) %a.break
+  br label %exit
+
+
+loop.body:
+  %x.plus.counter = getelementptr inbounds i32, ptr addrspace(1) %x, i32 %counter
+  %x.val = load i32, ptr addrspace(1) %x.plus.counter
+  %x.val.plus.1 = add i32 %x.val, 1
+  store i32 %x.val.plus.1, ptr addrspace(1) %x.plus.counter
+  %counter.plus.1 = add i32 %counter, 1
+  %x.cond = icmp ult i32 %counter, 100
+  br i1 %x.cond, label %exit, label %A
+
+exit:
+  ret void
+}
+
+; Snippet from test generated by the GraphicsFuzz tool, frontend generates ir
+; with irreducible control flow graph. FixIrreducible converts it into natural
+; loop and in the process creates i1 phi with three incoming values.
+
+; int loop(int x, int y, int a0, int a1, int a2, int a3, int a4) {
+;   do {
+;     if (y < a2) {
+;       do {
+;       } while (x < a2);
+;     }
+;     if (x < a3) {
+;       return a1;
+;     }
+;   } while (y < a2);
+;   return a0;
+; }
+
+; This test is also interesting because it has phi with three incomings
+;define amdgpu_ps i32 @irreducible_cfg(i32 %x, i32 %y, i32 %a0, i32 %a1, i32 %a2, i32 %a3) {
+;.entry:
+; %.y_lt_a2 = icmp sgt i32 %a2, %y
+; %.x_lt_a2 = icmp sgt i32 %a2, %x
+; %.x_lt_a3 = icmp sgt i32 %a3, %x
+; br i1 %.y_lt_a2, label %.preheader, label %.loopexit ; first iteration, jump to inner loop if 'y < a2' or start with 'if (x < a3)'
+;
+;.preheader: ; if (y < a2),
+; br label %.inner_loop
+;
+;.inner_loop: ; do while x < a2
+; br i1 %.x_lt_a2, label %.inner_loop, label %.loopexit
+;
+;.loopexit: ; if x < a3
+; %not.inner_loop = xor i1 %.y_lt_a2, true
+; %brmerge = select i1 %.x_lt_a3, i1 true, i1 %not.inner_loop ; exit loop if 'x < a3' or 'loop ends since !(y < a2)'
+; %.ret = select i1 %.x_lt_a3, i32 %a1, i32 %a0               ; select retrun value a1 'x < a3' or a0 'loop ends'
+; br i1 %brmerge, label %.exit, label %.preheader
+;
+;.exit:
+; ret i32 %.ret
+;}
+
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-structurizer.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-structurizer.mir
new file mode 100644
index 0000000000000000000000000000000000000000..9461d558684e856821f1fb551ad61f6121498880
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-structurizer.mir
@@ -0,0 +1,1004 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 4
+# RUN: llc -global-isel -mtriple=amdgcn-mesa-amdpal -mcpu=gfx1010 -run-pass=amdgpu-global-isel-divergence-lowering %s -o - | FileCheck -check-prefix=GFX10 %s
+
+---
+name: divergent_i1_phi_if_then
+legalized: true
+tracksRegLiveness: true
+body: |
+  ; GFX10-LABEL: name: divergent_i1_phi_if_then
+  ; GFX10: bb.0:
+  ; GFX10-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; GFX10-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GFX10-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GFX10-NEXT:   [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY]](s32), [[COPY1]](s32)
+  ; GFX10-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; GFX10-NEXT:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; GFX10-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 6
+  ; GFX10-NEXT:   [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[COPY2]](s32), [[C]]
+  ; GFX10-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT:   [[ICMP1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_ICMP intpred(eq), [[COPY3]](s32), [[C1]]
+  ; GFX10-NEXT:   [[SI_IF:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[ICMP1]](s1), %bb.2, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX10-NEXT:   G_BR %bb.1
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.1:
+  ; GFX10-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; GFX10-NEXT:   [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[COPY2]](s32), [[C2]]
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.2:
+  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:_(s1) = G_PHI [[ICMP]](s1), %bb.0, [[ICMP2]](s1), %bb.1
+  ; GFX10-NEXT:   G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[SI_IF]](s32)
+  ; GFX10-NEXT:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; GFX10-NEXT:   [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; GFX10-NEXT:   [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[PHI]](s1), [[C4]], [[C3]]
+  ; GFX10-NEXT:   G_STORE [[SELECT]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+  ; GFX10-NEXT:   S_ENDPGM 0
+  bb.0:
+    successors: %bb.1(0x40000000), %bb.2(0x40000000)
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(p1) = G_MERGE_VALUES %0(s32), %1(s32)
+    %3:_(s32) = COPY $vgpr2
+    %4:_(s32) = COPY $vgpr3
+    %5:_(s32) = G_CONSTANT i32 6
+    %6:_(s1) = G_ICMP intpred(uge), %3(s32), %5
+    %7:_(s32) = G_CONSTANT i32 0
+    %8:sreg_32_xm0_xexec(s1) = G_ICMP intpred(eq), %4(s32), %7
+    %9:sreg_32_xm0_xexec(s32) = SI_IF %8(s1), %bb.2, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.1
+
+  bb.1:
+    successors: %bb.2(0x80000000)
+
+    %10:_(s32) = G_CONSTANT i32 1
+    %11:_(s1) = G_ICMP intpred(ult), %3(s32), %10
+
+  bb.2:
+    %12:_(s1) = G_PHI %6(s1), %bb.0, %11(s1), %bb.1
+    G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %9(s32)
+    %13:_(s32) = G_CONSTANT i32 2
+    %14:_(s32) = G_CONSTANT i32 1
+    %15:_(s32) = G_SELECT %12(s1), %14, %13
+    G_STORE %15(s32), %2(p1) :: (store (s32), addrspace 1)
+    S_ENDPGM 0
+...
+
+---
+name: divergent_i1_phi_if_else
+legalized: true
+tracksRegLiveness: true
+body: |
+  ; GFX10-LABEL: name: divergent_i1_phi_if_else
+  ; GFX10: bb.0:
+  ; GFX10-NEXT:   successors: %bb.3(0x40000000), %bb.1(0x40000000)
+  ; GFX10-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GFX10-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GFX10-NEXT:   [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY]](s32), [[COPY1]](s32)
+  ; GFX10-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; GFX10-NEXT:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; GFX10-NEXT:   [[DEF:%[0-9]+]]:_(s1) = G_IMPLICIT_DEF
+  ; GFX10-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT:   [[ICMP:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), [[COPY3]](s32), [[C]]
+  ; GFX10-NEXT:   [[SI_IF:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[ICMP]](s1), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX10-NEXT:   G_BR %bb.3
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.1:
+  ; GFX10-NEXT:   successors: %bb.2(0x40000000), %bb.4(0x40000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:_(s1) = G_PHI %10(s1), %bb.3, [[DEF]](s1), %bb.0
+  ; GFX10-NEXT:   [[SI_ELSE:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_ELSE [[SI_IF]](s32), %bb.4, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX10-NEXT:   G_BR %bb.2
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.2:
+  ; GFX10-NEXT:   successors: %bb.4(0x80000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; GFX10-NEXT:   [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(uge), [[COPY2]](s32), [[C1]]
+  ; GFX10-NEXT:   G_BR %bb.4
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.3:
+  ; GFX10-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; GFX10-NEXT:   [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[COPY2]](s32), [[C2]]
+  ; GFX10-NEXT:   G_BR %bb.1
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.4:
+  ; GFX10-NEXT:   [[PHI1:%[0-9]+]]:_(s1) = G_PHI [[PHI]](s1), %bb.1, [[ICMP1]](s1), %bb.2
+  ; GFX10-NEXT:   G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[SI_ELSE]](s32)
+  ; GFX10-NEXT:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; GFX10-NEXT:   [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; GFX10-NEXT:   [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[PHI1]](s1), [[C3]], [[C4]]
+  ; GFX10-NEXT:   G_STORE [[SELECT]](s32), [[MV]](p1) :: (store (s32), addrspace 1)
+  ; GFX10-NEXT:   S_ENDPGM 0
+  bb.0:
+    successors: %bb.3(0x40000000), %bb.1(0x40000000)
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(p1) = G_MERGE_VALUES %0(s32), %1(s32)
+    %3:_(s32) = COPY $vgpr2
+    %4:_(s32) = COPY $vgpr3
+    %5:_(s1) = G_IMPLICIT_DEF
+    %6:_(s32) = G_CONSTANT i32 0
+    %7:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), %4(s32), %6
+    %8:sreg_32_xm0_xexec(s32) = SI_IF %7(s1), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.3
+
+  bb.1:
+    successors: %bb.2(0x40000000), %bb.4(0x40000000)
+
+    %9:_(s1) = G_PHI %10(s1), %bb.3, %5(s1), %bb.0
+    %11:sreg_32_xm0_xexec(s32) = SI_ELSE %8(s32), %bb.4, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.2
+
+  bb.2:
+    successors: %bb.4(0x80000000)
+
+    %12:_(s32) = G_CONSTANT i32 1
+    %13:_(s1) = G_ICMP intpred(uge), %3(s32), %12
+    G_BR %bb.4
+
+  bb.3:
+    successors: %bb.1(0x80000000)
+
+    %14:_(s32) = G_CONSTANT i32 2
+    %10:_(s1) = G_ICMP intpred(ult), %3(s32), %14
+    G_BR %bb.1
+
+  bb.4:
+    %15:_(s1) = G_PHI %9(s1), %bb.1, %13(s1), %bb.2
+    G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %11(s32)
+    %16:_(s32) = G_CONSTANT i32 1
+    %17:_(s32) = G_CONSTANT i32 2
+    %18:_(s32) = G_SELECT %15(s1), %16, %17
+    G_STORE %18(s32), %2(p1) :: (store (s32), addrspace 1)
+    S_ENDPGM 0
+...
+
+---
+name: loop_with_1break
+legalized: true
+tracksRegLiveness: true
+body: |
+  ; GFX10-LABEL: name: loop_with_1break
+  ; GFX10: bb.0:
+  ; GFX10-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX10-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GFX10-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GFX10-NEXT:   [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY]](s32), [[COPY1]](s32)
+  ; GFX10-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; GFX10-NEXT:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; GFX10-NEXT:   [[MV1:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32)
+  ; GFX10-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.1:
+  ; GFX10-NEXT:   successors: %bb.2(0x40000000), %bb.3(0x40000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:_(s32) = G_PHI %9(s32), %bb.3, [[C]](s32), %bb.0
+  ; GFX10-NEXT:   [[PHI1:%[0-9]+]]:_(s32) = G_PHI [[C]](s32), %bb.0, %11(s32), %bb.3
+  ; GFX10-NEXT:   [[C1:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
+  ; GFX10-NEXT:   [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[PHI1]](s32)
+  ; GFX10-NEXT:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; GFX10-NEXT:   [[SHL:%[0-9]+]]:_(s64) = G_SHL [[SEXT]], [[C2]](s32)
+  ; GFX10-NEXT:   [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[MV1]], [[SHL]](s64)
+  ; GFX10-NEXT:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p1) :: (load (s32), addrspace 1)
+  ; GFX10-NEXT:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT:   [[ICMP:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), [[LOAD]](s32), [[C3]]
+  ; GFX10-NEXT:   [[SI_IF:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[ICMP]](s1), %bb.3, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX10-NEXT:   G_BR %bb.2
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.2:
+  ; GFX10-NEXT:   successors: %bb.3(0x80000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; GFX10-NEXT:   [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[SEXT]], [[C4]](s32)
+  ; GFX10-NEXT:   [[PTR_ADD1:%[0-9]+]]:_(p1) = G_PTR_ADD [[MV]], [[SHL1]](s64)
+  ; GFX10-NEXT:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p1) :: (load (s32), addrspace 1)
+  ; GFX10-NEXT:   [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; GFX10-NEXT:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[LOAD1]], [[C5]]
+  ; GFX10-NEXT:   G_STORE [[ADD]](s32), [[PTR_ADD1]](p1) :: (store (s32), addrspace 1)
+  ; GFX10-NEXT:   [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[PHI1]], [[C5]]
+  ; GFX10-NEXT:   [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 100
+  ; GFX10-NEXT:   [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[PHI1]](s32), [[C6]]
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.3:
+  ; GFX10-NEXT:   successors: %bb.4(0x04000000), %bb.1(0x7c000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[PHI2:%[0-9]+]]:_(s32) = G_PHI [[ADD1]](s32), %bb.2, [[DEF]](s32), %bb.1
+  ; GFX10-NEXT:   [[PHI3:%[0-9]+]]:_(s1) = G_PHI [[ICMP1]](s1), %bb.2, [[C1]](s1), %bb.1
+  ; GFX10-NEXT:   G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[SI_IF]](s32)
+  ; GFX10-NEXT:   [[INTRINSIC_CONVERGENT:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.if.break), [[PHI3]](s1), [[PHI]](s32)
+  ; GFX10-NEXT:   SI_LOOP [[INTRINSIC_CONVERGENT]](s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX10-NEXT:   G_BR %bb.4
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.4:
+  ; GFX10-NEXT:   [[PHI4:%[0-9]+]]:_(s32) = G_PHI [[INTRINSIC_CONVERGENT]](s32), %bb.3
+  ; GFX10-NEXT:   G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[PHI4]](s32)
+  ; GFX10-NEXT:   S_ENDPGM 0
+  bb.0:
+    successors: %bb.1(0x80000000)
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3
+
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(p1) = G_MERGE_VALUES %0(s32), %1(s32)
+    %3:_(s32) = COPY $vgpr2
+    %4:_(s32) = COPY $vgpr3
+    %5:_(p1) = G_MERGE_VALUES %3(s32), %4(s32)
+    %6:_(s32) = G_CONSTANT i32 0
+    %7:_(s32) = G_IMPLICIT_DEF
+
+  bb.1:
+    successors: %bb.2(0x40000000), %bb.3(0x40000000)
+
+    %8:_(s32) = G_PHI %9(s32), %bb.3, %6(s32), %bb.0
+    %10:_(s32) = G_PHI %6(s32), %bb.0, %11(s32), %bb.3
+    %12:_(s1) = G_CONSTANT i1 true
+    %13:_(s64) = G_SEXT %10(s32)
+    %14:_(s32) = G_CONSTANT i32 2
+    %15:_(s64) = G_SHL %13, %14(s32)
+    %16:_(p1) = G_PTR_ADD %5, %15(s64)
+    %17:_(s32) = G_LOAD %16(p1) :: (load (s32), addrspace 1)
+    %18:_(s32) = G_CONSTANT i32 0
+    %19:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), %17(s32), %18
+    %20:sreg_32_xm0_xexec(s32) = SI_IF %19(s1), %bb.3, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.2
+
+  bb.2:
+    successors: %bb.3(0x80000000)
+
+    %21:_(s32) = G_CONSTANT i32 2
+    %22:_(s64) = G_SHL %13, %21(s32)
+    %23:_(p1) = G_PTR_ADD %2, %22(s64)
+    %24:_(s32) = G_LOAD %23(p1) :: (load (s32), addrspace 1)
+    %25:_(s32) = G_CONSTANT i32 1
+    %26:_(s32) = G_ADD %24, %25
+    G_STORE %26(s32), %23(p1) :: (store (s32), addrspace 1)
+    %27:_(s32) = G_ADD %10, %25
+    %28:_(s32) = G_CONSTANT i32 100
+    %29:_(s1) = G_ICMP intpred(ult), %10(s32), %28
+
+  bb.3:
+    successors: %bb.4(0x04000000), %bb.1(0x7c000000)
+
+    %11:_(s32) = G_PHI %27(s32), %bb.2, %7(s32), %bb.1
+    %30:_(s1) = G_PHI %29(s1), %bb.2, %12(s1), %bb.1
+    G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %20(s32)
+    %9:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.if.break), %30(s1), %8(s32)
+    SI_LOOP %9(s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.4
+
+  bb.4:
+    %31:_(s32) = G_PHI %9(s32), %bb.3
+    G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %31(s32)
+    S_ENDPGM 0
+...
+
+---
+name: loop_with_2breaks
+legalized: true
+tracksRegLiveness: true
+body: |
+  ; GFX10-LABEL: name: loop_with_2breaks
+  ; GFX10: bb.0:
+  ; GFX10-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX10-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GFX10-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GFX10-NEXT:   [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY]](s32), [[COPY1]](s32)
+  ; GFX10-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; GFX10-NEXT:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; GFX10-NEXT:   [[MV1:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32)
+  ; GFX10-NEXT:   [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; GFX10-NEXT:   [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
+  ; GFX10-NEXT:   [[MV2:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY4]](s32), [[COPY5]](s32)
+  ; GFX10-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.1:
+  ; GFX10-NEXT:   successors: %bb.2(0x40000000), %bb.3(0x40000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:_(s32) = G_PHI %12(s32), %bb.3, [[C]](s32), %bb.0
+  ; GFX10-NEXT:   [[PHI1:%[0-9]+]]:_(s32) = G_PHI [[C]](s32), %bb.0, %14(s32), %bb.3
+  ; GFX10-NEXT:   [[C1:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
+  ; GFX10-NEXT:   [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[PHI1]](s32)
+  ; GFX10-NEXT:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; GFX10-NEXT:   [[SHL:%[0-9]+]]:_(s64) = G_SHL [[SEXT]], [[C2]](s32)
+  ; GFX10-NEXT:   [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[MV1]], [[SHL]](s64)
+  ; GFX10-NEXT:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p1) :: (load (s32), addrspace 1)
+  ; GFX10-NEXT:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT:   [[ICMP:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), [[LOAD]](s32), [[C3]]
+  ; GFX10-NEXT:   [[SI_IF:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[ICMP]](s1), %bb.3, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX10-NEXT:   G_BR %bb.2
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.2:
+  ; GFX10-NEXT:   successors: %bb.4(0x40000000), %bb.5(0x40000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[C4:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
+  ; GFX10-NEXT:   [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; GFX10-NEXT:   [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[SEXT]], [[C5]](s32)
+  ; GFX10-NEXT:   [[PTR_ADD1:%[0-9]+]]:_(p1) = G_PTR_ADD [[MV2]], [[SHL1]](s64)
+  ; GFX10-NEXT:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p1) :: (load (s32), addrspace 1)
+  ; GFX10-NEXT:   [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT:   [[ICMP1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), [[LOAD1]](s32), [[C6]]
+  ; GFX10-NEXT:   [[SI_IF1:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[ICMP1]](s1), %bb.5, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX10-NEXT:   G_BR %bb.4
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.3:
+  ; GFX10-NEXT:   successors: %bb.6(0x04000000), %bb.1(0x7c000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[PHI2:%[0-9]+]]:_(s32) = G_PHI %32(s32), %bb.5, [[DEF]](s32), %bb.1
+  ; GFX10-NEXT:   [[PHI3:%[0-9]+]]:_(s1) = G_PHI %34(s1), %bb.5, [[C1]](s1), %bb.1
+  ; GFX10-NEXT:   G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[SI_IF]](s32)
+  ; GFX10-NEXT:   [[INTRINSIC_CONVERGENT:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.if.break), [[PHI3]](s1), [[PHI]](s32)
+  ; GFX10-NEXT:   SI_LOOP [[INTRINSIC_CONVERGENT]](s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX10-NEXT:   G_BR %bb.6
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.4:
+  ; GFX10-NEXT:   successors: %bb.5(0x80000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; GFX10-NEXT:   [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[SEXT]], [[C7]](s32)
+  ; GFX10-NEXT:   [[PTR_ADD2:%[0-9]+]]:_(p1) = G_PTR_ADD [[MV]], [[SHL2]](s64)
+  ; GFX10-NEXT:   [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p1) :: (load (s32), addrspace 1)
+  ; GFX10-NEXT:   [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; GFX10-NEXT:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[LOAD2]], [[C8]]
+  ; GFX10-NEXT:   G_STORE [[ADD]](s32), [[PTR_ADD2]](p1) :: (store (s32), addrspace 1)
+  ; GFX10-NEXT:   [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[PHI1]], [[C8]]
+  ; GFX10-NEXT:   [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 100
+  ; GFX10-NEXT:   [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[PHI1]](s32), [[C9]]
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.5:
+  ; GFX10-NEXT:   successors: %bb.3(0x80000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[PHI4:%[0-9]+]]:_(s32) = G_PHI [[ADD1]](s32), %bb.4, [[DEF]](s32), %bb.2
+  ; GFX10-NEXT:   [[PHI5:%[0-9]+]]:_(s1) = G_PHI [[ICMP2]](s1), %bb.4, [[C4]](s1), %bb.2
+  ; GFX10-NEXT:   G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[SI_IF1]](s32)
+  ; GFX10-NEXT:   G_BR %bb.3
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.6:
+  ; GFX10-NEXT:   [[PHI6:%[0-9]+]]:_(s32) = G_PHI [[INTRINSIC_CONVERGENT]](s32), %bb.3
+  ; GFX10-NEXT:   G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[PHI6]](s32)
+  ; GFX10-NEXT:   S_ENDPGM 0
+  bb.0:
+    successors: %bb.1(0x80000000)
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(p1) = G_MERGE_VALUES %0(s32), %1(s32)
+    %3:_(s32) = COPY $vgpr2
+    %4:_(s32) = COPY $vgpr3
+    %5:_(p1) = G_MERGE_VALUES %3(s32), %4(s32)
+    %6:_(s32) = COPY $vgpr4
+    %7:_(s32) = COPY $vgpr5
+    %8:_(p1) = G_MERGE_VALUES %6(s32), %7(s32)
+    %9:_(s32) = G_CONSTANT i32 0
+    %10:_(s32) = G_IMPLICIT_DEF
+
+  bb.1:
+    successors: %bb.2(0x40000000), %bb.3(0x40000000)
+
+    %11:_(s32) = G_PHI %12(s32), %bb.3, %9(s32), %bb.0
+    %13:_(s32) = G_PHI %9(s32), %bb.0, %14(s32), %bb.3
+    %15:_(s1) = G_CONSTANT i1 true
+    %16:_(s64) = G_SEXT %13(s32)
+    %17:_(s32) = G_CONSTANT i32 2
+    %18:_(s64) = G_SHL %16, %17(s32)
+    %19:_(p1) = G_PTR_ADD %5, %18(s64)
+    %20:_(s32) = G_LOAD %19(p1) :: (load (s32), addrspace 1)
+    %21:_(s32) = G_CONSTANT i32 0
+    %22:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), %20(s32), %21
+    %23:sreg_32_xm0_xexec(s32) = SI_IF %22(s1), %bb.3, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.2
+
+  bb.2:
+    successors: %bb.4(0x40000000), %bb.5(0x40000000)
+
+    %24:_(s1) = G_CONSTANT i1 true
+    %25:_(s32) = G_CONSTANT i32 2
+    %26:_(s64) = G_SHL %16, %25(s32)
+    %27:_(p1) = G_PTR_ADD %8, %26(s64)
+    %28:_(s32) = G_LOAD %27(p1) :: (load (s32), addrspace 1)
+    %29:_(s32) = G_CONSTANT i32 0
+    %30:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), %28(s32), %29
+    %31:sreg_32_xm0_xexec(s32) = SI_IF %30(s1), %bb.5, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.4
+
+  bb.3:
+    successors: %bb.6(0x04000000), %bb.1(0x7c000000)
+
+    %14:_(s32) = G_PHI %32(s32), %bb.5, %10(s32), %bb.1
+    %33:_(s1) = G_PHI %34(s1), %bb.5, %15(s1), %bb.1
+    G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %23(s32)
+    %12:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.if.break), %33(s1), %11(s32)
+    SI_LOOP %12(s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.6
+
+  bb.4:
+    successors: %bb.5(0x80000000)
+
+    %35:_(s32) = G_CONSTANT i32 2
+    %36:_(s64) = G_SHL %16, %35(s32)
+    %37:_(p1) = G_PTR_ADD %2, %36(s64)
+    %38:_(s32) = G_LOAD %37(p1) :: (load (s32), addrspace 1)
+    %39:_(s32) = G_CONSTANT i32 1
+    %40:_(s32) = G_ADD %38, %39
+    G_STORE %40(s32), %37(p1) :: (store (s32), addrspace 1)
+    %41:_(s32) = G_ADD %13, %39
+    %42:_(s32) = G_CONSTANT i32 100
+    %43:_(s1) = G_ICMP intpred(ult), %13(s32), %42
+
+  bb.5:
+    successors: %bb.3(0x80000000)
+
+    %32:_(s32) = G_PHI %41(s32), %bb.4, %10(s32), %bb.2
+    %34:_(s1) = G_PHI %43(s1), %bb.4, %24(s1), %bb.2
+    G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %31(s32)
+    G_BR %bb.3
+
+  bb.6:
+    %44:_(s32) = G_PHI %12(s32), %bb.3
+    G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %44(s32)
+    S_ENDPGM 0
+...
+
+---
+name: loop_with_3breaks
+legalized: true
+tracksRegLiveness: true
+body: |
+  ; GFX10-LABEL: name: loop_with_3breaks
+  ; GFX10: bb.0:
+  ; GFX10-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX10-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GFX10-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GFX10-NEXT:   [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY]](s32), [[COPY1]](s32)
+  ; GFX10-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; GFX10-NEXT:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; GFX10-NEXT:   [[MV1:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32)
+  ; GFX10-NEXT:   [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; GFX10-NEXT:   [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
+  ; GFX10-NEXT:   [[MV2:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY4]](s32), [[COPY5]](s32)
+  ; GFX10-NEXT:   [[COPY6:%[0-9]+]]:_(s32) = COPY $vgpr6
+  ; GFX10-NEXT:   [[COPY7:%[0-9]+]]:_(s32) = COPY $vgpr7
+  ; GFX10-NEXT:   [[MV3:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY6]](s32), [[COPY7]](s32)
+  ; GFX10-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.1:
+  ; GFX10-NEXT:   successors: %bb.2(0x40000000), %bb.3(0x40000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:_(s32) = G_PHI %15(s32), %bb.3, [[C]](s32), %bb.0
+  ; GFX10-NEXT:   [[PHI1:%[0-9]+]]:_(s32) = G_PHI [[C]](s32), %bb.0, %17(s32), %bb.3
+  ; GFX10-NEXT:   [[C1:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
+  ; GFX10-NEXT:   [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[PHI1]](s32)
+  ; GFX10-NEXT:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; GFX10-NEXT:   [[SHL:%[0-9]+]]:_(s64) = G_SHL [[SEXT]], [[C2]](s32)
+  ; GFX10-NEXT:   [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[MV1]], [[SHL]](s64)
+  ; GFX10-NEXT:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p1) :: (load (s32), addrspace 1)
+  ; GFX10-NEXT:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT:   [[ICMP:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), [[LOAD]](s32), [[C3]]
+  ; GFX10-NEXT:   [[SI_IF:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[ICMP]](s1), %bb.3, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX10-NEXT:   G_BR %bb.2
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.2:
+  ; GFX10-NEXT:   successors: %bb.4(0x40000000), %bb.5(0x40000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[C4:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
+  ; GFX10-NEXT:   [[C5:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; GFX10-NEXT:   [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[SEXT]], [[C5]](s32)
+  ; GFX10-NEXT:   [[PTR_ADD1:%[0-9]+]]:_(p1) = G_PTR_ADD [[MV2]], [[SHL1]](s64)
+  ; GFX10-NEXT:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p1) :: (load (s32), addrspace 1)
+  ; GFX10-NEXT:   [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT:   [[ICMP1:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), [[LOAD1]](s32), [[C6]]
+  ; GFX10-NEXT:   [[SI_IF1:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[ICMP1]](s1), %bb.5, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX10-NEXT:   G_BR %bb.4
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.3:
+  ; GFX10-NEXT:   successors: %bb.8(0x04000000), %bb.1(0x7c000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[PHI2:%[0-9]+]]:_(s32) = G_PHI %35(s32), %bb.5, [[DEF]](s32), %bb.1
+  ; GFX10-NEXT:   [[PHI3:%[0-9]+]]:_(s1) = G_PHI %37(s1), %bb.5, [[C1]](s1), %bb.1
+  ; GFX10-NEXT:   G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[SI_IF]](s32)
+  ; GFX10-NEXT:   [[INTRINSIC_CONVERGENT:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.if.break), [[PHI3]](s1), [[PHI]](s32)
+  ; GFX10-NEXT:   SI_LOOP [[INTRINSIC_CONVERGENT]](s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX10-NEXT:   G_BR %bb.8
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.4:
+  ; GFX10-NEXT:   successors: %bb.6(0x40000000), %bb.7(0x40000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[C7:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
+  ; GFX10-NEXT:   [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; GFX10-NEXT:   [[SHL2:%[0-9]+]]:_(s64) = G_SHL [[SEXT]], [[C8]](s32)
+  ; GFX10-NEXT:   [[PTR_ADD2:%[0-9]+]]:_(p1) = G_PTR_ADD [[MV3]], [[SHL2]](s64)
+  ; GFX10-NEXT:   [[LOAD2:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD2]](p1) :: (load (s32), addrspace 1)
+  ; GFX10-NEXT:   [[C9:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT:   [[ICMP2:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), [[LOAD2]](s32), [[C9]]
+  ; GFX10-NEXT:   [[SI_IF2:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[ICMP2]](s1), %bb.7, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX10-NEXT:   G_BR %bb.6
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.5:
+  ; GFX10-NEXT:   successors: %bb.3(0x80000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[PHI4:%[0-9]+]]:_(s32) = G_PHI %46(s32), %bb.7, [[DEF]](s32), %bb.2
+  ; GFX10-NEXT:   [[PHI5:%[0-9]+]]:_(s1) = G_PHI %47(s1), %bb.7, [[C4]](s1), %bb.2
+  ; GFX10-NEXT:   G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[SI_IF1]](s32)
+  ; GFX10-NEXT:   G_BR %bb.3
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.6:
+  ; GFX10-NEXT:   successors: %bb.7(0x80000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[C10:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; GFX10-NEXT:   [[SHL3:%[0-9]+]]:_(s64) = G_SHL [[SEXT]], [[C10]](s32)
+  ; GFX10-NEXT:   [[PTR_ADD3:%[0-9]+]]:_(p1) = G_PTR_ADD [[MV]], [[SHL3]](s64)
+  ; GFX10-NEXT:   [[LOAD3:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD3]](p1) :: (load (s32), addrspace 1)
+  ; GFX10-NEXT:   [[C11:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; GFX10-NEXT:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[LOAD3]], [[C11]]
+  ; GFX10-NEXT:   G_STORE [[ADD]](s32), [[PTR_ADD3]](p1) :: (store (s32), addrspace 1)
+  ; GFX10-NEXT:   [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[PHI1]], [[C11]]
+  ; GFX10-NEXT:   [[C12:%[0-9]+]]:_(s32) = G_CONSTANT i32 100
+  ; GFX10-NEXT:   [[ICMP3:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[PHI1]](s32), [[C12]]
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.7:
+  ; GFX10-NEXT:   successors: %bb.5(0x80000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[PHI6:%[0-9]+]]:_(s32) = G_PHI [[ADD1]](s32), %bb.6, [[DEF]](s32), %bb.4
+  ; GFX10-NEXT:   [[PHI7:%[0-9]+]]:_(s1) = G_PHI [[ICMP3]](s1), %bb.6, [[C7]](s1), %bb.4
+  ; GFX10-NEXT:   G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[SI_IF2]](s32)
+  ; GFX10-NEXT:   G_BR %bb.5
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.8:
+  ; GFX10-NEXT:   [[PHI8:%[0-9]+]]:_(s32) = G_PHI [[INTRINSIC_CONVERGENT]](s32), %bb.3
+  ; GFX10-NEXT:   G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[PHI8]](s32)
+  ; GFX10-NEXT:   S_ENDPGM 0
+  bb.0:
+    successors: %bb.1(0x80000000)
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5, $vgpr6, $vgpr7
+
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(p1) = G_MERGE_VALUES %0(s32), %1(s32)
+    %3:_(s32) = COPY $vgpr2
+    %4:_(s32) = COPY $vgpr3
+    %5:_(p1) = G_MERGE_VALUES %3(s32), %4(s32)
+    %6:_(s32) = COPY $vgpr4
+    %7:_(s32) = COPY $vgpr5
+    %8:_(p1) = G_MERGE_VALUES %6(s32), %7(s32)
+    %9:_(s32) = COPY $vgpr6
+    %10:_(s32) = COPY $vgpr7
+    %11:_(p1) = G_MERGE_VALUES %9(s32), %10(s32)
+    %12:_(s32) = G_CONSTANT i32 0
+    %13:_(s32) = G_IMPLICIT_DEF
+
+  bb.1:
+    successors: %bb.2(0x40000000), %bb.3(0x40000000)
+
+    %14:_(s32) = G_PHI %15(s32), %bb.3, %12(s32), %bb.0
+    %16:_(s32) = G_PHI %12(s32), %bb.0, %17(s32), %bb.3
+    %18:_(s1) = G_CONSTANT i1 true
+    %19:_(s64) = G_SEXT %16(s32)
+    %20:_(s32) = G_CONSTANT i32 2
+    %21:_(s64) = G_SHL %19, %20(s32)
+    %22:_(p1) = G_PTR_ADD %5, %21(s64)
+    %23:_(s32) = G_LOAD %22(p1) :: (load (s32), addrspace 1)
+    %24:_(s32) = G_CONSTANT i32 0
+    %25:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), %23(s32), %24
+    %26:sreg_32_xm0_xexec(s32) = SI_IF %25(s1), %bb.3, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.2
+
+  bb.2:
+    successors: %bb.4(0x40000000), %bb.5(0x40000000)
+
+    %27:_(s1) = G_CONSTANT i1 true
+    %28:_(s32) = G_CONSTANT i32 2
+    %29:_(s64) = G_SHL %19, %28(s32)
+    %30:_(p1) = G_PTR_ADD %8, %29(s64)
+    %31:_(s32) = G_LOAD %30(p1) :: (load (s32), addrspace 1)
+    %32:_(s32) = G_CONSTANT i32 0
+    %33:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), %31(s32), %32
+    %34:sreg_32_xm0_xexec(s32) = SI_IF %33(s1), %bb.5, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.4
+
+  bb.3:
+    successors: %bb.8(0x04000000), %bb.1(0x7c000000)
+
+    %17:_(s32) = G_PHI %35(s32), %bb.5, %13(s32), %bb.1
+    %36:_(s1) = G_PHI %37(s1), %bb.5, %18(s1), %bb.1
+    G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %26(s32)
+    %15:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.if.break), %36(s1), %14(s32)
+    SI_LOOP %15(s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.8
+
+  bb.4:
+    successors: %bb.6(0x40000000), %bb.7(0x40000000)
+
+    %38:_(s1) = G_CONSTANT i1 true
+    %39:_(s32) = G_CONSTANT i32 2
+    %40:_(s64) = G_SHL %19, %39(s32)
+    %41:_(p1) = G_PTR_ADD %11, %40(s64)
+    %42:_(s32) = G_LOAD %41(p1) :: (load (s32), addrspace 1)
+    %43:_(s32) = G_CONSTANT i32 0
+    %44:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), %42(s32), %43
+    %45:sreg_32_xm0_xexec(s32) = SI_IF %44(s1), %bb.7, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.6
+
+  bb.5:
+    successors: %bb.3(0x80000000)
+
+    %35:_(s32) = G_PHI %46(s32), %bb.7, %13(s32), %bb.2
+    %37:_(s1) = G_PHI %47(s1), %bb.7, %27(s1), %bb.2
+    G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %34(s32)
+    G_BR %bb.3
+
+  bb.6:
+    successors: %bb.7(0x80000000)
+
+    %48:_(s32) = G_CONSTANT i32 2
+    %49:_(s64) = G_SHL %19, %48(s32)
+    %50:_(p1) = G_PTR_ADD %2, %49(s64)
+    %51:_(s32) = G_LOAD %50(p1) :: (load (s32), addrspace 1)
+    %52:_(s32) = G_CONSTANT i32 1
+    %53:_(s32) = G_ADD %51, %52
+    G_STORE %53(s32), %50(p1) :: (store (s32), addrspace 1)
+    %54:_(s32) = G_ADD %16, %52
+    %55:_(s32) = G_CONSTANT i32 100
+    %56:_(s1) = G_ICMP intpred(ult), %16(s32), %55
+
+  bb.7:
+    successors: %bb.5(0x80000000)
+
+    %46:_(s32) = G_PHI %54(s32), %bb.6, %13(s32), %bb.4
+    %47:_(s1) = G_PHI %56(s1), %bb.6, %38(s1), %bb.4
+    G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %45(s32)
+    G_BR %bb.5
+
+  bb.8:
+    %57:_(s32) = G_PHI %15(s32), %bb.3
+    G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %57(s32)
+    S_ENDPGM 0
+...
+
+---
+name: loop_with_div_break_with_body
+legalized: true
+tracksRegLiveness: true
+body: |
+  ; GFX10-LABEL: name: loop_with_div_break_with_body
+  ; GFX10: bb.0:
+  ; GFX10-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX10-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GFX10-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GFX10-NEXT:   [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY]](s32), [[COPY1]](s32)
+  ; GFX10-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; GFX10-NEXT:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; GFX10-NEXT:   [[MV1:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY2]](s32), [[COPY3]](s32)
+  ; GFX10-NEXT:   [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; GFX10-NEXT:   [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
+  ; GFX10-NEXT:   [[MV2:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY4]](s32), [[COPY5]](s32)
+  ; GFX10-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.1:
+  ; GFX10-NEXT:   successors: %bb.3(0x40000000), %bb.5(0x40000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:_(s32) = G_PHI %12(s32), %bb.5, [[C]](s32), %bb.0
+  ; GFX10-NEXT:   [[PHI1:%[0-9]+]]:_(s32) = G_PHI [[C]](s32), %bb.0, %14(s32), %bb.5
+  ; GFX10-NEXT:   [[C1:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
+  ; GFX10-NEXT:   [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[PHI1]](s32)
+  ; GFX10-NEXT:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; GFX10-NEXT:   [[SHL:%[0-9]+]]:_(s64) = G_SHL [[SEXT]], [[C2]](s32)
+  ; GFX10-NEXT:   [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[MV1]], [[SHL]](s64)
+  ; GFX10-NEXT:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p1) :: (load (s32), addrspace 1)
+  ; GFX10-NEXT:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT:   [[ICMP:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), [[LOAD]](s32), [[C3]]
+  ; GFX10-NEXT:   [[SI_IF:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[ICMP]](s1), %bb.5, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX10-NEXT:   G_BR %bb.3
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.2:
+  ; GFX10-NEXT:   successors: %bb.4(0x80000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; GFX10-NEXT:   G_STORE [[C4]](s32), [[MV2]](p1) :: (store (s32), addrspace 1)
+  ; GFX10-NEXT:   G_BR %bb.4
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.3:
+  ; GFX10-NEXT:   successors: %bb.5(0x80000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[C5:%[0-9]+]]:_(s1) = G_CONSTANT i1 false
+  ; GFX10-NEXT:   [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; GFX10-NEXT:   [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[SEXT]], [[C6]](s32)
+  ; GFX10-NEXT:   [[PTR_ADD1:%[0-9]+]]:_(p1) = G_PTR_ADD [[MV]], [[SHL1]](s64)
+  ; GFX10-NEXT:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p1) :: (load (s32), addrspace 1)
+  ; GFX10-NEXT:   [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; GFX10-NEXT:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[LOAD1]], [[C7]]
+  ; GFX10-NEXT:   G_STORE [[ADD]](s32), [[PTR_ADD1]](p1) :: (store (s32), addrspace 1)
+  ; GFX10-NEXT:   [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[PHI1]], [[C7]]
+  ; GFX10-NEXT:   [[C8:%[0-9]+]]:_(s32) = G_CONSTANT i32 100
+  ; GFX10-NEXT:   [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[PHI1]](s32), [[C8]]
+  ; GFX10-NEXT:   G_BR %bb.5
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.4:
+  ; GFX10-NEXT:   G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %35(s32)
+  ; GFX10-NEXT:   S_ENDPGM 0
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.5:
+  ; GFX10-NEXT:   successors: %bb.6(0x04000000), %bb.1(0x7c000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[PHI2:%[0-9]+]]:_(s32) = G_PHI [[ADD1]](s32), %bb.3, [[DEF]](s32), %bb.1
+  ; GFX10-NEXT:   [[PHI3:%[0-9]+]]:_(s1) = G_PHI [[C5]](s1), %bb.3, [[C1]](s1), %bb.1
+  ; GFX10-NEXT:   [[PHI4:%[0-9]+]]:_(s1) = G_PHI [[ICMP1]](s1), %bb.3, [[C1]](s1), %bb.1
+  ; GFX10-NEXT:   G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[SI_IF]](s32)
+  ; GFX10-NEXT:   [[INTRINSIC_CONVERGENT:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.if.break), [[PHI4]](s1), [[PHI]](s32)
+  ; GFX10-NEXT:   SI_LOOP [[INTRINSIC_CONVERGENT]](s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX10-NEXT:   G_BR %bb.6
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.6:
+  ; GFX10-NEXT:   successors: %bb.2(0x40000000), %bb.4(0x40000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[PHI5:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_PHI [[PHI3]](s1), %bb.5
+  ; GFX10-NEXT:   [[PHI6:%[0-9]+]]:_(s32) = G_PHI [[INTRINSIC_CONVERGENT]](s32), %bb.5
+  ; GFX10-NEXT:   G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[PHI6]](s32)
+  ; GFX10-NEXT:   [[SI_IF1:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[PHI5]](s1), %bb.4, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX10-NEXT:   G_BR %bb.2
+  bb.0:
+    successors: %bb.1(0x80000000)
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(p1) = G_MERGE_VALUES %0(s32), %1(s32)
+    %3:_(s32) = COPY $vgpr2
+    %4:_(s32) = COPY $vgpr3
+    %5:_(p1) = G_MERGE_VALUES %3(s32), %4(s32)
+    %6:_(s32) = COPY $vgpr4
+    %7:_(s32) = COPY $vgpr5
+    %8:_(p1) = G_MERGE_VALUES %6(s32), %7(s32)
+    %9:_(s32) = G_CONSTANT i32 0
+    %10:_(s32) = G_IMPLICIT_DEF
+
+  bb.1:
+    successors: %bb.3(0x40000000), %bb.5(0x40000000)
+
+    %11:_(s32) = G_PHI %12(s32), %bb.5, %9(s32), %bb.0
+    %13:_(s32) = G_PHI %9(s32), %bb.0, %14(s32), %bb.5
+    %15:_(s1) = G_CONSTANT i1 true
+    %16:_(s64) = G_SEXT %13(s32)
+    %17:_(s32) = G_CONSTANT i32 2
+    %18:_(s64) = G_SHL %16, %17(s32)
+    %19:_(p1) = G_PTR_ADD %5, %18(s64)
+    %20:_(s32) = G_LOAD %19(p1) :: (load (s32), addrspace 1)
+    %21:_(s32) = G_CONSTANT i32 0
+    %22:sreg_32_xm0_xexec(s1) = G_ICMP intpred(ne), %20(s32), %21
+    %23:sreg_32_xm0_xexec(s32) = SI_IF %22(s1), %bb.5, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.3
+
+  bb.2:
+    successors: %bb.4(0x80000000)
+
+    %24:_(s32) = G_CONSTANT i32 10
+    G_STORE %24(s32), %8(p1) :: (store (s32), addrspace 1)
+    G_BR %bb.4
+
+  bb.3:
+    successors: %bb.5(0x80000000)
+
+    %25:_(s1) = G_CONSTANT i1 false
+    %26:_(s32) = G_CONSTANT i32 2
+    %27:_(s64) = G_SHL %16, %26(s32)
+    %28:_(p1) = G_PTR_ADD %2, %27(s64)
+    %29:_(s32) = G_LOAD %28(p1) :: (load (s32), addrspace 1)
+    %30:_(s32) = G_CONSTANT i32 1
+    %31:_(s32) = G_ADD %29, %30
+    G_STORE %31(s32), %28(p1) :: (store (s32), addrspace 1)
+    %32:_(s32) = G_ADD %13, %30
+    %33:_(s32) = G_CONSTANT i32 100
+    %34:_(s1) = G_ICMP intpred(ult), %13(s32), %33
+    G_BR %bb.5
+
+  bb.4:
+    G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %35(s32)
+    S_ENDPGM 0
+
+  bb.5:
+    successors: %bb.6(0x04000000), %bb.1(0x7c000000)
+
+    %14:_(s32) = G_PHI %32(s32), %bb.3, %10(s32), %bb.1
+    %36:_(s1) = G_PHI %25(s1), %bb.3, %15(s1), %bb.1
+    %37:_(s1) = G_PHI %34(s1), %bb.3, %15(s1), %bb.1
+    G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %23(s32)
+    %12:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.if.break), %37(s1), %11(s32)
+    SI_LOOP %12(s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.6
+
+  bb.6:
+    successors: %bb.2(0x40000000), %bb.4(0x40000000)
+
+    %38:sreg_32_xm0_xexec(s1) = G_PHI %36(s1), %bb.5
+    %39:_(s32) = G_PHI %12(s32), %bb.5
+    G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %39(s32)
+    %35:sreg_32_xm0_xexec(s32) = SI_IF %38(s1), %bb.4, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.2
+...
+
+---
+name: irreducible_cfg
+legalized: true
+tracksRegLiveness: true
+body: |
+  ; GFX10-LABEL: name: irreducible_cfg
+  ; GFX10: bb.0:
+  ; GFX10-NEXT:   successors: %bb.7(0x80000000)
+  ; GFX10-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GFX10-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GFX10-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; GFX10-NEXT:   [[COPY3:%[0-9]+]]:_(s32) = COPY $vgpr3
+  ; GFX10-NEXT:   [[COPY4:%[0-9]+]]:_(s32) = COPY $vgpr4
+  ; GFX10-NEXT:   [[COPY5:%[0-9]+]]:_(s32) = COPY $vgpr5
+  ; GFX10-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT:   [[DEF:%[0-9]+]]:_(s1) = G_IMPLICIT_DEF
+  ; GFX10-NEXT:   [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(sgt), [[COPY4]](s32), [[COPY1]]
+  ; GFX10-NEXT:   G_BR %bb.7
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.1:
+  ; GFX10-NEXT:   successors: %bb.3(0x80000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT:   [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(sle), [[COPY4]](s32), [[COPY]]
+  ; GFX10-NEXT:   G_BR %bb.3
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.2:
+  ; GFX10-NEXT:   successors: %bb.4(0x40000000), %bb.7(0x40000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:_(s1) = G_PHI %12(s1), %bb.6, [[DEF]](s1), %bb.7
+  ; GFX10-NEXT:   [[PHI1:%[0-9]+]]:_(s1) = G_PHI %12(s1), %bb.6, %14(s1), %bb.7
+  ; GFX10-NEXT:   G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %15(s32)
+  ; GFX10-NEXT:   [[INTRINSIC_CONVERGENT:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.if.break), [[PHI1]](s1), %17(s32)
+  ; GFX10-NEXT:   SI_LOOP [[INTRINSIC_CONVERGENT]](s32), %bb.7, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX10-NEXT:   G_BR %bb.4
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.3:
+  ; GFX10-NEXT:   successors: %bb.6(0x04000000), %bb.3(0x7c000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[PHI2:%[0-9]+]]:_(s32) = G_PHI [[C1]](s32), %bb.1, %19(s32), %bb.3
+  ; GFX10-NEXT:   [[INTRINSIC_CONVERGENT1:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.if.break), [[ICMP1]](s1), [[PHI2]](s32)
+  ; GFX10-NEXT:   SI_LOOP [[INTRINSIC_CONVERGENT1]](s32), %bb.3, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX10-NEXT:   G_BR %bb.6
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.4:
+  ; GFX10-NEXT:   successors: %bb.5(0x04000000), %bb.7(0x7c000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[INTRINSIC_CONVERGENT]](s32)
+  ; GFX10-NEXT:   [[ICMP2:%[0-9]+]]:_(s1) = G_ICMP intpred(sgt), [[COPY5]](s32), [[COPY]]
+  ; GFX10-NEXT:   [[C2:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
+  ; GFX10-NEXT:   [[XOR:%[0-9]+]]:_(s1) = G_XOR [[ICMP]], [[C2]]
+  ; GFX10-NEXT:   [[OR:%[0-9]+]]:_(s1) = G_OR [[ICMP2]], [[XOR]]
+  ; GFX10-NEXT:   [[INTRINSIC_CONVERGENT2:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.if.break), [[OR]](s1), %25(s32)
+  ; GFX10-NEXT:   SI_LOOP [[INTRINSIC_CONVERGENT2]](s32), %bb.7, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX10-NEXT:   G_BR %bb.5
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.5:
+  ; GFX10-NEXT:   [[PHI3:%[0-9]+]]:_(s1) = G_PHI [[ICMP2]](s1), %bb.4
+  ; GFX10-NEXT:   [[PHI4:%[0-9]+]]:_(s32) = G_PHI [[INTRINSIC_CONVERGENT2]](s32), %bb.4
+  ; GFX10-NEXT:   G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[PHI4]](s32)
+  ; GFX10-NEXT:   [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[PHI3]](s1), [[COPY3]], [[COPY2]]
+  ; GFX10-NEXT:   [[INTRINSIC_CONVERGENT3:%[0-9]+]]:_(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), [[SELECT]](s32)
+  ; GFX10-NEXT:   $sgpr0 = COPY [[INTRINSIC_CONVERGENT3]](s32)
+  ; GFX10-NEXT:   SI_RETURN_TO_EPILOG implicit $sgpr0
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.6:
+  ; GFX10-NEXT:   successors: %bb.2(0x80000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[PHI5:%[0-9]+]]:_(s32) = G_PHI [[INTRINSIC_CONVERGENT1]](s32), %bb.3
+  ; GFX10-NEXT:   [[C3:%[0-9]+]]:_(s1) = G_CONSTANT i1 false
+  ; GFX10-NEXT:   G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[PHI5]](s32)
+  ; GFX10-NEXT:   G_BR %bb.2
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.7:
+  ; GFX10-NEXT:   successors: %bb.1(0x40000000), %bb.2(0x40000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[PHI6:%[0-9]+]]:_(s32) = G_PHI [[INTRINSIC_CONVERGENT2]](s32), %bb.4, [[PHI6]](s32), %bb.2, [[C]](s32), %bb.0
+  ; GFX10-NEXT:   [[PHI7:%[0-9]+]]:_(s32) = G_PHI [[C]](s32), %bb.4, [[INTRINSIC_CONVERGENT]](s32), %bb.2, [[C]](s32), %bb.0
+  ; GFX10-NEXT:   [[PHI8:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_PHI [[ICMP]](s1), %bb.0, [[PHI]](s1), %bb.2, [[C2]](s1), %bb.4
+  ; GFX10-NEXT:   [[C4:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
+  ; GFX10-NEXT:   [[SI_IF:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[PHI8]](s1), %bb.2, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX10-NEXT:   G_BR %bb.1
+  bb.0:
+    successors: %bb.7(0x80000000)
+    liveins: $vgpr0, $vgpr1, $vgpr2, $vgpr3, $vgpr4, $vgpr5
+
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $vgpr2
+    %3:_(s32) = COPY $vgpr3
+    %4:_(s32) = COPY $vgpr4
+    %5:_(s32) = COPY $vgpr5
+    %6:_(s32) = G_CONSTANT i32 0
+    %7:_(s1) = G_IMPLICIT_DEF
+    %8:_(s1) = G_ICMP intpred(sgt), %4(s32), %1
+    G_BR %bb.7
+
+  bb.1:
+    successors: %bb.3(0x80000000)
+
+    %9:_(s32) = G_CONSTANT i32 0
+    %10:_(s1) = G_ICMP intpred(sle), %4(s32), %0
+    G_BR %bb.3
+
+  bb.2:
+    successors: %bb.4(0x40000000), %bb.7(0x40000000)
+
+    %11:_(s1) = G_PHI %12(s1), %bb.6, %7(s1), %bb.7
+    %13:_(s1) = G_PHI %12(s1), %bb.6, %14(s1), %bb.7
+    G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %15(s32)
+    %16:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.if.break), %13(s1), %17(s32)
+    SI_LOOP %16(s32), %bb.7, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.4
+
+  bb.3:
+    successors: %bb.6(0x04000000), %bb.3(0x7c000000)
+
+    %18:_(s32) = G_PHI %9(s32), %bb.1, %19(s32), %bb.3
+    %19:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.if.break), %10(s1), %18(s32)
+    SI_LOOP %19(s32), %bb.3, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.6
+
+  bb.4:
+    successors: %bb.5(0x04000000), %bb.7(0x7c000000)
+
+    G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %16(s32)
+    %20:_(s1) = G_ICMP intpred(sgt), %5(s32), %0
+    %21:_(s1) = G_CONSTANT i1 true
+    %22:_(s1) = G_XOR %8, %21
+    %23:_(s1) = G_OR %20, %22
+    %24:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.if.break), %23(s1), %25(s32)
+    SI_LOOP %24(s32), %bb.7, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.5
+
+  bb.5:
+    %26:_(s1) = G_PHI %20(s1), %bb.4
+    %27:_(s32) = G_PHI %24(s32), %bb.4
+    G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %27(s32)
+    %28:_(s32) = G_SELECT %26(s1), %3, %2
+    %29:_(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.readfirstlane), %28(s32)
+    $sgpr0 = COPY %29(s32)
+    SI_RETURN_TO_EPILOG implicit $sgpr0
+
+  bb.6:
+    successors: %bb.2(0x80000000)
+
+    %30:_(s32) = G_PHI %19(s32), %bb.3
+    %12:_(s1) = G_CONSTANT i1 false
+    G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %30(s32)
+    G_BR %bb.2
+
+  bb.7:
+    successors: %bb.1(0x40000000), %bb.2(0x40000000)
+
+    %25:_(s32) = G_PHI %24(s32), %bb.4, %25(s32), %bb.2, %6(s32), %bb.0
+    %17:_(s32) = G_PHI %6(s32), %bb.4, %16(s32), %bb.2, %6(s32), %bb.0
+    %31:sreg_32_xm0_xexec(s1) = G_PHI %8(s1), %bb.0, %11(s1), %bb.2, %21(s1), %bb.4
+    %14:_(s1) = G_CONSTANT i1 true
+    %15:sreg_32_xm0_xexec(s32) = SI_IF %31(s1), %bb.2, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.1
+...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-temporal-divergent-i1.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-temporal-divergent-i1.ll
new file mode 100644
index 0000000000000000000000000000000000000000..54881f59096c1e1c10d72fe655dae9481e932f4b
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-temporal-divergent-i1.ll
@@ -0,0 +1,171 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
+; RUN: llc -global-isel -amdgpu-global-isel-risky-select -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 < %s | FileCheck -check-prefix=GFX10 %s
+
+define void @temporal_divergent_i1_phi(float %val, ptr %addr) {
+; GFX10-LABEL: temporal_divergent_i1_phi:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_mov_b32 s4, 0
+; GFX10-NEXT:    v_mov_b32_e32 v4, 1
+; GFX10-NEXT:    v_mov_b32_e32 v3, s4
+; GFX10-NEXT:  .LBB0_1: ; %loop
+; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX10-NEXT:    v_cvt_f32_u32_e32 v6, v3
+; GFX10-NEXT:    v_mov_b32_e32 v5, v4
+; GFX10-NEXT:    v_add_nc_u32_e32 v3, 1, v3
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v6, v0
+; GFX10-NEXT:    v_xor_b32_e32 v4, 1, v5
+; GFX10-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT:    s_cbranch_execnz .LBB0_1
+; GFX10-NEXT:  ; %bb.2: ; %exit
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT:    v_and_b32_e32 v0, 1, v5
+; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v0
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, vcc_lo
+; GFX10-NEXT:    flat_store_dword v[1:2], v0
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+entry:
+  br label %loop
+
+loop:
+  %counter = phi i32 [ 0, %entry ], [ %counterPlus1, %loop ]
+  %bool_counter = phi i1 [ true, %entry ], [ %neg_bool_counter, %loop ]
+  %neg_bool_counter = xor i1 %bool_counter, true
+  %fcounter = uitofp i32 %counter to float
+  %cond = fcmp ogt float %fcounter, %val
+  %counterPlus1 = add i32 %counter, 1
+  br i1 %cond, label %exit, label %loop
+
+exit:
+  %select = select i1 %bool_counter, float 1.000000e+00, float 0.000000e+00
+  store float %select, ptr %addr
+  ret void
+}
+
+define void @temporal_divergent_i1_non_phi(float %val, ptr %addr) {
+; GFX10-LABEL: temporal_divergent_i1_non_phi:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_mov_b32 s4, 0
+; GFX10-NEXT:    v_mov_b32_e32 v3, 1
+; GFX10-NEXT:    v_mov_b32_e32 v4, s4
+; GFX10-NEXT:  .LBB1_1: ; %loop
+; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX10-NEXT:    v_cvt_f32_u32_e32 v5, v4
+; GFX10-NEXT:    v_xor_b32_e32 v3, 1, v3
+; GFX10-NEXT:    v_add_nc_u32_e32 v4, 1, v4
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v5, v0
+; GFX10-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT:    s_cbranch_execnz .LBB1_1
+; GFX10-NEXT:  ; %bb.2: ; %exit
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT:    v_and_b32_e32 v0, 1, v3
+; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v0
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, 0, 1.0, vcc_lo
+; GFX10-NEXT:    flat_store_dword v[1:2], v0
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+entry:
+  br label %loop
+
+loop:
+  %counter = phi i32 [ 0, %entry ], [ %counterPlus1, %loop ]
+  %bool_counter = phi i1 [ true, %entry ], [ %neg_bool_counter, %loop ]
+  %neg_bool_counter = xor i1 %bool_counter, true
+  %fcounter = uitofp i32 %counter to float
+  %cond = fcmp ogt float %fcounter, %val
+  %counterPlus1 = add i32 %counter, 1
+  br i1 %cond, label %exit, label %loop
+
+exit:
+  %select = select i1 %neg_bool_counter, float 1.000000e+00, float 0.000000e+00
+  store float %select, ptr %addr
+  ret void
+}
+
+; This is temporal divergent uniform i1 structurize-cfg phi.
+; Loop has uniform condition if with body with break at the end.
+define amdgpu_cs void @loop_with_1break(ptr addrspace(1) %x, i32 %x.size, ptr addrspace(1) inreg %a, ptr addrspace(1) inreg %a.break) {
+; GFX10-LABEL: loop_with_1break:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:    s_mov_b32 s4, 0
+; GFX10-NEXT:    v_mov_b32_e32 v4, s1
+; GFX10-NEXT:    v_mov_b32_e32 v3, s0
+; GFX10-NEXT:    v_mov_b32_e32 v5, s4
+; GFX10-NEXT:    s_branch .LBB2_3
+; GFX10-NEXT:  .LBB2_1: ; %loop.body
+; GFX10-NEXT:    ; in Loop: Header=BB2_3 Depth=1
+; GFX10-NEXT:    v_add_co_u32 v6, vcc_lo, v0, v6
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, v1, v7, vcc_lo
+; GFX10-NEXT:    v_add_nc_u32_e32 v9, 1, v5
+; GFX10-NEXT:    v_cmp_lt_u32_e64 s0, v5, v2
+; GFX10-NEXT:    s_mov_b32 s1, 0
+; GFX10-NEXT:    global_load_dword v8, v[6:7], off
+; GFX10-NEXT:    v_mov_b32_e32 v5, v9
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_add_nc_u32_e32 v8, 1, v8
+; GFX10-NEXT:    global_store_dword v[6:7], v8, off
+; GFX10-NEXT:  .LBB2_2: ; %Flow
+; GFX10-NEXT:    ; in Loop: Header=BB2_3 Depth=1
+; GFX10-NEXT:    s_and_b32 s0, exec_lo, s0
+; GFX10-NEXT:    s_or_b32 s4, s0, s4
+; GFX10-NEXT:    s_and_b32 s0, 1, s1
+; GFX10-NEXT:    v_cmp_ne_u32_e64 s0, 0, s0
+; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT:    s_cbranch_execz .LBB2_5
+; GFX10-NEXT:  .LBB2_3: ; %A
+; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX10-NEXT:    v_ashrrev_i32_e32 v6, 31, v5
+; GFX10-NEXT:    v_lshlrev_b64 v[6:7], 2, v[5:6]
+; GFX10-NEXT:    v_add_co_u32 v8, vcc_lo, v3, v6
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v9, vcc_lo, v4, v7, vcc_lo
+; GFX10-NEXT:    global_load_dword v8, v[8:9], off
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    v_cmp_ne_u32_e32 vcc_lo, 0, v8
+; GFX10-NEXT:    s_cbranch_vccnz .LBB2_1
+; GFX10-NEXT:  ; %bb.4: ; in Loop: Header=BB2_3 Depth=1
+; GFX10-NEXT:    s_mov_b32 s0, -1
+; GFX10-NEXT:    s_mov_b32 s1, 1
+; GFX10-NEXT:    ; implicit-def: $vgpr5
+; GFX10-NEXT:    s_branch .LBB2_2
+; GFX10-NEXT:  .LBB2_5: ; %loop.exit.guard
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT:    s_and_saveexec_b32 s1, s0
+; GFX10-NEXT:    s_xor_b32 s1, exec_lo, s1
+; GFX10-NEXT:    s_cbranch_execz .LBB2_7
+; GFX10-NEXT:  ; %bb.6: ; %break.body
+; GFX10-NEXT:    v_mov_b32_e32 v0, 10
+; GFX10-NEXT:    v_mov_b32_e32 v1, 0
+; GFX10-NEXT:    global_store_dword v1, v0, s[2:3]
+; GFX10-NEXT:  .LBB2_7: ; %exit
+; GFX10-NEXT:    s_endpgm
+entry:
+  br label %A
+
+A:
+  %counter = phi i32 [ %counter.plus.1, %loop.body ], [ 0, %entry ]
+  %a.plus.counter = getelementptr inbounds i32, ptr addrspace(1) %a, i32 %counter
+  %a.val = load i32, ptr addrspace(1) %a.plus.counter
+  %a.cond = icmp eq i32 %a.val, 0
+  br i1 %a.cond, label %break.body, label %loop.body
+
+break.body:
+  store i32 10, ptr addrspace(1) %a.break
+  br label %exit
+
+loop.body:
+  %x.plus.counter = getelementptr inbounds i32, ptr addrspace(1) %x, i32 %counter
+  %x.val = load i32, ptr addrspace(1) %x.plus.counter
+  %x.val.plus.1 = add i32 %x.val, 1
+  store i32 %x.val.plus.1, ptr addrspace(1) %x.plus.counter
+  %counter.plus.1 = add i32 %counter, 1
+  %x.cond = icmp ult i32 %counter, %x.size
+  br i1 %x.cond, label %exit, label %A
+
+exit:
+  ret void
+}
+
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-temporal-divergent-i1.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-temporal-divergent-i1.mir
new file mode 100644
index 0000000000000000000000000000000000000000..9c2d083d0aa1d88d72fe3248c027a411e5385319
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-temporal-divergent-i1.mir
@@ -0,0 +1,324 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 4
+# RUN: llc -global-isel -mtriple=amdgcn-mesa-amdpal -mcpu=gfx1010 -run-pass=amdgpu-global-isel-divergence-lowering %s -o - | FileCheck -check-prefix=GFX10 %s
+
+---
+name: temporal_divergent_i1_phi
+legalized: true
+tracksRegLiveness: true
+body: |
+  ; GFX10-LABEL: name: temporal_divergent_i1_phi
+  ; GFX10: bb.0:
+  ; GFX10-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX10-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GFX10-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GFX10-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; GFX10-NEXT:   [[MV:%[0-9]+]]:_(p0) = G_MERGE_VALUES [[COPY1]](s32), [[COPY2]](s32)
+  ; GFX10-NEXT:   [[C:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
+  ; GFX10-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.1:
+  ; GFX10-NEXT:   successors: %bb.2(0x04000000), %bb.1(0x7c000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:_(s32) = G_PHI %7(s32), %bb.1, [[C1]](s32), %bb.0
+  ; GFX10-NEXT:   [[PHI1:%[0-9]+]]:_(s32) = G_PHI [[C1]](s32), %bb.0, %9(s32), %bb.1
+  ; GFX10-NEXT:   [[PHI2:%[0-9]+]]:_(s1) = G_PHI [[C]](s1), %bb.0, %11(s1), %bb.1
+  ; GFX10-NEXT:   [[C2:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
+  ; GFX10-NEXT:   [[XOR:%[0-9]+]]:_(s1) = G_XOR [[PHI2]], [[C2]]
+  ; GFX10-NEXT:   [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[PHI1]](s32)
+  ; GFX10-NEXT:   [[FCMP:%[0-9]+]]:_(s1) = G_FCMP floatpred(ogt), [[UITOFP]](s32), [[COPY]]
+  ; GFX10-NEXT:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; GFX10-NEXT:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[PHI1]], [[C3]]
+  ; GFX10-NEXT:   [[INTRINSIC_CONVERGENT:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.if.break), [[FCMP]](s1), [[PHI]](s32)
+  ; GFX10-NEXT:   SI_LOOP [[INTRINSIC_CONVERGENT]](s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX10-NEXT:   G_BR %bb.2
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.2:
+  ; GFX10-NEXT:   [[PHI3:%[0-9]+]]:_(s1) = G_PHI [[PHI2]](s1), %bb.1
+  ; GFX10-NEXT:   [[PHI4:%[0-9]+]]:_(s32) = G_PHI [[INTRINSIC_CONVERGENT]](s32), %bb.1
+  ; GFX10-NEXT:   G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[PHI4]](s32)
+  ; GFX10-NEXT:   [[C4:%[0-9]+]]:_(s32) = G_FCONSTANT float 0.000000e+00
+  ; GFX10-NEXT:   [[C5:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.000000e+00
+  ; GFX10-NEXT:   [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[PHI3]](s1), [[C5]], [[C4]]
+  ; GFX10-NEXT:   G_STORE [[SELECT]](s32), [[MV]](p0) :: (store (s32))
+  ; GFX10-NEXT:   SI_RETURN
+  bb.0:
+    successors: %bb.1(0x80000000)
+    liveins: $vgpr0, $vgpr1, $vgpr2
+
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $vgpr2
+    %3:_(p0) = G_MERGE_VALUES %1(s32), %2(s32)
+    %4:_(s1) = G_CONSTANT i1 true
+    %5:_(s32) = G_CONSTANT i32 0
+
+  bb.1:
+    successors: %bb.2(0x04000000), %bb.1(0x7c000000)
+
+    %6:_(s32) = G_PHI %7(s32), %bb.1, %5(s32), %bb.0
+    %8:_(s32) = G_PHI %5(s32), %bb.0, %9(s32), %bb.1
+    %10:_(s1) = G_PHI %4(s1), %bb.0, %11(s1), %bb.1
+    %12:_(s1) = G_CONSTANT i1 true
+    %11:_(s1) = G_XOR %10, %12
+    %13:_(s32) = G_UITOFP %8(s32)
+    %14:_(s1) = G_FCMP floatpred(ogt), %13(s32), %0
+    %15:_(s32) = G_CONSTANT i32 1
+    %9:_(s32) = G_ADD %8, %15
+    %7:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.if.break), %14(s1), %6(s32)
+    SI_LOOP %7(s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.2
+
+  bb.2:
+    %16:_(s1) = G_PHI %10(s1), %bb.1
+    %17:_(s32) = G_PHI %7(s32), %bb.1
+    G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %17(s32)
+    %18:_(s32) = G_FCONSTANT float 0.000000e+00
+    %19:_(s32) = G_FCONSTANT float 1.000000e+00
+    %20:_(s32) = G_SELECT %16(s1), %19, %18
+    G_STORE %20(s32), %3(p0) :: (store (s32))
+    SI_RETURN
+...
+
+---
+name: temporal_divergent_i1_non_phi
+legalized: true
+tracksRegLiveness: true
+body: |
+  ; GFX10-LABEL: name: temporal_divergent_i1_non_phi
+  ; GFX10: bb.0:
+  ; GFX10-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX10-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GFX10-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GFX10-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; GFX10-NEXT:   [[MV:%[0-9]+]]:_(p0) = G_MERGE_VALUES [[COPY1]](s32), [[COPY2]](s32)
+  ; GFX10-NEXT:   [[C:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
+  ; GFX10-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.1:
+  ; GFX10-NEXT:   successors: %bb.2(0x04000000), %bb.1(0x7c000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:_(s32) = G_PHI %7(s32), %bb.1, [[C1]](s32), %bb.0
+  ; GFX10-NEXT:   [[PHI1:%[0-9]+]]:_(s32) = G_PHI [[C1]](s32), %bb.0, %9(s32), %bb.1
+  ; GFX10-NEXT:   [[PHI2:%[0-9]+]]:_(s1) = G_PHI [[C]](s1), %bb.0, %11(s1), %bb.1
+  ; GFX10-NEXT:   [[C2:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
+  ; GFX10-NEXT:   [[XOR:%[0-9]+]]:_(s1) = G_XOR [[PHI2]], [[C2]]
+  ; GFX10-NEXT:   [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[PHI1]](s32)
+  ; GFX10-NEXT:   [[FCMP:%[0-9]+]]:_(s1) = G_FCMP floatpred(ogt), [[UITOFP]](s32), [[COPY]]
+  ; GFX10-NEXT:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; GFX10-NEXT:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[PHI1]], [[C3]]
+  ; GFX10-NEXT:   [[INTRINSIC_CONVERGENT:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.if.break), [[FCMP]](s1), [[PHI]](s32)
+  ; GFX10-NEXT:   SI_LOOP [[INTRINSIC_CONVERGENT]](s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX10-NEXT:   G_BR %bb.2
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.2:
+  ; GFX10-NEXT:   [[PHI3:%[0-9]+]]:_(s1) = G_PHI [[XOR]](s1), %bb.1
+  ; GFX10-NEXT:   [[PHI4:%[0-9]+]]:_(s32) = G_PHI [[INTRINSIC_CONVERGENT]](s32), %bb.1
+  ; GFX10-NEXT:   G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[PHI4]](s32)
+  ; GFX10-NEXT:   [[C4:%[0-9]+]]:_(s32) = G_FCONSTANT float 0.000000e+00
+  ; GFX10-NEXT:   [[C5:%[0-9]+]]:_(s32) = G_FCONSTANT float 1.000000e+00
+  ; GFX10-NEXT:   [[SELECT:%[0-9]+]]:_(s32) = G_SELECT [[PHI3]](s1), [[C5]], [[C4]]
+  ; GFX10-NEXT:   G_STORE [[SELECT]](s32), [[MV]](p0) :: (store (s32))
+  ; GFX10-NEXT:   SI_RETURN
+  bb.0:
+    successors: %bb.1(0x80000000)
+    liveins: $vgpr0, $vgpr1, $vgpr2
+
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $vgpr2
+    %3:_(p0) = G_MERGE_VALUES %1(s32), %2(s32)
+    %4:_(s1) = G_CONSTANT i1 true
+    %5:_(s32) = G_CONSTANT i32 0
+
+  bb.1:
+    successors: %bb.2(0x04000000), %bb.1(0x7c000000)
+
+    %6:_(s32) = G_PHI %7(s32), %bb.1, %5(s32), %bb.0
+    %8:_(s32) = G_PHI %5(s32), %bb.0, %9(s32), %bb.1
+    %10:_(s1) = G_PHI %4(s1), %bb.0, %11(s1), %bb.1
+    %12:_(s1) = G_CONSTANT i1 true
+    %11:_(s1) = G_XOR %10, %12
+    %13:_(s32) = G_UITOFP %8(s32)
+    %14:_(s1) = G_FCMP floatpred(ogt), %13(s32), %0
+    %15:_(s32) = G_CONSTANT i32 1
+    %9:_(s32) = G_ADD %8, %15
+    %7:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.if.break), %14(s1), %6(s32)
+    SI_LOOP %7(s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.2
+
+  bb.2:
+    %16:_(s1) = G_PHI %11(s1), %bb.1
+    %17:_(s32) = G_PHI %7(s32), %bb.1
+    G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %17(s32)
+    %18:_(s32) = G_FCONSTANT float 0.000000e+00
+    %19:_(s32) = G_FCONSTANT float 1.000000e+00
+    %20:_(s32) = G_SELECT %16(s1), %19, %18
+    G_STORE %20(s32), %3(p0) :: (store (s32))
+    SI_RETURN
+...
+
+---
+name: loop_with_1break
+legalized: true
+tracksRegLiveness: true
+body: |
+  ; GFX10-LABEL: name: loop_with_1break
+  ; GFX10: bb.0:
+  ; GFX10-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX10-NEXT:   liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GFX10-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GFX10-NEXT:   [[MV:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY]](s32), [[COPY1]](s32)
+  ; GFX10-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; GFX10-NEXT:   [[COPY3:%[0-9]+]]:_(s32) = COPY $sgpr0
+  ; GFX10-NEXT:   [[COPY4:%[0-9]+]]:_(s32) = COPY $sgpr1
+  ; GFX10-NEXT:   [[MV1:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY3]](s32), [[COPY4]](s32)
+  ; GFX10-NEXT:   [[COPY5:%[0-9]+]]:_(s32) = COPY $sgpr2
+  ; GFX10-NEXT:   [[COPY6:%[0-9]+]]:_(s32) = COPY $sgpr3
+  ; GFX10-NEXT:   [[MV2:%[0-9]+]]:_(p1) = G_MERGE_VALUES [[COPY5]](s32), [[COPY6]](s32)
+  ; GFX10-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT:   [[DEF:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.1:
+  ; GFX10-NEXT:   successors: %bb.3(0x50000000), %bb.5(0x30000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:_(s32) = G_PHI %13(s32), %bb.5, [[C]](s32), %bb.0
+  ; GFX10-NEXT:   [[PHI1:%[0-9]+]]:_(s32) = G_PHI [[C]](s32), %bb.0, %15(s32), %bb.5
+  ; GFX10-NEXT:   [[C1:%[0-9]+]]:_(s1) = G_CONSTANT i1 true
+  ; GFX10-NEXT:   [[SEXT:%[0-9]+]]:_(s64) = G_SEXT [[PHI1]](s32)
+  ; GFX10-NEXT:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; GFX10-NEXT:   [[SHL:%[0-9]+]]:_(s64) = G_SHL [[SEXT]], [[C2]](s32)
+  ; GFX10-NEXT:   [[PTR_ADD:%[0-9]+]]:_(p1) = G_PTR_ADD [[MV1]], [[SHL]](s64)
+  ; GFX10-NEXT:   [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p1) :: (load (s32), addrspace 1)
+  ; GFX10-NEXT:   [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT:   [[ICMP:%[0-9]+]]:_(s1) = G_ICMP intpred(ne), [[LOAD]](s32), [[C3]]
+  ; GFX10-NEXT:   G_BRCOND [[ICMP]](s1), %bb.3
+  ; GFX10-NEXT:   G_BR %bb.5
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.2:
+  ; GFX10-NEXT:   successors: %bb.4(0x80000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[C4:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
+  ; GFX10-NEXT:   G_STORE [[C4]](s32), [[MV2]](p1) :: (store (s32), addrspace 1)
+  ; GFX10-NEXT:   G_BR %bb.4
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.3:
+  ; GFX10-NEXT:   successors: %bb.5(0x80000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[C5:%[0-9]+]]:_(s1) = G_CONSTANT i1 false
+  ; GFX10-NEXT:   [[C6:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+  ; GFX10-NEXT:   [[SHL1:%[0-9]+]]:_(s64) = G_SHL [[SEXT]], [[C6]](s32)
+  ; GFX10-NEXT:   [[PTR_ADD1:%[0-9]+]]:_(p1) = G_PTR_ADD [[MV]], [[SHL1]](s64)
+  ; GFX10-NEXT:   [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD1]](p1) :: (load (s32), addrspace 1)
+  ; GFX10-NEXT:   [[C7:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; GFX10-NEXT:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[LOAD1]], [[C7]]
+  ; GFX10-NEXT:   G_STORE [[ADD]](s32), [[PTR_ADD1]](p1) :: (store (s32), addrspace 1)
+  ; GFX10-NEXT:   [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[PHI1]], [[C7]]
+  ; GFX10-NEXT:   [[ICMP1:%[0-9]+]]:_(s1) = G_ICMP intpred(ult), [[PHI1]](s32), [[COPY2]]
+  ; GFX10-NEXT:   G_BR %bb.5
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.4:
+  ; GFX10-NEXT:   G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %34(s32)
+  ; GFX10-NEXT:   S_ENDPGM 0
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.5:
+  ; GFX10-NEXT:   successors: %bb.6(0x04000000), %bb.1(0x7c000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[PHI2:%[0-9]+]]:_(s32) = G_PHI [[ADD1]](s32), %bb.3, [[DEF]](s32), %bb.1
+  ; GFX10-NEXT:   [[PHI3:%[0-9]+]]:_(s1) = G_PHI [[C5]](s1), %bb.3, [[C1]](s1), %bb.1
+  ; GFX10-NEXT:   [[PHI4:%[0-9]+]]:_(s1) = G_PHI [[ICMP1]](s1), %bb.3, [[C1]](s1), %bb.1
+  ; GFX10-NEXT:   [[INTRINSIC_CONVERGENT:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.if.break), [[PHI4]](s1), [[PHI]](s32)
+  ; GFX10-NEXT:   SI_LOOP [[INTRINSIC_CONVERGENT]](s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX10-NEXT:   G_BR %bb.6
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.6:
+  ; GFX10-NEXT:   successors: %bb.2(0x40000000), %bb.4(0x40000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[PHI5:%[0-9]+]]:sreg_32_xm0_xexec(s1) = G_PHI [[PHI3]](s1), %bb.5
+  ; GFX10-NEXT:   [[PHI6:%[0-9]+]]:_(s32) = G_PHI [[INTRINSIC_CONVERGENT]](s32), %bb.5
+  ; GFX10-NEXT:   G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[PHI6]](s32)
+  ; GFX10-NEXT:   [[SI_IF:%[0-9]+]]:sreg_32_xm0_xexec(s32) = SI_IF [[PHI5]](s1), %bb.4, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX10-NEXT:   G_BR %bb.2
+  bb.0:
+    successors: %bb.1(0x80000000)
+    liveins: $sgpr0, $sgpr1, $sgpr2, $sgpr3, $vgpr0, $vgpr1, $vgpr2
+
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(p1) = G_MERGE_VALUES %0(s32), %1(s32)
+    %3:_(s32) = COPY $vgpr2
+    %4:_(s32) = COPY $sgpr0
+    %5:_(s32) = COPY $sgpr1
+    %6:_(p1) = G_MERGE_VALUES %4(s32), %5(s32)
+    %7:_(s32) = COPY $sgpr2
+    %8:_(s32) = COPY $sgpr3
+    %9:_(p1) = G_MERGE_VALUES %7(s32), %8(s32)
+    %10:_(s32) = G_CONSTANT i32 0
+    %11:_(s32) = G_IMPLICIT_DEF
+
+  bb.1:
+    successors: %bb.3(0x50000000), %bb.5(0x30000000)
+
+    %12:_(s32) = G_PHI %13(s32), %bb.5, %10(s32), %bb.0
+    %14:_(s32) = G_PHI %10(s32), %bb.0, %15(s32), %bb.5
+    %16:_(s1) = G_CONSTANT i1 true
+    %17:_(s64) = G_SEXT %14(s32)
+    %18:_(s32) = G_CONSTANT i32 2
+    %19:_(s64) = G_SHL %17, %18(s32)
+    %20:_(p1) = G_PTR_ADD %6, %19(s64)
+    %21:_(s32) = G_LOAD %20(p1) :: (load (s32), addrspace 1)
+    %22:_(s32) = G_CONSTANT i32 0
+    %23:_(s1) = G_ICMP intpred(ne), %21(s32), %22
+    G_BRCOND %23(s1), %bb.3
+    G_BR %bb.5
+
+  bb.2:
+    successors: %bb.4(0x80000000)
+
+    %24:_(s32) = G_CONSTANT i32 10
+    G_STORE %24(s32), %9(p1) :: (store (s32), addrspace 1)
+    G_BR %bb.4
+
+  bb.3:
+    successors: %bb.5(0x80000000)
+
+    %25:_(s1) = G_CONSTANT i1 false
+    %26:_(s32) = G_CONSTANT i32 2
+    %27:_(s64) = G_SHL %17, %26(s32)
+    %28:_(p1) = G_PTR_ADD %2, %27(s64)
+    %29:_(s32) = G_LOAD %28(p1) :: (load (s32), addrspace 1)
+    %30:_(s32) = G_CONSTANT i32 1
+    %31:_(s32) = G_ADD %29, %30
+    G_STORE %31(s32), %28(p1) :: (store (s32), addrspace 1)
+    %32:_(s32) = G_ADD %14, %30
+    %33:_(s1) = G_ICMP intpred(ult), %14(s32), %3
+    G_BR %bb.5
+
+  bb.4:
+    G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %34(s32)
+    S_ENDPGM 0
+
+  bb.5:
+    successors: %bb.6(0x04000000), %bb.1(0x7c000000)
+
+    %15:_(s32) = G_PHI %32(s32), %bb.3, %11(s32), %bb.1
+    %35:_(s1) = G_PHI %25(s1), %bb.3, %16(s1), %bb.1
+    %36:_(s1) = G_PHI %33(s1), %bb.3, %16(s1), %bb.1
+    %13:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.if.break), %36(s1), %12(s32)
+    SI_LOOP %13(s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.6
+
+  bb.6:
+    successors: %bb.2(0x40000000), %bb.4(0x40000000)
+
+    %37:sreg_32_xm0_xexec(s1) = G_PHI %35(s1), %bb.5
+    %38:_(s32) = G_PHI %13(s32), %bb.5
+    G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %38(s32)
+    %34:sreg_32_xm0_xexec(s32) = SI_IF %37(s1), %bb.4, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.2
+...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-temporal-divergent-reg.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-temporal-divergent-reg.ll
new file mode 100644
index 0000000000000000000000000000000000000000..d065f228a4e4a35e64ceb30aa258cc9805b7208e
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-temporal-divergent-reg.ll
@@ -0,0 +1,37 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 3
+; RUN: llc -global-isel -amdgpu-global-isel-risky-select -mtriple=amdgcn-amd-amdpal -mcpu=gfx1010 < %s | FileCheck -check-prefix=GFX10 %s
+
+define void @temporal_divergent_i32(float %val, ptr %addr) {
+; GFX10-LABEL: temporal_divergent_i32:
+; GFX10:       ; %bb.0: ; %entry
+; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_mov_b32 s4, -1
+; GFX10-NEXT:    v_mov_b32_e32 v3, s4
+; GFX10-NEXT:    s_mov_b32 s4, 0
+; GFX10-NEXT:  .LBB0_1: ; %loop
+; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
+; GFX10-NEXT:    v_add_nc_u32_e32 v3, 1, v3
+; GFX10-NEXT:    v_cvt_f32_u32_e32 v4, v3
+; GFX10-NEXT:    v_cmp_gt_f32_e32 vcc_lo, v4, v0
+; GFX10-NEXT:    s_or_b32 s4, vcc_lo, s4
+; GFX10-NEXT:    s_andn2_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT:    s_cbranch_execnz .LBB0_1
+; GFX10-NEXT:  ; %bb.2: ; %exit
+; GFX10-NEXT:    s_or_b32 exec_lo, exec_lo, s4
+; GFX10-NEXT:    flat_store_dword v[1:2], v3
+; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX10-NEXT:    s_setpc_b64 s[30:31]
+entry:
+  br label %loop
+
+loop:
+  %counter = phi i32 [ 0, %entry ], [ %counter.plus.1, %loop ]
+  %f.counter = uitofp i32 %counter to float
+  %cond = fcmp ogt float %f.counter, %val
+  %counter.plus.1 = add i32 %counter, 1
+  br i1 %cond, label %exit, label %loop
+
+exit:
+  store i32 %counter, ptr %addr
+  ret void
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-temporal-divergent-reg.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-temporal-divergent-reg.mir
new file mode 100644
index 0000000000000000000000000000000000000000..4cc68029489e93b6516765fafcbeea1df79af05f
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/divergence-temporal-divergent-reg.mir
@@ -0,0 +1,70 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 4
+# RUN: llc -global-isel -mtriple=amdgcn-mesa-amdpal -mcpu=gfx1010 -run-pass=amdgpu-global-isel-divergence-lowering %s -o - | FileCheck -check-prefix=GFX10 %s
+
+---
+name: temporal_divergent_i32
+legalized: true
+tracksRegLiveness: true
+body: |
+  ; GFX10-LABEL: name: temporal_divergent_i32
+  ; GFX10: bb.0:
+  ; GFX10-NEXT:   successors: %bb.1(0x80000000)
+  ; GFX10-NEXT:   liveins: $vgpr0, $vgpr1, $vgpr2
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0
+  ; GFX10-NEXT:   [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1
+  ; GFX10-NEXT:   [[COPY2:%[0-9]+]]:_(s32) = COPY $vgpr2
+  ; GFX10-NEXT:   [[MV:%[0-9]+]]:_(p0) = G_MERGE_VALUES [[COPY1]](s32), [[COPY2]](s32)
+  ; GFX10-NEXT:   [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 0
+  ; GFX10-NEXT:   [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.1:
+  ; GFX10-NEXT:   successors: %bb.2(0x04000000), %bb.1(0x7c000000)
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT:   [[PHI:%[0-9]+]]:_(s32) = G_PHI %7(s32), %bb.1, [[C]](s32), %bb.0
+  ; GFX10-NEXT:   [[PHI1:%[0-9]+]]:_(s32) = G_PHI [[C1]](s32), %bb.0, %9(s32), %bb.1
+  ; GFX10-NEXT:   [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 1
+  ; GFX10-NEXT:   [[ADD:%[0-9]+]]:_(s32) = G_ADD [[PHI1]], [[C2]]
+  ; GFX10-NEXT:   [[UITOFP:%[0-9]+]]:_(s32) = G_UITOFP [[ADD]](s32)
+  ; GFX10-NEXT:   [[FCMP:%[0-9]+]]:_(s1) = G_FCMP floatpred(ogt), [[UITOFP]](s32), [[COPY]]
+  ; GFX10-NEXT:   [[INTRINSIC_CONVERGENT:%[0-9]+]]:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.if.break), [[FCMP]](s1), [[PHI]](s32)
+  ; GFX10-NEXT:   SI_LOOP [[INTRINSIC_CONVERGENT]](s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+  ; GFX10-NEXT:   G_BR %bb.2
+  ; GFX10-NEXT: {{  $}}
+  ; GFX10-NEXT: bb.2:
+  ; GFX10-NEXT:   [[PHI2:%[0-9]+]]:_(s32) = G_PHI [[ADD]](s32), %bb.1
+  ; GFX10-NEXT:   [[PHI3:%[0-9]+]]:_(s32) = G_PHI [[INTRINSIC_CONVERGENT]](s32), %bb.1
+  ; GFX10-NEXT:   G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), [[PHI3]](s32)
+  ; GFX10-NEXT:   G_STORE [[PHI2]](s32), [[MV]](p0) :: (store (s32))
+  ; GFX10-NEXT:   SI_RETURN
+  bb.0:
+    successors: %bb.1(0x80000000)
+    liveins: $vgpr0, $vgpr1, $vgpr2
+
+    %0:_(s32) = COPY $vgpr0
+    %1:_(s32) = COPY $vgpr1
+    %2:_(s32) = COPY $vgpr2
+    %3:_(p0) = G_MERGE_VALUES %1(s32), %2(s32)
+    %4:_(s32) = G_CONSTANT i32 0
+    %5:_(s32) = G_CONSTANT i32 -1
+
+  bb.1:
+    successors: %bb.2(0x04000000), %bb.1(0x7c000000)
+
+    %6:_(s32) = G_PHI %7(s32), %bb.1, %4(s32), %bb.0
+    %8:_(s32) = G_PHI %5(s32), %bb.0, %9(s32), %bb.1
+    %10:_(s32) = G_CONSTANT i32 1
+    %9:_(s32) = G_ADD %8, %10
+    %11:_(s32) = G_UITOFP %9(s32)
+    %12:_(s1) = G_FCMP floatpred(ogt), %11(s32), %0
+    %7:sreg_32_xm0_xexec(s32) = G_INTRINSIC_CONVERGENT intrinsic(@llvm.amdgcn.if.break), %12(s1), %6(s32)
+    SI_LOOP %7(s32), %bb.1, implicit-def $exec, implicit-def $scc, implicit $exec
+    G_BR %bb.2
+
+  bb.2:
+    %13:_(s32) = G_PHI %9(s32), %bb.1
+    %14:_(s32) = G_PHI %7(s32), %bb.1
+    G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS intrinsic(@llvm.amdgcn.end.cf), %14(s32)
+    G_STORE %13(s32), %3(p0) :: (store (s32))
+    SI_RETURN
+...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.ll
index d4c536bdd5ebe14581ac3713ee73413e66546e56..ac153183be642a269044a26b8cf4f38aa25cb096 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.ll
@@ -292,46 +292,46 @@ define i64 @dyn_extract_v8i64_const_s_v(i32 %sel) {
 ; GCN-LABEL: dyn_extract_v8i64_const_s_v:
 ; GCN:       ; %bb.0: ; %entry
 ; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-NEXT:    s_mov_b64 s[4:5], 1
-; GCN-NEXT:    s_mov_b64 s[6:7], 2
-; GCN-NEXT:    v_mov_b32_e32 v1, s4
-; GCN-NEXT:    v_mov_b32_e32 v2, s5
-; GCN-NEXT:    v_mov_b32_e32 v3, s6
-; GCN-NEXT:    v_mov_b32_e32 v4, s7
-; GCN-NEXT:    s_mov_b64 s[8:9], 3
+; GCN-NEXT:    s_mov_b64 s[16:17], 2
+; GCN-NEXT:    s_mov_b64 s[18:19], 1
+; GCN-NEXT:    s_mov_b64 s[14:15], 3
+; GCN-NEXT:    v_mov_b32_e32 v1, s18
+; GCN-NEXT:    v_mov_b32_e32 v2, s19
+; GCN-NEXT:    v_mov_b32_e32 v3, s16
+; GCN-NEXT:    v_mov_b32_e32 v4, s17
 ; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GCN-NEXT:    v_mov_b32_e32 v5, s8
-; GCN-NEXT:    v_mov_b32_e32 v6, s9
-; GCN-NEXT:    s_mov_b64 s[10:11], 4
+; GCN-NEXT:    s_mov_b64 s[12:13], 4
+; GCN-NEXT:    v_mov_b32_e32 v5, s14
+; GCN-NEXT:    v_mov_b32_e32 v6, s15
 ; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
 ; GCN-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
 ; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 2, v0
-; GCN-NEXT:    s_mov_b64 s[12:13], 5
-; GCN-NEXT:    v_mov_b32_e32 v7, s10
-; GCN-NEXT:    v_mov_b32_e32 v8, s11
+; GCN-NEXT:    s_mov_b64 s[10:11], 5
+; GCN-NEXT:    v_mov_b32_e32 v7, s12
+; GCN-NEXT:    v_mov_b32_e32 v8, s13
 ; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
 ; GCN-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc
 ; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 3, v0
-; GCN-NEXT:    s_mov_b64 s[14:15], 6
-; GCN-NEXT:    v_mov_b32_e32 v9, s12
-; GCN-NEXT:    v_mov_b32_e32 v10, s13
+; GCN-NEXT:    s_mov_b64 s[8:9], 6
+; GCN-NEXT:    v_mov_b32_e32 v9, s10
+; GCN-NEXT:    v_mov_b32_e32 v10, s11
 ; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v7, vcc
 ; GCN-NEXT:    v_cndmask_b32_e32 v2, v2, v8, vcc
 ; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 4, v0
-; GCN-NEXT:    s_mov_b64 s[16:17], 7
-; GCN-NEXT:    v_mov_b32_e32 v11, s14
-; GCN-NEXT:    v_mov_b32_e32 v12, s15
+; GCN-NEXT:    s_mov_b64 s[6:7], 7
+; GCN-NEXT:    v_mov_b32_e32 v11, s8
+; GCN-NEXT:    v_mov_b32_e32 v12, s9
 ; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc
 ; GCN-NEXT:    v_cndmask_b32_e32 v2, v2, v10, vcc
 ; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 5, v0
-; GCN-NEXT:    s_mov_b64 s[18:19], 8
-; GCN-NEXT:    v_mov_b32_e32 v13, s16
-; GCN-NEXT:    v_mov_b32_e32 v14, s17
+; GCN-NEXT:    s_mov_b64 s[4:5], 8
+; GCN-NEXT:    v_mov_b32_e32 v13, s6
+; GCN-NEXT:    v_mov_b32_e32 v14, s7
 ; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v11, vcc
 ; GCN-NEXT:    v_cndmask_b32_e32 v2, v2, v12, vcc
 ; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 6, v0
-; GCN-NEXT:    v_mov_b32_e32 v15, s18
-; GCN-NEXT:    v_mov_b32_e32 v16, s19
+; GCN-NEXT:    v_mov_b32_e32 v15, s4
+; GCN-NEXT:    v_mov_b32_e32 v16, s5
 ; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v13, vcc
 ; GCN-NEXT:    v_cndmask_b32_e32 v2, v2, v14, vcc
 ; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 7, v0
@@ -342,72 +342,72 @@ define i64 @dyn_extract_v8i64_const_s_v(i32 %sel) {
 ; GFX10-LABEL: dyn_extract_v8i64_const_s_v:
 ; GFX10:       ; %bb.0: ; %entry
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX10-NEXT:    s_mov_b64 s[6:7], 2
+; GFX10-NEXT:    s_mov_b64 s[4:5], 2
 ; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX10-NEXT:    v_mov_b32_e32 v1, s6
-; GFX10-NEXT:    v_mov_b32_e32 v2, s7
-; GFX10-NEXT:    s_mov_b64 s[4:5], 1
-; GFX10-NEXT:    s_mov_b64 s[8:9], 3
-; GFX10-NEXT:    s_mov_b64 s[10:11], 4
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, s4, v1, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, s5, v2, vcc_lo
+; GFX10-NEXT:    v_mov_b32_e32 v1, s4
+; GFX10-NEXT:    v_mov_b32_e32 v2, s5
+; GFX10-NEXT:    s_mov_b64 s[6:7], 1
+; GFX10-NEXT:    s_mov_b64 s[4:5], 3
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, s6, v1, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, s7, v2, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 2, v0
-; GFX10-NEXT:    s_mov_b64 s[12:13], 5
-; GFX10-NEXT:    s_mov_b64 s[14:15], 6
-; GFX10-NEXT:    s_mov_b64 s[16:17], 7
-; GFX10-NEXT:    s_mov_b64 s[18:19], 8
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, s8, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, s9, vcc_lo
+; GFX10-NEXT:    s_mov_b64 s[6:7], 4
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, s4, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, s5, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 3, v0
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, s10, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, s11, vcc_lo
+; GFX10-NEXT:    s_mov_b64 s[4:5], 5
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, s6, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, s7, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 4, v0
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, s12, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, s13, vcc_lo
+; GFX10-NEXT:    s_mov_b64 s[6:7], 6
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, s4, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, s5, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 5, v0
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, s14, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, s15, vcc_lo
+; GFX10-NEXT:    s_mov_b64 s[4:5], 7
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, s6, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, s7, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 6, v0
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, s16, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, s17, vcc_lo
+; GFX10-NEXT:    s_mov_b64 s[6:7], 8
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, s4, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, s5, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 7, v0
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, v1, s18, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v2, s19, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v1, s6, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v2, s7, vcc_lo
 ; GFX10-NEXT:    s_setpc_b64 s[30:31]
 ;
 ; GFX11-LABEL: dyn_extract_v8i64_const_s_v:
 ; GFX11:       ; %bb.0: ; %entry
 ; GFX11-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-NEXT:    s_mov_b64 s[2:3], 2
+; GFX11-NEXT:    s_mov_b64 s[0:1], 2
 ; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX11-NEXT:    v_dual_mov_b32 v1, s2 :: v_dual_mov_b32 v2, s3
-; GFX11-NEXT:    s_mov_b64 s[0:1], 1
-; GFX11-NEXT:    s_mov_b64 s[4:5], 3
-; GFX11-NEXT:    s_mov_b64 s[6:7], 4
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, s0, v1, vcc_lo
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, s1, v2, vcc_lo
+; GFX11-NEXT:    v_dual_mov_b32 v1, s0 :: v_dual_mov_b32 v2, s1
+; GFX11-NEXT:    s_mov_b64 s[2:3], 1
+; GFX11-NEXT:    s_mov_b64 s[0:1], 3
+; GFX11-NEXT:    v_cndmask_b32_e32 v1, s2, v1, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e32 v2, s3, v2, vcc_lo
 ; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 2, v0
-; GFX11-NEXT:    s_mov_b64 s[8:9], 5
-; GFX11-NEXT:    s_mov_b64 s[10:11], 6
-; GFX11-NEXT:    s_mov_b64 s[12:13], 7
-; GFX11-NEXT:    s_mov_b64 s[14:15], 8
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, s4, vcc_lo
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v2, s5, vcc_lo
+; GFX11-NEXT:    s_mov_b64 s[2:3], 4
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, s0, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v2, v2, s1, vcc_lo
 ; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 3, v0
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, s6, vcc_lo
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v2, s7, vcc_lo
+; GFX11-NEXT:    s_mov_b64 s[0:1], 5
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, s2, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v2, v2, s3, vcc_lo
 ; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 4, v0
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, s8, vcc_lo
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v2, s9, vcc_lo
+; GFX11-NEXT:    s_mov_b64 s[2:3], 6
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, s0, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v2, v2, s1, vcc_lo
 ; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 5, v0
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, s10, vcc_lo
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v2, s11, vcc_lo
+; GFX11-NEXT:    s_mov_b64 s[0:1], 7
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, s2, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v2, v2, s3, vcc_lo
 ; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 6, v0
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, s12, vcc_lo
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v2, s13, vcc_lo
+; GFX11-NEXT:    s_mov_b64 s[2:3], 8
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, s0, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v2, v2, s1, vcc_lo
 ; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 7, v0
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, v1, s14, vcc_lo
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v2, s15, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, v1, s2, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, v2, s3, vcc_lo
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
 entry:
   %ext = extractelement <8 x i64> <i64 1, i64 2, i64 3, i64 4, i64 5, i64 6, i64 7, i64 8>, i32 %sel
@@ -492,40 +492,28 @@ entry:
 define amdgpu_ps void @dyn_extract_v8i64_s_v(<8 x i64> inreg %vec, i32 %sel) {
 ; GPRIDX-LABEL: dyn_extract_v8i64_s_v:
 ; GPRIDX:       ; %bb.0: ; %entry
-; GPRIDX-NEXT:    s_mov_b32 s0, s2
-; GPRIDX-NEXT:    s_mov_b32 s1, s3
-; GPRIDX-NEXT:    s_mov_b32 s2, s4
-; GPRIDX-NEXT:    s_mov_b32 s3, s5
-; GPRIDX-NEXT:    s_mov_b32 s4, s6
-; GPRIDX-NEXT:    s_mov_b32 s5, s7
-; GPRIDX-NEXT:    v_mov_b32_e32 v1, s0
-; GPRIDX-NEXT:    v_mov_b32_e32 v2, s1
-; GPRIDX-NEXT:    v_mov_b32_e32 v3, s2
-; GPRIDX-NEXT:    v_mov_b32_e32 v4, s3
+; GPRIDX-NEXT:    v_mov_b32_e32 v1, s2
+; GPRIDX-NEXT:    v_mov_b32_e32 v2, s3
+; GPRIDX-NEXT:    v_mov_b32_e32 v3, s4
+; GPRIDX-NEXT:    v_mov_b32_e32 v4, s5
 ; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GPRIDX-NEXT:    s_mov_b32 s6, s8
-; GPRIDX-NEXT:    s_mov_b32 s7, s9
-; GPRIDX-NEXT:    v_mov_b32_e32 v5, s4
-; GPRIDX-NEXT:    v_mov_b32_e32 v6, s5
+; GPRIDX-NEXT:    v_mov_b32_e32 v5, s6
+; GPRIDX-NEXT:    v_mov_b32_e32 v6, s7
 ; GPRIDX-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
 ; GPRIDX-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
 ; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, 2, v0
-; GPRIDX-NEXT:    s_mov_b32 s8, s10
-; GPRIDX-NEXT:    s_mov_b32 s9, s11
-; GPRIDX-NEXT:    v_mov_b32_e32 v7, s6
-; GPRIDX-NEXT:    v_mov_b32_e32 v8, s7
+; GPRIDX-NEXT:    v_mov_b32_e32 v7, s8
+; GPRIDX-NEXT:    v_mov_b32_e32 v8, s9
 ; GPRIDX-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
 ; GPRIDX-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc
 ; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, 3, v0
-; GPRIDX-NEXT:    s_mov_b32 s10, s12
-; GPRIDX-NEXT:    s_mov_b32 s11, s13
-; GPRIDX-NEXT:    v_mov_b32_e32 v9, s8
-; GPRIDX-NEXT:    v_mov_b32_e32 v10, s9
+; GPRIDX-NEXT:    v_mov_b32_e32 v9, s10
+; GPRIDX-NEXT:    v_mov_b32_e32 v10, s11
 ; GPRIDX-NEXT:    v_cndmask_b32_e32 v1, v1, v7, vcc
 ; GPRIDX-NEXT:    v_cndmask_b32_e32 v2, v2, v8, vcc
 ; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, 4, v0
-; GPRIDX-NEXT:    v_mov_b32_e32 v11, s10
-; GPRIDX-NEXT:    v_mov_b32_e32 v12, s11
+; GPRIDX-NEXT:    v_mov_b32_e32 v11, s12
+; GPRIDX-NEXT:    v_mov_b32_e32 v12, s13
 ; GPRIDX-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc
 ; GPRIDX-NEXT:    v_cndmask_b32_e32 v2, v2, v10, vcc
 ; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, 5, v0
@@ -546,40 +534,28 @@ define amdgpu_ps void @dyn_extract_v8i64_s_v(<8 x i64> inreg %vec, i32 %sel) {
 ;
 ; MOVREL-LABEL: dyn_extract_v8i64_s_v:
 ; MOVREL:       ; %bb.0: ; %entry
-; MOVREL-NEXT:    s_mov_b32 s0, s2
-; MOVREL-NEXT:    s_mov_b32 s1, s3
-; MOVREL-NEXT:    s_mov_b32 s2, s4
-; MOVREL-NEXT:    s_mov_b32 s3, s5
-; MOVREL-NEXT:    s_mov_b32 s4, s6
-; MOVREL-NEXT:    s_mov_b32 s5, s7
-; MOVREL-NEXT:    v_mov_b32_e32 v1, s0
-; MOVREL-NEXT:    v_mov_b32_e32 v2, s1
-; MOVREL-NEXT:    v_mov_b32_e32 v3, s2
-; MOVREL-NEXT:    v_mov_b32_e32 v4, s3
+; MOVREL-NEXT:    v_mov_b32_e32 v1, s2
+; MOVREL-NEXT:    v_mov_b32_e32 v2, s3
+; MOVREL-NEXT:    v_mov_b32_e32 v3, s4
+; MOVREL-NEXT:    v_mov_b32_e32 v4, s5
 ; MOVREL-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; MOVREL-NEXT:    s_mov_b32 s6, s8
-; MOVREL-NEXT:    s_mov_b32 s7, s9
-; MOVREL-NEXT:    v_mov_b32_e32 v5, s4
-; MOVREL-NEXT:    v_mov_b32_e32 v6, s5
+; MOVREL-NEXT:    v_mov_b32_e32 v5, s6
+; MOVREL-NEXT:    v_mov_b32_e32 v6, s7
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
 ; MOVREL-NEXT:    v_cmp_eq_u32_e32 vcc, 2, v0
-; MOVREL-NEXT:    s_mov_b32 s8, s10
-; MOVREL-NEXT:    s_mov_b32 s9, s11
-; MOVREL-NEXT:    v_mov_b32_e32 v7, s6
-; MOVREL-NEXT:    v_mov_b32_e32 v8, s7
+; MOVREL-NEXT:    v_mov_b32_e32 v7, s8
+; MOVREL-NEXT:    v_mov_b32_e32 v8, s9
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc
 ; MOVREL-NEXT:    v_cmp_eq_u32_e32 vcc, 3, v0
-; MOVREL-NEXT:    s_mov_b32 s10, s12
-; MOVREL-NEXT:    s_mov_b32 s11, s13
-; MOVREL-NEXT:    v_mov_b32_e32 v9, s8
-; MOVREL-NEXT:    v_mov_b32_e32 v10, s9
+; MOVREL-NEXT:    v_mov_b32_e32 v9, s10
+; MOVREL-NEXT:    v_mov_b32_e32 v10, s11
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v1, v1, v7, vcc
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v2, v2, v8, vcc
 ; MOVREL-NEXT:    v_cmp_eq_u32_e32 vcc, 4, v0
-; MOVREL-NEXT:    v_mov_b32_e32 v11, s10
-; MOVREL-NEXT:    v_mov_b32_e32 v12, s11
+; MOVREL-NEXT:    v_mov_b32_e32 v11, s12
+; MOVREL-NEXT:    v_mov_b32_e32 v12, s13
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v1, v1, v9, vcc
 ; MOVREL-NEXT:    v_cndmask_b32_e32 v2, v2, v10, vcc
 ; MOVREL-NEXT:    v_cmp_eq_u32_e32 vcc, 5, v0
@@ -600,88 +576,56 @@ define amdgpu_ps void @dyn_extract_v8i64_s_v(<8 x i64> inreg %vec, i32 %sel) {
 ;
 ; GFX10-LABEL: dyn_extract_v8i64_s_v:
 ; GFX10:       ; %bb.0: ; %entry
-; GFX10-NEXT:    s_mov_b32 s0, s2
-; GFX10-NEXT:    s_mov_b32 s2, s4
-; GFX10-NEXT:    s_mov_b32 s19, s5
-; GFX10-NEXT:    v_mov_b32_e32 v1, s2
-; GFX10-NEXT:    v_mov_b32_e32 v2, s19
+; GFX10-NEXT:    v_mov_b32_e32 v1, s4
+; GFX10-NEXT:    v_mov_b32_e32 v2, s5
 ; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX10-NEXT:    s_mov_b32 s1, s3
-; GFX10-NEXT:    s_mov_b32 s4, s6
-; GFX10-NEXT:    s_mov_b32 s5, s7
-; GFX10-NEXT:    s_mov_b32 s6, s8
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, s0, v1, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, s1, v2, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, s2, v1, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, s3, v2, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 2, v0
-; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    s_mov_b32 s8, s10
-; GFX10-NEXT:    s_mov_b32 s9, s11
-; GFX10-NEXT:    s_mov_b32 s10, s12
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, s4, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, s5, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 3, v0
-; GFX10-NEXT:    s_mov_b32 s11, s13
-; GFX10-NEXT:    s_mov_b32 s12, s14
-; GFX10-NEXT:    s_mov_b32 s13, s15
-; GFX10-NEXT:    s_mov_b32 s14, s16
 ; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, s6, vcc_lo
 ; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, s7, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 4, v0
-; GFX10-NEXT:    s_mov_b32 s15, s17
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 3, v0
 ; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, s8, vcc_lo
 ; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, s9, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 5, v0
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 4, v0
 ; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, s10, vcc_lo
 ; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, s11, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 6, v0
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 5, v0
 ; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, s12, vcc_lo
 ; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, s13, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 6, v0
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, s14, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, s15, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 7, v0
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, v1, s14, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v2, s15, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v1, s16, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v2, s17, vcc_lo
 ; GFX10-NEXT:    global_store_dwordx2 v[0:1], v[0:1], off
 ; GFX10-NEXT:    s_endpgm
 ;
 ; GFX11-LABEL: dyn_extract_v8i64_s_v:
 ; GFX11:       ; %bb.0: ; %entry
-; GFX11-NEXT:    s_mov_b32 s0, s2
-; GFX11-NEXT:    s_mov_b32 s2, s4
-; GFX11-NEXT:    s_mov_b32 s19, s5
-; GFX11-NEXT:    v_dual_mov_b32 v1, s2 :: v_dual_mov_b32 v2, s19
+; GFX11-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_mov_b32 v2, s5
 ; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX11-NEXT:    s_mov_b32 s1, s3
-; GFX11-NEXT:    s_mov_b32 s4, s6
-; GFX11-NEXT:    s_mov_b32 s5, s7
-; GFX11-NEXT:    s_mov_b32 s6, s8
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, s0, v1, vcc_lo
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, s1, v2, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e32 v1, s2, v1, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e32 v2, s3, v2, vcc_lo
 ; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 2, v0
-; GFX11-NEXT:    s_mov_b32 s7, s9
-; GFX11-NEXT:    s_mov_b32 s8, s10
-; GFX11-NEXT:    s_mov_b32 s9, s11
-; GFX11-NEXT:    s_mov_b32 s10, s12
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, s4, vcc_lo
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v2, s5, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 3, v0
-; GFX11-NEXT:    s_mov_b32 s11, s13
-; GFX11-NEXT:    s_mov_b32 s12, s14
-; GFX11-NEXT:    s_mov_b32 s13, s15
-; GFX11-NEXT:    s_mov_b32 s14, s16
 ; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, s6, vcc_lo
 ; GFX11-NEXT:    v_cndmask_b32_e64 v2, v2, s7, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 4, v0
-; GFX11-NEXT:    s_mov_b32 s15, s17
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 3, v0
 ; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, s8, vcc_lo
 ; GFX11-NEXT:    v_cndmask_b32_e64 v2, v2, s9, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 5, v0
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 4, v0
 ; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, s10, vcc_lo
 ; GFX11-NEXT:    v_cndmask_b32_e64 v2, v2, s11, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 6, v0
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 5, v0
 ; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, s12, vcc_lo
 ; GFX11-NEXT:    v_cndmask_b32_e64 v2, v2, s13, vcc_lo
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 6, v0
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, s14, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v2, v2, s15, vcc_lo
 ; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 7, v0
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, v1, s14, vcc_lo
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v2, s15, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, v1, s16, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, v2, s17, vcc_lo
 ; GFX11-NEXT:    global_store_b64 v[0:1], v[0:1], off
 ; GFX11-NEXT:    s_nop 0
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
@@ -2467,26 +2411,18 @@ entry:
 define amdgpu_ps double @dyn_extract_v6f64_s_v(<6 x double> inreg %vec, i32 %sel) {
 ; GCN-LABEL: dyn_extract_v6f64_s_v:
 ; GCN:       ; %bb.0: ; %entry
-; GCN-NEXT:    s_mov_b32 s0, s2
-; GCN-NEXT:    s_mov_b32 s1, s3
-; GCN-NEXT:    s_mov_b32 s2, s4
-; GCN-NEXT:    s_mov_b32 s3, s5
-; GCN-NEXT:    s_mov_b32 s4, s6
-; GCN-NEXT:    s_mov_b32 s5, s7
-; GCN-NEXT:    v_mov_b32_e32 v1, s0
-; GCN-NEXT:    v_mov_b32_e32 v2, s1
-; GCN-NEXT:    v_mov_b32_e32 v3, s2
-; GCN-NEXT:    v_mov_b32_e32 v4, s3
+; GCN-NEXT:    v_mov_b32_e32 v1, s2
+; GCN-NEXT:    v_mov_b32_e32 v2, s3
+; GCN-NEXT:    v_mov_b32_e32 v3, s4
+; GCN-NEXT:    v_mov_b32_e32 v4, s5
 ; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GCN-NEXT:    s_mov_b32 s6, s8
-; GCN-NEXT:    s_mov_b32 s7, s9
-; GCN-NEXT:    v_mov_b32_e32 v5, s4
-; GCN-NEXT:    v_mov_b32_e32 v6, s5
+; GCN-NEXT:    v_mov_b32_e32 v5, s6
+; GCN-NEXT:    v_mov_b32_e32 v6, s7
 ; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
 ; GCN-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
 ; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 2, v0
-; GCN-NEXT:    v_mov_b32_e32 v7, s6
-; GCN-NEXT:    v_mov_b32_e32 v8, s7
+; GCN-NEXT:    v_mov_b32_e32 v7, s8
+; GCN-NEXT:    v_mov_b32_e32 v8, s9
 ; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
 ; GCN-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc
 ; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 3, v0
@@ -2508,69 +2444,45 @@ define amdgpu_ps double @dyn_extract_v6f64_s_v(<6 x double> inreg %vec, i32 %sel
 ;
 ; GFX10-LABEL: dyn_extract_v6f64_s_v:
 ; GFX10:       ; %bb.0: ; %entry
-; GFX10-NEXT:    s_mov_b32 s0, s2
-; GFX10-NEXT:    s_mov_b32 s2, s4
-; GFX10-NEXT:    s_mov_b32 s15, s5
-; GFX10-NEXT:    v_mov_b32_e32 v1, s2
-; GFX10-NEXT:    v_mov_b32_e32 v2, s15
+; GFX10-NEXT:    v_mov_b32_e32 v1, s4
+; GFX10-NEXT:    v_mov_b32_e32 v2, s5
 ; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX10-NEXT:    s_mov_b32 s1, s3
-; GFX10-NEXT:    s_mov_b32 s4, s6
-; GFX10-NEXT:    s_mov_b32 s5, s7
-; GFX10-NEXT:    s_mov_b32 s6, s8
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, s0, v1, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, s1, v2, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, s2, v1, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, s3, v2, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 2, v0
-; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    s_mov_b32 s8, s10
-; GFX10-NEXT:    s_mov_b32 s9, s11
-; GFX10-NEXT:    s_mov_b32 s10, s12
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, s4, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, s5, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 3, v0
-; GFX10-NEXT:    s_mov_b32 s11, s13
 ; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, s6, vcc_lo
 ; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, s7, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 4, v0
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 3, v0
 ; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, s8, vcc_lo
 ; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, s9, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 4, v0
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, s10, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, s11, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 5, v0
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, v1, s10, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v2, s11, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v1, s12, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v2, s13, vcc_lo
 ; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX10-NEXT:    ; return to shader part epilog
 ;
 ; GFX11-LABEL: dyn_extract_v6f64_s_v:
 ; GFX11:       ; %bb.0: ; %entry
-; GFX11-NEXT:    s_mov_b32 s0, s2
-; GFX11-NEXT:    s_mov_b32 s2, s4
-; GFX11-NEXT:    s_mov_b32 s15, s5
-; GFX11-NEXT:    v_dual_mov_b32 v1, s2 :: v_dual_mov_b32 v2, s15
+; GFX11-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_mov_b32 v2, s5
 ; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX11-NEXT:    s_mov_b32 s1, s3
-; GFX11-NEXT:    s_mov_b32 s4, s6
-; GFX11-NEXT:    s_mov_b32 s5, s7
-; GFX11-NEXT:    s_mov_b32 s6, s8
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, s0, v1, vcc_lo
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, s1, v2, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e32 v1, s2, v1, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e32 v2, s3, v2, vcc_lo
 ; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 2, v0
-; GFX11-NEXT:    s_mov_b32 s7, s9
-; GFX11-NEXT:    s_mov_b32 s8, s10
-; GFX11-NEXT:    s_mov_b32 s9, s11
-; GFX11-NEXT:    s_mov_b32 s10, s12
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, s4, vcc_lo
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v2, s5, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 3, v0
-; GFX11-NEXT:    s_mov_b32 s11, s13
 ; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, s6, vcc_lo
 ; GFX11-NEXT:    v_cndmask_b32_e64 v2, v2, s7, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 4, v0
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 3, v0
 ; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, s8, vcc_lo
 ; GFX11-NEXT:    v_cndmask_b32_e64 v2, v2, s9, vcc_lo
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 4, v0
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, s10, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v2, v2, s11, vcc_lo
 ; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 5, v0
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, v1, s10, vcc_lo
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v2, s11, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, v1, s12, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, v2, s13, vcc_lo
 ; GFX11-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX11-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX11-NEXT:    ; return to shader part epilog
@@ -2717,33 +2629,23 @@ entry:
 define amdgpu_ps double @dyn_extract_v7f64_s_v(<7 x double> inreg %vec, i32 %sel) {
 ; GCN-LABEL: dyn_extract_v7f64_s_v:
 ; GCN:       ; %bb.0: ; %entry
-; GCN-NEXT:    s_mov_b32 s0, s2
-; GCN-NEXT:    s_mov_b32 s1, s3
-; GCN-NEXT:    s_mov_b32 s2, s4
-; GCN-NEXT:    s_mov_b32 s3, s5
-; GCN-NEXT:    s_mov_b32 s4, s6
-; GCN-NEXT:    s_mov_b32 s5, s7
-; GCN-NEXT:    v_mov_b32_e32 v1, s0
-; GCN-NEXT:    v_mov_b32_e32 v2, s1
-; GCN-NEXT:    v_mov_b32_e32 v3, s2
-; GCN-NEXT:    v_mov_b32_e32 v4, s3
+; GCN-NEXT:    v_mov_b32_e32 v1, s2
+; GCN-NEXT:    v_mov_b32_e32 v2, s3
+; GCN-NEXT:    v_mov_b32_e32 v3, s4
+; GCN-NEXT:    v_mov_b32_e32 v4, s5
 ; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
-; GCN-NEXT:    s_mov_b32 s6, s8
-; GCN-NEXT:    s_mov_b32 s7, s9
-; GCN-NEXT:    v_mov_b32_e32 v5, s4
-; GCN-NEXT:    v_mov_b32_e32 v6, s5
+; GCN-NEXT:    v_mov_b32_e32 v5, s6
+; GCN-NEXT:    v_mov_b32_e32 v6, s7
 ; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
 ; GCN-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
 ; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 2, v0
-; GCN-NEXT:    s_mov_b32 s8, s10
-; GCN-NEXT:    s_mov_b32 s9, s11
-; GCN-NEXT:    v_mov_b32_e32 v7, s6
-; GCN-NEXT:    v_mov_b32_e32 v8, s7
+; GCN-NEXT:    v_mov_b32_e32 v7, s8
+; GCN-NEXT:    v_mov_b32_e32 v8, s9
 ; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v5, vcc
 ; GCN-NEXT:    v_cndmask_b32_e32 v2, v2, v6, vcc
 ; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 3, v0
-; GCN-NEXT:    v_mov_b32_e32 v9, s8
-; GCN-NEXT:    v_mov_b32_e32 v10, s9
+; GCN-NEXT:    v_mov_b32_e32 v9, s10
+; GCN-NEXT:    v_mov_b32_e32 v10, s11
 ; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v7, vcc
 ; GCN-NEXT:    v_cndmask_b32_e32 v2, v2, v8, vcc
 ; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 4, v0
@@ -2760,8 +2662,8 @@ define amdgpu_ps double @dyn_extract_v7f64_s_v(<7 x double> inreg %vec, i32 %sel
 ; GCN-NEXT:    v_cndmask_b32_e32 v1, v1, v13, vcc
 ; GCN-NEXT:    v_cndmask_b32_e32 v2, v2, v14, vcc
 ; GCN-NEXT:    v_cmp_eq_u32_e32 vcc, 7, v0
-; GCN-NEXT:    ; kill: def $vgpr15 killed $sgpr14 killed $exec
-; GCN-NEXT:    ; kill: def $vgpr16 killed $sgpr15 killed $exec
+; GCN-NEXT:    ; kill: def $vgpr15 killed $sgpr2 killed $exec
+; GCN-NEXT:    ; kill: def $vgpr16 killed $sgpr3 killed $exec
 ; GCN-NEXT:    v_cndmask_b32_e32 v0, v1, v15, vcc
 ; GCN-NEXT:    v_cndmask_b32_e32 v1, v2, v16, vcc
 ; GCN-NEXT:    v_readfirstlane_b32 s0, v0
@@ -2770,85 +2672,59 @@ define amdgpu_ps double @dyn_extract_v7f64_s_v(<7 x double> inreg %vec, i32 %sel
 ;
 ; GFX10-LABEL: dyn_extract_v7f64_s_v:
 ; GFX10:       ; %bb.0: ; %entry
-; GFX10-NEXT:    s_mov_b32 s0, s2
-; GFX10-NEXT:    s_mov_b32 s2, s4
-; GFX10-NEXT:    s_mov_b32 s19, s5
-; GFX10-NEXT:    v_mov_b32_e32 v1, s2
-; GFX10-NEXT:    v_mov_b32_e32 v2, s19
+; GFX10-NEXT:    v_mov_b32_e32 v1, s4
+; GFX10-NEXT:    v_mov_b32_e32 v2, s5
 ; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX10-NEXT:    s_mov_b32 s1, s3
-; GFX10-NEXT:    s_mov_b32 s4, s6
-; GFX10-NEXT:    s_mov_b32 s5, s7
-; GFX10-NEXT:    s_mov_b32 s6, s8
-; GFX10-NEXT:    v_cndmask_b32_e32 v1, s0, v1, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, s1, v2, vcc_lo
+; GFX10-NEXT:    s_mov_b32 s0, s14
+; GFX10-NEXT:    v_cndmask_b32_e32 v1, s2, v1, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, s3, v2, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 2, v0
-; GFX10-NEXT:    s_mov_b32 s7, s9
-; GFX10-NEXT:    s_mov_b32 s8, s10
-; GFX10-NEXT:    s_mov_b32 s9, s11
-; GFX10-NEXT:    s_mov_b32 s10, s12
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, s4, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, s5, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 3, v0
-; GFX10-NEXT:    s_mov_b32 s11, s13
-; GFX10-NEXT:    s_mov_b32 s12, s14
-; GFX10-NEXT:    s_mov_b32 s13, s15
 ; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, s6, vcc_lo
 ; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, s7, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 4, v0
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 3, v0
 ; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, s8, vcc_lo
 ; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, s9, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 5, v0
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 4, v0
 ; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, s10, vcc_lo
 ; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, s11, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 6, v0
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 5, v0
 ; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, s12, vcc_lo
 ; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, s13, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 6, v0
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, s0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, s15, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 7, v0
-; GFX10-NEXT:    v_cndmask_b32_e64 v0, v1, s14, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v1, v2, s15, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v1, s2, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v2, s3, vcc_lo
 ; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX10-NEXT:    ; return to shader part epilog
 ;
 ; GFX11-LABEL: dyn_extract_v7f64_s_v:
 ; GFX11:       ; %bb.0: ; %entry
-; GFX11-NEXT:    s_mov_b32 s0, s2
-; GFX11-NEXT:    s_mov_b32 s2, s4
-; GFX11-NEXT:    s_mov_b32 s19, s5
-; GFX11-NEXT:    v_dual_mov_b32 v1, s2 :: v_dual_mov_b32 v2, s19
+; GFX11-NEXT:    v_dual_mov_b32 v1, s4 :: v_dual_mov_b32 v2, s5
 ; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v0
-; GFX11-NEXT:    s_mov_b32 s1, s3
-; GFX11-NEXT:    s_mov_b32 s4, s6
-; GFX11-NEXT:    s_mov_b32 s5, s7
-; GFX11-NEXT:    s_mov_b32 s6, s8
-; GFX11-NEXT:    v_cndmask_b32_e32 v1, s0, v1, vcc_lo
-; GFX11-NEXT:    v_cndmask_b32_e32 v2, s1, v2, vcc_lo
+; GFX11-NEXT:    s_mov_b32 s0, s14
+; GFX11-NEXT:    v_cndmask_b32_e32 v1, s2, v1, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e32 v2, s3, v2, vcc_lo
 ; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 2, v0
-; GFX11-NEXT:    s_mov_b32 s7, s9
-; GFX11-NEXT:    s_mov_b32 s8, s10
-; GFX11-NEXT:    s_mov_b32 s9, s11
-; GFX11-NEXT:    s_mov_b32 s10, s12
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, s4, vcc_lo
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v2, s5, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 3, v0
-; GFX11-NEXT:    s_mov_b32 s11, s13
-; GFX11-NEXT:    s_mov_b32 s12, s14
-; GFX11-NEXT:    s_mov_b32 s13, s15
 ; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, s6, vcc_lo
 ; GFX11-NEXT:    v_cndmask_b32_e64 v2, v2, s7, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 4, v0
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 3, v0
 ; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, s8, vcc_lo
 ; GFX11-NEXT:    v_cndmask_b32_e64 v2, v2, s9, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 5, v0
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 4, v0
 ; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, s10, vcc_lo
 ; GFX11-NEXT:    v_cndmask_b32_e64 v2, v2, s11, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 6, v0
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 5, v0
 ; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, s12, vcc_lo
 ; GFX11-NEXT:    v_cndmask_b32_e64 v2, v2, s13, vcc_lo
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 6, v0
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, s0, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v2, v2, s15, vcc_lo
 ; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 7, v0
-; GFX11-NEXT:    v_cndmask_b32_e64 v0, v1, s14, vcc_lo
-; GFX11-NEXT:    v_cndmask_b32_e64 v1, v2, s15, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, v1, s2, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, v2, s3, vcc_lo
 ; GFX11-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX11-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX11-NEXT:    ; return to shader part epilog
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/fmed3-min-max-const-combine.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/fmed3-min-max-const-combine.ll
index d6d36fe1acf35f7c17a7c6ef507a4cd5e71b3e89..096ca5bc8705f53c1d8b36232492f0aceb541454 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/fmed3-min-max-const-combine.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/fmed3-min-max-const-combine.ll
@@ -1,6 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -global-isel -mtriple=amdgcn-amd-mesa3d -mcpu=gfx1010 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX10 %s
 ; RUN: llc -global-isel -mtriple=amdgcn-amd-mesa3d -mcpu=gfx803 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX8 %s
+; RUN: llc -global-isel -mtriple=amdgcn-amd-mesa3d -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX12 %s
 
 define float @test_min_max_ValK0_K1_f32(float %a) #0 {
 ; GFX10-LABEL: test_min_max_ValK0_K1_f32:
@@ -14,6 +15,12 @@ define float @test_min_max_ValK0_K1_f32(float %a) #0 {
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    v_med3_f32 v0, v0, 2.0, 4.0
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: test_min_max_ValK0_K1_f32:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX12-NEXT:    v_med3_num_f32 v0, v0, 2.0, 4.0
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %maxnum = call nnan float @llvm.maxnum.f32(float %a, float 2.0)
   %fmed = call nnan float @llvm.minnum.f32(float %maxnum, float 4.0)
   ret float %fmed
@@ -31,6 +38,12 @@ define float @test_min_max_K0Val_K1_f32(float %a) #1 {
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    v_med3_f32 v0, v0, 2.0, 4.0
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: test_min_max_K0Val_K1_f32:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX12-NEXT:    v_med3_num_f32 v0, v0, 2.0, 4.0
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %maxnum = call nnan float @llvm.maxnum.f32(float 2.0, float %a)
   %fmed = call nnan float @llvm.minnum.f32(float %maxnum, float 4.0)
   ret float %fmed
@@ -53,6 +66,14 @@ define half @test_min_K1max_ValK0_f16(half %a) #0 {
 ; GFX8-NEXT:    v_max_f16_e32 v0, 2.0, v0
 ; GFX8-NEXT:    v_min_f16_e32 v0, 4.0, v0
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: test_min_K1max_ValK0_f16:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX12-NEXT:    v_max_num_f16_e32 v0, v0, v0
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT:    v_med3_num_f16 v0, v0, 2.0, 4.0
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %maxnum = call half @llvm.maxnum.f16(half %a, half 2.0)
   %fmed = call half @llvm.minnum.f16(half 4.0, half %maxnum)
   ret half %fmed
@@ -71,6 +92,12 @@ define half @test_min_K1max_K0Val_f16(half %a) #1 {
 ; GFX8-NEXT:    v_max_f16_e32 v0, 2.0, v0
 ; GFX8-NEXT:    v_min_f16_e32 v0, 4.0, v0
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: test_min_K1max_K0Val_f16:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX12-NEXT:    v_med3_num_f16 v0, v0, 2.0, 4.0
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %maxnum = call nnan half @llvm.maxnum.f16(half 2.0, half %a)
   %fmed = call nnan half @llvm.minnum.f16(half 4.0, half %maxnum)
   ret half %fmed
@@ -89,6 +116,12 @@ define float @test_max_min_ValK1_K0_f32(float %a) #0 {
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    v_med3_f32 v0, v0, 2.0, 4.0
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: test_max_min_ValK1_K0_f32:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX12-NEXT:    v_med3_num_f32 v0, v0, 2.0, 4.0
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %minnum = call nnan float @llvm.minnum.f32(float %a, float 4.0)
   %fmed = call nnan float @llvm.maxnum.f32(float %minnum, float 2.0)
   ret float %fmed
@@ -106,6 +139,12 @@ define float @test_max_min_K1Val_K0_f32(float %a) #1 {
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    v_med3_f32 v0, v0, 2.0, 4.0
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: test_max_min_K1Val_K0_f32:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX12-NEXT:    v_med3_num_f32 v0, v0, 2.0, 4.0
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %minnum = call nnan float @llvm.minnum.f32(float 4.0, float %a)
   %fmed = call nnan float @llvm.maxnum.f32(float %minnum, float 2.0)
   ret float %fmed
@@ -124,6 +163,12 @@ define half @test_max_K0min_ValK1_f16(half %a) #0 {
 ; GFX8-NEXT:    v_min_f16_e32 v0, 4.0, v0
 ; GFX8-NEXT:    v_max_f16_e32 v0, 2.0, v0
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: test_max_K0min_ValK1_f16:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX12-NEXT:    v_med3_num_f16 v0, v0, 2.0, 4.0
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %minnum = call nnan half @llvm.minnum.f16(half %a, half 4.0)
   %fmed = call nnan half @llvm.maxnum.f16(half 2.0, half %minnum)
   ret half %fmed
@@ -142,6 +187,12 @@ define half @test_max_K0min_K1Val_f16(half %a) #1 {
 ; GFX8-NEXT:    v_min_f16_e32 v0, 4.0, v0
 ; GFX8-NEXT:    v_max_f16_e32 v0, 2.0, v0
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: test_max_K0min_K1Val_f16:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX12-NEXT:    v_med3_num_f16 v0, v0, 2.0, 4.0
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %minnum = call nnan half @llvm.minnum.f16(half 4.0, half %a)
   %fmed = call nnan half @llvm.maxnum.f16(half 2.0, half %minnum)
   ret half %fmed
@@ -161,6 +212,12 @@ define float @test_min_max_global_nnan(float %a) #2 {
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    v_med3_f32 v0, v0, 2.0, 4.0
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: test_min_max_global_nnan:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX12-NEXT:    v_med3_num_f32 v0, v0, 2.0, 4.0
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %maxnum = call float @llvm.maxnum.f32(float %a, float 2.0)
   %fmed = call float @llvm.minnum.f32(float %maxnum, float 4.0)
   ret float %fmed
@@ -178,6 +235,12 @@ define float @test_max_min_global_nnan(float %a) #2 {
 ; GFX8-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; GFX8-NEXT:    v_med3_f32 v0, v0, 2.0, 4.0
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: test_max_min_global_nnan:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX12-NEXT:    v_med3_num_f32 v0, v0, 2.0, 4.0
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %minnum = call float @llvm.minnum.f32(float %a, float 4.0)
   %fmed = call float @llvm.maxnum.f32(float %minnum, float 2.0)
   ret float %fmed
@@ -202,6 +265,12 @@ define float @test_min_max_K0_gt_K1(float %a) #0 {
 ; GFX8-NEXT:    v_max_f32_e32 v0, 4.0, v0
 ; GFX8-NEXT:    v_min_f32_e32 v0, 2.0, v0
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: test_min_max_K0_gt_K1:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX12-NEXT:    v_maxmin_num_f32 v0, v0, 4.0, 2.0
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %maxnum = call nnan float @llvm.maxnum.f32(float %a, float 4.0)
   %fmed = call nnan float @llvm.minnum.f32(float %maxnum, float 2.0)
   ret float %fmed
@@ -222,6 +291,12 @@ define float @test_max_min_K0_gt_K1(float %a) #0 {
 ; GFX8-NEXT:    v_min_f32_e32 v0, 2.0, v0
 ; GFX8-NEXT:    v_max_f32_e32 v0, 4.0, v0
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: test_max_min_K0_gt_K1:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX12-NEXT:    v_minmax_num_f32 v0, v0, 2.0, 4.0
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %minnum = call nnan float @llvm.minnum.f32(float %a, float 2.0)
   %fmed = call nnan float @llvm.maxnum.f32(float %minnum, float 4.0)
   ret float %fmed
@@ -242,6 +317,12 @@ define float @test_min_max_non_inline_const(float %a) #0 {
 ; GFX8-NEXT:    v_max_f32_e32 v0, 2.0, v0
 ; GFX8-NEXT:    v_min_f32_e32 v0, 0x41000000, v0
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: test_min_max_non_inline_const:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX12-NEXT:    v_maxmin_num_f32 v0, v0, 2.0, 0x41000000
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %maxnum = call nnan float @llvm.maxnum.f32(float %a, float 2.0)
   %fmed = call nnan float @llvm.minnum.f32(float %maxnum, float 8.0)
   ret float %fmed
@@ -263,6 +344,14 @@ define double @test_min_max_f64(double %a) #0 {
 ; GFX8-NEXT:    v_max_f64 v[0:1], v[0:1], 2.0
 ; GFX8-NEXT:    v_min_f64 v[0:1], v[0:1], 4.0
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: test_min_max_f64:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX12-NEXT:    v_max_num_f64_e32 v[0:1], 2.0, v[0:1]
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT:    v_min_num_f64_e32 v[0:1], 4.0, v[0:1]
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %maxnum = call nnan double @llvm.maxnum.f64(double %a, double 2.0)
   %fmed = call nnan double @llvm.minnum.f64(double %maxnum, double 4.0)
   ret double %fmed
@@ -287,6 +376,14 @@ define <2 x half> @test_min_max_v2f16(<2 x half> %a) #0 {
 ; GFX8-NEXT:    v_min_f16_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
 ; GFX8-NEXT:    v_or_b32_e32 v0, v1, v0
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: test_min_max_v2f16:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX12-NEXT:    v_pk_max_num_f16 v0, v0, 2.0 op_sel_hi:[1,0]
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT:    v_pk_min_num_f16 v0, v0, 4.0 op_sel_hi:[1,0]
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %maxnum = call nnan <2 x half> @llvm.maxnum.v2f16(<2 x half> %a, <2 x half> <half 2.0, half 2.0>)
   %fmed = call nnan <2 x half> @llvm.minnum.v2f16(<2 x half> %maxnum, <2 x half> <half 4.0, half 4.0>)
   ret <2 x half> %fmed
@@ -309,6 +406,14 @@ define float @test_min_max_maybe_NaN_input_ieee_false(float %a) #1 {
 ; GFX8-NEXT:    v_max_f32_e32 v0, 2.0, v0
 ; GFX8-NEXT:    v_min_f32_e32 v0, 4.0, v0
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: test_min_max_maybe_NaN_input_ieee_false:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX12-NEXT:    v_max_num_f32_e32 v0, v0, v0
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT:    v_med3_num_f32 v0, v0, 2.0, 4.0
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %maxnum = call float @llvm.maxnum.f32(float %a, float 2.0)
   %fmed = call float @llvm.minnum.f32(float %maxnum, float 4.0)
   ret float %fmed
@@ -330,6 +435,14 @@ define float @test_max_min_maybe_NaN_input_ieee_false(float %a) #1 {
 ; GFX8-NEXT:    v_min_f32_e32 v0, 4.0, v0
 ; GFX8-NEXT:    v_max_f32_e32 v0, 2.0, v0
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: test_max_min_maybe_NaN_input_ieee_false:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX12-NEXT:    v_max_num_f32_e32 v0, v0, v0
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT:    v_minmax_num_f32 v0, v0, 4.0, 2.0
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %minnum = call float @llvm.minnum.f32(float %a, float 4.0)
   %fmed = call float @llvm.maxnum.f32(float %minnum, float 2.0)
   ret float %fmed
@@ -352,6 +465,14 @@ define float @test_max_min_maybe_NaN_input_ieee_true(float %a) #0 {
 ; GFX8-NEXT:    v_min_f32_e32 v0, 4.0, v0
 ; GFX8-NEXT:    v_max_f32_e32 v0, 2.0, v0
 ; GFX8-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: test_max_min_maybe_NaN_input_ieee_true:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX12-NEXT:    v_max_num_f32_e32 v0, v0, v0
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT:    v_minmax_num_f32 v0, v0, 4.0, 2.0
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %minnum = call float @llvm.minnum.f32(float %a, float 4.0)
   %fmed = call float @llvm.maxnum.f32(float %minnum, float 2.0)
   ret float %fmed
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.i16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.i16.ll
index 30b66e1fdd3405f0a534861b8dc505b693c83156..676a02aaf8a19c221bb55a5ee757b7bc0b20a418 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.i16.ll
@@ -3296,6 +3296,10 @@ define amdgpu_ps void @insertelement_v_v16i16_s_s(ptr addrspace(1) %ptr, i16 inr
 ; GFX9-NEXT:    v_cmp_eq_u32_e64 s[6:7], s12, 5
 ; GFX9-NEXT:    v_cmp_eq_u32_e64 s[8:9], s12, 6
 ; GFX9-NEXT:    v_cmp_eq_u32_e64 s[10:11], s12, 7
+; GFX9-NEXT:    v_mov_b32_e32 v10, 0
+; GFX9-NEXT:    v_mov_b32_e32 v11, 0
+; GFX9-NEXT:    v_mov_b32_e32 v12, 16
+; GFX9-NEXT:    v_mov_b32_e32 v13, 0
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_cndmask_b32_e32 v1, v2, v3, vcc
 ; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, v4, s[0:1]
@@ -3305,22 +3309,18 @@ define amdgpu_ps void @insertelement_v_v16i16_s_s(ptr addrspace(1) %ptr, i16 inr
 ; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, v7, s[6:7]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, v8, s[8:9]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, v9, s[10:11]
-; GFX9-NEXT:    v_and_or_b32 v10, v1, s13, v0
+; GFX9-NEXT:    v_and_or_b32 v14, v1, s13, v0
 ; GFX9-NEXT:    v_cmp_eq_u32_e64 s[12:13], s12, 0
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v2, v10, s[12:13]
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v10, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, v4, v10, s[0:1]
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, v5, v10, s[2:3]
-; GFX9-NEXT:    v_cndmask_b32_e64 v4, v6, v10, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v5, v7, v10, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v6, v8, v10, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e64 v7, v9, v10, s[10:11]
-; GFX9-NEXT:    v_mov_b32_e32 v8, 0
-; GFX9-NEXT:    v_mov_b32_e32 v9, 0
-; GFX9-NEXT:    v_mov_b32_e32 v10, 16
-; GFX9-NEXT:    v_mov_b32_e32 v11, 0
-; GFX9-NEXT:    global_store_dwordx4 v[8:9], v[0:3], off
-; GFX9-NEXT:    global_store_dwordx4 v[10:11], v[4:7], off
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v2, v14, s[12:13]
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v14, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, v4, v14, s[0:1]
+; GFX9-NEXT:    v_cndmask_b32_e64 v3, v5, v14, s[2:3]
+; GFX9-NEXT:    v_cndmask_b32_e64 v4, v6, v14, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v5, v7, v14, s[6:7]
+; GFX9-NEXT:    v_cndmask_b32_e64 v6, v8, v14, s[8:9]
+; GFX9-NEXT:    v_cndmask_b32_e64 v7, v9, v14, s[10:11]
+; GFX9-NEXT:    global_store_dwordx4 v[10:11], v[0:3], off
+; GFX9-NEXT:    global_store_dwordx4 v[12:13], v[4:7], off
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: insertelement_v_v16i16_s_s:
@@ -3436,52 +3436,52 @@ define amdgpu_ps void @insertelement_s_v16i16_v_s(ptr addrspace(4) inreg %ptr, i
 ; GFX9-LABEL: insertelement_s_v16i16_v_s:
 ; GFX9:       ; %bb.0:
 ; GFX9-NEXT:    s_load_dwordx8 s[8:15], s[2:3], 0x0
-; GFX9-NEXT:    s_lshr_b32 s2, s4, 1
-; GFX9-NEXT:    s_cmp_eq_u32 s2, 1
+; GFX9-NEXT:    s_lshr_b32 s0, s4, 1
+; GFX9-NEXT:    s_cmp_eq_u32 s0, 1
 ; GFX9-NEXT:    v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NEXT:    v_cmp_eq_u32_e64 vcc, s2, 0
+; GFX9-NEXT:    v_cmp_eq_u32_e64 vcc, s0, 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_cselect_b32 s0, s9, s8
-; GFX9-NEXT:    s_cmp_eq_u32 s2, 2
-; GFX9-NEXT:    s_cselect_b32 s0, s10, s0
-; GFX9-NEXT:    s_cmp_eq_u32 s2, 3
-; GFX9-NEXT:    s_cselect_b32 s0, s11, s0
-; GFX9-NEXT:    s_cmp_eq_u32 s2, 4
-; GFX9-NEXT:    s_cselect_b32 s0, s12, s0
-; GFX9-NEXT:    s_cmp_eq_u32 s2, 5
-; GFX9-NEXT:    s_cselect_b32 s0, s13, s0
-; GFX9-NEXT:    s_cmp_eq_u32 s2, 6
-; GFX9-NEXT:    s_cselect_b32 s0, s14, s0
-; GFX9-NEXT:    s_cmp_eq_u32 s2, 7
-; GFX9-NEXT:    s_cselect_b32 s0, s15, s0
-; GFX9-NEXT:    s_and_b32 s1, s4, 1
-; GFX9-NEXT:    s_lshl_b32 s1, s1, 4
-; GFX9-NEXT:    s_lshl_b32 s3, 0xffff, s1
-; GFX9-NEXT:    s_andn2_b32 s0, s0, s3
-; GFX9-NEXT:    v_mov_b32_e32 v1, s0
-; GFX9-NEXT:    v_lshl_or_b32 v8, v0, s1, v1
+; GFX9-NEXT:    s_cselect_b32 s1, s9, s8
+; GFX9-NEXT:    s_cmp_eq_u32 s0, 2
+; GFX9-NEXT:    s_cselect_b32 s1, s10, s1
+; GFX9-NEXT:    s_cmp_eq_u32 s0, 3
+; GFX9-NEXT:    s_cselect_b32 s1, s11, s1
+; GFX9-NEXT:    s_cmp_eq_u32 s0, 4
+; GFX9-NEXT:    s_cselect_b32 s1, s12, s1
+; GFX9-NEXT:    s_cmp_eq_u32 s0, 5
+; GFX9-NEXT:    s_cselect_b32 s1, s13, s1
+; GFX9-NEXT:    s_cmp_eq_u32 s0, 6
+; GFX9-NEXT:    s_cselect_b32 s1, s14, s1
+; GFX9-NEXT:    s_cmp_eq_u32 s0, 7
+; GFX9-NEXT:    s_cselect_b32 s1, s15, s1
+; GFX9-NEXT:    s_and_b32 s2, s4, 1
+; GFX9-NEXT:    s_lshl_b32 s2, s2, 4
+; GFX9-NEXT:    s_lshl_b32 s3, 0xffff, s2
+; GFX9-NEXT:    s_andn2_b32 s1, s1, s3
+; GFX9-NEXT:    v_mov_b32_e32 v1, s1
+; GFX9-NEXT:    v_lshl_or_b32 v8, v0, s2, v1
 ; GFX9-NEXT:    v_mov_b32_e32 v0, s8
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s9
 ; GFX9-NEXT:    v_cndmask_b32_e32 v0, v0, v8, vcc
-; GFX9-NEXT:    v_cmp_eq_u32_e64 vcc, s2, 1
+; GFX9-NEXT:    v_cmp_eq_u32_e64 vcc, s0, 1
 ; GFX9-NEXT:    v_mov_b32_e32 v2, s10
 ; GFX9-NEXT:    v_cndmask_b32_e32 v1, v1, v8, vcc
-; GFX9-NEXT:    v_cmp_eq_u32_e64 vcc, s2, 2
+; GFX9-NEXT:    v_cmp_eq_u32_e64 vcc, s0, 2
 ; GFX9-NEXT:    v_mov_b32_e32 v3, s11
 ; GFX9-NEXT:    v_cndmask_b32_e32 v2, v2, v8, vcc
-; GFX9-NEXT:    v_cmp_eq_u32_e64 vcc, s2, 3
-; GFX9-NEXT:    v_mov_b32_e32 v5, s13
+; GFX9-NEXT:    v_cmp_eq_u32_e64 vcc, s0, 3
+; GFX9-NEXT:    v_mov_b32_e32 v4, s12
 ; GFX9-NEXT:    v_cndmask_b32_e32 v3, v3, v8, vcc
-; GFX9-NEXT:    v_cmp_eq_u32_e64 vcc, s2, 5
+; GFX9-NEXT:    v_cmp_eq_u32_e64 vcc, s0, 4
+; GFX9-NEXT:    v_mov_b32_e32 v5, s13
+; GFX9-NEXT:    v_cndmask_b32_e32 v4, v4, v8, vcc
+; GFX9-NEXT:    v_cmp_eq_u32_e64 vcc, s0, 5
 ; GFX9-NEXT:    v_mov_b32_e32 v6, s14
 ; GFX9-NEXT:    v_cndmask_b32_e32 v5, v5, v8, vcc
-; GFX9-NEXT:    v_cmp_eq_u32_e64 vcc, s2, 6
-; GFX9-NEXT:    v_mov_b32_e32 v4, s12
+; GFX9-NEXT:    v_cmp_eq_u32_e64 vcc, s0, 6
 ; GFX9-NEXT:    v_mov_b32_e32 v7, s15
-; GFX9-NEXT:    v_cmp_eq_u32_e64 s[0:1], s2, 4
 ; GFX9-NEXT:    v_cndmask_b32_e32 v6, v6, v8, vcc
-; GFX9-NEXT:    v_cmp_eq_u32_e64 vcc, s2, 7
-; GFX9-NEXT:    v_cndmask_b32_e64 v4, v4, v8, s[0:1]
+; GFX9-NEXT:    v_cmp_eq_u32_e64 vcc, s0, 7
 ; GFX9-NEXT:    v_cndmask_b32_e32 v7, v7, v8, vcc
 ; GFX9-NEXT:    v_mov_b32_e32 v8, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v9, 0
@@ -4215,7 +4215,7 @@ define amdgpu_ps void @insertelement_v_v16i16_s_v(ptr addrspace(1) %ptr, i16 inr
 ; GFX9-NEXT:    s_and_b32 s0, s2, 0xffff
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v2, 4, v2
 ; GFX9-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v1
-; GFX9-NEXT:    v_lshlrev_b32_e64 v11, v2, s0
+; GFX9-NEXT:    v_lshlrev_b32_e64 v15, v2, s0
 ; GFX9-NEXT:    v_lshlrev_b32_e32 v0, v2, v0
 ; GFX9-NEXT:    v_cmp_eq_u32_e64 s[0:1], 2, v1
 ; GFX9-NEXT:    v_cmp_eq_u32_e64 s[2:3], 3, v1
@@ -4224,7 +4224,11 @@ define amdgpu_ps void @insertelement_v_v16i16_s_v(ptr addrspace(1) %ptr, i16 inr
 ; GFX9-NEXT:    v_cmp_eq_u32_e64 s[8:9], 6, v1
 ; GFX9-NEXT:    v_cmp_eq_u32_e64 s[10:11], 7, v1
 ; GFX9-NEXT:    v_not_b32_e32 v0, v0
+; GFX9-NEXT:    v_mov_b32_e32 v11, 0
 ; GFX9-NEXT:    v_cmp_eq_u32_e64 s[12:13], 0, v1
+; GFX9-NEXT:    v_mov_b32_e32 v12, 0
+; GFX9-NEXT:    v_mov_b32_e32 v13, 16
+; GFX9-NEXT:    v_mov_b32_e32 v14, 0
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_cndmask_b32_e32 v2, v3, v4, vcc
 ; GFX9-NEXT:    v_cndmask_b32_e64 v2, v2, v5, s[0:1]
@@ -4234,21 +4238,17 @@ define amdgpu_ps void @insertelement_v_v16i16_s_v(ptr addrspace(1) %ptr, i16 inr
 ; GFX9-NEXT:    v_cndmask_b32_e64 v2, v2, v8, s[6:7]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v2, v2, v9, s[8:9]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v2, v2, v10, s[10:11]
-; GFX9-NEXT:    v_and_or_b32 v11, v2, v0, v11
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v3, v11, s[12:13]
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, v5, v11, s[0:1]
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, v6, v11, s[2:3]
-; GFX9-NEXT:    v_cndmask_b32_e64 v5, v8, v11, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v6, v9, v11, s[8:9]
-; GFX9-NEXT:    v_mov_b32_e32 v8, 0
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v4, v11, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v4, v7, v11, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v7, v10, v11, s[10:11]
-; GFX9-NEXT:    v_mov_b32_e32 v9, 0
-; GFX9-NEXT:    v_mov_b32_e32 v10, 16
-; GFX9-NEXT:    v_mov_b32_e32 v11, 0
-; GFX9-NEXT:    global_store_dwordx4 v[8:9], v[0:3], off
-; GFX9-NEXT:    global_store_dwordx4 v[10:11], v[4:7], off
+; GFX9-NEXT:    v_and_or_b32 v15, v2, v0, v15
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v3, v15, s[12:13]
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v4, v15, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, v5, v15, s[0:1]
+; GFX9-NEXT:    v_cndmask_b32_e64 v3, v6, v15, s[2:3]
+; GFX9-NEXT:    v_cndmask_b32_e64 v4, v7, v15, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v5, v8, v15, s[6:7]
+; GFX9-NEXT:    v_cndmask_b32_e64 v6, v9, v15, s[8:9]
+; GFX9-NEXT:    v_cndmask_b32_e64 v7, v10, v15, s[10:11]
+; GFX9-NEXT:    global_store_dwordx4 v[11:12], v[0:3], off
+; GFX9-NEXT:    global_store_dwordx4 v[13:14], v[4:7], off
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: insertelement_v_v16i16_s_v:
@@ -4263,7 +4263,7 @@ define amdgpu_ps void @insertelement_v_v16i16_s_v(ptr addrspace(1) %ptr, i16 inr
 ; GFX8-NEXT:    s_and_b32 s0, s2, 0xffff
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v2, 4, v2
 ; GFX8-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v1
-; GFX8-NEXT:    v_lshlrev_b32_e64 v11, v2, s0
+; GFX8-NEXT:    v_lshlrev_b32_e64 v15, v2, s0
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v0, v2, v0
 ; GFX8-NEXT:    v_cmp_eq_u32_e64 s[0:1], 2, v1
 ; GFX8-NEXT:    v_cmp_eq_u32_e64 s[2:3], 3, v1
@@ -4272,7 +4272,11 @@ define amdgpu_ps void @insertelement_v_v16i16_s_v(ptr addrspace(1) %ptr, i16 inr
 ; GFX8-NEXT:    v_cmp_eq_u32_e64 s[8:9], 6, v1
 ; GFX8-NEXT:    v_cmp_eq_u32_e64 s[10:11], 7, v1
 ; GFX8-NEXT:    v_not_b32_e32 v0, v0
+; GFX8-NEXT:    v_mov_b32_e32 v11, 0
 ; GFX8-NEXT:    v_cmp_eq_u32_e64 s[12:13], 0, v1
+; GFX8-NEXT:    v_mov_b32_e32 v12, 0
+; GFX8-NEXT:    v_mov_b32_e32 v13, 16
+; GFX8-NEXT:    v_mov_b32_e32 v14, 0
 ; GFX8-NEXT:    s_waitcnt vmcnt(1)
 ; GFX8-NEXT:    v_cndmask_b32_e32 v2, v3, v4, vcc
 ; GFX8-NEXT:    v_cndmask_b32_e64 v2, v2, v5, s[0:1]
@@ -4283,30 +4287,26 @@ define amdgpu_ps void @insertelement_v_v16i16_s_v(ptr addrspace(1) %ptr, i16 inr
 ; GFX8-NEXT:    v_cndmask_b32_e64 v2, v2, v9, s[8:9]
 ; GFX8-NEXT:    v_cndmask_b32_e64 v2, v2, v10, s[10:11]
 ; GFX8-NEXT:    v_and_b32_e32 v0, v2, v0
-; GFX8-NEXT:    v_or_b32_e32 v11, v0, v11
-; GFX8-NEXT:    v_cndmask_b32_e64 v0, v3, v11, s[12:13]
-; GFX8-NEXT:    v_cndmask_b32_e64 v2, v5, v11, s[0:1]
-; GFX8-NEXT:    v_cndmask_b32_e64 v3, v6, v11, s[2:3]
-; GFX8-NEXT:    v_cndmask_b32_e64 v5, v8, v11, s[6:7]
-; GFX8-NEXT:    v_cndmask_b32_e64 v6, v9, v11, s[8:9]
-; GFX8-NEXT:    v_mov_b32_e32 v8, 0
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v4, v11, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v4, v7, v11, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v7, v10, v11, s[10:11]
-; GFX8-NEXT:    v_mov_b32_e32 v9, 0
-; GFX8-NEXT:    v_mov_b32_e32 v10, 16
-; GFX8-NEXT:    v_mov_b32_e32 v11, 0
-; GFX8-NEXT:    flat_store_dwordx4 v[8:9], v[0:3]
-; GFX8-NEXT:    flat_store_dwordx4 v[10:11], v[4:7]
+; GFX8-NEXT:    v_or_b32_e32 v15, v0, v15
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v3, v15, s[12:13]
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v4, v15, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, v5, v15, s[0:1]
+; GFX8-NEXT:    v_cndmask_b32_e64 v3, v6, v15, s[2:3]
+; GFX8-NEXT:    v_cndmask_b32_e64 v4, v7, v15, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v5, v8, v15, s[6:7]
+; GFX8-NEXT:    v_cndmask_b32_e64 v6, v9, v15, s[8:9]
+; GFX8-NEXT:    v_cndmask_b32_e64 v7, v10, v15, s[10:11]
+; GFX8-NEXT:    flat_store_dwordx4 v[11:12], v[0:3]
+; GFX8-NEXT:    flat_store_dwordx4 v[13:14], v[4:7]
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX7-LABEL: insertelement_v_v16i16_s_v:
 ; GFX7:       ; %bb.0:
-; GFX7-NEXT:    s_mov_b32 s14, 0
-; GFX7-NEXT:    s_mov_b32 s15, 0xf000
-; GFX7-NEXT:    s_mov_b64 s[12:13], 0
-; GFX7-NEXT:    buffer_load_dwordx4 v[3:6], v[0:1], s[12:15], 0 addr64
-; GFX7-NEXT:    buffer_load_dwordx4 v[7:10], v[0:1], s[12:15], 0 addr64 offset:16
+; GFX7-NEXT:    s_mov_b32 s18, 0
+; GFX7-NEXT:    s_mov_b32 s19, 0xf000
+; GFX7-NEXT:    s_mov_b64 s[16:17], 0
+; GFX7-NEXT:    buffer_load_dwordx4 v[3:6], v[0:1], s[16:19], 0 addr64
+; GFX7-NEXT:    buffer_load_dwordx4 v[7:10], v[0:1], s[16:19], 0 addr64 offset:16
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v0, 1, v2
 ; GFX7-NEXT:    v_and_b32_e32 v1, 1, v2
 ; GFX7-NEXT:    s_and_b32 s0, s2, 0xffff
@@ -4322,7 +4322,8 @@ define amdgpu_ps void @insertelement_v_v16i16_s_v(ptr addrspace(1) %ptr, i16 inr
 ; GFX7-NEXT:    v_cmp_eq_u32_e64 s[10:11], 7, v0
 ; GFX7-NEXT:    v_not_b32_e32 v1, v1
 ; GFX7-NEXT:    v_cmp_eq_u32_e64 s[12:13], 0, v0
-; GFX7-NEXT:    s_mov_b32 s14, -1
+; GFX7-NEXT:    s_mov_b64 s[16:17], 0
+; GFX7-NEXT:    s_mov_b32 s18, -1
 ; GFX7-NEXT:    s_waitcnt vmcnt(1)
 ; GFX7-NEXT:    v_cndmask_b32_e32 v11, v3, v4, vcc
 ; GFX7-NEXT:    v_cndmask_b32_e64 v11, v11, v5, s[0:1]
@@ -4338,14 +4339,13 @@ define amdgpu_ps void @insertelement_v_v16i16_s_v(ptr addrspace(1) %ptr, i16 inr
 ; GFX7-NEXT:    v_cndmask_b32_e32 v1, v4, v11, vcc
 ; GFX7-NEXT:    v_cndmask_b32_e64 v2, v5, v11, s[0:1]
 ; GFX7-NEXT:    v_cndmask_b32_e64 v3, v6, v11, s[2:3]
-; GFX7-NEXT:    s_mov_b64 s[12:13], 0
 ; GFX7-NEXT:    v_cndmask_b32_e64 v4, v7, v11, s[4:5]
 ; GFX7-NEXT:    v_cndmask_b32_e64 v5, v8, v11, s[6:7]
 ; GFX7-NEXT:    v_cndmask_b32_e64 v6, v9, v11, s[8:9]
 ; GFX7-NEXT:    v_cndmask_b32_e64 v7, v10, v11, s[10:11]
-; GFX7-NEXT:    buffer_store_dwordx4 v[0:3], off, s[12:15], 0
-; GFX7-NEXT:    s_mov_b64 s[12:13], 16
-; GFX7-NEXT:    buffer_store_dwordx4 v[4:7], off, s[12:15], 0
+; GFX7-NEXT:    buffer_store_dwordx4 v[0:3], off, s[16:19], 0
+; GFX7-NEXT:    s_mov_b64 s[16:17], 16
+; GFX7-NEXT:    buffer_store_dwordx4 v[4:7], off, s[16:19], 0
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX10-LABEL: insertelement_v_v16i16_s_v:
@@ -4470,6 +4470,10 @@ define amdgpu_ps void @insertelement_v_v16i16_v_s(ptr addrspace(1) %ptr, i16 %va
 ; GFX9-NEXT:    v_cmp_eq_u32_e64 s[6:7], s12, 5
 ; GFX9-NEXT:    v_cmp_eq_u32_e64 s[8:9], s12, 6
 ; GFX9-NEXT:    v_cmp_eq_u32_e64 s[10:11], s12, 7
+; GFX9-NEXT:    v_mov_b32_e32 v11, 0
+; GFX9-NEXT:    v_mov_b32_e32 v12, 0
+; GFX9-NEXT:    v_mov_b32_e32 v13, 16
+; GFX9-NEXT:    v_mov_b32_e32 v14, 0
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_cndmask_b32_e32 v1, v3, v4, vcc
 ; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, v5, s[0:1]
@@ -4479,22 +4483,18 @@ define amdgpu_ps void @insertelement_v_v16i16_v_s(ptr addrspace(1) %ptr, i16 %va
 ; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, v8, s[6:7]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, v9, s[8:9]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v1, v1, v10, s[10:11]
-; GFX9-NEXT:    v_and_or_b32 v11, v1, s13, v0
+; GFX9-NEXT:    v_and_or_b32 v15, v1, s13, v0
 ; GFX9-NEXT:    v_cmp_eq_u32_e64 s[12:13], s12, 0
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v3, v11, s[12:13]
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, v5, v11, s[0:1]
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, v6, v11, s[2:3]
-; GFX9-NEXT:    v_cndmask_b32_e64 v5, v8, v11, s[6:7]
-; GFX9-NEXT:    v_cndmask_b32_e64 v6, v9, v11, s[8:9]
-; GFX9-NEXT:    v_mov_b32_e32 v8, 0
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v4, v11, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v4, v7, v11, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v7, v10, v11, s[10:11]
-; GFX9-NEXT:    v_mov_b32_e32 v9, 0
-; GFX9-NEXT:    v_mov_b32_e32 v10, 16
-; GFX9-NEXT:    v_mov_b32_e32 v11, 0
-; GFX9-NEXT:    global_store_dwordx4 v[8:9], v[0:3], off
-; GFX9-NEXT:    global_store_dwordx4 v[10:11], v[4:7], off
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v3, v15, s[12:13]
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v4, v15, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, v5, v15, s[0:1]
+; GFX9-NEXT:    v_cndmask_b32_e64 v3, v6, v15, s[2:3]
+; GFX9-NEXT:    v_cndmask_b32_e64 v4, v7, v15, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v5, v8, v15, s[6:7]
+; GFX9-NEXT:    v_cndmask_b32_e64 v6, v9, v15, s[8:9]
+; GFX9-NEXT:    v_cndmask_b32_e64 v7, v10, v15, s[10:11]
+; GFX9-NEXT:    global_store_dwordx4 v[11:12], v[0:3], off
+; GFX9-NEXT:    global_store_dwordx4 v[13:14], v[4:7], off
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: insertelement_v_v16i16_v_s:
@@ -4624,7 +4624,11 @@ define amdgpu_ps void @insertelement_v_v16i16_v_v(ptr addrspace(1) %ptr, i16 %va
 ; GFX9-NEXT:    v_cmp_eq_u32_e64 s[8:9], 6, v1
 ; GFX9-NEXT:    v_cmp_eq_u32_e64 s[10:11], 7, v1
 ; GFX9-NEXT:    v_not_b32_e32 v0, v0
+; GFX9-NEXT:    v_mov_b32_e32 v12, 0
 ; GFX9-NEXT:    v_cmp_eq_u32_e64 s[12:13], 0, v1
+; GFX9-NEXT:    v_mov_b32_e32 v13, 0
+; GFX9-NEXT:    v_mov_b32_e32 v14, 16
+; GFX9-NEXT:    v_mov_b32_e32 v15, 0
 ; GFX9-NEXT:    s_waitcnt vmcnt(1)
 ; GFX9-NEXT:    v_cndmask_b32_e32 v3, v4, v5, vcc
 ; GFX9-NEXT:    v_cndmask_b32_e64 v3, v3, v6, s[0:1]
@@ -4634,21 +4638,17 @@ define amdgpu_ps void @insertelement_v_v16i16_v_v(ptr addrspace(1) %ptr, i16 %va
 ; GFX9-NEXT:    v_cndmask_b32_e64 v3, v3, v9, s[6:7]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v3, v3, v10, s[8:9]
 ; GFX9-NEXT:    v_cndmask_b32_e64 v3, v3, v11, s[10:11]
-; GFX9-NEXT:    v_and_or_b32 v12, v3, v0, v2
-; GFX9-NEXT:    v_cndmask_b32_e64 v0, v4, v12, s[12:13]
-; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v12, vcc
-; GFX9-NEXT:    v_cndmask_b32_e64 v4, v8, v12, s[4:5]
-; GFX9-NEXT:    v_cndmask_b32_e64 v5, v9, v12, s[6:7]
-; GFX9-NEXT:    v_mov_b32_e32 v8, 0
-; GFX9-NEXT:    v_cndmask_b32_e64 v2, v6, v12, s[0:1]
-; GFX9-NEXT:    v_cndmask_b32_e64 v3, v7, v12, s[2:3]
-; GFX9-NEXT:    v_cndmask_b32_e64 v6, v10, v12, s[8:9]
-; GFX9-NEXT:    v_cndmask_b32_e64 v7, v11, v12, s[10:11]
-; GFX9-NEXT:    v_mov_b32_e32 v9, 0
-; GFX9-NEXT:    v_mov_b32_e32 v10, 16
-; GFX9-NEXT:    v_mov_b32_e32 v11, 0
-; GFX9-NEXT:    global_store_dwordx4 v[8:9], v[0:3], off
-; GFX9-NEXT:    global_store_dwordx4 v[10:11], v[4:7], off
+; GFX9-NEXT:    v_and_or_b32 v16, v3, v0, v2
+; GFX9-NEXT:    v_cndmask_b32_e64 v0, v4, v16, s[12:13]
+; GFX9-NEXT:    v_cndmask_b32_e32 v1, v5, v16, vcc
+; GFX9-NEXT:    v_cndmask_b32_e64 v2, v6, v16, s[0:1]
+; GFX9-NEXT:    v_cndmask_b32_e64 v3, v7, v16, s[2:3]
+; GFX9-NEXT:    v_cndmask_b32_e64 v4, v8, v16, s[4:5]
+; GFX9-NEXT:    v_cndmask_b32_e64 v5, v9, v16, s[6:7]
+; GFX9-NEXT:    v_cndmask_b32_e64 v6, v10, v16, s[8:9]
+; GFX9-NEXT:    v_cndmask_b32_e64 v7, v11, v16, s[10:11]
+; GFX9-NEXT:    global_store_dwordx4 v[12:13], v[0:3], off
+; GFX9-NEXT:    global_store_dwordx4 v[14:15], v[4:7], off
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: insertelement_v_v16i16_v_v:
@@ -4671,7 +4671,11 @@ define amdgpu_ps void @insertelement_v_v16i16_v_v(ptr addrspace(1) %ptr, i16 %va
 ; GFX8-NEXT:    v_cmp_eq_u32_e64 s[8:9], 6, v1
 ; GFX8-NEXT:    v_cmp_eq_u32_e64 s[10:11], 7, v1
 ; GFX8-NEXT:    v_not_b32_e32 v0, v0
+; GFX8-NEXT:    v_mov_b32_e32 v12, 0
 ; GFX8-NEXT:    v_cmp_eq_u32_e64 s[12:13], 0, v1
+; GFX8-NEXT:    v_mov_b32_e32 v13, 0
+; GFX8-NEXT:    v_mov_b32_e32 v14, 16
+; GFX8-NEXT:    v_mov_b32_e32 v15, 0
 ; GFX8-NEXT:    s_waitcnt vmcnt(1)
 ; GFX8-NEXT:    v_cndmask_b32_e32 v3, v4, v5, vcc
 ; GFX8-NEXT:    v_cndmask_b32_e64 v3, v3, v6, s[0:1]
@@ -4682,30 +4686,26 @@ define amdgpu_ps void @insertelement_v_v16i16_v_v(ptr addrspace(1) %ptr, i16 %va
 ; GFX8-NEXT:    v_cndmask_b32_e64 v3, v3, v10, s[8:9]
 ; GFX8-NEXT:    v_cndmask_b32_e64 v3, v3, v11, s[10:11]
 ; GFX8-NEXT:    v_and_b32_e32 v0, v3, v0
-; GFX8-NEXT:    v_or_b32_e32 v12, v0, v2
-; GFX8-NEXT:    v_cndmask_b32_e64 v0, v4, v12, s[12:13]
-; GFX8-NEXT:    v_cndmask_b32_e32 v1, v5, v12, vcc
-; GFX8-NEXT:    v_cndmask_b32_e64 v4, v8, v12, s[4:5]
-; GFX8-NEXT:    v_cndmask_b32_e64 v5, v9, v12, s[6:7]
-; GFX8-NEXT:    v_mov_b32_e32 v8, 0
-; GFX8-NEXT:    v_cndmask_b32_e64 v2, v6, v12, s[0:1]
-; GFX8-NEXT:    v_cndmask_b32_e64 v3, v7, v12, s[2:3]
-; GFX8-NEXT:    v_cndmask_b32_e64 v6, v10, v12, s[8:9]
-; GFX8-NEXT:    v_cndmask_b32_e64 v7, v11, v12, s[10:11]
-; GFX8-NEXT:    v_mov_b32_e32 v9, 0
-; GFX8-NEXT:    v_mov_b32_e32 v10, 16
-; GFX8-NEXT:    v_mov_b32_e32 v11, 0
-; GFX8-NEXT:    flat_store_dwordx4 v[8:9], v[0:3]
-; GFX8-NEXT:    flat_store_dwordx4 v[10:11], v[4:7]
+; GFX8-NEXT:    v_or_b32_e32 v16, v0, v2
+; GFX8-NEXT:    v_cndmask_b32_e64 v0, v4, v16, s[12:13]
+; GFX8-NEXT:    v_cndmask_b32_e32 v1, v5, v16, vcc
+; GFX8-NEXT:    v_cndmask_b32_e64 v2, v6, v16, s[0:1]
+; GFX8-NEXT:    v_cndmask_b32_e64 v3, v7, v16, s[2:3]
+; GFX8-NEXT:    v_cndmask_b32_e64 v4, v8, v16, s[4:5]
+; GFX8-NEXT:    v_cndmask_b32_e64 v5, v9, v16, s[6:7]
+; GFX8-NEXT:    v_cndmask_b32_e64 v6, v10, v16, s[8:9]
+; GFX8-NEXT:    v_cndmask_b32_e64 v7, v11, v16, s[10:11]
+; GFX8-NEXT:    flat_store_dwordx4 v[12:13], v[0:3]
+; GFX8-NEXT:    flat_store_dwordx4 v[14:15], v[4:7]
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX7-LABEL: insertelement_v_v16i16_v_v:
 ; GFX7:       ; %bb.0:
-; GFX7-NEXT:    s_mov_b32 s14, 0
-; GFX7-NEXT:    s_mov_b32 s15, 0xf000
-; GFX7-NEXT:    s_mov_b64 s[12:13], 0
-; GFX7-NEXT:    buffer_load_dwordx4 v[4:7], v[0:1], s[12:15], 0 addr64
-; GFX7-NEXT:    buffer_load_dwordx4 v[8:11], v[0:1], s[12:15], 0 addr64 offset:16
+; GFX7-NEXT:    s_mov_b32 s18, 0
+; GFX7-NEXT:    s_mov_b32 s19, 0xf000
+; GFX7-NEXT:    s_mov_b64 s[16:17], 0
+; GFX7-NEXT:    buffer_load_dwordx4 v[4:7], v[0:1], s[16:19], 0 addr64
+; GFX7-NEXT:    buffer_load_dwordx4 v[8:11], v[0:1], s[16:19], 0 addr64 offset:16
 ; GFX7-NEXT:    v_lshrrev_b32_e32 v0, 1, v3
 ; GFX7-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v0
 ; GFX7-NEXT:    v_and_b32_e32 v1, 1, v3
@@ -4721,7 +4721,8 @@ define amdgpu_ps void @insertelement_v_v16i16_v_v(ptr addrspace(1) %ptr, i16 %va
 ; GFX7-NEXT:    v_cmp_eq_u32_e64 s[10:11], 7, v0
 ; GFX7-NEXT:    v_not_b32_e32 v1, v1
 ; GFX7-NEXT:    v_cmp_eq_u32_e64 s[12:13], 0, v0
-; GFX7-NEXT:    s_mov_b32 s14, -1
+; GFX7-NEXT:    s_mov_b64 s[16:17], 0
+; GFX7-NEXT:    s_mov_b32 s18, -1
 ; GFX7-NEXT:    s_waitcnt vmcnt(1)
 ; GFX7-NEXT:    v_cndmask_b32_e32 v3, v4, v5, vcc
 ; GFX7-NEXT:    v_cndmask_b32_e64 v3, v3, v6, s[0:1]
@@ -4737,14 +4738,13 @@ define amdgpu_ps void @insertelement_v_v16i16_v_v(ptr addrspace(1) %ptr, i16 %va
 ; GFX7-NEXT:    v_cndmask_b32_e32 v1, v5, v12, vcc
 ; GFX7-NEXT:    v_cndmask_b32_e64 v2, v6, v12, s[0:1]
 ; GFX7-NEXT:    v_cndmask_b32_e64 v3, v7, v12, s[2:3]
-; GFX7-NEXT:    s_mov_b64 s[12:13], 0
 ; GFX7-NEXT:    v_cndmask_b32_e64 v4, v8, v12, s[4:5]
 ; GFX7-NEXT:    v_cndmask_b32_e64 v5, v9, v12, s[6:7]
 ; GFX7-NEXT:    v_cndmask_b32_e64 v6, v10, v12, s[8:9]
 ; GFX7-NEXT:    v_cndmask_b32_e64 v7, v11, v12, s[10:11]
-; GFX7-NEXT:    buffer_store_dwordx4 v[0:3], off, s[12:15], 0
-; GFX7-NEXT:    s_mov_b64 s[12:13], 16
-; GFX7-NEXT:    buffer_store_dwordx4 v[4:7], off, s[12:15], 0
+; GFX7-NEXT:    buffer_store_dwordx4 v[0:3], off, s[16:19], 0
+; GFX7-NEXT:    s_mov_b64 s[16:17], 16
+; GFX7-NEXT:    buffer_store_dwordx4 v[4:7], off, s[16:19], 0
 ; GFX7-NEXT:    s_endpgm
 ;
 ; GFX10-LABEL: insertelement_v_v16i16_v_v:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.large.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.large.ll
index d910058f0ebd921658e4a007783e8245db670f7b..3abc21f812e1453a032454c94ef17e5055c2dbfa 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.large.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.large.ll
@@ -9,43 +9,34 @@ define amdgpu_kernel void @v_insert_v64i32_37(ptr addrspace(1) %ptr.in, ptr addr
 ; GCN-NEXT:    s_load_dwordx4 s[0:3], s[4:5], 0x0
 ; GCN-NEXT:    v_lshlrev_b32_e32 v64, 8, v0
 ; GCN-NEXT:    s_waitcnt lgkmcnt(0)
-; GCN-NEXT:    global_load_dwordx4 v[4:7], v64, s[0:1] offset:144
 ; GCN-NEXT:    global_load_dwordx4 v[0:3], v64, s[0:1]
-; GCN-NEXT:    global_load_dwordx4 v[8:11], v64, s[0:1] offset:16
-; GCN-NEXT:    global_load_dwordx4 v[12:15], v64, s[0:1] offset:32
-; GCN-NEXT:    global_load_dwordx4 v[16:19], v64, s[0:1] offset:48
-; GCN-NEXT:    global_load_dwordx4 v[20:23], v64, s[0:1] offset:64
-; GCN-NEXT:    global_load_dwordx4 v[24:27], v64, s[0:1] offset:80
-; GCN-NEXT:    global_load_dwordx4 v[28:31], v64, s[0:1] offset:96
-; GCN-NEXT:    global_load_dwordx4 v[32:35], v64, s[0:1] offset:112
-; GCN-NEXT:    global_load_dwordx4 v[36:39], v64, s[0:1] offset:128
+; GCN-NEXT:    global_load_dwordx4 v[4:7], v64, s[0:1] offset:16
+; GCN-NEXT:    global_load_dwordx4 v[8:11], v64, s[0:1] offset:32
+; GCN-NEXT:    global_load_dwordx4 v[12:15], v64, s[0:1] offset:48
+; GCN-NEXT:    global_load_dwordx4 v[16:19], v64, s[0:1] offset:64
+; GCN-NEXT:    global_load_dwordx4 v[20:23], v64, s[0:1] offset:80
+; GCN-NEXT:    global_load_dwordx4 v[24:27], v64, s[0:1] offset:96
+; GCN-NEXT:    global_load_dwordx4 v[28:31], v64, s[0:1] offset:112
+; GCN-NEXT:    global_load_dwordx4 v[32:35], v64, s[0:1] offset:128
+; GCN-NEXT:    global_load_dwordx4 v[36:39], v64, s[0:1] offset:144
 ; GCN-NEXT:    global_load_dwordx4 v[40:43], v64, s[0:1] offset:160
 ; GCN-NEXT:    global_load_dwordx4 v[44:47], v64, s[0:1] offset:176
 ; GCN-NEXT:    global_load_dwordx4 v[48:51], v64, s[0:1] offset:192
 ; GCN-NEXT:    global_load_dwordx4 v[52:55], v64, s[0:1] offset:208
 ; GCN-NEXT:    global_load_dwordx4 v[56:59], v64, s[0:1] offset:224
 ; GCN-NEXT:    global_load_dwordx4 v[60:63], v64, s[0:1] offset:240
-; GCN-NEXT:    s_waitcnt vmcnt(15)
-; GCN-NEXT:    v_mov_b32_e32 v5, 0x3e7
-; GCN-NEXT:    global_store_dwordx4 v64, v[4:7], s[2:3] offset:144
-; GCN-NEXT:    s_waitcnt vmcnt(15)
+; GCN-NEXT:    s_waitcnt vmcnt(6)
+; GCN-NEXT:    v_mov_b32_e32 v37, 0x3e7
 ; GCN-NEXT:    global_store_dwordx4 v64, v[0:3], s[2:3]
-; GCN-NEXT:    s_waitcnt vmcnt(15)
-; GCN-NEXT:    global_store_dwordx4 v64, v[8:11], s[2:3] offset:16
-; GCN-NEXT:    s_waitcnt vmcnt(15)
-; GCN-NEXT:    global_store_dwordx4 v64, v[12:15], s[2:3] offset:32
-; GCN-NEXT:    s_waitcnt vmcnt(15)
-; GCN-NEXT:    global_store_dwordx4 v64, v[16:19], s[2:3] offset:48
-; GCN-NEXT:    s_waitcnt vmcnt(15)
-; GCN-NEXT:    global_store_dwordx4 v64, v[20:23], s[2:3] offset:64
-; GCN-NEXT:    s_waitcnt vmcnt(15)
-; GCN-NEXT:    global_store_dwordx4 v64, v[24:27], s[2:3] offset:80
-; GCN-NEXT:    s_waitcnt vmcnt(15)
-; GCN-NEXT:    global_store_dwordx4 v64, v[28:31], s[2:3] offset:96
-; GCN-NEXT:    s_waitcnt vmcnt(15)
-; GCN-NEXT:    global_store_dwordx4 v64, v[32:35], s[2:3] offset:112
-; GCN-NEXT:    s_waitcnt vmcnt(15)
-; GCN-NEXT:    global_store_dwordx4 v64, v[36:39], s[2:3] offset:128
+; GCN-NEXT:    global_store_dwordx4 v64, v[4:7], s[2:3] offset:16
+; GCN-NEXT:    global_store_dwordx4 v64, v[8:11], s[2:3] offset:32
+; GCN-NEXT:    global_store_dwordx4 v64, v[12:15], s[2:3] offset:48
+; GCN-NEXT:    global_store_dwordx4 v64, v[16:19], s[2:3] offset:64
+; GCN-NEXT:    global_store_dwordx4 v64, v[20:23], s[2:3] offset:80
+; GCN-NEXT:    global_store_dwordx4 v64, v[24:27], s[2:3] offset:96
+; GCN-NEXT:    global_store_dwordx4 v64, v[28:31], s[2:3] offset:112
+; GCN-NEXT:    global_store_dwordx4 v64, v[32:35], s[2:3] offset:128
+; GCN-NEXT:    global_store_dwordx4 v64, v[36:39], s[2:3] offset:144
 ; GCN-NEXT:    s_waitcnt vmcnt(15)
 ; GCN-NEXT:    global_store_dwordx4 v64, v[40:43], s[2:3] offset:160
 ; GCN-NEXT:    s_waitcnt vmcnt(15)
@@ -67,39 +58,45 @@ define amdgpu_kernel void @v_insert_v64i32_37(ptr addrspace(1) %ptr.in, ptr addr
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_clause 0xf
 ; GFX10-NEXT:    global_load_dwordx4 v[0:3], v64, s[0:1]
-; GFX10-NEXT:    global_load_dwordx4 v[8:11], v64, s[0:1] offset:16
-; GFX10-NEXT:    global_load_dwordx4 v[12:15], v64, s[0:1] offset:32
-; GFX10-NEXT:    global_load_dwordx4 v[16:19], v64, s[0:1] offset:48
-; GFX10-NEXT:    global_load_dwordx4 v[20:23], v64, s[0:1] offset:64
-; GFX10-NEXT:    global_load_dwordx4 v[24:27], v64, s[0:1] offset:80
-; GFX10-NEXT:    global_load_dwordx4 v[28:31], v64, s[0:1] offset:96
-; GFX10-NEXT:    global_load_dwordx4 v[32:35], v64, s[0:1] offset:112
-; GFX10-NEXT:    global_load_dwordx4 v[36:39], v64, s[0:1] offset:160
-; GFX10-NEXT:    global_load_dwordx4 v[40:43], v64, s[0:1] offset:176
-; GFX10-NEXT:    global_load_dwordx4 v[44:47], v64, s[0:1] offset:192
-; GFX10-NEXT:    global_load_dwordx4 v[48:51], v64, s[0:1] offset:208
-; GFX10-NEXT:    global_load_dwordx4 v[52:55], v64, s[0:1] offset:224
-; GFX10-NEXT:    global_load_dwordx4 v[56:59], v64, s[0:1] offset:240
-; GFX10-NEXT:    global_load_dwordx4 v[60:63], v64, s[0:1] offset:128
-; GFX10-NEXT:    global_load_dwordx4 v[4:7], v64, s[0:1] offset:144
-; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    v_mov_b32_e32 v5, 0x3e7
+; GFX10-NEXT:    global_load_dwordx4 v[4:7], v64, s[0:1] offset:16
+; GFX10-NEXT:    global_load_dwordx4 v[8:11], v64, s[0:1] offset:32
+; GFX10-NEXT:    global_load_dwordx4 v[12:15], v64, s[0:1] offset:48
+; GFX10-NEXT:    global_load_dwordx4 v[16:19], v64, s[0:1] offset:64
+; GFX10-NEXT:    global_load_dwordx4 v[20:23], v64, s[0:1] offset:80
+; GFX10-NEXT:    global_load_dwordx4 v[24:27], v64, s[0:1] offset:96
+; GFX10-NEXT:    global_load_dwordx4 v[28:31], v64, s[0:1] offset:112
+; GFX10-NEXT:    global_load_dwordx4 v[32:35], v64, s[0:1] offset:128
+; GFX10-NEXT:    global_load_dwordx4 v[36:39], v64, s[0:1] offset:144
+; GFX10-NEXT:    global_load_dwordx4 v[40:43], v64, s[0:1] offset:160
+; GFX10-NEXT:    global_load_dwordx4 v[44:47], v64, s[0:1] offset:176
+; GFX10-NEXT:    global_load_dwordx4 v[48:51], v64, s[0:1] offset:192
+; GFX10-NEXT:    global_load_dwordx4 v[52:55], v64, s[0:1] offset:208
+; GFX10-NEXT:    global_load_dwordx4 v[56:59], v64, s[0:1] offset:224
+; GFX10-NEXT:    global_load_dwordx4 v[60:63], v64, s[0:1] offset:240
+; GFX10-NEXT:    s_waitcnt vmcnt(6)
+; GFX10-NEXT:    v_mov_b32_e32 v37, 0x3e7
 ; GFX10-NEXT:    global_store_dwordx4 v64, v[0:3], s[2:3]
-; GFX10-NEXT:    global_store_dwordx4 v64, v[8:11], s[2:3] offset:16
-; GFX10-NEXT:    global_store_dwordx4 v64, v[12:15], s[2:3] offset:32
-; GFX10-NEXT:    global_store_dwordx4 v64, v[16:19], s[2:3] offset:48
-; GFX10-NEXT:    global_store_dwordx4 v64, v[20:23], s[2:3] offset:64
-; GFX10-NEXT:    global_store_dwordx4 v64, v[24:27], s[2:3] offset:80
-; GFX10-NEXT:    global_store_dwordx4 v64, v[28:31], s[2:3] offset:96
-; GFX10-NEXT:    global_store_dwordx4 v64, v[32:35], s[2:3] offset:112
-; GFX10-NEXT:    global_store_dwordx4 v64, v[60:63], s[2:3] offset:128
-; GFX10-NEXT:    global_store_dwordx4 v64, v[4:7], s[2:3] offset:144
-; GFX10-NEXT:    global_store_dwordx4 v64, v[36:39], s[2:3] offset:160
-; GFX10-NEXT:    global_store_dwordx4 v64, v[40:43], s[2:3] offset:176
-; GFX10-NEXT:    global_store_dwordx4 v64, v[44:47], s[2:3] offset:192
-; GFX10-NEXT:    global_store_dwordx4 v64, v[48:51], s[2:3] offset:208
-; GFX10-NEXT:    global_store_dwordx4 v64, v[52:55], s[2:3] offset:224
-; GFX10-NEXT:    global_store_dwordx4 v64, v[56:59], s[2:3] offset:240
+; GFX10-NEXT:    global_store_dwordx4 v64, v[4:7], s[2:3] offset:16
+; GFX10-NEXT:    global_store_dwordx4 v64, v[8:11], s[2:3] offset:32
+; GFX10-NEXT:    global_store_dwordx4 v64, v[12:15], s[2:3] offset:48
+; GFX10-NEXT:    global_store_dwordx4 v64, v[16:19], s[2:3] offset:64
+; GFX10-NEXT:    global_store_dwordx4 v64, v[20:23], s[2:3] offset:80
+; GFX10-NEXT:    global_store_dwordx4 v64, v[24:27], s[2:3] offset:96
+; GFX10-NEXT:    global_store_dwordx4 v64, v[28:31], s[2:3] offset:112
+; GFX10-NEXT:    global_store_dwordx4 v64, v[32:35], s[2:3] offset:128
+; GFX10-NEXT:    global_store_dwordx4 v64, v[36:39], s[2:3] offset:144
+; GFX10-NEXT:    s_waitcnt vmcnt(5)
+; GFX10-NEXT:    global_store_dwordx4 v64, v[40:43], s[2:3] offset:160
+; GFX10-NEXT:    s_waitcnt vmcnt(4)
+; GFX10-NEXT:    global_store_dwordx4 v64, v[44:47], s[2:3] offset:176
+; GFX10-NEXT:    s_waitcnt vmcnt(3)
+; GFX10-NEXT:    global_store_dwordx4 v64, v[48:51], s[2:3] offset:192
+; GFX10-NEXT:    s_waitcnt vmcnt(2)
+; GFX10-NEXT:    global_store_dwordx4 v64, v[52:55], s[2:3] offset:208
+; GFX10-NEXT:    s_waitcnt vmcnt(1)
+; GFX10-NEXT:    global_store_dwordx4 v64, v[56:59], s[2:3] offset:224
+; GFX10-NEXT:    s_waitcnt vmcnt(0)
+; GFX10-NEXT:    global_store_dwordx4 v64, v[60:63], s[2:3] offset:240
 ; GFX10-NEXT:    s_endpgm
 ;
 ; GFX11-LABEL: v_insert_v64i32_37:
@@ -109,15 +106,15 @@ define amdgpu_kernel void @v_insert_v64i32_37(ptr addrspace(1) %ptr.in, ptr addr
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-NEXT:    s_clause 0xf
 ; GFX11-NEXT:    global_load_b128 v[0:3], v64, s[0:1]
-; GFX11-NEXT:    global_load_b128 v[8:11], v64, s[0:1] offset:16
-; GFX11-NEXT:    global_load_b128 v[12:15], v64, s[0:1] offset:32
-; GFX11-NEXT:    global_load_b128 v[16:19], v64, s[0:1] offset:48
-; GFX11-NEXT:    global_load_b128 v[20:23], v64, s[0:1] offset:64
-; GFX11-NEXT:    global_load_b128 v[24:27], v64, s[0:1] offset:80
-; GFX11-NEXT:    global_load_b128 v[28:31], v64, s[0:1] offset:96
-; GFX11-NEXT:    global_load_b128 v[32:35], v64, s[0:1] offset:112
-; GFX11-NEXT:    global_load_b128 v[36:39], v64, s[0:1] offset:128
-; GFX11-NEXT:    global_load_b128 v[4:7], v64, s[0:1] offset:144
+; GFX11-NEXT:    global_load_b128 v[4:7], v64, s[0:1] offset:16
+; GFX11-NEXT:    global_load_b128 v[8:11], v64, s[0:1] offset:32
+; GFX11-NEXT:    global_load_b128 v[12:15], v64, s[0:1] offset:48
+; GFX11-NEXT:    global_load_b128 v[16:19], v64, s[0:1] offset:64
+; GFX11-NEXT:    global_load_b128 v[20:23], v64, s[0:1] offset:80
+; GFX11-NEXT:    global_load_b128 v[24:27], v64, s[0:1] offset:96
+; GFX11-NEXT:    global_load_b128 v[28:31], v64, s[0:1] offset:112
+; GFX11-NEXT:    global_load_b128 v[32:35], v64, s[0:1] offset:128
+; GFX11-NEXT:    global_load_b128 v[36:39], v64, s[0:1] offset:144
 ; GFX11-NEXT:    global_load_b128 v[40:43], v64, s[0:1] offset:160
 ; GFX11-NEXT:    global_load_b128 v[44:47], v64, s[0:1] offset:176
 ; GFX11-NEXT:    global_load_b128 v[48:51], v64, s[0:1] offset:192
@@ -125,18 +122,18 @@ define amdgpu_kernel void @v_insert_v64i32_37(ptr addrspace(1) %ptr.in, ptr addr
 ; GFX11-NEXT:    global_load_b128 v[56:59], v64, s[0:1] offset:224
 ; GFX11-NEXT:    global_load_b128 v[60:63], v64, s[0:1] offset:240
 ; GFX11-NEXT:    s_waitcnt vmcnt(6)
-; GFX11-NEXT:    v_mov_b32_e32 v5, 0x3e7
+; GFX11-NEXT:    v_mov_b32_e32 v37, 0x3e7
 ; GFX11-NEXT:    s_clause 0x9
 ; GFX11-NEXT:    global_store_b128 v64, v[0:3], s[2:3]
-; GFX11-NEXT:    global_store_b128 v64, v[8:11], s[2:3] offset:16
-; GFX11-NEXT:    global_store_b128 v64, v[12:15], s[2:3] offset:32
-; GFX11-NEXT:    global_store_b128 v64, v[16:19], s[2:3] offset:48
-; GFX11-NEXT:    global_store_b128 v64, v[20:23], s[2:3] offset:64
-; GFX11-NEXT:    global_store_b128 v64, v[24:27], s[2:3] offset:80
-; GFX11-NEXT:    global_store_b128 v64, v[28:31], s[2:3] offset:96
-; GFX11-NEXT:    global_store_b128 v64, v[32:35], s[2:3] offset:112
-; GFX11-NEXT:    global_store_b128 v64, v[36:39], s[2:3] offset:128
-; GFX11-NEXT:    global_store_b128 v64, v[4:7], s[2:3] offset:144
+; GFX11-NEXT:    global_store_b128 v64, v[4:7], s[2:3] offset:16
+; GFX11-NEXT:    global_store_b128 v64, v[8:11], s[2:3] offset:32
+; GFX11-NEXT:    global_store_b128 v64, v[12:15], s[2:3] offset:48
+; GFX11-NEXT:    global_store_b128 v64, v[16:19], s[2:3] offset:64
+; GFX11-NEXT:    global_store_b128 v64, v[20:23], s[2:3] offset:80
+; GFX11-NEXT:    global_store_b128 v64, v[24:27], s[2:3] offset:96
+; GFX11-NEXT:    global_store_b128 v64, v[28:31], s[2:3] offset:112
+; GFX11-NEXT:    global_store_b128 v64, v[32:35], s[2:3] offset:128
+; GFX11-NEXT:    global_store_b128 v64, v[36:39], s[2:3] offset:144
 ; GFX11-NEXT:    s_waitcnt vmcnt(5)
 ; GFX11-NEXT:    global_store_b128 v64, v[40:43], s[2:3] offset:160
 ; GFX11-NEXT:    s_waitcnt vmcnt(4)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.ll
index dc9cbb498dab4d87d20a20d7e21f842130312276..49d9ad1c0f79744650ac4c86e4a4bc7595bf646a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.ll
@@ -715,27 +715,27 @@ define void @dyn_insertelement_v8f64_const_s_v_v(double %val, i32 %idx) {
 ; GPRIDX-NEXT:    v_mov_b32_e32 v16, s17
 ; GPRIDX-NEXT:    v_mov_b32_e32 v17, s18
 ; GPRIDX-NEXT:    v_mov_b32_e32 v18, s19
-; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[16:17], 0, v2
-; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[4:5], 2, v2
-; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[6:7], 3, v2
-; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[8:9], 4, v2
+; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
+; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[4:5], 1, v2
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v3, v3, v0, vcc
+; GPRIDX-NEXT:    v_cndmask_b32_e64 v5, v5, v0, s[4:5]
+; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[6:7], 2, v2
+; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[8:9], 3, v2
 ; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[10:11], 5, v2
 ; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[12:13], 6, v2
 ; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[14:15], 7, v2
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v3, v3, v0, s[16:17]
-; GPRIDX-NEXT:    v_cndmask_b32_e32 v5, v5, v0, vcc
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v4, v4, v1, s[16:17]
-; GPRIDX-NEXT:    v_cndmask_b32_e32 v6, v6, v1, vcc
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v7, v7, v0, s[4:5]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v9, v9, v0, s[6:7]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v11, v11, v0, s[8:9]
+; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[16:17], 4, v2
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
+; GPRIDX-NEXT:    v_cndmask_b32_e64 v6, v6, v1, s[4:5]
+; GPRIDX-NEXT:    v_cndmask_b32_e64 v7, v7, v0, s[6:7]
+; GPRIDX-NEXT:    v_cndmask_b32_e64 v9, v9, v0, s[8:9]
+; GPRIDX-NEXT:    v_cndmask_b32_e64 v11, v11, v0, s[16:17]
 ; GPRIDX-NEXT:    v_cndmask_b32_e64 v13, v13, v0, s[10:11]
 ; GPRIDX-NEXT:    v_cndmask_b32_e64 v15, v15, v0, s[12:13]
 ; GPRIDX-NEXT:    v_cndmask_b32_e64 v17, v17, v0, s[14:15]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v8, v8, v1, s[4:5]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v10, v10, v1, s[6:7]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v12, v12, v1, s[8:9]
+; GPRIDX-NEXT:    v_cndmask_b32_e64 v8, v8, v1, s[6:7]
+; GPRIDX-NEXT:    v_cndmask_b32_e64 v10, v10, v1, s[8:9]
+; GPRIDX-NEXT:    v_cndmask_b32_e64 v12, v12, v1, s[16:17]
 ; GPRIDX-NEXT:    v_cndmask_b32_e64 v14, v14, v1, s[10:11]
 ; GPRIDX-NEXT:    v_cndmask_b32_e64 v16, v16, v1, s[12:13]
 ; GPRIDX-NEXT:    v_cndmask_b32_e64 v18, v18, v1, s[14:15]
@@ -752,12 +752,12 @@ define void @dyn_insertelement_v8f64_const_s_v_v(double %val, i32 %idx) {
 ; GFX10-LABEL: dyn_insertelement_v8f64_const_s_v_v:
 ; GFX10:       ; %bb.0: ; %entry
 ; GFX10-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT:    s_mov_b64 s[4:5], 1.0
 ; GFX10-NEXT:    s_mov_b32 s18, 0
 ; GFX10-NEXT:    s_mov_b32 s16, 0
 ; GFX10-NEXT:    s_mov_b32 s14, 0
 ; GFX10-NEXT:    s_mov_b32 s12, 0
 ; GFX10-NEXT:    s_mov_b32 s8, 0
-; GFX10-NEXT:    s_mov_b64 s[4:5], 1.0
 ; GFX10-NEXT:    s_mov_b32 s19, 0x40200000
 ; GFX10-NEXT:    s_mov_b32 s17, 0x401c0000
 ; GFX10-NEXT:    s_mov_b32 s15, 0x40180000
@@ -767,6 +767,7 @@ define void @dyn_insertelement_v8f64_const_s_v_v(double %val, i32 %idx) {
 ; GFX10-NEXT:    s_mov_b64 s[6:7], 2.0
 ; GFX10-NEXT:    v_mov_b32_e32 v3, s4
 ; GFX10-NEXT:    v_mov_b32_e32 v4, s5
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v2
 ; GFX10-NEXT:    v_mov_b32_e32 v5, s6
 ; GFX10-NEXT:    v_mov_b32_e32 v6, s7
 ; GFX10-NEXT:    v_mov_b32_e32 v7, s8
@@ -781,30 +782,29 @@ define void @dyn_insertelement_v8f64_const_s_v_v(double %val, i32 %idx) {
 ; GFX10-NEXT:    v_mov_b32_e32 v16, s17
 ; GFX10-NEXT:    v_mov_b32_e32 v17, s18
 ; GFX10-NEXT:    v_mov_b32_e32 v18, s19
-; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v2
 ; GFX10-NEXT:    v_cmp_eq_u32_e64 s4, 1, v2
-; GFX10-NEXT:    v_cmp_eq_u32_e64 s5, 2, v2
-; GFX10-NEXT:    v_cmp_eq_u32_e64 s6, 3, v2
-; GFX10-NEXT:    v_cmp_eq_u32_e64 s7, 5, v2
-; GFX10-NEXT:    v_cmp_eq_u32_e64 s10, 4, v2
-; GFX10-NEXT:    v_cmp_eq_u32_e64 s8, 6, v2
-; GFX10-NEXT:    v_cmp_eq_u32_e64 s9, 7, v2
 ; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v0, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v5, v5, v0, s4
 ; GFX10-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 2, v2
+; GFX10-NEXT:    v_cmp_eq_u32_e64 s5, 7, v2
+; GFX10-NEXT:    v_cndmask_b32_e64 v5, v5, v0, s4
 ; GFX10-NEXT:    v_cndmask_b32_e64 v6, v6, v1, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v7, v7, v0, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v9, v9, v0, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v8, v8, v1, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v10, v10, v1, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v11, v11, v0, s10
-; GFX10-NEXT:    v_cndmask_b32_e64 v13, v13, v0, s7
-; GFX10-NEXT:    v_cndmask_b32_e64 v12, v12, v1, s10
-; GFX10-NEXT:    v_cndmask_b32_e64 v14, v14, v1, s7
-; GFX10-NEXT:    v_cndmask_b32_e64 v15, v15, v0, s8
-; GFX10-NEXT:    v_cndmask_b32_e64 v17, v17, v0, s9
-; GFX10-NEXT:    v_cndmask_b32_e64 v16, v16, v1, s8
-; GFX10-NEXT:    v_cndmask_b32_e64 v18, v18, v1, s9
+; GFX10-NEXT:    v_cmp_eq_u32_e64 s4, 3, v2
+; GFX10-NEXT:    v_cndmask_b32_e32 v7, v7, v0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v8, v8, v1, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 4, v2
+; GFX10-NEXT:    v_cndmask_b32_e64 v17, v17, v0, s5
+; GFX10-NEXT:    v_cndmask_b32_e64 v9, v9, v0, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v10, v10, v1, s4
+; GFX10-NEXT:    v_cmp_eq_u32_e64 s4, 5, v2
+; GFX10-NEXT:    v_cndmask_b32_e32 v11, v11, v0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v12, v12, v1, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 6, v2
+; GFX10-NEXT:    v_cndmask_b32_e64 v18, v18, v1, s5
+; GFX10-NEXT:    v_cndmask_b32_e64 v13, v13, v0, s4
+; GFX10-NEXT:    v_cndmask_b32_e64 v14, v14, v1, s4
+; GFX10-NEXT:    v_cndmask_b32_e32 v15, v15, v0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v16, v16, v1, vcc_lo
 ; GFX10-NEXT:    global_store_dwordx4 v[0:1], v[3:6], off
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    global_store_dwordx4 v[0:1], v[7:10], off
@@ -990,26 +990,26 @@ define amdgpu_ps void @dyn_insertelement_v8f64_s_s_v(<8 x double> inreg %vec, do
 ; GFX10-NEXT:    v_cmp_eq_u32_e64 s0, 1, v0
 ; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, s18, vcc_lo
 ; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, s19, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 3, v0
-; GFX10-NEXT:    v_cmp_eq_u32_e64 s1, 2, v0
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 2, v0
+; GFX10-NEXT:    v_cmp_eq_u32_e64 s1, 7, v0
 ; GFX10-NEXT:    v_cndmask_b32_e64 v3, v3, s18, s0
 ; GFX10-NEXT:    v_cndmask_b32_e64 v4, v4, s19, s0
-; GFX10-NEXT:    v_cmp_eq_u32_e64 s0, 4, v0
-; GFX10-NEXT:    v_cmp_eq_u32_e64 s2, 5, v0
-; GFX10-NEXT:    v_cmp_eq_u32_e64 s3, 6, v0
-; GFX10-NEXT:    v_cmp_eq_u32_e64 s4, 7, v0
-; GFX10-NEXT:    v_cndmask_b32_e64 v5, v5, s18, s1
-; GFX10-NEXT:    v_cndmask_b32_e64 v6, v6, s19, s1
-; GFX10-NEXT:    v_cndmask_b32_e64 v7, v7, s18, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v8, v8, s19, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v9, v9, s18, s0
-; GFX10-NEXT:    v_cndmask_b32_e64 v10, v10, s19, s0
-; GFX10-NEXT:    v_cndmask_b32_e64 v11, v11, s18, s2
-; GFX10-NEXT:    v_cndmask_b32_e64 v12, v12, s19, s2
-; GFX10-NEXT:    v_cndmask_b32_e64 v13, v13, s18, s3
-; GFX10-NEXT:    v_cndmask_b32_e64 v14, v14, s19, s3
-; GFX10-NEXT:    v_cndmask_b32_e64 v15, v15, s18, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v16, v16, s19, s4
+; GFX10-NEXT:    v_cmp_eq_u32_e64 s0, 3, v0
+; GFX10-NEXT:    v_cndmask_b32_e64 v5, v5, s18, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v6, v6, s19, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 4, v0
+; GFX10-NEXT:    v_cndmask_b32_e64 v15, v15, s18, s1
+; GFX10-NEXT:    v_cndmask_b32_e64 v7, v7, s18, s0
+; GFX10-NEXT:    v_cndmask_b32_e64 v8, v8, s19, s0
+; GFX10-NEXT:    v_cmp_eq_u32_e64 s0, 5, v0
+; GFX10-NEXT:    v_cndmask_b32_e64 v9, v9, s18, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v10, v10, s19, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 6, v0
+; GFX10-NEXT:    v_cndmask_b32_e64 v16, v16, s19, s1
+; GFX10-NEXT:    v_cndmask_b32_e64 v11, v11, s18, s0
+; GFX10-NEXT:    v_cndmask_b32_e64 v12, v12, s19, s0
+; GFX10-NEXT:    v_cndmask_b32_e64 v13, v13, s18, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v14, v14, s19, vcc_lo
 ; GFX10-NEXT:    global_store_dwordx4 v[0:1], v[1:4], off
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    global_store_dwordx4 v[0:1], v[5:8], off
@@ -1394,6 +1394,9 @@ define amdgpu_ps void @dyn_insertelement_v8f64_s_v_v(<8 x double> inreg %vec, do
 ; GFX10-NEXT:    s_mov_b32 s12, s14
 ; GFX10-NEXT:    s_mov_b32 s14, s16
 ; GFX10-NEXT:    v_mov_b32_e32 v18, s15
+; GFX10-NEXT:    v_mov_b32_e32 v4, s1
+; GFX10-NEXT:    v_mov_b32_e32 v3, s0
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v2
 ; GFX10-NEXT:    v_mov_b32_e32 v17, s14
 ; GFX10-NEXT:    v_mov_b32_e32 v16, s13
 ; GFX10-NEXT:    v_mov_b32_e32 v15, s12
@@ -1407,32 +1410,29 @@ define amdgpu_ps void @dyn_insertelement_v8f64_s_v_v(<8 x double> inreg %vec, do
 ; GFX10-NEXT:    v_mov_b32_e32 v7, s4
 ; GFX10-NEXT:    v_mov_b32_e32 v6, s3
 ; GFX10-NEXT:    v_mov_b32_e32 v5, s2
-; GFX10-NEXT:    v_mov_b32_e32 v4, s1
-; GFX10-NEXT:    v_mov_b32_e32 v3, s0
-; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v2
 ; GFX10-NEXT:    v_cmp_eq_u32_e64 s0, 1, v2
-; GFX10-NEXT:    v_cmp_eq_u32_e64 s1, 3, v2
-; GFX10-NEXT:    v_cmp_eq_u32_e64 s6, 2, v2
-; GFX10-NEXT:    v_cmp_eq_u32_e64 s2, 4, v2
-; GFX10-NEXT:    v_cmp_eq_u32_e64 s3, 5, v2
-; GFX10-NEXT:    v_cmp_eq_u32_e64 s4, 6, v2
-; GFX10-NEXT:    v_cmp_eq_u32_e64 s5, 7, v2
 ; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v0, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v5, v5, v0, s0
 ; GFX10-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 2, v2
+; GFX10-NEXT:    v_cmp_eq_u32_e64 s1, 7, v2
+; GFX10-NEXT:    v_cndmask_b32_e64 v5, v5, v0, s0
 ; GFX10-NEXT:    v_cndmask_b32_e64 v6, v6, v1, s0
-; GFX10-NEXT:    v_cndmask_b32_e64 v7, v7, v0, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v9, v9, v0, s1
-; GFX10-NEXT:    v_cndmask_b32_e64 v8, v8, v1, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v10, v10, v1, s1
-; GFX10-NEXT:    v_cndmask_b32_e64 v11, v11, v0, s2
-; GFX10-NEXT:    v_cndmask_b32_e64 v13, v13, v0, s3
-; GFX10-NEXT:    v_cndmask_b32_e64 v12, v12, v1, s2
-; GFX10-NEXT:    v_cndmask_b32_e64 v14, v14, v1, s3
-; GFX10-NEXT:    v_cndmask_b32_e64 v15, v15, v0, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v17, v17, v0, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v16, v16, v1, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v18, v18, v1, s5
+; GFX10-NEXT:    v_cmp_eq_u32_e64 s0, 3, v2
+; GFX10-NEXT:    v_cndmask_b32_e32 v7, v7, v0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v8, v8, v1, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 4, v2
+; GFX10-NEXT:    v_cndmask_b32_e64 v17, v17, v0, s1
+; GFX10-NEXT:    v_cndmask_b32_e64 v9, v9, v0, s0
+; GFX10-NEXT:    v_cndmask_b32_e64 v10, v10, v1, s0
+; GFX10-NEXT:    v_cmp_eq_u32_e64 s0, 5, v2
+; GFX10-NEXT:    v_cndmask_b32_e32 v11, v11, v0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v12, v12, v1, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 6, v2
+; GFX10-NEXT:    v_cndmask_b32_e64 v18, v18, v1, s1
+; GFX10-NEXT:    v_cndmask_b32_e64 v13, v13, v0, s0
+; GFX10-NEXT:    v_cndmask_b32_e64 v14, v14, v1, s0
+; GFX10-NEXT:    v_cndmask_b32_e32 v15, v15, v0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v16, v16, v1, vcc_lo
 ; GFX10-NEXT:    global_store_dwordx4 v[0:1], v[3:6], off
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    global_store_dwordx4 v[0:1], v[7:10], off
@@ -1517,31 +1517,31 @@ define amdgpu_ps void @dyn_insertelement_v8f64_v_s_v(<8 x double> %vec, double i
 ; GPRIDX-LABEL: dyn_insertelement_v8f64_v_s_v:
 ; GPRIDX:       ; %bb.0: ; %entry
 ; GPRIDX-NEXT:    v_mov_b32_e32 v17, s2
+; GPRIDX-NEXT:    v_mov_b32_e32 v18, s3
 ; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v16
-; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[0:1], 1, v16
-; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[14:15], 2, v16
-; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[4:5], 3, v16
-; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[6:7], 4, v16
-; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[8:9], 5, v16
-; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[10:11], 7, v16
-; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[12:13], 6, v16
-; GPRIDX-NEXT:    v_mov_b32_e32 v16, s3
 ; GPRIDX-NEXT:    v_cndmask_b32_e32 v0, v0, v17, vcc
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v2, v2, v17, s[0:1]
-; GPRIDX-NEXT:    v_cndmask_b32_e32 v1, v1, v16, vcc
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v3, v3, v16, s[0:1]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v4, v4, v17, s[14:15]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v6, v6, v17, s[4:5]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v8, v8, v17, s[6:7]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v10, v10, v17, s[8:9]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v12, v12, v17, s[12:13]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v14, v14, v17, s[10:11]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v5, v5, v16, s[14:15]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v7, v7, v16, s[4:5]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v9, v9, v16, s[6:7]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v11, v11, v16, s[8:9]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v13, v13, v16, s[12:13]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v15, v15, v16, s[10:11]
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v1, v1, v18, vcc
+; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v16
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v2, v2, v17, vcc
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v3, v3, v18, vcc
+; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, 2, v16
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v4, v4, v17, vcc
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v5, v5, v18, vcc
+; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, 3, v16
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v6, v6, v17, vcc
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v7, v7, v18, vcc
+; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, 4, v16
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v8, v8, v17, vcc
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v9, v9, v18, vcc
+; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, 5, v16
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v10, v10, v17, vcc
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v11, v11, v18, vcc
+; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, 6, v16
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v12, v12, v17, vcc
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v13, v13, v18, vcc
+; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, 7, v16
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v14, v14, v17, vcc
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v15, v15, v18, vcc
 ; GPRIDX-NEXT:    global_store_dwordx4 v[0:1], v[0:3], off
 ; GPRIDX-NEXT:    s_waitcnt vmcnt(0)
 ; GPRIDX-NEXT:    global_store_dwordx4 v[0:1], v[4:7], off
@@ -1555,29 +1555,29 @@ define amdgpu_ps void @dyn_insertelement_v8f64_v_s_v(<8 x double> %vec, double i
 ; GFX10-LABEL: dyn_insertelement_v8f64_v_s_v:
 ; GFX10:       ; %bb.0: ; %entry
 ; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v16
+; GFX10-NEXT:    v_cmp_eq_u32_e64 s0, 1, v16
+; GFX10-NEXT:    v_cmp_eq_u32_e64 s1, 7, v16
 ; GFX10-NEXT:    v_cndmask_b32_e64 v0, v0, s2, vcc_lo
 ; GFX10-NEXT:    v_cndmask_b32_e64 v1, v1, s3, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v16
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, s2, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v3, v3, s3, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 2, v16
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, s2, s0
+; GFX10-NEXT:    v_cndmask_b32_e64 v3, v3, s3, s0
+; GFX10-NEXT:    v_cmp_eq_u32_e64 s0, 3, v16
+; GFX10-NEXT:    v_cndmask_b32_e64 v14, v14, s2, s1
 ; GFX10-NEXT:    v_cndmask_b32_e64 v4, v4, s2, vcc_lo
 ; GFX10-NEXT:    v_cndmask_b32_e64 v5, v5, s3, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 3, v16
-; GFX10-NEXT:    v_cndmask_b32_e64 v6, v6, s2, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v7, v7, s3, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 4, v16
+; GFX10-NEXT:    v_cndmask_b32_e64 v6, v6, s2, s0
+; GFX10-NEXT:    v_cndmask_b32_e64 v7, v7, s3, s0
+; GFX10-NEXT:    v_cmp_eq_u32_e64 s0, 5, v16
+; GFX10-NEXT:    v_cndmask_b32_e64 v15, v15, s3, s1
 ; GFX10-NEXT:    v_cndmask_b32_e64 v8, v8, s2, vcc_lo
 ; GFX10-NEXT:    v_cndmask_b32_e64 v9, v9, s3, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 5, v16
-; GFX10-NEXT:    v_cndmask_b32_e64 v10, v10, s2, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v11, v11, s3, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 6, v16
+; GFX10-NEXT:    v_cndmask_b32_e64 v10, v10, s2, s0
+; GFX10-NEXT:    v_cndmask_b32_e64 v11, v11, s3, s0
 ; GFX10-NEXT:    v_cndmask_b32_e64 v12, v12, s2, vcc_lo
 ; GFX10-NEXT:    v_cndmask_b32_e64 v13, v13, s3, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 7, v16
-; GFX10-NEXT:    v_cndmask_b32_e64 v14, v14, s2, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v15, v15, s3, vcc_lo
 ; GFX10-NEXT:    global_store_dwordx4 v[0:1], v[0:3], off
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    global_store_dwordx4 v[0:1], v[4:7], off
@@ -1591,29 +1591,29 @@ define amdgpu_ps void @dyn_insertelement_v8f64_v_s_v(<8 x double> %vec, double i
 ; GFX11-LABEL: dyn_insertelement_v8f64_v_s_v:
 ; GFX11:       ; %bb.0: ; %entry
 ; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v16
+; GFX11-NEXT:    v_cmp_eq_u32_e64 s0, 1, v16
+; GFX11-NEXT:    v_cmp_eq_u32_e64 s1, 7, v16
 ; GFX11-NEXT:    v_cndmask_b32_e64 v0, v0, s2, vcc_lo
 ; GFX11-NEXT:    v_cndmask_b32_e64 v1, v1, s3, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v16
-; GFX11-NEXT:    v_cndmask_b32_e64 v2, v2, s2, vcc_lo
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v3, s3, vcc_lo
 ; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 2, v16
+; GFX11-NEXT:    v_cndmask_b32_e64 v2, v2, s2, s0
+; GFX11-NEXT:    v_cndmask_b32_e64 v3, v3, s3, s0
+; GFX11-NEXT:    v_cmp_eq_u32_e64 s0, 3, v16
+; GFX11-NEXT:    v_cndmask_b32_e64 v14, v14, s2, s1
 ; GFX11-NEXT:    v_cndmask_b32_e64 v4, v4, s2, vcc_lo
 ; GFX11-NEXT:    v_cndmask_b32_e64 v5, v5, s3, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 3, v16
-; GFX11-NEXT:    v_cndmask_b32_e64 v6, v6, s2, vcc_lo
-; GFX11-NEXT:    v_cndmask_b32_e64 v7, v7, s3, vcc_lo
 ; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 4, v16
+; GFX11-NEXT:    v_cndmask_b32_e64 v6, v6, s2, s0
+; GFX11-NEXT:    v_cndmask_b32_e64 v7, v7, s3, s0
+; GFX11-NEXT:    v_cmp_eq_u32_e64 s0, 5, v16
+; GFX11-NEXT:    v_cndmask_b32_e64 v15, v15, s3, s1
 ; GFX11-NEXT:    v_cndmask_b32_e64 v8, v8, s2, vcc_lo
 ; GFX11-NEXT:    v_cndmask_b32_e64 v9, v9, s3, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 5, v16
-; GFX11-NEXT:    v_cndmask_b32_e64 v10, v10, s2, vcc_lo
-; GFX11-NEXT:    v_cndmask_b32_e64 v11, v11, s3, vcc_lo
 ; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 6, v16
+; GFX11-NEXT:    v_cndmask_b32_e64 v10, v10, s2, s0
+; GFX11-NEXT:    v_cndmask_b32_e64 v11, v11, s3, s0
 ; GFX11-NEXT:    v_cndmask_b32_e64 v12, v12, s2, vcc_lo
 ; GFX11-NEXT:    v_cndmask_b32_e64 v13, v13, s3, vcc_lo
-; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 7, v16
-; GFX11-NEXT:    v_cndmask_b32_e64 v14, v14, s2, vcc_lo
-; GFX11-NEXT:    v_cndmask_b32_e64 v15, v15, s3, vcc_lo
 ; GFX11-NEXT:    global_store_b128 v[0:1], v[0:3], off dlc
 ; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-NEXT:    global_store_b128 v[0:1], v[4:7], off dlc
@@ -1704,29 +1704,29 @@ define amdgpu_ps void @dyn_insertelement_v8f64_v_v_v(<8 x double> %vec, double %
 ; GPRIDX-LABEL: dyn_insertelement_v8f64_v_v_v:
 ; GPRIDX:       ; %bb.0: ; %entry
 ; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v18
-; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[0:1], 1, v18
 ; GPRIDX-NEXT:    v_cndmask_b32_e32 v0, v0, v16, vcc
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v2, v2, v16, s[0:1]
-; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[2:3], 2, v18
-; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[4:5], 3, v18
-; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[6:7], 4, v18
-; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[8:9], 5, v18
-; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[10:11], 7, v18
-; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[12:13], 6, v18
 ; GPRIDX-NEXT:    v_cndmask_b32_e32 v1, v1, v17, vcc
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v3, v3, v17, s[0:1]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v4, v4, v16, s[2:3]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v6, v6, v16, s[4:5]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v8, v8, v16, s[6:7]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v10, v10, v16, s[8:9]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v12, v12, v16, s[12:13]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v14, v14, v16, s[10:11]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v5, v5, v17, s[2:3]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v7, v7, v17, s[4:5]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v9, v9, v17, s[6:7]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v11, v11, v17, s[8:9]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v13, v13, v17, s[12:13]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v15, v15, v17, s[10:11]
+; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v18
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v2, v2, v16, vcc
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v3, v3, v17, vcc
+; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, 2, v18
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v4, v4, v16, vcc
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v5, v5, v17, vcc
+; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, 3, v18
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v6, v6, v16, vcc
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v7, v7, v17, vcc
+; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, 4, v18
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v8, v8, v16, vcc
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v9, v9, v17, vcc
+; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, 5, v18
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v10, v10, v16, vcc
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v11, v11, v17, vcc
+; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, 6, v18
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v12, v12, v16, vcc
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v13, v13, v17, vcc
+; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, 7, v18
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v14, v14, v16, vcc
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v15, v15, v17, vcc
 ; GPRIDX-NEXT:    global_store_dwordx4 v[0:1], v[0:3], off
 ; GPRIDX-NEXT:    s_waitcnt vmcnt(0)
 ; GPRIDX-NEXT:    global_store_dwordx4 v[0:1], v[4:7], off
@@ -1741,28 +1741,28 @@ define amdgpu_ps void @dyn_insertelement_v8f64_v_v_v(<8 x double> %vec, double %
 ; GFX10:       ; %bb.0: ; %entry
 ; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v18
 ; GFX10-NEXT:    v_cmp_eq_u32_e64 s0, 1, v18
-; GFX10-NEXT:    v_cmp_eq_u32_e64 s1, 2, v18
-; GFX10-NEXT:    v_cmp_eq_u32_e64 s2, 3, v18
-; GFX10-NEXT:    v_cmp_eq_u32_e64 s3, 4, v18
-; GFX10-NEXT:    v_cmp_eq_u32_e64 s4, 5, v18
-; GFX10-NEXT:    v_cmp_eq_u32_e64 s5, 7, v18
-; GFX10-NEXT:    v_cmp_eq_u32_e64 s6, 6, v18
+; GFX10-NEXT:    v_cmp_eq_u32_e64 s1, 7, v18
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v16, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, v16, s0
 ; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v17, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 2, v18
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, v16, s0
 ; GFX10-NEXT:    v_cndmask_b32_e64 v3, v3, v17, s0
-; GFX10-NEXT:    v_cndmask_b32_e64 v4, v4, v16, s1
-; GFX10-NEXT:    v_cndmask_b32_e64 v6, v6, v16, s2
-; GFX10-NEXT:    v_cndmask_b32_e64 v5, v5, v17, s1
-; GFX10-NEXT:    v_cndmask_b32_e64 v7, v7, v17, s2
-; GFX10-NEXT:    v_cndmask_b32_e64 v8, v8, v16, s3
-; GFX10-NEXT:    v_cndmask_b32_e64 v10, v10, v16, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v9, v9, v17, s3
-; GFX10-NEXT:    v_cndmask_b32_e64 v11, v11, v17, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v12, v12, v16, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v14, v14, v16, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v13, v13, v17, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v15, v15, v17, s5
+; GFX10-NEXT:    v_cmp_eq_u32_e64 s0, 3, v18
+; GFX10-NEXT:    v_cndmask_b32_e64 v14, v14, v16, s1
+; GFX10-NEXT:    v_cndmask_b32_e32 v4, v4, v16, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v5, v5, v17, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 4, v18
+; GFX10-NEXT:    v_cndmask_b32_e64 v6, v6, v16, s0
+; GFX10-NEXT:    v_cndmask_b32_e64 v7, v7, v17, s0
+; GFX10-NEXT:    v_cmp_eq_u32_e64 s0, 5, v18
+; GFX10-NEXT:    v_cndmask_b32_e64 v15, v15, v17, s1
+; GFX10-NEXT:    v_cndmask_b32_e32 v8, v8, v16, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v9, v9, v17, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 6, v18
+; GFX10-NEXT:    v_cndmask_b32_e64 v10, v10, v16, s0
+; GFX10-NEXT:    v_cndmask_b32_e64 v11, v11, v17, s0
+; GFX10-NEXT:    v_cndmask_b32_e32 v12, v12, v16, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v13, v13, v17, vcc_lo
 ; GFX10-NEXT:    global_store_dwordx4 v[0:1], v[0:3], off
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    global_store_dwordx4 v[0:1], v[4:7], off
@@ -1777,27 +1777,24 @@ define amdgpu_ps void @dyn_insertelement_v8f64_v_v_v(<8 x double> %vec, double %
 ; GFX11:       ; %bb.0: ; %entry
 ; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v18
 ; GFX11-NEXT:    v_cmp_eq_u32_e64 s0, 1, v18
-; GFX11-NEXT:    v_cmp_eq_u32_e64 s1, 2, v18
-; GFX11-NEXT:    v_cmp_eq_u32_e64 s2, 3, v18
-; GFX11-NEXT:    v_cmp_eq_u32_e64 s3, 4, v18
-; GFX11-NEXT:    v_cmp_eq_u32_e64 s4, 5, v18
-; GFX11-NEXT:    v_cmp_eq_u32_e64 s5, 7, v18
-; GFX11-NEXT:    v_cmp_eq_u32_e64 s6, 6, v18
+; GFX11-NEXT:    v_cmp_eq_u32_e64 s1, 7, v18
 ; GFX11-NEXT:    v_dual_cndmask_b32 v0, v0, v16 :: v_dual_cndmask_b32 v1, v1, v17
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 2, v18
 ; GFX11-NEXT:    v_cndmask_b32_e64 v2, v2, v16, s0
 ; GFX11-NEXT:    v_cndmask_b32_e64 v3, v3, v17, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v4, v4, v16, s1
-; GFX11-NEXT:    v_cndmask_b32_e64 v6, v6, v16, s2
-; GFX11-NEXT:    v_cndmask_b32_e64 v5, v5, v17, s1
-; GFX11-NEXT:    v_cndmask_b32_e64 v7, v7, v17, s2
-; GFX11-NEXT:    v_cndmask_b32_e64 v8, v8, v16, s3
-; GFX11-NEXT:    v_cndmask_b32_e64 v10, v10, v16, s4
-; GFX11-NEXT:    v_cndmask_b32_e64 v9, v9, v17, s3
-; GFX11-NEXT:    v_cndmask_b32_e64 v11, v11, v17, s4
-; GFX11-NEXT:    v_cndmask_b32_e64 v12, v12, v16, s6
-; GFX11-NEXT:    v_cndmask_b32_e64 v14, v14, v16, s5
-; GFX11-NEXT:    v_cndmask_b32_e64 v13, v13, v17, s6
-; GFX11-NEXT:    v_cndmask_b32_e64 v15, v15, v17, s5
+; GFX11-NEXT:    v_cmp_eq_u32_e64 s0, 3, v18
+; GFX11-NEXT:    v_cndmask_b32_e64 v14, v14, v16, s1
+; GFX11-NEXT:    v_dual_cndmask_b32 v4, v4, v16 :: v_dual_cndmask_b32 v5, v5, v17
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 4, v18
+; GFX11-NEXT:    v_cndmask_b32_e64 v6, v6, v16, s0
+; GFX11-NEXT:    v_cndmask_b32_e64 v7, v7, v17, s0
+; GFX11-NEXT:    v_cmp_eq_u32_e64 s0, 5, v18
+; GFX11-NEXT:    v_cndmask_b32_e64 v15, v15, v17, s1
+; GFX11-NEXT:    v_dual_cndmask_b32 v8, v8, v16 :: v_dual_cndmask_b32 v9, v9, v17
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 6, v18
+; GFX11-NEXT:    v_cndmask_b32_e64 v10, v10, v16, s0
+; GFX11-NEXT:    v_cndmask_b32_e64 v11, v11, v17, s0
+; GFX11-NEXT:    v_dual_cndmask_b32 v12, v12, v16 :: v_dual_cndmask_b32 v13, v13, v17
 ; GFX11-NEXT:    global_store_b128 v[0:1], v[0:3], off dlc
 ; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-NEXT:    global_store_b128 v[0:1], v[4:7], off dlc
@@ -2436,29 +2433,29 @@ define amdgpu_ps void @dyn_insertelement_v8f64_v_v_v_add_1(<8 x double> %vec, do
 ; GPRIDX:       ; %bb.0: ; %entry
 ; GPRIDX-NEXT:    v_add_u32_e32 v18, 1, v18
 ; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v18
-; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[0:1], 1, v18
 ; GPRIDX-NEXT:    v_cndmask_b32_e32 v0, v0, v16, vcc
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v2, v2, v16, s[0:1]
-; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[2:3], 2, v18
-; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[4:5], 3, v18
-; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[6:7], 4, v18
-; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[8:9], 5, v18
-; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[10:11], 7, v18
-; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[12:13], 6, v18
 ; GPRIDX-NEXT:    v_cndmask_b32_e32 v1, v1, v17, vcc
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v3, v3, v17, s[0:1]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v4, v4, v16, s[2:3]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v6, v6, v16, s[4:5]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v8, v8, v16, s[6:7]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v10, v10, v16, s[8:9]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v12, v12, v16, s[12:13]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v14, v14, v16, s[10:11]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v5, v5, v17, s[2:3]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v7, v7, v17, s[4:5]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v9, v9, v17, s[6:7]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v11, v11, v17, s[8:9]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v13, v13, v17, s[12:13]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v15, v15, v17, s[10:11]
+; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v18
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v2, v2, v16, vcc
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v3, v3, v17, vcc
+; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, 2, v18
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v4, v4, v16, vcc
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v5, v5, v17, vcc
+; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, 3, v18
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v6, v6, v16, vcc
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v7, v7, v17, vcc
+; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, 4, v18
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v8, v8, v16, vcc
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v9, v9, v17, vcc
+; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, 5, v18
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v10, v10, v16, vcc
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v11, v11, v17, vcc
+; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, 6, v18
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v12, v12, v16, vcc
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v13, v13, v17, vcc
+; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, 7, v18
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v14, v14, v16, vcc
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v15, v15, v17, vcc
 ; GPRIDX-NEXT:    global_store_dwordx4 v[0:1], v[0:3], off
 ; GPRIDX-NEXT:    s_waitcnt vmcnt(0)
 ; GPRIDX-NEXT:    global_store_dwordx4 v[0:1], v[4:7], off
@@ -2474,28 +2471,28 @@ define amdgpu_ps void @dyn_insertelement_v8f64_v_v_v_add_1(<8 x double> %vec, do
 ; GFX10-NEXT:    v_add_nc_u32_e32 v18, 1, v18
 ; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v18
 ; GFX10-NEXT:    v_cmp_eq_u32_e64 s0, 1, v18
-; GFX10-NEXT:    v_cmp_eq_u32_e64 s1, 2, v18
-; GFX10-NEXT:    v_cmp_eq_u32_e64 s2, 3, v18
-; GFX10-NEXT:    v_cmp_eq_u32_e64 s3, 4, v18
-; GFX10-NEXT:    v_cmp_eq_u32_e64 s4, 5, v18
-; GFX10-NEXT:    v_cmp_eq_u32_e64 s5, 7, v18
-; GFX10-NEXT:    v_cmp_eq_u32_e64 s6, 6, v18
+; GFX10-NEXT:    v_cmp_eq_u32_e64 s1, 7, v18
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v16, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, v16, s0
 ; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v17, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 2, v18
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, v16, s0
 ; GFX10-NEXT:    v_cndmask_b32_e64 v3, v3, v17, s0
-; GFX10-NEXT:    v_cndmask_b32_e64 v4, v4, v16, s1
-; GFX10-NEXT:    v_cndmask_b32_e64 v6, v6, v16, s2
-; GFX10-NEXT:    v_cndmask_b32_e64 v5, v5, v17, s1
-; GFX10-NEXT:    v_cndmask_b32_e64 v7, v7, v17, s2
-; GFX10-NEXT:    v_cndmask_b32_e64 v8, v8, v16, s3
-; GFX10-NEXT:    v_cndmask_b32_e64 v10, v10, v16, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v9, v9, v17, s3
-; GFX10-NEXT:    v_cndmask_b32_e64 v11, v11, v17, s4
-; GFX10-NEXT:    v_cndmask_b32_e64 v12, v12, v16, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v14, v14, v16, s5
-; GFX10-NEXT:    v_cndmask_b32_e64 v13, v13, v17, s6
-; GFX10-NEXT:    v_cndmask_b32_e64 v15, v15, v17, s5
+; GFX10-NEXT:    v_cmp_eq_u32_e64 s0, 3, v18
+; GFX10-NEXT:    v_cndmask_b32_e64 v14, v14, v16, s1
+; GFX10-NEXT:    v_cndmask_b32_e32 v4, v4, v16, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v5, v5, v17, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 4, v18
+; GFX10-NEXT:    v_cndmask_b32_e64 v6, v6, v16, s0
+; GFX10-NEXT:    v_cndmask_b32_e64 v7, v7, v17, s0
+; GFX10-NEXT:    v_cmp_eq_u32_e64 s0, 5, v18
+; GFX10-NEXT:    v_cndmask_b32_e64 v15, v15, v17, s1
+; GFX10-NEXT:    v_cndmask_b32_e32 v8, v8, v16, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v9, v9, v17, vcc_lo
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 6, v18
+; GFX10-NEXT:    v_cndmask_b32_e64 v10, v10, v16, s0
+; GFX10-NEXT:    v_cndmask_b32_e64 v11, v11, v17, s0
+; GFX10-NEXT:    v_cndmask_b32_e32 v12, v12, v16, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v13, v13, v17, vcc_lo
 ; GFX10-NEXT:    global_store_dwordx4 v[0:1], v[0:3], off
 ; GFX10-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX10-NEXT:    global_store_dwordx4 v[0:1], v[4:7], off
@@ -2512,27 +2509,26 @@ define amdgpu_ps void @dyn_insertelement_v8f64_v_v_v_add_1(<8 x double> %vec, do
 ; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v18
 ; GFX11-NEXT:    v_cndmask_b32_e32 v1, v1, v17, vcc_lo
 ; GFX11-NEXT:    v_cmp_eq_u32_e64 s0, 1, v18
-; GFX11-NEXT:    v_cmp_eq_u32_e64 s1, 2, v18
-; GFX11-NEXT:    v_cmp_eq_u32_e64 s2, 3, v18
-; GFX11-NEXT:    v_cmp_eq_u32_e64 s3, 4, v18
-; GFX11-NEXT:    v_cmp_eq_u32_e64 s4, 5, v18
-; GFX11-NEXT:    v_cmp_eq_u32_e64 s5, 7, v18
-; GFX11-NEXT:    v_cmp_eq_u32_e64 s6, 6, v18
 ; GFX11-NEXT:    v_cndmask_b32_e32 v0, v0, v16, vcc_lo
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 2, v18
+; GFX11-NEXT:    v_cmp_eq_u32_e64 s1, 7, v18
 ; GFX11-NEXT:    v_cndmask_b32_e64 v2, v2, v16, s0
 ; GFX11-NEXT:    v_cndmask_b32_e64 v3, v3, v17, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v4, v4, v16, s1
-; GFX11-NEXT:    v_cndmask_b32_e64 v6, v6, v16, s2
-; GFX11-NEXT:    v_cndmask_b32_e64 v5, v5, v17, s1
-; GFX11-NEXT:    v_cndmask_b32_e64 v7, v7, v17, s2
-; GFX11-NEXT:    v_cndmask_b32_e64 v8, v8, v16, s3
-; GFX11-NEXT:    v_cndmask_b32_e64 v10, v10, v16, s4
-; GFX11-NEXT:    v_cndmask_b32_e64 v9, v9, v17, s3
-; GFX11-NEXT:    v_cndmask_b32_e64 v11, v11, v17, s4
-; GFX11-NEXT:    v_cndmask_b32_e64 v12, v12, v16, s6
-; GFX11-NEXT:    v_cndmask_b32_e64 v14, v14, v16, s5
-; GFX11-NEXT:    v_cndmask_b32_e64 v13, v13, v17, s6
-; GFX11-NEXT:    v_cndmask_b32_e64 v15, v15, v17, s5
+; GFX11-NEXT:    v_cndmask_b32_e32 v5, v5, v17, vcc_lo
+; GFX11-NEXT:    v_cmp_eq_u32_e64 s0, 3, v18
+; GFX11-NEXT:    v_cndmask_b32_e32 v4, v4, v16, vcc_lo
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 4, v18
+; GFX11-NEXT:    v_cndmask_b32_e64 v14, v14, v16, s1
+; GFX11-NEXT:    v_cndmask_b32_e64 v15, v15, v17, s1
+; GFX11-NEXT:    v_cndmask_b32_e64 v6, v6, v16, s0
+; GFX11-NEXT:    v_cndmask_b32_e64 v7, v7, v17, s0
+; GFX11-NEXT:    v_cndmask_b32_e32 v9, v9, v17, vcc_lo
+; GFX11-NEXT:    v_cmp_eq_u32_e64 s0, 5, v18
+; GFX11-NEXT:    v_cndmask_b32_e32 v8, v8, v16, vcc_lo
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 6, v18
+; GFX11-NEXT:    v_cndmask_b32_e64 v10, v10, v16, s0
+; GFX11-NEXT:    v_cndmask_b32_e64 v11, v11, v17, s0
+; GFX11-NEXT:    v_dual_cndmask_b32 v13, v13, v17 :: v_dual_cndmask_b32 v12, v12, v16
 ; GFX11-NEXT:    global_store_b128 v[0:1], v[0:3], off dlc
 ; GFX11-NEXT:    s_waitcnt_vscnt null, 0x0
 ; GFX11-NEXT:    global_store_b128 v[0:1], v[4:7], off dlc
@@ -5675,10 +5671,6 @@ define amdgpu_ps <7 x double> @dyn_insertelement_v7f64_s_v_v(<7 x double> inreg
 ; GPRIDX-NEXT:    s_mov_b32 s12, s14
 ; GPRIDX-NEXT:    s_mov_b32 s13, s15
 ; GPRIDX-NEXT:    v_mov_b32_e32 v18, s15
-; GPRIDX-NEXT:    v_mov_b32_e32 v3, s0
-; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
-; GPRIDX-NEXT:    v_mov_b32_e32 v4, s1
-; GPRIDX-NEXT:    v_cndmask_b32_e32 v3, v3, v0, vcc
 ; GPRIDX-NEXT:    v_mov_b32_e32 v17, s14
 ; GPRIDX-NEXT:    v_mov_b32_e32 v16, s13
 ; GPRIDX-NEXT:    v_mov_b32_e32 v15, s12
@@ -5692,39 +5684,43 @@ define amdgpu_ps <7 x double> @dyn_insertelement_v7f64_s_v_v(<7 x double> inreg
 ; GPRIDX-NEXT:    v_mov_b32_e32 v7, s4
 ; GPRIDX-NEXT:    v_mov_b32_e32 v6, s3
 ; GPRIDX-NEXT:    v_mov_b32_e32 v5, s2
+; GPRIDX-NEXT:    v_mov_b32_e32 v4, s1
+; GPRIDX-NEXT:    v_mov_b32_e32 v3, s0
+; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v2
+; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[0:1], 2, v2
+; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[2:3], 3, v2
+; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[4:5], 4, v2
+; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[6:7], 5, v2
+; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[8:9], 6, v2
+; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[10:11], 1, v2
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v3, v3, v0, vcc
+; GPRIDX-NEXT:    v_cndmask_b32_e64 v2, v5, v0, s[10:11]
+; GPRIDX-NEXT:    v_cndmask_b32_e64 v5, v7, v0, s[0:1]
+; GPRIDX-NEXT:    v_cndmask_b32_e64 v7, v9, v0, s[2:3]
+; GPRIDX-NEXT:    v_cndmask_b32_e64 v9, v11, v0, s[4:5]
+; GPRIDX-NEXT:    v_cndmask_b32_e64 v11, v13, v0, s[6:7]
+; GPRIDX-NEXT:    v_cndmask_b32_e64 v0, v15, v0, s[8:9]
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc
+; GPRIDX-NEXT:    v_cndmask_b32_e64 v6, v6, v1, s[10:11]
+; GPRIDX-NEXT:    v_cndmask_b32_e64 v8, v8, v1, s[0:1]
+; GPRIDX-NEXT:    v_cndmask_b32_e64 v10, v10, v1, s[2:3]
+; GPRIDX-NEXT:    v_cndmask_b32_e64 v12, v12, v1, s[4:5]
+; GPRIDX-NEXT:    v_cndmask_b32_e64 v13, v14, v1, s[6:7]
+; GPRIDX-NEXT:    v_cndmask_b32_e64 v1, v16, v1, s[8:9]
 ; GPRIDX-NEXT:    v_readfirstlane_b32 s0, v3
-; GPRIDX-NEXT:    v_cndmask_b32_e32 v3, v4, v1, vcc
-; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v2
-; GPRIDX-NEXT:    v_readfirstlane_b32 s1, v3
-; GPRIDX-NEXT:    v_cndmask_b32_e32 v3, v5, v0, vcc
-; GPRIDX-NEXT:    v_readfirstlane_b32 s2, v3
-; GPRIDX-NEXT:    v_cndmask_b32_e32 v3, v6, v1, vcc
-; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, 2, v2
-; GPRIDX-NEXT:    v_cndmask_b32_e32 v4, v7, v0, vcc
-; GPRIDX-NEXT:    v_cndmask_b32_e32 v5, v8, v1, vcc
-; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, 3, v2
-; GPRIDX-NEXT:    v_cndmask_b32_e32 v6, v9, v0, vcc
-; GPRIDX-NEXT:    v_cndmask_b32_e32 v7, v10, v1, vcc
-; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, 4, v2
-; GPRIDX-NEXT:    v_cndmask_b32_e32 v8, v11, v0, vcc
-; GPRIDX-NEXT:    v_cndmask_b32_e32 v9, v12, v1, vcc
-; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, 5, v2
-; GPRIDX-NEXT:    v_cndmask_b32_e32 v10, v13, v0, vcc
-; GPRIDX-NEXT:    v_cndmask_b32_e32 v11, v14, v1, vcc
-; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, 6, v2
-; GPRIDX-NEXT:    v_cndmask_b32_e32 v12, v15, v0, vcc
-; GPRIDX-NEXT:    v_cndmask_b32_e32 v13, v16, v1, vcc
-; GPRIDX-NEXT:    v_readfirstlane_b32 s3, v3
-; GPRIDX-NEXT:    v_readfirstlane_b32 s4, v4
-; GPRIDX-NEXT:    v_readfirstlane_b32 s5, v5
-; GPRIDX-NEXT:    v_readfirstlane_b32 s6, v6
-; GPRIDX-NEXT:    v_readfirstlane_b32 s7, v7
-; GPRIDX-NEXT:    v_readfirstlane_b32 s8, v8
-; GPRIDX-NEXT:    v_readfirstlane_b32 s9, v9
-; GPRIDX-NEXT:    v_readfirstlane_b32 s10, v10
-; GPRIDX-NEXT:    v_readfirstlane_b32 s11, v11
-; GPRIDX-NEXT:    v_readfirstlane_b32 s12, v12
-; GPRIDX-NEXT:    v_readfirstlane_b32 s13, v13
+; GPRIDX-NEXT:    v_readfirstlane_b32 s1, v4
+; GPRIDX-NEXT:    v_readfirstlane_b32 s2, v2
+; GPRIDX-NEXT:    v_readfirstlane_b32 s3, v6
+; GPRIDX-NEXT:    v_readfirstlane_b32 s4, v5
+; GPRIDX-NEXT:    v_readfirstlane_b32 s5, v8
+; GPRIDX-NEXT:    v_readfirstlane_b32 s6, v7
+; GPRIDX-NEXT:    v_readfirstlane_b32 s7, v10
+; GPRIDX-NEXT:    v_readfirstlane_b32 s8, v9
+; GPRIDX-NEXT:    v_readfirstlane_b32 s9, v12
+; GPRIDX-NEXT:    v_readfirstlane_b32 s10, v11
+; GPRIDX-NEXT:    v_readfirstlane_b32 s11, v13
+; GPRIDX-NEXT:    v_readfirstlane_b32 s12, v0
+; GPRIDX-NEXT:    v_readfirstlane_b32 s13, v1
 ; GPRIDX-NEXT:    ; return to shader part epilog
 ;
 ; GFX10-LABEL: dyn_insertelement_v7f64_s_v_v:
@@ -5744,13 +5740,9 @@ define amdgpu_ps <7 x double> @dyn_insertelement_v7f64_s_v_v(<7 x double> inreg
 ; GFX10-NEXT:    s_mov_b32 s12, s14
 ; GFX10-NEXT:    s_mov_b32 s13, s15
 ; GFX10-NEXT:    v_mov_b32_e32 v18, s15
-; GFX10-NEXT:    v_mov_b32_e32 v3, s0
-; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v2
-; GFX10-NEXT:    v_mov_b32_e32 v4, s1
 ; GFX10-NEXT:    v_mov_b32_e32 v17, s14
 ; GFX10-NEXT:    v_mov_b32_e32 v16, s13
 ; GFX10-NEXT:    v_mov_b32_e32 v15, s12
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v0, vcc_lo
 ; GFX10-NEXT:    v_mov_b32_e32 v14, s11
 ; GFX10-NEXT:    v_mov_b32_e32 v13, s10
 ; GFX10-NEXT:    v_mov_b32_e32 v12, s9
@@ -5761,39 +5753,43 @@ define amdgpu_ps <7 x double> @dyn_insertelement_v7f64_s_v_v(<7 x double> inreg
 ; GFX10-NEXT:    v_mov_b32_e32 v7, s4
 ; GFX10-NEXT:    v_mov_b32_e32 v6, s3
 ; GFX10-NEXT:    v_mov_b32_e32 v5, s2
-; GFX10-NEXT:    v_readfirstlane_b32 s0, v3
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v4, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v2
-; GFX10-NEXT:    v_readfirstlane_b32 s1, v3
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v5, v0, vcc_lo
-; GFX10-NEXT:    v_readfirstlane_b32 s2, v3
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v6, v1, vcc_lo
+; GFX10-NEXT:    v_mov_b32_e32 v4, s1
+; GFX10-NEXT:    v_mov_b32_e32 v3, s0
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v2
+; GFX10-NEXT:    v_cmp_eq_u32_e64 s0, 1, v2
+; GFX10-NEXT:    v_cmp_eq_u32_e64 s1, 6, v2
+; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v4, v4, v1, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 2, v2
-; GFX10-NEXT:    v_readfirstlane_b32 s3, v3
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v7, v0, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v8, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 3, v2
-; GFX10-NEXT:    v_readfirstlane_b32 s4, v4
-; GFX10-NEXT:    v_readfirstlane_b32 s5, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v9, v0, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v10, v1, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v5, v5, v0, s0
+; GFX10-NEXT:    v_cndmask_b32_e64 v6, v6, v1, s0
+; GFX10-NEXT:    v_cmp_eq_u32_e64 s0, 3, v2
+; GFX10-NEXT:    v_cndmask_b32_e32 v7, v7, v0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v8, v8, v1, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 4, v2
-; GFX10-NEXT:    v_readfirstlane_b32 s6, v6
-; GFX10-NEXT:    v_readfirstlane_b32 s7, v7
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, v11, v0, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e32 v9, v12, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 5, v2
-; GFX10-NEXT:    v_readfirstlane_b32 s8, v8
-; GFX10-NEXT:    v_readfirstlane_b32 s9, v9
-; GFX10-NEXT:    v_cndmask_b32_e32 v10, v13, v0, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e32 v11, v14, v1, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 6, v2
-; GFX10-NEXT:    v_readfirstlane_b32 s10, v10
-; GFX10-NEXT:    v_readfirstlane_b32 s11, v11
-; GFX10-NEXT:    v_cndmask_b32_e32 v12, v15, v0, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e32 v13, v16, v1, vcc_lo
-; GFX10-NEXT:    v_readfirstlane_b32 s12, v12
-; GFX10-NEXT:    v_readfirstlane_b32 s13, v13
+; GFX10-NEXT:    v_cndmask_b32_e64 v9, v9, v0, s0
+; GFX10-NEXT:    v_cndmask_b32_e64 v10, v10, v1, s0
+; GFX10-NEXT:    v_cmp_eq_u32_e64 s0, 5, v2
+; GFX10-NEXT:    v_readfirstlane_b32 s2, v5
+; GFX10-NEXT:    v_cndmask_b32_e32 v11, v11, v0, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v2, v12, v1, vcc_lo
+; GFX10-NEXT:    v_readfirstlane_b32 s3, v6
+; GFX10-NEXT:    v_cndmask_b32_e64 v12, v13, v0, s0
+; GFX10-NEXT:    v_cndmask_b32_e64 v13, v14, v1, s0
+; GFX10-NEXT:    v_cndmask_b32_e64 v0, v15, v0, s1
+; GFX10-NEXT:    v_cndmask_b32_e64 v1, v16, v1, s1
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v3
+; GFX10-NEXT:    v_readfirstlane_b32 s1, v4
+; GFX10-NEXT:    v_readfirstlane_b32 s4, v7
+; GFX10-NEXT:    v_readfirstlane_b32 s5, v8
+; GFX10-NEXT:    v_readfirstlane_b32 s6, v9
+; GFX10-NEXT:    v_readfirstlane_b32 s7, v10
+; GFX10-NEXT:    v_readfirstlane_b32 s8, v11
+; GFX10-NEXT:    v_readfirstlane_b32 s9, v2
+; GFX10-NEXT:    v_readfirstlane_b32 s10, v12
+; GFX10-NEXT:    v_readfirstlane_b32 s11, v13
+; GFX10-NEXT:    v_readfirstlane_b32 s12, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s13, v1
 ; GFX10-NEXT:    ; return to shader part epilog
 ;
 ; GFX11-LABEL: dyn_insertelement_v7f64_s_v_v:
@@ -5813,44 +5809,45 @@ define amdgpu_ps <7 x double> @dyn_insertelement_v7f64_s_v_v(<7 x double> inreg
 ; GFX11-NEXT:    s_mov_b32 s12, s14
 ; GFX11-NEXT:    s_mov_b32 s13, s15
 ; GFX11-NEXT:    v_dual_mov_b32 v18, s15 :: v_dual_mov_b32 v17, s14
-; GFX11-NEXT:    v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v3, s0
-; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v2
 ; GFX11-NEXT:    v_dual_mov_b32 v16, s13 :: v_dual_mov_b32 v15, s12
 ; GFX11-NEXT:    v_dual_mov_b32 v14, s11 :: v_dual_mov_b32 v13, s10
 ; GFX11-NEXT:    v_dual_mov_b32 v12, s9 :: v_dual_mov_b32 v11, s8
 ; GFX11-NEXT:    v_dual_mov_b32 v10, s7 :: v_dual_mov_b32 v9, s6
 ; GFX11-NEXT:    v_dual_mov_b32 v8, s5 :: v_dual_mov_b32 v7, s4
 ; GFX11-NEXT:    v_dual_mov_b32 v6, s3 :: v_dual_mov_b32 v5, s2
+; GFX11-NEXT:    v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v3, s0
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v2
 ; GFX11-NEXT:    v_cmp_eq_u32_e64 s0, 1, v2
-; GFX11-NEXT:    v_dual_cndmask_b32 v18, v3, v0 :: v_dual_cndmask_b32 v17, v4, v1
+; GFX11-NEXT:    v_cmp_eq_u32_e64 s1, 6, v2
+; GFX11-NEXT:    v_dual_cndmask_b32 v3, v3, v0 :: v_dual_cndmask_b32 v4, v4, v1
 ; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 2, v2
-; GFX11-NEXT:    v_cmp_eq_u32_e64 s6, 4, v2
-; GFX11-NEXT:    v_cndmask_b32_e64 v3, v5, v0, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v4, v6, v1, s0
-; GFX11-NEXT:    v_cmp_eq_u32_e64 s9, 6, v2
-; GFX11-NEXT:    v_dual_cndmask_b32 v5, v7, v0 :: v_dual_cndmask_b32 v6, v8, v1
-; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 3, v2
-; GFX11-NEXT:    v_readfirstlane_b32 s0, v18
-; GFX11-NEXT:    v_readfirstlane_b32 s1, v17
-; GFX11-NEXT:    v_readfirstlane_b32 s2, v3
-; GFX11-NEXT:    v_readfirstlane_b32 s3, v4
-; GFX11-NEXT:    v_dual_cndmask_b32 v7, v9, v0 :: v_dual_cndmask_b32 v8, v10, v1
-; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 5, v2
-; GFX11-NEXT:    v_cndmask_b32_e64 v10, v11, v0, s6
-; GFX11-NEXT:    v_cndmask_b32_e64 v9, v12, v1, s6
-; GFX11-NEXT:    v_readfirstlane_b32 s4, v5
-; GFX11-NEXT:    v_readfirstlane_b32 s5, v6
-; GFX11-NEXT:    v_dual_cndmask_b32 v11, v13, v0 :: v_dual_cndmask_b32 v12, v14, v1
-; GFX11-NEXT:    v_cndmask_b32_e64 v14, v15, v0, s9
-; GFX11-NEXT:    v_cndmask_b32_e64 v13, v16, v1, s9
-; GFX11-NEXT:    v_readfirstlane_b32 s6, v7
-; GFX11-NEXT:    v_readfirstlane_b32 s7, v8
-; GFX11-NEXT:    v_readfirstlane_b32 s8, v10
-; GFX11-NEXT:    v_readfirstlane_b32 s9, v9
-; GFX11-NEXT:    v_readfirstlane_b32 s10, v11
-; GFX11-NEXT:    v_readfirstlane_b32 s11, v12
-; GFX11-NEXT:    v_readfirstlane_b32 s12, v14
-; GFX11-NEXT:    v_readfirstlane_b32 s13, v13
+; GFX11-NEXT:    v_cndmask_b32_e64 v5, v5, v0, s0
+; GFX11-NEXT:    v_cndmask_b32_e64 v6, v6, v1, s0
+; GFX11-NEXT:    v_cmp_eq_u32_e64 s0, 3, v2
+; GFX11-NEXT:    v_dual_cndmask_b32 v7, v7, v0 :: v_dual_cndmask_b32 v8, v8, v1
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 4, v2
+; GFX11-NEXT:    v_cndmask_b32_e64 v9, v9, v0, s0
+; GFX11-NEXT:    v_cndmask_b32_e64 v10, v10, v1, s0
+; GFX11-NEXT:    v_cmp_eq_u32_e64 s0, 5, v2
+; GFX11-NEXT:    v_readfirstlane_b32 s2, v5
+; GFX11-NEXT:    v_dual_cndmask_b32 v11, v11, v0 :: v_dual_cndmask_b32 v2, v12, v1
+; GFX11-NEXT:    v_readfirstlane_b32 s3, v6
+; GFX11-NEXT:    v_cndmask_b32_e64 v12, v13, v0, s0
+; GFX11-NEXT:    v_cndmask_b32_e64 v13, v14, v1, s0
+; GFX11-NEXT:    v_cndmask_b32_e64 v0, v15, v0, s1
+; GFX11-NEXT:    v_cndmask_b32_e64 v1, v16, v1, s1
+; GFX11-NEXT:    v_readfirstlane_b32 s0, v3
+; GFX11-NEXT:    v_readfirstlane_b32 s1, v4
+; GFX11-NEXT:    v_readfirstlane_b32 s4, v7
+; GFX11-NEXT:    v_readfirstlane_b32 s5, v8
+; GFX11-NEXT:    v_readfirstlane_b32 s6, v9
+; GFX11-NEXT:    v_readfirstlane_b32 s7, v10
+; GFX11-NEXT:    v_readfirstlane_b32 s8, v11
+; GFX11-NEXT:    v_readfirstlane_b32 s9, v2
+; GFX11-NEXT:    v_readfirstlane_b32 s10, v12
+; GFX11-NEXT:    v_readfirstlane_b32 s11, v13
+; GFX11-NEXT:    v_readfirstlane_b32 s12, v0
+; GFX11-NEXT:    v_readfirstlane_b32 s13, v1
 ; GFX11-NEXT:    ; return to shader part epilog
 entry:
   %insert = insertelement <7 x double> %vec, double %val, i32 %idx
@@ -5951,38 +5948,38 @@ define amdgpu_ps <7 x double> @dyn_insertelement_v7f64_v_v_v(<7 x double> %vec,
 ; GFX10-LABEL: dyn_insertelement_v7f64_v_v_v:
 ; GFX10:       ; %bb.0: ; %entry
 ; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v16
+; GFX10-NEXT:    v_cmp_eq_u32_e64 s0, 1, v16
+; GFX10-NEXT:    v_cmp_eq_u32_e64 s1, 6, v16
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v14, vcc_lo
 ; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v15, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v16
-; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
-; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v14, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v15, vcc_lo
 ; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 2, v16
-; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
-; GFX10-NEXT:    v_readfirstlane_b32 s3, v3
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, v14, s0
+; GFX10-NEXT:    v_cndmask_b32_e64 v3, v3, v15, s0
+; GFX10-NEXT:    v_cmp_eq_u32_e64 s0, 3, v16
+; GFX10-NEXT:    v_cndmask_b32_e64 v12, v12, v14, s1
 ; GFX10-NEXT:    v_cndmask_b32_e32 v4, v4, v14, vcc_lo
 ; GFX10-NEXT:    v_cndmask_b32_e32 v5, v5, v15, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 3, v16
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 4, v16
+; GFX10-NEXT:    v_cndmask_b32_e64 v6, v6, v14, s0
+; GFX10-NEXT:    v_cndmask_b32_e64 v7, v7, v15, s0
+; GFX10-NEXT:    v_cmp_eq_u32_e64 s0, 5, v16
+; GFX10-NEXT:    v_cndmask_b32_e64 v13, v13, v15, s1
+; GFX10-NEXT:    v_cndmask_b32_e32 v8, v8, v14, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v9, v9, v15, vcc_lo
+; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX10-NEXT:    v_cndmask_b32_e64 v10, v10, v14, s0
+; GFX10-NEXT:    v_cndmask_b32_e64 v11, v11, v15, s0
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX10-NEXT:    v_readfirstlane_b32 s3, v3
 ; GFX10-NEXT:    v_readfirstlane_b32 s4, v4
 ; GFX10-NEXT:    v_readfirstlane_b32 s5, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v6, v14, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v7, v15, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 4, v16
 ; GFX10-NEXT:    v_readfirstlane_b32 s6, v6
 ; GFX10-NEXT:    v_readfirstlane_b32 s7, v7
-; GFX10-NEXT:    v_cndmask_b32_e32 v8, v8, v14, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e32 v9, v9, v15, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 5, v16
 ; GFX10-NEXT:    v_readfirstlane_b32 s8, v8
 ; GFX10-NEXT:    v_readfirstlane_b32 s9, v9
-; GFX10-NEXT:    v_cndmask_b32_e32 v10, v10, v14, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e32 v11, v11, v15, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 6, v16
 ; GFX10-NEXT:    v_readfirstlane_b32 s10, v10
 ; GFX10-NEXT:    v_readfirstlane_b32 s11, v11
-; GFX10-NEXT:    v_cndmask_b32_e32 v12, v12, v14, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e32 v13, v13, v15, vcc_lo
 ; GFX10-NEXT:    v_readfirstlane_b32 s12, v12
 ; GFX10-NEXT:    v_readfirstlane_b32 s13, v13
 ; GFX10-NEXT:    ; return to shader part epilog
@@ -5990,35 +5987,37 @@ define amdgpu_ps <7 x double> @dyn_insertelement_v7f64_v_v_v(<7 x double> %vec,
 ; GFX11-LABEL: dyn_insertelement_v7f64_v_v_v:
 ; GFX11:       ; %bb.0: ; %entry
 ; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v16
-; GFX11-NEXT:    v_cmp_eq_u32_e64 s9, 5, v16
-; GFX11-NEXT:    v_cmp_eq_u32_e64 s10, 6, v16
+; GFX11-NEXT:    v_cmp_eq_u32_e64 s0, 1, v16
+; GFX11-NEXT:    v_cmp_eq_u32_e64 s1, 6, v16
 ; GFX11-NEXT:    v_dual_cndmask_b32 v0, v0, v14 :: v_dual_cndmask_b32 v1, v1, v15
-; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v16
-; GFX11-NEXT:    v_cndmask_b32_e64 v10, v10, v14, s9
-; GFX11-NEXT:    v_cndmask_b32_e64 v11, v11, v15, s9
-; GFX11-NEXT:    v_cndmask_b32_e64 v12, v12, v14, s10
-; GFX11-NEXT:    v_cndmask_b32_e64 v13, v13, v15, s10
-; GFX11-NEXT:    v_dual_cndmask_b32 v2, v2, v14 :: v_dual_cndmask_b32 v3, v3, v15
 ; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 2, v16
-; GFX11-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX11-NEXT:    v_cndmask_b32_e64 v2, v2, v14, s0
+; GFX11-NEXT:    v_cndmask_b32_e64 v3, v3, v15, s0
+; GFX11-NEXT:    v_cmp_eq_u32_e64 s0, 3, v16
+; GFX11-NEXT:    v_cndmask_b32_e64 v12, v12, v14, s1
+; GFX11-NEXT:    v_dual_cndmask_b32 v4, v4, v14 :: v_dual_cndmask_b32 v5, v5, v15
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 4, v16
+; GFX11-NEXT:    v_cndmask_b32_e64 v6, v6, v14, s0
+; GFX11-NEXT:    v_cndmask_b32_e64 v7, v7, v15, s0
+; GFX11-NEXT:    v_cmp_eq_u32_e64 s0, 5, v16
+; GFX11-NEXT:    v_cndmask_b32_e64 v13, v13, v15, s1
+; GFX11-NEXT:    v_dual_cndmask_b32 v8, v8, v14 :: v_dual_cndmask_b32 v9, v9, v15
 ; GFX11-NEXT:    v_readfirstlane_b32 s1, v1
+; GFX11-NEXT:    v_cndmask_b32_e64 v10, v10, v14, s0
+; GFX11-NEXT:    v_cndmask_b32_e64 v11, v11, v15, s0
+; GFX11-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX11-NEXT:    v_readfirstlane_b32 s2, v2
 ; GFX11-NEXT:    v_readfirstlane_b32 s3, v3
-; GFX11-NEXT:    v_dual_cndmask_b32 v4, v4, v14 :: v_dual_cndmask_b32 v5, v5, v15
-; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 3, v16
-; GFX11-NEXT:    v_readfirstlane_b32 s10, v10
-; GFX11-NEXT:    v_readfirstlane_b32 s11, v11
 ; GFX11-NEXT:    v_readfirstlane_b32 s4, v4
 ; GFX11-NEXT:    v_readfirstlane_b32 s5, v5
-; GFX11-NEXT:    v_dual_cndmask_b32 v6, v6, v14 :: v_dual_cndmask_b32 v7, v7, v15
-; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 4, v16
-; GFX11-NEXT:    v_readfirstlane_b32 s12, v12
-; GFX11-NEXT:    v_readfirstlane_b32 s13, v13
 ; GFX11-NEXT:    v_readfirstlane_b32 s6, v6
 ; GFX11-NEXT:    v_readfirstlane_b32 s7, v7
-; GFX11-NEXT:    v_dual_cndmask_b32 v8, v8, v14 :: v_dual_cndmask_b32 v9, v9, v15
 ; GFX11-NEXT:    v_readfirstlane_b32 s8, v8
 ; GFX11-NEXT:    v_readfirstlane_b32 s9, v9
+; GFX11-NEXT:    v_readfirstlane_b32 s10, v10
+; GFX11-NEXT:    v_readfirstlane_b32 s11, v11
+; GFX11-NEXT:    v_readfirstlane_b32 s12, v12
+; GFX11-NEXT:    v_readfirstlane_b32 s13, v13
 ; GFX11-NEXT:    ; return to shader part epilog
 entry:
   %insert = insertelement <7 x double> %vec, double %val, i32 %idx
@@ -6352,20 +6351,20 @@ define amdgpu_ps <5 x double> @dyn_insertelement_v5f64_v_v_s(<5 x double> %vec,
 ; GPRIDX-LABEL: dyn_insertelement_v5f64_v_v_s:
 ; GPRIDX:       ; %bb.0: ; %entry
 ; GPRIDX-NEXT:    v_cmp_eq_u32_e64 vcc, s2, 0
-; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[0:1], s2, 1
-; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[8:9], s2, 2
-; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[4:5], s2, 3
-; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[6:7], s2, 4
 ; GPRIDX-NEXT:    v_cndmask_b32_e32 v0, v0, v10, vcc
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v2, v2, v10, s[0:1]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v4, v4, v10, s[8:9]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v6, v6, v10, s[4:5]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v8, v8, v10, s[6:7]
 ; GPRIDX-NEXT:    v_cndmask_b32_e32 v1, v1, v11, vcc
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v3, v3, v11, s[0:1]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v5, v5, v11, s[8:9]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v7, v7, v11, s[4:5]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v9, v9, v11, s[6:7]
+; GPRIDX-NEXT:    v_cmp_eq_u32_e64 vcc, s2, 1
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v2, v2, v10, vcc
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v3, v3, v11, vcc
+; GPRIDX-NEXT:    v_cmp_eq_u32_e64 vcc, s2, 2
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v4, v4, v10, vcc
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v5, v5, v11, vcc
+; GPRIDX-NEXT:    v_cmp_eq_u32_e64 vcc, s2, 3
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v6, v6, v10, vcc
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v7, v7, v11, vcc
+; GPRIDX-NEXT:    v_cmp_eq_u32_e64 vcc, s2, 4
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v8, v8, v10, vcc
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v9, v9, v11, vcc
 ; GPRIDX-NEXT:    v_readfirstlane_b32 s0, v0
 ; GPRIDX-NEXT:    v_readfirstlane_b32 s1, v1
 ; GPRIDX-NEXT:    v_readfirstlane_b32 s2, v2
@@ -6381,55 +6380,56 @@ define amdgpu_ps <5 x double> @dyn_insertelement_v5f64_v_v_s(<5 x double> %vec,
 ; GFX10-LABEL: dyn_insertelement_v5f64_v_v_s:
 ; GFX10:       ; %bb.0: ; %entry
 ; GFX10-NEXT:    v_cmp_eq_u32_e64 vcc_lo, s2, 0
-; GFX10-NEXT:    v_cmp_eq_u32_e64 s0, s2, 4
+; GFX10-NEXT:    v_cmp_eq_u32_e64 s0, s2, 1
+; GFX10-NEXT:    v_cmp_eq_u32_e64 s1, s2, 4
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v10, vcc_lo
 ; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v11, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u32_e64 vcc_lo, s2, 1
-; GFX10-NEXT:    v_cndmask_b32_e64 v8, v8, v10, s0
-; GFX10-NEXT:    v_cndmask_b32_e64 v9, v9, v11, s0
-; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
-; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v10, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v11, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, v10, s0
 ; GFX10-NEXT:    v_cmp_eq_u32_e64 vcc_lo, s2, 2
-; GFX10-NEXT:    v_readfirstlane_b32 s8, v8
-; GFX10-NEXT:    v_readfirstlane_b32 s9, v9
-; GFX10-NEXT:    v_readfirstlane_b32 s3, v3
+; GFX10-NEXT:    v_cndmask_b32_e64 v3, v3, v11, s0
+; GFX10-NEXT:    v_cmp_eq_u32_e64 s0, s2, 3
+; GFX10-NEXT:    v_cndmask_b32_e64 v8, v8, v10, s1
+; GFX10-NEXT:    v_cndmask_b32_e64 v9, v9, v11, s1
 ; GFX10-NEXT:    v_cndmask_b32_e32 v4, v4, v10, vcc_lo
 ; GFX10-NEXT:    v_cndmask_b32_e32 v5, v5, v11, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u32_e64 vcc_lo, s2, 3
+; GFX10-NEXT:    v_cndmask_b32_e64 v6, v6, v10, s0
+; GFX10-NEXT:    v_cndmask_b32_e64 v7, v7, v11, s0
+; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
+; GFX10-NEXT:    v_readfirstlane_b32 s3, v3
 ; GFX10-NEXT:    v_readfirstlane_b32 s4, v4
 ; GFX10-NEXT:    v_readfirstlane_b32 s5, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v6, v10, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v7, v11, vcc_lo
 ; GFX10-NEXT:    v_readfirstlane_b32 s6, v6
 ; GFX10-NEXT:    v_readfirstlane_b32 s7, v7
+; GFX10-NEXT:    v_readfirstlane_b32 s8, v8
+; GFX10-NEXT:    v_readfirstlane_b32 s9, v9
 ; GFX10-NEXT:    ; return to shader part epilog
 ;
 ; GFX11-LABEL: dyn_insertelement_v5f64_v_v_s:
 ; GFX11:       ; %bb.0: ; %entry
 ; GFX11-NEXT:    v_cmp_eq_u32_e64 vcc_lo, s2, 0
-; GFX11-NEXT:    v_cmp_eq_u32_e64 s0, s2, 2
+; GFX11-NEXT:    v_cmp_eq_u32_e64 s0, s2, 1
 ; GFX11-NEXT:    v_cmp_eq_u32_e64 s1, s2, 4
 ; GFX11-NEXT:    v_dual_cndmask_b32 v0, v0, v10 :: v_dual_cndmask_b32 v1, v1, v11
-; GFX11-NEXT:    v_cmp_eq_u32_e64 vcc_lo, s2, 1
-; GFX11-NEXT:    v_cndmask_b32_e64 v4, v4, v10, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v5, v5, v11, s0
+; GFX11-NEXT:    v_cndmask_b32_e64 v2, v2, v10, s0
+; GFX11-NEXT:    v_cmp_eq_u32_e64 vcc_lo, s2, 2
+; GFX11-NEXT:    v_cndmask_b32_e64 v3, v3, v11, s0
+; GFX11-NEXT:    v_cmp_eq_u32_e64 s0, s2, 3
 ; GFX11-NEXT:    v_cndmask_b32_e64 v8, v8, v10, s1
 ; GFX11-NEXT:    v_cndmask_b32_e64 v9, v9, v11, s1
-; GFX11-NEXT:    v_dual_cndmask_b32 v2, v2, v10 :: v_dual_cndmask_b32 v3, v3, v11
-; GFX11-NEXT:    v_cmp_eq_u32_e64 vcc_lo, s2, 3
+; GFX11-NEXT:    v_dual_cndmask_b32 v4, v4, v10 :: v_dual_cndmask_b32 v5, v5, v11
+; GFX11-NEXT:    v_cndmask_b32_e64 v6, v6, v10, s0
+; GFX11-NEXT:    v_cndmask_b32_e64 v7, v7, v11, s0
 ; GFX11-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX11-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX11-NEXT:    v_readfirstlane_b32 s2, v2
 ; GFX11-NEXT:    v_readfirstlane_b32 s3, v3
-; GFX11-NEXT:    v_dual_cndmask_b32 v6, v6, v10 :: v_dual_cndmask_b32 v7, v7, v11
 ; GFX11-NEXT:    v_readfirstlane_b32 s4, v4
 ; GFX11-NEXT:    v_readfirstlane_b32 s5, v5
-; GFX11-NEXT:    v_readfirstlane_b32 s8, v8
 ; GFX11-NEXT:    v_readfirstlane_b32 s6, v6
 ; GFX11-NEXT:    v_readfirstlane_b32 s7, v7
+; GFX11-NEXT:    v_readfirstlane_b32 s8, v8
 ; GFX11-NEXT:    v_readfirstlane_b32 s9, v9
 ; GFX11-NEXT:    ; return to shader part epilog
 entry:
@@ -6441,20 +6441,20 @@ define amdgpu_ps <5 x double> @dyn_insertelement_v5f64_v_v_v(<5 x double> %vec,
 ; GPRIDX-LABEL: dyn_insertelement_v5f64_v_v_v:
 ; GPRIDX:       ; %bb.0: ; %entry
 ; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, 0, v12
-; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[0:1], 1, v12
-; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[2:3], 2, v12
-; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[4:5], 3, v12
-; GPRIDX-NEXT:    v_cmp_eq_u32_e64 s[6:7], 4, v12
 ; GPRIDX-NEXT:    v_cndmask_b32_e32 v0, v0, v10, vcc
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v2, v2, v10, s[0:1]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v4, v4, v10, s[2:3]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v6, v6, v10, s[4:5]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v8, v8, v10, s[6:7]
 ; GPRIDX-NEXT:    v_cndmask_b32_e32 v1, v1, v11, vcc
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v3, v3, v11, s[0:1]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v5, v5, v11, s[2:3]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v7, v7, v11, s[4:5]
-; GPRIDX-NEXT:    v_cndmask_b32_e64 v9, v9, v11, s[6:7]
+; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, 1, v12
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v2, v2, v10, vcc
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v3, v3, v11, vcc
+; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, 2, v12
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v4, v4, v10, vcc
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v5, v5, v11, vcc
+; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, 3, v12
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v6, v6, v10, vcc
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v7, v7, v11, vcc
+; GPRIDX-NEXT:    v_cmp_eq_u32_e32 vcc, 4, v12
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v8, v8, v10, vcc
+; GPRIDX-NEXT:    v_cndmask_b32_e32 v9, v9, v11, vcc
 ; GPRIDX-NEXT:    v_readfirstlane_b32 s0, v0
 ; GPRIDX-NEXT:    v_readfirstlane_b32 s1, v1
 ; GPRIDX-NEXT:    v_readfirstlane_b32 s2, v2
@@ -6470,55 +6470,56 @@ define amdgpu_ps <5 x double> @dyn_insertelement_v5f64_v_v_v(<5 x double> %vec,
 ; GFX10-LABEL: dyn_insertelement_v5f64_v_v_v:
 ; GFX10:       ; %bb.0: ; %entry
 ; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v12
-; GFX10-NEXT:    v_cmp_eq_u32_e64 s0, 4, v12
+; GFX10-NEXT:    v_cmp_eq_u32_e64 s0, 1, v12
+; GFX10-NEXT:    v_cmp_eq_u32_e64 s1, 4, v12
 ; GFX10-NEXT:    v_cndmask_b32_e32 v0, v0, v10, vcc_lo
 ; GFX10-NEXT:    v_cndmask_b32_e32 v1, v1, v11, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v12
-; GFX10-NEXT:    v_cndmask_b32_e64 v8, v8, v10, s0
-; GFX10-NEXT:    v_cndmask_b32_e64 v9, v9, v11, s0
+; GFX10-NEXT:    v_cndmask_b32_e64 v2, v2, v10, s0
+; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 2, v12
+; GFX10-NEXT:    v_cndmask_b32_e64 v3, v3, v11, s0
+; GFX10-NEXT:    v_cmp_eq_u32_e64 s0, 3, v12
+; GFX10-NEXT:    v_cndmask_b32_e64 v8, v8, v10, s1
+; GFX10-NEXT:    v_cndmask_b32_e64 v9, v9, v11, s1
+; GFX10-NEXT:    v_cndmask_b32_e32 v4, v4, v10, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v5, v5, v11, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e64 v6, v6, v10, s0
+; GFX10-NEXT:    v_cndmask_b32_e64 v7, v7, v11, s0
 ; GFX10-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX10-NEXT:    v_readfirstlane_b32 s1, v1
-; GFX10-NEXT:    v_cndmask_b32_e32 v2, v2, v10, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e32 v3, v3, v11, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 2, v12
-; GFX10-NEXT:    v_readfirstlane_b32 s8, v8
-; GFX10-NEXT:    v_readfirstlane_b32 s9, v9
 ; GFX10-NEXT:    v_readfirstlane_b32 s2, v2
 ; GFX10-NEXT:    v_readfirstlane_b32 s3, v3
-; GFX10-NEXT:    v_cndmask_b32_e32 v4, v4, v10, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v5, v11, vcc_lo
-; GFX10-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 3, v12
 ; GFX10-NEXT:    v_readfirstlane_b32 s4, v4
 ; GFX10-NEXT:    v_readfirstlane_b32 s5, v5
-; GFX10-NEXT:    v_cndmask_b32_e32 v6, v6, v10, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e32 v7, v7, v11, vcc_lo
 ; GFX10-NEXT:    v_readfirstlane_b32 s6, v6
 ; GFX10-NEXT:    v_readfirstlane_b32 s7, v7
+; GFX10-NEXT:    v_readfirstlane_b32 s8, v8
+; GFX10-NEXT:    v_readfirstlane_b32 s9, v9
 ; GFX10-NEXT:    ; return to shader part epilog
 ;
 ; GFX11-LABEL: dyn_insertelement_v5f64_v_v_v:
 ; GFX11:       ; %bb.0: ; %entry
 ; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 0, v12
-; GFX11-NEXT:    v_cmp_eq_u32_e64 s0, 2, v12
+; GFX11-NEXT:    v_cmp_eq_u32_e64 s0, 1, v12
 ; GFX11-NEXT:    v_cmp_eq_u32_e64 s1, 4, v12
 ; GFX11-NEXT:    v_dual_cndmask_b32 v0, v0, v10 :: v_dual_cndmask_b32 v1, v1, v11
-; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 1, v12
-; GFX11-NEXT:    v_cndmask_b32_e64 v4, v4, v10, s0
-; GFX11-NEXT:    v_cndmask_b32_e64 v5, v5, v11, s0
+; GFX11-NEXT:    v_cndmask_b32_e64 v2, v2, v10, s0
+; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 2, v12
+; GFX11-NEXT:    v_cndmask_b32_e64 v3, v3, v11, s0
+; GFX11-NEXT:    v_cmp_eq_u32_e64 s0, 3, v12
 ; GFX11-NEXT:    v_cndmask_b32_e64 v8, v8, v10, s1
 ; GFX11-NEXT:    v_cndmask_b32_e64 v9, v9, v11, s1
-; GFX11-NEXT:    v_dual_cndmask_b32 v2, v2, v10 :: v_dual_cndmask_b32 v3, v3, v11
-; GFX11-NEXT:    v_cmp_eq_u32_e32 vcc_lo, 3, v12
+; GFX11-NEXT:    v_dual_cndmask_b32 v4, v4, v10 :: v_dual_cndmask_b32 v5, v5, v11
+; GFX11-NEXT:    v_cndmask_b32_e64 v6, v6, v10, s0
+; GFX11-NEXT:    v_cndmask_b32_e64 v7, v7, v11, s0
 ; GFX11-NEXT:    v_readfirstlane_b32 s0, v0
 ; GFX11-NEXT:    v_readfirstlane_b32 s1, v1
 ; GFX11-NEXT:    v_readfirstlane_b32 s2, v2
 ; GFX11-NEXT:    v_readfirstlane_b32 s3, v3
-; GFX11-NEXT:    v_dual_cndmask_b32 v6, v6, v10 :: v_dual_cndmask_b32 v7, v7, v11
 ; GFX11-NEXT:    v_readfirstlane_b32 s4, v4
 ; GFX11-NEXT:    v_readfirstlane_b32 s5, v5
-; GFX11-NEXT:    v_readfirstlane_b32 s8, v8
 ; GFX11-NEXT:    v_readfirstlane_b32 s6, v6
 ; GFX11-NEXT:    v_readfirstlane_b32 s7, v7
+; GFX11-NEXT:    v_readfirstlane_b32 s8, v8
 ; GFX11-NEXT:    v_readfirstlane_b32 s9, v9
 ; GFX11-NEXT:    ; return to shader part epilog
 entry:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.rsq.clamp.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.rsq.clamp.ll
index bd570df3d83b3f6bdbf3eba9006b5927f5d194e8..ed298796937c718f780079e2eba75e7bf5c55e93 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.rsq.clamp.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.rsq.clamp.ll
@@ -1,6 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -global-isel -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -check-prefix=SI %s
 ; RUN: llc -global-isel -march=amdgcn -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=VI %s
+; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX12 %s
 
 define float @v_rsq_clamp_f32(float %src) #0 {
 ; SI-LABEL: v_rsq_clamp_f32:
@@ -16,6 +17,15 @@ define float @v_rsq_clamp_f32(float %src) #0 {
 ; VI-NEXT:    v_min_f32_e32 v0, 0x7f7fffff, v0
 ; VI-NEXT:    v_max_f32_e32 v0, 0xff7fffff, v0
 ; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: v_rsq_clamp_f32:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX12-NEXT:    v_rsq_f32_e32 v0, v0
+; GFX12-NEXT:    v_mov_b32_e32 v1, 0xff7fffff
+; GFX12-NEXT:    s_delay_alu instid0(TRANS32_DEP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    v_minmax_num_f32 v0, v0, 0x7f7fffff, v1
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %rsq_clamp = call float @llvm.amdgcn.rsq.clamp.f32(float %src)
   ret float %rsq_clamp
 }
@@ -34,6 +44,15 @@ define float @v_rsq_clamp_fabs_f32(float %src) #0 {
 ; VI-NEXT:    v_min_f32_e32 v0, 0x7f7fffff, v0
 ; VI-NEXT:    v_max_f32_e32 v0, 0xff7fffff, v0
 ; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: v_rsq_clamp_fabs_f32:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX12-NEXT:    v_rsq_f32_e64 v0, |v0|
+; GFX12-NEXT:    v_mov_b32_e32 v1, 0xff7fffff
+; GFX12-NEXT:    s_delay_alu instid0(TRANS32_DEP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    v_minmax_num_f32 v0, v0, 0x7f7fffff, v1
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %fabs.src = call float @llvm.fabs.f32(float %src)
   %rsq_clamp = call float @llvm.amdgcn.rsq.clamp.f32(float %fabs.src)
   ret float %rsq_clamp
@@ -56,6 +75,19 @@ define double @v_rsq_clamp_f64(double %src) #0 {
 ; VI-NEXT:    s_mov_b32 s5, 0xffefffff
 ; VI-NEXT:    v_max_f64 v[0:1], v[0:1], s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: v_rsq_clamp_f64:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX12-NEXT:    v_rsq_f64_e32 v[0:1], v[0:1]
+; GFX12-NEXT:    s_mov_b32 s0, -1
+; GFX12-NEXT:    s_mov_b32 s1, 0x7fefffff
+; GFX12-NEXT:    s_delay_alu instid0(TRANS32_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT:    v_min_num_f64_e32 v[0:1], s[0:1], v[0:1]
+; GFX12-NEXT:    s_mov_b32 s1, 0xffefffff
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT:    v_max_num_f64_e32 v[0:1], s[0:1], v[0:1]
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %rsq_clamp = call double @llvm.amdgcn.rsq.clamp.f64(double %src)
   ret double %rsq_clamp
 }
@@ -77,6 +109,19 @@ define double @v_rsq_clamp_fabs_f64(double %src) #0 {
 ; VI-NEXT:    s_mov_b32 s5, 0xffefffff
 ; VI-NEXT:    v_max_f64 v[0:1], v[0:1], s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: v_rsq_clamp_fabs_f64:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX12-NEXT:    v_rsq_f64_e64 v[0:1], |v[0:1]|
+; GFX12-NEXT:    s_mov_b32 s0, -1
+; GFX12-NEXT:    s_mov_b32 s1, 0x7fefffff
+; GFX12-NEXT:    s_delay_alu instid0(TRANS32_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT:    v_min_num_f64_e32 v[0:1], s[0:1], v[0:1]
+; GFX12-NEXT:    s_mov_b32 s1, 0xffefffff
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT:    v_max_num_f64_e32 v[0:1], s[0:1], v[0:1]
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %fabs.src = call double @llvm.fabs.f64(double %src)
   %rsq_clamp = call double @llvm.amdgcn.rsq.clamp.f64(double %fabs.src)
   ret double %rsq_clamp
@@ -96,6 +141,15 @@ define float @v_rsq_clamp_undef_f32() #0 {
 ; VI-NEXT:    v_min_f32_e32 v0, 0x7f7fffff, v0
 ; VI-NEXT:    v_max_f32_e32 v0, 0xff7fffff, v0
 ; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: v_rsq_clamp_undef_f32:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX12-NEXT:    v_rsq_f32_e32 v0, s0
+; GFX12-NEXT:    v_mov_b32_e32 v1, 0xff7fffff
+; GFX12-NEXT:    s_delay_alu instid0(TRANS32_DEP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    v_minmax_num_f32 v0, v0, 0x7f7fffff, v1
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %rsq_clamp = call float @llvm.amdgcn.rsq.clamp.f32(float undef)
   ret float %rsq_clamp
 }
@@ -117,6 +171,19 @@ define double @v_rsq_clamp_undef_f64() #0 {
 ; VI-NEXT:    s_mov_b32 s5, 0xffefffff
 ; VI-NEXT:    v_max_f64 v[0:1], v[0:1], s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: v_rsq_clamp_undef_f64:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX12-NEXT:    v_rsq_f64_e32 v[0:1], s[0:1]
+; GFX12-NEXT:    s_mov_b32 s0, -1
+; GFX12-NEXT:    s_mov_b32 s1, 0x7fefffff
+; GFX12-NEXT:    s_delay_alu instid0(TRANS32_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT:    v_min_num_f64_e32 v[0:1], s[0:1], v[0:1]
+; GFX12-NEXT:    s_mov_b32 s1, 0xffefffff
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT:    v_max_num_f64_e32 v[0:1], s[0:1], v[0:1]
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %rsq_clamp = call double @llvm.amdgcn.rsq.clamp.f64(double undef)
   ret double %rsq_clamp
 }
@@ -135,6 +202,15 @@ define float @v_rsq_clamp_f32_non_ieee(float %src) #2 {
 ; VI-NEXT:    v_min_f32_e32 v0, 0x7f7fffff, v0
 ; VI-NEXT:    v_max_f32_e32 v0, 0xff7fffff, v0
 ; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: v_rsq_clamp_f32_non_ieee:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX12-NEXT:    v_rsq_f32_e32 v0, v0
+; GFX12-NEXT:    v_mov_b32_e32 v1, 0xff7fffff
+; GFX12-NEXT:    s_delay_alu instid0(TRANS32_DEP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    v_minmax_num_f32 v0, v0, 0x7f7fffff, v1
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %rsq_clamp = call float @llvm.amdgcn.rsq.clamp.f32(float %src)
   ret float %rsq_clamp
 }
@@ -156,6 +232,19 @@ define double @v_rsq_clamp_f64_non_ieee(double %src) #2 {
 ; VI-NEXT:    s_mov_b32 s5, 0xffefffff
 ; VI-NEXT:    v_max_f64 v[0:1], v[0:1], s[4:5]
 ; VI-NEXT:    s_setpc_b64 s[30:31]
+;
+; GFX12-LABEL: v_rsq_clamp_f64_non_ieee:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX12-NEXT:    v_rsq_f64_e32 v[0:1], v[0:1]
+; GFX12-NEXT:    s_mov_b32 s0, -1
+; GFX12-NEXT:    s_mov_b32 s1, 0x7fefffff
+; GFX12-NEXT:    s_delay_alu instid0(TRANS32_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT:    v_min_num_f64_e32 v[0:1], s[0:1], v[0:1]
+; GFX12-NEXT:    s_mov_b32 s1, 0xffefffff
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT:    v_max_num_f64_e32 v[0:1], s[0:1], v[0:1]
+; GFX12-NEXT:    s_setpc_b64 s[30:31]
   %rsq_clamp = call double @llvm.amdgcn.rsq.clamp.f64(double %src)
   ret double %rsq_clamp
 }
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankcombiner-clamp-fmed3-const.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankcombiner-clamp-fmed3-const.mir
index 95f55a6d800ecee14d29ce76135cbb4bcc99e565..a97d905f2a978cdb405c53fb50abb954f90cdf4e 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankcombiner-clamp-fmed3-const.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/regbankcombiner-clamp-fmed3-const.mir
@@ -1,5 +1,6 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
 # RUN: llc -mtriple=amdgcn-amd-mesa3d -mcpu=gfx1010 -run-pass=amdgpu-regbank-combiner -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple=amdgcn-amd-mesa3d -mcpu=gfx1200 -run-pass=amdgpu-regbank-combiner -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=GFX12
 ---
 name: test_fmed3_f32_known_nnan_ieee_true
 legalized: true
@@ -22,6 +23,16 @@ body: |
     ; CHECK-NEXT: [[FMUL:%[0-9]+]]:vgpr(s32) = G_FMUL [[COPY]], [[COPY1]]
     ; CHECK-NEXT: [[AMDGPU_CLAMP:%[0-9]+]]:vgpr(s32) = nnan G_AMDGPU_CLAMP [[FMUL]]
     ; CHECK-NEXT: $vgpr0 = COPY [[AMDGPU_CLAMP]](s32)
+    ;
+    ; GFX12-LABEL: name: test_fmed3_f32_known_nnan_ieee_true
+    ; GFX12: liveins: $vgpr0
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; GFX12-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_FCONSTANT float 2.000000e+00
+    ; GFX12-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
+    ; GFX12-NEXT: [[FMUL:%[0-9]+]]:vgpr(s32) = G_FMUL [[COPY]], [[COPY1]]
+    ; GFX12-NEXT: [[AMDGPU_CLAMP:%[0-9]+]]:vgpr(s32) = nnan G_AMDGPU_CLAMP [[FMUL]]
+    ; GFX12-NEXT: $vgpr0 = COPY [[AMDGPU_CLAMP]](s32)
     %0:vgpr(s32) = COPY $vgpr0
     %2:sgpr(s32) = G_FCONSTANT float 2.000000e+00
     %8:vgpr(s32) = COPY %2(s32)
@@ -58,6 +69,18 @@ body: |
     ; CHECK-NEXT: [[AMDGPU_CLAMP:%[0-9]+]]:vgpr(s16) = nnan G_AMDGPU_CLAMP [[FMUL]]
     ; CHECK-NEXT: [[ANYEXT:%[0-9]+]]:vgpr(s32) = G_ANYEXT [[AMDGPU_CLAMP]](s16)
     ; CHECK-NEXT: $vgpr0 = COPY [[ANYEXT]](s32)
+    ;
+    ; GFX12-LABEL: name: test_fmed3_f16_known_nnan_ieee_false
+    ; GFX12: liveins: $vgpr0
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; GFX12-NEXT: [[TRUNC:%[0-9]+]]:vgpr(s16) = G_TRUNC [[COPY]](s32)
+    ; GFX12-NEXT: [[C:%[0-9]+]]:sgpr(s16) = G_FCONSTANT half 0xH4000
+    ; GFX12-NEXT: [[COPY1:%[0-9]+]]:vgpr(s16) = COPY [[C]](s16)
+    ; GFX12-NEXT: [[FMUL:%[0-9]+]]:vgpr(s16) = G_FMUL [[TRUNC]], [[COPY1]]
+    ; GFX12-NEXT: [[AMDGPU_CLAMP:%[0-9]+]]:vgpr(s16) = nnan G_AMDGPU_CLAMP [[FMUL]]
+    ; GFX12-NEXT: [[ANYEXT:%[0-9]+]]:vgpr(s32) = G_ANYEXT [[AMDGPU_CLAMP]](s16)
+    ; GFX12-NEXT: $vgpr0 = COPY [[ANYEXT]](s32)
     %2:vgpr(s32) = COPY $vgpr0
     %0:vgpr(s16) = G_TRUNC %2(s32)
     %3:sgpr(s16) = G_FCONSTANT half 0xH4000
@@ -95,6 +118,17 @@ body: |
     ; CHECK-NEXT: [[FMINNUM_IEEE:%[0-9]+]]:vgpr(s32) = G_FMINNUM_IEEE [[FCANONICALIZE]], [[COPY1]]
     ; CHECK-NEXT: [[AMDGPU_CLAMP:%[0-9]+]]:vgpr(s32) = G_AMDGPU_CLAMP [[FMINNUM_IEEE]]
     ; CHECK-NEXT: $vgpr0 = COPY [[AMDGPU_CLAMP]](s32)
+    ;
+    ; GFX12-LABEL: name: test_fmed3_non_SNaN_input_ieee_true_dx10clamp_true
+    ; GFX12: liveins: $vgpr0
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; GFX12-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_FCONSTANT float 1.000000e+01
+    ; GFX12-NEXT: [[FCANONICALIZE:%[0-9]+]]:vgpr(s32) = G_FCANONICALIZE [[COPY]]
+    ; GFX12-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
+    ; GFX12-NEXT: [[FMINNUM_IEEE:%[0-9]+]]:vgpr(s32) = G_FMINNUM_IEEE [[FCANONICALIZE]], [[COPY1]]
+    ; GFX12-NEXT: [[AMDGPU_CLAMP:%[0-9]+]]:vgpr(s32) = G_AMDGPU_CLAMP [[FMINNUM_IEEE]]
+    ; GFX12-NEXT: $vgpr0 = COPY [[AMDGPU_CLAMP]](s32)
     %0:vgpr(s32) = COPY $vgpr0
     %2:sgpr(s32) = G_FCONSTANT float 1.000000e+01
     %8:vgpr(s32) = G_FCANONICALIZE %0
@@ -130,6 +164,16 @@ body: |
     ; CHECK-NEXT: [[FMUL:%[0-9]+]]:vgpr(s32) = G_FMUL [[COPY]], [[COPY1]]
     ; CHECK-NEXT: [[AMDGPU_CLAMP:%[0-9]+]]:vgpr(s32) = G_AMDGPU_CLAMP [[FMUL]]
     ; CHECK-NEXT: $vgpr0 = COPY [[AMDGPU_CLAMP]](s32)
+    ;
+    ; GFX12-LABEL: name: test_fmed3_maybe_SNaN_input_zero_third_operand_ieee_true_dx10clamp_true
+    ; GFX12: liveins: $vgpr0
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; GFX12-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_FCONSTANT float 2.000000e+00
+    ; GFX12-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
+    ; GFX12-NEXT: [[FMUL:%[0-9]+]]:vgpr(s32) = G_FMUL [[COPY]], [[COPY1]]
+    ; GFX12-NEXT: [[AMDGPU_CLAMP:%[0-9]+]]:vgpr(s32) = G_AMDGPU_CLAMP [[FMUL]]
+    ; GFX12-NEXT: $vgpr0 = COPY [[AMDGPU_CLAMP]](s32)
     %0:vgpr(s32) = COPY $vgpr0
     %2:sgpr(s32) = G_FCONSTANT float 2.000000e+00
     %8:vgpr(s32) = COPY %2(s32)
@@ -170,6 +214,16 @@ body: |
     ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
     ; CHECK-NEXT: [[AMDGPU_FMED3_:%[0-9]+]]:vgpr(s32) = G_AMDGPU_FMED3 [[FMUL]], [[COPY2]], [[COPY3]]
     ; CHECK-NEXT: $vgpr0 = COPY [[AMDGPU_FMED3_]](s32)
+    ;
+    ; GFX12-LABEL: name: test_fmed3_f32_maybe_NaN_ieee_false
+    ; GFX12: liveins: $vgpr0
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; GFX12-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_FCONSTANT float 2.000000e+00
+    ; GFX12-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
+    ; GFX12-NEXT: [[FMUL:%[0-9]+]]:vgpr(s32) = G_FMUL [[COPY]], [[COPY1]]
+    ; GFX12-NEXT: [[AMDGPU_CLAMP:%[0-9]+]]:vgpr(s32) = G_AMDGPU_CLAMP [[FMUL]]
+    ; GFX12-NEXT: $vgpr0 = COPY [[AMDGPU_CLAMP]](s32)
     %0:vgpr(s32) = COPY $vgpr0
     %2:sgpr(s32) = G_FCONSTANT float 2.000000e+00
     %8:vgpr(s32) = COPY %2(s32)
@@ -209,6 +263,17 @@ body: |
     ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr(s32) = COPY [[C1]](s32)
     ; CHECK-NEXT: [[AMDGPU_FMED3_:%[0-9]+]]:vgpr(s32) = G_AMDGPU_FMED3 [[FMINNUM_IEEE]], [[COPY2]], [[COPY3]]
     ; CHECK-NEXT: $vgpr0 = COPY [[AMDGPU_FMED3_]](s32)
+    ;
+    ; GFX12-LABEL: name: test_fmed3_non_SNaN_input_ieee_true_dx10clamp_false
+    ; GFX12: liveins: $vgpr0
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; GFX12-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_FCONSTANT float 1.000000e+01
+    ; GFX12-NEXT: [[FCANONICALIZE:%[0-9]+]]:vgpr(s32) = G_FCANONICALIZE [[COPY]]
+    ; GFX12-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
+    ; GFX12-NEXT: [[FMINNUM_IEEE:%[0-9]+]]:vgpr(s32) = G_FMINNUM_IEEE [[FCANONICALIZE]], [[COPY1]]
+    ; GFX12-NEXT: [[AMDGPU_CLAMP:%[0-9]+]]:vgpr(s32) = G_AMDGPU_CLAMP [[FMINNUM_IEEE]]
+    ; GFX12-NEXT: $vgpr0 = COPY [[AMDGPU_CLAMP]](s32)
     %0:vgpr(s32) = COPY $vgpr0
     %2:sgpr(s32) = G_FCONSTANT float 1.000000e+01
     %8:vgpr(s32) = G_FCANONICALIZE %0
@@ -244,6 +309,16 @@ body: |
     ; CHECK-NEXT: [[FMUL:%[0-9]+]]:vgpr(s32) = G_FMUL [[COPY]], [[COPY1]]
     ; CHECK-NEXT: [[AMDGPU_CLAMP:%[0-9]+]]:vgpr(s32) = G_AMDGPU_CLAMP [[FMUL]]
     ; CHECK-NEXT: $vgpr0 = COPY [[AMDGPU_CLAMP]](s32)
+    ;
+    ; GFX12-LABEL: name: test_fmed3_maybe_SNaN_input_ieee_true_dx10clamp_true
+    ; GFX12: liveins: $vgpr0
+    ; GFX12-NEXT: {{  $}}
+    ; GFX12-NEXT: [[COPY:%[0-9]+]]:vgpr(s32) = COPY $vgpr0
+    ; GFX12-NEXT: [[C:%[0-9]+]]:sgpr(s32) = G_FCONSTANT float 2.000000e+00
+    ; GFX12-NEXT: [[COPY1:%[0-9]+]]:vgpr(s32) = COPY [[C]](s32)
+    ; GFX12-NEXT: [[FMUL:%[0-9]+]]:vgpr(s32) = G_FMUL [[COPY]], [[COPY1]]
+    ; GFX12-NEXT: [[AMDGPU_CLAMP:%[0-9]+]]:vgpr(s32) = G_AMDGPU_CLAMP [[FMUL]]
+    ; GFX12-NEXT: $vgpr0 = COPY [[AMDGPU_CLAMP]](s32)
     %0:vgpr(s32) = COPY $vgpr0
     %2:sgpr(s32) = G_FCONSTANT float 2.000000e+00
     %8:vgpr(s32) = COPY %2(s32)
diff --git a/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll b/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll
index bf5843ea8047d5fe6c420df77fae370f748a4a46..60f61a67ccf0be656a878cdd58a396505a1c882e 100644
--- a/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll
+++ b/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll
@@ -590,35 +590,35 @@ define amdgpu_kernel void @introduced_copy_to_sgpr(i64 %arg, i32 %arg1, i32 %arg
 ; GFX908-NEXT:    v_readfirstlane_b32 s9, v3
 ; GFX908-NEXT:    s_add_u32 s5, s5, 1
 ; GFX908-NEXT:    s_addc_u32 s9, s9, 0
-; GFX908-NEXT:    s_mul_hi_u32 s21, s2, s5
-; GFX908-NEXT:    s_mul_i32 s22, s3, s5
-; GFX908-NEXT:    s_mul_i32 s20, s2, s5
-; GFX908-NEXT:    s_mul_i32 s5, s2, s9
-; GFX908-NEXT:    s_add_i32 s5, s21, s5
-; GFX908-NEXT:    s_add_i32 s5, s5, s22
+; GFX908-NEXT:    s_mul_hi_u32 s20, s2, s5
+; GFX908-NEXT:    s_mul_i32 s9, s2, s9
+; GFX908-NEXT:    s_mul_i32 s21, s3, s5
+; GFX908-NEXT:    s_add_i32 s9, s20, s9
+; GFX908-NEXT:    s_mul_i32 s5, s2, s5
+; GFX908-NEXT:    s_add_i32 s9, s9, s21
 ; GFX908-NEXT:    s_branch .LBB3_5
 ; GFX908-NEXT:  .LBB3_4: ; %bb58
 ; GFX908-NEXT:    ; in Loop: Header=BB3_5 Depth=2
 ; GFX908-NEXT:    v_add_co_u32_sdwa v2, vcc, v2, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
 ; GFX908-NEXT:    v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
 ; GFX908-NEXT:    s_add_u32 s18, s18, s14
-; GFX908-NEXT:    v_cmp_lt_i64_e64 s[24:25], -1, v[2:3]
+; GFX908-NEXT:    v_cmp_lt_i64_e64 s[22:23], -1, v[2:3]
 ; GFX908-NEXT:    s_addc_u32 s19, s19, s15
-; GFX908-NEXT:    s_mov_b64 s[22:23], 0
-; GFX908-NEXT:    s_andn2_b64 vcc, exec, s[24:25]
+; GFX908-NEXT:    s_mov_b64 s[20:21], 0
+; GFX908-NEXT:    s_andn2_b64 vcc, exec, s[22:23]
 ; GFX908-NEXT:    s_cbranch_vccz .LBB3_9
 ; GFX908-NEXT:  .LBB3_5: ; %bb16
 ; GFX908-NEXT:    ; Parent Loop BB3_2 Depth=1
 ; GFX908-NEXT:    ; => This Inner Loop Header: Depth=2
-; GFX908-NEXT:    s_add_u32 s22, s18, s20
-; GFX908-NEXT:    s_addc_u32 s23, s19, s5
-; GFX908-NEXT:    global_load_dword v21, v19, s[22:23] offset:-12 glc
+; GFX908-NEXT:    s_add_u32 s20, s18, s5
+; GFX908-NEXT:    s_addc_u32 s21, s19, s9
+; GFX908-NEXT:    global_load_dword v21, v19, s[20:21] offset:-12 glc
 ; GFX908-NEXT:    s_waitcnt vmcnt(0)
-; GFX908-NEXT:    global_load_dword v20, v19, s[22:23] offset:-8 glc
+; GFX908-NEXT:    global_load_dword v20, v19, s[20:21] offset:-8 glc
 ; GFX908-NEXT:    s_waitcnt vmcnt(0)
-; GFX908-NEXT:    global_load_dword v12, v19, s[22:23] offset:-4 glc
+; GFX908-NEXT:    global_load_dword v12, v19, s[20:21] offset:-4 glc
 ; GFX908-NEXT:    s_waitcnt vmcnt(0)
-; GFX908-NEXT:    global_load_dword v12, v19, s[22:23] glc
+; GFX908-NEXT:    global_load_dword v12, v19, s[20:21] glc
 ; GFX908-NEXT:    s_waitcnt vmcnt(0)
 ; GFX908-NEXT:    ds_read_b64 v[12:13], v19
 ; GFX908-NEXT:    ds_read_b64 v[14:15], v0
@@ -647,11 +647,11 @@ define amdgpu_kernel void @introduced_copy_to_sgpr(i64 %arg, i32 %arg1, i32 %arg
 ; GFX908-NEXT:    v_add_f32_e32 v7, v7, v15
 ; GFX908-NEXT:    v_add_f32_e32 v10, v10, v12
 ; GFX908-NEXT:    v_add_f32_e32 v11, v11, v13
-; GFX908-NEXT:    s_mov_b64 s[22:23], -1
+; GFX908-NEXT:    s_mov_b64 s[20:21], -1
 ; GFX908-NEXT:    s_branch .LBB3_4
 ; GFX908-NEXT:  .LBB3_7: ; in Loop: Header=BB3_5 Depth=2
-; GFX908-NEXT:    s_mov_b64 s[22:23], s[16:17]
-; GFX908-NEXT:    s_andn2_b64 vcc, exec, s[22:23]
+; GFX908-NEXT:    s_mov_b64 s[20:21], s[16:17]
+; GFX908-NEXT:    s_andn2_b64 vcc, exec, s[20:21]
 ; GFX908-NEXT:    s_cbranch_vccz .LBB3_4
 ; GFX908-NEXT:  ; %bb.8: ; in Loop: Header=BB3_2 Depth=1
 ; GFX908-NEXT:    ; implicit-def: $vgpr10_vgpr11
@@ -662,7 +662,7 @@ define amdgpu_kernel void @introduced_copy_to_sgpr(i64 %arg, i32 %arg1, i32 %arg
 ; GFX908-NEXT:    ; implicit-def: $sgpr18_sgpr19
 ; GFX908-NEXT:  .LBB3_9: ; %loop.exit.guard
 ; GFX908-NEXT:    ; in Loop: Header=BB3_2 Depth=1
-; GFX908-NEXT:    s_xor_b64 s[16:17], s[22:23], -1
+; GFX908-NEXT:    s_xor_b64 s[16:17], s[20:21], -1
 ; GFX908-NEXT:  .LBB3_10: ; %Flow19
 ; GFX908-NEXT:    ; in Loop: Header=BB3_2 Depth=1
 ; GFX908-NEXT:    s_mov_b64 s[0:1], -1
@@ -753,12 +753,12 @@ define amdgpu_kernel void @introduced_copy_to_sgpr(i64 %arg, i32 %arg1, i32 %arg
 ; GFX90A-NEXT:    v_readfirstlane_b32 s9, v5
 ; GFX90A-NEXT:    s_add_u32 s5, s5, 1
 ; GFX90A-NEXT:    s_addc_u32 s9, s9, 0
-; GFX90A-NEXT:    s_mul_hi_u32 s21, s2, s5
-; GFX90A-NEXT:    s_mul_i32 s22, s3, s5
-; GFX90A-NEXT:    s_mul_i32 s20, s2, s5
-; GFX90A-NEXT:    s_mul_i32 s5, s2, s9
-; GFX90A-NEXT:    s_add_i32 s5, s21, s5
-; GFX90A-NEXT:    s_add_i32 s5, s5, s22
+; GFX90A-NEXT:    s_mul_hi_u32 s20, s2, s5
+; GFX90A-NEXT:    s_mul_i32 s9, s2, s9
+; GFX90A-NEXT:    s_mul_i32 s21, s3, s5
+; GFX90A-NEXT:    s_add_i32 s9, s20, s9
+; GFX90A-NEXT:    s_mul_i32 s5, s2, s5
+; GFX90A-NEXT:    s_add_i32 s9, s9, s21
 ; GFX90A-NEXT:    s_branch .LBB3_5
 ; GFX90A-NEXT:  .LBB3_4: ; %bb58
 ; GFX90A-NEXT:    ; in Loop: Header=BB3_5 Depth=2
@@ -766,27 +766,27 @@ define amdgpu_kernel void @introduced_copy_to_sgpr(i64 %arg, i32 %arg1, i32 %arg
 ; GFX90A-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
 ; GFX90A-NEXT:    s_add_u32 s18, s18, s14
 ; GFX90A-NEXT:    s_addc_u32 s19, s19, s15
-; GFX90A-NEXT:    v_cmp_lt_i64_e64 s[24:25], -1, v[4:5]
-; GFX90A-NEXT:    s_mov_b64 s[22:23], 0
-; GFX90A-NEXT:    s_andn2_b64 vcc, exec, s[24:25]
+; GFX90A-NEXT:    v_cmp_lt_i64_e64 s[22:23], -1, v[4:5]
+; GFX90A-NEXT:    s_mov_b64 s[20:21], 0
+; GFX90A-NEXT:    s_andn2_b64 vcc, exec, s[22:23]
 ; GFX90A-NEXT:    s_cbranch_vccz .LBB3_9
 ; GFX90A-NEXT:  .LBB3_5: ; %bb16
 ; GFX90A-NEXT:    ; Parent Loop BB3_2 Depth=1
 ; GFX90A-NEXT:    ; => This Inner Loop Header: Depth=2
-; GFX90A-NEXT:    s_add_u32 s22, s18, s20
-; GFX90A-NEXT:    s_addc_u32 s23, s19, s5
-; GFX90A-NEXT:    global_load_dword v21, v19, s[22:23] offset:-12 glc
+; GFX90A-NEXT:    s_add_u32 s20, s18, s5
+; GFX90A-NEXT:    s_addc_u32 s21, s19, s9
+; GFX90A-NEXT:    global_load_dword v21, v19, s[20:21] offset:-12 glc
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
-; GFX90A-NEXT:    global_load_dword v20, v19, s[22:23] offset:-8 glc
+; GFX90A-NEXT:    global_load_dword v20, v19, s[20:21] offset:-8 glc
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
-; GFX90A-NEXT:    global_load_dword v14, v19, s[22:23] offset:-4 glc
+; GFX90A-NEXT:    global_load_dword v14, v19, s[20:21] offset:-4 glc
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
-; GFX90A-NEXT:    global_load_dword v14, v19, s[22:23] glc
+; GFX90A-NEXT:    global_load_dword v14, v19, s[20:21] glc
 ; GFX90A-NEXT:    s_waitcnt vmcnt(0)
 ; GFX90A-NEXT:    ds_read_b64 v[14:15], v19
 ; GFX90A-NEXT:    ds_read_b64 v[16:17], v0
 ; GFX90A-NEXT:    s_and_b64 vcc, exec, s[0:1]
-; GFX90A-NEXT:    ; kill: killed $sgpr22 killed $sgpr23
+; GFX90A-NEXT:    ; kill: killed $sgpr20 killed $sgpr21
 ; GFX90A-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX90A-NEXT:    s_cbranch_vccnz .LBB3_7
 ; GFX90A-NEXT:  ; %bb.6: ; %bb51
@@ -803,11 +803,11 @@ define amdgpu_kernel void @introduced_copy_to_sgpr(i64 %arg, i32 %arg1, i32 %arg
 ; GFX90A-NEXT:    v_pk_add_f32 v[10:11], v[10:11], v[26:27]
 ; GFX90A-NEXT:    v_pk_add_f32 v[8:9], v[8:9], v[16:17]
 ; GFX90A-NEXT:    v_pk_add_f32 v[12:13], v[12:13], v[14:15]
-; GFX90A-NEXT:    s_mov_b64 s[22:23], -1
+; GFX90A-NEXT:    s_mov_b64 s[20:21], -1
 ; GFX90A-NEXT:    s_branch .LBB3_4
 ; GFX90A-NEXT:  .LBB3_7: ; in Loop: Header=BB3_5 Depth=2
-; GFX90A-NEXT:    s_mov_b64 s[22:23], s[16:17]
-; GFX90A-NEXT:    s_andn2_b64 vcc, exec, s[22:23]
+; GFX90A-NEXT:    s_mov_b64 s[20:21], s[16:17]
+; GFX90A-NEXT:    s_andn2_b64 vcc, exec, s[20:21]
 ; GFX90A-NEXT:    s_cbranch_vccz .LBB3_4
 ; GFX90A-NEXT:  ; %bb.8: ; in Loop: Header=BB3_2 Depth=1
 ; GFX90A-NEXT:    ; implicit-def: $vgpr12_vgpr13
@@ -818,7 +818,7 @@ define amdgpu_kernel void @introduced_copy_to_sgpr(i64 %arg, i32 %arg1, i32 %arg
 ; GFX90A-NEXT:    ; implicit-def: $sgpr18_sgpr19
 ; GFX90A-NEXT:  .LBB3_9: ; %loop.exit.guard
 ; GFX90A-NEXT:    ; in Loop: Header=BB3_2 Depth=1
-; GFX90A-NEXT:    s_xor_b64 s[16:17], s[22:23], -1
+; GFX90A-NEXT:    s_xor_b64 s[16:17], s[20:21], -1
 ; GFX90A-NEXT:  .LBB3_10: ; %Flow19
 ; GFX90A-NEXT:    ; in Loop: Header=BB3_2 Depth=1
 ; GFX90A-NEXT:    s_mov_b64 s[0:1], -1
diff --git a/llvm/test/CodeGen/AMDGPU/amdpal-msgpack-ieee.ll b/llvm/test/CodeGen/AMDGPU/amdpal-msgpack-ieee.ll
index 4da9080e2c69b83385665f5dd8dc2ac4bd61d6b4..95d533544c3082104193701cb9b3d3db399ed625 100644
--- a/llvm/test/CodeGen/AMDGPU/amdpal-msgpack-ieee.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdpal-msgpack-ieee.ll
@@ -1,11 +1,13 @@
 ; RUN: llc -mtriple=amdgcn--amdpal -verify-machineinstrs < %s | FileCheck -check-prefix=SI %s
 ; RUN: llc -mtriple=amdgcn--amdpal -mcpu=tonga -verify-machineinstrs < %s | FileCheck -check-prefix=VI %s
 ; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX9 -enable-var-scope %s
+; RUN: llc -mtriple=amdgcn--amdpal -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck -check-prefix=GFX12 -enable-var-scope %s
 
 ; amdpal compute shader: check for 0x2e12 (COMPUTE_PGM_RSRC1) in pal metadata
 ; SI-DAG: 0x2e12 (COMPUTE_PGM_RSRC1): 0xaf0000{{$}}
 ; VI-DAG: 0x2e12 (COMPUTE_PGM_RSRC1): 0xaf02c0{{$}}
 ; GFX9-DAG: 0x2e12 (COMPUTE_PGM_RSRC1): 0xaf0000{{$}}
+; GFX12-DAG: 0x2e12 (COMPUTE_PGM_RSRC1): 0x600f0000{{$}}
 define amdgpu_cs half @cs_amdpal(half %arg0) #0 {
   %add = fadd half %arg0, 1.0
   ret half %add
@@ -15,6 +17,7 @@ define amdgpu_cs half @cs_amdpal(half %arg0) #0 {
 ; SI-DAG: 0x2cca (SPI_SHADER_PGM_RSRC1_ES): 0xaf0000{{$}}
 ; VI-DAG: 0x2cca (SPI_SHADER_PGM_RSRC1_ES): 0xaf02c0{{$}}
 ; GFX9-DAG: 0x2cca (SPI_SHADER_PGM_RSRC1_ES): 0xaf0000{{$}}
+; GFX12-DAG: 0x2cca (SPI_SHADER_PGM_RSRC1_ES): 0xf0000{{$}}
 define amdgpu_es half @es_amdpal(half %arg0) #0 {
   %add = fadd half %arg0, 1.0
   ret half %add
@@ -24,6 +27,7 @@ define amdgpu_es half @es_amdpal(half %arg0) #0 {
 ; SI-DAG: 0x2c8a (SPI_SHADER_PGM_RSRC1_GS): 0xaf0000{{$}}
 ; VI-DAG: 0x2c8a (SPI_SHADER_PGM_RSRC1_GS): 0xaf02c0{{$}}
 ; GFX9-DAG: 0x2c8a (SPI_SHADER_PGM_RSRC1_GS): 0xaf0000{{$}}
+; GFX12-DAG: 0x2c8a (SPI_SHADER_PGM_RSRC1_GS): 0xa0f0000{{$}}
 define amdgpu_gs half @gs_amdpal(half %arg0) #0 {
   %add = fadd half %arg0, 1.0
   ret half %add
@@ -33,6 +37,7 @@ define amdgpu_gs half @gs_amdpal(half %arg0) #0 {
 ; SI-DAG: 0x2d0a (SPI_SHADER_PGM_RSRC1_HS): 0xaf0000{{$}}
 ; VI-DAG: 0x2d0a (SPI_SHADER_PGM_RSRC1_HS): 0xaf02c0{{$}}
 ; GFX9-DAG: 0x2d0a (SPI_SHADER_PGM_RSRC1_HS): 0xaf0000{{$}}
+; GFX12-DAG: 0x2d0a (SPI_SHADER_PGM_RSRC1_HS): 0x50f0000{{$}}
 define amdgpu_hs half @hs_amdpal(half %arg0) #0 {
   %add = fadd half %arg0, 1.0
   ret half %add
@@ -42,6 +47,7 @@ define amdgpu_hs half @hs_amdpal(half %arg0) #0 {
 ; SI-DAG: 0x2d4a (SPI_SHADER_PGM_RSRC1_LS): 0xaf0000{{$}}
 ; VI-DAG: 0x2d4a (SPI_SHADER_PGM_RSRC1_LS): 0xaf02c0{{$}}
 ; GFX9-DAG: 0x2d4a (SPI_SHADER_PGM_RSRC1_LS): 0xaf0000{{$}}
+; GFX12-DAG: 0x2d4a (SPI_SHADER_PGM_RSRC1_LS): 0xf0000{{$}}
 define amdgpu_ls half @ls_amdpal(half %arg0) #0 {
   %add = fadd half %arg0, 1.0
   ret half %add
@@ -52,6 +58,7 @@ define amdgpu_ls half @ls_amdpal(half %arg0) #0 {
 ; SI-DAG:           0x2c0a (SPI_SHADER_PGM_RSRC1_PS): 0xaf0000{{$}}
 ; VI-DAG:           0x2c0a (SPI_SHADER_PGM_RSRC1_PS): 0xaf02c0{{$}}
 ; GFX9-DAG:         0x2c0a (SPI_SHADER_PGM_RSRC1_PS): 0xaf0000{{$}}
+; GFX12-DAG:        0x2c0a (SPI_SHADER_PGM_RSRC1_PS): 0x20f0000{{$}}
 define amdgpu_ps half @ps_amdpal(half %arg0) #0 {
   %add = fadd half %arg0, 1.0
   ret half %add
@@ -61,6 +68,7 @@ define amdgpu_ps half @ps_amdpal(half %arg0) #0 {
 ; SI-DAG: 0x2c4a (SPI_SHADER_PGM_RSRC1_VS): 0xaf0000{{$}}
 ; VI-DAG: 0x2c4a (SPI_SHADER_PGM_RSRC1_VS): 0xaf02c0{{$}}
 ; GFX9-DAG: 0x2c4a (SPI_SHADER_PGM_RSRC1_VS): 0xaf0000{{$}}
+; GFX12-DAG: 0x2c4a (SPI_SHADER_PGM_RSRC1_VS): 0x80f0000{{$}}
 define amdgpu_vs half @vs_amdpal(half %arg0) #0 {
   %add = fadd half %arg0, 1.0
   ret half %add
diff --git a/llvm/test/CodeGen/AMDGPU/clamp.ll b/llvm/test/CodeGen/AMDGPU/clamp.ll
index 20b248875487ecc2ae2413b9b5841dad9a686bdf..3c0b8f7712e19846390526e8477962a92c658f4e 100644
--- a/llvm/test/CodeGen/AMDGPU/clamp.ll
+++ b/llvm/test/CodeGen/AMDGPU/clamp.ll
@@ -3,6 +3,7 @@
 ; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX8 %s
 ; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9 %s
 ; RUN: llc -march=amdgcn -mcpu=gfx1100 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX11 %s
+; RUN: llc -march=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX12 %s
 
 define amdgpu_kernel void @v_clamp_f32(ptr addrspace(1) %out, ptr addrspace(1) %aptr) #0 {
 ; GFX6-LABEL: v_clamp_f32:
@@ -61,6 +62,19 @@ define amdgpu_kernel void @v_clamp_f32(ptr addrspace(1) %out, ptr addrspace(1) %
 ; GFX11-NEXT:    s_nop 0
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: v_clamp_f32:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX12-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX12-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX12-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX12-NEXT:    s_waitcnt vmcnt(0)
+; GFX12-NEXT:    v_max_num_f32_e64 v1, v1, v1 clamp
+; GFX12-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX12-NEXT:    s_nop 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep0 = getelementptr float, ptr addrspace(1) %aptr, i32 %tid
   %out.gep = getelementptr float, ptr addrspace(1) %out, i32 %tid
@@ -129,6 +143,19 @@ define amdgpu_kernel void @v_clamp_neg_f32(ptr addrspace(1) %out, ptr addrspace(
 ; GFX11-NEXT:    s_nop 0
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: v_clamp_neg_f32:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX12-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX12-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX12-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX12-NEXT:    s_waitcnt vmcnt(0)
+; GFX12-NEXT:    v_max_num_f32_e64 v1, -v1, -v1 clamp
+; GFX12-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX12-NEXT:    s_nop 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep0 = getelementptr float, ptr addrspace(1) %aptr, i32 %tid
   %out.gep = getelementptr float, ptr addrspace(1) %out, i32 %tid
@@ -198,6 +225,19 @@ define amdgpu_kernel void @v_clamp_negabs_f32(ptr addrspace(1) %out, ptr addrspa
 ; GFX11-NEXT:    s_nop 0
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: v_clamp_negabs_f32:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX12-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX12-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX12-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX12-NEXT:    s_waitcnt vmcnt(0)
+; GFX12-NEXT:    v_max_num_f32_e64 v1, -|v1|, -|v1| clamp
+; GFX12-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX12-NEXT:    s_nop 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep0 = getelementptr float, ptr addrspace(1) %aptr, i32 %tid
   %out.gep = getelementptr float, ptr addrspace(1) %out, i32 %tid
@@ -277,6 +317,21 @@ define amdgpu_kernel void @v_clamp_negzero_f32(ptr addrspace(1) %out, ptr addrsp
 ; GFX11-NEXT:    s_nop 0
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: v_clamp_negzero_f32:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX12-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX12-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX12-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX12-NEXT:    s_waitcnt vmcnt(0)
+; GFX12-NEXT:    v_add_f32_e32 v1, 0.5, v1
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT:    v_maxmin_num_f32 v1, v1, 0x80000000, 1.0
+; GFX12-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX12-NEXT:    s_nop 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep0 = getelementptr float, ptr addrspace(1) %aptr, i32 %tid
   %out.gep = getelementptr float, ptr addrspace(1) %out, i32 %tid
@@ -356,6 +411,21 @@ define amdgpu_kernel void @v_clamp_negzero_maybe_snan_f32(ptr addrspace(1) %out,
 ; GFX11-NEXT:    s_nop 0
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: v_clamp_negzero_maybe_snan_f32:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX12-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX12-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX12-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX12-NEXT:    s_waitcnt vmcnt(0)
+; GFX12-NEXT:    v_max_num_f32_e32 v1, v1, v1
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT:    v_maxmin_num_f32 v1, v1, 0x80000000, 1.0
+; GFX12-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX12-NEXT:    s_nop 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep0 = getelementptr float, ptr addrspace(1) %aptr, i32 %tid
   %out.gep = getelementptr float, ptr addrspace(1) %out, i32 %tid
@@ -442,6 +512,24 @@ define amdgpu_kernel void @v_clamp_multi_use_max_f32(ptr addrspace(1) %out, ptr
 ; GFX11-NEXT:    s_nop 0
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: v_clamp_multi_use_max_f32:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX12-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX12-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX12-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX12-NEXT:    s_waitcnt vmcnt(0)
+; GFX12-NEXT:    v_max_num_f32_e32 v1, v1, v1
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    v_max_num_f32_e32 v1, 0, v1
+; GFX12-NEXT:    v_min_num_f32_e32 v2, 1.0, v1
+; GFX12-NEXT:    global_store_b32 v0, v2, s[0:1]
+; GFX12-NEXT:    global_store_b32 v[0:1], v1, off th:TH_STORE_NT_RT
+; GFX12-NEXT:    s_waitcnt_vscnt null, 0x0
+; GFX12-NEXT:    s_nop 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep0 = getelementptr float, ptr addrspace(1) %aptr, i32 %tid
   %out.gep = getelementptr float, ptr addrspace(1) %out, i32 %tid
@@ -512,6 +600,19 @@ define amdgpu_kernel void @v_clamp_f16(ptr addrspace(1) %out, ptr addrspace(1) %
 ; GFX11-NEXT:    s_nop 0
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: v_clamp_f16:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX12-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
+; GFX12-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX12-NEXT:    global_load_u16 v1, v0, s[2:3]
+; GFX12-NEXT:    s_waitcnt vmcnt(0)
+; GFX12-NEXT:    v_max_num_f16_e64 v1, v1, v1 clamp
+; GFX12-NEXT:    global_store_b16 v0, v1, s[0:1]
+; GFX12-NEXT:    s_nop 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep0 = getelementptr half, ptr addrspace(1) %aptr, i32 %tid
   %out.gep = getelementptr half, ptr addrspace(1) %out, i32 %tid
@@ -581,6 +682,19 @@ define amdgpu_kernel void @v_clamp_neg_f16(ptr addrspace(1) %out, ptr addrspace(
 ; GFX11-NEXT:    s_nop 0
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: v_clamp_neg_f16:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX12-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
+; GFX12-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX12-NEXT:    global_load_u16 v1, v0, s[2:3]
+; GFX12-NEXT:    s_waitcnt vmcnt(0)
+; GFX12-NEXT:    v_max_num_f16_e64 v1, -v1, -v1 clamp
+; GFX12-NEXT:    global_store_b16 v0, v1, s[0:1]
+; GFX12-NEXT:    s_nop 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep0 = getelementptr half, ptr addrspace(1) %aptr, i32 %tid
   %out.gep = getelementptr half, ptr addrspace(1) %out, i32 %tid
@@ -651,6 +765,19 @@ define amdgpu_kernel void @v_clamp_negabs_f16(ptr addrspace(1) %out, ptr addrspa
 ; GFX11-NEXT:    s_nop 0
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: v_clamp_negabs_f16:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX12-NEXT:    v_lshlrev_b32_e32 v0, 1, v0
+; GFX12-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX12-NEXT:    global_load_u16 v1, v0, s[2:3]
+; GFX12-NEXT:    s_waitcnt vmcnt(0)
+; GFX12-NEXT:    v_max_num_f16_e64 v1, -|v1|, -|v1| clamp
+; GFX12-NEXT:    global_store_b16 v0, v1, s[0:1]
+; GFX12-NEXT:    s_nop 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep0 = getelementptr half, ptr addrspace(1) %aptr, i32 %tid
   %out.gep = getelementptr half, ptr addrspace(1) %out, i32 %tid
@@ -722,6 +849,19 @@ define amdgpu_kernel void @v_clamp_f64(ptr addrspace(1) %out, ptr addrspace(1) %
 ; GFX11-NEXT:    s_nop 0
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: v_clamp_f64:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX12-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; GFX12-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX12-NEXT:    global_load_b64 v[0:1], v2, s[2:3]
+; GFX12-NEXT:    s_waitcnt vmcnt(0)
+; GFX12-NEXT:    v_max_f64 v[0:1], v[0:1], v[0:1] clamp
+; GFX12-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
+; GFX12-NEXT:    s_nop 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep0 = getelementptr double, ptr addrspace(1) %aptr, i32 %tid
   %out.gep = getelementptr double, ptr addrspace(1) %out, i32 %tid
@@ -790,6 +930,19 @@ define amdgpu_kernel void @v_clamp_neg_f64(ptr addrspace(1) %out, ptr addrspace(
 ; GFX11-NEXT:    s_nop 0
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: v_clamp_neg_f64:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX12-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; GFX12-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX12-NEXT:    global_load_b64 v[0:1], v2, s[2:3]
+; GFX12-NEXT:    s_waitcnt vmcnt(0)
+; GFX12-NEXT:    v_max_f64 v[0:1], -v[0:1], -v[0:1] clamp
+; GFX12-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
+; GFX12-NEXT:    s_nop 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep0 = getelementptr double, ptr addrspace(1) %aptr, i32 %tid
   %out.gep = getelementptr double, ptr addrspace(1) %out, i32 %tid
@@ -859,6 +1012,19 @@ define amdgpu_kernel void @v_clamp_negabs_f64(ptr addrspace(1) %out, ptr addrspa
 ; GFX11-NEXT:    s_nop 0
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: v_clamp_negabs_f64:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX12-NEXT:    v_lshlrev_b32_e32 v2, 3, v0
+; GFX12-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX12-NEXT:    global_load_b64 v[0:1], v2, s[2:3]
+; GFX12-NEXT:    s_waitcnt vmcnt(0)
+; GFX12-NEXT:    v_max_f64 v[0:1], -|v[0:1]|, -|v[0:1]| clamp
+; GFX12-NEXT:    global_store_b64 v2, v[0:1], s[0:1]
+; GFX12-NEXT:    s_nop 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep0 = getelementptr double, ptr addrspace(1) %aptr, i32 %tid
   %out.gep = getelementptr double, ptr addrspace(1) %out, i32 %tid
@@ -933,6 +1099,19 @@ define amdgpu_kernel void @v_clamp_med3_aby_negzero_f32(ptr addrspace(1) %out, p
 ; GFX11-NEXT:    s_nop 0
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: v_clamp_med3_aby_negzero_f32:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX12-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX12-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX12-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX12-NEXT:    s_waitcnt vmcnt(0)
+; GFX12-NEXT:    v_med3_num_f32 v1, 0x80000000, 1.0, v1
+; GFX12-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX12-NEXT:    s_nop 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep0 = getelementptr float, ptr addrspace(1) %aptr, i32 %tid
   %out.gep = getelementptr float, ptr addrspace(1) %out, i32 %tid
@@ -999,6 +1178,19 @@ define amdgpu_kernel void @v_clamp_med3_aby_f32(ptr addrspace(1) %out, ptr addrs
 ; GFX11-NEXT:    s_nop 0
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: v_clamp_med3_aby_f32:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX12-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX12-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX12-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX12-NEXT:    s_waitcnt vmcnt(0)
+; GFX12-NEXT:    v_max_num_f32_e64 v1, v1, v1 clamp
+; GFX12-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX12-NEXT:    s_nop 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep0 = getelementptr float, ptr addrspace(1) %aptr, i32 %tid
   %out.gep = getelementptr float, ptr addrspace(1) %out, i32 %tid
@@ -1065,6 +1257,19 @@ define amdgpu_kernel void @v_clamp_med3_bay_f32(ptr addrspace(1) %out, ptr addrs
 ; GFX11-NEXT:    s_nop 0
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: v_clamp_med3_bay_f32:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX12-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX12-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX12-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX12-NEXT:    s_waitcnt vmcnt(0)
+; GFX12-NEXT:    v_max_num_f32_e64 v1, v1, v1 clamp
+; GFX12-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX12-NEXT:    s_nop 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep0 = getelementptr float, ptr addrspace(1) %aptr, i32 %tid
   %out.gep = getelementptr float, ptr addrspace(1) %out, i32 %tid
@@ -1131,6 +1336,19 @@ define amdgpu_kernel void @v_clamp_med3_yab_f32(ptr addrspace(1) %out, ptr addrs
 ; GFX11-NEXT:    s_nop 0
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: v_clamp_med3_yab_f32:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX12-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX12-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX12-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX12-NEXT:    s_waitcnt vmcnt(0)
+; GFX12-NEXT:    v_max_num_f32_e64 v1, v1, v1 clamp
+; GFX12-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX12-NEXT:    s_nop 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep0 = getelementptr float, ptr addrspace(1) %aptr, i32 %tid
   %out.gep = getelementptr float, ptr addrspace(1) %out, i32 %tid
@@ -1197,6 +1415,19 @@ define amdgpu_kernel void @v_clamp_med3_yba_f32(ptr addrspace(1) %out, ptr addrs
 ; GFX11-NEXT:    s_nop 0
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: v_clamp_med3_yba_f32:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX12-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX12-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX12-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX12-NEXT:    s_waitcnt vmcnt(0)
+; GFX12-NEXT:    v_max_num_f32_e64 v1, v1, v1 clamp
+; GFX12-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX12-NEXT:    s_nop 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep0 = getelementptr float, ptr addrspace(1) %aptr, i32 %tid
   %out.gep = getelementptr float, ptr addrspace(1) %out, i32 %tid
@@ -1263,6 +1494,19 @@ define amdgpu_kernel void @v_clamp_med3_ayb_f32(ptr addrspace(1) %out, ptr addrs
 ; GFX11-NEXT:    s_nop 0
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: v_clamp_med3_ayb_f32:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX12-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX12-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX12-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX12-NEXT:    s_waitcnt vmcnt(0)
+; GFX12-NEXT:    v_max_num_f32_e64 v1, v1, v1 clamp
+; GFX12-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX12-NEXT:    s_nop 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep0 = getelementptr float, ptr addrspace(1) %aptr, i32 %tid
   %out.gep = getelementptr float, ptr addrspace(1) %out, i32 %tid
@@ -1329,6 +1573,19 @@ define amdgpu_kernel void @v_clamp_med3_bya_f32(ptr addrspace(1) %out, ptr addrs
 ; GFX11-NEXT:    s_nop 0
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: v_clamp_med3_bya_f32:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX12-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX12-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX12-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX12-NEXT:    s_waitcnt vmcnt(0)
+; GFX12-NEXT:    v_max_num_f32_e64 v1, v1, v1 clamp
+; GFX12-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX12-NEXT:    s_nop 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep0 = getelementptr float, ptr addrspace(1) %aptr, i32 %tid
   %out.gep = getelementptr float, ptr addrspace(1) %out, i32 %tid
@@ -1381,6 +1638,16 @@ define amdgpu_kernel void @v_clamp_constants_to_one_f32(ptr addrspace(1) %out) #
 ; GFX11-NEXT:    s_nop 0
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: v_clamp_constants_to_one_f32:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GFX12-NEXT:    v_dual_mov_b32 v1, 1.0 :: v_dual_lshlrev_b32 v0, 2, v0
+; GFX12-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX12-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX12-NEXT:    s_nop 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %out.gep = getelementptr float, ptr addrspace(1) %out, i32 %tid
   %med = call float @llvm.amdgcn.fmed3.f32(float 0.0, float 1.0, float 4.0)
@@ -1430,6 +1697,16 @@ define amdgpu_kernel void @v_clamp_constants_to_zero_f32(ptr addrspace(1) %out)
 ; GFX11-NEXT:    s_nop 0
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: v_clamp_constants_to_zero_f32:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GFX12-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_lshlrev_b32 v0, 2, v0
+; GFX12-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX12-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX12-NEXT:    s_nop 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %out.gep = getelementptr float, ptr addrspace(1) %out, i32 %tid
   %med = call float @llvm.amdgcn.fmed3.f32(float 0.0, float 1.0, float -4.0)
@@ -1480,6 +1757,16 @@ define amdgpu_kernel void @v_clamp_constant_preserve_f32(ptr addrspace(1) %out)
 ; GFX11-NEXT:    s_nop 0
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: v_clamp_constant_preserve_f32:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GFX12-NEXT:    v_dual_mov_b32 v1, 0.5 :: v_dual_lshlrev_b32 v0, 2, v0
+; GFX12-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX12-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX12-NEXT:    s_nop 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %out.gep = getelementptr float, ptr addrspace(1) %out, i32 %tid
   %med = call float @llvm.amdgcn.fmed3.f32(float 0.0, float 1.0, float 0.5)
@@ -1530,6 +1817,16 @@ define amdgpu_kernel void @v_clamp_constant_preserve_denorm_f32(ptr addrspace(1)
 ; GFX11-NEXT:    s_nop 0
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: v_clamp_constant_preserve_denorm_f32:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GFX12-NEXT:    v_dual_mov_b32 v1, 0x7fffff :: v_dual_lshlrev_b32 v0, 2, v0
+; GFX12-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX12-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX12-NEXT:    s_nop 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %out.gep = getelementptr float, ptr addrspace(1) %out, i32 %tid
   %med = call float @llvm.amdgcn.fmed3.f32(float 0.0, float 1.0, float bitcast (i32 8388607 to float))
@@ -1579,6 +1876,16 @@ define amdgpu_kernel void @v_clamp_constant_qnan_f32(ptr addrspace(1) %out) #0 {
 ; GFX11-NEXT:    s_nop 0
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: v_clamp_constant_qnan_f32:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GFX12-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_lshlrev_b32 v0, 2, v0
+; GFX12-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX12-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX12-NEXT:    s_nop 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %out.gep = getelementptr float, ptr addrspace(1) %out, i32 %tid
   %med = call float @llvm.amdgcn.fmed3.f32(float 0.0, float 1.0, float 0x7FF8000000000000)
@@ -1628,6 +1935,16 @@ define amdgpu_kernel void @v_clamp_constant_snan_f32(ptr addrspace(1) %out) #0 {
 ; GFX11-NEXT:    s_nop 0
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: v_clamp_constant_snan_f32:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GFX12-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_lshlrev_b32 v0, 2, v0
+; GFX12-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX12-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX12-NEXT:    s_nop 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %out.gep = getelementptr float, ptr addrspace(1) %out, i32 %tid
   %med = call float @llvm.amdgcn.fmed3.f32(float 0.0, float 1.0, float bitcast (i32 2139095041 to float))
@@ -1701,6 +2018,19 @@ define amdgpu_kernel void @v_clamp_f32_no_dx10_clamp(ptr addrspace(1) %out, ptr
 ; GFX11-NEXT:    s_nop 0
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: v_clamp_f32_no_dx10_clamp:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX12-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX12-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX12-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX12-NEXT:    s_waitcnt vmcnt(0)
+; GFX12-NEXT:    v_add_f32_e64 v1, v1, 0.5 clamp
+; GFX12-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX12-NEXT:    s_nop 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep0 = getelementptr float, ptr addrspace(1) %aptr, i32 %tid
   %out.gep = getelementptr float, ptr addrspace(1) %out, i32 %tid
@@ -1770,6 +2100,19 @@ define amdgpu_kernel void @v_clamp_f32_snan_dx10clamp(ptr addrspace(1) %out, ptr
 ; GFX11-NEXT:    s_nop 0
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: v_clamp_f32_snan_dx10clamp:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX12-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX12-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX12-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX12-NEXT:    s_waitcnt vmcnt(0)
+; GFX12-NEXT:    v_add_f32_e64 v1, v1, 0.5 clamp
+; GFX12-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX12-NEXT:    s_nop 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep0 = getelementptr float, ptr addrspace(1) %aptr, i32 %tid
   %out.gep = getelementptr float, ptr addrspace(1) %out, i32 %tid
@@ -1844,6 +2187,19 @@ define amdgpu_kernel void @v_clamp_f32_snan_no_dx10clamp(ptr addrspace(1) %out,
 ; GFX11-NEXT:    s_nop 0
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: v_clamp_f32_snan_no_dx10clamp:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX12-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX12-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX12-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX12-NEXT:    s_waitcnt vmcnt(0)
+; GFX12-NEXT:    v_max_num_f32_e64 v1, v1, v1 clamp
+; GFX12-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX12-NEXT:    s_nop 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep0 = getelementptr float, ptr addrspace(1) %aptr, i32 %tid
   %out.gep = getelementptr float, ptr addrspace(1) %out, i32 %tid
@@ -1917,6 +2273,19 @@ define amdgpu_kernel void @v_clamp_f32_snan_no_dx10clamp_nnan_src(ptr addrspace(
 ; GFX11-NEXT:    s_nop 0
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: v_clamp_f32_snan_no_dx10clamp_nnan_src:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX12-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX12-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX12-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX12-NEXT:    s_waitcnt vmcnt(0)
+; GFX12-NEXT:    v_add_f32_e64 v1, v1, 1.0 clamp
+; GFX12-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX12-NEXT:    s_nop 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep0 = getelementptr float, ptr addrspace(1) %aptr, i32 %tid
   %out.gep = getelementptr float, ptr addrspace(1) %out, i32 %tid
@@ -1986,6 +2355,19 @@ define amdgpu_kernel void @v_clamp_med3_aby_f32_no_dx10_clamp(ptr addrspace(1) %
 ; GFX11-NEXT:    s_nop 0
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: v_clamp_med3_aby_f32_no_dx10_clamp:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX12-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX12-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX12-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX12-NEXT:    s_waitcnt vmcnt(0)
+; GFX12-NEXT:    v_max_num_f32_e64 v1, v1, v1 clamp
+; GFX12-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX12-NEXT:    s_nop 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep0 = getelementptr float, ptr addrspace(1) %aptr, i32 %tid
   %out.gep = getelementptr float, ptr addrspace(1) %out, i32 %tid
@@ -2052,6 +2434,19 @@ define amdgpu_kernel void @v_clamp_med3_bay_f32_no_dx10_clamp(ptr addrspace(1) %
 ; GFX11-NEXT:    s_nop 0
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: v_clamp_med3_bay_f32_no_dx10_clamp:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX12-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX12-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX12-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX12-NEXT:    s_waitcnt vmcnt(0)
+; GFX12-NEXT:    v_max_num_f32_e64 v1, v1, v1 clamp
+; GFX12-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX12-NEXT:    s_nop 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep0 = getelementptr float, ptr addrspace(1) %aptr, i32 %tid
   %out.gep = getelementptr float, ptr addrspace(1) %out, i32 %tid
@@ -2118,6 +2513,19 @@ define amdgpu_kernel void @v_clamp_med3_yab_f32_no_dx10_clamp(ptr addrspace(1) %
 ; GFX11-NEXT:    s_nop 0
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: v_clamp_med3_yab_f32_no_dx10_clamp:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX12-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX12-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX12-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX12-NEXT:    s_waitcnt vmcnt(0)
+; GFX12-NEXT:    v_max_num_f32_e64 v1, v1, v1 clamp
+; GFX12-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX12-NEXT:    s_nop 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep0 = getelementptr float, ptr addrspace(1) %aptr, i32 %tid
   %out.gep = getelementptr float, ptr addrspace(1) %out, i32 %tid
@@ -2184,6 +2592,19 @@ define amdgpu_kernel void @v_clamp_med3_yba_f32_no_dx10_clamp(ptr addrspace(1) %
 ; GFX11-NEXT:    s_nop 0
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: v_clamp_med3_yba_f32_no_dx10_clamp:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX12-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX12-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX12-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX12-NEXT:    s_waitcnt vmcnt(0)
+; GFX12-NEXT:    v_max_num_f32_e64 v1, v1, v1 clamp
+; GFX12-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX12-NEXT:    s_nop 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep0 = getelementptr float, ptr addrspace(1) %aptr, i32 %tid
   %out.gep = getelementptr float, ptr addrspace(1) %out, i32 %tid
@@ -2250,6 +2671,19 @@ define amdgpu_kernel void @v_clamp_med3_ayb_f32_no_dx10_clamp(ptr addrspace(1) %
 ; GFX11-NEXT:    s_nop 0
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: v_clamp_med3_ayb_f32_no_dx10_clamp:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX12-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX12-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX12-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX12-NEXT:    s_waitcnt vmcnt(0)
+; GFX12-NEXT:    v_max_num_f32_e64 v1, v1, v1 clamp
+; GFX12-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX12-NEXT:    s_nop 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep0 = getelementptr float, ptr addrspace(1) %aptr, i32 %tid
   %out.gep = getelementptr float, ptr addrspace(1) %out, i32 %tid
@@ -2316,6 +2750,19 @@ define amdgpu_kernel void @v_clamp_med3_bya_f32_no_dx10_clamp(ptr addrspace(1) %
 ; GFX11-NEXT:    s_nop 0
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: v_clamp_med3_bya_f32_no_dx10_clamp:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX12-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX12-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX12-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX12-NEXT:    s_waitcnt vmcnt(0)
+; GFX12-NEXT:    v_max_num_f32_e64 v1, v1, v1 clamp
+; GFX12-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX12-NEXT:    s_nop 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep0 = getelementptr float, ptr addrspace(1) %aptr, i32 %tid
   %out.gep = getelementptr float, ptr addrspace(1) %out, i32 %tid
@@ -2368,6 +2815,16 @@ define amdgpu_kernel void @v_clamp_constant_qnan_f32_no_dx10_clamp(ptr addrspace
 ; GFX11-NEXT:    s_nop 0
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: v_clamp_constant_qnan_f32_no_dx10_clamp:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GFX12-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_lshlrev_b32 v0, 2, v0
+; GFX12-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX12-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX12-NEXT:    s_nop 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %out.gep = getelementptr float, ptr addrspace(1) %out, i32 %tid
   %med = call float @llvm.amdgcn.fmed3.f32(float 0.0, float 1.0, float 0x7FF8000000000000)
@@ -2418,6 +2875,16 @@ define amdgpu_kernel void @v_clamp_constant_snan_f32_no_dx10_clamp(ptr addrspace
 ; GFX11-NEXT:    s_nop 0
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: v_clamp_constant_snan_f32_no_dx10_clamp:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GFX12-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_lshlrev_b32 v0, 2, v0
+; GFX12-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX12-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX12-NEXT:    s_nop 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %out.gep = getelementptr float, ptr addrspace(1) %out, i32 %tid
   %med = call float @llvm.amdgcn.fmed3.f32(float 0.0, float 1.0, float bitcast (i32 2139095041 to float))
@@ -2490,6 +2957,19 @@ define amdgpu_kernel void @v_clamp_v2f16(ptr addrspace(1) %out, ptr addrspace(1)
 ; GFX11-NEXT:    s_nop 0
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: v_clamp_v2f16:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX12-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX12-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX12-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX12-NEXT:    s_waitcnt vmcnt(0)
+; GFX12-NEXT:    v_pk_max_num_f16 v1, v1, v1 clamp
+; GFX12-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX12-NEXT:    s_nop 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep0 = getelementptr <2 x half>, ptr addrspace(1) %aptr, i32 %tid
   %out.gep = getelementptr <2 x half>, ptr addrspace(1) %out, i32 %tid
@@ -2577,6 +3057,19 @@ define amdgpu_kernel void @v_clamp_v2f16_undef_elt(ptr addrspace(1) %out, ptr ad
 ; GFX11-NEXT:    s_nop 0
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: v_clamp_v2f16_undef_elt:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX12-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX12-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX12-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX12-NEXT:    s_waitcnt vmcnt(0)
+; GFX12-NEXT:    v_pk_max_num_f16 v1, v1, v1 clamp
+; GFX12-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX12-NEXT:    s_nop 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep0 = getelementptr <2 x half>, ptr addrspace(1) %aptr, i32 %tid
   %out.gep = getelementptr <2 x half>, ptr addrspace(1) %out, i32 %tid
@@ -2663,6 +3156,22 @@ define amdgpu_kernel void @v_clamp_v2f16_not_zero(ptr addrspace(1) %out, ptr add
 ; GFX11-NEXT:    s_nop 0
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: v_clamp_v2f16_not_zero:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX12-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX12-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX12-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX12-NEXT:    s_waitcnt vmcnt(0)
+; GFX12-NEXT:    v_pk_max_num_f16 v1, v1, v1
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    v_pk_max_num_f16 v1, v1, 2.0
+; GFX12-NEXT:    v_pk_min_num_f16 v1, v1, 1.0 op_sel_hi:[1,0]
+; GFX12-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX12-NEXT:    s_nop 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep0 = getelementptr <2 x half>, ptr addrspace(1) %aptr, i32 %tid
   %out.gep = getelementptr <2 x half>, ptr addrspace(1) %out, i32 %tid
@@ -2748,6 +3257,22 @@ define amdgpu_kernel void @v_clamp_v2f16_not_one(ptr addrspace(1) %out, ptr addr
 ; GFX11-NEXT:    s_nop 0
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: v_clamp_v2f16_not_one:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX12-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX12-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX12-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX12-NEXT:    s_waitcnt vmcnt(0)
+; GFX12-NEXT:    v_pk_max_num_f16 v1, v1, v1
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT:    v_pk_max_num_f16 v1, v1, 0
+; GFX12-NEXT:    v_pk_min_num_f16 v1, v1, 1.0 op_sel:[0,1] op_sel_hi:[1,0]
+; GFX12-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX12-NEXT:    s_nop 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep0 = getelementptr <2 x half>, ptr addrspace(1) %aptr, i32 %tid
   %out.gep = getelementptr <2 x half>, ptr addrspace(1) %out, i32 %tid
@@ -2825,6 +3350,19 @@ define amdgpu_kernel void @v_clamp_neg_v2f16(ptr addrspace(1) %out, ptr addrspac
 ; GFX11-NEXT:    s_nop 0
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: v_clamp_neg_v2f16:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX12-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX12-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX12-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX12-NEXT:    s_waitcnt vmcnt(0)
+; GFX12-NEXT:    v_pk_max_num_f16 v1, v1, v1 neg_lo:[1,1] neg_hi:[1,1] clamp
+; GFX12-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX12-NEXT:    s_nop 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep0 = getelementptr <2 x half>, ptr addrspace(1) %aptr, i32 %tid
   %out.gep = getelementptr <2 x half>, ptr addrspace(1) %out, i32 %tid
@@ -2906,6 +3444,21 @@ define amdgpu_kernel void @v_clamp_negabs_v2f16(ptr addrspace(1) %out, ptr addrs
 ; GFX11-NEXT:    s_nop 0
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: v_clamp_negabs_v2f16:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX12-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX12-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX12-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX12-NEXT:    s_waitcnt vmcnt(0)
+; GFX12-NEXT:    v_and_b32_e32 v1, 0x7fff7fff, v1
+; GFX12-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT:    v_pk_max_num_f16 v1, v1, v1 neg_lo:[1,1] neg_hi:[1,1] clamp
+; GFX12-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX12-NEXT:    s_nop 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep0 = getelementptr <2 x half>, ptr addrspace(1) %aptr, i32 %tid
   %out.gep = getelementptr <2 x half>, ptr addrspace(1) %out, i32 %tid
@@ -2986,6 +3539,19 @@ define amdgpu_kernel void @v_clamp_neglo_v2f16(ptr addrspace(1) %out, ptr addrsp
 ; GFX11-NEXT:    s_nop 0
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: v_clamp_neglo_v2f16:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX12-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX12-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX12-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX12-NEXT:    s_waitcnt vmcnt(0)
+; GFX12-NEXT:    v_pk_max_num_f16 v1, v1, v1 neg_lo:[1,1] clamp
+; GFX12-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX12-NEXT:    s_nop 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep0 = getelementptr <2 x half>, ptr addrspace(1) %aptr, i32 %tid
   %out.gep = getelementptr <2 x half>, ptr addrspace(1) %out, i32 %tid
@@ -3065,6 +3631,19 @@ define amdgpu_kernel void @v_clamp_neghi_v2f16(ptr addrspace(1) %out, ptr addrsp
 ; GFX11-NEXT:    s_nop 0
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: v_clamp_neghi_v2f16:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX12-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX12-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX12-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX12-NEXT:    s_waitcnt vmcnt(0)
+; GFX12-NEXT:    v_pk_max_num_f16 v1, v1, v1 neg_hi:[1,1] clamp
+; GFX12-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX12-NEXT:    s_nop 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep0 = getelementptr <2 x half>, ptr addrspace(1) %aptr, i32 %tid
   %out.gep = getelementptr <2 x half>, ptr addrspace(1) %out, i32 %tid
@@ -3144,6 +3723,19 @@ define amdgpu_kernel void @v_clamp_v2f16_shuffle(ptr addrspace(1) %out, ptr addr
 ; GFX11-NEXT:    s_nop 0
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: v_clamp_v2f16_shuffle:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX12-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX12-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX12-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX12-NEXT:    s_waitcnt vmcnt(0)
+; GFX12-NEXT:    v_pk_max_num_f16 v1, v1, v1 op_sel:[1,1] op_sel_hi:[0,0] clamp
+; GFX12-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX12-NEXT:    s_nop 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep0 = getelementptr <2 x half>, ptr addrspace(1) %aptr, i32 %tid
   %out.gep = getelementptr <2 x half>, ptr addrspace(1) %out, i32 %tid
@@ -3232,6 +3824,19 @@ define amdgpu_kernel void @v_clamp_v2f16_undef_limit_elts0(ptr addrspace(1) %out
 ; GFX11-NEXT:    s_nop 0
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: v_clamp_v2f16_undef_limit_elts0:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX12-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX12-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX12-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX12-NEXT:    s_waitcnt vmcnt(0)
+; GFX12-NEXT:    v_pk_max_num_f16 v1, v1, v1 clamp
+; GFX12-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX12-NEXT:    s_nop 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep0 = getelementptr <2 x half>, ptr addrspace(1) %aptr, i32 %tid
   %out.gep = getelementptr <2 x half>, ptr addrspace(1) %out, i32 %tid
@@ -3319,6 +3924,19 @@ define amdgpu_kernel void @v_clamp_v2f16_undef_limit_elts1(ptr addrspace(1) %out
 ; GFX11-NEXT:    s_nop 0
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: v_clamp_v2f16_undef_limit_elts1:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX12-NEXT:    v_lshlrev_b32_e32 v0, 2, v0
+; GFX12-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX12-NEXT:    global_load_b32 v1, v0, s[2:3]
+; GFX12-NEXT:    s_waitcnt vmcnt(0)
+; GFX12-NEXT:    v_pk_max_num_f16 v1, v1, v1 clamp
+; GFX12-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GFX12-NEXT:    s_nop 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %gep0 = getelementptr <2 x half>, ptr addrspace(1) %aptr, i32 %tid
   %out.gep = getelementptr <2 x half>, ptr addrspace(1) %out, i32 %tid
@@ -3400,6 +4018,25 @@ define amdgpu_kernel void @v_clamp_diff_source_f32(ptr addrspace(1) %out, ptr ad
 ; GFX11-NEXT:    s_nop 0
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
+;
+; GFX12-LABEL: v_clamp_diff_source_f32:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GFX12-NEXT:    v_mov_b32_e32 v0, 0
+; GFX12-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX12-NEXT:    s_clause 0x1
+; GFX12-NEXT:    s_load_b64 s[4:5], s[2:3], 0x0
+; GFX12-NEXT:    s_load_b32 s2, s[2:3], 0x8
+; GFX12-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX12-NEXT:    s_add_f32 s3, s4, s5
+; GFX12-NEXT:    s_add_f32 s2, s4, s2
+; GFX12-NEXT:    s_delay_alu instid0(SALU_CYCLE_3) | instskip(NEXT) | instid1(SALU_CYCLE_3)
+; GFX12-NEXT:    s_max_num_f32 s2, s3, s2
+; GFX12-NEXT:    v_max_num_f32_e64 v1, s2, s2 clamp
+; GFX12-NEXT:    global_store_b32 v0, v1, s[0:1] offset:12
+; GFX12-NEXT:    s_nop 0
+; GFX12-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX12-NEXT:    s_endpgm
 {
   %gep1 = getelementptr float, ptr addrspace(1) %aptr, i32 1
   %gep2 = getelementptr float, ptr addrspace(1) %aptr, i32 2
diff --git a/llvm/test/CodeGen/AMDGPU/coalesce-identity-copies-undef-subregs.mir b/llvm/test/CodeGen/AMDGPU/coalesce-identity-copies-undef-subregs.mir
index 263d68294cd058b323e3e92cad0a6fae0527ab9d..23590cad83271c0a282f7d6ae9f1213c8015131c 100644
--- a/llvm/test/CodeGen/AMDGPU/coalesce-identity-copies-undef-subregs.mir
+++ b/llvm/test/CodeGen/AMDGPU/coalesce-identity-copies-undef-subregs.mir
@@ -1,5 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx906 -verify-coalescing -verify-machineinstrs -start-before=register-coalescer -stop-after=machine-scheduler -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx906 -verify-coalescing -verify-machineinstrs -amdgpu-enable-rewrite-partial-reg-uses=false -start-before=register-coalescer -stop-after=machine-scheduler -o - %s | FileCheck %s
 
 # Tests that break due to the handling of partially undef registers
 # when whole register identity copies are erased.
diff --git a/llvm/test/CodeGen/AMDGPU/dead-lane.mir b/llvm/test/CodeGen/AMDGPU/dead-lane.mir
index a18e647ad485a45e9e8f4c024dd4c3e71daf3417..66944983c20c885e8c6bed031b075d412464344e 100644
--- a/llvm/test/CodeGen/AMDGPU/dead-lane.mir
+++ b/llvm/test/CodeGen/AMDGPU/dead-lane.mir
@@ -3,8 +3,8 @@
 
 # GCN-LABEL: name: dead_lane
 # GCN:      bb.0:
-# GCN-NEXT: undef %3.sub0:vreg_64 = nofpexcept V_MAC_F32_e32 undef %1:vgpr_32, undef %1:vgpr_32, undef %3.sub0, implicit $mode, implicit $exec
-# GCN-NEXT: FLAT_STORE_DWORD undef %4:vreg_64, %3.sub0,
+# GCN-NEXT: %5:vgpr_32 = nofpexcept V_MAC_F32_e32 undef %1:vgpr_32, undef %1:vgpr_32, undef %5, implicit $mode, implicit $exec
+# GCN-NEXT: FLAT_STORE_DWORD undef %4:vreg_64, %5,
 ---
 name:            dead_lane
 tracksRegLiveness: true
diff --git a/llvm/test/CodeGen/AMDGPU/fmaximum.ll b/llvm/test/CodeGen/AMDGPU/fmaximum.ll
new file mode 100644
index 0000000000000000000000000000000000000000..4fe2a4ad2a2fba1dde33a125919ee1a14ae12ab8
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/fmaximum.ll
@@ -0,0 +1,311 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -march=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX12-SDAG %s
+; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX12-GISEL %s
+
+define amdgpu_ps float @test_fmaximum_f32_vv(float %a, float %b) {
+; GCN-LABEL: test_fmaximum_f32_vv:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    v_maximum_f32 v0, v0, v1
+; GCN-NEXT:    ; return to shader part epilog
+  %val = call float @llvm.maximum.f32(float %a, float %b)
+  ret float %val
+}
+
+define amdgpu_ps float @test_fmaximum_f32_ss(float inreg %a, float inreg %b) {
+; GCN-LABEL: test_fmaximum_f32_ss:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_maximum_f32 s0, s0, s1
+; GCN-NEXT:    s_delay_alu instid0(SALU_CYCLE_3)
+; GCN-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-NEXT:    ; return to shader part epilog
+  %val = call float @llvm.maximum.f32(float %a, float %b)
+  ret float %val
+}
+
+define amdgpu_ps float @test_fmaximum_f32_vs(float %a, float inreg %b) {
+; GCN-LABEL: test_fmaximum_f32_vs:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    v_maximum_f32 v0, v0, s0
+; GCN-NEXT:    ; return to shader part epilog
+  %val = call float @llvm.maximum.f32(float %a, float %b)
+  ret float %val
+}
+
+define amdgpu_ps float @test_fmaximum_nnan_f32(float %a, float %b) {
+; GCN-LABEL: test_fmaximum_nnan_f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    v_maximum_f32 v0, v0, v1
+; GCN-NEXT:    ; return to shader part epilog
+  %val = call nnan float @llvm.maximum.f32(float %a, float %b)
+  ret float %val
+}
+
+define amdgpu_ps <2 x float> @test_fmaximum_v2f32(<2 x float> %a, <2 x float> %b) {
+; GCN-LABEL: test_fmaximum_v2f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    v_maximum_f32 v0, v0, v2
+; GCN-NEXT:    v_maximum_f32 v1, v1, v3
+; GCN-NEXT:    ; return to shader part epilog
+  %val = call <2 x float> @llvm.maximum.v2f32(<2 x float> %a, <2 x float> %b)
+  ret <2 x float> %val
+}
+
+define amdgpu_ps <2 x float> @test_fmaximum_v2f32_ss(<2 x float> inreg %a, <2 x float> inreg %b) {
+; GCN-LABEL: test_fmaximum_v2f32_ss:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_maximum_f32 s0, s0, s2
+; GCN-NEXT:    s_maximum_f32 s1, s1, s3
+; GCN-NEXT:    s_delay_alu instid0(SALU_CYCLE_3)
+; GCN-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GCN-NEXT:    ; return to shader part epilog
+  %val = call <2 x float> @llvm.maximum.v2f32(<2 x float> %a, <2 x float> %b)
+  ret <2 x float> %val
+}
+
+define amdgpu_ps <3 x float> @test_fmaximum_v3f32(<3 x float> %a, <3 x float> %b) {
+; GCN-LABEL: test_fmaximum_v3f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    v_maximum_f32 v0, v0, v3
+; GCN-NEXT:    v_maximum_f32 v1, v1, v4
+; GCN-NEXT:    v_maximum_f32 v2, v2, v5
+; GCN-NEXT:    ; return to shader part epilog
+  %val = call <3 x float> @llvm.maximum.v3f32(<3 x float> %a, <3 x float> %b)
+  ret <3 x float> %val
+}
+
+define amdgpu_ps <4 x float> @test_fmaximum_v4f32(<4 x float> %a, <4 x float> %b) {
+; GCN-LABEL: test_fmaximum_v4f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    v_maximum_f32 v0, v0, v4
+; GCN-NEXT:    v_maximum_f32 v1, v1, v5
+; GCN-NEXT:    v_maximum_f32 v2, v2, v6
+; GCN-NEXT:    v_maximum_f32 v3, v3, v7
+; GCN-NEXT:    ; return to shader part epilog
+  %val = call <4 x float> @llvm.maximum.v4f32(<4 x float> %a, <4 x float> %b)
+  ret <4 x float> %val
+}
+
+define amdgpu_ps <16 x float> @test_fmaximum_v16f32(<16 x float> %a, <16 x float> %b) {
+; GCN-LABEL: test_fmaximum_v16f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    v_maximum_f32 v0, v0, v16
+; GCN-NEXT:    v_maximum_f32 v1, v1, v17
+; GCN-NEXT:    v_maximum_f32 v2, v2, v18
+; GCN-NEXT:    v_maximum_f32 v3, v3, v19
+; GCN-NEXT:    v_maximum_f32 v4, v4, v20
+; GCN-NEXT:    v_maximum_f32 v5, v5, v21
+; GCN-NEXT:    v_maximum_f32 v6, v6, v22
+; GCN-NEXT:    v_maximum_f32 v7, v7, v23
+; GCN-NEXT:    v_maximum_f32 v8, v8, v24
+; GCN-NEXT:    v_maximum_f32 v9, v9, v25
+; GCN-NEXT:    v_maximum_f32 v10, v10, v26
+; GCN-NEXT:    v_maximum_f32 v11, v11, v27
+; GCN-NEXT:    v_maximum_f32 v12, v12, v28
+; GCN-NEXT:    v_maximum_f32 v13, v13, v29
+; GCN-NEXT:    v_maximum_f32 v14, v14, v30
+; GCN-NEXT:    v_maximum_f32 v15, v15, v31
+; GCN-NEXT:    ; return to shader part epilog
+  %val = call <16 x float> @llvm.maximum.v16f32(<16 x float> %a, <16 x float> %b)
+  ret <16 x float> %val
+}
+
+define amdgpu_ps half @test_fmaximum_f16_vv(half %a, half %b) {
+; GCN-LABEL: test_fmaximum_f16_vv:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    v_maximum_f16 v0, v0, v1
+; GCN-NEXT:    ; return to shader part epilog
+  %val = call half @llvm.maximum.f16(half %a, half %b)
+  ret half %val
+}
+
+define amdgpu_ps half @test_fmaximum_f16_ss(half inreg %a, half inreg %b) {
+; GCN-LABEL: test_fmaximum_f16_ss:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_maximum_f16 s0, s0, s1
+; GCN-NEXT:    s_delay_alu instid0(SALU_CYCLE_3)
+; GCN-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-NEXT:    ; return to shader part epilog
+  %val = call half @llvm.maximum.f16(half %a, half %b)
+  ret half %val
+}
+
+define amdgpu_ps <2 x half> @test_fmaximum_v2f16_vv(<2 x half> %a, <2 x half> %b) {
+; GCN-LABEL: test_fmaximum_v2f16_vv:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    v_pk_maximum_f16 v0, v0, v1
+; GCN-NEXT:    ; return to shader part epilog
+  %val = call <2 x half> @llvm.maximum.v2f16(<2 x half> %a, <2 x half> %b)
+  ret <2 x half> %val
+}
+
+define amdgpu_ps <2 x half> @test_fmaximum_v2f16_ss(<2 x half> inreg %a, <2 x half> inreg %b) {
+; GCN-LABEL: test_fmaximum_v2f16_ss:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    v_pk_maximum_f16 v0, s0, s1
+; GCN-NEXT:    ; return to shader part epilog
+  %val = call <2 x half> @llvm.maximum.v2f16(<2 x half> %a, <2 x half> %b)
+  ret <2 x half> %val
+}
+
+define amdgpu_ps <3 x half> @test_fmaximum_v3f16_vv(<3 x half> %a, <3 x half> %b) {
+; GCN-LABEL: test_fmaximum_v3f16_vv:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    v_pk_maximum_f16 v0, v0, v2
+; GCN-NEXT:    v_maximum_f16 v1, v1, v3
+; GCN-NEXT:    ; return to shader part epilog
+  %val = call <3 x half> @llvm.maximum.v3f16(<3 x half> %a, <3 x half> %b)
+  ret <3 x half> %val
+}
+
+define amdgpu_ps <3 x half> @test_fmaximum_v3f16_ss(<3 x half> inreg %a, <3 x half> inreg %b) {
+; GCN-LABEL: test_fmaximum_v3f16_ss:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    v_pk_maximum_f16 v0, s0, s2
+; GCN-NEXT:    s_maximum_f16 s0, s1, s3
+; GCN-NEXT:    s_delay_alu instid0(SALU_CYCLE_3)
+; GCN-NEXT:    v_mov_b32_e32 v1, s0
+; GCN-NEXT:    ; return to shader part epilog
+  %val = call <3 x half> @llvm.maximum.v3f16(<3 x half> %a, <3 x half> %b)
+  ret <3 x half> %val
+}
+
+define amdgpu_ps <4 x half> @test_fmaximum_v4f16(<4 x half> %a, <4 x half> %b) {
+; GCN-LABEL: test_fmaximum_v4f16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    v_pk_maximum_f16 v0, v0, v2
+; GCN-NEXT:    v_pk_maximum_f16 v1, v1, v3
+; GCN-NEXT:    ; return to shader part epilog
+  %val = call <4 x half> @llvm.maximum.v4f16(<4 x half> %a, <4 x half> %b)
+  ret <4 x half> %val
+}
+
+define amdgpu_ps <4 x half> @test_fmaximum_v4f16_ss(<4 x half> inreg %a, <4 x half> inreg %b) {
+; GCN-LABEL: test_fmaximum_v4f16_ss:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    v_pk_maximum_f16 v0, s0, s2
+; GCN-NEXT:    v_pk_maximum_f16 v1, s1, s3
+; GCN-NEXT:    ; return to shader part epilog
+  %val = call <4 x half> @llvm.maximum.v4f16(<4 x half> %a, <4 x half> %b)
+  ret <4 x half> %val
+}
+
+define amdgpu_ps <2 x float> @test_fmaximum_f64_vv(double %a, double %b) {
+; GCN-LABEL: test_fmaximum_f64_vv:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    v_maximum_f64 v[0:1], v[0:1], v[2:3]
+; GCN-NEXT:    ; return to shader part epilog
+  %val = call double @llvm.maximum.f64(double %a, double %b)
+  %ret = bitcast double %val to <2 x float>
+  ret <2 x float> %ret
+}
+
+define amdgpu_ps <2 x float> @test_fmaximum_f64_ss(double inreg %a, double inreg %b) {
+; GCN-LABEL: test_fmaximum_f64_ss:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    v_maximum_f64 v[0:1], s[0:1], s[2:3]
+; GCN-NEXT:    ; return to shader part epilog
+  %val = call double @llvm.maximum.f64(double %a, double %b)
+  %ret = bitcast double %val to <2 x float>
+  ret <2 x float> %ret
+}
+
+define amdgpu_ps <4 x float> @test_fmaximum_v2f64_ss(<2 x double> inreg %a, <2 x double> inreg %b) {
+; GCN-LABEL: test_fmaximum_v2f64_ss:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    v_maximum_f64 v[0:1], s[0:1], s[4:5]
+; GCN-NEXT:    v_maximum_f64 v[2:3], s[2:3], s[6:7]
+; GCN-NEXT:    ; return to shader part epilog
+  %val = call <2 x double> @llvm.maximum.v2f64(<2 x double> %a, <2 x double> %b)
+  %ret = bitcast <2 x double> %val to <4 x float>
+  ret <4 x float> %ret
+}
+
+define amdgpu_ps <8 x float> @test_fmaximum_v4f64(<4 x double> %a, <4 x double> %b) {
+; GCN-LABEL: test_fmaximum_v4f64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    v_maximum_f64 v[0:1], v[0:1], v[8:9]
+; GCN-NEXT:    v_maximum_f64 v[2:3], v[2:3], v[10:11]
+; GCN-NEXT:    v_maximum_f64 v[4:5], v[4:5], v[12:13]
+; GCN-NEXT:    v_maximum_f64 v[6:7], v[6:7], v[14:15]
+; GCN-NEXT:    ; return to shader part epilog
+  %val = call <4 x double> @llvm.maximum.v4f64(<4 x double> %a, <4 x double> %b)
+  %ret = bitcast <4 x double> %val to <8 x float>
+  ret <8 x float> %ret
+}
+
+define amdgpu_ps <8 x float> @test_fmaximum_v4f64_ss(<4 x double> inreg %a, <4 x double> inreg %b) {
+; GCN-LABEL: test_fmaximum_v4f64_ss:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    v_maximum_f64 v[0:1], s[0:1], s[8:9]
+; GCN-NEXT:    v_maximum_f64 v[2:3], s[2:3], s[10:11]
+; GCN-NEXT:    v_maximum_f64 v[4:5], s[4:5], s[12:13]
+; GCN-NEXT:    v_maximum_f64 v[6:7], s[6:7], s[14:15]
+; GCN-NEXT:    ; return to shader part epilog
+  %val = call <4 x double> @llvm.maximum.v4f64(<4 x double> %a, <4 x double> %b)
+  %ret = bitcast <4 x double> %val to <8 x float>
+  ret <8 x float> %ret
+}
+
+define amdgpu_kernel void @fmaximumi_f32_move_to_valu(ptr addrspace(1) %out, ptr addrspace(1) %aptr, ptr addrspace(1) %bptr) {
+; GCN-LABEL: fmaximumi_f32_move_to_valu:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_clause 0x1
+; GCN-NEXT:    s_load_b128 s[4:7], s[0:1], 0x24
+; GCN-NEXT:    s_load_b64 s[0:1], s[0:1], 0x34
+; GCN-NEXT:    v_mov_b32_e32 v0, 0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    global_load_b32 v1, v0, s[6:7] th:TH_LOAD_RT_NT
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    global_load_b32 v2, v0, s[0:1] th:TH_LOAD_RT_NT
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_maximum_f32 v1, v1, v2
+; GCN-NEXT:    global_store_b32 v0, v1, s[4:5]
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GCN-NEXT:    s_endpgm
+  %a = load volatile float, ptr addrspace(1) %aptr, align 4
+  %b = load volatile float, ptr addrspace(1) %bptr, align 4
+  %v = call float @llvm.maximum.f32(float %a, float %b)
+  store float %v, ptr addrspace(1) %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @fmaximum_f16_move_to_valu(ptr addrspace(1) %out, ptr addrspace(1) %aptr, ptr addrspace(1) %bptr) {
+; GCN-LABEL: fmaximum_f16_move_to_valu:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_clause 0x1
+; GCN-NEXT:    s_load_b128 s[4:7], s[0:1], 0x24
+; GCN-NEXT:    s_load_b64 s[0:1], s[0:1], 0x34
+; GCN-NEXT:    v_mov_b32_e32 v0, 0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    global_load_u16 v1, v0, s[6:7] th:TH_LOAD_RT_NT
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    global_load_u16 v2, v0, s[0:1] th:TH_LOAD_RT_NT
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_maximum_f16 v1, v1, v2
+; GCN-NEXT:    global_store_b16 v0, v1, s[4:5]
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GCN-NEXT:    s_endpgm
+  %a = load volatile half, ptr addrspace(1) %aptr, align 4
+  %b = load volatile half, ptr addrspace(1) %bptr, align 4
+  %v = call half @llvm.maximum.f16(half %a, half %b)
+  store half %v, ptr addrspace(1) %out, align 4
+  ret void
+}
+
+declare float @llvm.maximum.f32(float, float)
+declare <2 x float> @llvm.maximum.v2f32(<2 x float>, <2 x float>)
+declare <3 x float> @llvm.maximum.v3f32(<3 x float>, <3 x float>)
+declare <4 x float> @llvm.maximum.v4f32(<4 x float>, <4 x float>)
+declare <16 x float> @llvm.maximum.v16f32(<16 x float>, <16 x float>)
+declare half @llvm.maximum.f16(half, half)
+declare <2 x half> @llvm.maximum.v2f16(<2 x half>, <2 x half>)
+declare <3 x half> @llvm.maximum.v3f16(<3 x half>, <3 x half>)
+declare <4 x half> @llvm.maximum.v4f16(<4 x half>, <4 x half>)
+declare double @llvm.maximum.f64(double, double)
+declare <2 x double> @llvm.maximum.v2f64(<2 x double>, <2 x double>)
+declare <4 x double> @llvm.maximum.v4f64(<4 x double>, <4 x double>)
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX12-GISEL: {{.*}}
+; GFX12-SDAG: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/fmaximum3.ll b/llvm/test/CodeGen/AMDGPU/fmaximum3.ll
new file mode 100644
index 0000000000000000000000000000000000000000..2d057e6feac410b292e79cf3e22aa8b36cccacac
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/fmaximum3.ll
@@ -0,0 +1,98 @@
+; RUN: llc -march=amdgcn -mcpu=gfx1200 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN %s
+
+; GCN-LABEL: {{^}}test_fmaximum3_olt_0_f32:
+; GCN: buffer_load_b32 [[REGC:v[0-9]+]]
+; GCN: buffer_load_b32 [[REGB:v[0-9]+]]
+; GCN: buffer_load_b32 [[REGA:v[0-9]+]]
+; GCN: v_maximum3_f32 [[RESULT:v[0-9]+]], [[REGC]], [[REGB]], [[REGA]]
+; GCN: buffer_store_b32 [[RESULT]],
+define amdgpu_kernel void @test_fmaximum3_olt_0_f32(ptr addrspace(1) %out, ptr addrspace(1) %aptr, ptr addrspace(1) %bptr, ptr addrspace(1) %cptr) {
+  %a = load volatile float, ptr addrspace(1) %aptr, align 4
+  %b = load volatile float, ptr addrspace(1) %bptr, align 4
+  %c = load volatile float, ptr addrspace(1) %cptr, align 4
+  %f0 = call float @llvm.maximum.f32(float %a, float %b)
+  %f1 = call float @llvm.maximum.f32(float %f0, float %c)
+  store float %f1, ptr addrspace(1) %out, align 4
+  ret void
+}
+
+; Commute operand of second fmaximum
+; GCN-LABEL: {{^}}test_fmaximum3_olt_1_f32:
+; GCN: buffer_load_b32 [[REGB:v[0-9]+]]
+; GCN: buffer_load_b32 [[REGA:v[0-9]+]]
+; GCN: buffer_load_b32 [[REGC:v[0-9]+]]
+; GCN: v_maximum3_f32 [[RESULT:v[0-9]+]], [[REGC]], [[REGB]], [[REGA]]
+; GCN: buffer_store_b32 [[RESULT]],
+define amdgpu_kernel void @test_fmaximum3_olt_1_f32(ptr addrspace(1) %out, ptr addrspace(1) %aptr, ptr addrspace(1) %bptr, ptr addrspace(1) %cptr) {
+  %a = load volatile float, ptr addrspace(1) %aptr, align 4
+  %b = load volatile float, ptr addrspace(1) %bptr, align 4
+  %c = load volatile float, ptr addrspace(1) %cptr, align 4
+  %f0 = call float @llvm.maximum.f32(float %a, float %b)
+  %f1 = call float @llvm.maximum.f32(float %c, float %f0)
+  store float %f1, ptr addrspace(1) %out, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_fmaximum3_olt_0_f16:
+; GCN: buffer_load_u16 [[REGC:v[0-9]+]]
+; GCN: buffer_load_u16 [[REGB:v[0-9]+]]
+; GCN: buffer_load_u16 [[REGA:v[0-9]+]]
+; GCN: v_maximum3_f16 [[RESULT:v[0-9]+]], [[REGC]], [[REGB]], [[REGA]]
+; GCN: buffer_store_b16 [[RESULT]],
+define amdgpu_kernel void @test_fmaximum3_olt_0_f16(ptr addrspace(1) %out, ptr addrspace(1) %aptr, ptr addrspace(1) %bptr, ptr addrspace(1) %cptr) {
+  %a = load volatile half, ptr addrspace(1) %aptr, align 2
+  %b = load volatile half, ptr addrspace(1) %bptr, align 2
+  %c = load volatile half, ptr addrspace(1) %cptr, align 2
+  %f0 = call half @llvm.maximum.f16(half %a, half %b)
+  %f1 = call half @llvm.maximum.f16(half %f0, half %c)
+  store half %f1, ptr addrspace(1) %out, align 2
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_fmaximum3_olt_1_f16:
+; GCN: buffer_load_u16 [[REGA:v[0-9]+]]
+; GCN: buffer_load_u16 [[REGB:v[0-9]+]]
+; GCN: buffer_load_u16 [[REGC:v[0-9]+]]
+; GCN: v_maximum3_f16 [[RESULT:v[0-9]+]], [[REGC]], [[REGA]], [[REGB]]
+; GCN: buffer_store_b16 [[RESULT]],
+define amdgpu_kernel void @test_fmaximum3_olt_1_f16(ptr addrspace(1) %out, ptr addrspace(1) %aptr, ptr addrspace(1) %bptr, ptr addrspace(1) %cptr) {
+  %a = load volatile half, ptr addrspace(1) %aptr, align 2
+  %b = load volatile half, ptr addrspace(1) %bptr, align 2
+  %c = load volatile half, ptr addrspace(1) %cptr, align 2
+  %f0 = call half @llvm.maximum.f16(half %a, half %b)
+  %f1 = call half @llvm.maximum.f16(half %c, half %f0)
+  store half %f1, ptr addrspace(1) %out, align 2
+  ret void
+}
+
+; Checks whether the test passes; performMinMaxCombine() should not optimize vector patterns of maximum3
+; since there are no pack instructions for fmaximum3.
+; GCN-LABEL: {{^}}no_fmaximum3_v2f16:
+; GCN: v_pk_maximum_f16 v0, v0, v1
+; GCN: v_pk_maximum_f16 v0, v2, v0
+; GCN: v_pk_maximum_f16 v0, v0, v3
+; GCN-NEXT: s_setpc_b64
+define <2 x half> @no_fmaximum3_v2f16(<2 x half> %a, <2 x half> %b, <2 x half> %c, <2 x half> %d) {
+entry:
+  %max = call <2 x half> @llvm.maximum.v2f16(<2 x half> %a, <2 x half> %b)
+  %max1 = call <2 x half> @llvm.maximum.v2f16(<2 x half> %c, <2 x half> %max)
+  %res = call <2 x half> @llvm.maximum.v2f16(<2 x half> %max1, <2 x half> %d)
+  ret <2 x half> %res
+}
+
+; GCN-LABEL: {{^}}no_fmaximum3_olt_0_f64:
+; GCN-COUNT-2: v_maximum_f64
+define amdgpu_kernel void @no_fmaximum3_olt_0_f64(ptr addrspace(1) %out, ptr addrspace(1) %aptr, ptr addrspace(1) %bptr, ptr addrspace(1) %cptr) {
+  %a = load volatile double, ptr addrspace(1) %aptr, align 4
+  %b = load volatile double, ptr addrspace(1) %bptr, align 4
+  %c = load volatile double, ptr addrspace(1) %cptr, align 4
+  %f0 = call double @llvm.maximum.f64(double %a, double %b)
+  %f1 = call double @llvm.maximum.f64(double %f0, double %c)
+  store double %f1, ptr addrspace(1) %out, align 4
+  ret void
+}
+
+declare double @llvm.maximum.f64(double, double)
+declare float @llvm.maximum.f32(float, float)
+declare half @llvm.maximum.f16(half, half)
+declare <2 x half> @llvm.maximum.v2f16(<2 x half>, <2 x half>)
diff --git a/llvm/test/CodeGen/AMDGPU/fminimum.ll b/llvm/test/CodeGen/AMDGPU/fminimum.ll
new file mode 100644
index 0000000000000000000000000000000000000000..b63a4fa40b5913111bcddf2d2b52df223e58481d
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/fminimum.ll
@@ -0,0 +1,311 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -march=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX12-SDAG %s
+; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX12-GISEL %s
+
+define amdgpu_ps float @test_fminimum_f32_vv(float %a, float %b) {
+; GCN-LABEL: test_fminimum_f32_vv:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    v_minimum_f32 v0, v0, v1
+; GCN-NEXT:    ; return to shader part epilog
+  %val = call float @llvm.minimum.f32(float %a, float %b)
+  ret float %val
+}
+
+define amdgpu_ps float @test_fminimum_f32_ss(float inreg %a, float inreg %b) {
+; GCN-LABEL: test_fminimum_f32_ss:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_minimum_f32 s0, s0, s1
+; GCN-NEXT:    s_delay_alu instid0(SALU_CYCLE_3)
+; GCN-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-NEXT:    ; return to shader part epilog
+  %val = call float @llvm.minimum.f32(float %a, float %b)
+  ret float %val
+}
+
+define amdgpu_ps float @test_fminimum_f32_vs(float %a, float inreg %b) {
+; GCN-LABEL: test_fminimum_f32_vs:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    v_minimum_f32 v0, v0, s0
+; GCN-NEXT:    ; return to shader part epilog
+  %val = call float @llvm.minimum.f32(float %a, float %b)
+  ret float %val
+}
+
+define amdgpu_ps float @test_fminimum_nnan_f32(float %a, float %b) {
+; GCN-LABEL: test_fminimum_nnan_f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    v_minimum_f32 v0, v0, v1
+; GCN-NEXT:    ; return to shader part epilog
+  %val = call nnan float @llvm.minimum.f32(float %a, float %b)
+  ret float %val
+}
+
+define amdgpu_ps <2 x float> @test_fminimum_v2f32(<2 x float> %a, <2 x float> %b) {
+; GCN-LABEL: test_fminimum_v2f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    v_minimum_f32 v0, v0, v2
+; GCN-NEXT:    v_minimum_f32 v1, v1, v3
+; GCN-NEXT:    ; return to shader part epilog
+  %val = call <2 x float> @llvm.minimum.v2f32(<2 x float> %a, <2 x float> %b)
+  ret <2 x float> %val
+}
+
+define amdgpu_ps <2 x float> @test_fminimum_v2f32_ss(<2 x float> inreg %a, <2 x float> inreg %b) {
+; GCN-LABEL: test_fminimum_v2f32_ss:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_minimum_f32 s0, s0, s2
+; GCN-NEXT:    s_minimum_f32 s1, s1, s3
+; GCN-NEXT:    s_delay_alu instid0(SALU_CYCLE_3)
+; GCN-NEXT:    v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GCN-NEXT:    ; return to shader part epilog
+  %val = call <2 x float> @llvm.minimum.v2f32(<2 x float> %a, <2 x float> %b)
+  ret <2 x float> %val
+}
+
+define amdgpu_ps <3 x float> @test_fminimum_v3f32(<3 x float> %a, <3 x float> %b) {
+; GCN-LABEL: test_fminimum_v3f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    v_minimum_f32 v0, v0, v3
+; GCN-NEXT:    v_minimum_f32 v1, v1, v4
+; GCN-NEXT:    v_minimum_f32 v2, v2, v5
+; GCN-NEXT:    ; return to shader part epilog
+  %val = call <3 x float> @llvm.minimum.v3f32(<3 x float> %a, <3 x float> %b)
+  ret <3 x float> %val
+}
+
+define amdgpu_ps <4 x float> @test_fminimum_v4f32(<4 x float> %a, <4 x float> %b) {
+; GCN-LABEL: test_fminimum_v4f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    v_minimum_f32 v0, v0, v4
+; GCN-NEXT:    v_minimum_f32 v1, v1, v5
+; GCN-NEXT:    v_minimum_f32 v2, v2, v6
+; GCN-NEXT:    v_minimum_f32 v3, v3, v7
+; GCN-NEXT:    ; return to shader part epilog
+  %val = call <4 x float> @llvm.minimum.v4f32(<4 x float> %a, <4 x float> %b)
+  ret <4 x float> %val
+}
+
+define amdgpu_ps <16 x float> @test_fminimum_v16f32(<16 x float> %a, <16 x float> %b) {
+; GCN-LABEL: test_fminimum_v16f32:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    v_minimum_f32 v0, v0, v16
+; GCN-NEXT:    v_minimum_f32 v1, v1, v17
+; GCN-NEXT:    v_minimum_f32 v2, v2, v18
+; GCN-NEXT:    v_minimum_f32 v3, v3, v19
+; GCN-NEXT:    v_minimum_f32 v4, v4, v20
+; GCN-NEXT:    v_minimum_f32 v5, v5, v21
+; GCN-NEXT:    v_minimum_f32 v6, v6, v22
+; GCN-NEXT:    v_minimum_f32 v7, v7, v23
+; GCN-NEXT:    v_minimum_f32 v8, v8, v24
+; GCN-NEXT:    v_minimum_f32 v9, v9, v25
+; GCN-NEXT:    v_minimum_f32 v10, v10, v26
+; GCN-NEXT:    v_minimum_f32 v11, v11, v27
+; GCN-NEXT:    v_minimum_f32 v12, v12, v28
+; GCN-NEXT:    v_minimum_f32 v13, v13, v29
+; GCN-NEXT:    v_minimum_f32 v14, v14, v30
+; GCN-NEXT:    v_minimum_f32 v15, v15, v31
+; GCN-NEXT:    ; return to shader part epilog
+  %val = call <16 x float> @llvm.minimum.v16f32(<16 x float> %a, <16 x float> %b)
+  ret <16 x float> %val
+}
+
+define amdgpu_ps half @test_fminimum_f16_vv(half %a, half %b) {
+; GCN-LABEL: test_fminimum_f16_vv:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    v_minimum_f16 v0, v0, v1
+; GCN-NEXT:    ; return to shader part epilog
+  %val = call half @llvm.minimum.f16(half %a, half %b)
+  ret half %val
+}
+
+define amdgpu_ps half @test_fminimum_f16_ss(half inreg %a, half inreg %b) {
+; GCN-LABEL: test_fminimum_f16_ss:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_minimum_f16 s0, s0, s1
+; GCN-NEXT:    s_delay_alu instid0(SALU_CYCLE_3)
+; GCN-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-NEXT:    ; return to shader part epilog
+  %val = call half @llvm.minimum.f16(half %a, half %b)
+  ret half %val
+}
+
+define amdgpu_ps <2 x half> @test_fminimum_v2f16_vv(<2 x half> %a, <2 x half> %b) {
+; GCN-LABEL: test_fminimum_v2f16_vv:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    v_pk_minimum_f16 v0, v0, v1
+; GCN-NEXT:    ; return to shader part epilog
+  %val = call <2 x half> @llvm.minimum.v2f16(<2 x half> %a, <2 x half> %b)
+  ret <2 x half> %val
+}
+
+define amdgpu_ps <2 x half> @test_fminimum_v2f16_ss(<2 x half> inreg %a, <2 x half> inreg %b) {
+; GCN-LABEL: test_fminimum_v2f16_ss:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    v_pk_minimum_f16 v0, s0, s1
+; GCN-NEXT:    ; return to shader part epilog
+  %val = call <2 x half> @llvm.minimum.v2f16(<2 x half> %a, <2 x half> %b)
+  ret <2 x half> %val
+}
+
+define amdgpu_ps <3 x half> @test_fminimum_v3f16_vv(<3 x half> %a, <3 x half> %b) {
+; GCN-LABEL: test_fminimum_v3f16_vv:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    v_pk_minimum_f16 v0, v0, v2
+; GCN-NEXT:    v_minimum_f16 v1, v1, v3
+; GCN-NEXT:    ; return to shader part epilog
+  %val = call <3 x half> @llvm.minimum.v3f16(<3 x half> %a, <3 x half> %b)
+  ret <3 x half> %val
+}
+
+define amdgpu_ps <3 x half> @test_fminimum_v3f16_ss(<3 x half> inreg %a, <3 x half> inreg %b) {
+; GCN-LABEL: test_fminimum_v3f16_ss:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    v_pk_minimum_f16 v0, s0, s2
+; GCN-NEXT:    s_minimum_f16 s0, s1, s3
+; GCN-NEXT:    s_delay_alu instid0(SALU_CYCLE_3)
+; GCN-NEXT:    v_mov_b32_e32 v1, s0
+; GCN-NEXT:    ; return to shader part epilog
+  %val = call <3 x half> @llvm.minimum.v3f16(<3 x half> %a, <3 x half> %b)
+  ret <3 x half> %val
+}
+
+define amdgpu_ps <4 x half> @test_fminimum_v4f16(<4 x half> %a, <4 x half> %b) {
+; GCN-LABEL: test_fminimum_v4f16:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    v_pk_minimum_f16 v0, v0, v2
+; GCN-NEXT:    v_pk_minimum_f16 v1, v1, v3
+; GCN-NEXT:    ; return to shader part epilog
+  %val = call <4 x half> @llvm.minimum.v4f16(<4 x half> %a, <4 x half> %b)
+  ret <4 x half> %val
+}
+
+define amdgpu_ps <4 x half> @test_fminimum_v4f16_ss(<4 x half> inreg %a, <4 x half> inreg %b) {
+; GCN-LABEL: test_fminimum_v4f16_ss:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    v_pk_minimum_f16 v0, s0, s2
+; GCN-NEXT:    v_pk_minimum_f16 v1, s1, s3
+; GCN-NEXT:    ; return to shader part epilog
+  %val = call <4 x half> @llvm.minimum.v4f16(<4 x half> %a, <4 x half> %b)
+  ret <4 x half> %val
+}
+
+define amdgpu_ps <2 x float> @test_fminimum_f64_vv(double %a, double %b) {
+; GCN-LABEL: test_fminimum_f64_vv:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    v_minimum_f64 v[0:1], v[0:1], v[2:3]
+; GCN-NEXT:    ; return to shader part epilog
+  %val = call double @llvm.minimum.f64(double %a, double %b)
+  %ret = bitcast double %val to <2 x float>
+  ret <2 x float> %ret
+}
+
+define amdgpu_ps <2 x float> @test_fminimum_f64_ss(double inreg %a, double inreg %b) {
+; GCN-LABEL: test_fminimum_f64_ss:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    v_minimum_f64 v[0:1], s[0:1], s[2:3]
+; GCN-NEXT:    ; return to shader part epilog
+  %val = call double @llvm.minimum.f64(double %a, double %b)
+  %ret = bitcast double %val to <2 x float>
+  ret <2 x float> %ret
+}
+
+define amdgpu_ps <4 x float> @test_fminimum_v2f64_ss(<2 x double> inreg %a, <2 x double> inreg %b) {
+; GCN-LABEL: test_fminimum_v2f64_ss:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    v_minimum_f64 v[0:1], s[0:1], s[4:5]
+; GCN-NEXT:    v_minimum_f64 v[2:3], s[2:3], s[6:7]
+; GCN-NEXT:    ; return to shader part epilog
+  %val = call <2 x double> @llvm.minimum.v2f64(<2 x double> %a, <2 x double> %b)
+  %ret = bitcast <2 x double> %val to <4 x float>
+  ret <4 x float> %ret
+}
+
+define amdgpu_ps <8 x float> @test_fminimum_v4f64(<4 x double> %a, <4 x double> %b) {
+; GCN-LABEL: test_fminimum_v4f64:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    v_minimum_f64 v[0:1], v[0:1], v[8:9]
+; GCN-NEXT:    v_minimum_f64 v[2:3], v[2:3], v[10:11]
+; GCN-NEXT:    v_minimum_f64 v[4:5], v[4:5], v[12:13]
+; GCN-NEXT:    v_minimum_f64 v[6:7], v[6:7], v[14:15]
+; GCN-NEXT:    ; return to shader part epilog
+  %val = call <4 x double> @llvm.minimum.v4f64(<4 x double> %a, <4 x double> %b)
+  %ret = bitcast <4 x double> %val to <8 x float>
+  ret <8 x float> %ret
+}
+
+define amdgpu_ps <8 x float> @test_fminimum_v4f64_ss(<4 x double> inreg %a, <4 x double> inreg %b) {
+; GCN-LABEL: test_fminimum_v4f64_ss:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    v_minimum_f64 v[0:1], s[0:1], s[8:9]
+; GCN-NEXT:    v_minimum_f64 v[2:3], s[2:3], s[10:11]
+; GCN-NEXT:    v_minimum_f64 v[4:5], s[4:5], s[12:13]
+; GCN-NEXT:    v_minimum_f64 v[6:7], s[6:7], s[14:15]
+; GCN-NEXT:    ; return to shader part epilog
+  %val = call <4 x double> @llvm.minimum.v4f64(<4 x double> %a, <4 x double> %b)
+  %ret = bitcast <4 x double> %val to <8 x float>
+  ret <8 x float> %ret
+}
+
+define amdgpu_kernel void @fminimumi_f32_move_to_valu(ptr addrspace(1) %out, ptr addrspace(1) %aptr, ptr addrspace(1) %bptr) {
+; GCN-LABEL: fminimumi_f32_move_to_valu:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_clause 0x1
+; GCN-NEXT:    s_load_b128 s[4:7], s[0:1], 0x24
+; GCN-NEXT:    s_load_b64 s[0:1], s[0:1], 0x34
+; GCN-NEXT:    v_mov_b32_e32 v0, 0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    global_load_b32 v1, v0, s[6:7] th:TH_LOAD_RT_NT
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    global_load_b32 v2, v0, s[0:1] th:TH_LOAD_RT_NT
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_minimum_f32 v1, v1, v2
+; GCN-NEXT:    global_store_b32 v0, v1, s[4:5]
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GCN-NEXT:    s_endpgm
+  %a = load volatile float, ptr addrspace(1) %aptr, align 4
+  %b = load volatile float, ptr addrspace(1) %bptr, align 4
+  %v = call float @llvm.minimum.f32(float %a, float %b)
+  store float %v, ptr addrspace(1) %out, align 4
+  ret void
+}
+
+define amdgpu_kernel void @fminimum_f16_move_to_valu(ptr addrspace(1) %out, ptr addrspace(1) %aptr, ptr addrspace(1) %bptr) {
+; GCN-LABEL: fminimum_f16_move_to_valu:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_clause 0x1
+; GCN-NEXT:    s_load_b128 s[4:7], s[0:1], 0x24
+; GCN-NEXT:    s_load_b64 s[0:1], s[0:1], 0x34
+; GCN-NEXT:    v_mov_b32_e32 v0, 0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    global_load_u16 v1, v0, s[6:7] th:TH_LOAD_RT_NT
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    global_load_u16 v2, v0, s[0:1] th:TH_LOAD_RT_NT
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_minimum_f16 v1, v1, v2
+; GCN-NEXT:    global_store_b16 v0, v1, s[4:5]
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GCN-NEXT:    s_endpgm
+  %a = load volatile half, ptr addrspace(1) %aptr, align 4
+  %b = load volatile half, ptr addrspace(1) %bptr, align 4
+  %v = call half @llvm.minimum.f16(half %a, half %b)
+  store half %v, ptr addrspace(1) %out, align 4
+  ret void
+}
+
+declare float @llvm.minimum.f32(float, float)
+declare <2 x float> @llvm.minimum.v2f32(<2 x float>, <2 x float>)
+declare <3 x float> @llvm.minimum.v3f32(<3 x float>, <3 x float>)
+declare <4 x float> @llvm.minimum.v4f32(<4 x float>, <4 x float>)
+declare <16 x float> @llvm.minimum.v16f32(<16 x float>, <16 x float>)
+declare half @llvm.minimum.f16(half, half)
+declare <2 x half> @llvm.minimum.v2f16(<2 x half>, <2 x half>)
+declare <3 x half> @llvm.minimum.v3f16(<3 x half>, <3 x half>)
+declare <4 x half> @llvm.minimum.v4f16(<4 x half>, <4 x half>)
+declare double @llvm.minimum.f64(double, double)
+declare <2 x double> @llvm.minimum.v2f64(<2 x double>, <2 x double>)
+declare <4 x double> @llvm.minimum.v4f64(<4 x double>, <4 x double>)
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX12-GISEL: {{.*}}
+; GFX12-SDAG: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/fminimum3.ll b/llvm/test/CodeGen/AMDGPU/fminimum3.ll
new file mode 100644
index 0000000000000000000000000000000000000000..e28b32403215706260276150c33912d322a927bc
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/fminimum3.ll
@@ -0,0 +1,98 @@
+; RUN: llc -march=amdgcn -mcpu=gfx1200 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN %s
+
+; GCN-LABEL: {{^}}test_fminimum3_olt_0_f32:
+; GCN: buffer_load_b32 [[REGC:v[0-9]+]]
+; GCN: buffer_load_b32 [[REGB:v[0-9]+]]
+; GCN: buffer_load_b32 [[REGA:v[0-9]+]]
+; GCN: v_minimum3_f32 [[RESULT:v[0-9]+]], [[REGC]], [[REGB]], [[REGA]]
+; GCN: buffer_store_b32 [[RESULT]],
+define amdgpu_kernel void @test_fminimum3_olt_0_f32(ptr addrspace(1) %out, ptr addrspace(1) %aptr, ptr addrspace(1) %bptr, ptr addrspace(1) %cptr) {
+  %a = load volatile float, ptr addrspace(1) %aptr, align 4
+  %b = load volatile float, ptr addrspace(1) %bptr, align 4
+  %c = load volatile float, ptr addrspace(1) %cptr, align 4
+  %f0 = call float @llvm.minimum.f32(float %a, float %b)
+  %f1 = call float @llvm.minimum.f32(float %f0, float %c)
+  store float %f1, ptr addrspace(1) %out, align 4
+  ret void
+}
+
+; Commute operand of second fminimum
+; GCN-LABEL: {{^}}test_fminimum3_olt_1_f32:
+; GCN: buffer_load_b32 [[REGB:v[0-9]+]]
+; GCN: buffer_load_b32 [[REGA:v[0-9]+]]
+; GCN: buffer_load_b32 [[REGC:v[0-9]+]]
+; GCN: v_minimum3_f32 [[RESULT:v[0-9]+]], [[REGC]], [[REGB]], [[REGA]]
+; GCN: buffer_store_b32 [[RESULT]],
+define amdgpu_kernel void @test_fminimum3_olt_1_f32(ptr addrspace(1) %out, ptr addrspace(1) %aptr, ptr addrspace(1) %bptr, ptr addrspace(1) %cptr) {
+  %a = load volatile float, ptr addrspace(1) %aptr, align 4
+  %b = load volatile float, ptr addrspace(1) %bptr, align 4
+  %c = load volatile float, ptr addrspace(1) %cptr, align 4
+  %f0 = call float @llvm.minimum.f32(float %a, float %b)
+  %f1 = call float @llvm.minimum.f32(float %c, float %f0)
+  store float %f1, ptr addrspace(1) %out, align 4
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_fminimum3_olt_0_f16:
+; GCN: buffer_load_u16 [[REGC:v[0-9]+]]
+; GCN: buffer_load_u16 [[REGB:v[0-9]+]]
+; GCN: buffer_load_u16 [[REGA:v[0-9]+]]
+; GCN: v_minimum3_f16 [[RESULT:v[0-9]+]], [[REGC]], [[REGB]], [[REGA]]
+; GCN: buffer_store_b16 [[RESULT]],
+define amdgpu_kernel void @test_fminimum3_olt_0_f16(ptr addrspace(1) %out, ptr addrspace(1) %aptr, ptr addrspace(1) %bptr, ptr addrspace(1) %cptr) {
+  %a = load volatile half, ptr addrspace(1) %aptr, align 2
+  %b = load volatile half, ptr addrspace(1) %bptr, align 2
+  %c = load volatile half, ptr addrspace(1) %cptr, align 2
+  %f0 = call half @llvm.minimum.f16(half %a, half %b)
+  %f1 = call half @llvm.minimum.f16(half %f0, half %c)
+  store half %f1, ptr addrspace(1) %out, align 2
+  ret void
+}
+
+; GCN-LABEL: {{^}}test_fminimum3_olt_1_f16:
+; GCN: buffer_load_u16 [[REGA:v[0-9]+]]
+; GCN: buffer_load_u16 [[REGB:v[0-9]+]]
+; GCN: buffer_load_u16 [[REGC:v[0-9]+]]
+; GCN: v_minimum3_f16 [[RESULT:v[0-9]+]], [[REGC]], [[REGA]], [[REGB]]
+; GCN: buffer_store_b16 [[RESULT]],
+define amdgpu_kernel void @test_fminimum3_olt_1_f16(ptr addrspace(1) %out, ptr addrspace(1) %aptr, ptr addrspace(1) %bptr, ptr addrspace(1) %cptr) {
+  %a = load volatile half, ptr addrspace(1) %aptr, align 2
+  %b = load volatile half, ptr addrspace(1) %bptr, align 2
+  %c = load volatile half, ptr addrspace(1) %cptr, align 2
+  %f0 = call half @llvm.minimum.f16(half %a, half %b)
+  %f1 = call half @llvm.minimum.f16(half %c, half %f0)
+  store half %f1, ptr addrspace(1) %out, align 2
+  ret void
+}
+
+; Checks whether the test passes; performMinMaxCombine() should not optimize vector patterns of minimum3
+; since there are no pack instructions for fminimum3.
+; GCN-LABEL: {{^}}no_fminimum3_v2f16:
+; GCN: v_pk_minimum_f16 v0, v0, v1
+; GCN: v_pk_minimum_f16 v0, v2, v0
+; GCN: v_pk_minimum_f16 v0, v0, v3
+; GCN-NEXT: s_setpc_b64
+define <2 x half> @no_fminimum3_v2f16(<2 x half> %a, <2 x half> %b, <2 x half> %c, <2 x half> %d) {
+entry:
+  %min = call <2 x half> @llvm.minimum.v2f16(<2 x half> %a, <2 x half> %b)
+  %min1 = call <2 x half> @llvm.minimum.v2f16(<2 x half> %c, <2 x half> %min)
+  %res = call <2 x half> @llvm.minimum.v2f16(<2 x half> %min1, <2 x half> %d)
+  ret <2 x half> %res
+}
+
+; GCN-LABEL: {{^}}no_fminimum3_olt_0_f64:
+; GCN-COUNT-2: v_minimum_f64
+define amdgpu_kernel void @no_fminimum3_olt_0_f64(ptr addrspace(1) %out, ptr addrspace(1) %aptr, ptr addrspace(1) %bptr, ptr addrspace(1) %cptr) {
+  %a = load volatile double, ptr addrspace(1) %aptr, align 4
+  %b = load volatile double, ptr addrspace(1) %bptr, align 4
+  %c = load volatile double, ptr addrspace(1) %cptr, align 4
+  %f0 = call double @llvm.minimum.f64(double %a, double %b)
+  %f1 = call double @llvm.minimum.f64(double %f0, double %c)
+  store double %f1, ptr addrspace(1) %out, align 4
+  ret void
+}
+
+declare double @llvm.minimum.f64(double, double)
+declare float @llvm.minimum.f32(float, float)
+declare half @llvm.minimum.f16(half, half)
+declare <2 x half> @llvm.minimum.v2f16(<2 x half>, <2 x half>)
diff --git a/llvm/test/CodeGen/AMDGPU/fneg-combines-gfx1200.ll b/llvm/test/CodeGen/AMDGPU/fneg-combines-gfx1200.ll
new file mode 100644
index 0000000000000000000000000000000000000000..368503725f6840e0f662e4b74b7ed21c84db4abc
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/fneg-combines-gfx1200.ll
@@ -0,0 +1,270 @@
+; RUN: llc -march=amdgcn -mcpu=gfx1200 -start-before=amdgpu-unify-divergent-exit-nodes --verify-machineinstrs < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GCN-SDAG %s
+; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1200 -start-before=amdgpu-unify-divergent-exit-nodes --verify-machineinstrs < %s | FileCheck -enable-var-scope --check-prefixes=GCN,GCN-GISEL %s
+
+; --------------------------------------------------------------------------------
+; fminimum tests
+; --------------------------------------------------------------------------------
+
+; GCN-LABEL: {{^}}v_fneg_minimum_f32:
+; GCN: global_load_b32 [[A:v[0-9]+]]
+; GCN: global_load_b32 [[B:v[0-9]+]]
+; GCN: v_maximum_f32 [[RESULT:v[0-9]+]], -[[A]], -[[B]]
+; GCN: global_store_b32 v[{{[0-9:]+}}], [[RESULT]]
+define void @v_fneg_minimum_f32(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, ptr addrspace(1) %b.ptr) {
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %tid.ext = sext i32 %tid to i64
+  %a.gep = getelementptr inbounds float, ptr addrspace(1) %a.ptr, i64 %tid.ext
+  %b.gep = getelementptr inbounds float, ptr addrspace(1) %b.ptr, i64 %tid.ext
+  %out.gep = getelementptr inbounds float, ptr addrspace(1) %out, i64 %tid.ext
+  %a = load volatile float, ptr addrspace(1) %a.gep
+  %b = load volatile float, ptr addrspace(1) %b.gep
+  %min = call float @llvm.minimum.f32(float %a, float %b)
+  %fneg = fneg float %min
+  store float %fneg, ptr addrspace(1) %out.gep
+  ret void
+}
+
+; GCN-LABEL: {{^}}v_fneg_self_minimum_f32:
+; GCN: global_load_b32 [[A:v[0-9]+]]
+; GCN: v_maximum_f32 [[RESULT:v[0-9]+]], -[[A]], -[[A]]
+; GCN: global_store_b32 v[{{[0-9:]+}}], [[RESULT]]
+define void @v_fneg_self_minimum_f32(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr) {
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %tid.ext = sext i32 %tid to i64
+  %a.gep = getelementptr inbounds float, ptr addrspace(1) %a.ptr, i64 %tid.ext
+  %out.gep = getelementptr inbounds float, ptr addrspace(1) %out, i64 %tid.ext
+  %a = load volatile float, ptr addrspace(1) %a.gep
+  %min = call float @llvm.minimum.f32(float %a, float %a)
+  %min.fneg = fneg float %min
+  store float %min.fneg, ptr addrspace(1) %out.gep
+  ret void
+}
+
+; GCN-LABEL: {{^}}v_fneg_posk_minimum_f32:
+; GCN: global_load_b32 [[A:v[0-9]+]]
+; GCN: v_maximum_f32 [[RESULT:v[0-9]+]], -[[A]], -4.0
+; GCN: global_store_b32 v[{{[0-9:]+}}], [[RESULT]]
+define void @v_fneg_posk_minimum_f32(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr) {
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %tid.ext = sext i32 %tid to i64
+  %a.gep = getelementptr inbounds float, ptr addrspace(1) %a.ptr, i64 %tid.ext
+  %out.gep = getelementptr inbounds float, ptr addrspace(1) %out, i64 %tid.ext
+  %a = load volatile float, ptr addrspace(1) %a.gep
+  %min = call float @llvm.minimum.f32(float %a, float 4.0)
+  %fneg = fneg float %min
+  store float %fneg, ptr addrspace(1) %out.gep
+  ret void
+}
+
+; GCN-LABEL: {{^}}v_fneg_negk_minimum_f32:
+; GCN: global_load_b32 [[A:v[0-9]+]]
+; GCN: v_maximum_f32 [[RESULT:v[0-9]+]], -[[A]], 4.0
+; GCN: global_store_b32 v[{{[0-9:]+}}], [[RESULT]]
+define void @v_fneg_negk_minimum_f32(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr) {
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %tid.ext = sext i32 %tid to i64
+  %a.gep = getelementptr inbounds float, ptr addrspace(1) %a.ptr, i64 %tid.ext
+  %out.gep = getelementptr inbounds float, ptr addrspace(1) %out, i64 %tid.ext
+  %a = load volatile float, ptr addrspace(1) %a.gep
+  %min = call float @llvm.minimum.f32(float %a, float -4.0)
+  %fneg = fneg float %min
+  store float %fneg, ptr addrspace(1) %out.gep
+  ret void
+}
+
+; GCN-LABEL: {{^}}v_fneg_0_minimum_f32:
+; GCN: global_load_b32 [[A:v[0-9]+]]
+; GCN: v_minimum_f32 [[RESULT:v[0-9]+]], [[A]], 0
+; GCN: global_store_b32 v[{{[0-9:]+}}], [[RESULT]]
+define void @v_fneg_0_minimum_f32(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr) {
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %tid.ext = sext i32 %tid to i64
+  %a.gep = getelementptr inbounds float, ptr addrspace(1) %a.ptr, i64 %tid.ext
+  %out.gep = getelementptr inbounds float, ptr addrspace(1) %out, i64 %tid.ext
+  %a = load volatile float, ptr addrspace(1) %a.gep
+  %min = call float @llvm.minimum.f32(float %a, float 0.0)
+  %fneg = fneg float %min
+  store float %fneg, ptr addrspace(1) %out.gep
+  ret void
+}
+
+; GCN-LABEL: {{^}}v_fneg_0_minimum_foldable_use_f32:
+; GCN: global_load_b32 [[A:v[0-9]+]]
+; GCN: global_load_b32 [[B:v[0-9]+]]
+; GCN: v_minimum_f32 [[MIN:v[0-9]+]], [[A]], 0
+; GCN: v_mul_f32_e64 [[RESULT:v[0-9]+]], -[[MIN]], [[B]]
+; GCN: global_store_b32 v[{{[0-9:]+}}], [[RESULT]]
+define void @v_fneg_0_minimum_foldable_use_f32(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, ptr addrspace(1) %b.ptr) {
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %tid.ext = sext i32 %tid to i64
+  %a.gep = getelementptr inbounds float, ptr addrspace(1) %a.ptr, i64 %tid.ext
+  %b.gep = getelementptr inbounds float, ptr addrspace(1) %b.ptr, i64 %tid.ext
+  %out.gep = getelementptr inbounds float, ptr addrspace(1) %out, i64 %tid.ext
+  %a = load volatile float, ptr addrspace(1) %a.gep
+  %b = load volatile float, ptr addrspace(1) %b.gep
+  %min = call float @llvm.minimum.f32(float %a, float 0.0)
+  %fneg = fneg float %min
+  %mul = fmul float %fneg, %b
+  store float %mul, ptr addrspace(1) %out.gep
+  ret void
+}
+
+; GCN-LABEL: {{^}}v_fneg_minimum_multi_use_minimum_f32:
+; GCN: global_load_b32 [[A:v[0-9]+]]
+; GCN: global_load_b32 [[B:v[0-9]+]]
+; GCN: v_maximum_f32 [[MAX0:v[0-9]+]], -[[A]], -[[B]]
+; GCN-SDAG:  v_mul_f32_e32 [[MUL1:v[0-9]+]], -4.0, [[MAX0]]
+; GCN-GISEL: v_mul_f32_e64 [[MUL1:v[0-9]+]], -[[MAX0]], 4.0
+; GCN: global_store_b32 v[{{[0-9:]+}}], [[MAX0]]
+; GCN: global_store_b32 v[{{[0-9:]+}}], [[MUL1]]
+define void @v_fneg_minimum_multi_use_minimum_f32(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, ptr addrspace(1) %b.ptr) {
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %tid.ext = sext i32 %tid to i64
+  %a.gep = getelementptr inbounds float, ptr addrspace(1) %a.ptr, i64 %tid.ext
+  %b.gep = getelementptr inbounds float, ptr addrspace(1) %b.ptr, i64 %tid.ext
+  %out.gep = getelementptr inbounds float, ptr addrspace(1) %out, i64 %tid.ext
+  %a = load volatile float, ptr addrspace(1) %a.gep
+  %b = load volatile float, ptr addrspace(1) %b.gep
+  %min = call float @llvm.minimum.f32(float %a, float %b)
+  %fneg = fneg float %min
+  %use1 = fmul float %min, 4.0
+  store volatile float %fneg, ptr addrspace(1) %out
+  store volatile float %use1, ptr addrspace(1) %out
+  ret void
+}
+
+; --------------------------------------------------------------------------------
+; fmaximum tests
+; --------------------------------------------------------------------------------
+
+; GCN-LABEL: {{^}}v_fneg_maximum_f32:
+; GCN: global_load_b32 [[A:v[0-9]+]]
+; GCN: global_load_b32 [[B:v[0-9]+]]
+; GCN: v_minimum_f32 [[RESULT:v[0-9]+]], -[[A]], -[[B]]
+; GCN: global_store_b32 v[{{[0-9:]+}}], [[RESULT]]
+define void @v_fneg_maximum_f32(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, ptr addrspace(1) %b.ptr) {
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %tid.ext = sext i32 %tid to i64
+  %a.gep = getelementptr inbounds float, ptr addrspace(1) %a.ptr, i64 %tid.ext
+  %b.gep = getelementptr inbounds float, ptr addrspace(1) %b.ptr, i64 %tid.ext
+  %out.gep = getelementptr inbounds float, ptr addrspace(1) %out, i64 %tid.ext
+  %a = load volatile float, ptr addrspace(1) %a.gep
+  %b = load volatile float, ptr addrspace(1) %b.gep
+  %min = call float @llvm.maximum.f32(float %a, float %b)
+  %fneg = fneg float %min
+  store float %fneg, ptr addrspace(1) %out.gep
+  ret void
+}
+
+; GCN-LABEL: {{^}}v_fneg_self_maximum_f32:
+; GCN: global_load_b32 [[A:v[0-9]+]]
+; GCN: v_minimum_f32 [[RESULT:v[0-9]+]], -[[A]], -[[A]]
+; GCN: global_store_b32 v[{{[0-9:]+}}], [[RESULT]]
+define void @v_fneg_self_maximum_f32(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr) {
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %tid.ext = sext i32 %tid to i64
+  %a.gep = getelementptr inbounds float, ptr addrspace(1) %a.ptr, i64 %tid.ext
+  %out.gep = getelementptr inbounds float, ptr addrspace(1) %out, i64 %tid.ext
+  %a = load volatile float, ptr addrspace(1) %a.gep
+  %min = call float @llvm.maximum.f32(float %a, float %a)
+  %min.fneg = fneg float %min
+  store float %min.fneg, ptr addrspace(1) %out.gep
+  ret void
+}
+
+; GCN-LABEL: {{^}}v_fneg_posk_maximum_f32:
+; GCN: global_load_b32 [[A:v[0-9]+]]
+; GCN: v_minimum_f32 [[RESULT:v[0-9]+]], -[[A]], -4.0
+; GCN: global_store_b32 v[{{[0-9:]+}}], [[RESULT]]
+define void @v_fneg_posk_maximum_f32(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr) {
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %tid.ext = sext i32 %tid to i64
+  %a.gep = getelementptr inbounds float, ptr addrspace(1) %a.ptr, i64 %tid.ext
+  %out.gep = getelementptr inbounds float, ptr addrspace(1) %out, i64 %tid.ext
+  %a = load volatile float, ptr addrspace(1) %a.gep
+  %min = call float @llvm.maximum.f32(float %a, float 4.0)
+  %fneg = fneg float %min
+  store float %fneg, ptr addrspace(1) %out.gep
+  ret void
+}
+
+; GCN-LABEL: {{^}}v_fneg_negk_maximum_f32:
+; GCN: global_load_b32 [[A:v[0-9]+]]
+; GCN: v_minimum_f32 [[RESULT:v[0-9]+]], -[[A]], 4.0
+; GCN: global_store_b32 v[{{[0-9:]+}}], [[RESULT]]
+define void @v_fneg_negk_maximum_f32(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr) {
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %tid.ext = sext i32 %tid to i64
+  %a.gep = getelementptr inbounds float, ptr addrspace(1) %a.ptr, i64 %tid.ext
+  %out.gep = getelementptr inbounds float, ptr addrspace(1) %out, i64 %tid.ext
+  %a = load volatile float, ptr addrspace(1) %a.gep
+  %min = call float @llvm.maximum.f32(float %a, float -4.0)
+  %fneg = fneg float %min
+  store float %fneg, ptr addrspace(1) %out.gep
+  ret void
+}
+
+; GCN-LABEL: {{^}}v_fneg_0_maximum_f32:
+; GCN: global_load_b32 [[A:v[0-9]+]]
+; GCN: v_maximum_f32 [[RESULT:v[0-9]+]], [[A]], 0
+; GCN: global_store_b32 v[{{[0-9:]+}}], [[RESULT]]
+define void @v_fneg_0_maximum_f32(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr) {
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %tid.ext = sext i32 %tid to i64
+  %a.gep = getelementptr inbounds float, ptr addrspace(1) %a.ptr, i64 %tid.ext
+  %out.gep = getelementptr inbounds float, ptr addrspace(1) %out, i64 %tid.ext
+  %a = load volatile float, ptr addrspace(1) %a.gep
+  %max = call float @llvm.maximum.f32(float %a, float 0.0)
+  %fneg = fneg float %max
+  store float %fneg, ptr addrspace(1) %out.gep
+  ret void
+}
+
+; GCN-LABEL: {{^}}v_fneg_0_maximum_foldable_use_f32:
+; GCN: global_load_b32 [[A:v[0-9]+]]
+; GCN: global_load_b32 [[B:v[0-9]+]]
+; GCN: v_maximum_f32 [[MAX:v[0-9]+]], [[A]], 0
+; GCN: v_mul_f32_e64 [[RESULT:v[0-9]+]], -[[MAX]], [[B]]
+; GCN: global_store_b32 v[{{[0-9:]+}}], [[RESULT]]
+define void @v_fneg_0_maximum_foldable_use_f32(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, ptr addrspace(1) %b.ptr) {
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %tid.ext = sext i32 %tid to i64
+  %a.gep = getelementptr inbounds float, ptr addrspace(1) %a.ptr, i64 %tid.ext
+  %b.gep = getelementptr inbounds float, ptr addrspace(1) %b.ptr, i64 %tid.ext
+  %out.gep = getelementptr inbounds float, ptr addrspace(1) %out, i64 %tid.ext
+  %a = load volatile float, ptr addrspace(1) %a.gep
+  %b = load volatile float, ptr addrspace(1) %b.gep
+  %max = call float @llvm.maximum.f32(float %a, float 0.0)
+  %fneg = fneg float %max
+  %mul = fmul float %fneg, %b
+  store float %mul, ptr addrspace(1) %out.gep
+  ret void
+}
+
+; GCN-LABEL: {{^}}v_fneg_maximum_multi_use_maximum_f32:
+; GCN: global_load_b32 [[A:v[0-9]+]]
+; GCN: global_load_b32 [[B:v[0-9]+]]
+; GCN: v_minimum_f32 [[MAX0:v[0-9]+]], -[[A]], -[[B]]
+; GCN-SDAG:  v_mul_f32_e32 [[MUL1:v[0-9]+]], -4.0, [[MAX0]]
+; GCN-GISEL: v_mul_f32_e64 [[MUL1:v[0-9]+]], -[[MAX0]], 4.0
+; GCN: global_store_b32 v[{{[0-9:]+}}], [[MAX0]]
+; GCN: global_store_b32 v[{{[0-9:]+}}], [[MUL1]]
+define void @v_fneg_maximum_multi_use_maximum_f32(ptr addrspace(1) %out, ptr addrspace(1) %a.ptr, ptr addrspace(1) %b.ptr) {
+  %tid = call i32 @llvm.amdgcn.workitem.id.x()
+  %tid.ext = sext i32 %tid to i64
+  %a.gep = getelementptr inbounds float, ptr addrspace(1) %a.ptr, i64 %tid.ext
+  %b.gep = getelementptr inbounds float, ptr addrspace(1) %b.ptr, i64 %tid.ext
+  %out.gep = getelementptr inbounds float, ptr addrspace(1) %out, i64 %tid.ext
+  %a = load volatile float, ptr addrspace(1) %a.gep
+  %b = load volatile float, ptr addrspace(1) %b.gep
+  %min = call float @llvm.maximum.f32(float %a, float %b)
+  %fneg = fneg float %min
+  %use1 = fmul float %min, 4.0
+  store volatile float %fneg, ptr addrspace(1) %out
+  store volatile float %use1, ptr addrspace(1) %out
+  ret void
+}
+
+declare i32 @llvm.amdgcn.workitem.id.x()
+declare float @llvm.minimum.f32(float, float)
+declare float @llvm.maximum.f32(float, float)
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_i64_system.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_i64_system.ll
index 5a827584aa1062ad19daa2f8ebe9f61b3a372e2a..c2c1af3143b43e2981e7f8cc2678b9356b73d642 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_i64_system.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_i64_system.ll
@@ -2652,15 +2652,15 @@ define i64 @global_atomic_nand_i64_ret(ptr addrspace(1) %ptr, i64 %in) {
 ; SI-LABEL: global_atomic_nand_i64_ret:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT:    v_mov_b32_e32 v4, v3
-; SI-NEXT:    v_mov_b32_e32 v5, v2
-; SI-NEXT:    v_mov_b32_e32 v7, v1
-; SI-NEXT:    v_mov_b32_e32 v6, v0
+; SI-NEXT:    v_mov_b32_e32 v6, v3
+; SI-NEXT:    v_mov_b32_e32 v7, v2
+; SI-NEXT:    v_mov_b32_e32 v5, v1
+; SI-NEXT:    v_mov_b32_e32 v4, v0
 ; SI-NEXT:    s_mov_b32 s6, 0
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s4, s6
 ; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_load_dwordx2 v[0:1], v[6:7], s[4:7], 0 addr64
+; SI-NEXT:    buffer_load_dwordx2 v[0:1], v[4:5], s[4:7], 0 addr64
 ; SI-NEXT:    s_mov_b64 s[8:9], 0
 ; SI-NEXT:  .LBB42_1: ; %atomicrmw.start
 ; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -2668,8 +2668,8 @@ define i64 @global_atomic_nand_i64_ret(ptr addrspace(1) %ptr, i64 %in) {
 ; SI-NEXT:    v_mov_b32_e32 v11, v1
 ; SI-NEXT:    v_mov_b32_e32 v10, v0
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v0, v11, v4
-; SI-NEXT:    v_and_b32_e32 v1, v10, v5
+; SI-NEXT:    v_and_b32_e32 v0, v11, v6
+; SI-NEXT:    v_and_b32_e32 v1, v10, v7
 ; SI-NEXT:    v_not_b32_e32 v9, v0
 ; SI-NEXT:    v_not_b32_e32 v8, v1
 ; SI-NEXT:    v_mov_b32_e32 v0, v8
@@ -2677,7 +2677,7 @@ define i64 @global_atomic_nand_i64_ret(ptr addrspace(1) %ptr, i64 %in) {
 ; SI-NEXT:    v_mov_b32_e32 v2, v10
 ; SI-NEXT:    v_mov_b32_e32 v3, v11
 ; SI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; SI-NEXT:    buffer_atomic_cmpswap_x2 v[0:3], v[6:7], s[4:7], 0 addr64 glc
+; SI-NEXT:    buffer_atomic_cmpswap_x2 v[0:3], v[4:5], s[4:7], 0 addr64 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
 ; SI-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]
@@ -2752,15 +2752,15 @@ define i64 @global_atomic_nand_i64_ret_offset(ptr addrspace(1) %out, i64 %in) {
 ; SI-LABEL: global_atomic_nand_i64_ret_offset:
 ; SI:       ; %bb.0:
 ; SI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SI-NEXT:    v_mov_b32_e32 v4, v3
-; SI-NEXT:    v_mov_b32_e32 v5, v2
-; SI-NEXT:    v_mov_b32_e32 v7, v1
-; SI-NEXT:    v_mov_b32_e32 v6, v0
+; SI-NEXT:    v_mov_b32_e32 v6, v3
+; SI-NEXT:    v_mov_b32_e32 v7, v2
+; SI-NEXT:    v_mov_b32_e32 v5, v1
+; SI-NEXT:    v_mov_b32_e32 v4, v0
 ; SI-NEXT:    s_mov_b32 s6, 0
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s4, s6
 ; SI-NEXT:    s_mov_b32 s5, s6
-; SI-NEXT:    buffer_load_dwordx2 v[0:1], v[6:7], s[4:7], 0 addr64 offset:32
+; SI-NEXT:    buffer_load_dwordx2 v[0:1], v[4:5], s[4:7], 0 addr64 offset:32
 ; SI-NEXT:    s_mov_b64 s[8:9], 0
 ; SI-NEXT:  .LBB43_1: ; %atomicrmw.start
 ; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
@@ -2768,8 +2768,8 @@ define i64 @global_atomic_nand_i64_ret_offset(ptr addrspace(1) %out, i64 %in) {
 ; SI-NEXT:    v_mov_b32_e32 v11, v1
 ; SI-NEXT:    v_mov_b32_e32 v10, v0
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v0, v11, v4
-; SI-NEXT:    v_and_b32_e32 v1, v10, v5
+; SI-NEXT:    v_and_b32_e32 v0, v11, v6
+; SI-NEXT:    v_and_b32_e32 v1, v10, v7
 ; SI-NEXT:    v_not_b32_e32 v9, v0
 ; SI-NEXT:    v_not_b32_e32 v8, v1
 ; SI-NEXT:    v_mov_b32_e32 v0, v8
@@ -2777,7 +2777,7 @@ define i64 @global_atomic_nand_i64_ret_offset(ptr addrspace(1) %out, i64 %in) {
 ; SI-NEXT:    v_mov_b32_e32 v2, v10
 ; SI-NEXT:    v_mov_b32_e32 v3, v11
 ; SI-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
-; SI-NEXT:    buffer_atomic_cmpswap_x2 v[0:3], v[6:7], s[4:7], 0 addr64 offset:32 glc
+; SI-NEXT:    buffer_atomic_cmpswap_x2 v[0:3], v[4:5], s[4:7], 0 addr64 offset:32 glc
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    buffer_wbinvl1
 ; SI-NEXT:    v_cmp_eq_u64_e32 vcc, v[0:1], v[10:11]
@@ -2859,8 +2859,8 @@ define amdgpu_gfx void @global_atomic_nand_i64_noret_scalar(ptr addrspace(1) inr
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_writelane_b32 v0, s6, 0
 ; SI-NEXT:    v_writelane_b32 v0, s7, 1
-; SI-NEXT:    s_mov_b32 s35, s7
-; SI-NEXT:    s_mov_b32 s34, s6
+; SI-NEXT:    s_mov_b32 s34, s7
+; SI-NEXT:    s_mov_b32 s35, s6
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:    buffer_load_dwordx2 v[3:4], off, s[4:7], 0
@@ -2868,9 +2868,9 @@ define amdgpu_gfx void @global_atomic_nand_i64_noret_scalar(ptr addrspace(1) inr
 ; SI-NEXT:  .LBB44_1: ; %atomicrmw.start
 ; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, s35, v4
+; SI-NEXT:    v_and_b32_e32 v1, s34, v4
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v5, s34, v3
+; SI-NEXT:    v_and_b32_e32 v5, s35, v3
 ; SI-NEXT:    v_not_b32_e32 v2, v1
 ; SI-NEXT:    v_not_b32_e32 v1, v5
 ; SI-NEXT:    v_mov_b32_e32 v8, v4
@@ -2967,8 +2967,8 @@ define amdgpu_gfx void @global_atomic_nand_i64_noret_offset_scalar(ptr addrspace
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_writelane_b32 v0, s6, 0
 ; SI-NEXT:    v_writelane_b32 v0, s7, 1
-; SI-NEXT:    s_mov_b32 s35, s7
-; SI-NEXT:    s_mov_b32 s34, s6
+; SI-NEXT:    s_mov_b32 s34, s7
+; SI-NEXT:    s_mov_b32 s35, s6
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:    buffer_load_dwordx2 v[3:4], off, s[4:7], 0 offset:32
@@ -2976,9 +2976,9 @@ define amdgpu_gfx void @global_atomic_nand_i64_noret_offset_scalar(ptr addrspace
 ; SI-NEXT:  .LBB45_1: ; %atomicrmw.start
 ; SI-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; SI-NEXT:    s_waitcnt vmcnt(0)
-; SI-NEXT:    v_and_b32_e32 v1, s35, v4
+; SI-NEXT:    v_and_b32_e32 v1, s34, v4
 ; SI-NEXT:    s_waitcnt expcnt(0)
-; SI-NEXT:    v_and_b32_e32 v5, s34, v3
+; SI-NEXT:    v_and_b32_e32 v5, s35, v3
 ; SI-NEXT:    v_not_b32_e32 v2, v1
 ; SI-NEXT:    v_not_b32_e32 v1, v5
 ; SI-NEXT:    v_mov_b32_e32 v8, v4
@@ -3076,8 +3076,8 @@ define amdgpu_gfx i64 @global_atomic_nand_i64_ret_scalar(ptr addrspace(1) inreg
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_writelane_b32 v2, s6, 0
 ; SI-NEXT:    v_writelane_b32 v2, s7, 1
-; SI-NEXT:    s_mov_b32 s35, s7
-; SI-NEXT:    s_mov_b32 s34, s6
+; SI-NEXT:    s_mov_b32 s34, s7
+; SI-NEXT:    s_mov_b32 s35, s6
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:    buffer_load_dwordx2 v[3:4], off, s[4:7], 0
@@ -3087,8 +3087,8 @@ define amdgpu_gfx i64 @global_atomic_nand_i64_ret_scalar(ptr addrspace(1) inreg
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_mov_b32_e32 v8, v4
 ; SI-NEXT:    v_mov_b32_e32 v7, v3
-; SI-NEXT:    v_and_b32_e32 v0, s35, v8
-; SI-NEXT:    v_and_b32_e32 v1, s34, v7
+; SI-NEXT:    v_and_b32_e32 v0, s34, v8
+; SI-NEXT:    v_and_b32_e32 v1, s35, v7
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_not_b32_e32 v6, v0
 ; SI-NEXT:    v_not_b32_e32 v5, v1
@@ -3186,8 +3186,8 @@ define amdgpu_gfx i64 @global_atomic_nand_i64_ret_offset_scalar(ptr addrspace(1)
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_writelane_b32 v2, s6, 0
 ; SI-NEXT:    v_writelane_b32 v2, s7, 1
-; SI-NEXT:    s_mov_b32 s35, s7
-; SI-NEXT:    s_mov_b32 s34, s6
+; SI-NEXT:    s_mov_b32 s34, s7
+; SI-NEXT:    s_mov_b32 s35, s6
 ; SI-NEXT:    s_mov_b32 s7, 0xf000
 ; SI-NEXT:    s_mov_b32 s6, -1
 ; SI-NEXT:    buffer_load_dwordx2 v[3:4], off, s[4:7], 0 offset:32
@@ -3197,8 +3197,8 @@ define amdgpu_gfx i64 @global_atomic_nand_i64_ret_offset_scalar(ptr addrspace(1)
 ; SI-NEXT:    s_waitcnt vmcnt(0)
 ; SI-NEXT:    v_mov_b32_e32 v8, v4
 ; SI-NEXT:    v_mov_b32_e32 v7, v3
-; SI-NEXT:    v_and_b32_e32 v0, s35, v8
-; SI-NEXT:    v_and_b32_e32 v1, s34, v7
+; SI-NEXT:    v_and_b32_e32 v0, s34, v8
+; SI-NEXT:    v_and_b32_e32 v1, s35, v7
 ; SI-NEXT:    s_waitcnt expcnt(0)
 ; SI-NEXT:    v_not_b32_e32 v6, v0
 ; SI-NEXT:    v_not_b32_e32 v5, v1
diff --git a/llvm/test/CodeGen/AMDGPU/idiv-licm.ll b/llvm/test/CodeGen/AMDGPU/idiv-licm.ll
index 8382624af5d749e68ce9e7a82d81df8bf89e4c5e..637fcd0a1e6b6ffe11a55ecfec3916ca84d11471 100644
--- a/llvm/test/CodeGen/AMDGPU/idiv-licm.ll
+++ b/llvm/test/CodeGen/AMDGPU/idiv-licm.ll
@@ -6,41 +6,41 @@
 define amdgpu_kernel void @udiv32_invariant_denom(ptr addrspace(1) nocapture %arg, i32 %arg1) {
 ; GFX9-LABEL: udiv32_invariant_denom:
 ; GFX9:       ; %bb.0: ; %bb
-; GFX9-NEXT:    s_load_dword s5, s[0:1], 0x2c
-; GFX9-NEXT:    s_mov_b32 s8, 0
+; GFX9-NEXT:    s_load_dword s6, s[0:1], 0x2c
+; GFX9-NEXT:    s_mov_b32 s7, 0
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX9-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s5
-; GFX9-NEXT:    s_sub_i32 s4, 0, s5
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s6
+; GFX9-NEXT:    s_sub_i32 s4, 0, s6
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v0, v0
 ; GFX9-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GFX9-NEXT:    v_cvt_u32_f32_e32 v1, v0
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 0
-; GFX9-NEXT:    v_readfirstlane_b32 s6, v1
-; GFX9-NEXT:    s_mul_i32 s4, s4, s6
-; GFX9-NEXT:    s_mul_hi_u32 s4, s6, s4
-; GFX9-NEXT:    s_add_i32 s4, s6, s4
-; GFX9-NEXT:    s_mov_b64 s[6:7], 0
+; GFX9-NEXT:    v_readfirstlane_b32 s5, v1
+; GFX9-NEXT:    s_mul_i32 s4, s4, s5
+; GFX9-NEXT:    s_mul_hi_u32 s4, s5, s4
+; GFX9-NEXT:    s_add_i32 s8, s5, s4
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX9-NEXT:  .LBB0_1: ; %bb3
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT:    s_not_b32 s10, s7
-; GFX9-NEXT:    s_mul_i32 s9, s5, s7
-; GFX9-NEXT:    s_mul_i32 s10, s5, s10
-; GFX9-NEXT:    s_add_i32 s11, s7, 1
-; GFX9-NEXT:    s_sub_i32 s9, s8, s9
-; GFX9-NEXT:    s_add_i32 s10, s8, s10
-; GFX9-NEXT:    s_cmp_ge_u32 s9, s5
-; GFX9-NEXT:    s_cselect_b32 s11, s11, s7
+; GFX9-NEXT:    s_not_b32 s10, s5
+; GFX9-NEXT:    s_mul_i32 s9, s6, s5
+; GFX9-NEXT:    s_mul_i32 s10, s6, s10
+; GFX9-NEXT:    s_add_i32 s11, s5, 1
+; GFX9-NEXT:    s_sub_i32 s9, s7, s9
+; GFX9-NEXT:    s_add_i32 s10, s7, s10
+; GFX9-NEXT:    s_cmp_ge_u32 s9, s6
+; GFX9-NEXT:    s_cselect_b32 s11, s11, s5
 ; GFX9-NEXT:    s_cselect_b32 s9, s10, s9
 ; GFX9-NEXT:    s_add_i32 s10, s11, 1
-; GFX9-NEXT:    s_cmp_ge_u32 s9, s5
+; GFX9-NEXT:    s_cmp_ge_u32 s9, s6
 ; GFX9-NEXT:    s_cselect_b32 s9, s10, s11
 ; GFX9-NEXT:    s_add_u32 s10, s0, s2
 ; GFX9-NEXT:    s_addc_u32 s11, s1, s3
-; GFX9-NEXT:    s_add_i32 s8, s8, 1
-; GFX9-NEXT:    s_add_u32 s6, s6, s4
-; GFX9-NEXT:    s_addc_u32 s7, s7, 0
+; GFX9-NEXT:    s_add_i32 s7, s7, 1
+; GFX9-NEXT:    s_add_u32 s4, s4, s8
+; GFX9-NEXT:    s_addc_u32 s5, s5, 0
 ; GFX9-NEXT:    s_add_u32 s2, s2, 4
 ; GFX9-NEXT:    s_addc_u32 s3, s3, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s9
@@ -52,43 +52,43 @@ define amdgpu_kernel void @udiv32_invariant_denom(ptr addrspace(1) nocapture %ar
 ;
 ; GFX10-LABEL: udiv32_invariant_denom:
 ; GFX10:       ; %bb.0: ; %bb
-; GFX10-NEXT:    s_load_dword s5, s[0:1], 0x2c
-; GFX10-NEXT:    s_mov_b32 s8, 0
+; GFX10-NEXT:    s_load_dword s6, s[0:1], 0x2c
+; GFX10-NEXT:    s_mov_b32 s7, 0
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    v_cvt_f32_u32_e32 v0, s5
-; GFX10-NEXT:    s_sub_i32 s2, 0, s5
+; GFX10-NEXT:    v_cvt_f32_u32_e32 v0, s6
+; GFX10-NEXT:    s_sub_i32 s2, 0, s6
 ; GFX10-NEXT:    v_rcp_iflag_f32_e32 v0, v0
 ; GFX10-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GFX10-NEXT:    v_cvt_u32_f32_e32 v0, v0
 ; GFX10-NEXT:    v_readfirstlane_b32 s4, v0
 ; GFX10-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX10-NEXT:    s_mul_i32 s2, s2, s4
-; GFX10-NEXT:    s_mul_hi_u32 s6, s4, s2
+; GFX10-NEXT:    s_mul_hi_u32 s5, s4, s2
 ; GFX10-NEXT:    s_mov_b64 s[2:3], 0
-; GFX10-NEXT:    s_add_i32 s4, s4, s6
-; GFX10-NEXT:    s_mov_b64 s[6:7], 0
+; GFX10-NEXT:    s_add_i32 s8, s4, s5
+; GFX10-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX10-NEXT:  .LBB0_1: ; %bb3
 ; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
-; GFX10-NEXT:    s_not_b32 s10, s7
-; GFX10-NEXT:    s_mul_i32 s9, s5, s7
-; GFX10-NEXT:    s_mul_i32 s10, s5, s10
-; GFX10-NEXT:    s_sub_i32 s9, s8, s9
-; GFX10-NEXT:    s_add_i32 s11, s7, 1
-; GFX10-NEXT:    s_add_i32 s10, s8, s10
-; GFX10-NEXT:    s_cmp_ge_u32 s9, s5
-; GFX10-NEXT:    s_cselect_b32 s11, s11, s7
+; GFX10-NEXT:    s_not_b32 s10, s5
+; GFX10-NEXT:    s_mul_i32 s9, s6, s5
+; GFX10-NEXT:    s_mul_i32 s10, s6, s10
+; GFX10-NEXT:    s_sub_i32 s9, s7, s9
+; GFX10-NEXT:    s_add_i32 s11, s5, 1
+; GFX10-NEXT:    s_add_i32 s10, s7, s10
+; GFX10-NEXT:    s_cmp_ge_u32 s9, s6
+; GFX10-NEXT:    s_cselect_b32 s11, s11, s5
 ; GFX10-NEXT:    s_cselect_b32 s9, s10, s9
 ; GFX10-NEXT:    s_add_i32 s10, s11, 1
-; GFX10-NEXT:    s_cmp_ge_u32 s9, s5
+; GFX10-NEXT:    s_cmp_ge_u32 s9, s6
 ; GFX10-NEXT:    s_cselect_b32 s9, s10, s11
 ; GFX10-NEXT:    s_add_u32 s10, s0, s2
 ; GFX10-NEXT:    s_addc_u32 s11, s1, s3
-; GFX10-NEXT:    s_add_i32 s8, s8, 1
-; GFX10-NEXT:    s_add_u32 s6, s6, s4
+; GFX10-NEXT:    s_add_i32 s7, s7, 1
+; GFX10-NEXT:    s_add_u32 s4, s4, s8
 ; GFX10-NEXT:    v_mov_b32_e32 v1, s9
-; GFX10-NEXT:    s_addc_u32 s7, s7, 0
+; GFX10-NEXT:    s_addc_u32 s5, s5, 0
 ; GFX10-NEXT:    s_add_u32 s2, s2, 4
 ; GFX10-NEXT:    s_addc_u32 s3, s3, 0
 ; GFX10-NEXT:    s_cmpk_eq_i32 s2, 0x1000
@@ -100,12 +100,12 @@ define amdgpu_kernel void @udiv32_invariant_denom(ptr addrspace(1) nocapture %ar
 ; GFX11-LABEL: udiv32_invariant_denom:
 ; GFX11:       ; %bb.0: ; %bb
 ; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b32 s5, s[0:1], 0x2c
+; GFX11-NEXT:    s_load_b32 s6, s[0:1], 0x2c
 ; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
-; GFX11-NEXT:    s_mov_b32 s8, 0
+; GFX11-NEXT:    s_mov_b32 s7, 0
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    v_cvt_f32_u32_e32 v0, s5
-; GFX11-NEXT:    s_sub_i32 s2, 0, s5
+; GFX11-NEXT:    v_cvt_f32_u32_e32 v0, s6
+; GFX11-NEXT:    s_sub_i32 s2, 0, s6
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX11-NEXT:    v_rcp_iflag_f32_e32 v0, v0
 ; GFX11-NEXT:    s_waitcnt_depctr 0xfff
@@ -116,32 +116,32 @@ define amdgpu_kernel void @udiv32_invariant_denom(ptr addrspace(1) nocapture %ar
 ; GFX11-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX11-NEXT:    s_mul_i32 s2, s2, s4
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT:    s_mul_hi_u32 s6, s4, s2
+; GFX11-NEXT:    s_mul_hi_u32 s5, s4, s2
 ; GFX11-NEXT:    s_mov_b64 s[2:3], 0
-; GFX11-NEXT:    s_add_i32 s4, s4, s6
-; GFX11-NEXT:    s_mov_b64 s[6:7], 0
+; GFX11-NEXT:    s_add_i32 s8, s4, s5
+; GFX11-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX11-NEXT:    .p2align 6
 ; GFX11-NEXT:  .LBB0_1: ; %bb3
 ; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT:    s_not_b32 s10, s7
-; GFX11-NEXT:    s_mul_i32 s9, s5, s7
-; GFX11-NEXT:    s_mul_i32 s10, s5, s10
-; GFX11-NEXT:    s_sub_i32 s9, s8, s9
-; GFX11-NEXT:    s_add_i32 s11, s7, 1
-; GFX11-NEXT:    s_add_i32 s10, s8, s10
-; GFX11-NEXT:    s_cmp_ge_u32 s9, s5
-; GFX11-NEXT:    s_cselect_b32 s11, s11, s7
+; GFX11-NEXT:    s_not_b32 s10, s5
+; GFX11-NEXT:    s_mul_i32 s9, s6, s5
+; GFX11-NEXT:    s_mul_i32 s10, s6, s10
+; GFX11-NEXT:    s_sub_i32 s9, s7, s9
+; GFX11-NEXT:    s_add_i32 s11, s5, 1
+; GFX11-NEXT:    s_add_i32 s10, s7, s10
+; GFX11-NEXT:    s_cmp_ge_u32 s9, s6
+; GFX11-NEXT:    s_cselect_b32 s11, s11, s5
 ; GFX11-NEXT:    s_cselect_b32 s9, s10, s9
 ; GFX11-NEXT:    s_add_i32 s10, s11, 1
-; GFX11-NEXT:    s_cmp_ge_u32 s9, s5
+; GFX11-NEXT:    s_cmp_ge_u32 s9, s6
 ; GFX11-NEXT:    s_cselect_b32 s9, s10, s11
 ; GFX11-NEXT:    s_add_u32 s10, s0, s2
 ; GFX11-NEXT:    s_addc_u32 s11, s1, s3
-; GFX11-NEXT:    s_add_i32 s8, s8, 1
-; GFX11-NEXT:    s_add_u32 s6, s6, s4
+; GFX11-NEXT:    s_add_i32 s7, s7, 1
+; GFX11-NEXT:    s_add_u32 s4, s4, s8
 ; GFX11-NEXT:    v_mov_b32_e32 v1, s9
-; GFX11-NEXT:    s_addc_u32 s7, s7, 0
+; GFX11-NEXT:    s_addc_u32 s5, s5, 0
 ; GFX11-NEXT:    s_add_u32 s2, s2, 4
 ; GFX11-NEXT:    s_addc_u32 s3, s3, 0
 ; GFX11-NEXT:    s_cmpk_eq_i32 s2, 0x1000
@@ -171,39 +171,39 @@ bb3:                                              ; preds = %bb3, %bb
 define amdgpu_kernel void @urem32_invariant_denom(ptr addrspace(1) nocapture %arg, i32 %arg1) {
 ; GFX9-LABEL: urem32_invariant_denom:
 ; GFX9:       ; %bb.0: ; %bb
-; GFX9-NEXT:    s_load_dword s5, s[0:1], 0x2c
-; GFX9-NEXT:    s_mov_b32 s8, 0
+; GFX9-NEXT:    s_load_dword s6, s[0:1], 0x2c
+; GFX9-NEXT:    s_mov_b32 s7, 0
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX9-NEXT:    s_mov_b64 s[2:3], 0
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s5
-; GFX9-NEXT:    s_sub_i32 s4, 0, s5
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s6
+; GFX9-NEXT:    s_sub_i32 s4, 0, s6
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v0, v0
 ; GFX9-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GFX9-NEXT:    v_cvt_u32_f32_e32 v1, v0
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 0
-; GFX9-NEXT:    v_readfirstlane_b32 s6, v1
-; GFX9-NEXT:    s_mul_i32 s4, s4, s6
-; GFX9-NEXT:    s_mul_hi_u32 s4, s6, s4
-; GFX9-NEXT:    s_add_i32 s4, s6, s4
-; GFX9-NEXT:    s_mov_b64 s[6:7], 0
+; GFX9-NEXT:    v_readfirstlane_b32 s5, v1
+; GFX9-NEXT:    s_mul_i32 s4, s4, s5
+; GFX9-NEXT:    s_mul_hi_u32 s4, s5, s4
+; GFX9-NEXT:    s_add_i32 s8, s5, s4
+; GFX9-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX9-NEXT:  .LBB1_1: ; %bb3
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT:    s_not_b32 s10, s7
-; GFX9-NEXT:    s_mul_i32 s9, s5, s7
-; GFX9-NEXT:    s_mul_i32 s10, s5, s10
-; GFX9-NEXT:    s_sub_i32 s9, s8, s9
-; GFX9-NEXT:    s_add_i32 s10, s8, s10
-; GFX9-NEXT:    s_cmp_ge_u32 s9, s5
+; GFX9-NEXT:    s_not_b32 s10, s5
+; GFX9-NEXT:    s_mul_i32 s9, s6, s5
+; GFX9-NEXT:    s_mul_i32 s10, s6, s10
+; GFX9-NEXT:    s_sub_i32 s9, s7, s9
+; GFX9-NEXT:    s_add_i32 s10, s7, s10
+; GFX9-NEXT:    s_cmp_ge_u32 s9, s6
 ; GFX9-NEXT:    s_cselect_b32 s9, s10, s9
-; GFX9-NEXT:    s_sub_i32 s10, s9, s5
-; GFX9-NEXT:    s_cmp_ge_u32 s9, s5
+; GFX9-NEXT:    s_sub_i32 s10, s9, s6
+; GFX9-NEXT:    s_cmp_ge_u32 s9, s6
 ; GFX9-NEXT:    s_cselect_b32 s9, s10, s9
 ; GFX9-NEXT:    s_add_u32 s10, s0, s2
 ; GFX9-NEXT:    s_addc_u32 s11, s1, s3
-; GFX9-NEXT:    s_add_i32 s8, s8, 1
-; GFX9-NEXT:    s_add_u32 s6, s6, s4
-; GFX9-NEXT:    s_addc_u32 s7, s7, 0
+; GFX9-NEXT:    s_add_i32 s7, s7, 1
+; GFX9-NEXT:    s_add_u32 s4, s4, s8
+; GFX9-NEXT:    s_addc_u32 s5, s5, 0
 ; GFX9-NEXT:    s_add_u32 s2, s2, 4
 ; GFX9-NEXT:    s_addc_u32 s3, s3, 0
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s9
@@ -215,41 +215,41 @@ define amdgpu_kernel void @urem32_invariant_denom(ptr addrspace(1) nocapture %ar
 ;
 ; GFX10-LABEL: urem32_invariant_denom:
 ; GFX10:       ; %bb.0: ; %bb
-; GFX10-NEXT:    s_load_dword s5, s[0:1], 0x2c
-; GFX10-NEXT:    s_mov_b32 s8, 0
+; GFX10-NEXT:    s_load_dword s6, s[0:1], 0x2c
+; GFX10-NEXT:    s_mov_b32 s7, 0
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    v_cvt_f32_u32_e32 v0, s5
-; GFX10-NEXT:    s_sub_i32 s2, 0, s5
+; GFX10-NEXT:    v_cvt_f32_u32_e32 v0, s6
+; GFX10-NEXT:    s_sub_i32 s2, 0, s6
 ; GFX10-NEXT:    v_rcp_iflag_f32_e32 v0, v0
 ; GFX10-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GFX10-NEXT:    v_cvt_u32_f32_e32 v0, v0
 ; GFX10-NEXT:    v_readfirstlane_b32 s4, v0
 ; GFX10-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX10-NEXT:    s_mul_i32 s2, s2, s4
-; GFX10-NEXT:    s_mul_hi_u32 s6, s4, s2
+; GFX10-NEXT:    s_mul_hi_u32 s5, s4, s2
 ; GFX10-NEXT:    s_mov_b64 s[2:3], 0
-; GFX10-NEXT:    s_add_i32 s4, s4, s6
-; GFX10-NEXT:    s_mov_b64 s[6:7], 0
+; GFX10-NEXT:    s_add_i32 s8, s4, s5
+; GFX10-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX10-NEXT:  .LBB1_1: ; %bb3
 ; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT:    s_not_b32 s9, s7
+; GFX10-NEXT:    s_not_b32 s9, s5
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
-; GFX10-NEXT:    s_mul_i32 s10, s5, s7
-; GFX10-NEXT:    s_mul_i32 s9, s5, s9
-; GFX10-NEXT:    s_sub_i32 s10, s8, s10
-; GFX10-NEXT:    s_add_i32 s9, s8, s9
-; GFX10-NEXT:    s_cmp_ge_u32 s10, s5
+; GFX10-NEXT:    s_mul_i32 s10, s6, s5
+; GFX10-NEXT:    s_mul_i32 s9, s6, s9
+; GFX10-NEXT:    s_sub_i32 s10, s7, s10
+; GFX10-NEXT:    s_add_i32 s9, s7, s9
+; GFX10-NEXT:    s_cmp_ge_u32 s10, s6
 ; GFX10-NEXT:    s_cselect_b32 s9, s9, s10
-; GFX10-NEXT:    s_sub_i32 s10, s9, s5
-; GFX10-NEXT:    s_cmp_ge_u32 s9, s5
+; GFX10-NEXT:    s_sub_i32 s10, s9, s6
+; GFX10-NEXT:    s_cmp_ge_u32 s9, s6
 ; GFX10-NEXT:    s_cselect_b32 s9, s10, s9
 ; GFX10-NEXT:    s_add_u32 s10, s0, s2
 ; GFX10-NEXT:    s_addc_u32 s11, s1, s3
-; GFX10-NEXT:    s_add_i32 s8, s8, 1
-; GFX10-NEXT:    s_add_u32 s6, s6, s4
+; GFX10-NEXT:    s_add_i32 s7, s7, 1
+; GFX10-NEXT:    s_add_u32 s4, s4, s8
 ; GFX10-NEXT:    v_mov_b32_e32 v1, s9
-; GFX10-NEXT:    s_addc_u32 s7, s7, 0
+; GFX10-NEXT:    s_addc_u32 s5, s5, 0
 ; GFX10-NEXT:    s_add_u32 s2, s2, 4
 ; GFX10-NEXT:    s_addc_u32 s3, s3, 0
 ; GFX10-NEXT:    s_cmpk_eq_i32 s2, 0x1000
@@ -261,12 +261,12 @@ define amdgpu_kernel void @urem32_invariant_denom(ptr addrspace(1) nocapture %ar
 ; GFX11-LABEL: urem32_invariant_denom:
 ; GFX11:       ; %bb.0: ; %bb
 ; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b32 s5, s[0:1], 0x2c
+; GFX11-NEXT:    s_load_b32 s6, s[0:1], 0x2c
 ; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
-; GFX11-NEXT:    s_mov_b32 s8, 0
+; GFX11-NEXT:    s_mov_b32 s7, 0
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    v_cvt_f32_u32_e32 v0, s5
-; GFX11-NEXT:    s_sub_i32 s2, 0, s5
+; GFX11-NEXT:    v_cvt_f32_u32_e32 v0, s6
+; GFX11-NEXT:    s_sub_i32 s2, 0, s6
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX11-NEXT:    v_rcp_iflag_f32_e32 v0, v0
 ; GFX11-NEXT:    s_waitcnt_depctr 0xfff
@@ -277,31 +277,31 @@ define amdgpu_kernel void @urem32_invariant_denom(ptr addrspace(1) nocapture %ar
 ; GFX11-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX11-NEXT:    s_mul_i32 s2, s2, s4
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT:    s_mul_hi_u32 s6, s4, s2
+; GFX11-NEXT:    s_mul_hi_u32 s5, s4, s2
 ; GFX11-NEXT:    s_mov_b64 s[2:3], 0
-; GFX11-NEXT:    s_add_i32 s4, s4, s6
-; GFX11-NEXT:    s_mov_b64 s[6:7], 0
+; GFX11-NEXT:    s_add_i32 s8, s4, s5
+; GFX11-NEXT:    s_mov_b64 s[4:5], 0
 ; GFX11-NEXT:    .p2align 6
 ; GFX11-NEXT:  .LBB1_1: ; %bb3
 ; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT:    s_not_b32 s9, s7
-; GFX11-NEXT:    s_mul_i32 s10, s5, s7
-; GFX11-NEXT:    s_mul_i32 s9, s5, s9
-; GFX11-NEXT:    s_sub_i32 s10, s8, s10
-; GFX11-NEXT:    s_add_i32 s9, s8, s9
-; GFX11-NEXT:    s_cmp_ge_u32 s10, s5
+; GFX11-NEXT:    s_not_b32 s9, s5
+; GFX11-NEXT:    s_mul_i32 s10, s6, s5
+; GFX11-NEXT:    s_mul_i32 s9, s6, s9
+; GFX11-NEXT:    s_sub_i32 s10, s7, s10
+; GFX11-NEXT:    s_add_i32 s9, s7, s9
+; GFX11-NEXT:    s_cmp_ge_u32 s10, s6
 ; GFX11-NEXT:    s_cselect_b32 s9, s9, s10
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT:    s_sub_i32 s10, s9, s5
-; GFX11-NEXT:    s_cmp_ge_u32 s9, s5
+; GFX11-NEXT:    s_sub_i32 s10, s9, s6
+; GFX11-NEXT:    s_cmp_ge_u32 s9, s6
 ; GFX11-NEXT:    s_cselect_b32 s9, s10, s9
 ; GFX11-NEXT:    s_add_u32 s10, s0, s2
 ; GFX11-NEXT:    s_addc_u32 s11, s1, s3
-; GFX11-NEXT:    s_add_i32 s8, s8, 1
-; GFX11-NEXT:    s_add_u32 s6, s6, s4
+; GFX11-NEXT:    s_add_i32 s7, s7, 1
+; GFX11-NEXT:    s_add_u32 s4, s4, s8
 ; GFX11-NEXT:    v_mov_b32_e32 v1, s9
-; GFX11-NEXT:    s_addc_u32 s7, s7, 0
+; GFX11-NEXT:    s_addc_u32 s5, s5, 0
 ; GFX11-NEXT:    s_add_u32 s2, s2, 4
 ; GFX11-NEXT:    s_addc_u32 s3, s3, 0
 ; GFX11-NEXT:    s_cmpk_eq_i32 s2, 0x1000
@@ -331,90 +331,90 @@ bb3:                                              ; preds = %bb3, %bb
 define amdgpu_kernel void @sdiv32_invariant_denom(ptr addrspace(1) nocapture %arg, i32 %arg1) {
 ; GFX9-LABEL: sdiv32_invariant_denom:
 ; GFX9:       ; %bb.0: ; %bb
-; GFX9-NEXT:    s_load_dword s2, s[0:1], 0x2c
-; GFX9-NEXT:    s_mov_b32 s5, 0
+; GFX9-NEXT:    s_load_dword s3, s[0:1], 0x2c
+; GFX9-NEXT:    s_mov_b32 s4, 0
 ; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-NEXT:    s_ashr_i32 s3, s2, 31
-; GFX9-NEXT:    s_add_i32 s2, s2, s3
-; GFX9-NEXT:    s_xor_b32 s4, s2, s3
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s4
-; GFX9-NEXT:    s_sub_i32 s2, 0, s4
+; GFX9-NEXT:    s_ashr_i32 s2, s3, 31
+; GFX9-NEXT:    s_add_i32 s3, s3, s2
+; GFX9-NEXT:    s_xor_b32 s3, s3, s2
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s3
+; GFX9-NEXT:    s_sub_i32 s5, 0, s3
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v0, v0
 ; GFX9-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GFX9-NEXT:    v_cvt_u32_f32_e32 v0, v0
 ; GFX9-NEXT:    v_readfirstlane_b32 s6, v0
-; GFX9-NEXT:    s_mul_i32 s2, s2, s6
-; GFX9-NEXT:    s_mul_hi_u32 s2, s6, s2
-; GFX9-NEXT:    s_add_i32 s2, s6, s2
+; GFX9-NEXT:    s_mul_i32 s5, s5, s6
+; GFX9-NEXT:    s_mul_hi_u32 s5, s6, s5
+; GFX9-NEXT:    s_add_i32 s5, s6, s5
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX9-NEXT:  .LBB2_1: ; %bb3
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT:    s_mul_hi_u32 s6, s5, s2
-; GFX9-NEXT:    s_mul_i32 s7, s6, s4
-; GFX9-NEXT:    s_sub_i32 s7, s5, s7
+; GFX9-NEXT:    s_mul_hi_u32 s6, s4, s5
+; GFX9-NEXT:    s_mul_i32 s7, s6, s3
+; GFX9-NEXT:    s_sub_i32 s7, s4, s7
 ; GFX9-NEXT:    s_add_i32 s8, s6, 1
-; GFX9-NEXT:    s_sub_i32 s9, s7, s4
-; GFX9-NEXT:    s_cmp_ge_u32 s7, s4
+; GFX9-NEXT:    s_sub_i32 s9, s7, s3
+; GFX9-NEXT:    s_cmp_ge_u32 s7, s3
 ; GFX9-NEXT:    s_cselect_b32 s6, s8, s6
 ; GFX9-NEXT:    s_cselect_b32 s7, s9, s7
 ; GFX9-NEXT:    s_add_i32 s8, s6, 1
-; GFX9-NEXT:    s_cmp_ge_u32 s7, s4
+; GFX9-NEXT:    s_cmp_ge_u32 s7, s3
 ; GFX9-NEXT:    s_cselect_b32 s6, s8, s6
-; GFX9-NEXT:    s_xor_b32 s6, s6, s3
-; GFX9-NEXT:    s_sub_i32 s6, s6, s3
-; GFX9-NEXT:    s_add_i32 s5, s5, 1
+; GFX9-NEXT:    s_xor_b32 s6, s6, s2
+; GFX9-NEXT:    s_sub_i32 s6, s6, s2
+; GFX9-NEXT:    s_add_i32 s4, s4, 1
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s6
 ; GFX9-NEXT:    global_store_dword v0, v1, s[0:1]
 ; GFX9-NEXT:    s_add_u32 s0, s0, 4
 ; GFX9-NEXT:    s_addc_u32 s1, s1, 0
-; GFX9-NEXT:    s_cmpk_eq_i32 s5, 0x400
+; GFX9-NEXT:    s_cmpk_eq_i32 s4, 0x400
 ; GFX9-NEXT:    s_cbranch_scc0 .LBB2_1
 ; GFX9-NEXT:  ; %bb.2: ; %bb2
 ; GFX9-NEXT:    s_endpgm
 ;
 ; GFX10-LABEL: sdiv32_invariant_denom:
 ; GFX10:       ; %bb.0: ; %bb
-; GFX10-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; GFX10-NEXT:    s_load_dword s3, s[0:1], 0x2c
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX10-NEXT:    s_ashr_i32 s3, s2, 31
+; GFX10-NEXT:    s_ashr_i32 s2, s3, 31
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GFX10-NEXT:    s_add_i32 s2, s2, s3
-; GFX10-NEXT:    s_xor_b32 s4, s2, s3
-; GFX10-NEXT:    v_cvt_f32_u32_e32 v0, s4
-; GFX10-NEXT:    s_sub_i32 s5, 0, s4
+; GFX10-NEXT:    s_add_i32 s3, s3, s2
+; GFX10-NEXT:    s_xor_b32 s3, s3, s2
+; GFX10-NEXT:    v_cvt_f32_u32_e32 v0, s3
+; GFX10-NEXT:    s_sub_i32 s4, 0, s3
 ; GFX10-NEXT:    v_rcp_iflag_f32_e32 v0, v0
 ; GFX10-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GFX10-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GFX10-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s5, v0
 ; GFX10-NEXT:    v_mov_b32_e32 v0, 0
-; GFX10-NEXT:    s_mul_i32 s5, s5, s2
-; GFX10-NEXT:    s_mul_hi_u32 s6, s2, s5
-; GFX10-NEXT:    s_mov_b32 s5, 0
-; GFX10-NEXT:    s_add_i32 s2, s2, s6
+; GFX10-NEXT:    s_mul_i32 s4, s4, s5
+; GFX10-NEXT:    s_mul_hi_u32 s6, s5, s4
+; GFX10-NEXT:    s_mov_b32 s4, 0
+; GFX10-NEXT:    s_add_i32 s5, s5, s6
 ; GFX10-NEXT:  .LBB2_1: ; %bb3
 ; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT:    s_mul_hi_u32 s6, s5, s2
-; GFX10-NEXT:    s_mul_i32 s7, s6, s4
+; GFX10-NEXT:    s_mul_hi_u32 s6, s4, s5
+; GFX10-NEXT:    s_mul_i32 s7, s6, s3
 ; GFX10-NEXT:    s_add_i32 s8, s6, 1
-; GFX10-NEXT:    s_sub_i32 s7, s5, s7
-; GFX10-NEXT:    s_sub_i32 s9, s7, s4
-; GFX10-NEXT:    s_cmp_ge_u32 s7, s4
+; GFX10-NEXT:    s_sub_i32 s7, s4, s7
+; GFX10-NEXT:    s_sub_i32 s9, s7, s3
+; GFX10-NEXT:    s_cmp_ge_u32 s7, s3
 ; GFX10-NEXT:    s_cselect_b32 s6, s8, s6
 ; GFX10-NEXT:    s_cselect_b32 s7, s9, s7
 ; GFX10-NEXT:    s_add_i32 s8, s6, 1
-; GFX10-NEXT:    s_cmp_ge_u32 s7, s4
+; GFX10-NEXT:    s_cmp_ge_u32 s7, s3
 ; GFX10-NEXT:    s_cselect_b32 s6, s8, s6
-; GFX10-NEXT:    s_add_i32 s5, s5, 1
-; GFX10-NEXT:    s_xor_b32 s6, s6, s3
-; GFX10-NEXT:    s_sub_i32 s6, s6, s3
+; GFX10-NEXT:    s_add_i32 s4, s4, 1
+; GFX10-NEXT:    s_xor_b32 s6, s6, s2
+; GFX10-NEXT:    s_sub_i32 s6, s6, s2
 ; GFX10-NEXT:    v_mov_b32_e32 v1, s6
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_store_dword v0, v1, s[0:1]
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-NEXT:    s_add_u32 s0, s0, 4
 ; GFX10-NEXT:    s_addc_u32 s1, s1, 0
-; GFX10-NEXT:    s_cmpk_eq_i32 s5, 0x400
+; GFX10-NEXT:    s_cmpk_eq_i32 s4, 0x400
 ; GFX10-NEXT:    s_cbranch_scc0 .LBB2_1
 ; GFX10-NEXT:  ; %bb.2: ; %bb2
 ; GFX10-NEXT:    s_endpgm
@@ -422,53 +422,53 @@ define amdgpu_kernel void @sdiv32_invariant_denom(ptr addrspace(1) nocapture %ar
 ; GFX11-LABEL: sdiv32_invariant_denom:
 ; GFX11:       ; %bb.0: ; %bb
 ; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    s_load_b32 s2, s[0:1], 0x2c
+; GFX11-NEXT:    s_load_b32 s3, s[0:1], 0x2c
 ; GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX11-NEXT:    s_ashr_i32 s3, s2, 31
+; GFX11-NEXT:    s_ashr_i32 s2, s3, 31
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT:    s_add_i32 s2, s2, s3
-; GFX11-NEXT:    s_xor_b32 s4, s2, s3
+; GFX11-NEXT:    s_add_i32 s3, s3, s2
+; GFX11-NEXT:    s_xor_b32 s3, s3, s2
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_cvt_f32_u32_e32 v0, s4
-; GFX11-NEXT:    s_sub_i32 s5, 0, s4
+; GFX11-NEXT:    v_cvt_f32_u32_e32 v0, s3
+; GFX11-NEXT:    s_sub_i32 s4, 0, s3
 ; GFX11-NEXT:    v_rcp_iflag_f32_e32 v0, v0
 ; GFX11-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX11-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GFX11-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX11-NEXT:    v_readfirstlane_b32 s5, v0
 ; GFX11-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT:    s_mul_i32 s5, s5, s2
-; GFX11-NEXT:    s_mul_hi_u32 s6, s2, s5
-; GFX11-NEXT:    s_mov_b32 s5, 0
-; GFX11-NEXT:    s_add_i32 s2, s2, s6
+; GFX11-NEXT:    s_mul_i32 s4, s4, s5
+; GFX11-NEXT:    s_mul_hi_u32 s6, s5, s4
+; GFX11-NEXT:    s_mov_b32 s4, 0
+; GFX11-NEXT:    s_add_i32 s5, s5, s6
 ; GFX11-NEXT:    .p2align 6
 ; GFX11-NEXT:  .LBB2_1: ; %bb3
 ; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT:    s_mul_hi_u32 s6, s5, s2
-; GFX11-NEXT:    s_mul_i32 s7, s6, s4
+; GFX11-NEXT:    s_mul_hi_u32 s6, s4, s5
+; GFX11-NEXT:    s_mul_i32 s7, s6, s3
 ; GFX11-NEXT:    s_add_i32 s8, s6, 1
-; GFX11-NEXT:    s_sub_i32 s7, s5, s7
+; GFX11-NEXT:    s_sub_i32 s7, s4, s7
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT:    s_sub_i32 s9, s7, s4
-; GFX11-NEXT:    s_cmp_ge_u32 s7, s4
+; GFX11-NEXT:    s_sub_i32 s9, s7, s3
+; GFX11-NEXT:    s_cmp_ge_u32 s7, s3
 ; GFX11-NEXT:    s_cselect_b32 s6, s8, s6
 ; GFX11-NEXT:    s_cselect_b32 s7, s9, s7
 ; GFX11-NEXT:    s_add_i32 s8, s6, 1
-; GFX11-NEXT:    s_cmp_ge_u32 s7, s4
+; GFX11-NEXT:    s_cmp_ge_u32 s7, s3
 ; GFX11-NEXT:    s_cselect_b32 s6, s8, s6
-; GFX11-NEXT:    s_add_i32 s5, s5, 1
-; GFX11-NEXT:    s_xor_b32 s6, s6, s3
+; GFX11-NEXT:    s_add_i32 s4, s4, 1
+; GFX11-NEXT:    s_xor_b32 s6, s6, s2
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT:    s_sub_i32 s6, s6, s3
+; GFX11-NEXT:    s_sub_i32 s6, s6, s2
 ; GFX11-NEXT:    v_mov_b32_e32 v1, s6
 ; GFX11-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX11-NEXT:    s_add_u32 s0, s0, 4
 ; GFX11-NEXT:    s_addc_u32 s1, s1, 0
-; GFX11-NEXT:    s_cmpk_eq_i32 s5, 0x400
+; GFX11-NEXT:    s_cmpk_eq_i32 s4, 0x400
 ; GFX11-NEXT:    s_cbranch_scc0 .LBB2_1
 ; GFX11-NEXT:  ; %bb.2: ; %bb2
 ; GFX11-NEXT:    s_nop 0
@@ -495,39 +495,40 @@ define amdgpu_kernel void @srem32_invariant_denom(ptr addrspace(1) nocapture %ar
 ; GFX9-LABEL: srem32_invariant_denom:
 ; GFX9:       ; %bb.0: ; %bb
 ; GFX9-NEXT:    s_load_dword s2, s[0:1], 0x2c
-; GFX9-NEXT:    s_mov_b32 s4, 0
-; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    s_ashr_i32 s3, s2, 31
 ; GFX9-NEXT:    s_add_i32 s2, s2, s3
-; GFX9-NEXT:    s_xor_b32 s3, s2, s3
-; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s3
-; GFX9-NEXT:    s_sub_i32 s2, 0, s3
+; GFX9-NEXT:    s_xor_b32 s2, s2, s3
+; GFX9-NEXT:    v_cvt_f32_u32_e32 v0, s2
+; GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-NEXT:    s_sub_i32 s4, 0, s2
+; GFX9-NEXT:    s_mov_b32 s3, 0
 ; GFX9-NEXT:    v_rcp_iflag_f32_e32 v0, v0
 ; GFX9-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GFX9-NEXT:    v_cvt_u32_f32_e32 v0, v0
 ; GFX9-NEXT:    v_readfirstlane_b32 s5, v0
-; GFX9-NEXT:    s_mul_i32 s2, s2, s5
-; GFX9-NEXT:    s_mul_hi_u32 s2, s5, s2
-; GFX9-NEXT:    s_add_i32 s2, s5, s2
+; GFX9-NEXT:    s_mul_i32 s4, s4, s5
+; GFX9-NEXT:    s_mul_hi_u32 s4, s5, s4
+; GFX9-NEXT:    s_add_i32 s4, s5, s4
 ; GFX9-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX9-NEXT:  .LBB3_1: ; %bb3
 ; GFX9-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT:    s_mul_hi_u32 s5, s4, s2
-; GFX9-NEXT:    s_mul_i32 s5, s5, s3
-; GFX9-NEXT:    s_sub_i32 s5, s4, s5
-; GFX9-NEXT:    s_sub_i32 s6, s5, s3
-; GFX9-NEXT:    s_cmp_ge_u32 s5, s3
+; GFX9-NEXT:    s_mul_hi_u32 s5, s3, s4
+; GFX9-NEXT:    s_mul_i32 s5, s5, s2
+; GFX9-NEXT:    s_sub_i32 s5, s3, s5
+; GFX9-NEXT:    s_sub_i32 s6, s5, s2
+; GFX9-NEXT:    s_cmp_ge_u32 s5, s2
 ; GFX9-NEXT:    s_cselect_b32 s5, s6, s5
-; GFX9-NEXT:    s_sub_i32 s6, s5, s3
-; GFX9-NEXT:    s_cmp_ge_u32 s5, s3
+; GFX9-NEXT:    s_sub_i32 s6, s5, s2
+; GFX9-NEXT:    s_cmp_ge_u32 s5, s2
 ; GFX9-NEXT:    s_cselect_b32 s5, s6, s5
-; GFX9-NEXT:    s_add_i32 s4, s4, 1
+; GFX9-NEXT:    s_add_i32 s3, s3, 1
 ; GFX9-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-NEXT:    global_store_dword v0, v1, s[0:1]
 ; GFX9-NEXT:    s_add_u32 s0, s0, 4
 ; GFX9-NEXT:    s_addc_u32 s1, s1, 0
-; GFX9-NEXT:    s_cmpk_eq_i32 s4, 0x400
+; GFX9-NEXT:    s_cmpk_eq_i32 s3, 0x400
 ; GFX9-NEXT:    s_cbranch_scc0 .LBB3_1
 ; GFX9-NEXT:  ; %bb.2: ; %bb2
 ; GFX9-NEXT:    s_endpgm
@@ -539,37 +540,37 @@ define amdgpu_kernel void @srem32_invariant_denom(ptr addrspace(1) nocapture %ar
 ; GFX10-NEXT:    s_ashr_i32 s3, s2, 31
 ; GFX10-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
 ; GFX10-NEXT:    s_add_i32 s2, s2, s3
-; GFX10-NEXT:    s_xor_b32 s3, s2, s3
-; GFX10-NEXT:    v_cvt_f32_u32_e32 v0, s3
-; GFX10-NEXT:    s_sub_i32 s4, 0, s3
+; GFX10-NEXT:    s_xor_b32 s2, s2, s3
+; GFX10-NEXT:    v_cvt_f32_u32_e32 v0, s2
+; GFX10-NEXT:    s_sub_i32 s3, 0, s2
 ; GFX10-NEXT:    v_rcp_iflag_f32_e32 v0, v0
 ; GFX10-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GFX10-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GFX10-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX10-NEXT:    v_readfirstlane_b32 s4, v0
 ; GFX10-NEXT:    v_mov_b32_e32 v0, 0
-; GFX10-NEXT:    s_mul_i32 s4, s4, s2
-; GFX10-NEXT:    s_mul_hi_u32 s5, s2, s4
-; GFX10-NEXT:    s_mov_b32 s4, 0
-; GFX10-NEXT:    s_add_i32 s2, s2, s5
+; GFX10-NEXT:    s_mul_i32 s3, s3, s4
+; GFX10-NEXT:    s_mul_hi_u32 s5, s4, s3
+; GFX10-NEXT:    s_mov_b32 s3, 0
+; GFX10-NEXT:    s_add_i32 s4, s4, s5
 ; GFX10-NEXT:  .LBB3_1: ; %bb3
 ; GFX10-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT:    s_mul_hi_u32 s5, s4, s2
-; GFX10-NEXT:    s_mul_i32 s5, s5, s3
-; GFX10-NEXT:    s_sub_i32 s5, s4, s5
-; GFX10-NEXT:    s_sub_i32 s6, s5, s3
-; GFX10-NEXT:    s_cmp_ge_u32 s5, s3
+; GFX10-NEXT:    s_mul_hi_u32 s5, s3, s4
+; GFX10-NEXT:    s_mul_i32 s5, s5, s2
+; GFX10-NEXT:    s_sub_i32 s5, s3, s5
+; GFX10-NEXT:    s_sub_i32 s6, s5, s2
+; GFX10-NEXT:    s_cmp_ge_u32 s5, s2
 ; GFX10-NEXT:    s_cselect_b32 s5, s6, s5
-; GFX10-NEXT:    s_sub_i32 s6, s5, s3
-; GFX10-NEXT:    s_cmp_ge_u32 s5, s3
+; GFX10-NEXT:    s_sub_i32 s6, s5, s2
+; GFX10-NEXT:    s_cmp_ge_u32 s5, s2
 ; GFX10-NEXT:    s_cselect_b32 s5, s6, s5
-; GFX10-NEXT:    s_add_i32 s4, s4, 1
+; GFX10-NEXT:    s_add_i32 s3, s3, 1
 ; GFX10-NEXT:    v_mov_b32_e32 v1, s5
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    global_store_dword v0, v1, s[0:1]
 ; GFX10-NEXT:    s_waitcnt_depctr 0xffe3
 ; GFX10-NEXT:    s_add_u32 s0, s0, 4
 ; GFX10-NEXT:    s_addc_u32 s1, s1, 0
-; GFX10-NEXT:    s_cmpk_eq_i32 s4, 0x400
+; GFX10-NEXT:    s_cmpk_eq_i32 s3, 0x400
 ; GFX10-NEXT:    s_cbranch_scc0 .LBB3_1
 ; GFX10-NEXT:  ; %bb.2: ; %bb2
 ; GFX10-NEXT:    s_endpgm
@@ -583,43 +584,43 @@ define amdgpu_kernel void @srem32_invariant_denom(ptr addrspace(1) nocapture %ar
 ; GFX11-NEXT:    s_ashr_i32 s3, s2, 31
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
 ; GFX11-NEXT:    s_add_i32 s2, s2, s3
-; GFX11-NEXT:    s_xor_b32 s3, s2, s3
+; GFX11-NEXT:    s_xor_b32 s2, s2, s3
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_cvt_f32_u32_e32 v0, s3
-; GFX11-NEXT:    s_sub_i32 s4, 0, s3
+; GFX11-NEXT:    v_cvt_f32_u32_e32 v0, s2
+; GFX11-NEXT:    s_sub_i32 s3, 0, s2
 ; GFX11-NEXT:    v_rcp_iflag_f32_e32 v0, v0
 ; GFX11-NEXT:    s_waitcnt_depctr 0xfff
 ; GFX11-NEXT:    v_mul_f32_e32 v0, 0x4f7ffffe, v0
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-NEXT:    v_cvt_u32_f32_e32 v0, v0
-; GFX11-NEXT:    v_readfirstlane_b32 s2, v0
+; GFX11-NEXT:    v_readfirstlane_b32 s4, v0
 ; GFX11-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT:    s_mul_i32 s4, s4, s2
-; GFX11-NEXT:    s_mul_hi_u32 s5, s2, s4
-; GFX11-NEXT:    s_mov_b32 s4, 0
-; GFX11-NEXT:    s_add_i32 s2, s2, s5
+; GFX11-NEXT:    s_mul_i32 s3, s3, s4
+; GFX11-NEXT:    s_mul_hi_u32 s5, s4, s3
+; GFX11-NEXT:    s_mov_b32 s3, 0
+; GFX11-NEXT:    s_add_i32 s4, s4, s5
 ; GFX11-NEXT:    .p2align 6
 ; GFX11-NEXT:  .LBB3_1: ; %bb3
 ; GFX11-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT:    s_mul_hi_u32 s5, s4, s2
-; GFX11-NEXT:    s_mul_i32 s5, s5, s3
+; GFX11-NEXT:    s_mul_hi_u32 s5, s3, s4
+; GFX11-NEXT:    s_mul_i32 s5, s5, s2
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT:    s_sub_i32 s5, s4, s5
-; GFX11-NEXT:    s_sub_i32 s6, s5, s3
-; GFX11-NEXT:    s_cmp_ge_u32 s5, s3
+; GFX11-NEXT:    s_sub_i32 s5, s3, s5
+; GFX11-NEXT:    s_sub_i32 s6, s5, s2
+; GFX11-NEXT:    s_cmp_ge_u32 s5, s2
 ; GFX11-NEXT:    s_cselect_b32 s5, s6, s5
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT:    s_sub_i32 s6, s5, s3
-; GFX11-NEXT:    s_cmp_ge_u32 s5, s3
+; GFX11-NEXT:    s_sub_i32 s6, s5, s2
+; GFX11-NEXT:    s_cmp_ge_u32 s5, s2
 ; GFX11-NEXT:    s_cselect_b32 s5, s6, s5
-; GFX11-NEXT:    s_add_i32 s4, s4, 1
+; GFX11-NEXT:    s_add_i32 s3, s3, 1
 ; GFX11-NEXT:    v_mov_b32_e32 v1, s5
 ; GFX11-NEXT:    global_store_b32 v0, v1, s[0:1]
 ; GFX11-NEXT:    s_add_u32 s0, s0, 4
 ; GFX11-NEXT:    s_addc_u32 s1, s1, 0
-; GFX11-NEXT:    s_cmpk_eq_i32 s4, 0x400
+; GFX11-NEXT:    s_cmpk_eq_i32 s3, 0x400
 ; GFX11-NEXT:    s_cbranch_scc0 .LBB3_1
 ; GFX11-NEXT:  ; %bb.2: ; %bb2
 ; GFX11-NEXT:    s_nop 0
diff --git a/llvm/test/CodeGen/AMDGPU/indirect-addressing-term.ll b/llvm/test/CodeGen/AMDGPU/indirect-addressing-term.ll
index b03d705b43e97ef1c782a427e34dd0ce6d4fc1ae..ceaba61da6fbba81593a9c8d1d59bb0ed1976917 100644
--- a/llvm/test/CodeGen/AMDGPU/indirect-addressing-term.ll
+++ b/llvm/test/CodeGen/AMDGPU/indirect-addressing-term.ll
@@ -56,22 +56,22 @@ define amdgpu_kernel void @extract_w_offset_vgpr(ptr addrspace(1) %out) {
   ; GCN-NEXT:   [[COPY14:%[0-9]+]]:vgpr_32 = COPY killed renamable $sgpr2
   ; GCN-NEXT:   [[COPY15:%[0-9]+]]:vgpr_32 = COPY killed renamable $sgpr1
   ; GCN-NEXT:   [[COPY16:%[0-9]+]]:vgpr_32 = COPY killed renamable $sgpr0
-  ; GCN-NEXT:   undef %35.sub0:vreg_512 = COPY [[COPY1]]
-  ; GCN-NEXT:   %35.sub1:vreg_512 = COPY [[COPY2]]
-  ; GCN-NEXT:   %35.sub2:vreg_512 = COPY [[COPY3]]
-  ; GCN-NEXT:   %35.sub3:vreg_512 = COPY [[COPY4]]
-  ; GCN-NEXT:   %35.sub4:vreg_512 = COPY [[COPY5]]
-  ; GCN-NEXT:   %35.sub5:vreg_512 = COPY [[COPY6]]
-  ; GCN-NEXT:   %35.sub6:vreg_512 = COPY [[COPY7]]
-  ; GCN-NEXT:   %35.sub7:vreg_512 = COPY [[COPY8]]
-  ; GCN-NEXT:   %35.sub8:vreg_512 = COPY [[COPY9]]
-  ; GCN-NEXT:   %35.sub9:vreg_512 = COPY [[COPY10]]
-  ; GCN-NEXT:   %35.sub10:vreg_512 = COPY [[COPY11]]
-  ; GCN-NEXT:   %35.sub11:vreg_512 = COPY [[COPY12]]
-  ; GCN-NEXT:   %35.sub12:vreg_512 = COPY [[COPY13]]
-  ; GCN-NEXT:   %35.sub13:vreg_512 = COPY [[COPY14]]
-  ; GCN-NEXT:   %35.sub14:vreg_512 = COPY [[COPY15]]
-  ; GCN-NEXT:   %35.sub15:vreg_512 = COPY [[COPY16]]
+  ; GCN-NEXT:   undef [[COPY17:%[0-9]+]].sub0:vreg_512 = COPY [[COPY1]]
+  ; GCN-NEXT:   [[COPY17:%[0-9]+]].sub1:vreg_512 = COPY [[COPY2]]
+  ; GCN-NEXT:   [[COPY17:%[0-9]+]].sub2:vreg_512 = COPY [[COPY3]]
+  ; GCN-NEXT:   [[COPY17:%[0-9]+]].sub3:vreg_512 = COPY [[COPY4]]
+  ; GCN-NEXT:   [[COPY17:%[0-9]+]].sub4:vreg_512 = COPY [[COPY5]]
+  ; GCN-NEXT:   [[COPY17:%[0-9]+]].sub5:vreg_512 = COPY [[COPY6]]
+  ; GCN-NEXT:   [[COPY17:%[0-9]+]].sub6:vreg_512 = COPY [[COPY7]]
+  ; GCN-NEXT:   [[COPY17:%[0-9]+]].sub7:vreg_512 = COPY [[COPY8]]
+  ; GCN-NEXT:   [[COPY17:%[0-9]+]].sub8:vreg_512 = COPY [[COPY9]]
+  ; GCN-NEXT:   [[COPY17:%[0-9]+]].sub9:vreg_512 = COPY [[COPY10]]
+  ; GCN-NEXT:   [[COPY17:%[0-9]+]].sub10:vreg_512 = COPY [[COPY11]]
+  ; GCN-NEXT:   [[COPY17:%[0-9]+]].sub11:vreg_512 = COPY [[COPY12]]
+  ; GCN-NEXT:   [[COPY17:%[0-9]+]].sub12:vreg_512 = COPY [[COPY13]]
+  ; GCN-NEXT:   [[COPY17:%[0-9]+]].sub13:vreg_512 = COPY [[COPY14]]
+  ; GCN-NEXT:   [[COPY17:%[0-9]+]].sub14:vreg_512 = COPY [[COPY15]]
+  ; GCN-NEXT:   [[COPY17:%[0-9]+]].sub15:vreg_512 = COPY [[COPY16]]
   ; GCN-NEXT:   renamable $sgpr0_sgpr1 = S_MOV_B64 $exec
   ; GCN-NEXT:   SI_SPILL_S64_SAVE killed $sgpr0_sgpr1, %stack.0, implicit $exec, implicit $sgpr32 :: (store (s64) into %stack.0, align 4, addrspace 5)
   ; GCN-NEXT:   [[DEF:%[0-9]+]]:vgpr_32 = IMPLICIT_DEF
@@ -81,12 +81,12 @@ define amdgpu_kernel void @extract_w_offset_vgpr(ptr addrspace(1) %out) {
   ; GCN-NEXT:   successors: %bb.1(0x40000000), %bb.3(0x40000000)
   ; GCN-NEXT: {{  $}}
   ; GCN-NEXT:   $sgpr0_sgpr1 = SI_SPILL_S64_RESTORE %stack.2, implicit $exec, implicit $sgpr32 :: (load (s64) from %stack.2, align 4, addrspace 5)
-  ; GCN-NEXT:   dead [[COPY17:%[0-9]+]]:vgpr_32 = COPY [[DEF]]
+  ; GCN-NEXT:   dead [[COPY18:%[0-9]+]]:vgpr_32 = COPY [[DEF]]
   ; GCN-NEXT:   renamable $sgpr2 = V_READFIRSTLANE_B32 [[COPY]](s32), implicit $exec
   ; GCN-NEXT:   renamable $sgpr0_sgpr1 = V_CMP_EQ_U32_e64 $sgpr2, [[COPY]](s32), implicit $exec
   ; GCN-NEXT:   renamable $sgpr0_sgpr1 = S_AND_SAVEEXEC_B64 killed renamable $sgpr0_sgpr1, implicit-def $exec, implicit-def dead $scc, implicit $exec
-  ; GCN-NEXT:   [[V_INDIRECT_REG_READ_GPR_IDX_B32_V16_:%[0-9]+]]:vgpr_32 = V_INDIRECT_REG_READ_GPR_IDX_B32_V16 %35, killed $sgpr2, 11, implicit-def $m0, implicit $m0, implicit $exec
-  ; GCN-NEXT:   [[COPY18:%[0-9]+]]:vgpr_32 = COPY [[V_INDIRECT_REG_READ_GPR_IDX_B32_V16_]]
+  ; GCN-NEXT:   [[V_INDIRECT_REG_READ_GPR_IDX_B32_V16_:%[0-9]+]]:vgpr_32 = V_INDIRECT_REG_READ_GPR_IDX_B32_V16 [[COPY17]], killed $sgpr2, 11, implicit-def $m0, implicit $m0, implicit $exec
+  ; GCN-NEXT:   [[DEF:%[0-9]+]]:vgpr_32 = COPY [[V_INDIRECT_REG_READ_GPR_IDX_B32_V16_]]
   ; GCN-NEXT:   renamable $sgpr2_sgpr3 = COPY renamable $sgpr0_sgpr1
   ; GCN-NEXT:   SI_SPILL_S64_SAVE killed $sgpr2_sgpr3, %stack.2, implicit $exec, implicit $sgpr32 :: (store (s64) into %stack.2, align 4, addrspace 5)
   ; GCN-NEXT:   $exec = S_XOR_B64_term $exec, killed renamable $sgpr0_sgpr1, implicit-def dead $scc
diff --git a/llvm/test/CodeGen/AMDGPU/lds-dma-waits.ll b/llvm/test/CodeGen/AMDGPU/lds-dma-waits.ll
index 688f0314f7d8f4b3d84cf415584ae3891dc026a9..58c6406fac7aa12324ce1d68e502677a32bfc5aa 100644
--- a/llvm/test/CodeGen/AMDGPU/lds-dma-waits.ll
+++ b/llvm/test/CodeGen/AMDGPU/lds-dma-waits.ll
@@ -9,15 +9,10 @@ declare void @llvm.amdgcn.global.load.lds(ptr addrspace(1) nocapture %gptr, ptr
 
 ; FIXME: vmcnt(0) is too strong, it shall use vmcnt(2) before the first
 ;        ds_read_b32 and vmcnt(0) before the second.
-; FIXME: GFX10 does not get a waitcount at all.
 
 ; GCN-LABEL: {{^}}buffer_load_lds_dword_2_arrays:
 ; GCN-COUNT-4: buffer_load_dword
-; GFX9: s_waitcnt vmcnt(0)
-
-; FIXME:
-; GFX10-NOT: s_waitcnt
-
+; GCN: s_waitcnt vmcnt(0)
 ; GCN: ds_read_b32
 
 ; FIXME:
@@ -49,9 +44,9 @@ main_body:
 ; GFX9: s_waitcnt vmcnt(0)
 ; GFX9-COUNT-2: ds_read_b32
 
-; FIXME:
-; GFX10-NOT: s_waitcnt
+; FIXME: can be vmcnt(2)
 
+; GFX10: s_waitcnt vmcnt(0)
 ; GFX10: ds_read_b32
 
 ; FIXME:
diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
index 7abb789019f1f0cc2eb8e72bfc3a58ff2e3ccade..488dbe2e3189bf8b7f552d86b88d30ec02dfdcd5 100644
--- a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
+++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
@@ -334,6 +334,7 @@
 ; GCN-O1-NEXT:        Machine Natural Loop Construction
 ; GCN-O1-NEXT:        Register Coalescer
 ; GCN-O1-NEXT:        Rename Disconnected Subregister Components
+; GCN-O1-NEXT:        Rewrite Partial Register Uses
 ; GCN-O1-NEXT:        Machine Instruction Scheduler
 ; GCN-O1-NEXT:        SI Whole Quad Mode
 ; GCN-O1-NEXT:        SI optimize exec mask operations pre-RA
@@ -622,6 +623,7 @@
 ; GCN-O1-OPTS-NEXT:        Machine Natural Loop Construction
 ; GCN-O1-OPTS-NEXT:        Register Coalescer
 ; GCN-O1-OPTS-NEXT:        Rename Disconnected Subregister Components
+; GCN-O1-OPTS-NEXT:        Rewrite Partial Register Uses
 ; GCN-O1-OPTS-NEXT:        AMDGPU Pre-RA optimizations
 ; GCN-O1-OPTS-NEXT:        Machine Instruction Scheduler
 ; GCN-O1-OPTS-NEXT:        SI Whole Quad Mode
@@ -922,6 +924,7 @@
 ; GCN-O2-NEXT:        Machine Natural Loop Construction
 ; GCN-O2-NEXT:        Register Coalescer
 ; GCN-O2-NEXT:        Rename Disconnected Subregister Components
+; GCN-O2-NEXT:        Rewrite Partial Register Uses
 ; GCN-O2-NEXT:        AMDGPU Pre-RA optimizations
 ; GCN-O2-NEXT:        Machine Instruction Scheduler
 ; GCN-O2-NEXT:        SI Whole Quad Mode
@@ -1235,6 +1238,7 @@
 ; GCN-O3-NEXT:        Machine Natural Loop Construction
 ; GCN-O3-NEXT:        Register Coalescer
 ; GCN-O3-NEXT:        Rename Disconnected Subregister Components
+; GCN-O3-NEXT:        Rewrite Partial Register Uses
 ; GCN-O3-NEXT:        AMDGPU Pre-RA optimizations
 ; GCN-O3-NEXT:        Machine Instruction Scheduler
 ; GCN-O3-NEXT:        SI Whole Quad Mode
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.barrier.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.barrier.ll
index 48c4e0276edda3a7beaf5872edea6a9a2880543f..4e65b376339490654a02a7f88833aa11bb99e284 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.barrier.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.barrier.ll
@@ -3,6 +3,9 @@
 ; RUN: llc -march=amdgcn -mattr=+auto-waitcnt-before-barrier -verify-machineinstrs < %s | FileCheck --check-prefix=VARIANT1 %s
 ; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck --check-prefix=VARIANT2 %s
 ; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=+auto-waitcnt-before-barrier -verify-machineinstrs < %s | FileCheck --check-prefix=VARIANT3 %s
+; RUN: llc -march=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck --check-prefix=VARIANT4 %s
+; RUN: llc -march=amdgcn -mcpu=gfx1200 -mattr=+auto-waitcnt-before-barrier -verify-machineinstrs < %s | FileCheck --check-prefix=VARIANT5 %s
+; RUN: llc -global-isel=1 -march=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck --check-prefix=VARIANT6 %s
 
 define amdgpu_kernel void @test_barrier(ptr addrspace(1) %out, i32 %size) #0 {
 ; VARIANT0-LABEL: test_barrier:
@@ -85,6 +88,80 @@ define amdgpu_kernel void @test_barrier(ptr addrspace(1) %out, i32 %size) #0 {
 ; VARIANT3-NEXT:    s_waitcnt vmcnt(0)
 ; VARIANT3-NEXT:    global_store_dword v2, v0, s[2:3]
 ; VARIANT3-NEXT:    s_endpgm
+;
+; VARIANT4-LABEL: test_barrier:
+; VARIANT4:       ; %bb.0: ; %entry
+; VARIANT4-NEXT:    s_clause 0x1
+; VARIANT4-NEXT:    s_load_b32 s2, s[0:1], 0x2c
+; VARIANT4-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; VARIANT4-NEXT:    v_lshlrev_b32_e32 v3, 2, v0
+; VARIANT4-NEXT:    s_waitcnt lgkmcnt(0)
+; VARIANT4-NEXT:    v_xad_u32 v1, v0, -1, s2
+; VARIANT4-NEXT:    global_store_b32 v3, v0, s[0:1]
+; VARIANT4-NEXT:    s_barrier_signal -1
+; VARIANT4-NEXT:    s_barrier_wait -1
+; VARIANT4-NEXT:    v_ashrrev_i32_e32 v2, 31, v1
+; VARIANT4-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; VARIANT4-NEXT:    v_lshlrev_b64_e32 v[1:2], 2, v[1:2]
+; VARIANT4-NEXT:    v_add_co_u32 v1, vcc_lo, s0, v1
+; VARIANT4-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; VARIANT4-NEXT:    v_add_co_ci_u32_e32 v2, vcc_lo, s1, v2, vcc_lo
+; VARIANT4-NEXT:    global_load_b32 v0, v[1:2], off
+; VARIANT4-NEXT:    s_waitcnt vmcnt(0)
+; VARIANT4-NEXT:    global_store_b32 v3, v0, s[0:1]
+; VARIANT4-NEXT:    s_nop 0
+; VARIANT4-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; VARIANT4-NEXT:    s_endpgm
+;
+; VARIANT5-LABEL: test_barrier:
+; VARIANT5:       ; %bb.0: ; %entry
+; VARIANT5-NEXT:    s_clause 0x1
+; VARIANT5-NEXT:    s_load_b32 s2, s[0:1], 0x2c
+; VARIANT5-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; VARIANT5-NEXT:    v_lshlrev_b32_e32 v3, 2, v0
+; VARIANT5-NEXT:    s_waitcnt lgkmcnt(0)
+; VARIANT5-NEXT:    v_xad_u32 v1, v0, -1, s2
+; VARIANT5-NEXT:    global_store_b32 v3, v0, s[0:1]
+; VARIANT5-NEXT:    s_barrier_signal -1
+; VARIANT5-NEXT:    s_barrier_wait -1
+; VARIANT5-NEXT:    v_ashrrev_i32_e32 v2, 31, v1
+; VARIANT5-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; VARIANT5-NEXT:    v_lshlrev_b64_e32 v[1:2], 2, v[1:2]
+; VARIANT5-NEXT:    v_add_co_u32 v1, vcc_lo, s0, v1
+; VARIANT5-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; VARIANT5-NEXT:    v_add_co_ci_u32_e32 v2, vcc_lo, s1, v2, vcc_lo
+; VARIANT5-NEXT:    global_load_b32 v0, v[1:2], off
+; VARIANT5-NEXT:    s_waitcnt vmcnt(0)
+; VARIANT5-NEXT:    global_store_b32 v3, v0, s[0:1]
+; VARIANT5-NEXT:    s_nop 0
+; VARIANT5-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; VARIANT5-NEXT:    s_endpgm
+;
+; VARIANT6-LABEL: test_barrier:
+; VARIANT6:       ; %bb.0: ; %entry
+; VARIANT6-NEXT:    s_clause 0x1
+; VARIANT6-NEXT:    s_load_b32 s2, s[0:1], 0x2c
+; VARIANT6-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; VARIANT6-NEXT:    v_lshlrev_b32_e32 v5, 2, v0
+; VARIANT6-NEXT:    s_waitcnt lgkmcnt(0)
+; VARIANT6-NEXT:    s_sub_co_i32 s2, s2, 1
+; VARIANT6-NEXT:    v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v3, s0
+; VARIANT6-NEXT:    v_sub_nc_u32_e32 v1, s2, v0
+; VARIANT6-NEXT:    global_store_b32 v5, v0, s[0:1]
+; VARIANT6-NEXT:    s_barrier_signal -1
+; VARIANT6-NEXT:    s_barrier_wait -1
+; VARIANT6-NEXT:    v_ashrrev_i32_e32 v2, 31, v1
+; VARIANT6-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; VARIANT6-NEXT:    v_lshlrev_b64_e32 v[1:2], 2, v[1:2]
+; VARIANT6-NEXT:    v_add_co_u32 v1, vcc_lo, v3, v1
+; VARIANT6-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; VARIANT6-NEXT:    v_add_co_ci_u32_e32 v2, vcc_lo, v4, v2, vcc_lo
+; VARIANT6-NEXT:    global_load_b32 v0, v[1:2], off
+; VARIANT6-NEXT:    s_waitcnt vmcnt(0)
+; VARIANT6-NEXT:    global_store_b32 v5, v0, s[0:1]
+; VARIANT6-NEXT:    s_nop 0
+; VARIANT6-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; VARIANT6-NEXT:    s_endpgm
 entry:
   %tmp = call i32 @llvm.amdgcn.workitem.id.x()
   %tmp1 = getelementptr i32, ptr addrspace(1) %out, i32 %tmp
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.barrier.wait.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.barrier.wait.ll
new file mode 100644
index 0000000000000000000000000000000000000000..1ad3e58ce7fc35044d78509c7e008d5b1bc2108c
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.barrier.wait.ll
@@ -0,0 +1,1366 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -global-isel=0 -march=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN %s
+; RUN: llc -global-isel=1 -march=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck -check-prefixes=GLOBAL-ISEL %s
+
+define amdgpu_kernel void @test1_s_barrier_signal(ptr addrspace(1) %out) #0 {
+; GCN-LABEL: test1_s_barrier_signal:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GCN-NEXT:    v_mul_u32_u24_e32 v1, v0, v0
+; GCN-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_lshlrev_b32 v3, 2, v0
+; GCN-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GCN-NEXT:    v_sub_nc_u32_e32 v0, v1, v0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    global_store_b32 v3, v2, s[0:1]
+; GCN-NEXT:    s_barrier_signal -1
+; GCN-NEXT:    s_barrier_wait -1
+; GCN-NEXT:    global_store_b32 v3, v0, s[0:1]
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GCN-NEXT:    s_endpgm
+;
+; GLOBAL-ISEL-LABEL: test1_s_barrier_signal:
+; GLOBAL-ISEL:       ; %bb.0: ; %entry
+; GLOBAL-ISEL-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GLOBAL-ISEL-NEXT:    v_mul_lo_u32 v1, v0, v0
+; GLOBAL-ISEL-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_lshlrev_b32 v3, 2, v0
+; GLOBAL-ISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GLOBAL-ISEL-NEXT:    v_sub_nc_u32_e32 v0, v1, v0
+; GLOBAL-ISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GLOBAL-ISEL-NEXT:    global_store_b32 v3, v2, s[0:1]
+; GLOBAL-ISEL-NEXT:    s_barrier_signal -1
+; GLOBAL-ISEL-NEXT:    s_barrier_wait -1
+; GLOBAL-ISEL-NEXT:    global_store_b32 v3, v0, s[0:1]
+; GLOBAL-ISEL-NEXT:    s_nop 0
+; GLOBAL-ISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GLOBAL-ISEL-NEXT:    s_endpgm
+entry:
+  %tmp = call i32 @llvm.amdgcn.workitem.id.x()
+  %tmp1 = getelementptr i32, ptr addrspace(1) %out, i32 %tmp
+  store i32 0, ptr addrspace(1) %tmp1
+  call void @llvm.amdgcn.s.barrier.signal(i32 -1)
+  call void @llvm.amdgcn.s.barrier.wait(i16 -1)
+  %tmp3 = mul i32 %tmp, %tmp
+  %tmp4 = sub i32 %tmp3, %tmp
+  store i32 %tmp4, ptr addrspace(1) %tmp1
+  ret void
+}
+
+define amdgpu_kernel void @test2_s_barrier_signal(ptr addrspace(1) %out) #0 {
+; GCN-LABEL: test2_s_barrier_signal:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GCN-NEXT:    v_mul_u32_u24_e32 v1, v0, v0
+; GCN-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_lshlrev_b32 v3, 2, v0
+; GCN-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GCN-NEXT:    v_sub_nc_u32_e32 v0, v1, v0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    global_store_b32 v3, v2, s[0:1]
+; GCN-NEXT:    s_barrier_signal 1
+; GCN-NEXT:    s_barrier_wait 1
+; GCN-NEXT:    global_store_b32 v3, v0, s[0:1]
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GCN-NEXT:    s_endpgm
+;
+; GLOBAL-ISEL-LABEL: test2_s_barrier_signal:
+; GLOBAL-ISEL:       ; %bb.0: ; %entry
+; GLOBAL-ISEL-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GLOBAL-ISEL-NEXT:    v_mul_lo_u32 v1, v0, v0
+; GLOBAL-ISEL-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_lshlrev_b32 v3, 2, v0
+; GLOBAL-ISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GLOBAL-ISEL-NEXT:    v_sub_nc_u32_e32 v0, v1, v0
+; GLOBAL-ISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GLOBAL-ISEL-NEXT:    global_store_b32 v3, v2, s[0:1]
+; GLOBAL-ISEL-NEXT:    s_barrier_signal 1
+; GLOBAL-ISEL-NEXT:    s_barrier_wait 1
+; GLOBAL-ISEL-NEXT:    global_store_b32 v3, v0, s[0:1]
+; GLOBAL-ISEL-NEXT:    s_nop 0
+; GLOBAL-ISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GLOBAL-ISEL-NEXT:    s_endpgm
+entry:
+  %tmp = call i32 @llvm.amdgcn.workitem.id.x()
+  %tmp1 = getelementptr i32, ptr addrspace(1) %out, i32 %tmp
+  store i32 0, ptr addrspace(1) %tmp1
+  call void @llvm.amdgcn.s.barrier.signal(i32 1)
+  call void @llvm.amdgcn.s.barrier.wait(i16 1)
+  %tmp3 = mul i32 %tmp, %tmp
+  %tmp4 = sub i32 %tmp3, %tmp
+  store i32 %tmp4, ptr addrspace(1) %tmp1
+  ret void
+}
+
+define amdgpu_kernel void @test3_s_barrier_signal(ptr addrspace(1) %out) #0 {
+; GCN-LABEL: test3_s_barrier_signal:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GCN-NEXT:    v_mul_u32_u24_e32 v1, v0, v0
+; GCN-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_lshlrev_b32 v3, 2, v0
+; GCN-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GCN-NEXT:    v_sub_nc_u32_e32 v0, v1, v0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    global_store_b32 v3, v2, s[0:1]
+; GCN-NEXT:    s_barrier_signal 0
+; GCN-NEXT:    s_barrier_wait 0
+; GCN-NEXT:    global_store_b32 v3, v0, s[0:1]
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GCN-NEXT:    s_endpgm
+;
+; GLOBAL-ISEL-LABEL: test3_s_barrier_signal:
+; GLOBAL-ISEL:       ; %bb.0: ; %entry
+; GLOBAL-ISEL-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GLOBAL-ISEL-NEXT:    v_mul_lo_u32 v1, v0, v0
+; GLOBAL-ISEL-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_lshlrev_b32 v3, 2, v0
+; GLOBAL-ISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GLOBAL-ISEL-NEXT:    v_sub_nc_u32_e32 v0, v1, v0
+; GLOBAL-ISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GLOBAL-ISEL-NEXT:    global_store_b32 v3, v2, s[0:1]
+; GLOBAL-ISEL-NEXT:    s_barrier_signal 0
+; GLOBAL-ISEL-NEXT:    s_barrier_wait 0
+; GLOBAL-ISEL-NEXT:    global_store_b32 v3, v0, s[0:1]
+; GLOBAL-ISEL-NEXT:    s_nop 0
+; GLOBAL-ISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GLOBAL-ISEL-NEXT:    s_endpgm
+entry:
+  %tmp = call i32 @llvm.amdgcn.workitem.id.x()
+  %tmp1 = getelementptr i32, ptr addrspace(1) %out, i32 %tmp
+  store i32 0, ptr addrspace(1) %tmp1
+  call void @llvm.amdgcn.s.barrier.signal(i32 0)
+  call void @llvm.amdgcn.s.barrier.wait(i16 0)
+  %tmp3 = mul i32 %tmp, %tmp
+  %tmp4 = sub i32 %tmp3, %tmp
+  store i32 %tmp4, ptr addrspace(1) %tmp1
+  ret void
+}
+
+define amdgpu_kernel void @test1_s_barrier_signal_var(ptr addrspace(1) %out) #0 {
+; GCN-LABEL: test1_s_barrier_signal_var:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GCN-NEXT:    v_mul_u32_u24_e32 v2, v0, v0
+; GCN-NEXT:    v_mov_b32_e32 v1, 0
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 2, v0
+; GCN-NEXT:    s_mov_b32 m0, 1
+; GCN-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GCN-NEXT:    v_sub_nc_u32_e32 v0, v2, v0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    global_store_b32 v3, v1, s[0:1]
+; GCN-NEXT:    s_barrier_signal m0
+; GCN-NEXT:    s_barrier_wait 1
+; GCN-NEXT:    global_store_b32 v3, v0, s[0:1]
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GCN-NEXT:    s_endpgm
+;
+; GLOBAL-ISEL-LABEL: test1_s_barrier_signal_var:
+; GLOBAL-ISEL:       ; %bb.0: ; %entry
+; GLOBAL-ISEL-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GLOBAL-ISEL-NEXT:    v_mul_lo_u32 v1, v0, v0
+; GLOBAL-ISEL-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_lshlrev_b32 v3, 2, v0
+; GLOBAL-ISEL-NEXT:    s_mov_b32 m0, 1
+; GLOBAL-ISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GLOBAL-ISEL-NEXT:    v_sub_nc_u32_e32 v0, v1, v0
+; GLOBAL-ISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GLOBAL-ISEL-NEXT:    global_store_b32 v3, v2, s[0:1]
+; GLOBAL-ISEL-NEXT:    s_barrier_signal m0
+; GLOBAL-ISEL-NEXT:    s_barrier_wait 1
+; GLOBAL-ISEL-NEXT:    global_store_b32 v3, v0, s[0:1]
+; GLOBAL-ISEL-NEXT:    s_nop 0
+; GLOBAL-ISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GLOBAL-ISEL-NEXT:    s_endpgm
+entry:
+  %tmp = call i32 @llvm.amdgcn.workitem.id.x()
+  %tmp1 = getelementptr i32, ptr addrspace(1) %out, i32 %tmp
+  store i32 0, ptr addrspace(1) %tmp1
+  call void @llvm.amdgcn.s.barrier.signal.var(i32 1)
+  call void @llvm.amdgcn.s.barrier.wait(i16 1)
+  %tmp3 = mul i32 %tmp, %tmp
+  %tmp4 = sub i32 %tmp3, %tmp
+  store i32 %tmp4, ptr addrspace(1) %tmp1
+  ret void
+}
+
+define void @test2_s_barrier_signal_var(i32 %arg) {
+; GCN-LABEL: test2_s_barrier_signal_var:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_readfirstlane_b32 s0, v0
+; GCN-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GCN-NEXT:    s_mov_b32 m0, s0
+; GCN-NEXT:    s_barrier_signal m0
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; GLOBAL-ISEL-LABEL: test2_s_barrier_signal_var:
+; GLOBAL-ISEL:       ; %bb.0:
+; GLOBAL-ISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GLOBAL-ISEL-NEXT:    v_readfirstlane_b32 m0, v0
+; GLOBAL-ISEL-NEXT:    s_barrier_signal m0
+; GLOBAL-ISEL-NEXT:    s_setpc_b64 s[30:31]
+  call void @llvm.amdgcn.s.barrier.signal.var(i32 %arg)
+  ret void
+}
+
+define amdgpu_kernel void @test1_s_barrier_signal_isfirst(ptr addrspace(1) %a, ptr addrspace(1) %b, ptr addrspace(1) %c, ptr addrspace(1) %out) #0 {
+; GCN-LABEL: test1_s_barrier_signal_isfirst:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_load_b256 s[0:7], s[0:1], 0x24
+; GCN-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_lshlrev_b32 v0, 2, v0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    global_store_b32 v0, v1, s[6:7]
+; GCN-NEXT:    s_barrier_signal_isfirst -1
+; GCN-NEXT:    s_cselect_b32 s3, s3, s5
+; GCN-NEXT:    s_cselect_b32 s2, s2, s4
+; GCN-NEXT:    s_clause 0x1
+; GCN-NEXT:    global_load_b32 v2, v1, s[0:1]
+; GCN-NEXT:    global_load_b32 v1, v1, s[2:3]
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_mul_lo_u32 v1, v1, v2
+; GCN-NEXT:    global_store_b32 v0, v1, s[6:7]
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GCN-NEXT:    s_endpgm
+;
+; GLOBAL-ISEL-LABEL: test1_s_barrier_signal_isfirst:
+; GLOBAL-ISEL:       ; %bb.0: ; %entry
+; GLOBAL-ISEL-NEXT:    s_load_b256 s[0:7], s[0:1], 0x24
+; GLOBAL-ISEL-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_lshlrev_b32 v0, 2, v0
+; GLOBAL-ISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GLOBAL-ISEL-NEXT:    global_store_b32 v0, v1, s[6:7]
+; GLOBAL-ISEL-NEXT:    s_barrier_signal_isfirst -1
+; GLOBAL-ISEL-NEXT:    s_cselect_b32 s8, 1, 0
+; GLOBAL-ISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GLOBAL-ISEL-NEXT:    s_and_b32 s8, s8, 1
+; GLOBAL-ISEL-NEXT:    s_cmp_lg_u32 s8, 0
+; GLOBAL-ISEL-NEXT:    s_cselect_b64 s[2:3], s[2:3], s[4:5]
+; GLOBAL-ISEL-NEXT:    s_clause 0x1
+; GLOBAL-ISEL-NEXT:    global_load_b32 v2, v1, s[0:1]
+; GLOBAL-ISEL-NEXT:    global_load_b32 v1, v1, s[2:3]
+; GLOBAL-ISEL-NEXT:    s_waitcnt vmcnt(0)
+; GLOBAL-ISEL-NEXT:    v_mul_lo_u32 v1, v1, v2
+; GLOBAL-ISEL-NEXT:    global_store_b32 v0, v1, s[6:7]
+; GLOBAL-ISEL-NEXT:    s_nop 0
+; GLOBAL-ISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GLOBAL-ISEL-NEXT:    s_endpgm
+entry:
+  %tmp = call i32 @llvm.amdgcn.workitem.id.x()
+  %tmp1 = getelementptr i32, ptr addrspace(1) %out, i32 %tmp
+  store i32 0, ptr addrspace(1) %tmp1
+  %isfirst = call i1 @llvm.amdgcn.s.barrier.signal.isfirst(i32 -1)
+  %0 = load i32, ptr addrspace(1) %a, align 4
+  %b.c = select i1 %isfirst, ptr addrspace(1) %b, ptr addrspace(1) %c
+  %1 = load i32, ptr addrspace(1) %b.c, align 4
+  %mul1 = mul nsw i32 %1, %0
+  store i32 %mul1, ptr addrspace(1) %tmp1
+  ret void
+}
+
+define amdgpu_kernel void @test2_s_barrier_signal_isfirst(ptr addrspace(1) %a, ptr addrspace(1) %b, ptr addrspace(1) %c, ptr addrspace(1) %out) #0 {
+; GCN-LABEL: test2_s_barrier_signal_isfirst:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_load_b256 s[0:7], s[0:1], 0x24
+; GCN-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_lshlrev_b32 v0, 2, v0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    global_store_b32 v0, v1, s[6:7]
+; GCN-NEXT:    s_barrier_signal_isfirst 1
+; GCN-NEXT:    s_cselect_b32 s3, s3, s5
+; GCN-NEXT:    s_cselect_b32 s2, s2, s4
+; GCN-NEXT:    s_clause 0x1
+; GCN-NEXT:    global_load_b32 v2, v1, s[0:1]
+; GCN-NEXT:    global_load_b32 v1, v1, s[2:3]
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_mul_lo_u32 v1, v1, v2
+; GCN-NEXT:    global_store_b32 v0, v1, s[6:7]
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GCN-NEXT:    s_endpgm
+;
+; GLOBAL-ISEL-LABEL: test2_s_barrier_signal_isfirst:
+; GLOBAL-ISEL:       ; %bb.0: ; %entry
+; GLOBAL-ISEL-NEXT:    s_load_b256 s[0:7], s[0:1], 0x24
+; GLOBAL-ISEL-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_lshlrev_b32 v0, 2, v0
+; GLOBAL-ISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GLOBAL-ISEL-NEXT:    global_store_b32 v0, v1, s[6:7]
+; GLOBAL-ISEL-NEXT:    s_barrier_signal_isfirst 1
+; GLOBAL-ISEL-NEXT:    s_cselect_b32 s8, 1, 0
+; GLOBAL-ISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GLOBAL-ISEL-NEXT:    s_and_b32 s8, s8, 1
+; GLOBAL-ISEL-NEXT:    s_cmp_lg_u32 s8, 0
+; GLOBAL-ISEL-NEXT:    s_cselect_b64 s[2:3], s[2:3], s[4:5]
+; GLOBAL-ISEL-NEXT:    s_clause 0x1
+; GLOBAL-ISEL-NEXT:    global_load_b32 v2, v1, s[0:1]
+; GLOBAL-ISEL-NEXT:    global_load_b32 v1, v1, s[2:3]
+; GLOBAL-ISEL-NEXT:    s_waitcnt vmcnt(0)
+; GLOBAL-ISEL-NEXT:    v_mul_lo_u32 v1, v1, v2
+; GLOBAL-ISEL-NEXT:    global_store_b32 v0, v1, s[6:7]
+; GLOBAL-ISEL-NEXT:    s_nop 0
+; GLOBAL-ISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GLOBAL-ISEL-NEXT:    s_endpgm
+entry:
+  %tmp = call i32 @llvm.amdgcn.workitem.id.x()
+  %tmp1 = getelementptr i32, ptr addrspace(1) %out, i32 %tmp
+  store i32 0, ptr addrspace(1) %tmp1
+  %isfirst = call i1 @llvm.amdgcn.s.barrier.signal.isfirst(i32 1)
+  %0 = load i32, ptr addrspace(1) %a, align 4
+  %b.c = select i1 %isfirst, ptr addrspace(1) %b, ptr addrspace(1) %c
+  %1 = load i32, ptr addrspace(1) %b.c, align 4
+  %mul1 = mul nsw i32 %1, %0
+  store i32 %mul1, ptr addrspace(1) %tmp1
+  ret void
+}
+
+define amdgpu_kernel void @test3_s_barrier_signal_isfirst(ptr addrspace(1) %a, ptr addrspace(1) %b, ptr addrspace(1) %c, ptr addrspace(1) %out) #0 {
+; GCN-LABEL: test3_s_barrier_signal_isfirst:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_load_b256 s[0:7], s[0:1], 0x24
+; GCN-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_lshlrev_b32 v0, 2, v0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    global_store_b32 v0, v1, s[6:7]
+; GCN-NEXT:    s_barrier_signal_isfirst 1
+; GCN-NEXT:    s_cselect_b32 s3, s3, s5
+; GCN-NEXT:    s_cselect_b32 s2, s2, s4
+; GCN-NEXT:    s_clause 0x1
+; GCN-NEXT:    global_load_b32 v2, v1, s[0:1]
+; GCN-NEXT:    global_load_b32 v1, v1, s[2:3]
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_mul_lo_u32 v1, v1, v2
+; GCN-NEXT:    global_store_b32 v0, v1, s[6:7]
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GCN-NEXT:    s_endpgm
+;
+; GLOBAL-ISEL-LABEL: test3_s_barrier_signal_isfirst:
+; GLOBAL-ISEL:       ; %bb.0: ; %entry
+; GLOBAL-ISEL-NEXT:    s_load_b256 s[0:7], s[0:1], 0x24
+; GLOBAL-ISEL-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_lshlrev_b32 v0, 2, v0
+; GLOBAL-ISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GLOBAL-ISEL-NEXT:    global_store_b32 v0, v1, s[6:7]
+; GLOBAL-ISEL-NEXT:    s_barrier_signal_isfirst 1
+; GLOBAL-ISEL-NEXT:    s_cselect_b32 s8, 1, 0
+; GLOBAL-ISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GLOBAL-ISEL-NEXT:    s_and_b32 s8, s8, 1
+; GLOBAL-ISEL-NEXT:    s_cmp_lg_u32 s8, 0
+; GLOBAL-ISEL-NEXT:    s_cselect_b64 s[2:3], s[2:3], s[4:5]
+; GLOBAL-ISEL-NEXT:    s_clause 0x1
+; GLOBAL-ISEL-NEXT:    global_load_b32 v2, v1, s[0:1]
+; GLOBAL-ISEL-NEXT:    global_load_b32 v1, v1, s[2:3]
+; GLOBAL-ISEL-NEXT:    s_waitcnt vmcnt(0)
+; GLOBAL-ISEL-NEXT:    v_mul_lo_u32 v1, v1, v2
+; GLOBAL-ISEL-NEXT:    global_store_b32 v0, v1, s[6:7]
+; GLOBAL-ISEL-NEXT:    s_nop 0
+; GLOBAL-ISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GLOBAL-ISEL-NEXT:    s_endpgm
+entry:
+  %tmp = call i32 @llvm.amdgcn.workitem.id.x()
+  %tmp1 = getelementptr i32, ptr addrspace(1) %out, i32 %tmp
+  store i32 0, ptr addrspace(1) %tmp1
+  %isfirst = call i1 @llvm.amdgcn.s.barrier.signal.isfirst(i32 1)
+  %0 = load i32, ptr addrspace(1) %a, align 4
+  %b.c = select i1 %isfirst, ptr addrspace(1) %b, ptr addrspace(1) %c
+  %1 = load i32, ptr addrspace(1) %b.c, align 4
+  %mul1 = mul nsw i32 %1, %0
+  store i32 %mul1, ptr addrspace(1) %tmp1
+  ret void
+}
+
+define amdgpu_kernel void @test1_s_barrier_signal_isfirst_var(ptr addrspace(1) %a, ptr addrspace(1) %b, ptr addrspace(1) %c, ptr addrspace(1) %out) #0 {
+; GCN-LABEL: test1_s_barrier_signal_isfirst_var:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_load_b256 s[0:7], s[0:1], 0x24
+; GCN-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_lshlrev_b32 v0, 2, v0
+; GCN-NEXT:    s_mov_b32 m0, 1
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    global_store_b32 v0, v1, s[6:7]
+; GCN-NEXT:    s_barrier_signal_isfirst m0
+; GCN-NEXT:    s_cselect_b32 s3, s3, s5
+; GCN-NEXT:    s_cselect_b32 s2, s2, s4
+; GCN-NEXT:    s_clause 0x1
+; GCN-NEXT:    global_load_b32 v2, v1, s[0:1]
+; GCN-NEXT:    global_load_b32 v1, v1, s[2:3]
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_mul_lo_u32 v1, v1, v2
+; GCN-NEXT:    global_store_b32 v0, v1, s[6:7]
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GCN-NEXT:    s_endpgm
+;
+; GLOBAL-ISEL-LABEL: test1_s_barrier_signal_isfirst_var:
+; GLOBAL-ISEL:       ; %bb.0: ; %entry
+; GLOBAL-ISEL-NEXT:    s_load_b256 s[0:7], s[0:1], 0x24
+; GLOBAL-ISEL-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_lshlrev_b32 v0, 2, v0
+; GLOBAL-ISEL-NEXT:    s_mov_b32 m0, 1
+; GLOBAL-ISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GLOBAL-ISEL-NEXT:    global_store_b32 v0, v1, s[6:7]
+; GLOBAL-ISEL-NEXT:    s_barrier_signal_isfirst m0
+; GLOBAL-ISEL-NEXT:    s_cselect_b32 s8, 1, 0
+; GLOBAL-ISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GLOBAL-ISEL-NEXT:    s_and_b32 s8, s8, 1
+; GLOBAL-ISEL-NEXT:    s_cmp_lg_u32 s8, 0
+; GLOBAL-ISEL-NEXT:    s_cselect_b64 s[2:3], s[2:3], s[4:5]
+; GLOBAL-ISEL-NEXT:    s_clause 0x1
+; GLOBAL-ISEL-NEXT:    global_load_b32 v2, v1, s[0:1]
+; GLOBAL-ISEL-NEXT:    global_load_b32 v1, v1, s[2:3]
+; GLOBAL-ISEL-NEXT:    s_waitcnt vmcnt(0)
+; GLOBAL-ISEL-NEXT:    v_mul_lo_u32 v1, v1, v2
+; GLOBAL-ISEL-NEXT:    global_store_b32 v0, v1, s[6:7]
+; GLOBAL-ISEL-NEXT:    s_nop 0
+; GLOBAL-ISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GLOBAL-ISEL-NEXT:    s_endpgm
+entry:
+  %tmp = call i32 @llvm.amdgcn.workitem.id.x()
+  %tmp1 = getelementptr i32, ptr addrspace(1) %out, i32 %tmp
+  store i32 0, ptr addrspace(1) %tmp1
+  %isfirst = call i1 @llvm.amdgcn.s.barrier.signal.isfirst.var(i32 1)
+  %0 = load i32, ptr addrspace(1) %a, align 4
+  %b.c = select i1 %isfirst, ptr addrspace(1) %b, ptr addrspace(1) %c
+  %1 = load i32, ptr addrspace(1) %b.c, align 4
+  %mul1 = mul nsw i32 %1, %0
+  store i32 %mul1, ptr addrspace(1) %tmp1
+  ret void
+}
+
+define void @test2_s_barrier_signal_isfirst_var(ptr addrspace(1) %a, ptr addrspace(1) %b, ptr addrspace(1) %c, i32 %arg, ptr addrspace(1) %out) {
+; GCN-LABEL: test2_s_barrier_signal_isfirst_var:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_dual_mov_b32 v10, 0 :: v_dual_and_b32 v9, 0x3ff, v31
+; GCN-NEXT:    v_readfirstlane_b32 s0, v6
+; GCN-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GCN-NEXT:    v_lshlrev_b32_e32 v9, 2, v9
+; GCN-NEXT:    s_mov_b32 m0, s0
+; GCN-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GCN-NEXT:    v_add_co_u32 v7, vcc_lo, v7, v9
+; GCN-NEXT:    v_add_co_ci_u32_e32 v8, vcc_lo, 0, v8, vcc_lo
+; GCN-NEXT:    global_store_b32 v[7:8], v10, off
+; GCN-NEXT:    s_barrier_signal_isfirst m0
+; GCN-NEXT:    s_cselect_b32 vcc_lo, -1, 0
+; GCN-NEXT:    v_dual_cndmask_b32 v2, v4, v2 :: v_dual_cndmask_b32 v3, v5, v3
+; GCN-NEXT:    global_load_b32 v0, v[0:1], off
+; GCN-NEXT:    global_load_b32 v1, v[2:3], off
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_mul_lo_u32 v0, v1, v0
+; GCN-NEXT:    global_store_b32 v[7:8], v0, off
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; GLOBAL-ISEL-LABEL: test2_s_barrier_signal_isfirst_var:
+; GLOBAL-ISEL:       ; %bb.0:
+; GLOBAL-ISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GLOBAL-ISEL-NEXT:    v_and_b32_e32 v9, 0x3ff, v31
+; GLOBAL-ISEL-NEXT:    v_readfirstlane_b32 m0, v6
+; GLOBAL-ISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GLOBAL-ISEL-NEXT:    v_lshlrev_b32_e32 v9, 2, v9
+; GLOBAL-ISEL-NEXT:    v_add_co_u32 v7, vcc_lo, v7, v9
+; GLOBAL-ISEL-NEXT:    v_add_co_ci_u32_e32 v8, vcc_lo, 0, v8, vcc_lo
+; GLOBAL-ISEL-NEXT:    v_mov_b32_e32 v9, 0
+; GLOBAL-ISEL-NEXT:    global_store_b32 v[7:8], v9, off
+; GLOBAL-ISEL-NEXT:    s_barrier_signal_isfirst m0
+; GLOBAL-ISEL-NEXT:    s_cselect_b32 s0, 1, 0
+; GLOBAL-ISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GLOBAL-ISEL-NEXT:    s_and_b32 s0, 1, s0
+; GLOBAL-ISEL-NEXT:    v_cmp_ne_u32_e64 vcc_lo, 0, s0
+; GLOBAL-ISEL-NEXT:    v_dual_cndmask_b32 v2, v4, v2 :: v_dual_cndmask_b32 v3, v5, v3
+; GLOBAL-ISEL-NEXT:    global_load_b32 v0, v[0:1], off
+; GLOBAL-ISEL-NEXT:    global_load_b32 v1, v[2:3], off
+; GLOBAL-ISEL-NEXT:    s_waitcnt vmcnt(0)
+; GLOBAL-ISEL-NEXT:    v_mul_lo_u32 v0, v1, v0
+; GLOBAL-ISEL-NEXT:    global_store_b32 v[7:8], v0, off
+; GLOBAL-ISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GLOBAL-ISEL-NEXT:    s_setpc_b64 s[30:31]
+  %tmp = call i32 @llvm.amdgcn.workitem.id.x()
+  %tmp1 = getelementptr i32, ptr addrspace(1) %out, i32 %tmp
+  store i32 0, ptr addrspace(1) %tmp1
+  %isfirst = call i1 @llvm.amdgcn.s.barrier.signal.isfirst.var(i32 %arg)
+  %1 = load i32, ptr addrspace(1) %a, align 4
+  %b.c = select i1 %isfirst, ptr addrspace(1) %b, ptr addrspace(1) %c
+  %2 = load i32, ptr addrspace(1) %b.c, align 4
+  %mul1 = mul nsw i32 %2, %1
+  store i32 %mul1, ptr addrspace(1) %tmp1
+  ret void
+}
+
+define amdgpu_kernel void @test1_s_barrier_init(ptr addrspace(1) %out, i32 %mbrCnt) #0 {
+; GCN-LABEL: test1_s_barrier_init:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_clause 0x1
+; GCN-NEXT:    s_load_b32 s2, s[0:1], 0x2c
+; GCN-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GCN-NEXT:    v_mul_u32_u24_e32 v1, v0, v0
+; GCN-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_lshlrev_b32 v3, 2, v0
+; GCN-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GCN-NEXT:    v_sub_nc_u32_e32 v0, v1, v0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_lshl_b32 s2, s2, 16
+; GCN-NEXT:    global_store_b32 v3, v2, s[0:1]
+; GCN-NEXT:    s_mov_b32 m0, s2
+; GCN-NEXT:    s_barrier_init -1
+; GCN-NEXT:    global_store_b32 v3, v0, s[0:1]
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GCN-NEXT:    s_endpgm
+;
+; GLOBAL-ISEL-LABEL: test1_s_barrier_init:
+; GLOBAL-ISEL:       ; %bb.0: ; %entry
+; GLOBAL-ISEL-NEXT:    s_clause 0x1
+; GLOBAL-ISEL-NEXT:    s_load_b32 s2, s[0:1], 0x2c
+; GLOBAL-ISEL-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GLOBAL-ISEL-NEXT:    v_mul_lo_u32 v1, v0, v0
+; GLOBAL-ISEL-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_lshlrev_b32 v3, 2, v0
+; GLOBAL-ISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GLOBAL-ISEL-NEXT:    v_sub_nc_u32_e32 v0, v1, v0
+; GLOBAL-ISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GLOBAL-ISEL-NEXT:    s_lshl_b32 m0, 16, s2
+; GLOBAL-ISEL-NEXT:    global_store_b32 v3, v2, s[0:1]
+; GLOBAL-ISEL-NEXT:    s_barrier_init -1
+; GLOBAL-ISEL-NEXT:    global_store_b32 v3, v0, s[0:1]
+; GLOBAL-ISEL-NEXT:    s_nop 0
+; GLOBAL-ISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GLOBAL-ISEL-NEXT:    s_endpgm
+entry:
+  %tmp = call i32 @llvm.amdgcn.workitem.id.x()
+  %tmp1 = getelementptr i32, ptr addrspace(1) %out, i32 %tmp
+  store i32 0, ptr addrspace(1) %tmp1
+  call void @llvm.amdgcn.s.barrier.init(i32 -1, i32 %mbrCnt)
+  %tmp2 = mul i32 %tmp, %tmp
+  %tmp3 = sub i32 %tmp2, %tmp
+  store i32 %tmp3, ptr addrspace(1) %tmp1
+  ret void
+}
+
+define amdgpu_kernel void @test2_s_barrier_init(ptr addrspace(1) %out, i32 %mbrCnt) #0 {
+; GCN-LABEL: test2_s_barrier_init:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_clause 0x1
+; GCN-NEXT:    s_load_b32 s2, s[0:1], 0x2c
+; GCN-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GCN-NEXT:    v_mul_u32_u24_e32 v1, v0, v0
+; GCN-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_lshlrev_b32 v3, 2, v0
+; GCN-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GCN-NEXT:    v_sub_nc_u32_e32 v0, v1, v0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_lshl_b32 s2, s2, 16
+; GCN-NEXT:    global_store_b32 v3, v2, s[0:1]
+; GCN-NEXT:    s_mov_b32 m0, s2
+; GCN-NEXT:    s_barrier_init 1
+; GCN-NEXT:    global_store_b32 v3, v0, s[0:1]
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GCN-NEXT:    s_endpgm
+;
+; GLOBAL-ISEL-LABEL: test2_s_barrier_init:
+; GLOBAL-ISEL:       ; %bb.0: ; %entry
+; GLOBAL-ISEL-NEXT:    s_clause 0x1
+; GLOBAL-ISEL-NEXT:    s_load_b32 s2, s[0:1], 0x2c
+; GLOBAL-ISEL-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GLOBAL-ISEL-NEXT:    v_mul_lo_u32 v1, v0, v0
+; GLOBAL-ISEL-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_lshlrev_b32 v3, 2, v0
+; GLOBAL-ISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GLOBAL-ISEL-NEXT:    v_sub_nc_u32_e32 v0, v1, v0
+; GLOBAL-ISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GLOBAL-ISEL-NEXT:    s_lshl_b32 m0, 16, s2
+; GLOBAL-ISEL-NEXT:    global_store_b32 v3, v2, s[0:1]
+; GLOBAL-ISEL-NEXT:    s_barrier_init 1
+; GLOBAL-ISEL-NEXT:    global_store_b32 v3, v0, s[0:1]
+; GLOBAL-ISEL-NEXT:    s_nop 0
+; GLOBAL-ISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GLOBAL-ISEL-NEXT:    s_endpgm
+entry:
+  %tmp = call i32 @llvm.amdgcn.workitem.id.x()
+  %tmp1 = getelementptr i32, ptr addrspace(1) %out, i32 %tmp
+  store i32 0, ptr addrspace(1) %tmp1
+  call void @llvm.amdgcn.s.barrier.init(i32 1, i32 %mbrCnt)
+  %tmp3 = mul i32 %tmp, %tmp
+  %tmp4 = sub i32 %tmp3, %tmp
+  store i32 %tmp4, ptr addrspace(1) %tmp1
+  ret void
+}
+
+define amdgpu_kernel void @test3_s_barrier_init(ptr addrspace(1) %out, i32 %mbrCnt) #0 {
+; GCN-LABEL: test3_s_barrier_init:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_clause 0x1
+; GCN-NEXT:    s_load_b32 s2, s[0:1], 0x2c
+; GCN-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GCN-NEXT:    v_mul_u32_u24_e32 v1, v0, v0
+; GCN-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_lshlrev_b32 v3, 2, v0
+; GCN-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GCN-NEXT:    v_sub_nc_u32_e32 v0, v1, v0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_lshl_b32 s2, s2, 16
+; GCN-NEXT:    global_store_b32 v3, v2, s[0:1]
+; GCN-NEXT:    s_mov_b32 m0, s2
+; GCN-NEXT:    s_barrier_init 0
+; GCN-NEXT:    global_store_b32 v3, v0, s[0:1]
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GCN-NEXT:    s_endpgm
+;
+; GLOBAL-ISEL-LABEL: test3_s_barrier_init:
+; GLOBAL-ISEL:       ; %bb.0: ; %entry
+; GLOBAL-ISEL-NEXT:    s_clause 0x1
+; GLOBAL-ISEL-NEXT:    s_load_b32 s2, s[0:1], 0x2c
+; GLOBAL-ISEL-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GLOBAL-ISEL-NEXT:    v_mul_lo_u32 v1, v0, v0
+; GLOBAL-ISEL-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_lshlrev_b32 v3, 2, v0
+; GLOBAL-ISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GLOBAL-ISEL-NEXT:    v_sub_nc_u32_e32 v0, v1, v0
+; GLOBAL-ISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GLOBAL-ISEL-NEXT:    s_lshl_b32 m0, 16, s2
+; GLOBAL-ISEL-NEXT:    global_store_b32 v3, v2, s[0:1]
+; GLOBAL-ISEL-NEXT:    s_barrier_init 0
+; GLOBAL-ISEL-NEXT:    global_store_b32 v3, v0, s[0:1]
+; GLOBAL-ISEL-NEXT:    s_nop 0
+; GLOBAL-ISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GLOBAL-ISEL-NEXT:    s_endpgm
+entry:
+  %tmp = call i32 @llvm.amdgcn.workitem.id.x()
+  %tmp1 = getelementptr i32, ptr addrspace(1) %out, i32 %tmp
+  store i32 0, ptr addrspace(1) %tmp1
+  call void @llvm.amdgcn.s.barrier.init(i32 0, i32 %mbrCnt)
+  %tmp3 = mul i32 %tmp, %tmp
+  %tmp4 = sub i32 %tmp3, %tmp
+  store i32 %tmp4, ptr addrspace(1) %tmp1
+  ret void
+}
+
+define amdgpu_kernel void @test4_s_barrier_init(ptr addrspace(1) %out, i32 %bar, i32 %mbrCnt) #0 {
+; GCN-LABEL: test4_s_barrier_init:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GCN-NEXT:    v_mul_u32_u24_e32 v1, v0, v0
+; GCN-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_lshlrev_b32 v3, 2, v0
+; GCN-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(SALU_CYCLE_1)
+; GCN-NEXT:    v_sub_nc_u32_e32 v0, v1, v0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_lshl_b32 s3, s3, 16
+; GCN-NEXT:    global_store_b32 v3, v2, s[0:1]
+; GCN-NEXT:    s_or_b32 s2, s2, s3
+; GCN-NEXT:    s_mov_b32 m0, s2
+; GCN-NEXT:    s_barrier_init m0
+; GCN-NEXT:    global_store_b32 v3, v0, s[0:1]
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GCN-NEXT:    s_endpgm
+;
+; GLOBAL-ISEL-LABEL: test4_s_barrier_init:
+; GLOBAL-ISEL:       ; %bb.0: ; %entry
+; GLOBAL-ISEL-NEXT:    s_load_b128 s[0:3], s[0:1], 0x24
+; GLOBAL-ISEL-NEXT:    v_mul_lo_u32 v1, v0, v0
+; GLOBAL-ISEL-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_lshlrev_b32 v3, 2, v0
+; GLOBAL-ISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GLOBAL-ISEL-NEXT:    v_sub_nc_u32_e32 v0, v1, v0
+; GLOBAL-ISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GLOBAL-ISEL-NEXT:    s_lshl_b32 s3, 16, s3
+; GLOBAL-ISEL-NEXT:    global_store_b32 v3, v2, s[0:1]
+; GLOBAL-ISEL-NEXT:    s_or_b32 m0, s2, s3
+; GLOBAL-ISEL-NEXT:    s_barrier_init m0
+; GLOBAL-ISEL-NEXT:    global_store_b32 v3, v0, s[0:1]
+; GLOBAL-ISEL-NEXT:    s_nop 0
+; GLOBAL-ISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GLOBAL-ISEL-NEXT:    s_endpgm
+entry:
+  %tmp = call i32 @llvm.amdgcn.workitem.id.x()
+  %tmp1 = getelementptr i32, ptr addrspace(1) %out, i32 %tmp
+  store i32 0, ptr addrspace(1) %tmp1
+  call void @llvm.amdgcn.s.barrier.init(i32 %bar, i32 %mbrCnt)
+  %tmp3 = mul i32 %tmp, %tmp
+  %tmp4 = sub i32 %tmp3, %tmp
+  store i32 %tmp4, ptr addrspace(1) %tmp1
+  ret void
+}
+
+define void @test5_s_barrier_init_m0(i32 %arg1 ,i32 %arg2) {
+; GCN-LABEL: test5_s_barrier_init_m0:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GCN-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GCN-NEXT:    v_or_b32_e32 v0, v0, v1
+; GCN-NEXT:    v_readfirstlane_b32 s0, v0
+; GCN-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GCN-NEXT:    s_mov_b32 m0, s0
+; GCN-NEXT:    s_barrier_init m0
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; GLOBAL-ISEL-LABEL: test5_s_barrier_init_m0:
+; GLOBAL-ISEL:       ; %bb.0:
+; GLOBAL-ISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GLOBAL-ISEL-NEXT:    v_readfirstlane_b32 s0, v1
+; GLOBAL-ISEL-NEXT:    v_readfirstlane_b32 s1, v0
+; GLOBAL-ISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GLOBAL-ISEL-NEXT:    s_lshl_b32 s0, 16, s0
+; GLOBAL-ISEL-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GLOBAL-ISEL-NEXT:    s_or_b32 m0, s1, s0
+; GLOBAL-ISEL-NEXT:    s_barrier_init m0
+; GLOBAL-ISEL-NEXT:    s_setpc_b64 s[30:31]
+  call void @llvm.amdgcn.s.barrier.init(i32 %arg1, i32 %arg2)
+  ret void
+}
+
+define amdgpu_kernel void @test1_s_barrier_join(ptr addrspace(1) %out) #0 {
+; GCN-LABEL: test1_s_barrier_join:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GCN-NEXT:    v_mul_u32_u24_e32 v1, v0, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GCN-NEXT:    s_barrier_join -1
+; GCN-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GCN-NEXT:    v_sub_nc_u32_e32 v0, v1, v0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    global_store_b32 v2, v0, s[0:1]
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GCN-NEXT:    s_endpgm
+;
+; GLOBAL-ISEL-LABEL: test1_s_barrier_join:
+; GLOBAL-ISEL:       ; %bb.0: ; %entry
+; GLOBAL-ISEL-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GLOBAL-ISEL-NEXT:    v_mul_lo_u32 v1, v0, v0
+; GLOBAL-ISEL-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_lshlrev_b32 v3, 2, v0
+; GLOBAL-ISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GLOBAL-ISEL-NEXT:    v_sub_nc_u32_e32 v0, v1, v0
+; GLOBAL-ISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GLOBAL-ISEL-NEXT:    global_store_b32 v3, v2, s[0:1]
+; GLOBAL-ISEL-NEXT:    s_barrier_join -1
+; GLOBAL-ISEL-NEXT:    global_store_b32 v3, v0, s[0:1]
+; GLOBAL-ISEL-NEXT:    s_nop 0
+; GLOBAL-ISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GLOBAL-ISEL-NEXT:    s_endpgm
+entry:
+  %tmp = call i32 @llvm.amdgcn.workitem.id.x()
+  %tmp1 = getelementptr i32, ptr addrspace(1) %out, i32 %tmp
+  store i32 0, ptr addrspace(1) %tmp1
+  call void @llvm.amdgcn.s.barrier.join(i32 -1)
+  %tmp3 = mul i32 %tmp, %tmp
+  %tmp4 = sub i32 %tmp3, %tmp
+  store i32 %tmp4, ptr addrspace(1) %tmp1
+  ret void
+}
+
+define amdgpu_kernel void @test2_s_barrier_join(ptr addrspace(1) %out) #0 {
+; GCN-LABEL: test2_s_barrier_join:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GCN-NEXT:    v_mul_u32_u24_e32 v1, v0, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GCN-NEXT:    s_barrier_join 1
+; GCN-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GCN-NEXT:    v_sub_nc_u32_e32 v0, v1, v0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    global_store_b32 v2, v0, s[0:1]
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GCN-NEXT:    s_endpgm
+;
+; GLOBAL-ISEL-LABEL: test2_s_barrier_join:
+; GLOBAL-ISEL:       ; %bb.0: ; %entry
+; GLOBAL-ISEL-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GLOBAL-ISEL-NEXT:    v_mul_lo_u32 v1, v0, v0
+; GLOBAL-ISEL-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_lshlrev_b32 v3, 2, v0
+; GLOBAL-ISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GLOBAL-ISEL-NEXT:    v_sub_nc_u32_e32 v0, v1, v0
+; GLOBAL-ISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GLOBAL-ISEL-NEXT:    global_store_b32 v3, v2, s[0:1]
+; GLOBAL-ISEL-NEXT:    s_barrier_join 1
+; GLOBAL-ISEL-NEXT:    global_store_b32 v3, v0, s[0:1]
+; GLOBAL-ISEL-NEXT:    s_nop 0
+; GLOBAL-ISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GLOBAL-ISEL-NEXT:    s_endpgm
+entry:
+  %tmp = call i32 @llvm.amdgcn.workitem.id.x()
+  %tmp1 = getelementptr i32, ptr addrspace(1) %out, i32 %tmp
+  store i32 0, ptr addrspace(1) %tmp1
+  call void @llvm.amdgcn.s.barrier.join(i32 1)
+  %tmp3 = mul i32 %tmp, %tmp
+  %tmp4 = sub i32 %tmp3, %tmp
+  store i32 %tmp4, ptr addrspace(1) %tmp1
+  ret void
+}
+
+define amdgpu_kernel void @test3_s_barrier_join(ptr addrspace(1) %out) #0 {
+; GCN-LABEL: test3_s_barrier_join:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GCN-NEXT:    v_mul_u32_u24_e32 v1, v0, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GCN-NEXT:    s_barrier_join 0
+; GCN-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GCN-NEXT:    v_sub_nc_u32_e32 v0, v1, v0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    global_store_b32 v2, v0, s[0:1]
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GCN-NEXT:    s_endpgm
+;
+; GLOBAL-ISEL-LABEL: test3_s_barrier_join:
+; GLOBAL-ISEL:       ; %bb.0: ; %entry
+; GLOBAL-ISEL-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GLOBAL-ISEL-NEXT:    v_mul_lo_u32 v1, v0, v0
+; GLOBAL-ISEL-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_lshlrev_b32 v3, 2, v0
+; GLOBAL-ISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GLOBAL-ISEL-NEXT:    v_sub_nc_u32_e32 v0, v1, v0
+; GLOBAL-ISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GLOBAL-ISEL-NEXT:    global_store_b32 v3, v2, s[0:1]
+; GLOBAL-ISEL-NEXT:    s_barrier_join 0
+; GLOBAL-ISEL-NEXT:    global_store_b32 v3, v0, s[0:1]
+; GLOBAL-ISEL-NEXT:    s_nop 0
+; GLOBAL-ISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GLOBAL-ISEL-NEXT:    s_endpgm
+entry:
+  %tmp = call i32 @llvm.amdgcn.workitem.id.x()
+  %tmp1 = getelementptr i32, ptr addrspace(1) %out, i32 %tmp
+  store i32 0, ptr addrspace(1) %tmp1
+  call void @llvm.amdgcn.s.barrier.join(i32 0)
+  %tmp3 = mul i32 %tmp, %tmp
+  %tmp4 = sub i32 %tmp3, %tmp
+  store i32 %tmp4, ptr addrspace(1) %tmp1
+  ret void
+}
+
+define amdgpu_kernel void @test4_s_barrier_join_m0(ptr addrspace(1) %out, i32 %bar) #0 {
+; GCN-LABEL: test4_s_barrier_join_m0:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_clause 0x1
+; GCN-NEXT:    s_load_b64 s[2:3], s[0:1], 0x24
+; GCN-NEXT:    s_load_b32 s0, s[0:1], 0x2c
+; GCN-NEXT:    v_mul_u32_u24_e32 v2, v0, v0
+; GCN-NEXT:    v_mov_b32_e32 v1, 0
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 2, v0
+; GCN-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GCN-NEXT:    v_sub_nc_u32_e32 v0, v2, v0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    global_store_b32 v3, v1, s[2:3]
+; GCN-NEXT:    s_mov_b32 m0, s0
+; GCN-NEXT:    s_barrier_join m0
+; GCN-NEXT:    global_store_b32 v3, v0, s[2:3]
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GCN-NEXT:    s_endpgm
+;
+; GLOBAL-ISEL-LABEL: test4_s_barrier_join_m0:
+; GLOBAL-ISEL:       ; %bb.0: ; %entry
+; GLOBAL-ISEL-NEXT:    s_clause 0x1
+; GLOBAL-ISEL-NEXT:    s_load_b64 s[2:3], s[0:1], 0x24
+; GLOBAL-ISEL-NEXT:    s_load_b32 s0, s[0:1], 0x2c
+; GLOBAL-ISEL-NEXT:    v_mul_lo_u32 v1, v0, v0
+; GLOBAL-ISEL-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_lshlrev_b32 v3, 2, v0
+; GLOBAL-ISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GLOBAL-ISEL-NEXT:    v_sub_nc_u32_e32 v0, v1, v0
+; GLOBAL-ISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GLOBAL-ISEL-NEXT:    global_store_b32 v3, v2, s[2:3]
+; GLOBAL-ISEL-NEXT:    s_mov_b32 m0, s0
+; GLOBAL-ISEL-NEXT:    s_barrier_join m0
+; GLOBAL-ISEL-NEXT:    global_store_b32 v3, v0, s[2:3]
+; GLOBAL-ISEL-NEXT:    s_nop 0
+; GLOBAL-ISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GLOBAL-ISEL-NEXT:    s_endpgm
+entry:
+  %tmp = call i32 @llvm.amdgcn.workitem.id.x()
+  %tmp1 = getelementptr i32, ptr addrspace(1) %out, i32 %tmp
+  store i32 0, ptr addrspace(1) %tmp1
+  call void @llvm.amdgcn.s.barrier.join(i32 %bar)
+  %tmp3 = mul i32 %tmp, %tmp
+  %tmp4 = sub i32 %tmp3, %tmp
+  store i32 %tmp4, ptr addrspace(1) %tmp1
+  ret void
+}
+
+define void @test5_s_barrier_join_m0(i32 %arg) {
+; GCN-LABEL: test5_s_barrier_join_m0:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_readfirstlane_b32 s0, v0
+; GCN-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GCN-NEXT:    s_mov_b32 m0, s0
+; GCN-NEXT:    s_barrier_join m0
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; GLOBAL-ISEL-LABEL: test5_s_barrier_join_m0:
+; GLOBAL-ISEL:       ; %bb.0:
+; GLOBAL-ISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GLOBAL-ISEL-NEXT:    v_readfirstlane_b32 m0, v0
+; GLOBAL-ISEL-NEXT:    s_barrier_join m0
+; GLOBAL-ISEL-NEXT:    s_setpc_b64 s[30:31]
+  call void @llvm.amdgcn.s.barrier.join(i32 %arg)
+  ret void
+}
+
+define amdgpu_kernel void @test1_s_barrier_leave(ptr addrspace(1) %a, ptr addrspace(1) %b, ptr addrspace(1) %c, ptr addrspace(1) %out) #0 {
+; GCN-LABEL: test1_s_barrier_leave:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_load_b256 s[0:7], s[0:1], 0x24
+; GCN-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_lshlrev_b32 v0, 2, v0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    global_store_b32 v0, v1, s[6:7]
+; GCN-NEXT:    s_barrier_leave
+; GCN-NEXT:    s_cselect_b32 s3, s3, s5
+; GCN-NEXT:    s_cselect_b32 s2, s2, s4
+; GCN-NEXT:    s_clause 0x1
+; GCN-NEXT:    global_load_b32 v2, v1, s[0:1]
+; GCN-NEXT:    global_load_b32 v1, v1, s[2:3]
+; GCN-NEXT:    s_waitcnt vmcnt(0)
+; GCN-NEXT:    v_mul_lo_u32 v1, v1, v2
+; GCN-NEXT:    global_store_b32 v0, v1, s[6:7]
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GCN-NEXT:    s_endpgm
+;
+; GLOBAL-ISEL-LABEL: test1_s_barrier_leave:
+; GLOBAL-ISEL:       ; %bb.0: ; %entry
+; GLOBAL-ISEL-NEXT:    s_load_b256 s[0:7], s[0:1], 0x24
+; GLOBAL-ISEL-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_lshlrev_b32 v0, 2, v0
+; GLOBAL-ISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GLOBAL-ISEL-NEXT:    global_store_b32 v0, v1, s[6:7]
+; GLOBAL-ISEL-NEXT:    s_barrier_leave
+; GLOBAL-ISEL-NEXT:    s_cselect_b32 s8, 1, 0
+; GLOBAL-ISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GLOBAL-ISEL-NEXT:    s_and_b32 s8, s8, 1
+; GLOBAL-ISEL-NEXT:    s_cmp_lg_u32 s8, 0
+; GLOBAL-ISEL-NEXT:    s_cselect_b64 s[2:3], s[2:3], s[4:5]
+; GLOBAL-ISEL-NEXT:    s_clause 0x1
+; GLOBAL-ISEL-NEXT:    global_load_b32 v2, v1, s[0:1]
+; GLOBAL-ISEL-NEXT:    global_load_b32 v1, v1, s[2:3]
+; GLOBAL-ISEL-NEXT:    s_waitcnt vmcnt(0)
+; GLOBAL-ISEL-NEXT:    v_mul_lo_u32 v1, v1, v2
+; GLOBAL-ISEL-NEXT:    global_store_b32 v0, v1, s[6:7]
+; GLOBAL-ISEL-NEXT:    s_nop 0
+; GLOBAL-ISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GLOBAL-ISEL-NEXT:    s_endpgm
+entry:
+  %tmp = call i32 @llvm.amdgcn.workitem.id.x()
+  %tmp1 = getelementptr i32, ptr addrspace(1) %out, i32 %tmp
+  store i32 0, ptr addrspace(1) %tmp1
+  %0 = call i1 @llvm.amdgcn.s.barrier.leave()
+  %1 = load i32, ptr addrspace(1) %a, align 4
+  %b.c = select i1 %0, ptr addrspace(1) %b, ptr addrspace(1) %c
+  %2 = load i32, ptr addrspace(1) %b.c, align 4
+  %mul1 = mul nsw i32 %2, %1
+  store i32 %mul1, ptr addrspace(1) %tmp1
+  ret void
+}
+
+define amdgpu_kernel void @test1_s_wakeup_barrier(ptr addrspace(1) %out) #0 {
+; GCN-LABEL: test1_s_wakeup_barrier:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GCN-NEXT:    v_mul_u32_u24_e32 v1, v0, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GCN-NEXT:    s_wakeup_barrier -1
+; GCN-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GCN-NEXT:    v_sub_nc_u32_e32 v0, v1, v0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    global_store_b32 v2, v0, s[0:1]
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GCN-NEXT:    s_endpgm
+;
+; GLOBAL-ISEL-LABEL: test1_s_wakeup_barrier:
+; GLOBAL-ISEL:       ; %bb.0: ; %entry
+; GLOBAL-ISEL-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GLOBAL-ISEL-NEXT:    v_mul_lo_u32 v1, v0, v0
+; GLOBAL-ISEL-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_lshlrev_b32 v3, 2, v0
+; GLOBAL-ISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GLOBAL-ISEL-NEXT:    v_sub_nc_u32_e32 v0, v1, v0
+; GLOBAL-ISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GLOBAL-ISEL-NEXT:    global_store_b32 v3, v2, s[0:1]
+; GLOBAL-ISEL-NEXT:    s_wakeup_barrier -1
+; GLOBAL-ISEL-NEXT:    global_store_b32 v3, v0, s[0:1]
+; GLOBAL-ISEL-NEXT:    s_nop 0
+; GLOBAL-ISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GLOBAL-ISEL-NEXT:    s_endpgm
+entry:
+  %tmp = call i32 @llvm.amdgcn.workitem.id.x()
+  %tmp1 = getelementptr i32, ptr addrspace(1) %out, i32 %tmp
+  store i32 0, ptr addrspace(1) %tmp1
+  call void @llvm.amdgcn.s.wakeup.barrier(i32 -1)
+  %tmp3 = mul i32 %tmp, %tmp
+  %tmp4 = sub i32 %tmp3, %tmp
+  store i32 %tmp4, ptr addrspace(1) %tmp1
+  ret void
+}
+
+define amdgpu_kernel void @test2_s_wakeup_barrier(ptr addrspace(1) %out) #0 {
+; GCN-LABEL: test2_s_wakeup_barrier:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GCN-NEXT:    v_mul_u32_u24_e32 v1, v0, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GCN-NEXT:    s_wakeup_barrier 1
+; GCN-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GCN-NEXT:    v_sub_nc_u32_e32 v0, v1, v0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    global_store_b32 v2, v0, s[0:1]
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GCN-NEXT:    s_endpgm
+;
+; GLOBAL-ISEL-LABEL: test2_s_wakeup_barrier:
+; GLOBAL-ISEL:       ; %bb.0: ; %entry
+; GLOBAL-ISEL-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GLOBAL-ISEL-NEXT:    v_mul_lo_u32 v1, v0, v0
+; GLOBAL-ISEL-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_lshlrev_b32 v3, 2, v0
+; GLOBAL-ISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GLOBAL-ISEL-NEXT:    v_sub_nc_u32_e32 v0, v1, v0
+; GLOBAL-ISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GLOBAL-ISEL-NEXT:    global_store_b32 v3, v2, s[0:1]
+; GLOBAL-ISEL-NEXT:    s_wakeup_barrier 1
+; GLOBAL-ISEL-NEXT:    global_store_b32 v3, v0, s[0:1]
+; GLOBAL-ISEL-NEXT:    s_nop 0
+; GLOBAL-ISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GLOBAL-ISEL-NEXT:    s_endpgm
+entry:
+  %tmp = call i32 @llvm.amdgcn.workitem.id.x()
+  %tmp1 = getelementptr i32, ptr addrspace(1) %out, i32 %tmp
+  store i32 0, ptr addrspace(1) %tmp1
+  call void @llvm.amdgcn.s.wakeup.barrier(i32 1)
+  %tmp3 = mul i32 %tmp, %tmp
+  %tmp4 = sub i32 %tmp3, %tmp
+  store i32 %tmp4, ptr addrspace(1) %tmp1
+  ret void
+}
+
+define amdgpu_kernel void @test3_s_wakeup_barrier(ptr addrspace(1) %out) #0 {
+; GCN-LABEL: test3_s_wakeup_barrier:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GCN-NEXT:    v_mul_u32_u24_e32 v1, v0, v0
+; GCN-NEXT:    v_lshlrev_b32_e32 v2, 2, v0
+; GCN-NEXT:    s_wakeup_barrier 0
+; GCN-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GCN-NEXT:    v_sub_nc_u32_e32 v0, v1, v0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    global_store_b32 v2, v0, s[0:1]
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GCN-NEXT:    s_endpgm
+;
+; GLOBAL-ISEL-LABEL: test3_s_wakeup_barrier:
+; GLOBAL-ISEL:       ; %bb.0: ; %entry
+; GLOBAL-ISEL-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GLOBAL-ISEL-NEXT:    v_mul_lo_u32 v1, v0, v0
+; GLOBAL-ISEL-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_lshlrev_b32 v3, 2, v0
+; GLOBAL-ISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GLOBAL-ISEL-NEXT:    v_sub_nc_u32_e32 v0, v1, v0
+; GLOBAL-ISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GLOBAL-ISEL-NEXT:    global_store_b32 v3, v2, s[0:1]
+; GLOBAL-ISEL-NEXT:    s_wakeup_barrier 0
+; GLOBAL-ISEL-NEXT:    global_store_b32 v3, v0, s[0:1]
+; GLOBAL-ISEL-NEXT:    s_nop 0
+; GLOBAL-ISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GLOBAL-ISEL-NEXT:    s_endpgm
+entry:
+  %tmp = call i32 @llvm.amdgcn.workitem.id.x()
+  %tmp1 = getelementptr i32, ptr addrspace(1) %out, i32 %tmp
+  store i32 0, ptr addrspace(1) %tmp1
+  call void @llvm.amdgcn.s.wakeup.barrier(i32 0)
+  %tmp3 = mul i32 %tmp, %tmp
+  %tmp4 = sub i32 %tmp3, %tmp
+  store i32 %tmp4, ptr addrspace(1) %tmp1
+  ret void
+}
+
+define amdgpu_kernel void @test4_s_wakeup_barrier_m0(ptr addrspace(1) %out, i32 %bar) #0 {
+; GCN-LABEL: test4_s_wakeup_barrier_m0:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_clause 0x1
+; GCN-NEXT:    s_load_b64 s[2:3], s[0:1], 0x24
+; GCN-NEXT:    s_load_b32 s0, s[0:1], 0x2c
+; GCN-NEXT:    v_mul_u32_u24_e32 v2, v0, v0
+; GCN-NEXT:    v_mov_b32_e32 v1, 0
+; GCN-NEXT:    v_lshlrev_b32_e32 v3, 2, v0
+; GCN-NEXT:    s_delay_alu instid0(VALU_DEP_3)
+; GCN-NEXT:    v_sub_nc_u32_e32 v0, v2, v0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    global_store_b32 v3, v1, s[2:3]
+; GCN-NEXT:    s_mov_b32 m0, s0
+; GCN-NEXT:    s_wakeup_barrier m0
+; GCN-NEXT:    global_store_b32 v3, v0, s[2:3]
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GCN-NEXT:    s_endpgm
+;
+; GLOBAL-ISEL-LABEL: test4_s_wakeup_barrier_m0:
+; GLOBAL-ISEL:       ; %bb.0: ; %entry
+; GLOBAL-ISEL-NEXT:    s_clause 0x1
+; GLOBAL-ISEL-NEXT:    s_load_b64 s[2:3], s[0:1], 0x24
+; GLOBAL-ISEL-NEXT:    s_load_b32 s0, s[0:1], 0x2c
+; GLOBAL-ISEL-NEXT:    v_mul_lo_u32 v1, v0, v0
+; GLOBAL-ISEL-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_lshlrev_b32 v3, 2, v0
+; GLOBAL-ISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GLOBAL-ISEL-NEXT:    v_sub_nc_u32_e32 v0, v1, v0
+; GLOBAL-ISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GLOBAL-ISEL-NEXT:    global_store_b32 v3, v2, s[2:3]
+; GLOBAL-ISEL-NEXT:    s_mov_b32 m0, s0
+; GLOBAL-ISEL-NEXT:    s_wakeup_barrier m0
+; GLOBAL-ISEL-NEXT:    global_store_b32 v3, v0, s[2:3]
+; GLOBAL-ISEL-NEXT:    s_nop 0
+; GLOBAL-ISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GLOBAL-ISEL-NEXT:    s_endpgm
+entry:
+  %tmp = call i32 @llvm.amdgcn.workitem.id.x()
+  %tmp1 = getelementptr i32, ptr addrspace(1) %out, i32 %tmp
+  store i32 0, ptr addrspace(1) %tmp1
+  call void @llvm.amdgcn.s.wakeup.barrier(i32 %bar)
+  %tmp3 = mul i32 %tmp, %tmp
+  %tmp4 = sub i32 %tmp3, %tmp
+  store i32 %tmp4, ptr addrspace(1) %tmp1
+  ret void
+}
+
+define void @test5_s_wakeup_barrier_m0(i32 %arg) {
+; GCN-LABEL: test5_s_wakeup_barrier_m0:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_readfirstlane_b32 s0, v0
+; GCN-NEXT:    s_delay_alu instid0(VALU_DEP_1)
+; GCN-NEXT:    s_mov_b32 m0, s0
+; GCN-NEXT:    s_wakeup_barrier m0
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; GLOBAL-ISEL-LABEL: test5_s_wakeup_barrier_m0:
+; GLOBAL-ISEL:       ; %bb.0:
+; GLOBAL-ISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GLOBAL-ISEL-NEXT:    v_readfirstlane_b32 m0, v0
+; GLOBAL-ISEL-NEXT:    s_wakeup_barrier m0
+; GLOBAL-ISEL-NEXT:    s_setpc_b64 s[30:31]
+  call void @llvm.amdgcn.s.wakeup.barrier(i32 %arg)
+  ret void
+}
+
+define amdgpu_kernel void @test1_s_get_barrier_state(ptr addrspace(1) %out) #0 {
+; GCN-LABEL: test1_s_get_barrier_state:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_get_barrier_state s2, -1
+; GCN-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GCN-NEXT:    v_dual_mov_b32 v1, s2 :: v_dual_lshlrev_b32 v0, 2, v0
+; GCN-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GCN-NEXT:    s_endpgm
+;
+; GLOBAL-ISEL-LABEL: test1_s_get_barrier_state:
+; GLOBAL-ISEL:       ; %bb.0: ; %entry
+; GLOBAL-ISEL-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GLOBAL-ISEL-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_lshlrev_b32 v0, 2, v0
+; GLOBAL-ISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GLOBAL-ISEL-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GLOBAL-ISEL-NEXT:    s_get_barrier_state s2, -1
+; GLOBAL-ISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GLOBAL-ISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_2)
+; GLOBAL-ISEL-NEXT:    v_mov_b32_e32 v1, s2
+; GLOBAL-ISEL-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GLOBAL-ISEL-NEXT:    s_nop 0
+; GLOBAL-ISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GLOBAL-ISEL-NEXT:    s_endpgm
+entry:
+  %tmp = call i32 @llvm.amdgcn.workitem.id.x()
+  %tmp1 = getelementptr i32, ptr addrspace(1) %out, i32 %tmp
+  store i32 0, ptr addrspace(1) %tmp1
+  %state = call i32 @llvm.amdgcn.s.get.barrier.state(i32 -1)
+  store i32 %state, ptr addrspace(1) %tmp1
+  ret void
+}
+
+define amdgpu_kernel void @test2_s_get_barrier_state(ptr addrspace(1) %out) #0 {
+; GCN-LABEL: test2_s_get_barrier_state:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_get_barrier_state s2, 1
+; GCN-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GCN-NEXT:    v_dual_mov_b32 v1, s2 :: v_dual_lshlrev_b32 v0, 2, v0
+; GCN-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GCN-NEXT:    s_endpgm
+;
+; GLOBAL-ISEL-LABEL: test2_s_get_barrier_state:
+; GLOBAL-ISEL:       ; %bb.0: ; %entry
+; GLOBAL-ISEL-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GLOBAL-ISEL-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_lshlrev_b32 v0, 2, v0
+; GLOBAL-ISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GLOBAL-ISEL-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GLOBAL-ISEL-NEXT:    s_get_barrier_state s2, 1
+; GLOBAL-ISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GLOBAL-ISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_2)
+; GLOBAL-ISEL-NEXT:    v_mov_b32_e32 v1, s2
+; GLOBAL-ISEL-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GLOBAL-ISEL-NEXT:    s_nop 0
+; GLOBAL-ISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GLOBAL-ISEL-NEXT:    s_endpgm
+entry:
+  %tmp = call i32 @llvm.amdgcn.workitem.id.x()
+  %tmp1 = getelementptr i32, ptr addrspace(1) %out, i32 %tmp
+  store i32 0, ptr addrspace(1) %tmp1
+  %state = call i32 @llvm.amdgcn.s.get.barrier.state(i32 1)
+  store i32 %state, ptr addrspace(1) %tmp1
+  ret void
+}
+
+define amdgpu_kernel void @test3_s_get_barrier_state(ptr addrspace(1) %out) #0 {
+; GCN-LABEL: test3_s_get_barrier_state:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_get_barrier_state s2, 0
+; GCN-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_delay_alu instid0(SALU_CYCLE_1)
+; GCN-NEXT:    v_dual_mov_b32 v1, s2 :: v_dual_lshlrev_b32 v0, 2, v0
+; GCN-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GCN-NEXT:    s_endpgm
+;
+; GLOBAL-ISEL-LABEL: test3_s_get_barrier_state:
+; GLOBAL-ISEL:       ; %bb.0: ; %entry
+; GLOBAL-ISEL-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GLOBAL-ISEL-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_lshlrev_b32 v0, 2, v0
+; GLOBAL-ISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GLOBAL-ISEL-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GLOBAL-ISEL-NEXT:    s_get_barrier_state s2, 0
+; GLOBAL-ISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GLOBAL-ISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_2)
+; GLOBAL-ISEL-NEXT:    v_mov_b32_e32 v1, s2
+; GLOBAL-ISEL-NEXT:    global_store_b32 v0, v1, s[0:1]
+; GLOBAL-ISEL-NEXT:    s_nop 0
+; GLOBAL-ISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GLOBAL-ISEL-NEXT:    s_endpgm
+entry:
+  %tmp = call i32 @llvm.amdgcn.workitem.id.x()
+  %tmp1 = getelementptr i32, ptr addrspace(1) %out, i32 %tmp
+  store i32 0, ptr addrspace(1) %tmp1
+  %state = call i32 @llvm.amdgcn.s.get.barrier.state(i32 0)
+  store i32 %state, ptr addrspace(1) %tmp1
+  ret void
+}
+
+define amdgpu_kernel void @test4_s_get_barrier_state_m0(ptr addrspace(1) %out, i32 %bar) #0 {
+; GCN-LABEL: test4_s_get_barrier_state_m0:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_clause 0x1
+; GCN-NEXT:    s_load_b64 s[2:3], s[0:1], 0x24
+; GCN-NEXT:    s_load_b32 s0, s[0:1], 0x2c
+; GCN-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_lshlrev_b32 v0, 2, v0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GCN-NEXT:    s_mov_b32 m0, s0
+; GCN-NEXT:    s_get_barrier_state s0, m0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    s_delay_alu instid0(SALU_CYCLE_2)
+; GCN-NEXT:    v_mov_b32_e32 v1, s0
+; GCN-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GCN-NEXT:    s_endpgm
+;
+; GLOBAL-ISEL-LABEL: test4_s_get_barrier_state_m0:
+; GLOBAL-ISEL:       ; %bb.0: ; %entry
+; GLOBAL-ISEL-NEXT:    s_clause 0x1
+; GLOBAL-ISEL-NEXT:    s_load_b64 s[2:3], s[0:1], 0x24
+; GLOBAL-ISEL-NEXT:    s_load_b32 s0, s[0:1], 0x2c
+; GLOBAL-ISEL-NEXT:    v_dual_mov_b32 v1, 0 :: v_dual_lshlrev_b32 v0, 2, v0
+; GLOBAL-ISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GLOBAL-ISEL-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GLOBAL-ISEL-NEXT:    s_mov_b32 m0, s0
+; GLOBAL-ISEL-NEXT:    s_get_barrier_state s0, m0
+; GLOBAL-ISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GLOBAL-ISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_2)
+; GLOBAL-ISEL-NEXT:    v_mov_b32_e32 v1, s0
+; GLOBAL-ISEL-NEXT:    global_store_b32 v0, v1, s[2:3]
+; GLOBAL-ISEL-NEXT:    s_nop 0
+; GLOBAL-ISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GLOBAL-ISEL-NEXT:    s_endpgm
+entry:
+  %tmp = call i32 @llvm.amdgcn.workitem.id.x()
+  %tmp1 = getelementptr i32, ptr addrspace(1) %out, i32 %tmp
+  store i32 0, ptr addrspace(1) %tmp1
+  %state = call i32 @llvm.amdgcn.s.get.barrier.state(i32 %bar)
+  store i32 %state, ptr addrspace(1) %tmp1
+  ret void
+}
+
+define i32 @test5_s_get_barrier_state_m0(i32 %arg) {
+; GCN-LABEL: test5_s_get_barrier_state_m0:
+; GCN:       ; %bb.0:
+; GCN-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT:    v_readfirstlane_b32 s0, v0
+; GCN-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_2)
+; GCN-NEXT:    s_mov_b32 m0, s0
+; GCN-NEXT:    s_get_barrier_state s0, m0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    v_mov_b32_e32 v0, s0
+; GCN-NEXT:    s_setpc_b64 s[30:31]
+;
+; GLOBAL-ISEL-LABEL: test5_s_get_barrier_state_m0:
+; GLOBAL-ISEL:       ; %bb.0:
+; GLOBAL-ISEL-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GLOBAL-ISEL-NEXT:    v_readfirstlane_b32 m0, v0
+; GLOBAL-ISEL-NEXT:    s_get_barrier_state s0, m0
+; GLOBAL-ISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GLOBAL-ISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_2)
+; GLOBAL-ISEL-NEXT:    v_mov_b32_e32 v0, s0
+; GLOBAL-ISEL-NEXT:    s_setpc_b64 s[30:31]
+  %state = call i32 @llvm.amdgcn.s.get.barrier.state(i32 %arg)
+  ret i32 %state
+}
+
+define amdgpu_kernel void @test_barrier_convert(ptr addrspace(1) %out) #0 {
+; GCN-LABEL: test_barrier_convert:
+; GCN:       ; %bb.0: ; %entry
+; GCN-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GCN-NEXT:    v_mul_u32_u24_e32 v1, v0, v0
+; GCN-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_lshlrev_b32 v3, 2, v0
+; GCN-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GCN-NEXT:    v_sub_nc_u32_e32 v0, v1, v0
+; GCN-NEXT:    s_waitcnt lgkmcnt(0)
+; GCN-NEXT:    global_store_b32 v3, v2, s[0:1]
+; GCN-NEXT:    s_barrier_signal -1
+; GCN-NEXT:    s_barrier_wait -1
+; GCN-NEXT:    global_store_b32 v3, v0, s[0:1]
+; GCN-NEXT:    s_nop 0
+; GCN-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GCN-NEXT:    s_endpgm
+;
+; GLOBAL-ISEL-LABEL: test_barrier_convert:
+; GLOBAL-ISEL:       ; %bb.0: ; %entry
+; GLOBAL-ISEL-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GLOBAL-ISEL-NEXT:    v_mul_lo_u32 v1, v0, v0
+; GLOBAL-ISEL-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_lshlrev_b32 v3, 2, v0
+; GLOBAL-ISEL-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GLOBAL-ISEL-NEXT:    v_sub_nc_u32_e32 v0, v1, v0
+; GLOBAL-ISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GLOBAL-ISEL-NEXT:    global_store_b32 v3, v2, s[0:1]
+; GLOBAL-ISEL-NEXT:    s_barrier_signal -1
+; GLOBAL-ISEL-NEXT:    s_barrier_wait -1
+; GLOBAL-ISEL-NEXT:    global_store_b32 v3, v0, s[0:1]
+; GLOBAL-ISEL-NEXT:    s_nop 0
+; GLOBAL-ISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GLOBAL-ISEL-NEXT:    s_endpgm
+entry:
+  %tmp = call i32 @llvm.amdgcn.workitem.id.x()
+  %tmp1 = getelementptr i32, ptr addrspace(1) %out, i32 %tmp
+  store i32 0, ptr addrspace(1) %tmp1
+  call void @llvm.amdgcn.s.barrier()
+  %tmp3 = mul i32 %tmp, %tmp
+  %tmp4 = sub i32 %tmp3, %tmp
+  store i32 %tmp4, ptr addrspace(1) %tmp1
+  ret void
+}
+declare void @llvm.amdgcn.s.barrier() #1
+declare void @llvm.amdgcn.s.barrier.wait(i16) #1
+declare void @llvm.amdgcn.s.barrier.signal(i32) #1
+declare void @llvm.amdgcn.s.barrier.signal.var(i32) #1
+declare i1 @llvm.amdgcn.s.barrier.signal.isfirst(i32) #1
+declare i1 @llvm.amdgcn.s.barrier.signal.isfirst.var(i32) #1
+declare void @llvm.amdgcn.s.barrier.init(i32, i32) #1
+declare void @llvm.amdgcn.s.barrier.join(i32) #1
+declare i1 @llvm.amdgcn.s.barrier.leave() #1
+declare void @llvm.amdgcn.s.wakeup.barrier(i32) #1
+declare i32 @llvm.amdgcn.s.get.barrier.state(i32) #1
+declare i32 @llvm.amdgcn.s.get.barrier.state.var(i32) #1
+declare i32 @llvm.amdgcn.workitem.id.x() #2
+
+attributes #0 = { nounwind }
+attributes #1 = { convergent nounwind }
+attributes #2 = { nounwind readnone }
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.mulo.ll b/llvm/test/CodeGen/AMDGPU/llvm.mulo.ll
index ab527481235ca9484a60f70711c829cf1d64cb05..249acec639540b3ebb7667157e641bcc4c5bb75d 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.mulo.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.mulo.ll
@@ -181,22 +181,22 @@ define { i64, i1 } @smulo_i64_v_v(i64 %x, i64 %y) {
 ; GFX10-NEXT:    v_mov_b32_e32 v5, v1
 ; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s4, v4, v2, 0
 ; GFX10-NEXT:    v_mad_u64_u32 v[6:7], s4, v4, v3, 0
-; GFX10-NEXT:    v_mad_u64_u32 v[9:10], s4, v5, v2, 0
-; GFX10-NEXT:    v_mad_i64_i32 v[11:12], s4, v5, v3, 0
-; GFX10-NEXT:    v_mov_b32_e32 v8, v1
-; GFX10-NEXT:    v_add3_u32 v1, v1, v6, v9
-; GFX10-NEXT:    v_add_co_u32 v8, vcc_lo, v8, v6
+; GFX10-NEXT:    v_mad_u64_u32 v[8:9], s4, v5, v2, 0
+; GFX10-NEXT:    v_mad_i64_i32 v[10:11], s4, v5, v3, 0
+; GFX10-NEXT:    v_mov_b32_e32 v12, v1
+; GFX10-NEXT:    v_add3_u32 v1, v1, v6, v8
+; GFX10-NEXT:    v_add_co_u32 v12, vcc_lo, v12, v6
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, 0, v7, vcc_lo
-; GFX10-NEXT:    v_add_co_u32 v8, vcc_lo, v8, v9
-; GFX10-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, v7, v10, vcc_lo
-; GFX10-NEXT:    v_add_co_ci_u32_e32 v8, vcc_lo, 0, v12, vcc_lo
-; GFX10-NEXT:    v_add_co_u32 v7, vcc_lo, v7, v11
-; GFX10-NEXT:    v_add_co_ci_u32_e32 v8, vcc_lo, 0, v8, vcc_lo
+; GFX10-NEXT:    v_add_co_u32 v12, vcc_lo, v12, v8
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, v7, v9, vcc_lo
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v9, vcc_lo, 0, v11, vcc_lo
+; GFX10-NEXT:    v_add_co_u32 v7, vcc_lo, v7, v10
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v9, vcc_lo, 0, v9, vcc_lo
 ; GFX10-NEXT:    v_sub_co_u32 v2, vcc_lo, v7, v2
-; GFX10-NEXT:    v_subrev_co_ci_u32_e32 v10, vcc_lo, 0, v8, vcc_lo
+; GFX10-NEXT:    v_subrev_co_ci_u32_e32 v10, vcc_lo, 0, v9, vcc_lo
 ; GFX10-NEXT:    v_cmp_gt_i32_e32 vcc_lo, 0, v5
 ; GFX10-NEXT:    v_cndmask_b32_e32 v6, v7, v2, vcc_lo
-; GFX10-NEXT:    v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX10-NEXT:    v_cndmask_b32_e32 v5, v9, v10, vcc_lo
 ; GFX10-NEXT:    v_ashrrev_i32_e32 v2, 31, v1
 ; GFX10-NEXT:    v_sub_co_u32 v4, vcc_lo, v6, v4
 ; GFX10-NEXT:    v_subrev_co_ci_u32_e32 v7, vcc_lo, 0, v5, vcc_lo
@@ -215,28 +215,28 @@ define { i64, i1 } @smulo_i64_v_v(i64 %x, i64 %y) {
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
 ; GFX11-NEXT:    v_mad_u64_u32 v[0:1], null, v4, v2, 0
 ; GFX11-NEXT:    v_mad_u64_u32 v[6:7], null, v4, v3, 0
-; GFX11-NEXT:    v_mad_u64_u32 v[9:10], null, v5, v2, 0
-; GFX11-NEXT:    v_mad_i64_i32 v[11:12], null, v5, v3, 0
+; GFX11-NEXT:    v_mad_u64_u32 v[8:9], null, v5, v2, 0
+; GFX11-NEXT:    v_mad_i64_i32 v[10:11], null, v5, v3, 0
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_mov_b32_e32 v8, v1
-; GFX11-NEXT:    v_add3_u32 v1, v1, v6, v9
+; GFX11-NEXT:    v_mov_b32_e32 v12, v1
+; GFX11-NEXT:    v_add3_u32 v1, v1, v6, v8
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_add_co_u32 v8, vcc_lo, v8, v6
+; GFX11-NEXT:    v_add_co_u32 v12, vcc_lo, v12, v6
 ; GFX11-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, 0, v7, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v8, vcc_lo, v8, v9
+; GFX11-NEXT:    v_add_co_u32 v12, vcc_lo, v12, v8
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, v7, v10, vcc_lo
-; GFX11-NEXT:    v_add_co_ci_u32_e32 v8, vcc_lo, 0, v12, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v7, vcc_lo, v7, v11
+; GFX11-NEXT:    v_add_co_ci_u32_e32 v7, vcc_lo, v7, v9, vcc_lo
+; GFX11-NEXT:    v_add_co_ci_u32_e32 v9, vcc_lo, 0, v11, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v7, vcc_lo, v7, v10
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_add_co_ci_u32_e32 v8, vcc_lo, 0, v8, vcc_lo
+; GFX11-NEXT:    v_add_co_ci_u32_e32 v9, vcc_lo, 0, v9, vcc_lo
 ; GFX11-NEXT:    v_sub_co_u32 v2, vcc_lo, v7, v2
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_subrev_co_ci_u32_e32 v10, vcc_lo, 0, v8, vcc_lo
+; GFX11-NEXT:    v_subrev_co_ci_u32_e32 v10, vcc_lo, 0, v9, vcc_lo
 ; GFX11-NEXT:    v_cmp_gt_i32_e32 vcc_lo, 0, v5
 ; GFX11-NEXT:    v_cndmask_b32_e32 v6, v7, v2, vcc_lo
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-NEXT:    v_cndmask_b32_e32 v5, v8, v10, vcc_lo
+; GFX11-NEXT:    v_cndmask_b32_e32 v5, v9, v10, vcc_lo
 ; GFX11-NEXT:    v_ashrrev_i32_e32 v2, 31, v1
 ; GFX11-NEXT:    v_sub_co_u32 v4, vcc_lo, v6, v4
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
diff --git a/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll b/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll
index 2e7fa86e8ab8b6c2acd2ca434a009ae0d6a75d10..d100cadb8ee57948c4b691e8408a8149e2cffa7e 100644
--- a/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll
@@ -3562,13 +3562,13 @@ define amdgpu_kernel void @constant_sextload_i1_to_i64(ptr addrspace(1) %out, pt
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s3
-; GFX8-NEXT:    flat_load_ubyte v0, v[0:1]
-; GFX8-NEXT:    v_mov_b32_e32 v1, s0
-; GFX8-NEXT:    v_mov_b32_e32 v2, s1
+; GFX8-NEXT:    flat_load_ubyte v2, v[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
-; GFX8-NEXT:    v_bfe_i32 v3, v0, 0, 1
-; GFX8-NEXT:    v_ashrrev_i32_e32 v4, 31, v3
-; GFX8-NEXT:    flat_store_dwordx2 v[1:2], v[3:4]
+; GFX8-NEXT:    v_bfe_i32 v2, v2, 0, 1
+; GFX8-NEXT:    v_ashrrev_i32_e32 v3, 31, v2
+; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX8-NEXT:    s_endpgm
 ;
 ; EG-LABEL: constant_sextload_i1_to_i64:
@@ -3677,13 +3677,13 @@ define amdgpu_kernel void @constant_sextload_v1i1_to_v1i64(ptr addrspace(1) %out
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s3
-; GFX8-NEXT:    flat_load_ubyte v0, v[0:1]
-; GFX8-NEXT:    v_mov_b32_e32 v1, s0
-; GFX8-NEXT:    v_mov_b32_e32 v2, s1
+; GFX8-NEXT:    flat_load_ubyte v2, v[0:1]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
-; GFX8-NEXT:    v_bfe_i32 v3, v0, 0, 1
-; GFX8-NEXT:    v_ashrrev_i32_e32 v4, 31, v3
-; GFX8-NEXT:    flat_store_dwordx2 v[1:2], v[3:4]
+; GFX8-NEXT:    v_bfe_i32 v2, v2, 0, 1
+; GFX8-NEXT:    v_ashrrev_i32_e32 v3, 31, v2
+; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX8-NEXT:    s_endpgm
 ;
 ; EG-LABEL: constant_sextload_v1i1_to_v1i64:
@@ -4755,66 +4755,66 @@ define amdgpu_kernel void @constant_sextload_v16i1_to_v16i64(ptr addrspace(1) %o
 ; GFX6-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX6-NEXT:    s_mov_b32 s8, s6
 ; GFX6-NEXT:    s_mov_b32 s9, s7
-; GFX6-NEXT:    buffer_load_ushort v5, off, s[8:11], 0
+; GFX6-NEXT:    buffer_load_ushort v1, off, s[8:11], 0
 ; GFX6-NEXT:    s_mov_b32 s0, s4
 ; GFX6-NEXT:    s_mov_b32 s1, s5
 ; GFX6-NEXT:    s_waitcnt vmcnt(0)
-; GFX6-NEXT:    v_lshrrev_b32_e32 v0, 14, v5
-; GFX6-NEXT:    v_lshrrev_b32_e32 v1, 15, v5
-; GFX6-NEXT:    v_lshrrev_b32_e32 v7, 12, v5
-; GFX6-NEXT:    v_lshrrev_b32_e32 v9, 10, v5
-; GFX6-NEXT:    v_lshrrev_b32_e32 v11, 11, v5
-; GFX6-NEXT:    v_lshrrev_b32_e32 v12, 8, v5
-; GFX6-NEXT:    v_lshrrev_b32_e32 v13, 9, v5
-; GFX6-NEXT:    v_lshrrev_b32_e32 v8, 6, v5
-; GFX6-NEXT:    v_lshrrev_b32_e32 v10, 7, v5
-; GFX6-NEXT:    v_lshrrev_b32_e32 v4, 4, v5
-; GFX6-NEXT:    v_lshrrev_b32_e32 v6, 5, v5
-; GFX6-NEXT:    v_lshrrev_b32_e32 v14, 2, v5
-; GFX6-NEXT:    v_bfe_i32 v2, v1, 0, 1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v3, 14, v1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v5, 15, v1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v7, 12, v1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v9, 13, v1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v11, 10, v1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v12, 8, v1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v13, 9, v1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v8, 6, v1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v10, 7, v1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v4, 4, v1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v6, 5, v1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v0, 2, v1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v2, 3, v1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v16, 1, v1
+; GFX6-NEXT:    v_bfe_i32 v2, v2, 0, 1
 ; GFX6-NEXT:    v_bfe_i32 v0, v0, 0, 1
-; GFX6-NEXT:    v_lshrrev_b32_e32 v15, 3, v5
-; GFX6-NEXT:    v_ashrrev_i32_e32 v3, 31, v2
-; GFX6-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; GFX6-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:112
-; GFX6-NEXT:    s_waitcnt expcnt(0)
-; GFX6-NEXT:    v_lshrrev_b32_e32 v1, 1, v5
-; GFX6-NEXT:    v_bfe_i32 v2, v15, 0, 1
-; GFX6-NEXT:    v_bfe_i32 v0, v14, 0, 1
 ; GFX6-NEXT:    v_bfe_i32 v6, v6, 0, 1
 ; GFX6-NEXT:    v_bfe_i32 v4, v4, 0, 1
 ; GFX6-NEXT:    v_bfe_i32 v10, v10, 0, 1
 ; GFX6-NEXT:    v_bfe_i32 v8, v8, 0, 1
 ; GFX6-NEXT:    v_bfe_i32 v14, v13, 0, 1
 ; GFX6-NEXT:    v_bfe_i32 v12, v12, 0, 1
-; GFX6-NEXT:    v_bfe_i32 v18, v11, 0, 1
-; GFX6-NEXT:    v_bfe_i32 v16, v9, 0, 1
-; GFX6-NEXT:    v_bfe_i32 v22, v1, 0, 1
-; GFX6-NEXT:    v_bfe_i32 v20, v5, 0, 1
-; GFX6-NEXT:    v_lshrrev_b32_e32 v1, 13, v5
-; GFX6-NEXT:    v_bfe_i32 v26, v1, 0, 1
-; GFX6-NEXT:    v_bfe_i32 v24, v7, 0, 1
-; GFX6-NEXT:    v_ashrrev_i32_e32 v21, 31, v20
-; GFX6-NEXT:    v_ashrrev_i32_e32 v23, 31, v22
+; GFX6-NEXT:    v_bfe_i32 v17, v5, 0, 1
+; GFX6-NEXT:    v_bfe_i32 v15, v3, 0, 1
+; GFX6-NEXT:    v_bfe_i32 v21, v16, 0, 1
+; GFX6-NEXT:    v_bfe_i32 v19, v1, 0, 1
+; GFX6-NEXT:    v_lshrrev_b32_e32 v1, 11, v1
+; GFX6-NEXT:    v_ashrrev_i32_e32 v18, 31, v17
+; GFX6-NEXT:    v_ashrrev_i32_e32 v16, 31, v15
+; GFX6-NEXT:    buffer_store_dwordx4 v[15:18], off, s[0:3], 0 offset:112
+; GFX6-NEXT:    v_bfe_i32 v25, v1, 0, 1
+; GFX6-NEXT:    v_bfe_i32 v23, v11, 0, 1
+; GFX6-NEXT:    v_bfe_i32 v29, v9, 0, 1
+; GFX6-NEXT:    v_bfe_i32 v27, v7, 0, 1
+; GFX6-NEXT:    v_ashrrev_i32_e32 v20, 31, v19
+; GFX6-NEXT:    v_ashrrev_i32_e32 v22, 31, v21
 ; GFX6-NEXT:    v_ashrrev_i32_e32 v3, 31, v2
 ; GFX6-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GFX6-NEXT:    v_ashrrev_i32_e32 v7, 31, v6
 ; GFX6-NEXT:    v_ashrrev_i32_e32 v5, 31, v4
 ; GFX6-NEXT:    v_ashrrev_i32_e32 v11, 31, v10
 ; GFX6-NEXT:    v_ashrrev_i32_e32 v9, 31, v8
+; GFX6-NEXT:    s_waitcnt expcnt(0)
 ; GFX6-NEXT:    v_ashrrev_i32_e32 v15, 31, v14
 ; GFX6-NEXT:    v_ashrrev_i32_e32 v13, 31, v12
-; GFX6-NEXT:    v_ashrrev_i32_e32 v19, 31, v18
-; GFX6-NEXT:    v_ashrrev_i32_e32 v17, 31, v16
-; GFX6-NEXT:    v_ashrrev_i32_e32 v27, 31, v26
-; GFX6-NEXT:    v_ashrrev_i32_e32 v25, 31, v24
-; GFX6-NEXT:    buffer_store_dwordx4 v[24:27], off, s[0:3], 0 offset:96
-; GFX6-NEXT:    buffer_store_dwordx4 v[16:19], off, s[0:3], 0 offset:80
+; GFX6-NEXT:    v_ashrrev_i32_e32 v26, 31, v25
+; GFX6-NEXT:    v_ashrrev_i32_e32 v24, 31, v23
+; GFX6-NEXT:    v_ashrrev_i32_e32 v30, 31, v29
+; GFX6-NEXT:    v_ashrrev_i32_e32 v28, 31, v27
+; GFX6-NEXT:    buffer_store_dwordx4 v[27:30], off, s[0:3], 0 offset:96
+; GFX6-NEXT:    buffer_store_dwordx4 v[23:26], off, s[0:3], 0 offset:80
 ; GFX6-NEXT:    buffer_store_dwordx4 v[12:15], off, s[0:3], 0 offset:64
 ; GFX6-NEXT:    buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:48
 ; GFX6-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:32
 ; GFX6-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:16
-; GFX6-NEXT:    buffer_store_dwordx4 v[20:23], off, s[0:3], 0
+; GFX6-NEXT:    buffer_store_dwordx4 v[19:22], off, s[0:3], 0
 ; GFX6-NEXT:    s_endpgm
 ;
 ; GFX8-LABEL: constant_sextload_v16i1_to_v16i64:
@@ -4838,78 +4838,78 @@ define amdgpu_kernel void @constant_sextload_v16i1_to_v16i64(ptr addrspace(1) %o
 ; GFX8-NEXT:    v_mov_b32_e32 v9, s2
 ; GFX8-NEXT:    s_add_u32 s2, s0, 64
 ; GFX8-NEXT:    s_addc_u32 s3, s1, 0
-; GFX8-NEXT:    v_mov_b32_e32 v12, s3
-; GFX8-NEXT:    v_mov_b32_e32 v11, s2
+; GFX8-NEXT:    v_mov_b32_e32 v16, s3
+; GFX8-NEXT:    v_mov_b32_e32 v15, s2
 ; GFX8-NEXT:    s_add_u32 s2, s0, 48
 ; GFX8-NEXT:    s_addc_u32 s3, s1, 0
-; GFX8-NEXT:    v_mov_b32_e32 v19, s3
-; GFX8-NEXT:    v_mov_b32_e32 v18, s2
+; GFX8-NEXT:    v_mov_b32_e32 v23, s3
+; GFX8-NEXT:    v_mov_b32_e32 v22, s2
 ; GFX8-NEXT:    s_add_u32 s2, s0, 32
-; GFX8-NEXT:    v_mov_b32_e32 v17, s1
+; GFX8-NEXT:    v_mov_b32_e32 v21, s1
 ; GFX8-NEXT:    s_addc_u32 s3, s1, 0
-; GFX8-NEXT:    v_mov_b32_e32 v16, s0
+; GFX8-NEXT:    v_mov_b32_e32 v20, s0
 ; GFX8-NEXT:    s_add_u32 s0, s0, 16
 ; GFX8-NEXT:    s_addc_u32 s1, s1, 0
-; GFX8-NEXT:    v_mov_b32_e32 v21, s3
-; GFX8-NEXT:    v_mov_b32_e32 v23, s1
-; GFX8-NEXT:    v_mov_b32_e32 v20, s2
-; GFX8-NEXT:    v_mov_b32_e32 v22, s0
+; GFX8-NEXT:    v_mov_b32_e32 v25, s3
+; GFX8-NEXT:    v_mov_b32_e32 v27, s1
+; GFX8-NEXT:    v_mov_b32_e32 v24, s2
+; GFX8-NEXT:    v_mov_b32_e32 v26, s0
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
 ; GFX8-NEXT:    v_lshrrev_b16_e32 v1, 14, v0
 ; GFX8-NEXT:    v_lshrrev_b16_e32 v2, 15, v0
 ; GFX8-NEXT:    v_bfe_i32 v3, v2, 0, 1
 ; GFX8-NEXT:    v_bfe_i32 v1, v1, 0, 1
-; GFX8-NEXT:    v_lshrrev_b16_e32 v13, 12, v0
-; GFX8-NEXT:    v_lshrrev_b16_e32 v14, 13, v0
+; GFX8-NEXT:    v_lshrrev_b16_e32 v11, 12, v0
+; GFX8-NEXT:    v_lshrrev_b16_e32 v12, 13, v0
 ; GFX8-NEXT:    v_ashrrev_i32_e32 v4, 31, v3
 ; GFX8-NEXT:    v_ashrrev_i32_e32 v2, 31, v1
 ; GFX8-NEXT:    flat_store_dwordx4 v[5:6], v[1:4]
-; GFX8-NEXT:    v_lshrrev_b16_e32 v15, 10, v0
-; GFX8-NEXT:    v_bfe_i32 v3, v14, 0, 1
-; GFX8-NEXT:    v_bfe_i32 v1, v13, 0, 1
-; GFX8-NEXT:    v_lshrrev_b16_e32 v5, 11, v0
+; GFX8-NEXT:    v_lshrrev_b16_e32 v13, 10, v0
+; GFX8-NEXT:    v_bfe_i32 v3, v12, 0, 1
+; GFX8-NEXT:    v_bfe_i32 v1, v11, 0, 1
+; GFX8-NEXT:    v_lshrrev_b16_e32 v14, 11, v0
+; GFX8-NEXT:    v_lshrrev_b16_e32 v6, 3, v0
 ; GFX8-NEXT:    v_ashrrev_i32_e32 v4, 31, v3
 ; GFX8-NEXT:    v_ashrrev_i32_e32 v2, 31, v1
+; GFX8-NEXT:    v_lshrrev_b16_e32 v5, 2, v0
 ; GFX8-NEXT:    flat_store_dwordx4 v[7:8], v[1:4]
-; GFX8-NEXT:    v_lshrrev_b16_e32 v6, 8, v0
-; GFX8-NEXT:    v_bfe_i32 v3, v5, 0, 1
-; GFX8-NEXT:    v_bfe_i32 v1, v15, 0, 1
-; GFX8-NEXT:    v_lshrrev_b16_e32 v13, 9, v0
-; GFX8-NEXT:    v_ashrrev_i32_e32 v4, 31, v3
-; GFX8-NEXT:    v_ashrrev_i32_e32 v2, 31, v1
-; GFX8-NEXT:    flat_store_dwordx4 v[9:10], v[1:4]
-; GFX8-NEXT:    v_lshrrev_b16_e32 v7, 6, v0
-; GFX8-NEXT:    v_bfe_i32 v3, v13, 0, 1
-; GFX8-NEXT:    v_bfe_i32 v1, v6, 0, 1
-; GFX8-NEXT:    v_lshrrev_b16_e32 v14, 7, v0
-; GFX8-NEXT:    v_ashrrev_i32_e32 v4, 31, v3
-; GFX8-NEXT:    v_ashrrev_i32_e32 v2, 31, v1
-; GFX8-NEXT:    v_lshrrev_b16_e32 v5, 4, v0
-; GFX8-NEXT:    v_lshrrev_b16_e32 v8, 5, v0
-; GFX8-NEXT:    v_lshrrev_b16_e32 v9, 2, v0
-; GFX8-NEXT:    v_lshrrev_b16_e32 v6, 3, v0
-; GFX8-NEXT:    flat_store_dwordx4 v[11:12], v[1:4]
-; GFX8-NEXT:    v_bfe_i32 v14, v14, 0, 1
-; GFX8-NEXT:    v_lshrrev_b16_e32 v2, 1, v0
-; GFX8-NEXT:    v_bfe_i32 v12, v7, 0, 1
-; GFX8-NEXT:    v_bfe_i32 v0, v0, 0, 1
-; GFX8-NEXT:    v_bfe_i32 v2, v2, 0, 1
-; GFX8-NEXT:    v_bfe_i32 v6, v6, 0, 1
-; GFX8-NEXT:    v_bfe_i32 v4, v9, 0, 1
-; GFX8-NEXT:    v_bfe_i32 v10, v8, 0, 1
-; GFX8-NEXT:    v_bfe_i32 v8, v5, 0, 1
-; GFX8-NEXT:    v_ashrrev_i32_e32 v15, 31, v14
-; GFX8-NEXT:    v_ashrrev_i32_e32 v13, 31, v12
-; GFX8-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; GFX8-NEXT:    v_ashrrev_i32_e32 v3, 31, v2
+; GFX8-NEXT:    v_lshrrev_b16_e32 v17, 8, v0
+; GFX8-NEXT:    v_bfe_i32 v3, v6, 0, 1
+; GFX8-NEXT:    v_bfe_i32 v6, v14, 0, 1
+; GFX8-NEXT:    v_bfe_i32 v4, v13, 0, 1
+; GFX8-NEXT:    v_lshrrev_b16_e32 v18, 9, v0
+; GFX8-NEXT:    v_bfe_i32 v1, v5, 0, 1
 ; GFX8-NEXT:    v_ashrrev_i32_e32 v7, 31, v6
 ; GFX8-NEXT:    v_ashrrev_i32_e32 v5, 31, v4
+; GFX8-NEXT:    flat_store_dwordx4 v[9:10], v[4:7]
+; GFX8-NEXT:    v_bfe_i32 v10, v18, 0, 1
+; GFX8-NEXT:    v_bfe_i32 v8, v17, 0, 1
+; GFX8-NEXT:    v_lshrrev_b16_e32 v19, 6, v0
+; GFX8-NEXT:    v_lshrrev_b16_e32 v28, 4, v0
+; GFX8-NEXT:    v_lshrrev_b16_e32 v29, 5, v0
+; GFX8-NEXT:    v_lshrrev_b16_e32 v2, 1, v0
+; GFX8-NEXT:    v_bfe_i32 v12, v0, 0, 1
+; GFX8-NEXT:    v_lshrrev_b16_e32 v0, 7, v0
 ; GFX8-NEXT:    v_ashrrev_i32_e32 v11, 31, v10
 ; GFX8-NEXT:    v_ashrrev_i32_e32 v9, 31, v8
-; GFX8-NEXT:    flat_store_dwordx4 v[18:19], v[12:15]
-; GFX8-NEXT:    flat_store_dwordx4 v[20:21], v[8:11]
-; GFX8-NEXT:    flat_store_dwordx4 v[22:23], v[4:7]
-; GFX8-NEXT:    flat_store_dwordx4 v[16:17], v[0:3]
+; GFX8-NEXT:    flat_store_dwordx4 v[15:16], v[8:11]
+; GFX8-NEXT:    v_bfe_i32 v18, v0, 0, 1
+; GFX8-NEXT:    v_bfe_i32 v16, v19, 0, 1
+; GFX8-NEXT:    v_bfe_i32 v7, v29, 0, 1
+; GFX8-NEXT:    v_bfe_i32 v5, v28, 0, 1
+; GFX8-NEXT:    v_bfe_i32 v14, v2, 0, 1
+; GFX8-NEXT:    v_ashrrev_i32_e32 v19, 31, v18
+; GFX8-NEXT:    v_ashrrev_i32_e32 v17, 31, v16
+; GFX8-NEXT:    v_ashrrev_i32_e32 v13, 31, v12
+; GFX8-NEXT:    v_ashrrev_i32_e32 v15, 31, v14
+; GFX8-NEXT:    v_ashrrev_i32_e32 v4, 31, v3
+; GFX8-NEXT:    v_ashrrev_i32_e32 v2, 31, v1
+; GFX8-NEXT:    v_ashrrev_i32_e32 v8, 31, v7
+; GFX8-NEXT:    v_ashrrev_i32_e32 v6, 31, v5
+; GFX8-NEXT:    flat_store_dwordx4 v[22:23], v[16:19]
+; GFX8-NEXT:    flat_store_dwordx4 v[24:25], v[5:8]
+; GFX8-NEXT:    flat_store_dwordx4 v[26:27], v[1:4]
+; GFX8-NEXT:    flat_store_dwordx4 v[20:21], v[12:15]
 ; GFX8-NEXT:    s_endpgm
 ;
 ; EG-LABEL: constant_sextload_v16i1_to_v16i64:
@@ -5635,187 +5635,187 @@ define amdgpu_kernel void @constant_sextload_v32i1_to_v32i64(ptr addrspace(1) %o
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_load_dword s8, s[2:3], 0x0
+; GFX8-NEXT:    s_load_dword s4, s[2:3], 0x0
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_lshr_b32 s10, s8, 22
-; GFX8-NEXT:    s_lshr_b32 s12, s8, 23
-; GFX8-NEXT:    s_lshr_b32 s14, s8, 20
-; GFX8-NEXT:    s_lshr_b32 s16, s8, 21
-; GFX8-NEXT:    s_lshr_b32 s18, s8, 18
-; GFX8-NEXT:    s_lshr_b32 s20, s8, 19
-; GFX8-NEXT:    s_lshr_b32 s22, s8, 16
-; GFX8-NEXT:    s_lshr_b32 s24, s8, 17
-; GFX8-NEXT:    s_lshr_b32 s6, s8, 24
-; GFX8-NEXT:    s_bfe_i64 s[2:3], s[6:7], 0x10000
-; GFX8-NEXT:    s_bfe_i64 s[4:5], s[8:9], 0x10000
-; GFX8-NEXT:    s_bfe_i64 s[24:25], s[24:25], 0x10000
-; GFX8-NEXT:    s_bfe_i64 s[22:23], s[22:23], 0x10000
+; GFX8-NEXT:    s_lshr_b32 s6, s4, 22
+; GFX8-NEXT:    s_lshr_b32 s8, s4, 23
+; GFX8-NEXT:    s_lshr_b32 s10, s4, 20
+; GFX8-NEXT:    s_lshr_b32 s12, s4, 21
+; GFX8-NEXT:    s_lshr_b32 s14, s4, 18
+; GFX8-NEXT:    s_lshr_b32 s16, s4, 19
+; GFX8-NEXT:    s_lshr_b32 s18, s4, 16
+; GFX8-NEXT:    s_lshr_b32 s20, s4, 17
+; GFX8-NEXT:    s_lshr_b32 s2, s4, 24
+; GFX8-NEXT:    v_lshrrev_b16_e64 v2, 14, s4
+; GFX8-NEXT:    v_lshrrev_b16_e64 v3, 15, s4
+; GFX8-NEXT:    v_lshrrev_b16_e64 v15, 12, s4
+; GFX8-NEXT:    v_lshrrev_b16_e64 v17, 13, s4
+; GFX8-NEXT:    v_lshrrev_b16_e64 v13, 10, s4
+; GFX8-NEXT:    v_lshrrev_b16_e64 v14, 11, s4
+; GFX8-NEXT:    v_lshrrev_b16_e64 v9, 8, s4
+; GFX8-NEXT:    v_lshrrev_b16_e64 v10, 9, s4
+; GFX8-NEXT:    v_lshrrev_b16_e64 v6, 6, s4
+; GFX8-NEXT:    v_lshrrev_b16_e64 v8, 7, s4
+; GFX8-NEXT:    v_lshrrev_b16_e64 v4, 4, s4
+; GFX8-NEXT:    v_lshrrev_b16_e64 v5, 5, s4
+; GFX8-NEXT:    v_lshrrev_b16_e64 v0, 2, s4
+; GFX8-NEXT:    v_lshrrev_b16_e64 v1, 3, s4
+; GFX8-NEXT:    v_lshrrev_b16_e64 v7, 1, s4
+; GFX8-NEXT:    v_lshrrev_b16_e64 v11, 6, s2
+; GFX8-NEXT:    v_lshrrev_b16_e64 v12, 7, s2
+; GFX8-NEXT:    v_lshrrev_b16_e64 v16, 4, s2
+; GFX8-NEXT:    v_lshrrev_b16_e64 v18, 5, s2
+; GFX8-NEXT:    v_lshrrev_b16_e64 v19, 2, s2
+; GFX8-NEXT:    v_lshrrev_b16_e64 v20, 3, s2
+; GFX8-NEXT:    v_lshrrev_b16_e64 v27, 1, s2
+; GFX8-NEXT:    s_bfe_i64 s[2:3], s[2:3], 0x10000
+; GFX8-NEXT:    s_bfe_i64 s[4:5], s[4:5], 0x10000
 ; GFX8-NEXT:    s_bfe_i64 s[20:21], s[20:21], 0x10000
 ; GFX8-NEXT:    s_bfe_i64 s[18:19], s[18:19], 0x10000
 ; GFX8-NEXT:    s_bfe_i64 s[16:17], s[16:17], 0x10000
 ; GFX8-NEXT:    s_bfe_i64 s[14:15], s[14:15], 0x10000
 ; GFX8-NEXT:    s_bfe_i64 s[12:13], s[12:13], 0x10000
 ; GFX8-NEXT:    s_bfe_i64 s[10:11], s[10:11], 0x10000
-; GFX8-NEXT:    v_mov_b32_e32 v11, s10
-; GFX8-NEXT:    s_add_u32 s10, s0, 0xb0
-; GFX8-NEXT:    v_mov_b32_e32 v12, s11
-; GFX8-NEXT:    s_addc_u32 s11, s1, 0
-; GFX8-NEXT:    v_mov_b32_e32 v16, s11
-; GFX8-NEXT:    v_mov_b32_e32 v15, s10
-; GFX8-NEXT:    s_add_u32 s10, s0, 0xa0
-; GFX8-NEXT:    v_mov_b32_e32 v13, s12
-; GFX8-NEXT:    v_mov_b32_e32 v14, s13
-; GFX8-NEXT:    s_addc_u32 s11, s1, 0
-; GFX8-NEXT:    flat_store_dwordx4 v[15:16], v[11:14]
-; GFX8-NEXT:    v_mov_b32_e32 v16, s11
-; GFX8-NEXT:    v_mov_b32_e32 v15, s10
-; GFX8-NEXT:    s_add_u32 s10, s0, 0x90
-; GFX8-NEXT:    v_mov_b32_e32 v11, s14
-; GFX8-NEXT:    v_mov_b32_e32 v12, s15
-; GFX8-NEXT:    v_mov_b32_e32 v13, s16
-; GFX8-NEXT:    v_mov_b32_e32 v14, s17
-; GFX8-NEXT:    s_addc_u32 s11, s1, 0
-; GFX8-NEXT:    flat_store_dwordx4 v[15:16], v[11:14]
-; GFX8-NEXT:    v_mov_b32_e32 v16, s11
-; GFX8-NEXT:    v_mov_b32_e32 v15, s10
-; GFX8-NEXT:    s_add_u32 s10, s0, 0x80
-; GFX8-NEXT:    v_mov_b32_e32 v11, s18
-; GFX8-NEXT:    v_mov_b32_e32 v12, s19
-; GFX8-NEXT:    v_mov_b32_e32 v13, s20
-; GFX8-NEXT:    v_mov_b32_e32 v14, s21
-; GFX8-NEXT:    s_addc_u32 s11, s1, 0
-; GFX8-NEXT:    flat_store_dwordx4 v[15:16], v[11:14]
-; GFX8-NEXT:    v_mov_b32_e32 v16, s11
-; GFX8-NEXT:    v_mov_b32_e32 v15, s10
-; GFX8-NEXT:    s_add_u32 s10, s0, 0x70
-; GFX8-NEXT:    v_lshrrev_b16_e64 v9, 14, s8
-; GFX8-NEXT:    v_lshrrev_b16_e64 v10, 15, s8
-; GFX8-NEXT:    v_mov_b32_e32 v11, s22
-; GFX8-NEXT:    v_mov_b32_e32 v12, s23
-; GFX8-NEXT:    v_mov_b32_e32 v13, s24
-; GFX8-NEXT:    v_mov_b32_e32 v14, s25
-; GFX8-NEXT:    s_addc_u32 s11, s1, 0
-; GFX8-NEXT:    v_lshrrev_b16_e64 v7, 12, s8
-; GFX8-NEXT:    v_lshrrev_b16_e64 v8, 13, s8
-; GFX8-NEXT:    v_lshrrev_b16_e64 v5, 10, s8
-; GFX8-NEXT:    v_lshrrev_b16_e64 v6, 11, s8
-; GFX8-NEXT:    v_lshrrev_b16_e64 v3, 8, s8
-; GFX8-NEXT:    v_lshrrev_b16_e64 v4, 9, s8
-; GFX8-NEXT:    v_lshrrev_b16_e64 v1, 6, s8
-; GFX8-NEXT:    v_lshrrev_b16_e64 v2, 7, s8
-; GFX8-NEXT:    v_lshrrev_b16_e64 v0, 4, s8
-; GFX8-NEXT:    flat_store_dwordx4 v[15:16], v[11:14]
-; GFX8-NEXT:    v_lshrrev_b16_e64 v16, 5, s8
-; GFX8-NEXT:    v_lshrrev_b16_e64 v20, 2, s8
-; GFX8-NEXT:    v_bfe_i32 v11, v10, 0, 1
-; GFX8-NEXT:    v_bfe_i32 v9, v9, 0, 1
-; GFX8-NEXT:    v_mov_b32_e32 v14, s11
-; GFX8-NEXT:    v_lshrrev_b16_e64 v21, 3, s8
-; GFX8-NEXT:    v_lshrrev_b16_e64 v22, 1, s8
-; GFX8-NEXT:    s_add_u32 s8, s0, 0x60
-; GFX8-NEXT:    v_ashrrev_i32_e32 v12, 31, v11
-; GFX8-NEXT:    v_ashrrev_i32_e32 v10, 31, v9
-; GFX8-NEXT:    v_mov_b32_e32 v13, s10
-; GFX8-NEXT:    s_addc_u32 s9, s1, 0
-; GFX8-NEXT:    flat_store_dwordx4 v[13:14], v[9:12]
-; GFX8-NEXT:    v_bfe_i32 v7, v7, 0, 1
-; GFX8-NEXT:    v_mov_b32_e32 v12, s9
-; GFX8-NEXT:    v_bfe_i32 v9, v8, 0, 1
-; GFX8-NEXT:    v_mov_b32_e32 v11, s8
-; GFX8-NEXT:    s_add_u32 s8, s0, 0x50
-; GFX8-NEXT:    v_ashrrev_i32_e32 v10, 31, v9
-; GFX8-NEXT:    v_ashrrev_i32_e32 v8, 31, v7
-; GFX8-NEXT:    s_addc_u32 s9, s1, 0
-; GFX8-NEXT:    flat_store_dwordx4 v[11:12], v[7:10]
-; GFX8-NEXT:    v_bfe_i32 v5, v5, 0, 1
-; GFX8-NEXT:    v_mov_b32_e32 v10, s9
-; GFX8-NEXT:    v_bfe_i32 v7, v6, 0, 1
-; GFX8-NEXT:    v_mov_b32_e32 v9, s8
-; GFX8-NEXT:    s_add_u32 s8, s0, 64
-; GFX8-NEXT:    v_ashrrev_i32_e32 v8, 31, v7
-; GFX8-NEXT:    v_ashrrev_i32_e32 v6, 31, v5
-; GFX8-NEXT:    s_addc_u32 s9, s1, 0
-; GFX8-NEXT:    flat_store_dwordx4 v[9:10], v[5:8]
-; GFX8-NEXT:    v_mov_b32_e32 v11, s9
-; GFX8-NEXT:    v_bfe_i32 v5, v4, 0, 1
-; GFX8-NEXT:    v_bfe_i32 v3, v3, 0, 1
-; GFX8-NEXT:    v_mov_b32_e32 v10, s8
-; GFX8-NEXT:    s_add_u32 s8, s0, 48
-; GFX8-NEXT:    v_ashrrev_i32_e32 v6, 31, v5
-; GFX8-NEXT:    v_ashrrev_i32_e32 v4, 31, v3
-; GFX8-NEXT:    s_addc_u32 s9, s1, 0
-; GFX8-NEXT:    flat_store_dwordx4 v[10:11], v[3:6]
-; GFX8-NEXT:    v_bfe_i32 v1, v1, 0, 1
-; GFX8-NEXT:    v_bfe_i32 v3, v2, 0, 1
-; GFX8-NEXT:    v_mov_b32_e32 v11, s9
-; GFX8-NEXT:    v_ashrrev_i32_e32 v4, 31, v3
-; GFX8-NEXT:    v_ashrrev_i32_e32 v2, 31, v1
-; GFX8-NEXT:    v_mov_b32_e32 v10, s8
-; GFX8-NEXT:    v_lshrrev_b16_e64 v12, 6, s6
-; GFX8-NEXT:    v_lshrrev_b16_e64 v13, 7, s6
-; GFX8-NEXT:    v_lshrrev_b16_e64 v8, 4, s6
-; GFX8-NEXT:    v_lshrrev_b16_e64 v9, 5, s6
-; GFX8-NEXT:    v_lshrrev_b16_e64 v5, 2, s6
-; GFX8-NEXT:    v_lshrrev_b16_e64 v6, 3, s6
-; GFX8-NEXT:    flat_store_dwordx4 v[10:11], v[1:4]
-; GFX8-NEXT:    v_bfe_i32 v18, v16, 0, 1
-; GFX8-NEXT:    v_lshrrev_b16_e64 v1, 1, s6
+; GFX8-NEXT:    s_bfe_i64 s[8:9], s[8:9], 0x10000
+; GFX8-NEXT:    s_bfe_i64 s[6:7], s[6:7], 0x10000
+; GFX8-NEXT:    v_mov_b32_e32 v21, s6
+; GFX8-NEXT:    s_add_u32 s6, s0, 0xb0
+; GFX8-NEXT:    v_mov_b32_e32 v22, s7
+; GFX8-NEXT:    s_addc_u32 s7, s1, 0
+; GFX8-NEXT:    v_mov_b32_e32 v26, s7
+; GFX8-NEXT:    v_mov_b32_e32 v25, s6
+; GFX8-NEXT:    s_add_u32 s6, s0, 0xa0
+; GFX8-NEXT:    v_mov_b32_e32 v23, s8
+; GFX8-NEXT:    v_mov_b32_e32 v24, s9
+; GFX8-NEXT:    s_addc_u32 s7, s1, 0
+; GFX8-NEXT:    flat_store_dwordx4 v[25:26], v[21:24]
+; GFX8-NEXT:    v_mov_b32_e32 v26, s7
+; GFX8-NEXT:    v_mov_b32_e32 v25, s6
+; GFX8-NEXT:    s_add_u32 s6, s0, 0x90
+; GFX8-NEXT:    v_mov_b32_e32 v21, s10
+; GFX8-NEXT:    v_mov_b32_e32 v22, s11
+; GFX8-NEXT:    v_mov_b32_e32 v23, s12
+; GFX8-NEXT:    v_mov_b32_e32 v24, s13
+; GFX8-NEXT:    s_addc_u32 s7, s1, 0
+; GFX8-NEXT:    flat_store_dwordx4 v[25:26], v[21:24]
+; GFX8-NEXT:    v_mov_b32_e32 v26, s7
+; GFX8-NEXT:    v_mov_b32_e32 v25, s6
+; GFX8-NEXT:    s_add_u32 s6, s0, 0x80
+; GFX8-NEXT:    v_mov_b32_e32 v21, s14
+; GFX8-NEXT:    v_mov_b32_e32 v22, s15
+; GFX8-NEXT:    v_mov_b32_e32 v23, s16
+; GFX8-NEXT:    v_mov_b32_e32 v24, s17
+; GFX8-NEXT:    s_addc_u32 s7, s1, 0
+; GFX8-NEXT:    flat_store_dwordx4 v[25:26], v[21:24]
+; GFX8-NEXT:    v_mov_b32_e32 v26, s7
+; GFX8-NEXT:    v_mov_b32_e32 v21, s18
+; GFX8-NEXT:    v_mov_b32_e32 v22, s19
+; GFX8-NEXT:    v_mov_b32_e32 v23, s20
+; GFX8-NEXT:    v_mov_b32_e32 v24, s21
+; GFX8-NEXT:    v_mov_b32_e32 v25, s6
+; GFX8-NEXT:    s_add_u32 s6, s0, 0x70
+; GFX8-NEXT:    flat_store_dwordx4 v[25:26], v[21:24]
+; GFX8-NEXT:    s_addc_u32 s7, s1, 0
+; GFX8-NEXT:    v_bfe_i32 v23, v3, 0, 1
+; GFX8-NEXT:    v_bfe_i32 v21, v2, 0, 1
+; GFX8-NEXT:    v_mov_b32_e32 v2, s6
+; GFX8-NEXT:    v_mov_b32_e32 v3, s7
+; GFX8-NEXT:    s_add_u32 s6, s0, 0x60
+; GFX8-NEXT:    s_addc_u32 s7, s1, 0
+; GFX8-NEXT:    v_ashrrev_i32_e32 v24, 31, v23
+; GFX8-NEXT:    v_ashrrev_i32_e32 v22, 31, v21
+; GFX8-NEXT:    v_mov_b32_e32 v26, s7
+; GFX8-NEXT:    flat_store_dwordx4 v[2:3], v[21:24]
+; GFX8-NEXT:    v_mov_b32_e32 v25, s6
+; GFX8-NEXT:    v_bfe_i32 v23, v17, 0, 1
+; GFX8-NEXT:    v_bfe_i32 v21, v15, 0, 1
+; GFX8-NEXT:    s_add_u32 s6, s0, 0x50
+; GFX8-NEXT:    v_ashrrev_i32_e32 v24, 31, v23
+; GFX8-NEXT:    v_ashrrev_i32_e32 v22, 31, v21
+; GFX8-NEXT:    s_addc_u32 s7, s1, 0
+; GFX8-NEXT:    flat_store_dwordx4 v[25:26], v[21:24]
+; GFX8-NEXT:    v_bfe_i32 v25, v14, 0, 1
+; GFX8-NEXT:    v_bfe_i32 v23, v13, 0, 1
+; GFX8-NEXT:    v_mov_b32_e32 v14, s7
+; GFX8-NEXT:    v_mov_b32_e32 v13, s6
+; GFX8-NEXT:    s_add_u32 s6, s0, 64
+; GFX8-NEXT:    v_ashrrev_i32_e32 v26, 31, v25
+; GFX8-NEXT:    v_ashrrev_i32_e32 v24, 31, v23
+; GFX8-NEXT:    s_addc_u32 s7, s1, 0
+; GFX8-NEXT:    flat_store_dwordx4 v[13:14], v[23:26]
+; GFX8-NEXT:    v_bfe_i32 v12, v12, 0, 1
+; GFX8-NEXT:    v_bfe_i32 v25, v10, 0, 1
+; GFX8-NEXT:    v_bfe_i32 v23, v9, 0, 1
+; GFX8-NEXT:    v_mov_b32_e32 v10, s7
+; GFX8-NEXT:    v_mov_b32_e32 v9, s6
+; GFX8-NEXT:    s_add_u32 s6, s0, 48
+; GFX8-NEXT:    v_ashrrev_i32_e32 v26, 31, v25
+; GFX8-NEXT:    v_ashrrev_i32_e32 v24, 31, v23
+; GFX8-NEXT:    s_addc_u32 s7, s1, 0
+; GFX8-NEXT:    flat_store_dwordx4 v[9:10], v[23:26]
+; GFX8-NEXT:    v_bfe_i32 v10, v11, 0, 1
+; GFX8-NEXT:    v_bfe_i32 v25, v8, 0, 1
+; GFX8-NEXT:    v_bfe_i32 v23, v6, 0, 1
+; GFX8-NEXT:    v_mov_b32_e32 v9, s7
+; GFX8-NEXT:    v_ashrrev_i32_e32 v26, 31, v25
+; GFX8-NEXT:    v_ashrrev_i32_e32 v24, 31, v23
+; GFX8-NEXT:    v_mov_b32_e32 v8, s6
 ; GFX8-NEXT:    s_add_u32 s6, s0, 32
-; GFX8-NEXT:    v_bfe_i32 v2, v1, 0, 1
-; GFX8-NEXT:    v_bfe_i32 v16, v0, 0, 1
+; GFX8-NEXT:    flat_store_dwordx4 v[8:9], v[23:26]
 ; GFX8-NEXT:    s_addc_u32 s7, s1, 0
-; GFX8-NEXT:    v_mov_b32_e32 v0, s6
-; GFX8-NEXT:    v_ashrrev_i32_e32 v19, 31, v18
-; GFX8-NEXT:    v_ashrrev_i32_e32 v17, 31, v16
-; GFX8-NEXT:    v_mov_b32_e32 v1, s7
+; GFX8-NEXT:    v_bfe_i32 v25, v5, 0, 1
+; GFX8-NEXT:    v_bfe_i32 v23, v4, 0, 1
+; GFX8-NEXT:    v_mov_b32_e32 v4, s6
+; GFX8-NEXT:    v_ashrrev_i32_e32 v26, 31, v25
+; GFX8-NEXT:    v_ashrrev_i32_e32 v24, 31, v23
+; GFX8-NEXT:    v_mov_b32_e32 v5, s7
 ; GFX8-NEXT:    s_add_u32 s6, s0, 16
-; GFX8-NEXT:    flat_store_dwordx4 v[0:1], v[16:19]
-; GFX8-NEXT:    v_bfe_i32 v20, v20, 0, 1
-; GFX8-NEXT:    v_bfe_i32 v18, v22, 0, 1
-; GFX8-NEXT:    v_bfe_i32 v22, v21, 0, 1
+; GFX8-NEXT:    flat_store_dwordx4 v[4:5], v[23:26]
 ; GFX8-NEXT:    s_addc_u32 s7, s1, 0
+; GFX8-NEXT:    v_bfe_i32 v25, v1, 0, 1
+; GFX8-NEXT:    v_bfe_i32 v23, v0, 0, 1
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s6
-; GFX8-NEXT:    v_ashrrev_i32_e32 v23, 31, v22
-; GFX8-NEXT:    v_ashrrev_i32_e32 v21, 31, v20
+; GFX8-NEXT:    v_ashrrev_i32_e32 v26, 31, v25
+; GFX8-NEXT:    v_ashrrev_i32_e32 v24, 31, v23
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s7
-; GFX8-NEXT:    flat_store_dwordx4 v[0:1], v[20:23]
+; GFX8-NEXT:    v_bfe_i32 v6, v7, 0, 1
+; GFX8-NEXT:    flat_store_dwordx4 v[0:1], v[23:26]
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s0
-; GFX8-NEXT:    v_ashrrev_i32_e32 v19, 31, v18
-; GFX8-NEXT:    v_mov_b32_e32 v16, s4
-; GFX8-NEXT:    v_mov_b32_e32 v17, s5
+; GFX8-NEXT:    v_ashrrev_i32_e32 v7, 31, v6
+; GFX8-NEXT:    v_mov_b32_e32 v4, s4
+; GFX8-NEXT:    v_mov_b32_e32 v5, s5
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s1
 ; GFX8-NEXT:    s_add_u32 s4, s0, 0xf0
-; GFX8-NEXT:    v_bfe_i32 v14, v13, 0, 1
-; GFX8-NEXT:    v_bfe_i32 v12, v12, 0, 1
-; GFX8-NEXT:    flat_store_dwordx4 v[0:1], v[16:19]
+; GFX8-NEXT:    flat_store_dwordx4 v[0:1], v[4:7]
 ; GFX8-NEXT:    s_addc_u32 s5, s1, 0
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s4
-; GFX8-NEXT:    v_ashrrev_i32_e32 v15, 31, v14
 ; GFX8-NEXT:    v_ashrrev_i32_e32 v13, 31, v12
+; GFX8-NEXT:    v_ashrrev_i32_e32 v11, 31, v10
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s5
 ; GFX8-NEXT:    s_add_u32 s4, s0, 0xe0
-; GFX8-NEXT:    flat_store_dwordx4 v[0:1], v[12:15]
+; GFX8-NEXT:    flat_store_dwordx4 v[0:1], v[10:13]
 ; GFX8-NEXT:    s_addc_u32 s5, s1, 0
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s4
-; GFX8-NEXT:    v_bfe_i32 v10, v9, 0, 1
-; GFX8-NEXT:    v_bfe_i32 v8, v8, 0, 1
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s5
 ; GFX8-NEXT:    s_add_u32 s4, s0, 0xd0
-; GFX8-NEXT:    v_ashrrev_i32_e32 v11, 31, v10
-; GFX8-NEXT:    v_ashrrev_i32_e32 v9, 31, v8
+; GFX8-NEXT:    v_bfe_i32 v17, v18, 0, 1
+; GFX8-NEXT:    v_bfe_i32 v15, v16, 0, 1
 ; GFX8-NEXT:    s_addc_u32 s5, s1, 0
-; GFX8-NEXT:    v_bfe_i32 v6, v6, 0, 1
-; GFX8-NEXT:    v_bfe_i32 v4, v5, 0, 1
-; GFX8-NEXT:    flat_store_dwordx4 v[0:1], v[8:11]
-; GFX8-NEXT:    v_mov_b32_e32 v0, s4
+; GFX8-NEXT:    v_ashrrev_i32_e32 v18, 31, v17
+; GFX8-NEXT:    v_ashrrev_i32_e32 v16, 31, v15
 ; GFX8-NEXT:    s_add_u32 s0, s0, 0xc0
-; GFX8-NEXT:    v_ashrrev_i32_e32 v7, 31, v6
-; GFX8-NEXT:    v_ashrrev_i32_e32 v5, 31, v4
-; GFX8-NEXT:    v_mov_b32_e32 v1, s5
+; GFX8-NEXT:    v_bfe_i32 v21, v20, 0, 1
+; GFX8-NEXT:    v_bfe_i32 v19, v19, 0, 1
+; GFX8-NEXT:    flat_store_dwordx4 v[0:1], v[15:18]
+; GFX8-NEXT:    v_mov_b32_e32 v0, s4
 ; GFX8-NEXT:    s_addc_u32 s1, s1, 0
-; GFX8-NEXT:    flat_store_dwordx4 v[0:1], v[4:7]
-; GFX8-NEXT:    v_ashrrev_i32_e32 v3, 31, v2
+; GFX8-NEXT:    v_bfe_i32 v2, v27, 0, 1
+; GFX8-NEXT:    v_ashrrev_i32_e32 v22, 31, v21
+; GFX8-NEXT:    v_ashrrev_i32_e32 v20, 31, v19
+; GFX8-NEXT:    v_mov_b32_e32 v1, s5
 ; GFX8-NEXT:    v_mov_b32_e32 v5, s1
+; GFX8-NEXT:    v_ashrrev_i32_e32 v3, 31, v2
+; GFX8-NEXT:    flat_store_dwordx4 v[0:1], v[19:22]
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s3
 ; GFX8-NEXT:    v_mov_b32_e32 v4, s0
@@ -7204,34 +7204,36 @@ define amdgpu_kernel void @constant_sextload_v64i1_to_v64i64(ptr addrspace(1) %o
 ; GFX8-LABEL: constant_sextload_v64i1_to_v64i64:
 ; GFX8:       ; %bb.0:
 ; GFX8-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GFX8-NEXT:    s_mov_b32 s13, 0
-; GFX8-NEXT:    s_mov_b32 s11, s13
+; GFX8-NEXT:    s_mov_b32 s7, 0
+; GFX8-NEXT:    s_mov_b32 s13, s7
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_load_dwordx2 s[8:9], s[2:3], 0x0
+; GFX8-NEXT:    s_load_dwordx2 s[10:11], s[2:3], 0x0
+; GFX8-NEXT:    v_mov_b32_e32 v29, s1
+; GFX8-NEXT:    v_mov_b32_e32 v28, s0
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NEXT:    s_lshr_b32 s16, s9, 22
-; GFX8-NEXT:    s_lshr_b32 s18, s9, 23
-; GFX8-NEXT:    s_lshr_b32 s20, s9, 20
-; GFX8-NEXT:    s_lshr_b32 s22, s9, 21
-; GFX8-NEXT:    s_lshr_b32 s24, s9, 18
-; GFX8-NEXT:    s_lshr_b32 s26, s9, 19
-; GFX8-NEXT:    s_lshr_b32 s28, s9, 16
-; GFX8-NEXT:    s_lshr_b32 s30, s9, 17
-; GFX8-NEXT:    s_lshr_b32 s34, s8, 22
-; GFX8-NEXT:    s_lshr_b32 s36, s8, 23
-; GFX8-NEXT:    s_lshr_b32 s38, s8, 20
-; GFX8-NEXT:    s_lshr_b32 s40, s8, 21
-; GFX8-NEXT:    s_lshr_b32 s42, s8, 18
-; GFX8-NEXT:    s_lshr_b32 s44, s8, 19
-; GFX8-NEXT:    s_lshr_b32 s46, s8, 16
-; GFX8-NEXT:    s_lshr_b32 s48, s8, 17
-; GFX8-NEXT:    s_mov_b32 s12, s9
-; GFX8-NEXT:    s_lshr_b32 s10, s9, 24
-; GFX8-NEXT:    s_lshr_b32 s6, s8, 24
-; GFX8-NEXT:    s_bfe_i64 s[2:3], s[6:7], 0x10000
-; GFX8-NEXT:    s_bfe_i64 s[4:5], s[10:11], 0x10000
-; GFX8-NEXT:    s_bfe_i64 s[12:13], s[12:13], 0x10000
-; GFX8-NEXT:    s_bfe_i64 s[14:15], s[8:9], 0x10000
+; GFX8-NEXT:    s_lshr_b32 s16, s11, 22
+; GFX8-NEXT:    s_lshr_b32 s18, s11, 23
+; GFX8-NEXT:    s_lshr_b32 s20, s11, 20
+; GFX8-NEXT:    s_lshr_b32 s22, s11, 21
+; GFX8-NEXT:    s_lshr_b32 s24, s11, 18
+; GFX8-NEXT:    s_lshr_b32 s26, s11, 19
+; GFX8-NEXT:    s_lshr_b32 s28, s11, 16
+; GFX8-NEXT:    s_lshr_b32 s30, s11, 17
+; GFX8-NEXT:    s_lshr_b32 s34, s10, 22
+; GFX8-NEXT:    s_lshr_b32 s36, s10, 23
+; GFX8-NEXT:    s_lshr_b32 s38, s10, 20
+; GFX8-NEXT:    s_lshr_b32 s40, s10, 21
+; GFX8-NEXT:    s_lshr_b32 s42, s10, 18
+; GFX8-NEXT:    s_lshr_b32 s44, s10, 19
+; GFX8-NEXT:    s_lshr_b32 s46, s10, 16
+; GFX8-NEXT:    s_lshr_b32 s48, s10, 17
+; GFX8-NEXT:    s_mov_b32 s6, s11
+; GFX8-NEXT:    s_lshr_b32 s12, s11, 24
+; GFX8-NEXT:    s_lshr_b32 s8, s10, 24
+; GFX8-NEXT:    s_bfe_i64 s[2:3], s[8:9], 0x10000
+; GFX8-NEXT:    s_bfe_i64 s[4:5], s[12:13], 0x10000
+; GFX8-NEXT:    s_bfe_i64 s[6:7], s[6:7], 0x10000
+; GFX8-NEXT:    s_bfe_i64 s[14:15], s[10:11], 0x10000
 ; GFX8-NEXT:    s_bfe_i64 s[48:49], s[48:49], 0x10000
 ; GFX8-NEXT:    s_bfe_i64 s[46:47], s[46:47], 0x10000
 ; GFX8-NEXT:    s_bfe_i64 s[44:45], s[44:45], 0x10000
@@ -7248,323 +7250,321 @@ define amdgpu_kernel void @constant_sextload_v64i1_to_v64i64(ptr addrspace(1) %o
 ; GFX8-NEXT:    s_bfe_i64 s[20:21], s[20:21], 0x10000
 ; GFX8-NEXT:    s_bfe_i64 s[18:19], s[18:19], 0x10000
 ; GFX8-NEXT:    s_bfe_i64 s[16:17], s[16:17], 0x10000
-; GFX8-NEXT:    v_mov_b32_e32 v11, s16
+; GFX8-NEXT:    v_mov_b32_e32 v22, s16
 ; GFX8-NEXT:    s_add_u32 s16, s0, 0x1b0
-; GFX8-NEXT:    v_mov_b32_e32 v12, s17
+; GFX8-NEXT:    v_mov_b32_e32 v23, s17
 ; GFX8-NEXT:    s_addc_u32 s17, s1, 0
-; GFX8-NEXT:    v_mov_b32_e32 v15, s16
-; GFX8-NEXT:    v_mov_b32_e32 v13, s18
-; GFX8-NEXT:    v_mov_b32_e32 v14, s19
-; GFX8-NEXT:    v_mov_b32_e32 v16, s17
+; GFX8-NEXT:    v_mov_b32_e32 v27, s17
+; GFX8-NEXT:    v_mov_b32_e32 v26, s16
 ; GFX8-NEXT:    s_add_u32 s16, s0, 0x1a0
-; GFX8-NEXT:    flat_store_dwordx4 v[15:16], v[11:14]
+; GFX8-NEXT:    v_mov_b32_e32 v24, s18
+; GFX8-NEXT:    v_mov_b32_e32 v25, s19
 ; GFX8-NEXT:    s_addc_u32 s17, s1, 0
-; GFX8-NEXT:    v_mov_b32_e32 v15, s16
-; GFX8-NEXT:    v_mov_b32_e32 v11, s20
-; GFX8-NEXT:    v_mov_b32_e32 v12, s21
-; GFX8-NEXT:    v_mov_b32_e32 v13, s22
-; GFX8-NEXT:    v_mov_b32_e32 v14, s23
-; GFX8-NEXT:    v_mov_b32_e32 v16, s17
+; GFX8-NEXT:    flat_store_dwordx4 v[26:27], v[22:25]
+; GFX8-NEXT:    v_mov_b32_e32 v27, s17
+; GFX8-NEXT:    v_mov_b32_e32 v26, s16
 ; GFX8-NEXT:    s_add_u32 s16, s0, 0x190
-; GFX8-NEXT:    flat_store_dwordx4 v[15:16], v[11:14]
+; GFX8-NEXT:    v_mov_b32_e32 v22, s20
+; GFX8-NEXT:    v_mov_b32_e32 v23, s21
+; GFX8-NEXT:    v_mov_b32_e32 v24, s22
+; GFX8-NEXT:    v_mov_b32_e32 v25, s23
 ; GFX8-NEXT:    s_addc_u32 s17, s1, 0
-; GFX8-NEXT:    v_mov_b32_e32 v15, s16
-; GFX8-NEXT:    v_mov_b32_e32 v11, s24
-; GFX8-NEXT:    v_mov_b32_e32 v12, s25
-; GFX8-NEXT:    v_mov_b32_e32 v13, s26
-; GFX8-NEXT:    v_mov_b32_e32 v14, s27
-; GFX8-NEXT:    v_mov_b32_e32 v16, s17
+; GFX8-NEXT:    flat_store_dwordx4 v[26:27], v[22:25]
+; GFX8-NEXT:    v_mov_b32_e32 v27, s17
+; GFX8-NEXT:    v_mov_b32_e32 v26, s16
 ; GFX8-NEXT:    s_add_u32 s16, s0, 0x180
-; GFX8-NEXT:    flat_store_dwordx4 v[15:16], v[11:14]
+; GFX8-NEXT:    v_mov_b32_e32 v22, s24
+; GFX8-NEXT:    v_mov_b32_e32 v23, s25
+; GFX8-NEXT:    v_mov_b32_e32 v24, s26
+; GFX8-NEXT:    v_mov_b32_e32 v25, s27
 ; GFX8-NEXT:    s_addc_u32 s17, s1, 0
-; GFX8-NEXT:    v_mov_b32_e32 v15, s16
-; GFX8-NEXT:    v_mov_b32_e32 v11, s28
-; GFX8-NEXT:    v_mov_b32_e32 v12, s29
-; GFX8-NEXT:    v_mov_b32_e32 v13, s30
-; GFX8-NEXT:    v_mov_b32_e32 v14, s31
-; GFX8-NEXT:    v_mov_b32_e32 v16, s17
+; GFX8-NEXT:    flat_store_dwordx4 v[26:27], v[22:25]
+; GFX8-NEXT:    v_mov_b32_e32 v27, s17
+; GFX8-NEXT:    v_mov_b32_e32 v26, s16
 ; GFX8-NEXT:    s_add_u32 s16, s0, 0xb0
-; GFX8-NEXT:    flat_store_dwordx4 v[15:16], v[11:14]
+; GFX8-NEXT:    v_mov_b32_e32 v22, s28
+; GFX8-NEXT:    v_mov_b32_e32 v23, s29
+; GFX8-NEXT:    v_mov_b32_e32 v24, s30
+; GFX8-NEXT:    v_mov_b32_e32 v25, s31
 ; GFX8-NEXT:    s_addc_u32 s17, s1, 0
-; GFX8-NEXT:    v_mov_b32_e32 v15, s16
-; GFX8-NEXT:    v_mov_b32_e32 v11, s34
-; GFX8-NEXT:    v_mov_b32_e32 v12, s35
-; GFX8-NEXT:    v_mov_b32_e32 v13, s36
-; GFX8-NEXT:    v_mov_b32_e32 v14, s37
-; GFX8-NEXT:    v_mov_b32_e32 v16, s17
+; GFX8-NEXT:    flat_store_dwordx4 v[26:27], v[22:25]
+; GFX8-NEXT:    v_mov_b32_e32 v27, s17
+; GFX8-NEXT:    v_mov_b32_e32 v26, s16
 ; GFX8-NEXT:    s_add_u32 s16, s0, 0xa0
-; GFX8-NEXT:    flat_store_dwordx4 v[15:16], v[11:14]
+; GFX8-NEXT:    v_mov_b32_e32 v22, s34
+; GFX8-NEXT:    v_mov_b32_e32 v23, s35
+; GFX8-NEXT:    v_mov_b32_e32 v24, s36
+; GFX8-NEXT:    v_mov_b32_e32 v25, s37
 ; GFX8-NEXT:    s_addc_u32 s17, s1, 0
-; GFX8-NEXT:    v_mov_b32_e32 v15, s16
-; GFX8-NEXT:    v_mov_b32_e32 v11, s38
-; GFX8-NEXT:    v_mov_b32_e32 v12, s39
-; GFX8-NEXT:    v_mov_b32_e32 v13, s40
-; GFX8-NEXT:    v_mov_b32_e32 v14, s41
-; GFX8-NEXT:    v_mov_b32_e32 v16, s17
+; GFX8-NEXT:    flat_store_dwordx4 v[26:27], v[22:25]
+; GFX8-NEXT:    v_mov_b32_e32 v27, s17
+; GFX8-NEXT:    v_mov_b32_e32 v26, s16
 ; GFX8-NEXT:    s_add_u32 s16, s0, 0x90
-; GFX8-NEXT:    flat_store_dwordx4 v[15:16], v[11:14]
+; GFX8-NEXT:    v_mov_b32_e32 v22, s38
+; GFX8-NEXT:    v_mov_b32_e32 v23, s39
+; GFX8-NEXT:    v_mov_b32_e32 v24, s40
+; GFX8-NEXT:    v_mov_b32_e32 v25, s41
 ; GFX8-NEXT:    s_addc_u32 s17, s1, 0
-; GFX8-NEXT:    v_mov_b32_e32 v15, s16
-; GFX8-NEXT:    v_mov_b32_e32 v11, s42
-; GFX8-NEXT:    v_mov_b32_e32 v12, s43
-; GFX8-NEXT:    v_mov_b32_e32 v13, s44
-; GFX8-NEXT:    v_mov_b32_e32 v14, s45
-; GFX8-NEXT:    v_mov_b32_e32 v16, s17
+; GFX8-NEXT:    flat_store_dwordx4 v[26:27], v[22:25]
+; GFX8-NEXT:    v_mov_b32_e32 v27, s17
+; GFX8-NEXT:    v_mov_b32_e32 v26, s16
 ; GFX8-NEXT:    s_add_u32 s16, s0, 0x80
-; GFX8-NEXT:    flat_store_dwordx4 v[15:16], v[11:14]
+; GFX8-NEXT:    v_mov_b32_e32 v22, s42
+; GFX8-NEXT:    v_mov_b32_e32 v23, s43
+; GFX8-NEXT:    v_mov_b32_e32 v24, s44
+; GFX8-NEXT:    v_mov_b32_e32 v25, s45
 ; GFX8-NEXT:    s_addc_u32 s17, s1, 0
-; GFX8-NEXT:    v_mov_b32_e32 v15, s16
-; GFX8-NEXT:    v_lshrrev_b16_e64 v9, 14, s8
-; GFX8-NEXT:    v_lshrrev_b16_e64 v10, 15, s8
-; GFX8-NEXT:    v_mov_b32_e32 v11, s46
-; GFX8-NEXT:    v_mov_b32_e32 v12, s47
-; GFX8-NEXT:    v_mov_b32_e32 v13, s48
-; GFX8-NEXT:    v_mov_b32_e32 v14, s49
-; GFX8-NEXT:    v_mov_b32_e32 v16, s17
+; GFX8-NEXT:    flat_store_dwordx4 v[26:27], v[22:25]
+; GFX8-NEXT:    v_mov_b32_e32 v27, s17
+; GFX8-NEXT:    v_mov_b32_e32 v26, s16
 ; GFX8-NEXT:    s_add_u32 s16, s0, 0x70
-; GFX8-NEXT:    flat_store_dwordx4 v[15:16], v[11:14]
-; GFX8-NEXT:    v_bfe_i32 v9, v9, 0, 1
-; GFX8-NEXT:    v_bfe_i32 v11, v10, 0, 1
+; GFX8-NEXT:    v_lshrrev_b16_e64 v20, 14, s10
+; GFX8-NEXT:    v_lshrrev_b16_e64 v21, 15, s10
+; GFX8-NEXT:    v_mov_b32_e32 v22, s46
+; GFX8-NEXT:    v_mov_b32_e32 v23, s47
+; GFX8-NEXT:    v_mov_b32_e32 v24, s48
+; GFX8-NEXT:    v_mov_b32_e32 v25, s49
 ; GFX8-NEXT:    s_addc_u32 s17, s1, 0
-; GFX8-NEXT:    v_mov_b32_e32 v15, s16
-; GFX8-NEXT:    v_lshrrev_b16_e64 v7, 12, s8
-; GFX8-NEXT:    v_lshrrev_b16_e64 v8, 13, s8
-; GFX8-NEXT:    v_ashrrev_i32_e32 v12, 31, v11
-; GFX8-NEXT:    v_ashrrev_i32_e32 v10, 31, v9
-; GFX8-NEXT:    v_mov_b32_e32 v16, s17
+; GFX8-NEXT:    flat_store_dwordx4 v[26:27], v[22:25]
+; GFX8-NEXT:    v_bfe_i32 v26, v21, 0, 1
+; GFX8-NEXT:    v_bfe_i32 v24, v20, 0, 1
+; GFX8-NEXT:    v_mov_b32_e32 v21, s17
+; GFX8-NEXT:    v_mov_b32_e32 v20, s16
 ; GFX8-NEXT:    s_add_u32 s16, s0, 0x60
-; GFX8-NEXT:    flat_store_dwordx4 v[15:16], v[9:12]
-; GFX8-NEXT:    v_bfe_i32 v7, v7, 0, 1
-; GFX8-NEXT:    v_bfe_i32 v9, v8, 0, 1
+; GFX8-NEXT:    v_lshrrev_b16_e64 v18, 12, s10
+; GFX8-NEXT:    v_lshrrev_b16_e64 v19, 13, s10
+; GFX8-NEXT:    v_ashrrev_i32_e32 v27, 31, v26
+; GFX8-NEXT:    v_ashrrev_i32_e32 v25, 31, v24
 ; GFX8-NEXT:    s_addc_u32 s17, s1, 0
-; GFX8-NEXT:    v_mov_b32_e32 v15, s16
-; GFX8-NEXT:    v_lshrrev_b16_e64 v5, 10, s8
-; GFX8-NEXT:    v_lshrrev_b16_e64 v6, 11, s8
-; GFX8-NEXT:    v_ashrrev_i32_e32 v10, 31, v9
-; GFX8-NEXT:    v_ashrrev_i32_e32 v8, 31, v7
-; GFX8-NEXT:    v_mov_b32_e32 v16, s17
+; GFX8-NEXT:    flat_store_dwordx4 v[20:21], v[24:27]
+; GFX8-NEXT:    v_lshrrev_b16_e64 v16, 10, s10
+; GFX8-NEXT:    v_bfe_i32 v26, v19, 0, 1
+; GFX8-NEXT:    v_bfe_i32 v24, v18, 0, 1
+; GFX8-NEXT:    v_mov_b32_e32 v19, s17
+; GFX8-NEXT:    v_lshrrev_b16_e64 v17, 11, s10
+; GFX8-NEXT:    v_ashrrev_i32_e32 v27, 31, v26
+; GFX8-NEXT:    v_ashrrev_i32_e32 v25, 31, v24
+; GFX8-NEXT:    v_mov_b32_e32 v18, s16
 ; GFX8-NEXT:    s_add_u32 s16, s0, 0x50
-; GFX8-NEXT:    flat_store_dwordx4 v[15:16], v[7:10]
-; GFX8-NEXT:    v_bfe_i32 v5, v5, 0, 1
-; GFX8-NEXT:    v_bfe_i32 v7, v6, 0, 1
+; GFX8-NEXT:    flat_store_dwordx4 v[18:19], v[24:27]
 ; GFX8-NEXT:    s_addc_u32 s17, s1, 0
-; GFX8-NEXT:    v_mov_b32_e32 v15, s16
-; GFX8-NEXT:    v_lshrrev_b16_e64 v3, 8, s8
-; GFX8-NEXT:    v_lshrrev_b16_e64 v4, 9, s8
-; GFX8-NEXT:    v_ashrrev_i32_e32 v8, 31, v7
-; GFX8-NEXT:    v_ashrrev_i32_e32 v6, 31, v5
-; GFX8-NEXT:    v_mov_b32_e32 v16, s17
-; GFX8-NEXT:    s_add_u32 s16, s0, 64
-; GFX8-NEXT:    flat_store_dwordx4 v[15:16], v[5:8]
+; GFX8-NEXT:    v_bfe_i32 v26, v17, 0, 1
+; GFX8-NEXT:    v_bfe_i32 v24, v16, 0, 1
+; GFX8-NEXT:    v_mov_b32_e32 v16, s16
+; GFX8-NEXT:    v_lshrrev_b16_e64 v14, 8, s10
+; GFX8-NEXT:    v_lshrrev_b16_e64 v15, 9, s10
+; GFX8-NEXT:    v_lshrrev_b16_e64 v12, 6, s10
+; GFX8-NEXT:    v_lshrrev_b16_e64 v13, 7, s10
+; GFX8-NEXT:    v_lshrrev_b16_e64 v10, 4, s10
+; GFX8-NEXT:    v_lshrrev_b16_e64 v11, 5, s10
+; GFX8-NEXT:    v_lshrrev_b16_e64 v8, 2, s10
+; GFX8-NEXT:    v_lshrrev_b16_e64 v9, 3, s10
+; GFX8-NEXT:    v_lshrrev_b16_e64 v7, 1, s10
+; GFX8-NEXT:    v_ashrrev_i32_e32 v27, 31, v26
+; GFX8-NEXT:    v_ashrrev_i32_e32 v25, 31, v24
+; GFX8-NEXT:    v_mov_b32_e32 v17, s17
+; GFX8-NEXT:    s_add_u32 s10, s0, 64
+; GFX8-NEXT:    v_lshrrev_b16_e64 v5, 14, s11
+; GFX8-NEXT:    v_lshrrev_b16_e64 v6, 15, s11
+; GFX8-NEXT:    v_lshrrev_b16_e64 v3, 12, s11
+; GFX8-NEXT:    v_lshrrev_b16_e64 v4, 13, s11
+; GFX8-NEXT:    v_lshrrev_b16_e64 v1, 10, s11
+; GFX8-NEXT:    v_lshrrev_b16_e64 v2, 11, s11
+; GFX8-NEXT:    v_lshrrev_b16_e64 v0, 8, s11
+; GFX8-NEXT:    v_lshrrev_b16_e64 v23, 9, s11
+; GFX8-NEXT:    v_lshrrev_b16_e64 v22, 6, s11
+; GFX8-NEXT:    v_lshrrev_b16_e64 v21, 7, s11
+; GFX8-NEXT:    v_lshrrev_b16_e64 v20, 4, s11
+; GFX8-NEXT:    v_lshrrev_b16_e64 v19, 5, s11
+; GFX8-NEXT:    v_lshrrev_b16_e64 v18, 2, s11
+; GFX8-NEXT:    flat_store_dwordx4 v[16:17], v[24:27]
+; GFX8-NEXT:    v_lshrrev_b16_e64 v17, 3, s11
+; GFX8-NEXT:    v_lshrrev_b16_e64 v16, 1, s11
+; GFX8-NEXT:    s_addc_u32 s11, s1, 0
+; GFX8-NEXT:    v_bfe_i32 v26, v15, 0, 1
+; GFX8-NEXT:    v_bfe_i32 v24, v14, 0, 1
+; GFX8-NEXT:    v_mov_b32_e32 v15, s11
+; GFX8-NEXT:    v_mov_b32_e32 v14, s10
+; GFX8-NEXT:    s_add_u32 s10, s0, 48
+; GFX8-NEXT:    v_ashrrev_i32_e32 v27, 31, v26
+; GFX8-NEXT:    v_ashrrev_i32_e32 v25, 31, v24
+; GFX8-NEXT:    s_addc_u32 s11, s1, 0
+; GFX8-NEXT:    flat_store_dwordx4 v[14:15], v[24:27]
 ; GFX8-NEXT:    v_bfe_i32 v3, v3, 0, 1
+; GFX8-NEXT:    v_bfe_i32 v26, v13, 0, 1
+; GFX8-NEXT:    v_bfe_i32 v24, v12, 0, 1
+; GFX8-NEXT:    v_mov_b32_e32 v13, s11
+; GFX8-NEXT:    v_ashrrev_i32_e32 v27, 31, v26
+; GFX8-NEXT:    v_ashrrev_i32_e32 v25, 31, v24
+; GFX8-NEXT:    v_mov_b32_e32 v12, s10
+; GFX8-NEXT:    s_add_u32 s10, s0, 32
+; GFX8-NEXT:    flat_store_dwordx4 v[12:13], v[24:27]
+; GFX8-NEXT:    s_addc_u32 s11, s1, 0
+; GFX8-NEXT:    v_bfe_i32 v26, v11, 0, 1
+; GFX8-NEXT:    v_bfe_i32 v24, v10, 0, 1
+; GFX8-NEXT:    v_mov_b32_e32 v10, s10
+; GFX8-NEXT:    v_ashrrev_i32_e32 v27, 31, v26
+; GFX8-NEXT:    v_ashrrev_i32_e32 v25, 31, v24
+; GFX8-NEXT:    v_mov_b32_e32 v11, s11
+; GFX8-NEXT:    s_add_u32 s10, s0, 16
+; GFX8-NEXT:    flat_store_dwordx4 v[10:11], v[24:27]
+; GFX8-NEXT:    s_addc_u32 s11, s1, 0
+; GFX8-NEXT:    v_bfe_i32 v26, v9, 0, 1
+; GFX8-NEXT:    v_bfe_i32 v24, v8, 0, 1
+; GFX8-NEXT:    v_mov_b32_e32 v8, s10
+; GFX8-NEXT:    v_ashrrev_i32_e32 v27, 31, v26
+; GFX8-NEXT:    v_ashrrev_i32_e32 v25, 31, v24
+; GFX8-NEXT:    v_mov_b32_e32 v9, s11
+; GFX8-NEXT:    flat_store_dwordx4 v[8:9], v[24:27]
+; GFX8-NEXT:    s_add_u32 s10, s0, 0x170
+; GFX8-NEXT:    v_bfe_i32 v26, v7, 0, 1
+; GFX8-NEXT:    v_ashrrev_i32_e32 v27, 31, v26
+; GFX8-NEXT:    v_mov_b32_e32 v24, s14
+; GFX8-NEXT:    v_mov_b32_e32 v25, s15
+; GFX8-NEXT:    flat_store_dwordx4 v[28:29], v[24:27]
+; GFX8-NEXT:    s_addc_u32 s11, s1, 0
+; GFX8-NEXT:    v_bfe_i32 v26, v6, 0, 1
+; GFX8-NEXT:    v_bfe_i32 v24, v5, 0, 1
+; GFX8-NEXT:    v_mov_b32_e32 v5, s10
+; GFX8-NEXT:    v_mov_b32_e32 v6, s11
+; GFX8-NEXT:    s_add_u32 s10, s0, 0x160
+; GFX8-NEXT:    v_ashrrev_i32_e32 v27, 31, v26
+; GFX8-NEXT:    v_ashrrev_i32_e32 v25, 31, v24
+; GFX8-NEXT:    s_addc_u32 s11, s1, 0
+; GFX8-NEXT:    flat_store_dwordx4 v[5:6], v[24:27]
 ; GFX8-NEXT:    v_bfe_i32 v5, v4, 0, 1
-; GFX8-NEXT:    s_addc_u32 s17, s1, 0
-; GFX8-NEXT:    v_mov_b32_e32 v15, s16
-; GFX8-NEXT:    v_lshrrev_b16_e64 v1, 6, s8
-; GFX8-NEXT:    v_lshrrev_b16_e64 v2, 7, s8
+; GFX8-NEXT:    v_mov_b32_e32 v25, s11
 ; GFX8-NEXT:    v_ashrrev_i32_e32 v6, 31, v5
 ; GFX8-NEXT:    v_ashrrev_i32_e32 v4, 31, v3
-; GFX8-NEXT:    v_mov_b32_e32 v16, s17
-; GFX8-NEXT:    s_add_u32 s16, s0, 48
-; GFX8-NEXT:    flat_store_dwordx4 v[15:16], v[3:6]
+; GFX8-NEXT:    v_mov_b32_e32 v24, s10
+; GFX8-NEXT:    s_add_u32 s10, s0, 0x150
+; GFX8-NEXT:    flat_store_dwordx4 v[24:25], v[3:6]
 ; GFX8-NEXT:    v_bfe_i32 v1, v1, 0, 1
 ; GFX8-NEXT:    v_bfe_i32 v3, v2, 0, 1
-; GFX8-NEXT:    s_addc_u32 s17, s1, 0
-; GFX8-NEXT:    v_mov_b32_e32 v15, s16
+; GFX8-NEXT:    s_addc_u32 s11, s1, 0
+; GFX8-NEXT:    v_mov_b32_e32 v5, s10
 ; GFX8-NEXT:    v_ashrrev_i32_e32 v4, 31, v3
 ; GFX8-NEXT:    v_ashrrev_i32_e32 v2, 31, v1
-; GFX8-NEXT:    v_mov_b32_e32 v16, s17
-; GFX8-NEXT:    s_add_u32 s16, s0, 32
-; GFX8-NEXT:    v_lshrrev_b16_e64 v0, 4, s8
-; GFX8-NEXT:    v_lshrrev_b16_e64 v13, 5, s8
-; GFX8-NEXT:    flat_store_dwordx4 v[15:16], v[1:4]
-; GFX8-NEXT:    s_addc_u32 s17, s1, 0
-; GFX8-NEXT:    v_mov_b32_e32 v16, s16
-; GFX8-NEXT:    v_bfe_i32 v2, v13, 0, 1
-; GFX8-NEXT:    v_bfe_i32 v0, v0, 0, 1
-; GFX8-NEXT:    v_mov_b32_e32 v17, s17
-; GFX8-NEXT:    s_add_u32 s16, s0, 16
-; GFX8-NEXT:    v_lshrrev_b16_e64 v14, 2, s8
-; GFX8-NEXT:    v_lshrrev_b16_e64 v11, 3, s8
-; GFX8-NEXT:    v_ashrrev_i32_e32 v3, 31, v2
-; GFX8-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; GFX8-NEXT:    s_addc_u32 s17, s1, 0
-; GFX8-NEXT:    flat_store_dwordx4 v[16:17], v[0:3]
-; GFX8-NEXT:    v_mov_b32_e32 v18, s17
-; GFX8-NEXT:    v_bfe_i32 v2, v11, 0, 1
-; GFX8-NEXT:    v_bfe_i32 v0, v14, 0, 1
-; GFX8-NEXT:    v_lshrrev_b16_e64 v12, 1, s8
-; GFX8-NEXT:    v_ashrrev_i32_e32 v3, 31, v2
-; GFX8-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; GFX8-NEXT:    v_mov_b32_e32 v17, s16
-; GFX8-NEXT:    flat_store_dwordx4 v[17:18], v[0:3]
-; GFX8-NEXT:    v_mov_b32_e32 v18, s1
-; GFX8-NEXT:    v_bfe_i32 v2, v12, 0, 1
-; GFX8-NEXT:    v_lshrrev_b16_e64 v9, 14, s9
-; GFX8-NEXT:    v_lshrrev_b16_e64 v10, 15, s9
-; GFX8-NEXT:    v_ashrrev_i32_e32 v3, 31, v2
-; GFX8-NEXT:    v_mov_b32_e32 v0, s14
-; GFX8-NEXT:    v_mov_b32_e32 v1, s15
-; GFX8-NEXT:    v_mov_b32_e32 v17, s0
-; GFX8-NEXT:    s_add_u32 s14, s0, 0x170
-; GFX8-NEXT:    flat_store_dwordx4 v[17:18], v[0:3]
-; GFX8-NEXT:    s_addc_u32 s15, s1, 0
-; GFX8-NEXT:    v_bfe_i32 v2, v10, 0, 1
-; GFX8-NEXT:    v_bfe_i32 v0, v9, 0, 1
-; GFX8-NEXT:    v_mov_b32_e32 v9, s14
-; GFX8-NEXT:    v_lshrrev_b16_e64 v7, 12, s9
-; GFX8-NEXT:    v_lshrrev_b16_e64 v8, 13, s9
-; GFX8-NEXT:    v_ashrrev_i32_e32 v3, 31, v2
-; GFX8-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; GFX8-NEXT:    v_mov_b32_e32 v10, s15
-; GFX8-NEXT:    s_add_u32 s8, s0, 0x160
-; GFX8-NEXT:    v_lshrrev_b16_e64 v5, 10, s9
-; GFX8-NEXT:    v_lshrrev_b16_e64 v6, 11, s9
-; GFX8-NEXT:    v_lshrrev_b16_e64 v4, 8, s9
-; GFX8-NEXT:    v_lshrrev_b16_e64 v15, 9, s9
-; GFX8-NEXT:    v_lshrrev_b16_e64 v13, 6, s9
-; GFX8-NEXT:    v_lshrrev_b16_e64 v16, 7, s9
-; GFX8-NEXT:    v_lshrrev_b16_e64 v11, 4, s9
-; GFX8-NEXT:    v_lshrrev_b16_e64 v12, 5, s9
-; GFX8-NEXT:    v_lshrrev_b16_e64 v14, 2, s9
-; GFX8-NEXT:    flat_store_dwordx4 v[9:10], v[0:3]
-; GFX8-NEXT:    v_lshrrev_b16_e64 v9, 3, s9
-; GFX8-NEXT:    v_lshrrev_b16_e64 v10, 1, s9
-; GFX8-NEXT:    v_bfe_i32 v2, v8, 0, 1
-; GFX8-NEXT:    v_bfe_i32 v0, v7, 0, 1
-; GFX8-NEXT:    s_addc_u32 s9, s1, 0
-; GFX8-NEXT:    v_mov_b32_e32 v7, s8
-; GFX8-NEXT:    v_ashrrev_i32_e32 v3, 31, v2
-; GFX8-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; GFX8-NEXT:    v_mov_b32_e32 v8, s9
-; GFX8-NEXT:    s_add_u32 s8, s0, 0x150
-; GFX8-NEXT:    flat_store_dwordx4 v[7:8], v[0:3]
-; GFX8-NEXT:    s_addc_u32 s9, s1, 0
-; GFX8-NEXT:    v_bfe_i32 v2, v6, 0, 1
-; GFX8-NEXT:    v_bfe_i32 v0, v5, 0, 1
-; GFX8-NEXT:    v_mov_b32_e32 v5, s8
-; GFX8-NEXT:    v_ashrrev_i32_e32 v3, 31, v2
-; GFX8-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; GFX8-NEXT:    v_mov_b32_e32 v6, s9
+; GFX8-NEXT:    v_mov_b32_e32 v6, s11
+; GFX8-NEXT:    v_lshrrev_b16_e64 v9, 6, s8
+; GFX8-NEXT:    v_lshrrev_b16_e64 v28, 7, s8
+; GFX8-NEXT:    v_lshrrev_b16_e64 v26, 4, s8
+; GFX8-NEXT:    v_lshrrev_b16_e64 v27, 5, s8
+; GFX8-NEXT:    v_lshrrev_b16_e64 v24, 2, s8
+; GFX8-NEXT:    v_lshrrev_b16_e64 v25, 3, s8
+; GFX8-NEXT:    flat_store_dwordx4 v[5:6], v[1:4]
+; GFX8-NEXT:    v_lshrrev_b16_e64 v6, 1, s8
 ; GFX8-NEXT:    s_add_u32 s8, s0, 0x140
-; GFX8-NEXT:    flat_store_dwordx4 v[5:6], v[0:3]
+; GFX8-NEXT:    v_bfe_i32 v2, v23, 0, 1
+; GFX8-NEXT:    v_bfe_i32 v0, v0, 0, 1
 ; GFX8-NEXT:    s_addc_u32 s9, s1, 0
-; GFX8-NEXT:    v_bfe_i32 v2, v15, 0, 1
-; GFX8-NEXT:    v_bfe_i32 v0, v4, 0, 1
 ; GFX8-NEXT:    v_mov_b32_e32 v4, s8
 ; GFX8-NEXT:    v_ashrrev_i32_e32 v3, 31, v2
 ; GFX8-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GFX8-NEXT:    v_mov_b32_e32 v5, s9
 ; GFX8-NEXT:    s_add_u32 s8, s0, 0x130
 ; GFX8-NEXT:    flat_store_dwordx4 v[4:5], v[0:3]
+; GFX8-NEXT:    v_bfe_i32 v4, v22, 0, 1
+; GFX8-NEXT:    v_bfe_i32 v2, v6, 0, 1
+; GFX8-NEXT:    v_bfe_i32 v6, v21, 0, 1
 ; GFX8-NEXT:    s_addc_u32 s9, s1, 0
-; GFX8-NEXT:    v_bfe_i32 v2, v16, 0, 1
-; GFX8-NEXT:    v_bfe_i32 v0, v13, 0, 1
-; GFX8-NEXT:    v_mov_b32_e32 v4, s8
-; GFX8-NEXT:    v_ashrrev_i32_e32 v3, 31, v2
-; GFX8-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; GFX8-NEXT:    v_mov_b32_e32 v5, s9
+; GFX8-NEXT:    v_mov_b32_e32 v0, s8
+; GFX8-NEXT:    v_ashrrev_i32_e32 v7, 31, v6
+; GFX8-NEXT:    v_ashrrev_i32_e32 v5, 31, v4
+; GFX8-NEXT:    v_mov_b32_e32 v1, s9
 ; GFX8-NEXT:    s_add_u32 s8, s0, 0x120
-; GFX8-NEXT:    flat_store_dwordx4 v[4:5], v[0:3]
+; GFX8-NEXT:    flat_store_dwordx4 v[0:1], v[4:7]
+; GFX8-NEXT:    v_bfe_i32 v21, v19, 0, 1
+; GFX8-NEXT:    v_bfe_i32 v19, v20, 0, 1
 ; GFX8-NEXT:    s_addc_u32 s9, s1, 0
-; GFX8-NEXT:    v_bfe_i32 v2, v12, 0, 1
-; GFX8-NEXT:    v_bfe_i32 v0, v11, 0, 1
-; GFX8-NEXT:    v_mov_b32_e32 v4, s8
-; GFX8-NEXT:    v_ashrrev_i32_e32 v3, 31, v2
-; GFX8-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; GFX8-NEXT:    v_mov_b32_e32 v5, s9
+; GFX8-NEXT:    v_mov_b32_e32 v0, s8
+; GFX8-NEXT:    v_ashrrev_i32_e32 v22, 31, v21
+; GFX8-NEXT:    v_ashrrev_i32_e32 v20, 31, v19
+; GFX8-NEXT:    v_mov_b32_e32 v1, s9
 ; GFX8-NEXT:    s_add_u32 s8, s0, 0x110
-; GFX8-NEXT:    flat_store_dwordx4 v[4:5], v[0:3]
-; GFX8-NEXT:    s_addc_u32 s9, s1, 0
-; GFX8-NEXT:    v_bfe_i32 v2, v9, 0, 1
-; GFX8-NEXT:    v_bfe_i32 v0, v14, 0, 1
-; GFX8-NEXT:    v_mov_b32_e32 v4, s8
-; GFX8-NEXT:    v_ashrrev_i32_e32 v3, 31, v2
-; GFX8-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; GFX8-NEXT:    v_mov_b32_e32 v5, s9
-; GFX8-NEXT:    s_add_u32 s8, s0, 0x100
-; GFX8-NEXT:    flat_store_dwordx4 v[4:5], v[0:3]
-; GFX8-NEXT:    s_addc_u32 s9, s1, 0
-; GFX8-NEXT:    v_bfe_i32 v2, v10, 0, 1
-; GFX8-NEXT:    v_mov_b32_e32 v4, s8
-; GFX8-NEXT:    v_lshrrev_b16_e64 v7, 6, s10
-; GFX8-NEXT:    v_lshrrev_b16_e64 v8, 7, s10
-; GFX8-NEXT:    v_ashrrev_i32_e32 v3, 31, v2
-; GFX8-NEXT:    v_mov_b32_e32 v0, s12
-; GFX8-NEXT:    v_mov_b32_e32 v1, s13
-; GFX8-NEXT:    v_mov_b32_e32 v5, s9
-; GFX8-NEXT:    s_add_u32 s8, s0, 0x1f0
-; GFX8-NEXT:    flat_store_dwordx4 v[4:5], v[0:3]
+; GFX8-NEXT:    v_bfe_i32 v6, v25, 0, 1
+; GFX8-NEXT:    flat_store_dwordx4 v[0:1], v[19:22]
+; GFX8-NEXT:    v_bfe_i32 v25, v17, 0, 1
+; GFX8-NEXT:    v_bfe_i32 v23, v18, 0, 1
 ; GFX8-NEXT:    s_addc_u32 s9, s1, 0
-; GFX8-NEXT:    v_bfe_i32 v2, v8, 0, 1
-; GFX8-NEXT:    v_bfe_i32 v0, v7, 0, 1
-; GFX8-NEXT:    v_mov_b32_e32 v6, s8
-; GFX8-NEXT:    v_ashrrev_i32_e32 v3, 31, v2
-; GFX8-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; GFX8-NEXT:    v_mov_b32_e32 v7, s9
-; GFX8-NEXT:    v_lshrrev_b16_e64 v17, 4, s10
-; GFX8-NEXT:    v_lshrrev_b16_e64 v18, 5, s10
-; GFX8-NEXT:    v_lshrrev_b16_e64 v13, 6, s6
-; GFX8-NEXT:    v_lshrrev_b16_e64 v12, 7, s6
-; GFX8-NEXT:    v_lshrrev_b16_e64 v15, 4, s6
-; GFX8-NEXT:    v_lshrrev_b16_e64 v9, 5, s6
-; GFX8-NEXT:    v_lshrrev_b16_e64 v4, 2, s6
-; GFX8-NEXT:    v_lshrrev_b16_e64 v5, 3, s6
-; GFX8-NEXT:    flat_store_dwordx4 v[6:7], v[0:3]
-; GFX8-NEXT:    v_bfe_i32 v18, v18, 0, 1
-; GFX8-NEXT:    v_lshrrev_b16_e64 v0, 1, s6
-; GFX8-NEXT:    s_add_u32 s6, s0, 0x1e0
-; GFX8-NEXT:    v_bfe_i32 v2, v0, 0, 1
-; GFX8-NEXT:    v_bfe_i32 v16, v17, 0, 1
+; GFX8-NEXT:    v_mov_b32_e32 v0, s8
+; GFX8-NEXT:    v_bfe_i32 v4, v24, 0, 1
+; GFX8-NEXT:    v_bfe_i32 v19, v26, 0, 1
+; GFX8-NEXT:    v_ashrrev_i32_e32 v26, 31, v25
+; GFX8-NEXT:    v_ashrrev_i32_e32 v24, 31, v23
+; GFX8-NEXT:    v_mov_b32_e32 v1, s9
+; GFX8-NEXT:    flat_store_dwordx4 v[0:1], v[23:26]
+; GFX8-NEXT:    v_lshrrev_b16_e64 v14, 6, s12
+; GFX8-NEXT:    v_mov_b32_e32 v23, s6
+; GFX8-NEXT:    s_add_u32 s6, s0, 0x100
+; GFX8-NEXT:    v_bfe_i32 v25, v16, 0, 1
+; GFX8-NEXT:    v_mov_b32_e32 v24, s7
 ; GFX8-NEXT:    s_addc_u32 s7, s1, 0
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s6
-; GFX8-NEXT:    v_lshrrev_b16_e64 v20, 2, s10
-; GFX8-NEXT:    v_lshrrev_b16_e64 v21, 3, s10
-; GFX8-NEXT:    v_lshrrev_b16_e64 v22, 1, s10
-; GFX8-NEXT:    v_ashrrev_i32_e32 v19, 31, v18
+; GFX8-NEXT:    v_lshrrev_b16_e64 v15, 7, s12
+; GFX8-NEXT:    v_ashrrev_i32_e32 v26, 31, v25
+; GFX8-NEXT:    v_mov_b32_e32 v1, s7
+; GFX8-NEXT:    s_add_u32 s6, s0, 0x1f0
+; GFX8-NEXT:    flat_store_dwordx4 v[0:1], v[23:26]
+; GFX8-NEXT:    v_bfe_i32 v16, v15, 0, 1
+; GFX8-NEXT:    v_bfe_i32 v14, v14, 0, 1
+; GFX8-NEXT:    s_addc_u32 s7, s1, 0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s6
+; GFX8-NEXT:    v_lshrrev_b16_e64 v12, 4, s12
+; GFX8-NEXT:    v_lshrrev_b16_e64 v13, 5, s12
 ; GFX8-NEXT:    v_ashrrev_i32_e32 v17, 31, v16
+; GFX8-NEXT:    v_ashrrev_i32_e32 v15, 31, v14
+; GFX8-NEXT:    v_mov_b32_e32 v1, s7
+; GFX8-NEXT:    s_add_u32 s6, s0, 0x1e0
+; GFX8-NEXT:    v_bfe_i32 v21, v27, 0, 1
+; GFX8-NEXT:    flat_store_dwordx4 v[0:1], v[14:17]
+; GFX8-NEXT:    v_bfe_i32 v29, v13, 0, 1
+; GFX8-NEXT:    v_bfe_i32 v27, v12, 0, 1
+; GFX8-NEXT:    s_addc_u32 s7, s1, 0
+; GFX8-NEXT:    v_mov_b32_e32 v0, s6
+; GFX8-NEXT:    v_lshrrev_b16_e64 v10, 2, s12
+; GFX8-NEXT:    v_lshrrev_b16_e64 v11, 3, s12
+; GFX8-NEXT:    v_bfe_i32 v25, v28, 0, 1
+; GFX8-NEXT:    v_ashrrev_i32_e32 v30, 31, v29
+; GFX8-NEXT:    v_ashrrev_i32_e32 v28, 31, v27
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s7
 ; GFX8-NEXT:    s_add_u32 s6, s0, 0x1d0
-; GFX8-NEXT:    flat_store_dwordx4 v[0:1], v[16:19]
-; GFX8-NEXT:    v_bfe_i32 v20, v20, 0, 1
-; GFX8-NEXT:    v_bfe_i32 v18, v22, 0, 1
-; GFX8-NEXT:    v_bfe_i32 v22, v21, 0, 1
+; GFX8-NEXT:    v_bfe_i32 v23, v9, 0, 1
+; GFX8-NEXT:    flat_store_dwordx4 v[0:1], v[27:30]
+; GFX8-NEXT:    v_bfe_i32 v11, v11, 0, 1
+; GFX8-NEXT:    v_bfe_i32 v9, v10, 0, 1
 ; GFX8-NEXT:    s_addc_u32 s7, s1, 0
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s6
-; GFX8-NEXT:    v_ashrrev_i32_e32 v23, 31, v22
-; GFX8-NEXT:    v_ashrrev_i32_e32 v21, 31, v20
+; GFX8-NEXT:    v_ashrrev_i32_e32 v12, 31, v11
+; GFX8-NEXT:    v_ashrrev_i32_e32 v10, 31, v9
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s7
-; GFX8-NEXT:    v_mov_b32_e32 v16, s4
+; GFX8-NEXT:    v_lshrrev_b16_e64 v8, 1, s12
+; GFX8-NEXT:    flat_store_dwordx4 v[0:1], v[9:12]
+; GFX8-NEXT:    v_bfe_i32 v14, v8, 0, 1
+; GFX8-NEXT:    v_mov_b32_e32 v12, s4
 ; GFX8-NEXT:    s_add_u32 s4, s0, 0x1c0
-; GFX8-NEXT:    flat_store_dwordx4 v[0:1], v[20:23]
-; GFX8-NEXT:    v_mov_b32_e32 v17, s5
+; GFX8-NEXT:    v_mov_b32_e32 v13, s5
 ; GFX8-NEXT:    s_addc_u32 s5, s1, 0
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s4
-; GFX8-NEXT:    v_ashrrev_i32_e32 v19, 31, v18
+; GFX8-NEXT:    v_ashrrev_i32_e32 v15, 31, v14
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s5
 ; GFX8-NEXT:    s_add_u32 s4, s0, 0xf0
-; GFX8-NEXT:    v_bfe_i32 v14, v12, 0, 1
-; GFX8-NEXT:    v_bfe_i32 v12, v13, 0, 1
-; GFX8-NEXT:    flat_store_dwordx4 v[0:1], v[16:19]
+; GFX8-NEXT:    flat_store_dwordx4 v[0:1], v[12:15]
 ; GFX8-NEXT:    s_addc_u32 s5, s1, 0
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s4
-; GFX8-NEXT:    v_bfe_i32 v8, v15, 0, 1
-; GFX8-NEXT:    v_ashrrev_i32_e32 v15, 31, v14
-; GFX8-NEXT:    v_ashrrev_i32_e32 v13, 31, v12
+; GFX8-NEXT:    v_ashrrev_i32_e32 v26, 31, v25
+; GFX8-NEXT:    v_ashrrev_i32_e32 v24, 31, v23
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s5
 ; GFX8-NEXT:    s_add_u32 s4, s0, 0xe0
-; GFX8-NEXT:    flat_store_dwordx4 v[0:1], v[12:15]
+; GFX8-NEXT:    flat_store_dwordx4 v[0:1], v[23:26]
 ; GFX8-NEXT:    s_addc_u32 s5, s1, 0
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s4
-; GFX8-NEXT:    v_bfe_i32 v10, v9, 0, 1
 ; GFX8-NEXT:    v_mov_b32_e32 v1, s5
 ; GFX8-NEXT:    s_add_u32 s4, s0, 0xd0
-; GFX8-NEXT:    v_ashrrev_i32_e32 v11, 31, v10
-; GFX8-NEXT:    v_ashrrev_i32_e32 v9, 31, v8
+; GFX8-NEXT:    v_ashrrev_i32_e32 v22, 31, v21
+; GFX8-NEXT:    v_ashrrev_i32_e32 v20, 31, v19
 ; GFX8-NEXT:    s_addc_u32 s5, s1, 0
-; GFX8-NEXT:    v_bfe_i32 v6, v5, 0, 1
-; GFX8-NEXT:    v_bfe_i32 v4, v4, 0, 1
-; GFX8-NEXT:    flat_store_dwordx4 v[0:1], v[8:11]
+; GFX8-NEXT:    flat_store_dwordx4 v[0:1], v[19:22]
 ; GFX8-NEXT:    v_mov_b32_e32 v0, s4
 ; GFX8-NEXT:    s_add_u32 s0, s0, 0xc0
 ; GFX8-NEXT:    v_ashrrev_i32_e32 v7, 31, v6
diff --git a/llvm/test/CodeGen/AMDGPU/load-constant-i32.ll b/llvm/test/CodeGen/AMDGPU/load-constant-i32.ll
index 87913841012184c12eb1d7691f0bfb2b4b221bc9..ffc2cd23ec251fcd3847101ea57c0a9b6df4ae70 100644
--- a/llvm/test/CodeGen/AMDGPU/load-constant-i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-constant-i32.ll
@@ -1603,13 +1603,13 @@ define amdgpu_kernel void @constant_sextload_v2i32_to_v2i64(ptr addrspace(1) %ou
 ; GFX6-NOHSA-NEXT:    s_load_dwordx2 s[4:5], s[2:3], 0x0
 ; GFX6-NOHSA-NEXT:    s_mov_b32 s3, 0xf000
 ; GFX6-NOHSA-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX6-NOHSA-NEXT:    s_ashr_i32 s7, s5, 31
-; GFX6-NOHSA-NEXT:    s_ashr_i32 s8, s4, 31
+; GFX6-NOHSA-NEXT:    s_ashr_i32 s6, s5, 31
+; GFX6-NOHSA-NEXT:    s_ashr_i32 s7, s4, 31
 ; GFX6-NOHSA-NEXT:    s_mov_b32 s2, -1
 ; GFX6-NOHSA-NEXT:    v_mov_b32_e32 v0, s4
 ; GFX6-NOHSA-NEXT:    v_mov_b32_e32 v2, s5
-; GFX6-NOHSA-NEXT:    v_mov_b32_e32 v1, s8
-; GFX6-NOHSA-NEXT:    v_mov_b32_e32 v3, s7
+; GFX6-NOHSA-NEXT:    v_mov_b32_e32 v1, s7
+; GFX6-NOHSA-NEXT:    v_mov_b32_e32 v3, s6
 ; GFX6-NOHSA-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
 ; GFX6-NOHSA-NEXT:    s_endpgm
 ;
@@ -1621,13 +1621,13 @@ define amdgpu_kernel void @constant_sextload_v2i32_to_v2i64(ptr addrspace(1) %ou
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v4, s0
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v5, s1
 ; GFX7-HSA-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX7-HSA-NEXT:    s_ashr_i32 s1, s3, 31
-; GFX7-HSA-NEXT:    s_mov_b32 s0, s3
+; GFX7-HSA-NEXT:    s_ashr_i32 s0, s3, 31
+; GFX7-HSA-NEXT:    s_mov_b32 s1, s3
 ; GFX7-HSA-NEXT:    s_ashr_i32 s3, s2, 31
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX7-HSA-NEXT:    v_mov_b32_e32 v1, s3
-; GFX7-HSA-NEXT:    v_mov_b32_e32 v2, s0
-; GFX7-HSA-NEXT:    v_mov_b32_e32 v3, s1
+; GFX7-HSA-NEXT:    v_mov_b32_e32 v2, s1
+; GFX7-HSA-NEXT:    v_mov_b32_e32 v3, s0
 ; GFX7-HSA-NEXT:    flat_store_dwordx4 v[4:5], v[0:3]
 ; GFX7-HSA-NEXT:    s_endpgm
 ;
@@ -1639,13 +1639,13 @@ define amdgpu_kernel void @constant_sextload_v2i32_to_v2i64(ptr addrspace(1) %ou
 ; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v4, s0
 ; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v5, s1
 ; GFX8-NOHSA-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX8-NOHSA-NEXT:    s_ashr_i32 s1, s3, 31
-; GFX8-NOHSA-NEXT:    s_mov_b32 s0, s3
+; GFX8-NOHSA-NEXT:    s_ashr_i32 s0, s3, 31
+; GFX8-NOHSA-NEXT:    s_mov_b32 s1, s3
 ; GFX8-NOHSA-NEXT:    s_ashr_i32 s3, s2, 31
 ; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v1, s3
-; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v2, s0
-; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v3, s1
+; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v2, s1
+; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v3, s0
 ; GFX8-NOHSA-NEXT:    flat_store_dwordx4 v[4:5], v[0:3]
 ; GFX8-NOHSA-NEXT:    s_endpgm
 ;
@@ -1678,13 +1678,12 @@ define amdgpu_kernel void @constant_sextload_v2i32_to_v2i64(ptr addrspace(1) %ou
 ; GFX9-HSA-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX9-HSA-NEXT:    s_load_dwordx2 s[2:3], s[2:3], 0x0
 ; GFX9-HSA-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX9-HSA-NEXT:    s_ashr_i32 s5, s3, 31
-; GFX9-HSA-NEXT:    s_mov_b32 s4, s3
-; GFX9-HSA-NEXT:    s_ashr_i32 s3, s2, 31
+; GFX9-HSA-NEXT:    s_ashr_i32 s4, s3, 31
+; GFX9-HSA-NEXT:    s_ashr_i32 s5, s2, 31
 ; GFX9-HSA-NEXT:    v_mov_b32_e32 v0, s2
-; GFX9-HSA-NEXT:    v_mov_b32_e32 v1, s3
-; GFX9-HSA-NEXT:    v_mov_b32_e32 v2, s4
-; GFX9-HSA-NEXT:    v_mov_b32_e32 v3, s5
+; GFX9-HSA-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-HSA-NEXT:    v_mov_b32_e32 v2, s3
+; GFX9-HSA-NEXT:    v_mov_b32_e32 v3, s4
 ; GFX9-HSA-NEXT:    global_store_dwordx4 v4, v[0:3], s[0:1]
 ; GFX9-HSA-NEXT:    s_endpgm
   %ld = load <2 x i32>, ptr addrspace(4) %in
diff --git a/llvm/test/CodeGen/AMDGPU/load-constant-i8.ll b/llvm/test/CodeGen/AMDGPU/load-constant-i8.ll
index 66fc322e5e04b57ac536515e36e872b807dfd5e0..9ebd2018798253a10c823099f9ac16217ab80a35 100644
--- a/llvm/test/CodeGen/AMDGPU/load-constant-i8.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-constant-i8.ll
@@ -7203,10 +7203,10 @@ define amdgpu_kernel void @constant_sextload_v32i8_to_v32i64(ptr addrspace(1) %o
 ; GFX8-NOHSA-NEXT:    s_mov_b32 s50, s5
 ; GFX8-NOHSA-NEXT:    s_mov_b32 s52, s3
 ; GFX8-NOHSA-NEXT:    s_mov_b32 s54, s1
-; GFX8-NOHSA-NEXT:    v_lshrrev_b16_e64 v5, 8, s7
+; GFX8-NOHSA-NEXT:    v_lshrrev_b16_e64 v4, 8, s7
 ; GFX8-NOHSA-NEXT:    v_lshrrev_b16_e64 v1, 8, s6
 ; GFX8-NOHSA-NEXT:    v_lshrrev_b16_e64 v0, 8, s5
-; GFX8-NOHSA-NEXT:    v_lshrrev_b16_e64 v4, 8, s4
+; GFX8-NOHSA-NEXT:    v_lshrrev_b16_e64 v5, 8, s4
 ; GFX8-NOHSA-NEXT:    v_lshrrev_b16_e64 v8, 8, s3
 ; GFX8-NOHSA-NEXT:    v_lshrrev_b16_e64 v9, 8, s2
 ; GFX8-NOHSA-NEXT:    v_lshrrev_b16_e64 v6, 8, s1
@@ -7235,93 +7235,93 @@ define amdgpu_kernel void @constant_sextload_v32i8_to_v32i64(ptr addrspace(1) %o
 ; GFX8-NOHSA-NEXT:    s_bfe_i64 s[34:35], s[34:35], 0x80000
 ; GFX8-NOHSA-NEXT:    s_bfe_i64 s[30:31], s[30:31], 0x80000
 ; GFX8-NOHSA-NEXT:    s_bfe_i64 s[28:29], s[28:29], 0x80000
-; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v12, s28
-; GFX8-NOHSA-NEXT:    s_add_u32 s28, s8, 0xf0
-; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v13, s29
-; GFX8-NOHSA-NEXT:    s_addc_u32 s29, s9, 0
 ; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v16, s28
-; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v14, s58
-; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v15, s59
+; GFX8-NOHSA-NEXT:    s_add_u32 s28, s8, 0xf0
+; GFX8-NOHSA-NEXT:    v_bfe_i32 v10, v9, 0, 8
+; GFX8-NOHSA-NEXT:    v_bfe_i32 v14, v8, 0, 8
 ; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v17, s29
+; GFX8-NOHSA-NEXT:    s_addc_u32 s29, s9, 0
+; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v8, s28
+; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v18, s58
+; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v19, s59
+; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v9, s29
 ; GFX8-NOHSA-NEXT:    s_add_u32 s28, s8, 0xd0
-; GFX8-NOHSA-NEXT:    flat_store_dwordx4 v[16:17], v[12:15]
+; GFX8-NOHSA-NEXT:    flat_store_dwordx4 v[8:9], v[16:19]
 ; GFX8-NOHSA-NEXT:    s_addc_u32 s29, s9, 0
-; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v16, s28
-; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v12, s30
-; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v13, s31
-; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v14, s34
-; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v15, s35
-; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v17, s29
+; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v8, s28
+; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v16, s30
+; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v17, s31
+; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v18, s34
+; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v19, s35
+; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v9, s29
 ; GFX8-NOHSA-NEXT:    s_add_u32 s28, s8, 0xb0
-; GFX8-NOHSA-NEXT:    flat_store_dwordx4 v[16:17], v[12:15]
+; GFX8-NOHSA-NEXT:    flat_store_dwordx4 v[8:9], v[16:19]
 ; GFX8-NOHSA-NEXT:    s_addc_u32 s29, s9, 0
-; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v16, s28
-; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v12, s36
-; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v13, s37
-; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v14, s56
-; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v15, s57
-; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v17, s29
+; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v8, s28
+; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v16, s36
+; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v17, s37
+; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v18, s56
+; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v19, s57
+; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v9, s29
 ; GFX8-NOHSA-NEXT:    s_add_u32 s28, s8, 0x90
-; GFX8-NOHSA-NEXT:    flat_store_dwordx4 v[16:17], v[12:15]
+; GFX8-NOHSA-NEXT:    flat_store_dwordx4 v[8:9], v[16:19]
 ; GFX8-NOHSA-NEXT:    s_addc_u32 s29, s9, 0
-; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v16, s28
-; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v12, s38
-; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v13, s39
-; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v14, s40
-; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v15, s41
-; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v17, s29
-; GFX8-NOHSA-NEXT:    flat_store_dwordx4 v[16:17], v[12:15]
-; GFX8-NOHSA-NEXT:    v_bfe_i32 v10, v9, 0, 8
-; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v14, s24
+; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v8, s28
+; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v16, s38
+; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v17, s39
+; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v18, s40
+; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v19, s41
+; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v9, s29
+; GFX8-NOHSA-NEXT:    flat_store_dwordx4 v[8:9], v[16:19]
+; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v20, s6
+; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v18, s24
 ; GFX8-NOHSA-NEXT:    s_add_u32 s24, s8, 0x70
-; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v15, s25
+; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v19, s25
 ; GFX8-NOHSA-NEXT:    s_addc_u32 s25, s9, 0
-; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v16, s24
-; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v12, s42
-; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v13, s43
-; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v17, s25
+; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v8, s24
+; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v16, s42
+; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v17, s43
+; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v9, s25
 ; GFX8-NOHSA-NEXT:    s_add_u32 s24, s8, 0x50
-; GFX8-NOHSA-NEXT:    flat_store_dwordx4 v[16:17], v[12:15]
+; GFX8-NOHSA-NEXT:    flat_store_dwordx4 v[8:9], v[16:19]
 ; GFX8-NOHSA-NEXT:    s_addc_u32 s25, s9, 0
-; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v16, s24
-; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v12, s44
-; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v13, s45
-; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v14, s46
-; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v15, s47
-; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v17, s25
-; GFX8-NOHSA-NEXT:    flat_store_dwordx4 v[16:17], v[12:15]
-; GFX8-NOHSA-NEXT:    v_bfe_i32 v18, v5, 0, 8
-; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v14, s18
+; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v8, s24
+; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v16, s44
+; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v17, s45
+; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v18, s46
+; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v19, s47
+; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v9, s25
+; GFX8-NOHSA-NEXT:    flat_store_dwordx4 v[8:9], v[16:19]
+; GFX8-NOHSA-NEXT:    v_bfe_i32 v22, v4, 0, 8
+; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v18, s18
 ; GFX8-NOHSA-NEXT:    s_add_u32 s18, s8, 48
-; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v15, s19
+; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v19, s19
 ; GFX8-NOHSA-NEXT:    s_addc_u32 s19, s9, 0
-; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v16, s18
-; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v12, s26
-; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v13, s27
-; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v17, s19
+; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v8, s18
+; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v16, s26
+; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v17, s27
+; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v9, s19
 ; GFX8-NOHSA-NEXT:    s_add_u32 s18, s8, 16
-; GFX8-NOHSA-NEXT:    flat_store_dwordx4 v[16:17], v[12:15]
+; GFX8-NOHSA-NEXT:    flat_store_dwordx4 v[8:9], v[16:19]
 ; GFX8-NOHSA-NEXT:    s_addc_u32 s19, s9, 0
-; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v16, s18
-; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v12, s22
-; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v13, s23
-; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v14, s20
-; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v15, s21
-; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v17, s19
-; GFX8-NOHSA-NEXT:    flat_store_dwordx4 v[16:17], v[12:15]
-; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v16, s6
+; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v8, s18
+; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v16, s22
+; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v17, s23
+; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v18, s20
+; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v19, s21
+; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v9, s19
 ; GFX8-NOHSA-NEXT:    s_add_u32 s6, s8, 0xe0
-; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v17, s7
+; GFX8-NOHSA-NEXT:    flat_store_dwordx4 v[8:9], v[16:19]
+; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v21, s7
+; GFX8-NOHSA-NEXT:    v_bfe_i32 v18, v5, 0, 8
 ; GFX8-NOHSA-NEXT:    s_addc_u32 s7, s9, 0
-; GFX8-NOHSA-NEXT:    v_bfe_i32 v14, v8, 0, 8
-; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v9, s7
-; GFX8-NOHSA-NEXT:    v_ashrrev_i32_e32 v19, 31, v18
-; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v8, s6
+; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v4, s6
+; GFX8-NOHSA-NEXT:    v_ashrrev_i32_e32 v23, 31, v22
+; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v5, s7
 ; GFX8-NOHSA-NEXT:    s_add_u32 s6, s8, 0xc0
-; GFX8-NOHSA-NEXT:    flat_store_dwordx4 v[8:9], v[16:19]
-; GFX8-NOHSA-NEXT:    v_bfe_i32 v22, v1, 0, 8
-; GFX8-NOHSA-NEXT:    v_bfe_i32 v18, v4, 0, 8
+; GFX8-NOHSA-NEXT:    flat_store_dwordx4 v[4:5], v[20:23]
 ; GFX8-NOHSA-NEXT:    s_addc_u32 s7, s9, 0
+; GFX8-NOHSA-NEXT:    v_bfe_i32 v22, v1, 0, 8
 ; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v4, s6
 ; GFX8-NOHSA-NEXT:    v_ashrrev_i32_e32 v23, 31, v22
 ; GFX8-NOHSA-NEXT:    v_mov_b32_e32 v20, s16
diff --git a/llvm/test/CodeGen/AMDGPU/load-global-i16.ll b/llvm/test/CodeGen/AMDGPU/load-global-i16.ll
index 7c80a220b72d7b0dfedb1dcd9729cfccf21d8669..6352a98f10a117a459307526d699c3755413292d 100644
--- a/llvm/test/CodeGen/AMDGPU/load-global-i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-global-i16.ll
@@ -6299,13 +6299,13 @@ define amdgpu_kernel void @global_sextload_v8i16_to_v8i64(ptr addrspace(1) %out,
 ; GCN-HSA-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v7, v3
 ; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v10, 16, v2
-; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v5, 16, v0
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v6, 16, v0
 ; GCN-HSA-NEXT:    v_ashr_i64 v[14:15], v[0:1], 48
 ; GCN-HSA-NEXT:    v_bfe_i32 v12, v1, 0, 16
 ; GCN-HSA-NEXT:    v_bfe_i32 v4, v0, 0, 16
 ; GCN-HSA-NEXT:    v_bfe_i32 v8, v2, 0, 16
-; GCN-HSA-NEXT:    v_bfe_i32 v6, v5, 0, 16
 ; GCN-HSA-NEXT:    v_ashr_i64 v[2:3], v[2:3], 48
+; GCN-HSA-NEXT:    v_bfe_i32 v6, v6, 0, 16
 ; GCN-HSA-NEXT:    v_bfe_i32 v10, v10, 0, 16
 ; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v13, 31, v12
 ; GCN-HSA-NEXT:    v_bfe_i32 v0, v7, 0, 16
@@ -6864,16 +6864,16 @@ define amdgpu_kernel void @global_sextload_v16i16_to_v16i64(ptr addrspace(1) %ou
 ; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v10, v3
 ; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v11, 16, v4
 ; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v13, 16, v2
-; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v17, 16, v0
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v14, 16, v0
 ; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v8, v0, 0, 16
-; GCN-NOHSA-SI-NEXT:    v_ashr_i64 v[14:15], v[0:1], 48
-; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v16, v10, 0, 16
-; GCN-NOHSA-SI-NEXT:    v_ashr_i64 v[18:19], v[2:3], 48
-; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v12, v1, 0, 16
-; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v0, v2, 0, 16
-; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v10, v17, 0, 16
-; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v2, v13, 0, 16
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v12, v2, 0, 16
+; GCN-NOHSA-SI-NEXT:    v_ashr_i64 v[18:19], v[0:1], 48
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v16, v1, 0, 16
+; GCN-NOHSA-SI-NEXT:    v_ashr_i64 v[2:3], v[2:3], 48
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v0, v10, 0, 16
 ; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v20, v4, 0, 16
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v10, v14, 0, 16
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v14, v13, 0, 16
 ; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v22, v11, 0, 16
 ; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v23, v9, 0, 16
 ; GCN-NOHSA-SI-NEXT:    v_ashr_i64 v[25:26], v[6:7], 48
@@ -6886,22 +6886,22 @@ define amdgpu_kernel void @global_sextload_v16i16_to_v16i64(ptr addrspace(1) %ou
 ; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v4, v6, 0, 16
 ; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v6, v1, 0, 16
 ; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v9, 31, v8
-; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v13, 31, v12
 ; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v21, 31, v20
 ; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v5, 31, v4
-; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v13, 31, v12
 ; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v17, 31, v16
+; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v25, 31, v24
 ; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v11, 31, v10
-; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v3, 31, v2
+; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v15, 31, v14
 ; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v23, 31, v22
 ; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v7, 31, v6
 ; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[24:27], off, s[0:3], 0 offset:80
-; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[16:19], off, s[0:3], 0 offset:48
-; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[12:15], off, s[0:3], 0 offset:16
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:48
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[16:19], off, s[0:3], 0 offset:16
 ; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:96
 ; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[20:23], off, s[0:3], 0 offset:64
-; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:32
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[12:15], off, s[0:3], 0 offset:32
 ; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[8:11], off, s[0:3], 0
 ; GCN-NOHSA-SI-NEXT:    s_endpgm
 ;
@@ -6957,13 +6957,13 @@ define amdgpu_kernel void @global_sextload_v16i16_to_v16i64(ptr addrspace(1) %ou
 ; GCN-HSA-NEXT:    v_bfe_i32 v8, v5, 0, 16
 ; GCN-HSA-NEXT:    v_ashr_i64 v[10:11], v[6:7], 48
 ; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v9, 31, v8
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v17, 16, v4
 ; GCN-HSA-NEXT:    flat_store_dwordx4 v[14:15], v[8:11]
 ; GCN-HSA-NEXT:    v_bfe_i32 v7, v6, 0, 16
-; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v9, 16, v4
-; GCN-HSA-NEXT:    v_bfe_i32 v6, v9, 0, 16
 ; GCN-HSA-NEXT:    v_bfe_i32 v9, v16, 0, 16
 ; GCN-HSA-NEXT:    v_bfe_i32 v4, v4, 0, 16
 ; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v8, 31, v7
+; GCN-HSA-NEXT:    v_bfe_i32 v6, v17, 0, 16
 ; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v10, 31, v9
 ; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v5, 31, v4
 ; GCN-HSA-NEXT:    flat_store_dwordx4 v[22:23], v[7:10]
@@ -6971,14 +6971,14 @@ define amdgpu_kernel void @global_sextload_v16i16_to_v16i64(ptr addrspace(1) %ou
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v11, v3
 ; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v7, 31, v6
 ; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v16, 16, v2
-; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v9, 16, v0
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v17, 16, v0
 ; GCN-HSA-NEXT:    flat_store_dwordx4 v[12:13], v[4:7]
 ; GCN-HSA-NEXT:    v_ashr_i64 v[14:15], v[0:1], 48
 ; GCN-HSA-NEXT:    v_bfe_i32 v12, v1, 0, 16
 ; GCN-HSA-NEXT:    v_bfe_i32 v8, v0, 0, 16
 ; GCN-HSA-NEXT:    v_bfe_i32 v4, v2, 0, 16
-; GCN-HSA-NEXT:    v_bfe_i32 v10, v9, 0, 16
 ; GCN-HSA-NEXT:    v_ashr_i64 v[2:3], v[2:3], 48
+; GCN-HSA-NEXT:    v_bfe_i32 v10, v17, 0, 16
 ; GCN-HSA-NEXT:    v_bfe_i32 v6, v16, 0, 16
 ; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v13, 31, v12
 ; GCN-HSA-NEXT:    v_bfe_i32 v0, v11, 0, 16
@@ -7003,62 +7003,62 @@ define amdgpu_kernel void @global_sextload_v16i16_to_v16i64(ptr addrspace(1) %ou
 ; GCN-NOHSA-VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NOHSA-VI-NEXT:    s_mov_b32 s8, s6
 ; GCN-NOHSA-VI-NEXT:    s_mov_b32 s9, s7
-; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[1:4], off, s[8:11], 0
-; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[5:8], off, s[8:11], 0 offset:16
+; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[0:3], off, s[8:11], 0
+; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[4:7], off, s[8:11], 0 offset:16
 ; GCN-NOHSA-VI-NEXT:    s_mov_b32 s0, s4
 ; GCN-NOHSA-VI-NEXT:    s_mov_b32 s1, s5
 ; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(1)
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v9, v2, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v10, 16, v2
-; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(0)
-; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v2, 16, v6
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v8, v0, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v9, 16, v0
 ; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v0, v1, 0, 16
 ; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v1, 16, v1
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v16, v2, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v14, v6, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v15, 31, v14
-; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v17, 31, v16
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v12, v2, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v11, 16, v2
 ; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v2, v1, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v1, v8
-; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[14:17], off, s[0:3], 0 offset:80
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v13, v3, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v17, v1, 0, 16
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(0)
 ; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v1, 16, v5
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v17, v1, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v1, 16, v4
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v13, v3
 ; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
-; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v6, v4
-; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v12, 16, v4
-; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v14, 16, v7
-; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v8, 16, v8
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v25, v5, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v27, v1, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v11, v10, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v15, v3, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v4, v6, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v6, v12, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v19, v8, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v23, v14, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v21, v7, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v26, 31, v25
-; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v28, 31, v27
-; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
-; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v10, 31, v9
-; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v14, 31, v13
-; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v22, 31, v21
-; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v3, 31, v2
-; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v12, 31, v11
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v20, 16, v6
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v22, v7
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v7, 16, v7
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v15, v5, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v27, v6, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v4, v4, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v6, v1, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v10, v9, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v14, v11, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v19, v13, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v21, v3, 0, 16
 ; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v16, 31, v15
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v18, 31, v17
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v23, v22, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v25, v7, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v29, v20, 0, 16
 ; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v5, 31, v4
 ; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v7, 31, v6
-; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v24, 31, v23
-; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v18, 31, v17
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[15:18], off, s[0:3], 0 offset:80
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v9, 31, v8
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v13, 31, v12
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v28, 31, v27
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v11, 31, v10
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v3, 31, v2
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v15, 31, v14
 ; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v20, 31, v19
-; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[25:28], off, s[0:3], 0 offset:64
-; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[17:20], off, s[0:3], 0 offset:112
-; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[21:24], off, s[0:3], 0 offset:96
-; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:48
-; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[13:16], off, s[0:3], 0 offset:32
-; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[9:12], off, s[0:3], 0 offset:16
-; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v22, 31, v21
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v30, 31, v29
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v24, 31, v23
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v26, 31, v25
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:64
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[23:26], off, s[0:3], 0 offset:112
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[27:30], off, s[0:3], 0 offset:96
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[19:22], off, s[0:3], 0 offset:48
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[12:15], off, s[0:3], 0 offset:32
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:16
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[8:11], off, s[0:3], 0
 ; GCN-NOHSA-VI-NEXT:    s_endpgm
 ;
 ; EG-LABEL: global_sextload_v16i16_to_v16i64:
@@ -7980,71 +7980,70 @@ define amdgpu_kernel void @global_sextload_v32i16_to_v32i64(ptr addrspace(1) %ou
 ; GCN-NOHSA-SI-NEXT:    s_mov_b32 s0, s4
 ; GCN-NOHSA-SI-NEXT:    s_mov_b32 s1, s5
 ; GCN-NOHSA-SI-NEXT:    s_waitcnt vmcnt(0)
-; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v18, v3
-; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v22, v7
-; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v23, v11
-; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v17, v15
-; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v16, 16, v2
-; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v18, v18, 0, 16
-; GCN-NOHSA-SI-NEXT:    v_ashr_i64 v[20:21], v[2:3], 48
-; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v19, 31, v18
-; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[18:21], off, s[0:3], 0 offset:240
-; GCN-NOHSA-SI-NEXT:    s_waitcnt expcnt(0)
-; GCN-NOHSA-SI-NEXT:    v_ashr_i64 v[20:21], v[0:1], 48
-; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v18, v1, 0, 16
-; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v19, 31, v18
-; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[18:21], off, s[0:3], 0 offset:208
-; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v24, 16, v4
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v22, v3
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v26, v7
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v27, v11
+; GCN-NOHSA-SI-NEXT:    v_mov_b32_e32 v21, v15
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v20, 16, v2
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v16, 16, v4
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v17, 16, v10
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v18, 16, v8
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v19, 16, v14
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v22, v22, 0, 16
+; GCN-NOHSA-SI-NEXT:    v_ashr_i64 v[24:25], v[2:3], 48
+; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v23, 31, v22
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[22:25], off, s[0:3], 0 offset:240
 ; GCN-NOHSA-SI-NEXT:    s_waitcnt expcnt(0)
-; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v18, v22, 0, 16
-; GCN-NOHSA-SI-NEXT:    v_ashr_i64 v[20:21], v[6:7], 48
-; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v19, 31, v18
-; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[18:21], off, s[0:3], 0 offset:176
+; GCN-NOHSA-SI-NEXT:    v_ashr_i64 v[24:25], v[0:1], 48
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v22, v1, 0, 16
+; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v23, 31, v22
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[22:25], off, s[0:3], 0 offset:208
+; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v3, 16, v12
 ; GCN-NOHSA-SI-NEXT:    s_waitcnt expcnt(0)
-; GCN-NOHSA-SI-NEXT:    v_ashr_i64 v[20:21], v[4:5], 48
-; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v18, v5, 0, 16
-; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v19, 31, v18
-; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[18:21], off, s[0:3], 0 offset:144
-; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v22, 16, v10
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v22, v26, 0, 16
+; GCN-NOHSA-SI-NEXT:    v_ashr_i64 v[24:25], v[6:7], 48
+; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v23, 31, v22
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[22:25], off, s[0:3], 0 offset:176
 ; GCN-NOHSA-SI-NEXT:    s_waitcnt expcnt(0)
-; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v18, v23, 0, 16
-; GCN-NOHSA-SI-NEXT:    v_ashr_i64 v[20:21], v[10:11], 48
-; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v19, 31, v18
-; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[18:21], off, s[0:3], 0 offset:112
+; GCN-NOHSA-SI-NEXT:    v_ashr_i64 v[24:25], v[4:5], 48
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v22, v5, 0, 16
+; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v23, 31, v22
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[22:25], off, s[0:3], 0 offset:144
 ; GCN-NOHSA-SI-NEXT:    s_waitcnt expcnt(0)
-; GCN-NOHSA-SI-NEXT:    v_ashr_i64 v[20:21], v[8:9], 48
-; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v18, v9, 0, 16
-; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v19, 31, v18
-; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[18:21], off, s[0:3], 0 offset:80
-; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v9, 16, v8
-; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v17, v17, 0, 16
+; GCN-NOHSA-SI-NEXT:    v_ashr_i64 v[24:25], v[10:11], 48
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v22, v27, 0, 16
+; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v23, 31, v22
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[22:25], off, s[0:3], 0 offset:112
 ; GCN-NOHSA-SI-NEXT:    s_waitcnt expcnt(0)
-; GCN-NOHSA-SI-NEXT:    v_ashr_i64 v[19:20], v[14:15], 48
-; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v18, 31, v17
-; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[17:20], off, s[0:3], 0 offset:48
+; GCN-NOHSA-SI-NEXT:    v_ashr_i64 v[24:25], v[8:9], 48
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v22, v9, 0, 16
+; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v23, 31, v22
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[22:25], off, s[0:3], 0 offset:80
 ; GCN-NOHSA-SI-NEXT:    s_waitcnt expcnt(0)
-; GCN-NOHSA-SI-NEXT:    v_ashr_i64 v[19:20], v[12:13], 48
-; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v17, v13, 0, 16
-; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v18, 31, v17
-; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[17:20], off, s[0:3], 0 offset:16
-; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v7, 16, v14
+; GCN-NOHSA-SI-NEXT:    v_ashr_i64 v[23:24], v[14:15], 48
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v21, v21, 0, 16
+; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v22, 31, v21
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[21:24], off, s[0:3], 0 offset:48
 ; GCN-NOHSA-SI-NEXT:    s_waitcnt expcnt(0)
-; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v17, v16, 0, 16
-; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v15, v2, 0, 16
-; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v16, 31, v15
-; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v18, 31, v17
-; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[15:18], off, s[0:3], 0 offset:224
-; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v1, 16, v12
-; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v3, v1, 0, 16
+; GCN-NOHSA-SI-NEXT:    v_ashr_i64 v[23:24], v[12:13], 48
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v21, v13, 0, 16
+; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v22, 31, v21
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[21:24], off, s[0:3], 0 offset:16
 ; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v1, v12, 0, 16
 ; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v5, v14, 0, 16
-; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v7, v7, 0, 16
-; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v11, v9, 0, 16
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v3, v3, 0, 16
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v13, v20, 0, 16
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v11, v2, 0, 16
+; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v12, 31, v11
+; GCN-NOHSA-SI-NEXT:    v_ashrrev_i32_e32 v14, 31, v13
+; GCN-NOHSA-SI-NEXT:    buffer_store_dwordx4 v[11:14], off, s[0:3], 0 offset:224
 ; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v9, v8, 0, 16
-; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v13, v10, 0, 16
 ; GCN-NOHSA-SI-NEXT:    s_waitcnt expcnt(0)
-; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v15, v22, 0, 16
-; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v19, v24, 0, 16
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v13, v10, 0, 16
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v7, v19, 0, 16
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v11, v18, 0, 16
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v15, v17, 0, 16
+; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v19, v16, 0, 16
 ; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v17, v4, 0, 16
 ; GCN-NOHSA-SI-NEXT:    v_lshrrev_b32_e32 v2, 16, v6
 ; GCN-NOHSA-SI-NEXT:    v_bfe_i32 v21, v6, 0, 16
@@ -8099,154 +8098,154 @@ define amdgpu_kernel void @global_sextload_v32i16_to_v32i64(ptr addrspace(1) %ou
 ; GCN-HSA-NEXT:    flat_load_dwordx4 v[12:15], v[12:13]
 ; GCN-HSA-NEXT:    s_add_u32 s2, s0, 48
 ; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v21, s3
+; GCN-HSA-NEXT:    v_mov_b32_e32 v20, s2
+; GCN-HSA-NEXT:    s_add_u32 s2, s0, 16
+; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v23, s3
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v22, s2
-; GCN-HSA-NEXT:    s_add_u32 s2, s0, 16
+; GCN-HSA-NEXT:    s_add_u32 s2, s0, 0xf0
 ; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v25, s3
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v24, s2
-; GCN-HSA-NEXT:    s_add_u32 s2, s0, 0xf0
+; GCN-HSA-NEXT:    s_add_u32 s2, s0, 0xd0
 ; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v27, s3
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v26, s2
-; GCN-HSA-NEXT:    s_add_u32 s2, s0, 0xd0
+; GCN-HSA-NEXT:    s_add_u32 s2, s0, 0xb0
 ; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
-; GCN-HSA-NEXT:    v_mov_b32_e32 v21, s1
-; GCN-HSA-NEXT:    v_mov_b32_e32 v20, s0
 ; GCN-HSA-NEXT:    s_waitcnt vmcnt(3)
 ; GCN-HSA-NEXT:    v_ashr_i64 v[18:19], v[8:9], 48
 ; GCN-HSA-NEXT:    v_bfe_i32 v16, v9, 0, 16
 ; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v17, 31, v16
-; GCN-HSA-NEXT:    flat_store_dwordx4 v[24:25], v[16:19]
-; GCN-HSA-NEXT:    v_mov_b32_e32 v25, s3
-; GCN-HSA-NEXT:    v_mov_b32_e32 v24, s2
-; GCN-HSA-NEXT:    s_add_u32 s2, s0, 0xb0
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[22:23], v[16:19]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v23, s3
+; GCN-HSA-NEXT:    v_mov_b32_e32 v22, s2
+; GCN-HSA-NEXT:    s_add_u32 s2, s0, 0x90
 ; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
-; GCN-HSA-NEXT:    s_add_u32 s4, s0, 0x90
-; GCN-HSA-NEXT:    v_mov_b32_e32 v9, v11
+; GCN-HSA-NEXT:    s_add_u32 s4, s0, 0x70
 ; GCN-HSA-NEXT:    s_addc_u32 s5, s1, 0
+; GCN-HSA-NEXT:    s_add_u32 s6, s0, 0x50
+; GCN-HSA-NEXT:    s_addc_u32 s7, s1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v9, v11
+; GCN-HSA-NEXT:    s_add_u32 s8, s0, 32
 ; GCN-HSA-NEXT:    v_bfe_i32 v16, v9, 0, 16
 ; GCN-HSA-NEXT:    v_ashr_i64 v[18:19], v[10:11], 48
-; GCN-HSA-NEXT:    s_add_u32 s6, s0, 0x70
+; GCN-HSA-NEXT:    s_addc_u32 s9, s1, 0
 ; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v17, 31, v16
 ; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v9, 16, v10
-; GCN-HSA-NEXT:    s_addc_u32 s7, s1, 0
-; GCN-HSA-NEXT:    flat_store_dwordx4 v[22:23], v[16:19]
-; GCN-HSA-NEXT:    s_add_u32 s8, s0, 0x50
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[20:21], v[16:19]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v21, s9
 ; GCN-HSA-NEXT:    v_bfe_i32 v18, v9, 0, 16
-; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v9, 16, v8
 ; GCN-HSA-NEXT:    v_bfe_i32 v16, v10, 0, 16
-; GCN-HSA-NEXT:    v_bfe_i32 v8, v8, 0, 16
-; GCN-HSA-NEXT:    v_bfe_i32 v10, v9, 0, 16
-; GCN-HSA-NEXT:    s_addc_u32 s9, s1, 0
-; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v9, 31, v8
-; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v11, 31, v10
-; GCN-HSA-NEXT:    s_add_u32 s10, s0, 32
-; GCN-HSA-NEXT:    flat_store_dwordx4 v[20:21], v[8:11]
-; GCN-HSA-NEXT:    s_addc_u32 s11, s1, 0
+; GCN-HSA-NEXT:    v_mov_b32_e32 v20, s8
+; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v17, 31, v16
+; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v19, 31, v18
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[20:21], v[16:19]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v10, s3
 ; GCN-HSA-NEXT:    s_waitcnt vmcnt(5)
-; GCN-HSA-NEXT:    v_ashr_i64 v[10:11], v[0:1], 48
-; GCN-HSA-NEXT:    v_bfe_i32 v8, v1, 0, 16
-; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v9, 31, v8
+; GCN-HSA-NEXT:    v_ashr_i64 v[18:19], v[0:1], 48
+; GCN-HSA-NEXT:    v_bfe_i32 v16, v1, 0, 16
+; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v17, 31, v16
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v1, v3
-; GCN-HSA-NEXT:    v_mov_b32_e32 v23, s11
-; GCN-HSA-NEXT:    flat_store_dwordx4 v[24:25], v[8:11]
-; GCN-HSA-NEXT:    v_mov_b32_e32 v22, s10
-; GCN-HSA-NEXT:    v_bfe_i32 v8, v1, 0, 16
-; GCN-HSA-NEXT:    v_ashr_i64 v[10:11], v[2:3], 48
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[26:27], v[16:19]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v9, s2
+; GCN-HSA-NEXT:    v_bfe_i32 v16, v1, 0, 16
+; GCN-HSA-NEXT:    v_ashr_i64 v[18:19], v[2:3], 48
+; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v17, 31, v16
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[24:25], v[16:19]
+; GCN-HSA-NEXT:    s_waitcnt vmcnt(6)
+; GCN-HSA-NEXT:    v_mov_b32_e32 v3, v7
+; GCN-HSA-NEXT:    v_ashr_i64 v[18:19], v[4:5], 48
+; GCN-HSA-NEXT:    v_bfe_i32 v16, v5, 0, 16
+; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v17, 31, v16
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[9:10], v[16:19]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v25, s7
+; GCN-HSA-NEXT:    v_bfe_i32 v16, v3, 0, 16
+; GCN-HSA-NEXT:    v_ashr_i64 v[18:19], v[6:7], 48
 ; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v17, 31, v16
-; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v19, 31, v18
-; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v9, 31, v8
 ; GCN-HSA-NEXT:    flat_store_dwordx4 v[22:23], v[16:19]
-; GCN-HSA-NEXT:    flat_store_dwordx4 v[26:27], v[8:11]
-; GCN-HSA-NEXT:    v_mov_b32_e32 v19, s5
+; GCN-HSA-NEXT:    v_mov_b32_e32 v24, s6
 ; GCN-HSA-NEXT:    s_waitcnt vmcnt(7)
-; GCN-HSA-NEXT:    v_ashr_i64 v[10:11], v[4:5], 48
-; GCN-HSA-NEXT:    v_bfe_i32 v8, v5, 0, 16
-; GCN-HSA-NEXT:    v_mov_b32_e32 v18, s4
-; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v9, 31, v8
-; GCN-HSA-NEXT:    v_mov_b32_e32 v3, v7
-; GCN-HSA-NEXT:    v_mov_b32_e32 v17, s3
-; GCN-HSA-NEXT:    flat_store_dwordx4 v[18:19], v[8:11]
-; GCN-HSA-NEXT:    v_mov_b32_e32 v16, s2
-; GCN-HSA-NEXT:    v_bfe_i32 v8, v3, 0, 16
-; GCN-HSA-NEXT:    v_ashr_i64 v[10:11], v[6:7], 48
-; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v9, 31, v8
-; GCN-HSA-NEXT:    v_mov_b32_e32 v21, s9
-; GCN-HSA-NEXT:    flat_store_dwordx4 v[16:17], v[8:11]
-; GCN-HSA-NEXT:    s_waitcnt vmcnt(8)
-; GCN-HSA-NEXT:    v_bfe_i32 v7, v13, 0, 16
-; GCN-HSA-NEXT:    v_ashr_i64 v[9:10], v[12:13], 48
-; GCN-HSA-NEXT:    v_mov_b32_e32 v20, s8
-; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v8, 31, v7
+; GCN-HSA-NEXT:    v_ashr_i64 v[18:19], v[12:13], 48
+; GCN-HSA-NEXT:    v_bfe_i32 v16, v13, 0, 16
+; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v17, 31, v16
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v3, v15
-; GCN-HSA-NEXT:    v_mov_b32_e32 v23, s7
-; GCN-HSA-NEXT:    flat_store_dwordx4 v[20:21], v[7:10]
-; GCN-HSA-NEXT:    v_mov_b32_e32 v22, s6
-; GCN-HSA-NEXT:    v_bfe_i32 v7, v3, 0, 16
-; GCN-HSA-NEXT:    v_ashr_i64 v[9:10], v[14:15], 48
-; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v25, 16, v2
-; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v1, 16, v0
-; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v8, 31, v7
+; GCN-HSA-NEXT:    v_mov_b32_e32 v21, s5
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[24:25], v[16:19]
+; GCN-HSA-NEXT:    v_mov_b32_e32 v20, s4
+; GCN-HSA-NEXT:    v_bfe_i32 v16, v3, 0, 16
+; GCN-HSA-NEXT:    v_ashr_i64 v[18:19], v[14:15], 48
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v1, 16, v8
+; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v17, 31, v16
+; GCN-HSA-NEXT:    v_bfe_i32 v8, v8, 0, 16
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v24, 16, v14
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[20:21], v[16:19]
+; GCN-HSA-NEXT:    v_bfe_i32 v10, v1, 0, 16
+; GCN-HSA-NEXT:    v_bfe_i32 v16, v14, 0, 16
+; GCN-HSA-NEXT:    v_mov_b32_e32 v15, s1
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v26, 16, v2
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v18, 16, v12
+; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v9, 31, v8
+; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v11, 31, v10
+; GCN-HSA-NEXT:    v_mov_b32_e32 v14, s0
 ; GCN-HSA-NEXT:    s_add_u32 s2, s0, 0xe0
-; GCN-HSA-NEXT:    flat_store_dwordx4 v[22:23], v[7:10]
-; GCN-HSA-NEXT:    v_bfe_i32 v19, v0, 0, 16
-; GCN-HSA-NEXT:    v_bfe_i32 v23, v2, 0, 16
-; GCN-HSA-NEXT:    v_bfe_i32 v21, v1, 0, 16
-; GCN-HSA-NEXT:    v_bfe_i32 v25, v25, 0, 16
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v22, 16, v0
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[14:15], v[8:11]
+; GCN-HSA-NEXT:    v_bfe_i32 v14, v18, 0, 16
+; GCN-HSA-NEXT:    v_bfe_i32 v18, v24, 0, 16
+; GCN-HSA-NEXT:    v_bfe_i32 v20, v0, 0, 16
+; GCN-HSA-NEXT:    v_bfe_i32 v24, v2, 0, 16
+; GCN-HSA-NEXT:    v_bfe_i32 v26, v26, 0, 16
 ; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
-; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v24, 31, v23
-; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v26, 31, v25
+; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v25, 31, v24
+; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v27, 31, v26
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN-HSA-NEXT:    s_add_u32 s2, s0, 0xc0
-; GCN-HSA-NEXT:    flat_store_dwordx4 v[0:1], v[23:26]
+; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v23, 16, v6
+; GCN-HSA-NEXT:    v_bfe_i32 v22, v22, 0, 16
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[0:1], v[24:27]
 ; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
-; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v18, 16, v6
-; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v20, 31, v19
-; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v22, 31, v21
+; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v21, 31, v20
+; GCN-HSA-NEXT:    v_bfe_i32 v9, v23, 0, 16
+; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v23, 31, v22
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN-HSA-NEXT:    s_add_u32 s2, s0, 0xa0
-; GCN-HSA-NEXT:    v_bfe_i32 v15, v6, 0, 16
-; GCN-HSA-NEXT:    v_bfe_i32 v17, v18, 0, 16
-; GCN-HSA-NEXT:    flat_store_dwordx4 v[0:1], v[19:22]
+; GCN-HSA-NEXT:    v_bfe_i32 v7, v6, 0, 16
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[0:1], v[20:23]
 ; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
 ; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v5, 16, v4
-; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v3, 16, v12
-; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v16, 31, v15
-; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v18, 31, v17
+; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v8, 31, v7
+; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v10, 31, v9
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN-HSA-NEXT:    s_add_u32 s2, s0, 0x80
-; GCN-HSA-NEXT:    v_bfe_i32 v9, v3, 0, 16
 ; GCN-HSA-NEXT:    v_bfe_i32 v3, v4, 0, 16
 ; GCN-HSA-NEXT:    v_bfe_i32 v5, v5, 0, 16
-; GCN-HSA-NEXT:    flat_store_dwordx4 v[0:1], v[15:18]
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[0:1], v[7:10]
 ; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
-; GCN-HSA-NEXT:    v_lshrrev_b32_e32 v13, 16, v14
 ; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v4, 31, v3
 ; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v6, 31, v5
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN-HSA-NEXT:    s_add_u32 s2, s0, 0x60
-; GCN-HSA-NEXT:    v_bfe_i32 v11, v14, 0, 16
-; GCN-HSA-NEXT:    v_bfe_i32 v13, v13, 0, 16
 ; GCN-HSA-NEXT:    flat_store_dwordx4 v[0:1], v[3:6]
 ; GCN-HSA-NEXT:    s_addc_u32 s3, s1, 0
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s2
-; GCN-HSA-NEXT:    v_bfe_i32 v7, v12, 0, 16
-; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v12, 31, v11
-; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v14, 31, v13
+; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v17, 31, v16
+; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v19, 31, v18
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s3
 ; GCN-HSA-NEXT:    s_add_u32 s0, s0, 64
-; GCN-HSA-NEXT:    flat_store_dwordx4 v[0:1], v[11:14]
+; GCN-HSA-NEXT:    v_bfe_i32 v12, v12, 0, 16
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[0:1], v[16:19]
 ; GCN-HSA-NEXT:    s_addc_u32 s1, s1, 0
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v0, s0
-; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v8, 31, v7
-; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v10, 31, v9
+; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v13, 31, v12
+; GCN-HSA-NEXT:    v_ashrrev_i32_e32 v15, 31, v14
 ; GCN-HSA-NEXT:    v_mov_b32_e32 v1, s1
-; GCN-HSA-NEXT:    flat_store_dwordx4 v[0:1], v[7:10]
+; GCN-HSA-NEXT:    flat_store_dwordx4 v[0:1], v[12:15]
 ; GCN-HSA-NEXT:    s_endpgm
 ;
 ; GCN-NOHSA-VI-LABEL: global_sextload_v32i16_to_v32i64:
@@ -8259,115 +8258,115 @@ define amdgpu_kernel void @global_sextload_v32i16_to_v32i64(ptr addrspace(1) %ou
 ; GCN-NOHSA-VI-NEXT:    s_waitcnt lgkmcnt(0)
 ; GCN-NOHSA-VI-NEXT:    s_mov_b32 s8, s6
 ; GCN-NOHSA-VI-NEXT:    s_mov_b32 s9, s7
-; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[7:10], off, s[8:11], 0
-; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[15:18], off, s[8:11], 0 offset:48
-; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[11:14], off, s[8:11], 0 offset:32
-; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[3:6], off, s[8:11], 0 offset:16
+; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[5:8], off, s[8:11], 0
+; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[13:16], off, s[8:11], 0 offset:48
+; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[9:12], off, s[8:11], 0 offset:32
+; GCN-NOHSA-VI-NEXT:    buffer_load_dwordx4 v[1:4], off, s[8:11], 0 offset:16
 ; GCN-NOHSA-VI-NEXT:    s_mov_b32 s0, s4
 ; GCN-NOHSA-VI-NEXT:    s_mov_b32 s1, s5
 ; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(3)
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v0, v8, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v1, 16, v8
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v0, v6, 0, 16
 ; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(2)
-; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v8, 16, v17
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v21, v8, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v19, v17, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v2, v18
-; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v20, 31, v19
-; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v22, 31, v21
-; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v8, 16, v18
-; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[19:22], off, s[0:3], 0 offset:224
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v17, v2, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v19, v8, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v18, 31, v17
-; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v20, 31, v19
-; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v2, 16, v15
-; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[17:20], off, s[0:3], 0 offset:240
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v8, v7, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v17, v15, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v19, v2, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v18, 31, v17
-; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v20, 31, v19
-; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v2, 16, v16
-; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[17:20], off, s[0:3], 0 offset:192
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v15, v16, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v17, v2, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v16, 31, v15
-; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v18, 31, v17
-; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[15:18], off, s[0:3], 0 offset:208
-; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(5)
-; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v2, v14
-; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v15, 16, v13
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v17, v15, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v15, v13, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v16, 31, v15
-; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v18, 31, v17
-; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v14, 16, v14
-; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[15:18], off, s[0:3], 0 offset:160
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v13, v2, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v15, v14, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v18, v15, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v15, 16, v15
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v20, v15, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v17, 16, v6
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v6, v16
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v16, 16, v16
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v19, 31, v18
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v21, 31, v20
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[18:21], off, s[0:3], 0 offset:224
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v15, 16, v14
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v18, v6, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v20, v16, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v19, 31, v18
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v21, 31, v20
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[18:21], off, s[0:3], 0 offset:240
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v6, 16, v13
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v18, v13, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v13, v14, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v15, v15, 0, 16
 ; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v14, 31, v13
 ; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v16, 31, v15
-; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v2, 16, v11
-; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[13:16], off, s[0:3], 0 offset:176
-; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v7, 16, v7
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v15, v2, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v20, v6, 0, 16
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[13:16], off, s[0:3], 0 offset:208
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v19, 31, v18
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(4)
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v13, 16, v11
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v21, 31, v20
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v15, v13, 0, 16
 ; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v13, v11, 0, 16
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[18:21], off, s[0:3], 0 offset:192
 ; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v14, 31, v13
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v19, v12
 ; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v16, 31, v15
-; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v2, 16, v12
-; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[13:16], off, s[0:3], 0 offset:128
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v11, v12, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v13, v2, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v17, v10
-; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v18, 16, v10
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v12, 16, v12
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[13:16], off, s[0:3], 0 offset:160
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v11, v19, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v13, v12, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v18, v8
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v16, 16, v8
 ; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v12, 31, v11
 ; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v14, 31, v13
-; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(7)
-; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v10, 16, v5
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v2, v1, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v1, 16, v4
-; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[11:14], off, s[0:3], 0 offset:144
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v15, v10, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v13, v5, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v10, v7, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v22, v18, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v18, v1, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v1, v6
-; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v7, 16, v6
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v12, v9, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v9, 16, v9
-; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v14, 31, v13
-; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v16, 31, v15
-; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v5, 16, v3
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v24, v1, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v26, v7, 0, 16
-; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[13:16], off, s[0:3], 0 offset:96
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v20, v17, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v16, v4, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v14, v9, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v4, v3, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v8, 16, v9
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[11:14], off, s[0:3], 0 offset:176
 ; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v6, v5, 0, 16
-; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v25, 31, v24
-; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v27, 31, v26
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v13, v8, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v11, v9, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v12, 31, v11
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v14, 31, v13
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v9, 16, v10
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[11:14], off, s[0:3], 0 offset:128
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v9, v9, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v11, v7, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v12, 16, v7
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v7, v10, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v8, 31, v7
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v10, 31, v9
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[7:10], off, s[0:3], 0 offset:144
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v5, 16, v5
+; GCN-NOHSA-VI-NEXT:    s_waitcnt vmcnt(8)
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v7, v3, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v3, 16, v3
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v9, v3, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v8, 31, v7
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v10, 31, v9
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[7:10], off, s[0:3], 0 offset:96
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v3, 16, v2
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v8, v5, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_mov_b32_e32 v5, v4
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v4, 16, v4
+; GCN-NOHSA-VI-NEXT:    v_lshrrev_b32_e32 v7, 16, v1
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v27, v5, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v29, v4, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v15, v2, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v2, v17, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v19, v18, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v21, v16, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v13, v12, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v23, v1, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v17, v3, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_bfe_i32 v25, v7, 0, 16
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v28, 31, v27
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v30, 31, v29
 ; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v1, 31, v0
 ; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v3, 31, v2
-; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v9, 31, v8
-; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v11, 31, v10
-; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v21, 31, v20
-; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v23, 31, v22
-; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v13, 31, v12
-; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v15, 31, v14
-; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v17, 31, v16
-; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v19, 31, v18
-; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v5, 31, v4
 ; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v7, 31, v6
-; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[24:27], off, s[0:3], 0 offset:112
-; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:64
-; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[16:19], off, s[0:3], 0 offset:80
-; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[12:15], off, s[0:3], 0 offset:32
-; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[20:23], off, s[0:3], 0 offset:48
-; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[8:11], off, s[0:3], 0
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v9, 31, v8
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v20, 31, v19
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v22, 31, v21
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v12, 31, v11
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v14, 31, v13
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v16, 31, v15
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v18, 31, v17
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v24, 31, v23
+; GCN-NOHSA-VI-NEXT:    v_ashrrev_i32_e32 v26, 31, v25
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[27:30], off, s[0:3], 0 offset:112
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[23:26], off, s[0:3], 0 offset:64
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[15:18], off, s[0:3], 0 offset:80
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[11:14], off, s[0:3], 0 offset:32
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[19:22], off, s[0:3], 0 offset:48
+; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[6:9], off, s[0:3], 0
 ; GCN-NOHSA-VI-NEXT:    buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:16
 ; GCN-NOHSA-VI-NEXT:    s_endpgm
 ;
diff --git a/llvm/test/CodeGen/AMDGPU/loop-live-out-copy-undef-subrange.ll b/llvm/test/CodeGen/AMDGPU/loop-live-out-copy-undef-subrange.ll
index 7738ec689dc7ebbb9ab04989d40600dc89a346f6..2d3c03bbe5317990c077a8158a422803c78e493e 100644
--- a/llvm/test/CodeGen/AMDGPU/loop-live-out-copy-undef-subrange.ll
+++ b/llvm/test/CodeGen/AMDGPU/loop-live-out-copy-undef-subrange.ll
@@ -1,5 +1,5 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx906 -verify-machineinstrs < %s | FileCheck %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx906 -amdgpu-enable-rewrite-partial-reg-uses=false -verify-machineinstrs < %s | FileCheck %s
 
 ; This example used to produce a verifier error resulting from the
 ; register coalescer leaving behind a false live interval when a live
diff --git a/llvm/test/CodeGen/AMDGPU/machine-sink-temporal-divergence-swdev407790.ll b/llvm/test/CodeGen/AMDGPU/machine-sink-temporal-divergence-swdev407790.ll
index cf588601016007fb0673ddc3aac0331f146083ee..1c75a2fc3dce67099f104f2da8898f491ab793b2 100644
--- a/llvm/test/CodeGen/AMDGPU/machine-sink-temporal-divergence-swdev407790.ll
+++ b/llvm/test/CodeGen/AMDGPU/machine-sink-temporal-divergence-swdev407790.ll
@@ -47,7 +47,7 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
 ; CHECK-NEXT:    s_add_u32 s0, s0, s15
 ; CHECK-NEXT:    s_mov_b64 s[34:35], s[6:7]
 ; CHECK-NEXT:    s_addc_u32 s1, s1, 0
-; CHECK-NEXT:    v_mov_b32_e32 v41, v0
+; CHECK-NEXT:    v_mov_b32_e32 v40, v0
 ; CHECK-NEXT:    s_add_u32 s42, s34, 40
 ; CHECK-NEXT:    v_mov_b32_e32 v31, v0
 ; CHECK-NEXT:    v_mov_b32_e32 v0, 0
@@ -65,7 +65,7 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
 ; CHECK-NEXT:    v_mov_b32_e32 v45, 0
 ; CHECK-NEXT:    s_swappc_b64 s[30:31], s[6:7]
 ; CHECK-NEXT:    v_mov_b32_e32 v43, v0
-; CHECK-NEXT:    v_mov_b32_e32 v31, v41
+; CHECK-NEXT:    v_mov_b32_e32 v31, v40
 ; CHECK-NEXT:    v_mov_b32_e32 v0, 0
 ; CHECK-NEXT:    s_mov_b64 s[4:5], s[38:39]
 ; CHECK-NEXT:    s_mov_b64 s[8:9], s[42:43]
@@ -77,8 +77,8 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
 ; CHECK-NEXT:    s_add_u32 s6, s6, _Z12get_local_idj@rel32@lo+4
 ; CHECK-NEXT:    s_addc_u32 s7, s7, _Z12get_local_idj@rel32@hi+12
 ; CHECK-NEXT:    s_swappc_b64 s[30:31], s[6:7]
-; CHECK-NEXT:    v_mov_b32_e32 v40, v0
-; CHECK-NEXT:    v_mov_b32_e32 v31, v41
+; CHECK-NEXT:    v_mov_b32_e32 v41, v0
+; CHECK-NEXT:    v_mov_b32_e32 v31, v40
 ; CHECK-NEXT:    v_mov_b32_e32 v0, 1
 ; CHECK-NEXT:    s_mov_b64 s[4:5], s[38:39]
 ; CHECK-NEXT:    s_mov_b64 s[8:9], s[42:43]
@@ -93,7 +93,7 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
 ; CHECK-NEXT:    s_swappc_b64 s[30:31], s[52:53]
 ; CHECK-NEXT:    v_lshrrev_b32_e32 v0, 1, v43
 ; CHECK-NEXT:    v_lshlrev_b32_e32 v1, 2, v43
-; CHECK-NEXT:    v_mov_b32_e32 v31, v41
+; CHECK-NEXT:    v_mov_b32_e32 v31, v40
 ; CHECK-NEXT:    s_mov_b64 s[4:5], s[38:39]
 ; CHECK-NEXT:    s_mov_b64 s[8:9], s[42:43]
 ; CHECK-NEXT:    v_and_b32_e32 v0, 0x7ffffffc, v0
@@ -115,7 +115,7 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
 ; CHECK-NEXT:    v_cmpx_ne_u32_e32 0, v42
 ; CHECK-NEXT:    s_cbranch_execz .LBB0_25
 ; CHECK-NEXT:  ; %bb.1: ; %.preheader5
-; CHECK-NEXT:    v_mul_lo_u32 v0, v40, 14
+; CHECK-NEXT:    v_mul_lo_u32 v0, v41, 14
 ; CHECK-NEXT:    s_mov_b32 s4, 0
 ; CHECK-NEXT:    s_mov_b32 s5, 0
 ; CHECK-NEXT:    v_add_nc_u32_e32 v44, 0x3c04, v0
@@ -184,7 +184,7 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
 ; CHECK-NEXT:    v_cmpx_eq_u16_e64 v56, v0
 ; CHECK-NEXT:    s_cbranch_execz .LBB0_10
 ; CHECK-NEXT:  ; %bb.9: ; in Loop: Header=BB0_8 Depth=2
-; CHECK-NEXT:    v_mov_b32_e32 v31, v41
+; CHECK-NEXT:    v_mov_b32_e32 v31, v40
 ; CHECK-NEXT:    v_mov_b32_e32 v0, 0x3c00
 ; CHECK-NEXT:    s_add_u32 s8, s34, 40
 ; CHECK-NEXT:    s_addc_u32 s9, s35, 0
@@ -205,7 +205,7 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
 ; CHECK-NEXT:    v_cmpx_eq_u16_e64 v56, v0
 ; CHECK-NEXT:    s_cbranch_execz .LBB0_12
 ; CHECK-NEXT:  ; %bb.11: ; in Loop: Header=BB0_8 Depth=2
-; CHECK-NEXT:    v_mov_b32_e32 v31, v41
+; CHECK-NEXT:    v_mov_b32_e32 v31, v40
 ; CHECK-NEXT:    v_mov_b32_e32 v0, 0x3c00
 ; CHECK-NEXT:    s_add_u32 s8, s34, 40
 ; CHECK-NEXT:    s_addc_u32 s9, s35, 0
@@ -227,7 +227,7 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
 ; CHECK-NEXT:    v_cmpx_eq_u16_e64 v56, v0
 ; CHECK-NEXT:    s_cbranch_execz .LBB0_14
 ; CHECK-NEXT:  ; %bb.13: ; in Loop: Header=BB0_8 Depth=2
-; CHECK-NEXT:    v_mov_b32_e32 v31, v41
+; CHECK-NEXT:    v_mov_b32_e32 v31, v40
 ; CHECK-NEXT:    v_mov_b32_e32 v0, 0x3c00
 ; CHECK-NEXT:    s_add_u32 s8, s34, 40
 ; CHECK-NEXT:    s_addc_u32 s9, s35, 0
@@ -249,7 +249,7 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
 ; CHECK-NEXT:    v_cmpx_eq_u16_e64 v56, v0
 ; CHECK-NEXT:    s_cbranch_execz .LBB0_7
 ; CHECK-NEXT:  ; %bb.15: ; in Loop: Header=BB0_8 Depth=2
-; CHECK-NEXT:    v_mov_b32_e32 v31, v41
+; CHECK-NEXT:    v_mov_b32_e32 v31, v40
 ; CHECK-NEXT:    v_mov_b32_e32 v0, 0x3c00
 ; CHECK-NEXT:    s_add_u32 s8, s34, 40
 ; CHECK-NEXT:    s_addc_u32 s9, s35, 0
@@ -297,7 +297,7 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
 ; CHECK-NEXT:    v_cmpx_eq_u16_e64 v56, v0
 ; CHECK-NEXT:    s_cbranch_execz .LBB0_19
 ; CHECK-NEXT:  ; %bb.21: ; in Loop: Header=BB0_20 Depth=2
-; CHECK-NEXT:    v_mov_b32_e32 v31, v41
+; CHECK-NEXT:    v_mov_b32_e32 v31, v40
 ; CHECK-NEXT:    v_mov_b32_e32 v0, 0x3c00
 ; CHECK-NEXT:    s_add_u32 s8, s34, 40
 ; CHECK-NEXT:    s_addc_u32 s9, s35, 0
@@ -330,7 +330,7 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
 ; CHECK-NEXT:    s_cbranch_execnz .LBB0_5
 ; CHECK-NEXT:  .LBB0_25: ; %Flow49
 ; CHECK-NEXT:    s_or_b32 exec_lo, exec_lo, s48
-; CHECK-NEXT:    v_mov_b32_e32 v31, v41
+; CHECK-NEXT:    v_mov_b32_e32 v31, v40
 ; CHECK-NEXT:    v_mov_b32_e32 v0, 1
 ; CHECK-NEXT:    s_add_u32 s8, s34, 40
 ; CHECK-NEXT:    s_addc_u32 s9, s35, 0
@@ -344,7 +344,7 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
 ; CHECK-NEXT:    s_mov_b32 s4, exec_lo
 ; CHECK-NEXT:    ds_read_b32 v47, v0 offset:15360
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    v_cmpx_gt_u32_e64 v47, v40
+; CHECK-NEXT:    v_cmpx_gt_u32_e64 v47, v41
 ; CHECK-NEXT:    s_cbranch_execz .LBB0_33
 ; CHECK-NEXT:  ; %bb.26:
 ; CHECK-NEXT:    s_add_u32 s52, s44, 8
@@ -362,7 +362,7 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
 ; CHECK-NEXT:    s_branch .LBB0_28
 ; CHECK-NEXT:  .LBB0_27: ; in Loop: Header=BB0_28 Depth=1
 ; CHECK-NEXT:    s_or_b32 exec_lo, exec_lo, s55
-; CHECK-NEXT:    v_mov_b32_e32 v31, v41
+; CHECK-NEXT:    v_mov_b32_e32 v31, v40
 ; CHECK-NEXT:    v_mov_b32_e32 v0, 0
 ; CHECK-NEXT:    s_add_u32 s8, s34, 40
 ; CHECK-NEXT:    s_addc_u32 s9, s35, 0
@@ -372,13 +372,13 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
 ; CHECK-NEXT:    s_mov_b32 s13, s40
 ; CHECK-NEXT:    s_mov_b32 s14, s33
 ; CHECK-NEXT:    s_swappc_b64 s[30:31], s[48:49]
-; CHECK-NEXT:    v_add_co_u32 v40, vcc_lo, v0, v40
-; CHECK-NEXT:    v_cmp_le_u32_e32 vcc_lo, v47, v40
+; CHECK-NEXT:    v_add_co_u32 v41, vcc_lo, v0, v41
+; CHECK-NEXT:    v_cmp_le_u32_e32 vcc_lo, v47, v41
 ; CHECK-NEXT:    s_or_b32 s54, vcc_lo, s54
 ; CHECK-NEXT:    s_andn2_b32 exec_lo, exec_lo, s54
 ; CHECK-NEXT:    s_cbranch_execz .LBB0_33
 ; CHECK-NEXT:  .LBB0_28: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT:    v_lshlrev_b32_e32 v0, 2, v40
+; CHECK-NEXT:    v_lshlrev_b32_e32 v0, 2, v41
 ; CHECK-NEXT:    s_mov_b32 s55, exec_lo
 ; CHECK-NEXT:    ds_read_b32 v0, v0
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
@@ -413,7 +413,7 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
 ; CHECK-NEXT:    v_lshlrev_b32_e32 v0, 4, v45
 ; CHECK-NEXT:    v_alignbit_b32 v1, v46, v45, 12
 ; CHECK-NEXT:    v_and_b32_e32 v2, 0xf0000, v45
-; CHECK-NEXT:    v_mov_b32_e32 v31, v41
+; CHECK-NEXT:    v_mov_b32_e32 v31, v40
 ; CHECK-NEXT:    s_add_u32 s8, s34, 40
 ; CHECK-NEXT:    v_and_b32_e32 v3, 0xf000, v0
 ; CHECK-NEXT:    v_and_b32_e32 v4, 0xf00, v1
@@ -471,7 +471,7 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
 ; CHECK-NEXT:    s_andn2_saveexec_b32 s4, s4
 ; CHECK-NEXT:    s_cbranch_execz .LBB0_27
 ; CHECK-NEXT:  ; %bb.32: ; in Loop: Header=BB0_28 Depth=1
-; CHECK-NEXT:    v_mov_b32_e32 v31, v41
+; CHECK-NEXT:    v_mov_b32_e32 v31, v40
 ; CHECK-NEXT:    v_mov_b32_e32 v0, v42
 ; CHECK-NEXT:    v_mov_b32_e32 v1, v43
 ; CHECK-NEXT:    v_mov_b32_e32 v2, v44
diff --git a/llvm/test/CodeGen/AMDGPU/mad_64_32.ll b/llvm/test/CodeGen/AMDGPU/mad_64_32.ll
index 3a337ec7a8733d5c804fbbb3e19eaa44ffc6635d..bd2e5cc5952bff0a2350432b72f65cdc3a8e060b 100644
--- a/llvm/test/CodeGen/AMDGPU/mad_64_32.ll
+++ b/llvm/test/CodeGen/AMDGPU/mad_64_32.ll
@@ -155,24 +155,24 @@ define i128 @mad_i64_i32_sextops_i32_i128(i32 %arg0, i32 %arg1, i128 %arg2) #0 {
 ; CI:       ; %bb.0:
 ; CI-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
 ; CI-NEXT:    v_mad_u64_u32 v[6:7], s[4:5], v0, v1, 0
-; CI-NEXT:    v_ashrrev_i32_e32 v13, 31, v0
+; CI-NEXT:    v_ashrrev_i32_e32 v12, 31, v0
 ; CI-NEXT:    v_mov_b32_e32 v8, 0
-; CI-NEXT:    v_mad_u64_u32 v[9:10], s[4:5], v13, v1, v[7:8]
-; CI-NEXT:    v_ashrrev_i32_e32 v14, 31, v1
-; CI-NEXT:    v_mad_i64_i32 v[11:12], s[4:5], v1, v13, 0
-; CI-NEXT:    v_mov_b32_e32 v7, v10
+; CI-NEXT:    v_mad_u64_u32 v[9:10], s[4:5], v12, v1, v[7:8]
+; CI-NEXT:    v_ashrrev_i32_e32 v13, 31, v1
+; CI-NEXT:    v_mov_b32_e32 v11, v10
 ; CI-NEXT:    v_mov_b32_e32 v10, v8
-; CI-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], v0, v14, v[9:10]
-; CI-NEXT:    v_mad_i64_i32 v[0:1], s[4:5], v14, v0, v[11:12]
-; CI-NEXT:    v_add_i32_e32 v9, vcc, v7, v9
-; CI-NEXT:    v_addc_u32_e64 v10, s[4:5], 0, 0, vcc
-; CI-NEXT:    v_mad_u64_u32 v[9:10], s[4:5], v13, v14, v[9:10]
-; CI-NEXT:    v_add_i32_e32 v7, vcc, v9, v0
-; CI-NEXT:    v_addc_u32_e32 v9, vcc, v10, v1, vcc
-; CI-NEXT:    v_mov_b32_e32 v1, v8
+; CI-NEXT:    v_mad_u64_u32 v[7:8], s[4:5], v0, v13, v[9:10]
+; CI-NEXT:    v_add_i32_e32 v8, vcc, v11, v8
+; CI-NEXT:    v_mad_i64_i32 v[10:11], s[4:5], v1, v12, 0
+; CI-NEXT:    v_addc_u32_e64 v9, s[4:5], 0, 0, vcc
+; CI-NEXT:    v_mad_u64_u32 v[8:9], s[4:5], v12, v13, v[8:9]
+; CI-NEXT:    v_mad_i64_i32 v[0:1], s[4:5], v13, v0, v[10:11]
+; CI-NEXT:    v_add_i32_e32 v8, vcc, v8, v0
+; CI-NEXT:    v_addc_u32_e32 v9, vcc, v9, v1, vcc
+; CI-NEXT:    v_mov_b32_e32 v1, v7
 ; CI-NEXT:    v_add_i32_e32 v0, vcc, v6, v2
 ; CI-NEXT:    v_addc_u32_e32 v1, vcc, v1, v3, vcc
-; CI-NEXT:    v_addc_u32_e32 v2, vcc, v7, v4, vcc
+; CI-NEXT:    v_addc_u32_e32 v2, vcc, v8, v4, vcc
 ; CI-NEXT:    v_addc_u32_e32 v3, vcc, v9, v5, vcc
 ; CI-NEXT:    s_setpc_b64 s[30:31]
 ;
@@ -245,26 +245,24 @@ define i128 @mad_i64_i32_sextops_i32_i128(i32 %arg0, i32 %arg1, i128 %arg2) #0 {
 ; GFX11-NEXT:    v_ashrrev_i32_e32 v15, 31, v1
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-NEXT:    v_mad_u64_u32 v[9:10], null, v14, v1, v[7:8]
-; GFX11-NEXT:    v_dual_mov_b32 v7, v10 :: v_dual_mov_b32 v10, v8
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_mad_u64_u32 v[11:12], null, v0, v15, v[9:10]
-; GFX11-NEXT:    v_mad_i64_i32 v[9:10], null, v1, v14, 0
-; GFX11-NEXT:    v_mov_b32_e32 v8, v12
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_mad_i64_i32 v[12:13], null, v15, v0, v[9:10]
-; GFX11-NEXT:    v_add_co_u32 v7, s0, v7, v8
+; GFX11-NEXT:    v_dual_mov_b32 v11, v10 :: v_dual_mov_b32 v10, v8
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v8, null, 0, 0, s0
-; GFX11-NEXT:    v_mad_u64_u32 v[0:1], null, v14, v15, v[7:8]
-; GFX11-NEXT:    v_mov_b32_e32 v7, v11
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_mad_u64_u32 v[7:8], null, v0, v15, v[9:10]
+; GFX11-NEXT:    v_mov_b32_e32 v10, v8
+; GFX11-NEXT:    v_mad_i64_i32 v[8:9], null, v1, v14, 0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_add_co_u32 v10, s0, v11, v10
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v11, null, 0, 0, s0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT:    v_mad_i64_i32 v[12:13], null, v15, v0, v[8:9]
+; GFX11-NEXT:    v_mad_u64_u32 v[0:1], null, v14, v15, v[10:11]
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_add_co_u32 v8, vcc_lo, v0, v12
 ; GFX11-NEXT:    v_add_co_ci_u32_e32 v9, vcc_lo, v1, v13, vcc_lo
 ; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, v6, v2
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX11-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, v7, v3, vcc_lo
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
 ; GFX11-NEXT:    v_add_co_ci_u32_e32 v2, vcc_lo, v8, v4, vcc_lo
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4)
 ; GFX11-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, v9, v5, vcc_lo
 ; GFX11-NEXT:    s_setpc_b64 s[30:31]
   %sext0 = sext i32 %arg0 to i128
diff --git a/llvm/test/CodeGen/AMDGPU/memcpy-crash-issue63986.ll b/llvm/test/CodeGen/AMDGPU/memcpy-crash-issue63986.ll
index 615039500962da99534b20f3ede4d9a9b25d60ad..8cf70eaf29e85ffcdb352355506c4cdba8b4db79 100644
--- a/llvm/test/CodeGen/AMDGPU/memcpy-crash-issue63986.ll
+++ b/llvm/test/CodeGen/AMDGPU/memcpy-crash-issue63986.ll
@@ -69,11 +69,11 @@ define void @issue63986(i64 %0, i64 %idxprom) {
 ; CHECK-NEXT:    v_lshlrev_b64 v[2:3], 6, v[2:3]
 ; CHECK-NEXT:    s_cbranch_execnz .LBB0_9
 ; CHECK-NEXT:  .LBB0_6: ; %loop-memcpy-residual.preheader
-; CHECK-NEXT:    v_mov_b32_e32 v7, s5
-; CHECK-NEXT:    v_or_b32_e32 v2, 32, v4
-; CHECK-NEXT:    v_mov_b32_e32 v3, v5
+; CHECK-NEXT:    v_mov_b32_e32 v2, s4
+; CHECK-NEXT:    v_or_b32_e32 v6, 32, v4
+; CHECK-NEXT:    v_mov_b32_e32 v7, v5
 ; CHECK-NEXT:    s_mov_b64 s[6:7], 0
-; CHECK-NEXT:    v_mov_b32_e32 v6, s4
+; CHECK-NEXT:    v_mov_b32_e32 v3, s5
 ; CHECK-NEXT:  .LBB0_7: ; %loop-memcpy-residual
 ; CHECK-NEXT:    ; =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    s_add_u32 s4, 32, s6
@@ -82,11 +82,11 @@ define void @issue63986(i64 %0, i64 %idxprom) {
 ; CHECK-NEXT:    v_mov_b32_e32 v8, s4
 ; CHECK-NEXT:    flat_load_ubyte v10, v[8:9]
 ; CHECK-NEXT:    v_mov_b32_e32 v9, s7
-; CHECK-NEXT:    v_add_co_u32_e32 v8, vcc, s6, v2
+; CHECK-NEXT:    v_add_co_u32_e32 v8, vcc, s6, v6
 ; CHECK-NEXT:    s_add_u32 s6, s6, 1
-; CHECK-NEXT:    v_addc_co_u32_e32 v9, vcc, v3, v9, vcc
+; CHECK-NEXT:    v_addc_co_u32_e32 v9, vcc, v7, v9, vcc
 ; CHECK-NEXT:    s_addc_u32 s7, s7, 0
-; CHECK-NEXT:    v_cmp_lt_u64_e32 vcc, s[6:7], v[6:7]
+; CHECK-NEXT:    v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
 ; CHECK-NEXT:    s_waitcnt vmcnt(0) lgkmcnt(0)
 ; CHECK-NEXT:    flat_store_byte v[8:9], v10
 ; CHECK-NEXT:    s_cbranch_vccnz .LBB0_7
diff --git a/llvm/test/CodeGen/AMDGPU/minimummaximum.ll b/llvm/test/CodeGen/AMDGPU/minimummaximum.ll
new file mode 100644
index 0000000000000000000000000000000000000000..e73ce7a4826c045c2cfb0e38aa1a8df107a89534
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/minimummaximum.ll
@@ -0,0 +1,144 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -march=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX12,SDAG %s
+; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1200 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX12,GISEL %s
+
+define amdgpu_ps float @test_minmax_f32(float %a, float %b, float %c) {
+; GFX12-LABEL: test_minmax_f32:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    v_maximumminimum_f32 v0, v0, v1, v2
+; GFX12-NEXT:    ; return to shader part epilog
+  %max = call float @llvm.maximum.f32(float %a, float %b)
+  %minmax = call float @llvm.minimum.f32(float %max, float %c)
+  ret float %minmax
+}
+
+define amdgpu_ps void @s_test_minmax_f32(float inreg %a, float inreg %b, float inreg %c, ptr addrspace(1) inreg %out) {
+; SDAG-LABEL: s_test_minmax_f32:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    s_maximum_f32 s0, s0, s1
+; SDAG-NEXT:    s_mov_b32 s5, s4
+; SDAG-NEXT:    s_mov_b32 s4, s3
+; SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_3)
+; SDAG-NEXT:    s_minimum_f32 s0, s0, s2
+; SDAG-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
+; SDAG-NEXT:    global_store_b32 v0, v1, s[4:5]
+; SDAG-NEXT:    s_nop 0
+; SDAG-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; SDAG-NEXT:    s_endpgm
+;
+; GISEL-LABEL: s_test_minmax_f32:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    s_maximum_f32 s0, s0, s1
+; GISEL-NEXT:    s_mov_b32 s6, s3
+; GISEL-NEXT:    s_mov_b32 s7, s4
+; GISEL-NEXT:    v_mov_b32_e32 v1, 0
+; GISEL-NEXT:    s_minimum_f32 s0, s0, s2
+; GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_3)
+; GISEL-NEXT:    v_mov_b32_e32 v0, s0
+; GISEL-NEXT:    global_store_b32 v1, v0, s[6:7]
+; GISEL-NEXT:    s_nop 0
+; GISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GISEL-NEXT:    s_endpgm
+  %smax = call float @llvm.maximum.f32(float %a, float %b)
+  %sminmax = call float @llvm.minimum.f32(float %smax, float %c)
+  store float %sminmax, ptr addrspace(1) %out
+  ret void
+}
+
+define amdgpu_ps float @test_minmax_commuted_f32(float %a, float %b, float %c) {
+; GFX12-LABEL: test_minmax_commuted_f32:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    v_maximumminimum_f32 v0, v0, v1, v2
+; GFX12-NEXT:    ; return to shader part epilog
+  %max = call float @llvm.maximum.f32(float %a, float %b)
+  %minmax = call float @llvm.minimum.f32(float %c, float %max)
+  ret float %minmax
+}
+
+define amdgpu_ps float @test_maxmin_f32(float %a, float %b, float %c) {
+; GFX12-LABEL: test_maxmin_f32:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    v_minimummaximum_f32 v0, v0, v1, v2
+; GFX12-NEXT:    ; return to shader part epilog
+  %min = call float @llvm.minimum.f32(float %a, float %b)
+  %maxmin = call float @llvm.maximum.f32(float %min, float %c)
+  ret float %maxmin
+}
+
+define amdgpu_ps float @test_maxmin_commuted_f32(float %a, float %b, float %c) {
+; GFX12-LABEL: test_maxmin_commuted_f32:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    v_minimummaximum_f32 v0, v0, v1, v2
+; GFX12-NEXT:    ; return to shader part epilog
+  %min = call float @llvm.minimum.f32(float %a, float %b)
+  %maxmin = call float @llvm.maximum.f32(float %c, float %min)
+  ret float %maxmin
+}
+
+define amdgpu_ps half @test_minmax_f16(half %a, half %b, half %c) {
+; GFX12-LABEL: test_minmax_f16:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    v_maximumminimum_f16 v0, v0, v1, v2
+; GFX12-NEXT:    ; return to shader part epilog
+  %max = call half @llvm.maximum.f16(half %a, half %b)
+  %minmax = call half @llvm.minimum.f16(half %max, half %c)
+  ret half %minmax
+}
+
+define amdgpu_ps half @test_minmax_commuted_f16(half %a, half %b, half %c) {
+; GFX12-LABEL: test_minmax_commuted_f16:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    v_maximumminimum_f16 v0, v0, v1, v2
+; GFX12-NEXT:    ; return to shader part epilog
+  %max = call half @llvm.maximum.f16(half %a, half %b)
+  %minmax = call half @llvm.minimum.f16(half %c, half %max)
+  ret half %minmax
+}
+
+define amdgpu_ps half @test_maxmin_commuted_f16(half %a, half %b, half %c) {
+; GFX12-LABEL: test_maxmin_commuted_f16:
+; GFX12:       ; %bb.0:
+; GFX12-NEXT:    v_minimummaximum_f16 v0, v0, v1, v2
+; GFX12-NEXT:    ; return to shader part epilog
+  %min = call half @llvm.minimum.f16(half %a, half %b)
+  %maxmin = call half @llvm.maximum.f16(half %c, half %min)
+  ret half %maxmin
+}
+
+define amdgpu_ps void @s_test_minmax_f16(half inreg %a, half inreg %b, half inreg %c, ptr addrspace(1) inreg %out) {
+; SDAG-LABEL: s_test_minmax_f16:
+; SDAG:       ; %bb.0:
+; SDAG-NEXT:    s_maximum_f16 s0, s0, s1
+; SDAG-NEXT:    s_mov_b32 s5, s4
+; SDAG-NEXT:    s_mov_b32 s4, s3
+; SDAG-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_3)
+; SDAG-NEXT:    s_minimum_f16 s0, s0, s2
+; SDAG-NEXT:    v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
+; SDAG-NEXT:    global_store_b16 v0, v1, s[4:5]
+; SDAG-NEXT:    s_nop 0
+; SDAG-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; SDAG-NEXT:    s_endpgm
+;
+; GISEL-LABEL: s_test_minmax_f16:
+; GISEL:       ; %bb.0:
+; GISEL-NEXT:    s_maximum_f16 s0, s0, s1
+; GISEL-NEXT:    s_mov_b32 s6, s3
+; GISEL-NEXT:    s_mov_b32 s7, s4
+; GISEL-NEXT:    v_mov_b32_e32 v1, 0
+; GISEL-NEXT:    s_minimum_f16 s0, s0, s2
+; GISEL-NEXT:    s_delay_alu instid0(SALU_CYCLE_3)
+; GISEL-NEXT:    v_mov_b32_e32 v0, s0
+; GISEL-NEXT:    global_store_b16 v1, v0, s[6:7]
+; GISEL-NEXT:    s_nop 0
+; GISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GISEL-NEXT:    s_endpgm
+  %smax = call half @llvm.maximum.f16(half %a, half %b)
+  %sminmax = call half @llvm.minimum.f16(half %smax, half %c)
+  store half %sminmax, ptr addrspace(1) %out
+  ret void
+}
+
+declare half @llvm.minimum.f16(half, half)
+declare half @llvm.maximum.f16(half, half)
+declare float @llvm.minimum.f32(float, float)
+declare float @llvm.maximum.f32(float, float)
diff --git a/llvm/test/CodeGen/AMDGPU/mul.ll b/llvm/test/CodeGen/AMDGPU/mul.ll
index 179b0a855182ce2d364d02460dab31e6ebc7127c..5e90c33f3c8cb796ad52bea0a99821fa1df6df9f 100644
--- a/llvm/test/CodeGen/AMDGPU/mul.ll
+++ b/llvm/test/CodeGen/AMDGPU/mul.ll
@@ -2448,22 +2448,22 @@ define amdgpu_kernel void @v_mul_i128(ptr addrspace(1) %out, ptr addrspace(1) %a
 ; VI-NEXT:    v_mad_u64_u32 v[12:13], s[0:1], v4, v2, 0
 ; VI-NEXT:    v_mul_lo_u32 v14, v5, v2
 ; VI-NEXT:    v_mad_u64_u32 v[2:3], s[0:1], v0, v4, 0
-; VI-NEXT:    v_mul_lo_u32 v15, v7, v0
-; VI-NEXT:    v_add_u32_e32 v7, vcc, v13, v10
+; VI-NEXT:    v_add_u32_e32 v13, vcc, v13, v10
 ; VI-NEXT:    v_mov_b32_e32 v10, v3
 ; VI-NEXT:    v_mad_u64_u32 v[3:4], s[0:1], v1, v4, v[10:11]
-; VI-NEXT:    v_add_u32_e32 v13, vcc, v7, v14
-; VI-NEXT:    v_mov_b32_e32 v7, v4
+; VI-NEXT:    v_add_u32_e32 v13, vcc, v13, v14
+; VI-NEXT:    v_mov_b32_e32 v10, v4
 ; VI-NEXT:    v_mov_b32_e32 v4, v11
+; VI-NEXT:    v_mul_lo_u32 v7, v7, v0
 ; VI-NEXT:    v_mad_u64_u32 v[12:13], s[0:1], v6, v0, v[12:13]
 ; VI-NEXT:    v_mad_u64_u32 v[3:4], s[0:1], v0, v5, v[3:4]
-; VI-NEXT:    v_add_u32_e32 v11, vcc, v15, v13
+; VI-NEXT:    v_add_u32_e32 v13, vcc, v7, v13
 ; VI-NEXT:    v_mov_b32_e32 v0, v4
-; VI-NEXT:    v_mul_lo_u32 v10, v6, v1
-; VI-NEXT:    v_add_u32_e32 v6, vcc, v7, v0
+; VI-NEXT:    v_mul_lo_u32 v11, v6, v1
+; VI-NEXT:    v_add_u32_e32 v6, vcc, v10, v0
 ; VI-NEXT:    v_addc_u32_e64 v7, s[0:1], 0, 0, vcc
 ; VI-NEXT:    v_mad_u64_u32 v[0:1], s[0:1], v1, v5, v[6:7]
-; VI-NEXT:    v_add_u32_e32 v5, vcc, v10, v11
+; VI-NEXT:    v_add_u32_e32 v5, vcc, v11, v13
 ; VI-NEXT:    v_add_u32_e32 v4, vcc, v0, v12
 ; VI-NEXT:    v_addc_u32_e32 v5, vcc, v1, v5, vcc
 ; VI-NEXT:    flat_store_dwordx4 v[8:9], v[2:5]
@@ -2483,18 +2483,18 @@ define amdgpu_kernel void @v_mul_i128(ptr addrspace(1) %out, ptr addrspace(1) %a
 ; GFX9-NEXT:    v_mul_lo_u32 v15, v4, v3
 ; GFX9-NEXT:    v_mad_u64_u32 v[11:12], s[0:1], v1, v4, v[9:10]
 ; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[0:1], v4, v2, 0
-; GFX9-NEXT:    v_mov_b32_e32 v4, v12
+; GFX9-NEXT:    v_mul_lo_u32 v16, v7, v0
+; GFX9-NEXT:    v_mov_b32_e32 v7, v12
 ; GFX9-NEXT:    v_mov_b32_e32 v12, v10
 ; GFX9-NEXT:    v_mad_u64_u32 v[9:10], s[0:1], v0, v5, v[11:12]
 ; GFX9-NEXT:    v_add3_u32 v3, v3, v15, v14
-; GFX9-NEXT:    v_mul_lo_u32 v17, v7, v0
 ; GFX9-NEXT:    v_mad_u64_u32 v[2:3], s[0:1], v6, v0, v[2:3]
 ; GFX9-NEXT:    v_mov_b32_e32 v0, v10
-; GFX9-NEXT:    v_mul_lo_u32 v16, v6, v1
-; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, v4, v0
+; GFX9-NEXT:    v_mul_lo_u32 v4, v6, v1
+; GFX9-NEXT:    v_add_co_u32_e32 v6, vcc, v7, v0
 ; GFX9-NEXT:    v_addc_co_u32_e64 v7, s[0:1], 0, 0, vcc
 ; GFX9-NEXT:    v_mad_u64_u32 v[0:1], s[0:1], v1, v5, v[6:7]
-; GFX9-NEXT:    v_add3_u32 v3, v17, v3, v16
+; GFX9-NEXT:    v_add3_u32 v3, v16, v3, v4
 ; GFX9-NEXT:    v_add_co_u32_e32 v10, vcc, v0, v2
 ; GFX9-NEXT:    v_addc_co_u32_e32 v11, vcc, v1, v3, vcc
 ; GFX9-NEXT:    global_store_dwordx4 v13, v[8:11], s[2:3]
@@ -2503,72 +2503,69 @@ define amdgpu_kernel void @v_mul_i128(ptr addrspace(1) %out, ptr addrspace(1) %a
 ; GFX10-LABEL: v_mul_i128:
 ; GFX10:       ; %bb.0: ; %entry
 ; GFX10-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x2c
-; GFX10-NEXT:    v_lshlrev_b32_e32 v14, 4, v0
+; GFX10-NEXT:    v_lshlrev_b32_e32 v13, 4, v0
 ; GFX10-NEXT:    v_mov_b32_e32 v10, 0
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_clause 0x1
-; GFX10-NEXT:    global_load_dwordx4 v[0:3], v14, s[0:1]
-; GFX10-NEXT:    global_load_dwordx4 v[4:7], v14, s[2:3]
+; GFX10-NEXT:    global_load_dwordx4 v[0:3], v13, s[0:1]
+; GFX10-NEXT:    global_load_dwordx4 v[4:7], v13, s[2:3]
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
 ; GFX10-NEXT:    v_mad_u64_u32 v[8:9], s0, v0, v4, 0
+; GFX10-NEXT:    v_mul_lo_u32 v15, v5, v2
 ; GFX10-NEXT:    v_mul_lo_u32 v7, v7, v0
 ; GFX10-NEXT:    v_mad_u64_u32 v[11:12], s0, v1, v4, v[9:10]
-; GFX10-NEXT:    v_mov_b32_e32 v9, v12
+; GFX10-NEXT:    v_mov_b32_e32 v14, v12
 ; GFX10-NEXT:    v_mov_b32_e32 v12, v10
-; GFX10-NEXT:    v_mul_lo_u32 v10, v5, v2
-; GFX10-NEXT:    v_mad_u64_u32 v[12:13], s0, v0, v5, v[11:12]
+; GFX10-NEXT:    v_mad_u64_u32 v[9:10], s0, v0, v5, v[11:12]
 ; GFX10-NEXT:    v_mul_lo_u32 v11, v4, v3
 ; GFX10-NEXT:    v_mad_u64_u32 v[2:3], s0, v4, v2, 0
-; GFX10-NEXT:    v_mov_b32_e32 v4, v13
-; GFX10-NEXT:    v_mul_lo_u32 v13, v6, v1
-; GFX10-NEXT:    v_add3_u32 v3, v3, v11, v10
-; GFX10-NEXT:    v_add_co_u32 v9, s0, v9, v4
-; GFX10-NEXT:    v_add_co_ci_u32_e64 v10, s0, 0, 0, s0
+; GFX10-NEXT:    v_mul_lo_u32 v12, v6, v1
+; GFX10-NEXT:    v_mov_b32_e32 v4, v10
+; GFX10-NEXT:    v_add3_u32 v3, v3, v11, v15
+; GFX10-NEXT:    v_add_co_u32 v10, s0, v14, v4
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v11, s0, 0, 0, s0
 ; GFX10-NEXT:    v_mad_u64_u32 v[2:3], s0, v6, v0, v[2:3]
-; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s0, v1, v5, v[9:10]
-; GFX10-NEXT:    v_mov_b32_e32 v9, v12
-; GFX10-NEXT:    v_add3_u32 v3, v7, v3, v13
+; GFX10-NEXT:    v_mad_u64_u32 v[0:1], s0, v1, v5, v[10:11]
+; GFX10-NEXT:    v_add3_u32 v3, v7, v3, v12
 ; GFX10-NEXT:    v_add_co_u32 v10, vcc_lo, v0, v2
 ; GFX10-NEXT:    v_add_co_ci_u32_e32 v11, vcc_lo, v1, v3, vcc_lo
-; GFX10-NEXT:    global_store_dwordx4 v14, v[8:11], s[2:3]
+; GFX10-NEXT:    global_store_dwordx4 v13, v[8:11], s[2:3]
 ; GFX10-NEXT:    s_endpgm
 ;
 ; GFX11-LABEL: v_mul_i128:
 ; GFX11:       ; %bb.0: ; %entry
 ; GFX11-NEXT:    s_load_b128 s[0:3], s[0:1], 0x2c
-; GFX11-NEXT:    v_lshlrev_b32_e32 v16, 4, v0
-; GFX11-NEXT:    v_mov_b32_e32 v10, 0
+; GFX11-NEXT:    v_dual_mov_b32 v10, 0 :: v_dual_lshlrev_b32 v15, 4, v0
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-NEXT:    s_clause 0x1
-; GFX11-NEXT:    global_load_b128 v[0:3], v16, s[0:1]
-; GFX11-NEXT:    global_load_b128 v[4:7], v16, s[2:3]
+; GFX11-NEXT:    global_load_b128 v[0:3], v15, s[0:1]
+; GFX11-NEXT:    global_load_b128 v[4:7], v15, s[2:3]
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    v_mad_u64_u32 v[8:9], null, v0, v4, 0
-; GFX11-NEXT:    v_mul_lo_u32 v15, v5, v2
+; GFX11-NEXT:    v_mul_lo_u32 v14, v5, v2
 ; GFX11-NEXT:    v_mul_lo_u32 v3, v4, v3
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
 ; GFX11-NEXT:    v_mad_u64_u32 v[11:12], null, v1, v4, v[9:10]
-; GFX11-NEXT:    v_dual_mov_b32 v9, v12 :: v_dual_mov_b32 v12, v10
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
-; GFX11-NEXT:    v_mad_u64_u32 v[13:14], null, v0, v5, v[11:12]
-; GFX11-NEXT:    v_mad_u64_u32 v[10:11], null, v4, v2, 0
+; GFX11-NEXT:    v_dual_mov_b32 v13, v12 :: v_dual_mov_b32 v12, v10
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_mad_u64_u32 v[9:10], null, v0, v5, v[11:12]
+; GFX11-NEXT:    v_mad_u64_u32 v[11:12], null, v4, v2, 0
 ; GFX11-NEXT:    v_mul_lo_u32 v4, v6, v1
-; GFX11-NEXT:    v_mul_lo_u32 v12, v7, v0
-; GFX11-NEXT:    v_mov_b32_e32 v2, v14
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_add3_u32 v11, v11, v3, v15
-; GFX11-NEXT:    v_add_co_u32 v2, s0, v9, v2
-; GFX11-NEXT:    v_mov_b32_e32 v9, v13
+; GFX11-NEXT:    v_mov_b32_e32 v2, v10
+; GFX11-NEXT:    v_mul_lo_u32 v10, v7, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-NEXT:    v_add3_u32 v12, v12, v3, v14
+; GFX11-NEXT:    v_add_co_u32 v2, s0, v13, v2
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
 ; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, null, 0, 0, s0
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_mad_u64_u32 v[14:15], null, v6, v0, v[10:11]
+; GFX11-NEXT:    v_mad_u64_u32 v[13:14], null, v6, v0, v[11:12]
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
 ; GFX11-NEXT:    v_mad_u64_u32 v[6:7], null, v1, v5, v[2:3]
+; GFX11-NEXT:    v_add3_u32 v0, v10, v14, v4
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_add3_u32 v0, v12, v15, v4
-; GFX11-NEXT:    v_add_co_u32 v10, vcc_lo, v6, v14
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT:    v_add_co_u32 v10, vcc_lo, v6, v13
 ; GFX11-NEXT:    v_add_co_ci_u32_e32 v11, vcc_lo, v7, v0, vcc_lo
-; GFX11-NEXT:    global_store_b128 v16, v[8:11], s[2:3]
+; GFX11-NEXT:    global_store_b128 v15, v[8:11], s[2:3]
 ; GFX11-NEXT:    s_nop 0
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm.ll b/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm.ll
index 85cd00cbfc5367f0ca2880ad6f371441e2f18b89..8081d40f7e665c365e3516a9c3b2e9f6c3038329 100644
--- a/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm.ll
+++ b/llvm/test/CodeGen/AMDGPU/promote-constOffset-to-imm.ll
@@ -363,96 +363,96 @@ define hidden amdgpu_kernel void @clmem_read(ptr addrspace(1)  %buffer) {
 ; GFX8-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX8-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX8-NEXT:    v_lshlrev_b32_e32 v1, 17, v0
-; GFX8-NEXT:    v_mov_b32_e32 v2, 3
-; GFX8-NEXT:    v_and_b32_e32 v1, 0xfe000000, v1
-; GFX8-NEXT:    v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX8-NEXT:    v_or_b32_e32 v0, v1, v0
-; GFX8-NEXT:    v_mov_b32_e32 v2, s35
+; GFX8-NEXT:    v_and_b32_e32 v6, 0xfe000000, v1
+; GFX8-NEXT:    v_mov_b32_e32 v1, 3
+; GFX8-NEXT:    v_lshlrev_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX8-NEXT:    v_or_b32_e32 v0, v6, v0
+; GFX8-NEXT:    v_mov_b32_e32 v1, s35
 ; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s34, v0
-; GFX8-NEXT:    v_addc_u32_e32 v3, vcc, 0, v2, vcc
+; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; GFX8-NEXT:    s_movk_i32 s0, 0x5000
-; GFX8-NEXT:    v_add_u32_e32 v2, vcc, s0, v0
-; GFX8-NEXT:    v_mov_b32_e32 v4, 0
-; GFX8-NEXT:    v_addc_u32_e32 v3, vcc, 0, v3, vcc
-; GFX8-NEXT:    v_mov_b32_e32 v5, 0
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s0, v0
+; GFX8-NEXT:    v_mov_b32_e32 v2, 0
+; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT:    v_mov_b32_e32 v3, 0
 ; GFX8-NEXT:    s_movk_i32 s0, 0x7f
 ; GFX8-NEXT:  .LBB1_1: ; %for.cond.preheader
 ; GFX8-NEXT:    ; =>This Loop Header: Depth=1
 ; GFX8-NEXT:    ; Child Loop BB1_2 Depth 2
-; GFX8-NEXT:    v_mov_b32_e32 v7, v3
-; GFX8-NEXT:    v_mov_b32_e32 v6, v2
+; GFX8-NEXT:    v_mov_b32_e32 v5, v1
+; GFX8-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX8-NEXT:    s_mov_b32 s1, 0
 ; GFX8-NEXT:  .LBB1_2: ; %for.body
 ; GFX8-NEXT:    ; Parent Loop BB1_1 Depth=1
 ; GFX8-NEXT:    ; => This Inner Loop Header: Depth=2
-; GFX8-NEXT:    v_add_u32_e32 v8, vcc, 0xffffb000, v6
-; GFX8-NEXT:    v_addc_u32_e32 v9, vcc, -1, v7, vcc
-; GFX8-NEXT:    v_add_u32_e32 v10, vcc, 0xffffb800, v6
-; GFX8-NEXT:    v_addc_u32_e32 v11, vcc, -1, v7, vcc
-; GFX8-NEXT:    v_add_u32_e32 v12, vcc, 0xffffc000, v6
-; GFX8-NEXT:    flat_load_dwordx2 v[8:9], v[8:9]
-; GFX8-NEXT:    flat_load_dwordx2 v[10:11], v[10:11]
-; GFX8-NEXT:    v_addc_u32_e32 v13, vcc, -1, v7, vcc
-; GFX8-NEXT:    v_add_u32_e32 v14, vcc, 0xffffc800, v6
-; GFX8-NEXT:    v_addc_u32_e32 v15, vcc, -1, v7, vcc
-; GFX8-NEXT:    v_add_u32_e32 v16, vcc, 0xffffd000, v6
-; GFX8-NEXT:    flat_load_dwordx2 v[12:13], v[12:13]
-; GFX8-NEXT:    flat_load_dwordx2 v[14:15], v[14:15]
-; GFX8-NEXT:    v_addc_u32_e32 v17, vcc, -1, v7, vcc
-; GFX8-NEXT:    v_add_u32_e32 v18, vcc, 0xffffd800, v6
-; GFX8-NEXT:    v_addc_u32_e32 v19, vcc, -1, v7, vcc
-; GFX8-NEXT:    flat_load_dwordx2 v[16:17], v[16:17]
-; GFX8-NEXT:    flat_load_dwordx2 v[18:19], v[18:19]
-; GFX8-NEXT:    v_add_u32_e32 v20, vcc, 0xffffe000, v6
-; GFX8-NEXT:    v_addc_u32_e32 v21, vcc, -1, v7, vcc
-; GFX8-NEXT:    v_add_u32_e32 v22, vcc, 0xffffe800, v6
-; GFX8-NEXT:    flat_load_dwordx2 v[20:21], v[20:21]
-; GFX8-NEXT:    v_addc_u32_e32 v23, vcc, -1, v7, vcc
-; GFX8-NEXT:    flat_load_dwordx2 v[22:23], v[22:23]
-; GFX8-NEXT:    v_add_u32_e32 v24, vcc, 0xfffff000, v6
-; GFX8-NEXT:    v_addc_u32_e32 v25, vcc, -1, v7, vcc
-; GFX8-NEXT:    flat_load_dwordx2 v[24:25], v[24:25]
-; GFX8-NEXT:    v_add_u32_e32 v26, vcc, 0xfffff800, v6
-; GFX8-NEXT:    v_addc_u32_e32 v27, vcc, -1, v7, vcc
-; GFX8-NEXT:    flat_load_dwordx2 v[26:27], v[26:27]
-; GFX8-NEXT:    flat_load_dwordx2 v[28:29], v[6:7]
-; GFX8-NEXT:    v_add_u32_e32 v6, vcc, 0x10000, v6
-; GFX8-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; GFX8-NEXT:    v_add_u32_e32 v7, vcc, 0xffffb000, v4
+; GFX8-NEXT:    v_addc_u32_e32 v8, vcc, -1, v5, vcc
+; GFX8-NEXT:    v_add_u32_e32 v9, vcc, 0xffffb800, v4
+; GFX8-NEXT:    v_addc_u32_e32 v10, vcc, -1, v5, vcc
+; GFX8-NEXT:    v_add_u32_e32 v11, vcc, 0xffffc000, v4
+; GFX8-NEXT:    flat_load_dwordx2 v[7:8], v[7:8]
+; GFX8-NEXT:    flat_load_dwordx2 v[9:10], v[9:10]
+; GFX8-NEXT:    v_addc_u32_e32 v12, vcc, -1, v5, vcc
+; GFX8-NEXT:    v_add_u32_e32 v13, vcc, 0xffffc800, v4
+; GFX8-NEXT:    v_addc_u32_e32 v14, vcc, -1, v5, vcc
+; GFX8-NEXT:    v_add_u32_e32 v15, vcc, 0xffffd000, v4
+; GFX8-NEXT:    flat_load_dwordx2 v[11:12], v[11:12]
+; GFX8-NEXT:    flat_load_dwordx2 v[13:14], v[13:14]
+; GFX8-NEXT:    v_addc_u32_e32 v16, vcc, -1, v5, vcc
+; GFX8-NEXT:    v_add_u32_e32 v17, vcc, 0xffffd800, v4
+; GFX8-NEXT:    v_addc_u32_e32 v18, vcc, -1, v5, vcc
+; GFX8-NEXT:    flat_load_dwordx2 v[15:16], v[15:16]
+; GFX8-NEXT:    flat_load_dwordx2 v[17:18], v[17:18]
+; GFX8-NEXT:    v_add_u32_e32 v19, vcc, 0xffffe000, v4
+; GFX8-NEXT:    v_addc_u32_e32 v20, vcc, -1, v5, vcc
+; GFX8-NEXT:    v_add_u32_e32 v21, vcc, 0xffffe800, v4
+; GFX8-NEXT:    flat_load_dwordx2 v[19:20], v[19:20]
+; GFX8-NEXT:    v_addc_u32_e32 v22, vcc, -1, v5, vcc
+; GFX8-NEXT:    flat_load_dwordx2 v[21:22], v[21:22]
+; GFX8-NEXT:    v_add_u32_e32 v23, vcc, 0xfffff000, v4
+; GFX8-NEXT:    v_addc_u32_e32 v24, vcc, -1, v5, vcc
+; GFX8-NEXT:    flat_load_dwordx2 v[23:24], v[23:24]
+; GFX8-NEXT:    v_add_u32_e32 v25, vcc, 0xfffff800, v4
+; GFX8-NEXT:    v_addc_u32_e32 v26, vcc, -1, v5, vcc
+; GFX8-NEXT:    flat_load_dwordx2 v[25:26], v[25:26]
+; GFX8-NEXT:    flat_load_dwordx2 v[27:28], v[4:5]
+; GFX8-NEXT:    v_add_u32_e32 v4, vcc, 0x10000, v4
+; GFX8-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
 ; GFX8-NEXT:    s_addk_i32 s1, 0x2000
 ; GFX8-NEXT:    s_cmp_gt_u32 s1, 0x3fffff
 ; GFX8-NEXT:    s_waitcnt vmcnt(10)
-; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v8, v4
-; GFX8-NEXT:    v_addc_u32_e32 v4, vcc, v9, v5, vcc
+; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v7, v2
+; GFX8-NEXT:    v_addc_u32_e32 v3, vcc, v8, v3, vcc
 ; GFX8-NEXT:    s_waitcnt vmcnt(9)
-; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v10, v0
-; GFX8-NEXT:    v_addc_u32_e32 v4, vcc, v11, v4, vcc
+; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v9, v2
+; GFX8-NEXT:    v_addc_u32_e32 v3, vcc, v10, v3, vcc
 ; GFX8-NEXT:    s_waitcnt vmcnt(8)
-; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v12, v0
-; GFX8-NEXT:    v_addc_u32_e32 v4, vcc, v13, v4, vcc
+; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v11, v2
+; GFX8-NEXT:    v_addc_u32_e32 v3, vcc, v12, v3, vcc
 ; GFX8-NEXT:    s_waitcnt vmcnt(7)
-; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v14, v0
-; GFX8-NEXT:    v_addc_u32_e32 v4, vcc, v15, v4, vcc
+; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v13, v2
+; GFX8-NEXT:    v_addc_u32_e32 v3, vcc, v14, v3, vcc
 ; GFX8-NEXT:    s_waitcnt vmcnt(6)
-; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v16, v0
-; GFX8-NEXT:    v_addc_u32_e32 v4, vcc, v17, v4, vcc
+; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v15, v2
+; GFX8-NEXT:    v_addc_u32_e32 v3, vcc, v16, v3, vcc
 ; GFX8-NEXT:    s_waitcnt vmcnt(5)
-; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v18, v0
-; GFX8-NEXT:    v_addc_u32_e32 v4, vcc, v19, v4, vcc
+; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v17, v2
+; GFX8-NEXT:    v_addc_u32_e32 v3, vcc, v18, v3, vcc
 ; GFX8-NEXT:    s_waitcnt vmcnt(4)
-; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v20, v0
-; GFX8-NEXT:    v_addc_u32_e32 v4, vcc, v21, v4, vcc
+; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v19, v2
+; GFX8-NEXT:    v_addc_u32_e32 v3, vcc, v20, v3, vcc
 ; GFX8-NEXT:    s_waitcnt vmcnt(3)
-; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v22, v0
-; GFX8-NEXT:    v_addc_u32_e32 v4, vcc, v23, v4, vcc
+; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v21, v2
+; GFX8-NEXT:    v_addc_u32_e32 v3, vcc, v22, v3, vcc
 ; GFX8-NEXT:    s_waitcnt vmcnt(2)
-; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v24, v0
-; GFX8-NEXT:    v_addc_u32_e32 v4, vcc, v25, v4, vcc
+; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v23, v2
+; GFX8-NEXT:    v_addc_u32_e32 v3, vcc, v24, v3, vcc
 ; GFX8-NEXT:    s_waitcnt vmcnt(1)
-; GFX8-NEXT:    v_add_u32_e32 v0, vcc, v26, v0
-; GFX8-NEXT:    v_addc_u32_e32 v5, vcc, v27, v4, vcc
+; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v25, v2
+; GFX8-NEXT:    v_addc_u32_e32 v3, vcc, v26, v3, vcc
 ; GFX8-NEXT:    s_waitcnt vmcnt(0)
-; GFX8-NEXT:    v_add_u32_e32 v4, vcc, v28, v0
-; GFX8-NEXT:    v_addc_u32_e32 v5, vcc, v29, v5, vcc
+; GFX8-NEXT:    v_add_u32_e32 v2, vcc, v27, v2
+; GFX8-NEXT:    v_addc_u32_e32 v3, vcc, v28, v3, vcc
 ; GFX8-NEXT:    s_cbranch_scc0 .LBB1_2
 ; GFX8-NEXT:  ; %bb.3: ; %while.cond.loopexit
 ; GFX8-NEXT:    ; in Loop: Header=BB1_1 Depth=1
@@ -463,10 +463,10 @@ define hidden amdgpu_kernel void @clmem_read(ptr addrspace(1)  %buffer) {
 ; GFX8-NEXT:    s_mov_b32 s0, s1
 ; GFX8-NEXT:    s_branch .LBB1_1
 ; GFX8-NEXT:  .LBB1_5: ; %while.end
-; GFX8-NEXT:    v_mov_b32_e32 v2, s35
-; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s34, v1
-; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v2, vcc
-; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[4:5]
+; GFX8-NEXT:    v_mov_b32_e32 v1, s35
+; GFX8-NEXT:    v_add_u32_e32 v0, vcc, s34, v6
+; GFX8-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; GFX8-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
 ; GFX8-NEXT:    s_endpgm
 ;
 ; GFX900-LABEL: clmem_read:
@@ -491,84 +491,84 @@ define hidden amdgpu_kernel void @clmem_read(ptr addrspace(1)  %buffer) {
 ; GFX900-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX900-NEXT:    v_and_b32_e32 v1, 0xff, v0
 ; GFX900-NEXT:    v_lshlrev_b32_e32 v0, 17, v0
-; GFX900-NEXT:    v_and_b32_e32 v0, 0xfe000000, v0
-; GFX900-NEXT:    v_lshl_or_b32 v1, v1, 3, v0
-; GFX900-NEXT:    v_mov_b32_e32 v2, s35
-; GFX900-NEXT:    v_add_co_u32_e32 v1, vcc, s34, v1
-; GFX900-NEXT:    v_addc_co_u32_e32 v2, vcc, 0, v2, vcc
+; GFX900-NEXT:    v_and_b32_e32 v6, 0xfe000000, v0
+; GFX900-NEXT:    v_lshl_or_b32 v0, v1, 3, v6
+; GFX900-NEXT:    v_mov_b32_e32 v1, s35
+; GFX900-NEXT:    v_add_co_u32_e32 v0, vcc, s34, v0
+; GFX900-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
 ; GFX900-NEXT:    s_movk_i32 s0, 0x5000
-; GFX900-NEXT:    v_add_co_u32_e32 v1, vcc, s0, v1
-; GFX900-NEXT:    v_mov_b32_e32 v3, 0
-; GFX900-NEXT:    v_addc_co_u32_e32 v2, vcc, 0, v2, vcc
+; GFX900-NEXT:    v_add_co_u32_e32 v0, vcc, s0, v0
+; GFX900-NEXT:    v_mov_b32_e32 v2, 0
+; GFX900-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
 ; GFX900-NEXT:    s_movk_i32 s2, 0x7f
-; GFX900-NEXT:    v_mov_b32_e32 v4, 0
+; GFX900-NEXT:    v_mov_b32_e32 v3, 0
 ; GFX900-NEXT:    s_movk_i32 s0, 0xd000
 ; GFX900-NEXT:    s_movk_i32 s1, 0xe000
 ; GFX900-NEXT:    s_movk_i32 s3, 0xf000
 ; GFX900-NEXT:  .LBB1_1: ; %for.cond.preheader
 ; GFX900-NEXT:    ; =>This Loop Header: Depth=1
 ; GFX900-NEXT:    ; Child Loop BB1_2 Depth 2
-; GFX900-NEXT:    v_mov_b32_e32 v6, v2
 ; GFX900-NEXT:    v_mov_b32_e32 v5, v1
+; GFX900-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX900-NEXT:    s_mov_b32 s4, 0
 ; GFX900-NEXT:  .LBB1_2: ; %for.body
 ; GFX900-NEXT:    ; Parent Loop BB1_1 Depth=1
 ; GFX900-NEXT:    ; => This Inner Loop Header: Depth=2
-; GFX900-NEXT:    v_add_co_u32_e32 v7, vcc, 0xffffb000, v5
-; GFX900-NEXT:    v_addc_co_u32_e32 v8, vcc, -1, v6, vcc
-; GFX900-NEXT:    global_load_dwordx2 v[9:10], v[5:6], off offset:-4096
-; GFX900-NEXT:    global_load_dwordx2 v[11:12], v[5:6], off offset:-2048
-; GFX900-NEXT:    v_add_co_u32_e32 v13, vcc, 0xffffc000, v5
+; GFX900-NEXT:    v_add_co_u32_e32 v7, vcc, 0xffffb000, v4
+; GFX900-NEXT:    v_addc_co_u32_e32 v8, vcc, -1, v5, vcc
+; GFX900-NEXT:    global_load_dwordx2 v[9:10], v[4:5], off offset:-4096
+; GFX900-NEXT:    global_load_dwordx2 v[11:12], v[4:5], off offset:-2048
+; GFX900-NEXT:    v_add_co_u32_e32 v13, vcc, 0xffffc000, v4
 ; GFX900-NEXT:    global_load_dwordx2 v[7:8], v[7:8], off
-; GFX900-NEXT:    v_addc_co_u32_e32 v14, vcc, -1, v6, vcc
+; GFX900-NEXT:    v_addc_co_u32_e32 v14, vcc, -1, v5, vcc
 ; GFX900-NEXT:    global_load_dwordx2 v[17:18], v[13:14], off offset:-2048
-; GFX900-NEXT:    v_add_co_u32_e32 v15, vcc, s0, v5
-; GFX900-NEXT:    v_addc_co_u32_e32 v16, vcc, -1, v6, vcc
+; GFX900-NEXT:    v_add_co_u32_e32 v15, vcc, s0, v4
+; GFX900-NEXT:    v_addc_co_u32_e32 v16, vcc, -1, v5, vcc
 ; GFX900-NEXT:    global_load_dwordx2 v[15:16], v[15:16], off offset:-2048
-; GFX900-NEXT:    v_add_co_u32_e32 v19, vcc, s1, v5
+; GFX900-NEXT:    v_add_co_u32_e32 v19, vcc, s1, v4
 ; GFX900-NEXT:    global_load_dwordx2 v[13:14], v[13:14], off
-; GFX900-NEXT:    v_addc_co_u32_e32 v20, vcc, -1, v6, vcc
+; GFX900-NEXT:    v_addc_co_u32_e32 v20, vcc, -1, v5, vcc
 ; GFX900-NEXT:    global_load_dwordx2 v[23:24], v[19:20], off offset:-4096
 ; GFX900-NEXT:    global_load_dwordx2 v[25:26], v[19:20], off offset:-2048
 ; GFX900-NEXT:    global_load_dwordx2 v[27:28], v[19:20], off
-; GFX900-NEXT:    v_add_co_u32_e32 v21, vcc, s3, v5
-; GFX900-NEXT:    v_addc_co_u32_e32 v22, vcc, -1, v6, vcc
+; GFX900-NEXT:    v_add_co_u32_e32 v21, vcc, s3, v4
+; GFX900-NEXT:    v_addc_co_u32_e32 v22, vcc, -1, v5, vcc
 ; GFX900-NEXT:    global_load_dwordx2 v[19:20], v[21:22], off offset:-2048
-; GFX900-NEXT:    global_load_dwordx2 v[29:30], v[5:6], off
-; GFX900-NEXT:    v_add_co_u32_e32 v5, vcc, 0x10000, v5
-; GFX900-NEXT:    v_addc_co_u32_e32 v6, vcc, 0, v6, vcc
+; GFX900-NEXT:    global_load_dwordx2 v[29:30], v[4:5], off
+; GFX900-NEXT:    v_add_co_u32_e32 v4, vcc, 0x10000, v4
+; GFX900-NEXT:    v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
 ; GFX900-NEXT:    s_addk_i32 s4, 0x2000
 ; GFX900-NEXT:    s_cmp_gt_u32 s4, 0x3fffff
 ; GFX900-NEXT:    s_waitcnt vmcnt(8)
-; GFX900-NEXT:    v_add_co_u32_e32 v3, vcc, v7, v3
-; GFX900-NEXT:    v_addc_co_u32_e32 v4, vcc, v8, v4, vcc
+; GFX900-NEXT:    v_add_co_u32_e32 v2, vcc, v7, v2
+; GFX900-NEXT:    v_addc_co_u32_e32 v3, vcc, v8, v3, vcc
 ; GFX900-NEXT:    s_waitcnt vmcnt(7)
-; GFX900-NEXT:    v_add_co_u32_e32 v3, vcc, v17, v3
-; GFX900-NEXT:    v_addc_co_u32_e32 v4, vcc, v18, v4, vcc
+; GFX900-NEXT:    v_add_co_u32_e32 v2, vcc, v17, v2
+; GFX900-NEXT:    v_addc_co_u32_e32 v3, vcc, v18, v3, vcc
 ; GFX900-NEXT:    s_waitcnt vmcnt(5)
-; GFX900-NEXT:    v_add_co_u32_e32 v3, vcc, v13, v3
-; GFX900-NEXT:    v_addc_co_u32_e32 v4, vcc, v14, v4, vcc
-; GFX900-NEXT:    v_add_co_u32_e32 v3, vcc, v15, v3
-; GFX900-NEXT:    v_addc_co_u32_e32 v4, vcc, v16, v4, vcc
+; GFX900-NEXT:    v_add_co_u32_e32 v2, vcc, v13, v2
+; GFX900-NEXT:    v_addc_co_u32_e32 v3, vcc, v14, v3, vcc
+; GFX900-NEXT:    v_add_co_u32_e32 v2, vcc, v15, v2
+; GFX900-NEXT:    v_addc_co_u32_e32 v3, vcc, v16, v3, vcc
 ; GFX900-NEXT:    s_waitcnt vmcnt(4)
-; GFX900-NEXT:    v_add_co_u32_e32 v3, vcc, v23, v3
-; GFX900-NEXT:    v_addc_co_u32_e32 v4, vcc, v24, v4, vcc
+; GFX900-NEXT:    v_add_co_u32_e32 v2, vcc, v23, v2
+; GFX900-NEXT:    v_addc_co_u32_e32 v3, vcc, v24, v3, vcc
 ; GFX900-NEXT:    s_waitcnt vmcnt(3)
-; GFX900-NEXT:    v_add_co_u32_e32 v3, vcc, v25, v3
-; GFX900-NEXT:    v_addc_co_u32_e32 v4, vcc, v26, v4, vcc
+; GFX900-NEXT:    v_add_co_u32_e32 v2, vcc, v25, v2
+; GFX900-NEXT:    v_addc_co_u32_e32 v3, vcc, v26, v3, vcc
 ; GFX900-NEXT:    s_waitcnt vmcnt(2)
-; GFX900-NEXT:    v_add_co_u32_e32 v3, vcc, v27, v3
-; GFX900-NEXT:    v_addc_co_u32_e32 v4, vcc, v28, v4, vcc
+; GFX900-NEXT:    v_add_co_u32_e32 v2, vcc, v27, v2
+; GFX900-NEXT:    v_addc_co_u32_e32 v3, vcc, v28, v3, vcc
 ; GFX900-NEXT:    s_waitcnt vmcnt(1)
-; GFX900-NEXT:    v_add_co_u32_e32 v3, vcc, v19, v3
-; GFX900-NEXT:    v_addc_co_u32_e32 v4, vcc, v20, v4, vcc
-; GFX900-NEXT:    v_add_co_u32_e32 v3, vcc, v9, v3
-; GFX900-NEXT:    v_addc_co_u32_e32 v4, vcc, v10, v4, vcc
-; GFX900-NEXT:    v_add_co_u32_e32 v3, vcc, v11, v3
-; GFX900-NEXT:    v_addc_co_u32_e32 v4, vcc, v12, v4, vcc
+; GFX900-NEXT:    v_add_co_u32_e32 v2, vcc, v19, v2
+; GFX900-NEXT:    v_addc_co_u32_e32 v3, vcc, v20, v3, vcc
+; GFX900-NEXT:    v_add_co_u32_e32 v2, vcc, v9, v2
+; GFX900-NEXT:    v_addc_co_u32_e32 v3, vcc, v10, v3, vcc
+; GFX900-NEXT:    v_add_co_u32_e32 v2, vcc, v11, v2
+; GFX900-NEXT:    v_addc_co_u32_e32 v3, vcc, v12, v3, vcc
 ; GFX900-NEXT:    s_waitcnt vmcnt(0)
-; GFX900-NEXT:    v_add_co_u32_e32 v3, vcc, v29, v3
-; GFX900-NEXT:    v_addc_co_u32_e32 v4, vcc, v30, v4, vcc
+; GFX900-NEXT:    v_add_co_u32_e32 v2, vcc, v29, v2
+; GFX900-NEXT:    v_addc_co_u32_e32 v3, vcc, v30, v3, vcc
 ; GFX900-NEXT:    s_cbranch_scc0 .LBB1_2
 ; GFX900-NEXT:  ; %bb.3: ; %while.cond.loopexit
 ; GFX900-NEXT:    ; in Loop: Header=BB1_1 Depth=1
@@ -580,9 +580,9 @@ define hidden amdgpu_kernel void @clmem_read(ptr addrspace(1)  %buffer) {
 ; GFX900-NEXT:    s_branch .LBB1_1
 ; GFX900-NEXT:  .LBB1_5: ; %while.end
 ; GFX900-NEXT:    v_mov_b32_e32 v1, s35
-; GFX900-NEXT:    v_add_co_u32_e32 v0, vcc, s34, v0
+; GFX900-NEXT:    v_add_co_u32_e32 v0, vcc, s34, v6
 ; GFX900-NEXT:    v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
-; GFX900-NEXT:    global_store_dwordx2 v[0:1], v[3:4], off
+; GFX900-NEXT:    global_store_dwordx2 v[0:1], v[2:3], off
 ; GFX900-NEXT:    s_endpgm
 ;
 ; GFX10-LABEL: clmem_read:
@@ -606,39 +606,39 @@ define hidden amdgpu_kernel void @clmem_read(ptr addrspace(1)  %buffer) {
 ; GFX10-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX10-NEXT:    s_swappc_b64 s[30:31], s[4:5]
 ; GFX10-NEXT:    v_lshlrev_b32_e32 v1, 17, v0
-; GFX10-NEXT:    v_and_b32_e32 v2, 0xff, v0
+; GFX10-NEXT:    v_and_b32_e32 v0, 0xff, v0
+; GFX10-NEXT:    v_mov_b32_e32 v2, 0
 ; GFX10-NEXT:    v_mov_b32_e32 v3, 0
-; GFX10-NEXT:    v_mov_b32_e32 v4, 0
 ; GFX10-NEXT:    s_movk_i32 s1, 0x7f
-; GFX10-NEXT:    v_and_b32_e32 v0, 0xfe000000, v1
-; GFX10-NEXT:    v_lshl_or_b32 v1, v2, 3, v0
-; GFX10-NEXT:    v_add_co_u32 v1, s0, v1, s34
-; GFX10-NEXT:    v_add_co_ci_u32_e64 v2, s0, 0, s35, s0
-; GFX10-NEXT:    v_add_co_u32 v1, vcc_lo, 0x5000, v1
-; GFX10-NEXT:    v_add_co_ci_u32_e32 v2, vcc_lo, 0, v2, vcc_lo
+; GFX10-NEXT:    v_and_b32_e32 v6, 0xfe000000, v1
+; GFX10-NEXT:    v_lshl_or_b32 v0, v0, 3, v6
+; GFX10-NEXT:    v_add_co_u32 v0, s0, v0, s34
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, 0, s35, s0
+; GFX10-NEXT:    v_add_co_u32 v0, vcc_lo, 0x5000, v0
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX10-NEXT:  .LBB1_1: ; %for.cond.preheader
 ; GFX10-NEXT:    ; =>This Loop Header: Depth=1
 ; GFX10-NEXT:    ; Child Loop BB1_2 Depth 2
-; GFX10-NEXT:    v_mov_b32_e32 v6, v2
 ; GFX10-NEXT:    v_mov_b32_e32 v5, v1
+; GFX10-NEXT:    v_mov_b32_e32 v4, v0
 ; GFX10-NEXT:    s_mov_b32 s2, 0
 ; GFX10-NEXT:  .LBB1_2: ; %for.body
 ; GFX10-NEXT:    ; Parent Loop BB1_1 Depth=1
 ; GFX10-NEXT:    ; => This Inner Loop Header: Depth=2
-; GFX10-NEXT:    v_add_co_u32 v7, vcc_lo, v5, 0xffffb800
-; GFX10-NEXT:    v_add_co_ci_u32_e32 v8, vcc_lo, -1, v6, vcc_lo
-; GFX10-NEXT:    v_add_co_u32 v9, vcc_lo, v5, 0xffffc800
-; GFX10-NEXT:    v_add_co_ci_u32_e32 v10, vcc_lo, -1, v6, vcc_lo
-; GFX10-NEXT:    v_add_co_u32 v13, vcc_lo, v5, 0xffffd800
-; GFX10-NEXT:    v_add_co_ci_u32_e32 v14, vcc_lo, -1, v6, vcc_lo
-; GFX10-NEXT:    v_add_co_u32 v17, vcc_lo, v5, 0xffffe800
+; GFX10-NEXT:    v_add_co_u32 v7, vcc_lo, v4, 0xffffb800
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v8, vcc_lo, -1, v5, vcc_lo
+; GFX10-NEXT:    v_add_co_u32 v9, vcc_lo, v4, 0xffffc800
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v10, vcc_lo, -1, v5, vcc_lo
+; GFX10-NEXT:    v_add_co_u32 v13, vcc_lo, v4, 0xffffd800
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v14, vcc_lo, -1, v5, vcc_lo
+; GFX10-NEXT:    v_add_co_u32 v17, vcc_lo, v4, 0xffffe800
 ; GFX10-NEXT:    s_clause 0x2
 ; GFX10-NEXT:    global_load_dwordx2 v[11:12], v[7:8], off offset:-2048
 ; GFX10-NEXT:    global_load_dwordx2 v[15:16], v[9:10], off offset:-2048
 ; GFX10-NEXT:    global_load_dwordx2 v[19:20], v[13:14], off offset:-2048
-; GFX10-NEXT:    v_add_co_ci_u32_e32 v18, vcc_lo, -1, v6, vcc_lo
-; GFX10-NEXT:    v_add_co_u32 v21, vcc_lo, 0xfffff000, v5
-; GFX10-NEXT:    v_add_co_ci_u32_e32 v22, vcc_lo, -1, v6, vcc_lo
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v18, vcc_lo, -1, v5, vcc_lo
+; GFX10-NEXT:    v_add_co_u32 v21, vcc_lo, 0xfffff000, v4
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v22, vcc_lo, -1, v5, vcc_lo
 ; GFX10-NEXT:    s_clause 0x7
 ; GFX10-NEXT:    global_load_dwordx2 v[23:24], v[17:18], off offset:-2048
 ; GFX10-NEXT:    global_load_dwordx2 v[7:8], v[7:8], off
@@ -646,42 +646,42 @@ define hidden amdgpu_kernel void @clmem_read(ptr addrspace(1)  %buffer) {
 ; GFX10-NEXT:    global_load_dwordx2 v[13:14], v[13:14], off
 ; GFX10-NEXT:    global_load_dwordx2 v[25:26], v[17:18], off
 ; GFX10-NEXT:    global_load_dwordx2 v[27:28], v[21:22], off
-; GFX10-NEXT:    global_load_dwordx2 v[29:30], v[5:6], off offset:-2048
-; GFX10-NEXT:    global_load_dwordx2 v[31:32], v[5:6], off
-; GFX10-NEXT:    v_add_co_u32 v5, vcc_lo, 0x10000, v5
-; GFX10-NEXT:    v_add_co_ci_u32_e32 v6, vcc_lo, 0, v6, vcc_lo
+; GFX10-NEXT:    global_load_dwordx2 v[29:30], v[4:5], off offset:-2048
+; GFX10-NEXT:    global_load_dwordx2 v[31:32], v[4:5], off
+; GFX10-NEXT:    v_add_co_u32 v4, vcc_lo, 0x10000, v4
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v5, vcc_lo, 0, v5, vcc_lo
 ; GFX10-NEXT:    s_addk_i32 s2, 0x2000
 ; GFX10-NEXT:    s_cmp_gt_u32 s2, 0x3fffff
 ; GFX10-NEXT:    s_waitcnt vmcnt(10)
-; GFX10-NEXT:    v_add_co_u32 v3, s0, v11, v3
-; GFX10-NEXT:    v_add_co_ci_u32_e64 v4, s0, v12, v4, s0
+; GFX10-NEXT:    v_add_co_u32 v2, s0, v11, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v3, s0, v12, v3, s0
 ; GFX10-NEXT:    s_waitcnt vmcnt(6)
-; GFX10-NEXT:    v_add_co_u32 v3, s0, v7, v3
-; GFX10-NEXT:    v_add_co_ci_u32_e64 v4, s0, v8, v4, s0
-; GFX10-NEXT:    v_add_co_u32 v3, s0, v15, v3
-; GFX10-NEXT:    v_add_co_ci_u32_e64 v4, s0, v16, v4, s0
+; GFX10-NEXT:    v_add_co_u32 v2, s0, v7, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v3, s0, v8, v3, s0
+; GFX10-NEXT:    v_add_co_u32 v2, s0, v15, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v3, s0, v16, v3, s0
 ; GFX10-NEXT:    s_waitcnt vmcnt(5)
-; GFX10-NEXT:    v_add_co_u32 v3, s0, v9, v3
-; GFX10-NEXT:    v_add_co_ci_u32_e64 v4, s0, v10, v4, s0
-; GFX10-NEXT:    v_add_co_u32 v3, s0, v19, v3
-; GFX10-NEXT:    v_add_co_ci_u32_e64 v4, s0, v20, v4, s0
+; GFX10-NEXT:    v_add_co_u32 v2, s0, v9, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v3, s0, v10, v3, s0
+; GFX10-NEXT:    v_add_co_u32 v2, s0, v19, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v3, s0, v20, v3, s0
 ; GFX10-NEXT:    s_waitcnt vmcnt(4)
-; GFX10-NEXT:    v_add_co_u32 v3, s0, v13, v3
-; GFX10-NEXT:    v_add_co_ci_u32_e64 v4, s0, v14, v4, s0
-; GFX10-NEXT:    v_add_co_u32 v3, s0, v23, v3
-; GFX10-NEXT:    v_add_co_ci_u32_e64 v4, s0, v24, v4, s0
+; GFX10-NEXT:    v_add_co_u32 v2, s0, v13, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v3, s0, v14, v3, s0
+; GFX10-NEXT:    v_add_co_u32 v2, s0, v23, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v3, s0, v24, v3, s0
 ; GFX10-NEXT:    s_waitcnt vmcnt(3)
-; GFX10-NEXT:    v_add_co_u32 v3, s0, v25, v3
-; GFX10-NEXT:    v_add_co_ci_u32_e64 v4, s0, v26, v4, s0
+; GFX10-NEXT:    v_add_co_u32 v2, s0, v25, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v3, s0, v26, v3, s0
 ; GFX10-NEXT:    s_waitcnt vmcnt(2)
-; GFX10-NEXT:    v_add_co_u32 v3, s0, v27, v3
-; GFX10-NEXT:    v_add_co_ci_u32_e64 v4, s0, v28, v4, s0
+; GFX10-NEXT:    v_add_co_u32 v2, s0, v27, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v3, s0, v28, v3, s0
 ; GFX10-NEXT:    s_waitcnt vmcnt(1)
-; GFX10-NEXT:    v_add_co_u32 v3, s0, v29, v3
-; GFX10-NEXT:    v_add_co_ci_u32_e64 v4, s0, v30, v4, s0
+; GFX10-NEXT:    v_add_co_u32 v2, s0, v29, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e64 v3, s0, v30, v3, s0
 ; GFX10-NEXT:    s_waitcnt vmcnt(0)
-; GFX10-NEXT:    v_add_co_u32 v3, vcc_lo, v31, v3
-; GFX10-NEXT:    v_add_co_ci_u32_e32 v4, vcc_lo, v32, v4, vcc_lo
+; GFX10-NEXT:    v_add_co_u32 v2, vcc_lo, v31, v2
+; GFX10-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, v32, v3, vcc_lo
 ; GFX10-NEXT:    s_cbranch_scc0 .LBB1_2
 ; GFX10-NEXT:  ; %bb.3: ; %while.cond.loopexit
 ; GFX10-NEXT:    ; in Loop: Header=BB1_1 Depth=1
@@ -692,9 +692,9 @@ define hidden amdgpu_kernel void @clmem_read(ptr addrspace(1)  %buffer) {
 ; GFX10-NEXT:    s_mov_b32 s1, s0
 ; GFX10-NEXT:    s_branch .LBB1_1
 ; GFX10-NEXT:  .LBB1_5: ; %while.end
-; GFX10-NEXT:    v_add_co_u32 v0, s0, s34, v0
+; GFX10-NEXT:    v_add_co_u32 v0, s0, s34, v6
 ; GFX10-NEXT:    v_add_co_ci_u32_e64 v1, s0, s35, 0, s0
-; GFX10-NEXT:    global_store_dwordx2 v[0:1], v[3:4], off
+; GFX10-NEXT:    global_store_dwordx2 v[0:1], v[2:3], off
 ; GFX10-NEXT:    s_endpgm
 ;
 ; GFX90A-LABEL: clmem_read:
@@ -822,101 +822,100 @@ define hidden amdgpu_kernel void @clmem_read(ptr addrspace(1)  %buffer) {
 ; GFX11-NEXT:    s_mov_b32 s32, 0
 ; GFX11-NEXT:    s_waitcnt lgkmcnt(0)
 ; GFX11-NEXT:    s_swappc_b64 s[30:31], s[2:3]
-; GFX11-NEXT:    v_lshlrev_b32_e32 v1, 17, v0
-; GFX11-NEXT:    v_dual_mov_b32 v3, 0 :: v_dual_and_b32 v2, 0xff, v0
-; GFX11-NEXT:    v_mov_b32_e32 v4, 0
+; GFX11-NEXT:    v_dual_mov_b32 v2, 0 :: v_dual_lshlrev_b32 v1, 17, v0
+; GFX11-NEXT:    v_dual_mov_b32 v3, 0 :: v_dual_and_b32 v0, 0xff, v0
 ; GFX11-NEXT:    s_movk_i32 s1, 0x7f
-; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_and_b32_e32 v0, 0xfe000000, v1
-; GFX11-NEXT:    v_lshl_or_b32 v1, v2, 3, v0
+; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT:    v_and_b32_e32 v6, 0xfe000000, v1
+; GFX11-NEXT:    v_lshl_or_b32 v0, v0, 3, v6
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_u32 v1, s0, v1, s34
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v2, null, 0, s35, s0
+; GFX11-NEXT:    v_add_co_u32 v0, s0, v0, s34
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, 0, s35, s0
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_add_co_u32 v1, vcc_lo, 0x5000, v1
-; GFX11-NEXT:    v_add_co_ci_u32_e32 v2, vcc_lo, 0, v2, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v0, vcc_lo, 0x5000, v0
+; GFX11-NEXT:    v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo
 ; GFX11-NEXT:  .LBB1_1: ; %for.cond.preheader
 ; GFX11-NEXT:    ; =>This Loop Header: Depth=1
 ; GFX11-NEXT:    ; Child Loop BB1_2 Depth 2
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT:    v_dual_mov_b32 v6, v2 :: v_dual_mov_b32 v5, v1
+; GFX11-NEXT:    v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v4, v0
 ; GFX11-NEXT:    s_mov_b32 s2, 0
 ; GFX11-NEXT:  .LBB1_2: ; %for.body
 ; GFX11-NEXT:    ; Parent Loop BB1_1 Depth=1
 ; GFX11-NEXT:    ; => This Inner Loop Header: Depth=2
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_add_co_u32 v7, vcc_lo, v5, 0xffffc000
-; GFX11-NEXT:    v_add_co_ci_u32_e32 v8, vcc_lo, -1, v6, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v9, vcc_lo, 0xffffc000, v5
-; GFX11-NEXT:    v_add_co_ci_u32_e32 v10, vcc_lo, -1, v6, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v11, vcc_lo, 0xffffd000, v5
+; GFX11-NEXT:    v_add_co_u32 v7, vcc_lo, v4, 0xffffc000
+; GFX11-NEXT:    v_add_co_ci_u32_e32 v8, vcc_lo, -1, v5, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v9, vcc_lo, 0xffffc000, v4
+; GFX11-NEXT:    v_add_co_ci_u32_e32 v10, vcc_lo, -1, v5, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v11, vcc_lo, 0xffffd000, v4
 ; GFX11-NEXT:    s_clause 0x1
 ; GFX11-NEXT:    global_load_b64 v[13:14], v[7:8], off offset:-4096
 ; GFX11-NEXT:    global_load_b64 v[9:10], v[9:10], off offset:-2048
-; GFX11-NEXT:    v_add_co_ci_u32_e32 v12, vcc_lo, -1, v6, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v15, vcc_lo, v5, 0xffffe000
-; GFX11-NEXT:    v_add_co_ci_u32_e32 v16, vcc_lo, -1, v6, vcc_lo
+; GFX11-NEXT:    v_add_co_ci_u32_e32 v12, vcc_lo, -1, v5, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v15, vcc_lo, v4, 0xffffe000
+; GFX11-NEXT:    v_add_co_ci_u32_e32 v16, vcc_lo, -1, v5, vcc_lo
 ; GFX11-NEXT:    global_load_b64 v[11:12], v[11:12], off offset:-2048
-; GFX11-NEXT:    v_add_co_u32 v17, vcc_lo, 0xffffe000, v5
+; GFX11-NEXT:    v_add_co_u32 v17, vcc_lo, 0xffffe000, v4
 ; GFX11-NEXT:    s_clause 0x1
 ; GFX11-NEXT:    global_load_b64 v[19:20], v[15:16], off offset:-4096
 ; GFX11-NEXT:    global_load_b64 v[7:8], v[7:8], off
-; GFX11-NEXT:    v_add_co_ci_u32_e32 v18, vcc_lo, -1, v6, vcc_lo
-; GFX11-NEXT:    v_add_co_u32 v21, vcc_lo, 0xfffff000, v5
-; GFX11-NEXT:    v_add_co_ci_u32_e32 v22, vcc_lo, -1, v6, vcc_lo
+; GFX11-NEXT:    v_add_co_ci_u32_e32 v18, vcc_lo, -1, v5, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v21, vcc_lo, 0xfffff000, v4
+; GFX11-NEXT:    v_add_co_ci_u32_e32 v22, vcc_lo, -1, v5, vcc_lo
 ; GFX11-NEXT:    s_clause 0x5
 ; GFX11-NEXT:    global_load_b64 v[17:18], v[17:18], off offset:-2048
 ; GFX11-NEXT:    global_load_b64 v[15:16], v[15:16], off
 ; GFX11-NEXT:    global_load_b64 v[21:22], v[21:22], off offset:-2048
-; GFX11-NEXT:    global_load_b64 v[23:24], v[5:6], off offset:-4096
-; GFX11-NEXT:    global_load_b64 v[25:26], v[5:6], off offset:-2048
-; GFX11-NEXT:    global_load_b64 v[27:28], v[5:6], off
-; GFX11-NEXT:    v_add_co_u32 v5, vcc_lo, 0x10000, v5
-; GFX11-NEXT:    v_add_co_ci_u32_e32 v6, vcc_lo, 0, v6, vcc_lo
+; GFX11-NEXT:    global_load_b64 v[23:24], v[4:5], off offset:-4096
+; GFX11-NEXT:    global_load_b64 v[25:26], v[4:5], off offset:-2048
+; GFX11-NEXT:    global_load_b64 v[27:28], v[4:5], off
+; GFX11-NEXT:    v_add_co_u32 v4, vcc_lo, 0x10000, v4
+; GFX11-NEXT:    v_add_co_ci_u32_e32 v5, vcc_lo, 0, v5, vcc_lo
 ; GFX11-NEXT:    s_addk_i32 s2, 0x2000
 ; GFX11-NEXT:    s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
 ; GFX11-NEXT:    s_cmp_gt_u32 s2, 0x3fffff
 ; GFX11-NEXT:    s_waitcnt vmcnt(10)
-; GFX11-NEXT:    v_add_co_u32 v3, s0, v13, v3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v4, s0, v14, v4, s0
+; GFX11-NEXT:    v_add_co_u32 v2, s0, v13, v2
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, s0, v14, v3, s0
 ; GFX11-NEXT:    s_waitcnt vmcnt(9)
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_u32 v3, s0, v9, v3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v4, s0, v10, v4, s0
+; GFX11-NEXT:    v_add_co_u32 v2, s0, v9, v2
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, s0, v10, v3, s0
 ; GFX11-NEXT:    s_waitcnt vmcnt(6)
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_u32 v3, s0, v7, v3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v4, s0, v8, v4, s0
+; GFX11-NEXT:    v_add_co_u32 v2, s0, v7, v2
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, s0, v8, v3, s0
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_u32 v3, s0, v11, v3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v4, s0, v12, v4, s0
+; GFX11-NEXT:    v_add_co_u32 v2, s0, v11, v2
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, s0, v12, v3, s0
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_u32 v3, s0, v19, v3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v4, s0, v20, v4, s0
+; GFX11-NEXT:    v_add_co_u32 v2, s0, v19, v2
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, s0, v20, v3, s0
 ; GFX11-NEXT:    s_waitcnt vmcnt(5)
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_u32 v3, s0, v17, v3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v4, s0, v18, v4, s0
+; GFX11-NEXT:    v_add_co_u32 v2, s0, v17, v2
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, s0, v18, v3, s0
 ; GFX11-NEXT:    s_waitcnt vmcnt(4)
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_u32 v3, s0, v15, v3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v4, s0, v16, v4, s0
+; GFX11-NEXT:    v_add_co_u32 v2, s0, v15, v2
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, s0, v16, v3, s0
 ; GFX11-NEXT:    s_waitcnt vmcnt(3)
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_u32 v3, s0, v21, v3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v4, s0, v22, v4, s0
+; GFX11-NEXT:    v_add_co_u32 v2, s0, v21, v2
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, s0, v22, v3, s0
 ; GFX11-NEXT:    s_waitcnt vmcnt(2)
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_u32 v3, s0, v23, v3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v4, s0, v24, v4, s0
+; GFX11-NEXT:    v_add_co_u32 v2, s0, v23, v2
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, s0, v24, v3, s0
 ; GFX11-NEXT:    s_waitcnt vmcnt(1)
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT:    v_add_co_u32 v3, s0, v25, v3
-; GFX11-NEXT:    v_add_co_ci_u32_e64 v4, s0, v26, v4, s0
+; GFX11-NEXT:    v_add_co_u32 v2, s0, v25, v2
+; GFX11-NEXT:    v_add_co_ci_u32_e64 v3, s0, v26, v3, s0
 ; GFX11-NEXT:    s_waitcnt vmcnt(0)
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT:    v_add_co_u32 v3, vcc_lo, v27, v3
-; GFX11-NEXT:    v_add_co_ci_u32_e32 v4, vcc_lo, v28, v4, vcc_lo
+; GFX11-NEXT:    v_add_co_u32 v2, vcc_lo, v27, v2
+; GFX11-NEXT:    v_add_co_ci_u32_e32 v3, vcc_lo, v28, v3, vcc_lo
 ; GFX11-NEXT:    s_cbranch_scc0 .LBB1_2
 ; GFX11-NEXT:  ; %bb.3: ; %while.cond.loopexit
 ; GFX11-NEXT:    ; in Loop: Header=BB1_1 Depth=1
@@ -927,10 +926,10 @@ define hidden amdgpu_kernel void @clmem_read(ptr addrspace(1)  %buffer) {
 ; GFX11-NEXT:    s_mov_b32 s1, s0
 ; GFX11-NEXT:    s_branch .LBB1_1
 ; GFX11-NEXT:  .LBB1_5: ; %while.end
-; GFX11-NEXT:    v_add_co_u32 v0, s0, s34, v0
+; GFX11-NEXT:    v_add_co_u32 v0, s0, s34, v6
 ; GFX11-NEXT:    s_delay_alu instid0(VALU_DEP_1)
 ; GFX11-NEXT:    v_add_co_ci_u32_e64 v1, null, s35, 0, s0
-; GFX11-NEXT:    global_store_b64 v[0:1], v[3:4], off
+; GFX11-NEXT:    global_store_b64 v[0:1], v[2:3], off
 ; GFX11-NEXT:    s_nop 0
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/sdiv64.ll b/llvm/test/CodeGen/AMDGPU/sdiv64.ll
index adc4c85aa60dc075220fce2e22e6e59a13f06d9c..c48370a9c6c75b79cb0e6be20b0297daf54279b4 100644
--- a/llvm/test/CodeGen/AMDGPU/sdiv64.ll
+++ b/llvm/test/CodeGen/AMDGPU/sdiv64.ll
@@ -164,33 +164,33 @@ define amdgpu_kernel void @s_test_sdiv(ptr addrspace(1) %out, i64 %x, i64 %y) {
 ; GCN-IR-NEXT:    s_flbit_i32_b32 s8, s12
 ; GCN-IR-NEXT:    s_add_i32 s8, s8, 32
 ; GCN-IR-NEXT:    s_flbit_i32_b32 s9, s13
-; GCN-IR-NEXT:    s_min_u32 s18, s8, s9
-; GCN-IR-NEXT:    s_sub_u32 s16, s14, s18
+; GCN-IR-NEXT:    s_min_u32 s20, s8, s9
+; GCN-IR-NEXT:    s_sub_u32 s16, s14, s20
 ; GCN-IR-NEXT:    s_subb_u32 s17, 0, 0
-; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[20:21], s[16:17], 63
+; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[18:19], s[16:17], 63
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[22:23], s[16:17], 63
-; GCN-IR-NEXT:    s_or_b64 s[20:21], s[10:11], s[20:21]
-; GCN-IR-NEXT:    s_and_b64 s[10:11], s[20:21], exec
+; GCN-IR-NEXT:    s_or_b64 s[18:19], s[10:11], s[18:19]
+; GCN-IR-NEXT:    s_and_b64 s[10:11], s[18:19], exec
 ; GCN-IR-NEXT:    s_cselect_b32 s11, 0, s13
 ; GCN-IR-NEXT:    s_cselect_b32 s10, 0, s12
-; GCN-IR-NEXT:    s_or_b64 s[20:21], s[20:21], s[22:23]
+; GCN-IR-NEXT:    s_or_b64 s[18:19], s[18:19], s[22:23]
 ; GCN-IR-NEXT:    s_mov_b64 s[8:9], 0
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[20:21]
+; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[18:19]
 ; GCN-IR-NEXT:    s_cbranch_vccz .LBB0_5
 ; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
-; GCN-IR-NEXT:    s_add_u32 s20, s16, 1
-; GCN-IR-NEXT:    s_addc_u32 s21, s17, 0
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[10:11], s[20:21], 0
+; GCN-IR-NEXT:    s_add_u32 s18, s16, 1
+; GCN-IR-NEXT:    s_addc_u32 s19, s17, 0
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[10:11], s[18:19], 0
 ; GCN-IR-NEXT:    s_sub_i32 s16, 63, s16
 ; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[10:11]
 ; GCN-IR-NEXT:    s_lshl_b64 s[10:11], s[12:13], s16
 ; GCN-IR-NEXT:    s_cbranch_vccz .LBB0_4
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
-; GCN-IR-NEXT:    s_lshr_b64 s[16:17], s[12:13], s20
-; GCN-IR-NEXT:    s_add_u32 s19, s6, -1
-; GCN-IR-NEXT:    s_addc_u32 s20, s7, -1
+; GCN-IR-NEXT:    s_lshr_b64 s[16:17], s[12:13], s18
+; GCN-IR-NEXT:    s_add_u32 s18, s6, -1
+; GCN-IR-NEXT:    s_addc_u32 s19, s7, -1
 ; GCN-IR-NEXT:    s_not_b64 s[8:9], s[14:15]
-; GCN-IR-NEXT:    s_add_u32 s12, s8, s18
+; GCN-IR-NEXT:    s_add_u32 s12, s8, s20
 ; GCN-IR-NEXT:    s_addc_u32 s13, s9, 0
 ; GCN-IR-NEXT:    s_mov_b64 s[14:15], 0
 ; GCN-IR-NEXT:    s_mov_b32 s9, 0
@@ -201,8 +201,8 @@ define amdgpu_kernel void @s_test_sdiv(ptr addrspace(1) %out, i64 %x, i64 %y) {
 ; GCN-IR-NEXT:    s_lshl_b64 s[10:11], s[10:11], 1
 ; GCN-IR-NEXT:    s_or_b64 s[16:17], s[16:17], s[8:9]
 ; GCN-IR-NEXT:    s_or_b64 s[10:11], s[14:15], s[10:11]
-; GCN-IR-NEXT:    s_sub_u32 s8, s19, s16
-; GCN-IR-NEXT:    s_subb_u32 s8, s20, s17
+; GCN-IR-NEXT:    s_sub_u32 s8, s18, s16
+; GCN-IR-NEXT:    s_subb_u32 s8, s19, s17
 ; GCN-IR-NEXT:    s_ashr_i32 s14, s8, 31
 ; GCN-IR-NEXT:    s_mov_b32 s15, s14
 ; GCN-IR-NEXT:    s_and_b32 s8, s14, 1
@@ -211,9 +211,9 @@ define amdgpu_kernel void @s_test_sdiv(ptr addrspace(1) %out, i64 %x, i64 %y) {
 ; GCN-IR-NEXT:    s_subb_u32 s17, s17, s15
 ; GCN-IR-NEXT:    s_add_u32 s12, s12, 1
 ; GCN-IR-NEXT:    s_addc_u32 s13, s13, 0
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[22:23], s[12:13], 0
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[20:21], s[12:13], 0
 ; GCN-IR-NEXT:    s_mov_b64 s[14:15], s[8:9]
-; GCN-IR-NEXT:    s_and_b64 vcc, exec, s[22:23]
+; GCN-IR-NEXT:    s_and_b64 vcc, exec, s[20:21]
 ; GCN-IR-NEXT:    s_cbranch_vccz .LBB0_3
 ; GCN-IR-NEXT:  .LBB0_4: ; %Flow7
 ; GCN-IR-NEXT:    s_lshl_b64 s[6:7], s[10:11], 1
@@ -362,85 +362,85 @@ define i64 @v_test_sdiv(i64 %x, i64 %y) {
 ; GCN-IR-LABEL: v_test_sdiv:
 ; GCN-IR:       ; %bb.0: ; %_udiv-special-cases
 ; GCN-IR-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-IR-NEXT:    v_ashrrev_i32_e32 v4, 31, v1
-; GCN-IR-NEXT:    v_xor_b32_e32 v0, v4, v0
-; GCN-IR-NEXT:    v_ashrrev_i32_e32 v5, 31, v3
-; GCN-IR-NEXT:    v_xor_b32_e32 v1, v4, v1
-; GCN-IR-NEXT:    v_sub_i32_e32 v10, vcc, v0, v4
-; GCN-IR-NEXT:    v_subb_u32_e32 v11, vcc, v1, v4, vcc
-; GCN-IR-NEXT:    v_xor_b32_e32 v0, v5, v2
-; GCN-IR-NEXT:    v_xor_b32_e32 v1, v5, v3
-; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, v0, v5
-; GCN-IR-NEXT:    v_subb_u32_e32 v1, vcc, v1, v5, vcc
+; GCN-IR-NEXT:    v_ashrrev_i32_e32 v12, 31, v1
+; GCN-IR-NEXT:    v_xor_b32_e32 v0, v12, v0
+; GCN-IR-NEXT:    v_ashrrev_i32_e32 v13, 31, v3
+; GCN-IR-NEXT:    v_xor_b32_e32 v1, v12, v1
+; GCN-IR-NEXT:    v_sub_i32_e32 v6, vcc, v0, v12
+; GCN-IR-NEXT:    v_subb_u32_e32 v7, vcc, v1, v12, vcc
+; GCN-IR-NEXT:    v_xor_b32_e32 v0, v13, v2
+; GCN-IR-NEXT:    v_xor_b32_e32 v1, v13, v3
+; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, v0, v13
+; GCN-IR-NEXT:    v_subb_u32_e32 v1, vcc, v1, v13, vcc
 ; GCN-IR-NEXT:    v_ffbh_u32_e32 v2, v0
 ; GCN-IR-NEXT:    v_add_i32_e64 v2, s[6:7], 32, v2
 ; GCN-IR-NEXT:    v_ffbh_u32_e32 v3, v1
-; GCN-IR-NEXT:    v_min_u32_e32 v12, v2, v3
-; GCN-IR-NEXT:    v_ffbh_u32_e32 v2, v10
+; GCN-IR-NEXT:    v_min_u32_e32 v10, v2, v3
+; GCN-IR-NEXT:    v_ffbh_u32_e32 v2, v6
 ; GCN-IR-NEXT:    v_add_i32_e64 v2, s[6:7], 32, v2
-; GCN-IR-NEXT:    v_ffbh_u32_e32 v3, v11
-; GCN-IR-NEXT:    v_min_u32_e32 v13, v2, v3
-; GCN-IR-NEXT:    v_sub_i32_e64 v2, s[6:7], v12, v13
+; GCN-IR-NEXT:    v_ffbh_u32_e32 v3, v7
+; GCN-IR-NEXT:    v_min_u32_e32 v11, v2, v3
+; GCN-IR-NEXT:    v_sub_i32_e64 v2, s[6:7], v10, v11
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[0:1]
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[4:5], 0, v[10:11]
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[4:5], 0, v[6:7]
 ; GCN-IR-NEXT:    v_subb_u32_e64 v3, s[6:7], 0, 0, s[6:7]
 ; GCN-IR-NEXT:    v_cmp_lt_u64_e64 s[6:7], 63, v[2:3]
 ; GCN-IR-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
 ; GCN-IR-NEXT:    s_or_b64 s[4:5], s[4:5], s[6:7]
 ; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 63, v[2:3]
 ; GCN-IR-NEXT:    s_xor_b64 s[6:7], s[4:5], -1
-; GCN-IR-NEXT:    v_mov_b32_e32 v6, v4
-; GCN-IR-NEXT:    v_mov_b32_e32 v7, v5
-; GCN-IR-NEXT:    v_cndmask_b32_e64 v9, v11, 0, s[4:5]
-; GCN-IR-NEXT:    v_cndmask_b32_e64 v8, v10, 0, s[4:5]
+; GCN-IR-NEXT:    v_mov_b32_e32 v14, v12
+; GCN-IR-NEXT:    v_mov_b32_e32 v15, v13
+; GCN-IR-NEXT:    v_cndmask_b32_e64 v5, v7, 0, s[4:5]
+; GCN-IR-NEXT:    v_cndmask_b32_e64 v4, v6, 0, s[4:5]
 ; GCN-IR-NEXT:    s_and_b64 s[4:5], s[6:7], vcc
 ; GCN-IR-NEXT:    s_and_saveexec_b64 s[6:7], s[4:5]
 ; GCN-IR-NEXT:    s_cbranch_execz .LBB1_6
 ; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
-; GCN-IR-NEXT:    v_add_i32_e32 v14, vcc, 1, v2
-; GCN-IR-NEXT:    v_addc_u32_e32 v15, vcc, 0, v3, vcc
+; GCN-IR-NEXT:    v_add_i32_e32 v8, vcc, 1, v2
+; GCN-IR-NEXT:    v_addc_u32_e32 v9, vcc, 0, v3, vcc
 ; GCN-IR-NEXT:    v_sub_i32_e64 v2, s[4:5], 63, v2
-; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[14:15]
-; GCN-IR-NEXT:    v_lshl_b64 v[2:3], v[10:11], v2
-; GCN-IR-NEXT:    v_mov_b32_e32 v8, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, 0
+; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[8:9]
+; GCN-IR-NEXT:    v_lshl_b64 v[2:3], v[6:7], v2
+; GCN-IR-NEXT:    v_mov_b32_e32 v4, 0
+; GCN-IR-NEXT:    v_mov_b32_e32 v5, 0
 ; GCN-IR-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-IR-NEXT:    s_xor_b64 s[8:9], exec, s[4:5]
 ; GCN-IR-NEXT:    s_cbranch_execz .LBB1_5
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
 ; GCN-IR-NEXT:    v_add_i32_e32 v16, vcc, -1, v0
 ; GCN-IR-NEXT:    v_addc_u32_e32 v17, vcc, -1, v1, vcc
-; GCN-IR-NEXT:    v_not_b32_e32 v9, v12
-; GCN-IR-NEXT:    v_lshr_b64 v[14:15], v[10:11], v14
-; GCN-IR-NEXT:    v_not_b32_e32 v8, 0
-; GCN-IR-NEXT:    v_add_i32_e32 v10, vcc, v9, v13
-; GCN-IR-NEXT:    v_mov_b32_e32 v12, 0
-; GCN-IR-NEXT:    v_addc_u32_e32 v11, vcc, 0, v8, vcc
+; GCN-IR-NEXT:    v_not_b32_e32 v5, v10
+; GCN-IR-NEXT:    v_lshr_b64 v[8:9], v[6:7], v8
+; GCN-IR-NEXT:    v_not_b32_e32 v4, 0
+; GCN-IR-NEXT:    v_add_i32_e32 v6, vcc, v5, v11
+; GCN-IR-NEXT:    v_mov_b32_e32 v10, 0
+; GCN-IR-NEXT:    v_addc_u32_e32 v7, vcc, 0, v4, vcc
 ; GCN-IR-NEXT:    s_mov_b64 s[10:11], 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v13, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, 0
+; GCN-IR-NEXT:    v_mov_b32_e32 v11, 0
+; GCN-IR-NEXT:    v_mov_b32_e32 v5, 0
 ; GCN-IR-NEXT:  .LBB1_3: ; %udiv-do-while
 ; GCN-IR-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN-IR-NEXT:    v_lshl_b64 v[14:15], v[14:15], 1
-; GCN-IR-NEXT:    v_lshrrev_b32_e32 v8, 31, v3
-; GCN-IR-NEXT:    v_or_b32_e32 v14, v14, v8
+; GCN-IR-NEXT:    v_lshl_b64 v[8:9], v[8:9], 1
+; GCN-IR-NEXT:    v_lshrrev_b32_e32 v4, 31, v3
+; GCN-IR-NEXT:    v_or_b32_e32 v8, v8, v4
 ; GCN-IR-NEXT:    v_lshl_b64 v[2:3], v[2:3], 1
-; GCN-IR-NEXT:    v_sub_i32_e32 v8, vcc, v16, v14
-; GCN-IR-NEXT:    v_subb_u32_e32 v8, vcc, v17, v15, vcc
-; GCN-IR-NEXT:    v_or_b32_e32 v2, v12, v2
-; GCN-IR-NEXT:    v_ashrrev_i32_e32 v12, 31, v8
-; GCN-IR-NEXT:    v_add_i32_e32 v10, vcc, 1, v10
-; GCN-IR-NEXT:    v_or_b32_e32 v3, v13, v3
-; GCN-IR-NEXT:    v_and_b32_e32 v8, 1, v12
-; GCN-IR-NEXT:    v_and_b32_e32 v13, v12, v1
-; GCN-IR-NEXT:    v_and_b32_e32 v12, v12, v0
-; GCN-IR-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
-; GCN-IR-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[10:11]
-; GCN-IR-NEXT:    v_sub_i32_e64 v14, s[4:5], v14, v12
-; GCN-IR-NEXT:    v_subb_u32_e64 v15, s[4:5], v15, v13, s[4:5]
-; GCN-IR-NEXT:    v_mov_b32_e32 v13, v9
+; GCN-IR-NEXT:    v_sub_i32_e32 v4, vcc, v16, v8
+; GCN-IR-NEXT:    v_subb_u32_e32 v4, vcc, v17, v9, vcc
+; GCN-IR-NEXT:    v_or_b32_e32 v2, v10, v2
+; GCN-IR-NEXT:    v_ashrrev_i32_e32 v10, 31, v4
+; GCN-IR-NEXT:    v_add_i32_e32 v6, vcc, 1, v6
+; GCN-IR-NEXT:    v_or_b32_e32 v3, v11, v3
+; GCN-IR-NEXT:    v_and_b32_e32 v4, 1, v10
+; GCN-IR-NEXT:    v_and_b32_e32 v11, v10, v1
+; GCN-IR-NEXT:    v_and_b32_e32 v10, v10, v0
+; GCN-IR-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; GCN-IR-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[6:7]
+; GCN-IR-NEXT:    v_sub_i32_e64 v8, s[4:5], v8, v10
+; GCN-IR-NEXT:    v_subb_u32_e64 v9, s[4:5], v9, v11, s[4:5]
+; GCN-IR-NEXT:    v_mov_b32_e32 v11, v5
 ; GCN-IR-NEXT:    s_or_b64 s[10:11], vcc, s[10:11]
-; GCN-IR-NEXT:    v_mov_b32_e32 v12, v8
+; GCN-IR-NEXT:    v_mov_b32_e32 v10, v4
 ; GCN-IR-NEXT:    s_andn2_b64 exec, exec, s[10:11]
 ; GCN-IR-NEXT:    s_cbranch_execnz .LBB1_3
 ; GCN-IR-NEXT:  ; %bb.4: ; %Flow
@@ -448,14 +448,14 @@ define i64 @v_test_sdiv(i64 %x, i64 %y) {
 ; GCN-IR-NEXT:  .LBB1_5: ; %Flow4
 ; GCN-IR-NEXT:    s_or_b64 exec, exec, s[8:9]
 ; GCN-IR-NEXT:    v_lshl_b64 v[0:1], v[2:3], 1
-; GCN-IR-NEXT:    v_or_b32_e32 v9, v9, v1
-; GCN-IR-NEXT:    v_or_b32_e32 v8, v8, v0
+; GCN-IR-NEXT:    v_or_b32_e32 v5, v5, v1
+; GCN-IR-NEXT:    v_or_b32_e32 v4, v4, v0
 ; GCN-IR-NEXT:  .LBB1_6: ; %Flow5
 ; GCN-IR-NEXT:    s_or_b64 exec, exec, s[6:7]
-; GCN-IR-NEXT:    v_xor_b32_e32 v0, v5, v4
-; GCN-IR-NEXT:    v_xor_b32_e32 v1, v7, v6
-; GCN-IR-NEXT:    v_xor_b32_e32 v3, v8, v0
-; GCN-IR-NEXT:    v_xor_b32_e32 v2, v9, v1
+; GCN-IR-NEXT:    v_xor_b32_e32 v0, v13, v12
+; GCN-IR-NEXT:    v_xor_b32_e32 v1, v15, v14
+; GCN-IR-NEXT:    v_xor_b32_e32 v3, v4, v0
+; GCN-IR-NEXT:    v_xor_b32_e32 v2, v5, v1
 ; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, v3, v0
 ; GCN-IR-NEXT:    v_subb_u32_e32 v1, vcc, v2, v1, vcc
 ; GCN-IR-NEXT:    s_setpc_b64 s[30:31]
@@ -1001,33 +1001,33 @@ define amdgpu_kernel void @s_test_sdiv24_48(ptr addrspace(1) %out, i48 %x, i48 %
 ; GCN-IR-NEXT:    s_flbit_i32_b32 s8, s12
 ; GCN-IR-NEXT:    s_add_i32 s8, s8, 32
 ; GCN-IR-NEXT:    s_flbit_i32_b32 s9, s13
-; GCN-IR-NEXT:    s_min_u32 s18, s8, s9
-; GCN-IR-NEXT:    s_sub_u32 s16, s14, s18
+; GCN-IR-NEXT:    s_min_u32 s20, s8, s9
+; GCN-IR-NEXT:    s_sub_u32 s16, s14, s20
 ; GCN-IR-NEXT:    s_subb_u32 s17, 0, 0
-; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[20:21], s[16:17], 63
+; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[18:19], s[16:17], 63
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[22:23], s[16:17], 63
-; GCN-IR-NEXT:    s_or_b64 s[20:21], s[10:11], s[20:21]
-; GCN-IR-NEXT:    s_and_b64 s[10:11], s[20:21], exec
+; GCN-IR-NEXT:    s_or_b64 s[18:19], s[10:11], s[18:19]
+; GCN-IR-NEXT:    s_and_b64 s[10:11], s[18:19], exec
 ; GCN-IR-NEXT:    s_cselect_b32 s11, 0, s13
 ; GCN-IR-NEXT:    s_cselect_b32 s10, 0, s12
-; GCN-IR-NEXT:    s_or_b64 s[20:21], s[20:21], s[22:23]
+; GCN-IR-NEXT:    s_or_b64 s[18:19], s[18:19], s[22:23]
 ; GCN-IR-NEXT:    s_mov_b64 s[8:9], 0
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[20:21]
+; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[18:19]
 ; GCN-IR-NEXT:    s_cbranch_vccz .LBB9_5
 ; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
-; GCN-IR-NEXT:    s_add_u32 s20, s16, 1
-; GCN-IR-NEXT:    s_addc_u32 s21, s17, 0
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[10:11], s[20:21], 0
+; GCN-IR-NEXT:    s_add_u32 s18, s16, 1
+; GCN-IR-NEXT:    s_addc_u32 s19, s17, 0
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[10:11], s[18:19], 0
 ; GCN-IR-NEXT:    s_sub_i32 s16, 63, s16
 ; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[10:11]
 ; GCN-IR-NEXT:    s_lshl_b64 s[10:11], s[12:13], s16
 ; GCN-IR-NEXT:    s_cbranch_vccz .LBB9_4
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
-; GCN-IR-NEXT:    s_lshr_b64 s[16:17], s[12:13], s20
-; GCN-IR-NEXT:    s_add_u32 s19, s6, -1
-; GCN-IR-NEXT:    s_addc_u32 s20, s7, -1
+; GCN-IR-NEXT:    s_lshr_b64 s[16:17], s[12:13], s18
+; GCN-IR-NEXT:    s_add_u32 s18, s6, -1
+; GCN-IR-NEXT:    s_addc_u32 s19, s7, -1
 ; GCN-IR-NEXT:    s_not_b64 s[8:9], s[14:15]
-; GCN-IR-NEXT:    s_add_u32 s12, s8, s18
+; GCN-IR-NEXT:    s_add_u32 s12, s8, s20
 ; GCN-IR-NEXT:    s_addc_u32 s13, s9, 0
 ; GCN-IR-NEXT:    s_mov_b64 s[14:15], 0
 ; GCN-IR-NEXT:    s_mov_b32 s9, 0
@@ -1038,8 +1038,8 @@ define amdgpu_kernel void @s_test_sdiv24_48(ptr addrspace(1) %out, i48 %x, i48 %
 ; GCN-IR-NEXT:    s_lshl_b64 s[10:11], s[10:11], 1
 ; GCN-IR-NEXT:    s_or_b64 s[16:17], s[16:17], s[8:9]
 ; GCN-IR-NEXT:    s_or_b64 s[10:11], s[14:15], s[10:11]
-; GCN-IR-NEXT:    s_sub_u32 s8, s19, s16
-; GCN-IR-NEXT:    s_subb_u32 s8, s20, s17
+; GCN-IR-NEXT:    s_sub_u32 s8, s18, s16
+; GCN-IR-NEXT:    s_subb_u32 s8, s19, s17
 ; GCN-IR-NEXT:    s_ashr_i32 s14, s8, 31
 ; GCN-IR-NEXT:    s_mov_b32 s15, s14
 ; GCN-IR-NEXT:    s_and_b32 s8, s14, 1
@@ -1048,9 +1048,9 @@ define amdgpu_kernel void @s_test_sdiv24_48(ptr addrspace(1) %out, i48 %x, i48 %
 ; GCN-IR-NEXT:    s_subb_u32 s17, s17, s15
 ; GCN-IR-NEXT:    s_add_u32 s12, s12, 1
 ; GCN-IR-NEXT:    s_addc_u32 s13, s13, 0
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[22:23], s[12:13], 0
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[20:21], s[12:13], 0
 ; GCN-IR-NEXT:    s_mov_b64 s[14:15], s[8:9]
-; GCN-IR-NEXT:    s_and_b64 vcc, exec, s[22:23]
+; GCN-IR-NEXT:    s_and_b64 vcc, exec, s[20:21]
 ; GCN-IR-NEXT:    s_cbranch_vccz .LBB9_3
 ; GCN-IR-NEXT:  .LBB9_4: ; %Flow4
 ; GCN-IR-NEXT:    s_lshl_b64 s[6:7], s[10:11], 1
@@ -1206,32 +1206,32 @@ define amdgpu_kernel void @s_test_sdiv_k_num_i64(ptr addrspace(1) %out, i64 %x)
 ; GCN-IR-NEXT:    s_flbit_i32_b32 s10, s2
 ; GCN-IR-NEXT:    s_add_i32 s10, s10, 32
 ; GCN-IR-NEXT:    s_flbit_i32_b32 s11, s3
-; GCN-IR-NEXT:    s_min_u32 s10, s10, s11
-; GCN-IR-NEXT:    s_add_u32 s12, s10, 0xffffffc5
-; GCN-IR-NEXT:    s_addc_u32 s13, 0, -1
+; GCN-IR-NEXT:    s_min_u32 s14, s10, s11
+; GCN-IR-NEXT:    s_add_u32 s10, s14, 0xffffffc5
+; GCN-IR-NEXT:    s_addc_u32 s11, 0, -1
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[8:9], s[2:3], 0
-; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[14:15], s[12:13], 63
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[16:17], s[12:13], 63
-; GCN-IR-NEXT:    s_or_b64 s[14:15], s[8:9], s[14:15]
-; GCN-IR-NEXT:    s_and_b64 s[8:9], s[14:15], exec
+; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[12:13], s[10:11], 63
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[16:17], s[10:11], 63
+; GCN-IR-NEXT:    s_or_b64 s[12:13], s[8:9], s[12:13]
+; GCN-IR-NEXT:    s_and_b64 s[8:9], s[12:13], exec
 ; GCN-IR-NEXT:    s_cselect_b32 s8, 0, 24
-; GCN-IR-NEXT:    s_or_b64 s[14:15], s[14:15], s[16:17]
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[14:15]
+; GCN-IR-NEXT:    s_or_b64 s[12:13], s[12:13], s[16:17]
+; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[12:13]
 ; GCN-IR-NEXT:    s_mov_b32 s9, 0
 ; GCN-IR-NEXT:    s_cbranch_vccz .LBB10_5
 ; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
-; GCN-IR-NEXT:    s_add_u32 s14, s12, 1
-; GCN-IR-NEXT:    s_addc_u32 s15, s13, 0
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[8:9], s[14:15], 0
-; GCN-IR-NEXT:    s_sub_i32 s11, 63, s12
+; GCN-IR-NEXT:    s_add_u32 s12, s10, 1
+; GCN-IR-NEXT:    s_addc_u32 s13, s11, 0
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[8:9], s[12:13], 0
+; GCN-IR-NEXT:    s_sub_i32 s10, 63, s10
 ; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[8:9]
-; GCN-IR-NEXT:    s_lshl_b64 s[8:9], 24, s11
+; GCN-IR-NEXT:    s_lshl_b64 s[8:9], 24, s10
 ; GCN-IR-NEXT:    s_cbranch_vccz .LBB10_4
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
-; GCN-IR-NEXT:    s_lshr_b64 s[12:13], 24, s14
+; GCN-IR-NEXT:    s_lshr_b64 s[12:13], 24, s12
 ; GCN-IR-NEXT:    s_add_u32 s16, s2, -1
 ; GCN-IR-NEXT:    s_addc_u32 s17, s3, -1
-; GCN-IR-NEXT:    s_sub_u32 s10, 58, s10
+; GCN-IR-NEXT:    s_sub_u32 s10, 58, s14
 ; GCN-IR-NEXT:    s_subb_u32 s11, 0, 0
 ; GCN-IR-NEXT:    s_mov_b64 s[14:15], 0
 ; GCN-IR-NEXT:    s_mov_b32 s7, 0
@@ -1385,87 +1385,87 @@ define i64 @v_test_sdiv_k_num_i64(i64 %x) {
 ; GCN-IR-LABEL: v_test_sdiv_k_num_i64:
 ; GCN-IR:       ; %bb.0: ; %_udiv-special-cases
 ; GCN-IR-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-IR-NEXT:    v_ashrrev_i32_e32 v2, 31, v1
-; GCN-IR-NEXT:    v_xor_b32_e32 v0, v2, v0
-; GCN-IR-NEXT:    v_xor_b32_e32 v1, v2, v1
-; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, v0, v2
-; GCN-IR-NEXT:    v_subb_u32_e32 v1, vcc, v1, v2, vcc
-; GCN-IR-NEXT:    v_ffbh_u32_e32 v4, v0
-; GCN-IR-NEXT:    v_add_i32_e32 v4, vcc, 32, v4
-; GCN-IR-NEXT:    v_ffbh_u32_e32 v5, v1
-; GCN-IR-NEXT:    v_min_u32_e32 v8, v4, v5
+; GCN-IR-NEXT:    v_ashrrev_i32_e32 v12, 31, v1
+; GCN-IR-NEXT:    v_xor_b32_e32 v0, v12, v0
+; GCN-IR-NEXT:    v_xor_b32_e32 v1, v12, v1
+; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, v0, v12
+; GCN-IR-NEXT:    v_subb_u32_e32 v1, vcc, v1, v12, vcc
+; GCN-IR-NEXT:    v_ffbh_u32_e32 v2, v0
+; GCN-IR-NEXT:    v_add_i32_e32 v2, vcc, 32, v2
+; GCN-IR-NEXT:    v_ffbh_u32_e32 v3, v1
+; GCN-IR-NEXT:    v_min_u32_e32 v10, v2, v3
 ; GCN-IR-NEXT:    s_movk_i32 s6, 0xffc5
-; GCN-IR-NEXT:    v_add_i32_e32 v5, vcc, s6, v8
-; GCN-IR-NEXT:    v_addc_u32_e64 v6, s[6:7], 0, -1, vcc
+; GCN-IR-NEXT:    v_add_i32_e32 v2, vcc, s6, v10
+; GCN-IR-NEXT:    v_addc_u32_e64 v3, s[6:7], 0, -1, vcc
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[4:5], 0, v[0:1]
-; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, 63, v[5:6]
-; GCN-IR-NEXT:    v_cmp_ne_u64_e64 s[6:7], 63, v[5:6]
+; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, 63, v[2:3]
+; GCN-IR-NEXT:    v_cmp_ne_u64_e64 s[6:7], 63, v[2:3]
 ; GCN-IR-NEXT:    s_or_b64 s[4:5], s[4:5], vcc
-; GCN-IR-NEXT:    v_cndmask_b32_e64 v7, 24, 0, s[4:5]
+; GCN-IR-NEXT:    v_cndmask_b32_e64 v4, 24, 0, s[4:5]
 ; GCN-IR-NEXT:    s_xor_b64 s[4:5], s[4:5], -1
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, v2
-; GCN-IR-NEXT:    v_mov_b32_e32 v4, 0
+; GCN-IR-NEXT:    v_mov_b32_e32 v13, v12
+; GCN-IR-NEXT:    v_mov_b32_e32 v5, 0
 ; GCN-IR-NEXT:    s_and_b64 s[4:5], s[4:5], s[6:7]
 ; GCN-IR-NEXT:    s_and_saveexec_b64 s[6:7], s[4:5]
 ; GCN-IR-NEXT:    s_cbranch_execz .LBB11_6
 ; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
-; GCN-IR-NEXT:    v_add_i32_e32 v9, vcc, 1, v5
-; GCN-IR-NEXT:    v_addc_u32_e32 v10, vcc, 0, v6, vcc
-; GCN-IR-NEXT:    v_sub_i32_e64 v4, s[4:5], 63, v5
-; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[9:10]
-; GCN-IR-NEXT:    v_lshl_b64 v[4:5], 24, v4
-; GCN-IR-NEXT:    v_mov_b32_e32 v6, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v7, 0
+; GCN-IR-NEXT:    v_add_i32_e32 v6, vcc, 1, v2
+; GCN-IR-NEXT:    v_addc_u32_e32 v7, vcc, 0, v3, vcc
+; GCN-IR-NEXT:    v_sub_i32_e64 v2, s[4:5], 63, v2
+; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[6:7]
+; GCN-IR-NEXT:    v_lshl_b64 v[2:3], 24, v2
+; GCN-IR-NEXT:    v_mov_b32_e32 v4, 0
+; GCN-IR-NEXT:    v_mov_b32_e32 v5, 0
 ; GCN-IR-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-IR-NEXT:    s_xor_b64 s[8:9], exec, s[4:5]
 ; GCN-IR-NEXT:    s_cbranch_execz .LBB11_5
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
 ; GCN-IR-NEXT:    v_add_i32_e32 v14, vcc, -1, v0
 ; GCN-IR-NEXT:    v_addc_u32_e32 v15, vcc, -1, v1, vcc
-; GCN-IR-NEXT:    v_lshr_b64 v[10:11], 24, v9
-; GCN-IR-NEXT:    v_sub_i32_e32 v8, vcc, 58, v8
-; GCN-IR-NEXT:    v_mov_b32_e32 v12, 0
-; GCN-IR-NEXT:    v_subb_u32_e64 v9, s[4:5], 0, 0, vcc
+; GCN-IR-NEXT:    v_lshr_b64 v[8:9], 24, v6
+; GCN-IR-NEXT:    v_sub_i32_e32 v6, vcc, 58, v10
+; GCN-IR-NEXT:    v_mov_b32_e32 v10, 0
+; GCN-IR-NEXT:    v_subb_u32_e64 v7, s[4:5], 0, 0, vcc
 ; GCN-IR-NEXT:    s_mov_b64 s[10:11], 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v13, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v7, 0
+; GCN-IR-NEXT:    v_mov_b32_e32 v11, 0
+; GCN-IR-NEXT:    v_mov_b32_e32 v5, 0
 ; GCN-IR-NEXT:  .LBB11_3: ; %udiv-do-while
 ; GCN-IR-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN-IR-NEXT:    v_lshl_b64 v[10:11], v[10:11], 1
-; GCN-IR-NEXT:    v_lshrrev_b32_e32 v6, 31, v5
-; GCN-IR-NEXT:    v_or_b32_e32 v10, v10, v6
-; GCN-IR-NEXT:    v_lshl_b64 v[4:5], v[4:5], 1
-; GCN-IR-NEXT:    v_sub_i32_e32 v6, vcc, v14, v10
-; GCN-IR-NEXT:    v_subb_u32_e32 v6, vcc, v15, v11, vcc
-; GCN-IR-NEXT:    v_or_b32_e32 v4, v12, v4
-; GCN-IR-NEXT:    v_ashrrev_i32_e32 v12, 31, v6
-; GCN-IR-NEXT:    v_add_i32_e32 v8, vcc, 1, v8
-; GCN-IR-NEXT:    v_or_b32_e32 v5, v13, v5
-; GCN-IR-NEXT:    v_and_b32_e32 v6, 1, v12
-; GCN-IR-NEXT:    v_and_b32_e32 v13, v12, v1
-; GCN-IR-NEXT:    v_and_b32_e32 v12, v12, v0
-; GCN-IR-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
-; GCN-IR-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[8:9]
-; GCN-IR-NEXT:    v_sub_i32_e64 v10, s[4:5], v10, v12
-; GCN-IR-NEXT:    v_subb_u32_e64 v11, s[4:5], v11, v13, s[4:5]
-; GCN-IR-NEXT:    v_mov_b32_e32 v13, v7
+; GCN-IR-NEXT:    v_lshl_b64 v[8:9], v[8:9], 1
+; GCN-IR-NEXT:    v_lshrrev_b32_e32 v4, 31, v3
+; GCN-IR-NEXT:    v_or_b32_e32 v8, v8, v4
+; GCN-IR-NEXT:    v_lshl_b64 v[2:3], v[2:3], 1
+; GCN-IR-NEXT:    v_sub_i32_e32 v4, vcc, v14, v8
+; GCN-IR-NEXT:    v_subb_u32_e32 v4, vcc, v15, v9, vcc
+; GCN-IR-NEXT:    v_or_b32_e32 v2, v10, v2
+; GCN-IR-NEXT:    v_ashrrev_i32_e32 v10, 31, v4
+; GCN-IR-NEXT:    v_add_i32_e32 v6, vcc, 1, v6
+; GCN-IR-NEXT:    v_or_b32_e32 v3, v11, v3
+; GCN-IR-NEXT:    v_and_b32_e32 v4, 1, v10
+; GCN-IR-NEXT:    v_and_b32_e32 v11, v10, v1
+; GCN-IR-NEXT:    v_and_b32_e32 v10, v10, v0
+; GCN-IR-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; GCN-IR-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[6:7]
+; GCN-IR-NEXT:    v_sub_i32_e64 v8, s[4:5], v8, v10
+; GCN-IR-NEXT:    v_subb_u32_e64 v9, s[4:5], v9, v11, s[4:5]
+; GCN-IR-NEXT:    v_mov_b32_e32 v11, v5
 ; GCN-IR-NEXT:    s_or_b64 s[10:11], vcc, s[10:11]
-; GCN-IR-NEXT:    v_mov_b32_e32 v12, v6
+; GCN-IR-NEXT:    v_mov_b32_e32 v10, v4
 ; GCN-IR-NEXT:    s_andn2_b64 exec, exec, s[10:11]
 ; GCN-IR-NEXT:    s_cbranch_execnz .LBB11_3
 ; GCN-IR-NEXT:  ; %bb.4: ; %Flow
 ; GCN-IR-NEXT:    s_or_b64 exec, exec, s[10:11]
 ; GCN-IR-NEXT:  .LBB11_5: ; %Flow4
 ; GCN-IR-NEXT:    s_or_b64 exec, exec, s[8:9]
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], v[4:5], 1
-; GCN-IR-NEXT:    v_or_b32_e32 v4, v7, v1
-; GCN-IR-NEXT:    v_or_b32_e32 v7, v6, v0
+; GCN-IR-NEXT:    v_lshl_b64 v[0:1], v[2:3], 1
+; GCN-IR-NEXT:    v_or_b32_e32 v5, v5, v1
+; GCN-IR-NEXT:    v_or_b32_e32 v4, v4, v0
 ; GCN-IR-NEXT:  .LBB11_6: ; %Flow5
 ; GCN-IR-NEXT:    s_or_b64 exec, exec, s[6:7]
-; GCN-IR-NEXT:    v_xor_b32_e32 v0, v7, v2
-; GCN-IR-NEXT:    v_xor_b32_e32 v1, v4, v3
-; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, v0, v2
-; GCN-IR-NEXT:    v_subb_u32_e32 v1, vcc, v1, v3, vcc
+; GCN-IR-NEXT:    v_xor_b32_e32 v0, v4, v12
+; GCN-IR-NEXT:    v_xor_b32_e32 v1, v5, v13
+; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, v0, v12
+; GCN-IR-NEXT:    v_subb_u32_e32 v1, vcc, v1, v13, vcc
 ; GCN-IR-NEXT:    s_setpc_b64 s[30:31]
   %result = sdiv i64 24, %x
   ret i64 %result
@@ -1578,89 +1578,89 @@ define i64 @v_test_sdiv_pow2_k_num_i64(i64 %x) {
 ; GCN-IR-LABEL: v_test_sdiv_pow2_k_num_i64:
 ; GCN-IR:       ; %bb.0: ; %_udiv-special-cases
 ; GCN-IR-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-IR-NEXT:    v_ashrrev_i32_e32 v2, 31, v1
-; GCN-IR-NEXT:    v_xor_b32_e32 v0, v2, v0
-; GCN-IR-NEXT:    v_xor_b32_e32 v1, v2, v1
-; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, v0, v2
-; GCN-IR-NEXT:    v_subb_u32_e32 v1, vcc, v1, v2, vcc
-; GCN-IR-NEXT:    v_ffbh_u32_e32 v4, v0
-; GCN-IR-NEXT:    v_add_i32_e32 v4, vcc, 32, v4
-; GCN-IR-NEXT:    v_ffbh_u32_e32 v5, v1
-; GCN-IR-NEXT:    v_min_u32_e32 v8, v4, v5
+; GCN-IR-NEXT:    v_ashrrev_i32_e32 v12, 31, v1
+; GCN-IR-NEXT:    v_xor_b32_e32 v0, v12, v0
+; GCN-IR-NEXT:    v_xor_b32_e32 v1, v12, v1
+; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, v0, v12
+; GCN-IR-NEXT:    v_subb_u32_e32 v1, vcc, v1, v12, vcc
+; GCN-IR-NEXT:    v_ffbh_u32_e32 v2, v0
+; GCN-IR-NEXT:    v_add_i32_e32 v2, vcc, 32, v2
+; GCN-IR-NEXT:    v_ffbh_u32_e32 v3, v1
+; GCN-IR-NEXT:    v_min_u32_e32 v10, v2, v3
 ; GCN-IR-NEXT:    s_movk_i32 s6, 0xffd0
-; GCN-IR-NEXT:    v_add_i32_e32 v5, vcc, s6, v8
-; GCN-IR-NEXT:    v_addc_u32_e64 v6, s[6:7], 0, -1, vcc
+; GCN-IR-NEXT:    v_add_i32_e32 v2, vcc, s6, v10
+; GCN-IR-NEXT:    v_addc_u32_e64 v3, s[6:7], 0, -1, vcc
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[4:5], 0, v[0:1]
-; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, 63, v[5:6]
-; GCN-IR-NEXT:    v_cmp_ne_u64_e64 s[6:7], 63, v[5:6]
-; GCN-IR-NEXT:    v_mov_b32_e32 v7, 0x8000
+; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, 63, v[2:3]
+; GCN-IR-NEXT:    v_cmp_ne_u64_e64 s[6:7], 63, v[2:3]
+; GCN-IR-NEXT:    v_mov_b32_e32 v4, 0x8000
 ; GCN-IR-NEXT:    s_or_b64 s[4:5], s[4:5], vcc
-; GCN-IR-NEXT:    v_cndmask_b32_e64 v7, v7, 0, s[4:5]
+; GCN-IR-NEXT:    v_cndmask_b32_e64 v4, v4, 0, s[4:5]
 ; GCN-IR-NEXT:    s_xor_b64 s[4:5], s[4:5], -1
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, v2
-; GCN-IR-NEXT:    v_mov_b32_e32 v4, 0
+; GCN-IR-NEXT:    v_mov_b32_e32 v13, v12
+; GCN-IR-NEXT:    v_mov_b32_e32 v5, 0
 ; GCN-IR-NEXT:    s_and_b64 s[4:5], s[4:5], s[6:7]
 ; GCN-IR-NEXT:    s_and_saveexec_b64 s[6:7], s[4:5]
 ; GCN-IR-NEXT:    s_cbranch_execz .LBB12_6
 ; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
-; GCN-IR-NEXT:    v_add_i32_e32 v9, vcc, 1, v5
-; GCN-IR-NEXT:    v_sub_i32_e64 v4, s[4:5], 63, v5
-; GCN-IR-NEXT:    v_addc_u32_e32 v10, vcc, 0, v6, vcc
+; GCN-IR-NEXT:    v_add_i32_e32 v6, vcc, 1, v2
+; GCN-IR-NEXT:    v_sub_i32_e64 v2, s[4:5], 63, v2
+; GCN-IR-NEXT:    v_addc_u32_e32 v7, vcc, 0, v3, vcc
 ; GCN-IR-NEXT:    s_mov_b64 s[4:5], 0x8000
-; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[9:10]
-; GCN-IR-NEXT:    v_lshl_b64 v[4:5], s[4:5], v4
-; GCN-IR-NEXT:    v_mov_b32_e32 v6, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v7, 0
+; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[6:7]
+; GCN-IR-NEXT:    v_lshl_b64 v[2:3], s[4:5], v2
+; GCN-IR-NEXT:    v_mov_b32_e32 v4, 0
+; GCN-IR-NEXT:    v_mov_b32_e32 v5, 0
 ; GCN-IR-NEXT:    s_and_saveexec_b64 s[8:9], vcc
 ; GCN-IR-NEXT:    s_xor_b64 s[8:9], exec, s[8:9]
 ; GCN-IR-NEXT:    s_cbranch_execz .LBB12_5
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
 ; GCN-IR-NEXT:    v_add_i32_e32 v14, vcc, -1, v0
 ; GCN-IR-NEXT:    v_addc_u32_e32 v15, vcc, -1, v1, vcc
-; GCN-IR-NEXT:    v_lshr_b64 v[10:11], s[4:5], v9
-; GCN-IR-NEXT:    v_sub_i32_e32 v8, vcc, 47, v8
-; GCN-IR-NEXT:    v_mov_b32_e32 v12, 0
-; GCN-IR-NEXT:    v_subb_u32_e64 v9, s[4:5], 0, 0, vcc
+; GCN-IR-NEXT:    v_lshr_b64 v[8:9], s[4:5], v6
+; GCN-IR-NEXT:    v_sub_i32_e32 v6, vcc, 47, v10
+; GCN-IR-NEXT:    v_mov_b32_e32 v10, 0
+; GCN-IR-NEXT:    v_subb_u32_e64 v7, s[4:5], 0, 0, vcc
 ; GCN-IR-NEXT:    s_mov_b64 s[10:11], 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v13, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v7, 0
+; GCN-IR-NEXT:    v_mov_b32_e32 v11, 0
+; GCN-IR-NEXT:    v_mov_b32_e32 v5, 0
 ; GCN-IR-NEXT:  .LBB12_3: ; %udiv-do-while
 ; GCN-IR-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN-IR-NEXT:    v_lshl_b64 v[10:11], v[10:11], 1
-; GCN-IR-NEXT:    v_lshrrev_b32_e32 v6, 31, v5
-; GCN-IR-NEXT:    v_or_b32_e32 v10, v10, v6
-; GCN-IR-NEXT:    v_lshl_b64 v[4:5], v[4:5], 1
-; GCN-IR-NEXT:    v_sub_i32_e32 v6, vcc, v14, v10
-; GCN-IR-NEXT:    v_subb_u32_e32 v6, vcc, v15, v11, vcc
-; GCN-IR-NEXT:    v_or_b32_e32 v4, v12, v4
-; GCN-IR-NEXT:    v_ashrrev_i32_e32 v12, 31, v6
-; GCN-IR-NEXT:    v_add_i32_e32 v8, vcc, 1, v8
-; GCN-IR-NEXT:    v_or_b32_e32 v5, v13, v5
-; GCN-IR-NEXT:    v_and_b32_e32 v6, 1, v12
-; GCN-IR-NEXT:    v_and_b32_e32 v13, v12, v1
-; GCN-IR-NEXT:    v_and_b32_e32 v12, v12, v0
-; GCN-IR-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
-; GCN-IR-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[8:9]
-; GCN-IR-NEXT:    v_sub_i32_e64 v10, s[4:5], v10, v12
-; GCN-IR-NEXT:    v_subb_u32_e64 v11, s[4:5], v11, v13, s[4:5]
-; GCN-IR-NEXT:    v_mov_b32_e32 v13, v7
+; GCN-IR-NEXT:    v_lshl_b64 v[8:9], v[8:9], 1
+; GCN-IR-NEXT:    v_lshrrev_b32_e32 v4, 31, v3
+; GCN-IR-NEXT:    v_or_b32_e32 v8, v8, v4
+; GCN-IR-NEXT:    v_lshl_b64 v[2:3], v[2:3], 1
+; GCN-IR-NEXT:    v_sub_i32_e32 v4, vcc, v14, v8
+; GCN-IR-NEXT:    v_subb_u32_e32 v4, vcc, v15, v9, vcc
+; GCN-IR-NEXT:    v_or_b32_e32 v2, v10, v2
+; GCN-IR-NEXT:    v_ashrrev_i32_e32 v10, 31, v4
+; GCN-IR-NEXT:    v_add_i32_e32 v6, vcc, 1, v6
+; GCN-IR-NEXT:    v_or_b32_e32 v3, v11, v3
+; GCN-IR-NEXT:    v_and_b32_e32 v4, 1, v10
+; GCN-IR-NEXT:    v_and_b32_e32 v11, v10, v1
+; GCN-IR-NEXT:    v_and_b32_e32 v10, v10, v0
+; GCN-IR-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; GCN-IR-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[6:7]
+; GCN-IR-NEXT:    v_sub_i32_e64 v8, s[4:5], v8, v10
+; GCN-IR-NEXT:    v_subb_u32_e64 v9, s[4:5], v9, v11, s[4:5]
+; GCN-IR-NEXT:    v_mov_b32_e32 v11, v5
 ; GCN-IR-NEXT:    s_or_b64 s[10:11], vcc, s[10:11]
-; GCN-IR-NEXT:    v_mov_b32_e32 v12, v6
+; GCN-IR-NEXT:    v_mov_b32_e32 v10, v4
 ; GCN-IR-NEXT:    s_andn2_b64 exec, exec, s[10:11]
 ; GCN-IR-NEXT:    s_cbranch_execnz .LBB12_3
 ; GCN-IR-NEXT:  ; %bb.4: ; %Flow
 ; GCN-IR-NEXT:    s_or_b64 exec, exec, s[10:11]
 ; GCN-IR-NEXT:  .LBB12_5: ; %Flow4
 ; GCN-IR-NEXT:    s_or_b64 exec, exec, s[8:9]
-; GCN-IR-NEXT:    v_lshl_b64 v[0:1], v[4:5], 1
-; GCN-IR-NEXT:    v_or_b32_e32 v4, v7, v1
-; GCN-IR-NEXT:    v_or_b32_e32 v7, v6, v0
+; GCN-IR-NEXT:    v_lshl_b64 v[0:1], v[2:3], 1
+; GCN-IR-NEXT:    v_or_b32_e32 v5, v5, v1
+; GCN-IR-NEXT:    v_or_b32_e32 v4, v4, v0
 ; GCN-IR-NEXT:  .LBB12_6: ; %Flow5
 ; GCN-IR-NEXT:    s_or_b64 exec, exec, s[6:7]
-; GCN-IR-NEXT:    v_xor_b32_e32 v0, v7, v2
-; GCN-IR-NEXT:    v_xor_b32_e32 v1, v4, v3
-; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, v0, v2
-; GCN-IR-NEXT:    v_subb_u32_e32 v1, vcc, v1, v3, vcc
+; GCN-IR-NEXT:    v_xor_b32_e32 v0, v4, v12
+; GCN-IR-NEXT:    v_xor_b32_e32 v1, v5, v13
+; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, v0, v12
+; GCN-IR-NEXT:    v_subb_u32_e32 v1, vcc, v1, v13, vcc
 ; GCN-IR-NEXT:    s_setpc_b64 s[30:31]
   %result = sdiv i64 32768, %x
   ret i64 %result
@@ -1680,84 +1680,84 @@ define i64 @v_test_sdiv_pow2_k_den_i64(i64 %x) {
 ; GCN-IR-LABEL: v_test_sdiv_pow2_k_den_i64:
 ; GCN-IR:       ; %bb.0: ; %_udiv-special-cases
 ; GCN-IR-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-IR-NEXT:    v_ashrrev_i32_e32 v2, 31, v1
-; GCN-IR-NEXT:    v_xor_b32_e32 v0, v2, v0
-; GCN-IR-NEXT:    v_xor_b32_e32 v1, v2, v1
-; GCN-IR-NEXT:    v_sub_i32_e32 v7, vcc, v0, v2
-; GCN-IR-NEXT:    v_subb_u32_e32 v8, vcc, v1, v2, vcc
-; GCN-IR-NEXT:    v_ffbh_u32_e32 v0, v7
+; GCN-IR-NEXT:    v_ashrrev_i32_e32 v10, 31, v1
+; GCN-IR-NEXT:    v_xor_b32_e32 v0, v10, v0
+; GCN-IR-NEXT:    v_xor_b32_e32 v1, v10, v1
+; GCN-IR-NEXT:    v_sub_i32_e32 v4, vcc, v0, v10
+; GCN-IR-NEXT:    v_subb_u32_e32 v5, vcc, v1, v10, vcc
+; GCN-IR-NEXT:    v_ffbh_u32_e32 v0, v4
 ; GCN-IR-NEXT:    v_add_i32_e64 v0, s[4:5], 32, v0
-; GCN-IR-NEXT:    v_ffbh_u32_e32 v1, v8
-; GCN-IR-NEXT:    v_min_u32_e32 v0, v0, v1
-; GCN-IR-NEXT:    v_sub_i32_e64 v3, s[4:5], 48, v0
-; GCN-IR-NEXT:    v_subb_u32_e64 v4, s[4:5], 0, 0, s[4:5]
-; GCN-IR-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[7:8]
-; GCN-IR-NEXT:    v_cmp_lt_u64_e64 s[4:5], 63, v[3:4]
-; GCN-IR-NEXT:    v_mov_b32_e32 v1, v2
+; GCN-IR-NEXT:    v_ffbh_u32_e32 v1, v5
+; GCN-IR-NEXT:    v_min_u32_e32 v8, v0, v1
+; GCN-IR-NEXT:    v_sub_i32_e64 v0, s[4:5], 48, v8
+; GCN-IR-NEXT:    v_subb_u32_e64 v1, s[4:5], 0, 0, s[4:5]
+; GCN-IR-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[4:5]
+; GCN-IR-NEXT:    v_cmp_lt_u64_e64 s[4:5], 63, v[0:1]
+; GCN-IR-NEXT:    v_mov_b32_e32 v11, v10
 ; GCN-IR-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 63, v[3:4]
+; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 63, v[0:1]
 ; GCN-IR-NEXT:    s_xor_b64 s[6:7], s[4:5], -1
-; GCN-IR-NEXT:    v_cndmask_b32_e64 v6, v8, 0, s[4:5]
-; GCN-IR-NEXT:    v_cndmask_b32_e64 v5, v7, 0, s[4:5]
+; GCN-IR-NEXT:    v_cndmask_b32_e64 v3, v5, 0, s[4:5]
+; GCN-IR-NEXT:    v_cndmask_b32_e64 v2, v4, 0, s[4:5]
 ; GCN-IR-NEXT:    s_and_b64 s[4:5], s[6:7], vcc
 ; GCN-IR-NEXT:    s_and_saveexec_b64 s[6:7], s[4:5]
 ; GCN-IR-NEXT:    s_cbranch_execz .LBB13_6
 ; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
-; GCN-IR-NEXT:    v_add_i32_e32 v9, vcc, 1, v3
-; GCN-IR-NEXT:    v_addc_u32_e32 v10, vcc, 0, v4, vcc
-; GCN-IR-NEXT:    v_sub_i32_e64 v3, s[4:5], 63, v3
-; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[9:10]
-; GCN-IR-NEXT:    v_lshl_b64 v[3:4], v[7:8], v3
-; GCN-IR-NEXT:    v_mov_b32_e32 v5, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v6, 0
+; GCN-IR-NEXT:    v_add_i32_e32 v6, vcc, 1, v0
+; GCN-IR-NEXT:    v_addc_u32_e32 v7, vcc, 0, v1, vcc
+; GCN-IR-NEXT:    v_sub_i32_e64 v0, s[4:5], 63, v0
+; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[6:7]
+; GCN-IR-NEXT:    v_lshl_b64 v[0:1], v[4:5], v0
+; GCN-IR-NEXT:    v_mov_b32_e32 v2, 0
+; GCN-IR-NEXT:    v_mov_b32_e32 v3, 0
 ; GCN-IR-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-IR-NEXT:    s_xor_b64 s[8:9], exec, s[4:5]
 ; GCN-IR-NEXT:    s_cbranch_execz .LBB13_5
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
-; GCN-IR-NEXT:    v_lshr_b64 v[9:10], v[7:8], v9
-; GCN-IR-NEXT:    v_add_i32_e32 v7, vcc, 0xffffffcf, v0
-; GCN-IR-NEXT:    v_mov_b32_e32 v11, 0
-; GCN-IR-NEXT:    v_addc_u32_e64 v8, s[4:5], 0, -1, vcc
+; GCN-IR-NEXT:    v_lshr_b64 v[6:7], v[4:5], v6
+; GCN-IR-NEXT:    v_add_i32_e32 v4, vcc, 0xffffffcf, v8
+; GCN-IR-NEXT:    v_mov_b32_e32 v8, 0
+; GCN-IR-NEXT:    v_addc_u32_e64 v5, s[4:5], 0, -1, vcc
 ; GCN-IR-NEXT:    s_mov_b64 s[10:11], 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v12, 0
+; GCN-IR-NEXT:    v_mov_b32_e32 v9, 0
 ; GCN-IR-NEXT:    s_movk_i32 s12, 0x7fff
-; GCN-IR-NEXT:    v_mov_b32_e32 v6, 0
+; GCN-IR-NEXT:    v_mov_b32_e32 v3, 0
 ; GCN-IR-NEXT:  .LBB13_3: ; %udiv-do-while
 ; GCN-IR-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN-IR-NEXT:    v_lshl_b64 v[9:10], v[9:10], 1
-; GCN-IR-NEXT:    v_lshrrev_b32_e32 v0, 31, v4
-; GCN-IR-NEXT:    v_or_b32_e32 v0, v9, v0
-; GCN-IR-NEXT:    v_sub_i32_e32 v5, vcc, s12, v0
-; GCN-IR-NEXT:    v_subb_u32_e32 v5, vcc, 0, v10, vcc
-; GCN-IR-NEXT:    v_add_i32_e32 v7, vcc, 1, v7
-; GCN-IR-NEXT:    v_lshl_b64 v[3:4], v[3:4], 1
-; GCN-IR-NEXT:    v_ashrrev_i32_e32 v9, 31, v5
-; GCN-IR-NEXT:    v_addc_u32_e32 v8, vcc, 0, v8, vcc
-; GCN-IR-NEXT:    v_and_b32_e32 v5, 1, v9
-; GCN-IR-NEXT:    v_and_b32_e32 v9, 0x8000, v9
-; GCN-IR-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[7:8]
-; GCN-IR-NEXT:    v_or_b32_e32 v4, v12, v4
-; GCN-IR-NEXT:    v_or_b32_e32 v3, v11, v3
-; GCN-IR-NEXT:    v_sub_i32_e64 v9, s[4:5], v0, v9
-; GCN-IR-NEXT:    v_mov_b32_e32 v12, v6
-; GCN-IR-NEXT:    v_subbrev_u32_e64 v10, s[4:5], 0, v10, s[4:5]
+; GCN-IR-NEXT:    v_lshl_b64 v[6:7], v[6:7], 1
+; GCN-IR-NEXT:    v_lshrrev_b32_e32 v2, 31, v1
+; GCN-IR-NEXT:    v_or_b32_e32 v6, v6, v2
+; GCN-IR-NEXT:    v_sub_i32_e32 v2, vcc, s12, v6
+; GCN-IR-NEXT:    v_lshl_b64 v[0:1], v[0:1], 1
+; GCN-IR-NEXT:    v_subb_u32_e32 v2, vcc, 0, v7, vcc
+; GCN-IR-NEXT:    v_add_i32_e32 v4, vcc, 1, v4
+; GCN-IR-NEXT:    v_or_b32_e32 v0, v8, v0
+; GCN-IR-NEXT:    v_ashrrev_i32_e32 v8, 31, v2
+; GCN-IR-NEXT:    v_addc_u32_e32 v5, vcc, 0, v5, vcc
+; GCN-IR-NEXT:    v_and_b32_e32 v2, 1, v8
+; GCN-IR-NEXT:    v_and_b32_e32 v8, 0x8000, v8
+; GCN-IR-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[4:5]
+; GCN-IR-NEXT:    v_or_b32_e32 v1, v9, v1
+; GCN-IR-NEXT:    v_sub_i32_e64 v6, s[4:5], v6, v8
+; GCN-IR-NEXT:    v_mov_b32_e32 v9, v3
+; GCN-IR-NEXT:    v_subbrev_u32_e64 v7, s[4:5], 0, v7, s[4:5]
 ; GCN-IR-NEXT:    s_or_b64 s[10:11], vcc, s[10:11]
-; GCN-IR-NEXT:    v_mov_b32_e32 v11, v5
+; GCN-IR-NEXT:    v_mov_b32_e32 v8, v2
 ; GCN-IR-NEXT:    s_andn2_b64 exec, exec, s[10:11]
 ; GCN-IR-NEXT:    s_cbranch_execnz .LBB13_3
 ; GCN-IR-NEXT:  ; %bb.4: ; %Flow
 ; GCN-IR-NEXT:    s_or_b64 exec, exec, s[10:11]
 ; GCN-IR-NEXT:  .LBB13_5: ; %Flow4
 ; GCN-IR-NEXT:    s_or_b64 exec, exec, s[8:9]
-; GCN-IR-NEXT:    v_lshl_b64 v[3:4], v[3:4], 1
-; GCN-IR-NEXT:    v_or_b32_e32 v6, v6, v4
-; GCN-IR-NEXT:    v_or_b32_e32 v5, v5, v3
+; GCN-IR-NEXT:    v_lshl_b64 v[0:1], v[0:1], 1
+; GCN-IR-NEXT:    v_or_b32_e32 v3, v3, v1
+; GCN-IR-NEXT:    v_or_b32_e32 v2, v2, v0
 ; GCN-IR-NEXT:  .LBB13_6: ; %Flow5
 ; GCN-IR-NEXT:    s_or_b64 exec, exec, s[6:7]
-; GCN-IR-NEXT:    v_xor_b32_e32 v0, v5, v2
-; GCN-IR-NEXT:    v_xor_b32_e32 v3, v6, v1
-; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, v0, v2
-; GCN-IR-NEXT:    v_subb_u32_e32 v1, vcc, v3, v1, vcc
+; GCN-IR-NEXT:    v_xor_b32_e32 v0, v2, v10
+; GCN-IR-NEXT:    v_xor_b32_e32 v1, v3, v11
+; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, v0, v10
+; GCN-IR-NEXT:    v_subb_u32_e32 v1, vcc, v1, v11, vcc
 ; GCN-IR-NEXT:    s_setpc_b64 s[30:31]
   %result = sdiv i64 %x, 32768
   ret i64 %result
diff --git a/llvm/test/CodeGen/AMDGPU/spill-empty-live-interval.mir b/llvm/test/CodeGen/AMDGPU/spill-empty-live-interval.mir
index b3b6319d70878d244e837e49f77b1491db7f02ec..9bac6bbd975957a8a6f75da1eb1a0a0b37935ad8 100644
--- a/llvm/test/CodeGen/AMDGPU/spill-empty-live-interval.mir
+++ b/llvm/test/CodeGen/AMDGPU/spill-empty-live-interval.mir
@@ -1,5 +1,5 @@
 # NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
-# RUN: llc -mtriple=amdgcn-amd-amdhsa -verify-machineinstrs -amdgpu-dce-in-ra=0 -stress-regalloc=1 -start-before=register-coalescer -stop-after=greedy,1 -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn-amd-amdhsa -verify-machineinstrs -amdgpu-enable-rewrite-partial-reg-uses=false -amdgpu-dce-in-ra=0 -stress-regalloc=1 -start-before=register-coalescer -stop-after=greedy,1 -o - %s | FileCheck %s
 # https://bugs.llvm.org/show_bug.cgi?id=33620
 
 ---
diff --git a/llvm/test/CodeGen/AMDGPU/spill-vgpr.ll b/llvm/test/CodeGen/AMDGPU/spill-vgpr.ll
index 314785cdbefd612122c685b1192e9e4ae82d7fc1..f1db395b3518248e08722643530b97560ba78d70 100644
--- a/llvm/test/CodeGen/AMDGPU/spill-vgpr.ll
+++ b/llvm/test/CodeGen/AMDGPU/spill-vgpr.ll
@@ -136,13 +136,13 @@ define amdgpu_kernel void @max_256_vgprs_spill_9x32(ptr addrspace(1) %p) #1 {
 ; GFX908-DAG: v_accvgpr_read_b32
 
 ; GFX900: NumVgprs: 256
-; GFX908: NumVgprs: 254
-; GFX900: ScratchSize: 1796
+; GFX908: NumVgprs: 252
+; GFX900: ScratchSize: 132
 ; GFX908: ScratchSize: 0
 ; GFX900: VGPRBlocks: 63
-; GFX908: VGPRBlocks: 63
+; GFX908: VGPRBlocks: 62
 ; GFX900: NumVGPRsForWavesPerEU: 256
-; GFX908: NumVGPRsForWavesPerEU: 254
+; GFX908: NumVGPRsForWavesPerEU: 252
 define amdgpu_kernel void @max_256_vgprs_spill_9x32_2bb(ptr addrspace(1) %p) #1 {
   %tid = call i32 @llvm.amdgcn.workitem.id.x()
   %p1 = getelementptr inbounds <32 x float>, ptr addrspace(1) %p, i32 %tid
diff --git a/llvm/test/CodeGen/AMDGPU/srem64.ll b/llvm/test/CodeGen/AMDGPU/srem64.ll
index bfb409438757c737ea4d5949dddd7e6edd9211b9..ac212d22e9cfa71b2f27f452651c1765d89efce5 100644
--- a/llvm/test/CodeGen/AMDGPU/srem64.ll
+++ b/llvm/test/CodeGen/AMDGPU/srem64.ll
@@ -135,34 +135,34 @@ define amdgpu_kernel void @s_test_srem(ptr addrspace(1) %out, i64 %x, i64 %y) {
 ; GCN-IR-NEXT:    s_add_i32 s6, s6, 32
 ; GCN-IR-NEXT:    s_flbit_i32_b32 s7, s3
 ; GCN-IR-NEXT:    s_min_u32 s10, s10, s11
-; GCN-IR-NEXT:    s_min_u32 s14, s6, s7
-; GCN-IR-NEXT:    s_sub_u32 s12, s10, s14
+; GCN-IR-NEXT:    s_min_u32 s18, s6, s7
+; GCN-IR-NEXT:    s_sub_u32 s12, s10, s18
 ; GCN-IR-NEXT:    s_subb_u32 s13, 0, 0
-; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[16:17], s[12:13], 63
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[18:19], s[12:13], 63
-; GCN-IR-NEXT:    s_or_b64 s[16:17], s[8:9], s[16:17]
-; GCN-IR-NEXT:    s_and_b64 s[8:9], s[16:17], exec
+; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[14:15], s[12:13], 63
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[16:17], s[12:13], 63
+; GCN-IR-NEXT:    s_or_b64 s[14:15], s[8:9], s[14:15]
+; GCN-IR-NEXT:    s_and_b64 s[8:9], s[14:15], exec
 ; GCN-IR-NEXT:    s_cselect_b32 s9, 0, s3
 ; GCN-IR-NEXT:    s_cselect_b32 s8, 0, s2
-; GCN-IR-NEXT:    s_or_b64 s[16:17], s[16:17], s[18:19]
+; GCN-IR-NEXT:    s_or_b64 s[14:15], s[14:15], s[16:17]
 ; GCN-IR-NEXT:    s_mov_b64 s[6:7], 0
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[16:17]
+; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[14:15]
 ; GCN-IR-NEXT:    s_mov_b32 s11, 0
 ; GCN-IR-NEXT:    s_cbranch_vccz .LBB0_5
 ; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
-; GCN-IR-NEXT:    s_add_u32 s16, s12, 1
-; GCN-IR-NEXT:    s_addc_u32 s17, s13, 0
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[8:9], s[16:17], 0
+; GCN-IR-NEXT:    s_add_u32 s14, s12, 1
+; GCN-IR-NEXT:    s_addc_u32 s15, s13, 0
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[8:9], s[14:15], 0
 ; GCN-IR-NEXT:    s_sub_i32 s12, 63, s12
 ; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[8:9]
 ; GCN-IR-NEXT:    s_lshl_b64 s[8:9], s[2:3], s12
 ; GCN-IR-NEXT:    s_cbranch_vccz .LBB0_4
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
-; GCN-IR-NEXT:    s_lshr_b64 s[12:13], s[2:3], s16
+; GCN-IR-NEXT:    s_lshr_b64 s[12:13], s[2:3], s14
 ; GCN-IR-NEXT:    s_add_u32 s16, s4, -1
 ; GCN-IR-NEXT:    s_addc_u32 s17, s5, -1
 ; GCN-IR-NEXT:    s_not_b64 s[6:7], s[10:11]
-; GCN-IR-NEXT:    s_add_u32 s10, s6, s14
+; GCN-IR-NEXT:    s_add_u32 s10, s6, s18
 ; GCN-IR-NEXT:    s_addc_u32 s11, s7, 0
 ; GCN-IR-NEXT:    s_mov_b64 s[14:15], 0
 ; GCN-IR-NEXT:    s_mov_b32 s7, 0
@@ -339,107 +339,107 @@ define i64 @v_test_srem(i64 %x, i64 %y) {
 ; GCN-IR-LABEL: v_test_srem:
 ; GCN-IR:       ; %bb.0: ; %_udiv-special-cases
 ; GCN-IR-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-IR-NEXT:    v_ashrrev_i32_e32 v4, 31, v1
-; GCN-IR-NEXT:    v_xor_b32_e32 v0, v0, v4
-; GCN-IR-NEXT:    v_ashrrev_i32_e32 v6, 31, v3
-; GCN-IR-NEXT:    v_xor_b32_e32 v1, v1, v4
-; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, v0, v4
-; GCN-IR-NEXT:    v_xor_b32_e32 v2, v2, v6
-; GCN-IR-NEXT:    v_subb_u32_e32 v1, vcc, v1, v4, vcc
-; GCN-IR-NEXT:    v_xor_b32_e32 v3, v3, v6
-; GCN-IR-NEXT:    v_sub_i32_e32 v2, vcc, v2, v6
-; GCN-IR-NEXT:    v_subb_u32_e32 v3, vcc, v3, v6, vcc
-; GCN-IR-NEXT:    v_ffbh_u32_e32 v6, v2
-; GCN-IR-NEXT:    v_add_i32_e64 v6, s[6:7], 32, v6
-; GCN-IR-NEXT:    v_ffbh_u32_e32 v7, v3
-; GCN-IR-NEXT:    v_min_u32_e32 v10, v6, v7
-; GCN-IR-NEXT:    v_ffbh_u32_e32 v6, v0
-; GCN-IR-NEXT:    v_add_i32_e64 v6, s[6:7], 32, v6
-; GCN-IR-NEXT:    v_ffbh_u32_e32 v7, v1
-; GCN-IR-NEXT:    v_min_u32_e32 v11, v6, v7
-; GCN-IR-NEXT:    v_sub_i32_e64 v6, s[6:7], v10, v11
+; GCN-IR-NEXT:    v_ashrrev_i32_e32 v14, 31, v1
+; GCN-IR-NEXT:    v_xor_b32_e32 v0, v0, v14
+; GCN-IR-NEXT:    v_ashrrev_i32_e32 v4, 31, v3
+; GCN-IR-NEXT:    v_xor_b32_e32 v1, v1, v14
+; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, v0, v14
+; GCN-IR-NEXT:    v_xor_b32_e32 v2, v2, v4
+; GCN-IR-NEXT:    v_subb_u32_e32 v1, vcc, v1, v14, vcc
+; GCN-IR-NEXT:    v_xor_b32_e32 v3, v3, v4
+; GCN-IR-NEXT:    v_sub_i32_e32 v2, vcc, v2, v4
+; GCN-IR-NEXT:    v_subb_u32_e32 v3, vcc, v3, v4, vcc
+; GCN-IR-NEXT:    v_ffbh_u32_e32 v4, v2
+; GCN-IR-NEXT:    v_add_i32_e64 v4, s[6:7], 32, v4
+; GCN-IR-NEXT:    v_ffbh_u32_e32 v5, v3
+; GCN-IR-NEXT:    v_min_u32_e32 v12, v4, v5
+; GCN-IR-NEXT:    v_ffbh_u32_e32 v4, v0
+; GCN-IR-NEXT:    v_add_i32_e64 v4, s[6:7], 32, v4
+; GCN-IR-NEXT:    v_ffbh_u32_e32 v5, v1
+; GCN-IR-NEXT:    v_min_u32_e32 v13, v4, v5
+; GCN-IR-NEXT:    v_sub_i32_e64 v4, s[6:7], v12, v13
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[2:3]
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[4:5], 0, v[0:1]
-; GCN-IR-NEXT:    v_subb_u32_e64 v7, s[6:7], 0, 0, s[6:7]
-; GCN-IR-NEXT:    v_cmp_lt_u64_e64 s[6:7], 63, v[6:7]
+; GCN-IR-NEXT:    v_subb_u32_e64 v5, s[6:7], 0, 0, s[6:7]
+; GCN-IR-NEXT:    v_cmp_lt_u64_e64 s[6:7], 63, v[4:5]
 ; GCN-IR-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
 ; GCN-IR-NEXT:    s_or_b64 s[4:5], s[4:5], s[6:7]
-; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 63, v[6:7]
+; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 63, v[4:5]
 ; GCN-IR-NEXT:    s_xor_b64 s[6:7], s[4:5], -1
-; GCN-IR-NEXT:    v_mov_b32_e32 v5, v4
-; GCN-IR-NEXT:    v_cndmask_b32_e64 v9, v1, 0, s[4:5]
-; GCN-IR-NEXT:    v_cndmask_b32_e64 v8, v0, 0, s[4:5]
+; GCN-IR-NEXT:    v_mov_b32_e32 v15, v14
+; GCN-IR-NEXT:    v_cndmask_b32_e64 v7, v1, 0, s[4:5]
+; GCN-IR-NEXT:    v_cndmask_b32_e64 v6, v0, 0, s[4:5]
 ; GCN-IR-NEXT:    s_and_b64 s[4:5], s[6:7], vcc
 ; GCN-IR-NEXT:    s_and_saveexec_b64 s[6:7], s[4:5]
 ; GCN-IR-NEXT:    s_cbranch_execz .LBB1_6
 ; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
-; GCN-IR-NEXT:    v_add_i32_e32 v12, vcc, 1, v6
-; GCN-IR-NEXT:    v_addc_u32_e32 v13, vcc, 0, v7, vcc
-; GCN-IR-NEXT:    v_sub_i32_e64 v6, s[4:5], 63, v6
-; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[12:13]
-; GCN-IR-NEXT:    v_lshl_b64 v[6:7], v[0:1], v6
-; GCN-IR-NEXT:    v_mov_b32_e32 v8, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, 0
+; GCN-IR-NEXT:    v_add_i32_e32 v8, vcc, 1, v4
+; GCN-IR-NEXT:    v_addc_u32_e32 v9, vcc, 0, v5, vcc
+; GCN-IR-NEXT:    v_sub_i32_e64 v4, s[4:5], 63, v4
+; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[8:9]
+; GCN-IR-NEXT:    v_lshl_b64 v[4:5], v[0:1], v4
+; GCN-IR-NEXT:    v_mov_b32_e32 v6, 0
+; GCN-IR-NEXT:    v_mov_b32_e32 v7, 0
 ; GCN-IR-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-IR-NEXT:    s_xor_b64 s[8:9], exec, s[4:5]
 ; GCN-IR-NEXT:    s_cbranch_execz .LBB1_5
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
 ; GCN-IR-NEXT:    v_add_i32_e32 v16, vcc, -1, v2
 ; GCN-IR-NEXT:    v_addc_u32_e32 v17, vcc, -1, v3, vcc
-; GCN-IR-NEXT:    v_not_b32_e32 v9, v10
-; GCN-IR-NEXT:    v_lshr_b64 v[12:13], v[0:1], v12
-; GCN-IR-NEXT:    v_not_b32_e32 v8, 0
-; GCN-IR-NEXT:    v_add_i32_e32 v10, vcc, v9, v11
-; GCN-IR-NEXT:    v_mov_b32_e32 v14, 0
-; GCN-IR-NEXT:    v_addc_u32_e32 v11, vcc, 0, v8, vcc
+; GCN-IR-NEXT:    v_not_b32_e32 v7, v12
+; GCN-IR-NEXT:    v_lshr_b64 v[10:11], v[0:1], v8
+; GCN-IR-NEXT:    v_not_b32_e32 v6, 0
+; GCN-IR-NEXT:    v_add_i32_e32 v8, vcc, v7, v13
+; GCN-IR-NEXT:    v_mov_b32_e32 v12, 0
+; GCN-IR-NEXT:    v_addc_u32_e32 v9, vcc, 0, v6, vcc
 ; GCN-IR-NEXT:    s_mov_b64 s[10:11], 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v15, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, 0
+; GCN-IR-NEXT:    v_mov_b32_e32 v13, 0
+; GCN-IR-NEXT:    v_mov_b32_e32 v7, 0
 ; GCN-IR-NEXT:  .LBB1_3: ; %udiv-do-while
 ; GCN-IR-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN-IR-NEXT:    v_lshl_b64 v[12:13], v[12:13], 1
-; GCN-IR-NEXT:    v_lshrrev_b32_e32 v8, 31, v7
-; GCN-IR-NEXT:    v_or_b32_e32 v12, v12, v8
-; GCN-IR-NEXT:    v_lshl_b64 v[6:7], v[6:7], 1
-; GCN-IR-NEXT:    v_sub_i32_e32 v8, vcc, v16, v12
-; GCN-IR-NEXT:    v_subb_u32_e32 v8, vcc, v17, v13, vcc
-; GCN-IR-NEXT:    v_or_b32_e32 v6, v14, v6
-; GCN-IR-NEXT:    v_ashrrev_i32_e32 v14, 31, v8
-; GCN-IR-NEXT:    v_add_i32_e32 v10, vcc, 1, v10
-; GCN-IR-NEXT:    v_or_b32_e32 v7, v15, v7
-; GCN-IR-NEXT:    v_and_b32_e32 v8, 1, v14
-; GCN-IR-NEXT:    v_and_b32_e32 v15, v14, v3
-; GCN-IR-NEXT:    v_and_b32_e32 v14, v14, v2
-; GCN-IR-NEXT:    v_addc_u32_e32 v11, vcc, 0, v11, vcc
-; GCN-IR-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[10:11]
-; GCN-IR-NEXT:    v_sub_i32_e64 v12, s[4:5], v12, v14
-; GCN-IR-NEXT:    v_subb_u32_e64 v13, s[4:5], v13, v15, s[4:5]
-; GCN-IR-NEXT:    v_mov_b32_e32 v15, v9
+; GCN-IR-NEXT:    v_lshl_b64 v[10:11], v[10:11], 1
+; GCN-IR-NEXT:    v_lshrrev_b32_e32 v6, 31, v5
+; GCN-IR-NEXT:    v_or_b32_e32 v10, v10, v6
+; GCN-IR-NEXT:    v_lshl_b64 v[4:5], v[4:5], 1
+; GCN-IR-NEXT:    v_sub_i32_e32 v6, vcc, v16, v10
+; GCN-IR-NEXT:    v_subb_u32_e32 v6, vcc, v17, v11, vcc
+; GCN-IR-NEXT:    v_or_b32_e32 v4, v12, v4
+; GCN-IR-NEXT:    v_ashrrev_i32_e32 v12, 31, v6
+; GCN-IR-NEXT:    v_add_i32_e32 v8, vcc, 1, v8
+; GCN-IR-NEXT:    v_or_b32_e32 v5, v13, v5
+; GCN-IR-NEXT:    v_and_b32_e32 v6, 1, v12
+; GCN-IR-NEXT:    v_and_b32_e32 v13, v12, v3
+; GCN-IR-NEXT:    v_and_b32_e32 v12, v12, v2
+; GCN-IR-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
+; GCN-IR-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[8:9]
+; GCN-IR-NEXT:    v_sub_i32_e64 v10, s[4:5], v10, v12
+; GCN-IR-NEXT:    v_subb_u32_e64 v11, s[4:5], v11, v13, s[4:5]
+; GCN-IR-NEXT:    v_mov_b32_e32 v13, v7
 ; GCN-IR-NEXT:    s_or_b64 s[10:11], vcc, s[10:11]
-; GCN-IR-NEXT:    v_mov_b32_e32 v14, v8
+; GCN-IR-NEXT:    v_mov_b32_e32 v12, v6
 ; GCN-IR-NEXT:    s_andn2_b64 exec, exec, s[10:11]
 ; GCN-IR-NEXT:    s_cbranch_execnz .LBB1_3
 ; GCN-IR-NEXT:  ; %bb.4: ; %Flow
 ; GCN-IR-NEXT:    s_or_b64 exec, exec, s[10:11]
 ; GCN-IR-NEXT:  .LBB1_5: ; %Flow4
 ; GCN-IR-NEXT:    s_or_b64 exec, exec, s[8:9]
-; GCN-IR-NEXT:    v_lshl_b64 v[6:7], v[6:7], 1
-; GCN-IR-NEXT:    v_or_b32_e32 v9, v9, v7
-; GCN-IR-NEXT:    v_or_b32_e32 v8, v8, v6
+; GCN-IR-NEXT:    v_lshl_b64 v[4:5], v[4:5], 1
+; GCN-IR-NEXT:    v_or_b32_e32 v7, v7, v5
+; GCN-IR-NEXT:    v_or_b32_e32 v6, v6, v4
 ; GCN-IR-NEXT:  .LBB1_6: ; %Flow5
 ; GCN-IR-NEXT:    s_or_b64 exec, exec, s[6:7]
-; GCN-IR-NEXT:    v_mul_lo_u32 v6, v2, v9
-; GCN-IR-NEXT:    v_mul_hi_u32 v7, v2, v8
-; GCN-IR-NEXT:    v_mul_lo_u32 v3, v3, v8
-; GCN-IR-NEXT:    v_mul_lo_u32 v2, v2, v8
-; GCN-IR-NEXT:    v_add_i32_e32 v6, vcc, v7, v6
-; GCN-IR-NEXT:    v_add_i32_e32 v3, vcc, v6, v3
+; GCN-IR-NEXT:    v_mul_lo_u32 v4, v2, v7
+; GCN-IR-NEXT:    v_mul_hi_u32 v5, v2, v6
+; GCN-IR-NEXT:    v_mul_lo_u32 v3, v3, v6
+; GCN-IR-NEXT:    v_mul_lo_u32 v2, v2, v6
+; GCN-IR-NEXT:    v_add_i32_e32 v4, vcc, v5, v4
+; GCN-IR-NEXT:    v_add_i32_e32 v3, vcc, v4, v3
 ; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, v0, v2
 ; GCN-IR-NEXT:    v_subb_u32_e32 v1, vcc, v1, v3, vcc
-; GCN-IR-NEXT:    v_xor_b32_e32 v0, v0, v4
-; GCN-IR-NEXT:    v_xor_b32_e32 v1, v1, v5
-; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, v0, v4
-; GCN-IR-NEXT:    v_subb_u32_e32 v1, vcc, v1, v5, vcc
+; GCN-IR-NEXT:    v_xor_b32_e32 v0, v0, v14
+; GCN-IR-NEXT:    v_xor_b32_e32 v1, v1, v15
+; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, v0, v14
+; GCN-IR-NEXT:    v_subb_u32_e32 v1, vcc, v1, v15, vcc
 ; GCN-IR-NEXT:    s_setpc_b64 s[30:31]
   %result = srem i64 %x, %y
   ret i64 %result
@@ -1037,33 +1037,33 @@ define amdgpu_kernel void @s_test_srem33_64(ptr addrspace(1) %out, i64 %x, i64 %
 ; GCN-IR-NEXT:    s_flbit_i32_b32 s6, s2
 ; GCN-IR-NEXT:    s_add_i32 s6, s6, 32
 ; GCN-IR-NEXT:    s_flbit_i32_b32 s7, s3
-; GCN-IR-NEXT:    s_min_u32 s16, s6, s7
-; GCN-IR-NEXT:    s_sub_u32 s14, s12, s16
+; GCN-IR-NEXT:    s_min_u32 s20, s6, s7
+; GCN-IR-NEXT:    s_sub_u32 s14, s12, s20
 ; GCN-IR-NEXT:    s_subb_u32 s15, 0, 0
-; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[18:19], s[14:15], 63
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[20:21], s[14:15], 63
-; GCN-IR-NEXT:    s_or_b64 s[18:19], s[10:11], s[18:19]
-; GCN-IR-NEXT:    s_and_b64 s[10:11], s[18:19], exec
+; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[16:17], s[14:15], 63
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[18:19], s[14:15], 63
+; GCN-IR-NEXT:    s_or_b64 s[16:17], s[10:11], s[16:17]
+; GCN-IR-NEXT:    s_and_b64 s[10:11], s[16:17], exec
 ; GCN-IR-NEXT:    s_cselect_b32 s11, 0, s3
 ; GCN-IR-NEXT:    s_cselect_b32 s10, 0, s2
-; GCN-IR-NEXT:    s_or_b64 s[18:19], s[18:19], s[20:21]
+; GCN-IR-NEXT:    s_or_b64 s[16:17], s[16:17], s[18:19]
 ; GCN-IR-NEXT:    s_mov_b64 s[6:7], 0
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[18:19]
+; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[16:17]
 ; GCN-IR-NEXT:    s_cbranch_vccz .LBB8_5
 ; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
-; GCN-IR-NEXT:    s_add_u32 s18, s14, 1
-; GCN-IR-NEXT:    s_addc_u32 s19, s15, 0
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[10:11], s[18:19], 0
+; GCN-IR-NEXT:    s_add_u32 s16, s14, 1
+; GCN-IR-NEXT:    s_addc_u32 s17, s15, 0
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[10:11], s[16:17], 0
 ; GCN-IR-NEXT:    s_sub_i32 s14, 63, s14
 ; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[10:11]
 ; GCN-IR-NEXT:    s_lshl_b64 s[10:11], s[2:3], s14
 ; GCN-IR-NEXT:    s_cbranch_vccz .LBB8_4
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
-; GCN-IR-NEXT:    s_lshr_b64 s[14:15], s[2:3], s18
+; GCN-IR-NEXT:    s_lshr_b64 s[14:15], s[2:3], s16
 ; GCN-IR-NEXT:    s_add_u32 s18, s8, -1
 ; GCN-IR-NEXT:    s_addc_u32 s19, s9, -1
 ; GCN-IR-NEXT:    s_not_b64 s[6:7], s[12:13]
-; GCN-IR-NEXT:    s_add_u32 s12, s6, s16
+; GCN-IR-NEXT:    s_add_u32 s12, s6, s20
 ; GCN-IR-NEXT:    s_addc_u32 s13, s7, 0
 ; GCN-IR-NEXT:    s_mov_b64 s[16:17], 0
 ; GCN-IR-NEXT:    s_mov_b32 s7, 0
@@ -1188,33 +1188,33 @@ define amdgpu_kernel void @s_test_srem24_48(ptr addrspace(1) %out, i48 %x, i48 %
 ; GCN-IR-NEXT:    s_flbit_i32_b32 s8, s4
 ; GCN-IR-NEXT:    s_add_i32 s8, s8, 32
 ; GCN-IR-NEXT:    s_flbit_i32_b32 s9, s5
-; GCN-IR-NEXT:    s_min_u32 s16, s8, s9
-; GCN-IR-NEXT:    s_sub_u32 s14, s12, s16
+; GCN-IR-NEXT:    s_min_u32 s20, s8, s9
+; GCN-IR-NEXT:    s_sub_u32 s14, s12, s20
 ; GCN-IR-NEXT:    s_subb_u32 s15, 0, 0
-; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[18:19], s[14:15], 63
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[20:21], s[14:15], 63
-; GCN-IR-NEXT:    s_or_b64 s[18:19], s[10:11], s[18:19]
-; GCN-IR-NEXT:    s_and_b64 s[10:11], s[18:19], exec
+; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[16:17], s[14:15], 63
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[18:19], s[14:15], 63
+; GCN-IR-NEXT:    s_or_b64 s[16:17], s[10:11], s[16:17]
+; GCN-IR-NEXT:    s_and_b64 s[10:11], s[16:17], exec
 ; GCN-IR-NEXT:    s_cselect_b32 s11, 0, s5
 ; GCN-IR-NEXT:    s_cselect_b32 s10, 0, s4
-; GCN-IR-NEXT:    s_or_b64 s[18:19], s[18:19], s[20:21]
+; GCN-IR-NEXT:    s_or_b64 s[16:17], s[16:17], s[18:19]
 ; GCN-IR-NEXT:    s_mov_b64 s[8:9], 0
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[18:19]
+; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[16:17]
 ; GCN-IR-NEXT:    s_cbranch_vccz .LBB9_5
 ; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
-; GCN-IR-NEXT:    s_add_u32 s18, s14, 1
-; GCN-IR-NEXT:    s_addc_u32 s19, s15, 0
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[10:11], s[18:19], 0
+; GCN-IR-NEXT:    s_add_u32 s16, s14, 1
+; GCN-IR-NEXT:    s_addc_u32 s17, s15, 0
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[10:11], s[16:17], 0
 ; GCN-IR-NEXT:    s_sub_i32 s14, 63, s14
 ; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[10:11]
 ; GCN-IR-NEXT:    s_lshl_b64 s[10:11], s[4:5], s14
 ; GCN-IR-NEXT:    s_cbranch_vccz .LBB9_4
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
-; GCN-IR-NEXT:    s_lshr_b64 s[14:15], s[4:5], s18
+; GCN-IR-NEXT:    s_lshr_b64 s[14:15], s[4:5], s16
 ; GCN-IR-NEXT:    s_add_u32 s18, s6, -1
 ; GCN-IR-NEXT:    s_addc_u32 s19, s7, -1
 ; GCN-IR-NEXT:    s_not_b64 s[8:9], s[12:13]
-; GCN-IR-NEXT:    s_add_u32 s12, s8, s16
+; GCN-IR-NEXT:    s_add_u32 s12, s8, s20
 ; GCN-IR-NEXT:    s_addc_u32 s13, s9, 0
 ; GCN-IR-NEXT:    s_mov_b64 s[16:17], 0
 ; GCN-IR-NEXT:    s_mov_b32 s9, 0
@@ -1396,32 +1396,32 @@ define amdgpu_kernel void @s_test_srem_k_num_i64(ptr addrspace(1) %out, i64 %x)
 ; GCN-IR-NEXT:    s_flbit_i32_b32 s2, s4
 ; GCN-IR-NEXT:    s_add_i32 s2, s2, 32
 ; GCN-IR-NEXT:    s_flbit_i32_b32 s3, s5
-; GCN-IR-NEXT:    s_min_u32 s8, s2, s3
-; GCN-IR-NEXT:    s_add_u32 s2, s8, 0xffffffc5
+; GCN-IR-NEXT:    s_min_u32 s12, s2, s3
+; GCN-IR-NEXT:    s_add_u32 s2, s12, 0xffffffc5
 ; GCN-IR-NEXT:    s_addc_u32 s3, 0, -1
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[10:11], s[4:5], 0
-; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[12:13], s[2:3], 63
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[8:9], s[4:5], 0
+; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[10:11], s[2:3], 63
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[14:15], s[2:3], 63
-; GCN-IR-NEXT:    s_or_b64 s[12:13], s[10:11], s[12:13]
-; GCN-IR-NEXT:    s_and_b64 s[10:11], s[12:13], exec
-; GCN-IR-NEXT:    s_cselect_b32 s10, 0, 24
-; GCN-IR-NEXT:    s_or_b64 s[12:13], s[12:13], s[14:15]
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[12:13]
-; GCN-IR-NEXT:    s_mov_b32 s11, 0
+; GCN-IR-NEXT:    s_or_b64 s[10:11], s[8:9], s[10:11]
+; GCN-IR-NEXT:    s_and_b64 s[8:9], s[10:11], exec
+; GCN-IR-NEXT:    s_cselect_b32 s8, 0, 24
+; GCN-IR-NEXT:    s_or_b64 s[10:11], s[10:11], s[14:15]
+; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[10:11]
+; GCN-IR-NEXT:    s_mov_b32 s9, 0
 ; GCN-IR-NEXT:    s_cbranch_vccz .LBB10_5
 ; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
-; GCN-IR-NEXT:    s_add_u32 s10, s2, 1
-; GCN-IR-NEXT:    s_addc_u32 s11, s3, 0
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[12:13], s[10:11], 0
+; GCN-IR-NEXT:    s_add_u32 s8, s2, 1
+; GCN-IR-NEXT:    s_addc_u32 s9, s3, 0
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[10:11], s[8:9], 0
 ; GCN-IR-NEXT:    s_sub_i32 s2, 63, s2
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[12:13]
+; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[10:11]
 ; GCN-IR-NEXT:    s_lshl_b64 s[2:3], 24, s2
 ; GCN-IR-NEXT:    s_cbranch_vccz .LBB10_4
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
-; GCN-IR-NEXT:    s_lshr_b64 s[10:11], 24, s10
+; GCN-IR-NEXT:    s_lshr_b64 s[10:11], 24, s8
 ; GCN-IR-NEXT:    s_add_u32 s14, s4, -1
 ; GCN-IR-NEXT:    s_addc_u32 s15, s5, -1
-; GCN-IR-NEXT:    s_sub_u32 s8, 58, s8
+; GCN-IR-NEXT:    s_sub_u32 s8, 58, s12
 ; GCN-IR-NEXT:    s_subb_u32 s9, 0, 0
 ; GCN-IR-NEXT:    s_mov_b64 s[12:13], 0
 ; GCN-IR-NEXT:    s_mov_b32 s7, 0
@@ -1448,13 +1448,13 @@ define amdgpu_kernel void @s_test_srem_k_num_i64(ptr addrspace(1) %out, i64 %x)
 ; GCN-IR-NEXT:    s_cbranch_vccz .LBB10_3
 ; GCN-IR-NEXT:  .LBB10_4: ; %Flow6
 ; GCN-IR-NEXT:    s_lshl_b64 s[2:3], s[2:3], 1
-; GCN-IR-NEXT:    s_or_b64 s[10:11], s[6:7], s[2:3]
+; GCN-IR-NEXT:    s_or_b64 s[8:9], s[6:7], s[2:3]
 ; GCN-IR-NEXT:  .LBB10_5: ; %udiv-end
-; GCN-IR-NEXT:    v_mov_b32_e32 v0, s10
+; GCN-IR-NEXT:    v_mov_b32_e32 v0, s8
 ; GCN-IR-NEXT:    v_mul_hi_u32 v0, s4, v0
-; GCN-IR-NEXT:    s_mul_i32 s6, s4, s11
-; GCN-IR-NEXT:    s_mul_i32 s5, s5, s10
-; GCN-IR-NEXT:    s_mul_i32 s4, s4, s10
+; GCN-IR-NEXT:    s_mul_i32 s6, s4, s9
+; GCN-IR-NEXT:    s_mul_i32 s5, s5, s8
+; GCN-IR-NEXT:    s_mul_i32 s4, s4, s8
 ; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, s6, v0
 ; GCN-IR-NEXT:    v_add_i32_e32 v1, vcc, s5, v0
 ; GCN-IR-NEXT:    v_sub_i32_e64 v0, vcc, 24, s4
@@ -1582,25 +1582,25 @@ define i64 @v_test_srem_k_num_i64(i64 %x) {
 ; GCN-IR-NEXT:    v_ffbh_u32_e32 v2, v0
 ; GCN-IR-NEXT:    v_add_i32_e32 v2, vcc, 32, v2
 ; GCN-IR-NEXT:    v_ffbh_u32_e32 v3, v1
-; GCN-IR-NEXT:    v_min_u32_e32 v6, v2, v3
+; GCN-IR-NEXT:    v_min_u32_e32 v10, v2, v3
 ; GCN-IR-NEXT:    s_movk_i32 s6, 0xffc5
-; GCN-IR-NEXT:    v_add_i32_e32 v3, vcc, s6, v6
-; GCN-IR-NEXT:    v_addc_u32_e64 v4, s[6:7], 0, -1, vcc
+; GCN-IR-NEXT:    v_add_i32_e32 v2, vcc, s6, v10
+; GCN-IR-NEXT:    v_addc_u32_e64 v3, s[6:7], 0, -1, vcc
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[4:5], 0, v[0:1]
-; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, 63, v[3:4]
-; GCN-IR-NEXT:    v_cmp_ne_u64_e64 s[6:7], 63, v[3:4]
+; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, 63, v[2:3]
+; GCN-IR-NEXT:    v_cmp_ne_u64_e64 s[6:7], 63, v[2:3]
 ; GCN-IR-NEXT:    s_or_b64 s[4:5], s[4:5], vcc
-; GCN-IR-NEXT:    v_cndmask_b32_e64 v5, 24, 0, s[4:5]
+; GCN-IR-NEXT:    v_cndmask_b32_e64 v4, 24, 0, s[4:5]
 ; GCN-IR-NEXT:    s_xor_b64 s[4:5], s[4:5], -1
-; GCN-IR-NEXT:    v_mov_b32_e32 v2, 0
+; GCN-IR-NEXT:    v_mov_b32_e32 v5, 0
 ; GCN-IR-NEXT:    s_and_b64 s[4:5], s[4:5], s[6:7]
 ; GCN-IR-NEXT:    s_and_saveexec_b64 s[6:7], s[4:5]
 ; GCN-IR-NEXT:    s_cbranch_execz .LBB11_6
 ; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
-; GCN-IR-NEXT:    v_add_i32_e32 v7, vcc, 1, v3
-; GCN-IR-NEXT:    v_addc_u32_e32 v8, vcc, 0, v4, vcc
-; GCN-IR-NEXT:    v_sub_i32_e64 v2, s[4:5], 63, v3
-; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[7:8]
+; GCN-IR-NEXT:    v_add_i32_e32 v6, vcc, 1, v2
+; GCN-IR-NEXT:    v_addc_u32_e32 v7, vcc, 0, v3, vcc
+; GCN-IR-NEXT:    v_sub_i32_e64 v2, s[4:5], 63, v2
+; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[6:7]
 ; GCN-IR-NEXT:    v_lshl_b64 v[2:3], 24, v2
 ; GCN-IR-NEXT:    v_mov_b32_e32 v4, 0
 ; GCN-IR-NEXT:    v_mov_b32_e32 v5, 0
@@ -1610,8 +1610,8 @@ define i64 @v_test_srem_k_num_i64(i64 %x) {
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
 ; GCN-IR-NEXT:    v_add_i32_e32 v12, vcc, -1, v0
 ; GCN-IR-NEXT:    v_addc_u32_e32 v13, vcc, -1, v1, vcc
-; GCN-IR-NEXT:    v_lshr_b64 v[8:9], 24, v7
-; GCN-IR-NEXT:    v_sub_i32_e32 v6, vcc, 58, v6
+; GCN-IR-NEXT:    v_lshr_b64 v[8:9], 24, v6
+; GCN-IR-NEXT:    v_sub_i32_e32 v6, vcc, 58, v10
 ; GCN-IR-NEXT:    v_mov_b32_e32 v10, 0
 ; GCN-IR-NEXT:    v_subb_u32_e64 v7, s[4:5], 0, 0, vcc
 ; GCN-IR-NEXT:    s_mov_b64 s[10:11], 0
@@ -1645,15 +1645,15 @@ define i64 @v_test_srem_k_num_i64(i64 %x) {
 ; GCN-IR-NEXT:    s_or_b64 exec, exec, s[10:11]
 ; GCN-IR-NEXT:  .LBB11_5: ; %Flow4
 ; GCN-IR-NEXT:    s_or_b64 exec, exec, s[8:9]
-; GCN-IR-NEXT:    v_lshl_b64 v[6:7], v[2:3], 1
-; GCN-IR-NEXT:    v_or_b32_e32 v2, v5, v7
-; GCN-IR-NEXT:    v_or_b32_e32 v5, v4, v6
+; GCN-IR-NEXT:    v_lshl_b64 v[2:3], v[2:3], 1
+; GCN-IR-NEXT:    v_or_b32_e32 v5, v5, v3
+; GCN-IR-NEXT:    v_or_b32_e32 v4, v4, v2
 ; GCN-IR-NEXT:  .LBB11_6: ; %Flow5
 ; GCN-IR-NEXT:    s_or_b64 exec, exec, s[6:7]
-; GCN-IR-NEXT:    v_mul_lo_u32 v2, v0, v2
-; GCN-IR-NEXT:    v_mul_hi_u32 v3, v0, v5
-; GCN-IR-NEXT:    v_mul_lo_u32 v1, v1, v5
-; GCN-IR-NEXT:    v_mul_lo_u32 v0, v0, v5
+; GCN-IR-NEXT:    v_mul_lo_u32 v2, v0, v5
+; GCN-IR-NEXT:    v_mul_hi_u32 v3, v0, v4
+; GCN-IR-NEXT:    v_mul_lo_u32 v1, v1, v4
+; GCN-IR-NEXT:    v_mul_lo_u32 v0, v0, v4
 ; GCN-IR-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
 ; GCN-IR-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
 ; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, 24, v0
@@ -1773,27 +1773,27 @@ define i64 @v_test_srem_pow2_k_num_i64(i64 %x) {
 ; GCN-IR-NEXT:    v_ffbh_u32_e32 v2, v0
 ; GCN-IR-NEXT:    v_add_i32_e32 v2, vcc, 32, v2
 ; GCN-IR-NEXT:    v_ffbh_u32_e32 v3, v1
-; GCN-IR-NEXT:    v_min_u32_e32 v6, v2, v3
+; GCN-IR-NEXT:    v_min_u32_e32 v10, v2, v3
 ; GCN-IR-NEXT:    s_movk_i32 s6, 0xffd0
-; GCN-IR-NEXT:    v_add_i32_e32 v3, vcc, s6, v6
-; GCN-IR-NEXT:    v_addc_u32_e64 v4, s[6:7], 0, -1, vcc
+; GCN-IR-NEXT:    v_add_i32_e32 v2, vcc, s6, v10
+; GCN-IR-NEXT:    v_addc_u32_e64 v3, s[6:7], 0, -1, vcc
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[4:5], 0, v[0:1]
-; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, 63, v[3:4]
-; GCN-IR-NEXT:    v_cmp_ne_u64_e64 s[6:7], 63, v[3:4]
-; GCN-IR-NEXT:    v_mov_b32_e32 v5, 0x8000
+; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, 63, v[2:3]
+; GCN-IR-NEXT:    v_cmp_ne_u64_e64 s[6:7], 63, v[2:3]
+; GCN-IR-NEXT:    v_mov_b32_e32 v4, 0x8000
 ; GCN-IR-NEXT:    s_or_b64 s[4:5], s[4:5], vcc
-; GCN-IR-NEXT:    v_cndmask_b32_e64 v5, v5, 0, s[4:5]
+; GCN-IR-NEXT:    v_cndmask_b32_e64 v4, v4, 0, s[4:5]
 ; GCN-IR-NEXT:    s_xor_b64 s[4:5], s[4:5], -1
-; GCN-IR-NEXT:    v_mov_b32_e32 v2, 0
+; GCN-IR-NEXT:    v_mov_b32_e32 v5, 0
 ; GCN-IR-NEXT:    s_and_b64 s[4:5], s[4:5], s[6:7]
 ; GCN-IR-NEXT:    s_and_saveexec_b64 s[6:7], s[4:5]
 ; GCN-IR-NEXT:    s_cbranch_execz .LBB12_6
 ; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
-; GCN-IR-NEXT:    v_add_i32_e32 v7, vcc, 1, v3
-; GCN-IR-NEXT:    v_sub_i32_e64 v2, s[4:5], 63, v3
-; GCN-IR-NEXT:    v_addc_u32_e32 v8, vcc, 0, v4, vcc
+; GCN-IR-NEXT:    v_add_i32_e32 v6, vcc, 1, v2
+; GCN-IR-NEXT:    v_sub_i32_e64 v2, s[4:5], 63, v2
+; GCN-IR-NEXT:    v_addc_u32_e32 v7, vcc, 0, v3, vcc
 ; GCN-IR-NEXT:    s_mov_b64 s[4:5], 0x8000
-; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[7:8]
+; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[6:7]
 ; GCN-IR-NEXT:    v_lshl_b64 v[2:3], s[4:5], v2
 ; GCN-IR-NEXT:    v_mov_b32_e32 v4, 0
 ; GCN-IR-NEXT:    v_mov_b32_e32 v5, 0
@@ -1803,8 +1803,8 @@ define i64 @v_test_srem_pow2_k_num_i64(i64 %x) {
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
 ; GCN-IR-NEXT:    v_add_i32_e32 v12, vcc, -1, v0
 ; GCN-IR-NEXT:    v_addc_u32_e32 v13, vcc, -1, v1, vcc
-; GCN-IR-NEXT:    v_lshr_b64 v[8:9], s[4:5], v7
-; GCN-IR-NEXT:    v_sub_i32_e32 v6, vcc, 47, v6
+; GCN-IR-NEXT:    v_lshr_b64 v[8:9], s[4:5], v6
+; GCN-IR-NEXT:    v_sub_i32_e32 v6, vcc, 47, v10
 ; GCN-IR-NEXT:    v_mov_b32_e32 v10, 0
 ; GCN-IR-NEXT:    v_subb_u32_e64 v7, s[4:5], 0, 0, vcc
 ; GCN-IR-NEXT:    s_mov_b64 s[10:11], 0
@@ -1838,15 +1838,15 @@ define i64 @v_test_srem_pow2_k_num_i64(i64 %x) {
 ; GCN-IR-NEXT:    s_or_b64 exec, exec, s[10:11]
 ; GCN-IR-NEXT:  .LBB12_5: ; %Flow4
 ; GCN-IR-NEXT:    s_or_b64 exec, exec, s[8:9]
-; GCN-IR-NEXT:    v_lshl_b64 v[6:7], v[2:3], 1
-; GCN-IR-NEXT:    v_or_b32_e32 v2, v5, v7
-; GCN-IR-NEXT:    v_or_b32_e32 v5, v4, v6
+; GCN-IR-NEXT:    v_lshl_b64 v[2:3], v[2:3], 1
+; GCN-IR-NEXT:    v_or_b32_e32 v5, v5, v3
+; GCN-IR-NEXT:    v_or_b32_e32 v4, v4, v2
 ; GCN-IR-NEXT:  .LBB12_6: ; %Flow5
 ; GCN-IR-NEXT:    s_or_b64 exec, exec, s[6:7]
-; GCN-IR-NEXT:    v_mul_lo_u32 v2, v0, v2
-; GCN-IR-NEXT:    v_mul_hi_u32 v3, v0, v5
-; GCN-IR-NEXT:    v_mul_lo_u32 v1, v1, v5
-; GCN-IR-NEXT:    v_mul_lo_u32 v0, v0, v5
+; GCN-IR-NEXT:    v_mul_lo_u32 v2, v0, v5
+; GCN-IR-NEXT:    v_mul_hi_u32 v3, v0, v4
+; GCN-IR-NEXT:    v_mul_lo_u32 v1, v1, v4
+; GCN-IR-NEXT:    v_mul_lo_u32 v0, v0, v4
 ; GCN-IR-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
 ; GCN-IR-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
 ; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, 0x8000, v0
@@ -1872,87 +1872,87 @@ define i64 @v_test_srem_pow2_k_den_i64(i64 %x) {
 ; GCN-IR-LABEL: v_test_srem_pow2_k_den_i64:
 ; GCN-IR:       ; %bb.0: ; %_udiv-special-cases
 ; GCN-IR-NEXT:    s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GCN-IR-NEXT:    v_ashrrev_i32_e32 v2, 31, v1
-; GCN-IR-NEXT:    v_xor_b32_e32 v0, v0, v2
-; GCN-IR-NEXT:    v_xor_b32_e32 v1, v1, v2
-; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, v0, v2
-; GCN-IR-NEXT:    v_subb_u32_e32 v1, vcc, v1, v2, vcc
-; GCN-IR-NEXT:    v_ffbh_u32_e32 v3, v0
-; GCN-IR-NEXT:    v_add_i32_e64 v3, s[4:5], 32, v3
-; GCN-IR-NEXT:    v_ffbh_u32_e32 v4, v1
-; GCN-IR-NEXT:    v_min_u32_e32 v8, v3, v4
-; GCN-IR-NEXT:    v_sub_i32_e64 v4, s[4:5], 48, v8
-; GCN-IR-NEXT:    v_subb_u32_e64 v5, s[4:5], 0, 0, s[4:5]
+; GCN-IR-NEXT:    v_ashrrev_i32_e32 v12, 31, v1
+; GCN-IR-NEXT:    v_xor_b32_e32 v0, v0, v12
+; GCN-IR-NEXT:    v_xor_b32_e32 v1, v1, v12
+; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, v0, v12
+; GCN-IR-NEXT:    v_subb_u32_e32 v1, vcc, v1, v12, vcc
+; GCN-IR-NEXT:    v_ffbh_u32_e32 v2, v0
+; GCN-IR-NEXT:    v_add_i32_e64 v2, s[4:5], 32, v2
+; GCN-IR-NEXT:    v_ffbh_u32_e32 v3, v1
+; GCN-IR-NEXT:    v_min_u32_e32 v10, v2, v3
+; GCN-IR-NEXT:    v_sub_i32_e64 v2, s[4:5], 48, v10
+; GCN-IR-NEXT:    v_subb_u32_e64 v3, s[4:5], 0, 0, s[4:5]
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[0:1]
-; GCN-IR-NEXT:    v_cmp_lt_u64_e64 s[4:5], 63, v[4:5]
-; GCN-IR-NEXT:    v_mov_b32_e32 v3, v2
+; GCN-IR-NEXT:    v_cmp_lt_u64_e64 s[4:5], 63, v[2:3]
+; GCN-IR-NEXT:    v_mov_b32_e32 v13, v12
 ; GCN-IR-NEXT:    s_or_b64 s[4:5], vcc, s[4:5]
-; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 63, v[4:5]
+; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 63, v[2:3]
 ; GCN-IR-NEXT:    s_xor_b64 s[6:7], s[4:5], -1
-; GCN-IR-NEXT:    v_cndmask_b32_e64 v7, v1, 0, s[4:5]
-; GCN-IR-NEXT:    v_cndmask_b32_e64 v6, v0, 0, s[4:5]
+; GCN-IR-NEXT:    v_cndmask_b32_e64 v5, v1, 0, s[4:5]
+; GCN-IR-NEXT:    v_cndmask_b32_e64 v4, v0, 0, s[4:5]
 ; GCN-IR-NEXT:    s_and_b64 s[4:5], s[6:7], vcc
 ; GCN-IR-NEXT:    s_and_saveexec_b64 s[6:7], s[4:5]
 ; GCN-IR-NEXT:    s_cbranch_execz .LBB13_6
 ; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
-; GCN-IR-NEXT:    v_add_i32_e32 v9, vcc, 1, v4
-; GCN-IR-NEXT:    v_addc_u32_e32 v10, vcc, 0, v5, vcc
-; GCN-IR-NEXT:    v_sub_i32_e64 v4, s[4:5], 63, v4
-; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[9:10]
-; GCN-IR-NEXT:    v_lshl_b64 v[4:5], v[0:1], v4
-; GCN-IR-NEXT:    v_mov_b32_e32 v6, 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v7, 0
+; GCN-IR-NEXT:    v_add_i32_e32 v6, vcc, 1, v2
+; GCN-IR-NEXT:    v_addc_u32_e32 v7, vcc, 0, v3, vcc
+; GCN-IR-NEXT:    v_sub_i32_e64 v2, s[4:5], 63, v2
+; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[6:7]
+; GCN-IR-NEXT:    v_lshl_b64 v[2:3], v[0:1], v2
+; GCN-IR-NEXT:    v_mov_b32_e32 v4, 0
+; GCN-IR-NEXT:    v_mov_b32_e32 v5, 0
 ; GCN-IR-NEXT:    s_and_saveexec_b64 s[4:5], vcc
 ; GCN-IR-NEXT:    s_xor_b64 s[8:9], exec, s[4:5]
 ; GCN-IR-NEXT:    s_cbranch_execz .LBB13_5
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
-; GCN-IR-NEXT:    v_lshr_b64 v[10:11], v[0:1], v9
-; GCN-IR-NEXT:    v_add_i32_e32 v8, vcc, 0xffffffcf, v8
-; GCN-IR-NEXT:    v_mov_b32_e32 v12, 0
-; GCN-IR-NEXT:    v_addc_u32_e64 v9, s[4:5], 0, -1, vcc
+; GCN-IR-NEXT:    v_lshr_b64 v[8:9], v[0:1], v6
+; GCN-IR-NEXT:    v_add_i32_e32 v6, vcc, 0xffffffcf, v10
+; GCN-IR-NEXT:    v_mov_b32_e32 v10, 0
+; GCN-IR-NEXT:    v_addc_u32_e64 v7, s[4:5], 0, -1, vcc
 ; GCN-IR-NEXT:    s_mov_b64 s[10:11], 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v13, 0
+; GCN-IR-NEXT:    v_mov_b32_e32 v11, 0
 ; GCN-IR-NEXT:    s_movk_i32 s12, 0x7fff
-; GCN-IR-NEXT:    v_mov_b32_e32 v7, 0
+; GCN-IR-NEXT:    v_mov_b32_e32 v5, 0
 ; GCN-IR-NEXT:  .LBB13_3: ; %udiv-do-while
 ; GCN-IR-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN-IR-NEXT:    v_lshl_b64 v[10:11], v[10:11], 1
-; GCN-IR-NEXT:    v_lshrrev_b32_e32 v6, 31, v5
-; GCN-IR-NEXT:    v_or_b32_e32 v10, v10, v6
-; GCN-IR-NEXT:    v_sub_i32_e32 v6, vcc, s12, v10
-; GCN-IR-NEXT:    v_lshl_b64 v[4:5], v[4:5], 1
-; GCN-IR-NEXT:    v_subb_u32_e32 v6, vcc, 0, v11, vcc
-; GCN-IR-NEXT:    v_add_i32_e32 v8, vcc, 1, v8
-; GCN-IR-NEXT:    v_or_b32_e32 v4, v12, v4
-; GCN-IR-NEXT:    v_ashrrev_i32_e32 v12, 31, v6
-; GCN-IR-NEXT:    v_addc_u32_e32 v9, vcc, 0, v9, vcc
-; GCN-IR-NEXT:    v_and_b32_e32 v6, 1, v12
-; GCN-IR-NEXT:    v_and_b32_e32 v12, 0x8000, v12
-; GCN-IR-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[8:9]
-; GCN-IR-NEXT:    v_or_b32_e32 v5, v13, v5
-; GCN-IR-NEXT:    v_sub_i32_e64 v10, s[4:5], v10, v12
-; GCN-IR-NEXT:    v_mov_b32_e32 v13, v7
-; GCN-IR-NEXT:    v_subbrev_u32_e64 v11, s[4:5], 0, v11, s[4:5]
+; GCN-IR-NEXT:    v_lshl_b64 v[8:9], v[8:9], 1
+; GCN-IR-NEXT:    v_lshrrev_b32_e32 v4, 31, v3
+; GCN-IR-NEXT:    v_or_b32_e32 v8, v8, v4
+; GCN-IR-NEXT:    v_sub_i32_e32 v4, vcc, s12, v8
+; GCN-IR-NEXT:    v_lshl_b64 v[2:3], v[2:3], 1
+; GCN-IR-NEXT:    v_subb_u32_e32 v4, vcc, 0, v9, vcc
+; GCN-IR-NEXT:    v_add_i32_e32 v6, vcc, 1, v6
+; GCN-IR-NEXT:    v_or_b32_e32 v2, v10, v2
+; GCN-IR-NEXT:    v_ashrrev_i32_e32 v10, 31, v4
+; GCN-IR-NEXT:    v_addc_u32_e32 v7, vcc, 0, v7, vcc
+; GCN-IR-NEXT:    v_and_b32_e32 v4, 1, v10
+; GCN-IR-NEXT:    v_and_b32_e32 v10, 0x8000, v10
+; GCN-IR-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[6:7]
+; GCN-IR-NEXT:    v_or_b32_e32 v3, v11, v3
+; GCN-IR-NEXT:    v_sub_i32_e64 v8, s[4:5], v8, v10
+; GCN-IR-NEXT:    v_mov_b32_e32 v11, v5
+; GCN-IR-NEXT:    v_subbrev_u32_e64 v9, s[4:5], 0, v9, s[4:5]
 ; GCN-IR-NEXT:    s_or_b64 s[10:11], vcc, s[10:11]
-; GCN-IR-NEXT:    v_mov_b32_e32 v12, v6
+; GCN-IR-NEXT:    v_mov_b32_e32 v10, v4
 ; GCN-IR-NEXT:    s_andn2_b64 exec, exec, s[10:11]
 ; GCN-IR-NEXT:    s_cbranch_execnz .LBB13_3
 ; GCN-IR-NEXT:  ; %bb.4: ; %Flow
 ; GCN-IR-NEXT:    s_or_b64 exec, exec, s[10:11]
 ; GCN-IR-NEXT:  .LBB13_5: ; %Flow4
 ; GCN-IR-NEXT:    s_or_b64 exec, exec, s[8:9]
-; GCN-IR-NEXT:    v_lshl_b64 v[4:5], v[4:5], 1
-; GCN-IR-NEXT:    v_or_b32_e32 v7, v7, v5
-; GCN-IR-NEXT:    v_or_b32_e32 v6, v6, v4
+; GCN-IR-NEXT:    v_lshl_b64 v[2:3], v[2:3], 1
+; GCN-IR-NEXT:    v_or_b32_e32 v5, v5, v3
+; GCN-IR-NEXT:    v_or_b32_e32 v4, v4, v2
 ; GCN-IR-NEXT:  .LBB13_6: ; %Flow5
 ; GCN-IR-NEXT:    s_or_b64 exec, exec, s[6:7]
-; GCN-IR-NEXT:    v_lshl_b64 v[4:5], v[6:7], 15
-; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, v0, v4
-; GCN-IR-NEXT:    v_subb_u32_e32 v1, vcc, v1, v5, vcc
-; GCN-IR-NEXT:    v_xor_b32_e32 v0, v0, v2
-; GCN-IR-NEXT:    v_xor_b32_e32 v1, v1, v3
+; GCN-IR-NEXT:    v_lshl_b64 v[2:3], v[4:5], 15
 ; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, v0, v2
 ; GCN-IR-NEXT:    v_subb_u32_e32 v1, vcc, v1, v3, vcc
+; GCN-IR-NEXT:    v_xor_b32_e32 v0, v0, v12
+; GCN-IR-NEXT:    v_xor_b32_e32 v1, v1, v13
+; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, v0, v12
+; GCN-IR-NEXT:    v_subb_u32_e32 v1, vcc, v1, v13, vcc
 ; GCN-IR-NEXT:    s_setpc_b64 s[30:31]
   %result = srem i64 %x, 32768
   ret i64 %result
diff --git a/llvm/test/CodeGen/AMDGPU/udiv64.ll b/llvm/test/CodeGen/AMDGPU/udiv64.ll
index 6102d652402a21030503dc0abe3226e6a8815ffa..9301170c034d89d8e21a9f2bb9740f8df8e7dc02 100644
--- a/llvm/test/CodeGen/AMDGPU/udiv64.ll
+++ b/llvm/test/CodeGen/AMDGPU/udiv64.ll
@@ -136,34 +136,34 @@ define amdgpu_kernel void @s_test_udiv_i64(ptr addrspace(1) %out, i64 %x, i64 %y
 ; GCN-IR-NEXT:    s_add_i32 s6, s6, 32
 ; GCN-IR-NEXT:    s_flbit_i32_b32 s7, s3
 ; GCN-IR-NEXT:    s_min_u32 s10, s10, s11
-; GCN-IR-NEXT:    s_min_u32 s14, s6, s7
-; GCN-IR-NEXT:    s_sub_u32 s12, s10, s14
+; GCN-IR-NEXT:    s_min_u32 s16, s6, s7
+; GCN-IR-NEXT:    s_sub_u32 s12, s10, s16
 ; GCN-IR-NEXT:    s_subb_u32 s13, 0, 0
-; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[16:17], s[12:13], 63
+; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[14:15], s[12:13], 63
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[18:19], s[12:13], 63
-; GCN-IR-NEXT:    s_or_b64 s[16:17], s[8:9], s[16:17]
-; GCN-IR-NEXT:    s_and_b64 s[8:9], s[16:17], exec
+; GCN-IR-NEXT:    s_or_b64 s[14:15], s[8:9], s[14:15]
+; GCN-IR-NEXT:    s_and_b64 s[8:9], s[14:15], exec
 ; GCN-IR-NEXT:    s_cselect_b32 s9, 0, s3
 ; GCN-IR-NEXT:    s_cselect_b32 s8, 0, s2
-; GCN-IR-NEXT:    s_or_b64 s[16:17], s[16:17], s[18:19]
+; GCN-IR-NEXT:    s_or_b64 s[14:15], s[14:15], s[18:19]
 ; GCN-IR-NEXT:    s_mov_b64 s[6:7], 0
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[16:17]
+; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[14:15]
 ; GCN-IR-NEXT:    s_mov_b32 s11, 0
 ; GCN-IR-NEXT:    s_cbranch_vccz .LBB0_5
 ; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
-; GCN-IR-NEXT:    s_add_u32 s16, s12, 1
-; GCN-IR-NEXT:    s_addc_u32 s17, s13, 0
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[8:9], s[16:17], 0
+; GCN-IR-NEXT:    s_add_u32 s14, s12, 1
+; GCN-IR-NEXT:    s_addc_u32 s15, s13, 0
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[8:9], s[14:15], 0
 ; GCN-IR-NEXT:    s_sub_i32 s12, 63, s12
 ; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[8:9]
 ; GCN-IR-NEXT:    s_lshl_b64 s[8:9], s[2:3], s12
 ; GCN-IR-NEXT:    s_cbranch_vccz .LBB0_4
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
-; GCN-IR-NEXT:    s_lshr_b64 s[12:13], s[2:3], s16
-; GCN-IR-NEXT:    s_add_u32 s15, s4, -1
-; GCN-IR-NEXT:    s_addc_u32 s16, s5, -1
+; GCN-IR-NEXT:    s_lshr_b64 s[12:13], s[2:3], s14
+; GCN-IR-NEXT:    s_add_u32 s14, s4, -1
+; GCN-IR-NEXT:    s_addc_u32 s15, s5, -1
 ; GCN-IR-NEXT:    s_not_b64 s[2:3], s[10:11]
-; GCN-IR-NEXT:    s_add_u32 s2, s2, s14
+; GCN-IR-NEXT:    s_add_u32 s2, s2, s16
 ; GCN-IR-NEXT:    s_addc_u32 s3, s3, 0
 ; GCN-IR-NEXT:    s_mov_b64 s[10:11], 0
 ; GCN-IR-NEXT:    s_mov_b32 s7, 0
@@ -174,8 +174,8 @@ define amdgpu_kernel void @s_test_udiv_i64(ptr addrspace(1) %out, i64 %x, i64 %y
 ; GCN-IR-NEXT:    s_lshl_b64 s[8:9], s[8:9], 1
 ; GCN-IR-NEXT:    s_or_b64 s[12:13], s[12:13], s[6:7]
 ; GCN-IR-NEXT:    s_or_b64 s[8:9], s[10:11], s[8:9]
-; GCN-IR-NEXT:    s_sub_u32 s6, s15, s12
-; GCN-IR-NEXT:    s_subb_u32 s6, s16, s13
+; GCN-IR-NEXT:    s_sub_u32 s6, s14, s12
+; GCN-IR-NEXT:    s_subb_u32 s6, s15, s13
 ; GCN-IR-NEXT:    s_ashr_i32 s10, s6, 31
 ; GCN-IR-NEXT:    s_mov_b32 s11, s10
 ; GCN-IR-NEXT:    s_and_b32 s6, s10, 1
@@ -184,9 +184,9 @@ define amdgpu_kernel void @s_test_udiv_i64(ptr addrspace(1) %out, i64 %x, i64 %y
 ; GCN-IR-NEXT:    s_subb_u32 s13, s13, s11
 ; GCN-IR-NEXT:    s_add_u32 s2, s2, 1
 ; GCN-IR-NEXT:    s_addc_u32 s3, s3, 0
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[18:19], s[2:3], 0
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[16:17], s[2:3], 0
 ; GCN-IR-NEXT:    s_mov_b64 s[10:11], s[6:7]
-; GCN-IR-NEXT:    s_and_b64 vcc, exec, s[18:19]
+; GCN-IR-NEXT:    s_and_b64 vcc, exec, s[16:17]
 ; GCN-IR-NEXT:    s_cbranch_vccz .LBB0_3
 ; GCN-IR-NEXT:  .LBB0_4: ; %Flow7
 ; GCN-IR-NEXT:    s_lshl_b64 s[2:3], s[8:9], 1
@@ -319,12 +319,12 @@ define i64 @v_test_udiv_i64(i64 %x, i64 %y) {
 ; GCN-IR-NEXT:    v_ffbh_u32_e32 v4, v2
 ; GCN-IR-NEXT:    v_add_i32_e64 v4, s[6:7], 32, v4
 ; GCN-IR-NEXT:    v_ffbh_u32_e32 v5, v3
-; GCN-IR-NEXT:    v_min_u32_e32 v8, v4, v5
+; GCN-IR-NEXT:    v_min_u32_e32 v10, v4, v5
 ; GCN-IR-NEXT:    v_ffbh_u32_e32 v4, v0
 ; GCN-IR-NEXT:    v_add_i32_e64 v4, s[6:7], 32, v4
 ; GCN-IR-NEXT:    v_ffbh_u32_e32 v5, v1
-; GCN-IR-NEXT:    v_min_u32_e32 v9, v4, v5
-; GCN-IR-NEXT:    v_sub_i32_e64 v6, s[6:7], v8, v9
+; GCN-IR-NEXT:    v_min_u32_e32 v11, v4, v5
+; GCN-IR-NEXT:    v_sub_i32_e64 v6, s[6:7], v10, v11
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[2:3]
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[4:5], 0, v[0:1]
 ; GCN-IR-NEXT:    v_subb_u32_e64 v7, s[6:7], 0, 0, s[6:7]
@@ -339,10 +339,10 @@ define i64 @v_test_udiv_i64(i64 %x, i64 %y) {
 ; GCN-IR-NEXT:    s_and_saveexec_b64 s[6:7], s[4:5]
 ; GCN-IR-NEXT:    s_cbranch_execz .LBB1_6
 ; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
-; GCN-IR-NEXT:    v_add_i32_e32 v10, vcc, 1, v6
-; GCN-IR-NEXT:    v_addc_u32_e32 v11, vcc, 0, v7, vcc
+; GCN-IR-NEXT:    v_add_i32_e32 v8, vcc, 1, v6
+; GCN-IR-NEXT:    v_addc_u32_e32 v9, vcc, 0, v7, vcc
 ; GCN-IR-NEXT:    v_sub_i32_e64 v4, s[4:5], 63, v6
-; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[10:11]
+; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[8:9]
 ; GCN-IR-NEXT:    v_lshl_b64 v[4:5], v[0:1], v4
 ; GCN-IR-NEXT:    v_mov_b32_e32 v6, 0
 ; GCN-IR-NEXT:    v_mov_b32_e32 v7, 0
@@ -351,38 +351,38 @@ define i64 @v_test_udiv_i64(i64 %x, i64 %y) {
 ; GCN-IR-NEXT:    s_cbranch_execz .LBB1_5
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
 ; GCN-IR-NEXT:    v_add_i32_e32 v12, vcc, -1, v2
-; GCN-IR-NEXT:    v_lshr_b64 v[10:11], v[0:1], v10
+; GCN-IR-NEXT:    v_lshr_b64 v[8:9], v[0:1], v8
 ; GCN-IR-NEXT:    v_addc_u32_e32 v13, vcc, -1, v3, vcc
-; GCN-IR-NEXT:    v_not_b32_e32 v0, v8
+; GCN-IR-NEXT:    v_not_b32_e32 v0, v10
 ; GCN-IR-NEXT:    v_not_b32_e32 v1, 0
-; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, v0, v9
-; GCN-IR-NEXT:    v_mov_b32_e32 v8, 0
+; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, v0, v11
+; GCN-IR-NEXT:    v_mov_b32_e32 v10, 0
 ; GCN-IR-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; GCN-IR-NEXT:    s_mov_b64 s[10:11], 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, 0
+; GCN-IR-NEXT:    v_mov_b32_e32 v11, 0
 ; GCN-IR-NEXT:    v_mov_b32_e32 v7, 0
 ; GCN-IR-NEXT:  .LBB1_3: ; %udiv-do-while
 ; GCN-IR-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN-IR-NEXT:    v_lshl_b64 v[10:11], v[10:11], 1
+; GCN-IR-NEXT:    v_lshl_b64 v[8:9], v[8:9], 1
 ; GCN-IR-NEXT:    v_lshrrev_b32_e32 v6, 31, v5
-; GCN-IR-NEXT:    v_or_b32_e32 v10, v10, v6
+; GCN-IR-NEXT:    v_or_b32_e32 v8, v8, v6
 ; GCN-IR-NEXT:    v_lshl_b64 v[4:5], v[4:5], 1
-; GCN-IR-NEXT:    v_sub_i32_e32 v6, vcc, v12, v10
-; GCN-IR-NEXT:    v_subb_u32_e32 v6, vcc, v13, v11, vcc
-; GCN-IR-NEXT:    v_or_b32_e32 v4, v8, v4
-; GCN-IR-NEXT:    v_ashrrev_i32_e32 v8, 31, v6
+; GCN-IR-NEXT:    v_sub_i32_e32 v6, vcc, v12, v8
+; GCN-IR-NEXT:    v_subb_u32_e32 v6, vcc, v13, v9, vcc
+; GCN-IR-NEXT:    v_or_b32_e32 v4, v10, v4
+; GCN-IR-NEXT:    v_ashrrev_i32_e32 v10, 31, v6
 ; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, 1, v0
-; GCN-IR-NEXT:    v_or_b32_e32 v5, v9, v5
-; GCN-IR-NEXT:    v_and_b32_e32 v6, 1, v8
-; GCN-IR-NEXT:    v_and_b32_e32 v9, v8, v3
-; GCN-IR-NEXT:    v_and_b32_e32 v8, v8, v2
+; GCN-IR-NEXT:    v_or_b32_e32 v5, v11, v5
+; GCN-IR-NEXT:    v_and_b32_e32 v6, 1, v10
+; GCN-IR-NEXT:    v_and_b32_e32 v11, v10, v3
+; GCN-IR-NEXT:    v_and_b32_e32 v10, v10, v2
 ; GCN-IR-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[0:1]
-; GCN-IR-NEXT:    v_sub_i32_e64 v10, s[4:5], v10, v8
-; GCN-IR-NEXT:    v_subb_u32_e64 v11, s[4:5], v11, v9, s[4:5]
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, v7
+; GCN-IR-NEXT:    v_sub_i32_e64 v8, s[4:5], v8, v10
+; GCN-IR-NEXT:    v_subb_u32_e64 v9, s[4:5], v9, v11, s[4:5]
+; GCN-IR-NEXT:    v_mov_b32_e32 v11, v7
 ; GCN-IR-NEXT:    s_or_b64 s[10:11], vcc, s[10:11]
-; GCN-IR-NEXT:    v_mov_b32_e32 v8, v6
+; GCN-IR-NEXT:    v_mov_b32_e32 v10, v6
 ; GCN-IR-NEXT:    s_andn2_b64 exec, exec, s[10:11]
 ; GCN-IR-NEXT:    s_cbranch_execnz .LBB1_3
 ; GCN-IR-NEXT:  ; %bb.4: ; %Flow
@@ -804,33 +804,33 @@ define amdgpu_kernel void @s_test_udiv24_i48(ptr addrspace(1) %out, i48 %x, i48
 ; GCN-IR-NEXT:    s_flbit_i32_b32 s4, s8
 ; GCN-IR-NEXT:    s_add_i32 s4, s4, 32
 ; GCN-IR-NEXT:    s_flbit_i32_b32 s5, s9
-; GCN-IR-NEXT:    s_min_u32 s14, s4, s5
-; GCN-IR-NEXT:    s_sub_u32 s12, s10, s14
+; GCN-IR-NEXT:    s_min_u32 s16, s4, s5
+; GCN-IR-NEXT:    s_sub_u32 s12, s10, s16
 ; GCN-IR-NEXT:    s_subb_u32 s13, 0, 0
-; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[16:17], s[12:13], 63
+; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[14:15], s[12:13], 63
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[18:19], s[12:13], 63
-; GCN-IR-NEXT:    s_or_b64 s[16:17], s[6:7], s[16:17]
-; GCN-IR-NEXT:    s_and_b64 s[6:7], s[16:17], exec
+; GCN-IR-NEXT:    s_or_b64 s[14:15], s[6:7], s[14:15]
+; GCN-IR-NEXT:    s_and_b64 s[6:7], s[14:15], exec
 ; GCN-IR-NEXT:    s_cselect_b32 s7, 0, s9
 ; GCN-IR-NEXT:    s_cselect_b32 s6, 0, s8
-; GCN-IR-NEXT:    s_or_b64 s[16:17], s[16:17], s[18:19]
+; GCN-IR-NEXT:    s_or_b64 s[14:15], s[14:15], s[18:19]
 ; GCN-IR-NEXT:    s_mov_b64 s[4:5], 0
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[16:17]
+; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[14:15]
 ; GCN-IR-NEXT:    s_cbranch_vccz .LBB7_5
 ; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
-; GCN-IR-NEXT:    s_add_u32 s16, s12, 1
-; GCN-IR-NEXT:    s_addc_u32 s17, s13, 0
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[6:7], s[16:17], 0
+; GCN-IR-NEXT:    s_add_u32 s14, s12, 1
+; GCN-IR-NEXT:    s_addc_u32 s15, s13, 0
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[6:7], s[14:15], 0
 ; GCN-IR-NEXT:    s_sub_i32 s12, 63, s12
 ; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
 ; GCN-IR-NEXT:    s_lshl_b64 s[6:7], s[8:9], s12
 ; GCN-IR-NEXT:    s_cbranch_vccz .LBB7_4
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
-; GCN-IR-NEXT:    s_lshr_b64 s[12:13], s[8:9], s16
-; GCN-IR-NEXT:    s_add_u32 s15, s2, -1
-; GCN-IR-NEXT:    s_addc_u32 s16, s3, -1
+; GCN-IR-NEXT:    s_lshr_b64 s[12:13], s[8:9], s14
+; GCN-IR-NEXT:    s_add_u32 s14, s2, -1
+; GCN-IR-NEXT:    s_addc_u32 s15, s3, -1
 ; GCN-IR-NEXT:    s_not_b64 s[4:5], s[10:11]
-; GCN-IR-NEXT:    s_add_u32 s8, s4, s14
+; GCN-IR-NEXT:    s_add_u32 s8, s4, s16
 ; GCN-IR-NEXT:    s_addc_u32 s9, s5, 0
 ; GCN-IR-NEXT:    s_mov_b64 s[10:11], 0
 ; GCN-IR-NEXT:    s_mov_b32 s5, 0
@@ -841,8 +841,8 @@ define amdgpu_kernel void @s_test_udiv24_i48(ptr addrspace(1) %out, i48 %x, i48
 ; GCN-IR-NEXT:    s_lshl_b64 s[6:7], s[6:7], 1
 ; GCN-IR-NEXT:    s_or_b64 s[12:13], s[12:13], s[4:5]
 ; GCN-IR-NEXT:    s_or_b64 s[6:7], s[10:11], s[6:7]
-; GCN-IR-NEXT:    s_sub_u32 s4, s15, s12
-; GCN-IR-NEXT:    s_subb_u32 s4, s16, s13
+; GCN-IR-NEXT:    s_sub_u32 s4, s14, s12
+; GCN-IR-NEXT:    s_subb_u32 s4, s15, s13
 ; GCN-IR-NEXT:    s_ashr_i32 s10, s4, 31
 ; GCN-IR-NEXT:    s_mov_b32 s11, s10
 ; GCN-IR-NEXT:    s_and_b32 s4, s10, 1
@@ -851,9 +851,9 @@ define amdgpu_kernel void @s_test_udiv24_i48(ptr addrspace(1) %out, i48 %x, i48
 ; GCN-IR-NEXT:    s_subb_u32 s13, s13, s11
 ; GCN-IR-NEXT:    s_add_u32 s8, s8, 1
 ; GCN-IR-NEXT:    s_addc_u32 s9, s9, 0
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[18:19], s[8:9], 0
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[16:17], s[8:9], 0
 ; GCN-IR-NEXT:    s_mov_b64 s[10:11], s[4:5]
-; GCN-IR-NEXT:    s_and_b64 vcc, exec, s[18:19]
+; GCN-IR-NEXT:    s_and_b64 vcc, exec, s[16:17]
 ; GCN-IR-NEXT:    s_cbranch_vccz .LBB7_3
 ; GCN-IR-NEXT:  .LBB7_4: ; %Flow4
 ; GCN-IR-NEXT:    s_lshl_b64 s[2:3], s[6:7], 1
@@ -989,32 +989,32 @@ define amdgpu_kernel void @s_test_udiv_k_num_i64(ptr addrspace(1) %out, i64 %x)
 ; GCN-IR-NEXT:    s_flbit_i32_b32 s8, s2
 ; GCN-IR-NEXT:    s_flbit_i32_b32 s9, s3
 ; GCN-IR-NEXT:    s_add_i32 s8, s8, 32
-; GCN-IR-NEXT:    s_min_u32 s8, s8, s9
-; GCN-IR-NEXT:    s_add_u32 s10, s8, 0xffffffc5
-; GCN-IR-NEXT:    s_addc_u32 s11, 0, -1
+; GCN-IR-NEXT:    s_min_u32 s12, s8, s9
+; GCN-IR-NEXT:    s_add_u32 s8, s12, 0xffffffc5
+; GCN-IR-NEXT:    s_addc_u32 s9, 0, -1
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[6:7], s[2:3], 0
-; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[12:13], s[10:11], 63
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[14:15], s[10:11], 63
-; GCN-IR-NEXT:    s_or_b64 s[12:13], s[6:7], s[12:13]
-; GCN-IR-NEXT:    s_and_b64 s[6:7], s[12:13], exec
+; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[10:11], s[8:9], 63
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[14:15], s[8:9], 63
+; GCN-IR-NEXT:    s_or_b64 s[10:11], s[6:7], s[10:11]
+; GCN-IR-NEXT:    s_and_b64 s[6:7], s[10:11], exec
 ; GCN-IR-NEXT:    s_cselect_b32 s6, 0, 24
-; GCN-IR-NEXT:    s_or_b64 s[12:13], s[12:13], s[14:15]
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[12:13]
+; GCN-IR-NEXT:    s_or_b64 s[10:11], s[10:11], s[14:15]
+; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[10:11]
 ; GCN-IR-NEXT:    s_mov_b32 s7, 0
 ; GCN-IR-NEXT:    s_cbranch_vccz .LBB8_5
 ; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
-; GCN-IR-NEXT:    s_add_u32 s12, s10, 1
-; GCN-IR-NEXT:    s_addc_u32 s13, s11, 0
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[6:7], s[12:13], 0
-; GCN-IR-NEXT:    s_sub_i32 s9, 63, s10
+; GCN-IR-NEXT:    s_add_u32 s10, s8, 1
+; GCN-IR-NEXT:    s_addc_u32 s11, s9, 0
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[6:7], s[10:11], 0
+; GCN-IR-NEXT:    s_sub_i32 s8, 63, s8
 ; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
-; GCN-IR-NEXT:    s_lshl_b64 s[6:7], 24, s9
+; GCN-IR-NEXT:    s_lshl_b64 s[6:7], 24, s8
 ; GCN-IR-NEXT:    s_cbranch_vccz .LBB8_4
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
-; GCN-IR-NEXT:    s_lshr_b64 s[10:11], 24, s12
+; GCN-IR-NEXT:    s_lshr_b64 s[10:11], 24, s10
 ; GCN-IR-NEXT:    s_add_u32 s14, s2, -1
 ; GCN-IR-NEXT:    s_addc_u32 s15, s3, -1
-; GCN-IR-NEXT:    s_sub_u32 s8, 58, s8
+; GCN-IR-NEXT:    s_sub_u32 s8, 58, s12
 ; GCN-IR-NEXT:    s_subb_u32 s9, 0, 0
 ; GCN-IR-NEXT:    s_mov_b64 s[12:13], 0
 ; GCN-IR-NEXT:    s_mov_b32 s5, 0
@@ -1160,8 +1160,8 @@ define i64 @v_test_udiv_pow2_k_num_i64(i64 %x) {
 ; GCN-IR-NEXT:    v_ffbh_u32_e32 v2, v0
 ; GCN-IR-NEXT:    v_add_i32_e32 v2, vcc, 32, v2
 ; GCN-IR-NEXT:    v_ffbh_u32_e32 v3, v1
-; GCN-IR-NEXT:    v_min_u32_e32 v6, v2, v3
-; GCN-IR-NEXT:    v_add_i32_e32 v4, vcc, 0xffffffd0, v6
+; GCN-IR-NEXT:    v_min_u32_e32 v10, v2, v3
+; GCN-IR-NEXT:    v_add_i32_e32 v4, vcc, 0xffffffd0, v10
 ; GCN-IR-NEXT:    v_addc_u32_e64 v5, s[6:7], 0, -1, vcc
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[4:5], 0, v[0:1]
 ; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, 63, v[4:5]
@@ -1175,11 +1175,11 @@ define i64 @v_test_udiv_pow2_k_num_i64(i64 %x) {
 ; GCN-IR-NEXT:    s_and_saveexec_b64 s[6:7], s[4:5]
 ; GCN-IR-NEXT:    s_cbranch_execz .LBB9_6
 ; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
-; GCN-IR-NEXT:    v_add_i32_e32 v7, vcc, 1, v4
+; GCN-IR-NEXT:    v_add_i32_e32 v6, vcc, 1, v4
 ; GCN-IR-NEXT:    v_sub_i32_e64 v2, s[4:5], 63, v4
-; GCN-IR-NEXT:    v_addc_u32_e32 v8, vcc, 0, v5, vcc
+; GCN-IR-NEXT:    v_addc_u32_e32 v7, vcc, 0, v5, vcc
 ; GCN-IR-NEXT:    s_mov_b64 s[4:5], 0x8000
-; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[7:8]
+; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[6:7]
 ; GCN-IR-NEXT:    v_lshl_b64 v[2:3], s[4:5], v2
 ; GCN-IR-NEXT:    v_mov_b32_e32 v4, 0
 ; GCN-IR-NEXT:    v_mov_b32_e32 v5, 0
@@ -1189,8 +1189,8 @@ define i64 @v_test_udiv_pow2_k_num_i64(i64 %x) {
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
 ; GCN-IR-NEXT:    v_add_i32_e32 v12, vcc, -1, v0
 ; GCN-IR-NEXT:    v_addc_u32_e32 v13, vcc, -1, v1, vcc
-; GCN-IR-NEXT:    v_lshr_b64 v[8:9], s[4:5], v7
-; GCN-IR-NEXT:    v_sub_i32_e32 v6, vcc, 47, v6
+; GCN-IR-NEXT:    v_lshr_b64 v[8:9], s[4:5], v6
+; GCN-IR-NEXT:    v_sub_i32_e32 v6, vcc, 47, v10
 ; GCN-IR-NEXT:    v_mov_b32_e32 v10, 0
 ; GCN-IR-NEXT:    v_subb_u32_e64 v7, s[4:5], 0, 0, vcc
 ; GCN-IR-NEXT:    s_mov_b64 s[10:11], 0
@@ -1250,8 +1250,8 @@ define i64 @v_test_udiv_pow2_k_den_i64(i64 %x) {
 ; GCN-IR-NEXT:    v_ffbh_u32_e32 v2, v0
 ; GCN-IR-NEXT:    v_add_i32_e64 v2, s[4:5], 32, v2
 ; GCN-IR-NEXT:    v_ffbh_u32_e32 v3, v1
-; GCN-IR-NEXT:    v_min_u32_e32 v6, v2, v3
-; GCN-IR-NEXT:    v_sub_i32_e64 v4, s[4:5], 48, v6
+; GCN-IR-NEXT:    v_min_u32_e32 v8, v2, v3
+; GCN-IR-NEXT:    v_sub_i32_e64 v4, s[4:5], 48, v8
 ; GCN-IR-NEXT:    v_subb_u32_e64 v5, s[4:5], 0, 0, s[4:5]
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[0:1]
 ; GCN-IR-NEXT:    v_cmp_lt_u64_e64 s[4:5], 63, v[4:5]
@@ -1264,10 +1264,10 @@ define i64 @v_test_udiv_pow2_k_den_i64(i64 %x) {
 ; GCN-IR-NEXT:    s_and_saveexec_b64 s[6:7], s[4:5]
 ; GCN-IR-NEXT:    s_cbranch_execz .LBB10_6
 ; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
-; GCN-IR-NEXT:    v_add_i32_e32 v7, vcc, 1, v4
-; GCN-IR-NEXT:    v_addc_u32_e32 v8, vcc, 0, v5, vcc
+; GCN-IR-NEXT:    v_add_i32_e32 v6, vcc, 1, v4
+; GCN-IR-NEXT:    v_addc_u32_e32 v7, vcc, 0, v5, vcc
 ; GCN-IR-NEXT:    v_sub_i32_e64 v2, s[4:5], 63, v4
-; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[7:8]
+; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[6:7]
 ; GCN-IR-NEXT:    v_lshl_b64 v[2:3], v[0:1], v2
 ; GCN-IR-NEXT:    v_mov_b32_e32 v4, 0
 ; GCN-IR-NEXT:    v_mov_b32_e32 v5, 0
@@ -1275,35 +1275,35 @@ define i64 @v_test_udiv_pow2_k_den_i64(i64 %x) {
 ; GCN-IR-NEXT:    s_xor_b64 s[8:9], exec, s[4:5]
 ; GCN-IR-NEXT:    s_cbranch_execz .LBB10_5
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
-; GCN-IR-NEXT:    v_lshr_b64 v[7:8], v[0:1], v7
-; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, 0xffffffcf, v6
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, 0
+; GCN-IR-NEXT:    v_lshr_b64 v[6:7], v[0:1], v6
+; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, 0xffffffcf, v8
+; GCN-IR-NEXT:    v_mov_b32_e32 v8, 0
 ; GCN-IR-NEXT:    v_addc_u32_e64 v1, s[4:5], 0, -1, vcc
 ; GCN-IR-NEXT:    s_mov_b64 s[10:11], 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v10, 0
+; GCN-IR-NEXT:    v_mov_b32_e32 v9, 0
 ; GCN-IR-NEXT:    s_movk_i32 s12, 0x7fff
 ; GCN-IR-NEXT:    v_mov_b32_e32 v5, 0
 ; GCN-IR-NEXT:  .LBB10_3: ; %udiv-do-while
 ; GCN-IR-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN-IR-NEXT:    v_lshl_b64 v[7:8], v[7:8], 1
+; GCN-IR-NEXT:    v_lshl_b64 v[6:7], v[6:7], 1
 ; GCN-IR-NEXT:    v_lshrrev_b32_e32 v4, 31, v3
-; GCN-IR-NEXT:    v_or_b32_e32 v6, v7, v4
+; GCN-IR-NEXT:    v_or_b32_e32 v6, v6, v4
 ; GCN-IR-NEXT:    v_sub_i32_e32 v4, vcc, s12, v6
-; GCN-IR-NEXT:    v_subb_u32_e32 v4, vcc, 0, v8, vcc
-; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, 1, v0
 ; GCN-IR-NEXT:    v_lshl_b64 v[2:3], v[2:3], 1
-; GCN-IR-NEXT:    v_ashrrev_i32_e32 v7, 31, v4
+; GCN-IR-NEXT:    v_subb_u32_e32 v4, vcc, 0, v7, vcc
+; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, 1, v0
+; GCN-IR-NEXT:    v_or_b32_e32 v2, v8, v2
+; GCN-IR-NEXT:    v_ashrrev_i32_e32 v8, 31, v4
 ; GCN-IR-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GCN-IR-NEXT:    v_and_b32_e32 v4, 1, v7
-; GCN-IR-NEXT:    v_and_b32_e32 v7, 0x8000, v7
+; GCN-IR-NEXT:    v_and_b32_e32 v4, 1, v8
+; GCN-IR-NEXT:    v_and_b32_e32 v8, 0x8000, v8
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[0:1]
-; GCN-IR-NEXT:    v_or_b32_e32 v3, v10, v3
-; GCN-IR-NEXT:    v_or_b32_e32 v2, v9, v2
-; GCN-IR-NEXT:    v_sub_i32_e64 v7, s[4:5], v6, v7
-; GCN-IR-NEXT:    v_mov_b32_e32 v10, v5
-; GCN-IR-NEXT:    v_subbrev_u32_e64 v8, s[4:5], 0, v8, s[4:5]
+; GCN-IR-NEXT:    v_or_b32_e32 v3, v9, v3
+; GCN-IR-NEXT:    v_sub_i32_e64 v6, s[4:5], v6, v8
+; GCN-IR-NEXT:    v_mov_b32_e32 v9, v5
+; GCN-IR-NEXT:    v_subbrev_u32_e64 v7, s[4:5], 0, v7, s[4:5]
 ; GCN-IR-NEXT:    s_or_b64 s[10:11], vcc, s[10:11]
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, v4
+; GCN-IR-NEXT:    v_mov_b32_e32 v8, v4
 ; GCN-IR-NEXT:    s_andn2_b64 exec, exec, s[10:11]
 ; GCN-IR-NEXT:    s_cbranch_execnz .LBB10_3
 ; GCN-IR-NEXT:  ; %bb.4: ; %Flow
@@ -1409,31 +1409,31 @@ define amdgpu_kernel void @s_test_udiv_k_den_i64(ptr addrspace(1) %out, i64 %x)
 ; GCN-IR-NEXT:    s_flbit_i32_b32 s6, s2
 ; GCN-IR-NEXT:    s_flbit_i32_b32 s7, s3
 ; GCN-IR-NEXT:    s_add_i32 s6, s6, 32
-; GCN-IR-NEXT:    s_min_u32 s10, s6, s7
-; GCN-IR-NEXT:    s_sub_u32 s8, 59, s10
+; GCN-IR-NEXT:    s_min_u32 s12, s6, s7
+; GCN-IR-NEXT:    s_sub_u32 s8, 59, s12
 ; GCN-IR-NEXT:    s_subb_u32 s9, 0, 0
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[4:5], s[2:3], 0
 ; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[6:7], s[8:9], 63
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[12:13], s[8:9], 63
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[10:11], s[8:9], 63
 ; GCN-IR-NEXT:    s_or_b64 s[4:5], s[4:5], s[6:7]
 ; GCN-IR-NEXT:    s_and_b64 s[6:7], s[4:5], exec
 ; GCN-IR-NEXT:    s_cselect_b32 s7, 0, s3
 ; GCN-IR-NEXT:    s_cselect_b32 s6, 0, s2
-; GCN-IR-NEXT:    s_or_b64 s[4:5], s[4:5], s[12:13]
+; GCN-IR-NEXT:    s_or_b64 s[4:5], s[4:5], s[10:11]
 ; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
 ; GCN-IR-NEXT:    s_mov_b64 s[4:5], 0
 ; GCN-IR-NEXT:    s_cbranch_vccz .LBB11_5
 ; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
-; GCN-IR-NEXT:    s_add_u32 s12, s8, 1
-; GCN-IR-NEXT:    s_addc_u32 s13, s9, 0
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[6:7], s[12:13], 0
+; GCN-IR-NEXT:    s_add_u32 s10, s8, 1
+; GCN-IR-NEXT:    s_addc_u32 s11, s9, 0
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[6:7], s[10:11], 0
 ; GCN-IR-NEXT:    s_sub_i32 s8, 63, s8
 ; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
 ; GCN-IR-NEXT:    s_lshl_b64 s[6:7], s[2:3], s8
 ; GCN-IR-NEXT:    s_cbranch_vccz .LBB11_4
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
-; GCN-IR-NEXT:    s_lshr_b64 s[8:9], s[2:3], s12
-; GCN-IR-NEXT:    s_add_u32 s2, s10, 0xffffffc4
+; GCN-IR-NEXT:    s_lshr_b64 s[8:9], s[2:3], s10
+; GCN-IR-NEXT:    s_add_u32 s2, s12, 0xffffffc4
 ; GCN-IR-NEXT:    s_addc_u32 s3, 0, -1
 ; GCN-IR-NEXT:    s_mov_b64 s[10:11], 0
 ; GCN-IR-NEXT:    s_mov_b32 s5, 0
@@ -1551,8 +1551,8 @@ define i64 @v_test_udiv_k_den_i64(i64 %x) {
 ; GCN-IR-NEXT:    v_ffbh_u32_e32 v2, v0
 ; GCN-IR-NEXT:    v_add_i32_e64 v2, s[4:5], 32, v2
 ; GCN-IR-NEXT:    v_ffbh_u32_e32 v3, v1
-; GCN-IR-NEXT:    v_min_u32_e32 v6, v2, v3
-; GCN-IR-NEXT:    v_sub_i32_e64 v4, s[4:5], 59, v6
+; GCN-IR-NEXT:    v_min_u32_e32 v8, v2, v3
+; GCN-IR-NEXT:    v_sub_i32_e64 v4, s[4:5], 59, v8
 ; GCN-IR-NEXT:    v_subb_u32_e64 v5, s[4:5], 0, 0, s[4:5]
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[0:1]
 ; GCN-IR-NEXT:    v_cmp_lt_u64_e64 s[4:5], 63, v[4:5]
@@ -1565,10 +1565,10 @@ define i64 @v_test_udiv_k_den_i64(i64 %x) {
 ; GCN-IR-NEXT:    s_and_saveexec_b64 s[6:7], s[4:5]
 ; GCN-IR-NEXT:    s_cbranch_execz .LBB12_6
 ; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
-; GCN-IR-NEXT:    v_add_i32_e32 v7, vcc, 1, v4
-; GCN-IR-NEXT:    v_addc_u32_e32 v8, vcc, 0, v5, vcc
+; GCN-IR-NEXT:    v_add_i32_e32 v6, vcc, 1, v4
+; GCN-IR-NEXT:    v_addc_u32_e32 v7, vcc, 0, v5, vcc
 ; GCN-IR-NEXT:    v_sub_i32_e64 v2, s[4:5], 63, v4
-; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[7:8]
+; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[6:7]
 ; GCN-IR-NEXT:    v_lshl_b64 v[2:3], v[0:1], v2
 ; GCN-IR-NEXT:    v_mov_b32_e32 v4, 0
 ; GCN-IR-NEXT:    v_mov_b32_e32 v5, 0
@@ -1576,34 +1576,34 @@ define i64 @v_test_udiv_k_den_i64(i64 %x) {
 ; GCN-IR-NEXT:    s_xor_b64 s[8:9], exec, s[4:5]
 ; GCN-IR-NEXT:    s_cbranch_execz .LBB12_5
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
-; GCN-IR-NEXT:    v_lshr_b64 v[7:8], v[0:1], v7
-; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, 0xffffffc4, v6
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, 0
+; GCN-IR-NEXT:    v_lshr_b64 v[6:7], v[0:1], v6
+; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, 0xffffffc4, v8
+; GCN-IR-NEXT:    v_mov_b32_e32 v8, 0
 ; GCN-IR-NEXT:    v_addc_u32_e64 v1, s[4:5], 0, -1, vcc
 ; GCN-IR-NEXT:    s_mov_b64 s[10:11], 0
-; GCN-IR-NEXT:    v_mov_b32_e32 v10, 0
+; GCN-IR-NEXT:    v_mov_b32_e32 v9, 0
 ; GCN-IR-NEXT:    v_mov_b32_e32 v5, 0
 ; GCN-IR-NEXT:  .LBB12_3: ; %udiv-do-while
 ; GCN-IR-NEXT:    ; =>This Inner Loop Header: Depth=1
-; GCN-IR-NEXT:    v_lshl_b64 v[7:8], v[7:8], 1
+; GCN-IR-NEXT:    v_lshl_b64 v[6:7], v[6:7], 1
 ; GCN-IR-NEXT:    v_lshrrev_b32_e32 v4, 31, v3
-; GCN-IR-NEXT:    v_or_b32_e32 v6, v7, v4
+; GCN-IR-NEXT:    v_or_b32_e32 v6, v6, v4
 ; GCN-IR-NEXT:    v_sub_i32_e32 v4, vcc, 23, v6
-; GCN-IR-NEXT:    v_subb_u32_e32 v4, vcc, 0, v8, vcc
-; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, 1, v0
 ; GCN-IR-NEXT:    v_lshl_b64 v[2:3], v[2:3], 1
-; GCN-IR-NEXT:    v_ashrrev_i32_e32 v7, 31, v4
+; GCN-IR-NEXT:    v_subb_u32_e32 v4, vcc, 0, v7, vcc
+; GCN-IR-NEXT:    v_add_i32_e32 v0, vcc, 1, v0
+; GCN-IR-NEXT:    v_or_b32_e32 v2, v8, v2
+; GCN-IR-NEXT:    v_ashrrev_i32_e32 v8, 31, v4
 ; GCN-IR-NEXT:    v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GCN-IR-NEXT:    v_and_b32_e32 v4, 1, v7
-; GCN-IR-NEXT:    v_and_b32_e32 v7, 24, v7
+; GCN-IR-NEXT:    v_and_b32_e32 v4, 1, v8
+; GCN-IR-NEXT:    v_and_b32_e32 v8, 24, v8
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[0:1]
-; GCN-IR-NEXT:    v_or_b32_e32 v3, v10, v3
-; GCN-IR-NEXT:    v_or_b32_e32 v2, v9, v2
-; GCN-IR-NEXT:    v_sub_i32_e64 v7, s[4:5], v6, v7
-; GCN-IR-NEXT:    v_mov_b32_e32 v10, v5
-; GCN-IR-NEXT:    v_subbrev_u32_e64 v8, s[4:5], 0, v8, s[4:5]
+; GCN-IR-NEXT:    v_or_b32_e32 v3, v9, v3
+; GCN-IR-NEXT:    v_sub_i32_e64 v6, s[4:5], v6, v8
+; GCN-IR-NEXT:    v_mov_b32_e32 v9, v5
+; GCN-IR-NEXT:    v_subbrev_u32_e64 v7, s[4:5], 0, v7, s[4:5]
 ; GCN-IR-NEXT:    s_or_b64 s[10:11], vcc, s[10:11]
-; GCN-IR-NEXT:    v_mov_b32_e32 v9, v4
+; GCN-IR-NEXT:    v_mov_b32_e32 v8, v4
 ; GCN-IR-NEXT:    s_andn2_b64 exec, exec, s[10:11]
 ; GCN-IR-NEXT:    s_cbranch_execnz .LBB12_3
 ; GCN-IR-NEXT:  ; %bb.4: ; %Flow
diff --git a/llvm/test/CodeGen/AMDGPU/urem64.ll b/llvm/test/CodeGen/AMDGPU/urem64.ll
index 7e8661b3ff32dd1b4b2d4f306dfa2ba495892b5a..784993ccd3bd1aab1e8ca768f880ff335c45e5da 100644
--- a/llvm/test/CodeGen/AMDGPU/urem64.ll
+++ b/llvm/test/CodeGen/AMDGPU/urem64.ll
@@ -135,34 +135,34 @@ define amdgpu_kernel void @s_test_urem_i64(ptr addrspace(1) %out, i64 %x, i64 %y
 ; GCN-IR-NEXT:    s_add_i32 s6, s6, 32
 ; GCN-IR-NEXT:    s_flbit_i32_b32 s7, s3
 ; GCN-IR-NEXT:    s_min_u32 s10, s10, s11
-; GCN-IR-NEXT:    s_min_u32 s14, s6, s7
-; GCN-IR-NEXT:    s_sub_u32 s12, s10, s14
+; GCN-IR-NEXT:    s_min_u32 s18, s6, s7
+; GCN-IR-NEXT:    s_sub_u32 s12, s10, s18
 ; GCN-IR-NEXT:    s_subb_u32 s13, 0, 0
-; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[16:17], s[12:13], 63
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[18:19], s[12:13], 63
-; GCN-IR-NEXT:    s_or_b64 s[16:17], s[8:9], s[16:17]
-; GCN-IR-NEXT:    s_and_b64 s[8:9], s[16:17], exec
+; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[14:15], s[12:13], 63
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[16:17], s[12:13], 63
+; GCN-IR-NEXT:    s_or_b64 s[14:15], s[8:9], s[14:15]
+; GCN-IR-NEXT:    s_and_b64 s[8:9], s[14:15], exec
 ; GCN-IR-NEXT:    s_cselect_b32 s9, 0, s3
 ; GCN-IR-NEXT:    s_cselect_b32 s8, 0, s2
-; GCN-IR-NEXT:    s_or_b64 s[16:17], s[16:17], s[18:19]
+; GCN-IR-NEXT:    s_or_b64 s[14:15], s[14:15], s[16:17]
 ; GCN-IR-NEXT:    s_mov_b64 s[6:7], 0
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[16:17]
+; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[14:15]
 ; GCN-IR-NEXT:    s_mov_b32 s11, 0
 ; GCN-IR-NEXT:    s_cbranch_vccz .LBB0_5
 ; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
-; GCN-IR-NEXT:    s_add_u32 s16, s12, 1
-; GCN-IR-NEXT:    s_addc_u32 s17, s13, 0
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[8:9], s[16:17], 0
+; GCN-IR-NEXT:    s_add_u32 s14, s12, 1
+; GCN-IR-NEXT:    s_addc_u32 s15, s13, 0
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[8:9], s[14:15], 0
 ; GCN-IR-NEXT:    s_sub_i32 s12, 63, s12
 ; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[8:9]
 ; GCN-IR-NEXT:    s_lshl_b64 s[8:9], s[2:3], s12
 ; GCN-IR-NEXT:    s_cbranch_vccz .LBB0_4
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
-; GCN-IR-NEXT:    s_lshr_b64 s[12:13], s[2:3], s16
+; GCN-IR-NEXT:    s_lshr_b64 s[12:13], s[2:3], s14
 ; GCN-IR-NEXT:    s_add_u32 s16, s4, -1
 ; GCN-IR-NEXT:    s_addc_u32 s17, s5, -1
 ; GCN-IR-NEXT:    s_not_b64 s[6:7], s[10:11]
-; GCN-IR-NEXT:    s_add_u32 s10, s6, s14
+; GCN-IR-NEXT:    s_add_u32 s10, s6, s18
 ; GCN-IR-NEXT:    s_addc_u32 s11, s7, 0
 ; GCN-IR-NEXT:    s_mov_b64 s[14:15], 0
 ; GCN-IR-NEXT:    s_mov_b32 s7, 0
@@ -328,12 +328,12 @@ define i64 @v_test_urem_i64(i64 %x, i64 %y) {
 ; GCN-IR-NEXT:    v_ffbh_u32_e32 v4, v2
 ; GCN-IR-NEXT:    v_add_i32_e64 v4, s[6:7], 32, v4
 ; GCN-IR-NEXT:    v_ffbh_u32_e32 v5, v3
-; GCN-IR-NEXT:    v_min_u32_e32 v8, v4, v5
+; GCN-IR-NEXT:    v_min_u32_e32 v12, v4, v5
 ; GCN-IR-NEXT:    v_ffbh_u32_e32 v4, v0
 ; GCN-IR-NEXT:    v_add_i32_e64 v4, s[6:7], 32, v4
 ; GCN-IR-NEXT:    v_ffbh_u32_e32 v5, v1
-; GCN-IR-NEXT:    v_min_u32_e32 v9, v4, v5
-; GCN-IR-NEXT:    v_sub_i32_e64 v4, s[6:7], v8, v9
+; GCN-IR-NEXT:    v_min_u32_e32 v13, v4, v5
+; GCN-IR-NEXT:    v_sub_i32_e64 v4, s[6:7], v12, v13
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[2:3]
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[4:5], 0, v[0:1]
 ; GCN-IR-NEXT:    v_subb_u32_e64 v5, s[6:7], 0, 0, s[6:7]
@@ -348,10 +348,10 @@ define i64 @v_test_urem_i64(i64 %x, i64 %y) {
 ; GCN-IR-NEXT:    s_and_saveexec_b64 s[6:7], s[4:5]
 ; GCN-IR-NEXT:    s_cbranch_execz .LBB1_6
 ; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
-; GCN-IR-NEXT:    v_add_i32_e32 v10, vcc, 1, v4
-; GCN-IR-NEXT:    v_addc_u32_e32 v11, vcc, 0, v5, vcc
+; GCN-IR-NEXT:    v_add_i32_e32 v8, vcc, 1, v4
+; GCN-IR-NEXT:    v_addc_u32_e32 v9, vcc, 0, v5, vcc
 ; GCN-IR-NEXT:    v_sub_i32_e64 v4, s[4:5], 63, v4
-; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[10:11]
+; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[8:9]
 ; GCN-IR-NEXT:    v_lshl_b64 v[4:5], v[0:1], v4
 ; GCN-IR-NEXT:    v_mov_b32_e32 v6, 0
 ; GCN-IR-NEXT:    v_mov_b32_e32 v7, 0
@@ -361,10 +361,10 @@ define i64 @v_test_urem_i64(i64 %x, i64 %y) {
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
 ; GCN-IR-NEXT:    v_add_i32_e32 v14, vcc, -1, v2
 ; GCN-IR-NEXT:    v_addc_u32_e32 v15, vcc, -1, v3, vcc
-; GCN-IR-NEXT:    v_not_b32_e32 v7, v8
-; GCN-IR-NEXT:    v_lshr_b64 v[10:11], v[0:1], v10
+; GCN-IR-NEXT:    v_not_b32_e32 v7, v12
+; GCN-IR-NEXT:    v_lshr_b64 v[10:11], v[0:1], v8
 ; GCN-IR-NEXT:    v_not_b32_e32 v6, 0
-; GCN-IR-NEXT:    v_add_i32_e32 v8, vcc, v7, v9
+; GCN-IR-NEXT:    v_add_i32_e32 v8, vcc, v7, v13
 ; GCN-IR-NEXT:    v_mov_b32_e32 v12, 0
 ; GCN-IR-NEXT:    v_addc_u32_e32 v9, vcc, 0, v6, vcc
 ; GCN-IR-NEXT:    s_mov_b64 s[10:11], 0
@@ -817,32 +817,32 @@ define amdgpu_kernel void @s_test_urem_k_num_i64(ptr addrspace(1) %out, i64 %x)
 ; GCN-IR-NEXT:    s_flbit_i32_b32 s8, s2
 ; GCN-IR-NEXT:    s_flbit_i32_b32 s9, s3
 ; GCN-IR-NEXT:    s_add_i32 s8, s8, 32
-; GCN-IR-NEXT:    s_min_u32 s8, s8, s9
-; GCN-IR-NEXT:    s_add_u32 s10, s8, 0xffffffc5
-; GCN-IR-NEXT:    s_addc_u32 s11, 0, -1
+; GCN-IR-NEXT:    s_min_u32 s12, s8, s9
+; GCN-IR-NEXT:    s_add_u32 s8, s12, 0xffffffc5
+; GCN-IR-NEXT:    s_addc_u32 s9, 0, -1
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[6:7], s[2:3], 0
-; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[12:13], s[10:11], 63
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[14:15], s[10:11], 63
-; GCN-IR-NEXT:    s_or_b64 s[12:13], s[6:7], s[12:13]
-; GCN-IR-NEXT:    s_and_b64 s[6:7], s[12:13], exec
+; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[10:11], s[8:9], 63
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[14:15], s[8:9], 63
+; GCN-IR-NEXT:    s_or_b64 s[10:11], s[6:7], s[10:11]
+; GCN-IR-NEXT:    s_and_b64 s[6:7], s[10:11], exec
 ; GCN-IR-NEXT:    s_cselect_b32 s6, 0, 24
-; GCN-IR-NEXT:    s_or_b64 s[12:13], s[12:13], s[14:15]
-; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[12:13]
+; GCN-IR-NEXT:    s_or_b64 s[10:11], s[10:11], s[14:15]
+; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[10:11]
 ; GCN-IR-NEXT:    s_mov_b32 s7, 0
 ; GCN-IR-NEXT:    s_cbranch_vccz .LBB6_5
 ; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
-; GCN-IR-NEXT:    s_add_u32 s12, s10, 1
-; GCN-IR-NEXT:    s_addc_u32 s13, s11, 0
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[6:7], s[12:13], 0
-; GCN-IR-NEXT:    s_sub_i32 s9, 63, s10
+; GCN-IR-NEXT:    s_add_u32 s10, s8, 1
+; GCN-IR-NEXT:    s_addc_u32 s11, s9, 0
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[6:7], s[10:11], 0
+; GCN-IR-NEXT:    s_sub_i32 s8, 63, s8
 ; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
-; GCN-IR-NEXT:    s_lshl_b64 s[6:7], 24, s9
+; GCN-IR-NEXT:    s_lshl_b64 s[6:7], 24, s8
 ; GCN-IR-NEXT:    s_cbranch_vccz .LBB6_4
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
-; GCN-IR-NEXT:    s_lshr_b64 s[10:11], 24, s12
+; GCN-IR-NEXT:    s_lshr_b64 s[10:11], 24, s10
 ; GCN-IR-NEXT:    s_add_u32 s14, s2, -1
 ; GCN-IR-NEXT:    s_addc_u32 s15, s3, -1
-; GCN-IR-NEXT:    s_sub_u32 s8, 58, s8
+; GCN-IR-NEXT:    s_sub_u32 s8, 58, s12
 ; GCN-IR-NEXT:    s_subb_u32 s9, 0, 0
 ; GCN-IR-NEXT:    s_mov_b64 s[12:13], 0
 ; GCN-IR-NEXT:    s_mov_b32 s5, 0
@@ -976,31 +976,31 @@ define amdgpu_kernel void @s_test_urem_k_den_i64(ptr addrspace(1) %out, i64 %x)
 ; GCN-IR-NEXT:    s_flbit_i32_b32 s6, s2
 ; GCN-IR-NEXT:    s_flbit_i32_b32 s7, s3
 ; GCN-IR-NEXT:    s_add_i32 s6, s6, 32
-; GCN-IR-NEXT:    s_min_u32 s8, s6, s7
-; GCN-IR-NEXT:    s_sub_u32 s10, 59, s8
-; GCN-IR-NEXT:    s_subb_u32 s11, 0, 0
+; GCN-IR-NEXT:    s_min_u32 s12, s6, s7
+; GCN-IR-NEXT:    s_sub_u32 s8, 59, s12
+; GCN-IR-NEXT:    s_subb_u32 s9, 0, 0
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[4:5], s[2:3], 0
-; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[6:7], s[10:11], 63
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[12:13], s[10:11], 63
+; GCN-IR-NEXT:    v_cmp_gt_u64_e64 s[6:7], s[8:9], 63
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[10:11], s[8:9], 63
 ; GCN-IR-NEXT:    s_or_b64 s[4:5], s[4:5], s[6:7]
 ; GCN-IR-NEXT:    s_and_b64 s[6:7], s[4:5], exec
 ; GCN-IR-NEXT:    s_cselect_b32 s7, 0, s3
 ; GCN-IR-NEXT:    s_cselect_b32 s6, 0, s2
-; GCN-IR-NEXT:    s_or_b64 s[4:5], s[4:5], s[12:13]
+; GCN-IR-NEXT:    s_or_b64 s[4:5], s[4:5], s[10:11]
 ; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[4:5]
 ; GCN-IR-NEXT:    s_mov_b64 s[4:5], 0
 ; GCN-IR-NEXT:    s_cbranch_vccz .LBB7_5
 ; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
-; GCN-IR-NEXT:    s_add_u32 s12, s10, 1
-; GCN-IR-NEXT:    s_addc_u32 s13, s11, 0
-; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[6:7], s[12:13], 0
-; GCN-IR-NEXT:    s_sub_i32 s9, 63, s10
+; GCN-IR-NEXT:    s_add_u32 s10, s8, 1
+; GCN-IR-NEXT:    s_addc_u32 s11, s9, 0
+; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[6:7], s[10:11], 0
+; GCN-IR-NEXT:    s_sub_i32 s8, 63, s8
 ; GCN-IR-NEXT:    s_andn2_b64 vcc, exec, s[6:7]
-; GCN-IR-NEXT:    s_lshl_b64 s[6:7], s[2:3], s9
+; GCN-IR-NEXT:    s_lshl_b64 s[6:7], s[2:3], s8
 ; GCN-IR-NEXT:    s_cbranch_vccz .LBB7_4
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
-; GCN-IR-NEXT:    s_lshr_b64 s[10:11], s[2:3], s12
-; GCN-IR-NEXT:    s_add_u32 s8, s8, 0xffffffc4
+; GCN-IR-NEXT:    s_lshr_b64 s[10:11], s[2:3], s10
+; GCN-IR-NEXT:    s_add_u32 s8, s12, 0xffffffc4
 ; GCN-IR-NEXT:    s_addc_u32 s9, 0, -1
 ; GCN-IR-NEXT:    s_mov_b64 s[12:13], 0
 ; GCN-IR-NEXT:    s_mov_b32 s5, 0
@@ -1153,26 +1153,26 @@ define i64 @v_test_urem_pow2_k_num_i64(i64 %x) {
 ; GCN-IR-NEXT:    v_ffbh_u32_e32 v2, v0
 ; GCN-IR-NEXT:    v_add_i32_e32 v2, vcc, 32, v2
 ; GCN-IR-NEXT:    v_ffbh_u32_e32 v3, v1
-; GCN-IR-NEXT:    v_min_u32_e32 v6, v2, v3
-; GCN-IR-NEXT:    v_add_i32_e32 v3, vcc, 0xffffffd0, v6
-; GCN-IR-NEXT:    v_addc_u32_e64 v4, s[6:7], 0, -1, vcc
+; GCN-IR-NEXT:    v_min_u32_e32 v10, v2, v3
+; GCN-IR-NEXT:    v_add_i32_e32 v2, vcc, 0xffffffd0, v10
+; GCN-IR-NEXT:    v_addc_u32_e64 v3, s[6:7], 0, -1, vcc
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e64 s[4:5], 0, v[0:1]
-; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, 63, v[3:4]
-; GCN-IR-NEXT:    v_cmp_ne_u64_e64 s[6:7], 63, v[3:4]
-; GCN-IR-NEXT:    v_mov_b32_e32 v5, 0x8000
+; GCN-IR-NEXT:    v_cmp_lt_u64_e32 vcc, 63, v[2:3]
+; GCN-IR-NEXT:    v_cmp_ne_u64_e64 s[6:7], 63, v[2:3]
+; GCN-IR-NEXT:    v_mov_b32_e32 v4, 0x8000
 ; GCN-IR-NEXT:    s_or_b64 s[4:5], s[4:5], vcc
-; GCN-IR-NEXT:    v_cndmask_b32_e64 v5, v5, 0, s[4:5]
+; GCN-IR-NEXT:    v_cndmask_b32_e64 v4, v4, 0, s[4:5]
 ; GCN-IR-NEXT:    s_xor_b64 s[4:5], s[4:5], -1
-; GCN-IR-NEXT:    v_mov_b32_e32 v2, 0
+; GCN-IR-NEXT:    v_mov_b32_e32 v5, 0
 ; GCN-IR-NEXT:    s_and_b64 s[4:5], s[4:5], s[6:7]
 ; GCN-IR-NEXT:    s_and_saveexec_b64 s[6:7], s[4:5]
 ; GCN-IR-NEXT:    s_cbranch_execz .LBB8_6
 ; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
-; GCN-IR-NEXT:    v_add_i32_e32 v7, vcc, 1, v3
-; GCN-IR-NEXT:    v_sub_i32_e64 v2, s[4:5], 63, v3
-; GCN-IR-NEXT:    v_addc_u32_e32 v8, vcc, 0, v4, vcc
+; GCN-IR-NEXT:    v_add_i32_e32 v6, vcc, 1, v2
+; GCN-IR-NEXT:    v_sub_i32_e64 v2, s[4:5], 63, v2
+; GCN-IR-NEXT:    v_addc_u32_e32 v7, vcc, 0, v3, vcc
 ; GCN-IR-NEXT:    s_mov_b64 s[4:5], 0x8000
-; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[7:8]
+; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[6:7]
 ; GCN-IR-NEXT:    v_lshl_b64 v[2:3], s[4:5], v2
 ; GCN-IR-NEXT:    v_mov_b32_e32 v4, 0
 ; GCN-IR-NEXT:    v_mov_b32_e32 v5, 0
@@ -1182,8 +1182,8 @@ define i64 @v_test_urem_pow2_k_num_i64(i64 %x) {
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
 ; GCN-IR-NEXT:    v_add_i32_e32 v12, vcc, -1, v0
 ; GCN-IR-NEXT:    v_addc_u32_e32 v13, vcc, -1, v1, vcc
-; GCN-IR-NEXT:    v_lshr_b64 v[8:9], s[4:5], v7
-; GCN-IR-NEXT:    v_sub_i32_e32 v6, vcc, 47, v6
+; GCN-IR-NEXT:    v_lshr_b64 v[8:9], s[4:5], v6
+; GCN-IR-NEXT:    v_sub_i32_e32 v6, vcc, 47, v10
 ; GCN-IR-NEXT:    v_mov_b32_e32 v10, 0
 ; GCN-IR-NEXT:    v_subb_u32_e64 v7, s[4:5], 0, 0, vcc
 ; GCN-IR-NEXT:    s_mov_b64 s[10:11], 0
@@ -1217,15 +1217,15 @@ define i64 @v_test_urem_pow2_k_num_i64(i64 %x) {
 ; GCN-IR-NEXT:    s_or_b64 exec, exec, s[10:11]
 ; GCN-IR-NEXT:  .LBB8_5: ; %Flow4
 ; GCN-IR-NEXT:    s_or_b64 exec, exec, s[8:9]
-; GCN-IR-NEXT:    v_lshl_b64 v[6:7], v[2:3], 1
-; GCN-IR-NEXT:    v_or_b32_e32 v2, v5, v7
-; GCN-IR-NEXT:    v_or_b32_e32 v5, v4, v6
+; GCN-IR-NEXT:    v_lshl_b64 v[2:3], v[2:3], 1
+; GCN-IR-NEXT:    v_or_b32_e32 v5, v5, v3
+; GCN-IR-NEXT:    v_or_b32_e32 v4, v4, v2
 ; GCN-IR-NEXT:  .LBB8_6: ; %Flow5
 ; GCN-IR-NEXT:    s_or_b64 exec, exec, s[6:7]
-; GCN-IR-NEXT:    v_mul_lo_u32 v2, v0, v2
-; GCN-IR-NEXT:    v_mul_hi_u32 v3, v0, v5
-; GCN-IR-NEXT:    v_mul_lo_u32 v1, v1, v5
-; GCN-IR-NEXT:    v_mul_lo_u32 v0, v0, v5
+; GCN-IR-NEXT:    v_mul_lo_u32 v2, v0, v5
+; GCN-IR-NEXT:    v_mul_hi_u32 v3, v0, v4
+; GCN-IR-NEXT:    v_mul_lo_u32 v1, v1, v4
+; GCN-IR-NEXT:    v_mul_lo_u32 v0, v0, v4
 ; GCN-IR-NEXT:    v_add_i32_e32 v2, vcc, v3, v2
 ; GCN-IR-NEXT:    v_add_i32_e32 v1, vcc, v2, v1
 ; GCN-IR-NEXT:    v_sub_i32_e32 v0, vcc, 0x8000, v0
@@ -1249,8 +1249,8 @@ define i64 @v_test_urem_pow2_k_den_i64(i64 %x) {
 ; GCN-IR-NEXT:    v_ffbh_u32_e32 v2, v0
 ; GCN-IR-NEXT:    v_add_i32_e64 v2, s[4:5], 32, v2
 ; GCN-IR-NEXT:    v_ffbh_u32_e32 v3, v1
-; GCN-IR-NEXT:    v_min_u32_e32 v6, v2, v3
-; GCN-IR-NEXT:    v_sub_i32_e64 v2, s[4:5], 48, v6
+; GCN-IR-NEXT:    v_min_u32_e32 v10, v2, v3
+; GCN-IR-NEXT:    v_sub_i32_e64 v2, s[4:5], 48, v10
 ; GCN-IR-NEXT:    v_subb_u32_e64 v3, s[4:5], 0, 0, s[4:5]
 ; GCN-IR-NEXT:    v_cmp_eq_u64_e32 vcc, 0, v[0:1]
 ; GCN-IR-NEXT:    v_cmp_lt_u64_e64 s[4:5], 63, v[2:3]
@@ -1263,10 +1263,10 @@ define i64 @v_test_urem_pow2_k_den_i64(i64 %x) {
 ; GCN-IR-NEXT:    s_and_saveexec_b64 s[6:7], s[4:5]
 ; GCN-IR-NEXT:    s_cbranch_execz .LBB9_6
 ; GCN-IR-NEXT:  ; %bb.1: ; %udiv-bb1
-; GCN-IR-NEXT:    v_add_i32_e32 v7, vcc, 1, v2
-; GCN-IR-NEXT:    v_addc_u32_e32 v8, vcc, 0, v3, vcc
+; GCN-IR-NEXT:    v_add_i32_e32 v6, vcc, 1, v2
+; GCN-IR-NEXT:    v_addc_u32_e32 v7, vcc, 0, v3, vcc
 ; GCN-IR-NEXT:    v_sub_i32_e64 v2, s[4:5], 63, v2
-; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[7:8]
+; GCN-IR-NEXT:    v_cmp_ne_u64_e32 vcc, 0, v[6:7]
 ; GCN-IR-NEXT:    v_lshl_b64 v[2:3], v[0:1], v2
 ; GCN-IR-NEXT:    v_mov_b32_e32 v4, 0
 ; GCN-IR-NEXT:    v_mov_b32_e32 v5, 0
@@ -1274,8 +1274,8 @@ define i64 @v_test_urem_pow2_k_den_i64(i64 %x) {
 ; GCN-IR-NEXT:    s_xor_b64 s[8:9], exec, s[4:5]
 ; GCN-IR-NEXT:    s_cbranch_execz .LBB9_5
 ; GCN-IR-NEXT:  ; %bb.2: ; %udiv-preheader
-; GCN-IR-NEXT:    v_lshr_b64 v[8:9], v[0:1], v7
-; GCN-IR-NEXT:    v_add_i32_e32 v6, vcc, 0xffffffcf, v6
+; GCN-IR-NEXT:    v_lshr_b64 v[8:9], v[0:1], v6
+; GCN-IR-NEXT:    v_add_i32_e32 v6, vcc, 0xffffffcf, v10
 ; GCN-IR-NEXT:    v_mov_b32_e32 v10, 0
 ; GCN-IR-NEXT:    v_addc_u32_e64 v7, s[4:5], 0, -1, vcc
 ; GCN-IR-NEXT:    s_mov_b64 s[10:11], 0
diff --git a/llvm/test/CodeGen/ARM/loopvectorize_pr33804.ll b/llvm/test/CodeGen/ARM/loopvectorize_pr33804.ll
index 540cbbfe96e81232d8d6abb771fd63e9f9bc3c7d..8038dad3fe92f11afa2a558c9c491cc059f843cb 100644
--- a/llvm/test/CodeGen/ARM/loopvectorize_pr33804.ll
+++ b/llvm/test/CodeGen/ARM/loopvectorize_pr33804.ll
@@ -15,16 +15,16 @@ target triple = "armv7-unknown-linux-gnueabihf"
 ; CHECK-LABEL: @cvCalcEMD2
 ; CHECK: vector.body
 ; CHECK: store <{{[0-9]+}} x ptr>
-define void @cvCalcEMD2() local_unnamed_addr #0 personality ptr @__gxx_personality_v0 {
+define void @cvCalcEMD2(ptr %dst) {
 entry:
   br label %for.body14.i.i
 
 for.body14.i.i:                                   ; preds = %for.body14.i.i, %entry
   %i.1424.i.i = phi i32 [ %inc21.i.i, %for.body14.i.i ], [ 0, %entry ]
-  %arrayidx15.i.i1427 = getelementptr inbounds %struct.CvNode1D, ptr undef, i32 %i.1424.i.i
+  %arrayidx15.i.i1427 = getelementptr inbounds %struct.CvNode1D, ptr %dst, i32 %i.1424.i.i
   store float 0xC415AF1D80000000, ptr %arrayidx15.i.i1427, align 4
-  %next19.i.i = getelementptr inbounds %struct.CvNode1D, ptr undef, i32 %i.1424.i.i, i32 1
-  store ptr undef, ptr %next19.i.i, align 4
+  %next19.i.i = getelementptr inbounds %struct.CvNode1D, ptr %dst, i32 %i.1424.i.i, i32 1
+  store ptr %dst, ptr %next19.i.i, align 4
   %inc21.i.i = add nuw nsw i32 %i.1424.i.i, 1
   %exitcond438.i.i = icmp eq i32 %inc21.i.i, 0
   br i1 %exitcond438.i.i, label %for.end22.i.i, label %for.body14.i.i
@@ -40,15 +40,15 @@ for.end22.i.i:                                    ; preds = %for.body14.i.i
 ; CHECK-LABEL: @cvCalcEMD2_2
 ; CHECK: vector.body
 ; CHECK: store <{{[0-9]+}} x float>
-define void @cvCalcEMD2_2() local_unnamed_addr #0 personality ptr @__gxx_personality_v0 {
+define void @cvCalcEMD2_2(ptr %dst) {
 entry:
   br label %for.body14.i.i
 
 for.body14.i.i:                                   ; preds = %for.body14.i.i, %entry
   %i.1424.i.i = phi i32 [ %inc21.i.i, %for.body14.i.i ], [ 0, %entry ]
-  %next19.i.i = getelementptr inbounds %struct.CvNode1D2, ptr undef, i32 %i.1424.i.i, i32 0
-  store ptr undef, ptr %next19.i.i, align 4
-  %arrayidx15.i.i1427 = getelementptr inbounds %struct.CvNode1D2, ptr undef, i32 %i.1424.i.i
+  %next19.i.i = getelementptr inbounds %struct.CvNode1D2, ptr %dst, i32 %i.1424.i.i, i32 0
+  store ptr %dst, ptr %next19.i.i, align 4
+  %arrayidx15.i.i1427 = getelementptr inbounds %struct.CvNode1D2, ptr %dst, i32 %i.1424.i.i
   %val.i.i = getelementptr inbounds %struct.CvNode1D2, ptr %arrayidx15.i.i1427, i32 0, i32 1
   store float 0xC415AF1D80000000, ptr %val.i.i, align 4
   %inc21.i.i = add nuw nsw i32 %i.1424.i.i, 1
@@ -64,16 +64,20 @@ for.end22.i.i:                                    ; preds = %for.body14.i.i
 ; CHECK-LABEL: @cvCalcEMD3
 ; CHECK: vector.body
 ; CHECK: inttoptr <{{[0-9]+}} x i32>
-define void @cvCalcEMD3() local_unnamed_addr #0 personality ptr @__gxx_personality_v0 {
+define void @cvCalcEMD3(ptr %src, ptr %dst) {
 entry:
   br label %for.body14.i.i
 
 for.body14.i.i:                                   ; preds = %for.body14.i.i, %entry
   %i.1424.i.i = phi i32 [ %inc21.i.i, %for.body14.i.i ], [ 0, %entry ]
-  %arrayidx15.i.i1427 = getelementptr inbounds %struct.CvNode1D, ptr undef, i32 %i.1424.i.i
+  %arrayidx15.i.i1427 = getelementptr inbounds %struct.CvNode1D, ptr %src, i32 %i.1424.i.i
   %loadf = load float, ptr %arrayidx15.i.i1427, align 4
-  %next19.i.i = getelementptr inbounds %struct.CvNode1D, ptr undef, i32 %i.1424.i.i, i32 1
+  %next19.i.i = getelementptr inbounds %struct.CvNode1D, ptr %src, i32 %i.1424.i.i, i32 1
   %loadp = load ptr, ptr %next19.i.i, align 4
+  %dst.ptr = getelementptr inbounds %struct.CvNode1D, ptr %dst, i32 %i.1424.i.i
+  %dst.ptr.1 = getelementptr inbounds %struct.CvNode1D, ptr %dst, i32 %i.1424.i.i, i32 1
+  store float %loadf, ptr %dst.ptr, align 4
+  store ptr %loadp, ptr %dst.ptr.1, align 4
   %inc21.i.i = add nuw nsw i32 %i.1424.i.i, 1
   %exitcond438.i.i = icmp eq i32 %inc21.i.i, 0
   br i1 %exitcond438.i.i, label %for.end22.i.i, label %for.body14.i.i
@@ -87,17 +91,21 @@ for.end22.i.i:                                    ; preds = %for.body14.i.i
 ; CHECK-LABEL: @cvCalcEMD3_2
 ; CHECK: vector.body
 ; CHECK: ptrtoint <{{[0-9]+}} x ptr>
-define void @cvCalcEMD3_2() local_unnamed_addr #0 personality ptr @__gxx_personality_v0 {
+define void @cvCalcEMD3_2(ptr %src, ptr %dst) {
 entry:
   br label %for.body14.i.i
 
 for.body14.i.i:                                   ; preds = %for.body14.i.i, %entry
   %i.1424.i.i = phi i32 [ %inc21.i.i, %for.body14.i.i ], [ 0, %entry ]
-  %next19.i.i = getelementptr inbounds %struct.CvNode1D2, ptr undef, i32 %i.1424.i.i, i32 0
+  %next19.i.i = getelementptr inbounds %struct.CvNode1D2, ptr %src, i32 %i.1424.i.i, i32 0
   %loadp = load ptr, ptr %next19.i.i, align 4
-  %arrayidx15.i.i1427 = getelementptr inbounds %struct.CvNode1D2, ptr undef, i32 %i.1424.i.i
+  %arrayidx15.i.i1427 = getelementptr inbounds %struct.CvNode1D2, ptr %src, i32 %i.1424.i.i
   %val.i.i = getelementptr inbounds %struct.CvNode1D2, ptr %arrayidx15.i.i1427, i32 0, i32 1
   %loadf = load float, ptr %val.i.i, align 4
+  %dst.ptr = getelementptr inbounds %struct.CvNode1D, ptr %dst, i32 %i.1424.i.i
+  %dst.ptr.1 = getelementptr inbounds %struct.CvNode1D, ptr %dst, i32 %i.1424.i.i, i32 1
+  store float %loadf, ptr %dst.ptr, align 4
+  store ptr %loadp, ptr %dst.ptr.1, align 4
   %inc21.i.i = add nuw nsw i32 %i.1424.i.i, 1
   %exitcond438.i.i = icmp eq i32 %inc21.i.i, 0
   br i1 %exitcond438.i.i, label %for.end22.i.i, label %for.body14.i.i
@@ -105,8 +113,3 @@ for.body14.i.i:                                   ; preds = %for.body14.i.i, %en
 for.end22.i.i:                                    ; preds = %for.body14.i.i
   unreachable
 }
-
-declare i32 @__gxx_personality_v0(...)
-
-attributes #0 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="generic" "target-features"="+dsp,+neon,+vfp3,-thumb-mode" "unsafe-fp-math"="false" "use-soft-float"="false" }
-
diff --git a/llvm/test/CodeGen/ARM/machine-outliner-noreturn.mir b/llvm/test/CodeGen/ARM/machine-outliner-noreturn.mir
new file mode 100644
index 0000000000000000000000000000000000000000..cd8313f27d73c7383173abcf93581e2be7c18b46
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/machine-outliner-noreturn.mir
@@ -0,0 +1,99 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -run-pass=machine-outliner %s -o - | FileCheck %s
+
+--- |
+  target datalayout = "e-m:e-p:32:32-Fi8-i64:64-v128:64:128-a:0:32-n32-S64"
+  target triple = "thumbv8.1m.main-arm-none-eabi"
+
+  @__stack_chk_guard = external dso_local global ptr
+
+  define hidden void @test1(i32 %P0, i32 %P1) {
+  entry:
+    ret void
+  }
+  define hidden void @test2(ptr %agg.result) { ret void }
+  define hidden void @test3(ptr %agg.result) { ret void }
+  define hidden void @test4(ptr %agg.result) { ret void }
+
+  declare void @noreturn(ptr, ptr, ptr) noreturn
+
+...
+---
+name:            test1
+tracksRegLiveness: true
+stack:
+  - { id: 1, name: '', type: spill-slot, offset: -12, size: 4, alignment: 4,
+      stack-id: default, callee-saved-register: '$lr', callee-saved-restored: true,
+      debug-info-variable: '', debug-info-expression: '', debug-info-location: '' }
+entry_values:    []
+callSites:       []
+debugValueSubstitutions: []
+constants:       []
+machineFunctionInfo:
+  isLRSpilled:     true
+body:             |
+  ; CHECK-LABEL: name: test1
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT:   liveins: $lr, $r0, $r1, $r2, $r3, $r4
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   early-clobber $sp = t2STR_PRE killed $lr, $sp, -8, 14 /* CC::al */, $noreg
+  ; CHECK-NEXT:   tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_0, implicit-def $lr, implicit $sp, implicit-def $lr, implicit-def $sp, implicit-def $r1, implicit $noreg, implicit $sp
+  ; CHECK-NEXT:   $lr, $sp = t2LDR_POST $sp, 8, 14 /* CC::al */, $noreg
+  ; CHECK-NEXT:   tCMPi8 killed renamable $r0, 0, 14 /* CC::al */, $noreg, implicit-def $cpsr
+  ; CHECK-NEXT:   t2STRDi8 killed $r2, killed $r3, $sp, 16, 14 /* CC::al */, $noreg
+  ; CHECK-NEXT:   INLINEASM &"", 1 /* sideeffect attdialect */, 327689 /* reguse:GPR */, killed renamable $lr
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.2(0x80000000)
+  ; CHECK-NEXT:   liveins: $r4
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   tBL 14 /* CC::al */, $noreg, @noreturn, csr_aapcs, implicit-def dead $lr, implicit $sp, implicit undef $r0, implicit undef $r1, implicit undef $r2, implicit-def $sp
+  bb.0:
+    liveins: $lr, $r0, $r1, $r2, $r3, $r4
+
+    $r1 = t2MOVi16 target-flags(arm-lo16) @__stack_chk_guard, 14 /* CC::al */, $noreg
+    $r1 = t2MOVTi16 killed $r1, target-flags(arm-hi16) @__stack_chk_guard, 14 /* CC::al */, $noreg
+
+    tCMPi8 killed renamable $r0, 0, 14 /* CC::al */, $noreg, implicit-def $cpsr
+    t2STRDi8 killed $r2, killed $r3, $sp, 16, 14 /* CC::al */, $noreg
+    INLINEASM &"", 1 /* sideeffect attdialect */, 327689 /* reguse:GPR */, killed renamable $lr
+
+  bb.1:
+    liveins: $r4
+
+  bb.2:
+    tBL 14 /* CC::al */, $noreg, @noreturn, csr_aapcs, implicit-def dead $lr, implicit $sp, implicit undef $r0, implicit undef $r1, implicit undef $r2, implicit-def $sp
+
+...
+---
+name:            test2
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name: test2
+    ; CHECK: tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_0, implicit-def $lr, implicit $sp, implicit-def $r1, implicit-def $lr, implicit $noreg, implicit $sp
+    $r1 = t2MOVi16 target-flags(arm-lo16) @__stack_chk_guard, 14 /* CC::al */, $noreg
+    $r1 = t2MOVTi16 killed $r1, target-flags(arm-hi16) @__stack_chk_guard, 14 /* CC::al */, $noreg
+...
+---
+name:            test3
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name: test3
+    ; CHECK: tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_0, implicit-def $lr, implicit $sp, implicit-def $r1, implicit-def $lr, implicit $noreg, implicit $sp
+    $r1 = t2MOVi16 target-flags(arm-lo16) @__stack_chk_guard, 14 /* CC::al */, $noreg
+    $r1 = t2MOVTi16 killed $r1, target-flags(arm-hi16) @__stack_chk_guard, 14 /* CC::al */, $noreg
+...
+---
+name:            test4
+tracksRegLiveness: true
+body:             |
+  bb.0:
+    ; CHECK-LABEL: name: test4
+    ; CHECK: tBL 14 /* CC::al */, $noreg, @OUTLINED_FUNCTION_0, implicit-def $lr, implicit $sp, implicit-def $r1, implicit-def $lr, implicit $noreg, implicit $sp
+    $r1 = t2MOVi16 target-flags(arm-lo16) @__stack_chk_guard, 14 /* CC::al */, $noreg
+    $r1 = t2MOVTi16 killed $r1, target-flags(arm-hi16) @__stack_chk_guard, 14 /* CC::al */, $noreg
diff --git a/llvm/test/CodeGen/BPF/dynamic-stack.ll b/llvm/test/CodeGen/BPF/dynamic-stack.ll
new file mode 100644
index 0000000000000000000000000000000000000000..046523ded8001c2682dac548dacd8eb1ac47206d
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/dynamic-stack.ll
@@ -0,0 +1,8 @@
+; RUN: not llc < %s -mtriple=bpfel 2>&1 | FileCheck %s
+
+define i64 @vla(i64 %num) {
+; CHECK: unsupported dynamic stack allocation
+    %vla = alloca i32, i64 %num
+    %ret = ptrtoint ptr %vla to i64
+    ret i64 %ret
+}
diff --git a/llvm/test/CodeGen/BPF/sdiv_error.ll b/llvm/test/CodeGen/BPF/sdiv_error.ll
index fc79a1d44c16ba17443b4e4aaeb2bd27479b867d..a5dcc6271224b0c0ddf9b70521b812abc3259b1f 100644
--- a/llvm/test/CodeGen/BPF/sdiv_error.ll
+++ b/llvm/test/CodeGen/BPF/sdiv_error.ll
@@ -1,9 +1,8 @@
-; RUN: not --crash llc -march=bpf < %s 2> %t1
+; RUN: not llc -mtriple=bpf < %s 2> %t1
 ; RUN: FileCheck %s < %t1
-; CHECK: Unsupport signed division
+; CHECK: unsupported signed division
 
-; Function Attrs: norecurse nounwind readnone
-define i32 @test(i32 %len) #0 {
-  %1 = srem i32 %len, 15
+define i32 @test(i32 %len) {
+  %1 = sdiv i32 %len, 15
   ret i32 %1
 }
diff --git a/llvm/test/CodeGen/BPF/srem_error.ll b/llvm/test/CodeGen/BPF/srem_error.ll
new file mode 100644
index 0000000000000000000000000000000000000000..883e72d32dc7edc71e536839ca46776ada166ee7
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/srem_error.ll
@@ -0,0 +1,8 @@
+; RUN: not llc -mtriple=bpf < %s 2> %t1
+; RUN: FileCheck %s < %t1
+; CHECK: unsupported signed division
+
+define i32 @test(i32 %len) {
+  %1 = srem i32 %len, 15
+  ret i32 %1
+}
diff --git a/llvm/test/CodeGen/PowerPC/memcmp-mergeexpand.ll b/llvm/test/CodeGen/PowerPC/memcmp-mergeexpand.ll
index b209d5809c573184fd1951716e9532c1609eb097..29910646c89371d7cf0d34902f924cc4186de05e 100644
--- a/llvm/test/CodeGen/PowerPC/memcmp-mergeexpand.ll
+++ b/llvm/test/CodeGen/PowerPC/memcmp-mergeexpand.ll
@@ -1,7 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -verify-machineinstrs -mcpu=pwr8 -mtriple=powerpc64le-unknown-gnu-linux  < %s | FileCheck %s -check-prefix=PPC64LE
 
-; This tests interaction between MergeICmp and ExpandMemCmp.
+; This tests interaction between MergeICmp and expand-memcmp.
 
 %"struct.std::pair" = type { i32, i32 }
 
diff --git a/llvm/test/CodeGen/RISCV/attributes.ll b/llvm/test/CodeGen/RISCV/attributes.ll
index 030ae06af6d28281a8a1e7b179e3919db91db8c2..b34ccb3ff0f937085c2cdd7d604a56472e80caad 100644
--- a/llvm/test/CodeGen/RISCV/attributes.ll
+++ b/llvm/test/CodeGen/RISCV/attributes.ll
@@ -271,7 +271,7 @@
 ; RV32ZVFBFMIN: .attribute 5, "rv32i2p1_f2p2_zicsr2p0_zfbfmin0p8_zve32f1p0_zve32x1p0_zvfbfmin0p8_zvl32b1p0"
 ; RV32ZVFBFWMA: .attribute 5, "rv32i2p1_f2p2_zicsr2p0_zfbfmin0p8_zve32f1p0_zve32x1p0_zvfbfmin0p8_zvfbfwma0p8_zvl32b1p0"
 ; RV32ZACAS: .attribute 5, "rv32i2p1_a2p1_zacas1p0"
-; RV32ZICFILP: .attribute 5, "rv32i2p1_zicfilp0p2"
+; RV32ZICFILP: .attribute 5, "rv32i2p1_zicfilp0p4"
 
 ; RV64M: .attribute 5, "rv64i2p1_m2p0"
 ; RV64ZMMUL: .attribute 5, "rv64i2p1_zmmul1p0"
@@ -360,7 +360,7 @@
 ; RV64ZVFBFMIN: .attribute 5, "rv64i2p1_f2p2_zicsr2p0_zfbfmin0p8_zve32f1p0_zve32x1p0_zvfbfmin0p8_zvl32b1p0"
 ; RV64ZVFBFWMA: .attribute 5, "rv64i2p1_f2p2_zicsr2p0_zfbfmin0p8_zve32f1p0_zve32x1p0_zvfbfmin0p8_zvfbfwma0p8_zvl32b1p0"
 ; RV64ZACAS: .attribute 5, "rv64i2p1_a2p1_zacas1p0"
-; RV64ZICFILP: .attribute 5, "rv64i2p1_zicfilp0p2"
+; RV64ZICFILP: .attribute 5, "rv64i2p1_zicfilp0p4"
 
 define i32 @addi(i32 %a) {
   %1 = add i32 %a, 1
diff --git a/llvm/test/CodeGen/RISCV/double-previous-failure.ll b/llvm/test/CodeGen/RISCV/double-previous-failure.ll
index d9d3442043a90ad04c60dbb224c44de90ad1ae0d..aec27b58e1fc490edd3eb7dd60e4a1e130cf96d5 100644
--- a/llvm/test/CodeGen/RISCV/double-previous-failure.ll
+++ b/llvm/test/CodeGen/RISCV/double-previous-failure.ll
@@ -31,16 +31,17 @@ define i32 @main() nounwind {
 ; RV32IFD-NEXT:    fld fa5, 0(sp)
 ; RV32IFD-NEXT:    lui a0, %hi(.LCPI1_0)
 ; RV32IFD-NEXT:    fld fa4, %lo(.LCPI1_0)(a0)
-; RV32IFD-NEXT:    lui a0, %hi(.LCPI1_1)
-; RV32IFD-NEXT:    fld fa3, %lo(.LCPI1_1)(a0)
 ; RV32IFD-NEXT:    flt.d a0, fa5, fa4
-; RV32IFD-NEXT:    flt.d a1, fa3, fa5
-; RV32IFD-NEXT:    or a0, a0, a1
-; RV32IFD-NEXT:    beqz a0, .LBB1_2
-; RV32IFD-NEXT:  # %bb.1: # %if.then
-; RV32IFD-NEXT:    call abort@plt
-; RV32IFD-NEXT:  .LBB1_2: # %if.end
+; RV32IFD-NEXT:    bnez a0, .LBB1_3
+; RV32IFD-NEXT:  # %bb.1: # %entry
+; RV32IFD-NEXT:    lui a0, %hi(.LCPI1_1)
+; RV32IFD-NEXT:    fld fa4, %lo(.LCPI1_1)(a0)
+; RV32IFD-NEXT:    flt.d a0, fa4, fa5
+; RV32IFD-NEXT:    bnez a0, .LBB1_3
+; RV32IFD-NEXT:  # %bb.2: # %if.end
 ; RV32IFD-NEXT:    call exit@plt
+; RV32IFD-NEXT:  .LBB1_3: # %if.then
+; RV32IFD-NEXT:    call abort@plt
 ;
 ; RV32IZFINXZDINX-LABEL: main:
 ; RV32IZFINXZDINX:       # %bb.0: # %entry
@@ -56,17 +57,18 @@ define i32 @main() nounwind {
 ; RV32IZFINXZDINX-NEXT:    lui a2, %hi(.LCPI1_0)
 ; RV32IZFINXZDINX-NEXT:    lw a3, %lo(.LCPI1_0+4)(a2)
 ; RV32IZFINXZDINX-NEXT:    lw a2, %lo(.LCPI1_0)(a2)
-; RV32IZFINXZDINX-NEXT:    lui a4, %hi(.LCPI1_1)
-; RV32IZFINXZDINX-NEXT:    lw a5, %lo(.LCPI1_1+4)(a4)
-; RV32IZFINXZDINX-NEXT:    lw a4, %lo(.LCPI1_1)(a4)
 ; RV32IZFINXZDINX-NEXT:    flt.d a2, a0, a2
-; RV32IZFINXZDINX-NEXT:    flt.d a0, a4, a0
-; RV32IZFINXZDINX-NEXT:    or a0, a2, a0
-; RV32IZFINXZDINX-NEXT:    beqz a0, .LBB1_2
-; RV32IZFINXZDINX-NEXT:  # %bb.1: # %if.then
-; RV32IZFINXZDINX-NEXT:    call abort@plt
-; RV32IZFINXZDINX-NEXT:  .LBB1_2: # %if.end
+; RV32IZFINXZDINX-NEXT:    bnez a2, .LBB1_3
+; RV32IZFINXZDINX-NEXT:  # %bb.1: # %entry
+; RV32IZFINXZDINX-NEXT:    lui a2, %hi(.LCPI1_1)
+; RV32IZFINXZDINX-NEXT:    lw a3, %lo(.LCPI1_1+4)(a2)
+; RV32IZFINXZDINX-NEXT:    lw a2, %lo(.LCPI1_1)(a2)
+; RV32IZFINXZDINX-NEXT:    flt.d a0, a2, a0
+; RV32IZFINXZDINX-NEXT:    bnez a0, .LBB1_3
+; RV32IZFINXZDINX-NEXT:  # %bb.2: # %if.end
 ; RV32IZFINXZDINX-NEXT:    call exit@plt
+; RV32IZFINXZDINX-NEXT:  .LBB1_3: # %if.then
+; RV32IZFINXZDINX-NEXT:    call abort@plt
 entry:
   %call = call double @test(double 2.000000e+00)
   %cmp = fcmp olt double %call, 2.400000e-01
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-extract-subvector.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-extract-subvector.ll
index b4260b04604cd06ac9164039f1ce8ef554472abc..0b9db09aab3a9c3821a0b2b4a7e156b1b35f5ed2 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-extract-subvector.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-extract-subvector.ll
@@ -284,14 +284,22 @@ define void @extract_v8i32_nxv16i32_8(<vscale x 16 x i32> %x, ptr %y) {
 }
 
 define void @extract_v8i1_v64i1_0(ptr %x, ptr %y) {
-; CHECK-LABEL: extract_v8i1_v64i1_0:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a2, 64
-; CHECK-NEXT:    vsetvli zero, a2, e8, m4, ta, ma
-; CHECK-NEXT:    vlm.v v8, (a0)
-; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
-; CHECK-NEXT:    vsm.v v8, (a1)
-; CHECK-NEXT:    ret
+; CHECK-V-LABEL: extract_v8i1_v64i1_0:
+; CHECK-V:       # %bb.0:
+; CHECK-V-NEXT:    li a2, 64
+; CHECK-V-NEXT:    vsetvli zero, a2, e8, m4, ta, ma
+; CHECK-V-NEXT:    vlm.v v8, (a0)
+; CHECK-V-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-V-NEXT:    vsm.v v8, (a1)
+; CHECK-V-NEXT:    ret
+;
+; CHECK-KNOWNVLEN128-LABEL: extract_v8i1_v64i1_0:
+; CHECK-KNOWNVLEN128:       # %bb.0:
+; CHECK-KNOWNVLEN128-NEXT:    vsetvli a2, zero, e8, m4, ta, ma
+; CHECK-KNOWNVLEN128-NEXT:    vlm.v v8, (a0)
+; CHECK-KNOWNVLEN128-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-KNOWNVLEN128-NEXT:    vsm.v v8, (a1)
+; CHECK-KNOWNVLEN128-NEXT:    ret
   %a = load <64 x i1>, ptr %x
   %c = call <8 x i1> @llvm.vector.extract.v8i1.v64i1(<64 x i1> %a, i64 0)
   store <8 x i1> %c, ptr %y
@@ -299,16 +307,26 @@ define void @extract_v8i1_v64i1_0(ptr %x, ptr %y) {
 }
 
 define void @extract_v8i1_v64i1_8(ptr %x, ptr %y) {
-; CHECK-LABEL: extract_v8i1_v64i1_8:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a2, 64
-; CHECK-NEXT:    vsetvli zero, a2, e8, m4, ta, ma
-; CHECK-NEXT:    vlm.v v8, (a0)
-; CHECK-NEXT:    vsetivli zero, 1, e8, mf2, ta, ma
-; CHECK-NEXT:    vslidedown.vi v8, v8, 1
-; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
-; CHECK-NEXT:    vsm.v v8, (a1)
-; CHECK-NEXT:    ret
+; CHECK-V-LABEL: extract_v8i1_v64i1_8:
+; CHECK-V:       # %bb.0:
+; CHECK-V-NEXT:    li a2, 64
+; CHECK-V-NEXT:    vsetvli zero, a2, e8, m4, ta, ma
+; CHECK-V-NEXT:    vlm.v v8, (a0)
+; CHECK-V-NEXT:    vsetivli zero, 1, e8, mf2, ta, ma
+; CHECK-V-NEXT:    vslidedown.vi v8, v8, 1
+; CHECK-V-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-V-NEXT:    vsm.v v8, (a1)
+; CHECK-V-NEXT:    ret
+;
+; CHECK-KNOWNVLEN128-LABEL: extract_v8i1_v64i1_8:
+; CHECK-KNOWNVLEN128:       # %bb.0:
+; CHECK-KNOWNVLEN128-NEXT:    vsetvli a2, zero, e8, m4, ta, ma
+; CHECK-KNOWNVLEN128-NEXT:    vlm.v v8, (a0)
+; CHECK-KNOWNVLEN128-NEXT:    vsetivli zero, 1, e8, mf2, ta, ma
+; CHECK-KNOWNVLEN128-NEXT:    vslidedown.vi v8, v8, 1
+; CHECK-KNOWNVLEN128-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-KNOWNVLEN128-NEXT:    vsm.v v8, (a1)
+; CHECK-KNOWNVLEN128-NEXT:    ret
   %a = load <64 x i1>, ptr %x
   %c = call <8 x i1> @llvm.vector.extract.v8i1.v64i1(<64 x i1> %a, i64 8)
   store <8 x i1> %c, ptr %y
@@ -316,16 +334,26 @@ define void @extract_v8i1_v64i1_8(ptr %x, ptr %y) {
 }
 
 define void @extract_v8i1_v64i1_48(ptr %x, ptr %y) {
-; CHECK-LABEL: extract_v8i1_v64i1_48:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a2, 64
-; CHECK-NEXT:    vsetvli zero, a2, e8, m4, ta, ma
-; CHECK-NEXT:    vlm.v v8, (a0)
-; CHECK-NEXT:    vsetivli zero, 1, e8, mf2, ta, ma
-; CHECK-NEXT:    vslidedown.vi v8, v8, 6
-; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
-; CHECK-NEXT:    vsm.v v8, (a1)
-; CHECK-NEXT:    ret
+; CHECK-V-LABEL: extract_v8i1_v64i1_48:
+; CHECK-V:       # %bb.0:
+; CHECK-V-NEXT:    li a2, 64
+; CHECK-V-NEXT:    vsetvli zero, a2, e8, m4, ta, ma
+; CHECK-V-NEXT:    vlm.v v8, (a0)
+; CHECK-V-NEXT:    vsetivli zero, 1, e8, mf2, ta, ma
+; CHECK-V-NEXT:    vslidedown.vi v8, v8, 6
+; CHECK-V-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-V-NEXT:    vsm.v v8, (a1)
+; CHECK-V-NEXT:    ret
+;
+; CHECK-KNOWNVLEN128-LABEL: extract_v8i1_v64i1_48:
+; CHECK-KNOWNVLEN128:       # %bb.0:
+; CHECK-KNOWNVLEN128-NEXT:    vsetvli a2, zero, e8, m4, ta, ma
+; CHECK-KNOWNVLEN128-NEXT:    vlm.v v8, (a0)
+; CHECK-KNOWNVLEN128-NEXT:    vsetivli zero, 1, e8, mf2, ta, ma
+; CHECK-KNOWNVLEN128-NEXT:    vslidedown.vi v8, v8, 6
+; CHECK-KNOWNVLEN128-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-KNOWNVLEN128-NEXT:    vsm.v v8, (a1)
+; CHECK-KNOWNVLEN128-NEXT:    ret
   %a = load <64 x i1>, ptr %x
   %c = call <8 x i1> @llvm.vector.extract.v8i1.v64i1(<64 x i1> %a, i64 48)
   store <8 x i1> %c, ptr %y
@@ -407,22 +435,38 @@ define void @extract_v8i1_nxv64i1_192(<vscale x 64 x i1> %x, ptr %y) {
 }
 
 define void @extract_v2i1_v64i1_0(ptr %x, ptr %y) {
-; CHECK-LABEL: extract_v2i1_v64i1_0:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a2, 64
-; CHECK-NEXT:    vsetvli zero, a2, e8, m4, ta, ma
-; CHECK-NEXT:    vlm.v v0, (a0)
-; CHECK-NEXT:    vsetivli zero, 2, e8, mf8, ta, ma
-; CHECK-NEXT:    vmv.v.i v8, 0
-; CHECK-NEXT:    vmerge.vim v8, v8, 1, v0
-; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
-; CHECK-NEXT:    vmv.v.i v9, 0
-; CHECK-NEXT:    vsetivli zero, 2, e8, mf2, tu, ma
-; CHECK-NEXT:    vmv.v.v v9, v8
-; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
-; CHECK-NEXT:    vmsne.vi v8, v9, 0
-; CHECK-NEXT:    vsm.v v8, (a1)
-; CHECK-NEXT:    ret
+; CHECK-V-LABEL: extract_v2i1_v64i1_0:
+; CHECK-V:       # %bb.0:
+; CHECK-V-NEXT:    li a2, 64
+; CHECK-V-NEXT:    vsetvli zero, a2, e8, m4, ta, ma
+; CHECK-V-NEXT:    vlm.v v0, (a0)
+; CHECK-V-NEXT:    vsetivli zero, 2, e8, mf8, ta, ma
+; CHECK-V-NEXT:    vmv.v.i v8, 0
+; CHECK-V-NEXT:    vmerge.vim v8, v8, 1, v0
+; CHECK-V-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-V-NEXT:    vmv.v.i v9, 0
+; CHECK-V-NEXT:    vsetivli zero, 2, e8, mf2, tu, ma
+; CHECK-V-NEXT:    vmv.v.v v9, v8
+; CHECK-V-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-V-NEXT:    vmsne.vi v8, v9, 0
+; CHECK-V-NEXT:    vsm.v v8, (a1)
+; CHECK-V-NEXT:    ret
+;
+; CHECK-KNOWNVLEN128-LABEL: extract_v2i1_v64i1_0:
+; CHECK-KNOWNVLEN128:       # %bb.0:
+; CHECK-KNOWNVLEN128-NEXT:    vsetvli a2, zero, e8, m4, ta, ma
+; CHECK-KNOWNVLEN128-NEXT:    vlm.v v0, (a0)
+; CHECK-KNOWNVLEN128-NEXT:    vsetivli zero, 2, e8, mf8, ta, ma
+; CHECK-KNOWNVLEN128-NEXT:    vmv.v.i v8, 0
+; CHECK-KNOWNVLEN128-NEXT:    vmerge.vim v8, v8, 1, v0
+; CHECK-KNOWNVLEN128-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-KNOWNVLEN128-NEXT:    vmv.v.i v9, 0
+; CHECK-KNOWNVLEN128-NEXT:    vsetivli zero, 2, e8, mf2, tu, ma
+; CHECK-KNOWNVLEN128-NEXT:    vmv.v.v v9, v8
+; CHECK-KNOWNVLEN128-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-KNOWNVLEN128-NEXT:    vmsne.vi v8, v9, 0
+; CHECK-KNOWNVLEN128-NEXT:    vsm.v v8, (a1)
+; CHECK-KNOWNVLEN128-NEXT:    ret
   %a = load <64 x i1>, ptr %x
   %c = call <2 x i1> @llvm.vector.extract.v2i1.v64i1(<64 x i1> %a, i64 0)
   store <2 x i1> %c, ptr %y
@@ -430,27 +474,48 @@ define void @extract_v2i1_v64i1_0(ptr %x, ptr %y) {
 }
 
 define void @extract_v2i1_v64i1_2(ptr %x, ptr %y) {
-; CHECK-LABEL: extract_v2i1_v64i1_2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a2, 64
-; CHECK-NEXT:    vsetvli zero, a2, e8, m4, ta, ma
-; CHECK-NEXT:    vlm.v v0, (a0)
-; CHECK-NEXT:    vmv.v.i v8, 0
-; CHECK-NEXT:    vmerge.vim v8, v8, 1, v0
-; CHECK-NEXT:    vsetivli zero, 2, e8, m1, ta, ma
-; CHECK-NEXT:    vslidedown.vi v8, v8, 2
-; CHECK-NEXT:    vsetivli zero, 2, e8, mf8, ta, ma
-; CHECK-NEXT:    vmsne.vi v0, v8, 0
-; CHECK-NEXT:    vmv.v.i v8, 0
-; CHECK-NEXT:    vmerge.vim v8, v8, 1, v0
-; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
-; CHECK-NEXT:    vmv.v.i v9, 0
-; CHECK-NEXT:    vsetivli zero, 2, e8, mf2, tu, ma
-; CHECK-NEXT:    vmv.v.v v9, v8
-; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
-; CHECK-NEXT:    vmsne.vi v8, v9, 0
-; CHECK-NEXT:    vsm.v v8, (a1)
-; CHECK-NEXT:    ret
+; CHECK-V-LABEL: extract_v2i1_v64i1_2:
+; CHECK-V:       # %bb.0:
+; CHECK-V-NEXT:    li a2, 64
+; CHECK-V-NEXT:    vsetvli zero, a2, e8, m4, ta, ma
+; CHECK-V-NEXT:    vlm.v v0, (a0)
+; CHECK-V-NEXT:    vmv.v.i v8, 0
+; CHECK-V-NEXT:    vmerge.vim v8, v8, 1, v0
+; CHECK-V-NEXT:    vsetivli zero, 2, e8, m1, ta, ma
+; CHECK-V-NEXT:    vslidedown.vi v8, v8, 2
+; CHECK-V-NEXT:    vsetivli zero, 2, e8, mf8, ta, ma
+; CHECK-V-NEXT:    vmsne.vi v0, v8, 0
+; CHECK-V-NEXT:    vmv.v.i v8, 0
+; CHECK-V-NEXT:    vmerge.vim v8, v8, 1, v0
+; CHECK-V-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-V-NEXT:    vmv.v.i v9, 0
+; CHECK-V-NEXT:    vsetivli zero, 2, e8, mf2, tu, ma
+; CHECK-V-NEXT:    vmv.v.v v9, v8
+; CHECK-V-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-V-NEXT:    vmsne.vi v8, v9, 0
+; CHECK-V-NEXT:    vsm.v v8, (a1)
+; CHECK-V-NEXT:    ret
+;
+; CHECK-KNOWNVLEN128-LABEL: extract_v2i1_v64i1_2:
+; CHECK-KNOWNVLEN128:       # %bb.0:
+; CHECK-KNOWNVLEN128-NEXT:    vsetvli a2, zero, e8, m4, ta, ma
+; CHECK-KNOWNVLEN128-NEXT:    vlm.v v0, (a0)
+; CHECK-KNOWNVLEN128-NEXT:    vmv.v.i v8, 0
+; CHECK-KNOWNVLEN128-NEXT:    vmerge.vim v8, v8, 1, v0
+; CHECK-KNOWNVLEN128-NEXT:    vsetivli zero, 2, e8, m1, ta, ma
+; CHECK-KNOWNVLEN128-NEXT:    vslidedown.vi v8, v8, 2
+; CHECK-KNOWNVLEN128-NEXT:    vsetivli zero, 2, e8, mf8, ta, ma
+; CHECK-KNOWNVLEN128-NEXT:    vmsne.vi v0, v8, 0
+; CHECK-KNOWNVLEN128-NEXT:    vmv.v.i v8, 0
+; CHECK-KNOWNVLEN128-NEXT:    vmerge.vim v8, v8, 1, v0
+; CHECK-KNOWNVLEN128-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-KNOWNVLEN128-NEXT:    vmv.v.i v9, 0
+; CHECK-KNOWNVLEN128-NEXT:    vsetivli zero, 2, e8, mf2, tu, ma
+; CHECK-KNOWNVLEN128-NEXT:    vmv.v.v v9, v8
+; CHECK-KNOWNVLEN128-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-KNOWNVLEN128-NEXT:    vmsne.vi v8, v9, 0
+; CHECK-KNOWNVLEN128-NEXT:    vsm.v v8, (a1)
+; CHECK-KNOWNVLEN128-NEXT:    ret
   %a = load <64 x i1>, ptr %x
   %c = call <2 x i1> @llvm.vector.extract.v2i1.v64i1(<64 x i1> %a, i64 2)
   store <2 x i1> %c, ptr %y
@@ -458,28 +523,50 @@ define void @extract_v2i1_v64i1_2(ptr %x, ptr %y) {
 }
 
 define void @extract_v2i1_v64i1_42(ptr %x, ptr %y) {
-; CHECK-LABEL: extract_v2i1_v64i1_42:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a2, 64
-; CHECK-NEXT:    vsetvli zero, a2, e8, m4, ta, ma
-; CHECK-NEXT:    vlm.v v0, (a0)
-; CHECK-NEXT:    vmv.v.i v8, 0
-; CHECK-NEXT:    vmerge.vim v8, v8, 1, v0
-; CHECK-NEXT:    li a0, 42
-; CHECK-NEXT:    vsetivli zero, 2, e8, m4, ta, ma
-; CHECK-NEXT:    vslidedown.vx v8, v8, a0
-; CHECK-NEXT:    vsetivli zero, 2, e8, mf8, ta, ma
-; CHECK-NEXT:    vmsne.vi v0, v8, 0
-; CHECK-NEXT:    vmv.v.i v8, 0
-; CHECK-NEXT:    vmerge.vim v8, v8, 1, v0
-; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
-; CHECK-NEXT:    vmv.v.i v9, 0
-; CHECK-NEXT:    vsetivli zero, 2, e8, mf2, tu, ma
-; CHECK-NEXT:    vmv.v.v v9, v8
-; CHECK-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
-; CHECK-NEXT:    vmsne.vi v8, v9, 0
-; CHECK-NEXT:    vsm.v v8, (a1)
-; CHECK-NEXT:    ret
+; CHECK-V-LABEL: extract_v2i1_v64i1_42:
+; CHECK-V:       # %bb.0:
+; CHECK-V-NEXT:    li a2, 64
+; CHECK-V-NEXT:    vsetvli zero, a2, e8, m4, ta, ma
+; CHECK-V-NEXT:    vlm.v v0, (a0)
+; CHECK-V-NEXT:    vmv.v.i v8, 0
+; CHECK-V-NEXT:    vmerge.vim v8, v8, 1, v0
+; CHECK-V-NEXT:    li a0, 42
+; CHECK-V-NEXT:    vsetivli zero, 2, e8, m4, ta, ma
+; CHECK-V-NEXT:    vslidedown.vx v8, v8, a0
+; CHECK-V-NEXT:    vsetivli zero, 2, e8, mf8, ta, ma
+; CHECK-V-NEXT:    vmsne.vi v0, v8, 0
+; CHECK-V-NEXT:    vmv.v.i v8, 0
+; CHECK-V-NEXT:    vmerge.vim v8, v8, 1, v0
+; CHECK-V-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-V-NEXT:    vmv.v.i v9, 0
+; CHECK-V-NEXT:    vsetivli zero, 2, e8, mf2, tu, ma
+; CHECK-V-NEXT:    vmv.v.v v9, v8
+; CHECK-V-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-V-NEXT:    vmsne.vi v8, v9, 0
+; CHECK-V-NEXT:    vsm.v v8, (a1)
+; CHECK-V-NEXT:    ret
+;
+; CHECK-KNOWNVLEN128-LABEL: extract_v2i1_v64i1_42:
+; CHECK-KNOWNVLEN128:       # %bb.0:
+; CHECK-KNOWNVLEN128-NEXT:    vsetvli a2, zero, e8, m4, ta, ma
+; CHECK-KNOWNVLEN128-NEXT:    vlm.v v0, (a0)
+; CHECK-KNOWNVLEN128-NEXT:    vmv.v.i v8, 0
+; CHECK-KNOWNVLEN128-NEXT:    vmerge.vim v8, v8, 1, v0
+; CHECK-KNOWNVLEN128-NEXT:    li a0, 42
+; CHECK-KNOWNVLEN128-NEXT:    vsetivli zero, 2, e8, m4, ta, ma
+; CHECK-KNOWNVLEN128-NEXT:    vslidedown.vx v8, v8, a0
+; CHECK-KNOWNVLEN128-NEXT:    vsetivli zero, 2, e8, mf8, ta, ma
+; CHECK-KNOWNVLEN128-NEXT:    vmsne.vi v0, v8, 0
+; CHECK-KNOWNVLEN128-NEXT:    vmv.v.i v8, 0
+; CHECK-KNOWNVLEN128-NEXT:    vmerge.vim v8, v8, 1, v0
+; CHECK-KNOWNVLEN128-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-KNOWNVLEN128-NEXT:    vmv.v.i v9, 0
+; CHECK-KNOWNVLEN128-NEXT:    vsetivli zero, 2, e8, mf2, tu, ma
+; CHECK-KNOWNVLEN128-NEXT:    vmv.v.v v9, v8
+; CHECK-KNOWNVLEN128-NEXT:    vsetivli zero, 8, e8, mf2, ta, ma
+; CHECK-KNOWNVLEN128-NEXT:    vmsne.vi v8, v9, 0
+; CHECK-KNOWNVLEN128-NEXT:    vsm.v v8, (a1)
+; CHECK-KNOWNVLEN128-NEXT:    ret
   %a = load <64 x i1>, ptr %x
   %c = call <2 x i1> @llvm.vector.extract.v2i1.v64i1(<64 x i1> %a, i64 42)
   store <2 x i1> %c, ptr %y
@@ -660,6 +747,3 @@ declare <2 x i8> @llvm.vector.extract.v2i8.nxv2i8(<vscale x 2 x i8> %vec, i64 %i
 
 declare <2 x i32> @llvm.vector.extract.v2i32.nxv16i32(<vscale x 16 x i32> %vec, i64 %idx)
 declare <8 x i32> @llvm.vector.extract.v8i32.nxv16i32(<vscale x 16 x i32> %vec, i64 %idx)
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; CHECK-KNOWNVLEN128: {{.*}}
-; CHECK-V: {{.*}}
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-insert.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-insert.ll
index a3f41fd842222cc8b884af49f2c57174a4331f61..6ebbc37f4afd7901966ad82bd92f246d7a08def3 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-insert.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-insert.ll
@@ -51,10 +51,8 @@ define <32 x i32> @insertelt_v32i32_0(<32 x i32> %a, i32 %y) {
 define <32 x i32> @insertelt_v32i32_4(<32 x i32> %a, i32 %y) {
 ; CHECK-LABEL: insertelt_v32i32_4:
 ; CHECK:       # %bb.0:
-; CHECK-NEXT:    li a1, 32
-; CHECK-NEXT:    vsetvli zero, a1, e32, m1, ta, ma
-; CHECK-NEXT:    vmv.s.x v16, a0
 ; CHECK-NEXT:    vsetivli zero, 5, e32, m2, tu, ma
+; CHECK-NEXT:    vmv.s.x v16, a0
 ; CHECK-NEXT:    vslideup.vi v8, v16, 4
 ; CHECK-NEXT:    ret
   %b = insertelement <32 x i32> %a, i32 %y, i32 4
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-load.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-load.ll
index caf0ae603fda9c0d78e10a993c42bed1dd3e8589..bcf2b0655b1763877fa1b2012442061afeb97a3e 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-load.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-load.ll
@@ -135,3 +135,54 @@ define <6 x i1> @load_v6i1(ptr %p) {
   %x = load <6 x i1>, ptr %p
   ret <6 x i1> %x
 }
+
+
+define <4 x i32> @exact_vlen_i32_m1(ptr %p) vscale_range(2,2) {
+; CHECK-LABEL: exact_vlen_i32_m1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vle32.v v8, (a0)
+; CHECK-NEXT:    ret
+  %v = load <4 x i32>, ptr %p
+  ret <4 x i32> %v
+}
+
+define <16 x i8> @exact_vlen_i8_m1(ptr %p) vscale_range(2,2) {
+; CHECK-LABEL: exact_vlen_i8_m1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-NEXT:    vle8.v v8, (a0)
+; CHECK-NEXT:    ret
+  %v = load <16 x i8>, ptr %p
+  ret <16 x i8> %v
+}
+
+define <32 x i8> @exact_vlen_i8_m2(ptr %p) vscale_range(2,2) {
+; CHECK-LABEL: exact_vlen_i8_m2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a1, zero, e8, m2, ta, ma
+; CHECK-NEXT:    vle8.v v8, (a0)
+; CHECK-NEXT:    ret
+  %v = load <32 x i8>, ptr %p
+  ret <32 x i8> %v
+}
+
+define <128 x i8> @exact_vlen_i8_m8(ptr %p) vscale_range(2,2) {
+; CHECK-LABEL: exact_vlen_i8_m8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a1, zero, e8, m8, ta, ma
+; CHECK-NEXT:    vle8.v v8, (a0)
+; CHECK-NEXT:    ret
+  %v = load <128 x i8>, ptr %p
+  ret <128 x i8> %v
+}
+
+define <16 x i64> @exact_vlen_i64_m8(ptr %p) vscale_range(2,2) {
+; CHECK-LABEL: exact_vlen_i64_m8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
+; CHECK-NEXT:    vle64.v v8, (a0)
+; CHECK-NEXT:    ret
+  %v = load <16 x i64>, ptr %p
+  ret <16 x i64> %v
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-gather.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-gather.ll
index ac3bf0d89b5ed9f7f0655172628c6cc1ce73c83e..d72ee25adb98d44e34035de878736cb0eec4c8e1 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-gather.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-masked-gather.ll
@@ -12432,7 +12432,6 @@ define <32 x i8> @mgather_baseidx_v32i8(ptr %base, <32 x i8> %idxs, <32 x i1> %m
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v12
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
 ; RV64ZVE32F-NEXT:    lbu a2, 0(a2)
-; RV64ZVE32F-NEXT:    li a3, 32
 ; RV64ZVE32F-NEXT:    vmv.s.x v12, a2
 ; RV64ZVE32F-NEXT:    vsetivli zero, 2, e8, m1, tu, ma
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 1
@@ -12450,14 +12449,11 @@ define <32 x i8> @mgather_baseidx_v32i8(ptr %base, <32 x i8> %idxs, <32 x i1> %m
 ; RV64ZVE32F-NEXT:    andi a2, a1, 16
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB98_8
 ; RV64ZVE32F-NEXT:  .LBB98_7: # %cond.load10
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
+; RV64ZVE32F-NEXT:    vsetivli zero, 5, e8, m1, tu, ma
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v13
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
 ; RV64ZVE32F-NEXT:    lbu a2, 0(a2)
-; RV64ZVE32F-NEXT:    li a3, 32
-; RV64ZVE32F-NEXT:    vsetvli zero, a3, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.s.x v12, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 5, e8, m1, tu, ma
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 4
 ; RV64ZVE32F-NEXT:  .LBB98_8: # %else11
 ; RV64ZVE32F-NEXT:    vsetivli zero, 8, e8, m1, ta, ma
@@ -12470,7 +12466,6 @@ define <32 x i8> @mgather_baseidx_v32i8(ptr %base, <32 x i8> %idxs, <32 x i1> %m
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v14
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
 ; RV64ZVE32F-NEXT:    lbu a2, 0(a2)
-; RV64ZVE32F-NEXT:    li a3, 32
 ; RV64ZVE32F-NEXT:    vmv.s.x v14, a2
 ; RV64ZVE32F-NEXT:    vsetivli zero, 6, e8, m1, tu, ma
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v14, 5
@@ -12494,7 +12489,6 @@ define <32 x i8> @mgather_baseidx_v32i8(ptr %base, <32 x i8> %idxs, <32 x i1> %m
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v13
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
 ; RV64ZVE32F-NEXT:    lbu a2, 0(a2)
-; RV64ZVE32F-NEXT:    li a3, 32
 ; RV64ZVE32F-NEXT:    vmv.s.x v13, a2
 ; RV64ZVE32F-NEXT:    vsetivli zero, 10, e8, m1, tu, ma
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v13, 9
@@ -12509,7 +12503,6 @@ define <32 x i8> @mgather_baseidx_v32i8(ptr %base, <32 x i8> %idxs, <32 x i1> %m
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v12
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
 ; RV64ZVE32F-NEXT:    lbu a2, 0(a2)
-; RV64ZVE32F-NEXT:    li a3, 32
 ; RV64ZVE32F-NEXT:    vmv.s.x v14, a2
 ; RV64ZVE32F-NEXT:    vsetivli zero, 11, e8, m1, tu, ma
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v14, 10
@@ -12522,7 +12515,6 @@ define <32 x i8> @mgather_baseidx_v32i8(ptr %base, <32 x i8> %idxs, <32 x i1> %m
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v12
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
 ; RV64ZVE32F-NEXT:    lbu a2, 0(a2)
-; RV64ZVE32F-NEXT:    li a3, 32
 ; RV64ZVE32F-NEXT:    vmv.s.x v12, a2
 ; RV64ZVE32F-NEXT:    vsetivli zero, 12, e8, m1, tu, ma
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 11
@@ -12535,7 +12527,6 @@ define <32 x i8> @mgather_baseidx_v32i8(ptr %base, <32 x i8> %idxs, <32 x i1> %m
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v13
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
 ; RV64ZVE32F-NEXT:    lbu a2, 0(a2)
-; RV64ZVE32F-NEXT:    li a3, 32
 ; RV64ZVE32F-NEXT:    vmv.s.x v9, a2
 ; RV64ZVE32F-NEXT:    vsetivli zero, 13, e8, m1, tu, ma
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v9, 12
@@ -12548,7 +12539,6 @@ define <32 x i8> @mgather_baseidx_v32i8(ptr %base, <32 x i8> %idxs, <32 x i1> %m
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v9
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
 ; RV64ZVE32F-NEXT:    lbu a2, 0(a2)
-; RV64ZVE32F-NEXT:    li a3, 32
 ; RV64ZVE32F-NEXT:    vmv.s.x v9, a2
 ; RV64ZVE32F-NEXT:    vsetivli zero, 14, e8, m1, tu, ma
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v9, 13
@@ -12572,7 +12562,6 @@ define <32 x i8> @mgather_baseidx_v32i8(ptr %base, <32 x i8> %idxs, <32 x i1> %m
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v9
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
 ; RV64ZVE32F-NEXT:    lbu a2, 0(a2)
-; RV64ZVE32F-NEXT:    li a3, 32
 ; RV64ZVE32F-NEXT:    vmv.s.x v12, a2
 ; RV64ZVE32F-NEXT:    vsetivli zero, 18, e8, m2, tu, ma
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 17
@@ -12590,14 +12579,11 @@ define <32 x i8> @mgather_baseidx_v32i8(ptr %base, <32 x i8> %idxs, <32 x i1> %m
 ; RV64ZVE32F-NEXT:    slli a2, a1, 43
 ; RV64ZVE32F-NEXT:    bgez a2, .LBB98_32
 ; RV64ZVE32F-NEXT:  .LBB98_31: # %cond.load58
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
+; RV64ZVE32F-NEXT:    vsetivli zero, 21, e8, m2, tu, ma
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v9
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
 ; RV64ZVE32F-NEXT:    lbu a2, 0(a2)
-; RV64ZVE32F-NEXT:    li a3, 32
-; RV64ZVE32F-NEXT:    vsetvli zero, a3, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.s.x v12, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 21, e8, m2, tu, ma
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 20
 ; RV64ZVE32F-NEXT:  .LBB98_32: # %else59
 ; RV64ZVE32F-NEXT:    vsetivli zero, 8, e8, m1, ta, ma
@@ -12610,7 +12596,6 @@ define <32 x i8> @mgather_baseidx_v32i8(ptr %base, <32 x i8> %idxs, <32 x i1> %m
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v12
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
 ; RV64ZVE32F-NEXT:    lbu a2, 0(a2)
-; RV64ZVE32F-NEXT:    li a3, 32
 ; RV64ZVE32F-NEXT:    vmv.s.x v12, a2
 ; RV64ZVE32F-NEXT:    vsetivli zero, 22, e8, m2, tu, ma
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 21
@@ -12634,7 +12619,6 @@ define <32 x i8> @mgather_baseidx_v32i8(ptr %base, <32 x i8> %idxs, <32 x i1> %m
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v9
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
 ; RV64ZVE32F-NEXT:    lbu a2, 0(a2)
-; RV64ZVE32F-NEXT:    li a3, 32
 ; RV64ZVE32F-NEXT:    vmv.s.x v12, a2
 ; RV64ZVE32F-NEXT:    vsetivli zero, 26, e8, m2, tu, ma
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 25
@@ -12660,7 +12644,6 @@ define <32 x i8> @mgather_baseidx_v32i8(ptr %base, <32 x i8> %idxs, <32 x i1> %m
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
 ; RV64ZVE32F-NEXT:    lbu a2, 0(a2)
-; RV64ZVE32F-NEXT:    li a3, 32
 ; RV64ZVE32F-NEXT:    vmv.s.x v12, a2
 ; RV64ZVE32F-NEXT:    vsetivli zero, 30, e8, m2, tu, ma
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 29
@@ -12673,7 +12656,6 @@ define <32 x i8> @mgather_baseidx_v32i8(ptr %base, <32 x i8> %idxs, <32 x i1> %m
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
 ; RV64ZVE32F-NEXT:    lbu a2, 0(a2)
-; RV64ZVE32F-NEXT:    li a3, 32
 ; RV64ZVE32F-NEXT:    vmv.s.x v12, a2
 ; RV64ZVE32F-NEXT:    vsetivli zero, 31, e8, m2, tu, ma
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 30
@@ -12698,7 +12680,6 @@ define <32 x i8> @mgather_baseidx_v32i8(ptr %base, <32 x i8> %idxs, <32 x i1> %m
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v12
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
 ; RV64ZVE32F-NEXT:    lbu a2, 0(a2)
-; RV64ZVE32F-NEXT:    li a3, 32
 ; RV64ZVE32F-NEXT:    vmv.s.x v14, a2
 ; RV64ZVE32F-NEXT:    vsetivli zero, 3, e8, m1, tu, ma
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v14, 2
@@ -12710,7 +12691,6 @@ define <32 x i8> @mgather_baseidx_v32i8(ptr %base, <32 x i8> %idxs, <32 x i1> %m
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v12
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
 ; RV64ZVE32F-NEXT:    lbu a2, 0(a2)
-; RV64ZVE32F-NEXT:    li a3, 32
 ; RV64ZVE32F-NEXT:    vmv.s.x v12, a2
 ; RV64ZVE32F-NEXT:    vsetivli zero, 4, e8, m1, tu, ma
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 3
@@ -12721,7 +12701,6 @@ define <32 x i8> @mgather_baseidx_v32i8(ptr %base, <32 x i8> %idxs, <32 x i1> %m
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v13
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
 ; RV64ZVE32F-NEXT:    lbu a2, 0(a2)
-; RV64ZVE32F-NEXT:    li a3, 32
 ; RV64ZVE32F-NEXT:    vmv.s.x v14, a2
 ; RV64ZVE32F-NEXT:    vsetivli zero, 7, e8, m1, tu, ma
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v14, 6
@@ -12733,21 +12712,17 @@ define <32 x i8> @mgather_baseidx_v32i8(ptr %base, <32 x i8> %idxs, <32 x i1> %m
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v13
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
 ; RV64ZVE32F-NEXT:    lbu a2, 0(a2)
-; RV64ZVE32F-NEXT:    li a3, 32
 ; RV64ZVE32F-NEXT:    vmv.s.x v13, a2
 ; RV64ZVE32F-NEXT:    vsetivli zero, 8, e8, m1, tu, ma
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v13, 7
 ; RV64ZVE32F-NEXT:    andi a2, a1, 256
 ; RV64ZVE32F-NEXT:    beqz a2, .LBB98_13
 ; RV64ZVE32F-NEXT:  .LBB98_53: # %cond.load22
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
+; RV64ZVE32F-NEXT:    vsetivli zero, 9, e8, m1, tu, ma
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v12
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
 ; RV64ZVE32F-NEXT:    lbu a2, 0(a2)
-; RV64ZVE32F-NEXT:    li a3, 32
-; RV64ZVE32F-NEXT:    vsetvli zero, a3, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.s.x v13, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 9, e8, m1, tu, ma
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v13, 8
 ; RV64ZVE32F-NEXT:    andi a2, a1, 512
 ; RV64ZVE32F-NEXT:    bnez a2, .LBB98_14
@@ -12756,7 +12731,6 @@ define <32 x i8> @mgather_baseidx_v32i8(ptr %base, <32 x i8> %idxs, <32 x i1> %m
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v9
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
 ; RV64ZVE32F-NEXT:    lbu a2, 0(a2)
-; RV64ZVE32F-NEXT:    li a3, 32
 ; RV64ZVE32F-NEXT:    vmv.s.x v12, a2
 ; RV64ZVE32F-NEXT:    vsetivli zero, 15, e8, m1, tu, ma
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 14
@@ -12768,21 +12742,17 @@ define <32 x i8> @mgather_baseidx_v32i8(ptr %base, <32 x i8> %idxs, <32 x i1> %m
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v9
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
 ; RV64ZVE32F-NEXT:    lbu a2, 0(a2)
-; RV64ZVE32F-NEXT:    li a3, 32
 ; RV64ZVE32F-NEXT:    vmv.s.x v9, a2
 ; RV64ZVE32F-NEXT:    vsetivli zero, 16, e8, m1, tu, ma
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v9, 15
 ; RV64ZVE32F-NEXT:    slli a2, a1, 47
 ; RV64ZVE32F-NEXT:    bgez a2, .LBB98_26
 ; RV64ZVE32F-NEXT:  .LBB98_56: # %cond.load46
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
+; RV64ZVE32F-NEXT:    vsetivli zero, 17, e8, m2, tu, ma
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
 ; RV64ZVE32F-NEXT:    lbu a2, 0(a2)
-; RV64ZVE32F-NEXT:    li a3, 32
-; RV64ZVE32F-NEXT:    vsetvli zero, a3, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.s.x v12, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 17, e8, m2, tu, ma
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 16
 ; RV64ZVE32F-NEXT:    slli a2, a1, 46
 ; RV64ZVE32F-NEXT:    bltz a2, .LBB98_27
@@ -12791,7 +12761,6 @@ define <32 x i8> @mgather_baseidx_v32i8(ptr %base, <32 x i8> %idxs, <32 x i1> %m
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v12
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
 ; RV64ZVE32F-NEXT:    lbu a2, 0(a2)
-; RV64ZVE32F-NEXT:    li a3, 32
 ; RV64ZVE32F-NEXT:    vmv.s.x v14, a2
 ; RV64ZVE32F-NEXT:    vsetivli zero, 19, e8, m2, tu, ma
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v14, 18
@@ -12803,7 +12772,6 @@ define <32 x i8> @mgather_baseidx_v32i8(ptr %base, <32 x i8> %idxs, <32 x i1> %m
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v12
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
 ; RV64ZVE32F-NEXT:    lbu a2, 0(a2)
-; RV64ZVE32F-NEXT:    li a3, 32
 ; RV64ZVE32F-NEXT:    vmv.s.x v12, a2
 ; RV64ZVE32F-NEXT:    vsetivli zero, 20, e8, m2, tu, ma
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 19
@@ -12814,7 +12782,6 @@ define <32 x i8> @mgather_baseidx_v32i8(ptr %base, <32 x i8> %idxs, <32 x i1> %m
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v9
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
 ; RV64ZVE32F-NEXT:    lbu a2, 0(a2)
-; RV64ZVE32F-NEXT:    li a3, 32
 ; RV64ZVE32F-NEXT:    vmv.s.x v12, a2
 ; RV64ZVE32F-NEXT:    vsetivli zero, 23, e8, m2, tu, ma
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 22
@@ -12826,21 +12793,17 @@ define <32 x i8> @mgather_baseidx_v32i8(ptr %base, <32 x i8> %idxs, <32 x i1> %m
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v9
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
 ; RV64ZVE32F-NEXT:    lbu a2, 0(a2)
-; RV64ZVE32F-NEXT:    li a3, 32
 ; RV64ZVE32F-NEXT:    vmv.s.x v12, a2
 ; RV64ZVE32F-NEXT:    vsetivli zero, 24, e8, m2, tu, ma
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 23
 ; RV64ZVE32F-NEXT:    slli a2, a1, 39
 ; RV64ZVE32F-NEXT:    bgez a2, .LBB98_37
 ; RV64ZVE32F-NEXT:  .LBB98_61: # %cond.load70
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
+; RV64ZVE32F-NEXT:    vsetivli zero, 25, e8, m2, tu, ma
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
 ; RV64ZVE32F-NEXT:    lbu a2, 0(a2)
-; RV64ZVE32F-NEXT:    li a3, 32
-; RV64ZVE32F-NEXT:    vsetvli zero, a3, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.s.x v12, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 25, e8, m2, tu, ma
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 24
 ; RV64ZVE32F-NEXT:    slli a2, a1, 38
 ; RV64ZVE32F-NEXT:    bltz a2, .LBB98_38
@@ -12849,7 +12812,6 @@ define <32 x i8> @mgather_baseidx_v32i8(ptr %base, <32 x i8> %idxs, <32 x i1> %m
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
 ; RV64ZVE32F-NEXT:    lbu a2, 0(a2)
-; RV64ZVE32F-NEXT:    li a3, 32
 ; RV64ZVE32F-NEXT:    vmv.s.x v12, a2
 ; RV64ZVE32F-NEXT:    vsetivli zero, 27, e8, m2, tu, ma
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 26
@@ -12861,21 +12823,17 @@ define <32 x i8> @mgather_baseidx_v32i8(ptr %base, <32 x i8> %idxs, <32 x i1> %m
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v8
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
 ; RV64ZVE32F-NEXT:    lbu a2, 0(a2)
-; RV64ZVE32F-NEXT:    li a3, 32
 ; RV64ZVE32F-NEXT:    vmv.s.x v12, a2
 ; RV64ZVE32F-NEXT:    vsetivli zero, 28, e8, m2, tu, ma
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 27
 ; RV64ZVE32F-NEXT:    slli a2, a1, 35
 ; RV64ZVE32F-NEXT:    bgez a2, .LBB98_42
 ; RV64ZVE32F-NEXT:  .LBB98_64: # %cond.load82
-; RV64ZVE32F-NEXT:    vsetivli zero, 1, e8, mf4, ta, ma
+; RV64ZVE32F-NEXT:    vsetivli zero, 29, e8, m2, tu, ma
 ; RV64ZVE32F-NEXT:    vmv.x.s a2, v9
 ; RV64ZVE32F-NEXT:    add a2, a0, a2
 ; RV64ZVE32F-NEXT:    lbu a2, 0(a2)
-; RV64ZVE32F-NEXT:    li a3, 32
-; RV64ZVE32F-NEXT:    vsetvli zero, a3, e8, mf4, ta, ma
 ; RV64ZVE32F-NEXT:    vmv.s.x v12, a2
-; RV64ZVE32F-NEXT:    vsetivli zero, 29, e8, m2, tu, ma
 ; RV64ZVE32F-NEXT:    vslideup.vi v10, v12, 28
 ; RV64ZVE32F-NEXT:    slli a2, a1, 34
 ; RV64ZVE32F-NEXT:    bltz a2, .LBB98_43
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-store.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-store.ll
index 32d26827f989e04a5e4258f6b61b9667b9fc08b2..8e3ef615428f35977c707aa914d4ccaa6e0124e8 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-store.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-store.ll
@@ -237,6 +237,63 @@ define void @store_constant_v2i8_volatile(ptr %p) {
   store volatile <2 x i8> <i8 1, i8 1>, ptr %p
   ret void
 }
+
+
+define void @exact_vlen_i32_m1(ptr %p) vscale_range(2,2) {
+; CHECK-LABEL: exact_vlen_i32_m1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 4, e32, m1, ta, ma
+; CHECK-NEXT:    vmv.v.i v8, 0
+; CHECK-NEXT:    vse32.v v8, (a0)
+; CHECK-NEXT:    ret
+  store <4 x i32> zeroinitializer, ptr %p
+  ret void
+}
+
+define void @exact_vlen_i8_m1(ptr %p) vscale_range(2,2) {
+; CHECK-LABEL: exact_vlen_i8_m1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 16, e8, m1, ta, ma
+; CHECK-NEXT:    vmv.v.i v8, 0
+; CHECK-NEXT:    vse8.v v8, (a0)
+; CHECK-NEXT:    ret
+  store <16 x i8> zeroinitializer, ptr %p
+  ret void
+}
+
+define void @exact_vlen_i8_m2(ptr %p) vscale_range(2,2) {
+; CHECK-LABEL: exact_vlen_i8_m2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a1, zero, e8, m2, ta, ma
+; CHECK-NEXT:    vmv.v.i v8, 0
+; CHECK-NEXT:    vse8.v v8, (a0)
+; CHECK-NEXT:    ret
+  store <32 x i8> zeroinitializer, ptr %p
+  ret void
+}
+
+define void @exact_vlen_i8_m8(ptr %p) vscale_range(2,2) {
+; CHECK-LABEL: exact_vlen_i8_m8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a1, zero, e8, m8, ta, ma
+; CHECK-NEXT:    vmv.v.i v8, 0
+; CHECK-NEXT:    vse8.v v8, (a0)
+; CHECK-NEXT:    ret
+  store <128 x i8> zeroinitializer, ptr %p
+  ret void
+}
+
+define void @exact_vlen_i64_m8(ptr %p) vscale_range(2,2) {
+; CHECK-LABEL: exact_vlen_i64_m8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
+; CHECK-NEXT:    vmv.v.i v8, 0
+; CHECK-NEXT:    vse64.v v8, (a0)
+; CHECK-NEXT:    ret
+  store <16 x i64> zeroinitializer, ptr %p
+  ret void
+}
+
 ;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
 ; RV32: {{.*}}
 ; RV64: {{.*}}
diff --git a/llvm/test/CodeGen/RISCV/rvv/load-add-store-16.ll b/llvm/test/CodeGen/RISCV/rvv/load-add-store-16.ll
deleted file mode 100644
index 4bff79ba935daede248c70728a0b5291548c91d2..0000000000000000000000000000000000000000
--- a/llvm/test/CodeGen/RISCV/rvv/load-add-store-16.ll
+++ /dev/null
@@ -1,101 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple riscv32 -mattr=+v %s -o - \
-; RUN:     -verify-machineinstrs | FileCheck %s
-; RUN: llc -mtriple riscv64 -mattr=+v %s -o - \
-; RUN:     -verify-machineinstrs | FileCheck %s
-
-define void @vadd_vint16m1(<vscale x 4 x i16> *%pc, <vscale x 4 x i16> *%pa, <vscale x 4 x i16> *%pb) nounwind {
-; CHECK-LABEL: vadd_vint16m1:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vl1re16.v v8, (a1)
-; CHECK-NEXT:    vl1re16.v v9, (a2)
-; CHECK-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
-; CHECK-NEXT:    vadd.vv v8, v8, v9
-; CHECK-NEXT:    vs1r.v v8, (a0)
-; CHECK-NEXT:    ret
-  %va = load <vscale x 4 x i16>, <vscale x 4 x i16>* %pa
-  %vb = load <vscale x 4 x i16>, <vscale x 4 x i16>* %pb
-  %vc = add <vscale x 4 x i16> %va, %vb
-  store <vscale x 4 x i16> %vc, <vscale x 4 x i16> *%pc
-  ret void
-}
-
-define void @vadd_vint16m2(<vscale x 8 x i16> *%pc, <vscale x 8 x i16> *%pa, <vscale x 8 x i16> *%pb) nounwind {
-; CHECK-LABEL: vadd_vint16m2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vl2re16.v v8, (a1)
-; CHECK-NEXT:    vl2re16.v v10, (a2)
-; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
-; CHECK-NEXT:    vadd.vv v8, v8, v10
-; CHECK-NEXT:    vs2r.v v8, (a0)
-; CHECK-NEXT:    ret
-  %va = load <vscale x 8 x i16>, <vscale x 8 x i16>* %pa
-  %vb = load <vscale x 8 x i16>, <vscale x 8 x i16>* %pb
-  %vc = add <vscale x 8 x i16> %va, %vb
-  store <vscale x 8 x i16> %vc, <vscale x 8 x i16> *%pc
-  ret void
-}
-
-define void @vadd_vint16m4(<vscale x 16 x i16> *%pc, <vscale x 16 x i16> *%pa, <vscale x 16 x i16> *%pb) nounwind {
-; CHECK-LABEL: vadd_vint16m4:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vl4re16.v v8, (a1)
-; CHECK-NEXT:    vl4re16.v v12, (a2)
-; CHECK-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
-; CHECK-NEXT:    vadd.vv v8, v8, v12
-; CHECK-NEXT:    vs4r.v v8, (a0)
-; CHECK-NEXT:    ret
-  %va = load <vscale x 16 x i16>, <vscale x 16 x i16>* %pa
-  %vb = load <vscale x 16 x i16>, <vscale x 16 x i16>* %pb
-  %vc = add <vscale x 16 x i16> %va, %vb
-  store <vscale x 16 x i16> %vc, <vscale x 16 x i16> *%pc
-  ret void
-}
-
-define void @vadd_vint16m8(<vscale x 32 x i16> *%pc, <vscale x 32 x i16> *%pa, <vscale x 32 x i16> *%pb) nounwind {
-; CHECK-LABEL: vadd_vint16m8:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vl8re16.v v8, (a1)
-; CHECK-NEXT:    vl8re16.v v16, (a2)
-; CHECK-NEXT:    vsetvli a1, zero, e16, m8, ta, ma
-; CHECK-NEXT:    vadd.vv v8, v8, v16
-; CHECK-NEXT:    vs8r.v v8, (a0)
-; CHECK-NEXT:    ret
-  %va = load <vscale x 32 x i16>, <vscale x 32 x i16>* %pa
-  %vb = load <vscale x 32 x i16>, <vscale x 32 x i16>* %pb
-  %vc = add <vscale x 32 x i16> %va, %vb
-  store <vscale x 32 x i16> %vc, <vscale x 32 x i16> *%pc
-  ret void
-}
-
-define void @vadd_vint16mf2(<vscale x 2 x i16> *%pc, <vscale x 2 x i16> *%pa, <vscale x 2 x i16> *%pb) nounwind {
-; CHECK-LABEL: vadd_vint16mf2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a3, zero, e16, mf2, ta, ma
-; CHECK-NEXT:    vle16.v v8, (a1)
-; CHECK-NEXT:    vle16.v v9, (a2)
-; CHECK-NEXT:    vadd.vv v8, v8, v9
-; CHECK-NEXT:    vse16.v v8, (a0)
-; CHECK-NEXT:    ret
-  %va = load <vscale x 2 x i16>, <vscale x 2 x i16>* %pa
-  %vb = load <vscale x 2 x i16>, <vscale x 2 x i16>* %pb
-  %vc = add <vscale x 2 x i16> %va, %vb
-  store <vscale x 2 x i16> %vc, <vscale x 2 x i16> *%pc
-  ret void
-}
-
-define void @vadd_vint16mf4(<vscale x 1 x i16> *%pc, <vscale x 1 x i16> *%pa, <vscale x 1 x i16> *%pb) nounwind {
-; CHECK-LABEL: vadd_vint16mf4:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a3, zero, e16, mf4, ta, ma
-; CHECK-NEXT:    vle16.v v8, (a1)
-; CHECK-NEXT:    vle16.v v9, (a2)
-; CHECK-NEXT:    vadd.vv v8, v8, v9
-; CHECK-NEXT:    vse16.v v8, (a0)
-; CHECK-NEXT:    ret
-  %va = load <vscale x 1 x i16>, <vscale x 1 x i16>* %pa
-  %vb = load <vscale x 1 x i16>, <vscale x 1 x i16>* %pb
-  %vc = add <vscale x 1 x i16> %va, %vb
-  store <vscale x 1 x i16> %vc, <vscale x 1 x i16> *%pc
-  ret void
-}
diff --git a/llvm/test/CodeGen/RISCV/rvv/load-add-store-32.ll b/llvm/test/CodeGen/RISCV/rvv/load-add-store-32.ll
deleted file mode 100644
index a312ce74c2ee3e8068b54a63b88040f291a89950..0000000000000000000000000000000000000000
--- a/llvm/test/CodeGen/RISCV/rvv/load-add-store-32.ll
+++ /dev/null
@@ -1,85 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple riscv32 -mattr=+v %s -o - \
-; RUN:     -verify-machineinstrs | FileCheck %s
-; RUN: llc -mtriple riscv64 -mattr=+v %s -o - \
-; RUN:     -verify-machineinstrs | FileCheck %s
-
-define void @vadd_vint32m1(<vscale x 2 x i32> *%pc, <vscale x 2 x i32> *%pa, <vscale x 2 x i32> *%pb) nounwind {
-; CHECK-LABEL: vadd_vint32m1:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vl1re32.v v8, (a1)
-; CHECK-NEXT:    vl1re32.v v9, (a2)
-; CHECK-NEXT:    vsetvli a1, zero, e32, m1, ta, ma
-; CHECK-NEXT:    vadd.vv v8, v8, v9
-; CHECK-NEXT:    vs1r.v v8, (a0)
-; CHECK-NEXT:    ret
-  %va = load <vscale x 2 x i32>, <vscale x 2 x i32>* %pa
-  %vb = load <vscale x 2 x i32>, <vscale x 2 x i32>* %pb
-  %vc = add <vscale x 2 x i32> %va, %vb
-  store <vscale x 2 x i32> %vc, <vscale x 2 x i32> *%pc
-  ret void
-}
-
-define void @vadd_vint32m2(<vscale x 4 x i32> *%pc, <vscale x 4 x i32> *%pa, <vscale x 4 x i32> *%pb) nounwind {
-; CHECK-LABEL: vadd_vint32m2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vl2re32.v v8, (a1)
-; CHECK-NEXT:    vl2re32.v v10, (a2)
-; CHECK-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
-; CHECK-NEXT:    vadd.vv v8, v8, v10
-; CHECK-NEXT:    vs2r.v v8, (a0)
-; CHECK-NEXT:    ret
-  %va = load <vscale x 4 x i32>, <vscale x 4 x i32>* %pa
-  %vb = load <vscale x 4 x i32>, <vscale x 4 x i32>* %pb
-  %vc = add <vscale x 4 x i32> %va, %vb
-  store <vscale x 4 x i32> %vc, <vscale x 4 x i32> *%pc
-  ret void
-}
-
-define void @vadd_vint32m4(<vscale x 8 x i32> *%pc, <vscale x 8 x i32> *%pa, <vscale x 8 x i32> *%pb) nounwind {
-; CHECK-LABEL: vadd_vint32m4:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vl4re32.v v8, (a1)
-; CHECK-NEXT:    vl4re32.v v12, (a2)
-; CHECK-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
-; CHECK-NEXT:    vadd.vv v8, v8, v12
-; CHECK-NEXT:    vs4r.v v8, (a0)
-; CHECK-NEXT:    ret
-  %va = load <vscale x 8 x i32>, <vscale x 8 x i32>* %pa
-  %vb = load <vscale x 8 x i32>, <vscale x 8 x i32>* %pb
-  %vc = add <vscale x 8 x i32> %va, %vb
-  store <vscale x 8 x i32> %vc, <vscale x 8 x i32> *%pc
-  ret void
-}
-
-define void @vadd_vint32m8(<vscale x 16 x i32> *%pc, <vscale x 16 x i32> *%pa, <vscale x 16 x i32> *%pb) nounwind {
-; CHECK-LABEL: vadd_vint32m8:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vl8re32.v v8, (a1)
-; CHECK-NEXT:    vl8re32.v v16, (a2)
-; CHECK-NEXT:    vsetvli a1, zero, e32, m8, ta, ma
-; CHECK-NEXT:    vadd.vv v8, v8, v16
-; CHECK-NEXT:    vs8r.v v8, (a0)
-; CHECK-NEXT:    ret
-  %va = load <vscale x 16 x i32>, <vscale x 16 x i32>* %pa
-  %vb = load <vscale x 16 x i32>, <vscale x 16 x i32>* %pb
-  %vc = add <vscale x 16 x i32> %va, %vb
-  store <vscale x 16 x i32> %vc, <vscale x 16 x i32> *%pc
-  ret void
-}
-
-define void @vadd_vint32mf2(<vscale x 1 x i32> *%pc, <vscale x 1 x i32> *%pa, <vscale x 1 x i32> *%pb) nounwind {
-; CHECK-LABEL: vadd_vint32mf2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a3, zero, e32, mf2, ta, ma
-; CHECK-NEXT:    vle32.v v8, (a1)
-; CHECK-NEXT:    vle32.v v9, (a2)
-; CHECK-NEXT:    vadd.vv v8, v8, v9
-; CHECK-NEXT:    vse32.v v8, (a0)
-; CHECK-NEXT:    ret
-  %va = load <vscale x 1 x i32>, <vscale x 1 x i32>* %pa
-  %vb = load <vscale x 1 x i32>, <vscale x 1 x i32>* %pb
-  %vc = add <vscale x 1 x i32> %va, %vb
-  store <vscale x 1 x i32> %vc, <vscale x 1 x i32> *%pc
-  ret void
-}
diff --git a/llvm/test/CodeGen/RISCV/rvv/load-add-store-64.ll b/llvm/test/CodeGen/RISCV/rvv/load-add-store-64.ll
deleted file mode 100644
index 2105197eded1584b1b534700a155a977de7a5084..0000000000000000000000000000000000000000
--- a/llvm/test/CodeGen/RISCV/rvv/load-add-store-64.ll
+++ /dev/null
@@ -1,69 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple riscv32 -mattr=+v %s -o - \
-; RUN:     -verify-machineinstrs | FileCheck %s
-; RUN: llc -mtriple riscv64 -mattr=+v %s -o - \
-; RUN:     -verify-machineinstrs | FileCheck %s
-
-define void @vadd_vint64m1(<vscale x 1 x i64> *%pc, <vscale x 1 x i64> *%pa, <vscale x 1 x i64> *%pb) nounwind {
-; CHECK-LABEL: vadd_vint64m1:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vl1re64.v v8, (a1)
-; CHECK-NEXT:    vl1re64.v v9, (a2)
-; CHECK-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
-; CHECK-NEXT:    vadd.vv v8, v8, v9
-; CHECK-NEXT:    vs1r.v v8, (a0)
-; CHECK-NEXT:    ret
-  %va = load <vscale x 1 x i64>, <vscale x 1 x i64>* %pa
-  %vb = load <vscale x 1 x i64>, <vscale x 1 x i64>* %pb
-  %vc = add <vscale x 1 x i64> %va, %vb
-  store <vscale x 1 x i64> %vc, <vscale x 1 x i64> *%pc
-  ret void
-}
-
-define void @vadd_vint64m2(<vscale x 2 x i64> *%pc, <vscale x 2 x i64> *%pa, <vscale x 2 x i64> *%pb) nounwind {
-; CHECK-LABEL: vadd_vint64m2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vl2re64.v v8, (a1)
-; CHECK-NEXT:    vl2re64.v v10, (a2)
-; CHECK-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
-; CHECK-NEXT:    vadd.vv v8, v8, v10
-; CHECK-NEXT:    vs2r.v v8, (a0)
-; CHECK-NEXT:    ret
-  %va = load <vscale x 2 x i64>, <vscale x 2 x i64>* %pa
-  %vb = load <vscale x 2 x i64>, <vscale x 2 x i64>* %pb
-  %vc = add <vscale x 2 x i64> %va, %vb
-  store <vscale x 2 x i64> %vc, <vscale x 2 x i64> *%pc
-  ret void
-}
-
-define void @vadd_vint64m4(<vscale x 4 x i64> *%pc, <vscale x 4 x i64> *%pa, <vscale x 4 x i64> *%pb) nounwind {
-; CHECK-LABEL: vadd_vint64m4:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vl4re64.v v8, (a1)
-; CHECK-NEXT:    vl4re64.v v12, (a2)
-; CHECK-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
-; CHECK-NEXT:    vadd.vv v8, v8, v12
-; CHECK-NEXT:    vs4r.v v8, (a0)
-; CHECK-NEXT:    ret
-  %va = load <vscale x 4 x i64>, <vscale x 4 x i64>* %pa
-  %vb = load <vscale x 4 x i64>, <vscale x 4 x i64>* %pb
-  %vc = add <vscale x 4 x i64> %va, %vb
-  store <vscale x 4 x i64> %vc, <vscale x 4 x i64> *%pc
-  ret void
-}
-
-define void @vadd_vint64m8(<vscale x 8 x i64> *%pc, <vscale x 8 x i64> *%pa, <vscale x 8 x i64> *%pb) nounwind {
-; CHECK-LABEL: vadd_vint64m8:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vl8re64.v v8, (a1)
-; CHECK-NEXT:    vl8re64.v v16, (a2)
-; CHECK-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
-; CHECK-NEXT:    vadd.vv v8, v8, v16
-; CHECK-NEXT:    vs8r.v v8, (a0)
-; CHECK-NEXT:    ret
-  %va = load <vscale x 8 x i64>, <vscale x 8 x i64>* %pa
-  %vb = load <vscale x 8 x i64>, <vscale x 8 x i64>* %pb
-  %vc = add <vscale x 8 x i64> %va, %vb
-  store <vscale x 8 x i64> %vc, <vscale x 8 x i64> *%pc
-  ret void
-}
diff --git a/llvm/test/CodeGen/RISCV/rvv/load-add-store-8.ll b/llvm/test/CodeGen/RISCV/rvv/load-add-store-8.ll
deleted file mode 100644
index 2a1231249803a290c887512ffea48de41be4fa23..0000000000000000000000000000000000000000
--- a/llvm/test/CodeGen/RISCV/rvv/load-add-store-8.ll
+++ /dev/null
@@ -1,117 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple riscv32 -mattr=+v %s -o - \
-; RUN:     -verify-machineinstrs | FileCheck %s
-; RUN: llc -mtriple riscv64 -mattr=+v %s -o - \
-; RUN:     -verify-machineinstrs | FileCheck %s
-
-define void @vadd_vint8m1(<vscale x 8 x i8> *%pc, <vscale x 8 x i8> *%pa, <vscale x 8 x i8> *%pb) nounwind {
-; CHECK-LABEL: vadd_vint8m1:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vl1r.v v8, (a1)
-; CHECK-NEXT:    vl1r.v v9, (a2)
-; CHECK-NEXT:    vsetvli a1, zero, e8, m1, ta, ma
-; CHECK-NEXT:    vadd.vv v8, v8, v9
-; CHECK-NEXT:    vs1r.v v8, (a0)
-; CHECK-NEXT:    ret
-  %va = load <vscale x 8 x i8>, <vscale x 8 x i8>* %pa
-  %vb = load <vscale x 8 x i8>, <vscale x 8 x i8>* %pb
-  %vc = add <vscale x 8 x i8> %va, %vb
-  store <vscale x 8 x i8> %vc, <vscale x 8 x i8> *%pc
-  ret void
-}
-
-define void @vadd_vint8m2(<vscale x 16 x i8> *%pc, <vscale x 16 x i8> *%pa, <vscale x 16 x i8> *%pb) nounwind {
-; CHECK-LABEL: vadd_vint8m2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vl2r.v v8, (a1)
-; CHECK-NEXT:    vl2r.v v10, (a2)
-; CHECK-NEXT:    vsetvli a1, zero, e8, m2, ta, ma
-; CHECK-NEXT:    vadd.vv v8, v8, v10
-; CHECK-NEXT:    vs2r.v v8, (a0)
-; CHECK-NEXT:    ret
-  %va = load <vscale x 16 x i8>, <vscale x 16 x i8>* %pa
-  %vb = load <vscale x 16 x i8>, <vscale x 16 x i8>* %pb
-  %vc = add <vscale x 16 x i8> %va, %vb
-  store <vscale x 16 x i8> %vc, <vscale x 16 x i8> *%pc
-  ret void
-}
-
-define void @vadd_vint8m4(<vscale x 32 x i8> *%pc, <vscale x 32 x i8> *%pa, <vscale x 32 x i8> *%pb) nounwind {
-; CHECK-LABEL: vadd_vint8m4:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vl4r.v v8, (a1)
-; CHECK-NEXT:    vl4r.v v12, (a2)
-; CHECK-NEXT:    vsetvli a1, zero, e8, m4, ta, ma
-; CHECK-NEXT:    vadd.vv v8, v8, v12
-; CHECK-NEXT:    vs4r.v v8, (a0)
-; CHECK-NEXT:    ret
-  %va = load <vscale x 32 x i8>, <vscale x 32 x i8>* %pa
-  %vb = load <vscale x 32 x i8>, <vscale x 32 x i8>* %pb
-  %vc = add <vscale x 32 x i8> %va, %vb
-  store <vscale x 32 x i8> %vc, <vscale x 32 x i8> *%pc
-  ret void
-}
-
-define void @vadd_vint8m8(<vscale x 64 x i8> *%pc, <vscale x 64 x i8> *%pa, <vscale x 64 x i8> *%pb) nounwind {
-; CHECK-LABEL: vadd_vint8m8:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vl8r.v v8, (a1)
-; CHECK-NEXT:    vl8r.v v16, (a2)
-; CHECK-NEXT:    vsetvli a1, zero, e8, m8, ta, ma
-; CHECK-NEXT:    vadd.vv v8, v8, v16
-; CHECK-NEXT:    vs8r.v v8, (a0)
-; CHECK-NEXT:    ret
-  %va = load <vscale x 64 x i8>, <vscale x 64 x i8>* %pa
-  %vb = load <vscale x 64 x i8>, <vscale x 64 x i8>* %pb
-  %vc = add <vscale x 64 x i8> %va, %vb
-  store <vscale x 64 x i8> %vc, <vscale x 64 x i8> *%pc
-  ret void
-}
-
-define void @vadd_vint8mf2(<vscale x 4 x i8> *%pc, <vscale x 4 x i8> *%pa, <vscale x 4 x i8> *%pb) nounwind {
-; CHECK-LABEL: vadd_vint8mf2:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a3, zero, e8, mf2, ta, ma
-; CHECK-NEXT:    vle8.v v8, (a1)
-; CHECK-NEXT:    vle8.v v9, (a2)
-; CHECK-NEXT:    vadd.vv v8, v8, v9
-; CHECK-NEXT:    vse8.v v8, (a0)
-; CHECK-NEXT:    ret
-  %va = load <vscale x 4 x i8>, <vscale x 4 x i8>* %pa
-  %vb = load <vscale x 4 x i8>, <vscale x 4 x i8>* %pb
-  %vc = add <vscale x 4 x i8> %va, %vb
-  store <vscale x 4 x i8> %vc, <vscale x 4 x i8> *%pc
-  ret void
-}
-
-define void @vadd_vint8mf4(<vscale x 2 x i8> *%pc, <vscale x 2 x i8> *%pa, <vscale x 2 x i8> *%pb) nounwind {
-; CHECK-LABEL: vadd_vint8mf4:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a3, zero, e8, mf4, ta, ma
-; CHECK-NEXT:    vle8.v v8, (a1)
-; CHECK-NEXT:    vle8.v v9, (a2)
-; CHECK-NEXT:    vadd.vv v8, v8, v9
-; CHECK-NEXT:    vse8.v v8, (a0)
-; CHECK-NEXT:    ret
-  %va = load <vscale x 2 x i8>, <vscale x 2 x i8>* %pa
-  %vb = load <vscale x 2 x i8>, <vscale x 2 x i8>* %pb
-  %vc = add <vscale x 2 x i8> %va, %vb
-  store <vscale x 2 x i8> %vc, <vscale x 2 x i8> *%pc
-  ret void
-}
-
-define void @vadd_vint8mf8(<vscale x 1 x i8> *%pc, <vscale x 1 x i8> *%pa, <vscale x 1 x i8> *%pb) nounwind {
-; CHECK-LABEL: vadd_vint8mf8:
-; CHECK:       # %bb.0:
-; CHECK-NEXT:    vsetvli a3, zero, e8, mf8, ta, ma
-; CHECK-NEXT:    vle8.v v8, (a1)
-; CHECK-NEXT:    vle8.v v9, (a2)
-; CHECK-NEXT:    vadd.vv v8, v8, v9
-; CHECK-NEXT:    vse8.v v8, (a0)
-; CHECK-NEXT:    ret
-  %va = load <vscale x 1 x i8>, <vscale x 1 x i8>* %pa
-  %vb = load <vscale x 1 x i8>, <vscale x 1 x i8>* %pb
-  %vc = add <vscale x 1 x i8> %va, %vb
-  store <vscale x 1 x i8> %vc, <vscale x 1 x i8> *%pc
-  ret void
-}
diff --git a/llvm/test/CodeGen/RISCV/rvv/load-add-store.ll b/llvm/test/CodeGen/RISCV/rvv/load-add-store.ll
new file mode 100644
index 0000000000000000000000000000000000000000..3bb465ba998a232384b542bf757d979da170f881
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/rvv/load-add-store.ll
@@ -0,0 +1,502 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple riscv32 -mattr=+v %s -o - \
+; RUN:     -verify-machineinstrs | FileCheck %s
+; RUN: llc -mtriple riscv64 -mattr=+v %s -o - \
+; RUN:     -verify-machineinstrs | FileCheck %s
+
+define void @vadd_vint8m1(ptr %pc, ptr %pa, ptr %pb) nounwind {
+; CHECK-LABEL: vadd_vint8m1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vl1r.v v8, (a1)
+; CHECK-NEXT:    vl1r.v v9, (a2)
+; CHECK-NEXT:    vsetvli a1, zero, e8, m1, ta, ma
+; CHECK-NEXT:    vadd.vv v8, v8, v9
+; CHECK-NEXT:    vs1r.v v8, (a0)
+; CHECK-NEXT:    ret
+  %va = load <vscale x 8 x i8>, ptr %pa
+  %vb = load <vscale x 8 x i8>, ptr %pb
+  %vc = add <vscale x 8 x i8> %va, %vb
+  store <vscale x 8 x i8> %vc, ptr %pc
+  ret void
+}
+
+define void @vadd_vint8m2(ptr %pc, ptr %pa, ptr %pb) nounwind {
+; CHECK-LABEL: vadd_vint8m2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vl2r.v v8, (a1)
+; CHECK-NEXT:    vl2r.v v10, (a2)
+; CHECK-NEXT:    vsetvli a1, zero, e8, m2, ta, ma
+; CHECK-NEXT:    vadd.vv v8, v8, v10
+; CHECK-NEXT:    vs2r.v v8, (a0)
+; CHECK-NEXT:    ret
+  %va = load <vscale x 16 x i8>, ptr %pa
+  %vb = load <vscale x 16 x i8>, ptr %pb
+  %vc = add <vscale x 16 x i8> %va, %vb
+  store <vscale x 16 x i8> %vc, ptr %pc
+  ret void
+}
+
+define void @vadd_vint8m4(ptr %pc, ptr %pa, ptr %pb) nounwind {
+; CHECK-LABEL: vadd_vint8m4:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vl4r.v v8, (a1)
+; CHECK-NEXT:    vl4r.v v12, (a2)
+; CHECK-NEXT:    vsetvli a1, zero, e8, m4, ta, ma
+; CHECK-NEXT:    vadd.vv v8, v8, v12
+; CHECK-NEXT:    vs4r.v v8, (a0)
+; CHECK-NEXT:    ret
+  %va = load <vscale x 32 x i8>, ptr %pa
+  %vb = load <vscale x 32 x i8>, ptr %pb
+  %vc = add <vscale x 32 x i8> %va, %vb
+  store <vscale x 32 x i8> %vc, ptr %pc
+  ret void
+}
+
+define void @vadd_vint8m8(ptr %pc, ptr %pa, ptr %pb) nounwind {
+; CHECK-LABEL: vadd_vint8m8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vl8r.v v8, (a1)
+; CHECK-NEXT:    vl8r.v v16, (a2)
+; CHECK-NEXT:    vsetvli a1, zero, e8, m8, ta, ma
+; CHECK-NEXT:    vadd.vv v8, v8, v16
+; CHECK-NEXT:    vs8r.v v8, (a0)
+; CHECK-NEXT:    ret
+  %va = load <vscale x 64 x i8>, ptr %pa
+  %vb = load <vscale x 64 x i8>, ptr %pb
+  %vc = add <vscale x 64 x i8> %va, %vb
+  store <vscale x 64 x i8> %vc, ptr %pc
+  ret void
+}
+
+define void @vadd_vint8mf2(ptr %pc, ptr %pa, ptr %pb) nounwind {
+; CHECK-LABEL: vadd_vint8mf2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a3, zero, e8, mf2, ta, ma
+; CHECK-NEXT:    vle8.v v8, (a1)
+; CHECK-NEXT:    vle8.v v9, (a2)
+; CHECK-NEXT:    vadd.vv v8, v8, v9
+; CHECK-NEXT:    vse8.v v8, (a0)
+; CHECK-NEXT:    ret
+  %va = load <vscale x 4 x i8>, ptr %pa
+  %vb = load <vscale x 4 x i8>, ptr %pb
+  %vc = add <vscale x 4 x i8> %va, %vb
+  store <vscale x 4 x i8> %vc, ptr %pc
+  ret void
+}
+
+define void @vadd_vint8mf4(ptr %pc, ptr %pa, ptr %pb) nounwind {
+; CHECK-LABEL: vadd_vint8mf4:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a3, zero, e8, mf4, ta, ma
+; CHECK-NEXT:    vle8.v v8, (a1)
+; CHECK-NEXT:    vle8.v v9, (a2)
+; CHECK-NEXT:    vadd.vv v8, v8, v9
+; CHECK-NEXT:    vse8.v v8, (a0)
+; CHECK-NEXT:    ret
+  %va = load <vscale x 2 x i8>, ptr %pa
+  %vb = load <vscale x 2 x i8>, ptr %pb
+  %vc = add <vscale x 2 x i8> %va, %vb
+  store <vscale x 2 x i8> %vc, ptr %pc
+  ret void
+}
+
+define void @vadd_vint8mf8(ptr %pc, ptr %pa, ptr %pb) nounwind {
+; CHECK-LABEL: vadd_vint8mf8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a3, zero, e8, mf8, ta, ma
+; CHECK-NEXT:    vle8.v v8, (a1)
+; CHECK-NEXT:    vle8.v v9, (a2)
+; CHECK-NEXT:    vadd.vv v8, v8, v9
+; CHECK-NEXT:    vse8.v v8, (a0)
+; CHECK-NEXT:    ret
+  %va = load <vscale x 1 x i8>, ptr %pa
+  %vb = load <vscale x 1 x i8>, ptr %pb
+  %vc = add <vscale x 1 x i8> %va, %vb
+  store <vscale x 1 x i8> %vc, ptr %pc
+  ret void
+}
+
+define void @vadd_vint16m1(ptr %pc, ptr %pa, ptr %pb) nounwind {
+; CHECK-LABEL: vadd_vint16m1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vl1re16.v v8, (a1)
+; CHECK-NEXT:    vl1re16.v v9, (a2)
+; CHECK-NEXT:    vsetvli a1, zero, e16, m1, ta, ma
+; CHECK-NEXT:    vadd.vv v8, v8, v9
+; CHECK-NEXT:    vs1r.v v8, (a0)
+; CHECK-NEXT:    ret
+  %va = load <vscale x 4 x i16>, ptr %pa
+  %vb = load <vscale x 4 x i16>, ptr %pb
+  %vc = add <vscale x 4 x i16> %va, %vb
+  store <vscale x 4 x i16> %vc, ptr %pc
+  ret void
+}
+
+define void @vadd_vint16m2(ptr %pc, ptr %pa, ptr %pb) nounwind {
+; CHECK-LABEL: vadd_vint16m2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vl2re16.v v8, (a1)
+; CHECK-NEXT:    vl2re16.v v10, (a2)
+; CHECK-NEXT:    vsetvli a1, zero, e16, m2, ta, ma
+; CHECK-NEXT:    vadd.vv v8, v8, v10
+; CHECK-NEXT:    vs2r.v v8, (a0)
+; CHECK-NEXT:    ret
+  %va = load <vscale x 8 x i16>, ptr %pa
+  %vb = load <vscale x 8 x i16>, ptr %pb
+  %vc = add <vscale x 8 x i16> %va, %vb
+  store <vscale x 8 x i16> %vc, ptr %pc
+  ret void
+}
+
+define void @vadd_vint16m4(ptr %pc, ptr %pa, ptr %pb) nounwind {
+; CHECK-LABEL: vadd_vint16m4:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vl4re16.v v8, (a1)
+; CHECK-NEXT:    vl4re16.v v12, (a2)
+; CHECK-NEXT:    vsetvli a1, zero, e16, m4, ta, ma
+; CHECK-NEXT:    vadd.vv v8, v8, v12
+; CHECK-NEXT:    vs4r.v v8, (a0)
+; CHECK-NEXT:    ret
+  %va = load <vscale x 16 x i16>, ptr %pa
+  %vb = load <vscale x 16 x i16>, ptr %pb
+  %vc = add <vscale x 16 x i16> %va, %vb
+  store <vscale x 16 x i16> %vc, ptr %pc
+  ret void
+}
+
+define void @vadd_vint16m8(ptr %pc, ptr %pa, ptr %pb) nounwind {
+; CHECK-LABEL: vadd_vint16m8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vl8re16.v v8, (a1)
+; CHECK-NEXT:    vl8re16.v v16, (a2)
+; CHECK-NEXT:    vsetvli a1, zero, e16, m8, ta, ma
+; CHECK-NEXT:    vadd.vv v8, v8, v16
+; CHECK-NEXT:    vs8r.v v8, (a0)
+; CHECK-NEXT:    ret
+  %va = load <vscale x 32 x i16>, ptr %pa
+  %vb = load <vscale x 32 x i16>, ptr %pb
+  %vc = add <vscale x 32 x i16> %va, %vb
+  store <vscale x 32 x i16> %vc, ptr %pc
+  ret void
+}
+
+define void @vadd_vint16mf2(ptr %pc, ptr %pa, ptr %pb) nounwind {
+; CHECK-LABEL: vadd_vint16mf2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a3, zero, e16, mf2, ta, ma
+; CHECK-NEXT:    vle16.v v8, (a1)
+; CHECK-NEXT:    vle16.v v9, (a2)
+; CHECK-NEXT:    vadd.vv v8, v8, v9
+; CHECK-NEXT:    vse16.v v8, (a0)
+; CHECK-NEXT:    ret
+  %va = load <vscale x 2 x i16>, ptr %pa
+  %vb = load <vscale x 2 x i16>, ptr %pb
+  %vc = add <vscale x 2 x i16> %va, %vb
+  store <vscale x 2 x i16> %vc, ptr %pc
+  ret void
+}
+
+define void @vadd_vint16mf4(ptr %pc, ptr %pa, ptr %pb) nounwind {
+; CHECK-LABEL: vadd_vint16mf4:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a3, zero, e16, mf4, ta, ma
+; CHECK-NEXT:    vle16.v v8, (a1)
+; CHECK-NEXT:    vle16.v v9, (a2)
+; CHECK-NEXT:    vadd.vv v8, v8, v9
+; CHECK-NEXT:    vse16.v v8, (a0)
+; CHECK-NEXT:    ret
+  %va = load <vscale x 1 x i16>, ptr %pa
+  %vb = load <vscale x 1 x i16>, ptr %pb
+  %vc = add <vscale x 1 x i16> %va, %vb
+  store <vscale x 1 x i16> %vc, ptr %pc
+  ret void
+}
+
+define void @vadd_vint32m1(ptr %pc, ptr %pa, ptr %pb) nounwind {
+; CHECK-LABEL: vadd_vint32m1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vl1re32.v v8, (a1)
+; CHECK-NEXT:    vl1re32.v v9, (a2)
+; CHECK-NEXT:    vsetvli a1, zero, e32, m1, ta, ma
+; CHECK-NEXT:    vadd.vv v8, v8, v9
+; CHECK-NEXT:    vs1r.v v8, (a0)
+; CHECK-NEXT:    ret
+  %va = load <vscale x 2 x i32>, ptr %pa
+  %vb = load <vscale x 2 x i32>, ptr %pb
+  %vc = add <vscale x 2 x i32> %va, %vb
+  store <vscale x 2 x i32> %vc, ptr %pc
+  ret void
+}
+
+define void @vadd_vint32m2(ptr %pc, ptr %pa, ptr %pb) nounwind {
+; CHECK-LABEL: vadd_vint32m2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vl2re32.v v8, (a1)
+; CHECK-NEXT:    vl2re32.v v10, (a2)
+; CHECK-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vadd.vv v8, v8, v10
+; CHECK-NEXT:    vs2r.v v8, (a0)
+; CHECK-NEXT:    ret
+  %va = load <vscale x 4 x i32>, ptr %pa
+  %vb = load <vscale x 4 x i32>, ptr %pb
+  %vc = add <vscale x 4 x i32> %va, %vb
+  store <vscale x 4 x i32> %vc, ptr %pc
+  ret void
+}
+
+define void @vadd_vint32m4(ptr %pc, ptr %pa, ptr %pb) nounwind {
+; CHECK-LABEL: vadd_vint32m4:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vl4re32.v v8, (a1)
+; CHECK-NEXT:    vl4re32.v v12, (a2)
+; CHECK-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vadd.vv v8, v8, v12
+; CHECK-NEXT:    vs4r.v v8, (a0)
+; CHECK-NEXT:    ret
+  %va = load <vscale x 8 x i32>, ptr %pa
+  %vb = load <vscale x 8 x i32>, ptr %pb
+  %vc = add <vscale x 8 x i32> %va, %vb
+  store <vscale x 8 x i32> %vc, ptr %pc
+  ret void
+}
+
+define void @vadd_vint32m8(ptr %pc, ptr %pa, ptr %pb) nounwind {
+; CHECK-LABEL: vadd_vint32m8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vl8re32.v v8, (a1)
+; CHECK-NEXT:    vl8re32.v v16, (a2)
+; CHECK-NEXT:    vsetvli a1, zero, e32, m8, ta, ma
+; CHECK-NEXT:    vadd.vv v8, v8, v16
+; CHECK-NEXT:    vs8r.v v8, (a0)
+; CHECK-NEXT:    ret
+  %va = load <vscale x 16 x i32>, ptr %pa
+  %vb = load <vscale x 16 x i32>, ptr %pb
+  %vc = add <vscale x 16 x i32> %va, %vb
+  store <vscale x 16 x i32> %vc, ptr %pc
+  ret void
+}
+
+define void @vadd_vint32mf2(ptr %pc, ptr %pa, ptr %pb) nounwind {
+; CHECK-LABEL: vadd_vint32mf2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a3, zero, e32, mf2, ta, ma
+; CHECK-NEXT:    vle32.v v8, (a1)
+; CHECK-NEXT:    vle32.v v9, (a2)
+; CHECK-NEXT:    vadd.vv v8, v8, v9
+; CHECK-NEXT:    vse32.v v8, (a0)
+; CHECK-NEXT:    ret
+  %va = load <vscale x 1 x i32>, ptr %pa
+  %vb = load <vscale x 1 x i32>, ptr %pb
+  %vc = add <vscale x 1 x i32> %va, %vb
+  store <vscale x 1 x i32> %vc, ptr %pc
+  ret void
+}
+
+define void @vadd_vint64m1(ptr %pc, ptr %pa, ptr %pb) nounwind {
+; CHECK-LABEL: vadd_vint64m1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vl1re64.v v8, (a1)
+; CHECK-NEXT:    vl1re64.v v9, (a2)
+; CHECK-NEXT:    vsetvli a1, zero, e64, m1, ta, ma
+; CHECK-NEXT:    vadd.vv v8, v8, v9
+; CHECK-NEXT:    vs1r.v v8, (a0)
+; CHECK-NEXT:    ret
+  %va = load <vscale x 1 x i64>, ptr %pa
+  %vb = load <vscale x 1 x i64>, ptr %pb
+  %vc = add <vscale x 1 x i64> %va, %vb
+  store <vscale x 1 x i64> %vc, ptr %pc
+  ret void
+}
+
+define void @vadd_vint64m2(ptr %pc, ptr %pa, ptr %pb) nounwind {
+; CHECK-LABEL: vadd_vint64m2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vl2re64.v v8, (a1)
+; CHECK-NEXT:    vl2re64.v v10, (a2)
+; CHECK-NEXT:    vsetvli a1, zero, e64, m2, ta, ma
+; CHECK-NEXT:    vadd.vv v8, v8, v10
+; CHECK-NEXT:    vs2r.v v8, (a0)
+; CHECK-NEXT:    ret
+  %va = load <vscale x 2 x i64>, ptr %pa
+  %vb = load <vscale x 2 x i64>, ptr %pb
+  %vc = add <vscale x 2 x i64> %va, %vb
+  store <vscale x 2 x i64> %vc, ptr %pc
+  ret void
+}
+
+define void @vadd_vint64m4(ptr %pc, ptr %pa, ptr %pb) nounwind {
+; CHECK-LABEL: vadd_vint64m4:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vl4re64.v v8, (a1)
+; CHECK-NEXT:    vl4re64.v v12, (a2)
+; CHECK-NEXT:    vsetvli a1, zero, e64, m4, ta, ma
+; CHECK-NEXT:    vadd.vv v8, v8, v12
+; CHECK-NEXT:    vs4r.v v8, (a0)
+; CHECK-NEXT:    ret
+  %va = load <vscale x 4 x i64>, ptr %pa
+  %vb = load <vscale x 4 x i64>, ptr %pb
+  %vc = add <vscale x 4 x i64> %va, %vb
+  store <vscale x 4 x i64> %vc, ptr %pc
+  ret void
+}
+
+define void @vadd_vint64m8(ptr %pc, ptr %pa, ptr %pb) nounwind {
+; CHECK-LABEL: vadd_vint64m8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vl8re64.v v8, (a1)
+; CHECK-NEXT:    vl8re64.v v16, (a2)
+; CHECK-NEXT:    vsetvli a1, zero, e64, m8, ta, ma
+; CHECK-NEXT:    vadd.vv v8, v8, v16
+; CHECK-NEXT:    vs8r.v v8, (a0)
+; CHECK-NEXT:    ret
+  %va = load <vscale x 8 x i64>, ptr %pa
+  %vb = load <vscale x 8 x i64>, ptr %pb
+  %vc = add <vscale x 8 x i64> %va, %vb
+  store <vscale x 8 x i64> %vc, ptr %pc
+  ret void
+}
+
+
+define void @exact_vlen_vadd_vint8m1(ptr %pc, ptr %pa, ptr %pb) nounwind vscale_range(2,2) {
+; CHECK-LABEL: exact_vlen_vadd_vint8m1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vl1r.v v8, (a1)
+; CHECK-NEXT:    vl1r.v v9, (a2)
+; CHECK-NEXT:    vsetvli a1, zero, e8, m1, ta, ma
+; CHECK-NEXT:    vadd.vv v8, v8, v9
+; CHECK-NEXT:    vs1r.v v8, (a0)
+; CHECK-NEXT:    ret
+  %va = load <vscale x 8 x i8>, ptr %pa
+  %vb = load <vscale x 8 x i8>, ptr %pb
+  %vc = add <vscale x 8 x i8> %va, %vb
+  store <vscale x 8 x i8> %vc, ptr %pc
+  ret void
+}
+
+define void @exact_vlen_vadd_vint8m2(ptr %pc, ptr %pa, ptr %pb) nounwind vscale_range(2,2) {
+; CHECK-LABEL: exact_vlen_vadd_vint8m2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vl2r.v v8, (a1)
+; CHECK-NEXT:    vl2r.v v10, (a2)
+; CHECK-NEXT:    vsetvli a1, zero, e8, m2, ta, ma
+; CHECK-NEXT:    vadd.vv v8, v8, v10
+; CHECK-NEXT:    vs2r.v v8, (a0)
+; CHECK-NEXT:    ret
+  %va = load <vscale x 16 x i8>, ptr %pa
+  %vb = load <vscale x 16 x i8>, ptr %pb
+  %vc = add <vscale x 16 x i8> %va, %vb
+  store <vscale x 16 x i8> %vc, ptr %pc
+  ret void
+}
+
+define void @exact_vlen_vadd_vint8mf2(ptr %pc, ptr %pa, ptr %pb) nounwind vscale_range(2,2) {
+; CHECK-LABEL: exact_vlen_vadd_vint8mf2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a3, zero, e8, mf2, ta, ma
+; CHECK-NEXT:    vle8.v v8, (a1)
+; CHECK-NEXT:    vle8.v v9, (a2)
+; CHECK-NEXT:    vadd.vv v8, v8, v9
+; CHECK-NEXT:    vse8.v v8, (a0)
+; CHECK-NEXT:    ret
+  %va = load <vscale x 4 x i8>, ptr %pa
+  %vb = load <vscale x 4 x i8>, ptr %pb
+  %vc = add <vscale x 4 x i8> %va, %vb
+  store <vscale x 4 x i8> %vc, ptr %pc
+  ret void
+}
+
+define void @exact_vlen_vadd_vint8mf4(ptr %pc, ptr %pa, ptr %pb) nounwind vscale_range(2,2) {
+; CHECK-LABEL: exact_vlen_vadd_vint8mf4:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a3, zero, e8, mf4, ta, ma
+; CHECK-NEXT:    vle8.v v8, (a1)
+; CHECK-NEXT:    vle8.v v9, (a2)
+; CHECK-NEXT:    vadd.vv v8, v8, v9
+; CHECK-NEXT:    vse8.v v8, (a0)
+; CHECK-NEXT:    ret
+  %va = load <vscale x 2 x i8>, ptr %pa
+  %vb = load <vscale x 2 x i8>, ptr %pb
+  %vc = add <vscale x 2 x i8> %va, %vb
+  store <vscale x 2 x i8> %vc, ptr %pc
+  ret void
+}
+
+define void @exact_vlen_vadd_vint8mf8(ptr %pc, ptr %pa, ptr %pb) nounwind vscale_range(2,2) {
+; CHECK-LABEL: exact_vlen_vadd_vint8mf8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a3, zero, e8, mf8, ta, ma
+; CHECK-NEXT:    vle8.v v8, (a1)
+; CHECK-NEXT:    vle8.v v9, (a2)
+; CHECK-NEXT:    vadd.vv v8, v8, v9
+; CHECK-NEXT:    vse8.v v8, (a0)
+; CHECK-NEXT:    ret
+  %va = load <vscale x 1 x i8>, ptr %pa
+  %vb = load <vscale x 1 x i8>, ptr %pb
+  %vc = add <vscale x 1 x i8> %va, %vb
+  store <vscale x 1 x i8> %vc, ptr %pc
+  ret void
+}
+
+define void @exact_vlen_vadd_vint32m1(ptr %pc, ptr %pa, ptr %pb) nounwind vscale_range(2,2) {
+; CHECK-LABEL: exact_vlen_vadd_vint32m1:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vl1re32.v v8, (a1)
+; CHECK-NEXT:    vl1re32.v v9, (a2)
+; CHECK-NEXT:    vsetvli a1, zero, e32, m1, ta, ma
+; CHECK-NEXT:    vadd.vv v8, v8, v9
+; CHECK-NEXT:    vs1r.v v8, (a0)
+; CHECK-NEXT:    ret
+  %va = load <vscale x 2 x i32>, ptr %pa
+  %vb = load <vscale x 2 x i32>, ptr %pb
+  %vc = add <vscale x 2 x i32> %va, %vb
+  store <vscale x 2 x i32> %vc, ptr %pc
+  ret void
+}
+
+define void @exact_vlen_vadd_vint32m2(ptr %pc, ptr %pa, ptr %pb) nounwind vscale_range(2,2) {
+; CHECK-LABEL: exact_vlen_vadd_vint32m2:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vl2re32.v v8, (a1)
+; CHECK-NEXT:    vl2re32.v v10, (a2)
+; CHECK-NEXT:    vsetvli a1, zero, e32, m2, ta, ma
+; CHECK-NEXT:    vadd.vv v8, v8, v10
+; CHECK-NEXT:    vs2r.v v8, (a0)
+; CHECK-NEXT:    ret
+  %va = load <vscale x 4 x i32>, ptr %pa
+  %vb = load <vscale x 4 x i32>, ptr %pb
+  %vc = add <vscale x 4 x i32> %va, %vb
+  store <vscale x 4 x i32> %vc, ptr %pc
+  ret void
+}
+
+define void @exact_vlen_vadd_vint32m4(ptr %pc, ptr %pa, ptr %pb) nounwind vscale_range(2,2) {
+; CHECK-LABEL: exact_vlen_vadd_vint32m4:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vl4re32.v v8, (a1)
+; CHECK-NEXT:    vl4re32.v v12, (a2)
+; CHECK-NEXT:    vsetvli a1, zero, e32, m4, ta, ma
+; CHECK-NEXT:    vadd.vv v8, v8, v12
+; CHECK-NEXT:    vs4r.v v8, (a0)
+; CHECK-NEXT:    ret
+  %va = load <vscale x 8 x i32>, ptr %pa
+  %vb = load <vscale x 8 x i32>, ptr %pb
+  %vc = add <vscale x 8 x i32> %va, %vb
+  store <vscale x 8 x i32> %vc, ptr %pc
+  ret void
+}
+
+define void @exact_vlen_vadd_vint32m8(ptr %pc, ptr %pa, ptr %pb) nounwind vscale_range(2,2) {
+; CHECK-LABEL: exact_vlen_vadd_vint32m8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vl8re32.v v8, (a1)
+; CHECK-NEXT:    vl8re32.v v16, (a2)
+; CHECK-NEXT:    vsetvli a1, zero, e32, m8, ta, ma
+; CHECK-NEXT:    vadd.vv v8, v8, v16
+; CHECK-NEXT:    vs8r.v v8, (a0)
+; CHECK-NEXT:    ret
+  %va = load <vscale x 16 x i32>, ptr %pa
+  %vb = load <vscale x 16 x i32>, ptr %pb
+  %vc = add <vscale x 16 x i32> %va, %vb
+  store <vscale x 16 x i32> %vc, ptr %pc
+  ret void
+}
diff --git a/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.ll b/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.ll
index 1114c7657c63d34aa401731eb4c39cc71ccd0862..a79180ed45b42bde33e83104c656a8c8acfecfa4 100644
--- a/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/vsetvli-insert.ll
@@ -594,6 +594,39 @@ bb:
   ret i64 %tmp2
 }
 
+
+define void @add_v128i8(ptr %x, ptr %y) vscale_range(2,2) {
+; CHECK-LABEL: add_v128i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetvli a2, zero, e8, m8, ta, ma
+; CHECK-NEXT:    vle8.v v8, (a0)
+; CHECK-NEXT:    vle8.v v16, (a1)
+; CHECK-NEXT:    vadd.vv v8, v8, v16
+; CHECK-NEXT:    vse8.v v8, (a0)
+; CHECK-NEXT:    ret
+  %a = load <128 x i8>, ptr %x
+  %b = load <128 x i8>, ptr %y
+  %c = add <128 x i8> %a, %b
+  store <128 x i8> %c, ptr %x
+  ret void
+}
+
+define void @add_v16i64(ptr %x, ptr %y) vscale_range(2,2) {
+; CHECK-LABEL: add_v16i64:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vsetivli zero, 16, e64, m8, ta, ma
+; CHECK-NEXT:    vle64.v v8, (a0)
+; CHECK-NEXT:    vle64.v v16, (a1)
+; CHECK-NEXT:    vadd.vv v8, v8, v16
+; CHECK-NEXT:    vse64.v v8, (a0)
+; CHECK-NEXT:    ret
+  %a = load <16 x i64>, ptr %x
+  %b = load <16 x i64>, ptr %y
+  %c = add <16 x i64> %a, %b
+  store <16 x i64> %c, ptr %x
+  ret void
+}
+
 declare <vscale x 1 x i64> @llvm.riscv.vadd.mask.nxv1i64.nxv1i64(
   <vscale x 1 x i64>,
   <vscale x 1 x i64>,
diff --git a/llvm/test/CodeGen/RISCV/select-and.ll b/llvm/test/CodeGen/RISCV/select-and.ll
index 51f411bd96548c7e503a356ffb0a941d26abf39d..5ba489043f0b4414ddcfeea54f2ed4f332731fd7 100644
--- a/llvm/test/CodeGen/RISCV/select-and.ll
+++ b/llvm/test/CodeGen/RISCV/select-and.ll
@@ -40,14 +40,15 @@ define signext i32 @if_of_and(i1 zeroext %a, i1 zeroext %b) nounwind {
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32I-NEXT:    and a0, a0, a1
-; RV32I-NEXT:    beqz a0, .LBB1_2
-; RV32I-NEXT:  # %bb.1: # %if.then
+; RV32I-NEXT:    beqz a0, .LBB1_3
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    beqz a1, .LBB1_3
+; RV32I-NEXT:  # %bb.2: # %if.then
 ; RV32I-NEXT:    call both@plt
-; RV32I-NEXT:    j .LBB1_3
-; RV32I-NEXT:  .LBB1_2: # %if.else
+; RV32I-NEXT:    j .LBB1_4
+; RV32I-NEXT:  .LBB1_3: # %if.else
 ; RV32I-NEXT:    call neither@plt
-; RV32I-NEXT:  .LBB1_3: # %if.end
+; RV32I-NEXT:  .LBB1_4: # %if.end
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
@@ -56,14 +57,15 @@ define signext i32 @if_of_and(i1 zeroext %a, i1 zeroext %b) nounwind {
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    and a0, a0, a1
-; RV64I-NEXT:    beqz a0, .LBB1_2
-; RV64I-NEXT:  # %bb.1: # %if.then
+; RV64I-NEXT:    beqz a0, .LBB1_3
+; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    beqz a1, .LBB1_3
+; RV64I-NEXT:  # %bb.2: # %if.then
 ; RV64I-NEXT:    call both@plt
-; RV64I-NEXT:    j .LBB1_3
-; RV64I-NEXT:  .LBB1_2: # %if.else
+; RV64I-NEXT:    j .LBB1_4
+; RV64I-NEXT:  .LBB1_3: # %if.else
 ; RV64I-NEXT:    call neither@plt
-; RV64I-NEXT:  .LBB1_3: # %if.end
+; RV64I-NEXT:  .LBB1_4: # %if.end
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/select-or.ll b/llvm/test/CodeGen/RISCV/select-or.ll
index 9ba284c611a6b371e3db8428156fc746ddd0e6f7..d378bb4dd563debeb860f96e022749261aee598c 100644
--- a/llvm/test/CodeGen/RISCV/select-or.ll
+++ b/llvm/test/CodeGen/RISCV/select-or.ll
@@ -40,14 +40,15 @@ define signext i32 @if_of_or(i1 zeroext %a, i1 zeroext %b) nounwind {
 ; RV32I:       # %bb.0:
 ; RV32I-NEXT:    addi sp, sp, -16
 ; RV32I-NEXT:    sw ra, 12(sp) # 4-byte Folded Spill
-; RV32I-NEXT:    or a0, a0, a1
-; RV32I-NEXT:    beqz a0, .LBB1_2
-; RV32I-NEXT:  # %bb.1: # %if.then
-; RV32I-NEXT:    call either@plt
-; RV32I-NEXT:    j .LBB1_3
-; RV32I-NEXT:  .LBB1_2: # %if.else
+; RV32I-NEXT:    bnez a0, .LBB1_3
+; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    bnez a1, .LBB1_3
+; RV32I-NEXT:  # %bb.2: # %if.else
 ; RV32I-NEXT:    call neither@plt
-; RV32I-NEXT:  .LBB1_3: # %if.end
+; RV32I-NEXT:    j .LBB1_4
+; RV32I-NEXT:  .LBB1_3: # %if.then
+; RV32I-NEXT:    call either@plt
+; RV32I-NEXT:  .LBB1_4: # %if.end
 ; RV32I-NEXT:    lw ra, 12(sp) # 4-byte Folded Reload
 ; RV32I-NEXT:    addi sp, sp, 16
 ; RV32I-NEXT:    ret
@@ -56,14 +57,15 @@ define signext i32 @if_of_or(i1 zeroext %a, i1 zeroext %b) nounwind {
 ; RV64I:       # %bb.0:
 ; RV64I-NEXT:    addi sp, sp, -16
 ; RV64I-NEXT:    sd ra, 8(sp) # 8-byte Folded Spill
-; RV64I-NEXT:    or a0, a0, a1
-; RV64I-NEXT:    beqz a0, .LBB1_2
-; RV64I-NEXT:  # %bb.1: # %if.then
-; RV64I-NEXT:    call either@plt
-; RV64I-NEXT:    j .LBB1_3
-; RV64I-NEXT:  .LBB1_2: # %if.else
+; RV64I-NEXT:    bnez a0, .LBB1_3
+; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    bnez a1, .LBB1_3
+; RV64I-NEXT:  # %bb.2: # %if.else
 ; RV64I-NEXT:    call neither@plt
-; RV64I-NEXT:  .LBB1_3: # %if.end
+; RV64I-NEXT:    j .LBB1_4
+; RV64I-NEXT:  .LBB1_3: # %if.then
+; RV64I-NEXT:    call either@plt
+; RV64I-NEXT:  .LBB1_4: # %if.end
 ; RV64I-NEXT:    ld ra, 8(sp) # 8-byte Folded Reload
 ; RV64I-NEXT:    addi sp, sp, 16
 ; RV64I-NEXT:    ret
diff --git a/llvm/test/CodeGen/RISCV/setcc-logic.ll b/llvm/test/CodeGen/RISCV/setcc-logic.ll
index 6a6c1bcd8dec1388d6a1ed890498ba8a37942811..14e76d1ff17288ba91276abf5554db767862554a 100644
--- a/llvm/test/CodeGen/RISCV/setcc-logic.ll
+++ b/llvm/test/CodeGen/RISCV/setcc-logic.ll
@@ -298,26 +298,22 @@ declare void @bar(...)
 define void @and_sge_eq(i32 signext %0, i32 signext %1, i32 signext %2, i32 signext %3) {
 ; RV32I-LABEL: and_sge_eq:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    slt a0, a0, a1
-; RV32I-NEXT:    xor a2, a2, a3
-; RV32I-NEXT:    snez a1, a2
-; RV32I-NEXT:    or a0, a1, a0
-; RV32I-NEXT:    bnez a0, .LBB13_2
+; RV32I-NEXT:    blt a0, a1, .LBB13_3
 ; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    bne a2, a3, .LBB13_3
+; RV32I-NEXT:  # %bb.2:
 ; RV32I-NEXT:    ret
-; RV32I-NEXT:  .LBB13_2:
+; RV32I-NEXT:  .LBB13_3:
 ; RV32I-NEXT:    tail bar@plt
 ;
 ; RV64I-LABEL: and_sge_eq:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    slt a0, a0, a1
-; RV64I-NEXT:    xor a2, a2, a3
-; RV64I-NEXT:    snez a1, a2
-; RV64I-NEXT:    or a0, a1, a0
-; RV64I-NEXT:    bnez a0, .LBB13_2
+; RV64I-NEXT:    blt a0, a1, .LBB13_3
 ; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    bne a2, a3, .LBB13_3
+; RV64I-NEXT:  # %bb.2:
 ; RV64I-NEXT:    ret
-; RV64I-NEXT:  .LBB13_2:
+; RV64I-NEXT:  .LBB13_3:
 ; RV64I-NEXT:    tail bar@plt
   %5 = icmp sge i32 %0, %1
   %6 = icmp eq i32 %2, %3
@@ -335,26 +331,22 @@ define void @and_sge_eq(i32 signext %0, i32 signext %1, i32 signext %2, i32 sign
 define void @and_sle_eq(i32 signext %0, i32 signext %1, i32 signext %2, i32 signext %3) {
 ; RV32I-LABEL: and_sle_eq:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    slt a0, a1, a0
-; RV32I-NEXT:    xor a2, a2, a3
-; RV32I-NEXT:    snez a1, a2
-; RV32I-NEXT:    or a0, a1, a0
-; RV32I-NEXT:    bnez a0, .LBB14_2
+; RV32I-NEXT:    blt a1, a0, .LBB14_3
 ; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    bne a2, a3, .LBB14_3
+; RV32I-NEXT:  # %bb.2:
 ; RV32I-NEXT:    ret
-; RV32I-NEXT:  .LBB14_2:
+; RV32I-NEXT:  .LBB14_3:
 ; RV32I-NEXT:    tail bar@plt
 ;
 ; RV64I-LABEL: and_sle_eq:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    slt a0, a1, a0
-; RV64I-NEXT:    xor a2, a2, a3
-; RV64I-NEXT:    snez a1, a2
-; RV64I-NEXT:    or a0, a1, a0
-; RV64I-NEXT:    bnez a0, .LBB14_2
+; RV64I-NEXT:    blt a1, a0, .LBB14_3
 ; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    bne a2, a3, .LBB14_3
+; RV64I-NEXT:  # %bb.2:
 ; RV64I-NEXT:    ret
-; RV64I-NEXT:  .LBB14_2:
+; RV64I-NEXT:  .LBB14_3:
 ; RV64I-NEXT:    tail bar@plt
   %5 = icmp sle i32 %0, %1
   %6 = icmp eq i32 %2, %3
@@ -372,26 +364,22 @@ define void @and_sle_eq(i32 signext %0, i32 signext %1, i32 signext %2, i32 sign
 define void @and_uge_eq(i32 signext %0, i32 signext %1, i32 signext %2, i32 signext %3) {
 ; RV32I-LABEL: and_uge_eq:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    sltu a0, a0, a1
-; RV32I-NEXT:    xor a2, a2, a3
-; RV32I-NEXT:    snez a1, a2
-; RV32I-NEXT:    or a0, a1, a0
-; RV32I-NEXT:    bnez a0, .LBB15_2
+; RV32I-NEXT:    bltu a0, a1, .LBB15_3
 ; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    bne a2, a3, .LBB15_3
+; RV32I-NEXT:  # %bb.2:
 ; RV32I-NEXT:    ret
-; RV32I-NEXT:  .LBB15_2:
+; RV32I-NEXT:  .LBB15_3:
 ; RV32I-NEXT:    tail bar@plt
 ;
 ; RV64I-LABEL: and_uge_eq:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sltu a0, a0, a1
-; RV64I-NEXT:    xor a2, a2, a3
-; RV64I-NEXT:    snez a1, a2
-; RV64I-NEXT:    or a0, a1, a0
-; RV64I-NEXT:    bnez a0, .LBB15_2
+; RV64I-NEXT:    bltu a0, a1, .LBB15_3
 ; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    bne a2, a3, .LBB15_3
+; RV64I-NEXT:  # %bb.2:
 ; RV64I-NEXT:    ret
-; RV64I-NEXT:  .LBB15_2:
+; RV64I-NEXT:  .LBB15_3:
 ; RV64I-NEXT:    tail bar@plt
   %5 = icmp uge i32 %0, %1
   %6 = icmp eq i32 %2, %3
@@ -409,26 +397,22 @@ define void @and_uge_eq(i32 signext %0, i32 signext %1, i32 signext %2, i32 sign
 define void @and_ule_eq(i32 signext %0, i32 signext %1, i32 signext %2, i32 signext %3) {
 ; RV32I-LABEL: and_ule_eq:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    sltu a0, a1, a0
-; RV32I-NEXT:    xor a2, a2, a3
-; RV32I-NEXT:    snez a1, a2
-; RV32I-NEXT:    or a0, a1, a0
-; RV32I-NEXT:    bnez a0, .LBB16_2
+; RV32I-NEXT:    bltu a1, a0, .LBB16_3
 ; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    bne a2, a3, .LBB16_3
+; RV32I-NEXT:  # %bb.2:
 ; RV32I-NEXT:    ret
-; RV32I-NEXT:  .LBB16_2:
+; RV32I-NEXT:  .LBB16_3:
 ; RV32I-NEXT:    tail bar@plt
 ;
 ; RV64I-LABEL: and_ule_eq:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sltu a0, a1, a0
-; RV64I-NEXT:    xor a2, a2, a3
-; RV64I-NEXT:    snez a1, a2
-; RV64I-NEXT:    or a0, a1, a0
-; RV64I-NEXT:    bnez a0, .LBB16_2
+; RV64I-NEXT:    bltu a1, a0, .LBB16_3
 ; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    bne a2, a3, .LBB16_3
+; RV64I-NEXT:  # %bb.2:
 ; RV64I-NEXT:    ret
-; RV64I-NEXT:  .LBB16_2:
+; RV64I-NEXT:  .LBB16_3:
 ; RV64I-NEXT:    tail bar@plt
   %5 = icmp ule i32 %0, %1
   %6 = icmp eq i32 %2, %3
@@ -446,26 +430,22 @@ define void @and_ule_eq(i32 signext %0, i32 signext %1, i32 signext %2, i32 sign
 define void @and_sge_ne(i32 signext %0, i32 signext %1, i32 signext %2, i32 signext %3) {
 ; RV32I-LABEL: and_sge_ne:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    slt a0, a0, a1
-; RV32I-NEXT:    xor a2, a2, a3
-; RV32I-NEXT:    seqz a1, a2
-; RV32I-NEXT:    or a0, a1, a0
-; RV32I-NEXT:    bnez a0, .LBB17_2
+; RV32I-NEXT:    blt a0, a1, .LBB17_3
 ; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    beq a2, a3, .LBB17_3
+; RV32I-NEXT:  # %bb.2:
 ; RV32I-NEXT:    ret
-; RV32I-NEXT:  .LBB17_2:
+; RV32I-NEXT:  .LBB17_3:
 ; RV32I-NEXT:    tail bar@plt
 ;
 ; RV64I-LABEL: and_sge_ne:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    slt a0, a0, a1
-; RV64I-NEXT:    xor a2, a2, a3
-; RV64I-NEXT:    seqz a1, a2
-; RV64I-NEXT:    or a0, a1, a0
-; RV64I-NEXT:    bnez a0, .LBB17_2
+; RV64I-NEXT:    blt a0, a1, .LBB17_3
 ; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    beq a2, a3, .LBB17_3
+; RV64I-NEXT:  # %bb.2:
 ; RV64I-NEXT:    ret
-; RV64I-NEXT:  .LBB17_2:
+; RV64I-NEXT:  .LBB17_3:
 ; RV64I-NEXT:    tail bar@plt
   %5 = icmp sge i32 %0, %1
   %6 = icmp ne i32 %2, %3
@@ -483,26 +463,22 @@ define void @and_sge_ne(i32 signext %0, i32 signext %1, i32 signext %2, i32 sign
 define void @and_sle_ne(i32 signext %0, i32 signext %1, i32 signext %2, i32 signext %3) {
 ; RV32I-LABEL: and_sle_ne:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    slt a0, a1, a0
-; RV32I-NEXT:    xor a2, a2, a3
-; RV32I-NEXT:    seqz a1, a2
-; RV32I-NEXT:    or a0, a1, a0
-; RV32I-NEXT:    bnez a0, .LBB18_2
+; RV32I-NEXT:    blt a1, a0, .LBB18_3
 ; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    beq a2, a3, .LBB18_3
+; RV32I-NEXT:  # %bb.2:
 ; RV32I-NEXT:    ret
-; RV32I-NEXT:  .LBB18_2:
+; RV32I-NEXT:  .LBB18_3:
 ; RV32I-NEXT:    tail bar@plt
 ;
 ; RV64I-LABEL: and_sle_ne:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    slt a0, a1, a0
-; RV64I-NEXT:    xor a2, a2, a3
-; RV64I-NEXT:    seqz a1, a2
-; RV64I-NEXT:    or a0, a1, a0
-; RV64I-NEXT:    bnez a0, .LBB18_2
+; RV64I-NEXT:    blt a1, a0, .LBB18_3
 ; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    beq a2, a3, .LBB18_3
+; RV64I-NEXT:  # %bb.2:
 ; RV64I-NEXT:    ret
-; RV64I-NEXT:  .LBB18_2:
+; RV64I-NEXT:  .LBB18_3:
 ; RV64I-NEXT:    tail bar@plt
   %5 = icmp sle i32 %0, %1
   %6 = icmp ne i32 %2, %3
@@ -520,26 +496,22 @@ define void @and_sle_ne(i32 signext %0, i32 signext %1, i32 signext %2, i32 sign
 define void @and_uge_ne(i32 signext %0, i32 signext %1, i32 signext %2, i32 signext %3) {
 ; RV32I-LABEL: and_uge_ne:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    sltu a0, a0, a1
-; RV32I-NEXT:    xor a2, a2, a3
-; RV32I-NEXT:    seqz a1, a2
-; RV32I-NEXT:    or a0, a1, a0
-; RV32I-NEXT:    bnez a0, .LBB19_2
+; RV32I-NEXT:    bltu a0, a1, .LBB19_3
 ; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    beq a2, a3, .LBB19_3
+; RV32I-NEXT:  # %bb.2:
 ; RV32I-NEXT:    ret
-; RV32I-NEXT:  .LBB19_2:
+; RV32I-NEXT:  .LBB19_3:
 ; RV32I-NEXT:    tail bar@plt
 ;
 ; RV64I-LABEL: and_uge_ne:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sltu a0, a0, a1
-; RV64I-NEXT:    xor a2, a2, a3
-; RV64I-NEXT:    seqz a1, a2
-; RV64I-NEXT:    or a0, a1, a0
-; RV64I-NEXT:    bnez a0, .LBB19_2
+; RV64I-NEXT:    bltu a0, a1, .LBB19_3
 ; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    beq a2, a3, .LBB19_3
+; RV64I-NEXT:  # %bb.2:
 ; RV64I-NEXT:    ret
-; RV64I-NEXT:  .LBB19_2:
+; RV64I-NEXT:  .LBB19_3:
 ; RV64I-NEXT:    tail bar@plt
   %5 = icmp uge i32 %0, %1
   %6 = icmp ne i32 %2, %3
@@ -557,26 +529,22 @@ define void @and_uge_ne(i32 signext %0, i32 signext %1, i32 signext %2, i32 sign
 define void @and_ule_ne(i32 signext %0, i32 signext %1, i32 signext %2, i32 signext %3) {
 ; RV32I-LABEL: and_ule_ne:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    sltu a0, a1, a0
-; RV32I-NEXT:    xor a2, a2, a3
-; RV32I-NEXT:    seqz a1, a2
-; RV32I-NEXT:    or a0, a1, a0
-; RV32I-NEXT:    bnez a0, .LBB20_2
+; RV32I-NEXT:    bltu a1, a0, .LBB20_3
 ; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    beq a2, a3, .LBB20_3
+; RV32I-NEXT:  # %bb.2:
 ; RV32I-NEXT:    ret
-; RV32I-NEXT:  .LBB20_2:
+; RV32I-NEXT:  .LBB20_3:
 ; RV32I-NEXT:    tail bar@plt
 ;
 ; RV64I-LABEL: and_ule_ne:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sltu a0, a1, a0
-; RV64I-NEXT:    xor a2, a2, a3
-; RV64I-NEXT:    seqz a1, a2
-; RV64I-NEXT:    or a0, a1, a0
-; RV64I-NEXT:    bnez a0, .LBB20_2
+; RV64I-NEXT:    bltu a1, a0, .LBB20_3
 ; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    beq a2, a3, .LBB20_3
+; RV64I-NEXT:  # %bb.2:
 ; RV64I-NEXT:    ret
-; RV64I-NEXT:  .LBB20_2:
+; RV64I-NEXT:  .LBB20_3:
 ; RV64I-NEXT:    tail bar@plt
   %5 = icmp ule i32 %0, %1
   %6 = icmp ne i32 %2, %3
@@ -594,27 +562,23 @@ define void @and_ule_ne(i32 signext %0, i32 signext %1, i32 signext %2, i32 sign
 define void @or_sge_eq(i32 signext %0, i32 signext %1, i32 signext %2, i32 signext %3) {
 ; RV32I-LABEL: or_sge_eq:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    slt a0, a0, a1
-; RV32I-NEXT:    xor a2, a2, a3
-; RV32I-NEXT:    snez a1, a2
-; RV32I-NEXT:    and a0, a1, a0
-; RV32I-NEXT:    bnez a0, .LBB21_2
+; RV32I-NEXT:    bge a0, a1, .LBB21_3
 ; RV32I-NEXT:  # %bb.1:
-; RV32I-NEXT:    ret
-; RV32I-NEXT:  .LBB21_2:
+; RV32I-NEXT:    beq a2, a3, .LBB21_3
+; RV32I-NEXT:  # %bb.2:
 ; RV32I-NEXT:    tail bar@plt
+; RV32I-NEXT:  .LBB21_3:
+; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: or_sge_eq:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    slt a0, a0, a1
-; RV64I-NEXT:    xor a2, a2, a3
-; RV64I-NEXT:    snez a1, a2
-; RV64I-NEXT:    and a0, a1, a0
-; RV64I-NEXT:    bnez a0, .LBB21_2
+; RV64I-NEXT:    bge a0, a1, .LBB21_3
 ; RV64I-NEXT:  # %bb.1:
-; RV64I-NEXT:    ret
-; RV64I-NEXT:  .LBB21_2:
+; RV64I-NEXT:    beq a2, a3, .LBB21_3
+; RV64I-NEXT:  # %bb.2:
 ; RV64I-NEXT:    tail bar@plt
+; RV64I-NEXT:  .LBB21_3:
+; RV64I-NEXT:    ret
   %5 = icmp sge i32 %0, %1
   %6 = icmp eq i32 %2, %3
   %7 = or i1 %5, %6
@@ -631,27 +595,23 @@ define void @or_sge_eq(i32 signext %0, i32 signext %1, i32 signext %2, i32 signe
 define void @or_sle_eq(i32 signext %0, i32 signext %1, i32 signext %2, i32 signext %3) {
 ; RV32I-LABEL: or_sle_eq:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    slt a0, a1, a0
-; RV32I-NEXT:    xor a2, a2, a3
-; RV32I-NEXT:    snez a1, a2
-; RV32I-NEXT:    and a0, a1, a0
-; RV32I-NEXT:    bnez a0, .LBB22_2
+; RV32I-NEXT:    bge a1, a0, .LBB22_3
 ; RV32I-NEXT:  # %bb.1:
-; RV32I-NEXT:    ret
-; RV32I-NEXT:  .LBB22_2:
+; RV32I-NEXT:    beq a2, a3, .LBB22_3
+; RV32I-NEXT:  # %bb.2:
 ; RV32I-NEXT:    tail bar@plt
+; RV32I-NEXT:  .LBB22_3:
+; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: or_sle_eq:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    slt a0, a1, a0
-; RV64I-NEXT:    xor a2, a2, a3
-; RV64I-NEXT:    snez a1, a2
-; RV64I-NEXT:    and a0, a1, a0
-; RV64I-NEXT:    bnez a0, .LBB22_2
+; RV64I-NEXT:    bge a1, a0, .LBB22_3
 ; RV64I-NEXT:  # %bb.1:
-; RV64I-NEXT:    ret
-; RV64I-NEXT:  .LBB22_2:
+; RV64I-NEXT:    beq a2, a3, .LBB22_3
+; RV64I-NEXT:  # %bb.2:
 ; RV64I-NEXT:    tail bar@plt
+; RV64I-NEXT:  .LBB22_3:
+; RV64I-NEXT:    ret
   %5 = icmp sle i32 %0, %1
   %6 = icmp eq i32 %2, %3
   %7 = or i1 %5, %6
@@ -668,27 +628,23 @@ define void @or_sle_eq(i32 signext %0, i32 signext %1, i32 signext %2, i32 signe
 define void @or_uge_eq(i32 signext %0, i32 signext %1, i32 signext %2, i32 signext %3) {
 ; RV32I-LABEL: or_uge_eq:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    sltu a0, a0, a1
-; RV32I-NEXT:    xor a2, a2, a3
-; RV32I-NEXT:    snez a1, a2
-; RV32I-NEXT:    and a0, a1, a0
-; RV32I-NEXT:    bnez a0, .LBB23_2
+; RV32I-NEXT:    bgeu a0, a1, .LBB23_3
 ; RV32I-NEXT:  # %bb.1:
-; RV32I-NEXT:    ret
-; RV32I-NEXT:  .LBB23_2:
+; RV32I-NEXT:    beq a2, a3, .LBB23_3
+; RV32I-NEXT:  # %bb.2:
 ; RV32I-NEXT:    tail bar@plt
+; RV32I-NEXT:  .LBB23_3:
+; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: or_uge_eq:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sltu a0, a0, a1
-; RV64I-NEXT:    xor a2, a2, a3
-; RV64I-NEXT:    snez a1, a2
-; RV64I-NEXT:    and a0, a1, a0
-; RV64I-NEXT:    bnez a0, .LBB23_2
+; RV64I-NEXT:    bgeu a0, a1, .LBB23_3
 ; RV64I-NEXT:  # %bb.1:
-; RV64I-NEXT:    ret
-; RV64I-NEXT:  .LBB23_2:
+; RV64I-NEXT:    beq a2, a3, .LBB23_3
+; RV64I-NEXT:  # %bb.2:
 ; RV64I-NEXT:    tail bar@plt
+; RV64I-NEXT:  .LBB23_3:
+; RV64I-NEXT:    ret
   %5 = icmp uge i32 %0, %1
   %6 = icmp eq i32 %2, %3
   %7 = or i1 %5, %6
@@ -705,27 +661,23 @@ define void @or_uge_eq(i32 signext %0, i32 signext %1, i32 signext %2, i32 signe
 define void @or_ule_eq(i32 signext %0, i32 signext %1, i32 signext %2, i32 signext %3) {
 ; RV32I-LABEL: or_ule_eq:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    sltu a0, a1, a0
-; RV32I-NEXT:    xor a2, a2, a3
-; RV32I-NEXT:    snez a1, a2
-; RV32I-NEXT:    and a0, a1, a0
-; RV32I-NEXT:    bnez a0, .LBB24_2
+; RV32I-NEXT:    bgeu a1, a0, .LBB24_3
 ; RV32I-NEXT:  # %bb.1:
-; RV32I-NEXT:    ret
-; RV32I-NEXT:  .LBB24_2:
+; RV32I-NEXT:    beq a2, a3, .LBB24_3
+; RV32I-NEXT:  # %bb.2:
 ; RV32I-NEXT:    tail bar@plt
+; RV32I-NEXT:  .LBB24_3:
+; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: or_ule_eq:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sltu a0, a1, a0
-; RV64I-NEXT:    xor a2, a2, a3
-; RV64I-NEXT:    snez a1, a2
-; RV64I-NEXT:    and a0, a1, a0
-; RV64I-NEXT:    bnez a0, .LBB24_2
+; RV64I-NEXT:    bgeu a1, a0, .LBB24_3
 ; RV64I-NEXT:  # %bb.1:
-; RV64I-NEXT:    ret
-; RV64I-NEXT:  .LBB24_2:
+; RV64I-NEXT:    beq a2, a3, .LBB24_3
+; RV64I-NEXT:  # %bb.2:
 ; RV64I-NEXT:    tail bar@plt
+; RV64I-NEXT:  .LBB24_3:
+; RV64I-NEXT:    ret
   %5 = icmp ule i32 %0, %1
   %6 = icmp eq i32 %2, %3
   %7 = or i1 %5, %6
@@ -742,27 +694,23 @@ define void @or_ule_eq(i32 signext %0, i32 signext %1, i32 signext %2, i32 signe
 define void @or_sge_ne(i32 signext %0, i32 signext %1, i32 signext %2, i32 signext %3) {
 ; RV32I-LABEL: or_sge_ne:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    slt a0, a0, a1
-; RV32I-NEXT:    xor a2, a2, a3
-; RV32I-NEXT:    seqz a1, a2
-; RV32I-NEXT:    and a0, a1, a0
-; RV32I-NEXT:    bnez a0, .LBB25_2
+; RV32I-NEXT:    bge a0, a1, .LBB25_3
 ; RV32I-NEXT:  # %bb.1:
-; RV32I-NEXT:    ret
-; RV32I-NEXT:  .LBB25_2:
+; RV32I-NEXT:    bne a2, a3, .LBB25_3
+; RV32I-NEXT:  # %bb.2:
 ; RV32I-NEXT:    tail bar@plt
+; RV32I-NEXT:  .LBB25_3:
+; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: or_sge_ne:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    slt a0, a0, a1
-; RV64I-NEXT:    xor a2, a2, a3
-; RV64I-NEXT:    seqz a1, a2
-; RV64I-NEXT:    and a0, a1, a0
-; RV64I-NEXT:    bnez a0, .LBB25_2
+; RV64I-NEXT:    bge a0, a1, .LBB25_3
 ; RV64I-NEXT:  # %bb.1:
-; RV64I-NEXT:    ret
-; RV64I-NEXT:  .LBB25_2:
+; RV64I-NEXT:    bne a2, a3, .LBB25_3
+; RV64I-NEXT:  # %bb.2:
 ; RV64I-NEXT:    tail bar@plt
+; RV64I-NEXT:  .LBB25_3:
+; RV64I-NEXT:    ret
   %5 = icmp sge i32 %0, %1
   %6 = icmp ne i32 %2, %3
   %7 = or i1 %5, %6
@@ -779,27 +727,23 @@ define void @or_sge_ne(i32 signext %0, i32 signext %1, i32 signext %2, i32 signe
 define void @or_sle_ne(i32 signext %0, i32 signext %1, i32 signext %2, i32 signext %3) {
 ; RV32I-LABEL: or_sle_ne:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    slt a0, a1, a0
-; RV32I-NEXT:    xor a2, a2, a3
-; RV32I-NEXT:    seqz a1, a2
-; RV32I-NEXT:    and a0, a1, a0
-; RV32I-NEXT:    bnez a0, .LBB26_2
+; RV32I-NEXT:    bge a1, a0, .LBB26_3
 ; RV32I-NEXT:  # %bb.1:
-; RV32I-NEXT:    ret
-; RV32I-NEXT:  .LBB26_2:
+; RV32I-NEXT:    bne a2, a3, .LBB26_3
+; RV32I-NEXT:  # %bb.2:
 ; RV32I-NEXT:    tail bar@plt
+; RV32I-NEXT:  .LBB26_3:
+; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: or_sle_ne:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    slt a0, a1, a0
-; RV64I-NEXT:    xor a2, a2, a3
-; RV64I-NEXT:    seqz a1, a2
-; RV64I-NEXT:    and a0, a1, a0
-; RV64I-NEXT:    bnez a0, .LBB26_2
+; RV64I-NEXT:    bge a1, a0, .LBB26_3
 ; RV64I-NEXT:  # %bb.1:
-; RV64I-NEXT:    ret
-; RV64I-NEXT:  .LBB26_2:
+; RV64I-NEXT:    bne a2, a3, .LBB26_3
+; RV64I-NEXT:  # %bb.2:
 ; RV64I-NEXT:    tail bar@plt
+; RV64I-NEXT:  .LBB26_3:
+; RV64I-NEXT:    ret
   %5 = icmp sle i32 %0, %1
   %6 = icmp ne i32 %2, %3
   %7 = or i1 %5, %6
@@ -816,27 +760,23 @@ define void @or_sle_ne(i32 signext %0, i32 signext %1, i32 signext %2, i32 signe
 define void @or_uge_ne(i32 signext %0, i32 signext %1, i32 signext %2, i32 signext %3) {
 ; RV32I-LABEL: or_uge_ne:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    sltu a0, a0, a1
-; RV32I-NEXT:    xor a2, a2, a3
-; RV32I-NEXT:    seqz a1, a2
-; RV32I-NEXT:    and a0, a1, a0
-; RV32I-NEXT:    bnez a0, .LBB27_2
+; RV32I-NEXT:    bgeu a0, a1, .LBB27_3
 ; RV32I-NEXT:  # %bb.1:
-; RV32I-NEXT:    ret
-; RV32I-NEXT:  .LBB27_2:
+; RV32I-NEXT:    bne a2, a3, .LBB27_3
+; RV32I-NEXT:  # %bb.2:
 ; RV32I-NEXT:    tail bar@plt
+; RV32I-NEXT:  .LBB27_3:
+; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: or_uge_ne:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sltu a0, a0, a1
-; RV64I-NEXT:    xor a2, a2, a3
-; RV64I-NEXT:    seqz a1, a2
-; RV64I-NEXT:    and a0, a1, a0
-; RV64I-NEXT:    bnez a0, .LBB27_2
+; RV64I-NEXT:    bgeu a0, a1, .LBB27_3
 ; RV64I-NEXT:  # %bb.1:
-; RV64I-NEXT:    ret
-; RV64I-NEXT:  .LBB27_2:
+; RV64I-NEXT:    bne a2, a3, .LBB27_3
+; RV64I-NEXT:  # %bb.2:
 ; RV64I-NEXT:    tail bar@plt
+; RV64I-NEXT:  .LBB27_3:
+; RV64I-NEXT:    ret
   %5 = icmp uge i32 %0, %1
   %6 = icmp ne i32 %2, %3
   %7 = or i1 %5, %6
@@ -853,27 +793,23 @@ define void @or_uge_ne(i32 signext %0, i32 signext %1, i32 signext %2, i32 signe
 define void @or_ule_ne(i32 signext %0, i32 signext %1, i32 signext %2, i32 signext %3) {
 ; RV32I-LABEL: or_ule_ne:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    sltu a0, a1, a0
-; RV32I-NEXT:    xor a2, a2, a3
-; RV32I-NEXT:    seqz a1, a2
-; RV32I-NEXT:    and a0, a1, a0
-; RV32I-NEXT:    bnez a0, .LBB28_2
+; RV32I-NEXT:    bgeu a1, a0, .LBB28_3
 ; RV32I-NEXT:  # %bb.1:
-; RV32I-NEXT:    ret
-; RV32I-NEXT:  .LBB28_2:
+; RV32I-NEXT:    bne a2, a3, .LBB28_3
+; RV32I-NEXT:  # %bb.2:
 ; RV32I-NEXT:    tail bar@plt
+; RV32I-NEXT:  .LBB28_3:
+; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: or_ule_ne:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sltu a0, a1, a0
-; RV64I-NEXT:    xor a2, a2, a3
-; RV64I-NEXT:    seqz a1, a2
-; RV64I-NEXT:    and a0, a1, a0
-; RV64I-NEXT:    bnez a0, .LBB28_2
+; RV64I-NEXT:    bgeu a1, a0, .LBB28_3
 ; RV64I-NEXT:  # %bb.1:
-; RV64I-NEXT:    ret
-; RV64I-NEXT:  .LBB28_2:
+; RV64I-NEXT:    bne a2, a3, .LBB28_3
+; RV64I-NEXT:  # %bb.2:
 ; RV64I-NEXT:    tail bar@plt
+; RV64I-NEXT:  .LBB28_3:
+; RV64I-NEXT:    ret
   %5 = icmp ule i32 %0, %1
   %6 = icmp ne i32 %2, %3
   %7 = or i1 %5, %6
@@ -890,26 +826,22 @@ define void @or_ule_ne(i32 signext %0, i32 signext %1, i32 signext %2, i32 signe
 define void @and_eq_sge(i32 signext %0, i32 signext %1, i32 signext %2, i32 signext %3) {
 ; RV32I-LABEL: and_eq_sge:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    xor a0, a0, a1
-; RV32I-NEXT:    snez a0, a0
-; RV32I-NEXT:    slt a1, a2, a3
-; RV32I-NEXT:    or a0, a0, a1
-; RV32I-NEXT:    bnez a0, .LBB29_2
+; RV32I-NEXT:    bne a0, a1, .LBB29_3
 ; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    blt a2, a3, .LBB29_3
+; RV32I-NEXT:  # %bb.2:
 ; RV32I-NEXT:    ret
-; RV32I-NEXT:  .LBB29_2:
+; RV32I-NEXT:  .LBB29_3:
 ; RV32I-NEXT:    tail bar@plt
 ;
 ; RV64I-LABEL: and_eq_sge:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    xor a0, a0, a1
-; RV64I-NEXT:    snez a0, a0
-; RV64I-NEXT:    slt a1, a2, a3
-; RV64I-NEXT:    or a0, a0, a1
-; RV64I-NEXT:    bnez a0, .LBB29_2
+; RV64I-NEXT:    bne a0, a1, .LBB29_3
 ; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    blt a2, a3, .LBB29_3
+; RV64I-NEXT:  # %bb.2:
 ; RV64I-NEXT:    ret
-; RV64I-NEXT:  .LBB29_2:
+; RV64I-NEXT:  .LBB29_3:
 ; RV64I-NEXT:    tail bar@plt
   %5 = icmp eq  i32 %0, %1
   %6 = icmp sge i32 %2, %3
@@ -927,26 +859,22 @@ define void @and_eq_sge(i32 signext %0, i32 signext %1, i32 signext %2, i32 sign
 define void @and_eq_sle(i32 signext %0, i32 signext %1, i32 signext %2, i32 signext %3) {
 ; RV32I-LABEL: and_eq_sle:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    xor a0, a0, a1
-; RV32I-NEXT:    snez a0, a0
-; RV32I-NEXT:    slt a1, a3, a2
-; RV32I-NEXT:    or a0, a0, a1
-; RV32I-NEXT:    bnez a0, .LBB30_2
+; RV32I-NEXT:    bne a0, a1, .LBB30_3
 ; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    blt a3, a2, .LBB30_3
+; RV32I-NEXT:  # %bb.2:
 ; RV32I-NEXT:    ret
-; RV32I-NEXT:  .LBB30_2:
+; RV32I-NEXT:  .LBB30_3:
 ; RV32I-NEXT:    tail bar@plt
 ;
 ; RV64I-LABEL: and_eq_sle:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    xor a0, a0, a1
-; RV64I-NEXT:    snez a0, a0
-; RV64I-NEXT:    slt a1, a3, a2
-; RV64I-NEXT:    or a0, a0, a1
-; RV64I-NEXT:    bnez a0, .LBB30_2
+; RV64I-NEXT:    bne a0, a1, .LBB30_3
 ; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    blt a3, a2, .LBB30_3
+; RV64I-NEXT:  # %bb.2:
 ; RV64I-NEXT:    ret
-; RV64I-NEXT:  .LBB30_2:
+; RV64I-NEXT:  .LBB30_3:
 ; RV64I-NEXT:    tail bar@plt
   %5 = icmp eq  i32 %0, %1
   %6 = icmp sle i32 %2, %3
@@ -964,26 +892,22 @@ define void @and_eq_sle(i32 signext %0, i32 signext %1, i32 signext %2, i32 sign
 define void @and_eq_uge(i32 signext %0, i32 signext %1, i32 signext %2, i32 signext %3) {
 ; RV32I-LABEL: and_eq_uge:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    xor a0, a0, a1
-; RV32I-NEXT:    snez a0, a0
-; RV32I-NEXT:    sltu a1, a2, a3
-; RV32I-NEXT:    or a0, a0, a1
-; RV32I-NEXT:    bnez a0, .LBB31_2
+; RV32I-NEXT:    bne a0, a1, .LBB31_3
 ; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    bltu a2, a3, .LBB31_3
+; RV32I-NEXT:  # %bb.2:
 ; RV32I-NEXT:    ret
-; RV32I-NEXT:  .LBB31_2:
+; RV32I-NEXT:  .LBB31_3:
 ; RV32I-NEXT:    tail bar@plt
 ;
 ; RV64I-LABEL: and_eq_uge:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    xor a0, a0, a1
-; RV64I-NEXT:    snez a0, a0
-; RV64I-NEXT:    sltu a1, a2, a3
-; RV64I-NEXT:    or a0, a0, a1
-; RV64I-NEXT:    bnez a0, .LBB31_2
+; RV64I-NEXT:    bne a0, a1, .LBB31_3
 ; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    bltu a2, a3, .LBB31_3
+; RV64I-NEXT:  # %bb.2:
 ; RV64I-NEXT:    ret
-; RV64I-NEXT:  .LBB31_2:
+; RV64I-NEXT:  .LBB31_3:
 ; RV64I-NEXT:    tail bar@plt
   %5 = icmp eq  i32 %0, %1
   %6 = icmp uge i32 %2, %3
@@ -1001,26 +925,22 @@ define void @and_eq_uge(i32 signext %0, i32 signext %1, i32 signext %2, i32 sign
 define void @and_eq_ule(i32 signext %0, i32 signext %1, i32 signext %2, i32 signext %3) {
 ; RV32I-LABEL: and_eq_ule:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    xor a0, a0, a1
-; RV32I-NEXT:    snez a0, a0
-; RV32I-NEXT:    sltu a1, a3, a2
-; RV32I-NEXT:    or a0, a0, a1
-; RV32I-NEXT:    bnez a0, .LBB32_2
+; RV32I-NEXT:    bne a0, a1, .LBB32_3
 ; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    bltu a3, a2, .LBB32_3
+; RV32I-NEXT:  # %bb.2:
 ; RV32I-NEXT:    ret
-; RV32I-NEXT:  .LBB32_2:
+; RV32I-NEXT:  .LBB32_3:
 ; RV32I-NEXT:    tail bar@plt
 ;
 ; RV64I-LABEL: and_eq_ule:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    xor a0, a0, a1
-; RV64I-NEXT:    snez a0, a0
-; RV64I-NEXT:    sltu a1, a3, a2
-; RV64I-NEXT:    or a0, a0, a1
-; RV64I-NEXT:    bnez a0, .LBB32_2
+; RV64I-NEXT:    bne a0, a1, .LBB32_3
 ; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    bltu a3, a2, .LBB32_3
+; RV64I-NEXT:  # %bb.2:
 ; RV64I-NEXT:    ret
-; RV64I-NEXT:  .LBB32_2:
+; RV64I-NEXT:  .LBB32_3:
 ; RV64I-NEXT:    tail bar@plt
   %5 = icmp eq  i32 %0, %1
   %6 = icmp ule i32 %2, %3
@@ -1038,26 +958,22 @@ define void @and_eq_ule(i32 signext %0, i32 signext %1, i32 signext %2, i32 sign
 define void @and_ne_sge(i32 signext %0, i32 signext %1, i32 signext %2, i32 signext %3) {
 ; RV32I-LABEL: and_ne_sge:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    xor a0, a0, a1
-; RV32I-NEXT:    seqz a0, a0
-; RV32I-NEXT:    slt a1, a2, a3
-; RV32I-NEXT:    or a0, a0, a1
-; RV32I-NEXT:    bnez a0, .LBB33_2
+; RV32I-NEXT:    beq a0, a1, .LBB33_3
 ; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    blt a2, a3, .LBB33_3
+; RV32I-NEXT:  # %bb.2:
 ; RV32I-NEXT:    ret
-; RV32I-NEXT:  .LBB33_2:
+; RV32I-NEXT:  .LBB33_3:
 ; RV32I-NEXT:    tail bar@plt
 ;
 ; RV64I-LABEL: and_ne_sge:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    xor a0, a0, a1
-; RV64I-NEXT:    seqz a0, a0
-; RV64I-NEXT:    slt a1, a2, a3
-; RV64I-NEXT:    or a0, a0, a1
-; RV64I-NEXT:    bnez a0, .LBB33_2
+; RV64I-NEXT:    beq a0, a1, .LBB33_3
 ; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    blt a2, a3, .LBB33_3
+; RV64I-NEXT:  # %bb.2:
 ; RV64I-NEXT:    ret
-; RV64I-NEXT:  .LBB33_2:
+; RV64I-NEXT:  .LBB33_3:
 ; RV64I-NEXT:    tail bar@plt
   %5 = icmp ne  i32 %0, %1
   %6 = icmp sge i32 %2, %3
@@ -1075,26 +991,22 @@ define void @and_ne_sge(i32 signext %0, i32 signext %1, i32 signext %2, i32 sign
 define void @and_ne_sle(i32 signext %0, i32 signext %1, i32 signext %2, i32 signext %3) {
 ; RV32I-LABEL: and_ne_sle:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    xor a0, a0, a1
-; RV32I-NEXT:    seqz a0, a0
-; RV32I-NEXT:    slt a1, a3, a2
-; RV32I-NEXT:    or a0, a0, a1
-; RV32I-NEXT:    bnez a0, .LBB34_2
+; RV32I-NEXT:    beq a0, a1, .LBB34_3
 ; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    blt a3, a2, .LBB34_3
+; RV32I-NEXT:  # %bb.2:
 ; RV32I-NEXT:    ret
-; RV32I-NEXT:  .LBB34_2:
+; RV32I-NEXT:  .LBB34_3:
 ; RV32I-NEXT:    tail bar@plt
 ;
 ; RV64I-LABEL: and_ne_sle:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    xor a0, a0, a1
-; RV64I-NEXT:    seqz a0, a0
-; RV64I-NEXT:    slt a1, a3, a2
-; RV64I-NEXT:    or a0, a0, a1
-; RV64I-NEXT:    bnez a0, .LBB34_2
+; RV64I-NEXT:    beq a0, a1, .LBB34_3
 ; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    blt a3, a2, .LBB34_3
+; RV64I-NEXT:  # %bb.2:
 ; RV64I-NEXT:    ret
-; RV64I-NEXT:  .LBB34_2:
+; RV64I-NEXT:  .LBB34_3:
 ; RV64I-NEXT:    tail bar@plt
   %5 = icmp ne  i32 %0, %1
   %6 = icmp sle i32 %2, %3
@@ -1112,26 +1024,22 @@ define void @and_ne_sle(i32 signext %0, i32 signext %1, i32 signext %2, i32 sign
 define void @and_ne_uge(i32 signext %0, i32 signext %1, i32 signext %2, i32 signext %3) {
 ; RV32I-LABEL: and_ne_uge:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    xor a0, a0, a1
-; RV32I-NEXT:    seqz a0, a0
-; RV32I-NEXT:    sltu a1, a2, a3
-; RV32I-NEXT:    or a0, a0, a1
-; RV32I-NEXT:    bnez a0, .LBB35_2
+; RV32I-NEXT:    beq a0, a1, .LBB35_3
 ; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    bltu a2, a3, .LBB35_3
+; RV32I-NEXT:  # %bb.2:
 ; RV32I-NEXT:    ret
-; RV32I-NEXT:  .LBB35_2:
+; RV32I-NEXT:  .LBB35_3:
 ; RV32I-NEXT:    tail bar@plt
 ;
 ; RV64I-LABEL: and_ne_uge:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    xor a0, a0, a1
-; RV64I-NEXT:    seqz a0, a0
-; RV64I-NEXT:    sltu a1, a2, a3
-; RV64I-NEXT:    or a0, a0, a1
-; RV64I-NEXT:    bnez a0, .LBB35_2
+; RV64I-NEXT:    beq a0, a1, .LBB35_3
 ; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    bltu a2, a3, .LBB35_3
+; RV64I-NEXT:  # %bb.2:
 ; RV64I-NEXT:    ret
-; RV64I-NEXT:  .LBB35_2:
+; RV64I-NEXT:  .LBB35_3:
 ; RV64I-NEXT:    tail bar@plt
   %5 = icmp ne  i32 %0, %1
   %6 = icmp uge i32 %2, %3
@@ -1149,26 +1057,22 @@ define void @and_ne_uge(i32 signext %0, i32 signext %1, i32 signext %2, i32 sign
 define void @and_ne_ule(i32 signext %0, i32 signext %1, i32 signext %2, i32 signext %3) {
 ; RV32I-LABEL: and_ne_ule:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    xor a0, a0, a1
-; RV32I-NEXT:    seqz a0, a0
-; RV32I-NEXT:    sltu a1, a3, a2
-; RV32I-NEXT:    or a0, a0, a1
-; RV32I-NEXT:    bnez a0, .LBB36_2
+; RV32I-NEXT:    beq a0, a1, .LBB36_3
 ; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    bltu a3, a2, .LBB36_3
+; RV32I-NEXT:  # %bb.2:
 ; RV32I-NEXT:    ret
-; RV32I-NEXT:  .LBB36_2:
+; RV32I-NEXT:  .LBB36_3:
 ; RV32I-NEXT:    tail bar@plt
 ;
 ; RV64I-LABEL: and_ne_ule:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    xor a0, a0, a1
-; RV64I-NEXT:    seqz a0, a0
-; RV64I-NEXT:    sltu a1, a3, a2
-; RV64I-NEXT:    or a0, a0, a1
-; RV64I-NEXT:    bnez a0, .LBB36_2
+; RV64I-NEXT:    beq a0, a1, .LBB36_3
 ; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    bltu a3, a2, .LBB36_3
+; RV64I-NEXT:  # %bb.2:
 ; RV64I-NEXT:    ret
-; RV64I-NEXT:  .LBB36_2:
+; RV64I-NEXT:  .LBB36_3:
 ; RV64I-NEXT:    tail bar@plt
   %5 = icmp ne  i32 %0, %1
   %6 = icmp ule i32 %2, %3
@@ -1186,24 +1090,22 @@ define void @and_ne_ule(i32 signext %0, i32 signext %1, i32 signext %2, i32 sign
 define void @and_sge_gt0(i32 signext %0, i32 signext %1, i32 signext %2) {
 ; RV32I-LABEL: and_sge_gt0:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    slt a0, a0, a1
-; RV32I-NEXT:    slti a1, a2, 1
-; RV32I-NEXT:    or a0, a1, a0
-; RV32I-NEXT:    bnez a0, .LBB37_2
+; RV32I-NEXT:    blt a0, a1, .LBB37_3
 ; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    blez a2, .LBB37_3
+; RV32I-NEXT:  # %bb.2:
 ; RV32I-NEXT:    ret
-; RV32I-NEXT:  .LBB37_2:
+; RV32I-NEXT:  .LBB37_3:
 ; RV32I-NEXT:    tail bar@plt
 ;
 ; RV64I-LABEL: and_sge_gt0:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    slt a0, a0, a1
-; RV64I-NEXT:    slti a1, a2, 1
-; RV64I-NEXT:    or a0, a1, a0
-; RV64I-NEXT:    bnez a0, .LBB37_2
+; RV64I-NEXT:    blt a0, a1, .LBB37_3
 ; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    blez a2, .LBB37_3
+; RV64I-NEXT:  # %bb.2:
 ; RV64I-NEXT:    ret
-; RV64I-NEXT:  .LBB37_2:
+; RV64I-NEXT:  .LBB37_3:
 ; RV64I-NEXT:    tail bar@plt
   %4 = icmp sge i32 %0, %1
   %5 = icmp sgt i32 %2, 0
@@ -1221,24 +1123,22 @@ define void @and_sge_gt0(i32 signext %0, i32 signext %1, i32 signext %2) {
 define void @and_sle_lt1(i32 signext %0, i32 signext %1, i32 signext %2) {
 ; RV32I-LABEL: and_sle_lt1:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    slt a0, a1, a0
-; RV32I-NEXT:    sgtz a1, a2
-; RV32I-NEXT:    or a0, a1, a0
-; RV32I-NEXT:    bnez a0, .LBB38_2
+; RV32I-NEXT:    blt a1, a0, .LBB38_3
 ; RV32I-NEXT:  # %bb.1:
+; RV32I-NEXT:    bgtz a2, .LBB38_3
+; RV32I-NEXT:  # %bb.2:
 ; RV32I-NEXT:    ret
-; RV32I-NEXT:  .LBB38_2:
+; RV32I-NEXT:  .LBB38_3:
 ; RV32I-NEXT:    tail bar@plt
 ;
 ; RV64I-LABEL: and_sle_lt1:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    slt a0, a1, a0
-; RV64I-NEXT:    sgtz a1, a2
-; RV64I-NEXT:    or a0, a1, a0
-; RV64I-NEXT:    bnez a0, .LBB38_2
+; RV64I-NEXT:    blt a1, a0, .LBB38_3
 ; RV64I-NEXT:  # %bb.1:
+; RV64I-NEXT:    bgtz a2, .LBB38_3
+; RV64I-NEXT:  # %bb.2:
 ; RV64I-NEXT:    ret
-; RV64I-NEXT:  .LBB38_2:
+; RV64I-NEXT:  .LBB38_3:
 ; RV64I-NEXT:    tail bar@plt
   %4 = icmp sle i32 %0, %1
   %5 = icmp slt i32 %2, 1
@@ -1256,25 +1156,23 @@ define void @and_sle_lt1(i32 signext %0, i32 signext %1, i32 signext %2) {
 define void @or_uge_gt0(i32 signext %0, i32 signext %1, i32 signext %2) {
 ; RV32I-LABEL: or_uge_gt0:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    sltu a0, a0, a1
-; RV32I-NEXT:    slti a1, a2, 1
-; RV32I-NEXT:    and a0, a1, a0
-; RV32I-NEXT:    bnez a0, .LBB39_2
+; RV32I-NEXT:    bgeu a0, a1, .LBB39_3
 ; RV32I-NEXT:  # %bb.1:
-; RV32I-NEXT:    ret
-; RV32I-NEXT:  .LBB39_2:
+; RV32I-NEXT:    bgtz a2, .LBB39_3
+; RV32I-NEXT:  # %bb.2:
 ; RV32I-NEXT:    tail bar@plt
+; RV32I-NEXT:  .LBB39_3:
+; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: or_uge_gt0:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sltu a0, a0, a1
-; RV64I-NEXT:    slti a1, a2, 1
-; RV64I-NEXT:    and a0, a1, a0
-; RV64I-NEXT:    bnez a0, .LBB39_2
+; RV64I-NEXT:    bgeu a0, a1, .LBB39_3
 ; RV64I-NEXT:  # %bb.1:
-; RV64I-NEXT:    ret
-; RV64I-NEXT:  .LBB39_2:
+; RV64I-NEXT:    bgtz a2, .LBB39_3
+; RV64I-NEXT:  # %bb.2:
 ; RV64I-NEXT:    tail bar@plt
+; RV64I-NEXT:  .LBB39_3:
+; RV64I-NEXT:    ret
   %4 = icmp uge i32 %0, %1
   %5 = icmp sgt i32 %2, 0
   %6 = or i1 %4, %5
@@ -1291,25 +1189,23 @@ define void @or_uge_gt0(i32 signext %0, i32 signext %1, i32 signext %2) {
 define void @or_ule_lt1(i32 signext %0, i32 signext %1, i32 signext %2) {
 ; RV32I-LABEL: or_ule_lt1:
 ; RV32I:       # %bb.0:
-; RV32I-NEXT:    sltu a0, a1, a0
-; RV32I-NEXT:    sgtz a1, a2
-; RV32I-NEXT:    and a0, a1, a0
-; RV32I-NEXT:    bnez a0, .LBB40_2
+; RV32I-NEXT:    bgeu a1, a0, .LBB40_3
 ; RV32I-NEXT:  # %bb.1:
-; RV32I-NEXT:    ret
-; RV32I-NEXT:  .LBB40_2:
+; RV32I-NEXT:    blez a2, .LBB40_3
+; RV32I-NEXT:  # %bb.2:
 ; RV32I-NEXT:    tail bar@plt
+; RV32I-NEXT:  .LBB40_3:
+; RV32I-NEXT:    ret
 ;
 ; RV64I-LABEL: or_ule_lt1:
 ; RV64I:       # %bb.0:
-; RV64I-NEXT:    sltu a0, a1, a0
-; RV64I-NEXT:    sgtz a1, a2
-; RV64I-NEXT:    and a0, a1, a0
-; RV64I-NEXT:    bnez a0, .LBB40_2
+; RV64I-NEXT:    bgeu a1, a0, .LBB40_3
 ; RV64I-NEXT:  # %bb.1:
-; RV64I-NEXT:    ret
-; RV64I-NEXT:  .LBB40_2:
+; RV64I-NEXT:    blez a2, .LBB40_3
+; RV64I-NEXT:  # %bb.2:
 ; RV64I-NEXT:    tail bar@plt
+; RV64I-NEXT:  .LBB40_3:
+; RV64I-NEXT:    ret
   %4 = icmp ule i32 %0, %1
   %5 = icmp slt i32 %2, 1
   %6 = or i1 %4, %5
diff --git a/llvm/test/CodeGen/RISCV/zext-with-load-is-free.ll b/llvm/test/CodeGen/RISCV/zext-with-load-is-free.ll
index 07b921e3d90f33db8b5361f6282cb68720cac900..771a72f8d55f0b83066bc2eb444919cd10c1f305 100644
--- a/llvm/test/CodeGen/RISCV/zext-with-load-is-free.ll
+++ b/llvm/test/CodeGen/RISCV/zext-with-load-is-free.ll
@@ -10,16 +10,19 @@ define dso_local i32 @test_zext_i8() nounwind {
 ; RV32I-LABEL: test_zext_i8:
 ; RV32I:       # %bb.0: # %entry
 ; RV32I-NEXT:    lui a0, %hi(bytes)
-; RV32I-NEXT:    addi a1, a0, %lo(bytes)
-; RV32I-NEXT:    lbu a0, %lo(bytes)(a0)
-; RV32I-NEXT:    lbu a1, 1(a1)
-; RV32I-NEXT:    xori a0, a0, 136
-; RV32I-NEXT:    xori a1, a1, 7
-; RV32I-NEXT:    or a0, a0, a1
-; RV32I-NEXT:    beqz a0, .LBB0_2
-; RV32I-NEXT:  # %bb.1: # %if.then
+; RV32I-NEXT:    lbu a1, %lo(bytes)(a0)
+; RV32I-NEXT:    li a2, 136
+; RV32I-NEXT:    bne a1, a2, .LBB0_3
+; RV32I-NEXT:  # %bb.1: # %entry
+; RV32I-NEXT:    addi a0, a0, %lo(bytes)
+; RV32I-NEXT:    lbu a0, 1(a0)
+; RV32I-NEXT:    li a1, 7
+; RV32I-NEXT:    bne a0, a1, .LBB0_3
+; RV32I-NEXT:  # %bb.2: # %if.end
+; RV32I-NEXT:    li a0, 0
+; RV32I-NEXT:    ret
+; RV32I-NEXT:  .LBB0_3: # %if.then
 ; RV32I-NEXT:    li a0, 1
-; RV32I-NEXT:  .LBB0_2: # %if.end
 ; RV32I-NEXT:    ret
 entry:
   %0 = load i8, ptr @bytes, align 1
@@ -42,18 +45,20 @@ define dso_local i32 @test_zext_i16() nounwind {
 ; RV32I-LABEL: test_zext_i16:
 ; RV32I:       # %bb.0: # %entry
 ; RV32I-NEXT:    lui a0, %hi(shorts)
-; RV32I-NEXT:    addi a1, a0, %lo(shorts)
-; RV32I-NEXT:    lhu a0, %lo(shorts)(a0)
-; RV32I-NEXT:    lhu a1, 2(a1)
+; RV32I-NEXT:    lhu a1, %lo(shorts)(a0)
 ; RV32I-NEXT:    lui a2, 16
 ; RV32I-NEXT:    addi a2, a2, -120
-; RV32I-NEXT:    xor a0, a0, a2
-; RV32I-NEXT:    xori a1, a1, 7
-; RV32I-NEXT:    or a0, a0, a1
-; RV32I-NEXT:    beqz a0, .LBB1_2
-; RV32I-NEXT:  # %bb.1: # %if.then
+; RV32I-NEXT:    bne a1, a2, .LBB1_3
+; RV32I-NEXT:  # %bb.1: # %entry
+; RV32I-NEXT:    addi a0, a0, %lo(shorts)
+; RV32I-NEXT:    lhu a0, 2(a0)
+; RV32I-NEXT:    li a1, 7
+; RV32I-NEXT:    bne a0, a1, .LBB1_3
+; RV32I-NEXT:  # %bb.2: # %if.end
+; RV32I-NEXT:    li a0, 0
+; RV32I-NEXT:    ret
+; RV32I-NEXT:  .LBB1_3: # %if.then
 ; RV32I-NEXT:    li a0, 1
-; RV32I-NEXT:  .LBB1_2: # %if.end
 ; RV32I-NEXT:    ret
 entry:
   %0 = load i16, ptr @shorts, align 2
diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-ops-i128.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-ops-i128.ll
index 0e8f04468022202c78efa04c2435f221f42f7c4f..41b0964be05e5641c62aaac746aa1b2ef316fb07 100644
--- a/llvm/test/CodeGen/SystemZ/atomicrmw-ops-i128.ll
+++ b/llvm/test/CodeGen/SystemZ/atomicrmw-ops-i128.ll
@@ -14,13 +14,11 @@ define i128 @atomicrmw_xchg(ptr %src, i128 %b) {
 ; CHECK-NEXT:    stmg %r12, %r15, 96(%r15)
 ; CHECK-NEXT:    .cfi_offset %r12, -64
 ; CHECK-NEXT:    .cfi_offset %r13, -56
-; CHECK-NEXT:    .cfi_offset %r14, -48
 ; CHECK-NEXT:    .cfi_offset %r15, -40
-; CHECK-NEXT:    lg %r14, 8(%r4)
+; CHECK-NEXT:    lg %r1, 8(%r4)
 ; CHECK-NEXT:    lg %r0, 0(%r4)
 ; CHECK-NEXT:    lg %r4, 8(%r3)
 ; CHECK-NEXT:    lg %r5, 0(%r3)
-; CHECK-NEXT:    lgr %r1, %r14
 ; CHECK-NEXT:  .LBB0_1: # %atomicrmw.start
 ; CHECK-NEXT:    # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:    lgr %r12, %r5
diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-xchg-07.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-xchg-07.ll
index b9e29599af7ee78fad8f720827533bce5c9c6027..18fa89e6ca6cb8af6eaca02880d0cc3430b204fe 100644
--- a/llvm/test/CodeGen/SystemZ/atomicrmw-xchg-07.ll
+++ b/llvm/test/CodeGen/SystemZ/atomicrmw-xchg-07.ll
@@ -4,11 +4,10 @@
 
 define void @f1(ptr align 16 %ret, ptr align 16 %src, ptr align 16 %b) {
 ; CHECK-LABEL: f1:
-; CHECK:       lg      %r14, 8(%r4)
+; CHECK:       lg      %r1, 8(%r4)
 ; CHECK-NEXT:  lg      %r0, 0(%r4)
 ; CHECK-NEXT:  lg      %r4, 8(%r3)
 ; CHECK-NEXT:  lg      %r5, 0(%r3)
-; CHECK-NEXT:  lgr     %r1, %r14
 ; CHECK-NEXT:.LBB0_1:                          # %atomicrmw.start
 ; CHECK-NEXT:                                  # =>This Inner Loop Header: Depth=1
 ; CHECK-NEXT:  lgr     %r12, %r5
diff --git a/llvm/test/CodeGen/X86/broadcast-elm-cross-splat-vec.ll b/llvm/test/CodeGen/X86/broadcast-elm-cross-splat-vec.ll
index 13f1451bbc8b027b86cc4e57d77236c551bc1554..9125f2492ebb803923b0b5d8673586a103799ea2 100644
--- a/llvm/test/CodeGen/X86/broadcast-elm-cross-splat-vec.ll
+++ b/llvm/test/CodeGen/X86/broadcast-elm-cross-splat-vec.ll
@@ -1459,17 +1459,16 @@ define <8 x i64> @f8xi64_i128(<8 x i64> %a) {
 ;
 ; AVX-64-LABEL: f8xi64_i128:
 ; AVX-64:       # %bb.0:
-; AVX-64-NEXT:    vextractf128 $1, %ymm1, %xmm2
-; AVX-64-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0]
-; AVX-64-NEXT:    vpaddq %xmm3, %xmm2, %xmm2
-; AVX-64-NEXT:    vpaddq %xmm3, %xmm1, %xmm1
-; AVX-64-NEXT:    vinsertf128 $1, %xmm2, %ymm1, %ymm1
-; AVX-64-NEXT:    vextractf128 $1, %ymm0, %xmm2
-; AVX-64-NEXT:    vpaddq %xmm3, %xmm2, %xmm2
-; AVX-64-NEXT:    vpaddq %xmm3, %xmm0, %xmm0
-; AVX-64-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; AVX-64-NEXT:    vbroadcastf128 {{.*#+}} ymm2 = [0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0]
+; AVX-64-NEXT:    vbroadcastf128 {{.*#+}} ymm2 = [0,1,0,1]
 ; AVX-64-NEXT:    # ymm2 = mem[0,1,0,1]
+; AVX-64-NEXT:    vextractf128 $1, %ymm1, %xmm3
+; AVX-64-NEXT:    vpaddq %xmm2, %xmm3, %xmm3
+; AVX-64-NEXT:    vpaddq %xmm2, %xmm1, %xmm1
+; AVX-64-NEXT:    vinsertf128 $1, %xmm3, %ymm1, %ymm1
+; AVX-64-NEXT:    vextractf128 $1, %ymm0, %xmm3
+; AVX-64-NEXT:    vpaddq %xmm2, %xmm3, %xmm3
+; AVX-64-NEXT:    vpaddq %xmm2, %xmm0, %xmm0
+; AVX-64-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm0
 ; AVX-64-NEXT:    vandps %ymm2, %ymm0, %ymm0
 ; AVX-64-NEXT:    vandps %ymm2, %ymm1, %ymm1
 ; AVX-64-NEXT:    retq
@@ -1955,13 +1954,6 @@ define <8 x double> @f8xf64_f128(<8 x double> %a) {
 }
 
 
-; AVX512:       .LCPI37
-; AVX512-NEXT:  .quad	0x4010000000000000      # double 4
-; AVX512-NEXT:  .quad	0x3ff0000000000000      # double 1
-; AVX512-NEXT:  .quad	0x4000000000000000      # double 2
-; AVX512-NEXT:  .quad	0x4008000000000000      # double 3
-; AVX512-NOT:   .quad
-
 define <8 x double> @f8xf64_f256(<8 x double> %a) {
 ; AVX-LABEL: f8xf64_f256:
 ; AVX:       # %bb.0:
diff --git a/llvm/test/CodeGen/X86/coalescer-dead-flag-verifier-error.ll b/llvm/test/CodeGen/X86/coalescer-dead-flag-verifier-error.ll
new file mode 100644
index 0000000000000000000000000000000000000000..4a396599a4444dc1e510c17a330684b889e99723
--- /dev/null
+++ b/llvm/test/CodeGen/X86/coalescer-dead-flag-verifier-error.ll
@@ -0,0 +1,131 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
+; RUN: llc -mtriple=x86_64-pc-linux-gnu -verify-coalescing < %s | FileCheck %s
+
+%"class.llvm::APInt." = type <{ %union.anon., i32, [4 x i8] }>
+%union.anon. = type { i64 }
+
+define void @_ZNK4llvm5APInt21multiplicativeInverseERKS0_(ptr %r) {
+; CHECK-LABEL: _ZNK4llvm5APInt21multiplicativeInverseERKS0_:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    xorl %ecx, %ecx
+; CHECK-NEXT:    jmp .LBB0_1
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB0_4: # %_ZNK4llvm5APInt13getActiveBitsEv.exit.i.i
+; CHECK-NEXT:    # in Loop: Header=BB0_1 Depth=1
+; CHECK-NEXT:    movl %edx, %edx
+; CHECK-NEXT:    shlq $4, %rdx
+; CHECK-NEXT:    movl $0, (%rdi,%rdx)
+; CHECK-NEXT:    movl %ecx, %edx
+; CHECK-NEXT:  .LBB0_1: # %bb
+; CHECK-NEXT:    # =>This Loop Header: Depth=1
+; CHECK-NEXT:    # Child Loop BB0_3 Depth 2
+; CHECK-NEXT:    xorl $1, %ecx
+; CHECK-NEXT:    xorl %esi, %esi
+; CHECK-NEXT:    movq %rcx, %r8
+; CHECK-NEXT:    testb %al, %al
+; CHECK-NEXT:    jne .LBB0_4
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB0_3: # %for.body.i.i.i.i.i.3
+; CHECK-NEXT:    # Parent Loop BB0_1 Depth=1
+; CHECK-NEXT:    # => This Inner Loop Header: Depth=2
+; CHECK-NEXT:    orq $1, %r8
+; CHECK-NEXT:    orq $1, %rsi
+; CHECK-NEXT:    testb %al, %al
+; CHECK-NEXT:    je .LBB0_3
+; CHECK-NEXT:    jmp .LBB0_4
+entry:
+  br label %bb
+
+bb:                                               ; preds = %_ZNK4llvm5APInt13getActiveBitsEv.exit.i.i, %entry
+  %i.0 = phi i32 [ 0, %entry ], [ %xor, %_ZNK4llvm5APInt13getActiveBitsEv.exit.i.i ]
+  %xor = xor i32 %i.0, 1
+  %idxprom = zext nneg i32 %xor to i64
+  br label %for.body.i.i.i.i.i
+
+for.body.i.i.i.i.i:                               ; preds = %for.body.i.i.i.i.i.3, %bb
+  %lsr.iv37 = phi i64 [ %lsr.iv.next38, %for.body.i.i.i.i.i.3 ], [ 0, %bb ]
+  %lsr.iv = phi i64 [ %lsr.iv.next, %for.body.i.i.i.i.i.3 ], [ %idxprom, %bb ]
+  %exitcond.not.i.i.i.i.i.2 = icmp eq i64 0, 1
+  br i1 %exitcond.not.i.i.i.i.i.2, label %_ZNK4llvm5APInt13getActiveBitsEv.exit.i.i, label %for.body.i.i.i.i.i.3
+
+for.body.i.i.i.i.i.3:                             ; preds = %for.body.i.i.i.i.i
+  %sunkaddr55 = mul i64 %lsr.iv37, 0
+  %i = xor i64 %lsr.iv, 0
+  %lsr.iv.next = or i64 %lsr.iv, 1
+  %lsr.iv.next38 = or i64 %lsr.iv37, 1
+  br label %for.body.i.i.i.i.i
+
+_ZNK4llvm5APInt13getActiveBitsEv.exit.i.i:        ; preds = %for.body.i.i.i.i.i
+  %idxprom12 = zext nneg i32 %i.0 to i64
+  %arrayidx13 = getelementptr [2 x %"class.llvm::APInt."], ptr %r, i64 0, i64 %idxprom12
+  store i32 0, ptr %arrayidx13, align 4
+  br label %bb
+}
+
+; This variant hit an assert and never reached the verifier error
+define void @_ZNK4llvm5APInt21multiplicativeInverseERKS0__assert(ptr %r) {
+; CHECK-LABEL: _ZNK4llvm5APInt21multiplicativeInverseERKS0__assert:
+; CHECK:       # %bb.0: # %entry
+; CHECK-NEXT:    xorl %eax, %eax
+; CHECK-NEXT:    xorl %edx, %edx
+; CHECK-NEXT:    xorl %ecx, %ecx
+; CHECK-NEXT:    jmp .LBB1_1
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB1_4: # %_ZNK4llvm5APInt13getActiveBitsEv.exit.i.i
+; CHECK-NEXT:    # in Loop: Header=BB1_1 Depth=1
+; CHECK-NEXT:    movl %edx, %edx
+; CHECK-NEXT:    shlq $4, %rdx
+; CHECK-NEXT:    movl $0, (%rdi,%rdx)
+; CHECK-NEXT:    movl %ecx, %edx
+; CHECK-NEXT:  .LBB1_1: # %bb
+; CHECK-NEXT:    # =>This Loop Header: Depth=1
+; CHECK-NEXT:    # Child Loop BB1_3 Depth 2
+; CHECK-NEXT:    xorl $1, %ecx
+; CHECK-NEXT:    movq %rcx, %rsi
+; CHECK-NEXT:    shlq $4, %rsi
+; CHECK-NEXT:    movq (%rsi), %rsi
+; CHECK-NEXT:    xorl %r8d, %r8d
+; CHECK-NEXT:    testb %al, %al
+; CHECK-NEXT:    jne .LBB1_4
+; CHECK-NEXT:    .p2align 4, 0x90
+; CHECK-NEXT:  .LBB1_3: # %for.body.i.i.i.i.i.3
+; CHECK-NEXT:    # Parent Loop BB1_1 Depth=1
+; CHECK-NEXT:    # => This Inner Loop Header: Depth=2
+; CHECK-NEXT:    orq $1, %rsi
+; CHECK-NEXT:    orq $1, %r8
+; CHECK-NEXT:    testb %al, %al
+; CHECK-NEXT:    je .LBB1_3
+; CHECK-NEXT:    jmp .LBB1_4
+entry:
+  br label %bb
+
+bb:                                               ; preds = %_ZNK4llvm5APInt13getActiveBitsEv.exit.i.i, %entry
+  %i.0 = phi i32 [ 0, %entry ], [ %xor, %_ZNK4llvm5APInt13getActiveBitsEv.exit.i.i ]
+  %xor = xor i32 %i.0, 1
+  %idxprom = zext nneg i32 %xor to i64
+  %arrayidx = getelementptr [2 x %"class.llvm::APInt."], ptr null, i64 0, i64 %idxprom
+  %i.i.i.i.i.i = load ptr, ptr %arrayidx, align 16
+  %i.i.i.i.i.i36 = ptrtoint ptr %i.i.i.i.i.i to i64
+  br label %for.body.i.i.i.i.i
+
+for.body.i.i.i.i.i:                               ; preds = %for.body.i.i.i.i.i.3, %bb
+  %lsr.iv37 = phi i64 [ %lsr.iv.next38, %for.body.i.i.i.i.i.3 ], [ 0, %bb ]
+  %lsr.iv = phi i64 [ %lsr.iv.next, %for.body.i.i.i.i.i.3 ], [ %i.i.i.i.i.i36, %bb ]
+  %exitcond.not.i.i.i.i.i.2 = icmp eq i64 0, 1
+  br i1 %exitcond.not.i.i.i.i.i.2, label %_ZNK4llvm5APInt13getActiveBitsEv.exit.i.i, label %for.body.i.i.i.i.i.3
+
+for.body.i.i.i.i.i.3:                             ; preds = %for.body.i.i.i.i.i
+  %sunkaddr55 = mul i64 %lsr.iv37, 0
+  %i = xor i64 %lsr.iv, 0
+  %lsr.iv.next = or i64 %lsr.iv, 1
+  %lsr.iv.next38 = or i64 %lsr.iv37, 1
+  br label %for.body.i.i.i.i.i
+
+_ZNK4llvm5APInt13getActiveBitsEv.exit.i.i:        ; preds = %for.body.i.i.i.i.i
+  %idxprom12 = zext nneg i32 %i.0 to i64
+  %arrayidx13 = getelementptr [2 x %"class.llvm::APInt."], ptr %r, i64 0, i64 %idxprom12
+  store i32 0, ptr %arrayidx13, align 4
+  br label %bb
+}
diff --git a/llvm/test/CodeGen/X86/coalescer-partial-redundancy-clear-dead-flag-undef-copy.mir b/llvm/test/CodeGen/X86/coalescer-partial-redundancy-clear-dead-flag-undef-copy.mir
new file mode 100644
index 0000000000000000000000000000000000000000..5f33be0bc1555963b4cc45a5eac4a0215ccc32da
--- /dev/null
+++ b/llvm/test/CodeGen/X86/coalescer-partial-redundancy-clear-dead-flag-undef-copy.mir
@@ -0,0 +1,47 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 4
+# RUN: llc -mtriple=x86_64-pc-linux-gnu -run-pass=register-coalescer -verify-coalescing -o - %s | FileCheck %s
+
+# Check for "Live range continues after dead def flag".
+
+# There are 2 copies of undef, but the registers also appear to be
+# live due to block live outs, and thus were not deleted as
+# eliminateUndefCopy only considered the live range, and not the undef
+# flag.
+#
+# removePartialRedundancy would move the COPY undef %0 in bb.1 to
+# bb.0.  The live range of %1 would then be extended to be live out of
+# %bb.1 for the backedge phi. This would then fail the verifier, since
+# the dead flag was no longer valid. This was fixed by directly
+# considering the undef flag to avoid considering this special case.
+
+---
+name: partial_redundancy_coalesce_undef_copy_live_out
+tracksRegLiveness: true
+body:             |
+  ; CHECK-LABEL: name: partial_redundancy_coalesce_undef_copy_live_out
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT:   liveins: $rdi
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gr32 = COPY $rdi
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   dead [[XOR32ri:%[0-9]+]]:gr32 = XOR32ri undef [[XOR32ri]], 1, implicit-def dead $eflags
+  ; CHECK-NEXT:   dead [[MOV32rr:%[0-9]+]]:gr32 = MOV32rr [[COPY]]
+  ; CHECK-NEXT:   [[COPY:%[0-9]+]]:gr32 = IMPLICIT_DEF
+  ; CHECK-NEXT:   JMP_1 %bb.1
+  bb.0:
+    liveins: $rdi
+
+    %0:gr32 = COPY $rdi
+
+  bb.1:
+    %1:gr32 = COPY undef %0
+    dead %1:gr32 = XOR32ri %1, 1, implicit-def dead $eflags
+    dead %2:gr32 = MOV32rr killed %0
+    %0:gr32 = COPY killed undef %1
+    JMP_1 %bb.1
+
+...
diff --git a/llvm/test/CodeGen/X86/coalescer-remat-with-undef-implicit-def-operand.mir b/llvm/test/CodeGen/X86/coalescer-remat-with-undef-implicit-def-operand.mir
new file mode 100644
index 0000000000000000000000000000000000000000..42d17130412b6121ba1577e02dc0212fd8736a89
--- /dev/null
+++ b/llvm/test/CodeGen/X86/coalescer-remat-with-undef-implicit-def-operand.mir
@@ -0,0 +1,123 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 4
+# RUN: llc -mtriple=x86_64-pc-linux-gnu -verify-coalescing -run-pass=register-coalescer -o - %s | FileCheck %s
+
+# The %1 = MOV32r0 is rematerialized as a subregister of %2. The
+# implicit-def %1 operand needs to have an undef added, just like the
+# main result operand.
+
+---
+name:           remat_into_subregister_set_undef_implicit_operand_subregisters
+tracksRegLiveness: true
+body:             |
+  ; CHECK-LABEL: name: remat_into_subregister_set_undef_implicit_operand_subregisters
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT:   liveins: $rdi
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   undef [[MOV32r0_:%[0-9]+]].sub_32bit:gr64_with_sub_8bit = MOV32r0 implicit-def dead $eflags, implicit-def [[MOV32r0_]]
+  ; CHECK-NEXT:   [[MOV32r0_1:%[0-9]+]]:gr32 = MOV32r0 implicit-def dead $eflags, implicit-def [[MOV32r0_1]]
+  ; CHECK-NEXT:   undef [[MOV32r0_2:%[0-9]+]].sub_32bit:gr64_with_sub_8bit = MOV32r0 implicit-def dead $eflags, implicit-def undef [[MOV32r0_2]].sub_32bit, implicit-def [[MOV32r0_2]]
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.2(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[MOV32r0_2:%[0-9]+]].sub_32bit:gr64_with_sub_8bit = XOR32ri [[MOV32r0_2]].sub_32bit, 1, implicit-def dead $eflags
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   successors: %bb.4(0x40000000), %bb.3(0x40000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   JCC_1 %bb.4, 5, implicit killed undef $eflags
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.3:
+  ; CHECK-NEXT:   successors: %bb.4(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.4:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   dead [[MOV32rr:%[0-9]+]]:gr32 = MOV32rr [[MOV32r0_1]]
+  ; CHECK-NEXT:   dead [[SHL64ri:%[0-9]+]]:gr64_nosp = SHL64ri [[MOV32r0_]], 4, implicit-def dead $eflags
+  ; CHECK-NEXT:   [[MOV32r0_1:%[0-9]+]]:gr32 = COPY [[MOV32r0_2]].sub_32bit
+  ; CHECK-NEXT:   JMP_1 %bb.1
+  bb.0:
+    liveins: $rdi
+
+    undef %0.sub_32bit:gr64_with_sub_8bit = MOV32r0 implicit-def dead $eflags, implicit-def %0
+    %1:gr32 = MOV32r0 implicit-def dead $eflags, implicit-def %1
+    undef %2.sub_32bit:gr64_with_sub_8bit = COPY %1, implicit-def %2
+
+  bb.1:
+    %2.sub_32bit:gr64_with_sub_8bit = XOR32ri %2.sub_32bit, 1, implicit-def dead $eflags
+
+  bb.2:
+    JCC_1 %bb.4, 5, implicit killed undef $eflags
+
+  bb.3:
+
+  bb.4:
+    dead %3:gr32 = MOV32rr %1
+    dead %4:gr64_nosp = SHL64ri %0, 4, implicit-def dead $eflags
+    %1:gr32 = COPY %2.sub_32bit
+    JMP_1 %bb.1
+
+...
+
+# Same, except the implicit-def on the original instruction already
+# has a subregister index.
+
+---
+name:           remat_into_subregister_set_undef_implicit_operand_subregisters_with_subreg
+tracksRegLiveness: true
+body:             |
+  ; CHECK-LABEL: name: remat_into_subregister_set_undef_implicit_operand_subregisters_with_subreg
+  ; CHECK: bb.0:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT:   liveins: $rdi
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   undef [[MOV32r0_:%[0-9]+]].sub_32bit:gr64_with_sub_8bit = MOV32r0 implicit-def dead $eflags, implicit-def [[MOV32r0_]]
+  ; CHECK-NEXT:   [[MOV32r0_1:%[0-9]+]]:gr32 = MOV32r0 implicit-def dead $eflags, implicit-def undef [[MOV32r0_1]].sub_8bit
+  ; CHECK-NEXT:   undef [[MOV32r0_2:%[0-9]+]].sub_32bit:gr64_with_sub_8bit = MOV32r0 implicit-def dead $eflags, implicit-def undef [[MOV32r0_2]].sub_8bit, implicit-def [[MOV32r0_2]]
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.1:
+  ; CHECK-NEXT:   successors: %bb.2(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   [[MOV32r0_2:%[0-9]+]].sub_32bit:gr64_with_sub_8bit = XOR32ri [[MOV32r0_2]].sub_32bit, 1, implicit-def dead $eflags
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.2:
+  ; CHECK-NEXT:   successors: %bb.4(0x40000000), %bb.3(0x40000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   JCC_1 %bb.4, 5, implicit killed undef $eflags
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.3:
+  ; CHECK-NEXT:   successors: %bb.4(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT: bb.4:
+  ; CHECK-NEXT:   successors: %bb.1(0x80000000)
+  ; CHECK-NEXT: {{  $}}
+  ; CHECK-NEXT:   dead [[MOV32rr:%[0-9]+]]:gr32 = MOV32rr [[MOV32r0_1]]
+  ; CHECK-NEXT:   dead [[SHL64ri:%[0-9]+]]:gr64_nosp = SHL64ri [[MOV32r0_]], 4, implicit-def dead $eflags
+  ; CHECK-NEXT:   [[MOV32r0_1:%[0-9]+]]:gr32 = COPY [[MOV32r0_2]].sub_32bit
+  ; CHECK-NEXT:   JMP_1 %bb.1
+  bb.0:
+    liveins: $rdi
+
+    undef %0.sub_32bit:gr64_with_sub_8bit = MOV32r0 implicit-def dead $eflags, implicit-def %0
+    %1:gr32 = MOV32r0 implicit-def dead $eflags, undef implicit-def %1.sub_8bit
+    undef %2.sub_32bit:gr64_with_sub_8bit = COPY %1, implicit-def %2
+
+  bb.1:
+    %2.sub_32bit:gr64_with_sub_8bit = XOR32ri %2.sub_32bit, 1, implicit-def dead $eflags
+
+  bb.2:
+    JCC_1 %bb.4, 5, implicit killed undef $eflags
+
+  bb.3:
+
+  bb.4:
+    dead %3:gr32 = MOV32rr %1
+    dead %4:gr64_nosp = SHL64ri %0, 4, implicit-def dead $eflags
+    %1:gr32 = COPY %2.sub_32bit
+    JMP_1 %bb.1
+
+...
diff --git a/llvm/test/CodeGen/X86/code-model-elf-sections.ll b/llvm/test/CodeGen/X86/code-model-elf-sections.ll
index 749d5b6bf904e5ea7e8d6a39ecb214ba1dee74d3..cb19f0d34f59f276f24ed95677c4f37642d4549c 100644
--- a/llvm/test/CodeGen/X86/code-model-elf-sections.ll
+++ b/llvm/test/CodeGen/X86/code-model-elf-sections.ll
@@ -21,16 +21,16 @@
 ; SMALL: .data {{.*}} WA {{.*}}
 ; SMALL: .data.x {{.*}} WA {{.*}}
 ; SMALL: .data0 {{.*}} WA {{.*}}
-; SMALL: .ldata {{.*}} WAl {{.*}}
-; SMALL: .ldata.x {{.*}} WAl {{.*}}
+; SMALL: .ldata {{.*}} WA {{.*}}
+; SMALL: .ldata.x {{.*}} WA {{.*}}
 ; SMALL: .ldata0 {{.*}} WA {{.*}}
 ; SMALL: force_small {{.*}} WA {{.*}}
 ; SMALL: force_large {{.*}} WAl {{.*}}
 ; SMALL: foo {{.*}} WA {{.*}}
 ; SMALL: .bss {{.*}} WA {{.*}}
-; SMALL: .lbss {{.*}} WAl {{.*}}
+; SMALL: .lbss {{.*}} WA {{.*}}
 ; SMALL: .rodata {{.*}} A {{.*}}
-; SMALL: .lrodata {{.*}} Al {{.*}}
+; SMALL: .lrodata {{.*}} A {{.*}}
 ; SMALL: .data.rel.ro {{.*}} WA {{.*}}
 ; SMALL: .tbss {{.*}} WAT {{.*}}
 ; SMALL: .tdata {{.*}} WAT {{.*}}
@@ -38,16 +38,16 @@
 ; SMALL-DS: .data {{.*}} WA {{.*}}
 ; SMALL-DS: .data.x {{.*}} WA {{.*}}
 ; SMALL-DS: .data0 {{.*}} WA {{.*}}
-; SMALL-DS: .ldata {{.*}} WAl {{.*}}
-; SMALL-DS: .ldata.x {{.*}} WAl {{.*}}
+; SMALL-DS: .ldata {{.*}} WA {{.*}}
+; SMALL-DS: .ldata.x {{.*}} WA {{.*}}
 ; SMALL-DS: .ldata0 {{.*}} WA {{.*}}
 ; SMALL-DS: .data.data {{.*}} WA {{.*}}
 ; SMALL-DS: force_small {{.*}} WA {{.*}}
 ; SMALL-DS: force_large {{.*}} WAl {{.*}}
 ; SMALL-DS: foo {{.*}} WA {{.*}}
-; SMALL-DS: .lbss {{.*}} WAl {{.*}}
+; SMALL-DS: .lbss {{.*}} WA {{.*}}
 ; SMALL-DS: .bss.bss {{.*}} WA {{.*}}
-; SMALL-DS: .lrodata {{.*}} Al {{.*}}
+; SMALL-DS: .lrodata {{.*}} A {{.*}}
 ; SMALL-DS: .rodata.rodata {{.*}} A {{.*}}
 ; SMALL-DS: .data.rel.ro.relro {{.*}} WA {{.*}}
 ; SMALL-DS: .tbss.tbss {{.*}} WAT {{.*}}
diff --git a/llvm/test/CodeGen/X86/code-model-elf.ll b/llvm/test/CodeGen/X86/code-model-elf.ll
index 06989b0ffd22cd2150190749fc99aafe450b8048..a43736e2a5884ad25dc775c42cd90b66ed86b2de 100644
--- a/llvm/test/CodeGen/X86/code-model-elf.ll
+++ b/llvm/test/CodeGen/X86/code-model-elf.ll
@@ -9,7 +9,7 @@
 ; RUN: llc -verify-machineinstrs < %s -relocation-model=pic    -code-model=medium -large-data-threshold=1000 | FileCheck %s --check-prefix=CHECK --check-prefix=MEDIUM-SMALL-DATA-PIC
 ; RUN: llc -verify-machineinstrs < %s -relocation-model=pic    -code-model=medium | FileCheck %s --check-prefix=CHECK --check-prefix=MEDIUM-PIC
 ; RUN: llc -verify-machineinstrs < %s -relocation-model=pic    -code-model=large  | FileCheck %s --check-prefix=CHECK --check-prefix=LARGE-PIC
-; RUN: llc -verify-machineinstrs < %s -relocation-model=pic    -code-model=large  -large-data-threshold=1000 | FileCheck %s --check-prefix=CHECK --check-prefix=LARGE-PIC
+; RUN: llc -verify-machineinstrs < %s -relocation-model=pic    -code-model=large  -large-data-threshold=1000 | FileCheck %s --check-prefix=CHECK --check-prefix=LARGE-SMALL-DATA-PIC
 
 ; Generated from this C source:
 ;
@@ -40,6 +40,7 @@ target triple = "x86_64--linux"
 
 @global_data = dso_local global [10 x i32] [i32 1, i32 2, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0], align 16
 @static_data = internal global [10 x i32] zeroinitializer, align 16
+@static_data_alias = internal constant ptr getelementptr inbounds ([10 x i32], ptr @static_data, i64 0, i64 2), align 8
 @extern_data = external global [10 x i32], align 16
 @thread_data = external thread_local global i32, align 4
 @unknown_size_data = dso_local global [0 x i32] zeroinitializer, align 16
@@ -89,9 +90,74 @@ define dso_local ptr @lea_static_data() #0 {
 ; LARGE-PIC-NEXT:    movabsq $static_data@GOTOFF, %rax
 ; LARGE-PIC-NEXT:    addq %rcx, %rax
 ; LARGE-PIC-NEXT:    retq
+;
+; LARGE-SMALL-DATA-PIC-LABEL: lea_static_data:
+; LARGE-SMALL-DATA-PIC:       # %bb.0:
+; LARGE-SMALL-DATA-PIC-NEXT:  .L0$pb:
+; LARGE-SMALL-DATA-PIC-NEXT:    leaq .L0$pb(%rip), %rax
+; LARGE-SMALL-DATA-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L0$pb, %rcx
+; LARGE-SMALL-DATA-PIC-NEXT:    addq %rax, %rcx
+; LARGE-SMALL-DATA-PIC-NEXT:    movl $static_data@GOTOFF, %eax
+; LARGE-SMALL-DATA-PIC-NEXT:    addq %rcx, %rax
+; LARGE-SMALL-DATA-PIC-NEXT:    retq
   ret ptr @static_data
 }
 
+define dso_local ptr @lea_static_data_alias() #0 {
+; SMALL-STATIC-LABEL: lea_static_data_alias:
+; SMALL-STATIC:       # %bb.0:
+; SMALL-STATIC-NEXT:    movl $static_data_alias, %eax
+; SMALL-STATIC-NEXT:    retq
+;
+; MEDIUM-STATIC-LABEL: lea_static_data_alias:
+; MEDIUM-STATIC:       # %bb.0:
+; MEDIUM-STATIC-NEXT:    movabsq $static_data_alias, %rax
+; MEDIUM-STATIC-NEXT:    retq
+;
+; LARGE-STATIC-LABEL: lea_static_data_alias:
+; LARGE-STATIC:       # %bb.0:
+; LARGE-STATIC-NEXT:    movabsq $static_data_alias, %rax
+; LARGE-STATIC-NEXT:    retq
+;
+; SMALL-PIC-LABEL: lea_static_data_alias:
+; SMALL-PIC:       # %bb.0:
+; SMALL-PIC-NEXT:    leaq static_data_alias(%rip), %rax
+; SMALL-PIC-NEXT:    retq
+;
+; MEDIUM-SMALL-DATA-PIC-LABEL: lea_static_data_alias:
+; MEDIUM-SMALL-DATA-PIC:       # %bb.0:
+; MEDIUM-SMALL-DATA-PIC-NEXT:    leaq static_data_alias(%rip), %rax
+; MEDIUM-SMALL-DATA-PIC-NEXT:    retq
+;
+; MEDIUM-PIC-LABEL: lea_static_data_alias:
+; MEDIUM-PIC:       # %bb.0:
+; MEDIUM-PIC-NEXT:    leaq _GLOBAL_OFFSET_TABLE_(%rip), %rcx
+; MEDIUM-PIC-NEXT:    movabsq $static_data_alias@GOTOFF, %rax
+; MEDIUM-PIC-NEXT:    addq %rcx, %rax
+; MEDIUM-PIC-NEXT:    retq
+;
+; LARGE-PIC-LABEL: lea_static_data_alias:
+; LARGE-PIC:       # %bb.0:
+; LARGE-PIC-NEXT:  .L1$pb:
+; LARGE-PIC-NEXT:    leaq .L1$pb(%rip), %rax
+; LARGE-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L1$pb, %rcx
+; LARGE-PIC-NEXT:    addq %rax, %rcx
+; LARGE-PIC-NEXT:    movabsq $static_data_alias@GOTOFF, %rax
+; LARGE-PIC-NEXT:    addq %rcx, %rax
+; LARGE-PIC-NEXT:    retq
+;
+; LARGE-SMALL-DATA-PIC-LABEL: lea_static_data_alias:
+; LARGE-SMALL-DATA-PIC:       # %bb.0:
+; LARGE-SMALL-DATA-PIC-NEXT:  .L1$pb:
+; LARGE-SMALL-DATA-PIC-NEXT:    leaq .L1$pb(%rip), %rax
+; LARGE-SMALL-DATA-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L1$pb, %rcx
+; LARGE-SMALL-DATA-PIC-NEXT:    addq %rax, %rcx
+; LARGE-SMALL-DATA-PIC-NEXT:    movl $static_data_alias@GOTOFF, %eax
+; LARGE-SMALL-DATA-PIC-NEXT:    addq %rcx, %rax
+; LARGE-SMALL-DATA-PIC-NEXT:    retq
+  ret ptr @static_data_alias
+}
+
 define dso_local ptr @lea_global_data() #0 {
 ; SMALL-STATIC-LABEL: lea_global_data:
 ; SMALL-STATIC:       # %bb.0:
@@ -127,13 +193,23 @@ define dso_local ptr @lea_global_data() #0 {
 ;
 ; LARGE-PIC-LABEL: lea_global_data:
 ; LARGE-PIC:       # %bb.0:
-; LARGE-PIC-NEXT:  .L1$pb:
-; LARGE-PIC-NEXT:    leaq .L1$pb(%rip), %rax
-; LARGE-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L1$pb, %rcx
+; LARGE-PIC-NEXT:  .L2$pb:
+; LARGE-PIC-NEXT:    leaq .L2$pb(%rip), %rax
+; LARGE-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L2$pb, %rcx
 ; LARGE-PIC-NEXT:    addq %rax, %rcx
 ; LARGE-PIC-NEXT:    movabsq $global_data@GOTOFF, %rax
 ; LARGE-PIC-NEXT:    addq %rcx, %rax
 ; LARGE-PIC-NEXT:    retq
+;
+; LARGE-SMALL-DATA-PIC-LABEL: lea_global_data:
+; LARGE-SMALL-DATA-PIC:       # %bb.0:
+; LARGE-SMALL-DATA-PIC-NEXT:  .L2$pb:
+; LARGE-SMALL-DATA-PIC-NEXT:    leaq .L2$pb(%rip), %rax
+; LARGE-SMALL-DATA-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L2$pb, %rcx
+; LARGE-SMALL-DATA-PIC-NEXT:    addq %rax, %rcx
+; LARGE-SMALL-DATA-PIC-NEXT:    movl $global_data@GOTOFF, %eax
+; LARGE-SMALL-DATA-PIC-NEXT:    addq %rcx, %rax
+; LARGE-SMALL-DATA-PIC-NEXT:    retq
   ret ptr @global_data
 }
 
@@ -170,13 +246,23 @@ define dso_local ptr @lea_extern_data() #0 {
 ;
 ; LARGE-PIC-LABEL: lea_extern_data:
 ; LARGE-PIC:       # %bb.0:
-; LARGE-PIC-NEXT:  .L2$pb:
-; LARGE-PIC-NEXT:    leaq .L2$pb(%rip), %rax
-; LARGE-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L2$pb, %rcx
+; LARGE-PIC-NEXT:  .L3$pb:
+; LARGE-PIC-NEXT:    leaq .L3$pb(%rip), %rax
+; LARGE-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L3$pb, %rcx
 ; LARGE-PIC-NEXT:    addq %rax, %rcx
 ; LARGE-PIC-NEXT:    movabsq $extern_data@GOT, %rax
 ; LARGE-PIC-NEXT:    movq (%rcx,%rax), %rax
 ; LARGE-PIC-NEXT:    retq
+;
+; LARGE-SMALL-DATA-PIC-LABEL: lea_extern_data:
+; LARGE-SMALL-DATA-PIC:       # %bb.0:
+; LARGE-SMALL-DATA-PIC-NEXT:  .L3$pb:
+; LARGE-SMALL-DATA-PIC-NEXT:    leaq .L3$pb(%rip), %rax
+; LARGE-SMALL-DATA-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L3$pb, %rcx
+; LARGE-SMALL-DATA-PIC-NEXT:    addq %rax, %rcx
+; LARGE-SMALL-DATA-PIC-NEXT:    movl $extern_data@GOT, %eax
+; LARGE-SMALL-DATA-PIC-NEXT:    movq (%rcx,%rax), %rax
+; LARGE-SMALL-DATA-PIC-NEXT:    retq
   ret ptr @extern_data
 }
 
@@ -217,17 +303,26 @@ define dso_local ptr @lea_unknown_size_data() #0 {
 ;
 ; LARGE-PIC-LABEL: lea_unknown_size_data:
 ; LARGE-PIC:       # %bb.0:
-; LARGE-PIC-NEXT:  .L3$pb:
-; LARGE-PIC-NEXT:    leaq .L3$pb(%rip), %rax
-; LARGE-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L3$pb, %rcx
+; LARGE-PIC-NEXT:  .L4$pb:
+; LARGE-PIC-NEXT:    leaq .L4$pb(%rip), %rax
+; LARGE-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L4$pb, %rcx
 ; LARGE-PIC-NEXT:    addq %rax, %rcx
 ; LARGE-PIC-NEXT:    movabsq $unknown_size_data@GOTOFF, %rax
 ; LARGE-PIC-NEXT:    addq %rcx, %rax
 ; LARGE-PIC-NEXT:    retq
+;
+; LARGE-SMALL-DATA-PIC-LABEL: lea_unknown_size_data:
+; LARGE-SMALL-DATA-PIC:       # %bb.0:
+; LARGE-SMALL-DATA-PIC-NEXT:  .L4$pb:
+; LARGE-SMALL-DATA-PIC-NEXT:    leaq .L4$pb(%rip), %rax
+; LARGE-SMALL-DATA-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L4$pb, %rcx
+; LARGE-SMALL-DATA-PIC-NEXT:    addq %rax, %rcx
+; LARGE-SMALL-DATA-PIC-NEXT:    movabsq $unknown_size_data@GOTOFF, %rax
+; LARGE-SMALL-DATA-PIC-NEXT:    addq %rcx, %rax
+; LARGE-SMALL-DATA-PIC-NEXT:    retq
   ret ptr @unknown_size_data
 }
 
-; TODO: make nopic small and medium instruction sequence the same
 define dso_local ptr @lea_forced_small_data() #0 {
 ; SMALL-STATIC-LABEL: lea_forced_small_data:
 ; SMALL-STATIC:       # %bb.0:
@@ -236,12 +331,12 @@ define dso_local ptr @lea_forced_small_data() #0 {
 ;
 ; MEDIUM-STATIC-LABEL: lea_forced_small_data:
 ; MEDIUM-STATIC:       # %bb.0:
-; MEDIUM-STATIC-NEXT:    movabsq $forced_small_data, %rax
+; MEDIUM-STATIC-NEXT:    movl $forced_small_data, %eax
 ; MEDIUM-STATIC-NEXT:    retq
 ;
 ; LARGE-STATIC-LABEL: lea_forced_small_data:
 ; LARGE-STATIC:       # %bb.0:
-; LARGE-STATIC-NEXT:    movabsq $forced_small_data, %rax
+; LARGE-STATIC-NEXT:    movl $forced_small_data, %eax
 ; LARGE-STATIC-NEXT:    retq
 ;
 ; SMALL-PIC-LABEL: lea_forced_small_data:
@@ -261,13 +356,23 @@ define dso_local ptr @lea_forced_small_data() #0 {
 ;
 ; LARGE-PIC-LABEL: lea_forced_small_data:
 ; LARGE-PIC:       # %bb.0:
-; LARGE-PIC-NEXT:  .L4$pb:
-; LARGE-PIC-NEXT:    leaq .L4$pb(%rip), %rax
-; LARGE-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L4$pb, %rcx
+; LARGE-PIC-NEXT:  .L5$pb:
+; LARGE-PIC-NEXT:    leaq .L5$pb(%rip), %rax
+; LARGE-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L5$pb, %rcx
 ; LARGE-PIC-NEXT:    addq %rax, %rcx
-; LARGE-PIC-NEXT:    movabsq $forced_small_data@GOTOFF, %rax
+; LARGE-PIC-NEXT:    movl $forced_small_data@GOTOFF, %eax
 ; LARGE-PIC-NEXT:    addq %rcx, %rax
 ; LARGE-PIC-NEXT:    retq
+;
+; LARGE-SMALL-DATA-PIC-LABEL: lea_forced_small_data:
+; LARGE-SMALL-DATA-PIC:       # %bb.0:
+; LARGE-SMALL-DATA-PIC-NEXT:  .L5$pb:
+; LARGE-SMALL-DATA-PIC-NEXT:    leaq .L5$pb(%rip), %rax
+; LARGE-SMALL-DATA-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L5$pb, %rcx
+; LARGE-SMALL-DATA-PIC-NEXT:    addq %rax, %rcx
+; LARGE-SMALL-DATA-PIC-NEXT:    movl $forced_small_data@GOTOFF, %eax
+; LARGE-SMALL-DATA-PIC-NEXT:    addq %rcx, %rax
+; LARGE-SMALL-DATA-PIC-NEXT:    retq
   ret ptr @forced_small_data
 }
 
@@ -280,13 +385,13 @@ define dso_local i32 @load_forced_small_data() #0 {
 ;
 ; MEDIUM-STATIC-LABEL: load_forced_small_data:
 ; MEDIUM-STATIC:       # %bb.0:
-; MEDIUM-STATIC-NEXT:    movabsq $forced_small_data, %rax
+; MEDIUM-STATIC-NEXT:    movl $forced_small_data, %eax
 ; MEDIUM-STATIC-NEXT:    movl 8(%rax), %eax
 ; MEDIUM-STATIC-NEXT:    retq
 ;
 ; LARGE-STATIC-LABEL: load_forced_small_data:
 ; LARGE-STATIC:       # %bb.0:
-; LARGE-STATIC-NEXT:    movabsq $forced_small_data, %rax
+; LARGE-STATIC-NEXT:    movl $forced_small_data, %eax
 ; LARGE-STATIC-NEXT:    movl 8(%rax), %eax
 ; LARGE-STATIC-NEXT:    retq
 ;
@@ -309,13 +414,23 @@ define dso_local i32 @load_forced_small_data() #0 {
 ;
 ; LARGE-PIC-LABEL: load_forced_small_data:
 ; LARGE-PIC:       # %bb.0:
-; LARGE-PIC-NEXT:  .L5$pb:
-; LARGE-PIC-NEXT:    leaq .L5$pb(%rip), %rax
-; LARGE-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L5$pb, %rcx
+; LARGE-PIC-NEXT:  .L6$pb:
+; LARGE-PIC-NEXT:    leaq .L6$pb(%rip), %rax
+; LARGE-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L6$pb, %rcx
 ; LARGE-PIC-NEXT:    addq %rax, %rcx
-; LARGE-PIC-NEXT:    movabsq $forced_small_data@GOTOFF, %rax
+; LARGE-PIC-NEXT:    movl $forced_small_data@GOTOFF, %eax
 ; LARGE-PIC-NEXT:    movl 8(%rcx,%rax), %eax
 ; LARGE-PIC-NEXT:    retq
+;
+; LARGE-SMALL-DATA-PIC-LABEL: load_forced_small_data:
+; LARGE-SMALL-DATA-PIC:       # %bb.0:
+; LARGE-SMALL-DATA-PIC-NEXT:  .L6$pb:
+; LARGE-SMALL-DATA-PIC-NEXT:    leaq .L6$pb(%rip), %rax
+; LARGE-SMALL-DATA-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L6$pb, %rcx
+; LARGE-SMALL-DATA-PIC-NEXT:    addq %rax, %rcx
+; LARGE-SMALL-DATA-PIC-NEXT:    movl $forced_small_data@GOTOFF, %eax
+; LARGE-SMALL-DATA-PIC-NEXT:    movl 8(%rcx,%rax), %eax
+; LARGE-SMALL-DATA-PIC-NEXT:    retq
   %rv = load i32, ptr getelementptr inbounds (i32, ptr @forced_small_data, i64 2)
   ret i32 %rv
 }
@@ -324,7 +439,7 @@ define dso_local i32 @load_forced_small_data() #0 {
 define dso_local ptr @lea_forced_large_data() #0 {
 ; SMALL-STATIC-LABEL: lea_forced_large_data:
 ; SMALL-STATIC:       # %bb.0:
-; SMALL-STATIC-NEXT:    movl $forced_large_data, %eax
+; SMALL-STATIC-NEXT:    movabsq $forced_large_data, %rax
 ; SMALL-STATIC-NEXT:    retq
 ;
 ; MEDIUM-STATIC-LABEL: lea_forced_large_data:
@@ -359,13 +474,23 @@ define dso_local ptr @lea_forced_large_data() #0 {
 ;
 ; LARGE-PIC-LABEL: lea_forced_large_data:
 ; LARGE-PIC:       # %bb.0:
-; LARGE-PIC-NEXT:  .L6$pb:
-; LARGE-PIC-NEXT:    leaq .L6$pb(%rip), %rax
-; LARGE-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L6$pb, %rcx
+; LARGE-PIC-NEXT:  .L7$pb:
+; LARGE-PIC-NEXT:    leaq .L7$pb(%rip), %rax
+; LARGE-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L7$pb, %rcx
 ; LARGE-PIC-NEXT:    addq %rax, %rcx
 ; LARGE-PIC-NEXT:    movabsq $forced_large_data@GOTOFF, %rax
 ; LARGE-PIC-NEXT:    addq %rcx, %rax
 ; LARGE-PIC-NEXT:    retq
+;
+; LARGE-SMALL-DATA-PIC-LABEL: lea_forced_large_data:
+; LARGE-SMALL-DATA-PIC:       # %bb.0:
+; LARGE-SMALL-DATA-PIC-NEXT:  .L7$pb:
+; LARGE-SMALL-DATA-PIC-NEXT:    leaq .L7$pb(%rip), %rax
+; LARGE-SMALL-DATA-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L7$pb, %rcx
+; LARGE-SMALL-DATA-PIC-NEXT:    addq %rax, %rcx
+; LARGE-SMALL-DATA-PIC-NEXT:    movabsq $forced_large_data@GOTOFF, %rax
+; LARGE-SMALL-DATA-PIC-NEXT:    addq %rcx, %rax
+; LARGE-SMALL-DATA-PIC-NEXT:    retq
   ret ptr @forced_large_data
 }
 
@@ -409,13 +534,23 @@ define dso_local i32 @load_forced_large_data() #0 {
 ;
 ; LARGE-PIC-LABEL: load_forced_large_data:
 ; LARGE-PIC:       # %bb.0:
-; LARGE-PIC-NEXT:  .L7$pb:
-; LARGE-PIC-NEXT:    leaq .L7$pb(%rip), %rax
-; LARGE-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L7$pb, %rcx
+; LARGE-PIC-NEXT:  .L8$pb:
+; LARGE-PIC-NEXT:    leaq .L8$pb(%rip), %rax
+; LARGE-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L8$pb, %rcx
 ; LARGE-PIC-NEXT:    addq %rax, %rcx
 ; LARGE-PIC-NEXT:    movabsq $forced_large_data@GOTOFF, %rax
 ; LARGE-PIC-NEXT:    movl 8(%rcx,%rax), %eax
 ; LARGE-PIC-NEXT:    retq
+;
+; LARGE-SMALL-DATA-PIC-LABEL: load_forced_large_data:
+; LARGE-SMALL-DATA-PIC:       # %bb.0:
+; LARGE-SMALL-DATA-PIC-NEXT:  .L8$pb:
+; LARGE-SMALL-DATA-PIC-NEXT:    leaq .L8$pb(%rip), %rax
+; LARGE-SMALL-DATA-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L8$pb, %rcx
+; LARGE-SMALL-DATA-PIC-NEXT:    addq %rax, %rcx
+; LARGE-SMALL-DATA-PIC-NEXT:    movabsq $forced_large_data@GOTOFF, %rax
+; LARGE-SMALL-DATA-PIC-NEXT:    movl 8(%rcx,%rax), %eax
+; LARGE-SMALL-DATA-PIC-NEXT:    retq
   %rv = load i32, ptr getelementptr inbounds (i32, ptr @forced_large_data, i64 2)
   ret i32 %rv
 }
@@ -458,13 +593,23 @@ define dso_local i32 @load_global_data() #0 {
 ;
 ; LARGE-PIC-LABEL: load_global_data:
 ; LARGE-PIC:       # %bb.0:
-; LARGE-PIC-NEXT:  .L8$pb:
-; LARGE-PIC-NEXT:    leaq .L8$pb(%rip), %rax
-; LARGE-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L8$pb, %rcx
+; LARGE-PIC-NEXT:  .L9$pb:
+; LARGE-PIC-NEXT:    leaq .L9$pb(%rip), %rax
+; LARGE-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L9$pb, %rcx
 ; LARGE-PIC-NEXT:    addq %rax, %rcx
 ; LARGE-PIC-NEXT:    movabsq $global_data@GOTOFF, %rax
 ; LARGE-PIC-NEXT:    movl 8(%rcx,%rax), %eax
 ; LARGE-PIC-NEXT:    retq
+;
+; LARGE-SMALL-DATA-PIC-LABEL: load_global_data:
+; LARGE-SMALL-DATA-PIC:       # %bb.0:
+; LARGE-SMALL-DATA-PIC-NEXT:  .L9$pb:
+; LARGE-SMALL-DATA-PIC-NEXT:    leaq .L9$pb(%rip), %rax
+; LARGE-SMALL-DATA-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L9$pb, %rcx
+; LARGE-SMALL-DATA-PIC-NEXT:    addq %rax, %rcx
+; LARGE-SMALL-DATA-PIC-NEXT:    movl $global_data@GOTOFF, %eax
+; LARGE-SMALL-DATA-PIC-NEXT:    movl 8(%rcx,%rax), %eax
+; LARGE-SMALL-DATA-PIC-NEXT:    retq
   %rv = load i32, ptr getelementptr inbounds ([10 x i32], ptr @global_data, i64 0, i64 2)
   ret i32 %rv
 }
@@ -508,14 +653,25 @@ define dso_local i32 @load_extern_data() #0 {
 ;
 ; LARGE-PIC-LABEL: load_extern_data:
 ; LARGE-PIC:       # %bb.0:
-; LARGE-PIC-NEXT:  .L9$pb:
-; LARGE-PIC-NEXT:    leaq .L9$pb(%rip), %rax
-; LARGE-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L9$pb, %rcx
+; LARGE-PIC-NEXT:  .L10$pb:
+; LARGE-PIC-NEXT:    leaq .L10$pb(%rip), %rax
+; LARGE-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L10$pb, %rcx
 ; LARGE-PIC-NEXT:    addq %rax, %rcx
 ; LARGE-PIC-NEXT:    movabsq $extern_data@GOT, %rax
 ; LARGE-PIC-NEXT:    movq (%rcx,%rax), %rax
 ; LARGE-PIC-NEXT:    movl 8(%rax), %eax
 ; LARGE-PIC-NEXT:    retq
+;
+; LARGE-SMALL-DATA-PIC-LABEL: load_extern_data:
+; LARGE-SMALL-DATA-PIC:       # %bb.0:
+; LARGE-SMALL-DATA-PIC-NEXT:  .L10$pb:
+; LARGE-SMALL-DATA-PIC-NEXT:    leaq .L10$pb(%rip), %rax
+; LARGE-SMALL-DATA-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L10$pb, %rcx
+; LARGE-SMALL-DATA-PIC-NEXT:    addq %rax, %rcx
+; LARGE-SMALL-DATA-PIC-NEXT:    movl $extern_data@GOT, %eax
+; LARGE-SMALL-DATA-PIC-NEXT:    movq (%rcx,%rax), %rax
+; LARGE-SMALL-DATA-PIC-NEXT:    movl 8(%rax), %eax
+; LARGE-SMALL-DATA-PIC-NEXT:    retq
   %rv = load i32, ptr getelementptr inbounds ([10 x i32], ptr @extern_data, i64 0, i64 2)
   ret i32 %rv
 }
@@ -559,13 +715,23 @@ define dso_local i32 @load_unknown_size_data() #0 {
 ;
 ; LARGE-PIC-LABEL: load_unknown_size_data:
 ; LARGE-PIC:       # %bb.0:
-; LARGE-PIC-NEXT:  .L10$pb:
-; LARGE-PIC-NEXT:    leaq .L10$pb(%rip), %rax
-; LARGE-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L10$pb, %rcx
+; LARGE-PIC-NEXT:  .L11$pb:
+; LARGE-PIC-NEXT:    leaq .L11$pb(%rip), %rax
+; LARGE-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L11$pb, %rcx
 ; LARGE-PIC-NEXT:    addq %rax, %rcx
 ; LARGE-PIC-NEXT:    movabsq $unknown_size_data@GOTOFF, %rax
 ; LARGE-PIC-NEXT:    movl 8(%rcx,%rax), %eax
 ; LARGE-PIC-NEXT:    retq
+;
+; LARGE-SMALL-DATA-PIC-LABEL: load_unknown_size_data:
+; LARGE-SMALL-DATA-PIC:       # %bb.0:
+; LARGE-SMALL-DATA-PIC-NEXT:  .L11$pb:
+; LARGE-SMALL-DATA-PIC-NEXT:    leaq .L11$pb(%rip), %rax
+; LARGE-SMALL-DATA-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L11$pb, %rcx
+; LARGE-SMALL-DATA-PIC-NEXT:    addq %rax, %rcx
+; LARGE-SMALL-DATA-PIC-NEXT:    movabsq $unknown_size_data@GOTOFF, %rax
+; LARGE-SMALL-DATA-PIC-NEXT:    movl 8(%rcx,%rax), %eax
+; LARGE-SMALL-DATA-PIC-NEXT:    retq
   %rv = load i32, ptr getelementptr inbounds (i32, ptr @unknown_size_data, i64 2)
   ret i32 %rv
 }
@@ -607,13 +773,23 @@ define dso_local ptr @lea_opaque() #0 {
 ;
 ; LARGE-PIC-LABEL: lea_opaque:
 ; LARGE-PIC:       # %bb.0:
-; LARGE-PIC-NEXT:  .L11$pb:
-; LARGE-PIC-NEXT:    leaq .L11$pb(%rip), %rax
-; LARGE-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L11$pb, %rcx
+; LARGE-PIC-NEXT:  .L12$pb:
+; LARGE-PIC-NEXT:    leaq .L12$pb(%rip), %rax
+; LARGE-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L12$pb, %rcx
 ; LARGE-PIC-NEXT:    addq %rax, %rcx
 ; LARGE-PIC-NEXT:    movabsq $opaque@GOTOFF, %rax
 ; LARGE-PIC-NEXT:    addq %rcx, %rax
 ; LARGE-PIC-NEXT:    retq
+;
+; LARGE-SMALL-DATA-PIC-LABEL: lea_opaque:
+; LARGE-SMALL-DATA-PIC:       # %bb.0:
+; LARGE-SMALL-DATA-PIC-NEXT:  .L12$pb:
+; LARGE-SMALL-DATA-PIC-NEXT:    leaq .L12$pb(%rip), %rax
+; LARGE-SMALL-DATA-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L12$pb, %rcx
+; LARGE-SMALL-DATA-PIC-NEXT:    addq %rax, %rcx
+; LARGE-SMALL-DATA-PIC-NEXT:    movabsq $opaque@GOTOFF, %rax
+; LARGE-SMALL-DATA-PIC-NEXT:    addq %rcx, %rax
+; LARGE-SMALL-DATA-PIC-NEXT:    retq
   ret ptr @opaque
 }
 
@@ -678,13 +854,23 @@ define dso_local ptr @lea_static_fn() #0 {
 ;
 ; LARGE-PIC-LABEL: lea_static_fn:
 ; LARGE-PIC:       # %bb.0:
-; LARGE-PIC-NEXT:  .L15$pb:
-; LARGE-PIC-NEXT:    leaq .L15$pb(%rip), %rax
-; LARGE-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L15$pb, %rcx
+; LARGE-PIC-NEXT:  .L16$pb:
+; LARGE-PIC-NEXT:    leaq .L16$pb(%rip), %rax
+; LARGE-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L16$pb, %rcx
 ; LARGE-PIC-NEXT:    addq %rax, %rcx
 ; LARGE-PIC-NEXT:    movabsq $static_fn@GOTOFF, %rax
 ; LARGE-PIC-NEXT:    addq %rcx, %rax
 ; LARGE-PIC-NEXT:    retq
+;
+; LARGE-SMALL-DATA-PIC-LABEL: lea_static_fn:
+; LARGE-SMALL-DATA-PIC:       # %bb.0:
+; LARGE-SMALL-DATA-PIC-NEXT:  .L16$pb:
+; LARGE-SMALL-DATA-PIC-NEXT:    leaq .L16$pb(%rip), %rax
+; LARGE-SMALL-DATA-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L16$pb, %rcx
+; LARGE-SMALL-DATA-PIC-NEXT:    addq %rax, %rcx
+; LARGE-SMALL-DATA-PIC-NEXT:    movabsq $static_fn@GOTOFF, %rax
+; LARGE-SMALL-DATA-PIC-NEXT:    addq %rcx, %rax
+; LARGE-SMALL-DATA-PIC-NEXT:    retq
   ret ptr @static_fn
 }
 
@@ -721,13 +907,23 @@ define dso_local ptr @lea_global_fn() #0 {
 ;
 ; LARGE-PIC-LABEL: lea_global_fn:
 ; LARGE-PIC:       # %bb.0:
-; LARGE-PIC-NEXT:  .L16$pb:
-; LARGE-PIC-NEXT:    leaq .L16$pb(%rip), %rax
-; LARGE-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L16$pb, %rcx
+; LARGE-PIC-NEXT:  .L17$pb:
+; LARGE-PIC-NEXT:    leaq .L17$pb(%rip), %rax
+; LARGE-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L17$pb, %rcx
 ; LARGE-PIC-NEXT:    addq %rax, %rcx
 ; LARGE-PIC-NEXT:    movabsq $global_fn@GOTOFF, %rax
 ; LARGE-PIC-NEXT:    addq %rcx, %rax
 ; LARGE-PIC-NEXT:    retq
+;
+; LARGE-SMALL-DATA-PIC-LABEL: lea_global_fn:
+; LARGE-SMALL-DATA-PIC:       # %bb.0:
+; LARGE-SMALL-DATA-PIC-NEXT:  .L17$pb:
+; LARGE-SMALL-DATA-PIC-NEXT:    leaq .L17$pb(%rip), %rax
+; LARGE-SMALL-DATA-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L17$pb, %rcx
+; LARGE-SMALL-DATA-PIC-NEXT:    addq %rax, %rcx
+; LARGE-SMALL-DATA-PIC-NEXT:    movabsq $global_fn@GOTOFF, %rax
+; LARGE-SMALL-DATA-PIC-NEXT:    addq %rcx, %rax
+; LARGE-SMALL-DATA-PIC-NEXT:    retq
   ret ptr @global_fn
 }
 
@@ -764,13 +960,23 @@ define dso_local ptr @lea_extern_fn() #0 {
 ;
 ; LARGE-PIC-LABEL: lea_extern_fn:
 ; LARGE-PIC:       # %bb.0:
-; LARGE-PIC-NEXT:  .L17$pb:
-; LARGE-PIC-NEXT:    leaq .L17$pb(%rip), %rax
-; LARGE-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L17$pb, %rcx
+; LARGE-PIC-NEXT:  .L18$pb:
+; LARGE-PIC-NEXT:    leaq .L18$pb(%rip), %rax
+; LARGE-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L18$pb, %rcx
 ; LARGE-PIC-NEXT:    addq %rax, %rcx
 ; LARGE-PIC-NEXT:    movabsq $extern_fn@GOT, %rax
 ; LARGE-PIC-NEXT:    movq (%rcx,%rax), %rax
 ; LARGE-PIC-NEXT:    retq
+;
+; LARGE-SMALL-DATA-PIC-LABEL: lea_extern_fn:
+; LARGE-SMALL-DATA-PIC:       # %bb.0:
+; LARGE-SMALL-DATA-PIC-NEXT:  .L18$pb:
+; LARGE-SMALL-DATA-PIC-NEXT:    leaq .L18$pb(%rip), %rax
+; LARGE-SMALL-DATA-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L18$pb, %rcx
+; LARGE-SMALL-DATA-PIC-NEXT:    addq %rax, %rcx
+; LARGE-SMALL-DATA-PIC-NEXT:    movabsq $extern_fn@GOT, %rax
+; LARGE-SMALL-DATA-PIC-NEXT:    movq (%rcx,%rax), %rax
+; LARGE-SMALL-DATA-PIC-NEXT:    retq
   ret ptr @extern_fn
 }
 
@@ -807,13 +1013,23 @@ define dso_local ptr @lea_ifunc() #0 {
 ;
 ; LARGE-PIC-LABEL: lea_ifunc:
 ; LARGE-PIC:       # %bb.0:
-; LARGE-PIC-NEXT:  .L18$pb:
-; LARGE-PIC-NEXT:    leaq .L18$pb(%rip), %rax
-; LARGE-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L18$pb, %rcx
+; LARGE-PIC-NEXT:  .L19$pb:
+; LARGE-PIC-NEXT:    leaq .L19$pb(%rip), %rax
+; LARGE-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L19$pb, %rcx
 ; LARGE-PIC-NEXT:    addq %rax, %rcx
 ; LARGE-PIC-NEXT:    movabsq $ifunc_func@GOT, %rax
 ; LARGE-PIC-NEXT:    movq (%rcx,%rax), %rax
 ; LARGE-PIC-NEXT:    retq
+;
+; LARGE-SMALL-DATA-PIC-LABEL: lea_ifunc:
+; LARGE-SMALL-DATA-PIC:       # %bb.0:
+; LARGE-SMALL-DATA-PIC-NEXT:  .L19$pb:
+; LARGE-SMALL-DATA-PIC-NEXT:    leaq .L19$pb(%rip), %rax
+; LARGE-SMALL-DATA-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L19$pb, %rcx
+; LARGE-SMALL-DATA-PIC-NEXT:    addq %rax, %rcx
+; LARGE-SMALL-DATA-PIC-NEXT:    movabsq $ifunc_func@GOT, %rax
+; LARGE-SMALL-DATA-PIC-NEXT:    movq (%rcx,%rax), %rax
+; LARGE-SMALL-DATA-PIC-NEXT:    retq
   ret ptr @ifunc_func
 }
 
@@ -825,7 +1041,7 @@ define dso_local ptr @lea_dso_local_ifunc() #0 {
 ;
 ; MEDIUM-STATIC-LABEL: lea_dso_local_ifunc:
 ; MEDIUM-STATIC:       # %bb.0:
-; MEDIUM-STATIC-NEXT:    movabsq $dso_local_ifunc_func, %rax
+; MEDIUM-STATIC-NEXT:    movl $dso_local_ifunc_func, %eax
 ; MEDIUM-STATIC-NEXT:    retq
 ;
 ; LARGE-STATIC-LABEL: lea_dso_local_ifunc:
@@ -850,13 +1066,23 @@ define dso_local ptr @lea_dso_local_ifunc() #0 {
 ;
 ; LARGE-PIC-LABEL: lea_dso_local_ifunc:
 ; LARGE-PIC:       # %bb.0:
-; LARGE-PIC-NEXT:  .L19$pb:
-; LARGE-PIC-NEXT:    leaq .L19$pb(%rip), %rax
-; LARGE-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L19$pb, %rcx
+; LARGE-PIC-NEXT:  .L20$pb:
+; LARGE-PIC-NEXT:    leaq .L20$pb(%rip), %rax
+; LARGE-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L20$pb, %rcx
 ; LARGE-PIC-NEXT:    addq %rax, %rcx
 ; LARGE-PIC-NEXT:    movabsq $dso_local_ifunc_func@GOTOFF, %rax
 ; LARGE-PIC-NEXT:    addq %rcx, %rax
 ; LARGE-PIC-NEXT:    retq
+;
+; LARGE-SMALL-DATA-PIC-LABEL: lea_dso_local_ifunc:
+; LARGE-SMALL-DATA-PIC:       # %bb.0:
+; LARGE-SMALL-DATA-PIC-NEXT:  .L20$pb:
+; LARGE-SMALL-DATA-PIC-NEXT:    leaq .L20$pb(%rip), %rax
+; LARGE-SMALL-DATA-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L20$pb, %rcx
+; LARGE-SMALL-DATA-PIC-NEXT:    addq %rax, %rcx
+; LARGE-SMALL-DATA-PIC-NEXT:    movabsq $dso_local_ifunc_func@GOTOFF, %rax
+; LARGE-SMALL-DATA-PIC-NEXT:    addq %rcx, %rax
+; LARGE-SMALL-DATA-PIC-NEXT:    retq
   ret ptr @dso_local_ifunc_func
 }
 
@@ -901,7 +1127,7 @@ define dso_local float @load_constant_pool(float %x) #0 {
 ;
 ; MEDIUM-STATIC-LABEL: load_constant_pool:
 ; MEDIUM-STATIC:       # %bb.0:
-; MEDIUM-STATIC-NEXT:    movabsq ${{\.?LCPI[0-9]+_[0-9]+}}, %rax
+; MEDIUM-STATIC-NEXT:    movl ${{\.?LCPI[0-9]+_[0-9]+}}, %eax
 ; MEDIUM-STATIC-NEXT:    addss (%rax), %xmm0
 ; MEDIUM-STATIC-NEXT:    retq
 ;
@@ -928,13 +1154,23 @@ define dso_local float @load_constant_pool(float %x) #0 {
 ;
 ; LARGE-PIC-LABEL: load_constant_pool:
 ; LARGE-PIC:       # %bb.0:
-; LARGE-PIC-NEXT:  .L21$pb:
-; LARGE-PIC-NEXT:    leaq .L21$pb(%rip), %rax
-; LARGE-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L21$pb, %rcx
+; LARGE-PIC-NEXT:  .L22$pb:
+; LARGE-PIC-NEXT:    leaq .L22$pb(%rip), %rax
+; LARGE-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L22$pb, %rcx
 ; LARGE-PIC-NEXT:    addq %rax, %rcx
 ; LARGE-PIC-NEXT:    movabsq ${{\.?LCPI[0-9]+_[0-9]+}}@GOTOFF, %rax
 ; LARGE-PIC-NEXT:    addss (%rcx,%rax), %xmm0
 ; LARGE-PIC-NEXT:    retq
+;
+; LARGE-SMALL-DATA-PIC-LABEL: load_constant_pool:
+; LARGE-SMALL-DATA-PIC:       # %bb.0:
+; LARGE-SMALL-DATA-PIC-NEXT:  .L22$pb:
+; LARGE-SMALL-DATA-PIC-NEXT:    leaq .L22$pb(%rip), %rax
+; LARGE-SMALL-DATA-PIC-NEXT:    movabsq $_GLOBAL_OFFSET_TABLE_-.L22$pb, %rcx
+; LARGE-SMALL-DATA-PIC-NEXT:    addq %rax, %rcx
+; LARGE-SMALL-DATA-PIC-NEXT:    movabsq ${{\.?LCPI[0-9]+_[0-9]+}}@GOTOFF, %rax
+; LARGE-SMALL-DATA-PIC-NEXT:    addss (%rcx,%rax), %xmm0
+; LARGE-SMALL-DATA-PIC-NEXT:    retq
   %a = fadd float %x, 1.0
   ret float %a
 }
diff --git a/llvm/test/CodeGen/X86/fast-isel-constpool.ll b/llvm/test/CodeGen/X86/fast-isel-constpool.ll
index 9f70fd55907377fcb9b865507735c92e2ac7d6b1..9e4cbb61308eaa24c2fffaec7f67a9aaf2079f2e 100644
--- a/llvm/test/CodeGen/X86/fast-isel-constpool.ll
+++ b/llvm/test/CodeGen/X86/fast-isel-constpool.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc -mtriple=x86_64-apple-darwin -fast-isel -code-model=small < %s | FileCheck %s
+; RUN: llc -mtriple=x86_64-apple-darwin -fast-isel -code-model=medium < %s | FileCheck %s
 ; RUN: llc -mtriple=x86_64-apple-darwin -fast-isel -code-model=large < %s | FileCheck %s --check-prefix=LARGE
 ; RUN: llc -mtriple=x86_64 -fast-isel -code-model=large -relocation-model=pic < %s | FileCheck %s --check-prefix=LARGE_PIC
 ; RUN: llc -mtriple=x86_64-apple-darwin -fast-isel -code-model=small -mattr=avx < %s | FileCheck %s --check-prefix=AVX
diff --git a/llvm/test/CodeGen/X86/fast-isel-large-object.ll b/llvm/test/CodeGen/X86/fast-isel-large-object.ll
new file mode 100644
index 0000000000000000000000000000000000000000..6ca2c42407237947e13c34835322e1630983de32
--- /dev/null
+++ b/llvm/test/CodeGen/X86/fast-isel-large-object.ll
@@ -0,0 +1,13 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
+; RUN: llc -fast-isel -mtriple=x86_64-unknown-unknown -relocation-model=pic < %s | FileCheck %s
+
+@g = external dso_local global i32, code_model "large"
+
+define ptr @f() {
+; CHECK-LABEL: f:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    leaq _GLOBAL_OFFSET_TABLE_(%rip), %rax
+; CHECK-NEXT:    leaq g@GOTOFF(%rax), %rax
+; CHECK-NEXT:    retq
+  ret ptr @g
+}
diff --git a/llvm/test/CodeGen/X86/fast-isel-medium-code-model.ll b/llvm/test/CodeGen/X86/fast-isel-medium-code-model.ll
new file mode 100644
index 0000000000000000000000000000000000000000..4aa230a209e285b2b014a9833a7d5cb4af8c1c36
--- /dev/null
+++ b/llvm/test/CodeGen/X86/fast-isel-medium-code-model.ll
@@ -0,0 +1,44 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=x86_64-linux-gnu -fast-isel -fast-isel-abort=3 -code-model=medium -large-data-threshold=5 < %s | FileCheck %s
+; RUN: llc -mtriple=x86_64-linux-gnu -fast-isel -code-model=medium -large-data-threshold=3 < %s -o /dev/null \
+; RUN:   -pass-remarks-output=- -pass-remarks-filter=sdagisel | FileCheck %s --check-prefix=FALLBACK --implicit-check-not=missed
+
+declare void @foo()
+
+define void @call_foo() {
+; CHECK-LABEL: call_foo:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    callq foo@PLT
+; CHECK-NEXT:    popq %rax
+; CHECK-NEXT:    .cfi_def_cfa_offset 8
+; CHECK-NEXT:    retq
+  call void @foo()
+  ret void
+}
+
+@g = internal global i32 42
+
+; FALLBACK: FastISel missed terminator
+; FALLBACK: in function: g_addr
+
+define ptr @g_addr() {
+; CHECK-LABEL: g_addr:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movabsq $g, %rax
+; CHECK-NEXT:    retq
+  ret ptr @g
+}
+
+; FALLBACK: FastISel missed
+; FALLBACK: in function: load_g
+
+define i32 @load_g() {
+; CHECK-LABEL: load_g:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl g, %eax
+; CHECK-NEXT:    retq
+  %i = load i32, ptr @g
+  ret i32 %i
+}
diff --git a/llvm/test/CodeGen/X86/splat-for-size.ll b/llvm/test/CodeGen/X86/splat-for-size.ll
index 6325f083d59bd1023d0241cba8739cf055d38a53..76a95e250457004dc42241975e3bd0c834e226a8 100644
--- a/llvm/test/CodeGen/X86/splat-for-size.ll
+++ b/llvm/test/CodeGen/X86/splat-for-size.ll
@@ -387,10 +387,9 @@ define <32 x i8> @splat_v32i8_pgso(<32 x i8> %x) !prof !14 {
 define <8 x i64> @pr23259() #1 {
 ; AVX-LABEL: pr23259:
 ; AVX:       # %bb.0: # %entry
-; AVX-NEXT:    vbroadcastsd {{.*#+}} ymm0 = [1,1,1,1]
-; AVX-NEXT:    vblendps {{.*#+}} xmm1 = mem[0,1],xmm0[2,3]
-; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
 ; AVX-NEXT:    vbroadcastsd {{.*#+}} ymm1 = [1,1,1,1]
+; AVX-NEXT:    vblendps {{.*#+}} xmm0 = mem[0,1],xmm1[2,3]
+; AVX-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],mem[4,5,6,7]
 ; AVX-NEXT:    retq
 ;
 ; AVX2-LABEL: pr23259:
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-3.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-3.ll
index 8aed554ff58f5404f066b08635bbd991b1170b0d..384e9cf36cf85d2e41a86afa0f9dfc95f73c8fb7 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-3.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-3.ll
@@ -603,9 +603,10 @@ define void @load_i16_stride3_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX2-ONLY-NEXT:    vpblendvb %ymm6, %ymm1, %ymm0, %ymm0
 ; AVX2-ONLY-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-ONLY-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1,2],ymm1[3],ymm0[4,5],ymm1[6],ymm0[7],ymm1[8],ymm0[9,10],ymm1[11],ymm0[12,13],ymm1[14],ymm0[15]
-; AVX2-ONLY-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15,20,21,26,27,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX2-ONLY-NEXT:    vpblendw {{.*#+}} xmm1 = xmm4[0],xmm3[1],xmm4[2,3],xmm3[4],xmm4[5,6],xmm3[7]
-; AVX2-ONLY-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[u,u,u,u,0,1,6,7,12,13,2,3,8,9,14,15]
+; AVX2-ONLY-NEXT:    vmovdqa {{.*#+}} ymm1 = <4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15,20,21,26,27,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX2-ONLY-NEXT:    vpshufb %ymm1, %ymm0, %ymm0
+; AVX2-ONLY-NEXT:    vpblendw {{.*#+}} xmm3 = xmm4[0],xmm3[1],xmm4[2,3],xmm3[4],xmm4[5,6],xmm3[7]
+; AVX2-ONLY-NEXT:    vpshufb %xmm1, %xmm3, %xmm1
 ; AVX2-ONLY-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
 ; AVX2-ONLY-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]
 ; AVX2-ONLY-NEXT:    vmovdqa %ymm2, (%rsi)
@@ -646,9 +647,10 @@ define void @load_i16_stride3_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512F-NEXT:    vpternlogq $202, %ymm2, %ymm1, %ymm0
 ; AVX512F-NEXT:    vmovdqa 16(%rdi), %xmm1
 ; AVX512F-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1,2],ymm1[3],ymm0[4,5],ymm1[6],ymm0[7],ymm1[8],ymm0[9,10],ymm1[11],ymm0[12,13],ymm1[14],ymm0[15]
-; AVX512F-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15,20,21,26,27,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512F-NEXT:    vpblendw {{.*#+}} xmm1 = xmm5[0],xmm4[1],xmm5[2,3],xmm4[4],xmm5[5,6],xmm4[7]
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[u,u,u,u,0,1,6,7,12,13,2,3,8,9,14,15]
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm1 = <4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15,20,21,26,27,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX512F-NEXT:    vpshufb %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT:    vpblendw {{.*#+}} xmm2 = xmm5[0],xmm4[1],xmm5[2,3],xmm4[4],xmm5[5,6],xmm4[7]
+; AVX512F-NEXT:    vpshufb %xmm1, %xmm2, %xmm1
 ; AVX512F-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
 ; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]
 ; AVX512F-NEXT:    vmovdqa %ymm3, (%rsi)
@@ -1111,22 +1113,22 @@ define void @load_i16_stride3_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ;
 ; AVX512F-LABEL: load_i16_stride3_vf32:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm0 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm1 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535]
 ; AVX512F-NEXT:    vmovdqa 128(%rdi), %ymm5
 ; AVX512F-NEXT:    vmovdqa 160(%rdi), %ymm6
-; AVX512F-NEXT:    vmovdqa %ymm0, %ymm1
-; AVX512F-NEXT:    vpternlogq $202, %ymm5, %ymm6, %ymm1
-; AVX512F-NEXT:    vpermq {{.*#+}} ymm2 = ymm1[2,3,0,1]
-; AVX512F-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm2[1],ymm1[2,3],ymm2[4],ymm1[5,6],ymm2[7],ymm1[8],ymm2[9],ymm1[10,11],ymm2[12],ymm1[13,14],ymm2[15]
-; AVX512F-NEXT:    vpshufb {{.*#+}} ymm3 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,4,5,10,11,16,17,22,23,28,29,18,19,24,25,30,31,20,21,26,27]
-; AVX512F-NEXT:    vmovdqa 112(%rdi), %xmm1
+; AVX512F-NEXT:    vmovdqa %ymm1, %ymm0
+; AVX512F-NEXT:    vpternlogq $202, %ymm5, %ymm6, %ymm0
+; AVX512F-NEXT:    vpermq {{.*#+}} ymm2 = ymm0[2,3,0,1]
+; AVX512F-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm2[1],ymm0[2,3],ymm2[4],ymm0[5,6],ymm2[7],ymm0[8],ymm2[9],ymm0[10,11],ymm2[12],ymm0[13,14],ymm2[15]
+; AVX512F-NEXT:    vpshufb {{.*#+}} ymm3 = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,4,5,10,11,16,17,22,23,28,29,18,19,24,25,30,31,20,21,26,27]
+; AVX512F-NEXT:    vmovdqa 112(%rdi), %xmm0
 ; AVX512F-NEXT:    vmovdqa 96(%rdi), %xmm2
-; AVX512F-NEXT:    vpblendw {{.*#+}} xmm4 = xmm2[0],xmm1[1],xmm2[2,3],xmm1[4],xmm2[5,6],xmm1[7]
+; AVX512F-NEXT:    vpblendw {{.*#+}} xmm4 = xmm2[0],xmm0[1],xmm2[2,3],xmm0[4],xmm2[5,6],xmm0[7]
 ; AVX512F-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[0,1,6,7,12,13,2,3,8,9,14,15,u,u,u,u]
 ; AVX512F-NEXT:    vpblendd {{.*#+}} ymm7 = ymm4[0,1,2],ymm3[3,4,5,6,7]
 ; AVX512F-NEXT:    vmovdqa (%rdi), %ymm8
 ; AVX512F-NEXT:    vmovdqa 32(%rdi), %ymm9
-; AVX512F-NEXT:    vmovdqa %ymm0, %ymm3
+; AVX512F-NEXT:    vmovdqa %ymm1, %ymm3
 ; AVX512F-NEXT:    vpternlogq $202, %ymm9, %ymm8, %ymm3
 ; AVX512F-NEXT:    vpermq {{.*#+}} ymm4 = ymm3[2,3,0,1]
 ; AVX512F-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm4[1],ymm3[2,3],ymm4[4],ymm3[5,6],ymm4[7],ymm3[8],ymm4[9],ymm3[10,11],ymm4[12],ymm3[13,14],ymm4[15]
@@ -1140,48 +1142,49 @@ define void @load_i16_stride3_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512F-NEXT:    vpshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,6,5,4,7]
 ; AVX512F-NEXT:    vpblendd {{.*#+}} ymm10 = ymm10[0,1,2,3],ymm11[4,5,6,7]
 ; AVX512F-NEXT:    vinserti64x4 $1, %ymm7, %zmm10, %zmm7
-; AVX512F-NEXT:    vmovdqa %ymm0, %ymm10
+; AVX512F-NEXT:    vmovdqa %ymm1, %ymm10
 ; AVX512F-NEXT:    vpternlogq $202, %ymm6, %ymm5, %ymm10
 ; AVX512F-NEXT:    vpermq {{.*#+}} ymm11 = ymm10[2,3,0,1]
 ; AVX512F-NEXT:    vpblendw {{.*#+}} ymm10 = ymm10[0,1],ymm11[2],ymm10[3,4],ymm11[5],ymm10[6,7,8,9],ymm11[10],ymm10[11,12],ymm11[13],ymm10[14,15]
-; AVX512F-NEXT:    vpshufb {{.*#+}} ymm10 = ymm10[2,3,8,9,14,15,4,5,10,11,0,1,6,7,12,13,18,19,24,25,30,31,20,21,26,27,16,17,22,23,28,29]
-; AVX512F-NEXT:    vpblendw {{.*#+}} xmm11 = xmm2[0,1],xmm1[2],xmm2[3,4],xmm1[5],xmm2[6,7]
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm11 = xmm11[2,3,8,9,14,15,4,5,10,11,u,u,u,u,u,u]
-; AVX512F-NEXT:    vpblendw {{.*#+}} xmm11 = xmm11[0,1,2,3,4],xmm10[5,6,7]
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm10 = ymm11[0,1,2,3],ymm10[4,5,6,7]
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm11 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
-; AVX512F-NEXT:    vmovdqa %ymm11, %ymm12
-; AVX512F-NEXT:    vpternlogq $202, %ymm8, %ymm9, %ymm12
-; AVX512F-NEXT:    vpermq {{.*#+}} ymm13 = ymm12[2,3,0,1]
-; AVX512F-NEXT:    vpblendw {{.*#+}} ymm12 = ymm12[0,1],ymm13[2],ymm12[3,4],ymm13[5],ymm12[6,7,8,9],ymm13[10],ymm12[11,12],ymm13[13],ymm12[14,15]
-; AVX512F-NEXT:    vpshufb {{.*#+}} ymm12 = ymm12[2,3,8,9,14,15,4,5,12,13,10,11,0,1,6,7,18,19,24,25,30,31,20,21,28,29,26,27,16,17,22,23]
-; AVX512F-NEXT:    vpblendw {{.*#+}} xmm13 = xmm3[0,1],xmm4[2],xmm3[3,4],xmm4[5],xmm3[6,7]
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm13 = xmm13[u,u,u,u,u,u,4,5,10,11,0,1,6,7,12,13]
-; AVX512F-NEXT:    vinserti128 $1, %xmm13, %ymm0, %ymm13
-; AVX512F-NEXT:    vpblendw {{.*#+}} ymm13 = ymm12[0,1,2],ymm13[3,4,5,6,7],ymm12[8,9,10],ymm13[11,12,13,14,15]
-; AVX512F-NEXT:    vpshufhw {{.*#+}} xmm12 = xmm12[0,1,2,3,5,6,7,4]
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm12 = ymm12[0,1,2,3],ymm13[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $1, %ymm10, %zmm12, %zmm10
-; AVX512F-NEXT:    vpternlogq $202, %ymm5, %ymm6, %ymm11
-; AVX512F-NEXT:    vpermq {{.*#+}} ymm5 = ymm11[2,3,0,1]
-; AVX512F-NEXT:    vpblendw {{.*#+}} ymm5 = ymm5[0],ymm11[1,2],ymm5[3],ymm11[4,5],ymm5[6],ymm11[7],ymm5[8],ymm11[9,10],ymm5[11],ymm11[12,13],ymm5[14],ymm11[15]
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm11 = [2,3,8,9,14,15,4,5,10,11,0,1,6,7,12,13,18,19,24,25,30,31,20,21,26,27,16,17,22,23,28,29]
+; AVX512F-NEXT:    vpshufb %ymm11, %ymm10, %ymm10
+; AVX512F-NEXT:    vpblendw {{.*#+}} xmm12 = xmm2[0,1],xmm0[2],xmm2[3,4],xmm0[5],xmm2[6,7]
+; AVX512F-NEXT:    vpshufb %xmm11, %xmm12, %xmm12
+; AVX512F-NEXT:    vpblendw {{.*#+}} xmm12 = xmm12[0,1,2,3,4],xmm10[5,6,7]
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm10 = ymm12[0,1,2,3],ymm10[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm12 = [65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535]
+; AVX512F-NEXT:    vmovdqa %ymm12, %ymm13
+; AVX512F-NEXT:    vpternlogq $202, %ymm8, %ymm9, %ymm13
+; AVX512F-NEXT:    vpermq {{.*#+}} ymm14 = ymm13[2,3,0,1]
+; AVX512F-NEXT:    vpblendw {{.*#+}} ymm13 = ymm13[0,1],ymm14[2],ymm13[3,4],ymm14[5],ymm13[6,7,8,9],ymm14[10],ymm13[11,12],ymm14[13],ymm13[14,15]
+; AVX512F-NEXT:    vpshufb {{.*#+}} ymm13 = ymm13[2,3,8,9,14,15,4,5,12,13,10,11,0,1,6,7,18,19,24,25,30,31,20,21,28,29,26,27,16,17,22,23]
+; AVX512F-NEXT:    vpblendw {{.*#+}} xmm14 = xmm3[0,1],xmm4[2],xmm3[3,4],xmm4[5],xmm3[6,7]
+; AVX512F-NEXT:    vpshufb %xmm11, %xmm14, %xmm11
+; AVX512F-NEXT:    vinserti128 $1, %xmm11, %ymm0, %ymm11
+; AVX512F-NEXT:    vpblendw {{.*#+}} ymm11 = ymm13[0,1,2],ymm11[3,4,5,6,7],ymm13[8,9,10],ymm11[11,12,13,14,15]
+; AVX512F-NEXT:    vpshufhw {{.*#+}} xmm13 = xmm13[0,1,2,3,5,6,7,4]
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm11 = ymm13[0,1,2,3],ymm11[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm10, %zmm11, %zmm10
+; AVX512F-NEXT:    vpternlogq $202, %ymm5, %ymm6, %ymm12
+; AVX512F-NEXT:    vpermq {{.*#+}} ymm5 = ymm12[2,3,0,1]
+; AVX512F-NEXT:    vpblendw {{.*#+}} ymm5 = ymm5[0],ymm12[1,2],ymm5[3],ymm12[4,5],ymm5[6],ymm12[7],ymm5[8],ymm12[9,10],ymm5[11],ymm12[12,13],ymm5[14],ymm12[15]
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm6 = [4,5,10,11,0,1,6,7,12,13,2,3,8,9,14,15,20,21,26,27,16,17,22,23,28,29,18,19,24,25,30,31]
 ; AVX512F-NEXT:    vpshufb %ymm6, %ymm5, %ymm5
-; AVX512F-NEXT:    vpternlogq $202, %ymm8, %ymm9, %ymm0
-; AVX512F-NEXT:    vpermq {{.*#+}} ymm8 = ymm0[2,3,0,1]
-; AVX512F-NEXT:    vpblendw {{.*#+}} ymm0 = ymm8[0],ymm0[1,2],ymm8[3],ymm0[4,5],ymm8[6],ymm0[7],ymm8[8],ymm0[9,10],ymm8[11],ymm0[12,13],ymm8[14],ymm0[15]
-; AVX512F-NEXT:    vpshufb %ymm6, %ymm0, %ymm0
+; AVX512F-NEXT:    vpternlogq $202, %ymm8, %ymm9, %ymm1
+; AVX512F-NEXT:    vpermq {{.*#+}} ymm8 = ymm1[2,3,0,1]
+; AVX512F-NEXT:    vpblendw {{.*#+}} ymm1 = ymm8[0],ymm1[1,2],ymm8[3],ymm1[4,5],ymm8[6],ymm1[7],ymm8[8],ymm1[9,10],ymm8[11],ymm1[12,13],ymm8[14],ymm1[15]
+; AVX512F-NEXT:    vpshufb %ymm6, %ymm1, %ymm1
 ; AVX512F-NEXT:    vpblendw {{.*#+}} xmm3 = xmm4[0],xmm3[1],xmm4[2,3],xmm3[4],xmm4[5,6],xmm3[7]
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u,u,u,0,1,6,7,12,13,2,3,8,9,14,15]
+; AVX512F-NEXT:    vpshufb %xmm6, %xmm3, %xmm3
 ; AVX512F-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm3[5,6,7]
-; AVX512F-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2],xmm1[3,4],xmm2[5],xmm1[6,7]
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[4,5,10,11,0,1,6,7,12,13,14,15,0,1,2,3]
-; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm1
-; AVX512F-NEXT:    vextracti32x4 $2, %zmm1, %xmm1
-; AVX512F-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3,4],xmm5[5,6,7]
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm5[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm3[5,6,7]
+; AVX512F-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2],xmm0[3,4],xmm2[5],xmm0[6,7]
+; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[4,5,10,11,0,1,6,7,12,13,14,15,0,1,2,3]
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512F-NEXT:    vextracti32x4 $2, %zmm0, %xmm0
+; AVX512F-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4],xmm5[5,6,7]
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; AVX512F-NEXT:    vmovdqa64 %zmm7, (%rsi)
 ; AVX512F-NEXT:    vmovdqa64 %zmm10, (%rdx)
 ; AVX512F-NEXT:    vmovdqa64 %zmm0, (%rcx)
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-5.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-5.ll
index f5399e791f6e843de0ec10a045160e25273487ff..30f96e0b3e1492d5d5341c64f966e3a3227e1787 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-5.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-5.ll
@@ -3184,176 +3184,192 @@ define void @load_i16_stride5_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512F-SLOW-NEXT:    vmovdqa 256(%rdi), %ymm0
 ; AVX512F-SLOW-NEXT:    vmovdqa 288(%rdi), %ymm1
 ; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm2 = ymm1[0,1],ymm0[2],ymm1[3],ymm0[4],ymm1[5,6],ymm0[7],ymm1[8,9],ymm0[10],ymm1[11],ymm0[12],ymm1[13,14],ymm0[15]
+; AVX512F-SLOW-NEXT:    vmovdqa64 %ymm1, %ymm17
 ; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm2, %xmm3
 ; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1,2],xmm3[3,4],xmm2[5,6,7]
 ; AVX512F-SLOW-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,u,u,u,4,5,14,15,8,9,2,3,12,13,6,7]
-; AVX512F-SLOW-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512F-SLOW-NEXT:    vmovdqa 192(%rdi), %ymm7
-; AVX512F-SLOW-NEXT:    vmovdqa 224(%rdi), %ymm8
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm3 = ymm8[0],ymm7[1],ymm8[2,3],ymm7[4],ymm8[5],ymm7[6],ymm8[7,8],ymm7[9],ymm8[10,11],ymm7[12],ymm8[13],ymm7[14],ymm8[15]
-; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm3, %xmm4
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0],ymm3[1,2],ymm4[3],ymm3[4,5,6,7]
-; AVX512F-SLOW-NEXT:    vpshufb {{.*#+}} ymm3 = ymm3[u,u,u,u,u,u,u,u,8,9,2,3,12,13,6,7,16,17,26,27,26,27,30,31,24,25,18,19,28,29,22,23]
-; AVX512F-SLOW-NEXT:    vmovdqa 176(%rdi), %xmm9
-; AVX512F-SLOW-NEXT:    vpshufd {{.*#+}} xmm4 = xmm9[3,1,2,3]
-; AVX512F-SLOW-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm4[2,1,2,3,4,5,6,7]
-; AVX512F-SLOW-NEXT:    vmovdqa 160(%rdi), %xmm10
-; AVX512F-SLOW-NEXT:    vpshufd {{.*#+}} xmm5 = xmm10[0,2,2,3]
-; AVX512F-SLOW-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm5[0,3,2,3,4,5,6,7]
-; AVX512F-SLOW-NEXT:    vpunpckldq {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1],ymm3[2,3,4,5,6,7]
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm3[0,1,2,3,4],ymm2[5,6,7]
-; AVX512F-SLOW-NEXT:    vmovdqa (%rdi), %ymm5
-; AVX512F-SLOW-NEXT:    vmovdqa 32(%rdi), %ymm6
-; AVX512F-SLOW-NEXT:    vmovdqa 64(%rdi), %ymm2
-; AVX512F-SLOW-NEXT:    vmovdqa 96(%rdi), %ymm3
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm11 = ymm2[0],ymm3[1,2],ymm2[3],ymm3[4],ymm2[5],ymm3[6,7],ymm2[8],ymm3[9,10],ymm2[11],ymm3[12],ymm2[13],ymm3[14,15]
+; AVX512F-SLOW-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm4
+; AVX512F-SLOW-NEXT:    vmovdqa 192(%rdi), %ymm3
+; AVX512F-SLOW-NEXT:    vmovdqa 224(%rdi), %ymm9
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm5 = ymm9[0],ymm3[1],ymm9[2,3],ymm3[4],ymm9[5],ymm3[6],ymm9[7,8],ymm3[9],ymm9[10,11],ymm3[12],ymm9[13],ymm3[14],ymm9[15]
+; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm5, %xmm6
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm5 = ymm6[0],ymm5[1,2],ymm6[3],ymm5[4,5,6,7]
+; AVX512F-SLOW-NEXT:    vmovdqa {{.*#+}} ymm7 = <u,u,u,u,u,u,u,u,8,9,2,3,12,13,6,7,16,17,26,27,26,27,30,31,24,25,18,19,28,29,22,23>
+; AVX512F-SLOW-NEXT:    vpshufb %ymm7, %ymm5, %ymm5
+; AVX512F-SLOW-NEXT:    vmovdqa 176(%rdi), %xmm1
+; AVX512F-SLOW-NEXT:    vpshufd {{.*#+}} xmm8 = xmm1[3,1,2,3]
+; AVX512F-SLOW-NEXT:    vmovdqa64 %xmm1, %xmm20
+; AVX512F-SLOW-NEXT:    vpshuflw {{.*#+}} xmm8 = xmm8[2,1,2,3,4,5,6,7]
+; AVX512F-SLOW-NEXT:    vmovdqa 160(%rdi), %xmm6
+; AVX512F-SLOW-NEXT:    vpshufd {{.*#+}} xmm10 = xmm6[0,2,2,3]
+; AVX512F-SLOW-NEXT:    vpshuflw {{.*#+}} xmm10 = xmm10[0,3,2,3,4,5,6,7]
+; AVX512F-SLOW-NEXT:    vpunpckldq {{.*#+}} xmm8 = xmm10[0],xmm8[0],xmm10[1],xmm8[1]
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm5 = ymm8[0,1],ymm5[2,3,4,5,6,7]
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm14 = ymm5[0,1,2,3,4],ymm4[5,6,7]
+; AVX512F-SLOW-NEXT:    vmovdqa (%rdi), %ymm8
+; AVX512F-SLOW-NEXT:    vmovdqa 32(%rdi), %ymm10
+; AVX512F-SLOW-NEXT:    vmovdqa 64(%rdi), %ymm4
+; AVX512F-SLOW-NEXT:    vmovdqa 96(%rdi), %ymm5
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm11 = ymm4[0],ymm5[1,2],ymm4[3],ymm5[4],ymm4[5],ymm5[6,7],ymm4[8],ymm5[9,10],ymm4[11],ymm5[12],ymm4[13],ymm5[14,15]
 ; AVX512F-SLOW-NEXT:    vpermq {{.*#+}} ymm12 = ymm11[2,3,0,1]
 ; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm11 = ymm11[0,1,2,3,4],ymm12[5],ymm11[6],ymm12[7]
 ; AVX512F-SLOW-NEXT:    vpshufb {{.*#+}} ymm11 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm11[6,7,16,17,26,27,20,21,30,31,24,25],zero,zero,zero,zero,zero,zero
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm12 = ymm5[0],ymm6[1],ymm5[2,3],ymm6[4],ymm5[5],ymm6[6],ymm5[7,8],ymm6[9],ymm5[10,11],ymm6[12],ymm5[13],ymm6[14],ymm5[15]
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm12 = ymm8[0],ymm10[1],ymm8[2,3],ymm10[4],ymm8[5],ymm10[6],ymm8[7,8],ymm10[9],ymm8[10,11],ymm10[12],ymm8[13],ymm10[14],ymm8[15]
 ; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm12, %xmm13
 ; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} xmm12 = xmm12[0],xmm13[1,2,3],xmm12[4,5],xmm13[6,7]
 ; AVX512F-SLOW-NEXT:    vpshufb {{.*#+}} ymm12 = ymm12[0,1,10,11,4,5,14,15,8,9,2,3,12,13],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm12[u,u,u,u,u,u]
-; AVX512F-SLOW-NEXT:    vpor %ymm11, %ymm12, %ymm13
-; AVX512F-SLOW-NEXT:    vmovdqa 144(%rdi), %xmm12
-; AVX512F-SLOW-NEXT:    vmovdqa 128(%rdi), %xmm11
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} xmm14 = xmm11[0],xmm12[1],xmm11[2,3]
-; AVX512F-SLOW-NEXT:    vpshufb {{.*#+}} xmm14 = xmm14[u,u,u,u,u,u,u,u,u,u,2,3,12,13,6,7]
-; AVX512F-SLOW-NEXT:    vinserti128 $1, %xmm14, %ymm0, %ymm15
-; AVX512F-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm17 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535]
-; AVX512F-SLOW-NEXT:    vpternlogq $184, %zmm13, %zmm17, %zmm15
-; AVX512F-SLOW-NEXT:    vinserti64x4 $1, %ymm4, %zmm15, %zmm16
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm13 = ymm7[0],ymm8[1],ymm7[2,3],ymm8[4],ymm7[5],ymm8[6],ymm7[7,8],ymm8[9],ymm7[10,11],ymm8[12],ymm7[13],ymm8[14],ymm7[15]
-; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm13, %xmm15
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm13 = ymm13[0],ymm15[1],ymm13[2],ymm15[3],ymm13[4,5,6,7]
-; AVX512F-SLOW-NEXT:    vpshufb {{.*#+}} ymm13 = ymm13[2,3,12,13,0,1,0,1,10,11,4,5,14,15,8,9,18,19,28,29,16,17,16,17,26,27,20,21,30,31,24,25]
-; AVX512F-SLOW-NEXT:    vpsrlq $48, %xmm9, %xmm15
-; AVX512F-SLOW-NEXT:    vpshufd {{.*#+}} xmm4 = xmm10[0,3,2,3]
-; AVX512F-SLOW-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm4[1,2,2,3,4,5,6,7]
-; AVX512F-SLOW-NEXT:    vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm15[0],xmm4[1],xmm15[1]
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1,2],xmm13[3,4,5,6,7]
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm13[4,5,6,7]
+; AVX512F-SLOW-NEXT:    vpor %ymm11, %ymm12, %ymm15
+; AVX512F-SLOW-NEXT:    vmovdqa 144(%rdi), %xmm11
+; AVX512F-SLOW-NEXT:    vmovdqa 128(%rdi), %xmm12
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} xmm13 = xmm12[0],xmm11[1],xmm12[2,3]
+; AVX512F-SLOW-NEXT:    vpshufb %xmm7, %xmm13, %xmm7
+; AVX512F-SLOW-NEXT:    vinserti128 $1, %xmm7, %ymm0, %ymm7
+; AVX512F-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm18 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535]
+; AVX512F-SLOW-NEXT:    vpternlogq $184, %zmm15, %zmm18, %zmm7
+; AVX512F-SLOW-NEXT:    vinserti64x4 $1, %ymm14, %zmm7, %zmm16
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm7 = ymm3[0],ymm9[1],ymm3[2,3],ymm9[4],ymm3[5],ymm9[6],ymm3[7,8],ymm9[9],ymm3[10,11],ymm9[12],ymm3[13],ymm9[14],ymm3[15]
+; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm7, %xmm14
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm7 = ymm7[0],ymm14[1],ymm7[2],ymm14[3],ymm7[4,5,6,7]
+; AVX512F-SLOW-NEXT:    vmovdqa {{.*#+}} ymm14 = [2,3,12,13,0,1,0,1,10,11,4,5,14,15,8,9,18,19,28,29,16,17,16,17,26,27,20,21,30,31,24,25]
+; AVX512F-SLOW-NEXT:    vpshufb %ymm14, %ymm7, %ymm7
+; AVX512F-SLOW-NEXT:    vpsrlq $48, %xmm1, %xmm15
+; AVX512F-SLOW-NEXT:    vpshufd {{.*#+}} xmm13 = xmm6[0,3,2,3]
+; AVX512F-SLOW-NEXT:    vpshuflw {{.*#+}} xmm13 = xmm13[1,2,2,3,4,5,6,7]
+; AVX512F-SLOW-NEXT:    vpunpckldq {{.*#+}} xmm13 = xmm13[0],xmm15[0],xmm13[1],xmm15[1]
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} xmm13 = xmm13[0,1,2],xmm7[3,4,5,6,7]
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm7 = ymm13[0,1,2,3],ymm7[4,5,6,7]
+; AVX512F-SLOW-NEXT:    vmovdqa %ymm0, %ymm2
+; AVX512F-SLOW-NEXT:    vmovdqa64 %ymm17, %ymm1
 ; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm13 = ymm0[0],ymm1[1,2],ymm0[3],ymm1[4],ymm0[5],ymm1[6,7],ymm0[8],ymm1[9,10],ymm0[11],ymm1[12],ymm0[13],ymm1[14,15]
 ; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm13, %xmm15
 ; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} xmm13 = xmm15[0],xmm13[1],xmm15[2],xmm13[3]
 ; AVX512F-SLOW-NEXT:    vpshufb {{.*#+}} xmm13 = xmm13[u,u,u,u,6,7,0,1,10,11,4,5,14,15,8,9]
 ; AVX512F-SLOW-NEXT:    vinserti128 $1, %xmm13, %ymm0, %ymm13
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3,4],ymm13[5,6,7]
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm13 = ymm3[0],ymm2[1],ymm3[2,3],ymm2[4],ymm3[5],ymm2[6],ymm3[7,8],ymm2[9],ymm3[10,11],ymm2[12],ymm3[13],ymm2[14],ymm3[15]
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm7 = ymm7[0,1,2,3,4],ymm13[5,6,7]
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm13 = ymm5[0],ymm4[1],ymm5[2,3],ymm4[4],ymm5[5],ymm4[6],ymm5[7,8],ymm4[9],ymm5[10,11],ymm4[12],ymm5[13],ymm4[14],ymm5[15]
 ; AVX512F-SLOW-NEXT:    vpermq {{.*#+}} ymm15 = ymm13[2,3,0,1]
 ; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm13 = ymm13[0,1,2,3,4],ymm15[5],ymm13[6,7]
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm15 = ymm10[0],ymm8[1],ymm10[2],ymm8[3],ymm10[4,5],ymm8[6],ymm10[7,8],ymm8[9],ymm10[10],ymm8[11],ymm10[12,13],ymm8[14],ymm10[15]
+; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm15, %xmm0
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm15[0,1],xmm0[2,3],xmm15[4,5,6],xmm0[7]
 ; AVX512F-SLOW-NEXT:    vpshufb {{.*#+}} ymm13 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm13[8,9,18,19,28,29,22,23,16,17,26,27],zero,zero,zero,zero,zero,zero
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm15 = ymm6[0],ymm5[1],ymm6[2],ymm5[3],ymm6[4,5],ymm5[6],ymm6[7,8],ymm5[9],ymm6[10],ymm5[11],ymm6[12,13],ymm5[14],ymm6[15]
-; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm15, %xmm14
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} xmm14 = xmm15[0,1],xmm14[2,3],xmm15[4,5,6],xmm14[7]
-; AVX512F-SLOW-NEXT:    vpshufb {{.*#+}} ymm14 = ymm14[2,3,12,13,6,7,0,1,10,11,4,5,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm14[u,u,u,u,u,u]
-; AVX512F-SLOW-NEXT:    vpor %ymm13, %ymm14, %ymm13
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} xmm14 = xmm11[0,1],xmm12[2],xmm11[3]
-; AVX512F-SLOW-NEXT:    vpshufb {{.*#+}} xmm14 = xmm14[u,u,u,u,u,u,u,u,u,u,4,5,14,15,8,9]
-; AVX512F-SLOW-NEXT:    vinserti128 $1, %xmm14, %ymm0, %ymm14
-; AVX512F-SLOW-NEXT:    vpternlogq $184, %zmm13, %zmm17, %zmm14
-; AVX512F-SLOW-NEXT:    vinserti64x4 $1, %ymm4, %zmm14, %zmm18
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm4 = ymm8[0],ymm7[1],ymm8[2],ymm7[3],ymm8[4,5],ymm7[6],ymm8[7,8],ymm7[9],ymm8[10],ymm7[11],ymm8[12,13],ymm7[14],ymm8[15]
-; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm4, %xmm14
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0],ymm14[1],ymm4[2,3,4,5,6,7]
-; AVX512F-SLOW-NEXT:    vpshufb {{.*#+}} ymm4 = ymm4[4,5,14,15,12,13,2,3,12,13,6,7,0,1,10,11,20,21,30,31,28,29,18,19,28,29,22,23,16,17,26,27]
-; AVX512F-SLOW-NEXT:    vpshufd {{.*#+}} xmm14 = xmm10[0,1,1,3]
-; AVX512F-SLOW-NEXT:    vpshufhw {{.*#+}} xmm14 = xmm14[0,1,2,3,4,7,6,7]
-; AVX512F-SLOW-NEXT:    vpunpckhdq {{.*#+}} xmm14 = xmm14[2],xmm9[2],xmm14[3],xmm9[3]
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} xmm14 = xmm14[0,1,2],xmm4[3,4,5,6,7]
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm14[0,1,2,3],ymm4[4,5,6,7]
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm14 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5],ymm0[6],ymm1[7,8],ymm0[9],ymm1[10,11],ymm0[12],ymm1[13],ymm0[14],ymm1[15]
-; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm14, %xmm15
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} xmm14 = xmm15[0,1,2],xmm14[3,4],xmm15[5,6,7]
-; AVX512F-SLOW-NEXT:    vpshufb {{.*#+}} xmm14 = xmm14[u,u,u,u,8,9,2,3,12,13,6,7,0,1,10,11]
-; AVX512F-SLOW-NEXT:    vinserti128 $1, %xmm14, %ymm0, %ymm14
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3,4],ymm14[5,6,7]
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm14 = ymm6[0,1],ymm5[2],ymm6[3],ymm5[4],ymm6[5,6],ymm5[7],ymm6[8,9],ymm5[10],ymm6[11],ymm5[12],ymm6[13,14],ymm5[15]
-; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm14, %xmm15
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} xmm14 = xmm14[0,1,2],xmm15[3,4],xmm14[5,6,7]
-; AVX512F-SLOW-NEXT:    vpshufb {{.*#+}} xmm14 = xmm14[4,5,14,15,8,9,2,3,12,13,6,7,u,u,u,u]
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm15 = ymm2[0],ymm3[1],ymm2[2,3],ymm3[4],ymm2[5],ymm3[6],ymm2[7,8],ymm3[9],ymm2[10,11],ymm3[12],ymm2[13],ymm3[14],ymm2[15]
-; AVX512F-SLOW-NEXT:    vpermq {{.*#+}} ymm13 = ymm15[2,3,0,1]
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm13 = ymm15[0,1,2,3,4,5],ymm13[6],ymm15[7]
-; AVX512F-SLOW-NEXT:    vpshufb {{.*#+}} ymm13 = ymm13[u,u,u,u,u,u,u,u,u,u,u,u,0,1,10,11,20,21,30,31,24,25,18,19,28,29,26,27,16,17,26,27]
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm13 = ymm14[0,1,2],ymm13[3,4,5,6,7]
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} xmm14 = xmm12[0],xmm11[1],xmm12[2,3]
-; AVX512F-SLOW-NEXT:    vpshufb {{.*#+}} xmm14 = xmm14[u,u,u,u,u,u,u,u,u,u,6,7,0,1,10,11]
-; AVX512F-SLOW-NEXT:    vinserti128 $1, %xmm14, %ymm0, %ymm14
-; AVX512F-SLOW-NEXT:    vpternlogq $184, %zmm13, %zmm17, %zmm14
-; AVX512F-SLOW-NEXT:    vinserti64x4 $1, %ymm4, %zmm14, %zmm17
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm4 = ymm8[0,1],ymm7[2],ymm8[3],ymm7[4],ymm8[5,6],ymm7[7],ymm8[8,9],ymm7[10],ymm8[11],ymm7[12],ymm8[13,14],ymm7[15]
-; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm4, %xmm13
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0,1],ymm13[2],ymm4[3,4,5,6,7]
-; AVX512F-SLOW-NEXT:    vpshufb {{.*#+}} ymm4 = ymm4[6,7,6,7,8,9,4,5,14,15,8,9,2,3,12,13,22,23,22,23,24,25,20,21,30,31,24,25,18,19,28,29]
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} xmm13 = xmm9[0],xmm10[1],xmm9[2,3]
-; AVX512F-SLOW-NEXT:    vpshufb {{.*#+}} xmm13 = xmm13[6,7,0,1,10,11,u,u,u,u,u,u,u,u,u,u]
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} xmm13 = xmm13[0,1,2],xmm4[3,4,5,6,7]
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm13[0,1,2,3],ymm4[4,5,6,7]
+; AVX512F-SLOW-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[2,3,12,13,6,7,0,1,10,11,4,5,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm0[u,u,u,u,u,u]
+; AVX512F-SLOW-NEXT:    vpor %ymm0, %ymm13, %ymm0
 ; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} xmm13 = xmm12[0,1],xmm11[2],xmm12[3]
-; AVX512F-SLOW-NEXT:    vpshufb {{.*#+}} xmm13 = xmm13[u,u,u,u,u,u,u,u,u,u,8,9,2,3,12,13]
+; AVX512F-SLOW-NEXT:    vpshufb %xmm14, %xmm13, %xmm13
 ; AVX512F-SLOW-NEXT:    vinserti128 $1, %xmm13, %ymm0, %ymm13
-; AVX512F-SLOW-NEXT:    vinserti64x4 $1, %ymm4, %zmm13, %zmm4
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm13 = ymm5[0],ymm6[1,2],ymm5[3],ymm6[4],ymm5[5],ymm6[6,7],ymm5[8],ymm6[9,10],ymm5[11],ymm6[12],ymm5[13],ymm6[14,15]
-; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm13, %xmm15
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} xmm13 = xmm15[0],xmm13[1],xmm15[2],xmm13[3]
+; AVX512F-SLOW-NEXT:    vpternlogq $184, %zmm0, %zmm18, %zmm13
+; AVX512F-SLOW-NEXT:    vinserti64x4 $1, %ymm7, %zmm13, %zmm19
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm9[0],ymm3[1],ymm9[2],ymm3[3],ymm9[4,5],ymm3[6],ymm9[7,8],ymm3[9],ymm9[10],ymm3[11],ymm9[12,13],ymm3[14],ymm9[15]
+; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm7
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm7[1],ymm0[2,3,4,5,6,7]
+; AVX512F-SLOW-NEXT:    vmovdqa {{.*#+}} ymm7 = [4,5,14,15,12,13,2,3,12,13,6,7,0,1,10,11,20,21,30,31,28,29,18,19,28,29,22,23,16,17,26,27]
+; AVX512F-SLOW-NEXT:    vpshufb %ymm7, %ymm0, %ymm0
+; AVX512F-SLOW-NEXT:    vmovdqa64 %ymm7, %ymm21
+; AVX512F-SLOW-NEXT:    vpshufd {{.*#+}} xmm7 = xmm6[0,1,1,3]
+; AVX512F-SLOW-NEXT:    vpshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,4,7,6,7]
+; AVX512F-SLOW-NEXT:    vmovdqa64 %xmm20, %xmm15
+; AVX512F-SLOW-NEXT:    vpunpckhdq {{.*#+}} xmm7 = xmm7[2],xmm20[2],xmm7[3],xmm20[3]
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} xmm7 = xmm7[0,1,2],xmm0[3,4,5,6,7]
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm7[0,1,2,3],ymm0[4,5,6,7]
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm7 = ymm1[0],ymm2[1],ymm1[2,3],ymm2[4],ymm1[5],ymm2[6],ymm1[7,8],ymm2[9],ymm1[10,11],ymm2[12],ymm1[13],ymm2[14],ymm1[15]
+; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm7, %xmm13
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} xmm7 = xmm13[0,1,2],xmm7[3,4],xmm13[5,6,7]
+; AVX512F-SLOW-NEXT:    vpshufb {{.*#+}} xmm7 = xmm7[u,u,u,u,8,9,2,3,12,13,6,7,0,1,10,11]
+; AVX512F-SLOW-NEXT:    vinserti128 $1, %xmm7, %ymm0, %ymm7
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm7 = ymm0[0,1,2,3,4],ymm7[5,6,7]
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm9[0,1],ymm3[2],ymm9[3],ymm3[4],ymm9[5,6],ymm3[7],ymm9[8,9],ymm3[10],ymm9[11],ymm3[12],ymm9[13,14],ymm3[15]
+; AVX512F-SLOW-NEXT:    vmovdqa64 %ymm9, %ymm20
+; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm13
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm13[2],ymm0[3,4,5,6,7]
+; AVX512F-SLOW-NEXT:    vmovdqa {{.*#+}} ymm13 = [6,7,6,7,8,9,4,5,14,15,8,9,2,3,12,13,22,23,22,23,24,25,20,21,30,31,24,25,18,19,28,29]
+; AVX512F-SLOW-NEXT:    vpshufb %ymm13, %ymm0, %ymm0
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} xmm14 = xmm15[0],xmm6[1],xmm15[2,3]
+; AVX512F-SLOW-NEXT:    vmovdqa64 %xmm15, %xmm22
+; AVX512F-SLOW-NEXT:    vpshufb {{.*#+}} xmm14 = xmm14[6,7,0,1,10,11,u,u,u,u,u,u,u,u,u,u]
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} xmm14 = xmm14[0,1,2],xmm0[3,4,5,6,7]
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm14[0,1,2,3],ymm0[4,5,6,7]
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} xmm14 = xmm11[0,1],xmm12[2],xmm11[3]
+; AVX512F-SLOW-NEXT:    vpshufb %xmm13, %xmm14, %xmm13
+; AVX512F-SLOW-NEXT:    vinserti128 $1, %xmm13, %ymm0, %ymm13
+; AVX512F-SLOW-NEXT:    vinserti64x4 $1, %ymm0, %zmm13, %zmm0
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm13 = ymm8[0],ymm10[1,2],ymm8[3],ymm10[4],ymm8[5],ymm10[6,7],ymm8[8],ymm10[9,10],ymm8[11],ymm10[12],ymm8[13],ymm10[14,15]
+; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm13, %xmm14
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} xmm13 = xmm14[0],xmm13[1],xmm14[2],xmm13[3]
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm14 = ymm5[0],ymm4[1],ymm5[2],ymm4[3],ymm5[4,5],ymm4[6],ymm5[7,8],ymm4[9],ymm5[10],ymm4[11],ymm5[12,13],ymm4[14],ymm5[15]
+; AVX512F-SLOW-NEXT:    vpermq {{.*#+}} ymm15 = ymm14[2,3,0,1]
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm14 = ymm14[0,1,2,3],ymm15[4],ymm14[5],ymm15[6],ymm14[7]
 ; AVX512F-SLOW-NEXT:    vpshufb {{.*#+}} xmm13 = xmm13[6,7,0,1,10,11,4,5,14,15,8,9,u,u,u,u]
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm15 = ymm3[0],ymm2[1],ymm3[2],ymm2[3],ymm3[4,5],ymm2[6],ymm3[7,8],ymm2[9],ymm3[10],ymm2[11],ymm3[12,13],ymm2[14],ymm3[15]
-; AVX512F-SLOW-NEXT:    vpermq {{.*#+}} ymm14 = ymm15[2,3,0,1]
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm14 = ymm15[0,1,2,3],ymm14[4],ymm15[5],ymm14[6],ymm15[7]
 ; AVX512F-SLOW-NEXT:    vpshufb {{.*#+}} ymm14 = ymm14[u,u,u,u,u,u,u,u,u,u,u,u,2,3,12,13,22,23,16,17,26,27,20,21,30,31,30,31,18,19,28,29]
 ; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm13 = ymm13[0,1,2],ymm14[3,4,5,6,7]
-; AVX512F-SLOW-NEXT:    vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm4, %zmm13
-; AVX512F-SLOW-NEXT:    vextracti64x4 $1, %zmm13, %ymm4
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm14 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5],ymm1[6],ymm0[7,8],ymm1[9],ymm0[10,11],ymm1[12],ymm0[13],ymm1[14],ymm0[15]
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm14 = ymm2[0],ymm1[1],ymm2[2,3],ymm1[4],ymm2[5],ymm1[6],ymm2[7,8],ymm1[9],ymm2[10,11],ymm1[12],ymm2[13],ymm1[14],ymm2[15]
+; AVX512F-SLOW-NEXT:    vmovdqa %ymm2, %ymm9
 ; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm14, %xmm15
 ; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} xmm14 = xmm14[0],xmm15[1,2,3],xmm14[4,5],xmm15[6,7]
-; AVX512F-SLOW-NEXT:    vpshufb {{.*#+}} xmm14 = xmm14[u,u,0,1,10,11,4,5,14,15,8,9,2,3,12,13]
-; AVX512F-SLOW-NEXT:    vinserti128 $1, %xmm14, %ymm0, %ymm14
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm14 = ymm4[0],ymm14[1,2,3,4,5,6,7],ymm4[8],ymm14[9,10,11,12,13,14,15]
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm14[4,5,6,7]
-; AVX512F-SLOW-NEXT:    vinserti64x4 $1, %ymm4, %zmm13, %zmm4
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm7 = ymm7[0],ymm8[1,2],ymm7[3],ymm8[4],ymm7[5],ymm8[6,7],ymm7[8],ymm8[9,10],ymm7[11],ymm8[12],ymm7[13],ymm8[14,15]
-; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm7, %xmm8
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm7 = ymm8[0],ymm7[1],ymm8[2],ymm7[3,4,5,6,7]
-; AVX512F-SLOW-NEXT:    vpshufb {{.*#+}} ymm7 = ymm7[8,9,14,15,4,5,6,7,0,1,10,11,4,5,14,15,24,25,30,31,20,21,22,23,16,17,26,27,20,21,30,31]
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} xmm8 = xmm9[0,1],xmm10[2],xmm9[3]
-; AVX512F-SLOW-NEXT:    vpshufb {{.*#+}} xmm8 = xmm8[8,9,2,3,12,13,u,u,u,u,u,u,u,u,u,u]
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} xmm8 = xmm8[0,1,2],xmm7[3,4,5,6,7]
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm7 = ymm8[0,1,2,3],ymm7[4,5,6,7]
-; AVX512F-SLOW-NEXT:    vpshufd {{.*#+}} xmm8 = xmm12[3,1,2,3]
-; AVX512F-SLOW-NEXT:    vpshuflw {{.*#+}} xmm8 = xmm8[0,1,2,1,4,5,6,7]
-; AVX512F-SLOW-NEXT:    vpshufd {{.*#+}} xmm9 = xmm11[0,2,2,3]
-; AVX512F-SLOW-NEXT:    vpshuflw {{.*#+}} xmm9 = xmm9[0,1,0,3,4,5,6,7]
-; AVX512F-SLOW-NEXT:    vpunpckldq {{.*#+}} xmm8 = xmm9[0],xmm8[0],xmm9[1],xmm8[1]
-; AVX512F-SLOW-NEXT:    vinserti128 $1, %xmm8, %ymm0, %ymm8
-; AVX512F-SLOW-NEXT:    vinserti64x4 $1, %ymm7, %zmm8, %zmm7
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm5 = ymm6[0],ymm5[1],ymm6[2,3],ymm5[4],ymm6[5],ymm5[6],ymm6[7,8],ymm5[9],ymm6[10,11],ymm5[12],ymm6[13],ymm5[14],ymm6[15]
-; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm5, %xmm6
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} xmm5 = xmm6[0,1,2],xmm5[3,4],xmm6[5,6,7]
-; AVX512F-SLOW-NEXT:    vpshufb {{.*#+}} xmm5 = xmm5[8,9,2,3,12,13,6,7,0,1,10,11,u,u,u,u]
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm2 = ymm3[0,1],ymm2[2],ymm3[3],ymm2[4],ymm3[5,6],ymm2[7],ymm3[8,9],ymm2[10],ymm3[11],ymm2[12],ymm3[13,14],ymm2[15]
-; AVX512F-SLOW-NEXT:    vpermq {{.*#+}} ymm3 = ymm2[2,3,0,1]
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4],ymm2[5,6],ymm3[7]
-; AVX512F-SLOW-NEXT:    vpshufb {{.*#+}} ymm2 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,4,5,14,15,24,25,18,19,28,29,22,23,20,21,22,23,20,21,30,31]
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm5[0,1,2],ymm2[3,4,5,6,7]
+; AVX512F-SLOW-NEXT:    vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm13
+; AVX512F-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm14[u,u,0,1,10,11,4,5,14,15,8,9,2,3,12,13]
+; AVX512F-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512F-SLOW-NEXT:    vextracti64x4 $1, %zmm13, %ymm14
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm14[0],ymm0[1,2,3,4,5,6,7],ymm14[8],ymm0[9,10,11,12,13,14,15]
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm14[0,1,2,3],ymm0[4,5,6,7]
+; AVX512F-SLOW-NEXT:    vinserti64x4 $1, %ymm0, %zmm13, %zmm17
+; AVX512F-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm11[3,1,2,3]
+; AVX512F-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,1,2,1,4,5,6,7]
+; AVX512F-SLOW-NEXT:    vpshufd {{.*#+}} xmm13 = xmm12[0,2,2,3]
+; AVX512F-SLOW-NEXT:    vpshuflw {{.*#+}} xmm13 = xmm13[0,1,0,3,4,5,6,7]
+; AVX512F-SLOW-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm13[0],xmm0[0],xmm13[1],xmm0[1]
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm13 = ymm10[0,1],ymm8[2],ymm10[3],ymm8[4],ymm10[5,6],ymm8[7],ymm10[8,9],ymm8[10],ymm10[11],ymm8[12],ymm10[13,14],ymm8[15]
+; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm13, %xmm14
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} xmm13 = xmm13[0,1,2],xmm14[3,4],xmm13[5,6,7]
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm14 = ymm4[0],ymm5[1],ymm4[2,3],ymm5[4],ymm4[5],ymm5[6],ymm4[7,8],ymm5[9],ymm4[10,11],ymm5[12],ymm4[13],ymm5[14],ymm4[15]
+; AVX512F-SLOW-NEXT:    vpermq {{.*#+}} ymm15 = ymm14[2,3,0,1]
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm14 = ymm14[0,1,2,3,4,5],ymm15[6],ymm14[7]
+; AVX512F-SLOW-NEXT:    vpshufb {{.*#+}} xmm13 = xmm13[4,5,14,15,8,9,2,3,12,13,6,7,u,u,u,u]
+; AVX512F-SLOW-NEXT:    vpshufb {{.*#+}} ymm14 = ymm14[u,u,u,u,u,u,u,u,u,u,u,u,0,1,10,11,20,21,30,31,24,25,18,19,28,29,26,27,16,17,26,27]
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm13 = ymm13[0,1,2],ymm14[3,4,5,6,7]
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} xmm11 = xmm11[0],xmm12[1],xmm11[2,3]
+; AVX512F-SLOW-NEXT:    vmovdqa64 %ymm21, %ymm2
+; AVX512F-SLOW-NEXT:    vpshufb %xmm2, %xmm11, %xmm11
+; AVX512F-SLOW-NEXT:    vinserti128 $1, %xmm11, %ymm0, %ymm11
+; AVX512F-SLOW-NEXT:    vpternlogq $184, %zmm13, %zmm18, %zmm11
+; AVX512F-SLOW-NEXT:    vinserti64x4 $1, %ymm7, %zmm11, %zmm7
+; AVX512F-SLOW-NEXT:    vmovdqa64 %ymm20, %ymm2
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm2 = ymm3[0],ymm2[1,2],ymm3[3],ymm2[4],ymm3[5],ymm2[6,7],ymm3[8],ymm2[9,10],ymm3[11],ymm2[12],ymm3[13],ymm2[14,15]
+; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm2, %xmm3
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2],ymm2[3,4,5,6,7]
+; AVX512F-SLOW-NEXT:    vmovdqa64 %xmm22, %xmm3
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} xmm3 = xmm3[0,1],xmm6[2],xmm3[3]
+; AVX512F-SLOW-NEXT:    vpshufb {{.*#+}} ymm2 = ymm2[8,9,14,15,4,5,6,7,0,1,10,11,4,5,14,15,24,25,30,31,20,21,22,23,16,17,26,27,20,21,30,31]
+; AVX512F-SLOW-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[8,9,2,3,12,13,u,u,u,u,u,u,u,u,u,u]
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1,2],xmm2[3,4,5,6,7]
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
+; AVX512F-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512F-SLOW-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm2 = ymm10[0],ymm8[1],ymm10[2,3],ymm8[4],ymm10[5],ymm8[6],ymm10[7,8],ymm8[9],ymm10[10,11],ymm8[12],ymm10[13],ymm8[14],ymm10[15]
+; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm2, %xmm3
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2],xmm2[3,4],xmm3[5,6,7]
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm3 = ymm5[0,1],ymm4[2],ymm5[3],ymm4[4],ymm5[5,6],ymm4[7],ymm5[8,9],ymm4[10],ymm5[11],ymm4[12],ymm5[13,14],ymm4[15]
+; AVX512F-SLOW-NEXT:    vpermq {{.*#+}} ymm4 = ymm3[2,3,0,1]
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm4[4],ymm3[5,6],ymm4[7]
+; AVX512F-SLOW-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[8,9,2,3,12,13,6,7,0,1,10,11,u,u,u,u]
+; AVX512F-SLOW-NEXT:    vpshufb {{.*#+}} ymm3 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,4,5,14,15,24,25,18,19,28,29,22,23,20,21,22,23,20,21,30,31]
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2],ymm3[3,4,5,6,7]
 ; AVX512F-SLOW-NEXT:    movb $7, %al
 ; AVX512F-SLOW-NEXT:    kmovw %eax, %k1
-; AVX512F-SLOW-NEXT:    vinserti64x4 $0, %ymm2, %zmm0, %zmm7 {%k1}
-; AVX512F-SLOW-NEXT:    vextracti64x4 $1, %zmm7, %ymm2
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4,5],ymm0[6],ymm1[7,8],ymm0[9],ymm1[10],ymm0[11],ymm1[12,13],ymm0[14],ymm1[15]
-; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5,6],xmm1[7]
-; AVX512F-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,2,3,12,13,6,7,0,1,10,11,4,5,14,15]
-; AVX512F-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm2[0],ymm0[1,2,3,4,5,6,7],ymm2[8],ymm0[9,10,11,12,13,14,15]
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
-; AVX512F-SLOW-NEXT:    vinserti64x4 $1, %ymm0, %zmm7, %zmm0
+; AVX512F-SLOW-NEXT:    vinserti64x4 $0, %ymm2, %zmm0, %zmm0 {%k1}
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0],ymm9[1],ymm1[2],ymm9[3],ymm1[4,5],ymm9[6],ymm1[7,8],ymm9[9],ymm1[10],ymm9[11],ymm1[12,13],ymm9[14],ymm1[15]
+; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3],xmm1[4,5,6],xmm2[7]
+; AVX512F-SLOW-NEXT:    vextracti64x4 $1, %zmm0, %ymm2
+; AVX512F-SLOW-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[u,u,2,3,12,13,6,7,0,1,10,11,4,5,14,15]
+; AVX512F-SLOW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0],ymm1[1,2,3,4,5,6,7],ymm2[8],ymm1[9,10,11,12,13,14,15]
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-SLOW-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
 ; AVX512F-SLOW-NEXT:    vmovdqa64 %zmm16, (%rsi)
-; AVX512F-SLOW-NEXT:    vmovdqa64 %zmm18, (%rdx)
-; AVX512F-SLOW-NEXT:    vmovdqa64 %zmm17, (%rcx)
-; AVX512F-SLOW-NEXT:    vmovdqa64 %zmm4, (%r8)
+; AVX512F-SLOW-NEXT:    vmovdqa64 %zmm19, (%rdx)
+; AVX512F-SLOW-NEXT:    vmovdqa64 %zmm7, (%rcx)
+; AVX512F-SLOW-NEXT:    vmovdqa64 %zmm17, (%r8)
 ; AVX512F-SLOW-NEXT:    vmovdqa64 %zmm0, (%r9)
 ; AVX512F-SLOW-NEXT:    vzeroupper
 ; AVX512F-SLOW-NEXT:    retq
@@ -3396,104 +3412,103 @@ define void @load_i16_stride5_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512F-FAST-NEXT:    vpor %ymm7, %ymm12, %ymm12
 ; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} ymm13 = <0,3,1,u,0,3,5,u>
 ; AVX512F-FAST-NEXT:    vmovdqa 128(%rdi), %ymm7
-; AVX512F-FAST-NEXT:    vpermd %ymm7, %ymm13, %ymm14
-; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} ymm13 = [4,5,2,3,4,5,6,7,8,9,2,3,4,5,10,11,20,21,18,19,20,21,22,23,24,25,18,19,20,21,26,27]
-; AVX512F-FAST-NEXT:    vpshufb %ymm13, %ymm14, %ymm14
-; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm15 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535]
-; AVX512F-FAST-NEXT:    vpternlogq $184, %zmm12, %zmm15, %zmm14
-; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm6, %zmm14, %zmm16
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm12 = ymm9[0],ymm8[1],ymm9[2,3],ymm8[4],ymm9[5],ymm8[6],ymm9[7,8],ymm8[9],ymm9[10,11],ymm8[12],ymm9[13],ymm8[14],ymm9[15]
-; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} ymm14 = <2,u,u,u,4,7,1,6>
-; AVX512F-FAST-NEXT:    vpermd %ymm12, %ymm14, %ymm12
-; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} ymm12 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm12[0,1,18,19,20,21,26,27,16,17,30,31],zero,zero,zero,zero,zero,zero
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm14 = ymm11[0],ymm10[1],ymm11[2],ymm10[3],ymm11[4,5],ymm10[6],ymm11[7,8],ymm10[9],ymm11[10],ymm10[11],ymm11[12,13],ymm10[14],ymm11[15]
-; AVX512F-FAST-NEXT:    vextracti128 $1, %ymm14, %xmm6
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} xmm6 = xmm14[0,1],xmm6[2,3],xmm14[4,5,6],xmm6[7]
-; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} ymm6 = ymm6[2,3,12,13,6,7,0,1,10,11,4,5,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm6[u,u,u,u,u,u]
-; AVX512F-FAST-NEXT:    vpor %ymm6, %ymm12, %ymm6
-; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} ymm12 = <1,3,2,u,1,3,6,u>
-; AVX512F-FAST-NEXT:    vpermd %ymm7, %ymm12, %ymm12
-; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} ymm14 = [0,1,6,7,4,5,6,7,8,9,0,1,6,7,8,9,16,17,22,23,20,21,22,23,24,25,16,17,22,23,24,25]
-; AVX512F-FAST-NEXT:    vpshufb %ymm14, %ymm12, %ymm12
-; AVX512F-FAST-NEXT:    vmovdqa64 %ymm14, %ymm19
-; AVX512F-FAST-NEXT:    vpternlogq $184, %zmm6, %zmm15, %zmm12
-; AVX512F-FAST-NEXT:    vpsrlq $48, %xmm2, %xmm6
-; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} xmm14 = xmm3[2,3,12,13,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512F-FAST-NEXT:    vpunpckldq {{.*#+}} xmm6 = xmm14[0],xmm6[0],xmm14[1],xmm6[1]
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm14 = ymm4[0],ymm5[1],ymm4[2,3],ymm5[4],ymm4[5],ymm5[6],ymm4[7,8],ymm5[9],ymm4[10,11],ymm5[12],ymm4[13],ymm5[14],ymm4[15]
+; AVX512F-FAST-NEXT:    vpermd %ymm7, %ymm13, %ymm13
+; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} ymm14 = [4,5,2,3,4,5,6,7,8,9,2,3,4,5,10,11,20,21,18,19,20,21,22,23,24,25,18,19,20,21,26,27]
+; AVX512F-FAST-NEXT:    vpshufb %ymm14, %ymm13, %ymm13
+; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm18 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535]
+; AVX512F-FAST-NEXT:    vpternlogq $184, %zmm12, %zmm18, %zmm13
+; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm6, %zmm13, %zmm16
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm12 = ymm11[0],ymm10[1],ymm11[2],ymm10[3],ymm11[4,5],ymm10[6],ymm11[7,8],ymm10[9],ymm11[10],ymm10[11],ymm11[12,13],ymm10[14],ymm11[15]
+; AVX512F-FAST-NEXT:    vextracti128 $1, %ymm12, %xmm13
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} xmm12 = xmm12[0,1],xmm13[2,3],xmm12[4,5,6],xmm13[7]
+; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} ymm6 = <2,3,12,13,6,7,0,1,10,11,4,5,14,15,128,128,128,128,128,128,128,128,128,128,128,128,u,u,u,u,u,u>
+; AVX512F-FAST-NEXT:    vpshufb %ymm6, %ymm12, %ymm12
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm13 = ymm9[0],ymm8[1],ymm9[2,3],ymm8[4],ymm9[5],ymm8[6],ymm9[7,8],ymm8[9],ymm9[10,11],ymm8[12],ymm9[13],ymm8[14],ymm9[15]
+; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} ymm17 = <2,u,u,u,4,7,1,6>
+; AVX512F-FAST-NEXT:    vpermd %ymm13, %ymm17, %ymm13
+; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} ymm13 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm13[0,1,18,19,20,21,26,27,16,17,30,31],zero,zero,zero,zero,zero,zero
+; AVX512F-FAST-NEXT:    vpor %ymm13, %ymm12, %ymm12
+; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} ymm13 = <1,3,2,u,1,3,6,u>
+; AVX512F-FAST-NEXT:    vpermd %ymm7, %ymm13, %ymm15
+; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} ymm13 = [0,1,6,7,4,5,6,7,8,9,0,1,6,7,8,9,16,17,22,23,20,21,22,23,24,25,16,17,22,23,24,25]
+; AVX512F-FAST-NEXT:    vpshufb %ymm13, %ymm15, %ymm15
+; AVX512F-FAST-NEXT:    vpternlogq $184, %zmm12, %zmm18, %zmm15
+; AVX512F-FAST-NEXT:    vpshufb %xmm6, %xmm3, %xmm6
+; AVX512F-FAST-NEXT:    vpsrlq $48, %xmm2, %xmm12
+; AVX512F-FAST-NEXT:    vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm12[0],xmm6[1],xmm12[1]
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm12 = ymm4[0],ymm5[1],ymm4[2,3],ymm5[4],ymm4[5],ymm5[6],ymm4[7,8],ymm5[9],ymm4[10,11],ymm5[12],ymm4[13],ymm5[14],ymm4[15]
 ; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} ymm17 = <0,2,5,7,4,7,u,u>
-; AVX512F-FAST-NEXT:    vpermd %ymm14, %ymm17, %ymm14
-; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} ymm14 = ymm14[2,3,4,5,4,5,0,1,6,7,8,9,14,15,4,5,18,19,20,21,20,21,16,17,22,23,24,25,30,31,20,21]
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} xmm6 = xmm6[0,1,2],xmm14[3,4,5,6,7]
-; AVX512F-FAST-NEXT:    vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3],ymm14[4,5,6,7]
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm14 = ymm0[0],ymm1[1,2],ymm0[3],ymm1[4],ymm0[5],ymm1[6,7],ymm0[8],ymm1[9,10],ymm0[11],ymm1[12],ymm0[13],ymm1[14,15]
+; AVX512F-FAST-NEXT:    vpermd %ymm12, %ymm17, %ymm12
+; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} ymm12 = ymm12[2,3,4,5,4,5,0,1,6,7,8,9,14,15,4,5,18,19,20,21,20,21,16,17,22,23,24,25,30,31,20,21]
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} xmm6 = xmm6[0,1,2],xmm12[3,4,5,6,7]
+; AVX512F-FAST-NEXT:    vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3],ymm12[4,5,6,7]
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm12 = ymm0[0],ymm1[1,2],ymm0[3],ymm1[4],ymm0[5],ymm1[6,7],ymm0[8],ymm1[9,10],ymm0[11],ymm1[12],ymm0[13],ymm1[14,15]
 ; AVX512F-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} ymm17 = [1,4,6,3,1,4,6,3]
 ; AVX512F-FAST-NEXT:    # ymm17 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-FAST-NEXT:    vpermd %ymm14, %ymm17, %ymm14
-; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} ymm14 = ymm14[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,19,20,21,26,27,16,17,30,31,24,25]
-; AVX512F-FAST-NEXT:    vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3,4],ymm14[5,6,7]
-; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm6, %zmm12, %zmm17
+; AVX512F-FAST-NEXT:    vpermd %ymm12, %ymm17, %ymm12
+; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} ymm12 = ymm12[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,19,20,21,26,27,16,17,30,31,24,25]
+; AVX512F-FAST-NEXT:    vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3,4],ymm12[5,6,7]
+; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm6, %zmm15, %zmm17
 ; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm6 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5],ymm0[6],ymm1[7,8],ymm0[9],ymm1[10,11],ymm0[12],ymm1[13],ymm0[14],ymm1[15]
-; AVX512F-FAST-NEXT:    vextracti128 $1, %ymm6, %xmm14
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} xmm6 = xmm14[0,1,2],xmm6[3,4],xmm14[5,6,7]
+; AVX512F-FAST-NEXT:    vextracti128 $1, %ymm6, %xmm15
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} xmm6 = xmm15[0,1,2],xmm6[3,4],xmm15[5,6,7]
 ; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} xmm6 = xmm6[u,u,u,u,8,9,2,3,12,13,6,7,0,1,10,11]
 ; AVX512F-FAST-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
-; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} xmm14 = xmm3[u,u,u,u,u,u,u,u,4,5,14,15,u,u,u,u]
-; AVX512F-FAST-NEXT:    vpunpckhdq {{.*#+}} xmm14 = xmm14[2],xmm2[2],xmm14[3],xmm2[3]
+; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} xmm15 = xmm3[u,u,u,u,u,u,u,u,4,5,14,15,u,u,u,u]
+; AVX512F-FAST-NEXT:    vpunpckhdq {{.*#+}} xmm15 = xmm15[2],xmm2[2],xmm15[3],xmm2[3]
 ; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm12 = ymm5[0],ymm4[1],ymm5[2],ymm4[3],ymm5[4,5],ymm4[6],ymm5[7,8],ymm4[9],ymm5[10],ymm4[11],ymm5[12,13],ymm4[14],ymm5[15]
-; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} ymm18 = <0,3,5,2,5,7,u,u>
-; AVX512F-FAST-NEXT:    vpermd %ymm12, %ymm18, %ymm12
+; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} ymm19 = <0,3,5,2,5,7,u,u>
+; AVX512F-FAST-NEXT:    vpermd %ymm12, %ymm19, %ymm12
 ; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} ymm12 = ymm12[0,1,6,7,2,3,2,3,4,5,10,11,0,1,14,15,16,17,22,23,18,19,18,19,20,21,26,27,16,17,30,31]
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} xmm14 = xmm14[0,1,2],xmm12[3,4,5,6,7]
-; AVX512F-FAST-NEXT:    vpblendd {{.*#+}} ymm12 = ymm14[0,1,2,3],ymm12[4,5,6,7]
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} xmm15 = xmm15[0,1,2],xmm12[3,4,5,6,7]
+; AVX512F-FAST-NEXT:    vpblendd {{.*#+}} ymm12 = ymm15[0,1,2,3],ymm12[4,5,6,7]
 ; AVX512F-FAST-NEXT:    vpblendd {{.*#+}} ymm6 = ymm12[0,1,2,3,4],ymm6[5,6,7]
 ; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm12 = ymm11[0,1],ymm10[2],ymm11[3],ymm10[4],ymm11[5,6],ymm10[7],ymm11[8,9],ymm10[10],ymm11[11],ymm10[12],ymm11[13,14],ymm10[15]
-; AVX512F-FAST-NEXT:    vextracti128 $1, %ymm12, %xmm14
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} xmm12 = xmm12[0,1,2],xmm14[3,4],xmm12[5,6,7]
+; AVX512F-FAST-NEXT:    vextracti128 $1, %ymm12, %xmm15
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} xmm12 = xmm12[0,1,2],xmm15[3,4],xmm12[5,6,7]
 ; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} xmm12 = xmm12[4,5,14,15,8,9,2,3,12,13,6,7,u,u,u,u]
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm14 = ymm8[0],ymm9[1],ymm8[2,3],ymm9[4],ymm8[5],ymm9[6],ymm8[7,8],ymm9[9],ymm8[10,11],ymm9[12],ymm8[13],ymm9[14],ymm8[15]
-; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} ymm18 = <0,2,u,u,5,7,2,4>
-; AVX512F-FAST-NEXT:    vpermd %ymm14, %ymm18, %ymm14
-; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} ymm14 = ymm14[u,u,u,u,u,u,u,u,u,u,u,u,0,1,6,7,16,17,22,23,24,25,30,31,20,21,22,23,16,17,22,23]
-; AVX512F-FAST-NEXT:    vpblendd {{.*#+}} ymm12 = ymm12[0,1,2],ymm14[3,4,5,6,7]
-; AVX512F-FAST-NEXT:    vbroadcasti128 {{.*#+}} ymm14 = [1,4,6,0,1,4,6,0]
-; AVX512F-FAST-NEXT:    # ymm14 = mem[0,1,0,1]
-; AVX512F-FAST-NEXT:    vpermd %ymm7, %ymm14, %ymm14
-; AVX512F-FAST-NEXT:    vpshufb %ymm13, %ymm14, %ymm13
-; AVX512F-FAST-NEXT:    vpternlogq $184, %zmm12, %zmm15, %zmm13
-; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm6, %zmm13, %zmm13
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm15 = ymm8[0],ymm9[1],ymm8[2,3],ymm9[4],ymm8[5],ymm9[6],ymm8[7,8],ymm9[9],ymm8[10,11],ymm9[12],ymm8[13],ymm9[14],ymm8[15]
+; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} ymm19 = <0,2,u,u,5,7,2,4>
+; AVX512F-FAST-NEXT:    vpermd %ymm15, %ymm19, %ymm15
+; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} ymm15 = ymm15[u,u,u,u,u,u,u,u,u,u,u,u,0,1,6,7,16,17,22,23,24,25,30,31,20,21,22,23,16,17,22,23]
+; AVX512F-FAST-NEXT:    vpblendd {{.*#+}} ymm12 = ymm12[0,1,2],ymm15[3,4,5,6,7]
+; AVX512F-FAST-NEXT:    vbroadcasti128 {{.*#+}} ymm15 = [1,4,6,0,1,4,6,0]
+; AVX512F-FAST-NEXT:    # ymm15 = mem[0,1,0,1]
+; AVX512F-FAST-NEXT:    vpermd %ymm7, %ymm15, %ymm15
+; AVX512F-FAST-NEXT:    vpshufb %ymm14, %ymm15, %ymm14
+; AVX512F-FAST-NEXT:    vpternlogq $184, %zmm12, %zmm18, %zmm14
+; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm6, %zmm14, %zmm14
 ; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm6 = ymm10[0],ymm11[1,2],ymm10[3],ymm11[4],ymm10[5],ymm11[6,7],ymm10[8],ymm11[9,10],ymm10[11],ymm11[12],ymm10[13],ymm11[14,15]
 ; AVX512F-FAST-NEXT:    vextracti128 $1, %ymm6, %xmm12
 ; AVX512F-FAST-NEXT:    vpblendd {{.*#+}} xmm6 = xmm12[0],xmm6[1],xmm12[2],xmm6[3]
 ; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} xmm6 = xmm6[6,7,0,1,10,11,4,5,14,15,8,9,u,u,u,u]
 ; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm12 = ymm9[0],ymm8[1],ymm9[2],ymm8[3],ymm9[4,5],ymm8[6],ymm9[7,8],ymm8[9],ymm9[10],ymm8[11],ymm9[12,13],ymm8[14],ymm9[15]
-; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} ymm14 = <0,3,u,u,5,0,2,7>
-; AVX512F-FAST-NEXT:    vpermd %ymm12, %ymm14, %ymm12
+; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} ymm15 = <0,3,u,u,5,0,2,7>
+; AVX512F-FAST-NEXT:    vpermd %ymm12, %ymm15, %ymm12
 ; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} ymm12 = ymm12[u,u,u,u,u,u,u,u,u,u,u,u,2,3,4,5,18,19,20,21,26,27,16,17,30,31,30,31,18,19,20,21]
 ; AVX512F-FAST-NEXT:    vpblendd {{.*#+}} ymm6 = ymm6[0,1,2],ymm12[3,4,5,6,7]
 ; AVX512F-FAST-NEXT:    vpblendd {{.*#+}} xmm12 = xmm2[0],xmm3[1],xmm2[2,3]
 ; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} xmm12 = xmm12[6,7,0,1,10,11,u,u,u,u,u,u,u,u,u,u]
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm14 = ymm5[0,1],ymm4[2],ymm5[3],ymm4[4],ymm5[5,6],ymm4[7],ymm5[8,9],ymm4[10],ymm5[11],ymm4[12],ymm5[13,14],ymm4[15]
-; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} ymm15 = <1,3,6,0,5,u,u,u>
-; AVX512F-FAST-NEXT:    vpermd %ymm14, %ymm15, %ymm14
-; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} ymm14 = ymm14[2,3,2,3,4,5,0,1,6,7,8,9,14,15,4,5,18,19,18,19,20,21,16,17,22,23,24,25,30,31,20,21]
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} xmm12 = xmm12[0,1,2],xmm14[3,4,5,6,7]
-; AVX512F-FAST-NEXT:    vpblendd {{.*#+}} ymm12 = ymm12[0,1,2,3],ymm14[4,5,6,7]
-; AVX512F-FAST-NEXT:    vbroadcasti128 {{.*#+}} ymm14 = [2,4,7,0,2,4,7,0]
-; AVX512F-FAST-NEXT:    # ymm14 = mem[0,1,0,1]
-; AVX512F-FAST-NEXT:    vpermd %ymm7, %ymm14, %ymm14
-; AVX512F-FAST-NEXT:    vmovdqa64 %ymm19, %ymm15
-; AVX512F-FAST-NEXT:    vpshufb %ymm15, %ymm14, %ymm14
-; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm12, %zmm14, %zmm12
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm15 = ymm5[0,1],ymm4[2],ymm5[3],ymm4[4],ymm5[5,6],ymm4[7],ymm5[8,9],ymm4[10],ymm5[11],ymm4[12],ymm5[13,14],ymm4[15]
+; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} ymm18 = <1,3,6,0,5,u,u,u>
+; AVX512F-FAST-NEXT:    vpermd %ymm15, %ymm18, %ymm15
+; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} ymm15 = ymm15[2,3,2,3,4,5,0,1,6,7,8,9,14,15,4,5,18,19,18,19,20,21,16,17,22,23,24,25,30,31,20,21]
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} xmm12 = xmm12[0,1,2],xmm15[3,4,5,6,7]
+; AVX512F-FAST-NEXT:    vpblendd {{.*#+}} ymm12 = ymm12[0,1,2,3],ymm15[4,5,6,7]
+; AVX512F-FAST-NEXT:    vbroadcasti128 {{.*#+}} ymm15 = [2,4,7,0,2,4,7,0]
+; AVX512F-FAST-NEXT:    # ymm15 = mem[0,1,0,1]
+; AVX512F-FAST-NEXT:    vpermd %ymm7, %ymm15, %ymm15
+; AVX512F-FAST-NEXT:    vpshufb %ymm13, %ymm15, %ymm13
+; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm12, %zmm13, %zmm12
 ; AVX512F-FAST-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm6, %zmm12
 ; AVX512F-FAST-NEXT:    vextracti64x4 $1, %zmm12, %ymm6
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm14 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5],ymm1[6],ymm0[7,8],ymm1[9],ymm0[10,11],ymm1[12],ymm0[13],ymm1[14],ymm0[15]
-; AVX512F-FAST-NEXT:    vextracti128 $1, %ymm14, %xmm15
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} xmm14 = xmm14[0],xmm15[1,2,3],xmm14[4,5],xmm15[6,7]
-; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} xmm14 = xmm14[u,u,0,1,10,11,4,5,14,15,8,9,2,3,12,13]
-; AVX512F-FAST-NEXT:    vinserti128 $1, %xmm14, %ymm0, %ymm14
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm14 = ymm6[0],ymm14[1,2,3,4,5,6,7],ymm6[8],ymm14[9,10,11,12,13,14,15]
-; AVX512F-FAST-NEXT:    vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3],ymm14[4,5,6,7]
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm13 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5],ymm1[6],ymm0[7,8],ymm1[9],ymm0[10,11],ymm1[12],ymm0[13],ymm1[14],ymm0[15]
+; AVX512F-FAST-NEXT:    vextracti128 $1, %ymm13, %xmm15
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} xmm13 = xmm13[0],xmm15[1,2,3],xmm13[4,5],xmm15[6,7]
+; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} xmm13 = xmm13[u,u,0,1,10,11,4,5,14,15,8,9,2,3,12,13]
+; AVX512F-FAST-NEXT:    vinserti128 $1, %xmm13, %ymm0, %ymm13
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm13 = ymm6[0],ymm13[1,2,3,4,5,6,7],ymm6[8],ymm13[9,10,11,12,13,14,15]
+; AVX512F-FAST-NEXT:    vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3],ymm13[4,5,6,7]
 ; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm6, %zmm12, %zmm6
 ; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm10 = ymm11[0],ymm10[1],ymm11[2,3],ymm10[4],ymm11[5],ymm10[6],ymm11[7,8],ymm10[9],ymm11[10,11],ymm10[12],ymm11[13],ymm10[14],ymm11[15]
 ; AVX512F-FAST-NEXT:    vextracti128 $1, %ymm10, %xmm11
@@ -3530,7 +3545,7 @@ define void @load_i16_stride5_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm0, %zmm2, %zmm0
 ; AVX512F-FAST-NEXT:    vmovdqa64 %zmm16, (%rsi)
 ; AVX512F-FAST-NEXT:    vmovdqa64 %zmm17, (%rdx)
-; AVX512F-FAST-NEXT:    vmovdqa64 %zmm13, (%rcx)
+; AVX512F-FAST-NEXT:    vmovdqa64 %zmm14, (%rcx)
 ; AVX512F-FAST-NEXT:    vmovdqa64 %zmm6, (%r8)
 ; AVX512F-FAST-NEXT:    vmovdqa64 %zmm0, (%r9)
 ; AVX512F-FAST-NEXT:    vzeroupper
@@ -6686,13 +6701,13 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ;
 ; AVX512F-SLOW-LABEL: load_i16_stride5_vf64:
 ; AVX512F-SLOW:       # %bb.0:
-; AVX512F-SLOW-NEXT:    subq $584, %rsp # imm = 0x248
+; AVX512F-SLOW-NEXT:    subq $552, %rsp # imm = 0x228
 ; AVX512F-SLOW-NEXT:    vmovdqa 384(%rdi), %ymm1
 ; AVX512F-SLOW-NEXT:    vmovdqa 416(%rdi), %ymm2
 ; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0],ymm2[1,2],ymm1[3],ymm2[4],ymm1[5],ymm2[6,7],ymm1[8],ymm2[9,10],ymm1[11],ymm2[12],ymm1[13],ymm2[14,15]
-; AVX512F-SLOW-NEXT:    vmovdqa %ymm2, %ymm6
+; AVX512F-SLOW-NEXT:    vmovdqa %ymm2, %ymm11
 ; AVX512F-SLOW-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512F-SLOW-NEXT:    vmovdqa %ymm1, %ymm7
+; AVX512F-SLOW-NEXT:    vmovdqa %ymm1, %ymm6
 ; AVX512F-SLOW-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512F-SLOW-NEXT:    vpermq {{.*#+}} ymm1 = ymm0[2,3,0,1]
 ; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3,4],ymm1[5],ymm0[6],ymm1[7]
@@ -6701,25 +6716,25 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512F-SLOW-NEXT:    vmovdqa 352(%rdi), %ymm3
 ; AVX512F-SLOW-NEXT:    vmovdqa 320(%rdi), %ymm4
 ; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm1 = ymm4[0],ymm3[1],ymm4[2,3],ymm3[4],ymm4[5],ymm3[6],ymm4[7,8],ymm3[9],ymm4[10,11],ymm3[12],ymm4[13],ymm3[14],ymm4[15]
-; AVX512F-SLOW-NEXT:    vmovdqa %ymm4, %ymm8
+; AVX512F-SLOW-NEXT:    vmovdqa %ymm4, %ymm7
 ; AVX512F-SLOW-NEXT:    vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512F-SLOW-NEXT:    vmovdqa %ymm3, %ymm9
+; AVX512F-SLOW-NEXT:    vmovdqa %ymm3, %ymm8
 ; AVX512F-SLOW-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm1, %xmm3
 ; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} xmm3 = xmm1[0],xmm3[1,2,3],xmm1[4,5],xmm3[6,7]
 ; AVX512F-SLOW-NEXT:    vmovdqa {{.*#+}} ymm1 = <0,1,10,11,4,5,14,15,8,9,2,3,12,13,128,128,128,128,128,128,128,128,128,128,128,128,u,u,u,u,u,u>
 ; AVX512F-SLOW-NEXT:    vpshufb %ymm1, %ymm3, %ymm3
 ; AVX512F-SLOW-NEXT:    vporq %ymm2, %ymm3, %ymm19
-; AVX512F-SLOW-NEXT:    vmovdqa 192(%rdi), %ymm12
-; AVX512F-SLOW-NEXT:    vmovdqa 224(%rdi), %ymm15
-; AVX512F-SLOW-NEXT:    vmovdqa 176(%rdi), %xmm13
+; AVX512F-SLOW-NEXT:    vmovdqa 192(%rdi), %ymm15
+; AVX512F-SLOW-NEXT:    vmovdqa 224(%rdi), %ymm13
+; AVX512F-SLOW-NEXT:    vmovdqa 176(%rdi), %xmm12
 ; AVX512F-SLOW-NEXT:    vmovdqa 160(%rdi), %xmm14
 ; AVX512F-SLOW-NEXT:    vmovdqa (%rdi), %ymm4
 ; AVX512F-SLOW-NEXT:    vmovdqa 32(%rdi), %ymm5
 ; AVX512F-SLOW-NEXT:    vmovdqa 64(%rdi), %ymm3
-; AVX512F-SLOW-NEXT:    vmovdqa 96(%rdi), %ymm10
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm2 = ymm3[0],ymm10[1,2],ymm3[3],ymm10[4],ymm3[5],ymm10[6,7],ymm3[8],ymm10[9,10],ymm3[11],ymm10[12],ymm3[13],ymm10[14,15]
-; AVX512F-SLOW-NEXT:    vmovdqa %ymm3, %ymm11
+; AVX512F-SLOW-NEXT:    vmovdqa 96(%rdi), %ymm9
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm2 = ymm3[0],ymm9[1,2],ymm3[3],ymm9[4],ymm3[5],ymm9[6,7],ymm3[8],ymm9[9,10],ymm3[11],ymm9[12],ymm3[13],ymm9[14,15]
+; AVX512F-SLOW-NEXT:    vmovdqa %ymm3, %ymm10
 ; AVX512F-SLOW-NEXT:    vpermq {{.*#+}} ymm3 = ymm2[2,3,0,1]
 ; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm3[5],ymm2[6],ymm3[7]
 ; AVX512F-SLOW-NEXT:    vpshufb %ymm0, %ymm2, %ymm0
@@ -6729,21 +6744,21 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512F-SLOW-NEXT:    vpshufb %ymm1, %ymm2, %ymm1
 ; AVX512F-SLOW-NEXT:    vpor %ymm0, %ymm1, %ymm0
 ; AVX512F-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm6[0],ymm7[1],ymm6[2,3],ymm7[4],ymm6[5],ymm7[6],ymm6[7,8],ymm7[9],ymm6[10,11],ymm7[12],ymm6[13],ymm7[14],ymm6[15]
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm11[0],ymm6[1],ymm11[2,3],ymm6[4],ymm11[5],ymm6[6],ymm11[7,8],ymm6[9],ymm11[10,11],ymm6[12],ymm11[13],ymm6[14],ymm11[15]
 ; AVX512F-SLOW-NEXT:    vpermq {{.*#+}} ymm1 = ymm0[2,3,0,1]
 ; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3,4],ymm1[5],ymm0[6,7]
 ; AVX512F-SLOW-NEXT:    vmovdqa {{.*#+}} ymm0 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,8,9,18,19,28,29,22,23,16,17,26,27,128,128,128,128,128,128]
 ; AVX512F-SLOW-NEXT:    vpshufb %ymm0, %ymm1, %ymm2
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm1 = ymm9[0],ymm8[1],ymm9[2],ymm8[3],ymm9[4,5],ymm8[6],ymm9[7,8],ymm8[9],ymm9[10],ymm8[11],ymm9[12,13],ymm8[14],ymm9[15]
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm1 = ymm8[0],ymm7[1],ymm8[2],ymm7[3],ymm8[4,5],ymm7[6],ymm8[7,8],ymm7[9],ymm8[10],ymm7[11],ymm8[12,13],ymm7[14],ymm8[15]
 ; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm1, %xmm3
 ; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} xmm3 = xmm1[0,1],xmm3[2,3],xmm1[4,5,6],xmm3[7]
 ; AVX512F-SLOW-NEXT:    vmovdqa {{.*#+}} ymm1 = <2,3,12,13,6,7,0,1,10,11,4,5,14,15,128,128,128,128,128,128,128,128,128,128,128,128,u,u,u,u,u,u>
 ; AVX512F-SLOW-NEXT:    vpshufb %ymm1, %ymm3, %ymm3
 ; AVX512F-SLOW-NEXT:    vpor %ymm2, %ymm3, %ymm2
 ; AVX512F-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm2 = ymm10[0],ymm11[1],ymm10[2,3],ymm11[4],ymm10[5],ymm11[6],ymm10[7,8],ymm11[9],ymm10[10,11],ymm11[12],ymm10[13],ymm11[14],ymm10[15]
-; AVX512F-SLOW-NEXT:    vmovdqu %ymm11, (%rsp) # 32-byte Spill
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm2 = ymm9[0],ymm10[1],ymm9[2,3],ymm10[4],ymm9[5],ymm10[6],ymm9[7,8],ymm10[9],ymm9[10,11],ymm10[12],ymm9[13],ymm10[14],ymm9[15]
 ; AVX512F-SLOW-NEXT:    vmovdqu %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512F-SLOW-NEXT:    vmovdqu %ymm9, (%rsp) # 32-byte Spill
 ; AVX512F-SLOW-NEXT:    vpermq {{.*#+}} ymm3 = ymm2[2,3,0,1]
 ; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm3[5],ymm2[6,7]
 ; AVX512F-SLOW-NEXT:    vpshufb %ymm0, %ymm2, %ymm0
@@ -6753,39 +6768,40 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm2, %xmm3
 ; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3],xmm2[4,5,6],xmm3[7]
 ; AVX512F-SLOW-NEXT:    vpshufb %ymm1, %ymm2, %ymm1
-; AVX512F-SLOW-NEXT:    vpor %ymm0, %ymm1, %ymm0
-; AVX512F-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm15[0,1],ymm12[2],ymm15[3],ymm12[4],ymm15[5,6],ymm12[7],ymm15[8,9],ymm12[10],ymm15[11],ymm12[12],ymm15[13,14],ymm12[15]
+; AVX512F-SLOW-NEXT:    vporq %ymm0, %ymm1, %ymm28
+; AVX512F-SLOW-NEXT:    vmovdqa {{.*#+}} ymm6 = [6,7,0,1,10,11,4,5,14,15,14,15,2,3,12,13,22,23,16,17,26,27,20,21,30,31,30,31,18,19,28,29]
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm13[0,1],ymm15[2],ymm13[3],ymm15[4],ymm13[5,6],ymm15[7],ymm13[8,9],ymm15[10],ymm13[11],ymm15[12],ymm13[13,14],ymm15[15]
+; AVX512F-SLOW-NEXT:    vmovdqa64 %ymm15, %ymm18
+; AVX512F-SLOW-NEXT:    vmovdqa64 %ymm13, %ymm24
 ; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2],ymm0[3,4,5,6,7]
 ; AVX512F-SLOW-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[6,7,6,7,8,9,4,5,14,15,8,9,2,3,12,13,22,23,22,23,24,25,20,21,30,31,24,25,18,19,28,29]
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} xmm1 = xmm13[0],xmm14[1],xmm13[2,3]
-; AVX512F-SLOW-NEXT:    vmovdqa64 %xmm14, %xmm26
-; AVX512F-SLOW-NEXT:    vmovdqa64 %xmm13, %xmm24
-; AVX512F-SLOW-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[6,7,0,1,10,11,u,u,u,u,u,u,u,u,u,u]
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} xmm1 = xmm12[0],xmm14[1],xmm12[2,3]
+; AVX512F-SLOW-NEXT:    vmovdqa64 %xmm12, %xmm16
+; AVX512F-SLOW-NEXT:    vmovdqa64 %xmm14, %xmm30
+; AVX512F-SLOW-NEXT:    vpshufb %xmm6, %xmm1, %xmm1
 ; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2],xmm0[3,4,5,6,7]
 ; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
 ; AVX512F-SLOW-NEXT:    vmovdqa 144(%rdi), %xmm2
 ; AVX512F-SLOW-NEXT:    vmovdqa 128(%rdi), %xmm3
 ; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm3[2],xmm2[3]
-; AVX512F-SLOW-NEXT:    vmovdqa64 %xmm3, %xmm18
-; AVX512F-SLOW-NEXT:    vmovdqa %xmm2, %xmm6
+; AVX512F-SLOW-NEXT:    vmovdqa %xmm3, %xmm7
+; AVX512F-SLOW-NEXT:    vmovdqa %xmm2, %xmm11
 ; AVX512F-SLOW-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,6,7,8,9,8,9,2,3,12,13]
 ; AVX512F-SLOW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
 ; AVX512F-SLOW-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm1 = ymm10[0],ymm11[1],ymm10[2],ymm11[3],ymm10[4,5],ymm11[6],ymm10[7,8],ymm11[9],ymm10[10],ymm11[11],ymm10[12,13],ymm11[14],ymm10[15]
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm1 = ymm9[0],ymm10[1],ymm9[2],ymm10[3],ymm9[4,5],ymm10[6],ymm9[7,8],ymm10[9],ymm9[10],ymm10[11],ymm9[12,13],ymm10[14],ymm9[15]
 ; AVX512F-SLOW-NEXT:    vpermq {{.*#+}} ymm2 = ymm1[2,3,0,1]
 ; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4],ymm1[5],ymm2[6],ymm1[7]
 ; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm2 = ymm4[0],ymm5[1,2],ymm4[3],ymm5[4],ymm4[5],ymm5[6,7],ymm4[8],ymm5[9,10],ymm4[11],ymm5[12],ymm4[13],ymm5[14,15]
 ; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm2, %xmm3
 ; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} xmm2 = xmm3[0],xmm2[1],xmm3[2],xmm2[3]
-; AVX512F-SLOW-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[6,7,0,1,10,11,4,5,14,15,14,15,2,3,12,13,22,23,16,17,26,27,20,21,30,31,30,31,18,19,28,29]
+; AVX512F-SLOW-NEXT:    vpshufb %ymm6, %ymm1, %ymm1
 ; AVX512F-SLOW-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[6,7,0,1,10,11,4,5,14,15,8,9,12,13,14,15]
 ; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2],ymm1[3,4,5,6,7]
-; AVX512F-SLOW-NEXT:    vmovdqa 256(%rdi), %ymm3
-; AVX512F-SLOW-NEXT:    vmovdqa 288(%rdi), %ymm4
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm2 = ymm3[0],ymm4[1],ymm3[2,3],ymm4[4],ymm3[5],ymm4[6],ymm3[7,8],ymm4[9],ymm3[10,11],ymm4[12],ymm3[13],ymm4[14],ymm3[15]
-; AVX512F-SLOW-NEXT:    vmovdqa %ymm3, %ymm5
+; AVX512F-SLOW-NEXT:    vmovdqa 256(%rdi), %ymm12
+; AVX512F-SLOW-NEXT:    vmovdqa 288(%rdi), %ymm15
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm2 = ymm12[0],ymm15[1],ymm12[2,3],ymm15[4],ymm12[5],ymm15[6],ymm12[7,8],ymm15[9],ymm12[10,11],ymm15[12],ymm12[13],ymm15[14],ymm12[15]
 ; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm2, %xmm3
 ; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0],xmm3[1,2,3],xmm2[4,5],xmm3[6,7]
 ; AVX512F-SLOW-NEXT:    vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm1
@@ -6796,294 +6812,294 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
 ; AVX512F-SLOW-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
 ; AVX512F-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-SLOW-NEXT:    vmovdqa 464(%rdi), %xmm13
-; AVX512F-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm13[3,1,2,3]
+; AVX512F-SLOW-NEXT:    vmovdqa 464(%rdi), %xmm8
+; AVX512F-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm8[3,1,2,3]
 ; AVX512F-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,1,2,1,4,5,6,7]
 ; AVX512F-SLOW-NEXT:    vmovdqa 448(%rdi), %xmm3
 ; AVX512F-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm3[0,2,2,3]
-; AVX512F-SLOW-NEXT:    vmovdqa %xmm3, %xmm7
 ; AVX512F-SLOW-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,1,0,3,4,5,6,7]
 ; AVX512F-SLOW-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
 ; AVX512F-SLOW-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512F-SLOW-NEXT:    vmovdqa %xmm6, %xmm3
-; AVX512F-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm6[3,1,2,3]
+; AVX512F-SLOW-NEXT:    vmovdqa %xmm11, %xmm6
+; AVX512F-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm11[3,1,2,3]
 ; AVX512F-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,1,2,1,4,5,6,7]
-; AVX512F-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm18[0,2,2,3]
+; AVX512F-SLOW-NEXT:    vmovdqa %xmm7, %xmm9
+; AVX512F-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm7[0,2,2,3]
 ; AVX512F-SLOW-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,1,0,3,4,5,6,7]
 ; AVX512F-SLOW-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
 ; AVX512F-SLOW-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm7[0],xmm13[1],xmm7[2,3]
-; AVX512F-SLOW-NEXT:    vmovdqa %xmm7, %xmm10
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm3[0],xmm8[1],xmm3[2,3]
+; AVX512F-SLOW-NEXT:    vmovdqa64 %xmm3, %xmm22
 ; AVX512F-SLOW-NEXT:    vmovdqa {{.*#+}} xmm1 = [6,7,2,3,4,5,6,7,6,7,2,3,12,13,6,7]
 ; AVX512F-SLOW-NEXT:    vpshufb %xmm1, %xmm0, %xmm0
-; AVX512F-SLOW-NEXT:    vmovdqa64 %xmm1, %xmm20
-; AVX512F-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm7
+; AVX512F-SLOW-NEXT:    vmovdqa64 %xmm1, %xmm23
+; AVX512F-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm10
 ; AVX512F-SLOW-NEXT:    vmovdqa 576(%rdi), %ymm1
 ; AVX512F-SLOW-NEXT:    vmovdqa 608(%rdi), %ymm2
 ; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm2[0,1],ymm1[2],ymm2[3],ymm1[4],ymm2[5,6],ymm1[7],ymm2[8,9],ymm1[10],ymm2[11],ymm1[12],ymm2[13,14],ymm1[15]
-; AVX512F-SLOW-NEXT:    vmovdqa64 %ymm2, %ymm17
-; AVX512F-SLOW-NEXT:    vmovdqa64 %ymm1, %ymm16
+; AVX512F-SLOW-NEXT:    vmovdqa64 %ymm2, %ymm20
+; AVX512F-SLOW-NEXT:    vmovdqa64 %ymm1, %ymm17
 ; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} xmm8 = xmm0[0,1,2],xmm1[3,4],xmm0[5,6,7]
-; AVX512F-SLOW-NEXT:    vmovdqa 512(%rdi), %ymm6
-; AVX512F-SLOW-NEXT:    vmovdqa 544(%rdi), %ymm2
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm2[0],ymm6[1],ymm2[2,3],ymm6[4],ymm2[5],ymm6[6],ymm2[7,8],ymm6[9],ymm2[10,11],ymm6[12],ymm2[13],ymm6[14],ymm2[15]
-; AVX512F-SLOW-NEXT:    vmovdqa %ymm2, %ymm11
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} xmm11 = xmm0[0,1,2],xmm1[3,4],xmm0[5,6,7]
+; AVX512F-SLOW-NEXT:    vmovdqa 512(%rdi), %ymm5
+; AVX512F-SLOW-NEXT:    vmovdqa 544(%rdi), %ymm13
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm13[0],ymm5[1],ymm13[2,3],ymm5[4],ymm13[5],ymm5[6],ymm13[7,8],ymm5[9],ymm13[10,11],ymm5[12],ymm13[13],ymm5[14],ymm13[15]
 ; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm1[0],ymm0[1,2],ymm1[3],ymm0[4,5,6,7]
-; AVX512F-SLOW-NEXT:    vmovdqa64 496(%rdi), %xmm27
-; AVX512F-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm27[3,1,2,3]
+; AVX512F-SLOW-NEXT:    vmovdqa64 496(%rdi), %xmm21
+; AVX512F-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm21[3,1,2,3]
 ; AVX512F-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm1[2,1,2,3,4,5,6,7]
-; AVX512F-SLOW-NEXT:    vmovdqa64 480(%rdi), %xmm28
-; AVX512F-SLOW-NEXT:    vpshufd {{.*#+}} xmm14 = xmm28[0,2,2,3]
+; AVX512F-SLOW-NEXT:    vmovdqa 480(%rdi), %xmm1
+; AVX512F-SLOW-NEXT:    vpshufd {{.*#+}} xmm14 = xmm1[0,2,2,3]
+; AVX512F-SLOW-NEXT:    vmovdqa %xmm1, %xmm7
 ; AVX512F-SLOW-NEXT:    vpshuflw {{.*#+}} xmm14 = xmm14[0,3,2,3,4,5,6,7]
 ; AVX512F-SLOW-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm14[0],xmm0[0],xmm14[1],xmm0[1]
 ; AVX512F-SLOW-NEXT:    vmovdqa {{.*#+}} ymm14 = <u,u,u,u,u,u,u,u,8,9,2,3,12,13,6,7,16,17,26,27,26,27,30,31,24,25,18,19,28,29,22,23>
 ; AVX512F-SLOW-NEXT:    vpshufb %ymm14, %ymm2, %ymm2
 ; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm2[2,3,4,5,6,7]
 ; AVX512F-SLOW-NEXT:    vmovdqa {{.*#+}} xmm2 = [12,13,14,15,4,5,14,15,8,9,2,3,12,13,6,7]
-; AVX512F-SLOW-NEXT:    vpshufb %xmm2, %xmm8, %xmm8
-; AVX512F-SLOW-NEXT:    vinserti128 $1, %xmm8, %ymm0, %ymm8
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm8[5,6,7]
-; AVX512F-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm25 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535]
-; AVX512F-SLOW-NEXT:    vpternlogq $184, %zmm19, %zmm25, %zmm7
-; AVX512F-SLOW-NEXT:    vinserti64x4 $1, %ymm0, %zmm7, %zmm0
+; AVX512F-SLOW-NEXT:    vpshufb %xmm2, %xmm11, %xmm11
+; AVX512F-SLOW-NEXT:    vinserti128 $1, %xmm11, %ymm0, %ymm11
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm11[5,6,7]
+; AVX512F-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm27 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535]
+; AVX512F-SLOW-NEXT:    vpternlogq $184, %zmm19, %zmm27, %zmm10
+; AVX512F-SLOW-NEXT:    vinserti64x4 $1, %ymm0, %zmm10, %zmm0
 ; AVX512F-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm4[0,1],ymm5[2],ymm4[3],ymm5[4],ymm4[5,6],ymm5[7],ymm4[8,9],ymm5[10],ymm4[11],ymm5[12],ymm4[13,14],ymm5[15]
-; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm7
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2],xmm7[3,4],xmm0[5,6,7]
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm15[0,1],ymm12[2],ymm15[3],ymm12[4],ymm15[5,6],ymm12[7],ymm15[8,9],ymm12[10],ymm15[11],ymm12[12],ymm15[13,14],ymm12[15]
+; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm10
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2],xmm10[3,4],xmm0[5,6,7]
 ; AVX512F-SLOW-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm2 = ymm15[0],ymm12[1],ymm15[2,3],ymm12[4],ymm15[5],ymm12[6],ymm15[7,8],ymm12[9],ymm15[10,11],ymm12[12],ymm15[13],ymm12[14],ymm15[15]
-; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm2, %xmm7
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm7[0],ymm2[1,2],ymm7[3],ymm2[4,5,6,7]
+; AVX512F-SLOW-NEXT:    vmovdqa64 %ymm18, %ymm3
+; AVX512F-SLOW-NEXT:    vmovdqa64 %ymm24, %ymm1
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm2 = ymm1[0],ymm3[1],ymm1[2,3],ymm3[4],ymm1[5],ymm3[6],ymm1[7,8],ymm3[9],ymm1[10,11],ymm3[12],ymm1[13],ymm3[14],ymm1[15]
+; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm2, %xmm10
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm10[0],ymm2[1,2],ymm10[3],ymm2[4,5,6,7]
 ; AVX512F-SLOW-NEXT:    vpshufb %ymm14, %ymm2, %ymm2
-; AVX512F-SLOW-NEXT:    vpshufd {{.*#+}} xmm7 = xmm24[3,1,2,3]
-; AVX512F-SLOW-NEXT:    vpshuflw {{.*#+}} xmm7 = xmm7[2,1,2,3,4,5,6,7]
-; AVX512F-SLOW-NEXT:    vpshufd {{.*#+}} xmm8 = xmm26[0,2,2,3]
-; AVX512F-SLOW-NEXT:    vpshuflw {{.*#+}} xmm8 = xmm8[0,3,2,3,4,5,6,7]
-; AVX512F-SLOW-NEXT:    vpunpckldq {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1]
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm7[0,1],ymm2[2,3,4,5,6,7]
+; AVX512F-SLOW-NEXT:    vpshufd {{.*#+}} xmm10 = xmm16[3,1,2,3]
+; AVX512F-SLOW-NEXT:    vpshuflw {{.*#+}} xmm10 = xmm10[2,1,2,3,4,5,6,7]
+; AVX512F-SLOW-NEXT:    vmovdqa64 %xmm30, %xmm24
+; AVX512F-SLOW-NEXT:    vpshufd {{.*#+}} xmm11 = xmm30[0,2,2,3]
+; AVX512F-SLOW-NEXT:    vpshuflw {{.*#+}} xmm11 = xmm11[0,3,2,3,4,5,6,7]
+; AVX512F-SLOW-NEXT:    vpunpckldq {{.*#+}} xmm10 = xmm11[0],xmm10[0],xmm11[1],xmm10[1]
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm10[0,1],ymm2[2,3,4,5,6,7]
 ; AVX512F-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
 ; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4],ymm0[5,6,7]
-; AVX512F-SLOW-NEXT:    vmovdqa64 %xmm18, %xmm1
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} xmm2 = xmm1[0],xmm3[1],xmm1[2,3]
-; AVX512F-SLOW-NEXT:    vmovdqa %xmm3, %xmm7
-; AVX512F-SLOW-NEXT:    vmovdqa64 %xmm20, %xmm3
-; AVX512F-SLOW-NEXT:    vpshufb %xmm3, %xmm2, %xmm2
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} xmm2 = xmm9[0],xmm6[1],xmm9[2,3]
+; AVX512F-SLOW-NEXT:    vmovdqa64 %xmm9, %xmm25
+; AVX512F-SLOW-NEXT:    vmovdqa64 %xmm23, %xmm4
+; AVX512F-SLOW-NEXT:    vpshufb %xmm4, %xmm2, %xmm2
 ; AVX512F-SLOW-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512F-SLOW-NEXT:    vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm25, %zmm2 # 64-byte Folded Reload
+; AVX512F-SLOW-NEXT:    vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm27, %zmm2 # 64-byte Folded Reload
 ; AVX512F-SLOW-NEXT:    vinserti64x4 $1, %ymm0, %zmm2, %zmm0
 ; AVX512F-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm10[0,1],xmm13[2],xmm10[3]
-; AVX512F-SLOW-NEXT:    vmovdqa64 %xmm10, %xmm29
-; AVX512F-SLOW-NEXT:    vmovdqa64 %xmm13, %xmm30
-; AVX512F-SLOW-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,1,2,3,4,5,6,7,8,9,4,5,14,15,8,9]
-; AVX512F-SLOW-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
-; AVX512F-SLOW-NEXT:    vmovdqa64 %xmm2, %xmm31
-; AVX512F-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm2 = ymm6[0],ymm11[1],ymm6[2,3],ymm11[4],ymm6[5],ymm11[6],ymm6[7,8],ymm11[9],ymm6[10,11],ymm11[12],ymm6[13],ymm11[14],ymm6[15]
-; AVX512F-SLOW-NEXT:    vmovdqa64 %ymm6, %ymm23
-; AVX512F-SLOW-NEXT:    vmovdqa64 %ymm11, %ymm19
-; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm2, %xmm8
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0],ymm8[1],ymm2[2],ymm8[3],ymm2[4,5,6,7]
-; AVX512F-SLOW-NEXT:    vpshufd {{.*#+}} xmm8 = xmm28[0,3,2,3]
-; AVX512F-SLOW-NEXT:    vpshuflw {{.*#+}} xmm8 = xmm8[1,2,2,3,4,5,6,7]
-; AVX512F-SLOW-NEXT:    vpsrlq $48, %xmm27, %xmm13
-; AVX512F-SLOW-NEXT:    vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm13[0],xmm8[1],xmm13[1]
+; AVX512F-SLOW-NEXT:    vmovdqa64 %xmm22, %xmm4
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} xmm2 = xmm4[0,1],xmm8[2],xmm4[3]
+; AVX512F-SLOW-NEXT:    vmovdqa {{.*#+}} xmm0 = [0,1,2,3,4,5,6,7,8,9,4,5,14,15,8,9]
+; AVX512F-SLOW-NEXT:    vpshufb %xmm0, %xmm2, %xmm2
+; AVX512F-SLOW-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm10 = ymm5[0],ymm13[1],ymm5[2,3],ymm13[4],ymm5[5],ymm13[6],ymm5[7,8],ymm13[9],ymm5[10,11],ymm13[12],ymm5[13],ymm13[14],ymm5[15]
+; AVX512F-SLOW-NEXT:    vmovdqa64 %ymm13, %ymm26
+; AVX512F-SLOW-NEXT:    vmovdqa64 %ymm5, %ymm22
+; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm10, %xmm11
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm10 = ymm10[0],ymm11[1],ymm10[2],ymm11[3],ymm10[4,5,6,7]
+; AVX512F-SLOW-NEXT:    vpshufd {{.*#+}} xmm11 = xmm7[0,3,2,3]
+; AVX512F-SLOW-NEXT:    vmovdqa64 %xmm7, %xmm30
+; AVX512F-SLOW-NEXT:    vpshuflw {{.*#+}} xmm11 = xmm11[1,2,2,3,4,5,6,7]
+; AVX512F-SLOW-NEXT:    vpsrlq $48, %xmm21, %xmm13
+; AVX512F-SLOW-NEXT:    vpunpckldq {{.*#+}} xmm11 = xmm11[0],xmm13[0],xmm11[1],xmm13[1]
 ; AVX512F-SLOW-NEXT:    vmovdqa {{.*#+}} ymm13 = [2,3,12,13,0,1,0,1,10,11,4,5,14,15,8,9,18,19,28,29,16,17,16,17,26,27,20,21,30,31,24,25]
-; AVX512F-SLOW-NEXT:    vpshufb %ymm13, %ymm2, %ymm2
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} xmm8 = xmm8[0,1,2],xmm2[3,4,5,6,7]
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm8[0,1,2,3],ymm2[4,5,6,7]
-; AVX512F-SLOW-NEXT:    vmovdqa64 %ymm16, %ymm6
-; AVX512F-SLOW-NEXT:    vmovdqa64 %ymm17, %ymm3
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm8 = ymm6[0],ymm3[1,2],ymm6[3],ymm3[4],ymm6[5],ymm3[6,7],ymm6[8],ymm3[9,10],ymm6[11],ymm3[12],ymm6[13],ymm3[14,15]
-; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm8, %xmm14
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} xmm8 = xmm14[0],xmm8[1],xmm14[2],xmm8[3]
+; AVX512F-SLOW-NEXT:    vpshufb %ymm13, %ymm10, %ymm10
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} xmm11 = xmm11[0,1,2],xmm10[3,4,5,6,7]
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm10 = ymm11[0,1,2,3],ymm10[4,5,6,7]
+; AVX512F-SLOW-NEXT:    vmovdqa64 %ymm20, %ymm9
+; AVX512F-SLOW-NEXT:    vmovdqa64 %ymm17, %ymm7
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm11 = ymm7[0],ymm9[1,2],ymm7[3],ymm9[4],ymm7[5],ymm9[6,7],ymm7[8],ymm9[9,10],ymm7[11],ymm9[12],ymm7[13],ymm9[14,15]
+; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm11, %xmm14
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} xmm11 = xmm14[0],xmm11[1],xmm14[2],xmm11[3]
 ; AVX512F-SLOW-NEXT:    vmovdqa {{.*#+}} xmm14 = [0,1,2,3,6,7,0,1,10,11,4,5,14,15,8,9]
-; AVX512F-SLOW-NEXT:    vpshufb %xmm14, %xmm8, %xmm8
-; AVX512F-SLOW-NEXT:    vinserti128 $1, %xmm8, %ymm0, %ymm8
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm8[5,6,7]
-; AVX512F-SLOW-NEXT:    vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm25, %zmm0 # 64-byte Folded Reload
+; AVX512F-SLOW-NEXT:    vpshufb %xmm14, %xmm11, %xmm11
+; AVX512F-SLOW-NEXT:    vinserti128 $1, %xmm11, %ymm0, %ymm11
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm10 = ymm10[0,1,2,3,4],ymm11[5,6,7]
+; AVX512F-SLOW-NEXT:    vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm27, %zmm2 # 64-byte Folded Reload
+; AVX512F-SLOW-NEXT:    vinserti64x4 $1, %ymm10, %zmm2, %zmm2
+; AVX512F-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm2 = ymm3[0],ymm1[1],ymm3[2,3],ymm1[4],ymm3[5],ymm1[6],ymm3[7,8],ymm1[9],ymm3[10,11],ymm1[12],ymm3[13],ymm1[14],ymm3[15]
+; AVX512F-SLOW-NEXT:    vmovdqa64 %ymm1, %ymm17
+; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm2, %xmm10
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0],ymm10[1],ymm2[2],ymm10[3],ymm2[4,5,6,7]
+; AVX512F-SLOW-NEXT:    vpshufb %ymm13, %ymm2, %ymm2
+; AVX512F-SLOW-NEXT:    vpshufd {{.*#+}} xmm10 = xmm24[0,3,2,3]
+; AVX512F-SLOW-NEXT:    vmovdqa64 %xmm24, %xmm20
+; AVX512F-SLOW-NEXT:    vpshuflw {{.*#+}} xmm10 = xmm10[1,2,2,3,4,5,6,7]
+; AVX512F-SLOW-NEXT:    vmovdqa64 %xmm16, %xmm1
+; AVX512F-SLOW-NEXT:    vpsrlq $48, %xmm16, %xmm11
+; AVX512F-SLOW-NEXT:    vpunpckldq {{.*#+}} xmm10 = xmm10[0],xmm11[0],xmm10[1],xmm11[1]
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} xmm10 = xmm10[0,1,2],xmm2[3,4,5,6,7]
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm10[0,1,2,3],ymm2[4,5,6,7]
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm10 = ymm12[0],ymm15[1,2],ymm12[3],ymm15[4],ymm12[5],ymm15[6,7],ymm12[8],ymm15[9,10],ymm12[11],ymm15[12],ymm12[13],ymm15[14,15]
+; AVX512F-SLOW-NEXT:    vmovdqa64 %ymm12, %ymm19
+; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm10, %xmm11
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} xmm10 = xmm11[0],xmm10[1],xmm11[2],xmm10[3]
+; AVX512F-SLOW-NEXT:    vpshufb %xmm14, %xmm10, %xmm10
+; AVX512F-SLOW-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm10[5,6,7]
+; AVX512F-SLOW-NEXT:    vmovdqa64 %xmm25, %xmm5
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} xmm10 = xmm5[0,1],xmm6[2],xmm5[3]
+; AVX512F-SLOW-NEXT:    vmovdqa64 %xmm6, %xmm23
+; AVX512F-SLOW-NEXT:    vpshufb %xmm0, %xmm10, %xmm0
+; AVX512F-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512F-SLOW-NEXT:    vpternlogq $184, %zmm28, %zmm27, %zmm0
 ; AVX512F-SLOW-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
 ; AVX512F-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm12[0],ymm15[1],ymm12[2,3],ymm15[4],ymm12[5],ymm15[6],ymm12[7,8],ymm15[9],ymm12[10,11],ymm15[12],ymm12[13],ymm15[14],ymm12[15]
-; AVX512F-SLOW-NEXT:    vmovdqa %ymm15, %ymm10
-; AVX512F-SLOW-NEXT:    vmovdqa %ymm12, %ymm9
-; AVX512F-SLOW-NEXT:    vmovdqu %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm2
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0],ymm2[1],ymm0[2],ymm2[3],ymm0[4,5,6,7]
-; AVX512F-SLOW-NEXT:    vpshufb %ymm13, %ymm0, %ymm0
-; AVX512F-SLOW-NEXT:    vpshufd {{.*#+}} xmm2 = xmm26[0,3,2,3]
-; AVX512F-SLOW-NEXT:    vmovdqa64 %xmm26, %xmm20
-; AVX512F-SLOW-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm2[1,2,2,3,4,5,6,7]
-; AVX512F-SLOW-NEXT:    vpsrlq $48, %xmm24, %xmm8
-; AVX512F-SLOW-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm8[0],xmm2[1],xmm8[1]
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1,2],xmm0[3,4,5,6,7]
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm2 = ymm5[0],ymm4[1,2],ymm5[3],ymm4[4],ymm5[5],ymm4[6,7],ymm5[8],ymm4[9,10],ymm5[11],ymm4[12],ymm5[13],ymm4[14,15]
-; AVX512F-SLOW-NEXT:    vmovdqa64 %ymm5, %ymm18
-; AVX512F-SLOW-NEXT:    vmovdqa64 %ymm4, %ymm26
-; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm2, %xmm8
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} xmm2 = xmm8[0],xmm2[1],xmm8[2],xmm2[3]
-; AVX512F-SLOW-NEXT:    vpshufb %xmm14, %xmm2, %xmm2
-; AVX512F-SLOW-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm2[5,6,7]
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} xmm2 = xmm1[0,1],xmm7[2],xmm1[3]
-; AVX512F-SLOW-NEXT:    vmovdqa64 %xmm7, %xmm21
-; AVX512F-SLOW-NEXT:    vmovdqa64 %xmm1, %xmm22
-; AVX512F-SLOW-NEXT:    vmovdqa64 %xmm31, %xmm1
-; AVX512F-SLOW-NEXT:    vpshufb %xmm1, %xmm2, %xmm2
-; AVX512F-SLOW-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512F-SLOW-NEXT:    vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm25, %zmm2 # 64-byte Folded Reload
-; AVX512F-SLOW-NEXT:    vinserti64x4 $1, %ymm0, %zmm2, %zmm0
-; AVX512F-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload
-; AVX512F-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm11[0],ymm5[1],ymm11[2,3],ymm5[4],ymm11[5],ymm5[6],ymm11[7,8],ymm5[9],ymm11[10,11],ymm5[12],ymm11[13],ymm5[14],ymm11[15]
+; AVX512F-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload
+; AVX512F-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm18 # 32-byte Reload
+; AVX512F-SLOW-NEXT:    vmovdqa64 %ymm18, %ymm0
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm12[1],ymm0[2,3],ymm12[4],ymm0[5],ymm12[6],ymm0[7,8],ymm12[9],ymm0[10,11],ymm12[12],ymm0[13],ymm12[14],ymm0[15]
 ; AVX512F-SLOW-NEXT:    vpermq {{.*#+}} ymm2 = ymm0[2,3,0,1]
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm2[6],ymm0[7]
-; AVX512F-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm17 # 32-byte Reload
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm0[0,1,2,3,4,5],ymm2[6],ymm0[7]
+; AVX512F-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm24 # 32-byte Reload
 ; AVX512F-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm16 # 32-byte Reload
-; AVX512F-SLOW-NEXT:    vmovdqa64 %ymm17, %ymm1
-; AVX512F-SLOW-NEXT:    vmovdqa64 %ymm16, %ymm2
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm2 = ymm1[0,1],ymm2[2],ymm1[3],ymm2[4],ymm1[5,6],ymm2[7],ymm1[8,9],ymm2[10],ymm1[11],ymm2[12],ymm1[13,14],ymm2[15]
-; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm2, %xmm7
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1,2],xmm7[3,4],xmm2[5,6,7]
-; AVX512F-SLOW-NEXT:    vmovdqa {{.*#+}} ymm7 = [4,5,14,15,8,9,2,3,12,13,10,11,0,1,10,11,20,21,30,31,24,25,18,19,28,29,26,27,16,17,26,27]
-; AVX512F-SLOW-NEXT:    vpshufb %ymm7, %ymm0, %ymm8
-; AVX512F-SLOW-NEXT:    vmovdqa {{.*#+}} xmm0 = [4,5,14,15,8,9,2,3,12,13,6,7,12,13,14,15]
-; AVX512F-SLOW-NEXT:    vpshufb %xmm0, %xmm2, %xmm2
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2],ymm8[3,4,5,6,7]
-; AVX512F-SLOW-NEXT:    vmovdqa64 %xmm29, %xmm8
-; AVX512F-SLOW-NEXT:    vmovdqa64 %xmm30, %xmm12
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} xmm13 = xmm12[0],xmm8[1],xmm12[2,3]
-; AVX512F-SLOW-NEXT:    vmovdqa {{.*#+}} xmm8 = [0,1,6,7,4,5,6,7,8,9,6,7,0,1,10,11]
-; AVX512F-SLOW-NEXT:    vpshufb %xmm8, %xmm13, %xmm13
+; AVX512F-SLOW-NEXT:    vmovdqa64 %ymm24, %ymm0
+; AVX512F-SLOW-NEXT:    vmovdqa64 %ymm16, %ymm5
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1],ymm5[2],ymm0[3],ymm5[4],ymm0[5,6],ymm5[7],ymm0[8,9],ymm5[10],ymm0[11],ymm5[12],ymm0[13,14],ymm5[15]
+; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm10
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} xmm10 = xmm0[0,1,2],xmm10[3,4],xmm0[5,6,7]
+; AVX512F-SLOW-NEXT:    vmovdqa {{.*#+}} ymm0 = [4,5,14,15,8,9,2,3,12,13,10,11,0,1,10,11,20,21,30,31,24,25,18,19,28,29,26,27,16,17,26,27]
+; AVX512F-SLOW-NEXT:    vpshufb %ymm0, %ymm2, %ymm2
+; AVX512F-SLOW-NEXT:    vpshufb {{.*#+}} xmm10 = xmm10[4,5,14,15,8,9,2,3,12,13,6,7,12,13,14,15]
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm10[0,1,2],ymm2[3,4,5,6,7]
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} xmm13 = xmm8[0],xmm4[1],xmm8[2,3]
+; AVX512F-SLOW-NEXT:    vmovdqa64 %xmm4, %xmm29
+; AVX512F-SLOW-NEXT:    vmovdqa64 %xmm8, %xmm28
+; AVX512F-SLOW-NEXT:    vpshufb {{.*#+}} xmm13 = xmm13[0,1,6,7,4,5,6,7,8,9,6,7,0,1,10,11]
 ; AVX512F-SLOW-NEXT:    vinserti128 $1, %xmm13, %ymm0, %ymm13
-; AVX512F-SLOW-NEXT:    vpternlogq $184, %zmm2, %zmm25, %zmm13
-; AVX512F-SLOW-NEXT:    vmovdqa64 %ymm23, %ymm1
-; AVX512F-SLOW-NEXT:    vmovdqa64 %ymm19, %ymm4
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm2 = ymm4[0],ymm1[1],ymm4[2],ymm1[3],ymm4[4,5],ymm1[6],ymm4[7,8],ymm1[9],ymm4[10],ymm1[11],ymm4[12,13],ymm1[14],ymm4[15]
+; AVX512F-SLOW-NEXT:    vpternlogq $184, %zmm2, %zmm27, %zmm13
+; AVX512F-SLOW-NEXT:    vmovdqa64 %ymm26, %ymm8
+; AVX512F-SLOW-NEXT:    vmovdqa64 %ymm22, %ymm4
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm2 = ymm8[0],ymm4[1],ymm8[2],ymm4[3],ymm8[4,5],ymm4[6],ymm8[7,8],ymm4[9],ymm8[10],ymm4[11],ymm8[12,13],ymm4[14],ymm8[15]
 ; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm2, %xmm14
 ; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0],ymm14[1],ymm2[2,3,4,5,6,7]
 ; AVX512F-SLOW-NEXT:    vmovdqa {{.*#+}} ymm14 = [4,5,14,15,12,13,2,3,12,13,6,7,0,1,10,11,20,21,30,31,28,29,18,19,28,29,22,23,16,17,26,27]
 ; AVX512F-SLOW-NEXT:    vpshufb %ymm14, %ymm2, %ymm2
-; AVX512F-SLOW-NEXT:    vpshufd {{.*#+}} xmm15 = xmm28[0,1,1,3]
-; AVX512F-SLOW-NEXT:    vpshufhw {{.*#+}} xmm15 = xmm15[0,1,2,3,4,7,6,7]
-; AVX512F-SLOW-NEXT:    vpunpckhdq {{.*#+}} xmm15 = xmm15[2],xmm27[2],xmm15[3],xmm27[3]
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} xmm15 = xmm15[0,1,2],xmm2[3,4,5,6,7]
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm15[0,1,2,3],ymm2[4,5,6,7]
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm15 = ymm3[0],ymm6[1],ymm3[2,3],ymm6[4],ymm3[5],ymm6[6],ymm3[7,8],ymm6[9],ymm3[10,11],ymm6[12],ymm3[13],ymm6[14],ymm3[15]
-; AVX512F-SLOW-NEXT:    vmovdqa64 %ymm3, %ymm19
-; AVX512F-SLOW-NEXT:    vmovdqa64 %ymm6, %ymm23
-; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm15, %xmm12
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} xmm12 = xmm12[0,1,2],xmm15[3,4],xmm12[5,6,7]
-; AVX512F-SLOW-NEXT:    vmovdqa {{.*#+}} xmm15 = [0,1,2,3,8,9,2,3,12,13,6,7,0,1,10,11]
-; AVX512F-SLOW-NEXT:    vpshufb %xmm15, %xmm12, %xmm12
-; AVX512F-SLOW-NEXT:    vinserti128 $1, %xmm12, %ymm0, %ymm12
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm12[5,6,7]
+; AVX512F-SLOW-NEXT:    vmovdqa64 %xmm30, %xmm6
+; AVX512F-SLOW-NEXT:    vpshufd {{.*#+}} xmm11 = xmm30[0,1,1,3]
+; AVX512F-SLOW-NEXT:    vpshufhw {{.*#+}} xmm11 = xmm11[0,1,2,3,4,7,6,7]
+; AVX512F-SLOW-NEXT:    vmovdqa64 %xmm21, %xmm5
+; AVX512F-SLOW-NEXT:    vpunpckhdq {{.*#+}} xmm11 = xmm11[2],xmm21[2],xmm11[3],xmm21[3]
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} xmm11 = xmm11[0,1,2],xmm2[3,4,5,6,7]
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm11[0,1,2,3],ymm2[4,5,6,7]
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm11 = ymm9[0],ymm7[1],ymm9[2,3],ymm7[4],ymm9[5],ymm7[6],ymm9[7,8],ymm7[9],ymm9[10,11],ymm7[12],ymm9[13],ymm7[14],ymm9[15]
+; AVX512F-SLOW-NEXT:    vmovdqa64 %ymm7, %ymm21
+; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm11, %xmm10
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} xmm10 = xmm10[0,1,2],xmm11[3,4],xmm10[5,6,7]
+; AVX512F-SLOW-NEXT:    vmovdqa {{.*#+}} xmm11 = [0,1,2,3,8,9,2,3,12,13,6,7,0,1,10,11]
+; AVX512F-SLOW-NEXT:    vpshufb %xmm11, %xmm10, %xmm10
+; AVX512F-SLOW-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm10[5,6,7]
 ; AVX512F-SLOW-NEXT:    vinserti64x4 $1, %ymm2, %zmm13, %zmm2
 ; AVX512F-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm2 = ymm10[0],ymm9[1],ymm10[2],ymm9[3],ymm10[4,5],ymm9[6],ymm10[7,8],ymm9[9],ymm10[10],ymm9[11],ymm10[12,13],ymm9[14],ymm10[15]
-; AVX512F-SLOW-NEXT:    vmovdqa64 %ymm10, %ymm31
-; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm2, %xmm12
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0],ymm12[1],ymm2[2,3,4,5,6,7]
+; AVX512F-SLOW-NEXT:    vmovdqa64 %ymm17, %ymm2
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0],ymm3[1],ymm2[2],ymm3[3],ymm2[4,5],ymm3[6],ymm2[7,8],ymm3[9],ymm2[10],ymm3[11],ymm2[12,13],ymm3[14],ymm2[15]
+; AVX512F-SLOW-NEXT:    vmovdqa64 %ymm3, %ymm22
+; AVX512F-SLOW-NEXT:    vmovdqa64 %ymm17, %ymm26
+; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm2, %xmm10
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0],ymm10[1],ymm2[2,3,4,5,6,7]
 ; AVX512F-SLOW-NEXT:    vpshufb %ymm14, %ymm2, %ymm2
-; AVX512F-SLOW-NEXT:    vpshufd {{.*#+}} xmm12 = xmm20[0,1,1,3]
-; AVX512F-SLOW-NEXT:    vpshufhw {{.*#+}} xmm12 = xmm12[0,1,2,3,4,7,6,7]
-; AVX512F-SLOW-NEXT:    vmovdqa64 %xmm24, %xmm10
-; AVX512F-SLOW-NEXT:    vpunpckhdq {{.*#+}} xmm12 = xmm12[2],xmm24[2],xmm12[3],xmm24[3]
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} xmm12 = xmm12[0,1,2],xmm2[3,4,5,6,7]
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm12[0,1,2,3],ymm2[4,5,6,7]
-; AVX512F-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
-; AVX512F-SLOW-NEXT:    vpblendw $173, (%rsp), %ymm3, %ymm12 # 32-byte Folded Reload
-; AVX512F-SLOW-NEXT:    # ymm12 = mem[0],ymm3[1],mem[2,3],ymm3[4],mem[5],ymm3[6],mem[7,8],ymm3[9],mem[10,11],ymm3[12],mem[13],ymm3[14],mem[15]
-; AVX512F-SLOW-NEXT:    vpermq {{.*#+}} ymm13 = ymm12[2,3,0,1]
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm12 = ymm12[0,1,2,3,4,5],ymm13[6],ymm12[7]
-; AVX512F-SLOW-NEXT:    vpshufb %ymm7, %ymm12, %ymm7
-; AVX512F-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
-; AVX512F-SLOW-NEXT:    vpblendw $148, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm12 # 32-byte Folded Reload
-; AVX512F-SLOW-NEXT:    # ymm12 = ymm3[0,1],mem[2],ymm3[3],mem[4],ymm3[5,6],mem[7],ymm3[8,9],mem[10],ymm3[11],mem[12],ymm3[13,14],mem[15]
-; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm12, %xmm13
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} xmm12 = xmm12[0,1,2],xmm13[3,4],xmm12[5,6,7]
-; AVX512F-SLOW-NEXT:    vpshufb %xmm0, %xmm12, %xmm0
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2],ymm7[3,4,5,6,7]
-; AVX512F-SLOW-NEXT:    vmovdqa64 %xmm21, %xmm3
-; AVX512F-SLOW-NEXT:    vmovdqa64 %xmm22, %xmm6
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} xmm7 = xmm3[0],xmm6[1],xmm3[2,3]
-; AVX512F-SLOW-NEXT:    vpshufb %xmm8, %xmm7, %xmm7
-; AVX512F-SLOW-NEXT:    vinserti128 $1, %xmm7, %ymm0, %ymm7
-; AVX512F-SLOW-NEXT:    vpternlogq $184, %zmm0, %zmm25, %zmm7
-; AVX512F-SLOW-NEXT:    vmovdqa64 %ymm18, %ymm0
-; AVX512F-SLOW-NEXT:    vmovdqa64 %ymm26, %ymm3
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm3[0],ymm0[1],ymm3[2,3],ymm0[4],ymm3[5],ymm0[6],ymm3[7,8],ymm0[9],ymm3[10,11],ymm0[12],ymm3[13],ymm0[14],ymm3[15]
-; AVX512F-SLOW-NEXT:    vmovdqa64 %ymm18, %ymm24
-; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm8
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm8[0,1,2],xmm0[3,4],xmm8[5,6,7]
-; AVX512F-SLOW-NEXT:    vpshufb %xmm15, %xmm0, %xmm0
-; AVX512F-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4],ymm0[5,6,7]
-; AVX512F-SLOW-NEXT:    vinserti64x4 $1, %ymm0, %zmm7, %zmm18
-; AVX512F-SLOW-NEXT:    vmovdqa %ymm5, %ymm12
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm5[0],ymm11[1],ymm5[2],ymm11[3],ymm5[4,5],ymm11[6],ymm5[7,8],ymm11[9],ymm5[10],ymm11[11],ymm5[12,13],ymm11[14],ymm5[15]
-; AVX512F-SLOW-NEXT:    vpermq {{.*#+}} ymm2 = ymm0[2,3,0,1]
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4],ymm0[5],ymm2[6],ymm0[7]
-; AVX512F-SLOW-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[6,7,0,1,10,11,4,5,14,15,14,15,2,3,12,13,22,23,16,17,26,27,20,21,30,31,30,31,18,19,28,29]
+; AVX512F-SLOW-NEXT:    vpshufd {{.*#+}} xmm10 = xmm20[0,1,1,3]
+; AVX512F-SLOW-NEXT:    vpshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,4,7,6,7]
+; AVX512F-SLOW-NEXT:    vpunpckhdq {{.*#+}} xmm10 = xmm10[2],xmm1[2],xmm10[3],xmm1[3]
+; AVX512F-SLOW-NEXT:    vmovdqa64 %xmm1, %xmm17
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} xmm10 = xmm10[0,1,2],xmm2[3,4,5,6,7]
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm10[0,1,2,3],ymm2[4,5,6,7]
+; AVX512F-SLOW-NEXT:    vmovdqa64 %ymm19, %ymm1
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm10 = ymm15[0],ymm1[1],ymm15[2,3],ymm1[4],ymm15[5],ymm1[6],ymm15[7,8],ymm1[9],ymm15[10,11],ymm1[12],ymm15[13],ymm1[14],ymm15[15]
+; AVX512F-SLOW-NEXT:    vmovdqa64 %ymm15, %ymm31
+; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm10, %xmm13
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} xmm10 = xmm13[0,1,2],xmm10[3,4],xmm13[5,6,7]
+; AVX512F-SLOW-NEXT:    vpshufb %xmm11, %xmm10, %xmm10
+; AVX512F-SLOW-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm10[5,6,7]
+; AVX512F-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX512F-SLOW-NEXT:    vpblendw $82, (%rsp), %ymm1, %ymm10 # 32-byte Folded Reload
+; AVX512F-SLOW-NEXT:    # ymm10 = ymm1[0],mem[1],ymm1[2,3],mem[4],ymm1[5],mem[6],ymm1[7,8],mem[9],ymm1[10,11],mem[12],ymm1[13],mem[14],ymm1[15]
+; AVX512F-SLOW-NEXT:    vpermq {{.*#+}} ymm11 = ymm10[2,3,0,1]
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm10 = ymm10[0,1,2,3,4,5],ymm11[6],ymm10[7]
+; AVX512F-SLOW-NEXT:    vpshufb %ymm0, %ymm10, %ymm0
+; AVX512F-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX512F-SLOW-NEXT:    vpblendw $148, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm10 # 32-byte Folded Reload
+; AVX512F-SLOW-NEXT:    # ymm10 = ymm1[0,1],mem[2],ymm1[3],mem[4],ymm1[5,6],mem[7],ymm1[8,9],mem[10],ymm1[11],mem[12],ymm1[13,14],mem[15]
+; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm10, %xmm11
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} xmm10 = xmm10[0,1,2],xmm11[3,4],xmm10[5,6,7]
+; AVX512F-SLOW-NEXT:    vpshufb {{.*#+}} xmm10 = xmm10[4,5,14,15,8,9,2,3,12,13,6,7,12,13,14,15]
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm10[0,1,2],ymm0[3,4,5,6,7]
+; AVX512F-SLOW-NEXT:    vmovdqa64 %xmm25, %xmm1
+; AVX512F-SLOW-NEXT:    vmovdqa64 %xmm23, %xmm3
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} xmm10 = xmm3[0],xmm1[1],xmm3[2,3]
+; AVX512F-SLOW-NEXT:    vpshufb {{.*#+}} xmm10 = xmm10[0,1,6,7,4,5,6,7,8,9,6,7,0,1,10,11]
+; AVX512F-SLOW-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512F-SLOW-NEXT:    vpternlogq $184, %zmm0, %zmm27, %zmm10
+; AVX512F-SLOW-NEXT:    vinserti64x4 $1, %ymm2, %zmm10, %zmm23
+; AVX512F-SLOW-NEXT:    vmovdqa64 %ymm18, %ymm13
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm12[0],ymm13[1],ymm12[2],ymm13[3],ymm12[4,5],ymm13[6],ymm12[7,8],ymm13[9],ymm12[10],ymm13[11],ymm12[12,13],ymm13[14],ymm12[15]
+; AVX512F-SLOW-NEXT:    vpermq {{.*#+}} ymm10 = ymm0[2,3,0,1]
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm10[4],ymm0[5],ymm10[6],ymm0[7]
 ; AVX512F-SLOW-NEXT:    vmovdqa64 %ymm16, %ymm14
-; AVX512F-SLOW-NEXT:    vmovdqa64 %ymm17, %ymm15
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm2 = ymm14[0],ymm15[1,2],ymm14[3],ymm15[4],ymm14[5],ymm15[6,7],ymm14[8],ymm15[9,10],ymm14[11],ymm15[12],ymm14[13],ymm15[14,15]
-; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm2, %xmm7
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} xmm2 = xmm7[0],xmm2[1],xmm7[2],xmm2[3]
-; AVX512F-SLOW-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[6,7,0,1,10,11,4,5,14,15,8,9,12,13,14,15]
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1,2],ymm0[3,4,5,6,7]
-; AVX512F-SLOW-NEXT:    vmovdqa %ymm1, %ymm3
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm2 = ymm4[0,1],ymm1[2],ymm4[3],ymm1[4],ymm4[5,6],ymm1[7],ymm4[8,9],ymm1[10],ymm4[11],ymm1[12],ymm4[13,14],ymm1[15]
-; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm2, %xmm7
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1],ymm7[2],ymm2[3,4,5,6,7]
-; AVX512F-SLOW-NEXT:    vpshufb {{.*#+}} ymm2 = ymm2[6,7,6,7,8,9,4,5,14,15,8,9,2,3,12,13,22,23,22,23,24,25,20,21,30,31,24,25,18,19,28,29]
-; AVX512F-SLOW-NEXT:    vmovdqa64 %xmm27, %xmm6
-; AVX512F-SLOW-NEXT:    vmovdqa64 %xmm28, %xmm1
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} xmm7 = xmm6[0],xmm1[1],xmm6[2,3]
-; AVX512F-SLOW-NEXT:    vpshufb {{.*#+}} xmm7 = xmm7[6,7,0,1,10,11,u,u,u,u,u,u,u,u,u,u]
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} xmm7 = xmm7[0,1,2],xmm2[3,4,5,6,7]
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm7[0,1,2,3],ymm2[4,5,6,7]
-; AVX512F-SLOW-NEXT:    vmovdqa64 %xmm29, %xmm5
+; AVX512F-SLOW-NEXT:    vmovdqa64 %ymm24, %ymm15
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm10 = ymm14[0],ymm15[1,2],ymm14[3],ymm15[4],ymm14[5],ymm15[6,7],ymm14[8],ymm15[9,10],ymm14[11],ymm15[12],ymm14[13],ymm15[14,15]
+; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm10, %xmm11
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} xmm10 = xmm11[0],xmm10[1],xmm11[2],xmm10[3]
+; AVX512F-SLOW-NEXT:    vpshufb {{.*#+}} xmm10 = xmm10[6,7,0,1,10,11,4,5,14,15,8,9,12,13,14,15]
+; AVX512F-SLOW-NEXT:    vmovdqa {{.*#+}} ymm1 = [6,7,0,1,10,11,4,5,14,15,14,15,2,3,12,13,22,23,16,17,26,27,20,21,30,31,30,31,18,19,28,29]
+; AVX512F-SLOW-NEXT:    vpshufb %ymm1, %ymm0, %ymm0
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm10[0,1,2],ymm0[3,4,5,6,7]
+; AVX512F-SLOW-NEXT:    vmovdqa %ymm4, %ymm3
+; AVX512F-SLOW-NEXT:    vmovdqa %ymm8, %ymm4
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm10 = ymm8[0,1],ymm3[2],ymm8[3],ymm3[4],ymm8[5,6],ymm3[7],ymm8[8,9],ymm3[10],ymm8[11],ymm3[12],ymm8[13,14],ymm3[15]
+; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm10, %xmm11
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm10 = ymm10[0,1],ymm11[2],ymm10[3,4,5,6,7]
+; AVX512F-SLOW-NEXT:    vpshufb {{.*#+}} ymm10 = ymm10[6,7,6,7,8,9,4,5,14,15,8,9,2,3,12,13,22,23,22,23,24,25,20,21,30,31,24,25,18,19,28,29]
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} xmm11 = xmm5[0],xmm6[1],xmm5[2,3]
 ; AVX512F-SLOW-NEXT:    vmovdqa64 %xmm30, %xmm7
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} xmm7 = xmm7[0,1],xmm5[2],xmm7[3]
-; AVX512F-SLOW-NEXT:    vpshufb {{.*#+}} xmm7 = xmm7[0,1,2,3,4,5,6,7,8,9,8,9,2,3,12,13]
-; AVX512F-SLOW-NEXT:    vinserti128 $1, %xmm7, %ymm0, %ymm7
-; AVX512F-SLOW-NEXT:    vinserti64x4 $1, %ymm2, %zmm7, %zmm2
-; AVX512F-SLOW-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm2
-; AVX512F-SLOW-NEXT:    vmovdqa64 %ymm19, %ymm9
-; AVX512F-SLOW-NEXT:    vmovdqa64 %ymm23, %ymm8
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm8[0],ymm9[1],ymm8[2,3],ymm9[4],ymm8[5],ymm9[6],ymm8[7,8],ymm9[9],ymm8[10,11],ymm9[12],ymm8[13],ymm9[14],ymm8[15]
-; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm7
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm7[1,2,3],xmm0[4,5],xmm7[6,7]
+; AVX512F-SLOW-NEXT:    vmovdqa %xmm5, %xmm6
+; AVX512F-SLOW-NEXT:    vpshufb %xmm1, %xmm11, %xmm11
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} xmm11 = xmm11[0,1,2],xmm10[3,4,5,6,7]
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm10 = ymm11[0,1,2,3],ymm10[4,5,6,7]
+; AVX512F-SLOW-NEXT:    vmovdqa64 %xmm29, %xmm1
+; AVX512F-SLOW-NEXT:    vmovdqa64 %xmm28, %xmm2
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} xmm8 = xmm2[0,1],xmm1[2],xmm2[3]
+; AVX512F-SLOW-NEXT:    vpshufb {{.*#+}} xmm8 = xmm8[0,1,2,3,4,5,6,7,8,9,8,9,2,3,12,13]
+; AVX512F-SLOW-NEXT:    vinserti128 $1, %xmm8, %ymm0, %ymm8
+; AVX512F-SLOW-NEXT:    vinserti64x4 $1, %ymm10, %zmm8, %zmm8
+; AVX512F-SLOW-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm8
+; AVX512F-SLOW-NEXT:    vmovdqa %ymm9, %ymm2
+; AVX512F-SLOW-NEXT:    vmovdqa64 %ymm21, %ymm10
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm10[0],ymm9[1],ymm10[2,3],ymm9[4],ymm10[5],ymm9[6],ymm10[7,8],ymm9[9],ymm10[10,11],ymm9[12],ymm10[13],ymm9[14],ymm10[15]
+; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm9
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm9[1,2,3],xmm0[4,5],xmm9[6,7]
 ; AVX512F-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,0,1,10,11,4,5,14,15,8,9,2,3,12,13]
 ; AVX512F-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512F-SLOW-NEXT:    vextracti64x4 $1, %zmm2, %ymm7
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm7[0],ymm0[1,2,3,4,5,6,7],ymm7[8],ymm0[9,10,11,12,13,14,15]
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm7[0,1,2,3],ymm0[4,5,6,7]
-; AVX512F-SLOW-NEXT:    vinserti64x4 $1, %ymm0, %zmm2, %zmm21
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm12[0,1],ymm11[2],ymm12[3],ymm11[4],ymm12[5,6],ymm11[7],ymm12[8,9],ymm11[10],ymm12[11],ymm11[12],ymm12[13,14],ymm11[15]
-; AVX512F-SLOW-NEXT:    vpermq {{.*#+}} ymm2 = ymm0[2,3,0,1]
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4],ymm0[5,6],ymm2[7]
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm2 = ymm15[0],ymm14[1],ymm15[2,3],ymm14[4],ymm15[5],ymm14[6],ymm15[7,8],ymm14[9],ymm15[10,11],ymm14[12],ymm15[13],ymm14[14],ymm15[15]
-; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm2, %xmm7
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} xmm11 = xmm7[0,1,2],xmm2[3,4],xmm7[5,6,7]
-; AVX512F-SLOW-NEXT:    vmovdqa {{.*#+}} ymm2 = [8,9,2,3,12,13,6,7,4,5,6,7,4,5,14,15,24,25,18,19,28,29,22,23,20,21,22,23,20,21,30,31]
-; AVX512F-SLOW-NEXT:    vpshufb %ymm2, %ymm0, %ymm0
-; AVX512F-SLOW-NEXT:    vmovdqa {{.*#+}} xmm7 = [8,9,2,3,12,13,6,7,0,1,10,11,0,1,6,7]
-; AVX512F-SLOW-NEXT:    vpshufb %xmm7, %xmm11, %xmm11
+; AVX512F-SLOW-NEXT:    vextracti64x4 $1, %zmm8, %ymm9
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm9[0],ymm0[1,2,3,4,5,6,7],ymm9[8],ymm0[9,10,11,12,13,14,15]
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm9[0,1,2,3],ymm0[4,5,6,7]
+; AVX512F-SLOW-NEXT:    vinserti64x4 $1, %ymm0, %zmm8, %zmm25
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm12[0,1],ymm13[2],ymm12[3],ymm13[4],ymm12[5,6],ymm13[7],ymm12[8,9],ymm13[10],ymm12[11],ymm13[12],ymm12[13,14],ymm13[15]
+; AVX512F-SLOW-NEXT:    vpermq {{.*#+}} ymm8 = ymm0[2,3,0,1]
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm8[4],ymm0[5,6],ymm8[7]
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm8 = ymm15[0],ymm14[1],ymm15[2,3],ymm14[4],ymm15[5],ymm14[6],ymm15[7,8],ymm14[9],ymm15[10,11],ymm14[12],ymm15[13],ymm14[14],ymm15[15]
+; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm8, %xmm9
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} xmm11 = xmm9[0,1,2],xmm8[3,4],xmm9[5,6,7]
+; AVX512F-SLOW-NEXT:    vmovdqa {{.*#+}} ymm9 = [8,9,2,3,12,13,6,7,4,5,6,7,4,5,14,15,24,25,18,19,28,29,22,23,20,21,22,23,20,21,30,31]
+; AVX512F-SLOW-NEXT:    vpshufb %ymm9, %ymm0, %ymm0
+; AVX512F-SLOW-NEXT:    vmovdqa {{.*#+}} xmm8 = [8,9,2,3,12,13,6,7,0,1,10,11,0,1,6,7]
+; AVX512F-SLOW-NEXT:    vpshufb %xmm8, %xmm11, %xmm11
 ; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm11[0,1,2],ymm0[3,4,5,6,7]
 ; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm4 = ymm3[0],ymm4[1,2],ymm3[3],ymm4[4],ymm3[5],ymm4[6,7],ymm3[8],ymm4[9,10],ymm3[11],ymm4[12],ymm3[13],ymm4[14,15]
 ; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm4, %xmm5
 ; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm5[0],ymm4[1],ymm5[2],ymm4[3,4,5,6,7]
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} xmm1 = xmm6[0,1],xmm1[2],xmm6[3]
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} xmm1 = xmm6[0,1],xmm7[2],xmm6[3]
 ; AVX512F-SLOW-NEXT:    vmovdqa {{.*#+}} ymm3 = [8,9,14,15,4,5,6,7,0,1,10,11,4,5,14,15,24,25,30,31,20,21,22,23,16,17,26,27,20,21,30,31]
 ; AVX512F-SLOW-NEXT:    vpshufb %ymm3, %ymm4, %ymm4
-; AVX512F-SLOW-NEXT:    vpbroadcastq {{.*#+}} xmm5 = [8,9,2,3,12,13,0,0,8,9,2,3,12,13,0,0]
-; AVX512F-SLOW-NEXT:    vpshufb %xmm5, %xmm1, %xmm1
+; AVX512F-SLOW-NEXT:    vpshufb %xmm9, %xmm1, %xmm1
 ; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2],xmm4[3,4,5,6,7]
 ; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm4[4,5,6,7]
 ; AVX512F-SLOW-NEXT:    vinserti128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm4 # 16-byte Folded Reload
@@ -7091,53 +7107,54 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512F-SLOW-NEXT:    movb $7, %al
 ; AVX512F-SLOW-NEXT:    kmovw %eax, %k1
 ; AVX512F-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm1 {%k1}
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm9[0],ymm8[1],ymm9[2],ymm8[3],ymm9[4,5],ymm8[6],ymm9[7,8],ymm8[9],ymm9[10],ymm8[11],ymm9[12,13],ymm8[14],ymm9[15]
-; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm4
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} xmm4 = xmm0[0,1],xmm4[2,3],xmm0[4,5,6],xmm4[7]
+; AVX512F-SLOW-NEXT:    vextracti64x4 $1, %zmm1, %ymm4
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm2[0],ymm10[1],ymm2[2],ymm10[3],ymm2[4,5],ymm10[6],ymm2[7,8],ymm10[9],ymm2[10],ymm10[11],ymm2[12,13],ymm10[14],ymm2[15]
+; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm5
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} xmm5 = xmm0[0,1],xmm5[2,3],xmm0[4,5,6],xmm5[7]
 ; AVX512F-SLOW-NEXT:    vmovdqa {{.*#+}} xmm0 = [6,7,2,3,12,13,6,7,0,1,10,11,4,5,14,15]
-; AVX512F-SLOW-NEXT:    vpshufb %xmm0, %xmm4, %xmm4
-; AVX512F-SLOW-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512F-SLOW-NEXT:    vextracti64x4 $1, %zmm1, %ymm6
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm4 = ymm6[0],ymm4[1,2,3,4,5,6,7],ymm6[8],ymm4[9,10,11,12,13,14,15]
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm6[0,1,2,3],ymm4[4,5,6,7]
+; AVX512F-SLOW-NEXT:    vpshufb %xmm0, %xmm5, %xmm5
+; AVX512F-SLOW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm5 = ymm4[0],ymm5[1,2,3,4,5,6,7],ymm4[8],ymm5[9,10,11,12,13,14,15]
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm5[4,5,6,7]
 ; AVX512F-SLOW-NEXT:    vinserti64x4 $1, %ymm4, %zmm1, %zmm1
-; AVX512F-SLOW-NEXT:    vmovdqa64 %ymm31, %ymm6
-; AVX512F-SLOW-NEXT:    vpblendw $41, {{[-0-9]+}}(%r{{[sb]}}p), %ymm6, %ymm4 # 32-byte Folded Reload
-; AVX512F-SLOW-NEXT:    # ymm4 = mem[0],ymm6[1,2],mem[3],ymm6[4],mem[5],ymm6[6,7],mem[8],ymm6[9,10],mem[11],ymm6[12],mem[13],ymm6[14,15]
-; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm4, %xmm6
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm6[0],ymm4[1],ymm6[2],ymm4[3,4,5,6,7]
+; AVX512F-SLOW-NEXT:    vmovdqa64 %ymm22, %ymm2
+; AVX512F-SLOW-NEXT:    vmovdqa64 %ymm26, %ymm4
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm4 = ymm2[0],ymm4[1,2],ymm2[3],ymm4[4],ymm2[5],ymm4[6,7],ymm2[8],ymm4[9,10],ymm2[11],ymm4[12],ymm2[13],ymm4[14,15]
+; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm4, %xmm5
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm5[0],ymm4[1],ymm5[2],ymm4[3,4,5,6,7]
 ; AVX512F-SLOW-NEXT:    vpshufb %ymm3, %ymm4, %ymm3
-; AVX512F-SLOW-NEXT:    vmovdqa64 %xmm20, %xmm4
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} xmm4 = xmm10[0,1],xmm4[2],xmm10[3]
-; AVX512F-SLOW-NEXT:    vpshufb %xmm5, %xmm4, %xmm4
+; AVX512F-SLOW-NEXT:    vmovdqa64 %xmm20, %xmm2
+; AVX512F-SLOW-NEXT:    vmovdqa64 %xmm17, %xmm4
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} xmm4 = xmm4[0,1],xmm2[2],xmm4[3]
+; AVX512F-SLOW-NEXT:    vpshufb %xmm9, %xmm4, %xmm4
 ; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1,2],xmm3[3,4,5,6,7]
 ; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
 ; AVX512F-SLOW-NEXT:    vinserti128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm4 # 16-byte Folded Reload
 ; AVX512F-SLOW-NEXT:    vinserti64x4 $1, %ymm3, %zmm4, %zmm3
-; AVX512F-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
-; AVX512F-SLOW-NEXT:    vpblendw $148, (%rsp), %ymm4, %ymm4 # 32-byte Folded Reload
-; AVX512F-SLOW-NEXT:    # ymm4 = ymm4[0,1],mem[2],ymm4[3],mem[4],ymm4[5,6],mem[7],ymm4[8,9],mem[10],ymm4[11],mem[12],ymm4[13,14],mem[15]
+; AVX512F-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX512F-SLOW-NEXT:    vpblendw $107, (%rsp), %ymm2, %ymm4 # 32-byte Folded Reload
+; AVX512F-SLOW-NEXT:    # ymm4 = mem[0,1],ymm2[2],mem[3],ymm2[4],mem[5,6],ymm2[7],mem[8,9],ymm2[10],mem[11],ymm2[12],mem[13,14],ymm2[15]
 ; AVX512F-SLOW-NEXT:    vpermq {{.*#+}} ymm5 = ymm4[2,3,0,1]
 ; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm5[4],ymm4[5,6],ymm5[7]
-; AVX512F-SLOW-NEXT:    vpshufb %ymm2, %ymm4, %ymm2
-; AVX512F-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
-; AVX512F-SLOW-NEXT:    vpblendw $82, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm4 # 32-byte Folded Reload
-; AVX512F-SLOW-NEXT:    # ymm4 = ymm4[0],mem[1],ymm4[2,3],mem[4],ymm4[5],mem[6],ymm4[7,8],mem[9],ymm4[10,11],mem[12],ymm4[13],mem[14],ymm4[15]
+; AVX512F-SLOW-NEXT:    vpshufb %ymm9, %ymm4, %ymm4
+; AVX512F-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX512F-SLOW-NEXT:    vpblendw $82, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm5 # 32-byte Folded Reload
+; AVX512F-SLOW-NEXT:    # ymm5 = ymm2[0],mem[1],ymm2[2,3],mem[4],ymm2[5],mem[6],ymm2[7,8],mem[9],ymm2[10,11],mem[12],ymm2[13],mem[14],ymm2[15]
+; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm5, %xmm6
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} xmm5 = xmm6[0,1,2],xmm5[3,4],xmm6[5,6,7]
+; AVX512F-SLOW-NEXT:    vpshufb %xmm8, %xmm5, %xmm5
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm5[0,1,2],ymm4[3,4,5,6,7]
+; AVX512F-SLOW-NEXT:    vinserti64x4 $0, %ymm4, %zmm0, %zmm3 {%k1}
+; AVX512F-SLOW-NEXT:    vmovdqa64 %ymm19, %ymm2
+; AVX512F-SLOW-NEXT:    vmovdqa64 %ymm31, %ymm4
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm4 = ymm4[0],ymm2[1],ymm4[2],ymm2[3],ymm4[4,5],ymm2[6],ymm4[7,8],ymm2[9],ymm4[10],ymm2[11],ymm4[12,13],ymm2[14],ymm4[15]
 ; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm4, %xmm5
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} xmm4 = xmm5[0,1,2],xmm4[3,4],xmm5[5,6,7]
-; AVX512F-SLOW-NEXT:    vpshufb %xmm7, %xmm4, %xmm4
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm4[0,1,2],ymm2[3,4,5,6,7]
-; AVX512F-SLOW-NEXT:    vinserti64x4 $0, %ymm2, %zmm0, %zmm3 {%k1}
-; AVX512F-SLOW-NEXT:    vmovdqa64 %ymm26, %ymm2
-; AVX512F-SLOW-NEXT:    vmovdqa64 %ymm24, %ymm4
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0],ymm4[1],ymm2[2],ymm4[3],ymm2[4,5],ymm4[6],ymm2[7,8],ymm4[9],ymm2[10],ymm4[11],ymm2[12,13],ymm4[14],ymm2[15]
-; AVX512F-SLOW-NEXT:    vextracti128 $1, %ymm2, %xmm4
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1],xmm4[2,3],xmm2[4,5,6],xmm4[7]
-; AVX512F-SLOW-NEXT:    vpshufb %xmm0, %xmm2, %xmm0
-; AVX512F-SLOW-NEXT:    vextracti64x4 $1, %zmm3, %ymm2
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1],xmm5[2,3],xmm4[4,5,6],xmm5[7]
+; AVX512F-SLOW-NEXT:    vpshufb %xmm0, %xmm4, %xmm0
+; AVX512F-SLOW-NEXT:    vextracti64x4 $1, %zmm3, %ymm4
 ; AVX512F-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm2[0],ymm0[1,2,3,4,5,6,7],ymm2[8],ymm0[9,10,11,12,13,14,15]
-; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
+; AVX512F-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm4[0],ymm0[1,2,3,4,5,6,7],ymm4[8],ymm0[9,10,11,12,13,14,15]
+; AVX512F-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm0[4,5,6,7]
 ; AVX512F-SLOW-NEXT:    vinserti64x4 $1, %ymm0, %zmm3, %zmm0
 ; AVX512F-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
 ; AVX512F-SLOW-NEXT:    vmovaps %zmm2, (%rsi)
@@ -7149,19 +7166,19 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512F-SLOW-NEXT:    vmovaps %zmm2, (%rdx)
 ; AVX512F-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
 ; AVX512F-SLOW-NEXT:    vmovaps %zmm2, 64(%rcx)
-; AVX512F-SLOW-NEXT:    vmovdqa64 %zmm18, (%rcx)
-; AVX512F-SLOW-NEXT:    vmovdqa64 %zmm21, 64(%r8)
+; AVX512F-SLOW-NEXT:    vmovdqa64 %zmm23, (%rcx)
+; AVX512F-SLOW-NEXT:    vmovdqa64 %zmm25, 64(%r8)
 ; AVX512F-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
 ; AVX512F-SLOW-NEXT:    vmovaps %zmm2, (%r8)
 ; AVX512F-SLOW-NEXT:    vmovdqa64 %zmm1, 64(%r9)
 ; AVX512F-SLOW-NEXT:    vmovdqa64 %zmm0, (%r9)
-; AVX512F-SLOW-NEXT:    addq $584, %rsp # imm = 0x248
+; AVX512F-SLOW-NEXT:    addq $552, %rsp # imm = 0x228
 ; AVX512F-SLOW-NEXT:    vzeroupper
 ; AVX512F-SLOW-NEXT:    retq
 ;
 ; AVX512F-FAST-LABEL: load_i16_stride5_vf64:
 ; AVX512F-FAST:       # %bb.0:
-; AVX512F-FAST-NEXT:    subq $712, %rsp # imm = 0x2C8
+; AVX512F-FAST-NEXT:    subq $552, %rsp # imm = 0x228
 ; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} xmm0 = [4,5,14,15,4,5,6,7,8,9,10,11,12,13,14,15]
 ; AVX512F-FAST-NEXT:    vmovdqa 496(%rdi), %xmm1
 ; AVX512F-FAST-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
@@ -7178,20 +7195,21 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512F-FAST-NEXT:    vmovdqa 576(%rdi), %ymm7
 ; AVX512F-FAST-NEXT:    vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512F-FAST-NEXT:    vmovdqa 608(%rdi), %ymm8
-; AVX512F-FAST-NEXT:    vmovdqu %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512F-FAST-NEXT:    vmovdqa 352(%rdi), %ymm1
-; AVX512F-FAST-NEXT:    vmovdqu %ymm1, (%rsp) # 32-byte Spill
-; AVX512F-FAST-NEXT:    vmovdqa 320(%rdi), %ymm4
-; AVX512F-FAST-NEXT:    vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm1 = ymm4[0],ymm1[1],ymm4[2,3],ymm1[4],ymm4[5],ymm1[6],ymm4[7,8],ymm1[9],ymm4[10,11],ymm1[12],ymm4[13],ymm1[14],ymm4[15]
+; AVX512F-FAST-NEXT:    vmovdqu %ymm8, (%rsp) # 32-byte Spill
+; AVX512F-FAST-NEXT:    vmovdqa 352(%rdi), %ymm4
+; AVX512F-FAST-NEXT:    vmovdqa 320(%rdi), %ymm5
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm1 = ymm5[0],ymm4[1],ymm5[2,3],ymm4[4],ymm5[5],ymm4[6],ymm5[7,8],ymm4[9],ymm5[10,11],ymm4[12],ymm5[13],ymm4[14],ymm5[15]
+; AVX512F-FAST-NEXT:    vmovdqa64 %ymm5, %ymm28
+; AVX512F-FAST-NEXT:    vmovdqa64 %ymm4, %ymm30
 ; AVX512F-FAST-NEXT:    vextracti128 $1, %ymm1, %xmm4
 ; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} xmm4 = xmm1[0],xmm4[1,2,3],xmm1[4,5],xmm4[6,7]
 ; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} ymm1 = <0,1,10,11,4,5,14,15,8,9,2,3,12,13,128,128,128,128,128,128,128,128,128,128,128,128,u,u,u,u,u,u>
 ; AVX512F-FAST-NEXT:    vpshufb %ymm1, %ymm4, %ymm4
 ; AVX512F-FAST-NEXT:    vmovdqa 384(%rdi), %ymm6
-; AVX512F-FAST-NEXT:    vmovdqa 416(%rdi), %ymm5
-; AVX512F-FAST-NEXT:    vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm5 = ymm6[0],ymm5[1,2],ymm6[3],ymm5[4],ymm6[5],ymm5[6,7],ymm6[8],ymm5[9,10],ymm6[11],ymm5[12],ymm6[13],ymm5[14,15]
+; AVX512F-FAST-NEXT:    vmovdqa 416(%rdi), %ymm9
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm5 = ymm6[0],ymm9[1,2],ymm6[3],ymm9[4],ymm6[5],ymm9[6,7],ymm6[8],ymm9[9,10],ymm6[11],ymm9[12],ymm6[13],ymm9[14,15]
+; AVX512F-FAST-NEXT:    vmovdqa64 %ymm9, %ymm26
+; AVX512F-FAST-NEXT:    vmovdqu %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512F-FAST-NEXT:    vmovdqa64 %ymm6, %ymm27
 ; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} ymm6 = [1,3,0,2,4,6,1,3]
 ; AVX512F-FAST-NEXT:    vpermd %ymm5, %ymm6, %ymm5
@@ -7213,46 +7231,46 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512F-FAST-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
 ; AVX512F-FAST-NEXT:    vpblendd {{.*#+}} ymm5 = ymm7[0,1,2,3,4],ymm5[5,6,7]
 ; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} ymm18 = [0,3,1,3,0,3,5,7]
-; AVX512F-FAST-NEXT:    vmovdqa64 448(%rdi), %ymm29
-; AVX512F-FAST-NEXT:    vpermd %ymm29, %ymm18, %ymm7
-; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} ymm8 = [4,5,2,3,4,5,6,7,8,9,2,3,4,5,10,11,20,21,18,19,20,21,22,23,24,25,18,19,20,21,26,27]
-; AVX512F-FAST-NEXT:    vpshufb %ymm8, %ymm7, %ymm7
-; AVX512F-FAST-NEXT:    vmovdqa %ymm8, %ymm14
-; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm26 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535]
-; AVX512F-FAST-NEXT:    vpternlogq $184, %zmm4, %zmm26, %zmm7
+; AVX512F-FAST-NEXT:    vmovdqa 448(%rdi), %ymm7
+; AVX512F-FAST-NEXT:    vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512F-FAST-NEXT:    vpermd %ymm7, %ymm18, %ymm7
+; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} ymm11 = [4,5,2,3,4,5,6,7,8,9,2,3,4,5,10,11,20,21,18,19,20,21,22,23,24,25,18,19,20,21,26,27]
+; AVX512F-FAST-NEXT:    vpshufb %ymm11, %ymm7, %ymm7
+; AVX512F-FAST-NEXT:    vmovdqa %ymm11, %ymm14
+; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm8 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535]
+; AVX512F-FAST-NEXT:    vpternlogq $184, %zmm4, %zmm8, %zmm7
+; AVX512F-FAST-NEXT:    vmovdqa64 %zmm8, %zmm16
 ; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm5, %zmm7, %zmm4
 ; AVX512F-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-FAST-NEXT:    vmovdqa 176(%rdi), %xmm4
 ; AVX512F-FAST-NEXT:    vpshufb %xmm0, %xmm4, %xmm0
 ; AVX512F-FAST-NEXT:    vmovdqa64 %xmm4, %xmm31
-; AVX512F-FAST-NEXT:    vmovdqa 160(%rdi), %xmm4
-; AVX512F-FAST-NEXT:    vpshufb %xmm3, %xmm4, %xmm3
-; AVX512F-FAST-NEXT:    vmovdqa %xmm4, %xmm15
+; AVX512F-FAST-NEXT:    vmovdqa 160(%rdi), %xmm15
+; AVX512F-FAST-NEXT:    vpshufb %xmm3, %xmm15, %xmm3
 ; AVX512F-FAST-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm3[0],xmm0[0],xmm3[1],xmm0[1]
-; AVX512F-FAST-NEXT:    vmovdqa 64(%rdi), %ymm7
-; AVX512F-FAST-NEXT:    vmovdqa 96(%rdi), %ymm8
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm3 = ymm7[0],ymm8[1,2],ymm7[3],ymm8[4],ymm7[5],ymm8[6,7],ymm7[8],ymm8[9,10],ymm7[11],ymm8[12],ymm7[13],ymm8[14,15]
-; AVX512F-FAST-NEXT:    vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512F-FAST-NEXT:    vmovdqa 64(%rdi), %ymm11
+; AVX512F-FAST-NEXT:    vmovdqa 96(%rdi), %ymm5
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm3 = ymm11[0],ymm5[1,2],ymm11[3],ymm5[4],ymm11[5],ymm5[6,7],ymm11[8],ymm5[9,10],ymm11[11],ymm5[12],ymm11[13],ymm5[14,15]
+; AVX512F-FAST-NEXT:    vmovdqa %ymm5, %ymm7
 ; AVX512F-FAST-NEXT:    vpermd %ymm3, %ymm6, %ymm3
 ; AVX512F-FAST-NEXT:    vpshufb %ymm13, %ymm3, %ymm3
 ; AVX512F-FAST-NEXT:    vmovdqa (%rdi), %ymm5
 ; AVX512F-FAST-NEXT:    vmovdqa 32(%rdi), %ymm6
 ; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm4 = ymm5[0],ymm6[1],ymm5[2,3],ymm6[4],ymm5[5],ymm6[6],ymm5[7,8],ymm6[9],ymm5[10,11],ymm6[12],ymm5[13],ymm6[14],ymm5[15]
-; AVX512F-FAST-NEXT:    vmovdqa %ymm5, %ymm11
+; AVX512F-FAST-NEXT:    vmovdqa %ymm5, %ymm12
 ; AVX512F-FAST-NEXT:    vextracti128 $1, %ymm4, %xmm5
 ; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0],xmm5[1,2,3],xmm4[4,5],xmm5[6,7]
 ; AVX512F-FAST-NEXT:    vpshufb %ymm1, %ymm4, %ymm1
 ; AVX512F-FAST-NEXT:    vpor %ymm3, %ymm1, %ymm1
-; AVX512F-FAST-NEXT:    vmovdqa 192(%rdi), %ymm4
-; AVX512F-FAST-NEXT:    vmovdqa 224(%rdi), %ymm5
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm3 = ymm5[0],ymm4[1],ymm5[2,3],ymm4[4],ymm5[5],ymm4[6],ymm5[7,8],ymm4[9],ymm5[10,11],ymm4[12],ymm5[13],ymm4[14],ymm5[15]
-; AVX512F-FAST-NEXT:    vmovdqa %ymm5, %ymm13
-; AVX512F-FAST-NEXT:    vmovdqa %ymm4, %ymm5
+; AVX512F-FAST-NEXT:    vmovdqa 192(%rdi), %ymm8
+; AVX512F-FAST-NEXT:    vmovdqa 224(%rdi), %ymm13
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm3 = ymm13[0],ymm8[1],ymm13[2,3],ymm8[4],ymm13[5],ymm8[6],ymm13[7,8],ymm8[9],ymm13[10,11],ymm8[12],ymm13[13],ymm8[14],ymm13[15]
 ; AVX512F-FAST-NEXT:    vpermd %ymm3, %ymm10, %ymm3
 ; AVX512F-FAST-NEXT:    vpshufb %ymm9, %ymm3, %ymm3
-; AVX512F-FAST-NEXT:    vmovdqa 256(%rdi), %ymm12
+; AVX512F-FAST-NEXT:    vmovdqa 256(%rdi), %ymm5
 ; AVX512F-FAST-NEXT:    vmovdqa 288(%rdi), %ymm9
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm4 = ymm9[0,1],ymm12[2],ymm9[3],ymm12[4],ymm9[5,6],ymm12[7],ymm9[8,9],ymm12[10],ymm9[11],ymm12[12],ymm9[13,14],ymm12[15]
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm4 = ymm9[0,1],ymm5[2],ymm9[3],ymm5[4],ymm9[5,6],ymm5[7],ymm9[8,9],ymm5[10],ymm9[11],ymm5[12],ymm9[13,14],ymm5[15]
+; AVX512F-FAST-NEXT:    vmovdqa64 %ymm5, %ymm25
 ; AVX512F-FAST-NEXT:    vextracti128 $1, %ymm4, %xmm10
 ; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1,2],xmm10[3,4],xmm4[5,6,7]
 ; AVX512F-FAST-NEXT:    vpshufb %xmm2, %xmm4, %xmm2
@@ -7262,123 +7280,124 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512F-FAST-NEXT:    vmovdqa64 128(%rdi), %ymm23
 ; AVX512F-FAST-NEXT:    vpermd %ymm23, %ymm18, %ymm2
 ; AVX512F-FAST-NEXT:    vpshufb %ymm14, %ymm2, %ymm2
-; AVX512F-FAST-NEXT:    vpternlogq $184, %zmm1, %zmm26, %zmm2
+; AVX512F-FAST-NEXT:    vpternlogq $184, %zmm1, %zmm16, %zmm2
 ; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm0, %zmm2, %zmm0
 ; AVX512F-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm0 = ymm6[0],ymm11[1],ymm6[2],ymm11[3],ymm6[4,5],ymm11[6],ymm6[7,8],ymm11[9],ymm6[10],ymm11[11],ymm6[12,13],ymm11[14],ymm6[15]
-; AVX512F-FAST-NEXT:    vmovdqa64 %ymm11, %ymm30
-; AVX512F-FAST-NEXT:    vmovdqu %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512F-FAST-NEXT:    vmovdqa64 %ymm6, %ymm17
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm0 = ymm6[0],ymm12[1],ymm6[2],ymm12[3],ymm6[4,5],ymm12[6],ymm6[7,8],ymm12[9],ymm6[10],ymm12[11],ymm6[12,13],ymm12[14],ymm6[15]
+; AVX512F-FAST-NEXT:    vmovdqa64 %ymm12, %ymm21
+; AVX512F-FAST-NEXT:    vmovdqu %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512F-FAST-NEXT:    vmovdqa64 %ymm6, %ymm29
 ; AVX512F-FAST-NEXT:    vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512F-FAST-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5,6],xmm1[7]
-; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} ymm21 = <2,u,u,u,4,7,1,6>
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm1 = ymm8[0],ymm7[1],ymm8[2,3],ymm7[4],ymm8[5],ymm7[6],ymm8[7,8],ymm7[9],ymm8[10,11],ymm7[12],ymm8[13],ymm7[14],ymm8[15]
-; AVX512F-FAST-NEXT:    vmovdqa64 %ymm8, %ymm22
-; AVX512F-FAST-NEXT:    vmovdqu %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512F-FAST-NEXT:    vpermd %ymm1, %ymm21, %ymm1
+; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} ymm17 = <2,u,u,u,4,7,1,6>
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm1 = ymm7[0],ymm11[1],ymm7[2,3],ymm11[4],ymm7[5],ymm11[6],ymm7[7,8],ymm11[9],ymm7[10,11],ymm11[12],ymm7[13],ymm11[14],ymm7[15]
+; AVX512F-FAST-NEXT:    vmovdqa64 %ymm11, %ymm19
+; AVX512F-FAST-NEXT:    vmovdqa64 %ymm7, %ymm22
+; AVX512F-FAST-NEXT:    vpermd %ymm1, %ymm17, %ymm1
 ; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} ymm6 = [128,128,128,128,128,128,128,128,128,128,128,128,128,128,0,1,18,19,20,21,26,27,16,17,30,31,128,128,128,128,128,128]
 ; AVX512F-FAST-NEXT:    vpshufb %ymm6, %ymm1, %ymm1
-; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} ymm3 = <2,3,12,13,6,7,0,1,10,11,4,5,14,15,128,128,128,128,128,128,128,128,128,128,128,128,u,u,u,u,u,u>
-; AVX512F-FAST-NEXT:    vpshufb %ymm3, %ymm0, %ymm0
-; AVX512F-FAST-NEXT:    vporq %ymm1, %ymm0, %ymm18
+; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} ymm2 = <2,3,12,13,6,7,0,1,10,11,4,5,14,15,128,128,128,128,128,128,128,128,128,128,128,128,u,u,u,u,u,u>
+; AVX512F-FAST-NEXT:    vpshufb %ymm2, %ymm0, %ymm0
+; AVX512F-FAST-NEXT:    vpor %ymm1, %ymm0, %ymm10
 ; AVX512F-FAST-NEXT:    vpsrlq $48, %xmm31, %xmm0
-; AVX512F-FAST-NEXT:    vpbroadcastd {{.*#+}} xmm4 = [2,3,12,13,2,3,12,13,2,3,12,13,2,3,12,13]
-; AVX512F-FAST-NEXT:    vpshufb %xmm4, %xmm15, %xmm1
-; AVX512F-FAST-NEXT:    vmovdqa %xmm15, %xmm11
+; AVX512F-FAST-NEXT:    vpshufb %xmm2, %xmm15, %xmm1
 ; AVX512F-FAST-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
 ; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} ymm20 = <0,2,5,7,4,7,u,u>
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm0 = ymm5[0],ymm13[1],ymm5[2,3],ymm13[4],ymm5[5],ymm13[6],ymm5[7,8],ymm13[9],ymm5[10,11],ymm13[12],ymm5[13],ymm13[14],ymm5[15]
-; AVX512F-FAST-NEXT:    vmovdqa64 %ymm13, %ymm19
-; AVX512F-FAST-NEXT:    vpermd %ymm0, %ymm20, %ymm10
-; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} ymm2 = [2,3,4,5,4,5,0,1,6,7,8,9,14,15,4,5,18,19,20,21,20,21,16,17,22,23,24,25,30,31,20,21]
-; AVX512F-FAST-NEXT:    vpshufb %ymm2, %ymm10, %ymm10
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2],xmm10[3,4,5,6,7]
-; AVX512F-FAST-NEXT:    vpblendd {{.*#+}} ymm10 = ymm1[0,1,2,3],ymm10[4,5,6,7]
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm0 = ymm8[0],ymm13[1],ymm8[2,3],ymm13[4],ymm8[5],ymm13[6],ymm8[7,8],ymm13[9],ymm8[10,11],ymm13[12],ymm8[13],ymm13[14],ymm8[15]
+; AVX512F-FAST-NEXT:    vmovdqa %ymm8, %ymm5
+; AVX512F-FAST-NEXT:    vpermd %ymm0, %ymm20, %ymm4
+; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} ymm3 = [2,3,4,5,4,5,0,1,6,7,8,9,14,15,4,5,18,19,20,21,20,21,16,17,22,23,24,25,30,31,20,21]
+; AVX512F-FAST-NEXT:    vpshufb %ymm3, %ymm4, %ymm4
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2],xmm4[3,4,5,6,7]
+; AVX512F-FAST-NEXT:    vpblendd {{.*#+}} ymm4 = ymm1[0,1,2,3],ymm4[4,5,6,7]
 ; AVX512F-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} ymm24 = [1,4,6,3,1,4,6,3]
 ; AVX512F-FAST-NEXT:    # ymm24 = mem[0,1,2,3,0,1,2,3]
 ; AVX512F-FAST-NEXT:    vmovdqu %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512F-FAST-NEXT:    vmovdqa64 %ymm25, %ymm12
+; AVX512F-FAST-NEXT:    vmovdqu64 %ymm25, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm1 = ymm12[0],ymm9[1,2],ymm12[3],ymm9[4],ymm12[5],ymm9[6,7],ymm12[8],ymm9[9,10],ymm12[11],ymm9[12],ymm12[13],ymm9[14,15]
-; AVX512F-FAST-NEXT:    vmovdqu %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512F-FAST-NEXT:    vpermd %ymm1, %ymm24, %ymm0
 ; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,1,2,3,2,3,4,5,10,11,0,1,14,15,8,9,16,17,18,19,18,19,20,21,26,27,16,17,30,31,24,25]
 ; AVX512F-FAST-NEXT:    vpshufb %ymm1, %ymm0, %ymm0
-; AVX512F-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm10[0,1,2,3,4],ymm0[5,6,7]
+; AVX512F-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3,4],ymm0[5,6,7]
 ; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} ymm25 = [1,3,2,3,1,3,6,7]
-; AVX512F-FAST-NEXT:    vpermd %ymm23, %ymm25, %ymm10
+; AVX512F-FAST-NEXT:    vpermd %ymm23, %ymm25, %ymm4
 ; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} ymm14 = [0,1,6,7,4,5,6,7,8,9,0,1,6,7,8,9,16,17,22,23,20,21,22,23,24,25,16,17,22,23,24,25]
-; AVX512F-FAST-NEXT:    vpshufb %ymm14, %ymm10, %ymm10
-; AVX512F-FAST-NEXT:    vpternlogq $184, %zmm18, %zmm26, %zmm10
-; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm0, %zmm10, %zmm0
+; AVX512F-FAST-NEXT:    vpshufb %ymm14, %ymm4, %ymm4
+; AVX512F-FAST-NEXT:    vpternlogq $184, %zmm10, %zmm16, %zmm4
+; AVX512F-FAST-NEXT:    vmovdqa64 %zmm16, %zmm10
+; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm0, %zmm4, %zmm0
 ; AVX512F-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm18 # 32-byte Reload
+; AVX512F-FAST-NEXT:    vmovdqa64 %ymm28, %ymm18
+; AVX512F-FAST-NEXT:    vmovdqa64 %ymm30, %ymm28
 ; AVX512F-FAST-NEXT:    vmovdqa64 %ymm18, %ymm0
-; AVX512F-FAST-NEXT:    vpblendw $181, (%rsp), %ymm0, %ymm0 # 32-byte Folded Reload
-; AVX512F-FAST-NEXT:    # ymm0 = mem[0],ymm0[1],mem[2],ymm0[3],mem[4,5],ymm0[6],mem[7,8],ymm0[9],mem[10],ymm0[11],mem[12,13],ymm0[14],mem[15]
-; AVX512F-FAST-NEXT:    vextracti128 $1, %ymm0, %xmm10
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm10[2,3],xmm0[4,5,6],xmm10[7]
-; AVX512F-FAST-NEXT:    vpshufb %ymm3, %ymm0, %ymm0
-; AVX512F-FAST-NEXT:    vmovdqa64 %ymm27, %ymm13
-; AVX512F-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm3 = ymm7[0],ymm13[1],ymm7[2,3],ymm13[4],ymm7[5],ymm13[6],ymm7[7,8],ymm13[9],ymm7[10,11],ymm13[12],ymm7[13],ymm13[14],ymm7[15]
-; AVX512F-FAST-NEXT:    vmovdqa64 %ymm7, %ymm28
-; AVX512F-FAST-NEXT:    vpermd %ymm3, %ymm21, %ymm3
-; AVX512F-FAST-NEXT:    vpshufb %ymm6, %ymm3, %ymm3
-; AVX512F-FAST-NEXT:    vpor %ymm3, %ymm0, %ymm0
+; AVX512F-FAST-NEXT:    vmovdqa64 %ymm30, %ymm4
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm0 = ymm4[0],ymm0[1],ymm4[2],ymm0[3],ymm4[4,5],ymm0[6],ymm4[7,8],ymm0[9],ymm4[10],ymm0[11],ymm4[12,13],ymm0[14],ymm4[15]
+; AVX512F-FAST-NEXT:    vextracti128 $1, %ymm0, %xmm4
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm4[2,3],xmm0[4,5,6],xmm4[7]
+; AVX512F-FAST-NEXT:    vmovdqa64 %ymm27, %ymm7
+; AVX512F-FAST-NEXT:    vmovdqa64 %ymm26, %ymm4
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm4 = ymm4[0],ymm7[1],ymm4[2,3],ymm7[4],ymm4[5],ymm7[6],ymm4[7,8],ymm7[9],ymm4[10,11],ymm7[12],ymm4[13],ymm7[14],ymm4[15]
+; AVX512F-FAST-NEXT:    vpermd %ymm4, %ymm17, %ymm4
+; AVX512F-FAST-NEXT:    vpshufb %ymm6, %ymm4, %ymm4
+; AVX512F-FAST-NEXT:    vpshufb %ymm2, %ymm0, %ymm0
+; AVX512F-FAST-NEXT:    vpor %ymm4, %ymm0, %ymm0
+; AVX512F-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload
 ; AVX512F-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm16 # 32-byte Reload
-; AVX512F-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload
-; AVX512F-FAST-NEXT:    vmovdqa64 %ymm16, %ymm3
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm3 = ymm15[0],ymm3[1],ymm15[2,3],ymm3[4],ymm15[5],ymm3[6],ymm15[7,8],ymm3[9],ymm15[10,11],ymm3[12],ymm15[13],ymm3[14],ymm15[15]
-; AVX512F-FAST-NEXT:    vpermd %ymm3, %ymm20, %ymm3
-; AVX512F-FAST-NEXT:    vpshufb %ymm2, %ymm3, %ymm2
-; AVX512F-FAST-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
-; AVX512F-FAST-NEXT:    vpshufb %xmm4, %xmm7, %xmm3
+; AVX512F-FAST-NEXT:    vmovdqa64 %ymm16, %ymm4
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm4 = ymm11[0],ymm4[1],ymm11[2,3],ymm4[4],ymm11[5],ymm4[6],ymm11[7,8],ymm4[9],ymm11[10,11],ymm4[12],ymm11[13],ymm4[14],ymm11[15]
+; AVX512F-FAST-NEXT:    vpermd %ymm4, %ymm20, %ymm4
+; AVX512F-FAST-NEXT:    vpshufb %ymm3, %ymm4, %ymm3
+; AVX512F-FAST-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
+; AVX512F-FAST-NEXT:    vpshufb %xmm2, %xmm6, %xmm2
 ; AVX512F-FAST-NEXT:    vmovdqa64 {{[-0-9]+}}(%r{{[sb]}}p), %xmm27 # 16-byte Reload
 ; AVX512F-FAST-NEXT:    vpsrlq $48, %xmm27, %xmm4
-; AVX512F-FAST-NEXT:    vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1,2],xmm2[3,4,5,6,7]
-; AVX512F-FAST-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
-; AVX512F-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
-; AVX512F-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm3 = ymm6[0],ymm8[1,2],ymm6[3],ymm8[4],ymm6[5],ymm8[6,7],ymm6[8],ymm8[9,10],ymm6[11],ymm8[12],ymm6[13],ymm8[14,15]
+; AVX512F-FAST-NEXT:    vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1,2],xmm3[3,4,5,6,7]
+; AVX512F-FAST-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5,6,7]
+; AVX512F-FAST-NEXT:    vmovdqu (%rsp), %ymm8 # 32-byte Reload
+; AVX512F-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm3 = ymm4[0],ymm8[1,2],ymm4[3],ymm8[4],ymm4[5],ymm8[6,7],ymm4[8],ymm8[9,10],ymm4[11],ymm8[12],ymm4[13],ymm8[14,15]
 ; AVX512F-FAST-NEXT:    vpermd %ymm3, %ymm24, %ymm3
 ; AVX512F-FAST-NEXT:    vpshufb %ymm1, %ymm3, %ymm1
 ; AVX512F-FAST-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4],ymm1[5,6,7]
-; AVX512F-FAST-NEXT:    vpermd %ymm29, %ymm25, %ymm2
+; AVX512F-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm17 # 32-byte Reload
+; AVX512F-FAST-NEXT:    vpermd %ymm17, %ymm25, %ymm2
 ; AVX512F-FAST-NEXT:    vpshufb %ymm14, %ymm2, %ymm2
-; AVX512F-FAST-NEXT:    vpternlogq $184, %zmm0, %zmm26, %zmm2
-; AVX512F-FAST-NEXT:    vmovdqa64 %zmm26, %zmm21
+; AVX512F-FAST-NEXT:    vpternlogq $184, %zmm0, %zmm10, %zmm2
 ; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm0
 ; AVX512F-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm0 = ymm9[0],ymm12[1],ymm9[2,3],ymm12[4],ymm9[5],ymm12[6],ymm9[7,8],ymm12[9],ymm9[10,11],ymm12[12],ymm9[13],ymm12[14],ymm9[15]
 ; AVX512F-FAST-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2],xmm0[3,4],xmm1[5,6,7]
-; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm11[u,u,u,u,u,u,u,u,4,5,14,15,u,u,u,u]
-; AVX512F-FAST-NEXT:    vmovdqa64 %xmm11, %xmm20
-; AVX512F-FAST-NEXT:    vpunpckhdq {{.*#+}} xmm3 = xmm0[2],xmm31[2],xmm0[3],xmm31[3]
+; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm15[u,u,u,u,u,u,u,u,4,5,14,15,u,u,u,u]
+; AVX512F-FAST-NEXT:    vmovdqa64 %xmm15, %xmm20
+; AVX512F-FAST-NEXT:    vpunpckhdq {{.*#+}} xmm0 = xmm0[2],xmm31[2],xmm0[3],xmm31[3]
 ; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} ymm24 = <0,3,5,2,5,7,u,u>
-; AVX512F-FAST-NEXT:    vmovdqa64 %ymm19, %ymm12
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm0 = ymm12[0],ymm5[1],ymm12[2],ymm5[3],ymm12[4,5],ymm5[6],ymm12[7,8],ymm5[9],ymm12[10],ymm5[11],ymm12[12,13],ymm5[14],ymm12[15]
-; AVX512F-FAST-NEXT:    vmovdqa64 %ymm5, %ymm26
-; AVX512F-FAST-NEXT:    vpermd %ymm0, %ymm24, %ymm4
-; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,1,6,7,2,3,2,3,4,5,10,11,0,1,14,15,16,17,22,23,18,19,18,19,20,21,26,27,16,17,30,31]
-; AVX512F-FAST-NEXT:    vpshufb %ymm2, %ymm4, %ymm4
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1,2],xmm4[3,4,5,6,7]
-; AVX512F-FAST-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm4[4,5,6,7]
-; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} xmm4 = [0,1,2,3,8,9,2,3,12,13,6,7,0,1,10,11]
-; AVX512F-FAST-NEXT:    vpshufb %xmm4, %xmm1, %xmm1
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm2 = ymm13[0],ymm5[1],ymm13[2],ymm5[3],ymm13[4,5],ymm5[6],ymm13[7,8],ymm5[9],ymm13[10],ymm5[11],ymm13[12,13],ymm5[14],ymm13[15]
+; AVX512F-FAST-NEXT:    vmovdqa64 %ymm5, %ymm30
+; AVX512F-FAST-NEXT:    vmovdqa64 %ymm13, %ymm26
+; AVX512F-FAST-NEXT:    vpermd %ymm2, %ymm24, %ymm3
+; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} ymm12 = [0,1,6,7,2,3,2,3,4,5,10,11,0,1,14,15,16,17,22,23,18,19,18,19,20,21,26,27,16,17,30,31]
+; AVX512F-FAST-NEXT:    vpshufb %ymm12, %ymm3, %ymm3
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2],xmm3[3,4,5,6,7]
+; AVX512F-FAST-NEXT:    vpblendd {{.*#+}} ymm3 = ymm0[0,1,2,3],ymm3[4,5,6,7]
+; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} xmm2 = [0,1,2,3,8,9,2,3,12,13,6,7,0,1,10,11]
+; AVX512F-FAST-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
 ; AVX512F-FAST-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
 ; AVX512F-FAST-NEXT:    vpblendd {{.*#+}} ymm10 = ymm3[0,1,2,3,4],ymm1[5,6,7]
-; AVX512F-FAST-NEXT:    vmovdqa64 %ymm30, %ymm0
-; AVX512F-FAST-NEXT:    vmovdqa64 %ymm17, %ymm1
+; AVX512F-FAST-NEXT:    vmovdqa64 %ymm21, %ymm0
+; AVX512F-FAST-NEXT:    vmovdqa64 %ymm29, %ymm1
 ; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0,1],ymm0[2],ymm1[3],ymm0[4],ymm1[5,6],ymm0[7],ymm1[8,9],ymm0[10],ymm1[11],ymm0[12],ymm1[13,14],ymm0[15]
 ; AVX512F-FAST-NEXT:    vextracti128 $1, %ymm1, %xmm3
 ; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} xmm5 = xmm1[0,1,2],xmm3[3,4],xmm1[5,6,7]
 ; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} ymm25 = <0,2,u,u,5,7,2,4>
-; AVX512F-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %ymm19 # 32-byte Reload
 ; AVX512F-FAST-NEXT:    vmovdqa64 %ymm19, %ymm0
 ; AVX512F-FAST-NEXT:    vmovdqa64 %ymm22, %ymm1
 ; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm1 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5],ymm1[6],ymm0[7,8],ymm1[9],ymm0[10,11],ymm1[12],ymm0[13],ymm1[14],ymm0[15]
+; AVX512F-FAST-NEXT:    vmovdqa64 %ymm22, %ymm29
+; AVX512F-FAST-NEXT:    vmovdqa64 %ymm19, %ymm21
 ; AVX512F-FAST-NEXT:    vpermd %ymm1, %ymm25, %ymm3
 ; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,1,6,7,8,9,14,15,4,5,6,7,0,1,6,7,16,17,22,23,24,25,30,31,20,21,22,23,16,17,22,23]
 ; AVX512F-FAST-NEXT:    vpshufb %ymm1, %ymm3, %ymm0
@@ -7388,56 +7407,55 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512F-FAST-NEXT:    vbroadcasti128 {{.*#+}} ymm5 = [1,4,6,0,1,4,6,0]
 ; AVX512F-FAST-NEXT:    # ymm5 = mem[0,1,0,1]
 ; AVX512F-FAST-NEXT:    vpermd %ymm23, %ymm5, %ymm14
-; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} ymm11 = [4,5,2,3,4,5,6,7,8,9,2,3,4,5,10,11,20,21,18,19,20,21,22,23,24,25,18,19,20,21,26,27]
-; AVX512F-FAST-NEXT:    vpshufb %ymm11, %ymm14, %ymm14
-; AVX512F-FAST-NEXT:    vpternlogq $184, %zmm0, %zmm21, %zmm14
+; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} ymm9 = [4,5,2,3,4,5,6,7,8,9,2,3,4,5,10,11,20,21,18,19,20,21,22,23,24,25,18,19,20,21,26,27]
+; AVX512F-FAST-NEXT:    vpshufb %ymm9, %ymm14, %ymm14
+; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm15 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535]
+; AVX512F-FAST-NEXT:    vpternlogq $184, %zmm0, %zmm15, %zmm14
 ; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm10, %zmm14, %zmm0
 ; AVX512F-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm0 = ymm8[0],ymm6[1],ymm8[2,3],ymm6[4],ymm8[5],ymm6[6],ymm8[7,8],ymm6[9],ymm8[10,11],ymm6[12],ymm8[13],ymm6[14],ymm8[15]
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm0 = ymm8[0],ymm4[1],ymm8[2,3],ymm4[4],ymm8[5],ymm4[6],ymm8[7,8],ymm4[9],ymm8[10,11],ymm4[12],ymm8[13],ymm4[14],ymm8[15]
 ; AVX512F-FAST-NEXT:    vextracti128 $1, %ymm0, %xmm10
 ; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} xmm0 = xmm10[0,1,2],xmm0[3,4],xmm10[5,6,7]
-; AVX512F-FAST-NEXT:    vpshufb %xmm4, %xmm0, %xmm0
-; AVX512F-FAST-NEXT:    vmovdqa64 %ymm16, %ymm6
-; AVX512F-FAST-NEXT:    vmovdqa %ymm15, %ymm8
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm4 = ymm6[0],ymm15[1],ymm6[2],ymm15[3],ymm6[4,5],ymm15[6],ymm6[7,8],ymm15[9],ymm6[10],ymm15[11],ymm6[12,13],ymm15[14],ymm6[15]
-; AVX512F-FAST-NEXT:    vpermd %ymm4, %ymm24, %ymm4
-; AVX512F-FAST-NEXT:    vpshufb %ymm2, %ymm4, %ymm2
-; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} xmm4 = xmm7[u,u,u,u,u,u,u,u,4,5,14,15,u,u,u,u]
-; AVX512F-FAST-NEXT:    vmovdqa %xmm7, %xmm14
+; AVX512F-FAST-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; AVX512F-FAST-NEXT:    vmovdqa64 %ymm16, %ymm8
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm2 = ymm8[0],ymm11[1],ymm8[2],ymm11[3],ymm8[4,5],ymm11[6],ymm8[7,8],ymm11[9],ymm8[10],ymm11[11],ymm8[12,13],ymm11[14],ymm8[15]
+; AVX512F-FAST-NEXT:    vmovdqa %ymm11, %ymm13
+; AVX512F-FAST-NEXT:    vpermd %ymm2, %ymm24, %ymm2
+; AVX512F-FAST-NEXT:    vpshufb %ymm12, %ymm2, %ymm2
+; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} xmm4 = xmm6[u,u,u,u,u,u,u,u,4,5,14,15,u,u,u,u]
+; AVX512F-FAST-NEXT:    vmovdqa %xmm6, %xmm14
 ; AVX512F-FAST-NEXT:    vpunpckhdq {{.*#+}} xmm4 = xmm4[2],xmm27[2],xmm4[3],xmm27[3]
-; AVX512F-FAST-NEXT:    vmovdqa64 %xmm27, %xmm15
+; AVX512F-FAST-NEXT:    vmovdqa64 %xmm27, %xmm6
 ; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1,2],xmm2[3,4,5,6,7]
 ; AVX512F-FAST-NEXT:    vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3],ymm2[4,5,6,7]
 ; AVX512F-FAST-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
 ; AVX512F-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4],ymm0[5,6,7]
-; AVX512F-FAST-NEXT:    vmovdqa64 %ymm18, %ymm4
-; AVX512F-FAST-NEXT:    vmovdqu (%rsp), %ymm7 # 32-byte Reload
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm2 = ymm7[0,1],ymm4[2],ymm7[3],ymm4[4],ymm7[5,6],ymm4[7],ymm7[8,9],ymm4[10],ymm7[11],ymm4[12],ymm7[13,14],ymm4[15]
-; AVX512F-FAST-NEXT:    vmovdqa64 %ymm7, %ymm30
-; AVX512F-FAST-NEXT:    vmovdqa64 %ymm18, %ymm22
+; AVX512F-FAST-NEXT:    vmovdqa64 %ymm18, %ymm11
+; AVX512F-FAST-NEXT:    vmovdqa64 %ymm28, %ymm12
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm2 = ymm12[0,1],ymm11[2],ymm12[3],ymm11[4],ymm12[5,6],ymm11[7],ymm12[8,9],ymm11[10],ymm12[11],ymm11[12],ymm12[13,14],ymm11[15]
 ; AVX512F-FAST-NEXT:    vextracti128 $1, %ymm2, %xmm4
 ; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1,2],xmm4[3,4],xmm2[5,6,7]
 ; AVX512F-FAST-NEXT:    vpshufb %xmm3, %xmm2, %xmm2
-; AVX512F-FAST-NEXT:    vmovdqa64 %ymm28, %ymm4
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm3 = ymm13[0],ymm4[1],ymm13[2,3],ymm4[4],ymm13[5],ymm4[6],ymm13[7,8],ymm4[9],ymm13[10,11],ymm4[12],ymm13[13],ymm4[14],ymm13[15]
-; AVX512F-FAST-NEXT:    vmovdqa64 %ymm13, %ymm18
+; AVX512F-FAST-NEXT:    vpblendw $82, {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm3 # 32-byte Folded Reload
+; AVX512F-FAST-NEXT:    # ymm3 = ymm7[0],mem[1],ymm7[2,3],mem[4],ymm7[5],mem[6],ymm7[7,8],mem[9],ymm7[10,11],mem[12],ymm7[13],mem[14],ymm7[15]
+; AVX512F-FAST-NEXT:    vmovdqa64 %ymm7, %ymm16
 ; AVX512F-FAST-NEXT:    vpermd %ymm3, %ymm25, %ymm3
 ; AVX512F-FAST-NEXT:    vpshufb %ymm1, %ymm3, %ymm1
 ; AVX512F-FAST-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2],ymm1[3,4,5,6,7]
-; AVX512F-FAST-NEXT:    vpermd %ymm29, %ymm5, %ymm2
-; AVX512F-FAST-NEXT:    vpshufb %ymm11, %ymm2, %ymm2
-; AVX512F-FAST-NEXT:    vpternlogq $184, %zmm1, %zmm21, %zmm2
-; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm0, %zmm2, %zmm0
-; AVX512F-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-FAST-NEXT:    vmovdqa64 %xmm31, %xmm7
-; AVX512F-FAST-NEXT:    vmovdqa64 %xmm20, %xmm9
-; AVX512F-FAST-NEXT:    vpblendd {{.*#+}} xmm1 = xmm7[0],xmm9[1],xmm7[2,3]
+; AVX512F-FAST-NEXT:    vpermd %ymm17, %ymm5, %ymm2
+; AVX512F-FAST-NEXT:    vpshufb %ymm9, %ymm2, %ymm2
+; AVX512F-FAST-NEXT:    vpternlogq $184, %zmm1, %zmm15, %zmm2
+; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm0, %zmm2, %zmm28
+; AVX512F-FAST-NEXT:    vmovdqa64 %xmm31, %xmm15
+; AVX512F-FAST-NEXT:    vmovdqa64 %xmm20, %xmm7
+; AVX512F-FAST-NEXT:    vpblendd {{.*#+}} xmm1 = xmm15[0],xmm7[1],xmm15[2,3]
 ; AVX512F-FAST-NEXT:    vpbroadcastq {{.*#+}} xmm0 = [6,7,0,1,10,11,0,0,6,7,0,1,10,11,0,0]
 ; AVX512F-FAST-NEXT:    vpshufb %xmm0, %xmm1, %xmm2
-; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} ymm3 = <1,3,6,0,5,u,u,u>
+; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} ymm18 = <1,3,6,0,5,u,u,u>
+; AVX512F-FAST-NEXT:    vmovdqa64 %ymm30, %ymm9
 ; AVX512F-FAST-NEXT:    vmovdqa64 %ymm26, %ymm10
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm1 = ymm12[0,1],ymm10[2],ymm12[3],ymm10[4],ymm12[5,6],ymm10[7],ymm12[8,9],ymm10[10],ymm12[11],ymm10[12],ymm12[13,14],ymm10[15]
-; AVX512F-FAST-NEXT:    vpermd %ymm1, %ymm3, %ymm4
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm1 = ymm10[0,1],ymm9[2],ymm10[3],ymm9[4],ymm10[5,6],ymm9[7],ymm10[8,9],ymm9[10],ymm10[11],ymm9[12],ymm10[13,14],ymm9[15]
+; AVX512F-FAST-NEXT:    vpermd %ymm1, %ymm18, %ymm4
 ; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} ymm1 = [2,3,2,3,4,5,0,1,6,7,8,9,14,15,4,5,18,19,18,19,20,21,16,17,22,23,24,25,30,31,20,21]
 ; AVX512F-FAST-NEXT:    vpshufb %ymm1, %ymm4, %ymm4
 ; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1,2],xmm4[3,4,5,6,7]
@@ -7445,24 +7463,25 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512F-FAST-NEXT:    vbroadcasti128 {{.*#+}} ymm4 = [2,4,7,0,2,4,7,0]
 ; AVX512F-FAST-NEXT:    # ymm4 = mem[0,1,0,1]
 ; AVX512F-FAST-NEXT:    vpermd %ymm23, %ymm4, %ymm5
-; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} ymm13 = [0,1,6,7,4,5,6,7,8,9,0,1,6,7,8,9,16,17,22,23,20,21,22,23,24,25,16,17,22,23,24,25]
-; AVX512F-FAST-NEXT:    vpshufb %ymm13, %ymm5, %ymm5
+; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,6,7,4,5,6,7,8,9,0,1,6,7,8,9,16,17,22,23,20,21,22,23,24,25,16,17,22,23,24,25]
+; AVX512F-FAST-NEXT:    vpshufb %ymm3, %ymm5, %ymm5
 ; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm2, %zmm5, %zmm25
-; AVX512F-FAST-NEXT:    vpblendd {{.*#+}} xmm2 = xmm15[0],xmm14[1],xmm15[2,3]
-; AVX512F-FAST-NEXT:    vmovdqa64 %xmm27, %xmm31
-; AVX512F-FAST-NEXT:    vmovdqa64 %xmm14, %xmm17
+; AVX512F-FAST-NEXT:    vpblendd {{.*#+}} xmm2 = xmm6[0],xmm14[1],xmm6[2,3]
+; AVX512F-FAST-NEXT:    vmovdqa64 %xmm27, %xmm22
+; AVX512F-FAST-NEXT:    vmovdqa64 %xmm14, %xmm31
 ; AVX512F-FAST-NEXT:    vpshufb %xmm0, %xmm2, %xmm0
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm2 = ymm6[0,1],ymm8[2],ymm6[3],ymm8[4],ymm6[5,6],ymm8[7],ymm6[8,9],ymm8[10],ymm6[11],ymm8[12],ymm6[13,14],ymm8[15]
-; AVX512F-FAST-NEXT:    vmovdqa64 %ymm8, %ymm20
-; AVX512F-FAST-NEXT:    vpermd %ymm2, %ymm3, %ymm2
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm2 = ymm8[0,1],ymm13[2],ymm8[3],ymm13[4],ymm8[5,6],ymm13[7],ymm8[8,9],ymm13[10],ymm8[11],ymm13[12],ymm8[13,14],ymm13[15]
+; AVX512F-FAST-NEXT:    vmovdqa64 %ymm13, %ymm20
+; AVX512F-FAST-NEXT:    vmovdqa64 %ymm8, %ymm30
+; AVX512F-FAST-NEXT:    vpermd %ymm2, %ymm18, %ymm2
 ; AVX512F-FAST-NEXT:    vpshufb %ymm1, %ymm2, %ymm1
 ; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[3,4,5,6,7]
 ; AVX512F-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; AVX512F-FAST-NEXT:    vpermd %ymm29, %ymm4, %ymm1
-; AVX512F-FAST-NEXT:    vpshufb %ymm13, %ymm1, %ymm1
+; AVX512F-FAST-NEXT:    vpermd %ymm17, %ymm4, %ymm1
+; AVX512F-FAST-NEXT:    vpshufb %ymm3, %ymm1, %ymm1
 ; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm24
-; AVX512F-FAST-NEXT:    vpblendd {{.*#+}} xmm0 = xmm7[0,1],xmm9[2],xmm7[3]
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm1 = ymm10[0],ymm12[1,2],ymm10[3],ymm12[4],ymm10[5],ymm12[6,7],ymm10[8],ymm12[9,10],ymm10[11],ymm12[12],ymm10[13],ymm12[14,15]
+; AVX512F-FAST-NEXT:    vpblendd {{.*#+}} xmm0 = xmm15[0,1],xmm7[2],xmm15[3]
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm1 = ymm9[0],ymm10[1,2],ymm9[3],ymm10[4],ymm9[5],ymm10[6,7],ymm9[8],ymm10[9,10],ymm9[11],ymm10[12],ymm9[13],ymm10[14,15]
 ; AVX512F-FAST-NEXT:    vpbroadcastq {{.*#+}} xmm2 = [8,9,2,3,12,13,0,0,8,9,2,3,12,13,0,0]
 ; AVX512F-FAST-NEXT:    vpshufb %xmm2, %xmm0, %xmm3
 ; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} ymm27 = <1,4,6,3,6,u,u,u>
@@ -7473,18 +7492,20 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512F-FAST-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
 ; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} ymm26 = [0,2,1,3,0,2,5,7]
 ; AVX512F-FAST-NEXT:    vpermd %ymm23, %ymm26, %ymm3
-; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} ymm3 = ymm3[0,1,6,7,4,5,6,7,0,1,6,7,8,9,14,15,16,17,22,23,20,21,22,23,16,17,22,23,24,25,30,31]
+; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} ymm4 = [0,1,6,7,4,5,6,7,0,1,6,7,8,9,14,15,16,17,22,23,20,21,22,23,16,17,22,23,24,25,30,31]
+; AVX512F-FAST-NEXT:    vpshufb %ymm4, %ymm3, %ymm3
+; AVX512F-FAST-NEXT:    vmovdqa64 %ymm4, %ymm19
 ; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm1, %zmm3, %zmm1
 ; AVX512F-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
-; AVX512F-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm3 = ymm4[0],ymm15[1,2],ymm4[3],ymm15[4],ymm4[5],ymm15[6,7],ymm4[8],ymm15[9,10],ymm4[11],ymm15[12],ymm4[13],ymm15[14,15]
-; AVX512F-FAST-NEXT:    vmovdqa64 %ymm4, %ymm23
+; AVX512F-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm3 = ymm4[0],ymm14[1,2],ymm4[3],ymm14[4],ymm4[5],ymm14[6,7],ymm4[8],ymm14[9,10],ymm4[11],ymm14[12],ymm4[13],ymm14[14,15]
+; AVX512F-FAST-NEXT:    vmovdqa64 %ymm4, %ymm18
 ; AVX512F-FAST-NEXT:    vextracti128 $1, %ymm3, %xmm4
 ; AVX512F-FAST-NEXT:    vpblendd {{.*#+}} xmm3 = xmm4[0],xmm3[1],xmm4[2],xmm3[3]
 ; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} ymm4 = <0,3,u,u,5,0,2,7>
-; AVX512F-FAST-NEXT:    vmovdqa64 %ymm19, %ymm12
-; AVX512F-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm5 = ymm10[0],ymm12[1],ymm10[2],ymm12[3],ymm10[4,5],ymm12[6],ymm10[7,8],ymm12[9],ymm10[10],ymm12[11],ymm10[12,13],ymm12[14],ymm10[15]
+; AVX512F-FAST-NEXT:    vmovdqa64 %ymm21, %ymm15
+; AVX512F-FAST-NEXT:    vmovdqa64 %ymm29, %ymm10
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm5 = ymm10[0],ymm15[1],ymm10[2],ymm15[3],ymm10[4,5],ymm15[6],ymm10[7,8],ymm15[9],ymm10[10],ymm15[11],ymm10[12,13],ymm15[14],ymm10[15]
 ; AVX512F-FAST-NEXT:    vpermd %ymm5, %ymm4, %ymm5
 ; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} ymm6 = [2,3,4,5,10,11,0,1,14,15,14,15,2,3,4,5,18,19,20,21,26,27,16,17,30,31,30,31,18,19,20,21]
 ; AVX512F-FAST-NEXT:    vpshufb %ymm6, %ymm5, %ymm5
@@ -7493,9 +7514,9 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512F-FAST-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2],ymm5[3,4,5,6,7]
 ; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535]
 ; AVX512F-FAST-NEXT:    vpternlogq $184, %zmm3, %zmm5, %zmm25
-; AVX512F-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload
-; AVX512F-FAST-NEXT:    vpblendw $82, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm3 # 32-byte Folded Reload
-; AVX512F-FAST-NEXT:    # ymm3 = ymm14[0],mem[1],ymm14[2,3],mem[4],ymm14[5],mem[6],ymm14[7,8],mem[9],ymm14[10,11],mem[12],ymm14[13],mem[14],ymm14[15]
+; AVX512F-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
+; AVX512F-FAST-NEXT:    vpblendw $82, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm3 # 32-byte Folded Reload
+; AVX512F-FAST-NEXT:    # ymm3 = ymm3[0],mem[1],ymm3[2,3],mem[4],ymm3[5],mem[6],ymm3[7,8],mem[9],ymm3[10,11],mem[12],ymm3[13],mem[14],ymm3[15]
 ; AVX512F-FAST-NEXT:    vextracti128 $1, %ymm3, %xmm8
 ; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} xmm8 = xmm3[0],xmm8[1,2,3],xmm3[4,5],xmm8[6,7]
 ; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,1,0,1,10,11,4,5,14,15,8,9,2,3,12,13]
@@ -7505,34 +7526,35 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm8 = ymm9[0],ymm8[1,2,3,4,5,6,7],ymm9[8],ymm8[9,10,11,12,13,14,15]
 ; AVX512F-FAST-NEXT:    vpblendd {{.*#+}} ymm8 = ymm9[0,1,2,3],ymm8[4,5,6,7]
 ; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm8, %zmm25, %zmm21
-; AVX512F-FAST-NEXT:    vmovdqa64 %ymm22, %ymm9
-; AVX512F-FAST-NEXT:    vmovdqa64 %ymm30, %ymm11
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm8 = ymm9[0],ymm11[1,2],ymm9[3],ymm11[4],ymm9[5],ymm11[6,7],ymm9[8],ymm11[9,10],ymm9[11],ymm11[12],ymm9[13],ymm11[14,15]
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm8 = ymm11[0],ymm12[1,2],ymm11[3],ymm12[4],ymm11[5],ymm12[6,7],ymm11[8],ymm12[9,10],ymm11[11],ymm12[12],ymm11[13],ymm12[14,15]
+; AVX512F-FAST-NEXT:    vmovdqa64 %ymm12, %ymm23
+; AVX512F-FAST-NEXT:    vmovdqa64 %ymm11, %ymm25
 ; AVX512F-FAST-NEXT:    vextracti128 $1, %ymm8, %xmm9
 ; AVX512F-FAST-NEXT:    vpblendd {{.*#+}} xmm8 = xmm9[0],xmm8[1],xmm9[2],xmm8[3]
 ; AVX512F-FAST-NEXT:    vpshufb %xmm7, %xmm8, %xmm7
-; AVX512F-FAST-NEXT:    vmovdqa64 %ymm28, %ymm13
-; AVX512F-FAST-NEXT:    vmovdqa64 %ymm18, %ymm8
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm8 = ymm13[0],ymm8[1],ymm13[2],ymm8[3],ymm13[4,5],ymm8[6],ymm13[7,8],ymm8[9],ymm13[10],ymm8[11],ymm13[12,13],ymm8[14],ymm13[15]
+; AVX512F-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload
+; AVX512F-FAST-NEXT:    vmovdqa64 %ymm16, %ymm12
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm8 = ymm13[0],ymm12[1],ymm13[2],ymm12[3],ymm13[4,5],ymm12[6],ymm13[7,8],ymm12[9],ymm13[10],ymm12[11],ymm13[12,13],ymm12[14],ymm13[15]
 ; AVX512F-FAST-NEXT:    vpermd %ymm8, %ymm4, %ymm4
 ; AVX512F-FAST-NEXT:    vpshufb %ymm6, %ymm4, %ymm4
 ; AVX512F-FAST-NEXT:    vpblendd {{.*#+}} ymm4 = ymm7[0,1,2],ymm4[3,4,5,6,7]
 ; AVX512F-FAST-NEXT:    vpternlogq $184, %zmm4, %zmm5, %zmm24
-; AVX512F-FAST-NEXT:    vmovdqa64 %xmm31, %xmm4
-; AVX512F-FAST-NEXT:    vmovdqa64 %xmm17, %xmm5
+; AVX512F-FAST-NEXT:    vmovdqa64 %xmm22, %xmm4
+; AVX512F-FAST-NEXT:    vmovdqa64 %xmm31, %xmm5
 ; AVX512F-FAST-NEXT:    vpblendd {{.*#+}} xmm4 = xmm4[0,1],xmm5[2],xmm4[3]
 ; AVX512F-FAST-NEXT:    vpshufb %xmm2, %xmm4, %xmm2
 ; AVX512F-FAST-NEXT:    vmovdqa64 %ymm20, %ymm4
-; AVX512F-FAST-NEXT:    vmovdqa64 %ymm16, %ymm5
+; AVX512F-FAST-NEXT:    vmovdqa64 %ymm30, %ymm5
 ; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm4 = ymm4[0],ymm5[1,2],ymm4[3],ymm5[4],ymm4[5],ymm5[6,7],ymm4[8],ymm5[9,10],ymm4[11],ymm5[12],ymm4[13],ymm5[14,15]
 ; AVX512F-FAST-NEXT:    vpermd %ymm4, %ymm27, %ymm4
 ; AVX512F-FAST-NEXT:    vpshufb %ymm0, %ymm4, %ymm0
 ; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1,2],xmm0[3,4,5,6,7]
 ; AVX512F-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
-; AVX512F-FAST-NEXT:    vpermd %ymm29, %ymm26, %ymm2
-; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} ymm2 = ymm2[0,1,6,7,4,5,6,7,0,1,6,7,8,9,14,15,16,17,22,23,20,21,22,23,16,17,22,23,24,25,30,31]
+; AVX512F-FAST-NEXT:    vpermd %ymm17, %ymm26, %ymm2
+; AVX512F-FAST-NEXT:    vmovdqa64 %ymm19, %ymm4
+; AVX512F-FAST-NEXT:    vpshufb %ymm4, %ymm2, %ymm2
 ; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm0, %zmm2, %zmm0
-; AVX512F-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
+; AVX512F-FAST-NEXT:    vmovdqu (%rsp), %ymm9 # 32-byte Reload
 ; AVX512F-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload
 ; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm2 = ymm11[0],ymm9[1],ymm11[2,3],ymm9[4],ymm11[5],ymm9[6],ymm11[7,8],ymm9[9],ymm11[10,11],ymm9[12],ymm11[13],ymm9[14],ymm11[15]
 ; AVX512F-FAST-NEXT:    vextracti128 $1, %ymm2, %xmm4
@@ -7543,13 +7565,13 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm2 = ymm3[0],ymm2[1,2,3,4,5,6,7],ymm3[8],ymm2[9,10,11,12,13,14,15]
 ; AVX512F-FAST-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
 ; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm2, %zmm24, %zmm2
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm3 = ymm10[0,1],ymm12[2],ymm10[3],ymm12[4],ymm10[5,6],ymm12[7],ymm10[8,9],ymm12[10],ymm10[11],ymm12[12],ymm10[13,14],ymm12[15]
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm3 = ymm10[0,1],ymm15[2],ymm10[3],ymm15[4],ymm10[5,6],ymm15[7],ymm10[8,9],ymm15[10],ymm10[11],ymm15[12],ymm10[13,14],ymm15[15]
 ; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} ymm4 = <1,3,u,u,6,0,3,5>
 ; AVX512F-FAST-NEXT:    vpermd %ymm3, %ymm4, %ymm3
 ; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} ymm5 = [0,1,6,7,8,9,14,15,0,1,6,7,0,1,6,7,16,17,22,23,24,25,30,31,16,17,22,23,16,17,22,23]
 ; AVX512F-FAST-NEXT:    vpshufb %ymm5, %ymm3, %ymm3
-; AVX512F-FAST-NEXT:    vmovdqa64 %ymm23, %ymm6
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm6 = ymm15[0],ymm6[1],ymm15[2,3],ymm6[4],ymm15[5],ymm6[6],ymm15[7,8],ymm6[9],ymm15[10,11],ymm6[12],ymm15[13],ymm6[14],ymm15[15]
+; AVX512F-FAST-NEXT:    vmovdqa64 %ymm18, %ymm6
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm6 = ymm14[0],ymm6[1],ymm14[2,3],ymm6[4],ymm14[5],ymm6[6],ymm14[7,8],ymm6[9],ymm14[10,11],ymm6[12],ymm14[13],ymm6[14],ymm14[15]
 ; AVX512F-FAST-NEXT:    vextracti128 $1, %ymm6, %xmm7
 ; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} xmm6 = xmm7[0,1,2],xmm6[3,4],xmm7[5,6,7]
 ; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} xmm7 = [8,9,2,3,12,13,6,7,0,1,10,11,0,1,6,7]
@@ -7559,8 +7581,9 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512F-FAST-NEXT:    kmovw %eax, %k1
 ; AVX512F-FAST-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm1 {%k1}
 ; AVX512F-FAST-NEXT:    vextracti64x4 $1, %zmm1, %ymm3
-; AVX512F-FAST-NEXT:    vpblendw $181, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm6 # 32-byte Folded Reload
-; AVX512F-FAST-NEXT:    # ymm6 = mem[0],ymm14[1],mem[2],ymm14[3],mem[4,5],ymm14[6],mem[7,8],ymm14[9],mem[10],ymm14[11],mem[12,13],ymm14[14],mem[15]
+; AVX512F-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
+; AVX512F-FAST-NEXT:    vpblendw $181, {{[-0-9]+}}(%r{{[sb]}}p), %ymm6, %ymm6 # 32-byte Folded Reload
+; AVX512F-FAST-NEXT:    # ymm6 = mem[0],ymm6[1],mem[2],ymm6[3],mem[4,5],ymm6[6],mem[7,8],ymm6[9],mem[10],ymm6[11],mem[12,13],ymm6[14],mem[15]
 ; AVX512F-FAST-NEXT:    vextracti128 $1, %ymm6, %xmm8
 ; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} xmm6 = xmm6[0,1],xmm8[2,3],xmm6[4,5,6],xmm8[7]
 ; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} xmm8 = [6,7,2,3,12,13,6,7,0,1,10,11,4,5,14,15]
@@ -7569,14 +7592,13 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm6 = ymm3[0],ymm6[1,2,3,4,5,6,7],ymm3[8],ymm6[9,10,11,12,13,14,15]
 ; AVX512F-FAST-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm6[4,5,6,7]
 ; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm3, %zmm1, %zmm1
-; AVX512F-FAST-NEXT:    vmovdqa64 %ymm30, %ymm3
-; AVX512F-FAST-NEXT:    vmovdqa64 %ymm22, %ymm6
+; AVX512F-FAST-NEXT:    vmovdqa64 %ymm23, %ymm3
+; AVX512F-FAST-NEXT:    vmovdqa64 %ymm25, %ymm6
 ; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm6[1],ymm3[2,3],ymm6[4],ymm3[5],ymm6[6],ymm3[7,8],ymm6[9],ymm3[10,11],ymm6[12],ymm3[13],ymm6[14],ymm3[15]
 ; AVX512F-FAST-NEXT:    vextracti128 $1, %ymm3, %xmm6
 ; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} xmm3 = xmm6[0,1,2],xmm3[3,4],xmm6[5,6,7]
 ; AVX512F-FAST-NEXT:    vpshufb %xmm7, %xmm3, %xmm3
-; AVX512F-FAST-NEXT:    vmovdqa64 %ymm18, %ymm6
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm6 = ymm13[0,1],ymm6[2],ymm13[3],ymm6[4],ymm13[5,6],ymm6[7],ymm13[8,9],ymm6[10],ymm13[11],ymm6[12],ymm13[13,14],ymm6[15]
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm6 = ymm13[0,1],ymm12[2],ymm13[3],ymm12[4],ymm13[5,6],ymm12[7],ymm13[8,9],ymm12[10],ymm13[11],ymm12[12],ymm13[13,14],ymm12[15]
 ; AVX512F-FAST-NEXT:    vpermd %ymm6, %ymm4, %ymm4
 ; AVX512F-FAST-NEXT:    vpshufb %ymm5, %ymm4, %ymm4
 ; AVX512F-FAST-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2],ymm4[3,4,5,6,7]
@@ -7598,15 +7620,14 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512F-FAST-NEXT:    vmovaps %zmm3, 64(%rdx)
 ; AVX512F-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
 ; AVX512F-FAST-NEXT:    vmovaps %zmm3, (%rdx)
-; AVX512F-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-FAST-NEXT:    vmovaps %zmm3, 64(%rcx)
+; AVX512F-FAST-NEXT:    vmovdqa64 %zmm28, 64(%rcx)
 ; AVX512F-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
 ; AVX512F-FAST-NEXT:    vmovaps %zmm3, (%rcx)
 ; AVX512F-FAST-NEXT:    vmovdqa64 %zmm2, 64(%r8)
 ; AVX512F-FAST-NEXT:    vmovdqa64 %zmm21, (%r8)
 ; AVX512F-FAST-NEXT:    vmovdqa64 %zmm0, 64(%r9)
 ; AVX512F-FAST-NEXT:    vmovdqa64 %zmm1, (%r9)
-; AVX512F-FAST-NEXT:    addq $712, %rsp # imm = 0x2C8
+; AVX512F-FAST-NEXT:    addq $552, %rsp # imm = 0x228
 ; AVX512F-FAST-NEXT:    vzeroupper
 ; AVX512F-FAST-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-7.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-7.ll
index 893bff29b21e45a6960e824d40c5026acd55deea..eb44ca1800986e4766530d0e4c91f1191371bef9 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-7.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-7.ll
@@ -2573,42 +2573,44 @@ define void @load_i16_stride7_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX2-FAST-PERLANE-NEXT:    vmovdqa 208(%rdi), %xmm12
 ; AVX2-FAST-PERLANE-NEXT:    vmovdqa 192(%rdi), %xmm13
 ; AVX2-FAST-PERLANE-NEXT:    vpblendw {{.*#+}} xmm14 = xmm13[0,1,2,3,4,5],xmm12[6],xmm13[7]
-; AVX2-FAST-PERLANE-NEXT:    vpshufb {{.*#+}} xmm14 = xmm14[u,u,u,u,u,u,u,u,u,u,0,1,14,15,12,13]
+; AVX2-FAST-PERLANE-NEXT:    vmovdqa {{.*#+}} ymm15 = <u,u,u,u,u,u,u,u,2,3,0,1,14,15,12,13,26,27,26,27,26,27,26,27,18,19,16,17,30,31,28,29>
+; AVX2-FAST-PERLANE-NEXT:    vpshufb %xmm15, %xmm14, %xmm14
 ; AVX2-FAST-PERLANE-NEXT:    vinserti128 $1, %xmm14, %ymm0, %ymm14
 ; AVX2-FAST-PERLANE-NEXT:    vpblendw {{.*#+}} ymm4 = ymm4[0,1,2,3,4],ymm14[5,6,7],ymm4[8,9,10,11,12],ymm14[13,14,15]
 ; AVX2-FAST-PERLANE-NEXT:    vpblendd {{.*#+}} ymm14 = ymm2[0,1,2],ymm3[3],ymm2[4,5],ymm3[6],ymm2[7]
-; AVX2-FAST-PERLANE-NEXT:    vpermq {{.*#+}} ymm15 = ymm14[2,3,0,1]
-; AVX2-FAST-PERLANE-NEXT:    vpblendw {{.*#+}} ymm14 = ymm15[0],ymm14[1,2,3,4,5,6],ymm15[7,8],ymm14[9,10,11,12,13,14],ymm15[15]
-; AVX2-FAST-PERLANE-NEXT:    vpblendd {{.*#+}} ymm15 = ymm1[0,1],ymm0[2,3],ymm1[4,5],ymm0[6,7]
-; AVX2-FAST-PERLANE-NEXT:    vextracti128 $1, %ymm15, %xmm7
-; AVX2-FAST-PERLANE-NEXT:    vpshufb %xmm11, %xmm7, %xmm7
-; AVX2-FAST-PERLANE-NEXT:    vpshufb {{.*#+}} xmm11 = xmm15[10,11,6,7,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX2-FAST-PERLANE-NEXT:    vpunpcklwd {{.*#+}} xmm7 = xmm11[0],xmm7[0],xmm11[1],xmm7[1],xmm11[2],xmm7[2],xmm11[3],xmm7[3]
-; AVX2-FAST-PERLANE-NEXT:    vpshufb {{.*#+}} ymm11 = ymm14[u,u,u,u,u,u,u,u,2,3,0,1,14,15,12,13,26,27,26,27,26,27,26,27,18,19,16,17,30,31,28,29]
-; AVX2-FAST-PERLANE-NEXT:    vpblendw {{.*#+}} ymm4 = ymm11[0],ymm4[1,2,3,4,5,6,7],ymm11[8],ymm4[9,10,11,12,13,14,15]
-; AVX2-FAST-PERLANE-NEXT:    vpblendd {{.*#+}} xmm7 = xmm7[0,1],xmm11[2,3]
+; AVX2-FAST-PERLANE-NEXT:    vpermq {{.*#+}} ymm7 = ymm14[2,3,0,1]
+; AVX2-FAST-PERLANE-NEXT:    vpblendw {{.*#+}} ymm7 = ymm7[0],ymm14[1,2,3,4,5,6],ymm7[7,8],ymm14[9,10,11,12,13,14],ymm7[15]
+; AVX2-FAST-PERLANE-NEXT:    vpshufb %ymm15, %ymm7, %ymm7
+; AVX2-FAST-PERLANE-NEXT:    vpblendd {{.*#+}} ymm14 = ymm1[0,1],ymm0[2,3],ymm1[4,5],ymm0[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vextracti128 $1, %ymm14, %xmm15
+; AVX2-FAST-PERLANE-NEXT:    vpshufb %xmm11, %xmm15, %xmm11
+; AVX2-FAST-PERLANE-NEXT:    vpshufb {{.*#+}} xmm14 = xmm14[10,11,6,7,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX2-FAST-PERLANE-NEXT:    vpunpcklwd {{.*#+}} xmm11 = xmm14[0],xmm11[0],xmm14[1],xmm11[1],xmm14[2],xmm11[2],xmm14[3],xmm11[3]
+; AVX2-FAST-PERLANE-NEXT:    vpblendw {{.*#+}} ymm4 = ymm7[0],ymm4[1,2,3,4,5,6,7],ymm7[8],ymm4[9,10,11,12,13,14,15]
+; AVX2-FAST-PERLANE-NEXT:    vpblendd {{.*#+}} xmm7 = xmm11[0,1],xmm7[2,3]
 ; AVX2-FAST-PERLANE-NEXT:    vpblendd {{.*#+}} ymm11 = ymm7[0,1,2,3],ymm4[4,5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vpblendd {{.*#+}} ymm4 = ymm6[0,1],ymm5[2,3],ymm6[4,5],ymm5[6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vextracti128 $1, %ymm4, %xmm5
 ; AVX2-FAST-PERLANE-NEXT:    vpblendw {{.*#+}} xmm4 = xmm5[0,1,2],xmm4[3],xmm5[4],xmm4[5],xmm5[6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpblendw {{.*#+}} xmm5 = xmm12[0],xmm13[1],xmm12[2,3,4,5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[u,u,10,11,8,9,6,7,4,5,u,u,u,u,u,u]
 ; AVX2-FAST-PERLANE-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX2-FAST-PERLANE-NEXT:    vpshufb {{.*#+}} xmm5 = xmm5[u,u,u,u,u,u,u,u,u,u,2,3,0,1,14,15]
+; AVX2-FAST-PERLANE-NEXT:    vpblendw {{.*#+}} xmm5 = xmm12[0],xmm13[1],xmm12[2,3,4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovdqa {{.*#+}} ymm6 = <u,u,u,u,u,u,u,u,4,5,2,3,0,1,14,15,28,29,28,29,28,29,28,29,20,21,18,19,16,17,30,31>
+; AVX2-FAST-PERLANE-NEXT:    vpshufb %xmm6, %xmm5, %xmm5
 ; AVX2-FAST-PERLANE-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
 ; AVX2-FAST-PERLANE-NEXT:    vpblendw {{.*#+}} ymm4 = ymm4[0,1,2,3,4],ymm5[5,6,7],ymm4[8,9,10,11,12],ymm5[13,14,15]
 ; AVX2-FAST-PERLANE-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2,3],ymm2[4],ymm3[5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vpermq {{.*#+}} ymm3 = ymm2[2,3,0,1]
 ; AVX2-FAST-PERLANE-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0],ymm3[1],ymm2[2,3,4,5,6,7,8],ymm3[9],ymm2[10,11,12,13,14,15]
+; AVX2-FAST-PERLANE-NEXT:    vpshufb %ymm6, %ymm2, %ymm2
 ; AVX2-FAST-PERLANE-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3],ymm1[4,5],ymm0[6],ymm1[7]
 ; AVX2-FAST-PERLANE-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX2-FAST-PERLANE-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[u,u,u,u,u,u,u,u,10,11,6,7,u,u,u,u]
 ; AVX2-FAST-PERLANE-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,4,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
-; AVX2-FAST-PERLANE-NEXT:    vpshufb {{.*#+}} ymm1 = ymm2[u,u,u,u,u,u,u,u,4,5,2,3,0,1,14,15,28,29,28,29,28,29,28,29,20,21,18,19,16,17,30,31]
-; AVX2-FAST-PERLANE-NEXT:    vpblendw {{.*#+}} ymm2 = ymm1[0],ymm4[1,2,3,4,5,6,7],ymm1[8],ymm4[9,10,11,12,13,14,15]
-; AVX2-FAST-PERLANE-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
-; AVX2-FAST-PERLANE-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0],ymm4[1,2,3,4,5,6,7],ymm2[8],ymm4[9,10,11,12,13,14,15]
+; AVX2-FAST-PERLANE-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3]
+; AVX2-FAST-PERLANE-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
 ; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm1, (%rsi)
 ; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
@@ -5686,16 +5688,16 @@ define void @load_i16_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ;
 ; AVX512F-ONLY-SLOW-LABEL: load_i16_stride7_vf32:
 ; AVX512F-ONLY-SLOW:       # %bb.0:
-; AVX512F-ONLY-SLOW-NEXT:    pushq %rax
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqa (%rdi), %ymm2
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 32(%rdi), %ymm3
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 64(%rdi), %ymm12
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 96(%rdi), %ymm4
-; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm4[0],ymm12[1],ymm4[2,3,4],ymm12[5],ymm4[6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 96(%rdi), %ymm11
+; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm11[0],ymm12[1],ymm11[2,3,4],ymm12[5],ymm11[6,7]
 ; AVX512F-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
 ; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm0[6,7,12,13,2,3,16,17,30,31],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1],ymm3[2],ymm2[3,4,5],ymm3[6],ymm2[7]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm2, %ymm13
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm3, %ymm4
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm2, %ymm8
 ; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm1, %xmm2
 ; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm2[4],xmm1[5],xmm2[6],xmm1[7]
 ; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[0,1,14,15,12,13,10,11,8,9],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm1[u,u,u,u,u,u,u,u,u,u,u,u]
@@ -5711,662 +5713,655 @@ define void @load_i16_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm2
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 192(%rdi), %ymm0
 ; AVX512F-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm1 = ymm0[0,1,0,2]
-; AVX512F-ONLY-SLOW-NEXT:    vpshufd {{.*#+}} ymm5 = ymm1[0,1,2,1,4,5,6,5]
-; AVX512F-ONLY-SLOW-NEXT:    vpshufhw {{.*#+}} ymm5 = ymm5[0,1,2,3,4,5,6,5,8,9,10,11,12,13,14,13]
-; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5,6],ymm5[7]
-; AVX512F-ONLY-SLOW-NEXT:    vpbroadcastw 252(%rdi), %xmm5
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 224(%rdi), %xmm7
-; AVX512F-ONLY-SLOW-NEXT:    vpshufd {{.*#+}} xmm6 = xmm7[0,1,0,3]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %xmm7, %xmm14
-; AVX512F-ONLY-SLOW-NEXT:    vpshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,4,7,6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vpunpckhdq {{.*#+}} xmm5 = xmm6[2],xmm5[2],xmm6[3],xmm5[3]
+; AVX512F-ONLY-SLOW-NEXT:    vpshufd {{.*#+}} ymm3 = ymm1[0,1,2,1,4,5,6,5]
+; AVX512F-ONLY-SLOW-NEXT:    vpshufhw {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5,6,5,8,9,10,11,12,13,14,13]
+; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5,6],ymm3[7]
+; AVX512F-ONLY-SLOW-NEXT:    vpbroadcastw 252(%rdi), %xmm3
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 224(%rdi), %xmm6
+; AVX512F-ONLY-SLOW-NEXT:    vpshufd {{.*#+}} xmm5 = xmm6[0,1,0,3]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %xmm6, %xmm13
+; AVX512F-ONLY-SLOW-NEXT:    vpshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,7,6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vpunpckhdq {{.*#+}} xmm3 = xmm5[2],xmm3[2],xmm5[3],xmm3[3]
 ; AVX512F-ONLY-SLOW-NEXT:    movw $992, %ax # imm = 0x3E0
 ; AVX512F-ONLY-SLOW-NEXT:    kmovw %eax, %k1
-; AVX512F-ONLY-SLOW-NEXT:    vinserti32x4 $2, %xmm5, %zmm2, %zmm21 {%k1}
+; AVX512F-ONLY-SLOW-NEXT:    vinserti32x4 $2, %xmm3, %zmm2, %zmm21 {%k1}
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 256(%rdi), %ymm5
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 288(%rdi), %ymm6
-; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm6[0,1],ymm5[2,3],ymm6[4,5],ymm5[6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %ymm5, %ymm27
-; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm2, %xmm7
-; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} xmm7 = xmm7[0,1,2],xmm2[3],xmm7[4],xmm2[5],xmm7[6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 240(%rdi), %xmm15
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 240(%rdi), %xmm14
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 80(%rdi), %xmm2
-; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm4[0,1],ymm12[2],ymm4[3,4],ymm12[5],ymm4[6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} ymm2 = ymm8[0,1,2],ymm2[3],ymm8[4,5,6,7,8,9,10],ymm2[11],ymm8[12,13,14,15]
+; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm11[0,1],ymm12[2],ymm11[3,4],ymm12[5],ymm11[6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} ymm2 = ymm3[0,1,2],ymm2[3],ymm3[4,5,6,7,8,9,10],ymm2[11],ymm3[12,13,14,15]
 ; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} ymm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[8,9,6,7,4,5,18,19],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm13[0,1,2],ymm3[3],ymm13[4,5],ymm3[6],ymm13[7]
-; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm8, %xmm11
-; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} xmm8 = xmm11[0],xmm8[1],xmm11[2,3,4,5],xmm8[6],xmm11[7]
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} ymm8 = ymm8[2,3,0,1,14,15,12,13,10,11],zero,zero,zero,zero,zero,zero,zero,zero,ymm8[u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vpor %ymm2, %ymm8, %ymm2
+; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm8[0,1,2],ymm4[3],ymm8[4,5],ymm4[6],ymm8[7]
+; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm3, %xmm7
+; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} xmm3 = xmm7[0],xmm3[1],xmm7[2,3,4,5],xmm3[6],xmm7[7]
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} ymm3 = ymm3[2,3,0,1,14,15,12,13,10,11],zero,zero,zero,zero,zero,zero,zero,zero,ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512F-ONLY-SLOW-NEXT:    vpor %ymm2, %ymm3, %ymm2
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm9[0,1],ymm10[2],ymm9[3,4,5],ymm10[6],ymm9[7]
-; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm2, %xmm8
-; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm8[4],xmm2[5],xmm8[6],xmm2[7]
+; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm2, %xmm3
+; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm3[4],xmm2[5],xmm3[6],xmm2[7]
 ; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[0,1,0,1,14,15,12,13,10,11,8,9,u,u,u,u]
 ; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
 ; AVX512F-ONLY-SLOW-NEXT:    vpshufd {{.*#+}} ymm1 = ymm1[0,1,1,3,4,5,5,7]
 ; AVX512F-ONLY-SLOW-NEXT:    vpshufhw {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5,5,6,8,9,10,11,12,13,13,14]
 ; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5,6],ymm1[7]
-; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} xmm2 = xmm15[0],xmm14[1],xmm15[2,3,4,5,6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} xmm2 = xmm14[0],xmm13[1],xmm14[2,3,4,5,6,7]
 ; AVX512F-ONLY-SLOW-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[0,3,2,3]
 ; AVX512F-ONLY-SLOW-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm2[1,0,3,3,4,5,6,7]
 ; AVX512F-ONLY-SLOW-NEXT:    vinserti32x4 $2, %xmm2, %zmm1, %zmm22
-; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm4[0,1],ymm12[2,3],ymm4[4,5],ymm12[6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm11[0,1],ymm12[2,3],ymm11[4,5],ymm12[6,7]
 ; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm1, %xmm2
 ; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4],ymm1[5,6,7,8,9,10,11],ymm2[12],ymm1[13,14,15]
 ; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} ymm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm1[10,11,8,9,6,7,20,21],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0],ymm13[1],ymm3[2,3],ymm13[4],ymm3[5,6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm2, %xmm8
-; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0],xmm8[1],xmm2[2,3,4,5],xmm8[6],xmm2[7]
+; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm4[0],ymm8[1],ymm4[2,3],ymm8[4],ymm4[5,6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm2, %xmm3
+; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0],xmm3[1],xmm2[2,3,4,5],xmm3[6],xmm2[7]
 ; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} ymm2 = ymm2[4,5,2,3,0,1,14,15,12,13],zero,zero,zero,zero,zero,zero,zero,zero,ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vpor %ymm1, %ymm2, %ymm1
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vporq %ymm1, %ymm2, %ymm19
 ; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm9[0,1,2],ymm10[3],ymm9[4,5],ymm10[6],ymm9[7]
 ; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm1, %xmm2
 ; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2,3,4,5],xmm1[6],xmm2[7]
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0,1,2,3,0,1,14,15,12,13,10,11,u,u,u,u]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa {{.*#+}} ymm2 = <0,1,2,3,0,1,14,15,12,13,10,11,8,9,128,128,128,128,128,128,128,128,128,128,u,u,u,u,u,u,u,u>
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %ymm2, %ymm28
 ; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
 ; AVX512F-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm2 = ymm0[0,1,1,2]
 ; AVX512F-ONLY-SLOW-NEXT:    vpshufd {{.*#+}} ymm2 = ymm2[0,1,0,3,4,5,4,7]
 ; AVX512F-ONLY-SLOW-NEXT:    vpshufhw {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5,4,7,8,9,10,11,12,13,12,15]
 ; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5,6],ymm2[7]
-; AVX512F-ONLY-SLOW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm14[0],xmm15[0],xmm14[1],xmm15[1],xmm14[2],xmm15[2],xmm14[3],xmm15[3]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %xmm14, %xmm17
-; AVX512F-ONLY-SLOW-NEXT:    vpshufd {{.*#+}} xmm2 = xmm5[2,1,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %xmm5, %xmm29
+; AVX512F-ONLY-SLOW-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm13[0],xmm14[0],xmm13[1],xmm14[1],xmm13[2],xmm14[2],xmm13[3],xmm14[3]
+; AVX512F-ONLY-SLOW-NEXT:    vpshufd {{.*#+}} xmm2 = xmm3[2,1,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %xmm3, %xmm27
 ; AVX512F-ONLY-SLOW-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm2[0,3,2,3,4,5,6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vinserti32x4 $2, %xmm2, %zmm1, %zmm24
-; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm4[0,1,2],ymm12[3],ymm4[4,5],ymm12[6],ymm4[7]
+; AVX512F-ONLY-SLOW-NEXT:    vinserti32x4 $2, %xmm2, %zmm1, %zmm23
+; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm11[0,1,2],ymm12[3],ymm11[4,5],ymm12[6],ymm11[7]
 ; AVX512F-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[1,3,2,3]
 ; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} ymm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm1[4,5,10,11,0,1,22,23],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0],ymm13[1],ymm3[2,3,4],ymm13[5],ymm3[6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %ymm13, %ymm25
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %ymm3, %ymm26
-; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm2, %xmm8
-; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} xmm2 = xmm8[0],xmm2[1],xmm8[2],xmm2[3],xmm8[4,5,6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm4[0],ymm8[1],ymm4[2,3,4],ymm8[5],ymm4[6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %ymm8, %ymm25
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %ymm4, %ymm26
+; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm2, %xmm3
+; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0],xmm2[1],xmm3[2],xmm2[3],xmm3[4,5,6,7]
 ; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} ymm2 = ymm2[6,7,4,5,2,3,0,1,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u]
 ; AVX512F-ONLY-SLOW-NEXT:    vporq %ymm1, %ymm2, %ymm20
-; AVX512F-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm23 = ymm0[0,1,1,3]
+; AVX512F-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm24 = ymm0[0,1,1,3]
 ; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm10[0],ymm9[1],ymm10[2,3,4],ymm9[5],ymm10[6,7]
 ; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4,5,6,7]
 ; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,6,7,4,5,2,3,0,1,14,15,u,u,u,u]
 ; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512F-ONLY-SLOW-NEXT:    vpshufd {{.*#+}} ymm1 = ymm23[0,1,2,1,4,5,6,5]
+; AVX512F-ONLY-SLOW-NEXT:    vpshufd {{.*#+}} ymm1 = ymm24[0,1,2,1,4,5,6,5]
 ; AVX512F-ONLY-SLOW-NEXT:    vpshufhw {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5,6,5,8,9,10,11,12,13,14,13]
 ; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,6],ymm1[7]
 ; AVX512F-ONLY-SLOW-NEXT:    vpbroadcastw 232(%rdi), %xmm1
-; AVX512F-ONLY-SLOW-NEXT:    vpsrlq $48, %xmm15, %xmm2
+; AVX512F-ONLY-SLOW-NEXT:    vpsrlq $48, %xmm14, %xmm2
 ; AVX512F-ONLY-SLOW-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
-; AVX512F-ONLY-SLOW-NEXT:    vinserti32x4 $2, %xmm1, %zmm0, %zmm19
+; AVX512F-ONLY-SLOW-NEXT:    vinserti32x4 $2, %xmm1, %zmm0, %zmm18
 ; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm10[0,1],ymm9[2],ymm10[3,4],ymm9[5],ymm10[6,7]
 ; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2],xmm1[3],xmm0[4],xmm1[5,6,7]
 ; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[8,9,8,9,6,7,4,5,2,3,u,u,u,u,u,u]
 ; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 208(%rdi), %xmm13
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 208(%rdi), %xmm2
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 192(%rdi), %xmm1
-; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} xmm2 = xmm1[0,1,2,3,4,5],xmm13[6],xmm1[7]
-; AVX512F-ONLY-SLOW-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[0,1,0,3]
-; AVX512F-ONLY-SLOW-NEXT:    vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,4,7,6]
-; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm2[5,6,7],ymm0[8,9,10,11,12],ymm2[13,14,15]
-; AVX512F-ONLY-SLOW-NEXT:    vpsrld $16, %xmm14, %xmm2
-; AVX512F-ONLY-SLOW-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm2[4],xmm15[4],xmm2[5],xmm15[5],xmm2[6],xmm15[6],xmm2[7],xmm15[7]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %xmm15, %xmm16
-; AVX512F-ONLY-SLOW-NEXT:    vinserti32x4 $2, %xmm2, %zmm0, %zmm18
+; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} xmm3 = xmm1[0,1,2,3,4,5],xmm2[6],xmm1[7]
+; AVX512F-ONLY-SLOW-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[0,1,0,3]
+; AVX512F-ONLY-SLOW-NEXT:    vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,4,7,6]
+; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm3[5,6,7],ymm0[8,9,10,11,12],ymm3[13,14,15]
+; AVX512F-ONLY-SLOW-NEXT:    vpsrld $16, %xmm13, %xmm3
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %xmm13, %xmm31
+; AVX512F-ONLY-SLOW-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm3[4],xmm14[4],xmm3[5],xmm14[5],xmm3[6],xmm14[6],xmm3[7],xmm14[7]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %xmm14, %xmm16
+; AVX512F-ONLY-SLOW-NEXT:    vinserti32x4 $2, %xmm3, %zmm0, %zmm17
+; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm6[0,1],ymm5[2,3],ymm6[4,5],ymm5[6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1,2],xmm0[3],xmm3[4],xmm0[5],xmm3[6,7]
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 352(%rdi), %ymm15
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 320(%rdi), %ymm3
-; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2],ymm15[3],ymm3[4,5],ymm15[6],ymm3[7]
-; AVX512F-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm8 = ymm2[2,3,0,1]
-; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm8[6],ymm2[7,8,9,10,11,12,13],ymm8[14],ymm2[15]
-; AVX512F-ONLY-SLOW-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[0,2,2,1]
-; AVX512F-ONLY-SLOW-NEXT:    vpshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,4,7,6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} ymm2 = ymm2[0,1,14,15,12,13,10,11,4,5,6,7,8,9,2,3,16,17,30,31,28,29,26,27,20,21,22,23,24,25,18,19]
-; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} xmm7 = xmm2[0,1,2],xmm7[3,4,5,6],xmm2[7]
-; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm7[0,1,2,3],ymm2[4,5,6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 384(%rdi), %ymm7
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 416(%rdi), %ymm8
-; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm8[0,1],ymm7[2],ymm8[3,4],ymm7[5],ymm8[6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm11
-; AVX512F-ONLY-SLOW-NEXT:    vpshuflw {{.*#+}} xmm11 = xmm11[0,1,3,1,4,5,6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 320(%rdi), %ymm4
+; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm4[0,1,2],ymm15[3],ymm4[4,5],ymm15[6],ymm4[7]
+; AVX512F-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm13 = ymm8[2,3,0,1]
+; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],ymm13[6],ymm8[7,8,9,10,11,12,13],ymm13[14],ymm8[15]
+; AVX512F-ONLY-SLOW-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[0,2,2,1]
+; AVX512F-ONLY-SLOW-NEXT:    vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,7,6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} ymm8 = ymm8[0,1,14,15,12,13,10,11,4,5,6,7,8,9,2,3,16,17,30,31,28,29,26,27,20,21,22,23,24,25,18,19]
+; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} xmm3 = xmm8[0,1,2],xmm3[3,4,5,6],xmm8[7]
+; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm8[4,5,6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 384(%rdi), %ymm8
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 416(%rdi), %ymm13
+; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm13[0,1],ymm8[2],ymm13[3,4],ymm8[5],ymm13[6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm7
+; AVX512F-ONLY-SLOW-NEXT:    vpshuflw {{.*#+}} xmm7 = xmm7[0,1,3,1,4,5,6,7]
 ; AVX512F-ONLY-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[2,1,2,3]
 ; AVX512F-ONLY-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,1,0,2,4,5,6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm11[0],xmm0[1],xmm11[1],xmm0[2],xmm11[2],xmm0[3],xmm11[3]
+; AVX512F-ONLY-SLOW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm7[0],xmm0[1],xmm7[1],xmm0[2],xmm7[2],xmm0[3],xmm7[3]
 ; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5],ymm0[6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %ymm0, %ymm30
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %ymm27, %ymm5
+; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3,4,5],ymm0[6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %ymm0, %ymm29
 ; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm6[0,1,2],ymm5[3],ymm6[4,5],ymm5[6],ymm6[7]
-; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm2
-; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm2[0,1,2,3],xmm0[4],xmm2[5],xmm0[6],xmm2[7]
-; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm15[0],ymm3[1],ymm15[2,3],ymm3[4],ymm15[5,6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm11 = ymm2[2,3,0,1]
-; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} ymm2 = ymm11[0],ymm2[1,2,3,4,5,6],ymm11[7,8],ymm2[9,10,11,12,13,14],ymm11[15]
+; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm3[0,1,2,3],xmm0[4],xmm3[5],xmm0[6],xmm3[7]
+; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm15[0],ymm4[1],ymm15[2,3],ymm4[4],ymm15[5,6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm7 = ymm3[2,3,0,1]
+; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} ymm3 = ymm7[0],ymm3[1,2,3,4,5,6],ymm7[7,8],ymm3[9,10,11,12,13,14],ymm7[15]
 ; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,u,12,13,10,11,8,9,6,7,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} ymm2 = ymm2[2,3,0,1,14,15,12,13,4,5,4,5,4,5,4,5,18,19,16,17,30,31,28,29,20,21,20,21,20,21,20,21]
-; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3,4,5,6],xmm2[7]
-; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm8[0,1],ymm7[2,3],ymm8[4,5],ymm7[6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm2, %xmm11
-; AVX512F-ONLY-SLOW-NEXT:    vpshufd {{.*#+}} xmm11 = xmm11[2,1,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vpshuflw {{.*#+}} xmm11 = xmm11[0,1,0,2,4,5,6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,1,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm2[0,1,1,3,4,5,6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm11[0],xmm2[1],xmm11[1],xmm2[2],xmm11[2],xmm2[3],xmm11[3]
-; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm2[6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %ymm0, %ymm28
-; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm15[0],ymm3[1],ymm15[2,3,4],ymm3[5],ymm15[6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm2 = ymm0[2,3,0,1]
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} ymm3 = ymm3[2,3,0,1,14,15,12,13,4,5,4,5,4,5,4,5,18,19,16,17,30,31,28,29,20,21,20,21,20,21,20,21]
+; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm3[0,1,2],xmm0[3,4,5,6],xmm3[7]
+; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm3[4,5,6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm13[0,1],ymm8[2,3],ymm13[4,5],ymm8[6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm3, %xmm7
+; AVX512F-ONLY-SLOW-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[2,1,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vpshuflw {{.*#+}} xmm7 = xmm7[0,1,0,2,4,5,6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,1,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm3[0,1,1,3,4,5,6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm7[0],xmm3[1],xmm7[1],xmm3[2],xmm7[2],xmm3[3],xmm7[3]
+; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm3[6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %ymm0, %ymm30
+; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm15[0],ymm4[1],ymm15[2,3,4],ymm4[5],ymm15[6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm3 = ymm0[2,3,0,1]
 ; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,u,u,6,7,20,21,u,u,16,17,30,31,u,u,u,u,u,u,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm2[1],ymm0[2,3,4,5,6,7,8],ymm2[9],ymm0[10,11,12,13,14,15]
-; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm5[0,1],ymm6[2],ymm5[3,4,5],ymm6[6],ymm5[7]
-; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm2, %xmm11
-; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm11[4],xmm2[5],xmm11[6],xmm2[7]
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} ymm2 = ymm2[0,1,2,3,0,1,14,15,12,13,10,11,8,9],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[u,u,u,u,u,u,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $244, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm2
-; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm8[0,1,2],ymm7[3],ymm8[4,5],ymm7[6],ymm8[7]
-; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm11
+; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm3[1],ymm0[2,3,4,5,6,7,8],ymm3[9],ymm0[10,11,12,13,14,15]
+; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm5[0,1],ymm6[2],ymm5[3,4,5],ymm6[6],ymm5[7]
+; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm3, %xmm7
+; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm7[4],xmm3[5],xmm7[6],xmm3[7]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %ymm28, %ymm7
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb %ymm7, %ymm3, %ymm3
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $244, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm3
+; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm13[0,1,2],ymm8[3],ymm13[4,5],ymm8[6],ymm13[7]
+; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm7
 ; AVX512F-ONLY-SLOW-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,6,4]
-; AVX512F-ONLY-SLOW-NEXT:    vpshufd {{.*#+}} xmm11 = xmm11[0,1,2,1]
-; AVX512F-ONLY-SLOW-NEXT:    vpshufhw {{.*#+}} xmm11 = xmm11[0,1,2,3,4,5,5,7]
-; AVX512F-ONLY-SLOW-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm11[4],xmm0[5],xmm11[5],xmm0[6],xmm11[6],xmm0[7],xmm11[7]
+; AVX512F-ONLY-SLOW-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[0,1,2,1]
+; AVX512F-ONLY-SLOW-NEXT:    vpshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,4,5,5,7]
+; AVX512F-ONLY-SLOW-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm7[4],xmm0[5],xmm7[5],xmm0[6],xmm7[6],xmm0[7],xmm7[7]
 ; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5],ymm0[6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %ymm0, %ymm31
+; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3,4,5],ymm0[6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %ymm0, %ymm28
 ; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm10[0],ymm9[1],ymm10[2,3],ymm9[4],ymm10[5,6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm2
-; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2,3,4,5],xmm2[6],xmm0[7]
+; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm3[1],xmm0[2,3,4,5],xmm3[6],xmm0[7]
 ; AVX512F-ONLY-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,1,0,4,5,6,7]
 ; AVX512F-ONLY-SLOW-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,6,6,7]
 ; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512F-ONLY-SLOW-NEXT:    vpshufd {{.*#+}} ymm2 = ymm23[0,1,2,0,4,5,6,4]
-; AVX512F-ONLY-SLOW-NEXT:    vpshufhw {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5,7,4,8,9,10,11,12,13,15,12]
-; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,6],ymm2[7]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %xmm29, %xmm2
-; AVX512F-ONLY-SLOW-NEXT:    vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,6,5,6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,2,2,2]
-; AVX512F-ONLY-SLOW-NEXT:    vinserti32x4 $2, %xmm2, %zmm0, %zmm27
+; AVX512F-ONLY-SLOW-NEXT:    vpshufd {{.*#+}} ymm3 = ymm24[0,1,2,0,4,5,6,4]
+; AVX512F-ONLY-SLOW-NEXT:    vpshufhw {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5,7,4,8,9,10,11,12,13,15,12]
+; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,6],ymm3[7]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %xmm27, %xmm3
+; AVX512F-ONLY-SLOW-NEXT:    vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,6,5,6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,2,2,2]
+; AVX512F-ONLY-SLOW-NEXT:    vinserti32x4 $2, %xmm3, %zmm0, %zmm27
 ; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm5[0,1,2],ymm6[3],ymm5[4,5],ymm6[6],ymm5[7]
-; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm2
-; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm2[0],xmm0[1],xmm2[2,3,4,5],xmm0[6],xmm2[7]
-; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm7[0,1],ymm8[2],ymm7[3,4,5],ymm8[6],ymm7[7]
-; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm2, %xmm14
-; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm14[4],xmm2[5],xmm14[6],xmm2[7]
+; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm3[0],xmm0[1],xmm3[2,3,4,5],xmm0[6],xmm3[7]
+; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm8[0,1],ymm13[2],ymm8[3,4,5],ymm13[6],ymm8[7]
+; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm3, %xmm14
+; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1,2,3],xmm14[4],xmm3[5],xmm14[6],xmm3[7]
 ; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,2,3,2,3,0,1,14,15,12,13,10,11],zero,zero
-; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm14 = ymm15[0,1],ymm3[2],ymm15[3,4],ymm3[5],ymm15[6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm14 = ymm15[0,1],ymm4[2],ymm15[3,4],ymm4[5],ymm15[6,7]
 ; AVX512F-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm14 = ymm14[1,1,2,0]
 ; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} ymm14 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm14[0,1,22,23,28,29,18,19],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; AVX512F-ONLY-SLOW-NEXT:    vpor %ymm0, %ymm14, %ymm0
-; AVX512F-ONLY-SLOW-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm2[0,0,0,0,4,5,6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,6,5,4]
-; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} ymm2 = ymm14[0,1,2],ymm2[3,4,5,6,7],ymm14[8,9,10],ymm2[11,12,13,14,15]
-; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %ymm0, %ymm29
+; AVX512F-ONLY-SLOW-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm3[0,0,0,0,4,5,6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,7,6,5,4]
+; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} ymm3 = ymm14[0,1,2],ymm3[3,4,5,6,7],ymm14[8,9,10],ymm3[11,12,13,14,15]
+; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm3[4,5,6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %ymm0, %ymm24
 ; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm10[0,1],ymm9[2,3],ymm10[4,5],ymm9[6,7]
 ; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm9
 ; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm9[0,1,2],xmm0[3],xmm9[4],xmm0[5],xmm9[6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} xmm1 = xmm13[0],xmm1[1],xmm13[2,3,4,5,6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2,3,4,5,6,7]
 ; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[8,9,10,11,8,9,6,7,4,5,u,u,u,u,u,u]
 ; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
 ; AVX512F-ONLY-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,1,0,3]
 ; AVX512F-ONLY-SLOW-NEXT:    vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,4,7]
 ; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
 ; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7],ymm0[8,9,10,11,12],ymm1[13,14,15]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %xmm17, %xmm1
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %xmm31, %xmm1
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %xmm16, %xmm2
 ; AVX512F-ONLY-SLOW-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
 ; AVX512F-ONLY-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,1,2,3]
 ; AVX512F-ONLY-SLOW-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,3,2,3,4,5,6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vinserti32x4 $2, %xmm1, %zmm0, %zmm23
-; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm12[0,1],ymm4[2],ymm12[3,4,5],ymm4[6],ymm12[7]
+; AVX512F-ONLY-SLOW-NEXT:    vinserti32x4 $2, %xmm1, %zmm0, %zmm16
+; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm12[0,1],ymm11[2],ymm12[3,4,5],ymm11[6],ymm12[7]
 ; AVX512F-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm1 = ymm0[2,3,0,1]
 ; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[u,u,u,u,u,u,u,u,0,1,14,15,u,u,10,11,24,25,24,25,24,25,24,25,16,17,30,31,u,u,26,27]
 ; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6],ymm0[7,8,9,10,11,12,13],ymm1[14],ymm0[15]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %ymm25, %ymm2
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %ymm26, %ymm11
-; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm11[0,1],ymm2[2],ymm11[3,4],ymm2[5],ymm11[6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm1, %xmm10
-; AVX512F-ONLY-SLOW-NEXT:    vpshuflw {{.*#+}} xmm10 = xmm10[3,1,2,3,4,5,6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %ymm25, %ymm3
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %ymm26, %ymm7
+; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm7[0,1],ymm3[2],ymm7[3,4],ymm3[5],ymm7[6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; AVX512F-ONLY-SLOW-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7]
 ; AVX512F-ONLY-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[2,1,2,3]
 ; AVX512F-ONLY-SLOW-NEXT:    vpshuflw {{.*#+}} xmm1 = xmm1[0,2,2,3,4,5,6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm10[0],xmm1[1],xmm10[1],xmm1[2],xmm10[2],xmm1[3],xmm10[3]
+; AVX512F-ONLY-SLOW-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
 ; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1],ymm0[2,3,4,5,6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm7[0,1,2],ymm8[3],ymm7[4,5],ymm8[6],ymm7[7]
-; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm10
-; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm10[0],xmm0[1],xmm10[2,3,4,5],xmm0[6],xmm10[7]
-; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm10 = ymm15[0,1],ymm3[2,3],ymm15[4,5],ymm3[6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm13 = ymm15[0,1,0,1]
-; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} ymm10 = ymm10[0,1,2],ymm13[3],ymm10[4,5,6,7,8,9,10],ymm13[11],ymm10[12,13,14,15]
-; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm13 = ymm6[0],ymm5[1],ymm6[2,3],ymm5[4],ymm6[5,6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm13, %xmm14
-; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} xmm13 = xmm13[0],xmm14[1],xmm13[2,3,4,5],xmm14[6],xmm13[7]
+; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm8[0,1,2],ymm13[3],ymm8[4,5],ymm13[6],ymm8[7]
+; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm2
+; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm2[0],xmm0[1],xmm2[2,3,4,5],xmm0[6],xmm2[7]
+; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm15[0,1],ymm4[2,3],ymm15[4,5],ymm4[6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm10 = ymm15[0,1,0,1]
+; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1,2],ymm10[3],ymm2[4,5,6,7,8,9,10],ymm10[11],ymm2[12,13,14,15]
+; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm10 = ymm6[0],ymm5[1],ymm6[2,3],ymm5[4],ymm6[5,6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm10, %xmm14
+; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} xmm10 = xmm10[0],xmm14[1],xmm10[2,3,4,5],xmm14[6],xmm10[7]
 ; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,u,2,3,0,1,14,15,12,13,10,11]
 ; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} ymm10 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm10[10,11,24,25,22,23,20,21],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm10[0,1,2],ymm0[3,4,5,6,7],ymm10[8,9,10],ymm0[11,12,13,14,15]
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm13 = xmm13[0,1,2,3,4,5,2,3,0,1,14,15,12,13],zero,zero
-; AVX512F-ONLY-SLOW-NEXT:    vpor %ymm10, %ymm13, %ymm10
-; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm13 = ymm10[0,1,2,3],ymm0[4,5,6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm8[0],ymm7[1],ymm8[2,3],ymm7[4],ymm8[5,6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm10
-; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm10[1],xmm0[2,3,4,5],xmm10[6],xmm0[7]
-; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm10 = ymm15[0,1,2],ymm3[3],ymm15[4,5],ymm3[6],ymm15[7]
-; AVX512F-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm14 = ymm10[2,3,0,1]
-; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} ymm10 = ymm10[0,1,2,3],ymm14[4],ymm10[5,6,7,8,9,10,11],ymm14[12],ymm10[13,14,15]
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} ymm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[10,11,24,25,22,23,20,21],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm2[0,1,2],ymm0[3,4,5,6,7],ymm2[8,9,10],ymm0[11,12,13,14,15]
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm10 = xmm10[0,1,2,3,4,5,2,3,0,1,14,15,12,13],zero,zero
+; AVX512F-ONLY-SLOW-NEXT:    vpor %ymm2, %ymm10, %ymm2
+; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm10 = ymm2[0,1,2,3],ymm0[4,5,6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm13[0],ymm8[1],ymm13[2,3],ymm8[4],ymm13[5,6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm2
+; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2,3,4,5],xmm2[6],xmm0[7]
+; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm15[0,1,2],ymm4[3],ymm15[4,5],ymm4[6],ymm15[7]
+; AVX512F-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm14 = ymm2[2,3,0,1]
+; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm14[4],ymm2[5,6,7,8,9,10,11],ymm14[12],ymm2[13,14,15]
 ; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm14 = ymm6[0],ymm5[1],ymm6[2,3,4],ymm5[5],ymm6[6,7]
 ; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm14, %xmm9
 ; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} xmm9 = xmm9[0],xmm14[1],xmm9[2],xmm14[3],xmm9[4,5,6,7]
 ; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,u,4,5,2,3,0,1,14,15,12,13]
 ; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} ymm10 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm10[12,13,26,27,24,25,22,23],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm10[0,1,2],ymm0[3,4,5,6,7],ymm10[8,9,10],ymm0[11,12,13,14,15]
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} ymm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[12,13,26,27,24,25,22,23],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm2[0,1,2],ymm0[3,4,5,6,7],ymm2[8,9,10],ymm0[11,12,13,14,15]
 ; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm9 = xmm9[0,1,2,3,6,7,4,5,2,3,0,1,14,15],zero,zero
-; AVX512F-ONLY-SLOW-NEXT:    vpor %ymm10, %ymm9, %ymm9
-; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm9[0,1,2,3],ymm0[4,5,6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm9 = ymm12[0,1,2],ymm4[3],ymm12[4,5],ymm4[6],ymm12[7]
-; AVX512F-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm10 = ymm9[2,3,0,1]
-; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} ymm9 = ymm10[0],ymm9[1,2,3,4,5,6],ymm10[7,8],ymm9[9,10,11,12,13,14],ymm10[15]
-; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm10 = ymm11[0,1],ymm2[2,3],ymm11[4,5],ymm2[6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm10, %xmm14
+; AVX512F-ONLY-SLOW-NEXT:    vpor %ymm2, %ymm9, %ymm2
+; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm12[0,1,2],ymm11[3],ymm12[4,5],ymm11[6],ymm12[7]
+; AVX512F-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm9 = ymm2[2,3,0,1]
+; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} ymm2 = ymm9[0],ymm2[1,2,3,4,5,6],ymm9[7,8],ymm2[9,10,11,12,13,14],ymm9[15]
+; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm9 = ymm7[0,1],ymm3[2,3],ymm7[4,5],ymm3[6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm9, %xmm14
 ; AVX512F-ONLY-SLOW-NEXT:    vpshufd {{.*#+}} xmm14 = xmm14[2,1,2,3]
 ; AVX512F-ONLY-SLOW-NEXT:    vpshuflw {{.*#+}} xmm14 = xmm14[0,2,2,3,4,5,6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vpshufd {{.*#+}} xmm10 = xmm10[2,1,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vpshuflw {{.*#+}} xmm10 = xmm10[1,3,2,3,4,5,6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm14[0],xmm10[1],xmm14[1],xmm10[2],xmm14[2],xmm10[3],xmm14[3]
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} ymm9 = ymm9[u,u,u,u,u,u,u,u,2,3,0,1,14,15,12,13,26,27,26,27,26,27,26,27,18,19,16,17,30,31,28,29]
-; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm10 = ymm10[0,1],ymm9[2,3,4,5,6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0],ymm12[1],ymm4[2,3],ymm12[4],ymm4[5,6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm9 = ymm4[2,3,0,1]
-; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} ymm4 = ymm4[0],ymm9[1],ymm4[2,3,4,5,6,7,8],ymm9[9],ymm4[10,11,12,13,14,15]
-; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $1, %ymm30, %zmm0, %zmm9
-; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $1, %ymm28, %zmm0, %zmm12
-; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $1, %ymm31, %zmm0, %zmm14
-; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $1, %ymm29, %zmm0, %zmm25
-; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $1, %ymm13, %zmm0, %zmm13
+; AVX512F-ONLY-SLOW-NEXT:    vpshufd {{.*#+}} xmm9 = xmm9[2,1,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vpshuflw {{.*#+}} xmm9 = xmm9[1,3,2,3,4,5,6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm9[0],xmm14[0],xmm9[1],xmm14[1],xmm9[2],xmm14[2],xmm9[3],xmm14[3]
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} ymm2 = ymm2[u,u,u,u,u,u,u,u,2,3,0,1,14,15,12,13,26,27,26,27,26,27,26,27,18,19,16,17,30,31,28,29]
+; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm9[0,1],ymm2[2,3,4,5,6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm9 = ymm11[0],ymm12[1],ymm11[2,3],ymm12[4],ymm11[5,6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm11 = ymm9[2,3,0,1]
+; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} ymm9 = ymm9[0],ymm11[1],ymm9[2,3,4,5,6,7,8],ymm11[9],ymm9[10,11,12,13,14,15]
+; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $1, %ymm29, %zmm0, %zmm11
+; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $1, %ymm30, %zmm0, %zmm12
+; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $1, %ymm28, %zmm0, %zmm14
+; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $1, %ymm24, %zmm0, %zmm25
+; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $1, %ymm10, %zmm0, %zmm10
 ; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm26
-; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm11[0,1,2],ymm2[3],ymm11[4,5],ymm2[6],ymm11[7]
-; AVX512F-ONLY-SLOW-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm2[0,1,2,3,6,4,6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm2, %xmm2
-; AVX512F-ONLY-SLOW-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[0,1,2,1]
-; AVX512F-ONLY-SLOW-NEXT:    vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,7,6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} ymm2 = ymm4[u,u,u,u,u,u,u,u,4,5,2,3,0,1,14,15,28,29,28,29,28,29,28,29,20,21,18,19,16,17,30,31]
-; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm0[0,1],ymm2[2,3,4,5,6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm8[0],ymm7[1],ymm8[2,3,4],ymm7[5],ymm8[6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm4
-; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm4[0],xmm0[1],xmm4[2],xmm0[3],xmm4[4,5,6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm6[0,1],ymm5[2],ymm6[3,4],ymm5[5],ymm6[6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm4, %xmm5
-; AVX512F-ONLY-SLOW-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm5[0,3,1,3,4,5,6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[2,1,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm4[0,0,2,3,4,5,6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
-; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1],ymm15[2],ymm3[3,4,5],ymm15[6],ymm3[7]
-; AVX512F-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,1,3,1]
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} ymm3 = ymm3[4,5,10,11,u,u,u,u,u,u,u,u,0,1,14,15,20,21,26,27,16,17,26,27,16,17,20,21,16,17,30,31]
-; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0],ymm4[1,2],ymm3[3,4,5,6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm7[0,1,2],ymm3[3],ymm7[4,5],ymm3[6],ymm7[7]
+; AVX512F-ONLY-SLOW-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm3[0,1,2,3,6,4,6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm3, %xmm3
+; AVX512F-ONLY-SLOW-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[0,1,2,1]
+; AVX512F-ONLY-SLOW-NEXT:    vpshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,5,7,6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7]
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} ymm3 = ymm9[u,u,u,u,u,u,u,u,4,5,2,3,0,1,14,15,28,29,28,29,28,29,28,29,20,21,18,19,16,17,30,31]
+; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm0[0,1],ymm3[2,3,4,5,6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm13[0],ymm8[1],ymm13[2,3,4],ymm8[5],ymm13[6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm8
+; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm8[0],xmm0[1],xmm8[2],xmm0[3],xmm8[4,5,6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm5 = ymm6[0,1],ymm5[2],ymm6[3,4],ymm5[5],ymm6[6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm5, %xmm6
+; AVX512F-ONLY-SLOW-NEXT:    vpshuflw {{.*#+}} xmm6 = xmm6[0,3,1,3,4,5,6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[2,1,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm5[0,0,2,3,4,5,6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3]
+; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0,1],ymm15[2],ymm4[3,4,5],ymm15[6],ymm4[7]
+; AVX512F-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[0,1,3,1]
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} ymm4 = ymm4[4,5,10,11,u,u,u,u,u,u,u,u,0,1,14,15,20,21,26,27,16,17,26,27,16,17,20,21,16,17,30,31]
+; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0],ymm5[1,2],ymm4[3,4,5,6,7]
 ; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,u,6,7,4,5,2,3,0,1,14,15]
 ; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm3[0,1,2],ymm0[3,4,5,6,7],ymm3[8,9,10],ymm0[11,12,13,14,15]
-; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3],ymm0[4,5,6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm4[0,1,2],ymm0[3,4,5,6,7],ymm4[8,9,10],ymm0[11,12,13,14,15]
+; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm0[4,5,6,7]
 ; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm0
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $184, %zmm21, %zmm3, %zmm9
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm4, %zmm22
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $184, %zmm22, %zmm3, %zmm12
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm24 # 64-byte Folded Reload
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $184, %zmm21, %zmm4, %zmm11
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm22
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $184, %zmm22, %zmm4, %zmm12
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $184, %zmm19, %zmm4, %zmm23
 ; AVX512F-ONLY-SLOW-NEXT:    movw $-512, %ax # imm = 0xFE00
 ; AVX512F-ONLY-SLOW-NEXT:    kmovw %eax, %k1
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa32 %zmm14, %zmm24 {%k1}
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $184, %zmm20, %zmm3, %zmm27
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa32 %zmm14, %zmm23 {%k1}
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $184, %zmm20, %zmm4, %zmm27
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqa32 %zmm25, %zmm27 {%k1}
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $226, %zmm19, %zmm3, %zmm1
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa32 %zmm13, %zmm1 {%k1}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm9, (%rsi)
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $226, %zmm18, %zmm4, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa32 %zmm10, %zmm1 {%k1}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm11, (%rsi)
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm12, (%rdx)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm24, (%rcx)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm23, (%rcx)
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm27, (%r8)
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, (%r9)
 ; AVX512F-ONLY-SLOW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $226, %zmm18, %zmm3, %zmm10
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa32 %zmm26, %zmm10 {%k1}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm10, (%rax)
-; AVX512F-ONLY-SLOW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $226, %zmm23, %zmm3, %zmm2
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa32 %zmm0, %zmm2 {%k1}
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $226, %zmm17, %zmm4, %zmm2
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa32 %zmm26, %zmm2 {%k1}
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, (%rax)
-; AVX512F-ONLY-SLOW-NEXT:    popq %rax
+; AVX512F-ONLY-SLOW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $226, %zmm16, %zmm4, %zmm3
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa32 %zmm0, %zmm3 {%k1}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, (%rax)
 ; AVX512F-ONLY-SLOW-NEXT:    vzeroupper
 ; AVX512F-ONLY-SLOW-NEXT:    retq
 ;
 ; AVX512F-ONLY-FAST-LABEL: load_i16_stride7_vf32:
 ; AVX512F-ONLY-FAST:       # %bb.0:
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 320(%rdi), %zmm28
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 64(%rdi), %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [2,6,9,13,2,6,9,13]
-; AVX512F-ONLY-FAST-NEXT:    # ymm2 = mem[0,1,0,1]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 128(%rdi), %zmm23
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 320(%rdi), %zmm30
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 64(%rdi), %zmm31
+; AVX512F-ONLY-FAST-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [2,6,9,13,2,6,9,13]
+; AVX512F-ONLY-FAST-NEXT:    # ymm1 = mem[0,1,0,1]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 128(%rdi), %zmm25
 ; AVX512F-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} ymm16 = [2,5,9,12,2,5,9,12]
 ; AVX512F-ONLY-FAST-NEXT:    # ymm16 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} ymm19 = [10,3,6,15,12,13,6,15]
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} ymm27 = [3,6,10,13,3,6,10,13]
-; AVX512F-ONLY-FAST-NEXT:    # ymm27 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 384(%rdi), %zmm18
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} ymm26 = <1,u,u,u,5,8,12,15>
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} ymm18 = [10,3,6,15,12,13,6,15]
+; AVX512F-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} ymm19 = [3,6,10,13,3,6,10,13]
+; AVX512F-ONLY-FAST-NEXT:    # ymm19 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 384(%rdi), %zmm14
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqa {{.*#+}} ymm0 = <2,6,9,u,13,u,u,u>
-; AVX512F-ONLY-FAST-NEXT:    vpermd %zmm1, %zmm0, %zmm14
-; AVX512F-ONLY-FAST-NEXT:    vpermd %zmm18, %zmm2, %zmm13
+; AVX512F-ONLY-FAST-NEXT:    vpermd %zmm31, %zmm0, %zmm10
+; AVX512F-ONLY-FAST-NEXT:    vpermd %zmm14, %zmm1, %zmm12
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqa {{.*#+}} ymm0 = <1,u,u,u,4,8,11,15>
-; AVX512F-ONLY-FAST-NEXT:    vpermd %zmm28, %zmm0, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vpermd %zmm30, %zmm0, %zmm15
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqa {{.*#+}} ymm0 = <2,5,9,u,12,u,u,u>
-; AVX512F-ONLY-FAST-NEXT:    vpermd %zmm1, %zmm0, %zmm10
-; AVX512F-ONLY-FAST-NEXT:    vpermd %zmm18, %zmm16, %zmm0
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa {{.*#+}} ymm3 = <0,u,u,u,4,7,11,14>
-; AVX512F-ONLY-FAST-NEXT:    vpermd %zmm28, %zmm3, %zmm6
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa {{.*#+}} ymm3 = [8,1,12,5,12,5,14,15]
-; AVX512F-ONLY-FAST-NEXT:    vpermd %zmm1, %zmm3, %zmm3
-; AVX512F-ONLY-FAST-NEXT:    vpermd %zmm23, %zmm27, %zmm4
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 192(%rdi), %ymm25
-; AVX512F-ONLY-FAST-NEXT:    vpermq {{.*#+}} ymm8 = ymm25[0,1,0,2]
-; AVX512F-ONLY-FAST-NEXT:    vpbroadcastd {{.*#+}} ymm7 = [20,21,26,27,20,21,26,27,20,21,26,27,20,21,26,27,20,21,26,27,20,21,26,27,20,21,26,27,20,21,26,27]
-; AVX512F-ONLY-FAST-NEXT:    vpshufb %ymm7, %ymm8, %ymm5
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %ymm7, %ymm30
+; AVX512F-ONLY-FAST-NEXT:    vpermd %zmm31, %zmm0, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vpermd %zmm14, %zmm16, %zmm7
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa {{.*#+}} ymm0 = <0,u,u,u,4,7,11,14>
+; AVX512F-ONLY-FAST-NEXT:    vpermd %zmm30, %zmm0, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa {{.*#+}} ymm0 = [8,1,12,5,12,5,14,15]
+; AVX512F-ONLY-FAST-NEXT:    vpermd %zmm31, %zmm0, %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vpermd %zmm25, %zmm19, %zmm4
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 192(%rdi), %ymm28
+; AVX512F-ONLY-FAST-NEXT:    vpermq {{.*#+}} ymm11 = ymm28[0,1,0,2]
+; AVX512F-ONLY-FAST-NEXT:    vpbroadcastd {{.*#+}} ymm1 = [20,21,26,27,20,21,26,27,20,21,26,27,20,21,26,27,20,21,26,27,20,21,26,27,20,21,26,27,20,21,26,27]
+; AVX512F-ONLY-FAST-NEXT:    vpshufb %ymm1, %ymm11, %ymm5
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %ymm1, %ymm23
 ; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm4 = ymm4[0,1,2,3,0,1,6,7,8,9,14,15,12,13,14,15,16,17,18,19,16,17,22,23,24,25,30,31,u,u,u,u]
-; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm5 = ymm4[0,1,2,3,4,5,6],ymm5[7]
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm7 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm3[6,7,12,13,2,3,16,17,30,31],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa (%rdi), %ymm3
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa 32(%rdi), %ymm4
-; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm11 = ymm3[0,1],ymm4[2],ymm3[3,4,5],ymm4[6],ymm3[7]
-; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm11, %xmm12
-; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} xmm11 = xmm11[0,1,2,3],xmm12[4],xmm11[5],xmm12[6],xmm11[7]
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm11 = ymm11[0,1,14,15,12,13,10,11,8,9],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm11[u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512F-ONLY-FAST-NEXT:    vporq %ymm7, %ymm11, %ymm22
-; AVX512F-ONLY-FAST-NEXT:    vpbroadcastw 252(%rdi), %xmm7
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa 224(%rdi), %xmm12
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm11 = xmm12[u,u,u,u,u,u,u,u,0,1,14,15,12,13,14,15]
-; AVX512F-ONLY-FAST-NEXT:    vpunpckhdq {{.*#+}} xmm7 = xmm11[2],xmm7[2],xmm11[3],xmm7[3]
+; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm6 = ymm4[0,1,2,3,4,5,6],ymm5[7]
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm0[6,7,12,13,2,3,16,17,30,31],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa (%rdi), %ymm4
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa 32(%rdi), %ymm5
+; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm8 = ymm4[0,1],ymm5[2],ymm4[3,4,5],ymm5[6],ymm4[7]
+; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm8, %xmm9
+; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} xmm8 = xmm8[0,1,2,3],xmm9[4],xmm8[5],xmm9[6],xmm8[7]
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm8 = ymm8[0,1,14,15,12,13,10,11,8,9],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm8[u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512F-ONLY-FAST-NEXT:    vporq %ymm0, %ymm8, %ymm22
+; AVX512F-ONLY-FAST-NEXT:    vpbroadcastw 252(%rdi), %xmm0
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa 224(%rdi), %xmm13
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm8 = xmm13[u,u,u,u,u,u,u,u,0,1,14,15,12,13,14,15]
+; AVX512F-ONLY-FAST-NEXT:    vpunpckhdq {{.*#+}} xmm0 = xmm8[2],xmm0[2],xmm8[3],xmm0[3]
 ; AVX512F-ONLY-FAST-NEXT:    movw $992, %ax # imm = 0x3E0
 ; AVX512F-ONLY-FAST-NEXT:    kmovw %eax, %k1
-; AVX512F-ONLY-FAST-NEXT:    vinserti32x4 $2, %xmm7, %zmm5, %zmm22 {%k1}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa 256(%rdi), %ymm5
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa 288(%rdi), %ymm7
-; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm11 = ymm7[0,1],ymm5[2,3],ymm7[4,5],ymm5[6,7]
-; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm11, %xmm15
-; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} xmm11 = xmm15[0,1,2],xmm11[3],xmm15[4],xmm11[5],xmm15[6,7]
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm11 = xmm11[u,u,u,u,u,u,10,11,8,9,6,7,4,5,u,u]
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm6 = ymm6[0,1,6,7,8,9,14,15,8,9,14,15,4,5,2,3,16,17,22,23,24,25,30,31,24,25,30,31,20,21,18,19]
-; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} xmm11 = xmm6[0,1,2],xmm11[3,4,5,6],xmm6[7]
-; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm11 = ymm11[0,1,2,3],ymm6[4,5,6,7]
-; AVX512F-ONLY-FAST-NEXT:    vpbroadcastq {{.*#+}} ymm6 = [16,17,22,23,24,25,30,31,16,17,22,23,24,25,30,31,16,17,22,23,24,25,30,31,16,17,22,23,24,25,30,31]
-; AVX512F-ONLY-FAST-NEXT:    vpshufb %ymm6, %ymm0, %ymm0
-; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm11[0,1,2,3,4,5],ymm0[6,7]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %ymm0, %ymm29
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm10[0,1,6,7,8,9,18,19],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm10 = ymm3[0,1,2],ymm4[3],ymm3[4,5],ymm4[6],ymm3[7]
-; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm10, %xmm11
-; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} xmm10 = xmm11[0],xmm10[1],xmm11[2,3,4,5],xmm10[6],xmm11[7]
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm10 = ymm10[2,3,0,1,14,15,12,13,10,11],zero,zero,zero,zero,zero,zero,zero,zero,ymm10[u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512F-ONLY-FAST-NEXT:    vpor %ymm0, %ymm10, %ymm0
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa 160(%rdi), %ymm11
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa 128(%rdi), %ymm10
-; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm10[0,1],ymm11[2],ymm10[3,4,5],ymm11[6],ymm10[7]
-; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm0, %xmm15
-; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm15[4],xmm0[5],xmm15[6],xmm0[7]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa 240(%rdi), %xmm15
+; AVX512F-ONLY-FAST-NEXT:    vinserti32x4 $2, %xmm0, %zmm6, %zmm22 {%k1}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa 256(%rdi), %ymm6
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa 288(%rdi), %ymm8
+; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm8[0,1],ymm6[2,3],ymm8[4,5],ymm6[6,7]
+; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm0, %xmm9
+; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} xmm0 = xmm9[0,1,2],xmm0[3],xmm9[4],xmm0[5],xmm9[6,7]
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,u,10,11,8,9,6,7,4,5,u,u]
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm2 = ymm2[0,1,6,7,8,9,14,15,8,9,14,15,4,5,2,3,16,17,22,23,24,25,30,31,24,25,30,31,20,21,18,19]
+; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3,4,5,6],xmm2[7]
+; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
+; AVX512F-ONLY-FAST-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [16,17,22,23,24,25,30,31,16,17,22,23,24,25,30,31,16,17,22,23,24,25,30,31,16,17,22,23,24,25,30,31]
+; AVX512F-ONLY-FAST-NEXT:    vpshufb %ymm2, %ymm7, %ymm7
+; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm7[6,7]
+; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm24
+; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm4[0,1,2],ymm5[3],ymm4[4,5],ymm5[6],ymm4[7]
+; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm0, %xmm7
+; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} xmm0 = xmm7[0],xmm0[1],xmm7[2,3,4,5],xmm0[6],xmm7[7]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa {{.*#+}} ymm1 = <2,3,0,1,14,15,12,13,10,11,128,128,128,128,128,128,128,128,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX512F-ONLY-FAST-NEXT:    vpshufb %ymm1, %ymm0, %ymm0
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm3[0,1,6,7,8,9,18,19],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512F-ONLY-FAST-NEXT:    vporq %ymm3, %ymm0, %ymm20
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa 160(%rdi), %ymm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa 128(%rdi), %ymm9
+; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm9[0,1],ymm3[2],ymm9[3,4,5],ymm3[6],ymm9[7]
+; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm0, %xmm7
+; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm7[4],xmm0[5],xmm7[6],xmm0[7]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa 240(%rdi), %xmm7
 ; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,0,1,14,15,12,13,10,11,8,9,u,u,u,u]
 ; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm8 = ymm8[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,22,23,28,29]
-; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,6],ymm8[7]
-; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} xmm8 = xmm15[0],xmm12[1],xmm15[2,3,4,5,6,7]
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm8 = xmm8[2,3,0,1,14,15,u,u,u,u,u,u,u,u,u,u]
-; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $1, %ymm8, %zmm0, %zmm24
-; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm7[0,1,2],ymm5[3],ymm7[4,5],ymm5[6],ymm7[7]
-; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm0, %xmm8
-; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} xmm0 = xmm8[0,1,2,3],xmm0[4],xmm8[5],xmm0[6],xmm8[7]
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm11 = ymm11[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,22,23,28,29]
+; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,6],ymm11[7]
+; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} xmm11 = xmm7[0],xmm13[1],xmm7[2,3,4,5,6,7]
+; AVX512F-ONLY-FAST-NEXT:    vpshufb %xmm1, %xmm11, %xmm1
+; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm27
+; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm8[0,1,2],ymm6[3],ymm8[4,5],ymm6[6],ymm8[7]
+; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4],xmm1[5],xmm0[6],xmm1[7]
 ; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,u,12,13,10,11,8,9,6,7,u,u]
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm2 = ymm2[2,3,4,5,10,11,12,13,0,1,0,1,0,1,0,1,18,19,20,21,26,27,28,29,16,17,16,17,16,17,16,17]
-; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3,4,5,6],xmm2[7]
-; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm2 = ymm13[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,19,20,21,26,27,28,29]
-; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm2[6,7]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %ymm0, %ymm20
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm14[2,3,4,5,10,11,16,17],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm2 = ymm4[0],ymm3[1],ymm4[2,3],ymm3[4],ymm4[5,6,7]
-; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm2, %xmm8
-; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0],xmm8[1],xmm2[2,3,4,5],xmm8[6],xmm2[7]
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm2 = ymm2[4,5,2,3,0,1,14,15,12,13],zero,zero,zero,zero,zero,zero,zero,zero,ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512F-ONLY-FAST-NEXT:    vporq %ymm0, %ymm2, %ymm31
-; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm10[0,1,2],ymm11[3],ymm10[4,5],ymm11[6],ymm10[7]
-; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm0, %xmm2
-; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} xmm0 = xmm2[0],xmm0[1],xmm2[2,3,4,5],xmm0[6],xmm2[7]
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,2,3,0,1,14,15,12,13,10,11,u,u,u,u]
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm1 = ymm15[2,3,4,5,10,11,12,13,0,1,0,1,0,1,0,1,18,19,20,21,26,27,28,29,16,17,16,17,16,17,16,17]
+; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3,4,5,6],xmm1[7]
+; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm1 = ymm12[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,19,20,21,26,27,28,29]
+; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %ymm0, %ymm26
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm10[2,3,4,5,10,11,16,17],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm1 = ymm5[0],ymm4[1],ymm5[2,3],ymm4[4],ymm5[5,6,7]
+; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm1, %xmm10
+; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm10[1],xmm1[2,3,4,5],xmm10[6],xmm1[7]
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[4,5,2,3,0,1,14,15,12,13],zero,zero,zero,zero,zero,zero,zero,zero,ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512F-ONLY-FAST-NEXT:    vporq %ymm0, %ymm1, %ymm21
+; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm9[0,1,2],ymm3[3],ymm9[4,5],ymm3[6],ymm9[7]
+; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2,3,4,5],xmm0[6],xmm1[7]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa {{.*#+}} ymm1 = <0,1,2,3,0,1,14,15,12,13,10,11,8,9,128,128,128,128,128,128,128,128,128,128,u,u,u,u,u,u,u,u>
+; AVX512F-ONLY-FAST-NEXT:    vpshufb %xmm1, %xmm0, %xmm0
 ; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512F-ONLY-FAST-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [2,5,2,5,2,5,2,5]
-; AVX512F-ONLY-FAST-NEXT:    vpermd %ymm25, %ymm2, %ymm2
-; AVX512F-ONLY-FAST-NEXT:    vpshufhw {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5,4,7,8,9,10,11,12,13,12,15]
-; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,6],ymm2[7]
-; AVX512F-ONLY-FAST-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm12[0],xmm15[0],xmm12[1],xmm15[1],xmm12[2],xmm15[2],xmm12[3],xmm15[3]
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm8 = xmm2[8,9,6,7,4,5,6,7,8,9,10,11,12,13,14,15]
-; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $1, %ymm8, %zmm0, %zmm14
-; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm5[0,1],ymm7[2],ymm5[3,4,5],ymm7[6],ymm5[7]
-; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm0, %xmm8
-; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm8[4],xmm0[5],xmm8[6],xmm0[7]
-; AVX512F-ONLY-FAST-NEXT:    vpermd %zmm28, %zmm26, %zmm8
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,2,3,0,1,14,15,12,13,10,11,8,9],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm0[u,u,u,u,u,u,u,u]
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm8 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm8[2,3,16,17,22,23,24,25,30,31,u,u,u,u,u,u,u,u]
-; AVX512F-ONLY-FAST-NEXT:    vpor %ymm0, %ymm8, %ymm0
-; AVX512F-ONLY-FAST-NEXT:    vpermd %zmm18, %zmm27, %zmm8
-; AVX512F-ONLY-FAST-NEXT:    vpshufb %ymm6, %ymm8, %ymm6
-; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm6[6,7]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %ymm0, %ymm27
-; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm4[0],ymm3[1],ymm4[2,3,4],ymm3[5],ymm4[6,7]
-; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm0, %xmm6
-; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} xmm0 = xmm6[0],xmm0[1],xmm6[2],xmm0[3],xmm6[4,5,6,7]
-; AVX512F-ONLY-FAST-NEXT:    vpermd %zmm1, %zmm19, %zmm6
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm6 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm6[4,5,10,11,0,1,22,23],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[6,7,4,5,2,3,0,1,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm0[u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512F-ONLY-FAST-NEXT:    vporq %ymm6, %ymm0, %ymm21
-; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm11[0],ymm10[1],ymm11[2,3],ymm10[4],ymm11[5,6,7]
-; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm0, %xmm6
-; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm6[1],xmm0[2,3,4,5],xmm6[6],xmm0[7]
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,2,3,0,1,14,15,12,13,u,u,u,u]
-; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm6
-; AVX512F-ONLY-FAST-NEXT:    vpermq {{.*#+}} ymm0 = ymm25[0,1,1,3]
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm8 = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,19,24,25]
-; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3,4,5,6],ymm8[7]
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[12,13,10,11,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $1, %ymm2, %zmm6, %zmm19
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa 416(%rdi), %ymm2
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa 384(%rdi), %ymm8
-; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm6 = ymm8[0,1],ymm2[2],ymm8[3,4,5],ymm2[6],ymm8[7]
-; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm6, %xmm9
-; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} xmm6 = xmm6[0,1,2,3],xmm9[4],xmm6[5],xmm9[6],xmm6[7]
-; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm9 = ymm5[0,1,2],ymm7[3],ymm5[4,5],ymm7[6],ymm5[7]
-; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm9, %xmm13
-; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} xmm9 = xmm13[0],xmm9[1],xmm13[2,3,4,5],xmm9[6],xmm13[7]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa {{.*#+}} ymm13 = [2,11,2,11,12,5,8,9]
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm6 = xmm6[u,u,u,u,u,u,0,1,14,15,12,13,10,11,8,9]
-; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
-; AVX512F-ONLY-FAST-NEXT:    vpermd %zmm28, %zmm13, %zmm13
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm13 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm13[0,1,22,23,28,29,18,19],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} ymm6 = ymm13[0,1,2],ymm6[3,4,5,6,7],ymm13[8,9,10],ymm6[11,12,13,14,15]
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm9 = xmm9[0,1,2,3,2,3,0,1,14,15,12,13,10,11],zero,zero
-; AVX512F-ONLY-FAST-NEXT:    vpor %ymm13, %ymm9, %ymm9
-; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm6 = ymm9[0,1,2,3],ymm6[4,5,6,7]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %ymm6, %ymm17
-; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm9 = ymm11[0],ymm10[1],ymm11[2,3,4],ymm10[5],ymm11[6,7]
-; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm9, %xmm10
-; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} xmm9 = xmm10[0],xmm9[1],xmm10[2],xmm9[3],xmm10[4,5,6,7]
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm9 = xmm9[0,1,6,7,4,5,2,3,0,1,14,15,u,u,u,u]
-; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %ymm30, %ymm6
-; AVX512F-ONLY-FAST-NEXT:    vpshufb %ymm6, %ymm0, %ymm0
-; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm9[0,1,2,3,4,5,6],ymm0[7]
-; AVX512F-ONLY-FAST-NEXT:    vpbroadcastw 232(%rdi), %xmm9
-; AVX512F-ONLY-FAST-NEXT:    vpsrlq $48, %xmm15, %xmm10
-; AVX512F-ONLY-FAST-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1],xmm9[2],xmm10[2],xmm9[3],xmm10[3]
-; AVX512F-ONLY-FAST-NEXT:    vpermd %zmm23, %zmm16, %zmm10
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa {{.*#+}} ymm11 = <0,3,7,10,14,u,u,u>
-; AVX512F-ONLY-FAST-NEXT:    vinserti32x4 $2, %xmm9, %zmm0, %zmm9
-; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm4[0,1],ymm3[2],ymm4[3,4],ymm3[5],ymm4[6,7]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa {{.*#+}} ymm13 = <0,3,3,u,0,3,7,u>
-; AVX512F-ONLY-FAST-NEXT:    vpermd %ymm25, %ymm13, %ymm13
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm13 = ymm13[u,u,u,u,u,u,u,u,u,u,0,1,6,7,8,9,u,u,u,u,u,u,u,u,u,u,16,17,22,23,24,25]
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm10 = ymm10[0,1,0,1,6,7,8,9,14,15,u,u,u,u,u,u,16,17,16,17,22,23,24,25,30,31,u,u,u,u,u,u]
-; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} ymm10 = ymm10[0,1,2,3,4],ymm13[5,6,7],ymm10[8,9,10,11,12],ymm13[13,14,15]
-; AVX512F-ONLY-FAST-NEXT:    vpsrld $16, %xmm12, %xmm13
-; AVX512F-ONLY-FAST-NEXT:    vpunpckhwd {{.*#+}} xmm13 = xmm13[4],xmm15[4],xmm13[5],xmm15[5],xmm13[6],xmm15[6],xmm13[7],xmm15[7]
-; AVX512F-ONLY-FAST-NEXT:    vinserti32x4 $2, %xmm13, %zmm10, %zmm26
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa {{.*#+}} xmm13 = [8,9,4,5,4,5,6,7,8,9,10,11,12,13,14,15]
-; AVX512F-ONLY-FAST-NEXT:    vpshufb %xmm13, %xmm0, %xmm10
-; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; AVX512F-ONLY-FAST-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[3,1,2,3,4,5,6,7]
-; AVX512F-ONLY-FAST-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm10[0],xmm0[0],xmm10[1],xmm0[1],xmm10[2],xmm0[2],xmm10[3],xmm0[3]
-; AVX512F-ONLY-FAST-NEXT:    vpermd %zmm1, %zmm11, %zmm10
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa {{.*#+}} ymm6 = <u,u,u,u,u,u,u,u,0,1,6,7,8,9,14,15,16,17,22,23,20,21,22,23,16,17,22,23,24,25,30,31>
-; AVX512F-ONLY-FAST-NEXT:    vpshufb %ymm6, %ymm10, %ymm10
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %ymm6, %ymm18
-; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm10[2,3,4,5,6,7]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm30 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $184, %zmm0, %zmm30, %zmm9
-; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm8[0,1,2],ymm2[3],ymm8[4,5],ymm2[6],ymm8[7]
+; AVX512F-ONLY-FAST-NEXT:    vpbroadcastq {{.*#+}} ymm10 = [2,5,2,5,2,5,2,5]
+; AVX512F-ONLY-FAST-NEXT:    vpermd %ymm28, %ymm10, %ymm10
+; AVX512F-ONLY-FAST-NEXT:    vpshufhw {{.*#+}} ymm10 = ymm10[0,1,2,3,4,5,4,7,8,9,10,11,12,13,12,15]
+; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,6],ymm10[7]
+; AVX512F-ONLY-FAST-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm13[0],xmm7[0],xmm13[1],xmm7[1],xmm13[2],xmm7[2],xmm13[3],xmm7[3]
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm11 = xmm10[8,9,6,7,4,5,6,7,8,9,10,11,12,13,14,15]
+; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $1, %ymm11, %zmm0, %zmm15
+; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm6[0,1],ymm8[2],ymm6[3,4,5],ymm8[6],ymm6[7]
 ; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm0, %xmm11
-; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} xmm0 = xmm11[0],xmm0[1],xmm11[2,3,4,5],xmm0[6],xmm11[7]
-; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm11 = ymm7[0],ymm5[1],ymm7[2,3],ymm5[4],ymm7[5,6,7]
-; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm11, %xmm6
-; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} xmm6 = xmm11[0],xmm6[1],xmm11[2,3,4,5],xmm6[6],xmm11[7]
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,u,2,3,0,1,14,15,12,13,10,11]
-; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa {{.*#+}} ymm11 = <2,u,u,u,6,9,13,u>
-; AVX512F-ONLY-FAST-NEXT:    vpermd %zmm28, %zmm11, %zmm11
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm11 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm11[2,3,16,17,22,23,24,25],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} ymm0 = ymm11[0,1,2],ymm0[3,4,5,6,7],ymm11[8,9,10],ymm0[11,12,13,14,15]
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm6 = xmm6[0,1,2,3,4,5,2,3,0,1,14,15,12,13],zero,zero
-; AVX512F-ONLY-FAST-NEXT:    vpor %ymm6, %ymm11, %ymm6
-; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3],ymm0[4,5,6,7]
-; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm11 = ymm4[0,1],ymm3[2,3],ymm4[4,5],ymm3[6,7]
-; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm11, %xmm0
-; AVX512F-ONLY-FAST-NEXT:    vpshufb %xmm13, %xmm0, %xmm10
-; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $1, %ymm29, %zmm0, %zmm16
-; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $1, %ymm20, %zmm0, %zmm13
-; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $1, %ymm27, %zmm0, %zmm27
-; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $1, %ymm17, %zmm0, %zmm29
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = [2,6,9,13,2,6,9,13]
-; AVX512F-ONLY-FAST-NEXT:    # ymm0 = mem[0,1,0,1]
-; AVX512F-ONLY-FAST-NEXT:    vpermd %zmm23, %zmm0, %zmm0
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} ymm23 = <0,4,7,11,14,u,u,u>
-; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $1, %ymm6, %zmm0, %zmm17
-; AVX512F-ONLY-FAST-NEXT:    vpermd %zmm1, %zmm23, %zmm6
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm11 = xmm11[10,11,6,7,4,5,6,7,u,u,u,u,u,u,u,u]
-; AVX512F-ONLY-FAST-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm11[0],xmm10[0],xmm11[1],xmm10[1],xmm11[2],xmm10[2],xmm11[3],xmm10[3]
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm6 = ymm6[u,u,u,u,u,u,u,u,2,3,4,5,10,11,12,13,18,19,18,19,18,19,18,19,18,19,20,21,26,27,28,29]
-; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm11 = ymm10[0,1],ymm6[2,3,4,5,6,7]
-; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm6 = ymm2[0],ymm8[1],ymm2[2,3],ymm8[4],ymm2[5,6,7]
-; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm6, %xmm10
-; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} xmm6 = xmm6[0],xmm10[1],xmm6[2,3,4,5],xmm10[6],xmm6[7]
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti128 {{.*#+}} ymm10 = [0,4,7,0,0,4,7,0]
-; AVX512F-ONLY-FAST-NEXT:    # ymm10 = mem[0,1,0,1]
-; AVX512F-ONLY-FAST-NEXT:    vpermd %ymm25, %ymm10, %ymm10
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm10 = ymm10[u,u,u,u,u,u,u,u,u,u,2,3,4,5,10,11,u,u,u,u,u,u,u,u,u,u,18,19,20,21,26,27]
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[4,5,2,3,4,5,10,11,12,13,u,u,u,u,u,u,20,21,18,19,20,21,26,27,28,29,u,u,u,u,u,u]
-; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm10[5,6,7],ymm0[8,9,10,11,12],ymm10[13,14,15]
-; AVX512F-ONLY-FAST-NEXT:    vpunpckhwd {{.*#+}} xmm10 = xmm12[4],xmm15[4],xmm12[5],xmm15[5],xmm12[6],xmm15[6],xmm12[7],xmm15[7]
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm10 = xmm10[8,9,6,7,4,5,6,7,8,9,10,11,12,13,14,15]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa {{.*#+}} ymm15 = <3,u,u,u,6,10,13,u>
-; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $1, %ymm10, %zmm0, %zmm12
-; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm7[0],ymm5[1],ymm7[2,3,4],ymm5[5],ymm7[6,7]
-; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm0, %xmm10
-; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} xmm0 = xmm10[0],xmm0[1],xmm10[2],xmm0[3],xmm10[4,5,6,7]
-; AVX512F-ONLY-FAST-NEXT:    vpermd %zmm28, %zmm15, %zmm10
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm6 = xmm6[u,u,u,u,u,u,4,5,2,3,0,1,14,15,12,13]
-; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm10 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm10[0,1,18,19,20,21,26,27],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} ymm6 = ymm10[0,1,2],ymm6[3,4,5,6,7],ymm10[8,9,10],ymm6[11,12,13,14,15]
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,2,3,6,7,4,5,2,3,0,1,14,15],zero,zero
-; AVX512F-ONLY-FAST-NEXT:    vpor %ymm0, %ymm10, %ymm0
-; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm6[4,5,6,7]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa {{.*#+}} ymm6 = <1,4,8,11,15,u,u,u>
-; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm10
-; AVX512F-ONLY-FAST-NEXT:    vpermd %zmm1, %zmm6, %zmm0
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %ymm18, %ymm1
+; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm11[4],xmm0[5],xmm11[6],xmm0[7]
+; AVX512F-ONLY-FAST-NEXT:    vpermd %zmm31, %zmm18, %zmm11
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa {{.*#+}} ymm12 = <1,u,u,u,5,8,12,15>
 ; AVX512F-ONLY-FAST-NEXT:    vpshufb %ymm1, %ymm0, %ymm0
-; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm1 = ymm4[0,1,2],ymm3[3],ymm4[4,5],ymm3[6],ymm4[7]
-; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm1, %xmm3
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u,u,u,u,u,u,u,10,11,6,7,4,5,6,7]
-; AVX512F-ONLY-FAST-NEXT:    vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,4,6,7]
-; AVX512F-ONLY-FAST-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]
-; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3,4,5,6,7]
-; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0],ymm8[1],ymm2[2,3,4],ymm8[5],ymm2[6,7]
+; AVX512F-ONLY-FAST-NEXT:    vpermd %zmm30, %zmm12, %zmm1
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm1[2,3,16,17,22,23,24,25,30,31,u,u,u,u,u,u,u,u]
+; AVX512F-ONLY-FAST-NEXT:    vpor %ymm1, %ymm0, %ymm0
+; AVX512F-ONLY-FAST-NEXT:    vpermd %zmm14, %zmm19, %zmm1
+; AVX512F-ONLY-FAST-NEXT:    vpshufb %ymm2, %ymm1, %ymm1
+; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %ymm0, %ymm17
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm11[4,5,10,11,0,1,22,23],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm1 = ymm5[0],ymm4[1],ymm5[2,3,4],ymm4[5],ymm5[6,7]
 ; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm1, %xmm2
 ; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2],xmm1[3],xmm2[4,5,6,7]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,1,10,3,14,7,10,3]
-; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm3 = ymm7[0,1],ymm5[2],ymm7[3,4],ymm5[5],ymm7[6,7]
-; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm3, %xmm4
-; AVX512F-ONLY-FAST-NEXT:    vpshuflw {{.*#+}} xmm4 = xmm4[0,3,1,3,4,5,6,7]
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[8,9,8,9,4,5,6,7,u,u,u,u,u,u,u,u]
-; AVX512F-ONLY-FAST-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
-; AVX512F-ONLY-FAST-NEXT:    vpermd %zmm28, %zmm2, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm2 = ymm2[4,5,10,11,u,u,u,u,u,u,u,u,0,1,14,15,20,21,26,27,16,17,26,27,16,17,20,21,16,17,30,31]
-; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0],ymm3[1,2],ymm2[3,4,5,6,7]
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[u,u,u,u,u,u,6,7,4,5,2,3,0,1,14,15]
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[6,7,4,5,2,3,0,1,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512F-ONLY-FAST-NEXT:    vporq %ymm0, %ymm1, %ymm18
+; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm3[0],ymm9[1],ymm3[2,3],ymm9[4],ymm3[5,6,7]
+; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3,4,5],xmm1[6],xmm0[7]
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,2,3,0,1,14,15,12,13,u,u,u,u]
+; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512F-ONLY-FAST-NEXT:    vpermq {{.*#+}} ymm11 = ymm28[0,1,1,3]
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm1 = ymm11[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,19,24,25]
+; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,6],ymm1[7]
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm1 = xmm10[12,13,10,11,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm19
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa 416(%rdi), %ymm14
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa 384(%rdi), %ymm2
+; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1],ymm14[2],ymm2[3,4,5],ymm14[6],ymm2[7]
+; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm1[4],xmm0[5],xmm1[6],xmm0[7]
+; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm1 = ymm6[0,1,2],ymm8[3],ymm6[4,5],ymm8[6],ymm6[7]
+; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm1, %xmm10
+; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} xmm1 = xmm10[0],xmm1[1],xmm10[2,3,4,5],xmm1[6],xmm10[7]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa {{.*#+}} ymm10 = [2,11,2,11,12,5,8,9]
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,u,0,1,14,15,12,13,10,11,8,9]
+; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512F-ONLY-FAST-NEXT:    vpermd %zmm30, %zmm10, %zmm10
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm10 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm10[0,1,22,23,28,29,18,19],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} ymm0 = ymm10[0,1,2],ymm0[3,4,5,6,7],ymm10[8,9,10],ymm0[11,12,13,14,15]
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0,1,2,3,2,3,0,1,14,15,12,13,10,11],zero,zero
+; AVX512F-ONLY-FAST-NEXT:    vpor %ymm1, %ymm10, %ymm1
+; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm10 = ymm1[0,1,2,3],ymm0[4,5,6,7]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa {{.*#+}} ymm0 = <0,3,7,10,14,u,u,u>
+; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0],ymm9[1],ymm3[2,3,4],ymm9[5],ymm3[6,7]
+; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm1, %xmm3
+; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} xmm1 = xmm3[0],xmm1[1],xmm3[2],xmm1[3],xmm3[4,5,6,7]
+; AVX512F-ONLY-FAST-NEXT:    vpermd %zmm31, %zmm0, %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0,1,6,7,4,5,2,3,0,1,14,15,u,u,u,u]
 ; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0,1,2],ymm1[3,4,5,6,7],ymm2[8,9,10],ymm1[11,12,13,14,15]
-; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm1
-; AVX512F-ONLY-FAST-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $184, %zmm22, %zmm2, %zmm16
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm24
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $184, %zmm24, %zmm2, %zmm13
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $184, %zmm31, %zmm30, %zmm14
-; AVX512F-ONLY-FAST-NEXT:    movw $-512, %di # imm = 0xFE00
-; AVX512F-ONLY-FAST-NEXT:    kmovw %edi, %k1
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa32 %zmm27, %zmm14 {%k1}
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $184, %zmm21, %zmm30, %zmm19
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa32 %zmm29, %zmm19 {%k1}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa32 %zmm17, %zmm9 {%k1}
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $226, %zmm26, %zmm30, %zmm11
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa32 %zmm10, %zmm11 {%k1}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm16, (%rsi)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm13, (%rdx)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm14, (%rcx)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %ymm23, %ymm3
+; AVX512F-ONLY-FAST-NEXT:    vpshufb %ymm3, %ymm11, %ymm3
+; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5,6],ymm3[7]
+; AVX512F-ONLY-FAST-NEXT:    vpbroadcastw 232(%rdi), %xmm3
+; AVX512F-ONLY-FAST-NEXT:    vpsrlq $48, %xmm7, %xmm9
+; AVX512F-ONLY-FAST-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm9[0],xmm3[1],xmm9[1],xmm3[2],xmm9[2],xmm3[3],xmm9[3]
+; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm9 = ymm5[0,1],ymm4[2],ymm5[3,4],ymm4[5],ymm5[6,7]
+; AVX512F-ONLY-FAST-NEXT:    vinserti32x4 $2, %xmm3, %zmm1, %zmm23
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa {{.*#+}} xmm12 = [8,9,4,5,4,5,6,7,8,9,10,11,12,13,14,15]
+; AVX512F-ONLY-FAST-NEXT:    vpshufb %xmm12, %xmm9, %xmm1
+; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm9, %xmm3
+; AVX512F-ONLY-FAST-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm3[3,1,2,3,4,5,6,7]
+; AVX512F-ONLY-FAST-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa {{.*#+}} ymm9 = <u,u,u,u,u,u,u,u,0,1,6,7,8,9,14,15,16,17,22,23,20,21,22,23,16,17,22,23,24,25,30,31>
+; AVX512F-ONLY-FAST-NEXT:    vpshufb %ymm9, %ymm0, %ymm0
+; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3,4,5,6,7]
+; AVX512F-ONLY-FAST-NEXT:    vpermd %zmm25, %zmm16, %zmm1
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm16 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $184, %zmm0, %zmm16, %zmm23
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa {{.*#+}} ymm0 = <0,3,3,u,0,3,7,u>
+; AVX512F-ONLY-FAST-NEXT:    vpermd %ymm28, %ymm0, %ymm0
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[u,u,u,u,u,u,u,u,u,u,0,1,6,7,8,9,u,u,u,u,u,u,u,u,u,u,16,17,22,23,24,25]
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[0,1,0,1,6,7,8,9,14,15,u,u,u,u,u,u,16,17,16,17,22,23,24,25,30,31,u,u,u,u,u,u]
+; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0,1,2,3,4],ymm0[5,6,7],ymm1[8,9,10,11,12],ymm0[13,14,15]
+; AVX512F-ONLY-FAST-NEXT:    vpsrld $16, %xmm13, %xmm1
+; AVX512F-ONLY-FAST-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm1[4],xmm7[4],xmm1[5],xmm7[5],xmm1[6],xmm7[6],xmm1[7],xmm7[7]
+; AVX512F-ONLY-FAST-NEXT:    vinserti32x4 $2, %xmm1, %zmm0, %zmm29
+; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1,2],ymm14[3],ymm2[4,5],ymm14[6],ymm2[7]
+; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2,3,4,5],xmm0[6],xmm1[7]
+; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm1 = ymm8[0],ymm6[1],ymm8[2,3],ymm6[4],ymm8[5,6,7]
+; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm1, %xmm3
+; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm3[1],xmm1[2,3,4,5],xmm3[6],xmm1[7]
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,u,2,3,0,1,14,15,12,13,10,11]
+; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa {{.*#+}} ymm3 = <2,u,u,u,6,9,13,u>
+; AVX512F-ONLY-FAST-NEXT:    vpermd %zmm30, %zmm3, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm3[2,3,16,17,22,23,24,25],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} ymm0 = ymm3[0,1,2],ymm0[3,4,5,6,7],ymm3[8,9,10],ymm0[11,12,13,14,15]
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,2,3,0,1,14,15,12,13],zero,zero
+; AVX512F-ONLY-FAST-NEXT:    vpor %ymm3, %ymm1, %ymm1
+; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm11 = ymm1[0,1,2,3],ymm0[4,5,6,7]
+; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm5[0,1],ymm4[2,3],ymm5[4,5],ymm4[6,7]
+; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-ONLY-FAST-NEXT:    vpshufb %xmm12, %xmm1, %xmm1
+; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $1, %ymm26, %zmm0, %zmm26
+; AVX512F-ONLY-FAST-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [2,6,9,13,2,6,9,13]
+; AVX512F-ONLY-FAST-NEXT:    # ymm3 = mem[0,1,0,1]
+; AVX512F-ONLY-FAST-NEXT:    vpermd %zmm25, %zmm3, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa {{.*#+}} ymm12 = <0,4,7,11,14,u,u,u>
+; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $1, %ymm17, %zmm0, %zmm17
+; AVX512F-ONLY-FAST-NEXT:    vpermd %zmm31, %zmm12, %zmm12
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[10,11,6,7,4,5,6,7,u,u,u,u,u,u,u,u]
+; AVX512F-ONLY-FAST-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm1 = ymm12[u,u,u,u,u,u,u,u,2,3,4,5,10,11,12,13,18,19,18,19,18,19,18,19,18,19,20,21,26,27,28,29]
+; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm12 = ymm0[0,1],ymm1[2,3,4,5,6,7]
+; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm14[0],ymm2[1],ymm14[2,3],ymm2[4],ymm14[5,6,7]
+; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3,4,5],xmm1[6],xmm0[7]
+; AVX512F-ONLY-FAST-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [0,4,7,0,0,4,7,0]
+; AVX512F-ONLY-FAST-NEXT:    # ymm1 = mem[0,1,0,1]
+; AVX512F-ONLY-FAST-NEXT:    vpermd %ymm28, %ymm1, %ymm1
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[u,u,u,u,u,u,u,u,u,u,2,3,4,5,10,11,u,u,u,u,u,u,u,u,u,u,18,19,20,21,26,27]
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm3 = ymm3[4,5,2,3,4,5,10,11,12,13,u,u,u,u,u,u,20,21,18,19,20,21,26,27,28,29,u,u,u,u,u,u]
+; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} ymm1 = ymm3[0,1,2,3,4],ymm1[5,6,7],ymm3[8,9,10,11,12],ymm1[13,14,15]
+; AVX512F-ONLY-FAST-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm13[4],xmm7[4],xmm13[5],xmm7[5],xmm13[6],xmm7[6],xmm13[7],xmm7[7]
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[8,9,6,7,4,5,6,7,8,9,10,11,12,13,14,15]
+; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $1, %ymm3, %zmm1, %zmm7
+; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm1 = ymm8[0],ymm6[1],ymm8[2,3,4],ymm6[5],ymm8[6,7]
+; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm1, %xmm3
+; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} xmm1 = xmm3[0],xmm1[1],xmm3[2],xmm1[3],xmm3[4,5,6,7]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa {{.*#+}} ymm3 = <3,u,u,u,6,10,13,u>
+; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $1, %ymm10, %zmm0, %zmm10
+; AVX512F-ONLY-FAST-NEXT:    vpermd %zmm30, %zmm3, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,u,4,5,2,3,0,1,14,15,12,13]
+; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm3[0,1,18,19,20,21,26,27],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} ymm0 = ymm3[0,1,2],ymm0[3,4,5,6,7],ymm3[8,9,10],ymm0[11,12,13,14,15]
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0,1,2,3,6,7,4,5,2,3,0,1,14,15],zero,zero
+; AVX512F-ONLY-FAST-NEXT:    vpor %ymm3, %ymm1, %ymm1
+; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa {{.*#+}} ymm1 = <1,4,8,11,15,u,u,u>
+; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $1, %ymm11, %zmm0, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vpermd %zmm31, %zmm1, %zmm1
+; AVX512F-ONLY-FAST-NEXT:    vpshufb %ymm9, %ymm1, %ymm1
+; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm4 = ymm5[0,1,2],ymm4[3],ymm5[4,5],ymm4[6],ymm5[7]
+; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm4, %xmm5
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm5 = xmm5[u,u,u,u,u,u,u,u,10,11,6,7,4,5,6,7]
+; AVX512F-ONLY-FAST-NEXT:    vpshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,6,4,6,7]
+; AVX512F-ONLY-FAST-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm4[4],xmm5[4],xmm4[5],xmm5[5],xmm4[6],xmm5[6],xmm4[7],xmm5[7]
+; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm1 = ymm4[0,1],ymm1[2,3,4,5,6,7]
+; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm2 = ymm14[0],ymm2[1],ymm14[2,3,4],ymm2[5],ymm14[6,7]
+; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm2, %xmm4
+; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} xmm2 = xmm4[0],xmm2[1],xmm4[2],xmm2[3],xmm4[4,5,6,7]
+; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm4 = ymm8[0,1],ymm6[2],ymm8[3,4],ymm6[5],ymm8[6,7]
+; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm4, %xmm5
+; AVX512F-ONLY-FAST-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm5[0,3,1,3,4,5,6,7]
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[8,9,8,9,4,5,6,7,u,u,u,u,u,u,u,u]
+; AVX512F-ONLY-FAST-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa {{.*#+}} ymm5 = [0,1,10,3,14,7,10,3]
+; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vpermd %zmm30, %zmm5, %zmm5
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm5 = ymm5[4,5,10,11,u,u,u,u,u,u,u,u,0,1,14,15,20,21,26,27,16,17,26,27,16,17,20,21,16,17,30,31]
+; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm4 = ymm5[0],ymm4[1,2],ymm5[3,4,5,6,7]
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,u,u,u,u,u,6,7,4,5,2,3,0,1,14,15]
+; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} ymm2 = ymm4[0,1,2],ymm2[3,4,5,6,7],ymm4[8,9,10],ymm2[11,12,13,14,15]
+; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm2 = ymm4[0,1,2,3],ymm2[4,5,6,7]
+; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $184, %zmm22, %zmm4, %zmm24
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm20, %zmm27
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $184, %zmm27, %zmm4, %zmm26
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $184, %zmm21, %zmm16, %zmm15
+; AVX512F-ONLY-FAST-NEXT:    movw $-512, %ax # imm = 0xFE00
+; AVX512F-ONLY-FAST-NEXT:    kmovw %eax, %k1
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa32 %zmm17, %zmm15 {%k1}
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $184, %zmm18, %zmm16, %zmm19
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa32 %zmm10, %zmm19 {%k1}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa32 %zmm3, %zmm23 {%k1}
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $226, %zmm29, %zmm16, %zmm12
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa32 %zmm0, %zmm12 {%k1}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm24, (%rsi)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm26, (%rdx)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm15, (%rcx)
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm19, (%r8)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm9, (%r9)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm11, (%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm23, (%r9)
 ; AVX512F-ONLY-FAST-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $226, %zmm12, %zmm30, %zmm0
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa32 %zmm1, %zmm0 {%k1}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, (%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm12, (%rax)
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $226, %zmm7, %zmm16, %zmm1
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa32 %zmm2, %zmm1 {%k1}
+; AVX512F-ONLY-FAST-NEXT:    movq {{[0-9]+}}(%rsp), %rax
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, (%rax)
 ; AVX512F-ONLY-FAST-NEXT:    vzeroupper
 ; AVX512F-ONLY-FAST-NEXT:    retq
 ;
 ; AVX512DQ-SLOW-LABEL: load_i16_stride7_vf32:
 ; AVX512DQ-SLOW:       # %bb.0:
-; AVX512DQ-SLOW-NEXT:    pushq %rax
 ; AVX512DQ-SLOW-NEXT:    vmovdqa (%rdi), %ymm2
 ; AVX512DQ-SLOW-NEXT:    vmovdqa 32(%rdi), %ymm3
 ; AVX512DQ-SLOW-NEXT:    vmovdqa 64(%rdi), %ymm1
-; AVX512DQ-SLOW-NEXT:    vmovdqa 96(%rdi), %ymm4
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm4[0],ymm1[1],ymm4[2,3,4],ymm1[5],ymm4[6,7]
-; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm4, %ymm7
-; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm1, %ymm12
+; AVX512DQ-SLOW-NEXT:    vmovdqa 96(%rdi), %ymm10
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm10[0],ymm1[1],ymm10[2,3,4],ymm1[5],ymm10[6,7]
+; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm1, %ymm4
 ; AVX512DQ-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,2,3]
 ; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm0[6,7,12,13,2,3,16,17,30,31],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1],ymm3[2],ymm2[3,4,5],ymm3[6],ymm2[7]
-; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm3, %ymm10
+; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm3, %ymm12
 ; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm2, %ymm11
 ; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm1, %xmm2
 ; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1,2,3],xmm2[4],xmm1[5],xmm2[6],xmm1[7]
 ; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[0,1,14,15,12,13,10,11,8,9],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm1[u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-SLOW-NEXT:    vpor %ymm0, %ymm1, %ymm6
+; AVX512DQ-SLOW-NEXT:    vporq %ymm0, %ymm1, %ymm18
 ; AVX512DQ-SLOW-NEXT:    vmovdqa 128(%rdi), %ymm8
 ; AVX512DQ-SLOW-NEXT:    vmovdqa 160(%rdi), %ymm9
 ; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm9[0,1,2],ymm8[3],ymm9[4,5],ymm8[6],ymm9[7]
@@ -6382,29 +6377,24 @@ define void @load_i16_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-SLOW-NEXT:    vpshufhw {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5,6,5,8,9,10,11,12,13,14,13]
 ; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5,6],ymm3[7]
 ; AVX512DQ-SLOW-NEXT:    vpbroadcastw 252(%rdi), %xmm3
-; AVX512DQ-SLOW-NEXT:    vmovdqa 224(%rdi), %xmm5
-; AVX512DQ-SLOW-NEXT:    vpshufd {{.*#+}} xmm4 = xmm5[0,1,0,3]
-; AVX512DQ-SLOW-NEXT:    vpshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,7,6,7]
-; AVX512DQ-SLOW-NEXT:    vpunpckhdq {{.*#+}} xmm3 = xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; AVX512DQ-SLOW-NEXT:    vmovdqa 224(%rdi), %xmm6
+; AVX512DQ-SLOW-NEXT:    vpshufd {{.*#+}} xmm5 = xmm6[0,1,0,3]
+; AVX512DQ-SLOW-NEXT:    vmovdqa %xmm6, %xmm13
+; AVX512DQ-SLOW-NEXT:    vpshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,7,6,7]
+; AVX512DQ-SLOW-NEXT:    vpunpckhdq {{.*#+}} xmm3 = xmm5[2],xmm3[2],xmm5[3],xmm3[3]
 ; AVX512DQ-SLOW-NEXT:    movw $992, %ax # imm = 0x3E0
 ; AVX512DQ-SLOW-NEXT:    kmovw %eax, %k1
-; AVX512DQ-SLOW-NEXT:    vinserti32x4 $2, %xmm3, %zmm2, %zmm6 {%k1}
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa 256(%rdi), %ymm3
-; AVX512DQ-SLOW-NEXT:    vmovdqa 288(%rdi), %ymm4
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm4[0,1],ymm3[2,3],ymm4[4,5],ymm3[6,7]
-; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm4, %ymm6
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %ymm3, %ymm25
-; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm2, %xmm3
-; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} xmm14 = xmm3[0,1,2],xmm2[3],xmm3[4],xmm2[5],xmm3[6,7]
-; AVX512DQ-SLOW-NEXT:    vmovdqa 240(%rdi), %xmm15
+; AVX512DQ-SLOW-NEXT:    vinserti32x4 $2, %xmm3, %zmm2, %zmm18 {%k1}
+; AVX512DQ-SLOW-NEXT:    vmovdqa 256(%rdi), %ymm6
+; AVX512DQ-SLOW-NEXT:    vmovdqa 288(%rdi), %ymm5
+; AVX512DQ-SLOW-NEXT:    vmovdqa 240(%rdi), %xmm14
 ; AVX512DQ-SLOW-NEXT:    vmovdqa 80(%rdi), %xmm2
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm7[0,1],ymm12[2],ymm7[3,4],ymm12[5],ymm7[6,7]
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm10[0,1],ymm4[2],ymm10[3,4],ymm4[5],ymm10[6,7]
 ; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} ymm2 = ymm3[0,1,2],ymm2[3],ymm3[4,5,6,7,8,9,10],ymm2[11],ymm3[12,13,14,15]
 ; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} ymm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[8,9,6,7,4,5,18,19],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm11[0,1,2],ymm10[3],ymm11[4,5],ymm10[6],ymm11[7]
-; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm3, %xmm4
-; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} xmm3 = xmm4[0],xmm3[1],xmm4[2,3,4,5],xmm3[6],xmm4[7]
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm11[0,1,2],ymm12[3],ymm11[4,5],ymm12[6],ymm11[7]
+; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm3, %xmm7
+; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} xmm3 = xmm7[0],xmm3[1],xmm7[2,3,4,5],xmm3[6],xmm7[7]
 ; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} ymm3 = ymm3[2,3,0,1,14,15,12,13,10,11],zero,zero,zero,zero,zero,zero,zero,zero,ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u]
 ; AVX512DQ-SLOW-NEXT:    vpor %ymm2, %ymm3, %ymm2
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -6416,87 +6406,92 @@ define void @load_i16_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-SLOW-NEXT:    vpshufd {{.*#+}} ymm1 = ymm1[0,1,1,3,4,5,5,7]
 ; AVX512DQ-SLOW-NEXT:    vpshufhw {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5,5,6,8,9,10,11,12,13,13,14]
 ; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5,6],ymm1[7]
-; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} xmm2 = xmm15[0],xmm5[1],xmm15[2,3,4,5,6,7]
+; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} xmm2 = xmm14[0],xmm13[1],xmm14[2,3,4,5,6,7]
 ; AVX512DQ-SLOW-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[0,3,2,3]
 ; AVX512DQ-SLOW-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm2[1,0,3,3,4,5,6,7]
-; AVX512DQ-SLOW-NEXT:    vinserti32x4 $2, %xmm2, %zmm1, %zmm20
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm7[0,1],ymm12[2,3],ymm7[4,5],ymm12[6,7]
+; AVX512DQ-SLOW-NEXT:    vinserti32x4 $2, %xmm2, %zmm1, %zmm21
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm10[0,1],ymm4[2,3],ymm10[4,5],ymm4[6,7]
 ; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm1, %xmm2
 ; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4],ymm1[5,6,7,8,9,10,11],ymm2[12],ymm1[13,14,15]
 ; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} ymm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm1[10,11,8,9,6,7,20,21],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm10[0],ymm11[1],ymm10[2,3],ymm11[4],ymm10[5,6,7]
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm12[0],ymm11[1],ymm12[2,3],ymm11[4],ymm12[5,6,7]
 ; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm2, %xmm3
 ; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0],xmm3[1],xmm2[2,3,4,5],xmm3[6],xmm2[7]
 ; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} ymm2 = ymm2[4,5,2,3,0,1,14,15,12,13],zero,zero,zero,zero,zero,zero,zero,zero,ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-SLOW-NEXT:    vporq %ymm1, %ymm2, %ymm18
+; AVX512DQ-SLOW-NEXT:    vporq %ymm1, %ymm2, %ymm19
 ; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm8[0,1,2],ymm9[3],ymm8[4,5],ymm9[6],ymm8[7]
 ; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm1, %xmm2
 ; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2,3,4,5],xmm1[6],xmm2[7]
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0,1,2,3,0,1,14,15,12,13,10,11,u,u,u,u]
+; AVX512DQ-SLOW-NEXT:    vmovdqa {{.*#+}} ymm2 = <0,1,2,3,0,1,14,15,12,13,10,11,8,9,128,128,128,128,128,128,128,128,128,128,u,u,u,u,u,u,u,u>
+; AVX512DQ-SLOW-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %ymm2, %ymm25
 ; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
 ; AVX512DQ-SLOW-NEXT:    vpermq {{.*#+}} ymm2 = ymm0[0,1,1,2]
 ; AVX512DQ-SLOW-NEXT:    vpshufd {{.*#+}} ymm2 = ymm2[0,1,0,3,4,5,4,7]
 ; AVX512DQ-SLOW-NEXT:    vpshufhw {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5,4,7,8,9,10,11,12,13,12,15]
 ; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5,6],ymm2[7]
-; AVX512DQ-SLOW-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm5[0],xmm15[0],xmm5[1],xmm15[1],xmm5[2],xmm15[2],xmm5[3],xmm15[3]
+; AVX512DQ-SLOW-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm13[0],xmm14[0],xmm13[1],xmm14[1],xmm13[2],xmm14[2],xmm13[3],xmm14[3]
 ; AVX512DQ-SLOW-NEXT:    vpshufd {{.*#+}} xmm2 = xmm3[2,1,2,3]
-; AVX512DQ-SLOW-NEXT:    vmovdqa %xmm3, %xmm13
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %xmm3, %xmm26
 ; AVX512DQ-SLOW-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm2[0,3,2,3,4,5,6,7]
-; AVX512DQ-SLOW-NEXT:    vinserti32x4 $2, %xmm2, %zmm1, %zmm24
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm7[0,1,2],ymm12[3],ymm7[4,5],ymm12[6],ymm7[7]
+; AVX512DQ-SLOW-NEXT:    vinserti32x4 $2, %xmm2, %zmm1, %zmm22
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm10[0,1,2],ymm4[3],ymm10[4,5],ymm4[6],ymm10[7]
 ; AVX512DQ-SLOW-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[1,3,2,3]
 ; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} ymm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm1[4,5,10,11,0,1,22,23],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm10[0],ymm11[1],ymm10[2,3,4],ymm11[5],ymm10[6,7]
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %ymm11, %ymm26
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm12[0],ymm11[1],ymm12[2,3,4],ymm11[5],ymm12[6,7]
+; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm12, %ymm15
 ; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm2, %xmm3
 ; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0],xmm2[1],xmm3[2],xmm2[3],xmm3[4,5,6,7]
 ; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} ymm2 = ymm2[6,7,4,5,2,3,0,1,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-SLOW-NEXT:    vporq %ymm1, %ymm2, %ymm19
-; AVX512DQ-SLOW-NEXT:    vpermq {{.*#+}} ymm22 = ymm0[0,1,1,3]
+; AVX512DQ-SLOW-NEXT:    vporq %ymm1, %ymm2, %ymm17
+; AVX512DQ-SLOW-NEXT:    vpermq {{.*#+}} ymm23 = ymm0[0,1,1,3]
 ; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm9[0],ymm8[1],ymm9[2,3,4],ymm8[5],ymm9[6,7]
 ; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2],xmm0[3],xmm1[4,5,6,7]
 ; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,6,7,4,5,2,3,0,1,14,15,u,u,u,u]
 ; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512DQ-SLOW-NEXT:    vpshufd {{.*#+}} ymm1 = ymm22[0,1,2,1,4,5,6,5]
+; AVX512DQ-SLOW-NEXT:    vpshufd {{.*#+}} ymm1 = ymm23[0,1,2,1,4,5,6,5]
 ; AVX512DQ-SLOW-NEXT:    vpshufhw {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5,6,5,8,9,10,11,12,13,14,13]
 ; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,6],ymm1[7]
 ; AVX512DQ-SLOW-NEXT:    vpbroadcastw 232(%rdi), %xmm1
-; AVX512DQ-SLOW-NEXT:    vpsrlq $48, %xmm15, %xmm2
+; AVX512DQ-SLOW-NEXT:    vpsrlq $48, %xmm14, %xmm2
 ; AVX512DQ-SLOW-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
-; AVX512DQ-SLOW-NEXT:    vinserti32x4 $2, %xmm1, %zmm0, %zmm21
+; AVX512DQ-SLOW-NEXT:    vinserti32x4 $2, %xmm1, %zmm0, %zmm20
 ; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm9[0,1],ymm8[2],ymm9[3,4],ymm8[5],ymm9[6,7]
 ; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm1
 ; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2],xmm1[3],xmm0[4],xmm1[5,6,7]
 ; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[8,9,8,9,6,7,4,5,2,3,u,u,u,u,u,u]
 ; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
 ; AVX512DQ-SLOW-NEXT:    vmovdqa 208(%rdi), %xmm2
-; AVX512DQ-SLOW-NEXT:    vmovdqa 192(%rdi), %xmm3
-; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} xmm1 = xmm3[0,1,2,3,4,5],xmm2[6],xmm3[7]
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %xmm3, %xmm17
+; AVX512DQ-SLOW-NEXT:    vmovdqa 192(%rdi), %xmm7
+; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} xmm1 = xmm7[0,1,2,3,4,5],xmm2[6],xmm7[7]
 ; AVX512DQ-SLOW-NEXT:    vmovdqa64 %xmm2, %xmm29
 ; AVX512DQ-SLOW-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[0,1,0,3]
 ; AVX512DQ-SLOW-NEXT:    vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,4,7,6]
 ; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
 ; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7],ymm0[8,9,10,11,12],ymm1[13,14,15]
-; AVX512DQ-SLOW-NEXT:    vpsrld $16, %xmm5, %xmm1
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %xmm5, %xmm27
-; AVX512DQ-SLOW-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm1[4],xmm15[4],xmm1[5],xmm15[5],xmm1[6],xmm15[6],xmm1[7],xmm15[7]
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %xmm15, %xmm28
-; AVX512DQ-SLOW-NEXT:    vinserti32x4 $2, %xmm1, %zmm0, %zmm23
-; AVX512DQ-SLOW-NEXT:    vmovdqa 352(%rdi), %ymm11
-; AVX512DQ-SLOW-NEXT:    vmovdqa 320(%rdi), %ymm15
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm15[0,1,2],ymm11[3],ymm15[4,5],ymm11[6],ymm15[7]
-; AVX512DQ-SLOW-NEXT:    vpermq {{.*#+}} ymm2 = ymm0[2,3,0,1]
-; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm2[6],ymm0[7,8,9,10,11,12,13],ymm2[14],ymm0[15]
-; AVX512DQ-SLOW-NEXT:    vpshufd {{.*#+}} xmm2 = xmm14[0,2,2,1]
-; AVX512DQ-SLOW-NEXT:    vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,7,6,7]
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[0,1,14,15,12,13,10,11,4,5,6,7,8,9,2,3,16,17,30,31,28,29,26,27,20,21,22,23,24,25,18,19]
-; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} xmm2 = xmm0[0,1,2],xmm2[3,4,5,6],xmm0[7]
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
-; AVX512DQ-SLOW-NEXT:    vmovdqa 384(%rdi), %ymm14
-; AVX512DQ-SLOW-NEXT:    vmovdqa 416(%rdi), %ymm5
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm5[0,1],ymm14[2],ymm5[3,4],ymm14[5],ymm5[6,7]
+; AVX512DQ-SLOW-NEXT:    vpsrld $16, %xmm13, %xmm1
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %xmm13, %xmm27
+; AVX512DQ-SLOW-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm1[4],xmm14[4],xmm1[5],xmm14[5],xmm1[6],xmm14[6],xmm1[7],xmm14[7]
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %xmm14, %xmm28
+; AVX512DQ-SLOW-NEXT:    vinserti32x4 $2, %xmm1, %zmm0, %zmm24
+; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm5, %ymm12
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm5[0,1],ymm6[2,3],ymm5[4,5],ymm6[6,7]
+; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3],xmm1[4],xmm0[5],xmm1[6,7]
+; AVX512DQ-SLOW-NEXT:    vmovdqa 352(%rdi), %ymm1
+; AVX512DQ-SLOW-NEXT:    vmovdqa 320(%rdi), %ymm14
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm14[0,1,2],ymm1[3],ymm14[4,5],ymm1[6],ymm14[7]
+; AVX512DQ-SLOW-NEXT:    vpermq {{.*#+}} ymm3 = ymm2[2,3,0,1]
+; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm3[6],ymm2[7,8,9,10,11,12,13],ymm3[14],ymm2[15]
+; AVX512DQ-SLOW-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[0,2,2,1]
+; AVX512DQ-SLOW-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,7,6,7]
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} ymm2 = ymm2[0,1,14,15,12,13,10,11,4,5,6,7,8,9,2,3,16,17,30,31,28,29,26,27,20,21,22,23,24,25,18,19]
+; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3,4,5,6],xmm2[7]
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
+; AVX512DQ-SLOW-NEXT:    vmovdqa 384(%rdi), %ymm5
+; AVX512DQ-SLOW-NEXT:    vmovdqa 416(%rdi), %ymm13
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm13[0,1],ymm5[2],ymm13[3,4],ymm5[5],ymm13[6,7]
 ; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm2, %xmm3
 ; AVX512DQ-SLOW-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm3[0,1,3,1,4,5,6,7]
 ; AVX512DQ-SLOW-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,1,2,3]
@@ -6505,18 +6500,17 @@ define void @load_i16_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
 ; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm2[6,7]
 ; AVX512DQ-SLOW-NEXT:    vmovdqa64 %ymm0, %ymm30
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %ymm25, %ymm4
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm6[0,1,2],ymm4[3],ymm6[4,5],ymm4[6],ymm6[7]
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm12[0,1,2],ymm6[3],ymm12[4,5],ymm6[6],ymm12[7]
 ; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm2
 ; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm2[0,1,2,3],xmm0[4],xmm2[5],xmm0[6],xmm2[7]
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm11[0],ymm15[1],ymm11[2,3],ymm15[4],ymm11[5,6,7]
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm1[0],ymm14[1],ymm1[2,3],ymm14[4],ymm1[5,6,7]
 ; AVX512DQ-SLOW-NEXT:    vpermq {{.*#+}} ymm3 = ymm2[2,3,0,1]
 ; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} ymm2 = ymm3[0],ymm2[1,2,3,4,5,6],ymm3[7,8],ymm2[9,10,11,12,13,14],ymm3[15]
 ; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,u,12,13,10,11,8,9,6,7,u,u]
 ; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} ymm2 = ymm2[2,3,0,1,14,15,12,13,4,5,4,5,4,5,4,5,18,19,16,17,30,31,28,29,20,21,20,21,20,21,20,21]
 ; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm2[0,1,2],xmm0[3,4,5,6],xmm2[7]
 ; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm5[0,1],ymm14[2,3],ymm5[4,5],ymm14[6,7]
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm13[0,1],ymm5[2,3],ymm13[4,5],ymm5[6,7]
 ; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm2, %xmm3
 ; AVX512DQ-SLOW-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[2,1,2,3]
 ; AVX512DQ-SLOW-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm3[0,1,0,2,4,5,6,7]
@@ -6525,17 +6519,18 @@ define void @load_i16_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-SLOW-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
 ; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
 ; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm2[6,7]
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %ymm0, %ymm16
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm11[0],ymm15[1],ymm11[2,3,4],ymm15[5],ymm11[6,7]
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %ymm0, %ymm31
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0],ymm14[1],ymm1[2,3,4],ymm14[5],ymm1[6,7]
 ; AVX512DQ-SLOW-NEXT:    vpermq {{.*#+}} ymm2 = ymm0[2,3,0,1]
 ; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,u,u,6,7,20,21,u,u,16,17,30,31,u,u,u,u,u,u,u,u]
 ; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm2[1],ymm0[2,3,4,5,6,7,8],ymm2[9],ymm0[10,11,12,13,14,15]
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm4[0,1],ymm6[2],ymm4[3,4,5],ymm6[6],ymm4[7]
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm6[0,1],ymm12[2],ymm6[3,4,5],ymm12[6],ymm6[7]
 ; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm2, %xmm3
 ; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm3[4],xmm2[5],xmm3[6],xmm2[7]
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} ymm2 = ymm2[0,1,2,3,0,1,14,15,12,13,10,11,8,9],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[u,u,u,u,u,u,u,u]
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %ymm25, %ymm3
+; AVX512DQ-SLOW-NEXT:    vpshufb %ymm3, %ymm2, %ymm2
 ; AVX512DQ-SLOW-NEXT:    vpternlogq $244, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm2
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm5[0,1,2],ymm14[3],ymm5[4,5],ymm14[6],ymm5[7]
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm13[0,1,2],ymm5[3],ymm13[4,5],ymm5[6],ymm13[7]
 ; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm3
 ; AVX512DQ-SLOW-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,6,4]
 ; AVX512DQ-SLOW-NEXT:    vpshufd {{.*#+}} xmm3 = xmm3[0,1,2,1]
@@ -6544,31 +6539,31 @@ define void @load_i16_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
 ; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5],ymm0[6,7]
 ; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm25 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
-; AVX512DQ-SLOW-NEXT:    vpternlogq $184, %zmm18, %zmm25, %zmm24
+; AVX512DQ-SLOW-NEXT:    vpternlogq $184, %zmm19, %zmm25, %zmm22
 ; AVX512DQ-SLOW-NEXT:    movw $-512, %ax # imm = 0xFE00
 ; AVX512DQ-SLOW-NEXT:    kmovw %eax, %k1
-; AVX512DQ-SLOW-NEXT:    vinserti32x8 $1, %ymm0, %zmm0, %zmm24 {%k1}
+; AVX512DQ-SLOW-NEXT:    vinserti32x8 $1, %ymm0, %zmm0, %zmm22 {%k1}
 ; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm9[0],ymm8[1],ymm9[2,3],ymm8[4],ymm9[5,6,7]
 ; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm2
 ; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2,3,4,5],xmm2[6],xmm0[7]
 ; AVX512DQ-SLOW-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[0,2,1,0,4,5,6,7]
 ; AVX512DQ-SLOW-NEXT:    vpshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,6,6,7]
 ; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512DQ-SLOW-NEXT:    vpshufd {{.*#+}} ymm2 = ymm22[0,1,2,0,4,5,6,4]
+; AVX512DQ-SLOW-NEXT:    vpshufd {{.*#+}} ymm2 = ymm23[0,1,2,0,4,5,6,4]
 ; AVX512DQ-SLOW-NEXT:    vpshufhw {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5,7,4,8,9,10,11,12,13,15,12]
 ; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,6],ymm2[7]
-; AVX512DQ-SLOW-NEXT:    vpshufhw {{.*#+}} xmm2 = xmm13[0,1,2,3,6,5,6,7]
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %xmm26, %xmm2
+; AVX512DQ-SLOW-NEXT:    vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,6,5,6,7]
 ; AVX512DQ-SLOW-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,2,2,2]
-; AVX512DQ-SLOW-NEXT:    vinserti32x4 $2, %xmm2, %zmm0, %zmm18
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm4[0,1,2],ymm6[3],ymm4[4,5],ymm6[6],ymm4[7]
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %ymm6, %ymm31
+; AVX512DQ-SLOW-NEXT:    vinserti32x4 $2, %xmm2, %zmm0, %zmm19
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm6[0,1,2],ymm12[3],ymm6[4,5],ymm12[6],ymm6[7]
 ; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm2
 ; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} xmm0 = xmm2[0],xmm0[1],xmm2[2,3,4,5],xmm0[6],xmm2[7]
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm14[0,1],ymm5[2],ymm14[3,4,5],ymm5[6],ymm14[7]
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm5[0,1],ymm13[2],ymm5[3,4,5],ymm13[6],ymm5[7]
 ; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm2, %xmm3
 ; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm3[4],xmm2[5],xmm3[6],xmm2[7]
 ; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,2,3,2,3,0,1,14,15,12,13,10,11],zero,zero
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm11[0,1],ymm15[2],ymm11[3,4],ymm15[5],ymm11[6,7]
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm1[0,1],ymm14[2],ymm1[3,4],ymm14[5],ymm1[6,7]
 ; AVX512DQ-SLOW-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[1,1,2,0]
 ; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} ymm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm3[0,1,22,23,28,29,18,19],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; AVX512DQ-SLOW-NEXT:    vpor %ymm3, %ymm0, %ymm0
@@ -6577,446 +6572,434 @@ define void @load_i16_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
 ; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} ymm2 = ymm3[0,1,2],ymm2[3,4,5,6,7],ymm3[8,9,10],ymm2[11,12,13,14,15]
 ; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
-; AVX512DQ-SLOW-NEXT:    vpternlogq $184, %zmm19, %zmm25, %zmm18
-; AVX512DQ-SLOW-NEXT:    vinserti32x8 $1, %ymm0, %zmm0, %zmm18 {%k1}
-; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm12, %ymm1
-; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm7, %ymm6
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm12[0,1],ymm7[2],ymm12[3,4,5],ymm7[6],ymm12[7]
+; AVX512DQ-SLOW-NEXT:    vpternlogq $184, %zmm17, %zmm25, %zmm19
+; AVX512DQ-SLOW-NEXT:    vinserti32x8 $1, %ymm0, %zmm0, %zmm19 {%k1}
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm4[0,1],ymm10[2],ymm4[3,4,5],ymm10[6],ymm4[7]
 ; AVX512DQ-SLOW-NEXT:    vpermq {{.*#+}} ymm2 = ymm0[2,3,0,1]
 ; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[u,u,u,u,u,u,u,u,0,1,14,15,u,u,10,11,24,25,24,25,24,25,24,25,16,17,30,31,u,u,26,27]
 ; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm2[6],ymm0[7,8,9,10,11,12,13],ymm2[14],ymm0[15]
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %ymm26, %ymm13
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm10[0,1],ymm13[2],ymm10[3,4],ymm13[5],ymm10[6,7]
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %ymm10, %ymm22
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %ymm15, %ymm23
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm15[0,1],ymm11[2],ymm15[3,4],ymm11[5],ymm15[6,7]
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %ymm11, %ymm26
 ; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm2, %xmm3
 ; AVX512DQ-SLOW-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm3[3,1,2,3,4,5,6,7]
 ; AVX512DQ-SLOW-NEXT:    vpshufd {{.*#+}} xmm2 = xmm2[2,1,2,3]
 ; AVX512DQ-SLOW-NEXT:    vpshuflw {{.*#+}} xmm2 = xmm2[0,2,2,3,4,5,6,7]
 ; AVX512DQ-SLOW-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1],ymm0[2,3,4,5,6,7]
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm14[0,1,2],ymm5[3],ymm14[4,5],ymm5[6],ymm14[7]
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm11 = ymm2[0,1],ymm0[2,3,4,5,6,7]
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm5[0,1,2],ymm13[3],ymm5[4,5],ymm13[6],ymm5[7]
 ; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm2, %xmm3
 ; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0],xmm2[1],xmm3[2,3,4,5],xmm2[6],xmm3[7]
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm11[0,1],ymm15[2,3],ymm11[4,5],ymm15[6,7]
-; AVX512DQ-SLOW-NEXT:    vpermq {{.*#+}} ymm12 = ymm11[0,1,0,1]
-; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0,1,2],ymm12[3],ymm3[4,5,6,7,8,9,10],ymm12[11],ymm3[12,13,14,15]
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %ymm31, %ymm7
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm12 = ymm7[0],ymm4[1],ymm7[2,3],ymm4[4],ymm7[5,6,7]
-; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm12, %xmm10
-; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} xmm10 = xmm12[0],xmm10[1],xmm12[2,3,4,5],xmm10[6],xmm12[7]
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm1[0,1],ymm14[2,3],ymm1[4,5],ymm14[6,7]
+; AVX512DQ-SLOW-NEXT:    vpermq {{.*#+}} ymm15 = ymm1[0,1,0,1]
+; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0,1,2],ymm15[3],ymm3[4,5,6,7,8,9,10],ymm15[11],ymm3[12,13,14,15]
+; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm12, %ymm0
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm15 = ymm12[0],ymm6[1],ymm12[2,3],ymm6[4],ymm12[5,6,7]
+; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm15, %xmm12
+; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} xmm12 = xmm15[0],xmm12[1],xmm15[2,3,4,5],xmm12[6],xmm15[7]
 ; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,u,u,u,u,u,2,3,0,1,14,15,12,13,10,11]
 ; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
 ; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} ymm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm3[10,11,24,25,22,23,20,21],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} ymm2 = ymm3[0,1,2],ymm2[3,4,5,6,7],ymm3[8,9,10],ymm2[11,12,13,14,15]
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm10 = xmm10[0,1,2,3,4,5,2,3,0,1,14,15,12,13],zero,zero
-; AVX512DQ-SLOW-NEXT:    vpor %ymm3, %ymm10, %ymm3
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm12 = xmm12[0,1,2,3,4,5,2,3,0,1,14,15,12,13],zero,zero
+; AVX512DQ-SLOW-NEXT:    vpor %ymm3, %ymm12, %ymm3
 ; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
-; AVX512DQ-SLOW-NEXT:    vpternlogq $226, %zmm21, %zmm25, %zmm0
-; AVX512DQ-SLOW-NEXT:    vinserti32x8 $1, %ymm2, %zmm0, %zmm0 {%k1}
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm5[0],ymm14[1],ymm5[2,3],ymm14[4],ymm5[5,6,7]
+; AVX512DQ-SLOW-NEXT:    vpternlogq $226, %zmm20, %zmm25, %zmm11
+; AVX512DQ-SLOW-NEXT:    vinserti32x8 $1, %ymm2, %zmm0, %zmm11 {%k1}
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm13[0],ymm5[1],ymm13[2,3],ymm5[4],ymm13[5,6,7]
 ; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm2, %xmm3
 ; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0],xmm3[1],xmm2[2,3,4,5],xmm3[6],xmm2[7]
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm11[0,1,2],ymm15[3],ymm11[4,5],ymm15[6],ymm11[7]
-; AVX512DQ-SLOW-NEXT:    vpermq {{.*#+}} ymm10 = ymm3[2,3,0,1]
-; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm10[4],ymm3[5,6,7,8,9,10,11],ymm10[12],ymm3[13,14,15]
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm10 = ymm7[0],ymm4[1],ymm7[2,3,4],ymm4[5],ymm7[6,7]
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %ymm4, %ymm21
-; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm10, %xmm12
-; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} xmm10 = xmm12[0],xmm10[1],xmm12[2],xmm10[3],xmm12[4,5,6,7]
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm1[0,1,2],ymm14[3],ymm1[4,5],ymm14[6],ymm1[7]
+; AVX512DQ-SLOW-NEXT:    vpermq {{.*#+}} ymm12 = ymm3[2,3,0,1]
+; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm12[4],ymm3[5,6,7,8,9,10,11],ymm12[12],ymm3[13,14,15]
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm12 = ymm0[0],ymm6[1],ymm0[2,3,4],ymm6[5],ymm0[6,7]
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %ymm0, %ymm20
+; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm12, %xmm15
+; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} xmm12 = xmm15[0],xmm12[1],xmm15[2],xmm12[3],xmm15[4,5,6,7]
 ; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,u,u,u,u,u,4,5,2,3,0,1,14,15,12,13]
 ; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
 ; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} ymm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm3[12,13,26,27,24,25,22,23],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} ymm2 = ymm3[0,1,2],ymm2[3,4,5,6,7],ymm3[8,9,10],ymm2[11,12,13,14,15]
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm10 = xmm10[0,1,2,3,6,7,4,5,2,3,0,1,14,15],zero,zero
-; AVX512DQ-SLOW-NEXT:    vpor %ymm3, %ymm10, %ymm3
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm7 = ymm3[0,1,2,3],ymm2[4,5,6,7]
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm1[0,1,2],ymm6[3],ymm1[4,5],ymm6[6],ymm1[7]
-; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm1, %ymm3
-; AVX512DQ-SLOW-NEXT:    vpermq {{.*#+}} ymm10 = ymm2[2,3,0,1]
-; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} ymm2 = ymm10[0],ymm2[1,2,3,4,5,6],ymm10[7,8],ymm2[9,10,11,12,13,14],ymm10[15]
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm12 = xmm12[0,1,2,3,6,7,4,5,2,3,0,1,14,15],zero,zero
+; AVX512DQ-SLOW-NEXT:    vpor %ymm3, %ymm12, %ymm3
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3],ymm2[4,5,6,7]
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %ymm0, %ymm17
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm4[0,1,2],ymm10[3],ymm4[4,5],ymm10[6],ymm4[7]
+; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm4, %ymm3
+; AVX512DQ-SLOW-NEXT:    vpermq {{.*#+}} ymm12 = ymm2[2,3,0,1]
+; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} ymm2 = ymm12[0],ymm2[1,2,3,4,5,6],ymm12[7,8],ymm2[9,10,11,12,13,14],ymm12[15]
 ; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm9[0,1],ymm8[2,3],ymm9[4,5],ymm8[6,7]
 ; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm8, %xmm9
-; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} xmm10 = xmm9[0,1,2],xmm8[3],xmm9[4],xmm8[5],xmm9[6,7]
+; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} xmm12 = xmm9[0,1,2],xmm8[3],xmm9[4],xmm8[5],xmm9[6,7]
 ; AVX512DQ-SLOW-NEXT:    vinserti64x4 $1, %ymm30, %zmm0, %zmm8
-; AVX512DQ-SLOW-NEXT:    vinserti64x4 $1, %ymm16, %zmm0, %zmm9
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %xmm29, %xmm1
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %xmm17, %xmm4
-; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} xmm4 = xmm1[0],xmm4[1],xmm1[2,3,4,5,6,7]
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm10 = xmm10[8,9,10,11,8,9,6,7,4,5,u,u,u,u,u,u]
-; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
-; AVX512DQ-SLOW-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[0,1,0,3]
-; AVX512DQ-SLOW-NEXT:    vpshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,5,4,7]
-; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} ymm4 = ymm10[0,1,2,3,4],ymm4[5,6,7],ymm10[8,9,10,11,12],ymm4[13,14,15]
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %xmm27, %xmm1
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %xmm28, %xmm10
-; AVX512DQ-SLOW-NEXT:    vpunpckhwd {{.*#+}} xmm10 = xmm1[4],xmm10[4],xmm1[5],xmm10[5],xmm1[6],xmm10[6],xmm1[7],xmm10[7]
-; AVX512DQ-SLOW-NEXT:    vpshufd {{.*#+}} xmm10 = xmm10[2,1,2,3]
-; AVX512DQ-SLOW-NEXT:    vpshuflw {{.*#+}} xmm10 = xmm10[0,3,2,3,4,5,6,7]
-; AVX512DQ-SLOW-NEXT:    vinserti32x4 $2, %xmm10, %zmm4, %zmm19
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %ymm22, %ymm4
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm10 = ymm4[0,1],ymm13[2,3],ymm4[4,5],ymm13[6,7]
-; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm10, %xmm12
+; AVX512DQ-SLOW-NEXT:    vinserti64x4 $1, %ymm31, %zmm0, %zmm9
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %xmm29, %xmm0
+; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} xmm7 = xmm0[0],xmm7[1],xmm0[2,3,4,5,6,7]
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm12 = xmm12[8,9,10,11,8,9,6,7,4,5,u,u,u,u,u,u]
+; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm12, %ymm0, %ymm12
+; AVX512DQ-SLOW-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[0,1,0,3]
+; AVX512DQ-SLOW-NEXT:    vpshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,4,5,4,7]
+; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm7, %ymm0, %ymm7
+; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} ymm7 = ymm12[0,1,2,3,4],ymm7[5,6,7],ymm12[8,9,10,11,12],ymm7[13,14,15]
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %xmm27, %xmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %xmm28, %xmm4
+; AVX512DQ-SLOW-NEXT:    vpunpckhwd {{.*#+}} xmm12 = xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
+; AVX512DQ-SLOW-NEXT:    vpshufd {{.*#+}} xmm12 = xmm12[2,1,2,3]
+; AVX512DQ-SLOW-NEXT:    vpshuflw {{.*#+}} xmm12 = xmm12[0,3,2,3,4,5,6,7]
+; AVX512DQ-SLOW-NEXT:    vinserti32x4 $2, %xmm12, %zmm7, %zmm27
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %ymm23, %ymm7
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %ymm26, %ymm0
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm12 = ymm7[0,1],ymm0[2,3],ymm7[4,5],ymm0[6,7]
+; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm12, %xmm15
+; AVX512DQ-SLOW-NEXT:    vpshufd {{.*#+}} xmm15 = xmm15[2,1,2,3]
+; AVX512DQ-SLOW-NEXT:    vpshuflw {{.*#+}} xmm15 = xmm15[0,2,2,3,4,5,6,7]
 ; AVX512DQ-SLOW-NEXT:    vpshufd {{.*#+}} xmm12 = xmm12[2,1,2,3]
-; AVX512DQ-SLOW-NEXT:    vpshuflw {{.*#+}} xmm12 = xmm12[0,2,2,3,4,5,6,7]
-; AVX512DQ-SLOW-NEXT:    vpshufd {{.*#+}} xmm10 = xmm10[2,1,2,3]
-; AVX512DQ-SLOW-NEXT:    vpshuflw {{.*#+}} xmm10 = xmm10[1,3,2,3,4,5,6,7]
-; AVX512DQ-SLOW-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm12[0],xmm10[1],xmm12[1],xmm10[2],xmm12[2],xmm10[3],xmm12[3]
+; AVX512DQ-SLOW-NEXT:    vpshuflw {{.*#+}} xmm12 = xmm12[1,3,2,3,4,5,6,7]
+; AVX512DQ-SLOW-NEXT:    vpunpcklwd {{.*#+}} xmm12 = xmm12[0],xmm15[0],xmm12[1],xmm15[1],xmm12[2],xmm15[2],xmm12[3],xmm15[3]
 ; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} ymm2 = ymm2[u,u,u,u,u,u,u,u,2,3,0,1,14,15,12,13,26,27,26,27,26,27,26,27,18,19,16,17,30,31,28,29]
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm10[0,1],ymm2[2,3,4,5,6,7]
-; AVX512DQ-SLOW-NEXT:    vpternlogq $226, %zmm23, %zmm25, %zmm2
-; AVX512DQ-SLOW-NEXT:    vinserti32x8 $1, %ymm7, %zmm0, %zmm2 {%k1}
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm6[0],ymm3[1],ymm6[2,3],ymm3[4],ymm6[5,6,7]
-; AVX512DQ-SLOW-NEXT:    vpermq {{.*#+}} ymm7 = ymm3[2,3,0,1]
-; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm7[1],ymm3[2,3,4,5,6,7,8],ymm7[9],ymm3[10,11,12,13,14,15]
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm7 = ymm4[0,1,2],ymm13[3],ymm4[4,5],ymm13[6],ymm4[7]
-; AVX512DQ-SLOW-NEXT:    vpshufhw {{.*#+}} xmm10 = xmm7[0,1,2,3,6,4,6,7]
-; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm7, %xmm7
-; AVX512DQ-SLOW-NEXT:    vpshufd {{.*#+}} xmm7 = xmm7[0,1,2,1]
-; AVX512DQ-SLOW-NEXT:    vpshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,5,7,6,7]
-; AVX512DQ-SLOW-NEXT:    vpunpckhwd {{.*#+}} xmm7 = xmm10[4],xmm7[4],xmm10[5],xmm7[5],xmm10[6],xmm7[6],xmm10[7],xmm7[7]
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm12[0,1],ymm2[2,3,4,5,6,7]
+; AVX512DQ-SLOW-NEXT:    vpternlogq $226, %zmm24, %zmm25, %zmm2
+; AVX512DQ-SLOW-NEXT:    vinserti32x8 $1, %ymm17, %zmm0, %zmm2 {%k1}
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm10[0],ymm3[1],ymm10[2,3],ymm3[4],ymm10[5,6,7]
+; AVX512DQ-SLOW-NEXT:    vpermq {{.*#+}} ymm4 = ymm3[2,3,0,1]
+; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm4[1],ymm3[2,3,4,5,6,7,8],ymm4[9],ymm3[10,11,12,13,14,15]
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm7[0,1,2],ymm0[3],ymm7[4,5],ymm0[6],ymm7[7]
+; AVX512DQ-SLOW-NEXT:    vpshufhw {{.*#+}} xmm10 = xmm4[0,1,2,3,6,4,6,7]
+; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm4, %xmm4
+; AVX512DQ-SLOW-NEXT:    vpshufd {{.*#+}} xmm4 = xmm4[0,1,2,1]
+; AVX512DQ-SLOW-NEXT:    vpshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,5,7,6,7]
+; AVX512DQ-SLOW-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm10[4],xmm4[4],xmm10[5],xmm4[5],xmm10[6],xmm4[6],xmm10[7],xmm4[7]
 ; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} ymm3 = ymm3[u,u,u,u,u,u,u,u,4,5,2,3,0,1,14,15,28,29,28,29,28,29,28,29,20,21,18,19,16,17,30,31]
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm7[0,1],ymm3[2,3,4,5,6,7]
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm7 = ymm5[0],ymm14[1],ymm5[2,3,4],ymm14[5],ymm5[6,7]
-; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm7, %xmm10
-; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} xmm7 = xmm10[0],xmm7[1],xmm10[2],xmm7[3],xmm10[4,5,6,7]
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %ymm21, %ymm1
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %ymm31, %ymm4
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm5 = ymm4[0,1],ymm1[2],ymm4[3,4],ymm1[5],ymm4[6,7]
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1],ymm3[2,3,4,5,6,7]
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm13[0],ymm5[1],ymm13[2,3,4],ymm5[5],ymm13[6,7]
+; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm4, %xmm10
+; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} xmm4 = xmm10[0],xmm4[1],xmm10[2],xmm4[3],xmm10[4,5,6,7]
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %ymm20, %ymm0
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm5 = ymm0[0,1],ymm6[2],ymm0[3,4],ymm6[5],ymm0[6,7]
 ; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm5, %xmm6
 ; AVX512DQ-SLOW-NEXT:    vpshuflw {{.*#+}} xmm6 = xmm6[0,3,1,3,4,5,6,7]
 ; AVX512DQ-SLOW-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[2,1,2,3]
 ; AVX512DQ-SLOW-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm5[0,0,2,3,4,5,6,7]
 ; AVX512DQ-SLOW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3]
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm15[0,1],ymm11[2],ymm15[3,4,5],ymm11[6],ymm15[7]
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm14[0,1],ymm1[2],ymm14[3,4,5],ymm1[6],ymm14[7]
 ; AVX512DQ-SLOW-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,1,3,1]
 ; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[4,5,10,11,u,u,u,u,u,u,u,u,0,1,14,15,20,21,26,27,16,17,26,27,16,17,20,21,16,17,30,31]
 ; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0],ymm5[1,2],ymm1[3,4,5,6,7]
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm5 = xmm7[u,u,u,u,u,u,6,7,4,5,2,3,0,1,14,15]
-; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} ymm5 = ymm1[0,1,2],ymm5[3,4,5,6,7],ymm1[8,9,10],ymm5[11,12,13,14,15]
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm5[4,5,6,7]
-; AVX512DQ-SLOW-NEXT:    vpternlogq $226, %zmm19, %zmm25, %zmm3
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[u,u,u,u,u,u,6,7,4,5,2,3,0,1,14,15]
+; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} ymm4 = ymm1[0,1,2],ymm4[3,4,5,6,7],ymm1[8,9,10],ymm4[11,12,13,14,15]
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm4[4,5,6,7]
+; AVX512DQ-SLOW-NEXT:    vpternlogq $226, %zmm27, %zmm25, %zmm3
 ; AVX512DQ-SLOW-NEXT:    vinserti32x8 $1, %ymm1, %zmm0, %zmm3 {%k1}
 ; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512DQ-SLOW-NEXT:    vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm8 # 64-byte Folded Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm4, %zmm20
-; AVX512DQ-SLOW-NEXT:    vpternlogq $184, %zmm20, %zmm1, %zmm9
+; AVX512DQ-SLOW-NEXT:    vpternlogq $184, %zmm18, %zmm1, %zmm8
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm21
+; AVX512DQ-SLOW-NEXT:    vpternlogq $184, %zmm21, %zmm1, %zmm9
 ; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm8, (%rsi)
 ; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm9, (%rdx)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm24, (%rcx)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm18, (%r8)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, (%r9)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm22, (%rcx)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm19, (%r8)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm11, (%r9)
 ; AVX512DQ-SLOW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm2, (%rax)
 ; AVX512DQ-SLOW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm3, (%rax)
-; AVX512DQ-SLOW-NEXT:    popq %rax
 ; AVX512DQ-SLOW-NEXT:    vzeroupper
 ; AVX512DQ-SLOW-NEXT:    retq
 ;
 ; AVX512DQ-FAST-LABEL: load_i16_stride7_vf32:
 ; AVX512DQ-FAST:       # %bb.0:
-; AVX512DQ-FAST-NEXT:    vmovdqa64 320(%rdi), %zmm27
-; AVX512DQ-FAST-NEXT:    vmovdqa64 64(%rdi), %zmm29
-; AVX512DQ-FAST-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [2,6,9,13,2,6,9,13]
-; AVX512DQ-FAST-NEXT:    # ymm1 = mem[0,1,0,1]
-; AVX512DQ-FAST-NEXT:    vmovdqa64 128(%rdi), %zmm24
-; AVX512DQ-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} ymm30 = [2,5,9,12,2,5,9,12]
-; AVX512DQ-FAST-NEXT:    # ymm30 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-FAST-NEXT:    vmovdqa {{.*#+}} ymm2 = [10,3,6,15,12,13,6,15]
-; AVX512DQ-FAST-NEXT:    vpermd %zmm29, %zmm2, %zmm0
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} ymm21 = [3,6,10,13,3,6,10,13]
-; AVX512DQ-FAST-NEXT:    # ymm21 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-FAST-NEXT:    vmovdqa64 384(%rdi), %zmm17
-; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} ymm19 = <1,u,u,u,5,8,12,15>
-; AVX512DQ-FAST-NEXT:    vmovdqa {{.*#+}} ymm2 = <2,6,9,u,13,u,u,u>
-; AVX512DQ-FAST-NEXT:    vpermd %zmm29, %zmm2, %zmm12
-; AVX512DQ-FAST-NEXT:    vpermd %zmm17, %zmm1, %zmm15
-; AVX512DQ-FAST-NEXT:    vmovdqa {{.*#+}} ymm2 = <1,u,u,u,4,8,11,15>
-; AVX512DQ-FAST-NEXT:    vpermd %zmm27, %zmm2, %zmm2
-; AVX512DQ-FAST-NEXT:    vmovdqa {{.*#+}} ymm3 = <2,5,9,u,12,u,u,u>
-; AVX512DQ-FAST-NEXT:    vpermd %zmm29, %zmm3, %zmm3
-; AVX512DQ-FAST-NEXT:    vpermd %zmm17, %zmm30, %zmm9
-; AVX512DQ-FAST-NEXT:    vmovdqa {{.*#+}} ymm4 = <0,u,u,u,4,7,11,14>
-; AVX512DQ-FAST-NEXT:    vpermd %zmm27, %zmm4, %zmm8
-; AVX512DQ-FAST-NEXT:    vmovdqa {{.*#+}} ymm4 = [8,1,12,5,12,5,14,15]
-; AVX512DQ-FAST-NEXT:    vpermd %zmm29, %zmm4, %zmm4
-; AVX512DQ-FAST-NEXT:    vpermd %zmm24, %zmm21, %zmm5
-; AVX512DQ-FAST-NEXT:    vmovdqa64 192(%rdi), %ymm26
-; AVX512DQ-FAST-NEXT:    vpermq {{.*#+}} ymm13 = ymm26[0,1,0,2]
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm6 = ymm13[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,20,21,26,27]
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm5 = ymm5[0,1,2,3,0,1,6,7,8,9,14,15,12,13,14,15,16,17,18,19,16,17,22,23,24,25,30,31,u,u,u,u]
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm6 = ymm5[0,1,2,3,4,5,6],ymm6[7]
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm7 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm4[6,7,12,13,2,3,16,17,30,31],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-FAST-NEXT:    vmovdqa (%rdi), %ymm0
-; AVX512DQ-FAST-NEXT:    vmovdqa 32(%rdi), %ymm1
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm10 = ymm0[0,1],ymm1[2],ymm0[3,4,5],ymm1[6],ymm0[7]
-; AVX512DQ-FAST-NEXT:    vmovdqa %ymm1, %ymm4
-; AVX512DQ-FAST-NEXT:    vmovdqa %ymm0, %ymm1
-; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm10, %xmm11
-; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} xmm10 = xmm10[0,1,2,3],xmm11[4],xmm10[5],xmm11[6],xmm10[7]
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm10 = ymm10[0,1,14,15,12,13,10,11,8,9],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm10[u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-FAST-NEXT:    vporq %ymm7, %ymm10, %ymm22
-; AVX512DQ-FAST-NEXT:    vpbroadcastw 252(%rdi), %xmm7
-; AVX512DQ-FAST-NEXT:    vmovdqa 224(%rdi), %xmm0
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm10 = xmm0[u,u,u,u,u,u,u,u,0,1,14,15,12,13,14,15]
-; AVX512DQ-FAST-NEXT:    vpunpckhdq {{.*#+}} xmm7 = xmm10[2],xmm7[2],xmm10[3],xmm7[3]
+; AVX512DQ-FAST-NEXT:    vmovdqa64 320(%rdi), %zmm26
+; AVX512DQ-FAST-NEXT:    vmovdqa64 64(%rdi), %zmm1
+; AVX512DQ-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} ymm17 = [2,6,9,13,2,6,9,13]
+; AVX512DQ-FAST-NEXT:    # ymm17 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-FAST-NEXT:    vmovdqa64 128(%rdi), %zmm23
+; AVX512DQ-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} ymm18 = [2,5,9,12,2,5,9,12]
+; AVX512DQ-FAST-NEXT:    # ymm18 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} ymm19 = [3,6,10,13,3,6,10,13]
+; AVX512DQ-FAST-NEXT:    # ymm19 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-FAST-NEXT:    vmovdqa64 384(%rdi), %zmm20
+; AVX512DQ-FAST-NEXT:    vmovdqa {{.*#+}} ymm0 = <2,6,9,u,13,u,u,u>
+; AVX512DQ-FAST-NEXT:    vpermd %zmm1, %zmm0, %zmm7
+; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} ymm27 = <1,u,u,u,4,8,11,15>
+; AVX512DQ-FAST-NEXT:    vmovdqa {{.*#+}} ymm0 = <2,5,9,u,12,u,u,u>
+; AVX512DQ-FAST-NEXT:    vpermd %zmm1, %zmm0, %zmm10
+; AVX512DQ-FAST-NEXT:    vpermd %zmm20, %zmm18, %zmm0
+; AVX512DQ-FAST-NEXT:    vmovdqa {{.*#+}} ymm3 = <0,u,u,u,4,7,11,14>
+; AVX512DQ-FAST-NEXT:    vpermd %zmm26, %zmm3, %zmm9
+; AVX512DQ-FAST-NEXT:    vmovdqa {{.*#+}} ymm3 = [8,1,12,5,12,5,14,15]
+; AVX512DQ-FAST-NEXT:    vpermd %zmm1, %zmm3, %zmm3
+; AVX512DQ-FAST-NEXT:    vpermd %zmm23, %zmm19, %zmm4
+; AVX512DQ-FAST-NEXT:    vmovdqa64 192(%rdi), %ymm25
+; AVX512DQ-FAST-NEXT:    vpermq {{.*#+}} ymm8 = ymm25[0,1,0,2]
+; AVX512DQ-FAST-NEXT:    vpbroadcastd {{.*#+}} ymm2 = [20,21,26,27,20,21,26,27,20,21,26,27,20,21,26,27,20,21,26,27,20,21,26,27,20,21,26,27,20,21,26,27]
+; AVX512DQ-FAST-NEXT:    vpshufb %ymm2, %ymm8, %ymm5
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %ymm2, %ymm30
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm4 = ymm4[0,1,2,3,0,1,6,7,8,9,14,15,12,13,14,15,16,17,18,19,16,17,22,23,24,25,30,31,u,u,u,u]
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm5 = ymm4[0,1,2,3,4,5,6],ymm5[7]
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm6 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm3[6,7,12,13,2,3,16,17,30,31],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512DQ-FAST-NEXT:    vmovdqa (%rdi), %ymm3
+; AVX512DQ-FAST-NEXT:    vmovdqa 32(%rdi), %ymm4
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm11 = ymm3[0,1],ymm4[2],ymm3[3,4,5],ymm4[6],ymm3[7]
+; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm11, %xmm12
+; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} xmm11 = xmm11[0,1,2,3],xmm12[4],xmm11[5],xmm12[6],xmm11[7]
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm11 = ymm11[0,1,14,15,12,13,10,11,8,9],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm11[u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-FAST-NEXT:    vporq %ymm6, %ymm11, %ymm22
+; AVX512DQ-FAST-NEXT:    vpbroadcastw 252(%rdi), %xmm6
+; AVX512DQ-FAST-NEXT:    vmovdqa 224(%rdi), %xmm13
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm11 = xmm13[u,u,u,u,u,u,u,u,0,1,14,15,12,13,14,15]
+; AVX512DQ-FAST-NEXT:    vpunpckhdq {{.*#+}} xmm6 = xmm11[2],xmm6[2],xmm11[3],xmm6[3]
 ; AVX512DQ-FAST-NEXT:    movw $992, %ax # imm = 0x3E0
 ; AVX512DQ-FAST-NEXT:    kmovw %eax, %k1
-; AVX512DQ-FAST-NEXT:    vinserti32x4 $2, %xmm7, %zmm6, %zmm22 {%k1}
+; AVX512DQ-FAST-NEXT:    vinserti32x4 $2, %xmm6, %zmm5, %zmm22 {%k1}
 ; AVX512DQ-FAST-NEXT:    vmovdqa 256(%rdi), %ymm5
-; AVX512DQ-FAST-NEXT:    vmovdqa 288(%rdi), %ymm7
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm10 = ymm7[0,1],ymm5[2,3],ymm7[4,5],ymm5[6,7]
-; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm10, %xmm11
-; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} xmm10 = xmm11[0,1,2],xmm10[3],xmm11[4],xmm10[5],xmm11[6,7]
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm10 = xmm10[u,u,u,u,u,u,10,11,8,9,6,7,4,5,u,u]
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm8 = ymm8[0,1,6,7,8,9,14,15,8,9,14,15,4,5,2,3,16,17,22,23,24,25,30,31,24,25,30,31,20,21,18,19]
-; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} xmm10 = xmm8[0,1,2],xmm10[3,4,5,6],xmm8[7]
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm10 = ymm10[0,1,2,3],ymm8[4,5,6,7]
-; AVX512DQ-FAST-NEXT:    vpbroadcastq {{.*#+}} ymm6 = [16,17,22,23,24,25,30,31,16,17,22,23,24,25,30,31,16,17,22,23,24,25,30,31,16,17,22,23,24,25,30,31]
-; AVX512DQ-FAST-NEXT:    vpshufb %ymm6, %ymm9, %ymm9
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm9 = ymm10[0,1,2,3,4,5],ymm9[6,7]
-; AVX512DQ-FAST-NEXT:    vinserti64x4 $1, %ymm9, %zmm0, %zmm23
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm3[0,1,6,7,8,9,18,19],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm9 = ymm1[0,1,2],ymm4[3],ymm1[4,5],ymm4[6],ymm1[7]
-; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm9, %xmm10
-; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} xmm9 = xmm10[0],xmm9[1],xmm10[2,3,4,5],xmm9[6],xmm10[7]
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm9 = ymm9[2,3,0,1,14,15,12,13,10,11],zero,zero,zero,zero,zero,zero,zero,zero,ymm9[u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-FAST-NEXT:    vporq %ymm3, %ymm9, %ymm20
-; AVX512DQ-FAST-NEXT:    vmovdqa 160(%rdi), %ymm3
-; AVX512DQ-FAST-NEXT:    vmovdqa 128(%rdi), %ymm9
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm10 = ymm9[0,1],ymm3[2],ymm9[3,4,5],ymm3[6],ymm9[7]
-; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm10, %xmm11
-; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} xmm11 = xmm10[0,1,2,3],xmm11[4],xmm10[5],xmm11[6],xmm10[7]
-; AVX512DQ-FAST-NEXT:    vmovdqa 240(%rdi), %xmm10
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm11 = xmm11[0,1,0,1,14,15,12,13,10,11,8,9,u,u,u,u]
-; AVX512DQ-FAST-NEXT:    vinserti128 $1, %xmm11, %ymm0, %ymm11
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm13 = ymm13[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,22,23,28,29]
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm11 = ymm11[0,1,2,3,4,5,6],ymm13[7]
-; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} xmm13 = xmm10[0],xmm0[1],xmm10[2,3,4,5,6,7]
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm13 = xmm13[2,3,0,1,14,15,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-FAST-NEXT:    vinserti64x4 $1, %ymm13, %zmm11, %zmm25
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm11 = ymm7[0,1,2],ymm5[3],ymm7[4,5],ymm5[6],ymm7[7]
-; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm11, %xmm13
-; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} xmm11 = xmm13[0,1,2,3],xmm11[4],xmm13[5],xmm11[6],xmm13[7]
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm11 = xmm11[u,u,u,u,u,u,12,13,10,11,8,9,6,7,u,u]
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm2 = ymm2[2,3,4,5,10,11,12,13,0,1,0,1,0,1,0,1,18,19,20,21,26,27,28,29,16,17,16,17,16,17,16,17]
-; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} xmm11 = xmm2[0,1,2],xmm11[3,4,5,6],xmm2[7]
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm2 = ymm11[0,1,2,3],ymm2[4,5,6,7]
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm11 = ymm15[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,19,20,21,26,27,28,29]
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm11[6,7]
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %ymm2, %ymm16
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm12[2,3,4,5,10,11,16,17],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm11 = ymm4[0],ymm1[1],ymm4[2,3],ymm1[4],ymm4[5,6,7]
+; AVX512DQ-FAST-NEXT:    vmovdqa 288(%rdi), %ymm6
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm11 = ymm6[0,1],ymm5[2,3],ymm6[4,5],ymm5[6,7]
 ; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm11, %xmm12
-; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} xmm11 = xmm11[0],xmm12[1],xmm11[2,3,4,5],xmm12[6],xmm11[7]
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm11 = ymm11[4,5,2,3,0,1,14,15,12,13],zero,zero,zero,zero,zero,zero,zero,zero,ymm11[u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-FAST-NEXT:    vpor %ymm2, %ymm11, %ymm11
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm2 = ymm9[0,1,2],ymm3[3],ymm9[4,5],ymm3[6],ymm9[7]
-; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm2, %xmm12
-; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} xmm2 = xmm12[0],xmm2[1],xmm12[2,3,4,5],xmm2[6],xmm12[7]
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[0,1,2,3,0,1,14,15,12,13,10,11,u,u,u,u]
-; AVX512DQ-FAST-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512DQ-FAST-NEXT:    vpbroadcastq {{.*#+}} ymm12 = [2,5,2,5,2,5,2,5]
-; AVX512DQ-FAST-NEXT:    vpermd %ymm26, %ymm12, %ymm12
-; AVX512DQ-FAST-NEXT:    vpshufhw {{.*#+}} ymm12 = ymm12[0,1,2,3,4,5,4,7,8,9,10,11,12,13,12,15]
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm12 = ymm2[0,1,2,3,4,5,6],ymm12[7]
-; AVX512DQ-FAST-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm0[0],xmm10[0],xmm0[1],xmm10[1],xmm0[2],xmm10[2],xmm0[3],xmm10[3]
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %xmm0, %xmm31
-; AVX512DQ-FAST-NEXT:    vmovdqa {{.*#+}} xmm0 = [8,9,6,7,4,5,6,7,8,9,10,11,12,13,14,15]
-; AVX512DQ-FAST-NEXT:    vpshufb %xmm0, %xmm2, %xmm15
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %xmm0, %xmm18
-; AVX512DQ-FAST-NEXT:    vinserti64x4 $1, %ymm15, %zmm12, %zmm15
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm12 = ymm5[0,1],ymm7[2],ymm5[3,4,5],ymm7[6],ymm5[7]
-; AVX512DQ-FAST-NEXT:    vmovdqa %ymm5, %ymm14
-; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm12, %xmm8
-; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} xmm8 = xmm12[0,1,2,3],xmm8[4],xmm12[5],xmm8[6],xmm12[7]
-; AVX512DQ-FAST-NEXT:    vpermd %zmm27, %zmm19, %zmm12
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm8 = ymm8[0,1,2,3,0,1,14,15,12,13,10,11,8,9],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm8[u,u,u,u,u,u,u,u]
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm12 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm12[2,3,16,17,22,23,24,25,30,31,u,u,u,u,u,u,u,u]
-; AVX512DQ-FAST-NEXT:    vpor %ymm12, %ymm8, %ymm8
-; AVX512DQ-FAST-NEXT:    vpermd %zmm17, %zmm21, %zmm12
-; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm17 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
-; AVX512DQ-FAST-NEXT:    vpternlogq $184, %zmm11, %zmm17, %zmm15
-; AVX512DQ-FAST-NEXT:    vpshufb %ymm6, %ymm12, %ymm11
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],ymm11[6,7]
+; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} xmm11 = xmm12[0,1,2],xmm11[3],xmm12[4],xmm11[5],xmm12[6,7]
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm11 = xmm11[u,u,u,u,u,u,10,11,8,9,6,7,4,5,u,u]
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm9 = ymm9[0,1,6,7,8,9,14,15,8,9,14,15,4,5,2,3,16,17,22,23,24,25,30,31,24,25,30,31,20,21,18,19]
+; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} xmm11 = xmm9[0,1,2],xmm11[3,4,5,6],xmm9[7]
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm11 = ymm11[0,1,2,3],ymm9[4,5,6,7]
+; AVX512DQ-FAST-NEXT:    vpbroadcastq {{.*#+}} ymm9 = [16,17,22,23,24,25,30,31,16,17,22,23,24,25,30,31,16,17,22,23,24,25,30,31,16,17,22,23,24,25,30,31]
+; AVX512DQ-FAST-NEXT:    vpshufb %ymm9, %ymm0, %ymm0
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm11[0,1,2,3,4,5],ymm0[6,7]
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %ymm0, %ymm28
+; AVX512DQ-FAST-NEXT:    vmovdqa 240(%rdi), %xmm15
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm3[0,1,2],ymm4[3],ymm3[4,5],ymm4[6],ymm3[7]
+; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm0, %xmm11
+; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} xmm0 = xmm11[0],xmm0[1],xmm11[2,3,4,5],xmm0[6],xmm11[7]
+; AVX512DQ-FAST-NEXT:    vmovdqa {{.*#+}} ymm11 = <2,3,0,1,14,15,12,13,10,11,128,128,128,128,128,128,128,128,u,u,u,u,u,u,u,u,u,u,u,u,u,u>
+; AVX512DQ-FAST-NEXT:    vpshufb %ymm11, %ymm0, %ymm0
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm10 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm10[0,1,6,7,8,9,18,19],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512DQ-FAST-NEXT:    vporq %ymm10, %ymm0, %ymm21
+; AVX512DQ-FAST-NEXT:    vmovdqa 160(%rdi), %ymm12
+; AVX512DQ-FAST-NEXT:    vmovdqa 128(%rdi), %ymm10
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm10[0,1],ymm12[2],ymm10[3,4,5],ymm12[6],ymm10[7]
+; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm0, %xmm14
+; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3],xmm14[4],xmm0[5],xmm14[6],xmm0[7]
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,0,1,14,15,12,13,10,11,8,9,u,u,u,u]
+; AVX512DQ-FAST-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm8 = ymm8[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,22,23,28,29]
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,6],ymm8[7]
+; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} xmm8 = xmm15[0],xmm13[1],xmm15[2,3,4,5,6,7]
+; AVX512DQ-FAST-NEXT:    vpshufb %xmm11, %xmm8, %xmm8
+; AVX512DQ-FAST-NEXT:    vinserti64x4 $1, %ymm8, %zmm0, %zmm24
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm7[2,3,4,5,10,11,16,17],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm7 = ymm4[0],ymm3[1],ymm4[2,3],ymm3[4],ymm4[5,6,7]
+; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm7, %xmm8
+; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} xmm7 = xmm7[0],xmm8[1],xmm7[2,3,4,5],xmm8[6],xmm7[7]
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm7 = ymm7[4,5,2,3,0,1,14,15,12,13],zero,zero,zero,zero,zero,zero,zero,zero,ymm7[u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-FAST-NEXT:    vpor %ymm0, %ymm7, %ymm8
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm10[0,1,2],ymm12[3],ymm10[4,5],ymm12[6],ymm10[7]
+; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm0, %xmm7
+; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} xmm7 = xmm7[0],xmm0[1],xmm7[2,3,4,5],xmm0[6],xmm7[7]
+; AVX512DQ-FAST-NEXT:    vmovdqa {{.*#+}} ymm0 = <0,1,2,3,0,1,14,15,12,13,10,11,8,9,128,128,128,128,128,128,128,128,128,128,u,u,u,u,u,u,u,u>
+; AVX512DQ-FAST-NEXT:    vpshufb %xmm0, %xmm7, %xmm7
+; AVX512DQ-FAST-NEXT:    vinserti128 $1, %xmm7, %ymm0, %ymm7
+; AVX512DQ-FAST-NEXT:    vpbroadcastq {{.*#+}} ymm11 = [2,5,2,5,2,5,2,5]
+; AVX512DQ-FAST-NEXT:    vpermd %ymm25, %ymm11, %ymm11
+; AVX512DQ-FAST-NEXT:    vpshufhw {{.*#+}} ymm11 = ymm11[0,1,2,3,4,5,4,7,8,9,10,11,12,13,12,15]
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm11 = ymm7[0,1,2,3,4,5,6],ymm11[7]
+; AVX512DQ-FAST-NEXT:    vpunpcklwd {{.*#+}} xmm7 = xmm13[0],xmm15[0],xmm13[1],xmm15[1],xmm13[2],xmm15[2],xmm13[3],xmm15[3]
+; AVX512DQ-FAST-NEXT:    vmovdqa {{.*#+}} xmm14 = [8,9,6,7,4,5,6,7,8,9,10,11,12,13,14,15]
+; AVX512DQ-FAST-NEXT:    vpshufb %xmm14, %xmm7, %xmm2
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %xmm14, %xmm31
+; AVX512DQ-FAST-NEXT:    vinserti64x4 $1, %ymm2, %zmm11, %zmm16
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm2 = ymm6[0,1,2],ymm5[3],ymm6[4,5],ymm5[6],ymm6[7]
+; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm2, %xmm11
+; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} xmm2 = xmm11[0,1,2,3],xmm2[4],xmm11[5],xmm2[6],xmm11[7]
+; AVX512DQ-FAST-NEXT:    vpermd %zmm26, %zmm27, %zmm11
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,u,u,u,u,u,12,13,10,11,8,9,6,7,u,u]
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm11 = ymm11[2,3,4,5,10,11,12,13,0,1,0,1,0,1,0,1,18,19,20,21,26,27,28,29,16,17,16,17,16,17,16,17]
+; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} xmm2 = xmm11[0,1,2],xmm2[3,4,5,6],xmm11[7]
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm11[4,5,6,7]
+; AVX512DQ-FAST-NEXT:    vpermd %zmm20, %zmm17, %zmm11
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm11 = ymm11[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,19,20,21,26,27,28,29]
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm11[6,7]
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %ymm2, %ymm29
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm2 = ymm5[0,1],ymm6[2],ymm5[3,4,5],ymm6[6],ymm5[7]
+; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm2, %xmm14
+; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1,2,3],xmm14[4],xmm2[5],xmm14[6],xmm2[7]
+; AVX512DQ-FAST-NEXT:    vmovdqa {{.*#+}} ymm14 = [10,3,6,15,12,13,6,15]
+; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} ymm27 = <1,u,u,u,5,8,12,15>
+; AVX512DQ-FAST-NEXT:    vpshufb %ymm0, %ymm2, %ymm0
+; AVX512DQ-FAST-NEXT:    vpermd %zmm26, %zmm27, %zmm2
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[2,3,16,17,22,23,24,25,30,31,u,u,u,u,u,u,u,u]
+; AVX512DQ-FAST-NEXT:    vpor %ymm2, %ymm0, %ymm0
+; AVX512DQ-FAST-NEXT:    vpermd %zmm20, %zmm19, %zmm2
+; AVX512DQ-FAST-NEXT:    vpshufb %ymm9, %ymm2, %ymm2
+; AVX512DQ-FAST-NEXT:    vpermd %zmm1, %zmm14, %zmm9
+; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm19 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
+; AVX512DQ-FAST-NEXT:    vpternlogq $184, %zmm8, %zmm19, %zmm16
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm2[6,7]
 ; AVX512DQ-FAST-NEXT:    movw $-512, %ax # imm = 0xFE00
 ; AVX512DQ-FAST-NEXT:    kmovw %eax, %k1
-; AVX512DQ-FAST-NEXT:    vinserti32x8 $1, %ymm8, %zmm0, %zmm15 {%k1}
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm8 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm0[4,5,10,11,0,1,22,23],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm11 = ymm4[0],ymm1[1],ymm4[2,3,4],ymm1[5],ymm4[6,7]
-; AVX512DQ-FAST-NEXT:    vmovdqa %ymm4, %ymm6
-; AVX512DQ-FAST-NEXT:    vmovdqa %ymm1, %ymm4
+; AVX512DQ-FAST-NEXT:    vinserti32x8 $1, %ymm0, %zmm0, %zmm16 {%k1}
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm9[4,5,10,11,0,1,22,23],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm2 = ymm4[0],ymm3[1],ymm4[2,3,4],ymm3[5],ymm4[6,7]
+; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm2, %xmm8
+; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} xmm2 = xmm8[0],xmm2[1],xmm8[2],xmm2[3],xmm8[4,5,6,7]
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm2 = ymm2[6,7,4,5,2,3,0,1,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-FAST-NEXT:    vpor %ymm0, %ymm2, %ymm8
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm12[0],ymm10[1],ymm12[2,3],ymm10[4],ymm12[5,6,7]
+; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm0, %xmm2
+; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2,3,4,5],xmm2[6],xmm0[7]
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,4,5,2,3,0,1,14,15,12,13,u,u,u,u]
+; AVX512DQ-FAST-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm2
+; AVX512DQ-FAST-NEXT:    vpermq {{.*#+}} ymm0 = ymm25[0,1,1,3]
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm9 = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,19,24,25]
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5,6],ymm9[7]
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm7 = xmm7[12,13,10,11,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-FAST-NEXT:    vinserti64x4 $1, %ymm7, %zmm2, %zmm20
+; AVX512DQ-FAST-NEXT:    vmovdqa 416(%rdi), %ymm2
+; AVX512DQ-FAST-NEXT:    vmovdqa 384(%rdi), %ymm9
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm7 = ymm9[0,1],ymm2[2],ymm9[3,4,5],ymm2[6],ymm9[7]
+; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm7, %xmm14
+; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} xmm7 = xmm7[0,1,2,3],xmm14[4],xmm7[5],xmm14[6],xmm7[7]
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm14 = ymm5[0,1,2],ymm6[3],ymm5[4,5],ymm6[6],ymm5[7]
+; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm14, %xmm11
+; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} xmm11 = xmm11[0],xmm14[1],xmm11[2,3,4,5],xmm14[6],xmm11[7]
+; AVX512DQ-FAST-NEXT:    vmovdqa {{.*#+}} ymm14 = [2,11,2,11,12,5,8,9]
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm7 = xmm7[u,u,u,u,u,u,0,1,14,15,12,13,10,11,8,9]
+; AVX512DQ-FAST-NEXT:    vinserti128 $1, %xmm7, %ymm0, %ymm7
+; AVX512DQ-FAST-NEXT:    vpermd %zmm26, %zmm14, %zmm14
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm14 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm14[0,1,22,23,28,29,18,19],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} ymm7 = ymm14[0,1,2],ymm7[3,4,5,6,7],ymm14[8,9,10],ymm7[11,12,13,14,15]
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm11 = xmm11[0,1,2,3,2,3,0,1,14,15,12,13,10,11],zero,zero
+; AVX512DQ-FAST-NEXT:    vpor %ymm14, %ymm11, %ymm11
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm7 = ymm11[0,1,2,3],ymm7[4,5,6,7]
+; AVX512DQ-FAST-NEXT:    vpternlogq $184, %zmm8, %zmm19, %zmm20
+; AVX512DQ-FAST-NEXT:    vinserti32x8 $1, %ymm7, %zmm0, %zmm20 {%k1}
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm8 = ymm4[0,1],ymm3[2],ymm4[3,4],ymm3[5],ymm4[6,7]
+; AVX512DQ-FAST-NEXT:    vmovdqa {{.*#+}} xmm7 = [8,9,4,5,4,5,6,7,8,9,10,11,12,13,14,15]
+; AVX512DQ-FAST-NEXT:    vpshufb %xmm7, %xmm8, %xmm11
+; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm8, %xmm8
+; AVX512DQ-FAST-NEXT:    vpshuflw {{.*#+}} xmm8 = xmm8[3,1,2,3,4,5,6,7]
+; AVX512DQ-FAST-NEXT:    vpunpcklwd {{.*#+}} xmm11 = xmm11[0],xmm8[0],xmm11[1],xmm8[1],xmm11[2],xmm8[2],xmm11[3],xmm8[3]
+; AVX512DQ-FAST-NEXT:    vmovdqa {{.*#+}} ymm8 = <0,3,7,10,14,u,u,u>
+; AVX512DQ-FAST-NEXT:    vpermd %zmm1, %zmm8, %zmm14
+; AVX512DQ-FAST-NEXT:    vmovdqa {{.*#+}} ymm8 = <u,u,u,u,u,u,u,u,0,1,6,7,8,9,14,15,16,17,22,23,20,21,22,23,16,17,22,23,24,25,30,31>
+; AVX512DQ-FAST-NEXT:    vpshufb %ymm8, %ymm14, %ymm14
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm11 = ymm11[0,1],ymm14[2,3,4,5,6,7]
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm10 = ymm12[0],ymm10[1],ymm12[2,3,4],ymm10[5],ymm12[6,7]
+; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm10, %xmm12
+; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} xmm10 = xmm12[0],xmm10[1],xmm12[2],xmm10[3],xmm12[4,5,6,7]
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %ymm30, %ymm12
+; AVX512DQ-FAST-NEXT:    vpshufb %ymm12, %ymm0, %ymm0
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm10 = xmm10[0,1,6,7,4,5,2,3,0,1,14,15,u,u,u,u]
+; AVX512DQ-FAST-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm10[0,1,2,3,4,5,6],ymm0[7]
+; AVX512DQ-FAST-NEXT:    vpbroadcastw 232(%rdi), %xmm10
+; AVX512DQ-FAST-NEXT:    vpsrlq $48, %xmm15, %xmm12
+; AVX512DQ-FAST-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm10[0],xmm12[0],xmm10[1],xmm12[1],xmm10[2],xmm12[2],xmm10[3],xmm12[3]
+; AVX512DQ-FAST-NEXT:    vinserti32x4 $2, %xmm10, %zmm0, %zmm10
+; AVX512DQ-FAST-NEXT:    vpternlogq $184, %zmm11, %zmm19, %zmm10
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm9[0,1,2],ymm2[3],ymm9[4,5],ymm2[6],ymm9[7]
+; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm0, %xmm11
+; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} xmm0 = xmm11[0],xmm0[1],xmm11[2,3,4,5],xmm0[6],xmm11[7]
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm11 = ymm6[0],ymm5[1],ymm6[2,3],ymm5[4],ymm6[5,6,7]
 ; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm11, %xmm12
-; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} xmm11 = xmm12[0],xmm11[1],xmm12[2],xmm11[3],xmm12[4,5,6,7]
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm11 = ymm11[6,7,4,5,2,3,0,1,14,15],zero,zero,zero,zero,zero,zero,zero,zero,ymm11[u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-FAST-NEXT:    vpor %ymm8, %ymm11, %ymm12
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm8 = ymm3[0],ymm9[1],ymm3[2,3],ymm9[4],ymm3[5,6,7]
-; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm8, %xmm11
-; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} xmm8 = xmm8[0],xmm11[1],xmm8[2,3,4,5],xmm11[6],xmm8[7]
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm8 = xmm8[0,1,4,5,2,3,0,1,14,15,12,13,u,u,u,u]
-; AVX512DQ-FAST-NEXT:    vinserti128 $1, %xmm8, %ymm0, %ymm8
-; AVX512DQ-FAST-NEXT:    vpermq {{.*#+}} ymm11 = ymm26[0,1,1,3]
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm13 = ymm11[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,19,24,25]
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5,6],ymm13[7]
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[12,13,10,11,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-FAST-NEXT:    vinserti64x4 $1, %ymm2, %zmm8, %zmm19
-; AVX512DQ-FAST-NEXT:    vmovdqa 416(%rdi), %ymm8
-; AVX512DQ-FAST-NEXT:    vmovdqa 384(%rdi), %ymm5
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm13 = ymm5[0,1],ymm8[2],ymm5[3,4,5],ymm8[6],ymm5[7]
-; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm13, %xmm0
-; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} xmm0 = xmm13[0,1,2,3],xmm0[4],xmm13[5],xmm0[6],xmm13[7]
-; AVX512DQ-FAST-NEXT:    vmovdqa %ymm14, %ymm2
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm13 = ymm14[0,1,2],ymm7[3],ymm14[4,5],ymm7[6],ymm14[7]
-; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm13, %xmm14
-; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} xmm13 = xmm14[0],xmm13[1],xmm14[2,3,4,5],xmm13[6],xmm14[7]
-; AVX512DQ-FAST-NEXT:    vmovdqa {{.*#+}} ymm14 = <0,3,7,10,14,u,u,u>
-; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} ymm28 = [2,11,2,11,12,5,8,9]
-; AVX512DQ-FAST-NEXT:    vinserti64x4 $1, %ymm16, %zmm0, %zmm21
-; AVX512DQ-FAST-NEXT:    vpermd %zmm27, %zmm28, %zmm1
-; AVX512DQ-FAST-NEXT:    vpternlogq $184, %zmm12, %zmm17, %zmm19
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,u,0,1,14,15,12,13,10,11,8,9]
-; AVX512DQ-FAST-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm1[0,1,22,23,28,29,18,19],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6,7],ymm1[8,9,10],ymm0[11,12,13,14,15]
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm12 = xmm13[0,1,2,3,2,3,0,1,14,15,12,13,10,11],zero,zero
-; AVX512DQ-FAST-NEXT:    vpor %ymm1, %ymm12, %ymm1
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
-; AVX512DQ-FAST-NEXT:    vinserti32x8 $1, %ymm0, %zmm0, %zmm19 {%k1}
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm6[0,1],ymm4[2],ymm6[3,4],ymm4[5],ymm6[6,7]
-; AVX512DQ-FAST-NEXT:    vmovdqa {{.*#+}} xmm13 = [8,9,4,5,4,5,6,7,8,9,10,11,12,13,14,15]
-; AVX512DQ-FAST-NEXT:    vpshufb %xmm13, %xmm0, %xmm1
-; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; AVX512DQ-FAST-NEXT:    vpshuflw {{.*#+}} xmm0 = xmm0[3,1,2,3,4,5,6,7]
-; AVX512DQ-FAST-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; AVX512DQ-FAST-NEXT:    vpermd %zmm29, %zmm14, %zmm1
-; AVX512DQ-FAST-NEXT:    vmovdqa {{.*#+}} ymm12 = <u,u,u,u,u,u,u,u,0,1,6,7,8,9,14,15,16,17,22,23,20,21,22,23,16,17,22,23,24,25,30,31>
-; AVX512DQ-FAST-NEXT:    vpshufb %ymm12, %ymm1, %ymm1
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %ymm12, %ymm28
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3,4,5,6,7]
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0],ymm9[1],ymm3[2,3,4],ymm9[5],ymm3[6,7]
-; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm1, %xmm3
-; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} xmm1 = xmm3[0],xmm1[1],xmm3[2],xmm1[3],xmm3[4,5,6,7]
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0,1,6,7,4,5,2,3,0,1,14,15,u,u,u,u]
-; AVX512DQ-FAST-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm3 = ymm11[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,20,21,26,27]
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5,6],ymm3[7]
-; AVX512DQ-FAST-NEXT:    vpbroadcastw 232(%rdi), %xmm3
-; AVX512DQ-FAST-NEXT:    vpsrlq $48, %xmm10, %xmm9
-; AVX512DQ-FAST-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm9[0],xmm3[1],xmm9[1],xmm3[2],xmm9[2],xmm3[3],xmm9[3]
-; AVX512DQ-FAST-NEXT:    vmovdqa {{.*#+}} ymm9 = <2,u,u,u,6,9,13,u>
-; AVX512DQ-FAST-NEXT:    vinserti32x4 $2, %xmm3, %zmm1, %zmm3
-; AVX512DQ-FAST-NEXT:    vpternlogq $184, %zmm0, %zmm17, %zmm3
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm5[0,1,2],ymm8[3],ymm5[4,5],ymm8[6],ymm5[7]
-; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2,3,4,5],xmm0[6],xmm1[7]
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm1 = ymm7[0],ymm2[1],ymm7[2,3],ymm2[4],ymm7[5,6,7]
-; AVX512DQ-FAST-NEXT:    vmovdqa %ymm2, %ymm12
-; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm1, %xmm11
-; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm11[1],xmm1[2,3,4,5],xmm11[6],xmm1[7]
-; AVX512DQ-FAST-NEXT:    vpermd %zmm27, %zmm9, %zmm9
+; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} xmm11 = xmm11[0],xmm12[1],xmm11[2,3,4,5],xmm12[6],xmm11[7]
+; AVX512DQ-FAST-NEXT:    vmovdqa {{.*#+}} ymm12 = <2,u,u,u,6,9,13,u>
+; AVX512DQ-FAST-NEXT:    vpermd %zmm26, %zmm12, %zmm12
 ; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,u,2,3,0,1,14,15,12,13,10,11]
 ; AVX512DQ-FAST-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm9 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm9[2,3,16,17,22,23,24,25],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} ymm0 = ymm9[0,1,2],ymm0[3,4,5,6,7],ymm9[8,9,10],ymm0[11,12,13,14,15]
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,2,3,0,1,14,15,12,13],zero,zero
-; AVX512DQ-FAST-NEXT:    vpor %ymm1, %ymm9, %ymm1
-; AVX512DQ-FAST-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [2,6,9,13,2,6,9,13]
-; AVX512DQ-FAST-NEXT:    # ymm2 = mem[0,1,0,1]
-; AVX512DQ-FAST-NEXT:    vpermd %zmm24, %zmm2, %zmm11
-; AVX512DQ-FAST-NEXT:    vmovdqa {{.*#+}} ymm9 = <0,4,7,11,14,u,u,u>
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
-; AVX512DQ-FAST-NEXT:    vpermd %zmm24, %zmm30, %zmm1
-; AVX512DQ-FAST-NEXT:    vinserti32x8 $1, %ymm0, %zmm0, %zmm3 {%k1}
-; AVX512DQ-FAST-NEXT:    vmovdqa {{.*#+}} ymm0 = <0,3,3,u,0,3,7,u>
-; AVX512DQ-FAST-NEXT:    vpermd %ymm26, %ymm0, %ymm0
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[u,u,u,u,u,u,u,u,u,u,0,1,6,7,8,9,u,u,u,u,u,u,u,u,u,u,16,17,22,23,24,25]
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[0,1,0,1,6,7,8,9,14,15,u,u,u,u,u,u,16,17,16,17,22,23,24,25,30,31,u,u,u,u,u,u]
-; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0,1,2,3,4],ymm0[5,6,7],ymm1[8,9,10,11,12],ymm0[13,14,15]
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %xmm31, %xmm2
-; AVX512DQ-FAST-NEXT:    vpsrld $16, %xmm31, %xmm1
-; AVX512DQ-FAST-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm1[4],xmm10[4],xmm1[5],xmm10[5],xmm1[6],xmm10[6],xmm1[7],xmm10[7]
-; AVX512DQ-FAST-NEXT:    vinserti32x4 $2, %xmm1, %zmm0, %zmm0
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm1 = ymm6[0,1],ymm4[2,3],ymm6[4,5],ymm4[6,7]
-; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm1, %xmm14
-; AVX512DQ-FAST-NEXT:    vpshufb %xmm13, %xmm14, %xmm13
-; AVX512DQ-FAST-NEXT:    vpermd %zmm29, %zmm9, %zmm9
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[10,11,6,7,4,5,6,7,u,u,u,u,u,u,u,u]
-; AVX512DQ-FAST-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm13[0],xmm1[1],xmm13[1],xmm1[2],xmm13[2],xmm1[3],xmm13[3]
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm9 = ymm9[u,u,u,u,u,u,u,u,2,3,4,5,10,11,12,13,18,19,18,19,18,19,18,19,18,19,20,21,26,27,28,29]
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm9 = ymm1[0,1],ymm9[2,3,4,5,6,7]
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm1 = ymm8[0],ymm5[1],ymm8[2,3],ymm5[4],ymm8[5,6,7]
-; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm1, %xmm13
-; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0],xmm13[1],xmm1[2,3,4,5],xmm13[6],xmm1[7]
-; AVX512DQ-FAST-NEXT:    vbroadcasti128 {{.*#+}} ymm13 = [0,4,7,0,0,4,7,0]
-; AVX512DQ-FAST-NEXT:    # ymm13 = mem[0,1,0,1]
-; AVX512DQ-FAST-NEXT:    vpermd %ymm26, %ymm13, %ymm13
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm13 = ymm13[u,u,u,u,u,u,u,u,u,u,2,3,4,5,10,11,u,u,u,u,u,u,u,u,u,u,18,19,20,21,26,27]
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm11 = ymm11[4,5,2,3,4,5,10,11,12,13,u,u,u,u,u,u,20,21,18,19,20,21,26,27,28,29,u,u,u,u,u,u]
-; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} ymm11 = ymm11[0,1,2,3,4],ymm13[5,6,7],ymm11[8,9,10,11,12],ymm13[13,14,15]
-; AVX512DQ-FAST-NEXT:    vpunpckhwd {{.*#+}} xmm10 = xmm2[4],xmm10[4],xmm2[5],xmm10[5],xmm2[6],xmm10[6],xmm2[7],xmm10[7]
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %xmm18, %xmm2
-; AVX512DQ-FAST-NEXT:    vpshufb %xmm2, %xmm10, %xmm10
-; AVX512DQ-FAST-NEXT:    vmovdqa {{.*#+}} ymm13 = <3,u,u,u,6,10,13,u>
-; AVX512DQ-FAST-NEXT:    vinserti64x4 $1, %ymm10, %zmm11, %zmm10
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm11 = ymm7[0],ymm12[1],ymm7[2,3,4],ymm12[5],ymm7[6,7]
-; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm11, %xmm14
-; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} xmm11 = xmm14[0],xmm11[1],xmm14[2],xmm11[3],xmm14[4,5,6,7]
-; AVX512DQ-FAST-NEXT:    vpermd %zmm27, %zmm13, %zmm13
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[u,u,u,u,u,u,4,5,2,3,0,1,14,15,12,13]
-; AVX512DQ-FAST-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm13 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm13[0,1,18,19,20,21,26,27],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} ymm1 = ymm13[0,1,2],ymm1[3,4,5,6,7],ymm13[8,9,10],ymm1[11,12,13,14,15]
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm11 = xmm11[0,1,2,3,6,7,4,5,2,3,0,1,14,15],zero,zero
-; AVX512DQ-FAST-NEXT:    vpor %ymm13, %ymm11, %ymm11
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm1 = ymm11[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQ-FAST-NEXT:    vmovdqa {{.*#+}} ymm11 = <1,4,8,11,15,u,u,u>
-; AVX512DQ-FAST-NEXT:    vpternlogq $226, %zmm0, %zmm17, %zmm9
-; AVX512DQ-FAST-NEXT:    vinserti32x8 $1, %ymm1, %zmm0, %zmm9 {%k1}
-; AVX512DQ-FAST-NEXT:    vpermd %zmm29, %zmm11, %zmm0
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %ymm28, %ymm1
-; AVX512DQ-FAST-NEXT:    vpshufb %ymm1, %ymm0, %ymm0
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm1 = ymm6[0,1,2],ymm4[3],ymm6[4,5],ymm4[6],ymm6[7]
-; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm1, %xmm4
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[u,u,u,u,u,u,u,u,10,11,6,7,4,5,6,7]
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm12 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm12[2,3,16,17,22,23,24,25],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} ymm0 = ymm12[0,1,2],ymm0[3,4,5,6,7],ymm12[8,9,10],ymm0[11,12,13,14,15]
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm11 = xmm11[0,1,2,3,4,5,2,3,0,1,14,15,12,13],zero,zero
+; AVX512DQ-FAST-NEXT:    vpor %ymm12, %ymm11, %ymm11
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm11 = ymm11[0,1,2,3],ymm0[4,5,6,7]
+; AVX512DQ-FAST-NEXT:    vpermd %zmm23, %zmm17, %zmm0
+; AVX512DQ-FAST-NEXT:    vpermd %zmm23, %zmm18, %zmm12
+; AVX512DQ-FAST-NEXT:    vinserti32x8 $1, %ymm11, %zmm0, %zmm10 {%k1}
+; AVX512DQ-FAST-NEXT:    vmovdqa {{.*#+}} ymm11 = <0,3,3,u,0,3,7,u>
+; AVX512DQ-FAST-NEXT:    vpermd %ymm25, %ymm11, %ymm11
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm11 = ymm11[u,u,u,u,u,u,u,u,u,u,0,1,6,7,8,9,u,u,u,u,u,u,u,u,u,u,16,17,22,23,24,25]
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm12 = ymm12[0,1,0,1,6,7,8,9,14,15,u,u,u,u,u,u,16,17,16,17,22,23,24,25,30,31,u,u,u,u,u,u]
+; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} ymm11 = ymm12[0,1,2,3,4],ymm11[5,6,7],ymm12[8,9,10,11,12],ymm11[13,14,15]
+; AVX512DQ-FAST-NEXT:    vpsrld $16, %xmm13, %xmm12
+; AVX512DQ-FAST-NEXT:    vpunpckhwd {{.*#+}} xmm12 = xmm12[4],xmm15[4],xmm12[5],xmm15[5],xmm12[6],xmm15[6],xmm12[7],xmm15[7]
+; AVX512DQ-FAST-NEXT:    vinserti32x4 $2, %xmm12, %zmm11, %zmm11
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm12 = ymm4[0,1],ymm3[2,3],ymm4[4,5],ymm3[6,7]
+; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm12, %xmm14
+; AVX512DQ-FAST-NEXT:    vpshufb %xmm7, %xmm14, %xmm7
+; AVX512DQ-FAST-NEXT:    vmovdqa {{.*#+}} ymm14 = <0,4,7,11,14,u,u,u>
+; AVX512DQ-FAST-NEXT:    vpermd %zmm1, %zmm14, %zmm14
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm12 = xmm12[10,11,6,7,4,5,6,7,u,u,u,u,u,u,u,u]
+; AVX512DQ-FAST-NEXT:    vpunpcklwd {{.*#+}} xmm7 = xmm12[0],xmm7[0],xmm12[1],xmm7[1],xmm12[2],xmm7[2],xmm12[3],xmm7[3]
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm12 = ymm14[u,u,u,u,u,u,u,u,2,3,4,5,10,11,12,13,18,19,18,19,18,19,18,19,18,19,20,21,26,27,28,29]
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm12 = ymm7[0,1],ymm12[2,3,4,5,6,7]
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm7 = ymm2[0],ymm9[1],ymm2[2,3],ymm9[4],ymm2[5,6,7]
+; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm7, %xmm14
+; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} xmm7 = xmm7[0],xmm14[1],xmm7[2,3,4,5],xmm14[6],xmm7[7]
+; AVX512DQ-FAST-NEXT:    vbroadcasti128 {{.*#+}} ymm14 = [0,4,7,0,0,4,7,0]
+; AVX512DQ-FAST-NEXT:    # ymm14 = mem[0,1,0,1]
+; AVX512DQ-FAST-NEXT:    vpermd %ymm25, %ymm14, %ymm14
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm14 = ymm14[u,u,u,u,u,u,u,u,u,u,2,3,4,5,10,11,u,u,u,u,u,u,u,u,u,u,18,19,20,21,26,27]
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[4,5,2,3,4,5,10,11,12,13,u,u,u,u,u,u,20,21,18,19,20,21,26,27,28,29,u,u,u,u,u,u]
+; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm14[5,6,7],ymm0[8,9,10,11,12],ymm14[13,14,15]
+; AVX512DQ-FAST-NEXT:    vpunpckhwd {{.*#+}} xmm13 = xmm13[4],xmm15[4],xmm13[5],xmm15[5],xmm13[6],xmm15[6],xmm13[7],xmm15[7]
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %xmm31, %xmm14
+; AVX512DQ-FAST-NEXT:    vpshufb %xmm14, %xmm13, %xmm13
+; AVX512DQ-FAST-NEXT:    vinserti64x4 $1, %ymm13, %zmm0, %zmm13
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm6[0],ymm5[1],ymm6[2,3,4],ymm5[5],ymm6[6,7]
+; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm0, %xmm14
+; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} xmm0 = xmm14[0],xmm0[1],xmm14[2],xmm0[3],xmm14[4,5,6,7]
+; AVX512DQ-FAST-NEXT:    vmovdqa {{.*#+}} ymm14 = <3,u,u,u,6,10,13,u>
+; AVX512DQ-FAST-NEXT:    vpermd %zmm26, %zmm14, %zmm14
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm7 = xmm7[u,u,u,u,u,u,4,5,2,3,0,1,14,15,12,13]
+; AVX512DQ-FAST-NEXT:    vinserti128 $1, %xmm7, %ymm0, %ymm7
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm14 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm14[0,1,18,19,20,21,26,27],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} ymm7 = ymm14[0,1,2],ymm7[3,4,5,6,7],ymm14[8,9,10],ymm7[11,12,13,14,15]
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,2,3,6,7,4,5,2,3,0,1,14,15],zero,zero
+; AVX512DQ-FAST-NEXT:    vpor %ymm0, %ymm14, %ymm0
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm7[4,5,6,7]
+; AVX512DQ-FAST-NEXT:    vpternlogq $226, %zmm11, %zmm19, %zmm12
+; AVX512DQ-FAST-NEXT:    vinserti32x8 $1, %ymm0, %zmm0, %zmm12 {%k1}
+; AVX512DQ-FAST-NEXT:    vmovdqa {{.*#+}} ymm0 = <1,4,8,11,15,u,u,u>
+; AVX512DQ-FAST-NEXT:    vinserti64x4 $1, %ymm28, %zmm0, %zmm7
+; AVX512DQ-FAST-NEXT:    vpermd %zmm1, %zmm0, %zmm0
+; AVX512DQ-FAST-NEXT:    vpshufb %ymm8, %ymm0, %ymm0
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm1 = ymm4[0,1,2],ymm3[3],ymm4[4,5],ymm3[6],ymm4[7]
+; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm1, %xmm3
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u,u,u,u,u,u,u,10,11,6,7,4,5,6,7]
 ; AVX512DQ-FAST-NEXT:    vpshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,4,6,7]
-; AVX512DQ-FAST-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm1[4],xmm4[4],xmm1[5],xmm4[5],xmm1[6],xmm4[6],xmm1[7],xmm4[7]
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1],ymm0[2,3,4,5,6,7]
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm8[0],ymm5[1],ymm8[2,3,4],ymm5[5],ymm8[6,7]
-; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm0, %xmm2
-; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} xmm0 = xmm2[0],xmm0[1],xmm2[2],xmm0[3],xmm2[4,5,6,7]
-; AVX512DQ-FAST-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,1,10,3,14,7,10,3]
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm4 = ymm7[0,1],ymm12[2],ymm7[3,4],ymm12[5],ymm7[6,7]
-; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm4, %xmm5
-; AVX512DQ-FAST-NEXT:    vpshuflw {{.*#+}} xmm5 = xmm5[0,3,1,3,4,5,6,7]
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[8,9,8,9,4,5,6,7,u,u,u,u,u,u,u,u]
-; AVX512DQ-FAST-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3]
-; AVX512DQ-FAST-NEXT:    vpermd %zmm27, %zmm2, %zmm2
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm2 = ymm2[4,5,10,11,u,u,u,u,u,u,u,u,0,1,14,15,20,21,26,27,16,17,26,27,16,17,20,21,16,17,30,31]
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0],ymm4[1,2],ymm2[3,4,5,6,7]
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,u,6,7,4,5,2,3,0,1,14,15]
-; AVX512DQ-FAST-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} ymm0 = ymm2[0,1,2],ymm0[3,4,5,6,7],ymm2[8,9,10],ymm0[11,12,13,14,15]
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
-; AVX512DQ-FAST-NEXT:    vpternlogq $226, %zmm10, %zmm17, %zmm1
-; AVX512DQ-FAST-NEXT:    vinserti32x8 $1, %ymm0, %zmm0, %zmm1 {%k1}
+; AVX512DQ-FAST-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3,4,5,6,7]
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0],ymm9[1],ymm2[2,3,4],ymm9[5],ymm2[6,7]
+; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0],xmm1[1],xmm2[2],xmm1[3],xmm2[4,5,6,7]
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm2 = ymm6[0,1],ymm5[2],ymm6[3,4],ymm5[5],ymm6[6,7]
+; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm2, %xmm3
+; AVX512DQ-FAST-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm3[0,3,1,3,4,5,6,7]
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[8,9,8,9,4,5,6,7,u,u,u,u,u,u,u,u]
+; AVX512DQ-FAST-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
+; AVX512DQ-FAST-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,1,10,3,14,7,10,3]
+; AVX512DQ-FAST-NEXT:    vinserti64x4 $1, %ymm29, %zmm0, %zmm4
+; AVX512DQ-FAST-NEXT:    vpermd %zmm26, %zmm3, %zmm3
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm3 = ymm3[4,5,10,11,u,u,u,u,u,u,u,u,0,1,14,15,20,21,26,27,16,17,26,27,16,17,20,21,16,17,30,31]
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0],ymm2[1,2],ymm3[3,4,5,6,7]
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[u,u,u,u,u,u,6,7,4,5,2,3,0,1,14,15]
+; AVX512DQ-FAST-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
+; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0,1,2],ymm1[3,4,5,6,7],ymm2[8,9,10],ymm1[11,12,13,14,15]
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512DQ-FAST-NEXT:    vpternlogq $226, %zmm13, %zmm19, %zmm0
+; AVX512DQ-FAST-NEXT:    vinserti32x8 $1, %ymm1, %zmm0, %zmm0 {%k1}
+; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVX512DQ-FAST-NEXT:    vpternlogq $184, %zmm22, %zmm1, %zmm7
+; AVX512DQ-FAST-NEXT:    vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm21, %zmm24
+; AVX512DQ-FAST-NEXT:    vpternlogq $184, %zmm24, %zmm1, %zmm4
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm7, (%rsi)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm4, (%rdx)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm16, (%rcx)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm20, (%r8)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm10, (%r9)
 ; AVX512DQ-FAST-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512DQ-FAST-NEXT:    vpternlogq $184, %zmm22, %zmm0, %zmm23
-; AVX512DQ-FAST-NEXT:    vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm20, %zmm25
-; AVX512DQ-FAST-NEXT:    vpternlogq $184, %zmm25, %zmm0, %zmm21
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm23, (%rsi)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm21, (%rdx)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm15, (%rcx)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm19, (%r8)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm3, (%r9)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm9, (%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm12, (%rax)
 ; AVX512DQ-FAST-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm1, (%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, (%rax)
 ; AVX512DQ-FAST-NEXT:    vzeroupper
 ; AVX512DQ-FAST-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-8.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-8.ll
index c7fca67c75aea54e73bb32d8b35da6c6aec13446..78065bc73c1d3f3b527841122ae42aa7c2f5f936 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-8.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-8.ll
@@ -3842,117 +3842,108 @@ define void @load_i16_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %r10
 ; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %r11
 ; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm0
-; AVX512BW-NEXT:    vmovdqa64 64(%rdi), %zmm1
-; AVX512BW-NEXT:    vmovdqa64 128(%rdi), %zmm3
+; AVX512BW-NEXT:    vmovdqa64 64(%rdi), %zmm2
+; AVX512BW-NEXT:    vmovdqa64 128(%rdi), %zmm1
 ; AVX512BW-NEXT:    vmovdqa64 192(%rdi), %zmm4
 ; AVX512BW-NEXT:    vmovdqa64 320(%rdi), %zmm5
-; AVX512BW-NEXT:    vmovdqa64 256(%rdi), %zmm2
+; AVX512BW-NEXT:    vmovdqa64 256(%rdi), %zmm3
 ; AVX512BW-NEXT:    vmovdqa64 448(%rdi), %zmm7
 ; AVX512BW-NEXT:    vmovdqa64 384(%rdi), %zmm6
 ; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm8 = [0,8,16,24,32,40,48,56,0,8,16,24,32,40,48,56,0,8,16,24,32,40,48,56,0,8,16,24,32,40,48,56]
 ; AVX512BW-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm9
 ; AVX512BW-NEXT:    vpermt2w %zmm7, %zmm8, %zmm9
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm10
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm10
 ; AVX512BW-NEXT:    vpermt2w %zmm5, %zmm8, %zmm10
 ; AVX512BW-NEXT:    movb $-64, %dil
 ; AVX512BW-NEXT:    kmovd %edi, %k1
 ; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm10 {%k1}
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm9 = [0,8,16,24,32,40,48,56,0,8,16,24,32,40,48,56]
-; AVX512BW-NEXT:    # ymm9 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vpermi2w %zmm4, %zmm3, %zmm9
-; AVX512BW-NEXT:    vpermi2w %zmm1, %zmm0, %zmm8
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm9
+; AVX512BW-NEXT:    vpermt2w %zmm4, %zmm8, %zmm9
+; AVX512BW-NEXT:    vpermi2w %zmm2, %zmm0, %zmm8
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3],ymm9[4,5,6,7]
 ; AVX512BW-NEXT:    vinserti64x4 $0, %ymm8, %zmm10, %zmm8
 ; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm9 = [1,9,17,25,33,41,49,57,1,9,17,25,33,41,49,57,1,9,17,25,33,41,49,57,1,9,17,25,33,41,49,57]
 ; AVX512BW-NEXT:    # zmm9 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm10
 ; AVX512BW-NEXT:    vpermt2w %zmm7, %zmm9, %zmm10
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm11
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm11
 ; AVX512BW-NEXT:    vpermt2w %zmm5, %zmm9, %zmm11
 ; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm11 {%k1}
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm10 = [1,9,17,25,33,41,49,57,1,9,17,25,33,41,49,57]
-; AVX512BW-NEXT:    # ymm10 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vpermi2w %zmm4, %zmm3, %zmm10
-; AVX512BW-NEXT:    vpermi2w %zmm1, %zmm0, %zmm9
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm10
+; AVX512BW-NEXT:    vpermt2w %zmm4, %zmm9, %zmm10
+; AVX512BW-NEXT:    vpermi2w %zmm2, %zmm0, %zmm9
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3],ymm10[4,5,6,7]
 ; AVX512BW-NEXT:    vinserti64x4 $0, %ymm9, %zmm11, %zmm9
 ; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm10 = [2,10,18,26,34,42,50,58,2,10,18,26,34,42,50,58,2,10,18,26,34,42,50,58,2,10,18,26,34,42,50,58]
 ; AVX512BW-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm11
 ; AVX512BW-NEXT:    vpermt2w %zmm7, %zmm10, %zmm11
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm12
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm12
 ; AVX512BW-NEXT:    vpermt2w %zmm5, %zmm10, %zmm12
 ; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm12 {%k1}
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm11 = [2,10,18,26,34,42,50,58,2,10,18,26,34,42,50,58]
-; AVX512BW-NEXT:    # ymm11 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vpermi2w %zmm4, %zmm3, %zmm11
-; AVX512BW-NEXT:    vpermi2w %zmm1, %zmm0, %zmm10
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm11
+; AVX512BW-NEXT:    vpermt2w %zmm4, %zmm10, %zmm11
+; AVX512BW-NEXT:    vpermi2w %zmm2, %zmm0, %zmm10
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm10 = ymm10[0,1,2,3],ymm11[4,5,6,7]
 ; AVX512BW-NEXT:    vinserti64x4 $0, %ymm10, %zmm12, %zmm10
 ; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm11 = [3,11,19,27,35,43,51,59,3,11,19,27,35,43,51,59,3,11,19,27,35,43,51,59,3,11,19,27,35,43,51,59]
 ; AVX512BW-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm12
 ; AVX512BW-NEXT:    vpermt2w %zmm7, %zmm11, %zmm12
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm13
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm13
 ; AVX512BW-NEXT:    vpermt2w %zmm5, %zmm11, %zmm13
 ; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm13 {%k1}
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm12 = [3,11,19,27,35,43,51,59,3,11,19,27,35,43,51,59]
-; AVX512BW-NEXT:    # ymm12 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vpermi2w %zmm4, %zmm3, %zmm12
-; AVX512BW-NEXT:    vpermi2w %zmm1, %zmm0, %zmm11
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm12
+; AVX512BW-NEXT:    vpermt2w %zmm4, %zmm11, %zmm12
+; AVX512BW-NEXT:    vpermi2w %zmm2, %zmm0, %zmm11
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm11 = ymm11[0,1,2,3],ymm12[4,5,6,7]
 ; AVX512BW-NEXT:    vinserti64x4 $0, %ymm11, %zmm13, %zmm11
 ; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm12 = [4,12,20,28,36,44,52,60,4,12,20,28,36,44,52,60,4,12,20,28,36,44,52,60,4,12,20,28,36,44,52,60]
 ; AVX512BW-NEXT:    # zmm12 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm13
 ; AVX512BW-NEXT:    vpermt2w %zmm7, %zmm12, %zmm13
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm14
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm14
 ; AVX512BW-NEXT:    vpermt2w %zmm5, %zmm12, %zmm14
 ; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm14 {%k1}
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm13 = [4,12,20,28,36,44,52,60,4,12,20,28,36,44,52,60]
-; AVX512BW-NEXT:    # ymm13 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vpermi2w %zmm4, %zmm3, %zmm13
-; AVX512BW-NEXT:    vpermi2w %zmm1, %zmm0, %zmm12
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm13
+; AVX512BW-NEXT:    vpermt2w %zmm4, %zmm12, %zmm13
+; AVX512BW-NEXT:    vpermi2w %zmm2, %zmm0, %zmm12
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm12 = ymm12[0,1,2,3],ymm13[4,5,6,7]
 ; AVX512BW-NEXT:    vinserti64x4 $0, %ymm12, %zmm14, %zmm12
 ; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm13 = [5,13,21,29,37,45,53,61,5,13,21,29,37,45,53,61,5,13,21,29,37,45,53,61,5,13,21,29,37,45,53,61]
 ; AVX512BW-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm14
 ; AVX512BW-NEXT:    vpermt2w %zmm7, %zmm13, %zmm14
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm15
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm15
 ; AVX512BW-NEXT:    vpermt2w %zmm5, %zmm13, %zmm15
 ; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm15 {%k1}
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm14 = [5,13,21,29,37,45,53,61,5,13,21,29,37,45,53,61]
-; AVX512BW-NEXT:    # ymm14 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vpermi2w %zmm4, %zmm3, %zmm14
-; AVX512BW-NEXT:    vpermi2w %zmm1, %zmm0, %zmm13
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm14
+; AVX512BW-NEXT:    vpermt2w %zmm4, %zmm13, %zmm14
+; AVX512BW-NEXT:    vpermi2w %zmm2, %zmm0, %zmm13
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm13 = ymm13[0,1,2,3],ymm14[4,5,6,7]
 ; AVX512BW-NEXT:    vinserti64x4 $0, %ymm13, %zmm15, %zmm13
 ; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm14 = [6,14,22,30,38,46,54,62,6,14,22,30,38,46,54,62,6,14,22,30,38,46,54,62,6,14,22,30,38,46,54,62]
 ; AVX512BW-NEXT:    # zmm14 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm15
 ; AVX512BW-NEXT:    vpermt2w %zmm7, %zmm14, %zmm15
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm16
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm16
 ; AVX512BW-NEXT:    vpermt2w %zmm5, %zmm14, %zmm16
 ; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm16 {%k1}
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm15 = [6,14,22,30,38,46,54,62,6,14,22,30,38,46,54,62]
-; AVX512BW-NEXT:    # ymm15 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vpermi2w %zmm4, %zmm3, %zmm15
-; AVX512BW-NEXT:    vpermi2w %zmm1, %zmm0, %zmm14
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm15
+; AVX512BW-NEXT:    vpermt2w %zmm4, %zmm14, %zmm15
+; AVX512BW-NEXT:    vpermi2w %zmm2, %zmm0, %zmm14
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm14 = ymm14[0,1,2,3],ymm15[4,5,6,7]
 ; AVX512BW-NEXT:    vinserti64x4 $0, %ymm14, %zmm16, %zmm14
 ; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm15 = [7,15,23,31,39,47,55,63,7,15,23,31,39,47,55,63,7,15,23,31,39,47,55,63,7,15,23,31,39,47,55,63]
 ; AVX512BW-NEXT:    # zmm15 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; AVX512BW-NEXT:    vpermt2w %zmm7, %zmm15, %zmm6
-; AVX512BW-NEXT:    vpermt2w %zmm5, %zmm15, %zmm2
-; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm2 {%k1}
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm5 = [7,15,23,31,39,47,55,63,7,15,23,31,39,47,55,63]
-; AVX512BW-NEXT:    # ymm5 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vpermi2w %zmm4, %zmm3, %zmm5
-; AVX512BW-NEXT:    vpermt2w %zmm1, %zmm15, %zmm0
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm2, %zmm0
+; AVX512BW-NEXT:    vpermt2w %zmm5, %zmm15, %zmm3
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm3 {%k1}
+; AVX512BW-NEXT:    vpermt2w %zmm4, %zmm15, %zmm1
+; AVX512BW-NEXT:    vpermt2w %zmm2, %zmm15, %zmm0
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm3, %zmm0
 ; AVX512BW-NEXT:    vmovdqa64 %zmm8, (%rsi)
 ; AVX512BW-NEXT:    vmovdqa64 %zmm9, (%rdx)
 ; AVX512BW-NEXT:    vmovdqa64 %zmm10, (%rcx)
@@ -8746,282 +8737,220 @@ define void @load_i16_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ;
 ; AVX512BW-LABEL: load_i16_stride8_vf64:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    subq $1096, %rsp # imm = 0x448
-; AVX512BW-NEXT:    vmovdqa64 704(%rdi), %zmm6
-; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 640(%rdi), %zmm8
-; AVX512BW-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 576(%rdi), %zmm28
-; AVX512BW-NEXT:    vmovdqa64 512(%rdi), %zmm11
-; AVX512BW-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 832(%rdi), %zmm5
-; AVX512BW-NEXT:    vmovdqa64 768(%rdi), %zmm29
-; AVX512BW-NEXT:    vmovdqa64 960(%rdi), %zmm21
-; AVX512BW-NEXT:    vmovdqa64 896(%rdi), %zmm25
-; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm7
-; AVX512BW-NEXT:    vmovdqa64 64(%rdi), %zmm17
-; AVX512BW-NEXT:    vmovdqa64 128(%rdi), %zmm4
-; AVX512BW-NEXT:    vmovdqa64 192(%rdi), %zmm9
-; AVX512BW-NEXT:    vmovdqa64 320(%rdi), %zmm20
-; AVX512BW-NEXT:    vmovdqa64 256(%rdi), %zmm26
-; AVX512BW-NEXT:    vmovdqa64 448(%rdi), %zmm22
-; AVX512BW-NEXT:    vmovdqa64 384(%rdi), %zmm2
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [0,8,16,24,32,40,48,56,0,8,16,24,32,40,48,56,0,8,16,24,32,40,48,56,0,8,16,24,32,40,48,56]
-; AVX512BW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm1
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm12
-; AVX512BW-NEXT:    vpermt2w %zmm22, %zmm0, %zmm1
-; AVX512BW-NEXT:    vmovdqa64 %zmm26, %zmm2
-; AVX512BW-NEXT:    vpermt2w %zmm20, %zmm0, %zmm2
-; AVX512BW-NEXT:    movb $-64, %al
-; AVX512BW-NEXT:    kmovd %eax, %k1
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [0,8,16,24,32,40,48,56,0,8,16,24,32,40,48,56]
-; AVX512BW-NEXT:    # ymm1 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm3
-; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm10
-; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2w %zmm9, %zmm1, %zmm3
-; AVX512BW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm4
-; AVX512BW-NEXT:    vpermt2w %zmm17, %zmm0, %zmm4
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm3, %zmm2, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm25, %zmm2
-; AVX512BW-NEXT:    vpermt2w %zmm21, %zmm0, %zmm2
-; AVX512BW-NEXT:    vmovdqa64 %zmm29, %zmm3
-; AVX512BW-NEXT:    vpermt2w %zmm5, %zmm0, %zmm3
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}
-; AVX512BW-NEXT:    vpermi2w %zmm28, %zmm11, %zmm0
-; AVX512BW-NEXT:    vpermi2w %zmm6, %zmm8, %zmm1
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm3, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm4 = [1,9,17,25,33,41,49,57,1,9,17,25,33,41,49,57,1,9,17,25,33,41,49,57,1,9,17,25,33,41,49,57]
+; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
+; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %r10
+; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %r11
+; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm0
+; AVX512BW-NEXT:    vmovdqa64 64(%rdi), %zmm29
+; AVX512BW-NEXT:    vmovdqa64 128(%rdi), %zmm1
+; AVX512BW-NEXT:    vmovdqa64 192(%rdi), %zmm30
+; AVX512BW-NEXT:    vmovdqa64 320(%rdi), %zmm31
+; AVX512BW-NEXT:    vmovdqa64 256(%rdi), %zmm3
+; AVX512BW-NEXT:    vmovdqa64 448(%rdi), %zmm7
+; AVX512BW-NEXT:    vmovdqa64 384(%rdi), %zmm6
+; AVX512BW-NEXT:    vmovdqa64 576(%rdi), %zmm9
+; AVX512BW-NEXT:    vmovdqa64 512(%rdi), %zmm5
+; AVX512BW-NEXT:    vmovdqa64 704(%rdi), %zmm12
+; AVX512BW-NEXT:    vmovdqa64 640(%rdi), %zmm2
+; AVX512BW-NEXT:    vmovdqa64 832(%rdi), %zmm14
+; AVX512BW-NEXT:    vmovdqa64 768(%rdi), %zmm11
+; AVX512BW-NEXT:    vmovdqa64 960(%rdi), %zmm16
+; AVX512BW-NEXT:    vmovdqa64 896(%rdi), %zmm15
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm13 = [0,8,16,24,32,40,48,56,0,8,16,24,32,40,48,56,0,8,16,24,32,40,48,56,0,8,16,24,32,40,48,56]
+; AVX512BW-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm17
+; AVX512BW-NEXT:    vpermt2w %zmm16, %zmm13, %zmm17
+; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm18
+; AVX512BW-NEXT:    vpermt2w %zmm14, %zmm13, %zmm18
+; AVX512BW-NEXT:    movb $-64, %dil
+; AVX512BW-NEXT:    kmovd %edi, %k1
+; AVX512BW-NEXT:    vmovdqa64 %zmm17, %zmm18 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm10
+; AVX512BW-NEXT:    vpermt2w %zmm12, %zmm13, %zmm10
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm8
+; AVX512BW-NEXT:    vpermt2w %zmm9, %zmm13, %zmm8
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3],ymm10[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm8, %zmm18, %zmm28
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm8
+; AVX512BW-NEXT:    vpermt2w %zmm7, %zmm13, %zmm8
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm10
+; AVX512BW-NEXT:    vpermt2w %zmm31, %zmm13, %zmm10
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm10 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm8
+; AVX512BW-NEXT:    vpermt2w %zmm30, %zmm13, %zmm8
+; AVX512BW-NEXT:    vpermi2w %zmm29, %zmm0, %zmm13
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm13[0,1,2,3],ymm8[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm8, %zmm10, %zmm17
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm8 = [1,9,17,25,33,41,49,57,1,9,17,25,33,41,49,57,1,9,17,25,33,41,49,57,1,9,17,25,33,41,49,57]
+; AVX512BW-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm10
+; AVX512BW-NEXT:    vpermt2w %zmm16, %zmm8, %zmm10
+; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm13
+; AVX512BW-NEXT:    vpermt2w %zmm14, %zmm8, %zmm13
+; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm13 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm10
+; AVX512BW-NEXT:    vpermt2w %zmm12, %zmm8, %zmm10
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm4
+; AVX512BW-NEXT:    vpermt2w %zmm9, %zmm8, %zmm4
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm10[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm4, %zmm13, %zmm18
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm4
+; AVX512BW-NEXT:    vpermt2w %zmm7, %zmm8, %zmm4
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm10
+; AVX512BW-NEXT:    vpermt2w %zmm31, %zmm8, %zmm10
+; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm10 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm4
+; AVX512BW-NEXT:    vpermt2w %zmm30, %zmm8, %zmm4
+; AVX512BW-NEXT:    vpermi2w %zmm29, %zmm0, %zmm8
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm8[0,1,2,3],ymm4[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm4, %zmm10, %zmm19
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm4 = [2,10,18,26,34,42,50,58,2,10,18,26,34,42,50,58,2,10,18,26,34,42,50,58,2,10,18,26,34,42,50,58]
+; AVX512BW-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm8
+; AVX512BW-NEXT:    vpermt2w %zmm16, %zmm4, %zmm8
+; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm10
+; AVX512BW-NEXT:    vpermt2w %zmm14, %zmm4, %zmm10
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm10 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm8
+; AVX512BW-NEXT:    vpermt2w %zmm12, %zmm4, %zmm8
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm13
+; AVX512BW-NEXT:    vpermt2w %zmm9, %zmm4, %zmm13
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm13[0,1,2,3],ymm8[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm8, %zmm10, %zmm20
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm8
+; AVX512BW-NEXT:    vpermt2w %zmm7, %zmm4, %zmm8
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm10
+; AVX512BW-NEXT:    vpermt2w %zmm31, %zmm4, %zmm10
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm10 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm8
+; AVX512BW-NEXT:    vpermt2w %zmm30, %zmm4, %zmm8
+; AVX512BW-NEXT:    vpermi2w %zmm29, %zmm0, %zmm4
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm8[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm4, %zmm10, %zmm21
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm4 = [3,11,19,27,35,43,51,59,3,11,19,27,35,43,51,59,3,11,19,27,35,43,51,59,3,11,19,27,35,43,51,59]
+; AVX512BW-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm8
+; AVX512BW-NEXT:    vpermt2w %zmm16, %zmm4, %zmm8
+; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm10
+; AVX512BW-NEXT:    vpermt2w %zmm14, %zmm4, %zmm10
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm10 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm8
+; AVX512BW-NEXT:    vpermt2w %zmm12, %zmm4, %zmm8
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm13
+; AVX512BW-NEXT:    vpermt2w %zmm9, %zmm4, %zmm13
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm13[0,1,2,3],ymm8[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm8, %zmm10, %zmm22
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm8
+; AVX512BW-NEXT:    vpermt2w %zmm7, %zmm4, %zmm8
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm10
+; AVX512BW-NEXT:    vpermt2w %zmm31, %zmm4, %zmm10
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm10 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm8
+; AVX512BW-NEXT:    vpermt2w %zmm30, %zmm4, %zmm8
+; AVX512BW-NEXT:    vpermi2w %zmm29, %zmm0, %zmm4
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm8[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm4, %zmm10, %zmm23
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm4 = [4,12,20,28,36,44,52,60,4,12,20,28,36,44,52,60,4,12,20,28,36,44,52,60,4,12,20,28,36,44,52,60]
+; AVX512BW-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm8
+; AVX512BW-NEXT:    vpermt2w %zmm16, %zmm4, %zmm8
+; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm10
+; AVX512BW-NEXT:    vpermt2w %zmm14, %zmm4, %zmm10
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm10 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm8
+; AVX512BW-NEXT:    vpermt2w %zmm12, %zmm4, %zmm8
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm13
+; AVX512BW-NEXT:    vpermt2w %zmm9, %zmm4, %zmm13
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm13[0,1,2,3],ymm8[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm8, %zmm10, %zmm24
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm8
+; AVX512BW-NEXT:    vpermt2w %zmm7, %zmm4, %zmm8
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm10
+; AVX512BW-NEXT:    vpermt2w %zmm31, %zmm4, %zmm10
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm10 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm8
+; AVX512BW-NEXT:    vpermt2w %zmm30, %zmm4, %zmm8
+; AVX512BW-NEXT:    vpermi2w %zmm29, %zmm0, %zmm4
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm8[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm4, %zmm10, %zmm25
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm4 = [5,13,21,29,37,45,53,61,5,13,21,29,37,45,53,61,5,13,21,29,37,45,53,61,5,13,21,29,37,45,53,61]
 ; AVX512BW-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm0
-; AVX512BW-NEXT:    vpermt2w %zmm22, %zmm4, %zmm0
-; AVX512BW-NEXT:    vmovdqa64 %zmm26, %zmm1
-; AVX512BW-NEXT:    vpermt2w %zmm20, %zmm4, %zmm1
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [1,9,17,25,33,41,49,57,1,9,17,25,33,41,49,57]
-; AVX512BW-NEXT:    # ymm2 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm0
-; AVX512BW-NEXT:    vpermt2w %zmm9, %zmm2, %zmm0
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm2
-; AVX512BW-NEXT:    vpermt2w %zmm17, %zmm4, %zmm2
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm1, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [2,10,18,26,34,42,50,58,2,10,18,26,34,42,50,58,2,10,18,26,34,42,50,58,2,10,18,26,34,42,50,58]
-; AVX512BW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm1
-; AVX512BW-NEXT:    vpermt2w %zmm22, %zmm0, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm2 = [3,11,19,27,35,43,51,59,3,11,19,27,35,43,51,59,3,11,19,27,35,43,51,59,3,11,19,27,35,43,51,59]
-; AVX512BW-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm1
-; AVX512BW-NEXT:    vpermt2w %zmm22, %zmm2, %zmm1
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm10 = [4,12,20,28,36,44,52,60,4,12,20,28,36,44,52,60,4,12,20,28,36,44,52,60,4,12,20,28,36,44,52,60]
+; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm8
+; AVX512BW-NEXT:    vpermt2w %zmm16, %zmm4, %zmm8
+; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm10
+; AVX512BW-NEXT:    vpermt2w %zmm14, %zmm4, %zmm10
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm10 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm8
+; AVX512BW-NEXT:    vpermt2w %zmm12, %zmm4, %zmm8
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm13
+; AVX512BW-NEXT:    vpermt2w %zmm9, %zmm4, %zmm13
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm13[0,1,2,3],ymm8[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm8, %zmm10, %zmm26
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm8
+; AVX512BW-NEXT:    vpermt2w %zmm7, %zmm4, %zmm8
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm10
+; AVX512BW-NEXT:    vpermt2w %zmm31, %zmm4, %zmm10
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm10 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm8
+; AVX512BW-NEXT:    vpermt2w %zmm30, %zmm4, %zmm8
+; AVX512BW-NEXT:    vpermi2w %zmm29, %zmm0, %zmm4
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm8[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm4, %zmm10, %zmm27
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm4 = [6,14,22,30,38,46,54,62,6,14,22,30,38,46,54,62,6,14,22,30,38,46,54,62,6,14,22,30,38,46,54,62]
+; AVX512BW-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm8
+; AVX512BW-NEXT:    vpermt2w %zmm16, %zmm4, %zmm8
+; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm10
+; AVX512BW-NEXT:    vpermt2w %zmm14, %zmm4, %zmm10
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm10 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm8
+; AVX512BW-NEXT:    vpermt2w %zmm12, %zmm4, %zmm8
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm13
+; AVX512BW-NEXT:    vpermt2w %zmm9, %zmm4, %zmm13
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm13[0,1,2,3],ymm8[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm8, %zmm10, %zmm8
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm10
+; AVX512BW-NEXT:    vpermt2w %zmm7, %zmm4, %zmm10
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm13
+; AVX512BW-NEXT:    vpermt2w %zmm31, %zmm4, %zmm13
+; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm13 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm10
+; AVX512BW-NEXT:    vpermt2w %zmm30, %zmm4, %zmm10
+; AVX512BW-NEXT:    vpermi2w %zmm29, %zmm0, %zmm4
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm10[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm4, %zmm13, %zmm4
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm10 = [7,15,23,31,39,47,55,63,7,15,23,31,39,47,55,63,7,15,23,31,39,47,55,63,7,15,23,31,39,47,55,63]
 ; AVX512BW-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm2
-; AVX512BW-NEXT:    vpermt2w %zmm22, %zmm10, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm15 = [5,13,21,29,37,45,53,61,5,13,21,29,37,45,53,61,5,13,21,29,37,45,53,61,5,13,21,29,37,45,53,61]
-; AVX512BW-NEXT:    # zmm15 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm1
-; AVX512BW-NEXT:    vpermt2w %zmm22, %zmm15, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm13 = [6,14,22,30,38,46,54,62,6,14,22,30,38,46,54,62,6,14,22,30,38,46,54,62,6,14,22,30,38,46,54,62]
-; AVX512BW-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm1
-; AVX512BW-NEXT:    vpermt2w %zmm22, %zmm13, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm1 = [7,15,23,31,39,47,55,63,7,15,23,31,39,47,55,63,7,15,23,31,39,47,55,63,7,15,23,31,39,47,55,63]
-; AVX512BW-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermt2w %zmm22, %zmm1, %zmm12
-; AVX512BW-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm26, %zmm24
-; AVX512BW-NEXT:    vpermt2w %zmm20, %zmm0, %zmm24
-; AVX512BW-NEXT:    vmovdqa64 %zmm26, %zmm27
-; AVX512BW-NEXT:    vpermt2w %zmm20, %zmm3, %zmm27
-; AVX512BW-NEXT:    vmovdqa64 %zmm26, %zmm31
-; AVX512BW-NEXT:    vpermt2w %zmm20, %zmm10, %zmm31
-; AVX512BW-NEXT:    vmovdqa64 %zmm26, %zmm30
-; AVX512BW-NEXT:    vpermt2w %zmm20, %zmm15, %zmm30
-; AVX512BW-NEXT:    vmovdqa64 %zmm26, %zmm22
-; AVX512BW-NEXT:    vpermt2w %zmm20, %zmm13, %zmm22
-; AVX512BW-NEXT:    vpermt2w %zmm20, %zmm1, %zmm26
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm8
-; AVX512BW-NEXT:    vpermt2w %zmm17, %zmm0, %zmm8
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm12
-; AVX512BW-NEXT:    vpermt2w %zmm17, %zmm3, %zmm12
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm6
-; AVX512BW-NEXT:    vpermt2w %zmm17, %zmm10, %zmm6
-; AVX512BW-NEXT:    vmovdqu64 %zmm6, (%rsp) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm6
-; AVX512BW-NEXT:    vpermt2w %zmm17, %zmm15, %zmm6
-; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm6
-; AVX512BW-NEXT:    vpermt2w %zmm17, %zmm13, %zmm6
-; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2w %zmm17, %zmm1, %zmm7
-; AVX512BW-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm25, %zmm18
-; AVX512BW-NEXT:    vpermt2w %zmm21, %zmm4, %zmm18
-; AVX512BW-NEXT:    vmovdqa64 %zmm25, %zmm19
-; AVX512BW-NEXT:    vpermt2w %zmm21, %zmm0, %zmm19
-; AVX512BW-NEXT:    vmovdqa64 %zmm25, %zmm23
-; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm6
-; AVX512BW-NEXT:    vpermt2w %zmm21, %zmm3, %zmm23
-; AVX512BW-NEXT:    vmovdqa64 %zmm25, %zmm16
-; AVX512BW-NEXT:    vpermt2w %zmm21, %zmm10, %zmm16
-; AVX512BW-NEXT:    vmovdqa64 %zmm25, %zmm20
-; AVX512BW-NEXT:    vpermt2w %zmm21, %zmm15, %zmm20
-; AVX512BW-NEXT:    vmovdqa64 %zmm25, %zmm17
-; AVX512BW-NEXT:    vpermt2w %zmm21, %zmm13, %zmm17
-; AVX512BW-NEXT:    vpermt2w %zmm21, %zmm1, %zmm25
-; AVX512BW-NEXT:    vmovdqa64 %zmm29, %zmm9
-; AVX512BW-NEXT:    vpermt2w %zmm5, %zmm4, %zmm9
-; AVX512BW-NEXT:    vmovdqa64 %zmm29, %zmm11
-; AVX512BW-NEXT:    vpermt2w %zmm5, %zmm0, %zmm11
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm3
-; AVX512BW-NEXT:    vmovdqa64 %zmm29, %zmm14
-; AVX512BW-NEXT:    vpermt2w %zmm5, %zmm6, %zmm14
-; AVX512BW-NEXT:    vmovdqa64 %zmm29, %zmm21
-; AVX512BW-NEXT:    vpermt2w %zmm5, %zmm10, %zmm21
-; AVX512BW-NEXT:    vmovdqa64 %zmm29, %zmm7
-; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm2
-; AVX512BW-NEXT:    vpermt2w %zmm5, %zmm15, %zmm7
-; AVX512BW-NEXT:    vmovdqa64 %zmm29, %zmm15
-; AVX512BW-NEXT:    vpermt2w %zmm5, %zmm13, %zmm15
-; AVX512BW-NEXT:    vpermt2w %zmm5, %zmm1, %zmm29
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vpermi2w %zmm28, %zmm0, %zmm4
-; AVX512BW-NEXT:    vpermi2w %zmm28, %zmm0, %zmm3
-; AVX512BW-NEXT:    vpermi2w %zmm28, %zmm0, %zmm6
-; AVX512BW-NEXT:    vpermi2w %zmm28, %zmm0, %zmm10
-; AVX512BW-NEXT:    vpermi2w %zmm28, %zmm0, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermi2w %zmm28, %zmm0, %zmm13
-; AVX512BW-NEXT:    vpermt2w %zmm28, %zmm1, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm18, %zmm9 {%k1}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm28 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = [1,9,17,25,33,41,49,57,1,9,17,25,33,41,49,57]
-; AVX512BW-NEXT:    # ymm0 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vpermi2w %zmm28, %zmm2, %zmm0
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm0[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm9, %zmm18
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm24 {%k1}
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm9 = [2,10,18,26,34,42,50,58,2,10,18,26,34,42,50,58]
-; AVX512BW-NEXT:    # ymm9 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2w %zmm5, %zmm9, %zmm0
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm8[0,1,2,3],ymm0[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm24, %zmm24
-; AVX512BW-NEXT:    vmovdqa64 %zmm19, %zmm11 {%k1}
-; AVX512BW-NEXT:    vpermi2w %zmm28, %zmm2, %zmm9
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm9[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm11, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm27 {%k1}
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [3,11,19,27,35,43,51,59,3,11,19,27,35,43,51,59]
-; AVX512BW-NEXT:    # ymm3 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm0
-; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm9
-; AVX512BW-NEXT:    vpermt2w %zmm5, %zmm3, %zmm9
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm12[0,1,2,3],ymm9[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm4, %zmm27, %zmm4
-; AVX512BW-NEXT:    vmovdqa64 %zmm23, %zmm14 {%k1}
-; AVX512BW-NEXT:    vpermi2w %zmm28, %zmm2, %zmm3
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm6[0,1,2,3],ymm3[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm3, %zmm14, %zmm3
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm6 = [4,12,20,28,36,44,52,60,4,12,20,28,36,44,52,60]
-; AVX512BW-NEXT:    # ymm6 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm8
-; AVX512BW-NEXT:    vpermt2w %zmm5, %zmm6, %zmm8
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm9 = [5,13,21,29,37,45,53,61,5,13,21,29,37,45,53,61]
-; AVX512BW-NEXT:    # ymm9 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm11
-; AVX512BW-NEXT:    vpermt2w %zmm5, %zmm9, %zmm11
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm12 = [6,14,22,30,38,46,54,62,6,14,22,30,38,46,54,62]
-; AVX512BW-NEXT:    # ymm12 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm14
-; AVX512BW-NEXT:    vpermt2w %zmm5, %zmm12, %zmm14
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} ymm19 = [7,15,23,31,39,47,55,63,7,15,23,31,39,47,55,63]
-; AVX512BW-NEXT:    # ymm19 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermt2w %zmm5, %zmm19, %zmm0
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm5
-; AVX512BW-NEXT:    vpermi2w %zmm28, %zmm2, %zmm6
-; AVX512BW-NEXT:    vpermi2w %zmm28, %zmm2, %zmm9
-; AVX512BW-NEXT:    vpermi2w %zmm28, %zmm2, %zmm12
-; AVX512BW-NEXT:    vpermt2w %zmm28, %zmm19, %zmm2
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm0
-; AVX512BW-NEXT:    vpblendd $15, (%rsp), %ymm8, %ymm8 # 32-byte Folded Reload
-; AVX512BW-NEXT:    # ymm8 = mem[0,1,2,3],ymm8[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm31 {%k1}
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm8, %zmm31, %zmm8
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm6 = ymm10[0,1,2,3],ymm6[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm21 {%k1}
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm6, %zmm21, %zmm6
-; AVX512BW-NEXT:    vpblendd $15, {{[-0-9]+}}(%r{{[sb]}}p), %ymm11, %ymm10 # 32-byte Folded Reload
-; AVX512BW-NEXT:    # ymm10 = mem[0,1,2,3],ymm11[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm30 {%k1}
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm10, %zmm30, %zmm10
-; AVX512BW-NEXT:    vpblendd $15, {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm9 # 32-byte Folded Reload
-; AVX512BW-NEXT:    # ymm9 = mem[0,1,2,3],ymm9[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqa64 %zmm20, %zmm7 {%k1}
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm9, %zmm7, %zmm7
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm22 {%k1}
-; AVX512BW-NEXT:    vpblendd $15, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm9 # 32-byte Folded Reload
-; AVX512BW-NEXT:    # ymm9 = mem[0,1,2,3],ymm14[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm9, %zmm22, %zmm9
-; AVX512BW-NEXT:    vmovdqa64 %zmm17, %zmm15 {%k1}
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm13[0,1,2,3],ymm12[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm15, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm26 {%k1}
-; AVX512BW-NEXT:    vpblendd $15, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm11 # 32-byte Folded Reload
-; AVX512BW-NEXT:    # ymm11 = mem[0,1,2,3],ymm5[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm11, %zmm26, %zmm5
-; AVX512BW-NEXT:    vmovdqa64 %zmm25, %zmm29 {%k1}
-; AVX512BW-NEXT:    vpblendd $15, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm11 # 32-byte Folded Reload
-; AVX512BW-NEXT:    # ymm11 = mem[0,1,2,3],ymm0[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm11, %zmm29, %zmm11
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm0, 64(%rsi)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm0, (%rsi)
+; AVX512BW-NEXT:    vpermt2w %zmm16, %zmm10, %zmm15
+; AVX512BW-NEXT:    vpermt2w %zmm14, %zmm10, %zmm11
+; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm11 {%k1}
+; AVX512BW-NEXT:    vpermt2w %zmm12, %zmm10, %zmm2
+; AVX512BW-NEXT:    vpermt2w %zmm9, %zmm10, %zmm5
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm5[0,1,2,3],ymm2[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm11, %zmm2
+; AVX512BW-NEXT:    vpermt2w %zmm7, %zmm10, %zmm6
+; AVX512BW-NEXT:    vpermt2w %zmm31, %zmm10, %zmm3
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm3 {%k1}
+; AVX512BW-NEXT:    vpermt2w %zmm30, %zmm10, %zmm1
+; AVX512BW-NEXT:    vpermt2w %zmm29, %zmm10, %zmm0
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm3, %zmm0
+; AVX512BW-NEXT:    vmovdqa64 %zmm28, 64(%rsi)
+; AVX512BW-NEXT:    vmovdqa64 %zmm17, (%rsi)
 ; AVX512BW-NEXT:    vmovdqa64 %zmm18, 64(%rdx)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm0, (%rdx)
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, 64(%rcx)
-; AVX512BW-NEXT:    vmovdqa64 %zmm24, (%rcx)
-; AVX512BW-NEXT:    vmovdqa64 %zmm3, 64(%r8)
-; AVX512BW-NEXT:    vmovdqa64 %zmm4, (%r8)
-; AVX512BW-NEXT:    vmovdqa64 %zmm6, 64(%r9)
-; AVX512BW-NEXT:    vmovdqa64 %zmm8, (%r9)
-; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, 64(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, (%rax)
-; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
+; AVX512BW-NEXT:    vmovdqa64 %zmm19, (%rdx)
+; AVX512BW-NEXT:    vmovdqa64 %zmm20, 64(%rcx)
+; AVX512BW-NEXT:    vmovdqa64 %zmm21, (%rcx)
+; AVX512BW-NEXT:    vmovdqa64 %zmm22, 64(%r8)
+; AVX512BW-NEXT:    vmovdqa64 %zmm23, (%r8)
+; AVX512BW-NEXT:    vmovdqa64 %zmm24, 64(%r9)
+; AVX512BW-NEXT:    vmovdqa64 %zmm25, (%r9)
+; AVX512BW-NEXT:    vmovdqa64 %zmm26, 64(%r11)
+; AVX512BW-NEXT:    vmovdqa64 %zmm27, (%r11)
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, 64(%r10)
+; AVX512BW-NEXT:    vmovdqa64 %zmm4, (%r10)
 ; AVX512BW-NEXT:    vmovdqa64 %zmm2, 64(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm9, (%rax)
-; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512BW-NEXT:    vmovdqa64 %zmm11, 64(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, (%rax)
-; AVX512BW-NEXT:    addq $1096, %rsp # imm = 0x448
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rax)
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
   %wide.vec = load <512 x i16>, ptr %in.vec, align 64
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i32-stride-4.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i32-stride-4.ll
index a08234ffb482f5aa7ab46df01dec0168c206c47e..40355e3f7686f66e81ac92e55a6f10f13291bf0a 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i32-stride-4.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i32-stride-4.ll
@@ -305,56 +305,48 @@ define void @load_i32_stride4_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX2-ONLY-LABEL: load_i32_stride4_vf8:
 ; AVX2-ONLY:       # %bb.0:
 ; AVX2-ONLY-NEXT:    vmovaps (%rdi), %ymm0
-; AVX2-ONLY-NEXT:    vmovaps 32(%rdi), %ymm4
+; AVX2-ONLY-NEXT:    vmovaps 32(%rdi), %ymm3
 ; AVX2-ONLY-NEXT:    vmovaps 64(%rdi), %ymm1
 ; AVX2-ONLY-NEXT:    vmovaps 96(%rdi), %ymm2
-; AVX2-ONLY-NEXT:    vbroadcastsd {{.*#+}} ymm3 = [0,4,0,4,0,4,0,4]
-; AVX2-ONLY-NEXT:    vpermps %ymm2, %ymm3, %ymm5
-; AVX2-ONLY-NEXT:    vpermps %ymm1, %ymm3, %ymm3
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5],ymm5[6,7]
-; AVX2-ONLY-NEXT:    vmovddup {{.*#+}} xmm5 = [0,4,0,4]
-; AVX2-ONLY-NEXT:    # xmm5 = mem[0,0]
-; AVX2-ONLY-NEXT:    vpermps %ymm4, %ymm5, %ymm6
-; AVX2-ONLY-NEXT:    vmovaps (%rdi), %xmm7
-; AVX2-ONLY-NEXT:    vmovaps 16(%rdi), %xmm8
-; AVX2-ONLY-NEXT:    vmovaps 32(%rdi), %xmm5
-; AVX2-ONLY-NEXT:    vunpcklps {{.*#+}} xmm9 = xmm7[0],xmm8[0],xmm7[1],xmm8[1]
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm6 = xmm9[0,1],xmm6[2,3]
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm3 = ymm6[0,1,2,3],ymm3[4,5,6,7]
-; AVX2-ONLY-NEXT:    vbroadcastsd {{.*#+}} ymm6 = [1,5,1,5,1,5,1,5]
-; AVX2-ONLY-NEXT:    vpermps %ymm2, %ymm6, %ymm9
-; AVX2-ONLY-NEXT:    vpermps %ymm1, %ymm6, %ymm6
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm6 = ymm6[0,1,2,3,4,5],ymm9[6,7]
-; AVX2-ONLY-NEXT:    vmovaps 48(%rdi), %xmm9
-; AVX2-ONLY-NEXT:    vunpcklps {{.*#+}} xmm10 = xmm5[0],xmm9[0],xmm5[1],xmm9[1]
-; AVX2-ONLY-NEXT:    vmovddup {{.*#+}} xmm11 = [1,5,1,5]
-; AVX2-ONLY-NEXT:    # xmm11 = mem[0,0]
-; AVX2-ONLY-NEXT:    vpermps %ymm0, %ymm11, %ymm11
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm10 = xmm11[0,1],xmm10[2,3]
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm6 = ymm10[0,1,2,3],ymm6[4,5,6,7]
-; AVX2-ONLY-NEXT:    vbroadcastsd {{.*#+}} ymm10 = [2,6,2,6,2,6,2,6]
-; AVX2-ONLY-NEXT:    vpermps %ymm2, %ymm10, %ymm11
-; AVX2-ONLY-NEXT:    vpermps %ymm1, %ymm10, %ymm10
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm10 = ymm10[0,1,2,3,4,5],ymm11[6,7]
-; AVX2-ONLY-NEXT:    vmovddup {{.*#+}} xmm11 = [2,6,2,6]
-; AVX2-ONLY-NEXT:    # xmm11 = mem[0,0]
-; AVX2-ONLY-NEXT:    vpermps %ymm4, %ymm11, %ymm4
-; AVX2-ONLY-NEXT:    vunpckhps {{.*#+}} xmm7 = xmm7[2],xmm8[2],xmm7[3],xmm8[3]
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm4 = xmm7[0,1],xmm4[2,3]
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm10[4,5,6,7]
-; AVX2-ONLY-NEXT:    vbroadcastsd {{.*#+}} ymm7 = [3,7,3,7,3,7,3,7]
-; AVX2-ONLY-NEXT:    vpermps %ymm2, %ymm7, %ymm2
-; AVX2-ONLY-NEXT:    vpermps %ymm1, %ymm7, %ymm1
+; AVX2-ONLY-NEXT:    vbroadcastsd {{.*#+}} ymm4 = [0,4,0,4,0,4,0,4]
+; AVX2-ONLY-NEXT:    vpermps %ymm2, %ymm4, %ymm5
+; AVX2-ONLY-NEXT:    vpermps %ymm1, %ymm4, %ymm6
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm5 = ymm6[0,1,2,3,4,5],ymm5[6,7]
+; AVX2-ONLY-NEXT:    vpermps %ymm3, %ymm4, %ymm4
+; AVX2-ONLY-NEXT:    vmovaps (%rdi), %xmm6
+; AVX2-ONLY-NEXT:    vmovaps 16(%rdi), %xmm7
+; AVX2-ONLY-NEXT:    vmovaps 32(%rdi), %xmm8
+; AVX2-ONLY-NEXT:    vunpcklps {{.*#+}} xmm9 = xmm6[0],xmm7[0],xmm6[1],xmm7[1]
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm4 = xmm9[0,1],xmm4[2,3]
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm5[4,5,6,7]
+; AVX2-ONLY-NEXT:    vbroadcastsd {{.*#+}} ymm5 = [1,5,1,5,1,5,1,5]
+; AVX2-ONLY-NEXT:    vpermps %ymm2, %ymm5, %ymm9
+; AVX2-ONLY-NEXT:    vpermps %ymm1, %ymm5, %ymm10
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm9 = ymm10[0,1,2,3,4,5],ymm9[6,7]
+; AVX2-ONLY-NEXT:    vmovaps 48(%rdi), %xmm10
+; AVX2-ONLY-NEXT:    vunpcklps {{.*#+}} xmm11 = xmm8[0],xmm10[0],xmm8[1],xmm10[1]
+; AVX2-ONLY-NEXT:    vpermps %ymm0, %ymm5, %ymm5
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm5 = xmm5[0,1],xmm11[2,3]
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm9[4,5,6,7]
+; AVX2-ONLY-NEXT:    vbroadcastsd {{.*#+}} ymm9 = [2,6,2,6,2,6,2,6]
+; AVX2-ONLY-NEXT:    vpermps %ymm2, %ymm9, %ymm11
+; AVX2-ONLY-NEXT:    vpermps %ymm1, %ymm9, %ymm12
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm11 = ymm12[0,1,2,3,4,5],ymm11[6,7]
+; AVX2-ONLY-NEXT:    vpermps %ymm3, %ymm9, %ymm3
+; AVX2-ONLY-NEXT:    vunpckhps {{.*#+}} xmm6 = xmm6[2],xmm7[2],xmm6[3],xmm7[3]
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm3 = xmm6[0,1],xmm3[2,3]
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm11[4,5,6,7]
+; AVX2-ONLY-NEXT:    vbroadcastsd {{.*#+}} ymm6 = [3,7,3,7,3,7,3,7]
+; AVX2-ONLY-NEXT:    vpermps %ymm2, %ymm6, %ymm2
+; AVX2-ONLY-NEXT:    vpermps %ymm1, %ymm6, %ymm1
 ; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm2[6,7]
-; AVX2-ONLY-NEXT:    vunpckhps {{.*#+}} xmm2 = xmm5[2],xmm9[2],xmm5[3],xmm9[3]
-; AVX2-ONLY-NEXT:    vmovddup {{.*#+}} xmm5 = [3,7,3,7]
-; AVX2-ONLY-NEXT:    # xmm5 = mem[0,0]
-; AVX2-ONLY-NEXT:    vpermps %ymm0, %ymm5, %ymm0
+; AVX2-ONLY-NEXT:    vunpckhps {{.*#+}} xmm2 = xmm8[2],xmm10[2],xmm8[3],xmm10[3]
+; AVX2-ONLY-NEXT:    vpermps %ymm0, %ymm6, %ymm0
 ; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3]
 ; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; AVX2-ONLY-NEXT:    vmovaps %ymm3, (%rsi)
-; AVX2-ONLY-NEXT:    vmovaps %ymm6, (%rdx)
-; AVX2-ONLY-NEXT:    vmovaps %ymm4, (%rcx)
+; AVX2-ONLY-NEXT:    vmovaps %ymm4, (%rsi)
+; AVX2-ONLY-NEXT:    vmovaps %ymm5, (%rdx)
+; AVX2-ONLY-NEXT:    vmovaps %ymm3, (%rcx)
 ; AVX2-ONLY-NEXT:    vmovaps %ymm0, (%r8)
 ; AVX2-ONLY-NEXT:    vzeroupper
 ; AVX2-ONLY-NEXT:    retq
@@ -634,38 +626,36 @@ define void @load_i32_stride4_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX2-ONLY-LABEL: load_i32_stride4_vf16:
 ; AVX2-ONLY:       # %bb.0:
 ; AVX2-ONLY-NEXT:    subq $104, %rsp
-; AVX2-ONLY-NEXT:    vmovaps (%rdi), %ymm8
+; AVX2-ONLY-NEXT:    vmovaps (%rdi), %ymm3
+; AVX2-ONLY-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-ONLY-NEXT:    vmovaps 32(%rdi), %ymm8
 ; AVX2-ONLY-NEXT:    vmovups %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-ONLY-NEXT:    vmovaps 32(%rdi), %ymm7
-; AVX2-ONLY-NEXT:    vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-ONLY-NEXT:    vmovaps 64(%rdi), %ymm4
 ; AVX2-ONLY-NEXT:    vmovaps 96(%rdi), %ymm5
-; AVX2-ONLY-NEXT:    vmovaps 160(%rdi), %ymm12
-; AVX2-ONLY-NEXT:    vmovups %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-ONLY-NEXT:    vmovaps 192(%rdi), %ymm3
+; AVX2-ONLY-NEXT:    vmovaps 160(%rdi), %ymm9
+; AVX2-ONLY-NEXT:    vmovups %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-ONLY-NEXT:    vmovaps 192(%rdi), %ymm7
 ; AVX2-ONLY-NEXT:    vmovaps 224(%rdi), %ymm2
 ; AVX2-ONLY-NEXT:    vbroadcastsd {{.*#+}} ymm0 = [0,4,0,4,0,4,0,4]
 ; AVX2-ONLY-NEXT:    vpermps %ymm2, %ymm0, %ymm1
-; AVX2-ONLY-NEXT:    vpermps %ymm3, %ymm0, %ymm6
+; AVX2-ONLY-NEXT:    vpermps %ymm7, %ymm0, %ymm6
 ; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm1 = ymm6[0,1,2,3,4,5],ymm1[6,7]
 ; AVX2-ONLY-NEXT:    vmovaps 144(%rdi), %xmm10
 ; AVX2-ONLY-NEXT:    vmovaps 128(%rdi), %xmm11
 ; AVX2-ONLY-NEXT:    vunpcklps {{.*#+}} xmm6 = xmm11[0],xmm10[0],xmm11[1],xmm10[1]
-; AVX2-ONLY-NEXT:    vmovddup {{.*#+}} xmm9 = [0,4,0,4]
-; AVX2-ONLY-NEXT:    # xmm9 = mem[0,0]
-; AVX2-ONLY-NEXT:    vpermps %ymm12, %ymm9, %ymm12
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm6 = xmm6[0,1],xmm12[2,3]
+; AVX2-ONLY-NEXT:    vpermps %ymm9, %ymm0, %ymm9
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm6 = xmm6[0,1],xmm9[2,3]
 ; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm1 = ymm6[0,1,2,3],ymm1[4,5,6,7]
 ; AVX2-ONLY-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-ONLY-NEXT:    vpermps %ymm5, %ymm0, %ymm1
-; AVX2-ONLY-NEXT:    vpermps %ymm4, %ymm0, %ymm0
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
-; AVX2-ONLY-NEXT:    vpermps %ymm7, %ymm9, %ymm1
+; AVX2-ONLY-NEXT:    vpermps %ymm4, %ymm0, %ymm6
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm1 = ymm6[0,1,2,3,4,5],ymm1[6,7]
+; AVX2-ONLY-NEXT:    vpermps %ymm8, %ymm0, %ymm0
 ; AVX2-ONLY-NEXT:    vmovaps (%rdi), %xmm12
 ; AVX2-ONLY-NEXT:    vmovaps 16(%rdi), %xmm13
 ; AVX2-ONLY-NEXT:    vunpcklps {{.*#+}} xmm6 = xmm12[0],xmm13[0],xmm12[1],xmm13[1]
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm1 = xmm6[0,1],xmm1[2,3]
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm0 = xmm6[0,1],xmm0[2,3]
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
 ; AVX2-ONLY-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-ONLY-NEXT:    vbroadcastsd {{.*#+}} ymm0 = [1,5,1,5,1,5,1,5]
 ; AVX2-ONLY-NEXT:    vpermps %ymm5, %ymm0, %ymm1
@@ -674,68 +664,62 @@ define void @load_i32_stride4_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX2-ONLY-NEXT:    vmovaps 32(%rdi), %xmm14
 ; AVX2-ONLY-NEXT:    vmovaps 48(%rdi), %xmm6
 ; AVX2-ONLY-NEXT:    vunpcklps {{.*#+}} xmm9 = xmm14[0],xmm6[0],xmm14[1],xmm6[1]
-; AVX2-ONLY-NEXT:    vmovddup {{.*#+}} xmm7 = [1,5,1,5]
-; AVX2-ONLY-NEXT:    # xmm7 = mem[0,0]
-; AVX2-ONLY-NEXT:    vpermps %ymm8, %ymm7, %ymm15
+; AVX2-ONLY-NEXT:    vpermps %ymm3, %ymm0, %ymm15
 ; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm9 = xmm15[0,1],xmm9[2,3]
 ; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm1 = ymm9[0,1,2,3],ymm1[4,5,6,7]
 ; AVX2-ONLY-NEXT:    vmovups %ymm1, (%rsp) # 32-byte Spill
 ; AVX2-ONLY-NEXT:    vpermps %ymm2, %ymm0, %ymm1
-; AVX2-ONLY-NEXT:    vpermps %ymm3, %ymm0, %ymm0
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm9 = ymm0[0,1,2,3,4,5],ymm1[6,7]
+; AVX2-ONLY-NEXT:    vpermps %ymm7, %ymm0, %ymm9
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm9 = ymm9[0,1,2,3,4,5],ymm1[6,7]
 ; AVX2-ONLY-NEXT:    vmovaps 128(%rdi), %ymm15
-; AVX2-ONLY-NEXT:    vpermps %ymm15, %ymm7, %ymm7
-; AVX2-ONLY-NEXT:    vmovaps 176(%rdi), %xmm1
-; AVX2-ONLY-NEXT:    vmovaps 160(%rdi), %xmm0
-; AVX2-ONLY-NEXT:    vunpcklps {{.*#+}} xmm8 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm7 = xmm7[0,1],xmm8[2,3]
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm7 = ymm7[0,1,2,3],ymm9[4,5,6,7]
-; AVX2-ONLY-NEXT:    vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-ONLY-NEXT:    vbroadcastsd {{.*#+}} ymm7 = [2,6,2,6,2,6,2,6]
-; AVX2-ONLY-NEXT:    vpermps %ymm2, %ymm7, %ymm8
-; AVX2-ONLY-NEXT:    vpermps %ymm3, %ymm7, %ymm9
+; AVX2-ONLY-NEXT:    vpermps %ymm15, %ymm0, %ymm0
+; AVX2-ONLY-NEXT:    vmovaps 176(%rdi), %xmm3
+; AVX2-ONLY-NEXT:    vmovaps 160(%rdi), %xmm1
+; AVX2-ONLY-NEXT:    vunpcklps {{.*#+}} xmm8 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1],xmm8[2,3]
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm9[4,5,6,7]
+; AVX2-ONLY-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-ONLY-NEXT:    vbroadcastsd {{.*#+}} ymm0 = [2,6,2,6,2,6,2,6]
+; AVX2-ONLY-NEXT:    vpermps %ymm2, %ymm0, %ymm8
+; AVX2-ONLY-NEXT:    vpermps %ymm7, %ymm0, %ymm9
 ; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm8 = ymm9[0,1,2,3,4,5],ymm8[6,7]
 ; AVX2-ONLY-NEXT:    vunpckhps {{.*#+}} xmm9 = xmm11[2],xmm10[2],xmm11[3],xmm10[3]
-; AVX2-ONLY-NEXT:    vmovddup {{.*#+}} xmm10 = [2,6,2,6]
-; AVX2-ONLY-NEXT:    # xmm10 = mem[0,0]
-; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm10, %ymm11 # 32-byte Folded Reload
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm9 = xmm9[0,1],xmm11[2,3]
+; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm10 # 32-byte Folded Reload
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm9 = xmm9[0,1],xmm10[2,3]
 ; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm8 = ymm9[0,1,2,3],ymm8[4,5,6,7]
-; AVX2-ONLY-NEXT:    vpermps %ymm5, %ymm7, %ymm9
-; AVX2-ONLY-NEXT:    vpermps %ymm4, %ymm7, %ymm7
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm7 = ymm7[0,1,2,3,4,5],ymm9[6,7]
-; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm10, %ymm9 # 32-byte Folded Reload
+; AVX2-ONLY-NEXT:    vpermps %ymm5, %ymm0, %ymm9
+; AVX2-ONLY-NEXT:    vpermps %ymm4, %ymm0, %ymm10
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm9 = ymm10[0,1,2,3,4,5],ymm9[6,7]
+; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
 ; AVX2-ONLY-NEXT:    vunpckhps {{.*#+}} xmm10 = xmm12[2],xmm13[2],xmm12[3],xmm13[3]
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm9 = xmm10[0,1],xmm9[2,3]
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm7 = ymm9[0,1,2,3],ymm7[4,5,6,7]
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm0 = xmm10[0,1],xmm0[2,3]
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm9[4,5,6,7]
 ; AVX2-ONLY-NEXT:    vbroadcastsd {{.*#+}} ymm9 = [3,7,3,7,3,7,3,7]
 ; AVX2-ONLY-NEXT:    vpermps %ymm5, %ymm9, %ymm5
 ; AVX2-ONLY-NEXT:    vpermps %ymm4, %ymm9, %ymm4
 ; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1,2,3,4,5],ymm5[6,7]
 ; AVX2-ONLY-NEXT:    vunpckhps {{.*#+}} xmm5 = xmm14[2],xmm6[2],xmm14[3],xmm6[3]
-; AVX2-ONLY-NEXT:    vmovddup {{.*#+}} xmm6 = [3,7,3,7]
-; AVX2-ONLY-NEXT:    # xmm6 = mem[0,0]
-; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm6, %ymm10 # 32-byte Folded Reload
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm5 = xmm10[0,1],xmm5[2,3]
+; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm6 # 32-byte Folded Reload
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm5 = xmm6[0,1],xmm5[2,3]
 ; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm4 = ymm5[0,1,2,3],ymm4[4,5,6,7]
 ; AVX2-ONLY-NEXT:    vpermps %ymm2, %ymm9, %ymm2
-; AVX2-ONLY-NEXT:    vpermps %ymm3, %ymm9, %ymm3
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5],ymm2[6,7]
-; AVX2-ONLY-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX2-ONLY-NEXT:    vpermps %ymm15, %ymm6, %ymm1
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
-; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX2-ONLY-NEXT:    vmovaps %ymm1, 32(%rsi)
-; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX2-ONLY-NEXT:    vmovaps %ymm1, (%rsi)
-; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX2-ONLY-NEXT:    vmovaps %ymm1, 32(%rdx)
-; AVX2-ONLY-NEXT:    vmovups (%rsp), %ymm1 # 32-byte Reload
-; AVX2-ONLY-NEXT:    vmovaps %ymm1, (%rdx)
+; AVX2-ONLY-NEXT:    vpermps %ymm7, %ymm9, %ymm5
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm2 = ymm5[0,1,2,3,4,5],ymm2[6,7]
+; AVX2-ONLY-NEXT:    vunpckhps {{.*#+}} xmm1 = xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; AVX2-ONLY-NEXT:    vpermps %ymm15, %ymm9, %ymm3
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm1 = xmm3[0,1],xmm1[2,3]
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
+; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX2-ONLY-NEXT:    vmovaps %ymm2, 32(%rsi)
+; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX2-ONLY-NEXT:    vmovaps %ymm2, (%rsi)
+; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX2-ONLY-NEXT:    vmovaps %ymm2, 32(%rdx)
+; AVX2-ONLY-NEXT:    vmovups (%rsp), %ymm2 # 32-byte Reload
+; AVX2-ONLY-NEXT:    vmovaps %ymm2, (%rdx)
 ; AVX2-ONLY-NEXT:    vmovaps %ymm8, 32(%rcx)
-; AVX2-ONLY-NEXT:    vmovaps %ymm7, (%rcx)
-; AVX2-ONLY-NEXT:    vmovaps %ymm0, 32(%r8)
+; AVX2-ONLY-NEXT:    vmovaps %ymm0, (%rcx)
+; AVX2-ONLY-NEXT:    vmovaps %ymm1, 32(%r8)
 ; AVX2-ONLY-NEXT:    vmovaps %ymm4, (%r8)
 ; AVX2-ONLY-NEXT:    addq $104, %rsp
 ; AVX2-ONLY-NEXT:    vzeroupper
@@ -1337,243 +1321,228 @@ define void @load_i32_stride4_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX2-ONLY-LABEL: load_i32_stride4_vf32:
 ; AVX2-ONLY:       # %bb.0:
 ; AVX2-ONLY-NEXT:    subq $680, %rsp # imm = 0x2A8
+; AVX2-ONLY-NEXT:    vmovaps 192(%rdi), %ymm6
 ; AVX2-ONLY-NEXT:    vmovaps 416(%rdi), %ymm8
 ; AVX2-ONLY-NEXT:    vmovups %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-ONLY-NEXT:    vmovaps 448(%rdi), %ymm5
-; AVX2-ONLY-NEXT:    vmovups %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-ONLY-NEXT:    vmovaps 480(%rdi), %ymm6
-; AVX2-ONLY-NEXT:    vmovaps 288(%rdi), %ymm4
-; AVX2-ONLY-NEXT:    vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-ONLY-NEXT:    vmovaps 320(%rdi), %ymm11
-; AVX2-ONLY-NEXT:    vmovaps 352(%rdi), %ymm2
+; AVX2-ONLY-NEXT:    vmovaps 448(%rdi), %ymm4
+; AVX2-ONLY-NEXT:    vmovaps 480(%rdi), %ymm10
+; AVX2-ONLY-NEXT:    vmovaps 288(%rdi), %ymm11
+; AVX2-ONLY-NEXT:    vmovaps 320(%rdi), %ymm3
+; AVX2-ONLY-NEXT:    vmovaps 352(%rdi), %ymm9
 ; AVX2-ONLY-NEXT:    vbroadcastsd {{.*#+}} ymm0 = [0,4,0,4,0,4,0,4]
-; AVX2-ONLY-NEXT:    vpermps %ymm2, %ymm0, %ymm1
-; AVX2-ONLY-NEXT:    vmovaps %ymm2, %ymm7
-; AVX2-ONLY-NEXT:    vpermps %ymm11, %ymm0, %ymm2
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm1[6,7]
-; AVX2-ONLY-NEXT:    vmovaps 272(%rdi), %xmm1
-; AVX2-ONLY-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX2-ONLY-NEXT:    vmovaps 256(%rdi), %xmm3
+; AVX2-ONLY-NEXT:    vpermps %ymm9, %ymm0, %ymm1
+; AVX2-ONLY-NEXT:    vpermps %ymm3, %ymm0, %ymm2
+; AVX2-ONLY-NEXT:    vmovaps %ymm3, %ymm7
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
+; AVX2-ONLY-NEXT:    vmovaps 272(%rdi), %xmm3
 ; AVX2-ONLY-NEXT:    vmovaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX2-ONLY-NEXT:    vunpcklps {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
-; AVX2-ONLY-NEXT:    vmovddup {{.*#+}} xmm1 = [0,4,0,4]
-; AVX2-ONLY-NEXT:    # xmm1 = mem[0,0]
-; AVX2-ONLY-NEXT:    vpermps %ymm4, %ymm1, %ymm4
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm3 = xmm3[0,1],xmm4[2,3]
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
-; AVX2-ONLY-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-ONLY-NEXT:    vmovaps 256(%rdi), %xmm2
+; AVX2-ONLY-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX2-ONLY-NEXT:    vunpcklps {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; AVX2-ONLY-NEXT:    vpermps %ymm11, %ymm0, %ymm3
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3]
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-ONLY-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-ONLY-NEXT:    vpermps %ymm10, %ymm0, %ymm1
+; AVX2-ONLY-NEXT:    vpermps %ymm4, %ymm0, %ymm2
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
+; AVX2-ONLY-NEXT:    vmovaps 400(%rdi), %xmm3
+; AVX2-ONLY-NEXT:    vmovaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX2-ONLY-NEXT:    vmovaps 384(%rdi), %xmm2
+; AVX2-ONLY-NEXT:    vmovaps %xmm2, (%rsp) # 16-byte Spill
+; AVX2-ONLY-NEXT:    vunpcklps {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; AVX2-ONLY-NEXT:    vpermps %ymm8, %ymm0, %ymm3
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3]
+; AVX2-ONLY-NEXT:    vmovaps 224(%rdi), %ymm8
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-ONLY-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-ONLY-NEXT:    vpermps %ymm8, %ymm0, %ymm1
 ; AVX2-ONLY-NEXT:    vpermps %ymm6, %ymm0, %ymm2
 ; AVX2-ONLY-NEXT:    vmovaps %ymm6, %ymm12
-; AVX2-ONLY-NEXT:    vpermps %ymm5, %ymm0, %ymm3
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5],ymm2[6,7]
-; AVX2-ONLY-NEXT:    vmovaps 400(%rdi), %xmm4
-; AVX2-ONLY-NEXT:    vmovaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX2-ONLY-NEXT:    vmovaps 384(%rdi), %xmm3
-; AVX2-ONLY-NEXT:    vmovaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX2-ONLY-NEXT:    vunpcklps {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
-; AVX2-ONLY-NEXT:    vpermps %ymm8, %ymm1, %ymm4
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm3 = xmm3[0,1],xmm4[2,3]
-; AVX2-ONLY-NEXT:    vmovaps 192(%rdi), %ymm9
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
-; AVX2-ONLY-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-ONLY-NEXT:    vmovaps 224(%rdi), %ymm3
-; AVX2-ONLY-NEXT:    vpermps %ymm3, %ymm0, %ymm2
-; AVX2-ONLY-NEXT:    vmovaps %ymm3, %ymm13
-; AVX2-ONLY-NEXT:    vpermps %ymm9, %ymm0, %ymm3
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5],ymm2[6,7]
+; AVX2-ONLY-NEXT:    vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
 ; AVX2-ONLY-NEXT:    vmovaps 160(%rdi), %ymm5
 ; AVX2-ONLY-NEXT:    vmovups %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-ONLY-NEXT:    vmovaps 144(%rdi), %xmm4
-; AVX2-ONLY-NEXT:    vmovaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX2-ONLY-NEXT:    vmovaps 128(%rdi), %xmm3
+; AVX2-ONLY-NEXT:    vmovaps 144(%rdi), %xmm3
 ; AVX2-ONLY-NEXT:    vmovaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX2-ONLY-NEXT:    vunpcklps {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
-; AVX2-ONLY-NEXT:    vpermps %ymm5, %ymm1, %ymm4
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm3 = xmm3[0,1],xmm4[2,3]
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
-; AVX2-ONLY-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-ONLY-NEXT:    vmovaps 128(%rdi), %xmm2
+; AVX2-ONLY-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX2-ONLY-NEXT:    vunpcklps {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; AVX2-ONLY-NEXT:    vpermps %ymm5, %ymm0, %ymm3
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3]
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-ONLY-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-ONLY-NEXT:    vmovaps 64(%rdi), %ymm14
 ; AVX2-ONLY-NEXT:    vmovaps 96(%rdi), %ymm15
-; AVX2-ONLY-NEXT:    vpermps %ymm15, %ymm0, %ymm2
-; AVX2-ONLY-NEXT:    vpermps %ymm14, %ymm0, %ymm0
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm2[6,7]
+; AVX2-ONLY-NEXT:    vpermps %ymm15, %ymm0, %ymm1
+; AVX2-ONLY-NEXT:    vpermps %ymm14, %ymm0, %ymm2
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
 ; AVX2-ONLY-NEXT:    vmovaps 32(%rdi), %ymm2
 ; AVX2-ONLY-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-ONLY-NEXT:    vpermps %ymm2, %ymm1, %ymm1
+; AVX2-ONLY-NEXT:    vpermps %ymm2, %ymm0, %ymm0
 ; AVX2-ONLY-NEXT:    vmovaps (%rdi), %xmm3
 ; AVX2-ONLY-NEXT:    vmovaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX2-ONLY-NEXT:    vmovaps 16(%rdi), %xmm2
 ; AVX2-ONLY-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX2-ONLY-NEXT:    vunpcklps {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
 ; AVX2-ONLY-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-ONLY-NEXT:    vbroadcastsd {{.*#+}} ymm1 = [1,5,1,5,1,5,1,5]
-; AVX2-ONLY-NEXT:    vpermps %ymm7, %ymm1, %ymm0
-; AVX2-ONLY-NEXT:    vmovaps %ymm7, %ymm8
-; AVX2-ONLY-NEXT:    vpermps %ymm11, %ymm1, %ymm2
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5],ymm0[6,7]
-; AVX2-ONLY-NEXT:    vmovaps 256(%rdi), %ymm3
-; AVX2-ONLY-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-ONLY-NEXT:    vmovaps 304(%rdi), %xmm4
-; AVX2-ONLY-NEXT:    vmovaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX2-ONLY-NEXT:    vbroadcastsd {{.*#+}} ymm0 = [1,5,1,5,1,5,1,5]
+; AVX2-ONLY-NEXT:    vpermps %ymm9, %ymm0, %ymm1
+; AVX2-ONLY-NEXT:    vpermps %ymm7, %ymm0, %ymm2
+; AVX2-ONLY-NEXT:    vmovaps %ymm7, %ymm6
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
+; AVX2-ONLY-NEXT:    vmovaps 256(%rdi), %ymm5
+; AVX2-ONLY-NEXT:    vmovups %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-ONLY-NEXT:    vmovaps 304(%rdi), %xmm3
+; AVX2-ONLY-NEXT:    vmovaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX2-ONLY-NEXT:    vmovaps 288(%rdi), %xmm2
-; AVX2-ONLY-NEXT:    vmovaps %xmm2, (%rsp) # 16-byte Spill
-; AVX2-ONLY-NEXT:    vunpcklps {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
-; AVX2-ONLY-NEXT:    vmovddup {{.*#+}} xmm7 = [1,5,1,5]
-; AVX2-ONLY-NEXT:    # xmm7 = mem[0,0]
-; AVX2-ONLY-NEXT:    vpermps %ymm3, %ymm7, %ymm4
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm2 = xmm4[0,1],xmm2[2,3]
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
-; AVX2-ONLY-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-ONLY-NEXT:    vpermps %ymm15, %ymm1, %ymm0
-; AVX2-ONLY-NEXT:    vpermps %ymm14, %ymm1, %ymm2
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm5 = ymm2[0,1,2,3,4,5],ymm0[6,7]
-; AVX2-ONLY-NEXT:    vmovaps (%rdi), %ymm0
-; AVX2-ONLY-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-ONLY-NEXT:    vmovaps 32(%rdi), %xmm2
 ; AVX2-ONLY-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX2-ONLY-NEXT:    vmovaps 48(%rdi), %xmm4
-; AVX2-ONLY-NEXT:    vunpcklps {{.*#+}} xmm6 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
-; AVX2-ONLY-NEXT:    vpermps %ymm0, %ymm7, %ymm10
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm6 = xmm10[0,1],xmm6[2,3]
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm6[0,1,2,3],ymm5[4,5,6,7]
-; AVX2-ONLY-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-ONLY-NEXT:    vpermps %ymm13, %ymm1, %ymm5
-; AVX2-ONLY-NEXT:    vmovaps %ymm13, %ymm10
-; AVX2-ONLY-NEXT:    vpermps %ymm9, %ymm1, %ymm6
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm6 = ymm6[0,1,2,3,4,5],ymm5[6,7]
-; AVX2-ONLY-NEXT:    vmovaps 128(%rdi), %ymm2
-; AVX2-ONLY-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-ONLY-NEXT:    vmovaps 176(%rdi), %xmm3
-; AVX2-ONLY-NEXT:    vmovaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX2-ONLY-NEXT:    vmovaps 160(%rdi), %xmm0
-; AVX2-ONLY-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX2-ONLY-NEXT:    vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
-; AVX2-ONLY-NEXT:    vpermps %ymm2, %ymm7, %ymm13
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm0 = xmm13[0,1],xmm0[2,3]
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm6[4,5,6,7]
-; AVX2-ONLY-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-ONLY-NEXT:    vmovaps %ymm12, %ymm3
+; AVX2-ONLY-NEXT:    vunpcklps {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; AVX2-ONLY-NEXT:    vpermps %ymm5, %ymm0, %ymm3
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3]
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-ONLY-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-ONLY-NEXT:    vpermps %ymm15, %ymm0, %ymm1
+; AVX2-ONLY-NEXT:    vpermps %ymm14, %ymm0, %ymm2
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
+; AVX2-ONLY-NEXT:    vmovaps (%rdi), %ymm7
+; AVX2-ONLY-NEXT:    vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-ONLY-NEXT:    vmovaps 32(%rdi), %xmm3
+; AVX2-ONLY-NEXT:    vmovaps 48(%rdi), %xmm2
+; AVX2-ONLY-NEXT:    vunpcklps {{.*#+}} xmm5 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; AVX2-ONLY-NEXT:    vpermps %ymm7, %ymm0, %ymm7
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm5 = xmm7[0,1],xmm5[2,3]
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm1 = ymm5[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-ONLY-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-ONLY-NEXT:    vpermps %ymm8, %ymm0, %ymm1
+; AVX2-ONLY-NEXT:    vpermps %ymm12, %ymm0, %ymm5
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm1 = ymm5[0,1,2,3,4,5],ymm1[6,7]
+; AVX2-ONLY-NEXT:    vmovaps 128(%rdi), %ymm12
 ; AVX2-ONLY-NEXT:    vmovups %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-ONLY-NEXT:    vpermps %ymm12, %ymm1, %ymm0
-; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-ONLY-NEXT:    vpermps %ymm2, %ymm1, %ymm1
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
+; AVX2-ONLY-NEXT:    vmovaps 176(%rdi), %xmm7
+; AVX2-ONLY-NEXT:    vmovaps %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX2-ONLY-NEXT:    vmovaps 160(%rdi), %xmm5
+; AVX2-ONLY-NEXT:    vunpcklps {{.*#+}} xmm7 = xmm5[0],xmm7[0],xmm5[1],xmm7[1]
+; AVX2-ONLY-NEXT:    vpermps %ymm12, %ymm0, %ymm13
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm7 = xmm13[0,1],xmm7[2,3]
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm1 = ymm7[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-ONLY-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-ONLY-NEXT:    vmovups %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-ONLY-NEXT:    vpermps %ymm10, %ymm0, %ymm1
+; AVX2-ONLY-NEXT:    vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-ONLY-NEXT:    vpermps %ymm4, %ymm0, %ymm7
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm13 = ymm7[0,1,2,3,4,5],ymm1[6,7]
 ; AVX2-ONLY-NEXT:    vmovaps 384(%rdi), %ymm1
 ; AVX2-ONLY-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-ONLY-NEXT:    vpermps %ymm1, %ymm7, %ymm13
-; AVX2-ONLY-NEXT:    vmovaps 432(%rdi), %xmm6
-; AVX2-ONLY-NEXT:    vmovaps 416(%rdi), %xmm5
-; AVX2-ONLY-NEXT:    vunpcklps {{.*#+}} xmm12 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm12 = xmm13[0,1],xmm12[2,3]
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm12[0,1,2,3],ymm0[4,5,6,7]
+; AVX2-ONLY-NEXT:    vpermps %ymm1, %ymm0, %ymm0
+; AVX2-ONLY-NEXT:    vmovaps 432(%rdi), %xmm7
+; AVX2-ONLY-NEXT:    vmovaps 416(%rdi), %xmm1
+; AVX2-ONLY-NEXT:    vunpcklps {{.*#+}} xmm12 = xmm1[0],xmm7[0],xmm1[1],xmm7[1]
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1],xmm12[2,3]
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm13[4,5,6,7]
 ; AVX2-ONLY-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-ONLY-NEXT:    vbroadcastsd {{.*#+}} ymm0 = [2,6,2,6,2,6,2,6]
-; AVX2-ONLY-NEXT:    vpermps %ymm8, %ymm0, %ymm12
-; AVX2-ONLY-NEXT:    vmovaps %ymm11, %ymm7
-; AVX2-ONLY-NEXT:    vpermps %ymm11, %ymm0, %ymm13
+; AVX2-ONLY-NEXT:    vpermps %ymm9, %ymm0, %ymm12
+; AVX2-ONLY-NEXT:    vpermps %ymm6, %ymm0, %ymm13
 ; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm12 = ymm13[0,1,2,3,4,5],ymm12[6,7]
-; AVX2-ONLY-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; AVX2-ONLY-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm13 # 16-byte Folded Reload
-; AVX2-ONLY-NEXT:    # xmm13 = xmm1[2],mem[2],xmm1[3],mem[3]
-; AVX2-ONLY-NEXT:    vmovddup {{.*#+}} xmm1 = [2,6,2,6]
-; AVX2-ONLY-NEXT:    # xmm1 = mem[0,0]
-; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm11 # 32-byte Folded Reload
+; AVX2-ONLY-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
+; AVX2-ONLY-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm13, %xmm13 # 16-byte Folded Reload
+; AVX2-ONLY-NEXT:    # xmm13 = xmm13[2],mem[2],xmm13[3],mem[3]
+; AVX2-ONLY-NEXT:    vpermps %ymm11, %ymm0, %ymm11
 ; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm11 = xmm13[0,1],xmm11[2,3]
 ; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm11 = ymm11[0,1,2,3],ymm12[4,5,6,7]
 ; AVX2-ONLY-NEXT:    vmovups %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-ONLY-NEXT:    vpermps %ymm3, %ymm0, %ymm11
-; AVX2-ONLY-NEXT:    vpermps %ymm2, %ymm0, %ymm12
+; AVX2-ONLY-NEXT:    vpermps %ymm10, %ymm0, %ymm11
+; AVX2-ONLY-NEXT:    vpermps %ymm4, %ymm0, %ymm12
 ; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm11 = ymm12[0,1,2,3,4,5],ymm11[6,7]
-; AVX2-ONLY-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
-; AVX2-ONLY-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2, %xmm12 # 16-byte Folded Reload
-; AVX2-ONLY-NEXT:    # xmm12 = xmm2[2],mem[2],xmm2[3],mem[3]
-; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm13 # 32-byte Folded Reload
+; AVX2-ONLY-NEXT:    vmovaps (%rsp), %xmm4 # 16-byte Reload
+; AVX2-ONLY-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4, %xmm12 # 16-byte Folded Reload
+; AVX2-ONLY-NEXT:    # xmm12 = xmm4[2],mem[2],xmm4[3],mem[3]
+; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm13 # 32-byte Folded Reload
 ; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm12 = xmm12[0,1],xmm13[2,3]
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm2 = ymm12[0,1,2,3],ymm11[4,5,6,7]
-; AVX2-ONLY-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-ONLY-NEXT:    vpermps %ymm10, %ymm0, %ymm11
-; AVX2-ONLY-NEXT:    vmovaps %ymm9, %ymm3
-; AVX2-ONLY-NEXT:    vpermps %ymm9, %ymm0, %ymm12
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm4 = ymm12[0,1,2,3],ymm11[4,5,6,7]
+; AVX2-ONLY-NEXT:    vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-ONLY-NEXT:    vpermps %ymm8, %ymm0, %ymm11
+; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
+; AVX2-ONLY-NEXT:    vpermps %ymm4, %ymm0, %ymm12
 ; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm11 = ymm12[0,1,2,3,4,5],ymm11[6,7]
-; AVX2-ONLY-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
-; AVX2-ONLY-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2, %xmm12 # 16-byte Folded Reload
-; AVX2-ONLY-NEXT:    # xmm12 = xmm2[2],mem[2],xmm2[3],mem[3]
-; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm13 # 32-byte Folded Reload
+; AVX2-ONLY-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
+; AVX2-ONLY-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm10, %xmm12 # 16-byte Folded Reload
+; AVX2-ONLY-NEXT:    # xmm12 = xmm10[2],mem[2],xmm10[3],mem[3]
+; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm13 # 32-byte Folded Reload
 ; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm12 = xmm12[0,1],xmm13[2,3]
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm12 = ymm12[0,1,2,3],ymm11[4,5,6,7]
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm10 = ymm12[0,1,2,3],ymm11[4,5,6,7]
+; AVX2-ONLY-NEXT:    vmovups %ymm10, (%rsp) # 32-byte Spill
 ; AVX2-ONLY-NEXT:    vpermps %ymm15, %ymm0, %ymm11
-; AVX2-ONLY-NEXT:    vpermps %ymm14, %ymm0, %ymm0
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm11[6,7]
-; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
-; AVX2-ONLY-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
-; AVX2-ONLY-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2, %xmm11 # 16-byte Folded Reload
-; AVX2-ONLY-NEXT:    # xmm11 = xmm2[2],mem[2],xmm2[3],mem[3]
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm1 = xmm11[0,1],xmm1[2,3]
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm13 = ymm1[0,1,2,3],ymm0[4,5,6,7]
-; AVX2-ONLY-NEXT:    vbroadcastsd {{.*#+}} ymm1 = [3,7,3,7,3,7,3,7]
-; AVX2-ONLY-NEXT:    vpermps %ymm15, %ymm1, %ymm0
-; AVX2-ONLY-NEXT:    vpermps %ymm14, %ymm1, %ymm11
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm11[0,1,2,3,4,5],ymm0[6,7]
-; AVX2-ONLY-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
-; AVX2-ONLY-NEXT:    vunpckhps {{.*#+}} xmm2 = xmm2[2],xmm4[2],xmm2[3],xmm4[3]
-; AVX2-ONLY-NEXT:    vmovddup {{.*#+}} xmm4 = [3,7,3,7]
-; AVX2-ONLY-NEXT:    # xmm4 = mem[0,0]
-; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm9 # 32-byte Folded Reload
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm2 = xmm9[0,1],xmm2[2,3]
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
-; AVX2-ONLY-NEXT:    vpermps %ymm8, %ymm1, %ymm2
-; AVX2-ONLY-NEXT:    vpermps %ymm7, %ymm1, %ymm9
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm2 = ymm9[0,1,2,3,4,5],ymm2[6,7]
-; AVX2-ONLY-NEXT:    vmovaps (%rsp), %xmm7 # 16-byte Reload
-; AVX2-ONLY-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm7, %xmm9 # 16-byte Folded Reload
-; AVX2-ONLY-NEXT:    # xmm9 = xmm7[2],mem[2],xmm7[3],mem[3]
-; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm11 # 32-byte Folded Reload
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm9 = xmm11[0,1],xmm9[2,3]
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm2 = ymm9[0,1,2,3],ymm2[4,5,6,7]
-; AVX2-ONLY-NEXT:    vpermps %ymm10, %ymm1, %ymm8
-; AVX2-ONLY-NEXT:    vpermps %ymm3, %ymm1, %ymm7
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm7 = ymm7[0,1,2,3,4,5],ymm8[6,7]
-; AVX2-ONLY-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
-; AVX2-ONLY-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3, %xmm8 # 16-byte Folded Reload
-; AVX2-ONLY-NEXT:    # xmm8 = xmm3[2],mem[2],xmm3[3],mem[3]
-; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm9 # 32-byte Folded Reload
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm8 = xmm9[0,1],xmm8[2,3]
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm7 = ymm8[0,1,2,3],ymm7[4,5,6,7]
-; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm8 # 32-byte Folded Reload
-; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm8[6,7]
-; AVX2-ONLY-NEXT:    vunpckhps {{.*#+}} xmm3 = xmm5[2],xmm6[2],xmm5[3],xmm6[3]
-; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm4 # 32-byte Folded Reload
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm3 = xmm4[0,1],xmm3[2,3]
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
-; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
-; AVX2-ONLY-NEXT:    vmovaps %ymm3, 32(%rsi)
-; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
-; AVX2-ONLY-NEXT:    vmovaps %ymm3, 96(%rsi)
-; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
-; AVX2-ONLY-NEXT:    vmovaps %ymm3, 64(%rsi)
-; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
-; AVX2-ONLY-NEXT:    vmovaps %ymm3, (%rsi)
-; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
-; AVX2-ONLY-NEXT:    vmovaps %ymm3, 96(%rdx)
-; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
-; AVX2-ONLY-NEXT:    vmovaps %ymm3, 32(%rdx)
-; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
-; AVX2-ONLY-NEXT:    vmovaps %ymm3, (%rdx)
-; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
-; AVX2-ONLY-NEXT:    vmovaps %ymm3, 64(%rdx)
-; AVX2-ONLY-NEXT:    vmovaps %ymm12, 32(%rcx)
-; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
-; AVX2-ONLY-NEXT:    vmovaps %ymm3, 96(%rcx)
-; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
-; AVX2-ONLY-NEXT:    vmovaps %ymm3, 64(%rcx)
+; AVX2-ONLY-NEXT:    vpermps %ymm14, %ymm0, %ymm12
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm11 = ymm12[0,1,2,3,4,5],ymm11[6,7]
+; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; AVX2-ONLY-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
+; AVX2-ONLY-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm10, %xmm12 # 16-byte Folded Reload
+; AVX2-ONLY-NEXT:    # xmm12 = xmm10[2],mem[2],xmm10[3],mem[3]
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm0 = xmm12[0,1],xmm0[2,3]
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm13 = ymm0[0,1,2,3],ymm11[4,5,6,7]
+; AVX2-ONLY-NEXT:    vbroadcastsd {{.*#+}} ymm0 = [3,7,3,7,3,7,3,7]
+; AVX2-ONLY-NEXT:    vpermps %ymm15, %ymm0, %ymm11
+; AVX2-ONLY-NEXT:    vpermps %ymm14, %ymm0, %ymm12
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm11 = ymm12[0,1,2,3,4,5],ymm11[6,7]
+; AVX2-ONLY-NEXT:    vunpckhps {{.*#+}} xmm2 = xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm3 # 32-byte Folded Reload
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3]
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm11[4,5,6,7]
+; AVX2-ONLY-NEXT:    vpermps %ymm9, %ymm0, %ymm3
+; AVX2-ONLY-NEXT:    vpermps %ymm6, %ymm0, %ymm6
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm3 = ymm6[0,1,2,3,4,5],ymm3[6,7]
+; AVX2-ONLY-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
+; AVX2-ONLY-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm6, %xmm6 # 16-byte Folded Reload
+; AVX2-ONLY-NEXT:    # xmm6 = xmm6[2],mem[2],xmm6[3],mem[3]
+; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm9 # 32-byte Folded Reload
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm6 = xmm9[0,1],xmm6[2,3]
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm3 = ymm6[0,1,2,3],ymm3[4,5,6,7]
+; AVX2-ONLY-NEXT:    vpermps %ymm8, %ymm0, %ymm6
+; AVX2-ONLY-NEXT:    vpermps %ymm4, %ymm0, %ymm8
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm6 = ymm8[0,1,2,3,4,5],ymm6[6,7]
+; AVX2-ONLY-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm5, %xmm5 # 16-byte Folded Reload
+; AVX2-ONLY-NEXT:    # xmm5 = xmm5[2],mem[2],xmm5[3],mem[3]
+; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm8 # 32-byte Folded Reload
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm5 = xmm8[0,1],xmm5[2,3]
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm6[4,5,6,7]
+; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm6 # 32-byte Folded Reload
+; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm8 # 32-byte Folded Reload
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm6 = ymm8[0,1,2,3,4,5],ymm6[6,7]
+; AVX2-ONLY-NEXT:    vunpckhps {{.*#+}} xmm1 = xmm1[2],xmm7[2],xmm1[3],xmm7[3]
+; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm6[4,5,6,7]
+; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-ONLY-NEXT:    vmovaps %ymm1, 32(%rsi)
+; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-ONLY-NEXT:    vmovaps %ymm1, 96(%rsi)
+; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-ONLY-NEXT:    vmovaps %ymm1, 64(%rsi)
+; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-ONLY-NEXT:    vmovaps %ymm1, (%rsi)
+; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-ONLY-NEXT:    vmovaps %ymm1, 96(%rdx)
+; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-ONLY-NEXT:    vmovaps %ymm1, 32(%rdx)
+; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-ONLY-NEXT:    vmovaps %ymm1, (%rdx)
+; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-ONLY-NEXT:    vmovaps %ymm1, 64(%rdx)
+; AVX2-ONLY-NEXT:    vmovups (%rsp), %ymm1 # 32-byte Reload
+; AVX2-ONLY-NEXT:    vmovaps %ymm1, 32(%rcx)
+; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-ONLY-NEXT:    vmovaps %ymm1, 96(%rcx)
+; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-ONLY-NEXT:    vmovaps %ymm1, 64(%rcx)
 ; AVX2-ONLY-NEXT:    vmovaps %ymm13, (%rcx)
-; AVX2-ONLY-NEXT:    vmovaps %ymm1, 96(%r8)
-; AVX2-ONLY-NEXT:    vmovaps %ymm7, 32(%r8)
-; AVX2-ONLY-NEXT:    vmovaps %ymm2, 64(%r8)
-; AVX2-ONLY-NEXT:    vmovaps %ymm0, (%r8)
+; AVX2-ONLY-NEXT:    vmovaps %ymm0, 96(%r8)
+; AVX2-ONLY-NEXT:    vmovaps %ymm5, 32(%r8)
+; AVX2-ONLY-NEXT:    vmovaps %ymm3, 64(%r8)
+; AVX2-ONLY-NEXT:    vmovaps %ymm2, (%r8)
 ; AVX2-ONLY-NEXT:    addq $680, %rsp # imm = 0x2A8
 ; AVX2-ONLY-NEXT:    vzeroupper
 ; AVX2-ONLY-NEXT:    retq
@@ -2812,497 +2781,494 @@ define void @load_i32_stride4_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ;
 ; AVX2-ONLY-LABEL: load_i32_stride4_vf64:
 ; AVX2-ONLY:       # %bb.0:
-; AVX2-ONLY-NEXT:    subq $1992, %rsp # imm = 0x7C8
+; AVX2-ONLY-NEXT:    subq $1944, %rsp # imm = 0x798
+; AVX2-ONLY-NEXT:    vmovaps 704(%rdi), %ymm7
 ; AVX2-ONLY-NEXT:    vmovaps 416(%rdi), %ymm8
 ; AVX2-ONLY-NEXT:    vmovups %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-ONLY-NEXT:    vmovaps 448(%rdi), %ymm14
+; AVX2-ONLY-NEXT:    vmovaps 448(%rdi), %ymm4
 ; AVX2-ONLY-NEXT:    vmovaps 480(%rdi), %ymm15
-; AVX2-ONLY-NEXT:    vmovaps 160(%rdi), %ymm4
-; AVX2-ONLY-NEXT:    vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-ONLY-NEXT:    vmovaps 192(%rdi), %ymm3
-; AVX2-ONLY-NEXT:    vmovaps 224(%rdi), %ymm10
+; AVX2-ONLY-NEXT:    vmovaps 160(%rdi), %ymm9
+; AVX2-ONLY-NEXT:    vmovups %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-ONLY-NEXT:    vmovaps 192(%rdi), %ymm10
+; AVX2-ONLY-NEXT:    vmovaps 224(%rdi), %ymm14
 ; AVX2-ONLY-NEXT:    vbroadcastsd {{.*#+}} ymm2 = [0,4,0,4,0,4,0,4]
-; AVX2-ONLY-NEXT:    vpermps %ymm10, %ymm2, %ymm0
-; AVX2-ONLY-NEXT:    vpermps %ymm3, %ymm2, %ymm1
-; AVX2-ONLY-NEXT:    vmovaps %ymm3, %ymm13
-; AVX2-ONLY-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-ONLY-NEXT:    vpermps %ymm14, %ymm2, %ymm0
+; AVX2-ONLY-NEXT:    vpermps %ymm10, %ymm2, %ymm1
+; AVX2-ONLY-NEXT:    vmovups %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
 ; AVX2-ONLY-NEXT:    vmovaps 144(%rdi), %xmm3
 ; AVX2-ONLY-NEXT:    vmovaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX2-ONLY-NEXT:    vmovaps 128(%rdi), %xmm1
 ; AVX2-ONLY-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX2-ONLY-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
-; AVX2-ONLY-NEXT:    vmovddup {{.*#+}} xmm3 = [0,4,0,4]
-; AVX2-ONLY-NEXT:    # xmm3 = mem[0,0]
-; AVX2-ONLY-NEXT:    vpermps %ymm4, %ymm3, %ymm4
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm4[2,3]
+; AVX2-ONLY-NEXT:    vpermps %ymm9, %ymm2, %ymm3
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3]
 ; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
 ; AVX2-ONLY-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-ONLY-NEXT:    vpermps %ymm15, %ymm2, %ymm0
-; AVX2-ONLY-NEXT:    vpermps %ymm14, %ymm2, %ymm1
-; AVX2-ONLY-NEXT:    vmovups %ymm14, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-ONLY-NEXT:    vpermps %ymm4, %ymm2, %ymm1
+; AVX2-ONLY-NEXT:    vmovaps %ymm4, %ymm9
 ; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
-; AVX2-ONLY-NEXT:    vmovaps 400(%rdi), %xmm4
-; AVX2-ONLY-NEXT:    vmovaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX2-ONLY-NEXT:    vmovaps 400(%rdi), %xmm3
+; AVX2-ONLY-NEXT:    vmovaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX2-ONLY-NEXT:    vmovaps 384(%rdi), %xmm1
 ; AVX2-ONLY-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX2-ONLY-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
-; AVX2-ONLY-NEXT:    vpermps %ymm8, %ymm3, %ymm4
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm4[2,3]
-; AVX2-ONLY-NEXT:    vmovaps 704(%rdi), %ymm12
+; AVX2-ONLY-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
+; AVX2-ONLY-NEXT:    vpermps %ymm8, %ymm2, %ymm3
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3]
+; AVX2-ONLY-NEXT:    vmovaps 736(%rdi), %ymm3
 ; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
 ; AVX2-ONLY-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-ONLY-NEXT:    vmovaps 736(%rdi), %ymm0
-; AVX2-ONLY-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-ONLY-NEXT:    vpermps %ymm0, %ymm2, %ymm0
-; AVX2-ONLY-NEXT:    vpermps %ymm12, %ymm2, %ymm1
-; AVX2-ONLY-NEXT:    vmovups %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-ONLY-NEXT:    vpermps %ymm3, %ymm2, %ymm0
+; AVX2-ONLY-NEXT:    vmovaps %ymm3, %ymm11
+; AVX2-ONLY-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-ONLY-NEXT:    vpermps %ymm7, %ymm2, %ymm1
+; AVX2-ONLY-NEXT:    vmovaps %ymm7, %ymm13
+; AVX2-ONLY-NEXT:    vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
-; AVX2-ONLY-NEXT:    vmovaps 672(%rdi), %ymm5
-; AVX2-ONLY-NEXT:    vmovups %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-ONLY-NEXT:    vmovaps 656(%rdi), %xmm4
-; AVX2-ONLY-NEXT:    vmovaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX2-ONLY-NEXT:    vmovaps 672(%rdi), %ymm4
+; AVX2-ONLY-NEXT:    vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-ONLY-NEXT:    vmovaps 656(%rdi), %xmm3
+; AVX2-ONLY-NEXT:    vmovaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX2-ONLY-NEXT:    vmovaps 640(%rdi), %xmm1
 ; AVX2-ONLY-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX2-ONLY-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
-; AVX2-ONLY-NEXT:    vpermps %ymm5, %ymm3, %ymm4
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm4[2,3]
+; AVX2-ONLY-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
+; AVX2-ONLY-NEXT:    vpermps %ymm4, %ymm2, %ymm3
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3]
 ; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
 ; AVX2-ONLY-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-ONLY-NEXT:    vmovaps 960(%rdi), %ymm4
-; AVX2-ONLY-NEXT:    vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-ONLY-NEXT:    vmovaps 992(%rdi), %ymm11
-; AVX2-ONLY-NEXT:    vpermps %ymm11, %ymm2, %ymm0
-; AVX2-ONLY-NEXT:    vpermps %ymm4, %ymm2, %ymm1
+; AVX2-ONLY-NEXT:    vmovaps 960(%rdi), %ymm3
+; AVX2-ONLY-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-ONLY-NEXT:    vmovaps 992(%rdi), %ymm1
+; AVX2-ONLY-NEXT:    vpermps %ymm1, %ymm2, %ymm0
+; AVX2-ONLY-NEXT:    vmovaps %ymm1, %ymm8
+; AVX2-ONLY-NEXT:    vpermps %ymm3, %ymm2, %ymm1
 ; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
-; AVX2-ONLY-NEXT:    vmovaps 928(%rdi), %ymm5
-; AVX2-ONLY-NEXT:    vmovups %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-ONLY-NEXT:    vmovaps 912(%rdi), %xmm4
-; AVX2-ONLY-NEXT:    vmovaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX2-ONLY-NEXT:    vmovaps 928(%rdi), %ymm4
+; AVX2-ONLY-NEXT:    vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-ONLY-NEXT:    vmovaps 912(%rdi), %xmm3
+; AVX2-ONLY-NEXT:    vmovaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX2-ONLY-NEXT:    vmovaps 896(%rdi), %xmm1
 ; AVX2-ONLY-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX2-ONLY-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
-; AVX2-ONLY-NEXT:    vpermps %ymm5, %ymm3, %ymm4
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm4[2,3]
+; AVX2-ONLY-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
+; AVX2-ONLY-NEXT:    vpermps %ymm4, %ymm2, %ymm3
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3]
 ; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
 ; AVX2-ONLY-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-ONLY-NEXT:    vmovaps 320(%rdi), %ymm4
-; AVX2-ONLY-NEXT:    vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-ONLY-NEXT:    vmovaps 320(%rdi), %ymm3
+; AVX2-ONLY-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-ONLY-NEXT:    vmovaps 352(%rdi), %ymm0
 ; AVX2-ONLY-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-ONLY-NEXT:    vpermps %ymm0, %ymm2, %ymm0
-; AVX2-ONLY-NEXT:    vpermps %ymm4, %ymm2, %ymm1
+; AVX2-ONLY-NEXT:    vpermps %ymm3, %ymm2, %ymm1
 ; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
-; AVX2-ONLY-NEXT:    vmovaps 288(%rdi), %ymm5
-; AVX2-ONLY-NEXT:    vmovups %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-ONLY-NEXT:    vmovaps 272(%rdi), %xmm4
-; AVX2-ONLY-NEXT:    vmovaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX2-ONLY-NEXT:    vmovaps 288(%rdi), %ymm4
+; AVX2-ONLY-NEXT:    vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-ONLY-NEXT:    vmovaps 272(%rdi), %xmm3
+; AVX2-ONLY-NEXT:    vmovaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX2-ONLY-NEXT:    vmovaps 256(%rdi), %xmm1
 ; AVX2-ONLY-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX2-ONLY-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
-; AVX2-ONLY-NEXT:    vpermps %ymm5, %ymm3, %ymm4
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm4[2,3]
+; AVX2-ONLY-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
+; AVX2-ONLY-NEXT:    vpermps %ymm4, %ymm2, %ymm3
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3]
 ; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
 ; AVX2-ONLY-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-ONLY-NEXT:    vmovaps 576(%rdi), %ymm4
-; AVX2-ONLY-NEXT:    vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-ONLY-NEXT:    vmovaps 576(%rdi), %ymm3
+; AVX2-ONLY-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-ONLY-NEXT:    vmovaps 608(%rdi), %ymm0
 ; AVX2-ONLY-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-ONLY-NEXT:    vpermps %ymm0, %ymm2, %ymm0
-; AVX2-ONLY-NEXT:    vpermps %ymm4, %ymm2, %ymm1
+; AVX2-ONLY-NEXT:    vpermps %ymm3, %ymm2, %ymm1
 ; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
-; AVX2-ONLY-NEXT:    vmovaps 544(%rdi), %ymm5
-; AVX2-ONLY-NEXT:    vmovups %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-ONLY-NEXT:    vmovaps 528(%rdi), %xmm4
-; AVX2-ONLY-NEXT:    vmovaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX2-ONLY-NEXT:    vmovaps 544(%rdi), %ymm4
+; AVX2-ONLY-NEXT:    vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-ONLY-NEXT:    vmovaps 528(%rdi), %xmm3
+; AVX2-ONLY-NEXT:    vmovaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX2-ONLY-NEXT:    vmovaps 512(%rdi), %xmm1
 ; AVX2-ONLY-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX2-ONLY-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
-; AVX2-ONLY-NEXT:    vpermps %ymm5, %ymm3, %ymm4
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm4[2,3]
+; AVX2-ONLY-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
+; AVX2-ONLY-NEXT:    vpermps %ymm4, %ymm2, %ymm3
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3]
 ; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
 ; AVX2-ONLY-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-ONLY-NEXT:    vmovaps 832(%rdi), %ymm4
-; AVX2-ONLY-NEXT:    vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-ONLY-NEXT:    vmovaps 832(%rdi), %ymm3
+; AVX2-ONLY-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-ONLY-NEXT:    vmovaps 864(%rdi), %ymm0
 ; AVX2-ONLY-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-ONLY-NEXT:    vpermps %ymm0, %ymm2, %ymm0
-; AVX2-ONLY-NEXT:    vpermps %ymm4, %ymm2, %ymm1
+; AVX2-ONLY-NEXT:    vpermps %ymm3, %ymm2, %ymm1
 ; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
-; AVX2-ONLY-NEXT:    vmovaps 800(%rdi), %ymm5
-; AVX2-ONLY-NEXT:    vmovups %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-ONLY-NEXT:    vmovaps 784(%rdi), %xmm4
-; AVX2-ONLY-NEXT:    vmovaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX2-ONLY-NEXT:    vmovaps 800(%rdi), %ymm4
+; AVX2-ONLY-NEXT:    vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-ONLY-NEXT:    vmovaps 784(%rdi), %xmm3
+; AVX2-ONLY-NEXT:    vmovaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX2-ONLY-NEXT:    vmovaps 768(%rdi), %xmm1
 ; AVX2-ONLY-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX2-ONLY-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
-; AVX2-ONLY-NEXT:    vpermps %ymm5, %ymm3, %ymm4
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm4[2,3]
+; AVX2-ONLY-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
+; AVX2-ONLY-NEXT:    vpermps %ymm4, %ymm2, %ymm3
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3]
 ; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
 ; AVX2-ONLY-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-ONLY-NEXT:    vmovaps 64(%rdi), %ymm4
-; AVX2-ONLY-NEXT:    vmovaps 96(%rdi), %ymm1
-; AVX2-ONLY-NEXT:    vpermps %ymm1, %ymm2, %ymm0
-; AVX2-ONLY-NEXT:    vmovaps %ymm1, %ymm8
+; AVX2-ONLY-NEXT:    vmovaps 64(%rdi), %ymm1
+; AVX2-ONLY-NEXT:    vmovaps 96(%rdi), %ymm3
+; AVX2-ONLY-NEXT:    vpermps %ymm3, %ymm2, %ymm0
+; AVX2-ONLY-NEXT:    vmovaps %ymm3, %ymm7
+; AVX2-ONLY-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-ONLY-NEXT:    vpermps %ymm1, %ymm2, %ymm4
+; AVX2-ONLY-NEXT:    vmovaps %ymm1, %ymm3
 ; AVX2-ONLY-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-ONLY-NEXT:    vpermps %ymm4, %ymm2, %ymm1
-; AVX2-ONLY-NEXT:    vmovaps %ymm4, %ymm5
-; AVX2-ONLY-NEXT:    vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm4[0,1,2,3,4,5],ymm0[6,7]
 ; AVX2-ONLY-NEXT:    vmovaps 32(%rdi), %ymm1
 ; AVX2-ONLY-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-ONLY-NEXT:    vpermps %ymm1, %ymm3, %ymm1
-; AVX2-ONLY-NEXT:    vmovaps (%rdi), %xmm3
-; AVX2-ONLY-NEXT:    vmovaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX2-ONLY-NEXT:    vmovaps 16(%rdi), %xmm2
-; AVX2-ONLY-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX2-ONLY-NEXT:    vunpcklps {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
+; AVX2-ONLY-NEXT:    vpermps %ymm1, %ymm2, %ymm2
+; AVX2-ONLY-NEXT:    vmovaps (%rdi), %xmm1
+; AVX2-ONLY-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX2-ONLY-NEXT:    vmovaps 16(%rdi), %xmm4
+; AVX2-ONLY-NEXT:    vmovaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX2-ONLY-NEXT:    vunpcklps {{.*#+}} xmm4 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm2 = xmm4[0,1],xmm2[2,3]
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
 ; AVX2-ONLY-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-ONLY-NEXT:    vbroadcastsd {{.*#+}} ymm2 = [1,5,1,5,1,5,1,5]
-; AVX2-ONLY-NEXT:    vpermps %ymm10, %ymm2, %ymm0
-; AVX2-ONLY-NEXT:    vmovups %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-ONLY-NEXT:    vpermps %ymm13, %ymm2, %ymm1
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
-; AVX2-ONLY-NEXT:    vmovaps 128(%rdi), %ymm4
-; AVX2-ONLY-NEXT:    vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-ONLY-NEXT:    vmovaps 176(%rdi), %xmm3
-; AVX2-ONLY-NEXT:    vmovaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX2-ONLY-NEXT:    vmovaps 160(%rdi), %xmm1
-; AVX2-ONLY-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX2-ONLY-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
-; AVX2-ONLY-NEXT:    vmovddup {{.*#+}} xmm7 = [1,5,1,5]
-; AVX2-ONLY-NEXT:    # xmm7 = mem[0,0]
-; AVX2-ONLY-NEXT:    vpermps %ymm4, %ymm7, %ymm6
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm1 = xmm6[0,1],xmm1[2,3]
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
+; AVX2-ONLY-NEXT:    vpermps %ymm14, %ymm2, %ymm0
+; AVX2-ONLY-NEXT:    vpermps %ymm10, %ymm2, %ymm4
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm4[0,1,2,3,4,5],ymm0[6,7]
+; AVX2-ONLY-NEXT:    vmovaps 128(%rdi), %ymm1
+; AVX2-ONLY-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-ONLY-NEXT:    vmovaps 176(%rdi), %xmm5
+; AVX2-ONLY-NEXT:    vmovaps %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX2-ONLY-NEXT:    vmovaps 160(%rdi), %xmm4
+; AVX2-ONLY-NEXT:    vmovaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX2-ONLY-NEXT:    vunpcklps {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
+; AVX2-ONLY-NEXT:    vpermps %ymm1, %ymm2, %ymm5
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm4 = xmm5[0,1],xmm4[2,3]
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm0[4,5,6,7]
 ; AVX2-ONLY-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-ONLY-NEXT:    vpermps %ymm15, %ymm2, %ymm0
 ; AVX2-ONLY-NEXT:    vmovups %ymm15, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-ONLY-NEXT:    vpermps %ymm14, %ymm2, %ymm1
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
-; AVX2-ONLY-NEXT:    vmovaps 384(%rdi), %ymm4
-; AVX2-ONLY-NEXT:    vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-ONLY-NEXT:    vmovaps 432(%rdi), %xmm6
-; AVX2-ONLY-NEXT:    vmovaps %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX2-ONLY-NEXT:    vmovaps 416(%rdi), %xmm1
-; AVX2-ONLY-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX2-ONLY-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm1[0],xmm6[0],xmm1[1],xmm6[1]
-; AVX2-ONLY-NEXT:    vpermps %ymm4, %ymm7, %ymm6
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm1 = xmm6[0,1],xmm1[2,3]
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
-; AVX2-ONLY-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload
-; AVX2-ONLY-NEXT:    vpermps %ymm13, %ymm2, %ymm0
-; AVX2-ONLY-NEXT:    vpermps %ymm12, %ymm2, %ymm1
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
-; AVX2-ONLY-NEXT:    vmovaps 640(%rdi), %ymm4
-; AVX2-ONLY-NEXT:    vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-ONLY-NEXT:    vmovaps 688(%rdi), %xmm6
-; AVX2-ONLY-NEXT:    vmovaps %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX2-ONLY-NEXT:    vmovaps 672(%rdi), %xmm1
-; AVX2-ONLY-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX2-ONLY-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm1[0],xmm6[0],xmm1[1],xmm6[1]
-; AVX2-ONLY-NEXT:    vpermps %ymm4, %ymm7, %ymm6
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm1 = xmm6[0,1],xmm1[2,3]
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
+; AVX2-ONLY-NEXT:    vpermps %ymm9, %ymm2, %ymm4
+; AVX2-ONLY-NEXT:    vmovaps %ymm9, %ymm10
+; AVX2-ONLY-NEXT:    vmovups %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm4[0,1,2,3,4,5],ymm0[6,7]
+; AVX2-ONLY-NEXT:    vmovaps 384(%rdi), %ymm1
+; AVX2-ONLY-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-ONLY-NEXT:    vmovaps 432(%rdi), %xmm5
+; AVX2-ONLY-NEXT:    vmovaps %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX2-ONLY-NEXT:    vmovaps 416(%rdi), %xmm4
+; AVX2-ONLY-NEXT:    vmovaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX2-ONLY-NEXT:    vunpcklps {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
+; AVX2-ONLY-NEXT:    vpermps %ymm1, %ymm2, %ymm5
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm4 = xmm5[0,1],xmm4[2,3]
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm0[4,5,6,7]
 ; AVX2-ONLY-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-ONLY-NEXT:    vpermps %ymm11, %ymm2, %ymm0
-; AVX2-ONLY-NEXT:    vmovups %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload
-; AVX2-ONLY-NEXT:    vpermps %ymm14, %ymm2, %ymm1
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
-; AVX2-ONLY-NEXT:    vmovaps 896(%rdi), %ymm4
-; AVX2-ONLY-NEXT:    vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-ONLY-NEXT:    vmovaps 944(%rdi), %xmm6
-; AVX2-ONLY-NEXT:    vmovaps %xmm6, (%rsp) # 16-byte Spill
-; AVX2-ONLY-NEXT:    vmovaps 928(%rdi), %xmm1
-; AVX2-ONLY-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX2-ONLY-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm1[0],xmm6[0],xmm1[1],xmm6[1]
-; AVX2-ONLY-NEXT:    vpermps %ymm4, %ymm7, %ymm6
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm1 = xmm6[0,1],xmm1[2,3]
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
-; AVX2-ONLY-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-ONLY-NEXT:    vpermps %ymm8, %ymm2, %ymm0
-; AVX2-ONLY-NEXT:    vpermps %ymm5, %ymm2, %ymm1
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
-; AVX2-ONLY-NEXT:    vmovaps (%rdi), %ymm4
-; AVX2-ONLY-NEXT:    vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-ONLY-NEXT:    vmovaps 32(%rdi), %xmm5
+; AVX2-ONLY-NEXT:    vpermps %ymm13, %ymm2, %ymm4
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm4[0,1,2,3,4,5],ymm0[6,7]
+; AVX2-ONLY-NEXT:    vmovaps 640(%rdi), %ymm1
+; AVX2-ONLY-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-ONLY-NEXT:    vmovaps 688(%rdi), %xmm5
 ; AVX2-ONLY-NEXT:    vmovaps %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX2-ONLY-NEXT:    vmovaps 48(%rdi), %xmm1
-; AVX2-ONLY-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX2-ONLY-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm5[0],xmm1[0],xmm5[1],xmm1[1]
-; AVX2-ONLY-NEXT:    vpermps %ymm4, %ymm7, %ymm6
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm1 = xmm6[0,1],xmm1[2,3]
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
+; AVX2-ONLY-NEXT:    vmovaps 672(%rdi), %xmm4
+; AVX2-ONLY-NEXT:    vmovaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX2-ONLY-NEXT:    vunpcklps {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
+; AVX2-ONLY-NEXT:    vpermps %ymm1, %ymm2, %ymm5
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm4 = xmm5[0,1],xmm4[2,3]
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm0[4,5,6,7]
 ; AVX2-ONLY-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
-; AVX2-ONLY-NEXT:    vpermps %ymm9, %ymm2, %ymm0
+; AVX2-ONLY-NEXT:    vpermps %ymm8, %ymm2, %ymm0
+; AVX2-ONLY-NEXT:    vmovaps %ymm8, %ymm11
 ; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
-; AVX2-ONLY-NEXT:    vpermps %ymm8, %ymm2, %ymm1
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
-; AVX2-ONLY-NEXT:    vmovaps 256(%rdi), %ymm4
-; AVX2-ONLY-NEXT:    vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-ONLY-NEXT:    vmovaps 304(%rdi), %xmm5
-; AVX2-ONLY-NEXT:    vmovaps %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX2-ONLY-NEXT:    vmovaps 288(%rdi), %xmm1
-; AVX2-ONLY-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX2-ONLY-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1]
-; AVX2-ONLY-NEXT:    vpermps %ymm4, %ymm7, %ymm6
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm1 = xmm6[0,1],xmm1[2,3]
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
+; AVX2-ONLY-NEXT:    vpermps %ymm8, %ymm2, %ymm4
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm4[0,1,2,3,4,5],ymm0[6,7]
+; AVX2-ONLY-NEXT:    vmovaps 896(%rdi), %ymm1
+; AVX2-ONLY-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-ONLY-NEXT:    vmovaps 944(%rdi), %xmm5
+; AVX2-ONLY-NEXT:    vmovaps %xmm5, (%rsp) # 16-byte Spill
+; AVX2-ONLY-NEXT:    vmovaps 928(%rdi), %xmm4
+; AVX2-ONLY-NEXT:    vmovaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX2-ONLY-NEXT:    vunpcklps {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
+; AVX2-ONLY-NEXT:    vpermps %ymm1, %ymm2, %ymm5
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm4 = xmm5[0,1],xmm4[2,3]
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm0[4,5,6,7]
 ; AVX2-ONLY-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
-; AVX2-ONLY-NEXT:    vpermps %ymm5, %ymm2, %ymm0
-; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload
-; AVX2-ONLY-NEXT:    vpermps %ymm12, %ymm2, %ymm1
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm3 = ymm1[0,1,2,3,4,5],ymm0[6,7]
-; AVX2-ONLY-NEXT:    vmovaps 512(%rdi), %ymm4
-; AVX2-ONLY-NEXT:    vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-ONLY-NEXT:    vmovaps 560(%rdi), %xmm1
-; AVX2-ONLY-NEXT:    vmovaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX2-ONLY-NEXT:    vmovaps 544(%rdi), %xmm0
-; AVX2-ONLY-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX2-ONLY-NEXT:    vunpcklps {{.*#+}} xmm1 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; AVX2-ONLY-NEXT:    vpermps %ymm4, %ymm7, %ymm6
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm1 = xmm6[0,1],xmm1[2,3]
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm3[4,5,6,7]
+; AVX2-ONLY-NEXT:    vpermps %ymm7, %ymm2, %ymm0
+; AVX2-ONLY-NEXT:    vpermps %ymm3, %ymm2, %ymm4
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm4[0,1,2,3,4,5],ymm0[6,7]
+; AVX2-ONLY-NEXT:    vmovaps (%rdi), %ymm1
+; AVX2-ONLY-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-ONLY-NEXT:    vmovaps 32(%rdi), %xmm3
+; AVX2-ONLY-NEXT:    vmovaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX2-ONLY-NEXT:    vmovaps 48(%rdi), %xmm4
+; AVX2-ONLY-NEXT:    vmovaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX2-ONLY-NEXT:    vunpcklps {{.*#+}} xmm4 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
+; AVX2-ONLY-NEXT:    vpermps %ymm1, %ymm2, %ymm5
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm4 = xmm5[0,1],xmm4[2,3]
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm0[4,5,6,7]
 ; AVX2-ONLY-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
 ; AVX2-ONLY-NEXT:    vpermps %ymm6, %ymm2, %ymm0
-; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
-; AVX2-ONLY-NEXT:    vpermps %ymm4, %ymm2, %ymm1
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
-; AVX2-ONLY-NEXT:    vmovaps 768(%rdi), %ymm1
+; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
+; AVX2-ONLY-NEXT:    vpermps %ymm7, %ymm2, %ymm4
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm4[0,1,2,3,4,5],ymm0[6,7]
+; AVX2-ONLY-NEXT:    vmovaps 256(%rdi), %ymm1
 ; AVX2-ONLY-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-ONLY-NEXT:    vpermps %ymm1, %ymm7, %ymm1
-; AVX2-ONLY-NEXT:    vmovaps 816(%rdi), %xmm3
+; AVX2-ONLY-NEXT:    vmovaps 304(%rdi), %xmm3
 ; AVX2-ONLY-NEXT:    vmovaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX2-ONLY-NEXT:    vmovaps 800(%rdi), %xmm2
-; AVX2-ONLY-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX2-ONLY-NEXT:    vunpcklps {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3]
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
+; AVX2-ONLY-NEXT:    vmovaps 288(%rdi), %xmm4
+; AVX2-ONLY-NEXT:    vmovaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX2-ONLY-NEXT:    vunpcklps {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
+; AVX2-ONLY-NEXT:    vpermps %ymm1, %ymm2, %ymm5
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm4 = xmm5[0,1],xmm4[2,3]
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm0[4,5,6,7]
+; AVX2-ONLY-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload
+; AVX2-ONLY-NEXT:    vpermps %ymm13, %ymm2, %ymm0
+; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload
+; AVX2-ONLY-NEXT:    vpermps %ymm12, %ymm2, %ymm4
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm4[0,1,2,3,4,5],ymm0[6,7]
+; AVX2-ONLY-NEXT:    vmovaps 512(%rdi), %ymm1
+; AVX2-ONLY-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-ONLY-NEXT:    vmovaps 560(%rdi), %xmm3
+; AVX2-ONLY-NEXT:    vmovaps %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX2-ONLY-NEXT:    vmovaps 544(%rdi), %xmm4
+; AVX2-ONLY-NEXT:    vmovaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX2-ONLY-NEXT:    vunpcklps {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
+; AVX2-ONLY-NEXT:    vpermps %ymm1, %ymm2, %ymm5
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm4 = xmm5[0,1],xmm4[2,3]
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm0[4,5,6,7]
+; AVX2-ONLY-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
+; AVX2-ONLY-NEXT:    vpermps %ymm3, %ymm2, %ymm0
+; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-ONLY-NEXT:    vpermps %ymm1, %ymm2, %ymm4
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm4[0,1,2,3,4,5],ymm0[6,7]
+; AVX2-ONLY-NEXT:    vmovaps 768(%rdi), %ymm4
+; AVX2-ONLY-NEXT:    vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-ONLY-NEXT:    vpermps %ymm4, %ymm2, %ymm2
+; AVX2-ONLY-NEXT:    vmovaps 816(%rdi), %xmm5
+; AVX2-ONLY-NEXT:    vmovaps %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX2-ONLY-NEXT:    vmovaps 800(%rdi), %xmm4
+; AVX2-ONLY-NEXT:    vmovaps %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX2-ONLY-NEXT:    vunpcklps {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm2 = xmm2[0,1],xmm4[2,3]
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
 ; AVX2-ONLY-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-ONLY-NEXT:    vbroadcastsd {{.*#+}} ymm0 = [2,6,2,6,2,6,2,6]
-; AVX2-ONLY-NEXT:    vpermps %ymm10, %ymm0, %ymm1
-; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm2 # 32-byte Folded Reload
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm1[6,7]
-; AVX2-ONLY-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; AVX2-ONLY-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm3 # 16-byte Folded Reload
-; AVX2-ONLY-NEXT:    # xmm3 = xmm1[2],mem[2],xmm1[3],mem[3]
-; AVX2-ONLY-NEXT:    vmovddup {{.*#+}} xmm1 = [2,6,2,6]
-; AVX2-ONLY-NEXT:    # xmm1 = mem[0,0]
-; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm10 # 32-byte Folded Reload
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm3 = xmm3[0,1],xmm10[2,3]
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
+; AVX2-ONLY-NEXT:    vpermps %ymm14, %ymm0, %ymm2
+; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
+; AVX2-ONLY-NEXT:    vpermps %ymm5, %ymm0, %ymm4
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5],ymm2[6,7]
+; AVX2-ONLY-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
+; AVX2-ONLY-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4, %xmm4 # 16-byte Folded Reload
+; AVX2-ONLY-NEXT:    # xmm4 = xmm4[2],mem[2],xmm4[3],mem[3]
+; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm9 # 32-byte Folded Reload
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm4 = xmm4[0,1],xmm9[2,3]
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm2 = ymm4[0,1,2,3],ymm2[4,5,6,7]
 ; AVX2-ONLY-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-ONLY-NEXT:    vpermps %ymm15, %ymm0, %ymm2
-; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload
-; AVX2-ONLY-NEXT:    vpermps %ymm15, %ymm0, %ymm3
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5],ymm2[6,7]
-; AVX2-ONLY-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
-; AVX2-ONLY-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3, %xmm3 # 16-byte Folded Reload
-; AVX2-ONLY-NEXT:    # xmm3 = xmm3[2],mem[2],xmm3[3],mem[3]
-; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm10 # 32-byte Folded Reload
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm3 = xmm3[0,1],xmm10[2,3]
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
+; AVX2-ONLY-NEXT:    vpermps %ymm10, %ymm0, %ymm4
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5],ymm2[6,7]
+; AVX2-ONLY-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
+; AVX2-ONLY-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4, %xmm4 # 16-byte Folded Reload
+; AVX2-ONLY-NEXT:    # xmm4 = xmm4[2],mem[2],xmm4[3],mem[3]
+; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm9 # 32-byte Folded Reload
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm4 = xmm4[0,1],xmm9[2,3]
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm2 = ymm4[0,1,2,3],ymm2[4,5,6,7]
 ; AVX2-ONLY-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-ONLY-NEXT:    vpermps %ymm13, %ymm0, %ymm2
-; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload
-; AVX2-ONLY-NEXT:    vpermps %ymm13, %ymm0, %ymm3
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5],ymm2[6,7]
-; AVX2-ONLY-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
-; AVX2-ONLY-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3, %xmm3 # 16-byte Folded Reload
-; AVX2-ONLY-NEXT:    # xmm3 = xmm3[2],mem[2],xmm3[3],mem[3]
-; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm10 # 32-byte Folded Reload
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm3 = xmm3[0,1],xmm10[2,3]
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
+; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload
+; AVX2-ONLY-NEXT:    vpermps %ymm15, %ymm0, %ymm2
+; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload
+; AVX2-ONLY-NEXT:    vpermps %ymm10, %ymm0, %ymm4
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5],ymm2[6,7]
+; AVX2-ONLY-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
+; AVX2-ONLY-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4, %xmm4 # 16-byte Folded Reload
+; AVX2-ONLY-NEXT:    # xmm4 = xmm4[2],mem[2],xmm4[3],mem[3]
+; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm9 # 32-byte Folded Reload
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm4 = xmm4[0,1],xmm9[2,3]
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm2 = ymm4[0,1,2,3],ymm2[4,5,6,7]
 ; AVX2-ONLY-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-ONLY-NEXT:    vpermps %ymm11, %ymm0, %ymm2
-; AVX2-ONLY-NEXT:    vpermps %ymm14, %ymm0, %ymm3
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5],ymm2[6,7]
-; AVX2-ONLY-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
-; AVX2-ONLY-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3, %xmm3 # 16-byte Folded Reload
-; AVX2-ONLY-NEXT:    # xmm3 = xmm3[2],mem[2],xmm3[3],mem[3]
-; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm10 # 32-byte Folded Reload
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm3 = xmm3[0,1],xmm10[2,3]
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
+; AVX2-ONLY-NEXT:    vpermps %ymm8, %ymm0, %ymm4
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5],ymm2[6,7]
+; AVX2-ONLY-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
+; AVX2-ONLY-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4, %xmm4 # 16-byte Folded Reload
+; AVX2-ONLY-NEXT:    # xmm4 = xmm4[2],mem[2],xmm4[3],mem[3]
+; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm9 # 32-byte Folded Reload
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm4 = xmm4[0,1],xmm9[2,3]
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm2 = ymm4[0,1,2,3],ymm2[4,5,6,7]
 ; AVX2-ONLY-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-ONLY-NEXT:    vpermps %ymm9, %ymm0, %ymm2
-; AVX2-ONLY-NEXT:    vmovaps %ymm9, %ymm7
-; AVX2-ONLY-NEXT:    vpermps %ymm8, %ymm0, %ymm3
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5],ymm2[6,7]
-; AVX2-ONLY-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
-; AVX2-ONLY-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3, %xmm3 # 16-byte Folded Reload
-; AVX2-ONLY-NEXT:    # xmm3 = xmm3[2],mem[2],xmm3[3],mem[3]
-; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm10 # 32-byte Folded Reload
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm3 = xmm3[0,1],xmm10[2,3]
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm9 = ymm3[0,1,2,3],ymm2[4,5,6,7]
-; AVX2-ONLY-NEXT:    vpermps %ymm5, %ymm0, %ymm2
-; AVX2-ONLY-NEXT:    vpermps %ymm12, %ymm0, %ymm3
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5],ymm2[6,7]
-; AVX2-ONLY-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
-; AVX2-ONLY-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3, %xmm3 # 16-byte Folded Reload
-; AVX2-ONLY-NEXT:    # xmm3 = xmm3[2],mem[2],xmm3[3],mem[3]
-; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm10 # 32-byte Folded Reload
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm3 = xmm3[0,1],xmm10[2,3]
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
-; AVX2-ONLY-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-ONLY-NEXT:    vmovaps %ymm6, %ymm8
 ; AVX2-ONLY-NEXT:    vpermps %ymm6, %ymm0, %ymm2
-; AVX2-ONLY-NEXT:    vpermps %ymm4, %ymm0, %ymm3
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5],ymm2[6,7]
-; AVX2-ONLY-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
-; AVX2-ONLY-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3, %xmm3 # 16-byte Folded Reload
-; AVX2-ONLY-NEXT:    # xmm3 = xmm3[2],mem[2],xmm3[3],mem[3]
-; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm10 # 32-byte Folded Reload
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm3 = xmm3[0,1],xmm10[2,3]
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
+; AVX2-ONLY-NEXT:    vpermps %ymm7, %ymm0, %ymm4
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5],ymm2[6,7]
+; AVX2-ONLY-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
+; AVX2-ONLY-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4, %xmm4 # 16-byte Folded Reload
+; AVX2-ONLY-NEXT:    # xmm4 = xmm4[2],mem[2],xmm4[3],mem[3]
+; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm9 # 32-byte Folded Reload
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm4 = xmm4[0,1],xmm9[2,3]
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm2 = ymm4[0,1,2,3],ymm2[4,5,6,7]
 ; AVX2-ONLY-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
-; AVX2-ONLY-NEXT:    vpermps %ymm4, %ymm0, %ymm2
-; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
-; AVX2-ONLY-NEXT:    vpermps %ymm5, %ymm0, %ymm0
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm2[6,7]
-; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
-; AVX2-ONLY-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
-; AVX2-ONLY-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2, %xmm2 # 16-byte Folded Reload
-; AVX2-ONLY-NEXT:    # xmm2 = xmm2[2],mem[2],xmm2[3],mem[3]
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm10 = ymm1[0,1,2,3],ymm0[4,5,6,7]
-; AVX2-ONLY-NEXT:    vbroadcastsd {{.*#+}} ymm0 = [3,7,3,7,3,7,3,7]
-; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 32-byte Folded Reload
-; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm2 # 32-byte Folded Reload
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm1[6,7]
+; AVX2-ONLY-NEXT:    vpermps %ymm13, %ymm0, %ymm2
+; AVX2-ONLY-NEXT:    vpermps %ymm12, %ymm0, %ymm4
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5],ymm2[6,7]
+; AVX2-ONLY-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
+; AVX2-ONLY-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4, %xmm4 # 16-byte Folded Reload
+; AVX2-ONLY-NEXT:    # xmm4 = xmm4[2],mem[2],xmm4[3],mem[3]
+; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm9 # 32-byte Folded Reload
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm4 = xmm4[0,1],xmm9[2,3]
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm2 = ymm4[0,1,2,3],ymm2[4,5,6,7]
+; AVX2-ONLY-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-ONLY-NEXT:    vpermps %ymm3, %ymm0, %ymm2
+; AVX2-ONLY-NEXT:    vpermps %ymm1, %ymm0, %ymm4
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5],ymm2[6,7]
 ; AVX2-ONLY-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; AVX2-ONLY-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm3 # 16-byte Folded Reload
-; AVX2-ONLY-NEXT:    # xmm3 = xmm1[2],mem[2],xmm1[3],mem[3]
-; AVX2-ONLY-NEXT:    vmovddup {{.*#+}} xmm1 = [3,7,3,7]
-; AVX2-ONLY-NEXT:    # xmm1 = mem[0,0]
-; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm11 # 32-byte Folded Reload
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm3 = xmm11[0,1],xmm3[2,3]
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm11 = ymm3[0,1,2,3],ymm2[4,5,6,7]
+; AVX2-ONLY-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm4 # 16-byte Folded Reload
+; AVX2-ONLY-NEXT:    # xmm4 = xmm1[2],mem[2],xmm1[3],mem[3]
+; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm9 # 32-byte Folded Reload
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm4 = xmm4[0,1],xmm9[2,3]
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm1 = ymm4[0,1,2,3],ymm2[4,5,6,7]
+; AVX2-ONLY-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-ONLY-NEXT:    vpermps %ymm1, %ymm0, %ymm2
+; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
+; AVX2-ONLY-NEXT:    vpermps %ymm3, %ymm0, %ymm4
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5],ymm2[6,7]
+; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; AVX2-ONLY-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
+; AVX2-ONLY-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4, %xmm4 # 16-byte Folded Reload
+; AVX2-ONLY-NEXT:    # xmm4 = xmm4[2],mem[2],xmm4[3],mem[3]
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm0 = xmm4[0,1],xmm0[2,3]
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm9 = ymm0[0,1,2,3],ymm2[4,5,6,7]
+; AVX2-ONLY-NEXT:    vbroadcastsd {{.*#+}} ymm0 = [3,7,3,7,3,7,3,7]
+; AVX2-ONLY-NEXT:    vpermps %ymm14, %ymm0, %ymm2
+; AVX2-ONLY-NEXT:    vpermps %ymm5, %ymm0, %ymm4
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5],ymm2[6,7]
+; AVX2-ONLY-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
+; AVX2-ONLY-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4, %xmm4 # 16-byte Folded Reload
+; AVX2-ONLY-NEXT:    # xmm4 = xmm4[2],mem[2],xmm4[3],mem[3]
+; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm14 # 32-byte Folded Reload
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm4 = xmm14[0,1],xmm4[2,3]
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm2 = ymm4[0,1,2,3],ymm2[4,5,6,7]
+; AVX2-ONLY-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm2 # 32-byte Folded Reload
-; AVX2-ONLY-NEXT:    vpermps %ymm15, %ymm0, %ymm3
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5],ymm2[6,7]
+; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm4 # 32-byte Folded Reload
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5],ymm2[6,7]
+; AVX2-ONLY-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
+; AVX2-ONLY-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4, %xmm4 # 16-byte Folded Reload
+; AVX2-ONLY-NEXT:    # xmm4 = xmm4[2],mem[2],xmm4[3],mem[3]
+; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm14 # 32-byte Folded Reload
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm4 = xmm14[0,1],xmm4[2,3]
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm2[4,5,6,7]
+; AVX2-ONLY-NEXT:    vpermps %ymm15, %ymm0, %ymm2
+; AVX2-ONLY-NEXT:    vpermps %ymm10, %ymm0, %ymm14
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm2 = ymm14[0,1,2,3,4,5],ymm2[6,7]
+; AVX2-ONLY-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
+; AVX2-ONLY-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm5, %xmm14 # 16-byte Folded Reload
+; AVX2-ONLY-NEXT:    # xmm14 = xmm5[2],mem[2],xmm5[3],mem[3]
+; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm15 # 32-byte Folded Reload
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm14 = xmm15[0,1],xmm14[2,3]
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm2 = ymm14[0,1,2,3],ymm2[4,5,6,7]
+; AVX2-ONLY-NEXT:    vpermps %ymm11, %ymm0, %ymm11
+; AVX2-ONLY-NEXT:    vpermps %ymm8, %ymm0, %ymm14
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm11 = ymm14[0,1,2,3,4,5],ymm11[6,7]
+; AVX2-ONLY-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
+; AVX2-ONLY-NEXT:    vunpckhps (%rsp), %xmm5, %xmm14 # 16-byte Folded Reload
+; AVX2-ONLY-NEXT:    # xmm14 = xmm5[2],mem[2],xmm5[3],mem[3]
+; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm15 # 32-byte Folded Reload
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm14 = xmm15[0,1],xmm14[2,3]
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm11 = ymm14[0,1,2,3],ymm11[4,5,6,7]
+; AVX2-ONLY-NEXT:    vpermps %ymm1, %ymm0, %ymm1
+; AVX2-ONLY-NEXT:    vpermps %ymm3, %ymm0, %ymm3
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5],ymm1[6,7]
 ; AVX2-ONLY-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
 ; AVX2-ONLY-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3, %xmm3 # 16-byte Folded Reload
 ; AVX2-ONLY-NEXT:    # xmm3 = xmm3[2],mem[2],xmm3[3],mem[3]
-; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm14 # 32-byte Folded Reload
+; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm14 # 32-byte Folded Reload
 ; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm3 = xmm14[0,1],xmm3[2,3]
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm2[4,5,6,7]
-; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm2 # 32-byte Folded Reload
-; AVX2-ONLY-NEXT:    vpermps %ymm13, %ymm0, %ymm14
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm2 = ymm14[0,1,2,3,4,5],ymm2[6,7]
-; AVX2-ONLY-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX2-ONLY-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm6, %xmm14 # 16-byte Folded Reload
-; AVX2-ONLY-NEXT:    # xmm14 = xmm6[2],mem[2],xmm6[3],mem[3]
-; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm15 # 32-byte Folded Reload
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-ONLY-NEXT:    vpermps %ymm6, %ymm0, %ymm3
+; AVX2-ONLY-NEXT:    vpermps %ymm7, %ymm0, %ymm14
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm3 = ymm14[0,1,2,3,4,5],ymm3[6,7]
+; AVX2-ONLY-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
+; AVX2-ONLY-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm5, %xmm14 # 16-byte Folded Reload
+; AVX2-ONLY-NEXT:    # xmm14 = xmm5[2],mem[2],xmm5[3],mem[3]
+; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm15 # 32-byte Folded Reload
 ; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm14 = xmm15[0,1],xmm14[2,3]
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm2 = ymm14[0,1,2,3],ymm2[4,5,6,7]
-; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm13 # 32-byte Folded Reload
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm3 = ymm14[0,1,2,3],ymm3[4,5,6,7]
+; AVX2-ONLY-NEXT:    vpermps %ymm13, %ymm0, %ymm10
+; AVX2-ONLY-NEXT:    vpermps %ymm12, %ymm0, %ymm14
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm10 = ymm14[0,1,2,3,4,5],ymm10[6,7]
+; AVX2-ONLY-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
+; AVX2-ONLY-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm5, %xmm13 # 16-byte Folded Reload
+; AVX2-ONLY-NEXT:    # xmm13 = xmm5[2],mem[2],xmm5[3],mem[3]
 ; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm14 # 32-byte Folded Reload
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm13 = ymm14[0,1,2,3,4,5],ymm13[6,7]
-; AVX2-ONLY-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX2-ONLY-NEXT:    vunpckhps (%rsp), %xmm6, %xmm14 # 16-byte Folded Reload
-; AVX2-ONLY-NEXT:    # xmm14 = xmm6[2],mem[2],xmm6[3],mem[3]
-; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm15 # 32-byte Folded Reload
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm14 = xmm15[0,1],xmm14[2,3]
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm13 = ymm14[0,1,2,3],ymm13[4,5,6,7]
-; AVX2-ONLY-NEXT:    vpermps %ymm4, %ymm0, %ymm4
-; AVX2-ONLY-NEXT:    vpermps %ymm5, %ymm0, %ymm5
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm4 = ymm5[0,1,2,3,4,5],ymm4[6,7]
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm13 = xmm14[0,1],xmm13[2,3]
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm10 = ymm13[0,1,2,3],ymm10[4,5,6,7]
+; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm7 # 32-byte Folded Reload
+; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm8 # 32-byte Folded Reload
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm7 = ymm8[0,1,2,3,4,5],ymm7[6,7]
 ; AVX2-ONLY-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
 ; AVX2-ONLY-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm5, %xmm5 # 16-byte Folded Reload
 ; AVX2-ONLY-NEXT:    # xmm5 = xmm5[2],mem[2],xmm5[3],mem[3]
-; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm14 # 32-byte Folded Reload
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm5 = xmm14[0,1],xmm5[2,3]
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm4 = ymm5[0,1,2,3],ymm4[4,5,6,7]
-; AVX2-ONLY-NEXT:    vpermps %ymm7, %ymm0, %ymm5
-; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm14 # 32-byte Folded Reload
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm5 = ymm14[0,1,2,3,4,5],ymm5[6,7]
-; AVX2-ONLY-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX2-ONLY-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm6, %xmm14 # 16-byte Folded Reload
-; AVX2-ONLY-NEXT:    # xmm14 = xmm6[2],mem[2],xmm6[3],mem[3]
-; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm15 # 32-byte Folded Reload
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm14 = xmm15[0,1],xmm14[2,3]
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm5 = ymm14[0,1,2,3],ymm5[4,5,6,7]
-; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm12 # 32-byte Folded Reload
-; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm14 # 32-byte Folded Reload
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm12 = ymm14[0,1,2,3,4,5],ymm12[6,7]
-; AVX2-ONLY-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX2-ONLY-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm6, %xmm14 # 16-byte Folded Reload
-; AVX2-ONLY-NEXT:    # xmm14 = xmm6[2],mem[2],xmm6[3],mem[3]
-; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm15 # 32-byte Folded Reload
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm14 = xmm15[0,1],xmm14[2,3]
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm12 = ymm14[0,1,2,3],ymm12[4,5,6,7]
-; AVX2-ONLY-NEXT:    vpermps %ymm8, %ymm0, %ymm8
 ; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm8[6,7]
-; AVX2-ONLY-NEXT:    vmovaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX2-ONLY-NEXT:    vunpckhps {{[-0-9]+}}(%r{{[sb]}}p), %xmm6, %xmm6 # 16-byte Folded Reload
-; AVX2-ONLY-NEXT:    # xmm6 = xmm6[2],mem[2],xmm6[3],mem[3]
-; AVX2-ONLY-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm1 = xmm1[0,1],xmm6[2,3]
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
-; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX2-ONLY-NEXT:    vmovaps %ymm1, 192(%rsi)
-; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX2-ONLY-NEXT:    vmovaps %ymm1, 128(%rsi)
-; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX2-ONLY-NEXT:    vmovaps %ymm1, 64(%rsi)
-; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX2-ONLY-NEXT:    vmovaps %ymm1, (%rsi)
-; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX2-ONLY-NEXT:    vmovaps %ymm1, 224(%rsi)
-; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX2-ONLY-NEXT:    vmovaps %ymm1, 160(%rsi)
-; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX2-ONLY-NEXT:    vmovaps %ymm1, 96(%rsi)
-; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX2-ONLY-NEXT:    vmovaps %ymm1, 32(%rsi)
-; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX2-ONLY-NEXT:    vmovaps %ymm1, 192(%rdx)
-; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX2-ONLY-NEXT:    vmovaps %ymm1, 128(%rdx)
-; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX2-ONLY-NEXT:    vmovaps %ymm1, 64(%rdx)
-; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX2-ONLY-NEXT:    vmovaps %ymm1, (%rdx)
-; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX2-ONLY-NEXT:    vmovaps %ymm1, 224(%rdx)
-; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX2-ONLY-NEXT:    vmovaps %ymm1, 160(%rdx)
-; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX2-ONLY-NEXT:    vmovaps %ymm1, 96(%rdx)
-; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX2-ONLY-NEXT:    vmovaps %ymm1, 32(%rdx)
-; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX2-ONLY-NEXT:    vmovaps %ymm1, 192(%rcx)
-; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX2-ONLY-NEXT:    vmovaps %ymm1, 128(%rcx)
-; AVX2-ONLY-NEXT:    vmovaps %ymm9, 64(%rcx)
-; AVX2-ONLY-NEXT:    vmovaps %ymm10, (%rcx)
-; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX2-ONLY-NEXT:    vmovaps %ymm1, 224(%rcx)
-; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX2-ONLY-NEXT:    vmovaps %ymm1, 160(%rcx)
-; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX2-ONLY-NEXT:    vmovaps %ymm1, 96(%rcx)
-; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX2-ONLY-NEXT:    vmovaps %ymm1, 32(%rcx)
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm0 = xmm0[0,1],xmm5[2,3]
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm7[4,5,6,7]
+; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
+; AVX2-ONLY-NEXT:    vmovaps %ymm5, 192(%rsi)
+; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
+; AVX2-ONLY-NEXT:    vmovaps %ymm5, 128(%rsi)
+; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
+; AVX2-ONLY-NEXT:    vmovaps %ymm5, 64(%rsi)
+; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
+; AVX2-ONLY-NEXT:    vmovaps %ymm5, (%rsi)
+; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
+; AVX2-ONLY-NEXT:    vmovaps %ymm5, 224(%rsi)
+; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
+; AVX2-ONLY-NEXT:    vmovaps %ymm5, 160(%rsi)
+; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
+; AVX2-ONLY-NEXT:    vmovaps %ymm5, 96(%rsi)
+; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
+; AVX2-ONLY-NEXT:    vmovaps %ymm5, 32(%rsi)
+; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
+; AVX2-ONLY-NEXT:    vmovaps %ymm5, 192(%rdx)
+; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
+; AVX2-ONLY-NEXT:    vmovaps %ymm5, 128(%rdx)
+; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
+; AVX2-ONLY-NEXT:    vmovaps %ymm5, 64(%rdx)
+; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
+; AVX2-ONLY-NEXT:    vmovaps %ymm5, (%rdx)
+; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
+; AVX2-ONLY-NEXT:    vmovaps %ymm5, 224(%rdx)
+; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
+; AVX2-ONLY-NEXT:    vmovaps %ymm5, 160(%rdx)
+; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
+; AVX2-ONLY-NEXT:    vmovaps %ymm5, 96(%rdx)
+; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
+; AVX2-ONLY-NEXT:    vmovaps %ymm5, 32(%rdx)
+; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
+; AVX2-ONLY-NEXT:    vmovaps %ymm5, 192(%rcx)
+; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
+; AVX2-ONLY-NEXT:    vmovaps %ymm5, 128(%rcx)
+; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
+; AVX2-ONLY-NEXT:    vmovaps %ymm5, 64(%rcx)
+; AVX2-ONLY-NEXT:    vmovaps %ymm9, (%rcx)
+; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
+; AVX2-ONLY-NEXT:    vmovaps %ymm5, 224(%rcx)
+; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
+; AVX2-ONLY-NEXT:    vmovaps %ymm5, 160(%rcx)
+; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
+; AVX2-ONLY-NEXT:    vmovaps %ymm5, 96(%rcx)
+; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
+; AVX2-ONLY-NEXT:    vmovaps %ymm5, 32(%rcx)
 ; AVX2-ONLY-NEXT:    vmovaps %ymm0, 192(%r8)
-; AVX2-ONLY-NEXT:    vmovaps %ymm12, 128(%r8)
-; AVX2-ONLY-NEXT:    vmovaps %ymm5, 64(%r8)
-; AVX2-ONLY-NEXT:    vmovaps %ymm4, (%r8)
-; AVX2-ONLY-NEXT:    vmovaps %ymm13, 224(%r8)
+; AVX2-ONLY-NEXT:    vmovaps %ymm10, 128(%r8)
+; AVX2-ONLY-NEXT:    vmovaps %ymm3, 64(%r8)
+; AVX2-ONLY-NEXT:    vmovaps %ymm1, (%r8)
+; AVX2-ONLY-NEXT:    vmovaps %ymm11, 224(%r8)
 ; AVX2-ONLY-NEXT:    vmovaps %ymm2, 160(%r8)
-; AVX2-ONLY-NEXT:    vmovaps %ymm3, 96(%r8)
-; AVX2-ONLY-NEXT:    vmovaps %ymm11, 32(%r8)
-; AVX2-ONLY-NEXT:    addq $1992, %rsp # imm = 0x7C8
+; AVX2-ONLY-NEXT:    vmovaps %ymm4, 96(%r8)
+; AVX2-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX2-ONLY-NEXT:    vmovaps %ymm0, 32(%r8)
+; AVX2-ONLY-NEXT:    addq $1944, %rsp # imm = 0x798
 ; AVX2-ONLY-NEXT:    vzeroupper
 ; AVX2-ONLY-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i32-stride-6.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i32-stride-6.ll
index 5d3f85d6ab6924ee94e4ee3608f03999201beb25..0c229457fb6024332dc5865c1a98cfba63abf5ba 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i32-stride-6.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i32-stride-6.ll
@@ -569,82 +569,78 @@ define void @load_i32_stride6_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX2-SLOW-NEXT:    vmovaps 128(%rdi), %ymm0
 ; AVX2-SLOW-NEXT:    vmovaps 160(%rdi), %ymm1
 ; AVX2-SLOW-NEXT:    vmovaps 96(%rdi), %ymm5
-; AVX2-SLOW-NEXT:    vmovaps (%rdi), %ymm3
+; AVX2-SLOW-NEXT:    vmovaps (%rdi), %ymm2
 ; AVX2-SLOW-NEXT:    vmovaps 32(%rdi), %ymm4
-; AVX2-SLOW-NEXT:    vmovaps 64(%rdi), %ymm7
-; AVX2-SLOW-NEXT:    vmovaps {{.*#+}} xmm2 = <0,6,4,u>
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm6 = ymm3[0,1,2,3],ymm4[4,5],ymm3[6,7]
-; AVX2-SLOW-NEXT:    vpermps %ymm6, %ymm2, %ymm2
-; AVX2-SLOW-NEXT:    vperm2f128 {{.*#+}} ymm8 = ymm7[0,1],ymm5[0,1]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],ymm5[6,7]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm9 = ymm8[0,2,2,2,4,6,6,6]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2],ymm9[3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm9 = ymm1[0,1,2,3],ymm0[4,5,6,7]
-; AVX2-SLOW-NEXT:    vbroadcastsd {{.*#+}} ymm10 = [4,2,4,2,4,2,4,2]
-; AVX2-SLOW-NEXT:    vpermps %ymm9, %ymm10, %ymm10
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm10[6,7]
-; AVX2-SLOW-NEXT:    vmovaps {{.*#+}} xmm10 = <1,7,5,u>
-; AVX2-SLOW-NEXT:    vpermps %ymm6, %ymm10, %ymm6
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm8 = ymm8[1,3,2,3,5,7,6,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm6 = ymm6[0,1,2],ymm8[3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vbroadcastsd {{.*#+}} ymm8 = [5,3,5,3,5,3,5,3]
-; AVX2-SLOW-NEXT:    vpermps %ymm9, %ymm8, %ymm8
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm6 = ymm6[0,1,2,3,4,5],ymm8[6,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm8 = ymm7[0,1],ymm5[2,3],ymm7[4,5],ymm5[6,7]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm8 = ymm8[2,0,2,3,6,4,6,7]
-; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm8 = ymm8[0,2,0,3]
-; AVX2-SLOW-NEXT:    vmovaps (%rdi), %xmm9
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm10 = xmm9[2,3,2,3]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm11 = ymm4[0,0,2,3,4,4,6,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm10 = ymm10[0],ymm11[1,2,3],ymm10[4],ymm11[5,6,7]
-; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm10 = ymm10[0,3,2,3]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm8 = ymm10[0,1,2],ymm8[3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm10 = ymm1[0,0,0,0,4,4,4,4]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm11 = ymm0[0,0,2,3,4,4,6,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm10 = ymm11[0,1,2],ymm10[3],ymm11[4,5,6],ymm10[7]
-; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm10 = ymm10[0,1,0,3]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm8 = ymm8[0,1,2,3,4],ymm10[5,6,7]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm10 = ymm5[3,3,3,3,7,7,7,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm10 = ymm10[0],ymm7[1],ymm10[2,3,4],ymm7[5],ymm10[6,7]
+; AVX2-SLOW-NEXT:    vmovaps 64(%rdi), %ymm6
+; AVX2-SLOW-NEXT:    vmovaps {{.*#+}} xmm3 = <0,6,4,u>
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm7 = ymm2[0,1,2,3],ymm4[4,5],ymm2[6,7]
+; AVX2-SLOW-NEXT:    vpermps %ymm7, %ymm3, %ymm3
+; AVX2-SLOW-NEXT:    vperm2f128 {{.*#+}} ymm8 = ymm6[0,1],ymm5[0,1]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm9 = ymm8[0,1,2,3,4,5],ymm5[6,7]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm8 = ymm9[0,2,2,2,4,6,6,6]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1,2],ymm8[3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm10 = ymm1[0,1,2,3],ymm0[4,5,6,7]
+; AVX2-SLOW-NEXT:    vbroadcastsd {{.*#+}} ymm8 = [4,2,4,2,4,2,4,2]
+; AVX2-SLOW-NEXT:    vpermps %ymm10, %ymm8, %ymm11
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5],ymm11[6,7]
+; AVX2-SLOW-NEXT:    vmovaps {{.*#+}} xmm11 = <1,7,5,u>
+; AVX2-SLOW-NEXT:    vpermps %ymm7, %ymm11, %ymm7
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm9 = ymm9[1,3,2,3,5,7,6,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm7 = ymm7[0,1,2],ymm9[3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vbroadcastsd {{.*#+}} ymm9 = [5,3,5,3,5,3,5,3]
+; AVX2-SLOW-NEXT:    vpermps %ymm10, %ymm9, %ymm10
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm7 = ymm7[0,1,2,3,4,5],ymm10[6,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm10 = ymm6[0,1],ymm5[2,3],ymm6[4,5],ymm5[6,7]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm10 = ymm10[2,0,2,3,6,4,6,7]
 ; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm10 = ymm10[0,2,0,3]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm9 = xmm9[3,3,3,3]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm11 = ymm4[0,1,3,3,4,5,7,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm9 = ymm9[0],ymm11[1,2,3],ymm9[4],ymm11[5,6,7]
-; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm9 = ymm9[0,3,2,3]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm9 = ymm9[0,1,2],ymm10[3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm10 = ymm1[0,1,0,1,4,5,4,5]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm11 = ymm0[0,1,3,3,4,5,7,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm10 = ymm11[0,1,2],ymm10[3],ymm11[4,5,6],ymm10[7]
-; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm10 = ymm10[0,1,0,3]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm9 = ymm9[0,1,2,3,4],ymm10[5,6,7]
-; AVX2-SLOW-NEXT:    vmovaps 80(%rdi), %xmm10
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm5 = ymm7[0,1,2,3],ymm5[4,5,6,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm7 = ymm5[0,1],ymm10[2,3],ymm5[4,5,6,7]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm7 = ymm7[0,1,0,2,4,5,4,6]
-; AVX2-SLOW-NEXT:    vmovddup {{.*#+}} xmm11 = [4,2,4,2]
-; AVX2-SLOW-NEXT:    # xmm11 = mem[0,0]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
-; AVX2-SLOW-NEXT:    vpermps %ymm3, %ymm11, %ymm4
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1],ymm7[2,3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vmovaps (%rdi), %xmm11
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm12 = xmm11[2,3,2,3]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm13 = ymm4[0,0,2,3,4,4,6,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm12 = ymm12[0],ymm13[1,2,3],ymm12[4],ymm13[5,6,7]
+; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm12 = ymm12[0,3,2,3]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm10 = ymm12[0,1,2],ymm10[3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm12 = ymm1[0,0,0,0,4,4,4,4]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm13 = ymm0[0,0,2,3,4,4,6,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm12 = ymm13[0,1,2],ymm12[3],ymm13[4,5,6],ymm12[7]
+; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm12 = ymm12[0,1,0,3]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm10 = ymm10[0,1,2,3,4],ymm12[5,6,7]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm12 = ymm5[3,3,3,3,7,7,7,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm12 = ymm12[0],ymm6[1],ymm12[2,3,4],ymm6[5],ymm12[6,7]
+; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm12 = ymm12[0,2,0,3]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm11 = xmm11[3,3,3,3]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm13 = ymm4[0,1,3,3,4,5,7,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm11 = ymm11[0],ymm13[1,2,3],ymm11[4],ymm13[5,6,7]
+; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm11 = ymm11[0,3,2,3]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm11 = ymm11[0,1,2],ymm12[3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm12 = ymm1[0,1,0,1,4,5,4,5]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm13 = ymm0[0,1,3,3,4,5,7,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm12 = ymm13[0,1,2],ymm12[3],ymm13[4,5,6],ymm12[7]
+; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm12 = ymm12[0,1,0,3]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm11 = ymm11[0,1,2,3,4],ymm12[5,6,7]
+; AVX2-SLOW-NEXT:    vmovaps 80(%rdi), %xmm12
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm5 = ymm6[0,1,2,3],ymm5[4,5,6,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm6 = ymm5[0,1],ymm12[2,3],ymm5[4,5,6,7]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm6 = ymm6[0,1,0,2,4,5,4,6]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm4[0,1,2,3],ymm2[4,5,6,7]
+; AVX2-SLOW-NEXT:    vpermps %ymm2, %ymm8, %ymm4
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1],ymm6[2,3,4,5,6,7]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3],ymm1[4,5,6,7]
 ; AVX2-SLOW-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = [0,2,0,6,0,2,0,6]
 ; AVX2-SLOW-NEXT:    # ymm1 = mem[0,1,0,1]
 ; AVX2-SLOW-NEXT:    vpermps %ymm0, %ymm1, %ymm1
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm4[0,1,2,3,4],ymm1[5,6,7]
 ; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm4 = ymm5[1,1,1,1,5,5,5,5]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1,2],ymm10[3],ymm4[4,5,6,7]
-; AVX2-SLOW-NEXT:    vmovddup {{.*#+}} xmm5 = [5,3,5,3]
-; AVX2-SLOW-NEXT:    # xmm5 = mem[0,0]
-; AVX2-SLOW-NEXT:    vpermps %ymm3, %ymm5, %ymm3
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1],ymm4[2,3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1,2],ymm12[3],ymm4[4,5,6,7]
+; AVX2-SLOW-NEXT:    vpermps %ymm2, %ymm9, %ymm2
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1],ymm4[2,3,4,5,6,7]
 ; AVX2-SLOW-NEXT:    vbroadcastf128 {{.*#+}} ymm4 = [0,3,1,7,0,3,1,7]
 ; AVX2-SLOW-NEXT:    # ymm4 = mem[0,1,0,1]
 ; AVX2-SLOW-NEXT:    vpermps %ymm0, %ymm4, %ymm0
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm3[0,1,2,3,4],ymm0[5,6,7]
-; AVX2-SLOW-NEXT:    vmovaps %ymm2, (%rsi)
-; AVX2-SLOW-NEXT:    vmovaps %ymm6, (%rdx)
-; AVX2-SLOW-NEXT:    vmovaps %ymm8, (%rcx)
-; AVX2-SLOW-NEXT:    vmovaps %ymm9, (%r8)
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3,4],ymm0[5,6,7]
+; AVX2-SLOW-NEXT:    vmovaps %ymm3, (%rsi)
+; AVX2-SLOW-NEXT:    vmovaps %ymm7, (%rdx)
+; AVX2-SLOW-NEXT:    vmovaps %ymm10, (%rcx)
+; AVX2-SLOW-NEXT:    vmovaps %ymm11, (%r8)
 ; AVX2-SLOW-NEXT:    vmovaps %ymm1, (%r9)
 ; AVX2-SLOW-NEXT:    vmovaps %ymm0, (%rax)
 ; AVX2-SLOW-NEXT:    vzeroupper
@@ -656,82 +652,78 @@ define void @load_i32_stride6_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX2-FAST-NEXT:    vmovaps 128(%rdi), %ymm0
 ; AVX2-FAST-NEXT:    vmovaps 160(%rdi), %ymm1
 ; AVX2-FAST-NEXT:    vmovaps 96(%rdi), %ymm5
-; AVX2-FAST-NEXT:    vmovaps (%rdi), %ymm3
+; AVX2-FAST-NEXT:    vmovaps (%rdi), %ymm2
 ; AVX2-FAST-NEXT:    vmovaps 32(%rdi), %ymm4
-; AVX2-FAST-NEXT:    vmovaps 64(%rdi), %ymm7
-; AVX2-FAST-NEXT:    vmovaps {{.*#+}} xmm2 = <0,6,4,u>
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm6 = ymm3[0,1,2,3],ymm4[4,5],ymm3[6,7]
-; AVX2-FAST-NEXT:    vpermps %ymm6, %ymm2, %ymm2
-; AVX2-FAST-NEXT:    vperm2f128 {{.*#+}} ymm8 = ymm7[0,1],ymm5[0,1]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],ymm5[6,7]
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm9 = ymm8[0,2,2,2,4,6,6,6]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2],ymm9[3,4,5,6,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm9 = ymm1[0,1,2,3],ymm0[4,5,6,7]
-; AVX2-FAST-NEXT:    vbroadcastsd {{.*#+}} ymm10 = [4,2,4,2,4,2,4,2]
-; AVX2-FAST-NEXT:    vpermps %ymm9, %ymm10, %ymm10
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm10[6,7]
-; AVX2-FAST-NEXT:    vmovaps {{.*#+}} xmm10 = <1,7,5,u>
-; AVX2-FAST-NEXT:    vpermps %ymm6, %ymm10, %ymm6
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm8 = ymm8[1,3,2,3,5,7,6,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm6 = ymm6[0,1,2],ymm8[3,4,5,6,7]
-; AVX2-FAST-NEXT:    vbroadcastsd {{.*#+}} ymm8 = [5,3,5,3,5,3,5,3]
-; AVX2-FAST-NEXT:    vpermps %ymm9, %ymm8, %ymm8
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm6 = ymm6[0,1,2,3,4,5],ymm8[6,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm8 = ymm7[0,1],ymm5[2,3],ymm7[4,5],ymm5[6,7]
-; AVX2-FAST-NEXT:    vbroadcastsd {{.*#+}} ymm9 = [2,4,2,4,2,4,2,4]
-; AVX2-FAST-NEXT:    vpermps %ymm8, %ymm9, %ymm8
-; AVX2-FAST-NEXT:    vmovaps (%rdi), %xmm9
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} xmm10 = xmm9[2,3,2,3]
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm11 = ymm4[0,0,2,3,4,4,6,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm10 = ymm10[0],ymm11[1,2,3],ymm10[4],ymm11[5,6,7]
-; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm10 = ymm10[0,3,2,3]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm8 = ymm10[0,1,2],ymm8[3,4,5,6,7]
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm10 = ymm1[0,0,0,0,4,4,4,4]
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm11 = ymm0[0,0,2,3,4,4,6,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm10 = ymm11[0,1,2],ymm10[3],ymm11[4,5,6],ymm10[7]
-; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm10 = ymm10[0,1,0,3]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm8 = ymm8[0,1,2,3,4],ymm10[5,6,7]
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm10 = ymm5[3,3,3,3,7,7,7,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm10 = ymm10[0],ymm7[1],ymm10[2,3,4],ymm7[5],ymm10[6,7]
-; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm10 = ymm10[0,2,0,3]
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} xmm9 = xmm9[3,3,3,3]
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm11 = ymm4[0,1,3,3,4,5,7,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm9 = ymm9[0],ymm11[1,2,3],ymm9[4],ymm11[5,6,7]
-; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm9 = ymm9[0,3,2,3]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm9 = ymm9[0,1,2],ymm10[3,4,5,6,7]
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm10 = ymm1[0,1,0,1,4,5,4,5]
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm11 = ymm0[0,1,3,3,4,5,7,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm10 = ymm11[0,1,2],ymm10[3],ymm11[4,5,6],ymm10[7]
-; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm10 = ymm10[0,1,0,3]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm9 = ymm9[0,1,2,3,4],ymm10[5,6,7]
-; AVX2-FAST-NEXT:    vmovaps 80(%rdi), %xmm10
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm5 = ymm7[0,1,2,3],ymm5[4,5,6,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm7 = ymm5[0,1],ymm10[2,3],ymm5[4,5,6,7]
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm7 = ymm7[0,1,0,2,4,5,4,6]
-; AVX2-FAST-NEXT:    vmovddup {{.*#+}} xmm11 = [4,2,4,2]
-; AVX2-FAST-NEXT:    # xmm11 = mem[0,0]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
-; AVX2-FAST-NEXT:    vpermps %ymm3, %ymm11, %ymm4
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1],ymm7[2,3,4,5,6,7]
+; AVX2-FAST-NEXT:    vmovaps 64(%rdi), %ymm6
+; AVX2-FAST-NEXT:    vmovaps {{.*#+}} xmm3 = <0,6,4,u>
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm7 = ymm2[0,1,2,3],ymm4[4,5],ymm2[6,7]
+; AVX2-FAST-NEXT:    vpermps %ymm7, %ymm3, %ymm3
+; AVX2-FAST-NEXT:    vperm2f128 {{.*#+}} ymm8 = ymm6[0,1],ymm5[0,1]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm9 = ymm8[0,1,2,3,4,5],ymm5[6,7]
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm8 = ymm9[0,2,2,2,4,6,6,6]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1,2],ymm8[3,4,5,6,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm10 = ymm1[0,1,2,3],ymm0[4,5,6,7]
+; AVX2-FAST-NEXT:    vbroadcastsd {{.*#+}} ymm8 = [4,2,4,2,4,2,4,2]
+; AVX2-FAST-NEXT:    vpermps %ymm10, %ymm8, %ymm11
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5],ymm11[6,7]
+; AVX2-FAST-NEXT:    vmovaps {{.*#+}} xmm11 = <1,7,5,u>
+; AVX2-FAST-NEXT:    vpermps %ymm7, %ymm11, %ymm7
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm9 = ymm9[1,3,2,3,5,7,6,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm7 = ymm7[0,1,2],ymm9[3,4,5,6,7]
+; AVX2-FAST-NEXT:    vbroadcastsd {{.*#+}} ymm9 = [5,3,5,3,5,3,5,3]
+; AVX2-FAST-NEXT:    vpermps %ymm10, %ymm9, %ymm10
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm7 = ymm7[0,1,2,3,4,5],ymm10[6,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm10 = ymm6[0,1],ymm5[2,3],ymm6[4,5],ymm5[6,7]
+; AVX2-FAST-NEXT:    vbroadcastsd {{.*#+}} ymm11 = [2,4,2,4,2,4,2,4]
+; AVX2-FAST-NEXT:    vpermps %ymm10, %ymm11, %ymm10
+; AVX2-FAST-NEXT:    vmovaps (%rdi), %xmm11
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} xmm12 = xmm11[2,3,2,3]
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm13 = ymm4[0,0,2,3,4,4,6,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm12 = ymm12[0],ymm13[1,2,3],ymm12[4],ymm13[5,6,7]
+; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm12 = ymm12[0,3,2,3]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm10 = ymm12[0,1,2],ymm10[3,4,5,6,7]
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm12 = ymm1[0,0,0,0,4,4,4,4]
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm13 = ymm0[0,0,2,3,4,4,6,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm12 = ymm13[0,1,2],ymm12[3],ymm13[4,5,6],ymm12[7]
+; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm12 = ymm12[0,1,0,3]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm10 = ymm10[0,1,2,3,4],ymm12[5,6,7]
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm12 = ymm5[3,3,3,3,7,7,7,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm12 = ymm12[0],ymm6[1],ymm12[2,3,4],ymm6[5],ymm12[6,7]
+; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm12 = ymm12[0,2,0,3]
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} xmm11 = xmm11[3,3,3,3]
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm13 = ymm4[0,1,3,3,4,5,7,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm11 = ymm11[0],ymm13[1,2,3],ymm11[4],ymm13[5,6,7]
+; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm11 = ymm11[0,3,2,3]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm11 = ymm11[0,1,2],ymm12[3,4,5,6,7]
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm12 = ymm1[0,1,0,1,4,5,4,5]
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm13 = ymm0[0,1,3,3,4,5,7,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm12 = ymm13[0,1,2],ymm12[3],ymm13[4,5,6],ymm12[7]
+; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm12 = ymm12[0,1,0,3]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm11 = ymm11[0,1,2,3,4],ymm12[5,6,7]
+; AVX2-FAST-NEXT:    vmovaps 80(%rdi), %xmm12
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm5 = ymm6[0,1,2,3],ymm5[4,5,6,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm6 = ymm5[0,1],ymm12[2,3],ymm5[4,5,6,7]
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm6 = ymm6[0,1,0,2,4,5,4,6]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm4[0,1,2,3],ymm2[4,5,6,7]
+; AVX2-FAST-NEXT:    vpermps %ymm2, %ymm8, %ymm4
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1],ymm6[2,3,4,5,6,7]
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3],ymm1[4,5,6,7]
 ; AVX2-FAST-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = [0,2,0,6,0,2,0,6]
 ; AVX2-FAST-NEXT:    # ymm1 = mem[0,1,0,1]
 ; AVX2-FAST-NEXT:    vpermps %ymm0, %ymm1, %ymm1
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm4[0,1,2,3,4],ymm1[5,6,7]
 ; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm4 = ymm5[1,1,1,1,5,5,5,5]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1,2],ymm10[3],ymm4[4,5,6,7]
-; AVX2-FAST-NEXT:    vmovddup {{.*#+}} xmm5 = [5,3,5,3]
-; AVX2-FAST-NEXT:    # xmm5 = mem[0,0]
-; AVX2-FAST-NEXT:    vpermps %ymm3, %ymm5, %ymm3
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1],ymm4[2,3,4,5,6,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1,2],ymm12[3],ymm4[4,5,6,7]
+; AVX2-FAST-NEXT:    vpermps %ymm2, %ymm9, %ymm2
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1],ymm4[2,3,4,5,6,7]
 ; AVX2-FAST-NEXT:    vbroadcastf128 {{.*#+}} ymm4 = [0,3,1,7,0,3,1,7]
 ; AVX2-FAST-NEXT:    # ymm4 = mem[0,1,0,1]
 ; AVX2-FAST-NEXT:    vpermps %ymm0, %ymm4, %ymm0
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm3[0,1,2,3,4],ymm0[5,6,7]
-; AVX2-FAST-NEXT:    vmovaps %ymm2, (%rsi)
-; AVX2-FAST-NEXT:    vmovaps %ymm6, (%rdx)
-; AVX2-FAST-NEXT:    vmovaps %ymm8, (%rcx)
-; AVX2-FAST-NEXT:    vmovaps %ymm9, (%r8)
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3,4],ymm0[5,6,7]
+; AVX2-FAST-NEXT:    vmovaps %ymm3, (%rsi)
+; AVX2-FAST-NEXT:    vmovaps %ymm7, (%rdx)
+; AVX2-FAST-NEXT:    vmovaps %ymm10, (%rcx)
+; AVX2-FAST-NEXT:    vmovaps %ymm11, (%r8)
 ; AVX2-FAST-NEXT:    vmovaps %ymm1, (%r9)
 ; AVX2-FAST-NEXT:    vmovaps %ymm0, (%rax)
 ; AVX2-FAST-NEXT:    vzeroupper
@@ -743,82 +735,78 @@ define void @load_i32_stride6_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX2-FAST-PERLANE-NEXT:    vmovaps 128(%rdi), %ymm0
 ; AVX2-FAST-PERLANE-NEXT:    vmovaps 160(%rdi), %ymm1
 ; AVX2-FAST-PERLANE-NEXT:    vmovaps 96(%rdi), %ymm5
-; AVX2-FAST-PERLANE-NEXT:    vmovaps (%rdi), %ymm3
+; AVX2-FAST-PERLANE-NEXT:    vmovaps (%rdi), %ymm2
 ; AVX2-FAST-PERLANE-NEXT:    vmovaps 32(%rdi), %ymm4
-; AVX2-FAST-PERLANE-NEXT:    vmovaps 64(%rdi), %ymm7
-; AVX2-FAST-PERLANE-NEXT:    vmovaps {{.*#+}} xmm2 = <0,6,4,u>
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm6 = ymm3[0,1,2,3],ymm4[4,5],ymm3[6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm6, %ymm2, %ymm2
-; AVX2-FAST-PERLANE-NEXT:    vperm2f128 {{.*#+}} ymm8 = ymm7[0,1],ymm5[0,1]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],ymm5[6,7]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm9 = ymm8[0,2,2,2,4,6,6,6]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2],ymm9[3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm9 = ymm1[0,1,2,3],ymm0[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vbroadcastsd {{.*#+}} ymm10 = [4,2,4,2,4,2,4,2]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm9, %ymm10, %ymm10
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm10[6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovaps {{.*#+}} xmm10 = <1,7,5,u>
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm6, %ymm10, %ymm6
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm8 = ymm8[1,3,2,3,5,7,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm6 = ymm6[0,1,2],ymm8[3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vbroadcastsd {{.*#+}} ymm8 = [5,3,5,3,5,3,5,3]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm9, %ymm8, %ymm8
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm6 = ymm6[0,1,2,3,4,5],ymm8[6,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm8 = ymm7[0,1],ymm5[2,3],ymm7[4,5],ymm5[6,7]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm8 = ymm8[2,0,2,3,6,4,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm8 = ymm8[0,2,0,3]
-; AVX2-FAST-PERLANE-NEXT:    vmovaps (%rdi), %xmm9
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} xmm10 = xmm9[2,3,2,3]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm11 = ymm4[0,0,2,3,4,4,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm10 = ymm10[0],ymm11[1,2,3],ymm10[4],ymm11[5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm10 = ymm10[0,3,2,3]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm8 = ymm10[0,1,2],ymm8[3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm10 = ymm1[0,0,0,0,4,4,4,4]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm11 = ymm0[0,0,2,3,4,4,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm10 = ymm11[0,1,2],ymm10[3],ymm11[4,5,6],ymm10[7]
-; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm10 = ymm10[0,1,0,3]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm8 = ymm8[0,1,2,3,4],ymm10[5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm10 = ymm5[3,3,3,3,7,7,7,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm10 = ymm10[0],ymm7[1],ymm10[2,3,4],ymm7[5],ymm10[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovaps 64(%rdi), %ymm6
+; AVX2-FAST-PERLANE-NEXT:    vmovaps {{.*#+}} xmm3 = <0,6,4,u>
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm7 = ymm2[0,1,2,3],ymm4[4,5],ymm2[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm7, %ymm3, %ymm3
+; AVX2-FAST-PERLANE-NEXT:    vperm2f128 {{.*#+}} ymm8 = ymm6[0,1],ymm5[0,1]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm9 = ymm8[0,1,2,3,4,5],ymm5[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm8 = ymm9[0,2,2,2,4,6,6,6]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1,2],ymm8[3,4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm10 = ymm1[0,1,2,3],ymm0[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vbroadcastsd {{.*#+}} ymm8 = [4,2,4,2,4,2,4,2]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm10, %ymm8, %ymm11
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5],ymm11[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovaps {{.*#+}} xmm11 = <1,7,5,u>
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm7, %ymm11, %ymm7
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm9 = ymm9[1,3,2,3,5,7,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm7 = ymm7[0,1,2],ymm9[3,4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vbroadcastsd {{.*#+}} ymm9 = [5,3,5,3,5,3,5,3]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm10, %ymm9, %ymm10
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm7 = ymm7[0,1,2,3,4,5],ymm10[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm10 = ymm6[0,1],ymm5[2,3],ymm6[4,5],ymm5[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm10 = ymm10[2,0,2,3,6,4,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm10 = ymm10[0,2,0,3]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} xmm9 = xmm9[3,3,3,3]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm11 = ymm4[0,1,3,3,4,5,7,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm9 = ymm9[0],ymm11[1,2,3],ymm9[4],ymm11[5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm9 = ymm9[0,3,2,3]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm9 = ymm9[0,1,2],ymm10[3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm10 = ymm1[0,1,0,1,4,5,4,5]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm11 = ymm0[0,1,3,3,4,5,7,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm10 = ymm11[0,1,2],ymm10[3],ymm11[4,5,6],ymm10[7]
-; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm10 = ymm10[0,1,0,3]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm9 = ymm9[0,1,2,3,4],ymm10[5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovaps 80(%rdi), %xmm10
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm5 = ymm7[0,1,2,3],ymm5[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm7 = ymm5[0,1],ymm10[2,3],ymm5[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm7 = ymm7[0,1,0,2,4,5,4,6]
-; AVX2-FAST-PERLANE-NEXT:    vmovddup {{.*#+}} xmm11 = [4,2,4,2]
-; AVX2-FAST-PERLANE-NEXT:    # xmm11 = mem[0,0]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm3, %ymm11, %ymm4
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1],ymm7[2,3,4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovaps (%rdi), %xmm11
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} xmm12 = xmm11[2,3,2,3]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm13 = ymm4[0,0,2,3,4,4,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm12 = ymm12[0],ymm13[1,2,3],ymm12[4],ymm13[5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm12 = ymm12[0,3,2,3]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm10 = ymm12[0,1,2],ymm10[3,4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm12 = ymm1[0,0,0,0,4,4,4,4]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm13 = ymm0[0,0,2,3,4,4,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm12 = ymm13[0,1,2],ymm12[3],ymm13[4,5,6],ymm12[7]
+; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm12 = ymm12[0,1,0,3]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm10 = ymm10[0,1,2,3,4],ymm12[5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm12 = ymm5[3,3,3,3,7,7,7,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm12 = ymm12[0],ymm6[1],ymm12[2,3,4],ymm6[5],ymm12[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm12 = ymm12[0,2,0,3]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} xmm11 = xmm11[3,3,3,3]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm13 = ymm4[0,1,3,3,4,5,7,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm11 = ymm11[0],ymm13[1,2,3],ymm11[4],ymm13[5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm11 = ymm11[0,3,2,3]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm11 = ymm11[0,1,2],ymm12[3,4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm12 = ymm1[0,1,0,1,4,5,4,5]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm13 = ymm0[0,1,3,3,4,5,7,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm12 = ymm13[0,1,2],ymm12[3],ymm13[4,5,6],ymm12[7]
+; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm12 = ymm12[0,1,0,3]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm11 = ymm11[0,1,2,3,4],ymm12[5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovaps 80(%rdi), %xmm12
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm5 = ymm6[0,1,2,3],ymm5[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm6 = ymm5[0,1],ymm12[2,3],ymm5[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm6 = ymm6[0,1,0,2,4,5,4,6]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm4[0,1,2,3],ymm2[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm2, %ymm8, %ymm4
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1],ymm6[2,3,4,5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3],ymm1[4,5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vbroadcastf128 {{.*#+}} ymm1 = [0,2,0,6,0,2,0,6]
 ; AVX2-FAST-PERLANE-NEXT:    # ymm1 = mem[0,1,0,1]
 ; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm0, %ymm1, %ymm1
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm4[0,1,2,3,4],ymm1[5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm4 = ymm5[1,1,1,1,5,5,5,5]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1,2],ymm10[3],ymm4[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovddup {{.*#+}} xmm5 = [5,3,5,3]
-; AVX2-FAST-PERLANE-NEXT:    # xmm5 = mem[0,0]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm3, %ymm5, %ymm3
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1],ymm4[2,3,4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1,2],ymm12[3],ymm4[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm2, %ymm9, %ymm2
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1],ymm4[2,3,4,5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vbroadcastf128 {{.*#+}} ymm4 = [0,3,1,7,0,3,1,7]
 ; AVX2-FAST-PERLANE-NEXT:    # ymm4 = mem[0,1,0,1]
 ; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm0, %ymm4, %ymm0
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm3[0,1,2,3,4],ymm0[5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm2, (%rsi)
-; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm6, (%rdx)
-; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm8, (%rcx)
-; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm9, (%r8)
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3,4],ymm0[5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm3, (%rsi)
+; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm7, (%rdx)
+; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm10, (%rcx)
+; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm11, (%r8)
 ; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm1, (%r9)
 ; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm0, (%rax)
 ; AVX2-FAST-PERLANE-NEXT:    vzeroupper
@@ -1367,539 +1355,557 @@ define void @load_i32_stride6_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ;
 ; AVX2-SLOW-LABEL: load_i32_stride6_vf16:
 ; AVX2-SLOW:       # %bb.0:
-; AVX2-SLOW-NEXT:    subq $232, %rsp
-; AVX2-SLOW-NEXT:    vmovaps 288(%rdi), %ymm9
+; AVX2-SLOW-NEXT:    subq $360, %rsp # imm = 0x168
+; AVX2-SLOW-NEXT:    vmovaps 288(%rdi), %ymm8
 ; AVX2-SLOW-NEXT:    vmovaps 224(%rdi), %ymm11
-; AVX2-SLOW-NEXT:    vmovaps 192(%rdi), %ymm3
-; AVX2-SLOW-NEXT:    vmovups %ymm3, (%rsp) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vmovups %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vmovaps 192(%rdi), %ymm1
+; AVX2-SLOW-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-SLOW-NEXT:    vmovaps 128(%rdi), %ymm0
 ; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-SLOW-NEXT:    vmovaps 160(%rdi), %ymm2
 ; AVX2-SLOW-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vmovaps 96(%rdi), %ymm7
-; AVX2-SLOW-NEXT:    vmovaps (%rdi), %ymm1
-; AVX2-SLOW-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vmovaps 32(%rdi), %ymm12
-; AVX2-SLOW-NEXT:    vmovaps 64(%rdi), %ymm10
+; AVX2-SLOW-NEXT:    vmovaps 96(%rdi), %ymm15
+; AVX2-SLOW-NEXT:    vmovaps (%rdi), %ymm3
+; AVX2-SLOW-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vmovaps 32(%rdi), %ymm13
+; AVX2-SLOW-NEXT:    vmovaps 64(%rdi), %ymm6
 ; AVX2-SLOW-NEXT:    vmovaps {{.*#+}} xmm14 = <0,6,4,u>
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm5 = ymm1[0,1,2,3],ymm12[4,5],ymm1[6,7]
-; AVX2-SLOW-NEXT:    vpermps %ymm5, %ymm14, %ymm1
-; AVX2-SLOW-NEXT:    vperm2f128 {{.*#+}} ymm8 = ymm10[0,1],ymm7[0,1]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],ymm7[6,7]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm13 = ymm8[0,2,2,2,4,6,6,6]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2],ymm13[3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm13 = ymm2[0,1,2,3],ymm0[4,5,6,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm4 = ymm3[0,1,2,3],ymm13[4,5],ymm3[6,7]
+; AVX2-SLOW-NEXT:    vpermps %ymm4, %ymm14, %ymm3
+; AVX2-SLOW-NEXT:    vperm2f128 {{.*#+}} ymm7 = ymm6[0,1],ymm15[0,1]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm9 = ymm7[0,1,2,3,4,5],ymm15[6,7]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm7 = ymm9[0,2,2,2,4,6,6,6]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1,2],ymm7[3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm10 = ymm2[0,1,2,3],ymm0[4,5,6,7]
 ; AVX2-SLOW-NEXT:    vbroadcastsd {{.*#+}} ymm0 = [4,2,4,2,4,2,4,2]
-; AVX2-SLOW-NEXT:    vpermps %ymm13, %ymm0, %ymm15
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm15[6,7]
-; AVX2-SLOW-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm4 = ymm3[0,1,2,3],ymm11[4,5],ymm3[6,7]
-; AVX2-SLOW-NEXT:    vpermps %ymm4, %ymm14, %ymm14
-; AVX2-SLOW-NEXT:    vmovaps 256(%rdi), %ymm6
-; AVX2-SLOW-NEXT:    vperm2f128 {{.*#+}} ymm15 = ymm6[0,1],ymm9[0,1]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm15[0,1,2,3,4,5],ymm9[6,7]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm15 = ymm2[0,2,2,2,4,6,6,6]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm3 = ymm14[0,1,2],ymm15[3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vmovaps 320(%rdi), %ymm14
-; AVX2-SLOW-NEXT:    vmovaps 352(%rdi), %ymm15
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm15[0,1,2,3],ymm14[4,5,6,7]
-; AVX2-SLOW-NEXT:    vpermps %ymm1, %ymm0, %ymm0
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm3[0,1,2,3,4,5],ymm0[6,7]
+; AVX2-SLOW-NEXT:    vpermps %ymm10, %ymm0, %ymm12
+; AVX2-SLOW-NEXT:    vmovaps %ymm0, %ymm5
 ; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vmovaps {{.*#+}} xmm0 = <1,7,5,u>
-; AVX2-SLOW-NEXT:    vpermps %ymm5, %ymm0, %ymm3
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm5 = ymm8[1,3,2,3,5,7,6,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1,2],ymm5[3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm3[0,1,2,3,4,5],ymm12[6,7]
+; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm3 = ymm1[0,1,2,3],ymm11[4,5],ymm1[6,7]
+; AVX2-SLOW-NEXT:    vpermps %ymm3, %ymm14, %ymm12
+; AVX2-SLOW-NEXT:    vmovaps 256(%rdi), %ymm7
+; AVX2-SLOW-NEXT:    vperm2f128 {{.*#+}} ymm14 = ymm7[0,1],ymm8[0,1]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm14[0,1,2,3,4,5],ymm8[6,7]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm14 = ymm1[0,2,2,2,4,6,6,6]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm12[0,1,2],ymm14[3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vmovaps 320(%rdi), %ymm12
+; AVX2-SLOW-NEXT:    vmovaps 352(%rdi), %ymm14
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm14[0,1,2,3],ymm12[4,5,6,7]
+; AVX2-SLOW-NEXT:    vpermps %ymm0, %ymm5, %ymm11
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm11[6,7]
+; AVX2-SLOW-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vmovaps {{.*#+}} xmm2 = <1,7,5,u>
+; AVX2-SLOW-NEXT:    vpermps %ymm4, %ymm2, %ymm4
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm9 = ymm9[1,3,2,3,5,7,6,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1,2],ymm9[3,4,5,6,7]
 ; AVX2-SLOW-NEXT:    vbroadcastsd {{.*#+}} ymm5 = [5,3,5,3,5,3,5,3]
-; AVX2-SLOW-NEXT:    vpermps %ymm13, %ymm5, %ymm8
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5],ymm8[6,7]
-; AVX2-SLOW-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vpermps %ymm4, %ymm0, %ymm0
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm2 = ymm2[1,3,2,3,5,7,6,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm2[3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vpermps %ymm1, %ymm5, %ymm1
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
+; AVX2-SLOW-NEXT:    vmovups %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vpermps %ymm10, %ymm5, %ymm10
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1,2,3,4,5],ymm10[6,7]
+; AVX2-SLOW-NEXT:    vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vpermps %ymm3, %ymm2, %ymm2
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[1,3,2,3,5,7,6,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2],ymm1[3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vpermps %ymm0, %ymm5, %ymm0
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
 ; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vmovaps (%rdi), %xmm3
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm0 = xmm3[2,3,2,3]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm12[0,0,2,3,4,4,6,7]
+; AVX2-SLOW-NEXT:    vmovaps (%rdi), %xmm5
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm0 = xmm5[2,3,2,3]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm13[0,0,2,3,4,4,6,7]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm10[0,1],ymm7[2,3],ymm10[4,5],ymm7[6,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm6[0,1],ymm15[2,3],ymm6[4,5],ymm15[6,7]
 ; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[2,0,2,3,6,4,6,7]
 ; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,2,0,3]
 ; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,3,2,3]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm13[0,0,0,0,4,4,4,4]
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm2 = ymm8[0,0,2,3,4,4,6,7]
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm10[0,0,0,0,4,4,4,4]
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm2 = ymm4[0,0,2,3,4,4,6,7]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2],ymm1[3],ymm2[4,5,6],ymm1[7]
 ; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,1,0,3]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]
 ; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-SLOW-NEXT:    vmovaps 192(%rdi), %xmm0
 ; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm2 = ymm11[0,0,2,3,4,4,6,7]
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm2 = ymm9[0,0,2,3,4,4,6,7]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm2[1,2,3],ymm1[4],ymm2[5,6,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm6[0,1],ymm9[2,3],ymm6[4,5],ymm9[6,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm7[0,1],ymm8[2,3],ymm7[4,5],ymm8[6,7]
 ; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm2 = ymm2[2,0,2,3,6,4,6,7]
 ; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[0,2,0,3]
 ; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,3,2,3]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2],ymm2[3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm2 = ymm15[0,0,0,0,4,4,4,4]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm4 = ymm14[0,0,2,3,4,4,6,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm4[0,1,2],ymm2[3],ymm4[4,5,6],ymm2[7]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm2 = ymm14[0,0,0,0,4,4,4,4]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm3 = ymm12[0,0,2,3,4,4,6,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1,2],ymm2[3],ymm3[4,5,6],ymm2[7]
 ; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[0,1,0,3]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm2[5,6,7]
 ; AVX2-SLOW-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm1 = xmm3[3,3,3,3]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm2 = ymm12[0,1,3,3,4,5,7,7]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm1 = xmm5[3,3,3,3]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm2 = ymm13[0,1,3,3,4,5,7,7]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm2[1,2,3],ymm1[4],ymm2[5,6,7]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm2 = ymm7[3,3,3,3,7,7,7,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0],ymm10[1],ymm2[2,3,4],ymm10[5],ymm2[6,7]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm2 = ymm15[3,3,3,3,7,7,7,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0],ymm6[1],ymm2[2,3,4],ymm6[5],ymm2[6,7]
 ; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[0,2,0,3]
 ; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,3,2,3]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2],ymm2[3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm2 = ymm13[0,1,0,1,4,5,4,5]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm3 = ymm8[0,1,3,3,4,5,7,7]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm2 = ymm10[0,1,0,1,4,5,4,5]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm3 = ymm4[0,1,3,3,4,5,7,7]
+; AVX2-SLOW-NEXT:    vmovaps %ymm4, %ymm5
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1,2],ymm2[3],ymm3[4,5,6],ymm2[7]
 ; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[0,1,0,3]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm2[5,6,7]
-; AVX2-SLOW-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vmovups %ymm1, (%rsp) # 32-byte Spill
 ; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm11[0,1,3,3,4,5,7,7]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm9[0,1,3,3,4,5,7,7]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm9[3,3,3,3,7,7,7,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm6[1],ymm1[2,3,4],ymm6[5],ymm1[6,7]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm8[3,3,3,3,7,7,7,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm7[1],ymm1[2,3,4],ymm7[5],ymm1[6,7]
 ; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,2,0,3]
 ; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,3,2,3]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm15[0,1,0,1,4,5,4,5]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm2 = ymm14[0,1,3,3,4,5,7,7]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm14[0,1,0,1,4,5,4,5]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm2 = ymm12[0,1,3,3,4,5,7,7]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2],ymm1[3],ymm2[4,5,6],ymm1[7]
 ; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,1,0,3]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm3 = ymm0[0,1,2,3,4],ymm1[5,6,7]
-; AVX2-SLOW-NEXT:    vblendps $240, (%rsp), %ymm11, %ymm11 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm11 = ymm11[0,1,2,3],mem[4,5,6,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm6[0,1,2,3],ymm9[4,5,6,7]
-; AVX2-SLOW-NEXT:    vmovaps 272(%rdi), %xmm2
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm2[2,3],ymm1[4,5,6,7]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,1,0,2,4,5,4,6]
-; AVX2-SLOW-NEXT:    vmovddup {{.*#+}} xmm4 = [4,2,4,2]
-; AVX2-SLOW-NEXT:    # xmm4 = mem[0,0]
-; AVX2-SLOW-NEXT:    vpermps %ymm11, %ymm4, %ymm5
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm5[0,1],ymm0[2,3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm5 = ymm15[0,1],ymm14[2,3],ymm15[4,5,6,7]
-; AVX2-SLOW-NEXT:    vbroadcastf128 {{.*#+}} ymm6 = [0,2,0,6,0,2,0,6]
-; AVX2-SLOW-NEXT:    # ymm6 = mem[0,1,0,1]
-; AVX2-SLOW-NEXT:    vpermps %ymm5, %ymm6, %ymm9
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm9[5,6,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm7 = ymm10[0,1,2,3],ymm7[4,5,6,7]
-; AVX2-SLOW-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm12, %ymm9 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm9 = ymm12[0,1,2,3],mem[4,5,6,7]
-; AVX2-SLOW-NEXT:    vmovaps 80(%rdi), %xmm10
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm12 = ymm7[0,1],ymm10[2,3],ymm7[4,5,6,7]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm12 = ymm12[0,1,0,2,4,5,4,6]
-; AVX2-SLOW-NEXT:    vpermps %ymm9, %ymm4, %ymm4
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1],ymm12[2,3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm12 = ymm13[0,1],ymm8[2,3],ymm13[4,5,6,7]
-; AVX2-SLOW-NEXT:    vpermps %ymm12, %ymm6, %ymm6
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1,2,3,4],ymm6[5,6,7]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[1,1,1,1,5,5,5,5]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2],ymm2[3],ymm1[4,5,6,7]
-; AVX2-SLOW-NEXT:    vmovddup {{.*#+}} xmm2 = [5,3,5,3]
-; AVX2-SLOW-NEXT:    # xmm2 = mem[0,0]
-; AVX2-SLOW-NEXT:    vpermps %ymm11, %ymm2, %ymm6
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]
+; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm4 = ymm6[0,1,2,3],ymm15[4,5,6,7]
+; AVX2-SLOW-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm13, %ymm2 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm2 = ymm13[0,1,2,3],mem[4,5,6,7]
+; AVX2-SLOW-NEXT:    vmovaps 80(%rdi), %xmm3
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm4[0,1],ymm3[2,3],ymm4[4,5,6,7]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[0,1,0,2,4,5,4,6]
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vpermps %ymm2, %ymm0, %ymm6
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm6[0,1],ymm1[2,3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vbroadcastf128 {{.*#+}} ymm6 = [0,3,1,7,0,3,1,7]
-; AVX2-SLOW-NEXT:    # ymm6 = mem[0,1,0,1]
-; AVX2-SLOW-NEXT:    vpermps %ymm5, %ymm6, %ymm5
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm5[5,6,7]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm5 = ymm7[1,1,1,1,5,5,5,5]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0,1,2],ymm10[3],ymm5[4,5,6,7]
-; AVX2-SLOW-NEXT:    vpermps %ymm9, %ymm2, %ymm2
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1],ymm5[2,3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vpermps %ymm12, %ymm6, %ymm5
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm5[5,6,7]
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vmovaps %ymm5, 32(%rsi)
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vmovaps %ymm5, (%rsi)
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vmovaps %ymm5, 32(%rdx)
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vmovaps %ymm5, (%rdx)
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vmovaps %ymm5, 32(%rcx)
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vmovaps %ymm5, (%rcx)
-; AVX2-SLOW-NEXT:    vmovaps %ymm3, 32(%r8)
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm6 = ymm10[0,1],ymm5[2,3],ymm10[4,5,6,7]
+; AVX2-SLOW-NEXT:    vbroadcastf128 {{.*#+}} ymm11 = [0,2,0,6,0,2,0,6]
+; AVX2-SLOW-NEXT:    # ymm11 = mem[0,1,0,1]
+; AVX2-SLOW-NEXT:    vpermps %ymm6, %ymm11, %ymm13
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm13[5,6,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm5 = ymm7[0,1,2,3],ymm8[4,5,6,7]
+; AVX2-SLOW-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm7 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm7 = ymm9[0,1,2,3],mem[4,5,6,7]
+; AVX2-SLOW-NEXT:    vmovaps 272(%rdi), %xmm13
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm15 = ymm5[0,1],ymm13[2,3],ymm5[4,5,6,7]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm15 = ymm15[0,1,0,2,4,5,4,6]
+; AVX2-SLOW-NEXT:    vpermps %ymm7, %ymm0, %ymm8
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm8 = ymm8[0,1],ymm15[2,3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm12 = ymm14[0,1],ymm12[2,3],ymm14[4,5,6,7]
+; AVX2-SLOW-NEXT:    vpermps %ymm12, %ymm11, %ymm11
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm8 = ymm8[0,1,2,3,4],ymm11[5,6,7]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm0 = ymm4[1,1,1,1,5,5,5,5]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm3[3],ymm0[4,5,6,7]
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vpermps %ymm2, %ymm4, %ymm2
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm2[0,1],ymm0[2,3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vbroadcastf128 {{.*#+}} ymm2 = [0,3,1,7,0,3,1,7]
+; AVX2-SLOW-NEXT:    # ymm2 = mem[0,1,0,1]
+; AVX2-SLOW-NEXT:    vpermps %ymm6, %ymm2, %ymm3
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm3[5,6,7]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm3 = ymm5[1,1,1,1,5,5,5,5]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1,2],ymm13[3],ymm3[4,5,6,7]
+; AVX2-SLOW-NEXT:    vpermps %ymm7, %ymm4, %ymm5
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm3 = ymm5[0,1],ymm3[2,3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vpermps %ymm12, %ymm2, %ymm2
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5,6,7]
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vmovaps %ymm3, 32(%rsi)
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vmovaps %ymm3, (%rsi)
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vmovaps %ymm3, 32(%rdx)
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vmovaps %ymm3, (%rdx)
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vmovaps %ymm3, 32(%rcx)
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vmovaps %ymm3, (%rcx)
 ; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vmovaps %ymm3, 32(%r8)
+; AVX2-SLOW-NEXT:    vmovups (%rsp), %ymm3 # 32-byte Reload
 ; AVX2-SLOW-NEXT:    vmovaps %ymm3, (%r8)
-; AVX2-SLOW-NEXT:    vmovaps %ymm0, 32(%r9)
-; AVX2-SLOW-NEXT:    vmovaps %ymm4, (%r9)
+; AVX2-SLOW-NEXT:    vmovaps %ymm8, 32(%r9)
+; AVX2-SLOW-NEXT:    vmovaps %ymm1, (%r9)
 ; AVX2-SLOW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX2-SLOW-NEXT:    vmovaps %ymm1, 32(%rax)
-; AVX2-SLOW-NEXT:    vmovaps %ymm2, (%rax)
-; AVX2-SLOW-NEXT:    addq $232, %rsp
+; AVX2-SLOW-NEXT:    vmovaps %ymm2, 32(%rax)
+; AVX2-SLOW-NEXT:    vmovaps %ymm0, (%rax)
+; AVX2-SLOW-NEXT:    addq $360, %rsp # imm = 0x168
 ; AVX2-SLOW-NEXT:    vzeroupper
 ; AVX2-SLOW-NEXT:    retq
 ;
 ; AVX2-FAST-LABEL: load_i32_stride6_vf16:
 ; AVX2-FAST:       # %bb.0:
-; AVX2-FAST-NEXT:    subq $200, %rsp
-; AVX2-FAST-NEXT:    vmovaps 288(%rdi), %ymm9
-; AVX2-FAST-NEXT:    vmovaps 224(%rdi), %ymm11
-; AVX2-FAST-NEXT:    vmovaps 192(%rdi), %ymm6
-; AVX2-FAST-NEXT:    vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovaps 128(%rdi), %ymm0
-; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    subq $328, %rsp # imm = 0x148
+; AVX2-FAST-NEXT:    vmovaps 288(%rdi), %ymm10
+; AVX2-FAST-NEXT:    vmovaps 224(%rdi), %ymm14
+; AVX2-FAST-NEXT:    vmovups %ymm14, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vmovaps 192(%rdi), %ymm11
+; AVX2-FAST-NEXT:    vmovups %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vmovaps 128(%rdi), %ymm1
+; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-NEXT:    vmovaps 160(%rdi), %ymm2
 ; AVX2-FAST-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovaps 96(%rdi), %ymm7
+; AVX2-FAST-NEXT:    vmovaps 96(%rdi), %ymm15
 ; AVX2-FAST-NEXT:    vmovaps (%rdi), %ymm3
 ; AVX2-FAST-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovaps 32(%rdi), %ymm12
-; AVX2-FAST-NEXT:    vmovaps 64(%rdi), %ymm8
-; AVX2-FAST-NEXT:    vmovaps {{.*#+}} xmm1 = <0,6,4,u>
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm5 = ymm3[0,1,2,3],ymm12[4,5],ymm3[6,7]
-; AVX2-FAST-NEXT:    vpermps %ymm5, %ymm1, %ymm4
-; AVX2-FAST-NEXT:    vperm2f128 {{.*#+}} ymm10 = ymm8[0,1],ymm7[0,1]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm10 = ymm10[0,1,2,3,4,5],ymm7[6,7]
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm13 = ymm10[0,2,2,2,4,6,6,6]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1,2],ymm13[3,4,5,6,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm13 = ymm2[0,1,2,3],ymm0[4,5,6,7]
-; AVX2-FAST-NEXT:    vbroadcastsd {{.*#+}} ymm0 = [4,2,4,2,4,2,4,2]
-; AVX2-FAST-NEXT:    vpermps %ymm13, %ymm0, %ymm14
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm4[0,1,2,3,4,5],ymm14[6,7]
-; AVX2-FAST-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm4 = ymm6[0,1,2,3],ymm11[4,5],ymm6[6,7]
-; AVX2-FAST-NEXT:    vpermps %ymm4, %ymm1, %ymm14
-; AVX2-FAST-NEXT:    vmovaps 256(%rdi), %ymm6
-; AVX2-FAST-NEXT:    vperm2f128 {{.*#+}} ymm15 = ymm6[0,1],ymm9[0,1]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm15[0,1,2,3,4,5],ymm9[6,7]
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm15 = ymm2[0,2,2,2,4,6,6,6]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm3 = ymm14[0,1,2],ymm15[3,4,5,6,7]
-; AVX2-FAST-NEXT:    vmovaps 320(%rdi), %ymm14
-; AVX2-FAST-NEXT:    vmovaps 352(%rdi), %ymm15
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm15[0,1,2,3],ymm14[4,5,6,7]
-; AVX2-FAST-NEXT:    vpermps %ymm1, %ymm0, %ymm0
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm3[0,1,2,3,4,5],ymm0[6,7]
-; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovaps {{.*#+}} xmm0 = <1,7,5,u>
-; AVX2-FAST-NEXT:    vpermps %ymm5, %ymm0, %ymm3
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm5 = ymm10[1,3,2,3,5,7,6,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1,2],ymm5[3,4,5,6,7]
-; AVX2-FAST-NEXT:    vbroadcastsd {{.*#+}} ymm5 = [5,3,5,3,5,3,5,3]
-; AVX2-FAST-NEXT:    vpermps %ymm13, %ymm5, %ymm10
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5],ymm10[6,7]
+; AVX2-FAST-NEXT:    vmovaps 32(%rdi), %ymm13
+; AVX2-FAST-NEXT:    vmovaps 64(%rdi), %ymm6
+; AVX2-FAST-NEXT:    vmovaps {{.*#+}} xmm0 = <0,6,4,u>
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm4 = ymm3[0,1,2,3],ymm13[4,5],ymm3[6,7]
+; AVX2-FAST-NEXT:    vpermps %ymm4, %ymm0, %ymm7
+; AVX2-FAST-NEXT:    vperm2f128 {{.*#+}} ymm8 = ymm6[0,1],ymm15[0,1]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],ymm15[6,7]
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm9 = ymm8[0,2,2,2,4,6,6,6]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm9 = ymm7[0,1,2],ymm9[3,4,5,6,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm3 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-FAST-NEXT:    vbroadcastsd {{.*#+}} ymm1 = [4,2,4,2,4,2,4,2]
+; AVX2-FAST-NEXT:    vpermps %ymm3, %ymm1, %ymm12
+; AVX2-FAST-NEXT:    vmovaps %ymm1, %ymm5
+; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm9[0,1,2,3,4,5],ymm12[6,7]
+; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm11[0,1,2,3],ymm14[4,5],ymm11[6,7]
+; AVX2-FAST-NEXT:    vpermps %ymm2, %ymm0, %ymm9
+; AVX2-FAST-NEXT:    vmovaps 256(%rdi), %ymm7
+; AVX2-FAST-NEXT:    vperm2f128 {{.*#+}} ymm12 = ymm7[0,1],ymm10[0,1]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm12[0,1,2,3,4,5],ymm10[6,7]
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm12 = ymm1[0,2,2,2,4,6,6,6]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm9 = ymm9[0,1,2],ymm12[3,4,5,6,7]
+; AVX2-FAST-NEXT:    vmovaps 320(%rdi), %ymm12
+; AVX2-FAST-NEXT:    vmovaps 352(%rdi), %ymm14
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm14[0,1,2,3],ymm12[4,5,6,7]
+; AVX2-FAST-NEXT:    vpermps %ymm0, %ymm5, %ymm11
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm9 = ymm9[0,1,2,3,4,5],ymm11[6,7]
+; AVX2-FAST-NEXT:    vmovups %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vmovaps {{.*#+}} xmm11 = <1,7,5,u>
+; AVX2-FAST-NEXT:    vpermps %ymm4, %ymm11, %ymm4
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm8 = ymm8[1,3,2,3,5,7,6,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1,2],ymm8[3,4,5,6,7]
+; AVX2-FAST-NEXT:    vbroadcastsd {{.*#+}} ymm9 = [5,3,5,3,5,3,5,3]
+; AVX2-FAST-NEXT:    vpermps %ymm3, %ymm9, %ymm3
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5],ymm3[6,7]
 ; AVX2-FAST-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vpermps %ymm4, %ymm0, %ymm0
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm2 = ymm2[1,3,2,3,5,7,6,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm2[3,4,5,6,7]
-; AVX2-FAST-NEXT:    vpermps %ymm1, %ymm5, %ymm1
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
+; AVX2-FAST-NEXT:    vpermps %ymm2, %ymm11, %ymm2
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[1,3,2,3,5,7,6,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2],ymm1[3,4,5,6,7]
+; AVX2-FAST-NEXT:    vpermps %ymm0, %ymm9, %ymm0
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
 ; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-NEXT:    vmovaps (%rdi), %xmm0
 ; AVX2-FAST-NEXT:    vshufps {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm2 = ymm12[0,0,2,3,4,4,6,7]
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm2 = ymm13[0,0,2,3,4,4,6,7]
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm2[1,2,3],ymm1[4],ymm2[5,6,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm8[0,1],ymm7[2,3],ymm8[4,5],ymm7[6,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm6[0,1],ymm15[2,3],ymm6[4,5],ymm15[6,7]
 ; AVX2-FAST-NEXT:    vmovaps {{.*#+}} ymm3 = [2,0,6,4,2,0,6,7]
 ; AVX2-FAST-NEXT:    vpermps %ymm2, %ymm3, %ymm2
 ; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,3,2,3]
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2],ymm2[3,4,5,6,7]
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm2 = ymm10[0,0,0,0,4,4,4,4]
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm4 = ymm13[0,0,2,3,4,4,6,7]
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm2 = ymm8[0,0,0,0,4,4,4,4]
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm4 = ymm5[0,0,2,3,4,4,6,7]
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm4[0,1,2],ymm2[3],ymm4[4,5,6],ymm2[7]
 ; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[0,1,0,3]
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm2[5,6,7]
-; AVX2-FAST-NEXT:    vmovups %ymm1, (%rsp) # 32-byte Spill
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm6[0,1],ymm9[2,3],ymm6[4,5],ymm9[6,7]
+; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm7[0,1],ymm10[2,3],ymm7[4,5],ymm10[6,7]
 ; AVX2-FAST-NEXT:    vpermps %ymm1, %ymm3, %ymm1
 ; AVX2-FAST-NEXT:    vmovaps 192(%rdi), %xmm2
 ; AVX2-FAST-NEXT:    vshufps {{.*#+}} xmm3 = xmm2[2,3,2,3]
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload
 ; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm4 = ymm11[0,0,2,3,4,4,6,7]
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0],ymm4[1,2,3],ymm3[4],ymm4[5,6,7]
 ; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm3 = ymm3[0,3,2,3]
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm3[0,1,2],ymm1[3,4,5,6,7]
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm3 = ymm15[0,0,0,0,4,4,4,4]
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm4 = ymm14[0,0,2,3,4,4,6,7]
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm3 = ymm14[0,0,0,0,4,4,4,4]
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm4 = ymm12[0,0,2,3,4,4,6,7]
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm3 = ymm4[0,1,2],ymm3[3],ymm4[4,5,6],ymm3[7]
 ; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm3 = ymm3[0,1,0,3]
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm3[5,6,7]
 ; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm12[0,1,3,3,4,5,7,7]
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm13[0,1,3,3,4,5,7,7]
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7]
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm7[3,3,3,3,7,7,7,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm8[1],ymm1[2,3,4],ymm8[5],ymm1[6,7]
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm15[3,3,3,3,7,7,7,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm6[1],ymm1[2,3,4],ymm6[5],ymm1[6,7]
 ; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,2,0,3]
 ; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,3,2,3]
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm10[0,1,0,1,4,5,4,5]
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm3 = ymm13[0,1,3,3,4,5,7,7]
-; AVX2-FAST-NEXT:    vmovaps %ymm13, %ymm4
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm8[0,1,0,1,4,5,4,5]
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm3 = ymm5[0,1,3,3,4,5,7,7]
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm3[0,1,2],ymm1[3],ymm3[4,5,6],ymm1[7]
 ; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,1,0,3]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm13 = ymm0[0,1,2,3,4],ymm1[5,6,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]
+; AVX2-FAST-NEXT:    vmovups %ymm0, (%rsp) # 32-byte Spill
 ; AVX2-FAST-NEXT:    vshufps {{.*#+}} xmm0 = xmm2[3,3,3,3]
 ; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm11[0,1,3,3,4,5,7,7]
+; AVX2-FAST-NEXT:    vmovaps %ymm11, %ymm8
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7]
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm9[3,3,3,3,7,7,7,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm6[1],ymm1[2,3,4],ymm6[5],ymm1[6,7]
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm10[3,3,3,3,7,7,7,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm7[1],ymm1[2,3,4],ymm7[5],ymm1[6,7]
 ; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,2,0,3]
 ; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,3,2,3]
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm15[0,1,0,1,4,5,4,5]
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm2 = ymm14[0,1,3,3,4,5,7,7]
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm14[0,1,0,1,4,5,4,5]
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm2 = ymm12[0,1,3,3,4,5,7,7]
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2],ymm1[3],ymm2[4,5,6],ymm1[7]
 ; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,1,0,3]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm3 = ymm0[0,1,2,3,4],ymm1[5,6,7]
-; AVX2-FAST-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm11, %ymm11 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm11 = ymm11[0,1,2,3],mem[4,5,6,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm6[0,1,2,3],ymm9[4,5,6,7]
-; AVX2-FAST-NEXT:    vmovaps 272(%rdi), %xmm2
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm2[2,3],ymm1[4,5,6,7]
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,1,0,2,4,5,4,6]
-; AVX2-FAST-NEXT:    vmovddup {{.*#+}} xmm5 = [4,2,4,2]
-; AVX2-FAST-NEXT:    # xmm5 = mem[0,0]
-; AVX2-FAST-NEXT:    vpermps %ymm11, %ymm5, %ymm6
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm6[0,1],ymm0[2,3,4,5,6,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm6 = ymm15[0,1],ymm14[2,3],ymm15[4,5,6,7]
-; AVX2-FAST-NEXT:    vbroadcastf128 {{.*#+}} ymm9 = [0,2,0,6,0,2,0,6]
-; AVX2-FAST-NEXT:    # ymm9 = mem[0,1,0,1]
-; AVX2-FAST-NEXT:    vpermps %ymm6, %ymm9, %ymm14
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm14[5,6,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm7 = ymm8[0,1,2,3],ymm7[4,5,6,7]
-; AVX2-FAST-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm12, %ymm8 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm8 = ymm12[0,1,2,3],mem[4,5,6,7]
-; AVX2-FAST-NEXT:    vmovaps 80(%rdi), %xmm12
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm14 = ymm7[0,1],ymm12[2,3],ymm7[4,5,6,7]
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm14 = ymm14[0,1,0,2,4,5,4,6]
-; AVX2-FAST-NEXT:    vpermps %ymm8, %ymm5, %ymm5
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0,1],ymm14[2,3,4,5,6,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm14 = ymm10[0,1],ymm4[2,3],ymm10[4,5,6,7]
-; AVX2-FAST-NEXT:    vpermps %ymm14, %ymm9, %ymm9
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0,1,2,3,4],ymm9[5,6,7]
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[1,1,1,1,5,5,5,5]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2],ymm2[3],ymm1[4,5,6,7]
-; AVX2-FAST-NEXT:    vmovddup {{.*#+}} xmm2 = [5,3,5,3]
-; AVX2-FAST-NEXT:    # xmm2 = mem[0,0]
-; AVX2-FAST-NEXT:    vpermps %ymm11, %ymm2, %ymm9
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm9[0,1],ymm1[2,3,4,5,6,7]
-; AVX2-FAST-NEXT:    vbroadcastf128 {{.*#+}} ymm9 = [0,3,1,7,0,3,1,7]
-; AVX2-FAST-NEXT:    # ymm9 = mem[0,1,0,1]
-; AVX2-FAST-NEXT:    vpermps %ymm6, %ymm9, %ymm6
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm6[5,6,7]
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm6 = ymm7[1,1,1,1,5,5,5,5]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm6 = ymm6[0,1,2],ymm12[3],ymm6[4,5,6,7]
-; AVX2-FAST-NEXT:    vpermps %ymm8, %ymm2, %ymm2
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1],ymm6[2,3,4,5,6,7]
-; AVX2-FAST-NEXT:    vpermps %ymm14, %ymm9, %ymm6
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm6[5,6,7]
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
-; AVX2-FAST-NEXT:    vmovaps %ymm6, 32(%rsi)
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
-; AVX2-FAST-NEXT:    vmovaps %ymm6, (%rsi)
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
-; AVX2-FAST-NEXT:    vmovaps %ymm4, 32(%rdx)
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
-; AVX2-FAST-NEXT:    vmovaps %ymm6, (%rdx)
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
-; AVX2-FAST-NEXT:    vmovaps %ymm4, 32(%rcx)
-; AVX2-FAST-NEXT:    vmovups (%rsp), %ymm4 # 32-byte Reload
-; AVX2-FAST-NEXT:    vmovaps %ymm4, (%rcx)
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]
+; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm4 = ymm6[0,1,2,3],ymm15[4,5,6,7]
+; AVX2-FAST-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm13, %ymm2 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm2 = ymm13[0,1,2,3],mem[4,5,6,7]
+; AVX2-FAST-NEXT:    vmovaps 80(%rdi), %xmm3
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm4[0,1],ymm3[2,3],ymm4[4,5,6,7]
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[0,1,0,2,4,5,4,6]
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX2-FAST-NEXT:    vpermps %ymm2, %ymm0, %ymm6
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm6[0,1],ymm1[2,3,4,5,6,7]
+; AVX2-FAST-NEXT:    vblendps $243, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm6 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm6 = mem[0,1],ymm5[2,3],mem[4,5,6,7]
+; AVX2-FAST-NEXT:    vbroadcastf128 {{.*#+}} ymm11 = [0,2,0,6,0,2,0,6]
+; AVX2-FAST-NEXT:    # ymm11 = mem[0,1,0,1]
+; AVX2-FAST-NEXT:    vpermps %ymm6, %ymm11, %ymm13
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm13[5,6,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm5 = ymm7[0,1,2,3],ymm10[4,5,6,7]
+; AVX2-FAST-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm7 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm7 = ymm8[0,1,2,3],mem[4,5,6,7]
+; AVX2-FAST-NEXT:    vmovaps 272(%rdi), %xmm13
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm15 = ymm5[0,1],ymm13[2,3],ymm5[4,5,6,7]
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm15 = ymm15[0,1,0,2,4,5,4,6]
+; AVX2-FAST-NEXT:    vpermps %ymm7, %ymm0, %ymm10
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm10 = ymm10[0,1],ymm15[2,3,4,5,6,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm12 = ymm14[0,1],ymm12[2,3],ymm14[4,5,6,7]
+; AVX2-FAST-NEXT:    vpermps %ymm12, %ymm11, %ymm11
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm10 = ymm10[0,1,2,3,4],ymm11[5,6,7]
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm0 = ymm4[1,1,1,1,5,5,5,5]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm3[3],ymm0[4,5,6,7]
+; AVX2-FAST-NEXT:    vpermps %ymm2, %ymm9, %ymm2
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm2[0,1],ymm0[2,3,4,5,6,7]
+; AVX2-FAST-NEXT:    vbroadcastf128 {{.*#+}} ymm2 = [0,3,1,7,0,3,1,7]
+; AVX2-FAST-NEXT:    # ymm2 = mem[0,1,0,1]
+; AVX2-FAST-NEXT:    vpermps %ymm6, %ymm2, %ymm3
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm3[5,6,7]
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm3 = ymm5[1,1,1,1,5,5,5,5]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1,2],ymm13[3],ymm3[4,5,6,7]
+; AVX2-FAST-NEXT:    vpermps %ymm7, %ymm9, %ymm5
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm3 = ymm5[0,1],ymm3[2,3,4,5,6,7]
+; AVX2-FAST-NEXT:    vpermps %ymm12, %ymm2, %ymm2
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5,6,7]
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
+; AVX2-FAST-NEXT:    vmovaps %ymm3, 32(%rsi)
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
+; AVX2-FAST-NEXT:    vmovaps %ymm3, (%rsi)
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
+; AVX2-FAST-NEXT:    vmovaps %ymm3, 32(%rdx)
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
+; AVX2-FAST-NEXT:    vmovaps %ymm3, (%rdx)
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
+; AVX2-FAST-NEXT:    vmovaps %ymm3, 32(%rcx)
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
+; AVX2-FAST-NEXT:    vmovaps %ymm3, (%rcx)
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
 ; AVX2-FAST-NEXT:    vmovaps %ymm3, 32(%r8)
-; AVX2-FAST-NEXT:    vmovaps %ymm13, (%r8)
-; AVX2-FAST-NEXT:    vmovaps %ymm0, 32(%r9)
-; AVX2-FAST-NEXT:    vmovaps %ymm5, (%r9)
+; AVX2-FAST-NEXT:    vmovups (%rsp), %ymm3 # 32-byte Reload
+; AVX2-FAST-NEXT:    vmovaps %ymm3, (%r8)
+; AVX2-FAST-NEXT:    vmovaps %ymm10, 32(%r9)
+; AVX2-FAST-NEXT:    vmovaps %ymm1, (%r9)
 ; AVX2-FAST-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX2-FAST-NEXT:    vmovaps %ymm1, 32(%rax)
-; AVX2-FAST-NEXT:    vmovaps %ymm2, (%rax)
-; AVX2-FAST-NEXT:    addq $200, %rsp
+; AVX2-FAST-NEXT:    vmovaps %ymm2, 32(%rax)
+; AVX2-FAST-NEXT:    vmovaps %ymm0, (%rax)
+; AVX2-FAST-NEXT:    addq $328, %rsp # imm = 0x148
 ; AVX2-FAST-NEXT:    vzeroupper
 ; AVX2-FAST-NEXT:    retq
 ;
 ; AVX2-FAST-PERLANE-LABEL: load_i32_stride6_vf16:
 ; AVX2-FAST-PERLANE:       # %bb.0:
-; AVX2-FAST-PERLANE-NEXT:    subq $232, %rsp
-; AVX2-FAST-PERLANE-NEXT:    vmovaps 288(%rdi), %ymm9
+; AVX2-FAST-PERLANE-NEXT:    subq $360, %rsp # imm = 0x168
+; AVX2-FAST-PERLANE-NEXT:    vmovaps 288(%rdi), %ymm8
 ; AVX2-FAST-PERLANE-NEXT:    vmovaps 224(%rdi), %ymm11
-; AVX2-FAST-PERLANE-NEXT:    vmovaps 192(%rdi), %ymm3
-; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm3, (%rsp) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vmovaps 192(%rdi), %ymm1
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-PERLANE-NEXT:    vmovaps 128(%rdi), %ymm0
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-PERLANE-NEXT:    vmovaps 160(%rdi), %ymm2
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vmovaps 96(%rdi), %ymm7
-; AVX2-FAST-PERLANE-NEXT:    vmovaps (%rdi), %ymm1
-; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vmovaps 32(%rdi), %ymm12
-; AVX2-FAST-PERLANE-NEXT:    vmovaps 64(%rdi), %ymm10
+; AVX2-FAST-PERLANE-NEXT:    vmovaps 96(%rdi), %ymm15
+; AVX2-FAST-PERLANE-NEXT:    vmovaps (%rdi), %ymm3
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vmovaps 32(%rdi), %ymm13
+; AVX2-FAST-PERLANE-NEXT:    vmovaps 64(%rdi), %ymm6
 ; AVX2-FAST-PERLANE-NEXT:    vmovaps {{.*#+}} xmm14 = <0,6,4,u>
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm5 = ymm1[0,1,2,3],ymm12[4,5],ymm1[6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm5, %ymm14, %ymm1
-; AVX2-FAST-PERLANE-NEXT:    vperm2f128 {{.*#+}} ymm8 = ymm10[0,1],ymm7[0,1]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],ymm7[6,7]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm13 = ymm8[0,2,2,2,4,6,6,6]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2],ymm13[3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm13 = ymm2[0,1,2,3],ymm0[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm4 = ymm3[0,1,2,3],ymm13[4,5],ymm3[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm4, %ymm14, %ymm3
+; AVX2-FAST-PERLANE-NEXT:    vperm2f128 {{.*#+}} ymm7 = ymm6[0,1],ymm15[0,1]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm9 = ymm7[0,1,2,3,4,5],ymm15[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm7 = ymm9[0,2,2,2,4,6,6,6]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1,2],ymm7[3,4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm10 = ymm2[0,1,2,3],ymm0[4,5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vbroadcastsd {{.*#+}} ymm0 = [4,2,4,2,4,2,4,2]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm13, %ymm0, %ymm15
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm15[6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm4 = ymm3[0,1,2,3],ymm11[4,5],ymm3[6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm4, %ymm14, %ymm14
-; AVX2-FAST-PERLANE-NEXT:    vmovaps 256(%rdi), %ymm6
-; AVX2-FAST-PERLANE-NEXT:    vperm2f128 {{.*#+}} ymm15 = ymm6[0,1],ymm9[0,1]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm15[0,1,2,3,4,5],ymm9[6,7]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm15 = ymm2[0,2,2,2,4,6,6,6]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm3 = ymm14[0,1,2],ymm15[3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovaps 320(%rdi), %ymm14
-; AVX2-FAST-PERLANE-NEXT:    vmovaps 352(%rdi), %ymm15
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm15[0,1,2,3],ymm14[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm1, %ymm0, %ymm0
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm3[0,1,2,3,4,5],ymm0[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm10, %ymm0, %ymm12
+; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm0, %ymm5
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vmovaps {{.*#+}} xmm0 = <1,7,5,u>
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm5, %ymm0, %ymm3
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm5 = ymm8[1,3,2,3,5,7,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1,2],ymm5[3,4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm3[0,1,2,3,4,5],ymm12[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm3 = ymm1[0,1,2,3],ymm11[4,5],ymm1[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm3, %ymm14, %ymm12
+; AVX2-FAST-PERLANE-NEXT:    vmovaps 256(%rdi), %ymm7
+; AVX2-FAST-PERLANE-NEXT:    vperm2f128 {{.*#+}} ymm14 = ymm7[0,1],ymm8[0,1]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm14[0,1,2,3,4,5],ymm8[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm14 = ymm1[0,2,2,2,4,6,6,6]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm12[0,1,2],ymm14[3,4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovaps 320(%rdi), %ymm12
+; AVX2-FAST-PERLANE-NEXT:    vmovaps 352(%rdi), %ymm14
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm14[0,1,2,3],ymm12[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm0, %ymm5, %ymm11
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm11[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vmovaps {{.*#+}} xmm2 = <1,7,5,u>
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm4, %ymm2, %ymm4
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm9 = ymm9[1,3,2,3,5,7,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1,2],ymm9[3,4,5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vbroadcastsd {{.*#+}} ymm5 = [5,3,5,3,5,3,5,3]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm13, %ymm5, %ymm8
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5],ymm8[6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm4, %ymm0, %ymm0
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm2 = ymm2[1,3,2,3,5,7,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm2[3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm1, %ymm5, %ymm1
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm10, %ymm5, %ymm10
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1,2,3,4,5],ymm10[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm3, %ymm2, %ymm2
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[1,3,2,3,5,7,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2],ymm1[3,4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm0, %ymm5, %ymm0
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vmovaps (%rdi), %xmm3
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} xmm0 = xmm3[2,3,2,3]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm12[0,0,2,3,4,4,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovaps (%rdi), %xmm5
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} xmm0 = xmm5[2,3,2,3]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm13[0,0,2,3,4,4,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm10[0,1],ymm7[2,3],ymm10[4,5],ymm7[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm6[0,1],ymm15[2,3],ymm6[4,5],ymm15[6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[2,0,2,3,6,4,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,2,0,3]
 ; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,3,2,3]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm13[0,0,0,0,4,4,4,4]
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm2 = ymm8[0,0,2,3,4,4,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm10[0,0,0,0,4,4,4,4]
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm2 = ymm4[0,0,2,3,4,4,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2],ymm1[3],ymm2[4,5,6],ymm1[7]
 ; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,1,0,3]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-PERLANE-NEXT:    vmovaps 192(%rdi), %xmm0
 ; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm2 = ymm11[0,0,2,3,4,4,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm2 = ymm9[0,0,2,3,4,4,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm2[1,2,3],ymm1[4],ymm2[5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm6[0,1],ymm9[2,3],ymm6[4,5],ymm9[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm7[0,1],ymm8[2,3],ymm7[4,5],ymm8[6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm2 = ymm2[2,0,2,3,6,4,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[0,2,0,3]
 ; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,3,2,3]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2],ymm2[3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm2 = ymm15[0,0,0,0,4,4,4,4]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm4 = ymm14[0,0,2,3,4,4,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm4[0,1,2],ymm2[3],ymm4[4,5,6],ymm2[7]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm2 = ymm14[0,0,0,0,4,4,4,4]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm3 = ymm12[0,0,2,3,4,4,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1,2],ymm2[3],ymm3[4,5,6],ymm2[7]
 ; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[0,1,0,3]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm2[5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} xmm1 = xmm3[3,3,3,3]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm2 = ymm12[0,1,3,3,4,5,7,7]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} xmm1 = xmm5[3,3,3,3]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm2 = ymm13[0,1,3,3,4,5,7,7]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm2[1,2,3],ymm1[4],ymm2[5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm2 = ymm7[3,3,3,3,7,7,7,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0],ymm10[1],ymm2[2,3,4],ymm10[5],ymm2[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm2 = ymm15[3,3,3,3,7,7,7,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0],ymm6[1],ymm2[2,3,4],ymm6[5],ymm2[6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[0,2,0,3]
 ; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,3,2,3]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2],ymm2[3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm2 = ymm13[0,1,0,1,4,5,4,5]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm3 = ymm8[0,1,3,3,4,5,7,7]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm2 = ymm10[0,1,0,1,4,5,4,5]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm3 = ymm4[0,1,3,3,4,5,7,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm4, %ymm5
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1,2],ymm2[3],ymm3[4,5,6],ymm2[7]
 ; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[0,1,0,3]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm2[5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm1, (%rsp) # 32-byte Spill
 ; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm11[0,1,3,3,4,5,7,7]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm9[0,1,3,3,4,5,7,7]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm9[3,3,3,3,7,7,7,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm6[1],ymm1[2,3,4],ymm6[5],ymm1[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm8[3,3,3,3,7,7,7,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm7[1],ymm1[2,3,4],ymm7[5],ymm1[6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,2,0,3]
 ; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,3,2,3]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm15[0,1,0,1,4,5,4,5]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm2 = ymm14[0,1,3,3,4,5,7,7]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm14[0,1,0,1,4,5,4,5]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm2 = ymm12[0,1,3,3,4,5,7,7]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2],ymm1[3],ymm2[4,5,6],ymm1[7]
 ; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,1,0,3]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm3 = ymm0[0,1,2,3,4],ymm1[5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps $240, (%rsp), %ymm11, %ymm11 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm11 = ymm11[0,1,2,3],mem[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm6[0,1,2,3],ymm9[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovaps 272(%rdi), %xmm2
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm2[2,3],ymm1[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,1,0,2,4,5,4,6]
-; AVX2-FAST-PERLANE-NEXT:    vmovddup {{.*#+}} xmm4 = [4,2,4,2]
-; AVX2-FAST-PERLANE-NEXT:    # xmm4 = mem[0,0]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm11, %ymm4, %ymm5
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm5[0,1],ymm0[2,3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm5 = ymm15[0,1],ymm14[2,3],ymm15[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vbroadcastf128 {{.*#+}} ymm6 = [0,2,0,6,0,2,0,6]
-; AVX2-FAST-PERLANE-NEXT:    # ymm6 = mem[0,1,0,1]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm5, %ymm6, %ymm9
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm9[5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm7 = ymm10[0,1,2,3],ymm7[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm12, %ymm9 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm9 = ymm12[0,1,2,3],mem[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovaps 80(%rdi), %xmm10
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm12 = ymm7[0,1],ymm10[2,3],ymm7[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm12 = ymm12[0,1,0,2,4,5,4,6]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm9, %ymm4, %ymm4
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1],ymm12[2,3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm12 = ymm13[0,1],ymm8[2,3],ymm13[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm12, %ymm6, %ymm6
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1,2,3,4],ymm6[5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[1,1,1,1,5,5,5,5]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2],ymm2[3],ymm1[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovddup {{.*#+}} xmm2 = [5,3,5,3]
-; AVX2-FAST-PERLANE-NEXT:    # xmm2 = mem[0,0]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm11, %ymm2, %ymm6
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm4 = ymm6[0,1,2,3],ymm15[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm13, %ymm2 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm2 = ymm13[0,1,2,3],mem[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovaps 80(%rdi), %xmm3
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm4[0,1],ymm3[2,3],ymm4[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[0,1,0,2,4,5,4,6]
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm2, %ymm0, %ymm6
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm6[0,1],ymm1[2,3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vbroadcastf128 {{.*#+}} ymm6 = [0,3,1,7,0,3,1,7]
-; AVX2-FAST-PERLANE-NEXT:    # ymm6 = mem[0,1,0,1]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm5, %ymm6, %ymm5
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm5[5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm5 = ymm7[1,1,1,1,5,5,5,5]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0,1,2],ymm10[3],ymm5[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm9, %ymm2, %ymm2
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1],ymm5[2,3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm12, %ymm6, %ymm5
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm5[5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm5, 32(%rsi)
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm5, (%rsi)
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm5, 32(%rdx)
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm5, (%rdx)
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm5, 32(%rcx)
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm5, (%rcx)
-; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm3, 32(%r8)
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm6 = ymm10[0,1],ymm5[2,3],ymm10[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vbroadcastf128 {{.*#+}} ymm11 = [0,2,0,6,0,2,0,6]
+; AVX2-FAST-PERLANE-NEXT:    # ymm11 = mem[0,1,0,1]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm6, %ymm11, %ymm13
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm13[5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm5 = ymm7[0,1,2,3],ymm8[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm7 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm7 = ymm9[0,1,2,3],mem[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovaps 272(%rdi), %xmm13
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm15 = ymm5[0,1],ymm13[2,3],ymm5[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm15 = ymm15[0,1,0,2,4,5,4,6]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm7, %ymm0, %ymm8
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm8 = ymm8[0,1],ymm15[2,3,4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm12 = ymm14[0,1],ymm12[2,3],ymm14[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm12, %ymm11, %ymm11
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm8 = ymm8[0,1,2,3,4],ymm11[5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm0 = ymm4[1,1,1,1,5,5,5,5]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm3[3],ymm0[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm2, %ymm4, %ymm2
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm2[0,1],ymm0[2,3,4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vbroadcastf128 {{.*#+}} ymm2 = [0,3,1,7,0,3,1,7]
+; AVX2-FAST-PERLANE-NEXT:    # ymm2 = mem[0,1,0,1]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm6, %ymm2, %ymm3
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm3[5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm3 = ymm5[1,1,1,1,5,5,5,5]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1,2],ymm13[3],ymm3[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm7, %ymm4, %ymm5
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm3 = ymm5[0,1],ymm3[2,3,4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm12, %ymm2, %ymm2
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm3, 32(%rsi)
 ; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm3, (%rsi)
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm3, 32(%rdx)
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm3, (%rdx)
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm3, 32(%rcx)
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm3, (%rcx)
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm3, 32(%r8)
+; AVX2-FAST-PERLANE-NEXT:    vmovups (%rsp), %ymm3 # 32-byte Reload
 ; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm3, (%r8)
-; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm0, 32(%r9)
-; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm4, (%r9)
+; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm8, 32(%r9)
+; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm1, (%r9)
 ; AVX2-FAST-PERLANE-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm1, 32(%rax)
-; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm2, (%rax)
-; AVX2-FAST-PERLANE-NEXT:    addq $232, %rsp
+; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm2, 32(%rax)
+; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm0, (%rax)
+; AVX2-FAST-PERLANE-NEXT:    addq $360, %rsp # imm = 0x168
 ; AVX2-FAST-PERLANE-NEXT:    vzeroupper
 ; AVX2-FAST-PERLANE-NEXT:    retq
 ;
@@ -3151,12 +3157,12 @@ define void @load_i32_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ;
 ; AVX2-SLOW-LABEL: load_i32_stride6_vf32:
 ; AVX2-SLOW:       # %bb.0:
-; AVX2-SLOW-NEXT:    subq $1160, %rsp # imm = 0x488
-; AVX2-SLOW-NEXT:    vmovaps 480(%rdi), %ymm6
+; AVX2-SLOW-NEXT:    subq $1256, %rsp # imm = 0x4E8
+; AVX2-SLOW-NEXT:    vmovaps 480(%rdi), %ymm9
 ; AVX2-SLOW-NEXT:    vmovaps 448(%rdi), %ymm10
 ; AVX2-SLOW-NEXT:    vmovups %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vmovaps 416(%rdi), %ymm11
-; AVX2-SLOW-NEXT:    vmovups %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vmovaps 416(%rdi), %ymm7
+; AVX2-SLOW-NEXT:    vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-SLOW-NEXT:    vmovaps 128(%rdi), %ymm2
 ; AVX2-SLOW-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-SLOW-NEXT:    vmovaps 160(%rdi), %ymm3
@@ -3169,411 +3175,410 @@ define void @load_i32_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX2-SLOW-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-SLOW-NEXT:    vmovaps 64(%rdi), %ymm5
 ; AVX2-SLOW-NEXT:    vmovups %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vmovaps {{.*#+}} xmm12 = <0,6,4,u>
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm9 = ymm0[0,1,2,3],ymm1[4,5],ymm0[6,7]
-; AVX2-SLOW-NEXT:    vpermps %ymm9, %ymm12, %ymm0
+; AVX2-SLOW-NEXT:    vmovaps {{.*#+}} xmm8 = <0,6,4,u>
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm13 = ymm0[0,1,2,3],ymm1[4,5],ymm0[6,7]
+; AVX2-SLOW-NEXT:    vpermps %ymm13, %ymm8, %ymm0
 ; AVX2-SLOW-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm5[0,1],ymm4[0,1]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm7 = ymm1[0,1,2,3,4,5],ymm4[6,7]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm7[0,2,2,2,4,6,6,6]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm0[0,1,2],ymm1[3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm8 = ymm3[0,1,2,3],ymm2[4,5,6,7]
-; AVX2-SLOW-NEXT:    vbroadcastsd {{.*#+}} ymm4 = [4,2,4,2,4,2,4,2]
-; AVX2-SLOW-NEXT:    vpermps %ymm8, %ymm4, %ymm2
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm2[6,7]
-; AVX2-SLOW-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm10[0,1],ymm6[0,1]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm6 = ymm1[0,1,2,3,4,5],ymm6[6,7]
-; AVX2-SLOW-NEXT:    vmovaps 384(%rdi), %ymm0
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm5 = ymm1[0,1,2,3,4,5],ymm4[6,7]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm5[0,2,2,2,4,6,6,6]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm6 = ymm3[0,1,2,3],ymm2[4,5,6,7]
+; AVX2-SLOW-NEXT:    vbroadcastsd {{.*#+}} ymm2 = [4,2,4,2,4,2,4,2]
+; AVX2-SLOW-NEXT:    vpermps %ymm6, %ymm2, %ymm1
+; AVX2-SLOW-NEXT:    vmovaps %ymm2, %ymm11
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
 ; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm5 = ymm0[0,1,2,3],ymm11[4,5],ymm0[6,7]
-; AVX2-SLOW-NEXT:    vpermps %ymm5, %ymm12, %ymm1
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm2 = ymm6[0,2,2,2,4,6,6,6]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2],ymm2[3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vmovaps 512(%rdi), %ymm0
+; AVX2-SLOW-NEXT:    vmovups %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm10[0,1],ymm9[0,1]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm3 = ymm0[0,1,2,3,4,5],ymm9[6,7]
+; AVX2-SLOW-NEXT:    vmovaps 384(%rdi), %ymm0
 ; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vmovaps 544(%rdi), %ymm2
-; AVX2-SLOW-NEXT:    vmovups %ymm2, (%rsp) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm11 = ymm2[0,1,2,3],ymm0[4,5,6,7]
-; AVX2-SLOW-NEXT:    vpermps %ymm11, %ymm4, %ymm2
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm2[6,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm4 = ymm0[0,1,2,3],ymm7[4,5],ymm0[6,7]
+; AVX2-SLOW-NEXT:    vpermps %ymm4, %ymm8, %ymm0
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm3[0,2,2,2,4,6,6,6]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vmovaps 512(%rdi), %ymm1
 ; AVX2-SLOW-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vmovaps 288(%rdi), %ymm0
+; AVX2-SLOW-NEXT:    vmovaps 544(%rdi), %ymm2
+; AVX2-SLOW-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-SLOW-NEXT:    vpermps %ymm2, %ymm11, %ymm1
+; AVX2-SLOW-NEXT:    vmovaps %ymm11, %ymm12
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
 ; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vmovaps 256(%rdi), %ymm1
+; AVX2-SLOW-NEXT:    vmovaps 288(%rdi), %ymm1
 ; AVX2-SLOW-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm1[0,1],ymm0[0,1]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm3 = ymm1[0,1,2,3,4,5],ymm0[6,7]
-; AVX2-SLOW-NEXT:    vmovaps 224(%rdi), %ymm0
+; AVX2-SLOW-NEXT:    vmovaps 256(%rdi), %ymm0
 ; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vmovaps 192(%rdi), %ymm1
+; AVX2-SLOW-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[0,1],ymm1[0,1]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm10 = ymm0[0,1,2,3,4,5],ymm1[6,7]
+; AVX2-SLOW-NEXT:    vmovaps 224(%rdi), %ymm1
 ; AVX2-SLOW-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm1[0,1,2,3],ymm0[4,5],ymm1[6,7]
-; AVX2-SLOW-NEXT:    vpermps %ymm2, %ymm12, %ymm10
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm15 = ymm3[0,2,2,2,4,6,6,6]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm15 = ymm10[0,1,2],ymm15[3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vmovaps 320(%rdi), %ymm0
+; AVX2-SLOW-NEXT:    vmovaps 192(%rdi), %ymm0
 ; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vmovaps 352(%rdi), %ymm1
-; AVX2-SLOW-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm10 = ymm1[0,1,2,3],ymm0[4,5,6,7]
-; AVX2-SLOW-NEXT:    vpermps %ymm10, %ymm4, %ymm14
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm15[0,1,2,3,4,5],ymm14[6,7]
-; AVX2-SLOW-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vmovaps 608(%rdi), %ymm0
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5],ymm0[6,7]
+; AVX2-SLOW-NEXT:    vpermps %ymm1, %ymm8, %ymm14
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm9 = ymm10[0,2,2,2,4,6,6,6]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm9 = ymm14[0,1,2],ymm9[3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vmovaps 320(%rdi), %ymm0
 ; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vmovaps 576(%rdi), %ymm1
-; AVX2-SLOW-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm14 = ymm1[0,1,2,3],ymm0[4,5],ymm1[6,7]
-; AVX2-SLOW-NEXT:    vpermps %ymm14, %ymm12, %ymm15
-; AVX2-SLOW-NEXT:    vmovaps 672(%rdi), %ymm0
+; AVX2-SLOW-NEXT:    vmovaps 352(%rdi), %ymm7
+; AVX2-SLOW-NEXT:    vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm14 = ymm7[0,1,2,3],ymm0[4,5,6,7]
+; AVX2-SLOW-NEXT:    vpermps %ymm14, %ymm11, %ymm7
+; AVX2-SLOW-NEXT:    vmovups %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm7 = ymm9[0,1,2,3,4,5],ymm7[6,7]
+; AVX2-SLOW-NEXT:    vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vmovaps 608(%rdi), %ymm0
 ; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vmovaps 640(%rdi), %ymm1
-; AVX2-SLOW-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vperm2f128 {{.*#+}} ymm12 = ymm1[0,1],ymm0[0,1]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm12 = ymm12[0,1,2,3,4,5],ymm0[6,7]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm13 = ymm12[0,2,2,2,4,6,6,6]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm13 = ymm15[0,1,2],ymm13[3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vmovaps 704(%rdi), %ymm0
-; AVX2-SLOW-NEXT:    vmovaps 736(%rdi), %ymm1
-; AVX2-SLOW-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm0[4,5,6,7]
-; AVX2-SLOW-NEXT:    vmovaps %ymm0, %ymm15
+; AVX2-SLOW-NEXT:    vmovaps 576(%rdi), %ymm7
+; AVX2-SLOW-NEXT:    vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm7 = ymm7[0,1,2,3],ymm0[4,5],ymm7[6,7]
+; AVX2-SLOW-NEXT:    vpermps %ymm7, %ymm8, %ymm8
+; AVX2-SLOW-NEXT:    vmovaps 672(%rdi), %ymm0
 ; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vpermps %ymm1, %ymm4, %ymm0
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm13[0,1,2,3,4,5],ymm0[6,7]
+; AVX2-SLOW-NEXT:    vmovaps 640(%rdi), %ymm9
+; AVX2-SLOW-NEXT:    vmovups %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vperm2f128 {{.*#+}} ymm9 = ymm9[0,1],ymm0[0,1]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm9 = ymm9[0,1,2,3,4,5],ymm0[6,7]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm11 = ymm9[0,2,2,2,4,6,6,6]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm8 = ymm8[0,1,2],ymm11[3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vmovaps 704(%rdi), %ymm11
+; AVX2-SLOW-NEXT:    vmovups %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vmovaps 736(%rdi), %ymm0
 ; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vmovaps {{.*#+}} xmm0 = <1,7,5,u>
-; AVX2-SLOW-NEXT:    vpermps %ymm9, %ymm0, %ymm9
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm7 = ymm7[1,3,2,3,5,7,6,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm7 = ymm9[0,1,2],ymm7[3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vbroadcastsd {{.*#+}} ymm9 = [5,3,5,3,5,3,5,3]
-; AVX2-SLOW-NEXT:    vpermps %ymm8, %ymm9, %ymm8
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm7 = ymm7[0,1,2,3,4,5],ymm8[6,7]
-; AVX2-SLOW-NEXT:    vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vpermps %ymm5, %ymm0, %ymm5
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm6 = ymm6[1,3,2,3,5,7,6,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0,1,2],ymm6[3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vpermps %ymm11, %ymm9, %ymm4
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm4 = ymm5[0,1,2,3,4,5],ymm4[6,7]
-; AVX2-SLOW-NEXT:    vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vpermps %ymm2, %ymm0, %ymm2
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm11 = ymm0[0,1,2,3],ymm11[4,5,6,7]
+; AVX2-SLOW-NEXT:    vpermps %ymm11, %ymm12, %ymm15
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],ymm15[6,7]
+; AVX2-SLOW-NEXT:    vmovups %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vmovaps {{.*#+}} xmm8 = <1,7,5,u>
+; AVX2-SLOW-NEXT:    vpermps %ymm13, %ymm8, %ymm13
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm5 = ymm5[1,3,2,3,5,7,6,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm5 = ymm13[0,1,2],ymm5[3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vbroadcastsd {{.*#+}} ymm12 = [5,3,5,3,5,3,5,3]
+; AVX2-SLOW-NEXT:    vmovups %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vpermps %ymm6, %ymm12, %ymm6
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0,1,2,3,4,5],ymm6[6,7]
+; AVX2-SLOW-NEXT:    vmovups %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vpermps %ymm4, %ymm8, %ymm4
 ; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm3 = ymm3[1,3,2,3,5,7,6,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2],ymm3[3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vpermps %ymm10, %ymm9, %ymm3
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm3[6,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm3 = ymm4[0,1,2],ymm3[3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vpermps %ymm2, %ymm12, %ymm2
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5],ymm2[6,7]
 ; AVX2-SLOW-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vpermps %ymm14, %ymm0, %ymm0
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm2 = ymm12[1,3,2,3,5,7,6,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm2[3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vpermps %ymm1, %ymm9, %ymm1
+; AVX2-SLOW-NEXT:    vpermps %ymm1, %ymm8, %ymm0
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm10[1,3,2,3,5,7,6,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vpermps %ymm14, %ymm12, %ymm1
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
+; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vpermps %ymm7, %ymm8, %ymm0
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm9[1,3,2,3,5,7,6,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vpermps %ymm11, %ymm12, %ymm1
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
 ; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vmovaps (%rdi), %xmm1
+; AVX2-SLOW-NEXT:    vmovaps (%rdi), %xmm3
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm0 = xmm3[2,3,2,3]
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm7[0,0,2,3,4,4,6,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7]
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm12[0,1],ymm11[2,3],ymm12[4,5],ymm11[6,7]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[2,0,2,3,6,4,6,7]
+; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,2,0,3]
+; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,3,2,3]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm9[0,0,0,0,4,4,4,4]
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm2 = ymm8[0,0,2,3,4,4,6,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2],ymm1[3],ymm2[4,5,6],ymm1[7]
+; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,1,0,3]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]
+; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vmovaps 384(%rdi), %xmm1
 ; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm0 = xmm1[2,3,2,3]
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm2 = ymm10[0,0,2,3,4,4,6,7]
+; AVX2-SLOW-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm2 = mem[0,0,2,3,4,4,6,7]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm2[1,2,3],ymm0[4],ymm2[5,6,7]
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm9[0,1],ymm12[2,3],ymm9[4,5],ymm12[6,7]
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vblendps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm2 = mem[0,1],ymm2[2,3],mem[4,5],ymm2[6,7]
 ; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm2 = ymm2[2,0,2,3,6,4,6,7]
 ; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[0,2,0,3]
 ; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,3,2,3]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm2[3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm2 = ymm7[0,0,0,0,4,4,4,4]
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm3 = ymm8[0,0,2,3,4,4,6,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1,2],ymm2[3],ymm3[4,5,6],ymm2[7]
+; AVX2-SLOW-NEXT:    vpermilps $0, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm2 = mem[0,0,0,0,4,4,4,4]
+; AVX2-SLOW-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm4 = mem[0,0,2,3,4,4,6,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm4[0,1,2],ymm2[3],ymm4[4,5,6],ymm2[7]
 ; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[0,1,0,3]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm2[5,6,7]
 ; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vmovaps 384(%rdi), %xmm0
+; AVX2-SLOW-NEXT:    vmovaps 576(%rdi), %xmm0
+; AVX2-SLOW-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; AVX2-SLOW-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm2 = mem[0,0,2,3,4,4,6,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm2[1,2,3],ymm0[4],ymm2[5,6,7]
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vblendps $204, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm2 = ymm2[0,1],mem[2,3],ymm2[4,5],mem[6,7]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm2 = ymm2[2,0,2,3,6,4,6,7]
+; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[0,2,0,3]
+; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,3,2,3]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm2[3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vpermilps $0, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm2 = mem[0,0,0,0,4,4,4,4]
+; AVX2-SLOW-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm5 = mem[0,0,2,3,4,4,6,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm5[0,1,2],ymm2[3],ymm5[4,5,6],ymm2[7]
+; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[0,1,0,3]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm2[5,6,7]
+; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vmovaps 192(%rdi), %xmm0
 ; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm2 = xmm0[2,3,2,3]
-; AVX2-SLOW-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm3 = mem[0,0,2,3,4,4,6,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0],ymm3[1,2,3],ymm2[4],ymm3[5,6,7]
 ; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm5 = ymm13[0,0,2,3,4,4,6,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0],ymm5[1,2,3],ymm2[4],ymm5[5,6,7]
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload
 ; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm3 = ymm13[0,1],ymm14[2,3],ymm13[4,5],ymm14[6,7]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm3 = ymm3[2,0,2,3,6,4,6,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm5 = ymm15[0,1],ymm14[2,3],ymm15[4,5],ymm14[6,7]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm5 = ymm5[2,0,2,3,6,4,6,7]
+; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm5 = ymm5[0,2,0,3]
+; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[0,3,2,3]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2],ymm5[3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm5 = ymm10[0,0,0,0,4,4,4,4]
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm6 = ymm4[0,0,2,3,4,4,6,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm5 = ymm6[0,1,2],ymm5[3],ymm6[4,5,6],ymm5[7]
+; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm5 = ymm5[0,1,0,3]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm5[5,6,7]
+; AVX2-SLOW-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm2 = xmm3[3,3,3,3]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm3 = ymm7[0,1,3,3,4,5,7,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0],ymm3[1,2,3],ymm2[4],ymm3[5,6,7]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm3 = ymm11[3,3,3,3,7,7,7,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0],ymm12[1],ymm3[2,3,4],ymm12[5],ymm3[6,7]
+; AVX2-SLOW-NEXT:    vmovaps %ymm12, %ymm11
 ; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm3 = ymm3[0,2,0,3]
 ; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[0,3,2,3]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2],ymm3[3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vpermilps $0, (%rsp), %ymm3 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm3 = mem[0,0,0,0,4,4,4,4]
-; AVX2-SLOW-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm4 = mem[0,0,2,3,4,4,6,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm3 = ymm4[0,1,2],ymm3[3],ymm4[4,5,6],ymm3[7]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm3 = ymm9[0,1,0,1,4,5,4,5]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm5 = ymm8[0,1,3,3,4,5,7,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm3 = ymm5[0,1,2],ymm3[3],ymm5[4,5,6],ymm3[7]
 ; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm3 = ymm3[0,1,0,3]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm3[5,6,7]
 ; AVX2-SLOW-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vmovaps 576(%rdi), %xmm2
-; AVX2-SLOW-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm3 = xmm2[2,3,2,3]
-; AVX2-SLOW-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm4 = mem[0,0,2,3,4,4,6,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0],ymm4[1,2,3],ymm3[4],ymm4[5,6,7]
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vblendps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm4 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm4 = mem[0,1],ymm2[2,3],mem[4,5],ymm2[6,7]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm4 = ymm4[2,0,2,3,6,4,6,7]
-; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm4 = ymm4[0,2,0,3]
-; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm3 = ymm3[0,3,2,3]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1,2],ymm4[3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vpermilps $0, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm4 = mem[0,0,0,0,4,4,4,4]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm5 = ymm15[0,0,2,3,4,4,6,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm4 = ymm5[0,1,2],ymm4[3],ymm5[4,5,6],ymm4[7]
-; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm4 = ymm4[0,1,0,3]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm4[5,6,7]
-; AVX2-SLOW-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vmovaps 192(%rdi), %xmm15
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm4 = xmm15[2,3,2,3]
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm5 = ymm2[0,0,2,3,4,4,6,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0],ymm5[1,2,3],ymm4[4],ymm5[5,6,7]
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm5 = ymm3[0,1],ymm11[2,3],ymm3[4,5],ymm11[6,7]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm5 = ymm5[2,0,2,3,6,4,6,7]
-; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm5 = ymm5[0,2,0,3]
-; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm4 = ymm4[0,3,2,3]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1,2],ymm5[3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vpermilps $0, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm5 = mem[0,0,0,0,4,4,4,4]
-; AVX2-SLOW-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm6 = mem[0,0,2,3,4,4,6,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm5 = ymm6[0,1,2],ymm5[3],ymm6[4,5,6],ymm5[7]
-; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm5 = ymm5[0,1,0,3]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1,2,3,4],ymm5[5,6,7]
-; AVX2-SLOW-NEXT:    vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm4 = ymm10[0,1,3,3,4,5,7,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm4[1,2,3],ymm1[4],ymm4[5,6,7]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm4 = ymm12[3,3,3,3,7,7,7,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0],ymm9[1],ymm4[2,3,4],ymm9[5],ymm4[6,7]
-; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm4 = ymm4[0,2,0,3]
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm2 = ymm12[0,1,3,3,4,5,7,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm2[1,2,3],ymm1[4],ymm2[5,6,7]
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm2 = ymm6[3,3,3,3,7,7,7,7]
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0],ymm5[1],ymm2[2,3,4],ymm5[5],ymm2[6,7]
+; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[0,2,0,3]
 ; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,3,2,3]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2],ymm4[3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm4 = ymm7[0,1,0,1,4,5,4,5]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm5 = ymm8[0,1,3,3,4,5,7,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm4 = ymm5[0,1,2],ymm4[3],ymm5[4,5,6],ymm4[7]
-; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm4 = ymm4[0,1,0,3]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm4[5,6,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2],ymm2[3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm2 = ymm9[0,1,0,1,4,5,4,5]
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm3 = ymm7[0,1,3,3,4,5,7,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1,2],ymm2[3],ymm3[4,5,6],ymm2[7]
+; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[0,1,0,3]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm2[5,6,7]
 ; AVX2-SLOW-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm8[0,1,3,3,4,5,7,7]
+; AVX2-SLOW-NEXT:    vmovaps %ymm13, %ymm3
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm13[0,1,3,3,4,5,7,7]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7]
 ; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm14[3,3,3,3,7,7,7,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm13[1],ymm1[2,3,4],ymm13[5],ymm1[6,7]
-; AVX2-SLOW-NEXT:    vmovaps %ymm13, %ymm14
-; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,2,0,3]
-; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,3,2,3]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vmovups (%rsp), %ymm13 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm13[0,1,0,1,4,5,4,5]
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm4 = ymm10[0,1,3,3,4,5,7,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm4[0,1,2],ymm1[3],ymm4[4,5,6],ymm1[7]
-; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,1,0,3]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]
-; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm0 = xmm15[3,3,3,3]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm2[0,1,3,3,4,5,7,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm11[3,3,3,3,7,7,7,7]
-; AVX2-SLOW-NEXT:    vmovaps %ymm3, %ymm7
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm3[1],ymm1[2,3,4],ymm3[5],ymm1[6,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm15[1],ymm1[2,3,4],ymm15[5],ymm1[6,7]
+; AVX2-SLOW-NEXT:    vmovaps %ymm15, %ymm14
 ; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,2,0,3]
 ; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,3,2,3]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm9[0,1,0,1,4,5,4,5]
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm3 = ymm12[0,1,3,3,4,5,7,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm3[0,1,2],ymm1[3],ymm3[4,5,6],ymm1[7]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm10[0,1,0,1,4,5,4,5]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm2 = ymm4[0,1,3,3,4,5,7,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2],ymm1[3],ymm2[4,5,6],ymm1[7]
 ; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,1,0,3]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]
 ; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; AVX2-SLOW-NEXT:    vpermilps $255, (%rsp), %xmm0 # 16-byte Folded Reload
 ; AVX2-SLOW-NEXT:    # xmm0 = mem[3,3,3,3]
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm3[0,1,3,3,4,5,7,7]
+; AVX2-SLOW-NEXT:    vpermilps $244, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm1 = mem[0,1,3,3,4,5,7,7]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7]
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm15[3,3,3,3,7,7,7,7]
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm4[1],ymm1[2,3,4],ymm4[5],ymm1[6,7]
+; AVX2-SLOW-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm1 = mem[3,3,3,3,7,7,7,7]
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm8[1],ymm1[2,3,4],ymm8[5],ymm1[6,7]
 ; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,2,0,3]
 ; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,3,2,3]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm6[0,1,0,1,4,5,4,5]
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm2 = ymm5[0,1,3,3,4,5,7,7]
+; AVX2-SLOW-NEXT:    vpermilps $68, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm1 = mem[0,1,0,1,4,5,4,5]
+; AVX2-SLOW-NEXT:    vpermilps $244, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm2 = mem[0,1,3,3,4,5,7,7]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2],ymm1[3],ymm2[4,5,6],ymm1[7]
 ; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,1,0,3]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]
-; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm2 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm2 = ymm8[0,1,2,3],mem[4,5,6,7]
-; AVX2-SLOW-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm1 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm1 = ymm14[0,1,2,3],mem[4,5,6,7]
+; AVX2-SLOW-NEXT:    vmovups %ymm0, (%rsp) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm5[0,1,2,3],ymm6[4,5,6,7]
 ; AVX2-SLOW-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm12, %ymm2 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm2 = ymm12[0,1,2,3],mem[4,5,6,7]
+; AVX2-SLOW-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-SLOW-NEXT:    vmovaps 464(%rdi), %xmm0
 ; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3],ymm1[4,5,6,7]
 ; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,1,0,2,4,5,4,6]
-; AVX2-SLOW-NEXT:    vmovddup {{.*#+}} xmm11 = [4,2,4,2]
-; AVX2-SLOW-NEXT:    # xmm11 = mem[0,0]
-; AVX2-SLOW-NEXT:    vpermps %ymm2, %ymm11, %ymm1
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1],ymm0[2,3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm14 = ymm13[0,1],ymm10[2,3],ymm13[4,5,6,7]
-; AVX2-SLOW-NEXT:    vbroadcastf128 {{.*#+}} ymm0 = [0,2,0,6,0,2,0,6]
-; AVX2-SLOW-NEXT:    # ymm0 = mem[0,1,0,1]
-; AVX2-SLOW-NEXT:    vpermps %ymm14, %ymm0, %ymm2
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm2[5,6,7]
-; AVX2-SLOW-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm8 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm8 = ymm1[0,1,2,3],mem[4,5,6,7]
-; AVX2-SLOW-NEXT:    vmovups %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm2 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm2 = ymm7[0,1,2,3],mem[4,5,6,7]
-; AVX2-SLOW-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vmovaps 272(%rdi), %xmm1
-; AVX2-SLOW-NEXT:    vmovups %ymm1, (%rsp) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1],ymm1[2,3],ymm2[4,5,6,7]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[0,1,0,2,4,5,4,6]
-; AVX2-SLOW-NEXT:    vpermps %ymm8, %ymm11, %ymm2
-; AVX2-SLOW-NEXT:    vmovaps %ymm11, %ymm13
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1],ymm1[2,3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm7 = ymm9[0,1],ymm12[2,3],ymm9[4,5,6,7]
-; AVX2-SLOW-NEXT:    vpermps %ymm7, %ymm0, %ymm2
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm2[5,6,7]
-; AVX2-SLOW-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm11 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm11 = ymm3[0,1,2,3],mem[4,5,6,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm8 = ymm4[0,1,2,3],ymm15[4,5,6,7]
-; AVX2-SLOW-NEXT:    vmovaps 656(%rdi), %xmm12
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm8[0,1],ymm12[2,3],ymm8[4,5,6,7]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[0,1,0,2,4,5,4,6]
-; AVX2-SLOW-NEXT:    vpermps %ymm11, %ymm13, %ymm2
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1],ymm1[2,3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm15 = ymm6[0,1],ymm5[2,3],ymm6[4,5,6,7]
-; AVX2-SLOW-NEXT:    vpermps %ymm15, %ymm0, %ymm2
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm2[5,6,7]
-; AVX2-SLOW-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm9 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm9 = ymm1[0,1,2,3],mem[4,5,6,7]
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vpermps %ymm2, %ymm12, %ymm1
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm13 = ymm9[0,1],ymm7[2,3],ymm9[4,5,6,7]
+; AVX2-SLOW-NEXT:    vbroadcastf128 {{.*#+}} ymm9 = [0,2,0,6,0,2,0,6]
+; AVX2-SLOW-NEXT:    # ymm9 = mem[0,1,0,1]
+; AVX2-SLOW-NEXT:    vpermps %ymm13, %ymm9, %ymm1
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]
+; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm11, %ymm7 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm7 = ymm11[0,1,2,3],mem[4,5,6,7]
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm11 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm11 = ymm0[0,1,2,3],mem[4,5,6,7]
+; AVX2-SLOW-NEXT:    vmovaps 80(%rdi), %xmm6
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm7[0,1],ymm6[2,3],ymm7[4,5,6,7]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,1,0,2,4,5,4,6]
+; AVX2-SLOW-NEXT:    vpermps %ymm11, %ymm12, %ymm1
+; AVX2-SLOW-NEXT:    vmovaps %ymm12, %ymm15
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3,4,5,6,7]
 ; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm2 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm2 = ymm1[0,1,2,3],mem[4,5,6,7]
-; AVX2-SLOW-NEXT:    vmovaps 80(%rdi), %xmm1
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm10 = ymm9[0,1],ymm1[2,3],ymm9[4,5,6,7]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm10 = ymm10[0,1,0,2,4,5,4,6]
-; AVX2-SLOW-NEXT:    vpermps %ymm2, %ymm13, %ymm4
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1],ymm10[2,3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vblendps $243, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm10 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm10 = mem[0,1],ymm3[2,3],mem[4,5,6,7]
-; AVX2-SLOW-NEXT:    vpermps %ymm10, %ymm0, %ymm0
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm4[0,1,2,3,4],ymm0[5,6,7]
+; AVX2-SLOW-NEXT:    vblendps $243, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm4 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm4 = mem[0,1],ymm1[2,3],mem[4,5,6,7]
+; AVX2-SLOW-NEXT:    vpermps %ymm4, %ymm9, %ymm1
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]
 ; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vpermilps $85, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm0 = mem[1,1,1,1,5,5,5,5]
-; AVX2-SLOW-NEXT:    vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm0 = ymm0[0,1,2],mem[3],ymm0[4,5,6,7]
-; AVX2-SLOW-NEXT:    vmovddup {{.*#+}} xmm4 = [5,3,5,3]
-; AVX2-SLOW-NEXT:    # xmm4 = mem[0,0]
-; AVX2-SLOW-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm13 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm13[0,1],ymm0[2,3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vbroadcastf128 {{.*#+}} ymm13 = [0,3,1,7,0,3,1,7]
-; AVX2-SLOW-NEXT:    # ymm13 = mem[0,1,0,1]
-; AVX2-SLOW-NEXT:    vpermps %ymm14, %ymm13, %ymm14
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm14[5,6,7]
-; AVX2-SLOW-NEXT:    vpermilps $85, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm3 = mem[1,1,1,1,5,5,5,5]
-; AVX2-SLOW-NEXT:    vblendps $8, (%rsp), %ymm3, %ymm3 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm3 = ymm3[0,1,2],mem[3],ymm3[4,5,6,7]
-; AVX2-SLOW-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm5 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm10 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm10 = ymm14[0,1,2,3],mem[4,5,6,7]
+; AVX2-SLOW-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm5 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm5 = ymm3[0,1,2,3],mem[4,5,6,7]
+; AVX2-SLOW-NEXT:    vmovaps 272(%rdi), %xmm3
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm10[0,1],ymm3[2,3],ymm10[4,5,6,7]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,1,0,2,4,5,4,6]
+; AVX2-SLOW-NEXT:    vpermps %ymm5, %ymm12, %ymm14
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm14 = ymm14[0,1],ymm0[2,3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vblendps $243, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm2 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm2 = mem[0,1],ymm0[2,3],mem[4,5,6,7]
+; AVX2-SLOW-NEXT:    vpermps %ymm2, %ymm9, %ymm12
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm14[0,1,2,3,4],ymm12[5,6,7]
+; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm12 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm12 = ymm8[0,1,2,3],mem[4,5,6,7]
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm1 = ymm0[0,1,2,3],mem[4,5,6,7]
+; AVX2-SLOW-NEXT:    vmovaps 656(%rdi), %xmm0
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm14 = ymm12[0,1],ymm0[2,3],ymm12[4,5,6,7]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm14 = ymm14[0,1,0,2,4,5,4,6]
+; AVX2-SLOW-NEXT:    vpermps %ymm1, %ymm15, %ymm15
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm14 = ymm15[0,1],ymm14[2,3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vblendps $243, {{[-0-9]+}}(%r{{[sb]}}p), %ymm15, %ymm15 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm15 = mem[0,1],ymm15[2,3],mem[4,5,6,7]
+; AVX2-SLOW-NEXT:    vpermps %ymm15, %ymm9, %ymm9
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm9 = ymm14[0,1,2,3,4],ymm9[5,6,7]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm7 = ymm7[1,1,1,1,5,5,5,5]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm6 = ymm7[0,1,2],ymm6[3],ymm7[4,5,6,7]
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vpermps %ymm11, %ymm14, %ymm7
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm6 = ymm7[0,1],ymm6[2,3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vbroadcastf128 {{.*#+}} ymm7 = [0,3,1,7,0,3,1,7]
+; AVX2-SLOW-NEXT:    # ymm7 = mem[0,1,0,1]
+; AVX2-SLOW-NEXT:    vpermps %ymm4, %ymm7, %ymm4
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm4 = ymm6[0,1,2,3,4],ymm4[5,6,7]
+; AVX2-SLOW-NEXT:    vpermilps $85, {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm6 = mem[1,1,1,1,5,5,5,5]
+; AVX2-SLOW-NEXT:    vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm6, %ymm6 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm6 = ymm6[0,1,2],mem[3],ymm6[4,5,6,7]
+; AVX2-SLOW-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm11 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm6 = ymm11[0,1],ymm6[2,3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vpermps %ymm13, %ymm7, %ymm8
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm6 = ymm6[0,1,2,3,4],ymm8[5,6,7]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm8 = ymm10[1,1,1,1,5,5,5,5]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm3 = ymm8[0,1,2],ymm3[3],ymm8[4,5,6,7]
+; AVX2-SLOW-NEXT:    vpermps %ymm5, %ymm14, %ymm5
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm3 = ymm5[0,1],ymm3[2,3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vpermps %ymm7, %ymm13, %ymm5
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1,2,3,4],ymm5[5,6,7]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm5 = ymm8[1,1,1,1,5,5,5,5]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0,1,2],ymm12[3],ymm5[4,5,6,7]
-; AVX2-SLOW-NEXT:    vpermps %ymm11, %ymm4, %ymm6
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm5 = ymm6[0,1],ymm5[2,3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vpermps %ymm15, %ymm13, %ymm6
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0,1,2,3,4],ymm6[5,6,7]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm6 = ymm9[1,1,1,1,5,5,5,5]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm6[0,1,2],ymm1[3],ymm6[4,5,6,7]
-; AVX2-SLOW-NEXT:    vpermps %ymm2, %ymm4, %ymm2
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1],ymm1[2,3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vpermps %ymm10, %ymm13, %ymm2
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm2[5,6,7]
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vmovaps %ymm2, 96(%rsi)
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vmovaps %ymm2, 32(%rsi)
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vmovaps %ymm2, 64(%rsi)
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vmovaps %ymm2, (%rsi)
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vmovaps %ymm2, 96(%rdx)
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vmovaps %ymm2, 32(%rdx)
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vmovaps %ymm2, 64(%rdx)
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vmovaps %ymm2, (%rdx)
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vmovaps %ymm2, 32(%rcx)
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vmovaps %ymm2, 96(%rcx)
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vmovaps %ymm2, 64(%rcx)
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vmovaps %ymm2, (%rcx)
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vmovaps %ymm2, 96(%r8)
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vmovaps %ymm2, 32(%r8)
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vmovaps %ymm2, 64(%r8)
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vmovaps %ymm2, (%r8)
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vmovaps %ymm2, 96(%r9)
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vmovaps %ymm2, 32(%r9)
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vmovaps %ymm2, (%r9)
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vmovaps %ymm2, 64(%r9)
+; AVX2-SLOW-NEXT:    vpermps %ymm2, %ymm7, %ymm2
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5,6,7]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm3 = ymm12[1,1,1,1,5,5,5,5]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm3[0,1,2],ymm0[3],ymm3[4,5,6,7]
+; AVX2-SLOW-NEXT:    vpermps %ymm1, %ymm14, %ymm1
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vpermps %ymm15, %ymm7, %ymm1
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vmovaps %ymm1, 96(%rsi)
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vmovaps %ymm1, 32(%rsi)
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vmovaps %ymm1, 64(%rsi)
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vmovaps %ymm1, (%rsi)
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vmovaps %ymm1, 96(%rdx)
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vmovaps %ymm1, 32(%rdx)
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vmovaps %ymm1, 64(%rdx)
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vmovaps %ymm1, (%rdx)
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vmovaps %ymm1, 32(%rcx)
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vmovaps %ymm1, 96(%rcx)
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vmovaps %ymm1, 64(%rcx)
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vmovaps %ymm1, (%rcx)
+; AVX2-SLOW-NEXT:    vmovups (%rsp), %ymm1 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vmovaps %ymm1, 96(%r8)
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vmovaps %ymm1, 32(%r8)
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vmovaps %ymm1, 64(%r8)
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vmovaps %ymm1, (%r8)
+; AVX2-SLOW-NEXT:    vmovaps %ymm9, 96(%r9)
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vmovaps %ymm1, 32(%r9)
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vmovaps %ymm1, (%r9)
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vmovaps %ymm1, 64(%r9)
 ; AVX2-SLOW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX2-SLOW-NEXT:    vmovaps %ymm5, 96(%rax)
-; AVX2-SLOW-NEXT:    vmovaps %ymm3, 32(%rax)
-; AVX2-SLOW-NEXT:    vmovaps %ymm0, 64(%rax)
-; AVX2-SLOW-NEXT:    vmovaps %ymm1, (%rax)
-; AVX2-SLOW-NEXT:    addq $1160, %rsp # imm = 0x488
+; AVX2-SLOW-NEXT:    vmovaps %ymm0, 96(%rax)
+; AVX2-SLOW-NEXT:    vmovaps %ymm2, 32(%rax)
+; AVX2-SLOW-NEXT:    vmovaps %ymm6, 64(%rax)
+; AVX2-SLOW-NEXT:    vmovaps %ymm4, (%rax)
+; AVX2-SLOW-NEXT:    addq $1256, %rsp # imm = 0x4E8
 ; AVX2-SLOW-NEXT:    vzeroupper
 ; AVX2-SLOW-NEXT:    retq
 ;
 ; AVX2-FAST-LABEL: load_i32_stride6_vf32:
 ; AVX2-FAST:       # %bb.0:
-; AVX2-FAST-NEXT:    subq $1160, %rsp # imm = 0x488
-; AVX2-FAST-NEXT:    vmovaps 480(%rdi), %ymm6
+; AVX2-FAST-NEXT:    subq $1256, %rsp # imm = 0x4E8
+; AVX2-FAST-NEXT:    vmovaps 480(%rdi), %ymm9
 ; AVX2-FAST-NEXT:    vmovaps 448(%rdi), %ymm10
 ; AVX2-FAST-NEXT:    vmovups %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovaps 416(%rdi), %ymm11
-; AVX2-FAST-NEXT:    vmovups %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vmovaps 416(%rdi), %ymm7
+; AVX2-FAST-NEXT:    vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-NEXT:    vmovaps 128(%rdi), %ymm2
 ; AVX2-FAST-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-NEXT:    vmovaps 160(%rdi), %ymm3
@@ -3586,154 +3591,156 @@ define void @load_i32_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-NEXT:    vmovaps 64(%rdi), %ymm5
 ; AVX2-FAST-NEXT:    vmovups %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovaps {{.*#+}} xmm12 = <0,6,4,u>
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm9 = ymm0[0,1,2,3],ymm1[4,5],ymm0[6,7]
-; AVX2-FAST-NEXT:    vpermps %ymm9, %ymm12, %ymm0
+; AVX2-FAST-NEXT:    vmovaps {{.*#+}} xmm8 = <0,6,4,u>
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm13 = ymm0[0,1,2,3],ymm1[4,5],ymm0[6,7]
+; AVX2-FAST-NEXT:    vpermps %ymm13, %ymm8, %ymm0
 ; AVX2-FAST-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm5[0,1],ymm4[0,1]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm7 = ymm1[0,1,2,3,4,5],ymm4[6,7]
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm7[0,2,2,2,4,6,6,6]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm0[0,1,2],ymm1[3,4,5,6,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm8 = ymm3[0,1,2,3],ymm2[4,5,6,7]
-; AVX2-FAST-NEXT:    vbroadcastsd {{.*#+}} ymm4 = [4,2,4,2,4,2,4,2]
-; AVX2-FAST-NEXT:    vpermps %ymm8, %ymm4, %ymm2
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm2[6,7]
-; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm10[0,1],ymm6[0,1]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm6 = ymm1[0,1,2,3,4,5],ymm6[6,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm6 = ymm1[0,1,2,3,4,5],ymm4[6,7]
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm6[0,2,2,2,4,6,6,6]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm5 = ymm3[0,1,2,3],ymm2[4,5,6,7]
+; AVX2-FAST-NEXT:    vbroadcastsd {{.*#+}} ymm12 = [4,2,4,2,4,2,4,2]
+; AVX2-FAST-NEXT:    vpermps %ymm5, %ymm12, %ymm1
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
+; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vmovups %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm10[0,1],ymm9[0,1]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm4 = ymm0[0,1,2,3,4,5],ymm9[6,7]
 ; AVX2-FAST-NEXT:    vmovaps 384(%rdi), %ymm0
 ; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm5 = ymm0[0,1,2,3],ymm11[4,5],ymm0[6,7]
-; AVX2-FAST-NEXT:    vpermps %ymm5, %ymm12, %ymm1
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm2 = ymm6[0,2,2,2,4,6,6,6]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2],ymm2[3,4,5,6,7]
-; AVX2-FAST-NEXT:    vmovaps 512(%rdi), %ymm0
-; AVX2-FAST-NEXT:    vmovups %ymm0, (%rsp) # 32-byte Spill
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm3 = ymm0[0,1,2,3],ymm7[4,5],ymm0[6,7]
+; AVX2-FAST-NEXT:    vpermps %ymm3, %ymm8, %ymm0
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm4[0,2,2,2,4,6,6,6]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
+; AVX2-FAST-NEXT:    vmovaps 512(%rdi), %ymm1
+; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-NEXT:    vmovaps 544(%rdi), %ymm2
 ; AVX2-FAST-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm11 = ymm2[0,1,2,3],ymm0[4,5,6,7]
-; AVX2-FAST-NEXT:    vpermps %ymm11, %ymm4, %ymm2
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm2[6,7]
-; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovaps 288(%rdi), %ymm0
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-FAST-NEXT:    vpermps %ymm2, %ymm12, %ymm1
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
 ; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovaps 256(%rdi), %ymm1
+; AVX2-FAST-NEXT:    vmovaps 288(%rdi), %ymm1
 ; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm1[0,1],ymm0[0,1]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm3 = ymm1[0,1,2,3,4,5],ymm0[6,7]
-; AVX2-FAST-NEXT:    vmovaps 224(%rdi), %ymm0
+; AVX2-FAST-NEXT:    vmovaps 256(%rdi), %ymm0
 ; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovaps 192(%rdi), %ymm1
-; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm1[0,1,2,3],ymm0[4,5],ymm1[6,7]
-; AVX2-FAST-NEXT:    vpermps %ymm2, %ymm12, %ymm10
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm15 = ymm3[0,2,2,2,4,6,6,6]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm15 = ymm10[0,1,2],ymm15[3,4,5,6,7]
-; AVX2-FAST-NEXT:    vmovaps 320(%rdi), %ymm1
+; AVX2-FAST-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[0,1],ymm1[0,1]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm10 = ymm0[0,1,2,3,4,5],ymm1[6,7]
+; AVX2-FAST-NEXT:    vmovaps 224(%rdi), %ymm1
 ; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovaps 352(%rdi), %ymm0
+; AVX2-FAST-NEXT:    vmovaps 192(%rdi), %ymm0
 ; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm10 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; AVX2-FAST-NEXT:    vpermps %ymm10, %ymm4, %ymm14
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm15[0,1,2,3,4,5],ymm14[6,7]
-; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5],ymm0[6,7]
+; AVX2-FAST-NEXT:    vpermps %ymm1, %ymm8, %ymm14
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm9 = ymm10[0,2,2,2,4,6,6,6]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm9 = ymm14[0,1,2],ymm9[3,4,5,6,7]
+; AVX2-FAST-NEXT:    vmovaps 320(%rdi), %ymm0
+; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vmovaps 352(%rdi), %ymm7
+; AVX2-FAST-NEXT:    vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm14 = ymm7[0,1,2,3],ymm0[4,5,6,7]
+; AVX2-FAST-NEXT:    vpermps %ymm14, %ymm12, %ymm7
+; AVX2-FAST-NEXT:    vmovups %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm7 = ymm9[0,1,2,3,4,5],ymm7[6,7]
+; AVX2-FAST-NEXT:    vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-NEXT:    vmovaps 608(%rdi), %ymm0
 ; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovaps 576(%rdi), %ymm1
-; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm14 = ymm1[0,1,2,3],ymm0[4,5],ymm1[6,7]
-; AVX2-FAST-NEXT:    vpermps %ymm14, %ymm12, %ymm15
+; AVX2-FAST-NEXT:    vmovaps 576(%rdi), %ymm7
+; AVX2-FAST-NEXT:    vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm7 = ymm7[0,1,2,3],ymm0[4,5],ymm7[6,7]
+; AVX2-FAST-NEXT:    vpermps %ymm7, %ymm8, %ymm8
 ; AVX2-FAST-NEXT:    vmovaps 672(%rdi), %ymm0
 ; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovaps 640(%rdi), %ymm1
-; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vperm2f128 {{.*#+}} ymm12 = ymm1[0,1],ymm0[0,1]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm12 = ymm12[0,1,2,3,4,5],ymm0[6,7]
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm13 = ymm12[0,2,2,2,4,6,6,6]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm13 = ymm15[0,1,2],ymm13[3,4,5,6,7]
-; AVX2-FAST-NEXT:    vmovaps 704(%rdi), %ymm0
-; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovaps 736(%rdi), %ymm1
-; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm0[4,5,6,7]
-; AVX2-FAST-NEXT:    vpermps %ymm1, %ymm4, %ymm0
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm13[0,1,2,3,4,5],ymm0[6,7]
+; AVX2-FAST-NEXT:    vmovaps 640(%rdi), %ymm9
+; AVX2-FAST-NEXT:    vmovups %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vperm2f128 {{.*#+}} ymm9 = ymm9[0,1],ymm0[0,1]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm9 = ymm9[0,1,2,3,4,5],ymm0[6,7]
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm11 = ymm9[0,2,2,2,4,6,6,6]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm8 = ymm8[0,1,2],ymm11[3,4,5,6,7]
+; AVX2-FAST-NEXT:    vmovaps 704(%rdi), %ymm11
+; AVX2-FAST-NEXT:    vmovups %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vmovaps 736(%rdi), %ymm0
 ; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovaps {{.*#+}} xmm0 = <1,7,5,u>
-; AVX2-FAST-NEXT:    vpermps %ymm9, %ymm0, %ymm9
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm7 = ymm7[1,3,2,3,5,7,6,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm7 = ymm9[0,1,2],ymm7[3,4,5,6,7]
-; AVX2-FAST-NEXT:    vbroadcastsd {{.*#+}} ymm9 = [5,3,5,3,5,3,5,3]
-; AVX2-FAST-NEXT:    vpermps %ymm8, %ymm9, %ymm8
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm7 = ymm7[0,1,2,3,4,5],ymm8[6,7]
-; AVX2-FAST-NEXT:    vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vpermps %ymm5, %ymm0, %ymm5
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm11 = ymm0[0,1,2,3],ymm11[4,5,6,7]
+; AVX2-FAST-NEXT:    vpermps %ymm11, %ymm12, %ymm15
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],ymm15[6,7]
+; AVX2-FAST-NEXT:    vmovups %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vmovaps {{.*#+}} xmm8 = <1,7,5,u>
+; AVX2-FAST-NEXT:    vpermps %ymm13, %ymm8, %ymm13
 ; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm6 = ymm6[1,3,2,3,5,7,6,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0,1,2],ymm6[3,4,5,6,7]
-; AVX2-FAST-NEXT:    vpermps %ymm11, %ymm9, %ymm4
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm4 = ymm5[0,1,2,3,4,5],ymm4[6,7]
-; AVX2-FAST-NEXT:    vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vpermps %ymm2, %ymm0, %ymm2
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm3 = ymm3[1,3,2,3,5,7,6,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2],ymm3[3,4,5,6,7]
-; AVX2-FAST-NEXT:    vpermps %ymm10, %ymm9, %ymm3
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm3[6,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm6 = ymm13[0,1,2],ymm6[3,4,5,6,7]
+; AVX2-FAST-NEXT:    vbroadcastsd {{.*#+}} ymm13 = [5,3,5,3,5,3,5,3]
+; AVX2-FAST-NEXT:    vmovups %ymm13, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vpermps %ymm5, %ymm13, %ymm5
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm5 = ymm6[0,1,2,3,4,5],ymm5[6,7]
+; AVX2-FAST-NEXT:    vmovups %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vpermps %ymm3, %ymm8, %ymm3
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm4 = ymm4[1,3,2,3,5,7,6,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1,2],ymm4[3,4,5,6,7]
+; AVX2-FAST-NEXT:    vpermps %ymm2, %ymm13, %ymm2
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5],ymm2[6,7]
 ; AVX2-FAST-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vpermps %ymm14, %ymm0, %ymm0
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm2 = ymm12[1,3,2,3,5,7,6,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm2[3,4,5,6,7]
-; AVX2-FAST-NEXT:    vpermps %ymm1, %ymm9, %ymm1
+; AVX2-FAST-NEXT:    vpermps %ymm1, %ymm8, %ymm0
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm10[1,3,2,3,5,7,6,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
+; AVX2-FAST-NEXT:    vpermps %ymm14, %ymm13, %ymm1
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
+; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vpermps %ymm7, %ymm8, %ymm0
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm9[1,3,2,3,5,7,6,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
+; AVX2-FAST-NEXT:    vpermps %ymm11, %ymm13, %ymm1
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
 ; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-NEXT:    vmovaps (%rdi), %xmm0
 ; AVX2-FAST-NEXT:    vshufps {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm2 = ymm10[0,0,2,3,4,4,6,7]
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm2 = ymm7[0,0,2,3,4,4,6,7]
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm2[1,2,3],ymm1[4],ymm2[5,6,7]
 ; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm9[0,1],ymm12[2,3],ymm9[4,5],ymm12[6,7]
-; AVX2-FAST-NEXT:    vmovaps {{.*#+}} ymm3 = [2,0,6,4,2,0,6,7]
-; AVX2-FAST-NEXT:    vpermps %ymm2, %ymm3, %ymm2
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm3 = ymm9[0,1],ymm11[2,3],ymm9[4,5],ymm11[6,7]
+; AVX2-FAST-NEXT:    vmovaps {{.*#+}} ymm10 = [2,0,6,4,2,0,6,7]
+; AVX2-FAST-NEXT:    vpermps %ymm3, %ymm10, %ymm3
 ; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,3,2,3]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2],ymm2[3,4,5,6,7]
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm2 = ymm7[0,0,0,0,4,4,4,4]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2],ymm3[3,4,5,6,7]
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm3 = ymm2[0,0,0,0,4,4,4,4]
 ; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
 ; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm4 = ymm8[0,0,2,3,4,4,6,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm4[0,1,2],ymm2[3],ymm4[4,5,6],ymm2[7]
-; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[0,1,0,3]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm2[5,6,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm3 = ymm4[0,1,2],ymm3[3],ymm4[4,5,6],ymm3[7]
+; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm3 = ymm3[0,1,0,3]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm3[5,6,7]
 ; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-NEXT:    vmovaps 384(%rdi), %xmm1
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} xmm2 = xmm1[2,3,2,3]
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} xmm3 = xmm1[2,3,2,3]
 ; AVX2-FAST-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Folded Reload
 ; AVX2-FAST-NEXT:    # ymm4 = mem[0,0,2,3,4,4,6,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0],ymm4[1,2,3],ymm2[4],ymm4[5,6,7]
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm4 = ymm13[0,1],ymm14[2,3],ymm13[4,5],ymm14[6,7]
-; AVX2-FAST-NEXT:    vpermps %ymm4, %ymm3, %ymm4
-; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[0,3,2,3]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2],ymm4[3,4,5,6,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0],ymm4[1,2,3],ymm3[4],ymm4[5,6,7]
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
+; AVX2-FAST-NEXT:    vblendps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm4 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm4 = mem[0,1],ymm4[2,3],mem[4,5],ymm4[6,7]
+; AVX2-FAST-NEXT:    vpermps %ymm4, %ymm10, %ymm4
+; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm3 = ymm3[0,3,2,3]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1,2],ymm4[3,4,5,6,7]
 ; AVX2-FAST-NEXT:    vpermilps $0, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Folded Reload
 ; AVX2-FAST-NEXT:    # ymm4 = mem[0,0,0,0,4,4,4,4]
-; AVX2-FAST-NEXT:    vpermilps $224, (%rsp), %ymm5 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Folded Reload
 ; AVX2-FAST-NEXT:    # ymm5 = mem[0,0,2,3,4,4,6,7]
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm4 = ymm5[0,1,2],ymm4[3],ymm5[4,5,6],ymm4[7]
 ; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm4 = ymm4[0,1,0,3]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm4[5,6,7]
-; AVX2-FAST-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovaps 576(%rdi), %xmm2
-; AVX2-FAST-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} xmm4 = xmm2[2,3,2,3]
-; AVX2-FAST-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm5 = mem[0,0,2,3,4,4,6,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1,2,3,4],ymm4[5,6,7]
+; AVX2-FAST-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vmovaps 576(%rdi), %xmm3
+; AVX2-FAST-NEXT:    vmovaps %xmm3, (%rsp) # 16-byte Spill
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} xmm4 = xmm3[2,3,2,3]
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm5 = ymm13[0,0,2,3,4,4,6,7]
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0],ymm5[1,2,3],ymm4[4],ymm5[5,6,7]
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-FAST-NEXT:    vblendps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm5 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm5 = mem[0,1],ymm2[2,3],mem[4,5],ymm2[6,7]
-; AVX2-FAST-NEXT:    vpermps %ymm5, %ymm3, %ymm5
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
+; AVX2-FAST-NEXT:    vblendps $204, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm5 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm5 = ymm3[0,1],mem[2,3],ymm3[4,5],mem[6,7]
+; AVX2-FAST-NEXT:    vpermps %ymm5, %ymm10, %ymm5
 ; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm4 = ymm4[0,3,2,3]
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1,2],ymm5[3,4,5,6,7]
 ; AVX2-FAST-NEXT:    vpermilps $0, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Folded Reload
@@ -3742,36 +3749,37 @@ define void @load_i32_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX2-FAST-NEXT:    # ymm6 = mem[0,0,2,3,4,4,6,7]
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm5 = ymm6[0,1,2],ymm5[3],ymm6[4,5,6],ymm5[7]
 ; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm5 = ymm5[0,1,0,3]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm4[0,1,2,3,4],ymm5[5,6,7]
-; AVX2-FAST-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload
-; AVX2-FAST-NEXT:    vblendps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm11, %ymm4 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm4 = mem[0,1],ymm11[2,3],mem[4,5],ymm11[6,7]
-; AVX2-FAST-NEXT:    vpermps %ymm4, %ymm3, %ymm4
-; AVX2-FAST-NEXT:    vmovaps 192(%rdi), %xmm3
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} xmm5 = xmm3[2,3,2,3]
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm6 = ymm2[0,0,2,3,4,4,6,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm3 = ymm4[0,1,2,3,4],ymm5[5,6,7]
+; AVX2-FAST-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload
+; AVX2-FAST-NEXT:    vblendps $204, {{[-0-9]+}}(%r{{[sb]}}p), %ymm15, %ymm4 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm4 = ymm15[0,1],mem[2,3],ymm15[4,5],mem[6,7]
+; AVX2-FAST-NEXT:    vpermps %ymm4, %ymm10, %ymm4
+; AVX2-FAST-NEXT:    vmovaps 192(%rdi), %xmm12
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} xmm5 = xmm12[2,3,2,3]
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm6 = ymm14[0,0,2,3,4,4,6,7]
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0],ymm6[1,2,3],ymm5[4],ymm6[5,6,7]
 ; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm5 = ymm5[0,3,2,3]
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm4 = ymm5[0,1,2],ymm4[3,4,5,6,7]
-; AVX2-FAST-NEXT:    vpermilps $0, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm5 = mem[0,0,0,0,4,4,4,4]
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm6 = ymm15[0,0,2,3,4,4,6,7]
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm5 = ymm10[0,0,0,0,4,4,4,4]
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm6 = ymm3[0,0,2,3,4,4,6,7]
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm5 = ymm6[0,1,2],ymm5[3],ymm6[4,5,6],ymm5[7]
 ; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm5 = ymm5[0,1,0,3]
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1,2,3,4],ymm5[5,6,7]
 ; AVX2-FAST-NEXT:    vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm4 = ymm10[0,1,3,3,4,5,7,7]
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm4 = ymm7[0,1,3,3,4,5,7,7]
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm4[1,2,3],ymm0[4],ymm4[5,6,7]
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm4 = ymm12[3,3,3,3,7,7,7,7]
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm4 = ymm11[3,3,3,3,7,7,7,7]
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0],ymm9[1],ymm4[2,3,4],ymm9[5],ymm4[6,7]
+; AVX2-FAST-NEXT:    vmovaps %ymm9, %ymm11
 ; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm4 = ymm4[0,2,0,3]
 ; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,3,2,3]
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm4[3,4,5,6,7]
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm4 = ymm7[0,1,0,1,4,5,4,5]
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm4 = ymm2[0,1,0,1,4,5,4,5]
 ; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm5 = ymm8[0,1,3,3,4,5,7,7]
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm4 = ymm5[0,1,2],ymm4[3],ymm5[4,5,6],ymm4[7]
 ; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm4 = ymm4[0,1,0,3]
@@ -3781,213 +3789,206 @@ define void @load_i32_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
 ; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm8[0,1,3,3,4,5,7,7]
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7]
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm14[3,3,3,3,7,7,7,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm13[1],ymm1[2,3,4],ymm13[5],ymm1[6,7]
-; AVX2-FAST-NEXT:    vmovaps %ymm13, %ymm14
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm6[3,3,3,3,7,7,7,7]
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm5[1],ymm1[2,3,4],ymm5[5],ymm1[6,7]
 ; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,2,0,3]
 ; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,3,2,3]
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm13[0,1,0,1,4,5,4,5]
-; AVX2-FAST-NEXT:    vmovups (%rsp), %ymm10 # 32-byte Reload
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm4 = ymm10[0,1,3,3,4,5,7,7]
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm9[0,1,0,1,4,5,4,5]
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm4 = ymm7[0,1,3,3,4,5,7,7]
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm4[0,1,2],ymm1[3],ymm4[4,5,6],ymm1[7]
 ; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,1,0,3]
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]
 ; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} xmm0 = xmm3[3,3,3,3]
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm2[0,1,3,3,4,5,7,7]
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} xmm0 = xmm12[3,3,3,3]
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm14[0,1,3,3,4,5,7,7]
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7]
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm11[3,3,3,3,7,7,7,7]
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm7[1],ymm1[2,3,4],ymm7[5],ymm1[6,7]
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm14[3,3,3,3,7,7,7,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm15[1],ymm1[2,3,4],ymm15[5],ymm1[6,7]
 ; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,2,0,3]
 ; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,3,2,3]
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm9[0,1,0,1,4,5,4,5]
-; AVX2-FAST-NEXT:    vmovaps %ymm15, %ymm12
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm3 = ymm15[0,1,3,3,4,5,7,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm3[0,1,2],ymm1[3],ymm3[4,5,6],ymm1[7]
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm10[0,1,0,1,4,5,4,5]
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm2 = ymm3[0,1,3,3,4,5,7,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2],ymm1[3],ymm2[4,5,6],ymm1[7]
 ; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,1,0,3]
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]
 ; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; AVX2-FAST-NEXT:    vpermilps $255, (%rsp), %xmm0 # 16-byte Folded Reload
 ; AVX2-FAST-NEXT:    # xmm0 = mem[3,3,3,3]
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm3[0,1,3,3,4,5,7,7]
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm13[0,1,3,3,4,5,7,7]
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7]
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm15[3,3,3,3,7,7,7,7]
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm4[1],ymm1[2,3,4],ymm4[5],ymm1[6,7]
+; AVX2-FAST-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm1 = mem[3,3,3,3,7,7,7,7]
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm13[1],ymm1[2,3,4],ymm13[5],ymm1[6,7]
 ; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,2,0,3]
 ; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,3,2,3]
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm6[0,1,0,1,4,5,4,5]
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm2 = ymm5[0,1,3,3,4,5,7,7]
+; AVX2-FAST-NEXT:    vpermilps $68, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm1 = mem[0,1,0,1,4,5,4,5]
+; AVX2-FAST-NEXT:    vpermilps $244, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm2 = mem[0,1,3,3,4,5,7,7]
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2],ymm1[3],ymm2[4,5,6],ymm1[7]
 ; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,1,0,3]
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]
-; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vmovups %ymm0, (%rsp) # 32-byte Spill
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm5[0,1,2,3],ymm6[4,5,6,7]
+; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm2 # 32-byte Folded Reload
 ; AVX2-FAST-NEXT:    # ymm2 = ymm8[0,1,2,3],mem[4,5,6,7]
 ; AVX2-FAST-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm1 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm1 = ymm14[0,1,2,3],mem[4,5,6,7]
-; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-NEXT:    vmovaps 464(%rdi), %xmm0
 ; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3],ymm1[4,5,6,7]
 ; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,1,0,2,4,5,4,6]
-; AVX2-FAST-NEXT:    vmovddup {{.*#+}} xmm11 = [4,2,4,2]
-; AVX2-FAST-NEXT:    # xmm11 = mem[0,0]
-; AVX2-FAST-NEXT:    vpermps %ymm2, %ymm11, %ymm1
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1],ymm0[2,3,4,5,6,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm14 = ymm13[0,1],ymm10[2,3],ymm13[4,5,6,7]
-; AVX2-FAST-NEXT:    vbroadcastf128 {{.*#+}} ymm0 = [0,2,0,6,0,2,0,6]
-; AVX2-FAST-NEXT:    # ymm0 = mem[0,1,0,1]
-; AVX2-FAST-NEXT:    vpermps %ymm14, %ymm0, %ymm2
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm2[5,6,7]
-; AVX2-FAST-NEXT:    vmovups %ymm1, (%rsp) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX2-FAST-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm8 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm8 = ymm1[0,1,2,3],mem[4,5,6,7]
-; AVX2-FAST-NEXT:    vmovups %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm2 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm2 = ymm7[0,1,2,3],mem[4,5,6,7]
-; AVX2-FAST-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovaps 272(%rdi), %xmm1
-; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1],ymm1[2,3],ymm2[4,5,6,7]
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[0,1,0,2,4,5,4,6]
-; AVX2-FAST-NEXT:    vpermps %ymm8, %ymm11, %ymm2
-; AVX2-FAST-NEXT:    vmovaps %ymm11, %ymm13
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1],ymm1[2,3,4,5,6,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm7 = ymm9[0,1],ymm12[2,3],ymm9[4,5,6,7]
-; AVX2-FAST-NEXT:    vpermps %ymm7, %ymm0, %ymm2
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm2[5,6,7]
-; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm11 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm11 = ymm3[0,1,2,3],mem[4,5,6,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm8 = ymm4[0,1,2,3],ymm15[4,5,6,7]
-; AVX2-FAST-NEXT:    vmovaps 656(%rdi), %xmm12
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm8[0,1],ymm12[2,3],ymm8[4,5,6,7]
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[0,1,0,2,4,5,4,6]
-; AVX2-FAST-NEXT:    vpermps %ymm11, %ymm13, %ymm2
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1],ymm1[2,3,4,5,6,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm15 = ymm6[0,1],ymm5[2,3],ymm6[4,5,6,7]
-; AVX2-FAST-NEXT:    vpermps %ymm15, %ymm0, %ymm2
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm2[5,6,7]
-; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX2-FAST-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm9 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm9 = ymm1[0,1,2,3],mem[4,5,6,7]
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload
+; AVX2-FAST-NEXT:    vpermps %ymm2, %ymm12, %ymm1
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3,4,5,6,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm8 = ymm9[0,1],ymm7[2,3],ymm9[4,5,6,7]
+; AVX2-FAST-NEXT:    vbroadcastf128 {{.*#+}} ymm9 = [0,2,0,6,0,2,0,6]
+; AVX2-FAST-NEXT:    # ymm9 = mem[0,1,0,1]
+; AVX2-FAST-NEXT:    vpermps %ymm8, %ymm9, %ymm1
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]
+; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm11, %ymm7 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm7 = ymm11[0,1,2,3],mem[4,5,6,7]
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX2-FAST-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm11 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm11 = ymm0[0,1,2,3],mem[4,5,6,7]
+; AVX2-FAST-NEXT:    vmovaps 80(%rdi), %xmm6
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm7[0,1],ymm6[2,3],ymm7[4,5,6,7]
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,1,0,2,4,5,4,6]
+; AVX2-FAST-NEXT:    vpermps %ymm11, %ymm12, %ymm1
+; AVX2-FAST-NEXT:    vmovaps %ymm12, %ymm15
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3,4,5,6,7]
 ; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX2-FAST-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm2 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm2 = ymm1[0,1,2,3],mem[4,5,6,7]
-; AVX2-FAST-NEXT:    vmovaps 80(%rdi), %xmm1
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm10 = ymm9[0,1],ymm1[2,3],ymm9[4,5,6,7]
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm10 = ymm10[0,1,0,2,4,5,4,6]
-; AVX2-FAST-NEXT:    vpermps %ymm2, %ymm13, %ymm4
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1],ymm10[2,3,4,5,6,7]
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
-; AVX2-FAST-NEXT:    vblendps $243, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm10 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm10 = mem[0,1],ymm3[2,3],mem[4,5,6,7]
-; AVX2-FAST-NEXT:    vpermps %ymm10, %ymm0, %ymm0
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm4[0,1,2,3,4],ymm0[5,6,7]
+; AVX2-FAST-NEXT:    vblendps $243, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm4 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm4 = mem[0,1],ymm1[2,3],mem[4,5,6,7]
+; AVX2-FAST-NEXT:    vpermps %ymm4, %ymm9, %ymm1
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]
 ; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vpermilps $85, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm0 = mem[1,1,1,1,5,5,5,5]
-; AVX2-FAST-NEXT:    vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm0 = ymm0[0,1,2],mem[3],ymm0[4,5,6,7]
-; AVX2-FAST-NEXT:    vmovddup {{.*#+}} xmm4 = [5,3,5,3]
-; AVX2-FAST-NEXT:    # xmm4 = mem[0,0]
-; AVX2-FAST-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm13 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm13[0,1],ymm0[2,3,4,5,6,7]
-; AVX2-FAST-NEXT:    vbroadcastf128 {{.*#+}} ymm13 = [0,3,1,7,0,3,1,7]
-; AVX2-FAST-NEXT:    # ymm13 = mem[0,1,0,1]
-; AVX2-FAST-NEXT:    vpermps %ymm14, %ymm13, %ymm14
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm14[5,6,7]
-; AVX2-FAST-NEXT:    vpermilps $85, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm3 = mem[1,1,1,1,5,5,5,5]
-; AVX2-FAST-NEXT:    vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm3 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm3 = ymm3[0,1,2],mem[3],ymm3[4,5,6,7]
-; AVX2-FAST-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm5 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    vblendps $15, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm10 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm10 = mem[0,1,2,3],ymm14[4,5,6,7]
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX2-FAST-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm5 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm5 = ymm0[0,1,2,3],mem[4,5,6,7]
+; AVX2-FAST-NEXT:    vmovaps 272(%rdi), %xmm3
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm10[0,1],ymm3[2,3],ymm10[4,5,6,7]
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,1,0,2,4,5,4,6]
+; AVX2-FAST-NEXT:    vpermps %ymm5, %ymm12, %ymm14
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm14 = ymm14[0,1],ymm0[2,3,4,5,6,7]
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX2-FAST-NEXT:    vblendps $243, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm2 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm2 = mem[0,1],ymm0[2,3],mem[4,5,6,7]
+; AVX2-FAST-NEXT:    vpermps %ymm2, %ymm9, %ymm12
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm14[0,1,2,3,4],ymm12[5,6,7]
+; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm13, %ymm12 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm12 = ymm13[0,1,2,3],mem[4,5,6,7]
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX2-FAST-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm1 = ymm0[0,1,2,3],mem[4,5,6,7]
+; AVX2-FAST-NEXT:    vmovaps 656(%rdi), %xmm0
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm14 = ymm12[0,1],ymm0[2,3],ymm12[4,5,6,7]
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm14 = ymm14[0,1,0,2,4,5,4,6]
+; AVX2-FAST-NEXT:    vpermps %ymm1, %ymm15, %ymm15
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm14 = ymm15[0,1],ymm14[2,3,4,5,6,7]
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload
+; AVX2-FAST-NEXT:    vblendps $12, {{[-0-9]+}}(%r{{[sb]}}p), %ymm13, %ymm15 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm15 = ymm13[0,1],mem[2,3],ymm13[4,5,6,7]
+; AVX2-FAST-NEXT:    vpermps %ymm15, %ymm9, %ymm9
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm9 = ymm14[0,1,2,3,4],ymm9[5,6,7]
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm7 = ymm7[1,1,1,1,5,5,5,5]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm6 = ymm7[0,1,2],ymm6[3],ymm7[4,5,6,7]
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload
+; AVX2-FAST-NEXT:    vpermps %ymm11, %ymm14, %ymm7
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm6 = ymm7[0,1],ymm6[2,3,4,5,6,7]
+; AVX2-FAST-NEXT:    vbroadcastf128 {{.*#+}} ymm7 = [0,3,1,7,0,3,1,7]
+; AVX2-FAST-NEXT:    # ymm7 = mem[0,1,0,1]
+; AVX2-FAST-NEXT:    vpermps %ymm4, %ymm7, %ymm4
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm4 = ymm6[0,1,2,3,4],ymm4[5,6,7]
+; AVX2-FAST-NEXT:    vpermilps $85, {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm6 = mem[1,1,1,1,5,5,5,5]
+; AVX2-FAST-NEXT:    vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm6, %ymm6 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm6 = ymm6[0,1,2],mem[3],ymm6[4,5,6,7]
+; AVX2-FAST-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm11 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm6 = ymm11[0,1],ymm6[2,3,4,5,6,7]
+; AVX2-FAST-NEXT:    vpermps %ymm8, %ymm7, %ymm8
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm6 = ymm6[0,1,2,3,4],ymm8[5,6,7]
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm8 = ymm10[1,1,1,1,5,5,5,5]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm3 = ymm8[0,1,2],ymm3[3],ymm8[4,5,6,7]
+; AVX2-FAST-NEXT:    vpermps %ymm5, %ymm14, %ymm5
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm3 = ymm5[0,1],ymm3[2,3,4,5,6,7]
-; AVX2-FAST-NEXT:    vpermps %ymm7, %ymm13, %ymm5
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1,2,3,4],ymm5[5,6,7]
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm5 = ymm8[1,1,1,1,5,5,5,5]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0,1,2],ymm12[3],ymm5[4,5,6,7]
-; AVX2-FAST-NEXT:    vpermps %ymm11, %ymm4, %ymm6
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm5 = ymm6[0,1],ymm5[2,3,4,5,6,7]
-; AVX2-FAST-NEXT:    vpermps %ymm15, %ymm13, %ymm6
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0,1,2,3,4],ymm6[5,6,7]
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm6 = ymm9[1,1,1,1,5,5,5,5]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm6[0,1,2],ymm1[3],ymm6[4,5,6,7]
-; AVX2-FAST-NEXT:    vpermps %ymm2, %ymm4, %ymm2
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1],ymm1[2,3,4,5,6,7]
-; AVX2-FAST-NEXT:    vpermps %ymm10, %ymm13, %ymm2
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm2[5,6,7]
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-FAST-NEXT:    vmovaps %ymm2, 96(%rsi)
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-FAST-NEXT:    vmovaps %ymm2, 32(%rsi)
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-FAST-NEXT:    vmovaps %ymm2, 64(%rsi)
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-FAST-NEXT:    vmovaps %ymm2, (%rsi)
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-FAST-NEXT:    vmovaps %ymm2, 96(%rdx)
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-FAST-NEXT:    vmovaps %ymm2, 32(%rdx)
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-FAST-NEXT:    vmovaps %ymm2, 64(%rdx)
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-FAST-NEXT:    vmovaps %ymm2, (%rdx)
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-FAST-NEXT:    vmovaps %ymm2, 32(%rcx)
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-FAST-NEXT:    vmovaps %ymm2, 96(%rcx)
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-FAST-NEXT:    vmovaps %ymm2, 64(%rcx)
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-FAST-NEXT:    vmovaps %ymm2, (%rcx)
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-FAST-NEXT:    vmovaps %ymm2, 96(%r8)
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-FAST-NEXT:    vmovaps %ymm2, 32(%r8)
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-FAST-NEXT:    vmovaps %ymm2, 64(%r8)
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-FAST-NEXT:    vmovaps %ymm2, (%r8)
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-FAST-NEXT:    vmovaps %ymm2, 96(%r9)
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-FAST-NEXT:    vmovaps %ymm2, 32(%r9)
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-FAST-NEXT:    vmovaps %ymm2, (%r9)
-; AVX2-FAST-NEXT:    vmovups (%rsp), %ymm2 # 32-byte Reload
-; AVX2-FAST-NEXT:    vmovaps %ymm2, 64(%r9)
+; AVX2-FAST-NEXT:    vpermps %ymm2, %ymm7, %ymm2
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5,6,7]
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm3 = ymm12[1,1,1,1,5,5,5,5]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm3[0,1,2],ymm0[3],ymm3[4,5,6,7]
+; AVX2-FAST-NEXT:    vpermps %ymm1, %ymm14, %ymm1
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3,4,5,6,7]
+; AVX2-FAST-NEXT:    vpermps %ymm15, %ymm7, %ymm1
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-FAST-NEXT:    vmovaps %ymm1, 96(%rsi)
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-FAST-NEXT:    vmovaps %ymm1, 32(%rsi)
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-FAST-NEXT:    vmovaps %ymm1, 64(%rsi)
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-FAST-NEXT:    vmovaps %ymm1, (%rsi)
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-FAST-NEXT:    vmovaps %ymm1, 96(%rdx)
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-FAST-NEXT:    vmovaps %ymm1, 32(%rdx)
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-FAST-NEXT:    vmovaps %ymm1, 64(%rdx)
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-FAST-NEXT:    vmovaps %ymm1, (%rdx)
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-FAST-NEXT:    vmovaps %ymm1, 32(%rcx)
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-FAST-NEXT:    vmovaps %ymm1, 96(%rcx)
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-FAST-NEXT:    vmovaps %ymm1, 64(%rcx)
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-FAST-NEXT:    vmovaps %ymm1, (%rcx)
+; AVX2-FAST-NEXT:    vmovups (%rsp), %ymm1 # 32-byte Reload
+; AVX2-FAST-NEXT:    vmovaps %ymm1, 96(%r8)
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-FAST-NEXT:    vmovaps %ymm1, 32(%r8)
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-FAST-NEXT:    vmovaps %ymm1, 64(%r8)
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-FAST-NEXT:    vmovaps %ymm1, (%r8)
+; AVX2-FAST-NEXT:    vmovaps %ymm9, 96(%r9)
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-FAST-NEXT:    vmovaps %ymm1, 32(%r9)
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-FAST-NEXT:    vmovaps %ymm1, (%r9)
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-FAST-NEXT:    vmovaps %ymm1, 64(%r9)
 ; AVX2-FAST-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX2-FAST-NEXT:    vmovaps %ymm5, 96(%rax)
-; AVX2-FAST-NEXT:    vmovaps %ymm3, 32(%rax)
-; AVX2-FAST-NEXT:    vmovaps %ymm0, 64(%rax)
-; AVX2-FAST-NEXT:    vmovaps %ymm1, (%rax)
-; AVX2-FAST-NEXT:    addq $1160, %rsp # imm = 0x488
+; AVX2-FAST-NEXT:    vmovaps %ymm0, 96(%rax)
+; AVX2-FAST-NEXT:    vmovaps %ymm2, 32(%rax)
+; AVX2-FAST-NEXT:    vmovaps %ymm6, 64(%rax)
+; AVX2-FAST-NEXT:    vmovaps %ymm4, (%rax)
+; AVX2-FAST-NEXT:    addq $1256, %rsp # imm = 0x4E8
 ; AVX2-FAST-NEXT:    vzeroupper
 ; AVX2-FAST-NEXT:    retq
 ;
 ; AVX2-FAST-PERLANE-LABEL: load_i32_stride6_vf32:
 ; AVX2-FAST-PERLANE:       # %bb.0:
-; AVX2-FAST-PERLANE-NEXT:    subq $1160, %rsp # imm = 0x488
-; AVX2-FAST-PERLANE-NEXT:    vmovaps 480(%rdi), %ymm6
+; AVX2-FAST-PERLANE-NEXT:    subq $1256, %rsp # imm = 0x4E8
+; AVX2-FAST-PERLANE-NEXT:    vmovaps 480(%rdi), %ymm9
 ; AVX2-FAST-PERLANE-NEXT:    vmovaps 448(%rdi), %ymm10
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vmovaps 416(%rdi), %ymm11
-; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vmovaps 416(%rdi), %ymm7
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-PERLANE-NEXT:    vmovaps 128(%rdi), %ymm2
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-PERLANE-NEXT:    vmovaps 160(%rdi), %ymm3
@@ -4000,400 +4001,399 @@ define void @load_i32_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-PERLANE-NEXT:    vmovaps 64(%rdi), %ymm5
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vmovaps {{.*#+}} xmm12 = <0,6,4,u>
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm9 = ymm0[0,1,2,3],ymm1[4,5],ymm0[6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm9, %ymm12, %ymm0
+; AVX2-FAST-PERLANE-NEXT:    vmovaps {{.*#+}} xmm8 = <0,6,4,u>
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm13 = ymm0[0,1,2,3],ymm1[4,5],ymm0[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm13, %ymm8, %ymm0
 ; AVX2-FAST-PERLANE-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm5[0,1],ymm4[0,1]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm7 = ymm1[0,1,2,3,4,5],ymm4[6,7]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm7[0,2,2,2,4,6,6,6]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm0[0,1,2],ymm1[3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm8 = ymm3[0,1,2,3],ymm2[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vbroadcastsd {{.*#+}} ymm4 = [4,2,4,2,4,2,4,2]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm8, %ymm4, %ymm2
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm2[6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm10[0,1],ymm6[0,1]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm6 = ymm1[0,1,2,3,4,5],ymm6[6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovaps 384(%rdi), %ymm0
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm5 = ymm1[0,1,2,3,4,5],ymm4[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm5[0,2,2,2,4,6,6,6]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm6 = ymm3[0,1,2,3],ymm2[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vbroadcastsd {{.*#+}} ymm2 = [4,2,4,2,4,2,4,2]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm6, %ymm2, %ymm1
+; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm2, %ymm11
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm5 = ymm0[0,1,2,3],ymm11[4,5],ymm0[6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm5, %ymm12, %ymm1
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm2 = ymm6[0,2,2,2,4,6,6,6]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2],ymm2[3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovaps 512(%rdi), %ymm0
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm10[0,1],ymm9[0,1]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm3 = ymm0[0,1,2,3,4,5],ymm9[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovaps 384(%rdi), %ymm0
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vmovaps 544(%rdi), %ymm2
-; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm2, (%rsp) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm11 = ymm2[0,1,2,3],ymm0[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm11, %ymm4, %ymm2
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm2[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm4 = ymm0[0,1,2,3],ymm7[4,5],ymm0[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm4, %ymm8, %ymm0
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm3[0,2,2,2,4,6,6,6]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovaps 512(%rdi), %ymm1
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vmovaps 288(%rdi), %ymm0
+; AVX2-FAST-PERLANE-NEXT:    vmovaps 544(%rdi), %ymm2
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm2, %ymm11, %ymm1
+; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm11, %ymm12
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vmovaps 256(%rdi), %ymm1
+; AVX2-FAST-PERLANE-NEXT:    vmovaps 288(%rdi), %ymm1
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm1[0,1],ymm0[0,1]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm3 = ymm1[0,1,2,3,4,5],ymm0[6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovaps 224(%rdi), %ymm0
+; AVX2-FAST-PERLANE-NEXT:    vmovaps 256(%rdi), %ymm0
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vmovaps 192(%rdi), %ymm1
+; AVX2-FAST-PERLANE-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[0,1],ymm1[0,1]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm10 = ymm0[0,1,2,3,4,5],ymm1[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovaps 224(%rdi), %ymm1
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm1[0,1,2,3],ymm0[4,5],ymm1[6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm2, %ymm12, %ymm10
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm15 = ymm3[0,2,2,2,4,6,6,6]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm15 = ymm10[0,1,2],ymm15[3,4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovaps 192(%rdi), %ymm0
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm1[4,5],ymm0[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm1, %ymm8, %ymm14
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm9 = ymm10[0,2,2,2,4,6,6,6]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm9 = ymm14[0,1,2],ymm9[3,4,5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovaps 320(%rdi), %ymm0
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vmovaps 352(%rdi), %ymm1
-; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm10 = ymm1[0,1,2,3],ymm0[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm10, %ymm4, %ymm14
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm15[0,1,2,3,4,5],ymm14[6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vmovaps 352(%rdi), %ymm7
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm14 = ymm7[0,1,2,3],ymm0[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm14, %ymm11, %ymm7
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm7 = ymm9[0,1,2,3,4,5],ymm7[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-PERLANE-NEXT:    vmovaps 608(%rdi), %ymm0
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vmovaps 576(%rdi), %ymm1
-; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm14 = ymm1[0,1,2,3],ymm0[4,5],ymm1[6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm14, %ymm12, %ymm15
+; AVX2-FAST-PERLANE-NEXT:    vmovaps 576(%rdi), %ymm7
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm7 = ymm7[0,1,2,3],ymm0[4,5],ymm7[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm7, %ymm8, %ymm8
 ; AVX2-FAST-PERLANE-NEXT:    vmovaps 672(%rdi), %ymm0
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vmovaps 640(%rdi), %ymm1
-; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vperm2f128 {{.*#+}} ymm12 = ymm1[0,1],ymm0[0,1]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm12 = ymm12[0,1,2,3,4,5],ymm0[6,7]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm13 = ymm12[0,2,2,2,4,6,6,6]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm13 = ymm15[0,1,2],ymm13[3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovaps 704(%rdi), %ymm0
-; AVX2-FAST-PERLANE-NEXT:    vmovaps 736(%rdi), %ymm1
-; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm0[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm0, %ymm15
-; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm1, %ymm4, %ymm0
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm13[0,1,2,3,4,5],ymm0[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovaps 640(%rdi), %ymm9
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vperm2f128 {{.*#+}} ymm9 = ymm9[0,1],ymm0[0,1]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm9 = ymm9[0,1,2,3,4,5],ymm0[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm11 = ymm9[0,2,2,2,4,6,6,6]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm8 = ymm8[0,1,2],ymm11[3,4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovaps 704(%rdi), %ymm11
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vmovaps 736(%rdi), %ymm0
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vmovaps {{.*#+}} xmm0 = <1,7,5,u>
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm9, %ymm0, %ymm9
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm7 = ymm7[1,3,2,3,5,7,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm7 = ymm9[0,1,2],ymm7[3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vbroadcastsd {{.*#+}} ymm9 = [5,3,5,3,5,3,5,3]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm8, %ymm9, %ymm8
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm7 = ymm7[0,1,2,3,4,5],ymm8[6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm5, %ymm0, %ymm5
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm6 = ymm6[1,3,2,3,5,7,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0,1,2],ymm6[3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm11, %ymm9, %ymm4
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm4 = ymm5[0,1,2,3,4,5],ymm4[6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm2, %ymm0, %ymm2
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm11 = ymm0[0,1,2,3],ymm11[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm11, %ymm12, %ymm15
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],ymm15[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vmovaps {{.*#+}} xmm8 = <1,7,5,u>
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm13, %ymm8, %ymm13
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm5 = ymm5[1,3,2,3,5,7,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm5 = ymm13[0,1,2],ymm5[3,4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vbroadcastsd {{.*#+}} ymm12 = [5,3,5,3,5,3,5,3]
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm6, %ymm12, %ymm6
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0,1,2,3,4,5],ymm6[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm4, %ymm8, %ymm4
 ; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm3 = ymm3[1,3,2,3,5,7,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2],ymm3[3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm10, %ymm9, %ymm3
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm3[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm3 = ymm4[0,1,2],ymm3[3,4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm2, %ymm12, %ymm2
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5],ymm2[6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm14, %ymm0, %ymm0
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm2 = ymm12[1,3,2,3,5,7,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm2[3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm1, %ymm9, %ymm1
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm1, %ymm8, %ymm0
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm10[1,3,2,3,5,7,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm14, %ymm12, %ymm1
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm7, %ymm8, %ymm0
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm9[1,3,2,3,5,7,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm11, %ymm12, %ymm1
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vmovaps (%rdi), %xmm1
+; AVX2-FAST-PERLANE-NEXT:    vmovaps (%rdi), %xmm3
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} xmm0 = xmm3[2,3,2,3]
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm7[0,0,2,3,4,4,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm12[0,1],ymm11[2,3],ymm12[4,5],ymm11[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[2,0,2,3,6,4,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,2,0,3]
+; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,3,2,3]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm9[0,0,0,0,4,4,4,4]
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm2 = ymm8[0,0,2,3,4,4,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2],ymm1[3],ymm2[4,5,6],ymm1[7]
+; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,1,0,3]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vmovaps 384(%rdi), %xmm1
 ; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} xmm0 = xmm1[2,3,2,3]
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm2 = ymm10[0,0,2,3,4,4,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm2 = mem[0,0,2,3,4,4,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm2[1,2,3],ymm0[4],ymm2[5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm9[0,1],ymm12[2,3],ymm9[4,5],ymm12[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vblendps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm2 = mem[0,1],ymm2[2,3],mem[4,5],ymm2[6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm2 = ymm2[2,0,2,3,6,4,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[0,2,0,3]
 ; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,3,2,3]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm2[3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm2 = ymm7[0,0,0,0,4,4,4,4]
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm3 = ymm8[0,0,2,3,4,4,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1,2],ymm2[3],ymm3[4,5,6],ymm2[7]
+; AVX2-FAST-PERLANE-NEXT:    vpermilps $0, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm2 = mem[0,0,0,0,4,4,4,4]
+; AVX2-FAST-PERLANE-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm4 = mem[0,0,2,3,4,4,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm4[0,1,2],ymm2[3],ymm4[4,5,6],ymm2[7]
+; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[0,1,0,3]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm2[5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vmovaps 576(%rdi), %xmm0
+; AVX2-FAST-PERLANE-NEXT:    vmovaps %xmm0, (%rsp) # 16-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; AVX2-FAST-PERLANE-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm2 = mem[0,0,2,3,4,4,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm2[1,2,3],ymm0[4],ymm2[5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vblendps $204, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm2 = ymm2[0,1],mem[2,3],ymm2[4,5],mem[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm2 = ymm2[2,0,2,3,6,4,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[0,2,0,3]
+; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,3,2,3]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm2[3,4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermilps $0, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm2 = mem[0,0,0,0,4,4,4,4]
+; AVX2-FAST-PERLANE-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm5 = mem[0,0,2,3,4,4,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm5[0,1,2],ymm2[3],ymm5[4,5,6],ymm2[7]
 ; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[0,1,0,3]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm2[5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vmovaps 384(%rdi), %xmm0
+; AVX2-FAST-PERLANE-NEXT:    vmovaps 192(%rdi), %xmm0
 ; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} xmm2 = xmm0[2,3,2,3]
-; AVX2-FAST-PERLANE-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm3 = mem[0,0,2,3,4,4,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0],ymm3[1,2,3],ymm2[4],ymm3[5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm5 = ymm13[0,0,2,3,4,4,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0],ymm5[1,2,3],ymm2[4],ymm5[5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload
 ; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm3 = ymm13[0,1],ymm14[2,3],ymm13[4,5],ymm14[6,7]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm3 = ymm3[2,0,2,3,6,4,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm5 = ymm15[0,1],ymm14[2,3],ymm15[4,5],ymm14[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm5 = ymm5[2,0,2,3,6,4,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm5 = ymm5[0,2,0,3]
+; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[0,3,2,3]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2],ymm5[3,4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm5 = ymm10[0,0,0,0,4,4,4,4]
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm6 = ymm4[0,0,2,3,4,4,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm5 = ymm6[0,1,2],ymm5[3],ymm6[4,5,6],ymm5[7]
+; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm5 = ymm5[0,1,0,3]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm5[5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} xmm2 = xmm3[3,3,3,3]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm3 = ymm7[0,1,3,3,4,5,7,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0],ymm3[1,2,3],ymm2[4],ymm3[5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm3 = ymm11[3,3,3,3,7,7,7,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0],ymm12[1],ymm3[2,3,4],ymm12[5],ymm3[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm12, %ymm11
 ; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm3 = ymm3[0,2,0,3]
 ; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[0,3,2,3]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2],ymm3[3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermilps $0, (%rsp), %ymm3 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm3 = mem[0,0,0,0,4,4,4,4]
-; AVX2-FAST-PERLANE-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm4 = mem[0,0,2,3,4,4,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm3 = ymm4[0,1,2],ymm3[3],ymm4[4,5,6],ymm3[7]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm3 = ymm9[0,1,0,1,4,5,4,5]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm5 = ymm8[0,1,3,3,4,5,7,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm3 = ymm5[0,1,2],ymm3[3],ymm5[4,5,6],ymm3[7]
 ; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm3 = ymm3[0,1,0,3]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm3[5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vmovaps 576(%rdi), %xmm2
-; AVX2-FAST-PERLANE-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} xmm3 = xmm2[2,3,2,3]
-; AVX2-FAST-PERLANE-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm4 = mem[0,0,2,3,4,4,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0],ymm4[1,2,3],ymm3[4],ymm4[5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vblendps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm4 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm4 = mem[0,1],ymm2[2,3],mem[4,5],ymm2[6,7]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm4 = ymm4[2,0,2,3,6,4,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm4 = ymm4[0,2,0,3]
-; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm3 = ymm3[0,3,2,3]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1,2],ymm4[3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermilps $0, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm4 = mem[0,0,0,0,4,4,4,4]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm5 = ymm15[0,0,2,3,4,4,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm4 = ymm5[0,1,2],ymm4[3],ymm5[4,5,6],ymm4[7]
-; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm4 = ymm4[0,1,0,3]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm4[5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vmovaps 192(%rdi), %xmm15
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} xmm4 = xmm15[2,3,2,3]
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm5 = ymm2[0,0,2,3,4,4,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0],ymm5[1,2,3],ymm4[4],ymm5[5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm5 = ymm3[0,1],ymm11[2,3],ymm3[4,5],ymm11[6,7]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm5 = ymm5[2,0,2,3,6,4,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm5 = ymm5[0,2,0,3]
-; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm4 = ymm4[0,3,2,3]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1,2],ymm5[3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermilps $0, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm5 = mem[0,0,0,0,4,4,4,4]
-; AVX2-FAST-PERLANE-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm6 = mem[0,0,2,3,4,4,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm5 = ymm6[0,1,2],ymm5[3],ymm6[4,5,6],ymm5[7]
-; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm5 = ymm5[0,1,0,3]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1,2,3,4],ymm5[5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm4 = ymm10[0,1,3,3,4,5,7,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm4[1,2,3],ymm1[4],ymm4[5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm4 = ymm12[3,3,3,3,7,7,7,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0],ymm9[1],ymm4[2,3,4],ymm9[5],ymm4[6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm4 = ymm4[0,2,0,3]
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm2 = ymm12[0,1,3,3,4,5,7,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm2[1,2,3],ymm1[4],ymm2[5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm2 = ymm6[3,3,3,3,7,7,7,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0],ymm5[1],ymm2[2,3,4],ymm5[5],ymm2[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[0,2,0,3]
 ; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,3,2,3]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2],ymm4[3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm4 = ymm7[0,1,0,1,4,5,4,5]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm5 = ymm8[0,1,3,3,4,5,7,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm4 = ymm5[0,1,2],ymm4[3],ymm5[4,5,6],ymm4[7]
-; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm4 = ymm4[0,1,0,3]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm4[5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2],ymm2[3,4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm2 = ymm9[0,1,0,1,4,5,4,5]
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm3 = ymm7[0,1,3,3,4,5,7,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1,2],ymm2[3],ymm3[4,5,6],ymm2[7]
+; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[0,1,0,3]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm2[5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm8[0,1,3,3,4,5,7,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm13, %ymm3
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm13[0,1,3,3,4,5,7,7]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm14[3,3,3,3,7,7,7,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm13[1],ymm1[2,3,4],ymm13[5],ymm1[6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm13, %ymm14
-; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,2,0,3]
-; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,3,2,3]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups (%rsp), %ymm13 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm13[0,1,0,1,4,5,4,5]
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm4 = ymm10[0,1,3,3,4,5,7,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm4[0,1,2],ymm1[3],ymm4[4,5,6],ymm1[7]
-; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,1,0,3]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} xmm0 = xmm15[3,3,3,3]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm2[0,1,3,3,4,5,7,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm11[3,3,3,3,7,7,7,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm3, %ymm7
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm3[1],ymm1[2,3,4],ymm3[5],ymm1[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm15[1],ymm1[2,3,4],ymm15[5],ymm1[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm15, %ymm14
 ; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,2,0,3]
 ; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,3,2,3]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm9[0,1,0,1,4,5,4,5]
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm3 = ymm12[0,1,3,3,4,5,7,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm3[0,1,2],ymm1[3],ymm3[4,5,6],ymm1[7]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm10[0,1,0,1,4,5,4,5]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm2 = ymm4[0,1,3,3,4,5,7,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2],ymm1[3],ymm2[4,5,6],ymm1[7]
 ; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,1,0,3]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    vpermilps $255, (%rsp), %xmm0 # 16-byte Folded Reload
 ; AVX2-FAST-PERLANE-NEXT:    # xmm0 = mem[3,3,3,3]
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm3[0,1,3,3,4,5,7,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermilps $244, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm1 = mem[0,1,3,3,4,5,7,7]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm15[3,3,3,3,7,7,7,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm4[1],ymm1[2,3,4],ymm4[5],ymm1[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm1 = mem[3,3,3,3,7,7,7,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm8[1],ymm1[2,3,4],ymm8[5],ymm1[6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,2,0,3]
 ; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,3,2,3]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm6[0,1,0,1,4,5,4,5]
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm2 = ymm5[0,1,3,3,4,5,7,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermilps $68, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm1 = mem[0,1,0,1,4,5,4,5]
+; AVX2-FAST-PERLANE-NEXT:    vpermilps $244, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm2 = mem[0,1,3,3,4,5,7,7]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2],ymm1[3],ymm2[4,5,6],ymm1[7]
 ; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,1,0,3]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm2 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm2 = ymm8[0,1,2,3],mem[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm1 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm1 = ymm14[0,1,2,3],mem[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, (%rsp) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm5[0,1,2,3],ymm6[4,5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm12, %ymm2 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm2 = ymm12[0,1,2,3],mem[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-PERLANE-NEXT:    vmovaps 464(%rdi), %xmm0
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3],ymm1[4,5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,1,0,2,4,5,4,6]
-; AVX2-FAST-PERLANE-NEXT:    vmovddup {{.*#+}} xmm11 = [4,2,4,2]
-; AVX2-FAST-PERLANE-NEXT:    # xmm11 = mem[0,0]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm2, %ymm11, %ymm1
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1],ymm0[2,3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm14 = ymm13[0,1],ymm10[2,3],ymm13[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vbroadcastf128 {{.*#+}} ymm0 = [0,2,0,6,0,2,0,6]
-; AVX2-FAST-PERLANE-NEXT:    # ymm0 = mem[0,1,0,1]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm14, %ymm0, %ymm2
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm2[5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm8 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm8 = ymm1[0,1,2,3],mem[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm2 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm2 = ymm7[0,1,2,3],mem[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vmovaps 272(%rdi), %xmm1
-; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm1, (%rsp) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1],ymm1[2,3],ymm2[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[0,1,0,2,4,5,4,6]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm8, %ymm11, %ymm2
-; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm11, %ymm13
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1],ymm1[2,3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm7 = ymm9[0,1],ymm12[2,3],ymm9[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm7, %ymm0, %ymm2
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm2[5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm11 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm11 = ymm3[0,1,2,3],mem[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm8 = ymm4[0,1,2,3],ymm15[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovaps 656(%rdi), %xmm12
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm8[0,1],ymm12[2,3],ymm8[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[0,1,0,2,4,5,4,6]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm11, %ymm13, %ymm2
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1],ymm1[2,3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm15 = ymm6[0,1],ymm5[2,3],ymm6[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm15, %ymm0, %ymm2
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm2[5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm9 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm9 = ymm1[0,1,2,3],mem[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm2, %ymm12, %ymm1
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3,4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm13 = ymm9[0,1],ymm7[2,3],ymm9[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vbroadcastf128 {{.*#+}} ymm9 = [0,2,0,6,0,2,0,6]
+; AVX2-FAST-PERLANE-NEXT:    # ymm9 = mem[0,1,0,1]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm13, %ymm9, %ymm1
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm11, %ymm7 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm7 = ymm11[0,1,2,3],mem[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm11 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm11 = ymm0[0,1,2,3],mem[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovaps 80(%rdi), %xmm6
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm7[0,1],ymm6[2,3],ymm7[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,1,0,2,4,5,4,6]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm11, %ymm12, %ymm1
+; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm12, %ymm15
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3,4,5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm2 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm2 = ymm1[0,1,2,3],mem[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovaps 80(%rdi), %xmm1
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm10 = ymm9[0,1],ymm1[2,3],ymm9[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm10 = ymm10[0,1,0,2,4,5,4,6]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm2, %ymm13, %ymm4
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1],ymm10[2,3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vblendps $243, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm10 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm10 = mem[0,1],ymm3[2,3],mem[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm10, %ymm0, %ymm0
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm4[0,1,2,3,4],ymm0[5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps $243, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm4 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm4 = mem[0,1],ymm1[2,3],mem[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm4, %ymm9, %ymm1
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vpermilps $85, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm0 = mem[1,1,1,1,5,5,5,5]
-; AVX2-FAST-PERLANE-NEXT:    vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm0 = ymm0[0,1,2],mem[3],ymm0[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovddup {{.*#+}} xmm4 = [5,3,5,3]
-; AVX2-FAST-PERLANE-NEXT:    # xmm4 = mem[0,0]
-; AVX2-FAST-PERLANE-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm13 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm13[0,1],ymm0[2,3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vbroadcastf128 {{.*#+}} ymm13 = [0,3,1,7,0,3,1,7]
-; AVX2-FAST-PERLANE-NEXT:    # ymm13 = mem[0,1,0,1]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm14, %ymm13, %ymm14
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm14[5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermilps $85, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm3 = mem[1,1,1,1,5,5,5,5]
-; AVX2-FAST-PERLANE-NEXT:    vblendps $8, (%rsp), %ymm3, %ymm3 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm3 = ymm3[0,1,2],mem[3],ymm3[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm5 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm10 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm10 = ymm14[0,1,2,3],mem[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm5 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm5 = ymm3[0,1,2,3],mem[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovaps 272(%rdi), %xmm3
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm10[0,1],ymm3[2,3],ymm10[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,1,0,2,4,5,4,6]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm5, %ymm12, %ymm14
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm14 = ymm14[0,1],ymm0[2,3,4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vblendps $243, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm2 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm2 = mem[0,1],ymm0[2,3],mem[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm2, %ymm9, %ymm12
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm14[0,1,2,3,4],ymm12[5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm12 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm12 = ymm8[0,1,2,3],mem[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm1 = ymm0[0,1,2,3],mem[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovaps 656(%rdi), %xmm0
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm14 = ymm12[0,1],ymm0[2,3],ymm12[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm14 = ymm14[0,1,0,2,4,5,4,6]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm1, %ymm15, %ymm15
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm14 = ymm15[0,1],ymm14[2,3,4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vblendps $243, {{[-0-9]+}}(%r{{[sb]}}p), %ymm15, %ymm15 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm15 = mem[0,1],ymm15[2,3],mem[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm15, %ymm9, %ymm9
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm9 = ymm14[0,1,2,3,4],ymm9[5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm7 = ymm7[1,1,1,1,5,5,5,5]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm6 = ymm7[0,1,2],ymm6[3],ymm7[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm11, %ymm14, %ymm7
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm6 = ymm7[0,1],ymm6[2,3,4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vbroadcastf128 {{.*#+}} ymm7 = [0,3,1,7,0,3,1,7]
+; AVX2-FAST-PERLANE-NEXT:    # ymm7 = mem[0,1,0,1]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm4, %ymm7, %ymm4
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm4 = ymm6[0,1,2,3,4],ymm4[5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermilps $85, {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm6 = mem[1,1,1,1,5,5,5,5]
+; AVX2-FAST-PERLANE-NEXT:    vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm6, %ymm6 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm6 = ymm6[0,1,2],mem[3],ymm6[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm11 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm6 = ymm11[0,1],ymm6[2,3,4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm13, %ymm7, %ymm8
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm6 = ymm6[0,1,2,3,4],ymm8[5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm8 = ymm10[1,1,1,1,5,5,5,5]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm3 = ymm8[0,1,2],ymm3[3],ymm8[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm5, %ymm14, %ymm5
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm3 = ymm5[0,1],ymm3[2,3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm7, %ymm13, %ymm5
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1,2,3,4],ymm5[5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm5 = ymm8[1,1,1,1,5,5,5,5]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0,1,2],ymm12[3],ymm5[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm11, %ymm4, %ymm6
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm5 = ymm6[0,1],ymm5[2,3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm15, %ymm13, %ymm6
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0,1,2,3,4],ymm6[5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm6 = ymm9[1,1,1,1,5,5,5,5]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm6[0,1,2],ymm1[3],ymm6[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm2, %ymm4, %ymm2
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1],ymm1[2,3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm10, %ymm13, %ymm2
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm2[5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm2, 96(%rsi)
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm2, 32(%rsi)
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm2, 64(%rsi)
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm2, (%rsi)
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm2, 96(%rdx)
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm2, 32(%rdx)
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm2, 64(%rdx)
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm2, (%rdx)
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm2, 32(%rcx)
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm2, 96(%rcx)
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm2, 64(%rcx)
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm2, (%rcx)
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm2, 96(%r8)
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm2, 32(%r8)
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm2, 64(%r8)
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm2, (%r8)
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm2, 96(%r9)
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm2, 32(%r9)
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm2, (%r9)
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm2, 64(%r9)
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm2, %ymm7, %ymm2
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1,2,3,4],ymm2[5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm3 = ymm12[1,1,1,1,5,5,5,5]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm3[0,1,2],ymm0[3],ymm3[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm1, %ymm14, %ymm1
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3,4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm15, %ymm7, %ymm1
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm1, 96(%rsi)
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm1, 32(%rsi)
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm1, 64(%rsi)
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm1, (%rsi)
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm1, 96(%rdx)
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm1, 32(%rdx)
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm1, 64(%rdx)
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm1, (%rdx)
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm1, 32(%rcx)
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm1, 96(%rcx)
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm1, 64(%rcx)
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm1, (%rcx)
+; AVX2-FAST-PERLANE-NEXT:    vmovups (%rsp), %ymm1 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm1, 96(%r8)
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm1, 32(%r8)
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm1, 64(%r8)
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm1, (%r8)
+; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm9, 96(%r9)
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm1, 32(%r9)
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm1, (%r9)
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm1, 64(%r9)
 ; AVX2-FAST-PERLANE-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm5, 96(%rax)
-; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm3, 32(%rax)
-; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm0, 64(%rax)
-; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm1, (%rax)
-; AVX2-FAST-PERLANE-NEXT:    addq $1160, %rsp # imm = 0x488
+; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm0, 96(%rax)
+; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm2, 32(%rax)
+; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm6, 64(%rax)
+; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm4, (%rax)
+; AVX2-FAST-PERLANE-NEXT:    addq $1256, %rsp # imm = 0x4E8
 ; AVX2-FAST-PERLANE-NEXT:    vzeroupper
 ; AVX2-FAST-PERLANE-NEXT:    retq
 ;
@@ -6945,45 +6945,47 @@ define void @load_i32_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ;
 ; AVX2-SLOW-LABEL: load_i32_stride6_vf64:
 ; AVX2-SLOW:       # %bb.0:
-; AVX2-SLOW-NEXT:    subq $2504, %rsp # imm = 0x9C8
-; AVX2-SLOW-NEXT:    vmovaps 672(%rdi), %ymm2
-; AVX2-SLOW-NEXT:    vmovaps 640(%rdi), %ymm3
+; AVX2-SLOW-NEXT:    subq $2536, %rsp # imm = 0x9E8
+; AVX2-SLOW-NEXT:    vmovaps 672(%rdi), %ymm4
+; AVX2-SLOW-NEXT:    vmovaps 640(%rdi), %ymm5
+; AVX2-SLOW-NEXT:    vmovups %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vmovaps 608(%rdi), %ymm3
 ; AVX2-SLOW-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vmovaps 608(%rdi), %ymm4
-; AVX2-SLOW-NEXT:    vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vmovaps 320(%rdi), %ymm5
-; AVX2-SLOW-NEXT:    vmovups %ymm5, (%rsp) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vmovaps 320(%rdi), %ymm2
+; AVX2-SLOW-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-SLOW-NEXT:    vmovaps 352(%rdi), %ymm6
 ; AVX2-SLOW-NEXT:    vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-SLOW-NEXT:    vmovaps 288(%rdi), %ymm7
 ; AVX2-SLOW-NEXT:    vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-SLOW-NEXT:    vmovaps 256(%rdi), %ymm9
-; AVX2-SLOW-NEXT:    vmovups %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vmovups %ymm9, (%rsp) # 32-byte Spill
 ; AVX2-SLOW-NEXT:    vmovaps 224(%rdi), %ymm0
 ; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-SLOW-NEXT:    vmovaps 192(%rdi), %ymm1
 ; AVX2-SLOW-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-SLOW-NEXT:    vmovaps {{.*#+}} xmm8 = <0,6,4,u>
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm14 = ymm1[0,1,2,3],ymm0[4,5],ymm1[6,7]
-; AVX2-SLOW-NEXT:    vpermps %ymm14, %ymm8, %ymm0
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5],ymm1[6,7]
+; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vpermps %ymm0, %ymm8, %ymm0
 ; AVX2-SLOW-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm9[0,1],ymm7[0,1]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm7[6,7]
 ; AVX2-SLOW-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[0,2,2,2,4,6,6,6]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm6[0,1,2,3],ymm5[4,5,6,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm6[0,1,2,3],ymm2[4,5,6,7]
 ; AVX2-SLOW-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vbroadcastsd {{.*#+}} ymm5 = [4,2,4,2,4,2,4,2]
-; AVX2-SLOW-NEXT:    vpermps %ymm1, %ymm5, %ymm1
+; AVX2-SLOW-NEXT:    vbroadcastsd {{.*#+}} ymm2 = [4,2,4,2,4,2,4,2]
+; AVX2-SLOW-NEXT:    vpermps %ymm1, %ymm2, %ymm1
+; AVX2-SLOW-NEXT:    vmovaps %ymm2, %ymm6
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
 ; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm3[0,1],ymm2[0,1]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm0[0,1,2,3,4,5],ymm2[6,7]
+; AVX2-SLOW-NEXT:    vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm5[0,1],ymm4[0,1]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm0[0,1,2,3,4,5],ymm4[6,7]
 ; AVX2-SLOW-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-SLOW-NEXT:    vmovaps 576(%rdi), %ymm0
 ; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm4[4,5],ymm0[6,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm3[4,5],ymm0[6,7]
 ; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-SLOW-NEXT:    vpermps %ymm0, %ymm8, %ymm0
 ; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm2[0,2,2,2,4,6,6,6]
@@ -6994,7 +6996,7 @@ define void @load_i32_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX2-SLOW-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
 ; AVX2-SLOW-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vpermps %ymm1, %ymm5, %ymm1
+; AVX2-SLOW-NEXT:    vpermps %ymm1, %ymm6, %ymm1
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
 ; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-SLOW-NEXT:    vmovaps 1056(%rdi), %ymm1
@@ -7019,7 +7021,7 @@ define void @load_i32_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX2-SLOW-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
 ; AVX2-SLOW-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vpermps %ymm1, %ymm5, %ymm1
+; AVX2-SLOW-NEXT:    vpermps %ymm1, %ymm6, %ymm1
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
 ; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-SLOW-NEXT:    vmovaps 1440(%rdi), %ymm1
@@ -7044,7 +7046,7 @@ define void @load_i32_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX2-SLOW-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
 ; AVX2-SLOW-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vpermps %ymm1, %ymm5, %ymm1
+; AVX2-SLOW-NEXT:    vpermps %ymm1, %ymm6, %ymm1
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
 ; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-SLOW-NEXT:    vmovaps 96(%rdi), %ymm1
@@ -7052,21 +7054,21 @@ define void @load_i32_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX2-SLOW-NEXT:    vmovaps 64(%rdi), %ymm0
 ; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-SLOW-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[0,1],ymm1[0,1]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm15 = ymm0[0,1,2,3,4,5],ymm1[6,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm13 = ymm0[0,1,2,3,4,5],ymm1[6,7]
 ; AVX2-SLOW-NEXT:    vmovaps (%rdi), %ymm0
 ; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-SLOW-NEXT:    vmovaps 32(%rdi), %ymm1
 ; AVX2-SLOW-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm12 = ymm0[0,1,2,3],ymm1[4,5],ymm0[6,7]
-; AVX2-SLOW-NEXT:    vpermps %ymm12, %ymm8, %ymm0
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm15[0,2,2,2,4,6,6,6]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm11 = ymm0[0,1,2,3],ymm1[4,5],ymm0[6,7]
+; AVX2-SLOW-NEXT:    vpermps %ymm11, %ymm8, %ymm0
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm13[0,2,2,2,4,6,6,6]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
 ; AVX2-SLOW-NEXT:    vmovaps 128(%rdi), %ymm1
 ; AVX2-SLOW-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-SLOW-NEXT:    vmovaps 160(%rdi), %ymm2
 ; AVX2-SLOW-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm11 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVX2-SLOW-NEXT:    vpermps %ymm11, %ymm5, %ymm1
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm10 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-SLOW-NEXT:    vpermps %ymm10, %ymm6, %ymm1
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
 ; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-SLOW-NEXT:    vmovaps 480(%rdi), %ymm1
@@ -7074,21 +7076,22 @@ define void @load_i32_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX2-SLOW-NEXT:    vmovaps 448(%rdi), %ymm0
 ; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-SLOW-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[0,1],ymm1[0,1]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm10 = ymm0[0,1,2,3,4,5],ymm1[6,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm7 = ymm0[0,1,2,3,4,5],ymm1[6,7]
 ; AVX2-SLOW-NEXT:    vmovaps 416(%rdi), %ymm0
 ; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-SLOW-NEXT:    vmovaps 384(%rdi), %ymm1
 ; AVX2-SLOW-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm9 = ymm1[0,1,2,3],ymm0[4,5],ymm1[6,7]
-; AVX2-SLOW-NEXT:    vpermps %ymm9, %ymm8, %ymm0
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm10[0,2,2,2,4,6,6,6]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm5 = ymm1[0,1,2,3],ymm0[4,5],ymm1[6,7]
+; AVX2-SLOW-NEXT:    vpermps %ymm5, %ymm8, %ymm0
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm7[0,2,2,2,4,6,6,6]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
 ; AVX2-SLOW-NEXT:    vmovaps 512(%rdi), %ymm1
 ; AVX2-SLOW-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-SLOW-NEXT:    vmovaps 544(%rdi), %ymm2
 ; AVX2-SLOW-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm7 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVX2-SLOW-NEXT:    vpermps %ymm7, %ymm5, %ymm1
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm4 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-SLOW-NEXT:    vpermps %ymm4, %ymm6, %ymm1
+; AVX2-SLOW-NEXT:    vmovaps %ymm6, %ymm12
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
 ; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-SLOW-NEXT:    vmovaps 864(%rdi), %ymm1
@@ -7096,114 +7099,117 @@ define void @load_i32_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX2-SLOW-NEXT:    vmovaps 832(%rdi), %ymm0
 ; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-SLOW-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[0,1],ymm1[0,1]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm6 = ymm0[0,1,2,3,4,5],ymm1[6,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm3 = ymm0[0,1,2,3,4,5],ymm1[6,7]
 ; AVX2-SLOW-NEXT:    vmovaps 800(%rdi), %ymm0
 ; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-SLOW-NEXT:    vmovaps 768(%rdi), %ymm1
 ; AVX2-SLOW-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm4 = ymm1[0,1,2,3],ymm0[4,5],ymm1[6,7]
-; AVX2-SLOW-NEXT:    vpermps %ymm4, %ymm8, %ymm0
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm6[0,2,2,2,4,6,6,6]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm1[0,1,2,3],ymm0[4,5],ymm1[6,7]
+; AVX2-SLOW-NEXT:    vpermps %ymm2, %ymm8, %ymm0
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm3[0,2,2,2,4,6,6,6]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
 ; AVX2-SLOW-NEXT:    vmovaps 896(%rdi), %ymm1
 ; AVX2-SLOW-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vmovaps 928(%rdi), %ymm2
-; AVX2-SLOW-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm3 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVX2-SLOW-NEXT:    vpermps %ymm3, %ymm5, %ymm1
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
+; AVX2-SLOW-NEXT:    vmovaps 928(%rdi), %ymm6
+; AVX2-SLOW-NEXT:    vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm6[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-SLOW-NEXT:    vpermps %ymm1, %ymm12, %ymm6
+; AVX2-SLOW-NEXT:    vmovaps %ymm12, %ymm14
+; AVX2-SLOW-NEXT:    vmovups %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm6[6,7]
 ; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-SLOW-NEXT:    vmovaps 1184(%rdi), %ymm0
 ; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vmovaps 1152(%rdi), %ymm1
-; AVX2-SLOW-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm1[0,1,2,3],ymm0[4,5],ymm1[6,7]
-; AVX2-SLOW-NEXT:    vpermps %ymm2, %ymm8, %ymm0
-; AVX2-SLOW-NEXT:    vmovaps 1248(%rdi), %ymm8
+; AVX2-SLOW-NEXT:    vmovaps 1152(%rdi), %ymm6
+; AVX2-SLOW-NEXT:    vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm6 = ymm6[0,1,2,3],ymm0[4,5],ymm6[6,7]
+; AVX2-SLOW-NEXT:    vpermps %ymm6, %ymm8, %ymm0
+; AVX2-SLOW-NEXT:    vmovaps 1248(%rdi), %ymm9
+; AVX2-SLOW-NEXT:    vmovups %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vmovaps 1216(%rdi), %ymm8
 ; AVX2-SLOW-NEXT:    vmovups %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vmovaps 1216(%rdi), %ymm1
-; AVX2-SLOW-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm1[0,1],ymm8[0,1]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm8 = ymm1[0,1,2,3,4,5],ymm8[6,7]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm8[0,2,2,2,4,6,6,6]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vmovaps 1280(%rdi), %ymm1
-; AVX2-SLOW-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vmovaps 1312(%rdi), %ymm13
-; AVX2-SLOW-NEXT:    vmovups %ymm13, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm13[0,1,2,3],ymm1[4,5,6,7]
-; AVX2-SLOW-NEXT:    vpermps %ymm1, %ymm5, %ymm5
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm5[6,7]
+; AVX2-SLOW-NEXT:    vperm2f128 {{.*#+}} ymm8 = ymm8[0,1],ymm9[0,1]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],ymm9[6,7]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm9 = ymm8[0,2,2,2,4,6,6,6]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm9[3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vmovaps 1280(%rdi), %ymm9
+; AVX2-SLOW-NEXT:    vmovups %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vmovaps 1312(%rdi), %ymm12
+; AVX2-SLOW-NEXT:    vmovups %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm15 = ymm12[0,1,2,3],ymm9[4,5,6,7]
+; AVX2-SLOW-NEXT:    vpermps %ymm15, %ymm14, %ymm9
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm9[6,7]
 ; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vmovaps {{.*#+}} xmm5 = <1,7,5,u>
-; AVX2-SLOW-NEXT:    vpermps %ymm14, %ymm5, %ymm0
-; AVX2-SLOW-NEXT:    vpermilps $237, {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm13 = mem[1,3,2,3,5,7,6,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm13 = ymm0[0,1,2],ymm13[3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vbroadcastsd {{.*#+}} ymm0 = [5,3,5,3,5,3,5,3]
-; AVX2-SLOW-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm14 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm13 = ymm13[0,1,2,3,4,5],ymm14[6,7]
-; AVX2-SLOW-NEXT:    vmovups %ymm13, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm13 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    vmovaps {{.*#+}} xmm0 = <1,7,5,u>
+; AVX2-SLOW-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm9 # 32-byte Folded Reload
 ; AVX2-SLOW-NEXT:    vpermilps $237, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Folded Reload
 ; AVX2-SLOW-NEXT:    # ymm14 = mem[1,3,2,3,5,7,6,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm13 = ymm13[0,1,2],ymm14[3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm14 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm13 = ymm13[0,1,2,3,4,5],ymm14[6,7]
-; AVX2-SLOW-NEXT:    vmovups %ymm13, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm13 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm14 = ymm9[0,1,2],ymm14[3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vbroadcastsd {{.*#+}} ymm9 = [5,3,5,3,5,3,5,3]
+; AVX2-SLOW-NEXT:    vmovups %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm12 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm12 = ymm14[0,1,2,3,4,5],ymm12[6,7]
+; AVX2-SLOW-NEXT:    vmovups %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm12 # 32-byte Folded Reload
 ; AVX2-SLOW-NEXT:    vpermilps $237, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Folded Reload
 ; AVX2-SLOW-NEXT:    # ymm14 = mem[1,3,2,3,5,7,6,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm13 = ymm13[0,1,2],ymm14[3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm14 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm13 = ymm13[0,1,2,3,4,5],ymm14[6,7]
-; AVX2-SLOW-NEXT:    vmovups %ymm13, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm13 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm12 = ymm12[0,1,2],ymm14[3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm14 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm12 = ymm12[0,1,2,3,4,5],ymm14[6,7]
+; AVX2-SLOW-NEXT:    vmovups %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm12 # 32-byte Folded Reload
 ; AVX2-SLOW-NEXT:    vpermilps $237, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Folded Reload
 ; AVX2-SLOW-NEXT:    # ymm14 = mem[1,3,2,3,5,7,6,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm13 = ymm13[0,1,2],ymm14[3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm14 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm13 = ymm13[0,1,2,3,4,5],ymm14[6,7]
-; AVX2-SLOW-NEXT:    vmovups %ymm13, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vpermps %ymm12, %ymm5, %ymm12
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm13 = ymm15[1,3,2,3,5,7,6,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm12 = ymm12[0,1,2],ymm13[3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm12 = ymm12[0,1,2],ymm14[3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm14 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm12 = ymm12[0,1,2,3,4,5],ymm14[6,7]
+; AVX2-SLOW-NEXT:    vmovups %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm12 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    vpermilps $237, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm14 = mem[1,3,2,3,5,7,6,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm12 = ymm12[0,1,2],ymm14[3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm14 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm12 = ymm12[0,1,2,3,4,5],ymm14[6,7]
+; AVX2-SLOW-NEXT:    vmovups %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-SLOW-NEXT:    vpermps %ymm11, %ymm0, %ymm11
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm11 = ymm12[0,1,2,3,4,5],ymm11[6,7]
-; AVX2-SLOW-NEXT:    vmovups %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vpermps %ymm9, %ymm5, %ymm9
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm10 = ymm10[1,3,2,3,5,7,6,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm9 = ymm9[0,1,2],ymm10[3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vpermps %ymm7, %ymm0, %ymm7
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm7 = ymm9[0,1,2,3,4,5],ymm7[6,7]
-; AVX2-SLOW-NEXT:    vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vpermps %ymm4, %ymm5, %ymm4
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm6 = ymm6[1,3,2,3,5,7,6,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1,2],ymm6[3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vpermps %ymm3, %ymm0, %ymm3
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5],ymm3[6,7]
-; AVX2-SLOW-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vpermps %ymm2, %ymm5, %ymm2
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm3 = ymm8[1,3,2,3,5,7,6,7]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm12 = ymm13[1,3,2,3,5,7,6,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm11 = ymm11[0,1,2],ymm12[3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vpermps %ymm10, %ymm9, %ymm10
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm10 = ymm11[0,1,2,3,4,5],ymm10[6,7]
+; AVX2-SLOW-NEXT:    vmovups %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vpermps %ymm5, %ymm0, %ymm5
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm7 = ymm7[1,3,2,3,5,7,6,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0,1,2],ymm7[3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vpermps %ymm4, %ymm9, %ymm4
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm4 = ymm5[0,1,2,3,4,5],ymm4[6,7]
+; AVX2-SLOW-NEXT:    vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vpermps %ymm2, %ymm0, %ymm2
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm3 = ymm3[1,3,2,3,5,7,6,7]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2],ymm3[3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vpermps %ymm1, %ymm0, %ymm0
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5],ymm0[6,7]
+; AVX2-SLOW-NEXT:    vpermps %ymm1, %ymm9, %ymm1
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
+; AVX2-SLOW-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vpermps %ymm6, %ymm0, %ymm0
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm8[1,3,2,3,5,7,6,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vpermps %ymm15, %ymm9, %ymm1
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
 ; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vmovaps 192(%rdi), %xmm4
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm0 = xmm4[2,3,2,3]
+; AVX2-SLOW-NEXT:    vmovaps 192(%rdi), %xmm5
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm0 = xmm5[2,3,2,3]
 ; AVX2-SLOW-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Folded Reload
 ; AVX2-SLOW-NEXT:    # ymm1 = mem[0,0,2,3,4,4,6,7]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7]
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vblendps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm1 = mem[0,1],ymm1[2,3],mem[4,5],ymm1[6,7]
+; AVX2-SLOW-NEXT:    vmovups (%rsp), %ymm1 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vblendps $204, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm1 = ymm1[0,1],mem[2,3],ymm1[4,5],mem[6,7]
 ; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[2,0,2,3,6,4,6,7]
 ; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,2,0,3]
 ; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,3,2,3]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
 ; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
 ; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm3[0,0,0,0,4,4,4,4]
-; AVX2-SLOW-NEXT:    vpermilps $224, (%rsp), %ymm2 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Folded Reload
 ; AVX2-SLOW-NEXT:    # ymm2 = mem[0,0,2,3,4,4,6,7]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2],ymm1[3],ymm2[4,5,6],ymm1[7]
 ; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,1,0,3]
@@ -7216,8 +7222,8 @@ define void @load_i32_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX2-SLOW-NEXT:    # ymm1 = mem[0,0,2,3,4,4,6,7]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7]
 ; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vblendps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm1 = mem[0,1],ymm1[2,3],mem[4,5],ymm1[6,7]
+; AVX2-SLOW-NEXT:    vblendps $204, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm1 = ymm1[0,1],mem[2,3],ymm1[4,5],mem[6,7]
 ; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[2,0,2,3,6,4,6,7]
 ; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,2,0,3]
 ; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,3,2,3]
@@ -7232,8 +7238,8 @@ define void @load_i32_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-SLOW-NEXT:    vmovaps 960(%rdi), %xmm2
 ; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm0 = xmm2[2,3,2,3]
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[0,0,2,3,4,4,6,7]
+; AVX2-SLOW-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm1 = mem[0,0,2,3,4,4,6,7]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7]
 ; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
 ; AVX2-SLOW-NEXT:    vblendps $204, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
@@ -7244,52 +7250,52 @@ define void @load_i32_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
 ; AVX2-SLOW-NEXT:    vpermilps $0, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Folded Reload
 ; AVX2-SLOW-NEXT:    # ymm1 = mem[0,0,0,0,4,4,4,4]
-; AVX2-SLOW-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm5 = mem[0,0,2,3,4,4,6,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm5[0,1,2],ymm1[3],ymm5[4,5,6],ymm1[7]
+; AVX2-SLOW-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm4 = mem[0,0,2,3,4,4,6,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm4[0,1,2],ymm1[3],ymm4[4,5,6],ymm1[7]
 ; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,1,0,3]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]
 ; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-SLOW-NEXT:    vmovaps 1344(%rdi), %xmm1
 ; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm0 = xmm1[2,3,2,3]
-; AVX2-SLOW-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm5 = mem[0,0,2,3,4,4,6,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm5[1,2,3],ymm0[4],ymm5[5,6,7]
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm4 = ymm15[0,0,2,3,4,4,6,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm4[1,2,3],ymm0[4],ymm4[5,6,7]
 ; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0,1],ymm13[2,3],ymm5[4,5],ymm13[6,7]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm5 = ymm5[2,0,2,3,6,4,6,7]
-; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm5 = ymm5[0,2,0,3]
+; AVX2-SLOW-NEXT:    vblendps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm13, %ymm4 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm4 = mem[0,1],ymm13[2,3],mem[4,5],ymm13[6,7]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm4 = ymm4[2,0,2,3,6,4,6,7]
+; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm4 = ymm4[0,2,0,3]
 ; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,3,2,3]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm5[3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm4[3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vpermilps $0, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm4 = mem[0,0,0,0,4,4,4,4]
 ; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm5 = ymm14[0,0,0,0,4,4,4,4]
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm6 = ymm15[0,0,2,3,4,4,6,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm5 = ymm6[0,1,2],ymm5[3],ymm6[4,5,6],ymm5[7]
-; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm5 = ymm5[0,1,0,3]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm5[5,6,7]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm6 = ymm14[0,0,2,3,4,4,6,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm4 = ymm6[0,1,2],ymm4[3],ymm6[4,5,6],ymm4[7]
+; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm4 = ymm4[0,1,0,3]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm4[5,6,7]
 ; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-SLOW-NEXT:    vmovaps (%rdi), %xmm0
 ; AVX2-SLOW-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[2,3,2,3]
-; AVX2-SLOW-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm5 = mem[0,0,2,3,4,4,6,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm5[1,2,3],ymm0[4],ymm5[5,6,7]
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vblendps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm5 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm5 = mem[0,1],ymm5[2,3],mem[4,5],ymm5[6,7]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm5 = ymm5[2,0,2,3,6,4,6,7]
-; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm5 = ymm5[0,2,0,3]
+; AVX2-SLOW-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm4 = mem[0,0,2,3,4,4,6,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm4[1,2,3],ymm0[4],ymm4[5,6,7]
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vblendps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm4 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm4 = mem[0,1],ymm4[2,3],mem[4,5],ymm4[6,7]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm4 = ymm4[2,0,2,3,6,4,6,7]
+; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm4 = ymm4[0,2,0,3]
 ; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,3,2,3]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm5[3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vpermilps $0, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm5 = mem[0,0,0,0,4,4,4,4]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm4[3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vpermilps $0, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm4 = mem[0,0,0,0,4,4,4,4]
 ; AVX2-SLOW-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Folded Reload
 ; AVX2-SLOW-NEXT:    # ymm6 = mem[0,0,2,3,4,4,6,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm5 = ymm6[0,1,2],ymm5[3],ymm6[4,5,6],ymm5[7]
-; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm5 = ymm5[0,1,0,3]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm5[5,6,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm4 = ymm6[0,1,2],ymm4[3],ymm6[4,5,6],ymm4[7]
+; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm4 = ymm4[0,1,0,3]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm4[5,6,7]
 ; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-SLOW-NEXT:    vmovaps 384(%rdi), %xmm0
 ; AVX2-SLOW-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
@@ -7297,99 +7303,99 @@ define void @load_i32_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX2-SLOW-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Folded Reload
 ; AVX2-SLOW-NEXT:    # ymm6 = mem[0,0,2,3,4,4,6,7]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm6[1,2,3],ymm0[4],ymm6[5,6,7]
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vblendps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm6 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm6 = mem[0,1],ymm5[2,3],mem[4,5],ymm5[6,7]
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vblendps $204, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm6 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm6 = ymm4[0,1],mem[2,3],ymm4[4,5],mem[6,7]
 ; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm6 = ymm6[2,0,2,3,6,4,6,7]
 ; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm6 = ymm6[0,2,0,3]
 ; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,3,2,3]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm6[3,4,5,6,7]
 ; AVX2-SLOW-NEXT:    vpermilps $0, {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Folded Reload
 ; AVX2-SLOW-NEXT:    # ymm6 = mem[0,0,0,0,4,4,4,4]
-; AVX2-SLOW-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm7 = mem[0,0,2,3,4,4,6,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm6 = ymm7[0,1,2],ymm6[3],ymm7[4,5,6],ymm6[7]
+; AVX2-SLOW-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm8 = mem[0,0,2,3,4,4,6,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm6 = ymm8[0,1,2],ymm6[3],ymm8[4,5,6],ymm6[7]
 ; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm6 = ymm6[0,1,0,3]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm6[5,6,7]
 ; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-SLOW-NEXT:    vmovaps 768(%rdi), %xmm0
 ; AVX2-SLOW-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm6 = xmm0[2,3,2,3]
-; AVX2-SLOW-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm7 = mem[0,0,2,3,4,4,6,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm6 = ymm6[0],ymm7[1,2,3],ymm6[4],ymm7[5,6,7]
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vblendps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm7 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm7 = mem[0,1],ymm0[2,3],mem[4,5],ymm0[6,7]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm7 = ymm7[2,0,2,3,6,4,6,7]
-; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm7 = ymm7[0,2,0,3]
-; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm6 = ymm6[0,3,2,3]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm6 = ymm6[0,1,2],ymm7[3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vpermilps $0, {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm7 = mem[0,0,0,0,4,4,4,4]
 ; AVX2-SLOW-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Folded Reload
 ; AVX2-SLOW-NEXT:    # ymm8 = mem[0,0,2,3,4,4,6,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm7 = ymm8[0,1,2],ymm7[3],ymm8[4,5,6],ymm7[7]
-; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm7 = ymm7[0,1,0,3]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm6[0,1,2,3,4],ymm7[5,6,7]
-; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vmovaps 1152(%rdi), %xmm12
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm7 = xmm12[2,3,2,3]
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm8 = ymm9[0,0,2,3,4,4,6,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm7 = ymm7[0],ymm8[1,2,3],ymm7[4],ymm8[5,6,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm6 = ymm6[0],ymm8[1,2,3],ymm6[4],ymm8[5,6,7]
 ; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm8 = ymm11[0,1],ymm0[2,3],ymm11[4,5],ymm0[6,7]
+; AVX2-SLOW-NEXT:    vblendps $204, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm8 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm8 = ymm0[0,1],mem[2,3],ymm0[4,5],mem[6,7]
 ; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm8 = ymm8[2,0,2,3,6,4,6,7]
 ; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm8 = ymm8[0,2,0,3]
-; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm7 = ymm7[0,3,2,3]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm7 = ymm7[0,1,2],ymm8[3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm8 = ymm6[0,0,0,0,4,4,4,4]
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm10 = ymm5[0,0,2,3,4,4,6,7]
+; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm6 = ymm6[0,3,2,3]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm6 = ymm6[0,1,2],ymm8[3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vpermilps $0, {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm8 = mem[0,0,0,0,4,4,4,4]
+; AVX2-SLOW-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm10 = mem[0,0,2,3,4,4,6,7]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm8 = ymm10[0,1,2],ymm8[3],ymm10[4,5,6],ymm8[7]
 ; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm8 = ymm8[0,1,0,3]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm7 = ymm7[0,1,2,3,4],ymm8[5,6,7]
-; AVX2-SLOW-NEXT:    vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm4 = xmm4[3,3,3,3]
-; AVX2-SLOW-NEXT:    vpermilps $244, {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm7 = mem[0,1,3,3,4,5,7,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0],ymm7[1,2,3],ymm4[4],ymm7[5,6,7]
-; AVX2-SLOW-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm7 = mem[3,3,3,3,7,7,7,7]
-; AVX2-SLOW-NEXT:    vblendps $34, {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm7 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm7 = ymm7[0],mem[1],ymm7[2,3,4],mem[5],ymm7[6,7]
-; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm7 = ymm7[0,2,0,3]
-; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm4 = ymm4[0,3,2,3]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1,2],ymm7[3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm7 = ymm3[0,1,0,1,4,5,4,5]
-; AVX2-SLOW-NEXT:    vpermilps $244, (%rsp), %ymm8 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm6[0,1,2,3,4],ymm8[5,6,7]
+; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vmovaps 1152(%rdi), %xmm12
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm8 = xmm12[2,3,2,3]
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm10 = ymm6[0,0,2,3,4,4,6,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm8 = ymm8[0],ymm10[1,2,3],ymm8[4],ymm10[5,6,7]
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm10 = ymm9[0,1],ymm0[2,3],ymm9[4,5],ymm0[6,7]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm10 = ymm10[2,0,2,3,6,4,6,7]
+; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm10 = ymm10[0,2,0,3]
+; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm8 = ymm8[0,3,2,3]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm8 = ymm8[0,1,2],ymm10[3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm10 = ymm4[0,0,0,0,4,4,4,4]
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm11 = ymm7[0,0,2,3,4,4,6,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm10 = ymm11[0,1,2],ymm10[3],ymm11[4,5,6],ymm10[7]
+; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm10 = ymm10[0,1,0,3]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm8 = ymm8[0,1,2,3,4],ymm10[5,6,7]
+; AVX2-SLOW-NEXT:    vmovups %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm5 = xmm5[3,3,3,3]
+; AVX2-SLOW-NEXT:    vpermilps $244, {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Folded Reload
 ; AVX2-SLOW-NEXT:    # ymm8 = mem[0,1,3,3,4,5,7,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm7 = ymm8[0,1,2],ymm7[3],ymm8[4,5,6],ymm7[7]
-; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm7 = ymm7[0,1,0,3]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm3 = ymm4[0,1,2,3,4],ymm7[5,6,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0],ymm8[1,2,3],ymm5[4],ymm8[5,6,7]
+; AVX2-SLOW-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm8 = mem[3,3,3,3,7,7,7,7]
+; AVX2-SLOW-NEXT:    vblendps $34, (%rsp), %ymm8, %ymm8 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm8 = ymm8[0],mem[1],ymm8[2,3,4],mem[5],ymm8[6,7]
+; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm8 = ymm8[0,2,0,3]
+; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm5 = ymm5[0,3,2,3]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0,1,2],ymm8[3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm8 = ymm3[0,1,0,1,4,5,4,5]
+; AVX2-SLOW-NEXT:    vpermilps $244, {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm10 = mem[0,1,3,3,4,5,7,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm8 = ymm10[0,1,2],ymm8[3],ymm10[4,5,6],ymm8[7]
+; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm8 = ymm8[0,1,0,3]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm3 = ymm5[0,1,2,3,4],ymm8[5,6,7]
 ; AVX2-SLOW-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-SLOW-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
 ; AVX2-SLOW-NEXT:    # xmm3 = mem[3,3,3,3]
-; AVX2-SLOW-NEXT:    vpermilps $244, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm4 = mem[0,1,3,3,4,5,7,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0],ymm4[1,2,3],ymm3[4],ymm4[5,6,7]
-; AVX2-SLOW-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm4 = mem[3,3,3,3,7,7,7,7]
-; AVX2-SLOW-NEXT:    vblendps $34, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm4 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm4 = ymm4[0],mem[1],ymm4[2,3,4],mem[5],ymm4[6,7]
-; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm4 = ymm4[0,2,0,3]
+; AVX2-SLOW-NEXT:    vpermilps $244, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm5 = mem[0,1,3,3,4,5,7,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0],ymm5[1,2,3],ymm3[4],ymm5[5,6,7]
+; AVX2-SLOW-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm5 = mem[3,3,3,3,7,7,7,7]
+; AVX2-SLOW-NEXT:    vblendps $34, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm5 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm5 = ymm5[0],mem[1],ymm5[2,3,4],mem[5],ymm5[6,7]
+; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm5 = ymm5[0,2,0,3]
 ; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm3 = ymm3[0,3,2,3]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1,2],ymm4[3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vpermilps $68, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm4 = mem[0,1,0,1,4,5,4,5]
-; AVX2-SLOW-NEXT:    vpermilps $244, {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm7 = mem[0,1,3,3,4,5,7,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm4 = ymm7[0,1,2],ymm4[3],ymm7[4,5,6],ymm4[7]
-; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm4 = ymm4[0,1,0,3]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1,2,3,4],ymm4[5,6,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1,2],ymm5[3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vpermilps $68, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm5 = mem[0,1,0,1,4,5,4,5]
+; AVX2-SLOW-NEXT:    vpermilps $244, {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm8 = mem[0,1,3,3,4,5,7,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm5 = ymm8[0,1,2],ymm5[3],ymm8[4,5,6],ymm5[7]
+; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm5 = ymm5[0,1,0,3]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1,2,3,4],ymm5[5,6,7]
 ; AVX2-SLOW-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[3,3,3,3]
 ; AVX2-SLOW-NEXT:    vpermilps $244, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Folded Reload
@@ -7404,15 +7410,14 @@ define void @load_i32_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2],ymm3[3,4,5,6,7]
 ; AVX2-SLOW-NEXT:    vpermilps $68, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Folded Reload
 ; AVX2-SLOW-NEXT:    # ymm3 = mem[0,1,0,1,4,5,4,5]
-; AVX2-SLOW-NEXT:    vpermilps $244, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm4 = mem[0,1,3,3,4,5,7,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm3 = ymm4[0,1,2],ymm3[3],ymm4[4,5,6],ymm3[7]
+; AVX2-SLOW-NEXT:    vpermilps $244, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm5 = mem[0,1,3,3,4,5,7,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm3 = ymm5[0,1,2],ymm3[3],ymm5[4,5,6],ymm3[7]
 ; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm3 = ymm3[0,1,0,3]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm3[5,6,7]
 ; AVX2-SLOW-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
-; AVX2-SLOW-NEXT:    vpermilps $244, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm2 = mem[0,1,3,3,4,5,7,7]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm2 = ymm15[0,1,3,3,4,5,7,7]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm2[1,2,3],ymm1[4],ymm2[5,6,7]
 ; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm2 = ymm13[3,3,3,3,7,7,7,7]
 ; AVX2-SLOW-NEXT:    vblendps $34, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload
@@ -7420,302 +7425,298 @@ define void @load_i32_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[0,2,0,3]
 ; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,3,2,3]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2],ymm2[3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm2 = ymm14[0,1,0,1,4,5,4,5]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm3 = ymm15[0,1,3,3,4,5,7,7]
+; AVX2-SLOW-NEXT:    vpermilps $68, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm2 = mem[0,1,0,1,4,5,4,5]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm3 = ymm14[0,1,3,3,4,5,7,7]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1,2],ymm2[3],ymm3[4,5,6],ymm2[7]
 ; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[0,1,0,3]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm2[5,6,7]
 ; AVX2-SLOW-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-SLOW-NEXT:    vshufps {{.*#+}} xmm1 = xmm12[3,3,3,3]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm2 = ymm9[0,1,3,3,4,5,7,7]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm2 = ymm6[0,1,3,3,4,5,7,7]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm2[1,2,3],ymm1[4],ymm2[5,6,7]
 ; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm2 = ymm0[3,3,3,3,7,7,7,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0],ymm11[1],ymm2[2,3,4],ymm11[5],ymm2[6,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0],ymm9[1],ymm2[2,3,4],ymm9[5],ymm2[6,7]
 ; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[0,2,0,3]
 ; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,3,2,3]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2],ymm2[3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm2 = ymm6[0,1,0,1,4,5,4,5]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm3 = ymm5[0,1,3,3,4,5,7,7]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm2 = ymm4[0,1,0,1,4,5,4,5]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm3 = ymm7[0,1,3,3,4,5,7,7]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1,2],ymm2[3],ymm3[4,5,6],ymm2[7]
 ; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[0,1,0,3]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3,4],ymm2[5,6,7]
 ; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-SLOW-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
 ; AVX2-SLOW-NEXT:    # xmm0 = mem[3,3,3,3]
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm10[0,1,3,3,4,5,7,7]
+; AVX2-SLOW-NEXT:    vpermilps $244, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm1 = mem[0,1,3,3,4,5,7,7]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7]
 ; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload
 ; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm13[3,3,3,3,7,7,7,7]
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm11[1],ymm1[2,3,4],ymm11[5],ymm1[6,7]
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm12[1],ymm1[2,3,4],ymm12[5],ymm1[6,7]
 ; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,2,0,3]
 ; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,3,2,3]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm14[0,1,0,1,4,5,4,5]
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm2 = ymm12[0,1,3,3,4,5,7,7]
+; AVX2-SLOW-NEXT:    vpermilps $68, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm1 = mem[0,1,0,1,4,5,4,5]
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm2 = ymm15[0,1,3,3,4,5,7,7]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2],ymm1[3],ymm2[4,5,6],ymm1[7]
 ; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,1,0,3]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]
 ; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-SLOW-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
 ; AVX2-SLOW-NEXT:    # xmm0 = mem[3,3,3,3]
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm4[0,1,3,3,4,5,7,7]
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm11[0,1,3,3,4,5,7,7]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7]
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm8[3,3,3,3,7,7,7,7]
 ; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm7[3,3,3,3,7,7,7,7]
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm6[1],ymm1[2,3,4],ymm6[5],ymm1[6,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm7[1],ymm1[2,3,4],ymm7[5],ymm1[6,7]
 ; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,2,0,3]
 ; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,3,2,3]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm10[0,1,0,1,4,5,4,5]
 ; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm9[0,1,0,1,4,5,4,5]
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm2 = ymm8[0,1,3,3,4,5,7,7]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm2 = ymm9[0,1,3,3,4,5,7,7]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2],ymm1[3],ymm2[4,5,6],ymm1[7]
 ; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,1,0,3]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]
 ; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-SLOW-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
 ; AVX2-SLOW-NEXT:    # xmm0 = mem[3,3,3,3]
-; AVX2-SLOW-NEXT:    vpermilps $244, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm1 = mem[0,1,3,3,4,5,7,7]
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm6[0,1,3,3,4,5,7,7]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7]
-; AVX2-SLOW-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm1 = mem[3,3,3,3,7,7,7,7]
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm15[1],ymm1[2,3,4],ymm15[5],ymm1[6,7]
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm3[3,3,3,3,7,7,7,7]
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm14[1],ymm1[2,3,4],ymm14[5],ymm1[6,7]
 ; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,2,0,3]
 ; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,3,2,3]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vpermilps $68, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm1 = mem[0,1,0,1,4,5,4,5]
-; AVX2-SLOW-NEXT:    vpermilps $244, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm2 = mem[0,1,3,3,4,5,7,7]
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm5[0,1,0,1,4,5,4,5]
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm2 = ymm4[0,1,3,3,4,5,7,7]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2],ymm1[3],ymm2[4,5,6],ymm1[7]
 ; AVX2-SLOW-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,1,0,3]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]
 ; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm2 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm2 = ymm0[0,1,2,3],mem[4,5,6,7]
-; AVX2-SLOW-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vblendps $15, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm1 = mem[0,1,2,3],ymm0[4,5,6,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm14[0,1,2,3],ymm3[4,5,6,7]
 ; AVX2-SLOW-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vmovaps 272(%rdi), %xmm0
+; AVX2-SLOW-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm6, %ymm2 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm2 = ymm6[0,1,2,3],mem[4,5,6,7]
+; AVX2-SLOW-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vmovaps 80(%rdi), %xmm0
 ; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3],ymm1[4,5,6,7]
 ; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,1,0,2,4,5,4,6]
-; AVX2-SLOW-NEXT:    vmovddup {{.*#+}} xmm1 = [4,2,4,2]
-; AVX2-SLOW-NEXT:    # xmm1 = mem[0,0]
-; AVX2-SLOW-NEXT:    vpermps %ymm2, %ymm1, %ymm2
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1],ymm0[2,3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vblendps $243, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm0 = mem[0,1],ymm0[2,3],mem[4,5,6,7]
-; AVX2-SLOW-NEXT:    vmovups %ymm0, (%rsp) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vbroadcastf128 {{.*#+}} ymm5 = [0,2,0,6,0,2,0,6]
-; AVX2-SLOW-NEXT:    # ymm5 = mem[0,1,0,1]
-; AVX2-SLOW-NEXT:    vpermps %ymm0, %ymm5, %ymm3
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm3[5,6,7]
-; AVX2-SLOW-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm3 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm3 = ymm4[0,1,2,3],mem[4,5,6,7]
-; AVX2-SLOW-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm6[0,1,2,3],ymm7[4,5,6,7]
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vpermps %ymm2, %ymm14, %ymm1
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1],ymm0[2,3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm5[0,1],ymm4[2,3],ymm5[4,5,6,7]
 ; AVX2-SLOW-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vmovaps 464(%rdi), %xmm0
-; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1],ymm0[2,3],ymm2[4,5,6,7]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm2 = ymm2[0,1,0,2,4,5,4,6]
-; AVX2-SLOW-NEXT:    vpermps %ymm3, %ymm1, %ymm3
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1],ymm2[2,3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm9[0,1],ymm8[2,3],ymm9[4,5,6,7]
-; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vpermps %ymm0, %ymm5, %ymm3
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm3[5,6,7]
+; AVX2-SLOW-NEXT:    vbroadcastf128 {{.*#+}} ymm0 = [0,2,0,6,0,2,0,6]
+; AVX2-SLOW-NEXT:    # ymm0 = mem[0,1,0,1]
+; AVX2-SLOW-NEXT:    vpermps %ymm2, %ymm0, %ymm2
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm2[5,6,7]
+; AVX2-SLOW-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vmovups (%rsp), %ymm1 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm2 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm2 = ymm1[0,1,2,3],mem[4,5,6,7]
 ; AVX2-SLOW-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm0 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm0 = ymm2[0,1,2,3],mem[4,5,6,7]
-; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm3 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm3 = ymm2[0,1,2,3],mem[4,5,6,7]
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm3 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm3 = ymm1[0,1,2,3],mem[4,5,6,7]
 ; AVX2-SLOW-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vmovaps 656(%rdi), %xmm2
+; AVX2-SLOW-NEXT:    vmovaps 272(%rdi), %xmm1
+; AVX2-SLOW-NEXT:    vmovups %ymm1, (%rsp) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1],ymm1[2,3],ymm2[4,5,6,7]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[0,1,0,2,4,5,4,6]
+; AVX2-SLOW-NEXT:    vpermps %ymm3, %ymm14, %ymm2
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1],ymm1[2,3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vblendps $243, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm2 = mem[0,1],ymm2[2,3],mem[4,5,6,7]
 ; AVX2-SLOW-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1],ymm2[2,3],ymm3[4,5,6,7]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm2 = ymm2[0,1,0,2,4,5,4,6]
-; AVX2-SLOW-NEXT:    vpermps %ymm0, %ymm1, %ymm3
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1],ymm2[2,3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vblendps $243, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm0 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm0 = mem[0,1],ymm3[2,3],mem[4,5,6,7]
-; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vpermps %ymm0, %ymm5, %ymm3
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm3[5,6,7]
+; AVX2-SLOW-NEXT:    vpermps %ymm2, %ymm0, %ymm2
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm2[5,6,7]
+; AVX2-SLOW-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm7[0,1,2,3],ymm8[4,5,6,7]
 ; AVX2-SLOW-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm10, %ymm0 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm0 = ymm10[0,1,2,3],mem[4,5,6,7]
-; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm3 = ymm11[0,1,2,3],ymm13[4,5,6,7]
+; AVX2-SLOW-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm11, %ymm3 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm3 = ymm11[0,1,2,3],mem[4,5,6,7]
 ; AVX2-SLOW-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vmovaps 848(%rdi), %xmm2
+; AVX2-SLOW-NEXT:    vmovaps 464(%rdi), %xmm1
+; AVX2-SLOW-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1],ymm1[2,3],ymm2[4,5,6,7]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[0,1,0,2,4,5,4,6]
+; AVX2-SLOW-NEXT:    vpermps %ymm3, %ymm14, %ymm2
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1],ymm1[2,3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm10[0,1],ymm9[2,3],ymm10[4,5,6,7]
 ; AVX2-SLOW-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1],ymm2[2,3],ymm3[4,5,6,7]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm2 = ymm2[0,1,0,2,4,5,4,6]
-; AVX2-SLOW-NEXT:    vpermps %ymm0, %ymm1, %ymm3
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1],ymm2[2,3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm14[0,1],ymm12[2,3],ymm14[4,5,6,7]
-; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vpermps %ymm0, %ymm5, %ymm3
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm3[5,6,7]
+; AVX2-SLOW-NEXT:    vpermps %ymm2, %ymm0, %ymm2
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm2[5,6,7]
+; AVX2-SLOW-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm2 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm2 = ymm1[0,1,2,3],mem[4,5,6,7]
 ; AVX2-SLOW-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm0 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm0 = ymm2[0,1,2,3],mem[4,5,6,7]
-; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm3 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm3 = ymm2[0,1,2,3],mem[4,5,6,7]
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm3 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm3 = ymm1[0,1,2,3],mem[4,5,6,7]
 ; AVX2-SLOW-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vmovaps 1040(%rdi), %xmm2
-; AVX2-SLOW-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1],ymm2[2,3],ymm3[4,5,6,7]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm2 = ymm2[0,1,0,2,4,5,4,6]
-; AVX2-SLOW-NEXT:    vpermps %ymm0, %ymm1, %ymm3
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1],ymm2[2,3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vblendps $243, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm13 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm13 = mem[0,1],ymm3[2,3],mem[4,5,6,7]
-; AVX2-SLOW-NEXT:    vpermps %ymm13, %ymm5, %ymm3
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm3[5,6,7]
-; AVX2-SLOW-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm0 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm0 = ymm2[0,1,2,3],mem[4,5,6,7]
-; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vmovaps 656(%rdi), %xmm1
+; AVX2-SLOW-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1],ymm1[2,3],ymm2[4,5,6,7]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[0,1,0,2,4,5,4,6]
+; AVX2-SLOW-NEXT:    vpermps %ymm3, %ymm14, %ymm2
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1],ymm1[2,3,4,5,6,7]
 ; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm12 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm12 = ymm2[0,1,2,3],mem[4,5,6,7]
-; AVX2-SLOW-NEXT:    vmovaps 1232(%rdi), %xmm11
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm12[0,1],ymm11[2,3],ymm12[4,5,6,7]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm2 = ymm2[0,1,0,2,4,5,4,6]
-; AVX2-SLOW-NEXT:    vpermps %ymm0, %ymm1, %ymm3
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1],ymm2[2,3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vblendps $243, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm9 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm9 = mem[0,1],ymm3[2,3],mem[4,5,6,7]
-; AVX2-SLOW-NEXT:    vpermps %ymm9, %ymm5, %ymm3
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm3[5,6,7]
+; AVX2-SLOW-NEXT:    vblendps $243, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm2 = mem[0,1],ymm2[2,3],mem[4,5,6,7]
 ; AVX2-SLOW-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm0 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm0 = ymm2[0,1,2,3],mem[4,5,6,7]
-; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm7 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm7 = ymm2[0,1,2,3],mem[4,5,6,7]
-; AVX2-SLOW-NEXT:    vmovaps 1424(%rdi), %xmm8
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm7[0,1],ymm8[2,3],ymm7[4,5,6,7]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm2 = ymm2[0,1,0,2,4,5,4,6]
-; AVX2-SLOW-NEXT:    vpermps %ymm0, %ymm1, %ymm3
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1],ymm2[2,3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vblendps $243, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm6 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm6 = mem[0,1],ymm3[2,3],mem[4,5,6,7]
-; AVX2-SLOW-NEXT:    vpermps %ymm6, %ymm5, %ymm3
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm3[5,6,7]
+; AVX2-SLOW-NEXT:    vpermps %ymm2, %ymm0, %ymm2
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm2[5,6,7]
+; AVX2-SLOW-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm12[0,1,2,3],ymm13[4,5,6,7]
+; AVX2-SLOW-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm3 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm3 = ymm1[0,1,2,3],mem[4,5,6,7]
+; AVX2-SLOW-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vmovaps 848(%rdi), %xmm1
+; AVX2-SLOW-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1],ymm1[2,3],ymm2[4,5,6,7]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[0,1,0,2,4,5,4,6]
+; AVX2-SLOW-NEXT:    vpermps %ymm3, %ymm14, %ymm2
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1],ymm1[2,3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vblendps $243, {{[-0-9]+}}(%r{{[sb]}}p), %ymm15, %ymm2 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm2 = mem[0,1],ymm15[2,3],mem[4,5,6,7]
 ; AVX2-SLOW-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-SLOW-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm15, %ymm10 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm10 = ymm15[0,1,2,3],mem[4,5,6,7]
+; AVX2-SLOW-NEXT:    vpermps %ymm2, %ymm0, %ymm2
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm2[5,6,7]
+; AVX2-SLOW-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm13 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm13 = ymm1[0,1,2,3],mem[4,5,6,7]
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm12 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm12 = ymm1[0,1,2,3],mem[4,5,6,7]
+; AVX2-SLOW-NEXT:    vmovaps 1040(%rdi), %xmm11
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm13[0,1],ymm11[2,3],ymm13[4,5,6,7]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[0,1,0,2,4,5,4,6]
+; AVX2-SLOW-NEXT:    vpermps %ymm12, %ymm14, %ymm2
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1],ymm1[2,3,4,5,6,7]
 ; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm4 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm4 = ymm2[0,1,2,3],mem[4,5,6,7]
-; AVX2-SLOW-NEXT:    vmovaps 80(%rdi), %xmm3
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm10[0,1],ymm3[2,3],ymm10[4,5,6,7]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm2 = ymm2[0,1,0,2,4,5,4,6]
-; AVX2-SLOW-NEXT:    vpermps %ymm4, %ymm1, %ymm1
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1],ymm2[2,3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vblendps $12, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm10 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm10 = ymm2[0,1],mem[2,3],ymm2[4,5,6,7]
+; AVX2-SLOW-NEXT:    vpermps %ymm10, %ymm0, %ymm2
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm2[5,6,7]
+; AVX2-SLOW-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm8 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm8 = ymm1[0,1,2,3],mem[4,5,6,7]
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm7 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm7 = ymm1[0,1,2,3],mem[4,5,6,7]
+; AVX2-SLOW-NEXT:    vmovaps 1232(%rdi), %xmm6
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm8[0,1],ymm6[2,3],ymm8[4,5,6,7]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[0,1,0,2,4,5,4,6]
+; AVX2-SLOW-NEXT:    vpermps %ymm7, %ymm14, %ymm2
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1],ymm1[2,3,4,5,6,7]
 ; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vblendps $243, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    # ymm2 = mem[0,1],ymm2[2,3],mem[4,5,6,7]
-; AVX2-SLOW-NEXT:    vpermps %ymm2, %ymm5, %ymm0
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3,4],ymm0[5,6,7]
+; AVX2-SLOW-NEXT:    vblendps $243, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm5 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm5 = mem[0,1],ymm2[2,3],mem[4,5,6,7]
+; AVX2-SLOW-NEXT:    vpermps %ymm5, %ymm0, %ymm2
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm2[5,6,7]
+; AVX2-SLOW-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm4 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm4 = ymm1[0,1,2,3],mem[4,5,6,7]
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm3 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm3 = ymm1[0,1,2,3],mem[4,5,6,7]
+; AVX2-SLOW-NEXT:    vmovaps 1424(%rdi), %xmm2
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm4[0,1],ymm2[2,3],ymm4[4,5,6,7]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[0,1,0,2,4,5,4,6]
+; AVX2-SLOW-NEXT:    vpermps %ymm3, %ymm14, %ymm15
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm15 = ymm15[0,1],ymm1[2,3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vblendps $243, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    # ymm1 = mem[0,1],ymm1[2,3],mem[4,5,6,7]
+; AVX2-SLOW-NEXT:    vpermps %ymm1, %ymm0, %ymm0
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm15[0,1,2,3,4],ymm0[5,6,7]
 ; AVX2-SLOW-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-SLOW-NEXT:    vpermilps $85, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
 ; AVX2-SLOW-NEXT:    # ymm0 = mem[1,1,1,1,5,5,5,5]
 ; AVX2-SLOW-NEXT:    vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
 ; AVX2-SLOW-NEXT:    # ymm0 = ymm0[0,1,2],mem[3],ymm0[4,5,6,7]
-; AVX2-SLOW-NEXT:    vmovddup {{.*#+}} xmm1 = [5,3,5,3]
-; AVX2-SLOW-NEXT:    # xmm1 = mem[0,0]
-; AVX2-SLOW-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm14 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm14 = ymm14[0,1],ymm0[2,3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm15 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm15 = ymm15[0,1],ymm0[2,3,4,5,6,7]
 ; AVX2-SLOW-NEXT:    vbroadcastf128 {{.*#+}} ymm0 = [0,3,1,7,0,3,1,7]
 ; AVX2-SLOW-NEXT:    # ymm0 = mem[0,1,0,1]
-; AVX2-SLOW-NEXT:    vpermps (%rsp), %ymm0, %ymm15 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm5 = ymm14[0,1,2,3,4],ymm15[5,6,7]
-; AVX2-SLOW-NEXT:    vmovups %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm14 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm14 = ymm15[0,1,2,3,4],ymm14[5,6,7]
+; AVX2-SLOW-NEXT:    vmovups %ymm14, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-SLOW-NEXT:    vpermilps $85, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Folded Reload
 ; AVX2-SLOW-NEXT:    # ymm14 = mem[1,1,1,1,5,5,5,5]
-; AVX2-SLOW-NEXT:    vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm14 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    vblendps $8, (%rsp), %ymm14, %ymm14 # 32-byte Folded Reload
 ; AVX2-SLOW-NEXT:    # ymm14 = ymm14[0,1,2],mem[3],ymm14[4,5,6,7]
-; AVX2-SLOW-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm15 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm15 # 32-byte Folded Reload
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm14 = ymm15[0,1],ymm14[2,3,4,5,6,7]
 ; AVX2-SLOW-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm15 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm5 = ymm14[0,1,2,3,4],ymm15[5,6,7]
-; AVX2-SLOW-NEXT:    vmovups %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm14 = ymm14[0,1,2,3,4],ymm15[5,6,7]
+; AVX2-SLOW-NEXT:    vmovups %ymm14, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-SLOW-NEXT:    vpermilps $85, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Folded Reload
 ; AVX2-SLOW-NEXT:    # ymm14 = mem[1,1,1,1,5,5,5,5]
 ; AVX2-SLOW-NEXT:    vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm14 # 32-byte Folded Reload
 ; AVX2-SLOW-NEXT:    # ymm14 = ymm14[0,1,2],mem[3],ymm14[4,5,6,7]
-; AVX2-SLOW-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm15 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm15 # 32-byte Folded Reload
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm14 = ymm15[0,1],ymm14[2,3,4,5,6,7]
 ; AVX2-SLOW-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm15 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm5 = ymm14[0,1,2,3,4],ymm15[5,6,7]
-; AVX2-SLOW-NEXT:    vmovups %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm14 = ymm14[0,1,2,3,4],ymm15[5,6,7]
+; AVX2-SLOW-NEXT:    vmovups %ymm14, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-SLOW-NEXT:    vpermilps $85, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Folded Reload
 ; AVX2-SLOW-NEXT:    # ymm14 = mem[1,1,1,1,5,5,5,5]
 ; AVX2-SLOW-NEXT:    vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm14 # 32-byte Folded Reload
 ; AVX2-SLOW-NEXT:    # ymm14 = ymm14[0,1,2],mem[3],ymm14[4,5,6,7]
-; AVX2-SLOW-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm15 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm15 # 32-byte Folded Reload
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm14 = ymm15[0,1],ymm14[2,3,4,5,6,7]
 ; AVX2-SLOW-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm15 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm5 = ymm14[0,1,2,3,4],ymm15[5,6,7]
-; AVX2-SLOW-NEXT:    vmovups %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm14 = ymm14[0,1,2,3,4],ymm15[5,6,7]
+; AVX2-SLOW-NEXT:    vmovups %ymm14, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-SLOW-NEXT:    vpermilps $85, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Folded Reload
 ; AVX2-SLOW-NEXT:    # ymm14 = mem[1,1,1,1,5,5,5,5]
 ; AVX2-SLOW-NEXT:    vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm14 # 32-byte Folded Reload
 ; AVX2-SLOW-NEXT:    # ymm14 = ymm14[0,1,2],mem[3],ymm14[4,5,6,7]
-; AVX2-SLOW-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm15 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm15 # 32-byte Folded Reload
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm14 = ymm15[0,1],ymm14[2,3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vpermps %ymm13, %ymm0, %ymm13
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm13 = ymm14[0,1,2,3,4],ymm13[5,6,7]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm12 = ymm12[1,1,1,1,5,5,5,5]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm11 = ymm12[0,1,2],ymm11[3],ymm12[4,5,6,7]
-; AVX2-SLOW-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm12 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm15 # 32-byte Folded Reload
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm14 = ymm14[0,1,2,3,4],ymm15[5,6,7]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm13 = ymm13[1,1,1,1,5,5,5,5]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm11 = ymm13[0,1,2],ymm11[3],ymm13[4,5,6,7]
+; AVX2-SLOW-NEXT:    vpermps %ymm12, %ymm9, %ymm12
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm11 = ymm12[0,1],ymm11[2,3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vpermps %ymm9, %ymm0, %ymm9
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm9 = ymm11[0,1,2,3,4],ymm9[5,6,7]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm7 = ymm7[1,1,1,1,5,5,5,5]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm7 = ymm7[0,1,2],ymm8[3],ymm7[4,5,6,7]
-; AVX2-SLOW-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm8 # 32-byte Folded Reload
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm7 = ymm8[0,1],ymm7[2,3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vpermps %ymm6, %ymm0, %ymm6
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm6 = ymm7[0,1,2,3,4],ymm6[5,6,7]
-; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm5 = ymm10[1,1,1,1,5,5,5,5]
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm3 = ymm5[0,1,2],ymm3[3],ymm5[4,5,6,7]
-; AVX2-SLOW-NEXT:    vpermps %ymm4, %ymm1, %ymm1
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1],ymm3[2,3,4,5,6,7]
-; AVX2-SLOW-NEXT:    vpermps %ymm2, %ymm0, %ymm0
-; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3,4],ymm0[5,6,7]
+; AVX2-SLOW-NEXT:    vpermps %ymm10, %ymm0, %ymm10
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm10 = ymm11[0,1,2,3,4],ymm10[5,6,7]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm8 = ymm8[1,1,1,1,5,5,5,5]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm6 = ymm8[0,1,2],ymm6[3],ymm8[4,5,6,7]
+; AVX2-SLOW-NEXT:    vpermps %ymm7, %ymm9, %ymm7
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm6 = ymm7[0,1],ymm6[2,3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vpermps %ymm5, %ymm0, %ymm5
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm5 = ymm6[0,1,2,3,4],ymm5[5,6,7]
+; AVX2-SLOW-NEXT:    vshufps {{.*#+}} ymm4 = ymm4[1,1,1,1,5,5,5,5]
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm4[0,1,2],ymm2[3],ymm4[4,5,6,7]
+; AVX2-SLOW-NEXT:    vpermps %ymm3, %ymm9, %ymm3
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1],ymm2[2,3,4,5,6,7]
+; AVX2-SLOW-NEXT:    vpermps %ymm1, %ymm0, %ymm0
+; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3,4],ymm0[5,6,7]
 ; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
 ; AVX2-SLOW-NEXT:    vmovaps %ymm1, 192(%rsi)
 ; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
@@ -7797,788 +7798,785 @@ define void @load_i32_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
 ; AVX2-SLOW-NEXT:    vmovaps %ymm1, (%r9)
 ; AVX2-SLOW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX2-SLOW-NEXT:    vmovaps %ymm6, 224(%rax)
-; AVX2-SLOW-NEXT:    vmovaps %ymm9, 192(%rax)
-; AVX2-SLOW-NEXT:    vmovaps %ymm13, 160(%rax)
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vmovaps %ymm1, 128(%rax)
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vmovaps %ymm1, 96(%rax)
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vmovaps %ymm1, 64(%rax)
-; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX2-SLOW-NEXT:    vmovaps %ymm1, 32(%rax)
+; AVX2-SLOW-NEXT:    vmovaps %ymm0, 224(%rax)
+; AVX2-SLOW-NEXT:    vmovaps %ymm5, 192(%rax)
+; AVX2-SLOW-NEXT:    vmovaps %ymm10, 160(%rax)
+; AVX2-SLOW-NEXT:    vmovaps %ymm14, 128(%rax)
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vmovaps %ymm0, 96(%rax)
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vmovaps %ymm0, 64(%rax)
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX2-SLOW-NEXT:    vmovaps %ymm0, 32(%rax)
+; AVX2-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX2-SLOW-NEXT:    vmovaps %ymm0, (%rax)
-; AVX2-SLOW-NEXT:    addq $2504, %rsp # imm = 0x9C8
+; AVX2-SLOW-NEXT:    addq $2536, %rsp # imm = 0x9E8
 ; AVX2-SLOW-NEXT:    vzeroupper
 ; AVX2-SLOW-NEXT:    retq
 ;
 ; AVX2-FAST-LABEL: load_i32_stride6_vf64:
 ; AVX2-FAST:       # %bb.0:
 ; AVX2-FAST-NEXT:    subq $2504, %rsp # imm = 0x9C8
-; AVX2-FAST-NEXT:    vmovaps 672(%rdi), %ymm2
-; AVX2-FAST-NEXT:    vmovaps 640(%rdi), %ymm3
-; AVX2-FAST-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovaps 608(%rdi), %ymm4
-; AVX2-FAST-NEXT:    vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovaps 320(%rdi), %ymm5
+; AVX2-FAST-NEXT:    vmovaps 672(%rdi), %ymm4
+; AVX2-FAST-NEXT:    vmovaps 640(%rdi), %ymm5
 ; AVX2-FAST-NEXT:    vmovups %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovaps 352(%rdi), %ymm6
+; AVX2-FAST-NEXT:    vmovaps 608(%rdi), %ymm3
+; AVX2-FAST-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vmovaps 320(%rdi), %ymm6
 ; AVX2-FAST-NEXT:    vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovaps 288(%rdi), %ymm7
+; AVX2-FAST-NEXT:    vmovaps 352(%rdi), %ymm7
 ; AVX2-FAST-NEXT:    vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vmovaps 288(%rdi), %ymm8
+; AVX2-FAST-NEXT:    vmovups %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-NEXT:    vmovaps 256(%rdi), %ymm9
-; AVX2-FAST-NEXT:    vmovups %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovaps 224(%rdi), %ymm0
-; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovaps 192(%rdi), %ymm1
+; AVX2-FAST-NEXT:    vmovups %ymm9, (%rsp) # 32-byte Spill
+; AVX2-FAST-NEXT:    vmovaps 224(%rdi), %ymm1
+; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vmovaps 192(%rdi), %ymm2
+; AVX2-FAST-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vmovaps {{.*#+}} xmm0 = <0,6,4,u>
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm14 = ymm2[0,1,2,3],ymm1[4,5],ymm2[6,7]
+; AVX2-FAST-NEXT:    vpermps %ymm14, %ymm0, %ymm1
+; AVX2-FAST-NEXT:    vperm2f128 {{.*#+}} ymm2 = ymm9[0,1],ymm8[0,1]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm15 = ymm2[0,1,2,3,4,5],ymm8[6,7]
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm2 = ymm15[0,2,2,2,4,6,6,6]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2],ymm2[3,4,5,6,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm7[0,1,2,3],ymm6[4,5,6,7]
+; AVX2-FAST-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vbroadcastsd {{.*#+}} ymm13 = [4,2,4,2,4,2,4,2]
+; AVX2-FAST-NEXT:    vpermps %ymm2, %ymm13, %ymm2
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm2[6,7]
 ; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovaps {{.*#+}} xmm8 = <0,6,4,u>
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm15 = ymm1[0,1,2,3],ymm0[4,5],ymm1[6,7]
-; AVX2-FAST-NEXT:    vpermps %ymm15, %ymm8, %ymm0
-; AVX2-FAST-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm9[0,1],ymm7[0,1]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm7[6,7]
+; AVX2-FAST-NEXT:    vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm5[0,1],ymm4[0,1]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm4 = ymm1[0,1,2,3,4,5],ymm4[6,7]
+; AVX2-FAST-NEXT:    vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vmovaps 576(%rdi), %ymm1
 ; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[0,2,2,2,4,6,6,6]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm6[0,1,2,3],ymm5[4,5,6,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm3[4,5],ymm1[6,7]
 ; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vbroadcastsd {{.*#+}} ymm5 = [4,2,4,2,4,2,4,2]
-; AVX2-FAST-NEXT:    vpermps %ymm1, %ymm5, %ymm1
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
-; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vpermps %ymm1, %ymm0, %ymm1
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm2 = ymm4[0,2,2,2,4,6,6,6]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2],ymm2[3,4,5,6,7]
+; AVX2-FAST-NEXT:    vmovaps 704(%rdi), %ymm2
 ; AVX2-FAST-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm3[0,1],ymm2[0,1]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm0[0,1,2,3,4,5],ymm2[6,7]
+; AVX2-FAST-NEXT:    vmovaps 736(%rdi), %ymm3
+; AVX2-FAST-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
 ; AVX2-FAST-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovaps 576(%rdi), %ymm0
-; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm4[4,5],ymm0[6,7]
-; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vpermps %ymm0, %ymm8, %ymm0
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm2[0,2,2,2,4,6,6,6]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
-; AVX2-FAST-NEXT:    vmovaps 704(%rdi), %ymm1
+; AVX2-FAST-NEXT:    vpermps %ymm2, %ymm13, %ymm2
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm2[6,7]
 ; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovaps 736(%rdi), %ymm2
+; AVX2-FAST-NEXT:    vmovaps 1056(%rdi), %ymm2
 ; AVX2-FAST-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-FAST-NEXT:    vmovaps 1024(%rdi), %ymm1
 ; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vpermps %ymm1, %ymm5, %ymm1
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
-; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovaps 1056(%rdi), %ymm1
+; AVX2-FAST-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm1[0,1],ymm2[0,1]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm3 = ymm1[0,1,2,3,4,5],ymm2[6,7]
+; AVX2-FAST-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vmovaps 992(%rdi), %ymm1
 ; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovaps 1024(%rdi), %ymm0
-; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[0,1],ymm1[0,1]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm0[0,1,2,3,4,5],ymm1[6,7]
+; AVX2-FAST-NEXT:    vmovaps 960(%rdi), %ymm2
 ; AVX2-FAST-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovaps 992(%rdi), %ymm0
-; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovaps 960(%rdi), %ymm1
-; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5],ymm1[6,7]
-; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vpermps %ymm0, %ymm8, %ymm0
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm2[0,2,2,2,4,6,6,6]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
-; AVX2-FAST-NEXT:    vmovaps 1088(%rdi), %ymm1
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5],ymm2[6,7]
 ; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovaps 1120(%rdi), %ymm2
+; AVX2-FAST-NEXT:    vpermps %ymm1, %ymm0, %ymm1
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm2 = ymm3[0,2,2,2,4,6,6,6]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2],ymm2[3,4,5,6,7]
+; AVX2-FAST-NEXT:    vmovaps 1088(%rdi), %ymm2
 ; AVX2-FAST-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vpermps %ymm1, %ymm5, %ymm1
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
-; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovaps 1440(%rdi), %ymm1
+; AVX2-FAST-NEXT:    vmovaps 1120(%rdi), %ymm3
+; AVX2-FAST-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
+; AVX2-FAST-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vpermps %ymm2, %ymm13, %ymm2
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm2[6,7]
 ; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovaps 1408(%rdi), %ymm0
-; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[0,1],ymm1[0,1]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm0[0,1,2,3,4,5],ymm1[6,7]
+; AVX2-FAST-NEXT:    vmovaps 1440(%rdi), %ymm2
 ; AVX2-FAST-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovaps 1376(%rdi), %ymm0
-; AVX2-FAST-NEXT:    vmovups %ymm0, (%rsp) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovaps 1344(%rdi), %ymm1
+; AVX2-FAST-NEXT:    vmovaps 1408(%rdi), %ymm1
 ; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5],ymm1[6,7]
-; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vpermps %ymm0, %ymm8, %ymm0
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm2[0,2,2,2,4,6,6,6]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
-; AVX2-FAST-NEXT:    vmovaps 1472(%rdi), %ymm1
+; AVX2-FAST-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm1[0,1],ymm2[0,1]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm3 = ymm1[0,1,2,3,4,5],ymm2[6,7]
+; AVX2-FAST-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vmovaps 1376(%rdi), %ymm1
 ; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovaps 1504(%rdi), %ymm2
+; AVX2-FAST-NEXT:    vmovaps 1344(%rdi), %ymm2
 ; AVX2-FAST-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5],ymm2[6,7]
 ; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vpermps %ymm1, %ymm5, %ymm1
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
-; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovaps 96(%rdi), %ymm1
+; AVX2-FAST-NEXT:    vpermps %ymm1, %ymm0, %ymm1
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm2 = ymm3[0,2,2,2,4,6,6,6]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2],ymm2[3,4,5,6,7]
+; AVX2-FAST-NEXT:    vmovaps 1472(%rdi), %ymm2
+; AVX2-FAST-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vmovaps 1504(%rdi), %ymm3
+; AVX2-FAST-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
+; AVX2-FAST-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vpermps %ymm2, %ymm13, %ymm2
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm2[6,7]
 ; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovaps 64(%rdi), %ymm0
-; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[0,1],ymm1[0,1]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm14 = ymm0[0,1,2,3,4,5],ymm1[6,7]
-; AVX2-FAST-NEXT:    vmovaps (%rdi), %ymm0
-; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovaps 32(%rdi), %ymm1
+; AVX2-FAST-NEXT:    vmovaps 96(%rdi), %ymm2
+; AVX2-FAST-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vmovaps 64(%rdi), %ymm1
 ; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm12 = ymm0[0,1,2,3],ymm1[4,5],ymm0[6,7]
-; AVX2-FAST-NEXT:    vpermps %ymm12, %ymm8, %ymm0
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm14[0,2,2,2,4,6,6,6]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
-; AVX2-FAST-NEXT:    vmovaps 128(%rdi), %ymm1
+; AVX2-FAST-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm1[0,1],ymm2[0,1]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm3 = ymm1[0,1,2,3,4,5],ymm2[6,7]
+; AVX2-FAST-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vmovaps (%rdi), %ymm1
 ; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovaps 160(%rdi), %ymm2
+; AVX2-FAST-NEXT:    vmovaps 32(%rdi), %ymm2
 ; AVX2-FAST-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm11 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVX2-FAST-NEXT:    vpermps %ymm11, %ymm5, %ymm1
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
-; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovaps 480(%rdi), %ymm1
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm12 = ymm1[0,1,2,3],ymm2[4,5],ymm1[6,7]
+; AVX2-FAST-NEXT:    vpermps %ymm12, %ymm0, %ymm1
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm2 = ymm3[0,2,2,2,4,6,6,6]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2],ymm2[3,4,5,6,7]
+; AVX2-FAST-NEXT:    vmovaps 128(%rdi), %ymm2
+; AVX2-FAST-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vmovaps 160(%rdi), %ymm3
+; AVX2-FAST-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm11 = ymm3[0,1,2,3],ymm2[4,5,6,7]
+; AVX2-FAST-NEXT:    vpermps %ymm11, %ymm13, %ymm2
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm2[6,7]
 ; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovaps 448(%rdi), %ymm0
-; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[0,1],ymm1[0,1]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm10 = ymm0[0,1,2,3,4,5],ymm1[6,7]
-; AVX2-FAST-NEXT:    vmovaps 416(%rdi), %ymm0
-; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovaps 384(%rdi), %ymm1
+; AVX2-FAST-NEXT:    vmovaps 480(%rdi), %ymm2
+; AVX2-FAST-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vmovaps 448(%rdi), %ymm1
 ; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm9 = ymm1[0,1,2,3],ymm0[4,5],ymm1[6,7]
-; AVX2-FAST-NEXT:    vpermps %ymm9, %ymm8, %ymm0
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm10[0,2,2,2,4,6,6,6]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
-; AVX2-FAST-NEXT:    vmovaps 512(%rdi), %ymm1
+; AVX2-FAST-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm1[0,1],ymm2[0,1]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm10 = ymm1[0,1,2,3,4,5],ymm2[6,7]
+; AVX2-FAST-NEXT:    vmovaps 416(%rdi), %ymm1
 ; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovaps 544(%rdi), %ymm2
+; AVX2-FAST-NEXT:    vmovaps 384(%rdi), %ymm2
 ; AVX2-FAST-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm7 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVX2-FAST-NEXT:    vpermps %ymm7, %ymm5, %ymm1
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
-; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovaps 864(%rdi), %ymm1
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm9 = ymm2[0,1,2,3],ymm1[4,5],ymm2[6,7]
+; AVX2-FAST-NEXT:    vpermps %ymm9, %ymm0, %ymm1
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm2 = ymm10[0,2,2,2,4,6,6,6]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2],ymm2[3,4,5,6,7]
+; AVX2-FAST-NEXT:    vmovaps 512(%rdi), %ymm2
+; AVX2-FAST-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vmovaps 544(%rdi), %ymm3
+; AVX2-FAST-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm8 = ymm3[0,1,2,3],ymm2[4,5,6,7]
+; AVX2-FAST-NEXT:    vpermps %ymm8, %ymm13, %ymm2
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm2[6,7]
 ; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovaps 832(%rdi), %ymm0
-; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[0,1],ymm1[0,1]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm6 = ymm0[0,1,2,3,4,5],ymm1[6,7]
-; AVX2-FAST-NEXT:    vmovaps 800(%rdi), %ymm0
-; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovaps 768(%rdi), %ymm1
+; AVX2-FAST-NEXT:    vmovaps 864(%rdi), %ymm2
+; AVX2-FAST-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vmovaps 832(%rdi), %ymm1
 ; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm4 = ymm1[0,1,2,3],ymm0[4,5],ymm1[6,7]
-; AVX2-FAST-NEXT:    vpermps %ymm4, %ymm8, %ymm0
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm6[0,2,2,2,4,6,6,6]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
-; AVX2-FAST-NEXT:    vmovaps 896(%rdi), %ymm1
+; AVX2-FAST-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm1[0,1],ymm2[0,1]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm6 = ymm1[0,1,2,3,4,5],ymm2[6,7]
+; AVX2-FAST-NEXT:    vmovaps 800(%rdi), %ymm1
 ; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovaps 928(%rdi), %ymm2
+; AVX2-FAST-NEXT:    vmovaps 768(%rdi), %ymm2
 ; AVX2-FAST-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm3 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVX2-FAST-NEXT:    vpermps %ymm3, %ymm5, %ymm1
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
-; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovaps 1184(%rdi), %ymm0
-; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovaps 1152(%rdi), %ymm1
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm5 = ymm2[0,1,2,3],ymm1[4,5],ymm2[6,7]
+; AVX2-FAST-NEXT:    vpermps %ymm5, %ymm0, %ymm1
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm2 = ymm6[0,2,2,2,4,6,6,6]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2],ymm2[3,4,5,6,7]
+; AVX2-FAST-NEXT:    vmovaps 896(%rdi), %ymm2
+; AVX2-FAST-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vmovaps 928(%rdi), %ymm3
+; AVX2-FAST-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm4 = ymm3[0,1,2,3],ymm2[4,5,6,7]
+; AVX2-FAST-NEXT:    vpermps %ymm4, %ymm13, %ymm2
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm2[6,7]
 ; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm1[0,1,2,3],ymm0[4,5],ymm1[6,7]
-; AVX2-FAST-NEXT:    vpermps %ymm2, %ymm8, %ymm0
-; AVX2-FAST-NEXT:    vmovaps 1248(%rdi), %ymm8
-; AVX2-FAST-NEXT:    vmovups %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vmovaps 1184(%rdi), %ymm1
+; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vmovaps 1152(%rdi), %ymm2
+; AVX2-FAST-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm3 = ymm2[0,1,2,3],ymm1[4,5],ymm2[6,7]
+; AVX2-FAST-NEXT:    vpermps %ymm3, %ymm0, %ymm0
+; AVX2-FAST-NEXT:    vmovaps 1248(%rdi), %ymm2
+; AVX2-FAST-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-NEXT:    vmovaps 1216(%rdi), %ymm1
 ; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm1[0,1],ymm8[0,1]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm8 = ymm1[0,1,2,3,4,5],ymm8[6,7]
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm8[0,2,2,2,4,6,6,6]
+; AVX2-FAST-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm1[0,1],ymm2[0,1]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm1[0,1,2,3,4,5],ymm2[6,7]
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm2[0,2,2,2,4,6,6,6]
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
 ; AVX2-FAST-NEXT:    vmovaps 1280(%rdi), %ymm1
 ; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovaps 1312(%rdi), %ymm13
-; AVX2-FAST-NEXT:    vmovups %ymm13, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm13[0,1,2,3],ymm1[4,5,6,7]
-; AVX2-FAST-NEXT:    vpermps %ymm1, %ymm5, %ymm5
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm5[6,7]
+; AVX2-FAST-NEXT:    vmovaps 1312(%rdi), %ymm7
+; AVX2-FAST-NEXT:    vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm7[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-FAST-NEXT:    vpermps %ymm1, %ymm13, %ymm7
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm7[6,7]
 ; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovaps {{.*#+}} xmm5 = <1,7,5,u>
-; AVX2-FAST-NEXT:    vpermps %ymm15, %ymm5, %ymm0
-; AVX2-FAST-NEXT:    vpermilps $237, {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm15 = mem[1,3,2,3,5,7,6,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm15 = ymm0[0,1,2],ymm15[3,4,5,6,7]
-; AVX2-FAST-NEXT:    vbroadcastsd {{.*#+}} ymm0 = [5,3,5,3,5,3,5,3]
-; AVX2-FAST-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm13 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm13 = ymm15[0,1,2,3,4,5],ymm13[6,7]
-; AVX2-FAST-NEXT:    vmovups %ymm13, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm13 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    vmovaps {{.*#+}} xmm0 = <1,7,5,u>
+; AVX2-FAST-NEXT:    vpermps %ymm14, %ymm0, %ymm7
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm14 = ymm15[1,3,2,3,5,7,6,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm14 = ymm7[0,1,2],ymm14[3,4,5,6,7]
+; AVX2-FAST-NEXT:    vbroadcastsd {{.*#+}} ymm7 = [5,3,5,3,5,3,5,3]
+; AVX2-FAST-NEXT:    vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm15 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm14 = ymm14[0,1,2,3,4,5],ymm15[6,7]
+; AVX2-FAST-NEXT:    vmovups %ymm14, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm14 # 32-byte Folded Reload
 ; AVX2-FAST-NEXT:    vpermilps $237, {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Folded Reload
 ; AVX2-FAST-NEXT:    # ymm15 = mem[1,3,2,3,5,7,6,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm13 = ymm13[0,1,2],ymm15[3,4,5,6,7]
-; AVX2-FAST-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm15 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm13 = ymm13[0,1,2,3,4,5],ymm15[6,7]
-; AVX2-FAST-NEXT:    vmovups %ymm13, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm13 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm14 = ymm14[0,1,2],ymm15[3,4,5,6,7]
+; AVX2-FAST-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm15 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm14 = ymm14[0,1,2,3,4,5],ymm15[6,7]
+; AVX2-FAST-NEXT:    vmovups %ymm14, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm14 # 32-byte Folded Reload
 ; AVX2-FAST-NEXT:    vpermilps $237, {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Folded Reload
 ; AVX2-FAST-NEXT:    # ymm15 = mem[1,3,2,3,5,7,6,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm13 = ymm13[0,1,2],ymm15[3,4,5,6,7]
-; AVX2-FAST-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm15 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm13 = ymm13[0,1,2,3,4,5],ymm15[6,7]
-; AVX2-FAST-NEXT:    vmovups %ymm13, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm13 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm14 = ymm14[0,1,2],ymm15[3,4,5,6,7]
+; AVX2-FAST-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm15 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm14 = ymm14[0,1,2,3,4,5],ymm15[6,7]
+; AVX2-FAST-NEXT:    vmovups %ymm14, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm14 # 32-byte Folded Reload
 ; AVX2-FAST-NEXT:    vpermilps $237, {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Folded Reload
 ; AVX2-FAST-NEXT:    # ymm15 = mem[1,3,2,3,5,7,6,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm13 = ymm13[0,1,2],ymm15[3,4,5,6,7]
-; AVX2-FAST-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm15 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm13 = ymm13[0,1,2,3,4,5],ymm15[6,7]
-; AVX2-FAST-NEXT:    vmovups %ymm13, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vpermps %ymm12, %ymm5, %ymm12
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm13 = ymm14[1,3,2,3,5,7,6,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm12 = ymm12[0,1,2],ymm13[3,4,5,6,7]
-; AVX2-FAST-NEXT:    vpermps %ymm11, %ymm0, %ymm11
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm14 = ymm14[0,1,2],ymm15[3,4,5,6,7]
+; AVX2-FAST-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm15 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm14 = ymm14[0,1,2,3,4,5],ymm15[6,7]
+; AVX2-FAST-NEXT:    vmovups %ymm14, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vpermps %ymm12, %ymm0, %ymm12
+; AVX2-FAST-NEXT:    vpermilps $237, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm14 = mem[1,3,2,3,5,7,6,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm12 = ymm12[0,1,2],ymm14[3,4,5,6,7]
+; AVX2-FAST-NEXT:    vpermps %ymm11, %ymm7, %ymm11
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm11 = ymm12[0,1,2,3,4,5],ymm11[6,7]
 ; AVX2-FAST-NEXT:    vmovups %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vpermps %ymm9, %ymm5, %ymm9
+; AVX2-FAST-NEXT:    vpermps %ymm9, %ymm0, %ymm9
 ; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm10 = ymm10[1,3,2,3,5,7,6,7]
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm9 = ymm9[0,1,2],ymm10[3,4,5,6,7]
-; AVX2-FAST-NEXT:    vpermps %ymm7, %ymm0, %ymm7
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm7 = ymm9[0,1,2,3,4,5],ymm7[6,7]
-; AVX2-FAST-NEXT:    vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vpermps %ymm4, %ymm5, %ymm4
+; AVX2-FAST-NEXT:    vpermps %ymm8, %ymm7, %ymm8
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm8 = ymm9[0,1,2,3,4,5],ymm8[6,7]
+; AVX2-FAST-NEXT:    vmovups %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vpermps %ymm5, %ymm0, %ymm5
 ; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm6 = ymm6[1,3,2,3,5,7,6,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1,2],ymm6[3,4,5,6,7]
-; AVX2-FAST-NEXT:    vpermps %ymm3, %ymm0, %ymm3
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5],ymm3[6,7]
-; AVX2-FAST-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vpermps %ymm2, %ymm5, %ymm2
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm3 = ymm8[1,3,2,3,5,7,6,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2],ymm3[3,4,5,6,7]
-; AVX2-FAST-NEXT:    vpermps %ymm1, %ymm0, %ymm0
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5],ymm0[6,7]
-; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovaps 192(%rdi), %xmm2
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} xmm0 = xmm2[2,3,2,3]
-; AVX2-FAST-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm1 = mem[0,0,2,3,4,4,6,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7]
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX2-FAST-NEXT:    vblendps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm1 = mem[0,1],ymm1[2,3],mem[4,5],ymm1[6,7]
-; AVX2-FAST-NEXT:    vmovaps {{.*#+}} ymm6 = [2,0,6,4,2,0,6,7]
-; AVX2-FAST-NEXT:    vpermps %ymm1, %ymm6, %ymm1
-; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,3,2,3]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
-; AVX2-FAST-NEXT:    vpermilps $0, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm1 = mem[0,0,0,0,4,4,4,4]
-; AVX2-FAST-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm3 = mem[0,0,2,3,4,4,6,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm3[0,1,2],ymm1[3],ymm3[4,5,6],ymm1[7]
-; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,1,0,3]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0,1,2],ymm6[3,4,5,6,7]
+; AVX2-FAST-NEXT:    vpermps %ymm4, %ymm7, %ymm4
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm4 = ymm5[0,1,2,3,4,5],ymm4[6,7]
+; AVX2-FAST-NEXT:    vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vpermps %ymm3, %ymm0, %ymm0
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm2 = ymm2[1,3,2,3,5,7,6,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm2[3,4,5,6,7]
+; AVX2-FAST-NEXT:    vpermps %ymm1, %ymm7, %ymm1
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
 ; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovaps 576(%rdi), %xmm1
+; AVX2-FAST-NEXT:    vmovaps 192(%rdi), %xmm1
 ; AVX2-FAST-NEXT:    vshufps {{.*#+}} xmm0 = xmm1[2,3,2,3]
-; AVX2-FAST-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm3 = mem[0,0,2,3,4,4,6,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm3[1,2,3],ymm0[4],ymm3[5,6,7]
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm3 = ymm13[0,1],ymm12[2,3],ymm13[4,5],ymm12[6,7]
-; AVX2-FAST-NEXT:    vpermps %ymm3, %ymm6, %ymm3
-; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,3,2,3]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm3[3,4,5,6,7]
+; AVX2-FAST-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm2 = mem[0,0,2,3,4,4,6,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm0[0],ymm2[1,2,3],ymm0[4],ymm2[5,6,7]
+; AVX2-FAST-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
+; AVX2-FAST-NEXT:    vblendps $204, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm3 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm3 = ymm0[0,1],mem[2,3],ymm0[4,5],mem[6,7]
+; AVX2-FAST-NEXT:    vmovaps {{.*#+}} ymm0 = [2,0,6,4,2,0,6,7]
+; AVX2-FAST-NEXT:    vpermps %ymm3, %ymm0, %ymm3
+; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[0,3,2,3]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2],ymm3[3,4,5,6,7]
 ; AVX2-FAST-NEXT:    vpermilps $0, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Folded Reload
 ; AVX2-FAST-NEXT:    # ymm3 = mem[0,0,0,0,4,4,4,4]
 ; AVX2-FAST-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Folded Reload
 ; AVX2-FAST-NEXT:    # ymm4 = mem[0,0,2,3,4,4,6,7]
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm3 = ymm4[0,1,2],ymm3[3],ymm4[4,5,6],ymm3[7]
 ; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm3 = ymm3[0,1,0,3]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm3[5,6,7]
-; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovaps 960(%rdi), %xmm0
-; AVX2-FAST-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm3[5,6,7]
+; AVX2-FAST-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vmovaps 576(%rdi), %xmm3
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} xmm2 = xmm3[2,3,2,3]
 ; AVX2-FAST-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Folded Reload
 ; AVX2-FAST-NEXT:    # ymm4 = mem[0,0,2,3,4,4,6,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm4[1,2,3],ymm0[4],ymm4[5,6,7]
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
-; AVX2-FAST-NEXT:    vblendps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm4 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm4 = mem[0,1],ymm3[2,3],mem[4,5],ymm3[6,7]
-; AVX2-FAST-NEXT:    vpermps %ymm4, %ymm6, %ymm4
-; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,3,2,3]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm4[3,4,5,6,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0],ymm4[1,2,3],ymm2[4],ymm4[5,6,7]
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
+; AVX2-FAST-NEXT:    vblendps $204, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm4 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm4 = ymm4[0,1],mem[2,3],ymm4[4,5],mem[6,7]
+; AVX2-FAST-NEXT:    vpermps %ymm4, %ymm0, %ymm4
+; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[0,3,2,3]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2],ymm4[3,4,5,6,7]
 ; AVX2-FAST-NEXT:    vpermilps $0, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Folded Reload
 ; AVX2-FAST-NEXT:    # ymm4 = mem[0,0,0,0,4,4,4,4]
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm5 = ymm15[0,0,2,3,4,4,6,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm4 = ymm5[0,1,2],ymm4[3],ymm5[4,5,6],ymm4[7]
-; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm4 = ymm4[0,1,0,3]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm4[5,6,7]
-; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovaps 1344(%rdi), %xmm0
-; AVX2-FAST-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[2,3,2,3]
-; AVX2-FAST-NEXT:    vpermilps $224, (%rsp), %ymm5 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm5 = mem[0,0,2,3,4,4,6,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm5[1,2,3],ymm0[4],ymm5[5,6,7]
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm5 = ymm4[0,1],ymm14[2,3],ymm4[4,5],ymm14[6,7]
-; AVX2-FAST-NEXT:    vpermps %ymm5, %ymm6, %ymm5
-; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,3,2,3]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm5[3,4,5,6,7]
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm5 = ymm4[0,0,0,0,4,4,4,4]
-; AVX2-FAST-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm7 = mem[0,0,2,3,4,4,6,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm5 = ymm7[0,1,2],ymm5[3],ymm7[4,5,6],ymm5[7]
-; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm5 = ymm5[0,1,0,3]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm5[5,6,7]
-; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovaps (%rdi), %xmm0
-; AVX2-FAST-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[2,3,2,3]
 ; AVX2-FAST-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Folded Reload
 ; AVX2-FAST-NEXT:    # ymm5 = mem[0,0,2,3,4,4,6,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm5[1,2,3],ymm0[4],ymm5[5,6,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm4 = ymm5[0,1,2],ymm4[3],ymm5[4,5,6],ymm4[7]
+; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm4 = ymm4[0,1,0,3]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm4[5,6,7]
+; AVX2-FAST-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vmovaps 960(%rdi), %xmm2
+; AVX2-FAST-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[2,3,2,3]
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm4 = ymm5[0,0,2,3,4,4,6,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0],ymm4[1,2,3],ymm2[4],ymm4[5,6,7]
 ; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
-; AVX2-FAST-NEXT:    vblendps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm5 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm5 = mem[0,1],ymm4[2,3],mem[4,5],ymm4[6,7]
-; AVX2-FAST-NEXT:    vpermps %ymm5, %ymm6, %ymm5
-; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,3,2,3]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm5[3,4,5,6,7]
-; AVX2-FAST-NEXT:    vpermilps $0, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm5 = mem[0,0,0,0,4,4,4,4]
-; AVX2-FAST-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm7 = mem[0,0,2,3,4,4,6,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm5 = ymm7[0,1,2],ymm5[3],ymm7[4,5,6],ymm5[7]
-; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm5 = ymm5[0,1,0,3]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm5[5,6,7]
-; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovaps 384(%rdi), %xmm0
-; AVX2-FAST-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[2,3,2,3]
-; AVX2-FAST-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm7 = mem[0,0,2,3,4,4,6,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm7[1,2,3],ymm0[4],ymm7[5,6,7]
+; AVX2-FAST-NEXT:    vblendps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm4 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm4 = mem[0,1],ymm4[2,3],mem[4,5],ymm4[6,7]
+; AVX2-FAST-NEXT:    vpermps %ymm4, %ymm0, %ymm4
+; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[0,3,2,3]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2],ymm4[3,4,5,6,7]
+; AVX2-FAST-NEXT:    vpermilps $0, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm4 = mem[0,0,0,0,4,4,4,4]
+; AVX2-FAST-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm6 = mem[0,0,2,3,4,4,6,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm4 = ymm6[0,1,2],ymm4[3],ymm6[4,5,6],ymm4[7]
+; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm4 = ymm4[0,1,0,3]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm4[5,6,7]
+; AVX2-FAST-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vmovaps 1344(%rdi), %xmm2
+; AVX2-FAST-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[2,3,2,3]
 ; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
-; AVX2-FAST-NEXT:    vblendps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm7 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm7 = mem[0,1],ymm4[2,3],mem[4,5],ymm4[6,7]
-; AVX2-FAST-NEXT:    vpermps %ymm7, %ymm6, %ymm7
-; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,3,2,3]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm7[3,4,5,6,7]
-; AVX2-FAST-NEXT:    vpermilps $0, {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm7 = mem[0,0,0,0,4,4,4,4]
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm6 = ymm4[0,0,2,3,4,4,6,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0],ymm6[1,2,3],ymm2[4],ymm6[5,6,7]
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm6 = ymm15[0,1],ymm14[2,3],ymm15[4,5],ymm14[6,7]
+; AVX2-FAST-NEXT:    vpermps %ymm6, %ymm0, %ymm6
+; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[0,3,2,3]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2],ymm6[3,4,5,6,7]
+; AVX2-FAST-NEXT:    vpermilps $0, {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm6 = mem[0,0,0,0,4,4,4,4]
+; AVX2-FAST-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm8 = mem[0,0,2,3,4,4,6,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm6 = ymm8[0,1,2],ymm6[3],ymm8[4,5,6],ymm6[7]
+; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm6 = ymm6[0,1,0,3]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm6[5,6,7]
+; AVX2-FAST-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vmovaps (%rdi), %xmm2
+; AVX2-FAST-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[2,3,2,3]
+; AVX2-FAST-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm6 = mem[0,0,2,3,4,4,6,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0],ymm6[1,2,3],ymm2[4],ymm6[5,6,7]
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
+; AVX2-FAST-NEXT:    vblendps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm6, %ymm6 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm6 = mem[0,1],ymm6[2,3],mem[4,5],ymm6[6,7]
+; AVX2-FAST-NEXT:    vpermps %ymm6, %ymm0, %ymm6
+; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[0,3,2,3]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2],ymm6[3,4,5,6,7]
+; AVX2-FAST-NEXT:    vpermilps $0, {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm6 = mem[0,0,0,0,4,4,4,4]
 ; AVX2-FAST-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Folded Reload
 ; AVX2-FAST-NEXT:    # ymm9 = mem[0,0,2,3,4,4,6,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm7 = ymm9[0,1,2],ymm7[3],ymm9[4,5,6],ymm7[7]
-; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm7 = ymm7[0,1,0,3]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm7[5,6,7]
-; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovaps 768(%rdi), %xmm0
-; AVX2-FAST-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} xmm7 = xmm0[2,3,2,3]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm6 = ymm9[0,1,2],ymm6[3],ymm9[4,5,6],ymm6[7]
+; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm6 = ymm6[0,1,0,3]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm6[5,6,7]
+; AVX2-FAST-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vmovaps 384(%rdi), %xmm2
+; AVX2-FAST-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[2,3,2,3]
 ; AVX2-FAST-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Folded Reload
 ; AVX2-FAST-NEXT:    # ymm9 = mem[0,0,2,3,4,4,6,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm7 = ymm7[0],ymm9[1,2,3],ymm7[4],ymm9[5,6,7]
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX2-FAST-NEXT:    vblendps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm9 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm9 = mem[0,1],ymm0[2,3],mem[4,5],ymm0[6,7]
-; AVX2-FAST-NEXT:    vpermps %ymm9, %ymm6, %ymm9
-; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm7 = ymm7[0,3,2,3]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm7 = ymm7[0,1,2],ymm9[3,4,5,6,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0],ymm9[1,2,3],ymm2[4],ymm9[5,6,7]
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
+; AVX2-FAST-NEXT:    vblendps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm6, %ymm9 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm9 = mem[0,1],ymm6[2,3],mem[4,5],ymm6[6,7]
+; AVX2-FAST-NEXT:    vpermps %ymm9, %ymm0, %ymm9
+; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[0,3,2,3]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2],ymm9[3,4,5,6,7]
 ; AVX2-FAST-NEXT:    vpermilps $0, {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Folded Reload
 ; AVX2-FAST-NEXT:    # ymm9 = mem[0,0,0,0,4,4,4,4]
 ; AVX2-FAST-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Folded Reload
 ; AVX2-FAST-NEXT:    # ymm10 = mem[0,0,2,3,4,4,6,7]
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm9 = ymm10[0,1,2],ymm9[3],ymm10[4,5,6],ymm9[7]
 ; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm9 = ymm9[0,1,0,3]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm7[0,1,2,3,4],ymm9[5,6,7]
-; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm7 = ymm5[0,1],ymm4[2,3],ymm5[4,5],ymm4[6,7]
-; AVX2-FAST-NEXT:    vpermps %ymm7, %ymm6, %ymm7
-; AVX2-FAST-NEXT:    vmovaps 1152(%rdi), %xmm8
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} xmm9 = xmm8[2,3,2,3]
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm10 = ymm6[0,0,2,3,4,4,6,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm9[5,6,7]
+; AVX2-FAST-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vmovaps 768(%rdi), %xmm2
+; AVX2-FAST-NEXT:    vmovaps %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} xmm9 = xmm2[2,3,2,3]
+; AVX2-FAST-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm10 = mem[0,0,2,3,4,4,6,7]
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm9 = ymm9[0],ymm10[1,2,3],ymm9[4],ymm10[5,6,7]
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX2-FAST-NEXT:    vblendps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm10 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm10 = mem[0,1],ymm2[2,3],mem[4,5],ymm2[6,7]
+; AVX2-FAST-NEXT:    vpermps %ymm10, %ymm0, %ymm10
 ; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm9 = ymm9[0,3,2,3]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm7 = ymm9[0,1,2],ymm7[3,4,5,6,7]
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm9 = ymm11[0,0,0,0,4,4,4,4]
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm10 = ymm0[0,0,2,3,4,4,6,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm9 = ymm10[0,1,2],ymm9[3],ymm10[4,5,6],ymm9[7]
-; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm9 = ymm9[0,1,0,3]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm7 = ymm7[0,1,2,3,4],ymm9[5,6,7]
-; AVX2-FAST-NEXT:    vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[3,3,3,3]
-; AVX2-FAST-NEXT:    vpermilps $244, {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm7 = mem[0,1,3,3,4,5,7,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0],ymm7[1,2,3],ymm2[4],ymm7[5,6,7]
-; AVX2-FAST-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm7 = mem[3,3,3,3,7,7,7,7]
-; AVX2-FAST-NEXT:    vblendps $34, {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm7 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm7 = ymm7[0],mem[1],ymm7[2,3,4],mem[5],ymm7[6,7]
-; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm7 = ymm7[0,2,0,3]
-; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[0,3,2,3]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2],ymm7[3,4,5,6,7]
-; AVX2-FAST-NEXT:    vpermilps $68, {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm7 = mem[0,1,0,1,4,5,4,5]
-; AVX2-FAST-NEXT:    vpermilps $244, {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm9 = mem[0,1,3,3,4,5,7,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm7 = ymm9[0,1,2],ymm7[3],ymm9[4,5,6],ymm7[7]
-; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm7 = ymm7[0,1,0,3]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm7[5,6,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm9 = ymm9[0,1,2],ymm10[3,4,5,6,7]
+; AVX2-FAST-NEXT:    vpermilps $0, {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm10 = mem[0,0,0,0,4,4,4,4]
+; AVX2-FAST-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm11 = mem[0,0,2,3,4,4,6,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm10 = ymm11[0,1,2],ymm10[3],ymm11[4,5,6],ymm10[7]
+; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm10 = ymm10[0,1,0,3]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm9[0,1,2,3,4],ymm10[5,6,7]
 ; AVX2-FAST-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm9 = ymm12[0,1],ymm8[2,3],ymm12[4,5],ymm8[6,7]
+; AVX2-FAST-NEXT:    vpermps %ymm9, %ymm0, %ymm9
+; AVX2-FAST-NEXT:    vmovaps 1152(%rdi), %xmm0
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} xmm10 = xmm0[2,3,2,3]
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm11 = ymm2[0,0,2,3,4,4,6,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm10 = ymm10[0],ymm11[1,2,3],ymm10[4],ymm11[5,6,7]
+; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm10 = ymm10[0,3,2,3]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm9 = ymm10[0,1,2],ymm9[3,4,5,6,7]
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm10 = ymm6[0,0,0,0,4,4,4,4]
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm11 = ymm7[0,0,2,3,4,4,6,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm10 = ymm11[0,1,2],ymm10[3],ymm11[4,5,6],ymm10[7]
+; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm10 = ymm10[0,1,0,3]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm9 = ymm9[0,1,2,3,4],ymm10[5,6,7]
+; AVX2-FAST-NEXT:    vmovups %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
-; AVX2-FAST-NEXT:    vpermilps $244, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm2 = mem[0,1,3,3,4,5,7,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm2[1,2,3],ymm1[4],ymm2[5,6,7]
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm2 = ymm12[3,3,3,3,7,7,7,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0],ymm13[1],ymm2[2,3,4],ymm13[5],ymm2[6,7]
-; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[0,2,0,3]
+; AVX2-FAST-NEXT:    vpermilps $244, {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm9 = mem[0,1,3,3,4,5,7,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm9[1,2,3],ymm1[4],ymm9[5,6,7]
+; AVX2-FAST-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm9 = mem[3,3,3,3,7,7,7,7]
+; AVX2-FAST-NEXT:    vblendps $34, (%rsp), %ymm9, %ymm9 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm9 = ymm9[0],mem[1],ymm9[2,3,4],mem[5],ymm9[6,7]
+; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm9 = ymm9[0,2,0,3]
 ; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,3,2,3]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2],ymm2[3,4,5,6,7]
-; AVX2-FAST-NEXT:    vpermilps $68, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm2 = mem[0,1,0,1,4,5,4,5]
-; AVX2-FAST-NEXT:    vpermilps $244, {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm7 = mem[0,1,3,3,4,5,7,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm7[0,1,2],ymm2[3],ymm7[4,5,6],ymm2[7]
-; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[0,1,0,3]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm2[5,6,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2],ymm9[3,4,5,6,7]
+; AVX2-FAST-NEXT:    vpermilps $68, {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm9 = mem[0,1,0,1,4,5,4,5]
+; AVX2-FAST-NEXT:    vpermilps $244, {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm10 = mem[0,1,3,3,4,5,7,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm9 = ymm10[0,1,2],ymm9[3],ymm10[4,5,6],ymm9[7]
+; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm9 = ymm9[0,1,0,3]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm9[5,6,7]
+; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} xmm1 = xmm3[3,3,3,3]
+; AVX2-FAST-NEXT:    vpermilps $244, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm3 = mem[0,1,3,3,4,5,7,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm3[1,2,3],ymm1[4],ymm3[5,6,7]
+; AVX2-FAST-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm3 = mem[3,3,3,3,7,7,7,7]
+; AVX2-FAST-NEXT:    vblendps $34, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm3 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm3 = ymm3[0],mem[1],ymm3[2,3,4],mem[5],ymm3[6,7]
+; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm3 = ymm3[0,2,0,3]
+; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,3,2,3]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2],ymm3[3,4,5,6,7]
+; AVX2-FAST-NEXT:    vpermilps $68, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm3 = mem[0,1,0,1,4,5,4,5]
+; AVX2-FAST-NEXT:    vpermilps $244, {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm9 = mem[0,1,3,3,4,5,7,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm3 = ymm9[0,1,2],ymm3[3],ymm9[4,5,6],ymm3[7]
+; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm3 = ymm3[0,1,0,3]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm3[5,6,7]
 ; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
 ; AVX2-FAST-NEXT:    # xmm1 = mem[3,3,3,3]
-; AVX2-FAST-NEXT:    vpermilps $244, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm2 = mem[0,1,3,3,4,5,7,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm2[1,2,3],ymm1[4],ymm2[5,6,7]
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm2 = ymm3[3,3,3,3,7,7,7,7]
-; AVX2-FAST-NEXT:    vblendps $34, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm2 = ymm2[0],mem[1],ymm2[2,3,4],mem[5],ymm2[6,7]
-; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[0,2,0,3]
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm3 = ymm5[0,1,3,3,4,5,7,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm3[1,2,3],ymm1[4],ymm3[5,6,7]
+; AVX2-FAST-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm3 = mem[3,3,3,3,7,7,7,7]
+; AVX2-FAST-NEXT:    vblendps $34, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm3 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm3 = ymm3[0],mem[1],ymm3[2,3,4],mem[5],ymm3[6,7]
+; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm3 = ymm3[0,2,0,3]
 ; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,3,2,3]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2],ymm2[3,4,5,6,7]
-; AVX2-FAST-NEXT:    vpermilps $68, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm2 = mem[0,1,0,1,4,5,4,5]
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm3 = ymm15[0,1,3,3,4,5,7,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1,2],ymm2[3],ymm3[4,5,6],ymm2[7]
-; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[0,1,0,3]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm2[5,6,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2],ymm3[3,4,5,6,7]
+; AVX2-FAST-NEXT:    vpermilps $68, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm3 = mem[0,1,0,1,4,5,4,5]
+; AVX2-FAST-NEXT:    vpermilps $244, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm5 = mem[0,1,3,3,4,5,7,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm3 = ymm5[0,1,2],ymm3[3],ymm5[4,5,6],ymm3[7]
+; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm3 = ymm3[0,1,0,3]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm3[5,6,7]
 ; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
 ; AVX2-FAST-NEXT:    # xmm1 = mem[3,3,3,3]
-; AVX2-FAST-NEXT:    vpermilps $244, (%rsp), %ymm2 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm2 = mem[0,1,3,3,4,5,7,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm2[1,2,3],ymm1[4],ymm2[5,6,7]
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm2 = ymm14[3,3,3,3,7,7,7,7]
-; AVX2-FAST-NEXT:    vblendps $34, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm2 = ymm2[0],mem[1],ymm2[2,3,4],mem[5],ymm2[6,7]
-; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[0,2,0,3]
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm3 = ymm4[0,1,3,3,4,5,7,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm3[1,2,3],ymm1[4],ymm3[5,6,7]
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm3 = ymm14[3,3,3,3,7,7,7,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0],ymm15[1],ymm3[2,3,4],ymm15[5],ymm3[6,7]
+; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm3 = ymm3[0,2,0,3]
 ; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,3,2,3]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2],ymm2[3,4,5,6,7]
-; AVX2-FAST-NEXT:    vpermilps $68, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm2 = mem[0,1,0,1,4,5,4,5]
-; AVX2-FAST-NEXT:    vpermilps $244, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm3 = mem[0,1,3,3,4,5,7,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1,2],ymm2[3],ymm3[4,5,6],ymm2[7]
-; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[0,1,0,3]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm2[5,6,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2],ymm3[3,4,5,6,7]
+; AVX2-FAST-NEXT:    vpermilps $68, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm3 = mem[0,1,0,1,4,5,4,5]
+; AVX2-FAST-NEXT:    vpermilps $244, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm4 = mem[0,1,3,3,4,5,7,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm3 = ymm4[0,1,2],ymm3[3],ymm4[4,5,6],ymm3[7]
+; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm3 = ymm3[0,1,0,3]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm3[5,6,7]
 ; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} xmm1 = xmm8[3,3,3,3]
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm2 = ymm6[0,1,3,3,4,5,7,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm2[1,2,3],ymm1[4],ymm2[5,6,7]
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm2 = ymm4[3,3,3,3,7,7,7,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0],ymm5[1],ymm2[2,3,4],ymm5[5],ymm2[6,7]
-; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[0,2,0,3]
-; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,3,2,3]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2],ymm2[3,4,5,6,7]
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm2 = ymm11[0,1,0,1,4,5,4,5]
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm3 = ymm0[0,1,3,3,4,5,7,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1,2],ymm2[3],ymm3[4,5,6],ymm2[7]
-; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[0,1,0,3]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3,4],ymm2[5,6,7]
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm2[0,1,3,3,4,5,7,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7]
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm8[3,3,3,3,7,7,7,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm12[1],ymm1[2,3,4],ymm12[5],ymm1[6,7]
+; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,2,0,3]
+; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,3,2,3]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm6[0,1,0,1,4,5,4,5]
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm3 = ymm7[0,1,3,3,4,5,7,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm3[0,1,2],ymm1[3],ymm3[4,5,6],ymm1[7]
+; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,1,0,3]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]
 ; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
 ; AVX2-FAST-NEXT:    # xmm0 = mem[3,3,3,3]
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm10[0,1,3,3,4,5,7,7]
+; AVX2-FAST-NEXT:    vpermilps $244, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm1 = mem[0,1,3,3,4,5,7,7]
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7]
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm13[3,3,3,3,7,7,7,7]
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm11[1],ymm1[2,3,4],ymm11[5],ymm1[6,7]
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm15[3,3,3,3,7,7,7,7]
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm14[1],ymm1[2,3,4],ymm14[5],ymm1[6,7]
 ; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,2,0,3]
 ; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,3,2,3]
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm14[0,1,0,1,4,5,4,5]
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm2 = ymm12[0,1,3,3,4,5,7,7]
+; AVX2-FAST-NEXT:    vpermilps $68, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm1 = mem[0,1,0,1,4,5,4,5]
+; AVX2-FAST-NEXT:    vpermilps $244, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm2 = mem[0,1,3,3,4,5,7,7]
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2],ymm1[3],ymm2[4,5,6],ymm1[7]
 ; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,1,0,3]
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]
 ; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
 ; AVX2-FAST-NEXT:    # xmm0 = mem[3,3,3,3]
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm4[0,1,3,3,4,5,7,7]
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm12[0,1,3,3,4,5,7,7]
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7]
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm7[3,3,3,3,7,7,7,7]
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm6[1],ymm1[2,3,4],ymm6[5],ymm1[6,7]
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm9[3,3,3,3,7,7,7,7]
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm8[1],ymm1[2,3,4],ymm8[5],ymm1[6,7]
 ; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,2,0,3]
 ; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,3,2,3]
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm9[0,1,0,1,4,5,4,5]
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm2 = ymm8[0,1,3,3,4,5,7,7]
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm11[0,1,0,1,4,5,4,5]
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm2 = ymm10[0,1,3,3,4,5,7,7]
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2],ymm1[3],ymm2[4,5,6],ymm1[7]
 ; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,1,0,3]
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]
 ; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
 ; AVX2-FAST-NEXT:    # xmm0 = mem[3,3,3,3]
-; AVX2-FAST-NEXT:    vpermilps $244, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm1 = mem[0,1,3,3,4,5,7,7]
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm7[0,1,3,3,4,5,7,7]
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7]
-; AVX2-FAST-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm1 = mem[3,3,3,3,7,7,7,7]
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm15[1],ymm1[2,3,4],ymm15[5],ymm1[6,7]
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm4[3,3,3,3,7,7,7,7]
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm3[1],ymm1[2,3,4],ymm3[5],ymm1[6,7]
 ; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,2,0,3]
 ; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,3,2,3]
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
-; AVX2-FAST-NEXT:    vpermilps $68, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm1 = mem[0,1,0,1,4,5,4,5]
-; AVX2-FAST-NEXT:    vpermilps $244, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm2 = mem[0,1,3,3,4,5,7,7]
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm6[0,1,0,1,4,5,4,5]
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm2 = ymm5[0,1,3,3,4,5,7,7]
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2],ymm1[3],ymm2[4,5,6],ymm1[7]
 ; AVX2-FAST-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,1,0,3]
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]
 ; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm4[4,5,6,7]
+; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm2 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm2 = ymm7[0,1,2,3],mem[4,5,6,7]
+; AVX2-FAST-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vmovaps 80(%rdi), %xmm0
+; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3],ymm1[4,5,6,7]
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,1,0,2,4,5,4,6]
+; AVX2-FAST-NEXT:    vpermps %ymm2, %ymm13, %ymm1
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3,4,5,6,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm6[0,1],ymm5[2,3],ymm6[4,5,6,7]
+; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vbroadcastf128 {{.*#+}} ymm2 = [0,2,0,6,0,2,0,6]
+; AVX2-FAST-NEXT:    # ymm2 = mem[0,1,0,1]
+; AVX2-FAST-NEXT:    vpermps %ymm1, %ymm2, %ymm1
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]
+; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
+; AVX2-FAST-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm1 = ymm0[0,1,2,3],mem[4,5,6,7]
+; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX2-FAST-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm3 # 32-byte Folded Reload
 ; AVX2-FAST-NEXT:    # ymm3 = ymm0[0,1,2,3],mem[4,5,6,7]
 ; AVX2-FAST-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX2-FAST-NEXT:    vblendps $15, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm1 = mem[0,1,2,3],ymm0[4,5,6,7]
-; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-NEXT:    vmovaps 272(%rdi), %xmm0
-; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vmovups %ymm0, (%rsp) # 32-byte Spill
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3],ymm1[4,5,6,7]
 ; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,1,0,2,4,5,4,6]
-; AVX2-FAST-NEXT:    vmovddup {{.*#+}} xmm2 = [4,2,4,2]
-; AVX2-FAST-NEXT:    # xmm2 = mem[0,0]
-; AVX2-FAST-NEXT:    vpermps %ymm3, %ymm2, %ymm1
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1],ymm0[2,3,4,5,6,7]
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX2-FAST-NEXT:    vblendps $12, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm0 = ymm0[0,1],mem[2,3],ymm0[4,5,6,7]
+; AVX2-FAST-NEXT:    vpermps %ymm3, %ymm13, %ymm1
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3,4,5,6,7]
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-FAST-NEXT:    vblendps $243, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm1 = mem[0,1],ymm1[2,3],mem[4,5,6,7]
+; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vpermps %ymm1, %ymm2, %ymm1
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]
 ; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vbroadcastf128 {{.*#+}} ymm5 = [0,2,0,6,0,2,0,6]
-; AVX2-FAST-NEXT:    # ymm5 = mem[0,1,0,1]
-; AVX2-FAST-NEXT:    vpermps %ymm0, %ymm5, %ymm3
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm3[5,6,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm8[0,1,2,3],ymm9[4,5,6,7]
 ; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm3 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm3 = ymm4[0,1,2,3],mem[4,5,6,7]
+; AVX2-FAST-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm12, %ymm3 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm3 = ymm12[0,1,2,3],mem[4,5,6,7]
 ; AVX2-FAST-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm6[0,1,2,3],ymm7[4,5,6,7]
-; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-NEXT:    vmovaps 464(%rdi), %xmm0
 ; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1],ymm0[2,3],ymm1[4,5,6,7]
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[0,1,0,2,4,5,4,6]
-; AVX2-FAST-NEXT:    vpermps %ymm3, %ymm2, %ymm3
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm3[0,1],ymm1[2,3,4,5,6,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm9[0,1],ymm8[2,3],ymm9[4,5,6,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3],ymm1[4,5,6,7]
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,1,0,2,4,5,4,6]
+; AVX2-FAST-NEXT:    vpermps %ymm3, %ymm13, %ymm1
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3,4,5,6,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm11[0,1],ymm10[2,3],ymm11[4,5,6,7]
+; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vpermps %ymm1, %ymm2, %ymm1
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]
 ; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vpermps %ymm0, %ymm5, %ymm3
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm3[5,6,7]
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX2-FAST-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm1 = ymm0[0,1,2,3],mem[4,5,6,7]
 ; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX2-FAST-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm0 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm0 = ymm1[0,1,2,3],mem[4,5,6,7]
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX2-FAST-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm3 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm3 = ymm0[0,1,2,3],mem[4,5,6,7]
+; AVX2-FAST-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vmovaps 656(%rdi), %xmm0
 ; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3],ymm1[4,5,6,7]
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,1,0,2,4,5,4,6]
+; AVX2-FAST-NEXT:    vpermps %ymm3, %ymm13, %ymm1
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3,4,5,6,7]
 ; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX2-FAST-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm3 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm3 = ymm1[0,1,2,3],mem[4,5,6,7]
-; AVX2-FAST-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovaps 656(%rdi), %xmm1
+; AVX2-FAST-NEXT:    vblendps $12, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm1 = ymm1[0,1],mem[2,3],ymm1[4,5,6,7]
 ; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm3[0,1],ymm1[2,3],ymm3[4,5,6,7]
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[0,1,0,2,4,5,4,6]
-; AVX2-FAST-NEXT:    vpermps %ymm0, %ymm2, %ymm3
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm3[0,1],ymm1[2,3,4,5,6,7]
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
-; AVX2-FAST-NEXT:    vblendps $243, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm0 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm0 = mem[0,1],ymm3[2,3],mem[4,5,6,7]
+; AVX2-FAST-NEXT:    vpermps %ymm1, %ymm2, %ymm1
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]
 ; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vpermps %ymm0, %ymm5, %ymm3
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm3[5,6,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm14[0,1,2,3],ymm15[4,5,6,7]
 ; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm10, %ymm0 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm0 = ymm10[0,1,2,3],mem[4,5,6,7]
-; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm3 = ymm11[0,1,2,3],ymm13[4,5,6,7]
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX2-FAST-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm3 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm3 = ymm0[0,1,2,3],mem[4,5,6,7]
 ; AVX2-FAST-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovaps 848(%rdi), %xmm1
-; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm3[0,1],ymm1[2,3],ymm3[4,5,6,7]
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[0,1,0,2,4,5,4,6]
-; AVX2-FAST-NEXT:    vpermps %ymm0, %ymm2, %ymm3
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm3[0,1],ymm1[2,3,4,5,6,7]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm14[0,1],ymm12[2,3],ymm14[4,5,6,7]
-; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vpermps %ymm0, %ymm5, %ymm3
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm3[5,6,7]
-; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX2-FAST-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm0 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm0 = ymm1[0,1,2,3],mem[4,5,6,7]
+; AVX2-FAST-NEXT:    vmovaps 848(%rdi), %xmm0
 ; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3],ymm1[4,5,6,7]
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,1,0,2,4,5,4,6]
+; AVX2-FAST-NEXT:    vpermps %ymm3, %ymm13, %ymm1
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3,4,5,6,7]
 ; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX2-FAST-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm3 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm3 = ymm1[0,1,2,3],mem[4,5,6,7]
-; AVX2-FAST-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovaps 1040(%rdi), %xmm1
-; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm3[0,1],ymm1[2,3],ymm3[4,5,6,7]
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[0,1,0,2,4,5,4,6]
-; AVX2-FAST-NEXT:    vpermps %ymm0, %ymm2, %ymm3
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm3[0,1],ymm1[2,3,4,5,6,7]
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
-; AVX2-FAST-NEXT:    vblendps $243, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm13 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm13 = mem[0,1],ymm3[2,3],mem[4,5,6,7]
-; AVX2-FAST-NEXT:    vpermps %ymm13, %ymm5, %ymm3
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm3[5,6,7]
+; AVX2-FAST-NEXT:    vblendps $243, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm1 = mem[0,1],ymm1[2,3],mem[4,5,6,7]
 ; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX2-FAST-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm0 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm0 = ymm1[0,1,2,3],mem[4,5,6,7]
+; AVX2-FAST-NEXT:    vpermps %ymm1, %ymm2, %ymm1
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]
 ; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX2-FAST-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm14 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm14 = ymm0[0,1,2,3],mem[4,5,6,7]
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX2-FAST-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm12 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm12 = ymm0[0,1,2,3],mem[4,5,6,7]
+; AVX2-FAST-NEXT:    vmovaps 1040(%rdi), %xmm11
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm14[0,1],ymm11[2,3],ymm14[4,5,6,7]
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,1,0,2,4,5,4,6]
+; AVX2-FAST-NEXT:    vpermps %ymm12, %ymm13, %ymm1
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3,4,5,6,7]
 ; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX2-FAST-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm12 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm12 = ymm1[0,1,2,3],mem[4,5,6,7]
-; AVX2-FAST-NEXT:    vmovaps 1232(%rdi), %xmm11
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm12[0,1],ymm11[2,3],ymm12[4,5,6,7]
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[0,1,0,2,4,5,4,6]
-; AVX2-FAST-NEXT:    vpermps %ymm0, %ymm2, %ymm3
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm3[0,1],ymm1[2,3,4,5,6,7]
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
-; AVX2-FAST-NEXT:    vblendps $243, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm9 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm9 = mem[0,1],ymm3[2,3],mem[4,5,6,7]
-; AVX2-FAST-NEXT:    vpermps %ymm9, %ymm5, %ymm3
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm3[5,6,7]
-; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vmovups (%rsp), %ymm1 # 32-byte Reload
-; AVX2-FAST-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm0 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm0 = ymm1[0,1,2,3],mem[4,5,6,7]
+; AVX2-FAST-NEXT:    vblendps $243, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm10 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm10 = mem[0,1],ymm1[2,3],mem[4,5,6,7]
+; AVX2-FAST-NEXT:    vpermps %ymm10, %ymm2, %ymm1
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]
 ; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX2-FAST-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm9 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm9 = ymm0[0,1,2,3],mem[4,5,6,7]
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX2-FAST-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm8 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm8 = ymm0[0,1,2,3],mem[4,5,6,7]
+; AVX2-FAST-NEXT:    vmovaps 1232(%rdi), %xmm6
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm9[0,1],ymm6[2,3],ymm9[4,5,6,7]
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,1,0,2,4,5,4,6]
+; AVX2-FAST-NEXT:    vpermps %ymm8, %ymm13, %ymm1
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3,4,5,6,7]
 ; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX2-FAST-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm7 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm7 = ymm1[0,1,2,3],mem[4,5,6,7]
-; AVX2-FAST-NEXT:    vmovaps 1424(%rdi), %xmm8
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm7[0,1],ymm8[2,3],ymm7[4,5,6,7]
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[0,1,0,2,4,5,4,6]
-; AVX2-FAST-NEXT:    vpermps %ymm0, %ymm2, %ymm3
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm3[0,1],ymm1[2,3,4,5,6,7]
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
-; AVX2-FAST-NEXT:    vblendps $243, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm6 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm6 = mem[0,1],ymm3[2,3],mem[4,5,6,7]
-; AVX2-FAST-NEXT:    vpermps %ymm6, %ymm5, %ymm3
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm3[5,6,7]
-; AVX2-FAST-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm15, %ymm10 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm10 = ymm15[0,1,2,3],mem[4,5,6,7]
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX2-FAST-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm4 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm4 = ymm1[0,1,2,3],mem[4,5,6,7]
-; AVX2-FAST-NEXT:    vmovaps 80(%rdi), %xmm3
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm10[0,1],ymm3[2,3],ymm10[4,5,6,7]
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[0,1,0,2,4,5,4,6]
-; AVX2-FAST-NEXT:    vpermps %ymm4, %ymm2, %ymm2
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1],ymm1[2,3,4,5,6,7]
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-FAST-NEXT:    vblendps $243, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm2 = mem[0,1],ymm2[2,3],mem[4,5,6,7]
-; AVX2-FAST-NEXT:    vpermps %ymm2, %ymm5, %ymm0
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3,4],ymm0[5,6,7]
+; AVX2-FAST-NEXT:    vblendps $243, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm5 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm5 = mem[0,1],ymm1[2,3],mem[4,5,6,7]
+; AVX2-FAST-NEXT:    vpermps %ymm5, %ymm2, %ymm1
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]
+; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX2-FAST-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm4 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm4 = ymm0[0,1,2,3],mem[4,5,6,7]
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX2-FAST-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm3 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm3 = ymm0[0,1,2,3],mem[4,5,6,7]
+; AVX2-FAST-NEXT:    vmovaps 1424(%rdi), %xmm1
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm4[0,1],ymm1[2,3],ymm4[4,5,6,7]
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,1,0,2,4,5,4,6]
+; AVX2-FAST-NEXT:    vpermps %ymm3, %ymm13, %ymm13
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm13[0,1],ymm0[2,3,4,5,6,7]
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
+; AVX2-FAST-NEXT:    vblendps $243, {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm13 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm13 = mem[0,1],ymm7[2,3],mem[4,5,6,7]
+; AVX2-FAST-NEXT:    vpermps %ymm13, %ymm2, %ymm2
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm2[5,6,7]
 ; AVX2-FAST-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-NEXT:    vpermilps $85, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
 ; AVX2-FAST-NEXT:    # ymm0 = mem[1,1,1,1,5,5,5,5]
 ; AVX2-FAST-NEXT:    vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
 ; AVX2-FAST-NEXT:    # ymm0 = ymm0[0,1,2],mem[3],ymm0[4,5,6,7]
-; AVX2-FAST-NEXT:    vmovddup {{.*#+}} xmm1 = [5,3,5,3]
-; AVX2-FAST-NEXT:    # xmm1 = mem[0,0]
-; AVX2-FAST-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm14 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm14 = ymm14[0,1],ymm0[2,3,4,5,6,7]
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
+; AVX2-FAST-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm2 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1],ymm0[2,3,4,5,6,7]
 ; AVX2-FAST-NEXT:    vbroadcastf128 {{.*#+}} ymm0 = [0,3,1,7,0,3,1,7]
 ; AVX2-FAST-NEXT:    # ymm0 = mem[0,1,0,1]
 ; AVX2-FAST-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm15 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm5 = ymm14[0,1,2,3,4],ymm15[5,6,7]
-; AVX2-FAST-NEXT:    vmovups %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vpermilps $85, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm14 = mem[1,1,1,1,5,5,5,5]
-; AVX2-FAST-NEXT:    vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm14 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm14 = ymm14[0,1,2],mem[3],ymm14[4,5,6,7]
-; AVX2-FAST-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm15 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm14 = ymm15[0,1],ymm14[2,3,4,5,6,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm15[5,6,7]
+; AVX2-FAST-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vpermilps $85, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm2 = mem[1,1,1,1,5,5,5,5]
+; AVX2-FAST-NEXT:    vblendps $8, (%rsp), %ymm2, %ymm2 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm2 = ymm2[0,1,2],mem[3],ymm2[4,5,6,7]
+; AVX2-FAST-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm15 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm15[0,1],ymm2[2,3,4,5,6,7]
 ; AVX2-FAST-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm15 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm5 = ymm14[0,1,2,3,4],ymm15[5,6,7]
-; AVX2-FAST-NEXT:    vmovups %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vpermilps $85, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm14 = mem[1,1,1,1,5,5,5,5]
-; AVX2-FAST-NEXT:    vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm14 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm14 = ymm14[0,1,2],mem[3],ymm14[4,5,6,7]
-; AVX2-FAST-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm15 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm14 = ymm15[0,1],ymm14[2,3,4,5,6,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm15[5,6,7]
+; AVX2-FAST-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vpermilps $85, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm2 = mem[1,1,1,1,5,5,5,5]
+; AVX2-FAST-NEXT:    vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm2 = ymm2[0,1,2],mem[3],ymm2[4,5,6,7]
+; AVX2-FAST-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm15 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm15[0,1],ymm2[2,3,4,5,6,7]
 ; AVX2-FAST-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm15 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm5 = ymm14[0,1,2,3,4],ymm15[5,6,7]
-; AVX2-FAST-NEXT:    vmovups %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-NEXT:    vpermilps $85, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm14 = mem[1,1,1,1,5,5,5,5]
-; AVX2-FAST-NEXT:    vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm14 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm14 = ymm14[0,1,2],mem[3],ymm14[4,5,6,7]
-; AVX2-FAST-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm15 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm14 = ymm15[0,1],ymm14[2,3,4,5,6,7]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm15[5,6,7]
+; AVX2-FAST-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vpermilps $85, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm2 = mem[1,1,1,1,5,5,5,5]
+; AVX2-FAST-NEXT:    vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm2 = ymm2[0,1,2],mem[3],ymm2[4,5,6,7]
+; AVX2-FAST-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm15 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm15[0,1],ymm2[2,3,4,5,6,7]
 ; AVX2-FAST-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm15 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm5 = ymm14[0,1,2,3,4],ymm15[5,6,7]
-; AVX2-FAST-NEXT:    vmovups %ymm5, (%rsp) # 32-byte Spill
-; AVX2-FAST-NEXT:    vpermilps $85, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm14 = mem[1,1,1,1,5,5,5,5]
-; AVX2-FAST-NEXT:    vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm14 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    # ymm14 = ymm14[0,1,2],mem[3],ymm14[4,5,6,7]
-; AVX2-FAST-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm15 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm14 = ymm15[0,1],ymm14[2,3,4,5,6,7]
-; AVX2-FAST-NEXT:    vpermps %ymm13, %ymm0, %ymm13
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm13 = ymm14[0,1,2,3,4],ymm13[5,6,7]
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm12 = ymm12[1,1,1,1,5,5,5,5]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm11 = ymm12[0,1,2],ymm11[3],ymm12[4,5,6,7]
-; AVX2-FAST-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm12 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm11 = ymm12[0,1],ymm11[2,3,4,5,6,7]
-; AVX2-FAST-NEXT:    vpermps %ymm9, %ymm0, %ymm9
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm9 = ymm11[0,1,2,3,4],ymm9[5,6,7]
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm7 = ymm7[1,1,1,1,5,5,5,5]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm7 = ymm7[0,1,2],ymm8[3],ymm7[4,5,6,7]
-; AVX2-FAST-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm8 # 32-byte Folded Reload
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm7 = ymm8[0,1],ymm7[2,3,4,5,6,7]
-; AVX2-FAST-NEXT:    vpermps %ymm6, %ymm0, %ymm6
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm6 = ymm7[0,1,2,3,4],ymm6[5,6,7]
-; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm5 = ymm10[1,1,1,1,5,5,5,5]
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm3 = ymm5[0,1,2],ymm3[3],ymm5[4,5,6,7]
-; AVX2-FAST-NEXT:    vpermps %ymm4, %ymm1, %ymm1
-; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1],ymm3[2,3,4,5,6,7]
-; AVX2-FAST-NEXT:    vpermps %ymm2, %ymm0, %ymm0
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm15[5,6,7]
+; AVX2-FAST-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-NEXT:    vpermilps $85, {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm15 = mem[1,1,1,1,5,5,5,5]
+; AVX2-FAST-NEXT:    vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm15, %ymm15 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    # ymm15 = ymm15[0,1,2],mem[3],ymm15[4,5,6,7]
+; AVX2-FAST-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm2 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1],ymm15[2,3,4,5,6,7]
+; AVX2-FAST-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm15 # 32-byte Folded Reload
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm15 = ymm2[0,1,2,3,4],ymm15[5,6,7]
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm2 = ymm14[1,1,1,1,5,5,5,5]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2],ymm11[3],ymm2[4,5,6,7]
+; AVX2-FAST-NEXT:    vpermps %ymm12, %ymm7, %ymm11
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm11[0,1],ymm2[2,3,4,5,6,7]
+; AVX2-FAST-NEXT:    vpermps %ymm10, %ymm0, %ymm10
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm10 = ymm2[0,1,2,3,4],ymm10[5,6,7]
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm2 = ymm9[1,1,1,1,5,5,5,5]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2],ymm6[3],ymm2[4,5,6,7]
+; AVX2-FAST-NEXT:    vpermps %ymm8, %ymm7, %ymm6
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm6[0,1],ymm2[2,3,4,5,6,7]
+; AVX2-FAST-NEXT:    vpermps %ymm5, %ymm0, %ymm5
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm5 = ymm2[0,1,2,3,4],ymm5[5,6,7]
+; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm2 = ymm4[1,1,1,1,5,5,5,5]
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2],ymm1[3],ymm2[4,5,6,7]
+; AVX2-FAST-NEXT:    vpermps %ymm3, %ymm7, %ymm2
+; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1],ymm1[2,3,4,5,6,7]
+; AVX2-FAST-NEXT:    vpermps %ymm13, %ymm0, %ymm0
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3,4],ymm0[5,6,7]
 ; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
 ; AVX2-FAST-NEXT:    vmovaps %ymm1, 192(%rsi)
@@ -8661,17 +8659,17 @@ define void @load_i32_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
 ; AVX2-FAST-NEXT:    vmovaps %ymm1, (%r9)
 ; AVX2-FAST-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX2-FAST-NEXT:    vmovaps %ymm6, 224(%rax)
-; AVX2-FAST-NEXT:    vmovaps %ymm9, 192(%rax)
-; AVX2-FAST-NEXT:    vmovaps %ymm13, 160(%rax)
-; AVX2-FAST-NEXT:    vmovups (%rsp), %ymm1 # 32-byte Reload
-; AVX2-FAST-NEXT:    vmovaps %ymm1, 128(%rax)
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX2-FAST-NEXT:    vmovaps %ymm1, 96(%rax)
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX2-FAST-NEXT:    vmovaps %ymm1, 64(%rax)
-; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX2-FAST-NEXT:    vmovaps %ymm1, 32(%rax)
+; AVX2-FAST-NEXT:    vmovaps %ymm0, 224(%rax)
+; AVX2-FAST-NEXT:    vmovaps %ymm5, 192(%rax)
+; AVX2-FAST-NEXT:    vmovaps %ymm10, 160(%rax)
+; AVX2-FAST-NEXT:    vmovaps %ymm15, 128(%rax)
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX2-FAST-NEXT:    vmovaps %ymm0, 96(%rax)
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX2-FAST-NEXT:    vmovaps %ymm0, 64(%rax)
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX2-FAST-NEXT:    vmovaps %ymm0, 32(%rax)
+; AVX2-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX2-FAST-NEXT:    vmovaps %ymm0, (%rax)
 ; AVX2-FAST-NEXT:    addq $2504, %rsp # imm = 0x9C8
 ; AVX2-FAST-NEXT:    vzeroupper
@@ -8679,45 +8677,47 @@ define void @load_i32_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ;
 ; AVX2-FAST-PERLANE-LABEL: load_i32_stride6_vf64:
 ; AVX2-FAST-PERLANE:       # %bb.0:
-; AVX2-FAST-PERLANE-NEXT:    subq $2504, %rsp # imm = 0x9C8
-; AVX2-FAST-PERLANE-NEXT:    vmovaps 672(%rdi), %ymm2
-; AVX2-FAST-PERLANE-NEXT:    vmovaps 640(%rdi), %ymm3
+; AVX2-FAST-PERLANE-NEXT:    subq $2536, %rsp # imm = 0x9E8
+; AVX2-FAST-PERLANE-NEXT:    vmovaps 672(%rdi), %ymm4
+; AVX2-FAST-PERLANE-NEXT:    vmovaps 640(%rdi), %ymm5
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vmovaps 608(%rdi), %ymm3
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vmovaps 608(%rdi), %ymm4
-; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vmovaps 320(%rdi), %ymm5
-; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm5, (%rsp) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vmovaps 320(%rdi), %ymm2
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-PERLANE-NEXT:    vmovaps 352(%rdi), %ymm6
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-PERLANE-NEXT:    vmovaps 288(%rdi), %ymm7
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-PERLANE-NEXT:    vmovaps 256(%rdi), %ymm9
-; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm9, (%rsp) # 32-byte Spill
 ; AVX2-FAST-PERLANE-NEXT:    vmovaps 224(%rdi), %ymm0
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-PERLANE-NEXT:    vmovaps 192(%rdi), %ymm1
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-PERLANE-NEXT:    vmovaps {{.*#+}} xmm8 = <0,6,4,u>
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm14 = ymm1[0,1,2,3],ymm0[4,5],ymm1[6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm14, %ymm8, %ymm0
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5],ymm1[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm0, %ymm8, %ymm0
 ; AVX2-FAST-PERLANE-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm9[0,1],ymm7[0,1]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm7[6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[0,2,2,2,4,6,6,6]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm6[0,1,2,3],ymm5[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm6[0,1,2,3],ymm2[4,5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vbroadcastsd {{.*#+}} ymm5 = [4,2,4,2,4,2,4,2]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm1, %ymm5, %ymm1
+; AVX2-FAST-PERLANE-NEXT:    vbroadcastsd {{.*#+}} ymm2 = [4,2,4,2,4,2,4,2]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm1, %ymm2, %ymm1
+; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm2, %ymm6
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm3[0,1],ymm2[0,1]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm0[0,1,2,3,4,5],ymm2[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm5[0,1],ymm4[0,1]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm0[0,1,2,3,4,5],ymm4[6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-PERLANE-NEXT:    vmovaps 576(%rdi), %ymm0
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm4[4,5],ymm0[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm3[4,5],ymm0[6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm0, %ymm8, %ymm0
 ; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm2[0,2,2,2,4,6,6,6]
@@ -8728,7 +8728,7 @@ define void @load_i32_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm1, %ymm5, %ymm1
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm1, %ymm6, %ymm1
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-PERLANE-NEXT:    vmovaps 1056(%rdi), %ymm1
@@ -8753,7 +8753,7 @@ define void @load_i32_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm1, %ymm5, %ymm1
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm1, %ymm6, %ymm1
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-PERLANE-NEXT:    vmovaps 1440(%rdi), %ymm1
@@ -8778,7 +8778,7 @@ define void @load_i32_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm1, %ymm5, %ymm1
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm1, %ymm6, %ymm1
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-PERLANE-NEXT:    vmovaps 96(%rdi), %ymm1
@@ -8786,21 +8786,21 @@ define void @load_i32_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX2-FAST-PERLANE-NEXT:    vmovaps 64(%rdi), %ymm0
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-PERLANE-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[0,1],ymm1[0,1]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm15 = ymm0[0,1,2,3,4,5],ymm1[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm13 = ymm0[0,1,2,3,4,5],ymm1[6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovaps (%rdi), %ymm0
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-PERLANE-NEXT:    vmovaps 32(%rdi), %ymm1
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm12 = ymm0[0,1,2,3],ymm1[4,5],ymm0[6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm12, %ymm8, %ymm0
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm15[0,2,2,2,4,6,6,6]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm11 = ymm0[0,1,2,3],ymm1[4,5],ymm0[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm11, %ymm8, %ymm0
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm13[0,2,2,2,4,6,6,6]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovaps 128(%rdi), %ymm1
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-PERLANE-NEXT:    vmovaps 160(%rdi), %ymm2
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm11 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm11, %ymm5, %ymm1
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm10 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm10, %ymm6, %ymm1
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-PERLANE-NEXT:    vmovaps 480(%rdi), %ymm1
@@ -8808,21 +8808,22 @@ define void @load_i32_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX2-FAST-PERLANE-NEXT:    vmovaps 448(%rdi), %ymm0
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-PERLANE-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[0,1],ymm1[0,1]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm10 = ymm0[0,1,2,3,4,5],ymm1[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm7 = ymm0[0,1,2,3,4,5],ymm1[6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovaps 416(%rdi), %ymm0
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-PERLANE-NEXT:    vmovaps 384(%rdi), %ymm1
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm9 = ymm1[0,1,2,3],ymm0[4,5],ymm1[6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm9, %ymm8, %ymm0
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm10[0,2,2,2,4,6,6,6]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm5 = ymm1[0,1,2,3],ymm0[4,5],ymm1[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm5, %ymm8, %ymm0
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm7[0,2,2,2,4,6,6,6]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovaps 512(%rdi), %ymm1
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-PERLANE-NEXT:    vmovaps 544(%rdi), %ymm2
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm7 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm7, %ymm5, %ymm1
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm4 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm4, %ymm6, %ymm1
+; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm6, %ymm12
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-PERLANE-NEXT:    vmovaps 864(%rdi), %ymm1
@@ -8830,114 +8831,117 @@ define void @load_i32_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX2-FAST-PERLANE-NEXT:    vmovaps 832(%rdi), %ymm0
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-PERLANE-NEXT:    vperm2f128 {{.*#+}} ymm0 = ymm0[0,1],ymm1[0,1]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm6 = ymm0[0,1,2,3,4,5],ymm1[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm3 = ymm0[0,1,2,3,4,5],ymm1[6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovaps 800(%rdi), %ymm0
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-PERLANE-NEXT:    vmovaps 768(%rdi), %ymm1
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm4 = ymm1[0,1,2,3],ymm0[4,5],ymm1[6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm4, %ymm8, %ymm0
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm6[0,2,2,2,4,6,6,6]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm1[0,1,2,3],ymm0[4,5],ymm1[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm2, %ymm8, %ymm0
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm3[0,2,2,2,4,6,6,6]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovaps 896(%rdi), %ymm1
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vmovaps 928(%rdi), %ymm2
-; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm3 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm3, %ymm5, %ymm1
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovaps 928(%rdi), %ymm6
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm6[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm1, %ymm12, %ymm6
+; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm12, %ymm14
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm6[6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-PERLANE-NEXT:    vmovaps 1184(%rdi), %ymm0
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vmovaps 1152(%rdi), %ymm1
-; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm1[0,1,2,3],ymm0[4,5],ymm1[6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm2, %ymm8, %ymm0
-; AVX2-FAST-PERLANE-NEXT:    vmovaps 1248(%rdi), %ymm8
+; AVX2-FAST-PERLANE-NEXT:    vmovaps 1152(%rdi), %ymm6
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm6 = ymm6[0,1,2,3],ymm0[4,5],ymm6[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm6, %ymm8, %ymm0
+; AVX2-FAST-PERLANE-NEXT:    vmovaps 1248(%rdi), %ymm9
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vmovaps 1216(%rdi), %ymm8
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vmovaps 1216(%rdi), %ymm1
-; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vperm2f128 {{.*#+}} ymm1 = ymm1[0,1],ymm8[0,1]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm8 = ymm1[0,1,2,3,4,5],ymm8[6,7]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm8[0,2,2,2,4,6,6,6]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovaps 1280(%rdi), %ymm1
-; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vmovaps 1312(%rdi), %ymm13
-; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm13, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm13[0,1,2,3],ymm1[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm1, %ymm5, %ymm5
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm5[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vperm2f128 {{.*#+}} ymm8 = ymm8[0,1],ymm9[0,1]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],ymm9[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm9 = ymm8[0,2,2,2,4,6,6,6]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm9[3,4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovaps 1280(%rdi), %ymm9
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vmovaps 1312(%rdi), %ymm12
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm15 = ymm12[0,1,2,3],ymm9[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm15, %ymm14, %ymm9
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm9[6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vmovaps {{.*#+}} xmm5 = <1,7,5,u>
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm14, %ymm5, %ymm0
-; AVX2-FAST-PERLANE-NEXT:    vpermilps $237, {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm13 = mem[1,3,2,3,5,7,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm13 = ymm0[0,1,2],ymm13[3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vbroadcastsd {{.*#+}} ymm0 = [5,3,5,3,5,3,5,3]
-; AVX2-FAST-PERLANE-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm14 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm13 = ymm13[0,1,2,3,4,5],ymm14[6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm13, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm13 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    vmovaps {{.*#+}} xmm0 = <1,7,5,u>
+; AVX2-FAST-PERLANE-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm9 # 32-byte Folded Reload
 ; AVX2-FAST-PERLANE-NEXT:    vpermilps $237, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Folded Reload
 ; AVX2-FAST-PERLANE-NEXT:    # ymm14 = mem[1,3,2,3,5,7,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm13 = ymm13[0,1,2],ymm14[3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm14 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm13 = ymm13[0,1,2,3,4,5],ymm14[6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm13, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm13 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm14 = ymm9[0,1,2],ymm14[3,4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vbroadcastsd {{.*#+}} ymm9 = [5,3,5,3,5,3,5,3]
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm12 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm12 = ymm14[0,1,2,3,4,5],ymm12[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm12 # 32-byte Folded Reload
 ; AVX2-FAST-PERLANE-NEXT:    vpermilps $237, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Folded Reload
 ; AVX2-FAST-PERLANE-NEXT:    # ymm14 = mem[1,3,2,3,5,7,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm13 = ymm13[0,1,2],ymm14[3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm14 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm13 = ymm13[0,1,2,3,4,5],ymm14[6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm13, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm13 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm12 = ymm12[0,1,2],ymm14[3,4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm14 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm12 = ymm12[0,1,2,3,4,5],ymm14[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm12 # 32-byte Folded Reload
 ; AVX2-FAST-PERLANE-NEXT:    vpermilps $237, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Folded Reload
 ; AVX2-FAST-PERLANE-NEXT:    # ymm14 = mem[1,3,2,3,5,7,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm13 = ymm13[0,1,2],ymm14[3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm14 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm13 = ymm13[0,1,2,3,4,5],ymm14[6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm13, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm12, %ymm5, %ymm12
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm13 = ymm15[1,3,2,3,5,7,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm12 = ymm12[0,1,2],ymm13[3,4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm12 = ymm12[0,1,2],ymm14[3,4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm14 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm12 = ymm12[0,1,2,3,4,5],ymm14[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm12 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    vpermilps $237, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm14 = mem[1,3,2,3,5,7,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm12 = ymm12[0,1,2],ymm14[3,4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm14 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm12 = ymm12[0,1,2,3,4,5],ymm14[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm11, %ymm0, %ymm11
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm11 = ymm12[0,1,2,3,4,5],ymm11[6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm9, %ymm5, %ymm9
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm10 = ymm10[1,3,2,3,5,7,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm9 = ymm9[0,1,2],ymm10[3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm7, %ymm0, %ymm7
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm7 = ymm9[0,1,2,3,4,5],ymm7[6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm4, %ymm5, %ymm4
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm6 = ymm6[1,3,2,3,5,7,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1,2],ymm6[3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm3, %ymm0, %ymm3
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm3 = ymm4[0,1,2,3,4,5],ymm3[6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm2, %ymm5, %ymm2
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm3 = ymm8[1,3,2,3,5,7,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm12 = ymm13[1,3,2,3,5,7,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm11 = ymm11[0,1,2],ymm12[3,4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm10, %ymm9, %ymm10
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm10 = ymm11[0,1,2,3,4,5],ymm10[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm5, %ymm0, %ymm5
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm7 = ymm7[1,3,2,3,5,7,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0,1,2],ymm7[3,4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm4, %ymm9, %ymm4
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm4 = ymm5[0,1,2,3,4,5],ymm4[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm2, %ymm0, %ymm2
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm3 = ymm3[1,3,2,3,5,7,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2],ymm3[3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm1, %ymm0, %ymm0
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5],ymm0[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm1, %ymm9, %ymm1
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm6, %ymm0, %ymm0
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm8[1,3,2,3,5,7,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm15, %ymm9, %ymm1
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vmovaps 192(%rdi), %xmm4
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} xmm0 = xmm4[2,3,2,3]
+; AVX2-FAST-PERLANE-NEXT:    vmovaps 192(%rdi), %xmm5
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} xmm0 = xmm5[2,3,2,3]
 ; AVX2-FAST-PERLANE-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Folded Reload
 ; AVX2-FAST-PERLANE-NEXT:    # ymm1 = mem[0,0,2,3,4,4,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vblendps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm1 = mem[0,1],ymm1[2,3],mem[4,5],ymm1[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups (%rsp), %ymm1 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vblendps $204, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm1 = ymm1[0,1],mem[2,3],ymm1[4,5],mem[6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[2,0,2,3,6,4,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,2,0,3]
 ; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,3,2,3]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
 ; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm3[0,0,0,0,4,4,4,4]
-; AVX2-FAST-PERLANE-NEXT:    vpermilps $224, (%rsp), %ymm2 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Folded Reload
 ; AVX2-FAST-PERLANE-NEXT:    # ymm2 = mem[0,0,2,3,4,4,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2],ymm1[3],ymm2[4,5,6],ymm1[7]
 ; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,1,0,3]
@@ -8950,8 +8954,8 @@ define void @load_i32_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX2-FAST-PERLANE-NEXT:    # ymm1 = mem[0,0,2,3,4,4,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vblendps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm1 = mem[0,1],ymm1[2,3],mem[4,5],ymm1[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps $204, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm1 = ymm1[0,1],mem[2,3],ymm1[4,5],mem[6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[2,0,2,3,6,4,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,2,0,3]
 ; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,3,2,3]
@@ -8966,8 +8970,8 @@ define void @load_i32_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-PERLANE-NEXT:    vmovaps 960(%rdi), %xmm2
 ; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} xmm0 = xmm2[2,3,2,3]
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[0,0,2,3,4,4,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm1 = mem[0,0,2,3,4,4,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
 ; AVX2-FAST-PERLANE-NEXT:    vblendps $204, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
@@ -8978,52 +8982,52 @@ define void @load_i32_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vpermilps $0, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Folded Reload
 ; AVX2-FAST-PERLANE-NEXT:    # ymm1 = mem[0,0,0,0,4,4,4,4]
-; AVX2-FAST-PERLANE-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm5 = mem[0,0,2,3,4,4,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm5[0,1,2],ymm1[3],ymm5[4,5,6],ymm1[7]
+; AVX2-FAST-PERLANE-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm4 = mem[0,0,2,3,4,4,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm4[0,1,2],ymm1[3],ymm4[4,5,6],ymm1[7]
 ; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,1,0,3]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-PERLANE-NEXT:    vmovaps 1344(%rdi), %xmm1
 ; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} xmm0 = xmm1[2,3,2,3]
-; AVX2-FAST-PERLANE-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm5 = mem[0,0,2,3,4,4,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm5[1,2,3],ymm0[4],ymm5[5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm4 = ymm15[0,0,2,3,4,4,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm4[1,2,3],ymm0[4],ymm4[5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0,1],ymm13[2,3],ymm5[4,5],ymm13[6,7]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm5 = ymm5[2,0,2,3,6,4,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm5 = ymm5[0,2,0,3]
+; AVX2-FAST-PERLANE-NEXT:    vblendps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm13, %ymm4 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm4 = mem[0,1],ymm13[2,3],mem[4,5],ymm13[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm4 = ymm4[2,0,2,3,6,4,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm4 = ymm4[0,2,0,3]
 ; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,3,2,3]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm5[3,4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm4[3,4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermilps $0, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm4 = mem[0,0,0,0,4,4,4,4]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm5 = ymm14[0,0,0,0,4,4,4,4]
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm6 = ymm15[0,0,2,3,4,4,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm5 = ymm6[0,1,2],ymm5[3],ymm6[4,5,6],ymm5[7]
-; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm5 = ymm5[0,1,0,3]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm5[5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm6 = ymm14[0,0,2,3,4,4,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm4 = ymm6[0,1,2],ymm4[3],ymm6[4,5,6],ymm4[7]
+; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm4 = ymm4[0,1,0,3]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm4[5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-PERLANE-NEXT:    vmovaps (%rdi), %xmm0
 ; AVX2-FAST-PERLANE-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} xmm0 = xmm0[2,3,2,3]
-; AVX2-FAST-PERLANE-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm5 = mem[0,0,2,3,4,4,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm5[1,2,3],ymm0[4],ymm5[5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vblendps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm5 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm5 = mem[0,1],ymm5[2,3],mem[4,5],ymm5[6,7]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm5 = ymm5[2,0,2,3,6,4,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm5 = ymm5[0,2,0,3]
+; AVX2-FAST-PERLANE-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm4 = mem[0,0,2,3,4,4,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm4[1,2,3],ymm0[4],ymm4[5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vblendps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm4 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm4 = mem[0,1],ymm4[2,3],mem[4,5],ymm4[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm4 = ymm4[2,0,2,3,6,4,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm4 = ymm4[0,2,0,3]
 ; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,3,2,3]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm5[3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermilps $0, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm5 = mem[0,0,0,0,4,4,4,4]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm4[3,4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermilps $0, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm4 = mem[0,0,0,0,4,4,4,4]
 ; AVX2-FAST-PERLANE-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Folded Reload
 ; AVX2-FAST-PERLANE-NEXT:    # ymm6 = mem[0,0,2,3,4,4,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm5 = ymm6[0,1,2],ymm5[3],ymm6[4,5,6],ymm5[7]
-; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm5 = ymm5[0,1,0,3]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm5[5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm4 = ymm6[0,1,2],ymm4[3],ymm6[4,5,6],ymm4[7]
+; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm4 = ymm4[0,1,0,3]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm4[5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-PERLANE-NEXT:    vmovaps 384(%rdi), %xmm0
 ; AVX2-FAST-PERLANE-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
@@ -9031,99 +9035,99 @@ define void @load_i32_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX2-FAST-PERLANE-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Folded Reload
 ; AVX2-FAST-PERLANE-NEXT:    # ymm6 = mem[0,0,2,3,4,4,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm6[1,2,3],ymm0[4],ymm6[5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vblendps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm6 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm6 = mem[0,1],ymm5[2,3],mem[4,5],ymm5[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vblendps $204, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm6 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm6 = ymm4[0,1],mem[2,3],ymm4[4,5],mem[6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm6 = ymm6[2,0,2,3,6,4,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm6 = ymm6[0,2,0,3]
 ; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,3,2,3]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm6[3,4,5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vpermilps $0, {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Folded Reload
 ; AVX2-FAST-PERLANE-NEXT:    # ymm6 = mem[0,0,0,0,4,4,4,4]
-; AVX2-FAST-PERLANE-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm7 = mem[0,0,2,3,4,4,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm6 = ymm7[0,1,2],ymm6[3],ymm7[4,5,6],ymm6[7]
+; AVX2-FAST-PERLANE-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm8 = mem[0,0,2,3,4,4,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm6 = ymm8[0,1,2],ymm6[3],ymm8[4,5,6],ymm6[7]
 ; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm6 = ymm6[0,1,0,3]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm6[5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-PERLANE-NEXT:    vmovaps 768(%rdi), %xmm0
 ; AVX2-FAST-PERLANE-NEXT:    vmovaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} xmm6 = xmm0[2,3,2,3]
-; AVX2-FAST-PERLANE-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm7 = mem[0,0,2,3,4,4,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm6 = ymm6[0],ymm7[1,2,3],ymm6[4],ymm7[5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vblendps $51, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm7 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm7 = mem[0,1],ymm0[2,3],mem[4,5],ymm0[6,7]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm7 = ymm7[2,0,2,3,6,4,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm7 = ymm7[0,2,0,3]
-; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm6 = ymm6[0,3,2,3]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm6 = ymm6[0,1,2],ymm7[3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermilps $0, {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm7 = mem[0,0,0,0,4,4,4,4]
 ; AVX2-FAST-PERLANE-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Folded Reload
 ; AVX2-FAST-PERLANE-NEXT:    # ymm8 = mem[0,0,2,3,4,4,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm7 = ymm8[0,1,2],ymm7[3],ymm8[4,5,6],ymm7[7]
-; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm7 = ymm7[0,1,0,3]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm6[0,1,2,3,4],ymm7[5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vmovaps 1152(%rdi), %xmm12
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} xmm7 = xmm12[2,3,2,3]
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm8 = ymm9[0,0,2,3,4,4,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm7 = ymm7[0],ymm8[1,2,3],ymm7[4],ymm8[5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm6 = ymm6[0],ymm8[1,2,3],ymm6[4],ymm8[5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm8 = ymm11[0,1],ymm0[2,3],ymm11[4,5],ymm0[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps $204, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm8 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm8 = ymm0[0,1],mem[2,3],ymm0[4,5],mem[6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm8 = ymm8[2,0,2,3,6,4,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm8 = ymm8[0,2,0,3]
-; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm7 = ymm7[0,3,2,3]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm7 = ymm7[0,1,2],ymm8[3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm8 = ymm6[0,0,0,0,4,4,4,4]
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm10 = ymm5[0,0,2,3,4,4,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm6 = ymm6[0,3,2,3]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm6 = ymm6[0,1,2],ymm8[3,4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermilps $0, {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm8 = mem[0,0,0,0,4,4,4,4]
+; AVX2-FAST-PERLANE-NEXT:    vpermilps $224, {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm10 = mem[0,0,2,3,4,4,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm8 = ymm10[0,1,2],ymm8[3],ymm10[4,5,6],ymm8[7]
 ; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm8 = ymm8[0,1,0,3]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm7 = ymm7[0,1,2,3,4],ymm8[5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} xmm4 = xmm4[3,3,3,3]
-; AVX2-FAST-PERLANE-NEXT:    vpermilps $244, {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm7 = mem[0,1,3,3,4,5,7,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0],ymm7[1,2,3],ymm4[4],ymm7[5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm7 = mem[3,3,3,3,7,7,7,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps $34, {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm7 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm7 = ymm7[0],mem[1],ymm7[2,3,4],mem[5],ymm7[6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm7 = ymm7[0,2,0,3]
-; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm4 = ymm4[0,3,2,3]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1,2],ymm7[3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm7 = ymm3[0,1,0,1,4,5,4,5]
-; AVX2-FAST-PERLANE-NEXT:    vpermilps $244, (%rsp), %ymm8 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm6[0,1,2,3,4],ymm8[5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vmovaps 1152(%rdi), %xmm12
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} xmm8 = xmm12[2,3,2,3]
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm10 = ymm6[0,0,2,3,4,4,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm8 = ymm8[0],ymm10[1,2,3],ymm8[4],ymm10[5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm10 = ymm9[0,1],ymm0[2,3],ymm9[4,5],ymm0[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm10 = ymm10[2,0,2,3,6,4,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm10 = ymm10[0,2,0,3]
+; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm8 = ymm8[0,3,2,3]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm8 = ymm8[0,1,2],ymm10[3,4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm10 = ymm4[0,0,0,0,4,4,4,4]
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm11 = ymm7[0,0,2,3,4,4,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm10 = ymm11[0,1,2],ymm10[3],ymm11[4,5,6],ymm10[7]
+; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm10 = ymm10[0,1,0,3]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm8 = ymm8[0,1,2,3,4],ymm10[5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} xmm5 = xmm5[3,3,3,3]
+; AVX2-FAST-PERLANE-NEXT:    vpermilps $244, {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Folded Reload
 ; AVX2-FAST-PERLANE-NEXT:    # ymm8 = mem[0,1,3,3,4,5,7,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm7 = ymm8[0,1,2],ymm7[3],ymm8[4,5,6],ymm7[7]
-; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm7 = ymm7[0,1,0,3]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm3 = ymm4[0,1,2,3,4],ymm7[5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0],ymm8[1,2,3],ymm5[4],ymm8[5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm8 = mem[3,3,3,3,7,7,7,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps $34, (%rsp), %ymm8, %ymm8 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm8 = ymm8[0],mem[1],ymm8[2,3,4],mem[5],ymm8[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm8 = ymm8[0,2,0,3]
+; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm5 = ymm5[0,3,2,3]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0,1,2],ymm8[3,4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm8 = ymm3[0,1,0,1,4,5,4,5]
+; AVX2-FAST-PERLANE-NEXT:    vpermilps $244, {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm10 = mem[0,1,3,3,4,5,7,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm8 = ymm10[0,1,2],ymm8[3],ymm10[4,5,6],ymm8[7]
+; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm8 = ymm8[0,1,0,3]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm3 = ymm5[0,1,2,3,4],ymm8[5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-PERLANE-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
 ; AVX2-FAST-PERLANE-NEXT:    # xmm3 = mem[3,3,3,3]
-; AVX2-FAST-PERLANE-NEXT:    vpermilps $244, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm4 = mem[0,1,3,3,4,5,7,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0],ymm4[1,2,3],ymm3[4],ymm4[5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm4 = mem[3,3,3,3,7,7,7,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps $34, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm4 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm4 = ymm4[0],mem[1],ymm4[2,3,4],mem[5],ymm4[6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm4 = ymm4[0,2,0,3]
+; AVX2-FAST-PERLANE-NEXT:    vpermilps $244, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm5 = mem[0,1,3,3,4,5,7,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0],ymm5[1,2,3],ymm3[4],ymm5[5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm5 = mem[3,3,3,3,7,7,7,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps $34, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm5 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm5 = ymm5[0],mem[1],ymm5[2,3,4],mem[5],ymm5[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm5 = ymm5[0,2,0,3]
 ; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm3 = ymm3[0,3,2,3]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1,2],ymm4[3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermilps $68, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm4 = mem[0,1,0,1,4,5,4,5]
-; AVX2-FAST-PERLANE-NEXT:    vpermilps $244, {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm7 = mem[0,1,3,3,4,5,7,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm4 = ymm7[0,1,2],ymm4[3],ymm7[4,5,6],ymm4[7]
-; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm4 = ymm4[0,1,0,3]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1,2,3,4],ymm4[5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1,2],ymm5[3,4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermilps $68, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm5 = mem[0,1,0,1,4,5,4,5]
+; AVX2-FAST-PERLANE-NEXT:    vpermilps $244, {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm8 = mem[0,1,3,3,4,5,7,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm5 = ymm8[0,1,2],ymm5[3],ymm8[4,5,6],ymm5[7]
+; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm5 = ymm5[0,1,0,3]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1,2,3,4],ymm5[5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} xmm2 = xmm2[3,3,3,3]
 ; AVX2-FAST-PERLANE-NEXT:    vpermilps $244, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Folded Reload
@@ -9138,15 +9142,14 @@ define void @load_i32_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2],ymm3[3,4,5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vpermilps $68, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Folded Reload
 ; AVX2-FAST-PERLANE-NEXT:    # ymm3 = mem[0,1,0,1,4,5,4,5]
-; AVX2-FAST-PERLANE-NEXT:    vpermilps $244, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm4 = mem[0,1,3,3,4,5,7,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm3 = ymm4[0,1,2],ymm3[3],ymm4[4,5,6],ymm3[7]
+; AVX2-FAST-PERLANE-NEXT:    vpermilps $244, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm5 = mem[0,1,3,3,4,5,7,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm3 = ymm5[0,1,2],ymm3[3],ymm5[4,5,6],ymm3[7]
 ; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm3 = ymm3[0,1,0,3]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm3[5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
-; AVX2-FAST-PERLANE-NEXT:    vpermilps $244, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm2 = mem[0,1,3,3,4,5,7,7]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm2 = ymm15[0,1,3,3,4,5,7,7]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm2[1,2,3],ymm1[4],ymm2[5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm2 = ymm13[3,3,3,3,7,7,7,7]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps $34, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload
@@ -9154,302 +9157,298 @@ define void @load_i32_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[0,2,0,3]
 ; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,3,2,3]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2],ymm2[3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm2 = ymm14[0,1,0,1,4,5,4,5]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm3 = ymm15[0,1,3,3,4,5,7,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermilps $68, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm2 = mem[0,1,0,1,4,5,4,5]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm3 = ymm14[0,1,3,3,4,5,7,7]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1,2],ymm2[3],ymm3[4,5,6],ymm2[7]
 ; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[0,1,0,3]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm2[5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} xmm1 = xmm12[3,3,3,3]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm2 = ymm9[0,1,3,3,4,5,7,7]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm2 = ymm6[0,1,3,3,4,5,7,7]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm2[1,2,3],ymm1[4],ymm2[5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm2 = ymm0[3,3,3,3,7,7,7,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0],ymm11[1],ymm2[2,3,4],ymm11[5],ymm2[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0],ymm9[1],ymm2[2,3,4],ymm9[5],ymm2[6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[0,2,0,3]
 ; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,3,2,3]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2],ymm2[3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm2 = ymm6[0,1,0,1,4,5,4,5]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm3 = ymm5[0,1,3,3,4,5,7,7]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm2 = ymm4[0,1,0,1,4,5,4,5]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm3 = ymm7[0,1,3,3,4,5,7,7]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1,2],ymm2[3],ymm3[4,5,6],ymm2[7]
 ; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm2 = ymm2[0,1,0,3]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3,4],ymm2[5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-PERLANE-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
 ; AVX2-FAST-PERLANE-NEXT:    # xmm0 = mem[3,3,3,3]
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm10[0,1,3,3,4,5,7,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermilps $244, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm1 = mem[0,1,3,3,4,5,7,7]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload
 ; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm13[3,3,3,3,7,7,7,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm11[1],ymm1[2,3,4],ymm11[5],ymm1[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm12[1],ymm1[2,3,4],ymm12[5],ymm1[6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,2,0,3]
 ; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,3,2,3]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm14[0,1,0,1,4,5,4,5]
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm2 = ymm12[0,1,3,3,4,5,7,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermilps $68, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm1 = mem[0,1,0,1,4,5,4,5]
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm2 = ymm15[0,1,3,3,4,5,7,7]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2],ymm1[3],ymm2[4,5,6],ymm1[7]
 ; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,1,0,3]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-PERLANE-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
 ; AVX2-FAST-PERLANE-NEXT:    # xmm0 = mem[3,3,3,3]
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm4[0,1,3,3,4,5,7,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm11 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm11[0,1,3,3,4,5,7,7]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm8[3,3,3,3,7,7,7,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm7[3,3,3,3,7,7,7,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm6[1],ymm1[2,3,4],ymm6[5],ymm1[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm7[1],ymm1[2,3,4],ymm7[5],ymm1[6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,2,0,3]
 ; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,3,2,3]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm10[0,1,0,1,4,5,4,5]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm9[0,1,0,1,4,5,4,5]
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm2 = ymm8[0,1,3,3,4,5,7,7]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm2 = ymm9[0,1,3,3,4,5,7,7]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2],ymm1[3],ymm2[4,5,6],ymm1[7]
 ; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,1,0,3]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-PERLANE-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
 ; AVX2-FAST-PERLANE-NEXT:    # xmm0 = mem[3,3,3,3]
-; AVX2-FAST-PERLANE-NEXT:    vpermilps $244, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm1 = mem[0,1,3,3,4,5,7,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm6[0,1,3,3,4,5,7,7]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermilps $255, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm1 = mem[3,3,3,3,7,7,7,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm15[1],ymm1[2,3,4],ymm15[5],ymm1[6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm3[3,3,3,3,7,7,7,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0],ymm14[1],ymm1[2,3,4],ymm14[5],ymm1[6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,2,0,3]
 ; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm0 = ymm0[0,3,2,3]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2],ymm1[3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermilps $68, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm1 = mem[0,1,0,1,4,5,4,5]
-; AVX2-FAST-PERLANE-NEXT:    vpermilps $244, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm2 = mem[0,1,3,3,4,5,7,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm5[0,1,0,1,4,5,4,5]
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm2 = ymm4[0,1,3,3,4,5,7,7]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1,2],ymm1[3],ymm2[4,5,6],ymm1[7]
 ; AVX2-FAST-PERLANE-NEXT:    vpermpd {{.*#+}} ymm1 = ymm1[0,1,0,3]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm1[5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm2 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm2 = ymm0[0,1,2,3],mem[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vblendps $15, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm1 = mem[0,1,2,3],ymm0[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm14[0,1,2,3],ymm3[4,5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vmovaps 272(%rdi), %xmm0
+; AVX2-FAST-PERLANE-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm6, %ymm2 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm2 = ymm6[0,1,2,3],mem[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vmovaps 80(%rdi), %xmm0
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3],ymm1[4,5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm0 = ymm0[0,1,0,2,4,5,4,6]
-; AVX2-FAST-PERLANE-NEXT:    vmovddup {{.*#+}} xmm1 = [4,2,4,2]
-; AVX2-FAST-PERLANE-NEXT:    # xmm1 = mem[0,0]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm2, %ymm1, %ymm2
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1],ymm0[2,3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vblendps $243, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm0 = mem[0,1],ymm0[2,3],mem[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, (%rsp) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vbroadcastf128 {{.*#+}} ymm5 = [0,2,0,6,0,2,0,6]
-; AVX2-FAST-PERLANE-NEXT:    # ymm5 = mem[0,1,0,1]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm0, %ymm5, %ymm3
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm3[5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm3 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm3 = ymm4[0,1,2,3],mem[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm6[0,1,2,3],ymm7[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm2, %ymm14, %ymm1
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1],ymm0[2,3,4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm5[0,1],ymm4[2,3],ymm5[4,5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vmovaps 464(%rdi), %xmm0
-; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1],ymm0[2,3],ymm2[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm2 = ymm2[0,1,0,2,4,5,4,6]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm3, %ymm1, %ymm3
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1],ymm2[2,3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm9[0,1],ymm8[2,3],ymm9[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm0, %ymm5, %ymm3
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm3[5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vbroadcastf128 {{.*#+}} ymm0 = [0,2,0,6,0,2,0,6]
+; AVX2-FAST-PERLANE-NEXT:    # ymm0 = mem[0,1,0,1]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm2, %ymm0, %ymm2
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm2[5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vmovups (%rsp), %ymm1 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm2 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm2 = ymm1[0,1,2,3],mem[4,5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm0 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm0 = ymm2[0,1,2,3],mem[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm3 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm3 = ymm2[0,1,2,3],mem[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm3 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm3 = ymm1[0,1,2,3],mem[4,5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vmovaps 656(%rdi), %xmm2
+; AVX2-FAST-PERLANE-NEXT:    vmovaps 272(%rdi), %xmm1
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm1, (%rsp) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1],ymm1[2,3],ymm2[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[0,1,0,2,4,5,4,6]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm3, %ymm14, %ymm2
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1],ymm1[2,3,4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vblendps $243, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm2 = mem[0,1],ymm2[2,3],mem[4,5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1],ymm2[2,3],ymm3[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm2 = ymm2[0,1,0,2,4,5,4,6]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm0, %ymm1, %ymm3
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1],ymm2[2,3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vblendps $243, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm0 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm0 = mem[0,1],ymm3[2,3],mem[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm0, %ymm5, %ymm3
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm3[5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm2, %ymm0, %ymm2
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm2[5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm7[0,1,2,3],ymm8[4,5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm10, %ymm0 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm0 = ymm10[0,1,2,3],mem[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm3 = ymm11[0,1,2,3],ymm13[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm11, %ymm3 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm3 = ymm11[0,1,2,3],mem[4,5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vmovaps 848(%rdi), %xmm2
+; AVX2-FAST-PERLANE-NEXT:    vmovaps 464(%rdi), %xmm1
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1],ymm1[2,3],ymm2[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[0,1,0,2,4,5,4,6]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm3, %ymm14, %ymm2
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1],ymm1[2,3,4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm10[0,1],ymm9[2,3],ymm10[4,5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1],ymm2[2,3],ymm3[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm2 = ymm2[0,1,0,2,4,5,4,6]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm0, %ymm1, %ymm3
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1],ymm2[2,3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm14[0,1],ymm12[2,3],ymm14[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm0, %ymm5, %ymm3
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm3[5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm2, %ymm0, %ymm2
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm2[5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm2 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm2 = ymm1[0,1,2,3],mem[4,5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm0 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm0 = ymm2[0,1,2,3],mem[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm3 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm3 = ymm2[0,1,2,3],mem[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm3 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm3 = ymm1[0,1,2,3],mem[4,5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vmovaps 1040(%rdi), %xmm2
-; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1],ymm2[2,3],ymm3[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm2 = ymm2[0,1,0,2,4,5,4,6]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm0, %ymm1, %ymm3
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1],ymm2[2,3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vblendps $243, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm13 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm13 = mem[0,1],ymm3[2,3],mem[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm13, %ymm5, %ymm3
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm3[5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm0 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm0 = ymm2[0,1,2,3],mem[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vmovaps 656(%rdi), %xmm1
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1],ymm1[2,3],ymm2[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[0,1,0,2,4,5,4,6]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm3, %ymm14, %ymm2
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1],ymm1[2,3,4,5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm12 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm12 = ymm2[0,1,2,3],mem[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovaps 1232(%rdi), %xmm11
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm12[0,1],ymm11[2,3],ymm12[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm2 = ymm2[0,1,0,2,4,5,4,6]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm0, %ymm1, %ymm3
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1],ymm2[2,3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vblendps $243, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm9 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm9 = mem[0,1],ymm3[2,3],mem[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm9, %ymm5, %ymm3
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm3[5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps $243, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm2 = mem[0,1],ymm2[2,3],mem[4,5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm0 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm0 = ymm2[0,1,2,3],mem[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm7 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm7 = ymm2[0,1,2,3],mem[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovaps 1424(%rdi), %xmm8
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm7[0,1],ymm8[2,3],ymm7[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm2 = ymm2[0,1,0,2,4,5,4,6]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm0, %ymm1, %ymm3
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1],ymm2[2,3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vblendps $243, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm6 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm6 = mem[0,1],ymm3[2,3],mem[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm6, %ymm5, %ymm3
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3,4],ymm3[5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm2, %ymm0, %ymm2
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm2[5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm12[0,1,2,3],ymm13[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm3 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm3 = ymm1[0,1,2,3],mem[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vmovaps 848(%rdi), %xmm1
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1],ymm1[2,3],ymm2[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[0,1,0,2,4,5,4,6]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm3, %ymm14, %ymm2
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1],ymm1[2,3,4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps $243, {{[-0-9]+}}(%r{{[sb]}}p), %ymm15, %ymm2 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm2 = mem[0,1],ymm15[2,3],mem[4,5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FAST-PERLANE-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm15, %ymm10 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm10 = ymm15[0,1,2,3],mem[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm2, %ymm0, %ymm2
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm2[5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm13 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm13 = ymm1[0,1,2,3],mem[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm12 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm12 = ymm1[0,1,2,3],mem[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovaps 1040(%rdi), %xmm11
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm13[0,1],ymm11[2,3],ymm13[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[0,1,0,2,4,5,4,6]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm12, %ymm14, %ymm2
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1],ymm1[2,3,4,5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm4 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm4 = ymm2[0,1,2,3],mem[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovaps 80(%rdi), %xmm3
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm10[0,1],ymm3[2,3],ymm10[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm2 = ymm2[0,1,0,2,4,5,4,6]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm4, %ymm1, %ymm1
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1],ymm2[2,3,4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps $12, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm10 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm10 = ymm2[0,1],mem[2,3],ymm2[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm10, %ymm0, %ymm2
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm2[5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm8 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm8 = ymm1[0,1,2,3],mem[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm7 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm7 = ymm1[0,1,2,3],mem[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovaps 1232(%rdi), %xmm6
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm8[0,1],ymm6[2,3],ymm8[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[0,1,0,2,4,5,4,6]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm7, %ymm14, %ymm2
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm2[0,1],ymm1[2,3,4,5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vblendps $243, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    # ymm2 = mem[0,1],ymm2[2,3],mem[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm2, %ymm5, %ymm0
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3,4],ymm0[5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vblendps $243, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm5 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm5 = mem[0,1],ymm2[2,3],mem[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm5, %ymm0, %ymm2
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm2[5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm4 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm4 = ymm1[0,1,2,3],mem[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm3 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm3 = ymm1[0,1,2,3],mem[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovaps 1424(%rdi), %xmm2
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm4[0,1],ymm2[2,3],ymm4[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm1 = ymm1[0,1,0,2,4,5,4,6]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm3, %ymm14, %ymm15
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm15 = ymm15[0,1],ymm1[2,3,4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vblendps $243, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    # ymm1 = mem[0,1],ymm1[2,3],mem[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm1, %ymm0, %ymm0
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm15[0,1,2,3,4],ymm0[5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-PERLANE-NEXT:    vpermilps $85, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload
 ; AVX2-FAST-PERLANE-NEXT:    # ymm0 = mem[1,1,1,1,5,5,5,5]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
 ; AVX2-FAST-PERLANE-NEXT:    # ymm0 = ymm0[0,1,2],mem[3],ymm0[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovddup {{.*#+}} xmm1 = [5,3,5,3]
-; AVX2-FAST-PERLANE-NEXT:    # xmm1 = mem[0,0]
-; AVX2-FAST-PERLANE-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm14 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm14 = ymm14[0,1],ymm0[2,3,4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm15 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm15 = ymm15[0,1],ymm0[2,3,4,5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vbroadcastf128 {{.*#+}} ymm0 = [0,3,1,7,0,3,1,7]
 ; AVX2-FAST-PERLANE-NEXT:    # ymm0 = mem[0,1,0,1]
-; AVX2-FAST-PERLANE-NEXT:    vpermps (%rsp), %ymm0, %ymm15 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm5 = ymm14[0,1,2,3,4],ymm15[5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm14 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm14 = ymm15[0,1,2,3,4],ymm14[5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm14, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-PERLANE-NEXT:    vpermilps $85, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Folded Reload
 ; AVX2-FAST-PERLANE-NEXT:    # ymm14 = mem[1,1,1,1,5,5,5,5]
-; AVX2-FAST-PERLANE-NEXT:    vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm14 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    vblendps $8, (%rsp), %ymm14, %ymm14 # 32-byte Folded Reload
 ; AVX2-FAST-PERLANE-NEXT:    # ymm14 = ymm14[0,1,2],mem[3],ymm14[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm15 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm15 # 32-byte Folded Reload
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm14 = ymm15[0,1],ymm14[2,3,4,5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm15 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm5 = ymm14[0,1,2,3,4],ymm15[5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm14 = ymm14[0,1,2,3,4],ymm15[5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm14, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-PERLANE-NEXT:    vpermilps $85, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Folded Reload
 ; AVX2-FAST-PERLANE-NEXT:    # ymm14 = mem[1,1,1,1,5,5,5,5]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm14 # 32-byte Folded Reload
 ; AVX2-FAST-PERLANE-NEXT:    # ymm14 = ymm14[0,1,2],mem[3],ymm14[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm15 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm15 # 32-byte Folded Reload
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm14 = ymm15[0,1],ymm14[2,3,4,5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm15 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm5 = ymm14[0,1,2,3,4],ymm15[5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm14 = ymm14[0,1,2,3,4],ymm15[5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm14, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-PERLANE-NEXT:    vpermilps $85, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Folded Reload
 ; AVX2-FAST-PERLANE-NEXT:    # ymm14 = mem[1,1,1,1,5,5,5,5]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm14 # 32-byte Folded Reload
 ; AVX2-FAST-PERLANE-NEXT:    # ymm14 = ymm14[0,1,2],mem[3],ymm14[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm15 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm15 # 32-byte Folded Reload
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm14 = ymm15[0,1],ymm14[2,3,4,5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm15 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm5 = ymm14[0,1,2,3,4],ymm15[5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm14 = ymm14[0,1,2,3,4],ymm15[5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vmovups %ymm14, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX2-FAST-PERLANE-NEXT:    vpermilps $85, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Folded Reload
 ; AVX2-FAST-PERLANE-NEXT:    # ymm14 = mem[1,1,1,1,5,5,5,5]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm14 # 32-byte Folded Reload
 ; AVX2-FAST-PERLANE-NEXT:    # ymm14 = ymm14[0,1,2],mem[3],ymm14[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm15 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm15 # 32-byte Folded Reload
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm14 = ymm15[0,1],ymm14[2,3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm13, %ymm0, %ymm13
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm13 = ymm14[0,1,2,3,4],ymm13[5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm12 = ymm12[1,1,1,1,5,5,5,5]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm11 = ymm12[0,1,2],ymm11[3],ymm12[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm12 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm15 # 32-byte Folded Reload
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm14 = ymm14[0,1,2,3,4],ymm15[5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm13 = ymm13[1,1,1,1,5,5,5,5]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm11 = ymm13[0,1,2],ymm11[3],ymm13[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm12, %ymm9, %ymm12
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm11 = ymm12[0,1],ymm11[2,3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm9, %ymm0, %ymm9
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm9 = ymm11[0,1,2,3,4],ymm9[5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm7 = ymm7[1,1,1,1,5,5,5,5]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm7 = ymm7[0,1,2],ymm8[3],ymm7[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermps {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm8 # 32-byte Folded Reload
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm7 = ymm8[0,1],ymm7[2,3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm6, %ymm0, %ymm6
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm6 = ymm7[0,1,2,3,4],ymm6[5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm5 = ymm10[1,1,1,1,5,5,5,5]
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm3 = ymm5[0,1,2],ymm3[3],ymm5[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm4, %ymm1, %ymm1
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm1 = ymm1[0,1],ymm3[2,3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm2, %ymm0, %ymm0
-; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3,4],ymm0[5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm10, %ymm0, %ymm10
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm10 = ymm11[0,1,2,3,4],ymm10[5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm8 = ymm8[1,1,1,1,5,5,5,5]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm6 = ymm8[0,1,2],ymm6[3],ymm8[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm7, %ymm9, %ymm7
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm6 = ymm7[0,1],ymm6[2,3,4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm5, %ymm0, %ymm5
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm5 = ymm6[0,1,2,3,4],ymm5[5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vshufps {{.*#+}} ymm4 = ymm4[1,1,1,1,5,5,5,5]
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm4[0,1,2],ymm2[3],ymm4[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm3, %ymm9, %ymm3
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm2 = ymm3[0,1],ymm2[2,3,4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm1, %ymm0, %ymm0
+; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3,4],ymm0[5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
 ; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm1, 192(%rsi)
 ; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
@@ -9531,19 +9530,19 @@ define void @load_i32_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
 ; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm1, (%r9)
 ; AVX2-FAST-PERLANE-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm6, 224(%rax)
-; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm9, 192(%rax)
-; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm13, 160(%rax)
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm1, 128(%rax)
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm1, 96(%rax)
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm1, 64(%rax)
-; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm1, 32(%rax)
+; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm0, 224(%rax)
+; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm5, 192(%rax)
+; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm10, 160(%rax)
+; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm14, 128(%rax)
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm0, 96(%rax)
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm0, 64(%rax)
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm0, 32(%rax)
+; AVX2-FAST-PERLANE-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX2-FAST-PERLANE-NEXT:    vmovaps %ymm0, (%rax)
-; AVX2-FAST-PERLANE-NEXT:    addq $2504, %rsp # imm = 0x9C8
+; AVX2-FAST-PERLANE-NEXT:    addq $2536, %rsp # imm = 0x9E8
 ; AVX2-FAST-PERLANE-NEXT:    vzeroupper
 ; AVX2-FAST-PERLANE-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i32-stride-8.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i32-stride-8.ll
index 1bd7025307d2b95d96a7a75a93a7a0281828e714..faa1642831c1553ace0f74116cb7adb540303e5c 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i32-stride-8.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i32-stride-8.ll
@@ -1617,117 +1617,108 @@ define void @load_i32_stride8_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512F-NEXT:    movq {{[0-9]+}}(%rsp), %r10
 ; AVX512F-NEXT:    movq {{[0-9]+}}(%rsp), %r11
 ; AVX512F-NEXT:    vmovdqa64 (%rdi), %zmm0
-; AVX512F-NEXT:    vmovdqa64 64(%rdi), %zmm1
-; AVX512F-NEXT:    vmovdqa64 128(%rdi), %zmm3
+; AVX512F-NEXT:    vmovdqa64 64(%rdi), %zmm2
+; AVX512F-NEXT:    vmovdqa64 128(%rdi), %zmm1
 ; AVX512F-NEXT:    vmovdqa64 192(%rdi), %zmm4
 ; AVX512F-NEXT:    vmovdqa64 320(%rdi), %zmm5
-; AVX512F-NEXT:    vmovdqa64 256(%rdi), %zmm2
+; AVX512F-NEXT:    vmovdqa64 256(%rdi), %zmm3
 ; AVX512F-NEXT:    vmovdqa64 448(%rdi), %zmm7
 ; AVX512F-NEXT:    vmovdqa64 384(%rdi), %zmm6
 ; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm8 = [0,8,16,24,0,8,16,24,0,8,16,24,0,8,16,24]
 ; AVX512F-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm9
 ; AVX512F-NEXT:    vpermt2d %zmm7, %zmm8, %zmm9
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm10
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm10
 ; AVX512F-NEXT:    vpermt2d %zmm5, %zmm8, %zmm10
 ; AVX512F-NEXT:    movb $-64, %dil
 ; AVX512F-NEXT:    kmovw %edi, %k1
 ; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm10 {%k1}
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm9 = [0,8,16,24,0,8,16,24]
-; AVX512F-NEXT:    # ymm9 = mem[0,1,0,1]
-; AVX512F-NEXT:    vpermi2d %zmm4, %zmm3, %zmm9
-; AVX512F-NEXT:    vpermi2d %zmm1, %zmm0, %zmm8
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm9
+; AVX512F-NEXT:    vpermt2d %zmm4, %zmm8, %zmm9
+; AVX512F-NEXT:    vpermi2d %zmm2, %zmm0, %zmm8
 ; AVX512F-NEXT:    vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3],ymm9[4,5,6,7]
 ; AVX512F-NEXT:    vinserti64x4 $0, %ymm8, %zmm10, %zmm8
 ; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm9 = [1,9,17,25,1,9,17,25,1,9,17,25,1,9,17,25]
 ; AVX512F-NEXT:    # zmm9 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm10
 ; AVX512F-NEXT:    vpermt2d %zmm7, %zmm9, %zmm10
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm11
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm11
 ; AVX512F-NEXT:    vpermt2d %zmm5, %zmm9, %zmm11
 ; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm11 {%k1}
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm10 = [1,9,17,25,1,9,17,25]
-; AVX512F-NEXT:    # ymm10 = mem[0,1,0,1]
-; AVX512F-NEXT:    vpermi2d %zmm4, %zmm3, %zmm10
-; AVX512F-NEXT:    vpermi2d %zmm1, %zmm0, %zmm9
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm10
+; AVX512F-NEXT:    vpermt2d %zmm4, %zmm9, %zmm10
+; AVX512F-NEXT:    vpermi2d %zmm2, %zmm0, %zmm9
 ; AVX512F-NEXT:    vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3],ymm10[4,5,6,7]
 ; AVX512F-NEXT:    vinserti64x4 $0, %ymm9, %zmm11, %zmm9
 ; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm10 = [2,10,18,26,2,10,18,26,2,10,18,26,2,10,18,26]
 ; AVX512F-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm11
 ; AVX512F-NEXT:    vpermt2d %zmm7, %zmm10, %zmm11
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm12
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm12
 ; AVX512F-NEXT:    vpermt2d %zmm5, %zmm10, %zmm12
 ; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm12 {%k1}
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm11 = [2,10,18,26,2,10,18,26]
-; AVX512F-NEXT:    # ymm11 = mem[0,1,0,1]
-; AVX512F-NEXT:    vpermi2d %zmm4, %zmm3, %zmm11
-; AVX512F-NEXT:    vpermi2d %zmm1, %zmm0, %zmm10
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm11
+; AVX512F-NEXT:    vpermt2d %zmm4, %zmm10, %zmm11
+; AVX512F-NEXT:    vpermi2d %zmm2, %zmm0, %zmm10
 ; AVX512F-NEXT:    vpblendd {{.*#+}} ymm10 = ymm10[0,1,2,3],ymm11[4,5,6,7]
 ; AVX512F-NEXT:    vinserti64x4 $0, %ymm10, %zmm12, %zmm10
 ; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm11 = [3,11,19,27,3,11,19,27,3,11,19,27,3,11,19,27]
 ; AVX512F-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm12
 ; AVX512F-NEXT:    vpermt2d %zmm7, %zmm11, %zmm12
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm13
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm13
 ; AVX512F-NEXT:    vpermt2d %zmm5, %zmm11, %zmm13
 ; AVX512F-NEXT:    vmovdqa64 %zmm12, %zmm13 {%k1}
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm12 = [3,11,19,27,3,11,19,27]
-; AVX512F-NEXT:    # ymm12 = mem[0,1,0,1]
-; AVX512F-NEXT:    vpermi2d %zmm4, %zmm3, %zmm12
-; AVX512F-NEXT:    vpermi2d %zmm1, %zmm0, %zmm11
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm12
+; AVX512F-NEXT:    vpermt2d %zmm4, %zmm11, %zmm12
+; AVX512F-NEXT:    vpermi2d %zmm2, %zmm0, %zmm11
 ; AVX512F-NEXT:    vpblendd {{.*#+}} ymm11 = ymm11[0,1,2,3],ymm12[4,5,6,7]
 ; AVX512F-NEXT:    vinserti64x4 $0, %ymm11, %zmm13, %zmm11
 ; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm12 = [4,12,20,28,4,12,20,28,4,12,20,28,4,12,20,28]
 ; AVX512F-NEXT:    # zmm12 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm13
 ; AVX512F-NEXT:    vpermt2d %zmm7, %zmm12, %zmm13
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm14
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm14
 ; AVX512F-NEXT:    vpermt2d %zmm5, %zmm12, %zmm14
 ; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm14 {%k1}
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm13 = [4,12,20,28,4,12,20,28]
-; AVX512F-NEXT:    # ymm13 = mem[0,1,0,1]
-; AVX512F-NEXT:    vpermi2d %zmm4, %zmm3, %zmm13
-; AVX512F-NEXT:    vpermi2d %zmm1, %zmm0, %zmm12
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm13
+; AVX512F-NEXT:    vpermt2d %zmm4, %zmm12, %zmm13
+; AVX512F-NEXT:    vpermi2d %zmm2, %zmm0, %zmm12
 ; AVX512F-NEXT:    vpblendd {{.*#+}} ymm12 = ymm12[0,1,2,3],ymm13[4,5,6,7]
 ; AVX512F-NEXT:    vinserti64x4 $0, %ymm12, %zmm14, %zmm12
 ; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm13 = [5,13,21,29,5,13,21,29,5,13,21,29,5,13,21,29]
 ; AVX512F-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm14
 ; AVX512F-NEXT:    vpermt2d %zmm7, %zmm13, %zmm14
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm15
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm15
 ; AVX512F-NEXT:    vpermt2d %zmm5, %zmm13, %zmm15
 ; AVX512F-NEXT:    vmovdqa64 %zmm14, %zmm15 {%k1}
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm14 = [5,13,21,29,5,13,21,29]
-; AVX512F-NEXT:    # ymm14 = mem[0,1,0,1]
-; AVX512F-NEXT:    vpermi2d %zmm4, %zmm3, %zmm14
-; AVX512F-NEXT:    vpermi2d %zmm1, %zmm0, %zmm13
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm14
+; AVX512F-NEXT:    vpermt2d %zmm4, %zmm13, %zmm14
+; AVX512F-NEXT:    vpermi2d %zmm2, %zmm0, %zmm13
 ; AVX512F-NEXT:    vpblendd {{.*#+}} ymm13 = ymm13[0,1,2,3],ymm14[4,5,6,7]
 ; AVX512F-NEXT:    vinserti64x4 $0, %ymm13, %zmm15, %zmm13
 ; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm14 = [6,14,22,30,6,14,22,30,6,14,22,30,6,14,22,30]
 ; AVX512F-NEXT:    # zmm14 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm15
 ; AVX512F-NEXT:    vpermt2d %zmm7, %zmm14, %zmm15
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm16
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm16
 ; AVX512F-NEXT:    vpermt2d %zmm5, %zmm14, %zmm16
 ; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm16 {%k1}
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm15 = [6,14,22,30,6,14,22,30]
-; AVX512F-NEXT:    # ymm15 = mem[0,1,0,1]
-; AVX512F-NEXT:    vpermi2d %zmm4, %zmm3, %zmm15
-; AVX512F-NEXT:    vpermi2d %zmm1, %zmm0, %zmm14
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm15
+; AVX512F-NEXT:    vpermt2d %zmm4, %zmm14, %zmm15
+; AVX512F-NEXT:    vpermi2d %zmm2, %zmm0, %zmm14
 ; AVX512F-NEXT:    vpblendd {{.*#+}} ymm14 = ymm14[0,1,2,3],ymm15[4,5,6,7]
 ; AVX512F-NEXT:    vinserti64x4 $0, %ymm14, %zmm16, %zmm14
 ; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm15 = [7,15,23,31,7,15,23,31,7,15,23,31,7,15,23,31]
 ; AVX512F-NEXT:    # zmm15 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; AVX512F-NEXT:    vpermt2d %zmm7, %zmm15, %zmm6
-; AVX512F-NEXT:    vpermt2d %zmm5, %zmm15, %zmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm2 {%k1}
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm5 = [7,15,23,31,7,15,23,31]
-; AVX512F-NEXT:    # ymm5 = mem[0,1,0,1]
-; AVX512F-NEXT:    vpermi2d %zmm4, %zmm3, %zmm5
-; AVX512F-NEXT:    vpermt2d %zmm1, %zmm15, %zmm0
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm2, %zmm0
+; AVX512F-NEXT:    vpermt2d %zmm5, %zmm15, %zmm3
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm3 {%k1}
+; AVX512F-NEXT:    vpermt2d %zmm4, %zmm15, %zmm1
+; AVX512F-NEXT:    vpermt2d %zmm2, %zmm15, %zmm0
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm3, %zmm0
 ; AVX512F-NEXT:    vmovdqa64 %zmm8, (%rsi)
 ; AVX512F-NEXT:    vmovdqa64 %zmm9, (%rdx)
 ; AVX512F-NEXT:    vmovdqa64 %zmm10, (%rcx)
@@ -1745,117 +1736,108 @@ define void @load_i32_stride8_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %r10
 ; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %r11
 ; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm0
-; AVX512BW-NEXT:    vmovdqa64 64(%rdi), %zmm1
-; AVX512BW-NEXT:    vmovdqa64 128(%rdi), %zmm3
+; AVX512BW-NEXT:    vmovdqa64 64(%rdi), %zmm2
+; AVX512BW-NEXT:    vmovdqa64 128(%rdi), %zmm1
 ; AVX512BW-NEXT:    vmovdqa64 192(%rdi), %zmm4
 ; AVX512BW-NEXT:    vmovdqa64 320(%rdi), %zmm5
-; AVX512BW-NEXT:    vmovdqa64 256(%rdi), %zmm2
+; AVX512BW-NEXT:    vmovdqa64 256(%rdi), %zmm3
 ; AVX512BW-NEXT:    vmovdqa64 448(%rdi), %zmm7
 ; AVX512BW-NEXT:    vmovdqa64 384(%rdi), %zmm6
 ; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm8 = [0,8,16,24,0,8,16,24,0,8,16,24,0,8,16,24]
 ; AVX512BW-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm9
 ; AVX512BW-NEXT:    vpermt2d %zmm7, %zmm8, %zmm9
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm10
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm10
 ; AVX512BW-NEXT:    vpermt2d %zmm5, %zmm8, %zmm10
 ; AVX512BW-NEXT:    movb $-64, %dil
 ; AVX512BW-NEXT:    kmovd %edi, %k1
 ; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm10 {%k1}
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm9 = [0,8,16,24,0,8,16,24]
-; AVX512BW-NEXT:    # ymm9 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vpermi2d %zmm4, %zmm3, %zmm9
-; AVX512BW-NEXT:    vpermi2d %zmm1, %zmm0, %zmm8
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm9
+; AVX512BW-NEXT:    vpermt2d %zmm4, %zmm8, %zmm9
+; AVX512BW-NEXT:    vpermi2d %zmm2, %zmm0, %zmm8
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3],ymm9[4,5,6,7]
 ; AVX512BW-NEXT:    vinserti64x4 $0, %ymm8, %zmm10, %zmm8
 ; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm9 = [1,9,17,25,1,9,17,25,1,9,17,25,1,9,17,25]
 ; AVX512BW-NEXT:    # zmm9 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm10
 ; AVX512BW-NEXT:    vpermt2d %zmm7, %zmm9, %zmm10
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm11
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm11
 ; AVX512BW-NEXT:    vpermt2d %zmm5, %zmm9, %zmm11
 ; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm11 {%k1}
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm10 = [1,9,17,25,1,9,17,25]
-; AVX512BW-NEXT:    # ymm10 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vpermi2d %zmm4, %zmm3, %zmm10
-; AVX512BW-NEXT:    vpermi2d %zmm1, %zmm0, %zmm9
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm10
+; AVX512BW-NEXT:    vpermt2d %zmm4, %zmm9, %zmm10
+; AVX512BW-NEXT:    vpermi2d %zmm2, %zmm0, %zmm9
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3],ymm10[4,5,6,7]
 ; AVX512BW-NEXT:    vinserti64x4 $0, %ymm9, %zmm11, %zmm9
 ; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm10 = [2,10,18,26,2,10,18,26,2,10,18,26,2,10,18,26]
 ; AVX512BW-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm11
 ; AVX512BW-NEXT:    vpermt2d %zmm7, %zmm10, %zmm11
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm12
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm12
 ; AVX512BW-NEXT:    vpermt2d %zmm5, %zmm10, %zmm12
 ; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm12 {%k1}
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm11 = [2,10,18,26,2,10,18,26]
-; AVX512BW-NEXT:    # ymm11 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vpermi2d %zmm4, %zmm3, %zmm11
-; AVX512BW-NEXT:    vpermi2d %zmm1, %zmm0, %zmm10
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm11
+; AVX512BW-NEXT:    vpermt2d %zmm4, %zmm10, %zmm11
+; AVX512BW-NEXT:    vpermi2d %zmm2, %zmm0, %zmm10
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm10 = ymm10[0,1,2,3],ymm11[4,5,6,7]
 ; AVX512BW-NEXT:    vinserti64x4 $0, %ymm10, %zmm12, %zmm10
 ; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm11 = [3,11,19,27,3,11,19,27,3,11,19,27,3,11,19,27]
 ; AVX512BW-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm12
 ; AVX512BW-NEXT:    vpermt2d %zmm7, %zmm11, %zmm12
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm13
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm13
 ; AVX512BW-NEXT:    vpermt2d %zmm5, %zmm11, %zmm13
 ; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm13 {%k1}
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm12 = [3,11,19,27,3,11,19,27]
-; AVX512BW-NEXT:    # ymm12 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vpermi2d %zmm4, %zmm3, %zmm12
-; AVX512BW-NEXT:    vpermi2d %zmm1, %zmm0, %zmm11
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm12
+; AVX512BW-NEXT:    vpermt2d %zmm4, %zmm11, %zmm12
+; AVX512BW-NEXT:    vpermi2d %zmm2, %zmm0, %zmm11
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm11 = ymm11[0,1,2,3],ymm12[4,5,6,7]
 ; AVX512BW-NEXT:    vinserti64x4 $0, %ymm11, %zmm13, %zmm11
 ; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm12 = [4,12,20,28,4,12,20,28,4,12,20,28,4,12,20,28]
 ; AVX512BW-NEXT:    # zmm12 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm13
 ; AVX512BW-NEXT:    vpermt2d %zmm7, %zmm12, %zmm13
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm14
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm14
 ; AVX512BW-NEXT:    vpermt2d %zmm5, %zmm12, %zmm14
 ; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm14 {%k1}
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm13 = [4,12,20,28,4,12,20,28]
-; AVX512BW-NEXT:    # ymm13 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vpermi2d %zmm4, %zmm3, %zmm13
-; AVX512BW-NEXT:    vpermi2d %zmm1, %zmm0, %zmm12
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm13
+; AVX512BW-NEXT:    vpermt2d %zmm4, %zmm12, %zmm13
+; AVX512BW-NEXT:    vpermi2d %zmm2, %zmm0, %zmm12
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm12 = ymm12[0,1,2,3],ymm13[4,5,6,7]
 ; AVX512BW-NEXT:    vinserti64x4 $0, %ymm12, %zmm14, %zmm12
 ; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm13 = [5,13,21,29,5,13,21,29,5,13,21,29,5,13,21,29]
 ; AVX512BW-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm14
 ; AVX512BW-NEXT:    vpermt2d %zmm7, %zmm13, %zmm14
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm15
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm15
 ; AVX512BW-NEXT:    vpermt2d %zmm5, %zmm13, %zmm15
 ; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm15 {%k1}
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm14 = [5,13,21,29,5,13,21,29]
-; AVX512BW-NEXT:    # ymm14 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vpermi2d %zmm4, %zmm3, %zmm14
-; AVX512BW-NEXT:    vpermi2d %zmm1, %zmm0, %zmm13
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm14
+; AVX512BW-NEXT:    vpermt2d %zmm4, %zmm13, %zmm14
+; AVX512BW-NEXT:    vpermi2d %zmm2, %zmm0, %zmm13
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm13 = ymm13[0,1,2,3],ymm14[4,5,6,7]
 ; AVX512BW-NEXT:    vinserti64x4 $0, %ymm13, %zmm15, %zmm13
 ; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm14 = [6,14,22,30,6,14,22,30,6,14,22,30,6,14,22,30]
 ; AVX512BW-NEXT:    # zmm14 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm15
 ; AVX512BW-NEXT:    vpermt2d %zmm7, %zmm14, %zmm15
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm16
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm16
 ; AVX512BW-NEXT:    vpermt2d %zmm5, %zmm14, %zmm16
 ; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm16 {%k1}
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm15 = [6,14,22,30,6,14,22,30]
-; AVX512BW-NEXT:    # ymm15 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vpermi2d %zmm4, %zmm3, %zmm15
-; AVX512BW-NEXT:    vpermi2d %zmm1, %zmm0, %zmm14
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm15
+; AVX512BW-NEXT:    vpermt2d %zmm4, %zmm14, %zmm15
+; AVX512BW-NEXT:    vpermi2d %zmm2, %zmm0, %zmm14
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm14 = ymm14[0,1,2,3],ymm15[4,5,6,7]
 ; AVX512BW-NEXT:    vinserti64x4 $0, %ymm14, %zmm16, %zmm14
 ; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm15 = [7,15,23,31,7,15,23,31,7,15,23,31,7,15,23,31]
 ; AVX512BW-NEXT:    # zmm15 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; AVX512BW-NEXT:    vpermt2d %zmm7, %zmm15, %zmm6
-; AVX512BW-NEXT:    vpermt2d %zmm5, %zmm15, %zmm2
-; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm2 {%k1}
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm5 = [7,15,23,31,7,15,23,31]
-; AVX512BW-NEXT:    # ymm5 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vpermi2d %zmm4, %zmm3, %zmm5
-; AVX512BW-NEXT:    vpermt2d %zmm1, %zmm15, %zmm0
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm2, %zmm0
+; AVX512BW-NEXT:    vpermt2d %zmm5, %zmm15, %zmm3
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm3 {%k1}
+; AVX512BW-NEXT:    vpermt2d %zmm4, %zmm15, %zmm1
+; AVX512BW-NEXT:    vpermt2d %zmm2, %zmm15, %zmm0
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm3, %zmm0
 ; AVX512BW-NEXT:    vmovdqa64 %zmm8, (%rsi)
 ; AVX512BW-NEXT:    vmovdqa64 %zmm9, (%rdx)
 ; AVX512BW-NEXT:    vmovdqa64 %zmm10, (%rcx)
@@ -3689,563 +3671,439 @@ define void @load_i32_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ;
 ; AVX512F-LABEL: load_i32_stride8_vf32:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    subq $1096, %rsp # imm = 0x448
-; AVX512F-NEXT:    vmovdqa64 704(%rdi), %zmm6
-; AVX512F-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 640(%rdi), %zmm8
-; AVX512F-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 576(%rdi), %zmm28
-; AVX512F-NEXT:    vmovdqa64 512(%rdi), %zmm11
-; AVX512F-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 832(%rdi), %zmm5
-; AVX512F-NEXT:    vmovdqa64 768(%rdi), %zmm29
-; AVX512F-NEXT:    vmovdqa64 960(%rdi), %zmm21
-; AVX512F-NEXT:    vmovdqa64 896(%rdi), %zmm25
-; AVX512F-NEXT:    vmovdqa64 (%rdi), %zmm7
-; AVX512F-NEXT:    vmovdqa64 64(%rdi), %zmm17
-; AVX512F-NEXT:    vmovdqa64 128(%rdi), %zmm4
-; AVX512F-NEXT:    vmovdqa64 192(%rdi), %zmm9
-; AVX512F-NEXT:    vmovdqa64 320(%rdi), %zmm20
-; AVX512F-NEXT:    vmovdqa64 256(%rdi), %zmm26
-; AVX512F-NEXT:    vmovdqa64 448(%rdi), %zmm22
-; AVX512F-NEXT:    vmovdqa64 384(%rdi), %zmm2
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [0,8,16,24,0,8,16,24,0,8,16,24,0,8,16,24]
-; AVX512F-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm1
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm12
-; AVX512F-NEXT:    vpermt2d %zmm22, %zmm0, %zmm1
-; AVX512F-NEXT:    vmovdqa64 %zmm26, %zmm2
-; AVX512F-NEXT:    vpermt2d %zmm20, %zmm0, %zmm2
-; AVX512F-NEXT:    movb $-64, %al
-; AVX512F-NEXT:    kmovw %eax, %k1
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [0,8,16,24,0,8,16,24]
-; AVX512F-NEXT:    # ymm1 = mem[0,1,0,1]
-; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm3
-; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm10
-; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2d %zmm9, %zmm1, %zmm3
-; AVX512F-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm4
-; AVX512F-NEXT:    vpermt2d %zmm17, %zmm0, %zmm4
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm3, %zmm2, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm25, %zmm2
-; AVX512F-NEXT:    vpermt2d %zmm21, %zmm0, %zmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm29, %zmm3
-; AVX512F-NEXT:    vpermt2d %zmm5, %zmm0, %zmm3
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}
-; AVX512F-NEXT:    vpermi2d %zmm28, %zmm11, %zmm0
-; AVX512F-NEXT:    vpermi2d %zmm6, %zmm8, %zmm1
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm3, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm4 = [1,9,17,25,1,9,17,25,1,9,17,25,1,9,17,25]
+; AVX512F-NEXT:    movq {{[0-9]+}}(%rsp), %rax
+; AVX512F-NEXT:    movq {{[0-9]+}}(%rsp), %r10
+; AVX512F-NEXT:    movq {{[0-9]+}}(%rsp), %r11
+; AVX512F-NEXT:    vmovdqa64 (%rdi), %zmm0
+; AVX512F-NEXT:    vmovdqa64 64(%rdi), %zmm29
+; AVX512F-NEXT:    vmovdqa64 128(%rdi), %zmm1
+; AVX512F-NEXT:    vmovdqa64 192(%rdi), %zmm30
+; AVX512F-NEXT:    vmovdqa64 320(%rdi), %zmm31
+; AVX512F-NEXT:    vmovdqa64 256(%rdi), %zmm3
+; AVX512F-NEXT:    vmovdqa64 448(%rdi), %zmm7
+; AVX512F-NEXT:    vmovdqa64 384(%rdi), %zmm6
+; AVX512F-NEXT:    vmovdqa64 576(%rdi), %zmm9
+; AVX512F-NEXT:    vmovdqa64 512(%rdi), %zmm5
+; AVX512F-NEXT:    vmovdqa64 704(%rdi), %zmm12
+; AVX512F-NEXT:    vmovdqa64 640(%rdi), %zmm2
+; AVX512F-NEXT:    vmovdqa64 832(%rdi), %zmm14
+; AVX512F-NEXT:    vmovdqa64 768(%rdi), %zmm11
+; AVX512F-NEXT:    vmovdqa64 960(%rdi), %zmm16
+; AVX512F-NEXT:    vmovdqa64 896(%rdi), %zmm15
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm13 = [0,8,16,24,0,8,16,24,0,8,16,24,0,8,16,24]
+; AVX512F-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm17
+; AVX512F-NEXT:    vpermt2d %zmm16, %zmm13, %zmm17
+; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm18
+; AVX512F-NEXT:    vpermt2d %zmm14, %zmm13, %zmm18
+; AVX512F-NEXT:    movb $-64, %dil
+; AVX512F-NEXT:    kmovw %edi, %k1
+; AVX512F-NEXT:    vmovdqa64 %zmm17, %zmm18 {%k1}
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm10
+; AVX512F-NEXT:    vpermt2d %zmm12, %zmm13, %zmm10
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm8
+; AVX512F-NEXT:    vpermt2d %zmm9, %zmm13, %zmm8
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3],ymm10[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm8, %zmm18, %zmm28
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm8
+; AVX512F-NEXT:    vpermt2d %zmm7, %zmm13, %zmm8
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm10
+; AVX512F-NEXT:    vpermt2d %zmm31, %zmm13, %zmm10
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm10 {%k1}
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm8
+; AVX512F-NEXT:    vpermt2d %zmm30, %zmm13, %zmm8
+; AVX512F-NEXT:    vpermi2d %zmm29, %zmm0, %zmm13
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm8 = ymm13[0,1,2,3],ymm8[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm8, %zmm10, %zmm17
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm8 = [1,9,17,25,1,9,17,25,1,9,17,25,1,9,17,25]
+; AVX512F-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm10
+; AVX512F-NEXT:    vpermt2d %zmm16, %zmm8, %zmm10
+; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm13
+; AVX512F-NEXT:    vpermt2d %zmm14, %zmm8, %zmm13
+; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm13 {%k1}
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm10
+; AVX512F-NEXT:    vpermt2d %zmm12, %zmm8, %zmm10
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm4
+; AVX512F-NEXT:    vpermt2d %zmm9, %zmm8, %zmm4
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm10[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm4, %zmm13, %zmm18
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm4
+; AVX512F-NEXT:    vpermt2d %zmm7, %zmm8, %zmm4
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm10
+; AVX512F-NEXT:    vpermt2d %zmm31, %zmm8, %zmm10
+; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm10 {%k1}
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm4
+; AVX512F-NEXT:    vpermt2d %zmm30, %zmm8, %zmm4
+; AVX512F-NEXT:    vpermi2d %zmm29, %zmm0, %zmm8
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm4 = ymm8[0,1,2,3],ymm4[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm4, %zmm10, %zmm19
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm4 = [2,10,18,26,2,10,18,26,2,10,18,26,2,10,18,26]
 ; AVX512F-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm12, %zmm0
-; AVX512F-NEXT:    vpermt2d %zmm22, %zmm4, %zmm0
-; AVX512F-NEXT:    vmovdqa64 %zmm26, %zmm1
-; AVX512F-NEXT:    vpermt2d %zmm20, %zmm4, %zmm1
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [1,9,17,25,1,9,17,25]
-; AVX512F-NEXT:    # ymm2 = mem[0,1,0,1]
-; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm0
-; AVX512F-NEXT:    vpermt2d %zmm9, %zmm2, %zmm0
-; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm2
-; AVX512F-NEXT:    vpermt2d %zmm17, %zmm4, %zmm2
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm1, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [2,10,18,26,2,10,18,26,2,10,18,26,2,10,18,26]
-; AVX512F-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm12, %zmm1
-; AVX512F-NEXT:    vpermt2d %zmm22, %zmm0, %zmm1
-; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm2 = [3,11,19,27,3,11,19,27,3,11,19,27,3,11,19,27]
-; AVX512F-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm12, %zmm1
-; AVX512F-NEXT:    vpermt2d %zmm22, %zmm2, %zmm1
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm10 = [4,12,20,28,4,12,20,28,4,12,20,28,4,12,20,28]
+; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm8
+; AVX512F-NEXT:    vpermt2d %zmm16, %zmm4, %zmm8
+; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm10
+; AVX512F-NEXT:    vpermt2d %zmm14, %zmm4, %zmm10
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm10 {%k1}
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm8
+; AVX512F-NEXT:    vpermt2d %zmm12, %zmm4, %zmm8
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm13
+; AVX512F-NEXT:    vpermt2d %zmm9, %zmm4, %zmm13
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm8 = ymm13[0,1,2,3],ymm8[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm8, %zmm10, %zmm20
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm8
+; AVX512F-NEXT:    vpermt2d %zmm7, %zmm4, %zmm8
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm10
+; AVX512F-NEXT:    vpermt2d %zmm31, %zmm4, %zmm10
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm10 {%k1}
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm8
+; AVX512F-NEXT:    vpermt2d %zmm30, %zmm4, %zmm8
+; AVX512F-NEXT:    vpermi2d %zmm29, %zmm0, %zmm4
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm8[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm4, %zmm10, %zmm21
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm4 = [3,11,19,27,3,11,19,27,3,11,19,27,3,11,19,27]
+; AVX512F-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm8
+; AVX512F-NEXT:    vpermt2d %zmm16, %zmm4, %zmm8
+; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm10
+; AVX512F-NEXT:    vpermt2d %zmm14, %zmm4, %zmm10
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm10 {%k1}
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm8
+; AVX512F-NEXT:    vpermt2d %zmm12, %zmm4, %zmm8
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm13
+; AVX512F-NEXT:    vpermt2d %zmm9, %zmm4, %zmm13
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm8 = ymm13[0,1,2,3],ymm8[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm8, %zmm10, %zmm22
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm8
+; AVX512F-NEXT:    vpermt2d %zmm7, %zmm4, %zmm8
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm10
+; AVX512F-NEXT:    vpermt2d %zmm31, %zmm4, %zmm10
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm10 {%k1}
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm8
+; AVX512F-NEXT:    vpermt2d %zmm30, %zmm4, %zmm8
+; AVX512F-NEXT:    vpermi2d %zmm29, %zmm0, %zmm4
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm8[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm4, %zmm10, %zmm23
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm4 = [4,12,20,28,4,12,20,28,4,12,20,28,4,12,20,28]
+; AVX512F-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm8
+; AVX512F-NEXT:    vpermt2d %zmm16, %zmm4, %zmm8
+; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm10
+; AVX512F-NEXT:    vpermt2d %zmm14, %zmm4, %zmm10
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm10 {%k1}
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm8
+; AVX512F-NEXT:    vpermt2d %zmm12, %zmm4, %zmm8
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm13
+; AVX512F-NEXT:    vpermt2d %zmm9, %zmm4, %zmm13
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm8 = ymm13[0,1,2,3],ymm8[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm8, %zmm10, %zmm24
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm8
+; AVX512F-NEXT:    vpermt2d %zmm7, %zmm4, %zmm8
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm10
+; AVX512F-NEXT:    vpermt2d %zmm31, %zmm4, %zmm10
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm10 {%k1}
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm8
+; AVX512F-NEXT:    vpermt2d %zmm30, %zmm4, %zmm8
+; AVX512F-NEXT:    vpermi2d %zmm29, %zmm0, %zmm4
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm8[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm4, %zmm10, %zmm25
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm4 = [5,13,21,29,5,13,21,29,5,13,21,29,5,13,21,29]
+; AVX512F-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm8
+; AVX512F-NEXT:    vpermt2d %zmm16, %zmm4, %zmm8
+; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm10
+; AVX512F-NEXT:    vpermt2d %zmm14, %zmm4, %zmm10
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm10 {%k1}
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm8
+; AVX512F-NEXT:    vpermt2d %zmm12, %zmm4, %zmm8
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm13
+; AVX512F-NEXT:    vpermt2d %zmm9, %zmm4, %zmm13
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm8 = ymm13[0,1,2,3],ymm8[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm8, %zmm10, %zmm26
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm8
+; AVX512F-NEXT:    vpermt2d %zmm7, %zmm4, %zmm8
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm10
+; AVX512F-NEXT:    vpermt2d %zmm31, %zmm4, %zmm10
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm10 {%k1}
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm8
+; AVX512F-NEXT:    vpermt2d %zmm30, %zmm4, %zmm8
+; AVX512F-NEXT:    vpermi2d %zmm29, %zmm0, %zmm4
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm8[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm4, %zmm10, %zmm27
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm4 = [6,14,22,30,6,14,22,30,6,14,22,30,6,14,22,30]
+; AVX512F-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm8
+; AVX512F-NEXT:    vpermt2d %zmm16, %zmm4, %zmm8
+; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm10
+; AVX512F-NEXT:    vpermt2d %zmm14, %zmm4, %zmm10
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm10 {%k1}
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm8
+; AVX512F-NEXT:    vpermt2d %zmm12, %zmm4, %zmm8
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm13
+; AVX512F-NEXT:    vpermt2d %zmm9, %zmm4, %zmm13
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm8 = ymm13[0,1,2,3],ymm8[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm8, %zmm10, %zmm8
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm10
+; AVX512F-NEXT:    vpermt2d %zmm7, %zmm4, %zmm10
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm13
+; AVX512F-NEXT:    vpermt2d %zmm31, %zmm4, %zmm13
+; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm13 {%k1}
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm10
+; AVX512F-NEXT:    vpermt2d %zmm30, %zmm4, %zmm10
+; AVX512F-NEXT:    vpermi2d %zmm29, %zmm0, %zmm4
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm10[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm4, %zmm13, %zmm4
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm10 = [7,15,23,31,7,15,23,31,7,15,23,31,7,15,23,31]
 ; AVX512F-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm12, %zmm2
-; AVX512F-NEXT:    vpermt2d %zmm22, %zmm10, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm15 = [5,13,21,29,5,13,21,29,5,13,21,29,5,13,21,29]
-; AVX512F-NEXT:    # zmm15 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm12, %zmm1
-; AVX512F-NEXT:    vpermt2d %zmm22, %zmm15, %zmm1
-; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm13 = [6,14,22,30,6,14,22,30,6,14,22,30,6,14,22,30]
-; AVX512F-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm12, %zmm1
-; AVX512F-NEXT:    vpermt2d %zmm22, %zmm13, %zmm1
-; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm1 = [7,15,23,31,7,15,23,31,7,15,23,31,7,15,23,31]
-; AVX512F-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermt2d %zmm22, %zmm1, %zmm12
-; AVX512F-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm26, %zmm24
-; AVX512F-NEXT:    vpermt2d %zmm20, %zmm0, %zmm24
-; AVX512F-NEXT:    vmovdqa64 %zmm26, %zmm27
-; AVX512F-NEXT:    vpermt2d %zmm20, %zmm3, %zmm27
-; AVX512F-NEXT:    vmovdqa64 %zmm26, %zmm31
-; AVX512F-NEXT:    vpermt2d %zmm20, %zmm10, %zmm31
-; AVX512F-NEXT:    vmovdqa64 %zmm26, %zmm30
-; AVX512F-NEXT:    vpermt2d %zmm20, %zmm15, %zmm30
-; AVX512F-NEXT:    vmovdqa64 %zmm26, %zmm22
-; AVX512F-NEXT:    vpermt2d %zmm20, %zmm13, %zmm22
-; AVX512F-NEXT:    vpermt2d %zmm20, %zmm1, %zmm26
-; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm8
-; AVX512F-NEXT:    vpermt2d %zmm17, %zmm0, %zmm8
-; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm12
-; AVX512F-NEXT:    vpermt2d %zmm17, %zmm3, %zmm12
-; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm6
-; AVX512F-NEXT:    vpermt2d %zmm17, %zmm10, %zmm6
-; AVX512F-NEXT:    vmovdqu64 %zmm6, (%rsp) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm6
-; AVX512F-NEXT:    vpermt2d %zmm17, %zmm15, %zmm6
-; AVX512F-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm6
-; AVX512F-NEXT:    vpermt2d %zmm17, %zmm13, %zmm6
-; AVX512F-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2d %zmm17, %zmm1, %zmm7
-; AVX512F-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm25, %zmm18
-; AVX512F-NEXT:    vpermt2d %zmm21, %zmm4, %zmm18
-; AVX512F-NEXT:    vmovdqa64 %zmm25, %zmm19
-; AVX512F-NEXT:    vpermt2d %zmm21, %zmm0, %zmm19
-; AVX512F-NEXT:    vmovdqa64 %zmm25, %zmm23
-; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm6
-; AVX512F-NEXT:    vpermt2d %zmm21, %zmm3, %zmm23
-; AVX512F-NEXT:    vmovdqa64 %zmm25, %zmm16
-; AVX512F-NEXT:    vpermt2d %zmm21, %zmm10, %zmm16
-; AVX512F-NEXT:    vmovdqa64 %zmm25, %zmm20
-; AVX512F-NEXT:    vpermt2d %zmm21, %zmm15, %zmm20
-; AVX512F-NEXT:    vmovdqa64 %zmm25, %zmm17
-; AVX512F-NEXT:    vpermt2d %zmm21, %zmm13, %zmm17
-; AVX512F-NEXT:    vpermt2d %zmm21, %zmm1, %zmm25
-; AVX512F-NEXT:    vmovdqa64 %zmm29, %zmm9
-; AVX512F-NEXT:    vpermt2d %zmm5, %zmm4, %zmm9
-; AVX512F-NEXT:    vmovdqa64 %zmm29, %zmm11
-; AVX512F-NEXT:    vpermt2d %zmm5, %zmm0, %zmm11
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm3
-; AVX512F-NEXT:    vmovdqa64 %zmm29, %zmm14
-; AVX512F-NEXT:    vpermt2d %zmm5, %zmm6, %zmm14
-; AVX512F-NEXT:    vmovdqa64 %zmm29, %zmm21
-; AVX512F-NEXT:    vpermt2d %zmm5, %zmm10, %zmm21
-; AVX512F-NEXT:    vmovdqa64 %zmm29, %zmm7
-; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm2
-; AVX512F-NEXT:    vpermt2d %zmm5, %zmm15, %zmm7
-; AVX512F-NEXT:    vmovdqa64 %zmm29, %zmm15
-; AVX512F-NEXT:    vpermt2d %zmm5, %zmm13, %zmm15
-; AVX512F-NEXT:    vpermt2d %zmm5, %zmm1, %zmm29
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vpermi2d %zmm28, %zmm0, %zmm4
-; AVX512F-NEXT:    vpermi2d %zmm28, %zmm0, %zmm3
-; AVX512F-NEXT:    vpermi2d %zmm28, %zmm0, %zmm6
-; AVX512F-NEXT:    vpermi2d %zmm28, %zmm0, %zmm10
-; AVX512F-NEXT:    vpermi2d %zmm28, %zmm0, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermi2d %zmm28, %zmm0, %zmm13
-; AVX512F-NEXT:    vpermt2d %zmm28, %zmm1, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm18, %zmm9 {%k1}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm28 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = [1,9,17,25,1,9,17,25]
-; AVX512F-NEXT:    # ymm0 = mem[0,1,0,1]
-; AVX512F-NEXT:    vpermi2d %zmm28, %zmm2, %zmm0
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm0[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm9, %zmm18
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm24 {%k1}
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm9 = [2,10,18,26,2,10,18,26]
-; AVX512F-NEXT:    # ymm9 = mem[0,1,0,1]
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm0
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2d %zmm5, %zmm9, %zmm0
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm8[0,1,2,3],ymm0[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm24, %zmm24
-; AVX512F-NEXT:    vmovdqa64 %zmm19, %zmm11 {%k1}
-; AVX512F-NEXT:    vpermi2d %zmm28, %zmm2, %zmm9
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm9[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm11, %zmm1
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm27 {%k1}
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [3,11,19,27,3,11,19,27]
-; AVX512F-NEXT:    # ymm3 = mem[0,1,0,1]
-; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm0
-; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm9
-; AVX512F-NEXT:    vpermt2d %zmm5, %zmm3, %zmm9
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm4 = ymm12[0,1,2,3],ymm9[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm4, %zmm27, %zmm4
-; AVX512F-NEXT:    vmovdqa64 %zmm23, %zmm14 {%k1}
-; AVX512F-NEXT:    vpermi2d %zmm28, %zmm2, %zmm3
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm3 = ymm6[0,1,2,3],ymm3[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm3, %zmm14, %zmm3
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm6 = [4,12,20,28,4,12,20,28]
-; AVX512F-NEXT:    # ymm6 = mem[0,1,0,1]
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm8
-; AVX512F-NEXT:    vpermt2d %zmm5, %zmm6, %zmm8
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm9 = [5,13,21,29,5,13,21,29]
-; AVX512F-NEXT:    # ymm9 = mem[0,1,0,1]
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm11
-; AVX512F-NEXT:    vpermt2d %zmm5, %zmm9, %zmm11
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm12 = [6,14,22,30,6,14,22,30]
-; AVX512F-NEXT:    # ymm12 = mem[0,1,0,1]
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm14
-; AVX512F-NEXT:    vpermt2d %zmm5, %zmm12, %zmm14
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} ymm19 = [7,15,23,31,7,15,23,31]
-; AVX512F-NEXT:    # ymm19 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermt2d %zmm5, %zmm19, %zmm0
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm5
-; AVX512F-NEXT:    vpermi2d %zmm28, %zmm2, %zmm6
-; AVX512F-NEXT:    vpermi2d %zmm28, %zmm2, %zmm9
-; AVX512F-NEXT:    vpermi2d %zmm28, %zmm2, %zmm12
-; AVX512F-NEXT:    vpermt2d %zmm28, %zmm19, %zmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm0
-; AVX512F-NEXT:    vpblendd $15, (%rsp), %ymm8, %ymm8 # 32-byte Folded Reload
-; AVX512F-NEXT:    # ymm8 = mem[0,1,2,3],ymm8[4,5,6,7]
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm31 {%k1}
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm8, %zmm31, %zmm8
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm6 = ymm10[0,1,2,3],ymm6[4,5,6,7]
-; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm21 {%k1}
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm6, %zmm21, %zmm6
-; AVX512F-NEXT:    vpblendd $15, {{[-0-9]+}}(%r{{[sb]}}p), %ymm11, %ymm10 # 32-byte Folded Reload
-; AVX512F-NEXT:    # ymm10 = mem[0,1,2,3],ymm11[4,5,6,7]
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm30 {%k1}
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm10, %zmm30, %zmm10
-; AVX512F-NEXT:    vpblendd $15, {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm9 # 32-byte Folded Reload
-; AVX512F-NEXT:    # ymm9 = mem[0,1,2,3],ymm9[4,5,6,7]
-; AVX512F-NEXT:    vmovdqa64 %zmm20, %zmm7 {%k1}
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm9, %zmm7, %zmm7
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm22 {%k1}
-; AVX512F-NEXT:    vpblendd $15, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm9 # 32-byte Folded Reload
-; AVX512F-NEXT:    # ymm9 = mem[0,1,2,3],ymm14[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm9, %zmm22, %zmm9
-; AVX512F-NEXT:    vmovdqa64 %zmm17, %zmm15 {%k1}
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm13[0,1,2,3],ymm12[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm15, %zmm2
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm26 {%k1}
-; AVX512F-NEXT:    vpblendd $15, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm11 # 32-byte Folded Reload
-; AVX512F-NEXT:    # ymm11 = mem[0,1,2,3],ymm5[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm11, %zmm26, %zmm5
-; AVX512F-NEXT:    vmovdqa64 %zmm25, %zmm29 {%k1}
-; AVX512F-NEXT:    vpblendd $15, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm11 # 32-byte Folded Reload
-; AVX512F-NEXT:    # ymm11 = mem[0,1,2,3],ymm0[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm11, %zmm29, %zmm11
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 64(%rsi)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, (%rsi)
+; AVX512F-NEXT:    vpermt2d %zmm16, %zmm10, %zmm15
+; AVX512F-NEXT:    vpermt2d %zmm14, %zmm10, %zmm11
+; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm11 {%k1}
+; AVX512F-NEXT:    vpermt2d %zmm12, %zmm10, %zmm2
+; AVX512F-NEXT:    vpermt2d %zmm9, %zmm10, %zmm5
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm5[0,1,2,3],ymm2[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm11, %zmm2
+; AVX512F-NEXT:    vpermt2d %zmm7, %zmm10, %zmm6
+; AVX512F-NEXT:    vpermt2d %zmm31, %zmm10, %zmm3
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm3 {%k1}
+; AVX512F-NEXT:    vpermt2d %zmm30, %zmm10, %zmm1
+; AVX512F-NEXT:    vpermt2d %zmm29, %zmm10, %zmm0
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm3, %zmm0
+; AVX512F-NEXT:    vmovdqa64 %zmm28, 64(%rsi)
+; AVX512F-NEXT:    vmovdqa64 %zmm17, (%rsi)
 ; AVX512F-NEXT:    vmovdqa64 %zmm18, 64(%rdx)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, (%rdx)
-; AVX512F-NEXT:    vmovdqa64 %zmm1, 64(%rcx)
-; AVX512F-NEXT:    vmovdqa64 %zmm24, (%rcx)
-; AVX512F-NEXT:    vmovdqa64 %zmm3, 64(%r8)
-; AVX512F-NEXT:    vmovdqa64 %zmm4, (%r8)
-; AVX512F-NEXT:    vmovdqa64 %zmm6, 64(%r9)
-; AVX512F-NEXT:    vmovdqa64 %zmm8, (%r9)
-; AVX512F-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512F-NEXT:    vmovdqa64 %zmm7, 64(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm10, (%rax)
-; AVX512F-NEXT:    movq {{[0-9]+}}(%rsp), %rax
+; AVX512F-NEXT:    vmovdqa64 %zmm19, (%rdx)
+; AVX512F-NEXT:    vmovdqa64 %zmm20, 64(%rcx)
+; AVX512F-NEXT:    vmovdqa64 %zmm21, (%rcx)
+; AVX512F-NEXT:    vmovdqa64 %zmm22, 64(%r8)
+; AVX512F-NEXT:    vmovdqa64 %zmm23, (%r8)
+; AVX512F-NEXT:    vmovdqa64 %zmm24, 64(%r9)
+; AVX512F-NEXT:    vmovdqa64 %zmm25, (%r9)
+; AVX512F-NEXT:    vmovdqa64 %zmm26, 64(%r11)
+; AVX512F-NEXT:    vmovdqa64 %zmm27, (%r11)
+; AVX512F-NEXT:    vmovdqa64 %zmm8, 64(%r10)
+; AVX512F-NEXT:    vmovdqa64 %zmm4, (%r10)
 ; AVX512F-NEXT:    vmovdqa64 %zmm2, 64(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm9, (%rax)
-; AVX512F-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512F-NEXT:    vmovdqa64 %zmm11, 64(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm5, (%rax)
-; AVX512F-NEXT:    addq $1096, %rsp # imm = 0x448
+; AVX512F-NEXT:    vmovdqa64 %zmm0, (%rax)
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: load_i32_stride8_vf32:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    subq $1096, %rsp # imm = 0x448
-; AVX512BW-NEXT:    vmovdqa64 704(%rdi), %zmm6
-; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 640(%rdi), %zmm8
-; AVX512BW-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 576(%rdi), %zmm28
-; AVX512BW-NEXT:    vmovdqa64 512(%rdi), %zmm11
-; AVX512BW-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 832(%rdi), %zmm5
-; AVX512BW-NEXT:    vmovdqa64 768(%rdi), %zmm29
-; AVX512BW-NEXT:    vmovdqa64 960(%rdi), %zmm21
-; AVX512BW-NEXT:    vmovdqa64 896(%rdi), %zmm25
-; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm7
-; AVX512BW-NEXT:    vmovdqa64 64(%rdi), %zmm17
-; AVX512BW-NEXT:    vmovdqa64 128(%rdi), %zmm4
-; AVX512BW-NEXT:    vmovdqa64 192(%rdi), %zmm9
-; AVX512BW-NEXT:    vmovdqa64 320(%rdi), %zmm20
-; AVX512BW-NEXT:    vmovdqa64 256(%rdi), %zmm26
-; AVX512BW-NEXT:    vmovdqa64 448(%rdi), %zmm22
-; AVX512BW-NEXT:    vmovdqa64 384(%rdi), %zmm2
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [0,8,16,24,0,8,16,24,0,8,16,24,0,8,16,24]
-; AVX512BW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm1
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm12
-; AVX512BW-NEXT:    vpermt2d %zmm22, %zmm0, %zmm1
-; AVX512BW-NEXT:    vmovdqa64 %zmm26, %zmm2
-; AVX512BW-NEXT:    vpermt2d %zmm20, %zmm0, %zmm2
-; AVX512BW-NEXT:    movb $-64, %al
-; AVX512BW-NEXT:    kmovd %eax, %k1
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [0,8,16,24,0,8,16,24]
-; AVX512BW-NEXT:    # ymm1 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm3
-; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm10
-; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2d %zmm9, %zmm1, %zmm3
-; AVX512BW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm4
-; AVX512BW-NEXT:    vpermt2d %zmm17, %zmm0, %zmm4
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm3[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm3, %zmm2, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm25, %zmm2
-; AVX512BW-NEXT:    vpermt2d %zmm21, %zmm0, %zmm2
-; AVX512BW-NEXT:    vmovdqa64 %zmm29, %zmm3
-; AVX512BW-NEXT:    vpermt2d %zmm5, %zmm0, %zmm3
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}
-; AVX512BW-NEXT:    vpermi2d %zmm28, %zmm11, %zmm0
-; AVX512BW-NEXT:    vpermi2d %zmm6, %zmm8, %zmm1
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm3, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm4 = [1,9,17,25,1,9,17,25,1,9,17,25,1,9,17,25]
+; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
+; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %r10
+; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %r11
+; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm0
+; AVX512BW-NEXT:    vmovdqa64 64(%rdi), %zmm29
+; AVX512BW-NEXT:    vmovdqa64 128(%rdi), %zmm1
+; AVX512BW-NEXT:    vmovdqa64 192(%rdi), %zmm30
+; AVX512BW-NEXT:    vmovdqa64 320(%rdi), %zmm31
+; AVX512BW-NEXT:    vmovdqa64 256(%rdi), %zmm3
+; AVX512BW-NEXT:    vmovdqa64 448(%rdi), %zmm7
+; AVX512BW-NEXT:    vmovdqa64 384(%rdi), %zmm6
+; AVX512BW-NEXT:    vmovdqa64 576(%rdi), %zmm9
+; AVX512BW-NEXT:    vmovdqa64 512(%rdi), %zmm5
+; AVX512BW-NEXT:    vmovdqa64 704(%rdi), %zmm12
+; AVX512BW-NEXT:    vmovdqa64 640(%rdi), %zmm2
+; AVX512BW-NEXT:    vmovdqa64 832(%rdi), %zmm14
+; AVX512BW-NEXT:    vmovdqa64 768(%rdi), %zmm11
+; AVX512BW-NEXT:    vmovdqa64 960(%rdi), %zmm16
+; AVX512BW-NEXT:    vmovdqa64 896(%rdi), %zmm15
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm13 = [0,8,16,24,0,8,16,24,0,8,16,24,0,8,16,24]
+; AVX512BW-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm17
+; AVX512BW-NEXT:    vpermt2d %zmm16, %zmm13, %zmm17
+; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm18
+; AVX512BW-NEXT:    vpermt2d %zmm14, %zmm13, %zmm18
+; AVX512BW-NEXT:    movb $-64, %dil
+; AVX512BW-NEXT:    kmovd %edi, %k1
+; AVX512BW-NEXT:    vmovdqa64 %zmm17, %zmm18 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm10
+; AVX512BW-NEXT:    vpermt2d %zmm12, %zmm13, %zmm10
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm8
+; AVX512BW-NEXT:    vpermt2d %zmm9, %zmm13, %zmm8
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3],ymm10[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm8, %zmm18, %zmm28
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm8
+; AVX512BW-NEXT:    vpermt2d %zmm7, %zmm13, %zmm8
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm10
+; AVX512BW-NEXT:    vpermt2d %zmm31, %zmm13, %zmm10
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm10 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm8
+; AVX512BW-NEXT:    vpermt2d %zmm30, %zmm13, %zmm8
+; AVX512BW-NEXT:    vpermi2d %zmm29, %zmm0, %zmm13
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm13[0,1,2,3],ymm8[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm8, %zmm10, %zmm17
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm8 = [1,9,17,25,1,9,17,25,1,9,17,25,1,9,17,25]
+; AVX512BW-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm10
+; AVX512BW-NEXT:    vpermt2d %zmm16, %zmm8, %zmm10
+; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm13
+; AVX512BW-NEXT:    vpermt2d %zmm14, %zmm8, %zmm13
+; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm13 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm10
+; AVX512BW-NEXT:    vpermt2d %zmm12, %zmm8, %zmm10
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm4
+; AVX512BW-NEXT:    vpermt2d %zmm9, %zmm8, %zmm4
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm10[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm4, %zmm13, %zmm18
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm4
+; AVX512BW-NEXT:    vpermt2d %zmm7, %zmm8, %zmm4
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm10
+; AVX512BW-NEXT:    vpermt2d %zmm31, %zmm8, %zmm10
+; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm10 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm4
+; AVX512BW-NEXT:    vpermt2d %zmm30, %zmm8, %zmm4
+; AVX512BW-NEXT:    vpermi2d %zmm29, %zmm0, %zmm8
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm8[0,1,2,3],ymm4[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm4, %zmm10, %zmm19
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm4 = [2,10,18,26,2,10,18,26,2,10,18,26,2,10,18,26]
 ; AVX512BW-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm0
-; AVX512BW-NEXT:    vpermt2d %zmm22, %zmm4, %zmm0
-; AVX512BW-NEXT:    vmovdqa64 %zmm26, %zmm1
-; AVX512BW-NEXT:    vpermt2d %zmm20, %zmm4, %zmm1
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [1,9,17,25,1,9,17,25]
-; AVX512BW-NEXT:    # ymm2 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm0
-; AVX512BW-NEXT:    vpermt2d %zmm9, %zmm2, %zmm0
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm2
-; AVX512BW-NEXT:    vpermt2d %zmm17, %zmm4, %zmm2
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm1, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [2,10,18,26,2,10,18,26,2,10,18,26,2,10,18,26]
-; AVX512BW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm1
-; AVX512BW-NEXT:    vpermt2d %zmm22, %zmm0, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm2 = [3,11,19,27,3,11,19,27,3,11,19,27,3,11,19,27]
-; AVX512BW-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm1
-; AVX512BW-NEXT:    vpermt2d %zmm22, %zmm2, %zmm1
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm10 = [4,12,20,28,4,12,20,28,4,12,20,28,4,12,20,28]
+; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm8
+; AVX512BW-NEXT:    vpermt2d %zmm16, %zmm4, %zmm8
+; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm10
+; AVX512BW-NEXT:    vpermt2d %zmm14, %zmm4, %zmm10
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm10 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm8
+; AVX512BW-NEXT:    vpermt2d %zmm12, %zmm4, %zmm8
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm13
+; AVX512BW-NEXT:    vpermt2d %zmm9, %zmm4, %zmm13
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm13[0,1,2,3],ymm8[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm8, %zmm10, %zmm20
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm8
+; AVX512BW-NEXT:    vpermt2d %zmm7, %zmm4, %zmm8
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm10
+; AVX512BW-NEXT:    vpermt2d %zmm31, %zmm4, %zmm10
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm10 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm8
+; AVX512BW-NEXT:    vpermt2d %zmm30, %zmm4, %zmm8
+; AVX512BW-NEXT:    vpermi2d %zmm29, %zmm0, %zmm4
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm8[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm4, %zmm10, %zmm21
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm4 = [3,11,19,27,3,11,19,27,3,11,19,27,3,11,19,27]
+; AVX512BW-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm8
+; AVX512BW-NEXT:    vpermt2d %zmm16, %zmm4, %zmm8
+; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm10
+; AVX512BW-NEXT:    vpermt2d %zmm14, %zmm4, %zmm10
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm10 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm8
+; AVX512BW-NEXT:    vpermt2d %zmm12, %zmm4, %zmm8
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm13
+; AVX512BW-NEXT:    vpermt2d %zmm9, %zmm4, %zmm13
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm13[0,1,2,3],ymm8[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm8, %zmm10, %zmm22
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm8
+; AVX512BW-NEXT:    vpermt2d %zmm7, %zmm4, %zmm8
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm10
+; AVX512BW-NEXT:    vpermt2d %zmm31, %zmm4, %zmm10
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm10 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm8
+; AVX512BW-NEXT:    vpermt2d %zmm30, %zmm4, %zmm8
+; AVX512BW-NEXT:    vpermi2d %zmm29, %zmm0, %zmm4
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm8[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm4, %zmm10, %zmm23
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm4 = [4,12,20,28,4,12,20,28,4,12,20,28,4,12,20,28]
+; AVX512BW-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm8
+; AVX512BW-NEXT:    vpermt2d %zmm16, %zmm4, %zmm8
+; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm10
+; AVX512BW-NEXT:    vpermt2d %zmm14, %zmm4, %zmm10
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm10 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm8
+; AVX512BW-NEXT:    vpermt2d %zmm12, %zmm4, %zmm8
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm13
+; AVX512BW-NEXT:    vpermt2d %zmm9, %zmm4, %zmm13
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm13[0,1,2,3],ymm8[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm8, %zmm10, %zmm24
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm8
+; AVX512BW-NEXT:    vpermt2d %zmm7, %zmm4, %zmm8
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm10
+; AVX512BW-NEXT:    vpermt2d %zmm31, %zmm4, %zmm10
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm10 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm8
+; AVX512BW-NEXT:    vpermt2d %zmm30, %zmm4, %zmm8
+; AVX512BW-NEXT:    vpermi2d %zmm29, %zmm0, %zmm4
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm8[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm4, %zmm10, %zmm25
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm4 = [5,13,21,29,5,13,21,29,5,13,21,29,5,13,21,29]
+; AVX512BW-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm8
+; AVX512BW-NEXT:    vpermt2d %zmm16, %zmm4, %zmm8
+; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm10
+; AVX512BW-NEXT:    vpermt2d %zmm14, %zmm4, %zmm10
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm10 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm8
+; AVX512BW-NEXT:    vpermt2d %zmm12, %zmm4, %zmm8
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm13
+; AVX512BW-NEXT:    vpermt2d %zmm9, %zmm4, %zmm13
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm13[0,1,2,3],ymm8[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm8, %zmm10, %zmm26
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm8
+; AVX512BW-NEXT:    vpermt2d %zmm7, %zmm4, %zmm8
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm10
+; AVX512BW-NEXT:    vpermt2d %zmm31, %zmm4, %zmm10
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm10 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm8
+; AVX512BW-NEXT:    vpermt2d %zmm30, %zmm4, %zmm8
+; AVX512BW-NEXT:    vpermi2d %zmm29, %zmm0, %zmm4
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm8[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm4, %zmm10, %zmm27
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm4 = [6,14,22,30,6,14,22,30,6,14,22,30,6,14,22,30]
+; AVX512BW-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm8
+; AVX512BW-NEXT:    vpermt2d %zmm16, %zmm4, %zmm8
+; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm10
+; AVX512BW-NEXT:    vpermt2d %zmm14, %zmm4, %zmm10
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm10 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm8
+; AVX512BW-NEXT:    vpermt2d %zmm12, %zmm4, %zmm8
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm13
+; AVX512BW-NEXT:    vpermt2d %zmm9, %zmm4, %zmm13
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm13[0,1,2,3],ymm8[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm8, %zmm10, %zmm8
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm10
+; AVX512BW-NEXT:    vpermt2d %zmm7, %zmm4, %zmm10
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm13
+; AVX512BW-NEXT:    vpermt2d %zmm31, %zmm4, %zmm13
+; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm13 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm10
+; AVX512BW-NEXT:    vpermt2d %zmm30, %zmm4, %zmm10
+; AVX512BW-NEXT:    vpermi2d %zmm29, %zmm0, %zmm4
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm10[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm4, %zmm13, %zmm4
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm10 = [7,15,23,31,7,15,23,31,7,15,23,31,7,15,23,31]
 ; AVX512BW-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm2
-; AVX512BW-NEXT:    vpermt2d %zmm22, %zmm10, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm15 = [5,13,21,29,5,13,21,29,5,13,21,29,5,13,21,29]
-; AVX512BW-NEXT:    # zmm15 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm1
-; AVX512BW-NEXT:    vpermt2d %zmm22, %zmm15, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm13 = [6,14,22,30,6,14,22,30,6,14,22,30,6,14,22,30]
-; AVX512BW-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm1
-; AVX512BW-NEXT:    vpermt2d %zmm22, %zmm13, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm1 = [7,15,23,31,7,15,23,31,7,15,23,31,7,15,23,31]
-; AVX512BW-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermt2d %zmm22, %zmm1, %zmm12
-; AVX512BW-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm26, %zmm24
-; AVX512BW-NEXT:    vpermt2d %zmm20, %zmm0, %zmm24
-; AVX512BW-NEXT:    vmovdqa64 %zmm26, %zmm27
-; AVX512BW-NEXT:    vpermt2d %zmm20, %zmm3, %zmm27
-; AVX512BW-NEXT:    vmovdqa64 %zmm26, %zmm31
-; AVX512BW-NEXT:    vpermt2d %zmm20, %zmm10, %zmm31
-; AVX512BW-NEXT:    vmovdqa64 %zmm26, %zmm30
-; AVX512BW-NEXT:    vpermt2d %zmm20, %zmm15, %zmm30
-; AVX512BW-NEXT:    vmovdqa64 %zmm26, %zmm22
-; AVX512BW-NEXT:    vpermt2d %zmm20, %zmm13, %zmm22
-; AVX512BW-NEXT:    vpermt2d %zmm20, %zmm1, %zmm26
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm8
-; AVX512BW-NEXT:    vpermt2d %zmm17, %zmm0, %zmm8
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm12
-; AVX512BW-NEXT:    vpermt2d %zmm17, %zmm3, %zmm12
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm6
-; AVX512BW-NEXT:    vpermt2d %zmm17, %zmm10, %zmm6
-; AVX512BW-NEXT:    vmovdqu64 %zmm6, (%rsp) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm6
-; AVX512BW-NEXT:    vpermt2d %zmm17, %zmm15, %zmm6
-; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm6
-; AVX512BW-NEXT:    vpermt2d %zmm17, %zmm13, %zmm6
-; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2d %zmm17, %zmm1, %zmm7
-; AVX512BW-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm25, %zmm18
-; AVX512BW-NEXT:    vpermt2d %zmm21, %zmm4, %zmm18
-; AVX512BW-NEXT:    vmovdqa64 %zmm25, %zmm19
-; AVX512BW-NEXT:    vpermt2d %zmm21, %zmm0, %zmm19
-; AVX512BW-NEXT:    vmovdqa64 %zmm25, %zmm23
-; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm6
-; AVX512BW-NEXT:    vpermt2d %zmm21, %zmm3, %zmm23
-; AVX512BW-NEXT:    vmovdqa64 %zmm25, %zmm16
-; AVX512BW-NEXT:    vpermt2d %zmm21, %zmm10, %zmm16
-; AVX512BW-NEXT:    vmovdqa64 %zmm25, %zmm20
-; AVX512BW-NEXT:    vpermt2d %zmm21, %zmm15, %zmm20
-; AVX512BW-NEXT:    vmovdqa64 %zmm25, %zmm17
-; AVX512BW-NEXT:    vpermt2d %zmm21, %zmm13, %zmm17
-; AVX512BW-NEXT:    vpermt2d %zmm21, %zmm1, %zmm25
-; AVX512BW-NEXT:    vmovdqa64 %zmm29, %zmm9
-; AVX512BW-NEXT:    vpermt2d %zmm5, %zmm4, %zmm9
-; AVX512BW-NEXT:    vmovdqa64 %zmm29, %zmm11
-; AVX512BW-NEXT:    vpermt2d %zmm5, %zmm0, %zmm11
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm3
-; AVX512BW-NEXT:    vmovdqa64 %zmm29, %zmm14
-; AVX512BW-NEXT:    vpermt2d %zmm5, %zmm6, %zmm14
-; AVX512BW-NEXT:    vmovdqa64 %zmm29, %zmm21
-; AVX512BW-NEXT:    vpermt2d %zmm5, %zmm10, %zmm21
-; AVX512BW-NEXT:    vmovdqa64 %zmm29, %zmm7
-; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm2
-; AVX512BW-NEXT:    vpermt2d %zmm5, %zmm15, %zmm7
-; AVX512BW-NEXT:    vmovdqa64 %zmm29, %zmm15
-; AVX512BW-NEXT:    vpermt2d %zmm5, %zmm13, %zmm15
-; AVX512BW-NEXT:    vpermt2d %zmm5, %zmm1, %zmm29
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vpermi2d %zmm28, %zmm0, %zmm4
-; AVX512BW-NEXT:    vpermi2d %zmm28, %zmm0, %zmm3
-; AVX512BW-NEXT:    vpermi2d %zmm28, %zmm0, %zmm6
-; AVX512BW-NEXT:    vpermi2d %zmm28, %zmm0, %zmm10
-; AVX512BW-NEXT:    vpermi2d %zmm28, %zmm0, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermi2d %zmm28, %zmm0, %zmm13
-; AVX512BW-NEXT:    vpermt2d %zmm28, %zmm1, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm18, %zmm9 {%k1}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm28 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = [1,9,17,25,1,9,17,25]
-; AVX512BW-NEXT:    # ymm0 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vpermi2d %zmm28, %zmm2, %zmm0
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm0[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm9, %zmm18
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm24 {%k1}
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm9 = [2,10,18,26,2,10,18,26]
-; AVX512BW-NEXT:    # ymm9 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2d %zmm5, %zmm9, %zmm0
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm8[0,1,2,3],ymm0[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm24, %zmm24
-; AVX512BW-NEXT:    vmovdqa64 %zmm19, %zmm11 {%k1}
-; AVX512BW-NEXT:    vpermi2d %zmm28, %zmm2, %zmm9
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm9[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm11, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm27 {%k1}
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [3,11,19,27,3,11,19,27]
-; AVX512BW-NEXT:    # ymm3 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm0
-; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm9
-; AVX512BW-NEXT:    vpermt2d %zmm5, %zmm3, %zmm9
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm12[0,1,2,3],ymm9[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm4, %zmm27, %zmm4
-; AVX512BW-NEXT:    vmovdqa64 %zmm23, %zmm14 {%k1}
-; AVX512BW-NEXT:    vpermi2d %zmm28, %zmm2, %zmm3
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm6[0,1,2,3],ymm3[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm3, %zmm14, %zmm3
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm6 = [4,12,20,28,4,12,20,28]
-; AVX512BW-NEXT:    # ymm6 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm8
-; AVX512BW-NEXT:    vpermt2d %zmm5, %zmm6, %zmm8
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm9 = [5,13,21,29,5,13,21,29]
-; AVX512BW-NEXT:    # ymm9 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm11
-; AVX512BW-NEXT:    vpermt2d %zmm5, %zmm9, %zmm11
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm12 = [6,14,22,30,6,14,22,30]
-; AVX512BW-NEXT:    # ymm12 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm14
-; AVX512BW-NEXT:    vpermt2d %zmm5, %zmm12, %zmm14
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} ymm19 = [7,15,23,31,7,15,23,31]
-; AVX512BW-NEXT:    # ymm19 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermt2d %zmm5, %zmm19, %zmm0
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm5
-; AVX512BW-NEXT:    vpermi2d %zmm28, %zmm2, %zmm6
-; AVX512BW-NEXT:    vpermi2d %zmm28, %zmm2, %zmm9
-; AVX512BW-NEXT:    vpermi2d %zmm28, %zmm2, %zmm12
-; AVX512BW-NEXT:    vpermt2d %zmm28, %zmm19, %zmm2
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm0
-; AVX512BW-NEXT:    vpblendd $15, (%rsp), %ymm8, %ymm8 # 32-byte Folded Reload
-; AVX512BW-NEXT:    # ymm8 = mem[0,1,2,3],ymm8[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm31 {%k1}
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm8, %zmm31, %zmm8
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm6 = ymm10[0,1,2,3],ymm6[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm21 {%k1}
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm6, %zmm21, %zmm6
-; AVX512BW-NEXT:    vpblendd $15, {{[-0-9]+}}(%r{{[sb]}}p), %ymm11, %ymm10 # 32-byte Folded Reload
-; AVX512BW-NEXT:    # ymm10 = mem[0,1,2,3],ymm11[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm30 {%k1}
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm10, %zmm30, %zmm10
-; AVX512BW-NEXT:    vpblendd $15, {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm9 # 32-byte Folded Reload
-; AVX512BW-NEXT:    # ymm9 = mem[0,1,2,3],ymm9[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqa64 %zmm20, %zmm7 {%k1}
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm9, %zmm7, %zmm7
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm22 {%k1}
-; AVX512BW-NEXT:    vpblendd $15, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm9 # 32-byte Folded Reload
-; AVX512BW-NEXT:    # ymm9 = mem[0,1,2,3],ymm14[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm9, %zmm22, %zmm9
-; AVX512BW-NEXT:    vmovdqa64 %zmm17, %zmm15 {%k1}
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm13[0,1,2,3],ymm12[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm15, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm26 {%k1}
-; AVX512BW-NEXT:    vpblendd $15, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm11 # 32-byte Folded Reload
-; AVX512BW-NEXT:    # ymm11 = mem[0,1,2,3],ymm5[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm11, %zmm26, %zmm5
-; AVX512BW-NEXT:    vmovdqa64 %zmm25, %zmm29 {%k1}
-; AVX512BW-NEXT:    vpblendd $15, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm11 # 32-byte Folded Reload
-; AVX512BW-NEXT:    # ymm11 = mem[0,1,2,3],ymm0[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm11, %zmm29, %zmm11
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm0, 64(%rsi)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm0, (%rsi)
+; AVX512BW-NEXT:    vpermt2d %zmm16, %zmm10, %zmm15
+; AVX512BW-NEXT:    vpermt2d %zmm14, %zmm10, %zmm11
+; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm11 {%k1}
+; AVX512BW-NEXT:    vpermt2d %zmm12, %zmm10, %zmm2
+; AVX512BW-NEXT:    vpermt2d %zmm9, %zmm10, %zmm5
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm5[0,1,2,3],ymm2[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm11, %zmm2
+; AVX512BW-NEXT:    vpermt2d %zmm7, %zmm10, %zmm6
+; AVX512BW-NEXT:    vpermt2d %zmm31, %zmm10, %zmm3
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm3 {%k1}
+; AVX512BW-NEXT:    vpermt2d %zmm30, %zmm10, %zmm1
+; AVX512BW-NEXT:    vpermt2d %zmm29, %zmm10, %zmm0
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm3, %zmm0
+; AVX512BW-NEXT:    vmovdqa64 %zmm28, 64(%rsi)
+; AVX512BW-NEXT:    vmovdqa64 %zmm17, (%rsi)
 ; AVX512BW-NEXT:    vmovdqa64 %zmm18, 64(%rdx)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm0, (%rdx)
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, 64(%rcx)
-; AVX512BW-NEXT:    vmovdqa64 %zmm24, (%rcx)
-; AVX512BW-NEXT:    vmovdqa64 %zmm3, 64(%r8)
-; AVX512BW-NEXT:    vmovdqa64 %zmm4, (%r8)
-; AVX512BW-NEXT:    vmovdqa64 %zmm6, 64(%r9)
-; AVX512BW-NEXT:    vmovdqa64 %zmm8, (%r9)
-; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, 64(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, (%rax)
-; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
+; AVX512BW-NEXT:    vmovdqa64 %zmm19, (%rdx)
+; AVX512BW-NEXT:    vmovdqa64 %zmm20, 64(%rcx)
+; AVX512BW-NEXT:    vmovdqa64 %zmm21, (%rcx)
+; AVX512BW-NEXT:    vmovdqa64 %zmm22, 64(%r8)
+; AVX512BW-NEXT:    vmovdqa64 %zmm23, (%r8)
+; AVX512BW-NEXT:    vmovdqa64 %zmm24, 64(%r9)
+; AVX512BW-NEXT:    vmovdqa64 %zmm25, (%r9)
+; AVX512BW-NEXT:    vmovdqa64 %zmm26, 64(%r11)
+; AVX512BW-NEXT:    vmovdqa64 %zmm27, (%r11)
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, 64(%r10)
+; AVX512BW-NEXT:    vmovdqa64 %zmm4, (%r10)
 ; AVX512BW-NEXT:    vmovdqa64 %zmm2, 64(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm9, (%rax)
-; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512BW-NEXT:    vmovdqa64 %zmm11, 64(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, (%rax)
-; AVX512BW-NEXT:    addq $1096, %rsp # imm = 0x448
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rax)
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
   %wide.vec = load <256 x i32>, ptr %in.vec, align 64
@@ -8127,1227 +7985,1195 @@ define void @load_i32_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ;
 ; AVX512F-LABEL: load_i32_stride8_vf64:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    subq $3304, %rsp # imm = 0xCE8
-; AVX512F-NEXT:    vmovdqa64 1984(%rdi), %zmm21
-; AVX512F-NEXT:    vmovdqa64 1216(%rdi), %zmm7
-; AVX512F-NEXT:    vmovaps 1152(%rdi), %zmm0
+; AVX512F-NEXT:    subq $3144, %rsp # imm = 0xC48
+; AVX512F-NEXT:    vmovdqa64 320(%rdi), %zmm11
+; AVX512F-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 448(%rdi), %zmm18
+; AVX512F-NEXT:    vmovdqa64 1600(%rdi), %zmm31
+; AVX512F-NEXT:    vmovaps 1536(%rdi), %zmm0
+; AVX512F-NEXT:    vmovups %zmm0, (%rsp) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 1728(%rdi), %zmm24
+; AVX512F-NEXT:    vmovdqu64 %zmm24, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovaps 1664(%rdi), %zmm0
 ; AVX512F-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 1088(%rdi), %zmm23
-; AVX512F-NEXT:    vmovdqa64 1024(%rdi), %zmm8
-; AVX512F-NEXT:    vmovdqa64 1344(%rdi), %zmm22
-; AVX512F-NEXT:    vmovdqa64 1280(%rdi), %zmm28
-; AVX512F-NEXT:    vmovdqa64 1472(%rdi), %zmm30
-; AVX512F-NEXT:    vmovdqa64 1408(%rdi), %zmm9
-; AVX512F-NEXT:    vmovdqa64 704(%rdi), %zmm18
-; AVX512F-NEXT:    vmovdqa64 640(%rdi), %zmm15
-; AVX512F-NEXT:    vmovdqa64 576(%rdi), %zmm11
-; AVX512F-NEXT:    vmovdqa64 512(%rdi), %zmm14
-; AVX512F-NEXT:    vmovdqa64 832(%rdi), %zmm17
-; AVX512F-NEXT:    vmovdqa64 768(%rdi), %zmm27
-; AVX512F-NEXT:    vmovdqa64 960(%rdi), %zmm19
-; AVX512F-NEXT:    vmovdqa64 896(%rdi), %zmm16
-; AVX512F-NEXT:    vmovdqa64 (%rdi), %zmm10
-; AVX512F-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 64(%rdi), %zmm13
-; AVX512F-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 128(%rdi), %zmm5
-; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 192(%rdi), %zmm6
+; AVX512F-NEXT:    vmovdqa64 1856(%rdi), %zmm21
+; AVX512F-NEXT:    vmovdqa64 1792(%rdi), %zmm26
+; AVX512F-NEXT:    vmovdqa64 1984(%rdi), %zmm22
+; AVX512F-NEXT:    vmovdqa64 1920(%rdi), %zmm5
+; AVX512F-NEXT:    vmovdqa64 1088(%rdi), %zmm14
+; AVX512F-NEXT:    vmovdqa64 1024(%rdi), %zmm3
+; AVX512F-NEXT:    vmovdqa64 1216(%rdi), %zmm19
+; AVX512F-NEXT:    vmovdqa64 1152(%rdi), %zmm2
+; AVX512F-NEXT:    vmovdqa64 1344(%rdi), %zmm13
+; AVX512F-NEXT:    vmovdqa64 1280(%rdi), %zmm27
+; AVX512F-NEXT:    vmovdqa64 1472(%rdi), %zmm20
+; AVX512F-NEXT:    vmovdqa64 1408(%rdi), %zmm10
+; AVX512F-NEXT:    vmovdqa64 576(%rdi), %zmm17
+; AVX512F-NEXT:    vmovdqa64 512(%rdi), %zmm7
+; AVX512F-NEXT:    vmovdqa64 704(%rdi), %zmm9
+; AVX512F-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 640(%rdi), %zmm12
+; AVX512F-NEXT:    vmovdqa64 832(%rdi), %zmm6
 ; AVX512F-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 320(%rdi), %zmm4
-; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 256(%rdi), %zmm20
-; AVX512F-NEXT:    vmovdqa64 448(%rdi), %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 384(%rdi), %zmm1
-; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 768(%rdi), %zmm28
+; AVX512F-NEXT:    vmovdqa64 960(%rdi), %zmm23
+; AVX512F-NEXT:    vmovdqa64 896(%rdi), %zmm4
 ; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [0,8,16,24,0,8,16,24,0,8,16,24,0,8,16,24]
 ; AVX512F-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermt2d %zmm2, %zmm0, %zmm1
-; AVX512F-NEXT:    vmovdqa64 %zmm20, %zmm2
-; AVX512F-NEXT:    vpermt2d %zmm4, %zmm0, %zmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm16
+; AVX512F-NEXT:    vpermt2d %zmm23, %zmm0, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm28, %zmm4
+; AVX512F-NEXT:    vpermt2d %zmm6, %zmm0, %zmm4
 ; AVX512F-NEXT:    movb $-64, %al
 ; AVX512F-NEXT:    kmovw %eax, %k1
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [0,8,16,24,0,8,16,24]
-; AVX512F-NEXT:    # ymm1 = mem[0,1,0,1]
-; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm4
-; AVX512F-NEXT:    vpermt2d %zmm6, %zmm1, %zmm4
-; AVX512F-NEXT:    vpermt2d %zmm13, %zmm0, %zmm10
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm4 = ymm10[0,1,2,3],ymm4[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm4, %zmm2, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm2
-; AVX512F-NEXT:    vpermt2d %zmm19, %zmm0, %zmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm19, %zmm29
-; AVX512F-NEXT:    vmovdqu64 %zmm19, (%rsp) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm27, %zmm4
-; AVX512F-NEXT:    vpermt2d %zmm17, %zmm0, %zmm4
-; AVX512F-NEXT:    vmovdqa64 %zmm17, %zmm31
-; AVX512F-NEXT:    vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm4 {%k1}
-; AVX512F-NEXT:    vmovdqa64 %zmm14, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2d %zmm11, %zmm0, %zmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm6
-; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm13
-; AVX512F-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2d %zmm18, %zmm1, %zmm6
-; AVX512F-NEXT:    vmovdqa64 %zmm18, %zmm15
-; AVX512F-NEXT:    vmovdqu64 %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm6[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm4, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm17
-; AVX512F-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm2
-; AVX512F-NEXT:    vpermt2d %zmm30, %zmm0, %zmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm28, %zmm4
-; AVX512F-NEXT:    vmovdqa64 %zmm22, %zmm19
-; AVX512F-NEXT:    vpermt2d %zmm22, %zmm0, %zmm4
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm4 {%k1}
-; AVX512F-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm2
-; AVX512F-NEXT:    vpermt2d %zmm23, %zmm0, %zmm2
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm12, %zmm6
-; AVX512F-NEXT:    vpermt2d %zmm7, %zmm1, %zmm6
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm6[4,5,6,7]
-; AVX512F-NEXT:    vmovdqa64 1920(%rdi), %zmm3
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm4, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm18
-; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2d %zmm21, %zmm0, %zmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm21, %zmm22
-; AVX512F-NEXT:    vmovdqa64 1856(%rdi), %zmm10
-; AVX512F-NEXT:    vmovdqa64 1792(%rdi), %zmm4
-; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2d %zmm10, %zmm0, %zmm4
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
+; AVX512F-NEXT:    vmovdqa64 %zmm12, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm12, %zmm15
+; AVX512F-NEXT:    vpermt2d %zmm9, %zmm0, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm12
+; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm9
+; AVX512F-NEXT:    vpermt2d %zmm17, %zmm0, %zmm12
+; AVX512F-NEXT:    vmovdqa64 %zmm17, %zmm25
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm12[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm4, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm6
 ; AVX512F-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm4 {%k1}
-; AVX512F-NEXT:    vmovdqa64 1600(%rdi), %zmm2
-; AVX512F-NEXT:    vmovdqa64 1536(%rdi), %zmm3
-; AVX512F-NEXT:    vpermi2d %zmm2, %zmm3, %zmm0
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm25
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 1728(%rdi), %zmm26
-; AVX512F-NEXT:    vmovdqa64 1664(%rdi), %zmm9
-; AVX512F-NEXT:    vpermi2d %zmm26, %zmm9, %zmm1
-; AVX512F-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2d %zmm20, %zmm0, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm27, %zmm4
+; AVX512F-NEXT:    vpermt2d %zmm13, %zmm0, %zmm4
+; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm29
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm8
+; AVX512F-NEXT:    vpermt2d %zmm19, %zmm0, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm19, %zmm30
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm10
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm12
+; AVX512F-NEXT:    vpermt2d %zmm14, %zmm0, %zmm12
+; AVX512F-NEXT:    vmovdqa64 %zmm14, %zmm13
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm12[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm4, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm14
+; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm1
+; AVX512F-NEXT:    vpermt2d %zmm22, %zmm0, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm22, %zmm17
+; AVX512F-NEXT:    vmovdqu64 %zmm22, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm26, %zmm4
+; AVX512F-NEXT:    vpermt2d %zmm21, %zmm0, %zmm4
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm1
+; AVX512F-NEXT:    vpermt2d %zmm24, %zmm0, %zmm1
+; AVX512F-NEXT:    vmovdqu64 (%rsp), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm12
+; AVX512F-NEXT:    vmovdqa64 %zmm31, %zmm24
+; AVX512F-NEXT:    vpermt2d %zmm31, %zmm0, %zmm12
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm12[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 384(%rdi), %zmm5
+; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm4, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm1
+; AVX512F-NEXT:    vpermt2d %zmm18, %zmm0, %zmm1
+; AVX512F-NEXT:    vmovdqa64 256(%rdi), %zmm22
+; AVX512F-NEXT:    vmovdqa64 %zmm22, %zmm4
+; AVX512F-NEXT:    vpermt2d %zmm11, %zmm0, %zmm4
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
+; AVX512F-NEXT:    vmovdqa64 128(%rdi), %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 192(%rdi), %zmm5
+; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2d %zmm5, %zmm0, %zmm1
+; AVX512F-NEXT:    vmovdqa64 (%rdi), %zmm7
+; AVX512F-NEXT:    vmovdqa64 64(%rdi), %zmm19
+; AVX512F-NEXT:    vpermi2d %zmm19, %zmm7, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
 ; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm4, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [1,9,17,25,1,9,17,25,1,9,17,25,1,9,17,25]
 ; AVX512F-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm17, %zmm1
-; AVX512F-NEXT:    vpermt2d %zmm30, %zmm0, %zmm1
-; AVX512F-NEXT:    vmovdqa64 %zmm28, %zmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm28, %zmm24
-; AVX512F-NEXT:    vpermt2d %zmm19, %zmm0, %zmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}
-; AVX512F-NEXT:    vpermt2d %zmm23, %zmm0, %zmm8
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [1,9,17,25,1,9,17,25]
-; AVX512F-NEXT:    # ymm1 = mem[0,1,0,1]
-; AVX512F-NEXT:    vmovdqa64 %zmm12, %zmm6
-; AVX512F-NEXT:    vpermt2d %zmm7, %zmm1, %zmm6
-; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm17
-; AVX512F-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm4 = ymm8[0,1,2,3],ymm6[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm4, %zmm2, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm8
-; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm2
-; AVX512F-NEXT:    vpermt2d %zmm29, %zmm0, %zmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm1
+; AVX512F-NEXT:    vpermt2d %zmm20, %zmm0, %zmm1
 ; AVX512F-NEXT:    vmovdqa64 %zmm27, %zmm4
-; AVX512F-NEXT:    vpermt2d %zmm31, %zmm0, %zmm4
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm4 {%k1}
-; AVX512F-NEXT:    vpermt2d %zmm11, %zmm0, %zmm14
-; AVX512F-NEXT:    vpermt2d %zmm15, %zmm1, %zmm13
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm14[0,1,2,3],ymm13[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm4, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm12, %zmm2
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2d %zmm5, %zmm0, %zmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm20, %zmm4
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm29 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 %zmm29, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vpermt2d %zmm29, %zmm0, %zmm4
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm4 {%k1}
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
+; AVX512F-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm30, %zmm31
+; AVX512F-NEXT:    vpermt2d %zmm30, %zmm0, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm12
+; AVX512F-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm30
+; AVX512F-NEXT:    vpermt2d %zmm13, %zmm0, %zmm12
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm12[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm4, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm1
+; AVX512F-NEXT:    vpermt2d %zmm23, %zmm0, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm28, %zmm4
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2d %zmm5, %zmm0, %zmm4
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
+; AVX512F-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm1
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm2
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm31 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2d %zmm31, %zmm1, %zmm2
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm6
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2d %zmm16, %zmm0, %zmm6
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm6[0,1,2,3],ymm2[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm4, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm18, %zmm2
-; AVX512F-NEXT:    vpermt2d %zmm21, %zmm0, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm21 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm21, %zmm4
-; AVX512F-NEXT:    vpermt2d %zmm10, %zmm0, %zmm4
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm4 {%k1}
-; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermi2d %zmm25, %zmm3, %zmm0
-; AVX512F-NEXT:    vmovdqa64 %zmm26, %zmm10
-; AVX512F-NEXT:    vmovdqu64 %zmm26, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermi2d %zmm26, %zmm9, %zmm1
+; AVX512F-NEXT:    vpermt2d %zmm13, %zmm0, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm12
+; AVX512F-NEXT:    vpermt2d %zmm25, %zmm0, %zmm12
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm12[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm4, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm14, %zmm1
+; AVX512F-NEXT:    vpermt2d %zmm17, %zmm0, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm26, %zmm4
+; AVX512F-NEXT:    vpermt2d %zmm21, %zmm0, %zmm4
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm1
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2d %zmm3, %zmm0, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm12
+; AVX512F-NEXT:    vpermt2d %zmm24, %zmm0, %zmm12
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm12[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm4, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm1
+; AVX512F-NEXT:    vpermt2d %zmm18, %zmm0, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm22, %zmm4
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2d %zmm2, %zmm0, %zmm4
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm14, %zmm1
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2d %zmm11, %zmm0, %zmm1
+; AVX512F-NEXT:    vpermi2d %zmm19, %zmm7, %zmm0
 ; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
 ; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm4, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [2,10,18,26,2,10,18,26,2,10,18,26,2,10,18,26]
 ; AVX512F-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm28 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm28, %zmm1
-; AVX512F-NEXT:    vpermt2d %zmm30, %zmm0, %zmm1
-; AVX512F-NEXT:    vmovdqa64 %zmm24, %zmm18
-; AVX512F-NEXT:    vmovdqu64 %zmm24, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm24, %zmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm19, %zmm26
-; AVX512F-NEXT:    vpermt2d %zmm19, %zmm0, %zmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm4
-; AVX512F-NEXT:    vmovdqa64 %zmm23, %zmm14
-; AVX512F-NEXT:    vmovdqu64 %zmm23, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2d %zmm23, %zmm0, %zmm4
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [2,10,18,26,2,10,18,26]
-; AVX512F-NEXT:    # ymm1 = mem[0,1,0,1]
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm6
-; AVX512F-NEXT:    vpermt2d %zmm17, %zmm1, %zmm6
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm6[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm4, %zmm2, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm24
-; AVX512F-NEXT:    vmovdqu64 (%rsp), %zmm19 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2d %zmm19, %zmm0, %zmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm27, %zmm4
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2d %zmm17, %zmm0, %zmm4
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm4 {%k1}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2d %zmm11, %zmm0, %zmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm23
-; AVX512F-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm25 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2d %zmm25, %zmm1, %zmm6
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm6[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm4, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm12, %zmm2
-; AVX512F-NEXT:    vpermt2d %zmm5, %zmm0, %zmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm11
-; AVX512F-NEXT:    vmovdqa64 %zmm20, %zmm4
-; AVX512F-NEXT:    vmovdqa64 %zmm29, %zmm5
+; AVX512F-NEXT:    vmovdqa64 %zmm17, %zmm1
+; AVX512F-NEXT:    vpermt2d %zmm20, %zmm0, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm27, %zmm4
 ; AVX512F-NEXT:    vpermt2d %zmm29, %zmm0, %zmm4
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm4 {%k1}
-; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm8
-; AVX512F-NEXT:    vpermt2d %zmm31, %zmm1, %zmm2
-; AVX512F-NEXT:    vpermt2d %zmm16, %zmm0, %zmm7
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm7[0,1,2,3],ymm2[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm4, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm2
-; AVX512F-NEXT:    vpermt2d %zmm22, %zmm0, %zmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm21, %zmm4
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2d %zmm12, %zmm0, %zmm4
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm4 {%k1}
-; AVX512F-NEXT:    vpermi2d {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm0 # 64-byte Folded Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-NEXT:    vpermi2d %zmm10, %zmm3, %zmm1
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm4, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
+; AVX512F-NEXT:    vpermt2d %zmm31, %zmm0, %zmm8
+; AVX512F-NEXT:    vpermt2d %zmm30, %zmm0, %zmm10
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm10[0,1,2,3],ymm8[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm4, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm1
+; AVX512F-NEXT:    vpermt2d %zmm23, %zmm0, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm28, %zmm4
+; AVX512F-NEXT:    vpermt2d %zmm5, %zmm0, %zmm4
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
+; AVX512F-NEXT:    vpermt2d %zmm13, %zmm0, %zmm15
+; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm16
+; AVX512F-NEXT:    vpermt2d %zmm25, %zmm0, %zmm9
+; AVX512F-NEXT:    vmovdqa64 %zmm25, %zmm29
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm9[0,1,2,3],ymm15[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm4, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm1
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2d %zmm10, %zmm0, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm26, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm26, %zmm4
+; AVX512F-NEXT:    vmovdqa64 %zmm21, %zmm7
+; AVX512F-NEXT:    vpermt2d %zmm21, %zmm0, %zmm4
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm25
+; AVX512F-NEXT:    vpermt2d %zmm3, %zmm0, %zmm1
+; AVX512F-NEXT:    vmovdqu64 (%rsp), %zmm3 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm12
+; AVX512F-NEXT:    vmovdqa64 %zmm24, %zmm15
+; AVX512F-NEXT:    vpermt2d %zmm24, %zmm0, %zmm12
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm12[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm4, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm13
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm1
+; AVX512F-NEXT:    vpermt2d %zmm18, %zmm0, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm22, %zmm4
+; AVX512F-NEXT:    vpermt2d %zmm2, %zmm0, %zmm4
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
+; AVX512F-NEXT:    vmovdqa64 %zmm14, %zmm6
+; AVX512F-NEXT:    vmovdqa64 %zmm14, %zmm1
+; AVX512F-NEXT:    vpermt2d %zmm11, %zmm0, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm24
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
+; AVX512F-NEXT:    vpermi2d %zmm19, %zmm14, %zmm0
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm4, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [3,11,19,27,3,11,19,27,3,11,19,27,3,11,19,27]
 ; AVX512F-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm28, %zmm1
-; AVX512F-NEXT:    vpermt2d %zmm30, %zmm0, %zmm1
-; AVX512F-NEXT:    vpermt2d %zmm26, %zmm0, %zmm18
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm18 {%k1}
-; AVX512F-NEXT:    vpermt2d %zmm14, %zmm0, %zmm15
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [3,11,19,27,3,11,19,27]
-; AVX512F-NEXT:    # ymm1 = mem[0,1,0,1]
-; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm6
-; AVX512F-NEXT:    vpermt2d {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm6 # 64-byte Folded Reload
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm4 = ymm15[0,1,2,3],ymm6[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm4, %zmm18, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm24, %zmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm24, %zmm29
-; AVX512F-NEXT:    vmovdqu64 %zmm24, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2d %zmm19, %zmm0, %zmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm19, %zmm24
+; AVX512F-NEXT:    vmovdqa64 %zmm17, %zmm1
+; AVX512F-NEXT:    vpermt2d %zmm20, %zmm0, %zmm1
 ; AVX512F-NEXT:    vmovdqa64 %zmm27, %zmm4
-; AVX512F-NEXT:    vmovdqa64 %zmm27, %zmm16
-; AVX512F-NEXT:    vmovdqu64 %zmm27, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2d %zmm17, %zmm0, %zmm4
-; AVX512F-NEXT:    vmovdqa64 %zmm17, %zmm28
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm4 {%k1}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm2
-; AVX512F-NEXT:    vpermt2d %zmm23, %zmm0, %zmm2
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm14, %zmm6
-; AVX512F-NEXT:    vmovdqa64 %zmm25, %zmm15
-; AVX512F-NEXT:    vpermt2d %zmm25, %zmm1, %zmm6
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm6[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm4, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm23 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm23, %zmm2
-; AVX512F-NEXT:    vpermt2d %zmm11, %zmm0, %zmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm27
-; AVX512F-NEXT:    vmovdqa64 %zmm20, %zmm4
-; AVX512F-NEXT:    vmovdqa64 %zmm20, %zmm22
-; AVX512F-NEXT:    vmovdqu64 %zmm20, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2d %zmm2, %zmm0, %zmm4
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm31, %zmm21
+; AVX512F-NEXT:    vpermt2d %zmm31, %zmm0, %zmm1
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2d %zmm30, %zmm0, %zmm12
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm12[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm4, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm23, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2d %zmm23, %zmm0, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm28, %zmm4
 ; AVX512F-NEXT:    vpermt2d %zmm5, %zmm0, %zmm4
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm4 {%k1}
-; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm11
-; AVX512F-NEXT:    vpermt2d %zmm31, %zmm1, %zmm2
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm6
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm20 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2d %zmm20, %zmm0, %zmm6
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm6[0,1,2,3],ymm2[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm4, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm17
-; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm2
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2d %zmm18, %zmm0, %zmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm21, %zmm4
-; AVX512F-NEXT:    vmovdqa64 %zmm12, %zmm21
-; AVX512F-NEXT:    vpermt2d %zmm12, %zmm0, %zmm4
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm4 {%k1}
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm31
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2d %zmm16, %zmm0, %zmm1
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512F-NEXT:    vpermi2d %zmm12, %zmm7, %zmm0
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
-; AVX512F-NEXT:    vpermi2d %zmm8, %zmm3, %zmm1
+; AVX512F-NEXT:    vpermt2d %zmm29, %zmm0, %zmm12
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm12[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm4, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm1
+; AVX512F-NEXT:    vpermt2d %zmm10, %zmm0, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm26, %zmm4
+; AVX512F-NEXT:    vpermt2d %zmm7, %zmm0, %zmm4
+; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm26
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
+; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm25, %zmm9
+; AVX512F-NEXT:    vpermt2d %zmm25, %zmm0, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm12
+; AVX512F-NEXT:    vpermt2d %zmm15, %zmm0, %zmm12
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm12[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm4, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm1
+; AVX512F-NEXT:    vpermt2d %zmm18, %zmm0, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm18, %zmm25
+; AVX512F-NEXT:    vmovdqa64 %zmm22, %zmm4
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2d %zmm5, %zmm0, %zmm4
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm24, %zmm18
+; AVX512F-NEXT:    vpermt2d %zmm24, %zmm0, %zmm1
+; AVX512F-NEXT:    vpermi2d %zmm19, %zmm14, %zmm0
+; AVX512F-NEXT:    vmovdqa64 %zmm19, %zmm24
+; AVX512F-NEXT:    vmovdqa64 %zmm14, %zmm13
 ; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
 ; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm4, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [4,12,20,28,4,12,20,28,4,12,20,28,4,12,20,28]
 ; AVX512F-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm17, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm20, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2d %zmm20, %zmm0, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm27, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm27, %zmm4
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
+; AVX512F-NEXT:    vpermt2d %zmm2, %zmm0, %zmm4
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm21, %zmm6
+; AVX512F-NEXT:    vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2d %zmm21, %zmm0, %zmm1
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
 ; AVX512F-NEXT:    vmovdqu64 %zmm30, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2d %zmm30, %zmm0, %zmm1
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm25 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm25, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm26, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2d %zmm26, %zmm0, %zmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm4
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2d %zmm19, %zmm0, %zmm4
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [4,12,20,28,4,12,20,28]
-; AVX512F-NEXT:    # ymm1 = mem[0,1,0,1]
-; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm6
+; AVX512F-NEXT:    vpermt2d %zmm30, %zmm0, %zmm12
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm12[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm4, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm1
+; AVX512F-NEXT:    vpermt2d %zmm23, %zmm0, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm28, %zmm4
+; AVX512F-NEXT:    vpermt2d %zmm31, %zmm0, %zmm4
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm1
+; AVX512F-NEXT:    vpermt2d %zmm16, %zmm0, %zmm1
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm12
+; AVX512F-NEXT:    vmovdqa64 %zmm29, %zmm11
+; AVX512F-NEXT:    vpermt2d %zmm29, %zmm0, %zmm12
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm12[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm4, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm23 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm23, %zmm1
+; AVX512F-NEXT:    vpermt2d %zmm10, %zmm0, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm31
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm4
+; AVX512F-NEXT:    vmovdqa64 %zmm26, %zmm19
+; AVX512F-NEXT:    vpermt2d %zmm26, %zmm0, %zmm4
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2d %zmm9, %zmm0, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm26
+; AVX512F-NEXT:    vmovdqu64 (%rsp), %zmm12 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2d %zmm15, %zmm0, %zmm12
+; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm29
+; AVX512F-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm12[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm4, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm21 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm21, %zmm1
+; AVX512F-NEXT:    vpermt2d %zmm25, %zmm0, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm22, %zmm4
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm14
+; AVX512F-NEXT:    vpermt2d %zmm5, %zmm0, %zmm4
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2d %zmm9, %zmm1, %zmm6
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm6[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm4, %zmm2, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2d %zmm24, %zmm0, %zmm29
-; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm4
-; AVX512F-NEXT:    vpermt2d %zmm28, %zmm0, %zmm4
-; AVX512F-NEXT:    vmovdqa64 %zmm29, %zmm4 {%k1}
-; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm1
+; AVX512F-NEXT:    vpermt2d %zmm18, %zmm0, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm5
+; AVX512F-NEXT:    vmovdqa64 %zmm24, %zmm13
+; AVX512F-NEXT:    vmovdqu64 %zmm24, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermi2d %zmm24, %zmm5, %zmm0
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm4, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [5,13,21,29,5,13,21,29,5,13,21,29,5,13,21,29]
+; AVX512F-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpermt2d %zmm20, %zmm0, %zmm17
+; AVX512F-NEXT:    vpermt2d %zmm3, %zmm0, %zmm27
+; AVX512F-NEXT:    vmovdqa64 %zmm17, %zmm27 {%k1}
+; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm1
+; AVX512F-NEXT:    vpermt2d %zmm6, %zmm0, %zmm1
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2d %zmm30, %zmm0, %zmm12
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm12[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm27, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm24 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2d %zmm24, %zmm0, %zmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm14, %zmm6
-; AVX512F-NEXT:    vpermt2d %zmm15, %zmm1, %zmm6
-; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm16
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm6[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm4, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm23, %zmm2
-; AVX512F-NEXT:    vpermt2d %zmm27, %zmm0, %zmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm27, %zmm15
+; AVX512F-NEXT:    vmovdqa64 %zmm24, %zmm1
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2d %zmm3, %zmm0, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm28, %zmm4
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2d %zmm7, %zmm0, %zmm4
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm15
+; AVX512F-NEXT:    vpermt2d %zmm16, %zmm0, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm8
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm12
+; AVX512F-NEXT:    vpermt2d %zmm11, %zmm0, %zmm12
+; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm2
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm12[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm4, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm23, %zmm1
+; AVX512F-NEXT:    vpermt2d %zmm31, %zmm0, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm31, %zmm16
+; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm4
+; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm27
+; AVX512F-NEXT:    vpermt2d %zmm19, %zmm0, %zmm4
+; AVX512F-NEXT:    vmovdqa64 %zmm19, %zmm20
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm1
+; AVX512F-NEXT:    vpermt2d %zmm26, %zmm0, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm26, %zmm17
+; AVX512F-NEXT:    vmovdqu64 (%rsp), %zmm6 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm12
+; AVX512F-NEXT:    vpermt2d %zmm29, %zmm0, %zmm12
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm12[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm4, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm21, %zmm1
+; AVX512F-NEXT:    vpermt2d %zmm25, %zmm0, %zmm1
 ; AVX512F-NEXT:    vmovdqa64 %zmm22, %zmm4
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm27 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2d %zmm27, %zmm0, %zmm4
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm4 {%k1}
-; AVX512F-NEXT:    vpermt2d %zmm31, %zmm1, %zmm11
-; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm6
-; AVX512F-NEXT:    vpermt2d %zmm20, %zmm0, %zmm6
-; AVX512F-NEXT:    vmovdqa64 %zmm20, %zmm22
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm6[0,1,2,3],ymm11[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm4, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm17, %zmm2
-; AVX512F-NEXT:    vpermt2d %zmm18, %zmm0, %zmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm18, %zmm6
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm29 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm29, %zmm4
-; AVX512F-NEXT:    vpermt2d %zmm21, %zmm0, %zmm4
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm4 {%k1}
-; AVX512F-NEXT:    vpermi2d %zmm12, %zmm7, %zmm0
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vpermi2d %zmm8, %zmm2, %zmm1
+; AVX512F-NEXT:    vpermt2d %zmm14, %zmm0, %zmm4
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
+; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm1
+; AVX512F-NEXT:    vpermt2d %zmm18, %zmm0, %zmm1
+; AVX512F-NEXT:    vpermi2d %zmm13, %zmm5, %zmm0
 ; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
 ; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm4, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm7 = [5,13,21,29,5,13,21,29,5,13,21,29,5,13,21,29]
-; AVX512F-NEXT:    # zmm7 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm0
-; AVX512F-NEXT:    vpermt2d %zmm30, %zmm7, %zmm0
-; AVX512F-NEXT:    vmovdqa64 %zmm25, %zmm1
-; AVX512F-NEXT:    vmovdqa64 %zmm25, %zmm12
-; AVX512F-NEXT:    vpermt2d %zmm26, %zmm7, %zmm1
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm0
-; AVX512F-NEXT:    vpermt2d %zmm19, %zmm7, %zmm0
-; AVX512F-NEXT:    vmovdqa64 %zmm19, %zmm26
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [5,13,21,29,5,13,21,29]
-; AVX512F-NEXT:    # ymm2 = mem[0,1,0,1]
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2d %zmm9, %zmm2, %zmm4
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm4[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm1, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm0
-; AVX512F-NEXT:    vpermt2d %zmm24, %zmm7, %zmm0
-; AVX512F-NEXT:    vpermt2d %zmm16, %zmm2, %zmm14
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm14[4,5,6,7]
-; AVX512F-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm21 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm21, %zmm9
-; AVX512F-NEXT:    vmovdqu64 (%rsp), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2d %zmm0, %zmm7, %zmm9
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm10 = [6,14,22,30,6,14,22,30,6,14,22,30,6,14,22,30]
-; AVX512F-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm21, %zmm30
-; AVX512F-NEXT:    vpermt2d %zmm0, %zmm10, %zmm30
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm2 = [7,15,23,31,7,15,23,31,7,15,23,31,7,15,23,31]
-; AVX512F-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermt2d %zmm0, %zmm2, %zmm21
-; AVX512F-NEXT:    vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm20 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm20, %zmm25
-; AVX512F-NEXT:    vpermt2d %zmm28, %zmm7, %zmm25
-; AVX512F-NEXT:    vmovdqa64 %zmm20, %zmm31
-; AVX512F-NEXT:    vpermt2d %zmm28, %zmm10, %zmm31
-; AVX512F-NEXT:    vpermt2d %zmm28, %zmm2, %zmm20
-; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm4
-; AVX512F-NEXT:    vpermt2d %zmm24, %zmm10, %zmm4
-; AVX512F-NEXT:    vmovdqu64 %zmm4, (%rsp) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2d %zmm24, %zmm2, %zmm13
-; AVX512F-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm23, %zmm19
-; AVX512F-NEXT:    vmovdqa64 %zmm23, %zmm18
-; AVX512F-NEXT:    vpermt2d %zmm15, %zmm7, %zmm18
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm1 = [6,14,22,30,6,14,22,30,6,14,22,30,6,14,22,30]
+; AVX512F-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm24, %zmm4
+; AVX512F-NEXT:    vmovdqa64 %zmm24, %zmm30
+; AVX512F-NEXT:    vpermt2d %zmm3, %zmm1, %zmm30
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [7,15,23,31,7,15,23,31,7,15,23,31,7,15,23,31]
+; AVX512F-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpermt2d %zmm3, %zmm0, %zmm4
+; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm19
+; AVX512F-NEXT:    vmovdqa64 %zmm28, %zmm11
+; AVX512F-NEXT:    vpermt2d %zmm7, %zmm1, %zmm11
+; AVX512F-NEXT:    vpermt2d %zmm7, %zmm0, %zmm28
+; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm9
+; AVX512F-NEXT:    vpermt2d %zmm8, %zmm1, %zmm9
+; AVX512F-NEXT:    vpermt2d %zmm8, %zmm0, %zmm15
+; AVX512F-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm8
+; AVX512F-NEXT:    vpermt2d %zmm2, %zmm1, %zmm8
+; AVX512F-NEXT:    vpermt2d %zmm2, %zmm0, %zmm3
+; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm23, %zmm31
+; AVX512F-NEXT:    vpermt2d %zmm16, %zmm1, %zmm31
+; AVX512F-NEXT:    vpermt2d %zmm16, %zmm0, %zmm23
 ; AVX512F-NEXT:    vmovdqa64 %zmm23, %zmm16
-; AVX512F-NEXT:    vpermt2d %zmm15, %zmm10, %zmm16
-; AVX512F-NEXT:    vpermt2d %zmm15, %zmm2, %zmm19
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm23 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm23, %zmm11
-; AVX512F-NEXT:    vpermt2d %zmm27, %zmm7, %zmm11
-; AVX512F-NEXT:    vmovdqa64 %zmm23, %zmm17
-; AVX512F-NEXT:    vpermt2d %zmm27, %zmm10, %zmm17
-; AVX512F-NEXT:    vpermt2d %zmm27, %zmm2, %zmm23
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm14
-; AVX512F-NEXT:    vpermt2d %zmm22, %zmm7, %zmm14
-; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm1
-; AVX512F-NEXT:    vpermt2d %zmm22, %zmm10, %zmm1
-; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2d %zmm22, %zmm2, %zmm13
-; AVX512F-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm21
-; AVX512F-NEXT:    vpermt2d %zmm6, %zmm7, %zmm21
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm28
-; AVX512F-NEXT:    vpermt2d %zmm6, %zmm10, %zmm28
-; AVX512F-NEXT:    vpermt2d %zmm6, %zmm2, %zmm1
-; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm29, %zmm1
-; AVX512F-NEXT:    vmovdqa64 %zmm29, %zmm22
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2d %zmm0, %zmm7, %zmm22
-; AVX512F-NEXT:    vmovdqa64 %zmm29, %zmm24
-; AVX512F-NEXT:    vpermt2d %zmm0, %zmm10, %zmm24
-; AVX512F-NEXT:    vpermt2d %zmm0, %zmm2, %zmm1
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm29
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vpermi2d %zmm1, %zmm0, %zmm7
-; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm27
-; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm15
+; AVX512F-NEXT:    vmovdqa64 %zmm27, %zmm29
+; AVX512F-NEXT:    vpermt2d %zmm20, %zmm1, %zmm29
+; AVX512F-NEXT:    vpermt2d %zmm20, %zmm0, %zmm27
+; AVX512F-NEXT:    vmovdqa64 %zmm27, %zmm26
+; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm5
+; AVX512F-NEXT:    vpermt2d %zmm17, %zmm1, %zmm5
+; AVX512F-NEXT:    vpermt2d %zmm17, %zmm0, %zmm2
+; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm4
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2d %zmm2, %zmm1, %zmm4
+; AVX512F-NEXT:    vpermt2d %zmm2, %zmm0, %zmm6
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm15
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2d %zmm3, %zmm10, %zmm15
-; AVX512F-NEXT:    vmovdqa64 %zmm12, %zmm13
-; AVX512F-NEXT:    vmovdqa64 %zmm12, %zmm5
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2d %zmm4, %zmm10, %zmm5
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm12, %zmm8
-; AVX512F-NEXT:    vmovdqa64 %zmm26, %zmm6
-; AVX512F-NEXT:    vpermt2d %zmm26, %zmm10, %zmm8
-; AVX512F-NEXT:    vpermi2d %zmm1, %zmm0, %zmm10
-; AVX512F-NEXT:    vpermt2d %zmm1, %zmm2, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2d %zmm3, %zmm2, %zmm27
-; AVX512F-NEXT:    vpermt2d %zmm4, %zmm2, %zmm13
-; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm26
-; AVX512F-NEXT:    vpermt2d %zmm6, %zmm2, %zmm12
-; AVX512F-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm25 {%k1}
-; AVX512F-NEXT:    vinserti64x4 $0, {{[-0-9]+}}(%r{{[sb]}}p), %zmm25, %zmm25 # 32-byte Folded Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm18, %zmm11 {%k1}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm6
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = [5,13,21,29,5,13,21,29]
-; AVX512F-NEXT:    # ymm0 = mem[0,1,0,1]
-; AVX512F-NEXT:    vpermt2d %zmm4, %zmm0, %zmm6
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm3 = ymm14[0,1,2,3],ymm6[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm3, %zmm11, %zmm18
-; AVX512F-NEXT:    vmovdqa64 %zmm21, %zmm22 {%k1}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
-; AVX512F-NEXT:    vpermi2d %zmm14, %zmm1, %zmm0
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm7[0,1,2,3],ymm0[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm22, %zmm21
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [6,14,22,30,6,14,22,30]
-; AVX512F-NEXT:    # ymm3 = mem[0,1,0,1]
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm12, %zmm7
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm23
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2d %zmm2, %zmm3, %zmm7
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm11 = [7,15,23,31,7,15,23,31]
-; AVX512F-NEXT:    # ymm11 = mem[0,1,0,1]
-; AVX512F-NEXT:    vpermt2d %zmm2, %zmm11, %zmm12
-; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm0
-; AVX512F-NEXT:    vpermt2d %zmm4, %zmm3, %zmm0
-; AVX512F-NEXT:    vpermt2d %zmm4, %zmm11, %zmm9
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm2
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm22 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2d %zmm22, %zmm3, %zmm2
-; AVX512F-NEXT:    vpermi2d %zmm14, %zmm1, %zmm3
-; AVX512F-NEXT:    vpermt2d %zmm14, %zmm11, %zmm1
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm14
-; AVX512F-NEXT:    vpermt2d %zmm22, %zmm11, %zmm6
-; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm11
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm6 = ymm8[0,1,2,3],ymm2[4,5,6,7]
-; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm5 {%k1}
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm6, %zmm5, %zmm6
-; AVX512F-NEXT:    vpblendd $15, (%rsp), %ymm7, %ymm7 # 32-byte Folded Reload
-; AVX512F-NEXT:    # ymm7 = mem[0,1,2,3],ymm7[4,5,6,7]
-; AVX512F-NEXT:    vmovdqa64 %zmm30, %zmm31 {%k1}
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm7, %zmm31, %zmm13
-; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm17 {%k1}
-; AVX512F-NEXT:    vpblendd $15, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
-; AVX512F-NEXT:    # ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm17, %zmm0
-; AVX512F-NEXT:    vmovdqa64 %zmm28, %zmm24 {%k1}
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm10[0,1,2,3],ymm3[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm24, %zmm1
+; AVX512F-NEXT:    vpermt2d %zmm2, %zmm1, %zmm23
+; AVX512F-NEXT:    vpermt2d %zmm2, %zmm0, %zmm3
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm17
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm24
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm20 {%k1}
-; AVX512F-NEXT:    vpblendd $15, {{[-0-9]+}}(%r{{[sb]}}p), %ymm12, %ymm3 # 32-byte Folded Reload
-; AVX512F-NEXT:    # ymm3 = mem[0,1,2,3],ymm12[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm3, %zmm20, %zmm3
-; AVX512F-NEXT:    vmovdqa64 %zmm19, %zmm23 {%k1}
-; AVX512F-NEXT:    vpblendd $15, {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm4 # 32-byte Folded Reload
-; AVX512F-NEXT:    # ymm4 = mem[0,1,2,3],ymm9[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm4, %zmm23, %zmm4
+; AVX512F-NEXT:    vpermt2d %zmm2, %zmm1, %zmm24
+; AVX512F-NEXT:    vpermt2d %zmm2, %zmm0, %zmm3
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm27
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm12, %zmm13
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2d %zmm3, %zmm1, %zmm13
+; AVX512F-NEXT:    vpermt2d %zmm3, %zmm0, %zmm12
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm3
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2d %zmm6, %zmm1, %zmm3
+; AVX512F-NEXT:    vpermt2d %zmm6, %zmm0, %zmm10
+; AVX512F-NEXT:    vmovdqa64 %zmm21, %zmm6
+; AVX512F-NEXT:    vmovdqa64 %zmm21, %zmm20
+; AVX512F-NEXT:    vpermt2d %zmm25, %zmm1, %zmm20
+; AVX512F-NEXT:    vpermt2d %zmm25, %zmm0, %zmm6
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm21
+; AVX512F-NEXT:    vmovdqa64 %zmm22, %zmm25
+; AVX512F-NEXT:    vpermt2d %zmm14, %zmm1, %zmm25
+; AVX512F-NEXT:    vpermt2d %zmm14, %zmm0, %zmm22
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm14
+; AVX512F-NEXT:    vpermt2d %zmm18, %zmm1, %zmm14
+; AVX512F-NEXT:    vpermt2d %zmm18, %zmm0, %zmm7
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm29 {%k1}
-; AVX512F-NEXT:    vpblendd $15, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm7 # 32-byte Folded Reload
-; AVX512F-NEXT:    # ymm7 = mem[0,1,2,3],ymm14[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm7, %zmm29, %zmm7
-; AVX512F-NEXT:    vmovdqa64 %zmm27, %zmm26 {%k1}
-; AVX512F-NEXT:    vpblendd $15, {{[-0-9]+}}(%r{{[sb]}}p), %ymm11, %ymm8 # 32-byte Folded Reload
-; AVX512F-NEXT:    # ymm8 = mem[0,1,2,3],ymm11[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm8, %zmm26, %zmm8
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm2, 192(%rsi)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm2, 128(%rsi)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm2, 64(%rsi)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm2, (%rsi)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm2, 192(%rdx)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm2, (%rdx)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm2, 64(%rdx)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm2, 128(%rdx)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm2, 192(%rcx)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm2, (%rcx)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm2, 64(%rcx)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm2, 128(%rcx)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm2, 192(%r8)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm2, (%r8)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm2, 64(%r8)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm2, 128(%r8)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm2, 192(%r9)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm2, (%r9)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm2, 64(%r9)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm2, 128(%r9)
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512F-NEXT:    vpermi2d %zmm6, %zmm2, %zmm1
+; AVX512F-NEXT:    vpermt2d %zmm6, %zmm0, %zmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm6
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3],ymm13[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 %zmm23, %zmm24 {%k1}
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm24, %zmm0
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm8[0,1,2,3],ymm9[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 %zmm30, %zmm11 {%k1}
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm11, %zmm2
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm5[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 %zmm31, %zmm29 {%k1}
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm3, %zmm29, %zmm3
+; AVX512F-NEXT:    vmovdqa64 %zmm20, %zmm25 {%k1}
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm14[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm25, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm19, %zmm28 {%k1}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512F-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm4 # 32-byte Folded Reload
+; AVX512F-NEXT:    # ymm4 = ymm4[0,1,2,3],mem[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm4, %zmm28, %zmm4
+; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm26 {%k1}
+; AVX512F-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm15, %ymm5 # 32-byte Folded Reload
+; AVX512F-NEXT:    # ymm5 = ymm15[0,1,2,3],mem[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm5, %zmm26, %zmm5
+; AVX512F-NEXT:    vmovdqa64 %zmm17, %zmm27 {%k1}
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm8 = ymm10[0,1,2,3],ymm12[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm8, %zmm27, %zmm9
+; AVX512F-NEXT:    vmovdqa64 %zmm21, %zmm22 {%k1}
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm7 = ymm6[0,1,2,3],ymm7[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm7, %zmm22, %zmm7
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm8, 192(%rsi)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm8, 128(%rsi)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm8, 64(%rsi)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm6, (%rsi)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm6, 192(%rdx)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm6, (%rdx)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm6, 64(%rdx)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm6, 128(%rdx)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm6, 192(%rcx)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm6, (%rcx)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm6, 64(%rcx)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm6, 128(%rcx)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm6, 192(%r8)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm6, (%r8)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm6, 64(%r8)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm6, 128(%r8)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm6, 192(%r9)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm6, (%r9)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm6, 64(%r9)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm6, 128(%r9)
 ; AVX512F-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512F-NEXT:    vmovdqa64 %zmm21, 192(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm18, (%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm25, 64(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm2, 128(%rax)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm6, 192(%rax)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm6, (%rax)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm6, 64(%rax)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm6, 128(%rax)
 ; AVX512F-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512F-NEXT:    vmovdqa64 %zmm1, 192(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm0, (%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm13, 64(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm6, 128(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm3, 192(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm1, (%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm2, 64(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm0, 128(%rax)
 ; AVX512F-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512F-NEXT:    vmovdqa64 %zmm8, 128(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm7, 192(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm4, (%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm3, 64(%rax)
-; AVX512F-NEXT:    addq $3304, %rsp # imm = 0xCE8
+; AVX512F-NEXT:    vmovdqa64 %zmm9, 128(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm5, 192(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm7, (%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm4, 64(%rax)
+; AVX512F-NEXT:    addq $3144, %rsp # imm = 0xC48
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: load_i32_stride8_vf64:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    subq $3304, %rsp # imm = 0xCE8
-; AVX512BW-NEXT:    vmovdqa64 1984(%rdi), %zmm21
-; AVX512BW-NEXT:    vmovdqa64 1216(%rdi), %zmm7
-; AVX512BW-NEXT:    vmovaps 1152(%rdi), %zmm0
+; AVX512BW-NEXT:    subq $3144, %rsp # imm = 0xC48
+; AVX512BW-NEXT:    vmovdqa64 320(%rdi), %zmm11
+; AVX512BW-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 448(%rdi), %zmm18
+; AVX512BW-NEXT:    vmovdqa64 1600(%rdi), %zmm31
+; AVX512BW-NEXT:    vmovaps 1536(%rdi), %zmm0
+; AVX512BW-NEXT:    vmovups %zmm0, (%rsp) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 1728(%rdi), %zmm24
+; AVX512BW-NEXT:    vmovdqu64 %zmm24, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovaps 1664(%rdi), %zmm0
 ; AVX512BW-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 1088(%rdi), %zmm23
-; AVX512BW-NEXT:    vmovdqa64 1024(%rdi), %zmm8
-; AVX512BW-NEXT:    vmovdqa64 1344(%rdi), %zmm22
-; AVX512BW-NEXT:    vmovdqa64 1280(%rdi), %zmm28
-; AVX512BW-NEXT:    vmovdqa64 1472(%rdi), %zmm30
-; AVX512BW-NEXT:    vmovdqa64 1408(%rdi), %zmm9
-; AVX512BW-NEXT:    vmovdqa64 704(%rdi), %zmm18
-; AVX512BW-NEXT:    vmovdqa64 640(%rdi), %zmm15
-; AVX512BW-NEXT:    vmovdqa64 576(%rdi), %zmm11
-; AVX512BW-NEXT:    vmovdqa64 512(%rdi), %zmm14
-; AVX512BW-NEXT:    vmovdqa64 832(%rdi), %zmm17
-; AVX512BW-NEXT:    vmovdqa64 768(%rdi), %zmm27
-; AVX512BW-NEXT:    vmovdqa64 960(%rdi), %zmm19
-; AVX512BW-NEXT:    vmovdqa64 896(%rdi), %zmm16
-; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm10
-; AVX512BW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 64(%rdi), %zmm13
-; AVX512BW-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 128(%rdi), %zmm5
-; AVX512BW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 192(%rdi), %zmm6
+; AVX512BW-NEXT:    vmovdqa64 1856(%rdi), %zmm21
+; AVX512BW-NEXT:    vmovdqa64 1792(%rdi), %zmm26
+; AVX512BW-NEXT:    vmovdqa64 1984(%rdi), %zmm22
+; AVX512BW-NEXT:    vmovdqa64 1920(%rdi), %zmm5
+; AVX512BW-NEXT:    vmovdqa64 1088(%rdi), %zmm14
+; AVX512BW-NEXT:    vmovdqa64 1024(%rdi), %zmm3
+; AVX512BW-NEXT:    vmovdqa64 1216(%rdi), %zmm19
+; AVX512BW-NEXT:    vmovdqa64 1152(%rdi), %zmm2
+; AVX512BW-NEXT:    vmovdqa64 1344(%rdi), %zmm13
+; AVX512BW-NEXT:    vmovdqa64 1280(%rdi), %zmm27
+; AVX512BW-NEXT:    vmovdqa64 1472(%rdi), %zmm20
+; AVX512BW-NEXT:    vmovdqa64 1408(%rdi), %zmm10
+; AVX512BW-NEXT:    vmovdqa64 576(%rdi), %zmm17
+; AVX512BW-NEXT:    vmovdqa64 512(%rdi), %zmm7
+; AVX512BW-NEXT:    vmovdqa64 704(%rdi), %zmm9
+; AVX512BW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 640(%rdi), %zmm12
+; AVX512BW-NEXT:    vmovdqa64 832(%rdi), %zmm6
 ; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 320(%rdi), %zmm4
-; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 256(%rdi), %zmm20
-; AVX512BW-NEXT:    vmovdqa64 448(%rdi), %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 384(%rdi), %zmm1
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 768(%rdi), %zmm28
+; AVX512BW-NEXT:    vmovdqa64 960(%rdi), %zmm23
+; AVX512BW-NEXT:    vmovdqa64 896(%rdi), %zmm4
 ; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [0,8,16,24,0,8,16,24,0,8,16,24,0,8,16,24]
 ; AVX512BW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermt2d %zmm2, %zmm0, %zmm1
-; AVX512BW-NEXT:    vmovdqa64 %zmm20, %zmm2
-; AVX512BW-NEXT:    vpermt2d %zmm4, %zmm0, %zmm2
+; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm16
+; AVX512BW-NEXT:    vpermt2d %zmm23, %zmm0, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm28, %zmm4
+; AVX512BW-NEXT:    vpermt2d %zmm6, %zmm0, %zmm4
 ; AVX512BW-NEXT:    movb $-64, %al
 ; AVX512BW-NEXT:    kmovd %eax, %k1
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [0,8,16,24,0,8,16,24]
-; AVX512BW-NEXT:    # ymm1 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm4
-; AVX512BW-NEXT:    vpermt2d %zmm6, %zmm1, %zmm4
-; AVX512BW-NEXT:    vpermt2d %zmm13, %zmm0, %zmm10
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm10[0,1,2,3],ymm4[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm4, %zmm2, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm2
-; AVX512BW-NEXT:    vpermt2d %zmm19, %zmm0, %zmm2
-; AVX512BW-NEXT:    vmovdqa64 %zmm19, %zmm29
-; AVX512BW-NEXT:    vmovdqu64 %zmm19, (%rsp) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm27, %zmm4
-; AVX512BW-NEXT:    vpermt2d %zmm17, %zmm0, %zmm4
-; AVX512BW-NEXT:    vmovdqa64 %zmm17, %zmm31
-; AVX512BW-NEXT:    vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm4 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2d %zmm11, %zmm0, %zmm2
-; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm6
-; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm13
-; AVX512BW-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2d %zmm18, %zmm1, %zmm6
-; AVX512BW-NEXT:    vmovdqa64 %zmm18, %zmm15
-; AVX512BW-NEXT:    vmovdqu64 %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm6[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm4, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm17
-; AVX512BW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm2
-; AVX512BW-NEXT:    vpermt2d %zmm30, %zmm0, %zmm2
-; AVX512BW-NEXT:    vmovdqa64 %zmm28, %zmm4
-; AVX512BW-NEXT:    vmovdqa64 %zmm22, %zmm19
-; AVX512BW-NEXT:    vpermt2d %zmm22, %zmm0, %zmm4
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm4 {%k1}
-; AVX512BW-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm2
-; AVX512BW-NEXT:    vpermt2d %zmm23, %zmm0, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm6
-; AVX512BW-NEXT:    vpermt2d %zmm7, %zmm1, %zmm6
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm6[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqa64 1920(%rdi), %zmm3
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm4, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm2
-; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm18
-; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2d %zmm21, %zmm0, %zmm2
-; AVX512BW-NEXT:    vmovdqa64 %zmm21, %zmm22
-; AVX512BW-NEXT:    vmovdqa64 1856(%rdi), %zmm10
-; AVX512BW-NEXT:    vmovdqa64 1792(%rdi), %zmm4
-; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2d %zmm10, %zmm0, %zmm4
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm15
+; AVX512BW-NEXT:    vpermt2d %zmm9, %zmm0, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm12
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm9
+; AVX512BW-NEXT:    vpermt2d %zmm17, %zmm0, %zmm12
+; AVX512BW-NEXT:    vmovdqa64 %zmm17, %zmm25
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm12[0,1,2,3],ymm1[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm4, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm6
 ; AVX512BW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm4 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 1600(%rdi), %zmm2
-; AVX512BW-NEXT:    vmovdqa64 1536(%rdi), %zmm3
-; AVX512BW-NEXT:    vpermi2d %zmm2, %zmm3, %zmm0
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm25
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 1728(%rdi), %zmm26
-; AVX512BW-NEXT:    vmovdqa64 1664(%rdi), %zmm9
-; AVX512BW-NEXT:    vpermi2d %zmm26, %zmm9, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2d %zmm20, %zmm0, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm27, %zmm4
+; AVX512BW-NEXT:    vpermt2d %zmm13, %zmm0, %zmm4
+; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm29
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm8
+; AVX512BW-NEXT:    vpermt2d %zmm19, %zmm0, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm19, %zmm30
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm10
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm12
+; AVX512BW-NEXT:    vpermt2d %zmm14, %zmm0, %zmm12
+; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm13
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm12[0,1,2,3],ymm1[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm4, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm14
+; AVX512BW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm1
+; AVX512BW-NEXT:    vpermt2d %zmm22, %zmm0, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm22, %zmm17
+; AVX512BW-NEXT:    vmovdqu64 %zmm22, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm26, %zmm4
+; AVX512BW-NEXT:    vpermt2d %zmm21, %zmm0, %zmm4
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm1
+; AVX512BW-NEXT:    vpermt2d %zmm24, %zmm0, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 (%rsp), %zmm2 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm12
+; AVX512BW-NEXT:    vmovdqa64 %zmm31, %zmm24
+; AVX512BW-NEXT:    vpermt2d %zmm31, %zmm0, %zmm12
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm12[0,1,2,3],ymm1[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 384(%rdi), %zmm5
+; AVX512BW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm4, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm1
+; AVX512BW-NEXT:    vpermt2d %zmm18, %zmm0, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 256(%rdi), %zmm22
+; AVX512BW-NEXT:    vmovdqa64 %zmm22, %zmm4
+; AVX512BW-NEXT:    vpermt2d %zmm11, %zmm0, %zmm4
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 128(%rdi), %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 192(%rdi), %zmm5
+; AVX512BW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2d %zmm5, %zmm0, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm7
+; AVX512BW-NEXT:    vmovdqa64 64(%rdi), %zmm19
+; AVX512BW-NEXT:    vpermi2d %zmm19, %zmm7, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
 ; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm4, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [1,9,17,25,1,9,17,25,1,9,17,25,1,9,17,25]
 ; AVX512BW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm17, %zmm1
-; AVX512BW-NEXT:    vpermt2d %zmm30, %zmm0, %zmm1
-; AVX512BW-NEXT:    vmovdqa64 %zmm28, %zmm2
-; AVX512BW-NEXT:    vmovdqa64 %zmm28, %zmm24
-; AVX512BW-NEXT:    vpermt2d %zmm19, %zmm0, %zmm2
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}
-; AVX512BW-NEXT:    vpermt2d %zmm23, %zmm0, %zmm8
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [1,9,17,25,1,9,17,25]
-; AVX512BW-NEXT:    # ymm1 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm6
-; AVX512BW-NEXT:    vpermt2d %zmm7, %zmm1, %zmm6
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm17
-; AVX512BW-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm8[0,1,2,3],ymm6[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm4, %zmm2, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm8
-; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm2
-; AVX512BW-NEXT:    vpermt2d %zmm29, %zmm0, %zmm2
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm1
+; AVX512BW-NEXT:    vpermt2d %zmm20, %zmm0, %zmm1
 ; AVX512BW-NEXT:    vmovdqa64 %zmm27, %zmm4
-; AVX512BW-NEXT:    vpermt2d %zmm31, %zmm0, %zmm4
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm4 {%k1}
-; AVX512BW-NEXT:    vpermt2d %zmm11, %zmm0, %zmm14
-; AVX512BW-NEXT:    vpermt2d %zmm15, %zmm1, %zmm13
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm14[0,1,2,3],ymm13[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm4, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2d %zmm5, %zmm0, %zmm2
-; AVX512BW-NEXT:    vmovdqa64 %zmm20, %zmm4
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm29 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 %zmm29, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vpermt2d %zmm29, %zmm0, %zmm4
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm4 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
+; AVX512BW-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm30, %zmm31
+; AVX512BW-NEXT:    vpermt2d %zmm30, %zmm0, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm12
+; AVX512BW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm30
+; AVX512BW-NEXT:    vpermt2d %zmm13, %zmm0, %zmm12
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm12[0,1,2,3],ymm1[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm4, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm1
+; AVX512BW-NEXT:    vpermt2d %zmm23, %zmm0, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm28, %zmm4
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2d %zmm5, %zmm0, %zmm4
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
+; AVX512BW-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm1
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm31 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2d %zmm31, %zmm1, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm6
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2d %zmm16, %zmm0, %zmm6
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm6[0,1,2,3],ymm2[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm4, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm18, %zmm2
-; AVX512BW-NEXT:    vpermt2d %zmm21, %zmm0, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm21 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm21, %zmm4
-; AVX512BW-NEXT:    vpermt2d %zmm10, %zmm0, %zmm4
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm4 {%k1}
-; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermi2d %zmm25, %zmm3, %zmm0
-; AVX512BW-NEXT:    vmovdqa64 %zmm26, %zmm10
-; AVX512BW-NEXT:    vmovdqu64 %zmm26, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermi2d %zmm26, %zmm9, %zmm1
+; AVX512BW-NEXT:    vpermt2d %zmm13, %zmm0, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm12
+; AVX512BW-NEXT:    vpermt2d %zmm25, %zmm0, %zmm12
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm12[0,1,2,3],ymm1[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm4, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm1
+; AVX512BW-NEXT:    vpermt2d %zmm17, %zmm0, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm26, %zmm4
+; AVX512BW-NEXT:    vpermt2d %zmm21, %zmm0, %zmm4
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2d %zmm3, %zmm0, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm12
+; AVX512BW-NEXT:    vpermt2d %zmm24, %zmm0, %zmm12
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm12[0,1,2,3],ymm1[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm4, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm1
+; AVX512BW-NEXT:    vpermt2d %zmm18, %zmm0, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm22, %zmm4
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2d %zmm2, %zmm0, %zmm4
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2d %zmm11, %zmm0, %zmm1
+; AVX512BW-NEXT:    vpermi2d %zmm19, %zmm7, %zmm0
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
 ; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm4, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [2,10,18,26,2,10,18,26,2,10,18,26,2,10,18,26]
 ; AVX512BW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm28 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm28, %zmm1
-; AVX512BW-NEXT:    vpermt2d %zmm30, %zmm0, %zmm1
-; AVX512BW-NEXT:    vmovdqa64 %zmm24, %zmm18
-; AVX512BW-NEXT:    vmovdqu64 %zmm24, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm24, %zmm2
-; AVX512BW-NEXT:    vmovdqa64 %zmm19, %zmm26
-; AVX512BW-NEXT:    vpermt2d %zmm19, %zmm0, %zmm2
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm4
-; AVX512BW-NEXT:    vmovdqa64 %zmm23, %zmm14
-; AVX512BW-NEXT:    vmovdqu64 %zmm23, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2d %zmm23, %zmm0, %zmm4
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [2,10,18,26,2,10,18,26]
-; AVX512BW-NEXT:    # ymm1 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm6
-; AVX512BW-NEXT:    vpermt2d %zmm17, %zmm1, %zmm6
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm6[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm4, %zmm2, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm2
-; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm24
-; AVX512BW-NEXT:    vmovdqu64 (%rsp), %zmm19 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2d %zmm19, %zmm0, %zmm2
-; AVX512BW-NEXT:    vmovdqa64 %zmm27, %zmm4
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2d %zmm17, %zmm0, %zmm4
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm4 {%k1}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2d %zmm11, %zmm0, %zmm2
-; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm23
-; AVX512BW-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm25 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2d %zmm25, %zmm1, %zmm6
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm6[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm4, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm2
-; AVX512BW-NEXT:    vpermt2d %zmm5, %zmm0, %zmm2
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm11
-; AVX512BW-NEXT:    vmovdqa64 %zmm20, %zmm4
-; AVX512BW-NEXT:    vmovdqa64 %zmm29, %zmm5
+; AVX512BW-NEXT:    vmovdqa64 %zmm17, %zmm1
+; AVX512BW-NEXT:    vpermt2d %zmm20, %zmm0, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm27, %zmm4
 ; AVX512BW-NEXT:    vpermt2d %zmm29, %zmm0, %zmm4
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm4 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm2
-; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm8
-; AVX512BW-NEXT:    vpermt2d %zmm31, %zmm1, %zmm2
-; AVX512BW-NEXT:    vpermt2d %zmm16, %zmm0, %zmm7
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm7[0,1,2,3],ymm2[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm4, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm2
-; AVX512BW-NEXT:    vpermt2d %zmm22, %zmm0, %zmm2
-; AVX512BW-NEXT:    vmovdqa64 %zmm21, %zmm4
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2d %zmm12, %zmm0, %zmm4
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm4 {%k1}
-; AVX512BW-NEXT:    vpermi2d {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm0 # 64-byte Folded Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-NEXT:    vpermi2d %zmm10, %zmm3, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
+; AVX512BW-NEXT:    vpermt2d %zmm31, %zmm0, %zmm8
+; AVX512BW-NEXT:    vpermt2d %zmm30, %zmm0, %zmm10
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm10[0,1,2,3],ymm8[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm4, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm1
+; AVX512BW-NEXT:    vpermt2d %zmm23, %zmm0, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm28, %zmm4
+; AVX512BW-NEXT:    vpermt2d %zmm5, %zmm0, %zmm4
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
+; AVX512BW-NEXT:    vpermt2d %zmm13, %zmm0, %zmm15
+; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm16
+; AVX512BW-NEXT:    vpermt2d %zmm25, %zmm0, %zmm9
+; AVX512BW-NEXT:    vmovdqa64 %zmm25, %zmm29
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm9[0,1,2,3],ymm15[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm4, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2d %zmm10, %zmm0, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm26, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm26, %zmm4
+; AVX512BW-NEXT:    vmovdqa64 %zmm21, %zmm7
+; AVX512BW-NEXT:    vpermt2d %zmm21, %zmm0, %zmm4
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm25
+; AVX512BW-NEXT:    vpermt2d %zmm3, %zmm0, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 (%rsp), %zmm3 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm12
+; AVX512BW-NEXT:    vmovdqa64 %zmm24, %zmm15
+; AVX512BW-NEXT:    vpermt2d %zmm24, %zmm0, %zmm12
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm12[0,1,2,3],ymm1[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm4, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm13
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm1
+; AVX512BW-NEXT:    vpermt2d %zmm18, %zmm0, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm22, %zmm4
+; AVX512BW-NEXT:    vpermt2d %zmm2, %zmm0, %zmm4
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm6
+; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm1
+; AVX512BW-NEXT:    vpermt2d %zmm11, %zmm0, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm24
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
+; AVX512BW-NEXT:    vpermi2d %zmm19, %zmm14, %zmm0
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
 ; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm4, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [3,11,19,27,3,11,19,27,3,11,19,27,3,11,19,27]
 ; AVX512BW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm28, %zmm1
-; AVX512BW-NEXT:    vpermt2d %zmm30, %zmm0, %zmm1
-; AVX512BW-NEXT:    vpermt2d %zmm26, %zmm0, %zmm18
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm18 {%k1}
-; AVX512BW-NEXT:    vpermt2d %zmm14, %zmm0, %zmm15
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [3,11,19,27,3,11,19,27]
-; AVX512BW-NEXT:    # ymm1 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm6
-; AVX512BW-NEXT:    vpermt2d {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm6 # 64-byte Folded Reload
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm15[0,1,2,3],ymm6[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm4, %zmm18, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm24, %zmm2
-; AVX512BW-NEXT:    vmovdqa64 %zmm24, %zmm29
-; AVX512BW-NEXT:    vmovdqu64 %zmm24, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2d %zmm19, %zmm0, %zmm2
-; AVX512BW-NEXT:    vmovdqa64 %zmm19, %zmm24
+; AVX512BW-NEXT:    vmovdqa64 %zmm17, %zmm1
+; AVX512BW-NEXT:    vpermt2d %zmm20, %zmm0, %zmm1
 ; AVX512BW-NEXT:    vmovdqa64 %zmm27, %zmm4
-; AVX512BW-NEXT:    vmovdqa64 %zmm27, %zmm16
-; AVX512BW-NEXT:    vmovdqu64 %zmm27, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2d %zmm17, %zmm0, %zmm4
-; AVX512BW-NEXT:    vmovdqa64 %zmm17, %zmm28
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm4 {%k1}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm2
-; AVX512BW-NEXT:    vpermt2d %zmm23, %zmm0, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm6
-; AVX512BW-NEXT:    vmovdqa64 %zmm25, %zmm15
-; AVX512BW-NEXT:    vpermt2d %zmm25, %zmm1, %zmm6
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm6[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm4, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm23 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm23, %zmm2
-; AVX512BW-NEXT:    vpermt2d %zmm11, %zmm0, %zmm2
-; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm27
-; AVX512BW-NEXT:    vmovdqa64 %zmm20, %zmm4
-; AVX512BW-NEXT:    vmovdqa64 %zmm20, %zmm22
-; AVX512BW-NEXT:    vmovdqu64 %zmm20, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2d %zmm2, %zmm0, %zmm4
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm31, %zmm21
+; AVX512BW-NEXT:    vpermt2d %zmm31, %zmm0, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2d %zmm30, %zmm0, %zmm12
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm12[0,1,2,3],ymm1[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm4, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm23, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2d %zmm23, %zmm0, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm28, %zmm4
 ; AVX512BW-NEXT:    vpermt2d %zmm5, %zmm0, %zmm4
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm4 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm2
-; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm11
-; AVX512BW-NEXT:    vpermt2d %zmm31, %zmm1, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm6
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm20 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2d %zmm20, %zmm0, %zmm6
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm6[0,1,2,3],ymm2[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm4, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm17
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2d %zmm18, %zmm0, %zmm2
-; AVX512BW-NEXT:    vmovdqa64 %zmm21, %zmm4
-; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm21
-; AVX512BW-NEXT:    vpermt2d %zmm12, %zmm0, %zmm4
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm4 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm31
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2d %zmm16, %zmm0, %zmm1
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512BW-NEXT:    vpermi2d %zmm12, %zmm7, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
-; AVX512BW-NEXT:    vpermi2d %zmm8, %zmm3, %zmm1
+; AVX512BW-NEXT:    vpermt2d %zmm29, %zmm0, %zmm12
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm12[0,1,2,3],ymm1[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm4, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm1
+; AVX512BW-NEXT:    vpermt2d %zmm10, %zmm0, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm26, %zmm4
+; AVX512BW-NEXT:    vpermt2d %zmm7, %zmm0, %zmm4
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm26
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm25, %zmm9
+; AVX512BW-NEXT:    vpermt2d %zmm25, %zmm0, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm12
+; AVX512BW-NEXT:    vpermt2d %zmm15, %zmm0, %zmm12
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm12[0,1,2,3],ymm1[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm4, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm1
+; AVX512BW-NEXT:    vpermt2d %zmm18, %zmm0, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm18, %zmm25
+; AVX512BW-NEXT:    vmovdqa64 %zmm22, %zmm4
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2d %zmm5, %zmm0, %zmm4
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm24, %zmm18
+; AVX512BW-NEXT:    vpermt2d %zmm24, %zmm0, %zmm1
+; AVX512BW-NEXT:    vpermi2d %zmm19, %zmm14, %zmm0
+; AVX512BW-NEXT:    vmovdqa64 %zmm19, %zmm24
+; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm13
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
 ; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm4, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [4,12,20,28,4,12,20,28,4,12,20,28,4,12,20,28]
 ; AVX512BW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm17, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm20, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2d %zmm20, %zmm0, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm27, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm27, %zmm4
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm3
+; AVX512BW-NEXT:    vpermt2d %zmm2, %zmm0, %zmm4
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm21, %zmm6
+; AVX512BW-NEXT:    vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2d %zmm21, %zmm0, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
 ; AVX512BW-NEXT:    vmovdqu64 %zmm30, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2d %zmm30, %zmm0, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm25 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm25, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm26, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2d %zmm26, %zmm0, %zmm2
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm4
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2d %zmm19, %zmm0, %zmm4
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [4,12,20,28,4,12,20,28]
-; AVX512BW-NEXT:    # ymm1 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm6
+; AVX512BW-NEXT:    vpermt2d %zmm30, %zmm0, %zmm12
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm12[0,1,2,3],ymm1[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm4, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm1
+; AVX512BW-NEXT:    vpermt2d %zmm23, %zmm0, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm28, %zmm4
+; AVX512BW-NEXT:    vpermt2d %zmm31, %zmm0, %zmm4
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm1
+; AVX512BW-NEXT:    vpermt2d %zmm16, %zmm0, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm12
+; AVX512BW-NEXT:    vmovdqa64 %zmm29, %zmm11
+; AVX512BW-NEXT:    vpermt2d %zmm29, %zmm0, %zmm12
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm12[0,1,2,3],ymm1[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm4, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm23 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm23, %zmm1
+; AVX512BW-NEXT:    vpermt2d %zmm10, %zmm0, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm31
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm4
+; AVX512BW-NEXT:    vmovdqa64 %zmm26, %zmm19
+; AVX512BW-NEXT:    vpermt2d %zmm26, %zmm0, %zmm4
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2d %zmm9, %zmm0, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm26
+; AVX512BW-NEXT:    vmovdqu64 (%rsp), %zmm12 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2d %zmm15, %zmm0, %zmm12
+; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm29
+; AVX512BW-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm12[0,1,2,3],ymm1[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm4, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm21 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm21, %zmm1
+; AVX512BW-NEXT:    vpermt2d %zmm25, %zmm0, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm22, %zmm4
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm14
+; AVX512BW-NEXT:    vpermt2d %zmm5, %zmm0, %zmm4
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2d %zmm9, %zmm1, %zmm6
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm6[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm4, %zmm2, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2d %zmm24, %zmm0, %zmm29
-; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm4
-; AVX512BW-NEXT:    vpermt2d %zmm28, %zmm0, %zmm4
-; AVX512BW-NEXT:    vmovdqa64 %zmm29, %zmm4 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm2
+; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm1
+; AVX512BW-NEXT:    vpermt2d %zmm18, %zmm0, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm5
+; AVX512BW-NEXT:    vmovdqa64 %zmm24, %zmm13
+; AVX512BW-NEXT:    vmovdqu64 %zmm24, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermi2d %zmm24, %zmm5, %zmm0
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm4, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [5,13,21,29,5,13,21,29,5,13,21,29,5,13,21,29]
+; AVX512BW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpermt2d %zmm20, %zmm0, %zmm17
+; AVX512BW-NEXT:    vpermt2d %zmm3, %zmm0, %zmm27
+; AVX512BW-NEXT:    vmovdqa64 %zmm17, %zmm27 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm1
+; AVX512BW-NEXT:    vpermt2d %zmm6, %zmm0, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2d %zmm30, %zmm0, %zmm12
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm12[0,1,2,3],ymm1[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm27, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm24 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2d %zmm24, %zmm0, %zmm2
-; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm6
-; AVX512BW-NEXT:    vpermt2d %zmm15, %zmm1, %zmm6
-; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm16
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm6[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm4, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm23, %zmm2
-; AVX512BW-NEXT:    vpermt2d %zmm27, %zmm0, %zmm2
-; AVX512BW-NEXT:    vmovdqa64 %zmm27, %zmm15
+; AVX512BW-NEXT:    vmovdqa64 %zmm24, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2d %zmm3, %zmm0, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm28, %zmm4
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2d %zmm7, %zmm0, %zmm4
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm15
+; AVX512BW-NEXT:    vpermt2d %zmm16, %zmm0, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm8
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm12
+; AVX512BW-NEXT:    vpermt2d %zmm11, %zmm0, %zmm12
+; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm2
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm12[0,1,2,3],ymm1[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm4, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm23, %zmm1
+; AVX512BW-NEXT:    vpermt2d %zmm31, %zmm0, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm31, %zmm16
+; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm4
+; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm27
+; AVX512BW-NEXT:    vpermt2d %zmm19, %zmm0, %zmm4
+; AVX512BW-NEXT:    vmovdqa64 %zmm19, %zmm20
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm1
+; AVX512BW-NEXT:    vpermt2d %zmm26, %zmm0, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm26, %zmm17
+; AVX512BW-NEXT:    vmovdqu64 (%rsp), %zmm6 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm12
+; AVX512BW-NEXT:    vpermt2d %zmm29, %zmm0, %zmm12
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm12[0,1,2,3],ymm1[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm4, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm21, %zmm1
+; AVX512BW-NEXT:    vpermt2d %zmm25, %zmm0, %zmm1
 ; AVX512BW-NEXT:    vmovdqa64 %zmm22, %zmm4
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm27 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2d %zmm27, %zmm0, %zmm4
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm4 {%k1}
-; AVX512BW-NEXT:    vpermt2d %zmm31, %zmm1, %zmm11
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm6
-; AVX512BW-NEXT:    vpermt2d %zmm20, %zmm0, %zmm6
-; AVX512BW-NEXT:    vmovdqa64 %zmm20, %zmm22
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm6[0,1,2,3],ymm11[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm4, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm17, %zmm2
-; AVX512BW-NEXT:    vpermt2d %zmm18, %zmm0, %zmm2
-; AVX512BW-NEXT:    vmovdqa64 %zmm18, %zmm6
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm29 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm29, %zmm4
-; AVX512BW-NEXT:    vpermt2d %zmm21, %zmm0, %zmm4
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm4 {%k1}
-; AVX512BW-NEXT:    vpermi2d %zmm12, %zmm7, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vpermi2d %zmm8, %zmm2, %zmm1
+; AVX512BW-NEXT:    vpermt2d %zmm14, %zmm0, %zmm4
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm1
+; AVX512BW-NEXT:    vpermt2d %zmm18, %zmm0, %zmm1
+; AVX512BW-NEXT:    vpermi2d %zmm13, %zmm5, %zmm0
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
 ; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm4, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm7 = [5,13,21,29,5,13,21,29,5,13,21,29,5,13,21,29]
-; AVX512BW-NEXT:    # zmm7 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm0
-; AVX512BW-NEXT:    vpermt2d %zmm30, %zmm7, %zmm0
-; AVX512BW-NEXT:    vmovdqa64 %zmm25, %zmm1
-; AVX512BW-NEXT:    vmovdqa64 %zmm25, %zmm12
-; AVX512BW-NEXT:    vpermt2d %zmm26, %zmm7, %zmm1
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm0
-; AVX512BW-NEXT:    vpermt2d %zmm19, %zmm7, %zmm0
-; AVX512BW-NEXT:    vmovdqa64 %zmm19, %zmm26
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [5,13,21,29,5,13,21,29]
-; AVX512BW-NEXT:    # ymm2 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2d %zmm9, %zmm2, %zmm4
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm4[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm1, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm0
-; AVX512BW-NEXT:    vpermt2d %zmm24, %zmm7, %zmm0
-; AVX512BW-NEXT:    vpermt2d %zmm16, %zmm2, %zmm14
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm14[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm21 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm21, %zmm9
-; AVX512BW-NEXT:    vmovdqu64 (%rsp), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2d %zmm0, %zmm7, %zmm9
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm10 = [6,14,22,30,6,14,22,30,6,14,22,30,6,14,22,30]
-; AVX512BW-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm21, %zmm30
-; AVX512BW-NEXT:    vpermt2d %zmm0, %zmm10, %zmm30
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm2 = [7,15,23,31,7,15,23,31,7,15,23,31,7,15,23,31]
-; AVX512BW-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermt2d %zmm0, %zmm2, %zmm21
-; AVX512BW-NEXT:    vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm20 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm20, %zmm25
-; AVX512BW-NEXT:    vpermt2d %zmm28, %zmm7, %zmm25
-; AVX512BW-NEXT:    vmovdqa64 %zmm20, %zmm31
-; AVX512BW-NEXT:    vpermt2d %zmm28, %zmm10, %zmm31
-; AVX512BW-NEXT:    vpermt2d %zmm28, %zmm2, %zmm20
-; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm4
-; AVX512BW-NEXT:    vpermt2d %zmm24, %zmm10, %zmm4
-; AVX512BW-NEXT:    vmovdqu64 %zmm4, (%rsp) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2d %zmm24, %zmm2, %zmm13
-; AVX512BW-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm23, %zmm19
-; AVX512BW-NEXT:    vmovdqa64 %zmm23, %zmm18
-; AVX512BW-NEXT:    vpermt2d %zmm15, %zmm7, %zmm18
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm1 = [6,14,22,30,6,14,22,30,6,14,22,30,6,14,22,30]
+; AVX512BW-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm24, %zmm4
+; AVX512BW-NEXT:    vmovdqa64 %zmm24, %zmm30
+; AVX512BW-NEXT:    vpermt2d %zmm3, %zmm1, %zmm30
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [7,15,23,31,7,15,23,31,7,15,23,31,7,15,23,31]
+; AVX512BW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpermt2d %zmm3, %zmm0, %zmm4
+; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm19
+; AVX512BW-NEXT:    vmovdqa64 %zmm28, %zmm11
+; AVX512BW-NEXT:    vpermt2d %zmm7, %zmm1, %zmm11
+; AVX512BW-NEXT:    vpermt2d %zmm7, %zmm0, %zmm28
+; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm9
+; AVX512BW-NEXT:    vpermt2d %zmm8, %zmm1, %zmm9
+; AVX512BW-NEXT:    vpermt2d %zmm8, %zmm0, %zmm15
+; AVX512BW-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm8
+; AVX512BW-NEXT:    vpermt2d %zmm2, %zmm1, %zmm8
+; AVX512BW-NEXT:    vpermt2d %zmm2, %zmm0, %zmm3
+; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm23, %zmm31
+; AVX512BW-NEXT:    vpermt2d %zmm16, %zmm1, %zmm31
+; AVX512BW-NEXT:    vpermt2d %zmm16, %zmm0, %zmm23
 ; AVX512BW-NEXT:    vmovdqa64 %zmm23, %zmm16
-; AVX512BW-NEXT:    vpermt2d %zmm15, %zmm10, %zmm16
-; AVX512BW-NEXT:    vpermt2d %zmm15, %zmm2, %zmm19
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm23 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm23, %zmm11
-; AVX512BW-NEXT:    vpermt2d %zmm27, %zmm7, %zmm11
-; AVX512BW-NEXT:    vmovdqa64 %zmm23, %zmm17
-; AVX512BW-NEXT:    vpermt2d %zmm27, %zmm10, %zmm17
-; AVX512BW-NEXT:    vpermt2d %zmm27, %zmm2, %zmm23
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm14
-; AVX512BW-NEXT:    vpermt2d %zmm22, %zmm7, %zmm14
-; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm1
-; AVX512BW-NEXT:    vpermt2d %zmm22, %zmm10, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2d %zmm22, %zmm2, %zmm13
-; AVX512BW-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm21
-; AVX512BW-NEXT:    vpermt2d %zmm6, %zmm7, %zmm21
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm28
-; AVX512BW-NEXT:    vpermt2d %zmm6, %zmm10, %zmm28
-; AVX512BW-NEXT:    vpermt2d %zmm6, %zmm2, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm29, %zmm1
-; AVX512BW-NEXT:    vmovdqa64 %zmm29, %zmm22
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2d %zmm0, %zmm7, %zmm22
-; AVX512BW-NEXT:    vmovdqa64 %zmm29, %zmm24
-; AVX512BW-NEXT:    vpermt2d %zmm0, %zmm10, %zmm24
-; AVX512BW-NEXT:    vpermt2d %zmm0, %zmm2, %zmm1
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm29
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vpermi2d %zmm1, %zmm0, %zmm7
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm27
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm15
+; AVX512BW-NEXT:    vmovdqa64 %zmm27, %zmm29
+; AVX512BW-NEXT:    vpermt2d %zmm20, %zmm1, %zmm29
+; AVX512BW-NEXT:    vpermt2d %zmm20, %zmm0, %zmm27
+; AVX512BW-NEXT:    vmovdqa64 %zmm27, %zmm26
+; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm2
+; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm5
+; AVX512BW-NEXT:    vpermt2d %zmm17, %zmm1, %zmm5
+; AVX512BW-NEXT:    vpermt2d %zmm17, %zmm0, %zmm2
+; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm4
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2d %zmm2, %zmm1, %zmm4
+; AVX512BW-NEXT:    vpermt2d %zmm2, %zmm0, %zmm6
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm15
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2d %zmm3, %zmm10, %zmm15
-; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm13
-; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm5
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2d %zmm4, %zmm10, %zmm5
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm8
-; AVX512BW-NEXT:    vmovdqa64 %zmm26, %zmm6
-; AVX512BW-NEXT:    vpermt2d %zmm26, %zmm10, %zmm8
-; AVX512BW-NEXT:    vpermi2d %zmm1, %zmm0, %zmm10
-; AVX512BW-NEXT:    vpermt2d %zmm1, %zmm2, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2d %zmm3, %zmm2, %zmm27
-; AVX512BW-NEXT:    vpermt2d %zmm4, %zmm2, %zmm13
-; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm26
-; AVX512BW-NEXT:    vpermt2d %zmm6, %zmm2, %zmm12
-; AVX512BW-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm25 {%k1}
-; AVX512BW-NEXT:    vinserti64x4 $0, {{[-0-9]+}}(%r{{[sb]}}p), %zmm25, %zmm25 # 32-byte Folded Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm18, %zmm11 {%k1}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm6
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = [5,13,21,29,5,13,21,29]
-; AVX512BW-NEXT:    # ymm0 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vpermt2d %zmm4, %zmm0, %zmm6
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm14[0,1,2,3],ymm6[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm3, %zmm11, %zmm18
-; AVX512BW-NEXT:    vmovdqa64 %zmm21, %zmm22 {%k1}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
-; AVX512BW-NEXT:    vpermi2d %zmm14, %zmm1, %zmm0
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm7[0,1,2,3],ymm0[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm22, %zmm21
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [6,14,22,30,6,14,22,30]
-; AVX512BW-NEXT:    # ymm3 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm7
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm23
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2d %zmm2, %zmm3, %zmm7
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm11 = [7,15,23,31,7,15,23,31]
-; AVX512BW-NEXT:    # ymm11 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vpermt2d %zmm2, %zmm11, %zmm12
-; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm0
-; AVX512BW-NEXT:    vpermt2d %zmm4, %zmm3, %zmm0
-; AVX512BW-NEXT:    vpermt2d %zmm4, %zmm11, %zmm9
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm22 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2d %zmm22, %zmm3, %zmm2
-; AVX512BW-NEXT:    vpermi2d %zmm14, %zmm1, %zmm3
-; AVX512BW-NEXT:    vpermt2d %zmm14, %zmm11, %zmm1
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm14
-; AVX512BW-NEXT:    vpermt2d %zmm22, %zmm11, %zmm6
-; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm11
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm6 = ymm8[0,1,2,3],ymm2[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm5 {%k1}
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm6, %zmm5, %zmm6
-; AVX512BW-NEXT:    vpblendd $15, (%rsp), %ymm7, %ymm7 # 32-byte Folded Reload
-; AVX512BW-NEXT:    # ymm7 = mem[0,1,2,3],ymm7[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqa64 %zmm30, %zmm31 {%k1}
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm7, %zmm31, %zmm13
-; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm17 {%k1}
-; AVX512BW-NEXT:    vpblendd $15, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
-; AVX512BW-NEXT:    # ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm17, %zmm0
-; AVX512BW-NEXT:    vmovdqa64 %zmm28, %zmm24 {%k1}
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm10[0,1,2,3],ymm3[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm24, %zmm1
+; AVX512BW-NEXT:    vpermt2d %zmm2, %zmm1, %zmm23
+; AVX512BW-NEXT:    vpermt2d %zmm2, %zmm0, %zmm3
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm17
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm24
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm20 {%k1}
-; AVX512BW-NEXT:    vpblendd $15, {{[-0-9]+}}(%r{{[sb]}}p), %ymm12, %ymm3 # 32-byte Folded Reload
-; AVX512BW-NEXT:    # ymm3 = mem[0,1,2,3],ymm12[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm3, %zmm20, %zmm3
-; AVX512BW-NEXT:    vmovdqa64 %zmm19, %zmm23 {%k1}
-; AVX512BW-NEXT:    vpblendd $15, {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm4 # 32-byte Folded Reload
-; AVX512BW-NEXT:    # ymm4 = mem[0,1,2,3],ymm9[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm4, %zmm23, %zmm4
+; AVX512BW-NEXT:    vpermt2d %zmm2, %zmm1, %zmm24
+; AVX512BW-NEXT:    vpermt2d %zmm2, %zmm0, %zmm3
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm27
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm13
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2d %zmm3, %zmm1, %zmm13
+; AVX512BW-NEXT:    vpermt2d %zmm3, %zmm0, %zmm12
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm3
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2d %zmm6, %zmm1, %zmm3
+; AVX512BW-NEXT:    vpermt2d %zmm6, %zmm0, %zmm10
+; AVX512BW-NEXT:    vmovdqa64 %zmm21, %zmm6
+; AVX512BW-NEXT:    vmovdqa64 %zmm21, %zmm20
+; AVX512BW-NEXT:    vpermt2d %zmm25, %zmm1, %zmm20
+; AVX512BW-NEXT:    vpermt2d %zmm25, %zmm0, %zmm6
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm21
+; AVX512BW-NEXT:    vmovdqa64 %zmm22, %zmm25
+; AVX512BW-NEXT:    vpermt2d %zmm14, %zmm1, %zmm25
+; AVX512BW-NEXT:    vpermt2d %zmm14, %zmm0, %zmm22
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm14
+; AVX512BW-NEXT:    vpermt2d %zmm18, %zmm1, %zmm14
+; AVX512BW-NEXT:    vpermt2d %zmm18, %zmm0, %zmm7
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm29 {%k1}
-; AVX512BW-NEXT:    vpblendd $15, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm7 # 32-byte Folded Reload
-; AVX512BW-NEXT:    # ymm7 = mem[0,1,2,3],ymm14[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm7, %zmm29, %zmm7
-; AVX512BW-NEXT:    vmovdqa64 %zmm27, %zmm26 {%k1}
-; AVX512BW-NEXT:    vpblendd $15, {{[-0-9]+}}(%r{{[sb]}}p), %ymm11, %ymm8 # 32-byte Folded Reload
-; AVX512BW-NEXT:    # ymm8 = mem[0,1,2,3],ymm11[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm8, %zmm26, %zmm8
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm2, 192(%rsi)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm2, 128(%rsi)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm2, 64(%rsi)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm2, (%rsi)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm2, 192(%rdx)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm2, (%rdx)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm2, 64(%rdx)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm2, 128(%rdx)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm2, 192(%rcx)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm2, (%rcx)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm2, 64(%rcx)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm2, 128(%rcx)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm2, 192(%r8)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm2, (%r8)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm2, 64(%r8)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm2, 128(%r8)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm2, 192(%r9)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm2, (%r9)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm2, 64(%r9)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm2, 128(%r9)
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512BW-NEXT:    vpermi2d %zmm6, %zmm2, %zmm1
+; AVX512BW-NEXT:    vpermt2d %zmm6, %zmm0, %zmm2
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm6
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3],ymm13[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm23, %zmm24 {%k1}
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm24, %zmm0
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm8[0,1,2,3],ymm9[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm30, %zmm11 {%k1}
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm11, %zmm2
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm4[0,1,2,3],ymm5[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm31, %zmm29 {%k1}
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm3, %zmm29, %zmm3
+; AVX512BW-NEXT:    vmovdqa64 %zmm20, %zmm25 {%k1}
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm14[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm25, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm19, %zmm28 {%k1}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512BW-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm4 # 32-byte Folded Reload
+; AVX512BW-NEXT:    # ymm4 = ymm4[0,1,2,3],mem[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm4, %zmm28, %zmm4
+; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm26 {%k1}
+; AVX512BW-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm15, %ymm5 # 32-byte Folded Reload
+; AVX512BW-NEXT:    # ymm5 = ymm15[0,1,2,3],mem[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm5, %zmm26, %zmm5
+; AVX512BW-NEXT:    vmovdqa64 %zmm17, %zmm27 {%k1}
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm10[0,1,2,3],ymm12[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm8, %zmm27, %zmm9
+; AVX512BW-NEXT:    vmovdqa64 %zmm21, %zmm22 {%k1}
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm7 = ymm6[0,1,2,3],ymm7[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm7, %zmm22, %zmm7
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm8, 192(%rsi)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm8, 128(%rsi)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm8, 64(%rsi)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm6, (%rsi)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm6, 192(%rdx)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm6, (%rdx)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm6, 64(%rdx)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm6, 128(%rdx)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm6, 192(%rcx)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm6, (%rcx)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm6, 64(%rcx)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm6, 128(%rcx)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm6, 192(%r8)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm6, (%r8)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm6, 64(%r8)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm6, 128(%r8)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm6, 192(%r9)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm6, (%r9)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm6, 64(%r9)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm6, 128(%r9)
 ; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512BW-NEXT:    vmovdqa64 %zmm21, 192(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm18, (%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm25, 64(%rax)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm2, 128(%rax)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm6, 192(%rax)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm6, (%rax)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm6, 64(%rax)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm6, 128(%rax)
 ; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, 192(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm13, 64(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm6, 128(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, 192(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, (%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, 64(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, 128(%rax)
 ; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512BW-NEXT:    vmovdqa64 %zmm8, 128(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, 192(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm4, (%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm3, 64(%rax)
-; AVX512BW-NEXT:    addq $3304, %rsp # imm = 0xCE8
+; AVX512BW-NEXT:    vmovdqa64 %zmm9, 128(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, 192(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, (%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm4, 64(%rax)
+; AVX512BW-NEXT:    addq $3144, %rsp # imm = 0xC48
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
   %wide.vec = load <512 x i32>, ptr %in.vec, align 64
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i64-stride-5.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i64-stride-5.ll
index 4a2c38168bdf4208d1a100e327af109e045e8b64..ccc15a4119ba457278f47a1f6b42391cad5d792f 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i64-stride-5.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i64-stride-5.ll
@@ -520,118 +520,116 @@ define void @load_i64_stride5_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512F-LABEL: load_i64_stride5_vf8:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    vmovdqa64 256(%rdi), %zmm0
-; AVX512F-NEXT:    vmovdqa64 (%rdi), %zmm1
-; AVX512F-NEXT:    vmovdqa64 64(%rdi), %zmm2
-; AVX512F-NEXT:    vmovdqa64 128(%rdi), %zmm3
-; AVX512F-NEXT:    vmovdqa64 192(%rdi), %zmm4
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm5 = [12,1,6,0,12,1,6,0]
-; AVX512F-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermi2q %zmm3, %zmm4, %zmm5
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm6 = [0,5,10,15]
-; AVX512F-NEXT:    vpermi2q %zmm2, %zmm1, %zmm6
-; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm5 = zmm6[0,1,2,3],zmm5[4,5,6,7]
-; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [0,1,2,3,4,5,6,11]
-; AVX512F-NEXT:    vpermi2q %zmm0, %zmm5, %zmm6
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm5 = <1,6,11,u>
-; AVX512F-NEXT:    vpermi2q %zmm2, %zmm1, %zmm5
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm7 = [5,10,15,0,5,10,15,0]
+; AVX512F-NEXT:    vmovdqa64 (%rdi), %zmm2
+; AVX512F-NEXT:    vmovdqa64 64(%rdi), %zmm1
+; AVX512F-NEXT:    vmovdqa64 128(%rdi), %zmm4
+; AVX512F-NEXT:    vmovdqa64 192(%rdi), %zmm5
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm3 = [12,1,6,0,12,1,6,0]
+; AVX512F-NEXT:    # zmm3 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm6 = [5,10,15,0,5,10,15,0]
+; AVX512F-NEXT:    # zmm6 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpermi2q %zmm5, %zmm4, %zmm6
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm7 = [6,11,0,1,6,11,0,1]
 ; AVX512F-NEXT:    # zmm7 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermi2q %zmm4, %zmm3, %zmm7
+; AVX512F-NEXT:    vpermi2q %zmm5, %zmm4, %zmm7
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm8 = [7,12,0,2,7,12,0,2]
+; AVX512F-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpermi2q %zmm5, %zmm4, %zmm8
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm9 = [0,5,0,11,0,5,0,11]
+; AVX512F-NEXT:    # zmm9 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpermi2q %zmm4, %zmm5, %zmm9
+; AVX512F-NEXT:    vpermt2q %zmm4, %zmm3, %zmm5
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm4 = [0,5,10,15]
+; AVX512F-NEXT:    vpermi2q %zmm1, %zmm2, %zmm4
+; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm4 = zmm4[0,1,2,3],zmm5[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [0,1,2,3,4,5,6,11]
+; AVX512F-NEXT:    vpermi2q %zmm0, %zmm4, %zmm5
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm4 = <1,6,11,u>
+; AVX512F-NEXT:    vpermi2q %zmm1, %zmm2, %zmm4
 ; AVX512F-NEXT:    movb $7, %al
 ; AVX512F-NEXT:    kmovw %eax, %k1
-; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm7 {%k1}
-; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [0,1,2,3,4,5,6,12]
-; AVX512F-NEXT:    vpermi2q %zmm0, %zmm7, %zmm5
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm7 = [6,11,0,1,6,11,0,1]
-; AVX512F-NEXT:    # zmm7 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermi2q %zmm4, %zmm3, %zmm7
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm8 = <2,7,12,u>
-; AVX512F-NEXT:    vpermi2q %zmm2, %zmm1, %zmm8
+; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm6 {%k1}
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [0,1,2,3,4,5,6,12]
+; AVX512F-NEXT:    vpermi2q %zmm0, %zmm6, %zmm4
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm6 = <2,7,12,u>
+; AVX512F-NEXT:    vpermi2q %zmm1, %zmm2, %zmm6
 ; AVX512F-NEXT:    movb $56, %al
 ; AVX512F-NEXT:    kmovw %eax, %k1
-; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm8 {%k1}
+; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm6 {%k1}
 ; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm7 = [0,1,2,3,4,5,8,13]
-; AVX512F-NEXT:    vpermi2q %zmm0, %zmm8, %zmm7
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm8 = [7,12,0,2,7,12,0,2]
-; AVX512F-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermi2q %zmm4, %zmm3, %zmm8
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm9 = <11,0,5,u>
-; AVX512F-NEXT:    vpermi2q %zmm1, %zmm2, %zmm9
-; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm9 {%k1}
+; AVX512F-NEXT:    vpermi2q %zmm0, %zmm6, %zmm7
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm6 = <11,0,5,u>
+; AVX512F-NEXT:    vpermi2q %zmm2, %zmm1, %zmm6
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm6 {%k1}
 ; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm8 = [0,1,2,3,4,5,9,14]
-; AVX512F-NEXT:    vpermi2q %zmm0, %zmm9, %zmm8
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm9 = [0,5,0,11,0,5,0,11]
-; AVX512F-NEXT:    # zmm9 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermi2q %zmm3, %zmm4, %zmm9
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = <12,1,6,u>
-; AVX512F-NEXT:    vpermi2q %zmm1, %zmm2, %zmm3
-; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm3 {%k1}
-; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,1,2,3,4,5,10,15]
-; AVX512F-NEXT:    vpermi2q %zmm0, %zmm3, %zmm1
-; AVX512F-NEXT:    vmovdqa64 %zmm6, (%rsi)
-; AVX512F-NEXT:    vmovdqa64 %zmm5, (%rdx)
+; AVX512F-NEXT:    vpermi2q %zmm0, %zmm6, %zmm8
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm3, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm1 {%k1}
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [0,1,2,3,4,5,10,15]
+; AVX512F-NEXT:    vpermi2q %zmm0, %zmm1, %zmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm5, (%rsi)
+; AVX512F-NEXT:    vmovdqa64 %zmm4, (%rdx)
 ; AVX512F-NEXT:    vmovdqa64 %zmm7, (%rcx)
 ; AVX512F-NEXT:    vmovdqa64 %zmm8, (%r8)
-; AVX512F-NEXT:    vmovdqa64 %zmm1, (%r9)
+; AVX512F-NEXT:    vmovdqa64 %zmm2, (%r9)
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: load_i64_stride5_vf8:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    vmovdqa64 256(%rdi), %zmm0
-; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm1
-; AVX512BW-NEXT:    vmovdqa64 64(%rdi), %zmm2
-; AVX512BW-NEXT:    vmovdqa64 128(%rdi), %zmm3
-; AVX512BW-NEXT:    vmovdqa64 192(%rdi), %zmm4
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm5 = [12,1,6,0,12,1,6,0]
-; AVX512BW-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermi2q %zmm3, %zmm4, %zmm5
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm6 = [0,5,10,15]
-; AVX512BW-NEXT:    vpermi2q %zmm2, %zmm1, %zmm6
-; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm5 = zmm6[0,1,2,3],zmm5[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [0,1,2,3,4,5,6,11]
-; AVX512BW-NEXT:    vpermi2q %zmm0, %zmm5, %zmm6
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm5 = <1,6,11,u>
-; AVX512BW-NEXT:    vpermi2q %zmm2, %zmm1, %zmm5
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm7 = [5,10,15,0,5,10,15,0]
+; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm2
+; AVX512BW-NEXT:    vmovdqa64 64(%rdi), %zmm1
+; AVX512BW-NEXT:    vmovdqa64 128(%rdi), %zmm4
+; AVX512BW-NEXT:    vmovdqa64 192(%rdi), %zmm5
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm3 = [12,1,6,0,12,1,6,0]
+; AVX512BW-NEXT:    # zmm3 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm6 = [5,10,15,0,5,10,15,0]
+; AVX512BW-NEXT:    # zmm6 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpermi2q %zmm5, %zmm4, %zmm6
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm7 = [6,11,0,1,6,11,0,1]
 ; AVX512BW-NEXT:    # zmm7 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermi2q %zmm4, %zmm3, %zmm7
+; AVX512BW-NEXT:    vpermi2q %zmm5, %zmm4, %zmm7
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm8 = [7,12,0,2,7,12,0,2]
+; AVX512BW-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpermi2q %zmm5, %zmm4, %zmm8
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm9 = [0,5,0,11,0,5,0,11]
+; AVX512BW-NEXT:    # zmm9 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpermi2q %zmm4, %zmm5, %zmm9
+; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm3, %zmm5
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm4 = [0,5,10,15]
+; AVX512BW-NEXT:    vpermi2q %zmm1, %zmm2, %zmm4
+; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm4 = zmm4[0,1,2,3],zmm5[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [0,1,2,3,4,5,6,11]
+; AVX512BW-NEXT:    vpermi2q %zmm0, %zmm4, %zmm5
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm4 = <1,6,11,u>
+; AVX512BW-NEXT:    vpermi2q %zmm1, %zmm2, %zmm4
 ; AVX512BW-NEXT:    movb $7, %al
 ; AVX512BW-NEXT:    kmovd %eax, %k1
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm7 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [0,1,2,3,4,5,6,12]
-; AVX512BW-NEXT:    vpermi2q %zmm0, %zmm7, %zmm5
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm7 = [6,11,0,1,6,11,0,1]
-; AVX512BW-NEXT:    # zmm7 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermi2q %zmm4, %zmm3, %zmm7
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm8 = <2,7,12,u>
-; AVX512BW-NEXT:    vpermi2q %zmm2, %zmm1, %zmm8
+; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm6 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [0,1,2,3,4,5,6,12]
+; AVX512BW-NEXT:    vpermi2q %zmm0, %zmm6, %zmm4
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm6 = <2,7,12,u>
+; AVX512BW-NEXT:    vpermi2q %zmm1, %zmm2, %zmm6
 ; AVX512BW-NEXT:    movb $56, %al
 ; AVX512BW-NEXT:    kmovd %eax, %k1
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm8 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm6 {%k1}
 ; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm7 = [0,1,2,3,4,5,8,13]
-; AVX512BW-NEXT:    vpermi2q %zmm0, %zmm8, %zmm7
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm8 = [7,12,0,2,7,12,0,2]
-; AVX512BW-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermi2q %zmm4, %zmm3, %zmm8
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm9 = <11,0,5,u>
-; AVX512BW-NEXT:    vpermi2q %zmm1, %zmm2, %zmm9
-; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm9 {%k1}
+; AVX512BW-NEXT:    vpermi2q %zmm0, %zmm6, %zmm7
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm6 = <11,0,5,u>
+; AVX512BW-NEXT:    vpermi2q %zmm2, %zmm1, %zmm6
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm6 {%k1}
 ; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm8 = [0,1,2,3,4,5,9,14]
-; AVX512BW-NEXT:    vpermi2q %zmm0, %zmm9, %zmm8
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm9 = [0,5,0,11,0,5,0,11]
-; AVX512BW-NEXT:    # zmm9 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermi2q %zmm3, %zmm4, %zmm9
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm3 = <12,1,6,u>
-; AVX512BW-NEXT:    vpermi2q %zmm1, %zmm2, %zmm3
-; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm3 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,1,2,3,4,5,10,15]
-; AVX512BW-NEXT:    vpermi2q %zmm0, %zmm3, %zmm1
-; AVX512BW-NEXT:    vmovdqa64 %zmm6, (%rsi)
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, (%rdx)
+; AVX512BW-NEXT:    vpermi2q %zmm0, %zmm6, %zmm8
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm3, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm1 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [0,1,2,3,4,5,10,15]
+; AVX512BW-NEXT:    vpermi2q %zmm0, %zmm1, %zmm2
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, (%rsi)
+; AVX512BW-NEXT:    vmovdqa64 %zmm4, (%rdx)
 ; AVX512BW-NEXT:    vmovdqa64 %zmm7, (%rcx)
 ; AVX512BW-NEXT:    vmovdqa64 %zmm8, (%r8)
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, (%r9)
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, (%r9)
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
   %wide.vec = load <40 x i64>, ptr %in.vec, align 64
@@ -1183,90 +1181,90 @@ define void @load_i64_stride5_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512F-NEXT:    vmovdqa64 448(%rdi), %zmm5
 ; AVX512F-NEXT:    vmovdqa64 512(%rdi), %zmm4
 ; AVX512F-NEXT:    vmovdqa64 256(%rdi), %zmm6
-; AVX512F-NEXT:    vmovdqa64 (%rdi), %zmm8
+; AVX512F-NEXT:    vmovdqa64 (%rdi), %zmm9
 ; AVX512F-NEXT:    vmovdqa64 64(%rdi), %zmm1
 ; AVX512F-NEXT:    vmovdqa64 128(%rdi), %zmm11
-; AVX512F-NEXT:    vmovdqa64 192(%rdi), %zmm9
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm10 = [12,1,6,0,12,1,6,0]
-; AVX512F-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm7
-; AVX512F-NEXT:    vpermt2q %zmm11, %zmm10, %zmm7
+; AVX512F-NEXT:    vmovdqa64 192(%rdi), %zmm10
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm7 = [12,1,6,0,12,1,6,0]
+; AVX512F-NEXT:    # zmm7 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm8
+; AVX512F-NEXT:    vpermt2q %zmm11, %zmm7, %zmm8
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm12 = [0,5,10,15]
-; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm13
+; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm13
 ; AVX512F-NEXT:    vpermt2q %zmm1, %zmm12, %zmm13
-; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm7 = zmm13[0,1,2,3],zmm7[4,5,6,7]
+; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm8 = zmm13[0,1,2,3],zmm8[4,5,6,7]
 ; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm13 = [0,1,2,3,4,5,6,11]
-; AVX512F-NEXT:    vpermt2q %zmm6, %zmm13, %zmm7
-; AVX512F-NEXT:    vpermi2q %zmm5, %zmm4, %zmm10
+; AVX512F-NEXT:    vpermt2q %zmm6, %zmm13, %zmm8
+; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm14
+; AVX512F-NEXT:    vpermt2q %zmm5, %zmm7, %zmm14
 ; AVX512F-NEXT:    vpermi2q %zmm0, %zmm3, %zmm12
-; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm10 = zmm12[0,1,2,3],zmm10[4,5,6,7]
-; AVX512F-NEXT:    vpermt2q %zmm2, %zmm13, %zmm10
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm14 = <1,6,11,u>
-; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm15
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm14, %zmm15
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm13 = [5,10,15,0,5,10,15,0]
-; AVX512F-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm12
-; AVX512F-NEXT:    vpermt2q %zmm9, %zmm13, %zmm12
+; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm12 = zmm12[0,1,2,3],zmm14[4,5,6,7]
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm13, %zmm12
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm15 = <1,6,11,u>
+; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm16
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm15, %zmm16
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm14 = [5,10,15,0,5,10,15,0]
+; AVX512F-NEXT:    # zmm14 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm13
+; AVX512F-NEXT:    vpermt2q %zmm10, %zmm14, %zmm13
 ; AVX512F-NEXT:    movb $7, %al
 ; AVX512F-NEXT:    kmovw %eax, %k1
-; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm12 {%k1}
-; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm15 = [0,1,2,3,4,5,6,12]
-; AVX512F-NEXT:    vpermt2q %zmm6, %zmm15, %zmm12
-; AVX512F-NEXT:    vpermi2q %zmm4, %zmm5, %zmm13
-; AVX512F-NEXT:    vpermi2q %zmm0, %zmm3, %zmm14
-; AVX512F-NEXT:    vmovdqa64 %zmm14, %zmm13 {%k1}
-; AVX512F-NEXT:    vpermt2q %zmm2, %zmm15, %zmm13
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm14 = [6,11,0,1,6,11,0,1]
-; AVX512F-NEXT:    # zmm14 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm15
-; AVX512F-NEXT:    vpermt2q %zmm9, %zmm14, %zmm15
-; AVX512F-NEXT:    vmovdqa64 {{.*#+}} ymm16 = <2,7,12,u>
-; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm17
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm16, %zmm17
+; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm13 {%k1}
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm16 = [0,1,2,3,4,5,6,12]
+; AVX512F-NEXT:    vpermt2q %zmm6, %zmm16, %zmm13
+; AVX512F-NEXT:    vpermi2q %zmm4, %zmm5, %zmm14
+; AVX512F-NEXT:    vpermi2q %zmm0, %zmm3, %zmm15
+; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm14 {%k1}
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm16, %zmm14
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm15 = [6,11,0,1,6,11,0,1]
+; AVX512F-NEXT:    # zmm15 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm16
+; AVX512F-NEXT:    vpermt2q %zmm10, %zmm15, %zmm16
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} ymm17 = <2,7,12,u>
+; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm18
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm17, %zmm18
 ; AVX512F-NEXT:    movb $56, %al
 ; AVX512F-NEXT:    kmovw %eax, %k1
+; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm18 {%k1}
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm16 = [0,1,2,3,4,5,8,13]
+; AVX512F-NEXT:    vpermt2q %zmm6, %zmm16, %zmm18
+; AVX512F-NEXT:    vpermi2q %zmm4, %zmm5, %zmm15
+; AVX512F-NEXT:    vpermi2q %zmm0, %zmm3, %zmm17
 ; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm17 {%k1}
-; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm15 = [0,1,2,3,4,5,8,13]
-; AVX512F-NEXT:    vpermt2q %zmm6, %zmm15, %zmm17
-; AVX512F-NEXT:    vpermi2q %zmm4, %zmm5, %zmm14
-; AVX512F-NEXT:    vpermi2q %zmm0, %zmm3, %zmm16
-; AVX512F-NEXT:    vmovdqa64 %zmm14, %zmm16 {%k1}
-; AVX512F-NEXT:    vpermt2q %zmm2, %zmm15, %zmm16
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm14 = [7,12,0,2,7,12,0,2]
-; AVX512F-NEXT:    # zmm14 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm15
-; AVX512F-NEXT:    vpermt2q %zmm9, %zmm14, %zmm15
-; AVX512F-NEXT:    vmovdqa64 {{.*#+}} ymm18 = <11,0,5,u>
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm19
-; AVX512F-NEXT:    vpermt2q %zmm8, %zmm18, %zmm19
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm16, %zmm17
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm15 = [7,12,0,2,7,12,0,2]
+; AVX512F-NEXT:    # zmm15 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm16
+; AVX512F-NEXT:    vpermt2q %zmm10, %zmm15, %zmm16
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} ymm19 = <11,0,5,u>
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm20
+; AVX512F-NEXT:    vpermt2q %zmm9, %zmm19, %zmm20
+; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm20 {%k1}
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm16 = [0,1,2,3,4,5,9,14]
+; AVX512F-NEXT:    vpermt2q %zmm6, %zmm16, %zmm20
+; AVX512F-NEXT:    vpermi2q %zmm4, %zmm5, %zmm15
+; AVX512F-NEXT:    vpermi2q %zmm3, %zmm0, %zmm19
 ; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm19 {%k1}
-; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm15 = [0,1,2,3,4,5,9,14]
-; AVX512F-NEXT:    vpermt2q %zmm6, %zmm15, %zmm19
-; AVX512F-NEXT:    vpermi2q %zmm4, %zmm5, %zmm14
-; AVX512F-NEXT:    vpermi2q %zmm3, %zmm0, %zmm18
-; AVX512F-NEXT:    vmovdqa64 %zmm14, %zmm18 {%k1}
-; AVX512F-NEXT:    vpermt2q %zmm2, %zmm15, %zmm18
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm14 = [0,5,0,11,0,5,0,11]
-; AVX512F-NEXT:    # zmm14 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermt2q %zmm11, %zmm14, %zmm9
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm11 = <12,1,6,u>
-; AVX512F-NEXT:    vpermt2q %zmm8, %zmm11, %zmm1
-; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm1 {%k1}
-; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm8 = [0,1,2,3,4,5,10,15]
-; AVX512F-NEXT:    vpermt2q %zmm6, %zmm8, %zmm1
-; AVX512F-NEXT:    vpermt2q %zmm5, %zmm14, %zmm4
-; AVX512F-NEXT:    vpermt2q %zmm3, %zmm11, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm16, %zmm19
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm15 = [0,5,0,11,0,5,0,11]
+; AVX512F-NEXT:    # zmm15 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpermt2q %zmm11, %zmm15, %zmm10
+; AVX512F-NEXT:    vpermt2q %zmm9, %zmm7, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm1 {%k1}
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm9 = [0,1,2,3,4,5,10,15]
+; AVX512F-NEXT:    vpermt2q %zmm6, %zmm9, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm5, %zmm15, %zmm4
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm7, %zmm0
 ; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm0 {%k1}
-; AVX512F-NEXT:    vpermt2q %zmm2, %zmm8, %zmm0
-; AVX512F-NEXT:    vmovdqa64 %zmm10, 64(%rsi)
-; AVX512F-NEXT:    vmovdqa64 %zmm7, (%rsi)
-; AVX512F-NEXT:    vmovdqa64 %zmm13, 64(%rdx)
-; AVX512F-NEXT:    vmovdqa64 %zmm12, (%rdx)
-; AVX512F-NEXT:    vmovdqa64 %zmm16, 64(%rcx)
-; AVX512F-NEXT:    vmovdqa64 %zmm17, (%rcx)
-; AVX512F-NEXT:    vmovdqa64 %zmm18, 64(%r8)
-; AVX512F-NEXT:    vmovdqa64 %zmm19, (%r8)
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm9, %zmm0
+; AVX512F-NEXT:    vmovdqa64 %zmm12, 64(%rsi)
+; AVX512F-NEXT:    vmovdqa64 %zmm8, (%rsi)
+; AVX512F-NEXT:    vmovdqa64 %zmm14, 64(%rdx)
+; AVX512F-NEXT:    vmovdqa64 %zmm13, (%rdx)
+; AVX512F-NEXT:    vmovdqa64 %zmm17, 64(%rcx)
+; AVX512F-NEXT:    vmovdqa64 %zmm18, (%rcx)
+; AVX512F-NEXT:    vmovdqa64 %zmm19, 64(%r8)
+; AVX512F-NEXT:    vmovdqa64 %zmm20, (%r8)
 ; AVX512F-NEXT:    vmovdqa64 %zmm0, 64(%r9)
 ; AVX512F-NEXT:    vmovdqa64 %zmm1, (%r9)
 ; AVX512F-NEXT:    vzeroupper
@@ -1280,90 +1278,90 @@ define void @load_i64_stride5_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512BW-NEXT:    vmovdqa64 448(%rdi), %zmm5
 ; AVX512BW-NEXT:    vmovdqa64 512(%rdi), %zmm4
 ; AVX512BW-NEXT:    vmovdqa64 256(%rdi), %zmm6
-; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm8
+; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm9
 ; AVX512BW-NEXT:    vmovdqa64 64(%rdi), %zmm1
 ; AVX512BW-NEXT:    vmovdqa64 128(%rdi), %zmm11
-; AVX512BW-NEXT:    vmovdqa64 192(%rdi), %zmm9
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm10 = [12,1,6,0,12,1,6,0]
-; AVX512BW-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm7
-; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm10, %zmm7
+; AVX512BW-NEXT:    vmovdqa64 192(%rdi), %zmm10
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm7 = [12,1,6,0,12,1,6,0]
+; AVX512BW-NEXT:    # zmm7 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm8
+; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm7, %zmm8
 ; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm12 = [0,5,10,15]
-; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm13
+; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm13
 ; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm12, %zmm13
-; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm7 = zmm13[0,1,2,3],zmm7[4,5,6,7]
+; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm8 = zmm13[0,1,2,3],zmm8[4,5,6,7]
 ; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm13 = [0,1,2,3,4,5,6,11]
-; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm13, %zmm7
-; AVX512BW-NEXT:    vpermi2q %zmm5, %zmm4, %zmm10
+; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm13, %zmm8
+; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm14
+; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm7, %zmm14
 ; AVX512BW-NEXT:    vpermi2q %zmm0, %zmm3, %zmm12
-; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm10 = zmm12[0,1,2,3],zmm10[4,5,6,7]
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm13, %zmm10
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm14 = <1,6,11,u>
-; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm15
-; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm14, %zmm15
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm13 = [5,10,15,0,5,10,15,0]
-; AVX512BW-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm12
-; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm13, %zmm12
+; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm12 = zmm12[0,1,2,3],zmm14[4,5,6,7]
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm13, %zmm12
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm15 = <1,6,11,u>
+; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm16
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm15, %zmm16
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm14 = [5,10,15,0,5,10,15,0]
+; AVX512BW-NEXT:    # zmm14 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm13
+; AVX512BW-NEXT:    vpermt2q %zmm10, %zmm14, %zmm13
 ; AVX512BW-NEXT:    movb $7, %al
 ; AVX512BW-NEXT:    kmovd %eax, %k1
-; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm12 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm15 = [0,1,2,3,4,5,6,12]
-; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm15, %zmm12
-; AVX512BW-NEXT:    vpermi2q %zmm4, %zmm5, %zmm13
-; AVX512BW-NEXT:    vpermi2q %zmm0, %zmm3, %zmm14
-; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm13 {%k1}
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm15, %zmm13
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm14 = [6,11,0,1,6,11,0,1]
-; AVX512BW-NEXT:    # zmm14 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm15
-; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm14, %zmm15
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} ymm16 = <2,7,12,u>
-; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm17
-; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm16, %zmm17
+; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm13 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm16 = [0,1,2,3,4,5,6,12]
+; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm16, %zmm13
+; AVX512BW-NEXT:    vpermi2q %zmm4, %zmm5, %zmm14
+; AVX512BW-NEXT:    vpermi2q %zmm0, %zmm3, %zmm15
+; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm14 {%k1}
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm16, %zmm14
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm15 = [6,11,0,1,6,11,0,1]
+; AVX512BW-NEXT:    # zmm15 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm16
+; AVX512BW-NEXT:    vpermt2q %zmm10, %zmm15, %zmm16
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} ymm17 = <2,7,12,u>
+; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm18
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm17, %zmm18
 ; AVX512BW-NEXT:    movb $56, %al
 ; AVX512BW-NEXT:    kmovd %eax, %k1
+; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm18 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm16 = [0,1,2,3,4,5,8,13]
+; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm16, %zmm18
+; AVX512BW-NEXT:    vpermi2q %zmm4, %zmm5, %zmm15
+; AVX512BW-NEXT:    vpermi2q %zmm0, %zmm3, %zmm17
 ; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm17 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm15 = [0,1,2,3,4,5,8,13]
-; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm15, %zmm17
-; AVX512BW-NEXT:    vpermi2q %zmm4, %zmm5, %zmm14
-; AVX512BW-NEXT:    vpermi2q %zmm0, %zmm3, %zmm16
-; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm16 {%k1}
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm15, %zmm16
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm14 = [7,12,0,2,7,12,0,2]
-; AVX512BW-NEXT:    # zmm14 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm15
-; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm14, %zmm15
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} ymm18 = <11,0,5,u>
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm19
-; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm18, %zmm19
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm16, %zmm17
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm15 = [7,12,0,2,7,12,0,2]
+; AVX512BW-NEXT:    # zmm15 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm16
+; AVX512BW-NEXT:    vpermt2q %zmm10, %zmm15, %zmm16
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} ymm19 = <11,0,5,u>
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm20
+; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm19, %zmm20
+; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm20 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm16 = [0,1,2,3,4,5,9,14]
+; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm16, %zmm20
+; AVX512BW-NEXT:    vpermi2q %zmm4, %zmm5, %zmm15
+; AVX512BW-NEXT:    vpermi2q %zmm3, %zmm0, %zmm19
 ; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm19 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm15 = [0,1,2,3,4,5,9,14]
-; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm15, %zmm19
-; AVX512BW-NEXT:    vpermi2q %zmm4, %zmm5, %zmm14
-; AVX512BW-NEXT:    vpermi2q %zmm3, %zmm0, %zmm18
-; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm18 {%k1}
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm15, %zmm18
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm14 = [0,5,0,11,0,5,0,11]
-; AVX512BW-NEXT:    # zmm14 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm14, %zmm9
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm11 = <12,1,6,u>
-; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm11, %zmm1
-; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm1 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm8 = [0,1,2,3,4,5,10,15]
-; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm8, %zmm1
-; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm14, %zmm4
-; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm11, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm16, %zmm19
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm15 = [0,5,0,11,0,5,0,11]
+; AVX512BW-NEXT:    # zmm15 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm15, %zmm10
+; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm7, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm1 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm9 = [0,1,2,3,4,5,10,15]
+; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm9, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm15, %zmm4
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm7, %zmm0
 ; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm0 {%k1}
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm8, %zmm0
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, 64(%rsi)
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, (%rsi)
-; AVX512BW-NEXT:    vmovdqa64 %zmm13, 64(%rdx)
-; AVX512BW-NEXT:    vmovdqa64 %zmm12, (%rdx)
-; AVX512BW-NEXT:    vmovdqa64 %zmm16, 64(%rcx)
-; AVX512BW-NEXT:    vmovdqa64 %zmm17, (%rcx)
-; AVX512BW-NEXT:    vmovdqa64 %zmm18, 64(%r8)
-; AVX512BW-NEXT:    vmovdqa64 %zmm19, (%r8)
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm9, %zmm0
+; AVX512BW-NEXT:    vmovdqa64 %zmm12, 64(%rsi)
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, (%rsi)
+; AVX512BW-NEXT:    vmovdqa64 %zmm14, 64(%rdx)
+; AVX512BW-NEXT:    vmovdqa64 %zmm13, (%rdx)
+; AVX512BW-NEXT:    vmovdqa64 %zmm17, 64(%rcx)
+; AVX512BW-NEXT:    vmovdqa64 %zmm18, (%rcx)
+; AVX512BW-NEXT:    vmovdqa64 %zmm19, 64(%r8)
+; AVX512BW-NEXT:    vmovdqa64 %zmm20, (%r8)
 ; AVX512BW-NEXT:    vmovdqa64 %zmm0, 64(%r9)
 ; AVX512BW-NEXT:    vmovdqa64 %zmm1, (%r9)
 ; AVX512BW-NEXT:    vzeroupper
@@ -2559,397 +2557,403 @@ define void @load_i64_stride5_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ;
 ; AVX512F-LABEL: load_i64_stride5_vf32:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    subq $584, %rsp # imm = 0x248
+; AVX512F-NEXT:    subq $648, %rsp # imm = 0x288
 ; AVX512F-NEXT:    vmovdqa64 1088(%rdi), %zmm21
 ; AVX512F-NEXT:    vmovdqa64 1152(%rdi), %zmm1
-; AVX512F-NEXT:    vmovdqa64 768(%rdi), %zmm20
+; AVX512F-NEXT:    vmovdqa64 768(%rdi), %zmm18
 ; AVX512F-NEXT:    vmovdqa64 832(%rdi), %zmm0
-; AVX512F-NEXT:    vmovdqa64 384(%rdi), %zmm14
-; AVX512F-NEXT:    vmovdqa64 448(%rdi), %zmm25
+; AVX512F-NEXT:    vmovdqa64 384(%rdi), %zmm17
+; AVX512F-NEXT:    vmovdqa64 320(%rdi), %zmm19
+; AVX512F-NEXT:    vmovdqa64 448(%rdi), %zmm26
 ; AVX512F-NEXT:    vmovdqa64 512(%rdi), %zmm2
-; AVX512F-NEXT:    vmovdqa64 (%rdi), %zmm22
-; AVX512F-NEXT:    vmovdqa64 64(%rdi), %zmm13
-; AVX512F-NEXT:    vmovdqa64 128(%rdi), %zmm27
+; AVX512F-NEXT:    vmovdqa64 (%rdi), %zmm16
+; AVX512F-NEXT:    vmovdqa64 128(%rdi), %zmm25
 ; AVX512F-NEXT:    vmovdqa64 192(%rdi), %zmm3
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm26 = [12,1,6,0,12,1,6,0]
-; AVX512F-NEXT:    # zmm26 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm11 = [12,1,6,0,12,1,6,0]
+; AVX512F-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3]
 ; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm10
-; AVX512F-NEXT:    vpermt2q %zmm27, %zmm26, %zmm10
+; AVX512F-NEXT:    vpermt2q %zmm25, %zmm11, %zmm10
 ; AVX512F-NEXT:    vmovdqa64 {{.*#+}} ymm24 = [0,5,10,15]
-; AVX512F-NEXT:    vmovdqa64 %zmm22, %zmm11
-; AVX512F-NEXT:    vpermt2q %zmm13, %zmm24, %zmm11
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm12
 ; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm4
-; AVX512F-NEXT:    vpermt2q %zmm25, %zmm26, %zmm4
-; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm26, %zmm11, %zmm12
+; AVX512F-NEXT:    vmovdqa64 %zmm19, %zmm2
+; AVX512F-NEXT:    vpermt2q %zmm17, %zmm24, %zmm2
+; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm5
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm4
-; AVX512F-NEXT:    vpermt2q %zmm20, %zmm26, %zmm5
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm2
+; AVX512F-NEXT:    vpermt2q %zmm18, %zmm11, %zmm5
 ; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermi2q %zmm21, %zmm1, %zmm26
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm21, %zmm11, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vmovdqa64 {{.*#+}} ymm28 = <1,6,11,u>
-; AVX512F-NEXT:    vmovdqa64 %zmm22, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm13, %zmm28, %zmm0
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm15 = [5,10,15,0,5,10,15,0]
+; AVX512F-NEXT:    # zmm15 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm25, %zmm30
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm15, %zmm30
+; AVX512F-NEXT:    vmovdqa64 %zmm18, %zmm27
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm15, %zmm27
+; AVX512F-NEXT:    vmovdqa64 %zmm26, %zmm22
+; AVX512F-NEXT:    vpermt2q %zmm4, %zmm15, %zmm22
+; AVX512F-NEXT:    vmovdqa64 %zmm19, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm17, %zmm28, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm12 = [5,10,15,0,5,10,15,0]
-; AVX512F-NEXT:    # zmm12 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm27, %zmm30
-; AVX512F-NEXT:    vpermt2q %zmm3, %zmm12, %zmm30
-; AVX512F-NEXT:    vmovdqa64 %zmm20, %zmm19
-; AVX512F-NEXT:    vpermt2q %zmm4, %zmm12, %zmm19
-; AVX512F-NEXT:    vmovdqa64 %zmm25, %zmm18
-; AVX512F-NEXT:    vpermt2q %zmm2, %zmm12, %zmm18
-; AVX512F-NEXT:    vpermi2q %zmm1, %zmm21, %zmm12
+; AVX512F-NEXT:    vpermi2q %zmm1, %zmm21, %zmm15
 ; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm23 = [6,11,0,1,6,11,0,1]
 ; AVX512F-NEXT:    # zmm23 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm27, %zmm0
+; AVX512F-NEXT:    vmovdqa64 %zmm25, %zmm0
 ; AVX512F-NEXT:    vpermt2q %zmm3, %zmm23, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm15 = <2,7,12,u>
-; AVX512F-NEXT:    vmovdqa64 %zmm22, %zmm16
-; AVX512F-NEXT:    vpermt2q %zmm13, %zmm15, %zmm16
-; AVX512F-NEXT:    vmovdqa64 %zmm20, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm4, %zmm23, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm25, %zmm0
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm13 = <2,7,12,u>
+; AVX512F-NEXT:    vmovdqa64 %zmm18, %zmm0
 ; AVX512F-NEXT:    vpermt2q %zmm2, %zmm23, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm26, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm4, %zmm23, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vpermi2q %zmm1, %zmm21, %zmm23
 ; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm29 = [7,12,0,2,7,12,0,2]
 ; AVX512F-NEXT:    # zmm29 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm27, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm3, %zmm29, %zmm0
+; AVX512F-NEXT:    vmovdqa64 %zmm26, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm4, %zmm29, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [0,5,0,11,0,5,0,11]
 ; AVX512F-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermt2q %zmm27, %zmm0, %zmm3
-; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm25, %zmm27
-; AVX512F-NEXT:    vpermt2q %zmm2, %zmm29, %zmm27
-; AVX512F-NEXT:    vpermt2q %zmm25, %zmm0, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, (%rsp) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm20, %zmm17
-; AVX512F-NEXT:    vpermt2q %zmm4, %zmm29, %zmm17
+; AVX512F-NEXT:    vpermt2q %zmm26, %zmm0, %zmm4
+; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm25, %zmm26
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm29, %zmm26
+; AVX512F-NEXT:    vpermt2q %zmm25, %zmm0, %zmm3
+; AVX512F-NEXT:    vmovdqu64 %zmm3, (%rsp) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm18, %zmm20
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm29, %zmm20
 ; AVX512F-NEXT:    vpermi2q %zmm1, %zmm21, %zmm29
 ; AVX512F-NEXT:    vpermt2q %zmm21, %zmm0, %zmm1
 ; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 {{.*#+}} ymm21 = <11,0,5,u>
-; AVX512F-NEXT:    vpermt2q %zmm20, %zmm0, %zmm4
-; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm20
-; AVX512F-NEXT:    vpermt2q %zmm22, %zmm21, %zmm20
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm9 = <12,1,6,u>
-; AVX512F-NEXT:    vpermt2q %zmm22, %zmm9, %zmm13
-; AVX512F-NEXT:    vmovdqa64 320(%rdi), %zmm0
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm7
-; AVX512F-NEXT:    vpermt2q %zmm14, %zmm24, %zmm7
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm8
-; AVX512F-NEXT:    vpermt2q %zmm14, %zmm28, %zmm8
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm31
-; AVX512F-NEXT:    vpermt2q %zmm14, %zmm15, %zmm31
-; AVX512F-NEXT:    vmovdqa64 %zmm14, %zmm22
-; AVX512F-NEXT:    vpermt2q %zmm0, %zmm21, %zmm22
-; AVX512F-NEXT:    vpermt2q %zmm0, %zmm9, %zmm14
-; AVX512F-NEXT:    vmovdqa64 704(%rdi), %zmm25
+; AVX512F-NEXT:    vmovdqa64 %zmm19, %zmm21
+; AVX512F-NEXT:    vpermt2q %zmm17, %zmm13, %zmm21
+; AVX512F-NEXT:    vpermt2q %zmm18, %zmm0, %zmm2
+; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} ymm31 = <11,0,5,u>
+; AVX512F-NEXT:    vmovdqa64 %zmm17, %zmm25
+; AVX512F-NEXT:    vpermt2q %zmm19, %zmm31, %zmm25
+; AVX512F-NEXT:    vpermt2q %zmm19, %zmm11, %zmm17
+; AVX512F-NEXT:    vmovdqa64 64(%rdi), %zmm18
+; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm6
+; AVX512F-NEXT:    vpermt2q %zmm18, %zmm24, %zmm6
+; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm9
+; AVX512F-NEXT:    vpermt2q %zmm18, %zmm28, %zmm9
+; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm8
+; AVX512F-NEXT:    vpermt2q %zmm18, %zmm13, %zmm8
+; AVX512F-NEXT:    vmovdqa64 %zmm18, %zmm19
+; AVX512F-NEXT:    vpermt2q %zmm16, %zmm31, %zmm19
+; AVX512F-NEXT:    vpermt2q %zmm16, %zmm11, %zmm18
+; AVX512F-NEXT:    vmovdqa64 704(%rdi), %zmm16
 ; AVX512F-NEXT:    vmovdqa64 640(%rdi), %zmm1
 ; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm3
-; AVX512F-NEXT:    vpermt2q %zmm25, %zmm24, %zmm3
+; AVX512F-NEXT:    vpermt2q %zmm16, %zmm24, %zmm3
 ; AVX512F-NEXT:    vmovdqa64 1024(%rdi), %zmm2
 ; AVX512F-NEXT:    vmovdqa64 960(%rdi), %zmm0
 ; AVX512F-NEXT:    vpermi2q %zmm2, %zmm0, %zmm24
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm6
-; AVX512F-NEXT:    vpermt2q %zmm25, %zmm28, %zmm6
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm7
+; AVX512F-NEXT:    vpermt2q %zmm16, %zmm28, %zmm7
 ; AVX512F-NEXT:    vpermi2q %zmm2, %zmm0, %zmm28
 ; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm4
-; AVX512F-NEXT:    vpermt2q %zmm25, %zmm15, %zmm4
-; AVX512F-NEXT:    vpermi2q %zmm2, %zmm0, %zmm15
-; AVX512F-NEXT:    vmovdqa64 %zmm25, %zmm5
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm21, %zmm5
-; AVX512F-NEXT:    vpermi2q %zmm0, %zmm2, %zmm21
-; AVX512F-NEXT:    vpermt2q %zmm0, %zmm9, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm9, %zmm25
-; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm11 = zmm11[0,1,2,3],zmm10[4,5,6,7]
-; AVX512F-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm7, %zmm10 # 64-byte Folded Reload
-; AVX512F-NEXT:    # zmm10 = zmm7[0,1,2,3],mem[4,5,6,7]
-; AVX512F-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm3 # 64-byte Folded Reload
-; AVX512F-NEXT:    # zmm3 = zmm3[0,1,2,3],mem[4,5,6,7]
-; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm7 = zmm24[0,1,2,3],zmm26[4,5,6,7]
-; AVX512F-NEXT:    vmovdqa64 256(%rdi), %zmm9
+; AVX512F-NEXT:    vpermt2q %zmm16, %zmm13, %zmm4
+; AVX512F-NEXT:    vpermi2q %zmm2, %zmm0, %zmm13
+; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm5
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm31, %zmm5
+; AVX512F-NEXT:    vpermi2q %zmm0, %zmm2, %zmm31
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm11, %zmm2
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm11, %zmm16
+; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm14 = zmm6[0,1,2,3],zmm10[4,5,6,7]
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm12 = zmm0[0,1,2,3],zmm12[4,5,6,7]
+; AVX512F-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm10 # 64-byte Folded Reload
+; AVX512F-NEXT:    # zmm10 = zmm3[0,1,2,3],mem[4,5,6,7]
+; AVX512F-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm24, %zmm6 # 64-byte Folded Reload
+; AVX512F-NEXT:    # zmm6 = zmm24[0,1,2,3],mem[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 256(%rdi), %zmm11
 ; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm24 = [0,1,2,3,4,5,6,11]
-; AVX512F-NEXT:    vpermt2q %zmm9, %zmm24, %zmm11
-; AVX512F-NEXT:    vmovdqa64 576(%rdi), %zmm26
-; AVX512F-NEXT:    vpermt2q %zmm26, %zmm24, %zmm10
-; AVX512F-NEXT:    vmovdqa64 896(%rdi), %zmm1
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm24, %zmm3
-; AVX512F-NEXT:    vmovdqa64 1216(%rdi), %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm0, %zmm24, %zmm7
+; AVX512F-NEXT:    vpermt2q %zmm11, %zmm24, %zmm14
+; AVX512F-NEXT:    vmovdqa64 576(%rdi), %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm24, %zmm12
+; AVX512F-NEXT:    vmovdqa64 896(%rdi), %zmm3
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm24, %zmm10
+; AVX512F-NEXT:    vmovdqa64 1216(%rdi), %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm24, %zmm6
 ; AVX512F-NEXT:    movb $7, %al
 ; AVX512F-NEXT:    kmovw %eax, %k1
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm24 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm24, %zmm30 {%k1}
-; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm19 {%k1}
-; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm18 {%k1}
-; AVX512F-NEXT:    vmovdqa64 %zmm28, %zmm12 {%k1}
-; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [0,1,2,3,4,5,6,12]
-; AVX512F-NEXT:    vpermt2q %zmm9, %zmm6, %zmm30
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm6, %zmm19
-; AVX512F-NEXT:    vpermt2q %zmm26, %zmm6, %zmm18
-; AVX512F-NEXT:    vpermt2q %zmm0, %zmm6, %zmm12
+; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm30 {%k1}
+; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm27 {%k1}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm22 {%k1}
+; AVX512F-NEXT:    vmovdqa64 %zmm28, %zmm15 {%k1}
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm7 = [0,1,2,3,4,5,6,12]
+; AVX512F-NEXT:    vpermt2q %zmm11, %zmm7, %zmm30
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm7, %zmm27
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm7, %zmm22
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm7, %zmm15
 ; AVX512F-NEXT:    movb $56, %al
 ; AVX512F-NEXT:    kmovw %eax, %k1
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm16 {%k1}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm4 {%k1}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm31 {%k1}
-; AVX512F-NEXT:    vmovdqa64 %zmm23, %zmm15 {%k1}
-; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [0,1,2,3,4,5,8,13]
-; AVX512F-NEXT:    vpermt2q %zmm9, %zmm6, %zmm16
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm6, %zmm4
-; AVX512F-NEXT:    vpermt2q %zmm26, %zmm6, %zmm31
-; AVX512F-NEXT:    vpermt2q %zmm0, %zmm6, %zmm15
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm20 {%k1}
-; AVX512F-NEXT:    vmovdqa64 %zmm17, %zmm5 {%k1}
-; AVX512F-NEXT:    vmovdqa64 %zmm27, %zmm22 {%k1}
-; AVX512F-NEXT:    vmovdqa64 %zmm29, %zmm21 {%k1}
-; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [0,1,2,3,4,5,9,14]
-; AVX512F-NEXT:    vpermt2q %zmm9, %zmm6, %zmm20
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm6, %zmm5
-; AVX512F-NEXT:    vpermt2q %zmm26, %zmm6, %zmm22
-; AVX512F-NEXT:    vpermt2q %zmm0, %zmm6, %zmm21
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm13 {%k1}
-; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [0,1,2,3,4,5,10,15]
-; AVX512F-NEXT:    vpermt2q %zmm9, %zmm6, %zmm13
-; AVX512F-NEXT:    vmovdqu64 (%rsp), %zmm8 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm14 {%k1}
-; AVX512F-NEXT:    vpermt2q %zmm26, %zmm6, %zmm14
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm2 {%k1}
-; AVX512F-NEXT:    vpermt2q %zmm0, %zmm6, %zmm2
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm8 {%k1}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm4 {%k1}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm21 {%k1}
+; AVX512F-NEXT:    vmovdqa64 %zmm23, %zmm13 {%k1}
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm7 = [0,1,2,3,4,5,8,13]
+; AVX512F-NEXT:    vpermt2q %zmm11, %zmm7, %zmm8
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm7, %zmm4
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm7, %zmm21
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm7, %zmm13
+; AVX512F-NEXT:    vmovdqa64 %zmm26, %zmm19 {%k1}
+; AVX512F-NEXT:    vmovdqa64 %zmm20, %zmm5 {%k1}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm25 {%k1}
+; AVX512F-NEXT:    vmovdqa64 %zmm29, %zmm31 {%k1}
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm7 = [0,1,2,3,4,5,9,14]
+; AVX512F-NEXT:    vpermt2q %zmm11, %zmm7, %zmm19
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm7, %zmm5
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm7, %zmm25
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm7, %zmm31
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm17 {%k1}
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm7 = [0,1,2,3,4,5,10,15]
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm7, %zmm17
+; AVX512F-NEXT:    vmovdqu64 (%rsp), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm18 {%k1}
+; AVX512F-NEXT:    vpermt2q %zmm11, %zmm7, %zmm18
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm25 {%k1}
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm6, %zmm25
-; AVX512F-NEXT:    vmovdqa64 %zmm7, 192(%rsi)
-; AVX512F-NEXT:    vmovdqa64 %zmm3, 128(%rsi)
-; AVX512F-NEXT:    vmovdqa64 %zmm10, 64(%rsi)
-; AVX512F-NEXT:    vmovdqa64 %zmm11, (%rsi)
-; AVX512F-NEXT:    vmovdqa64 %zmm12, 192(%rdx)
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k1}
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm7, %zmm2
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm16 {%k1}
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm7, %zmm16
+; AVX512F-NEXT:    vmovdqa64 %zmm6, 192(%rsi)
+; AVX512F-NEXT:    vmovdqa64 %zmm10, 128(%rsi)
+; AVX512F-NEXT:    vmovdqa64 %zmm12, 64(%rsi)
+; AVX512F-NEXT:    vmovdqa64 %zmm14, (%rsi)
+; AVX512F-NEXT:    vmovdqa64 %zmm15, 192(%rdx)
 ; AVX512F-NEXT:    vmovdqa64 %zmm30, (%rdx)
-; AVX512F-NEXT:    vmovdqa64 %zmm18, 64(%rdx)
-; AVX512F-NEXT:    vmovdqa64 %zmm19, 128(%rdx)
-; AVX512F-NEXT:    vmovdqa64 %zmm15, 192(%rcx)
-; AVX512F-NEXT:    vmovdqa64 %zmm16, (%rcx)
-; AVX512F-NEXT:    vmovdqa64 %zmm31, 64(%rcx)
+; AVX512F-NEXT:    vmovdqa64 %zmm22, 64(%rdx)
+; AVX512F-NEXT:    vmovdqa64 %zmm27, 128(%rdx)
+; AVX512F-NEXT:    vmovdqa64 %zmm13, 192(%rcx)
+; AVX512F-NEXT:    vmovdqa64 %zmm8, (%rcx)
+; AVX512F-NEXT:    vmovdqa64 %zmm21, 64(%rcx)
 ; AVX512F-NEXT:    vmovdqa64 %zmm4, 128(%rcx)
-; AVX512F-NEXT:    vmovdqa64 %zmm21, 192(%r8)
-; AVX512F-NEXT:    vmovdqa64 %zmm20, (%r8)
-; AVX512F-NEXT:    vmovdqa64 %zmm22, 64(%r8)
+; AVX512F-NEXT:    vmovdqa64 %zmm31, 192(%r8)
+; AVX512F-NEXT:    vmovdqa64 %zmm19, (%r8)
+; AVX512F-NEXT:    vmovdqa64 %zmm25, 64(%r8)
 ; AVX512F-NEXT:    vmovdqa64 %zmm5, 128(%r8)
-; AVX512F-NEXT:    vmovdqa64 %zmm25, 128(%r9)
+; AVX512F-NEXT:    vmovdqa64 %zmm16, 128(%r9)
 ; AVX512F-NEXT:    vmovdqa64 %zmm2, 192(%r9)
-; AVX512F-NEXT:    vmovdqa64 %zmm13, (%r9)
-; AVX512F-NEXT:    vmovdqa64 %zmm14, 64(%r9)
-; AVX512F-NEXT:    addq $584, %rsp # imm = 0x248
+; AVX512F-NEXT:    vmovdqa64 %zmm18, (%r9)
+; AVX512F-NEXT:    vmovdqa64 %zmm17, 64(%r9)
+; AVX512F-NEXT:    addq $648, %rsp # imm = 0x288
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: load_i64_stride5_vf32:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    subq $584, %rsp # imm = 0x248
+; AVX512BW-NEXT:    subq $648, %rsp # imm = 0x288
 ; AVX512BW-NEXT:    vmovdqa64 1088(%rdi), %zmm21
 ; AVX512BW-NEXT:    vmovdqa64 1152(%rdi), %zmm1
-; AVX512BW-NEXT:    vmovdqa64 768(%rdi), %zmm20
+; AVX512BW-NEXT:    vmovdqa64 768(%rdi), %zmm18
 ; AVX512BW-NEXT:    vmovdqa64 832(%rdi), %zmm0
-; AVX512BW-NEXT:    vmovdqa64 384(%rdi), %zmm14
-; AVX512BW-NEXT:    vmovdqa64 448(%rdi), %zmm25
+; AVX512BW-NEXT:    vmovdqa64 384(%rdi), %zmm17
+; AVX512BW-NEXT:    vmovdqa64 320(%rdi), %zmm19
+; AVX512BW-NEXT:    vmovdqa64 448(%rdi), %zmm26
 ; AVX512BW-NEXT:    vmovdqa64 512(%rdi), %zmm2
-; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm22
-; AVX512BW-NEXT:    vmovdqa64 64(%rdi), %zmm13
-; AVX512BW-NEXT:    vmovdqa64 128(%rdi), %zmm27
+; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm16
+; AVX512BW-NEXT:    vmovdqa64 128(%rdi), %zmm25
 ; AVX512BW-NEXT:    vmovdqa64 192(%rdi), %zmm3
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm26 = [12,1,6,0,12,1,6,0]
-; AVX512BW-NEXT:    # zmm26 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm11 = [12,1,6,0,12,1,6,0]
+; AVX512BW-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3]
 ; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm10
-; AVX512BW-NEXT:    vpermt2q %zmm27, %zmm26, %zmm10
+; AVX512BW-NEXT:    vpermt2q %zmm25, %zmm11, %zmm10
 ; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} ymm24 = [0,5,10,15]
-; AVX512BW-NEXT:    vmovdqa64 %zmm22, %zmm11
-; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm24, %zmm11
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm12
 ; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm4
-; AVX512BW-NEXT:    vpermt2q %zmm25, %zmm26, %zmm4
-; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm26, %zmm11, %zmm12
+; AVX512BW-NEXT:    vmovdqa64 %zmm19, %zmm2
+; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm24, %zmm2
+; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm5
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm4
-; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm26, %zmm5
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm2
+; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm11, %zmm5
 ; AVX512BW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermi2q %zmm21, %zmm1, %zmm26
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm21, %zmm11, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} ymm28 = <1,6,11,u>
-; AVX512BW-NEXT:    vmovdqa64 %zmm22, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm28, %zmm0
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm15 = [5,10,15,0,5,10,15,0]
+; AVX512BW-NEXT:    # zmm15 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm25, %zmm30
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm15, %zmm30
+; AVX512BW-NEXT:    vmovdqa64 %zmm18, %zmm27
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm15, %zmm27
+; AVX512BW-NEXT:    vmovdqa64 %zmm26, %zmm22
+; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm15, %zmm22
+; AVX512BW-NEXT:    vmovdqa64 %zmm19, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm28, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm12 = [5,10,15,0,5,10,15,0]
-; AVX512BW-NEXT:    # zmm12 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm27, %zmm30
-; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm12, %zmm30
-; AVX512BW-NEXT:    vmovdqa64 %zmm20, %zmm19
-; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm12, %zmm19
-; AVX512BW-NEXT:    vmovdqa64 %zmm25, %zmm18
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm12, %zmm18
-; AVX512BW-NEXT:    vpermi2q %zmm1, %zmm21, %zmm12
+; AVX512BW-NEXT:    vpermi2q %zmm1, %zmm21, %zmm15
 ; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm23 = [6,11,0,1,6,11,0,1]
 ; AVX512BW-NEXT:    # zmm23 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm27, %zmm0
+; AVX512BW-NEXT:    vmovdqa64 %zmm25, %zmm0
 ; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm23, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm15 = <2,7,12,u>
-; AVX512BW-NEXT:    vmovdqa64 %zmm22, %zmm16
-; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm15, %zmm16
-; AVX512BW-NEXT:    vmovdqa64 %zmm20, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm23, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm25, %zmm0
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm13 = <2,7,12,u>
+; AVX512BW-NEXT:    vmovdqa64 %zmm18, %zmm0
 ; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm23, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm26, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm23, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vpermi2q %zmm1, %zmm21, %zmm23
 ; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm29 = [7,12,0,2,7,12,0,2]
 ; AVX512BW-NEXT:    # zmm29 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm27, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm29, %zmm0
+; AVX512BW-NEXT:    vmovdqa64 %zmm26, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm29, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [0,5,0,11,0,5,0,11]
 ; AVX512BW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermt2q %zmm27, %zmm0, %zmm3
-; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm25, %zmm27
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm29, %zmm27
-; AVX512BW-NEXT:    vpermt2q %zmm25, %zmm0, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, (%rsp) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm20, %zmm17
-; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm29, %zmm17
+; AVX512BW-NEXT:    vpermt2q %zmm26, %zmm0, %zmm4
+; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm25, %zmm26
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm29, %zmm26
+; AVX512BW-NEXT:    vpermt2q %zmm25, %zmm0, %zmm3
+; AVX512BW-NEXT:    vmovdqu64 %zmm3, (%rsp) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm18, %zmm20
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm29, %zmm20
 ; AVX512BW-NEXT:    vpermi2q %zmm1, %zmm21, %zmm29
 ; AVX512BW-NEXT:    vpermt2q %zmm21, %zmm0, %zmm1
 ; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} ymm21 = <11,0,5,u>
-; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm0, %zmm4
-; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm20
-; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm21, %zmm20
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm9 = <12,1,6,u>
-; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm9, %zmm13
-; AVX512BW-NEXT:    vmovdqa64 320(%rdi), %zmm0
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm7
-; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm24, %zmm7
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm8
-; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm28, %zmm8
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm31
-; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm15, %zmm31
-; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm22
-; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm21, %zmm22
-; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm9, %zmm14
-; AVX512BW-NEXT:    vmovdqa64 704(%rdi), %zmm25
+; AVX512BW-NEXT:    vmovdqa64 %zmm19, %zmm21
+; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm13, %zmm21
+; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm0, %zmm2
+; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} ymm31 = <11,0,5,u>
+; AVX512BW-NEXT:    vmovdqa64 %zmm17, %zmm25
+; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm31, %zmm25
+; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm11, %zmm17
+; AVX512BW-NEXT:    vmovdqa64 64(%rdi), %zmm18
+; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm6
+; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm24, %zmm6
+; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm9
+; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm28, %zmm9
+; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm8
+; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm13, %zmm8
+; AVX512BW-NEXT:    vmovdqa64 %zmm18, %zmm19
+; AVX512BW-NEXT:    vpermt2q %zmm16, %zmm31, %zmm19
+; AVX512BW-NEXT:    vpermt2q %zmm16, %zmm11, %zmm18
+; AVX512BW-NEXT:    vmovdqa64 704(%rdi), %zmm16
 ; AVX512BW-NEXT:    vmovdqa64 640(%rdi), %zmm1
 ; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm3
-; AVX512BW-NEXT:    vpermt2q %zmm25, %zmm24, %zmm3
+; AVX512BW-NEXT:    vpermt2q %zmm16, %zmm24, %zmm3
 ; AVX512BW-NEXT:    vmovdqa64 1024(%rdi), %zmm2
 ; AVX512BW-NEXT:    vmovdqa64 960(%rdi), %zmm0
 ; AVX512BW-NEXT:    vpermi2q %zmm2, %zmm0, %zmm24
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm6
-; AVX512BW-NEXT:    vpermt2q %zmm25, %zmm28, %zmm6
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm7
+; AVX512BW-NEXT:    vpermt2q %zmm16, %zmm28, %zmm7
 ; AVX512BW-NEXT:    vpermi2q %zmm2, %zmm0, %zmm28
 ; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm4
-; AVX512BW-NEXT:    vpermt2q %zmm25, %zmm15, %zmm4
-; AVX512BW-NEXT:    vpermi2q %zmm2, %zmm0, %zmm15
-; AVX512BW-NEXT:    vmovdqa64 %zmm25, %zmm5
-; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm21, %zmm5
-; AVX512BW-NEXT:    vpermi2q %zmm0, %zmm2, %zmm21
-; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm9, %zmm2
-; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm9, %zmm25
-; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm11 = zmm11[0,1,2,3],zmm10[4,5,6,7]
-; AVX512BW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm7, %zmm10 # 64-byte Folded Reload
-; AVX512BW-NEXT:    # zmm10 = zmm7[0,1,2,3],mem[4,5,6,7]
-; AVX512BW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm3 # 64-byte Folded Reload
-; AVX512BW-NEXT:    # zmm3 = zmm3[0,1,2,3],mem[4,5,6,7]
-; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm7 = zmm24[0,1,2,3],zmm26[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqa64 256(%rdi), %zmm9
+; AVX512BW-NEXT:    vpermt2q %zmm16, %zmm13, %zmm4
+; AVX512BW-NEXT:    vpermi2q %zmm2, %zmm0, %zmm13
+; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm5
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm31, %zmm5
+; AVX512BW-NEXT:    vpermi2q %zmm0, %zmm2, %zmm31
+; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm11, %zmm2
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm11, %zmm16
+; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm14 = zmm6[0,1,2,3],zmm10[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm12 = zmm0[0,1,2,3],zmm12[4,5,6,7]
+; AVX512BW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm10 # 64-byte Folded Reload
+; AVX512BW-NEXT:    # zmm10 = zmm3[0,1,2,3],mem[4,5,6,7]
+; AVX512BW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm24, %zmm6 # 64-byte Folded Reload
+; AVX512BW-NEXT:    # zmm6 = zmm24[0,1,2,3],mem[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 256(%rdi), %zmm11
 ; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm24 = [0,1,2,3,4,5,6,11]
-; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm24, %zmm11
-; AVX512BW-NEXT:    vmovdqa64 576(%rdi), %zmm26
-; AVX512BW-NEXT:    vpermt2q %zmm26, %zmm24, %zmm10
-; AVX512BW-NEXT:    vmovdqa64 896(%rdi), %zmm1
-; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm24, %zmm3
-; AVX512BW-NEXT:    vmovdqa64 1216(%rdi), %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm24, %zmm7
+; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm24, %zmm14
+; AVX512BW-NEXT:    vmovdqa64 576(%rdi), %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm24, %zmm12
+; AVX512BW-NEXT:    vmovdqa64 896(%rdi), %zmm3
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm24, %zmm10
+; AVX512BW-NEXT:    vmovdqa64 1216(%rdi), %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm24, %zmm6
 ; AVX512BW-NEXT:    movb $7, %al
 ; AVX512BW-NEXT:    kmovd %eax, %k1
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm24 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm24, %zmm30 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm19 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm18 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 %zmm28, %zmm12 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [0,1,2,3,4,5,6,12]
-; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm6, %zmm30
-; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm6, %zmm19
-; AVX512BW-NEXT:    vpermt2q %zmm26, %zmm6, %zmm18
-; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm6, %zmm12
+; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm30 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm27 {%k1}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm22 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm28, %zmm15 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm7 = [0,1,2,3,4,5,6,12]
+; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm7, %zmm30
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm7, %zmm27
+; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm7, %zmm22
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm7, %zmm15
 ; AVX512BW-NEXT:    movb $56, %al
 ; AVX512BW-NEXT:    kmovd %eax, %k1
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm16 {%k1}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm4 {%k1}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm31 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 %zmm23, %zmm15 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [0,1,2,3,4,5,8,13]
-; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm6, %zmm16
-; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm6, %zmm4
-; AVX512BW-NEXT:    vpermt2q %zmm26, %zmm6, %zmm31
-; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm6, %zmm15
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm20 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 %zmm17, %zmm5 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 %zmm27, %zmm22 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 %zmm29, %zmm21 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [0,1,2,3,4,5,9,14]
-; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm6, %zmm20
-; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm6, %zmm5
-; AVX512BW-NEXT:    vpermt2q %zmm26, %zmm6, %zmm22
-; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm6, %zmm21
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm13 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [0,1,2,3,4,5,10,15]
-; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm6, %zmm13
-; AVX512BW-NEXT:    vmovdqu64 (%rsp), %zmm8 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm14 {%k1}
-; AVX512BW-NEXT:    vpermt2q %zmm26, %zmm6, %zmm14
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm2 {%k1}
-; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm6, %zmm2
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm8 {%k1}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm4 {%k1}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm21 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm23, %zmm13 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm7 = [0,1,2,3,4,5,8,13]
+; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm7, %zmm8
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm7, %zmm4
+; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm7, %zmm21
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm7, %zmm13
+; AVX512BW-NEXT:    vmovdqa64 %zmm26, %zmm19 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm20, %zmm5 {%k1}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm25 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm29, %zmm31 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm7 = [0,1,2,3,4,5,9,14]
+; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm7, %zmm19
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm7, %zmm5
+; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm7, %zmm25
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm7, %zmm31
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm17 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm7 = [0,1,2,3,4,5,10,15]
+; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm7, %zmm17
+; AVX512BW-NEXT:    vmovdqu64 (%rsp), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm18 {%k1}
+; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm7, %zmm18
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k1}
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm7, %zmm2
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm25 {%k1}
-; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm6, %zmm25
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, 192(%rsi)
-; AVX512BW-NEXT:    vmovdqa64 %zmm3, 128(%rsi)
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, 64(%rsi)
-; AVX512BW-NEXT:    vmovdqa64 %zmm11, (%rsi)
-; AVX512BW-NEXT:    vmovdqa64 %zmm12, 192(%rdx)
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm16 {%k1}
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm7, %zmm16
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, 192(%rsi)
+; AVX512BW-NEXT:    vmovdqa64 %zmm10, 128(%rsi)
+; AVX512BW-NEXT:    vmovdqa64 %zmm12, 64(%rsi)
+; AVX512BW-NEXT:    vmovdqa64 %zmm14, (%rsi)
+; AVX512BW-NEXT:    vmovdqa64 %zmm15, 192(%rdx)
 ; AVX512BW-NEXT:    vmovdqa64 %zmm30, (%rdx)
-; AVX512BW-NEXT:    vmovdqa64 %zmm18, 64(%rdx)
-; AVX512BW-NEXT:    vmovdqa64 %zmm19, 128(%rdx)
-; AVX512BW-NEXT:    vmovdqa64 %zmm15, 192(%rcx)
-; AVX512BW-NEXT:    vmovdqa64 %zmm16, (%rcx)
-; AVX512BW-NEXT:    vmovdqa64 %zmm31, 64(%rcx)
+; AVX512BW-NEXT:    vmovdqa64 %zmm22, 64(%rdx)
+; AVX512BW-NEXT:    vmovdqa64 %zmm27, 128(%rdx)
+; AVX512BW-NEXT:    vmovdqa64 %zmm13, 192(%rcx)
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, (%rcx)
+; AVX512BW-NEXT:    vmovdqa64 %zmm21, 64(%rcx)
 ; AVX512BW-NEXT:    vmovdqa64 %zmm4, 128(%rcx)
-; AVX512BW-NEXT:    vmovdqa64 %zmm21, 192(%r8)
-; AVX512BW-NEXT:    vmovdqa64 %zmm20, (%r8)
-; AVX512BW-NEXT:    vmovdqa64 %zmm22, 64(%r8)
+; AVX512BW-NEXT:    vmovdqa64 %zmm31, 192(%r8)
+; AVX512BW-NEXT:    vmovdqa64 %zmm19, (%r8)
+; AVX512BW-NEXT:    vmovdqa64 %zmm25, 64(%r8)
 ; AVX512BW-NEXT:    vmovdqa64 %zmm5, 128(%r8)
-; AVX512BW-NEXT:    vmovdqa64 %zmm25, 128(%r9)
+; AVX512BW-NEXT:    vmovdqa64 %zmm16, 128(%r9)
 ; AVX512BW-NEXT:    vmovdqa64 %zmm2, 192(%r9)
-; AVX512BW-NEXT:    vmovdqa64 %zmm13, (%r9)
-; AVX512BW-NEXT:    vmovdqa64 %zmm14, 64(%r9)
-; AVX512BW-NEXT:    addq $584, %rsp # imm = 0x248
+; AVX512BW-NEXT:    vmovdqa64 %zmm18, (%r9)
+; AVX512BW-NEXT:    vmovdqa64 %zmm17, 64(%r9)
+; AVX512BW-NEXT:    addq $648, %rsp # imm = 0x288
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
   %wide.vec = load <160 x i64>, ptr %in.vec, align 64
@@ -5486,292 +5490,291 @@ define void @load_i64_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ;
 ; AVX512F-LABEL: load_i64_stride5_vf64:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    subq $3400, %rsp # imm = 0xD48
-; AVX512F-NEXT:    vmovdqa64 1728(%rdi), %zmm21
-; AVX512F-NEXT:    vmovdqa64 1792(%rdi), %zmm4
-; AVX512F-NEXT:    vmovdqa64 1408(%rdi), %zmm19
-; AVX512F-NEXT:    vmovdqa64 1088(%rdi), %zmm0
-; AVX512F-NEXT:    vmovdqa64 1152(%rdi), %zmm3
-; AVX512F-NEXT:    vmovdqa64 768(%rdi), %zmm26
-; AVX512F-NEXT:    vmovdqa64 832(%rdi), %zmm5
-; AVX512F-NEXT:    vmovdqa64 448(%rdi), %zmm1
-; AVX512F-NEXT:    vmovdqa64 512(%rdi), %zmm6
+; AVX512F-NEXT:    subq $3336, %rsp # imm = 0xD08
+; AVX512F-NEXT:    vmovdqa64 1728(%rdi), %zmm3
+; AVX512F-NEXT:    vmovdqa64 1792(%rdi), %zmm7
+; AVX512F-NEXT:    vmovdqa64 1408(%rdi), %zmm0
+; AVX512F-NEXT:    vmovdqa64 1088(%rdi), %zmm1
+; AVX512F-NEXT:    vmovdqa64 1152(%rdi), %zmm6
+; AVX512F-NEXT:    vmovdqa64 768(%rdi), %zmm5
+; AVX512F-NEXT:    vmovdqa64 832(%rdi), %zmm8
+; AVX512F-NEXT:    vmovdqa64 448(%rdi), %zmm4
+; AVX512F-NEXT:    vmovdqa64 512(%rdi), %zmm9
 ; AVX512F-NEXT:    vmovdqa64 128(%rdi), %zmm2
-; AVX512F-NEXT:    vmovdqa64 192(%rdi), %zmm7
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm15 = [12,1,6,0,12,1,6,0]
-; AVX512F-NEXT:    # zmm15 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm16
-; AVX512F-NEXT:    vpermt2q %zmm2, %zmm15, %zmm16
-; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm8
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm15, %zmm8
-; AVX512F-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm8
-; AVX512F-NEXT:    vpermt2q %zmm26, %zmm15, %zmm8
-; AVX512F-NEXT:    vmovdqu64 %zmm8, (%rsp) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm9
-; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm8
-; AVX512F-NEXT:    vpermt2q %zmm0, %zmm15, %zmm9
+; AVX512F-NEXT:    vmovdqa64 192(%rdi), %zmm10
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm16 = [12,1,6,0,12,1,6,0]
+; AVX512F-NEXT:    # zmm16 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm11
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm16, %zmm11
+; AVX512F-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm11
+; AVX512F-NEXT:    vpermt2q %zmm4, %zmm16, %zmm11
+; AVX512F-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm11
+; AVX512F-NEXT:    vpermt2q %zmm5, %zmm16, %zmm11
+; AVX512F-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm12
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm11
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm16, %zmm12
+; AVX512F-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm6
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm16, %zmm6
+; AVX512F-NEXT:    vmovdqu64 %zmm6, (%rsp) # 64-byte Spill
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm12 = [5,10,15,0,5,10,15,0]
+; AVX512F-NEXT:    # zmm12 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm6
+; AVX512F-NEXT:    vpermt2q %zmm10, %zmm12, %zmm6
+; AVX512F-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm6
+; AVX512F-NEXT:    vpermt2q %zmm9, %zmm12, %zmm6
+; AVX512F-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm6
+; AVX512F-NEXT:    vpermt2q %zmm11, %zmm12, %zmm6
+; AVX512F-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm6
+; AVX512F-NEXT:    vpermt2q %zmm8, %zmm12, %zmm6
+; AVX512F-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm6
+; AVX512F-NEXT:    vpermt2q %zmm7, %zmm12, %zmm6
+; AVX512F-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm6 = [6,11,0,1,6,11,0,1]
+; AVX512F-NEXT:    # zmm6 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm13
+; AVX512F-NEXT:    vpermt2q %zmm10, %zmm6, %zmm13
+; AVX512F-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm13
+; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm6
+; AVX512F-NEXT:    vpermt2q %zmm9, %zmm13, %zmm6
+; AVX512F-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm6
+; AVX512F-NEXT:    vpermt2q %zmm11, %zmm13, %zmm6
+; AVX512F-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm6
+; AVX512F-NEXT:    vpermt2q %zmm8, %zmm13, %zmm6
+; AVX512F-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm6
+; AVX512F-NEXT:    vpermt2q %zmm7, %zmm13, %zmm6
+; AVX512F-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm14 = [7,12,0,2,7,12,0,2]
+; AVX512F-NEXT:    # zmm14 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm6
+; AVX512F-NEXT:    vpermt2q %zmm9, %zmm14, %zmm6
+; AVX512F-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm6 = [0,5,0,11,0,5,0,11]
+; AVX512F-NEXT:    # zmm6 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpermt2q %zmm4, %zmm6, %zmm9
 ; AVX512F-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm3
-; AVX512F-NEXT:    vpermt2q %zmm21, %zmm15, %zmm3
-; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm9 = [5,10,15,0,5,10,15,0]
-; AVX512F-NEXT:    # zmm9 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vpermt2q %zmm7, %zmm9, %zmm3
-; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm3
-; AVX512F-NEXT:    vpermt2q %zmm6, %zmm9, %zmm3
-; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm3
-; AVX512F-NEXT:    vpermt2q %zmm8, %zmm9, %zmm3
-; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm26, %zmm3
-; AVX512F-NEXT:    vpermt2q %zmm5, %zmm9, %zmm3
-; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm21, %zmm3
-; AVX512F-NEXT:    vpermt2q %zmm4, %zmm9, %zmm3
-; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm10 = [6,11,0,1,6,11,0,1]
-; AVX512F-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vpermt2q %zmm7, %zmm10, %zmm3
-; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm3
-; AVX512F-NEXT:    vpermt2q %zmm6, %zmm10, %zmm3
-; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm3
-; AVX512F-NEXT:    vpermt2q %zmm8, %zmm10, %zmm3
-; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm26, %zmm3
-; AVX512F-NEXT:    vpermt2q %zmm5, %zmm10, %zmm3
-; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm21, %zmm3
-; AVX512F-NEXT:    vpermt2q %zmm4, %zmm10, %zmm3
-; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm11 = [7,12,0,2,7,12,0,2]
-; AVX512F-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vpermt2q %zmm7, %zmm11, %zmm3
-; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm3 = [0,5,0,11,0,5,0,11]
-; AVX512F-NEXT:    # zmm3 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermt2q %zmm2, %zmm3, %zmm7
-; AVX512F-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm4
+; AVX512F-NEXT:    vpermt2q %zmm10, %zmm14, %zmm4
+; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm6, %zmm10
+; AVX512F-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm6, %zmm11, %zmm2
+; AVX512F-NEXT:    vpermt2q %zmm11, %zmm14, %zmm2
 ; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm3, %zmm6
-; AVX512F-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm1
-; AVX512F-NEXT:    vpermt2q %zmm8, %zmm11, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm6, %zmm11
+; AVX512F-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm8, %zmm14, %zmm1
 ; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm0, %zmm3, %zmm8
+; AVX512F-NEXT:    vpermt2q %zmm5, %zmm6, %zmm8
 ; AVX512F-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm26, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm5, %zmm11, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm26, %zmm3, %zmm5
-; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm21, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm4, %zmm11, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm21, %zmm3, %zmm4
-; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 1472(%rdi), %zmm1
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm17
-; AVX512F-NEXT:    vpermt2q %zmm19, %zmm15, %zmm17
-; AVX512F-NEXT:    vmovdqa64 %zmm19, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm9, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm19, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm10, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm19, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm11, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm19, %zmm3, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm7, %zmm14, %zmm1
 ; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 2048(%rdi), %zmm0
-; AVX512F-NEXT:    vmovdqa64 2112(%rdi), %zmm4
-; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm18
-; AVX512F-NEXT:    vpermt2q %zmm0, %zmm15, %zmm18
-; AVX512F-NEXT:    vmovdqa64 2368(%rdi), %zmm1
-; AVX512F-NEXT:    vmovdqa64 2432(%rdi), %zmm5
-; AVX512F-NEXT:    vpermi2q %zmm1, %zmm5, %zmm15
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm4, %zmm9, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermi2q %zmm5, %zmm1, %zmm9
-; AVX512F-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm4, %zmm10, %zmm2
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm6, %zmm7
+; AVX512F-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 1472(%rdi), %zmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm19
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm16, %zmm19
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm12, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm13, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm14, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm6, %zmm2
 ; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermi2q %zmm5, %zmm1, %zmm10
-; AVX512F-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm4, %zmm11, %zmm2
+; AVX512F-NEXT:    vmovdqa64 2048(%rdi), %zmm0
+; AVX512F-NEXT:    vmovdqa64 2112(%rdi), %zmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm18
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm16, %zmm18
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm12, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm13, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm14, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm6, %zmm2
 ; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermi2q %zmm5, %zmm1, %zmm11
-; AVX512F-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm3, %zmm5
-; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm0, %zmm3, %zmm4
-; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 (%rdi), %zmm2
-; AVX512F-NEXT:    vmovdqa64 64(%rdi), %zmm31
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,5,10,15]
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm19
-; AVX512F-NEXT:    vpermt2q %zmm31, %zmm1, %zmm19
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm4 = <1,6,11,u>
+; AVX512F-NEXT:    vmovdqa64 2368(%rdi), %zmm0
+; AVX512F-NEXT:    vmovdqa64 2432(%rdi), %zmm1
+; AVX512F-NEXT:    vpermi2q %zmm1, %zmm0, %zmm12
+; AVX512F-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermi2q %zmm1, %zmm0, %zmm13
+; AVX512F-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermi2q %zmm1, %zmm0, %zmm14
+; AVX512F-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm17
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm6, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm16, %zmm17
+; AVX512F-NEXT:    vmovdqa64 384(%rdi), %zmm31
+; AVX512F-NEXT:    vmovdqa64 320(%rdi), %zmm2
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,5,10,15]
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm31, %zmm3, %zmm1
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} ymm29 = <1,6,11,u>
 ; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm31, %zmm4, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm31, %zmm29, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm8 = <2,7,12,u>
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm6 = <2,7,12,u>
 ; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm31, %zmm8, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm31, %zmm6, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm10 = <11,0,5,u>
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm7 = <11,0,5,u>
 ; AVX512F-NEXT:    vmovdqa64 %zmm31, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm2, %zmm10, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm7, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 {{.*#+}} ymm25 = <12,1,6,u>
-; AVX512F-NEXT:    vpermt2q %zmm2, %zmm25, %zmm31
-; AVX512F-NEXT:    vmovdqa64 384(%rdi), %zmm26
-; AVX512F-NEXT:    vmovdqa64 320(%rdi), %zmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm30
-; AVX512F-NEXT:    vpermt2q %zmm26, %zmm1, %zmm30
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm16, %zmm31
+; AVX512F-NEXT:    vmovdqa64 (%rdi), %zmm2
+; AVX512F-NEXT:    vmovdqa64 64(%rdi), %zmm23
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm4
+; AVX512F-NEXT:    vpermt2q %zmm23, %zmm3, %zmm4
 ; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm26, %zmm4, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm23, %zmm29, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm26, %zmm8, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm23, %zmm6, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm26, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm2, %zmm10, %zmm0
+; AVX512F-NEXT:    vmovdqa64 %zmm23, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm7, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm2, %zmm25, %zmm26
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm16, %zmm23
 ; AVX512F-NEXT:    vmovdqa64 1024(%rdi), %zmm21
 ; AVX512F-NEXT:    vmovdqa64 960(%rdi), %zmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vpermt2q %zmm21, %zmm1, %zmm3
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm10
+; AVX512F-NEXT:    vpermt2q %zmm21, %zmm3, %zmm10
 ; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm21, %zmm4, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm21, %zmm29, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm21, %zmm8, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm21, %zmm6, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vmovdqa64 %zmm21, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm2, %zmm10, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm7, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm2, %zmm25, %zmm21
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm16, %zmm21
 ; AVX512F-NEXT:    vmovdqa64 704(%rdi), %zmm20
 ; AVX512F-NEXT:    vmovdqa64 640(%rdi), %zmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm5
-; AVX512F-NEXT:    vpermt2q %zmm20, %zmm1, %zmm5
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm11
+; AVX512F-NEXT:    vpermt2q %zmm20, %zmm3, %zmm11
 ; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm20, %zmm4, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm20, %zmm29, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm20, %zmm8, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm20, %zmm6, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vmovdqa64 %zmm20, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm2, %zmm10, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm7, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm2, %zmm25, %zmm20
-; AVX512F-NEXT:    vmovdqa64 1664(%rdi), %zmm22
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm16, %zmm20
+; AVX512F-NEXT:    vmovdqa64 1664(%rdi), %zmm25
 ; AVX512F-NEXT:    vmovdqa64 1600(%rdi), %zmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm11
-; AVX512F-NEXT:    vpermt2q %zmm22, %zmm1, %zmm11
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm22, %zmm4, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm13
+; AVX512F-NEXT:    vpermt2q %zmm25, %zmm3, %zmm13
 ; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm22, %zmm8, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm25, %zmm29, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm22, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm2, %zmm10, %zmm0
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm30
+; AVX512F-NEXT:    vpermt2q %zmm25, %zmm6, %zmm30
+; AVX512F-NEXT:    vmovdqa64 %zmm25, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm7, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm2, %zmm25, %zmm22
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm16, %zmm25
 ; AVX512F-NEXT:    vmovdqa64 1344(%rdi), %zmm12
 ; AVX512F-NEXT:    vmovdqa64 1280(%rdi), %zmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm13
-; AVX512F-NEXT:    vpermt2q %zmm12, %zmm1, %zmm13
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm24
-; AVX512F-NEXT:    vpermt2q %zmm12, %zmm4, %zmm24
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm28
-; AVX512F-NEXT:    vpermt2q %zmm12, %zmm8, %zmm28
-; AVX512F-NEXT:    vmovdqa64 %zmm12, %zmm29
-; AVX512F-NEXT:    vpermt2q %zmm2, %zmm10, %zmm29
-; AVX512F-NEXT:    vpermt2q %zmm2, %zmm25, %zmm12
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm14
+; AVX512F-NEXT:    vpermt2q %zmm12, %zmm3, %zmm14
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm26
+; AVX512F-NEXT:    vpermt2q %zmm12, %zmm29, %zmm26
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm27
+; AVX512F-NEXT:    vpermt2q %zmm12, %zmm6, %zmm27
+; AVX512F-NEXT:    vmovdqa64 %zmm12, %zmm28
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm7, %zmm28
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm16, %zmm12
 ; AVX512F-NEXT:    vmovdqa64 1984(%rdi), %zmm9
-; AVX512F-NEXT:    vmovdqa64 1920(%rdi), %zmm14
-; AVX512F-NEXT:    vmovdqa64 %zmm14, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm9, %zmm1, %zmm2
-; AVX512F-NEXT:    vmovdqa64 2304(%rdi), %zmm6
+; AVX512F-NEXT:    vmovdqa64 1920(%rdi), %zmm15
+; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm2
+; AVX512F-NEXT:    vpermt2q %zmm9, %zmm3, %zmm2
+; AVX512F-NEXT:    vmovdqa64 2304(%rdi), %zmm5
 ; AVX512F-NEXT:    vmovdqa64 2240(%rdi), %zmm0
-; AVX512F-NEXT:    vpermi2q %zmm6, %zmm0, %zmm1
-; AVX512F-NEXT:    vmovdqa64 %zmm14, %zmm7
-; AVX512F-NEXT:    vpermt2q %zmm9, %zmm4, %zmm7
-; AVX512F-NEXT:    vpermi2q %zmm6, %zmm0, %zmm4
-; AVX512F-NEXT:    vmovdqa64 %zmm14, %zmm23
-; AVX512F-NEXT:    vpermt2q %zmm9, %zmm8, %zmm23
-; AVX512F-NEXT:    vpermi2q %zmm6, %zmm0, %zmm8
-; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm27
-; AVX512F-NEXT:    vpermt2q %zmm14, %zmm10, %zmm27
-; AVX512F-NEXT:    vpermi2q %zmm0, %zmm6, %zmm10
-; AVX512F-NEXT:    vpermt2q %zmm0, %zmm25, %zmm6
-; AVX512F-NEXT:    vpermt2q %zmm14, %zmm25, %zmm9
-; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm19[0,1,2,3],zmm16[4,5,6,7]
-; AVX512F-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm30, %zmm14 # 64-byte Folded Reload
-; AVX512F-NEXT:    # zmm14 = zmm30[0,1,2,3],mem[4,5,6,7]
-; AVX512F-NEXT:    vshufi64x2 $228, (%rsp), %zmm5, %zmm5 # 64-byte Folded Reload
-; AVX512F-NEXT:    # zmm5 = zmm5[0,1,2,3],mem[4,5,6,7]
-; AVX512F-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm3 # 64-byte Folded Reload
-; AVX512F-NEXT:    # zmm3 = zmm3[0,1,2,3],mem[4,5,6,7]
-; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm13 = zmm13[0,1,2,3],zmm17[4,5,6,7]
-; AVX512F-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm11, %zmm19 # 64-byte Folded Reload
-; AVX512F-NEXT:    # zmm19 = zmm11[0,1,2,3],mem[4,5,6,7]
+; AVX512F-NEXT:    vpermi2q %zmm5, %zmm0, %zmm3
+; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm8
+; AVX512F-NEXT:    vpermt2q %zmm9, %zmm29, %zmm8
+; AVX512F-NEXT:    vpermi2q %zmm5, %zmm0, %zmm29
+; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm22
+; AVX512F-NEXT:    vpermt2q %zmm9, %zmm6, %zmm22
+; AVX512F-NEXT:    vpermi2q %zmm5, %zmm0, %zmm6
+; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm24
+; AVX512F-NEXT:    vpermt2q %zmm15, %zmm7, %zmm24
+; AVX512F-NEXT:    vpermi2q %zmm0, %zmm5, %zmm7
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm16, %zmm5
+; AVX512F-NEXT:    vpermt2q %zmm15, %zmm16, %zmm9
+; AVX512F-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm0 # 64-byte Folded Reload
+; AVX512F-NEXT:    # zmm0 = zmm4[0,1,2,3],mem[4,5,6,7]
+; AVX512F-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm4 # 64-byte Folded Reload
+; AVX512F-NEXT:    # zmm4 = zmm1[0,1,2,3],mem[4,5,6,7]
+; AVX512F-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm11, %zmm11 # 64-byte Folded Reload
+; AVX512F-NEXT:    # zmm11 = zmm11[0,1,2,3],mem[4,5,6,7]
+; AVX512F-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm10 # 64-byte Folded Reload
+; AVX512F-NEXT:    # zmm10 = zmm10[0,1,2,3],mem[4,5,6,7]
+; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm19 = zmm14[0,1,2,3],zmm19[4,5,6,7]
+; AVX512F-NEXT:    vshufi64x2 $228, (%rsp), %zmm13, %zmm1 # 64-byte Folded Reload
+; AVX512F-NEXT:    # zmm1 = zmm13[0,1,2,3],mem[4,5,6,7]
 ; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm2[0,1,2,3],zmm18[4,5,6,7]
-; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm1[0,1,2,3],zmm15[4,5,6,7]
-; AVX512F-NEXT:    vmovdqa64 256(%rdi), %zmm16
-; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm25 = [0,1,2,3,4,5,6,11]
-; AVX512F-NEXT:    vpermt2q %zmm16, %zmm25, %zmm0
+; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm3 = zmm3[0,1,2,3],zmm17[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 256(%rdi), %zmm18
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm15 = [0,1,2,3,4,5,6,11]
+; AVX512F-NEXT:    vpermt2q %zmm18, %zmm15, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 576(%rdi), %zmm15
-; AVX512F-NEXT:    vpermt2q %zmm15, %zmm25, %zmm14
-; AVX512F-NEXT:    vmovdqu64 %zmm14, (%rsp) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 576(%rdi), %zmm14
+; AVX512F-NEXT:    vpermt2q %zmm14, %zmm15, %zmm4
+; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vmovdqa64 896(%rdi), %zmm17
-; AVX512F-NEXT:    vpermt2q %zmm17, %zmm25, %zmm5
-; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 1216(%rdi), %zmm11
-; AVX512F-NEXT:    vpermt2q %zmm11, %zmm25, %zmm3
-; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 1536(%rdi), %zmm18
-; AVX512F-NEXT:    vpermt2q %zmm18, %zmm25, %zmm13
-; AVX512F-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 1856(%rdi), %zmm14
-; AVX512F-NEXT:    vpermt2q %zmm14, %zmm25, %zmm19
+; AVX512F-NEXT:    vpermt2q %zmm17, %zmm15, %zmm11
+; AVX512F-NEXT:    vmovdqu64 %zmm11, (%rsp) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 1216(%rdi), %zmm13
+; AVX512F-NEXT:    vpermt2q %zmm13, %zmm15, %zmm10
+; AVX512F-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 1536(%rdi), %zmm16
+; AVX512F-NEXT:    vpermt2q %zmm16, %zmm15, %zmm19
 ; AVX512F-NEXT:    vmovdqu64 %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 1856(%rdi), %zmm11
+; AVX512F-NEXT:    vpermt2q %zmm11, %zmm15, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vmovdqa64 2176(%rdi), %zmm19
-; AVX512F-NEXT:    vpermt2q %zmm19, %zmm25, %zmm2
+; AVX512F-NEXT:    vpermt2q %zmm19, %zmm15, %zmm2
 ; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 2496(%rdi), %zmm13
-; AVX512F-NEXT:    vpermt2q %zmm13, %zmm25, %zmm1
-; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 2496(%rdi), %zmm10
+; AVX512F-NEXT:    vpermt2q %zmm10, %zmm15, %zmm3
+; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    movb $7, %al
 ; AVX512F-NEXT:    kmovw %eax, %k1
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm5 {%k1}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm3 {%k1}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm25 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm25 {%k1}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm15 {%k1}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm4 {%k1}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm7 {%k1}
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm8 {%k1}
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
 ; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm0 {%k1}
@@ -5779,467 +5782,467 @@ define void @load_i64_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
 ; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm1 {%k1}
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm24, %zmm2 {%k1}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm24 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm24 {%k1}
-; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [0,1,2,3,4,5,6,12]
-; AVX512F-NEXT:    vpermt2q %zmm16, %zmm4, %zmm5
-; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm19, %zmm4, %zmm3
-; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm15, %zmm4, %zmm25
-; AVX512F-NEXT:    vmovdqu64 %zmm25, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm11, %zmm4, %zmm7
-; AVX512F-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm17, %zmm4, %zmm0
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm2 {%k1}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm26, %zmm3 {%k1}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm26 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm29, %zmm26 {%k1}
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm29 = [0,1,2,3,4,5,6,12]
+; AVX512F-NEXT:    vpermt2q %zmm18, %zmm29, %zmm15
+; AVX512F-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm19, %zmm29, %zmm4
+; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm14, %zmm29, %zmm8
+; AVX512F-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm13, %zmm29, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm14, %zmm4, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm17, %zmm29, %zmm1
 ; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm18, %zmm4, %zmm2
+; AVX512F-NEXT:    vpermt2q %zmm11, %zmm29, %zmm2
 ; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm13, %zmm4, %zmm24
-; AVX512F-NEXT:    vmovdqu64 %zmm24, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm16, %zmm29, %zmm3
+; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm10, %zmm29, %zmm26
+; AVX512F-NEXT:    vmovdqu64 %zmm26, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    movb $56, %al
 ; AVX512F-NEXT:    kmovw %eax, %k1
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm23 {%k1}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm0 {%k1}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm30 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm30 {%k1}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm25 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm25 {%k1}
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm0 {%k1}
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm24 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm24 {%k1}
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm22 {%k1}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm8 {%k1}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm26 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm26 {%k1}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm15 {%k1}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm30 {%k1}
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm28 {%k1}
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm27 {%k1}
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm8 {%k1}
-; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [0,1,2,3,4,5,8,13]
-; AVX512F-NEXT:    vpermt2q %zmm16, %zmm3, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm19, %zmm3, %zmm23
-; AVX512F-NEXT:    vpermt2q %zmm15, %zmm3, %zmm0
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm6 {%k1}
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,1,2,3,4,5,8,13]
+; AVX512F-NEXT:    vpermt2q %zmm18, %zmm1, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm11, %zmm3, %zmm30
-; AVX512F-NEXT:    vpermt2q %zmm17, %zmm3, %zmm25
-; AVX512F-NEXT:    vpermt2q %zmm14, %zmm3, %zmm24
-; AVX512F-NEXT:    vpermt2q %zmm18, %zmm3, %zmm28
-; AVX512F-NEXT:    vpermt2q %zmm13, %zmm3, %zmm8
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm7 {%k1}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm27 {%k1}
+; AVX512F-NEXT:    vpermt2q %zmm19, %zmm1, %zmm22
+; AVX512F-NEXT:    vpermt2q %zmm14, %zmm1, %zmm8
+; AVX512F-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm13, %zmm1, %zmm26
+; AVX512F-NEXT:    vpermt2q %zmm17, %zmm1, %zmm15
+; AVX512F-NEXT:    vpermt2q %zmm11, %zmm1, %zmm30
+; AVX512F-NEXT:    vpermt2q %zmm16, %zmm1, %zmm27
+; AVX512F-NEXT:    vpermt2q %zmm10, %zmm1, %zmm6
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm8 {%k1}
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm0 {%k1}
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm24 {%k1}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm29 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm29 {%k1}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm1 {%k1}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm3 {%k1}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm5 {%k1}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm29 {%k1}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm10 {%k1}
-; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,1,2,3,4,5,9,14]
-; AVX512F-NEXT:    vpermt2q %zmm16, %zmm1, %zmm7
-; AVX512F-NEXT:    vpermt2q %zmm19, %zmm1, %zmm27
-; AVX512F-NEXT:    vpermt2q %zmm15, %zmm1, %zmm0
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm11, %zmm1, %zmm3
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm4 {%k1}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm28 {%k1}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm7 {%k1}
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [0,1,2,3,4,5,9,14]
+; AVX512F-NEXT:    vpermt2q %zmm18, %zmm2, %zmm8
+; AVX512F-NEXT:    vpermt2q %zmm19, %zmm2, %zmm24
+; AVX512F-NEXT:    vpermt2q %zmm14, %zmm2, %zmm29
+; AVX512F-NEXT:    vpermt2q %zmm13, %zmm2, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm17, %zmm2, %zmm3
 ; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm17, %zmm1, %zmm4
-; AVX512F-NEXT:    vpermt2q %zmm14, %zmm1, %zmm5
-; AVX512F-NEXT:    vpermt2q %zmm18, %zmm1, %zmm29
-; AVX512F-NEXT:    vpermt2q %zmm13, %zmm1, %zmm10
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm31 {%k1}
+; AVX512F-NEXT:    vpermt2q %zmm11, %zmm2, %zmm4
+; AVX512F-NEXT:    vpermt2q %zmm16, %zmm2, %zmm28
+; AVX512F-NEXT:    vpermt2q %zmm10, %zmm2, %zmm7
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm31 {%k1}
 ; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [0,1,2,3,4,5,10,15]
-; AVX512F-NEXT:    vpermt2q %zmm16, %zmm3, %zmm31
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm26 {%k1}
-; AVX512F-NEXT:    vpermt2q %zmm15, %zmm3, %zmm26
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm21 {%k1}
-; AVX512F-NEXT:    vpermt2q %zmm11, %zmm3, %zmm21
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm20 {%k1}
+; AVX512F-NEXT:    vpermt2q %zmm14, %zmm3, %zmm31
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm23 {%k1}
+; AVX512F-NEXT:    vpermt2q %zmm18, %zmm3, %zmm23
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm21 {%k1}
+; AVX512F-NEXT:    vpermt2q %zmm13, %zmm3, %zmm21
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm20 {%k1}
 ; AVX512F-NEXT:    vpermt2q %zmm17, %zmm3, %zmm20
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm22 {%k1}
-; AVX512F-NEXT:    vpermt2q %zmm14, %zmm3, %zmm22
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm12 {%k1}
-; AVX512F-NEXT:    vpermt2q %zmm18, %zmm3, %zmm12
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm6 {%k1}
-; AVX512F-NEXT:    vpermt2q %zmm13, %zmm3, %zmm6
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm9 {%k1}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm25 {%k1}
+; AVX512F-NEXT:    vpermt2q %zmm11, %zmm3, %zmm25
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm12 {%k1}
+; AVX512F-NEXT:    vpermt2q %zmm16, %zmm3, %zmm12
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm5 {%k1}
+; AVX512F-NEXT:    vpermt2q %zmm10, %zmm3, %zmm5
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm9 {%k1}
 ; AVX512F-NEXT:    vpermt2q %zmm19, %zmm3, %zmm9
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm1, 448(%rsi)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm1, 384(%rsi)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm1, 320(%rsi)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm1, 256(%rsi)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm1, 192(%rsi)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm1, 128(%rsi)
-; AVX512F-NEXT:    vmovups (%rsp), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm1, 64(%rsi)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm1, (%rsi)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm1, 448(%rdx)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm1, 256(%rdx)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm1, 320(%rdx)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm1, 128(%rdx)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm1, 192(%rdx)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm1, (%rdx)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm1, 64(%rdx)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm1, 384(%rdx)
-; AVX512F-NEXT:    vmovdqa64 %zmm8, 448(%rcx)
-; AVX512F-NEXT:    vmovdqa64 %zmm28, 256(%rcx)
-; AVX512F-NEXT:    vmovdqa64 %zmm24, 320(%rcx)
-; AVX512F-NEXT:    vmovdqa64 %zmm25, 128(%rcx)
-; AVX512F-NEXT:    vmovdqa64 %zmm30, 192(%rcx)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm1, (%rcx)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm1, 64(%rcx)
-; AVX512F-NEXT:    vmovdqa64 %zmm23, 384(%rcx)
-; AVX512F-NEXT:    vmovdqa64 %zmm10, 448(%r8)
-; AVX512F-NEXT:    vmovdqa64 %zmm29, 256(%r8)
-; AVX512F-NEXT:    vmovdqa64 %zmm5, 320(%r8)
-; AVX512F-NEXT:    vmovdqa64 %zmm4, 128(%r8)
-; AVX512F-NEXT:    vmovdqa64 %zmm0, 192(%r8)
-; AVX512F-NEXT:    vmovdqa64 %zmm7, (%r8)
-; AVX512F-NEXT:    vmovdqa64 %zmm2, 64(%r8)
-; AVX512F-NEXT:    vmovdqa64 %zmm27, 384(%r8)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm2, 448(%rsi)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm2, 384(%rsi)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm2, 320(%rsi)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm2, 256(%rsi)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm2, 192(%rsi)
+; AVX512F-NEXT:    vmovups (%rsp), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm2, 128(%rsi)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm2, 64(%rsi)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm2, (%rsi)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm2, 448(%rdx)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm2, 256(%rdx)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm2, 320(%rdx)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm2, 128(%rdx)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm2, 192(%rdx)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm2, (%rdx)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm2, 64(%rdx)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm2, 384(%rdx)
+; AVX512F-NEXT:    vmovdqa64 %zmm6, 448(%rcx)
+; AVX512F-NEXT:    vmovdqa64 %zmm27, 256(%rcx)
+; AVX512F-NEXT:    vmovdqa64 %zmm30, 320(%rcx)
+; AVX512F-NEXT:    vmovdqa64 %zmm15, 128(%rcx)
+; AVX512F-NEXT:    vmovdqa64 %zmm26, 192(%rcx)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm2, (%rcx)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm2, 64(%rcx)
+; AVX512F-NEXT:    vmovdqa64 %zmm22, 384(%rcx)
+; AVX512F-NEXT:    vmovdqa64 %zmm7, 448(%r8)
+; AVX512F-NEXT:    vmovdqa64 %zmm28, 256(%r8)
+; AVX512F-NEXT:    vmovdqa64 %zmm4, 320(%r8)
+; AVX512F-NEXT:    vmovdqa64 %zmm0, 128(%r8)
+; AVX512F-NEXT:    vmovdqa64 %zmm1, 192(%r8)
+; AVX512F-NEXT:    vmovdqa64 %zmm8, (%r8)
+; AVX512F-NEXT:    vmovdqa64 %zmm29, 64(%r8)
+; AVX512F-NEXT:    vmovdqa64 %zmm24, 384(%r8)
 ; AVX512F-NEXT:    vmovdqa64 %zmm9, 384(%r9)
-; AVX512F-NEXT:    vmovdqa64 %zmm6, 448(%r9)
+; AVX512F-NEXT:    vmovdqa64 %zmm5, 448(%r9)
 ; AVX512F-NEXT:    vmovdqa64 %zmm12, 256(%r9)
-; AVX512F-NEXT:    vmovdqa64 %zmm22, 320(%r9)
+; AVX512F-NEXT:    vmovdqa64 %zmm25, 320(%r9)
 ; AVX512F-NEXT:    vmovdqa64 %zmm20, 128(%r9)
 ; AVX512F-NEXT:    vmovdqa64 %zmm21, 192(%r9)
-; AVX512F-NEXT:    vmovdqa64 %zmm31, (%r9)
-; AVX512F-NEXT:    vmovdqa64 %zmm26, 64(%r9)
-; AVX512F-NEXT:    addq $3400, %rsp # imm = 0xD48
+; AVX512F-NEXT:    vmovdqa64 %zmm23, (%r9)
+; AVX512F-NEXT:    vmovdqa64 %zmm31, 64(%r9)
+; AVX512F-NEXT:    addq $3336, %rsp # imm = 0xD08
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: load_i64_stride5_vf64:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    subq $3400, %rsp # imm = 0xD48
-; AVX512BW-NEXT:    vmovdqa64 1728(%rdi), %zmm21
-; AVX512BW-NEXT:    vmovdqa64 1792(%rdi), %zmm4
-; AVX512BW-NEXT:    vmovdqa64 1408(%rdi), %zmm19
-; AVX512BW-NEXT:    vmovdqa64 1088(%rdi), %zmm0
-; AVX512BW-NEXT:    vmovdqa64 1152(%rdi), %zmm3
-; AVX512BW-NEXT:    vmovdqa64 768(%rdi), %zmm26
-; AVX512BW-NEXT:    vmovdqa64 832(%rdi), %zmm5
-; AVX512BW-NEXT:    vmovdqa64 448(%rdi), %zmm1
-; AVX512BW-NEXT:    vmovdqa64 512(%rdi), %zmm6
+; AVX512BW-NEXT:    subq $3336, %rsp # imm = 0xD08
+; AVX512BW-NEXT:    vmovdqa64 1728(%rdi), %zmm3
+; AVX512BW-NEXT:    vmovdqa64 1792(%rdi), %zmm7
+; AVX512BW-NEXT:    vmovdqa64 1408(%rdi), %zmm0
+; AVX512BW-NEXT:    vmovdqa64 1088(%rdi), %zmm1
+; AVX512BW-NEXT:    vmovdqa64 1152(%rdi), %zmm6
+; AVX512BW-NEXT:    vmovdqa64 768(%rdi), %zmm5
+; AVX512BW-NEXT:    vmovdqa64 832(%rdi), %zmm8
+; AVX512BW-NEXT:    vmovdqa64 448(%rdi), %zmm4
+; AVX512BW-NEXT:    vmovdqa64 512(%rdi), %zmm9
 ; AVX512BW-NEXT:    vmovdqa64 128(%rdi), %zmm2
-; AVX512BW-NEXT:    vmovdqa64 192(%rdi), %zmm7
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm15 = [12,1,6,0,12,1,6,0]
-; AVX512BW-NEXT:    # zmm15 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm16
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm15, %zmm16
-; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm8
-; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm15, %zmm8
-; AVX512BW-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm8
-; AVX512BW-NEXT:    vpermt2q %zmm26, %zmm15, %zmm8
-; AVX512BW-NEXT:    vmovdqu64 %zmm8, (%rsp) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm9
-; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm8
-; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm15, %zmm9
+; AVX512BW-NEXT:    vmovdqa64 192(%rdi), %zmm10
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm16 = [12,1,6,0,12,1,6,0]
+; AVX512BW-NEXT:    # zmm16 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm11
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm16, %zmm11
+; AVX512BW-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm11
+; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm16, %zmm11
+; AVX512BW-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm11
+; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm16, %zmm11
+; AVX512BW-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm12
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm11
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm16, %zmm12
+; AVX512BW-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm6
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm16, %zmm6
+; AVX512BW-NEXT:    vmovdqu64 %zmm6, (%rsp) # 64-byte Spill
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm12 = [5,10,15,0,5,10,15,0]
+; AVX512BW-NEXT:    # zmm12 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm6
+; AVX512BW-NEXT:    vpermt2q %zmm10, %zmm12, %zmm6
+; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm6
+; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm12, %zmm6
+; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm6
+; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm12, %zmm6
+; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm6
+; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm12, %zmm6
+; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm6
+; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm12, %zmm6
+; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm6 = [6,11,0,1,6,11,0,1]
+; AVX512BW-NEXT:    # zmm6 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm13
+; AVX512BW-NEXT:    vpermt2q %zmm10, %zmm6, %zmm13
+; AVX512BW-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm13
+; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm6
+; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm13, %zmm6
+; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm6
+; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm13, %zmm6
+; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm6
+; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm13, %zmm6
+; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm6
+; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm13, %zmm6
+; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm14 = [7,12,0,2,7,12,0,2]
+; AVX512BW-NEXT:    # zmm14 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm6
+; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm14, %zmm6
+; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm6 = [0,5,0,11,0,5,0,11]
+; AVX512BW-NEXT:    # zmm6 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm6, %zmm9
 ; AVX512BW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm3
-; AVX512BW-NEXT:    vpermt2q %zmm21, %zmm15, %zmm3
-; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm9 = [5,10,15,0,5,10,15,0]
-; AVX512BW-NEXT:    # zmm9 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm9, %zmm3
-; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm3
-; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm9, %zmm3
-; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm3
-; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm9, %zmm3
-; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm26, %zmm3
-; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm9, %zmm3
-; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm21, %zmm3
-; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm9, %zmm3
-; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm10 = [6,11,0,1,6,11,0,1]
-; AVX512BW-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm10, %zmm3
-; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm3
-; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm10, %zmm3
-; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm3
-; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm10, %zmm3
-; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm26, %zmm3
-; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm10, %zmm3
-; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm21, %zmm3
-; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm10, %zmm3
-; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm11 = [7,12,0,2,7,12,0,2]
-; AVX512BW-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm11, %zmm3
-; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm3 = [0,5,0,11,0,5,0,11]
-; AVX512BW-NEXT:    # zmm3 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm3, %zmm7
-; AVX512BW-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm4
+; AVX512BW-NEXT:    vpermt2q %zmm10, %zmm14, %zmm4
+; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm6, %zmm10
+; AVX512BW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm2
-; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm11, %zmm2
+; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm14, %zmm2
 ; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm3, %zmm6
-; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm1
-; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm11, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm6, %zmm11
+; AVX512BW-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm14, %zmm1
 ; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm3, %zmm8
+; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm6, %zmm8
 ; AVX512BW-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm26, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm11, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm26, %zmm3, %zmm5
-; AVX512BW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm21, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm11, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm21, %zmm3, %zmm4
-; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 1472(%rdi), %zmm1
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm17
-; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm15, %zmm17
-; AVX512BW-NEXT:    vmovdqa64 %zmm19, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm9, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm19, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm10, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm19, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm11, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm3, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm14, %zmm1
 ; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 2048(%rdi), %zmm0
-; AVX512BW-NEXT:    vmovdqa64 2112(%rdi), %zmm4
-; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm18
-; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm15, %zmm18
-; AVX512BW-NEXT:    vmovdqa64 2368(%rdi), %zmm1
-; AVX512BW-NEXT:    vmovdqa64 2432(%rdi), %zmm5
-; AVX512BW-NEXT:    vpermi2q %zmm1, %zmm5, %zmm15
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm2
-; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm9, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermi2q %zmm5, %zmm1, %zmm9
-; AVX512BW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm2
-; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm10, %zmm2
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm6, %zmm7
+; AVX512BW-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 1472(%rdi), %zmm2
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm19
+; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm16, %zmm19
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm12, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm13, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm14, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm6, %zmm2
 ; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermi2q %zmm5, %zmm1, %zmm10
-; AVX512BW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm2
-; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm11, %zmm2
+; AVX512BW-NEXT:    vmovdqa64 2048(%rdi), %zmm0
+; AVX512BW-NEXT:    vmovdqa64 2112(%rdi), %zmm2
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm18
+; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm16, %zmm18
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm12, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm13, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm14, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm6, %zmm2
 ; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermi2q %zmm5, %zmm1, %zmm11
-; AVX512BW-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm3, %zmm5
-; AVX512BW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm3, %zmm4
-; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm2
-; AVX512BW-NEXT:    vmovdqa64 64(%rdi), %zmm31
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm1 = [0,5,10,15]
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm19
-; AVX512BW-NEXT:    vpermt2q %zmm31, %zmm1, %zmm19
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm4 = <1,6,11,u>
+; AVX512BW-NEXT:    vmovdqa64 2368(%rdi), %zmm0
+; AVX512BW-NEXT:    vmovdqa64 2432(%rdi), %zmm1
+; AVX512BW-NEXT:    vpermi2q %zmm1, %zmm0, %zmm12
+; AVX512BW-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermi2q %zmm1, %zmm0, %zmm13
+; AVX512BW-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermi2q %zmm1, %zmm0, %zmm14
+; AVX512BW-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm17
+; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm6, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm16, %zmm17
+; AVX512BW-NEXT:    vmovdqa64 384(%rdi), %zmm31
+; AVX512BW-NEXT:    vmovdqa64 320(%rdi), %zmm2
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm3 = [0,5,10,15]
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm31, %zmm3, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} ymm29 = <1,6,11,u>
 ; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm31, %zmm4, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm31, %zmm29, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm8 = <2,7,12,u>
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm6 = <2,7,12,u>
 ; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm31, %zmm8, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm31, %zmm6, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm10 = <11,0,5,u>
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm7 = <11,0,5,u>
 ; AVX512BW-NEXT:    vmovdqa64 %zmm31, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm10, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm7, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} ymm25 = <12,1,6,u>
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm25, %zmm31
-; AVX512BW-NEXT:    vmovdqa64 384(%rdi), %zmm26
-; AVX512BW-NEXT:    vmovdqa64 320(%rdi), %zmm2
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm30
-; AVX512BW-NEXT:    vpermt2q %zmm26, %zmm1, %zmm30
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm16, %zmm31
+; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm2
+; AVX512BW-NEXT:    vmovdqa64 64(%rdi), %zmm23
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm4
+; AVX512BW-NEXT:    vpermt2q %zmm23, %zmm3, %zmm4
 ; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm26, %zmm4, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm23, %zmm29, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm26, %zmm8, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm23, %zmm6, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm26, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm10, %zmm0
+; AVX512BW-NEXT:    vmovdqa64 %zmm23, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm7, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm25, %zmm26
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm16, %zmm23
 ; AVX512BW-NEXT:    vmovdqa64 1024(%rdi), %zmm21
 ; AVX512BW-NEXT:    vmovdqa64 960(%rdi), %zmm2
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512BW-NEXT:    vpermt2q %zmm21, %zmm1, %zmm3
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm10
+; AVX512BW-NEXT:    vpermt2q %zmm21, %zmm3, %zmm10
 ; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm21, %zmm4, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm21, %zmm29, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm21, %zmm8, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm21, %zmm6, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vmovdqa64 %zmm21, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm10, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm7, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm25, %zmm21
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm16, %zmm21
 ; AVX512BW-NEXT:    vmovdqa64 704(%rdi), %zmm20
 ; AVX512BW-NEXT:    vmovdqa64 640(%rdi), %zmm2
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm5
-; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm1, %zmm5
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm11
+; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm3, %zmm11
 ; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm4, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm29, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm8, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm6, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vmovdqa64 %zmm20, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm10, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm7, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm25, %zmm20
-; AVX512BW-NEXT:    vmovdqa64 1664(%rdi), %zmm22
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm16, %zmm20
+; AVX512BW-NEXT:    vmovdqa64 1664(%rdi), %zmm25
 ; AVX512BW-NEXT:    vmovdqa64 1600(%rdi), %zmm2
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm11
-; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm1, %zmm11
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm4, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm13
+; AVX512BW-NEXT:    vpermt2q %zmm25, %zmm3, %zmm13
 ; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm8, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm25, %zmm29, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm22, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm10, %zmm0
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm30
+; AVX512BW-NEXT:    vpermt2q %zmm25, %zmm6, %zmm30
+; AVX512BW-NEXT:    vmovdqa64 %zmm25, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm7, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm25, %zmm22
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm16, %zmm25
 ; AVX512BW-NEXT:    vmovdqa64 1344(%rdi), %zmm12
 ; AVX512BW-NEXT:    vmovdqa64 1280(%rdi), %zmm2
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm13
-; AVX512BW-NEXT:    vpermt2q %zmm12, %zmm1, %zmm13
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm24
-; AVX512BW-NEXT:    vpermt2q %zmm12, %zmm4, %zmm24
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm28
-; AVX512BW-NEXT:    vpermt2q %zmm12, %zmm8, %zmm28
-; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm29
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm10, %zmm29
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm25, %zmm12
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm14
+; AVX512BW-NEXT:    vpermt2q %zmm12, %zmm3, %zmm14
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm26
+; AVX512BW-NEXT:    vpermt2q %zmm12, %zmm29, %zmm26
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm27
+; AVX512BW-NEXT:    vpermt2q %zmm12, %zmm6, %zmm27
+; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm28
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm7, %zmm28
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm16, %zmm12
 ; AVX512BW-NEXT:    vmovdqa64 1984(%rdi), %zmm9
-; AVX512BW-NEXT:    vmovdqa64 1920(%rdi), %zmm14
-; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm2
-; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm1, %zmm2
-; AVX512BW-NEXT:    vmovdqa64 2304(%rdi), %zmm6
+; AVX512BW-NEXT:    vmovdqa64 1920(%rdi), %zmm15
+; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm2
+; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm3, %zmm2
+; AVX512BW-NEXT:    vmovdqa64 2304(%rdi), %zmm5
 ; AVX512BW-NEXT:    vmovdqa64 2240(%rdi), %zmm0
-; AVX512BW-NEXT:    vpermi2q %zmm6, %zmm0, %zmm1
-; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm7
-; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm4, %zmm7
-; AVX512BW-NEXT:    vpermi2q %zmm6, %zmm0, %zmm4
-; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm23
-; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm8, %zmm23
-; AVX512BW-NEXT:    vpermi2q %zmm6, %zmm0, %zmm8
-; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm27
-; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm10, %zmm27
-; AVX512BW-NEXT:    vpermi2q %zmm0, %zmm6, %zmm10
-; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm25, %zmm6
-; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm25, %zmm9
-; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm19[0,1,2,3],zmm16[4,5,6,7]
-; AVX512BW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm30, %zmm14 # 64-byte Folded Reload
-; AVX512BW-NEXT:    # zmm14 = zmm30[0,1,2,3],mem[4,5,6,7]
-; AVX512BW-NEXT:    vshufi64x2 $228, (%rsp), %zmm5, %zmm5 # 64-byte Folded Reload
-; AVX512BW-NEXT:    # zmm5 = zmm5[0,1,2,3],mem[4,5,6,7]
-; AVX512BW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm3 # 64-byte Folded Reload
-; AVX512BW-NEXT:    # zmm3 = zmm3[0,1,2,3],mem[4,5,6,7]
-; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm13 = zmm13[0,1,2,3],zmm17[4,5,6,7]
-; AVX512BW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm11, %zmm19 # 64-byte Folded Reload
-; AVX512BW-NEXT:    # zmm19 = zmm11[0,1,2,3],mem[4,5,6,7]
+; AVX512BW-NEXT:    vpermi2q %zmm5, %zmm0, %zmm3
+; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm8
+; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm29, %zmm8
+; AVX512BW-NEXT:    vpermi2q %zmm5, %zmm0, %zmm29
+; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm22
+; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm6, %zmm22
+; AVX512BW-NEXT:    vpermi2q %zmm5, %zmm0, %zmm6
+; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm24
+; AVX512BW-NEXT:    vpermt2q %zmm15, %zmm7, %zmm24
+; AVX512BW-NEXT:    vpermi2q %zmm0, %zmm5, %zmm7
+; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm16, %zmm5
+; AVX512BW-NEXT:    vpermt2q %zmm15, %zmm16, %zmm9
+; AVX512BW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm0 # 64-byte Folded Reload
+; AVX512BW-NEXT:    # zmm0 = zmm4[0,1,2,3],mem[4,5,6,7]
+; AVX512BW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm4 # 64-byte Folded Reload
+; AVX512BW-NEXT:    # zmm4 = zmm1[0,1,2,3],mem[4,5,6,7]
+; AVX512BW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm11, %zmm11 # 64-byte Folded Reload
+; AVX512BW-NEXT:    # zmm11 = zmm11[0,1,2,3],mem[4,5,6,7]
+; AVX512BW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm10 # 64-byte Folded Reload
+; AVX512BW-NEXT:    # zmm10 = zmm10[0,1,2,3],mem[4,5,6,7]
+; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm19 = zmm14[0,1,2,3],zmm19[4,5,6,7]
+; AVX512BW-NEXT:    vshufi64x2 $228, (%rsp), %zmm13, %zmm1 # 64-byte Folded Reload
+; AVX512BW-NEXT:    # zmm1 = zmm13[0,1,2,3],mem[4,5,6,7]
 ; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm2[0,1,2,3],zmm18[4,5,6,7]
-; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm1[0,1,2,3],zmm15[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqa64 256(%rdi), %zmm16
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm25 = [0,1,2,3,4,5,6,11]
-; AVX512BW-NEXT:    vpermt2q %zmm16, %zmm25, %zmm0
+; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm3 = zmm3[0,1,2,3],zmm17[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 256(%rdi), %zmm18
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm15 = [0,1,2,3,4,5,6,11]
+; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm15, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 576(%rdi), %zmm15
-; AVX512BW-NEXT:    vpermt2q %zmm15, %zmm25, %zmm14
-; AVX512BW-NEXT:    vmovdqu64 %zmm14, (%rsp) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 576(%rdi), %zmm14
+; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm15, %zmm4
+; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vmovdqa64 896(%rdi), %zmm17
-; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm25, %zmm5
-; AVX512BW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 1216(%rdi), %zmm11
-; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm25, %zmm3
-; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 1536(%rdi), %zmm18
-; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm25, %zmm13
-; AVX512BW-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 1856(%rdi), %zmm14
-; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm25, %zmm19
+; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm15, %zmm11
+; AVX512BW-NEXT:    vmovdqu64 %zmm11, (%rsp) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 1216(%rdi), %zmm13
+; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm15, %zmm10
+; AVX512BW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 1536(%rdi), %zmm16
+; AVX512BW-NEXT:    vpermt2q %zmm16, %zmm15, %zmm19
 ; AVX512BW-NEXT:    vmovdqu64 %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 1856(%rdi), %zmm11
+; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm15, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vmovdqa64 2176(%rdi), %zmm19
-; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm25, %zmm2
+; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm15, %zmm2
 ; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 2496(%rdi), %zmm13
-; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm25, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 2496(%rdi), %zmm10
+; AVX512BW-NEXT:    vpermt2q %zmm10, %zmm15, %zmm3
+; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    movb $7, %al
 ; AVX512BW-NEXT:    kmovd %eax, %k1
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm5 {%k1}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm3 {%k1}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm25 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm25 {%k1}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm15 {%k1}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm4 {%k1}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm7 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm8 {%k1}
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
 ; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm0 {%k1}
@@ -6247,176 +6250,177 @@ define void @load_i64_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
 ; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm1 {%k1}
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm24, %zmm2 {%k1}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm24 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm24 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [0,1,2,3,4,5,6,12]
-; AVX512BW-NEXT:    vpermt2q %zmm16, %zmm4, %zmm5
-; AVX512BW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm4, %zmm3
-; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm15, %zmm4, %zmm25
-; AVX512BW-NEXT:    vmovdqu64 %zmm25, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm4, %zmm7
-; AVX512BW-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm4, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm2 {%k1}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm26, %zmm3 {%k1}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm26 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm29, %zmm26 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm29 = [0,1,2,3,4,5,6,12]
+; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm29, %zmm15
+; AVX512BW-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm29, %zmm4
+; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm29, %zmm8
+; AVX512BW-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm29, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm4, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm29, %zmm1
 ; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm4, %zmm2
+; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm29, %zmm2
 ; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm4, %zmm24
-; AVX512BW-NEXT:    vmovdqu64 %zmm24, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm16, %zmm29, %zmm3
+; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm10, %zmm29, %zmm26
+; AVX512BW-NEXT:    vmovdqu64 %zmm26, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    movb $56, %al
 ; AVX512BW-NEXT:    kmovd %eax, %k1
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm23 {%k1}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm0 {%k1}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm30 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm30 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm0 {%k1}
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm25 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm25 {%k1}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm24 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm24 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm22 {%k1}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm8 {%k1}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm26 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm26 {%k1}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm15 {%k1}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm30 {%k1}
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm28 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm27 {%k1}
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm8 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [0,1,2,3,4,5,8,13]
-; AVX512BW-NEXT:    vpermt2q %zmm16, %zmm3, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm3, %zmm23
-; AVX512BW-NEXT:    vpermt2q %zmm15, %zmm3, %zmm0
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm6 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,1,2,3,4,5,8,13]
+; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm1, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm3, %zmm30
-; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm3, %zmm25
-; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm3, %zmm24
-; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm3, %zmm28
-; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm3, %zmm8
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm7 {%k1}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm27 {%k1}
+; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm1, %zmm22
+; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm1, %zmm8
+; AVX512BW-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm1, %zmm26
+; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm1, %zmm15
+; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm1, %zmm30
+; AVX512BW-NEXT:    vpermt2q %zmm16, %zmm1, %zmm27
+; AVX512BW-NEXT:    vpermt2q %zmm10, %zmm1, %zmm6
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm8 {%k1}
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm0 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm24 {%k1}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm29 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm29 {%k1}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm1 {%k1}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm3 {%k1}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm5 {%k1}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm29 {%k1}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm10 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,1,2,3,4,5,9,14]
-; AVX512BW-NEXT:    vpermt2q %zmm16, %zmm1, %zmm7
-; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm1, %zmm27
-; AVX512BW-NEXT:    vpermt2q %zmm15, %zmm1, %zmm0
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm2
-; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm1, %zmm3
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm4 {%k1}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm28 {%k1}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm7 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [0,1,2,3,4,5,9,14]
+; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm2, %zmm8
+; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm2, %zmm24
+; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm2, %zmm29
+; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm2, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm2, %zmm3
 ; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm1, %zmm4
-; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm1, %zmm5
-; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm1, %zmm29
-; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm1, %zmm10
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm31 {%k1}
+; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm2, %zmm4
+; AVX512BW-NEXT:    vpermt2q %zmm16, %zmm2, %zmm28
+; AVX512BW-NEXT:    vpermt2q %zmm10, %zmm2, %zmm7
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm31 {%k1}
 ; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [0,1,2,3,4,5,10,15]
-; AVX512BW-NEXT:    vpermt2q %zmm16, %zmm3, %zmm31
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm26 {%k1}
-; AVX512BW-NEXT:    vpermt2q %zmm15, %zmm3, %zmm26
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm21 {%k1}
-; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm3, %zmm21
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm20 {%k1}
+; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm3, %zmm31
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm23 {%k1}
+; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm3, %zmm23
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm21 {%k1}
+; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm3, %zmm21
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm20 {%k1}
 ; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm3, %zmm20
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm22 {%k1}
-; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm3, %zmm22
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm12 {%k1}
-; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm3, %zmm12
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm6 {%k1}
-; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm3, %zmm6
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm9 {%k1}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm25 {%k1}
+; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm3, %zmm25
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm12 {%k1}
+; AVX512BW-NEXT:    vpermt2q %zmm16, %zmm3, %zmm12
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm5 {%k1}
+; AVX512BW-NEXT:    vpermt2q %zmm10, %zmm3, %zmm5
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm9 {%k1}
 ; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm3, %zmm9
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm1, 448(%rsi)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm1, 384(%rsi)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm1, 320(%rsi)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm1, 256(%rsi)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm1, 192(%rsi)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm1, 128(%rsi)
-; AVX512BW-NEXT:    vmovups (%rsp), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm1, 64(%rsi)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm1, (%rsi)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm1, 448(%rdx)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm1, 256(%rdx)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm1, 320(%rdx)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm1, 128(%rdx)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm1, 192(%rdx)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm1, (%rdx)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm1, 64(%rdx)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm1, 384(%rdx)
-; AVX512BW-NEXT:    vmovdqa64 %zmm8, 448(%rcx)
-; AVX512BW-NEXT:    vmovdqa64 %zmm28, 256(%rcx)
-; AVX512BW-NEXT:    vmovdqa64 %zmm24, 320(%rcx)
-; AVX512BW-NEXT:    vmovdqa64 %zmm25, 128(%rcx)
-; AVX512BW-NEXT:    vmovdqa64 %zmm30, 192(%rcx)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm1, (%rcx)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm1, 64(%rcx)
-; AVX512BW-NEXT:    vmovdqa64 %zmm23, 384(%rcx)
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, 448(%r8)
-; AVX512BW-NEXT:    vmovdqa64 %zmm29, 256(%r8)
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, 320(%r8)
-; AVX512BW-NEXT:    vmovdqa64 %zmm4, 128(%r8)
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, 192(%r8)
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, (%r8)
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, 64(%r8)
-; AVX512BW-NEXT:    vmovdqa64 %zmm27, 384(%r8)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm2, 448(%rsi)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm2, 384(%rsi)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm2, 320(%rsi)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm2, 256(%rsi)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm2, 192(%rsi)
+; AVX512BW-NEXT:    vmovups (%rsp), %zmm2 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm2, 128(%rsi)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm2, 64(%rsi)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm2, (%rsi)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm2, 448(%rdx)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm2, 256(%rdx)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm2, 320(%rdx)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm2, 128(%rdx)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm2, 192(%rdx)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm2, (%rdx)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm2, 64(%rdx)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm2, 384(%rdx)
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, 448(%rcx)
+; AVX512BW-NEXT:    vmovdqa64 %zmm27, 256(%rcx)
+; AVX512BW-NEXT:    vmovdqa64 %zmm30, 320(%rcx)
+; AVX512BW-NEXT:    vmovdqa64 %zmm15, 128(%rcx)
+; AVX512BW-NEXT:    vmovdqa64 %zmm26, 192(%rcx)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm2, (%rcx)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm2, 64(%rcx)
+; AVX512BW-NEXT:    vmovdqa64 %zmm22, 384(%rcx)
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, 448(%r8)
+; AVX512BW-NEXT:    vmovdqa64 %zmm28, 256(%r8)
+; AVX512BW-NEXT:    vmovdqa64 %zmm4, 320(%r8)
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, 128(%r8)
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, 192(%r8)
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, (%r8)
+; AVX512BW-NEXT:    vmovdqa64 %zmm29, 64(%r8)
+; AVX512BW-NEXT:    vmovdqa64 %zmm24, 384(%r8)
 ; AVX512BW-NEXT:    vmovdqa64 %zmm9, 384(%r9)
-; AVX512BW-NEXT:    vmovdqa64 %zmm6, 448(%r9)
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, 448(%r9)
 ; AVX512BW-NEXT:    vmovdqa64 %zmm12, 256(%r9)
-; AVX512BW-NEXT:    vmovdqa64 %zmm22, 320(%r9)
+; AVX512BW-NEXT:    vmovdqa64 %zmm25, 320(%r9)
 ; AVX512BW-NEXT:    vmovdqa64 %zmm20, 128(%r9)
 ; AVX512BW-NEXT:    vmovdqa64 %zmm21, 192(%r9)
-; AVX512BW-NEXT:    vmovdqa64 %zmm31, (%r9)
-; AVX512BW-NEXT:    vmovdqa64 %zmm26, 64(%r9)
-; AVX512BW-NEXT:    addq $3400, %rsp # imm = 0xD48
+; AVX512BW-NEXT:    vmovdqa64 %zmm23, (%r9)
+; AVX512BW-NEXT:    vmovdqa64 %zmm31, 64(%r9)
+; AVX512BW-NEXT:    addq $3336, %rsp # imm = 0xD08
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
   %wide.vec = load <320 x i64>, ptr %in.vec, align 64
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i64-stride-7.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i64-stride-7.ll
index 7d9c056716ceeb2fa0102f9d12fc7fe96ed5f591..91a70fb000dd65e4bcc3740bed7792e74976a119 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i64-stride-7.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i64-stride-7.ll
@@ -751,223 +751,221 @@ define void @load_i64_stride7_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ;
 ; AVX512F-LABEL: load_i64_stride7_vf8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa64 384(%rdi), %zmm1
-; AVX512F-NEXT:    vmovdqa64 320(%rdi), %zmm4
-; AVX512F-NEXT:    vmovdqa64 256(%rdi), %zmm5
-; AVX512F-NEXT:    vmovdqa64 192(%rdi), %zmm6
-; AVX512F-NEXT:    vmovdqa64 128(%rdi), %zmm7
-; AVX512F-NEXT:    vmovdqa64 64(%rdi), %zmm2
-; AVX512F-NEXT:    vmovdqa64 (%rdi), %zmm3
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm0 = <0,7,14,u>
-; AVX512F-NEXT:    vpermi2q %zmm2, %zmm3, %zmm0
-; AVX512F-NEXT:    movb $24, %al
-; AVX512F-NEXT:    kmovw %eax, %k2
-; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm0 {%k2} = zmm7[4,5,4,5],zmm6[4,5,4,5]
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm8 = [10,3,10,3,10,3,10,3]
-; AVX512F-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermi2q %zmm4, %zmm5, %zmm8
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm9 = [0,5,6,9,0,5,6,9]
-; AVX512F-NEXT:    # zmm9 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermi2q %zmm1, %zmm8, %zmm9
-; AVX512F-NEXT:    movb $-32, %al
-; AVX512F-NEXT:    kmovw %eax, %k1
-; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm0 {%k1}
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm8 = [11,4,11,4,11,4,11,4]
-; AVX512F-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermi2q %zmm4, %zmm5, %zmm8
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm10 = [0,5,6,10,0,5,6,10]
-; AVX512F-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermi2q %zmm1, %zmm8, %zmm10
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm9 = [13,6,13,6,13,6,13,6]
-; AVX512F-NEXT:    # zmm9 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm8 = [14,0,0,7,14,0,0,7]
+; AVX512F-NEXT:    vmovdqa64 384(%rdi), %zmm0
+; AVX512F-NEXT:    vmovdqa64 320(%rdi), %zmm1
+; AVX512F-NEXT:    vmovdqa64 256(%rdi), %zmm2
+; AVX512F-NEXT:    vmovdqa64 192(%rdi), %zmm4
+; AVX512F-NEXT:    vmovdqa64 128(%rdi), %zmm5
+; AVX512F-NEXT:    vmovdqa64 64(%rdi), %zmm9
+; AVX512F-NEXT:    vmovdqa64 (%rdi), %zmm10
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm8 = [0,7,14,0,0,7,14,0]
 ; AVX512F-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermi2q %zmm6, %zmm7, %zmm8
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm6 = [9,0,7,0,9,0,7,0]
+; AVX512F-NEXT:    # zmm6 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm3
+; AVX512F-NEXT:    vpermt2q %zmm10, %zmm6, %zmm3
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm7 = [14,0,0,7,14,0,0,7]
+; AVX512F-NEXT:    # zmm7 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpermi2q %zmm4, %zmm5, %zmm7
 ; AVX512F-NEXT:    vmovdqa 16(%rdi), %xmm11
 ; AVX512F-NEXT:    vpblendd {{.*#+}} xmm11 = xmm11[0,1],mem[2,3]
-; AVX512F-NEXT:    vinserti32x4 $0, %xmm11, %zmm8, %zmm8
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm11 = [7,0,9,0,7,0,9,0]
-; AVX512F-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermi2q %zmm7, %zmm6, %zmm11
-; AVX512F-NEXT:    vmovdqa 64(%rdi), %ymm12
-; AVX512F-NEXT:    vpalignr {{.*#+}} ymm12 = mem[8,9,10,11,12,13,14,15],ymm12[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm12[16,17,18,19,20,21,22,23]
-; AVX512F-NEXT:    vextracti128 $1, %ymm12, %xmm15
-; AVX512F-NEXT:    vinserti128 $1, 192(%rdi), %ymm0, %ymm13
-; AVX512F-NEXT:    vmovdqa 128(%rdi), %ymm12
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm13 = ymm12[0,1,2,3,4,5],ymm13[6,7]
+; AVX512F-NEXT:    vinserti32x4 $0, %xmm11, %zmm7, %zmm7
+; AVX512F-NEXT:    vmovdqa 64(%rdi), %ymm11
+; AVX512F-NEXT:    vpalignr {{.*#+}} ymm11 = mem[8,9,10,11,12,13,14,15],ymm11[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm11[16,17,18,19,20,21,22,23]
+; AVX512F-NEXT:    vextracti128 $1, %ymm11, %xmm11
+; AVX512F-NEXT:    vinserti128 $1, 192(%rdi), %ymm0, %ymm12
+; AVX512F-NEXT:    vmovdqa 128(%rdi), %ymm13
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm12 = ymm13[0,1,2,3,4,5],ymm12[6,7]
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm14 = [4,11]
-; AVX512F-NEXT:    vpermi2q %zmm2, %zmm3, %zmm14
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm13 = ymm14[0,1,2,3],ymm13[4,5,6,7]
+; AVX512F-NEXT:    vpermi2q %zmm9, %zmm10, %zmm14
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm15 = ymm14[0,1,2,3],ymm12[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm14 = [5,12]
+; AVX512F-NEXT:    vpermi2q %zmm9, %zmm10, %zmm14
+; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm12 = [6,13]
+; AVX512F-NEXT:    vpermi2q %zmm9, %zmm10, %zmm12
+; AVX512F-NEXT:    vpermt2q %zmm9, %zmm8, %zmm10
+; AVX512F-NEXT:    vpermi2q %zmm1, %zmm2, %zmm8
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm9 = [4,5,6,13,4,5,6,13]
+; AVX512F-NEXT:    # zmm9 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpermi2q %zmm0, %zmm8, %zmm9
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm15, %zmm9, %zmm8
+; AVX512F-NEXT:    vmovdqa 192(%rdi), %ymm9
+; AVX512F-NEXT:    vpalignr {{.*#+}} ymm9 = ymm13[8,9,10,11,12,13,14,15],ymm9[0,1,2,3,4,5,6,7],ymm13[24,25,26,27,28,29,30,31],ymm9[16,17,18,19,20,21,22,23]
+; AVX512F-NEXT:    movq {{[0-9]+}}(%rsp), %rax
+; AVX512F-NEXT:    movq {{[0-9]+}}(%rsp), %rdi
+; AVX512F-NEXT:    movb $24, %r10b
+; AVX512F-NEXT:    kmovw %r10d, %k2
+; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm10 {%k2} = zmm5[4,5,4,5],zmm4[4,5,4,5]
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm13 = [10,3,10,3,10,3,10,3]
+; AVX512F-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpermi2q %zmm1, %zmm2, %zmm13
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm15 = [0,5,6,9,0,5,6,9]
+; AVX512F-NEXT:    # zmm15 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpermi2q %zmm0, %zmm13, %zmm15
+; AVX512F-NEXT:    movb $-32, %r10b
+; AVX512F-NEXT:    kmovw %r10d, %k1
+; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm10 {%k1}
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm13 = [13,6,13,6,13,6,13,6]
+; AVX512F-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm15 = [7,0,9,0,7,0,9,0]
+; AVX512F-NEXT:    # zmm15 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpermi2q %zmm5, %zmm4, %zmm15
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm9 = ymm14[0,1,2,3],ymm9[4,5,6,7]
 ; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm14 = [4,11,4,11]
 ; AVX512F-NEXT:    # ymm14 = mem[0,1,0,1]
-; AVX512F-NEXT:    vpermi2q %zmm6, %zmm7, %zmm14
-; AVX512F-NEXT:    vpermt2q %zmm6, %zmm9, %zmm7
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm6 = <9,0,7,u>
-; AVX512F-NEXT:    vpermi2q %zmm3, %zmm2, %zmm6
-; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm6 {%k2}
-; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm6 {%k1}
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm7 = [12,5,12,5,12,5,12,5]
-; AVX512F-NEXT:    # zmm7 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermi2q %zmm4, %zmm5, %zmm7
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm10 = [0,5,6,11,0,5,6,11]
-; AVX512F-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermi2q %zmm1, %zmm7, %zmm10
-; AVX512F-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512F-NEXT:    movq {{[0-9]+}}(%rsp), %r10
-; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm8 {%k1}
-; AVX512F-NEXT:    vpermi2q %zmm4, %zmm5, %zmm9
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm7 = [0,5,6,12,0,5,6,12]
-; AVX512F-NEXT:    # zmm7 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermi2q %zmm1, %zmm9, %zmm7
-; AVX512F-NEXT:    vinserti32x4 $0, %xmm15, %zmm11, %zmm9
-; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm9 {%k1}
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm7 = [0,7,14,0,0,7,14,0]
-; AVX512F-NEXT:    # zmm7 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermi2q %zmm4, %zmm5, %zmm7
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm10 = [4,5,6,13,4,5,6,13]
-; AVX512F-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermi2q %zmm1, %zmm7, %zmm10
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm13, %zmm10, %zmm7
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm10 = [9,0,7,0,9,0,7,0]
-; AVX512F-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermi2q %zmm5, %zmm4, %zmm10
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm11 = [4,5,6,14,4,5,6,14]
-; AVX512F-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermi2q %zmm1, %zmm10, %zmm11
-; AVX512F-NEXT:    vmovdqa 192(%rdi), %ymm10
-; AVX512F-NEXT:    vpalignr {{.*#+}} ymm10 = ymm12[8,9,10,11,12,13,14,15],ymm10[0,1,2,3,4,5,6,7],ymm12[24,25,26,27,28,29,30,31],ymm10[16,17,18,19,20,21,22,23]
-; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm12 = [5,12]
-; AVX512F-NEXT:    vpermi2q %zmm2, %zmm3, %zmm12
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm10 = ymm12[0,1,2,3],ymm10[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm10, %zmm11, %zmm10
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm11 = [2,9,2,9,2,9,2,9]
-; AVX512F-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermi2q %zmm4, %zmm5, %zmm11
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [4,5,8,15,4,5,8,15]
+; AVX512F-NEXT:    vpermi2q %zmm4, %zmm5, %zmm14
+; AVX512F-NEXT:    vpermt2q %zmm4, %zmm13, %zmm5
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm3 {%k2}
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm4 = [11,4,11,4,11,4,11,4]
+; AVX512F-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpermi2q %zmm1, %zmm2, %zmm4
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm5 = [0,5,6,10,0,5,6,10]
+; AVX512F-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpermi2q %zmm0, %zmm4, %zmm5
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm3 {%k1}
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm4 = [12,5,12,5,12,5,12,5]
+; AVX512F-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpermi2q %zmm1, %zmm2, %zmm4
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm5 = [0,5,6,11,0,5,6,11]
+; AVX512F-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpermi2q %zmm0, %zmm4, %zmm5
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm7 {%k1}
+; AVX512F-NEXT:    vpermi2q %zmm1, %zmm2, %zmm13
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [0,5,6,12,0,5,6,12]
 ; AVX512F-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermi2q %zmm1, %zmm11, %zmm4
-; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm1 = [6,13]
-; AVX512F-NEXT:    vpermi2q %zmm2, %zmm3, %zmm1
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm14[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm4, %zmm1
-; AVX512F-NEXT:    vmovdqa64 %zmm0, (%rsi)
-; AVX512F-NEXT:    vmovdqa64 %zmm6, (%rdx)
-; AVX512F-NEXT:    vmovdqa64 %zmm8, (%rcx)
-; AVX512F-NEXT:    vmovdqa64 %zmm9, (%r8)
-; AVX512F-NEXT:    vmovdqa64 %zmm7, (%r9)
-; AVX512F-NEXT:    vmovdqa64 %zmm10, (%r10)
-; AVX512F-NEXT:    vmovdqa64 %zmm1, (%rax)
+; AVX512F-NEXT:    vpermi2q %zmm0, %zmm13, %zmm4
+; AVX512F-NEXT:    vinserti32x4 $0, %xmm11, %zmm15, %zmm5
+; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm5 {%k1}
+; AVX512F-NEXT:    vpermi2q %zmm2, %zmm1, %zmm6
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [4,5,6,14,4,5,6,14]
+; AVX512F-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpermi2q %zmm0, %zmm6, %zmm4
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm9, %zmm4, %zmm4
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm6 = [2,9,2,9,2,9,2,9]
+; AVX512F-NEXT:    # zmm6 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpermi2q %zmm1, %zmm2, %zmm6
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm1 = [4,5,8,15,4,5,8,15]
+; AVX512F-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpermi2q %zmm0, %zmm6, %zmm1
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm12[0,1,2,3],ymm14[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm1, %zmm0
+; AVX512F-NEXT:    vmovdqa64 %zmm10, (%rsi)
+; AVX512F-NEXT:    vmovdqa64 %zmm3, (%rdx)
+; AVX512F-NEXT:    vmovdqa64 %zmm7, (%rcx)
+; AVX512F-NEXT:    vmovdqa64 %zmm5, (%r8)
+; AVX512F-NEXT:    vmovdqa64 %zmm8, (%r9)
+; AVX512F-NEXT:    vmovdqa64 %zmm4, (%rdi)
+; AVX512F-NEXT:    vmovdqa64 %zmm0, (%rax)
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: load_i64_stride7_vf8:
 ; AVX512BW:       # %bb.0:
+; AVX512BW-NEXT:    vmovdqa64 384(%rdi), %zmm0
+; AVX512BW-NEXT:    vmovdqa64 320(%rdi), %zmm1
+; AVX512BW-NEXT:    vmovdqa64 256(%rdi), %zmm2
+; AVX512BW-NEXT:    vmovdqa64 192(%rdi), %zmm4
+; AVX512BW-NEXT:    vmovdqa64 128(%rdi), %zmm5
+; AVX512BW-NEXT:    vmovdqa64 64(%rdi), %zmm9
+; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm10
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm8 = [0,7,14,0,0,7,14,0]
+; AVX512BW-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm6 = [9,0,7,0,9,0,7,0]
+; AVX512BW-NEXT:    # zmm6 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm3
+; AVX512BW-NEXT:    vpermt2q %zmm10, %zmm6, %zmm3
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm7 = [14,0,0,7,14,0,0,7]
+; AVX512BW-NEXT:    # zmm7 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpermi2q %zmm4, %zmm5, %zmm7
+; AVX512BW-NEXT:    vmovdqa 16(%rdi), %xmm11
+; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm11 = xmm11[0,1],mem[2,3]
+; AVX512BW-NEXT:    vinserti32x4 $0, %xmm11, %zmm7, %zmm7
+; AVX512BW-NEXT:    vinserti128 $1, 192(%rdi), %ymm0, %ymm12
+; AVX512BW-NEXT:    vmovdqa 128(%rdi), %ymm11
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm12 = ymm11[0,1,2,3,4,5],ymm12[6,7]
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm13 = [4,11]
+; AVX512BW-NEXT:    vpermi2q %zmm9, %zmm10, %zmm13
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm14 = ymm13[0,1,2,3],ymm12[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm13 = [5,12]
+; AVX512BW-NEXT:    vpermi2q %zmm9, %zmm10, %zmm13
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm12 = [6,13]
+; AVX512BW-NEXT:    vpermi2q %zmm9, %zmm10, %zmm12
+; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm8, %zmm10
+; AVX512BW-NEXT:    vpermi2q %zmm1, %zmm2, %zmm8
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm9 = [4,5,6,13,4,5,6,13]
+; AVX512BW-NEXT:    # zmm9 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpermi2q %zmm0, %zmm8, %zmm9
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm14, %zmm9, %zmm8
 ; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %r10
-; AVX512BW-NEXT:    vmovdqa64 384(%rdi), %zmm1
-; AVX512BW-NEXT:    vmovdqa64 320(%rdi), %zmm4
-; AVX512BW-NEXT:    vmovdqa64 256(%rdi), %zmm5
-; AVX512BW-NEXT:    vmovdqa64 192(%rdi), %zmm6
-; AVX512BW-NEXT:    vmovdqa64 128(%rdi), %zmm7
-; AVX512BW-NEXT:    vmovdqa64 64(%rdi), %zmm2
-; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm3
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm0 = <0,7,14,u>
-; AVX512BW-NEXT:    vpermi2q %zmm2, %zmm3, %zmm0
 ; AVX512BW-NEXT:    movb $24, %r11b
 ; AVX512BW-NEXT:    kmovd %r11d, %k2
-; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm0 {%k2} = zmm7[4,5,4,5],zmm6[4,5,4,5]
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm8 = [10,3,10,3,10,3,10,3]
-; AVX512BW-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermi2q %zmm4, %zmm5, %zmm8
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm9 = [0,5,6,9,0,5,6,9]
-; AVX512BW-NEXT:    # zmm9 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermi2q %zmm1, %zmm8, %zmm9
+; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm10 {%k2} = zmm5[4,5,4,5],zmm4[4,5,4,5]
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm9 = [10,3,10,3,10,3,10,3]
+; AVX512BW-NEXT:    # zmm9 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpermi2q %zmm1, %zmm2, %zmm9
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm14 = [0,5,6,9,0,5,6,9]
+; AVX512BW-NEXT:    # zmm14 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpermi2q %zmm0, %zmm9, %zmm14
 ; AVX512BW-NEXT:    movb $-32, %r11b
 ; AVX512BW-NEXT:    kmovd %r11d, %k1
-; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm0 {%k1}
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm8 = [11,4,11,4,11,4,11,4]
-; AVX512BW-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermi2q %zmm4, %zmm5, %zmm8
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm10 = [0,5,6,10,0,5,6,10]
-; AVX512BW-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermi2q %zmm1, %zmm8, %zmm10
+; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm10 {%k1}
 ; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm9 = [13,6,13,6,13,6,13,6]
 ; AVX512BW-NEXT:    # zmm9 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm8 = [14,0,0,7,14,0,0,7]
-; AVX512BW-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermi2q %zmm6, %zmm7, %zmm8
-; AVX512BW-NEXT:    vmovdqa 16(%rdi), %xmm11
-; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm11 = xmm11[0,1],mem[2,3]
-; AVX512BW-NEXT:    vinserti32x4 $0, %xmm11, %zmm8, %zmm8
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm11 = [7,0,9,0,7,0,9,0]
-; AVX512BW-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermi2q %zmm7, %zmm6, %zmm11
-; AVX512BW-NEXT:    vinserti128 $1, 192(%rdi), %ymm0, %ymm13
-; AVX512BW-NEXT:    vmovdqa 128(%rdi), %ymm12
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm13 = ymm12[0,1,2,3,4,5],ymm13[6,7]
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm14 = [4,11]
-; AVX512BW-NEXT:    vpermi2q %zmm2, %zmm3, %zmm14
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm13 = ymm14[0,1,2,3],ymm13[4,5,6,7]
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm14 = [4,11,4,11]
-; AVX512BW-NEXT:    # ymm14 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vpermi2q %zmm6, %zmm7, %zmm14
-; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm9, %zmm7
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm6 = <9,0,7,u>
-; AVX512BW-NEXT:    vpermi2q %zmm3, %zmm2, %zmm6
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm6 {%k2}
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm6 {%k1}
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm7 = [12,5,12,5,12,5,12,5]
-; AVX512BW-NEXT:    # zmm7 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermi2q %zmm4, %zmm5, %zmm7
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm10 = [0,5,6,11,0,5,6,11]
-; AVX512BW-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermi2q %zmm1, %zmm7, %zmm10
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm8 {%k1}
-; AVX512BW-NEXT:    vpermi2q %zmm4, %zmm5, %zmm9
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm7 = [0,5,6,12,0,5,6,12]
-; AVX512BW-NEXT:    # zmm7 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermi2q %zmm1, %zmm9, %zmm7
-; AVX512BW-NEXT:    vmovdqa 64(%rdi), %ymm9
-; AVX512BW-NEXT:    vpalignr {{.*#+}} ymm9 = mem[8,9,10,11,12,13,14,15],ymm9[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm9[16,17,18,19,20,21,22,23]
-; AVX512BW-NEXT:    vextracti128 $1, %ymm9, %xmm9
-; AVX512BW-NEXT:    vinserti32x4 $0, %xmm9, %zmm11, %zmm9
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm9 {%k1}
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm7 = [0,7,14,0,0,7,14,0]
-; AVX512BW-NEXT:    # zmm7 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermi2q %zmm4, %zmm5, %zmm7
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm10 = [4,5,6,13,4,5,6,13]
-; AVX512BW-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermi2q %zmm1, %zmm7, %zmm10
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm13, %zmm10, %zmm7
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm10 = [9,0,7,0,9,0,7,0]
-; AVX512BW-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermi2q %zmm5, %zmm4, %zmm10
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm11 = [4,5,6,14,4,5,6,14]
-; AVX512BW-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermi2q %zmm1, %zmm10, %zmm11
-; AVX512BW-NEXT:    vmovdqa 192(%rdi), %ymm10
-; AVX512BW-NEXT:    vpalignr {{.*#+}} ymm10 = ymm12[8,9,10,11,12,13,14,15],ymm10[0,1,2,3,4,5,6,7],ymm12[24,25,26,27,28,29,30,31],ymm10[16,17,18,19,20,21,22,23]
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm12 = [5,12]
-; AVX512BW-NEXT:    vpermi2q %zmm2, %zmm3, %zmm12
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm10 = ymm12[0,1,2,3],ymm10[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm10, %zmm11, %zmm10
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm11 = [2,9,2,9,2,9,2,9]
-; AVX512BW-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermi2q %zmm4, %zmm5, %zmm11
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [4,5,8,15,4,5,8,15]
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm14 = [7,0,9,0,7,0,9,0]
+; AVX512BW-NEXT:    # zmm14 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpermi2q %zmm5, %zmm4, %zmm14
+; AVX512BW-NEXT:    vmovdqa 192(%rdi), %ymm15
+; AVX512BW-NEXT:    vpalignr {{.*#+}} ymm11 = ymm11[8,9,10,11,12,13,14,15],ymm15[0,1,2,3,4,5,6,7],ymm11[24,25,26,27,28,29,30,31],ymm15[16,17,18,19,20,21,22,23]
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm11 = ymm13[0,1,2,3],ymm11[4,5,6,7]
+; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm13 = [4,11,4,11]
+; AVX512BW-NEXT:    # ymm13 = mem[0,1,0,1]
+; AVX512BW-NEXT:    vpermi2q %zmm4, %zmm5, %zmm13
+; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm9, %zmm5
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm3 {%k2}
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm4 = [11,4,11,4,11,4,11,4]
+; AVX512BW-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpermi2q %zmm1, %zmm2, %zmm4
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm5 = [0,5,6,10,0,5,6,10]
+; AVX512BW-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpermi2q %zmm0, %zmm4, %zmm5
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm3 {%k1}
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm4 = [12,5,12,5,12,5,12,5]
+; AVX512BW-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpermi2q %zmm1, %zmm2, %zmm4
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm5 = [0,5,6,11,0,5,6,11]
+; AVX512BW-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpermi2q %zmm0, %zmm4, %zmm5
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm7 {%k1}
+; AVX512BW-NEXT:    vpermi2q %zmm1, %zmm2, %zmm9
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [0,5,6,12,0,5,6,12]
 ; AVX512BW-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermi2q %zmm1, %zmm11, %zmm4
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm1 = [6,13]
-; AVX512BW-NEXT:    vpermi2q %zmm2, %zmm3, %zmm1
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm14[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm4, %zmm1
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rsi)
-; AVX512BW-NEXT:    vmovdqa64 %zmm6, (%rdx)
-; AVX512BW-NEXT:    vmovdqa64 %zmm8, (%rcx)
-; AVX512BW-NEXT:    vmovdqa64 %zmm9, (%r8)
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, (%r9)
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, (%r10)
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, (%rax)
+; AVX512BW-NEXT:    vpermi2q %zmm0, %zmm9, %zmm4
+; AVX512BW-NEXT:    vmovdqa 64(%rdi), %ymm5
+; AVX512BW-NEXT:    vpalignr {{.*#+}} ymm5 = mem[8,9,10,11,12,13,14,15],ymm5[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm5[16,17,18,19,20,21,22,23]
+; AVX512BW-NEXT:    vextracti128 $1, %ymm5, %xmm5
+; AVX512BW-NEXT:    vinserti32x4 $0, %xmm5, %zmm14, %zmm5
+; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm5 {%k1}
+; AVX512BW-NEXT:    vpermi2q %zmm2, %zmm1, %zmm6
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [4,5,6,14,4,5,6,14]
+; AVX512BW-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpermi2q %zmm0, %zmm6, %zmm4
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm11, %zmm4, %zmm4
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm6 = [2,9,2,9,2,9,2,9]
+; AVX512BW-NEXT:    # zmm6 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpermi2q %zmm1, %zmm2, %zmm6
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm1 = [4,5,8,15,4,5,8,15]
+; AVX512BW-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpermi2q %zmm0, %zmm6, %zmm1
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm12[0,1,2,3],ymm13[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm1, %zmm0
+; AVX512BW-NEXT:    vmovdqa64 %zmm10, (%rsi)
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, (%rdx)
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, (%rcx)
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, (%r8)
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, (%r9)
+; AVX512BW-NEXT:    vmovdqa64 %zmm4, (%r10)
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rax)
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
   %wide.vec = load <56 x i64>, ptr %in.vec, align 64
@@ -1724,183 +1722,185 @@ define void @load_i64_stride7_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512F-NEXT:    movq {{[0-9]+}}(%rsp), %r10
-; AVX512F-NEXT:    vmovdqa64 832(%rdi), %zmm28
-; AVX512F-NEXT:    vmovdqa64 768(%rdi), %zmm30
+; AVX512F-NEXT:    vmovdqa64 832(%rdi), %zmm30
+; AVX512F-NEXT:    vmovdqa64 768(%rdi), %zmm6
 ; AVX512F-NEXT:    vmovdqa64 704(%rdi), %zmm0
-; AVX512F-NEXT:    vmovdqa64 640(%rdi), %zmm25
-; AVX512F-NEXT:    vmovdqa64 576(%rdi), %zmm2
-; AVX512F-NEXT:    vmovdqa64 512(%rdi), %zmm24
-; AVX512F-NEXT:    vmovdqa64 448(%rdi), %zmm6
-; AVX512F-NEXT:    vmovdqa64 384(%rdi), %zmm14
+; AVX512F-NEXT:    vmovdqa64 640(%rdi), %zmm26
+; AVX512F-NEXT:    vmovdqa64 576(%rdi), %zmm3
+; AVX512F-NEXT:    vmovdqa64 512(%rdi), %zmm27
+; AVX512F-NEXT:    vmovdqa64 448(%rdi), %zmm2
+; AVX512F-NEXT:    vmovdqa64 384(%rdi), %zmm13
 ; AVX512F-NEXT:    vmovdqa64 320(%rdi), %zmm15
-; AVX512F-NEXT:    vmovdqa64 256(%rdi), %zmm9
-; AVX512F-NEXT:    vmovdqa64 192(%rdi), %zmm27
-; AVX512F-NEXT:    vmovdqa64 128(%rdi), %zmm10
-; AVX512F-NEXT:    vmovdqa64 64(%rdi), %zmm26
+; AVX512F-NEXT:    vmovdqa64 256(%rdi), %zmm8
+; AVX512F-NEXT:    vmovdqa64 192(%rdi), %zmm29
+; AVX512F-NEXT:    vmovdqa64 128(%rdi), %zmm9
+; AVX512F-NEXT:    vmovdqa64 64(%rdi), %zmm28
 ; AVX512F-NEXT:    vmovdqa64 (%rdi), %zmm1
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm7 = <0,7,14,u>
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm29
-; AVX512F-NEXT:    vpermt2q %zmm26, %zmm7, %zmm29
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm18 = [0,7,14,0,0,7,14,0]
+; AVX512F-NEXT:    # zmm18 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm31
+; AVX512F-NEXT:    vpermt2q %zmm28, %zmm18, %zmm31
 ; AVX512F-NEXT:    movb $24, %r11b
 ; AVX512F-NEXT:    kmovw %r11d, %k2
-; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm29 {%k2} = zmm10[4,5,4,5],zmm27[4,5,4,5]
+; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm31 {%k2} = zmm9[4,5,4,5],zmm29[4,5,4,5]
 ; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm16 = [10,3,10,3,10,3,10,3]
 ; AVX512F-NEXT:    # zmm16 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm17
-; AVX512F-NEXT:    vpermt2q %zmm15, %zmm16, %zmm17
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm18 = [0,5,6,9,0,5,6,9]
-; AVX512F-NEXT:    # zmm18 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermt2q %zmm14, %zmm18, %zmm17
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm14
+; AVX512F-NEXT:    vpermt2q %zmm15, %zmm16, %zmm14
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm17 = [0,5,6,9,0,5,6,9]
+; AVX512F-NEXT:    # zmm17 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpermt2q %zmm13, %zmm17, %zmm14
 ; AVX512F-NEXT:    movb $-32, %r11b
 ; AVX512F-NEXT:    kmovw %r11d, %k1
-; AVX512F-NEXT:    vmovdqa64 %zmm17, %zmm29 {%k1}
-; AVX512F-NEXT:    vpermi2q %zmm24, %zmm6, %zmm7
-; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm7 {%k2} = zmm2[4,5,4,5],zmm25[4,5,4,5]
-; AVX512F-NEXT:    vpermi2q %zmm30, %zmm0, %zmm16
-; AVX512F-NEXT:    vpermt2q %zmm28, %zmm18, %zmm16
-; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm7 {%k1}
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm18 = [11,4,11,4,11,4,11,4]
-; AVX512F-NEXT:    # zmm18 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm19
-; AVX512F-NEXT:    vpermt2q %zmm15, %zmm18, %zmm19
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm20 = [0,5,6,10,0,5,6,10]
+; AVX512F-NEXT:    vmovdqa64 %zmm14, %zmm31 {%k1}
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm14
+; AVX512F-NEXT:    vpermt2q %zmm27, %zmm18, %zmm14
+; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm14 {%k2} = zmm3[4,5,4,5],zmm26[4,5,4,5]
+; AVX512F-NEXT:    vpermi2q %zmm6, %zmm0, %zmm16
+; AVX512F-NEXT:    vpermt2q %zmm30, %zmm17, %zmm16
+; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm14 {%k1}
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm24 = [13,6,13,6,13,6,13,6]
+; AVX512F-NEXT:    # zmm24 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm17
+; AVX512F-NEXT:    vpermt2q %zmm29, %zmm24, %zmm17
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm20 = [9,0,7,0,9,0,7,0]
 ; AVX512F-NEXT:    # zmm20 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermt2q %zmm14, %zmm20, %zmm19
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm22 = [13,6,13,6,13,6,13,6]
-; AVX512F-NEXT:    # zmm22 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm21
-; AVX512F-NEXT:    vpermt2q %zmm27, %zmm22, %zmm21
-; AVX512F-NEXT:    vmovdqa64 {{.*#+}} ymm16 = <9,0,7,u>
-; AVX512F-NEXT:    vmovdqa64 %zmm26, %zmm17
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm16, %zmm17
-; AVX512F-NEXT:    vmovdqa64 %zmm21, %zmm17 {%k2}
-; AVX512F-NEXT:    vmovdqa64 %zmm19, %zmm17 {%k1}
-; AVX512F-NEXT:    vpermi2q %zmm30, %zmm0, %zmm18
-; AVX512F-NEXT:    vpermt2q %zmm28, %zmm20, %zmm18
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm19
-; AVX512F-NEXT:    vpermt2q %zmm25, %zmm22, %zmm19
-; AVX512F-NEXT:    vpermi2q %zmm6, %zmm24, %zmm16
-; AVX512F-NEXT:    vmovdqa64 %zmm19, %zmm16 {%k2}
-; AVX512F-NEXT:    vmovdqa64 %zmm18, %zmm16 {%k1}
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm20 = [12,5,12,5,12,5,12,5]
-; AVX512F-NEXT:    # zmm20 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm19
-; AVX512F-NEXT:    vpermt2q %zmm15, %zmm20, %zmm19
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm21 = [0,5,6,11,0,5,6,11]
+; AVX512F-NEXT:    vmovdqa64 %zmm28, %zmm16
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm20, %zmm16
+; AVX512F-NEXT:    vmovdqa64 %zmm17, %zmm16 {%k2}
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm19 = [11,4,11,4,11,4,11,4]
+; AVX512F-NEXT:    # zmm19 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm17
+; AVX512F-NEXT:    vpermt2q %zmm15, %zmm19, %zmm17
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm21 = [0,5,6,10,0,5,6,10]
 ; AVX512F-NEXT:    # zmm21 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermt2q %zmm14, %zmm21, %zmm19
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm23 = [14,0,0,7,14,0,0,7]
+; AVX512F-NEXT:    vpermt2q %zmm13, %zmm21, %zmm17
+; AVX512F-NEXT:    vmovdqa64 %zmm17, %zmm16 {%k1}
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm22
+; AVX512F-NEXT:    vpermt2q %zmm26, %zmm24, %zmm22
+; AVX512F-NEXT:    vmovdqa64 %zmm27, %zmm17
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm20, %zmm17
+; AVX512F-NEXT:    vmovdqa64 %zmm22, %zmm17 {%k2}
+; AVX512F-NEXT:    vpermi2q %zmm6, %zmm0, %zmm19
+; AVX512F-NEXT:    vpermt2q %zmm30, %zmm21, %zmm19
+; AVX512F-NEXT:    vmovdqa64 %zmm19, %zmm17 {%k1}
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm22 = [12,5,12,5,12,5,12,5]
+; AVX512F-NEXT:    # zmm22 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm21
+; AVX512F-NEXT:    vpermt2q %zmm15, %zmm22, %zmm21
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm23 = [0,5,6,11,0,5,6,11]
 ; AVX512F-NEXT:    # zmm23 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm18
-; AVX512F-NEXT:    vpermt2q %zmm27, %zmm23, %zmm18
-; AVX512F-NEXT:    vmovdqa 16(%rdi), %xmm4
-; AVX512F-NEXT:    vpblendd {{.*#+}} xmm4 = xmm4[0,1],mem[2,3]
-; AVX512F-NEXT:    vinserti32x4 $0, %xmm4, %zmm18, %zmm18
-; AVX512F-NEXT:    vmovdqa64 %zmm19, %zmm18 {%k1}
-; AVX512F-NEXT:    vpermi2q %zmm30, %zmm0, %zmm20
-; AVX512F-NEXT:    vpermt2q %zmm28, %zmm21, %zmm20
-; AVX512F-NEXT:    vpermi2q %zmm25, %zmm2, %zmm23
-; AVX512F-NEXT:    vmovdqa 464(%rdi), %xmm4
-; AVX512F-NEXT:    vpblendd {{.*#+}} xmm4 = xmm4[0,1],mem[2,3]
-; AVX512F-NEXT:    vinserti32x4 $0, %xmm4, %zmm23, %zmm19
-; AVX512F-NEXT:    vmovdqa64 %zmm20, %zmm19 {%k1}
-; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm4
-; AVX512F-NEXT:    vpermt2q %zmm15, %zmm22, %zmm4
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm21 = [0,5,6,12,0,5,6,12]
-; AVX512F-NEXT:    # zmm21 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermt2q %zmm14, %zmm21, %zmm4
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm23 = [7,0,9,0,7,0,9,0]
+; AVX512F-NEXT:    vpermt2q %zmm13, %zmm23, %zmm21
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm25 = [14,0,0,7,14,0,0,7]
+; AVX512F-NEXT:    # zmm25 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm19
+; AVX512F-NEXT:    vpermt2q %zmm29, %zmm25, %zmm19
+; AVX512F-NEXT:    vmovdqa 16(%rdi), %xmm5
+; AVX512F-NEXT:    vpblendd {{.*#+}} xmm5 = xmm5[0,1],mem[2,3]
+; AVX512F-NEXT:    vinserti32x4 $0, %xmm5, %zmm19, %zmm19
+; AVX512F-NEXT:    vmovdqa64 %zmm21, %zmm19 {%k1}
+; AVX512F-NEXT:    vpermi2q %zmm6, %zmm0, %zmm22
+; AVX512F-NEXT:    vpermt2q %zmm30, %zmm23, %zmm22
+; AVX512F-NEXT:    vpermi2q %zmm26, %zmm3, %zmm25
+; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa 464(%rdi), %xmm5
+; AVX512F-NEXT:    vpblendd {{.*#+}} xmm5 = xmm5[0,1],mem[2,3]
+; AVX512F-NEXT:    vinserti32x4 $0, %xmm5, %zmm25, %zmm21
+; AVX512F-NEXT:    vmovdqa64 %zmm22, %zmm21 {%k1}
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm5
+; AVX512F-NEXT:    vpermt2q %zmm15, %zmm24, %zmm5
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm23 = [0,5,6,12,0,5,6,12]
 ; AVX512F-NEXT:    # zmm23 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm27, %zmm20
-; AVX512F-NEXT:    vpermt2q %zmm10, %zmm23, %zmm20
-; AVX512F-NEXT:    vmovdqa 64(%rdi), %ymm5
-; AVX512F-NEXT:    vpalignr {{.*#+}} ymm5 = mem[8,9,10,11,12,13,14,15],ymm5[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm5[16,17,18,19,20,21,22,23]
-; AVX512F-NEXT:    vextracti128 $1, %ymm5, %xmm5
-; AVX512F-NEXT:    vinserti32x4 $0, %xmm5, %zmm20, %zmm20
-; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm20 {%k1}
-; AVX512F-NEXT:    vpermi2q %zmm30, %zmm0, %zmm22
-; AVX512F-NEXT:    vpermt2q %zmm28, %zmm21, %zmm22
-; AVX512F-NEXT:    vpermi2q %zmm2, %zmm25, %zmm23
+; AVX512F-NEXT:    vpermt2q %zmm13, %zmm23, %zmm5
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm25 = [7,0,9,0,7,0,9,0]
+; AVX512F-NEXT:    # zmm25 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm29, %zmm22
+; AVX512F-NEXT:    vpermt2q %zmm9, %zmm25, %zmm22
+; AVX512F-NEXT:    vmovdqa 64(%rdi), %ymm4
+; AVX512F-NEXT:    vpalignr {{.*#+}} ymm4 = mem[8,9,10,11,12,13,14,15],ymm4[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm4[16,17,18,19,20,21,22,23]
+; AVX512F-NEXT:    vextracti128 $1, %ymm4, %xmm4
+; AVX512F-NEXT:    vinserti32x4 $0, %xmm4, %zmm22, %zmm22
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm22 {%k1}
+; AVX512F-NEXT:    vpermi2q %zmm6, %zmm0, %zmm24
+; AVX512F-NEXT:    vpermt2q %zmm30, %zmm23, %zmm24
+; AVX512F-NEXT:    vpermi2q %zmm3, %zmm26, %zmm25
 ; AVX512F-NEXT:    vmovdqa 512(%rdi), %ymm4
 ; AVX512F-NEXT:    vpalignr {{.*#+}} ymm4 = mem[8,9,10,11,12,13,14,15],ymm4[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm4[16,17,18,19,20,21,22,23]
 ; AVX512F-NEXT:    vextracti128 $1, %ymm4, %xmm4
-; AVX512F-NEXT:    vinserti32x4 $0, %xmm4, %zmm23, %zmm21
-; AVX512F-NEXT:    vmovdqa64 %zmm22, %zmm21 {%k1}
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [0,7,14,0,0,7,14,0]
-; AVX512F-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm5
-; AVX512F-NEXT:    vpermt2q %zmm15, %zmm4, %zmm5
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm23 = [4,5,6,13,4,5,6,13]
-; AVX512F-NEXT:    # zmm23 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermt2q %zmm14, %zmm23, %zmm5
-; AVX512F-NEXT:    vinserti128 $1, 192(%rdi), %ymm0, %ymm12
-; AVX512F-NEXT:    vmovdqa 128(%rdi), %ymm13
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm12 = ymm13[0,1,2,3,4,5],ymm12[6,7]
+; AVX512F-NEXT:    vinserti32x4 $0, %xmm4, %zmm25, %zmm23
+; AVX512F-NEXT:    vmovdqa64 %zmm24, %zmm23 {%k1}
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm4
+; AVX512F-NEXT:    vpermt2q %zmm15, %zmm18, %zmm4
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm5 = [4,5,6,13,4,5,6,13]
+; AVX512F-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpermt2q %zmm13, %zmm5, %zmm4
+; AVX512F-NEXT:    vinserti128 $1, 192(%rdi), %ymm0, %ymm11
+; AVX512F-NEXT:    vmovdqa 128(%rdi), %ymm12
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm11 = ymm12[0,1,2,3,4,5],ymm11[6,7]
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm3 = [4,11]
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm11
-; AVX512F-NEXT:    vpermt2q %zmm26, %zmm3, %zmm11
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm11 = ymm11[0,1,2,3],ymm12[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm11, %zmm5, %zmm22
-; AVX512F-NEXT:    vpermi2q %zmm30, %zmm0, %zmm4
-; AVX512F-NEXT:    vpermt2q %zmm28, %zmm23, %zmm4
-; AVX512F-NEXT:    vinserti128 $1, 640(%rdi), %ymm0, %ymm5
-; AVX512F-NEXT:    vmovdqa 576(%rdi), %ymm11
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm5 = ymm11[0,1,2,3,4,5],ymm5[6,7]
-; AVX512F-NEXT:    vpermi2q %zmm24, %zmm6, %zmm3
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm5[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm3, %zmm4, %zmm23
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm3 = [9,0,7,0,9,0,7,0]
-; AVX512F-NEXT:    # zmm3 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm10
+; AVX512F-NEXT:    vpermt2q %zmm28, %zmm3, %zmm10
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm10 = ymm10[0,1,2,3],ymm11[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm10, %zmm4, %zmm24
+; AVX512F-NEXT:    vpermi2q %zmm6, %zmm0, %zmm18
+; AVX512F-NEXT:    vpermt2q %zmm30, %zmm5, %zmm18
+; AVX512F-NEXT:    vinserti128 $1, 640(%rdi), %ymm0, %ymm4
+; AVX512F-NEXT:    vmovdqa 576(%rdi), %ymm5
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4,5],ymm4[6,7]
+; AVX512F-NEXT:    vpermi2q %zmm27, %zmm2, %zmm3
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm4[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm3, %zmm18, %zmm3
 ; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm4
-; AVX512F-NEXT:    vpermt2q %zmm9, %zmm3, %zmm4
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm5 = [4,5,6,14,4,5,6,14]
-; AVX512F-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermt2q %zmm14, %zmm5, %zmm4
-; AVX512F-NEXT:    vmovdqa 192(%rdi), %ymm12
-; AVX512F-NEXT:    vpalignr {{.*#+}} ymm12 = ymm13[8,9,10,11,12,13,14,15],ymm12[0,1,2,3,4,5,6,7],ymm13[24,25,26,27,28,29,30,31],ymm12[16,17,18,19,20,21,22,23]
-; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm13 = [5,12]
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm8
-; AVX512F-NEXT:    vpermt2q %zmm26, %zmm13, %zmm8
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3],ymm12[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm8, %zmm4, %zmm31
-; AVX512F-NEXT:    vpermi2q %zmm0, %zmm30, %zmm3
-; AVX512F-NEXT:    vpermt2q %zmm28, %zmm5, %zmm3
-; AVX512F-NEXT:    vmovdqa 640(%rdi), %ymm5
-; AVX512F-NEXT:    vpalignr {{.*#+}} ymm5 = ymm11[8,9,10,11,12,13,14,15],ymm5[0,1,2,3,4,5,6,7],ymm11[24,25,26,27,28,29,30,31],ymm5[16,17,18,19,20,21,22,23]
-; AVX512F-NEXT:    vpermi2q %zmm24, %zmm6, %zmm13
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm5 = ymm13[0,1,2,3],ymm5[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm5, %zmm3, %zmm3
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm5 = [2,9,2,9,2,9,2,9]
-; AVX512F-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermt2q %zmm15, %zmm5, %zmm9
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm8 = [4,5,8,15,4,5,8,15]
-; AVX512F-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermt2q %zmm14, %zmm8, %zmm9
+; AVX512F-NEXT:    vpermt2q %zmm8, %zmm20, %zmm4
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm10 = [4,5,6,14,4,5,6,14]
+; AVX512F-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpermt2q %zmm13, %zmm10, %zmm4
+; AVX512F-NEXT:    vmovdqa 192(%rdi), %ymm11
+; AVX512F-NEXT:    vpalignr {{.*#+}} ymm11 = ymm12[8,9,10,11,12,13,14,15],ymm11[0,1,2,3,4,5,6,7],ymm12[24,25,26,27,28,29,30,31],ymm11[16,17,18,19,20,21,22,23]
+; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm12 = [5,12]
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm7
+; AVX512F-NEXT:    vpermt2q %zmm28, %zmm12, %zmm7
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm7 = ymm7[0,1,2,3],ymm11[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm7, %zmm4, %zmm4
+; AVX512F-NEXT:    vpermi2q %zmm0, %zmm6, %zmm20
+; AVX512F-NEXT:    vpermt2q %zmm30, %zmm10, %zmm20
+; AVX512F-NEXT:    vmovdqa 640(%rdi), %ymm7
+; AVX512F-NEXT:    vpalignr {{.*#+}} ymm5 = ymm5[8,9,10,11,12,13,14,15],ymm7[0,1,2,3,4,5,6,7],ymm5[24,25,26,27,28,29,30,31],ymm7[16,17,18,19,20,21,22,23]
+; AVX512F-NEXT:    vpermi2q %zmm27, %zmm2, %zmm12
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm5 = ymm12[0,1,2,3],ymm5[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm5, %zmm20, %zmm5
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm7 = [2,9,2,9,2,9,2,9]
+; AVX512F-NEXT:    # zmm7 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpermt2q %zmm15, %zmm7, %zmm8
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm10 = [4,5,8,15,4,5,8,15]
+; AVX512F-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpermt2q %zmm13, %zmm10, %zmm8
 ; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm11 = [4,11,4,11]
 ; AVX512F-NEXT:    # ymm11 = mem[0,1,0,1]
-; AVX512F-NEXT:    vpermt2q %zmm27, %zmm11, %zmm10
+; AVX512F-NEXT:    vpermt2q %zmm29, %zmm11, %zmm9
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm12 = [6,13]
-; AVX512F-NEXT:    vpermt2q %zmm26, %zmm12, %zmm1
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm10[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm9, %zmm1
-; AVX512F-NEXT:    vpermt2q %zmm30, %zmm5, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm28, %zmm8, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm25, %zmm11, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm24, %zmm12, %zmm6
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm6[0,1,2,3],ymm2[4,5,6,7]
+; AVX512F-NEXT:    vpermt2q %zmm28, %zmm12, %zmm1
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm9[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm8, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm6, %zmm7, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm30, %zmm10, %zmm0
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm26, %zmm11, %zmm6
+; AVX512F-NEXT:    vpermt2q %zmm27, %zmm12, %zmm2
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm6[4,5,6,7]
 ; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm0, %zmm0
-; AVX512F-NEXT:    vmovdqa64 %zmm7, 64(%rsi)
-; AVX512F-NEXT:    vmovdqa64 %zmm29, (%rsi)
-; AVX512F-NEXT:    vmovdqa64 %zmm16, 64(%rdx)
-; AVX512F-NEXT:    vmovdqa64 %zmm17, (%rdx)
-; AVX512F-NEXT:    vmovdqa64 %zmm19, 64(%rcx)
-; AVX512F-NEXT:    vmovdqa64 %zmm18, (%rcx)
-; AVX512F-NEXT:    vmovdqa64 %zmm21, 64(%r8)
-; AVX512F-NEXT:    vmovdqa64 %zmm20, (%r8)
-; AVX512F-NEXT:    vmovdqa64 %zmm23, 64(%r9)
-; AVX512F-NEXT:    vmovdqa64 %zmm22, (%r9)
-; AVX512F-NEXT:    vmovdqa64 %zmm3, 64(%r10)
-; AVX512F-NEXT:    vmovdqa64 %zmm31, (%r10)
+; AVX512F-NEXT:    vmovdqa64 %zmm14, 64(%rsi)
+; AVX512F-NEXT:    vmovdqa64 %zmm31, (%rsi)
+; AVX512F-NEXT:    vmovdqa64 %zmm17, 64(%rdx)
+; AVX512F-NEXT:    vmovdqa64 %zmm16, (%rdx)
+; AVX512F-NEXT:    vmovdqa64 %zmm21, 64(%rcx)
+; AVX512F-NEXT:    vmovdqa64 %zmm19, (%rcx)
+; AVX512F-NEXT:    vmovdqa64 %zmm23, 64(%r8)
+; AVX512F-NEXT:    vmovdqa64 %zmm22, (%r8)
+; AVX512F-NEXT:    vmovdqa64 %zmm3, 64(%r9)
+; AVX512F-NEXT:    vmovdqa64 %zmm24, (%r9)
+; AVX512F-NEXT:    vmovdqa64 %zmm5, 64(%r10)
+; AVX512F-NEXT:    vmovdqa64 %zmm4, (%r10)
 ; AVX512F-NEXT:    vmovdqa64 %zmm0, 64(%rax)
 ; AVX512F-NEXT:    vmovdqa64 %zmm1, (%rax)
 ; AVX512F-NEXT:    vzeroupper
@@ -1910,183 +1910,185 @@ define void @load_i64_stride7_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %r10
-; AVX512BW-NEXT:    vmovdqa64 832(%rdi), %zmm30
-; AVX512BW-NEXT:    vmovdqa64 768(%rdi), %zmm31
+; AVX512BW-NEXT:    vmovdqa64 832(%rdi), %zmm3
+; AVX512BW-NEXT:    vmovdqa64 768(%rdi), %zmm6
 ; AVX512BW-NEXT:    vmovdqa64 704(%rdi), %zmm0
 ; AVX512BW-NEXT:    vmovdqa64 640(%rdi), %zmm26
-; AVX512BW-NEXT:    vmovdqa64 576(%rdi), %zmm2
-; AVX512BW-NEXT:    vmovdqa64 512(%rdi), %zmm24
-; AVX512BW-NEXT:    vmovdqa64 448(%rdi), %zmm6
-; AVX512BW-NEXT:    vmovdqa64 384(%rdi), %zmm14
+; AVX512BW-NEXT:    vmovdqa64 576(%rdi), %zmm4
+; AVX512BW-NEXT:    vmovdqa64 512(%rdi), %zmm28
+; AVX512BW-NEXT:    vmovdqa64 448(%rdi), %zmm2
+; AVX512BW-NEXT:    vmovdqa64 384(%rdi), %zmm13
 ; AVX512BW-NEXT:    vmovdqa64 320(%rdi), %zmm15
-; AVX512BW-NEXT:    vmovdqa64 256(%rdi), %zmm9
-; AVX512BW-NEXT:    vmovdqa64 192(%rdi), %zmm28
-; AVX512BW-NEXT:    vmovdqa64 128(%rdi), %zmm10
-; AVX512BW-NEXT:    vmovdqa64 64(%rdi), %zmm27
+; AVX512BW-NEXT:    vmovdqa64 256(%rdi), %zmm8
+; AVX512BW-NEXT:    vmovdqa64 192(%rdi), %zmm30
+; AVX512BW-NEXT:    vmovdqa64 128(%rdi), %zmm9
+; AVX512BW-NEXT:    vmovdqa64 64(%rdi), %zmm29
 ; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm1
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm7 = <0,7,14,u>
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm29
-; AVX512BW-NEXT:    vpermt2q %zmm27, %zmm7, %zmm29
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm18 = [0,7,14,0,0,7,14,0]
+; AVX512BW-NEXT:    # zmm18 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm31
+; AVX512BW-NEXT:    vpermt2q %zmm29, %zmm18, %zmm31
 ; AVX512BW-NEXT:    movb $24, %r11b
 ; AVX512BW-NEXT:    kmovd %r11d, %k2
-; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm29 {%k2} = zmm10[4,5,4,5],zmm28[4,5,4,5]
+; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm31 {%k2} = zmm9[4,5,4,5],zmm30[4,5,4,5]
 ; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm16 = [10,3,10,3,10,3,10,3]
 ; AVX512BW-NEXT:    # zmm16 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm17
-; AVX512BW-NEXT:    vpermt2q %zmm15, %zmm16, %zmm17
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm18 = [0,5,6,9,0,5,6,9]
-; AVX512BW-NEXT:    # zmm18 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm18, %zmm17
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm14
+; AVX512BW-NEXT:    vpermt2q %zmm15, %zmm16, %zmm14
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm17 = [0,5,6,9,0,5,6,9]
+; AVX512BW-NEXT:    # zmm17 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm17, %zmm14
 ; AVX512BW-NEXT:    movb $-32, %r11b
 ; AVX512BW-NEXT:    kmovd %r11d, %k1
-; AVX512BW-NEXT:    vmovdqa64 %zmm17, %zmm29 {%k1}
-; AVX512BW-NEXT:    vpermi2q %zmm24, %zmm6, %zmm7
-; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm7 {%k2} = zmm2[4,5,4,5],zmm26[4,5,4,5]
-; AVX512BW-NEXT:    vpermi2q %zmm31, %zmm0, %zmm16
-; AVX512BW-NEXT:    vpermt2q %zmm30, %zmm18, %zmm16
-; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm7 {%k1}
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm18 = [11,4,11,4,11,4,11,4]
-; AVX512BW-NEXT:    # zmm18 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm19
-; AVX512BW-NEXT:    vpermt2q %zmm15, %zmm18, %zmm19
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm20 = [0,5,6,10,0,5,6,10]
+; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm31 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm14
+; AVX512BW-NEXT:    vpermt2q %zmm28, %zmm18, %zmm14
+; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm14 {%k2} = zmm4[4,5,4,5],zmm26[4,5,4,5]
+; AVX512BW-NEXT:    vpermi2q %zmm6, %zmm0, %zmm16
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm17, %zmm16
+; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm14 {%k1}
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm24 = [13,6,13,6,13,6,13,6]
+; AVX512BW-NEXT:    # zmm24 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm17
+; AVX512BW-NEXT:    vpermt2q %zmm30, %zmm24, %zmm17
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm20 = [9,0,7,0,9,0,7,0]
 ; AVX512BW-NEXT:    # zmm20 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm20, %zmm19
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm22 = [13,6,13,6,13,6,13,6]
-; AVX512BW-NEXT:    # zmm22 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm21
-; AVX512BW-NEXT:    vpermt2q %zmm28, %zmm22, %zmm21
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} ymm16 = <9,0,7,u>
-; AVX512BW-NEXT:    vmovdqa64 %zmm27, %zmm17
-; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm16, %zmm17
-; AVX512BW-NEXT:    vmovdqa64 %zmm21, %zmm17 {%k2}
-; AVX512BW-NEXT:    vmovdqa64 %zmm19, %zmm17 {%k1}
-; AVX512BW-NEXT:    vpermi2q %zmm31, %zmm0, %zmm18
-; AVX512BW-NEXT:    vpermt2q %zmm30, %zmm20, %zmm18
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm19
-; AVX512BW-NEXT:    vpermt2q %zmm26, %zmm22, %zmm19
-; AVX512BW-NEXT:    vpermi2q %zmm6, %zmm24, %zmm16
-; AVX512BW-NEXT:    vmovdqa64 %zmm19, %zmm16 {%k2}
-; AVX512BW-NEXT:    vmovdqa64 %zmm18, %zmm16 {%k1}
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm20 = [12,5,12,5,12,5,12,5]
-; AVX512BW-NEXT:    # zmm20 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm19
-; AVX512BW-NEXT:    vpermt2q %zmm15, %zmm20, %zmm19
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm21 = [0,5,6,11,0,5,6,11]
-; AVX512BW-NEXT:    # zmm21 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm21, %zmm19
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm23 = [14,0,0,7,14,0,0,7]
-; AVX512BW-NEXT:    # zmm23 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm18
-; AVX512BW-NEXT:    vpermt2q %zmm28, %zmm23, %zmm18
-; AVX512BW-NEXT:    vmovdqa 16(%rdi), %xmm4
-; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm4 = xmm4[0,1],mem[2,3]
-; AVX512BW-NEXT:    vinserti32x4 $0, %xmm4, %zmm18, %zmm18
-; AVX512BW-NEXT:    vmovdqa64 %zmm19, %zmm18 {%k1}
-; AVX512BW-NEXT:    vpermi2q %zmm31, %zmm0, %zmm20
-; AVX512BW-NEXT:    vpermt2q %zmm30, %zmm21, %zmm20
-; AVX512BW-NEXT:    vpermi2q %zmm26, %zmm2, %zmm23
-; AVX512BW-NEXT:    vmovdqa 464(%rdi), %xmm4
-; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm4 = xmm4[0,1],mem[2,3]
-; AVX512BW-NEXT:    vinserti32x4 $0, %xmm4, %zmm23, %zmm19
-; AVX512BW-NEXT:    vmovdqa64 %zmm20, %zmm19 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm4
-; AVX512BW-NEXT:    vpermt2q %zmm15, %zmm22, %zmm4
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm21 = [0,5,6,12,0,5,6,12]
+; AVX512BW-NEXT:    vmovdqa64 %zmm29, %zmm16
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm20, %zmm16
+; AVX512BW-NEXT:    vmovdqa64 %zmm17, %zmm16 {%k2}
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm19 = [11,4,11,4,11,4,11,4]
+; AVX512BW-NEXT:    # zmm19 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm17
+; AVX512BW-NEXT:    vpermt2q %zmm15, %zmm19, %zmm17
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm21 = [0,5,6,10,0,5,6,10]
 ; AVX512BW-NEXT:    # zmm21 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm21, %zmm4
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm23 = [7,0,9,0,7,0,9,0]
+; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm21, %zmm17
+; AVX512BW-NEXT:    vmovdqa64 %zmm17, %zmm16 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm22
+; AVX512BW-NEXT:    vpermt2q %zmm26, %zmm24, %zmm22
+; AVX512BW-NEXT:    vmovdqa64 %zmm28, %zmm17
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm20, %zmm17
+; AVX512BW-NEXT:    vmovdqa64 %zmm22, %zmm17 {%k2}
+; AVX512BW-NEXT:    vpermi2q %zmm6, %zmm0, %zmm19
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm21, %zmm19
+; AVX512BW-NEXT:    vmovdqa64 %zmm19, %zmm17 {%k1}
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm22 = [12,5,12,5,12,5,12,5]
+; AVX512BW-NEXT:    # zmm22 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm21
+; AVX512BW-NEXT:    vpermt2q %zmm15, %zmm22, %zmm21
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm23 = [0,5,6,11,0,5,6,11]
 ; AVX512BW-NEXT:    # zmm23 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm28, %zmm20
-; AVX512BW-NEXT:    vpermt2q %zmm10, %zmm23, %zmm20
-; AVX512BW-NEXT:    vmovdqa64 64(%rdi), %ymm25
-; AVX512BW-NEXT:    vpalignr {{.*#+}} ymm25 = mem[8,9,10,11,12,13,14,15],ymm25[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm25[16,17,18,19,20,21,22,23]
-; AVX512BW-NEXT:    vextracti32x4 $1, %ymm25, %xmm25
-; AVX512BW-NEXT:    vinserti32x4 $0, %xmm25, %zmm20, %zmm20
-; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm20 {%k1}
-; AVX512BW-NEXT:    vpermi2q %zmm31, %zmm0, %zmm22
-; AVX512BW-NEXT:    vpermt2q %zmm30, %zmm21, %zmm22
-; AVX512BW-NEXT:    vpermi2q %zmm2, %zmm26, %zmm23
-; AVX512BW-NEXT:    vmovdqa 512(%rdi), %ymm4
-; AVX512BW-NEXT:    vpalignr {{.*#+}} ymm4 = mem[8,9,10,11,12,13,14,15],ymm4[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm4[16,17,18,19,20,21,22,23]
-; AVX512BW-NEXT:    vextracti128 $1, %ymm4, %xmm4
-; AVX512BW-NEXT:    vinserti32x4 $0, %xmm4, %zmm23, %zmm21
+; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm23, %zmm21
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm25 = [14,0,0,7,14,0,0,7]
+; AVX512BW-NEXT:    # zmm25 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm19
+; AVX512BW-NEXT:    vpermt2q %zmm30, %zmm25, %zmm19
+; AVX512BW-NEXT:    vmovdqa 16(%rdi), %xmm5
+; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm5 = xmm5[0,1],mem[2,3]
+; AVX512BW-NEXT:    vinserti32x4 $0, %xmm5, %zmm19, %zmm19
+; AVX512BW-NEXT:    vmovdqa64 %zmm21, %zmm19 {%k1}
+; AVX512BW-NEXT:    vpermi2q %zmm6, %zmm0, %zmm22
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm23, %zmm22
+; AVX512BW-NEXT:    vpermi2q %zmm26, %zmm4, %zmm25
+; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa 464(%rdi), %xmm5
+; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm5 = xmm5[0,1],mem[2,3]
+; AVX512BW-NEXT:    vinserti32x4 $0, %xmm5, %zmm25, %zmm21
 ; AVX512BW-NEXT:    vmovdqa64 %zmm22, %zmm21 {%k1}
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [0,7,14,0,0,7,14,0]
-; AVX512BW-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm22
-; AVX512BW-NEXT:    vpermt2q %zmm15, %zmm4, %zmm22
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm23 = [4,5,6,13,4,5,6,13]
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm5
+; AVX512BW-NEXT:    vpermt2q %zmm15, %zmm24, %zmm5
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm23 = [0,5,6,12,0,5,6,12]
 ; AVX512BW-NEXT:    # zmm23 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm23, %zmm22
-; AVX512BW-NEXT:    vinserti128 $1, 192(%rdi), %ymm0, %ymm5
-; AVX512BW-NEXT:    vmovdqa 128(%rdi), %ymm12
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm5 = ymm12[0,1,2,3,4,5],ymm5[6,7]
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm13 = [4,11]
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm11
-; AVX512BW-NEXT:    vpermt2q %zmm27, %zmm13, %zmm11
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm5 = ymm11[0,1,2,3],ymm5[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm5, %zmm22, %zmm22
-; AVX512BW-NEXT:    vpermi2q %zmm31, %zmm0, %zmm4
-; AVX512BW-NEXT:    vpermt2q %zmm30, %zmm23, %zmm4
-; AVX512BW-NEXT:    vinserti128 $1, 640(%rdi), %ymm0, %ymm5
-; AVX512BW-NEXT:    vmovdqa 576(%rdi), %ymm11
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm5 = ymm11[0,1,2,3,4,5],ymm5[6,7]
-; AVX512BW-NEXT:    vpermi2q %zmm24, %zmm6, %zmm13
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm5 = ymm13[0,1,2,3],ymm5[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm5, %zmm4, %zmm23
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [9,0,7,0,9,0,7,0]
-; AVX512BW-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm5
-; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm4, %zmm5
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm13 = [4,5,6,14,4,5,6,14]
-; AVX512BW-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm13, %zmm5
-; AVX512BW-NEXT:    vmovdqa64 192(%rdi), %ymm25
-; AVX512BW-NEXT:    vpalignr {{.*#+}} ymm12 = ymm12[8,9,10,11,12,13,14,15],ymm25[0,1,2,3,4,5,6,7],ymm12[24,25,26,27,28,29,30,31],ymm25[16,17,18,19,20,21,22,23]
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm3 = [5,12]
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm8
-; AVX512BW-NEXT:    vpermt2q %zmm27, %zmm3, %zmm8
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3],ymm12[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm8, %zmm5, %zmm5
-; AVX512BW-NEXT:    vpermi2q %zmm0, %zmm31, %zmm4
-; AVX512BW-NEXT:    vpermt2q %zmm30, %zmm13, %zmm4
-; AVX512BW-NEXT:    vmovdqa 640(%rdi), %ymm8
-; AVX512BW-NEXT:    vpalignr {{.*#+}} ymm8 = ymm11[8,9,10,11,12,13,14,15],ymm8[0,1,2,3,4,5,6,7],ymm11[24,25,26,27,28,29,30,31],ymm8[16,17,18,19,20,21,22,23]
-; AVX512BW-NEXT:    vpermi2q %zmm24, %zmm6, %zmm3
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm8[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm3, %zmm4, %zmm3
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm4 = [2,9,2,9,2,9,2,9]
-; AVX512BW-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermt2q %zmm15, %zmm4, %zmm9
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm8 = [4,5,8,15,4,5,8,15]
-; AVX512BW-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm8, %zmm9
+; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm23, %zmm5
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm25 = [7,0,9,0,7,0,9,0]
+; AVX512BW-NEXT:    # zmm25 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm30, %zmm22
+; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm25, %zmm22
+; AVX512BW-NEXT:    vmovdqa64 64(%rdi), %ymm27
+; AVX512BW-NEXT:    vpalignr {{.*#+}} ymm27 = mem[8,9,10,11,12,13,14,15],ymm27[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm27[16,17,18,19,20,21,22,23]
+; AVX512BW-NEXT:    vextracti32x4 $1, %ymm27, %xmm27
+; AVX512BW-NEXT:    vinserti32x4 $0, %xmm27, %zmm22, %zmm22
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm22 {%k1}
+; AVX512BW-NEXT:    vpermi2q %zmm6, %zmm0, %zmm24
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm23, %zmm24
+; AVX512BW-NEXT:    vpermi2q %zmm4, %zmm26, %zmm25
+; AVX512BW-NEXT:    vmovdqa 512(%rdi), %ymm5
+; AVX512BW-NEXT:    vpalignr {{.*#+}} ymm5 = mem[8,9,10,11,12,13,14,15],ymm5[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm5[16,17,18,19,20,21,22,23]
+; AVX512BW-NEXT:    vextracti128 $1, %ymm5, %xmm5
+; AVX512BW-NEXT:    vinserti32x4 $0, %xmm5, %zmm25, %zmm23
+; AVX512BW-NEXT:    vmovdqa64 %zmm24, %zmm23 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm5
+; AVX512BW-NEXT:    vpermt2q %zmm15, %zmm18, %zmm5
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm25 = [4,5,6,13,4,5,6,13]
+; AVX512BW-NEXT:    # zmm25 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm25, %zmm5
+; AVX512BW-NEXT:    vinserti128 $1, 192(%rdi), %ymm0, %ymm4
+; AVX512BW-NEXT:    vmovdqa 128(%rdi), %ymm11
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm11[0,1,2,3,4,5],ymm4[6,7]
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm12 = [4,11]
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm10
+; AVX512BW-NEXT:    vpermt2q %zmm29, %zmm12, %zmm10
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm10[0,1,2,3],ymm4[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm4, %zmm5, %zmm24
+; AVX512BW-NEXT:    vpermi2q %zmm6, %zmm0, %zmm18
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm25, %zmm18
+; AVX512BW-NEXT:    vinserti128 $1, 640(%rdi), %ymm0, %ymm4
+; AVX512BW-NEXT:    vmovdqa 576(%rdi), %ymm5
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3,4,5],ymm4[6,7]
+; AVX512BW-NEXT:    vpermi2q %zmm28, %zmm2, %zmm12
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm12[0,1,2,3],ymm4[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm4, %zmm18, %zmm25
+; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm10
+; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm20, %zmm10
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm12 = [4,5,6,14,4,5,6,14]
+; AVX512BW-NEXT:    # zmm12 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm12, %zmm10
+; AVX512BW-NEXT:    vmovdqa64 192(%rdi), %ymm18
+; AVX512BW-NEXT:    vpalignr {{.*#+}} ymm11 = ymm11[8,9,10,11,12,13,14,15],ymm18[0,1,2,3,4,5,6,7],ymm11[24,25,26,27,28,29,30,31],ymm18[16,17,18,19,20,21,22,23]
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm4 = [5,12]
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm7
+; AVX512BW-NEXT:    vpermt2q %zmm29, %zmm4, %zmm7
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm7 = ymm7[0,1,2,3],ymm11[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm7, %zmm10, %zmm7
+; AVX512BW-NEXT:    vpermi2q %zmm0, %zmm6, %zmm20
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm12, %zmm20
+; AVX512BW-NEXT:    vmovdqa 640(%rdi), %ymm10
+; AVX512BW-NEXT:    vpalignr {{.*#+}} ymm5 = ymm5[8,9,10,11,12,13,14,15],ymm10[0,1,2,3,4,5,6,7],ymm5[24,25,26,27,28,29,30,31],ymm10[16,17,18,19,20,21,22,23]
+; AVX512BW-NEXT:    vpermi2q %zmm28, %zmm2, %zmm4
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm5[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm4, %zmm20, %zmm4
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm5 = [2,9,2,9,2,9,2,9]
+; AVX512BW-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpermt2q %zmm15, %zmm5, %zmm8
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm10 = [4,5,8,15,4,5,8,15]
+; AVX512BW-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm10, %zmm8
 ; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm11 = [4,11,4,11]
 ; AVX512BW-NEXT:    # ymm11 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vpermt2q %zmm28, %zmm11, %zmm10
+; AVX512BW-NEXT:    vpermt2q %zmm30, %zmm11, %zmm9
 ; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm12 = [6,13]
-; AVX512BW-NEXT:    vpermt2q %zmm27, %zmm12, %zmm1
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm10[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm9, %zmm1
-; AVX512BW-NEXT:    vpermt2q %zmm31, %zmm4, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm30, %zmm8, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm26, %zmm11, %zmm2
-; AVX512BW-NEXT:    vpermt2q %zmm24, %zmm12, %zmm6
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm6[0,1,2,3],ymm2[4,5,6,7]
+; AVX512BW-NEXT:    vpermt2q %zmm29, %zmm12, %zmm1
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm9[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm8, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm5, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm10, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm26, %zmm11, %zmm3
+; AVX512BW-NEXT:    vpermt2q %zmm28, %zmm12, %zmm2
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5,6,7]
 ; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm0, %zmm0
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, 64(%rsi)
-; AVX512BW-NEXT:    vmovdqa64 %zmm29, (%rsi)
-; AVX512BW-NEXT:    vmovdqa64 %zmm16, 64(%rdx)
-; AVX512BW-NEXT:    vmovdqa64 %zmm17, (%rdx)
-; AVX512BW-NEXT:    vmovdqa64 %zmm19, 64(%rcx)
-; AVX512BW-NEXT:    vmovdqa64 %zmm18, (%rcx)
-; AVX512BW-NEXT:    vmovdqa64 %zmm21, 64(%r8)
-; AVX512BW-NEXT:    vmovdqa64 %zmm20, (%r8)
-; AVX512BW-NEXT:    vmovdqa64 %zmm23, 64(%r9)
-; AVX512BW-NEXT:    vmovdqa64 %zmm22, (%r9)
-; AVX512BW-NEXT:    vmovdqa64 %zmm3, 64(%r10)
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, (%r10)
+; AVX512BW-NEXT:    vmovdqa64 %zmm14, 64(%rsi)
+; AVX512BW-NEXT:    vmovdqa64 %zmm31, (%rsi)
+; AVX512BW-NEXT:    vmovdqa64 %zmm17, 64(%rdx)
+; AVX512BW-NEXT:    vmovdqa64 %zmm16, (%rdx)
+; AVX512BW-NEXT:    vmovdqa64 %zmm21, 64(%rcx)
+; AVX512BW-NEXT:    vmovdqa64 %zmm19, (%rcx)
+; AVX512BW-NEXT:    vmovdqa64 %zmm23, 64(%r8)
+; AVX512BW-NEXT:    vmovdqa64 %zmm22, (%r8)
+; AVX512BW-NEXT:    vmovdqa64 %zmm25, 64(%r9)
+; AVX512BW-NEXT:    vmovdqa64 %zmm24, (%r9)
+; AVX512BW-NEXT:    vmovdqa64 %zmm4, 64(%r10)
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, (%r10)
 ; AVX512BW-NEXT:    vmovdqa64 %zmm0, 64(%rax)
 ; AVX512BW-NEXT:    vmovdqa64 %zmm1, (%rax)
 ; AVX512BW-NEXT:    vzeroupper
@@ -3680,892 +3682,960 @@ define void @load_i64_stride7_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ;
 ; AVX512F-LABEL: load_i64_stride7_vf32:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    subq $2216, %rsp # imm = 0x8A8
-; AVX512F-NEXT:    vmovdqa64 1664(%rdi), %zmm21
-; AVX512F-NEXT:    vmovdqa64 1600(%rdi), %zmm3
-; AVX512F-NEXT:    vmovdqa64 1216(%rdi), %zmm1
-; AVX512F-NEXT:    vmovdqa64 1152(%rdi), %zmm6
-; AVX512F-NEXT:    vmovdqa64 1088(%rdi), %zmm11
-; AVX512F-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovaps 1024(%rdi), %zmm0
-; AVX512F-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    subq $2728, %rsp # imm = 0xAA8
+; AVX512F-NEXT:    vmovdqa64 1216(%rdi), %zmm25
+; AVX512F-NEXT:    vmovdqa64 1152(%rdi), %zmm1
+; AVX512F-NEXT:    vmovdqa64 960(%rdi), %zmm31
+; AVX512F-NEXT:    vmovdqa64 896(%rdi), %zmm12
 ; AVX512F-NEXT:    vmovdqa64 768(%rdi), %zmm2
-; AVX512F-NEXT:    vmovdqa64 704(%rdi), %zmm9
-; AVX512F-NEXT:    vmovdqa64 640(%rdi), %zmm15
-; AVX512F-NEXT:    vmovdqa64 576(%rdi), %zmm10
-; AVX512F-NEXT:    vmovdqa64 320(%rdi), %zmm0
-; AVX512F-NEXT:    vmovdqa64 256(%rdi), %zmm26
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm7 = [10,3,10,3,10,3,10,3]
-; AVX512F-NEXT:    # zmm7 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm26, %zmm8
-; AVX512F-NEXT:    vpermt2q %zmm0, %zmm7, %zmm8
-; AVX512F-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm8
-; AVX512F-NEXT:    vpermt2q %zmm2, %zmm7, %zmm8
-; AVX512F-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm8
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm7, %zmm8
-; AVX512F-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermi2q %zmm21, %zmm3, %zmm7
+; AVX512F-NEXT:    vmovdqa64 704(%rdi), %zmm20
+; AVX512F-NEXT:    vmovdqa64 512(%rdi), %zmm9
+; AVX512F-NEXT:    vmovdqa64 448(%rdi), %zmm7
 ; AVX512F-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm7 = [11,4,11,4,11,4,11,4]
-; AVX512F-NEXT:    # zmm7 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm8
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm7, %zmm8
-; AVX512F-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm13
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm7 = [13,6,13,6,13,6,13,6]
-; AVX512F-NEXT:    # zmm7 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm8
-; AVX512F-NEXT:    vpermt2q %zmm2, %zmm13, %zmm8
-; AVX512F-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm26, %zmm8
-; AVX512F-NEXT:    vpermt2q %zmm0, %zmm13, %zmm8
-; AVX512F-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm20 = [12,5,12,5,12,5,12,5]
-; AVX512F-NEXT:    # zmm20 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm18
-; AVX512F-NEXT:    vpermt2q %zmm2, %zmm20, %zmm18
-; AVX512F-NEXT:    vmovdqa64 %zmm26, %zmm23
-; AVX512F-NEXT:    vpermt2q %zmm0, %zmm20, %zmm23
-; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm17
-; AVX512F-NEXT:    vpermt2q %zmm2, %zmm7, %zmm17
-; AVX512F-NEXT:    vmovdqa64 %zmm26, %zmm25
-; AVX512F-NEXT:    vpermt2q %zmm0, %zmm7, %zmm25
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm28 = [0,7,14,0,0,7,14,0]
-; AVX512F-NEXT:    # zmm28 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm8
-; AVX512F-NEXT:    vpermt2q %zmm2, %zmm28, %zmm8
-; AVX512F-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm26, %zmm16
-; AVX512F-NEXT:    vpermt2q %zmm0, %zmm28, %zmm16
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm30 = [9,0,7,0,9,0,7,0]
+; AVX512F-NEXT:    vmovdqa64 320(%rdi), %zmm26
+; AVX512F-NEXT:    vmovdqa64 256(%rdi), %zmm19
+; AVX512F-NEXT:    vmovdqa64 64(%rdi), %zmm23
+; AVX512F-NEXT:    vmovdqa64 (%rdi), %zmm6
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm30 = [0,7,14,0,0,7,14,0]
 ; AVX512F-NEXT:    # zmm30 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm14
-; AVX512F-NEXT:    vpermt2q %zmm9, %zmm30, %zmm14
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm4 = [2,9,2,9,2,9,2,9]
-; AVX512F-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermt2q %zmm2, %zmm4, %zmm9
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm8
-; AVX512F-NEXT:    vpermt2q %zmm26, %zmm30, %zmm8
-; AVX512F-NEXT:    vpermt2q %zmm0, %zmm4, %zmm26
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm11, %zmm7, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm11
-; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm0
-; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm10
-; AVX512F-NEXT:    vpermt2q %zmm15, %zmm7, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 192(%rdi), %zmm24
-; AVX512F-NEXT:    vmovdqa64 128(%rdi), %zmm12
-; AVX512F-NEXT:    vmovdqa64 %zmm12, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm24, %zmm7, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm5
-; AVX512F-NEXT:    vpermi2q %zmm21, %zmm3, %zmm13
-; AVX512F-NEXT:    vmovdqa64 1536(%rdi), %zmm15
-; AVX512F-NEXT:    vmovdqa64 1472(%rdi), %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm15, %zmm7, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm29
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm20, %zmm29
-; AVX512F-NEXT:    vpermi2q %zmm21, %zmm3, %zmm20
-; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm31
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm7, %zmm31
-; AVX512F-NEXT:    vpermi2q %zmm21, %zmm3, %zmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm22
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm28, %zmm22
-; AVX512F-NEXT:    vpermi2q %zmm21, %zmm3, %zmm28
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm3
-; AVX512F-NEXT:    vpermt2q %zmm6, %zmm30, %zmm3
-; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm0
-; AVX512F-NEXT:    vpermi2q %zmm5, %zmm21, %zmm30
-; AVX512F-NEXT:    vpermt2q %zmm21, %zmm4, %zmm5
-; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm21
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm4, %zmm0
-; AVX512F-NEXT:    vmovdqa64 384(%rdi), %zmm1
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [0,5,6,9,0,5,6,9]
-; AVX512F-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm4, %zmm5
-; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 832(%rdi), %zmm5
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm5, %zmm4, %zmm6
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm3
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm13
 ; AVX512F-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 1280(%rdi), %zmm6
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm6, %zmm4, %zmm7
-; AVX512F-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 1728(%rdi), %zmm7
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm27 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm7, %zmm4, %zmm27
-; AVX512F-NEXT:    vmovdqu64 %zmm27, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [0,5,6,10,0,5,6,10]
-; AVX512F-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm6, %zmm4, %zmm19
-; AVX512F-NEXT:    vmovdqu64 %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm5, %zmm4, %zmm19
-; AVX512F-NEXT:    vmovdqu64 %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm4, %zmm19
-; AVX512F-NEXT:    vmovdqu64 %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm7, %zmm4, %zmm13
-; AVX512F-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [0,5,6,11,0,5,6,11]
-; AVX512F-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermt2q %zmm6, %zmm4, %zmm29
-; AVX512F-NEXT:    vpermt2q %zmm5, %zmm4, %zmm18
-; AVX512F-NEXT:    vmovdqu64 %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm4, %zmm23
+; AVX512F-NEXT:    vpermt2q %zmm23, %zmm30, %zmm3
+; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vmovdqu64 %zmm23, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm7, %zmm4, %zmm20
-; AVX512F-NEXT:    vmovdqu64 %zmm20, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [0,5,6,12,0,5,6,12]
-; AVX512F-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermt2q %zmm6, %zmm4, %zmm31
-; AVX512F-NEXT:    vpermt2q %zmm5, %zmm4, %zmm17
-; AVX512F-NEXT:    vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm4, %zmm25
-; AVX512F-NEXT:    vmovdqu64 %zmm25, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm7, %zmm4, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [4,5,6,13,4,5,6,13]
-; AVX512F-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermt2q %zmm6, %zmm4, %zmm22
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm5, %zmm4, %zmm19
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm4, %zmm16
-; AVX512F-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm7, %zmm4, %zmm28
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [4,5,6,14,4,5,6,14]
-; AVX512F-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermt2q %zmm6, %zmm4, %zmm3
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm3 = [10,3,10,3,10,3,10,3]
+; AVX512F-NEXT:    # zmm3 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm19, %zmm6
+; AVX512F-NEXT:    vpermt2q %zmm26, %zmm3, %zmm6
+; AVX512F-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm8
+; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm3
+; AVX512F-NEXT:    vpermt2q %zmm9, %zmm30, %zmm3
 ; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm5, %zmm4, %zmm14
-; AVX512F-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm4, %zmm8
-; AVX512F-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm7, %zmm4, %zmm30
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [4,5,8,15,4,5,8,15]
-; AVX512F-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermt2q %zmm5, %zmm4, %zmm9
+; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm14
 ; AVX512F-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm4, %zmm26
-; AVX512F-NEXT:    vmovdqu64 %zmm26, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm7, %zmm4, %zmm21
-; AVX512F-NEXT:    vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm6, %zmm4, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    movb $24, %al
-; AVX512F-NEXT:    kmovw %eax, %k1
-; AVX512F-NEXT:    vmovdqa64 512(%rdi), %zmm5
-; AVX512F-NEXT:    vmovdqa64 448(%rdi), %zmm7
-; AVX512F-NEXT:    vmovdqa64 {{.*#+}} ymm21 = <0,7,14,u>
-; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm18
-; AVX512F-NEXT:    vpermt2q %zmm5, %zmm21, %zmm18
-; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm0
-; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm18 {%k1} = zmm11[4,5,4,5],zmm10[4,5,4,5]
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm10 = [14,0,0,7,14,0,0,7]
-; AVX512F-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm26
-; AVX512F-NEXT:    vpermt2q %zmm0, %zmm10, %zmm26
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm16 = [7,0,9,0,7,0,9,0]
-; AVX512F-NEXT:    # zmm16 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm27
-; AVX512F-NEXT:    vpermt2q %zmm11, %zmm16, %zmm27
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} ymm20 = [4,11,4,11]
-; AVX512F-NEXT:    # ymm20 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermt2q %zmm0, %zmm20, %zmm11
-; AVX512F-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 64(%rdi), %zmm1
-; AVX512F-NEXT:    vmovdqa64 (%rdi), %zmm14
-; AVX512F-NEXT:    vmovdqa64 %zmm14, %zmm17
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm21, %zmm17
-; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm17 {%k1} = zmm12[4,5,4,5],zmm24[4,5,4,5]
-; AVX512F-NEXT:    vmovdqa64 %zmm12, %zmm23
-; AVX512F-NEXT:    vpermt2q %zmm24, %zmm10, %zmm23
-; AVX512F-NEXT:    vmovdqa64 %zmm24, %zmm25
-; AVX512F-NEXT:    vpermt2q %zmm12, %zmm16, %zmm25
-; AVX512F-NEXT:    vpermt2q %zmm24, %zmm20, %zmm12
-; AVX512F-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 960(%rdi), %zmm6
-; AVX512F-NEXT:    vmovdqa64 896(%rdi), %zmm11
-; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm12
-; AVX512F-NEXT:    vpermt2q %zmm6, %zmm21, %zmm12
-; AVX512F-NEXT:    vmovdqa64 1408(%rdi), %zmm4
-; AVX512F-NEXT:    vmovdqa64 1344(%rdi), %zmm8
-; AVX512F-NEXT:    vpermi2q %zmm4, %zmm8, %zmm21
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm21 {%k1} = zmm3[4,5,4,5],zmm15[4,5,4,5]
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm0
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm9, %zmm10, %zmm0
-; AVX512F-NEXT:    vpermi2q %zmm15, %zmm3, %zmm10
-; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm24
-; AVX512F-NEXT:    vpermt2q %zmm2, %zmm16, %zmm24
-; AVX512F-NEXT:    vpermi2q %zmm3, %zmm15, %zmm16
-; AVX512F-NEXT:    vpermt2q %zmm15, %zmm20, %zmm3
+; AVX512F-NEXT:    vmovdqa64 %zmm20, %zmm3
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm8, %zmm3
 ; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm12 {%k1} = zmm2[4,5,4,5],zmm9[4,5,4,5]
-; AVX512F-NEXT:    vpermt2q %zmm9, %zmm20, %zmm2
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm11 = [13,6,13,6,13,6,13,6]
+; AVX512F-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm9 = [9,0,7,0,9,0,7,0]
+; AVX512F-NEXT:    # zmm9 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm15 = [11,4,11,4,11,4,11,4]
+; AVX512F-NEXT:    # zmm15 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm20, %zmm3
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm15, %zmm3
+; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm19, %zmm3
+; AVX512F-NEXT:    vpermt2q %zmm26, %zmm15, %zmm3
+; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm22 = [12,5,12,5,12,5,12,5]
+; AVX512F-NEXT:    # zmm22 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm20, %zmm3
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm22, %zmm3
+; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm19, %zmm3
+; AVX512F-NEXT:    vpermt2q %zmm26, %zmm22, %zmm3
+; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm20, %zmm3
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm11, %zmm3
+; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm19, %zmm3
+; AVX512F-NEXT:    vpermt2q %zmm26, %zmm11, %zmm3
+; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm20, %zmm3
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm30, %zmm3
+; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm19, %zmm3
+; AVX512F-NEXT:    vpermt2q %zmm26, %zmm30, %zmm3
+; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
+; AVX512F-NEXT:    vpermt2q %zmm20, %zmm9, %zmm3
+; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [2,9,2,9,2,9,2,9]
+; AVX512F-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm0, %zmm20
+; AVX512F-NEXT:    vmovdqa64 %zmm26, %zmm21
+; AVX512F-NEXT:    vpermt2q %zmm19, %zmm9, %zmm21
+; AVX512F-NEXT:    vpermt2q %zmm26, %zmm0, %zmm19
+; AVX512F-NEXT:    vmovdqa64 %zmm12, %zmm3
+; AVX512F-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm12, %zmm2
+; AVX512F-NEXT:    vmovdqu64 %zmm31, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm31, %zmm30, %zmm2
 ; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa 912(%rdi), %xmm2
-; AVX512F-NEXT:    vpblendd {{.*#+}} xmm2 = xmm2[0,1],mem[2,3]
-; AVX512F-NEXT:    vinserti32x4 $0, %xmm2, %zmm0, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa 464(%rdi), %xmm0
-; AVX512F-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0,1],mem[2,3]
-; AVX512F-NEXT:    vinserti32x4 $0, %xmm0, %zmm26, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa 16(%rdi), %xmm0
-; AVX512F-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0,1],mem[2,3]
-; AVX512F-NEXT:    vinserti32x4 $0, %xmm0, %zmm23, %zmm26
-; AVX512F-NEXT:    vmovdqa 1360(%rdi), %xmm0
-; AVX512F-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0,1],mem[2,3]
-; AVX512F-NEXT:    vinserti32x4 $0, %xmm0, %zmm10, %zmm20
-; AVX512F-NEXT:    vinserti128 $1, 1088(%rdi), %ymm0, %ymm0
-; AVX512F-NEXT:    vmovdqa 1024(%rdi), %ymm2
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5],ymm0[6,7]
-; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm10 = [4,11]
-; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm13
-; AVX512F-NEXT:    vpermt2q %zmm6, %zmm10, %zmm13
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm13[0,1,2,3],ymm0[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm22, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vinserti128 $1, 640(%rdi), %ymm0, %ymm0
-; AVX512F-NEXT:    vmovdqa 576(%rdi), %ymm3
-; AVX512F-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3,4,5],ymm0[6,7]
-; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm3
-; AVX512F-NEXT:    vpermt2q %zmm5, %zmm10, %zmm3
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3],ymm0[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm19, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vinserti128 $1, 192(%rdi), %ymm0, %ymm0
-; AVX512F-NEXT:    vmovdqa 128(%rdi), %ymm3
-; AVX512F-NEXT:    vmovdqu %ymm3, (%rsp) # 32-byte Spill
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3,4,5],ymm0[6,7]
-; AVX512F-NEXT:    vmovdqa64 %zmm14, %zmm3
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm10, %zmm3
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3],ymm0[4,5,6,7]
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm3, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vinserti128 $1, 1536(%rdi), %ymm0, %ymm0
-; AVX512F-NEXT:    vmovdqa 1472(%rdi), %ymm3
-; AVX512F-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3,4,5],ymm0[6,7]
-; AVX512F-NEXT:    vpermi2q %zmm4, %zmm8, %zmm10
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm10[0,1,2,3],ymm0[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm28, %zmm28
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm12
+; AVX512F-NEXT:    vpermt2q %zmm25, %zmm8, %zmm12
+; AVX512F-NEXT:    vmovdqa64 1408(%rdi), %zmm28
+; AVX512F-NEXT:    vmovdqa64 1344(%rdi), %zmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm5
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm7
+; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm28, %zmm30, %zmm5
+; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 1664(%rdi), %zmm2
+; AVX512F-NEXT:    vmovdqa64 1600(%rdi), %zmm10
+; AVX512F-NEXT:    vpermi2q %zmm2, %zmm10, %zmm8
+; AVX512F-NEXT:    vmovdqa64 1088(%rdi), %zmm18
+; AVX512F-NEXT:    vmovdqa64 1024(%rdi), %zmm6
+; AVX512F-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm5
+; AVX512F-NEXT:    vpermt2q %zmm18, %zmm11, %zmm6
+; AVX512F-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm9, %zmm31
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm4
+; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm6
+; AVX512F-NEXT:    vpermt2q %zmm25, %zmm15, %zmm4
+; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm17
+; AVX512F-NEXT:    vmovdqa64 640(%rdi), %zmm15
+; AVX512F-NEXT:    vmovdqa64 576(%rdi), %zmm4
+; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm15, %zmm11, %zmm4
+; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm14, %zmm26
+; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm9, %zmm26 # 64-byte Folded Reload
+; AVX512F-NEXT:    vmovdqa64 192(%rdi), %zmm16
+; AVX512F-NEXT:    vmovdqa64 128(%rdi), %zmm4
+; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm16, %zmm11, %zmm4
+; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm13, %zmm9, %zmm23
+; AVX512F-NEXT:    vmovdqa64 1536(%rdi), %zmm11
+; AVX512F-NEXT:    vmovdqa64 1472(%rdi), %zmm4
+; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm11, %zmm5, %zmm4
+; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm28, %zmm27
+; AVX512F-NEXT:    vpermt2q %zmm7, %zmm9, %zmm27
+; AVX512F-NEXT:    vpermi2q %zmm2, %zmm10, %zmm6
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm29
+; AVX512F-NEXT:    vmovdqa64 %zmm25, %zmm3
+; AVX512F-NEXT:    vpermt2q %zmm25, %zmm22, %zmm29
+; AVX512F-NEXT:    vpermi2q %zmm2, %zmm10, %zmm22
+; AVX512F-NEXT:    vmovdqa64 %zmm22, %zmm14
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm22
+; AVX512F-NEXT:    vpermt2q %zmm25, %zmm5, %zmm22
+; AVX512F-NEXT:    vpermi2q %zmm2, %zmm10, %zmm5
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm13
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm25
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm30, %zmm25
+; AVX512F-NEXT:    vpermi2q %zmm2, %zmm10, %zmm30
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm24
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm9, %zmm24
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm4
+; AVX512F-NEXT:    vpermi2q %zmm10, %zmm2, %zmm9
+; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm7
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm0, %zmm10
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm0, %zmm4
+; AVX512F-NEXT:    vmovdqa64 384(%rdi), %zmm0
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [0,5,6,9,0,5,6,9]
+; AVX512F-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm2, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 832(%rdi), %zmm3
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm2, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 1280(%rdi), %zmm5
+; AVX512F-NEXT:    vpermt2q %zmm5, %zmm2, %zmm12
+; AVX512F-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 1728(%rdi), %zmm9
+; AVX512F-NEXT:    vpermt2q %zmm9, %zmm2, %zmm8
+; AVX512F-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [0,5,6,10,0,5,6,10]
+; AVX512F-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpermt2q %zmm5, %zmm2, %zmm17
+; AVX512F-NEXT:    vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm2, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm2, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm9, %zmm2, %zmm6
+; AVX512F-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [0,5,6,11,0,5,6,11]
+; AVX512F-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpermt2q %zmm5, %zmm2, %zmm29
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm2, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm2, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm9, %zmm2, %zmm14
+; AVX512F-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [0,5,6,12,0,5,6,12]
+; AVX512F-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpermt2q %zmm5, %zmm2, %zmm22
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm2, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm2, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm9, %zmm2, %zmm13
+; AVX512F-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [4,5,6,13,4,5,6,13]
+; AVX512F-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpermt2q %zmm5, %zmm2, %zmm25
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm2, %zmm8
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm2, %zmm14
+; AVX512F-NEXT:    vpermt2q %zmm9, %zmm2, %zmm30
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [4,5,6,14,4,5,6,14]
+; AVX512F-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpermt2q %zmm5, %zmm2, %zmm24
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm2, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm2, %zmm21
+; AVX512F-NEXT:    vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm9, %zmm2, %zmm7
+; AVX512F-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [4,5,8,15,4,5,8,15]
+; AVX512F-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm2, %zmm20
+; AVX512F-NEXT:    vmovdqu64 %zmm20, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm2, %zmm19
+; AVX512F-NEXT:    vmovdqu64 %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm9, %zmm2, %zmm10
+; AVX512F-NEXT:    vpermt2q %zmm5, %zmm2, %zmm4
+; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    movb $24, %al
+; AVX512F-NEXT:    kmovw %eax, %k1
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm0 {%k1} = zmm1[4,5,4,5],zmm15[4,5,4,5]
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [14,0,0,7,14,0,0,7]
+; AVX512F-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm2
+; AVX512F-NEXT:    vpermt2q %zmm15, %zmm0, %zmm2
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [7,0,9,0,7,0,9,0]
+; AVX512F-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm20
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm4, %zmm20
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} ymm21 = [4,11,4,11]
+; AVX512F-NEXT:    # ymm21 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpermt2q %zmm15, %zmm21, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm1 {%k1} = zmm3[4,5,4,5],zmm16[4,5,4,5]
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm15
+; AVX512F-NEXT:    vpermt2q %zmm16, %zmm0, %zmm15
+; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm19
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm4, %zmm19
+; AVX512F-NEXT:    vpermt2q %zmm16, %zmm21, %zmm3
+; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm1 {%k1} = zmm3[4,5,4,5],zmm11[4,5,4,5]
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm16
+; AVX512F-NEXT:    vpermt2q %zmm18, %zmm0, %zmm16
+; AVX512F-NEXT:    vpermi2q %zmm11, %zmm3, %zmm0
+; AVX512F-NEXT:    vmovdqa64 %zmm18, %zmm17
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm4, %zmm17
+; AVX512F-NEXT:    vpermi2q %zmm3, %zmm11, %zmm4
+; AVX512F-NEXT:    vpermt2q %zmm11, %zmm21, %zmm3
+; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
+; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm9 {%k1} = zmm1[4,5,4,5],zmm18[4,5,4,5]
+; AVX512F-NEXT:    vpermt2q %zmm18, %zmm21, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa 912(%rdi), %xmm11
+; AVX512F-NEXT:    vpblendd {{.*#+}} xmm11 = xmm11[0,1],mem[2,3]
+; AVX512F-NEXT:    vinserti32x4 $0, %xmm11, %zmm16, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa 464(%rdi), %xmm11
+; AVX512F-NEXT:    vpblendd {{.*#+}} xmm11 = xmm11[0,1],mem[2,3]
+; AVX512F-NEXT:    vinserti32x4 $0, %xmm11, %zmm2, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa 16(%rdi), %xmm2
+; AVX512F-NEXT:    vpblendd {{.*#+}} xmm2 = xmm2[0,1],mem[2,3]
+; AVX512F-NEXT:    vinserti32x4 $0, %xmm2, %zmm15, %zmm21
+; AVX512F-NEXT:    vmovdqa 1360(%rdi), %xmm2
+; AVX512F-NEXT:    vpblendd {{.*#+}} xmm2 = xmm2[0,1],mem[2,3]
+; AVX512F-NEXT:    vinserti32x4 $0, %xmm2, %zmm0, %zmm16
+; AVX512F-NEXT:    vinserti128 $1, 1088(%rdi), %ymm0, %ymm0
+; AVX512F-NEXT:    vmovdqa 1024(%rdi), %ymm7
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm11 = ymm7[0,1,2,3,4,5],ymm0[6,7]
+; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm0 = [4,11]
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm12
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm13, %zmm0, %zmm12
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm11 = ymm12[0,1,2,3],ymm11[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm11, %zmm25, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vinserti128 $1, 640(%rdi), %ymm0, %ymm11
+; AVX512F-NEXT:    vmovdqa 576(%rdi), %ymm1
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm11 = ymm1[0,1,2,3,4,5],ymm11[6,7]
+; AVX512F-NEXT:    vmovdqa64 %ymm1, %ymm25
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm15
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm12, %zmm0, %zmm15
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm11 = ymm15[0,1,2,3],ymm11[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm11, %zmm8, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vinserti128 $1, 192(%rdi), %ymm0, %ymm11
+; AVX512F-NEXT:    vmovdqa 128(%rdi), %ymm1
+; AVX512F-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm11 = ymm1[0,1,2,3,4,5],ymm11[6,7]
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm1
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm15, %zmm0, %zmm1
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm11[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm14, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vinserti128 $1, 1536(%rdi), %ymm0, %ymm1
+; AVX512F-NEXT:    vmovdqa 1472(%rdi), %ymm11
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm11[0,1,2,3,4,5],ymm1[6,7]
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-NEXT:    vpermi2q %zmm28, %zmm8, %zmm0
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm30, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vmovdqa 1088(%rdi), %ymm0
-; AVX512F-NEXT:    vpalignr {{.*#+}} ymm0 = ymm2[8,9,10,11,12,13,14,15],ymm0[0,1,2,3,4,5,6,7],ymm2[24,25,26,27,28,29,30,31],ymm0[16,17,18,19,20,21,22,23]
-; AVX512F-NEXT:    vmovdqa64 %ymm0, %ymm23
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm0 = <9,0,7,u>
-; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm3
-; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm9
-; AVX512F-NEXT:    vpermt2q %zmm7, %zmm0, %zmm3
-; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm2 = [5,12]
-; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm13
-; AVX512F-NEXT:    vpermt2q %zmm5, %zmm2, %zmm13
-; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm7 = [6,13]
-; AVX512F-NEXT:    vpermt2q %zmm5, %zmm7, %zmm9
-; AVX512F-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm5
-; AVX512F-NEXT:    vpermt2q %zmm14, %zmm0, %zmm5
-; AVX512F-NEXT:    vmovdqa64 %zmm14, %zmm10
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm2, %zmm10
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm7, %zmm14
-; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm1
-; AVX512F-NEXT:    vpermt2q %zmm11, %zmm0, %zmm1
-; AVX512F-NEXT:    vpermi2q %zmm8, %zmm4, %zmm0
+; AVX512F-NEXT:    vpalignr {{.*#+}} ymm14 = ymm7[8,9,10,11,12,13,14,15],ymm0[0,1,2,3,4,5,6,7],ymm7[24,25,26,27,28,29,30,31],ymm0[16,17,18,19,20,21,22,23]
+; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm0 = [5,12]
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm7
+; AVX512F-NEXT:    vpermt2q %zmm12, %zmm0, %zmm7
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} xmm30 = [6,13]
+; AVX512F-NEXT:    vpermt2q %zmm12, %zmm30, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm6
+; AVX512F-NEXT:    vpermt2q %zmm15, %zmm0, %zmm6
+; AVX512F-NEXT:    vpermt2q %zmm15, %zmm30, %zmm3
+; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vmovdqa 960(%rdi), %ymm15
 ; AVX512F-NEXT:    vpalignr {{.*#+}} ymm15 = mem[8,9,10,11,12,13,14,15],ymm15[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm15[16,17,18,19,20,21,22,23]
-; AVX512F-NEXT:    vextracti32x4 $1, %ymm15, %xmm19
-; AVX512F-NEXT:    vmovdqa 512(%rdi), %ymm15
-; AVX512F-NEXT:    vpalignr {{.*#+}} ymm15 = mem[8,9,10,11,12,13,14,15],ymm15[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm15[16,17,18,19,20,21,22,23]
 ; AVX512F-NEXT:    vextracti128 $1, %ymm15, %xmm15
-; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm9
-; AVX512F-NEXT:    vpermt2q %zmm6, %zmm2, %zmm9
-; AVX512F-NEXT:    vpermi2q %zmm4, %zmm8, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm4, %zmm7, %zmm8
-; AVX512F-NEXT:    vpermt2q %zmm6, %zmm7, %zmm11
+; AVX512F-NEXT:    vmovdqa 512(%rdi), %ymm12
+; AVX512F-NEXT:    vpalignr {{.*#+}} ymm12 = mem[8,9,10,11,12,13,14,15],ymm12[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm12[16,17,18,19,20,21,22,23]
+; AVX512F-NEXT:    vextracti128 $1, %ymm12, %xmm12
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm2
+; AVX512F-NEXT:    vpermt2q %zmm13, %zmm0, %zmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm3
+; AVX512F-NEXT:    vpermi2q %zmm28, %zmm8, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm28, %zmm30, %zmm3
+; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm13, %zmm30, %zmm5
+; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    movb $-32, %al
 ; AVX512F-NEXT:    kmovw %eax, %k2
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm17 {%k2}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm18 {%k2}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm12 {%k2}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm21 {%k2}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm1 {%k1}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm1 {%k2}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm3 {%k1}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm3 {%k2}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm5 {%k1}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm5 {%k2}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm0 {%k1}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm0 {%k2}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm22 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm29, %zmm22 {%k2}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm28 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm28 {%k2}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm30 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm30 {%k2}
+; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm13
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm13 {%k2}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm18 {%k2}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm31 {%k1}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm31 {%k2}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm26 {%k1}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm26 {%k2}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm23 {%k1}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm23 {%k2}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm27 {%k1}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm27 {%k2}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm29, %zmm5 {%k2}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm29 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm29 {%k2}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm26 {%k2}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm20 {%k2}
-; AVX512F-NEXT:    vinserti32x4 $0, %xmm19, %zmm24, %zmm4
-; AVX512F-NEXT:    vmovdqa64 %zmm31, %zmm4 {%k2}
-; AVX512F-NEXT:    vinserti32x4 $0, %xmm15, %zmm27, %zmm6
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm6 {%k2}
-; AVX512F-NEXT:    vmovdqa64 %ymm23, %ymm7
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm7 = ymm9[0,1,2,3],ymm7[4,5,6,7]
-; AVX512F-NEXT:    vmovdqa 64(%rdi), %ymm9
-; AVX512F-NEXT:    vpalignr {{.*#+}} ymm9 = mem[8,9,10,11,12,13,14,15],ymm9[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm9[16,17,18,19,20,21,22,23]
-; AVX512F-NEXT:    vextracti128 $1, %ymm9, %xmm9
-; AVX512F-NEXT:    vinserti32x4 $0, %xmm9, %zmm25, %zmm9
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm9 {%k2}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm7, %zmm15, %zmm19
-; AVX512F-NEXT:    vmovdqa 640(%rdi), %ymm15
-; AVX512F-NEXT:    vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm15, %ymm15 # 32-byte Folded Reload
-; AVX512F-NEXT:    # ymm15 = mem[8,9,10,11,12,13,14,15],ymm15[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm15[16,17,18,19,20,21,22,23]
-; AVX512F-NEXT:    vmovdqa 1408(%rdi), %ymm7
-; AVX512F-NEXT:    vpalignr {{.*#+}} ymm7 = mem[8,9,10,11,12,13,14,15],ymm7[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm7[16,17,18,19,20,21,22,23]
-; AVX512F-NEXT:    vextracti128 $1, %ymm7, %xmm7
-; AVX512F-NEXT:    vinserti32x4 $0, %xmm7, %zmm16, %zmm7
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm7 {%k2}
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm13 = ymm13[0,1,2,3],ymm15[4,5,6,7]
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm13, %zmm15, %zmm13
-; AVX512F-NEXT:    vmovdqa 192(%rdi), %ymm15
-; AVX512F-NEXT:    vpalignr $8, (%rsp), %ymm15, %ymm15 # 32-byte Folded Reload
-; AVX512F-NEXT:    # ymm15 = mem[8,9,10,11,12,13,14,15],ymm15[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm15[16,17,18,19,20,21,22,23]
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm10 = ymm10[0,1,2,3],ymm15[4,5,6,7]
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm10, %zmm15, %zmm10
-; AVX512F-NEXT:    vmovdqa 1536(%rdi), %ymm15
-; AVX512F-NEXT:    vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm15, %ymm15 # 32-byte Folded Reload
-; AVX512F-NEXT:    # ymm15 = mem[8,9,10,11,12,13,14,15],ymm15[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm15[16,17,18,19,20,21,22,23]
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm15[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm30, %zmm2
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
-; AVX512F-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm15, %ymm15 # 32-byte Folded Reload
-; AVX512F-NEXT:    # ymm15 = ymm15[0,1,2,3],mem[4,5,6,7]
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
-; AVX512F-NEXT:    vinsertf64x4 $0, %ymm15, %zmm16, %zmm15
-; AVX512F-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm14 # 32-byte Folded Reload
-; AVX512F-NEXT:    # ymm14 = ymm14[0,1,2,3],mem[4,5,6,7]
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm14, %zmm16, %zmm14
-; AVX512F-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm8 # 32-byte Folded Reload
-; AVX512F-NEXT:    # ymm8 = ymm8[0,1,2,3],mem[4,5,6,7]
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm8, %zmm16, %zmm8
-; AVX512F-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm11, %ymm11 # 32-byte Folded Reload
-; AVX512F-NEXT:    # ymm11 = ymm11[0,1,2,3],mem[4,5,6,7]
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm11, %zmm16, %zmm11
-; AVX512F-NEXT:    vmovdqa64 %zmm21, 192(%rsi)
-; AVX512F-NEXT:    vmovdqa64 %zmm12, 128(%rsi)
-; AVX512F-NEXT:    vmovdqa64 %zmm18, 64(%rsi)
-; AVX512F-NEXT:    vmovdqa64 %zmm17, (%rsi)
-; AVX512F-NEXT:    vmovdqa64 %zmm0, 192(%rdx)
-; AVX512F-NEXT:    vmovdqa64 %zmm5, (%rdx)
-; AVX512F-NEXT:    vmovdqa64 %zmm3, 64(%rdx)
-; AVX512F-NEXT:    vmovdqa64 %zmm1, 128(%rdx)
-; AVX512F-NEXT:    vmovdqa64 %zmm20, 192(%rcx)
-; AVX512F-NEXT:    vmovdqa64 %zmm26, (%rcx)
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm29 {%k2}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm21 {%k2}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm16 {%k2}
+; AVX512F-NEXT:    vinserti32x4 $0, %xmm15, %zmm17, %zmm17
+; AVX512F-NEXT:    vmovdqa64 %zmm22, %zmm17 {%k2}
+; AVX512F-NEXT:    vinserti32x4 $0, %xmm12, %zmm20, %zmm12
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm12 {%k2}
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm14[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa 64(%rdi), %ymm14
+; AVX512F-NEXT:    vpalignr {{.*#+}} ymm14 = mem[8,9,10,11,12,13,14,15],ymm14[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm14[16,17,18,19,20,21,22,23]
+; AVX512F-NEXT:    vextracti128 $1, %ymm14, %xmm14
+; AVX512F-NEXT:    vinserti32x4 $0, %xmm14, %zmm19, %zmm9
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm9 {%k2}
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm24, %zmm1
+; AVX512F-NEXT:    vmovdqa 640(%rdi), %ymm14
+; AVX512F-NEXT:    vmovdqa64 %ymm25, %ymm2
+; AVX512F-NEXT:    vpalignr {{.*#+}} ymm14 = ymm2[8,9,10,11,12,13,14,15],ymm14[0,1,2,3,4,5,6,7],ymm2[24,25,26,27,28,29,30,31],ymm14[16,17,18,19,20,21,22,23]
+; AVX512F-NEXT:    vmovdqa 1408(%rdi), %ymm15
+; AVX512F-NEXT:    vpalignr {{.*#+}} ymm15 = mem[8,9,10,11,12,13,14,15],ymm15[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm15[16,17,18,19,20,21,22,23]
+; AVX512F-NEXT:    vextracti128 $1, %ymm15, %xmm15
+; AVX512F-NEXT:    vinserti32x4 $0, %xmm15, %zmm4, %zmm4
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm4 {%k2}
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm3 = ymm7[0,1,2,3],ymm14[4,5,6,7]
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm3, %zmm2, %zmm3
+; AVX512F-NEXT:    vmovdqa 192(%rdi), %ymm14
+; AVX512F-NEXT:    vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm14 # 32-byte Folded Reload
+; AVX512F-NEXT:    # ymm14 = mem[8,9,10,11,12,13,14,15],ymm14[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm14[16,17,18,19,20,21,22,23]
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm6[0,1,2,3],ymm14[4,5,6,7]
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm6, %zmm2
+; AVX512F-NEXT:    vmovdqa 1536(%rdi), %ymm14
+; AVX512F-NEXT:    vpalignr {{.*#+}} ymm11 = ymm11[8,9,10,11,12,13,14,15],ymm14[0,1,2,3,4,5,6,7],ymm11[24,25,26,27,28,29,30,31],ymm14[16,17,18,19,20,21,22,23]
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm11[4,5,6,7]
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm6, %zmm0
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512F-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm6, %ymm11 # 32-byte Folded Reload
+; AVX512F-NEXT:    # ymm11 = ymm6[0,1,2,3],mem[4,5,6,7]
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512F-NEXT:    vinsertf64x4 $0, %ymm11, %zmm6, %zmm11
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512F-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm6, %ymm8 # 32-byte Folded Reload
+; AVX512F-NEXT:    # ymm8 = ymm6[0,1,2,3],mem[4,5,6,7]
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512F-NEXT:    vinsertf64x4 $0, %ymm8, %zmm6, %zmm8
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512F-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm6, %ymm7 # 32-byte Folded Reload
+; AVX512F-NEXT:    # ymm7 = ymm6[0,1,2,3],mem[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm7, %zmm10, %zmm6
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512F-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm7 # 32-byte Folded Reload
+; AVX512F-NEXT:    # ymm7 = ymm7[0,1,2,3],mem[4,5,6,7]
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
+; AVX512F-NEXT:    vinsertf64x4 $0, %ymm7, %zmm10, %zmm7
+; AVX512F-NEXT:    vmovdqa64 %zmm18, 192(%rsi)
+; AVX512F-NEXT:    vmovdqa64 %zmm13, 128(%rsi)
+; AVX512F-NEXT:    vmovdqa64 %zmm30, 64(%rsi)
+; AVX512F-NEXT:    vmovdqa64 %zmm28, (%rsi)
+; AVX512F-NEXT:    vmovdqa64 %zmm27, 192(%rdx)
+; AVX512F-NEXT:    vmovdqa64 %zmm23, (%rdx)
+; AVX512F-NEXT:    vmovdqa64 %zmm26, 64(%rdx)
+; AVX512F-NEXT:    vmovdqa64 %zmm31, 128(%rdx)
+; AVX512F-NEXT:    vmovdqa64 %zmm16, 192(%rcx)
+; AVX512F-NEXT:    vmovdqa64 %zmm21, (%rcx)
 ; AVX512F-NEXT:    vmovdqa64 %zmm29, 64(%rcx)
-; AVX512F-NEXT:    vmovdqa64 %zmm22, 128(%rcx)
-; AVX512F-NEXT:    vmovdqa64 %zmm7, 192(%r8)
+; AVX512F-NEXT:    vmovdqa64 %zmm5, 128(%rcx)
+; AVX512F-NEXT:    vmovdqa64 %zmm4, 192(%r8)
 ; AVX512F-NEXT:    vmovdqa64 %zmm9, (%r8)
-; AVX512F-NEXT:    vmovdqa64 %zmm6, 64(%r8)
-; AVX512F-NEXT:    vmovdqa64 %zmm4, 128(%r8)
-; AVX512F-NEXT:    vmovdqa64 %zmm28, 192(%r9)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, (%r9)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 64(%r9)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 128(%r9)
+; AVX512F-NEXT:    vmovdqa64 %zmm12, 64(%r8)
+; AVX512F-NEXT:    vmovdqa64 %zmm17, 128(%r8)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm4, 192(%r9)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm4, (%r9)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm4, 64(%r9)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm4, 128(%r9)
 ; AVX512F-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512F-NEXT:    vmovdqa64 %zmm2, 192(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm10, (%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm13, 64(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm19, 128(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm0, 192(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm2, (%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm3, 64(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm1, 128(%rax)
 ; AVX512F-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512F-NEXT:    vmovdqa64 %zmm11, 128(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm8, 192(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm14, (%rax)
-; AVX512F-NEXT:    vmovaps %zmm15, 64(%rax)
-; AVX512F-NEXT:    addq $2216, %rsp # imm = 0x8A8
+; AVX512F-NEXT:    vmovaps %zmm7, 128(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm6, 192(%rax)
+; AVX512F-NEXT:    vmovaps %zmm8, (%rax)
+; AVX512F-NEXT:    vmovaps %zmm11, 64(%rax)
+; AVX512F-NEXT:    addq $2728, %rsp # imm = 0xAA8
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: load_i64_stride7_vf32:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    subq $2152, %rsp # imm = 0x868
-; AVX512BW-NEXT:    vmovdqa64 1664(%rdi), %zmm21
-; AVX512BW-NEXT:    vmovdqa64 1600(%rdi), %zmm31
-; AVX512BW-NEXT:    vmovdqa64 1216(%rdi), %zmm2
-; AVX512BW-NEXT:    vmovdqa64 1152(%rdi), %zmm6
-; AVX512BW-NEXT:    vmovdqa64 1088(%rdi), %zmm12
+; AVX512BW-NEXT:    subq $2760, %rsp # imm = 0xAC8
+; AVX512BW-NEXT:    vmovdqa64 1216(%rdi), %zmm20
+; AVX512BW-NEXT:    vmovdqa64 1152(%rdi), %zmm2
+; AVX512BW-NEXT:    vmovdqa64 960(%rdi), %zmm29
+; AVX512BW-NEXT:    vmovdqa64 896(%rdi), %zmm4
+; AVX512BW-NEXT:    vmovdqa64 768(%rdi), %zmm1
+; AVX512BW-NEXT:    vmovdqa64 704(%rdi), %zmm30
+; AVX512BW-NEXT:    vmovdqa64 512(%rdi), %zmm28
+; AVX512BW-NEXT:    vmovdqa64 448(%rdi), %zmm12
+; AVX512BW-NEXT:    vmovdqa64 320(%rdi), %zmm26
+; AVX512BW-NEXT:    vmovdqa64 256(%rdi), %zmm17
+; AVX512BW-NEXT:    vmovdqa64 64(%rdi), %zmm6
+; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm3
+; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm25 = [0,7,14,0,0,7,14,0]
+; AVX512BW-NEXT:    # zmm25 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm25, %zmm3
+; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm8 = [10,3,10,3,10,3,10,3]
+; AVX512BW-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm17, %zmm6
+; AVX512BW-NEXT:    vpermt2q %zmm26, %zmm8, %zmm6
+; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm3
 ; AVX512BW-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 1024(%rdi), %zmm3
+; AVX512BW-NEXT:    vpermt2q %zmm28, %zmm25, %zmm3
 ; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 768(%rdi), %zmm1
-; AVX512BW-NEXT:    vmovdqa64 704(%rdi), %zmm10
-; AVX512BW-NEXT:    vmovdqa64 640(%rdi), %zmm11
-; AVX512BW-NEXT:    vmovaps 576(%rdi), %zmm0
-; AVX512BW-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 320(%rdi), %zmm0
-; AVX512BW-NEXT:    vmovdqa64 256(%rdi), %zmm5
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm7 = [10,3,10,3,10,3,10,3]
-; AVX512BW-NEXT:    # zmm7 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm8
-; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm7, %zmm8
-; AVX512BW-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm8
-; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm7, %zmm8
-; AVX512BW-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm8
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm7, %zmm8
-; AVX512BW-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermi2q %zmm21, %zmm31, %zmm7
-; AVX512BW-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 %zmm28, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm30, %zmm3
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm8, %zmm3
+; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm11 = [13,6,13,6,13,6,13,6]
+; AVX512BW-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm18 = [9,0,7,0,9,0,7,0]
+; AVX512BW-NEXT:    # zmm18 = mem[0,1,2,3,0,1,2,3]
 ; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm7 = [11,4,11,4,11,4,11,4]
 ; AVX512BW-NEXT:    # zmm7 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm8
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm7, %zmm8
-; AVX512BW-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm13
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm7 = [13,6,13,6,13,6,13,6]
-; AVX512BW-NEXT:    # zmm7 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm8
-; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm13, %zmm8
-; AVX512BW-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm8
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm18
-; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm13, %zmm18
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm19 = [12,5,12,5,12,5,12,5]
-; AVX512BW-NEXT:    # zmm19 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm17
-; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm19, %zmm17
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm22
-; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm19, %zmm22
-; AVX512BW-NEXT:    vmovdqa64 %zmm22, %zmm23
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm16
-; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm7, %zmm16
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm22
-; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm7, %zmm22
-; AVX512BW-NEXT:    vmovdqa64 %zmm22, %zmm24
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm28 = [0,7,14,0,0,7,14,0]
-; AVX512BW-NEXT:    # zmm28 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm15
-; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm28, %zmm15
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm25
-; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm28, %zmm25
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm29 = [9,0,7,0,9,0,7,0]
-; AVX512BW-NEXT:    # zmm29 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm26
-; AVX512BW-NEXT:    vpermt2q %zmm10, %zmm29, %zmm26
-; AVX512BW-NEXT:    vmovdqa64 %zmm26, %zmm30
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm4 = [2,9,2,9,2,9,2,9]
-; AVX512BW-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm4, %zmm10
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm9
-; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm29, %zmm9
-; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm4, %zmm8
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm1
-; AVX512BW-NEXT:    vpermt2q %zmm12, %zmm7, %zmm3
+; AVX512BW-NEXT:    vmovdqa64 %zmm30, %zmm3
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm7, %zmm3
 ; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm7, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 192(%rdi), %zmm22
-; AVX512BW-NEXT:    vmovdqa64 128(%rdi), %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm7, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm31, %zmm5
-; AVX512BW-NEXT:    vpermi2q %zmm21, %zmm31, %zmm13
-; AVX512BW-NEXT:    vmovdqa64 1536(%rdi), %zmm3
-; AVX512BW-NEXT:    vmovdqa64 1472(%rdi), %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm7, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm0
-; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm31
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm19, %zmm31
-; AVX512BW-NEXT:    vpermi2q %zmm21, %zmm5, %zmm19
-; AVX512BW-NEXT:    vmovdqa64 %zmm19, %zmm26
-; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm27
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm7, %zmm27
-; AVX512BW-NEXT:    vpermi2q %zmm21, %zmm5, %zmm1
-; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm20
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm28, %zmm20
-; AVX512BW-NEXT:    vpermi2q %zmm21, %zmm5, %zmm28
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm12
-; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm29, %zmm12
-; AVX512BW-NEXT:    vpermi2q %zmm5, %zmm21, %zmm29
-; AVX512BW-NEXT:    vpermt2q %zmm21, %zmm4, %zmm5
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm21
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm4, %zmm0
-; AVX512BW-NEXT:    vmovdqa64 384(%rdi), %zmm2
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [0,5,6,9,0,5,6,9]
-; AVX512BW-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm4, %zmm5
+; AVX512BW-NEXT:    vmovdqa64 %zmm17, %zmm3
+; AVX512BW-NEXT:    vpermt2q %zmm26, %zmm7, %zmm3
+; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm23 = [12,5,12,5,12,5,12,5]
+; AVX512BW-NEXT:    # zmm23 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm30, %zmm3
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm23, %zmm3
+; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm17, %zmm3
+; AVX512BW-NEXT:    vpermt2q %zmm26, %zmm23, %zmm3
+; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm30, %zmm3
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm11, %zmm3
+; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm17, %zmm3
+; AVX512BW-NEXT:    vpermt2q %zmm26, %zmm11, %zmm3
+; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm30, %zmm3
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm25, %zmm3
+; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm17, %zmm3
+; AVX512BW-NEXT:    vpermt2q %zmm26, %zmm25, %zmm3
+; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512BW-NEXT:    vpermt2q %zmm30, %zmm18, %zmm3
+; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [2,9,2,9,2,9,2,9]
+; AVX512BW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm0, %zmm30
+; AVX512BW-NEXT:    vmovdqa64 %zmm26, %zmm10
+; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm18, %zmm10
+; AVX512BW-NEXT:    vpermt2q %zmm26, %zmm0, %zmm17
+; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm29, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm29, %zmm25, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm6
+; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm8, %zmm6
+; AVX512BW-NEXT:    vmovdqa64 1408(%rdi), %zmm24
+; AVX512BW-NEXT:    vmovdqa64 1344(%rdi), %zmm15
+; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm3
+; AVX512BW-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm24, %zmm25, %zmm3
+; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 1664(%rdi), %zmm1
+; AVX512BW-NEXT:    vmovdqa64 1600(%rdi), %zmm16
+; AVX512BW-NEXT:    vpermi2q %zmm1, %zmm16, %zmm8
+; AVX512BW-NEXT:    vmovdqa64 1088(%rdi), %zmm13
+; AVX512BW-NEXT:    vmovdqa64 1024(%rdi), %zmm5
 ; AVX512BW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 832(%rdi), %zmm5
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm4, %zmm6
+; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm11, %zmm5
+; AVX512BW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm18, %zmm9
+; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm18, %zmm29
+; AVX512BW-NEXT:    vmovdqu64 %zmm29, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm4
+; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm7, %zmm4
+; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm14
+; AVX512BW-NEXT:    vmovdqa64 640(%rdi), %zmm18
+; AVX512BW-NEXT:    vmovdqa64 576(%rdi), %zmm4
+; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm11, %zmm4
+; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm12, %zmm9, %zmm28
+; AVX512BW-NEXT:    vmovdqa64 192(%rdi), %zmm19
+; AVX512BW-NEXT:    vmovdqa64 128(%rdi), %zmm4
+; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm11, %zmm4
+; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm26 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm9, %zmm26 # 64-byte Folded Reload
+; AVX512BW-NEXT:    vmovdqa64 1536(%rdi), %zmm12
+; AVX512BW-NEXT:    vmovdqa64 1472(%rdi), %zmm4
+; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm12, %zmm11, %zmm4
+; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm24, %zmm22
+; AVX512BW-NEXT:    vpermt2q %zmm15, %zmm9, %zmm22
+; AVX512BW-NEXT:    vpermi2q %zmm1, %zmm16, %zmm7
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm29
+; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm23, %zmm29
+; AVX512BW-NEXT:    vpermi2q %zmm1, %zmm16, %zmm23
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm31
+; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm11, %zmm31
+; AVX512BW-NEXT:    vpermi2q %zmm1, %zmm16, %zmm11
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm15
+; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm25, %zmm15
+; AVX512BW-NEXT:    vpermi2q %zmm1, %zmm16, %zmm25
+; AVX512BW-NEXT:    vmovdqa64 %zmm20, %zmm21
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm9, %zmm21
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm4
+; AVX512BW-NEXT:    vpermi2q %zmm16, %zmm1, %zmm9
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm0, %zmm16
+; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm0, %zmm4
+; AVX512BW-NEXT:    vmovdqa64 384(%rdi), %zmm0
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm1 = [0,5,6,9,0,5,6,9]
+; AVX512BW-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm1, %zmm2
+; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 832(%rdi), %zmm3
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm1, %zmm2
+; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 1280(%rdi), %zmm5
+; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm1, %zmm6
 ; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 1280(%rdi), %zmm6
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm4, %zmm7
+; AVX512BW-NEXT:    vmovdqa64 1728(%rdi), %zmm6
+; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm1, %zmm8
+; AVX512BW-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm1 = [0,5,6,10,0,5,6,10]
+; AVX512BW-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm1, %zmm14
+; AVX512BW-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm1, %zmm2
+; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm1, %zmm2
+; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm1, %zmm7
 ; AVX512BW-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 1728(%rdi), %zmm7
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm4, %zmm19
-; AVX512BW-NEXT:    vmovdqu64 %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [0,5,6,10,0,5,6,10]
-; AVX512BW-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm4, %zmm19
-; AVX512BW-NEXT:    vmovdqu64 %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm4, %zmm19
-; AVX512BW-NEXT:    vmovdqu64 %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm4, %zmm18
-; AVX512BW-NEXT:    vmovdqu64 %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm4, %zmm13
-; AVX512BW-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [0,5,6,11,0,5,6,11]
-; AVX512BW-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm4, %zmm31
-; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm4, %zmm17
-; AVX512BW-NEXT:    vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm4, %zmm23
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm1 = [0,5,6,11,0,5,6,11]
+; AVX512BW-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm1, %zmm29
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm1, %zmm2
+; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm1, %zmm2
+; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm1, %zmm23
 ; AVX512BW-NEXT:    vmovdqu64 %zmm23, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm4, %zmm26
-; AVX512BW-NEXT:    vmovdqu64 %zmm26, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [0,5,6,12,0,5,6,12]
-; AVX512BW-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm4, %zmm27
-; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm4, %zmm16
-; AVX512BW-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm4, %zmm24
-; AVX512BW-NEXT:    vmovdqu64 %zmm24, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm4, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [4,5,6,13,4,5,6,13]
-; AVX512BW-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm4, %zmm20
-; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm4, %zmm15
-; AVX512BW-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm4, %zmm25
-; AVX512BW-NEXT:    vmovdqu64 %zmm25, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm4, %zmm28
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [4,5,6,14,4,5,6,14]
-; AVX512BW-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm4, %zmm12
-; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm4, %zmm30
-; AVX512BW-NEXT:    vmovdqu64 %zmm30, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm4, %zmm9
-; AVX512BW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm4, %zmm29
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [4,5,8,15,4,5,8,15]
-; AVX512BW-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm4, %zmm10
-; AVX512BW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm4, %zmm8
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm1 = [0,5,6,12,0,5,6,12]
+; AVX512BW-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm1, %zmm31
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm1, %zmm2
+; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm1, %zmm2
+; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm1, %zmm11
+; AVX512BW-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm1 = [4,5,6,13,4,5,6,13]
+; AVX512BW-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm1, %zmm15
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm1, %zmm2
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm1, %zmm7
+; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm1, %zmm25
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm1 = [4,5,6,14,4,5,6,14]
+; AVX512BW-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm1, %zmm21
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm1, %zmm8
 ; AVX512BW-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm4, %zmm21
-; AVX512BW-NEXT:    vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm4, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm1, %zmm10
+; AVX512BW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm1, %zmm9
+; AVX512BW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm1 = [4,5,8,15,4,5,8,15]
+; AVX512BW-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm1, %zmm30
+; AVX512BW-NEXT:    vmovdqu64 %zmm30, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm1, %zmm17
+; AVX512BW-NEXT:    vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm1, %zmm16
+; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm1, %zmm4
+; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    movb $24, %al
 ; AVX512BW-NEXT:    kmovd %eax, %k1
-; AVX512BW-NEXT:    vmovdqa64 512(%rdi), %zmm2
-; AVX512BW-NEXT:    vmovdqa64 448(%rdi), %zmm15
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} ymm21 = <0,7,14,u>
-; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm24
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm21, %zmm24
-; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm0
-; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm24 {%k1} = zmm14[4,5,4,5],zmm11[4,5,4,5]
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm11 = [14,0,0,7,14,0,0,7]
-; AVX512BW-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm13
-; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm11, %zmm13
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm17 = [7,0,9,0,7,0,9,0]
-; AVX512BW-NEXT:    # zmm17 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm26
-; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm17, %zmm26
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} ymm19 = [4,11,4,11]
-; AVX512BW-NEXT:    # ymm19 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm19, %zmm14
-; AVX512BW-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 64(%rdi), %zmm4
-; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm14
-; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm18
-; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm21, %zmm18
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm18 {%k1} = zmm0[4,5,4,5],zmm22[4,5,4,5]
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm23
-; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm11, %zmm23
-; AVX512BW-NEXT:    vmovdqa64 %zmm22, %zmm30
-; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm17, %zmm30
-; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm19, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm0 {%k1} = zmm1[4,5,4,5],zmm18[4,5,4,5]
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 960(%rdi), %zmm7
-; AVX512BW-NEXT:    vmovdqa64 896(%rdi), %zmm10
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm16
-; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm21, %zmm16
-; AVX512BW-NEXT:    vmovdqa64 1408(%rdi), %zmm5
-; AVX512BW-NEXT:    vmovdqa64 1344(%rdi), %zmm8
-; AVX512BW-NEXT:    vpermi2q %zmm5, %zmm8, %zmm21
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
-; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm21 {%k1} = zmm6[4,5,4,5],zmm3[4,5,4,5]
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [14,0,0,7,14,0,0,7]
+; AVX512BW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm0, %zmm3
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm10 = [7,0,9,0,7,0,9,0]
+; AVX512BW-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm18, %zmm27
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm10, %zmm27
+; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm5 = [4,11,4,11]
+; AVX512BW-NEXT:    # ymm5 = mem[0,1,0,1]
+; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm5, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm22 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm11, %zmm0
-; AVX512BW-NEXT:    vpermi2q %zmm3, %zmm6, %zmm11
-; AVX512BW-NEXT:    vmovdqa64 %zmm22, %zmm25
-; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm17, %zmm25
-; AVX512BW-NEXT:    vpermi2q %zmm6, %zmm3, %zmm17
-; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm19, %zmm6
-; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm16 {%k1} = zmm1[4,5,4,5],zmm22[4,5,4,5]
-; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm19, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm1 {%k1} = zmm4[4,5,4,5],zmm19[4,5,4,5]
 ; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa 912(%rdi), %xmm1
-; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0,1],mem[2,3]
-; AVX512BW-NEXT:    vinserti32x4 $0, %xmm1, %zmm0, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa 464(%rdi), %xmm0
-; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0,1],mem[2,3]
-; AVX512BW-NEXT:    vinserti32x4 $0, %xmm0, %zmm13, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa 16(%rdi), %xmm0
-; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0,1],mem[2,3]
-; AVX512BW-NEXT:    vinserti32x4 $0, %xmm0, %zmm23, %zmm23
-; AVX512BW-NEXT:    vmovdqa 1360(%rdi), %xmm0
-; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm0 = xmm0[0,1],mem[2,3]
-; AVX512BW-NEXT:    vinserti32x4 $0, %xmm0, %zmm11, %zmm22
+; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm18
+; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm0, %zmm18
+; AVX512BW-NEXT:    vmovdqa64 %zmm19, %zmm23
+; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm10, %zmm23
+; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm5, %zmm4
+; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
+; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm17 {%k1} = zmm4[4,5,4,5],zmm12[4,5,4,5]
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm19
+; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm0, %zmm19
+; AVX512BW-NEXT:    vpermi2q %zmm12, %zmm4, %zmm0
+; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm20
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm10, %zmm20
+; AVX512BW-NEXT:    vpermi2q %zmm4, %zmm12, %zmm10
+; AVX512BW-NEXT:    vpermt2q %zmm12, %zmm5, %zmm4
+; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
+; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm14 {%k1} = zmm1[4,5,4,5],zmm13[4,5,4,5]
+; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm5, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa 912(%rdi), %xmm5
+; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm5 = xmm5[0,1],mem[2,3]
+; AVX512BW-NEXT:    vinserti32x4 $0, %xmm5, %zmm19, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa 464(%rdi), %xmm5
+; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm5 = xmm5[0,1],mem[2,3]
+; AVX512BW-NEXT:    vinserti32x4 $0, %xmm5, %zmm3, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa 16(%rdi), %xmm3
+; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm3 = xmm3[0,1],mem[2,3]
+; AVX512BW-NEXT:    vinserti32x4 $0, %xmm3, %zmm18, %zmm19
+; AVX512BW-NEXT:    vmovdqa 1360(%rdi), %xmm3
+; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm3 = xmm3[0,1],mem[2,3]
+; AVX512BW-NEXT:    vinserti32x4 $0, %xmm3, %zmm0, %zmm18
 ; AVX512BW-NEXT:    vinserti128 $1, 1088(%rdi), %ymm0, %ymm0
-; AVX512BW-NEXT:    vmovdqa 1024(%rdi), %ymm1
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
-; AVX512BW-NEXT:    vmovdqa64 %ymm1, %ymm19
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm1 = [4,11]
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm11
-; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm1, %zmm11
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm11[0,1,2,3],ymm0[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm20, %zmm0
+; AVX512BW-NEXT:    vmovdqa 1024(%rdi), %ymm13
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm13[0,1,2,3,4,5],ymm0[6,7]
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm3 = [4,11]
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm12
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm3, %zmm12
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm12[0,1,2,3],ymm0[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm15, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vinserti128 $1, 640(%rdi), %ymm0, %ymm0
-; AVX512BW-NEXT:    vmovdqa 576(%rdi), %ymm3
-; AVX512BW-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3,4,5],ymm0[6,7]
-; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm3
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm1, %zmm3
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3],ymm0[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm3, %zmm0
+; AVX512BW-NEXT:    vmovdqa 576(%rdi), %ymm1
+; AVX512BW-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm11
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm3, %zmm11
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm11[0,1,2,3],ymm0[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm2, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vinserti128 $1, 192(%rdi), %ymm0, %ymm0
-; AVX512BW-NEXT:    vmovdqa 128(%rdi), %ymm3
-; AVX512BW-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3,4,5],ymm0[6,7]
-; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm3
-; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm1, %zmm3
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3],ymm0[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm3, %zmm0
+; AVX512BW-NEXT:    vmovdqa 128(%rdi), %ymm12
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm12[0,1,2,3,4,5],ymm0[6,7]
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm11
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm3, %zmm11
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm11[0,1,2,3],ymm0[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm7, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vinserti128 $1, 1536(%rdi), %ymm0, %ymm0
-; AVX512BW-NEXT:    vmovdqa 1472(%rdi), %ymm3
-; AVX512BW-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3,4,5],ymm0[6,7]
-; AVX512BW-NEXT:    vpermi2q %zmm5, %zmm8, %zmm1
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm28, %zmm0
+; AVX512BW-NEXT:    vmovdqa 1472(%rdi), %ymm2
+; AVX512BW-NEXT:    vmovdqu %ymm2, (%rsp) # 32-byte Spill
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5],ymm0[6,7]
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-NEXT:    vpermi2q %zmm24, %zmm2, %zmm3
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3],ymm0[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm25, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm7 = [5,12]
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm11
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm7, %zmm11
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} xmm30 = [6,13]
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm30, %zmm6
+; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm3
+; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm7, %zmm3
+; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm30, %zmm5
+; AVX512BW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm0
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm4
+; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm7, %zmm0
+; AVX512BW-NEXT:    vpermi2q %zmm24, %zmm2, %zmm7
+; AVX512BW-NEXT:    vpermt2q %zmm24, %zmm30, %zmm2
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm8
+; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm30, %zmm9
+; AVX512BW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 1088(%rdi), %ymm24
+; AVX512BW-NEXT:    vpalignr {{.*#+}} ymm5 = ymm13[8,9,10,11,12,13,14,15],ymm24[0,1,2,3,4,5,6,7],ymm13[24,25,26,27,28,29,30,31],ymm24[16,17,18,19,20,21,22,23]
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3],ymm5[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm21, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm0 = <9,0,7,u>
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512BW-NEXT:    vpermt2q %zmm15, %zmm0, %zmm3
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm1 = [5,12]
-; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm13
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm1, %zmm13
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm6 = [6,13]
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm6, %zmm15
-; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm2
-; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm0, %zmm2
-; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm11
-; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm1, %zmm11
-; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm6, %zmm14
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm4
-; AVX512BW-NEXT:    vpermt2q %zmm10, %zmm0, %zmm4
-; AVX512BW-NEXT:    vpermi2q %zmm8, %zmm5, %zmm0
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm9
-; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm1, %zmm9
-; AVX512BW-NEXT:    vpermi2q %zmm5, %zmm8, %zmm1
-; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm6, %zmm8
-; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm6, %zmm10
-; AVX512BW-NEXT:    vmovdqa 1088(%rdi), %ymm5
-; AVX512BW-NEXT:    vpalignr {{.*#+}} ymm5 = ymm19[8,9,10,11,12,13,14,15],ymm5[0,1,2,3,4,5,6,7],ymm19[24,25,26,27,28,29,30,31],ymm5[16,17,18,19,20,21,22,23]
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm5 = ymm9[0,1,2,3],ymm5[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm5, %zmm12, %zmm28
 ; AVX512BW-NEXT:    movb $-32, %al
 ; AVX512BW-NEXT:    kmovd %eax, %k2
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm24 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm24 {%k2}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm25 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm25 {%k2}
+; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm2
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k2}
+; AVX512BW-NEXT:    vmovdqa64 %zmm17, %zmm14
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm14 {%k2}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k2}
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm30
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm28 {%k1}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm28 {%k2}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm26 {%k1}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm26 {%k2}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm22 {%k1}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm22 {%k2}
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm18 {%k2}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm24 {%k2}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm16 {%k2}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm21 {%k2}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm4 {%k1}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm4 {%k2}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm3 {%k1}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm3 {%k2}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm2 {%k1}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm2 {%k2}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm0 {%k1}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm0 {%k2}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm31, %zmm12 {%k2}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm19 {%k2}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm23 {%k2}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm22 {%k2}
-; AVX512BW-NEXT:    vmovdqa 960(%rdi), %ymm6
+; AVX512BW-NEXT:    vmovdqa64 %zmm29, %zmm5 {%k2}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm29 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm29 {%k2}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm19 {%k2}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm18 {%k2}
+; AVX512BW-NEXT:    vmovdqa 960(%rdi), %ymm1
+; AVX512BW-NEXT:    vpalignr {{.*#+}} ymm1 = mem[8,9,10,11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23]
+; AVX512BW-NEXT:    vextracti128 $1, %ymm1, %xmm1
+; AVX512BW-NEXT:    vinserti32x4 $0, %xmm1, %zmm20, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm31, %zmm1 {%k2}
+; AVX512BW-NEXT:    vmovdqa 512(%rdi), %ymm6
 ; AVX512BW-NEXT:    vpalignr {{.*#+}} ymm6 = mem[8,9,10,11,12,13,14,15],ymm6[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm6[16,17,18,19,20,21,22,23]
 ; AVX512BW-NEXT:    vextracti128 $1, %ymm6, %xmm6
-; AVX512BW-NEXT:    vinserti32x4 $0, %xmm6, %zmm25, %zmm6
-; AVX512BW-NEXT:    vmovdqa64 %zmm27, %zmm6 {%k2}
-; AVX512BW-NEXT:    vmovdqa 512(%rdi), %ymm7
-; AVX512BW-NEXT:    vpalignr {{.*#+}} ymm7 = mem[8,9,10,11,12,13,14,15],ymm7[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm7[16,17,18,19,20,21,22,23]
-; AVX512BW-NEXT:    vextracti128 $1, %ymm7, %xmm7
-; AVX512BW-NEXT:    vinserti32x4 $0, %xmm7, %zmm26, %zmm7
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm7 {%k2}
-; AVX512BW-NEXT:    vmovdqa 64(%rdi), %ymm9
-; AVX512BW-NEXT:    vpalignr {{.*#+}} ymm9 = mem[8,9,10,11,12,13,14,15],ymm9[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm9[16,17,18,19,20,21,22,23]
-; AVX512BW-NEXT:    vextracti128 $1, %ymm9, %xmm9
-; AVX512BW-NEXT:    vinserti32x4 $0, %xmm9, %zmm30, %zmm9
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm9 {%k2}
-; AVX512BW-NEXT:    vmovdqa64 1408(%rdi), %ymm20
-; AVX512BW-NEXT:    vpalignr {{.*#+}} ymm20 = mem[8,9,10,11,12,13,14,15],ymm20[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm20[16,17,18,19,20,21,22,23]
-; AVX512BW-NEXT:    vextracti32x4 $1, %ymm20, %xmm20
-; AVX512BW-NEXT:    vinserti32x4 $0, %xmm20, %zmm17, %zmm17
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm17 {%k2}
-; AVX512BW-NEXT:    vmovdqa64 640(%rdi), %ymm20
-; AVX512BW-NEXT:    vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm20, %ymm5 # 32-byte Folded Reload
-; AVX512BW-NEXT:    # ymm5 = mem[8,9,10,11,12,13,14,15],ymm20[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm20[16,17,18,19,20,21,22,23]
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm5 = ymm13[0,1,2,3],ymm5[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm5, %zmm13, %zmm5
+; AVX512BW-NEXT:    vinserti32x4 $0, %xmm6, %zmm27, %zmm6
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm6 {%k2}
+; AVX512BW-NEXT:    vmovdqa64 64(%rdi), %ymm17
+; AVX512BW-NEXT:    vpalignr {{.*#+}} ymm17 = mem[8,9,10,11,12,13,14,15],ymm17[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm17[16,17,18,19,20,21,22,23]
+; AVX512BW-NEXT:    vextracti32x4 $1, %ymm17, %xmm17
+; AVX512BW-NEXT:    vinserti32x4 $0, %xmm17, %zmm23, %zmm15
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm15 {%k2}
+; AVX512BW-NEXT:    vmovdqa64 1408(%rdi), %ymm17
+; AVX512BW-NEXT:    vpalignr {{.*#+}} ymm17 = mem[8,9,10,11,12,13,14,15],ymm17[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm17[16,17,18,19,20,21,22,23]
+; AVX512BW-NEXT:    vextracti32x4 $1, %ymm17, %xmm17
+; AVX512BW-NEXT:    vinserti32x4 $0, %xmm17, %zmm10, %zmm21
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm21 {%k2}
+; AVX512BW-NEXT:    vmovdqa64 640(%rdi), %ymm17
+; AVX512BW-NEXT:    vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm17, %ymm13 # 32-byte Folded Reload
+; AVX512BW-NEXT:    # ymm13 = mem[8,9,10,11,12,13,14,15],ymm17[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm17[16,17,18,19,20,21,22,23]
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm11[0,1,2,3],ymm13[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm4, %zmm0, %zmm4
 ; AVX512BW-NEXT:    vmovdqa 192(%rdi), %ymm13
-; AVX512BW-NEXT:    vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm13, %ymm13 # 32-byte Folded Reload
-; AVX512BW-NEXT:    # ymm13 = mem[8,9,10,11,12,13,14,15],ymm13[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm13[16,17,18,19,20,21,22,23]
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm11 = ymm11[0,1,2,3],ymm13[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm11, %zmm13, %zmm11
-; AVX512BW-NEXT:    vmovdqa 1536(%rdi), %ymm13
-; AVX512BW-NEXT:    vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm13, %ymm13 # 32-byte Folded Reload
-; AVX512BW-NEXT:    # ymm13 = mem[8,9,10,11,12,13,14,15],ymm13[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm13[16,17,18,19,20,21,22,23]
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm13[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm29, %zmm1
-; AVX512BW-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm15, %ymm13 # 32-byte Folded Reload
-; AVX512BW-NEXT:    # ymm13 = ymm15[0,1,2,3],mem[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm13, %zmm15, %zmm13
-; AVX512BW-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm14 # 32-byte Folded Reload
-; AVX512BW-NEXT:    # ymm14 = ymm14[0,1,2,3],mem[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm14, %zmm15, %zmm14
+; AVX512BW-NEXT:    vpalignr {{.*#+}} ymm12 = ymm12[8,9,10,11,12,13,14,15],ymm13[0,1,2,3,4,5,6,7],ymm12[24,25,26,27,28,29,30,31],ymm13[16,17,18,19,20,21,22,23]
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm12[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm3
+; AVX512BW-NEXT:    vmovdqa 1536(%rdi), %ymm12
+; AVX512BW-NEXT:    vpalignr $8, (%rsp), %ymm12, %ymm11 # 32-byte Folded Reload
+; AVX512BW-NEXT:    # ymm11 = mem[8,9,10,11,12,13,14,15],ymm12[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm12[16,17,18,19,20,21,22,23]
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm7[0,1,2,3],ymm11[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm7, %zmm0
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512BW-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm11 # 32-byte Folded Reload
+; AVX512BW-NEXT:    # ymm11 = ymm7[0,1,2,3],mem[4,5,6,7]
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512BW-NEXT:    vinsertf64x4 $0, %ymm11, %zmm7, %zmm11
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512BW-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm9 # 32-byte Folded Reload
+; AVX512BW-NEXT:    # ymm9 = ymm7[0,1,2,3],mem[4,5,6,7]
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512BW-NEXT:    vinsertf64x4 $0, %ymm9, %zmm7, %zmm9
 ; AVX512BW-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm8 # 32-byte Folded Reload
 ; AVX512BW-NEXT:    # ymm8 = ymm8[0,1,2,3],mem[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm8, %zmm15, %zmm8
-; AVX512BW-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm10, %ymm10 # 32-byte Folded Reload
-; AVX512BW-NEXT:    # ymm10 = ymm10[0,1,2,3],mem[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm10, %zmm15, %zmm10
-; AVX512BW-NEXT:    vmovdqa64 %zmm21, 192(%rsi)
-; AVX512BW-NEXT:    vmovdqa64 %zmm16, 128(%rsi)
-; AVX512BW-NEXT:    vmovdqa64 %zmm24, 64(%rsi)
-; AVX512BW-NEXT:    vmovdqa64 %zmm18, (%rsi)
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, 192(%rdx)
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, (%rdx)
-; AVX512BW-NEXT:    vmovdqa64 %zmm3, 64(%rdx)
-; AVX512BW-NEXT:    vmovdqa64 %zmm4, 128(%rdx)
-; AVX512BW-NEXT:    vmovdqa64 %zmm22, 192(%rcx)
-; AVX512BW-NEXT:    vmovdqa64 %zmm23, (%rcx)
-; AVX512BW-NEXT:    vmovdqa64 %zmm19, 64(%rcx)
-; AVX512BW-NEXT:    vmovdqa64 %zmm12, 128(%rcx)
-; AVX512BW-NEXT:    vmovdqa64 %zmm17, 192(%r8)
-; AVX512BW-NEXT:    vmovdqa64 %zmm9, (%r8)
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, 64(%r8)
-; AVX512BW-NEXT:    vmovdqa64 %zmm6, 128(%r8)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm0, 192(%r9)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm0, (%r9)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm0, 64(%r9)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm0, 128(%r9)
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm8, %zmm16, %zmm7
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512BW-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm8 # 32-byte Folded Reload
+; AVX512BW-NEXT:    # ymm8 = ymm8[0,1,2,3],mem[4,5,6,7]
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
+; AVX512BW-NEXT:    vinsertf64x4 $0, %ymm8, %zmm10, %zmm8
+; AVX512BW-NEXT:    vmovdqa64 %zmm14, 192(%rsi)
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, 128(%rsi)
+; AVX512BW-NEXT:    vmovdqa64 %zmm25, 64(%rsi)
+; AVX512BW-NEXT:    vmovdqa64 %zmm24, (%rsi)
+; AVX512BW-NEXT:    vmovdqa64 %zmm22, 192(%rdx)
+; AVX512BW-NEXT:    vmovdqa64 %zmm26, (%rdx)
+; AVX512BW-NEXT:    vmovdqa64 %zmm28, 64(%rdx)
+; AVX512BW-NEXT:    vmovdqa64 %zmm30, 128(%rdx)
+; AVX512BW-NEXT:    vmovdqa64 %zmm18, 192(%rcx)
+; AVX512BW-NEXT:    vmovdqa64 %zmm19, (%rcx)
+; AVX512BW-NEXT:    vmovdqa64 %zmm29, 64(%rcx)
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, 128(%rcx)
+; AVX512BW-NEXT:    vmovdqa64 %zmm21, 192(%r8)
+; AVX512BW-NEXT:    vmovdqa64 %zmm15, (%r8)
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, 64(%r8)
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, 128(%r8)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm1, 192(%r9)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm1, (%r9)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm1, 64(%r9)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm1, 128(%r9)
 ; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, 192(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm11, (%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, 64(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm28, 128(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, 192(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, (%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm4, 64(%rax)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm0, 128(%rax)
 ; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, 128(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm8, 192(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm14, (%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm13, 64(%rax)
-; AVX512BW-NEXT:    addq $2152, %rsp # imm = 0x868
+; AVX512BW-NEXT:    vmovaps %zmm8, 128(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, 192(%rax)
+; AVX512BW-NEXT:    vmovaps %zmm9, (%rax)
+; AVX512BW-NEXT:    vmovaps %zmm11, 64(%rax)
+; AVX512BW-NEXT:    addq $2760, %rsp # imm = 0xAC8
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
   %wide.vec = load <224 x i64>, ptr %in.vec, align 64
@@ -8015,45 +8085,46 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ;
 ; AVX512F-LABEL: load_i64_stride7_vf64:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    subq $6728, %rsp # imm = 0x1A48
-; AVX512F-NEXT:    vmovdqa64 3328(%rdi), %zmm17
-; AVX512F-NEXT:    vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 3264(%rdi), %zmm3
-; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    subq $7624, %rsp # imm = 0x1DC8
+; AVX512F-NEXT:    vmovdqa64 3328(%rdi), %zmm16
+; AVX512F-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 3264(%rdi), %zmm8
+; AVX512F-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vmovdqa64 3008(%rdi), %zmm19
-; AVX512F-NEXT:    vmovdqa64 2944(%rdi), %zmm22
+; AVX512F-NEXT:    vmovdqu64 %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 2944(%rdi), %zmm20
 ; AVX512F-NEXT:    vmovdqa64 2880(%rdi), %zmm2
 ; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vmovdqa64 2816(%rdi), %zmm1
 ; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 2752(%rdi), %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm20
-; AVX512F-NEXT:    vmovdqa64 2688(%rdi), %zmm15
-; AVX512F-NEXT:    vmovdqa64 2432(%rdi), %zmm4
-; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 2368(%rdi), %zmm5
-; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 1984(%rdi), %zmm7
+; AVX512F-NEXT:    vmovdqa64 2752(%rdi), %zmm18
+; AVX512F-NEXT:    vmovdqu64 %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 2688(%rdi), %zmm7
 ; AVX512F-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 1920(%rdi), %zmm8
-; AVX512F-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 1536(%rdi), %zmm9
+; AVX512F-NEXT:    vmovdqa64 2432(%rdi), %zmm17
+; AVX512F-NEXT:    vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 2368(%rdi), %zmm9
 ; AVX512F-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 1472(%rdi), %zmm10
-; AVX512F-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 1088(%rdi), %zmm13
-; AVX512F-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 1024(%rdi), %zmm11
+; AVX512F-NEXT:    vmovdqa64 1984(%rdi), %zmm11
 ; AVX512F-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 640(%rdi), %zmm14
+; AVX512F-NEXT:    vmovdqa64 1920(%rdi), %zmm3
+; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 1536(%rdi), %zmm12
+; AVX512F-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 1472(%rdi), %zmm13
+; AVX512F-NEXT:    vmovdqu64 %zmm13, (%rsp) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 1088(%rdi), %zmm14
 ; AVX512F-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 1024(%rdi), %zmm4
+; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 640(%rdi), %zmm10
+; AVX512F-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vmovdqa64 576(%rdi), %zmm6
 ; AVX512F-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 192(%rdi), %zmm18
-; AVX512F-NEXT:    vmovdqu64 %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 128(%rdi), %zmm12
-; AVX512F-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 192(%rdi), %zmm15
+; AVX512F-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 128(%rdi), %zmm5
+; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [14,0,0,7,14,0,0,7]
 ; AVX512F-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
 ; AVX512F-NEXT:    vpermt2q %zmm2, %zmm0, %zmm1
@@ -8061,972 +8132,997 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512F-NEXT:    vpblendd {{.*#+}} xmm2 = xmm2[0,1],mem[2,3]
 ; AVX512F-NEXT:    vinserti32x4 $0, %xmm2, %zmm1, %zmm1
 ; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm1
-; AVX512F-NEXT:    vpermt2q %zmm14, %zmm0, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm10, %zmm0, %zmm6
 ; AVX512F-NEXT:    vmovdqa 464(%rdi), %xmm2
 ; AVX512F-NEXT:    vpblendd {{.*#+}} xmm2 = xmm2[0,1],mem[2,3]
-; AVX512F-NEXT:    vinserti32x4 $0, %xmm2, %zmm1, %zmm1
+; AVX512F-NEXT:    vinserti32x4 $0, %xmm2, %zmm6, %zmm1
 ; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm12, %zmm1
-; AVX512F-NEXT:    vpermt2q %zmm18, %zmm0, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm15, %zmm0, %zmm1
 ; AVX512F-NEXT:    vmovdqa 16(%rdi), %xmm2
 ; AVX512F-NEXT:    vpblendd {{.*#+}} xmm2 = xmm2[0,1],mem[2,3]
 ; AVX512F-NEXT:    vinserti32x4 $0, %xmm2, %zmm1, %zmm1
 ; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm1
-; AVX512F-NEXT:    vpermt2q %zmm9, %zmm0, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm12, %zmm0, %zmm1
 ; AVX512F-NEXT:    vmovdqa 1360(%rdi), %xmm2
 ; AVX512F-NEXT:    vpblendd {{.*#+}} xmm2 = xmm2[0,1],mem[2,3]
 ; AVX512F-NEXT:    vinserti32x4 $0, %xmm2, %zmm1, %zmm1
 ; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm1
-; AVX512F-NEXT:    vpermt2q %zmm13, %zmm0, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm14, %zmm0, %zmm1
 ; AVX512F-NEXT:    vmovdqa 912(%rdi), %xmm2
 ; AVX512F-NEXT:    vpblendd {{.*#+}} xmm2 = xmm2[0,1],mem[2,3]
 ; AVX512F-NEXT:    vinserti32x4 $0, %xmm2, %zmm1, %zmm1
 ; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm1
-; AVX512F-NEXT:    vpermt2q %zmm4, %zmm0, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm17, %zmm0, %zmm1
 ; AVX512F-NEXT:    vmovdqa 2256(%rdi), %xmm2
 ; AVX512F-NEXT:    vpblendd {{.*#+}} xmm2 = xmm2[0,1],mem[2,3]
 ; AVX512F-NEXT:    vinserti32x4 $0, %xmm2, %zmm1, %zmm1
 ; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm1
-; AVX512F-NEXT:    vpermt2q %zmm7, %zmm0, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm11, %zmm0, %zmm1
 ; AVX512F-NEXT:    vmovdqa 1808(%rdi), %xmm2
 ; AVX512F-NEXT:    vpblendd {{.*#+}} xmm2 = xmm2[0,1],mem[2,3]
 ; AVX512F-NEXT:    vinserti32x4 $0, %xmm2, %zmm1, %zmm1
 ; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermi2q %zmm17, %zmm3, %zmm0
+; AVX512F-NEXT:    vpermi2q %zmm16, %zmm8, %zmm0
 ; AVX512F-NEXT:    vmovdqa 3152(%rdi), %xmm1
 ; AVX512F-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0,1],mem[2,3]
 ; AVX512F-NEXT:    vinserti32x4 $0, %xmm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vinserti128 $1, 2880(%rdi), %ymm0, %ymm1
-; AVX512F-NEXT:    vmovdqa 2816(%rdi), %ymm0
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3,4,5],ymm1[6,7]
-; AVX512F-NEXT:    vmovdqa64 %ymm0, %ymm16
-; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm5 = [4,11]
-; AVX512F-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm20, %zmm5, %zmm2
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [0,7,14,0,0,7,14,0]
-; AVX512F-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vinserti128 $1, 2880(%rdi), %ymm0, %ymm1
+; AVX512F-NEXT:    vmovdqa 2816(%rdi), %ymm0
+; AVX512F-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3,4,5],ymm1[6,7]
+; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm9 = [4,11]
+; AVX512F-NEXT:    vpermt2q %zmm18, %zmm9, %zmm7
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm7[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm11 = [0,7,14,0,0,7,14,0]
+; AVX512F-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm20, %zmm12
+; AVX512F-NEXT:    vmovdqu64 %zmm20, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm20, %zmm2
+; AVX512F-NEXT:    vpermt2q %zmm19, %zmm11, %zmm2
+; AVX512F-NEXT:    vmovdqa64 3072(%rdi), %zmm3
+; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [4,5,6,13,4,5,6,13]
+; AVX512F-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm4, %zmm2
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm2, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vinserti128 $1, 640(%rdi), %ymm0, %ymm2
+; AVX512F-NEXT:    vmovdqa 576(%rdi), %ymm0
+; AVX512F-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm0[0,1,2,3,4,5],ymm2[6,7]
+; AVX512F-NEXT:    vmovdqa64 512(%rdi), %zmm6
+; AVX512F-NEXT:    vmovdqa64 448(%rdi), %zmm3
+; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm6, %zmm9, %zmm3
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm16
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 768(%rdi), %zmm31
+; AVX512F-NEXT:    vmovdqa64 704(%rdi), %zmm3
+; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm31, %zmm11, %zmm3
+; AVX512F-NEXT:    vmovdqa64 832(%rdi), %zmm5
+; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm5, %zmm4, %zmm3
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm3, %zmm2
+; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vinserti128 $1, 192(%rdi), %ymm0, %ymm3
+; AVX512F-NEXT:    vmovdqa 128(%rdi), %ymm2
+; AVX512F-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm3 = ymm2[0,1,2,3,4,5],ymm3[6,7]
+; AVX512F-NEXT:    vmovdqa64 64(%rdi), %zmm8
+; AVX512F-NEXT:    vmovdqa64 (%rdi), %zmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm5
+; AVX512F-NEXT:    vpermt2q %zmm8, %zmm9, %zmm5
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm23
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm3 = ymm5[0,1,2,3],ymm3[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 320(%rdi), %zmm13
+; AVX512F-NEXT:    vmovdqa64 256(%rdi), %zmm6
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm5
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm24
+; AVX512F-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm13, %zmm11, %zmm5
+; AVX512F-NEXT:    vmovdqa64 384(%rdi), %zmm6
+; AVX512F-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm6, %zmm4, %zmm5
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm3, %zmm5, %zmm3
+; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vinserti128 $1, 1536(%rdi), %ymm0, %ymm5
+; AVX512F-NEXT:    vmovdqa 1472(%rdi), %ymm3
+; AVX512F-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm5 = ymm3[0,1,2,3,4,5],ymm5[6,7]
+; AVX512F-NEXT:    vmovdqa64 1408(%rdi), %zmm3
+; AVX512F-NEXT:    vmovdqa64 1344(%rdi), %zmm7
+; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm6
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm9, %zmm6
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm19
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3],ymm5[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 1664(%rdi), %zmm17
+; AVX512F-NEXT:    vmovdqa64 1600(%rdi), %zmm8
+; AVX512F-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm17, %zmm11, %zmm8
+; AVX512F-NEXT:    vmovdqa64 1728(%rdi), %zmm3
+; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm4, %zmm8
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm6, %zmm8, %zmm5
+; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vinserti128 $1, 1088(%rdi), %ymm0, %ymm6
+; AVX512F-NEXT:    vmovdqa 1024(%rdi), %ymm3
+; AVX512F-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm6 = ymm3[0,1,2,3,4,5],ymm6[6,7]
+; AVX512F-NEXT:    vmovdqa64 960(%rdi), %zmm5
+; AVX512F-NEXT:    vmovdqa64 896(%rdi), %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm8
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm5, %zmm9, %zmm8
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm30
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3],ymm6[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 1216(%rdi), %zmm29
+; AVX512F-NEXT:    vmovdqa64 1152(%rdi), %zmm10
+; AVX512F-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm29, %zmm11, %zmm10
+; AVX512F-NEXT:    vmovdqa64 1280(%rdi), %zmm3
+; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm4, %zmm10
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm8, %zmm10, %zmm5
+; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vinserti128 $1, 2432(%rdi), %ymm0, %ymm8
+; AVX512F-NEXT:    vmovdqa 2368(%rdi), %ymm3
+; AVX512F-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm8 = ymm3[0,1,2,3,4,5],ymm8[6,7]
+; AVX512F-NEXT:    vmovdqa64 2304(%rdi), %zmm18
+; AVX512F-NEXT:    vmovdqa64 2240(%rdi), %zmm0
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm10
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm18, %zmm9, %zmm10
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm10 = ymm10[0,1,2,3],ymm8[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 2560(%rdi), %zmm20
+; AVX512F-NEXT:    vmovdqa64 2496(%rdi), %zmm14
+; AVX512F-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm20, %zmm11, %zmm14
+; AVX512F-NEXT:    vmovdqa64 2624(%rdi), %zmm21
+; AVX512F-NEXT:    vpermt2q %zmm21, %zmm4, %zmm14
+; AVX512F-NEXT:    vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm10, %zmm14, %zmm5
+; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vinserti128 $1, 1984(%rdi), %ymm0, %ymm10
+; AVX512F-NEXT:    vmovdqa 1920(%rdi), %ymm3
+; AVX512F-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm10 = ymm3[0,1,2,3,4,5],ymm10[6,7]
+; AVX512F-NEXT:    vmovdqa64 1856(%rdi), %zmm22
+; AVX512F-NEXT:    vmovdqa64 1792(%rdi), %zmm5
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm15
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm6
+; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm22, %zmm9, %zmm15
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm10 = ymm15[0,1,2,3],ymm10[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 2112(%rdi), %zmm26
+; AVX512F-NEXT:    vmovdqa64 2048(%rdi), %zmm15
+; AVX512F-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm26, %zmm11, %zmm15
+; AVX512F-NEXT:    vmovdqa64 2176(%rdi), %zmm28
+; AVX512F-NEXT:    vpermt2q %zmm28, %zmm4, %zmm15
+; AVX512F-NEXT:    vmovdqu64 %zmm28, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm10, %zmm15, %zmm5
+; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vinserti128 $1, 3328(%rdi), %ymm0, %ymm10
+; AVX512F-NEXT:    vmovdqa 3264(%rdi), %ymm3
+; AVX512F-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm10 = ymm3[0,1,2,3,4,5],ymm10[6,7]
+; AVX512F-NEXT:    vmovdqa64 3200(%rdi), %zmm25
+; AVX512F-NEXT:    vmovdqa64 3136(%rdi), %zmm14
+; AVX512F-NEXT:    vpermi2q %zmm25, %zmm14, %zmm9
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm10 = ymm9[0,1,2,3],ymm10[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm5
+; AVX512F-NEXT:    vpermt2q %zmm23, %zmm11, %zmm5
+; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm5
+; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm27
+; AVX512F-NEXT:    vpermt2q %zmm16, %zmm11, %zmm5
+; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm30, %zmm11, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm3
+; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm5
+; AVX512F-NEXT:    vpermt2q %zmm19, %zmm11, %zmm5
+; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm5
+; AVX512F-NEXT:    vpermt2q %zmm22, %zmm11, %zmm5
+; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm18, %zmm11, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm9
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm7, %zmm11, %zmm9
+; AVX512F-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm14, %zmm9
+; AVX512F-NEXT:    vpermt2q %zmm25, %zmm11, %zmm9
+; AVX512F-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 3456(%rdi), %zmm9
+; AVX512F-NEXT:    vmovdqa64 3392(%rdi), %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermi2q %zmm9, %zmm0, %zmm11
+; AVX512F-NEXT:    vmovdqa64 3520(%rdi), %zmm6
+; AVX512F-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm6, %zmm4, %zmm11
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm10, %zmm11, %zmm4
+; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa 2880(%rdi), %ymm4
+; AVX512F-NEXT:    vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm0 # 32-byte Folded Reload
+; AVX512F-NEXT:    # ymm0 = mem[8,9,10,11,12,13,14,15],ymm4[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm4[16,17,18,19,20,21,22,23]
+; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm11 = [5,12]
+; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm4
+; AVX512F-NEXT:    vpermt2q %zmm7, %zmm11, %zmm4
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm10 = ymm4[0,1,2,3],ymm0[4,5,6,7]
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [9,0,7,0,9,0,7,0]
+; AVX512F-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm12, %zmm0, %zmm16
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [4,5,6,14,4,5,6,14]
+; AVX512F-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm16 # 64-byte Folded Reload
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm10, %zmm16, %zmm5
+; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa 640(%rdi), %ymm10
+; AVX512F-NEXT:    vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm10, %ymm1 # 32-byte Folded Reload
+; AVX512F-NEXT:    # ymm1 = mem[8,9,10,11,12,13,14,15],ymm10[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm10[16,17,18,19,20,21,22,23]
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm10
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm6
+; AVX512F-NEXT:    vpermt2q %zmm27, %zmm11, %zmm10
+; AVX512F-NEXT:    vmovdqu64 %zmm27, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm10[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 %zmm31, %zmm10
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm16, %zmm0, %zmm10
+; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm10 # 64-byte Folded Reload
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm10, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa 192(%rdi), %ymm1
+; AVX512F-NEXT:    vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
+; AVX512F-NEXT:    # ymm1 = mem[8,9,10,11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23]
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm5
+; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 %zmm23, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm23, %zmm11, %zmm2
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm2
+; AVX512F-NEXT:    vpermt2q %zmm24, %zmm0, %zmm2
+; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm2 # 64-byte Folded Reload
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm2, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa 1536(%rdi), %ymm1
+; AVX512F-NEXT:    vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
+; AVX512F-NEXT:    # ymm1 = mem[8,9,10,11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23]
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm8
+; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm2
+; AVX512F-NEXT:    vmovdqu64 %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm19, %zmm11, %zmm2
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 %zmm17, %zmm2
+; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 # 64-byte Folded Reload
+; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm2 # 64-byte Folded Reload
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm2, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa 1088(%rdi), %ymm1
+; AVX512F-NEXT:    vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
+; AVX512F-NEXT:    # ymm1 = mem[8,9,10,11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23]
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm2
+; AVX512F-NEXT:    vmovdqu64 %zmm30, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm30, %zmm11, %zmm2
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 %zmm29, %zmm2
+; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 # 64-byte Folded Reload
+; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm2 # 64-byte Folded Reload
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm2, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa 2432(%rdi), %ymm1
+; AVX512F-NEXT:    vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
+; AVX512F-NEXT:    # ymm1 = mem[8,9,10,11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23]
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm2
+; AVX512F-NEXT:    vmovdqu64 %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm18, %zmm11, %zmm2
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 %zmm20, %zmm2
+; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 # 64-byte Folded Reload
+; AVX512F-NEXT:    vpermt2q %zmm21, %zmm4, %zmm2
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm2, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa 1984(%rdi), %ymm1
+; AVX512F-NEXT:    vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
+; AVX512F-NEXT:    # ymm1 = mem[8,9,10,11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23]
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm12, %zmm2
+; AVX512F-NEXT:    vmovdqu64 %zmm22, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm22, %zmm11, %zmm2
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 %zmm26, %zmm2
+; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 # 64-byte Folded Reload
+; AVX512F-NEXT:    vpermt2q %zmm28, %zmm4, %zmm2
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm2, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa 3328(%rdi), %ymm1
+; AVX512F-NEXT:    vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
+; AVX512F-NEXT:    # ymm1 = mem[8,9,10,11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23]
+; AVX512F-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 %zmm25, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermi2q %zmm25, %zmm14, %zmm11
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm11[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    vpermt2q %zmm15, %zmm0, %zmm7
+; AVX512F-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm27, %zmm2
+; AVX512F-NEXT:    vpermt2q %zmm6, %zmm0, %zmm2
+; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm23, %zmm2
+; AVX512F-NEXT:    vpermt2q %zmm5, %zmm0, %zmm2
+; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm19, %zmm2
+; AVX512F-NEXT:    vpermt2q %zmm8, %zmm0, %zmm2
+; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm30, %zmm2
+; AVX512F-NEXT:    vpermt2q %zmm10, %zmm0, %zmm2
+; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm0, %zmm18
+; AVX512F-NEXT:    vmovdqu64 %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm12, %zmm0, %zmm22
+; AVX512F-NEXT:    vmovdqu64 %zmm22, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm14, %zmm0, %zmm25
+; AVX512F-NEXT:    vmovdqu64 %zmm25, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm25
+; AVX512F-NEXT:    vpermi2q %zmm6, %zmm9, %zmm0
+; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm0 # 64-byte Folded Reload
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm23 = [10,3,10,3,10,3,10,3]
+; AVX512F-NEXT:    # zmm23 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm12
+; AVX512F-NEXT:    vpermt2q %zmm31, %zmm23, %zmm12
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm24 = [11,4,11,4,11,4,11,4]
+; AVX512F-NEXT:    # zmm24 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm31, %zmm24, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm8 = [12,5,12,5,12,5,12,5]
+; AVX512F-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm31, %zmm8, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm16 = [13,6,13,6,13,6,13,6]
+; AVX512F-NEXT:    # zmm16 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm31, %zmm16, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm1 = [2,9,2,9,2,9,2,9]
+; AVX512F-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpermt2q %zmm31, %zmm1, %zmm2
+; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm28 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm28, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm13, %zmm23, %zmm0
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm10
+; AVX512F-NEXT:    vmovdqa64 %zmm28, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm13, %zmm24, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm28, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm13, %zmm8, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm28, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm13, %zmm16, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm13, %zmm1, %zmm28
+; AVX512F-NEXT:    vmovdqu64 %zmm28, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm22 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm22, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm17, %zmm23, %zmm0
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm14
+; AVX512F-NEXT:    vmovdqa64 %zmm22, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm17, %zmm24, %zmm0
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm18
+; AVX512F-NEXT:    vmovdqa64 %zmm22, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm17, %zmm8, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm22, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm17, %zmm16, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm17, %zmm1, %zmm22
 ; AVX512F-NEXT:    vmovdqu64 %zmm22, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm22, %zmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm19, %zmm18
-; AVX512F-NEXT:    vmovdqu64 %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm19, %zmm4, %zmm2
-; AVX512F-NEXT:    vmovdqa64 3072(%rdi), %zmm0
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm30 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm30, %zmm9
+; AVX512F-NEXT:    vpermt2q %zmm29, %zmm23, %zmm9
+; AVX512F-NEXT:    vmovdqa64 %zmm30, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm29, %zmm24, %zmm0
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm19
+; AVX512F-NEXT:    vmovdqa64 %zmm30, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm29, %zmm8, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm9 = [4,5,6,13,4,5,6,13]
-; AVX512F-NEXT:    # zmm9 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermt2q %zmm0, %zmm9, %zmm2
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm2, %zmm1
-; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vinserti128 $1, 640(%rdi), %ymm0, %ymm1
-; AVX512F-NEXT:    vmovdqa 576(%rdi), %ymm2
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
-; AVX512F-NEXT:    vmovdqa64 512(%rdi), %zmm28
-; AVX512F-NEXT:    vmovdqa64 448(%rdi), %zmm3
-; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm28, %zmm5, %zmm3
-; AVX512F-NEXT:    vmovdqu64 %zmm28, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
-; AVX512F-NEXT:    vmovdqa64 768(%rdi), %zmm13
-; AVX512F-NEXT:    vmovdqa64 704(%rdi), %zmm3
-; AVX512F-NEXT:    vmovdqu64 %zmm3, (%rsp) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm13, %zmm4, %zmm3
-; AVX512F-NEXT:    vmovdqa64 832(%rdi), %zmm7
-; AVX512F-NEXT:    vpermt2q %zmm7, %zmm9, %zmm3
-; AVX512F-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm3, %zmm1
-; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vinserti128 $1, 192(%rdi), %ymm0, %ymm1
-; AVX512F-NEXT:    vmovdqa 128(%rdi), %ymm0
-; AVX512F-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3,4,5],ymm1[6,7]
-; AVX512F-NEXT:    vmovdqa64 64(%rdi), %zmm6
-; AVX512F-NEXT:    vmovdqa64 (%rdi), %zmm3
+; AVX512F-NEXT:    vmovdqa64 %zmm30, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm29, %zmm16, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm29, %zmm1, %zmm30
+; AVX512F-NEXT:    vmovdqu64 %zmm30, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm7
+; AVX512F-NEXT:    vpermt2q %zmm20, %zmm23, %zmm7
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm3
+; AVX512F-NEXT:    vpermt2q %zmm20, %zmm24, %zmm3
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm21
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm3
+; AVX512F-NEXT:    vpermt2q %zmm20, %zmm8, %zmm3
 ; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm6, %zmm5, %zmm3
-; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm30
-; AVX512F-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
-; AVX512F-NEXT:    vmovdqa64 320(%rdi), %zmm10
-; AVX512F-NEXT:    vmovdqa64 256(%rdi), %zmm3
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm3
+; AVX512F-NEXT:    vpermt2q %zmm20, %zmm16, %zmm3
 ; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm10, %zmm4, %zmm3
-; AVX512F-NEXT:    vmovdqa64 384(%rdi), %zmm27
-; AVX512F-NEXT:    vpermt2q %zmm27, %zmm9, %zmm3
-; AVX512F-NEXT:    vmovdqu64 %zmm27, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm3, %zmm1
-; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vinserti128 $1, 1536(%rdi), %ymm0, %ymm1
-; AVX512F-NEXT:    vmovdqa 1472(%rdi), %ymm3
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5],ymm1[6,7]
-; AVX512F-NEXT:    vmovdqa64 1408(%rdi), %zmm25
-; AVX512F-NEXT:    vmovdqa64 1344(%rdi), %zmm6
-; AVX512F-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm25, %zmm5, %zmm6
-; AVX512F-NEXT:    vmovdqu64 %zmm25, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm6[0,1,2,3],ymm1[4,5,6,7]
-; AVX512F-NEXT:    vmovdqa64 1664(%rdi), %zmm14
-; AVX512F-NEXT:    vmovdqa64 1600(%rdi), %zmm6
-; AVX512F-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm14, %zmm4, %zmm6
-; AVX512F-NEXT:    vmovdqa64 1728(%rdi), %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm20, %zmm1, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm0, %zmm9, %zmm6
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm6, %zmm1
-; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vinserti128 $1, 1088(%rdi), %ymm0, %ymm1
-; AVX512F-NEXT:    vmovdqa 1024(%rdi), %ymm0
-; AVX512F-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3,4,5],ymm1[6,7]
-; AVX512F-NEXT:    vmovdqa64 960(%rdi), %zmm19
-; AVX512F-NEXT:    vmovdqa64 896(%rdi), %zmm8
-; AVX512F-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm19, %zmm5, %zmm8
-; AVX512F-NEXT:    vmovdqu64 %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm8[0,1,2,3],ymm1[4,5,6,7]
-; AVX512F-NEXT:    vmovdqa64 1216(%rdi), %zmm20
-; AVX512F-NEXT:    vmovdqa64 1152(%rdi), %zmm8
-; AVX512F-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm20, %zmm4, %zmm8
-; AVX512F-NEXT:    vmovdqa64 1280(%rdi), %zmm0
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm30
+; AVX512F-NEXT:    vpermt2q %zmm26, %zmm23, %zmm30
+; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm26, %zmm24, %zmm0
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm22
+; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm5
+; AVX512F-NEXT:    vpermt2q %zmm26, %zmm8, %zmm5
+; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm26, %zmm16, %zmm0
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm29
+; AVX512F-NEXT:    vpermt2q %zmm26, %zmm1, %zmm11
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm28
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm20 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm20, %zmm23, %zmm28
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm31
+; AVX512F-NEXT:    vmovdqa64 %zmm25, %zmm3
+; AVX512F-NEXT:    vpermi2q %zmm25, %zmm6, %zmm23
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm16, %zmm0 # 64-byte Folded Reload
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm0, %zmm9, %zmm8
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm8, %zmm1
-; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vinserti128 $1, 2432(%rdi), %ymm0, %ymm1
-; AVX512F-NEXT:    vmovdqa 2368(%rdi), %ymm0
-; AVX512F-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3,4,5],ymm1[6,7]
-; AVX512F-NEXT:    vmovdqa64 2304(%rdi), %zmm0
+; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm27
+; AVX512F-NEXT:    vpermt2q %zmm20, %zmm24, %zmm27
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm13, %zmm16, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 2240(%rdi), %zmm8
-; AVX512F-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm0, %zmm5, %zmm8
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm8[0,1,2,3],ymm1[4,5,6,7]
-; AVX512F-NEXT:    vmovdqa64 2560(%rdi), %zmm21
-; AVX512F-NEXT:    vmovdqa64 2496(%rdi), %zmm8
-; AVX512F-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm21, %zmm4, %zmm8
-; AVX512F-NEXT:    vmovdqa64 2624(%rdi), %zmm26
-; AVX512F-NEXT:    vpermt2q %zmm26, %zmm9, %zmm8
-; AVX512F-NEXT:    vmovdqu64 %zmm26, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm8, %zmm1
-; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vinserti128 $1, 1984(%rdi), %ymm0, %ymm1
-; AVX512F-NEXT:    vmovdqa 1920(%rdi), %ymm0
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3,4,5],ymm1[6,7]
-; AVX512F-NEXT:    vmovdqa64 %ymm0, %ymm29
-; AVX512F-NEXT:    vmovdqa64 1856(%rdi), %zmm17
-; AVX512F-NEXT:    vmovdqa64 1792(%rdi), %zmm8
-; AVX512F-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm17, %zmm5, %zmm8
-; AVX512F-NEXT:    vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3],ymm1[4,5,6,7]
-; AVX512F-NEXT:    vmovdqa64 2112(%rdi), %zmm1
-; AVX512F-NEXT:    vmovdqa64 2048(%rdi), %zmm11
-; AVX512F-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm4, %zmm11
-; AVX512F-NEXT:    vmovdqa64 2176(%rdi), %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm0, %zmm9, %zmm11
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm24
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm17, %zmm16, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm8, %zmm11, %zmm8
-; AVX512F-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vinserti128 $1, 3328(%rdi), %ymm0, %ymm8
-; AVX512F-NEXT:    vmovdqa 3264(%rdi), %ymm12
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm8 = ymm12[0,1,2,3,4,5],ymm8[6,7]
-; AVX512F-NEXT:    vmovdqa64 3200(%rdi), %zmm11
-; AVX512F-NEXT:    vmovdqa64 3136(%rdi), %zmm6
-; AVX512F-NEXT:    vpermi2q %zmm11, %zmm6, %zmm5
-; AVX512F-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm8 = ymm5[0,1,2,3],ymm8[4,5,6,7]
-; AVX512F-NEXT:    vmovdqa64 3456(%rdi), %zmm5
-; AVX512F-NEXT:    vmovdqa64 3392(%rdi), %zmm31
-; AVX512F-NEXT:    vpermi2q %zmm5, %zmm31, %zmm4
-; AVX512F-NEXT:    vmovdqa64 3520(%rdi), %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm0, %zmm9, %zmm4
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm23
+; AVX512F-NEXT:    vmovdqu64 (%rsp), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm26 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm26, %zmm16, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm8, %zmm4, %zmm4
-; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa 2880(%rdi), %ymm4
-; AVX512F-NEXT:    vmovdqa64 %ymm16, %ymm0
-; AVX512F-NEXT:    vpalignr {{.*#+}} ymm4 = ymm0[8,9,10,11,12,13,14,15],ymm4[0,1,2,3,4,5,6,7],ymm0[24,25,26,27,28,29,30,31],ymm4[16,17,18,19,20,21,22,23]
-; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm0 = [5,12]
-; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm15 # 64-byte Folded Reload
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm9 = ymm15[0,1,2,3],ymm4[4,5,6,7]
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [9,0,7,0,9,0,7,0]
-; AVX512F-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm18, %zmm16
-; AVX512F-NEXT:    vpermt2q %zmm22, %zmm4, %zmm16
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm8 = [4,5,6,14,4,5,6,14]
-; AVX512F-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm8, %zmm16 # 64-byte Folded Reload
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm9, %zmm16, %zmm9
-; AVX512F-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa 640(%rdi), %ymm9
-; AVX512F-NEXT:    vpalignr {{.*#+}} ymm2 = ymm2[8,9,10,11,12,13,14,15],ymm9[0,1,2,3,4,5,6,7],ymm2[24,25,26,27,28,29,30,31],ymm9[16,17,18,19,20,21,22,23]
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm28, %zmm0, %zmm9
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm9[0,1,2,3],ymm2[4,5,6,7]
-; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm9
-; AVX512F-NEXT:    vmovdqu64 (%rsp), %zmm15 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm15, %zmm4, %zmm9
-; AVX512F-NEXT:    vpermt2q %zmm7, %zmm8, %zmm9
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm9, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa 192(%rdi), %ymm2
-; AVX512F-NEXT:    vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload
-; AVX512F-NEXT:    # ymm2 = mem[8,9,10,11,12,13,14,15],ymm2[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm2[16,17,18,19,20,21,22,23]
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm30, %zmm0, %zmm7
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm7[0,1,2,3],ymm2[4,5,6,7]
-; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm7
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm9, %zmm4, %zmm7
-; AVX512F-NEXT:    vpermt2q %zmm27, %zmm8, %zmm7
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm7, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa 1536(%rdi), %ymm2
-; AVX512F-NEXT:    vpalignr {{.*#+}} ymm2 = ymm3[8,9,10,11,12,13,14,15],ymm2[0,1,2,3,4,5,6,7],ymm3[24,25,26,27,28,29,30,31],ymm2[16,17,18,19,20,21,22,23]
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm25, %zmm0, %zmm3
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
-; AVX512F-NEXT:    vmovdqa64 %zmm14, %zmm3
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm25 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm25, %zmm4, %zmm3
-; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm8, %zmm3 # 64-byte Folded Reload
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm3, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa 1088(%rdi), %ymm2
-; AVX512F-NEXT:    vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload
-; AVX512F-NEXT:    # ymm2 = mem[8,9,10,11,12,13,14,15],ymm2[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm2[16,17,18,19,20,21,22,23]
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm19, %zmm0, %zmm3
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
-; AVX512F-NEXT:    vmovdqa64 %zmm20, %zmm3
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm30 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm30, %zmm4, %zmm3
-; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm8, %zmm3 # 64-byte Folded Reload
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm3, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa 2432(%rdi), %ymm2
-; AVX512F-NEXT:    vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload
-; AVX512F-NEXT:    # ymm2 = mem[8,9,10,11,12,13,14,15],ymm2[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm2[16,17,18,19,20,21,22,23]
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm3 # 64-byte Folded Reload
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
-; AVX512F-NEXT:    vmovdqa64 %zmm21, %zmm3
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm19, %zmm4, %zmm3
-; AVX512F-NEXT:    vpermt2q %zmm26, %zmm8, %zmm3
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm3, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa 1984(%rdi), %ymm2
-; AVX512F-NEXT:    vmovdqa64 %ymm29, %ymm3
-; AVX512F-NEXT:    vpalignr {{.*#+}} ymm2 = ymm3[8,9,10,11,12,13,14,15],ymm2[0,1,2,3,4,5,6,7],ymm3[24,25,26,27,28,29,30,31],ymm2[16,17,18,19,20,21,22,23]
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm17, %zmm0, %zmm3
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm3
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm27 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm27, %zmm4, %zmm3
-; AVX512F-NEXT:    vpermt2q %zmm24, %zmm8, %zmm3
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm3, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa 3328(%rdi), %ymm2
-; AVX512F-NEXT:    vpalignr {{.*#+}} ymm2 = ymm12[8,9,10,11,12,13,14,15],ymm2[0,1,2,3,4,5,6,7],ymm12[24,25,26,27,28,29,30,31],ymm2[16,17,18,19,20,21,22,23]
-; AVX512F-NEXT:    vpermi2q %zmm11, %zmm6, %zmm0
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
-; AVX512F-NEXT:    vpermi2q %zmm31, %zmm5, %zmm4
-; AVX512F-NEXT:    vpermt2q %zmm23, %zmm8, %zmm4
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm4, %zmm0
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm4, %zmm16, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm26 = [10,3,10,3,10,3,10,3]
-; AVX512F-NEXT:    # zmm26 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm4
-; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm13, %zmm26, %zmm0
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm16, %zmm0 # 64-byte Folded Reload
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [11,4,11,4,11,4,11,4]
-; AVX512F-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm13, %zmm0, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm3
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm7 = [12,5,12,5,12,5,12,5]
-; AVX512F-NEXT:    # zmm7 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm13, %zmm7, %zmm0
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm16, %zmm6 # 64-byte Folded Reload
+; AVX512F-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm16, %zmm0 # 64-byte Folded Reload
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm12 = [13,6,13,6,13,6,13,6]
-; AVX512F-NEXT:    # zmm12 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpermi2q %zmm25, %zmm31, %zmm24
+; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm25
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm2
+; AVX512F-NEXT:    vpermt2q %zmm20, %zmm8, %zmm25
+; AVX512F-NEXT:    vpermi2q %zmm3, %zmm31, %zmm2
 ; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm13, %zmm12, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [2,9,2,9,2,9,2,9]
-; AVX512F-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermt2q %zmm13, %zmm0, %zmm4
-; AVX512F-NEXT:    vmovdqu64 %zmm4, (%rsp) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm17
-; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm10, %zmm26, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm10, %zmm3, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm10, %zmm7, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm10, %zmm12, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm10, %zmm0, %zmm17
-; AVX512F-NEXT:    vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm25, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm14, %zmm26, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm25, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm14, %zmm3, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm25, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm14, %zmm7, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm25, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm14, %zmm12, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm14, %zmm0, %zmm25
-; AVX512F-NEXT:    vmovdqu64 %zmm25, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm30, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm20, %zmm26, %zmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm13
-; AVX512F-NEXT:    vmovdqa64 %zmm30, %zmm24
-; AVX512F-NEXT:    vpermt2q %zmm20, %zmm3, %zmm24
-; AVX512F-NEXT:    vmovdqa64 %zmm30, %zmm25
-; AVX512F-NEXT:    vpermt2q %zmm20, %zmm7, %zmm25
-; AVX512F-NEXT:    vmovdqa64 %zmm30, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm20, %zmm12, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm20, %zmm0, %zmm30
-; AVX512F-NEXT:    vmovdqu64 %zmm30, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm19, %zmm15
-; AVX512F-NEXT:    vpermt2q %zmm21, %zmm26, %zmm15
-; AVX512F-NEXT:    vmovdqa64 %zmm19, %zmm18
-; AVX512F-NEXT:    vpermt2q %zmm21, %zmm3, %zmm18
-; AVX512F-NEXT:    vmovdqa64 %zmm19, %zmm22
-; AVX512F-NEXT:    vpermt2q %zmm21, %zmm7, %zmm22
-; AVX512F-NEXT:    vmovdqa64 %zmm19, %zmm23
-; AVX512F-NEXT:    vpermt2q %zmm21, %zmm12, %zmm23
-; AVX512F-NEXT:    vpermt2q %zmm21, %zmm0, %zmm19
-; AVX512F-NEXT:    vmovdqa64 %zmm27, %zmm9
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm26, %zmm9
-; AVX512F-NEXT:    vmovdqa64 %zmm27, %zmm11
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm3, %zmm11
-; AVX512F-NEXT:    vmovdqa64 %zmm27, %zmm17
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm7, %zmm17
-; AVX512F-NEXT:    vmovdqa64 %zmm27, %zmm8
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm12, %zmm8
-; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm21
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm0, %zmm27
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm20 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm20, %zmm4
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm26, %zmm4
-; AVX512F-NEXT:    vpermi2q %zmm5, %zmm31, %zmm26
-; AVX512F-NEXT:    vmovdqa64 %zmm20, %zmm30
-; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm3, %zmm30
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm16
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm12, %zmm1 # 64-byte Folded Reload
-; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm29 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm29, %zmm12, %zmm1
-; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm12, %zmm1 # 64-byte Folded Reload
-; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm12, %zmm1 # 64-byte Folded Reload
-; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm12, %zmm1 # 64-byte Folded Reload
-; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm12, %zmm1 # 64-byte Folded Reload
-; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm12, %zmm1 # 64-byte Folded Reload
-; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm31, %zmm8
-; AVX512F-NEXT:    vpermi2q %zmm5, %zmm31, %zmm2
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm12, %zmm1 # 64-byte Folded Reload
-; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm20, %zmm28
-; AVX512F-NEXT:    vpermt2q %zmm16, %zmm7, %zmm28
-; AVX512F-NEXT:    vpermi2q %zmm5, %zmm31, %zmm7
-; AVX512F-NEXT:    vmovdqa64 %zmm20, %zmm3
-; AVX512F-NEXT:    vpermt2q %zmm16, %zmm12, %zmm3
-; AVX512F-NEXT:    vpermi2q %zmm5, %zmm31, %zmm12
-; AVX512F-NEXT:    vpermt2q %zmm5, %zmm0, %zmm8
-; AVX512F-NEXT:    vpermt2q %zmm16, %zmm0, %zmm20
-; AVX512F-NEXT:    vmovdqa64 %zmm20, %zmm1
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [0,5,6,9,0,5,6,9]
-; AVX512F-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm10, %zmm0, %zmm5
-; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm20, %zmm16, %zmm0
+; AVX512F-NEXT:    vpermi2q %zmm3, %zmm31, %zmm16
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm1, %zmm31
+; AVX512F-NEXT:    vmovdqa64 %zmm31, %zmm3
+; AVX512F-NEXT:    vpermt2q %zmm20, %zmm1, %zmm15
+; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm31
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm1 = [0,5,6,9,0,5,6,9]
+; AVX512F-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm8, %zmm1, %zmm10
+; AVX512F-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm6, %zmm0, %zmm5
-; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm16, %zmm0, %zmm13
-; AVX512F-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm14, %zmm0, %zmm5
-; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm13, %zmm0, %zmm9
+; AVX512F-NEXT:    vpermt2q %zmm6, %zmm1, %zmm12
+; AVX512F-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm10, %zmm1, %zmm9
 ; AVX512F-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm31 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm31, %zmm0, %zmm15
-; AVX512F-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm5, %zmm0, %zmm4
-; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm9, %zmm1, %zmm14
+; AVX512F-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm15, %zmm0, %zmm26
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [0,5,6,10,0,5,6,10]
-; AVX512F-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermt2q %zmm5, %zmm0, %zmm30
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm20 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm6, %zmm0, %zmm20
-; AVX512F-NEXT:    vmovdqu64 %zmm20, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm20 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm10, %zmm0, %zmm20
-; AVX512F-NEXT:    vmovdqu64 %zmm20, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm15, %zmm1, %zmm30
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm14, %zmm1, %zmm7
+; AVX512F-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm7, %zmm1, %zmm28
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm20 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm14, %zmm0, %zmm20
-; AVX512F-NEXT:    vmovdqu64 %zmm20, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm16, %zmm0, %zmm24
-; AVX512F-NEXT:    vmovdqu64 %zmm24, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm31, %zmm0, %zmm18
+; AVX512F-NEXT:    vpermt2q %zmm20, %zmm1, %zmm23
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm1 = [0,5,6,10,0,5,6,10]
+; AVX512F-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpermt2q %zmm7, %zmm1, %zmm27
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm6, %zmm1, %zmm12
+; AVX512F-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm8, %zmm1, %zmm12
+; AVX512F-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm9, %zmm1, %zmm18
 ; AVX512F-NEXT:    vmovdqu64 %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm13, %zmm0, %zmm11
-; AVX512F-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm15, %zmm0, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [0,5,6,11,0,5,6,11]
-; AVX512F-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermt2q %zmm5, %zmm0, %zmm28
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm6, %zmm0, %zmm4
-; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm10, %zmm0, %zmm9
-; AVX512F-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm14, %zmm0, %zmm9
-; AVX512F-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm16, %zmm0, %zmm25
-; AVX512F-NEXT:    vmovdqu64 %zmm25, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm31, %zmm0, %zmm22
-; AVX512F-NEXT:    vmovdqu64 %zmm22, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm13, %zmm0, %zmm17
-; AVX512F-NEXT:    vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm15, %zmm0, %zmm7
-; AVX512F-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [0,5,6,12,0,5,6,12]
-; AVX512F-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermt2q %zmm5, %zmm0, %zmm3
-; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm6, %zmm0, %zmm4
-; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm10, %zmm0, %zmm4
-; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm14, %zmm0, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm16, %zmm0, %zmm4
-; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm31, %zmm0, %zmm23
-; AVX512F-NEXT:    vmovdqu64 %zmm23, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm13, %zmm0, %zmm21
+; AVX512F-NEXT:    vpermt2q %zmm10, %zmm1, %zmm19
+; AVX512F-NEXT:    vmovdqu64 %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm14, %zmm1, %zmm21
 ; AVX512F-NEXT:    vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm15, %zmm0, %zmm12
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [4,5,8,15,4,5,8,15]
-; AVX512F-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqu64 (%rsp), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm6, %zmm0, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, (%rsp) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm10, %zmm0, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm14, %zmm0, %zmm2
+; AVX512F-NEXT:    vpermt2q %zmm15, %zmm1, %zmm22
+; AVX512F-NEXT:    vmovdqu64 %zmm22, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm20, %zmm1, %zmm24
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm1 = [0,5,6,11,0,5,6,11]
+; AVX512F-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpermt2q %zmm7, %zmm1, %zmm25
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm6, %zmm1, %zmm12
+; AVX512F-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm8, %zmm1, %zmm12
+; AVX512F-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm9, %zmm1, %zmm12
+; AVX512F-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm10, %zmm1, %zmm12
+; AVX512F-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm14, %zmm1, %zmm12
+; AVX512F-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm15, %zmm1, %zmm5
+; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm20, %zmm1, %zmm2
 ; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm1 = [0,5,6,12,0,5,6,12]
+; AVX512F-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpermt2q %zmm7, %zmm1, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm6, %zmm1, %zmm12
+; AVX512F-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm8, %zmm1, %zmm12
+; AVX512F-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm9, %zmm1, %zmm12
+; AVX512F-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm10, %zmm1, %zmm12
+; AVX512F-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm14, %zmm1, %zmm12
+; AVX512F-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm15, %zmm1, %zmm29
+; AVX512F-NEXT:    vmovdqu64 %zmm29, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm20, %zmm1, %zmm16
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm1 = [4,5,8,15,4,5,8,15]
+; AVX512F-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm6, %zmm1, %zmm18
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm8, %zmm1, %zmm19
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm21 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm9, %zmm1, %zmm21
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm22 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm10, %zmm1, %zmm22
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm16, %zmm0, %zmm2
+; AVX512F-NEXT:    vpermt2q %zmm14, %zmm1, %zmm2
 ; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm31, %zmm0, %zmm19
-; AVX512F-NEXT:    vmovdqu64 %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm13, %zmm0, %zmm27
-; AVX512F-NEXT:    vmovdqu64 %zmm27, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm15, %zmm0, %zmm8
-; AVX512F-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm5, %zmm0, %zmm1
-; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm15, %zmm1, %zmm11
+; AVX512F-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm20, %zmm1, %zmm3
+; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm7, %zmm1, %zmm31
+; AVX512F-NEXT:    vmovdqu64 %zmm31, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    movb $24, %al
-; AVX512F-NEXT:    kmovw %eax, %k1
-; AVX512F-NEXT:    vmovdqa64 {{.*#+}} ymm23 = <0,7,14,u>
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm21 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm16, %zmm23, %zmm21
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm29, %zmm17
-; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm21 {%k1} = zmm13[4,5,4,5],zmm29[4,5,4,5]
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm29 = [7,0,9,0,7,0,9,0]
-; AVX512F-NEXT:    # zmm29 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm17, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm13, %zmm29, %zmm0
+; AVX512F-NEXT:    kmovw %eax, %k2
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm12 {%k2} = zmm0[4,5,4,5],zmm13[4,5,4,5]
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm15 = [7,0,9,0,7,0,9,0]
+; AVX512F-NEXT:    # zmm15 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm20
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm15, %zmm20
+; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [4,11,4,11]
+; AVX512F-NEXT:    # ymm3 = mem[0,1,0,1]
+; AVX512F-NEXT:    vpermt2q %zmm13, %zmm3, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm6 = [4,11,4,11]
-; AVX512F-NEXT:    # ymm6 = mem[0,1,0,1]
-; AVX512F-NEXT:    vpermt2q %zmm17, %zmm6, %zmm13
-; AVX512F-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm24 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm20 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm20, %zmm23, %zmm24
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm25 # 64-byte Reload
-; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm24 {%k1} = zmm1[4,5,4,5],zmm25[4,5,4,5]
-; AVX512F-NEXT:    vmovdqa64 %zmm25, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm29, %zmm0
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
+; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm10 {%k2} = zmm0[4,5,4,5],zmm17[4,5,4,5]
+; AVX512F-NEXT:    vmovdqa64 %zmm17, %zmm29
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm15, %zmm29
+; AVX512F-NEXT:    vpermt2q %zmm17, %zmm3, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm25, %zmm6, %zmm1
-; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm31 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 (%rsp), %zmm0 # 64-byte Reload
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm9, %zmm23, %zmm31
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
-; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm31 {%k1} = zmm1[4,5,4,5],zmm17[4,5,4,5]
-; AVX512F-NEXT:    vmovdqa64 %zmm17, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm29, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm17, %zmm6, %zmm1
-; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm25 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm7, %zmm23, %zmm25
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm25 {%k1} = zmm1[4,5,4,5],zmm5[4,5,4,5]
-; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm29, %zmm0
+; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm9 {%k2} = zmm0[4,5,4,5],zmm26[4,5,4,5]
+; AVX512F-NEXT:    vmovdqa64 %zmm26, %zmm17
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm15, %zmm17
+; AVX512F-NEXT:    vpermt2q %zmm26, %zmm3, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, (%rsp) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm8 {%k2} = zmm2[4,5,4,5],zmm4[4,5,4,5]
+; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm15, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm5, %zmm6, %zmm1
-; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm18, %zmm23, %zmm19
+; AVX512F-NEXT:    vpermt2q %zmm4, %zmm3, %zmm2
+; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm19 {%k1} = zmm1[4,5,4,5],zmm3[4,5,4,5]
-; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm29, %zmm0
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm7 {%k2} = zmm0[4,5,4,5],zmm1[4,5,4,5]
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm2
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm15, %zmm2
+; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm3, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm3, %zmm6, %zmm1
-; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm17
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm14, %zmm23, %zmm17
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm17 {%k1} = zmm1[4,5,4,5],zmm3[4,5,4,5]
-; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm29, %zmm0
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm6 {%k2} = zmm0[4,5,4,5],zmm1[4,5,4,5]
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm2
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm15, %zmm2
+; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm3, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm3, %zmm6, %zmm1
-; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm11
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm0, %zmm23, %zmm11
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
-; AVX512F-NEXT:    vpermi2q %zmm13, %zmm10, %zmm23
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm31 # 64-byte Reload
+; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm31 {%k2} = zmm0[4,5,4,5],zmm2[4,5,4,5]
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm23 {%k1} = zmm2[4,5,4,5],zmm4[4,5,4,5]
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm22
+; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm26
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm29, %zmm22
-; AVX512F-NEXT:    vpermi2q %zmm2, %zmm4, %zmm29
-; AVX512F-NEXT:    vpermt2q %zmm4, %zmm6, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm11 {%k1} = zmm1[4,5,4,5],zmm5[4,5,4,5]
-; AVX512F-NEXT:    vpermt2q %zmm5, %zmm6, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm15, %zmm26
+; AVX512F-NEXT:    vpermi2q %zmm0, %zmm2, %zmm15
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm3, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm0
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm4 {%k2} = zmm1[4,5,4,5],zmm0[4,5,4,5]
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm3, %zmm1
 ; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm3 = <9,0,7,u>
-; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm4
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm2, %zmm3, %zmm16
-; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm1 = [6,13]
-; AVX512F-NEXT:    vpermt2q %zmm4, %zmm1, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm20, %zmm4
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm2, %zmm3, %zmm20
-; AVX512F-NEXT:    vpermt2q %zmm4, %zmm1, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm4
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm2, %zmm3, %zmm9
-; AVX512F-NEXT:    vpermt2q %zmm4, %zmm1, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm4
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm2, %zmm3, %zmm7
-; AVX512F-NEXT:    vpermt2q %zmm4, %zmm1, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm18, %zmm6
+; AVX512F-NEXT:    vmovdqa {{.*#+}} xmm0 = [6,13]
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm1 # 64-byte Folded Reload
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm2, %zmm3, %zmm6
-; AVX512F-NEXT:    vpermt2q %zmm18, %zmm1, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm14, %zmm5
-; AVX512F-NEXT:    vpermt2q %zmm15, %zmm3, %zmm5
-; AVX512F-NEXT:    vpermt2q %zmm14, %zmm1, %zmm15
-; AVX512F-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm4
-; AVX512F-NEXT:    vpermt2q %zmm8, %zmm3, %zmm4
-; AVX512F-NEXT:    vpermi2q %zmm10, %zmm13, %zmm3
-; AVX512F-NEXT:    vpermt2q %zmm13, %zmm1, %zmm10
-; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm14
-; AVX512F-NEXT:    vpermt2q %zmm0, %zmm1, %zmm8
-; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm13
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 32-byte Folded Reload
-; AVX512F-NEXT:    # ymm1 = ymm0[0,1,2,3],mem[4,5,6,7]
-; AVX512F-NEXT:    vmovups (%rsp), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vinsertf64x4 $0, %ymm1, %zmm0, %zmm0
-; AVX512F-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 32-byte Folded Reload
-; AVX512F-NEXT:    # ymm1 = ymm0[0,1,2,3],mem[4,5,6,7]
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vinsertf64x4 $0, %ymm1, %zmm0, %zmm0
-; AVX512F-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 32-byte Folded Reload
-; AVX512F-NEXT:    # ymm1 = ymm0[0,1,2,3],mem[4,5,6,7]
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vinsertf64x4 $0, %ymm1, %zmm0, %zmm0
-; AVX512F-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 32-byte Folded Reload
-; AVX512F-NEXT:    # ymm1 = ymm0[0,1,2,3],mem[4,5,6,7]
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vinsertf64x4 $0, %ymm1, %zmm0, %zmm0
-; AVX512F-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 # 64-byte Folded Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm3 # 64-byte Folded Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm5 # 64-byte Folded Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm11 # 64-byte Folded Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm13 # 64-byte Folded Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm14 # 64-byte Folded Reload
+; AVX512F-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm14 # 64-byte Folded Reload
+; AVX512F-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm0 # 32-byte Folded Reload
+; AVX512F-NEXT:    # ymm0 = ymm1[0,1,2,3],mem[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm18, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm0 # 32-byte Folded Reload
+; AVX512F-NEXT:    # ymm0 = ymm2[0,1,2,3],mem[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm19, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpblendd $240, (%rsp), %ymm3, %ymm0 # 32-byte Folded Reload
+; AVX512F-NEXT:    # ymm0 = ymm3[0,1,2,3],mem[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm21, %zmm21
+; AVX512F-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm0 # 32-byte Folded Reload
+; AVX512F-NEXT:    # ymm0 = ymm5[0,1,2,3],mem[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm22, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, (%rsp) # 64-byte Spill
 ; AVX512F-NEXT:    movb $-32, %al
-; AVX512F-NEXT:    kmovw %eax, %k2
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm24 {%k2}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm21 {%k2}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm25 {%k2}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm31 {%k2}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm17 {%k2}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm19 {%k2}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm11 {%k2}
-; AVX512F-NEXT:    vmovdqa64 %zmm26, %zmm23 {%k2}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm4 {%k1}
-; AVX512F-NEXT:    vmovdqa64 %zmm30, %zmm4 {%k2}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm16 {%k1}
+; AVX512F-NEXT:    kmovw %eax, %k1
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm16 {%k2}
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm10 {%k1}
+; AVX512F-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm20 {%k1}
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm12 {%k1}
+; AVX512F-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm20 {%k2}
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm8 {%k1}
+; AVX512F-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm9 {%k1}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm9 {%k2}
+; AVX512F-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm30, %zmm6 {%k1}
+; AVX512F-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm7 {%k1}
+; AVX512F-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm28, %zmm4 {%k1}
+; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm23, %zmm31 {%k1}
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm7 {%k2}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm6 {%k1}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm6 {%k2}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm5 {%k1}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm5 {%k2}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm3 {%k1}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm3 {%k2}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm28, %zmm0 {%k2}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm0 {%k2}
+; AVX512F-NEXT:    vmovdqa64 %zmm27, %zmm0 {%k1}
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm30 # 64-byte Reload
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm30 {%k2}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm27 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm27 {%k2}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm10 {%k2}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm18 {%k2}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm0 {%k2}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm0 {%k1}
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm8 {%k2}
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm0 {%k2}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm0 {%k1}
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k2}
-; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm0 {%k2}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm0 {%k1}
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k2}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
 ; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 32-byte Folded Reload
-; AVX512F-NEXT:    # ymm1 = ymm0[0,1,2,3],mem[4,5,6,7]
-; AVX512F-NEXT:    vmovdqa 2752(%rdi), %ymm15
-; AVX512F-NEXT:    vpalignr {{.*#+}} ymm15 = mem[8,9,10,11,12,13,14,15],ymm15[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm15[16,17,18,19,20,21,22,23]
-; AVX512F-NEXT:    vextracti128 $1, %ymm15, %xmm15
-; AVX512F-NEXT:    vinserti32x4 $0, %xmm15, %zmm22, %zmm28
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm23 # 64-byte Reload
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm28 {%k2}
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vinsertf64x4 $0, %ymm1, %zmm0, %zmm0
-; AVX512F-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa 512(%rdi), %ymm15
-; AVX512F-NEXT:    vpalignr {{.*#+}} ymm15 = mem[8,9,10,11,12,13,14,15],ymm15[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm15[16,17,18,19,20,21,22,23]
-; AVX512F-NEXT:    vextracti128 $1, %ymm15, %xmm15
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm23 {%k2}
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vinserti32x4 $0, %xmm15, %zmm0, %zmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm23 {%k1}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k2}
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm15 # 32-byte Folded Reload
-; AVX512F-NEXT:    # ymm15 = ymm0[0,1,2,3],mem[4,5,6,7]
-; AVX512F-NEXT:    vmovdqa 64(%rdi), %ymm1
-; AVX512F-NEXT:    vpalignr {{.*#+}} ymm1 = mem[8,9,10,11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23]
-; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm1
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vinserti32x4 $0, %xmm1, %zmm0, %zmm22
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k1}
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm22 {%k2}
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vinsertf64x4 $0, %ymm15, %zmm0, %zmm0
-; AVX512F-NEXT:    vmovups %zmm0, (%rsp) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa 1408(%rdi), %ymm15
-; AVX512F-NEXT:    vpalignr {{.*#+}} ymm15 = mem[8,9,10,11,12,13,14,15],ymm15[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm15[16,17,18,19,20,21,22,23]
-; AVX512F-NEXT:    vextracti128 $1, %ymm15, %xmm15
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm0 {%k2}
+; AVX512F-NEXT:    vmovdqa64 %zmm24, %zmm0 {%k1}
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm18
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm25, %zmm7 {%k1}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm22 # 64-byte Reload
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vinserti32x4 $0, %xmm15, %zmm0, %zmm26
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm22 {%k1}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm26 {%k2}
-; AVX512F-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm1 # 32-byte Folded Reload
-; AVX512F-NEXT:    # ymm1 = ymm14[0,1,2,3],mem[4,5,6,7]
-; AVX512F-NEXT:    vmovdqa 960(%rdi), %ymm14
-; AVX512F-NEXT:    vpalignr {{.*#+}} ymm14 = mem[8,9,10,11,12,13,14,15],ymm14[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm14[16,17,18,19,20,21,22,23]
-; AVX512F-NEXT:    vextracti128 $1, %ymm14, %xmm14
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm19 {%k1}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm24 # 64-byte Reload
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vinserti32x4 $0, %xmm14, %zmm0, %zmm14
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm24 {%k1}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm30 # 64-byte Reload
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm14 {%k2}
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm30 {%k1}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm27 # 64-byte Reload
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa 2304(%rdi), %ymm1
-; AVX512F-NEXT:    vpalignr {{.*#+}} ymm1 = mem[8,9,10,11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23]
-; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm27 {%k1}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm28 # 64-byte Reload
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vinserti32x4 $0, %xmm1, %zmm0, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm28 {%k1}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm25 # 64-byte Reload
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k2}
-; AVX512F-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm13, %ymm15 # 32-byte Folded Reload
-; AVX512F-NEXT:    # ymm15 = ymm13[0,1,2,3],mem[4,5,6,7]
-; AVX512F-NEXT:    vmovdqa 1856(%rdi), %ymm13
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm25 {%k1}
+; AVX512F-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm11, %ymm0 # 32-byte Folded Reload
+; AVX512F-NEXT:    # ymm0 = ymm11[0,1,2,3],mem[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa 2752(%rdi), %ymm3
+; AVX512F-NEXT:    vpalignr {{.*#+}} ymm3 = mem[8,9,10,11,12,13,14,15],ymm3[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm3[16,17,18,19,20,21,22,23]
+; AVX512F-NEXT:    vextracti128 $1, %ymm3, %xmm3
+; AVX512F-NEXT:    vinserti32x4 $0, %xmm3, %zmm26, %zmm3
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm3 {%k1}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm1, %zmm0
+; AVX512F-NEXT:    vmovdqa 512(%rdi), %ymm1
+; AVX512F-NEXT:    vpalignr {{.*#+}} ymm1 = mem[8,9,10,11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23]
+; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm1
+; AVX512F-NEXT:    vinserti32x4 $0, %xmm1, %zmm20, %zmm6
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm6 {%k1}
+; AVX512F-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm13, %ymm1 # 32-byte Folded Reload
+; AVX512F-NEXT:    # ymm1 = ymm13[0,1,2,3],mem[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa 64(%rdi), %ymm4
+; AVX512F-NEXT:    vpalignr {{.*#+}} ymm4 = mem[8,9,10,11,12,13,14,15],ymm4[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm4[16,17,18,19,20,21,22,23]
+; AVX512F-NEXT:    vextracti128 $1, %ymm4, %xmm4
+; AVX512F-NEXT:    vinserti32x4 $0, %xmm4, %zmm29, %zmm14
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm14 {%k1}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm4, %zmm1
+; AVX512F-NEXT:    vmovdqa 1408(%rdi), %ymm4
+; AVX512F-NEXT:    vpalignr {{.*#+}} ymm4 = mem[8,9,10,11,12,13,14,15],ymm4[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm4[16,17,18,19,20,21,22,23]
+; AVX512F-NEXT:    vextracti128 $1, %ymm4, %xmm4
+; AVX512F-NEXT:    vinserti32x4 $0, %xmm4, %zmm17, %zmm5
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm5 {%k1}
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512F-NEXT:    vblendps $15, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm4 # 32-byte Folded Reload
+; AVX512F-NEXT:    # ymm4 = mem[0,1,2,3],ymm4[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa 960(%rdi), %ymm8
+; AVX512F-NEXT:    vpalignr {{.*#+}} ymm8 = mem[8,9,10,11,12,13,14,15],ymm8[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm8[16,17,18,19,20,21,22,23]
+; AVX512F-NEXT:    vextracti128 $1, %ymm8, %xmm8
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
+; AVX512F-NEXT:    vinserti32x4 $0, %xmm8, %zmm9, %zmm8
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm8 {%k1}
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
+; AVX512F-NEXT:    vinsertf64x4 $0, %ymm4, %zmm9, %zmm4
+; AVX512F-NEXT:    vmovdqa 2304(%rdi), %ymm10
+; AVX512F-NEXT:    vpalignr {{.*#+}} ymm10 = mem[8,9,10,11,12,13,14,15],ymm10[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm10[16,17,18,19,20,21,22,23]
+; AVX512F-NEXT:    vextracti128 $1, %ymm10, %xmm10
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
+; AVX512F-NEXT:    vinserti32x4 $0, %xmm10, %zmm9, %zmm10
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm10 {%k1}
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
+; AVX512F-NEXT:    vblendps $15, {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm11 # 32-byte Folded Reload
+; AVX512F-NEXT:    # ymm11 = mem[0,1,2,3],ymm9[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa 1856(%rdi), %ymm12
+; AVX512F-NEXT:    vpalignr {{.*#+}} ymm12 = mem[8,9,10,11,12,13,14,15],ymm12[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm12[16,17,18,19,20,21,22,23]
+; AVX512F-NEXT:    vextracti128 $1, %ymm12, %xmm12
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
+; AVX512F-NEXT:    vinserti32x4 $0, %xmm12, %zmm9, %zmm12
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm12 {%k1}
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
+; AVX512F-NEXT:    vinsertf64x4 $0, %ymm11, %zmm9, %zmm11
+; AVX512F-NEXT:    vmovdqa 3200(%rdi), %ymm13
 ; AVX512F-NEXT:    vpalignr {{.*#+}} ymm13 = mem[8,9,10,11,12,13,14,15],ymm13[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm13[16,17,18,19,20,21,22,23]
 ; AVX512F-NEXT:    vextracti128 $1, %ymm13, %xmm13
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vinserti32x4 $0, %xmm13, %zmm0, %zmm13
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm13 {%k2}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm15, %zmm0, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa 3200(%rdi), %ymm15
-; AVX512F-NEXT:    vpalignr {{.*#+}} ymm15 = mem[8,9,10,11,12,13,14,15],ymm15[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm15[16,17,18,19,20,21,22,23]
-; AVX512F-NEXT:    vextracti128 $1, %ymm15, %xmm15
-; AVX512F-NEXT:    vinserti32x4 $0, %xmm15, %zmm29, %zmm15
-; AVX512F-NEXT:    vmovdqa64 %zmm12, %zmm15 {%k2}
-; AVX512F-NEXT:    vmovdqa64 %zmm23, 448(%rsi)
-; AVX512F-NEXT:    vmovdqa64 %zmm11, 384(%rsi)
-; AVX512F-NEXT:    vmovdqa64 %zmm19, 320(%rsi)
-; AVX512F-NEXT:    vmovdqa64 %zmm17, 256(%rsi)
-; AVX512F-NEXT:    vmovdqa64 %zmm31, 192(%rsi)
-; AVX512F-NEXT:    vmovdqa64 %zmm25, 128(%rsi)
-; AVX512F-NEXT:    vmovdqa64 %zmm21, 64(%rsi)
-; AVX512F-NEXT:    vmovdqa64 %zmm24, (%rsi)
-; AVX512F-NEXT:    vmovdqa64 %zmm3, 448(%rdx)
-; AVX512F-NEXT:    vmovdqa64 %zmm5, 256(%rdx)
-; AVX512F-NEXT:    vmovdqa64 %zmm6, 320(%rdx)
-; AVX512F-NEXT:    vmovdqa64 %zmm7, 128(%rdx)
-; AVX512F-NEXT:    vmovdqa64 %zmm9, 192(%rdx)
-; AVX512F-NEXT:    vmovdqa64 %zmm20, (%rdx)
-; AVX512F-NEXT:    vmovdqa64 %zmm16, 64(%rdx)
-; AVX512F-NEXT:    vmovdqa64 %zmm4, 384(%rdx)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 448(%rcx)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 256(%rcx)
-; AVX512F-NEXT:    vmovdqa64 %zmm8, 320(%rcx)
-; AVX512F-NEXT:    vmovdqa64 %zmm18, 128(%rcx)
-; AVX512F-NEXT:    vmovdqa64 %zmm10, 192(%rcx)
-; AVX512F-NEXT:    vmovdqa64 %zmm27, (%rcx)
-; AVX512F-NEXT:    vmovdqa64 %zmm30, 64(%rcx)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 384(%rcx)
-; AVX512F-NEXT:    vmovdqa64 %zmm15, 448(%r8)
-; AVX512F-NEXT:    vmovdqa64 %zmm13, 256(%r8)
-; AVX512F-NEXT:    vmovdqa64 %zmm1, 320(%r8)
-; AVX512F-NEXT:    vmovdqa64 %zmm14, 128(%r8)
-; AVX512F-NEXT:    vmovdqa64 %zmm26, 192(%r8)
-; AVX512F-NEXT:    vmovdqa64 %zmm22, (%r8)
-; AVX512F-NEXT:    vmovdqa64 %zmm2, 64(%r8)
-; AVX512F-NEXT:    vmovdqa64 %zmm28, 384(%r8)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 448(%r9)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 256(%r9)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 320(%r9)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 128(%r9)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 192(%r9)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, (%r9)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 64(%r9)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 384(%r9)
+; AVX512F-NEXT:    vinserti32x4 $0, %xmm13, %zmm15, %zmm9
+; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm9 {%k1}
+; AVX512F-NEXT:    vmovdqa64 %zmm31, 448(%rsi)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm13, 384(%rsi)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm13, 320(%rsi)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm13, 256(%rsi)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm13, 192(%rsi)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm13, 128(%rsi)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm13, 64(%rsi)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm13, (%rsi)
+; AVX512F-NEXT:    vmovdqa64 %zmm18, 448(%rdx)
+; AVX512F-NEXT:    vmovdqa64 %zmm2, 256(%rdx)
+; AVX512F-NEXT:    vmovdqa64 %zmm23, 320(%rdx)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm2, 128(%rdx)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm2, 192(%rdx)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm2, (%rdx)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm2, 64(%rdx)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm2, 384(%rdx)
+; AVX512F-NEXT:    vmovdqa64 %zmm25, 448(%rcx)
+; AVX512F-NEXT:    vmovdqa64 %zmm28, 256(%rcx)
+; AVX512F-NEXT:    vmovdqa64 %zmm27, 320(%rcx)
+; AVX512F-NEXT:    vmovdqa64 %zmm30, 128(%rcx)
+; AVX512F-NEXT:    vmovdqa64 %zmm24, 192(%rcx)
+; AVX512F-NEXT:    vmovdqa64 %zmm19, (%rcx)
+; AVX512F-NEXT:    vmovdqa64 %zmm22, 64(%rcx)
+; AVX512F-NEXT:    vmovdqa64 %zmm7, 384(%rcx)
+; AVX512F-NEXT:    vmovdqa64 %zmm9, 448(%r8)
+; AVX512F-NEXT:    vmovdqa64 %zmm12, 256(%r8)
+; AVX512F-NEXT:    vmovdqa64 %zmm10, 320(%r8)
+; AVX512F-NEXT:    vmovdqa64 %zmm8, 128(%r8)
+; AVX512F-NEXT:    vmovdqa64 %zmm5, 192(%r8)
+; AVX512F-NEXT:    vmovdqa64 %zmm14, (%r8)
+; AVX512F-NEXT:    vmovdqa64 %zmm6, 64(%r8)
+; AVX512F-NEXT:    vmovdqa64 %zmm3, 384(%r8)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm3, 448(%r9)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm3, 256(%r9)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm3, 320(%r9)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm3, 128(%r9)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm3, 192(%r9)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm3, (%r9)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm3, 64(%r9)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm3, 384(%r9)
 ; AVX512F-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 448(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 256(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 320(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 128(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 192(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, (%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 64(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 384(%rax)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm2, 448(%rax)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm2, 256(%rax)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm2, 320(%rax)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm2, 128(%rax)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm2, 192(%rax)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm2, (%rax)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm2, 64(%rax)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm3, 384(%rax)
 ; AVX512F-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 384(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 448(%rax)
+; AVX512F-NEXT:    vmovaps %zmm11, 384(%rax)
+; AVX512F-NEXT:    vmovaps %zmm4, 448(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm1, 256(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm0, 320(%rax)
 ; AVX512F-NEXT:    vmovups (%rsp), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 256(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 320(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-NEXT:    vmovaps %zmm0, 128(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 192(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm21, 192(%rax)
 ; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-NEXT:    vmovaps %zmm0, (%rax)
 ; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-NEXT:    vmovaps %zmm0, 64(%rax)
-; AVX512F-NEXT:    addq $6728, %rsp # imm = 0x1A48
+; AVX512F-NEXT:    addq $7624, %rsp # imm = 0x1DC8
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: load_i64_stride7_vf64:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    subq $6664, %rsp # imm = 0x1A08
-; AVX512BW-NEXT:    vmovdqa64 3328(%rdi), %zmm17
-; AVX512BW-NEXT:    vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 3264(%rdi), %zmm4
+; AVX512BW-NEXT:    subq $7624, %rsp # imm = 0x1DC8
+; AVX512BW-NEXT:    vmovdqa64 3328(%rdi), %zmm4
 ; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 3008(%rdi), %zmm19
+; AVX512BW-NEXT:    vmovdqa64 3264(%rdi), %zmm6
+; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 3008(%rdi), %zmm25
 ; AVX512BW-NEXT:    vmovdqa64 2944(%rdi), %zmm18
-; AVX512BW-NEXT:    vmovdqu64 %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vmovdqa64 2880(%rdi), %zmm2
 ; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vmovdqa64 2816(%rdi), %zmm1
 ; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 2752(%rdi), %zmm20
-; AVX512BW-NEXT:    vmovdqa64 2688(%rdi), %zmm15
-; AVX512BW-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 2432(%rdi), %zmm5
+; AVX512BW-NEXT:    vmovdqa64 2752(%rdi), %zmm30
+; AVX512BW-NEXT:    vmovdqa64 2688(%rdi), %zmm5
 ; AVX512BW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 2368(%rdi), %zmm6
-; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 1984(%rdi), %zmm8
+; AVX512BW-NEXT:    vmovdqa64 2432(%rdi), %zmm19
+; AVX512BW-NEXT:    vmovdqu64 %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 2368(%rdi), %zmm8
 ; AVX512BW-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 1920(%rdi), %zmm9
+; AVX512BW-NEXT:    vmovdqa64 1984(%rdi), %zmm9
 ; AVX512BW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 1920(%rdi), %zmm3
+; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vmovdqa64 1536(%rdi), %zmm10
 ; AVX512BW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 1472(%rdi), %zmm11
+; AVX512BW-NEXT:    vmovdqa64 1472(%rdi), %zmm12
+; AVX512BW-NEXT:    vmovdqu64 %zmm12, (%rsp) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 1088(%rdi), %zmm11
 ; AVX512BW-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 1088(%rdi), %zmm12
-; AVX512BW-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vmovdqa64 1024(%rdi), %zmm13
 ; AVX512BW-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vmovdqa64 640(%rdi), %zmm16
 ; AVX512BW-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 576(%rdi), %zmm7
-; AVX512BW-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 192(%rdi), %zmm14
+; AVX512BW-NEXT:    vmovdqa64 576(%rdi), %zmm14
 ; AVX512BW-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 128(%rdi), %zmm3
-; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 192(%rdi), %zmm17
+; AVX512BW-NEXT:    vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 128(%rdi), %zmm15
+; AVX512BW-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [14,0,0,7,14,0,0,7]
 ; AVX512BW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
 ; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm0, %zmm1
@@ -9034,867 +9130,891 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm2 = xmm2[0,1],mem[2,3]
 ; AVX512BW-NEXT:    vinserti32x4 $0, %xmm2, %zmm1, %zmm1
 ; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm1
 ; AVX512BW-NEXT:    vpermt2q %zmm16, %zmm0, %zmm1
 ; AVX512BW-NEXT:    vmovdqa 464(%rdi), %xmm2
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm2 = xmm2[0,1],mem[2,3]
 ; AVX512BW-NEXT:    vinserti32x4 $0, %xmm2, %zmm1, %zmm1
 ; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm0, %zmm3
+; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm0, %zmm1
 ; AVX512BW-NEXT:    vmovdqa 16(%rdi), %xmm2
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm2 = xmm2[0,1],mem[2,3]
-; AVX512BW-NEXT:    vinserti32x4 $0, %xmm2, %zmm3, %zmm1
+; AVX512BW-NEXT:    vinserti32x4 $0, %xmm2, %zmm1, %zmm1
 ; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm1
 ; AVX512BW-NEXT:    vpermt2q %zmm10, %zmm0, %zmm1
 ; AVX512BW-NEXT:    vmovdqa 1360(%rdi), %xmm2
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm2 = xmm2[0,1],mem[2,3]
 ; AVX512BW-NEXT:    vinserti32x4 $0, %xmm2, %zmm1, %zmm1
 ; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm1
-; AVX512BW-NEXT:    vpermt2q %zmm12, %zmm0, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm0, %zmm1
 ; AVX512BW-NEXT:    vmovdqa 912(%rdi), %xmm2
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm2 = xmm2[0,1],mem[2,3]
 ; AVX512BW-NEXT:    vinserti32x4 $0, %xmm2, %zmm1, %zmm1
 ; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm1
-; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm0, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm0, %zmm1
 ; AVX512BW-NEXT:    vmovdqa 2256(%rdi), %xmm2
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm2 = xmm2[0,1],mem[2,3]
 ; AVX512BW-NEXT:    vinserti32x4 $0, %xmm2, %zmm1, %zmm1
 ; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm1
-; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm0, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm0, %zmm1
 ; AVX512BW-NEXT:    vmovdqa 1808(%rdi), %xmm2
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm2 = xmm2[0,1],mem[2,3]
 ; AVX512BW-NEXT:    vinserti32x4 $0, %xmm2, %zmm1, %zmm1
 ; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermi2q %zmm17, %zmm4, %zmm0
+; AVX512BW-NEXT:    vpermi2q %zmm4, %zmm6, %zmm0
 ; AVX512BW-NEXT:    vmovdqa 3152(%rdi), %xmm1
 ; AVX512BW-NEXT:    vpblendd {{.*#+}} xmm1 = xmm1[0,1],mem[2,3]
 ; AVX512BW-NEXT:    vinserti32x4 $0, %xmm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vinserti128 $1, 2880(%rdi), %ymm0, %ymm1
-; AVX512BW-NEXT:    vmovdqa 2816(%rdi), %ymm0
-; AVX512BW-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm0[0,1,2,3,4,5],ymm1[6,7]
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm5 = [4,11]
-; AVX512BW-NEXT:    vmovdqu64 %zmm20, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm5, %zmm15
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm15[0,1,2,3],ymm1[4,5,6,7]
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [0,7,14,0,0,7,14,0]
+; AVX512BW-NEXT:    vinserti128 $1, 2880(%rdi), %ymm0, %ymm0
+; AVX512BW-NEXT:    vmovdqa 2816(%rdi), %ymm1
+; AVX512BW-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm9 = [4,11]
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm30, %zmm9, %zmm1
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm10 = [0,7,14,0,0,7,14,0]
+; AVX512BW-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm18, %zmm16
+; AVX512BW-NEXT:    vmovdqu64 %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm18, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm25, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm25, %zmm10, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 3072(%rdi), %zmm2
+; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [4,5,6,13,4,5,6,13]
 ; AVX512BW-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm18, %zmm2
-; AVX512BW-NEXT:    vmovdqa64 %zmm19, %zmm15
-; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm4, %zmm2
-; AVX512BW-NEXT:    vmovdqa64 3072(%rdi), %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm4, %zmm1
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm1, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm8 = [4,5,6,13,4,5,6,13]
-; AVX512BW-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm8, %zmm2
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm2, %zmm1
+; AVX512BW-NEXT:    vinserti128 $1, 640(%rdi), %ymm0, %ymm0
+; AVX512BW-NEXT:    vmovdqa 576(%rdi), %ymm1
+; AVX512BW-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
+; AVX512BW-NEXT:    vmovdqa64 512(%rdi), %zmm3
+; AVX512BW-NEXT:    vmovdqa64 448(%rdi), %zmm2
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm12
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm9, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm27
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 768(%rdi), %zmm28
+; AVX512BW-NEXT:    vmovdqa64 704(%rdi), %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm28, %zmm10, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 832(%rdi), %zmm2
+; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm4, %zmm1
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm1, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vinserti128 $1, 192(%rdi), %ymm0, %ymm0
+; AVX512BW-NEXT:    vmovdqa 128(%rdi), %ymm1
+; AVX512BW-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
+; AVX512BW-NEXT:    vmovdqa64 64(%rdi), %zmm3
+; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm8
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm9, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm31
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 320(%rdi), %zmm18
+; AVX512BW-NEXT:    vmovdqa64 256(%rdi), %zmm29
+; AVX512BW-NEXT:    vmovdqa64 %zmm29, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm29, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm10, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 384(%rdi), %zmm2
+; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm4, %zmm1
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm1, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vinserti128 $1, 1536(%rdi), %ymm0, %ymm0
+; AVX512BW-NEXT:    vmovdqa 1472(%rdi), %ymm1
+; AVX512BW-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
+; AVX512BW-NEXT:    vmovdqa64 1408(%rdi), %zmm26
+; AVX512BW-NEXT:    vmovdqa64 1344(%rdi), %zmm7
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm26, %zmm9, %zmm1
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 1664(%rdi), %zmm13
+; AVX512BW-NEXT:    vmovdqa64 1600(%rdi), %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm10, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 1728(%rdi), %zmm2
+; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm4, %zmm1
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm1, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vinserti128 $1, 1088(%rdi), %ymm0, %ymm0
+; AVX512BW-NEXT:    vmovdqa 1024(%rdi), %ymm1
+; AVX512BW-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
+; AVX512BW-NEXT:    vmovdqa64 960(%rdi), %zmm19
+; AVX512BW-NEXT:    vmovdqa64 896(%rdi), %zmm6
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm9, %zmm1
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 1216(%rdi), %zmm22
+; AVX512BW-NEXT:    vmovdqa64 1152(%rdi), %zmm1
 ; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vinserti128 $1, 640(%rdi), %ymm0, %ymm2
-; AVX512BW-NEXT:    vmovdqa 576(%rdi), %ymm1
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm1[0,1,2,3,4,5],ymm2[6,7]
-; AVX512BW-NEXT:    vmovdqa64 512(%rdi), %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 448(%rdi), %zmm3
-; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm5, %zmm3
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqa64 768(%rdi), %zmm25
-; AVX512BW-NEXT:    vmovdqa64 704(%rdi), %zmm3
-; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm25, %zmm4, %zmm3
-; AVX512BW-NEXT:    vmovdqa64 832(%rdi), %zmm7
-; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm8, %zmm3
-; AVX512BW-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm3, %zmm2
+; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm10, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 1280(%rdi), %zmm2
 ; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vinserti128 $1, 192(%rdi), %ymm0, %ymm2
-; AVX512BW-NEXT:    vmovdqa 128(%rdi), %ymm0
-; AVX512BW-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm0[0,1,2,3,4,5],ymm2[6,7]
-; AVX512BW-NEXT:    vmovdqa64 64(%rdi), %zmm13
-; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm3
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm4, %zmm1
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm1, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vinserti128 $1, 2432(%rdi), %ymm0, %ymm0
+; AVX512BW-NEXT:    vmovdqa 2368(%rdi), %ymm1
+; AVX512BW-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
+; AVX512BW-NEXT:    vmovdqa64 2304(%rdi), %zmm20
+; AVX512BW-NEXT:    vmovdqa64 2240(%rdi), %zmm5
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm9, %zmm1
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 2560(%rdi), %zmm15
+; AVX512BW-NEXT:    vmovdqa64 2496(%rdi), %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm15, %zmm10, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 2624(%rdi), %zmm3
 ; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm5, %zmm3
-; AVX512BW-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqa64 320(%rdi), %zmm14
-; AVX512BW-NEXT:    vmovdqa64 256(%rdi), %zmm3
-; AVX512BW-NEXT:    vmovdqu64 %zmm3, (%rsp) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm4, %zmm3
-; AVX512BW-NEXT:    vmovdqa64 384(%rdi), %zmm30
-; AVX512BW-NEXT:    vpermt2q %zmm30, %zmm8, %zmm3
-; AVX512BW-NEXT:    vmovdqu64 %zmm30, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm3, %zmm2
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm4, %zmm1
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm1, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vinserti128 $1, 1984(%rdi), %ymm0, %ymm0
+; AVX512BW-NEXT:    vmovdqa 1920(%rdi), %ymm1
+; AVX512BW-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
+; AVX512BW-NEXT:    vmovdqa64 1856(%rdi), %zmm21
+; AVX512BW-NEXT:    vmovdqa64 1792(%rdi), %zmm2
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm1
 ; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vinserti128 $1, 1536(%rdi), %ymm0, %ymm2
-; AVX512BW-NEXT:    vmovdqa 1472(%rdi), %ymm3
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5],ymm2[6,7]
-; AVX512BW-NEXT:    vmovdqa64 1408(%rdi), %zmm29
-; AVX512BW-NEXT:    vmovdqa64 1344(%rdi), %zmm6
-; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm29, %zmm5, %zmm6
-; AVX512BW-NEXT:    vmovdqu64 %zmm29, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3],ymm2[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqa64 1664(%rdi), %zmm2
-; AVX512BW-NEXT:    vmovdqa64 1600(%rdi), %zmm9
+; AVX512BW-NEXT:    vpermt2q %zmm21, %zmm9, %zmm1
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 2112(%rdi), %zmm14
+; AVX512BW-NEXT:    vmovdqa64 2048(%rdi), %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm10, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 2176(%rdi), %zmm23
+; AVX512BW-NEXT:    vpermt2q %zmm23, %zmm4, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm23, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm1, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vinserti128 $1, 3328(%rdi), %ymm0, %ymm0
+; AVX512BW-NEXT:    vmovdqa 3264(%rdi), %ymm1
+; AVX512BW-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
+; AVX512BW-NEXT:    vmovdqa64 3200(%rdi), %zmm17
+; AVX512BW-NEXT:    vmovdqa64 3136(%rdi), %zmm11
+; AVX512BW-NEXT:    vpermi2q %zmm17, %zmm11, %zmm9
+; AVX512BW-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm9[0,1,2,3],ymm0[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm9
+; AVX512BW-NEXT:    vpermt2q %zmm31, %zmm10, %zmm9
 ; AVX512BW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm4, %zmm9
-; AVX512BW-NEXT:    vmovdqa64 1728(%rdi), %zmm28
-; AVX512BW-NEXT:    vpermt2q %zmm28, %zmm8, %zmm9
-; AVX512BW-NEXT:    vmovdqu64 %zmm28, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm6, %zmm9, %zmm6
+; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm9
+; AVX512BW-NEXT:    vpermt2q %zmm27, %zmm10, %zmm9
+; AVX512BW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm10, %zmm6
 ; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vinserti128 $1, 1088(%rdi), %ymm0, %ymm9
-; AVX512BW-NEXT:    vmovdqa 1024(%rdi), %ymm0
-; AVX512BW-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm9 = ymm0[0,1,2,3,4,5],ymm9[6,7]
-; AVX512BW-NEXT:    vmovdqa64 960(%rdi), %zmm27
-; AVX512BW-NEXT:    vmovdqa64 896(%rdi), %zmm10
-; AVX512BW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm27, %zmm5, %zmm10
-; AVX512BW-NEXT:    vmovdqu64 %zmm27, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm9 = ymm10[0,1,2,3],ymm9[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqa64 1216(%rdi), %zmm22
-; AVX512BW-NEXT:    vmovdqa64 1152(%rdi), %zmm10
-; AVX512BW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm4, %zmm10
-; AVX512BW-NEXT:    vmovdqa64 1280(%rdi), %zmm24
-; AVX512BW-NEXT:    vpermt2q %zmm24, %zmm8, %zmm10
-; AVX512BW-NEXT:    vmovdqu64 %zmm24, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm9, %zmm10, %zmm9
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm9
+; AVX512BW-NEXT:    vpermt2q %zmm26, %zmm10, %zmm9
 ; AVX512BW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vinserti128 $1, 2432(%rdi), %ymm0, %ymm9
-; AVX512BW-NEXT:    vmovdqa 2368(%rdi), %ymm0
-; AVX512BW-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm9 = ymm0[0,1,2,3,4,5],ymm9[6,7]
-; AVX512BW-NEXT:    vmovdqa64 2304(%rdi), %zmm21
-; AVX512BW-NEXT:    vmovdqa64 2240(%rdi), %zmm10
-; AVX512BW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm21, %zmm5, %zmm10
-; AVX512BW-NEXT:    vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm9 = ymm10[0,1,2,3],ymm9[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqa64 2560(%rdi), %zmm31
-; AVX512BW-NEXT:    vmovdqa64 2496(%rdi), %zmm10
-; AVX512BW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm31, %zmm4, %zmm10
-; AVX512BW-NEXT:    vmovdqa64 2624(%rdi), %zmm19
-; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm8, %zmm10
-; AVX512BW-NEXT:    vmovdqu64 %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm9, %zmm10, %zmm9
+; AVX512BW-NEXT:    vpermt2q %zmm21, %zmm10, %zmm2
+; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm10, %zmm5
+; AVX512BW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm9
+; AVX512BW-NEXT:    vmovdqa64 %zmm30, %zmm6
+; AVX512BW-NEXT:    vpermt2q %zmm30, %zmm10, %zmm9
 ; AVX512BW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vinserti128 $1, 1984(%rdi), %ymm0, %ymm9
-; AVX512BW-NEXT:    vmovdqa 1920(%rdi), %ymm0
-; AVX512BW-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm9 = ymm0[0,1,2,3,4,5],ymm9[6,7]
-; AVX512BW-NEXT:    vmovdqa64 1856(%rdi), %zmm18
-; AVX512BW-NEXT:    vmovdqa64 1792(%rdi), %zmm11
-; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm12
+; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm10, %zmm11
 ; AVX512BW-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm5, %zmm12
-; AVX512BW-NEXT:    vmovdqu64 %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm9 = ymm12[0,1,2,3],ymm9[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqa64 2112(%rdi), %zmm17
-; AVX512BW-NEXT:    vmovdqa64 2048(%rdi), %zmm12
-; AVX512BW-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm4, %zmm12
-; AVX512BW-NEXT:    vmovdqa64 2176(%rdi), %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm8, %zmm12
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm23
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm9, %zmm12, %zmm9
-; AVX512BW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vinserti128 $1, 3328(%rdi), %ymm0, %ymm9
-; AVX512BW-NEXT:    vmovdqa 3264(%rdi), %ymm12
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm9 = ymm12[0,1,2,3,4,5],ymm9[6,7]
-; AVX512BW-NEXT:    vmovdqa64 3200(%rdi), %zmm26
-; AVX512BW-NEXT:    vmovdqa64 3136(%rdi), %zmm10
-; AVX512BW-NEXT:    vpermi2q %zmm26, %zmm10, %zmm5
-; AVX512BW-NEXT:    vmovdqu64 %zmm26, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm9[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqa64 3456(%rdi), %zmm6
-; AVX512BW-NEXT:    vmovdqa64 3392(%rdi), %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermi2q %zmm6, %zmm0, %zmm4
-; AVX512BW-NEXT:    vmovdqa64 3520(%rdi), %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm8, %zmm4
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm16
+; AVX512BW-NEXT:    vmovdqa64 3456(%rdi), %zmm24
+; AVX512BW-NEXT:    vmovdqa64 3392(%rdi), %zmm2
+; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermi2q %zmm24, %zmm2, %zmm10
+; AVX512BW-NEXT:    vmovdqa64 3520(%rdi), %zmm30
+; AVX512BW-NEXT:    vpermt2q %zmm30, %zmm4, %zmm10
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm10, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm5, %zmm4, %zmm4
-; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa 2880(%rdi), %ymm4
-; AVX512BW-NEXT:    vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm4 # 32-byte Folded Reload
-; AVX512BW-NEXT:    # ymm4 = mem[8,9,10,11,12,13,14,15],ymm4[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm4[16,17,18,19,20,21,22,23]
+; AVX512BW-NEXT:    vmovdqa 2880(%rdi), %ymm0
+; AVX512BW-NEXT:    vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm4 # 32-byte Folded Reload
+; AVX512BW-NEXT:    # ymm4 = mem[8,9,10,11,12,13,14,15],ymm0[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm0[16,17,18,19,20,21,22,23]
 ; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm0 = [5,12]
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm0, %zmm5
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm5[0,1,2,3],ymm4[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm9
+; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm0, %zmm9
+; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3],ymm4[4,5,6,7]
 ; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [9,0,7,0,9,0,7,0]
 ; AVX512BW-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm15 # 64-byte Folded Reload
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm5 = [4,5,6,14,4,5,6,14]
-; AVX512BW-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm15 # 64-byte Folded Reload
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm8, %zmm15, %zmm8
+; AVX512BW-NEXT:    vmovdqa64 %zmm25, %zmm12
+; AVX512BW-NEXT:    vpermt2q %zmm16, %zmm4, %zmm12
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm10 = [4,5,6,14,4,5,6,14]
+; AVX512BW-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm12 # 64-byte Folded Reload
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm9, %zmm12, %zmm5
+; AVX512BW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa 640(%rdi), %ymm9
+; AVX512BW-NEXT:    vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm9 # 32-byte Folded Reload
+; AVX512BW-NEXT:    # ymm9 = mem[8,9,10,11,12,13,14,15],ymm9[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm9[16,17,18,19,20,21,22,23]
+; AVX512BW-NEXT:    vpermt2q %zmm27, %zmm0, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm27, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm9 = ymm1[0,1,2,3],ymm9[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm28, %zmm12
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm25 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm25, %zmm4, %zmm12
+; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm12 # 64-byte Folded Reload
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm9, %zmm12, %zmm5
+; AVX512BW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa 192(%rdi), %ymm9
+; AVX512BW-NEXT:    vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm9 # 32-byte Folded Reload
+; AVX512BW-NEXT:    # ymm9 = mem[8,9,10,11,12,13,14,15],ymm9[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm9[16,17,18,19,20,21,22,23]
 ; AVX512BW-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa 640(%rdi), %ymm8
-; AVX512BW-NEXT:    vpalignr {{.*#+}} ymm1 = ymm1[8,9,10,11,12,13,14,15],ymm8[0,1,2,3,4,5,6,7],ymm1[24,25,26,27,28,29,30,31],ymm8[16,17,18,19,20,21,22,23]
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm8 # 64-byte Folded Reload
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm8[0,1,2,3],ymm1[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqa64 %zmm25, %zmm8
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm15, %zmm4, %zmm8
-; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm5, %zmm8
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm8, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa 192(%rdi), %ymm1
-; AVX512BW-NEXT:    vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
-; AVX512BW-NEXT:    # ymm1 = mem[8,9,10,11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23]
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm12
+; AVX512BW-NEXT:    vmovdqu64 %zmm31, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm31, %zmm0, %zmm12
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm9 = ymm12[0,1,2,3],ymm9[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm18, %zmm12
+; AVX512BW-NEXT:    vpermt2q %zmm29, %zmm4, %zmm12
+; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm12 # 64-byte Folded Reload
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm9, %zmm12, %zmm5
+; AVX512BW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa 1536(%rdi), %ymm9
+; AVX512BW-NEXT:    vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm9 # 32-byte Folded Reload
+; AVX512BW-NEXT:    # ymm9 = mem[8,9,10,11,12,13,14,15],ymm9[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm9[16,17,18,19,20,21,22,23]
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm5
+; AVX512BW-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm12
+; AVX512BW-NEXT:    vmovdqa64 %zmm26, %zmm16
+; AVX512BW-NEXT:    vmovdqu64 %zmm26, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm26, %zmm0, %zmm12
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm9 = ymm12[0,1,2,3],ymm9[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm12
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm26 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm26, %zmm4, %zmm12
+; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm12 # 64-byte Folded Reload
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm9, %zmm12, %zmm7
+; AVX512BW-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa 1088(%rdi), %ymm9
+; AVX512BW-NEXT:    vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm9 # 32-byte Folded Reload
+; AVX512BW-NEXT:    # ymm9 = mem[8,9,10,11,12,13,14,15],ymm9[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm9[16,17,18,19,20,21,22,23]
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm0, %zmm7
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm7[0,1,2,3],ymm1[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm7
-; AVX512BW-NEXT:    vmovdqu64 (%rsp), %zmm9 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm4, %zmm7
-; AVX512BW-NEXT:    vpermt2q %zmm30, %zmm5, %zmm7
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm7, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm12
+; AVX512BW-NEXT:    vmovdqu64 %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm0, %zmm12
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm9 = ymm12[0,1,2,3],ymm9[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm22, %zmm12
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm29 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm29, %zmm4, %zmm12
+; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm12 # 64-byte Folded Reload
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm9, %zmm12, %zmm9
+; AVX512BW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa 2432(%rdi), %ymm9
+; AVX512BW-NEXT:    vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm2 # 32-byte Folded Reload
+; AVX512BW-NEXT:    # ymm2 = mem[8,9,10,11,12,13,14,15],ymm9[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm9[16,17,18,19,20,21,22,23]
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm9
+; AVX512BW-NEXT:    vmovdqu64 %zmm20, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm0, %zmm9
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm9[0,1,2,3],ymm2[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm9
+; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm9 # 64-byte Folded Reload
+; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm9 # 64-byte Folded Reload
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm9, %zmm2
+; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa 1984(%rdi), %ymm2
+; AVX512BW-NEXT:    vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload
+; AVX512BW-NEXT:    # ymm2 = mem[8,9,10,11,12,13,14,15],ymm2[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm2[16,17,18,19,20,21,22,23]
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm3
+; AVX512BW-NEXT:    vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm21, %zmm0, %zmm3
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm3
+; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm3 # 64-byte Folded Reload
+; AVX512BW-NEXT:    vpermt2q %zmm23, %zmm10, %zmm3
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm3, %zmm2
+; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa 3328(%rdi), %ymm2
+; AVX512BW-NEXT:    vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm1 # 32-byte Folded Reload
+; AVX512BW-NEXT:    # ymm1 = mem[8,9,10,11,12,13,14,15],ymm2[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm2[16,17,18,19,20,21,22,23]
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermi2q %zmm17, %zmm2, %zmm0
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm6 # 64-byte Folded Reload
+; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm27, %zmm1
+; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm1 # 64-byte Folded Reload
 ; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa 1536(%rdi), %ymm1
-; AVX512BW-NEXT:    vpalignr {{.*#+}} ymm1 = ymm3[8,9,10,11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7],ymm3[24,25,26,27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23]
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm29, %zmm0, %zmm3
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm4, %zmm3
-; AVX512BW-NEXT:    vpermt2q %zmm28, %zmm5, %zmm3
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm3, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm31, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm4, %zmm1
 ; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa 1088(%rdi), %ymm1
-; AVX512BW-NEXT:    vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
-; AVX512BW-NEXT:    # ymm1 = mem[8,9,10,11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23]
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm27, %zmm0, %zmm3
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqa64 %zmm22, %zmm3
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm27 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm27, %zmm4, %zmm3
-; AVX512BW-NEXT:    vpermt2q %zmm24, %zmm5, %zmm3
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm3, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm4, %zmm1
 ; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa 2432(%rdi), %ymm1
-; AVX512BW-NEXT:    vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
-; AVX512BW-NEXT:    # ymm1 = mem[8,9,10,11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23]
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm21, %zmm0, %zmm3
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm1[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqa64 %zmm31, %zmm3
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm28 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm28, %zmm4, %zmm3
-; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm5, %zmm3
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm3, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm4, %zmm19
+; AVX512BW-NEXT:    vmovdqu64 %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm4, %zmm20
+; AVX512BW-NEXT:    vmovdqu64 %zmm20, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm4, %zmm21
+; AVX512BW-NEXT:    vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm4, %zmm17
+; AVX512BW-NEXT:    vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512BW-NEXT:    vpermi2q %zmm8, %zmm24, %zmm4
+; AVX512BW-NEXT:    vpermt2q %zmm30, %zmm10, %zmm4
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm4, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm23 = [10,3,10,3,10,3,10,3]
+; AVX512BW-NEXT:    # zmm23 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm25, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm28, %zmm23, %zmm25
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm27 = [11,4,11,4,11,4,11,4]
+; AVX512BW-NEXT:    # zmm27 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm28, %zmm27, %zmm1
 ; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa 1984(%rdi), %ymm1
-; AVX512BW-NEXT:    vpalignr $8, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
-; AVX512BW-NEXT:    # ymm1 = mem[8,9,10,11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23]
-; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm0, %zmm11
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm11[0,1,2,3],ymm1[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqa64 %zmm17, %zmm3
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm30 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm30, %zmm4, %zmm3
-; AVX512BW-NEXT:    vpermt2q %zmm23, %zmm5, %zmm3
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm3, %zmm1
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm31 = [12,5,12,5,12,5,12,5]
+; AVX512BW-NEXT:    # zmm31 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm28, %zmm31, %zmm1
 ; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa 3328(%rdi), %ymm1
-; AVX512BW-NEXT:    vpalignr {{.*#+}} ymm1 = ymm12[8,9,10,11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7],ymm12[24,25,26,27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23]
-; AVX512BW-NEXT:    vpermi2q %zmm26, %zmm10, %zmm0
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512BW-NEXT:    vpermi2q %zmm7, %zmm6, %zmm4
-; AVX512BW-NEXT:    vpermt2q %zmm16, %zmm5, %zmm4
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm4, %zmm0
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm12 = [13,6,13,6,13,6,13,6]
+; AVX512BW-NEXT:    # zmm12 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm28, %zmm12, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm1 = [2,9,2,9,2,9,2,9]
+; AVX512BW-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpermt2q %zmm28, %zmm1, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm19 = [10,3,10,3,10,3,10,3]
-; AVX512BW-NEXT:    # zmm19 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm1
-; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm21
-; AVX512BW-NEXT:    vpermt2q %zmm25, %zmm19, %zmm21
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm20 = [11,4,11,4,11,4,11,4]
-; AVX512BW-NEXT:    # zmm20 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm25, %zmm20, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm28 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm28, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm23, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm18 = [12,5,12,5,12,5,12,5]
-; AVX512BW-NEXT:    # zmm18 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm25, %zmm18, %zmm0
+; AVX512BW-NEXT:    vmovdqa64 %zmm28, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm27, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm11 = [13,6,13,6,13,6,13,6]
-; AVX512BW-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm25, %zmm11, %zmm0
+; AVX512BW-NEXT:    vmovdqa64 %zmm28, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm31, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [2,9,2,9,2,9,2,9]
-; AVX512BW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermt2q %zmm25, %zmm0, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm29
-; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm12
-; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm19, %zmm12
-; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm1
-; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm20, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm1
-; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm18, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm1
-; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm11, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm0, %zmm29
-; AVX512BW-NEXT:    vmovdqu64 %zmm29, (%rsp) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm24
-; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm1
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm19, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm1
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm20, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm1
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm18, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm1
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm11, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm0, %zmm24
-; AVX512BW-NEXT:    vmovdqu64 %zmm24, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm27, %zmm9
-; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm19, %zmm9
-; AVX512BW-NEXT:    vmovdqa64 %zmm27, %zmm1
-; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm20, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm27, %zmm1
-; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm18, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm27, %zmm1
-; AVX512BW-NEXT:    vmovdqa64 %zmm27, %zmm2
-; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm11, %zmm1
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm27
-; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm0, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm28, %zmm10
-; AVX512BW-NEXT:    vpermt2q %zmm31, %zmm19, %zmm10
-; AVX512BW-NEXT:    vmovdqa64 %zmm28, %zmm14
-; AVX512BW-NEXT:    vpermt2q %zmm31, %zmm20, %zmm14
-; AVX512BW-NEXT:    vmovdqa64 %zmm28, %zmm15
-; AVX512BW-NEXT:    vpermt2q %zmm31, %zmm18, %zmm15
-; AVX512BW-NEXT:    vmovdqa64 %zmm28, %zmm16
-; AVX512BW-NEXT:    vpermt2q %zmm31, %zmm11, %zmm16
-; AVX512BW-NEXT:    vpermt2q %zmm31, %zmm0, %zmm28
+; AVX512BW-NEXT:    vmovdqa64 %zmm28, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm12, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm1, %zmm28
 ; AVX512BW-NEXT:    vmovdqu64 %zmm28, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm30, %zmm1
-; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm19, %zmm1
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm26
-; AVX512BW-NEXT:    vmovdqa64 %zmm30, %zmm5
-; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm20, %zmm5
-; AVX512BW-NEXT:    vmovdqa64 %zmm30, %zmm8
-; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm18, %zmm8
-; AVX512BW-NEXT:    vmovdqa64 %zmm30, %zmm13
-; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm11, %zmm13
-; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm0, %zmm30
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm22 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm22, %zmm4
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm19, %zmm4
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm24
-; AVX512BW-NEXT:    vpermi2q %zmm6, %zmm7, %zmm19
-; AVX512BW-NEXT:    vmovdqa64 %zmm22, %zmm3
-; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm20, %zmm3
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm17
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm11, %zmm1 # 64-byte Folded Reload
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm31 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm31, %zmm11, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm25 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm25, %zmm11, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm23 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm23, %zmm11, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm11, %zmm1 # 64-byte Folded Reload
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm11, %zmm1 # 64-byte Folded Reload
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm11, %zmm1 # 64-byte Folded Reload
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermi2q %zmm6, %zmm7, %zmm20
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm11, %zmm1 # 64-byte Folded Reload
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm22, %zmm29
-; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm18, %zmm29
-; AVX512BW-NEXT:    vpermi2q %zmm6, %zmm7, %zmm18
-; AVX512BW-NEXT:    vmovdqa64 %zmm22, %zmm1
-; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm11, %zmm1
-; AVX512BW-NEXT:    vpermi2q %zmm6, %zmm7, %zmm11
-; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm0, %zmm24
-; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm0, %zmm22
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [0,5,6,9,0,5,6,9]
-; AVX512BW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm0, %zmm12
-; AVX512BW-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm0, %zmm21
-; AVX512BW-NEXT:    vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm12, %zmm0, %zmm9
-; AVX512BW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm0, %zmm17
-; AVX512BW-NEXT:    vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm0, %zmm26
+; AVX512BW-NEXT:    vmovdqa64 %zmm26, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm23, %zmm0
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm28
+; AVX512BW-NEXT:    vmovdqa64 %zmm26, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm27, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm26, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm31, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm26, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm12, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm1, %zmm26
 ; AVX512BW-NEXT:    vmovdqu64 %zmm26, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm29, %zmm9
+; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm23, %zmm9
+; AVX512BW-NEXT:    vmovdqa64 %zmm29, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm27, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm29, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm31, %zmm0
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm17
+; AVX512BW-NEXT:    vmovdqa64 %zmm29, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm12, %zmm0
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm16
+; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm1, %zmm29
+; AVX512BW-NEXT:    vmovdqu64 %zmm29, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm21 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm21, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm15, %zmm23, %zmm0
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm19
+; AVX512BW-NEXT:    vmovdqa64 %zmm21, %zmm10
+; AVX512BW-NEXT:    vpermt2q %zmm15, %zmm27, %zmm10
+; AVX512BW-NEXT:    vmovdqa64 %zmm21, %zmm13
+; AVX512BW-NEXT:    vpermt2q %zmm15, %zmm31, %zmm13
+; AVX512BW-NEXT:    vmovdqa64 %zmm21, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm15, %zmm12, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm15, %zmm1, %zmm21
+; AVX512BW-NEXT:    vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm2
+; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm23, %zmm2
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm22
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm4
+; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm27, %zmm4
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm3
+; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm31, %zmm3
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm20
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm7
+; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm12, %zmm7
+; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm1, %zmm5
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm26 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm26, %zmm0, %zmm10
-; AVX512BW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm28 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm28, %zmm0, %zmm4
-; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm0, %zmm19
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [0,5,6,10,0,5,6,10]
-; AVX512BW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermt2q %zmm28, %zmm0, %zmm3
-; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm0, %zmm17
-; AVX512BW-NEXT:    vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm0, %zmm17
-; AVX512BW-NEXT:    vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm0, %zmm17
-; AVX512BW-NEXT:    vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm12, %zmm0, %zmm17
-; AVX512BW-NEXT:    vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm26, %zmm0, %zmm14
-; AVX512BW-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm0, %zmm5
-; AVX512BW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm0, %zmm20
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [0,5,6,11,0,5,6,11]
-; AVX512BW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermt2q %zmm28, %zmm0, %zmm29
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm0, %zmm17
-; AVX512BW-NEXT:    vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm0, %zmm17
-; AVX512BW-NEXT:    vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm0, %zmm17
-; AVX512BW-NEXT:    vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm12, %zmm0, %zmm17
-; AVX512BW-NEXT:    vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm26, %zmm0, %zmm15
-; AVX512BW-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm0, %zmm8
+; AVX512BW-NEXT:    vmovdqa64 %zmm26, %zmm3
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm23, %zmm3
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm29
+; AVX512BW-NEXT:    vpermi2q %zmm24, %zmm8, %zmm23
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm12, %zmm0 # 64-byte Folded Reload
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm26, %zmm2
+; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm27, %zmm2
+; AVX512BW-NEXT:    vmovdqa64 %zmm18, %zmm14
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm15, %zmm12, %zmm6
+; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm12, %zmm6 # 64-byte Folded Reload
+; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 (%rsp), %zmm6 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm12, %zmm6 # 64-byte Folded Reload
+; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm12, %zmm6 # 64-byte Folded Reload
+; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm12, %zmm6 # 64-byte Folded Reload
+; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm12, %zmm8
 ; AVX512BW-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm0, %zmm18
-; AVX512BW-NEXT:    vmovdqu64 %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [0,5,6,12,0,5,6,12]
-; AVX512BW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermt2q %zmm28, %zmm0, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm0, %zmm17
-; AVX512BW-NEXT:    vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm0, %zmm17
-; AVX512BW-NEXT:    vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm0, %zmm17
-; AVX512BW-NEXT:    vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm12, %zmm0, %zmm27
-; AVX512BW-NEXT:    vmovdqu64 %zmm27, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm26, %zmm0, %zmm16
-; AVX512BW-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm1
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm0, %zmm13
-; AVX512BW-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm0, %zmm11
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [4,5,8,15,4,5,8,15]
-; AVX512BW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm0, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 (%rsp), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm0, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, (%rsp) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm0, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm12, %zmm0, %zmm2
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm12, %zmm8 # 64-byte Folded Reload
+; AVX512BW-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermi2q %zmm24, %zmm29, %zmm27
+; AVX512BW-NEXT:    vmovdqa64 %zmm26, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm31, %zmm0
+; AVX512BW-NEXT:    vpermi2q %zmm24, %zmm29, %zmm31
+; AVX512BW-NEXT:    vmovdqa64 %zmm26, %zmm8
+; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm12, %zmm8
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm18
+; AVX512BW-NEXT:    vpermi2q %zmm24, %zmm29, %zmm12
+; AVX512BW-NEXT:    vpermt2q %zmm24, %zmm1, %zmm29
+; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm1, %zmm26
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm1 = [0,5,6,9,0,5,6,9]
+; AVX512BW-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm1, %zmm11
+; AVX512BW-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm21 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm21, %zmm1, %zmm25
+; AVX512BW-NEXT:    vmovdqu64 %zmm25, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm24 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm24, %zmm1, %zmm9
+; AVX512BW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm1, %zmm28
+; AVX512BW-NEXT:    vmovdqu64 %zmm28, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm25 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm25, %zmm1, %zmm22
+; AVX512BW-NEXT:    vmovdqu64 %zmm22, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm1, %zmm19
+; AVX512BW-NEXT:    vmovdqu64 %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm1, %zmm3
+; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm30, %zmm1, %zmm23
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm1 = [0,5,6,10,0,5,6,10]
+; AVX512BW-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm1, %zmm2
 ; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm28 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm21, %zmm1, %zmm28
+; AVX512BW-NEXT:    vmovdqu64 %zmm28, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm26, %zmm0, %zmm2
+; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm1, %zmm2
 ; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm0, %zmm30
-; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm0, %zmm24
-; AVX512BW-NEXT:    vpermt2q %zmm28, %zmm0, %zmm22
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm1, %zmm9
+; AVX512BW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm24, %zmm1, %zmm9
+; AVX512BW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm1, %zmm10
+; AVX512BW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm25, %zmm1, %zmm4
+; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm30, %zmm1, %zmm27
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm1 = [0,5,6,11,0,5,6,11]
+; AVX512BW-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm1, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm28 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm21, %zmm1, %zmm28
+; AVX512BW-NEXT:    vmovdqu64 %zmm28, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm28 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm1, %zmm28
+; AVX512BW-NEXT:    vmovdqu64 %zmm28, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm1, %zmm9
+; AVX512BW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm24, %zmm1, %zmm17
+; AVX512BW-NEXT:    vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm1, %zmm13
+; AVX512BW-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm25, %zmm1, %zmm20
+; AVX512BW-NEXT:    vmovdqu64 %zmm20, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm30, %zmm1, %zmm31
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm1 = [0,5,6,12,0,5,6,12]
+; AVX512BW-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm1, %zmm18
+; AVX512BW-NEXT:    vmovdqu64 %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm28 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm21, %zmm1, %zmm28
+; AVX512BW-NEXT:    vmovdqu64 %zmm28, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm28 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm1, %zmm28
+; AVX512BW-NEXT:    vmovdqu64 %zmm28, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm28 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm1, %zmm28
+; AVX512BW-NEXT:    vmovdqu64 %zmm28, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm24, %zmm1, %zmm16
+; AVX512BW-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm1, %zmm9
+; AVX512BW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm25, %zmm1, %zmm7
+; AVX512BW-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm30, %zmm1, %zmm12
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm1 = [4,5,8,15,4,5,8,15]
+; AVX512BW-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm21, %zmm1, %zmm16
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm1, %zmm17
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm1, %zmm18
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm20 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm24, %zmm1, %zmm20
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm21 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm1, %zmm21
+; AVX512BW-NEXT:    vpermt2q %zmm25, %zmm1, %zmm5
+; AVX512BW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm30, %zmm1, %zmm29
+; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm1, %zmm26
 ; AVX512BW-NEXT:    movb $24, %al
 ; AVX512BW-NEXT:    kmovd %eax, %k1
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} ymm16 = <0,7,14,u>
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm12, %zmm16, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm1 {%k1} = zmm0[4,5,4,5],zmm31[4,5,4,5]
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm26 = [7,0,9,0,7,0,9,0]
-; AVX512BW-NEXT:    # zmm26 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm31, %zmm1
-; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm26, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm1
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = [4,11,4,11]
-; AVX512BW-NEXT:    # ymm0 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vpermt2q %zmm31, %zmm0, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm22 # 64-byte Reload
+; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm22 {%k1} = zmm1[4,5,4,5],zmm15[4,5,4,5]
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm30 = [7,0,9,0,7,0,9,0]
+; AVX512BW-NEXT:    # zmm30 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm30, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [4,11,4,11]
+; AVX512BW-NEXT:    # ymm2 = mem[0,1,0,1]
+; AVX512BW-NEXT:    vpermt2q %zmm15, %zmm2, %zmm1
 ; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm16, %zmm18
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm18 {%k1} = zmm1[4,5,4,5],zmm25[4,5,4,5]
-; AVX512BW-NEXT:    vmovdqa64 %zmm25, %zmm2
-; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm26, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm25, %zmm0, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm27 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm31 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm31, %zmm16, %zmm27
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
+; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm19 {%k1} = zmm3[4,5,4,5],zmm1[4,5,4,5]
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm30, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm2, %zmm3
+; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 (%rsp), %zmm3 # 64-byte Reload
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm27 {%k1} = zmm1[4,5,4,5],zmm23[4,5,4,5]
-; AVX512BW-NEXT:    vmovdqa64 %zmm23, %zmm2
-; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm26, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm23, %zmm0, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm25 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm16, %zmm25
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
+; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm14 {%k1} = zmm3[4,5,4,5],zmm1[4,5,4,5]
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm30, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm2, %zmm3
+; AVX512BW-NEXT:    vmovdqu64 %zmm3, (%rsp) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm25 {%k1} = zmm1[4,5,4,5],zmm9[4,5,4,5]
-; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm2
-; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm26, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm0, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm23 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm16, %zmm23
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm24 # 64-byte Reload
+; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm24 {%k1} = zmm3[4,5,4,5],zmm1[4,5,4,5]
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm30, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm2, %zmm3
+; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
+; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm11 {%k1} = zmm0[4,5,4,5],zmm1[4,5,4,5]
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm30, %zmm3
+; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm2, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
+; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm10 {%k1} = zmm0[4,5,4,5],zmm6[4,5,4,5]
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm3
+; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm30, %zmm3
+; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm2, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
+; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm9 {%k1} = zmm15[4,5,4,5],zmm6[4,5,4,5]
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm23 {%k1} = zmm1[4,5,4,5],zmm3[4,5,4,5]
-; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm2
-; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm26, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm0, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm28
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm16, %zmm13
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm13 {%k1} = zmm1[4,5,4,5],zmm3[4,5,4,5]
-; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm2
-; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm26, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm0, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm10
+; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm30, %zmm28
+; AVX512BW-NEXT:    vpermi2q %zmm15, %zmm6, %zmm30
+; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm2, %zmm15
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm25 # 64-byte Reload
+; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm25 {%k1} = zmm13[4,5,4,5],zmm3[4,5,4,5]
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm2, %zmm13
+; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm0 = [6,13]
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 # 64-byte Folded Reload
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm16, %zmm10
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm28 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
-; AVX512BW-NEXT:    vpermi2q %zmm28, %zmm15, %zmm16
+; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm3 # 64-byte Folded Reload
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm16 {%k1} = zmm1[4,5,4,5],zmm2[4,5,4,5]
+; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm1 # 64-byte Folded Reload
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm21
+; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm4 # 64-byte Folded Reload
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm26, %zmm21
-; AVX512BW-NEXT:    vpermi2q %zmm1, %zmm2, %zmm26
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm0, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm1
-; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm10 {%k1} = zmm5[4,5,4,5],zmm4[4,5,4,5]
-; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm0, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} ymm0 = <9,0,7,u>
-; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm4
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm0, %zmm12
-; AVX512BW-NEXT:    vmovdqa {{.*#+}} xmm1 = [6,13]
-; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm1, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm9
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm0, %zmm9
-; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm1, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm31, %zmm7
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm0, %zmm7
-; AVX512BW-NEXT:    vpermt2q %zmm31, %zmm1, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm4
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm0, %zmm6
-; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm1, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm17, %zmm5
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm0, %zmm5
-; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm1, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm0, %zmm2
-; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm1, %zmm4
-; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm14
-; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm4
-; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm0, %zmm4
-; AVX512BW-NEXT:    vpermi2q %zmm15, %zmm28, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm28, %zmm1, %zmm15
-; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm1, %zmm8
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
-; AVX512BW-NEXT:    # ymm1 = ymm1[0,1,2,3],mem[4,5,6,7]
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-NEXT:    vinsertf64x4 $0, %ymm1, %zmm3, %zmm1
-; AVX512BW-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
-; AVX512BW-NEXT:    # ymm1 = ymm1[0,1,2,3],mem[4,5,6,7]
-; AVX512BW-NEXT:    vmovups (%rsp), %zmm3 # 64-byte Reload
-; AVX512BW-NEXT:    vinsertf64x4 $0, %ymm1, %zmm3, %zmm1
-; AVX512BW-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
-; AVX512BW-NEXT:    # ymm1 = ymm1[0,1,2,3],mem[4,5,6,7]
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-NEXT:    vinsertf64x4 $0, %ymm1, %zmm3, %zmm1
-; AVX512BW-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
-; AVX512BW-NEXT:    # ymm1 = ymm1[0,1,2,3],mem[4,5,6,7]
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-NEXT:    vinsertf64x4 $0, %ymm1, %zmm3, %zmm1
-; AVX512BW-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
-; AVX512BW-NEXT:    # ymm1 = ymm1[0,1,2,3],mem[4,5,6,7]
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-NEXT:    vinsertf64x4 $0, %ymm1, %zmm3, %zmm1
-; AVX512BW-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm1 # 32-byte Folded Reload
-; AVX512BW-NEXT:    # ymm1 = ymm14[0,1,2,3],mem[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm30, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm15, %ymm1 # 32-byte Folded Reload
-; AVX512BW-NEXT:    # ymm1 = ymm15[0,1,2,3],mem[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm24, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, (%rsp) # 64-byte Spill
-; AVX512BW-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm1 # 32-byte Folded Reload
-; AVX512BW-NEXT:    # ymm1 = ymm8[0,1,2,3],mem[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm22, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm5 # 64-byte Folded Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm6 # 64-byte Folded Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm8 # 64-byte Folded Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm7 # 64-byte Folded Reload
+; AVX512BW-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm0 # 32-byte Folded Reload
+; AVX512BW-NEXT:    # ymm0 = ymm2[0,1,2,3],mem[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm16, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm0 # 32-byte Folded Reload
+; AVX512BW-NEXT:    # ymm0 = ymm3[0,1,2,3],mem[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm17, %zmm17
+; AVX512BW-NEXT:    vpblendd $240, (%rsp), %ymm1, %ymm0 # 32-byte Folded Reload
+; AVX512BW-NEXT:    # ymm0 = ymm1[0,1,2,3],mem[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm18, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm0 # 32-byte Folded Reload
+; AVX512BW-NEXT:    # ymm0 = ymm4[0,1,2,3],mem[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm20, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, (%rsp) # 64-byte Spill
+; AVX512BW-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm0 # 32-byte Folded Reload
+; AVX512BW-NEXT:    # ymm0 = ymm5[0,1,2,3],mem[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm21, %zmm21
+; AVX512BW-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm6, %ymm0 # 32-byte Folded Reload
+; AVX512BW-NEXT:    # ymm0 = ymm6[0,1,2,3],mem[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm1, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm8[0,1,2,3],ymm15[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm29, %zmm29
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm7[0,1,2,3],ymm13[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm26, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    movb $-32, %al
 ; AVX512BW-NEXT:    kmovd %eax, %k2
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm19 {%k2}
+; AVX512BW-NEXT:    vmovdqu64 %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm22, %zmm19
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm19 {%k2}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm24 {%k2}
+; AVX512BW-NEXT:    vmovdqu64 %zmm24, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm24
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm24 {%k2}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm10 {%k2}
+; AVX512BW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm11 {%k2}
+; AVX512BW-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm25 {%k2}
+; AVX512BW-NEXT:    vmovdqu64 %zmm25, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm23, %zmm9 {%k2}
+; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm25
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm16 {%k1}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm16 {%k2}
+; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm22
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm18 {%k2}
-; AVX512BW-NEXT:    vmovdqu64 %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm1 {%k2}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k2}
 ; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm25 {%k2}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm27 {%k2}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm13 {%k2}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k2}
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm16 {%k1}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm16 {%k2}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm11 {%k1}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm11 {%k2}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm23 {%k2}
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm10 {%k1}
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
 ; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm10 {%k2}
-; AVX512BW-NEXT:    vmovdqa64 %zmm19, %zmm16 {%k2}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k2}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm12 {%k1}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm12 {%k2}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm9 {%k1}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm9 {%k2}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm7 {%k1}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm7 {%k2}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm6 {%k1}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm6 {%k2}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm5 {%k1}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm5 {%k2}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
 ; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
 ; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k2}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
 ; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm0 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 %zmm20, %zmm0 {%k2}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm31 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm29, %zmm31 {%k2}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm28 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm28 {%k2}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm29 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm29 {%k2}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm30 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm30 {%k2}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm15 {%k2}
+; AVX512BW-NEXT:    vmovdqa64 %zmm27, %zmm0 {%k2}
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm9
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm13 {%k2}
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm14 {%k2}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm8 {%k2}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm3 {%k2}
-; AVX512BW-NEXT:    vmovdqa 2752(%rdi), %ymm1
-; AVX512BW-NEXT:    vpalignr {{.*#+}} ymm1 = mem[8,9,10,11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23]
-; AVX512BW-NEXT:    vextracti128 $1, %ymm1, %xmm1
-; AVX512BW-NEXT:    vinserti32x4 $0, %xmm1, %zmm21, %zmm17
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm17 {%k2}
-; AVX512BW-NEXT:    vmovdqa 512(%rdi), %ymm1
-; AVX512BW-NEXT:    vpalignr {{.*#+}} ymm1 = mem[8,9,10,11,12,13,14,15],ymm1[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm1[16,17,18,19,20,21,22,23]
-; AVX512BW-NEXT:    vextracti128 $1, %ymm1, %xmm1
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
-; AVX512BW-NEXT:    vinserti32x4 $0, %xmm1, %zmm18, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm14 {%k2}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm15 {%k2}
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm18, %zmm1 {%k2}
-; AVX512BW-NEXT:    vmovdqa64 64(%rdi), %ymm18
-; AVX512BW-NEXT:    vpalignr {{.*#+}} ymm18 = mem[8,9,10,11,12,13,14,15],ymm18[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm18[16,17,18,19,20,21,22,23]
-; AVX512BW-NEXT:    vextracti32x4 $1, %ymm18, %xmm18
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
-; AVX512BW-NEXT:    vinserti32x4 $0, %xmm18, %zmm19, %zmm18
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm19, %zmm18 {%k2}
-; AVX512BW-NEXT:    vmovdqa64 1408(%rdi), %ymm19
-; AVX512BW-NEXT:    vpalignr {{.*#+}} ymm19 = mem[8,9,10,11,12,13,14,15],ymm19[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm19[16,17,18,19,20,21,22,23]
-; AVX512BW-NEXT:    vextracti32x4 $1, %ymm19, %xmm19
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm20 # 64-byte Reload
-; AVX512BW-NEXT:    vinserti32x4 $0, %xmm19, %zmm20, %zmm19
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm18 {%k2}
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm20 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm20, %zmm19 {%k2}
-; AVX512BW-NEXT:    vmovdqa64 960(%rdi), %ymm20
-; AVX512BW-NEXT:    vpalignr {{.*#+}} ymm20 = mem[8,9,10,11,12,13,14,15],ymm20[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm20[16,17,18,19,20,21,22,23]
-; AVX512BW-NEXT:    vextracti32x4 $1, %ymm20, %xmm20
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm21 # 64-byte Reload
-; AVX512BW-NEXT:    vinserti32x4 $0, %xmm20, %zmm21, %zmm20
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm21 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm21, %zmm20 {%k2}
-; AVX512BW-NEXT:    vmovdqa64 2304(%rdi), %ymm22
-; AVX512BW-NEXT:    vpalignr {{.*#+}} ymm22 = mem[8,9,10,11,12,13,14,15],ymm22[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm22[16,17,18,19,20,21,22,23]
-; AVX512BW-NEXT:    vextracti32x4 $1, %ymm22, %xmm22
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm21 # 64-byte Reload
-; AVX512BW-NEXT:    vinserti32x4 $0, %xmm22, %zmm21, %zmm22
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm21 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm21, %zmm22 {%k2}
-; AVX512BW-NEXT:    vmovdqa64 1856(%rdi), %ymm24
-; AVX512BW-NEXT:    vpalignr {{.*#+}} ymm24 = mem[8,9,10,11,12,13,14,15],ymm24[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm24[16,17,18,19,20,21,22,23]
-; AVX512BW-NEXT:    vextracti32x4 $1, %ymm24, %xmm24
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm21 # 64-byte Reload
-; AVX512BW-NEXT:    vinserti32x4 $0, %xmm24, %zmm21, %zmm21
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm24 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm24, %zmm21 {%k2}
-; AVX512BW-NEXT:    vmovdqa64 3200(%rdi), %ymm24
-; AVX512BW-NEXT:    vpalignr {{.*#+}} ymm24 = mem[8,9,10,11,12,13,14,15],ymm24[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm24[16,17,18,19,20,21,22,23]
-; AVX512BW-NEXT:    vextracti32x4 $1, %ymm24, %xmm24
-; AVX512BW-NEXT:    vinserti32x4 $0, %xmm24, %zmm26, %zmm24
-; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm24 {%k2}
-; AVX512BW-NEXT:    vmovdqa64 %zmm16, 448(%rsi)
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, 384(%rsi)
-; AVX512BW-NEXT:    vmovdqa64 %zmm23, 320(%rsi)
-; AVX512BW-NEXT:    vmovdqa64 %zmm13, 256(%rsi)
-; AVX512BW-NEXT:    vmovdqa64 %zmm27, 192(%rsi)
-; AVX512BW-NEXT:    vmovdqa64 %zmm25, 128(%rsi)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm10, 64(%rsi)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm10, (%rsi)
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, 448(%rdx)
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm20 {%k2}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm23 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm23 {%k2}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm26 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm26 {%k2}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm27 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm31, %zmm27 {%k2}
+; AVX512BW-NEXT:    vmovdqa 2752(%rdi), %ymm0
+; AVX512BW-NEXT:    vpalignr {{.*#+}} ymm0 = mem[8,9,10,11,12,13,14,15],ymm0[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm0[16,17,18,19,20,21,22,23]
+; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; AVX512BW-NEXT:    vinserti32x4 $0, %xmm0, %zmm28, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k2}
+; AVX512BW-NEXT:    vmovdqa 512(%rdi), %ymm0
+; AVX512BW-NEXT:    vpalignr {{.*#+}} ymm0 = mem[8,9,10,11,12,13,14,15],ymm0[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm0[16,17,18,19,20,21,22,23]
+; AVX512BW-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512BW-NEXT:    vinserti32x4 $0, %xmm0, %zmm3, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm0 {%k2}
+; AVX512BW-NEXT:    vmovdqa 64(%rdi), %ymm3
+; AVX512BW-NEXT:    vpalignr {{.*#+}} ymm3 = mem[8,9,10,11,12,13,14,15],ymm3[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm3[16,17,18,19,20,21,22,23]
+; AVX512BW-NEXT:    vextracti128 $1, %ymm3, %xmm3
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512BW-NEXT:    vinserti32x4 $0, %xmm3, %zmm4, %zmm3
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm3 {%k2}
+; AVX512BW-NEXT:    vmovdqa 1408(%rdi), %ymm4
+; AVX512BW-NEXT:    vpalignr {{.*#+}} ymm4 = mem[8,9,10,11,12,13,14,15],ymm4[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm4[16,17,18,19,20,21,22,23]
+; AVX512BW-NEXT:    vextracti128 $1, %ymm4, %xmm4
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512BW-NEXT:    vinserti32x4 $0, %xmm4, %zmm5, %zmm4
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm4 {%k2}
+; AVX512BW-NEXT:    vmovdqa 960(%rdi), %ymm5
+; AVX512BW-NEXT:    vpalignr {{.*#+}} ymm5 = mem[8,9,10,11,12,13,14,15],ymm5[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm5[16,17,18,19,20,21,22,23]
+; AVX512BW-NEXT:    vextracti128 $1, %ymm5, %xmm5
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512BW-NEXT:    vinserti32x4 $0, %xmm5, %zmm6, %zmm5
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm5 {%k2}
+; AVX512BW-NEXT:    vmovdqa 2304(%rdi), %ymm6
+; AVX512BW-NEXT:    vpalignr {{.*#+}} ymm6 = mem[8,9,10,11,12,13,14,15],ymm6[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm6[16,17,18,19,20,21,22,23]
+; AVX512BW-NEXT:    vextracti128 $1, %ymm6, %xmm6
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512BW-NEXT:    vinserti32x4 $0, %xmm6, %zmm7, %zmm6
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm6 {%k2}
+; AVX512BW-NEXT:    vmovdqa 1856(%rdi), %ymm7
+; AVX512BW-NEXT:    vpalignr {{.*#+}} ymm7 = mem[8,9,10,11,12,13,14,15],ymm7[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm7[16,17,18,19,20,21,22,23]
+; AVX512BW-NEXT:    vextracti128 $1, %ymm7, %xmm7
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512BW-NEXT:    vinserti32x4 $0, %xmm7, %zmm8, %zmm7
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm7 {%k2}
+; AVX512BW-NEXT:    vmovdqa 3200(%rdi), %ymm8
+; AVX512BW-NEXT:    vpalignr {{.*#+}} ymm8 = mem[8,9,10,11,12,13,14,15],ymm8[0,1,2,3,4,5,6,7],mem[24,25,26,27,28,29,30,31],ymm8[16,17,18,19,20,21,22,23]
+; AVX512BW-NEXT:    vextracti128 $1, %ymm8, %xmm8
+; AVX512BW-NEXT:    vinserti32x4 $0, %xmm8, %zmm30, %zmm8
+; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm8 {%k2}
+; AVX512BW-NEXT:    vmovdqa64 %zmm25, 448(%rsi)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm12, 384(%rsi)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm12, 320(%rsi)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm12, 256(%rsi)
+; AVX512BW-NEXT:    vmovdqa64 %zmm24, 192(%rsi)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm12, 128(%rsi)
+; AVX512BW-NEXT:    vmovdqa64 %zmm19, 64(%rsi)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm12, (%rsi)
+; AVX512BW-NEXT:    vmovdqa64 %zmm9, 448(%rdx)
 ; AVX512BW-NEXT:    vmovdqa64 %zmm2, 256(%rdx)
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, 320(%rdx)
-; AVX512BW-NEXT:    vmovdqa64 %zmm6, 128(%rdx)
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, 192(%rdx)
-; AVX512BW-NEXT:    vmovdqa64 %zmm9, (%rdx)
-; AVX512BW-NEXT:    vmovdqa64 %zmm12, 64(%rdx)
-; AVX512BW-NEXT:    vmovdqa64 %zmm4, 384(%rdx)
-; AVX512BW-NEXT:    vmovdqa64 %zmm3, 448(%rcx)
-; AVX512BW-NEXT:    vmovdqa64 %zmm8, 256(%rcx)
-; AVX512BW-NEXT:    vmovdqa64 %zmm14, 320(%rcx)
-; AVX512BW-NEXT:    vmovdqa64 %zmm15, 128(%rcx)
-; AVX512BW-NEXT:    vmovdqa64 %zmm30, 192(%rcx)
-; AVX512BW-NEXT:    vmovdqa64 %zmm29, (%rcx)
-; AVX512BW-NEXT:    vmovdqa64 %zmm28, 64(%rcx)
-; AVX512BW-NEXT:    vmovdqa64 %zmm31, 384(%rcx)
-; AVX512BW-NEXT:    vmovdqa64 %zmm24, 448(%r8)
-; AVX512BW-NEXT:    vmovdqa64 %zmm21, 256(%r8)
-; AVX512BW-NEXT:    vmovdqa64 %zmm22, 320(%r8)
-; AVX512BW-NEXT:    vmovdqa64 %zmm20, 128(%r8)
-; AVX512BW-NEXT:    vmovdqa64 %zmm19, 192(%r8)
-; AVX512BW-NEXT:    vmovdqa64 %zmm18, (%r8)
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, 64(%r8)
-; AVX512BW-NEXT:    vmovdqa64 %zmm17, 384(%r8)
+; AVX512BW-NEXT:    vmovdqa64 %zmm10, 320(%rdx)
+; AVX512BW-NEXT:    vmovdqa64 %zmm11, 128(%rdx)
+; AVX512BW-NEXT:    vmovdqa64 %zmm16, 192(%rdx)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm2, (%rdx)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm2, 64(%rdx)
+; AVX512BW-NEXT:    vmovdqa64 %zmm22, 384(%rdx)
+; AVX512BW-NEXT:    vmovdqa64 %zmm27, 448(%rcx)
+; AVX512BW-NEXT:    vmovdqa64 %zmm26, 256(%rcx)
+; AVX512BW-NEXT:    vmovdqa64 %zmm23, 320(%rcx)
+; AVX512BW-NEXT:    vmovdqa64 %zmm20, 128(%rcx)
+; AVX512BW-NEXT:    vmovdqa64 %zmm18, 192(%rcx)
+; AVX512BW-NEXT:    vmovdqa64 %zmm15, (%rcx)
+; AVX512BW-NEXT:    vmovdqa64 %zmm14, 64(%rcx)
+; AVX512BW-NEXT:    vmovdqa64 %zmm13, 384(%rcx)
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, 448(%r8)
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, 256(%r8)
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, 320(%r8)
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, 128(%r8)
+; AVX512BW-NEXT:    vmovdqa64 %zmm4, 192(%r8)
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, (%r8)
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, 64(%r8)
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, 384(%r8)
 ; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-NEXT:    vmovaps %zmm0, 448(%r9)
 ; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
@@ -9931,21 +10051,18 @@ define void @load_i64_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-NEXT:    vmovaps %zmm0, 384(%rax)
-; AVX512BW-NEXT:    vmovups (%rsp), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm0, 448(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm29, 448(%rax)
 ; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-NEXT:    vmovaps %zmm0, 256(%rax)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm0, 320(%rax)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm21, 320(%rax)
+; AVX512BW-NEXT:    vmovups (%rsp), %zmm0 # 64-byte Reload
 ; AVX512BW-NEXT:    vmovaps %zmm0, 128(%rax)
 ; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-NEXT:    vmovaps %zmm0, 192(%rax)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm0, (%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm17, (%rax)
 ; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-NEXT:    vmovaps %zmm0, 64(%rax)
-; AVX512BW-NEXT:    addq $6664, %rsp # imm = 0x1A08
+; AVX512BW-NEXT:    addq $7624, %rsp # imm = 0x1DC8
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
   %wide.vec = load <448 x i64>, ptr %in.vec, align 64
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i64-stride-8.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i64-stride-8.ll
index 4738b2344255b59cc3d8cf0340982d8d2de8e198..638f7f685319bdf750dc95d9ae564c664923bd25 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i64-stride-8.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i64-stride-8.ll
@@ -815,229 +815,225 @@ define void @load_i64_stride8_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ;
 ; AVX512F-LABEL: load_i64_stride8_vf8:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa64 192(%rdi), %zmm18
-; AVX512F-NEXT:    vmovdqa64 128(%rdi), %zmm1
-; AVX512F-NEXT:    vmovdqa64 64(%rdi), %zmm4
-; AVX512F-NEXT:    vmovdqa64 (%rdi), %zmm2
-; AVX512F-NEXT:    vmovdqa64 320(%rdi), %zmm12
-; AVX512F-NEXT:    vmovdqa64 256(%rdi), %zmm3
-; AVX512F-NEXT:    vmovdqa64 448(%rdi), %zmm7
-; AVX512F-NEXT:    vmovdqa64 384(%rdi), %zmm11
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm5 = [0,8,0,8,0,8,0,8]
-; AVX512F-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm6
-; AVX512F-NEXT:    vpermt2q %zmm7, %zmm5, %zmm6
-; AVX512F-NEXT:    vpermi2q %zmm12, %zmm3, %zmm5
-; AVX512F-NEXT:    movb $-64, %al
-; AVX512F-NEXT:    kmovw %eax, %k1
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm8 = [1,9,1,9,1,9,1,9]
+; AVX512F-NEXT:    pushq %rbx
+; AVX512F-NEXT:    movq {{[0-9]+}}(%rsp), %rax
+; AVX512F-NEXT:    movq {{[0-9]+}}(%rsp), %r10
+; AVX512F-NEXT:    movq {{[0-9]+}}(%rsp), %r11
+; AVX512F-NEXT:    vmovdqa64 64(%rdi), %zmm2
+; AVX512F-NEXT:    vmovdqa64 (%rdi), %zmm1
+; AVX512F-NEXT:    vmovdqa64 192(%rdi), %zmm7
+; AVX512F-NEXT:    vmovdqa64 128(%rdi), %zmm6
+; AVX512F-NEXT:    vmovdqa64 320(%rdi), %zmm3
+; AVX512F-NEXT:    vmovdqa64 256(%rdi), %zmm0
+; AVX512F-NEXT:    vmovdqa64 448(%rdi), %zmm4
+; AVX512F-NEXT:    vmovdqa64 384(%rdi), %zmm5
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm8 = [0,8,0,8,0,8,0,8]
 ; AVX512F-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm10
-; AVX512F-NEXT:    vpermt2q %zmm7, %zmm8, %zmm10
-; AVX512F-NEXT:    vpermi2q %zmm12, %zmm3, %zmm8
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm9 = [2,10,2,10,2,10,2,10]
-; AVX512F-NEXT:    # zmm9 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm13
-; AVX512F-NEXT:    vpermt2q %zmm7, %zmm9, %zmm13
-; AVX512F-NEXT:    vpermi2q %zmm12, %zmm3, %zmm9
-; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm9 {%k1}
-; AVX512F-NEXT:    vmovdqa 192(%rdi), %ymm13
-; AVX512F-NEXT:    vmovdqa 128(%rdi), %ymm14
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm15 = ymm14[0],ymm13[0],ymm14[2],ymm13[2]
-; AVX512F-NEXT:    vmovdqa64 64(%rdi), %ymm16
-; AVX512F-NEXT:    vmovdqa64 (%rdi), %ymm17
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm17[0],ymm16[0],ymm17[2],ymm16[2]
-; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm0[2,3],ymm15[2,3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm9, %zmm9
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [3,11,3,11,3,11,3,11]
-; AVX512F-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm15
-; AVX512F-NEXT:    vpermt2q %zmm7, %zmm0, %zmm15
-; AVX512F-NEXT:    vpermi2q %zmm12, %zmm3, %zmm0
-; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm0 {%k1}
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm13 = ymm14[1],ymm13[1],ymm14[3],ymm13[3]
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm14 = ymm17[1],ymm16[1],ymm17[3],ymm16[3]
-; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm13 = ymm14[2,3],ymm13[2,3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm13, %zmm0, %zmm13
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm14 = [4,12,4,12,4,12,4,12]
-; AVX512F-NEXT:    # zmm14 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm0 = zmm3[0],zmm12[0],zmm3[2],zmm12[2],zmm3[4],zmm12[4],zmm3[6],zmm12[6]
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm16 = zmm3[1],zmm12[1],zmm3[3],zmm12[3],zmm3[5],zmm12[5],zmm3[7],zmm12[7]
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm15 = [6,14,6,14,6,14,6,14]
-; AVX512F-NEXT:    # zmm15 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm17
-; AVX512F-NEXT:    vpermt2q %zmm12, %zmm15, %zmm17
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm19 = [7,15,7,15,7,15,7,15]
-; AVX512F-NEXT:    # zmm19 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermt2q %zmm12, %zmm19, %zmm3
-; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm12
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm17 {%k1} = zmm11[0],zmm7[0],zmm11[2],zmm7[2],zmm11[4],zmm7[4],zmm11[6],zmm7[6]
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm3 {%k1} = zmm11[1],zmm7[1],zmm11[3],zmm7[3],zmm11[5],zmm7[5],zmm11[7],zmm7[7]
-; AVX512F-NEXT:    vpermt2q %zmm7, %zmm14, %zmm11
-; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm0 {%k1}
-; AVX512F-NEXT:    vpermi2q %zmm4, %zmm2, %zmm14
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm11 = [4,12,4,12]
-; AVX512F-NEXT:    # ymm11 = mem[0,1,0,1]
-; AVX512F-NEXT:    vpermi2q %zmm18, %zmm1, %zmm11
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm11 = ymm14[0,1,2,3],ymm11[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm11, %zmm0, %zmm0
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm11 = [5,13,5,13,5,13,5,13]
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm9
+; AVX512F-NEXT:    vpermt2q %zmm4, %zmm8, %zmm9
+; AVX512F-NEXT:    vpermi2q %zmm3, %zmm0, %zmm8
+; AVX512F-NEXT:    movb $-64, %bl
+; AVX512F-NEXT:    kmovw %ebx, %k1
+; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm8 {%k1}
+; AVX512F-NEXT:    vmovdqa (%rdi), %xmm9
+; AVX512F-NEXT:    vmovdqa 64(%rdi), %xmm10
+; AVX512F-NEXT:    vinserti128 $1, 192(%rdi), %ymm10, %ymm10
+; AVX512F-NEXT:    vinserti128 $1, 128(%rdi), %ymm9, %ymm9
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm11 = ymm9[0],ymm10[0],ymm9[2],ymm10[2]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm11, %zmm8, %zmm16
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm11 = [1,9,1,9,1,9,1,9]
 ; AVX512F-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm12
+; AVX512F-NEXT:    vpermt2q %zmm4, %zmm11, %zmm12
+; AVX512F-NEXT:    vpermi2q %zmm3, %zmm0, %zmm11
+; AVX512F-NEXT:    vmovdqa64 %zmm12, %zmm11 {%k1}
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm9 = ymm9[1],ymm10[1],ymm9[3],ymm10[3]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm9, %zmm11, %zmm9
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm10 = [2,10,2,10,2,10,2,10]
+; AVX512F-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm11
+; AVX512F-NEXT:    vpermt2q %zmm4, %zmm10, %zmm11
+; AVX512F-NEXT:    vpermi2q %zmm3, %zmm0, %zmm10
+; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm10 {%k1}
+; AVX512F-NEXT:    vmovdqa 192(%rdi), %ymm11
+; AVX512F-NEXT:    vmovdqa 128(%rdi), %ymm12
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm13 = ymm12[0],ymm11[0],ymm12[2],ymm11[2]
+; AVX512F-NEXT:    vmovdqa 64(%rdi), %ymm14
+; AVX512F-NEXT:    vmovdqa (%rdi), %ymm15
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm8 = ymm15[0],ymm14[0],ymm15[2],ymm14[2]
+; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm8 = ymm8[2,3],ymm13[2,3]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm8, %zmm10, %zmm10
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm8 = [3,11,3,11,3,11,3,11]
+; AVX512F-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm13
+; AVX512F-NEXT:    vpermt2q %zmm4, %zmm8, %zmm13
+; AVX512F-NEXT:    vpermi2q %zmm3, %zmm0, %zmm8
+; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm8 {%k1}
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm11 = ymm12[1],ymm11[1],ymm12[3],ymm11[3]
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm12 = ymm15[1],ymm14[1],ymm15[3],ymm14[3]
+; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm11 = ymm12[2,3],ymm11[2,3]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm11, %zmm8, %zmm8
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm11 = [4,12,4,12,4,12,4,12]
+; AVX512F-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm12
 ; AVX512F-NEXT:    vpermt2q %zmm7, %zmm11, %zmm12
-; AVX512F-NEXT:    vpermi2q %zmm4, %zmm2, %zmm11
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm7 = [5,13,5,13]
-; AVX512F-NEXT:    # ymm7 = mem[0,1,0,1]
-; AVX512F-NEXT:    vpermi2q %zmm18, %zmm1, %zmm7
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm7 = ymm11[0,1,2,3],ymm7[4,5,6,7]
-; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm5 {%k1}
-; AVX512F-NEXT:    vmovdqa (%rdi), %xmm6
-; AVX512F-NEXT:    vmovdqa 64(%rdi), %xmm11
-; AVX512F-NEXT:    vinserti128 $1, 192(%rdi), %ymm11, %ymm11
-; AVX512F-NEXT:    vinserti128 $1, 128(%rdi), %ymm6, %ymm6
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm14 = ymm6[0],ymm11[0],ymm6[2],ymm11[2]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm14, %zmm5, %zmm5
-; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm8 {%k1}
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm6 = ymm6[1],ymm11[1],ymm6[3],ymm11[3]
-; AVX512F-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512F-NEXT:    movq {{[0-9]+}}(%rsp), %rdi
-; AVX512F-NEXT:    movq {{[0-9]+}}(%rsp), %r10
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm6, %zmm8, %zmm6
-; AVX512F-NEXT:    vmovdqa64 %zmm12, %zmm16 {%k1}
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm7, %zmm16, %zmm7
-; AVX512F-NEXT:    vpermi2q %zmm4, %zmm2, %zmm15
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm8 = [6,14,6,14]
-; AVX512F-NEXT:    # ymm8 = mem[0,1,0,1]
-; AVX512F-NEXT:    vpermi2q %zmm18, %zmm1, %zmm8
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm8 = ymm15[0,1,2,3],ymm8[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm8, %zmm17, %zmm8
-; AVX512F-NEXT:    vpermt2q %zmm4, %zmm19, %zmm2
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm4 = [7,15,7,15]
-; AVX512F-NEXT:    # ymm4 = mem[0,1,0,1]
-; AVX512F-NEXT:    vpermi2q %zmm18, %zmm1, %zmm4
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm4[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm3, %zmm1
-; AVX512F-NEXT:    vmovdqa64 %zmm5, (%rsi)
-; AVX512F-NEXT:    vmovdqa64 %zmm6, (%rdx)
-; AVX512F-NEXT:    vmovdqa64 %zmm9, (%rcx)
-; AVX512F-NEXT:    vmovdqa64 %zmm13, (%r8)
-; AVX512F-NEXT:    vmovdqa64 %zmm0, (%r9)
-; AVX512F-NEXT:    vmovdqa64 %zmm7, (%r10)
-; AVX512F-NEXT:    vmovdqa64 %zmm8, (%rdi)
-; AVX512F-NEXT:    vmovdqa64 %zmm1, (%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm13
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm11, %zmm13
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm12 = ymm13[0,1,2,3],ymm12[4,5,6,7]
+; AVX512F-NEXT:    vpermi2q %zmm4, %zmm5, %zmm11
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm13 = zmm0[0],zmm3[0],zmm0[2],zmm3[2],zmm0[4],zmm3[4],zmm0[6],zmm3[6]
+; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm13 {%k1}
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm12, %zmm13, %zmm11
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm12 = [5,13,5,13,5,13,5,13]
+; AVX512F-NEXT:    # zmm12 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm13
+; AVX512F-NEXT:    vpermt2q %zmm7, %zmm12, %zmm13
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm14
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm12, %zmm14
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm13 = ymm14[0,1,2,3],ymm13[4,5,6,7]
+; AVX512F-NEXT:    vpermi2q %zmm4, %zmm5, %zmm12
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm14 = zmm0[1],zmm3[1],zmm0[3],zmm3[3],zmm0[5],zmm3[5],zmm0[7],zmm3[7]
+; AVX512F-NEXT:    vmovdqa64 %zmm12, %zmm14 {%k1}
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm13, %zmm14, %zmm12
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm13 = [6,14,6,14,6,14,6,14]
+; AVX512F-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm14
+; AVX512F-NEXT:    vpermt2q %zmm7, %zmm13, %zmm14
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm15
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm13, %zmm15
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm14 = ymm15[0,1,2,3],ymm14[4,5,6,7]
+; AVX512F-NEXT:    vpermi2q %zmm3, %zmm0, %zmm13
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm13 {%k1} = zmm5[0],zmm4[0],zmm5[2],zmm4[2],zmm5[4],zmm4[4],zmm5[6],zmm4[6]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm14, %zmm13, %zmm13
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm14 = [7,15,7,15,7,15,7,15]
+; AVX512F-NEXT:    # zmm14 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpermt2q %zmm7, %zmm14, %zmm6
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm14, %zmm1
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm6[4,5,6,7]
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm14, %zmm0
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm0 {%k1} = zmm5[1],zmm4[1],zmm5[3],zmm4[3],zmm5[5],zmm4[5],zmm5[7],zmm4[7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vmovdqa64 %zmm16, (%rsi)
+; AVX512F-NEXT:    vmovdqa64 %zmm9, (%rdx)
+; AVX512F-NEXT:    vmovdqa64 %zmm10, (%rcx)
+; AVX512F-NEXT:    vmovdqa64 %zmm8, (%r8)
+; AVX512F-NEXT:    vmovdqa64 %zmm11, (%r9)
+; AVX512F-NEXT:    vmovdqa64 %zmm12, (%r11)
+; AVX512F-NEXT:    vmovdqa64 %zmm13, (%r10)
+; AVX512F-NEXT:    vmovdqa64 %zmm0, (%rax)
+; AVX512F-NEXT:    popq %rbx
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: load_i64_stride8_vf8:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    vmovdqa64 192(%rdi), %zmm18
-; AVX512BW-NEXT:    vmovdqa64 128(%rdi), %zmm1
-; AVX512BW-NEXT:    vmovdqa64 64(%rdi), %zmm4
-; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm2
-; AVX512BW-NEXT:    vmovdqa64 320(%rdi), %zmm12
-; AVX512BW-NEXT:    vmovdqa64 256(%rdi), %zmm3
-; AVX512BW-NEXT:    vmovdqa64 448(%rdi), %zmm7
-; AVX512BW-NEXT:    vmovdqa64 384(%rdi), %zmm11
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm5 = [0,8,0,8,0,8,0,8]
-; AVX512BW-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm6
-; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm5, %zmm6
-; AVX512BW-NEXT:    vpermi2q %zmm12, %zmm3, %zmm5
-; AVX512BW-NEXT:    movb $-64, %al
-; AVX512BW-NEXT:    kmovd %eax, %k1
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm8 = [1,9,1,9,1,9,1,9]
+; AVX512BW-NEXT:    pushq %rbx
+; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
+; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %r10
+; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %r11
+; AVX512BW-NEXT:    vmovdqa64 64(%rdi), %zmm2
+; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm1
+; AVX512BW-NEXT:    vmovdqa64 192(%rdi), %zmm7
+; AVX512BW-NEXT:    vmovdqa64 128(%rdi), %zmm6
+; AVX512BW-NEXT:    vmovdqa64 320(%rdi), %zmm3
+; AVX512BW-NEXT:    vmovdqa64 256(%rdi), %zmm0
+; AVX512BW-NEXT:    vmovdqa64 448(%rdi), %zmm4
+; AVX512BW-NEXT:    vmovdqa64 384(%rdi), %zmm5
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm8 = [0,8,0,8,0,8,0,8]
 ; AVX512BW-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm10
-; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm8, %zmm10
-; AVX512BW-NEXT:    vpermi2q %zmm12, %zmm3, %zmm8
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm9 = [2,10,2,10,2,10,2,10]
-; AVX512BW-NEXT:    # zmm9 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm13
-; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm9, %zmm13
-; AVX512BW-NEXT:    vpermi2q %zmm12, %zmm3, %zmm9
-; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm9 {%k1}
-; AVX512BW-NEXT:    vmovdqa 192(%rdi), %ymm13
-; AVX512BW-NEXT:    vmovdqa 128(%rdi), %ymm14
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm15 = ymm14[0],ymm13[0],ymm14[2],ymm13[2]
-; AVX512BW-NEXT:    vmovdqa64 64(%rdi), %ymm16
-; AVX512BW-NEXT:    vmovdqa64 (%rdi), %ymm17
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm17[0],ymm16[0],ymm17[2],ymm16[2]
-; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm0[2,3],ymm15[2,3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm9, %zmm9
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [3,11,3,11,3,11,3,11]
-; AVX512BW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm15
-; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm0, %zmm15
-; AVX512BW-NEXT:    vpermi2q %zmm12, %zmm3, %zmm0
-; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm0 {%k1}
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm13 = ymm14[1],ymm13[1],ymm14[3],ymm13[3]
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm14 = ymm17[1],ymm16[1],ymm17[3],ymm16[3]
-; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm13 = ymm14[2,3],ymm13[2,3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm13, %zmm0, %zmm13
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm14 = [4,12,4,12,4,12,4,12]
-; AVX512BW-NEXT:    # zmm14 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm0 = zmm3[0],zmm12[0],zmm3[2],zmm12[2],zmm3[4],zmm12[4],zmm3[6],zmm12[6]
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm16 = zmm3[1],zmm12[1],zmm3[3],zmm12[3],zmm3[5],zmm12[5],zmm3[7],zmm12[7]
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm15 = [6,14,6,14,6,14,6,14]
-; AVX512BW-NEXT:    # zmm15 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm17
-; AVX512BW-NEXT:    vpermt2q %zmm12, %zmm15, %zmm17
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm19 = [7,15,7,15,7,15,7,15]
-; AVX512BW-NEXT:    # zmm19 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermt2q %zmm12, %zmm19, %zmm3
-; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm12
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm17 {%k1} = zmm11[0],zmm7[0],zmm11[2],zmm7[2],zmm11[4],zmm7[4],zmm11[6],zmm7[6]
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm3 {%k1} = zmm11[1],zmm7[1],zmm11[3],zmm7[3],zmm11[5],zmm7[5],zmm11[7],zmm7[7]
-; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm14, %zmm11
-; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm0 {%k1}
-; AVX512BW-NEXT:    vpermi2q %zmm4, %zmm2, %zmm14
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm11 = [4,12,4,12]
-; AVX512BW-NEXT:    # ymm11 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vpermi2q %zmm18, %zmm1, %zmm11
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm11 = ymm14[0,1,2,3],ymm11[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm11, %zmm0, %zmm0
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm11 = [5,13,5,13,5,13,5,13]
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm9
+; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm8, %zmm9
+; AVX512BW-NEXT:    vpermi2q %zmm3, %zmm0, %zmm8
+; AVX512BW-NEXT:    movb $-64, %bl
+; AVX512BW-NEXT:    kmovd %ebx, %k1
+; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm8 {%k1}
+; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm9
+; AVX512BW-NEXT:    vmovdqa 64(%rdi), %xmm10
+; AVX512BW-NEXT:    vinserti128 $1, 192(%rdi), %ymm10, %ymm10
+; AVX512BW-NEXT:    vinserti128 $1, 128(%rdi), %ymm9, %ymm9
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm11 = ymm9[0],ymm10[0],ymm9[2],ymm10[2]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm11, %zmm8, %zmm16
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm11 = [1,9,1,9,1,9,1,9]
 ; AVX512BW-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm12
+; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm11, %zmm12
+; AVX512BW-NEXT:    vpermi2q %zmm3, %zmm0, %zmm11
+; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm11 {%k1}
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm9 = ymm9[1],ymm10[1],ymm9[3],ymm10[3]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm9, %zmm11, %zmm9
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm10 = [2,10,2,10,2,10,2,10]
+; AVX512BW-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm11
+; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm10, %zmm11
+; AVX512BW-NEXT:    vpermi2q %zmm3, %zmm0, %zmm10
+; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm10 {%k1}
+; AVX512BW-NEXT:    vmovdqa 192(%rdi), %ymm11
+; AVX512BW-NEXT:    vmovdqa 128(%rdi), %ymm12
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm13 = ymm12[0],ymm11[0],ymm12[2],ymm11[2]
+; AVX512BW-NEXT:    vmovdqa 64(%rdi), %ymm14
+; AVX512BW-NEXT:    vmovdqa (%rdi), %ymm15
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm8 = ymm15[0],ymm14[0],ymm15[2],ymm14[2]
+; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm8 = ymm8[2,3],ymm13[2,3]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm8, %zmm10, %zmm10
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm8 = [3,11,3,11,3,11,3,11]
+; AVX512BW-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm13
+; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm8, %zmm13
+; AVX512BW-NEXT:    vpermi2q %zmm3, %zmm0, %zmm8
+; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm8 {%k1}
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm11 = ymm12[1],ymm11[1],ymm12[3],ymm11[3]
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm12 = ymm15[1],ymm14[1],ymm15[3],ymm14[3]
+; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm11 = ymm12[2,3],ymm11[2,3]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm11, %zmm8, %zmm8
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm11 = [4,12,4,12,4,12,4,12]
+; AVX512BW-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm12
 ; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm11, %zmm12
-; AVX512BW-NEXT:    vpermi2q %zmm4, %zmm2, %zmm11
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm7 = [5,13,5,13]
-; AVX512BW-NEXT:    # ymm7 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vpermi2q %zmm18, %zmm1, %zmm7
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm7 = ymm11[0,1,2,3],ymm7[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm5 {%k1}
-; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm6
-; AVX512BW-NEXT:    vmovdqa 64(%rdi), %xmm11
-; AVX512BW-NEXT:    vinserti128 $1, 192(%rdi), %ymm11, %ymm11
-; AVX512BW-NEXT:    vinserti128 $1, 128(%rdi), %ymm6, %ymm6
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm14 = ymm6[0],ymm11[0],ymm6[2],ymm11[2]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm14, %zmm5, %zmm5
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm8 {%k1}
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm6 = ymm6[1],ymm11[1],ymm6[3],ymm11[3]
-; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %rdi
-; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %r10
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm6, %zmm8, %zmm6
-; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm16 {%k1}
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm7, %zmm16, %zmm7
-; AVX512BW-NEXT:    vpermi2q %zmm4, %zmm2, %zmm15
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm8 = [6,14,6,14]
-; AVX512BW-NEXT:    # ymm8 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vpermi2q %zmm18, %zmm1, %zmm8
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm15[0,1,2,3],ymm8[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm8, %zmm17, %zmm8
-; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm19, %zmm2
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm4 = [7,15,7,15]
-; AVX512BW-NEXT:    # ymm4 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vpermi2q %zmm18, %zmm1, %zmm4
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm4[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm3, %zmm1
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, (%rsi)
-; AVX512BW-NEXT:    vmovdqa64 %zmm6, (%rdx)
-; AVX512BW-NEXT:    vmovdqa64 %zmm9, (%rcx)
-; AVX512BW-NEXT:    vmovdqa64 %zmm13, (%r8)
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%r9)
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, (%r10)
-; AVX512BW-NEXT:    vmovdqa64 %zmm8, (%rdi)
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, (%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm13
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm11, %zmm13
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm12 = ymm13[0,1,2,3],ymm12[4,5,6,7]
+; AVX512BW-NEXT:    vpermi2q %zmm4, %zmm5, %zmm11
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm13 = zmm0[0],zmm3[0],zmm0[2],zmm3[2],zmm0[4],zmm3[4],zmm0[6],zmm3[6]
+; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm13 {%k1}
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm12, %zmm13, %zmm11
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm12 = [5,13,5,13,5,13,5,13]
+; AVX512BW-NEXT:    # zmm12 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm13
+; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm12, %zmm13
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm14
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm12, %zmm14
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm13 = ymm14[0,1,2,3],ymm13[4,5,6,7]
+; AVX512BW-NEXT:    vpermi2q %zmm4, %zmm5, %zmm12
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm14 = zmm0[1],zmm3[1],zmm0[3],zmm3[3],zmm0[5],zmm3[5],zmm0[7],zmm3[7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm14 {%k1}
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm13, %zmm14, %zmm12
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm13 = [6,14,6,14,6,14,6,14]
+; AVX512BW-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm14
+; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm13, %zmm14
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm15
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm13, %zmm15
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm14 = ymm15[0,1,2,3],ymm14[4,5,6,7]
+; AVX512BW-NEXT:    vpermi2q %zmm3, %zmm0, %zmm13
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm13 {%k1} = zmm5[0],zmm4[0],zmm5[2],zmm4[2],zmm5[4],zmm4[4],zmm5[6],zmm4[6]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm14, %zmm13, %zmm13
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm14 = [7,15,7,15,7,15,7,15]
+; AVX512BW-NEXT:    # zmm14 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm14, %zmm6
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm14, %zmm1
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm6[4,5,6,7]
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm14, %zmm0
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm0 {%k1} = zmm5[1],zmm4[1],zmm5[3],zmm4[3],zmm5[5],zmm4[5],zmm5[7],zmm4[7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vmovdqa64 %zmm16, (%rsi)
+; AVX512BW-NEXT:    vmovdqa64 %zmm9, (%rdx)
+; AVX512BW-NEXT:    vmovdqa64 %zmm10, (%rcx)
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, (%r8)
+; AVX512BW-NEXT:    vmovdqa64 %zmm11, (%r9)
+; AVX512BW-NEXT:    vmovdqa64 %zmm12, (%r11)
+; AVX512BW-NEXT:    vmovdqa64 %zmm13, (%r10)
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rax)
+; AVX512BW-NEXT:    popq %rbx
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
   %wide.vec = load <64 x i64>, ptr %in.vec, align 64
@@ -1926,221 +1922,222 @@ define void @load_i64_stride8_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512F-LABEL: load_i64_stride8_vf16:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    subq $264, %rsp # imm = 0x108
-; AVX512F-NEXT:    vmovdqa64 704(%rdi), %zmm18
-; AVX512F-NEXT:    vmovaps 640(%rdi), %zmm0
+; AVX512F-NEXT:    vmovaps 576(%rdi), %zmm0
 ; AVX512F-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 576(%rdi), %zmm31
-; AVX512F-NEXT:    vmovdqa64 512(%rdi), %zmm3
-; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 192(%rdi), %zmm17
-; AVX512F-NEXT:    vmovdqa64 128(%rdi), %zmm4
-; AVX512F-NEXT:    vmovdqa64 832(%rdi), %zmm11
-; AVX512F-NEXT:    vmovdqa64 768(%rdi), %zmm28
-; AVX512F-NEXT:    vmovdqa64 960(%rdi), %zmm9
-; AVX512F-NEXT:    vmovdqa64 896(%rdi), %zmm10
-; AVX512F-NEXT:    vmovdqa64 320(%rdi), %zmm14
-; AVX512F-NEXT:    vmovdqa64 256(%rdi), %zmm24
-; AVX512F-NEXT:    vmovdqa64 448(%rdi), %zmm12
-; AVX512F-NEXT:    vmovdqa64 384(%rdi), %zmm13
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm19 = [0,8,0,8,0,8,0,8]
-; AVX512F-NEXT:    # zmm19 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm5
-; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm7
-; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm6
-; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm12, %zmm19, %zmm0
-; AVX512F-NEXT:    vmovdqa64 %zmm24, %zmm15
-; AVX512F-NEXT:    vpermt2q %zmm14, %zmm19, %zmm15
+; AVX512F-NEXT:    vmovaps 512(%rdi), %zmm0
+; AVX512F-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 704(%rdi), %zmm6
+; AVX512F-NEXT:    vmovdqa64 640(%rdi), %zmm4
+; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 64(%rdi), %zmm7
+; AVX512F-NEXT:    vmovaps (%rdi), %zmm0
+; AVX512F-NEXT:    vmovups %zmm0, (%rsp) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 192(%rdi), %zmm31
+; AVX512F-NEXT:    vmovdqa64 128(%rdi), %zmm8
+; AVX512F-NEXT:    vmovdqa64 832(%rdi), %zmm10
+; AVX512F-NEXT:    vmovdqa64 768(%rdi), %zmm14
+; AVX512F-NEXT:    vmovdqa64 960(%rdi), %zmm30
+; AVX512F-NEXT:    vmovdqa64 896(%rdi), %zmm28
+; AVX512F-NEXT:    vmovdqa64 320(%rdi), %zmm16
+; AVX512F-NEXT:    vmovdqa64 256(%rdi), %zmm5
+; AVX512F-NEXT:    vmovdqa64 448(%rdi), %zmm3
+; AVX512F-NEXT:    vmovdqa64 384(%rdi), %zmm15
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm29 = [0,8,0,8,0,8,0,8]
+; AVX512F-NEXT:    # zmm29 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm17
+; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm20
+; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm9
+; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm19
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm29, %zmm19
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm18
+; AVX512F-NEXT:    vpermt2q %zmm16, %zmm29, %zmm18
 ; AVX512F-NEXT:    movb $-64, %al
 ; AVX512F-NEXT:    kmovw %eax, %k1
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm15 {%k1}
-; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512F-NEXT:    vmovdqa64 64(%rdi), %xmm16
-; AVX512F-NEXT:    vinserti32x4 $1, 192(%rdi), %ymm16, %ymm21
-; AVX512F-NEXT:    vinserti32x4 $1, 128(%rdi), %ymm0, %ymm16
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm16[0],ymm21[0],ymm16[2],ymm21[2]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm15, %zmm0
+; AVX512F-NEXT:    vmovdqa64 %zmm19, %zmm18 {%k1}
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm19 = [2,10,2,10,2,10,2,10]
+; AVX512F-NEXT:    # zmm19 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm19, %zmm17
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm21
+; AVX512F-NEXT:    vpermt2q %zmm16, %zmm19, %zmm21
+; AVX512F-NEXT:    vmovdqa64 %zmm17, %zmm21 {%k1}
+; AVX512F-NEXT:    vmovdqa64 192(%rdi), %ymm22
+; AVX512F-NEXT:    vmovdqa64 128(%rdi), %ymm23
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm11 = ymm23[0],ymm22[0],ymm23[2],ymm22[2]
+; AVX512F-NEXT:    vmovdqa64 64(%rdi), %ymm24
+; AVX512F-NEXT:    vmovdqa64 (%rdi), %ymm25
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm12 = ymm25[0],ymm24[0],ymm25[2],ymm24[2]
+; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm11 = ymm12[2,3],ymm11[2,3]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm11, %zmm21, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm9, %zmm19, %zmm0
-; AVX512F-NEXT:    vpermi2q %zmm11, %zmm28, %zmm19
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm19 {%k1}
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [2,10,2,10,2,10,2,10]
-; AVX512F-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermt2q %zmm12, %zmm0, %zmm5
-; AVX512F-NEXT:    vmovdqa64 %zmm24, %zmm15
-; AVX512F-NEXT:    vpermt2q %zmm14, %zmm0, %zmm15
-; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm15 {%k1}
-; AVX512F-NEXT:    vmovdqa 192(%rdi), %ymm5
-; AVX512F-NEXT:    vmovdqa64 128(%rdi), %ymm20
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm1 = ymm20[0],ymm5[0],ymm20[2],ymm5[2]
-; AVX512F-NEXT:    vmovdqa64 64(%rdi), %ymm22
-; AVX512F-NEXT:    vmovdqa64 (%rdi), %ymm23
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm2 = ymm23[0],ymm22[0],ymm23[2],ymm22[2]
-; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm1 = ymm2[2,3],ymm1[2,3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm15, %zmm1
-; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm1
-; AVX512F-NEXT:    vpermt2q %zmm9, %zmm0, %zmm1
-; AVX512F-NEXT:    vpermi2q %zmm11, %zmm28, %zmm0
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm0 {%k1}
-; AVX512F-NEXT:    vmovdqa 704(%rdi), %ymm1
-; AVX512F-NEXT:    vmovdqa 640(%rdi), %ymm2
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm15 = ymm2[0],ymm1[0],ymm2[2],ymm1[2]
-; AVX512F-NEXT:    vmovdqa64 576(%rdi), %ymm25
+; AVX512F-NEXT:    vmovdqa64 %zmm28, %zmm11
+; AVX512F-NEXT:    vpermt2q %zmm30, %zmm19, %zmm11
+; AVX512F-NEXT:    vpermi2q %zmm10, %zmm14, %zmm19
+; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm19 {%k1}
+; AVX512F-NEXT:    vmovdqa 704(%rdi), %ymm11
+; AVX512F-NEXT:    vmovdqa 640(%rdi), %ymm12
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm13 = ymm12[0],ymm11[0],ymm12[2],ymm11[2]
+; AVX512F-NEXT:    vmovdqa64 576(%rdi), %ymm26
 ; AVX512F-NEXT:    vmovdqa64 512(%rdi), %ymm27
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm8 = ymm27[0],ymm25[0],ymm27[2],ymm25[2]
-; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm8 = ymm8[2,3],ymm15[2,3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm8, %zmm0, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, (%rsp) # 64-byte Spill
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm27[0],ymm26[0],ymm27[2],ymm26[2]
+; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm0[2,3],ymm13[2,3]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm19, %zmm19
 ; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [3,11,3,11,3,11,3,11]
 ; AVX512F-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermt2q %zmm12, %zmm0, %zmm7
-; AVX512F-NEXT:    vmovdqa64 %zmm24, %zmm30
-; AVX512F-NEXT:    vpermt2q %zmm14, %zmm0, %zmm30
-; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm30 {%k1}
-; AVX512F-NEXT:    vmovdqa64 64(%rdi), %zmm26
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm5 = ymm20[1],ymm5[1],ymm20[3],ymm5[3]
-; AVX512F-NEXT:    vmovdqa64 (%rdi), %zmm15
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm7 = ymm23[1],ymm22[1],ymm23[3],ymm22[3]
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm23 = [1,9,1,9,1,9,1,9]
-; AVX512F-NEXT:    # zmm23 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm5 = ymm7[2,3],ymm5[2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm24, %zmm29
-; AVX512F-NEXT:    vpermt2q %zmm14, %zmm23, %zmm29
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm5, %zmm30, %zmm5
-; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm5
-; AVX512F-NEXT:    vpermt2q %zmm9, %zmm0, %zmm5
-; AVX512F-NEXT:    vpermi2q %zmm11, %zmm28, %zmm0
-; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm0 {%k1}
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm2[1],ymm1[1],ymm2[3],ymm1[3]
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm2 = ymm27[1],ymm25[1],ymm27[3],ymm25[3]
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm0, %zmm20
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm13
+; AVX512F-NEXT:    vpermt2q %zmm16, %zmm0, %zmm13
+; AVX512F-NEXT:    vmovdqa64 %zmm20, %zmm13 {%k1}
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm21 = [1,9,1,9,1,9,1,9]
+; AVX512F-NEXT:    # zmm21 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm23[1],ymm22[1],ymm23[3],ymm22[3]
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm23
+; AVX512F-NEXT:    vpermt2q %zmm16, %zmm21, %zmm23
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm2 = ymm25[1],ymm24[1],ymm25[3],ymm24[3]
+; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm1 = ymm2[2,3],ymm1[2,3]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm13, %zmm20
+; AVX512F-NEXT:    vmovdqa64 %zmm28, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm30, %zmm0, %zmm1
+; AVX512F-NEXT:    vpermi2q %zmm10, %zmm14, %zmm0
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm0 {%k1}
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm12[1],ymm11[1],ymm12[3],ymm11[3]
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm2 = ymm27[1],ymm26[1],ymm27[3],ymm26[3]
 ; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm1 = ymm2[2,3],ymm1[2,3]
 ; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm22
 ; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [4,12,4,12,4,12,4,12]
 ; AVX512F-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermt2q %zmm12, %zmm0, %zmm6
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm1 = zmm24[0],zmm14[0],zmm24[2],zmm14[2],zmm24[4],zmm14[4],zmm24[6],zmm14[6]
-; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm1 {%k1}
-; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm26, %zmm0, %zmm2
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm5 = [4,12,4,12]
-; AVX512F-NEXT:    # ymm5 = mem[0,1,0,1]
-; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm6
-; AVX512F-NEXT:    vpermt2q %zmm17, %zmm5, %zmm6
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm6[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm1, %zmm25
-; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm1
-; AVX512F-NEXT:    vpermt2q %zmm9, %zmm0, %zmm1
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm2 = zmm28[0],zmm11[0],zmm28[2],zmm11[2],zmm28[4],zmm11[4],zmm28[6],zmm11[6]
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}
-; AVX512F-NEXT:    vpermi2q %zmm31, %zmm3, %zmm0
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
-; AVX512F-NEXT:    vpermi2q %zmm18, %zmm8, %zmm5
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm2, %zmm27
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm6 = [5,13,5,13,5,13,5,13]
-; AVX512F-NEXT:    # zmm6 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm26, %zmm6, %zmm0
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm7 = [5,13,5,13]
-; AVX512F-NEXT:    # ymm7 = mem[0,1,0,1]
+; AVX512F-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm31, %zmm0, %zmm1
+; AVX512F-NEXT:    vmovdqu64 (%rsp), %zmm13 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm11
+; AVX512F-NEXT:    vpermt2q %zmm7, %zmm0, %zmm2
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm0, %zmm9
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm2 = zmm5[0],zmm16[0],zmm5[2],zmm16[2],zmm5[4],zmm16[4],zmm5[6],zmm16[6]
+; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm2 {%k1}
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm2, %zmm24
 ; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm1
-; AVX512F-NEXT:    vpermt2q %zmm17, %zmm7, %zmm1
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm5 = [6,14,6,14,6,14,6,14]
-; AVX512F-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm24, %zmm30
-; AVX512F-NEXT:    vpermt2q %zmm14, %zmm5, %zmm30
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm1 = zmm24[1],zmm14[1],zmm24[3],zmm14[3],zmm24[5],zmm14[5],zmm24[7],zmm14[7]
+; AVX512F-NEXT:    vpermt2q %zmm6, %zmm0, %zmm1
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm2
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm7, %zmm0, %zmm2
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    vpermi2q %zmm30, %zmm28, %zmm0
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm2 = zmm14[0],zmm10[0],zmm14[2],zmm10[2],zmm14[4],zmm10[4],zmm14[6],zmm10[6]
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k1}
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm2, %zmm25
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm27 = [5,13,5,13,5,13,5,13]
+; AVX512F-NEXT:    # zmm27 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm31, %zmm27, %zmm0
+; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm11, %zmm27, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm8
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm9 = ymm1[0,1,2,3],ymm0[4,5,6,7]
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm0 = zmm5[1],zmm16[1],zmm5[3],zmm16[3],zmm5[5],zmm16[5],zmm5[7],zmm16[7]
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm26 = [6,14,6,14,6,14,6,14]
+; AVX512F-NEXT:    # zmm26 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm16, %zmm26, %zmm1
 ; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm2 = [7,15,7,15,7,15,7,15]
 ; AVX512F-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermt2q %zmm14, %zmm2, %zmm24
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm30 {%k1} = zmm13[0],zmm12[0],zmm13[2],zmm12[2],zmm13[4],zmm12[4],zmm13[6],zmm12[6]
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm24 {%k1} = zmm13[1],zmm12[1],zmm13[3],zmm12[3],zmm13[5],zmm12[5],zmm13[7],zmm12[7]
-; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm14
-; AVX512F-NEXT:    vpermt2q %zmm12, %zmm23, %zmm14
-; AVX512F-NEXT:    vpermt2q %zmm12, %zmm6, %zmm13
-; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm1 {%k1}
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm1, %zmm20
-; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm3
-; AVX512F-NEXT:    vpermt2q %zmm9, %zmm23, %zmm3
-; AVX512F-NEXT:    vpermi2q %zmm11, %zmm28, %zmm23
+; AVX512F-NEXT:    vpermt2q %zmm16, %zmm2, %zmm5
+; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm12
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm1 {%k1} = zmm15[0],zmm3[0],zmm15[2],zmm3[2],zmm15[4],zmm3[4],zmm15[6],zmm3[6]
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm5 {%k1} = zmm15[1],zmm3[1],zmm15[3],zmm3[3],zmm15[5],zmm3[5],zmm15[7],zmm3[7]
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm21, %zmm12
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm27, %zmm15
+; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm0 {%k1}
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm9, %zmm0, %zmm17
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm6, %zmm27, %zmm0
+; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm15
+; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm9
+; AVX512F-NEXT:    vpermt2q %zmm7, %zmm27, %zmm9
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm9[0,1,2,3],ymm0[4,5,6,7]
+; AVX512F-NEXT:    vpermi2q %zmm30, %zmm28, %zmm27
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm9 = zmm14[1],zmm10[1],zmm14[3],zmm10[3],zmm14[5],zmm10[5],zmm14[7],zmm10[7]
+; AVX512F-NEXT:    vmovdqa64 %zmm27, %zmm9 {%k1}
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm9, %zmm27
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm31, %zmm26, %zmm0
+; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm9
+; AVX512F-NEXT:    vpermt2q %zmm8, %zmm26, %zmm9
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm9[0,1,2,3],ymm0[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm1, %zmm16
+; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512F-NEXT:    vpermt2q %zmm31, %zmm2, %zmm4
+; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vmovdqa64 %zmm28, %zmm1
-; AVX512F-NEXT:    vpermt2q %zmm11, %zmm5, %zmm1
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm13 = zmm28[1],zmm11[1],zmm28[3],zmm11[3],zmm28[5],zmm11[5],zmm28[7],zmm11[7]
-; AVX512F-NEXT:    vpermt2q %zmm11, %zmm2, %zmm28
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm1 {%k1} = zmm10[0],zmm9[0],zmm10[2],zmm9[2],zmm10[4],zmm9[4],zmm10[6],zmm9[6]
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm28 {%k1} = zmm10[1],zmm9[1],zmm10[3],zmm9[3],zmm10[5],zmm9[5],zmm10[7],zmm9[7]
-; AVX512F-NEXT:    vpermt2q %zmm9, %zmm6, %zmm10
-; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm13 {%k1}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
-; AVX512F-NEXT:    vpermi2q %zmm31, %zmm10, %zmm6
-; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm0
-; AVX512F-NEXT:    vpermi2q %zmm18, %zmm8, %zmm7
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3],ymm7[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm6, %zmm13, %zmm12
-; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm7
-; AVX512F-NEXT:    vpermt2q %zmm26, %zmm5, %zmm7
-; AVX512F-NEXT:    vpermt2q %zmm26, %zmm2, %zmm15
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm9 = [6,14,6,14]
-; AVX512F-NEXT:    # ymm9 = mem[0,1,0,1]
-; AVX512F-NEXT:    vpermi2q %zmm31, %zmm10, %zmm5
-; AVX512F-NEXT:    vpermt2q %zmm31, %zmm2, %zmm10
-; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm8
-; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm17, %zmm9, %zmm2
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm10 = [7,15,7,15]
-; AVX512F-NEXT:    # ymm10 = mem[0,1,0,1]
-; AVX512F-NEXT:    vpermt2q %zmm17, %zmm10, %zmm4
-; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm6
-; AVX512F-NEXT:    vmovdqa 576(%rdi), %xmm11
-; AVX512F-NEXT:    vinserti128 $1, 704(%rdi), %ymm11, %ymm11
-; AVX512F-NEXT:    vpermi2q %zmm18, %zmm0, %zmm9
-; AVX512F-NEXT:    vpermt2q %zmm18, %zmm10, %zmm0
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm4
-; AVX512F-NEXT:    vmovdqa 512(%rdi), %xmm10
-; AVX512F-NEXT:    vinserti128 $1, 640(%rdi), %ymm10, %ymm10
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm13 = ymm10[0],ymm11[0],ymm10[2],ymm11[2]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm13, %zmm19, %zmm13
-; AVX512F-NEXT:    vmovdqa64 %zmm14, %zmm29 {%k1}
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm14 = ymm16[1],ymm21[1],ymm16[3],ymm21[3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm14, %zmm29, %zmm14
-; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm23 {%k1}
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm0 = ymm10[1],ymm11[1],ymm10[3],ymm11[3]
+; AVX512F-NEXT:    vpermt2q %zmm30, %zmm29, %zmm1
+; AVX512F-NEXT:    vpermi2q %zmm10, %zmm14, %zmm29
+; AVX512F-NEXT:    vpermt2q %zmm8, %zmm2, %zmm13
+; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm4
+; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm9
+; AVX512F-NEXT:    vpermt2q %zmm6, %zmm26, %zmm9
+; AVX512F-NEXT:    vpermt2q %zmm6, %zmm2, %zmm11
+; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm8
+; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm13
+; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm11
+; AVX512F-NEXT:    vpermt2q %zmm7, %zmm26, %zmm11
+; AVX512F-NEXT:    vpermt2q %zmm7, %zmm2, %zmm13
+; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm15
+; AVX512F-NEXT:    vmovdqa64 %zmm28, %zmm13
+; AVX512F-NEXT:    vpermt2q %zmm30, %zmm21, %zmm13
+; AVX512F-NEXT:    vpermi2q %zmm10, %zmm14, %zmm21
+; AVX512F-NEXT:    vpermi2q %zmm10, %zmm14, %zmm26
+; AVX512F-NEXT:    vpermt2q %zmm10, %zmm2, %zmm14
+; AVX512F-NEXT:    vmovdqa 64(%rdi), %xmm2
+; AVX512F-NEXT:    vinserti128 $1, 192(%rdi), %ymm2, %ymm2
+; AVX512F-NEXT:    vinserti128 $1, 128(%rdi), %ymm0, %ymm0
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm10 = ymm0[0],ymm2[0],ymm0[2],ymm2[2]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm10, %zmm18, %zmm10
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm29 {%k1}
+; AVX512F-NEXT:    vmovdqa 576(%rdi), %xmm1
+; AVX512F-NEXT:    vinserti128 $1, 704(%rdi), %ymm1, %ymm1
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm26 {%k1} = zmm28[0],zmm30[0],zmm28[2],zmm30[2],zmm28[4],zmm30[4],zmm28[6],zmm30[6]
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm14 {%k1} = zmm28[1],zmm30[1],zmm28[3],zmm30[3],zmm28[5],zmm30[5],zmm28[7],zmm30[7]
+; AVX512F-NEXT:    vmovdqa 512(%rdi), %xmm6
+; AVX512F-NEXT:    vinserti128 $1, 640(%rdi), %ymm6, %ymm6
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm7 = ymm6[0],ymm1[0],ymm6[2],ymm1[2]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm7, %zmm29, %zmm7
+; AVX512F-NEXT:    vmovdqa64 %zmm12, %zmm23 {%k1}
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm0 = ymm0[1],ymm2[1],ymm0[3],ymm2[3]
 ; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm23, %zmm0
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm7[0,1,2,3],ymm2[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm30, %zmm2
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm9[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm5, %zmm1, %zmm1
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm5 = ymm15[0,1,2,3],ymm6[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm5, %zmm24, %zmm6
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm5 = ymm8[0,1,2,3],ymm4[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm5, %zmm28, %zmm3
-; AVX512F-NEXT:    vmovdqa64 %zmm13, 64(%rsi)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm4, (%rsi)
-; AVX512F-NEXT:    vmovdqa64 %zmm0, 64(%rdx)
-; AVX512F-NEXT:    vmovdqa64 %zmm14, (%rdx)
-; AVX512F-NEXT:    vmovups (%rsp), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 64(%rcx)
+; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm21 {%k1}
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm6[1],ymm1[1],ymm6[3],ymm1[3]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm21, %zmm1
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm11[0,1,2,3],ymm9[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm26, %zmm2
+; AVX512F-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm4 # 32-byte Folded Reload
+; AVX512F-NEXT:    # ymm4 = ymm4[0,1,2,3],mem[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm4, %zmm5, %zmm4
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm5 = ymm15[0,1,2,3],ymm8[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm5, %zmm14, %zmm3
+; AVX512F-NEXT:    vmovdqa64 %zmm7, 64(%rsi)
+; AVX512F-NEXT:    vmovdqa64 %zmm10, (%rsi)
+; AVX512F-NEXT:    vmovdqa64 %zmm1, 64(%rdx)
+; AVX512F-NEXT:    vmovdqa64 %zmm0, (%rdx)
+; AVX512F-NEXT:    vmovdqa64 %zmm19, 64(%rcx)
 ; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-NEXT:    vmovaps %zmm0, (%rcx)
 ; AVX512F-NEXT:    vmovdqa64 %zmm22, 64(%r8)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, (%r8)
-; AVX512F-NEXT:    vmovdqa64 %zmm27, 64(%r9)
-; AVX512F-NEXT:    vmovdqa64 %zmm25, (%r9)
+; AVX512F-NEXT:    vmovdqa64 %zmm20, (%r8)
+; AVX512F-NEXT:    vmovdqa64 %zmm25, 64(%r9)
+; AVX512F-NEXT:    vmovdqa64 %zmm24, (%r9)
 ; AVX512F-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512F-NEXT:    vmovdqa64 %zmm12, 64(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm20, (%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm27, 64(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm17, (%rax)
 ; AVX512F-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512F-NEXT:    vmovdqa64 %zmm1, 64(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm2, (%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm2, 64(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm16, (%rax)
 ; AVX512F-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512F-NEXT:    vmovdqa64 %zmm3, 64(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm6, (%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm4, (%rax)
 ; AVX512F-NEXT:    addq $264, %rsp # imm = 0x108
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
@@ -2148,221 +2145,222 @@ define void @load_i64_stride8_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512BW-LABEL: load_i64_stride8_vf16:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    subq $264, %rsp # imm = 0x108
-; AVX512BW-NEXT:    vmovdqa64 704(%rdi), %zmm18
-; AVX512BW-NEXT:    vmovaps 640(%rdi), %zmm0
+; AVX512BW-NEXT:    vmovaps 576(%rdi), %zmm0
 ; AVX512BW-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 576(%rdi), %zmm31
-; AVX512BW-NEXT:    vmovdqa64 512(%rdi), %zmm3
-; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 192(%rdi), %zmm17
-; AVX512BW-NEXT:    vmovdqa64 128(%rdi), %zmm4
-; AVX512BW-NEXT:    vmovdqa64 832(%rdi), %zmm11
-; AVX512BW-NEXT:    vmovdqa64 768(%rdi), %zmm28
-; AVX512BW-NEXT:    vmovdqa64 960(%rdi), %zmm9
-; AVX512BW-NEXT:    vmovdqa64 896(%rdi), %zmm10
-; AVX512BW-NEXT:    vmovdqa64 320(%rdi), %zmm14
-; AVX512BW-NEXT:    vmovdqa64 256(%rdi), %zmm24
-; AVX512BW-NEXT:    vmovdqa64 448(%rdi), %zmm12
-; AVX512BW-NEXT:    vmovdqa64 384(%rdi), %zmm13
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm19 = [0,8,0,8,0,8,0,8]
-; AVX512BW-NEXT:    # zmm19 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm5
-; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm7
-; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm6
-; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm12, %zmm19, %zmm0
-; AVX512BW-NEXT:    vmovdqa64 %zmm24, %zmm15
-; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm19, %zmm15
+; AVX512BW-NEXT:    vmovaps 512(%rdi), %zmm0
+; AVX512BW-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 704(%rdi), %zmm6
+; AVX512BW-NEXT:    vmovdqa64 640(%rdi), %zmm4
+; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 64(%rdi), %zmm7
+; AVX512BW-NEXT:    vmovaps (%rdi), %zmm0
+; AVX512BW-NEXT:    vmovups %zmm0, (%rsp) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 192(%rdi), %zmm31
+; AVX512BW-NEXT:    vmovdqa64 128(%rdi), %zmm8
+; AVX512BW-NEXT:    vmovdqa64 832(%rdi), %zmm10
+; AVX512BW-NEXT:    vmovdqa64 768(%rdi), %zmm14
+; AVX512BW-NEXT:    vmovdqa64 960(%rdi), %zmm30
+; AVX512BW-NEXT:    vmovdqa64 896(%rdi), %zmm28
+; AVX512BW-NEXT:    vmovdqa64 320(%rdi), %zmm16
+; AVX512BW-NEXT:    vmovdqa64 256(%rdi), %zmm5
+; AVX512BW-NEXT:    vmovdqa64 448(%rdi), %zmm3
+; AVX512BW-NEXT:    vmovdqa64 384(%rdi), %zmm15
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm29 = [0,8,0,8,0,8,0,8]
+; AVX512BW-NEXT:    # zmm29 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm17
+; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm20
+; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm9
+; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm19
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm29, %zmm19
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm18
+; AVX512BW-NEXT:    vpermt2q %zmm16, %zmm29, %zmm18
 ; AVX512BW-NEXT:    movb $-64, %al
 ; AVX512BW-NEXT:    kmovd %eax, %k1
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm15 {%k1}
-; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512BW-NEXT:    vmovdqa64 64(%rdi), %xmm16
-; AVX512BW-NEXT:    vinserti32x4 $1, 192(%rdi), %ymm16, %ymm21
-; AVX512BW-NEXT:    vinserti32x4 $1, 128(%rdi), %ymm0, %ymm16
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm16[0],ymm21[0],ymm16[2],ymm21[2]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm15, %zmm0
+; AVX512BW-NEXT:    vmovdqa64 %zmm19, %zmm18 {%k1}
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm19 = [2,10,2,10,2,10,2,10]
+; AVX512BW-NEXT:    # zmm19 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm19, %zmm17
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm21
+; AVX512BW-NEXT:    vpermt2q %zmm16, %zmm19, %zmm21
+; AVX512BW-NEXT:    vmovdqa64 %zmm17, %zmm21 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 192(%rdi), %ymm22
+; AVX512BW-NEXT:    vmovdqa64 128(%rdi), %ymm23
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm11 = ymm23[0],ymm22[0],ymm23[2],ymm22[2]
+; AVX512BW-NEXT:    vmovdqa64 64(%rdi), %ymm24
+; AVX512BW-NEXT:    vmovdqa64 (%rdi), %ymm25
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm12 = ymm25[0],ymm24[0],ymm25[2],ymm24[2]
+; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm11 = ymm12[2,3],ymm11[2,3]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm11, %zmm21, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm19, %zmm0
-; AVX512BW-NEXT:    vpermi2q %zmm11, %zmm28, %zmm19
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm19 {%k1}
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [2,10,2,10,2,10,2,10]
-; AVX512BW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermt2q %zmm12, %zmm0, %zmm5
-; AVX512BW-NEXT:    vmovdqa64 %zmm24, %zmm15
-; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm0, %zmm15
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm15 {%k1}
-; AVX512BW-NEXT:    vmovdqa 192(%rdi), %ymm5
-; AVX512BW-NEXT:    vmovdqa64 128(%rdi), %ymm20
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm1 = ymm20[0],ymm5[0],ymm20[2],ymm5[2]
-; AVX512BW-NEXT:    vmovdqa64 64(%rdi), %ymm22
-; AVX512BW-NEXT:    vmovdqa64 (%rdi), %ymm23
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm2 = ymm23[0],ymm22[0],ymm23[2],ymm22[2]
-; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm1 = ymm2[2,3],ymm1[2,3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm15, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm1
-; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm0, %zmm1
-; AVX512BW-NEXT:    vpermi2q %zmm11, %zmm28, %zmm0
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm0 {%k1}
-; AVX512BW-NEXT:    vmovdqa 704(%rdi), %ymm1
-; AVX512BW-NEXT:    vmovdqa 640(%rdi), %ymm2
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm15 = ymm2[0],ymm1[0],ymm2[2],ymm1[2]
-; AVX512BW-NEXT:    vmovdqa64 576(%rdi), %ymm25
+; AVX512BW-NEXT:    vmovdqa64 %zmm28, %zmm11
+; AVX512BW-NEXT:    vpermt2q %zmm30, %zmm19, %zmm11
+; AVX512BW-NEXT:    vpermi2q %zmm10, %zmm14, %zmm19
+; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm19 {%k1}
+; AVX512BW-NEXT:    vmovdqa 704(%rdi), %ymm11
+; AVX512BW-NEXT:    vmovdqa 640(%rdi), %ymm12
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm13 = ymm12[0],ymm11[0],ymm12[2],ymm11[2]
+; AVX512BW-NEXT:    vmovdqa64 576(%rdi), %ymm26
 ; AVX512BW-NEXT:    vmovdqa64 512(%rdi), %ymm27
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm8 = ymm27[0],ymm25[0],ymm27[2],ymm25[2]
-; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm8 = ymm8[2,3],ymm15[2,3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm8, %zmm0, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, (%rsp) # 64-byte Spill
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm27[0],ymm26[0],ymm27[2],ymm26[2]
+; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm0[2,3],ymm13[2,3]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm19, %zmm19
 ; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [3,11,3,11,3,11,3,11]
 ; AVX512BW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermt2q %zmm12, %zmm0, %zmm7
-; AVX512BW-NEXT:    vmovdqa64 %zmm24, %zmm30
-; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm0, %zmm30
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm30 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 64(%rdi), %zmm26
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm5 = ymm20[1],ymm5[1],ymm20[3],ymm5[3]
-; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm15
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm7 = ymm23[1],ymm22[1],ymm23[3],ymm22[3]
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm23 = [1,9,1,9,1,9,1,9]
-; AVX512BW-NEXT:    # zmm23 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm5 = ymm7[2,3],ymm5[2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm24, %zmm29
-; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm23, %zmm29
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm5, %zmm30, %zmm5
-; AVX512BW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm5
-; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm0, %zmm5
-; AVX512BW-NEXT:    vpermi2q %zmm11, %zmm28, %zmm0
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm0 {%k1}
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm2[1],ymm1[1],ymm2[3],ymm1[3]
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm2 = ymm27[1],ymm25[1],ymm27[3],ymm25[3]
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm0, %zmm20
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm13
+; AVX512BW-NEXT:    vpermt2q %zmm16, %zmm0, %zmm13
+; AVX512BW-NEXT:    vmovdqa64 %zmm20, %zmm13 {%k1}
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm21 = [1,9,1,9,1,9,1,9]
+; AVX512BW-NEXT:    # zmm21 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm23[1],ymm22[1],ymm23[3],ymm22[3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm23
+; AVX512BW-NEXT:    vpermt2q %zmm16, %zmm21, %zmm23
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm2 = ymm25[1],ymm24[1],ymm25[3],ymm24[3]
+; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm1 = ymm2[2,3],ymm1[2,3]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm13, %zmm20
+; AVX512BW-NEXT:    vmovdqa64 %zmm28, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm30, %zmm0, %zmm1
+; AVX512BW-NEXT:    vpermi2q %zmm10, %zmm14, %zmm0
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm0 {%k1}
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm12[1],ymm11[1],ymm12[3],ymm11[3]
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm2 = ymm27[1],ymm26[1],ymm27[3],ymm26[3]
 ; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm1 = ymm2[2,3],ymm1[2,3]
 ; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm22
 ; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [4,12,4,12,4,12,4,12]
 ; AVX512BW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermt2q %zmm12, %zmm0, %zmm6
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm1 = zmm24[0],zmm14[0],zmm24[2],zmm14[2],zmm24[4],zmm14[4],zmm24[6],zmm14[6]
-; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm1 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm2
-; AVX512BW-NEXT:    vpermt2q %zmm26, %zmm0, %zmm2
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm5 = [4,12,4,12]
-; AVX512BW-NEXT:    # ymm5 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm6
-; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm5, %zmm6
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm6[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm1, %zmm25
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm1
-; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm0, %zmm1
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm2 = zmm28[0],zmm11[0],zmm28[2],zmm11[2],zmm28[4],zmm11[4],zmm28[6],zmm11[6]
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}
-; AVX512BW-NEXT:    vpermi2q %zmm31, %zmm3, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
-; AVX512BW-NEXT:    vpermi2q %zmm18, %zmm8, %zmm5
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm2, %zmm27
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm6 = [5,13,5,13,5,13,5,13]
-; AVX512BW-NEXT:    # zmm6 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm26, %zmm6, %zmm0
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm7 = [5,13,5,13]
-; AVX512BW-NEXT:    # ymm7 = mem[0,1,0,1]
+; AVX512BW-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm31, %zmm0, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 (%rsp), %zmm13 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm2
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm11
+; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm0, %zmm2
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm0, %zmm9
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm2 = zmm5[0],zmm16[0],zmm5[2],zmm16[2],zmm5[4],zmm16[4],zmm5[6],zmm16[6]
+; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm2 {%k1}
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm2, %zmm24
 ; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm1
-; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm7, %zmm1
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm5 = [6,14,6,14,6,14,6,14]
-; AVX512BW-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm24, %zmm30
-; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm5, %zmm30
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm1 = zmm24[1],zmm14[1],zmm24[3],zmm14[3],zmm24[5],zmm14[5],zmm24[7],zmm14[7]
+; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm0, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm2
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm0, %zmm2
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512BW-NEXT:    vpermi2q %zmm30, %zmm28, %zmm0
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm2 = zmm14[0],zmm10[0],zmm14[2],zmm10[2],zmm14[4],zmm10[4],zmm14[6],zmm10[6]
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k1}
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm2, %zmm25
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm27 = [5,13,5,13,5,13,5,13]
+; AVX512BW-NEXT:    # zmm27 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm31, %zmm27, %zmm0
+; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm27, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm8
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm9 = ymm1[0,1,2,3],ymm0[4,5,6,7]
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm0 = zmm5[1],zmm16[1],zmm5[3],zmm16[3],zmm5[5],zmm16[5],zmm5[7],zmm16[7]
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm26 = [6,14,6,14,6,14,6,14]
+; AVX512BW-NEXT:    # zmm26 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm16, %zmm26, %zmm1
 ; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm2 = [7,15,7,15,7,15,7,15]
 ; AVX512BW-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm2, %zmm24
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm30 {%k1} = zmm13[0],zmm12[0],zmm13[2],zmm12[2],zmm13[4],zmm12[4],zmm13[6],zmm12[6]
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm24 {%k1} = zmm13[1],zmm12[1],zmm13[3],zmm12[3],zmm13[5],zmm12[5],zmm13[7],zmm12[7]
-; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm14
-; AVX512BW-NEXT:    vpermt2q %zmm12, %zmm23, %zmm14
-; AVX512BW-NEXT:    vpermt2q %zmm12, %zmm6, %zmm13
-; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm1 {%k1}
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm1, %zmm20
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm3
-; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm23, %zmm3
-; AVX512BW-NEXT:    vpermi2q %zmm11, %zmm28, %zmm23
+; AVX512BW-NEXT:    vpermt2q %zmm16, %zmm2, %zmm5
+; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm12
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm1 {%k1} = zmm15[0],zmm3[0],zmm15[2],zmm3[2],zmm15[4],zmm3[4],zmm15[6],zmm3[6]
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm5 {%k1} = zmm15[1],zmm3[1],zmm15[3],zmm3[3],zmm15[5],zmm3[5],zmm15[7],zmm3[7]
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm21, %zmm12
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm27, %zmm15
+; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm0 {%k1}
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm9, %zmm0, %zmm17
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm27, %zmm0
+; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm15
+; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm9
+; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm27, %zmm9
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm9[0,1,2,3],ymm0[4,5,6,7]
+; AVX512BW-NEXT:    vpermi2q %zmm30, %zmm28, %zmm27
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm9 = zmm14[1],zmm10[1],zmm14[3],zmm10[3],zmm14[5],zmm10[5],zmm14[7],zmm10[7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm27, %zmm9 {%k1}
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm9, %zmm27
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm31, %zmm26, %zmm0
+; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm9
+; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm26, %zmm9
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm9[0,1,2,3],ymm0[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm1, %zmm16
+; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512BW-NEXT:    vpermt2q %zmm31, %zmm2, %zmm4
+; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vmovdqa64 %zmm28, %zmm1
-; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm5, %zmm1
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm13 = zmm28[1],zmm11[1],zmm28[3],zmm11[3],zmm28[5],zmm11[5],zmm28[7],zmm11[7]
-; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm2, %zmm28
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm1 {%k1} = zmm10[0],zmm9[0],zmm10[2],zmm9[2],zmm10[4],zmm9[4],zmm10[6],zmm9[6]
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm28 {%k1} = zmm10[1],zmm9[1],zmm10[3],zmm9[3],zmm10[5],zmm9[5],zmm10[7],zmm9[7]
-; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm6, %zmm10
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm13 {%k1}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
-; AVX512BW-NEXT:    vpermi2q %zmm31, %zmm10, %zmm6
-; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm0
-; AVX512BW-NEXT:    vpermi2q %zmm18, %zmm8, %zmm7
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3],ymm7[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm6, %zmm13, %zmm12
-; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm7
-; AVX512BW-NEXT:    vpermt2q %zmm26, %zmm5, %zmm7
-; AVX512BW-NEXT:    vpermt2q %zmm26, %zmm2, %zmm15
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm9 = [6,14,6,14]
-; AVX512BW-NEXT:    # ymm9 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vpermi2q %zmm31, %zmm10, %zmm5
-; AVX512BW-NEXT:    vpermt2q %zmm31, %zmm2, %zmm10
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm8
-; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm2
-; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm9, %zmm2
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm10 = [7,15,7,15]
-; AVX512BW-NEXT:    # ymm10 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm10, %zmm4
-; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm6
-; AVX512BW-NEXT:    vmovdqa 576(%rdi), %xmm11
-; AVX512BW-NEXT:    vinserti128 $1, 704(%rdi), %ymm11, %ymm11
-; AVX512BW-NEXT:    vpermi2q %zmm18, %zmm0, %zmm9
-; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm10, %zmm0
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm4
-; AVX512BW-NEXT:    vmovdqa 512(%rdi), %xmm10
-; AVX512BW-NEXT:    vinserti128 $1, 640(%rdi), %ymm10, %ymm10
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm13 = ymm10[0],ymm11[0],ymm10[2],ymm11[2]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm13, %zmm19, %zmm13
-; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm29 {%k1}
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm14 = ymm16[1],ymm21[1],ymm16[3],ymm21[3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm14, %zmm29, %zmm14
-; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm23 {%k1}
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm0 = ymm10[1],ymm11[1],ymm10[3],ymm11[3]
+; AVX512BW-NEXT:    vpermt2q %zmm30, %zmm29, %zmm1
+; AVX512BW-NEXT:    vpermi2q %zmm10, %zmm14, %zmm29
+; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm2, %zmm13
+; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm4
+; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm9
+; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm26, %zmm9
+; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm2, %zmm11
+; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm8
+; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm13
+; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm11
+; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm26, %zmm11
+; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm2, %zmm13
+; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm15
+; AVX512BW-NEXT:    vmovdqa64 %zmm28, %zmm13
+; AVX512BW-NEXT:    vpermt2q %zmm30, %zmm21, %zmm13
+; AVX512BW-NEXT:    vpermi2q %zmm10, %zmm14, %zmm21
+; AVX512BW-NEXT:    vpermi2q %zmm10, %zmm14, %zmm26
+; AVX512BW-NEXT:    vpermt2q %zmm10, %zmm2, %zmm14
+; AVX512BW-NEXT:    vmovdqa 64(%rdi), %xmm2
+; AVX512BW-NEXT:    vinserti128 $1, 192(%rdi), %ymm2, %ymm2
+; AVX512BW-NEXT:    vinserti128 $1, 128(%rdi), %ymm0, %ymm0
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm10 = ymm0[0],ymm2[0],ymm0[2],ymm2[2]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm10, %zmm18, %zmm10
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm29 {%k1}
+; AVX512BW-NEXT:    vmovdqa 576(%rdi), %xmm1
+; AVX512BW-NEXT:    vinserti128 $1, 704(%rdi), %ymm1, %ymm1
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm26 {%k1} = zmm28[0],zmm30[0],zmm28[2],zmm30[2],zmm28[4],zmm30[4],zmm28[6],zmm30[6]
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm14 {%k1} = zmm28[1],zmm30[1],zmm28[3],zmm30[3],zmm28[5],zmm30[5],zmm28[7],zmm30[7]
+; AVX512BW-NEXT:    vmovdqa 512(%rdi), %xmm6
+; AVX512BW-NEXT:    vinserti128 $1, 640(%rdi), %ymm6, %ymm6
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm7 = ymm6[0],ymm1[0],ymm6[2],ymm1[2]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm7, %zmm29, %zmm7
+; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm23 {%k1}
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm0 = ymm0[1],ymm2[1],ymm0[3],ymm2[3]
 ; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm23, %zmm0
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm7[0,1,2,3],ymm2[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm30, %zmm2
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm9[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm5, %zmm1, %zmm1
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm5 = ymm15[0,1,2,3],ymm6[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm5, %zmm24, %zmm6
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm5 = ymm8[0,1,2,3],ymm4[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm5, %zmm28, %zmm3
-; AVX512BW-NEXT:    vmovdqa64 %zmm13, 64(%rsi)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm4, (%rsi)
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, 64(%rdx)
-; AVX512BW-NEXT:    vmovdqa64 %zmm14, (%rdx)
-; AVX512BW-NEXT:    vmovups (%rsp), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm0, 64(%rcx)
+; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm21 {%k1}
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm6[1],ymm1[1],ymm6[3],ymm1[3]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm21, %zmm1
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm11[0,1,2,3],ymm9[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm26, %zmm2
+; AVX512BW-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm4 # 32-byte Folded Reload
+; AVX512BW-NEXT:    # ymm4 = ymm4[0,1,2,3],mem[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm4, %zmm5, %zmm4
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm5 = ymm15[0,1,2,3],ymm8[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm5, %zmm14, %zmm3
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, 64(%rsi)
+; AVX512BW-NEXT:    vmovdqa64 %zmm10, (%rsi)
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, 64(%rdx)
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rdx)
+; AVX512BW-NEXT:    vmovdqa64 %zmm19, 64(%rcx)
 ; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-NEXT:    vmovaps %zmm0, (%rcx)
 ; AVX512BW-NEXT:    vmovdqa64 %zmm22, 64(%r8)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm0, (%r8)
-; AVX512BW-NEXT:    vmovdqa64 %zmm27, 64(%r9)
-; AVX512BW-NEXT:    vmovdqa64 %zmm25, (%r9)
+; AVX512BW-NEXT:    vmovdqa64 %zmm20, (%r8)
+; AVX512BW-NEXT:    vmovdqa64 %zmm25, 64(%r9)
+; AVX512BW-NEXT:    vmovdqa64 %zmm24, (%r9)
 ; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512BW-NEXT:    vmovdqa64 %zmm12, 64(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm20, (%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm27, 64(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm17, (%rax)
 ; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, 64(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, (%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, 64(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm16, (%rax)
 ; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512BW-NEXT:    vmovdqa64 %zmm3, 64(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm6, (%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm4, (%rax)
 ; AVX512BW-NEXT:    addq $264, %rsp # imm = 0x108
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
@@ -4240,1031 +4238,1043 @@ define void @load_i64_stride8_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ;
 ; AVX512F-LABEL: load_i64_stride8_vf32:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    subq $2632, %rsp # imm = 0xA48
-; AVX512F-NEXT:    vmovdqa64 1856(%rdi), %zmm14
-; AVX512F-NEXT:    vmovdqa64 1984(%rdi), %zmm28
-; AVX512F-NEXT:    vmovdqa64 1344(%rdi), %zmm21
+; AVX512F-NEXT:    subq $3208, %rsp # imm = 0xC88
+; AVX512F-NEXT:    vmovdqa64 1856(%rdi), %zmm12
+; AVX512F-NEXT:    vmovdqa64 1984(%rdi), %zmm16
+; AVX512F-NEXT:    vmovdqa64 1344(%rdi), %zmm19
 ; AVX512F-NEXT:    vmovdqa64 1280(%rdi), %zmm18
-; AVX512F-NEXT:    vmovdqa64 1472(%rdi), %zmm17
-; AVX512F-NEXT:    vmovdqa64 1408(%rdi), %zmm22
+; AVX512F-NEXT:    vmovdqa64 1472(%rdi), %zmm5
+; AVX512F-NEXT:    vmovdqa64 1408(%rdi), %zmm27
 ; AVX512F-NEXT:    vmovdqa64 832(%rdi), %zmm7
-; AVX512F-NEXT:    vmovdqa64 768(%rdi), %zmm8
-; AVX512F-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 960(%rdi), %zmm10
+; AVX512F-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 768(%rdi), %zmm11
+; AVX512F-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 960(%rdi), %zmm31
 ; AVX512F-NEXT:    vmovdqa64 896(%rdi), %zmm3
-; AVX512F-NEXT:    vmovdqa64 320(%rdi), %zmm15
-; AVX512F-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 256(%rdi), %zmm30
-; AVX512F-NEXT:    vmovdqa64 448(%rdi), %zmm19
-; AVX512F-NEXT:    vmovdqa64 384(%rdi), %zmm20
+; AVX512F-NEXT:    vmovdqa64 320(%rdi), %zmm20
+; AVX512F-NEXT:    vmovdqa64 256(%rdi), %zmm8
+; AVX512F-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 448(%rdi), %zmm25
+; AVX512F-NEXT:    vmovdqu64 %zmm25, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 384(%rdi), %zmm23
 ; AVX512F-NEXT:    movb $-64, %al
 ; AVX512F-NEXT:    kmovw %eax, %k1
 ; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [2,10,2,10,2,10,2,10]
 ; AVX512F-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm22, %zmm1
-; AVX512F-NEXT:    vpermt2q %zmm17, %zmm0, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm27, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm5, %zmm0, %zmm1
 ; AVX512F-NEXT:    vmovdqa64 %zmm18, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm21, %zmm0, %zmm2
+; AVX512F-NEXT:    vmovdqu64 %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm19, %zmm0, %zmm2
 ; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}
-; AVX512F-NEXT:    vmovdqa64 1216(%rdi), %ymm29
-; AVX512F-NEXT:    vmovdqa 1152(%rdi), %ymm13
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm5 = ymm13[0],ymm29[0],ymm13[2],ymm29[2]
-; AVX512F-NEXT:    vmovdqa 1088(%rdi), %ymm6
-; AVX512F-NEXT:    vmovdqa 1024(%rdi), %ymm4
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm9 = ymm4[0],ymm6[0],ymm4[2],ymm6[2]
-; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm5 = ymm9[2,3],ymm5[2,3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm5, %zmm2, %zmm1
+; AVX512F-NEXT:    vmovdqa64 1216(%rdi), %ymm22
+; AVX512F-NEXT:    vmovdqa 1152(%rdi), %ymm10
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm4 = ymm10[0],ymm22[0],ymm10[2],ymm22[2]
+; AVX512F-NEXT:    vmovdqa 1088(%rdi), %ymm9
+; AVX512F-NEXT:    vmovdqa 1024(%rdi), %ymm6
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm13 = ymm6[0],ymm9[0],ymm6[2],ymm9[2]
+; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm4 = ymm13[2,3],ymm4[2,3]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm4, %zmm2, %zmm1
 ; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm10, %zmm0, %zmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm24
-; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm5
-; AVX512F-NEXT:    vpermt2q %zmm7, %zmm0, %zmm5
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm5 {%k1}
-; AVX512F-NEXT:    vmovdqa 704(%rdi), %ymm8
-; AVX512F-NEXT:    vmovdqa 640(%rdi), %ymm9
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm10 = ymm9[0],ymm8[0],ymm9[2],ymm8[2]
-; AVX512F-NEXT:    vmovdqa 576(%rdi), %ymm12
-; AVX512F-NEXT:    vmovdqa64 512(%rdi), %ymm16
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm11 = ymm16[0],ymm12[0],ymm16[2],ymm12[2]
-; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm10 = ymm11[2,3],ymm10[2,3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm10, %zmm5, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm31, %zmm0, %zmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm4
+; AVX512F-NEXT:    vpermt2q %zmm7, %zmm0, %zmm4
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm4 {%k1}
+; AVX512F-NEXT:    vmovdqa 704(%rdi), %ymm7
+; AVX512F-NEXT:    vmovdqa 640(%rdi), %ymm13
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm14 = ymm13[0],ymm7[0],ymm13[2],ymm7[2]
+; AVX512F-NEXT:    vmovdqa64 576(%rdi), %ymm17
+; AVX512F-NEXT:    vmovdqa64 512(%rdi), %ymm21
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm11 = ymm21[0],ymm17[0],ymm21[2],ymm17[2]
+; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm11 = ymm11[2,3],ymm14[2,3]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm11, %zmm4, %zmm1
 ; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm20, %zmm5
-; AVX512F-NEXT:    vpermt2q %zmm19, %zmm0, %zmm5
-; AVX512F-NEXT:    vmovdqa64 %zmm30, %zmm10
-; AVX512F-NEXT:    vpermt2q %zmm15, %zmm0, %zmm10
-; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm10 {%k1}
-; AVX512F-NEXT:    vmovdqa 192(%rdi), %ymm5
-; AVX512F-NEXT:    vmovdqa64 128(%rdi), %ymm25
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm11 = ymm25[0],ymm5[0],ymm25[2],ymm5[2]
-; AVX512F-NEXT:    vmovdqa64 64(%rdi), %ymm26
-; AVX512F-NEXT:    vmovdqa64 (%rdi), %ymm27
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm15 = ymm27[0],ymm26[0],ymm27[2],ymm26[2]
-; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm15 = ymm15[2,3],ymm11[2,3]
-; AVX512F-NEXT:    vmovdqa64 1920(%rdi), %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm15, %zmm10, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm23, %zmm4
+; AVX512F-NEXT:    vpermt2q %zmm25, %zmm0, %zmm4
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm11
+; AVX512F-NEXT:    vpermt2q %zmm20, %zmm0, %zmm11
+; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm11 {%k1}
+; AVX512F-NEXT:    vmovdqa 192(%rdi), %ymm4
+; AVX512F-NEXT:    vmovdqa 128(%rdi), %ymm14
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm1 = ymm14[0],ymm4[0],ymm14[2],ymm4[2]
+; AVX512F-NEXT:    vmovdqa64 64(%rdi), %ymm25
+; AVX512F-NEXT:    vmovdqa64 (%rdi), %ymm28
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm8 = ymm28[0],ymm25[0],ymm28[2],ymm25[2]
+; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm1 = ymm8[2,3],ymm1[2,3]
+; AVX512F-NEXT:    vmovdqa64 1920(%rdi), %zmm29
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm11, %zmm1
 ; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm10
-; AVX512F-NEXT:    vmovdqa64 %zmm28, %zmm15
-; AVX512F-NEXT:    vpermt2q %zmm28, %zmm0, %zmm10
-; AVX512F-NEXT:    vmovdqa64 1792(%rdi), %zmm23
-; AVX512F-NEXT:    vmovdqa64 %zmm14, %zmm11
-; AVX512F-NEXT:    vpermi2q %zmm14, %zmm23, %zmm0
-; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm0 {%k1}
-; AVX512F-NEXT:    vmovdqa 1728(%rdi), %ymm10
-; AVX512F-NEXT:    vmovdqa64 1664(%rdi), %ymm28
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm1 = ymm28[0],ymm10[0],ymm28[2],ymm10[2]
-; AVX512F-NEXT:    vmovdqa64 1600(%rdi), %ymm31
-; AVX512F-NEXT:    vmovdqa 1536(%rdi), %ymm2
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm14 = ymm2[0],ymm31[0],ymm2[2],ymm31[2]
-; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm1 = ymm14[2,3],ymm1[2,3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vmovdqa64 %zmm29, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm16, %zmm0, %zmm1
+; AVX512F-NEXT:    vmovdqa64 1792(%rdi), %zmm26
+; AVX512F-NEXT:    vpermi2q %zmm12, %zmm26, %zmm0
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm0 {%k1}
+; AVX512F-NEXT:    vmovdqa 1728(%rdi), %ymm2
+; AVX512F-NEXT:    vmovdqa 1664(%rdi), %ymm8
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm11 = ymm8[0],ymm2[0],ymm8[2],ymm2[2]
+; AVX512F-NEXT:    vmovdqa64 1600(%rdi), %ymm30
+; AVX512F-NEXT:    vmovdqa 1536(%rdi), %ymm1
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm15 = ymm1[0],ymm30[0],ymm1[2],ymm30[2]
+; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm11 = ymm15[2,3],ymm11[2,3]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm11, %zmm0, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [3,11,3,11,3,11,3,11]
 ; AVX512F-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm22, %zmm1
-; AVX512F-NEXT:    vpermt2q %zmm17, %zmm0, %zmm1
-; AVX512F-NEXT:    vmovdqa64 %zmm18, %zmm14
-; AVX512F-NEXT:    vpermt2q %zmm21, %zmm0, %zmm14
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm14 {%k1}
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm13[1],ymm29[1],ymm13[3],ymm29[3]
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm4 = ymm4[1],ymm6[1],ymm4[3],ymm6[3]
-; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm1 = ymm4[2,3],ymm1[2,3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm14, %zmm1
-; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm1
-; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm13
-; AVX512F-NEXT:    vmovdqu64 %zmm3, (%rsp) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 %zmm24, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm24, %zmm0, %zmm1
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm4
-; AVX512F-NEXT:    vpermt2q %zmm7, %zmm0, %zmm4
-; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm29
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm9[1],ymm8[1],ymm9[3],ymm8[3]
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm3 = ymm16[1],ymm12[1],ymm16[3],ymm12[3]
-; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm1 = ymm3[2,3],ymm1[2,3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm4, %zmm1
-; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 %zmm20, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm20, %zmm1
-; AVX512F-NEXT:    vmovdqu64 %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm19, %zmm0, %zmm1
-; AVX512F-NEXT:    vmovdqa64 %zmm30, %zmm3
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm14, %zmm0, %zmm3
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm3 {%k1}
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm25[1],ymm5[1],ymm25[3],ymm5[3]
-; AVX512F-NEXT:    vmovdqa64 1216(%rdi), %zmm5
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm4 = ymm27[1],ymm26[1],ymm27[3],ymm26[3]
-; AVX512F-NEXT:    vmovdqa64 1152(%rdi), %zmm8
-; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm1 = ymm4[2,3],ymm1[2,3]
-; AVX512F-NEXT:    vmovdqa64 1088(%rdi), %zmm9
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm3, %zmm1
-; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm12, %zmm1
-; AVX512F-NEXT:    vpermt2q %zmm15, %zmm0, %zmm1
-; AVX512F-NEXT:    vmovdqa64 %zmm23, %zmm7
-; AVX512F-NEXT:    vmovdqu64 %zmm23, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermi2q %zmm11, %zmm23, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm0 {%k1}
-; AVX512F-NEXT:    vmovdqa64 1024(%rdi), %zmm3
+; AVX512F-NEXT:    vmovdqa64 %zmm27, %zmm11
+; AVX512F-NEXT:    vpermt2q %zmm5, %zmm0, %zmm11
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm24
+; AVX512F-NEXT:    vpermt2q %zmm19, %zmm0, %zmm18
+; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm18 {%k1}
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm5 = ymm10[1],ymm22[1],ymm10[3],ymm22[3]
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm6 = ymm6[1],ymm9[1],ymm6[3],ymm9[3]
+; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm5 = ymm6[2,3],ymm5[2,3]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm5, %zmm18, %zmm5
+; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm5
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm10
+; AVX512F-NEXT:    vpermt2q %zmm31, %zmm0, %zmm5
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm6
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm22 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm22, %zmm0, %zmm6
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm6 {%k1}
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm3 = ymm13[1],ymm7[1],ymm13[3],ymm7[3]
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm5 = ymm21[1],ymm17[1],ymm21[3],ymm17[3]
+; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm3 = ymm5[2,3],ymm3[2,3]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm3, %zmm6, %zmm3
 ; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm28[1],ymm10[1],ymm28[3],ymm10[3]
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm2 = ymm2[1],ymm31[1],ymm2[3],ymm31[3]
-; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm1 = ymm2[2,3],ymm1[2,3]
+; AVX512F-NEXT:    vmovdqu64 %zmm23, (%rsp) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm23, %zmm3
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm18, %zmm0, %zmm3
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm21 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm21, %zmm5
+; AVX512F-NEXT:    vpermt2q %zmm20, %zmm0, %zmm5
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm5 {%k1}
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm3 = ymm14[1],ymm4[1],ymm14[3],ymm4[3]
+; AVX512F-NEXT:    vmovdqa64 1088(%rdi), %zmm7
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm4 = ymm28[1],ymm25[1],ymm28[3],ymm25[3]
+; AVX512F-NEXT:    vmovdqa64 1024(%rdi), %zmm6
+; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm3 = ymm4[2,3],ymm3[2,3]
+; AVX512F-NEXT:    vmovdqa64 1216(%rdi), %zmm13
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm3, %zmm5, %zmm3
+; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm29, %zmm15
+; AVX512F-NEXT:    vmovdqu64 %zmm29, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm29, %zmm3
+; AVX512F-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm16, %zmm0, %zmm3
+; AVX512F-NEXT:    vmovdqa64 %zmm26, %zmm9
+; AVX512F-NEXT:    vmovdqu64 %zmm26, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermi2q %zmm12, %zmm26, %zmm0
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm0 {%k1}
+; AVX512F-NEXT:    vmovdqa64 1152(%rdi), %zmm4
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm2 = ymm8[1],ymm2[1],ymm8[3],ymm2[3]
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm1[1],ymm30[1],ymm1[3],ymm30[3]
+; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm1 = ymm1[2,3],ymm2[2,3]
 ; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [4,12,4,12,4,12,4,12]
 ; AVX512F-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm22, %zmm1
-; AVX512F-NEXT:    vmovdqa64 %zmm22, %zmm25
-; AVX512F-NEXT:    vpermt2q %zmm17, %zmm0, %zmm1
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm2 = zmm18[0],zmm21[0],zmm18[2],zmm21[2],zmm18[4],zmm21[4],zmm18[6],zmm21[6]
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}
-; AVX512F-NEXT:    vpermt2q %zmm9, %zmm0, %zmm3
-; AVX512F-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [4,12,4,12]
-; AVX512F-NEXT:    # ymm1 = mem[0,1,0,1]
-; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm4
-; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm10
-; AVX512F-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm5, %zmm1, %zmm4
-; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm8
-; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm4[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm3, %zmm2, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm24, %zmm0, %zmm13
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm3 = zmm6[0],zmm29[0],zmm6[2],zmm29[2],zmm6[4],zmm29[4],zmm6[6],zmm29[6]
-; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm16
-; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm3 {%k1}
-; AVX512F-NEXT:    vmovdqa64 576(%rdi), %zmm4
-; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 512(%rdi), %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm4, %zmm0, %zmm2
-; AVX512F-NEXT:    vmovdqa64 704(%rdi), %zmm22
-; AVX512F-NEXT:    vmovdqa64 640(%rdi), %zmm4
-; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm22, %zmm1, %zmm4
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm4[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm3, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm19, %zmm0, %zmm20
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm3 = zmm30[0],zmm14[0],zmm30[2],zmm14[2],zmm30[4],zmm14[4],zmm30[6],zmm14[6]
-; AVX512F-NEXT:    vmovdqa64 %zmm14, %zmm6
-; AVX512F-NEXT:    vmovdqa64 %zmm20, %zmm3 {%k1}
-; AVX512F-NEXT:    vmovdqa64 64(%rdi), %zmm4
-; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 (%rdi), %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm4, %zmm0, %zmm2
-; AVX512F-NEXT:    vmovdqa64 192(%rdi), %zmm5
-; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 128(%rdi), %zmm4
+; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm1
 ; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm5, %zmm1, %zmm4
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm4[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm3, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm12, %zmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm12, %zmm23
-; AVX512F-NEXT:    vpermt2q %zmm15, %zmm0, %zmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm28
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm3 = zmm7[0],zmm11[0],zmm7[2],zmm11[2],zmm7[4],zmm11[4],zmm7[6],zmm11[6]
+; AVX512F-NEXT:    vpermt2q %zmm13, %zmm0, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm8
+; AVX512F-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm5
+; AVX512F-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm7, %zmm0, %zmm2
+; AVX512F-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 %zmm27, %zmm2
+; AVX512F-NEXT:    vpermt2q %zmm24, %zmm0, %zmm2
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm3 = zmm17[0],zmm19[0],zmm17[2],zmm19[2],zmm17[4],zmm19[4],zmm17[6],zmm19[6]
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}
+; AVX512F-NEXT:    vmovdqa64 704(%rdi), %zmm6
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm3, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 640(%rdi), %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm6, %zmm0, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm25
+; AVX512F-NEXT:    vmovdqa64 576(%rdi), %zmm6
+; AVX512F-NEXT:    vmovdqa64 512(%rdi), %zmm14
+; AVX512F-NEXT:    vmovdqa64 %zmm14, %zmm2
+; AVX512F-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm6, %zmm0, %zmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm26
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm29
+; AVX512F-NEXT:    vpermt2q %zmm31, %zmm0, %zmm2
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm3 = zmm11[0],zmm22[0],zmm11[2],zmm22[2],zmm11[4],zmm22[4],zmm11[6],zmm22[6]
+; AVX512F-NEXT:    vmovdqa64 %zmm22, %zmm10
 ; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}
-; AVX512F-NEXT:    vmovdqa64 1600(%rdi), %zmm2
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm3, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 192(%rdi), %zmm2
 ; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 1536(%rdi), %zmm5
-; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermi2q %zmm2, %zmm5, %zmm0
-; AVX512F-NEXT:    vmovdqa64 1728(%rdi), %zmm2
+; AVX512F-NEXT:    vmovdqa64 128(%rdi), %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm0, %zmm1
+; AVX512F-NEXT:    vmovdqa64 64(%rdi), %zmm6
+; AVX512F-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 (%rdi), %zmm2
 ; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 1664(%rdi), %zmm5
-; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermi2q %zmm2, %zmm5, %zmm1
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm3, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm31 = [0,8,0,8,0,8,0,8]
-; AVX512F-NEXT:    # zmm31 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm18, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm21, %zmm31, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm6, %zmm0, %zmm2
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    vpermt2q %zmm18, %zmm0, %zmm23
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm3 = zmm21[0],zmm20[0],zmm21[2],zmm20[2],zmm21[4],zmm20[4],zmm21[6],zmm20[6]
+; AVX512F-NEXT:    vmovdqa64 %zmm21, %zmm22
+; AVX512F-NEXT:    vmovdqa64 %zmm23, %zmm3 {%k1}
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm3, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 1728(%rdi), %zmm18
+; AVX512F-NEXT:    vmovdqa64 1664(%rdi), %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm18, %zmm0, %zmm1
+; AVX512F-NEXT:    vmovdqa64 1600(%rdi), %zmm13
+; AVX512F-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 1536(%rdi), %zmm6
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm2
+; AVX512F-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm13, %zmm0, %zmm2
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    vpermi2q %zmm16, %zmm15, %zmm0
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm2 = zmm9[0],zmm12[0],zmm9[2],zmm12[2],zmm9[4],zmm12[4],zmm9[6],zmm12[6]
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k1}
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm2, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm24 = [1,9,1,9,1,9,1,9]
-; AVX512F-NEXT:    # zmm24 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm18, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm21, %zmm24, %zmm0
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm2 = [5,13,5,13,5,13,5,13]
+; AVX512F-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpermt2q %zmm8, %zmm2, %zmm4
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm7, %zmm2, %zmm1
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm4 = ymm1[0,1,2,3],ymm4[4,5,6,7]
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm21 = [0,8,0,8,0,8,0,8]
+; AVX512F-NEXT:    # zmm21 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm17, %zmm9
+; AVX512F-NEXT:    vmovdqa64 %zmm17, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm19, %zmm21, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm16 = [1,9,1,9,1,9,1,9]
+; AVX512F-NEXT:    # zmm16 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm17, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm19, %zmm16, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm23 = [6,14,6,14,6,14,6,14]
+; AVX512F-NEXT:    # zmm23 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm17, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm19, %zmm23, %zmm0
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm7 = zmm17[1],zmm19[1],zmm17[3],zmm19[3],zmm17[5],zmm19[5],zmm17[7],zmm19[7]
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm17 = [7,15,7,15,7,15,7,15]
+; AVX512F-NEXT:    # zmm17 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpermt2q %zmm19, %zmm17, %zmm9
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm0 {%k1} = zmm27[0],zmm24[0],zmm27[2],zmm24[2],zmm27[4],zmm24[4],zmm27[6],zmm24[6]
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm9 {%k1} = zmm27[1],zmm24[1],zmm27[3],zmm24[3],zmm27[5],zmm24[5],zmm27[7],zmm24[7]
+; AVX512F-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm27, %zmm0
+; AVX512F-NEXT:    vmovdqa64 %zmm27, %zmm8
+; AVX512F-NEXT:    vpermt2q %zmm24, %zmm21, %zmm27
+; AVX512F-NEXT:    vmovdqu64 %zmm27, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm24, %zmm16, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm24, %zmm2, %zmm8
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm7 {%k1}
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm4, %zmm7, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm4 = [6,14,6,14,6,14,6,14]
-; AVX512F-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm18, %zmm3
-; AVX512F-NEXT:    vpermt2q %zmm21, %zmm4, %zmm3
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm2 = zmm18[1],zmm21[1],zmm18[3],zmm21[3],zmm18[5],zmm21[5],zmm18[7],zmm21[7]
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm11 = [7,15,7,15,7,15,7,15]
-; AVX512F-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermt2q %zmm21, %zmm11, %zmm18
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm3 {%k1} = zmm25[0],zmm17[0],zmm25[2],zmm17[2],zmm25[4],zmm17[4],zmm25[6],zmm17[6]
-; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm18 {%k1} = zmm25[1],zmm17[1],zmm25[3],zmm17[3],zmm25[5],zmm17[5],zmm25[7],zmm17[7]
-; AVX512F-NEXT:    vmovdqu64 %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm25, %zmm3
-; AVX512F-NEXT:    vmovdqa64 %zmm25, %zmm1
-; AVX512F-NEXT:    vpermt2q %zmm17, %zmm31, %zmm25
-; AVX512F-NEXT:    vmovdqu64 %zmm25, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm17, %zmm24, %zmm3
-; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm5 = [5,13,5,13,5,13,5,13]
-; AVX512F-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermt2q %zmm17, %zmm5, %zmm1
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm9, %zmm5, %zmm3
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm9 = [5,13,5,13]
-; AVX512F-NEXT:    # ymm9 = mem[0,1,0,1]
-; AVX512F-NEXT:    vpermt2q %zmm8, %zmm9, %zmm10
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm10[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm3, %zmm2, %zmm0
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm7
+; AVX512F-NEXT:    vmovdqa64 %zmm25, %zmm9
+; AVX512F-NEXT:    vpermt2q %zmm25, %zmm2, %zmm7
+; AVX512F-NEXT:    vmovdqa64 %zmm26, %zmm13
+; AVX512F-NEXT:    vpermt2q %zmm26, %zmm2, %zmm14
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3],ymm7[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm0
+; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm4
+; AVX512F-NEXT:    vpermt2q %zmm10, %zmm21, %zmm4
+; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm4
+; AVX512F-NEXT:    vpermt2q %zmm10, %zmm16, %zmm4
+; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm30
+; AVX512F-NEXT:    vpermt2q %zmm10, %zmm23, %zmm30
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm11 = zmm11[1],zmm10[1],zmm11[3],zmm10[3],zmm11[5],zmm10[5],zmm11[7],zmm10[7]
+; AVX512F-NEXT:    vpermt2q %zmm10, %zmm17, %zmm0
+; AVX512F-NEXT:    vmovdqa64 %zmm29, %zmm4
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm30 {%k1} = zmm29[0],zmm31[0],zmm29[2],zmm31[2],zmm29[4],zmm31[4],zmm29[6],zmm31[6]
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm0 {%k1} = zmm29[1],zmm31[1],zmm29[3],zmm31[3],zmm29[5],zmm31[5],zmm29[7],zmm31[7]
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm18
-; AVX512F-NEXT:    vpermt2q %zmm29, %zmm31, %zmm18
-; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm20
-; AVX512F-NEXT:    vpermt2q %zmm29, %zmm24, %zmm20
-; AVX512F-NEXT:    vpermt2q %zmm29, %zmm4, %zmm16
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm15 = zmm2[1],zmm29[1],zmm2[3],zmm29[3],zmm2[5],zmm29[5],zmm2[7],zmm29[7]
-; AVX512F-NEXT:    vpermt2q %zmm29, %zmm11, %zmm2
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 (%rsp), %zmm26 # 64-byte Reload
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm16 {%k1} = zmm26[0],zmm0[0],zmm26[2],zmm0[2],zmm26[4],zmm0[4],zmm26[6],zmm0[6]
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm2 {%k1} = zmm26[1],zmm0[1],zmm26[3],zmm0[3],zmm26[5],zmm0[5],zmm26[7],zmm0[7]
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm26, %zmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm26, %zmm21
-; AVX512F-NEXT:    vpermt2q %zmm0, %zmm31, %zmm26
-; AVX512F-NEXT:    vpermt2q %zmm0, %zmm24, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, (%rsp) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm0, %zmm5, %zmm21
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm14, %zmm3
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm0, %zmm5, %zmm3
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm22, %zmm29
-; AVX512F-NEXT:    vpermt2q %zmm22, %zmm9, %zmm10
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm10[4,5,6,7]
-; AVX512F-NEXT:    vmovdqa64 %ymm1, %ymm27
-; AVX512F-NEXT:    vmovdqa64 %zmm30, %zmm12
-; AVX512F-NEXT:    vpermt2q %zmm6, %zmm31, %zmm12
-; AVX512F-NEXT:    vmovdqa64 %zmm30, %zmm17
-; AVX512F-NEXT:    vpermt2q %zmm6, %zmm24, %zmm17
-; AVX512F-NEXT:    vmovdqa64 %zmm30, %zmm19
-; AVX512F-NEXT:    vpermt2q %zmm6, %zmm4, %zmm19
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm13 = zmm30[1],zmm6[1],zmm30[3],zmm6[3],zmm30[5],zmm6[5],zmm30[7],zmm6[7]
-; AVX512F-NEXT:    vpermt2q %zmm6, %zmm11, %zmm30
+; AVX512F-NEXT:    vmovdqa64 %zmm29, %zmm0
+; AVX512F-NEXT:    vmovdqa64 %zmm29, %zmm14
+; AVX512F-NEXT:    vpermt2q %zmm31, %zmm21, %zmm4
+; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm31, %zmm16, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm31, %zmm2, %zmm14
+; AVX512F-NEXT:    vmovdqa64 %zmm14, %zmm11 {%k1}
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm8, %zmm11, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm28 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm28, %zmm2, %zmm8
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm11
+; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm11 # 64-byte Folded Reload
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm11 = ymm11[0,1,2,3],ymm8[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 %zmm22, %zmm0
+; AVX512F-NEXT:    vmovdqa64 %zmm22, %zmm25
+; AVX512F-NEXT:    vpermt2q %zmm20, %zmm21, %zmm25
+; AVX512F-NEXT:    vmovdqa64 %zmm22, %zmm5
+; AVX512F-NEXT:    vpermt2q %zmm20, %zmm16, %zmm5
+; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm22, %zmm27
+; AVX512F-NEXT:    vpermt2q %zmm20, %zmm23, %zmm27
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm14 = zmm22[1],zmm20[1],zmm22[3],zmm20[3],zmm22[5],zmm20[5],zmm22[7],zmm20[7]
+; AVX512F-NEXT:    vpermt2q %zmm20, %zmm17, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 (%rsp), %zmm31 # 64-byte Reload
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm27 {%k1} = zmm31[0],zmm1[0],zmm31[2],zmm1[2],zmm31[4],zmm1[4],zmm31[6],zmm1[6]
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm0 {%k1} = zmm31[1],zmm1[1],zmm31[3],zmm1[3],zmm31[5],zmm1[5],zmm31[7],zmm1[7]
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm31, %zmm0
+; AVX512F-NEXT:    vmovdqa64 %zmm31, %zmm24
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm21, %zmm31
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm16, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm2, %zmm24
+; AVX512F-NEXT:    vmovdqa64 %zmm24, %zmm14 {%k1}
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm11, %zmm14, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm11
+; AVX512F-NEXT:    vpermt2q %zmm18, %zmm2, %zmm11
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm19, %zmm2, %zmm6
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm11 = ymm6[0,1,2,3],ymm11[4,5,6,7]
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm20 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512F-NEXT:    vpermi2q %zmm20, %zmm5, %zmm2
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm22 # 64-byte Reload
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm19 {%k1} = zmm22[0],zmm1[0],zmm22[2],zmm1[2],zmm22[4],zmm1[4],zmm22[6],zmm1[6]
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm30 {%k1} = zmm22[1],zmm1[1],zmm22[3],zmm1[3],zmm22[5],zmm1[5],zmm22[7],zmm1[7]
-; AVX512F-NEXT:    vmovdqu64 %zmm30, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm22, %zmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm22, %zmm30
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm31, %zmm22
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm24, %zmm2
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm14 = zmm22[1],zmm12[1],zmm22[3],zmm12[3],zmm22[5],zmm12[5],zmm22[7],zmm12[7]
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm14 {%k1}
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm11, %zmm14, %zmm2
+; AVX512F-NEXT:    vmovdqu64 %zmm2, (%rsp) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm2
+; AVX512F-NEXT:    vpermt2q %zmm20, %zmm21, %zmm2
 ; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm5, %zmm30
-; AVX512F-NEXT:    vmovdqa64 %zmm23, %zmm7
-; AVX512F-NEXT:    vmovdqa64 %zmm23, %zmm25
-; AVX512F-NEXT:    vmovdqa64 %zmm28, %zmm1
-; AVX512F-NEXT:    vpermt2q %zmm28, %zmm31, %zmm25
+; AVX512F-NEXT:    vpermi2q %zmm12, %zmm22, %zmm21
+; AVX512F-NEXT:    vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm21
+; AVX512F-NEXT:    vpermt2q %zmm20, %zmm16, %zmm2
+; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermi2q %zmm12, %zmm22, %zmm16
+; AVX512F-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm12, %zmm16
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm14
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm24 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm24, %zmm23, %zmm14
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm26 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm26, %zmm23, %zmm11
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm12
+; AVX512F-NEXT:    vpermt2q %zmm9, %zmm23, %zmm12
+; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm29
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-NEXT:    vpermi2q %zmm2, %zmm3, %zmm31
-; AVX512F-NEXT:    vmovdqa64 %zmm23, %zmm28
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm24, %zmm28
-; AVX512F-NEXT:    vpermi2q %zmm2, %zmm3, %zmm24
-; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm23
-; AVX512F-NEXT:    vpermt2q %zmm2, %zmm4, %zmm23
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm8 = zmm3[1],zmm2[1],zmm3[3],zmm2[3],zmm3[5],zmm2[5],zmm3[7],zmm2[7]
-; AVX512F-NEXT:    vpermt2q %zmm2, %zmm11, %zmm3
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm23 {%k1} = zmm7[0],zmm1[0],zmm7[2],zmm1[2],zmm7[4],zmm1[4],zmm7[6],zmm1[6]
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm3 {%k1} = zmm7[1],zmm1[1],zmm7[3],zmm1[3],zmm7[5],zmm1[5],zmm7[7],zmm1[7]
-; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm3
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm5, %zmm3
-; AVX512F-NEXT:    vmovdqa64 %zmm14, %zmm10
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm8
+; AVX512F-NEXT:    vpermt2q %zmm13, %zmm23, %zmm8
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm10
+; AVX512F-NEXT:    vpermt2q %zmm28, %zmm23, %zmm10
+; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm5
 ; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm7
-; AVX512F-NEXT:    vpermt2q %zmm0, %zmm4, %zmm10
-; AVX512F-NEXT:    vpermt2q %zmm0, %zmm11, %zmm14
-; AVX512F-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm2
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm0, %zmm5, %zmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm4
-; AVX512F-NEXT:    vpermt2q %zmm0, %zmm7, %zmm4
-; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm6
-; AVX512F-NEXT:    vpermt2q %zmm0, %zmm11, %zmm1
-; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
-; AVX512F-NEXT:    vpermi2q %zmm14, %zmm0, %zmm5
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm7
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm6, %zmm7
-; AVX512F-NEXT:    vpermi2q %zmm14, %zmm0, %zmm6
-; AVX512F-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm14, %zmm11, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm11, %zmm4
-; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm21, %zmm15 {%k1}
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm27, %zmm15, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm4, %zmm23, %zmm7
+; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm18, %zmm23, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm30, %zmm13 {%k1}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm6
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm14, %zmm9, %zmm6
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm6[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm13, %zmm0
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm19, %zmm23, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm8 {%k1}
+; AVX512F-NEXT:    vpermi2q %zmm16, %zmm22, %zmm23
+; AVX512F-NEXT:    vpermt2q %zmm16, %zmm17, %zmm22
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm23 {%k1} = zmm21[0],zmm20[0],zmm21[2],zmm20[2],zmm21[4],zmm20[4],zmm21[6],zmm20[6]
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm22 {%k1} = zmm21[1],zmm20[1],zmm21[3],zmm20[3],zmm21[5],zmm20[5],zmm21[7],zmm20[7]
+; AVX512F-NEXT:    vpermt2q %zmm29, %zmm17, %zmm3
+; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm13, %zmm17, %zmm2
+; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm28, %zmm17, %zmm9
+; AVX512F-NEXT:    vpermt2q %zmm4, %zmm17, %zmm5
+; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm18, %zmm17, %zmm15
+; AVX512F-NEXT:    vpermt2q %zmm19, %zmm17, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm24, %zmm17, %zmm6
+; AVX512F-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm26, %zmm17, %zmm3
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm4 = ymm11[0,1,2,3],ymm14[4,5,6,7]
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm4, %zmm0, %zmm14
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm8[0,1,2,3],ymm12[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm30, %zmm4
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm7[0,1,2,3],ymm10[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm27, %zmm10
+; AVX512F-NEXT:    vmovdqa64 %zmm31, %zmm25 {%k1}
+; AVX512F-NEXT:    vmovdqa 64(%rdi), %xmm0
+; AVX512F-NEXT:    vinserti128 $1, 192(%rdi), %ymm0, %ymm0
+; AVX512F-NEXT:    vmovdqa (%rdi), %xmm1
+; AVX512F-NEXT:    vinserti128 $1, 128(%rdi), %ymm1, %ymm1
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm25, %zmm2
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm7 {%k1}
+; AVX512F-NEXT:    vmovdqa 576(%rdi), %xmm5
+; AVX512F-NEXT:    vinserti128 $1, 704(%rdi), %ymm5, %ymm5
+; AVX512F-NEXT:    vmovdqa 512(%rdi), %xmm6
+; AVX512F-NEXT:    vinserti128 $1, 640(%rdi), %ymm6, %ymm6
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm18 = ymm6[0],ymm5[0],ymm6[2],ymm5[2]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm18, %zmm7, %zmm27
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm7 {%k1}
+; AVX512F-NEXT:    vmovdqa 1088(%rdi), %xmm13
+; AVX512F-NEXT:    vinserti128 $1, 1216(%rdi), %ymm13, %ymm13
+; AVX512F-NEXT:    vmovdqa64 1024(%rdi), %xmm18
+; AVX512F-NEXT:    vinserti32x4 $1, 1152(%rdi), %ymm18, %ymm18
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm19 = ymm18[0],ymm13[0],ymm18[2],ymm13[2]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm19, %zmm7, %zmm19
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm7 {%k1}
+; AVX512F-NEXT:    vmovdqa64 1600(%rdi), %xmm21
+; AVX512F-NEXT:    vinserti32x4 $1, 1728(%rdi), %ymm21, %ymm21
+; AVX512F-NEXT:    vmovdqa64 1536(%rdi), %xmm25
+; AVX512F-NEXT:    vinserti32x4 $1, 1664(%rdi), %ymm25, %ymm25
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm26 = ymm25[0],ymm21[0],ymm25[2],ymm21[2]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm26, %zmm7, %zmm12
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm7 {%k1}
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm13 = ymm18[1],ymm13[1],ymm18[3],ymm13[3]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm13, %zmm7, %zmm13
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm7 {%k1}
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm5 = ymm6[1],ymm5[1],ymm6[3],ymm5[3]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm5, %zmm7, %zmm5
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm7 {%k1}
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm0 = ymm1[1],ymm0[1],ymm1[3],ymm0[3]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm7, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
-; AVX512F-NEXT:    vpermi2q %zmm15, %zmm6, %zmm9
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm5[0,1,2,3],ymm9[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm8, %zmm30
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [6,14,6,14]
-; AVX512F-NEXT:    # ymm2 = mem[0,1,0,1]
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm5
-; AVX512F-NEXT:    vpermt2q %zmm29, %zmm2, %zmm5
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = [7,15,7,15]
-; AVX512F-NEXT:    # ymm0 = mem[0,1,0,1]
-; AVX512F-NEXT:    vpermt2q %zmm29, %zmm0, %zmm1
-; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm9
-; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm1
-; AVX512F-NEXT:    vpermt2q %zmm14, %zmm2, %zmm1
-; AVX512F-NEXT:    vpermt2q %zmm14, %zmm0, %zmm9
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm8
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm4, %zmm2, %zmm8
-; AVX512F-NEXT:    vpermi2q %zmm15, %zmm6, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm15, %zmm0, %zmm6
-; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm3
-; AVX512F-NEXT:    vpermt2q %zmm4, %zmm0, %zmm11
-; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm6
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm7[0,1,2,3],ymm8[4,5,6,7]
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm4, %zmm21
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm4 = ymm10[0,1,2,3],ymm5[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm4, %zmm16, %zmm0
-; AVX512F-NEXT:    vmovdqa64 %zmm22, %zmm12 {%k1}
-; AVX512F-NEXT:    vmovdqa 64(%rdi), %xmm4
-; AVX512F-NEXT:    vinserti128 $1, 192(%rdi), %ymm4, %ymm4
-; AVX512F-NEXT:    vmovdqa (%rdi), %xmm5
-; AVX512F-NEXT:    vinserti128 $1, 128(%rdi), %ymm5, %ymm5
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm7 = ymm5[0],ymm4[0],ymm5[2],ymm4[2]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm7, %zmm12, %zmm7
-; AVX512F-NEXT:    vmovdqa64 %zmm26, %zmm18 {%k1}
-; AVX512F-NEXT:    vmovdqa 576(%rdi), %xmm8
-; AVX512F-NEXT:    vinserti128 $1, 704(%rdi), %ymm8, %ymm8
-; AVX512F-NEXT:    vmovdqa 512(%rdi), %xmm10
-; AVX512F-NEXT:    vinserti128 $1, 640(%rdi), %ymm10, %ymm10
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm11 = ymm10[0],ymm8[0],ymm10[2],ymm8[2]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm11, %zmm18, %zmm11
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm12, %zmm13 {%k1}
-; AVX512F-NEXT:    vmovdqa 1088(%rdi), %xmm12
-; AVX512F-NEXT:    vinserti128 $1, 1216(%rdi), %ymm12, %ymm12
-; AVX512F-NEXT:    vmovdqa64 1024(%rdi), %xmm16
-; AVX512F-NEXT:    vinserti32x4 $1, 1152(%rdi), %ymm16, %ymm16
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm22 = ymm16[0],ymm12[0],ymm16[2],ymm12[2]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm22, %zmm13, %zmm22
-; AVX512F-NEXT:    vmovdqa64 %zmm25, %zmm31 {%k1}
-; AVX512F-NEXT:    vmovdqa64 1600(%rdi), %xmm25
-; AVX512F-NEXT:    vinserti32x4 $1, 1728(%rdi), %ymm25, %ymm25
-; AVX512F-NEXT:    vmovdqa64 1536(%rdi), %xmm26
-; AVX512F-NEXT:    vinserti32x4 $1, 1664(%rdi), %ymm26, %ymm26
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm27 = ymm26[0],ymm25[0],ymm26[2],ymm25[2]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm27, %zmm31, %zmm27
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm14, %zmm13 {%k1}
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm12 = ymm16[1],ymm12[1],ymm16[3],ymm12[3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm12, %zmm13, %zmm12
-; AVX512F-NEXT:    vmovdqu64 (%rsp), %zmm13 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm20 {%k1}
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm8 = ymm10[1],ymm8[1],ymm10[3],ymm8[3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm8, %zmm20, %zmm8
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm17 {%k1}
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm4 = ymm5[1],ymm4[1],ymm5[3],ymm4[3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm4, %zmm17, %zmm4
-; AVX512F-NEXT:    vmovdqa64 %zmm28, %zmm24 {%k1}
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm5 = ymm26[1],ymm25[1],ymm26[3],ymm25[3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm5, %zmm24, %zmm5
-; AVX512F-NEXT:    vpblendd $15, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
-; AVX512F-NEXT:    # ymm1 = mem[0,1,2,3],ymm1[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm19, %zmm1
-; AVX512F-NEXT:    vpblendd $15, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload
-; AVX512F-NEXT:    # ymm2 = mem[0,1,2,3],ymm2[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm23, %zmm2
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
-; AVX512F-NEXT:    vblendps $15, {{[-0-9]+}}(%r{{[sb]}}p), %ymm10, %ymm10 # 32-byte Folded Reload
-; AVX512F-NEXT:    # ymm10 = mem[0,1,2,3],ymm10[4,5,6,7]
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512F-NEXT:    vinsertf64x4 $0, %ymm10, %zmm13, %zmm10
-; AVX512F-NEXT:    vpblendd $15, {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm13 # 32-byte Folded Reload
-; AVX512F-NEXT:    # ymm13 = mem[0,1,2,3],ymm9[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm6 {%k1}
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm25[1],ymm21[1],ymm25[3],ymm21[3]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm6, %zmm1
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512F-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm6, %ymm6 # 32-byte Folded Reload
+; AVX512F-NEXT:    # ymm6 = ymm6[0,1,2,3],mem[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm6, %zmm23, %zmm6
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512F-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm7 # 32-byte Folded Reload
+; AVX512F-NEXT:    # ymm7 = ymm7[0,1,2,3],mem[4,5,6,7]
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-NEXT:    vinsertf64x4 $0, %ymm7, %zmm8, %zmm7
+; AVX512F-NEXT:    vpblendd $15, {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm8 # 32-byte Folded Reload
+; AVX512F-NEXT:    # ymm8 = mem[0,1,2,3],ymm9[4,5,6,7]
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm13, %zmm9, %zmm13
-; AVX512F-NEXT:    vpblendd $15, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm14 # 32-byte Folded Reload
-; AVX512F-NEXT:    # ymm14 = mem[0,1,2,3],ymm3[4,5,6,7]
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm14, %zmm3, %zmm14
-; AVX512F-NEXT:    vpblendd $15, {{[-0-9]+}}(%r{{[sb]}}p), %ymm6, %ymm15 # 32-byte Folded Reload
-; AVX512F-NEXT:    # ymm15 = mem[0,1,2,3],ymm6[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm8, %zmm9, %zmm8
+; AVX512F-NEXT:    vpblendd $15, {{[-0-9]+}}(%r{{[sb]}}p), %ymm15, %ymm9 # 32-byte Folded Reload
+; AVX512F-NEXT:    # ymm9 = mem[0,1,2,3],ymm15[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm9, %zmm22, %zmm9
+; AVX512F-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm11 # 32-byte Folded Reload
+; AVX512F-NEXT:    # ymm11 = ymm3[0,1,2,3],mem[4,5,6,7]
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm15, %zmm3, %zmm15
-; AVX512F-NEXT:    vmovdqa64 %zmm27, 192(%rsi)
-; AVX512F-NEXT:    vmovdqa64 %zmm22, 128(%rsi)
-; AVX512F-NEXT:    vmovdqa64 %zmm11, 64(%rsi)
-; AVX512F-NEXT:    vmovdqa64 %zmm7, (%rsi)
-; AVX512F-NEXT:    vmovdqa64 %zmm5, 192(%rdx)
-; AVX512F-NEXT:    vmovdqa64 %zmm4, (%rdx)
-; AVX512F-NEXT:    vmovdqa64 %zmm8, 64(%rdx)
-; AVX512F-NEXT:    vmovdqa64 %zmm12, 128(%rdx)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm4, 192(%rcx)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm4, (%rcx)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm4, 64(%rcx)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm4, 128(%rcx)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm4, 192(%r8)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm4, (%r8)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm4, 64(%r8)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm4, 128(%r8)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm3, 192(%r9)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm4, (%r9)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm4, 64(%r9)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm4, 128(%r9)
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm11, %zmm3, %zmm11
+; AVX512F-NEXT:    vmovdqa64 %zmm12, 192(%rsi)
+; AVX512F-NEXT:    vmovdqa64 %zmm19, 128(%rsi)
+; AVX512F-NEXT:    vmovdqa64 %zmm27, 64(%rsi)
+; AVX512F-NEXT:    vmovdqa64 %zmm2, (%rsi)
+; AVX512F-NEXT:    vmovdqa64 %zmm1, 192(%rdx)
+; AVX512F-NEXT:    vmovdqa64 %zmm0, (%rdx)
+; AVX512F-NEXT:    vmovdqa64 %zmm5, 64(%rdx)
+; AVX512F-NEXT:    vmovdqa64 %zmm13, 128(%rdx)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm0, 192(%rcx)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm0, (%rcx)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm0, 64(%rcx)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm0, 128(%rcx)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm0, 192(%r8)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm0, (%r8)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm0, 64(%r8)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm0, 128(%r8)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm0, 192(%r9)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm0, (%r9)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm0, 64(%r9)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm0, 128(%r9)
 ; AVX512F-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512F-NEXT:    vmovdqa64 %zmm30, 192(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm3, (%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm3, 64(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm3, 128(%rax)
+; AVX512F-NEXT:    vmovups (%rsp), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm0, 192(%rax)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm0, (%rax)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm0, 64(%rax)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm0, 128(%rax)
 ; AVX512F-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512F-NEXT:    vmovdqa64 %zmm2, 192(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm1, (%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm0, 64(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm21, 128(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm6, 192(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm10, (%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm4, 64(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm14, 128(%rax)
 ; AVX512F-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512F-NEXT:    vmovdqa64 %zmm15, 128(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm14, 192(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm13, (%rax)
-; AVX512F-NEXT:    vmovaps %zmm10, 64(%rax)
-; AVX512F-NEXT:    addq $2632, %rsp # imm = 0xA48
+; AVX512F-NEXT:    vmovdqa64 %zmm11, 128(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm9, 192(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm8, (%rax)
+; AVX512F-NEXT:    vmovaps %zmm7, 64(%rax)
+; AVX512F-NEXT:    addq $3208, %rsp # imm = 0xC88
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: load_i64_stride8_vf32:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    subq $2632, %rsp # imm = 0xA48
-; AVX512BW-NEXT:    vmovdqa64 1856(%rdi), %zmm14
-; AVX512BW-NEXT:    vmovdqa64 1984(%rdi), %zmm28
-; AVX512BW-NEXT:    vmovdqa64 1344(%rdi), %zmm21
+; AVX512BW-NEXT:    subq $3208, %rsp # imm = 0xC88
+; AVX512BW-NEXT:    vmovdqa64 1856(%rdi), %zmm12
+; AVX512BW-NEXT:    vmovdqa64 1984(%rdi), %zmm16
+; AVX512BW-NEXT:    vmovdqa64 1344(%rdi), %zmm19
 ; AVX512BW-NEXT:    vmovdqa64 1280(%rdi), %zmm18
-; AVX512BW-NEXT:    vmovdqa64 1472(%rdi), %zmm17
-; AVX512BW-NEXT:    vmovdqa64 1408(%rdi), %zmm22
+; AVX512BW-NEXT:    vmovdqa64 1472(%rdi), %zmm5
+; AVX512BW-NEXT:    vmovdqa64 1408(%rdi), %zmm27
 ; AVX512BW-NEXT:    vmovdqa64 832(%rdi), %zmm7
-; AVX512BW-NEXT:    vmovdqa64 768(%rdi), %zmm8
-; AVX512BW-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 960(%rdi), %zmm10
+; AVX512BW-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 768(%rdi), %zmm11
+; AVX512BW-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 960(%rdi), %zmm31
 ; AVX512BW-NEXT:    vmovdqa64 896(%rdi), %zmm3
-; AVX512BW-NEXT:    vmovdqa64 320(%rdi), %zmm15
-; AVX512BW-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 256(%rdi), %zmm30
-; AVX512BW-NEXT:    vmovdqa64 448(%rdi), %zmm19
-; AVX512BW-NEXT:    vmovdqa64 384(%rdi), %zmm20
+; AVX512BW-NEXT:    vmovdqa64 320(%rdi), %zmm20
+; AVX512BW-NEXT:    vmovdqa64 256(%rdi), %zmm8
+; AVX512BW-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 448(%rdi), %zmm25
+; AVX512BW-NEXT:    vmovdqu64 %zmm25, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 384(%rdi), %zmm23
 ; AVX512BW-NEXT:    movb $-64, %al
 ; AVX512BW-NEXT:    kmovd %eax, %k1
 ; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [2,10,2,10,2,10,2,10]
 ; AVX512BW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm22, %zmm1
-; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm0, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm27, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm0, %zmm1
 ; AVX512BW-NEXT:    vmovdqa64 %zmm18, %zmm2
-; AVX512BW-NEXT:    vpermt2q %zmm21, %zmm0, %zmm2
+; AVX512BW-NEXT:    vmovdqu64 %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm0, %zmm2
 ; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 1216(%rdi), %ymm29
-; AVX512BW-NEXT:    vmovdqa 1152(%rdi), %ymm13
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm5 = ymm13[0],ymm29[0],ymm13[2],ymm29[2]
-; AVX512BW-NEXT:    vmovdqa 1088(%rdi), %ymm6
-; AVX512BW-NEXT:    vmovdqa 1024(%rdi), %ymm4
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm9 = ymm4[0],ymm6[0],ymm4[2],ymm6[2]
-; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm5 = ymm9[2,3],ymm5[2,3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm5, %zmm2, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 1216(%rdi), %ymm22
+; AVX512BW-NEXT:    vmovdqa 1152(%rdi), %ymm10
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm4 = ymm10[0],ymm22[0],ymm10[2],ymm22[2]
+; AVX512BW-NEXT:    vmovdqa 1088(%rdi), %ymm9
+; AVX512BW-NEXT:    vmovdqa 1024(%rdi), %ymm6
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm13 = ymm6[0],ymm9[0],ymm6[2],ymm9[2]
+; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm4 = ymm13[2,3],ymm4[2,3]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm4, %zmm2, %zmm1
 ; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm2
-; AVX512BW-NEXT:    vpermt2q %zmm10, %zmm0, %zmm2
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm24
-; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm5
-; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm0, %zmm5
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm5 {%k1}
-; AVX512BW-NEXT:    vmovdqa 704(%rdi), %ymm8
-; AVX512BW-NEXT:    vmovdqa 640(%rdi), %ymm9
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm10 = ymm9[0],ymm8[0],ymm9[2],ymm8[2]
-; AVX512BW-NEXT:    vmovdqa 576(%rdi), %ymm12
-; AVX512BW-NEXT:    vmovdqa64 512(%rdi), %ymm16
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm11 = ymm16[0],ymm12[0],ymm16[2],ymm12[2]
-; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm10 = ymm11[2,3],ymm10[2,3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm10, %zmm5, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm31, %zmm0, %zmm2
+; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm4
+; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm0, %zmm4
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm4 {%k1}
+; AVX512BW-NEXT:    vmovdqa 704(%rdi), %ymm7
+; AVX512BW-NEXT:    vmovdqa 640(%rdi), %ymm13
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm14 = ymm13[0],ymm7[0],ymm13[2],ymm7[2]
+; AVX512BW-NEXT:    vmovdqa64 576(%rdi), %ymm17
+; AVX512BW-NEXT:    vmovdqa64 512(%rdi), %ymm21
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm11 = ymm21[0],ymm17[0],ymm21[2],ymm17[2]
+; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm11 = ymm11[2,3],ymm14[2,3]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm11, %zmm4, %zmm1
 ; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm20, %zmm5
-; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm0, %zmm5
-; AVX512BW-NEXT:    vmovdqa64 %zmm30, %zmm10
-; AVX512BW-NEXT:    vpermt2q %zmm15, %zmm0, %zmm10
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm10 {%k1}
-; AVX512BW-NEXT:    vmovdqa 192(%rdi), %ymm5
-; AVX512BW-NEXT:    vmovdqa64 128(%rdi), %ymm25
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm11 = ymm25[0],ymm5[0],ymm25[2],ymm5[2]
-; AVX512BW-NEXT:    vmovdqa64 64(%rdi), %ymm26
-; AVX512BW-NEXT:    vmovdqa64 (%rdi), %ymm27
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm15 = ymm27[0],ymm26[0],ymm27[2],ymm26[2]
-; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm15 = ymm15[2,3],ymm11[2,3]
-; AVX512BW-NEXT:    vmovdqa64 1920(%rdi), %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm15, %zmm10, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm23, %zmm4
+; AVX512BW-NEXT:    vpermt2q %zmm25, %zmm0, %zmm4
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm11
+; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm0, %zmm11
+; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm11 {%k1}
+; AVX512BW-NEXT:    vmovdqa 192(%rdi), %ymm4
+; AVX512BW-NEXT:    vmovdqa 128(%rdi), %ymm14
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm1 = ymm14[0],ymm4[0],ymm14[2],ymm4[2]
+; AVX512BW-NEXT:    vmovdqa64 64(%rdi), %ymm25
+; AVX512BW-NEXT:    vmovdqa64 (%rdi), %ymm28
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm8 = ymm28[0],ymm25[0],ymm28[2],ymm25[2]
+; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm1 = ymm8[2,3],ymm1[2,3]
+; AVX512BW-NEXT:    vmovdqa64 1920(%rdi), %zmm29
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm11, %zmm1
 ; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm10
-; AVX512BW-NEXT:    vmovdqa64 %zmm28, %zmm15
-; AVX512BW-NEXT:    vpermt2q %zmm28, %zmm0, %zmm10
-; AVX512BW-NEXT:    vmovdqa64 1792(%rdi), %zmm23
-; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm11
-; AVX512BW-NEXT:    vpermi2q %zmm14, %zmm23, %zmm0
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm0 {%k1}
-; AVX512BW-NEXT:    vmovdqa 1728(%rdi), %ymm10
-; AVX512BW-NEXT:    vmovdqa64 1664(%rdi), %ymm28
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm1 = ymm28[0],ymm10[0],ymm28[2],ymm10[2]
-; AVX512BW-NEXT:    vmovdqa64 1600(%rdi), %ymm31
-; AVX512BW-NEXT:    vmovdqa 1536(%rdi), %ymm2
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm14 = ymm2[0],ymm31[0],ymm2[2],ymm31[2]
-; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm1 = ymm14[2,3],ymm1[2,3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vmovdqa64 %zmm29, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm16, %zmm0, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 1792(%rdi), %zmm26
+; AVX512BW-NEXT:    vpermi2q %zmm12, %zmm26, %zmm0
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm0 {%k1}
+; AVX512BW-NEXT:    vmovdqa 1728(%rdi), %ymm2
+; AVX512BW-NEXT:    vmovdqa 1664(%rdi), %ymm8
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm11 = ymm8[0],ymm2[0],ymm8[2],ymm2[2]
+; AVX512BW-NEXT:    vmovdqa64 1600(%rdi), %ymm30
+; AVX512BW-NEXT:    vmovdqa 1536(%rdi), %ymm1
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm15 = ymm1[0],ymm30[0],ymm1[2],ymm30[2]
+; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm11 = ymm15[2,3],ymm11[2,3]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm11, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [3,11,3,11,3,11,3,11]
 ; AVX512BW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm22, %zmm1
-; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm0, %zmm1
-; AVX512BW-NEXT:    vmovdqa64 %zmm18, %zmm14
-; AVX512BW-NEXT:    vpermt2q %zmm21, %zmm0, %zmm14
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm14 {%k1}
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm13[1],ymm29[1],ymm13[3],ymm29[3]
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm4 = ymm4[1],ymm6[1],ymm4[3],ymm6[3]
-; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm1 = ymm4[2,3],ymm1[2,3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm14, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm1
-; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm13
-; AVX512BW-NEXT:    vmovdqu64 %zmm3, (%rsp) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 %zmm24, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm24, %zmm0, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm4
-; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm0, %zmm4
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm29
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm9[1],ymm8[1],ymm9[3],ymm8[3]
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm3 = ymm16[1],ymm12[1],ymm16[3],ymm12[3]
-; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm1 = ymm3[2,3],ymm1[2,3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm4, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 %zmm20, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm20, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm0, %zmm1
-; AVX512BW-NEXT:    vmovdqa64 %zmm30, %zmm3
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm0, %zmm3
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm3 {%k1}
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm25[1],ymm5[1],ymm25[3],ymm5[3]
-; AVX512BW-NEXT:    vmovdqa64 1216(%rdi), %zmm5
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm4 = ymm27[1],ymm26[1],ymm27[3],ymm26[3]
-; AVX512BW-NEXT:    vmovdqa64 1152(%rdi), %zmm8
-; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm1 = ymm4[2,3],ymm1[2,3]
-; AVX512BW-NEXT:    vmovdqa64 1088(%rdi), %zmm9
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm3, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm1
-; AVX512BW-NEXT:    vpermt2q %zmm15, %zmm0, %zmm1
-; AVX512BW-NEXT:    vmovdqa64 %zmm23, %zmm7
-; AVX512BW-NEXT:    vmovdqu64 %zmm23, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermi2q %zmm11, %zmm23, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm0 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 1024(%rdi), %zmm3
+; AVX512BW-NEXT:    vmovdqa64 %zmm27, %zmm11
+; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm0, %zmm11
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm24
+; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm0, %zmm18
+; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm18 {%k1}
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm5 = ymm10[1],ymm22[1],ymm10[3],ymm22[3]
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm6 = ymm6[1],ymm9[1],ymm6[3],ymm9[3]
+; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm5 = ymm6[2,3],ymm5[2,3]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm5, %zmm18, %zmm5
+; AVX512BW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm5
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm10
+; AVX512BW-NEXT:    vpermt2q %zmm31, %zmm0, %zmm5
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm6
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm22 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm0, %zmm6
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm6 {%k1}
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm3 = ymm13[1],ymm7[1],ymm13[3],ymm7[3]
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm5 = ymm21[1],ymm17[1],ymm21[3],ymm17[3]
+; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm3 = ymm5[2,3],ymm3[2,3]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm3, %zmm6, %zmm3
 ; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm28[1],ymm10[1],ymm28[3],ymm10[3]
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm2 = ymm2[1],ymm31[1],ymm2[3],ymm31[3]
-; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm1 = ymm2[2,3],ymm1[2,3]
+; AVX512BW-NEXT:    vmovdqu64 %zmm23, (%rsp) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm23, %zmm3
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm0, %zmm3
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm21 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm21, %zmm5
+; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm0, %zmm5
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm5 {%k1}
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm3 = ymm14[1],ymm4[1],ymm14[3],ymm4[3]
+; AVX512BW-NEXT:    vmovdqa64 1088(%rdi), %zmm7
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm4 = ymm28[1],ymm25[1],ymm28[3],ymm25[3]
+; AVX512BW-NEXT:    vmovdqa64 1024(%rdi), %zmm6
+; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm3 = ymm4[2,3],ymm3[2,3]
+; AVX512BW-NEXT:    vmovdqa64 1216(%rdi), %zmm13
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm3, %zmm5, %zmm3
+; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm29, %zmm15
+; AVX512BW-NEXT:    vmovdqu64 %zmm29, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm29, %zmm3
+; AVX512BW-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm16, %zmm0, %zmm3
+; AVX512BW-NEXT:    vmovdqa64 %zmm26, %zmm9
+; AVX512BW-NEXT:    vmovdqu64 %zmm26, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermi2q %zmm12, %zmm26, %zmm0
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm0 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 1152(%rdi), %zmm4
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm2 = ymm8[1],ymm2[1],ymm8[3],ymm2[3]
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm1[1],ymm30[1],ymm1[3],ymm30[3]
+; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm1 = ymm1[2,3],ymm2[2,3]
 ; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [4,12,4,12,4,12,4,12]
 ; AVX512BW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm22, %zmm1
-; AVX512BW-NEXT:    vmovdqa64 %zmm22, %zmm25
-; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm0, %zmm1
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm2 = zmm18[0],zmm21[0],zmm18[2],zmm21[2],zmm18[4],zmm21[4],zmm18[6],zmm21[6]
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}
-; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm0, %zmm3
-; AVX512BW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [4,12,4,12]
-; AVX512BW-NEXT:    # ymm1 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm4
-; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm10
-; AVX512BW-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm1, %zmm4
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm8
-; AVX512BW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm4[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm3, %zmm2, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm24, %zmm0, %zmm13
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm3 = zmm6[0],zmm29[0],zmm6[2],zmm29[2],zmm6[4],zmm29[4],zmm6[6],zmm29[6]
-; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm16
-; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm3 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 576(%rdi), %zmm4
-; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 512(%rdi), %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm0, %zmm2
-; AVX512BW-NEXT:    vmovdqa64 704(%rdi), %zmm22
-; AVX512BW-NEXT:    vmovdqa64 640(%rdi), %zmm4
-; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm1, %zmm4
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm4[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm3, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm0, %zmm20
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm3 = zmm30[0],zmm14[0],zmm30[2],zmm14[2],zmm30[4],zmm14[4],zmm30[6],zmm14[6]
-; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm6
-; AVX512BW-NEXT:    vmovdqa64 %zmm20, %zmm3 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 64(%rdi), %zmm4
-; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm0, %zmm2
-; AVX512BW-NEXT:    vmovdqa64 192(%rdi), %zmm5
-; AVX512BW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 128(%rdi), %zmm4
+; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm1
 ; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm1, %zmm4
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm4[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm3, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm2
-; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm23
-; AVX512BW-NEXT:    vpermt2q %zmm15, %zmm0, %zmm2
-; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm28
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm3 = zmm7[0],zmm11[0],zmm7[2],zmm11[2],zmm7[4],zmm11[4],zmm7[6],zmm11[6]
+; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm0, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm8
+; AVX512BW-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm2
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm5
+; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm0, %zmm2
+; AVX512BW-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm27, %zmm2
+; AVX512BW-NEXT:    vpermt2q %zmm24, %zmm0, %zmm2
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm3 = zmm17[0],zmm19[0],zmm17[2],zmm19[2],zmm17[4],zmm19[4],zmm17[6],zmm19[6]
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 704(%rdi), %zmm6
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm3, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 640(%rdi), %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm0, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm25
+; AVX512BW-NEXT:    vmovdqa64 576(%rdi), %zmm6
+; AVX512BW-NEXT:    vmovdqa64 512(%rdi), %zmm14
+; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm2
+; AVX512BW-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm0, %zmm2
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm26
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm2
+; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm29
+; AVX512BW-NEXT:    vpermt2q %zmm31, %zmm0, %zmm2
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm3 = zmm11[0],zmm22[0],zmm11[2],zmm22[2],zmm11[4],zmm22[4],zmm11[6],zmm22[6]
+; AVX512BW-NEXT:    vmovdqa64 %zmm22, %zmm10
 ; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 1600(%rdi), %zmm2
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm3, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 192(%rdi), %zmm2
 ; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 1536(%rdi), %zmm5
-; AVX512BW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermi2q %zmm2, %zmm5, %zmm0
-; AVX512BW-NEXT:    vmovdqa64 1728(%rdi), %zmm2
+; AVX512BW-NEXT:    vmovdqa64 128(%rdi), %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm0, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 64(%rdi), %zmm6
+; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm2
 ; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 1664(%rdi), %zmm5
-; AVX512BW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermi2q %zmm2, %zmm5, %zmm1
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm3, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm0, %zmm2
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm0, %zmm23
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm3 = zmm21[0],zmm20[0],zmm21[2],zmm20[2],zmm21[4],zmm20[4],zmm21[6],zmm20[6]
+; AVX512BW-NEXT:    vmovdqa64 %zmm21, %zmm22
+; AVX512BW-NEXT:    vmovdqa64 %zmm23, %zmm3 {%k1}
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm3, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 1728(%rdi), %zmm18
+; AVX512BW-NEXT:    vmovdqa64 1664(%rdi), %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm0, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 1600(%rdi), %zmm13
+; AVX512BW-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 1536(%rdi), %zmm6
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm2
+; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm0, %zmm2
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512BW-NEXT:    vpermi2q %zmm16, %zmm15, %zmm0
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm2 = zmm9[0],zmm12[0],zmm9[2],zmm12[2],zmm9[4],zmm12[4],zmm9[6],zmm12[6]
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k1}
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm2, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm31 = [0,8,0,8,0,8,0,8]
-; AVX512BW-NEXT:    # zmm31 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm18, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm21, %zmm31, %zmm0
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm2 = [5,13,5,13,5,13,5,13]
+; AVX512BW-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm2, %zmm4
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm2, %zmm1
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm1[0,1,2,3],ymm4[4,5,6,7]
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm21 = [0,8,0,8,0,8,0,8]
+; AVX512BW-NEXT:    # zmm21 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm17, %zmm9
+; AVX512BW-NEXT:    vmovdqa64 %zmm17, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm21, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm24 = [1,9,1,9,1,9,1,9]
-; AVX512BW-NEXT:    # zmm24 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm18, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm21, %zmm24, %zmm0
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm16 = [1,9,1,9,1,9,1,9]
+; AVX512BW-NEXT:    # zmm16 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm17, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm16, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm23 = [6,14,6,14,6,14,6,14]
+; AVX512BW-NEXT:    # zmm23 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm17, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm23, %zmm0
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm7 = zmm17[1],zmm19[1],zmm17[3],zmm19[3],zmm17[5],zmm19[5],zmm17[7],zmm19[7]
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm17 = [7,15,7,15,7,15,7,15]
+; AVX512BW-NEXT:    # zmm17 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm17, %zmm9
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm0 {%k1} = zmm27[0],zmm24[0],zmm27[2],zmm24[2],zmm27[4],zmm24[4],zmm27[6],zmm24[6]
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm9 {%k1} = zmm27[1],zmm24[1],zmm27[3],zmm24[3],zmm27[5],zmm24[5],zmm27[7],zmm24[7]
+; AVX512BW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm27, %zmm0
+; AVX512BW-NEXT:    vmovdqa64 %zmm27, %zmm8
+; AVX512BW-NEXT:    vpermt2q %zmm24, %zmm21, %zmm27
+; AVX512BW-NEXT:    vmovdqu64 %zmm27, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm24, %zmm16, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm24, %zmm2, %zmm8
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm7 {%k1}
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm4, %zmm7, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm4 = [6,14,6,14,6,14,6,14]
-; AVX512BW-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm18, %zmm3
-; AVX512BW-NEXT:    vpermt2q %zmm21, %zmm4, %zmm3
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm2 = zmm18[1],zmm21[1],zmm18[3],zmm21[3],zmm18[5],zmm21[5],zmm18[7],zmm21[7]
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm11 = [7,15,7,15,7,15,7,15]
-; AVX512BW-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermt2q %zmm21, %zmm11, %zmm18
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm3 {%k1} = zmm25[0],zmm17[0],zmm25[2],zmm17[2],zmm25[4],zmm17[4],zmm25[6],zmm17[6]
-; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm18 {%k1} = zmm25[1],zmm17[1],zmm25[3],zmm17[3],zmm25[5],zmm17[5],zmm25[7],zmm17[7]
-; AVX512BW-NEXT:    vmovdqu64 %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm25, %zmm3
-; AVX512BW-NEXT:    vmovdqa64 %zmm25, %zmm1
-; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm31, %zmm25
-; AVX512BW-NEXT:    vmovdqu64 %zmm25, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm24, %zmm3
-; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm5 = [5,13,5,13,5,13,5,13]
-; AVX512BW-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm5, %zmm1
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm5, %zmm3
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm9 = [5,13,5,13]
-; AVX512BW-NEXT:    # ymm9 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm9, %zmm10
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm10[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm3, %zmm2, %zmm0
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm7
+; AVX512BW-NEXT:    vmovdqa64 %zmm25, %zmm9
+; AVX512BW-NEXT:    vpermt2q %zmm25, %zmm2, %zmm7
+; AVX512BW-NEXT:    vmovdqa64 %zmm26, %zmm13
+; AVX512BW-NEXT:    vpermt2q %zmm26, %zmm2, %zmm14
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm14[0,1,2,3],ymm7[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm0
+; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm4
+; AVX512BW-NEXT:    vpermt2q %zmm10, %zmm21, %zmm4
+; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm4
+; AVX512BW-NEXT:    vpermt2q %zmm10, %zmm16, %zmm4
+; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm30
+; AVX512BW-NEXT:    vpermt2q %zmm10, %zmm23, %zmm30
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm11 = zmm11[1],zmm10[1],zmm11[3],zmm10[3],zmm11[5],zmm10[5],zmm11[7],zmm10[7]
+; AVX512BW-NEXT:    vpermt2q %zmm10, %zmm17, %zmm0
+; AVX512BW-NEXT:    vmovdqa64 %zmm29, %zmm4
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm30 {%k1} = zmm29[0],zmm31[0],zmm29[2],zmm31[2],zmm29[4],zmm31[4],zmm29[6],zmm31[6]
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm0 {%k1} = zmm29[1],zmm31[1],zmm29[3],zmm31[3],zmm29[5],zmm31[5],zmm29[7],zmm31[7]
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm2
-; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm18
-; AVX512BW-NEXT:    vpermt2q %zmm29, %zmm31, %zmm18
-; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm20
-; AVX512BW-NEXT:    vpermt2q %zmm29, %zmm24, %zmm20
-; AVX512BW-NEXT:    vpermt2q %zmm29, %zmm4, %zmm16
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm15 = zmm2[1],zmm29[1],zmm2[3],zmm29[3],zmm2[5],zmm29[5],zmm2[7],zmm29[7]
-; AVX512BW-NEXT:    vpermt2q %zmm29, %zmm11, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 (%rsp), %zmm26 # 64-byte Reload
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm16 {%k1} = zmm26[0],zmm0[0],zmm26[2],zmm0[2],zmm26[4],zmm0[4],zmm26[6],zmm0[6]
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm2 {%k1} = zmm26[1],zmm0[1],zmm26[3],zmm0[3],zmm26[5],zmm0[5],zmm26[7],zmm0[7]
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm26, %zmm2
-; AVX512BW-NEXT:    vmovdqa64 %zmm26, %zmm21
-; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm31, %zmm26
-; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm24, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, (%rsp) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm5, %zmm21
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm3
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm5, %zmm3
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm22, %zmm29
-; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm9, %zmm10
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3],ymm10[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqa64 %ymm1, %ymm27
-; AVX512BW-NEXT:    vmovdqa64 %zmm30, %zmm12
-; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm31, %zmm12
-; AVX512BW-NEXT:    vmovdqa64 %zmm30, %zmm17
-; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm24, %zmm17
-; AVX512BW-NEXT:    vmovdqa64 %zmm30, %zmm19
-; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm4, %zmm19
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm13 = zmm30[1],zmm6[1],zmm30[3],zmm6[3],zmm30[5],zmm6[5],zmm30[7],zmm6[7]
-; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm11, %zmm30
+; AVX512BW-NEXT:    vmovdqa64 %zmm29, %zmm0
+; AVX512BW-NEXT:    vmovdqa64 %zmm29, %zmm14
+; AVX512BW-NEXT:    vpermt2q %zmm31, %zmm21, %zmm4
+; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm31, %zmm16, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm31, %zmm2, %zmm14
+; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm11 {%k1}
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm8, %zmm11, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm28 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm28, %zmm2, %zmm8
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm11
+; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm11 # 64-byte Folded Reload
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm11 = ymm11[0,1,2,3],ymm8[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm22, %zmm0
+; AVX512BW-NEXT:    vmovdqa64 %zmm22, %zmm25
+; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm21, %zmm25
+; AVX512BW-NEXT:    vmovdqa64 %zmm22, %zmm5
+; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm16, %zmm5
+; AVX512BW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm22, %zmm27
+; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm23, %zmm27
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm14 = zmm22[1],zmm20[1],zmm22[3],zmm20[3],zmm22[5],zmm20[5],zmm22[7],zmm20[7]
+; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm17, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 (%rsp), %zmm31 # 64-byte Reload
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm27 {%k1} = zmm31[0],zmm1[0],zmm31[2],zmm1[2],zmm31[4],zmm1[4],zmm31[6],zmm1[6]
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm0 {%k1} = zmm31[1],zmm1[1],zmm31[3],zmm1[3],zmm31[5],zmm1[5],zmm31[7],zmm1[7]
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm31, %zmm0
+; AVX512BW-NEXT:    vmovdqa64 %zmm31, %zmm24
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm21, %zmm31
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm16, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm2, %zmm24
+; AVX512BW-NEXT:    vmovdqa64 %zmm24, %zmm14 {%k1}
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm11, %zmm14, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm11
+; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm2, %zmm11
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm2, %zmm6
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm11 = ymm6[0,1,2,3],ymm11[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm20 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512BW-NEXT:    vpermi2q %zmm20, %zmm5, %zmm2
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm22 # 64-byte Reload
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm19 {%k1} = zmm22[0],zmm1[0],zmm22[2],zmm1[2],zmm22[4],zmm1[4],zmm22[6],zmm1[6]
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm30 {%k1} = zmm22[1],zmm1[1],zmm22[3],zmm1[3],zmm22[5],zmm1[5],zmm22[7],zmm1[7]
-; AVX512BW-NEXT:    vmovdqu64 %zmm30, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm22, %zmm2
-; AVX512BW-NEXT:    vmovdqa64 %zmm22, %zmm30
-; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm31, %zmm22
-; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm24, %zmm2
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm14 = zmm22[1],zmm12[1],zmm22[3],zmm12[3],zmm22[5],zmm12[5],zmm22[7],zmm12[7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm14 {%k1}
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm11, %zmm14, %zmm2
+; AVX512BW-NEXT:    vmovdqu64 %zmm2, (%rsp) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm2
+; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm21, %zmm2
 ; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm5, %zmm30
-; AVX512BW-NEXT:    vmovdqa64 %zmm23, %zmm7
-; AVX512BW-NEXT:    vmovdqa64 %zmm23, %zmm25
-; AVX512BW-NEXT:    vmovdqa64 %zmm28, %zmm1
-; AVX512BW-NEXT:    vpermt2q %zmm28, %zmm31, %zmm25
+; AVX512BW-NEXT:    vpermi2q %zmm12, %zmm22, %zmm21
+; AVX512BW-NEXT:    vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm2
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm21
+; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm16, %zmm2
+; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermi2q %zmm12, %zmm22, %zmm16
+; AVX512BW-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm16
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm14
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm24 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm24, %zmm23, %zmm14
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm26 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm26, %zmm23, %zmm11
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm12
+; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm23, %zmm12
+; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm29
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-NEXT:    vpermi2q %zmm2, %zmm3, %zmm31
-; AVX512BW-NEXT:    vmovdqa64 %zmm23, %zmm28
-; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm24, %zmm28
-; AVX512BW-NEXT:    vpermi2q %zmm2, %zmm3, %zmm24
-; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm23
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm4, %zmm23
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm8 = zmm3[1],zmm2[1],zmm3[3],zmm2[3],zmm3[5],zmm2[5],zmm3[7],zmm2[7]
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm11, %zmm3
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm23 {%k1} = zmm7[0],zmm1[0],zmm7[2],zmm1[2],zmm7[4],zmm1[4],zmm7[6],zmm1[6]
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm3 {%k1} = zmm7[1],zmm1[1],zmm7[3],zmm1[3],zmm7[5],zmm1[5],zmm7[7],zmm1[7]
-; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm3
-; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm5, %zmm3
-; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm10
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm8
+; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm23, %zmm8
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm10
+; AVX512BW-NEXT:    vpermt2q %zmm28, %zmm23, %zmm10
+; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm5
 ; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm7
-; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm4, %zmm10
-; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm11, %zmm14
-; AVX512BW-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm5, %zmm2
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm4
-; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm7, %zmm4
-; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm6
-; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm11, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
-; AVX512BW-NEXT:    vpermi2q %zmm14, %zmm0, %zmm5
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm7
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm6, %zmm7
-; AVX512BW-NEXT:    vpermi2q %zmm14, %zmm0, %zmm6
-; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm11, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm11, %zmm4
-; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm21, %zmm15 {%k1}
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm27, %zmm15, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm23, %zmm7
+; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm23, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm30, %zmm13 {%k1}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm6
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm9, %zmm6
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm6[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm13, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm23, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm8 {%k1}
+; AVX512BW-NEXT:    vpermi2q %zmm16, %zmm22, %zmm23
+; AVX512BW-NEXT:    vpermt2q %zmm16, %zmm17, %zmm22
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm23 {%k1} = zmm21[0],zmm20[0],zmm21[2],zmm20[2],zmm21[4],zmm20[4],zmm21[6],zmm20[6]
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm22 {%k1} = zmm21[1],zmm20[1],zmm21[3],zmm20[3],zmm21[5],zmm20[5],zmm21[7],zmm20[7]
+; AVX512BW-NEXT:    vpermt2q %zmm29, %zmm17, %zmm3
+; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm17, %zmm2
+; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm28, %zmm17, %zmm9
+; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm17, %zmm5
+; AVX512BW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm17, %zmm15
+; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm17, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm24, %zmm17, %zmm6
+; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm26, %zmm17, %zmm3
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm11[0,1,2,3],ymm14[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm4, %zmm0, %zmm14
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm8[0,1,2,3],ymm12[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm30, %zmm4
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm7[0,1,2,3],ymm10[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm27, %zmm10
+; AVX512BW-NEXT:    vmovdqa64 %zmm31, %zmm25 {%k1}
+; AVX512BW-NEXT:    vmovdqa 64(%rdi), %xmm0
+; AVX512BW-NEXT:    vinserti128 $1, 192(%rdi), %ymm0, %ymm0
+; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm1
+; AVX512BW-NEXT:    vinserti128 $1, 128(%rdi), %ymm1, %ymm1
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm25, %zmm2
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm7 {%k1}
+; AVX512BW-NEXT:    vmovdqa 576(%rdi), %xmm5
+; AVX512BW-NEXT:    vinserti128 $1, 704(%rdi), %ymm5, %ymm5
+; AVX512BW-NEXT:    vmovdqa 512(%rdi), %xmm6
+; AVX512BW-NEXT:    vinserti128 $1, 640(%rdi), %ymm6, %ymm6
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm18 = ymm6[0],ymm5[0],ymm6[2],ymm5[2]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm18, %zmm7, %zmm27
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm7 {%k1}
+; AVX512BW-NEXT:    vmovdqa 1088(%rdi), %xmm13
+; AVX512BW-NEXT:    vinserti128 $1, 1216(%rdi), %ymm13, %ymm13
+; AVX512BW-NEXT:    vmovdqa64 1024(%rdi), %xmm18
+; AVX512BW-NEXT:    vinserti32x4 $1, 1152(%rdi), %ymm18, %ymm18
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm19 = ymm18[0],ymm13[0],ymm18[2],ymm13[2]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm19, %zmm7, %zmm19
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm7 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 1600(%rdi), %xmm21
+; AVX512BW-NEXT:    vinserti32x4 $1, 1728(%rdi), %ymm21, %ymm21
+; AVX512BW-NEXT:    vmovdqa64 1536(%rdi), %xmm25
+; AVX512BW-NEXT:    vinserti32x4 $1, 1664(%rdi), %ymm25, %ymm25
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm26 = ymm25[0],ymm21[0],ymm25[2],ymm21[2]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm26, %zmm7, %zmm12
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm7 {%k1}
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm13 = ymm18[1],ymm13[1],ymm18[3],ymm13[3]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm13, %zmm7, %zmm13
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm7 {%k1}
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm5 = ymm6[1],ymm5[1],ymm6[3],ymm5[3]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm5, %zmm7, %zmm5
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm7 {%k1}
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm0 = ymm1[1],ymm0[1],ymm1[3],ymm0[3]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm7, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
-; AVX512BW-NEXT:    vpermi2q %zmm15, %zmm6, %zmm9
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm5[0,1,2,3],ymm9[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm8, %zmm30
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [6,14,6,14]
-; AVX512BW-NEXT:    # ymm2 = mem[0,1,0,1]
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm5
-; AVX512BW-NEXT:    vpermt2q %zmm29, %zmm2, %zmm5
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = [7,15,7,15]
-; AVX512BW-NEXT:    # ymm0 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vpermt2q %zmm29, %zmm0, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm9
-; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm1
-; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm2, %zmm1
-; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm0, %zmm9
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm8
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm2, %zmm8
-; AVX512BW-NEXT:    vpermi2q %zmm15, %zmm6, %zmm2
-; AVX512BW-NEXT:    vpermt2q %zmm15, %zmm0, %zmm6
-; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm3
-; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm0, %zmm11
-; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm6
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm7[0,1,2,3],ymm8[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm4, %zmm21
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm10[0,1,2,3],ymm5[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm4, %zmm16, %zmm0
-; AVX512BW-NEXT:    vmovdqa64 %zmm22, %zmm12 {%k1}
-; AVX512BW-NEXT:    vmovdqa 64(%rdi), %xmm4
-; AVX512BW-NEXT:    vinserti128 $1, 192(%rdi), %ymm4, %ymm4
-; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm5
-; AVX512BW-NEXT:    vinserti128 $1, 128(%rdi), %ymm5, %ymm5
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm7 = ymm5[0],ymm4[0],ymm5[2],ymm4[2]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm7, %zmm12, %zmm7
-; AVX512BW-NEXT:    vmovdqa64 %zmm26, %zmm18 {%k1}
-; AVX512BW-NEXT:    vmovdqa 576(%rdi), %xmm8
-; AVX512BW-NEXT:    vinserti128 $1, 704(%rdi), %ymm8, %ymm8
-; AVX512BW-NEXT:    vmovdqa 512(%rdi), %xmm10
-; AVX512BW-NEXT:    vinserti128 $1, 640(%rdi), %ymm10, %ymm10
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm11 = ymm10[0],ymm8[0],ymm10[2],ymm8[2]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm11, %zmm18, %zmm11
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm13 {%k1}
-; AVX512BW-NEXT:    vmovdqa 1088(%rdi), %xmm12
-; AVX512BW-NEXT:    vinserti128 $1, 1216(%rdi), %ymm12, %ymm12
-; AVX512BW-NEXT:    vmovdqa64 1024(%rdi), %xmm16
-; AVX512BW-NEXT:    vinserti32x4 $1, 1152(%rdi), %ymm16, %ymm16
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm22 = ymm16[0],ymm12[0],ymm16[2],ymm12[2]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm22, %zmm13, %zmm22
-; AVX512BW-NEXT:    vmovdqa64 %zmm25, %zmm31 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 1600(%rdi), %xmm25
-; AVX512BW-NEXT:    vinserti32x4 $1, 1728(%rdi), %ymm25, %ymm25
-; AVX512BW-NEXT:    vmovdqa64 1536(%rdi), %xmm26
-; AVX512BW-NEXT:    vinserti32x4 $1, 1664(%rdi), %ymm26, %ymm26
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm27 = ymm26[0],ymm25[0],ymm26[2],ymm25[2]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm27, %zmm31, %zmm27
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm13 {%k1}
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm12 = ymm16[1],ymm12[1],ymm16[3],ymm12[3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm12, %zmm13, %zmm12
-; AVX512BW-NEXT:    vmovdqu64 (%rsp), %zmm13 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm20 {%k1}
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm8 = ymm10[1],ymm8[1],ymm10[3],ymm8[3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm8, %zmm20, %zmm8
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm17 {%k1}
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm4 = ymm5[1],ymm4[1],ymm5[3],ymm4[3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm4, %zmm17, %zmm4
-; AVX512BW-NEXT:    vmovdqa64 %zmm28, %zmm24 {%k1}
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm5 = ymm26[1],ymm25[1],ymm26[3],ymm25[3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm5, %zmm24, %zmm5
-; AVX512BW-NEXT:    vpblendd $15, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
-; AVX512BW-NEXT:    # ymm1 = mem[0,1,2,3],ymm1[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm19, %zmm1
-; AVX512BW-NEXT:    vpblendd $15, {{[-0-9]+}}(%r{{[sb]}}p), %ymm2, %ymm2 # 32-byte Folded Reload
-; AVX512BW-NEXT:    # ymm2 = mem[0,1,2,3],ymm2[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm23, %zmm2
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
-; AVX512BW-NEXT:    vblendps $15, {{[-0-9]+}}(%r{{[sb]}}p), %ymm10, %ymm10 # 32-byte Folded Reload
-; AVX512BW-NEXT:    # ymm10 = mem[0,1,2,3],ymm10[4,5,6,7]
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512BW-NEXT:    vinsertf64x4 $0, %ymm10, %zmm13, %zmm10
-; AVX512BW-NEXT:    vpblendd $15, {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm13 # 32-byte Folded Reload
-; AVX512BW-NEXT:    # ymm13 = mem[0,1,2,3],ymm9[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm6 {%k1}
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm25[1],ymm21[1],ymm25[3],ymm21[3]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm6, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512BW-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm6, %ymm6 # 32-byte Folded Reload
+; AVX512BW-NEXT:    # ymm6 = ymm6[0,1,2,3],mem[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm6, %zmm23, %zmm6
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512BW-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm7 # 32-byte Folded Reload
+; AVX512BW-NEXT:    # ymm7 = ymm7[0,1,2,3],mem[4,5,6,7]
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512BW-NEXT:    vinsertf64x4 $0, %ymm7, %zmm8, %zmm7
+; AVX512BW-NEXT:    vpblendd $15, {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm8 # 32-byte Folded Reload
+; AVX512BW-NEXT:    # ymm8 = mem[0,1,2,3],ymm9[4,5,6,7]
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm13, %zmm9, %zmm13
-; AVX512BW-NEXT:    vpblendd $15, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm14 # 32-byte Folded Reload
-; AVX512BW-NEXT:    # ymm14 = mem[0,1,2,3],ymm3[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm8, %zmm9, %zmm8
+; AVX512BW-NEXT:    vpblendd $15, {{[-0-9]+}}(%r{{[sb]}}p), %ymm15, %ymm9 # 32-byte Folded Reload
+; AVX512BW-NEXT:    # ymm9 = mem[0,1,2,3],ymm15[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm9, %zmm22, %zmm9
+; AVX512BW-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm11 # 32-byte Folded Reload
+; AVX512BW-NEXT:    # ymm11 = ymm3[0,1,2,3],mem[4,5,6,7]
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm14, %zmm3, %zmm14
-; AVX512BW-NEXT:    vpblendd $15, {{[-0-9]+}}(%r{{[sb]}}p), %ymm6, %ymm15 # 32-byte Folded Reload
-; AVX512BW-NEXT:    # ymm15 = mem[0,1,2,3],ymm6[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm15, %zmm3, %zmm15
-; AVX512BW-NEXT:    vmovdqa64 %zmm27, 192(%rsi)
-; AVX512BW-NEXT:    vmovdqa64 %zmm22, 128(%rsi)
-; AVX512BW-NEXT:    vmovdqa64 %zmm11, 64(%rsi)
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, (%rsi)
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, 192(%rdx)
-; AVX512BW-NEXT:    vmovdqa64 %zmm4, (%rdx)
-; AVX512BW-NEXT:    vmovdqa64 %zmm8, 64(%rdx)
-; AVX512BW-NEXT:    vmovdqa64 %zmm12, 128(%rdx)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm4, 192(%rcx)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm4, (%rcx)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm4, 64(%rcx)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm4, 128(%rcx)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm4, 192(%r8)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm4, (%r8)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm4, 64(%r8)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm4, 128(%r8)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm3, 192(%r9)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm4, (%r9)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm4, 64(%r9)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm4, 128(%r9)
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm11, %zmm3, %zmm11
+; AVX512BW-NEXT:    vmovdqa64 %zmm12, 192(%rsi)
+; AVX512BW-NEXT:    vmovdqa64 %zmm19, 128(%rsi)
+; AVX512BW-NEXT:    vmovdqa64 %zmm27, 64(%rsi)
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, (%rsi)
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, 192(%rdx)
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rdx)
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, 64(%rdx)
+; AVX512BW-NEXT:    vmovdqa64 %zmm13, 128(%rdx)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm0, 192(%rcx)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm0, (%rcx)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm0, 64(%rcx)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm0, 128(%rcx)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm0, 192(%r8)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm0, (%r8)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm0, 64(%r8)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm0, 128(%r8)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm0, 192(%r9)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm0, (%r9)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm0, 64(%r9)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm0, 128(%r9)
 ; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512BW-NEXT:    vmovdqa64 %zmm30, 192(%rax)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm3, (%rax)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm3, 64(%rax)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm3, 128(%rax)
+; AVX512BW-NEXT:    vmovups (%rsp), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm0, 192(%rax)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm0, (%rax)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm0, 64(%rax)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm0, 128(%rax)
 ; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, 192(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, (%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, 64(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm21, 128(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, 192(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm10, (%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm4, 64(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm14, 128(%rax)
 ; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512BW-NEXT:    vmovdqa64 %zmm15, 128(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm14, 192(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm13, (%rax)
-; AVX512BW-NEXT:    vmovaps %zmm10, 64(%rax)
-; AVX512BW-NEXT:    addq $2632, %rsp # imm = 0xA48
+; AVX512BW-NEXT:    vmovdqa64 %zmm11, 128(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm9, 192(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, (%rax)
+; AVX512BW-NEXT:    vmovaps %zmm7, 64(%rax)
+; AVX512BW-NEXT:    addq $3208, %rsp # imm = 0xC88
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
   %wide.vec = load <256 x i64>, ptr %in.vec, align 64
@@ -9123,42 +9133,41 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ;
 ; AVX512F-LABEL: load_i64_stride8_vf64:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    subq $6600, %rsp # imm = 0x19C8
-; AVX512F-NEXT:    vmovdqa64 3392(%rdi), %zmm13
-; AVX512F-NEXT:    vmovdqa64 3328(%rdi), %zmm7
-; AVX512F-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 3520(%rdi), %zmm1
-; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 3456(%rdi), %zmm10
-; AVX512F-NEXT:    vmovdqa64 1856(%rdi), %zmm17
-; AVX512F-NEXT:    vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 1984(%rdi), %zmm12
-; AVX512F-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 832(%rdi), %zmm8
-; AVX512F-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 768(%rdi), %zmm4
-; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 960(%rdi), %zmm9
-; AVX512F-NEXT:    vmovdqa64 896(%rdi), %zmm11
+; AVX512F-NEXT:    subq $6664, %rsp # imm = 0x1A08
+; AVX512F-NEXT:    vmovdqa64 3392(%rdi), %zmm3
+; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 3328(%rdi), %zmm16
+; AVX512F-NEXT:    vmovdqa64 3520(%rdi), %zmm8
+; AVX512F-NEXT:    vmovdqa64 3456(%rdi), %zmm28
+; AVX512F-NEXT:    vmovdqa64 1856(%rdi), %zmm10
+; AVX512F-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 1984(%rdi), %zmm11
 ; AVX512F-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 320(%rdi), %zmm15
-; AVX512F-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 256(%rdi), %zmm5
+; AVX512F-NEXT:    vmovdqa64 832(%rdi), %zmm4
+; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 768(%rdi), %zmm5
 ; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 448(%rdi), %zmm16
-; AVX512F-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 384(%rdi), %zmm6
+; AVX512F-NEXT:    vmovdqa64 960(%rdi), %zmm6
+; AVX512F-NEXT:    vmovdqa64 896(%rdi), %zmm9
+; AVX512F-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 320(%rdi), %zmm12
+; AVX512F-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 256(%rdi), %zmm15
+; AVX512F-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 448(%rdi), %zmm13
+; AVX512F-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 384(%rdi), %zmm14
+; AVX512F-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    movb $-64, %al
 ; AVX512F-NEXT:    kmovw %eax, %k1
 ; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm2 = [2,10,2,10,2,10,2,10]
 ; AVX512F-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm0
-; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm29
-; AVX512F-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm2, %zmm0
-; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm1
-; AVX512F-NEXT:    vpermt2q %zmm13, %zmm2, %zmm1
-; AVX512F-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm28, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm28, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm8, %zmm2, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm2, %zmm1
 ; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
 ; AVX512F-NEXT:    vmovdqa 3264(%rdi), %ymm3
 ; AVX512F-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
@@ -9167,62 +9176,60 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm3[0],ymm0[2],ymm3[2]
 ; AVX512F-NEXT:    vmovdqa 3136(%rdi), %ymm3
 ; AVX512F-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512F-NEXT:    vmovdqa 3072(%rdi), %ymm14
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm3 = ymm14[0],ymm3[0],ymm14[2],ymm3[2]
+; AVX512F-NEXT:    vmovdqa 3072(%rdi), %ymm7
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm3 = ymm7[0],ymm3[0],ymm7[2],ymm3[2]
 ; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm3[2,3],ymm0[2,3]
 ; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm1, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm9, %zmm2, %zmm0
-; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm7
-; AVX512F-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm1
-; AVX512F-NEXT:    vpermt2q %zmm8, %zmm2, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm6, %zmm2, %zmm0
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm9
+; AVX512F-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm4, %zmm2, %zmm1
 ; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512F-NEXT:    vmovdqa 704(%rdi), %ymm3
-; AVX512F-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512F-NEXT:    vmovdqa 640(%rdi), %ymm0
+; AVX512F-NEXT:    vmovdqa 704(%rdi), %ymm0
 ; AVX512F-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm3[0],ymm0[2],ymm3[2]
-; AVX512F-NEXT:    vmovdqa64 576(%rdi), %ymm26
-; AVX512F-NEXT:    vmovdqa64 512(%rdi), %ymm23
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm3 = ymm23[0],ymm26[0],ymm23[2],ymm26[2]
+; AVX512F-NEXT:    vmovdqa64 640(%rdi), %ymm20
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm20[0],ymm0[0],ymm20[2],ymm0[2]
+; AVX512F-NEXT:    vmovdqa64 576(%rdi), %ymm22
+; AVX512F-NEXT:    vmovdqa64 512(%rdi), %ymm19
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm3 = ymm19[0],ymm22[0],ymm19[2],ymm22[2]
 ; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm3[2,3],ymm0[2,3]
 ; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm1, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm0
-; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm8
-; AVX512F-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm16, %zmm2, %zmm0
-; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm1
-; AVX512F-NEXT:    vpermt2q %zmm15, %zmm2, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm14, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm13, %zmm2, %zmm0
+; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm12, %zmm2, %zmm1
 ; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512F-NEXT:    vmovdqa 192(%rdi), %ymm0
-; AVX512F-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512F-NEXT:    vmovdqa64 128(%rdi), %ymm30
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm30[0],ymm0[0],ymm30[2],ymm0[2]
-; AVX512F-NEXT:    vmovdqa64 64(%rdi), %ymm20
-; AVX512F-NEXT:    vmovdqa64 (%rdi), %ymm16
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm3 = ymm16[0],ymm20[0],ymm16[2],ymm20[2]
+; AVX512F-NEXT:    vmovdqa 192(%rdi), %ymm14
+; AVX512F-NEXT:    vmovdqa 128(%rdi), %ymm15
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm15[0],ymm14[0],ymm15[2],ymm14[2]
+; AVX512F-NEXT:    vmovdqa64 64(%rdi), %ymm21
+; AVX512F-NEXT:    vmovdqa (%rdi), %ymm13
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm3 = ymm13[0],ymm21[0],ymm13[2],ymm21[2]
 ; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm3[2,3],ymm0[2,3]
 ; AVX512F-NEXT:    vmovdqa64 1920(%rdi), %zmm3
 ; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm1, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm12, %zmm2, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm11, %zmm2, %zmm0
 ; AVX512F-NEXT:    vmovdqa64 1792(%rdi), %zmm1
 ; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm17, %zmm2, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm10, %zmm2, %zmm1
 ; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
 ; AVX512F-NEXT:    vmovdqa 1728(%rdi), %ymm3
 ; AVX512F-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512F-NEXT:    vmovdqa 1664(%rdi), %ymm0
 ; AVX512F-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm3[0],ymm0[2],ymm3[2]
-; AVX512F-NEXT:    vmovdqa64 1600(%rdi), %ymm21
-; AVX512F-NEXT:    vmovdqa64 1536(%rdi), %ymm17
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm3 = ymm17[0],ymm21[0],ymm17[2],ymm21[2]
+; AVX512F-NEXT:    vmovdqa 1600(%rdi), %ymm4
+; AVX512F-NEXT:    vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512F-NEXT:    vmovdqa 1536(%rdi), %ymm3
+; AVX512F-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm3 = ymm3[0],ymm4[0],ymm3[2],ymm4[2]
 ; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm3[2,3],ymm0[2,3]
 ; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm1, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -9233,61 +9240,64 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512F-NEXT:    vpermt2q %zmm1, %zmm2, %zmm0
 ; AVX512F-NEXT:    vmovdqa64 1344(%rdi), %zmm3
 ; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 1280(%rdi), %zmm11
-; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm1
-; AVX512F-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 1280(%rdi), %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, (%rsp) # 64-byte Spill
 ; AVX512F-NEXT:    vpermt2q %zmm3, %zmm2, %zmm1
 ; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512F-NEXT:    vmovdqa 1216(%rdi), %ymm0
+; AVX512F-NEXT:    vmovdqa 1216(%rdi), %ymm3
+; AVX512F-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512F-NEXT:    vmovdqa 1152(%rdi), %ymm0
 ; AVX512F-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512F-NEXT:    vmovdqa64 1152(%rdi), %ymm25
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm25[0],ymm0[0],ymm25[2],ymm0[2]
-; AVX512F-NEXT:    vmovdqa64 1088(%rdi), %ymm24
-; AVX512F-NEXT:    vmovdqa64 1024(%rdi), %ymm22
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm3 = ymm22[0],ymm24[0],ymm22[2],ymm24[2]
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm3[0],ymm0[2],ymm3[2]
+; AVX512F-NEXT:    vmovdqa 1088(%rdi), %ymm4
+; AVX512F-NEXT:    vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512F-NEXT:    vmovdqa 1024(%rdi), %ymm3
+; AVX512F-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm3 = ymm3[0],ymm4[0],ymm3[2],ymm4[2]
 ; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm3[2,3],ymm0[2,3]
 ; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm1, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vmovdqa64 3008(%rdi), %zmm1
 ; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 2944(%rdi), %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 2944(%rdi), %zmm29
+; AVX512F-NEXT:    vmovdqa64 %zmm29, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm29, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vpermt2q %zmm1, %zmm2, %zmm0
-; AVX512F-NEXT:    vmovdqa64 2880(%rdi), %zmm3
-; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 2816(%rdi), %zmm1
-; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm3, %zmm2, %zmm1
+; AVX512F-NEXT:    vmovdqa64 2880(%rdi), %zmm24
+; AVX512F-NEXT:    vmovdqa64 2816(%rdi), %zmm25
+; AVX512F-NEXT:    vmovdqa64 %zmm25, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm25, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm24, %zmm2, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm24, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512F-NEXT:    vmovdqa 2752(%rdi), %ymm3
-; AVX512F-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512F-NEXT:    vmovdqa 2688(%rdi), %ymm0
-; AVX512F-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm3[0],ymm0[2],ymm3[2]
-; AVX512F-NEXT:    vmovdqa64 2624(%rdi), %ymm31
-; AVX512F-NEXT:    vmovdqa 2560(%rdi), %ymm10
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm3 = ymm10[0],ymm31[0],ymm10[2],ymm31[2]
+; AVX512F-NEXT:    vmovdqa64 2752(%rdi), %ymm27
+; AVX512F-NEXT:    vmovdqa64 2688(%rdi), %ymm26
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm26[0],ymm27[0],ymm26[2],ymm27[2]
+; AVX512F-NEXT:    vmovdqa64 2624(%rdi), %ymm30
+; AVX512F-NEXT:    vmovdqa64 2560(%rdi), %ymm18
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm3 = ymm18[0],ymm30[0],ymm18[2],ymm30[2]
 ; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm3[2,3],ymm0[2,3]
 ; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm1, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vmovdqa64 2496(%rdi), %zmm1
 ; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 2432(%rdi), %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 2432(%rdi), %zmm31
+; AVX512F-NEXT:    vmovdqa64 %zmm31, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm31, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vpermt2q %zmm1, %zmm2, %zmm0
-; AVX512F-NEXT:    vmovdqa64 2368(%rdi), %zmm18
+; AVX512F-NEXT:    vmovdqa64 2368(%rdi), %zmm3
+; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vmovdqa64 2304(%rdi), %zmm1
-; AVX512F-NEXT:    vmovdqu64 %zmm1, (%rsp) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm18, %zmm2, %zmm1
-; AVX512F-NEXT:    vmovdqu64 %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm2, %zmm1
 ; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512F-NEXT:    vmovdqa64 2240(%rdi), %ymm28
-; AVX512F-NEXT:    vmovdqa64 2176(%rdi), %ymm19
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm19[0],ymm28[0],ymm19[2],ymm28[2]
-; AVX512F-NEXT:    vmovdqa64 2112(%rdi), %ymm27
-; AVX512F-NEXT:    vmovdqa 2048(%rdi), %ymm6
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm3 = ymm6[0],ymm27[0],ymm6[2],ymm27[2]
-; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm3[2,3],ymm0[2,3]
+; AVX512F-NEXT:    vmovdqa 2240(%rdi), %ymm12
+; AVX512F-NEXT:    vmovdqa 2176(%rdi), %ymm11
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm11[0],ymm12[0],ymm11[2],ymm12[2]
+; AVX512F-NEXT:    vmovdqa 2112(%rdi), %ymm10
+; AVX512F-NEXT:    vmovdqa 2048(%rdi), %ymm3
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm5 = ymm3[0],ymm10[0],ymm3[2],ymm10[2]
+; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm5[2,3],ymm0[2,3]
 ; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm1, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vmovdqa64 4032(%rdi), %zmm1
@@ -9295,807 +9305,801 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512F-NEXT:    vmovdqa64 3968(%rdi), %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vpermt2q %zmm1, %zmm2, %zmm0
-; AVX512F-NEXT:    vmovdqa64 3904(%rdi), %zmm5
-; AVX512F-NEXT:    vmovdqa64 3840(%rdi), %zmm3
-; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermi2q %zmm5, %zmm3, %zmm2
+; AVX512F-NEXT:    vmovdqa64 3904(%rdi), %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 3840(%rdi), %zmm4
+; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermi2q %zmm1, %zmm4, %zmm2
 ; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k1}
-; AVX512F-NEXT:    vmovdqa 3776(%rdi), %ymm12
-; AVX512F-NEXT:    vmovdqa 3712(%rdi), %ymm9
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm9[0],ymm12[0],ymm9[2],ymm12[2]
-; AVX512F-NEXT:    vmovdqa 3648(%rdi), %ymm4
-; AVX512F-NEXT:    vmovdqa 3584(%rdi), %ymm1
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm15 = ymm1[0],ymm4[0],ymm1[2],ymm4[2]
-; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm15[2,3],ymm0[2,3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm2, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 3776(%rdi), %ymm17
+; AVX512F-NEXT:    vmovdqa64 3712(%rdi), %ymm23
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm5 = ymm23[0],ymm17[0],ymm23[2],ymm17[2]
+; AVX512F-NEXT:    vmovdqa 3648(%rdi), %ymm1
+; AVX512F-NEXT:    vmovdqa 3584(%rdi), %ymm0
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm6 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
+; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm5 = ymm6[2,3],ymm5[2,3]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm5, %zmm2, %zmm2
+; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm2 = [3,11,3,11,3,11,3,11]
 ; AVX512F-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm29, %zmm0
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm3, %zmm2, %zmm0
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm13, %zmm2, %zmm15
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm15 {%k1}
-; AVX512F-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX512F-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
-; AVX512F-NEXT:    # ymm0 = ymm0[1],mem[1],ymm0[3],mem[3]
-; AVX512F-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm14 # 32-byte Folded Reload
-; AVX512F-NEXT:    # ymm14 = ymm14[1],mem[1],ymm14[3],mem[3]
-; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm14[2,3],ymm0[2,3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm15, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm7, %zmm2, %zmm0
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm14
-; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm14 # 64-byte Folded Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm14 {%k1}
-; AVX512F-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX512F-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
-; AVX512F-NEXT:    # ymm0 = ymm0[1],mem[1],ymm0[3],mem[3]
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm15 = ymm23[1],ymm26[1],ymm23[3],ymm26[3]
-; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm15[2,3],ymm0[2,3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm14, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm23 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm23, %zmm2, %zmm8
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm29 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm29, %zmm14
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm26 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm26, %zmm2, %zmm14
-; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm14 {%k1}
-; AVX512F-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm30, %ymm0 # 32-byte Folded Reload
-; AVX512F-NEXT:    # ymm0 = ymm30[1],mem[1],ymm30[3],mem[3]
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm15 = ymm16[1],ymm20[1],ymm16[3],ymm20[3]
-; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm15[2,3],ymm0[2,3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm14, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm30 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm30, %zmm0
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm20 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm20, %zmm2, %zmm0
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm14
+; AVX512F-NEXT:    vpermt2q %zmm8, %zmm2, %zmm28
+; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm8
+; AVX512F-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm6
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm16, %zmm2, %zmm14
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm14 {%k1}
-; AVX512F-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX512F-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
-; AVX512F-NEXT:    # ymm0 = ymm0[1],mem[1],ymm0[3],mem[3]
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm15 = ymm17[1],ymm21[1],ymm17[3],ymm21[3]
-; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm15[2,3],ymm0[2,3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm14, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm21 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm21, %zmm0
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm17, %zmm2, %zmm0
-; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm14
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm11, %zmm2, %zmm14
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm14 {%k1}
-; AVX512F-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm25, %ymm0 # 32-byte Folded Reload
-; AVX512F-NEXT:    # ymm0 = ymm25[1],mem[1],ymm25[3],mem[3]
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm15 = ymm22[1],ymm24[1],ymm22[3],ymm24[3]
-; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm15[2,3],ymm0[2,3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm14, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm16, %zmm2, %zmm6
+; AVX512F-NEXT:    vmovdqa64 %zmm28, %zmm6 {%k1}
+; AVX512F-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
+; AVX512F-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm5 # 32-byte Folded Reload
+; AVX512F-NEXT:    # ymm5 = ymm4[1],mem[1],ymm4[3],mem[3]
+; AVX512F-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm7 # 32-byte Folded Reload
+; AVX512F-NEXT:    # ymm7 = ymm7[1],mem[1],ymm7[3],mem[3]
+; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm5 = ymm7[2,3],ymm5[2,3]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm5, %zmm6, %zmm5
+; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm28 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm28, %zmm5
+; AVX512F-NEXT:    vpermt2q %zmm9, %zmm2, %zmm5
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm6
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm4, %zmm2, %zmm6
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm6 {%k1}
+; AVX512F-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm20, %ymm5 # 32-byte Folded Reload
+; AVX512F-NEXT:    # ymm5 = ymm20[1],mem[1],ymm20[3],mem[3]
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm7 = ymm19[1],ymm22[1],ymm19[3],ymm22[3]
+; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm5 = ymm7[2,3],ymm5[2,3]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm5, %zmm6, %zmm5
+; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm20 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm20, %zmm5
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm19, %zmm2, %zmm5
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm22 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm22, %zmm0
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm25 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm25, %zmm2, %zmm0
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm24 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm24, %zmm14
+; AVX512F-NEXT:    vpermt2q %zmm22, %zmm2, %zmm6
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm6 {%k1}
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm5 = ymm15[1],ymm14[1],ymm15[3],ymm14[3]
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm7 = ymm13[1],ymm21[1],ymm13[3],ymm21[3]
+; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm5 = ymm7[2,3],ymm5[2,3]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm5, %zmm6, %zmm5
+; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm14, %zmm5
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm15, %zmm2, %zmm14
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm14 {%k1}
-; AVX512F-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX512F-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
-; AVX512F-NEXT:    # ymm0 = ymm0[1],mem[1],ymm0[3],mem[3]
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm10 = ymm10[1],ymm31[1],ymm10[3],ymm31[3]
-; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm10[2,3],ymm0[2,3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm14, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm31 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm31, %zmm0
-; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm0 # 64-byte Folded Reload
-; AVX512F-NEXT:    vmovdqu64 (%rsp), %zmm10 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm18, %zmm2, %zmm10
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm10 {%k1}
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm0 = ymm19[1],ymm28[1],ymm19[3],ymm28[3]
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm6 = ymm6[1],ymm27[1],ymm6[3],ymm27[3]
-; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm6[2,3],ymm0[2,3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm10, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm15, %zmm2, %zmm5
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm21 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm21, %zmm6
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm13, %zmm2, %zmm6
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm6 {%k1}
+; AVX512F-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
+; AVX512F-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm5 # 32-byte Folded Reload
+; AVX512F-NEXT:    # ymm5 = ymm5[1],mem[1],ymm5[3],mem[3]
+; AVX512F-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
+; AVX512F-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm7 # 32-byte Folded Reload
+; AVX512F-NEXT:    # ymm7 = ymm7[1],mem[1],ymm7[3],mem[3]
+; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm5 = ymm7[2,3],ymm5[2,3]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm5, %zmm6, %zmm5
+; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm5 # 64-byte Folded Reload
+; AVX512F-NEXT:    vmovdqu64 (%rsp), %zmm6 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm6 # 64-byte Folded Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm6 {%k1}
+; AVX512F-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
+; AVX512F-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm5 # 32-byte Folded Reload
+; AVX512F-NEXT:    # ymm5 = ymm5[1],mem[1],ymm5[3],mem[3]
+; AVX512F-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
+; AVX512F-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm7 # 32-byte Folded Reload
+; AVX512F-NEXT:    # ymm7 = ymm7[1],mem[1],ymm7[3],mem[3]
+; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm5 = ymm7[2,3],ymm5[2,3]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm5, %zmm6, %zmm5
+; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm29 # 64-byte Folded Reload
+; AVX512F-NEXT:    vpermt2q %zmm24, %zmm2, %zmm25
+; AVX512F-NEXT:    vmovdqa64 %zmm29, %zmm25 {%k1}
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm5 = ymm26[1],ymm27[1],ymm26[3],ymm27[3]
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm7 = ymm18[1],ymm30[1],ymm18[3],ymm30[3]
+; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm5 = ymm7[2,3],ymm5[2,3]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm5, %zmm25, %zmm5
+; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm24 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm24, %zmm2, %zmm31
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm26 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm26, %zmm6
+; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm6 # 64-byte Folded Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm31, %zmm6 {%k1}
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm5 = ymm11[1],ymm12[1],ymm11[3],ymm12[3]
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm3 = ymm3[1],ymm10[1],ymm3[3],ymm10[3]
+; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm3 = ymm3[2,3],ymm5[2,3]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm3, %zmm6, %zmm3
+; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm29 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm29, %zmm3
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm18, %zmm2, %zmm0
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
-; AVX512F-NEXT:    vpermi2q %zmm5, %zmm19, %zmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm28
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k1}
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm0 = ymm9[1],ymm12[1],ymm9[3],ymm12[3]
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm1[1],ymm4[1],ymm1[3],ymm4[3]
-; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm1[2,3],ymm0[2,3]
+; AVX512F-NEXT:    vpermt2q %zmm18, %zmm2, %zmm3
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512F-NEXT:    vpermi2q %zmm11, %zmm12, %zmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm2 {%k1}
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm3 = ymm23[1],ymm17[1],ymm23[3],ymm17[3]
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm0 = ymm0[1],ymm1[1],ymm0[3],ymm1[3]
+; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm0[2,3],ymm3[2,3]
 ; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm2, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 3264(%rdi), %zmm3
+; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 3200(%rdi), %zmm10
 ; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [4,12,4,12,4,12,4,12]
 ; AVX512F-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm1
 ; AVX512F-NEXT:    vpermt2q %zmm3, %zmm0, %zmm1
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm27 # 64-byte Reload
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm2 = zmm27[0],zmm12[0],zmm27[2],zmm12[2],zmm27[4],zmm12[4],zmm27[6],zmm12[6]
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}
-; AVX512F-NEXT:    vmovdqa64 3136(%rdi), %zmm1
-; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 3072(%rdi), %zmm3
-; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm0, %zmm3
-; AVX512F-NEXT:    vmovdqa64 3264(%rdi), %zmm5
-; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 3200(%rdi), %zmm4
-; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [4,12,4,12]
-; AVX512F-NEXT:    # ymm1 = mem[0,1,0,1]
-; AVX512F-NEXT:    vpermt2q %zmm5, %zmm1, %zmm4
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm4[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm3, %zmm2, %zmm2
+; AVX512F-NEXT:    vmovdqa64 3136(%rdi), %zmm23
+; AVX512F-NEXT:    vmovdqa64 3072(%rdi), %zmm2
 ; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm6
-; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm2
-; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 # 64-byte Folded Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm3 = zmm7[0],zmm13[0],zmm7[2],zmm13[2],zmm7[4],zmm13[4],zmm7[6],zmm13[6]
+; AVX512F-NEXT:    vpermt2q %zmm23, %zmm0, %zmm2
+; AVX512F-NEXT:    vmovdqu64 %zmm23, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm17, %zmm2
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm25 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm25, %zmm0, %zmm2
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm3 = zmm8[0],zmm16[0],zmm8[2],zmm16[2],zmm8[4],zmm16[4],zmm8[6],zmm16[6]
 ; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}
-; AVX512F-NEXT:    vmovdqa64 576(%rdi), %zmm5
-; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 512(%rdi), %zmm2
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm3, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 704(%rdi), %zmm2
 ; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm5, %zmm0, %zmm2
-; AVX512F-NEXT:    vmovdqa64 704(%rdi), %zmm5
-; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 640(%rdi), %zmm4
-; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm5, %zmm1, %zmm4
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm4[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm3, %zmm2
+; AVX512F-NEXT:    vmovdqa64 640(%rdi), %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm0, %zmm1
+; AVX512F-NEXT:    vmovdqa64 576(%rdi), %zmm3
+; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 512(%rdi), %zmm2
 ; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm23, %zmm0, %zmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm23, %zmm7
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm3 = zmm29[0],zmm26[0],zmm29[2],zmm26[2],zmm29[4],zmm26[4],zmm29[6],zmm26[6]
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm0, %zmm2
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 %zmm28, %zmm2
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm30 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm30, %zmm0, %zmm2
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm3 = zmm9[0],zmm4[0],zmm9[2],zmm4[2],zmm9[4],zmm4[4],zmm9[6],zmm4[6]
 ; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}
-; AVX512F-NEXT:    vmovdqa64 64(%rdi), %zmm4
-; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 (%rdi), %zmm2
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm3, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 192(%rdi), %zmm2
 ; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm4, %zmm0, %zmm2
-; AVX512F-NEXT:    vmovdqa64 192(%rdi), %zmm23
-; AVX512F-NEXT:    vmovdqa64 128(%rdi), %zmm5
-; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm4
-; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm23, %zmm1, %zmm4
-; AVX512F-NEXT:    vmovdqa64 %zmm23, %zmm26
-; AVX512F-NEXT:    vmovdqu64 %zmm23, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm4[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm3, %zmm2
+; AVX512F-NEXT:    vmovdqa64 128(%rdi), %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm0, %zmm1
+; AVX512F-NEXT:    vmovdqa64 64(%rdi), %zmm3
+; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 (%rdi), %zmm2
 ; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm30, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm20, %zmm0, %zmm2
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm3 = zmm8[0],zmm16[0],zmm8[2],zmm16[2],zmm8[4],zmm16[4],zmm8[6],zmm16[6]
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm0, %zmm2
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 %zmm20, %zmm2
+; AVX512F-NEXT:    vpermt2q %zmm19, %zmm0, %zmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm19, %zmm16
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm3 = zmm3[0],zmm22[0],zmm3[2],zmm22[2],zmm3[4],zmm22[4],zmm3[6],zmm22[6]
 ; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}
-; AVX512F-NEXT:    vmovdqa64 1600(%rdi), %zmm30
-; AVX512F-NEXT:    vmovdqa64 1536(%rdi), %zmm2
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm3, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 1728(%rdi), %zmm2
 ; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm30, %zmm0, %zmm2
-; AVX512F-NEXT:    vmovdqa64 1728(%rdi), %zmm20
-; AVX512F-NEXT:    vmovdqa64 1664(%rdi), %zmm4
-; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm20, %zmm1, %zmm4
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm4[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm3, %zmm2
+; AVX512F-NEXT:    vmovdqa64 1664(%rdi), %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm0, %zmm1
+; AVX512F-NEXT:    vmovdqa64 1600(%rdi), %zmm3
+; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 1536(%rdi), %zmm2
 ; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm21, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm17, %zmm0, %zmm2
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm3 = zmm9[0],zmm11[0],zmm9[2],zmm11[2],zmm9[4],zmm11[4],zmm9[6],zmm11[6]
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm0, %zmm2
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 %zmm14, %zmm2
+; AVX512F-NEXT:    vpermt2q %zmm15, %zmm0, %zmm2
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm3 = zmm21[0],zmm13[0],zmm21[2],zmm13[2],zmm21[4],zmm13[4],zmm21[6],zmm13[6]
 ; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}
-; AVX512F-NEXT:    vmovdqa64 1088(%rdi), %zmm4
-; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 1024(%rdi), %zmm2
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm3, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 1216(%rdi), %zmm2
 ; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm4, %zmm0, %zmm2
-; AVX512F-NEXT:    vmovdqa64 1216(%rdi), %zmm16
-; AVX512F-NEXT:    vmovdqa64 1152(%rdi), %zmm4
-; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm16, %zmm1, %zmm4
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm4[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm3, %zmm2
+; AVX512F-NEXT:    vmovdqa64 1152(%rdi), %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm0, %zmm1
+; AVX512F-NEXT:    vmovdqa64 1088(%rdi), %zmm3
+; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 1024(%rdi), %zmm2
 ; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm22, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm25, %zmm0, %zmm2
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm3 = zmm24[0],zmm15[0],zmm24[2],zmm15[2],zmm24[4],zmm15[4],zmm24[6],zmm15[6]
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm0, %zmm2
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm31 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm31, %zmm2
+; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 # 64-byte Folded Reload
+; AVX512F-NEXT:    vmovdqu64 (%rsp), %zmm27 # 64-byte Reload
+; AVX512F-NEXT:    vpunpcklqdq {{[-0-9]+}}(%r{{[sb]}}p), %zmm27, %zmm3 # 64-byte Folded Reload
+; AVX512F-NEXT:    # zmm3 = zmm27[0],mem[0],zmm27[2],mem[2],zmm27[4],mem[4],zmm27[6],mem[6]
 ; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}
-; AVX512F-NEXT:    vmovdqa64 2624(%rdi), %zmm4
-; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm3, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 2752(%rdi), %zmm3
+; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 2688(%rdi), %zmm8
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm0, %zmm1
+; AVX512F-NEXT:    vmovdqa64 2624(%rdi), %zmm3
+; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vmovdqa64 2560(%rdi), %zmm2
 ; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm4, %zmm0, %zmm2
-; AVX512F-NEXT:    vmovdqa64 2752(%rdi), %zmm14
-; AVX512F-NEXT:    vmovdqa64 2688(%rdi), %zmm4
-; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm14, %zmm1, %zmm4
-; AVX512F-NEXT:    vmovdqa64 %zmm14, %zmm25
-; AVX512F-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm4[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm3, %zmm2
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm0, %zmm2
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm2
+; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 # 64-byte Folded Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512F-NEXT:    vpunpcklqdq {{[-0-9]+}}(%r{{[sb]}}p), %zmm7, %zmm3 # 64-byte Folded Reload
+; AVX512F-NEXT:    # zmm3 = zmm7[0],mem[0],zmm7[2],mem[2],zmm7[4],mem[4],zmm7[6],mem[6]
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm3, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 2240(%rdi), %zmm2
 ; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 2176(%rdi), %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm0, %zmm1
 ; AVX512F-NEXT:    vmovdqa64 2112(%rdi), %zmm3
 ; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vmovdqa64 2048(%rdi), %zmm2
 ; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vpermt2q %zmm3, %zmm0, %zmm2
-; AVX512F-NEXT:    vmovdqa64 2240(%rdi), %zmm4
-; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 2176(%rdi), %zmm3
-; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm4, %zmm1, %zmm3
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5,6,7]
-; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm31 # 64-byte Folded Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 (%rsp), %zmm4 # 64-byte Reload
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm4 = zmm4[0],zmm8[0],zmm4[2],zmm8[2],zmm4[4],zmm8[4],zmm4[6],zmm8[6]
-; AVX512F-NEXT:    vmovdqa64 %zmm31, %zmm4 {%k1}
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm4, %zmm2
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm2
+; AVX512F-NEXT:    vpermt2q %zmm24, %zmm0, %zmm2
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm3 = zmm26[0],zmm4[0],zmm26[2],zmm4[2],zmm26[4],zmm4[4],zmm26[6],zmm4[6]
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm3, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 3776(%rdi), %zmm2
 ; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm18, %zmm0, %zmm10
-; AVX512F-NEXT:    vmovdqa64 3648(%rdi), %zmm4
-; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 3584(%rdi), %zmm3
-; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermi2q %zmm4, %zmm3, %zmm0
-; AVX512F-NEXT:    vmovdqa64 3776(%rdi), %zmm4
-; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 3712(%rdi), %zmm3
+; AVX512F-NEXT:    vmovdqa64 3712(%rdi), %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm0, %zmm1
+; AVX512F-NEXT:    vmovdqa64 3648(%rdi), %zmm3
 ; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermi2q %zmm4, %zmm3, %zmm1
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; AVX512F-NEXT:    vmovdqa64 %zmm28, %zmm18
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm1 = zmm19[0],zmm28[0],zmm19[2],zmm28[2],zmm19[4],zmm28[4],zmm19[6],zmm28[6]
-; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm1 {%k1}
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm1, %zmm0
+; AVX512F-NEXT:    vmovdqa64 3584(%rdi), %zmm2
+; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm0, %zmm2
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    vpermi2q %zmm18, %zmm29, %zmm0
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm2 = zmm12[0],zmm11[0],zmm12[2],zmm11[2],zmm12[4],zmm11[4],zmm12[6],zmm11[6]
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k1}
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm2, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [5,13,5,13,5,13,5,13]
 ; AVX512F-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm26 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm26, %zmm0, %zmm10
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 # 64-byte Folded Reload
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [5,13,5,13]
-; AVX512F-NEXT:    # ymm1 = mem[0,1,0,1]
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm3 # 64-byte Folded Reload
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5,6,7]
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm3 # 64-byte Folded Reload
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm4 = zmm27[1],zmm12[1],zmm27[3],zmm12[3],zmm27[5],zmm12[5],zmm27[7],zmm12[7]
-; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm4 {%k1}
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm4, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm23, %zmm0, %zmm2
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm10[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 %zmm17, %zmm2
+; AVX512F-NEXT:    vpermt2q %zmm25, %zmm0, %zmm2
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm25 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm3 = zmm19[1],zmm25[1],zmm19[3],zmm25[3],zmm19[5],zmm25[5],zmm19[7],zmm25[7]
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm3, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm22 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm22, %zmm0, %zmm1
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm24 # 64-byte Reload
 ; AVX512F-NEXT:    vpermt2q %zmm24, %zmm0, %zmm2
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 %zmm28, %zmm29
+; AVX512F-NEXT:    vmovdqa64 %zmm28, %zmm2
+; AVX512F-NEXT:    vpermt2q %zmm30, %zmm0, %zmm2
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm28 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm28, %zmm1, %zmm3
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5,6,7]
-; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm3
-; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm3 # 64-byte Folded Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm23
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm31 # 64-byte Reload
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm4 = zmm31[1],zmm13[1],zmm31[3],zmm13[3],zmm31[5],zmm13[5],zmm31[7],zmm13[7]
-; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm4 {%k1}
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm4, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm3 = zmm15[1],zmm28[1],zmm15[3],zmm28[3],zmm15[5],zmm28[5],zmm15[7],zmm28[7]
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm3, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm20 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm20, %zmm0, %zmm1
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm22 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm22, %zmm0, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm26, %zmm1, %zmm5
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm5[4,5,6,7]
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm7, %zmm0, %zmm3
-; AVX512F-NEXT:    vmovdqa64 %zmm29, %zmm14
-; AVX512F-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %zmm29, %zmm4 # 64-byte Folded Reload
-; AVX512F-NEXT:    # zmm4 = zmm29[1],mem[1],zmm29[3],mem[3],zmm29[5],mem[5],zmm29[7],mem[7]
-; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm4 {%k1}
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm4, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm21 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm21, %zmm0, %zmm2
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 %zmm30, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm30, %zmm0, %zmm2
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 %zmm20, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm20, %zmm1, %zmm3
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5,6,7]
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm3 # 64-byte Folded Reload
+; AVX512F-NEXT:    vpermt2q %zmm16, %zmm0, %zmm2
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm3 = zmm12[1],zmm18[1],zmm12[3],zmm18[3],zmm12[5],zmm18[5],zmm12[7],zmm18[7]
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm3, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
-; AVX512F-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %zmm17, %zmm4 # 64-byte Folded Reload
-; AVX512F-NEXT:    # zmm4 = zmm17[1],mem[1],zmm17[3],mem[3],zmm17[5],mem[5],zmm17[7],mem[7]
-; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm4 {%k1}
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm4, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm17, %zmm0, %zmm1
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm19, %zmm0, %zmm2
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm3
-; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm29
-; AVX512F-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm16, %zmm1, %zmm3
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5,6,7]
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm16, %zmm0, %zmm2
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 %zmm14, %zmm2
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm23 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm23, %zmm0, %zmm2
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm3 # 64-byte Folded Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm16
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm4 = zmm9[1],zmm11[1],zmm9[3],zmm11[3],zmm9[5],zmm11[5],zmm9[7],zmm11[7]
-; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm4 {%k1}
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm4, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm2
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm12, %zmm0, %zmm2
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm3 = zmm3[1],zmm13[1],zmm3[3],zmm13[3],zmm3[5],zmm13[5],zmm3[7],zmm13[7]
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm3, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm3
-; AVX512F-NEXT:    vpermt2q %zmm25, %zmm1, %zmm3
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5,6,7]
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm3 # 64-byte Folded Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm4 # 64-byte Folded Reload
-; AVX512F-NEXT:    # zmm4 = zmm4[1],mem[1],zmm4[3],mem[3],zmm4[5],mem[5],zmm4[7],mem[7]
-; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm4 {%k1}
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm4, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm13, %zmm0, %zmm1
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm7, %zmm0, %zmm2
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm3
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm10, %zmm1, %zmm3
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5,6,7]
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm26 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm26, %zmm3
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm21 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm21, %zmm0, %zmm3
-; AVX512F-NEXT:    vmovdqu64 (%rsp), %zmm4 # 64-byte Reload
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm4 = zmm4[1],zmm8[1],zmm4[3],zmm8[3],zmm4[5],zmm8[5],zmm4[7],zmm8[7]
-; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm4 {%k1}
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm4, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm14, %zmm0, %zmm2
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm31 # 64-byte Folded Reload
+; AVX512F-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %zmm27, %zmm3 # 64-byte Folded Reload
+; AVX512F-NEXT:    # zmm3 = zmm27[1],mem[1],zmm27[3],mem[3],zmm27[5],mem[5],zmm27[7],mem[7]
+; AVX512F-NEXT:    vmovdqa64 %zmm31, %zmm3 {%k1}
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm3, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm31 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm31, %zmm0, %zmm8
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 # 64-byte Folded Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm11, %zmm0, %zmm2
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm8[4,5,6,7]
+; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm9 # 64-byte Folded Reload
+; AVX512F-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %zmm7, %zmm3 # 64-byte Folded Reload
+; AVX512F-NEXT:    # zmm3 = zmm7[1],mem[1],zmm7[3],mem[3],zmm7[5],mem[5],zmm7[7],mem[7]
+; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm3 {%k1}
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm3, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm27 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm27, %zmm0, %zmm1
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm2
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm8, %zmm0, %zmm2
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm5 # 64-byte Folded Reload
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-NEXT:    vpermi2q %zmm6, %zmm3, %zmm0
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm3 = zmm3[1],zmm4[1],zmm3[3],zmm4[3],zmm3[5],zmm4[5],zmm3[7],zmm4[7]
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm3 {%k1}
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm3, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm1
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm5, %zmm0, %zmm1
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-NEXT:    vpermi2q %zmm4, %zmm5, %zmm1
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm25 # 64-byte Reload
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm1 = zmm25[1],zmm18[1],zmm25[3],zmm18[3],zmm25[5],zmm18[5],zmm25[7],zmm18[7]
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm1 {%k1}
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm1, %zmm0
+; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm2
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm6, %zmm0, %zmm2
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
+; AVX512F-NEXT:    vpermi2q %zmm10, %zmm9, %zmm0
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm2 # 64-byte Folded Reload
+; AVX512F-NEXT:    # zmm2 = zmm2[1],mem[1],zmm2[3],mem[3],zmm2[5],mem[5],zmm2[7],mem[7]
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k1}
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm2, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [6,14,6,14,6,14,6,14]
 ; AVX512F-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm26, %zmm0, %zmm1
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
 ; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 # 64-byte Folded Reload
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [6,14,6,14]
-; AVX512F-NEXT:    # ymm1 = mem[0,1,0,1]
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm3 # 64-byte Folded Reload
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5,6,7]
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm3 # 64-byte Folded Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm27 # 64-byte Reload
-; AVX512F-NEXT:    vpunpcklqdq {{[-0-9]+}}(%r{{[sb]}}p), %zmm27, %zmm3 {%k1} # 64-byte Folded Reload
-; AVX512F-NEXT:    # zmm3 {%k1} = zmm27[0],mem[0],zmm27[2],mem[2],zmm27[4],mem[4],zmm27[6],mem[6]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm3, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 %zmm19, %zmm2
+; AVX512F-NEXT:    vpermt2q %zmm25, %zmm0, %zmm2
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm25 # 64-byte Reload
+; AVX512F-NEXT:    vpunpcklqdq {{[-0-9]+}}(%r{{[sb]}}p), %zmm25, %zmm2 {%k1} # 64-byte Folded Reload
+; AVX512F-NEXT:    # zmm2 {%k1} = zmm25[0],mem[0],zmm25[2],mem[2],zmm25[4],mem[4],zmm25[6],mem[6]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm2, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm22, %zmm0, %zmm1
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm24, %zmm0, %zmm2
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    vpermt2q %zmm28, %zmm0, %zmm15
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm15 {%k1} = zmm29[0],zmm30[0],zmm29[2],zmm30[2],zmm29[4],zmm30[4],zmm29[6],zmm30[6]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm15, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm20, %zmm0, %zmm1
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm21, %zmm0, %zmm2
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 %zmm12, %zmm2
+; AVX512F-NEXT:    vpermt2q %zmm18, %zmm0, %zmm2
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm30 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm26 # 64-byte Reload
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm2 {%k1} = zmm30[0],zmm26[0],zmm30[2],zmm26[2],zmm30[4],zmm26[4],zmm30[6],zmm26[6]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm2, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm17, %zmm0, %zmm1
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm24, %zmm0, %zmm2
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm28, %zmm1, %zmm3
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5,6,7]
-; AVX512F-NEXT:    vpermt2q %zmm23, %zmm0, %zmm31
+; AVX512F-NEXT:    vpermt2q %zmm16, %zmm0, %zmm2
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm2
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm29 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm29, %zmm0, %zmm2
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm24 # 64-byte Reload
-; AVX512F-NEXT:    vpunpcklqdq {{[-0-9]+}}(%r{{[sb]}}p), %zmm24, %zmm31 {%k1} # 64-byte Folded Reload
-; AVX512F-NEXT:    # zmm31 {%k1} = zmm24[0],mem[0],zmm24[2],mem[2],zmm24[4],mem[4],zmm24[6],mem[6]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm31, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm2 {%k1} = zmm24[0],zmm23[0],zmm24[2],zmm23[2],zmm24[4],zmm23[4],zmm24[6],zmm23[6]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm2, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm13, %zmm0, %zmm1
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm22, %zmm0, %zmm2
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm3 # 64-byte Folded Reload
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5,6,7]
-; AVX512F-NEXT:    vmovdqa64 %zmm14, %zmm3
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm22 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm22, %zmm0, %zmm3
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm28 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm3 {%k1} = zmm14[0],zmm28[0],zmm14[2],zmm28[2],zmm14[4],zmm28[4],zmm14[6],zmm28[6]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm3, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm14, %zmm0, %zmm2
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    vmovdqu64 (%rsp), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm21 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm21, %zmm0, %zmm2
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm23 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm2 {%k1} = zmm23[0],zmm18[0],zmm23[2],zmm18[2],zmm23[4],zmm18[4],zmm23[6],zmm18[6]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm2, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm31, %zmm0, %zmm1
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm30, %zmm0, %zmm2
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm20, %zmm1, %zmm3
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5,6,7]
-; AVX512F-NEXT:    vmovdqa64 %zmm17, %zmm3
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm17, %zmm0, %zmm3
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm30 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm27 # 64-byte Reload
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm3 {%k1} = zmm30[0],zmm27[0],zmm30[2],zmm27[2],zmm30[4],zmm27[4],zmm30[6],zmm27[6]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm3, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm11, %zmm0, %zmm2
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm19, %zmm0, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm29, %zmm1, %zmm15
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm15[4,5,6,7]
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm20 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm20, %zmm3
-; AVX512F-NEXT:    vpermt2q %zmm16, %zmm0, %zmm3
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm31 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm3 {%k1} = zmm31[0],zmm16[0],zmm31[2],zmm16[2],zmm31[4],zmm16[4],zmm31[6],zmm16[6]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm3, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm12, %zmm0, %zmm11
-; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm13 # 64-byte Folded Reload
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm11[0,1,2,3],ymm13[4,5,6,7]
+; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 # 64-byte Folded Reload
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm3
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm13, %zmm0, %zmm3
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512F-NEXT:    vpunpcklqdq {{[-0-9]+}}(%r{{[sb]}}p), %zmm12, %zmm3 {%k1} # 64-byte Folded Reload
-; AVX512F-NEXT:    # zmm3 {%k1} = zmm12[0],mem[0],zmm12[2],mem[2],zmm12[4],mem[4],zmm12[6],mem[6]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm3, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm7, %zmm0, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm10, %zmm1, %zmm9
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm9[4,5,6,7]
-; AVX512F-NEXT:    vmovdqu64 (%rsp), %zmm10 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm3
-; AVX512F-NEXT:    vpermt2q %zmm8, %zmm0, %zmm3
-; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm7
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm3 {%k1} = zmm26[0],zmm21[0],zmm26[2],zmm21[2],zmm26[4],zmm21[4],zmm26[6],zmm21[6]
-; AVX512F-NEXT:    vmovdqa64 %zmm21, %zmm8
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm3, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm25, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm18, %zmm0, %zmm2
+; AVX512F-NEXT:    vpunpcklqdq {{[-0-9]+}}(%r{{[sb]}}p), %zmm11, %zmm2 {%k1} # 64-byte Folded Reload
+; AVX512F-NEXT:    # zmm2 {%k1} = zmm11[0],mem[0],zmm11[2],mem[2],zmm11[4],mem[4],zmm11[6],mem[6]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm2, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm27, %zmm0, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm8, %zmm0, %zmm7
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm7[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm22 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm22, %zmm2
+; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 # 64-byte Folded Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm28 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm2 {%k1} = zmm7[0],zmm28[0],zmm7[2],zmm28[2],zmm7[4],zmm28[4],zmm7[6],zmm28[6]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm2, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm5, %zmm0, %zmm3
+; AVX512F-NEXT:    vpermt2q %zmm6, %zmm0, %zmm4
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3],ymm3[4,5,6,7]
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm25 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
+; AVX512F-NEXT:    vpermi2q %zmm25, %zmm15, %zmm0
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm0 {%k1} = zmm9[0],zmm10[0],zmm9[2],zmm10[2],zmm9[4],zmm10[4],zmm9[6],zmm10[6]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm19, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm19, %zmm0
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm5 = [7,15,7,15,7,15,7,15]
+; AVX512F-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-NEXT:    vpermi2q %zmm6, %zmm3, %zmm0
-; AVX512F-NEXT:    vpermi2q %zmm4, %zmm5, %zmm1
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm2 {%k1} = zmm9[0],zmm5[0],zmm9[2],zmm5[2],zmm9[4],zmm5[4],zmm9[6],zmm5[6]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm2, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm5, %zmm19
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm19 {%k1} = zmm17[1],zmm2[1],zmm17[3],zmm2[3],zmm17[5],zmm2[5],zmm17[7],zmm2[7]
+; AVX512F-NEXT:    vmovdqu64 %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm17, %zmm6
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm12 = [0,8,0,8,0,8,0,8]
+; AVX512F-NEXT:    # zmm12 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm12, %zmm17
+; AVX512F-NEXT:    vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm4
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm2 = [1,9,1,9,1,9,1,9]
+; AVX512F-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpermt2q %zmm4, %zmm2, %zmm6
+; AVX512F-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm12, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm2, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm6
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm20
 ; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm0
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm4 = [7,15,7,15,7,15,7,15]
-; AVX512F-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm23, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm23, %zmm4, %zmm1
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm1 {%k1} = zmm24[1],zmm3[1],zmm24[3],zmm3[3],zmm24[5],zmm3[5],zmm24[7],zmm3[7]
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm4, %zmm5, %zmm1
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm1 {%k1} = zmm13[1],zmm11[1],zmm13[3],zmm11[3],zmm13[5],zmm11[5],zmm13[7],zmm11[7]
 ; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm24, %zmm1
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm14 = [0,8,0,8,0,8,0,8]
-; AVX512F-NEXT:    # zmm14 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermt2q %zmm3, %zmm14, %zmm24
-; AVX512F-NEXT:    vmovdqu64 %zmm24, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm23 = [1,9,1,9,1,9,1,9]
-; AVX512F-NEXT:    # zmm23 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermt2q %zmm3, %zmm23, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm11, %zmm12, %zmm13
+; AVX512F-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm11, %zmm2, %zmm1
 ; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm2, %zmm14, %zmm6
-; AVX512F-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm2, %zmm23, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm4, %zmm12, %zmm20
+; AVX512F-NEXT:    vpermt2q %zmm4, %zmm2, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm24
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm1
-; AVX512F-NEXT:    vpermt2q %zmm22, %zmm4, %zmm2
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm2 {%k1} = zmm15[1],zmm28[1],zmm15[3],zmm28[3],zmm15[5],zmm28[5],zmm15[7],zmm28[7]
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm6
-; AVX512F-NEXT:    vpermt2q %zmm28, %zmm14, %zmm15
-; AVX512F-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm28, %zmm23, %zmm6
-; AVX512F-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm22, %zmm14, %zmm24
-; AVX512F-NEXT:    vpermt2q %zmm22, %zmm23, %zmm1
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm17
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm0
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm9, %zmm5, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm26, %zmm10
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm1 {%k1} = zmm30[1],zmm26[1],zmm30[3],zmm26[3],zmm30[5],zmm26[5],zmm30[7],zmm26[7]
 ; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm19, %zmm29
-; AVX512F-NEXT:    vmovdqa64 %zmm19, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm17, %zmm4, %zmm19
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm19 {%k1} = zmm30[1],zmm27[1],zmm30[3],zmm27[3],zmm30[5],zmm27[5],zmm30[7],zmm27[7]
 ; AVX512F-NEXT:    vmovdqa64 %zmm30, %zmm1
-; AVX512F-NEXT:    vpermt2q %zmm27, %zmm14, %zmm30
-; AVX512F-NEXT:    vmovdqu64 %zmm30, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm27, %zmm23, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm30, %zmm26
+; AVX512F-NEXT:    vpermt2q %zmm10, %zmm12, %zmm26
+; AVX512F-NEXT:    vpermt2q %zmm10, %zmm2, %zmm1
 ; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm17, %zmm14, %zmm29
-; AVX512F-NEXT:    vpermt2q %zmm17, %zmm23, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm9, %zmm12, %zmm17
+; AVX512F-NEXT:    vpermt2q %zmm9, %zmm2, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm20, %zmm28
-; AVX512F-NEXT:    vmovdqa64 %zmm20, %zmm3
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm4, %zmm20
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm20 {%k1} = zmm31[1],zmm16[1],zmm31[3],zmm16[3],zmm31[5],zmm16[5],zmm31[7],zmm16[7]
-; AVX512F-NEXT:    vmovdqu64 %zmm20, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm31, %zmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm31, %zmm17
-; AVX512F-NEXT:    vpermt2q %zmm16, %zmm14, %zmm17
-; AVX512F-NEXT:    vpermt2q %zmm16, %zmm23, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm14, %zmm28
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm23, %zmm3
-; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm21
-; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm31
-; AVX512F-NEXT:    vpermt2q %zmm13, %zmm4, %zmm11
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm11 {%k1} = zmm3[1],zmm0[1],zmm3[3],zmm0[3],zmm3[5],zmm0[5],zmm3[7],zmm0[7]
-; AVX512F-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm0, %zmm14, %zmm3
-; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm0, %zmm23, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm13, %zmm14, %zmm21
-; AVX512F-NEXT:    vpermt2q %zmm13, %zmm23, %zmm31
-; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm20
-; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm25
-; AVX512F-NEXT:    vpermt2q %zmm7, %zmm4, %zmm2
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm2 {%k1} = zmm26[1],zmm8[1],zmm26[3],zmm8[3],zmm26[5],zmm8[5],zmm26[7],zmm8[7]
-; AVX512F-NEXT:    vmovdqu64 %zmm2, (%rsp) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm26, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm8, %zmm14, %zmm26
-; AVX512F-NEXT:    vpermt2q %zmm8, %zmm23, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm7, %zmm14, %zmm20
-; AVX512F-NEXT:    vpermt2q %zmm7, %zmm23, %zmm25
+; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm19
+; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm31
+; AVX512F-NEXT:    vmovdqa64 %zmm29, %zmm3
+; AVX512F-NEXT:    vpermt2q %zmm29, %zmm5, %zmm16
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm0
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm3, %zmm14, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm30 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm30, %zmm16
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm27 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm27, %zmm14, %zmm16
-; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm5, %zmm14, %zmm0
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm16 {%k1} = zmm24[1],zmm6[1],zmm24[3],zmm6[3],zmm24[5],zmm6[5],zmm24[7],zmm6[7]
+; AVX512F-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm24, %zmm0
+; AVX512F-NEXT:    vmovdqa64 %zmm24, %zmm29
+; AVX512F-NEXT:    vpermt2q %zmm6, %zmm12, %zmm29
+; AVX512F-NEXT:    vpermt2q %zmm6, %zmm2, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vpermi2q %zmm18, %zmm0, %zmm14
-; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm3, %zmm23, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm30, %zmm22
-; AVX512F-NEXT:    vpermt2q %zmm27, %zmm23, %zmm22
-; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm5, %zmm23, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermi2q %zmm18, %zmm0, %zmm23
-; AVX512F-NEXT:    vpermt2q %zmm18, %zmm4, %zmm0
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm0 {%k1} = zmm9[1],zmm5[1],zmm9[3],zmm5[3],zmm9[5],zmm5[5],zmm9[7],zmm5[7]
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm18
-; AVX512F-NEXT:    vpermt2q %zmm27, %zmm4, %zmm30
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm30 {%k1} = zmm6[1],zmm3[1],zmm6[3],zmm3[3],zmm6[5],zmm3[5],zmm6[7],zmm3[7]
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm7 # 64-byte Folded Reload
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm12, %zmm19
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm2, %zmm31
+; AVX512F-NEXT:    vmovdqu64 (%rsp), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm16
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm27
+; AVX512F-NEXT:    vpermt2q %zmm21, %zmm5, %zmm1
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm1 {%k1} = zmm23[1],zmm18[1],zmm23[3],zmm18[3],zmm23[5],zmm18[5],zmm23[7],zmm18[7]
+; AVX512F-NEXT:    vmovdqu64 %zmm1, (%rsp) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm23, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm23, %zmm24
+; AVX512F-NEXT:    vpermt2q %zmm18, %zmm12, %zmm24
+; AVX512F-NEXT:    vpermt2q %zmm18, %zmm2, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm21, %zmm12, %zmm16
+; AVX512F-NEXT:    vpermt2q %zmm21, %zmm2, %zmm27
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm18
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm30
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm8 # 64-byte Folded Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm10 # 64-byte Folded Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm11 # 64-byte Folded Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm12 # 64-byte Folded Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm13 # 64-byte Folded Reload
+; AVX512F-NEXT:    vpermt2q %zmm8, %zmm5, %zmm1
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm0 # 64-byte Folded Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm1 {%k1} = zmm10[1],zmm0[1],zmm10[3],zmm0[3],zmm10[5],zmm0[5],zmm10[7],zmm0[7]
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm12, %zmm10
+; AVX512F-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm2, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm8, %zmm12, %zmm18
+; AVX512F-NEXT:    vpermt2q %zmm8, %zmm2, %zmm30
+; AVX512F-NEXT:    vmovdqa64 %zmm22, %zmm6
+; AVX512F-NEXT:    vmovdqa64 %zmm22, %zmm23
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm8, %zmm5, %zmm6
+; AVX512F-NEXT:    vmovdqa64 %zmm28, %zmm0
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm21 # 64-byte Reload
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm6 {%k1} = zmm21[1],zmm28[1],zmm21[3],zmm28[3],zmm21[5],zmm28[5],zmm21[7],zmm28[7]
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm28
+; AVX512F-NEXT:    vmovdqa64 %zmm21, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm12, %zmm21
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm2, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm8, %zmm12, %zmm23
+; AVX512F-NEXT:    vpermt2q %zmm8, %zmm2, %zmm22
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm14, %zmm0
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm8, %zmm12, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm15 # 64-byte Folded Reload
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = [7,15,7,15]
-; AVX512F-NEXT:    # ymm0 = mem[0,1,0,1]
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm9 # 64-byte Folded Reload
+; AVX512F-NEXT:    vpermi2q %zmm25, %zmm15, %zmm12
+; AVX512F-NEXT:    vmovdqa64 %zmm14, %zmm3
+; AVX512F-NEXT:    vpermt2q %zmm8, %zmm2, %zmm3
+; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermi2q %zmm25, %zmm15, %zmm2
+; AVX512F-NEXT:    vpermt2q %zmm25, %zmm5, %zmm15
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm15 {%k1} = zmm14[1],zmm8[1],zmm14[3],zmm8[3],zmm14[5],zmm8[5],zmm14[7],zmm8[7]
+; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm25
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm7 # 64-byte Folded Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm0 # 64-byte Folded Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm8 # 64-byte Folded Reload
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm1 # 64-byte Folded Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 # 64-byte Folded Reload
+; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm1 # 64-byte Folded Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm13 # 64-byte Folded Reload
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm3 # 64-byte Folded Reload
+; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm3 # 64-byte Folded Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm9 # 64-byte Folded Reload
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm4 # 64-byte Folded Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm5 # 64-byte Folded Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm6 # 64-byte Folded Reload
-; AVX512F-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm4 # 64-byte Folded Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm11 # 64-byte Folded Reload
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm6 # 64-byte Folded Reload
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm7[0,1,2,3],ymm9[4,5,6,7]
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm7, %zmm0
+; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm6 # 64-byte Folded Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm15 # 64-byte Folded Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm10 # 64-byte Folded Reload
+; AVX512F-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm10 # 64-byte Folded Reload
+; AVX512F-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm10 # 64-byte Folded Reload
+; AVX512F-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm10 # 64-byte Folded Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm14 # 64-byte Folded Reload
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm7[4,5,6,7]
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm5, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm8[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm8[4,5,6,7]
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
 ; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm1, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm10[0,1,2,3],ymm2[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm19, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm11[0,1,2,3],ymm3[4,5,6,7]
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm1, %zmm11
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm12[0,1,2,3],ymm4[4,5,6,7]
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3],ymm13[4,5,6,7]
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
 ; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm1, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm13[0,1,2,3],ymm5[4,5,6,7]
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm9[4,5,6,7]
 ; AVX512F-NEXT:    vmovdqu64 (%rsp), %zmm1 # 64-byte Reload
 ; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm1, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, (%rsp) # 64-byte Spill
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm6[0,1,2,3],ymm11[4,5,6,7]
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm1, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpblendd $15, {{[-0-9]+}}(%r{{[sb]}}p), %ymm15, %ymm0 # 32-byte Folded Reload
+; AVX512F-NEXT:    # ymm0 = mem[0,1,2,3],ymm15[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm28, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
 ; AVX512F-NEXT:    # ymm0 = ymm0[0,1,2,3],mem[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm18, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm15[0,1,2,3],ymm6[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm30, %zmm0
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm25, %zmm25
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm14[0,1,2,3],ymm10[4,5,6,7]
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm1, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm24 {%k1}
-; AVX512F-NEXT:    vmovdqa 64(%rdi), %xmm0
-; AVX512F-NEXT:    vinserti128 $1, 192(%rdi), %ymm0, %ymm0
-; AVX512F-NEXT:    vmovdqa (%rdi), %xmm15
-; AVX512F-NEXT:    vinserti128 $1, 128(%rdi), %ymm15, %ymm15
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm30 = ymm15[0],ymm0[0],ymm15[2],ymm0[2]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm30, %zmm24, %zmm1
-; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm26, %zmm17 {%k1}
+; AVX512F-NEXT:    vmovdqa 64(%rdi), %xmm8
+; AVX512F-NEXT:    vinserti128 $1, 192(%rdi), %ymm8, %ymm8
+; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512F-NEXT:    vinserti128 $1, 128(%rdi), %ymm0, %ymm0
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm28 = ymm0[0],ymm8[0],ymm0[2],ymm8[2]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm28, %zmm17, %zmm26
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm1 {%k1}
-; AVX512F-NEXT:    vmovdqa64 576(%rdi), %xmm24
-; AVX512F-NEXT:    vinserti32x4 $1, 704(%rdi), %ymm24, %ymm30
-; AVX512F-NEXT:    vmovdqa64 512(%rdi), %xmm24
-; AVX512F-NEXT:    vinserti32x4 $1, 640(%rdi), %ymm24, %ymm24
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm27 = ymm24[0],ymm30[0],ymm24[2],ymm30[2]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm27, %zmm1, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm20 {%k1}
+; AVX512F-NEXT:    vmovdqa 576(%rdi), %xmm13
+; AVX512F-NEXT:    vinserti128 $1, 704(%rdi), %ymm13, %ymm13
+; AVX512F-NEXT:    vmovdqa64 512(%rdi), %xmm28
+; AVX512F-NEXT:    vinserti32x4 $1, 640(%rdi), %ymm28, %ymm28
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm4 = ymm28[0],ymm13[0],ymm28[2],ymm13[2]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm4, %zmm20, %zmm1
 ; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm17, %zmm28 {%k1}
-; AVX512F-NEXT:    vmovdqa 1088(%rdi), %xmm9
-; AVX512F-NEXT:    vinserti32x4 $1, 1216(%rdi), %ymm9, %ymm27
-; AVX512F-NEXT:    vmovdqa 1024(%rdi), %xmm9
-; AVX512F-NEXT:    vinserti128 $1, 1152(%rdi), %ymm9, %ymm9
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm4 = ymm9[0],ymm27[0],ymm9[2],ymm27[2]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm4, %zmm28, %zmm1
-; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm29 {%k1}
-; AVX512F-NEXT:    vmovdqa 1600(%rdi), %xmm4
-; AVX512F-NEXT:    vinserti128 $1, 1728(%rdi), %ymm4, %ymm4
-; AVX512F-NEXT:    vmovdqa64 1536(%rdi), %xmm17
-; AVX512F-NEXT:    vinserti32x4 $1, 1664(%rdi), %ymm17, %ymm17
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm19 = ymm17[0],ymm4[0],ymm17[2],ymm4[2]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm19, %zmm29, %zmm13
-; AVX512F-NEXT:    vmovdqa64 %zmm26, %zmm20 {%k1}
-; AVX512F-NEXT:    vmovdqa64 2112(%rdi), %xmm19
-; AVX512F-NEXT:    vinserti32x4 $1, 2240(%rdi), %ymm19, %ymm19
-; AVX512F-NEXT:    vmovdqa64 2048(%rdi), %xmm26
-; AVX512F-NEXT:    vinserti32x4 $1, 2176(%rdi), %ymm26, %ymm26
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm29 = ymm26[0],ymm19[0],ymm26[2],ymm19[2]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm29, %zmm20, %zmm5
+; AVX512F-NEXT:    vmovdqa64 %zmm24, %zmm16 {%k1}
+; AVX512F-NEXT:    vmovdqa 1088(%rdi), %xmm4
+; AVX512F-NEXT:    vinserti128 $1, 1216(%rdi), %ymm4, %ymm4
+; AVX512F-NEXT:    vmovdqa 1024(%rdi), %xmm7
+; AVX512F-NEXT:    vinserti128 $1, 1152(%rdi), %ymm7, %ymm7
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm5 = ymm7[0],ymm4[0],ymm7[2],ymm4[2]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm5, %zmm16, %zmm24
+; AVX512F-NEXT:    vmovdqa64 %zmm29, %zmm19 {%k1}
+; AVX512F-NEXT:    vmovdqa 1600(%rdi), %xmm5
+; AVX512F-NEXT:    vinserti128 $1, 1728(%rdi), %ymm5, %ymm5
+; AVX512F-NEXT:    vmovdqa 1536(%rdi), %xmm11
+; AVX512F-NEXT:    vinserti128 $1, 1664(%rdi), %ymm11, %ymm11
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm29 = ymm11[0],ymm5[0],ymm11[2],ymm5[2]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm29, %zmm19, %zmm19
+; AVX512F-NEXT:    vmovdqa64 %zmm21, %zmm23 {%k1}
+; AVX512F-NEXT:    vmovdqa 2112(%rdi), %xmm6
+; AVX512F-NEXT:    vinserti128 $1, 2240(%rdi), %ymm6, %ymm6
+; AVX512F-NEXT:    vmovdqa64 2048(%rdi), %xmm29
+; AVX512F-NEXT:    vinserti32x4 $1, 2176(%rdi), %ymm29, %ymm29
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm3 = ymm29[0],ymm6[0],ymm29[2],ymm6[2]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm3, %zmm23, %zmm3
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm21 {%k1}
-; AVX512F-NEXT:    vmovdqa 2624(%rdi), %xmm12
-; AVX512F-NEXT:    vinserti128 $1, 2752(%rdi), %ymm12, %ymm12
-; AVX512F-NEXT:    vmovdqa64 2560(%rdi), %xmm29
-; AVX512F-NEXT:    vinserti32x4 $1, 2688(%rdi), %ymm29, %ymm29
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm18 = ymm29[0],ymm12[0],ymm29[2],ymm12[2]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm18, %zmm21, %zmm18
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm18 {%k1}
+; AVX512F-NEXT:    vmovdqa 2624(%rdi), %xmm10
+; AVX512F-NEXT:    vinserti128 $1, 2752(%rdi), %ymm10, %ymm10
+; AVX512F-NEXT:    vmovdqa64 2560(%rdi), %xmm23
+; AVX512F-NEXT:    vinserti32x4 $1, 2688(%rdi), %ymm23, %ymm23
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm20 = ymm23[0],ymm10[0],ymm23[2],ymm10[2]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm20, %zmm18, %zmm18
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm16 {%k1}
-; AVX512F-NEXT:    vmovdqa 3136(%rdi), %xmm6
-; AVX512F-NEXT:    vinserti128 $1, 3264(%rdi), %ymm6, %ymm6
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm1 {%k1}
+; AVX512F-NEXT:    vmovdqa64 3136(%rdi), %xmm17
+; AVX512F-NEXT:    vinserti32x4 $1, 3264(%rdi), %ymm17, %ymm17
 ; AVX512F-NEXT:    vmovdqa64 3072(%rdi), %xmm20
 ; AVX512F-NEXT:    vinserti32x4 $1, 3200(%rdi), %ymm20, %ymm20
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm21 = ymm20[0],ymm6[0],ymm20[2],ymm6[2]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm21, %zmm16, %zmm2
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm15 = ymm20[0],ymm17[0],ymm20[2],ymm17[2]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm15, %zmm1, %zmm15
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm14 {%k1}
-; AVX512F-NEXT:    vmovdqa 3648(%rdi), %xmm7
-; AVX512F-NEXT:    vinserti128 $1, 3776(%rdi), %ymm7, %ymm7
-; AVX512F-NEXT:    vmovdqa64 3584(%rdi), %xmm21
-; AVX512F-NEXT:    vinserti32x4 $1, 3712(%rdi), %ymm21, %ymm21
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm28 = ymm21[0],ymm7[0],ymm21[2],ymm7[2]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm28, %zmm14, %zmm1
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm22 {%k1}
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm6 = ymm20[1],ymm6[1],ymm20[3],ymm6[3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm6, %zmm22, %zmm3
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm12 {%k1}
+; AVX512F-NEXT:    vmovdqa 3648(%rdi), %xmm9
+; AVX512F-NEXT:    vinserti128 $1, 3776(%rdi), %ymm9, %ymm9
+; AVX512F-NEXT:    vmovdqa 3584(%rdi), %xmm14
+; AVX512F-NEXT:    vinserti128 $1, 3712(%rdi), %ymm14, %ymm14
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm16 = ymm14[0],ymm9[0],ymm14[2],ymm9[2]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm16, %zmm12, %zmm1
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm12 {%k1}
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm16 = ymm20[1],ymm17[1],ymm20[3],ymm17[3]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm16, %zmm12, %zmm16
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm17, %zmm12 {%k1}
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm13 = ymm28[1],ymm13[1],ymm28[3],ymm13[3]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm13, %zmm12, %zmm13
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm17, %zmm12 {%k1}
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm0 = ymm0[1],ymm8[1],ymm0[3],ymm8[3]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm12, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm8 {%k1}
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm6 = ymm24[1],ymm30[1],ymm24[3],ymm30[3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm6, %zmm8, %zmm6
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm31 {%k1}
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm5 = ymm11[1],ymm5[1],ymm11[3],ymm5[3]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm5, %zmm31, %zmm5
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm8 {%k1}
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm0 = ymm15[1],ymm0[1],ymm15[3],ymm0[3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm8, %zmm0
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm27 {%k1}
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm4 = ymm7[1],ymm4[1],ymm7[3],ymm4[3]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm4, %zmm27, %zmm4
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm30 {%k1}
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm7 = ymm23[1],ymm10[1],ymm23[3],ymm10[3]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm7, %zmm30, %zmm7
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm8 {%k1}
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm4 = ymm17[1],ymm4[1],ymm17[3],ymm4[3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm4, %zmm8, %zmm4
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm22 {%k1}
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm6 = ymm29[1],ymm6[1],ymm29[3],ymm6[3]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm6, %zmm22, %zmm6
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm10 {%k1}
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm8 = ymm9[1],ymm27[1],ymm9[3],ymm27[3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm8, %zmm10, %zmm8
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm31 {%k1}
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm9 = ymm29[1],ymm12[1],ymm29[3],ymm12[3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm9, %zmm31, %zmm9
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm25 {%k1}
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm10 = ymm26[1],ymm19[1],ymm26[3],ymm19[3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm10, %zmm25, %zmm10
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm12, %zmm23 {%k1}
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm7 = ymm21[1],ymm7[1],ymm21[3],ymm7[3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm7, %zmm23, %zmm7
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm2 {%k1}
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm8 = ymm14[1],ymm9[1],ymm14[3],ymm9[3]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm8, %zmm2, %zmm2
 ; AVX512F-NEXT:    vmovdqa64 %zmm1, 448(%rsi)
-; AVX512F-NEXT:    vmovdqa64 %zmm2, 384(%rsi)
+; AVX512F-NEXT:    vmovdqa64 %zmm15, 384(%rsi)
 ; AVX512F-NEXT:    vmovdqa64 %zmm18, 320(%rsi)
-; AVX512F-NEXT:    vmovdqa64 %zmm5, 256(%rsi)
-; AVX512F-NEXT:    vmovdqa64 %zmm13, 192(%rsi)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm1, 128(%rsi)
+; AVX512F-NEXT:    vmovdqa64 %zmm3, 256(%rsi)
+; AVX512F-NEXT:    vmovdqa64 %zmm19, 192(%rsi)
+; AVX512F-NEXT:    vmovdqa64 %zmm24, 128(%rsi)
 ; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
 ; AVX512F-NEXT:    vmovaps %zmm1, 64(%rsi)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm1, (%rsi)
-; AVX512F-NEXT:    vmovdqa64 %zmm7, 448(%rdx)
-; AVX512F-NEXT:    vmovdqa64 %zmm10, 256(%rdx)
-; AVX512F-NEXT:    vmovdqa64 %zmm9, 320(%rdx)
-; AVX512F-NEXT:    vmovdqa64 %zmm8, 128(%rdx)
-; AVX512F-NEXT:    vmovdqa64 %zmm4, 192(%rdx)
+; AVX512F-NEXT:    vmovdqa64 %zmm26, (%rsi)
+; AVX512F-NEXT:    vmovdqa64 %zmm2, 448(%rdx)
+; AVX512F-NEXT:    vmovdqa64 %zmm6, 256(%rdx)
+; AVX512F-NEXT:    vmovdqa64 %zmm7, 320(%rdx)
+; AVX512F-NEXT:    vmovdqa64 %zmm4, 128(%rdx)
+; AVX512F-NEXT:    vmovdqa64 %zmm5, 192(%rdx)
 ; AVX512F-NEXT:    vmovdqa64 %zmm0, (%rdx)
-; AVX512F-NEXT:    vmovdqa64 %zmm6, 64(%rdx)
-; AVX512F-NEXT:    vmovdqa64 %zmm3, 384(%rdx)
+; AVX512F-NEXT:    vmovdqa64 %zmm13, 64(%rdx)
+; AVX512F-NEXT:    vmovdqa64 %zmm16, 384(%rdx)
 ; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-NEXT:    vmovaps %zmm0, 448(%rcx)
 ; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
@@ -10181,61 +10185,60 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512F-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-NEXT:    vmovaps %zmm0, 384(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm25, 448(%rax)
 ; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 448(%rax)
-; AVX512F-NEXT:    vmovups (%rsp), %zmm0 # 64-byte Reload
 ; AVX512F-NEXT:    vmovaps %zmm0, 256(%rax)
 ; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-NEXT:    vmovaps %zmm0, 320(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm11, 128(%rax)
+; AVX512F-NEXT:    vmovups (%rsp), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm0, 128(%rax)
 ; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-NEXT:    vmovaps %zmm0, 192(%rax)
 ; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-NEXT:    vmovaps %zmm0, (%rax)
 ; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-NEXT:    vmovaps %zmm0, 64(%rax)
-; AVX512F-NEXT:    addq $6600, %rsp # imm = 0x19C8
+; AVX512F-NEXT:    addq $6664, %rsp # imm = 0x1A08
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: load_i64_stride8_vf64:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    subq $6600, %rsp # imm = 0x19C8
-; AVX512BW-NEXT:    vmovdqa64 3392(%rdi), %zmm13
-; AVX512BW-NEXT:    vmovdqa64 3328(%rdi), %zmm7
-; AVX512BW-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 3520(%rdi), %zmm1
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 3456(%rdi), %zmm10
-; AVX512BW-NEXT:    vmovdqa64 1856(%rdi), %zmm17
-; AVX512BW-NEXT:    vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 1984(%rdi), %zmm12
-; AVX512BW-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 832(%rdi), %zmm8
-; AVX512BW-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 768(%rdi), %zmm4
-; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 960(%rdi), %zmm9
-; AVX512BW-NEXT:    vmovdqa64 896(%rdi), %zmm11
+; AVX512BW-NEXT:    subq $6664, %rsp # imm = 0x1A08
+; AVX512BW-NEXT:    vmovdqa64 3392(%rdi), %zmm3
+; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 3328(%rdi), %zmm16
+; AVX512BW-NEXT:    vmovdqa64 3520(%rdi), %zmm8
+; AVX512BW-NEXT:    vmovdqa64 3456(%rdi), %zmm28
+; AVX512BW-NEXT:    vmovdqa64 1856(%rdi), %zmm10
+; AVX512BW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 1984(%rdi), %zmm11
 ; AVX512BW-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 320(%rdi), %zmm15
-; AVX512BW-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 256(%rdi), %zmm5
+; AVX512BW-NEXT:    vmovdqa64 832(%rdi), %zmm4
+; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 768(%rdi), %zmm5
 ; AVX512BW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 448(%rdi), %zmm16
-; AVX512BW-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 384(%rdi), %zmm6
+; AVX512BW-NEXT:    vmovdqa64 960(%rdi), %zmm6
+; AVX512BW-NEXT:    vmovdqa64 896(%rdi), %zmm9
+; AVX512BW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 320(%rdi), %zmm12
+; AVX512BW-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 256(%rdi), %zmm15
+; AVX512BW-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 448(%rdi), %zmm13
+; AVX512BW-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 384(%rdi), %zmm14
+; AVX512BW-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    movb $-64, %al
 ; AVX512BW-NEXT:    kmovd %eax, %k1
 ; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm2 = [2,10,2,10,2,10,2,10]
 ; AVX512BW-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm0
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm29
-; AVX512BW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm2, %zmm0
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm1
-; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm2, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm28, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm28, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm2, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm2, %zmm1
 ; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
 ; AVX512BW-NEXT:    vmovdqa 3264(%rdi), %ymm3
 ; AVX512BW-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
@@ -10244,62 +10247,60 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm3[0],ymm0[2],ymm3[2]
 ; AVX512BW-NEXT:    vmovdqa 3136(%rdi), %ymm3
 ; AVX512BW-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512BW-NEXT:    vmovdqa 3072(%rdi), %ymm14
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm3 = ymm14[0],ymm3[0],ymm14[2],ymm3[2]
+; AVX512BW-NEXT:    vmovdqa 3072(%rdi), %ymm7
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm3 = ymm7[0],ymm3[0],ymm7[2],ymm3[2]
 ; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm3[2,3],ymm0[2,3]
 ; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm1, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm2, %zmm0
-; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm7
-; AVX512BW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm1
-; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm2, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm2, %zmm0
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm9
+; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm2, %zmm1
 ; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512BW-NEXT:    vmovdqa 704(%rdi), %ymm3
-; AVX512BW-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512BW-NEXT:    vmovdqa 640(%rdi), %ymm0
+; AVX512BW-NEXT:    vmovdqa 704(%rdi), %ymm0
 ; AVX512BW-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm3[0],ymm0[2],ymm3[2]
-; AVX512BW-NEXT:    vmovdqa64 576(%rdi), %ymm26
-; AVX512BW-NEXT:    vmovdqa64 512(%rdi), %ymm23
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm3 = ymm23[0],ymm26[0],ymm23[2],ymm26[2]
+; AVX512BW-NEXT:    vmovdqa64 640(%rdi), %ymm20
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm20[0],ymm0[0],ymm20[2],ymm0[2]
+; AVX512BW-NEXT:    vmovdqa64 576(%rdi), %ymm22
+; AVX512BW-NEXT:    vmovdqa64 512(%rdi), %ymm19
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm3 = ymm19[0],ymm22[0],ymm19[2],ymm22[2]
 ; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm3[2,3],ymm0[2,3]
 ; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm1, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm0
-; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm8
-; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm16, %zmm2, %zmm0
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm1
-; AVX512BW-NEXT:    vpermt2q %zmm15, %zmm2, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm2, %zmm0
+; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm12, %zmm2, %zmm1
 ; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512BW-NEXT:    vmovdqa 192(%rdi), %ymm0
-; AVX512BW-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 128(%rdi), %ymm30
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm30[0],ymm0[0],ymm30[2],ymm0[2]
-; AVX512BW-NEXT:    vmovdqa64 64(%rdi), %ymm20
-; AVX512BW-NEXT:    vmovdqa64 (%rdi), %ymm16
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm3 = ymm16[0],ymm20[0],ymm16[2],ymm20[2]
+; AVX512BW-NEXT:    vmovdqa 192(%rdi), %ymm14
+; AVX512BW-NEXT:    vmovdqa 128(%rdi), %ymm15
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm15[0],ymm14[0],ymm15[2],ymm14[2]
+; AVX512BW-NEXT:    vmovdqa64 64(%rdi), %ymm21
+; AVX512BW-NEXT:    vmovdqa (%rdi), %ymm13
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm3 = ymm13[0],ymm21[0],ymm13[2],ymm21[2]
 ; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm3[2,3],ymm0[2,3]
 ; AVX512BW-NEXT:    vmovdqa64 1920(%rdi), %zmm3
 ; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm1, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm12, %zmm2, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm2, %zmm0
 ; AVX512BW-NEXT:    vmovdqa64 1792(%rdi), %zmm1
 ; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm2, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm10, %zmm2, %zmm1
 ; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
 ; AVX512BW-NEXT:    vmovdqa 1728(%rdi), %ymm3
 ; AVX512BW-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512BW-NEXT:    vmovdqa 1664(%rdi), %ymm0
 ; AVX512BW-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm3[0],ymm0[2],ymm3[2]
-; AVX512BW-NEXT:    vmovdqa64 1600(%rdi), %ymm21
-; AVX512BW-NEXT:    vmovdqa64 1536(%rdi), %ymm17
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm3 = ymm17[0],ymm21[0],ymm17[2],ymm21[2]
+; AVX512BW-NEXT:    vmovdqa 1600(%rdi), %ymm4
+; AVX512BW-NEXT:    vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-NEXT:    vmovdqa 1536(%rdi), %ymm3
+; AVX512BW-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm3 = ymm3[0],ymm4[0],ymm3[2],ymm4[2]
 ; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm3[2,3],ymm0[2,3]
 ; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm1, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
@@ -10310,61 +10311,64 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm2, %zmm0
 ; AVX512BW-NEXT:    vmovdqa64 1344(%rdi), %zmm3
 ; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 1280(%rdi), %zmm11
-; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 1280(%rdi), %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, (%rsp) # 64-byte Spill
 ; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm2, %zmm1
 ; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512BW-NEXT:    vmovdqa 1216(%rdi), %ymm0
+; AVX512BW-NEXT:    vmovdqa 1216(%rdi), %ymm3
+; AVX512BW-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-NEXT:    vmovdqa 1152(%rdi), %ymm0
 ; AVX512BW-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 1152(%rdi), %ymm25
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm25[0],ymm0[0],ymm25[2],ymm0[2]
-; AVX512BW-NEXT:    vmovdqa64 1088(%rdi), %ymm24
-; AVX512BW-NEXT:    vmovdqa64 1024(%rdi), %ymm22
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm3 = ymm22[0],ymm24[0],ymm22[2],ymm24[2]
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm3[0],ymm0[2],ymm3[2]
+; AVX512BW-NEXT:    vmovdqa 1088(%rdi), %ymm4
+; AVX512BW-NEXT:    vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-NEXT:    vmovdqa 1024(%rdi), %ymm3
+; AVX512BW-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm3 = ymm3[0],ymm4[0],ymm3[2],ymm4[2]
 ; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm3[2,3],ymm0[2,3]
 ; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm1, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vmovdqa64 3008(%rdi), %zmm1
 ; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 2944(%rdi), %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 2944(%rdi), %zmm29
+; AVX512BW-NEXT:    vmovdqa64 %zmm29, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm29, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm2, %zmm0
-; AVX512BW-NEXT:    vmovdqa64 2880(%rdi), %zmm3
-; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 2816(%rdi), %zmm1
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm2, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 2880(%rdi), %zmm24
+; AVX512BW-NEXT:    vmovdqa64 2816(%rdi), %zmm25
+; AVX512BW-NEXT:    vmovdqa64 %zmm25, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm25, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm24, %zmm2, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm24, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512BW-NEXT:    vmovdqa 2752(%rdi), %ymm3
-; AVX512BW-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512BW-NEXT:    vmovdqa 2688(%rdi), %ymm0
-; AVX512BW-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm3[0],ymm0[2],ymm3[2]
-; AVX512BW-NEXT:    vmovdqa64 2624(%rdi), %ymm31
-; AVX512BW-NEXT:    vmovdqa 2560(%rdi), %ymm10
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm3 = ymm10[0],ymm31[0],ymm10[2],ymm31[2]
+; AVX512BW-NEXT:    vmovdqa64 2752(%rdi), %ymm27
+; AVX512BW-NEXT:    vmovdqa64 2688(%rdi), %ymm26
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm26[0],ymm27[0],ymm26[2],ymm27[2]
+; AVX512BW-NEXT:    vmovdqa64 2624(%rdi), %ymm30
+; AVX512BW-NEXT:    vmovdqa64 2560(%rdi), %ymm18
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm3 = ymm18[0],ymm30[0],ymm18[2],ymm30[2]
 ; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm3[2,3],ymm0[2,3]
 ; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm1, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vmovdqa64 2496(%rdi), %zmm1
 ; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 2432(%rdi), %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 2432(%rdi), %zmm31
+; AVX512BW-NEXT:    vmovdqa64 %zmm31, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm31, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm2, %zmm0
-; AVX512BW-NEXT:    vmovdqa64 2368(%rdi), %zmm18
+; AVX512BW-NEXT:    vmovdqa64 2368(%rdi), %zmm3
+; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vmovdqa64 2304(%rdi), %zmm1
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, (%rsp) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm2, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm2, %zmm1
 ; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 2240(%rdi), %ymm28
-; AVX512BW-NEXT:    vmovdqa64 2176(%rdi), %ymm19
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm19[0],ymm28[0],ymm19[2],ymm28[2]
-; AVX512BW-NEXT:    vmovdqa64 2112(%rdi), %ymm27
-; AVX512BW-NEXT:    vmovdqa 2048(%rdi), %ymm6
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm3 = ymm6[0],ymm27[0],ymm6[2],ymm27[2]
-; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm3[2,3],ymm0[2,3]
+; AVX512BW-NEXT:    vmovdqa 2240(%rdi), %ymm12
+; AVX512BW-NEXT:    vmovdqa 2176(%rdi), %ymm11
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm11[0],ymm12[0],ymm11[2],ymm12[2]
+; AVX512BW-NEXT:    vmovdqa 2112(%rdi), %ymm10
+; AVX512BW-NEXT:    vmovdqa 2048(%rdi), %ymm3
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm5 = ymm3[0],ymm10[0],ymm3[2],ymm10[2]
+; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm5[2,3],ymm0[2,3]
 ; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm1, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vmovdqa64 4032(%rdi), %zmm1
@@ -10372,807 +10376,801 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512BW-NEXT:    vmovdqa64 3968(%rdi), %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm2, %zmm0
-; AVX512BW-NEXT:    vmovdqa64 3904(%rdi), %zmm5
-; AVX512BW-NEXT:    vmovdqa64 3840(%rdi), %zmm3
-; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermi2q %zmm5, %zmm3, %zmm2
+; AVX512BW-NEXT:    vmovdqa64 3904(%rdi), %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 3840(%rdi), %zmm4
+; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermi2q %zmm1, %zmm4, %zmm2
 ; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k1}
-; AVX512BW-NEXT:    vmovdqa 3776(%rdi), %ymm12
-; AVX512BW-NEXT:    vmovdqa 3712(%rdi), %ymm9
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm9[0],ymm12[0],ymm9[2],ymm12[2]
-; AVX512BW-NEXT:    vmovdqa 3648(%rdi), %ymm4
-; AVX512BW-NEXT:    vmovdqa 3584(%rdi), %ymm1
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm15 = ymm1[0],ymm4[0],ymm1[2],ymm4[2]
-; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm15[2,3],ymm0[2,3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm2, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 3776(%rdi), %ymm17
+; AVX512BW-NEXT:    vmovdqa64 3712(%rdi), %ymm23
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm5 = ymm23[0],ymm17[0],ymm23[2],ymm17[2]
+; AVX512BW-NEXT:    vmovdqa 3648(%rdi), %ymm1
+; AVX512BW-NEXT:    vmovdqa 3584(%rdi), %ymm0
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm6 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
+; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm5 = ymm6[2,3],ymm5[2,3]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm5, %zmm2, %zmm2
+; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm2 = [3,11,3,11,3,11,3,11]
 ; AVX512BW-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm29, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm2, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm2, %zmm15
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm15 {%k1}
-; AVX512BW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX512BW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
-; AVX512BW-NEXT:    # ymm0 = ymm0[1],mem[1],ymm0[3],mem[3]
-; AVX512BW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm14, %ymm14 # 32-byte Folded Reload
-; AVX512BW-NEXT:    # ymm14 = ymm14[1],mem[1],ymm14[3],mem[3]
-; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm14[2,3],ymm0[2,3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm15, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm2, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm14
-; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm14 # 64-byte Folded Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm14 {%k1}
-; AVX512BW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX512BW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
-; AVX512BW-NEXT:    # ymm0 = ymm0[1],mem[1],ymm0[3],mem[3]
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm15 = ymm23[1],ymm26[1],ymm23[3],ymm26[3]
-; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm15[2,3],ymm0[2,3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm14, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm23 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm23, %zmm2, %zmm8
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm29 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm29, %zmm14
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm26 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm26, %zmm2, %zmm14
-; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm14 {%k1}
-; AVX512BW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm30, %ymm0 # 32-byte Folded Reload
-; AVX512BW-NEXT:    # ymm0 = ymm30[1],mem[1],ymm30[3],mem[3]
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm15 = ymm16[1],ymm20[1],ymm16[3],ymm20[3]
-; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm15[2,3],ymm0[2,3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm14, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm30 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm30, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm20 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm2, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm14
+; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm2, %zmm28
+; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm8
+; AVX512BW-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm6
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm16, %zmm2, %zmm14
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm14 {%k1}
-; AVX512BW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX512BW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
-; AVX512BW-NEXT:    # ymm0 = ymm0[1],mem[1],ymm0[3],mem[3]
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm15 = ymm17[1],ymm21[1],ymm17[3],ymm21[3]
-; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm15[2,3],ymm0[2,3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm14, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm21 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm21, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm2, %zmm0
-; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm14
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm2, %zmm14
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm14 {%k1}
-; AVX512BW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm25, %ymm0 # 32-byte Folded Reload
-; AVX512BW-NEXT:    # ymm0 = ymm25[1],mem[1],ymm25[3],mem[3]
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm15 = ymm22[1],ymm24[1],ymm22[3],ymm24[3]
-; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm15[2,3],ymm0[2,3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm14, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm16, %zmm2, %zmm6
+; AVX512BW-NEXT:    vmovdqa64 %zmm28, %zmm6 {%k1}
+; AVX512BW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
+; AVX512BW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm4, %ymm5 # 32-byte Folded Reload
+; AVX512BW-NEXT:    # ymm5 = ymm4[1],mem[1],ymm4[3],mem[3]
+; AVX512BW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm7 # 32-byte Folded Reload
+; AVX512BW-NEXT:    # ymm7 = ymm7[1],mem[1],ymm7[3],mem[3]
+; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm5 = ymm7[2,3],ymm5[2,3]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm5, %zmm6, %zmm5
+; AVX512BW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm28 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm28, %zmm5
+; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm2, %zmm5
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm6
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm2, %zmm6
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm6 {%k1}
+; AVX512BW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm20, %ymm5 # 32-byte Folded Reload
+; AVX512BW-NEXT:    # ymm5 = ymm20[1],mem[1],ymm20[3],mem[3]
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm7 = ymm19[1],ymm22[1],ymm19[3],ymm22[3]
+; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm5 = ymm7[2,3],ymm5[2,3]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm5, %zmm6, %zmm5
+; AVX512BW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm20 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm20, %zmm5
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm2, %zmm5
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm22 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm22, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm25 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm25, %zmm2, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm24 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm24, %zmm14
+; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm2, %zmm6
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm6 {%k1}
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm5 = ymm15[1],ymm14[1],ymm15[3],ymm14[3]
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm7 = ymm13[1],ymm21[1],ymm13[3],ymm21[3]
+; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm5 = ymm7[2,3],ymm5[2,3]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm5, %zmm6, %zmm5
+; AVX512BW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm5
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm15, %zmm2, %zmm14
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm14 {%k1}
-; AVX512BW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX512BW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
-; AVX512BW-NEXT:    # ymm0 = ymm0[1],mem[1],ymm0[3],mem[3]
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm10 = ymm10[1],ymm31[1],ymm10[3],ymm31[3]
-; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm10[2,3],ymm0[2,3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm14, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm31 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm31, %zmm0
-; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm0 # 64-byte Folded Reload
-; AVX512BW-NEXT:    vmovdqu64 (%rsp), %zmm10 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm2, %zmm10
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm10 {%k1}
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm0 = ymm19[1],ymm28[1],ymm19[3],ymm28[3]
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm6 = ymm6[1],ymm27[1],ymm6[3],ymm27[3]
-; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm6[2,3],ymm0[2,3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm10, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm15, %zmm2, %zmm5
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm21 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm21, %zmm6
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm2, %zmm6
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm6 {%k1}
+; AVX512BW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
+; AVX512BW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm5 # 32-byte Folded Reload
+; AVX512BW-NEXT:    # ymm5 = ymm5[1],mem[1],ymm5[3],mem[3]
+; AVX512BW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
+; AVX512BW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm7 # 32-byte Folded Reload
+; AVX512BW-NEXT:    # ymm7 = ymm7[1],mem[1],ymm7[3],mem[3]
+; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm5 = ymm7[2,3],ymm5[2,3]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm5, %zmm6, %zmm5
+; AVX512BW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm5 # 64-byte Folded Reload
+; AVX512BW-NEXT:    vmovdqu64 (%rsp), %zmm6 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm6 # 64-byte Folded Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm6 {%k1}
+; AVX512BW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm5 # 32-byte Reload
+; AVX512BW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm5, %ymm5 # 32-byte Folded Reload
+; AVX512BW-NEXT:    # ymm5 = ymm5[1],mem[1],ymm5[3],mem[3]
+; AVX512BW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
+; AVX512BW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm7 # 32-byte Folded Reload
+; AVX512BW-NEXT:    # ymm7 = ymm7[1],mem[1],ymm7[3],mem[3]
+; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm5 = ymm7[2,3],ymm5[2,3]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm5, %zmm6, %zmm5
+; AVX512BW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm29 # 64-byte Folded Reload
+; AVX512BW-NEXT:    vpermt2q %zmm24, %zmm2, %zmm25
+; AVX512BW-NEXT:    vmovdqa64 %zmm29, %zmm25 {%k1}
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm5 = ymm26[1],ymm27[1],ymm26[3],ymm27[3]
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm7 = ymm18[1],ymm30[1],ymm18[3],ymm30[3]
+; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm5 = ymm7[2,3],ymm5[2,3]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm5, %zmm25, %zmm5
+; AVX512BW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm24 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm24, %zmm2, %zmm31
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm26 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm26, %zmm6
+; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm6 # 64-byte Folded Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm31, %zmm6 {%k1}
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm5 = ymm11[1],ymm12[1],ymm11[3],ymm12[3]
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm3 = ymm3[1],ymm10[1],ymm3[3],ymm10[3]
+; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm3 = ymm3[2,3],ymm5[2,3]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm3, %zmm6, %zmm3
+; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm29 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm29, %zmm3
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm2, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
-; AVX512BW-NEXT:    vpermi2q %zmm5, %zmm19, %zmm2
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm28
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k1}
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm0 = ymm9[1],ymm12[1],ymm9[3],ymm12[3]
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm1[1],ymm4[1],ymm1[3],ymm4[3]
-; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm1[2,3],ymm0[2,3]
+; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm2, %zmm3
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512BW-NEXT:    vpermi2q %zmm11, %zmm12, %zmm2
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm2 {%k1}
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm3 = ymm23[1],ymm17[1],ymm23[3],ymm17[3]
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm0 = ymm0[1],ymm1[1],ymm0[3],ymm1[3]
+; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm0[2,3],ymm3[2,3]
 ; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm2, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 3264(%rdi), %zmm3
+; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 3200(%rdi), %zmm10
 ; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [4,12,4,12,4,12,4,12]
 ; AVX512BW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm1
 ; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm0, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm27 # 64-byte Reload
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm2 = zmm27[0],zmm12[0],zmm27[2],zmm12[2],zmm27[4],zmm12[4],zmm27[6],zmm12[6]
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 3136(%rdi), %zmm1
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 3072(%rdi), %zmm3
-; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm0, %zmm3
-; AVX512BW-NEXT:    vmovdqa64 3264(%rdi), %zmm5
-; AVX512BW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 3200(%rdi), %zmm4
-; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [4,12,4,12]
-; AVX512BW-NEXT:    # ymm1 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm1, %zmm4
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm4[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm3, %zmm2, %zmm2
+; AVX512BW-NEXT:    vmovdqa64 3136(%rdi), %zmm23
+; AVX512BW-NEXT:    vmovdqa64 3072(%rdi), %zmm2
 ; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm6
-; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm2
-; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 # 64-byte Folded Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm3 = zmm7[0],zmm13[0],zmm7[2],zmm13[2],zmm7[4],zmm13[4],zmm7[6],zmm13[6]
+; AVX512BW-NEXT:    vpermt2q %zmm23, %zmm0, %zmm2
+; AVX512BW-NEXT:    vmovdqu64 %zmm23, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm17, %zmm2
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm25 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm25, %zmm0, %zmm2
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm3 = zmm8[0],zmm16[0],zmm8[2],zmm16[2],zmm8[4],zmm16[4],zmm8[6],zmm16[6]
 ; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 576(%rdi), %zmm5
-; AVX512BW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 512(%rdi), %zmm2
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm3, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 704(%rdi), %zmm2
 ; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm0, %zmm2
-; AVX512BW-NEXT:    vmovdqa64 704(%rdi), %zmm5
-; AVX512BW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 640(%rdi), %zmm4
-; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm1, %zmm4
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm4[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm3, %zmm2
+; AVX512BW-NEXT:    vmovdqa64 640(%rdi), %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm0, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 576(%rdi), %zmm3
+; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 512(%rdi), %zmm2
 ; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm23, %zmm0, %zmm2
-; AVX512BW-NEXT:    vmovdqa64 %zmm23, %zmm7
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm3 = zmm29[0],zmm26[0],zmm29[2],zmm26[2],zmm29[4],zmm26[4],zmm29[6],zmm26[6]
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm0, %zmm2
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm28, %zmm2
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm30 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm30, %zmm0, %zmm2
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm3 = zmm9[0],zmm4[0],zmm9[2],zmm4[2],zmm9[4],zmm4[4],zmm9[6],zmm4[6]
 ; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 64(%rdi), %zmm4
-; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm2
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm3, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 192(%rdi), %zmm2
 ; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm0, %zmm2
-; AVX512BW-NEXT:    vmovdqa64 192(%rdi), %zmm23
-; AVX512BW-NEXT:    vmovdqa64 128(%rdi), %zmm5
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm4
-; AVX512BW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm23, %zmm1, %zmm4
-; AVX512BW-NEXT:    vmovdqa64 %zmm23, %zmm26
-; AVX512BW-NEXT:    vmovdqu64 %zmm23, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm4[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm3, %zmm2
+; AVX512BW-NEXT:    vmovdqa64 128(%rdi), %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm0, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 64(%rdi), %zmm3
+; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm2
 ; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm30, %zmm2
-; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm0, %zmm2
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm3 = zmm8[0],zmm16[0],zmm8[2],zmm16[2],zmm8[4],zmm16[4],zmm8[6],zmm16[6]
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm0, %zmm2
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm20, %zmm2
+; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm0, %zmm2
+; AVX512BW-NEXT:    vmovdqa64 %zmm19, %zmm16
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm3 = zmm3[0],zmm22[0],zmm3[2],zmm22[2],zmm3[4],zmm22[4],zmm3[6],zmm22[6]
 ; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 1600(%rdi), %zmm30
-; AVX512BW-NEXT:    vmovdqa64 1536(%rdi), %zmm2
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm3, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 1728(%rdi), %zmm2
 ; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm30, %zmm0, %zmm2
-; AVX512BW-NEXT:    vmovdqa64 1728(%rdi), %zmm20
-; AVX512BW-NEXT:    vmovdqa64 1664(%rdi), %zmm4
-; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm1, %zmm4
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm4[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm3, %zmm2
+; AVX512BW-NEXT:    vmovdqa64 1664(%rdi), %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm0, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 1600(%rdi), %zmm3
+; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 1536(%rdi), %zmm2
 ; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm21, %zmm2
-; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm0, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm3 = zmm9[0],zmm11[0],zmm9[2],zmm11[2],zmm9[4],zmm11[4],zmm9[6],zmm11[6]
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm0, %zmm2
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm2
+; AVX512BW-NEXT:    vpermt2q %zmm15, %zmm0, %zmm2
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm3 = zmm21[0],zmm13[0],zmm21[2],zmm13[2],zmm21[4],zmm13[4],zmm21[6],zmm13[6]
 ; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 1088(%rdi), %zmm4
-; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 1024(%rdi), %zmm2
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm3, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 1216(%rdi), %zmm2
 ; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm0, %zmm2
-; AVX512BW-NEXT:    vmovdqa64 1216(%rdi), %zmm16
-; AVX512BW-NEXT:    vmovdqa64 1152(%rdi), %zmm4
-; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm16, %zmm1, %zmm4
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm4[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm3, %zmm2
+; AVX512BW-NEXT:    vmovdqa64 1152(%rdi), %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm0, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 1088(%rdi), %zmm3
+; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 1024(%rdi), %zmm2
 ; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm22, %zmm2
-; AVX512BW-NEXT:    vpermt2q %zmm25, %zmm0, %zmm2
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm3 = zmm24[0],zmm15[0],zmm24[2],zmm15[2],zmm24[4],zmm15[4],zmm24[6],zmm15[6]
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm0, %zmm2
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm31 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm31, %zmm2
+; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 # 64-byte Folded Reload
+; AVX512BW-NEXT:    vmovdqu64 (%rsp), %zmm27 # 64-byte Reload
+; AVX512BW-NEXT:    vpunpcklqdq {{[-0-9]+}}(%r{{[sb]}}p), %zmm27, %zmm3 # 64-byte Folded Reload
+; AVX512BW-NEXT:    # zmm3 = zmm27[0],mem[0],zmm27[2],mem[2],zmm27[4],mem[4],zmm27[6],mem[6]
 ; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 2624(%rdi), %zmm4
-; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm3, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 2752(%rdi), %zmm3
+; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 2688(%rdi), %zmm8
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm0, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 2624(%rdi), %zmm3
+; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vmovdqa64 2560(%rdi), %zmm2
 ; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm0, %zmm2
-; AVX512BW-NEXT:    vmovdqa64 2752(%rdi), %zmm14
-; AVX512BW-NEXT:    vmovdqa64 2688(%rdi), %zmm4
-; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm1, %zmm4
-; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm25
-; AVX512BW-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm4[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm3, %zmm2
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm0, %zmm2
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm2
+; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 # 64-byte Folded Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512BW-NEXT:    vpunpcklqdq {{[-0-9]+}}(%r{{[sb]}}p), %zmm7, %zmm3 # 64-byte Folded Reload
+; AVX512BW-NEXT:    # zmm3 = zmm7[0],mem[0],zmm7[2],mem[2],zmm7[4],mem[4],zmm7[6],mem[6]
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm3, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 2240(%rdi), %zmm2
 ; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 2176(%rdi), %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm0, %zmm1
 ; AVX512BW-NEXT:    vmovdqa64 2112(%rdi), %zmm3
 ; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vmovdqa64 2048(%rdi), %zmm2
 ; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm0, %zmm2
-; AVX512BW-NEXT:    vmovdqa64 2240(%rdi), %zmm4
-; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 2176(%rdi), %zmm3
-; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm1, %zmm3
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5,6,7]
-; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm31 # 64-byte Folded Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 (%rsp), %zmm4 # 64-byte Reload
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm4 = zmm4[0],zmm8[0],zmm4[2],zmm8[2],zmm4[4],zmm8[4],zmm4[6],zmm8[6]
-; AVX512BW-NEXT:    vmovdqa64 %zmm31, %zmm4 {%k1}
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm4, %zmm2
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm2
+; AVX512BW-NEXT:    vpermt2q %zmm24, %zmm0, %zmm2
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm3 = zmm26[0],zmm4[0],zmm26[2],zmm4[2],zmm26[4],zmm4[4],zmm26[6],zmm4[6]
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm3, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 3776(%rdi), %zmm2
 ; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm0, %zmm10
-; AVX512BW-NEXT:    vmovdqa64 3648(%rdi), %zmm4
-; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 3584(%rdi), %zmm3
-; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermi2q %zmm4, %zmm3, %zmm0
-; AVX512BW-NEXT:    vmovdqa64 3776(%rdi), %zmm4
-; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 3712(%rdi), %zmm3
+; AVX512BW-NEXT:    vmovdqa64 3712(%rdi), %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm0, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 3648(%rdi), %zmm3
 ; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermi2q %zmm4, %zmm3, %zmm1
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqa64 %zmm28, %zmm18
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm1 = zmm19[0],zmm28[0],zmm19[2],zmm28[2],zmm19[4],zmm28[4],zmm19[6],zmm28[6]
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm1 {%k1}
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm1, %zmm0
+; AVX512BW-NEXT:    vmovdqa64 3584(%rdi), %zmm2
+; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm0, %zmm2
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512BW-NEXT:    vpermi2q %zmm18, %zmm29, %zmm0
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm2 = zmm12[0],zmm11[0],zmm12[2],zmm11[2],zmm12[4],zmm11[4],zmm12[6],zmm11[6]
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k1}
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm2, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [5,13,5,13,5,13,5,13]
 ; AVX512BW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm26 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm26, %zmm0, %zmm10
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 # 64-byte Folded Reload
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [5,13,5,13]
-; AVX512BW-NEXT:    # ymm1 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm3 # 64-byte Folded Reload
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm3 # 64-byte Folded Reload
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm4 = zmm27[1],zmm12[1],zmm27[3],zmm12[3],zmm27[5],zmm12[5],zmm27[7],zmm12[7]
-; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm4 {%k1}
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm4, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm23, %zmm0, %zmm2
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm10[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm17, %zmm2
+; AVX512BW-NEXT:    vpermt2q %zmm25, %zmm0, %zmm2
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm25 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm3 = zmm19[1],zmm25[1],zmm19[3],zmm25[3],zmm19[5],zmm25[5],zmm19[7],zmm25[7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm3, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm22 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm0, %zmm1
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm24 # 64-byte Reload
 ; AVX512BW-NEXT:    vpermt2q %zmm24, %zmm0, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm28, %zmm29
+; AVX512BW-NEXT:    vmovdqa64 %zmm28, %zmm2
+; AVX512BW-NEXT:    vpermt2q %zmm30, %zmm0, %zmm2
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm28 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm28, %zmm1, %zmm3
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm3
-; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm3 # 64-byte Folded Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm23
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm31 # 64-byte Reload
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm4 = zmm31[1],zmm13[1],zmm31[3],zmm13[3],zmm31[5],zmm13[5],zmm31[7],zmm13[7]
-; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm4 {%k1}
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm4, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm3 = zmm15[1],zmm28[1],zmm15[3],zmm28[3],zmm15[5],zmm28[5],zmm15[7],zmm28[7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm3, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm20 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm0, %zmm1
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm22 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm0, %zmm2
-; AVX512BW-NEXT:    vpermt2q %zmm26, %zmm1, %zmm5
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm5[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm0, %zmm3
-; AVX512BW-NEXT:    vmovdqa64 %zmm29, %zmm14
-; AVX512BW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %zmm29, %zmm4 # 64-byte Folded Reload
-; AVX512BW-NEXT:    # zmm4 = zmm29[1],mem[1],zmm29[3],mem[3],zmm29[5],mem[5],zmm29[7],mem[7]
-; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm4 {%k1}
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm4, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm21 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm21, %zmm0, %zmm2
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 %zmm30, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm30, %zmm0, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 %zmm20, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm1, %zmm3
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm3 # 64-byte Folded Reload
+; AVX512BW-NEXT:    vpermt2q %zmm16, %zmm0, %zmm2
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm3 = zmm12[1],zmm18[1],zmm12[3],zmm18[3],zmm12[5],zmm18[5],zmm12[7],zmm18[7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm3, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
-; AVX512BW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %zmm17, %zmm4 # 64-byte Folded Reload
-; AVX512BW-NEXT:    # zmm4 = zmm17[1],mem[1],zmm17[3],mem[3],zmm17[5],mem[5],zmm17[7],mem[7]
-; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm4 {%k1}
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm4, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm0, %zmm1
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm0, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm3
-; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm29
-; AVX512BW-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm16, %zmm1, %zmm3
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm16, %zmm0, %zmm2
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm2
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm23 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm23, %zmm0, %zmm2
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm3 # 64-byte Folded Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm16
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm4 = zmm9[1],zmm11[1],zmm9[3],zmm11[3],zmm9[5],zmm11[5],zmm9[7],zmm11[7]
-; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm4 {%k1}
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm4, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm12, %zmm0, %zmm2
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm3 = zmm3[1],zmm13[1],zmm3[3],zmm13[3],zmm3[5],zmm13[5],zmm3[7],zmm13[7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm3, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm3
-; AVX512BW-NEXT:    vpermt2q %zmm25, %zmm1, %zmm3
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm3 # 64-byte Folded Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512BW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm4 # 64-byte Folded Reload
-; AVX512BW-NEXT:    # zmm4 = zmm4[1],mem[1],zmm4[3],mem[3],zmm4[5],mem[5],zmm4[7],mem[7]
-; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm4 {%k1}
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm4, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm0, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm0, %zmm2
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm31 # 64-byte Folded Reload
+; AVX512BW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %zmm27, %zmm3 # 64-byte Folded Reload
+; AVX512BW-NEXT:    # zmm3 = zmm27[1],mem[1],zmm27[3],mem[3],zmm27[5],mem[5],zmm27[7],mem[7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm31, %zmm3 {%k1}
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm3, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm31 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm31, %zmm0, %zmm8
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm0, %zmm2
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm8[4,5,6,7]
+; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm9 # 64-byte Folded Reload
+; AVX512BW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %zmm7, %zmm3 # 64-byte Folded Reload
+; AVX512BW-NEXT:    # zmm3 = zmm7[1],mem[1],zmm7[3],mem[3],zmm7[5],mem[5],zmm7[7],mem[7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm3 {%k1}
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm3, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm27 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm27, %zmm0, %zmm1
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm0, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm3
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm10, %zmm1, %zmm3
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm26 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm26, %zmm3
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm21 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm21, %zmm0, %zmm3
-; AVX512BW-NEXT:    vmovdqu64 (%rsp), %zmm4 # 64-byte Reload
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm4 = zmm4[1],zmm8[1],zmm4[3],zmm8[3],zmm4[5],zmm8[5],zmm4[7],zmm8[7]
-; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm4 {%k1}
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm4, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 # 64-byte Folded Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm2
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm0, %zmm2
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm5 # 64-byte Folded Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm3 = zmm3[1],zmm4[1],zmm3[3],zmm4[3],zmm3[5],zmm4[5],zmm3[7],zmm4[7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm3 {%k1}
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm3, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-NEXT:    vpermi2q %zmm6, %zmm3, %zmm0
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm1
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm0, %zmm1
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512BW-NEXT:    vpermi2q %zmm4, %zmm5, %zmm1
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm25 # 64-byte Reload
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm1 = zmm25[1],zmm18[1],zmm25[3],zmm18[3],zmm25[5],zmm18[5],zmm25[7],zmm18[7]
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm1 {%k1}
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm1, %zmm0
+; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm2
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm0, %zmm2
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
+; AVX512BW-NEXT:    vpermi2q %zmm10, %zmm9, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm2 # 64-byte Folded Reload
+; AVX512BW-NEXT:    # zmm2 = zmm2[1],mem[1],zmm2[3],mem[3],zmm2[5],mem[5],zmm2[7],mem[7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k1}
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm2, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [6,14,6,14,6,14,6,14]
 ; AVX512BW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm26, %zmm0, %zmm1
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
 ; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 # 64-byte Folded Reload
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [6,14,6,14]
-; AVX512BW-NEXT:    # ymm1 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm3 # 64-byte Folded Reload
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm3 # 64-byte Folded Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm27 # 64-byte Reload
-; AVX512BW-NEXT:    vpunpcklqdq {{[-0-9]+}}(%r{{[sb]}}p), %zmm27, %zmm3 {%k1} # 64-byte Folded Reload
-; AVX512BW-NEXT:    # zmm3 {%k1} = zmm27[0],mem[0],zmm27[2],mem[2],zmm27[4],mem[4],zmm27[6],mem[6]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm3, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm19, %zmm2
+; AVX512BW-NEXT:    vpermt2q %zmm25, %zmm0, %zmm2
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm25 # 64-byte Reload
+; AVX512BW-NEXT:    vpunpcklqdq {{[-0-9]+}}(%r{{[sb]}}p), %zmm25, %zmm2 {%k1} # 64-byte Folded Reload
+; AVX512BW-NEXT:    # zmm2 {%k1} = zmm25[0],mem[0],zmm25[2],mem[2],zmm25[4],mem[4],zmm25[6],mem[6]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm2, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm0, %zmm1
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
 ; AVX512BW-NEXT:    vpermt2q %zmm24, %zmm0, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm28, %zmm1, %zmm3
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5,6,7]
-; AVX512BW-NEXT:    vpermt2q %zmm23, %zmm0, %zmm31
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm24 # 64-byte Reload
-; AVX512BW-NEXT:    vpunpcklqdq {{[-0-9]+}}(%r{{[sb]}}p), %zmm24, %zmm31 {%k1} # 64-byte Folded Reload
-; AVX512BW-NEXT:    # zmm31 {%k1} = zmm24[0],mem[0],zmm24[2],mem[2],zmm24[4],mem[4],zmm24[6],mem[6]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm31, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm0, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm3 # 64-byte Folded Reload
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm3
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm22 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm0, %zmm3
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm28 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm3 {%k1} = zmm14[0],zmm28[0],zmm14[2],zmm28[2],zmm14[4],zmm28[4],zmm14[6],zmm28[6]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm3, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512BW-NEXT:    vpermt2q %zmm28, %zmm0, %zmm15
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm15 {%k1} = zmm29[0],zmm30[0],zmm29[2],zmm30[2],zmm29[4],zmm30[4],zmm29[6],zmm30[6]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm15, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm0, %zmm1
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm30, %zmm0, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm1, %zmm3
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqa64 %zmm17, %zmm3
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm0, %zmm3
+; AVX512BW-NEXT:    vpermt2q %zmm21, %zmm0, %zmm2
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm2
+; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm0, %zmm2
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm30 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm27 # 64-byte Reload
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm3 {%k1} = zmm30[0],zmm27[0],zmm30[2],zmm27[2],zmm30[4],zmm27[4],zmm30[6],zmm27[6]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm3, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm26 # 64-byte Reload
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm2 {%k1} = zmm30[0],zmm26[0],zmm30[2],zmm26[2],zmm30[4],zmm26[4],zmm30[6],zmm26[6]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm2, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm0, %zmm1
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm0, %zmm2
-; AVX512BW-NEXT:    vpermt2q %zmm29, %zmm1, %zmm15
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm15[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm20 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm20, %zmm3
-; AVX512BW-NEXT:    vpermt2q %zmm16, %zmm0, %zmm3
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm31 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm16, %zmm0, %zmm2
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm3 {%k1} = zmm31[0],zmm16[0],zmm31[2],zmm16[2],zmm31[4],zmm16[4],zmm31[6],zmm16[6]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm3, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm12, %zmm0, %zmm11
-; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm13 # 64-byte Folded Reload
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm11[0,1,2,3],ymm13[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm3
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm0, %zmm3
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512BW-NEXT:    vpunpcklqdq {{[-0-9]+}}(%r{{[sb]}}p), %zmm12, %zmm3 {%k1} # 64-byte Folded Reload
-; AVX512BW-NEXT:    # zmm3 {%k1} = zmm12[0],mem[0],zmm12[2],mem[2],zmm12[4],mem[4],zmm12[6],mem[6]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm3, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm2
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm29 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm29, %zmm0, %zmm2
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm24 # 64-byte Reload
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm2 {%k1} = zmm24[0],zmm23[0],zmm24[2],zmm23[2],zmm24[4],zmm23[4],zmm24[6],zmm23[6]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm2, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm0, %zmm1
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm0, %zmm2
-; AVX512BW-NEXT:    vpermt2q %zmm10, %zmm1, %zmm9
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm9[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqu64 (%rsp), %zmm10 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm3
-; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm0, %zmm3
-; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm7
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm3 {%k1} = zmm26[0],zmm21[0],zmm26[2],zmm21[2],zmm26[4],zmm21[4],zmm26[6],zmm21[6]
-; AVX512BW-NEXT:    vmovdqa64 %zmm21, %zmm8
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm3, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm25, %zmm2
-; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm0, %zmm2
+; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm0, %zmm2
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqu64 (%rsp), %zmm2 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm21 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm21, %zmm0, %zmm2
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm23 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm2 {%k1} = zmm23[0],zmm18[0],zmm23[2],zmm18[2],zmm23[4],zmm18[4],zmm23[6],zmm18[6]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm2, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm31, %zmm0, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm0, %zmm2
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 # 64-byte Folded Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
+; AVX512BW-NEXT:    vpunpcklqdq {{[-0-9]+}}(%r{{[sb]}}p), %zmm11, %zmm2 {%k1} # 64-byte Folded Reload
+; AVX512BW-NEXT:    # zmm2 {%k1} = zmm11[0],mem[0],zmm11[2],mem[2],zmm11[4],mem[4],zmm11[6],mem[6]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm2, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm27, %zmm0, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm0, %zmm7
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm7[0,1,2,3],ymm1[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm22 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm22, %zmm2
+; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 # 64-byte Folded Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm28 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm2 {%k1} = zmm7[0],zmm28[0],zmm7[2],zmm28[2],zmm7[4],zmm28[4],zmm7[6],zmm28[6]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm2, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm0, %zmm3
+; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm0, %zmm4
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3],ymm3[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm25 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
+; AVX512BW-NEXT:    vpermi2q %zmm25, %zmm15, %zmm0
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm0 {%k1} = zmm9[0],zmm10[0],zmm9[2],zmm10[2],zmm9[4],zmm10[4],zmm9[6],zmm10[6]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm19, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm19, %zmm0
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm5 = [7,15,7,15,7,15,7,15]
+; AVX512BW-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-NEXT:    vpermi2q %zmm6, %zmm3, %zmm0
-; AVX512BW-NEXT:    vpermi2q %zmm4, %zmm5, %zmm1
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm2 {%k1} = zmm9[0],zmm5[0],zmm9[2],zmm5[2],zmm9[4],zmm5[4],zmm9[6],zmm5[6]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm2, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm5, %zmm19
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm19 {%k1} = zmm17[1],zmm2[1],zmm17[3],zmm2[3],zmm17[5],zmm2[5],zmm17[7],zmm2[7]
+; AVX512BW-NEXT:    vmovdqu64 %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm17, %zmm6
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm12 = [0,8,0,8,0,8,0,8]
+; AVX512BW-NEXT:    # zmm12 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm12, %zmm17
+; AVX512BW-NEXT:    vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm4
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm2 = [1,9,1,9,1,9,1,9]
+; AVX512BW-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm2, %zmm6
+; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm12, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm2, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm6
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm20
 ; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm0
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm4 = [7,15,7,15,7,15,7,15]
-; AVX512BW-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm23, %zmm2
-; AVX512BW-NEXT:    vpermt2q %zmm23, %zmm4, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm1 {%k1} = zmm24[1],zmm3[1],zmm24[3],zmm3[3],zmm24[5],zmm3[5],zmm24[7],zmm3[7]
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm5, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm1 {%k1} = zmm13[1],zmm11[1],zmm13[3],zmm11[3],zmm13[5],zmm11[5],zmm13[7],zmm11[7]
 ; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm24, %zmm1
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm14 = [0,8,0,8,0,8,0,8]
-; AVX512BW-NEXT:    # zmm14 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm14, %zmm24
-; AVX512BW-NEXT:    vmovdqu64 %zmm24, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm23 = [1,9,1,9,1,9,1,9]
-; AVX512BW-NEXT:    # zmm23 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm23, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm12, %zmm13
+; AVX512BW-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm2, %zmm1
 ; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm14, %zmm6
-; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm23, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm12, %zmm20
+; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm2, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm24
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm1
-; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm4, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm2 {%k1} = zmm15[1],zmm28[1],zmm15[3],zmm28[3],zmm15[5],zmm28[5],zmm15[7],zmm28[7]
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm6
-; AVX512BW-NEXT:    vpermt2q %zmm28, %zmm14, %zmm15
-; AVX512BW-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm28, %zmm23, %zmm6
-; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm14, %zmm24
-; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm23, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm17
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm5, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm26, %zmm10
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm1 {%k1} = zmm30[1],zmm26[1],zmm30[3],zmm26[3],zmm30[5],zmm26[5],zmm30[7],zmm26[7]
 ; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm19, %zmm29
-; AVX512BW-NEXT:    vmovdqa64 %zmm19, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm4, %zmm19
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm19 {%k1} = zmm30[1],zmm27[1],zmm30[3],zmm27[3],zmm30[5],zmm27[5],zmm30[7],zmm27[7]
 ; AVX512BW-NEXT:    vmovdqa64 %zmm30, %zmm1
-; AVX512BW-NEXT:    vpermt2q %zmm27, %zmm14, %zmm30
-; AVX512BW-NEXT:    vmovdqu64 %zmm30, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm27, %zmm23, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm30, %zmm26
+; AVX512BW-NEXT:    vpermt2q %zmm10, %zmm12, %zmm26
+; AVX512BW-NEXT:    vpermt2q %zmm10, %zmm2, %zmm1
 ; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm14, %zmm29
-; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm23, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm12, %zmm17
+; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm2, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm20, %zmm28
-; AVX512BW-NEXT:    vmovdqa64 %zmm20, %zmm3
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm4, %zmm20
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm20 {%k1} = zmm31[1],zmm16[1],zmm31[3],zmm16[3],zmm31[5],zmm16[5],zmm31[7],zmm16[7]
-; AVX512BW-NEXT:    vmovdqu64 %zmm20, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm31, %zmm2
-; AVX512BW-NEXT:    vmovdqa64 %zmm31, %zmm17
-; AVX512BW-NEXT:    vpermt2q %zmm16, %zmm14, %zmm17
-; AVX512BW-NEXT:    vpermt2q %zmm16, %zmm23, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm14, %zmm28
-; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm23, %zmm3
-; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm21
-; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm31
-; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm4, %zmm11
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm11 {%k1} = zmm3[1],zmm0[1],zmm3[3],zmm0[3],zmm3[5],zmm0[5],zmm3[7],zmm0[7]
-; AVX512BW-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm2
-; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm14, %zmm3
-; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm23, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm14, %zmm21
-; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm23, %zmm31
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm2
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm20
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm25
-; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm4, %zmm2
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm2 {%k1} = zmm26[1],zmm8[1],zmm26[3],zmm8[3],zmm26[5],zmm8[5],zmm26[7],zmm8[7]
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, (%rsp) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm26, %zmm2
-; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm14, %zmm26
-; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm23, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm14, %zmm20
-; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm23, %zmm25
+; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm19
+; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm31
+; AVX512BW-NEXT:    vmovdqa64 %zmm29, %zmm3
+; AVX512BW-NEXT:    vpermt2q %zmm29, %zmm5, %zmm16
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm14, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm30 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm30, %zmm16
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm27 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm27, %zmm14, %zmm16
-; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm14, %zmm0
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm16 {%k1} = zmm24[1],zmm6[1],zmm24[3],zmm6[3],zmm24[5],zmm6[5],zmm24[7],zmm6[7]
+; AVX512BW-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm24, %zmm0
+; AVX512BW-NEXT:    vmovdqa64 %zmm24, %zmm29
+; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm12, %zmm29
+; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm2, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vpermi2q %zmm18, %zmm0, %zmm14
-; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm2
-; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm23, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm30, %zmm22
-; AVX512BW-NEXT:    vpermt2q %zmm27, %zmm23, %zmm22
-; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm2
-; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm23, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermi2q %zmm18, %zmm0, %zmm23
-; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm4, %zmm0
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm0 {%k1} = zmm9[1],zmm5[1],zmm9[3],zmm5[3],zmm9[5],zmm5[5],zmm9[7],zmm5[7]
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm18
-; AVX512BW-NEXT:    vpermt2q %zmm27, %zmm4, %zmm30
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm30 {%k1} = zmm6[1],zmm3[1],zmm6[3],zmm3[3],zmm6[5],zmm3[5],zmm6[7],zmm3[7]
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm7 # 64-byte Folded Reload
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm12, %zmm19
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm2, %zmm31
+; AVX512BW-NEXT:    vmovdqu64 (%rsp), %zmm1 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm16
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm27
+; AVX512BW-NEXT:    vpermt2q %zmm21, %zmm5, %zmm1
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm1 {%k1} = zmm23[1],zmm18[1],zmm23[3],zmm18[3],zmm23[5],zmm18[5],zmm23[7],zmm18[7]
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, (%rsp) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm23, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm23, %zmm24
+; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm12, %zmm24
+; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm2, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm21, %zmm12, %zmm16
+; AVX512BW-NEXT:    vpermt2q %zmm21, %zmm2, %zmm27
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm18
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm30
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm8 # 64-byte Folded Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm10 # 64-byte Folded Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm11 # 64-byte Folded Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm12 # 64-byte Folded Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm13 # 64-byte Folded Reload
+; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm5, %zmm1
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm0 # 64-byte Folded Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm1 {%k1} = zmm10[1],zmm0[1],zmm10[3],zmm0[3],zmm10[5],zmm0[5],zmm10[7],zmm0[7]
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm12, %zmm10
+; AVX512BW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm2, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm12, %zmm18
+; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm2, %zmm30
+; AVX512BW-NEXT:    vmovdqa64 %zmm22, %zmm6
+; AVX512BW-NEXT:    vmovdqa64 %zmm22, %zmm23
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm5, %zmm6
+; AVX512BW-NEXT:    vmovdqa64 %zmm28, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm21 # 64-byte Reload
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm6 {%k1} = zmm21[1],zmm28[1],zmm21[3],zmm28[3],zmm21[5],zmm28[5],zmm21[7],zmm28[7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm28
+; AVX512BW-NEXT:    vmovdqa64 %zmm21, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm12, %zmm21
+; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm2, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm12, %zmm23
+; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm2, %zmm22
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm12, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm15 # 64-byte Folded Reload
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = [7,15,7,15]
-; AVX512BW-NEXT:    # ymm0 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm9 # 64-byte Folded Reload
+; AVX512BW-NEXT:    vpermi2q %zmm25, %zmm15, %zmm12
+; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm3
+; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm2, %zmm3
+; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermi2q %zmm25, %zmm15, %zmm2
+; AVX512BW-NEXT:    vpermt2q %zmm25, %zmm5, %zmm15
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm15 {%k1} = zmm14[1],zmm8[1],zmm14[3],zmm8[3],zmm14[5],zmm8[5],zmm14[7],zmm8[7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm25
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm7 # 64-byte Folded Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm0 # 64-byte Folded Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm8 # 64-byte Folded Reload
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm1 # 64-byte Folded Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 # 64-byte Folded Reload
+; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm1 # 64-byte Folded Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm13 # 64-byte Folded Reload
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm3 # 64-byte Folded Reload
+; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm3 # 64-byte Folded Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm9 # 64-byte Folded Reload
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm4 # 64-byte Folded Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm5 # 64-byte Folded Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm6 # 64-byte Folded Reload
-; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm4 # 64-byte Folded Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm11 # 64-byte Folded Reload
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm6 # 64-byte Folded Reload
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm7[0,1,2,3],ymm9[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm7, %zmm0
+; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm6 # 64-byte Folded Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm15 # 64-byte Folded Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm10 # 64-byte Folded Reload
+; AVX512BW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm10 # 64-byte Folded Reload
+; AVX512BW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm10 # 64-byte Folded Reload
+; AVX512BW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm10 # 64-byte Folded Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm14 # 64-byte Folded Reload
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm7[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm5, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm8[0,1,2,3],ymm1[4,5,6,7]
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm8[4,5,6,7]
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
 ; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm1, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm10[0,1,2,3],ymm2[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm19, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm11[0,1,2,3],ymm3[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm1, %zmm11
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm12[0,1,2,3],ymm4[4,5,6,7]
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3],ymm13[4,5,6,7]
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
 ; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm1, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm13[0,1,2,3],ymm5[4,5,6,7]
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3],ymm9[4,5,6,7]
 ; AVX512BW-NEXT:    vmovdqu64 (%rsp), %zmm1 # 64-byte Reload
 ; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm1, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, (%rsp) # 64-byte Spill
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm6[0,1,2,3],ymm11[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm1, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpblendd $15, {{[-0-9]+}}(%r{{[sb]}}p), %ymm15, %ymm0 # 32-byte Folded Reload
+; AVX512BW-NEXT:    # ymm0 = mem[0,1,2,3],ymm15[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm28, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
 ; AVX512BW-NEXT:    # ymm0 = ymm0[0,1,2,3],mem[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm18, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm15[0,1,2,3],ymm6[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm30, %zmm0
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm25, %zmm25
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm14[0,1,2,3],ymm10[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm1, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm24 {%k1}
-; AVX512BW-NEXT:    vmovdqa 64(%rdi), %xmm0
-; AVX512BW-NEXT:    vinserti128 $1, 192(%rdi), %ymm0, %ymm0
-; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm15
-; AVX512BW-NEXT:    vinserti128 $1, 128(%rdi), %ymm15, %ymm15
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm30 = ymm15[0],ymm0[0],ymm15[2],ymm0[2]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm30, %zmm24, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm26, %zmm17 {%k1}
+; AVX512BW-NEXT:    vmovdqa 64(%rdi), %xmm8
+; AVX512BW-NEXT:    vinserti128 $1, 192(%rdi), %ymm8, %ymm8
+; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512BW-NEXT:    vinserti128 $1, 128(%rdi), %ymm0, %ymm0
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm28 = ymm0[0],ymm8[0],ymm0[2],ymm8[2]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm28, %zmm17, %zmm26
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm1 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 576(%rdi), %xmm24
-; AVX512BW-NEXT:    vinserti32x4 $1, 704(%rdi), %ymm24, %ymm30
-; AVX512BW-NEXT:    vmovdqa64 512(%rdi), %xmm24
-; AVX512BW-NEXT:    vinserti32x4 $1, 640(%rdi), %ymm24, %ymm24
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm27 = ymm24[0],ymm30[0],ymm24[2],ymm30[2]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm27, %zmm1, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm17, %zmm28 {%k1}
-; AVX512BW-NEXT:    vmovdqa 1088(%rdi), %xmm9
-; AVX512BW-NEXT:    vinserti32x4 $1, 1216(%rdi), %ymm9, %ymm27
-; AVX512BW-NEXT:    vmovdqa 1024(%rdi), %xmm9
-; AVX512BW-NEXT:    vinserti128 $1, 1152(%rdi), %ymm9, %ymm9
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm4 = ymm9[0],ymm27[0],ymm9[2],ymm27[2]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm4, %zmm28, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm20 {%k1}
+; AVX512BW-NEXT:    vmovdqa 576(%rdi), %xmm13
+; AVX512BW-NEXT:    vinserti128 $1, 704(%rdi), %ymm13, %ymm13
+; AVX512BW-NEXT:    vmovdqa64 512(%rdi), %xmm28
+; AVX512BW-NEXT:    vinserti32x4 $1, 640(%rdi), %ymm28, %ymm28
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm4 = ymm28[0],ymm13[0],ymm28[2],ymm13[2]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm4, %zmm20, %zmm1
 ; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm24, %zmm16 {%k1}
+; AVX512BW-NEXT:    vmovdqa 1088(%rdi), %xmm4
+; AVX512BW-NEXT:    vinserti128 $1, 1216(%rdi), %ymm4, %ymm4
+; AVX512BW-NEXT:    vmovdqa 1024(%rdi), %xmm7
+; AVX512BW-NEXT:    vinserti128 $1, 1152(%rdi), %ymm7, %ymm7
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm5 = ymm7[0],ymm4[0],ymm7[2],ymm4[2]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm5, %zmm16, %zmm24
+; AVX512BW-NEXT:    vmovdqa64 %zmm29, %zmm19 {%k1}
+; AVX512BW-NEXT:    vmovdqa 1600(%rdi), %xmm5
+; AVX512BW-NEXT:    vinserti128 $1, 1728(%rdi), %ymm5, %ymm5
+; AVX512BW-NEXT:    vmovdqa 1536(%rdi), %xmm11
+; AVX512BW-NEXT:    vinserti128 $1, 1664(%rdi), %ymm11, %ymm11
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm29 = ymm11[0],ymm5[0],ymm11[2],ymm5[2]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm29, %zmm19, %zmm19
+; AVX512BW-NEXT:    vmovdqa64 %zmm21, %zmm23 {%k1}
+; AVX512BW-NEXT:    vmovdqa 2112(%rdi), %xmm6
+; AVX512BW-NEXT:    vinserti128 $1, 2240(%rdi), %ymm6, %ymm6
+; AVX512BW-NEXT:    vmovdqa64 2048(%rdi), %xmm29
+; AVX512BW-NEXT:    vinserti32x4 $1, 2176(%rdi), %ymm29, %ymm29
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm3 = ymm29[0],ymm6[0],ymm29[2],ymm6[2]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm3, %zmm23, %zmm3
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm29 {%k1}
-; AVX512BW-NEXT:    vmovdqa 1600(%rdi), %xmm4
-; AVX512BW-NEXT:    vinserti128 $1, 1728(%rdi), %ymm4, %ymm4
-; AVX512BW-NEXT:    vmovdqa64 1536(%rdi), %xmm17
-; AVX512BW-NEXT:    vinserti32x4 $1, 1664(%rdi), %ymm17, %ymm17
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm19 = ymm17[0],ymm4[0],ymm17[2],ymm4[2]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm19, %zmm29, %zmm13
-; AVX512BW-NEXT:    vmovdqa64 %zmm26, %zmm20 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 2112(%rdi), %xmm19
-; AVX512BW-NEXT:    vinserti32x4 $1, 2240(%rdi), %ymm19, %ymm19
-; AVX512BW-NEXT:    vmovdqa64 2048(%rdi), %xmm26
-; AVX512BW-NEXT:    vinserti32x4 $1, 2176(%rdi), %ymm26, %ymm26
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm29 = ymm26[0],ymm19[0],ymm26[2],ymm19[2]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm29, %zmm20, %zmm5
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm21 {%k1}
-; AVX512BW-NEXT:    vmovdqa 2624(%rdi), %xmm12
-; AVX512BW-NEXT:    vinserti128 $1, 2752(%rdi), %ymm12, %ymm12
-; AVX512BW-NEXT:    vmovdqa64 2560(%rdi), %xmm29
-; AVX512BW-NEXT:    vinserti32x4 $1, 2688(%rdi), %ymm29, %ymm29
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm18 = ymm29[0],ymm12[0],ymm29[2],ymm12[2]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm18, %zmm21, %zmm18
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm18 {%k1}
+; AVX512BW-NEXT:    vmovdqa 2624(%rdi), %xmm10
+; AVX512BW-NEXT:    vinserti128 $1, 2752(%rdi), %ymm10, %ymm10
+; AVX512BW-NEXT:    vmovdqa64 2560(%rdi), %xmm23
+; AVX512BW-NEXT:    vinserti32x4 $1, 2688(%rdi), %ymm23, %ymm23
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm20 = ymm23[0],ymm10[0],ymm23[2],ymm10[2]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm20, %zmm18, %zmm18
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm16 {%k1}
-; AVX512BW-NEXT:    vmovdqa 3136(%rdi), %xmm6
-; AVX512BW-NEXT:    vinserti128 $1, 3264(%rdi), %ymm6, %ymm6
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm1 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 3136(%rdi), %xmm17
+; AVX512BW-NEXT:    vinserti32x4 $1, 3264(%rdi), %ymm17, %ymm17
 ; AVX512BW-NEXT:    vmovdqa64 3072(%rdi), %xmm20
 ; AVX512BW-NEXT:    vinserti32x4 $1, 3200(%rdi), %ymm20, %ymm20
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm21 = ymm20[0],ymm6[0],ymm20[2],ymm6[2]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm21, %zmm16, %zmm2
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm15 = ymm20[0],ymm17[0],ymm20[2],ymm17[2]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm15, %zmm1, %zmm15
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm14 {%k1}
-; AVX512BW-NEXT:    vmovdqa 3648(%rdi), %xmm7
-; AVX512BW-NEXT:    vinserti128 $1, 3776(%rdi), %ymm7, %ymm7
-; AVX512BW-NEXT:    vmovdqa64 3584(%rdi), %xmm21
-; AVX512BW-NEXT:    vinserti32x4 $1, 3712(%rdi), %ymm21, %ymm21
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm28 = ymm21[0],ymm7[0],ymm21[2],ymm7[2]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm28, %zmm14, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm22 {%k1}
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm6 = ymm20[1],ymm6[1],ymm20[3],ymm6[3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm6, %zmm22, %zmm3
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm12 {%k1}
+; AVX512BW-NEXT:    vmovdqa 3648(%rdi), %xmm9
+; AVX512BW-NEXT:    vinserti128 $1, 3776(%rdi), %ymm9, %ymm9
+; AVX512BW-NEXT:    vmovdqa 3584(%rdi), %xmm14
+; AVX512BW-NEXT:    vinserti128 $1, 3712(%rdi), %ymm14, %ymm14
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm16 = ymm14[0],ymm9[0],ymm14[2],ymm9[2]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm16, %zmm12, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm12 {%k1}
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm16 = ymm20[1],ymm17[1],ymm20[3],ymm17[3]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm16, %zmm12, %zmm16
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm17, %zmm12 {%k1}
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm13 = ymm28[1],ymm13[1],ymm28[3],ymm13[3]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm13, %zmm12, %zmm13
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm17, %zmm12 {%k1}
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm0 = ymm0[1],ymm8[1],ymm0[3],ymm8[3]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm12, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm8 {%k1}
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm6 = ymm24[1],ymm30[1],ymm24[3],ymm30[3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm6, %zmm8, %zmm6
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm31 {%k1}
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm5 = ymm11[1],ymm5[1],ymm11[3],ymm5[3]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm5, %zmm31, %zmm5
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm8 {%k1}
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm0 = ymm15[1],ymm0[1],ymm15[3],ymm0[3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm8, %zmm0
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm27 {%k1}
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm4 = ymm7[1],ymm4[1],ymm7[3],ymm4[3]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm4, %zmm27, %zmm4
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm30 {%k1}
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm7 = ymm23[1],ymm10[1],ymm23[3],ymm10[3]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm7, %zmm30, %zmm7
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm8 {%k1}
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm4 = ymm17[1],ymm4[1],ymm17[3],ymm4[3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm4, %zmm8, %zmm4
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm22 {%k1}
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm6 = ymm29[1],ymm6[1],ymm29[3],ymm6[3]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm6, %zmm22, %zmm6
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm10 {%k1}
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm8 = ymm9[1],ymm27[1],ymm9[3],ymm27[3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm8, %zmm10, %zmm8
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm31 {%k1}
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm9 = ymm29[1],ymm12[1],ymm29[3],ymm12[3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm9, %zmm31, %zmm9
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm25 {%k1}
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm10 = ymm26[1],ymm19[1],ymm26[3],ymm19[3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm10, %zmm25, %zmm10
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm23 {%k1}
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm7 = ymm21[1],ymm7[1],ymm21[3],ymm7[3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm7, %zmm23, %zmm7
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm2 {%k1}
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm8 = ymm14[1],ymm9[1],ymm14[3],ymm9[3]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm8, %zmm2, %zmm2
 ; AVX512BW-NEXT:    vmovdqa64 %zmm1, 448(%rsi)
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, 384(%rsi)
+; AVX512BW-NEXT:    vmovdqa64 %zmm15, 384(%rsi)
 ; AVX512BW-NEXT:    vmovdqa64 %zmm18, 320(%rsi)
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, 256(%rsi)
-; AVX512BW-NEXT:    vmovdqa64 %zmm13, 192(%rsi)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm1, 128(%rsi)
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, 256(%rsi)
+; AVX512BW-NEXT:    vmovdqa64 %zmm19, 192(%rsi)
+; AVX512BW-NEXT:    vmovdqa64 %zmm24, 128(%rsi)
 ; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
 ; AVX512BW-NEXT:    vmovaps %zmm1, 64(%rsi)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm1, (%rsi)
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, 448(%rdx)
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, 256(%rdx)
-; AVX512BW-NEXT:    vmovdqa64 %zmm9, 320(%rdx)
-; AVX512BW-NEXT:    vmovdqa64 %zmm8, 128(%rdx)
-; AVX512BW-NEXT:    vmovdqa64 %zmm4, 192(%rdx)
+; AVX512BW-NEXT:    vmovdqa64 %zmm26, (%rsi)
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, 448(%rdx)
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, 256(%rdx)
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, 320(%rdx)
+; AVX512BW-NEXT:    vmovdqa64 %zmm4, 128(%rdx)
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, 192(%rdx)
 ; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rdx)
-; AVX512BW-NEXT:    vmovdqa64 %zmm6, 64(%rdx)
-; AVX512BW-NEXT:    vmovdqa64 %zmm3, 384(%rdx)
+; AVX512BW-NEXT:    vmovdqa64 %zmm13, 64(%rdx)
+; AVX512BW-NEXT:    vmovdqa64 %zmm16, 384(%rdx)
 ; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-NEXT:    vmovaps %zmm0, 448(%rcx)
 ; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
@@ -11258,20 +11256,20 @@ define void @load_i64_stride8_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
 ; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-NEXT:    vmovaps %zmm0, 384(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm25, 448(%rax)
 ; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm0, 448(%rax)
-; AVX512BW-NEXT:    vmovups (%rsp), %zmm0 # 64-byte Reload
 ; AVX512BW-NEXT:    vmovaps %zmm0, 256(%rax)
 ; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-NEXT:    vmovaps %zmm0, 320(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm11, 128(%rax)
+; AVX512BW-NEXT:    vmovups (%rsp), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm0, 128(%rax)
 ; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-NEXT:    vmovaps %zmm0, 192(%rax)
 ; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-NEXT:    vmovaps %zmm0, (%rax)
 ; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-NEXT:    vmovaps %zmm0, 64(%rax)
-; AVX512BW-NEXT:    addq $6600, %rsp # imm = 0x19C8
+; AVX512BW-NEXT:    addq $6664, %rsp # imm = 0x1A08
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
   %wide.vec = load <512 x i64>, ptr %in.vec, align 64
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-5.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-5.ll
index 4d5c8fe891e810b138dbe54e90d951e4b6e06d25..a060d29200ba85d5dffde1120089bad332d1bcc6 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-5.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-5.ll
@@ -1779,97 +1779,101 @@ define void @load_i8_stride5_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ;
 ; AVX512F-LABEL: load_i8_stride5_vf32:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm0 = [65535,0,65535,65535,0,65535,0,65535,65535,0,65535,0,65535,65535,0,65535]
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm2 = [65535,65535,0,65535,0,65535,65535,0,65535,0,65535,65535,0,65535,0,65535]
 ; AVX512F-NEXT:    vmovdqa (%rdi), %ymm3
-; AVX512F-NEXT:    vmovdqa 32(%rdi), %ymm4
-; AVX512F-NEXT:    vmovdqa 64(%rdi), %ymm1
-; AVX512F-NEXT:    vmovdqa 96(%rdi), %ymm2
-; AVX512F-NEXT:    vmovdqa %ymm0, %ymm5
-; AVX512F-NEXT:    vpternlogq $202, %ymm4, %ymm3, %ymm5
-; AVX512F-NEXT:    vextracti128 $1, %ymm5, %xmm6
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm6 = zero,zero,zero,zero,xmm6[4,9,14],zero,zero,zero,xmm6[2,7,12,u,u,u]
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm5 = xmm5[0,5,10,15],zero,zero,zero,xmm5[3,8,13],zero,zero,zero,xmm5[u,u,u]
-; AVX512F-NEXT:    vpor %xmm6, %xmm5, %xmm6
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm5 = [65535,65535,0,65535,0,65535,65535,0,65535,0,65535,65535,0,65535,0,65535]
-; AVX512F-NEXT:    vmovdqa %ymm5, %ymm7
-; AVX512F-NEXT:    vpternlogq $202, %ymm2, %ymm1, %ymm7
-; AVX512F-NEXT:    vpermq {{.*#+}} ymm8 = ymm7[2,3,0,1]
-; AVX512F-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm7, %ymm8
-; AVX512F-NEXT:    vpshufb {{.*#+}} ymm9 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm8[1,6,11,16,21,26,31,20,25,30,19,24,29],zero,zero,zero,zero,zero,zero
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm10 = [255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255]
-; AVX512F-NEXT:    vpternlogq $248, %ymm10, %ymm6, %ymm9
+; AVX512F-NEXT:    vmovdqa 32(%rdi), %ymm5
+; AVX512F-NEXT:    vmovdqa 64(%rdi), %ymm0
+; AVX512F-NEXT:    vmovdqa 96(%rdi), %ymm1
+; AVX512F-NEXT:    vmovdqa %ymm2, %ymm4
+; AVX512F-NEXT:    vpternlogq $202, %ymm1, %ymm0, %ymm4
+; AVX512F-NEXT:    vpermq {{.*#+}} ymm6 = ymm4[2,3,0,1]
+; AVX512F-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm4, %ymm6
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm8 = [128,128,128,128,128,128,128,128,128,128,128,128,128,1,6,11,16,21,26,31,20,25,30,19,24,29,128,128,128,128,128,128]
+; AVX512F-NEXT:    vpshufb %ymm8, %ymm6, %ymm6
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm4 = [65535,0,65535,65535,0,65535,0,65535,65535,0,65535,0,65535,65535,0,65535]
+; AVX512F-NEXT:    vmovdqa %ymm4, %ymm7
+; AVX512F-NEXT:    vpternlogq $202, %ymm5, %ymm3, %ymm7
+; AVX512F-NEXT:    vextracti128 $1, %ymm7, %xmm9
+; AVX512F-NEXT:    vpshufb {{.*#+}} xmm9 = zero,zero,zero,zero,xmm9[4,9,14],zero,zero,zero,xmm9[2,7,12,u,u,u]
+; AVX512F-NEXT:    vpshufb {{.*#+}} xmm7 = xmm7[0,5,10,15],zero,zero,zero,xmm7[3,8,13],zero,zero,zero,xmm7[u,u,u]
+; AVX512F-NEXT:    vpor %xmm7, %xmm9, %xmm9
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm11 = [255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255]
+; AVX512F-NEXT:    vpternlogq $236, %ymm11, %ymm6, %ymm9
 ; AVX512F-NEXT:    vmovdqa 144(%rdi), %xmm7
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm6 = xmm7[u,u,u,u,u,u,u,u,u,u],zero,zero,zero,xmm7[1,6,11]
+; AVX512F-NEXT:    vpshufb %xmm8, %xmm7, %xmm6
 ; AVX512F-NEXT:    vmovdqa 128(%rdi), %xmm8
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm11 = xmm8[u,u,u,u,u,u,u,u,u,u,2,7,12],zero,zero,zero
-; AVX512F-NEXT:    vpor %xmm6, %xmm11, %xmm6
+; AVX512F-NEXT:    vpshufb {{.*#+}} xmm10 = xmm8[u,u,u,u,u,u,u,u,u,u,2,7,12],zero,zero,zero
+; AVX512F-NEXT:    vpor %xmm6, %xmm10, %xmm6
 ; AVX512F-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
 ; AVX512F-NEXT:    vpblendw {{.*#+}} ymm6 = ymm9[0,1,2,3,4],ymm6[5,6,7],ymm9[8,9,10,11,12],ymm6[13,14,15]
 ; AVX512F-NEXT:    vpblendd {{.*#+}} ymm6 = ymm9[0,1,2,3],ymm6[4,5,6,7]
-; AVX512F-NEXT:    vmovdqa %ymm5, %ymm9
-; AVX512F-NEXT:    vpternlogq $202, %ymm4, %ymm3, %ymm9
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm11 = xmm9[1,6,11],zero,zero,zero,zero,xmm9[4,9,14],zero,zero,zero,xmm9[u,u,u]
-; AVX512F-NEXT:    vextracti128 $1, %ymm9, %xmm9
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm9 = zero,zero,zero,xmm9[0,5,10,15],zero,zero,zero,xmm9[3,8,13,u,u,u]
-; AVX512F-NEXT:    vpor %xmm11, %xmm9, %xmm9
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm11 = [65535,0,65535,0,65535,65535,0,65535,0,65535,65535,0,65535,0,65535,65535]
-; AVX512F-NEXT:    vmovdqa %ymm11, %ymm12
-; AVX512F-NEXT:    vpternlogq $202, %ymm1, %ymm2, %ymm12
-; AVX512F-NEXT:    vpermq {{.*#+}} ymm13 = ymm12[2,3,0,1]
-; AVX512F-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm12, %ymm13
-; AVX512F-NEXT:    vpshufb {{.*#+}} ymm12 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm13[2,7,12,17,22,27,16,21,26,31,20,25,30],zero,zero,zero,zero,zero,zero
-; AVX512F-NEXT:    vpternlogq $248, %ymm10, %ymm9, %ymm12
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm9 = xmm7[u,u,u,u,u,u,u,u,u,u],zero,zero,zero,xmm7[2,7,12]
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm13 = xmm8[u,u,u,u,u,u,u,u,u,u,3,8,13],zero,zero,zero
-; AVX512F-NEXT:    vpor %xmm9, %xmm13, %xmm9
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm10 = [65535,0,65535,0,65535,65535,0,65535,0,65535,65535,0,65535,0,65535,65535]
+; AVX512F-NEXT:    vmovdqa %ymm10, %ymm9
+; AVX512F-NEXT:    vpternlogq $202, %ymm0, %ymm1, %ymm9
+; AVX512F-NEXT:    vpermq {{.*#+}} ymm12 = ymm9[2,3,0,1]
+; AVX512F-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm9, %ymm12
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm9 = [128,128,128,128,128,128,128,128,128,128,128,128,128,2,7,12,17,22,27,16,21,26,31,20,25,30,128,128,128,128,128,128]
+; AVX512F-NEXT:    vpshufb %ymm9, %ymm12, %ymm12
+; AVX512F-NEXT:    vmovdqa %ymm2, %ymm13
+; AVX512F-NEXT:    vpternlogq $202, %ymm5, %ymm3, %ymm13
+; AVX512F-NEXT:    vpshufb {{.*#+}} xmm14 = xmm13[1,6,11],zero,zero,zero,zero,xmm13[4,9,14],zero,zero,zero,xmm13[u,u,u]
+; AVX512F-NEXT:    vextracti128 $1, %ymm13, %xmm13
+; AVX512F-NEXT:    vpshufb {{.*#+}} xmm13 = zero,zero,zero,xmm13[0,5,10,15],zero,zero,zero,xmm13[3,8,13,u,u,u]
+; AVX512F-NEXT:    vpor %xmm14, %xmm13, %xmm13
+; AVX512F-NEXT:    vpternlogq $236, %ymm11, %ymm12, %ymm13
+; AVX512F-NEXT:    vpshufb %xmm9, %xmm7, %xmm9
+; AVX512F-NEXT:    vpshufb {{.*#+}} xmm12 = xmm8[u,u,u,u,u,u,u,u,u,u,3,8,13],zero,zero,zero
+; AVX512F-NEXT:    vpor %xmm9, %xmm12, %xmm9
 ; AVX512F-NEXT:    vinserti128 $1, %xmm9, %ymm0, %ymm9
-; AVX512F-NEXT:    vpblendw {{.*#+}} ymm9 = ymm12[0,1,2,3,4],ymm9[5,6,7],ymm12[8,9,10,11,12],ymm9[13,14,15]
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm9 = ymm12[0,1,2,3],ymm9[4,5,6,7]
-; AVX512F-NEXT:    vpternlogq $202, %ymm3, %ymm4, %ymm11
-; AVX512F-NEXT:    vextracti128 $1, %ymm11, %xmm12
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm12 = zero,zero,zero,xmm12[1,6,11],zero,zero,zero,zero,xmm12[4,9,14,u,u,u]
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm11 = xmm11[2,7,12],zero,zero,zero,xmm11[0,5,10,15],zero,zero,zero,xmm11[u,u,u]
-; AVX512F-NEXT:    vpor %xmm12, %xmm11, %xmm11
-; AVX512F-NEXT:    vmovdqa %ymm0, %ymm12
-; AVX512F-NEXT:    vpternlogq $202, %ymm1, %ymm2, %ymm12
-; AVX512F-NEXT:    vpermq {{.*#+}} ymm13 = ymm12[2,3,0,1]
-; AVX512F-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm12, %ymm13
-; AVX512F-NEXT:    vpshufb {{.*#+}} ymm12 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm13[3,8,13,18,23,28,17,22,27,16,21,26,31],zero,zero,zero,zero,zero,zero
-; AVX512F-NEXT:    vpternlogq $248, %ymm10, %ymm11, %ymm12
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm10 = xmm7[u,u,u,u,u,u,u,u,u,u],zero,zero,zero,xmm7[3,8,13]
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm11 = xmm8[u,u,u,u,u,u,u,u,u,u,4,9,14],zero,zero,zero
-; AVX512F-NEXT:    vpor %xmm10, %xmm11, %xmm10
-; AVX512F-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
-; AVX512F-NEXT:    vpblendw {{.*#+}} ymm10 = ymm12[0,1,2,3,4],ymm10[5,6,7],ymm12[8,9,10,11,12],ymm10[13,14,15]
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm10 = ymm12[0,1,2,3],ymm10[4,5,6,7]
-; AVX512F-NEXT:    vmovdqa %ymm0, %ymm11
-; AVX512F-NEXT:    vpternlogq $202, %ymm3, %ymm4, %ymm11
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm12 = xmm11[3,8,13],zero,zero,zero,xmm11[1,6,11],zero,zero,zero,zero,xmm11[u,u,u]
-; AVX512F-NEXT:    vextracti128 $1, %ymm11, %xmm11
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm11 = zero,zero,zero,xmm11[2,7,12],zero,zero,zero,xmm11[0,5,10,15,u,u,u]
-; AVX512F-NEXT:    vpor %xmm12, %xmm11, %xmm11
-; AVX512F-NEXT:    vmovdqa %ymm5, %ymm12
-; AVX512F-NEXT:    vpternlogq $202, %ymm1, %ymm2, %ymm12
+; AVX512F-NEXT:    vpblendw {{.*#+}} ymm9 = ymm13[0,1,2,3,4],ymm9[5,6,7],ymm13[8,9,10,11,12],ymm9[13,14,15]
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm9 = ymm13[0,1,2,3],ymm9[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa %ymm4, %ymm12
+; AVX512F-NEXT:    vpternlogq $202, %ymm0, %ymm1, %ymm12
 ; AVX512F-NEXT:    vpermq {{.*#+}} ymm13 = ymm12[2,3,0,1]
 ; AVX512F-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm12, %ymm13
-; AVX512F-NEXT:    vpshufb {{.*#+}} ymm12 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm13[4,9,14,19,24,29,18,23,28,17,22,27,u,u,u,u,u,u,u]
-; AVX512F-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm11, %ymm12
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm7 = xmm7[u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,xmm7[4,9,14]
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm12 = [128,128,128,128,128,128,128,128,128,128,128,128,128,3,8,13,18,23,28,17,22,27,16,21,26,31,128,128,128,128,128,128]
+; AVX512F-NEXT:    vpshufb %ymm12, %ymm13, %ymm13
+; AVX512F-NEXT:    vpternlogq $202, %ymm3, %ymm5, %ymm10
+; AVX512F-NEXT:    vextracti128 $1, %ymm10, %xmm14
+; AVX512F-NEXT:    vpshufb {{.*#+}} xmm14 = zero,zero,zero,xmm14[1,6,11],zero,zero,zero,zero,xmm14[4,9,14,u,u,u]
+; AVX512F-NEXT:    vpshufb {{.*#+}} xmm10 = xmm10[2,7,12],zero,zero,zero,xmm10[0,5,10,15],zero,zero,zero,xmm10[u,u,u]
+; AVX512F-NEXT:    vpor %xmm14, %xmm10, %xmm10
+; AVX512F-NEXT:    vpternlogq $236, %ymm11, %ymm13, %ymm10
+; AVX512F-NEXT:    vpshufb %xmm12, %xmm7, %xmm11
+; AVX512F-NEXT:    vpshufb {{.*#+}} xmm12 = xmm8[u,u,u,u,u,u,u,u,u,u,4,9,14],zero,zero,zero
+; AVX512F-NEXT:    vpor %xmm11, %xmm12, %xmm11
+; AVX512F-NEXT:    vinserti128 $1, %xmm11, %ymm0, %ymm11
+; AVX512F-NEXT:    vpblendw {{.*#+}} ymm11 = ymm10[0,1,2,3,4],ymm11[5,6,7],ymm10[8,9,10,11,12],ymm11[13,14,15]
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm10 = ymm10[0,1,2,3],ymm11[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa %ymm2, %ymm11
+; AVX512F-NEXT:    vpternlogq $202, %ymm0, %ymm1, %ymm11
+; AVX512F-NEXT:    vpermq {{.*#+}} ymm12 = ymm11[2,3,0,1]
+; AVX512F-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm11, %ymm12
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm11 = <128,128,128,128,128,128,128,128,128,128,128,128,128,4,9,14,19,24,29,18,23,28,17,22,27,u,u,u,u,u,u,u>
+; AVX512F-NEXT:    vpshufb %ymm11, %ymm12, %ymm12
+; AVX512F-NEXT:    vmovdqa %ymm4, %ymm13
+; AVX512F-NEXT:    vpternlogq $202, %ymm3, %ymm5, %ymm13
+; AVX512F-NEXT:    vpshufb {{.*#+}} xmm14 = xmm13[3,8,13],zero,zero,zero,xmm13[1,6,11],zero,zero,zero,zero,xmm13[u,u,u]
+; AVX512F-NEXT:    vextracti128 $1, %ymm13, %xmm13
+; AVX512F-NEXT:    vpshufb {{.*#+}} xmm13 = zero,zero,zero,xmm13[2,7,12],zero,zero,zero,xmm13[0,5,10,15,u,u,u]
+; AVX512F-NEXT:    vpor %xmm14, %xmm13, %xmm13
+; AVX512F-NEXT:    vpternlogq $236, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm12, %ymm13
+; AVX512F-NEXT:    vpshufb %xmm11, %xmm7, %xmm7
 ; AVX512F-NEXT:    vpshufb {{.*#+}} xmm8 = xmm8[u,u,u,u,u,u,u,u,u,0,5,10,15],zero,zero,zero
 ; AVX512F-NEXT:    vpor %xmm7, %xmm8, %xmm7
 ; AVX512F-NEXT:    vinserti128 $1, %xmm7, %ymm0, %ymm7
 ; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm8 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0]
-; AVX512F-NEXT:    vpternlogq $184, %ymm12, %ymm8, %ymm7
-; AVX512F-NEXT:    vpternlogq $202, %ymm3, %ymm4, %ymm5
-; AVX512F-NEXT:    vextracti128 $1, %ymm5, %xmm3
+; AVX512F-NEXT:    vpternlogq $184, %ymm13, %ymm8, %ymm7
+; AVX512F-NEXT:    vpternlogq $202, %ymm3, %ymm5, %ymm2
+; AVX512F-NEXT:    vextracti128 $1, %ymm2, %xmm3
 ; AVX512F-NEXT:    vpshufb {{.*#+}} xmm3 = zero,zero,zero,xmm3[3,8,13],zero,zero,zero,xmm3[1,6,11,u,u,u,u]
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm4 = xmm5[4,9,14],zero,zero,zero,xmm5[2,7,12],zero,zero,zero,xmm5[u,u,u,u]
-; AVX512F-NEXT:    vpor %xmm3, %xmm4, %xmm3
-; AVX512F-NEXT:    vpternlogq $202, %ymm2, %ymm1, %ymm0
-; AVX512F-NEXT:    vpermq {{.*#+}} ymm1 = ymm0[2,3,0,1]
-; AVX512F-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm1
-; AVX512F-NEXT:    vpshufb {{.*#+}} ymm0 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,0,5,10,15,20,25,30,19,24,29,18,23,28,u,u,u,u,u,u,u]
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm3[0,1,2],ymm0[3,4,5,6,7]
+; AVX512F-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[4,9,14],zero,zero,zero,xmm2[2,7,12],zero,zero,zero,xmm2[u,u,u,u]
+; AVX512F-NEXT:    vpor %xmm3, %xmm2, %xmm2
+; AVX512F-NEXT:    vpternlogq $202, %ymm1, %ymm0, %ymm4
+; AVX512F-NEXT:    vpermq {{.*#+}} ymm0 = ymm4[2,3,0,1]
+; AVX512F-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm4, %ymm0
+; AVX512F-NEXT:    vpshufb {{.*#+}} ymm0 = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,0,5,10,15,20,25,30,19,24,29,18,23,28,u,u,u,u,u,u,u]
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1,2],ymm0[3,4,5,6,7]
 ; AVX512F-NEXT:    vmovdqa 128(%rdi), %ymm1
 ; AVX512F-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[u,1,6,11,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,21,26,31,u,u,u,u,u,u,u,u]
 ; AVX512F-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [0,5,0,5,0,5,0,5]
@@ -3821,202 +3825,206 @@ define void @load_i8_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ;
 ; AVX512F-LABEL: load_i8_stride5_vf64:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm0 = [65535,0,65535,65535,0,65535,0,65535,65535,0,65535,0,65535,65535,0,65535]
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm5 = [65535,65535,0,65535,0,65535,65535,0,65535,0,65535,65535,0,65535,0,65535]
 ; AVX512F-NEXT:    vmovdqa64 (%rdi), %ymm23
 ; AVX512F-NEXT:    vmovdqa64 32(%rdi), %ymm24
 ; AVX512F-NEXT:    vmovdqa64 64(%rdi), %ymm21
 ; AVX512F-NEXT:    vmovdqa64 96(%rdi), %ymm22
-; AVX512F-NEXT:    vmovdqa %ymm0, %ymm5
-; AVX512F-NEXT:    vpternlogq $202, %ymm24, %ymm23, %ymm5
-; AVX512F-NEXT:    vextracti128 $1, %ymm5, %xmm6
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm6 = zero,zero,zero,zero,xmm6[4,9,14],zero,zero,zero,xmm6[2,7,12,u,u,u]
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm5 = xmm5[0,5,10,15],zero,zero,zero,xmm5[3,8,13],zero,zero,zero,xmm5[u,u,u]
-; AVX512F-NEXT:    vpor %xmm6, %xmm5, %xmm6
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm5 = [65535,65535,0,65535,0,65535,65535,0,65535,0,65535,65535,0,65535,0,65535]
-; AVX512F-NEXT:    vmovdqa %ymm5, %ymm7
-; AVX512F-NEXT:    vpternlogq $202, %ymm22, %ymm21, %ymm7
-; AVX512F-NEXT:    vpermq {{.*#+}} ymm8 = ymm7[2,3,0,1]
-; AVX512F-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm7, %ymm8
-; AVX512F-NEXT:    vpshufb {{.*#+}} ymm9 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm8[1,6,11,16,21,26,31,20,25,30,19,24,29],zero,zero,zero,zero,zero,zero
-; AVX512F-NEXT:    vmovdqa64 {{.*#+}} ymm18 = [255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255]
-; AVX512F-NEXT:    vpternlogq $248, %ymm18, %ymm6, %ymm9
-; AVX512F-NEXT:    vmovdqa 192(%rdi), %ymm6
+; AVX512F-NEXT:    vmovdqa %ymm5, %ymm4
+; AVX512F-NEXT:    vpternlogq $202, %ymm22, %ymm21, %ymm4
+; AVX512F-NEXT:    vpermq {{.*#+}} ymm6 = ymm4[2,3,0,1]
+; AVX512F-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm4, %ymm6
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm10 = [128,128,128,128,128,128,128,128,128,128,128,128,128,1,6,11,16,21,26,31,20,25,30,19,24,29,128,128,128,128,128,128]
+; AVX512F-NEXT:    vpshufb %ymm10, %ymm6, %ymm6
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm4 = [65535,0,65535,65535,0,65535,0,65535,65535,0,65535,0,65535,65535,0,65535]
+; AVX512F-NEXT:    vmovdqa %ymm4, %ymm7
+; AVX512F-NEXT:    vpternlogq $202, %ymm24, %ymm23, %ymm7
+; AVX512F-NEXT:    vextracti128 $1, %ymm7, %xmm8
+; AVX512F-NEXT:    vpshufb {{.*#+}} xmm8 = zero,zero,zero,zero,xmm8[4,9,14],zero,zero,zero,xmm8[2,7,12,u,u,u]
+; AVX512F-NEXT:    vpshufb {{.*#+}} xmm7 = xmm7[0,5,10,15],zero,zero,zero,xmm7[3,8,13],zero,zero,zero,xmm7[u,u,u]
+; AVX512F-NEXT:    vpor %xmm7, %xmm8, %xmm12
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} ymm19 = [255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255]
+; AVX512F-NEXT:    vpternlogq $236, %ymm19, %ymm6, %ymm12
+; AVX512F-NEXT:    vmovdqa64 192(%rdi), %ymm25
 ; AVX512F-NEXT:    vmovdqa 224(%rdi), %ymm7
-; AVX512F-NEXT:    vmovdqa %ymm0, %ymm10
-; AVX512F-NEXT:    vpternlogq $202, %ymm6, %ymm7, %ymm10
+; AVX512F-NEXT:    vmovdqa %ymm4, %ymm9
+; AVX512F-NEXT:    vpternlogq $202, %ymm25, %ymm7, %ymm9
 ; AVX512F-NEXT:    vmovdqa 208(%rdi), %xmm8
-; AVX512F-NEXT:    vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm8, %ymm10
-; AVX512F-NEXT:    vpshufb {{.*#+}} ymm12 = zero,zero,zero,zero,zero,zero,zero,ymm10[3,8,13,2,7,12,1,6,11,16,21,26,31,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512F-NEXT:    vmovdqa 176(%rdi), %xmm10
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm13 = xmm10[u,u,u,u,u,u,u,u,4,9,14,u,u,u,u,u]
+; AVX512F-NEXT:    vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm8, %ymm9
+; AVX512F-NEXT:    vpshufb {{.*#+}} ymm13 = zero,zero,zero,zero,zero,zero,zero,ymm9[3,8,13,2,7,12,1,6,11,16,21,26,31,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512F-NEXT:    vmovdqa 176(%rdi), %xmm9
+; AVX512F-NEXT:    vpshufb {{.*#+}} xmm14 = xmm9[u,u,u,u,u,u,u,u,4,9,14,u,u,u,u,u]
 ; AVX512F-NEXT:    vmovdqa 160(%rdi), %xmm11
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm14 = xmm11[u,u,u,u,u,u,u,u,0,5,10,15,u,u,u,u]
-; AVX512F-NEXT:    vpunpckhdq {{.*#+}} xmm1 = xmm14[2],xmm13[2],xmm14[3],xmm13[3]
+; AVX512F-NEXT:    vpshufb {{.*#+}} xmm15 = xmm11[u,u,u,u,u,u,u,u,0,5,10,15,u,u,u,u]
+; AVX512F-NEXT:    vpunpckhdq {{.*#+}} xmm0 = xmm15[2],xmm14[2],xmm15[3],xmm14[3]
 ; AVX512F-NEXT:    vmovdqa64 {{.*#+}} ymm16 = [0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512F-NEXT:    vpternlogq $186, %ymm12, %ymm16, %ymm1
+; AVX512F-NEXT:    vpternlogq $186, %ymm13, %ymm16, %ymm0
 ; AVX512F-NEXT:    vmovdqa 144(%rdi), %xmm13
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm12 = xmm13[u,u,u,u,u,u,u,u,u,u],zero,zero,zero,xmm13[1,6,11]
+; AVX512F-NEXT:    vpshufb %xmm10, %xmm13, %xmm10
 ; AVX512F-NEXT:    vmovdqa 128(%rdi), %xmm14
 ; AVX512F-NEXT:    vpshufb {{.*#+}} xmm15 = xmm14[u,u,u,u,u,u,u,u,u,u,2,7,12],zero,zero,zero
-; AVX512F-NEXT:    vpor %xmm12, %xmm15, %xmm12
-; AVX512F-NEXT:    vinserti128 $1, %xmm12, %ymm0, %ymm12
-; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm12, %zmm17
+; AVX512F-NEXT:    vpor %xmm10, %xmm15, %xmm10
+; AVX512F-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm0, %zmm10, %zmm10
 ; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm20 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535]
-; AVX512F-NEXT:    vpternlogq $184, %zmm9, %zmm20, %zmm17
+; AVX512F-NEXT:    vpternlogq $184, %zmm12, %zmm20, %zmm10
 ; AVX512F-NEXT:    vmovdqa 256(%rdi), %ymm15
 ; AVX512F-NEXT:    vmovdqa 288(%rdi), %ymm12
-; AVX512F-NEXT:    vmovdqa %ymm5, %ymm9
-; AVX512F-NEXT:    vpternlogq $202, %ymm15, %ymm12, %ymm9
-; AVX512F-NEXT:    vextracti128 $1, %ymm9, %xmm2
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,u,u,u],zero,zero,zero,xmm2[3,8,13],zero,zero,zero,xmm2[1,6,11]
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm9 = xmm9[u,u,u,u,4,9,14],zero,zero,zero,xmm9[2,7,12],zero,zero,zero
-; AVX512F-NEXT:    vpor %xmm2, %xmm9, %xmm2
+; AVX512F-NEXT:    vmovdqa %ymm5, %ymm2
+; AVX512F-NEXT:    vpternlogq $202, %ymm15, %ymm12, %ymm2
+; AVX512F-NEXT:    vextracti128 $1, %ymm2, %xmm3
+; AVX512F-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u,u,u],zero,zero,zero,xmm3[3,8,13],zero,zero,zero,xmm3[1,6,11]
+; AVX512F-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,u,u,u,4,9,14],zero,zero,zero,xmm2[2,7,12],zero,zero,zero
+; AVX512F-NEXT:    vpor %xmm3, %xmm2, %xmm2
 ; AVX512F-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm2[5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm17, %zmm19
-; AVX512F-NEXT:    vmovdqa %ymm0, %ymm1
-; AVX512F-NEXT:    vpternlogq $202, %ymm12, %ymm15, %ymm1
-; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm2[5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm0, %zmm10, %zmm18
+; AVX512F-NEXT:    vmovdqa %ymm4, %ymm0
+; AVX512F-NEXT:    vpternlogq $202, %ymm12, %ymm15, %ymm0
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm2
 ; AVX512F-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,u,u],zero,zero,zero,zero,xmm2[4,9,14],zero,zero,zero,xmm2[2,7,12]
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[u,u,u,0,5,10,15],zero,zero,zero,xmm1[3,8,13],zero,zero,zero
-; AVX512F-NEXT:    vpor %xmm2, %xmm1, %xmm1
-; AVX512F-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
+; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,0,5,10,15],zero,zero,zero,xmm0[3,8,13],zero,zero,zero
+; AVX512F-NEXT:    vpor %xmm2, %xmm0, %xmm0
+; AVX512F-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
 ; AVX512F-NEXT:    vmovdqa %ymm5, %ymm2
-; AVX512F-NEXT:    vpternlogq $202, %ymm6, %ymm7, %ymm2
+; AVX512F-NEXT:    vpternlogq $202, %ymm25, %ymm7, %ymm2
 ; AVX512F-NEXT:    vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm8, %ymm2
 ; AVX512F-NEXT:    vpshufb {{.*#+}} ymm2 = zero,zero,zero,zero,zero,zero,zero,ymm2[4,9,14,3,8,13,2,7,12,17,22,27,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm9 = xmm11[1,6,11],zero,zero,zero,zero,xmm11[u,u,u,u,u,u,u,u,u]
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm3 = zero,zero,zero,xmm10[0,5,10,15,u,u,u,u,u,u,u,u,u]
-; AVX512F-NEXT:    vpor %xmm3, %xmm9, %xmm3
+; AVX512F-NEXT:    vpshufb {{.*#+}} xmm3 = xmm11[1,6,11],zero,zero,zero,zero,xmm11[u,u,u,u,u,u,u,u,u]
+; AVX512F-NEXT:    vpshufb {{.*#+}} xmm10 = zero,zero,zero,xmm9[0,5,10,15,u,u,u,u,u,u,u,u,u]
+; AVX512F-NEXT:    vpor %xmm3, %xmm10, %xmm3
 ; AVX512F-NEXT:    vpternlogq $186, %ymm2, %ymm16, %ymm3
 ; AVX512F-NEXT:    vmovdqa64 {{.*#+}} ymm16 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512F-NEXT:    vpternlogq $226, %ymm1, %ymm16, %ymm3
+; AVX512F-NEXT:    vpternlogq $226, %ymm0, %ymm16, %ymm3
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm10 = [65535,0,65535,0,65535,65535,0,65535,0,65535,65535,0,65535,0,65535,65535]
+; AVX512F-NEXT:    vmovdqa %ymm10, %ymm0
+; AVX512F-NEXT:    vpternlogq $202, %ymm21, %ymm22, %ymm0
+; AVX512F-NEXT:    vpermq {{.*#+}} ymm2 = ymm0[2,3,0,1]
+; AVX512F-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm2
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm0 = [128,128,128,128,128,128,128,128,128,128,128,128,128,2,7,12,17,22,27,16,21,26,31,20,25,30,128,128,128,128,128,128]
+; AVX512F-NEXT:    vpshufb %ymm0, %ymm2, %ymm2
 ; AVX512F-NEXT:    vmovdqa %ymm5, %ymm1
 ; AVX512F-NEXT:    vpternlogq $202, %ymm24, %ymm23, %ymm1
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm2 = xmm1[1,6,11],zero,zero,zero,zero,xmm1[4,9,14],zero,zero,zero,xmm1[u,u,u]
+; AVX512F-NEXT:    vpshufb {{.*#+}} xmm6 = xmm1[1,6,11],zero,zero,zero,zero,xmm1[4,9,14],zero,zero,zero,xmm1[u,u,u]
 ; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm1
 ; AVX512F-NEXT:    vpshufb {{.*#+}} xmm1 = zero,zero,zero,xmm1[0,5,10,15],zero,zero,zero,xmm1[3,8,13,u,u,u]
-; AVX512F-NEXT:    vpor %xmm2, %xmm1, %xmm1
-; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm9 = [65535,0,65535,0,65535,65535,0,65535,0,65535,65535,0,65535,0,65535,65535]
-; AVX512F-NEXT:    vmovdqa %ymm9, %ymm2
-; AVX512F-NEXT:    vpternlogq $202, %ymm21, %ymm22, %ymm2
-; AVX512F-NEXT:    vpermq {{.*#+}} ymm4 = ymm2[2,3,0,1]
-; AVX512F-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm4
-; AVX512F-NEXT:    vpshufb {{.*#+}} ymm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm4[2,7,12,17,22,27,16,21,26,31,20,25,30],zero,zero,zero,zero,zero,zero
-; AVX512F-NEXT:    vpternlogq $248, %ymm18, %ymm1, %ymm2
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm1 = xmm13[u,u,u,u,u,u,u,u,u,u],zero,zero,zero,xmm13[2,7,12]
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm4 = xmm14[u,u,u,u,u,u,u,u,u,u,3,8,13],zero,zero,zero
-; AVX512F-NEXT:    vpor %xmm1, %xmm4, %xmm1
-; AVX512F-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512F-NEXT:    vpternlogq $184, %zmm2, %zmm20, %zmm1
-; AVX512F-NEXT:    vinserti64x4 $1, %ymm3, %zmm1, %zmm17
+; AVX512F-NEXT:    vpor %xmm6, %xmm1, %xmm1
+; AVX512F-NEXT:    vpternlogq $236, %ymm19, %ymm2, %ymm1
+; AVX512F-NEXT:    vpshufb %xmm0, %xmm13, %xmm0
+; AVX512F-NEXT:    vpshufb {{.*#+}} xmm2 = xmm14[u,u,u,u,u,u,u,u,u,u,3,8,13],zero,zero,zero
+; AVX512F-NEXT:    vpor %xmm0, %xmm2, %xmm0
+; AVX512F-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512F-NEXT:    vpternlogq $184, %zmm1, %zmm20, %zmm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm17
+; AVX512F-NEXT:    vmovdqa %ymm5, %ymm0
+; AVX512F-NEXT:    vpternlogq $202, %ymm12, %ymm15, %ymm0
+; AVX512F-NEXT:    vpshufb {{.*#+}} xmm1 = xmm0[u,u,u,1,6,11],zero,zero,zero,zero,xmm0[4,9,14],zero,zero,zero
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,u],zero,zero,zero,xmm0[0,5,10,15],zero,zero,zero,xmm0[3,8,13]
+; AVX512F-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512F-NEXT:    vmovdqa %ymm4, %ymm1
+; AVX512F-NEXT:    vpternlogq $202, %ymm7, %ymm25, %ymm1
+; AVX512F-NEXT:    vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm8, %ymm1
+; AVX512F-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[u,u,u,u,u,u,0,5,10,15,4,9,14,3,8,13,18,23,28,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512F-NEXT:    vpshufb {{.*#+}} xmm2 = zero,zero,zero,xmm9[1,6,11,u,u,u,u,u,u,u,u,u,u]
+; AVX512F-NEXT:    vpshufb {{.*#+}} xmm3 = xmm11[2,7,12],zero,zero,zero,xmm11[u,u,u,u,u,u,u,u,u,u]
+; AVX512F-NEXT:    vpor %xmm2, %xmm3, %xmm2
+; AVX512F-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1,2],xmm1[3,4,5,6,7]
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    vpternlogq $226, %ymm0, %ymm16, %ymm1
+; AVX512F-NEXT:    vmovdqa %ymm4, %ymm0
+; AVX512F-NEXT:    vpternlogq $202, %ymm21, %ymm22, %ymm0
+; AVX512F-NEXT:    vpermq {{.*#+}} ymm2 = ymm0[2,3,0,1]
+; AVX512F-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm2
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm0 = [128,128,128,128,128,128,128,128,128,128,128,128,128,3,8,13,18,23,28,17,22,27,16,21,26,31,128,128,128,128,128,128]
+; AVX512F-NEXT:    vpshufb %ymm0, %ymm2, %ymm2
+; AVX512F-NEXT:    vmovdqa %ymm10, %ymm3
+; AVX512F-NEXT:    vpternlogq $202, %ymm23, %ymm24, %ymm3
+; AVX512F-NEXT:    vextracti128 $1, %ymm3, %xmm6
+; AVX512F-NEXT:    vpshufb {{.*#+}} xmm6 = zero,zero,zero,xmm6[1,6,11],zero,zero,zero,zero,xmm6[4,9,14,u,u,u]
+; AVX512F-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[2,7,12],zero,zero,zero,xmm3[0,5,10,15],zero,zero,zero,xmm3[u,u,u]
+; AVX512F-NEXT:    vpor %xmm6, %xmm3, %xmm3
+; AVX512F-NEXT:    vpternlogq $236, %ymm19, %ymm2, %ymm3
+; AVX512F-NEXT:    vpshufb %xmm0, %xmm13, %xmm0
+; AVX512F-NEXT:    vpshufb {{.*#+}} xmm2 = xmm14[u,u,u,u,u,u,u,u,u,u,4,9,14],zero,zero,zero
+; AVX512F-NEXT:    vpor %xmm0, %xmm2, %xmm0
+; AVX512F-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512F-NEXT:    vpternlogq $184, %zmm3, %zmm20, %zmm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm19
+; AVX512F-NEXT:    vmovdqa %ymm10, %ymm0
+; AVX512F-NEXT:    vpternlogq $202, %ymm15, %ymm12, %ymm0
+; AVX512F-NEXT:    vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[u,u,u],zero,zero,zero,xmm1[1,6,11],zero,zero,zero,zero,xmm1[4,9,14]
+; AVX512F-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,2,7,12],zero,zero,zero,xmm0[0,5,10,15],zero,zero,zero
+; AVX512F-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
 ; AVX512F-NEXT:    vmovdqa %ymm5, %ymm1
-; AVX512F-NEXT:    vpternlogq $202, %ymm12, %ymm15, %ymm1
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm2 = xmm1[u,u,u,1,6,11],zero,zero,zero,zero,xmm1[4,9,14],zero,zero,zero
-; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm1
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[u,u,u],zero,zero,zero,xmm1[0,5,10,15],zero,zero,zero,xmm1[3,8,13]
-; AVX512F-NEXT:    vpor %xmm2, %xmm1, %xmm1
-; AVX512F-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512F-NEXT:    vmovdqa %ymm0, %ymm2
-; AVX512F-NEXT:    vpternlogq $202, %ymm7, %ymm6, %ymm2
-; AVX512F-NEXT:    vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm8, %ymm2
-; AVX512F-NEXT:    vpshufb {{.*#+}} ymm2 = ymm2[u,u,u,u,u,u,0,5,10,15,4,9,14,3,8,13,18,23,28,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm3 = zero,zero,zero,xmm10[1,6,11,u,u,u,u,u,u,u,u,u,u]
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm4 = xmm11[2,7,12],zero,zero,zero,xmm11[u,u,u,u,u,u,u,u,u,u]
-; AVX512F-NEXT:    vpor %xmm3, %xmm4, %xmm3
-; AVX512F-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1,2],xmm2[3,4,5,6,7]
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
-; AVX512F-NEXT:    vpternlogq $226, %ymm1, %ymm16, %ymm2
-; AVX512F-NEXT:    vmovdqa %ymm9, %ymm1
-; AVX512F-NEXT:    vpternlogq $202, %ymm23, %ymm24, %ymm1
-; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm3
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm3 = zero,zero,zero,xmm3[1,6,11],zero,zero,zero,zero,xmm3[4,9,14,u,u,u]
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[2,7,12],zero,zero,zero,xmm1[0,5,10,15],zero,zero,zero,xmm1[u,u,u]
-; AVX512F-NEXT:    vpor %xmm3, %xmm1, %xmm1
-; AVX512F-NEXT:    vmovdqa %ymm0, %ymm3
-; AVX512F-NEXT:    vpternlogq $202, %ymm21, %ymm22, %ymm3
-; AVX512F-NEXT:    vpermq {{.*#+}} ymm4 = ymm3[2,3,0,1]
-; AVX512F-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm4
-; AVX512F-NEXT:    vpshufb {{.*#+}} ymm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm4[3,8,13,18,23,28,17,22,27,16,21,26,31],zero,zero,zero,zero,zero,zero
-; AVX512F-NEXT:    vpternlogq $248, %ymm18, %ymm1, %ymm3
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm1 = xmm13[u,u,u,u,u,u,u,u,u,u],zero,zero,zero,xmm13[3,8,13]
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm4 = xmm14[u,u,u,u,u,u,u,u,u,u,4,9,14],zero,zero,zero
-; AVX512F-NEXT:    vpor %xmm1, %xmm4, %xmm1
-; AVX512F-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512F-NEXT:    vpternlogq $184, %zmm3, %zmm20, %zmm1
-; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm18
-; AVX512F-NEXT:    vmovdqa %ymm9, %ymm1
-; AVX512F-NEXT:    vpternlogq $202, %ymm15, %ymm12, %ymm1
-; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm2
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,u,u],zero,zero,zero,xmm2[1,6,11],zero,zero,zero,zero,xmm2[4,9,14]
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[u,u,u,2,7,12],zero,zero,zero,xmm1[0,5,10,15],zero,zero,zero
-; AVX512F-NEXT:    vpor %xmm2, %xmm1, %xmm1
-; AVX512F-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512F-NEXT:    vmovdqa %ymm5, %ymm2
-; AVX512F-NEXT:    vpternlogq $202, %ymm7, %ymm6, %ymm2
-; AVX512F-NEXT:    vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm8, %ymm2
-; AVX512F-NEXT:    vpshufb {{.*#+}} ymm2 = ymm2[u,u,u,u,u,u,1,6,11,0,5,10,15,4,9,14,19,24,29,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm3 = zero,zero,zero,xmm10[2,7,12,u,u,u,u,u,u,u,u,u,u]
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm4 = xmm11[3,8,13],zero,zero,zero,xmm11[u,u,u,u,u,u,u,u,u,u]
-; AVX512F-NEXT:    vpor %xmm3, %xmm4, %xmm3
-; AVX512F-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1,2],xmm2[3,4,5,6,7]
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
-; AVX512F-NEXT:    vpternlogq $226, %ymm1, %ymm16, %ymm2
-; AVX512F-NEXT:    vmovdqa %ymm0, %ymm1
-; AVX512F-NEXT:    vpternlogq $202, %ymm23, %ymm24, %ymm1
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm3 = xmm1[3,8,13],zero,zero,zero,xmm1[1,6,11],zero,zero,zero,zero,xmm1[u,u,u]
-; AVX512F-NEXT:    vextracti128 $1, %ymm1, %xmm1
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm1 = zero,zero,zero,xmm1[2,7,12],zero,zero,zero,xmm1[0,5,10,15,u,u,u]
-; AVX512F-NEXT:    vpor %xmm3, %xmm1, %xmm1
-; AVX512F-NEXT:    vmovdqa %ymm5, %ymm3
-; AVX512F-NEXT:    vpternlogq $202, %ymm21, %ymm22, %ymm3
-; AVX512F-NEXT:    vpermq {{.*#+}} ymm4 = ymm3[2,3,0,1]
-; AVX512F-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm4
-; AVX512F-NEXT:    vpshufb {{.*#+}} ymm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm4[4,9,14,19,24,29,18,23,28,17,22,27,u,u,u,u,u,u,u]
-; AVX512F-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm3
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm1 = xmm13[u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,xmm13[4,9,14]
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm4 = xmm14[u,u,u,u,u,u,u,u,u,0,5,10,15],zero,zero,zero
-; AVX512F-NEXT:    vpor %xmm1, %xmm4, %xmm1
-; AVX512F-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512F-NEXT:    vpternlogq $226, %zmm3, %zmm4, %zmm1
-; AVX512F-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm1
-; AVX512F-NEXT:    vpternlogq $226, %ymm15, %ymm0, %ymm12
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm2 = xmm12[u,u,u,3,8,13],zero,zero,zero,xmm12[1,6,11],zero,zero,zero,zero
+; AVX512F-NEXT:    vpternlogq $202, %ymm7, %ymm25, %ymm1
+; AVX512F-NEXT:    vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm8, %ymm1
+; AVX512F-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[u,u,u,u,u,u,1,6,11,0,5,10,15,4,9,14,19,24,29,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512F-NEXT:    vpshufb {{.*#+}} xmm2 = zero,zero,zero,xmm9[2,7,12,u,u,u,u,u,u,u,u,u,u]
+; AVX512F-NEXT:    vpshufb {{.*#+}} xmm3 = xmm11[3,8,13],zero,zero,zero,xmm11[u,u,u,u,u,u,u,u,u,u]
+; AVX512F-NEXT:    vpor %xmm2, %xmm3, %xmm2
+; AVX512F-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1,2],xmm1[3,4,5,6,7]
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    vpternlogq $226, %ymm0, %ymm16, %ymm1
+; AVX512F-NEXT:    vmovdqa %ymm5, %ymm0
+; AVX512F-NEXT:    vpternlogq $202, %ymm21, %ymm22, %ymm0
+; AVX512F-NEXT:    vpermq {{.*#+}} ymm2 = ymm0[2,3,0,1]
+; AVX512F-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm2
+; AVX512F-NEXT:    vmovdqa {{.*#+}} ymm0 = <128,128,128,128,128,128,128,128,128,128,128,128,128,4,9,14,19,24,29,18,23,28,17,22,27,u,u,u,u,u,u,u>
+; AVX512F-NEXT:    vpshufb %ymm0, %ymm2, %ymm2
+; AVX512F-NEXT:    vmovdqa %ymm4, %ymm3
+; AVX512F-NEXT:    vpternlogq $202, %ymm23, %ymm24, %ymm3
+; AVX512F-NEXT:    vpshufb {{.*#+}} xmm6 = xmm3[3,8,13],zero,zero,zero,xmm3[1,6,11],zero,zero,zero,zero,xmm3[u,u,u]
+; AVX512F-NEXT:    vextracti128 $1, %ymm3, %xmm3
+; AVX512F-NEXT:    vpshufb {{.*#+}} xmm3 = zero,zero,zero,xmm3[2,7,12],zero,zero,zero,xmm3[0,5,10,15,u,u,u]
+; AVX512F-NEXT:    vpor %xmm6, %xmm3, %xmm3
+; AVX512F-NEXT:    vpternlogq $236, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm3
+; AVX512F-NEXT:    vpshufb %xmm0, %xmm13, %xmm0
+; AVX512F-NEXT:    vpshufb {{.*#+}} xmm2 = xmm14[u,u,u,u,u,u,u,u,u,0,5,10,15],zero,zero,zero
+; AVX512F-NEXT:    vpor %xmm0, %xmm2, %xmm0
+; AVX512F-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; AVX512F-NEXT:    vpternlogq $226, %zmm3, %zmm2, %zmm0
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vpternlogq $226, %ymm15, %ymm4, %ymm12
+; AVX512F-NEXT:    vpshufb {{.*#+}} xmm1 = xmm12[u,u,u,3,8,13],zero,zero,zero,xmm12[1,6,11],zero,zero,zero,zero
 ; AVX512F-NEXT:    vextracti128 $1, %ymm12, %xmm3
 ; AVX512F-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u,u],zero,zero,zero,xmm3[2,7,12],zero,zero,zero,xmm3[0,5,10,15]
-; AVX512F-NEXT:    vpor %xmm2, %xmm3, %xmm2
-; AVX512F-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512F-NEXT:    vpternlogq $202, %ymm6, %ymm7, %ymm9
-; AVX512F-NEXT:    vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm8, %ymm9
-; AVX512F-NEXT:    vpshufb {{.*#+}} ymm3 = ymm9[u,u,u,u,u,u,2,7,12,1,6,11,0,5,10,15,20,25,30,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm6 = zero,zero,zero,xmm10[3,8,13,u,u,u,u,u,u,u,u,u,u]
+; AVX512F-NEXT:    vpor %xmm1, %xmm3, %xmm1
+; AVX512F-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
+; AVX512F-NEXT:    vpternlogq $202, %ymm25, %ymm7, %ymm10
+; AVX512F-NEXT:    vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm8, %ymm10
+; AVX512F-NEXT:    vpshufb {{.*#+}} ymm3 = ymm10[u,u,u,u,u,u,2,7,12,1,6,11,0,5,10,15,20,25,30,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512F-NEXT:    vpshufb {{.*#+}} xmm6 = zero,zero,zero,xmm9[3,8,13,u,u,u,u,u,u,u,u,u,u]
 ; AVX512F-NEXT:    vpshufb {{.*#+}} xmm7 = xmm11[4,9,14],zero,zero,zero,xmm11[u,u,u,u,u,u,u,u,u,u]
 ; AVX512F-NEXT:    vpor %xmm6, %xmm7, %xmm6
 ; AVX512F-NEXT:    vpblendw {{.*#+}} xmm6 = xmm6[0,1,2],xmm3[3,4,5,6,7]
 ; AVX512F-NEXT:    vpblendd {{.*#+}} ymm3 = ymm6[0,1,2,3],ymm3[4,5,6,7]
-; AVX512F-NEXT:    vpternlogq $226, %ymm2, %ymm16, %ymm3
+; AVX512F-NEXT:    vpternlogq $226, %ymm1, %ymm16, %ymm3
 ; AVX512F-NEXT:    vpternlogq $202, %ymm23, %ymm24, %ymm5
-; AVX512F-NEXT:    vextracti128 $1, %ymm5, %xmm2
-; AVX512F-NEXT:    vpshufb {{.*#+}} xmm2 = zero,zero,zero,xmm2[3,8,13],zero,zero,zero,xmm2[1,6,11,u,u,u,u]
+; AVX512F-NEXT:    vextracti128 $1, %ymm5, %xmm1
+; AVX512F-NEXT:    vpshufb {{.*#+}} xmm1 = zero,zero,zero,xmm1[3,8,13],zero,zero,zero,xmm1[1,6,11,u,u,u,u]
 ; AVX512F-NEXT:    vpshufb {{.*#+}} xmm5 = xmm5[4,9,14],zero,zero,zero,xmm5[2,7,12],zero,zero,zero,xmm5[u,u,u,u]
-; AVX512F-NEXT:    vpor %xmm2, %xmm5, %xmm2
-; AVX512F-NEXT:    vpternlogq $202, %ymm22, %ymm21, %ymm0
-; AVX512F-NEXT:    vpermq {{.*#+}} ymm5 = ymm0[2,3,0,1]
-; AVX512F-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm5
-; AVX512F-NEXT:    vpshufb {{.*#+}} ymm0 = ymm5[u,u,u,u,u,u,u,u,u,u,u,u,0,5,10,15,20,25,30,19,24,29,18,23,28,u,u,u,u,u,u,u]
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm2[0,1,2],ymm0[3,4,5,6,7]
-; AVX512F-NEXT:    vmovdqa 128(%rdi), %ymm2
-; AVX512F-NEXT:    vpshufb {{.*#+}} ymm2 = ymm2[u,1,6,11,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,21,26,31,u,u,u,u,u,u,u,u]
+; AVX512F-NEXT:    vpor %xmm1, %xmm5, %xmm1
+; AVX512F-NEXT:    vpternlogq $202, %ymm22, %ymm21, %ymm4
+; AVX512F-NEXT:    vpermq {{.*#+}} ymm5 = ymm4[2,3,0,1]
+; AVX512F-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm4, %ymm5
+; AVX512F-NEXT:    vpshufb {{.*#+}} ymm4 = ymm5[u,u,u,u,u,u,u,u,u,u,u,u,0,5,10,15,20,25,30,19,24,29,18,23,28,u,u,u,u,u,u,u]
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2],ymm4[3,4,5,6,7]
+; AVX512F-NEXT:    vmovdqa 128(%rdi), %ymm4
+; AVX512F-NEXT:    vpshufb {{.*#+}} ymm4 = ymm4[u,1,6,11,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,21,26,31,u,u,u,u,u,u,u,u]
 ; AVX512F-NEXT:    vpbroadcastq {{.*#+}} ymm5 = [0,5,0,5,0,5,0,5]
-; AVX512F-NEXT:    vpermd %ymm2, %ymm5, %ymm2
-; AVX512F-NEXT:    vpternlogq $226, %zmm0, %zmm4, %zmm2
-; AVX512F-NEXT:    vinserti64x4 $1, %ymm3, %zmm2, %zmm0
-; AVX512F-NEXT:    vmovdqa64 %zmm19, (%rsi)
+; AVX512F-NEXT:    vpermd %ymm4, %ymm5, %ymm4
+; AVX512F-NEXT:    vpternlogq $226, %zmm1, %zmm2, %zmm4
+; AVX512F-NEXT:    vinserti64x4 $1, %ymm3, %zmm4, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm18, (%rsi)
 ; AVX512F-NEXT:    vmovdqa64 %zmm17, (%rdx)
-; AVX512F-NEXT:    vmovdqa64 %zmm18, (%rcx)
-; AVX512F-NEXT:    vmovdqa64 %zmm1, (%r8)
-; AVX512F-NEXT:    vmovdqa64 %zmm0, (%r9)
+; AVX512F-NEXT:    vmovdqa64 %zmm19, (%rcx)
+; AVX512F-NEXT:    vmovdqa64 %zmm0, (%r8)
+; AVX512F-NEXT:    vmovdqa64 %zmm1, (%r9)
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-6.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-6.ll
index be34bbd254022275e646a263c3a40a0eda2d6dc2..992b190ac17cc82290c1f069289612535bd30979 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-6.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-6.ll
@@ -1700,248 +1700,250 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ;
 ; AVX1-ONLY-LABEL: load_i8_stride6_vf32:
 ; AVX1-ONLY:       # %bb.0:
-; AVX1-ONLY-NEXT:    subq $168, %rsp
-; AVX1-ONLY-NEXT:    vmovdqa 128(%rdi), %xmm1
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm0 = xmm1[u,u,4,10,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX1-ONLY-NEXT:    vmovdqa %xmm1, %xmm4
-; AVX1-ONLY-NEXT:    vmovdqa 144(%rdi), %xmm2
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm1 = xmm2[u,u,u,u,0,6,12,u,u,u,u,u,u,u,u,u]
-; AVX1-ONLY-NEXT:    vmovdqa %xmm2, %xmm5
-; AVX1-ONLY-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; AVX1-ONLY-NEXT:    vmovdqa 176(%rdi), %xmm2
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm0 = xmm2[u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,xmm2[4,10]
-; AVX1-ONLY-NEXT:    vmovdqa %xmm2, %xmm6
-; AVX1-ONLY-NEXT:    vmovdqa 160(%rdi), %xmm3
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm2 = xmm3[u,u,u,u,u,u,u,u,u,u,u,2,8,14],zero,zero
-; AVX1-ONLY-NEXT:    vmovdqa %xmm3, %xmm7
+; AVX1-ONLY-NEXT:    subq $120, %rsp
+; AVX1-ONLY-NEXT:    vmovdqa (%rdi), %xmm9
+; AVX1-ONLY-NEXT:    vmovdqa 16(%rdi), %xmm7
+; AVX1-ONLY-NEXT:    vmovdqa 32(%rdi), %xmm6
+; AVX1-ONLY-NEXT:    vmovdqa 48(%rdi), %xmm8
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm0 = xmm8[2,8,14,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm1 = xmm6[u,u,u,u,u,0,6,12,u,u,u,u,u,u,u,u]
+; AVX1-ONLY-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm0 = zero,zero,zero,xmm7[4,10,u,u,u,u,u,u,u,u,u,u,u]
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm2 = xmm9[2,8,14],zero,zero,xmm9[u,u,u,u,u,u,u,u,u,u,u]
 ; AVX1-ONLY-NEXT:    vpor %xmm0, %xmm2, %xmm2
-; AVX1-ONLY-NEXT:    vmovdqa {{.*#+}} xmm0 = <u,u,u,u,u,u,255,255,255,255,255,0,0,0,0,0>
+; AVX1-ONLY-NEXT:    vmovdqa {{.*#+}} xmm0 = <0,0,0,0,0,255,255,255,255,255,255,u,u,u,u,u>
 ; AVX1-ONLY-NEXT:    vpblendvb %xmm0, %xmm1, %xmm2, %xmm1
-; AVX1-ONLY-NEXT:    vmovdqa %xmm1, (%rsp) # 16-byte Spill
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm1 = xmm4[u,u,5,11,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm2 = xmm5[u,u,u,u,1,7,13,u,u,u,u,u,u,u,u,u]
-; AVX1-ONLY-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm2 = xmm6[u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,xmm6[5,11]
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u,u,u,u,u,u,u,u,u,u,3,9,15],zero,zero
+; AVX1-ONLY-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm1 = xmm8[3,9,15,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm2 = xmm6[u,u,u,u,u,1,7,13,u,u,u,u,u,u,u,u]
+; AVX1-ONLY-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm2 = zero,zero,zero,xmm7[5,11,u,u,u,u,u,u,u,u,u,u,u]
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm3 = xmm9[3,9,15],zero,zero,xmm9[u,u,u,u,u,u,u,u,u,u,u]
 ; AVX1-ONLY-NEXT:    vpor %xmm2, %xmm3, %xmm2
 ; AVX1-ONLY-NEXT:    vpblendvb %xmm0, %xmm1, %xmm2, %xmm0
-; AVX1-ONLY-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX1-ONLY-NEXT:    vmovdqa 32(%rdi), %xmm14
-; AVX1-ONLY-NEXT:    vmovdqa 48(%rdi), %xmm15
-; AVX1-ONLY-NEXT:    vbroadcastss {{.*#+}} xmm0 = [2,8,14,0,2,8,14,0,2,8,14,0,2,8,14,0]
-; AVX1-ONLY-NEXT:    vpshufb %xmm0, %xmm15, %xmm2
-; AVX1-ONLY-NEXT:    vbroadcastss {{.*#+}} xmm1 = [0,0,6,12,0,0,6,12,0,0,6,12,0,0,6,12]
-; AVX1-ONLY-NEXT:    vpshufb %xmm1, %xmm14, %xmm3
-; AVX1-ONLY-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm2[0]
-; AVX1-ONLY-NEXT:    vmovdqa (%rdi), %xmm9
-; AVX1-ONLY-NEXT:    vmovdqa 16(%rdi), %xmm8
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm2 = zero,zero,zero,xmm8[4,10,u,u,u,u,u,u,u,u,u,u,u]
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm10 = xmm9[2,8,14],zero,zero,xmm9[u,u,u,u,u,u,u,u,u,u,u]
-; AVX1-ONLY-NEXT:    vpor %xmm2, %xmm10, %xmm10
-; AVX1-ONLY-NEXT:    vmovdqa {{.*#+}} xmm2 = <0,0,0,0,0,255,255,255,255,255,255,u,u,u,u,u>
-; AVX1-ONLY-NEXT:    vpblendvb %xmm2, %xmm3, %xmm10, %xmm3
-; AVX1-ONLY-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX1-ONLY-NEXT:    vbroadcastss {{.*#+}} xmm3 = [3,9,15,0,3,9,15,0,3,9,15,0,3,9,15,0]
-; AVX1-ONLY-NEXT:    vpshufb %xmm3, %xmm15, %xmm10
-; AVX1-ONLY-NEXT:    vbroadcastss {{.*#+}} xmm11 = [0,1,7,13,0,1,7,13,0,1,7,13,0,1,7,13]
-; AVX1-ONLY-NEXT:    vpshufb %xmm11, %xmm14, %xmm12
-; AVX1-ONLY-NEXT:    vpunpcklqdq {{.*#+}} xmm10 = xmm12[0],xmm10[0]
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm12 = zero,zero,zero,xmm8[5,11,u,u,u,u,u,u,u,u,u,u,u]
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm13 = xmm9[3,9,15],zero,zero,xmm9[u,u,u,u,u,u,u,u,u,u,u]
-; AVX1-ONLY-NEXT:    vpor %xmm12, %xmm13, %xmm12
-; AVX1-ONLY-NEXT:    vpblendvb %xmm2, %xmm10, %xmm12, %xmm2
-; AVX1-ONLY-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX1-ONLY-NEXT:    vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX1-ONLY-NEXT:    vpshufb %xmm0, %xmm5, %xmm0
-; AVX1-ONLY-NEXT:    vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX1-ONLY-NEXT:    vpshufb %xmm1, %xmm4, %xmm1
-; AVX1-ONLY-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX1-ONLY-NEXT:    vmovdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm1 = xmm7[u,u,u,u,u,u,u,u,u,u,u,4,10],zero,zero,zero
-; AVX1-ONLY-NEXT:    vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm2 = xmm6[u,u,u,u,u,u,u,u,u,u,u],zero,zero,xmm6[0,6,12]
-; AVX1-ONLY-NEXT:    vpor %xmm1, %xmm2, %xmm1
-; AVX1-ONLY-NEXT:    vmovdqa {{.*#+}} xmm2 = <u,u,u,u,u,255,255,255,255,255,255,0,0,0,0,0>
-; AVX1-ONLY-NEXT:    vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
-; AVX1-ONLY-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX1-ONLY-NEXT:    vpshufb %xmm3, %xmm5, %xmm0
-; AVX1-ONLY-NEXT:    vpshufb %xmm11, %xmm4, %xmm1
-; AVX1-ONLY-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm1 = xmm7[u,u,u,u,u,u,u,u,u,u,u,5,11],zero,zero,zero
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm3 = xmm6[u,u,u,u,u,u,u,u,u,u,u],zero,zero,xmm6[1,7,13]
-; AVX1-ONLY-NEXT:    vpor %xmm1, %xmm3, %xmm1
-; AVX1-ONLY-NEXT:    vpblendvb %xmm2, %xmm0, %xmm1, %xmm0
-; AVX1-ONLY-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-ONLY-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm0 = xmm9[4,10],zero,zero,zero,xmm9[u,u,u,u,u,u,u,u,u,u,u]
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm1 = zero,zero,xmm8[0,6,12,u,u,u,u,u,u,u,u,u,u,u]
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm1 = zero,zero,xmm7[0,6,12,u,u,u,u,u,u,u,u,u,u,u]
 ; AVX1-ONLY-NEXT:    vpor %xmm0, %xmm1, %xmm1
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm0 = xmm15[u,u,u,u,u,u,u,u,4,10,u,u,u,u,u,u]
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm2 = xmm14[u,u,u,u,u,u,u,u,u,u,u,u,u,2,8,14]
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm0 = xmm8[u,u,u,u,u,u,u,u,4,10,u,u,u,u,u,u]
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm2 = xmm6[u,u,u,u,u,u,u,u,u,u,u,u,u,2,8,14]
 ; AVX1-ONLY-NEXT:    vpunpckhqdq {{.*#+}} xmm2 = xmm2[1],xmm0[1]
 ; AVX1-ONLY-NEXT:    vmovdqa {{.*#+}} xmm0 = <255,255,255,255,255,0,0,0,0,0,u,u,u,u,u,u>
 ; AVX1-ONLY-NEXT:    vpblendvb %xmm0, %xmm1, %xmm2, %xmm1
 ; AVX1-ONLY-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm1 = xmm9[5,11],zero,zero,zero,xmm9[u,u,u,u,u,u,u,u,u,u,u]
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm2 = zero,zero,xmm8[1,7,13,u,u,u,u,u,u,u,u,u,u,u]
+; AVX1-ONLY-NEXT:    vmovdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm2 = zero,zero,xmm7[1,7,13,u,u,u,u,u,u,u,u,u,u,u]
 ; AVX1-ONLY-NEXT:    vpor %xmm1, %xmm2, %xmm1
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm2 = xmm15[u,u,u,u,u,u,u,u,5,11,u,u,u,u,u,u]
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm3 = xmm14[u,u,u,u,u,u,u,u,u,u,u,u,u,3,9,15]
+; AVX1-ONLY-NEXT:    vmovdqa %xmm8, %xmm5
+; AVX1-ONLY-NEXT:    vmovdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm2 = xmm8[u,u,u,u,u,u,u,u,5,11,u,u,u,u,u,u]
+; AVX1-ONLY-NEXT:    vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm3 = xmm6[u,u,u,u,u,u,u,u,u,u,u,u,u,3,9,15]
 ; AVX1-ONLY-NEXT:    vpunpckhqdq {{.*#+}} xmm2 = xmm3[1],xmm2[1]
 ; AVX1-ONLY-NEXT:    vpblendvb %xmm0, %xmm1, %xmm2, %xmm0
 ; AVX1-ONLY-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm10 = [128,128,128,2,8,14,0,0,128,128,128,2,8,14,0,0]
-; AVX1-ONLY-NEXT:    # xmm10 = mem[0,0]
-; AVX1-ONLY-NEXT:    vmovdqa 112(%rdi), %xmm5
-; AVX1-ONLY-NEXT:    vpshufb %xmm10, %xmm5, %xmm1
-; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm6 = [0,6,12,128,128,128,0,0,0,6,12,128,128,128,0,0]
-; AVX1-ONLY-NEXT:    # xmm6 = mem[0,0]
-; AVX1-ONLY-NEXT:    vmovdqa 96(%rdi), %xmm3
-; AVX1-ONLY-NEXT:    vpshufb %xmm6, %xmm3, %xmm2
-; AVX1-ONLY-NEXT:    vpor %xmm1, %xmm2, %xmm1
-; AVX1-ONLY-NEXT:    vmovdqa 80(%rdi), %xmm2
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm0 = xmm2[u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,xmm2[4,10]
-; AVX1-ONLY-NEXT:    vmovdqa 64(%rdi), %xmm4
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm13 = xmm4[u,u,u,u,u,u,u,u,u,u,u,2,8,14],zero,zero
-; AVX1-ONLY-NEXT:    vpor %xmm0, %xmm13, %xmm0
+; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm8 = [128,128,128,2,8,14,0,0,128,128,128,2,8,14,0,0]
+; AVX1-ONLY-NEXT:    # xmm8 = mem[0,0]
+; AVX1-ONLY-NEXT:    vmovdqa 112(%rdi), %xmm0
+; AVX1-ONLY-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-ONLY-NEXT:    vpshufb %xmm8, %xmm0, %xmm1
+; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm2 = [0,6,12,128,128,128,0,0,0,6,12,128,128,128,0,0]
+; AVX1-ONLY-NEXT:    # xmm2 = mem[0,0]
+; AVX1-ONLY-NEXT:    vmovdqa 96(%rdi), %xmm13
+; AVX1-ONLY-NEXT:    vpshufb %xmm2, %xmm13, %xmm3
+; AVX1-ONLY-NEXT:    vpor %xmm1, %xmm3, %xmm1
+; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm11 = [0,0,0,128,128,128,4,10,0,0,0,128,128,128,4,10]
+; AVX1-ONLY-NEXT:    # xmm11 = mem[0,0]
+; AVX1-ONLY-NEXT:    vmovdqa 80(%rdi), %xmm12
+; AVX1-ONLY-NEXT:    vpshufb %xmm11, %xmm12, %xmm4
+; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm3 = [0,0,0,2,8,14,128,128,0,0,0,2,8,14,128,128]
+; AVX1-ONLY-NEXT:    # xmm3 = mem[0,0]
+; AVX1-ONLY-NEXT:    vmovdqa 64(%rdi), %xmm14
+; AVX1-ONLY-NEXT:    vpshufb %xmm3, %xmm14, %xmm10
+; AVX1-ONLY-NEXT:    vpor %xmm4, %xmm10, %xmm4
+; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm1, %ymm4, %ymm1
+; AVX1-ONLY-NEXT:    vbroadcastss {{.*#+}} xmm15 = [0,0,4,10,0,0,4,10,0,0,4,10,0,0,4,10]
+; AVX1-ONLY-NEXT:    vpshufb %xmm15, %xmm6, %xmm4
+; AVX1-ONLY-NEXT:    vbroadcastss {{.*#+}} xmm0 = [0,6,12,0,0,6,12,0,0,6,12,0,0,6,12,0]
+; AVX1-ONLY-NEXT:    vpshufb %xmm0, %xmm5, %xmm10
+; AVX1-ONLY-NEXT:    vpunpckldq {{.*#+}} xmm4 = xmm10[0],xmm4[0],xmm10[1],xmm4[1]
+; AVX1-ONLY-NEXT:    vpshufb %xmm8, %xmm7, %xmm8
+; AVX1-ONLY-NEXT:    vmovdqa %xmm7, %xmm10
+; AVX1-ONLY-NEXT:    vpshufb %xmm2, %xmm9, %xmm2
+; AVX1-ONLY-NEXT:    vpor %xmm2, %xmm8, %xmm2
+; AVX1-ONLY-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1,2],xmm4[3,4,5],xmm2[6,7]
+; AVX1-ONLY-NEXT:    vmovaps {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255]
+; AVX1-ONLY-NEXT:    vandnps %ymm1, %ymm4, %ymm1
+; AVX1-ONLY-NEXT:    vandps %ymm4, %ymm2, %ymm2
+; AVX1-ONLY-NEXT:    vorps %ymm1, %ymm2, %ymm8
+; AVX1-ONLY-NEXT:    vmovdqa 128(%rdi), %xmm6
+; AVX1-ONLY-NEXT:    vpshufb %xmm15, %xmm6, %xmm1
+; AVX1-ONLY-NEXT:    vmovdqa 144(%rdi), %xmm5
+; AVX1-ONLY-NEXT:    vpshufb %xmm0, %xmm5, %xmm0
+; AVX1-ONLY-NEXT:    vpunpckldq {{.*#+}} xmm15 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX1-ONLY-NEXT:    vmovdqa 176(%rdi), %xmm4
+; AVX1-ONLY-NEXT:    vpshufb %xmm11, %xmm4, %xmm11
+; AVX1-ONLY-NEXT:    vmovdqa 160(%rdi), %xmm2
+; AVX1-ONLY-NEXT:    vpshufb %xmm3, %xmm2, %xmm3
+; AVX1-ONLY-NEXT:    vpor %xmm3, %xmm11, %xmm11
+; AVX1-ONLY-NEXT:    vmovdqa {{.*#+}} ymm3 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255]
+; AVX1-ONLY-NEXT:    vpblendvb %xmm3, %xmm15, %xmm11, %xmm15
+; AVX1-ONLY-NEXT:    vmovaps {{.*#+}} ymm11 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0]
+; AVX1-ONLY-NEXT:    vandps %ymm11, %ymm8, %ymm8
+; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm15, %ymm0, %ymm15
+; AVX1-ONLY-NEXT:    vandnps %ymm15, %ymm11, %ymm15
+; AVX1-ONLY-NEXT:    vorps %ymm15, %ymm8, %ymm0
+; AVX1-ONLY-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm0 = zero,zero,zero,xmm9[3,9,15,u,u,u,u,u,u,u,u,u,u]
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm15 = xmm13[1,7,13],zero,zero,zero,xmm13[u,u,u,u,u,u,u,u,u,u]
+; AVX1-ONLY-NEXT:    vpor %xmm0, %xmm15, %xmm1
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm0 = xmm12[u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,xmm12[5,11]
+; AVX1-ONLY-NEXT:    vmovdqa %xmm14, %xmm7
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm15 = xmm14[u,u,u,u,u,u,u,u,u,u,u,3,9,15],zero,zero
+; AVX1-ONLY-NEXT:    vpor %xmm0, %xmm15, %xmm0
 ; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm1 = xmm14[u,u,4,10,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm13 = xmm15[u,u,u,u,0,6,12,u,u,u,u,u,u,u,u,u]
-; AVX1-ONLY-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm13[0],xmm1[0],xmm13[1],xmm1[1]
-; AVX1-ONLY-NEXT:    vpshufb %xmm10, %xmm8, %xmm10
-; AVX1-ONLY-NEXT:    vpshufb %xmm6, %xmm9, %xmm6
-; AVX1-ONLY-NEXT:    vpor %xmm6, %xmm10, %xmm6
-; AVX1-ONLY-NEXT:    vpblendw {{.*#+}} xmm1 = xmm6[0,1,2],xmm1[3,4,5],xmm6[6,7]
-; AVX1-ONLY-NEXT:    vmovaps {{.*#+}} ymm6 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255]
-; AVX1-ONLY-NEXT:    vandnps %ymm0, %ymm6, %ymm0
-; AVX1-ONLY-NEXT:    vandps %ymm6, %ymm1, %ymm1
-; AVX1-ONLY-NEXT:    vorps %ymm0, %ymm1, %ymm0
-; AVX1-ONLY-NEXT:    vmovaps {{.*#+}} ymm1 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0]
-; AVX1-ONLY-NEXT:    vandps %ymm1, %ymm0, %ymm0
-; AVX1-ONLY-NEXT:    vinsertf128 $1, (%rsp), %ymm0, %ymm10 # 16-byte Folded Reload
-; AVX1-ONLY-NEXT:    vandnps %ymm10, %ymm1, %ymm10
+; AVX1-ONLY-NEXT:    vbroadcastss {{.*#+}} xmm8 = [0,0,5,11,0,0,5,11,0,0,5,11,0,0,5,11]
+; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; AVX1-ONLY-NEXT:    vpshufb %xmm8, %xmm1, %xmm15
+; AVX1-ONLY-NEXT:    vbroadcastss {{.*#+}} xmm1 = [1,7,13,0,1,7,13,0,1,7,13,0,1,7,13,0]
+; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
+; AVX1-ONLY-NEXT:    vpshufb %xmm1, %xmm14, %xmm14
+; AVX1-ONLY-NEXT:    vpunpckldq {{.*#+}} xmm14 = xmm14[0],xmm15[0],xmm14[1],xmm15[1]
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm15 = zero,zero,zero,xmm10[3,9,15,u,u,u,u,u,u,u,u,u,u]
+; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm10 = xmm10[1,7,13],zero,zero,zero,xmm10[u,u,u,u,u,u,u,u,u,u]
+; AVX1-ONLY-NEXT:    vpor %xmm15, %xmm10, %xmm10
+; AVX1-ONLY-NEXT:    vpblendw {{.*#+}} xmm10 = xmm10[0,1,2],xmm14[3,4,5],xmm10[6,7]
+; AVX1-ONLY-NEXT:    vmovaps {{.*#+}} ymm14 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255]
+; AVX1-ONLY-NEXT:    vandnps %ymm0, %ymm14, %ymm0
+; AVX1-ONLY-NEXT:    vandps %ymm14, %ymm10, %ymm10
 ; AVX1-ONLY-NEXT:    vorps %ymm0, %ymm10, %ymm0
-; AVX1-ONLY-NEXT:    vmovups %ymm0, (%rsp) # 32-byte Spill
-; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm7 = [128,128,128,3,9,15,0,0,128,128,128,3,9,15,0,0]
-; AVX1-ONLY-NEXT:    # xmm7 = mem[0,0]
-; AVX1-ONLY-NEXT:    vpshufb %xmm7, %xmm5, %xmm13
-; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm0 = [1,7,13,128,128,128,0,0,1,7,13,128,128,128,0,0]
-; AVX1-ONLY-NEXT:    # xmm0 = mem[0,0]
-; AVX1-ONLY-NEXT:    vpshufb %xmm0, %xmm3, %xmm12
-; AVX1-ONLY-NEXT:    vpor %xmm13, %xmm12, %xmm12
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm13 = xmm2[u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,xmm2[5,11]
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm11 = xmm4[u,u,u,u,u,u,u,u,u,u,u,3,9,15],zero,zero
-; AVX1-ONLY-NEXT:    vpor %xmm13, %xmm11, %xmm11
-; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm12, %ymm11, %ymm11
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm12 = xmm14[u,u,5,11,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm13 = xmm15[u,u,u,u,1,7,13,u,u,u,u,u,u,u,u,u]
-; AVX1-ONLY-NEXT:    vpunpckldq {{.*#+}} xmm12 = xmm13[0],xmm12[0],xmm13[1],xmm12[1]
-; AVX1-ONLY-NEXT:    vpshufb %xmm7, %xmm8, %xmm7
-; AVX1-ONLY-NEXT:    vpshufb %xmm0, %xmm9, %xmm0
-; AVX1-ONLY-NEXT:    vpor %xmm7, %xmm0, %xmm0
-; AVX1-ONLY-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2],xmm12[3,4,5],xmm0[6,7]
-; AVX1-ONLY-NEXT:    vandnps %ymm11, %ymm6, %ymm7
-; AVX1-ONLY-NEXT:    vandps %ymm6, %ymm0, %ymm0
-; AVX1-ONLY-NEXT:    vorps %ymm7, %ymm0, %ymm0
-; AVX1-ONLY-NEXT:    vandps %ymm1, %ymm0, %ymm0
-; AVX1-ONLY-NEXT:    vinsertf128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm6 # 16-byte Folded Reload
-; AVX1-ONLY-NEXT:    vandnps %ymm6, %ymm1, %ymm1
+; AVX1-ONLY-NEXT:    vpshufb %xmm8, %xmm6, %xmm8
+; AVX1-ONLY-NEXT:    vpshufb %xmm1, %xmm5, %xmm1
+; AVX1-ONLY-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm8[0],xmm1[1],xmm8[1]
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm8 = xmm4[u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,xmm4[5,11]
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm10 = xmm2[u,u,u,u,u,u,u,u,u,u,u,3,9,15],zero,zero
+; AVX1-ONLY-NEXT:    vpor %xmm8, %xmm10, %xmm8
+; AVX1-ONLY-NEXT:    vpblendvb %xmm3, %xmm1, %xmm8, %xmm1
+; AVX1-ONLY-NEXT:    vandps %ymm0, %ymm11, %ymm0
+; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1
+; AVX1-ONLY-NEXT:    vandnps %ymm1, %ymm11, %ymm1
 ; AVX1-ONLY-NEXT:    vorps %ymm1, %ymm0, %ymm0
 ; AVX1-ONLY-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm1 = zero,zero,zero,xmm5[4,10,u,u,u,u,u,u,u,u,u,u,u]
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm6 = xmm3[2,8,14],zero,zero,xmm3[u,u,u,u,u,u,u,u,u,u,u]
-; AVX1-ONLY-NEXT:    vpor %xmm1, %xmm6, %xmm1
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm6 = xmm4[u,u,u,u,u,u,u,u,u,u,u,4,10],zero,zero,zero
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm7 = xmm2[u,u,u,u,u,u,u,u,u,u,u],zero,zero,xmm2[0,6,12]
-; AVX1-ONLY-NEXT:    vpor %xmm6, %xmm7, %xmm6
-; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm1, %ymm6, %ymm1
-; AVX1-ONLY-NEXT:    vmovaps {{.*#+}} ymm6 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255]
-; AVX1-ONLY-NEXT:    vandnps %ymm1, %ymm6, %ymm1
-; AVX1-ONLY-NEXT:    vandps {{[-0-9]+}}(%r{{[sb]}}p), %ymm6, %ymm7 # 32-byte Folded Reload
-; AVX1-ONLY-NEXT:    vorps %ymm1, %ymm7, %ymm7
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm0 = zero,zero,zero,xmm9[4,10,u,u,u,u,u,u,u,u,u,u,u]
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm1 = xmm13[2,8,14],zero,zero,xmm13[u,u,u,u,u,u,u,u,u,u,u]
+; AVX1-ONLY-NEXT:    vpor %xmm0, %xmm1, %xmm0
+; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm1 = [0,0,0,4,10,128,128,128,0,0,0,4,10,128,128,128]
+; AVX1-ONLY-NEXT:    # xmm1 = mem[0,0]
+; AVX1-ONLY-NEXT:    vpshufb %xmm1, %xmm7, %xmm10
+; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm11 = [0,0,0,128,128,0,6,12,0,0,0,128,128,0,6,12]
+; AVX1-ONLY-NEXT:    # xmm11 = mem[0,0]
+; AVX1-ONLY-NEXT:    vpshufb %xmm11, %xmm12, %xmm14
+; AVX1-ONLY-NEXT:    vpor %xmm10, %xmm14, %xmm10
+; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm0, %ymm10, %ymm0
+; AVX1-ONLY-NEXT:    vandnps %ymm0, %ymm3, %ymm0
+; AVX1-ONLY-NEXT:    vandps {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm10 # 32-byte Folded Reload
+; AVX1-ONLY-NEXT:    vorps %ymm0, %ymm10, %ymm0
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm10 = xmm5[2,8,14,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm14 = xmm6[u,u,u,u,u,0,6,12,u,u,u,u,u,u,u,u]
+; AVX1-ONLY-NEXT:    vpunpcklqdq {{.*#+}} xmm10 = xmm14[0],xmm10[0]
+; AVX1-ONLY-NEXT:    vpshufb %xmm1, %xmm2, %xmm1
+; AVX1-ONLY-NEXT:    vpshufb %xmm11, %xmm4, %xmm11
+; AVX1-ONLY-NEXT:    vpor %xmm1, %xmm11, %xmm1
+; AVX1-ONLY-NEXT:    vpblendvb %xmm3, %xmm10, %xmm1, %xmm1
 ; AVX1-ONLY-NEXT:    vmovaps {{.*#+}} ymm10 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0]
-; AVX1-ONLY-NEXT:    vandps %ymm7, %ymm10, %ymm7
-; AVX1-ONLY-NEXT:    vinsertf128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm8 # 16-byte Folded Reload
-; AVX1-ONLY-NEXT:    vandnps %ymm8, %ymm10, %ymm8
-; AVX1-ONLY-NEXT:    vorps %ymm7, %ymm8, %ymm13
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm7 = zero,zero,zero,xmm5[5,11,u,u,u,u,u,u,u,u,u,u,u]
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm8 = xmm3[3,9,15],zero,zero,xmm3[u,u,u,u,u,u,u,u,u,u,u]
-; AVX1-ONLY-NEXT:    vpor %xmm7, %xmm8, %xmm7
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm8 = xmm4[u,u,u,u,u,u,u,u,u,u,u,5,11],zero,zero,zero
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm9 = xmm2[u,u,u,u,u,u,u,u,u,u,u],zero,zero,xmm2[1,7,13]
-; AVX1-ONLY-NEXT:    vpor %xmm8, %xmm9, %xmm8
-; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm7, %ymm8, %ymm7
-; AVX1-ONLY-NEXT:    vandnps %ymm7, %ymm6, %ymm7
-; AVX1-ONLY-NEXT:    vandps {{[-0-9]+}}(%r{{[sb]}}p), %ymm6, %ymm6 # 32-byte Folded Reload
-; AVX1-ONLY-NEXT:    vorps %ymm7, %ymm6, %ymm6
-; AVX1-ONLY-NEXT:    vandps %ymm6, %ymm10, %ymm6
-; AVX1-ONLY-NEXT:    vinsertf128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm7 # 16-byte Folded Reload
-; AVX1-ONLY-NEXT:    vandnps %ymm7, %ymm10, %ymm7
-; AVX1-ONLY-NEXT:    vorps %ymm7, %ymm6, %ymm14
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm6 = xmm3[4,10],zero,zero,zero,xmm3[u,u,u,u,u,u,u,u,u,u,u]
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm7 = zero,zero,xmm5[0,6,12,u,u,u,u,u,u,u,u,u,u,u]
-; AVX1-ONLY-NEXT:    vpor %xmm6, %xmm7, %xmm6
-; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm7 = [0,0,128,128,128,2,8,14,0,0,128,128,128,2,8,14]
-; AVX1-ONLY-NEXT:    # xmm7 = mem[0,0]
-; AVX1-ONLY-NEXT:    vpshufb %xmm7, %xmm2, %xmm8
-; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm9 = [0,0,0,6,12,128,128,128,0,0,0,6,12,128,128,128]
-; AVX1-ONLY-NEXT:    # xmm9 = mem[0,0]
-; AVX1-ONLY-NEXT:    vpshufb %xmm9, %xmm4, %xmm11
-; AVX1-ONLY-NEXT:    vpor %xmm8, %xmm11, %xmm8
-; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm6, %ymm8, %ymm6
+; AVX1-ONLY-NEXT:    vandps %ymm0, %ymm10, %ymm0
+; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1
+; AVX1-ONLY-NEXT:    vandnps %ymm1, %ymm10, %ymm1
+; AVX1-ONLY-NEXT:    vorps %ymm1, %ymm0, %ymm11
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm0 = zero,zero,zero,xmm9[5,11,u,u,u,u,u,u,u,u,u,u,u]
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm1 = xmm13[3,9,15],zero,zero,xmm13[u,u,u,u,u,u,u,u,u,u,u]
+; AVX1-ONLY-NEXT:    vpor %xmm0, %xmm1, %xmm0
+; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm1 = [0,0,0,5,11,128,128,128,0,0,0,5,11,128,128,128]
+; AVX1-ONLY-NEXT:    # xmm1 = mem[0,0]
+; AVX1-ONLY-NEXT:    vpshufb %xmm1, %xmm7, %xmm14
+; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm15 = [0,0,0,128,128,1,7,13,0,0,0,128,128,1,7,13]
+; AVX1-ONLY-NEXT:    # xmm15 = mem[0,0]
+; AVX1-ONLY-NEXT:    vpshufb %xmm15, %xmm12, %xmm8
+; AVX1-ONLY-NEXT:    vpor %xmm14, %xmm8, %xmm8
+; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm0, %ymm8, %ymm0
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm8 = xmm5[3,9,15,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm14 = xmm6[u,u,u,u,u,1,7,13,u,u,u,u,u,u,u,u]
+; AVX1-ONLY-NEXT:    vpunpcklqdq {{.*#+}} xmm8 = xmm14[0],xmm8[0]
+; AVX1-ONLY-NEXT:    vpshufb %xmm1, %xmm2, %xmm1
+; AVX1-ONLY-NEXT:    vpshufb %xmm15, %xmm4, %xmm14
+; AVX1-ONLY-NEXT:    vpor %xmm1, %xmm14, %xmm1
+; AVX1-ONLY-NEXT:    vpblendvb %xmm3, %xmm8, %xmm1, %xmm1
+; AVX1-ONLY-NEXT:    vandnps %ymm0, %ymm3, %ymm0
+; AVX1-ONLY-NEXT:    vandps {{[-0-9]+}}(%r{{[sb]}}p), %ymm3, %ymm3 # 32-byte Folded Reload
+; AVX1-ONLY-NEXT:    vorps %ymm0, %ymm3, %ymm0
+; AVX1-ONLY-NEXT:    vandps %ymm0, %ymm10, %ymm0
+; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1
+; AVX1-ONLY-NEXT:    vandnps %ymm1, %ymm10, %ymm1
+; AVX1-ONLY-NEXT:    vorps %ymm1, %ymm0, %ymm3
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm0 = xmm13[4,10],zero,zero,zero,xmm13[u,u,u,u,u,u,u,u,u,u,u]
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm1 = zero,zero,xmm9[0,6,12,u,u,u,u,u,u,u,u,u,u,u]
+; AVX1-ONLY-NEXT:    vpor %xmm0, %xmm1, %xmm0
+; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm1 = [0,0,128,128,128,2,8,14,0,0,128,128,128,2,8,14]
+; AVX1-ONLY-NEXT:    # xmm1 = mem[0,0]
+; AVX1-ONLY-NEXT:    vpshufb %xmm1, %xmm12, %xmm8
+; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm14 = [0,0,0,6,12,128,128,128,0,0,0,6,12,128,128,128]
+; AVX1-ONLY-NEXT:    # xmm14 = mem[0,0]
+; AVX1-ONLY-NEXT:    vpshufb %xmm14, %xmm7, %xmm15
+; AVX1-ONLY-NEXT:    vpor %xmm8, %xmm15, %xmm8
+; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm0, %ymm8, %ymm0
 ; AVX1-ONLY-NEXT:    vmovaps {{.*#+}} ymm8 = [0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
-; AVX1-ONLY-NEXT:    vandnps {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm11 # 32-byte Folded Reload
-; AVX1-ONLY-NEXT:    vandps %ymm6, %ymm8, %ymm6
-; AVX1-ONLY-NEXT:    vorps %ymm6, %ymm11, %ymm6
-; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; AVX1-ONLY-NEXT:    vpshufb %xmm7, %xmm1, %xmm7
-; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX1-ONLY-NEXT:    vpshufb %xmm9, %xmm0, %xmm9
-; AVX1-ONLY-NEXT:    vpor %xmm7, %xmm9, %xmm7
-; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm9 = xmm15[u,u,u,u,u,u,u,u,4,10,u,u,u,u,u,u]
-; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm11 = xmm12[u,u,u,u,u,u,u,u,u,u,u,u,u,2,8,14]
-; AVX1-ONLY-NEXT:    vpunpckhqdq {{.*#+}} xmm9 = xmm11[1],xmm9[1]
-; AVX1-ONLY-NEXT:    vpblendw {{.*#+}} xmm7 = xmm9[0,1,2,3,4],xmm7[5,6,7]
-; AVX1-ONLY-NEXT:    vandps %ymm6, %ymm10, %ymm6
-; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm7, %ymm0, %ymm7
-; AVX1-ONLY-NEXT:    vandnps %ymm7, %ymm10, %ymm7
-; AVX1-ONLY-NEXT:    vorps %ymm7, %ymm6, %ymm6
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[5,11],zero,zero,zero,xmm3[u,u,u,u,u,u,u,u,u,u,u]
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm5 = zero,zero,xmm5[1,7,13,u,u,u,u,u,u,u,u,u,u,u]
-; AVX1-ONLY-NEXT:    vpor %xmm3, %xmm5, %xmm3
-; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm5 = [0,0,128,128,128,3,9,15,0,0,128,128,128,3,9,15]
-; AVX1-ONLY-NEXT:    # xmm5 = mem[0,0]
-; AVX1-ONLY-NEXT:    vpshufb %xmm5, %xmm2, %xmm2
-; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm7 = [0,0,1,7,13,128,128,128,0,0,1,7,13,128,128,128]
-; AVX1-ONLY-NEXT:    # xmm7 = mem[0,0]
-; AVX1-ONLY-NEXT:    vpshufb %xmm7, %xmm4, %xmm4
-; AVX1-ONLY-NEXT:    vpor %xmm2, %xmm4, %xmm2
-; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm2
-; AVX1-ONLY-NEXT:    vandnps {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm3 # 32-byte Folded Reload
-; AVX1-ONLY-NEXT:    vandps %ymm2, %ymm8, %ymm2
-; AVX1-ONLY-NEXT:    vorps %ymm3, %ymm2, %ymm2
-; AVX1-ONLY-NEXT:    vpshufb %xmm5, %xmm1, %xmm3
-; AVX1-ONLY-NEXT:    vpshufb %xmm7, %xmm0, %xmm4
-; AVX1-ONLY-NEXT:    vpor %xmm3, %xmm4, %xmm3
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm4 = xmm15[u,u,u,u,u,u,u,u,5,11,u,u,u,u,u,u]
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm5 = xmm12[u,u,u,u,u,u,u,u,u,u,u,u,u,3,9,15]
+; AVX1-ONLY-NEXT:    vandnps {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm15 # 32-byte Folded Reload
+; AVX1-ONLY-NEXT:    vandps %ymm0, %ymm8, %ymm0
+; AVX1-ONLY-NEXT:    vorps %ymm0, %ymm15, %ymm0
+; AVX1-ONLY-NEXT:    vpshufb %xmm1, %xmm4, %xmm1
+; AVX1-ONLY-NEXT:    vpshufb %xmm14, %xmm2, %xmm14
+; AVX1-ONLY-NEXT:    vpor %xmm1, %xmm14, %xmm1
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm14 = xmm5[u,u,u,u,u,u,u,u,4,10,u,u,u,u,u,u]
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm15 = xmm6[u,u,u,u,u,u,u,u,u,u,u,u,u,2,8,14]
+; AVX1-ONLY-NEXT:    vpunpckhqdq {{.*#+}} xmm14 = xmm15[1],xmm14[1]
+; AVX1-ONLY-NEXT:    vpblendw {{.*#+}} xmm1 = xmm14[0,1,2,3,4],xmm1[5,6,7]
+; AVX1-ONLY-NEXT:    vandps %ymm0, %ymm10, %ymm0
+; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1
+; AVX1-ONLY-NEXT:    vandnps %ymm1, %ymm10, %ymm1
+; AVX1-ONLY-NEXT:    vorps %ymm1, %ymm0, %ymm0
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm1 = xmm13[5,11],zero,zero,zero,xmm13[u,u,u,u,u,u,u,u,u,u,u]
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm9 = zero,zero,xmm9[1,7,13,u,u,u,u,u,u,u,u,u,u,u]
+; AVX1-ONLY-NEXT:    vpor %xmm1, %xmm9, %xmm1
+; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm9 = [0,0,128,128,128,3,9,15,0,0,128,128,128,3,9,15]
+; AVX1-ONLY-NEXT:    # xmm9 = mem[0,0]
+; AVX1-ONLY-NEXT:    vpshufb %xmm9, %xmm12, %xmm12
+; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm13 = [0,0,1,7,13,128,128,128,0,0,1,7,13,128,128,128]
+; AVX1-ONLY-NEXT:    # xmm13 = mem[0,0]
+; AVX1-ONLY-NEXT:    vpshufb %xmm13, %xmm7, %xmm7
+; AVX1-ONLY-NEXT:    vpor %xmm7, %xmm12, %xmm7
+; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm1, %ymm7, %ymm1
+; AVX1-ONLY-NEXT:    vandnps {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm7 # 32-byte Folded Reload
+; AVX1-ONLY-NEXT:    vandps %ymm1, %ymm8, %ymm1
+; AVX1-ONLY-NEXT:    vorps %ymm7, %ymm1, %ymm1
+; AVX1-ONLY-NEXT:    vpshufb %xmm9, %xmm4, %xmm4
+; AVX1-ONLY-NEXT:    vpshufb %xmm13, %xmm2, %xmm2
+; AVX1-ONLY-NEXT:    vpor %xmm4, %xmm2, %xmm2
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm4 = xmm5[u,u,u,u,u,u,u,u,5,11,u,u,u,u,u,u]
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm5 = xmm6[u,u,u,u,u,u,u,u,u,u,u,u,u,3,9,15]
 ; AVX1-ONLY-NEXT:    vpunpckhqdq {{.*#+}} xmm4 = xmm5[1],xmm4[1]
-; AVX1-ONLY-NEXT:    vpblendw {{.*#+}} xmm3 = xmm4[0,1,2,3,4],xmm3[5,6,7]
-; AVX1-ONLY-NEXT:    vandps %ymm2, %ymm10, %ymm2
-; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm3
-; AVX1-ONLY-NEXT:    vandnps %ymm3, %ymm10, %ymm1
-; AVX1-ONLY-NEXT:    vorps %ymm1, %ymm2, %ymm1
-; AVX1-ONLY-NEXT:    vmovups (%rsp), %ymm0 # 32-byte Reload
-; AVX1-ONLY-NEXT:    vmovaps %ymm0, (%rsi)
-; AVX1-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX1-ONLY-NEXT:    vmovaps %ymm0, (%rdx)
-; AVX1-ONLY-NEXT:    vmovaps %ymm13, (%rcx)
-; AVX1-ONLY-NEXT:    vmovaps %ymm14, (%r8)
-; AVX1-ONLY-NEXT:    vmovaps %ymm6, (%r9)
+; AVX1-ONLY-NEXT:    vpblendw {{.*#+}} xmm2 = xmm4[0,1,2,3,4],xmm2[5,6,7]
+; AVX1-ONLY-NEXT:    vandps %ymm1, %ymm10, %ymm1
+; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm2
+; AVX1-ONLY-NEXT:    vandnps %ymm2, %ymm10, %ymm2
+; AVX1-ONLY-NEXT:    vorps %ymm2, %ymm1, %ymm1
+; AVX1-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX1-ONLY-NEXT:    vmovaps %ymm2, (%rsi)
+; AVX1-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX1-ONLY-NEXT:    vmovaps %ymm2, (%rdx)
+; AVX1-ONLY-NEXT:    vmovaps %ymm11, (%rcx)
+; AVX1-ONLY-NEXT:    vmovaps %ymm3, (%r8)
+; AVX1-ONLY-NEXT:    vmovaps %ymm0, (%r9)
 ; AVX1-ONLY-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX1-ONLY-NEXT:    vmovaps %ymm1, (%rax)
-; AVX1-ONLY-NEXT:    addq $168, %rsp
+; AVX1-ONLY-NEXT:    addq $120, %rsp
 ; AVX1-ONLY-NEXT:    vzeroupper
 ; AVX1-ONLY-NEXT:    retq
 ;
@@ -3586,646 +3588,627 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ;
 ; AVX1-ONLY-LABEL: load_i8_stride6_vf64:
 ; AVX1-ONLY:       # %bb.0:
-; AVX1-ONLY-NEXT:    subq $808, %rsp # imm = 0x328
-; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm6 = [0,0,0,128,128,128,4,10,0,0,0,128,128,128,4,10]
-; AVX1-ONLY-NEXT:    # xmm6 = mem[0,0]
-; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm7 = [0,0,0,2,8,14,128,128,0,0,0,2,8,14,128,128]
-; AVX1-ONLY-NEXT:    # xmm7 = mem[0,0]
-; AVX1-ONLY-NEXT:    vbroadcastss {{.*#+}} xmm2 = [0,0,4,10,0,0,4,10,0,0,4,10,0,0,4,10]
-; AVX1-ONLY-NEXT:    vbroadcastss {{.*#+}} xmm3 = [0,6,12,0,0,6,12,0,0,6,12,0,0,6,12,0]
-; AVX1-ONLY-NEXT:    vmovdqa 128(%rdi), %xmm1
-; AVX1-ONLY-NEXT:    vpshufb %xmm2, %xmm1, %xmm0
+; AVX1-ONLY-NEXT:    subq $616, %rsp # imm = 0x268
+; AVX1-ONLY-NEXT:    vmovdqa (%rdi), %xmm2
+; AVX1-ONLY-NEXT:    vmovdqa 16(%rdi), %xmm7
+; AVX1-ONLY-NEXT:    vmovdqa 32(%rdi), %xmm5
+; AVX1-ONLY-NEXT:    vmovdqa 48(%rdi), %xmm6
+; AVX1-ONLY-NEXT:    vmovdqa 224(%rdi), %xmm8
+; AVX1-ONLY-NEXT:    vmovdqa 240(%rdi), %xmm9
+; AVX1-ONLY-NEXT:    vmovdqa 208(%rdi), %xmm10
+; AVX1-ONLY-NEXT:    vmovdqa 192(%rdi), %xmm11
+; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm3 = [128,128,128,4,10,0,0,0,128,128,128,4,10,0,0,0]
+; AVX1-ONLY-NEXT:    # xmm3 = mem[0,0]
+; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm12 = [2,8,14,128,128,0,0,0,2,8,14,128,128,0,0,0]
+; AVX1-ONLY-NEXT:    # xmm12 = mem[0,0]
+; AVX1-ONLY-NEXT:    vbroadcastss {{.*#+}} xmm1 = [2,8,14,0,2,8,14,0,2,8,14,0,2,8,14,0]
+; AVX1-ONLY-NEXT:    vpshufb %xmm1, %xmm6, %xmm0
+; AVX1-ONLY-NEXT:    vmovdqa %xmm1, %xmm13
+; AVX1-ONLY-NEXT:    vbroadcastss {{.*#+}} xmm4 = [0,0,6,12,0,0,6,12,0,0,6,12,0,0,6,12]
+; AVX1-ONLY-NEXT:    vpshufb %xmm4, %xmm5, %xmm1
+; AVX1-ONLY-NEXT:    vmovdqa %xmm5, %xmm15
+; AVX1-ONLY-NEXT:    vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-ONLY-NEXT:    vmovdqa %xmm4, %xmm14
+; AVX1-ONLY-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; AVX1-ONLY-NEXT:    vpshufb %xmm3, %xmm7, %xmm0
 ; AVX1-ONLY-NEXT:    vmovdqa %xmm2, %xmm4
-; AVX1-ONLY-NEXT:    vmovdqa %xmm1, %xmm8
-; AVX1-ONLY-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX1-ONLY-NEXT:    vmovdqa 144(%rdi), %xmm2
-; AVX1-ONLY-NEXT:    vpshufb %xmm3, %xmm2, %xmm1
-; AVX1-ONLY-NEXT:    vmovdqa %xmm3, %xmm5
-; AVX1-ONLY-NEXT:    vmovdqa %xmm2, %xmm10
-; AVX1-ONLY-NEXT:    vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX1-ONLY-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; AVX1-ONLY-NEXT:    vmovdqa 176(%rdi), %xmm2
-; AVX1-ONLY-NEXT:    vpshufb %xmm6, %xmm2, %xmm0
-; AVX1-ONLY-NEXT:    vmovdqa %xmm2, %xmm9
-; AVX1-ONLY-NEXT:    vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX1-ONLY-NEXT:    vmovdqa 160(%rdi), %xmm3
-; AVX1-ONLY-NEXT:    vpshufb %xmm7, %xmm3, %xmm2
-; AVX1-ONLY-NEXT:    vmovdqa %xmm3, %xmm11
-; AVX1-ONLY-NEXT:    vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-ONLY-NEXT:    vpshufb %xmm12, %xmm2, %xmm2
 ; AVX1-ONLY-NEXT:    vpor %xmm0, %xmm2, %xmm2
-; AVX1-ONLY-NEXT:    vmovdqa {{.*#+}} xmm0 = <u,u,u,u,u,u,255,255,255,255,255,0,0,0,0,0>
-; AVX1-ONLY-NEXT:    vpblendvb %xmm0, %xmm1, %xmm2, %xmm1
-; AVX1-ONLY-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX1-ONLY-NEXT:    vmovdqa 320(%rdi), %xmm2
-; AVX1-ONLY-NEXT:    vpshufb %xmm4, %xmm2, %xmm1
-; AVX1-ONLY-NEXT:    vmovdqa %xmm2, %xmm12
-; AVX1-ONLY-NEXT:    vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX1-ONLY-NEXT:    vmovdqa 336(%rdi), %xmm3
-; AVX1-ONLY-NEXT:    vpshufb %xmm5, %xmm3, %xmm2
-; AVX1-ONLY-NEXT:    vmovdqa %xmm3, %xmm13
-; AVX1-ONLY-NEXT:    vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX1-ONLY-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; AVX1-ONLY-NEXT:    vmovdqa 368(%rdi), %xmm3
-; AVX1-ONLY-NEXT:    vpshufb %xmm6, %xmm3, %xmm2
-; AVX1-ONLY-NEXT:    vmovdqa %xmm3, %xmm14
-; AVX1-ONLY-NEXT:    vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX1-ONLY-NEXT:    vmovdqa 352(%rdi), %xmm4
-; AVX1-ONLY-NEXT:    vpshufb %xmm7, %xmm4, %xmm3
-; AVX1-ONLY-NEXT:    vmovdqa %xmm4, %xmm15
-; AVX1-ONLY-NEXT:    vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX1-ONLY-NEXT:    vpor %xmm2, %xmm3, %xmm2
-; AVX1-ONLY-NEXT:    vpblendvb %xmm0, %xmm1, %xmm2, %xmm1
-; AVX1-ONLY-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX1-ONLY-NEXT:    vbroadcastss {{.*#+}} xmm2 = [0,0,5,11,0,0,5,11,0,0,5,11,0,0,5,11]
-; AVX1-ONLY-NEXT:    vbroadcastss {{.*#+}} xmm3 = [1,7,13,0,1,7,13,0,1,7,13,0,1,7,13,0]
-; AVX1-ONLY-NEXT:    vpshufb %xmm2, %xmm8, %xmm1
-; AVX1-ONLY-NEXT:    vmovdqa %xmm2, %xmm4
+; AVX1-ONLY-NEXT:    vmovdqa {{.*#+}} xmm5 = <0,0,0,0,0,255,255,255,255,255,255,u,u,u,u,u>
+; AVX1-ONLY-NEXT:    vpblendvb %xmm5, %xmm1, %xmm2, %xmm1
+; AVX1-ONLY-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX1-ONLY-NEXT:    vpshufb %xmm13, %xmm9, %xmm1
+; AVX1-ONLY-NEXT:    vpshufb %xmm14, %xmm8, %xmm2
+; AVX1-ONLY-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
 ; AVX1-ONLY-NEXT:    vpshufb %xmm3, %xmm10, %xmm2
-; AVX1-ONLY-NEXT:    vmovdqa %xmm3, %xmm5
-; AVX1-ONLY-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm3 = [0,0,0,128,128,128,5,11,0,0,0,128,128,128,5,11]
+; AVX1-ONLY-NEXT:    vpshufb %xmm12, %xmm11, %xmm3
+; AVX1-ONLY-NEXT:    vpor %xmm2, %xmm3, %xmm2
+; AVX1-ONLY-NEXT:    vpblendvb %xmm5, %xmm1, %xmm2, %xmm1
+; AVX1-ONLY-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX1-ONLY-NEXT:    vbroadcastss {{.*#+}} xmm2 = [3,9,15,0,3,9,15,0,3,9,15,0,3,9,15,0]
+; AVX1-ONLY-NEXT:    vpshufb %xmm2, %xmm6, %xmm1
+; AVX1-ONLY-NEXT:    vmovdqa %xmm2, %xmm13
+; AVX1-ONLY-NEXT:    vbroadcastss {{.*#+}} xmm0 = [0,1,7,13,0,1,7,13,0,1,7,13,0,1,7,13]
+; AVX1-ONLY-NEXT:    vpshufb %xmm0, %xmm15, %xmm2
+; AVX1-ONLY-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm3 = [128,128,128,5,11,0,0,0,128,128,128,5,11,0,0,0]
 ; AVX1-ONLY-NEXT:    # xmm3 = mem[0,0]
-; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm6 = [0,0,0,3,9,15,128,128,0,0,0,3,9,15,128,128]
-; AVX1-ONLY-NEXT:    # xmm6 = mem[0,0]
-; AVX1-ONLY-NEXT:    vpshufb %xmm3, %xmm9, %xmm2
-; AVX1-ONLY-NEXT:    vmovdqa %xmm3, %xmm7
-; AVX1-ONLY-NEXT:    vpshufb %xmm6, %xmm11, %xmm3
+; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm12 = [3,9,15,128,128,0,0,0,3,9,15,128,128,0,0,0]
+; AVX1-ONLY-NEXT:    # xmm12 = mem[0,0]
+; AVX1-ONLY-NEXT:    vpshufb %xmm3, %xmm7, %xmm2
+; AVX1-ONLY-NEXT:    vmovdqa %xmm7, %xmm14
+; AVX1-ONLY-NEXT:    vmovdqa %xmm3, %xmm15
+; AVX1-ONLY-NEXT:    vpshufb %xmm12, %xmm4, %xmm3
+; AVX1-ONLY-NEXT:    vmovdqa %xmm4, %xmm7
 ; AVX1-ONLY-NEXT:    vpor %xmm2, %xmm3, %xmm2
-; AVX1-ONLY-NEXT:    vpblendvb %xmm0, %xmm1, %xmm2, %xmm1
-; AVX1-ONLY-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX1-ONLY-NEXT:    vpshufb %xmm4, %xmm12, %xmm1
-; AVX1-ONLY-NEXT:    vpshufb %xmm5, %xmm13, %xmm2
-; AVX1-ONLY-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; AVX1-ONLY-NEXT:    vpshufb %xmm7, %xmm14, %xmm2
-; AVX1-ONLY-NEXT:    vpshufb %xmm6, %xmm15, %xmm3
+; AVX1-ONLY-NEXT:    vpblendvb %xmm5, %xmm1, %xmm2, %xmm1
+; AVX1-ONLY-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX1-ONLY-NEXT:    vpshufb %xmm13, %xmm9, %xmm1
+; AVX1-ONLY-NEXT:    vmovdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-ONLY-NEXT:    vpshufb %xmm0, %xmm8, %xmm2
+; AVX1-ONLY-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX1-ONLY-NEXT:    vpshufb %xmm15, %xmm10, %xmm2
+; AVX1-ONLY-NEXT:    vpshufb %xmm12, %xmm11, %xmm3
 ; AVX1-ONLY-NEXT:    vpor %xmm2, %xmm3, %xmm2
-; AVX1-ONLY-NEXT:    vpblendvb %xmm0, %xmm1, %xmm2, %xmm0
-; AVX1-ONLY-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX1-ONLY-NEXT:    vmovdqa 32(%rdi), %xmm4
-; AVX1-ONLY-NEXT:    vmovdqa 48(%rdi), %xmm11
-; AVX1-ONLY-NEXT:    vbroadcastss {{.*#+}} xmm0 = [2,8,14,0,2,8,14,0,2,8,14,0,2,8,14,0]
-; AVX1-ONLY-NEXT:    vpshufb %xmm0, %xmm11, %xmm2
-; AVX1-ONLY-NEXT:    vmovdqa %xmm0, %xmm8
-; AVX1-ONLY-NEXT:    vmovdqa %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX1-ONLY-NEXT:    vbroadcastss {{.*#+}} xmm15 = [0,0,6,12,0,0,6,12,0,0,6,12,0,0,6,12]
-; AVX1-ONLY-NEXT:    vpshufb %xmm15, %xmm4, %xmm3
-; AVX1-ONLY-NEXT:    vmovdqa %xmm4, %xmm0
-; AVX1-ONLY-NEXT:    vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX1-ONLY-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm2[0]
-; AVX1-ONLY-NEXT:    vmovdqa (%rdi), %xmm10
-; AVX1-ONLY-NEXT:    vmovdqa 16(%rdi), %xmm14
-; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm4 = [128,128,128,4,10,0,0,0,128,128,128,4,10,0,0,0]
-; AVX1-ONLY-NEXT:    # xmm4 = mem[0,0]
-; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm5 = [2,8,14,128,128,0,0,0,2,8,14,128,128,0,0,0]
-; AVX1-ONLY-NEXT:    # xmm5 = mem[0,0]
-; AVX1-ONLY-NEXT:    vpshufb %xmm4, %xmm14, %xmm2
-; AVX1-ONLY-NEXT:    vmovdqa %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX1-ONLY-NEXT:    vmovdqa %xmm4, %xmm12
-; AVX1-ONLY-NEXT:    vpshufb %xmm5, %xmm10, %xmm4
-; AVX1-ONLY-NEXT:    vmovdqa %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX1-ONLY-NEXT:    vmovdqa %xmm5, %xmm6
-; AVX1-ONLY-NEXT:    vpor %xmm2, %xmm4, %xmm4
-; AVX1-ONLY-NEXT:    vmovdqa {{.*#+}} xmm13 = <0,0,0,0,0,255,255,255,255,255,255,u,u,u,u,u>
-; AVX1-ONLY-NEXT:    vpblendvb %xmm13, %xmm3, %xmm4, %xmm3
-; AVX1-ONLY-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX1-ONLY-NEXT:    vmovdqa 224(%rdi), %xmm5
-; AVX1-ONLY-NEXT:    vmovdqa 240(%rdi), %xmm9
-; AVX1-ONLY-NEXT:    vpshufb %xmm8, %xmm9, %xmm3
-; AVX1-ONLY-NEXT:    vmovdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX1-ONLY-NEXT:    vpshufb %xmm15, %xmm5, %xmm4
-; AVX1-ONLY-NEXT:    vmovdqa %xmm5, %xmm8
-; AVX1-ONLY-NEXT:    vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX1-ONLY-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm4[0],xmm3[0]
-; AVX1-ONLY-NEXT:    vmovdqa 208(%rdi), %xmm1
-; AVX1-ONLY-NEXT:    vmovdqa 192(%rdi), %xmm2
-; AVX1-ONLY-NEXT:    vpshufb %xmm12, %xmm1, %xmm4
-; AVX1-ONLY-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX1-ONLY-NEXT:    vpshufb %xmm6, %xmm2, %xmm5
-; AVX1-ONLY-NEXT:    vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX1-ONLY-NEXT:    vpor %xmm4, %xmm5, %xmm4
-; AVX1-ONLY-NEXT:    vpblendvb %xmm13, %xmm3, %xmm4, %xmm3
-; AVX1-ONLY-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX1-ONLY-NEXT:    vbroadcastss {{.*#+}} xmm3 = [3,9,15,0,3,9,15,0,3,9,15,0,3,9,15,0]
-; AVX1-ONLY-NEXT:    vpshufb %xmm3, %xmm11, %xmm5
-; AVX1-ONLY-NEXT:    vmovdqa %xmm3, %xmm11
-; AVX1-ONLY-NEXT:    vbroadcastss {{.*#+}} xmm12 = [0,1,7,13,0,1,7,13,0,1,7,13,0,1,7,13]
-; AVX1-ONLY-NEXT:    vpshufb %xmm12, %xmm0, %xmm6
-; AVX1-ONLY-NEXT:    vpunpcklqdq {{.*#+}} xmm5 = xmm6[0],xmm5[0]
-; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm4 = [128,128,128,5,11,0,0,0,128,128,128,5,11,0,0,0]
-; AVX1-ONLY-NEXT:    # xmm4 = mem[0,0]
-; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm3 = [3,9,15,128,128,0,0,0,3,9,15,128,128,0,0,0]
-; AVX1-ONLY-NEXT:    # xmm3 = mem[0,0]
-; AVX1-ONLY-NEXT:    vpshufb %xmm4, %xmm14, %xmm6
-; AVX1-ONLY-NEXT:    vpshufb %xmm3, %xmm10, %xmm7
-; AVX1-ONLY-NEXT:    vpor %xmm6, %xmm7, %xmm6
-; AVX1-ONLY-NEXT:    vpblendvb %xmm13, %xmm5, %xmm6, %xmm0
-; AVX1-ONLY-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX1-ONLY-NEXT:    vpshufb %xmm11, %xmm9, %xmm5
-; AVX1-ONLY-NEXT:    vmovdqa %xmm11, %xmm14
-; AVX1-ONLY-NEXT:    vpshufb %xmm12, %xmm8, %xmm6
-; AVX1-ONLY-NEXT:    vpunpcklqdq {{.*#+}} xmm5 = xmm6[0],xmm5[0]
-; AVX1-ONLY-NEXT:    vpshufb %xmm4, %xmm1, %xmm6
-; AVX1-ONLY-NEXT:    vpshufb %xmm3, %xmm2, %xmm7
-; AVX1-ONLY-NEXT:    vpor %xmm6, %xmm7, %xmm6
-; AVX1-ONLY-NEXT:    vpblendvb %xmm13, %xmm5, %xmm6, %xmm0
+; AVX1-ONLY-NEXT:    vpblendvb %xmm5, %xmm1, %xmm2, %xmm0
 ; AVX1-ONLY-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
-; AVX1-ONLY-NEXT:    vbroadcastss {{.*#+}} xmm1 = [2,8,14,0,2,8,14,0,2,8,14,0,2,8,14,0]
-; AVX1-ONLY-NEXT:    vpshufb %xmm1, %xmm9, %xmm2
-; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
-; AVX1-ONLY-NEXT:    vpshufb %xmm15, %xmm8, %xmm5
-; AVX1-ONLY-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm5[0],xmm2[0]
-; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm7 = [0,0,0,4,10,128,128,128,0,0,0,4,10,128,128,128]
-; AVX1-ONLY-NEXT:    # xmm7 = mem[0,0]
-; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm3 = [0,0,0,128,128,0,6,12,0,0,0,128,128,0,6,12]
-; AVX1-ONLY-NEXT:    # xmm3 = mem[0,0]
-; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
-; AVX1-ONLY-NEXT:    vpshufb %xmm7, %xmm11, %xmm5
-; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
-; AVX1-ONLY-NEXT:    vpshufb %xmm3, %xmm10, %xmm6
-; AVX1-ONLY-NEXT:    vpor %xmm5, %xmm6, %xmm5
-; AVX1-ONLY-NEXT:    vmovdqa {{.*#+}} xmm6 = <u,u,u,u,u,255,255,255,255,255,255,0,0,0,0,0>
-; AVX1-ONLY-NEXT:    vpblendvb %xmm6, %xmm2, %xmm5, %xmm0
-; AVX1-ONLY-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
-; AVX1-ONLY-NEXT:    vpshufb %xmm1, %xmm13, %xmm0
-; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
-; AVX1-ONLY-NEXT:    vpshufb %xmm15, %xmm5, %xmm1
-; AVX1-ONLY-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
-; AVX1-ONLY-NEXT:    vpshufb %xmm7, %xmm15, %xmm1
-; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
-; AVX1-ONLY-NEXT:    vpshufb %xmm3, %xmm7, %xmm2
-; AVX1-ONLY-NEXT:    vpor %xmm1, %xmm2, %xmm1
-; AVX1-ONLY-NEXT:    vpblendvb %xmm6, %xmm0, %xmm1, %xmm0
-; AVX1-ONLY-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX1-ONLY-NEXT:    vpshufb %xmm14, %xmm9, %xmm0
-; AVX1-ONLY-NEXT:    vpshufb %xmm12, %xmm8, %xmm1
-; AVX1-ONLY-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm2 = [0,0,0,5,11,128,128,128,0,0,0,5,11,128,128,128]
-; AVX1-ONLY-NEXT:    # xmm2 = mem[0,0]
-; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm8 = [0,0,0,128,128,1,7,13,0,0,0,128,128,1,7,13]
-; AVX1-ONLY-NEXT:    # xmm8 = mem[0,0]
-; AVX1-ONLY-NEXT:    vpshufb %xmm2, %xmm11, %xmm1
-; AVX1-ONLY-NEXT:    vmovdqa %xmm2, %xmm3
-; AVX1-ONLY-NEXT:    vpshufb %xmm8, %xmm10, %xmm2
-; AVX1-ONLY-NEXT:    vpor %xmm1, %xmm2, %xmm1
-; AVX1-ONLY-NEXT:    vpblendvb %xmm6, %xmm0, %xmm1, %xmm0
-; AVX1-ONLY-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX1-ONLY-NEXT:    vpshufb %xmm14, %xmm13, %xmm0
-; AVX1-ONLY-NEXT:    vpshufb %xmm12, %xmm5, %xmm1
-; AVX1-ONLY-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX1-ONLY-NEXT:    vpshufb %xmm3, %xmm15, %xmm1
-; AVX1-ONLY-NEXT:    vpshufb %xmm8, %xmm7, %xmm2
-; AVX1-ONLY-NEXT:    vpor %xmm1, %xmm2, %xmm1
-; AVX1-ONLY-NEXT:    vpblendvb %xmm6, %xmm0, %xmm1, %xmm0
-; AVX1-ONLY-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm1 = [4,10,128,128,128,0,0,0,4,10,128,128,128,0,0,0]
 ; AVX1-ONLY-NEXT:    # xmm1 = mem[0,0]
-; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
-; AVX1-ONLY-NEXT:    vpshufb %xmm1, %xmm10, %xmm0
-; AVX1-ONLY-NEXT:    vmovdqa %xmm1, %xmm5
+; AVX1-ONLY-NEXT:    vpshufb %xmm1, %xmm4, %xmm0
+; AVX1-ONLY-NEXT:    vmovdqa %xmm1, %xmm12
 ; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm2 = [128,128,0,6,12,0,0,0,128,128,0,6,12,0,0,0]
 ; AVX1-ONLY-NEXT:    # xmm2 = mem[0,0]
-; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
-; AVX1-ONLY-NEXT:    vpshufb %xmm2, %xmm4, %xmm1
-; AVX1-ONLY-NEXT:    vmovdqa %xmm2, %xmm9
+; AVX1-ONLY-NEXT:    vpshufb %xmm2, %xmm14, %xmm1
+; AVX1-ONLY-NEXT:    vmovdqa %xmm2, %xmm13
 ; AVX1-ONLY-NEXT:    vpor %xmm0, %xmm1, %xmm1
 ; AVX1-ONLY-NEXT:    vbroadcastss {{.*#+}} xmm2 = [4,10,0,0,4,10,0,0,4,10,0,0,4,10,0,0]
-; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
-; AVX1-ONLY-NEXT:    vpshufb %xmm2, %xmm12, %xmm0
-; AVX1-ONLY-NEXT:    vmovdqa %xmm2, %xmm6
+; AVX1-ONLY-NEXT:    vpshufb %xmm2, %xmm6, %xmm0
+; AVX1-ONLY-NEXT:    vmovdqa %xmm2, %xmm4
 ; AVX1-ONLY-NEXT:    vbroadcastss {{.*#+}} xmm3 = [0,2,8,14,0,2,8,14,0,2,8,14,0,2,8,14]
-; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
-; AVX1-ONLY-NEXT:    vpshufb %xmm3, %xmm14, %xmm2
+; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
+; AVX1-ONLY-NEXT:    vpshufb %xmm3, %xmm5, %xmm2
 ; AVX1-ONLY-NEXT:    vpunpckhqdq {{.*#+}} xmm2 = xmm2[1],xmm0[1]
 ; AVX1-ONLY-NEXT:    vmovdqa {{.*#+}} xmm0 = <255,255,255,255,255,0,0,0,0,0,u,u,u,u,u,u>
 ; AVX1-ONLY-NEXT:    vpblendvb %xmm0, %xmm1, %xmm2, %xmm1
 ; AVX1-ONLY-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
-; AVX1-ONLY-NEXT:    vpshufb %xmm5, %xmm15, %xmm1
-; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
-; AVX1-ONLY-NEXT:    vpshufb %xmm9, %xmm8, %xmm2
+; AVX1-ONLY-NEXT:    vpshufb %xmm12, %xmm11, %xmm1
+; AVX1-ONLY-NEXT:    vpshufb %xmm13, %xmm10, %xmm2
+; AVX1-ONLY-NEXT:    vmovdqa %xmm10, %xmm12
 ; AVX1-ONLY-NEXT:    vpor %xmm1, %xmm2, %xmm1
-; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
-; AVX1-ONLY-NEXT:    vpshufb %xmm6, %xmm7, %xmm2
-; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; AVX1-ONLY-NEXT:    vpshufb %xmm3, %xmm6, %xmm3
+; AVX1-ONLY-NEXT:    vpshufb %xmm4, %xmm9, %xmm2
+; AVX1-ONLY-NEXT:    vmovdqa %xmm9, %xmm15
+; AVX1-ONLY-NEXT:    vmovdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-ONLY-NEXT:    vpshufb %xmm3, %xmm8, %xmm3
 ; AVX1-ONLY-NEXT:    vpunpckhqdq {{.*#+}} xmm2 = xmm3[1],xmm2[1]
 ; AVX1-ONLY-NEXT:    vpblendvb %xmm0, %xmm1, %xmm2, %xmm1
 ; AVX1-ONLY-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm2 = [5,11,128,128,128,0,0,0,5,11,128,128,128,0,0,0]
 ; AVX1-ONLY-NEXT:    # xmm2 = mem[0,0]
-; AVX1-ONLY-NEXT:    vpshufb %xmm2, %xmm10, %xmm1
-; AVX1-ONLY-NEXT:    vmovdqa %xmm2, %xmm9
+; AVX1-ONLY-NEXT:    vmovdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-ONLY-NEXT:    vpshufb %xmm2, %xmm7, %xmm1
+; AVX1-ONLY-NEXT:    vmovdqa %xmm2, %xmm8
 ; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm3 = [128,128,1,7,13,0,0,0,128,128,1,7,13,0,0,0]
 ; AVX1-ONLY-NEXT:    # xmm3 = mem[0,0]
-; AVX1-ONLY-NEXT:    vpshufb %xmm3, %xmm4, %xmm2
-; AVX1-ONLY-NEXT:    vmovdqa %xmm3, %xmm10
-; AVX1-ONLY-NEXT:    vmovdqa %xmm4, %xmm13
+; AVX1-ONLY-NEXT:    vmovdqa %xmm14, %xmm13
+; AVX1-ONLY-NEXT:    vpshufb %xmm3, %xmm14, %xmm2
+; AVX1-ONLY-NEXT:    vmovdqa %xmm3, %xmm14
 ; AVX1-ONLY-NEXT:    vpor %xmm1, %xmm2, %xmm1
 ; AVX1-ONLY-NEXT:    vbroadcastss {{.*#+}} xmm3 = [5,11,0,0,5,11,0,0,5,11,0,0,5,11,0,0]
-; AVX1-ONLY-NEXT:    vpshufb %xmm3, %xmm12, %xmm2
-; AVX1-ONLY-NEXT:    vmovdqa %xmm3, %xmm11
+; AVX1-ONLY-NEXT:    vpshufb %xmm3, %xmm6, %xmm2
+; AVX1-ONLY-NEXT:    vmovdqa %xmm3, %xmm9
+; AVX1-ONLY-NEXT:    vmovdqa %xmm6, %xmm10
+; AVX1-ONLY-NEXT:    vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX1-ONLY-NEXT:    vbroadcastss {{.*#+}} xmm4 = [0,3,9,15,0,3,9,15,0,3,9,15,0,3,9,15]
-; AVX1-ONLY-NEXT:    vpshufb %xmm4, %xmm14, %xmm3
+; AVX1-ONLY-NEXT:    vpshufb %xmm4, %xmm5, %xmm3
 ; AVX1-ONLY-NEXT:    vpunpckhqdq {{.*#+}} xmm2 = xmm3[1],xmm2[1]
 ; AVX1-ONLY-NEXT:    vpblendvb %xmm0, %xmm1, %xmm2, %xmm1
 ; AVX1-ONLY-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX1-ONLY-NEXT:    vpshufb %xmm9, %xmm15, %xmm1
-; AVX1-ONLY-NEXT:    vpshufb %xmm10, %xmm8, %xmm2
+; AVX1-ONLY-NEXT:    vpshufb %xmm8, %xmm11, %xmm1
+; AVX1-ONLY-NEXT:    vmovdqa %xmm11, %xmm8
+; AVX1-ONLY-NEXT:    vmovdqa %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-ONLY-NEXT:    vpshufb %xmm14, %xmm12, %xmm2
+; AVX1-ONLY-NEXT:    vmovdqa %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX1-ONLY-NEXT:    vpor %xmm1, %xmm2, %xmm1
-; AVX1-ONLY-NEXT:    vpshufb %xmm11, %xmm7, %xmm2
+; AVX1-ONLY-NEXT:    vpshufb %xmm9, %xmm15, %xmm2
+; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
 ; AVX1-ONLY-NEXT:    vpshufb %xmm4, %xmm6, %xmm3
 ; AVX1-ONLY-NEXT:    vpunpckhqdq {{.*#+}} xmm2 = xmm3[1],xmm2[1]
 ; AVX1-ONLY-NEXT:    vpblendvb %xmm0, %xmm1, %xmm2, %xmm0
 ; AVX1-ONLY-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm1 = [128,128,128,2,8,14,0,0,128,128,128,2,8,14,0,0]
-; AVX1-ONLY-NEXT:    # xmm1 = mem[0,0]
+; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm3 = [128,128,128,2,8,14,0,0,128,128,128,2,8,14,0,0]
+; AVX1-ONLY-NEXT:    # xmm3 = mem[0,0]
 ; AVX1-ONLY-NEXT:    vmovdqa 112(%rdi), %xmm0
-; AVX1-ONLY-NEXT:    vmovdqa %xmm0, (%rsp) # 16-byte Spill
-; AVX1-ONLY-NEXT:    vpshufb %xmm1, %xmm0, %xmm0
-; AVX1-ONLY-NEXT:    vmovdqa %xmm1, %xmm3
-; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm2 = [0,6,12,128,128,128,0,0,0,6,12,128,128,128,0,0]
-; AVX1-ONLY-NEXT:    # xmm2 = mem[0,0]
+; AVX1-ONLY-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-ONLY-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
+; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm15 = [0,6,12,128,128,128,0,0,0,6,12,128,128,128,0,0]
+; AVX1-ONLY-NEXT:    # xmm15 = mem[0,0]
 ; AVX1-ONLY-NEXT:    vmovdqa 96(%rdi), %xmm1
 ; AVX1-ONLY-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX1-ONLY-NEXT:    vpshufb %xmm2, %xmm1, %xmm1
-; AVX1-ONLY-NEXT:    vmovdqa %xmm2, %xmm5
+; AVX1-ONLY-NEXT:    vpshufb %xmm15, %xmm1, %xmm1
 ; AVX1-ONLY-NEXT:    vpor %xmm0, %xmm1, %xmm0
-; AVX1-ONLY-NEXT:    vmovdqa 80(%rdi), %xmm2
-; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm10 = [0,0,0,128,128,128,4,10,0,0,0,128,128,128,4,10]
-; AVX1-ONLY-NEXT:    # xmm10 = mem[0,0]
-; AVX1-ONLY-NEXT:    vpshufb %xmm10, %xmm2, %xmm1
-; AVX1-ONLY-NEXT:    vmovdqa %xmm2, %xmm9
-; AVX1-ONLY-NEXT:    vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm4 = [0,0,0,128,128,128,4,10,0,0,0,128,128,128,4,10]
+; AVX1-ONLY-NEXT:    # xmm4 = mem[0,0]
+; AVX1-ONLY-NEXT:    vmovdqa 80(%rdi), %xmm1
+; AVX1-ONLY-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-ONLY-NEXT:    vpshufb %xmm4, %xmm1, %xmm1
+; AVX1-ONLY-NEXT:    vmovdqa %xmm4, %xmm11
 ; AVX1-ONLY-NEXT:    vmovdqa 64(%rdi), %xmm2
 ; AVX1-ONLY-NEXT:    vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm11 = [0,0,0,2,8,14,128,128,0,0,0,2,8,14,128,128]
-; AVX1-ONLY-NEXT:    # xmm11 = mem[0,0]
-; AVX1-ONLY-NEXT:    vpshufb %xmm11, %xmm2, %xmm4
-; AVX1-ONLY-NEXT:    vpor %xmm1, %xmm4, %xmm1
-; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm0 = xmm14[u,u,4,10,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm4 = xmm12[u,u,u,u,0,6,12,u,u,u,u,u,u,u,u,u]
-; AVX1-ONLY-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm4[0],xmm0[0],xmm4[1],xmm0[1]
-; AVX1-ONLY-NEXT:    vpshufb %xmm3, %xmm13, %xmm4
-; AVX1-ONLY-NEXT:    vmovdqa %xmm3, %xmm14
-; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
-; AVX1-ONLY-NEXT:    vmovdqa %xmm5, %xmm12
-; AVX1-ONLY-NEXT:    vpshufb %xmm5, %xmm2, %xmm5
-; AVX1-ONLY-NEXT:    vpor %xmm4, %xmm5, %xmm4
-; AVX1-ONLY-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1,2],xmm0[3,4,5],xmm4[6,7]
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,u,u,u,u,u,u,u,u,u,u,2,8,14],zero,zero
+; AVX1-ONLY-NEXT:    vpor %xmm1, %xmm2, %xmm1
+; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm2
+; AVX1-ONLY-NEXT:    vbroadcastss {{.*#+}} xmm14 = [0,0,4,10,0,0,4,10,0,0,4,10,0,0,4,10]
+; AVX1-ONLY-NEXT:    vpshufb %xmm14, %xmm5, %xmm4
+; AVX1-ONLY-NEXT:    vbroadcastss {{.*#+}} xmm1 = [0,6,12,0,0,6,12,0,0,6,12,0,0,6,12,0]
+; AVX1-ONLY-NEXT:    vpshufb %xmm1, %xmm10, %xmm5
+; AVX1-ONLY-NEXT:    vpunpckldq {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
+; AVX1-ONLY-NEXT:    vmovdqa %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-ONLY-NEXT:    vpshufb %xmm3, %xmm13, %xmm5
+; AVX1-ONLY-NEXT:    vpshufb %xmm15, %xmm7, %xmm9
+; AVX1-ONLY-NEXT:    vpor %xmm5, %xmm9, %xmm5
+; AVX1-ONLY-NEXT:    vpblendw {{.*#+}} xmm4 = xmm5[0,1,2],xmm4[3,4,5],xmm5[6,7]
 ; AVX1-ONLY-NEXT:    vmovaps {{.*#+}} ymm13 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255]
-; AVX1-ONLY-NEXT:    vandnps %ymm1, %ymm13, %ymm1
+; AVX1-ONLY-NEXT:    vandnps %ymm2, %ymm13, %ymm2
 ; AVX1-ONLY-NEXT:    vandps %ymm4, %ymm13, %ymm4
-; AVX1-ONLY-NEXT:    vorps %ymm1, %ymm4, %ymm4
-; AVX1-ONLY-NEXT:    vmovaps {{.*#+}} ymm3 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0]
-; AVX1-ONLY-NEXT:    vandps %ymm3, %ymm4, %ymm4
-; AVX1-ONLY-NEXT:    vinsertf128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm5 # 16-byte Folded Reload
-; AVX1-ONLY-NEXT:    vandnps %ymm5, %ymm3, %ymm5
-; AVX1-ONLY-NEXT:    vorps %ymm5, %ymm4, %ymm0
-; AVX1-ONLY-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX1-ONLY-NEXT:    vmovdqa 304(%rdi), %xmm0
-; AVX1-ONLY-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX1-ONLY-NEXT:    vpshufb %xmm14, %xmm0, %xmm4
-; AVX1-ONLY-NEXT:    vmovdqa 288(%rdi), %xmm0
-; AVX1-ONLY-NEXT:    vpshufb %xmm12, %xmm0, %xmm5
-; AVX1-ONLY-NEXT:    vmovdqa %xmm0, %xmm12
-; AVX1-ONLY-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX1-ONLY-NEXT:    vpor %xmm4, %xmm5, %xmm1
-; AVX1-ONLY-NEXT:    vmovdqa 272(%rdi), %xmm0
-; AVX1-ONLY-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX1-ONLY-NEXT:    vpshufb %xmm10, %xmm0, %xmm0
-; AVX1-ONLY-NEXT:    vmovdqa 256(%rdi), %xmm2
+; AVX1-ONLY-NEXT:    vorps %ymm2, %ymm4, %ymm9
+; AVX1-ONLY-NEXT:    vmovdqa 128(%rdi), %xmm2
+; AVX1-ONLY-NEXT:    vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-ONLY-NEXT:    vpshufb %xmm14, %xmm2, %xmm2
+; AVX1-ONLY-NEXT:    vmovdqa 144(%rdi), %xmm4
+; AVX1-ONLY-NEXT:    vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-ONLY-NEXT:    vpshufb %xmm1, %xmm4, %xmm4
+; AVX1-ONLY-NEXT:    vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
+; AVX1-ONLY-NEXT:    vmovdqa 176(%rdi), %xmm2
+; AVX1-ONLY-NEXT:    vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-ONLY-NEXT:    vpshufb %xmm11, %xmm2, %xmm2
+; AVX1-ONLY-NEXT:    vmovdqa 160(%rdi), %xmm5
+; AVX1-ONLY-NEXT:    vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm0 = [0,0,0,2,8,14,128,128,0,0,0,2,8,14,128,128]
+; AVX1-ONLY-NEXT:    # xmm0 = mem[0,0]
+; AVX1-ONLY-NEXT:    vpshufb %xmm0, %xmm5, %xmm10
+; AVX1-ONLY-NEXT:    vpor %xmm2, %xmm10, %xmm10
+; AVX1-ONLY-NEXT:    vmovdqa {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255]
+; AVX1-ONLY-NEXT:    vpblendvb %xmm2, %xmm4, %xmm10, %xmm10
+; AVX1-ONLY-NEXT:    vmovdqa %ymm2, %ymm5
+; AVX1-ONLY-NEXT:    vmovaps {{.*#+}} ymm2 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0]
+; AVX1-ONLY-NEXT:    vandps %ymm2, %ymm9, %ymm9
+; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm10, %ymm0, %ymm10
+; AVX1-ONLY-NEXT:    vandnps %ymm10, %ymm2, %ymm10
+; AVX1-ONLY-NEXT:    vmovaps %ymm2, %ymm7
+; AVX1-ONLY-NEXT:    vorps %ymm10, %ymm9, %ymm4
+; AVX1-ONLY-NEXT:    vmovups %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX1-ONLY-NEXT:    vmovdqa 304(%rdi), %xmm2
+; AVX1-ONLY-NEXT:    vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-ONLY-NEXT:    vpshufb %xmm3, %xmm2, %xmm9
+; AVX1-ONLY-NEXT:    vmovdqa 288(%rdi), %xmm2
+; AVX1-ONLY-NEXT:    vmovdqa %xmm2, (%rsp) # 16-byte Spill
+; AVX1-ONLY-NEXT:    vpshufb %xmm15, %xmm2, %xmm10
+; AVX1-ONLY-NEXT:    vpor %xmm9, %xmm10, %xmm9
+; AVX1-ONLY-NEXT:    vmovdqa 272(%rdi), %xmm2
 ; AVX1-ONLY-NEXT:    vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
 ; AVX1-ONLY-NEXT:    vpshufb %xmm11, %xmm2, %xmm10
-; AVX1-ONLY-NEXT:    vpor %xmm0, %xmm10, %xmm0
-; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm1 = xmm6[u,u,4,10,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm10 = xmm7[u,u,u,u,0,6,12,u,u,u,u,u,u,u,u,u]
-; AVX1-ONLY-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm10[0],xmm1[0],xmm10[1],xmm1[1]
-; AVX1-ONLY-NEXT:    vpshufb %xmm14, %xmm8, %xmm7
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm6 = xmm15[0,6,12],zero,zero,zero,xmm15[u,u,u,u,u,u,u,u,u,u]
-; AVX1-ONLY-NEXT:    vpor %xmm7, %xmm6, %xmm6
-; AVX1-ONLY-NEXT:    vpblendw {{.*#+}} xmm1 = xmm6[0,1,2],xmm1[3,4,5],xmm6[6,7]
-; AVX1-ONLY-NEXT:    vandnps %ymm0, %ymm13, %ymm0
-; AVX1-ONLY-NEXT:    vandps %ymm1, %ymm13, %ymm1
+; AVX1-ONLY-NEXT:    vmovdqa 256(%rdi), %xmm2
+; AVX1-ONLY-NEXT:    vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-ONLY-NEXT:    vpshufb %xmm0, %xmm2, %xmm11
+; AVX1-ONLY-NEXT:    vpor %xmm10, %xmm11, %xmm10
+; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm9, %ymm10, %ymm9
+; AVX1-ONLY-NEXT:    vpshufb %xmm14, %xmm6, %xmm10
+; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
+; AVX1-ONLY-NEXT:    vpshufb %xmm1, %xmm2, %xmm11
+; AVX1-ONLY-NEXT:    vpunpckldq {{.*#+}} xmm10 = xmm11[0],xmm10[0],xmm11[1],xmm10[1]
+; AVX1-ONLY-NEXT:    vpshufb %xmm3, %xmm12, %xmm3
+; AVX1-ONLY-NEXT:    vpshufb %xmm15, %xmm8, %xmm6
+; AVX1-ONLY-NEXT:    vpor %xmm3, %xmm6, %xmm3
+; AVX1-ONLY-NEXT:    vpblendw {{.*#+}} xmm3 = xmm3[0,1,2],xmm10[3,4,5],xmm3[6,7]
+; AVX1-ONLY-NEXT:    vandnps %ymm9, %ymm13, %ymm6
+; AVX1-ONLY-NEXT:    vandps %ymm3, %ymm13, %ymm3
+; AVX1-ONLY-NEXT:    vmovaps %ymm13, %ymm11
+; AVX1-ONLY-NEXT:    vorps %ymm6, %ymm3, %ymm3
+; AVX1-ONLY-NEXT:    vmovdqa 320(%rdi), %xmm4
+; AVX1-ONLY-NEXT:    vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-ONLY-NEXT:    vpshufb %xmm14, %xmm4, %xmm0
+; AVX1-ONLY-NEXT:    vmovdqa 336(%rdi), %xmm4
+; AVX1-ONLY-NEXT:    vpshufb %xmm1, %xmm4, %xmm1
+; AVX1-ONLY-NEXT:    vmovdqa %xmm4, %xmm10
+; AVX1-ONLY-NEXT:    vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-ONLY-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; AVX1-ONLY-NEXT:    vmovdqa 368(%rdi), %xmm1
+; AVX1-ONLY-NEXT:    vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,xmm1[4,10]
+; AVX1-ONLY-NEXT:    vmovdqa 352(%rdi), %xmm4
+; AVX1-ONLY-NEXT:    vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm6 = xmm4[u,u,u,u,u,u,u,u,u,u,u,2,8,14],zero,zero
+; AVX1-ONLY-NEXT:    vpor %xmm1, %xmm6, %xmm1
+; AVX1-ONLY-NEXT:    vpblendvb %xmm5, %xmm0, %xmm1, %xmm0
+; AVX1-ONLY-NEXT:    vandps %ymm7, %ymm3, %ymm1
+; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
+; AVX1-ONLY-NEXT:    vandnps %ymm0, %ymm7, %ymm0
 ; AVX1-ONLY-NEXT:    vorps %ymm0, %ymm1, %ymm0
-; AVX1-ONLY-NEXT:    vandps %ymm3, %ymm0, %ymm0
-; AVX1-ONLY-NEXT:    vinsertf128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 16-byte Folded Reload
-; AVX1-ONLY-NEXT:    vandnps %ymm1, %ymm3, %ymm1
-; AVX1-ONLY-NEXT:    vorps %ymm1, %ymm0, %ymm0
 ; AVX1-ONLY-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm0 = [128,128,128,3,9,15,0,0,128,128,128,3,9,15,0,0]
-; AVX1-ONLY-NEXT:    # xmm0 = mem[0,0]
-; AVX1-ONLY-NEXT:    vmovdqa (%rsp), %xmm14 # 16-byte Reload
-; AVX1-ONLY-NEXT:    vpshufb %xmm0, %xmm14, %xmm1
-; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm6 = [1,7,13,128,128,128,0,0,1,7,13,128,128,128,0,0]
-; AVX1-ONLY-NEXT:    # xmm6 = mem[0,0]
-; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
-; AVX1-ONLY-NEXT:    vpshufb %xmm6, %xmm15, %xmm7
-; AVX1-ONLY-NEXT:    vpor %xmm1, %xmm7, %xmm1
-; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm8 = [0,0,0,128,128,128,5,11,0,0,0,128,128,128,5,11]
+; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm8 = [128,128,128,3,9,15,0,0,128,128,128,3,9,15,0,0]
 ; AVX1-ONLY-NEXT:    # xmm8 = mem[0,0]
-; AVX1-ONLY-NEXT:    vpshufb %xmm8, %xmm9, %xmm7
+; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX1-ONLY-NEXT:    vpshufb %xmm8, %xmm0, %xmm0
+; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm4 = [1,7,13,128,128,128,0,0,1,7,13,128,128,128,0,0]
+; AVX1-ONLY-NEXT:    # xmm4 = mem[0,0]
+; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; AVX1-ONLY-NEXT:    vpshufb %xmm4, %xmm1, %xmm1
+; AVX1-ONLY-NEXT:    vpor %xmm0, %xmm1, %xmm0
+; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,xmm1[5,11]
 ; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm9 = [0,0,0,3,9,15,128,128,0,0,0,3,9,15,128,128]
 ; AVX1-ONLY-NEXT:    # xmm9 = mem[0,0]
-; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
-; AVX1-ONLY-NEXT:    vpshufb %xmm9, %xmm2, %xmm10
-; AVX1-ONLY-NEXT:    vpor %xmm7, %xmm10, %xmm7
-; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm1, %ymm7, %ymm1
-; AVX1-ONLY-NEXT:    vbroadcastss {{.*#+}} xmm4 = [0,0,5,11,0,0,5,11,0,0,5,11,0,0,5,11]
 ; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
-; AVX1-ONLY-NEXT:    vpshufb %xmm4, %xmm5, %xmm7
-; AVX1-ONLY-NEXT:    vbroadcastss {{.*#+}} xmm5 = [1,7,13,0,1,7,13,0,1,7,13,0,1,7,13,0]
-; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
-; AVX1-ONLY-NEXT:    vpshufb %xmm5, %xmm10, %xmm10
-; AVX1-ONLY-NEXT:    vpunpckldq {{.*#+}} xmm7 = xmm10[0],xmm7[0],xmm10[1],xmm7[1]
-; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
-; AVX1-ONLY-NEXT:    vpshufb %xmm0, %xmm10, %xmm10
-; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
-; AVX1-ONLY-NEXT:    vpshufb %xmm6, %xmm11, %xmm11
-; AVX1-ONLY-NEXT:    vpor %xmm10, %xmm11, %xmm10
-; AVX1-ONLY-NEXT:    vpblendw {{.*#+}} xmm7 = xmm10[0,1,2],xmm7[3,4,5],xmm10[6,7]
-; AVX1-ONLY-NEXT:    vandnps %ymm1, %ymm13, %ymm1
-; AVX1-ONLY-NEXT:    vandps %ymm7, %ymm13, %ymm7
-; AVX1-ONLY-NEXT:    vorps %ymm1, %ymm7, %ymm1
-; AVX1-ONLY-NEXT:    vandps %ymm3, %ymm1, %ymm1
-; AVX1-ONLY-NEXT:    vinsertf128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm7 # 16-byte Folded Reload
-; AVX1-ONLY-NEXT:    vandnps %ymm7, %ymm3, %ymm7
-; AVX1-ONLY-NEXT:    vorps %ymm7, %ymm1, %ymm1
-; AVX1-ONLY-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
-; AVX1-ONLY-NEXT:    vpshufb %xmm0, %xmm11, %xmm1
-; AVX1-ONLY-NEXT:    vpshufb %xmm6, %xmm12, %xmm7
-; AVX1-ONLY-NEXT:    vpor %xmm1, %xmm7, %xmm1
+; AVX1-ONLY-NEXT:    vpshufb %xmm9, %xmm5, %xmm14
+; AVX1-ONLY-NEXT:    vpor %xmm1, %xmm14, %xmm1
+; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm14
+; AVX1-ONLY-NEXT:    vbroadcastss {{.*#+}} xmm3 = [0,0,5,11,0,0,5,11,0,0,5,11,0,0,5,11]
+; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX1-ONLY-NEXT:    vpshufb %xmm3, %xmm0, %xmm0
+; AVX1-ONLY-NEXT:    vbroadcastss {{.*#+}} xmm7 = [1,7,13,0,1,7,13,0,1,7,13,0,1,7,13,0]
+; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
+; AVX1-ONLY-NEXT:    vpshufb %xmm7, %xmm6, %xmm15
+; AVX1-ONLY-NEXT:    vpunpckldq {{.*#+}} xmm0 = xmm15[0],xmm0[0],xmm15[1],xmm0[1]
+; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
+; AVX1-ONLY-NEXT:    vpshufb %xmm8, %xmm6, %xmm15
+; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
+; AVX1-ONLY-NEXT:    vpshufb %xmm4, %xmm6, %xmm13
+; AVX1-ONLY-NEXT:    vpor %xmm15, %xmm13, %xmm13
+; AVX1-ONLY-NEXT:    vpblendw {{.*#+}} xmm0 = xmm13[0,1,2],xmm0[3,4,5],xmm13[6,7]
+; AVX1-ONLY-NEXT:    vandnps %ymm14, %ymm11, %ymm13
+; AVX1-ONLY-NEXT:    vandps %ymm0, %ymm11, %ymm0
+; AVX1-ONLY-NEXT:    vorps %ymm0, %ymm13, %ymm0
 ; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
-; AVX1-ONLY-NEXT:    vpshufb %xmm8, %xmm12, %xmm7
-; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
-; AVX1-ONLY-NEXT:    vpshufb %xmm9, %xmm8, %xmm10
-; AVX1-ONLY-NEXT:    vpor %xmm7, %xmm10, %xmm7
-; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm1, %ymm7, %ymm1
-; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
-; AVX1-ONLY-NEXT:    vpshufb %xmm4, %xmm7, %xmm7
-; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
-; AVX1-ONLY-NEXT:    vpshufb %xmm5, %xmm4, %xmm10
-; AVX1-ONLY-NEXT:    vpunpckldq {{.*#+}} xmm7 = xmm10[0],xmm7[0],xmm10[1],xmm7[1]
-; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
-; AVX1-ONLY-NEXT:    vpshufb %xmm0, %xmm4, %xmm0
+; AVX1-ONLY-NEXT:    vpshufb %xmm3, %xmm12, %xmm13
+; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
+; AVX1-ONLY-NEXT:    vpshufb %xmm7, %xmm11, %xmm14
+; AVX1-ONLY-NEXT:    vpunpckldq {{.*#+}} xmm13 = xmm14[0],xmm13[0],xmm14[1],xmm13[1]
+; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
+; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm1 = [0,0,0,128,128,128,5,11,0,0,0,128,128,128,5,11]
+; AVX1-ONLY-NEXT:    # xmm1 = mem[0,0]
+; AVX1-ONLY-NEXT:    vpshufb %xmm1, %xmm6, %xmm14
+; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
+; AVX1-ONLY-NEXT:    vpshufb %xmm9, %xmm15, %xmm15
+; AVX1-ONLY-NEXT:    vpor %xmm14, %xmm15, %xmm14
+; AVX1-ONLY-NEXT:    vmovdqa {{.*#+}} ymm15 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255]
+; AVX1-ONLY-NEXT:    vpblendvb %xmm15, %xmm13, %xmm14, %xmm13
+; AVX1-ONLY-NEXT:    vmovaps {{.*#+}} ymm15 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0]
+; AVX1-ONLY-NEXT:    vandps %ymm0, %ymm15, %ymm0
+; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm13, %ymm0, %ymm13
+; AVX1-ONLY-NEXT:    vandnps %ymm13, %ymm15, %ymm13
+; AVX1-ONLY-NEXT:    vorps %ymm0, %ymm13, %ymm0
+; AVX1-ONLY-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX1-ONLY-NEXT:    vpshufb %xmm8, %xmm0, %xmm0
+; AVX1-ONLY-NEXT:    vmovdqa (%rsp), %xmm13 # 16-byte Reload
+; AVX1-ONLY-NEXT:    vpshufb %xmm4, %xmm13, %xmm13
+; AVX1-ONLY-NEXT:    vpor %xmm0, %xmm13, %xmm0
+; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
+; AVX1-ONLY-NEXT:    vpshufb %xmm1, %xmm13, %xmm13
+; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
+; AVX1-ONLY-NEXT:    vpshufb %xmm9, %xmm14, %xmm14
+; AVX1-ONLY-NEXT:    vpor %xmm13, %xmm14, %xmm13
+; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm0, %ymm13, %ymm0
+; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
+; AVX1-ONLY-NEXT:    vpshufb %xmm3, %xmm13, %xmm13
+; AVX1-ONLY-NEXT:    vpshufb %xmm7, %xmm2, %xmm14
+; AVX1-ONLY-NEXT:    vpunpckldq {{.*#+}} xmm13 = xmm14[0],xmm13[0],xmm14[1],xmm13[1]
+; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
+; AVX1-ONLY-NEXT:    vpshufb %xmm8, %xmm2, %xmm8
+; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
+; AVX1-ONLY-NEXT:    vpshufb %xmm4, %xmm2, %xmm4
+; AVX1-ONLY-NEXT:    vpor %xmm4, %xmm8, %xmm4
+; AVX1-ONLY-NEXT:    vpblendw {{.*#+}} xmm4 = xmm4[0,1,2],xmm13[3,4,5],xmm4[6,7]
+; AVX1-ONLY-NEXT:    vmovaps {{.*#+}} ymm1 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255]
+; AVX1-ONLY-NEXT:    vandnps %ymm0, %ymm1, %ymm0
+; AVX1-ONLY-NEXT:    vandps %ymm1, %ymm4, %ymm4
+; AVX1-ONLY-NEXT:    vorps %ymm0, %ymm4, %ymm0
+; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
+; AVX1-ONLY-NEXT:    vpshufb %xmm3, %xmm2, %xmm3
+; AVX1-ONLY-NEXT:    vpshufb %xmm7, %xmm10, %xmm1
+; AVX1-ONLY-NEXT:    vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
+; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm3 = xmm14[u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,xmm14[5,11]
 ; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
-; AVX1-ONLY-NEXT:    vpshufb %xmm6, %xmm4, %xmm6
-; AVX1-ONLY-NEXT:    vpor %xmm0, %xmm6, %xmm0
-; AVX1-ONLY-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1,2],xmm7[3,4,5],xmm0[6,7]
-; AVX1-ONLY-NEXT:    vandnps %ymm1, %ymm13, %ymm1
-; AVX1-ONLY-NEXT:    vandps %ymm0, %ymm13, %ymm0
-; AVX1-ONLY-NEXT:    vorps %ymm1, %ymm0, %ymm0
-; AVX1-ONLY-NEXT:    vandps %ymm3, %ymm0, %ymm0
-; AVX1-ONLY-NEXT:    vinsertf128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 16-byte Folded Reload
-; AVX1-ONLY-NEXT:    vandnps %ymm1, %ymm3, %ymm1
+; AVX1-ONLY-NEXT:    vpshufb %xmm9, %xmm4, %xmm4
+; AVX1-ONLY-NEXT:    vpor %xmm3, %xmm4, %xmm3
+; AVX1-ONLY-NEXT:    vmovdqa {{.*#+}} ymm8 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255]
+; AVX1-ONLY-NEXT:    vpblendvb %xmm8, %xmm1, %xmm3, %xmm1
+; AVX1-ONLY-NEXT:    vandps %ymm0, %ymm15, %ymm0
+; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1
+; AVX1-ONLY-NEXT:    vandnps %ymm1, %ymm15, %ymm1
 ; AVX1-ONLY-NEXT:    vorps %ymm1, %ymm0, %ymm0
 ; AVX1-ONLY-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm3 = [128,128,128,4,10,0,0,0,128,128,128,4,10,0,0,0]
+; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm9 = [128,128,128,4,10,0,0,0,128,128,128,4,10,0,0,0]
+; AVX1-ONLY-NEXT:    # xmm9 = mem[0,0]
+; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
+; AVX1-ONLY-NEXT:    vpshufb %xmm9, %xmm10, %xmm0
+; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm13 = [2,8,14,128,128,0,0,0,2,8,14,128,128,0,0,0]
+; AVX1-ONLY-NEXT:    # xmm13 = mem[0,0]
+; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; AVX1-ONLY-NEXT:    vpshufb %xmm13, %xmm1, %xmm1
+; AVX1-ONLY-NEXT:    vpor %xmm0, %xmm1, %xmm1
+; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm0 = [0,0,0,4,10,128,128,128,0,0,0,4,10,128,128,128]
+; AVX1-ONLY-NEXT:    # xmm0 = mem[0,0]
+; AVX1-ONLY-NEXT:    vpshufb %xmm0, %xmm5, %xmm4
+; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm3 = [0,0,0,128,128,0,6,12,0,0,0,128,128,0,6,12]
 ; AVX1-ONLY-NEXT:    # xmm3 = mem[0,0]
-; AVX1-ONLY-NEXT:    vmovdqa %xmm14, %xmm13
-; AVX1-ONLY-NEXT:    vpshufb %xmm3, %xmm14, %xmm0
-; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm4 = [2,8,14,128,128,0,0,0,2,8,14,128,128,0,0,0]
-; AVX1-ONLY-NEXT:    # xmm4 = mem[0,0]
-; AVX1-ONLY-NEXT:    vpshufb %xmm4, %xmm15, %xmm1
-; AVX1-ONLY-NEXT:    vmovdqa %xmm15, %xmm4
-; AVX1-ONLY-NEXT:    vpor %xmm0, %xmm1, %xmm0
-; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm15 = [0,0,0,4,10,128,128,128,0,0,0,4,10,128,128,128]
-; AVX1-ONLY-NEXT:    # xmm15 = mem[0,0]
-; AVX1-ONLY-NEXT:    vpshufb %xmm15, %xmm2, %xmm1
-; AVX1-ONLY-NEXT:    vmovdqa %xmm2, %xmm5
-; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm6 = [0,0,0,128,128,0,6,12,0,0,0,128,128,0,6,12]
-; AVX1-ONLY-NEXT:    # xmm6 = mem[0,0]
+; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
+; AVX1-ONLY-NEXT:    vpshufb %xmm3, %xmm5, %xmm5
+; AVX1-ONLY-NEXT:    vpor %xmm4, %xmm5, %xmm4
+; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm1, %ymm4, %ymm1
+; AVX1-ONLY-NEXT:    vmovdqa %ymm8, %ymm9
+; AVX1-ONLY-NEXT:    vandnps %ymm1, %ymm8, %ymm1
+; AVX1-ONLY-NEXT:    vandps {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm4 # 32-byte Folded Reload
+; AVX1-ONLY-NEXT:    vorps %ymm1, %ymm4, %ymm4
+; AVX1-ONLY-NEXT:    vbroadcastss {{.*#+}} xmm13 = [2,8,14,0,2,8,14,0,2,8,14,0,2,8,14,0]
+; AVX1-ONLY-NEXT:    vpshufb %xmm13, %xmm11, %xmm1
+; AVX1-ONLY-NEXT:    vbroadcastss {{.*#+}} xmm8 = [0,0,6,12,0,0,6,12,0,0,6,12,0,0,6,12]
+; AVX1-ONLY-NEXT:    vpshufb %xmm8, %xmm12, %xmm5
+; AVX1-ONLY-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm5[0],xmm1[0]
 ; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
-; AVX1-ONLY-NEXT:    vpshufb %xmm6, %xmm7, %xmm2
-; AVX1-ONLY-NEXT:    vpor %xmm1, %xmm2, %xmm1
-; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm0, %ymm1, %ymm1
-; AVX1-ONLY-NEXT:    vmovaps {{.*#+}} ymm0 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255]
-; AVX1-ONLY-NEXT:    vandnps %ymm1, %ymm0, %ymm1
-; AVX1-ONLY-NEXT:    vandps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm2 # 32-byte Folded Reload
-; AVX1-ONLY-NEXT:    vorps %ymm1, %ymm2, %ymm1
-; AVX1-ONLY-NEXT:    vmovaps {{.*#+}} ymm14 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0]
-; AVX1-ONLY-NEXT:    vandps %ymm1, %ymm14, %ymm1
-; AVX1-ONLY-NEXT:    vinsertf128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm2 # 16-byte Folded Reload
-; AVX1-ONLY-NEXT:    vandnps %ymm2, %ymm14, %ymm2
-; AVX1-ONLY-NEXT:    vorps %ymm2, %ymm1, %ymm1
-; AVX1-ONLY-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX1-ONLY-NEXT:    vmovdqa %xmm11, %xmm9
-; AVX1-ONLY-NEXT:    vpshufb %xmm3, %xmm11, %xmm2
-; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm3 = xmm10[2,8,14],zero,zero,xmm10[u,u,u,u,u,u,u,u,u,u,u]
-; AVX1-ONLY-NEXT:    vpor %xmm2, %xmm3, %xmm2
-; AVX1-ONLY-NEXT:    vpshufb %xmm15, %xmm8, %xmm3
-; AVX1-ONLY-NEXT:    vmovdqa %xmm8, %xmm15
-; AVX1-ONLY-NEXT:    vpshufb %xmm6, %xmm12, %xmm6
-; AVX1-ONLY-NEXT:    vmovdqa %xmm12, %xmm11
-; AVX1-ONLY-NEXT:    vpor %xmm3, %xmm6, %xmm3
-; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm2, %ymm3, %ymm2
-; AVX1-ONLY-NEXT:    vandnps %ymm2, %ymm0, %ymm2
-; AVX1-ONLY-NEXT:    vandps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm3 # 32-byte Folded Reload
-; AVX1-ONLY-NEXT:    vorps %ymm2, %ymm3, %ymm2
-; AVX1-ONLY-NEXT:    vandps %ymm2, %ymm14, %ymm2
-; AVX1-ONLY-NEXT:    vinsertf128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm3 # 16-byte Folded Reload
-; AVX1-ONLY-NEXT:    vandnps %ymm3, %ymm14, %ymm3
-; AVX1-ONLY-NEXT:    vorps %ymm3, %ymm2, %ymm1
+; AVX1-ONLY-NEXT:    vpshufb %xmm0, %xmm7, %xmm5
+; AVX1-ONLY-NEXT:    vmovdqa %xmm6, %xmm15
+; AVX1-ONLY-NEXT:    vpshufb %xmm3, %xmm6, %xmm6
+; AVX1-ONLY-NEXT:    vpor %xmm5, %xmm6, %xmm5
+; AVX1-ONLY-NEXT:    vpblendvb %xmm9, %xmm1, %xmm5, %xmm5
+; AVX1-ONLY-NEXT:    vmovaps {{.*#+}} ymm1 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0]
+; AVX1-ONLY-NEXT:    vandps %ymm1, %ymm4, %ymm4
+; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm5, %ymm0, %ymm5
+; AVX1-ONLY-NEXT:    vandnps %ymm5, %ymm1, %ymm5
+; AVX1-ONLY-NEXT:    vorps %ymm5, %ymm4, %ymm1
 ; AVX1-ONLY-NEXT:    vmovups %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm1 = [128,128,128,5,11,0,0,0,128,128,128,5,11,0,0,0]
-; AVX1-ONLY-NEXT:    # xmm1 = mem[0,0]
-; AVX1-ONLY-NEXT:    vpshufb %xmm1, %xmm13, %xmm3
-; AVX1-ONLY-NEXT:    vmovdqa %xmm13, %xmm1
-; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm2 = [3,9,15,128,128,0,0,0,3,9,15,128,128,0,0,0]
-; AVX1-ONLY-NEXT:    # xmm2 = mem[0,0]
-; AVX1-ONLY-NEXT:    vmovdqa %xmm4, %xmm13
-; AVX1-ONLY-NEXT:    vpshufb %xmm2, %xmm4, %xmm6
-; AVX1-ONLY-NEXT:    vpor %xmm3, %xmm6, %xmm3
-; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm4 = [0,0,0,5,11,128,128,128,0,0,0,5,11,128,128,128]
-; AVX1-ONLY-NEXT:    # xmm4 = mem[0,0]
-; AVX1-ONLY-NEXT:    vmovdqa %xmm5, %xmm12
-; AVX1-ONLY-NEXT:    vpshufb %xmm4, %xmm5, %xmm6
+; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm4 = zero,zero,zero,xmm12[4,10,u,u,u,u,u,u,u,u,u,u,u]
+; AVX1-ONLY-NEXT:    vmovdqa (%rsp), %xmm1 # 16-byte Reload
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm5 = xmm1[2,8,14],zero,zero,xmm1[u,u,u,u,u,u,u,u,u,u,u]
+; AVX1-ONLY-NEXT:    vpor %xmm4, %xmm5, %xmm4
+; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
+; AVX1-ONLY-NEXT:    vpshufb %xmm0, %xmm11, %xmm5
+; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
+; AVX1-ONLY-NEXT:    vpshufb %xmm3, %xmm6, %xmm6
+; AVX1-ONLY-NEXT:    vpor %xmm5, %xmm6, %xmm5
+; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm4, %ymm5, %ymm4
+; AVX1-ONLY-NEXT:    vandnps %ymm4, %ymm9, %ymm4
+; AVX1-ONLY-NEXT:    vandps {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm5 # 32-byte Folded Reload
+; AVX1-ONLY-NEXT:    vorps %ymm4, %ymm5, %ymm4
+; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; AVX1-ONLY-NEXT:    vpshufb %xmm13, %xmm1, %xmm5
+; AVX1-ONLY-NEXT:    vpshufb %xmm8, %xmm2, %xmm6
+; AVX1-ONLY-NEXT:    vpunpcklqdq {{.*#+}} xmm5 = xmm6[0],xmm5[0]
+; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
+; AVX1-ONLY-NEXT:    vpshufb %xmm0, %xmm2, %xmm0
+; AVX1-ONLY-NEXT:    vpshufb %xmm3, %xmm14, %xmm3
+; AVX1-ONLY-NEXT:    vpor %xmm0, %xmm3, %xmm0
+; AVX1-ONLY-NEXT:    vpblendvb %xmm9, %xmm5, %xmm0, %xmm0
+; AVX1-ONLY-NEXT:    vmovaps {{.*#+}} ymm5 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0]
+; AVX1-ONLY-NEXT:    vandps %ymm5, %ymm4, %ymm3
+; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
+; AVX1-ONLY-NEXT:    vandnps %ymm0, %ymm5, %ymm0
+; AVX1-ONLY-NEXT:    vorps %ymm0, %ymm3, %ymm0
+; AVX1-ONLY-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm14 = [128,128,128,5,11,0,0,0,128,128,128,5,11,0,0,0]
+; AVX1-ONLY-NEXT:    # xmm14 = mem[0,0]
+; AVX1-ONLY-NEXT:    vpshufb %xmm14, %xmm10, %xmm0
+; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm13 = [3,9,15,128,128,0,0,0,3,9,15,128,128,0,0,0]
+; AVX1-ONLY-NEXT:    # xmm13 = mem[0,0]
+; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
+; AVX1-ONLY-NEXT:    vpshufb %xmm13, %xmm14, %xmm3
+; AVX1-ONLY-NEXT:    vpor %xmm0, %xmm3, %xmm0
+; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm3 = [0,0,0,5,11,128,128,128,0,0,0,5,11,128,128,128]
+; AVX1-ONLY-NEXT:    # xmm3 = mem[0,0]
+; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
+; AVX1-ONLY-NEXT:    vpshufb %xmm3, %xmm4, %xmm4
 ; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm5 = [0,0,0,128,128,1,7,13,0,0,0,128,128,1,7,13]
 ; AVX1-ONLY-NEXT:    # xmm5 = mem[0,0]
-; AVX1-ONLY-NEXT:    vmovdqa %xmm7, %xmm8
-; AVX1-ONLY-NEXT:    vpshufb %xmm5, %xmm7, %xmm7
-; AVX1-ONLY-NEXT:    vpor %xmm6, %xmm7, %xmm6
-; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm3, %ymm6, %ymm3
-; AVX1-ONLY-NEXT:    vandnps %ymm3, %ymm0, %ymm3
-; AVX1-ONLY-NEXT:    vandps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm6 # 32-byte Folded Reload
-; AVX1-ONLY-NEXT:    vorps %ymm3, %ymm6, %ymm3
-; AVX1-ONLY-NEXT:    vandps %ymm3, %ymm14, %ymm3
-; AVX1-ONLY-NEXT:    vinsertf128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm6 # 16-byte Folded Reload
-; AVX1-ONLY-NEXT:    vandnps %ymm6, %ymm14, %ymm6
-; AVX1-ONLY-NEXT:    vorps %ymm6, %ymm3, %ymm3
-; AVX1-ONLY-NEXT:    vmovups %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm6 = zero,zero,zero,xmm9[5,11,u,u,u,u,u,u,u,u,u,u,u]
-; AVX1-ONLY-NEXT:    vpshufb %xmm2, %xmm10, %xmm7
-; AVX1-ONLY-NEXT:    vmovdqa %xmm10, %xmm9
+; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload
+; AVX1-ONLY-NEXT:    vpshufb %xmm5, %xmm10, %xmm6
+; AVX1-ONLY-NEXT:    vpor %xmm4, %xmm6, %xmm4
+; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm0, %ymm4, %ymm0
+; AVX1-ONLY-NEXT:    vmovdqa %ymm9, %ymm13
+; AVX1-ONLY-NEXT:    vandnps %ymm0, %ymm9, %ymm0
+; AVX1-ONLY-NEXT:    vandps {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm4 # 32-byte Folded Reload
+; AVX1-ONLY-NEXT:    vorps %ymm0, %ymm4, %ymm0
+; AVX1-ONLY-NEXT:    vbroadcastss {{.*#+}} xmm8 = [3,9,15,0,3,9,15,0,3,9,15,0,3,9,15,0]
+; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
+; AVX1-ONLY-NEXT:    vpshufb %xmm8, %xmm4, %xmm4
+; AVX1-ONLY-NEXT:    vbroadcastss {{.*#+}} xmm9 = [0,1,7,13,0,1,7,13,0,1,7,13,0,1,7,13]
+; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
+; AVX1-ONLY-NEXT:    vpshufb %xmm9, %xmm6, %xmm6
+; AVX1-ONLY-NEXT:    vpunpcklqdq {{.*#+}} xmm4 = xmm6[0],xmm4[0]
+; AVX1-ONLY-NEXT:    vpshufb %xmm3, %xmm7, %xmm6
+; AVX1-ONLY-NEXT:    vpshufb %xmm5, %xmm15, %xmm7
+; AVX1-ONLY-NEXT:    vmovdqa %xmm15, %xmm8
 ; AVX1-ONLY-NEXT:    vpor %xmm6, %xmm7, %xmm6
-; AVX1-ONLY-NEXT:    vpshufb %xmm4, %xmm15, %xmm7
-; AVX1-ONLY-NEXT:    vmovdqa %xmm15, %xmm4
-; AVX1-ONLY-NEXT:    vpshufb %xmm5, %xmm11, %xmm10
-; AVX1-ONLY-NEXT:    vmovdqa %xmm11, %xmm2
-; AVX1-ONLY-NEXT:    vpor %xmm7, %xmm10, %xmm7
-; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm6, %ymm7, %ymm6
-; AVX1-ONLY-NEXT:    vandnps %ymm6, %ymm0, %ymm6
-; AVX1-ONLY-NEXT:    vandps {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
-; AVX1-ONLY-NEXT:    vorps %ymm6, %ymm0, %ymm0
-; AVX1-ONLY-NEXT:    vandps %ymm0, %ymm14, %ymm0
-; AVX1-ONLY-NEXT:    vinsertf128 $1, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm6 # 16-byte Folded Reload
-; AVX1-ONLY-NEXT:    vandnps %ymm6, %ymm14, %ymm6
-; AVX1-ONLY-NEXT:    vmovaps %ymm14, %ymm11
-; AVX1-ONLY-NEXT:    vorps %ymm6, %ymm0, %ymm0
+; AVX1-ONLY-NEXT:    vpblendvb %xmm13, %xmm4, %xmm6, %xmm4
+; AVX1-ONLY-NEXT:    vmovaps {{.*#+}} ymm6 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0]
+; AVX1-ONLY-NEXT:    vandps %ymm6, %ymm0, %ymm0
+; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm4, %ymm0, %ymm4
+; AVX1-ONLY-NEXT:    vandnps %ymm4, %ymm6, %ymm4
+; AVX1-ONLY-NEXT:    vmovaps %ymm6, %ymm7
+; AVX1-ONLY-NEXT:    vorps %ymm4, %ymm0, %ymm0
 ; AVX1-ONLY-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm14 = [4,10,128,128,128,0,0,0,4,10,128,128,128,0,0,0]
-; AVX1-ONLY-NEXT:    # xmm14 = mem[0,0]
-; AVX1-ONLY-NEXT:    vpshufb %xmm14, %xmm13, %xmm0
-; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm15 = [128,128,0,6,12,0,0,0,128,128,0,6,12,0,0,0]
-; AVX1-ONLY-NEXT:    # xmm15 = mem[0,0]
-; AVX1-ONLY-NEXT:    vpshufb %xmm15, %xmm1, %xmm7
-; AVX1-ONLY-NEXT:    vpor %xmm0, %xmm7, %xmm0
-; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm1 = [0,0,128,128,128,2,8,14,0,0,128,128,128,2,8,14]
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm0 = zero,zero,zero,xmm12[5,11,u,u,u,u,u,u,u,u,u,u,u]
+; AVX1-ONLY-NEXT:    vmovdqa (%rsp), %xmm12 # 16-byte Reload
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm4 = xmm12[3,9,15],zero,zero,xmm12[u,u,u,u,u,u,u,u,u,u,u]
+; AVX1-ONLY-NEXT:    vpor %xmm0, %xmm4, %xmm0
+; AVX1-ONLY-NEXT:    vpshufb %xmm3, %xmm11, %xmm4
+; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
+; AVX1-ONLY-NEXT:    vpshufb %xmm5, %xmm15, %xmm6
+; AVX1-ONLY-NEXT:    vpor %xmm4, %xmm6, %xmm4
+; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm0, %ymm4, %ymm0
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm4 = xmm1[3,9,15,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; AVX1-ONLY-NEXT:    vpshufb %xmm9, %xmm1, %xmm6
+; AVX1-ONLY-NEXT:    vpunpcklqdq {{.*#+}} xmm4 = xmm6[0],xmm4[0]
+; AVX1-ONLY-NEXT:    vpshufb %xmm3, %xmm2, %xmm3
+; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
+; AVX1-ONLY-NEXT:    vpshufb %xmm5, %xmm11, %xmm5
+; AVX1-ONLY-NEXT:    vpor %xmm3, %xmm5, %xmm3
+; AVX1-ONLY-NEXT:    vpblendvb %xmm13, %xmm4, %xmm3, %xmm3
+; AVX1-ONLY-NEXT:    vandnps %ymm0, %ymm13, %ymm0
+; AVX1-ONLY-NEXT:    vandps {{[-0-9]+}}(%r{{[sb]}}p), %ymm13, %ymm2 # 32-byte Folded Reload
+; AVX1-ONLY-NEXT:    vorps %ymm0, %ymm2, %ymm0
+; AVX1-ONLY-NEXT:    vandps %ymm7, %ymm0, %ymm0
+; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm2
+; AVX1-ONLY-NEXT:    vandnps %ymm2, %ymm7, %ymm2
+; AVX1-ONLY-NEXT:    vorps %ymm2, %ymm0, %ymm0
+; AVX1-ONLY-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm1 = [4,10,128,128,128,0,0,0,4,10,128,128,128,0,0,0]
 ; AVX1-ONLY-NEXT:    # xmm1 = mem[0,0]
-; AVX1-ONLY-NEXT:    vpshufb %xmm1, %xmm8, %xmm7
-; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm5 = [0,0,0,6,12,128,128,128,0,0,0,6,12,128,128,128]
-; AVX1-ONLY-NEXT:    # xmm5 = mem[0,0]
-; AVX1-ONLY-NEXT:    vpshufb %xmm5, %xmm12, %xmm10
-; AVX1-ONLY-NEXT:    vpor %xmm7, %xmm10, %xmm7
-; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm0, %ymm7, %ymm0
+; AVX1-ONLY-NEXT:    vpshufb %xmm1, %xmm14, %xmm0
+; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm14 = [128,128,0,6,12,0,0,0,128,128,0,6,12,0,0,0]
+; AVX1-ONLY-NEXT:    # xmm14 = mem[0,0]
+; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
+; AVX1-ONLY-NEXT:    vpshufb %xmm14, %xmm13, %xmm2
+; AVX1-ONLY-NEXT:    vpor %xmm0, %xmm2, %xmm0
+; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm3 = [0,0,128,128,128,2,8,14,0,0,128,128,128,2,8,14]
+; AVX1-ONLY-NEXT:    # xmm3 = mem[0,0]
+; AVX1-ONLY-NEXT:    vpshufb %xmm3, %xmm10, %xmm2
+; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm4 = [0,0,0,6,12,128,128,128,0,0,0,6,12,128,128,128]
+; AVX1-ONLY-NEXT:    # xmm4 = mem[0,0]
+; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload
+; AVX1-ONLY-NEXT:    vpshufb %xmm4, %xmm5, %xmm5
+; AVX1-ONLY-NEXT:    vpor %xmm2, %xmm5, %xmm2
+; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm0, %ymm2, %ymm2
 ; AVX1-ONLY-NEXT:    vmovaps {{.*#+}} ymm10 = [0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
-; AVX1-ONLY-NEXT:    vandnps {{[-0-9]+}}(%r{{[sb]}}p), %ymm10, %ymm7 # 32-byte Folded Reload
-; AVX1-ONLY-NEXT:    vandps %ymm0, %ymm10, %ymm0
-; AVX1-ONLY-NEXT:    vorps %ymm7, %ymm0, %ymm0
-; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
-; AVX1-ONLY-NEXT:    vpshufb %xmm1, %xmm15, %xmm7
-; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
-; AVX1-ONLY-NEXT:    vpshufb %xmm5, %xmm8, %xmm3
-; AVX1-ONLY-NEXT:    vpor %xmm7, %xmm3, %xmm3
+; AVX1-ONLY-NEXT:    vandnps {{[-0-9]+}}(%r{{[sb]}}p), %ymm10, %ymm5 # 32-byte Folded Reload
+; AVX1-ONLY-NEXT:    vandps %ymm2, %ymm10, %ymm2
+; AVX1-ONLY-NEXT:    vorps %ymm5, %ymm2, %ymm2
+; AVX1-ONLY-NEXT:    vpshufb %xmm3, %xmm8, %xmm5
+; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX1-ONLY-NEXT:    vpshufb %xmm4, %xmm0, %xmm6
+; AVX1-ONLY-NEXT:    vpor %xmm5, %xmm6, %xmm5
+; AVX1-ONLY-NEXT:    vbroadcastss {{.*#+}} xmm8 = [4,10,0,0,4,10,0,0,4,10,0,0,4,10,0,0]
 ; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
-; AVX1-ONLY-NEXT:    vbroadcastss {{.*#+}} xmm12 = [4,10,0,0,4,10,0,0,4,10,0,0,4,10,0,0]
-; AVX1-ONLY-NEXT:    vpshufb %xmm12, %xmm14, %xmm7
+; AVX1-ONLY-NEXT:    vpshufb %xmm8, %xmm14, %xmm6
+; AVX1-ONLY-NEXT:    vbroadcastss {{.*#+}} xmm9 = [0,2,8,14,0,2,8,14,0,2,8,14,0,2,8,14]
+; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX1-ONLY-NEXT:    vpshufb %xmm9, %xmm0, %xmm7
+; AVX1-ONLY-NEXT:    vpunpckhqdq {{.*#+}} xmm6 = xmm7[1],xmm6[1]
+; AVX1-ONLY-NEXT:    vpblendw {{.*#+}} xmm5 = xmm6[0,1,2,3,4],xmm5[5,6,7]
+; AVX1-ONLY-NEXT:    vmovaps {{.*#+}} ymm6 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0]
+; AVX1-ONLY-NEXT:    vandps %ymm6, %ymm2, %ymm2
+; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm5, %ymm0, %ymm5
+; AVX1-ONLY-NEXT:    vandnps %ymm5, %ymm6, %ymm5
+; AVX1-ONLY-NEXT:    vmovaps %ymm6, %ymm0
+; AVX1-ONLY-NEXT:    vorps %ymm5, %ymm2, %ymm2
+; AVX1-ONLY-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX1-ONLY-NEXT:    vpshufb %xmm1, %xmm12, %xmm5
+; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm6 = zero,zero,xmm1[0,6,12,u,u,u,u,u,u,u,u,u,u,u]
+; AVX1-ONLY-NEXT:    vpor %xmm5, %xmm6, %xmm5
+; AVX1-ONLY-NEXT:    vpshufb %xmm3, %xmm15, %xmm6
+; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; AVX1-ONLY-NEXT:    vpshufb %xmm4, %xmm1, %xmm7
+; AVX1-ONLY-NEXT:    vpor %xmm6, %xmm7, %xmm6
+; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm5, %ymm6, %ymm5
+; AVX1-ONLY-NEXT:    vandnps {{[-0-9]+}}(%r{{[sb]}}p), %ymm10, %ymm6 # 32-byte Folded Reload
+; AVX1-ONLY-NEXT:    vandps %ymm5, %ymm10, %ymm5
+; AVX1-ONLY-NEXT:    vorps %ymm6, %ymm5, %ymm5
+; AVX1-ONLY-NEXT:    vpshufb %xmm3, %xmm11, %xmm3
+; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
+; AVX1-ONLY-NEXT:    vpshufb %xmm4, %xmm11, %xmm4
+; AVX1-ONLY-NEXT:    vpor %xmm3, %xmm4, %xmm3
 ; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
-; AVX1-ONLY-NEXT:    vbroadcastss {{.*#+}} xmm13 = [0,2,8,14,0,2,8,14,0,2,8,14,0,2,8,14]
-; AVX1-ONLY-NEXT:    vpshufb %xmm13, %xmm12, %xmm6
-; AVX1-ONLY-NEXT:    vpunpckhqdq {{.*#+}} xmm6 = xmm6[1],xmm7[1]
-; AVX1-ONLY-NEXT:    vpblendw {{.*#+}} xmm3 = xmm6[0,1,2,3,4],xmm3[5,6,7]
-; AVX1-ONLY-NEXT:    vandps %ymm0, %ymm11, %ymm0
+; AVX1-ONLY-NEXT:    vpshufb %xmm8, %xmm12, %xmm4
+; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; AVX1-ONLY-NEXT:    vpshufb %xmm9, %xmm1, %xmm6
+; AVX1-ONLY-NEXT:    vpunpckhqdq {{.*#+}} xmm4 = xmm6[1],xmm4[1]
+; AVX1-ONLY-NEXT:    vpblendw {{.*#+}} xmm3 = xmm4[0,1,2,3,4],xmm3[5,6,7]
+; AVX1-ONLY-NEXT:    vandps %ymm0, %ymm5, %ymm4
 ; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm3, %ymm0, %ymm3
-; AVX1-ONLY-NEXT:    vandnps %ymm3, %ymm11, %ymm3
-; AVX1-ONLY-NEXT:    vorps %ymm3, %ymm0, %ymm0
-; AVX1-ONLY-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm0 = xmm9[4,10],zero,zero,zero,xmm9[u,u,u,u,u,u,u,u,u,u,u]
-; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm3 = zero,zero,xmm7[0,6,12,u,u,u,u,u,u,u,u,u,u,u]
-; AVX1-ONLY-NEXT:    vpor %xmm0, %xmm3, %xmm0
-; AVX1-ONLY-NEXT:    vpshufb %xmm1, %xmm2, %xmm3
-; AVX1-ONLY-NEXT:    vpshufb %xmm5, %xmm4, %xmm6
-; AVX1-ONLY-NEXT:    vpor %xmm3, %xmm6, %xmm3
-; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm0, %ymm3, %ymm0
-; AVX1-ONLY-NEXT:    vandnps {{[-0-9]+}}(%r{{[sb]}}p), %ymm10, %ymm3 # 32-byte Folded Reload
-; AVX1-ONLY-NEXT:    vandps %ymm0, %ymm10, %ymm0
-; AVX1-ONLY-NEXT:    vorps %ymm3, %ymm0, %ymm0
-; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload
-; AVX1-ONLY-NEXT:    vpshufb %xmm1, %xmm11, %xmm2
-; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload
-; AVX1-ONLY-NEXT:    vpshufb %xmm5, %xmm9, %xmm1
-; AVX1-ONLY-NEXT:    vpor %xmm2, %xmm1, %xmm1
+; AVX1-ONLY-NEXT:    vandnps %ymm3, %ymm0, %ymm3
+; AVX1-ONLY-NEXT:    vorps %ymm3, %ymm4, %ymm2
+; AVX1-ONLY-NEXT:    vmovups %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm15 = [5,11,128,128,128,0,0,0,5,11,128,128,128,0,0,0]
+; AVX1-ONLY-NEXT:    # xmm15 = mem[0,0]
 ; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,u,u,u,u,u,u,u,4,10,u,u,u,u,u,u]
-; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
-; AVX1-ONLY-NEXT:    vpshufb %xmm13, %xmm4, %xmm3
-; AVX1-ONLY-NEXT:    vpunpckhqdq {{.*#+}} xmm2 = xmm3[1],xmm2[1]
-; AVX1-ONLY-NEXT:    vpblendw {{.*#+}} xmm1 = xmm2[0,1,2,3,4],xmm1[5,6,7]
+; AVX1-ONLY-NEXT:    vpshufb %xmm15, %xmm2, %xmm3
+; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm4 = [128,128,1,7,13,0,0,0,128,128,1,7,13,0,0,0]
+; AVX1-ONLY-NEXT:    # xmm4 = mem[0,0]
+; AVX1-ONLY-NEXT:    vpshufb %xmm4, %xmm13, %xmm5
+; AVX1-ONLY-NEXT:    vpor %xmm3, %xmm5, %xmm3
+; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm5 = [0,0,128,128,128,3,9,15,0,0,128,128,128,3,9,15]
+; AVX1-ONLY-NEXT:    # xmm5 = mem[0,0]
+; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
+; AVX1-ONLY-NEXT:    vpshufb %xmm5, %xmm2, %xmm7
+; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm6 = [0,0,1,7,13,128,128,128,0,0,1,7,13,128,128,128]
+; AVX1-ONLY-NEXT:    # xmm6 = mem[0,0]
+; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
+; AVX1-ONLY-NEXT:    vpshufb %xmm6, %xmm2, %xmm8
+; AVX1-ONLY-NEXT:    vpor %xmm7, %xmm8, %xmm7
+; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm3, %ymm7, %ymm3
+; AVX1-ONLY-NEXT:    vandnps {{[-0-9]+}}(%r{{[sb]}}p), %ymm10, %ymm7 # 32-byte Folded Reload
+; AVX1-ONLY-NEXT:    vandps %ymm3, %ymm10, %ymm3
+; AVX1-ONLY-NEXT:    vorps %ymm7, %ymm3, %ymm3
+; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
+; AVX1-ONLY-NEXT:    vpshufb %xmm5, %xmm2, %xmm7
+; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX1-ONLY-NEXT:    vpshufb %xmm6, %xmm0, %xmm8
+; AVX1-ONLY-NEXT:    vpor %xmm7, %xmm8, %xmm7
+; AVX1-ONLY-NEXT:    vbroadcastss {{.*#+}} xmm13 = [5,11,0,0,5,11,0,0,5,11,0,0,5,11,0,0]
+; AVX1-ONLY-NEXT:    vpshufb %xmm13, %xmm14, %xmm8
+; AVX1-ONLY-NEXT:    vbroadcastss {{.*#+}} xmm14 = [0,3,9,15,0,3,9,15,0,3,9,15,0,3,9,15]
+; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX1-ONLY-NEXT:    vpshufb %xmm14, %xmm0, %xmm9
+; AVX1-ONLY-NEXT:    vpunpckhqdq {{.*#+}} xmm8 = xmm9[1],xmm8[1]
+; AVX1-ONLY-NEXT:    vpblendw {{.*#+}} xmm7 = xmm8[0,1,2,3,4],xmm7[5,6,7]
 ; AVX1-ONLY-NEXT:    vmovaps {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0]
+; AVX1-ONLY-NEXT:    vandps %ymm2, %ymm3, %ymm3
+; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm7, %ymm0, %ymm7
+; AVX1-ONLY-NEXT:    vandnps %ymm7, %ymm2, %ymm7
+; AVX1-ONLY-NEXT:    vorps %ymm7, %ymm3, %ymm3
+; AVX1-ONLY-NEXT:    vmovdqa (%rsp), %xmm7 # 16-byte Reload
+; AVX1-ONLY-NEXT:    vpshufb %xmm15, %xmm7, %xmm7
+; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
+; AVX1-ONLY-NEXT:    vpshufb %xmm4, %xmm8, %xmm8
+; AVX1-ONLY-NEXT:    vpor %xmm7, %xmm8, %xmm7
+; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
+; AVX1-ONLY-NEXT:    vpshufb %xmm5, %xmm4, %xmm8
+; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
+; AVX1-ONLY-NEXT:    vpshufb %xmm6, %xmm4, %xmm9
+; AVX1-ONLY-NEXT:    vpor %xmm8, %xmm9, %xmm8
+; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm7, %ymm8, %ymm7
+; AVX1-ONLY-NEXT:    vandnps {{[-0-9]+}}(%r{{[sb]}}p), %ymm10, %ymm8 # 32-byte Folded Reload
+; AVX1-ONLY-NEXT:    vandps %ymm7, %ymm10, %ymm0
+; AVX1-ONLY-NEXT:    vorps %ymm0, %ymm8, %ymm0
+; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
+; AVX1-ONLY-NEXT:    vpshufb %xmm5, %xmm4, %xmm5
+; AVX1-ONLY-NEXT:    vpshufb %xmm6, %xmm11, %xmm6
+; AVX1-ONLY-NEXT:    vpor %xmm5, %xmm6, %xmm5
+; AVX1-ONLY-NEXT:    vpshufb %xmm13, %xmm12, %xmm6
+; AVX1-ONLY-NEXT:    vpshufb %xmm14, %xmm1, %xmm7
+; AVX1-ONLY-NEXT:    vpunpckhqdq {{.*#+}} xmm6 = xmm7[1],xmm6[1]
+; AVX1-ONLY-NEXT:    vpblendw {{.*#+}} xmm5 = xmm6[0,1,2,3,4],xmm5[5,6,7]
 ; AVX1-ONLY-NEXT:    vandps %ymm2, %ymm0, %ymm0
-; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm1
-; AVX1-ONLY-NEXT:    vandnps %ymm1, %ymm2, %ymm1
+; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm5, %ymm0, %ymm5
+; AVX1-ONLY-NEXT:    vandnps %ymm5, %ymm2, %ymm1
 ; AVX1-ONLY-NEXT:    vorps %ymm1, %ymm0, %ymm0
-; AVX1-ONLY-NEXT:    vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm1 = xmm0[5,11],zero,zero,zero,xmm0[u,u,u,u,u,u,u,u,u,u,u]
-; AVX1-ONLY-NEXT:    vmovdqa (%rsp), %xmm0 # 16-byte Reload
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm2 = zero,zero,xmm0[1,7,13,u,u,u,u,u,u,u,u,u,u,u]
-; AVX1-ONLY-NEXT:    vpor %xmm1, %xmm2, %xmm1
-; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm13 = [0,0,128,128,128,3,9,15,0,0,128,128,128,3,9,15]
-; AVX1-ONLY-NEXT:    # xmm13 = mem[0,0]
-; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX1-ONLY-NEXT:    vpshufb %xmm13, %xmm0, %xmm2
-; AVX1-ONLY-NEXT:    vmovddup {{.*#+}} xmm0 = [0,0,1,7,13,128,128,128,0,0,1,7,13,128,128,128]
-; AVX1-ONLY-NEXT:    # xmm0 = mem[0,0]
-; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
-; AVX1-ONLY-NEXT:    vpshufb %xmm0, %xmm3, %xmm3
-; AVX1-ONLY-NEXT:    vpor %xmm2, %xmm3, %xmm2
-; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
-; AVX1-ONLY-NEXT:    vandnps {{[-0-9]+}}(%r{{[sb]}}p), %ymm10, %ymm2 # 32-byte Folded Reload
-; AVX1-ONLY-NEXT:    vandps %ymm1, %ymm10, %ymm1
-; AVX1-ONLY-NEXT:    vorps %ymm2, %ymm1, %ymm1
-; AVX1-ONLY-NEXT:    vpshufb %xmm13, %xmm15, %xmm2
-; AVX1-ONLY-NEXT:    vpshufb %xmm0, %xmm8, %xmm3
-; AVX1-ONLY-NEXT:    vpor %xmm2, %xmm3, %xmm2
-; AVX1-ONLY-NEXT:    vbroadcastss {{.*#+}} xmm8 = [5,11,0,0,5,11,0,0,5,11,0,0,5,11,0,0]
-; AVX1-ONLY-NEXT:    vpshufb %xmm8, %xmm14, %xmm3
-; AVX1-ONLY-NEXT:    vbroadcastss {{.*#+}} xmm5 = [0,3,9,15,0,3,9,15,0,3,9,15,0,3,9,15]
-; AVX1-ONLY-NEXT:    vpshufb %xmm5, %xmm12, %xmm6
-; AVX1-ONLY-NEXT:    vpunpckhqdq {{.*#+}} xmm3 = xmm6[1],xmm3[1]
-; AVX1-ONLY-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3,4],xmm2[5,6,7]
-; AVX1-ONLY-NEXT:    vmovaps {{.*#+}} ymm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0]
-; AVX1-ONLY-NEXT:    vandps %ymm3, %ymm1, %ymm1
-; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm2
-; AVX1-ONLY-NEXT:    vandnps %ymm2, %ymm3, %ymm2
-; AVX1-ONLY-NEXT:    vmovaps %ymm3, %ymm12
-; AVX1-ONLY-NEXT:    vorps %ymm2, %ymm1, %ymm6
-; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[5,11],zero,zero,zero,xmm1[u,u,u,u,u,u,u,u,u,u,u]
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm2 = zero,zero,xmm7[1,7,13,u,u,u,u,u,u,u,u,u,u,u]
-; AVX1-ONLY-NEXT:    vpor %xmm1, %xmm2, %xmm1
-; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
-; AVX1-ONLY-NEXT:    vpshufb %xmm13, %xmm2, %xmm2
-; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
-; AVX1-ONLY-NEXT:    vpshufb %xmm0, %xmm3, %xmm3
-; AVX1-ONLY-NEXT:    vpor %xmm2, %xmm3, %xmm2
-; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm1, %ymm2, %ymm1
-; AVX1-ONLY-NEXT:    vandnps {{[-0-9]+}}(%r{{[sb]}}p), %ymm10, %ymm2 # 32-byte Folded Reload
-; AVX1-ONLY-NEXT:    vandps %ymm1, %ymm10, %ymm1
-; AVX1-ONLY-NEXT:    vorps %ymm2, %ymm1, %ymm1
-; AVX1-ONLY-NEXT:    vpshufb %xmm13, %xmm11, %xmm2
-; AVX1-ONLY-NEXT:    vpshufb %xmm0, %xmm9, %xmm3
-; AVX1-ONLY-NEXT:    vpor %xmm2, %xmm3, %xmm2
-; AVX1-ONLY-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX1-ONLY-NEXT:    vpshufb %xmm8, %xmm0, %xmm3
-; AVX1-ONLY-NEXT:    vpshufb %xmm5, %xmm4, %xmm4
-; AVX1-ONLY-NEXT:    vpunpckhqdq {{.*#+}} xmm3 = xmm4[1],xmm3[1]
-; AVX1-ONLY-NEXT:    vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3,4],xmm2[5,6,7]
-; AVX1-ONLY-NEXT:    vandps %ymm1, %ymm12, %ymm1
-; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm2, %ymm0, %ymm2
-; AVX1-ONLY-NEXT:    vandnps %ymm2, %ymm12, %ymm2
-; AVX1-ONLY-NEXT:    vorps %ymm2, %ymm1, %ymm1
-; AVX1-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX1-ONLY-NEXT:    vmovaps %ymm0, 32(%rsi)
-; AVX1-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX1-ONLY-NEXT:    vmovaps %ymm0, (%rsi)
-; AVX1-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX1-ONLY-NEXT:    vmovaps %ymm0, 32(%rdx)
-; AVX1-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX1-ONLY-NEXT:    vmovaps %ymm0, (%rdx)
-; AVX1-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX1-ONLY-NEXT:    vmovaps %ymm0, 32(%rcx)
-; AVX1-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX1-ONLY-NEXT:    vmovaps %ymm0, (%rcx)
-; AVX1-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX1-ONLY-NEXT:    vmovaps %ymm0, 32(%r8)
-; AVX1-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX1-ONLY-NEXT:    vmovaps %ymm0, (%r8)
-; AVX1-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX1-ONLY-NEXT:    vmovaps %ymm0, 32(%r9)
-; AVX1-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX1-ONLY-NEXT:    vmovaps %ymm0, (%r9)
+; AVX1-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX1-ONLY-NEXT:    vmovaps %ymm1, 32(%rsi)
+; AVX1-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX1-ONLY-NEXT:    vmovaps %ymm1, (%rsi)
+; AVX1-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX1-ONLY-NEXT:    vmovaps %ymm1, 32(%rdx)
+; AVX1-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX1-ONLY-NEXT:    vmovaps %ymm1, (%rdx)
+; AVX1-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX1-ONLY-NEXT:    vmovaps %ymm1, 32(%rcx)
+; AVX1-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX1-ONLY-NEXT:    vmovaps %ymm1, (%rcx)
+; AVX1-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX1-ONLY-NEXT:    vmovaps %ymm1, 32(%r8)
+; AVX1-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX1-ONLY-NEXT:    vmovaps %ymm1, (%r8)
+; AVX1-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX1-ONLY-NEXT:    vmovaps %ymm1, 32(%r9)
+; AVX1-ONLY-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX1-ONLY-NEXT:    vmovaps %ymm1, (%r9)
 ; AVX1-ONLY-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX1-ONLY-NEXT:    vmovaps %ymm1, 32(%rax)
-; AVX1-ONLY-NEXT:    vmovaps %ymm6, (%rax)
-; AVX1-ONLY-NEXT:    addq $808, %rsp # imm = 0x328
+; AVX1-ONLY-NEXT:    vmovaps %ymm0, 32(%rax)
+; AVX1-ONLY-NEXT:    vmovaps %ymm3, (%rax)
+; AVX1-ONLY-NEXT:    addq $616, %rsp # imm = 0x268
 ; AVX1-ONLY-NEXT:    vzeroupper
 ; AVX1-ONLY-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-7.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-7.ll
index 0e3f8121db1de0eeb4569d102d1800b0df23fe09..f82a192c60b570445f4c11ba641bad4204a7bab8 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-7.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-7.ll
@@ -9218,515 +9218,508 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ;
 ; AVX512F-ONLY-SLOW-LABEL: load_i8_stride7_vf64:
 ; AVX512F-ONLY-SLOW:       # %bb.0:
-; AVX512F-ONLY-SLOW-NEXT:    subq $72, %rsp
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa {{.*#+}} ymm1 = [65535,0,65535,65535,65535,0,65535,65535,0,65535,65535,65535,0,65535,65535,0]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 (%rdi), %ymm17
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 32(%rdi), %ymm15
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 64(%rdi), %ymm24
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm1, %ymm0
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm1, %ymm12
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm15, %ymm17, %ymm0
-; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm1 = zero,zero,zero,xmm1[5,12],zero,zero,xmm1[1,8,15,u,u,u,u,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,7,14],zero,zero,xmm0[3,10],zero,zero,zero,xmm0[u,u,u,u,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm1, %xmm0, %xmm1
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa {{.*#+}} ymm11 = [65535,65535,65535,0,65535,65535,0,65535,65535,65535,0,65535,65535,0,65535,65535]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa {{.*#+}} ymm0 = [65535,0,65535,65535,65535,0,65535,65535,0,65535,65535,65535,0,65535,65535,0]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa (%rdi), %ymm12
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 32(%rdi), %ymm13
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 64(%rdi), %ymm31
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm0, %ymm1
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %ymm0, %ymm24
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm13, %ymm12, %ymm1
+; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm2 = zero,zero,zero,xmm2[5,12],zero,zero,xmm2[1,8,15,u,u,u,u,u,u]
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0,7,14],zero,zero,xmm1[3,10],zero,zero,zero,xmm1[u,u,u,u,u,u]
+; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm2, %xmm1, %xmm1
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa {{.*#+}} ymm9 = [65535,65535,65535,0,65535,65535,0,65535,65535,65535,0,65535,65535,0,65535,65535]
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 96(%rdi), %ymm19
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm11, %ymm2
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm24, %ymm19, %ymm2
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 80(%rdi), %xmm0
-; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm0[2],ymm2[3,4],ymm0[5],ymm2[6,7,8,9],ymm0[10],ymm2[11,12],ymm0[13],ymm2[14,15]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm9, %ymm2
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm31, %ymm19, %ymm2
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 80(%rdi), %xmm11
+; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm11[2],ymm2[3,4],ymm11[5],ymm2[6,7,8,9],ymm11[10],ymm2[11,12],ymm11[13],ymm2[14,15]
 ; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} ymm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[6,13,4,11,2,9,16,23,30,u],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm2
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa {{.*#+}} ymm1 = [65535,65535,0,65535,65535,65535,0,65535,65535,0,65535,65535,65535,0,65535,65535]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 128(%rdi), %ymm31
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa {{.*#+}} ymm14 = [65535,65535,0,65535,65535,65535,0,65535,65535,0,65535,65535,65535,0,65535,65535]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 128(%rdi), %ymm21
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 160(%rdi), %ymm29
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm1, %ymm4
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm31, %ymm29, %ymm4
-; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm4, %xmm5
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm5 = xmm5[u,u,u],zero,zero,xmm5[3,10],zero,zero,zero,xmm5[6,13,u,u,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[u,u,u,5,12],zero,zero,xmm4[1,8,15],zero,zero,xmm4[u,u,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm5, %xmm4, %xmm4
-; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 192(%rdi), %xmm5
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm7 = xmm5[u,u,u,u,u,u,4,11,u,u,u,u,u,u,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %xmm5, %xmm6
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 208(%rdi), %xmm5
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm9 = xmm5[u,u,u,u,u,u,2,9,u,u,u,u,u,u,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %xmm5, %xmm23
-; AVX512F-ONLY-SLOW-NEXT:    vpunpcklwd {{.*#+}} xmm7 = xmm7[0],xmm9[0],xmm7[1],xmm9[1],xmm7[2],xmm9[2],xmm7[3],xmm9[3]
-; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm7, %ymm0, %ymm7
-; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3,4,5,6],ymm7[7]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 240(%rdi), %xmm14
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm7 = zero,zero,zero,xmm14[5,12,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 224(%rdi), %xmm8
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm9 = xmm8[0,7,14],zero,zero,xmm8[u,u,u,u,u,u,u,u,u,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm7, %xmm9, %xmm7
-; AVX512F-ONLY-SLOW-NEXT:    vinserti32x4 $2, %xmm7, %zmm4, %zmm7
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm26 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $184, %zmm2, %zmm26, %zmm7
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 288(%rdi), %ymm20
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 256(%rdi), %ymm27
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm11, %ymm4
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm20, %ymm27, %ymm4
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm9 = xmm4[u,u,u,u,u,3,10],zero,zero,zero,xmm4[6,13],zero,zero,xmm4[u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm4, %xmm4
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[u,u,u,u,u],zero,zero,xmm4[1,8,15],zero,zero,xmm4[4,11,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm4, %xmm9, %xmm9
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 352(%rdi), %ymm18
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 320(%rdi), %ymm16
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm1, %ymm10
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm18, %ymm16, %ymm10
-; AVX512F-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm13 = ymm10[2,3,0,1]
-; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} ymm10 = ymm10[0,1],ymm13[2],ymm10[3,4,5],ymm13[6],ymm10[7,8,9],ymm13[10],ymm10[11,12,13],ymm13[14],ymm10[15]
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} ymm10 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm10[2,9,16,23,30,21,28,19,26,u,u,u,u,u,u,u,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} ymm28 = [65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,65535,65535,65535,65535]
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $248, %ymm28, %ymm9, %ymm10
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa {{.*#+}} ymm9 = [65535,65535,0,65535,65535,0,65535,65535,65535,0,65535,65535,0,65535,65535,65535]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 416(%rdi), %ymm21
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 384(%rdi), %ymm22
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm9, %ymm13
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm21, %ymm22, %ymm13
-; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm13, %xmm5
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm5 = xmm5[u,u,u,u,u,u,u],zero,zero,zero,xmm5[6,13],zero,zero,xmm5[2,9]
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm13 = xmm13[u,u,u,u,u,u,u,1,8,15],zero,zero,xmm13[4,11],zero,zero
-; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm5, %xmm13, %xmm5
-; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa {{.*#+}} ymm3 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0]
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $184, %ymm10, %ymm3, %ymm5
-; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $1, %ymm5, %zmm0, %zmm2
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm30 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $184, %zmm7, %zmm30, %zmm2
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, (%rsp) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm9, %ymm5
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm15, %ymm17, %ymm5
-; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm5, %xmm7
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm7 = zero,zero,zero,xmm7[6,13],zero,zero,xmm7[2,9,u,u,u,u,u,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm5 = xmm5[1,8,15],zero,zero,xmm5[4,11],zero,zero,xmm5[u,u,u,u,u,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm7, %xmm5, %xmm5
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm12, %ymm4
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm12, %ymm7
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm19, %ymm24, %ymm7
-; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} ymm7 = ymm7[0,1],ymm0[2],ymm7[3,4,5],ymm0[6],ymm7[7,8,9],ymm0[10],ymm7[11,12,13],ymm0[14],ymm7[15]
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} ymm7 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm7[0,7,14,5,12,3,10,17,24,31,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $244, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm5, %ymm7
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm11, %ymm5
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm31, %ymm29, %ymm5
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm10 = xmm5[u,u,u,6,13],zero,zero,xmm5[2,9],zero,zero,zero,xmm5[u,u,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm5, %xmm5
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm5 = xmm5[u,u,u],zero,zero,xmm5[4,11],zero,zero,xmm5[0,7,14,u,u,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm5, %xmm10, %xmm5
-; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %xmm23, %xmm2
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm10 = xmm2[u,u,u,u,u,u,3,10,u,u,u,u,u,u,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm13 = xmm6[u,u,u,u,u,u,5,12,u,u,u,u,u,u,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm13[0],xmm10[0],xmm13[1],xmm10[1],xmm13[2],xmm10[2],xmm13[3],xmm10[3]
-; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
-; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3,4,5,6],ymm10[7]
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm10 = zero,zero,zero,xmm14[6,13,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm13 = xmm8[1,8,15],zero,zero,xmm8[u,u,u,u,u,u,u,u,u,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm10, %xmm13, %xmm10
-; AVX512F-ONLY-SLOW-NEXT:    vinserti32x4 $2, %xmm10, %zmm5, %zmm5
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $184, %zmm7, %zmm26, %zmm5
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm9, %ymm7
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm27, %ymm20, %ymm7
-; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm7, %xmm10
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm10 = xmm10[u,u,u,u,u],zero,zero,xmm10[2,9],zero,zero,zero,xmm10[5,12,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm7 = xmm7[u,u,u,u,u,4,11],zero,zero,xmm7[0,7,14],zero,zero,xmm7[u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm7, %xmm10, %xmm7
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm11, %ymm10
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm18, %ymm16, %ymm10
-; AVX512F-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm13 = ymm10[2,3,0,1]
-; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} ymm10 = ymm10[0,1,2],ymm13[3],ymm10[4,5],ymm13[6],ymm10[7,8,9,10],ymm13[11],ymm10[12,13],ymm13[14],ymm10[15]
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} ymm10 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm10[3,10,17,24,31,22,29,20,27,u,u,u,u,u,u,u,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $248, %ymm28, %ymm7, %ymm10
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm1, %ymm7
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm21, %ymm22, %ymm7
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm13 = xmm7[u,u,u,u,u,u,u,2,9],zero,zero,zero,xmm7[5,12],zero,zero
-; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm7, %xmm7
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm7 = xmm7[u,u,u,u,u,u,u],zero,zero,xmm7[0,7,14],zero,zero,xmm7[3,10]
-; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm7, %xmm13, %xmm7
-; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm7, %ymm0, %ymm7
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $184, %ymm10, %ymm3, %ymm7
-; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $1, %ymm7, %zmm0, %zmm7
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $184, %zmm5, %zmm30, %zmm7
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm1, %ymm5
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm15, %ymm17, %ymm5
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm7 = xmm5[2,9],zero,zero,zero,xmm5[5,12],zero,zero,xmm5[u,u,u,u,u,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm5, %xmm5
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm5 = zero,zero,xmm5[0,7,14],zero,zero,xmm5[3,10,u,u,u,u,u,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm7, %xmm5, %xmm5
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm9, %ymm7
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm19, %ymm24, %ymm7
-; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} ymm7 = ymm7[0,1,2],ymm0[3],ymm7[4,5],ymm0[6],ymm7[7,8,9,10],ymm0[11],ymm7[12,13],ymm0[14],ymm7[15]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} ymm7 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm7[1,8,15,6,13,4,11,18,25],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} ymm26 = [255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $248, %ymm26, %ymm5, %ymm7
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm4, %ymm5
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %ymm4, %ymm23
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm29, %ymm31, %ymm5
-; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm5, %xmm10
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm10 = xmm10[u,u],zero,zero,zero,xmm10[5,12],zero,zero,xmm10[1,8,15,u,u,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm5 = xmm5[u,u,0,7,14],zero,zero,xmm5[3,10],zero,zero,zero,xmm5[u,u,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm5, %xmm10, %xmm5
-; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm10 = xmm2[u,u,u,u,u,u,4,11,u,u,u,u,u,u,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm13 = xmm6[u,u,u,u,u,u,6,13,u,u,u,u,u,u,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm13[0],xmm10[0],xmm13[1],xmm10[1],xmm13[2],xmm10[2],xmm13[3],xmm10[3]
-; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
-; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3,4,5,6],ymm10[7]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm10 = xmm8[2,9],zero,zero,zero,xmm8[u,u,u,u,u,u,u,u,u,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %xmm14, %xmm4
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm13 = zero,zero,xmm14[0,7,14,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm10, %xmm13, %xmm10
-; AVX512F-ONLY-SLOW-NEXT:    vinserti32x4 $2, %xmm10, %zmm5, %zmm5
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm14 = [0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $226, %zmm7, %zmm14, %zmm5
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm1, %ymm7
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm27, %ymm20, %ymm7
-; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm7, %xmm10
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm10 = xmm10[u,u,u,u,u],zero,zero,xmm10[3,10],zero,zero,zero,xmm10[6,13,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm7 = xmm7[u,u,u,u,u,5,12],zero,zero,xmm7[1,8,15],zero,zero,xmm7[u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm7, %xmm10, %xmm7
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm9, %ymm10
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm16, %ymm18, %ymm10
-; AVX512F-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm13 = ymm10[2,3,0,1]
-; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} ymm10 = ymm13[0],ymm10[1,2],ymm13[3],ymm10[4,5,6],ymm13[7,8],ymm10[9,10],ymm13[11],ymm10[12,13,14],ymm13[15]
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} ymm10 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm10[4,11,18,25,16,23,30,21,28,u,u,u,u,u,u,u,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $248, %ymm28, %ymm7, %ymm10
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm11, %ymm7
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm21, %ymm22, %ymm7
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm13 = xmm7[u,u,u,u,u,u,u,3,10],zero,zero,zero,xmm7[6,13],zero,zero
-; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm7, %xmm7
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm7 = xmm7[u,u,u,u,u,u,u],zero,zero,xmm7[1,8,15],zero,zero,xmm7[4,11]
-; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm7, %xmm13, %xmm7
-; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm7, %ymm0, %ymm7
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $184, %ymm10, %ymm3, %ymm7
-; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $1, %ymm7, %zmm0, %zmm25
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $184, %zmm5, %zmm30, %zmm25
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm11, %ymm5
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm15, %ymm17, %ymm5
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm7 = xmm5[3,10],zero,zero,zero,xmm5[6,13],zero,zero,xmm5[u,u,u,u,u,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm5, %xmm5
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm5 = zero,zero,xmm5[1,8,15],zero,zero,xmm5[4,11,u,u,u,u,u,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm7, %xmm5, %xmm5
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm1, %ymm7
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm19, %ymm24, %ymm7
-; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} ymm7 = ymm0[0],ymm7[1,2],ymm0[3],ymm7[4,5,6],ymm0[7,8],ymm7[9,10],ymm0[11],ymm7[12,13,14],ymm0[15]
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} ymm7 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm7[2,9,0,7,14,5,12,19,26],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $248, %ymm26, %ymm5, %ymm7
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm9, %ymm5
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm29, %ymm31, %ymm5
-; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm5, %xmm10
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm10 = xmm10[u,u],zero,zero,zero,xmm10[6,13],zero,zero,xmm10[2,9,u,u,u,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm5 = xmm5[u,u,1,8,15],zero,zero,xmm5[4,11],zero,zero,xmm5[u,u,u,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm5, %xmm10, %xmm5
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm10 = xmm2[u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,xmm2[5,12]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %xmm2, %xmm12
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm13 = xmm6[u,u,u,u,u,u,u,u,u,u,u,0,7,14],zero,zero
-; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm10, %xmm13, %xmm10
-; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm5, %ymm10
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm5 = xmm8[3,10],zero,zero,zero,xmm8[u,u,u,u,u,u,u,u,u,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm13 = zero,zero,xmm4[1,8,15,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm5, %xmm13, %xmm5
-; AVX512F-ONLY-SLOW-NEXT:    vinserti32x4 $2, %xmm5, %zmm10, %zmm5
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $226, %zmm7, %zmm14, %zmm5
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm11, %ymm7
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm27, %ymm20, %ymm7
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm10 = xmm7[u,u,u,u,u,6,13],zero,zero,xmm7[2,9],zero,zero,zero,xmm7[u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm7, %xmm7
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm7 = xmm7[u,u,u,u,u],zero,zero,xmm7[4,11],zero,zero,xmm7[0,7,14,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm7, %xmm10, %xmm7
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm1, %ymm10
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm16, %ymm18, %ymm10
-; AVX512F-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm13 = ymm10[2,3,0,1]
-; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} ymm10 = ymm13[0],ymm10[1,2,3],ymm13[4],ymm10[5,6],ymm13[7,8],ymm10[9,10,11],ymm13[12],ymm10[13,14],ymm13[15]
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} ymm10 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm10[5,12,19,26,17,24,31,22,29,u,u,u,u,u,u,u,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $248, %ymm28, %ymm7, %ymm10
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm9, %ymm7
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm22, %ymm21, %ymm7
-; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm7, %xmm13
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm13 = xmm13[u,u,u,u,u,u,u],zero,zero,xmm13[2,9],zero,zero,zero,xmm13[5,12]
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm7 = xmm7[u,u,u,u,u,u,u,4,11],zero,zero,xmm7[0,7,14],zero,zero
-; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm7, %xmm13, %xmm7
-; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm7, %ymm0, %ymm7
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $184, %ymm10, %ymm3, %ymm7
-; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $1, %ymm7, %zmm0, %zmm26
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $184, %zmm5, %zmm30, %zmm26
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm9, %ymm5
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm17, %ymm15, %ymm5
-; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm5, %xmm7
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm7 = zero,zero,xmm7[2,9],zero,zero,zero,xmm7[5,12,u,u,u,u,u,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm5 = xmm5[4,11],zero,zero,xmm5[0,7,14],zero,zero,xmm5[u,u,u,u,u,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm7, %xmm5, %xmm2
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %ymm23, %ymm5
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm20, %ymm27, %ymm5
-; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm5, %xmm7
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm7 = xmm7[u,u,u,u],zero,zero,zero,xmm7[5,12],zero,zero,xmm7[1,8,15,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm5 = xmm5[u,u,u,u,0,7,14],zero,zero,xmm5[3,10],zero,zero,zero,xmm5[u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm7, %xmm5, %xmm5
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm11, %ymm7
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm16, %ymm18, %ymm7
-; AVX512F-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm10 = ymm7[2,3,0,1]
-; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} ymm7 = ymm7[0],ymm10[1],ymm7[2,3],ymm10[4],ymm7[5,6,7,8],ymm10[9],ymm7[10,11],ymm10[12],ymm7[13,14,15]
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} ymm7 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm7[6,13,20,27,18,25,16,23,30,u,u,u,u,u,u,u,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $248, %ymm28, %ymm5, %ymm7
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm1, %ymm5
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm22, %ymm21, %ymm5
-; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm5, %xmm10
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm10 = xmm10[u,u,u,u,u,u,u],zero,zero,xmm10[3,10],zero,zero,zero,xmm10[6,13]
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm5 = xmm5[u,u,u,u,u,u,u,5,12],zero,zero,xmm5[1,8,15],zero,zero
-; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm5, %xmm10, %xmm5
-; AVX512F-ONLY-SLOW-NEXT:    vinserti32x4 $1, %xmm5, %ymm0, %ymm28
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $184, %ymm7, %ymm3, %ymm28
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm9, %ymm5
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm20, %ymm27, %ymm5
-; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm5, %xmm7
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm7 = xmm7[u,u,u,u],zero,zero,zero,xmm7[6,13],zero,zero,xmm7[2,9,u,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm5 = xmm5[u,u,u,u,1,8,15],zero,zero,xmm5[4,11],zero,zero,xmm5[u,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm7, %xmm5, %xmm5
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %ymm23, %ymm7
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm18, %ymm16, %ymm7
-; AVX512F-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm10 = ymm7[2,3,0,1]
-; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} ymm7 = ymm7[0],ymm10[1],ymm7[2,3,4],ymm10[5],ymm7[6,7,8],ymm10[9],ymm7[10,11,12],ymm10[13],ymm7[14,15]
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} ymm7 = ymm7[u,u,u,u],zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm7[0,7,14,21,28,19,26,17,24,31,u,u,u,u,u,u,u,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $244, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm5, %ymm7
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm11, %ymm5
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm22, %ymm21, %ymm5
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm10 = xmm5[u,u,u,u,u,u,u,6,13],zero,zero,xmm5[2,9],zero,zero,zero
-; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm5, %xmm5
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm5 = xmm5[u,u,u,u,u,u,u],zero,zero,xmm5[4,11],zero,zero,xmm5[0,7,14]
-; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm5, %xmm10, %xmm5
-; AVX512F-ONLY-SLOW-NEXT:    vinserti32x4 $1, %xmm5, %ymm0, %ymm30
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $184, %ymm7, %ymm3, %ymm30
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm1, %ymm5
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm29, %ymm31, %ymm5
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm7 = xmm5[u,u,2,9],zero,zero,zero,xmm5[5,12],zero,zero,xmm5[u,u,u,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm5, %xmm5
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm5 = xmm5[u,u],zero,zero,xmm5[0,7,14],zero,zero,xmm5[3,10,u,u,u,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm7, %xmm5, %xmm5
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm11, %ymm10
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm21, %ymm22, %ymm23
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm1, %ymm13
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm17, %ymm15, %ymm13
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm11, %ymm7
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm17, %ymm15, %ymm11
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %xmm12, %xmm4
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm15 = xmm12[u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,xmm12[6,13]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %xmm6, %xmm3
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm6 = xmm6[u,u,u,u,u,u,u,u,u,u,u,1,8,15],zero,zero
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm14, %ymm1
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm21, %ymm29, %ymm1
+; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm1, %xmm3
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u,u],zero,zero,xmm3[3,10],zero,zero,zero,xmm3[6,13,u,u,u,u]
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[u,u,u,5,12],zero,zero,xmm1[1,8,15],zero,zero,xmm1[u,u,u,u]
+; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm3, %xmm1, %xmm1
+; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 192(%rdi), %xmm0
+; AVX512F-ONLY-SLOW-NEXT:    vpbroadcastd {{.*#+}} xmm4 = [0,0,4,11,0,0,4,11,0,0,4,11,0,0,4,11]
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb %xmm4, %xmm0, %xmm3
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %xmm4, %xmm27
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %xmm0, %xmm20
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 208(%rdi), %xmm10
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm5 = xmm10[u,u,u,u,u,u,2,9,u,u,u,u,u,u,u,u]
+; AVX512F-ONLY-SLOW-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]
+; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5,6],ymm3[7]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 240(%rdi), %xmm0
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm5 = zero,zero,zero,xmm0[5,12,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %xmm0, %xmm3
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 224(%rdi), %xmm0
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm7 = xmm0[0,7,14],zero,zero,xmm0[u,u,u,u,u,u,u,u,u,u,u]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %xmm0, %xmm6
+; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm5, %xmm7, %xmm5
+; AVX512F-ONLY-SLOW-NEXT:    vinserti32x4 $2, %xmm5, %zmm1, %zmm22
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $184, %zmm2, %zmm4, %zmm22
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 288(%rdi), %ymm18
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 256(%rdi), %ymm16
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm9, %ymm2
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm18, %ymm16, %ymm2
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm5 = xmm2[u,u,u,u,u,3,10],zero,zero,zero,xmm2[6,13],zero,zero,xmm2[u,u]
+; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm2, %xmm2
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,u,u,u,u],zero,zero,xmm2[1,8,15],zero,zero,xmm2[4,11,u,u]
+; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm5, %xmm2, %xmm2
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 352(%rdi), %ymm17
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 320(%rdi), %ymm28
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm14, %ymm7
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm17, %ymm28, %ymm7
+; AVX512F-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm8 = ymm7[2,3,0,1]
+; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} ymm7 = ymm7[0,1],ymm8[2],ymm7[3,4,5],ymm8[6],ymm7[7,8,9],ymm8[10],ymm7[11,12,13],ymm8[14],ymm7[15]
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} ymm8 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm7[2,9,16,23,30,21,28,19,26,u,u,u,u,u,u,u,u,u]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} ymm23 = [65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,65535,65535,65535,65535]
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $248, %ymm23, %ymm2, %ymm8
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa {{.*#+}} ymm7 = [65535,65535,0,65535,65535,0,65535,65535,65535,0,65535,65535,0,65535,65535,65535]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm7, %ymm2
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm13, %ymm12, %ymm2
+; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm2, %xmm15
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm15 = zero,zero,zero,xmm15[6,13],zero,zero,xmm15[2,9,u,u,u,u,u,u,u]
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[1,8,15],zero,zero,xmm2[4,11],zero,zero,xmm2[u,u,u,u,u,u,u]
+; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm2, %xmm15, %xmm2
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %ymm24, %ymm15
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %ymm24, %ymm5
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm19, %ymm31, %ymm15
+; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} ymm15 = ymm15[0,1],ymm11[2],ymm15[3,4,5],ymm11[6],ymm15[7,8,9],ymm11[10],ymm15[11,12,13],ymm11[14],ymm15[15]
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} ymm15 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm15[0,7,14,5,12,3,10,17,24,31,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $244, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm15
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm9, %ymm2
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm21, %ymm29, %ymm2
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm2[u,u,u,6,13],zero,zero,xmm2[2,9],zero,zero,zero,xmm2[u,u,u,u]
+; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm2, %xmm2
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,u,u],zero,zero,xmm2[4,11],zero,zero,xmm2[0,7,14,u,u,u,u]
+; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm0, %xmm2, %xmm0
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm2 = zero,zero,zero,xmm3[6,13,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %xmm3, %xmm25
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm3 = xmm6[1,8,15],zero,zero,xmm6[u,u,u,u,u,u,u,u,u,u,u]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %xmm6, %xmm26
+; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm2, %xmm3, %xmm2
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm3 = xmm10[u,u,u,u,u,u,3,10,u,u,u,u,u,u,u,u]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %xmm20, %xmm1
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm6 = xmm1[u,u,u,u,u,u,5,12,u,u,u,u,u,u,u,u]
+; AVX512F-ONLY-SLOW-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm6[0],xmm3[0],xmm6[1],xmm3[1],xmm6[2],xmm3[2],xmm6[3],xmm3[3]
+; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,6],ymm3[7]
+; AVX512F-ONLY-SLOW-NEXT:    vinserti32x4 $2, %xmm2, %zmm0, %zmm24
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $184, %zmm15, %zmm4, %zmm24
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm14, %ymm0
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm13, %ymm12, %ymm0
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm2 = xmm0[2,9],zero,zero,zero,xmm0[5,12],zero,zero,xmm0[u,u,u,u,u,u,u]
+; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = zero,zero,xmm0[0,7,14],zero,zero,xmm0[3,10,u,u,u,u,u,u,u]
+; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm2, %xmm0, %xmm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm7, %ymm2
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm19, %ymm31, %ymm2
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1,2],ymm11[3],ymm2[4,5],ymm11[6],ymm2[7,8,9,10],ymm11[11],ymm2[12,13],ymm11[14],ymm2[15]
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} ymm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[1,8,15,6,13,4,11,18,25],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa {{.*#+}} ymm3 = [255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $248, %ymm3, %ymm0, %ymm2
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm3, %ymm15
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm5, %ymm0
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm29, %ymm21, %ymm0
+; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u],zero,zero,zero,xmm3[5,12],zero,zero,xmm3[1,8,15,u,u,u,u]
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,0,7,14],zero,zero,xmm0[3,10],zero,zero,zero,xmm0[u,u,u,u]
+; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm3, %xmm0, %xmm0
+; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %xmm27, %xmm3
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb %xmm3, %xmm10, %xmm3
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm6 = xmm1[u,u,u,u,u,u,6,13,u,u,u,u,u,u,u,u]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %xmm20, %xmm5
+; AVX512F-ONLY-SLOW-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm6[0],xmm3[0],xmm6[1],xmm3[1],xmm6[2],xmm3[2],xmm6[3],xmm3[3]
+; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,6],ymm3[7]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %xmm26, %xmm1
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %xmm26, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm3 = xmm1[2,9],zero,zero,zero,xmm1[u,u,u,u,u,u,u,u,u,u,u]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %xmm25, %xmm4
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %xmm25, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm6 = zero,zero,xmm4[0,7,14,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm3, %xmm6, %xmm3
+; AVX512F-ONLY-SLOW-NEXT:    vinserti32x4 $2, %xmm3, %zmm0, %zmm25
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm20 = [0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $226, %zmm2, %zmm20, %zmm25
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm9, %ymm0
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm13, %ymm12, %ymm0
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm2 = xmm0[3,10],zero,zero,zero,xmm0[6,13],zero,zero,xmm0[u,u,u,u,u,u,u]
+; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = zero,zero,xmm0[1,8,15],zero,zero,xmm0[4,11,u,u,u,u,u,u,u]
+; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm2, %xmm0, %xmm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm14, %ymm2
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm19, %ymm31, %ymm2
+; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} ymm2 = ymm11[0],ymm2[1,2],ymm11[3],ymm2[4,5,6],ymm11[7,8],ymm2[9,10],ymm11[11],ymm2[12,13,14],ymm11[15]
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} ymm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[2,9,0,7,14,5,12,19,26],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $248, %ymm15, %ymm0, %ymm3
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm15, %ymm11
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm7, %ymm0
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm29, %ymm21, %ymm0
+; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm2
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,u],zero,zero,zero,xmm2[6,13],zero,zero,xmm2[2,9,u,u,u,u,u]
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,1,8,15],zero,zero,xmm0[4,11],zero,zero,xmm0[u,u,u,u,u]
+; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm2, %xmm0, %xmm0
+; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa {{.*#+}} ymm2 = <128,128,128,128,128,128,128,128,128,128,128,128,128,128,5,12,19,26,17,24,31,22,29,u,u,u,u,u,u,u,u,u>
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb %xmm2, %xmm10, %xmm6
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm15 = xmm5[u,u,u,u,u,u,u,u,u,u,u,0,7,14],zero,zero
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %xmm5, %xmm10
 ; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm6, %xmm15, %xmm6
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm19, %ymm24, %ymm10
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $226, %ymm20, %ymm1, %ymm27
+; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm6
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm1[3,10],zero,zero,zero,xmm1[u,u,u,u,u,u,u,u,u,u,u]
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm15 = zero,zero,xmm4[1,8,15,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm0, %xmm15, %xmm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 416(%rdi), %ymm26
+; AVX512F-ONLY-SLOW-NEXT:    vinserti32x4 $2, %xmm0, %zmm6, %zmm30
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 384(%rdi), %ymm27
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $226, %zmm3, %zmm20, %zmm30
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm7, %ymm0
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm26, %ymm27, %ymm0
+; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u,u,u,u,u,u],zero,zero,zero,xmm3[6,13],zero,zero,xmm3[2,9]
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,u,u,1,8,15],zero,zero,xmm0[4,11],zero,zero
+; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm3, %xmm0, %xmm0
+; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa {{.*#+}} ymm1 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0]
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $184, %ymm8, %ymm1, %ymm0
+; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm20
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm8 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $184, %zmm22, %zmm8, %zmm20
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm7, %ymm0
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm16, %ymm18, %ymm0
+; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u,u,u,u],zero,zero,xmm3[2,9],zero,zero,zero,xmm3[5,12,u,u]
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,4,11],zero,zero,xmm0[0,7,14],zero,zero,xmm0[u,u]
+; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm3, %xmm0, %xmm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm9, %ymm3
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm17, %ymm28, %ymm3
+; AVX512F-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm6 = ymm3[2,3,0,1]
+; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0,1,2],ymm6[3],ymm3[4,5],ymm6[6],ymm3[7,8,9,10],ymm6[11],ymm3[12,13],ymm6[14],ymm3[15]
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} ymm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm3[3,10,17,24,31,22,29,20,27,u,u,u,u,u,u,u,u,u]
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $248, %ymm23, %ymm0, %ymm3
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm14, %ymm0
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm26, %ymm27, %ymm0
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm6 = xmm0[u,u,u,u,u,u,u,2,9],zero,zero,zero,xmm0[5,12],zero,zero
+; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,u,u],zero,zero,xmm0[0,7,14],zero,zero,xmm0[3,10]
+; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm6, %xmm0, %xmm0
+; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $184, %ymm3, %ymm1, %ymm0
+; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm22
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $184, %zmm24, %zmm8, %zmm22
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm14, %ymm0
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm16, %ymm18, %ymm0
+; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u,u,u,u],zero,zero,xmm3[3,10],zero,zero,zero,xmm3[6,13,u,u]
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,5,12],zero,zero,xmm0[1,8,15],zero,zero,xmm0[u,u]
+; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm3, %xmm0, %xmm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm7, %ymm3
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm28, %ymm17, %ymm3
+; AVX512F-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm6 = ymm3[2,3,0,1]
+; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} ymm3 = ymm6[0],ymm3[1,2],ymm6[3],ymm3[4,5,6],ymm6[7,8],ymm3[9,10],ymm6[11],ymm3[12,13,14],ymm6[15]
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} ymm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm3[4,11,18,25,16,23,30,21,28,u,u,u,u,u,u,u,u,u]
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $248, %ymm23, %ymm0, %ymm3
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm9, %ymm0
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm26, %ymm27, %ymm0
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm6 = xmm0[u,u,u,u,u,u,u,3,10],zero,zero,zero,xmm0[6,13],zero,zero
+; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,u,u],zero,zero,xmm0[1,8,15],zero,zero,xmm0[4,11]
+; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm6, %xmm0, %xmm0
+; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $184, %ymm3, %ymm1, %ymm0
+; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm24
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $184, %zmm25, %zmm8, %zmm24
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm14, %ymm0
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm28, %ymm17, %ymm0
+; AVX512F-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm3 = ymm0[2,3,0,1]
+; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm3[0],ymm0[1,2,3],ymm3[4],ymm0[5,6],ymm3[7,8],ymm0[9,10,11],ymm3[12],ymm0[13,14],ymm3[15]
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb %ymm2, %ymm0, %ymm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm9, %ymm2
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm16, %ymm18, %ymm2
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm3 = xmm2[u,u,u,u,u,6,13],zero,zero,xmm2[2,9],zero,zero,zero,xmm2[u,u]
+; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm2, %xmm2
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,u,u,u,u],zero,zero,xmm2[4,11],zero,zero,xmm2[0,7,14,u,u]
+; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm3, %xmm2, %xmm2
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $236, %ymm23, %ymm0, %ymm2
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm7, %ymm0
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm27, %ymm26, %ymm0
+; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u,u,u,u,u,u],zero,zero,xmm3[2,9],zero,zero,zero,xmm3[5,12]
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,u,u,4,11],zero,zero,xmm0[0,7,14],zero,zero
+; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm3, %xmm0, %xmm0
+; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $184, %ymm2, %ymm1, %ymm0
+; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm25
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $184, %zmm30, %zmm8, %zmm25
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm9, %ymm0
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm28, %ymm17, %ymm0
+; AVX512F-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm2 = ymm0[2,3,0,1]
+; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm2[1],ymm0[2,3],ymm2[4],ymm0[5,6,7,8],ymm2[9],ymm0[10,11],ymm2[12],ymm0[13,14,15]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa {{.*#+}} ymm4 = [65535,0,65535,65535,65535,0,65535,65535,0,65535,65535,65535,0,65535,65535,0]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm4, %ymm2
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm18, %ymm16, %ymm2
+; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm2, %xmm3
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u,u,u],zero,zero,zero,xmm3[5,12],zero,zero,xmm3[1,8,15,u,u]
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,u,u,u,0,7,14],zero,zero,xmm2[3,10],zero,zero,zero,xmm2[u,u]
+; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm3, %xmm2, %xmm2
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm0[6,13,20,27,18,25,16,23,30,u,u,u,u,u,u,u,u,u]
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $236, %ymm23, %ymm0, %ymm2
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm14, %ymm0
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm27, %ymm26, %ymm0
+; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u,u,u,u,u,u],zero,zero,xmm3[3,10],zero,zero,zero,xmm3[6,13]
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,u,u,5,12],zero,zero,xmm0[1,8,15],zero,zero
+; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm3, %xmm0, %xmm0
+; AVX512F-ONLY-SLOW-NEXT:    vinserti32x4 $1, %xmm0, %ymm0, %ymm30
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $184, %ymm2, %ymm1, %ymm30
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm4, %ymm0
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm17, %ymm28, %ymm0
+; AVX512F-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm2 = ymm0[2,3,0,1]
+; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm2[1],ymm0[2,3,4],ymm2[5],ymm0[6,7,8],ymm2[9],ymm0[10,11,12],ymm2[13],ymm0[14,15]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm7, %ymm2
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm18, %ymm16, %ymm2
+; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm2, %xmm3
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u,u,u],zero,zero,zero,xmm3[6,13],zero,zero,xmm3[2,9,u,u,u]
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,u,u,u,1,8,15],zero,zero,xmm2[4,11],zero,zero,xmm2[u,u,u]
+; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm3, %xmm2, %xmm2
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa {{.*#+}} ymm5 = <u,u,u,u,128,128,128,128,128,128,128,128,128,0,7,14,21,28,19,26,17,24,31,u,u,u,u,u,u,u,u,u>
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb %ymm5, %ymm0, %ymm0
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $220, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm2
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm9, %ymm0
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm27, %ymm26, %ymm0
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm3 = xmm0[u,u,u,u,u,u,u,6,13],zero,zero,xmm0[2,9],zero,zero,zero
+; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,u,u],zero,zero,xmm0[4,11],zero,zero,xmm0[0,7,14]
+; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm3, %xmm0, %xmm0
+; AVX512F-ONLY-SLOW-NEXT:    vinserti32x4 $1, %xmm0, %ymm0, %ymm23
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $184, %ymm2, %ymm1, %ymm23
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm7, %ymm0
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm12, %ymm13, %ymm0
+; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm2
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm2 = zero,zero,xmm2[2,9],zero,zero,zero,xmm2[5,12,u,u,u,u,u,u,u]
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[4,11],zero,zero,xmm0[0,7,14],zero,zero,xmm0[u,u,u,u,u,u,u]
+; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm2, %xmm0, %xmm0
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm26, %ymm27, %ymm4
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm14, %ymm2
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm12, %ymm13, %ymm2
+; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm2, %xmm3
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm3 = zero,zero,xmm3[3,10],zero,zero,zero,xmm3[6,13,u,u,u,u,u,u,u]
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[5,12],zero,zero,xmm2[1,8,15],zero,zero,xmm2[u,u,u,u,u,u,u]
+; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm3, %xmm2, %xmm2
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm9, %ymm3
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm9, %ymm15
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm24, %ymm19, %ymm15
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm24, %ymm19, %ymm1
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm12, %ymm13, %ymm9
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm19, %ymm31, %ymm3
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm6 = xmm9[6,13],zero,zero,xmm9[2,9],zero,zero,zero,xmm9[u,u,u,u,u,u,u]
+; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm9, %xmm9
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm9 = zero,zero,xmm9[4,11],zero,zero,xmm9[0,7,14,u,u,u,u,u,u,u]
+; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm6, %xmm9, %xmm6
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm14, %ymm12
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $226, %ymm18, %ymm14, %ymm16
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm7, %ymm9
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm31, %ymm19, %ymm9
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm31, %ymm19, %ymm14
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} ymm10 = ymm8[0],ymm10[1,2,3],ymm8[4],ymm10[5,6],ymm8[7,8],ymm10[9,10,11],ymm8[12],ymm10[13,14],ymm8[15]
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} ymm14 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm10[3,10,1,8,15,6,13,20,27],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa {{.*#+}} ymm0 = [255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $248, %ymm0, %ymm2, %ymm14
-; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} ymm17 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0]
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $184, %ymm5, %ymm17, %ymm6
-; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm13, %xmm5
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm5 = zero,zero,xmm5[3,10],zero,zero,zero,xmm5[6,13,u,u,u,u,u,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm10 = xmm13[5,12],zero,zero,xmm13[1,8,15],zero,zero,xmm13[u,u,u,u,u,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm5, %xmm10, %xmm5
-; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} ymm10 = ymm15[0],ymm8[1],ymm15[2,3],ymm8[4],ymm15[5,6,7,8],ymm8[9],ymm15[10,11],ymm8[12],ymm15[13,14,15]
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} ymm12 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm10[4,11,2,9,0,7,14,21,28],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $248, %ymm0, %ymm5, %ymm12
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm0, %ymm2
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm5 = xmm11[6,13],zero,zero,xmm11[2,9],zero,zero,zero,xmm11[u,u,u,u,u,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm11, %xmm0
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = zero,zero,xmm0[4,11],zero,zero,xmm0[0,7,14,u,u,u,u,u,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm5, %xmm0, %xmm5
-; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0],ymm8[1],ymm1[2,3,4],ymm8[5],ymm1[6,7,8],ymm8[9],ymm1[10,11,12],ymm8[13],ymm1[14,15]
-; AVX512F-ONLY-SLOW-NEXT:    vpbroadcastd {{.*#+}} xmm11 = [4,11,0,0,4,11,0,0,4,11,0,0,4,11,0,0]
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm0[5,12,3,10,1,8,15,22,29],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $248, %ymm2, %ymm5, %ymm0
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb %xmm11, %xmm2, %xmm1
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm5 = xmm15[2,9,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1],xmm1[2],xmm5[2],xmm1[3],xmm5[3]
-; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $1, %ymm1, %zmm6, %zmm10
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $184, %zmm14, %zmm1, %zmm10
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm29, %ymm31, %ymm7
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm5 = xmm7[u,u,3,10],zero,zero,zero,xmm7[6,13],zero,zero,xmm7[u,u,u,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm7, %xmm6
+; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} ymm3 = ymm8[0],ymm3[1,2,3],ymm8[4],ymm3[5,6],ymm8[7,8],ymm3[9,10,11],ymm8[12],ymm3[13,14],ymm8[15]
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} ymm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm3[3,10,1,8,15,6,13,20,27],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %ymm11, %ymm1
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $248, %ymm11, %ymm0, %ymm3
+; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm9[0],ymm8[1],ymm9[2,3],ymm8[4],ymm9[5,6,7,8],ymm8[9],ymm9[10,11],ymm8[12],ymm9[13,14,15]
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} ymm11 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm0[4,11,2,9,0,7,14,21,28],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $248, %ymm1, %ymm2, %ymm11
+; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm14[0],ymm8[1],ymm14[2,3,4],ymm8[5],ymm14[6,7,8],ymm8[9],ymm14[10,11,12],ymm8[13],ymm14[14,15]
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} ymm9 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm0[5,12,3,10,1,8,15,22,29],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $248, %ymm1, %ymm6, %ymm9
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm29, %ymm21, %ymm12
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm12[u,u,2,9],zero,zero,zero,xmm12[5,12],zero,zero,xmm12[u,u,u,u,u]
+; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm12, %xmm2
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,u],zero,zero,xmm2[0,7,14],zero,zero,xmm2[3,10,u,u,u,u,u]
+; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm0, %xmm2, %xmm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa {{.*#+}} ymm1 = <128,128,128,128,128,128,128,128,128,128,128,128,128,128,6,13,20,27,18,25,16,23,30,u,u,u,u,u,u,u,u,u>
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb %xmm1, %xmm12, %xmm2
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %xmm10, %xmm13
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm6 = xmm10[u,u,u,u,u,u,u,u,u,u,u,1,8,15],zero,zero
+; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm2, %xmm6, %xmm2
+; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa {{.*#+}} ymm1 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0]
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $184, %ymm0, %ymm1, %ymm2
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $226, %ymm17, %ymm7, %ymm28
+; AVX512F-ONLY-SLOW-NEXT:    vpbroadcastd {{.*#+}} xmm10 = [4,11,0,0,4,11,0,0,4,11,0,0,4,11,0,0]
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm29, %ymm21, %ymm15
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm21, %ymm29, %ymm7
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb %xmm10, %xmm8, %xmm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm6 = xmm14[2,9,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512F-ONLY-SLOW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm6[0],xmm0[1],xmm6[1],xmm0[2],xmm6[2],xmm0[3],xmm6[3]
+; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $1, %ymm0, %zmm2, %zmm2
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $184, %zmm3, %zmm0, %zmm2
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm3 = xmm15[u,u,3,10],zero,zero,zero,xmm15[6,13],zero,zero,xmm15[u,u,u,u,u]
+; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm15, %xmm6
 ; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm6 = xmm6[u,u],zero,zero,xmm6[1,8,15],zero,zero,xmm6[4,11,u,u,u,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm5, %xmm6, %xmm5
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $226, %ymm18, %ymm9, %ymm16
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $202, %ymm31, %ymm29, %ymm9
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm6 = xmm3[u,u,u,u,u,u,u,u,u,u,u,2,9],zero,zero,zero
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm7 = xmm4[u,u,u,u,u,u,u,u,u,u,u],zero,zero,xmm4[0,7,14]
-; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm6, %xmm7, %xmm6
-; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm3, %xmm6, %xmm3
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb %xmm5, %xmm12, %xmm6
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa %xmm12, %xmm15
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm12 = xmm13[u,u,u,u,u,u,u,u,u,u,u,2,9],zero,zero,zero
+; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm6, %xmm12, %xmm6
+; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
 ; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $184, %ymm5, %ymm17, %ymm6
-; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm9, %xmm5
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm5 = xmm5[u,u],zero,zero,xmm5[2,9],zero,zero,zero,xmm5[5,12,u,u,u,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm7 = xmm9[u,u,4,11],zero,zero,xmm9[0,7,14],zero,zero,xmm9[u,u,u,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm5, %xmm7, %xmm5
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm7 = xmm3[u,u,u,u,u,u,u,u,u,u,u,3,10],zero,zero,zero
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm8 = xmm4[u,u,u,u,u,u,u,u,u,u,u],zero,zero,xmm4[1,8,15]
-; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm7, %xmm8, %xmm7
-; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm7, %ymm0, %ymm7
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $184, %ymm5, %ymm17, %ymm7
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm5 = xmm15[3,10,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm8 = xmm2[5,12,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm8[0],xmm5[0],xmm8[1],xmm5[1],xmm8[2],xmm5[2],xmm8[3],xmm5[3]
-; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $1, %ymm5, %zmm6, %zmm5
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $184, %zmm12, %zmm1, %zmm5
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb %xmm11, %xmm15, %xmm6
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm8 = xmm2[6,13,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm8[0],xmm6[0],xmm8[1],xmm6[1],xmm8[2],xmm6[2],xmm8[3],xmm6[3]
-; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $1, %ymm6, %zmm7, %zmm6
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $184, %zmm0, %zmm1, %zmm6
-; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $1, %ymm28, %zmm0, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $184, %ymm3, %ymm1, %ymm6
+; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm7, %xmm3
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u],zero,zero,xmm3[2,9],zero,zero,zero,xmm3[5,12,u,u,u,u,u]
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm7 = xmm7[u,u,4,11],zero,zero,xmm7[0,7,14],zero,zero,xmm7[u,u,u,u,u]
+; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm3, %xmm7, %xmm3
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa {{.*#+}} ymm7 = [128,128,128,128,128,128,128,128,128,128,128,128,128,1,8,15,22,29,20,27,18,25,128,128,128,128,128,128,128,128,128,128]
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb %xmm7, %xmm15, %xmm12
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm13 = xmm13[u,u,u,u,u,u,u,u,u,u,u,3,10],zero,zero,zero
+; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm13, %xmm12, %xmm12
+; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm12, %ymm0, %ymm12
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $184, %ymm3, %ymm1, %ymm12
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm3 = xmm14[3,10,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm13 = xmm8[5,12,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512F-ONLY-SLOW-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm13[0],xmm3[0],xmm13[1],xmm3[1],xmm13[2],xmm3[2],xmm13[3],xmm3[3]
+; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $1, %ymm3, %zmm6, %zmm3
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $184, %zmm11, %zmm0, %zmm3
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb %xmm10, %xmm14, %xmm6
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm10 = xmm8[6,13,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512F-ONLY-SLOW-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm10[0],xmm6[0],xmm10[1],xmm6[1],xmm10[2],xmm6[2],xmm10[3],xmm6[3]
+; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $1, %ymm6, %zmm12, %zmm6
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $184, %zmm9, %zmm0, %zmm6
+; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $1, %ymm30, %zmm0, %zmm0
 ; AVX512F-ONLY-SLOW-NEXT:    movw $-512, %ax # imm = 0xFE00
 ; AVX512F-ONLY-SLOW-NEXT:    kmovw %eax, %k1
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa32 %zmm0, %zmm10 {%k1}
-; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $1, %ymm30, %zmm0, %zmm0
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa32 %zmm0, %zmm5 {%k1}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %ymm27, %ymm1
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm1[u,u,u,u,2,9],zero,zero,zero,xmm1[5,12],zero,zero,xmm1[u,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vextracti32x4 $1, %ymm27, %xmm1
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa32 %zmm0, %zmm2 {%k1}
+; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $1, %ymm23, %zmm0, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa32 %zmm0, %zmm3 {%k1}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %ymm28, %ymm1
+; AVX512F-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm28[2,3,0,1]
+; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7,8,9],ymm0[10],ymm1[11,12],ymm0[13],ymm1[14,15]
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb %ymm7, %ymm0, %ymm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %ymm16, %ymm1
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm5 = xmm1[u,u,u,u,2,9],zero,zero,zero,xmm1[5,12],zero,zero,xmm1[u,u,u]
+; AVX512F-ONLY-SLOW-NEXT:    vextracti32x4 $1, %ymm16, %xmm1
 ; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[u,u,u,u],zero,zero,xmm1[0,7,14],zero,zero,xmm1[3,10,u,u,u]
-; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm0, %xmm1, %xmm0
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %ymm16, %ymm2
-; AVX512F-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm1 = ymm16[2,3,0,1]
-; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0,1],ymm1[2],ymm2[3,4],ymm1[5],ymm2[6,7,8,9],ymm1[10],ymm2[11,12],ymm1[13],ymm2[14,15]
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} ymm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm1[1,8,15,22,29,20,27,18,25],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm1
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %ymm23, %ymm2
-; AVX512F-ONLY-SLOW-NEXT:    vextracti32x4 $1, %ymm23, %xmm0
+; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm5, %xmm1, %xmm1
+; AVX512F-ONLY-SLOW-NEXT:    vpternlogq $236, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm1
+; AVX512F-ONLY-SLOW-NEXT:    vextracti128 $1, %ymm4, %xmm0
 ; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,u],zero,zero,zero,xmm0[5,12],zero,zero,xmm0[1,8,15]
-; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,u,u,u,u,u,0,7,14],zero,zero,xmm2[3,10],zero,zero,zero
-; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm0, %xmm2, %xmm0
+; AVX512F-ONLY-SLOW-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[u,u,u,u,u,u,0,7,14],zero,zero,xmm4[3,10],zero,zero,zero
+; AVX512F-ONLY-SLOW-NEXT:    vpor %xmm0, %xmm4, %xmm0
 ; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
 ; AVX512F-ONLY-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6,7],ymm1[8,9,10],ymm0[11,12,13,14,15]
 ; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
 ; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm0
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqa32 %zmm0, %zmm6 {%k1}
-; AVX512F-ONLY-SLOW-NEXT:    vmovups (%rsp), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovaps %zmm0, (%rsi)
-; AVX512F-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovaps %zmm0, (%rdx)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm25, (%rcx)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm26, (%r8)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm10, (%r9)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm20, (%rsi)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm22, (%rdx)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm24, (%rcx)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm25, (%r8)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, (%r9)
 ; AVX512F-ONLY-SLOW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm5, (%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, (%rax)
 ; AVX512F-ONLY-SLOW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, (%rax)
-; AVX512F-ONLY-SLOW-NEXT:    addq $72, %rsp
 ; AVX512F-ONLY-SLOW-NEXT:    vzeroupper
 ; AVX512F-ONLY-SLOW-NEXT:    retq
 ;
 ; AVX512F-ONLY-FAST-LABEL: load_i8_stride7_vf64:
 ; AVX512F-ONLY-FAST:       # %bb.0:
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} ymm26 = [65535,0,65535,65535,65535,0,65535,65535,0,65535,65535,65535,0,65535,65535,0]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa (%rdi), %ymm13
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa 32(%rdi), %ymm14
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa 64(%rdi), %ymm10
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %ymm26, %ymm1
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm14, %ymm13, %ymm1
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa {{.*#+}} ymm0 = [65535,0,65535,65535,65535,0,65535,65535,0,65535,65535,65535,0,65535,65535,0]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 (%rdi), %ymm19
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa 32(%rdi), %ymm10
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 64(%rdi), %ymm30
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm0, %ymm1
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm0, %ymm7
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm10, %ymm19, %ymm1
 ; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm1, %xmm2
 ; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm2 = zero,zero,zero,xmm2[5,12],zero,zero,xmm2[1,8,15,u,u,u,u,u,u]
 ; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0,7,14],zero,zero,xmm1[3,10],zero,zero,zero,xmm1[u,u,u,u,u,u]
 ; AVX512F-ONLY-FAST-NEXT:    vpor %xmm2, %xmm1, %xmm1
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqa {{.*#+}} ymm6 = [65535,65535,65535,0,65535,65535,0,65535,65535,65535,0,65535,65535,0,65535,65535]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa 96(%rdi), %ymm12
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 96(%rdi), %ymm27
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm6, %ymm2
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm10, %ymm12, %ymm2
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa 80(%rdi), %xmm3
-; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm3[2],ymm2[3,4],ymm3[5],ymm2[6,7,8,9],ymm3[10],ymm2[11,12],ymm3[13],ymm2[14,15]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm3, %ymm9
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm30, %ymm27, %ymm2
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa 80(%rdi), %xmm9
+; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm9[2],ymm2[3,4],ymm9[5],ymm2[6,7,8,9],ymm9[10],ymm2[11,12],ymm9[13],ymm2[14,15]
 ; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm4 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[6,13,4,11,2,9,16,23,30,u],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; AVX512F-ONLY-FAST-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm4
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa {{.*#+}} ymm11 = [65535,65535,0,65535,65535,65535,0,65535,65535,0,65535,65535,65535,0,65535,65535]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 128(%rdi), %ymm30
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 160(%rdi), %ymm31
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm11, %ymm1
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm30, %ymm31, %ymm1
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa {{.*#+}} ymm12 = [65535,65535,0,65535,65535,65535,0,65535,65535,0,65535,65535,65535,0,65535,65535]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 128(%rdi), %ymm31
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 160(%rdi), %ymm29
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm12, %ymm1
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm31, %ymm29, %ymm1
 ; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm1, %xmm2
 ; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,u,u],zero,zero,xmm2[3,10],zero,zero,zero,xmm2[6,13,u,u,u,u]
 ; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[u,u,u,5,12],zero,zero,xmm1[1,8,15],zero,zero,xmm1[u,u,u,u]
 ; AVX512F-ONLY-FAST-NEXT:    vpor %xmm2, %xmm1, %xmm1
 ; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [1,2,4,6,1,2,4,6]
-; AVX512F-ONLY-FAST-NEXT:    # ymm3 = mem[0,1,0,1]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa 192(%rdi), %ymm2
-; AVX512F-ONLY-FAST-NEXT:    vpermd %ymm2, %ymm3, %ymm3
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm3 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,23,26,29]
-; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5,6],ymm3[7]
+; AVX512F-ONLY-FAST-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [1,2,4,6,1,2,4,6]
+; AVX512F-ONLY-FAST-NEXT:    # ymm2 = mem[0,1,0,1]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 192(%rdi), %ymm18
+; AVX512F-ONLY-FAST-NEXT:    vpermd %ymm18, %ymm2, %ymm2
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm2 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,23,26,29]
+; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5,6],ymm2[7]
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqa 240(%rdi), %xmm3
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm5 = zero,zero,zero,xmm3[5,12,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %xmm3, %xmm21
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa 224(%rdi), %xmm3
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm7 = xmm3[0,7,14],zero,zero,xmm3[u,u,u,u,u,u,u,u,u,u,u]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %xmm3, %xmm23
-; AVX512F-ONLY-FAST-NEXT:    vpor %xmm5, %xmm7, %xmm5
-; AVX512F-ONLY-FAST-NEXT:    vinserti32x4 $2, %xmm5, %zmm1, %zmm5
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm17 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $184, %zmm4, %zmm17, %zmm5
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 288(%rdi), %ymm18
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 256(%rdi), %ymm28
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm2 = zero,zero,zero,xmm3[5,12,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa %xmm3, %xmm8
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa 224(%rdi), %xmm0
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm5 = xmm0[0,7,14],zero,zero,xmm0[u,u,u,u,u,u,u,u,u,u,u]
+; AVX512F-ONLY-FAST-NEXT:    vpor %xmm2, %xmm5, %xmm2
+; AVX512F-ONLY-FAST-NEXT:    vinserti32x4 $2, %xmm2, %zmm1, %zmm20
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $184, %zmm4, %zmm2, %zmm20
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa 288(%rdi), %ymm11
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 256(%rdi), %ymm26
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm6, %ymm4
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm18, %ymm28, %ymm4
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm8 = xmm4[u,u,u,u,u,3,10],zero,zero,zero,xmm4[6,13],zero,zero,xmm4[u,u]
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm11, %ymm26, %ymm4
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm5 = xmm4[u,u,u,u,u,3,10],zero,zero,zero,xmm4[6,13],zero,zero,xmm4[u,u]
 ; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm4, %xmm4
 ; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[u,u,u,u,u],zero,zero,xmm4[1,8,15],zero,zero,xmm4[4,11,u,u]
-; AVX512F-ONLY-FAST-NEXT:    vpor %xmm4, %xmm8, %xmm8
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 352(%rdi), %ymm16
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa 320(%rdi), %ymm4
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm11, %ymm15
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm16, %ymm4, %ymm15
-; AVX512F-ONLY-FAST-NEXT:    vpermq {{.*#+}} ymm7 = ymm15[2,3,0,1]
-; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} ymm7 = ymm15[0,1],ymm7[2],ymm15[3,4,5],ymm7[6],ymm15[7,8,9],ymm7[10],ymm15[11,12,13],ymm7[14],ymm15[15]
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm7 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm7[2,9,16,23,30,21,28,19,26,u,u,u,u,u,u,u,u,u]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} ymm27 = [65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,65535,65535,65535,65535]
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $248, %ymm27, %ymm8, %ymm7
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa {{.*#+}} ymm15 = [65535,65535,0,65535,65535,0,65535,65535,65535,0,65535,65535,0,65535,65535,65535]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 416(%rdi), %ymm19
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 384(%rdi), %ymm20
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm15, %ymm8
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm19, %ymm20, %ymm8
-; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm8, %xmm3
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u,u,u,u,u,u],zero,zero,zero,xmm3[6,13],zero,zero,xmm3[2,9]
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm8 = xmm8[u,u,u,u,u,u,u,1,8,15],zero,zero,xmm8[4,11],zero,zero
-; AVX512F-ONLY-FAST-NEXT:    vpor %xmm3, %xmm8, %xmm3
-; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} ymm22 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0]
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $184, %ymm7, %ymm22, %ymm3
-; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm0
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm29 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $184, %zmm5, %zmm29, %zmm0
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm15, %ymm3
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm14, %ymm13, %ymm3
-; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm3, %xmm5
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm5 = zero,zero,zero,xmm5[6,13],zero,zero,xmm5[2,9,u,u,u,u,u,u,u]
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[1,8,15],zero,zero,xmm3[4,11],zero,zero,xmm3[u,u,u,u,u,u,u]
-; AVX512F-ONLY-FAST-NEXT:    vpor %xmm5, %xmm3, %xmm3
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %ymm26, %ymm5
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm12, %ymm10, %ymm5
-; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} ymm5 = ymm5[0,1],ymm9[2],ymm5[3,4,5],ymm9[6],ymm5[7,8,9],ymm9[10],ymm5[11,12,13],ymm9[14],ymm5[15]
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm5 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm5[0,7,14,5,12,3,10,17,24,31,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $244, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm5
+; AVX512F-ONLY-FAST-NEXT:    vpor %xmm5, %xmm4, %xmm13
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa 352(%rdi), %ymm15
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 320(%rdi), %ymm16
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm12, %ymm5
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm15, %ymm16, %ymm5
+; AVX512F-ONLY-FAST-NEXT:    vpermq {{.*#+}} ymm14 = ymm5[2,3,0,1]
+; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} ymm5 = ymm5[0,1],ymm14[2],ymm5[3,4,5],ymm14[6],ymm5[7,8,9],ymm14[10],ymm5[11,12,13],ymm14[14],ymm5[15]
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm5 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm5[2,9,16,23,30,21,28,19,26,u,u,u,u,u,u,u,u,u]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} ymm21 = [65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,65535,65535,65535,65535]
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $248, %ymm21, %ymm13, %ymm5
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa {{.*#+}} ymm14 = [65535,65535,0,65535,65535,0,65535,65535,65535,0,65535,65535,0,65535,65535,65535]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm14, %ymm13
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm10, %ymm19, %ymm13
+; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm13, %xmm3
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = zero,zero,zero,xmm3[6,13],zero,zero,xmm3[2,9,u,u,u,u,u,u,u]
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm13 = xmm13[1,8,15],zero,zero,xmm13[4,11],zero,zero,xmm13[u,u,u,u,u,u,u]
+; AVX512F-ONLY-FAST-NEXT:    vpor %xmm3, %xmm13, %xmm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm7, %ymm1
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm7, %ymm13
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm27, %ymm30, %ymm13
+; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} ymm13 = ymm13[0,1],ymm9[2],ymm13[3,4,5],ymm9[6],ymm13[7,8,9],ymm9[10],ymm13[11,12,13],ymm9[14],ymm13[15]
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm13 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm13[0,7,14,5,12,3,10,17,24,31,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $244, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm13
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm6, %ymm3
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm30, %ymm31, %ymm3
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm31, %ymm29, %ymm3
 ; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm7 = xmm3[u,u,u,6,13],zero,zero,xmm3[2,9],zero,zero,zero,xmm3[u,u,u,u]
 ; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm3, %xmm3
 ; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u,u],zero,zero,xmm3[4,11],zero,zero,xmm3[0,7,14,u,u,u,u]
@@ -9734,822 +9727,849 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
 ; AVX512F-ONLY-FAST-NEXT:    vbroadcasti128 {{.*#+}} ymm7 = [1,3,4,6,1,3,4,6]
 ; AVX512F-ONLY-FAST-NEXT:    # ymm7 = mem[0,1,0,1]
-; AVX512F-ONLY-FAST-NEXT:    vpermd %ymm2, %ymm7, %ymm7
+; AVX512F-ONLY-FAST-NEXT:    vpermd %ymm18, %ymm7, %ymm7
 ; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm7 = ymm7[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,17,20,27,30]
 ; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5,6],ymm7[7]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %xmm21, %xmm1
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm7 = zero,zero,zero,xmm1[6,13,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %xmm23, %xmm0
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm7 = zero,zero,zero,xmm8[6,13,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %xmm8, %xmm23
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa %xmm0, %xmm4
 ; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm8 = xmm0[1,8,15],zero,zero,xmm0[u,u,u,u,u,u,u,u,u,u,u]
 ; AVX512F-ONLY-FAST-NEXT:    vpor %xmm7, %xmm8, %xmm7
-; AVX512F-ONLY-FAST-NEXT:    vinserti32x4 $2, %xmm7, %zmm3, %zmm3
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $184, %zmm5, %zmm17, %zmm3
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm15, %ymm5
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm28, %ymm18, %ymm5
-; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm5, %xmm7
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm7 = xmm7[u,u,u,u,u],zero,zero,xmm7[2,9],zero,zero,zero,xmm7[5,12,u,u]
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm5 = xmm5[u,u,u,u,u,4,11],zero,zero,xmm5[0,7,14],zero,zero,xmm5[u,u]
-; AVX512F-ONLY-FAST-NEXT:    vpor %xmm7, %xmm5, %xmm5
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm6, %ymm7
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm16, %ymm4, %ymm7
-; AVX512F-ONLY-FAST-NEXT:    vpermq {{.*#+}} ymm8 = ymm7[2,3,0,1]
-; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} ymm7 = ymm7[0,1,2],ymm8[3],ymm7[4,5],ymm8[6],ymm7[7,8,9,10],ymm8[11],ymm7[12,13],ymm8[14],ymm7[15]
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm7 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm7[3,10,17,24,31,22,29,20,27,u,u,u,u,u,u,u,u,u]
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $248, %ymm27, %ymm5, %ymm7
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm11, %ymm5
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm19, %ymm20, %ymm5
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm8 = xmm5[u,u,u,u,u,u,u,2,9],zero,zero,zero,xmm5[5,12],zero,zero
-; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm5, %xmm5
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm5 = xmm5[u,u,u,u,u,u,u],zero,zero,xmm5[0,7,14],zero,zero,xmm5[3,10]
-; AVX512F-ONLY-FAST-NEXT:    vpor %xmm5, %xmm8, %xmm5
-; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $184, %ymm7, %ymm22, %ymm5
-; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $1, %ymm5, %zmm0, %zmm17
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $184, %zmm3, %zmm29, %zmm17
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm11, %ymm3
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm14, %ymm13, %ymm3
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm5 = xmm3[2,9],zero,zero,zero,xmm3[5,12],zero,zero,xmm3[u,u,u,u,u,u,u]
-; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm3, %xmm3
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = zero,zero,xmm3[0,7,14],zero,zero,xmm3[3,10,u,u,u,u,u,u,u]
-; AVX512F-ONLY-FAST-NEXT:    vpor %xmm5, %xmm3, %xmm3
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm15, %ymm5
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm12, %ymm10, %ymm5
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} ymm5 = ymm5[0,1,2],ymm9[3],ymm5[4,5],ymm9[6],ymm5[7,8,9,10],ymm9[11],ymm5[12,13],ymm9[14],ymm5[15]
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm5 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm5[1,8,15,6,13,4,11,18,25],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} ymm23 = [255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $248, %ymm23, %ymm3, %ymm5
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %ymm26, %ymm3
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm31, %ymm30, %ymm3
-; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm3, %xmm7
+; AVX512F-ONLY-FAST-NEXT:    vinserti32x4 $2, %xmm7, %zmm3, %zmm22
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $184, %zmm13, %zmm2, %zmm22
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm12, %ymm2
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm10, %ymm19, %ymm2
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = xmm2[2,9],zero,zero,zero,xmm2[5,12],zero,zero,xmm2[u,u,u,u,u,u,u]
+; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm2, %xmm2
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm2 = zero,zero,xmm2[0,7,14],zero,zero,xmm2[3,10,u,u,u,u,u,u,u]
+; AVX512F-ONLY-FAST-NEXT:    vpor %xmm3, %xmm2, %xmm2
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm14, %ymm3
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm27, %ymm30, %ymm3
+; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0,1,2],ymm9[3],ymm3[4,5],ymm9[6],ymm3[7,8,9,10],ymm9[11],ymm3[12,13],ymm9[14],ymm3[15]
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm3[1,8,15,6,13,4,11,18,25],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} ymm17 = [255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $248, %ymm17, %ymm2, %ymm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm1, %ymm2
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm29, %ymm31, %ymm2
+; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm2, %xmm7
 ; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm7 = xmm7[u,u],zero,zero,zero,xmm7[5,12],zero,zero,xmm7[1,8,15,u,u,u,u]
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u,0,7,14],zero,zero,xmm3[3,10],zero,zero,zero,xmm3[u,u,u,u]
-; AVX512F-ONLY-FAST-NEXT:    vpor %xmm7, %xmm3, %xmm3
-; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,u,0,7,14],zero,zero,xmm2[3,10],zero,zero,zero,xmm2[u,u,u,u]
+; AVX512F-ONLY-FAST-NEXT:    vpor %xmm7, %xmm2, %xmm2
+; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
 ; AVX512F-ONLY-FAST-NEXT:    vbroadcasti128 {{.*#+}} ymm7 = [1,3,5,6,1,3,5,6]
 ; AVX512F-ONLY-FAST-NEXT:    # ymm7 = mem[0,1,0,1]
-; AVX512F-ONLY-FAST-NEXT:    vpermd %ymm2, %ymm7, %ymm2
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm2 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,21,24,31]
-; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5,6],ymm2[7]
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = xmm0[2,9],zero,zero,zero,xmm0[u,u,u,u,u,u,u,u,u,u,u]
+; AVX512F-ONLY-FAST-NEXT:    vpermd %ymm18, %ymm7, %ymm7
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm7 = ymm7[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,21,24,31]
+; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5,6],ymm7[7]
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm7 = xmm0[2,9],zero,zero,zero,xmm0[u,u,u,u,u,u,u,u,u,u,u]
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm7 = zero,zero,xmm1[0,7,14,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512F-ONLY-FAST-NEXT:    vpor %xmm3, %xmm7, %xmm3
-; AVX512F-ONLY-FAST-NEXT:    vinserti32x4 $2, %xmm3, %zmm2, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm24 = [0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $226, %zmm5, %zmm24, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm11, %ymm3
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm28, %ymm18, %ymm3
-; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm3, %xmm5
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm5 = xmm5[u,u,u,u,u],zero,zero,xmm5[3,10],zero,zero,zero,xmm5[6,13,u,u]
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u,u,u,u,5,12],zero,zero,xmm3[1,8,15],zero,zero,xmm3[u,u]
-; AVX512F-ONLY-FAST-NEXT:    vpor %xmm5, %xmm3, %xmm3
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm15, %ymm5
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm4, %ymm16, %ymm5
-; AVX512F-ONLY-FAST-NEXT:    vpermq {{.*#+}} ymm7 = ymm5[2,3,0,1]
-; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} ymm5 = ymm7[0],ymm5[1,2],ymm7[3],ymm5[4,5,6],ymm7[7,8],ymm5[9,10],ymm7[11],ymm5[12,13,14],ymm7[15]
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm5 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm5[4,11,18,25,16,23,30,21,28,u,u,u,u,u,u,u,u,u]
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $248, %ymm27, %ymm3, %ymm5
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm6, %ymm3
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm19, %ymm20, %ymm3
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm7 = xmm3[u,u,u,u,u,u,u,3,10],zero,zero,zero,xmm3[6,13],zero,zero
-; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm3, %xmm3
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u,u,u,u,u,u],zero,zero,xmm3[1,8,15],zero,zero,xmm3[4,11]
-; AVX512F-ONLY-FAST-NEXT:    vpor %xmm7, %xmm3, %xmm3
-; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $184, %ymm5, %ymm22, %ymm3
-; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm21
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $184, %zmm2, %zmm29, %zmm21
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %xmm23, %xmm1
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %xmm23, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm8 = zero,zero,xmm1[0,7,14,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512F-ONLY-FAST-NEXT:    vpor %xmm7, %xmm8, %xmm7
+; AVX512F-ONLY-FAST-NEXT:    vinserti32x4 $2, %xmm7, %zmm2, %zmm23
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm18 = [0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $226, %zmm3, %zmm18, %zmm23
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm6, %ymm2
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm14, %ymm13, %ymm2
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm10, %ymm19, %ymm2
 ; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = xmm2[3,10],zero,zero,zero,xmm2[6,13],zero,zero,xmm2[u,u,u,u,u,u,u]
 ; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm2, %xmm2
 ; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm2 = zero,zero,xmm2[1,8,15],zero,zero,xmm2[4,11,u,u,u,u,u,u,u]
 ; AVX512F-ONLY-FAST-NEXT:    vpor %xmm3, %xmm2, %xmm2
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm11, %ymm3
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm12, %ymm10, %ymm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm12, %ymm3
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm27, %ymm30, %ymm3
 ; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} ymm3 = ymm9[0],ymm3[1,2],ymm9[3],ymm3[4,5,6],ymm9[7,8],ymm3[9,10],ymm9[11],ymm3[12,13,14],ymm9[15]
 ; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm3[2,9,0,7,14,5,12,19,26],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $248, %ymm23, %ymm2, %ymm3
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm15, %ymm2
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm31, %ymm30, %ymm2
-; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm2, %xmm5
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm5 = xmm5[u,u],zero,zero,zero,xmm5[6,13],zero,zero,xmm5[2,9,u,u,u,u,u]
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $248, %ymm17, %ymm2, %ymm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm14, %ymm2
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm29, %ymm31, %ymm2
+; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm2, %xmm7
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm7 = xmm7[u,u],zero,zero,zero,xmm7[6,13],zero,zero,xmm7[2,9,u,u,u,u,u]
 ; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,u,1,8,15],zero,zero,xmm2[4,11],zero,zero,xmm2[u,u,u,u,u]
-; AVX512F-ONLY-FAST-NEXT:    vpor %xmm5, %xmm2, %xmm5
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa 208(%rdi), %xmm7
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm8 = xmm7[u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,xmm7[5,12]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa 192(%rdi), %xmm2
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm9 = xmm2[u,u,u,u,u,u,u,u,u,u,u,0,7,14],zero,zero
-; AVX512F-ONLY-FAST-NEXT:    vpor %xmm8, %xmm9, %xmm8
-; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512F-ONLY-FAST-NEXT:    vpor %xmm7, %xmm2, %xmm2
+; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm7
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa {{.*#+}} ymm2 = <128,128,128,128,128,128,128,128,128,128,128,128,128,128,5,12,19,26,17,24,31,22,29,u,u,u,u,u,u,u,u,u>
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa 208(%rdi), %xmm0
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vpshufb %xmm2, %xmm0, %xmm8
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa 192(%rdi), %xmm0
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm13 = xmm0[u,u,u,u,u,u,u,u,u,u,u,0,7,14],zero,zero
+; AVX512F-ONLY-FAST-NEXT:    vpor %xmm8, %xmm13, %xmm8
 ; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm8, %ymm0, %ymm8
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} ymm25 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0]
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $184, %ymm5, %ymm25, %ymm8
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm5 = xmm0[3,10],zero,zero,zero,xmm0[u,u,u,u,u,u,u,u,u,u,u]
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm9 = zero,zero,xmm1[1,8,15,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa %xmm1, %xmm0
-; AVX512F-ONLY-FAST-NEXT:    vpor %xmm5, %xmm9, %xmm5
-; AVX512F-ONLY-FAST-NEXT:    vinserti32x4 $2, %xmm5, %zmm8, %zmm5
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $226, %zmm3, %zmm24, %zmm5
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm7, %ymm8
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm7 = xmm4[3,10],zero,zero,zero,xmm4[u,u,u,u,u,u,u,u,u,u,u]
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm13 = zero,zero,xmm1[1,8,15,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512F-ONLY-FAST-NEXT:    vpor %xmm7, %xmm13, %xmm7
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 416(%rdi), %ymm24
+; AVX512F-ONLY-FAST-NEXT:    vinserti32x4 $2, %xmm7, %zmm8, %zmm28
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 384(%rdi), %ymm25
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $226, %zmm3, %zmm18, %zmm28
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm14, %ymm3
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm24, %ymm25, %ymm3
+; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm3, %xmm7
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm7 = xmm7[u,u,u,u,u,u,u],zero,zero,zero,xmm7[6,13],zero,zero,xmm7[2,9]
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u,u,u,u,u,u,1,8,15],zero,zero,xmm3[4,11],zero,zero
+; AVX512F-ONLY-FAST-NEXT:    vpor %xmm7, %xmm3, %xmm3
+; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa {{.*#+}} ymm0 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0]
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $184, %ymm5, %ymm0, %ymm3
+; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm18
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $184, %zmm20, %zmm5, %zmm18
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm14, %ymm3
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm26, %ymm11, %ymm3
+; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm3, %xmm7
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm7 = xmm7[u,u,u,u,u],zero,zero,xmm7[2,9],zero,zero,zero,xmm7[5,12,u,u]
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u,u,u,u,4,11],zero,zero,xmm3[0,7,14],zero,zero,xmm3[u,u]
+; AVX512F-ONLY-FAST-NEXT:    vpor %xmm7, %xmm3, %xmm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm6, %ymm7
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm15, %ymm16, %ymm7
+; AVX512F-ONLY-FAST-NEXT:    vpermq {{.*#+}} ymm8 = ymm7[2,3,0,1]
+; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} ymm7 = ymm7[0,1,2],ymm8[3],ymm7[4,5],ymm8[6],ymm7[7,8,9,10],ymm8[11],ymm7[12,13],ymm8[14],ymm7[15]
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm7 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm7[3,10,17,24,31,22,29,20,27,u,u,u,u,u,u,u,u,u]
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $248, %ymm21, %ymm3, %ymm7
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm12, %ymm3
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm24, %ymm25, %ymm3
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm8 = xmm3[u,u,u,u,u,u,u,2,9],zero,zero,zero,xmm3[5,12],zero,zero
+; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm3, %xmm3
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u,u,u,u,u,u],zero,zero,xmm3[0,7,14],zero,zero,xmm3[3,10]
+; AVX512F-ONLY-FAST-NEXT:    vpor %xmm3, %xmm8, %xmm3
+; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $184, %ymm7, %ymm0, %ymm3
+; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm20
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $184, %zmm22, %zmm5, %zmm20
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm12, %ymm3
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm26, %ymm11, %ymm3
+; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm3, %xmm7
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm7 = xmm7[u,u,u,u,u],zero,zero,xmm7[3,10],zero,zero,zero,xmm7[6,13,u,u]
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u,u,u,u,5,12],zero,zero,xmm3[1,8,15],zero,zero,xmm3[u,u]
+; AVX512F-ONLY-FAST-NEXT:    vpor %xmm7, %xmm3, %xmm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm14, %ymm7
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm16, %ymm15, %ymm7
+; AVX512F-ONLY-FAST-NEXT:    vpermq {{.*#+}} ymm8 = ymm7[2,3,0,1]
+; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} ymm7 = ymm8[0],ymm7[1,2],ymm8[3],ymm7[4,5,6],ymm8[7,8],ymm7[9,10],ymm8[11],ymm7[12,13,14],ymm8[15]
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm7 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm7[4,11,18,25,16,23,30,21,28,u,u,u,u,u,u,u,u,u]
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $248, %ymm21, %ymm3, %ymm7
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm6, %ymm3
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm28, %ymm18, %ymm3
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm8 = xmm3[u,u,u,u,u,6,13],zero,zero,xmm3[2,9],zero,zero,zero,xmm3[u,u]
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm24, %ymm25, %ymm3
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm8 = xmm3[u,u,u,u,u,u,u,3,10],zero,zero,zero,xmm3[6,13],zero,zero
 ; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm3, %xmm3
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u,u,u,u],zero,zero,xmm3[4,11],zero,zero,xmm3[0,7,14,u,u]
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u,u,u,u,u,u],zero,zero,xmm3[1,8,15],zero,zero,xmm3[4,11]
 ; AVX512F-ONLY-FAST-NEXT:    vpor %xmm3, %xmm8, %xmm3
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm11, %ymm8
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm4, %ymm16, %ymm8
-; AVX512F-ONLY-FAST-NEXT:    vpermq {{.*#+}} ymm9 = ymm8[2,3,0,1]
-; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} ymm8 = ymm9[0],ymm8[1,2,3],ymm9[4],ymm8[5,6],ymm9[7,8],ymm8[9,10,11],ymm9[12],ymm8[13,14],ymm9[15]
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm8 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm8[5,12,19,26,17,24,31,22,29,u,u,u,u,u,u,u,u,u]
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $248, %ymm27, %ymm3, %ymm8
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm15, %ymm3
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm20, %ymm19, %ymm3
-; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm3, %xmm9
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm9 = xmm9[u,u,u,u,u,u,u],zero,zero,xmm9[2,9],zero,zero,zero,xmm9[5,12]
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u,u,u,u,u,u,4,11],zero,zero,xmm3[0,7,14],zero,zero
-; AVX512F-ONLY-FAST-NEXT:    vpor %xmm3, %xmm9, %xmm3
 ; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $184, %ymm8, %ymm22, %ymm3
-; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm24
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $184, %zmm5, %zmm29, %zmm24
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm15, %ymm3
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm13, %ymm14, %ymm3
-; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm3, %xmm5
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm5 = zero,zero,xmm5[2,9],zero,zero,zero,xmm5[5,12,u,u,u,u,u,u,u]
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[4,11],zero,zero,xmm3[0,7,14],zero,zero,xmm3[u,u,u,u,u,u,u]
-; AVX512F-ONLY-FAST-NEXT:    vpor %xmm5, %xmm3, %xmm1
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %ymm26, %ymm3
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm18, %ymm28, %ymm3
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $184, %ymm7, %ymm0, %ymm3
+; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm22
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $184, %zmm23, %zmm5, %zmm22
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm12, %ymm3
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm16, %ymm15, %ymm3
+; AVX512F-ONLY-FAST-NEXT:    vpermq {{.*#+}} ymm7 = ymm3[2,3,0,1]
+; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} ymm3 = ymm7[0],ymm3[1,2,3],ymm7[4],ymm3[5,6],ymm7[7,8],ymm3[9,10,11],ymm7[12],ymm3[13,14],ymm7[15]
+; AVX512F-ONLY-FAST-NEXT:    vpshufb %ymm2, %ymm3, %ymm2
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm6, %ymm3
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm26, %ymm11, %ymm3
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm7 = xmm3[u,u,u,u,u,6,13],zero,zero,xmm3[2,9],zero,zero,zero,xmm3[u,u]
+; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm3, %xmm3
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u,u,u,u],zero,zero,xmm3[4,11],zero,zero,xmm3[0,7,14,u,u]
+; AVX512F-ONLY-FAST-NEXT:    vpor %xmm7, %xmm3, %xmm3
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $236, %ymm21, %ymm2, %ymm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm14, %ymm2
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm25, %ymm24, %ymm2
+; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm2, %xmm7
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm7 = xmm7[u,u,u,u,u,u,u],zero,zero,xmm7[2,9],zero,zero,zero,xmm7[5,12]
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,u,u,u,u,u,u,4,11],zero,zero,xmm2[0,7,14],zero,zero
+; AVX512F-ONLY-FAST-NEXT:    vpor %xmm7, %xmm2, %xmm2
+; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $184, %ymm3, %ymm0, %ymm2
+; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm23
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $184, %zmm28, %zmm5, %zmm23
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm6, %ymm2
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm16, %ymm15, %ymm2
+; AVX512F-ONLY-FAST-NEXT:    vpermq {{.*#+}} ymm3 = ymm2[2,3,0,1]
+; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0],ymm3[1],ymm2[2,3],ymm3[4],ymm2[5,6,7,8],ymm3[9],ymm2[10,11],ymm3[12],ymm2[13,14,15]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa {{.*#+}} ymm1 = [65535,0,65535,65535,65535,0,65535,65535,0,65535,65535,65535,0,65535,65535,0]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm1, %ymm3
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm11, %ymm26, %ymm3
 ; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm3, %xmm5
 ; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm5 = xmm5[u,u,u,u],zero,zero,zero,xmm5[5,12],zero,zero,xmm5[1,8,15,u,u]
 ; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u,u,u,0,7,14],zero,zero,xmm3[3,10],zero,zero,zero,xmm3[u,u]
 ; AVX512F-ONLY-FAST-NEXT:    vpor %xmm5, %xmm3, %xmm3
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm6, %ymm5
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm4, %ymm16, %ymm5
-; AVX512F-ONLY-FAST-NEXT:    vpermq {{.*#+}} ymm8 = ymm5[2,3,0,1]
-; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} ymm5 = ymm5[0],ymm8[1],ymm5[2,3],ymm8[4],ymm5[5,6,7,8],ymm8[9],ymm5[10,11],ymm8[12],ymm5[13,14,15]
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm5 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm5[6,13,20,27,18,25,16,23,30,u,u,u,u,u,u,u,u,u]
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $248, %ymm27, %ymm3, %ymm5
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm11, %ymm3
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm20, %ymm19, %ymm3
-; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm3, %xmm8
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm8 = xmm8[u,u,u,u,u,u,u],zero,zero,xmm8[3,10],zero,zero,zero,xmm8[6,13]
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u,u,u,u,u,u,5,12],zero,zero,xmm3[1,8,15],zero,zero
-; AVX512F-ONLY-FAST-NEXT:    vpor %xmm3, %xmm8, %xmm3
-; AVX512F-ONLY-FAST-NEXT:    vinserti32x4 $1, %xmm3, %ymm0, %ymm27
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $184, %ymm5, %ymm22, %ymm27
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm15, %ymm3
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm18, %ymm28, %ymm3
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[6,13,20,27,18,25,16,23,30,u,u,u,u,u,u,u,u,u]
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $236, %ymm21, %ymm2, %ymm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm12, %ymm2
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm25, %ymm24, %ymm2
+; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm2, %xmm5
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm5 = xmm5[u,u,u,u,u,u,u],zero,zero,xmm5[3,10],zero,zero,zero,xmm5[6,13]
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,u,u,u,u,u,u,5,12],zero,zero,xmm2[1,8,15],zero,zero
+; AVX512F-ONLY-FAST-NEXT:    vpor %xmm5, %xmm2, %xmm2
+; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm4
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $184, %ymm3, %ymm0, %ymm4
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm1, %ymm2
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %ymm1, %ymm21
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm15, %ymm16, %ymm2
+; AVX512F-ONLY-FAST-NEXT:    vpermq {{.*#+}} ymm3 = ymm2[2,3,0,1]
+; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0],ymm3[1],ymm2[2,3,4],ymm3[5],ymm2[6,7,8],ymm3[9],ymm2[10,11,12],ymm3[13],ymm2[14,15]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm14, %ymm3
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm11, %ymm26, %ymm3
 ; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm3, %xmm5
 ; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm5 = xmm5[u,u,u,u],zero,zero,zero,xmm5[6,13],zero,zero,xmm5[2,9,u,u,u]
 ; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u,u,u,1,8,15],zero,zero,xmm3[4,11],zero,zero,xmm3[u,u,u]
 ; AVX512F-ONLY-FAST-NEXT:    vpor %xmm5, %xmm3, %xmm3
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %ymm26, %ymm5
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm16, %ymm4, %ymm5
-; AVX512F-ONLY-FAST-NEXT:    vpermq {{.*#+}} ymm8 = ymm5[2,3,0,1]
-; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} ymm5 = ymm5[0],ymm8[1],ymm5[2,3,4],ymm8[5],ymm5[6,7,8],ymm8[9],ymm5[10,11,12],ymm8[13],ymm5[14,15]
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm5 = ymm5[u,u,u,u],zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm5[0,7,14,21,28,19,26,17,24,31,u,u,u,u,u,u,u,u,u]
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $244, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm5
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm6, %ymm3
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm20, %ymm19, %ymm3
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm8 = xmm3[u,u,u,u,u,u,u,6,13],zero,zero,xmm3[2,9],zero,zero,zero
-; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm3, %xmm3
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u,u,u,u,u,u],zero,zero,xmm3[4,11],zero,zero,xmm3[0,7,14]
-; AVX512F-ONLY-FAST-NEXT:    vpor %xmm3, %xmm8, %xmm3
-; AVX512F-ONLY-FAST-NEXT:    vinserti32x4 $1, %xmm3, %ymm0, %ymm29
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $184, %ymm5, %ymm22, %ymm29
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm11, %ymm3
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm31, %ymm30, %ymm3
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm5 = xmm3[u,u,2,9],zero,zero,zero,xmm3[5,12],zero,zero,xmm3[u,u,u,u,u]
-; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm3, %xmm3
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u],zero,zero,xmm3[0,7,14],zero,zero,xmm3[3,10,u,u,u,u,u]
-; AVX512F-ONLY-FAST-NEXT:    vpor %xmm5, %xmm3, %xmm3
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm6, %ymm8
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm19, %ymm20, %ymm26
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm11, %ymm9
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm13, %ymm14, %ymm9
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm6, %ymm5
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm13, %ymm14, %ymm6
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm13 = xmm7[u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,xmm7[6,13]
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm14 = xmm2[u,u,u,u,u,u,u,u,u,u,u,1,8,15],zero,zero
-; AVX512F-ONLY-FAST-NEXT:    vpor %xmm13, %xmm14, %xmm13
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm12, %ymm10, %ymm8
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $226, %ymm18, %ymm11, %ymm28
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm15, %ymm14
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm10, %ymm12, %ymm14
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm10, %ymm12, %ymm11
-; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm9, %xmm10
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm10 = zero,zero,xmm10[3,10],zero,zero,zero,xmm10[6,13,u,u,u,u,u,u,u]
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm9 = xmm9[5,12],zero,zero,xmm9[1,8,15],zero,zero,xmm9[u,u,u,u,u,u,u]
-; AVX512F-ONLY-FAST-NEXT:    vpor %xmm10, %xmm9, %xmm9
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} ymm8 = ymm10[0],ymm8[1,2,3],ymm10[4],ymm8[5,6],ymm10[7,8],ymm8[9,10,11],ymm10[12],ymm8[13,14],ymm10[15]
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm12 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm8[3,10,1,8,15,6,13,20,27],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $248, %ymm23, %ymm1, %ymm12
-; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm13, %ymm0, %ymm8
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $184, %ymm3, %ymm25, %ymm8
-; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} ymm3 = ymm14[0],ymm10[1],ymm14[2,3],ymm10[4],ymm14[5,6,7,8],ymm10[9],ymm14[10,11],ymm10[12],ymm14[13,14,15]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm10, %ymm13
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm10 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm3[4,11,2,9,0,7,14,21,28],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $248, %ymm23, %ymm9, %ymm10
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = xmm6[6,13],zero,zero,xmm6[2,9],zero,zero,zero,xmm6[u,u,u,u,u,u,u]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa {{.*#+}} ymm5 = <u,u,u,u,128,128,128,128,128,128,128,128,128,0,7,14,21,28,19,26,17,24,31,u,u,u,u,u,u,u,u,u>
+; AVX512F-ONLY-FAST-NEXT:    vpshufb %ymm5, %ymm2, %ymm2
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $220, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm6, %ymm2
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm25, %ymm24, %ymm2
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm7 = xmm2[u,u,u,u,u,u,u,6,13],zero,zero,xmm2[2,9],zero,zero,zero
+; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm2, %xmm2
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,u,u,u,u,u,u],zero,zero,xmm2[4,11],zero,zero,xmm2[0,7,14]
+; AVX512F-ONLY-FAST-NEXT:    vpor %xmm7, %xmm2, %xmm2
+; AVX512F-ONLY-FAST-NEXT:    vinserti32x4 $1, %xmm2, %ymm0, %ymm28
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $184, %ymm3, %ymm0, %ymm28
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm14, %ymm2
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm19, %ymm10, %ymm2
+; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm2, %xmm3
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = zero,zero,xmm3[2,9],zero,zero,zero,xmm3[5,12,u,u,u,u,u,u,u]
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[4,11],zero,zero,xmm2[0,7,14],zero,zero,xmm2[u,u,u,u,u,u,u]
+; AVX512F-ONLY-FAST-NEXT:    vpor %xmm3, %xmm2, %xmm2
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm24, %ymm25, %ymm21
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm12, %ymm3
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm19, %ymm10, %ymm3
+; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm3, %xmm7
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm7 = zero,zero,xmm7[3,10],zero,zero,zero,xmm7[6,13,u,u,u,u,u,u,u]
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[5,12],zero,zero,xmm3[1,8,15],zero,zero,xmm3[u,u,u,u,u,u,u]
+; AVX512F-ONLY-FAST-NEXT:    vpor %xmm7, %xmm3, %xmm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm6, %ymm7
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm6, %ymm13
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm19, %ymm10, %ymm6
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm27, %ymm30, %ymm7
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm8 = xmm6[6,13],zero,zero,xmm6[2,9],zero,zero,zero,xmm6[u,u,u,u,u,u,u]
 ; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm6, %xmm6
 ; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm6 = zero,zero,xmm6[4,11],zero,zero,xmm6[0,7,14,u,u,u,u,u,u,u]
-; AVX512F-ONLY-FAST-NEXT:    vpor %xmm3, %xmm6, %xmm3
-; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} ymm6 = ymm11[0],ymm13[1],ymm11[2,3,4],ymm13[5],ymm11[6,7,8],ymm13[9],ymm11[10,11,12],ymm13[13],ymm11[14,15]
-; AVX512F-ONLY-FAST-NEXT:    vpbroadcastd {{.*#+}} xmm9 = [4,11,0,0,4,11,0,0,4,11,0,0,4,11,0,0]
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm6 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm6[5,12,3,10,1,8,15,22,29],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $248, %ymm23, %ymm3, %ymm6
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vpshufb %xmm9, %xmm13, %xmm3
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa %xmm0, %xmm14
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm11 = xmm0[2,9,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512F-ONLY-FAST-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm11[0],xmm3[1],xmm11[1],xmm3[2],xmm11[2],xmm3[3],xmm11[3]
-; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $1, %ymm3, %zmm8, %zmm8
+; AVX512F-ONLY-FAST-NEXT:    vpor %xmm6, %xmm8, %xmm8
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm12, %ymm10
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $226, %ymm11, %ymm12, %ymm26
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm14, %ymm6
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm30, %ymm27, %ymm6
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm30, %ymm27, %ymm12
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm9, %ymm0
+; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} ymm7 = ymm9[0],ymm7[1,2,3],ymm9[4],ymm7[5,6],ymm9[7,8],ymm7[9,10,11],ymm9[12],ymm7[13,14],ymm9[15]
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm9 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm7[3,10,1,8,15,6,13,20,27],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $248, %ymm17, %ymm2, %ymm9
+; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} ymm2 = ymm6[0],ymm0[1],ymm6[2,3],ymm0[4],ymm6[5,6,7,8],ymm0[9],ymm6[10,11],ymm0[12],ymm6[13,14,15]
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm6 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[4,11,2,9,0,7,14,21,28],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $248, %ymm17, %ymm3, %ymm6
+; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} ymm2 = ymm12[0],ymm0[1],ymm12[2,3,4],ymm0[5],ymm12[6,7,8],ymm0[9],ymm12[10,11,12],ymm0[13],ymm12[14,15]
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm7 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[5,12,3,10,1,8,15,22,29],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $248, %ymm17, %ymm8, %ymm7
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm29, %ymm31, %ymm10
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm2 = xmm10[u,u,2,9],zero,zero,zero,xmm10[5,12],zero,zero,xmm10[u,u,u,u,u]
+; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm10, %xmm3
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u],zero,zero,xmm3[0,7,14],zero,zero,xmm3[3,10,u,u,u,u,u]
+; AVX512F-ONLY-FAST-NEXT:    vpor %xmm2, %xmm3, %xmm2
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa {{.*#+}} ymm0 = <128,128,128,128,128,128,128,128,128,128,128,128,128,128,6,13,20,27,18,25,16,23,30,u,u,u,u,u,u,u,u,u>
+; AVX512F-ONLY-FAST-NEXT:    vpshufb %xmm0, %xmm1, %xmm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm8 = xmm12[u,u,u,u,u,u,u,u,u,u,u,1,8,15],zero,zero
+; AVX512F-ONLY-FAST-NEXT:    vpor %xmm3, %xmm8, %xmm3
+; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa {{.*#+}} ymm11 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0]
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $184, %ymm2, %ymm11, %ymm3
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $226, %ymm15, %ymm14, %ymm16
+; AVX512F-ONLY-FAST-NEXT:    vpbroadcastd {{.*#+}} xmm8 = [4,11,0,0,4,11,0,0,4,11,0,0,4,11,0,0]
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm29, %ymm31, %ymm13
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm31, %ymm29, %ymm14
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vpshufb %xmm8, %xmm0, %xmm2
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm10 = xmm15[2,9,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512F-ONLY-FAST-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm10[0],xmm2[1],xmm10[1],xmm2[2],xmm10[2],xmm2[3],xmm10[3]
+; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm2
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $184, %zmm12, %zmm3, %zmm8
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm31, %ymm30, %ymm5
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm11 = xmm5[u,u,3,10],zero,zero,zero,xmm5[6,13],zero,zero,xmm5[u,u,u,u,u]
-; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm5, %xmm5
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm5 = xmm5[u,u],zero,zero,xmm5[1,8,15],zero,zero,xmm5[4,11,u,u,u,u,u]
-; AVX512F-ONLY-FAST-NEXT:    vpor %xmm5, %xmm11, %xmm5
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $226, %ymm16, %ymm15, %ymm4
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $202, %ymm30, %ymm31, %ymm15
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm11 = xmm2[u,u,u,u,u,u,u,u,u,u,u,2,9],zero,zero,zero
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm12 = xmm7[u,u,u,u,u,u,u,u,u,u,u],zero,zero,xmm7[0,7,14]
-; AVX512F-ONLY-FAST-NEXT:    vpor %xmm11, %xmm12, %xmm11
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $184, %zmm9, %zmm3, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm9 = xmm13[u,u,3,10],zero,zero,zero,xmm13[6,13],zero,zero,xmm13[u,u,u,u,u]
+; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm13, %xmm10
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm10 = xmm10[u,u],zero,zero,xmm10[1,8,15],zero,zero,xmm10[4,11,u,u,u,u,u]
+; AVX512F-ONLY-FAST-NEXT:    vpor %xmm9, %xmm10, %xmm9
+; AVX512F-ONLY-FAST-NEXT:    vpshufb %xmm5, %xmm1, %xmm5
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm10 = xmm12[u,u,u,u,u,u,u,u,u,u,u,2,9],zero,zero,zero
+; AVX512F-ONLY-FAST-NEXT:    vpor %xmm5, %xmm10, %xmm5
+; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm9, %ymm0, %ymm9
 ; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $184, %ymm9, %ymm11, %ymm5
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa %ymm11, %ymm13
+; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm14, %xmm9
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm9 = xmm9[u,u],zero,zero,xmm9[2,9],zero,zero,zero,xmm9[5,12,u,u,u,u,u]
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm10 = xmm14[u,u,4,11],zero,zero,xmm14[0,7,14],zero,zero,xmm14[u,u,u,u,u]
+; AVX512F-ONLY-FAST-NEXT:    vpor %xmm9, %xmm10, %xmm9
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa {{.*#+}} ymm10 = [128,128,128,128,128,128,128,128,128,128,128,128,128,1,8,15,22,29,20,27,18,25,128,128,128,128,128,128,128,128,128,128]
+; AVX512F-ONLY-FAST-NEXT:    vpshufb %xmm10, %xmm1, %xmm11
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm12 = xmm12[u,u,u,u,u,u,u,u,u,u,u,3,10],zero,zero,zero
+; AVX512F-ONLY-FAST-NEXT:    vpor %xmm12, %xmm11, %xmm11
+; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm9, %ymm0, %ymm9
 ; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm11, %ymm0, %ymm11
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $184, %ymm5, %ymm25, %ymm11
-; AVX512F-ONLY-FAST-NEXT:    vextracti128 $1, %ymm15, %xmm5
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm5 = xmm5[u,u],zero,zero,xmm5[2,9],zero,zero,zero,xmm5[5,12,u,u,u,u,u]
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm12 = xmm15[u,u,4,11],zero,zero,xmm15[0,7,14],zero,zero,xmm15[u,u,u,u,u]
-; AVX512F-ONLY-FAST-NEXT:    vpor %xmm5, %xmm12, %xmm5
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,u,u,u,u,u,u,u,u,u,u,3,10],zero,zero,zero
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm7 = xmm7[u,u,u,u,u,u,u,u,u,u,u],zero,zero,xmm7[1,8,15]
-; AVX512F-ONLY-FAST-NEXT:    vpor %xmm2, %xmm7, %xmm2
-; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $184, %ymm5, %ymm25, %ymm2
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm5 = xmm14[3,10,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm7 = xmm13[5,12,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512F-ONLY-FAST-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm7[0],xmm5[0],xmm7[1],xmm5[1],xmm7[2],xmm5[2],xmm7[3],xmm5[3]
-; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $1, %ymm5, %zmm11, %zmm5
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $184, %zmm10, %zmm3, %zmm5
-; AVX512F-ONLY-FAST-NEXT:    vpshufb %xmm9, %xmm14, %xmm7
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm9 = xmm13[6,13,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512F-ONLY-FAST-NEXT:    vpunpcklwd {{.*#+}} xmm7 = xmm9[0],xmm7[0],xmm9[1],xmm7[1],xmm9[2],xmm7[2],xmm9[3],xmm7[3]
-; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $1, %ymm7, %zmm2, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $184, %zmm6, %zmm3, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $1, %ymm27, %zmm0, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $184, %ymm9, %ymm13, %ymm11
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm9 = xmm15[3,10,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm12 = xmm0[5,12,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512F-ONLY-FAST-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm12[0],xmm9[0],xmm12[1],xmm9[1],xmm12[2],xmm9[2],xmm12[3],xmm9[3]
+; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $1, %ymm9, %zmm5, %zmm5
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $184, %zmm6, %zmm3, %zmm5
+; AVX512F-ONLY-FAST-NEXT:    vpshufb %xmm8, %xmm15, %xmm6
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm8 = xmm0[6,13,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512F-ONLY-FAST-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm8[0],xmm6[0],xmm8[1],xmm6[1],xmm8[2],xmm6[2],xmm8[3],xmm6[3]
+; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $1, %ymm6, %zmm11, %zmm6
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $184, %zmm7, %zmm3, %zmm6
+; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $1, %ymm4, %zmm0, %zmm3
 ; AVX512F-ONLY-FAST-NEXT:    movw $-512, %ax # imm = 0xFE00
 ; AVX512F-ONLY-FAST-NEXT:    kmovw %eax, %k1
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa32 %zmm3, %zmm8 {%k1}
-; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $1, %ymm29, %zmm0, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa32 %zmm3, %zmm2 {%k1}
+; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $1, %ymm28, %zmm0, %zmm3
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqa32 %zmm3, %zmm5 {%k1}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %ymm28, %ymm0
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = xmm0[u,u,u,u,2,9],zero,zero,zero,xmm0[5,12],zero,zero,xmm0[u,u,u]
-; AVX512F-ONLY-FAST-NEXT:    vextracti32x4 $1, %ymm28, %xmm1
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[u,u,u,u],zero,zero,xmm1[0,7,14],zero,zero,xmm1[3,10,u,u,u]
-; AVX512F-ONLY-FAST-NEXT:    vpor %xmm3, %xmm1, %xmm1
-; AVX512F-ONLY-FAST-NEXT:    vpermq {{.*#+}} ymm3 = ymm4[2,3,0,1]
-; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} ymm3 = ymm4[0,1],ymm3[2],ymm4[3,4],ymm3[5],ymm4[6,7,8,9],ymm3[10],ymm4[11,12],ymm3[13],ymm4[14,15]
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} ymm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm3[1,8,15,22,29,20,27,18,25],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512F-ONLY-FAST-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %ymm16, %ymm0
+; AVX512F-ONLY-FAST-NEXT:    vpermq {{.*#+}} ymm3 = ymm16[2,3,0,1]
+; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} ymm3 = ymm0[0,1],ymm3[2],ymm0[3,4],ymm3[5],ymm0[6,7,8,9],ymm3[10],ymm0[11,12],ymm3[13],ymm0[14,15]
+; AVX512F-ONLY-FAST-NEXT:    vpshufb %ymm10, %ymm3, %ymm3
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %ymm26, %ymm0
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm4 = xmm0[u,u,u,u,2,9],zero,zero,zero,xmm0[5,12],zero,zero,xmm0[u,u,u]
 ; AVX512F-ONLY-FAST-NEXT:    vextracti32x4 $1, %ymm26, %xmm1
-; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[u,u,u,u,u,u],zero,zero,zero,xmm1[5,12],zero,zero,xmm1[1,8,15]
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[u,u,u,u],zero,zero,xmm1[0,7,14],zero,zero,xmm1[3,10,u,u,u]
+; AVX512F-ONLY-FAST-NEXT:    vpor %xmm4, %xmm1, %xmm1
+; AVX512F-ONLY-FAST-NEXT:    vpternlogq $236, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm1
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %ymm21, %ymm0
+; AVX512F-ONLY-FAST-NEXT:    vextracti32x4 $1, %ymm21, %xmm3
+; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u,u,u,u,u],zero,zero,zero,xmm3[5,12],zero,zero,xmm3[1,8,15]
 ; AVX512F-ONLY-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,u,0,7,14],zero,zero,xmm0[3,10],zero,zero,zero
-; AVX512F-ONLY-FAST-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512F-ONLY-FAST-NEXT:    vpor %xmm3, %xmm0, %xmm0
 ; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} ymm0 = ymm3[0,1,2],ymm0[3,4,5,6,7],ymm3[8,9,10],ymm0[11,12,13,14,15]
-; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3],ymm0[4,5,6,7]
+; AVX512F-ONLY-FAST-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6,7],ymm1[8,9,10],ymm0[11,12,13,14,15]
+; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
 ; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm0
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa32 %zmm0, %zmm2 {%k1}
-; AVX512F-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovaps %zmm0, (%rsi)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm17, (%rdx)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm21, (%rcx)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm24, (%r8)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm8, (%r9)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa32 %zmm0, %zmm6 {%k1}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm18, (%rsi)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm20, (%rdx)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm22, (%rcx)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm23, (%r8)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, (%r9)
 ; AVX512F-ONLY-FAST-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm5, (%rax)
 ; AVX512F-ONLY-FAST-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, (%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm6, (%rax)
 ; AVX512F-ONLY-FAST-NEXT:    vzeroupper
 ; AVX512F-ONLY-FAST-NEXT:    retq
 ;
 ; AVX512DQ-SLOW-LABEL: load_i8_stride7_vf64:
 ; AVX512DQ-SLOW:       # %bb.0:
-; AVX512DQ-SLOW-NEXT:    subq $104, %rsp
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} ymm28 = [65535,0,65535,65535,65535,0,65535,65535,0,65535,65535,65535,0,65535,65535,0]
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 (%rdi), %ymm19
-; AVX512DQ-SLOW-NEXT:    vmovdqa 32(%rdi), %ymm14
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 64(%rdi), %ymm27
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %ymm28, %ymm0
-; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm14, %ymm19, %ymm0
-; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm1
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm1 = zero,zero,zero,xmm1[5,12],zero,zero,xmm1[1,8,15,u,u,u,u,u,u]
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,7,14],zero,zero,xmm0[3,10],zero,zero,zero,xmm0[u,u,u,u,u,u]
-; AVX512DQ-SLOW-NEXT:    vpor %xmm1, %xmm0, %xmm0
-; AVX512DQ-SLOW-NEXT:    vmovdqa {{.*#+}} ymm7 = [65535,65535,65535,0,65535,65535,0,65535,65535,65535,0,65535,65535,0,65535,65535]
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 96(%rdi), %ymm26
-; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm7, %ymm1
-; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm27, %ymm26, %ymm1
-; AVX512DQ-SLOW-NEXT:    vmovdqa 80(%rdi), %xmm2
-; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0,1],ymm2[2],ymm1[3,4],ymm2[5],ymm1[6,7,8,9],ymm2[10],ymm1[11,12],ymm2[13],ymm1[14,15]
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %ymm2, %ymm24
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} ymm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm1[6,13,4,11,2,9,16,23,30,u],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-SLOW-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm2
-; AVX512DQ-SLOW-NEXT:    vmovdqa {{.*#+}} ymm15 = [65535,65535,0,65535,65535,65535,0,65535,65535,0,65535,65535,65535,0,65535,65535]
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 128(%rdi), %ymm31
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 160(%rdi), %ymm30
-; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm15, %ymm0
-; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm31, %ymm30, %ymm0
-; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm4
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[u,u,u],zero,zero,xmm4[3,10],zero,zero,zero,xmm4[6,13,u,u,u,u]
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,5,12],zero,zero,xmm0[1,8,15],zero,zero,xmm0[u,u,u,u]
-; AVX512DQ-SLOW-NEXT:    vpor %xmm4, %xmm0, %xmm0
-; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512DQ-SLOW-NEXT:    vmovdqa 192(%rdi), %xmm4
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm5 = xmm4[u,u,u,u,u,u,4,11,u,u,u,u,u,u,u,u]
-; AVX512DQ-SLOW-NEXT:    vmovdqa %xmm4, %xmm6
-; AVX512DQ-SLOW-NEXT:    vmovdqa 208(%rdi), %xmm4
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm9 = xmm4[u,u,u,u,u,u,2,9,u,u,u,u,u,u,u,u]
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %xmm4, %xmm23
-; AVX512DQ-SLOW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm9[0],xmm5[1],xmm9[1],xmm5[2],xmm9[2],xmm5[3],xmm9[3]
-; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,6],ymm5[7]
-; AVX512DQ-SLOW-NEXT:    vmovdqa 240(%rdi), %xmm12
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm5 = zero,zero,zero,xmm12[5,12,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-SLOW-NEXT:    vmovdqa 224(%rdi), %xmm8
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm9 = xmm8[0,7,14],zero,zero,xmm8[u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-SLOW-NEXT:    vpor %xmm5, %xmm9, %xmm5
-; AVX512DQ-SLOW-NEXT:    vinserti32x4 $2, %xmm5, %zmm0, %zmm0
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm11 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512DQ-SLOW-NEXT:    vpternlogq $184, %zmm2, %zmm11, %zmm0
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 288(%rdi), %ymm20
+; AVX512DQ-SLOW-NEXT:    subq $24, %rsp
+; AVX512DQ-SLOW-NEXT:    vmovdqa {{.*#+}} ymm0 = [65535,0,65535,65535,65535,0,65535,65535,0,65535,65535,65535,0,65535,65535,0]
+; AVX512DQ-SLOW-NEXT:    vmovdqa (%rdi), %ymm12
+; AVX512DQ-SLOW-NEXT:    vmovdqa 32(%rdi), %ymm13
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 64(%rdi), %ymm31
+; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm0, %ymm1
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %ymm0, %ymm23
+; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm13, %ymm12, %ymm1
+; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm1, %xmm2
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm2 = zero,zero,zero,xmm2[5,12],zero,zero,xmm2[1,8,15,u,u,u,u,u,u]
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0,7,14],zero,zero,xmm1[3,10],zero,zero,zero,xmm1[u,u,u,u,u,u]
+; AVX512DQ-SLOW-NEXT:    vpor %xmm2, %xmm1, %xmm1
+; AVX512DQ-SLOW-NEXT:    vmovdqa {{.*#+}} ymm9 = [65535,65535,65535,0,65535,65535,0,65535,65535,65535,0,65535,65535,0,65535,65535]
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 96(%rdi), %ymm28
+; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm9, %ymm2
+; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm31, %ymm28, %ymm2
+; AVX512DQ-SLOW-NEXT:    vmovdqa 80(%rdi), %xmm11
+; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm11[2],ymm2[3,4],ymm11[5],ymm2[6,7,8,9],ymm11[10],ymm2[11,12],ymm11[13],ymm2[14,15]
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} ymm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[6,13,4,11,2,9,16,23,30,u],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512DQ-SLOW-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm2
+; AVX512DQ-SLOW-NEXT:    vmovdqa {{.*#+}} ymm14 = [65535,65535,0,65535,65535,65535,0,65535,65535,0,65535,65535,65535,0,65535,65535]
+; AVX512DQ-SLOW-NEXT:    vmovdqa 128(%rdi), %ymm0
+; AVX512DQ-SLOW-NEXT:    vmovdqa 160(%rdi), %ymm3
+; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm14, %ymm1
+; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm0, %ymm3, %ymm1
+; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm3, %ymm4
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %ymm0, %ymm25
+; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm1, %xmm3
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u,u],zero,zero,xmm3[3,10],zero,zero,zero,xmm3[6,13,u,u,u,u]
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[u,u,u,5,12],zero,zero,xmm1[1,8,15],zero,zero,xmm1[u,u,u,u]
+; AVX512DQ-SLOW-NEXT:    vpor %xmm3, %xmm1, %xmm1
+; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
+; AVX512DQ-SLOW-NEXT:    vmovdqa 192(%rdi), %xmm0
+; AVX512DQ-SLOW-NEXT:    vpbroadcastd {{.*#+}} xmm5 = [0,0,4,11,0,0,4,11,0,0,4,11,0,0,4,11]
+; AVX512DQ-SLOW-NEXT:    vpshufb %xmm5, %xmm0, %xmm3
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %xmm5, %xmm30
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %xmm0, %xmm20
+; AVX512DQ-SLOW-NEXT:    vmovdqa 208(%rdi), %xmm10
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm6 = xmm10[u,u,u,u,u,u,2,9,u,u,u,u,u,u,u,u]
+; AVX512DQ-SLOW-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm6[0],xmm3[1],xmm6[1],xmm3[2],xmm6[2],xmm3[3],xmm6[3]
+; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5,6],ymm3[7]
+; AVX512DQ-SLOW-NEXT:    vmovdqa 240(%rdi), %xmm0
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm6 = zero,zero,zero,xmm0[5,12,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-SLOW-NEXT:    vmovdqa %xmm0, %xmm3
+; AVX512DQ-SLOW-NEXT:    vmovdqa 224(%rdi), %xmm0
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm7 = xmm0[0,7,14],zero,zero,xmm0[u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-SLOW-NEXT:    vmovdqa %xmm0, %xmm5
+; AVX512DQ-SLOW-NEXT:    vpor %xmm6, %xmm7, %xmm6
+; AVX512DQ-SLOW-NEXT:    vinserti32x4 $2, %xmm6, %zmm1, %zmm22
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm19 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; AVX512DQ-SLOW-NEXT:    vpternlogq $184, %zmm2, %zmm19, %zmm22
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 288(%rdi), %ymm18
 ; AVX512DQ-SLOW-NEXT:    vmovdqa64 256(%rdi), %ymm17
-; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm7, %ymm5
-; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm20, %ymm17, %ymm5
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm9 = xmm5[u,u,u,u,u,3,10],zero,zero,zero,xmm5[6,13],zero,zero,xmm5[u,u]
-; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm5, %xmm5
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm5 = xmm5[u,u,u,u,u],zero,zero,xmm5[1,8,15],zero,zero,xmm5[4,11,u,u]
-; AVX512DQ-SLOW-NEXT:    vpor %xmm5, %xmm9, %xmm9
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 352(%rdi), %ymm18
+; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm9, %ymm2
+; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm18, %ymm17, %ymm2
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm6 = xmm2[u,u,u,u,u,3,10],zero,zero,zero,xmm2[6,13],zero,zero,xmm2[u,u]
+; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm2, %xmm2
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,u,u,u,u],zero,zero,xmm2[1,8,15],zero,zero,xmm2[4,11,u,u]
+; AVX512DQ-SLOW-NEXT:    vpor %xmm6, %xmm2, %xmm2
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 352(%rdi), %ymm21
 ; AVX512DQ-SLOW-NEXT:    vmovdqa64 320(%rdi), %ymm16
-; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm15, %ymm13
-; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm18, %ymm16, %ymm13
-; AVX512DQ-SLOW-NEXT:    vpermq {{.*#+}} ymm10 = ymm13[2,3,0,1]
-; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} ymm10 = ymm13[0,1],ymm10[2],ymm13[3,4,5],ymm10[6],ymm13[7,8,9],ymm10[10],ymm13[11,12,13],ymm10[14],ymm13[15]
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} ymm10 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm10[2,9,16,23,30,21,28,19,26,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-SLOW-NEXT:    vmovdqa {{.*#+}} ymm5 = [65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,65535,65535,65535,65535]
-; AVX512DQ-SLOW-NEXT:    vpternlogq $248, %ymm5, %ymm9, %ymm10
-; AVX512DQ-SLOW-NEXT:    vmovdqa {{.*#+}} ymm9 = [65535,65535,0,65535,65535,0,65535,65535,65535,0,65535,65535,0,65535,65535,65535]
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 416(%rdi), %ymm21
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 384(%rdi), %ymm22
-; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm9, %ymm13
-; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm21, %ymm22, %ymm13
-; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm13, %xmm4
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[u,u,u,u,u,u,u],zero,zero,zero,xmm4[6,13],zero,zero,xmm4[2,9]
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm13 = xmm13[u,u,u,u,u,u,u,1,8,15],zero,zero,xmm13[4,11],zero,zero
-; AVX512DQ-SLOW-NEXT:    vpor %xmm4, %xmm13, %xmm4
-; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} ymm25 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0]
-; AVX512DQ-SLOW-NEXT:    vpternlogq $184, %ymm10, %ymm25, %ymm4
-; AVX512DQ-SLOW-NEXT:    vinserti64x4 $1, %ymm4, %zmm0, %zmm2
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm29 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512DQ-SLOW-NEXT:    vpternlogq $184, %zmm0, %zmm29, %zmm2
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm9, %ymm0
-; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm14, %ymm19, %ymm0
-; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm4
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm4 = zero,zero,zero,xmm4[6,13],zero,zero,xmm4[2,9,u,u,u,u,u,u,u]
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[1,8,15],zero,zero,xmm0[4,11],zero,zero,xmm0[u,u,u,u,u,u,u]
-; AVX512DQ-SLOW-NEXT:    vpor %xmm4, %xmm0, %xmm0
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %ymm28, %ymm4
-; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm26, %ymm27, %ymm4
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %ymm24, %ymm1
-; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} ymm4 = ymm4[0,1],ymm1[2],ymm4[3,4,5],ymm1[6],ymm4[7,8,9],ymm1[10],ymm4[11,12,13],ymm1[14],ymm4[15]
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} ymm4 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm4[0,7,14,5,12,3,10,17,24,31,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-SLOW-NEXT:    vpternlogq $244, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm4
-; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm7, %ymm0
-; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm31, %ymm30, %ymm0
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm10 = xmm0[u,u,u,6,13],zero,zero,xmm0[2,9],zero,zero,zero,xmm0[u,u,u,u]
-; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,u],zero,zero,xmm0[4,11],zero,zero,xmm0[0,7,14,u,u,u,u]
-; AVX512DQ-SLOW-NEXT:    vpor %xmm0, %xmm10, %xmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm14, %ymm7
+; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm21, %ymm16, %ymm7
+; AVX512DQ-SLOW-NEXT:    vpermq {{.*#+}} ymm8 = ymm7[2,3,0,1]
+; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} ymm7 = ymm7[0,1],ymm8[2],ymm7[3,4,5],ymm8[6],ymm7[7,8,9],ymm8[10],ymm7[11,12,13],ymm8[14],ymm7[15]
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} ymm8 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm7[2,9,16,23,30,21,28,19,26,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} ymm24 = [65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,65535,65535,65535,65535]
+; AVX512DQ-SLOW-NEXT:    vpternlogq $248, %ymm24, %ymm2, %ymm8
+; AVX512DQ-SLOW-NEXT:    vmovdqa {{.*#+}} ymm7 = [65535,65535,0,65535,65535,0,65535,65535,65535,0,65535,65535,0,65535,65535,65535]
+; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm7, %ymm2
+; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm13, %ymm12, %ymm2
+; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm2, %xmm15
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm15 = zero,zero,zero,xmm15[6,13],zero,zero,xmm15[2,9,u,u,u,u,u,u,u]
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[1,8,15],zero,zero,xmm2[4,11],zero,zero,xmm2[u,u,u,u,u,u,u]
+; AVX512DQ-SLOW-NEXT:    vpor %xmm2, %xmm15, %xmm2
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %ymm23, %ymm15
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %ymm23, %ymm29
+; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm28, %ymm31, %ymm15
+; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} ymm15 = ymm15[0,1],ymm11[2],ymm15[3,4,5],ymm11[6],ymm15[7,8,9],ymm11[10],ymm15[11,12,13],ymm11[14],ymm15[15]
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} ymm15 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm15[0,7,14,5,12,3,10,17,24,31,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-SLOW-NEXT:    vpternlogq $244, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm15
+; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm9, %ymm2
+; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm4, %ymm6
+; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm25, %ymm4, %ymm2
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm2[u,u,u,6,13],zero,zero,xmm2[2,9],zero,zero,zero,xmm2[u,u,u,u]
+; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm2, %xmm2
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,u,u],zero,zero,xmm2[4,11],zero,zero,xmm2[0,7,14,u,u,u,u]
+; AVX512DQ-SLOW-NEXT:    vpor %xmm0, %xmm2, %xmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqa %xmm3, %xmm4
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm2 = zero,zero,zero,xmm3[6,13,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm3 = xmm5[1,8,15],zero,zero,xmm5[u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %xmm5, %xmm26
+; AVX512DQ-SLOW-NEXT:    vpor %xmm2, %xmm3, %xmm2
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm3 = xmm10[u,u,u,u,u,u,3,10,u,u,u,u,u,u,u,u]
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %xmm20, %xmm1
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm5 = xmm1[u,u,u,u,u,u,5,12,u,u,u,u,u,u,u,u]
+; AVX512DQ-SLOW-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
 ; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %xmm23, %xmm2
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm10 = xmm2[u,u,u,u,u,u,3,10,u,u,u,u,u,u,u,u]
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm13 = xmm6[u,u,u,u,u,u,5,12,u,u,u,u,u,u,u,u]
-; AVX512DQ-SLOW-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm13[0],xmm10[0],xmm13[1],xmm10[1],xmm13[2],xmm10[2],xmm13[3],xmm10[3]
-; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,6],ymm10[7]
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm10 = zero,zero,zero,xmm12[6,13,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm13 = xmm8[1,8,15],zero,zero,xmm8[u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-SLOW-NEXT:    vpor %xmm10, %xmm13, %xmm10
-; AVX512DQ-SLOW-NEXT:    vinserti32x4 $2, %xmm10, %zmm0, %zmm0
-; AVX512DQ-SLOW-NEXT:    vpternlogq $184, %zmm4, %zmm11, %zmm0
-; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm9, %ymm4
-; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm17, %ymm20, %ymm4
-; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm4, %xmm10
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm10 = xmm10[u,u,u,u,u],zero,zero,xmm10[2,9],zero,zero,zero,xmm10[5,12,u,u]
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[u,u,u,u,u,4,11],zero,zero,xmm4[0,7,14],zero,zero,xmm4[u,u]
-; AVX512DQ-SLOW-NEXT:    vpor %xmm4, %xmm10, %xmm4
-; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm7, %ymm10
-; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm18, %ymm16, %ymm10
-; AVX512DQ-SLOW-NEXT:    vpermq {{.*#+}} ymm13 = ymm10[2,3,0,1]
-; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} ymm10 = ymm10[0,1,2],ymm13[3],ymm10[4,5],ymm13[6],ymm10[7,8,9,10],ymm13[11],ymm10[12,13],ymm13[14],ymm10[15]
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} ymm10 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm10[3,10,17,24,31,22,29,20,27,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-SLOW-NEXT:    vpternlogq $248, %ymm5, %ymm4, %ymm10
-; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm15, %ymm4
-; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm21, %ymm22, %ymm4
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm13 = xmm4[u,u,u,u,u,u,u,2,9],zero,zero,zero,xmm4[5,12],zero,zero
-; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm4, %xmm4
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[u,u,u,u,u,u,u],zero,zero,xmm4[0,7,14],zero,zero,xmm4[3,10]
-; AVX512DQ-SLOW-NEXT:    vpor %xmm4, %xmm13, %xmm4
-; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512DQ-SLOW-NEXT:    vpternlogq $184, %ymm10, %ymm25, %ymm4
-; AVX512DQ-SLOW-NEXT:    vinserti64x4 $1, %ymm4, %zmm0, %zmm3
-; AVX512DQ-SLOW-NEXT:    vpternlogq $184, %zmm0, %zmm29, %zmm3
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm15, %ymm0
-; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm14, %ymm19, %ymm0
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm4 = xmm0[2,9],zero,zero,zero,xmm0[5,12],zero,zero,xmm0[u,u,u,u,u,u,u]
+; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,6],ymm3[7]
+; AVX512DQ-SLOW-NEXT:    vinserti32x4 $2, %xmm2, %zmm0, %zmm23
+; AVX512DQ-SLOW-NEXT:    vpternlogq $184, %zmm15, %zmm19, %zmm23
+; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm14, %ymm0
+; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm13, %ymm12, %ymm0
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm2 = xmm0[2,9],zero,zero,zero,xmm0[5,12],zero,zero,xmm0[u,u,u,u,u,u,u]
 ; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = zero,zero,xmm0[0,7,14],zero,zero,xmm0[3,10,u,u,u,u,u,u,u]
-; AVX512DQ-SLOW-NEXT:    vpor %xmm4, %xmm0, %xmm0
-; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm9, %ymm4
-; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm26, %ymm27, %ymm4
-; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} ymm4 = ymm4[0,1,2],ymm1[3],ymm4[4,5],ymm1[6],ymm4[7,8,9,10],ymm1[11],ymm4[12,13],ymm1[14],ymm4[15]
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %ymm24, %ymm11
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} ymm4 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm4[1,8,15,6,13,4,11,18,25],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-SLOW-NEXT:    vmovdqa {{.*#+}} ymm1 = [255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512DQ-SLOW-NEXT:    vpternlogq $248, %ymm1, %ymm0, %ymm4
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %ymm1, %ymm23
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %ymm28, %ymm0
-; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm30, %ymm31, %ymm0
-; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm10
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm10 = xmm10[u,u],zero,zero,zero,xmm10[5,12],zero,zero,xmm10[1,8,15,u,u,u,u]
+; AVX512DQ-SLOW-NEXT:    vpor %xmm2, %xmm0, %xmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm7, %ymm2
+; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm28, %ymm31, %ymm2
+; AVX512DQ-SLOW-NEXT:    vmovdqu %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1,2],ymm11[3],ymm2[4,5],ymm11[6],ymm2[7,8,9,10],ymm11[11],ymm2[12,13],ymm11[14],ymm2[15]
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} ymm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[1,8,15,6,13,4,11,18,25],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512DQ-SLOW-NEXT:    vmovdqa {{.*#+}} ymm3 = [255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; AVX512DQ-SLOW-NEXT:    vpternlogq $248, %ymm3, %ymm0, %ymm2
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %ymm3, %ymm27
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %ymm29, %ymm0
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %ymm25, %ymm15
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %ymm25, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm6, %ymm25, %ymm0
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %ymm6, %ymm19
+; AVX512DQ-SLOW-NEXT:    vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u],zero,zero,zero,xmm3[5,12],zero,zero,xmm3[1,8,15,u,u,u,u]
 ; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,0,7,14],zero,zero,xmm0[3,10],zero,zero,zero,xmm0[u,u,u,u]
-; AVX512DQ-SLOW-NEXT:    vpor %xmm0, %xmm10, %xmm0
+; AVX512DQ-SLOW-NEXT:    vpor %xmm3, %xmm0, %xmm0
 ; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm10 = xmm2[u,u,u,u,u,u,4,11,u,u,u,u,u,u,u,u]
-; AVX512DQ-SLOW-NEXT:    vmovdqa %xmm2, %xmm1
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm13 = xmm6[u,u,u,u,u,u,6,13,u,u,u,u,u,u,u,u]
-; AVX512DQ-SLOW-NEXT:    vmovdqa %xmm6, %xmm3
-; AVX512DQ-SLOW-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm13[0],xmm10[0],xmm13[1],xmm10[1],xmm13[2],xmm10[2],xmm13[3],xmm10[3]
-; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,6],ymm10[7]
-; AVX512DQ-SLOW-NEXT:    vmovdqa %xmm8, %xmm2
-; AVX512DQ-SLOW-NEXT:    vmovdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm10 = xmm8[2,9],zero,zero,zero,xmm8[u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-SLOW-NEXT:    vmovdqa %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm13 = zero,zero,xmm12[0,7,14,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-SLOW-NEXT:    vpor %xmm10, %xmm13, %xmm10
-; AVX512DQ-SLOW-NEXT:    vinserti32x4 $2, %xmm10, %zmm0, %zmm0
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm10 = [0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
-; AVX512DQ-SLOW-NEXT:    vpternlogq $226, %zmm4, %zmm10, %zmm0
-; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm15, %ymm4
-; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm17, %ymm20, %ymm4
-; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm4, %xmm13
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm13 = xmm13[u,u,u,u,u],zero,zero,xmm13[3,10],zero,zero,zero,xmm13[6,13,u,u]
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[u,u,u,u,u,5,12],zero,zero,xmm4[1,8,15],zero,zero,xmm4[u,u]
-; AVX512DQ-SLOW-NEXT:    vpor %xmm4, %xmm13, %xmm4
-; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm9, %ymm13
-; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm16, %ymm18, %ymm13
-; AVX512DQ-SLOW-NEXT:    vpermq {{.*#+}} ymm6 = ymm13[2,3,0,1]
-; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} ymm6 = ymm6[0],ymm13[1,2],ymm6[3],ymm13[4,5,6],ymm6[7,8],ymm13[9,10],ymm6[11],ymm13[12,13,14],ymm6[15]
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} ymm6 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm6[4,11,18,25,16,23,30,21,28,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-SLOW-NEXT:    vpternlogq $248, %ymm5, %ymm4, %ymm6
-; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm7, %ymm4
-; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm21, %ymm22, %ymm4
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm13 = xmm4[u,u,u,u,u,u,u,3,10],zero,zero,zero,xmm4[6,13],zero,zero
-; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm4, %xmm4
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[u,u,u,u,u,u,u],zero,zero,xmm4[1,8,15],zero,zero,xmm4[4,11]
-; AVX512DQ-SLOW-NEXT:    vpor %xmm4, %xmm13, %xmm4
-; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512DQ-SLOW-NEXT:    vpternlogq $184, %ymm6, %ymm25, %ymm4
-; AVX512DQ-SLOW-NEXT:    vinserti64x4 $1, %ymm4, %zmm0, %zmm4
-; AVX512DQ-SLOW-NEXT:    vpternlogq $184, %zmm0, %zmm29, %zmm4
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm7, %ymm0
-; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm14, %ymm19, %ymm0
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm4 = xmm0[3,10],zero,zero,zero,xmm0[6,13],zero,zero,xmm0[u,u,u,u,u,u,u]
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %xmm30, %xmm3
+; AVX512DQ-SLOW-NEXT:    vpshufb %xmm3, %xmm10, %xmm3
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm5 = xmm1[u,u,u,u,u,u,6,13,u,u,u,u,u,u,u,u]
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %xmm20, %xmm6
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %xmm20, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-SLOW-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
+; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5,6],ymm3[7]
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %xmm26, %xmm1
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %xmm26, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm3 = xmm1[2,9],zero,zero,zero,xmm1[u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm5 = zero,zero,xmm4[0,7,14,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-SLOW-NEXT:    vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-SLOW-NEXT:    vpor %xmm3, %xmm5, %xmm3
+; AVX512DQ-SLOW-NEXT:    vinserti32x4 $2, %xmm3, %zmm0, %zmm25
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm20 = [0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
+; AVX512DQ-SLOW-NEXT:    vpternlogq $226, %zmm2, %zmm20, %zmm25
+; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm9, %ymm0
+; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm13, %ymm12, %ymm0
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm2 = xmm0[3,10],zero,zero,zero,xmm0[6,13],zero,zero,xmm0[u,u,u,u,u,u,u]
 ; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm0
 ; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = zero,zero,xmm0[1,8,15],zero,zero,xmm0[4,11,u,u,u,u,u,u,u]
-; AVX512DQ-SLOW-NEXT:    vpor %xmm4, %xmm0, %xmm0
-; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm15, %ymm4
-; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm26, %ymm27, %ymm4
-; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} ymm4 = ymm11[0],ymm4[1,2],ymm11[3],ymm4[4,5,6],ymm11[7,8],ymm4[9,10],ymm11[11],ymm4[12,13,14],ymm11[15]
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} ymm4 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm4[2,9,0,7,14,5,12,19,26],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-SLOW-NEXT:    vpternlogq $248, %ymm23, %ymm0, %ymm4
-; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm9, %ymm0
-; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm30, %ymm31, %ymm0
-; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm6
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm6 = xmm6[u,u],zero,zero,zero,xmm6[6,13],zero,zero,xmm6[2,9,u,u,u,u,u]
+; AVX512DQ-SLOW-NEXT:    vpor %xmm2, %xmm0, %xmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm14, %ymm2
+; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm28, %ymm31, %ymm2
+; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} ymm2 = ymm11[0],ymm2[1,2],ymm11[3],ymm2[4,5,6],ymm11[7,8],ymm2[9,10],ymm11[11],ymm2[12,13,14],ymm11[15]
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} ymm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[2,9,0,7,14,5,12,19,26],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512DQ-SLOW-NEXT:    vpternlogq $248, %ymm27, %ymm0, %ymm3
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %ymm27, %ymm11
+; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm7, %ymm0
+; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm19, %ymm15, %ymm0
+; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm2
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,u],zero,zero,zero,xmm2[6,13],zero,zero,xmm2[2,9,u,u,u,u,u]
 ; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,1,8,15],zero,zero,xmm0[4,11],zero,zero,xmm0[u,u,u,u,u]
-; AVX512DQ-SLOW-NEXT:    vpor %xmm6, %xmm0, %xmm0
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm6 = xmm1[u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,xmm1[5,12]
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm13 = xmm3[u,u,u,u,u,u,u,u,u,u,u,0,7,14],zero,zero
-; AVX512DQ-SLOW-NEXT:    vpor %xmm6, %xmm13, %xmm6
+; AVX512DQ-SLOW-NEXT:    vpor %xmm2, %xmm0, %xmm0
 ; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
-; AVX512DQ-SLOW-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm6
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm2[3,10],zero,zero,zero,xmm2[u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm13 = zero,zero,xmm12[1,8,15,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-SLOW-NEXT:    vpor %xmm0, %xmm13, %xmm0
-; AVX512DQ-SLOW-NEXT:    vinserti32x4 $2, %xmm0, %zmm6, %zmm0
-; AVX512DQ-SLOW-NEXT:    vpternlogq $226, %zmm4, %zmm10, %zmm0
-; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm7, %ymm4
-; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm17, %ymm20, %ymm4
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm6 = xmm4[u,u,u,u,u,6,13],zero,zero,xmm4[2,9],zero,zero,zero,xmm4[u,u]
-; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm4, %xmm4
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[u,u,u,u,u],zero,zero,xmm4[4,11],zero,zero,xmm4[0,7,14,u,u]
-; AVX512DQ-SLOW-NEXT:    vpor %xmm6, %xmm4, %xmm4
-; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm15, %ymm6
-; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm16, %ymm18, %ymm6
-; AVX512DQ-SLOW-NEXT:    vpermq {{.*#+}} ymm10 = ymm6[2,3,0,1]
-; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} ymm6 = ymm10[0],ymm6[1,2,3],ymm10[4],ymm6[5,6],ymm10[7,8],ymm6[9,10,11],ymm10[12],ymm6[13,14],ymm10[15]
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} ymm6 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm6[5,12,19,26,17,24,31,22,29,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-SLOW-NEXT:    vpternlogq $248, %ymm5, %ymm4, %ymm6
-; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm9, %ymm4
-; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm22, %ymm21, %ymm4
-; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm4, %xmm10
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm10 = xmm10[u,u,u,u,u,u,u],zero,zero,xmm10[2,9],zero,zero,zero,xmm10[5,12]
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[u,u,u,u,u,u,u,4,11],zero,zero,xmm4[0,7,14],zero,zero
-; AVX512DQ-SLOW-NEXT:    vpor %xmm4, %xmm10, %xmm4
-; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512DQ-SLOW-NEXT:    vpternlogq $184, %ymm6, %ymm25, %ymm4
-; AVX512DQ-SLOW-NEXT:    vinserti64x4 $1, %ymm4, %zmm0, %zmm24
-; AVX512DQ-SLOW-NEXT:    vpternlogq $184, %zmm0, %zmm29, %zmm24
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %ymm28, %ymm0
-; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm20, %ymm17, %ymm0
-; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm4
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[u,u,u,u],zero,zero,zero,xmm4[5,12],zero,zero,xmm4[1,8,15,u,u]
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,0,7,14],zero,zero,xmm0[3,10],zero,zero,zero,xmm0[u,u]
-; AVX512DQ-SLOW-NEXT:    vpor %xmm4, %xmm0, %xmm0
-; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm7, %ymm4
-; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm16, %ymm18, %ymm4
-; AVX512DQ-SLOW-NEXT:    vpermq {{.*#+}} ymm6 = ymm4[2,3,0,1]
-; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} ymm4 = ymm4[0],ymm6[1],ymm4[2,3],ymm6[4],ymm4[5,6,7,8],ymm6[9],ymm4[10,11],ymm6[12],ymm4[13,14,15]
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} ymm4 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm4[6,13,20,27,18,25,16,23,30,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-SLOW-NEXT:    vpternlogq $248, %ymm5, %ymm0, %ymm4
-; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm15, %ymm0
-; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm22, %ymm21, %ymm0
-; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm6
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm6 = xmm6[u,u,u,u,u,u,u],zero,zero,xmm6[3,10],zero,zero,zero,xmm6[6,13]
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,u,u,5,12],zero,zero,xmm0[1,8,15],zero,zero
-; AVX512DQ-SLOW-NEXT:    vpor %xmm6, %xmm0, %xmm0
-; AVX512DQ-SLOW-NEXT:    vinserti32x4 $1, %xmm0, %ymm0, %ymm23
-; AVX512DQ-SLOW-NEXT:    vpternlogq $184, %ymm4, %ymm25, %ymm23
-; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm9, %ymm0
-; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm19, %ymm14, %ymm0
-; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm4
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm4 = zero,zero,xmm4[2,9],zero,zero,zero,xmm4[5,12,u,u,u,u,u,u,u]
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[4,11],zero,zero,xmm0[0,7,14],zero,zero,xmm0[u,u,u,u,u,u,u]
-; AVX512DQ-SLOW-NEXT:    vpor %xmm4, %xmm0, %xmm2
-; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm9, %ymm0
-; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm20, %ymm17, %ymm0
-; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm4
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[u,u,u,u],zero,zero,zero,xmm4[6,13],zero,zero,xmm4[2,9,u,u,u]
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,1,8,15],zero,zero,xmm0[4,11],zero,zero,xmm0[u,u,u]
-; AVX512DQ-SLOW-NEXT:    vpor %xmm4, %xmm0, %xmm0
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %ymm28, %ymm4
-; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm18, %ymm16, %ymm4
-; AVX512DQ-SLOW-NEXT:    vpermq {{.*#+}} ymm6 = ymm4[2,3,0,1]
-; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} ymm4 = ymm4[0],ymm6[1],ymm4[2,3,4],ymm6[5],ymm4[6,7,8],ymm6[9],ymm4[10,11,12],ymm6[13],ymm4[14,15]
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} ymm4 = ymm4[u,u,u,u],zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm4[0,7,14,21,28,19,26,17,24,31,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-SLOW-NEXT:    vpternlogq $244, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm4
+; AVX512DQ-SLOW-NEXT:    vmovdqa {{.*#+}} ymm2 = <128,128,128,128,128,128,128,128,128,128,128,128,128,128,5,12,19,26,17,24,31,22,29,u,u,u,u,u,u,u,u,u>
+; AVX512DQ-SLOW-NEXT:    vpshufb %xmm2, %xmm10, %xmm5
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm15 = xmm6[u,u,u,u,u,u,u,u,u,u,u,0,7,14],zero,zero
+; AVX512DQ-SLOW-NEXT:    vpor %xmm5, %xmm15, %xmm5
+; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512DQ-SLOW-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm5
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm1[3,10],zero,zero,zero,xmm1[u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm15 = zero,zero,xmm4[1,8,15,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-SLOW-NEXT:    vpor %xmm0, %xmm15, %xmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 416(%rdi), %ymm26
+; AVX512DQ-SLOW-NEXT:    vinserti32x4 $2, %xmm0, %zmm5, %zmm30
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 384(%rdi), %ymm27
+; AVX512DQ-SLOW-NEXT:    vpternlogq $226, %zmm3, %zmm20, %zmm30
 ; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm7, %ymm0
-; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm22, %ymm21, %ymm0
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm6 = xmm0[u,u,u,u,u,u,u,6,13],zero,zero,xmm0[2,9],zero,zero,zero
+; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm26, %ymm27, %ymm0
+; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u,u,u,u,u,u],zero,zero,zero,xmm3[6,13],zero,zero,xmm3[2,9]
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,u,u,1,8,15],zero,zero,xmm0[4,11],zero,zero
+; AVX512DQ-SLOW-NEXT:    vpor %xmm3, %xmm0, %xmm0
+; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} ymm29 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0]
+; AVX512DQ-SLOW-NEXT:    vpternlogq $184, %ymm8, %ymm29, %ymm0
+; AVX512DQ-SLOW-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm20
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm8 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVX512DQ-SLOW-NEXT:    vpternlogq $184, %zmm22, %zmm8, %zmm20
+; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm7, %ymm0
+; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm17, %ymm18, %ymm0
+; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u,u,u,u],zero,zero,xmm3[2,9],zero,zero,zero,xmm3[5,12,u,u]
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,4,11],zero,zero,xmm0[0,7,14],zero,zero,xmm0[u,u]
+; AVX512DQ-SLOW-NEXT:    vpor %xmm3, %xmm0, %xmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm9, %ymm3
+; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm21, %ymm16, %ymm3
+; AVX512DQ-SLOW-NEXT:    vpermq {{.*#+}} ymm5 = ymm3[2,3,0,1]
+; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0,1,2],ymm5[3],ymm3[4,5],ymm5[6],ymm3[7,8,9,10],ymm5[11],ymm3[12,13],ymm5[14],ymm3[15]
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} ymm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm3[3,10,17,24,31,22,29,20,27,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-SLOW-NEXT:    vpternlogq $248, %ymm24, %ymm0, %ymm3
+; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm14, %ymm0
+; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm26, %ymm27, %ymm0
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm5 = xmm0[u,u,u,u,u,u,u,2,9],zero,zero,zero,xmm0[5,12],zero,zero
 ; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,u,u],zero,zero,xmm0[4,11],zero,zero,xmm0[0,7,14]
-; AVX512DQ-SLOW-NEXT:    vpor %xmm6, %xmm0, %xmm0
-; AVX512DQ-SLOW-NEXT:    vinserti32x4 $1, %xmm0, %ymm0, %ymm29
-; AVX512DQ-SLOW-NEXT:    vpternlogq $184, %ymm4, %ymm25, %ymm29
-; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm15, %ymm0
-; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm30, %ymm31, %ymm0
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm4 = xmm0[u,u,2,9],zero,zero,zero,xmm0[5,12],zero,zero,xmm0[u,u,u,u,u]
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,u,u],zero,zero,xmm0[0,7,14],zero,zero,xmm0[3,10]
+; AVX512DQ-SLOW-NEXT:    vpor %xmm5, %xmm0, %xmm0
+; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512DQ-SLOW-NEXT:    vpternlogq $184, %ymm3, %ymm29, %ymm0
+; AVX512DQ-SLOW-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm22
+; AVX512DQ-SLOW-NEXT:    vpternlogq $184, %zmm23, %zmm8, %zmm22
+; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm14, %ymm0
+; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm17, %ymm18, %ymm0
+; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u,u,u,u],zero,zero,xmm3[3,10],zero,zero,zero,xmm3[6,13,u,u]
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,5,12],zero,zero,xmm0[1,8,15],zero,zero,xmm0[u,u]
+; AVX512DQ-SLOW-NEXT:    vpor %xmm3, %xmm0, %xmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm7, %ymm3
+; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm16, %ymm21, %ymm3
+; AVX512DQ-SLOW-NEXT:    vpermq {{.*#+}} ymm5 = ymm3[2,3,0,1]
+; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} ymm3 = ymm5[0],ymm3[1,2],ymm5[3],ymm3[4,5,6],ymm5[7,8],ymm3[9,10],ymm5[11],ymm3[12,13,14],ymm5[15]
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} ymm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm3[4,11,18,25,16,23,30,21,28,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-SLOW-NEXT:    vpternlogq $248, %ymm24, %ymm0, %ymm3
+; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm9, %ymm0
+; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm26, %ymm27, %ymm0
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm5 = xmm0[u,u,u,u,u,u,u,3,10],zero,zero,zero,xmm0[6,13],zero,zero
 ; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u],zero,zero,xmm0[0,7,14],zero,zero,xmm0[3,10,u,u,u,u,u]
-; AVX512DQ-SLOW-NEXT:    vporq %xmm4, %xmm0, %xmm25
-; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm7, %ymm10
-; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm21, %ymm22, %ymm28
-; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm15, %ymm13
-; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm19, %ymm14, %ymm13
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,u,u],zero,zero,xmm0[1,8,15],zero,zero,xmm0[4,11]
+; AVX512DQ-SLOW-NEXT:    vpor %xmm5, %xmm0, %xmm0
+; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512DQ-SLOW-NEXT:    vpternlogq $184, %ymm3, %ymm29, %ymm0
+; AVX512DQ-SLOW-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm23
+; AVX512DQ-SLOW-NEXT:    vpternlogq $184, %zmm25, %zmm8, %zmm23
+; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm14, %ymm0
+; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm16, %ymm21, %ymm0
+; AVX512DQ-SLOW-NEXT:    vpermq {{.*#+}} ymm3 = ymm0[2,3,0,1]
+; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm3[0],ymm0[1,2,3],ymm3[4],ymm0[5,6],ymm3[7,8],ymm0[9,10,11],ymm3[12],ymm0[13,14],ymm3[15]
+; AVX512DQ-SLOW-NEXT:    vpshufb %ymm2, %ymm0, %ymm0
+; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm9, %ymm2
+; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm17, %ymm18, %ymm2
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm3 = xmm2[u,u,u,u,u,6,13],zero,zero,xmm2[2,9],zero,zero,zero,xmm2[u,u]
+; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm2, %xmm2
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,u,u,u,u],zero,zero,xmm2[4,11],zero,zero,xmm2[0,7,14,u,u]
+; AVX512DQ-SLOW-NEXT:    vpor %xmm3, %xmm2, %xmm2
+; AVX512DQ-SLOW-NEXT:    vpternlogq $236, %ymm24, %ymm0, %ymm2
 ; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm7, %ymm0
-; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm19, %ymm14, %ymm7
-; AVX512DQ-SLOW-NEXT:    vmovdqa %xmm1, %xmm5
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm4 = xmm1[u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,xmm1[6,13]
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm6 = xmm3[u,u,u,u,u,u,u,u,u,u,u,1,8,15],zero,zero
-; AVX512DQ-SLOW-NEXT:    vpor %xmm4, %xmm6, %xmm4
-; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm26, %ymm27, %ymm10
-; AVX512DQ-SLOW-NEXT:    vpternlogq $226, %ymm20, %ymm15, %ymm17
-; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm9, %ymm6
-; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm27, %ymm26, %ymm6
-; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm27, %ymm26, %ymm15
-; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} ymm10 = ymm11[0],ymm10[1,2,3],ymm11[4],ymm10[5,6],ymm11[7,8],ymm10[9,10,11],ymm11[12],ymm10[13,14],ymm11[15]
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} ymm14 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm10[3,10,1,8,15,6,13,20,27],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-SLOW-NEXT:    vmovdqa {{.*#+}} ymm1 = [255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512DQ-SLOW-NEXT:    vpternlogq $248, %ymm1, %ymm2, %ymm14
-; AVX512DQ-SLOW-NEXT:    vinserti32x4 $1, %xmm25, %ymm0, %ymm10
-; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} ymm19 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0]
-; AVX512DQ-SLOW-NEXT:    vpternlogq $184, %ymm10, %ymm19, %ymm4
-; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm13, %xmm10
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm10 = zero,zero,xmm10[3,10],zero,zero,zero,xmm10[6,13,u,u,u,u,u,u,u]
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm12 = xmm13[5,12],zero,zero,xmm13[1,8,15],zero,zero,xmm13[u,u,u,u,u,u,u]
-; AVX512DQ-SLOW-NEXT:    vpor %xmm10, %xmm12, %xmm12
-; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} ymm6 = ymm6[0],ymm11[1],ymm6[2,3],ymm11[4],ymm6[5,6,7,8],ymm11[9],ymm6[10,11],ymm11[12],ymm6[13,14,15]
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} ymm10 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm6[4,11,2,9,0,7,14,21,28],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-SLOW-NEXT:    vpternlogq $248, %ymm1, %ymm12, %ymm10
-; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm1, %ymm2
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm6 = xmm7[6,13],zero,zero,xmm7[2,9],zero,zero,zero,xmm7[u,u,u,u,u,u,u]
-; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm7, %xmm7
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm7 = zero,zero,xmm7[4,11],zero,zero,xmm7[0,7,14,u,u,u,u,u,u,u]
-; AVX512DQ-SLOW-NEXT:    vpor %xmm6, %xmm7, %xmm6
-; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} ymm1 = ymm15[0],ymm11[1],ymm15[2,3,4],ymm11[5],ymm15[6,7,8],ymm11[9],ymm15[10,11,12],ymm11[13],ymm15[14,15]
-; AVX512DQ-SLOW-NEXT:    vpbroadcastd {{.*#+}} xmm12 = [4,11,0,0,4,11,0,0,4,11,0,0,4,11,0,0]
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} ymm11 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm1[5,12,3,10,1,8,15,22,29],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-SLOW-NEXT:    vpternlogq $248, %ymm2, %ymm6, %ymm11
-; AVX512DQ-SLOW-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload
-; AVX512DQ-SLOW-NEXT:    vpshufb %xmm12, %xmm2, %xmm1
-; AVX512DQ-SLOW-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm6 = xmm15[2,9,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-SLOW-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm6[0],xmm1[1],xmm6[1],xmm1[2],xmm6[2],xmm1[3],xmm6[3]
-; AVX512DQ-SLOW-NEXT:    vinserti64x4 $1, %ymm1, %zmm4, %zmm7
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
-; AVX512DQ-SLOW-NEXT:    vpternlogq $184, %zmm14, %zmm1, %zmm7
-; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm30, %ymm31, %ymm0
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm4 = xmm0[u,u,3,10],zero,zero,zero,xmm0[6,13],zero,zero,xmm0[u,u,u,u,u]
-; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm0
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u],zero,zero,xmm0[1,8,15],zero,zero,xmm0[4,11,u,u,u,u,u]
-; AVX512DQ-SLOW-NEXT:    vpor %xmm4, %xmm0, %xmm0
-; AVX512DQ-SLOW-NEXT:    vpternlogq $226, %ymm18, %ymm9, %ymm16
-; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm31, %ymm30, %ymm9
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm4 = xmm3[u,u,u,u,u,u,u,u,u,u,u,2,9],zero,zero,zero
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm6 = xmm5[u,u,u,u,u,u,u,u,u,u,u],zero,zero,xmm5[0,7,14]
-; AVX512DQ-SLOW-NEXT:    vpor %xmm4, %xmm6, %xmm4
+; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm27, %ymm26, %ymm0
+; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u,u,u,u,u,u],zero,zero,xmm3[2,9],zero,zero,zero,xmm3[5,12]
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,u,u,4,11],zero,zero,xmm0[0,7,14],zero,zero
+; AVX512DQ-SLOW-NEXT:    vpor %xmm3, %xmm0, %xmm0
 ; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512DQ-SLOW-NEXT:    vpternlogq $184, %ymm0, %ymm19, %ymm4
-; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm9, %xmm0
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u],zero,zero,xmm0[2,9],zero,zero,zero,xmm0[5,12,u,u,u,u,u]
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm6 = xmm9[u,u,4,11],zero,zero,xmm9[0,7,14],zero,zero,xmm9[u,u,u,u,u]
-; AVX512DQ-SLOW-NEXT:    vpor %xmm0, %xmm6, %xmm0
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm6 = xmm3[u,u,u,u,u,u,u,u,u,u,u,3,10],zero,zero,zero
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm8 = xmm5[u,u,u,u,u,u,u,u,u,u,u],zero,zero,xmm5[1,8,15]
-; AVX512DQ-SLOW-NEXT:    vpor %xmm6, %xmm8, %xmm6
+; AVX512DQ-SLOW-NEXT:    vpternlogq $184, %ymm2, %ymm29, %ymm0
+; AVX512DQ-SLOW-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm25
+; AVX512DQ-SLOW-NEXT:    vpternlogq $184, %zmm30, %zmm8, %zmm25
+; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm9, %ymm0
+; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm16, %ymm21, %ymm0
+; AVX512DQ-SLOW-NEXT:    vpermq {{.*#+}} ymm2 = ymm0[2,3,0,1]
+; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm2[1],ymm0[2,3],ymm2[4],ymm0[5,6,7,8],ymm2[9],ymm0[10,11],ymm2[12],ymm0[13,14,15]
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} ymm19 = [65535,0,65535,65535,65535,0,65535,65535,0,65535,65535,65535,0,65535,65535,0]
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %ymm19, %ymm2
+; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm18, %ymm17, %ymm2
+; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm2, %xmm3
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u,u,u],zero,zero,zero,xmm3[5,12],zero,zero,xmm3[1,8,15,u,u]
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,u,u,u,0,7,14],zero,zero,xmm2[3,10],zero,zero,zero,xmm2[u,u]
+; AVX512DQ-SLOW-NEXT:    vpor %xmm3, %xmm2, %xmm2
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} ymm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm0[6,13,20,27,18,25,16,23,30,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-SLOW-NEXT:    vpternlogq $236, %ymm24, %ymm0, %ymm2
+; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm14, %ymm0
+; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm27, %ymm26, %ymm0
+; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm3
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u,u,u,u,u,u],zero,zero,xmm3[3,10],zero,zero,zero,xmm3[6,13]
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,u,u,5,12],zero,zero,xmm0[1,8,15],zero,zero
+; AVX512DQ-SLOW-NEXT:    vpor %xmm3, %xmm0, %xmm0
+; AVX512DQ-SLOW-NEXT:    vinserti32x4 $1, %xmm0, %ymm0, %ymm24
+; AVX512DQ-SLOW-NEXT:    vpternlogq $184, %ymm2, %ymm29, %ymm24
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %ymm19, %ymm0
+; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm21, %ymm16, %ymm0
+; AVX512DQ-SLOW-NEXT:    vpermq {{.*#+}} ymm2 = ymm0[2,3,0,1]
+; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm2[1],ymm0[2,3,4],ymm2[5],ymm0[6,7,8],ymm2[9],ymm0[10,11,12],ymm2[13],ymm0[14,15]
+; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm7, %ymm2
+; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm18, %ymm17, %ymm2
+; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm2, %xmm3
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u,u,u],zero,zero,zero,xmm3[6,13],zero,zero,xmm3[2,9,u,u,u]
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,u,u,u,1,8,15],zero,zero,xmm2[4,11],zero,zero,xmm2[u,u,u]
+; AVX512DQ-SLOW-NEXT:    vpor %xmm3, %xmm2, %xmm2
+; AVX512DQ-SLOW-NEXT:    vmovdqa {{.*#+}} ymm8 = <u,u,u,u,128,128,128,128,128,128,128,128,128,0,7,14,21,28,19,26,17,24,31,u,u,u,u,u,u,u,u,u>
+; AVX512DQ-SLOW-NEXT:    vpshufb %ymm8, %ymm0, %ymm0
+; AVX512DQ-SLOW-NEXT:    vpternlogq $220, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm2
+; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm9, %ymm0
+; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm27, %ymm26, %ymm0
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm3 = xmm0[u,u,u,u,u,u,u,6,13],zero,zero,xmm0[2,9],zero,zero,zero
+; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm0
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,u,u],zero,zero,xmm0[4,11],zero,zero,xmm0[0,7,14]
+; AVX512DQ-SLOW-NEXT:    vpor %xmm3, %xmm0, %xmm0
+; AVX512DQ-SLOW-NEXT:    vinserti32x4 $1, %xmm0, %ymm0, %ymm30
+; AVX512DQ-SLOW-NEXT:    vpternlogq $184, %ymm2, %ymm29, %ymm30
+; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm7, %ymm0
+; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm12, %ymm13, %ymm0
+; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm2
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm2 = zero,zero,xmm2[2,9],zero,zero,zero,xmm2[5,12,u,u,u,u,u,u,u]
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[4,11],zero,zero,xmm0[0,7,14],zero,zero,xmm0[u,u,u,u,u,u,u]
+; AVX512DQ-SLOW-NEXT:    vporq %xmm2, %xmm0, %xmm29
+; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm26, %ymm27, %ymm19
+; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm14, %ymm0
+; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm12, %ymm13, %ymm0
+; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm0, %xmm2
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm2 = zero,zero,xmm2[3,10],zero,zero,zero,xmm2[6,13,u,u,u,u,u,u,u]
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[5,12],zero,zero,xmm0[1,8,15],zero,zero,xmm0[u,u,u,u,u,u,u]
+; AVX512DQ-SLOW-NEXT:    vpor %xmm2, %xmm0, %xmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm9, %ymm2
+; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm9, %ymm15
+; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm12, %ymm13, %ymm9
+; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm28, %ymm31, %ymm2
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm3 = xmm9[6,13],zero,zero,xmm9[2,9],zero,zero,zero,xmm9[u,u,u,u,u,u,u]
+; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm9, %xmm5
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm5 = zero,zero,xmm5[4,11],zero,zero,xmm5[0,7,14,u,u,u,u,u,u,u]
+; AVX512DQ-SLOW-NEXT:    vpor %xmm3, %xmm5, %xmm3
+; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm14, %ymm5
+; AVX512DQ-SLOW-NEXT:    vpternlogq $226, %ymm18, %ymm14, %ymm17
+; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm7, %ymm9
+; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm31, %ymm28, %ymm9
+; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm31, %ymm28, %ymm14
+; AVX512DQ-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
+; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} ymm2 = ymm4[0],ymm2[1,2,3],ymm4[4],ymm2[5,6],ymm4[7,8],ymm2[9,10,11],ymm4[12],ymm2[13,14],ymm4[15]
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} ymm12 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[3,10,1,8,15,6,13,20,27],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512DQ-SLOW-NEXT:    vmovdqa %ymm11, %ymm1
+; AVX512DQ-SLOW-NEXT:    vpternlogq $248, %ymm11, %ymm29, %ymm12
+; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} ymm2 = ymm9[0],ymm4[1],ymm9[2,3],ymm4[4],ymm9[5,6,7,8],ymm4[9],ymm9[10,11],ymm4[12],ymm9[13,14,15]
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} ymm11 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[4,11,2,9,0,7,14,21,28],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512DQ-SLOW-NEXT:    vpternlogq $248, %ymm1, %ymm0, %ymm11
+; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm14[0],ymm4[1],ymm14[2,3,4],ymm4[5],ymm14[6,7,8],ymm4[9],ymm14[10,11,12],ymm4[13],ymm14[14,15]
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} ymm9 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm0[5,12,3,10,1,8,15,22,29],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512DQ-SLOW-NEXT:    vpternlogq $248, %ymm1, %ymm3, %ymm9
+; AVX512DQ-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload
+; AVX512DQ-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
+; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm6, %ymm13, %ymm5
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm5[u,u,2,9],zero,zero,zero,xmm5[5,12],zero,zero,xmm5[u,u,u,u,u]
+; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm5, %xmm2
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,u],zero,zero,xmm2[0,7,14],zero,zero,xmm2[3,10,u,u,u,u,u]
+; AVX512DQ-SLOW-NEXT:    vpor %xmm0, %xmm2, %xmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqa %xmm10, %xmm1
+; AVX512DQ-SLOW-NEXT:    vmovdqa {{.*#+}} ymm2 = <128,128,128,128,128,128,128,128,128,128,128,128,128,128,6,13,20,27,18,25,16,23,30,u,u,u,u,u,u,u,u,u>
+; AVX512DQ-SLOW-NEXT:    vpshufb %xmm2, %xmm10, %xmm2
+; AVX512DQ-SLOW-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm3 = xmm4[u,u,u,u,u,u,u,u,u,u,u,1,8,15],zero,zero
+; AVX512DQ-SLOW-NEXT:    vpor %xmm2, %xmm3, %xmm2
 ; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
-; AVX512DQ-SLOW-NEXT:    vpternlogq $184, %ymm0, %ymm19, %ymm6
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm15[3,10,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm8 = xmm2[5,12,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-SLOW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm8[0],xmm0[0],xmm8[1],xmm0[1],xmm8[2],xmm0[2],xmm8[3],xmm0[3]
-; AVX512DQ-SLOW-NEXT:    vinserti64x4 $1, %ymm0, %zmm4, %zmm0
-; AVX512DQ-SLOW-NEXT:    vpternlogq $184, %zmm10, %zmm1, %zmm0
-; AVX512DQ-SLOW-NEXT:    vpshufb %xmm12, %xmm15, %xmm4
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm8 = xmm2[6,13,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-SLOW-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm8[0],xmm4[0],xmm8[1],xmm4[1],xmm8[2],xmm4[2],xmm8[3],xmm4[3]
-; AVX512DQ-SLOW-NEXT:    vinserti64x4 $1, %ymm4, %zmm6, %zmm4
-; AVX512DQ-SLOW-NEXT:    vpternlogq $184, %zmm11, %zmm1, %zmm4
+; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} ymm18 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0]
+; AVX512DQ-SLOW-NEXT:    vpternlogq $184, %ymm0, %ymm18, %ymm2
+; AVX512DQ-SLOW-NEXT:    vpternlogq $226, %ymm21, %ymm7, %ymm16
+; AVX512DQ-SLOW-NEXT:    vpbroadcastd {{.*#+}} xmm10 = [4,11,0,0,4,11,0,0,4,11,0,0,4,11,0,0]
+; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm6, %ymm13, %ymm15
+; AVX512DQ-SLOW-NEXT:    vpternlogq $202, %ymm13, %ymm6, %ymm7
+; AVX512DQ-SLOW-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload
+; AVX512DQ-SLOW-NEXT:    vpshufb %xmm10, %xmm14, %xmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm3 = xmm13[2,9,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-SLOW-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
+; AVX512DQ-SLOW-NEXT:    vinserti64x4 $1, %ymm0, %zmm2, %zmm2
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
+; AVX512DQ-SLOW-NEXT:    vpternlogq $184, %zmm12, %zmm0, %zmm2
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm3 = xmm15[u,u,3,10],zero,zero,zero,xmm15[6,13],zero,zero,xmm15[u,u,u,u,u]
+; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm15, %xmm5
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm5 = xmm5[u,u],zero,zero,xmm5[1,8,15],zero,zero,xmm5[4,11,u,u,u,u,u]
+; AVX512DQ-SLOW-NEXT:    vpor %xmm3, %xmm5, %xmm3
+; AVX512DQ-SLOW-NEXT:    vpshufb %xmm8, %xmm1, %xmm5
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm8 = xmm4[u,u,u,u,u,u,u,u,u,u,u,2,9],zero,zero,zero
+; AVX512DQ-SLOW-NEXT:    vpor %xmm5, %xmm8, %xmm5
+; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512DQ-SLOW-NEXT:    vpternlogq $184, %ymm3, %ymm18, %ymm5
+; AVX512DQ-SLOW-NEXT:    vextracti128 $1, %ymm7, %xmm3
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u],zero,zero,xmm3[2,9],zero,zero,zero,xmm3[5,12,u,u,u,u,u]
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm7 = xmm7[u,u,4,11],zero,zero,xmm7[0,7,14],zero,zero,xmm7[u,u,u,u,u]
+; AVX512DQ-SLOW-NEXT:    vpor %xmm3, %xmm7, %xmm3
+; AVX512DQ-SLOW-NEXT:    vmovdqa {{.*#+}} ymm7 = [128,128,128,128,128,128,128,128,128,128,128,128,128,1,8,15,22,29,20,27,18,25,128,128,128,128,128,128,128,128,128,128]
+; AVX512DQ-SLOW-NEXT:    vpshufb %xmm7, %xmm1, %xmm8
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm12 = xmm4[u,u,u,u,u,u,u,u,u,u,u,3,10],zero,zero,zero
+; AVX512DQ-SLOW-NEXT:    vpor %xmm12, %xmm8, %xmm8
+; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm8, %ymm0, %ymm8
+; AVX512DQ-SLOW-NEXT:    vpternlogq $184, %ymm3, %ymm18, %ymm8
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm3 = xmm13[3,10,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm12 = xmm14[5,12,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-SLOW-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm12[0],xmm3[0],xmm12[1],xmm3[1],xmm12[2],xmm3[2],xmm12[3],xmm3[3]
+; AVX512DQ-SLOW-NEXT:    vinserti64x4 $1, %ymm3, %zmm5, %zmm3
+; AVX512DQ-SLOW-NEXT:    vpternlogq $184, %zmm11, %zmm0, %zmm3
+; AVX512DQ-SLOW-NEXT:    vpshufb %xmm10, %xmm13, %xmm5
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm10 = xmm14[6,13,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-SLOW-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm10[0],xmm5[0],xmm10[1],xmm5[1],xmm10[2],xmm5[2],xmm10[3],xmm5[3]
+; AVX512DQ-SLOW-NEXT:    vinserti64x4 $1, %ymm5, %zmm8, %zmm5
+; AVX512DQ-SLOW-NEXT:    vpternlogq $184, %zmm9, %zmm0, %zmm5
 ; AVX512DQ-SLOW-NEXT:    movw $-512, %ax # imm = 0xFE00
 ; AVX512DQ-SLOW-NEXT:    kmovw %eax, %k1
-; AVX512DQ-SLOW-NEXT:    vinserti32x8 $1, %ymm23, %zmm0, %zmm7 {%k1}
-; AVX512DQ-SLOW-NEXT:    vinserti32x8 $1, %ymm29, %zmm0, %zmm0 {%k1}
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %ymm17, %ymm2
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm1 = xmm2[u,u,u,u,2,9],zero,zero,zero,xmm2[5,12],zero,zero,xmm2[u,u,u]
-; AVX512DQ-SLOW-NEXT:    vextracti32x4 $1, %ymm17, %xmm2
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,u,u,u],zero,zero,xmm2[0,7,14],zero,zero,xmm2[3,10,u,u,u]
-; AVX512DQ-SLOW-NEXT:    vpor %xmm1, %xmm2, %xmm1
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %ymm16, %ymm3
-; AVX512DQ-SLOW-NEXT:    vpermq {{.*#+}} ymm2 = ymm16[2,3,0,1]
-; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} ymm2 = ymm3[0,1],ymm2[2],ymm3[3,4],ymm2[5],ymm3[6,7,8,9],ymm2[10],ymm3[11,12],ymm2[13],ymm3[14,15]
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} ymm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[1,8,15,22,29,20,27,18,25],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-SLOW-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm2
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %ymm28, %ymm3
-; AVX512DQ-SLOW-NEXT:    vextracti32x4 $1, %ymm28, %xmm1
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[u,u,u,u,u,u],zero,zero,zero,xmm1[5,12],zero,zero,xmm1[1,8,15]
-; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u,u,u,u,u,0,7,14],zero,zero,xmm3[3,10],zero,zero,zero
-; AVX512DQ-SLOW-NEXT:    vpor %xmm1, %xmm3, %xmm1
-; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0,1,2],ymm1[3,4,5,6,7],ymm2[8,9,10],ymm1[11,12,13,14,15]
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQ-SLOW-NEXT:    vinserti32x8 $1, %ymm1, %zmm0, %zmm4 {%k1}
-; AVX512DQ-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovaps %zmm1, (%rsi)
-; AVX512DQ-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovaps %zmm1, (%rdx)
-; AVX512DQ-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovaps %zmm1, (%rcx)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm24, (%r8)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm7, (%r9)
+; AVX512DQ-SLOW-NEXT:    vinserti32x8 $1, %ymm24, %zmm0, %zmm2 {%k1}
+; AVX512DQ-SLOW-NEXT:    vinserti32x8 $1, %ymm30, %zmm0, %zmm3 {%k1}
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %ymm16, %ymm1
+; AVX512DQ-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm16[2,3,0,1]
+; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0,1],ymm0[2],ymm1[3,4],ymm0[5],ymm1[6,7,8,9],ymm0[10],ymm1[11,12],ymm0[13],ymm1[14,15]
+; AVX512DQ-SLOW-NEXT:    vpshufb %ymm7, %ymm0, %ymm0
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %ymm17, %ymm1
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm6 = xmm1[u,u,u,u,2,9],zero,zero,zero,xmm1[5,12],zero,zero,xmm1[u,u,u]
+; AVX512DQ-SLOW-NEXT:    vextracti32x4 $1, %ymm17, %xmm1
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[u,u,u,u],zero,zero,xmm1[0,7,14],zero,zero,xmm1[3,10,u,u,u]
+; AVX512DQ-SLOW-NEXT:    vpor %xmm6, %xmm1, %xmm1
+; AVX512DQ-SLOW-NEXT:    vpternlogq $236, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm1
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %ymm19, %ymm4
+; AVX512DQ-SLOW-NEXT:    vextracti32x4 $1, %ymm19, %xmm0
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,u],zero,zero,zero,xmm0[5,12],zero,zero,xmm0[1,8,15]
+; AVX512DQ-SLOW-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[u,u,u,u,u,u,0,7,14],zero,zero,xmm4[3,10],zero,zero,zero
+; AVX512DQ-SLOW-NEXT:    vpor %xmm0, %xmm4, %xmm0
+; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512DQ-SLOW-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6,7],ymm1[8,9,10],ymm0[11,12,13,14,15]
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
+; AVX512DQ-SLOW-NEXT:    vinserti32x8 $1, %ymm0, %zmm0, %zmm5 {%k1}
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm20, (%rsi)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm22, (%rdx)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm23, (%rcx)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm25, (%r8)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm2, (%r9)
 ; AVX512DQ-SLOW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, (%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm3, (%rax)
 ; AVX512DQ-SLOW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm4, (%rax)
-; AVX512DQ-SLOW-NEXT:    addq $104, %rsp
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm5, (%rax)
+; AVX512DQ-SLOW-NEXT:    addq $24, %rsp
 ; AVX512DQ-SLOW-NEXT:    vzeroupper
 ; AVX512DQ-SLOW-NEXT:    retq
 ;
 ; AVX512DQ-FAST-LABEL: load_i8_stride7_vf64:
 ; AVX512DQ-FAST:       # %bb.0:
-; AVX512DQ-FAST-NEXT:    pushq %rax
-; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} ymm22 = [65535,0,65535,65535,65535,0,65535,65535,0,65535,65535,65535,0,65535,65535,0]
-; AVX512DQ-FAST-NEXT:    vmovdqa (%rdi), %ymm13
-; AVX512DQ-FAST-NEXT:    vmovdqa 32(%rdi), %ymm14
-; AVX512DQ-FAST-NEXT:    vmovdqa64 64(%rdi), %ymm17
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %ymm22, %ymm1
-; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm14, %ymm13, %ymm1
+; AVX512DQ-FAST-NEXT:    vmovdqa {{.*#+}} ymm0 = [65535,0,65535,65535,65535,0,65535,65535,0,65535,65535,65535,0,65535,65535,0]
+; AVX512DQ-FAST-NEXT:    vmovdqa64 (%rdi), %ymm26
+; AVX512DQ-FAST-NEXT:    vmovdqa 32(%rdi), %ymm11
+; AVX512DQ-FAST-NEXT:    vmovdqa64 64(%rdi), %ymm30
+; AVX512DQ-FAST-NEXT:    vmovdqa %ymm0, %ymm1
+; AVX512DQ-FAST-NEXT:    vmovdqa %ymm0, %ymm7
+; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm11, %ymm26, %ymm1
 ; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm1, %xmm2
 ; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm2 = zero,zero,zero,xmm2[5,12],zero,zero,xmm2[1,8,15,u,u,u,u,u,u]
 ; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[0,7,14],zero,zero,xmm1[3,10],zero,zero,zero,xmm1[u,u,u,u,u,u]
 ; AVX512DQ-FAST-NEXT:    vpor %xmm2, %xmm1, %xmm1
-; AVX512DQ-FAST-NEXT:    vmovdqa {{.*#+}} ymm5 = [65535,65535,65535,0,65535,65535,0,65535,65535,65535,0,65535,65535,0,65535,65535]
-; AVX512DQ-FAST-NEXT:    vmovdqa 96(%rdi), %ymm12
-; AVX512DQ-FAST-NEXT:    vmovdqa %ymm5, %ymm2
-; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm17, %ymm12, %ymm2
-; AVX512DQ-FAST-NEXT:    vmovdqa 80(%rdi), %xmm10
-; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm10[2],ymm2[3,4],ymm10[5],ymm2[6,7,8,9],ymm10[10],ymm2[11,12],ymm10[13],ymm2[14,15]
+; AVX512DQ-FAST-NEXT:    vmovdqa {{.*#+}} ymm6 = [65535,65535,65535,0,65535,65535,0,65535,65535,65535,0,65535,65535,0,65535,65535]
+; AVX512DQ-FAST-NEXT:    vmovdqa64 96(%rdi), %ymm29
+; AVX512DQ-FAST-NEXT:    vmovdqa %ymm6, %ymm2
+; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm30, %ymm29, %ymm2
+; AVX512DQ-FAST-NEXT:    vmovdqa 80(%rdi), %xmm9
+; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm9[2],ymm2[3,4],ymm9[5],ymm2[6,7,8,9],ymm9[10],ymm2[11,12],ymm9[13],ymm2[14,15]
 ; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm4 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[6,13,4,11,2,9,16,23,30,u],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
 ; AVX512DQ-FAST-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm4
-; AVX512DQ-FAST-NEXT:    vmovdqa {{.*#+}} ymm11 = [65535,65535,0,65535,65535,65535,0,65535,65535,0,65535,65535,65535,0,65535,65535]
-; AVX512DQ-FAST-NEXT:    vmovdqa64 128(%rdi), %ymm30
+; AVX512DQ-FAST-NEXT:    vmovdqa {{.*#+}} ymm12 = [65535,65535,0,65535,65535,65535,0,65535,65535,0,65535,65535,65535,0,65535,65535]
+; AVX512DQ-FAST-NEXT:    vmovdqa 128(%rdi), %ymm0
 ; AVX512DQ-FAST-NEXT:    vmovdqa64 160(%rdi), %ymm31
-; AVX512DQ-FAST-NEXT:    vmovdqa %ymm11, %ymm1
-; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm30, %ymm31, %ymm1
+; AVX512DQ-FAST-NEXT:    vmovdqa %ymm12, %ymm1
+; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm0, %ymm31, %ymm1
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %ymm0, %ymm24
 ; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm1, %xmm2
 ; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,u,u],zero,zero,xmm2[3,10],zero,zero,zero,xmm2[6,13,u,u,u,u]
 ; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[u,u,u,5,12],zero,zero,xmm1[1,8,15],zero,zero,xmm1[u,u,u,u]
 ; AVX512DQ-FAST-NEXT:    vpor %xmm2, %xmm1, %xmm1
 ; AVX512DQ-FAST-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512DQ-FAST-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [1,2,4,6,1,2,4,6]
-; AVX512DQ-FAST-NEXT:    # ymm3 = mem[0,1,0,1]
-; AVX512DQ-FAST-NEXT:    vmovdqa 192(%rdi), %ymm2
-; AVX512DQ-FAST-NEXT:    vpermd %ymm2, %ymm3, %ymm3
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm3 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,23,26,29]
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5,6],ymm3[7]
-; AVX512DQ-FAST-NEXT:    vmovdqa 240(%rdi), %xmm0
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm6 = zero,zero,zero,xmm0[5,12,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-FAST-NEXT:    vmovdqa 224(%rdi), %xmm3
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm7 = xmm3[0,7,14],zero,zero,xmm3[u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %xmm3, %xmm21
-; AVX512DQ-FAST-NEXT:    vpor %xmm6, %xmm7, %xmm6
-; AVX512DQ-FAST-NEXT:    vinserti32x4 $2, %xmm6, %zmm1, %zmm6
-; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm24 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512DQ-FAST-NEXT:    vpternlogq $184, %zmm4, %zmm24, %zmm6
-; AVX512DQ-FAST-NEXT:    vmovdqa64 288(%rdi), %ymm18
-; AVX512DQ-FAST-NEXT:    vmovdqa64 256(%rdi), %ymm29
-; AVX512DQ-FAST-NEXT:    vmovdqa %ymm5, %ymm4
-; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm18, %ymm29, %ymm4
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm8 = xmm4[u,u,u,u,u,3,10],zero,zero,zero,xmm4[6,13],zero,zero,xmm4[u,u]
+; AVX512DQ-FAST-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [1,2,4,6,1,2,4,6]
+; AVX512DQ-FAST-NEXT:    # ymm2 = mem[0,1,0,1]
+; AVX512DQ-FAST-NEXT:    vmovdqa64 192(%rdi), %ymm18
+; AVX512DQ-FAST-NEXT:    vpermd %ymm18, %ymm2, %ymm2
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm2 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,16,23,26,29]
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5,6],ymm2[7]
+; AVX512DQ-FAST-NEXT:    vmovdqa 240(%rdi), %xmm3
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm2 = zero,zero,zero,xmm3[5,12,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-FAST-NEXT:    vmovdqa %xmm3, %xmm8
+; AVX512DQ-FAST-NEXT:    vmovdqa 224(%rdi), %xmm0
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm5 = xmm0[0,7,14],zero,zero,xmm0[u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-FAST-NEXT:    vpor %xmm2, %xmm5, %xmm2
+; AVX512DQ-FAST-NEXT:    vinserti32x4 $2, %xmm2, %zmm1, %zmm20
+; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; AVX512DQ-FAST-NEXT:    vpternlogq $184, %zmm4, %zmm2, %zmm20
+; AVX512DQ-FAST-NEXT:    vmovdqa 288(%rdi), %ymm10
+; AVX512DQ-FAST-NEXT:    vmovdqa64 256(%rdi), %ymm19
+; AVX512DQ-FAST-NEXT:    vmovdqa %ymm6, %ymm4
+; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm10, %ymm19, %ymm4
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm5 = xmm4[u,u,u,u,u,3,10],zero,zero,zero,xmm4[6,13],zero,zero,xmm4[u,u]
 ; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm4, %xmm4
 ; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[u,u,u,u,u],zero,zero,xmm4[1,8,15],zero,zero,xmm4[4,11,u,u]
-; AVX512DQ-FAST-NEXT:    vpor %xmm4, %xmm8, %xmm8
-; AVX512DQ-FAST-NEXT:    vmovdqa64 352(%rdi), %ymm16
-; AVX512DQ-FAST-NEXT:    vmovdqa 320(%rdi), %ymm4
-; AVX512DQ-FAST-NEXT:    vmovdqa %ymm11, %ymm15
-; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm16, %ymm4, %ymm15
-; AVX512DQ-FAST-NEXT:    vpermq {{.*#+}} ymm7 = ymm15[2,3,0,1]
-; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} ymm7 = ymm15[0,1],ymm7[2],ymm15[3,4,5],ymm7[6],ymm15[7,8,9],ymm7[10],ymm15[11,12,13],ymm7[14],ymm15[15]
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm7 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm7[2,9,16,23,30,21,28,19,26,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} ymm27 = [65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,65535,65535,65535,65535]
-; AVX512DQ-FAST-NEXT:    vpternlogq $248, %ymm27, %ymm8, %ymm7
-; AVX512DQ-FAST-NEXT:    vmovdqa {{.*#+}} ymm15 = [65535,65535,0,65535,65535,0,65535,65535,65535,0,65535,65535,0,65535,65535,65535]
-; AVX512DQ-FAST-NEXT:    vmovdqa64 416(%rdi), %ymm19
-; AVX512DQ-FAST-NEXT:    vmovdqa64 384(%rdi), %ymm20
-; AVX512DQ-FAST-NEXT:    vmovdqa %ymm15, %ymm8
-; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm19, %ymm20, %ymm8
-; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm8, %xmm3
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u,u,u,u,u,u],zero,zero,zero,xmm3[6,13],zero,zero,xmm3[2,9]
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm8 = xmm8[u,u,u,u,u,u,u,1,8,15],zero,zero,xmm8[4,11],zero,zero
-; AVX512DQ-FAST-NEXT:    vpor %xmm3, %xmm8, %xmm3
-; AVX512DQ-FAST-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} ymm23 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0]
-; AVX512DQ-FAST-NEXT:    vpternlogq $184, %ymm7, %ymm23, %ymm3
-; AVX512DQ-FAST-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm3
-; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm28 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
-; AVX512DQ-FAST-NEXT:    vpternlogq $184, %zmm6, %zmm28, %zmm3
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa %ymm15, %ymm3
-; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm14, %ymm13, %ymm3
-; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm3, %xmm6
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm6 = zero,zero,zero,xmm6[6,13],zero,zero,xmm6[2,9,u,u,u,u,u,u,u]
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[1,8,15],zero,zero,xmm3[4,11],zero,zero,xmm3[u,u,u,u,u,u,u]
-; AVX512DQ-FAST-NEXT:    vpor %xmm6, %xmm3, %xmm3
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %ymm22, %ymm6
-; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm12, %ymm17, %ymm6
-; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} ymm6 = ymm6[0,1],ymm10[2],ymm6[3,4,5],ymm10[6],ymm6[7,8,9],ymm10[10],ymm6[11,12,13],ymm10[14],ymm6[15]
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm6 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm6[0,7,14,5,12,3,10,17,24,31,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-FAST-NEXT:    vpternlogq $244, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm6
-; AVX512DQ-FAST-NEXT:    vmovdqa %ymm5, %ymm3
-; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm30, %ymm31, %ymm3
+; AVX512DQ-FAST-NEXT:    vpor %xmm5, %xmm4, %xmm13
+; AVX512DQ-FAST-NEXT:    vmovdqa 352(%rdi), %ymm15
+; AVX512DQ-FAST-NEXT:    vmovdqa64 320(%rdi), %ymm16
+; AVX512DQ-FAST-NEXT:    vmovdqa %ymm12, %ymm5
+; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm15, %ymm16, %ymm5
+; AVX512DQ-FAST-NEXT:    vpermq {{.*#+}} ymm14 = ymm5[2,3,0,1]
+; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} ymm5 = ymm5[0,1],ymm14[2],ymm5[3,4,5],ymm14[6],ymm5[7,8,9],ymm14[10],ymm5[11,12,13],ymm14[14],ymm5[15]
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm5 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm5[2,9,16,23,30,21,28,19,26,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} ymm23 = [65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,65535,65535,65535,65535]
+; AVX512DQ-FAST-NEXT:    vpternlogq $248, %ymm23, %ymm13, %ymm5
+; AVX512DQ-FAST-NEXT:    vmovdqa {{.*#+}} ymm14 = [65535,65535,0,65535,65535,0,65535,65535,65535,0,65535,65535,0,65535,65535,65535]
+; AVX512DQ-FAST-NEXT:    vmovdqa %ymm14, %ymm13
+; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm11, %ymm26, %ymm13
+; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm13, %xmm3
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = zero,zero,zero,xmm3[6,13],zero,zero,xmm3[2,9,u,u,u,u,u,u,u]
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm13 = xmm13[1,8,15],zero,zero,xmm13[4,11],zero,zero,xmm13[u,u,u,u,u,u,u]
+; AVX512DQ-FAST-NEXT:    vpor %xmm3, %xmm13, %xmm3
+; AVX512DQ-FAST-NEXT:    vmovdqa %ymm7, %ymm1
+; AVX512DQ-FAST-NEXT:    vmovdqa %ymm7, %ymm13
+; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm29, %ymm30, %ymm13
+; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} ymm13 = ymm13[0,1],ymm9[2],ymm13[3,4,5],ymm9[6],ymm13[7,8,9],ymm9[10],ymm13[11,12,13],ymm9[14],ymm13[15]
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm13 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm13[0,7,14,5,12,3,10,17,24,31,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-FAST-NEXT:    vpternlogq $244, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm13
+; AVX512DQ-FAST-NEXT:    vmovdqa %ymm6, %ymm3
+; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm24, %ymm31, %ymm3
 ; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm7 = xmm3[u,u,u,6,13],zero,zero,xmm3[2,9],zero,zero,zero,xmm3[u,u,u,u]
 ; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm3, %xmm3
 ; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u,u],zero,zero,xmm3[4,11],zero,zero,xmm3[0,7,14,u,u,u,u]
@@ -10557,307 +10577,334 @@ define void @load_i8_stride7_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr
 ; AVX512DQ-FAST-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
 ; AVX512DQ-FAST-NEXT:    vbroadcasti128 {{.*#+}} ymm7 = [1,3,4,6,1,3,4,6]
 ; AVX512DQ-FAST-NEXT:    # ymm7 = mem[0,1,0,1]
-; AVX512DQ-FAST-NEXT:    vpermd %ymm2, %ymm7, %ymm7
+; AVX512DQ-FAST-NEXT:    vpermd %ymm18, %ymm7, %ymm7
 ; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm7 = ymm7[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,17,20,27,30]
 ; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5,6],ymm7[7]
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm7 = zero,zero,zero,xmm0[6,13,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %xmm21, %xmm1
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm8 = xmm1[1,8,15],zero,zero,xmm1[u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm7 = zero,zero,zero,xmm8[6,13,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %xmm8, %xmm22
+; AVX512DQ-FAST-NEXT:    vmovdqa %xmm0, %xmm4
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm8 = xmm0[1,8,15],zero,zero,xmm0[u,u,u,u,u,u,u,u,u,u,u]
 ; AVX512DQ-FAST-NEXT:    vpor %xmm7, %xmm8, %xmm7
-; AVX512DQ-FAST-NEXT:    vinserti32x4 $2, %xmm7, %zmm3, %zmm3
-; AVX512DQ-FAST-NEXT:    vpternlogq $184, %zmm6, %zmm24, %zmm3
-; AVX512DQ-FAST-NEXT:    vmovdqa %ymm15, %ymm6
-; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm29, %ymm18, %ymm6
-; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm6, %xmm7
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm7 = xmm7[u,u,u,u,u],zero,zero,xmm7[2,9],zero,zero,zero,xmm7[5,12,u,u]
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm6 = xmm6[u,u,u,u,u,4,11],zero,zero,xmm6[0,7,14],zero,zero,xmm6[u,u]
-; AVX512DQ-FAST-NEXT:    vpor %xmm7, %xmm6, %xmm6
-; AVX512DQ-FAST-NEXT:    vmovdqa %ymm5, %ymm7
-; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm16, %ymm4, %ymm7
-; AVX512DQ-FAST-NEXT:    vpermq {{.*#+}} ymm8 = ymm7[2,3,0,1]
-; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} ymm7 = ymm7[0,1,2],ymm8[3],ymm7[4,5],ymm8[6],ymm7[7,8,9,10],ymm8[11],ymm7[12,13],ymm8[14],ymm7[15]
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm7 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm7[3,10,17,24,31,22,29,20,27,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-FAST-NEXT:    vpternlogq $248, %ymm27, %ymm6, %ymm7
-; AVX512DQ-FAST-NEXT:    vmovdqa %ymm11, %ymm6
-; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm19, %ymm20, %ymm6
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm8 = xmm6[u,u,u,u,u,u,u,2,9],zero,zero,zero,xmm6[5,12],zero,zero
-; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm6, %xmm6
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm6 = xmm6[u,u,u,u,u,u,u],zero,zero,xmm6[0,7,14],zero,zero,xmm6[3,10]
-; AVX512DQ-FAST-NEXT:    vpor %xmm6, %xmm8, %xmm6
-; AVX512DQ-FAST-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
-; AVX512DQ-FAST-NEXT:    vpternlogq $184, %ymm7, %ymm23, %ymm6
-; AVX512DQ-FAST-NEXT:    vinserti64x4 $1, %ymm6, %zmm0, %zmm25
-; AVX512DQ-FAST-NEXT:    vpternlogq $184, %zmm3, %zmm28, %zmm25
-; AVX512DQ-FAST-NEXT:    vmovdqa %ymm11, %ymm3
-; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm14, %ymm13, %ymm3
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm6 = xmm3[2,9],zero,zero,zero,xmm3[5,12],zero,zero,xmm3[u,u,u,u,u,u,u]
-; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm3, %xmm3
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = zero,zero,xmm3[0,7,14],zero,zero,xmm3[3,10,u,u,u,u,u,u,u]
-; AVX512DQ-FAST-NEXT:    vpor %xmm6, %xmm3, %xmm3
-; AVX512DQ-FAST-NEXT:    vmovdqa %ymm15, %ymm6
-; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm12, %ymm17, %ymm6
-; AVX512DQ-FAST-NEXT:    vmovdqu %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} ymm6 = ymm6[0,1,2],ymm10[3],ymm6[4,5],ymm10[6],ymm6[7,8,9,10],ymm10[11],ymm6[12,13],ymm10[14],ymm6[15]
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm6 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm6[1,8,15,6,13,4,11,18,25],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} ymm26 = [255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
-; AVX512DQ-FAST-NEXT:    vpternlogq $248, %ymm26, %ymm3, %ymm6
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %ymm22, %ymm3
-; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm31, %ymm30, %ymm3
-; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm3, %xmm7
+; AVX512DQ-FAST-NEXT:    vinserti32x4 $2, %xmm7, %zmm3, %zmm21
+; AVX512DQ-FAST-NEXT:    vpternlogq $184, %zmm13, %zmm2, %zmm21
+; AVX512DQ-FAST-NEXT:    vmovdqa %ymm12, %ymm2
+; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm11, %ymm26, %ymm2
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = xmm2[2,9],zero,zero,zero,xmm2[5,12],zero,zero,xmm2[u,u,u,u,u,u,u]
+; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm2, %xmm2
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm2 = zero,zero,xmm2[0,7,14],zero,zero,xmm2[3,10,u,u,u,u,u,u,u]
+; AVX512DQ-FAST-NEXT:    vpor %xmm3, %xmm2, %xmm2
+; AVX512DQ-FAST-NEXT:    vmovdqa %ymm14, %ymm3
+; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm29, %ymm30, %ymm3
+; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0,1,2],ymm9[3],ymm3[4,5],ymm9[6],ymm3[7,8,9,10],ymm9[11],ymm3[12,13],ymm9[14],ymm3[15]
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm3[1,8,15,6,13,4,11,18,25],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} ymm17 = [255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
+; AVX512DQ-FAST-NEXT:    vpternlogq $248, %ymm17, %ymm2, %ymm3
+; AVX512DQ-FAST-NEXT:    vmovdqa %ymm1, %ymm2
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %ymm24, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm31, %ymm24, %ymm2
+; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm2, %xmm7
 ; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm7 = xmm7[u,u],zero,zero,zero,xmm7[5,12],zero,zero,xmm7[1,8,15,u,u,u,u]
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u,0,7,14],zero,zero,xmm3[3,10],zero,zero,zero,xmm3[u,u,u,u]
-; AVX512DQ-FAST-NEXT:    vpor %xmm7, %xmm3, %xmm3
-; AVX512DQ-FAST-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,u,0,7,14],zero,zero,xmm2[3,10],zero,zero,zero,xmm2[u,u,u,u]
+; AVX512DQ-FAST-NEXT:    vpor %xmm7, %xmm2, %xmm2
+; AVX512DQ-FAST-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
 ; AVX512DQ-FAST-NEXT:    vbroadcasti128 {{.*#+}} ymm7 = [1,3,5,6,1,3,5,6]
 ; AVX512DQ-FAST-NEXT:    # ymm7 = mem[0,1,0,1]
-; AVX512DQ-FAST-NEXT:    vpermd %ymm2, %ymm7, %ymm2
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm2 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,21,24,31]
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5,6],ymm2[7]
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = xmm1[2,9],zero,zero,zero,xmm1[u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %xmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm7 = zero,zero,xmm0[0,7,14,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-FAST-NEXT:    vpermd %ymm18, %ymm7, %ymm7
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm7 = ymm7[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,21,24,31]
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5,6],ymm7[7]
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm7 = xmm0[2,9],zero,zero,zero,xmm0[u,u,u,u,u,u,u,u,u,u,u]
 ; AVX512DQ-FAST-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512DQ-FAST-NEXT:    vpor %xmm3, %xmm7, %xmm3
-; AVX512DQ-FAST-NEXT:    vinserti32x4 $2, %xmm3, %zmm2, %zmm2
-; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm24 = [0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
-; AVX512DQ-FAST-NEXT:    vpternlogq $226, %zmm6, %zmm24, %zmm2
-; AVX512DQ-FAST-NEXT:    vmovdqa %ymm11, %ymm3
-; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm29, %ymm18, %ymm3
-; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm3, %xmm6
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm6 = xmm6[u,u,u,u,u],zero,zero,xmm6[3,10],zero,zero,zero,xmm6[6,13,u,u]
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u,u,u,u,5,12],zero,zero,xmm3[1,8,15],zero,zero,xmm3[u,u]
-; AVX512DQ-FAST-NEXT:    vpor %xmm6, %xmm3, %xmm3
-; AVX512DQ-FAST-NEXT:    vmovdqa %ymm15, %ymm6
-; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm4, %ymm16, %ymm6
-; AVX512DQ-FAST-NEXT:    vpermq {{.*#+}} ymm7 = ymm6[2,3,0,1]
-; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} ymm6 = ymm7[0],ymm6[1,2],ymm7[3],ymm6[4,5,6],ymm7[7,8],ymm6[9,10],ymm7[11],ymm6[12,13,14],ymm7[15]
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm6 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm6[4,11,18,25,16,23,30,21,28,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-FAST-NEXT:    vpternlogq $248, %ymm27, %ymm3, %ymm6
-; AVX512DQ-FAST-NEXT:    vmovdqa %ymm5, %ymm3
-; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm19, %ymm20, %ymm3
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm7 = xmm3[u,u,u,u,u,u,u,3,10],zero,zero,zero,xmm3[6,13],zero,zero
-; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm3, %xmm3
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u,u,u,u,u,u],zero,zero,xmm3[1,8,15],zero,zero,xmm3[4,11]
-; AVX512DQ-FAST-NEXT:    vpor %xmm7, %xmm3, %xmm3
-; AVX512DQ-FAST-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-FAST-NEXT:    vpternlogq $184, %ymm6, %ymm23, %ymm3
-; AVX512DQ-FAST-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm21
-; AVX512DQ-FAST-NEXT:    vpternlogq $184, %zmm2, %zmm28, %zmm21
-; AVX512DQ-FAST-NEXT:    vmovdqa %ymm5, %ymm2
-; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm14, %ymm13, %ymm2
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %xmm22, %xmm1
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %xmm22, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm8 = zero,zero,xmm1[0,7,14,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-FAST-NEXT:    vpor %xmm7, %xmm8, %xmm7
+; AVX512DQ-FAST-NEXT:    vinserti32x4 $2, %xmm7, %zmm2, %zmm22
+; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm18 = [0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
+; AVX512DQ-FAST-NEXT:    vpternlogq $226, %zmm3, %zmm18, %zmm22
+; AVX512DQ-FAST-NEXT:    vmovdqa %ymm6, %ymm2
+; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm11, %ymm26, %ymm2
 ; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = xmm2[3,10],zero,zero,zero,xmm2[6,13],zero,zero,xmm2[u,u,u,u,u,u,u]
 ; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm2, %xmm2
 ; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm2 = zero,zero,xmm2[1,8,15],zero,zero,xmm2[4,11,u,u,u,u,u,u,u]
 ; AVX512DQ-FAST-NEXT:    vpor %xmm3, %xmm2, %xmm2
-; AVX512DQ-FAST-NEXT:    vmovdqa %ymm11, %ymm3
-; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm12, %ymm17, %ymm3
-; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} ymm3 = ymm10[0],ymm3[1,2],ymm10[3],ymm3[4,5,6],ymm10[7,8],ymm3[9,10],ymm10[11],ymm3[12,13,14],ymm10[15]
+; AVX512DQ-FAST-NEXT:    vmovdqa %ymm12, %ymm3
+; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm29, %ymm30, %ymm3
+; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} ymm3 = ymm9[0],ymm3[1,2],ymm9[3],ymm3[4,5,6],ymm9[7,8],ymm3[9,10],ymm9[11],ymm3[12,13,14],ymm9[15]
 ; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm3[2,9,0,7,14,5,12,19,26],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-FAST-NEXT:    vpternlogq $248, %ymm26, %ymm2, %ymm3
-; AVX512DQ-FAST-NEXT:    vmovdqa %ymm15, %ymm2
-; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm31, %ymm30, %ymm2
-; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm2, %xmm6
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm6 = xmm6[u,u],zero,zero,zero,xmm6[6,13],zero,zero,xmm6[2,9,u,u,u,u,u]
+; AVX512DQ-FAST-NEXT:    vpternlogq $248, %ymm17, %ymm2, %ymm3
+; AVX512DQ-FAST-NEXT:    vmovdqa %ymm14, %ymm2
+; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm31, %ymm24, %ymm2
+; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm2, %xmm7
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm7 = xmm7[u,u],zero,zero,zero,xmm7[6,13],zero,zero,xmm7[2,9,u,u,u,u,u]
 ; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,u,1,8,15],zero,zero,xmm2[4,11],zero,zero,xmm2[u,u,u,u,u]
-; AVX512DQ-FAST-NEXT:    vpor %xmm6, %xmm2, %xmm6
-; AVX512DQ-FAST-NEXT:    vmovdqa 208(%rdi), %xmm7
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm8 = xmm7[u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,xmm7[5,12]
-; AVX512DQ-FAST-NEXT:    vmovdqa 192(%rdi), %xmm2
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm9 = xmm2[u,u,u,u,u,u,u,u,u,u,u,0,7,14],zero,zero
-; AVX512DQ-FAST-NEXT:    vpor %xmm8, %xmm9, %xmm8
-; AVX512DQ-FAST-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
+; AVX512DQ-FAST-NEXT:    vpor %xmm7, %xmm2, %xmm2
+; AVX512DQ-FAST-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm7
+; AVX512DQ-FAST-NEXT:    vmovdqa {{.*#+}} ymm2 = <128,128,128,128,128,128,128,128,128,128,128,128,128,128,5,12,19,26,17,24,31,22,29,u,u,u,u,u,u,u,u,u>
+; AVX512DQ-FAST-NEXT:    vmovdqa 208(%rdi), %xmm0
+; AVX512DQ-FAST-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-FAST-NEXT:    vpshufb %xmm2, %xmm0, %xmm8
+; AVX512DQ-FAST-NEXT:    vmovdqa 192(%rdi), %xmm0
+; AVX512DQ-FAST-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm13 = xmm0[u,u,u,u,u,u,u,u,u,u,u,0,7,14],zero,zero
+; AVX512DQ-FAST-NEXT:    vpor %xmm8, %xmm13, %xmm8
 ; AVX512DQ-FAST-NEXT:    vinserti128 $1, %xmm8, %ymm0, %ymm8
-; AVX512DQ-FAST-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm6, %ymm8
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm6 = xmm1[3,10],zero,zero,zero,xmm1[u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm9 = zero,zero,xmm0[1,8,15,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-FAST-NEXT:    vpor %xmm6, %xmm9, %xmm6
-; AVX512DQ-FAST-NEXT:    vinserti32x4 $2, %xmm6, %zmm8, %zmm6
-; AVX512DQ-FAST-NEXT:    vpternlogq $226, %zmm3, %zmm24, %zmm6
-; AVX512DQ-FAST-NEXT:    vmovdqa %ymm5, %ymm3
-; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm29, %ymm18, %ymm3
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm8 = xmm3[u,u,u,u,u,6,13],zero,zero,xmm3[2,9],zero,zero,zero,xmm3[u,u]
+; AVX512DQ-FAST-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm7, %ymm8
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm7 = xmm4[3,10],zero,zero,zero,xmm4[u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm13 = zero,zero,xmm1[1,8,15,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-FAST-NEXT:    vpor %xmm7, %xmm13, %xmm7
+; AVX512DQ-FAST-NEXT:    vmovdqa64 416(%rdi), %ymm24
+; AVX512DQ-FAST-NEXT:    vinserti32x4 $2, %xmm7, %zmm8, %zmm28
+; AVX512DQ-FAST-NEXT:    vmovdqa64 384(%rdi), %ymm25
+; AVX512DQ-FAST-NEXT:    vpternlogq $226, %zmm3, %zmm18, %zmm28
+; AVX512DQ-FAST-NEXT:    vmovdqa %ymm14, %ymm3
+; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm24, %ymm25, %ymm3
+; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm3, %xmm7
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm7 = xmm7[u,u,u,u,u,u,u],zero,zero,zero,xmm7[6,13],zero,zero,xmm7[2,9]
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u,u,u,u,u,u,1,8,15],zero,zero,xmm3[4,11],zero,zero
+; AVX512DQ-FAST-NEXT:    vpor %xmm7, %xmm3, %xmm3
+; AVX512DQ-FAST-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} ymm27 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0]
+; AVX512DQ-FAST-NEXT:    vpternlogq $184, %ymm5, %ymm27, %ymm3
+; AVX512DQ-FAST-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm18
+; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
+; AVX512DQ-FAST-NEXT:    vpternlogq $184, %zmm20, %zmm5, %zmm18
+; AVX512DQ-FAST-NEXT:    vmovdqa %ymm14, %ymm3
+; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm19, %ymm10, %ymm3
+; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm3, %xmm7
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm7 = xmm7[u,u,u,u,u],zero,zero,xmm7[2,9],zero,zero,zero,xmm7[5,12,u,u]
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u,u,u,u,4,11],zero,zero,xmm3[0,7,14],zero,zero,xmm3[u,u]
+; AVX512DQ-FAST-NEXT:    vpor %xmm7, %xmm3, %xmm3
+; AVX512DQ-FAST-NEXT:    vmovdqa %ymm6, %ymm7
+; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm15, %ymm16, %ymm7
+; AVX512DQ-FAST-NEXT:    vpermq {{.*#+}} ymm8 = ymm7[2,3,0,1]
+; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} ymm7 = ymm7[0,1,2],ymm8[3],ymm7[4,5],ymm8[6],ymm7[7,8,9,10],ymm8[11],ymm7[12,13],ymm8[14],ymm7[15]
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm7 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm7[3,10,17,24,31,22,29,20,27,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-FAST-NEXT:    vpternlogq $248, %ymm23, %ymm3, %ymm7
+; AVX512DQ-FAST-NEXT:    vmovdqa %ymm12, %ymm3
+; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm24, %ymm25, %ymm3
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm8 = xmm3[u,u,u,u,u,u,u,2,9],zero,zero,zero,xmm3[5,12],zero,zero
 ; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm3, %xmm3
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u,u,u,u],zero,zero,xmm3[4,11],zero,zero,xmm3[0,7,14,u,u]
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u,u,u,u,u,u],zero,zero,xmm3[0,7,14],zero,zero,xmm3[3,10]
 ; AVX512DQ-FAST-NEXT:    vpor %xmm3, %xmm8, %xmm3
-; AVX512DQ-FAST-NEXT:    vmovdqa %ymm11, %ymm8
-; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm4, %ymm16, %ymm8
-; AVX512DQ-FAST-NEXT:    vpermq {{.*#+}} ymm9 = ymm8[2,3,0,1]
-; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} ymm8 = ymm9[0],ymm8[1,2,3],ymm9[4],ymm8[5,6],ymm9[7,8],ymm8[9,10,11],ymm9[12],ymm8[13,14],ymm9[15]
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm8 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm8[5,12,19,26,17,24,31,22,29,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-FAST-NEXT:    vpternlogq $248, %ymm27, %ymm3, %ymm8
-; AVX512DQ-FAST-NEXT:    vmovdqa %ymm15, %ymm3
-; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm20, %ymm19, %ymm3
-; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm3, %xmm9
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm9 = xmm9[u,u,u,u,u,u,u],zero,zero,xmm9[2,9],zero,zero,zero,xmm9[5,12]
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u,u,u,u,u,u,4,11],zero,zero,xmm3[0,7,14],zero,zero
-; AVX512DQ-FAST-NEXT:    vpor %xmm3, %xmm9, %xmm3
 ; AVX512DQ-FAST-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-FAST-NEXT:    vpternlogq $184, %ymm8, %ymm23, %ymm3
-; AVX512DQ-FAST-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm24
-; AVX512DQ-FAST-NEXT:    vpternlogq $184, %zmm6, %zmm28, %zmm24
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %ymm22, %ymm3
-; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm18, %ymm29, %ymm3
-; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm3, %xmm6
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm6 = xmm6[u,u,u,u],zero,zero,zero,xmm6[5,12],zero,zero,xmm6[1,8,15,u,u]
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u,u,u,0,7,14],zero,zero,xmm3[3,10],zero,zero,zero,xmm3[u,u]
-; AVX512DQ-FAST-NEXT:    vpor %xmm6, %xmm3, %xmm3
-; AVX512DQ-FAST-NEXT:    vmovdqa %ymm5, %ymm6
-; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm4, %ymm16, %ymm6
-; AVX512DQ-FAST-NEXT:    vpermq {{.*#+}} ymm8 = ymm6[2,3,0,1]
-; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} ymm6 = ymm6[0],ymm8[1],ymm6[2,3],ymm8[4],ymm6[5,6,7,8],ymm8[9],ymm6[10,11],ymm8[12],ymm6[13,14,15]
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm6 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm6[6,13,20,27,18,25,16,23,30,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-FAST-NEXT:    vpternlogq $248, %ymm27, %ymm3, %ymm6
-; AVX512DQ-FAST-NEXT:    vmovdqa %ymm11, %ymm3
-; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm20, %ymm19, %ymm3
-; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm3, %xmm8
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm8 = xmm8[u,u,u,u,u,u,u],zero,zero,xmm8[3,10],zero,zero,zero,xmm8[6,13]
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u,u,u,u,u,u,5,12],zero,zero,xmm3[1,8,15],zero,zero
-; AVX512DQ-FAST-NEXT:    vpor %xmm3, %xmm8, %xmm3
-; AVX512DQ-FAST-NEXT:    vinserti32x4 $1, %xmm3, %ymm0, %ymm27
-; AVX512DQ-FAST-NEXT:    vpternlogq $184, %ymm6, %ymm23, %ymm27
-; AVX512DQ-FAST-NEXT:    vmovdqa %ymm15, %ymm3
-; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm13, %ymm14, %ymm3
-; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm3, %xmm6
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm6 = zero,zero,xmm6[2,9],zero,zero,zero,xmm6[5,12,u,u,u,u,u,u,u]
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[4,11],zero,zero,xmm3[0,7,14],zero,zero,xmm3[u,u,u,u,u,u,u]
-; AVX512DQ-FAST-NEXT:    vpor %xmm6, %xmm3, %xmm1
-; AVX512DQ-FAST-NEXT:    vmovdqa %ymm15, %ymm3
-; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm18, %ymm29, %ymm3
-; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm3, %xmm6
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm6 = xmm6[u,u,u,u],zero,zero,zero,xmm6[6,13],zero,zero,xmm6[2,9,u,u,u]
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u,u,u,1,8,15],zero,zero,xmm3[4,11],zero,zero,xmm3[u,u,u]
-; AVX512DQ-FAST-NEXT:    vpor %xmm6, %xmm3, %xmm3
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %ymm22, %ymm6
-; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm16, %ymm4, %ymm6
-; AVX512DQ-FAST-NEXT:    vpermq {{.*#+}} ymm8 = ymm6[2,3,0,1]
-; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} ymm6 = ymm6[0],ymm8[1],ymm6[2,3,4],ymm8[5],ymm6[6,7,8],ymm8[9],ymm6[10,11,12],ymm8[13],ymm6[14,15]
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm6 = ymm6[u,u,u,u],zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm6[0,7,14,21,28,19,26,17,24,31,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-FAST-NEXT:    vpternlogq $244, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm6
-; AVX512DQ-FAST-NEXT:    vmovdqa %ymm5, %ymm3
-; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm20, %ymm19, %ymm3
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm8 = xmm3[u,u,u,u,u,u,u,6,13],zero,zero,xmm3[2,9],zero,zero,zero
+; AVX512DQ-FAST-NEXT:    vpternlogq $184, %ymm7, %ymm27, %ymm3
+; AVX512DQ-FAST-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm20
+; AVX512DQ-FAST-NEXT:    vpternlogq $184, %zmm21, %zmm5, %zmm20
+; AVX512DQ-FAST-NEXT:    vmovdqa %ymm12, %ymm3
+; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm19, %ymm10, %ymm3
+; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm3, %xmm7
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm7 = xmm7[u,u,u,u,u],zero,zero,xmm7[3,10],zero,zero,zero,xmm7[6,13,u,u]
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u,u,u,u,5,12],zero,zero,xmm3[1,8,15],zero,zero,xmm3[u,u]
+; AVX512DQ-FAST-NEXT:    vpor %xmm7, %xmm3, %xmm3
+; AVX512DQ-FAST-NEXT:    vmovdqa %ymm14, %ymm7
+; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm16, %ymm15, %ymm7
+; AVX512DQ-FAST-NEXT:    vpermq {{.*#+}} ymm8 = ymm7[2,3,0,1]
+; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} ymm7 = ymm8[0],ymm7[1,2],ymm8[3],ymm7[4,5,6],ymm8[7,8],ymm7[9,10],ymm8[11],ymm7[12,13,14],ymm8[15]
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm7 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm7[4,11,18,25,16,23,30,21,28,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-FAST-NEXT:    vpternlogq $248, %ymm23, %ymm3, %ymm7
+; AVX512DQ-FAST-NEXT:    vmovdqa %ymm6, %ymm3
+; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm24, %ymm25, %ymm3
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm8 = xmm3[u,u,u,u,u,u,u,3,10],zero,zero,zero,xmm3[6,13],zero,zero
 ; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm3, %xmm3
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u,u,u,u,u,u],zero,zero,xmm3[4,11],zero,zero,xmm3[0,7,14]
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u,u,u,u,u,u],zero,zero,xmm3[1,8,15],zero,zero,xmm3[4,11]
 ; AVX512DQ-FAST-NEXT:    vpor %xmm3, %xmm8, %xmm3
-; AVX512DQ-FAST-NEXT:    vinserti32x4 $1, %xmm3, %ymm0, %ymm28
-; AVX512DQ-FAST-NEXT:    vpternlogq $184, %ymm6, %ymm23, %ymm28
-; AVX512DQ-FAST-NEXT:    vmovdqa %ymm11, %ymm3
-; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm31, %ymm30, %ymm3
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm6 = xmm3[u,u,2,9],zero,zero,zero,xmm3[5,12],zero,zero,xmm3[u,u,u,u,u]
+; AVX512DQ-FAST-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512DQ-FAST-NEXT:    vpternlogq $184, %ymm7, %ymm27, %ymm3
+; AVX512DQ-FAST-NEXT:    vinserti64x4 $1, %ymm3, %zmm0, %zmm21
+; AVX512DQ-FAST-NEXT:    vpternlogq $184, %zmm22, %zmm5, %zmm21
+; AVX512DQ-FAST-NEXT:    vmovdqa %ymm12, %ymm3
+; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm16, %ymm15, %ymm3
+; AVX512DQ-FAST-NEXT:    vpermq {{.*#+}} ymm7 = ymm3[2,3,0,1]
+; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} ymm3 = ymm7[0],ymm3[1,2,3],ymm7[4],ymm3[5,6],ymm7[7,8],ymm3[9,10,11],ymm7[12],ymm3[13,14],ymm7[15]
+; AVX512DQ-FAST-NEXT:    vpshufb %ymm2, %ymm3, %ymm2
+; AVX512DQ-FAST-NEXT:    vmovdqa %ymm6, %ymm3
+; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm19, %ymm10, %ymm3
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm7 = xmm3[u,u,u,u,u,6,13],zero,zero,xmm3[2,9],zero,zero,zero,xmm3[u,u]
 ; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm3, %xmm3
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u],zero,zero,xmm3[0,7,14],zero,zero,xmm3[3,10,u,u,u,u,u]
-; AVX512DQ-FAST-NEXT:    vporq %xmm6, %xmm3, %xmm23
-; AVX512DQ-FAST-NEXT:    vmovdqa %ymm5, %ymm8
-; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm19, %ymm20, %ymm22
-; AVX512DQ-FAST-NEXT:    vmovdqa %ymm11, %ymm3
-; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm13, %ymm14, %ymm3
-; AVX512DQ-FAST-NEXT:    vmovdqa %ymm5, %ymm6
-; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm13, %ymm14, %ymm5
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm9 = xmm7[u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,xmm7[6,13]
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm13 = xmm2[u,u,u,u,u,u,u,u,u,u,u,1,8,15],zero,zero
-; AVX512DQ-FAST-NEXT:    vpor %xmm9, %xmm13, %xmm9
-; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm12, %ymm17, %ymm8
-; AVX512DQ-FAST-NEXT:    vpternlogq $226, %ymm18, %ymm11, %ymm29
-; AVX512DQ-FAST-NEXT:    vmovdqa %ymm15, %ymm13
-; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm17, %ymm12, %ymm13
-; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm17, %ymm12, %ymm11
-; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm3, %xmm10
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm10 = zero,zero,xmm10[3,10],zero,zero,zero,xmm10[6,13,u,u,u,u,u,u,u]
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[5,12],zero,zero,xmm3[1,8,15],zero,zero,xmm3[u,u,u,u,u,u,u]
-; AVX512DQ-FAST-NEXT:    vpor %xmm3, %xmm10, %xmm3
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u,u,u,u],zero,zero,xmm3[4,11],zero,zero,xmm3[0,7,14,u,u]
+; AVX512DQ-FAST-NEXT:    vpor %xmm7, %xmm3, %xmm3
+; AVX512DQ-FAST-NEXT:    vpternlogq $236, %ymm23, %ymm2, %ymm3
+; AVX512DQ-FAST-NEXT:    vmovdqa %ymm14, %ymm2
+; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm25, %ymm24, %ymm2
+; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm2, %xmm7
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm7 = xmm7[u,u,u,u,u,u,u],zero,zero,xmm7[2,9],zero,zero,zero,xmm7[5,12]
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,u,u,u,u,u,u,4,11],zero,zero,xmm2[0,7,14],zero,zero
+; AVX512DQ-FAST-NEXT:    vpor %xmm7, %xmm2, %xmm2
+; AVX512DQ-FAST-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512DQ-FAST-NEXT:    vpternlogq $184, %ymm3, %ymm27, %ymm2
+; AVX512DQ-FAST-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm22
+; AVX512DQ-FAST-NEXT:    vpternlogq $184, %zmm28, %zmm5, %zmm22
+; AVX512DQ-FAST-NEXT:    vmovdqa %ymm6, %ymm2
+; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm16, %ymm15, %ymm2
+; AVX512DQ-FAST-NEXT:    vpermq {{.*#+}} ymm3 = ymm2[2,3,0,1]
+; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0],ymm3[1],ymm2[2,3],ymm3[4],ymm2[5,6,7,8],ymm3[9],ymm2[10,11],ymm3[12],ymm2[13,14,15]
+; AVX512DQ-FAST-NEXT:    vmovdqa {{.*#+}} ymm0 = [65535,0,65535,65535,65535,0,65535,65535,0,65535,65535,65535,0,65535,65535,0]
+; AVX512DQ-FAST-NEXT:    vmovdqa %ymm0, %ymm3
+; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm10, %ymm19, %ymm3
+; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm3, %xmm5
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm5 = xmm5[u,u,u,u],zero,zero,zero,xmm5[5,12],zero,zero,xmm5[1,8,15,u,u]
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u,u,u,0,7,14],zero,zero,xmm3[3,10],zero,zero,zero,xmm3[u,u]
+; AVX512DQ-FAST-NEXT:    vpor %xmm5, %xmm3, %xmm3
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[6,13,20,27,18,25,16,23,30,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-FAST-NEXT:    vpternlogq $236, %ymm23, %ymm2, %ymm3
+; AVX512DQ-FAST-NEXT:    vmovdqa %ymm12, %ymm2
+; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm25, %ymm24, %ymm2
+; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm2, %xmm5
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm5 = xmm5[u,u,u,u,u,u,u],zero,zero,xmm5[3,10],zero,zero,zero,xmm5[6,13]
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,u,u,u,u,u,u,5,12],zero,zero,xmm2[1,8,15],zero,zero
+; AVX512DQ-FAST-NEXT:    vpor %xmm5, %xmm2, %xmm2
+; AVX512DQ-FAST-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm1
+; AVX512DQ-FAST-NEXT:    vpternlogq $184, %ymm3, %ymm27, %ymm1
+; AVX512DQ-FAST-NEXT:    vmovdqa %ymm0, %ymm2
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %ymm0, %ymm23
+; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm15, %ymm16, %ymm2
+; AVX512DQ-FAST-NEXT:    vpermq {{.*#+}} ymm3 = ymm2[2,3,0,1]
+; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0],ymm3[1],ymm2[2,3,4],ymm3[5],ymm2[6,7,8],ymm3[9],ymm2[10,11,12],ymm3[13],ymm2[14,15]
+; AVX512DQ-FAST-NEXT:    vmovdqa %ymm14, %ymm3
+; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm10, %ymm19, %ymm3
+; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm3, %xmm5
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm5 = xmm5[u,u,u,u],zero,zero,zero,xmm5[6,13],zero,zero,xmm5[2,9,u,u,u]
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u,u,u,1,8,15],zero,zero,xmm3[4,11],zero,zero,xmm3[u,u,u]
+; AVX512DQ-FAST-NEXT:    vpor %xmm5, %xmm3, %xmm3
+; AVX512DQ-FAST-NEXT:    vmovdqa {{.*#+}} ymm5 = <u,u,u,u,128,128,128,128,128,128,128,128,128,0,7,14,21,28,19,26,17,24,31,u,u,u,u,u,u,u,u,u>
+; AVX512DQ-FAST-NEXT:    vpshufb %ymm5, %ymm2, %ymm2
+; AVX512DQ-FAST-NEXT:    vpternlogq $220, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm3
+; AVX512DQ-FAST-NEXT:    vmovdqa %ymm6, %ymm2
+; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm25, %ymm24, %ymm2
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm7 = xmm2[u,u,u,u,u,u,u,6,13],zero,zero,xmm2[2,9],zero,zero,zero
+; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm2, %xmm2
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,u,u,u,u,u,u],zero,zero,xmm2[4,11],zero,zero,xmm2[0,7,14]
+; AVX512DQ-FAST-NEXT:    vpor %xmm7, %xmm2, %xmm2
+; AVX512DQ-FAST-NEXT:    vinserti32x4 $1, %xmm2, %ymm0, %ymm28
+; AVX512DQ-FAST-NEXT:    vpternlogq $184, %ymm3, %ymm27, %ymm28
+; AVX512DQ-FAST-NEXT:    vmovdqa %ymm14, %ymm2
+; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm26, %ymm11, %ymm2
+; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm2, %xmm3
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = zero,zero,xmm3[2,9],zero,zero,zero,xmm3[5,12,u,u,u,u,u,u,u]
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[4,11],zero,zero,xmm2[0,7,14],zero,zero,xmm2[u,u,u,u,u,u,u]
+; AVX512DQ-FAST-NEXT:    vporq %xmm3, %xmm2, %xmm27
+; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm24, %ymm25, %ymm23
+; AVX512DQ-FAST-NEXT:    vmovdqa %ymm12, %ymm2
+; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm26, %ymm11, %ymm2
+; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm2, %xmm3
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = zero,zero,xmm3[3,10],zero,zero,zero,xmm3[6,13,u,u,u,u,u,u,u]
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[5,12],zero,zero,xmm2[1,8,15],zero,zero,xmm2[u,u,u,u,u,u,u]
+; AVX512DQ-FAST-NEXT:    vpor %xmm3, %xmm2, %xmm2
+; AVX512DQ-FAST-NEXT:    vmovdqa %ymm6, %ymm3
+; AVX512DQ-FAST-NEXT:    vmovdqa %ymm6, %ymm13
+; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm26, %ymm11, %ymm6
+; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm29, %ymm30, %ymm3
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm7 = xmm6[6,13],zero,zero,xmm6[2,9],zero,zero,zero,xmm6[u,u,u,u,u,u,u]
+; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm6, %xmm6
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm6 = zero,zero,xmm6[4,11],zero,zero,xmm6[0,7,14,u,u,u,u,u,u,u]
+; AVX512DQ-FAST-NEXT:    vpor %xmm7, %xmm6, %xmm6
+; AVX512DQ-FAST-NEXT:    vmovdqa %ymm12, %ymm8
+; AVX512DQ-FAST-NEXT:    vpternlogq $226, %ymm10, %ymm12, %ymm19
+; AVX512DQ-FAST-NEXT:    vmovdqa %ymm14, %ymm7
+; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm30, %ymm29, %ymm7
+; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm30, %ymm29, %ymm12
+; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} ymm3 = ymm9[0],ymm3[1,2,3],ymm9[4],ymm3[5,6],ymm9[7,8],ymm3[9,10,11],ymm9[12],ymm3[13,14],ymm9[15]
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm3[3,10,1,8,15,6,13,20,27],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512DQ-FAST-NEXT:    vpternlogq $248, %ymm17, %ymm27, %ymm3
+; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} ymm7 = ymm7[0],ymm9[1],ymm7[2,3],ymm9[4],ymm7[5,6,7,8],ymm9[9],ymm7[10,11],ymm9[12],ymm7[13,14,15]
+; AVX512DQ-FAST-NEXT:    vmovdqa %ymm9, %ymm0
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm9 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm7[4,11,2,9,0,7,14,21,28],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512DQ-FAST-NEXT:    vpternlogq $248, %ymm17, %ymm2, %ymm9
+; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} ymm2 = ymm12[0],ymm0[1],ymm12[2,3,4],ymm0[5],ymm12[6,7,8],ymm0[9],ymm12[10,11,12],ymm0[13],ymm12[14,15]
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm7 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm2[5,12,3,10,1,8,15,22,29],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512DQ-FAST-NEXT:    vpternlogq $248, %ymm17, %ymm6, %ymm7
 ; AVX512DQ-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} ymm8 = ymm0[0],ymm8[1,2,3],ymm0[4],ymm8[5,6],ymm0[7,8],ymm8[9,10,11],ymm0[12],ymm8[13,14],ymm0[15]
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm12 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm8[3,10,1,8,15,6,13,20,27],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-FAST-NEXT:    vpternlogq $248, %ymm26, %ymm1, %ymm12
-; AVX512DQ-FAST-NEXT:    vinserti32x4 $1, %xmm23, %ymm0, %ymm8
-; AVX512DQ-FAST-NEXT:    vinserti128 $1, %xmm9, %ymm0, %ymm14
-; AVX512DQ-FAST-NEXT:    vmovdqa {{.*#+}} ymm1 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0]
-; AVX512DQ-FAST-NEXT:    vpternlogq $184, %ymm8, %ymm1, %ymm14
-; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} ymm8 = ymm13[0],ymm0[1],ymm13[2,3],ymm0[4],ymm13[5,6,7,8],ymm0[9],ymm13[10,11],ymm0[12],ymm13[13,14,15]
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm8 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm8[4,11,2,9,0,7,14,21,28],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-FAST-NEXT:    vpternlogq $248, %ymm26, %ymm3, %ymm8
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = xmm5[6,13],zero,zero,xmm5[2,9],zero,zero,zero,xmm5[u,u,u,u,u,u,u]
-; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm5, %xmm5
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm5 = zero,zero,xmm5[4,11],zero,zero,xmm5[0,7,14,u,u,u,u,u,u,u]
-; AVX512DQ-FAST-NEXT:    vpor %xmm3, %xmm5, %xmm3
-; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} ymm5 = ymm11[0],ymm0[1],ymm11[2,3,4],ymm0[5],ymm11[6,7,8],ymm0[9],ymm11[10,11,12],ymm0[13],ymm11[14,15]
-; AVX512DQ-FAST-NEXT:    vpbroadcastd {{.*#+}} xmm10 = [4,11,0,0,4,11,0,0,4,11,0,0,4,11,0,0]
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm9 = zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm5[5,12,3,10,1,8,15,22,29],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-FAST-NEXT:    vpternlogq $248, %ymm26, %ymm3, %ymm9
+; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm31, %ymm0, %ymm8
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm2 = xmm8[u,u,2,9],zero,zero,zero,xmm8[5,12],zero,zero,xmm8[u,u,u,u,u]
+; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm8, %xmm6
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm6 = xmm6[u,u],zero,zero,xmm6[0,7,14],zero,zero,xmm6[3,10,u,u,u,u,u]
+; AVX512DQ-FAST-NEXT:    vpor %xmm2, %xmm6, %xmm2
+; AVX512DQ-FAST-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
+; AVX512DQ-FAST-NEXT:    vmovdqa {{.*#+}} ymm6 = <128,128,128,128,128,128,128,128,128,128,128,128,128,128,6,13,20,27,18,25,16,23,30,u,u,u,u,u,u,u,u,u>
+; AVX512DQ-FAST-NEXT:    vpshufb %xmm6, %xmm4, %xmm6
+; AVX512DQ-FAST-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm8 = xmm12[u,u,u,u,u,u,u,u,u,u,u,1,8,15],zero,zero
+; AVX512DQ-FAST-NEXT:    vpor %xmm6, %xmm8, %xmm6
+; AVX512DQ-FAST-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512DQ-FAST-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm8
+; AVX512DQ-FAST-NEXT:    vmovdqa {{.*#+}} ymm11 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0]
+; AVX512DQ-FAST-NEXT:    vpternlogq $184, %ymm2, %ymm11, %ymm8
+; AVX512DQ-FAST-NEXT:    vpternlogq $226, %ymm15, %ymm14, %ymm16
+; AVX512DQ-FAST-NEXT:    vpbroadcastd {{.*#+}} xmm6 = [4,11,0,0,4,11,0,0,4,11,0,0,4,11,0,0]
+; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm31, %ymm0, %ymm13
+; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm0, %ymm31, %ymm14
 ; AVX512DQ-FAST-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX512DQ-FAST-NEXT:    vpshufb %xmm10, %xmm0, %xmm3
-; AVX512DQ-FAST-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm5 = xmm13[2,9,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-FAST-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]
-; AVX512DQ-FAST-NEXT:    vinserti64x4 $1, %ymm3, %zmm14, %zmm5
-; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
-; AVX512DQ-FAST-NEXT:    vpternlogq $184, %zmm12, %zmm3, %zmm5
-; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm31, %ymm30, %ymm6
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm11 = xmm6[u,u,3,10],zero,zero,zero,xmm6[6,13],zero,zero,xmm6[u,u,u,u,u]
-; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm6, %xmm6
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm6 = xmm6[u,u],zero,zero,xmm6[1,8,15],zero,zero,xmm6[4,11,u,u,u,u,u]
-; AVX512DQ-FAST-NEXT:    vpor %xmm6, %xmm11, %xmm6
-; AVX512DQ-FAST-NEXT:    vpternlogq $226, %ymm16, %ymm15, %ymm4
-; AVX512DQ-FAST-NEXT:    vpternlogq $202, %ymm30, %ymm31, %ymm15
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm11 = xmm2[u,u,u,u,u,u,u,u,u,u,u,2,9],zero,zero,zero
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm12 = xmm7[u,u,u,u,u,u,u,u,u,u,u],zero,zero,xmm7[0,7,14]
-; AVX512DQ-FAST-NEXT:    vpor %xmm11, %xmm12, %xmm11
-; AVX512DQ-FAST-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
+; AVX512DQ-FAST-NEXT:    vpshufb %xmm6, %xmm0, %xmm2
+; AVX512DQ-FAST-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm10 = xmm15[2,9,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-FAST-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm10[0],xmm2[1],xmm10[1],xmm2[2],xmm10[2],xmm2[3],xmm10[3]
+; AVX512DQ-FAST-NEXT:    vinserti64x4 $1, %ymm2, %zmm8, %zmm2
+; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm8 = [65535,65535,65535,65535,65535,65535,65535,65535,65535,0,0,0,0,0,0,0,0,0,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535,65535]
+; AVX512DQ-FAST-NEXT:    vpternlogq $184, %zmm3, %zmm8, %zmm2
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = xmm13[u,u,3,10],zero,zero,zero,xmm13[6,13],zero,zero,xmm13[u,u,u,u,u]
+; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm13, %xmm10
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm10 = xmm10[u,u],zero,zero,xmm10[1,8,15],zero,zero,xmm10[4,11,u,u,u,u,u]
+; AVX512DQ-FAST-NEXT:    vpor %xmm3, %xmm10, %xmm3
+; AVX512DQ-FAST-NEXT:    vpshufb %xmm5, %xmm4, %xmm5
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm10 = xmm12[u,u,u,u,u,u,u,u,u,u,u,2,9],zero,zero,zero
+; AVX512DQ-FAST-NEXT:    vpor %xmm5, %xmm10, %xmm5
+; AVX512DQ-FAST-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512DQ-FAST-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512DQ-FAST-NEXT:    vpternlogq $184, %ymm3, %ymm11, %ymm5
+; AVX512DQ-FAST-NEXT:    vmovdqa %ymm11, %ymm13
+; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm14, %xmm3
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[u,u],zero,zero,xmm3[2,9],zero,zero,zero,xmm3[5,12,u,u,u,u,u]
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm10 = xmm14[u,u,4,11],zero,zero,xmm14[0,7,14],zero,zero,xmm14[u,u,u,u,u]
+; AVX512DQ-FAST-NEXT:    vpor %xmm3, %xmm10, %xmm3
+; AVX512DQ-FAST-NEXT:    vmovdqa {{.*#+}} ymm10 = [128,128,128,128,128,128,128,128,128,128,128,128,128,1,8,15,22,29,20,27,18,25,128,128,128,128,128,128,128,128,128,128]
+; AVX512DQ-FAST-NEXT:    vpshufb %xmm10, %xmm4, %xmm11
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm12 = xmm12[u,u,u,u,u,u,u,u,u,u,u,3,10],zero,zero,zero
+; AVX512DQ-FAST-NEXT:    vpor %xmm12, %xmm11, %xmm11
+; AVX512DQ-FAST-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
 ; AVX512DQ-FAST-NEXT:    vinserti128 $1, %xmm11, %ymm0, %ymm11
-; AVX512DQ-FAST-NEXT:    vpternlogq $184, %ymm6, %ymm1, %ymm11
-; AVX512DQ-FAST-NEXT:    vextracti128 $1, %ymm15, %xmm6
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm6 = xmm6[u,u],zero,zero,xmm6[2,9],zero,zero,zero,xmm6[5,12,u,u,u,u,u]
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm12 = xmm15[u,u,4,11],zero,zero,xmm15[0,7,14],zero,zero,xmm15[u,u,u,u,u]
-; AVX512DQ-FAST-NEXT:    vpor %xmm6, %xmm12, %xmm6
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[u,u,u,u,u,u,u,u,u,u,u,3,10],zero,zero,zero
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm7 = xmm7[u,u,u,u,u,u,u,u,u,u,u],zero,zero,xmm7[1,8,15]
-; AVX512DQ-FAST-NEXT:    vpor %xmm2, %xmm7, %xmm2
-; AVX512DQ-FAST-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
-; AVX512DQ-FAST-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512DQ-FAST-NEXT:    vpternlogq $184, %ymm6, %ymm1, %ymm2
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm6 = xmm13[3,10,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm7 = xmm0[5,12,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-FAST-NEXT:    vpunpcklwd {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3]
-; AVX512DQ-FAST-NEXT:    vinserti64x4 $1, %ymm6, %zmm11, %zmm6
-; AVX512DQ-FAST-NEXT:    vpternlogq $184, %zmm8, %zmm3, %zmm6
-; AVX512DQ-FAST-NEXT:    vpshufb %xmm10, %xmm13, %xmm7
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm8 = xmm0[6,13,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
-; AVX512DQ-FAST-NEXT:    vpunpcklwd {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3]
-; AVX512DQ-FAST-NEXT:    vinserti64x4 $1, %ymm7, %zmm2, %zmm2
-; AVX512DQ-FAST-NEXT:    vpternlogq $184, %zmm9, %zmm3, %zmm2
+; AVX512DQ-FAST-NEXT:    vpternlogq $184, %ymm3, %ymm13, %ymm11
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = xmm15[3,10,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm12 = xmm0[5,12,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-FAST-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm12[0],xmm3[0],xmm12[1],xmm3[1],xmm12[2],xmm3[2],xmm12[3],xmm3[3]
+; AVX512DQ-FAST-NEXT:    vinserti64x4 $1, %ymm3, %zmm5, %zmm3
+; AVX512DQ-FAST-NEXT:    vpternlogq $184, %zmm9, %zmm8, %zmm3
+; AVX512DQ-FAST-NEXT:    vpshufb %xmm6, %xmm15, %xmm5
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm6 = xmm0[6,13,u,u,u,u,u,u,u,u,u,u,u,u,u,u]
+; AVX512DQ-FAST-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
+; AVX512DQ-FAST-NEXT:    vinserti64x4 $1, %ymm5, %zmm11, %zmm5
+; AVX512DQ-FAST-NEXT:    vpternlogq $184, %zmm7, %zmm8, %zmm5
 ; AVX512DQ-FAST-NEXT:    movw $-512, %ax # imm = 0xFE00
 ; AVX512DQ-FAST-NEXT:    kmovw %eax, %k1
-; AVX512DQ-FAST-NEXT:    vinserti32x8 $1, %ymm27, %zmm0, %zmm5 {%k1}
-; AVX512DQ-FAST-NEXT:    vinserti32x8 $1, %ymm28, %zmm0, %zmm6 {%k1}
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %ymm29, %ymm0
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm3 = xmm0[u,u,u,u,2,9],zero,zero,zero,xmm0[5,12],zero,zero,xmm0[u,u,u]
-; AVX512DQ-FAST-NEXT:    vextracti32x4 $1, %ymm29, %xmm1
+; AVX512DQ-FAST-NEXT:    vinserti32x8 $1, %ymm1, %zmm0, %zmm2 {%k1}
+; AVX512DQ-FAST-NEXT:    vinserti32x8 $1, %ymm28, %zmm0, %zmm3 {%k1}
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %ymm16, %ymm0
+; AVX512DQ-FAST-NEXT:    vpermq {{.*#+}} ymm6 = ymm16[2,3,0,1]
+; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} ymm4 = ymm0[0,1],ymm6[2],ymm0[3,4],ymm6[5],ymm0[6,7,8,9],ymm6[10],ymm0[11,12],ymm6[13],ymm0[14,15]
+; AVX512DQ-FAST-NEXT:    vpshufb %ymm10, %ymm4, %ymm4
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %ymm19, %ymm0
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm6 = xmm0[u,u,u,u,2,9],zero,zero,zero,xmm0[5,12],zero,zero,xmm0[u,u,u]
+; AVX512DQ-FAST-NEXT:    vextracti32x4 $1, %ymm19, %xmm1
 ; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[u,u,u,u],zero,zero,xmm1[0,7,14],zero,zero,xmm1[3,10,u,u,u]
-; AVX512DQ-FAST-NEXT:    vpor %xmm3, %xmm1, %xmm1
-; AVX512DQ-FAST-NEXT:    vpermq {{.*#+}} ymm3 = ymm4[2,3,0,1]
-; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} ymm3 = ymm4[0,1],ymm3[2],ymm4[3,4],ymm3[5],ymm4[6,7,8,9],ymm3[10],ymm4[11,12],ymm3[13],ymm4[14,15]
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} ymm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm3[1,8,15,22,29,20,27,18,25],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512DQ-FAST-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm3
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %ymm22, %ymm0
-; AVX512DQ-FAST-NEXT:    vextracti32x4 $1, %ymm22, %xmm1
-; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm1 = xmm1[u,u,u,u,u,u],zero,zero,zero,xmm1[5,12],zero,zero,xmm1[1,8,15]
+; AVX512DQ-FAST-NEXT:    vpor %xmm6, %xmm1, %xmm1
+; AVX512DQ-FAST-NEXT:    vpternlogq $236, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm4, %ymm1
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %ymm23, %ymm0
+; AVX512DQ-FAST-NEXT:    vextracti32x4 $1, %ymm23, %xmm4
+; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[u,u,u,u,u,u],zero,zero,zero,xmm4[5,12],zero,zero,xmm4[1,8,15]
 ; AVX512DQ-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[u,u,u,u,u,u,0,7,14],zero,zero,xmm0[3,10],zero,zero,zero
-; AVX512DQ-FAST-NEXT:    vpor %xmm1, %xmm0, %xmm0
+; AVX512DQ-FAST-NEXT:    vpor %xmm4, %xmm0, %xmm0
 ; AVX512DQ-FAST-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} ymm0 = ymm3[0,1,2],ymm0[3,4,5,6,7],ymm3[8,9,10],ymm0[11,12,13,14,15]
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3],ymm0[4,5,6,7]
-; AVX512DQ-FAST-NEXT:    vinserti32x8 $1, %ymm0, %zmm0, %zmm2 {%k1}
-; AVX512DQ-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovaps %zmm0, (%rsi)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm25, (%rdx)
+; AVX512DQ-FAST-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3,4,5,6,7],ymm1[8,9,10],ymm0[11,12,13,14,15]
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
+; AVX512DQ-FAST-NEXT:    vinserti32x8 $1, %ymm0, %zmm0, %zmm5 {%k1}
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm18, (%rsi)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm20, (%rdx)
 ; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm21, (%rcx)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm24, (%r8)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm5, (%r9)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm22, (%r8)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm2, (%r9)
 ; AVX512DQ-FAST-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm6, (%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm3, (%rax)
 ; AVX512DQ-FAST-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm2, (%rax)
-; AVX512DQ-FAST-NEXT:    popq %rax
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm5, (%rax)
 ; AVX512DQ-FAST-NEXT:    vzeroupper
 ; AVX512DQ-FAST-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-7.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-7.ll
index 0bb8e17e0ee484df705ff390161c2b38ceb3b422..b01532ff900b50ed10feb9581a2b4a84f7f0a78d 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-7.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-7.ll
@@ -5318,289 +5318,288 @@ define void @store_i16_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ;
 ; AVX512F-FAST-LABEL: store_i16_stride7_vf32:
 ; AVX512F-FAST:       # %bb.0:
-; AVX512F-FAST-NEXT:    subq $248, %rsp
+; AVX512F-FAST-NEXT:    subq $232, %rsp
 ; AVX512F-FAST-NEXT:    vmovdqa (%rcx), %ymm1
 ; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} ymm0 = [128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128,128,128,128,128,128,128]
 ; AVX512F-FAST-NEXT:    vpshufb %ymm0, %ymm1, %ymm2
-; AVX512F-FAST-NEXT:    vmovdqa %ymm1, %ymm10
+; AVX512F-FAST-NEXT:    vmovdqa %ymm1, %ymm9
 ; AVX512F-FAST-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512F-FAST-NEXT:    vmovdqa (%rdx), %ymm4
 ; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} ymm1 = <u,u,u,u,12,13,14,15,128,128,u,u,u,u,u,u,u,u,u,u,16,17,128,128,u,u,u,u,u,u,u,u>
 ; AVX512F-FAST-NEXT:    vpshufb %ymm1, %ymm4, %ymm3
-; AVX512F-FAST-NEXT:    vmovdqa64 %ymm4, %ymm18
+; AVX512F-FAST-NEXT:    vmovdqa %ymm4, %ymm11
 ; AVX512F-FAST-NEXT:    vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512F-FAST-NEXT:    vpor %ymm2, %ymm3, %ymm2
 ; AVX512F-FAST-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512F-FAST-NEXT:    vmovdqa (%rsi), %ymm4
-; AVX512F-FAST-NEXT:    vmovdqa 32(%rsi), %ymm11
-; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} ymm3 = [128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128,128,128,128,128,128,128,128,128,128,128]
-; AVX512F-FAST-NEXT:    vpshufb %ymm3, %ymm4, %ymm2
-; AVX512F-FAST-NEXT:    vmovdqa %ymm4, %ymm9
-; AVX512F-FAST-NEXT:    vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512F-FAST-NEXT:    vmovdqa (%rsi), %ymm8
+; AVX512F-FAST-NEXT:    vmovdqa 32(%rsi), %ymm10
+; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} ymm4 = [128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128,128,128,128,128,128,128,128,128,128,128]
+; AVX512F-FAST-NEXT:    vpshufb %ymm4, %ymm8, %ymm2
+; AVX512F-FAST-NEXT:    vmovdqu %ymm8, (%rsp) # 32-byte Spill
 ; AVX512F-FAST-NEXT:    vmovdqa (%rdi), %ymm6
 ; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} ymm5 = <12,13,14,15,128,128,u,u,u,u,u,u,u,u,u,u,16,17,128,128,u,u,u,u,u,u,u,u,16,17,18,19>
-; AVX512F-FAST-NEXT:    vpshufb %ymm5, %ymm6, %ymm4
-; AVX512F-FAST-NEXT:    vmovdqa64 %ymm6, %ymm16
+; AVX512F-FAST-NEXT:    vpshufb %ymm5, %ymm6, %ymm3
+; AVX512F-FAST-NEXT:    vmovdqa64 %ymm6, %ymm18
 ; AVX512F-FAST-NEXT:    vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512F-FAST-NEXT:    vpor %ymm2, %ymm4, %ymm2
+; AVX512F-FAST-NEXT:    vpor %ymm2, %ymm3, %ymm2
 ; AVX512F-FAST-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512F-FAST-NEXT:    vmovdqa (%r9), %ymm13
 ; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} ymm2 = [128,128,128,128,128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128,128,128]
 ; AVX512F-FAST-NEXT:    vpshufb %ymm2, %ymm13, %ymm6
 ; AVX512F-FAST-NEXT:    vmovdqa (%r8), %ymm14
-; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} ymm4 = <u,u,u,u,u,u,u,u,12,13,14,15,128,128,u,u,u,u,u,u,u,u,u,u,16,17,128,128,u,u,u,u>
-; AVX512F-FAST-NEXT:    vpshufb %ymm4, %ymm14, %ymm7
+; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} ymm3 = <u,u,u,u,u,u,u,u,12,13,14,15,128,128,u,u,u,u,u,u,u,u,u,u,16,17,128,128,u,u,u,u>
+; AVX512F-FAST-NEXT:    vpshufb %ymm3, %ymm14, %ymm7
 ; AVX512F-FAST-NEXT:    vporq %ymm6, %ymm7, %ymm25
-; AVX512F-FAST-NEXT:    vpshufb %ymm3, %ymm11, %ymm6
-; AVX512F-FAST-NEXT:    vmovdqa 32(%rdi), %ymm8
-; AVX512F-FAST-NEXT:    vpshufb %ymm5, %ymm8, %ymm5
-; AVX512F-FAST-NEXT:    vpor %ymm6, %ymm5, %ymm3
-; AVX512F-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-FAST-NEXT:    vmovdqa 32(%rcx), %ymm3
-; AVX512F-FAST-NEXT:    vpshufb %ymm0, %ymm3, %ymm0
-; AVX512F-FAST-NEXT:    vmovdqa64 %ymm3, %ymm29
+; AVX512F-FAST-NEXT:    vpshufb %ymm4, %ymm10, %ymm4
+; AVX512F-FAST-NEXT:    vmovdqa 32(%rdi), %ymm6
+; AVX512F-FAST-NEXT:    vpshufb %ymm5, %ymm6, %ymm5
+; AVX512F-FAST-NEXT:    vpor %ymm4, %ymm5, %ymm4
+; AVX512F-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-FAST-NEXT:    vmovdqa 32(%rcx), %ymm4
+; AVX512F-FAST-NEXT:    vpshufb %ymm0, %ymm4, %ymm0
+; AVX512F-FAST-NEXT:    vmovdqa64 %ymm4, %ymm31
 ; AVX512F-FAST-NEXT:    vmovdqa 32(%rdx), %ymm12
 ; AVX512F-FAST-NEXT:    vpshufb %ymm1, %ymm12, %ymm1
 ; AVX512F-FAST-NEXT:    vpor %ymm0, %ymm1, %ymm0
 ; AVX512F-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-FAST-NEXT:    vmovdqa 32(%r9), %ymm15
 ; AVX512F-FAST-NEXT:    vpshufb %ymm2, %ymm15, %ymm0
-; AVX512F-FAST-NEXT:    vmovdqa 32(%r8), %ymm6
-; AVX512F-FAST-NEXT:    vpshufb %ymm4, %ymm6, %ymm1
+; AVX512F-FAST-NEXT:    vmovdqa 32(%r8), %ymm4
+; AVX512F-FAST-NEXT:    vpshufb %ymm3, %ymm4, %ymm1
 ; AVX512F-FAST-NEXT:    vporq %ymm0, %ymm1, %ymm21
 ; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} ymm0 = <u,u,u,u,u,u,u,u,u,u,u,u,14,15,12,13,u,u,u,u,30,31,28,29,u,u,u,u,30,31,28,29>
-; AVX512F-FAST-NEXT:    vpshufb %ymm0, %ymm11, %ymm1
-; AVX512F-FAST-NEXT:    vpshufd {{.*#+}} ymm2 = ymm8[3,3,3,3,7,7,7,7]
+; AVX512F-FAST-NEXT:    vpshufb %ymm0, %ymm10, %ymm1
+; AVX512F-FAST-NEXT:    vpshufd {{.*#+}} ymm2 = ymm6[3,3,3,3,7,7,7,7]
 ; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0,1,2],ymm1[3],ymm2[4,5],ymm1[6],ymm2[7,8,9,10],ymm1[11],ymm2[12,13],ymm1[14],ymm2[15]
-; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} ymm2 = ymm11[10,11,8,9,u,u,u,u,u,u,u,u,u,u,u,u,26,27,24,25,u,u,u,u,26,27,24,25,u,u,u,u]
-; AVX512F-FAST-NEXT:    vmovdqa64 %ymm11, %ymm30
-; AVX512F-FAST-NEXT:    vpshufd {{.*#+}} ymm4 = ymm8[2,2,2,2,6,6,6,6]
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm2 = ymm4[0],ymm2[1],ymm4[2,3],ymm2[4],ymm4[5,6,7,8],ymm2[9],ymm4[10,11],ymm2[12],ymm4[13,14,15]
-; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm24 = <u,2,2,3,10,u,11,u>
-; AVX512F-FAST-NEXT:    vpermi2q %zmm1, %zmm2, %zmm24
-; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} ymm1 = ymm10[14,15,12,13,u,u,u,u,u,u,u,u,u,u,u,u,30,31,28,29,u,u,u,u,30,31,28,29,u,u,u,u]
-; AVX512F-FAST-NEXT:    vpshufd {{.*#+}} ymm2 = ymm18[3,3,3,3,7,7,7,7]
+; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} ymm2 = ymm10[10,11,8,9,u,u,u,u,u,u,u,u,u,u,u,u,26,27,24,25,u,u,u,u,26,27,24,25,u,u,u,u]
+; AVX512F-FAST-NEXT:    vmovdqa64 %ymm10, %ymm29
+; AVX512F-FAST-NEXT:    vpshufd {{.*#+}} ymm3 = ymm6[2,2,2,2,6,6,6,6]
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm2 = ymm3[0],ymm2[1],ymm3[2,3],ymm2[4],ymm3[5,6,7,8],ymm2[9],ymm3[10,11],ymm2[12],ymm3[13,14,15]
+; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm22 = <u,2,2,3,10,u,11,u>
+; AVX512F-FAST-NEXT:    vpermi2q %zmm1, %zmm2, %zmm22
+; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} ymm1 = ymm9[14,15,12,13,u,u,u,u,u,u,u,u,u,u,u,u,30,31,28,29,u,u,u,u,30,31,28,29,u,u,u,u]
+; AVX512F-FAST-NEXT:    vpshufd {{.*#+}} ymm2 = ymm11[3,3,3,3,7,7,7,7]
 ; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm1 = ymm2[0],ymm1[1],ymm2[2,3],ymm1[4],ymm2[5,6,7,8],ymm1[9],ymm2[10,11],ymm1[12],ymm2[13,14,15]
 ; AVX512F-FAST-NEXT:    vmovdqa 32(%rcx), %xmm10
-; AVX512F-FAST-NEXT:    vmovdqa 32(%rdx), %xmm5
-; AVX512F-FAST-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm5[0],xmm10[0],xmm5[1],xmm10[1],xmm5[2],xmm10[2],xmm5[3],xmm10[3]
+; AVX512F-FAST-NEXT:    vmovdqa 32(%rdx), %xmm11
+; AVX512F-FAST-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm11[0],xmm10[0],xmm11[1],xmm10[1],xmm11[2],xmm10[2],xmm11[3],xmm10[3]
 ; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[0,1,2,3,0,1,2,3,6,7,4,5,6,7,4,5]
 ; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm23 = <u,2,2,3,8,u,9,u>
 ; AVX512F-FAST-NEXT:    vpermi2q %zmm2, %zmm1, %zmm23
-; AVX512F-FAST-NEXT:    vpshufb %ymm0, %ymm9, %ymm0
-; AVX512F-FAST-NEXT:    vpshufd {{.*#+}} ymm1 = ymm16[3,3,3,3,7,7,7,7]
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm0 = ymm1[0,1,2],ymm0[3],ymm1[4,5],ymm0[6],ymm1[7,8,9,10],ymm0[11],ymm1[12,13],ymm0[14],ymm1[15]
-; AVX512F-FAST-NEXT:    vmovdqa 32(%rdi), %xmm1
-; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} xmm11 = [0,1,2,3,4,5,4,5,6,7,10,11,8,9,10,11]
-; AVX512F-FAST-NEXT:    vmovdqa 32(%rsi), %xmm2
-; AVX512F-FAST-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
-; AVX512F-FAST-NEXT:    vpshufb %xmm11, %xmm4, %xmm4
-; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm20 = [2,1,3,3,8,8,9,9]
-; AVX512F-FAST-NEXT:    vpermi2q %zmm4, %zmm0, %zmm20
-; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} ymm0 = ymm13[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,26,27,28,29,26,27,28,29,26,27,28,29,30,31,30,31]
-; AVX512F-FAST-NEXT:    vpshufd {{.*#+}} ymm4 = ymm14[3,3,3,3,7,7,7,7]
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm4 = ymm0[0,1],ymm4[2],ymm0[3,4],ymm4[5],ymm0[6,7,8,9],ymm4[10],ymm0[11,12],ymm4[13],ymm0[14,15]
-; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm22 = [2,2,2,3,8,8,8,9]
-; AVX512F-FAST-NEXT:    vmovdqa 32(%r9), %xmm0
-; AVX512F-FAST-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512F-FAST-NEXT:    vmovdqa 32(%r8), %xmm3
-; AVX512F-FAST-NEXT:    vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512F-FAST-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3]
+; AVX512F-FAST-NEXT:    vpshufb %ymm0, %ymm8, %ymm0
+; AVX512F-FAST-NEXT:    vpshufd {{.*#+}} ymm1 = ymm18[3,3,3,3,7,7,7,7]
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm2 = ymm1[0,1,2],ymm0[3],ymm1[4,5],ymm0[6],ymm1[7,8,9,10],ymm0[11],ymm1[12,13],ymm0[14],ymm1[15]
+; AVX512F-FAST-NEXT:    vmovdqa 32(%rdi), %xmm0
+; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} xmm5 = [0,1,2,3,4,5,4,5,6,7,10,11,8,9,10,11]
+; AVX512F-FAST-NEXT:    vmovdqa 32(%rsi), %xmm1
+; AVX512F-FAST-NEXT:    vpunpcklwd {{.*#+}} xmm3 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX512F-FAST-NEXT:    vpshufb %xmm5, %xmm3, %xmm3
+; AVX512F-FAST-NEXT:    vmovdqa64 %xmm5, %xmm30
+; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm19 = [2,1,3,3,8,8,9,9]
+; AVX512F-FAST-NEXT:    vpermi2q %zmm3, %zmm2, %zmm19
+; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} ymm2 = ymm13[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,26,27,28,29,26,27,28,29,26,27,28,29,30,31,30,31]
+; AVX512F-FAST-NEXT:    vpshufd {{.*#+}} ymm3 = ymm14[3,3,3,3,7,7,7,7]
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm3[2],ymm2[3,4],ymm3[5],ymm2[6,7,8,9],ymm3[10],ymm2[11,12],ymm3[13],ymm2[14,15]
+; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm20 = [2,2,2,3,8,8,8,9]
+; AVX512F-FAST-NEXT:    vmovdqa 32(%r9), %xmm3
 ; AVX512F-FAST-NEXT:    vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} xmm0 = [0,1,2,3,6,7,4,5,6,7,4,5,12,13,14,15]
-; AVX512F-FAST-NEXT:    vpshufb %xmm0, %xmm3, %xmm7
-; AVX512F-FAST-NEXT:    vpermi2q %zmm7, %zmm4, %zmm22
-; AVX512F-FAST-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
-; AVX512F-FAST-NEXT:    vprold $16, %xmm2, %xmm2
-; AVX512F-FAST-NEXT:    vpshufd {{.*#+}} xmm1 = xmm1[1,1,2,3]
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2],xmm1[3,4],xmm2[5],xmm1[6,7]
-; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} xmm7 = [4,5,2,3,4,5,6,7,8,9,10,11,10,11,8,9]
-; AVX512F-FAST-NEXT:    vpshufb %xmm7, %xmm4, %xmm2
+; AVX512F-FAST-NEXT:    vmovdqa 32(%r8), %xmm5
+; AVX512F-FAST-NEXT:    vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512F-FAST-NEXT:    vpunpcklwd {{.*#+}} xmm7 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
+; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} xmm3 = [0,1,2,3,6,7,4,5,6,7,4,5,12,13,14,15]
+; AVX512F-FAST-NEXT:    vpshufb %xmm3, %xmm7, %xmm5
+; AVX512F-FAST-NEXT:    vmovdqa64 %xmm7, %xmm24
+; AVX512F-FAST-NEXT:    vpermi2q %zmm5, %zmm2, %zmm20
+; AVX512F-FAST-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX512F-FAST-NEXT:    vprold $16, %xmm1, %xmm1
+; AVX512F-FAST-NEXT:    vpshufd {{.*#+}} xmm0 = xmm0[1,1,2,3]
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2],xmm0[3,4],xmm1[5],xmm0[6,7]
+; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} xmm9 = [4,5,2,3,4,5,6,7,8,9,10,11,10,11,8,9]
+; AVX512F-FAST-NEXT:    vpshufb %xmm9, %xmm2, %xmm1
 ; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm26 = <u,0,u,1,8,8,9,u>
-; AVX512F-FAST-NEXT:    vpermi2q %zmm2, %zmm1, %zmm26
+; AVX512F-FAST-NEXT:    vpermi2q %zmm1, %zmm0, %zmm26
 ; AVX512F-FAST-NEXT:    vmovdqa (%r9), %xmm2
-; AVX512F-FAST-NEXT:    vmovdqa (%r8), %xmm9
-; AVX512F-FAST-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm9[0],xmm2[0],xmm9[1],xmm2[1],xmm9[2],xmm2[2],xmm9[3],xmm2[3]
-; AVX512F-FAST-NEXT:    vpshufb %xmm0, %xmm4, %xmm0
-; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[0,1,2,3,8,9,10,11,14,15,12,13,14,15,12,13]
+; AVX512F-FAST-NEXT:    vmovdqa (%r8), %xmm1
+; AVX512F-FAST-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; AVX512F-FAST-NEXT:    vpshufb %xmm3, %xmm0, %xmm3
+; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,2,3,8,9,10,11,14,15,12,13,14,15,12,13]
 ; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm27 = [0,0,0,1,8,9,9,11]
-; AVX512F-FAST-NEXT:    vpermi2q %zmm4, %zmm0, %zmm27
+; AVX512F-FAST-NEXT:    vpermi2q %zmm0, %zmm3, %zmm27
 ; AVX512F-FAST-NEXT:    vprold $16, %ymm13, %ymm0
-; AVX512F-FAST-NEXT:    vpshufd {{.*#+}} ymm4 = ymm14[1,2,2,3,5,6,6,7]
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm0 = ymm4[0,1],ymm0[2],ymm4[3,4],ymm0[5],ymm4[6,7,8,9],ymm0[10],ymm4[11,12],ymm0[13],ymm4[14,15]
-; AVX512F-FAST-NEXT:    vpbroadcastd {{.*#+}} ymm3 = [18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21]
-; AVX512F-FAST-NEXT:    vpshufb %ymm3, %ymm13, %ymm4
-; AVX512F-FAST-NEXT:    vpshufd {{.*#+}} ymm13 = ymm14[0,0,2,1,4,4,6,5]
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm4 = ymm4[0,1,2],ymm13[3],ymm4[4,5],ymm13[6],ymm4[7,8,9,10],ymm13[11],ymm4[12,13],ymm13[14],ymm4[15]
+; AVX512F-FAST-NEXT:    vpshufd {{.*#+}} ymm3 = ymm14[1,2,2,3,5,6,6,7]
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm0 = ymm3[0,1],ymm0[2],ymm3[3,4],ymm0[5],ymm3[6,7,8,9],ymm0[10],ymm3[11,12],ymm0[13],ymm3[14,15]
+; AVX512F-FAST-NEXT:    vpbroadcastd {{.*#+}} ymm5 = [18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21,18,19,20,21]
+; AVX512F-FAST-NEXT:    vpshufb %ymm5, %ymm13, %ymm3
+; AVX512F-FAST-NEXT:    vpshufd {{.*#+}} ymm7 = ymm14[0,0,2,1,4,4,6,5]
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0,1,2],ymm7[3],ymm3[4,5],ymm7[6],ymm3[7,8,9,10],ymm7[11],ymm3[12,13],ymm7[14],ymm3[15]
 ; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm28 = [2,2,3,3,10,9,11,10]
-; AVX512F-FAST-NEXT:    vpermi2q %zmm0, %zmm4, %zmm28
-; AVX512F-FAST-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512F-FAST-NEXT:    vmovdqa (%rsi), %xmm14
-; AVX512F-FAST-NEXT:    vpunpckhwd {{.*#+}} xmm4 = xmm14[4],xmm0[4],xmm14[5],xmm0[5],xmm14[6],xmm0[6],xmm14[7],xmm0[7]
-; AVX512F-FAST-NEXT:    vpshufb %xmm7, %xmm4, %xmm1
-; AVX512F-FAST-NEXT:    vmovdqa64 %ymm1, %ymm19
-; AVX512F-FAST-NEXT:    vpunpckhwd {{.*#+}} xmm2 = xmm9[4],xmm2[4],xmm9[5],xmm2[5],xmm9[6],xmm2[6],xmm9[7],xmm2[7]
-; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm25, %zmm0, %zmm4
-; AVX512F-FAST-NEXT:    vpshufb %xmm11, %xmm2, %xmm2
+; AVX512F-FAST-NEXT:    vpermi2q %zmm0, %zmm3, %zmm28
+; AVX512F-FAST-NEXT:    vmovdqa (%rdi), %xmm8
+; AVX512F-FAST-NEXT:    vmovdqa (%rsi), %xmm0
+; AVX512F-FAST-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm0[4],xmm8[4],xmm0[5],xmm8[5],xmm0[6],xmm8[6],xmm0[7],xmm8[7]
+; AVX512F-FAST-NEXT:    vpshufb %xmm9, %xmm3, %xmm3
+; AVX512F-FAST-NEXT:    vmovdqa64 %ymm3, %ymm17
+; AVX512F-FAST-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm25, %zmm0, %zmm2
+; AVX512F-FAST-NEXT:    vmovdqa64 %xmm30, %xmm9
+; AVX512F-FAST-NEXT:    vpshufb %xmm9, %xmm1, %xmm1
 ; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm25 = <0,0,1,1,12,13,u,15>
-; AVX512F-FAST-NEXT:    vpermi2q %zmm4, %zmm2, %zmm25
+; AVX512F-FAST-NEXT:    vpermi2q %zmm2, %zmm1, %zmm25
 ; AVX512F-FAST-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512F-FAST-NEXT:    vpbroadcastd 8(%rax), %ymm2
-; AVX512F-FAST-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
-; AVX512F-FAST-NEXT:    vmovdqa (%rax), %ymm9
-; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} ymm1 = [12,13,128,128,128,128,128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128]
-; AVX512F-FAST-NEXT:    vpshufb %ymm1, %ymm9, %ymm4
-; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm4, %zmm2, %zmm31
-; AVX512F-FAST-NEXT:    vmovdqa64 %ymm30, %ymm2
-; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} ymm2 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,19,20,21,18,19,20,21,24,25,26,27,22,23,22,23]
-; AVX512F-FAST-NEXT:    vpshufd {{.*#+}} ymm4 = ymm8[1,1,1,1,5,5,5,5]
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm4[2],ymm2[3,4],ymm4[5],ymm2[6,7,8,9],ymm4[10],ymm2[11,12],ymm4[13],ymm2[14,15]
-; AVX512F-FAST-NEXT:    vmovdqa64 %ymm2, %ymm18
-; AVX512F-FAST-NEXT:    vmovdqa64 %ymm29, %ymm7
-; AVX512F-FAST-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm7[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11]
-; AVX512F-FAST-NEXT:    vpshufd {{.*#+}} ymm4 = ymm12[0,1,1,3,4,5,5,7]
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm2 = ymm4[0,1],ymm2[2],ymm4[3,4],ymm2[5],ymm4[6,7,8,9],ymm2[10],ymm4[11,12],ymm2[13],ymm4[14,15]
-; AVX512F-FAST-NEXT:    vmovdqa64 %ymm2, %ymm17
-; AVX512F-FAST-NEXT:    vpshufb %ymm3, %ymm15, %ymm2
-; AVX512F-FAST-NEXT:    vpshufd {{.*#+}} ymm3 = ymm6[0,0,2,1,4,4,6,5]
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1,2],ymm3[3],ymm2[4,5],ymm3[6],ymm2[7,8,9,10],ymm3[11],ymm2[12,13],ymm3[14],ymm2[15]
+; AVX512F-FAST-NEXT:    vpbroadcastd 8(%rax), %ymm1
+; AVX512F-FAST-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX512F-FAST-NEXT:    vmovdqa (%rax), %ymm3
+; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} ymm2 = [12,13,128,128,128,128,128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,128,128,128,128,16,17,128,128]
+; AVX512F-FAST-NEXT:    vpshufb %ymm2, %ymm3, %ymm7
+; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm7, %zmm1, %zmm30
+; AVX512F-FAST-NEXT:    vmovdqa64 %ymm29, %ymm1
+; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,19,20,21,18,19,20,21,24,25,26,27,22,23,22,23]
+; AVX512F-FAST-NEXT:    vpshufd {{.*#+}} ymm6 = ymm6[1,1,1,1,5,5,5,5]
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0,1],ymm6[2],ymm1[3,4],ymm6[5],ymm1[6,7,8,9],ymm6[10],ymm1[11,12],ymm6[13],ymm1[14,15]
+; AVX512F-FAST-NEXT:    vmovdqa64 %ymm1, %ymm16
+; AVX512F-FAST-NEXT:    vmovdqa64 %ymm31, %ymm13
+; AVX512F-FAST-NEXT:    vpunpcklwd {{.*#+}} ymm1 = ymm13[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11]
+; AVX512F-FAST-NEXT:    vpshufd {{.*#+}} ymm6 = ymm12[0,1,1,3,4,5,5,7]
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm7 = ymm6[0,1],ymm1[2],ymm6[3,4],ymm1[5],ymm6[6,7,8,9],ymm1[10],ymm6[11,12],ymm1[13],ymm6[14,15]
+; AVX512F-FAST-NEXT:    vpshufb %ymm5, %ymm15, %ymm1
+; AVX512F-FAST-NEXT:    vpshufd {{.*#+}} ymm5 = ymm4[0,0,2,1,4,4,6,5]
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm1 = ymm1[0,1,2],ymm5[3],ymm1[4,5],ymm5[6],ymm1[7,8,9,10],ymm5[11],ymm1[12,13],ymm5[14],ymm1[15]
 ; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm29 = <0,1,u,3,10,10,11,11>
-; AVX512F-FAST-NEXT:    vpermi2q %zmm2, %zmm21, %zmm29
-; AVX512F-FAST-NEXT:    vmovdqa 32(%rax), %ymm8
-; AVX512F-FAST-NEXT:    vpshufb %ymm1, %ymm8, %ymm1
-; AVX512F-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm21 = [0,13,4,0,0,14,5,0,0,13,4,0,0,14,5,0]
-; AVX512F-FAST-NEXT:    # zmm21 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-FAST-NEXT:    vpermd %ymm8, %ymm21, %ymm2
+; AVX512F-FAST-NEXT:    vpermi2q %zmm1, %zmm21, %zmm29
+; AVX512F-FAST-NEXT:    vmovdqa 32(%rax), %ymm6
+; AVX512F-FAST-NEXT:    vpshufb %ymm2, %ymm6, %ymm1
+; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} ymm2 = [0,1,4,5,4,5,5,7]
+; AVX512F-FAST-NEXT:    vpermd %ymm6, %ymm2, %ymm2
 ; AVX512F-FAST-NEXT:    vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
-; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm16
-; AVX512F-FAST-NEXT:    vmovdqa %ymm7, %ymm4
-; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} ymm2 = ymm7[14,15,12,13,u,u,u,u,u,u,u,u,u,u,u,u,30,31,28,29,u,u,u,u,30,31,28,29,u,u,u,u]
-; AVX512F-FAST-NEXT:    vpshufd {{.*#+}} ymm3 = ymm12[3,3,3,3,7,7,7,7]
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0],ymm2[1],ymm3[2,3],ymm2[4],ymm3[5,6,7,8],ymm2[9],ymm3[10,11],ymm2[12],ymm3[13,14,15]
-; AVX512F-FAST-NEXT:    vbroadcasti128 {{.*#+}} ymm7 = [22,23,26,27,0,0,24,25,26,27,0,0,26,27,26,27,22,23,26,27,0,0,24,25,26,27,0,0,26,27,26,27]
-; AVX512F-FAST-NEXT:    # ymm7 = mem[0,1,0,1]
-; AVX512F-FAST-NEXT:    vpshufb %ymm7, %ymm4, %ymm13
+; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm14
+; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} ymm2 = ymm13[14,15,12,13,u,u,u,u,u,u,u,u,u,u,u,u,30,31,28,29,u,u,u,u,30,31,28,29,u,u,u,u]
+; AVX512F-FAST-NEXT:    vpshufd {{.*#+}} ymm5 = ymm12[3,3,3,3,7,7,7,7]
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm2 = ymm5[0],ymm2[1],ymm5[2,3],ymm2[4],ymm5[5,6,7,8],ymm2[9],ymm5[10,11],ymm2[12],ymm5[13,14,15]
+; AVX512F-FAST-NEXT:    vbroadcasti128 {{.*#+}} ymm5 = [22,23,26,27,0,0,24,25,26,27,0,0,26,27,26,27,22,23,26,27,0,0,24,25,26,27,0,0,26,27,26,27]
+; AVX512F-FAST-NEXT:    # ymm5 = mem[0,1,0,1]
+; AVX512F-FAST-NEXT:    vpshufb %ymm5, %ymm13, %ymm13
 ; AVX512F-FAST-NEXT:    vpshufd {{.*#+}} ymm12 = ymm12[2,2,2,2,6,6,6,6]
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm12 = ymm13[0,1],ymm12[2],ymm13[3,4],ymm12[5],ymm13[6,7,8,9],ymm12[10],ymm13[11,12],ymm12[13],ymm13[14,15]
-; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm30 = [2,2,2,3,8,10,10,11]
-; AVX512F-FAST-NEXT:    vpermt2q %zmm3, %zmm30, %zmm12
-; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} ymm3 = ymm15[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,26,27,28,29,26,27,28,29,26,27,28,29,30,31,30,31]
-; AVX512F-FAST-NEXT:    vpshufd {{.*#+}} ymm13 = ymm6[3,3,3,3,7,7,7,7]
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0,1],ymm13[2],ymm3[3,4],ymm13[5],ymm3[6,7,8,9],ymm13[10],ymm3[11,12],ymm13[13],ymm3[14,15]
-; AVX512F-FAST-NEXT:    vprold $16, %ymm15, %ymm13
-; AVX512F-FAST-NEXT:    vpshufd {{.*#+}} ymm6 = ymm6[1,2,2,3,5,6,6,7]
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm6 = ymm6[0,1],ymm13[2],ymm6[3,4],ymm13[5],ymm6[6,7,8,9],ymm13[10],ymm6[11,12],ymm13[13],ymm6[14,15]
-; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm13 = [2,1,3,2,10,10,10,11]
-; AVX512F-FAST-NEXT:    vpermt2q %zmm3, %zmm13, %zmm6
-; AVX512F-FAST-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm5[4],xmm10[4],xmm5[5],xmm10[5],xmm5[6],xmm10[6],xmm5[7],xmm10[7]
-; AVX512F-FAST-NEXT:    vpbroadcastq {{.*#+}} xmm15 = [6,7,4,5,0,0,8,9,6,7,4,5,0,0,8,9]
-; AVX512F-FAST-NEXT:    vpshufb %xmm15, %xmm10, %xmm10
-; AVX512F-FAST-NEXT:    vpshufd {{.*#+}} xmm5 = xmm5[1,1,2,2]
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} xmm5 = xmm5[0],xmm10[1],xmm5[2,3],xmm10[4],xmm5[5,6],xmm10[7]
-; AVX512F-FAST-NEXT:    vpshuflw {{.*#+}} xmm3 = xmm3[0,2,3,3,4,5,6,7]
-; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [0,0,1,1,8,8,10,9]
-; AVX512F-FAST-NEXT:    vpermt2q %zmm3, %zmm2, %zmm5
-; AVX512F-FAST-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; AVX512F-FAST-NEXT:    vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm3 # 16-byte Folded Reload
-; AVX512F-FAST-NEXT:    # xmm3 = xmm1[4],mem[4],xmm1[5],mem[5],xmm1[6],mem[6],xmm1[7],mem[7]
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm13 = ymm13[0,1],ymm12[2],ymm13[3,4],ymm12[5],ymm13[6,7,8,9],ymm12[10],ymm13[11,12],ymm12[13],ymm13[14,15]
+; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm21 = [2,2,2,3,8,10,10,11]
+; AVX512F-FAST-NEXT:    vpermt2q %zmm2, %zmm21, %zmm13
+; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} ymm2 = ymm15[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,26,27,28,29,26,27,28,29,26,27,28,29,30,31,30,31]
+; AVX512F-FAST-NEXT:    vpshufd {{.*#+}} ymm12 = ymm4[3,3,3,3,7,7,7,7]
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm2 = ymm2[0,1],ymm12[2],ymm2[3,4],ymm12[5],ymm2[6,7,8,9],ymm12[10],ymm2[11,12],ymm12[13],ymm2[14,15]
+; AVX512F-FAST-NEXT:    vprold $16, %ymm15, %ymm12
+; AVX512F-FAST-NEXT:    vpshufd {{.*#+}} ymm4 = ymm4[1,2,2,3,5,6,6,7]
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm12 = ymm4[0,1],ymm12[2],ymm4[3,4],ymm12[5],ymm4[6,7,8,9],ymm12[10],ymm4[11,12],ymm12[13],ymm4[14,15]
+; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm31 = [2,1,3,2,10,10,10,11]
+; AVX512F-FAST-NEXT:    vpermt2q %zmm2, %zmm31, %zmm12
+; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm18 = [65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535]
+; AVX512F-FAST-NEXT:    vpternlogq $226, %zmm22, %zmm18, %zmm13
+; AVX512F-FAST-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm13, %zmm12
+; AVX512F-FAST-NEXT:    vmovdqa64 (%rax), %zmm15
+; AVX512F-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [30,5,0,0,31,6,0,31,30,5,0,0,31,6,0,31]
+; AVX512F-FAST-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-FAST-NEXT:    vpermi2d %zmm15, %zmm6, %zmm4
+; AVX512F-FAST-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm12, %zmm4
+; AVX512F-FAST-NEXT:    vpunpckhwd {{.*#+}} xmm6 = xmm11[4],xmm10[4],xmm11[5],xmm10[5],xmm11[6],xmm10[6],xmm11[7],xmm10[7]
+; AVX512F-FAST-NEXT:    vpbroadcastq {{.*#+}} xmm12 = [6,7,4,5,0,0,8,9,6,7,4,5,0,0,8,9]
+; AVX512F-FAST-NEXT:    vpshufb %xmm12, %xmm10, %xmm10
+; AVX512F-FAST-NEXT:    vpshufd {{.*#+}} xmm11 = xmm11[1,1,2,2]
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} xmm10 = xmm11[0],xmm10[1],xmm11[2,3],xmm10[4],xmm11[5,6],xmm10[7]
+; AVX512F-FAST-NEXT:    vpshuflw {{.*#+}} xmm6 = xmm6[0,2,3,3,4,5,6,7]
+; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm11 = [0,0,1,1,8,8,10,9]
+; AVX512F-FAST-NEXT:    vpermt2q %zmm6, %zmm11, %zmm10
 ; AVX512F-FAST-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} xmm10 = xmm1[0,1,2,3,8,9,10,11,14,15,12,13,14,15,12,13]
-; AVX512F-FAST-NEXT:    vmovdqa %xmm11, %xmm1
-; AVX512F-FAST-NEXT:    vpshufb %xmm11, %xmm3, %xmm3
-; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [0,1,1,3,8,8,9,9]
-; AVX512F-FAST-NEXT:    vpermt2q %zmm3, %zmm4, %zmm10
-; AVX512F-FAST-NEXT:    vprold $16, %xmm14, %xmm3
-; AVX512F-FAST-NEXT:    vpshufd {{.*#+}} xmm11 = xmm0[1,1,2,3]
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} xmm3 = xmm11[0,1],xmm3[2],xmm11[3,4],xmm3[5],xmm11[6,7]
-; AVX512F-FAST-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3]
-; AVX512F-FAST-NEXT:    vpshufb %xmm1, %xmm0, %xmm11
-; AVX512F-FAST-NEXT:    vpermt2q %zmm3, %zmm2, %zmm11
-; AVX512F-FAST-NEXT:    vmovdqa (%rcx), %xmm0
-; AVX512F-FAST-NEXT:    vpshufb %xmm15, %xmm0, %xmm2
-; AVX512F-FAST-NEXT:    vmovdqa (%rdx), %xmm3
-; AVX512F-FAST-NEXT:    vpshufd {{.*#+}} xmm14 = xmm3[1,1,2,2]
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} xmm2 = xmm14[0],xmm2[1],xmm14[2,3],xmm2[4],xmm14[5,6],xmm2[7]
-; AVX512F-FAST-NEXT:    vpunpckhwd {{.*#+}} xmm14 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
-; AVX512F-FAST-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3]
-; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[0,1,2,3,0,1,2,3,6,7,4,5,6,7,4,5]
-; AVX512F-FAST-NEXT:    vpermt2q %zmm2, %zmm4, %zmm0
-; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535]
-; AVX512F-FAST-NEXT:    vpternlogq $226, %zmm24, %zmm2, %zmm12
-; AVX512F-FAST-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm12, %zmm6
-; AVX512F-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload
-; AVX512F-FAST-NEXT:    vpshufb %ymm7, %ymm12, %ymm3
-; AVX512F-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX512F-FAST-NEXT:    vpshufd {{.*#+}} ymm4 = ymm1[2,2,2,2,6,6,6,6]
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm3 = ymm3[0,1],ymm4[2],ymm3[3,4],ymm4[5],ymm3[6,7,8,9],ymm4[10],ymm3[11,12],ymm4[13],ymm3[14,15]
-; AVX512F-FAST-NEXT:    vpunpcklwd {{.*#+}} ymm4 = ymm12[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11]
-; AVX512F-FAST-NEXT:    vpshufd {{.*#+}} ymm7 = ymm1[0,1,1,3,4,5,5,7]
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm4 = ymm7[0,1],ymm4[2],ymm7[3,4],ymm4[5],ymm7[6,7,8,9],ymm4[10],ymm7[11,12],ymm4[13],ymm7[14,15]
-; AVX512F-FAST-NEXT:    vpermt2q %zmm3, %zmm13, %zmm4
-; AVX512F-FAST-NEXT:    vmovdqa64 (%rax), %zmm3
-; AVX512F-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm7 = [30,5,0,0,31,6,0,31,30,5,0,0,31,6,0,31]
-; AVX512F-FAST-NEXT:    # zmm7 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-FAST-NEXT:    vpermi2d %zmm3, %zmm8, %zmm7
-; AVX512F-FAST-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm6, %zmm7
+; AVX512F-FAST-NEXT:    vpunpckhwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm6 # 16-byte Folded Reload
+; AVX512F-FAST-NEXT:    # xmm6 = xmm1[4],mem[4],xmm1[5],mem[5],xmm1[6],mem[6],xmm1[7],mem[7]
+; AVX512F-FAST-NEXT:    vmovdqa64 %xmm24, %xmm1
+; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} xmm13 = xmm1[0,1,2,3,8,9,10,11,14,15,12,13,14,15,12,13]
+; AVX512F-FAST-NEXT:    vpshufb %xmm9, %xmm6, %xmm6
+; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm22 = [0,1,1,3,8,8,9,9]
+; AVX512F-FAST-NEXT:    vpermt2q %zmm6, %zmm22, %zmm13
+; AVX512F-FAST-NEXT:    vprold $16, %xmm0, %xmm6
+; AVX512F-FAST-NEXT:    vpshufd {{.*#+}} xmm2 = xmm8[1,1,2,3]
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} xmm2 = xmm2[0,1],xmm6[2],xmm2[3,4],xmm6[5],xmm2[6,7]
+; AVX512F-FAST-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm8[0],xmm0[0],xmm8[1],xmm0[1],xmm8[2],xmm0[2],xmm8[3],xmm0[3]
+; AVX512F-FAST-NEXT:    vpshufb %xmm9, %xmm0, %xmm0
+; AVX512F-FAST-NEXT:    vpermt2q %zmm2, %zmm11, %zmm0
+; AVX512F-FAST-NEXT:    vmovdqa (%rcx), %xmm2
+; AVX512F-FAST-NEXT:    vpshufb %xmm12, %xmm2, %xmm6
+; AVX512F-FAST-NEXT:    vmovdqa (%rdx), %xmm11
+; AVX512F-FAST-NEXT:    vpshufd {{.*#+}} xmm12 = xmm11[1,1,2,2]
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} xmm6 = xmm12[0],xmm6[1],xmm12[2,3],xmm6[4],xmm12[5,6],xmm6[7]
+; AVX512F-FAST-NEXT:    vpunpckhwd {{.*#+}} xmm12 = xmm11[4],xmm2[4],xmm11[5],xmm2[5],xmm11[6],xmm2[6],xmm11[7],xmm2[7]
+; AVX512F-FAST-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm11[0],xmm2[0],xmm11[1],xmm2[1],xmm11[2],xmm2[2],xmm11[3],xmm2[3]
+; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} xmm2 = xmm2[0,1,2,3,0,1,2,3,6,7,4,5,6,7,4,5]
+; AVX512F-FAST-NEXT:    vpermt2q %zmm6, %zmm22, %zmm2
 ; AVX512F-FAST-NEXT:    vpbroadcastd 36(%rax), %ymm6
-; AVX512F-FAST-NEXT:    vpbroadcastd 40(%rax), %ymm8
-; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm8, %zmm6, %zmm6
-; AVX512F-FAST-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm10, %zmm6
-; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm8 = [65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535]
-; AVX512F-FAST-NEXT:    vpternlogq $226, %zmm26, %zmm8, %zmm5
-; AVX512F-FAST-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm6
-; AVX512F-FAST-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm11, %zmm0
-; AVX512F-FAST-NEXT:    vpbroadcastd (%rax), %ymm5
-; AVX512F-FAST-NEXT:    vpbroadcastd 4(%rax), %ymm10
-; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm10, %zmm5, %zmm5
-; AVX512F-FAST-NEXT:    vpshuflw {{.*#+}} xmm10 = xmm14[0,2,3,3,4,5,6,7]
+; AVX512F-FAST-NEXT:    vpbroadcastd 40(%rax), %ymm11
+; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm11, %zmm6, %zmm6
+; AVX512F-FAST-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm13, %zmm6
+; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm11 = [65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535]
+; AVX512F-FAST-NEXT:    vpternlogq $226, %zmm26, %zmm11, %zmm10
+; AVX512F-FAST-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm10, %zmm6
+; AVX512F-FAST-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm2
+; AVX512F-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload
+; AVX512F-FAST-NEXT:    vpshufb %ymm5, %ymm10, %ymm0
+; AVX512F-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX512F-FAST-NEXT:    vpshufd {{.*#+}} ymm5 = ymm1[2,2,2,2,6,6,6,6]
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0,1],ymm5[2],ymm0[3,4],ymm5[5],ymm0[6,7,8,9],ymm5[10],ymm0[11,12],ymm5[13],ymm0[14,15]
+; AVX512F-FAST-NEXT:    vpunpcklwd {{.*#+}} ymm5 = ymm10[0,0,1,1,2,2,3,3,8,8,9,9,10,10,11,11]
+; AVX512F-FAST-NEXT:    vpshufd {{.*#+}} ymm10 = ymm1[0,1,1,3,4,5,5,7]
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm5 = ymm10[0,1],ymm5[2],ymm10[3,4],ymm5[5],ymm10[6,7,8,9],ymm5[10],ymm10[11,12],ymm5[13],ymm10[14,15]
+; AVX512F-FAST-NEXT:    vpshuflw {{.*#+}} xmm10 = xmm12[0,2,3,3,4,5,6,7]
 ; AVX512F-FAST-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[0,0,2,1]
-; AVX512F-FAST-NEXT:    vpermq {{.*#+}} ymm11 = ymm19[0,0,1,3]
-; AVX512F-FAST-NEXT:    vpermq {{.*#+}} ymm12 = ymm18[2,2,2,3]
-; AVX512F-FAST-NEXT:    vpermq {{.*#+}} ymm13 = ymm17[2,1,3,2]
-; AVX512F-FAST-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm27, %zmm5
-; AVX512F-FAST-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm5
-; AVX512F-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload
-; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} ymm0 = ymm15[10,11,8,9,u,u,u,u,u,u,u,u,u,u,u,u,26,27,24,25,u,u,u,u,26,27,24,25,u,u,u,u]
+; AVX512F-FAST-NEXT:    vpermq {{.*#+}} ymm9 = ymm17[0,0,1,3]
+; AVX512F-FAST-NEXT:    vpermq {{.*#+}} ymm8 = ymm16[2,2,2,3]
+; AVX512F-FAST-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[2,1,3,2]
+; AVX512F-FAST-NEXT:    vpermt2q %zmm0, %zmm31, %zmm5
+; AVX512F-FAST-NEXT:    vpbroadcastd (%rax), %ymm0
+; AVX512F-FAST-NEXT:    vpbroadcastd 4(%rax), %ymm12
+; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm12, %zmm0, %zmm0
+; AVX512F-FAST-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm27, %zmm0
+; AVX512F-FAST-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm0
+; AVX512F-FAST-NEXT:    vmovdqu (%rsp), %ymm13 # 32-byte Reload
+; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} ymm2 = ymm13[10,11,8,9,u,u,u,u,u,u,u,u,u,u,u,u,26,27,24,25,u,u,u,u,26,27,24,25,u,u,u,u]
 ; AVX512F-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX512F-FAST-NEXT:    vpshufd {{.*#+}} ymm14 = ymm1[2,2,2,2,6,6,6,6]
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm0 = ymm14[0],ymm0[1],ymm14[2,3],ymm0[4],ymm14[5,6,7,8],ymm0[9],ymm14[10,11],ymm0[12],ymm14[13,14,15]
-; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} ymm14 = ymm15[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,19,20,21,18,19,20,21,24,25,26,27,22,23,22,23]
-; AVX512F-FAST-NEXT:    vpshufd {{.*#+}} ymm15 = ymm1[1,1,1,1,5,5,5,5]
-; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm14 = ymm14[0,1],ymm15[2],ymm14[3,4],ymm15[5],ymm14[6,7,8,9],ymm15[10],ymm14[11,12],ymm15[13],ymm14[14,15]
-; AVX512F-FAST-NEXT:    vpermt2q %zmm0, %zmm30, %zmm14
-; AVX512F-FAST-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm4, %zmm14
-; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} ymm0 = <6,u,u,u,7,u,u,7>
-; AVX512F-FAST-NEXT:    vpermd %ymm9, %ymm0, %ymm0
-; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm9, %zmm3, %zmm3
-; AVX512F-FAST-NEXT:    vpermd %zmm3, %zmm21, %zmm3
+; AVX512F-FAST-NEXT:    vpshufd {{.*#+}} ymm12 = ymm1[2,2,2,2,6,6,6,6]
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm2 = ymm12[0],ymm2[1],ymm12[2,3],ymm2[4],ymm12[5,6,7,8],ymm2[9],ymm12[10,11],ymm2[12],ymm12[13,14,15]
+; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} ymm12 = ymm13[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,19,20,21,18,19,20,21,24,25,26,27,22,23,22,23]
+; AVX512F-FAST-NEXT:    vpshufd {{.*#+}} ymm13 = ymm1[1,1,1,1,5,5,5,5]
+; AVX512F-FAST-NEXT:    vpblendw {{.*#+}} ymm12 = ymm12[0,1],ymm13[2],ymm12[3,4],ymm13[5],ymm12[6,7,8,9],ymm13[10],ymm12[11,12],ymm13[13],ymm12[14,15]
+; AVX512F-FAST-NEXT:    vpermt2q %zmm2, %zmm21, %zmm12
+; AVX512F-FAST-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm12
+; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} ymm2 = <6,u,u,u,7,u,u,7>
+; AVX512F-FAST-NEXT:    vpermd %ymm3, %ymm2, %ymm2
+; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm3, %zmm15, %zmm3
+; AVX512F-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm5 = [0,13,4,0,0,14,5,0,0,13,4,0,0,14,5,0]
+; AVX512F-FAST-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-FAST-NEXT:    vpermd %zmm3, %zmm5, %zmm3
 ; AVX512F-FAST-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm28, %zmm3
-; AVX512F-FAST-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm14, %zmm3
-; AVX512F-FAST-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm4 # 32-byte Folded Reload
-; AVX512F-FAST-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm11, %zmm9 # 32-byte Folded Reload
-; AVX512F-FAST-NEXT:    vpternlogq $226, %zmm4, %zmm8, %zmm9
-; AVX512F-FAST-NEXT:    vpternlogq $226, %zmm23, %zmm2, %zmm20
-; AVX512F-FAST-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm25, %zmm31
-; AVX512F-FAST-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm9, %zmm31
-; AVX512F-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm12, %zmm2, %zmm2
-; AVX512F-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm13, %zmm4, %zmm4
-; AVX512F-FAST-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm4
-; AVX512F-FAST-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm29, %zmm16
-; AVX512F-FAST-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm4, %zmm16
-; AVX512F-FAST-NEXT:    vpbroadcastd 32(%rax), %ymm2
-; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; AVX512F-FAST-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm22, %zmm0
-; AVX512F-FAST-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm20, %zmm0
+; AVX512F-FAST-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm12, %zmm3
+; AVX512F-FAST-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm5 # 32-byte Folded Reload
+; AVX512F-FAST-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm9, %zmm9 # 32-byte Folded Reload
+; AVX512F-FAST-NEXT:    vpternlogq $226, %zmm5, %zmm11, %zmm9
+; AVX512F-FAST-NEXT:    vpternlogq $226, %zmm23, %zmm18, %zmm19
+; AVX512F-FAST-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm25, %zmm30
+; AVX512F-FAST-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm9, %zmm30
+; AVX512F-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm8, %zmm5, %zmm5
+; AVX512F-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm7, %zmm8, %zmm7
+; AVX512F-FAST-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm7
+; AVX512F-FAST-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm29, %zmm14
+; AVX512F-FAST-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm7, %zmm14
+; AVX512F-FAST-NEXT:    vpbroadcastd 32(%rax), %ymm5
+; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm5, %zmm2, %zmm2
+; AVX512F-FAST-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm20, %zmm2
+; AVX512F-FAST-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm19, %zmm2
 ; AVX512F-FAST-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512F-FAST-NEXT:    vmovdqa64 %zmm3, 128(%rax)
-; AVX512F-FAST-NEXT:    vmovdqa64 %zmm5, (%rax)
-; AVX512F-FAST-NEXT:    vmovdqa64 %zmm16, 320(%rax)
+; AVX512F-FAST-NEXT:    vmovdqa64 %zmm0, (%rax)
+; AVX512F-FAST-NEXT:    vmovdqa64 %zmm14, 320(%rax)
 ; AVX512F-FAST-NEXT:    vmovdqa64 %zmm6, 256(%rax)
-; AVX512F-FAST-NEXT:    vmovdqa64 %zmm0, 192(%rax)
-; AVX512F-FAST-NEXT:    vmovdqa64 %zmm7, 384(%rax)
-; AVX512F-FAST-NEXT:    vmovdqa64 %zmm31, 64(%rax)
-; AVX512F-FAST-NEXT:    addq $248, %rsp
+; AVX512F-FAST-NEXT:    vmovdqa64 %zmm2, 192(%rax)
+; AVX512F-FAST-NEXT:    vmovdqa64 %zmm4, 384(%rax)
+; AVX512F-FAST-NEXT:    vmovdqa64 %zmm30, 64(%rax)
+; AVX512F-FAST-NEXT:    addq $232, %rsp
 ; AVX512F-FAST-NEXT:    vzeroupper
 ; AVX512F-FAST-NEXT:    retq
 ;
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-8.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-8.ll
index c02348af872917b0439d127516f9a54e0b28ce9c..bc77e91a2bd4041cda5c81a1727f087ffda0f9df 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-8.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-8.ll
@@ -3480,200 +3480,203 @@ define void @store_i16_stride8_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ;
 ; AVX512F-FAST-LABEL: store_i16_stride8_vf32:
 ; AVX512F-FAST:       # %bb.0:
-; AVX512F-FAST-NEXT:    subq $472, %rsp # imm = 0x1D8
+; AVX512F-FAST-NEXT:    subq $520, %rsp # imm = 0x208
 ; AVX512F-FAST-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512F-FAST-NEXT:    movq {{[0-9]+}}(%rsp), %r10
 ; AVX512F-FAST-NEXT:    vmovdqa (%r10), %xmm1
 ; AVX512F-FAST-NEXT:    vmovdqa (%rax), %xmm2
 ; AVX512F-FAST-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
-; AVX512F-FAST-NEXT:    vmovdqa64 %xmm2, %xmm26
-; AVX512F-FAST-NEXT:    vmovdqa64 %xmm1, %xmm27
+; AVX512F-FAST-NEXT:    vmovdqa64 %xmm2, %xmm25
+; AVX512F-FAST-NEXT:    vmovdqa64 %xmm1, %xmm26
 ; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm0, %zmm0, %zmm0
 ; AVX512F-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-FAST-NEXT:    vmovdqa (%r9), %xmm0
-; AVX512F-FAST-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX512F-FAST-NEXT:    vmovdqa (%r9), %xmm1
 ; AVX512F-FAST-NEXT:    vmovdqa (%r8), %xmm2
-; AVX512F-FAST-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
-; AVX512F-FAST-NEXT:    vmovdqa64 %xmm2, %xmm31
+; AVX512F-FAST-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; AVX512F-FAST-NEXT:    vmovdqa64 %xmm2, %xmm27
+; AVX512F-FAST-NEXT:    vmovdqa64 %xmm1, %xmm28
 ; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm0, %zmm0, %zmm0
 ; AVX512F-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-FAST-NEXT:    vmovdqa (%rcx), %xmm1
 ; AVX512F-FAST-NEXT:    vmovdqa (%rdx), %xmm2
 ; AVX512F-FAST-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
-; AVX512F-FAST-NEXT:    vmovdqa64 %xmm2, %xmm20
-; AVX512F-FAST-NEXT:    vmovdqa64 %xmm1, %xmm21
+; AVX512F-FAST-NEXT:    vmovdqa64 %xmm2, %xmm31
+; AVX512F-FAST-NEXT:    vmovdqa64 %xmm1, %xmm19
 ; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm0, %zmm0, %zmm0
 ; AVX512F-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-FAST-NEXT:    vmovdqa (%rsi), %xmm1
 ; AVX512F-FAST-NEXT:    vmovdqa (%rdi), %xmm2
 ; AVX512F-FAST-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
-; AVX512F-FAST-NEXT:    vmovdqa64 %xmm2, %xmm18
-; AVX512F-FAST-NEXT:    vmovdqa64 %xmm1, %xmm19
-; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm0, %zmm0, %zmm0
-; AVX512F-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-FAST-NEXT:    vmovdqa64 %xmm2, %xmm17
+; AVX512F-FAST-NEXT:    vmovdqa64 %xmm1, %xmm18
+; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm0, %zmm0, %zmm20
 ; AVX512F-FAST-NEXT:    vmovdqa 32(%r10), %ymm0
 ; AVX512F-FAST-NEXT:    vmovdqa 32(%rax), %ymm1
 ; AVX512F-FAST-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11]
-; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm2, %zmm2, %zmm24
-; AVX512F-FAST-NEXT:    vmovdqa 32(%r9), %ymm2
+; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm2, %zmm2, %zmm23
+; AVX512F-FAST-NEXT:    vmovdqa 32(%r9), %ymm3
 ; AVX512F-FAST-NEXT:    vmovdqa 32(%r8), %ymm4
-; AVX512F-FAST-NEXT:    vpunpcklwd {{.*#+}} ymm3 = ymm4[0],ymm2[0],ymm4[1],ymm2[1],ymm4[2],ymm2[2],ymm4[3],ymm2[3],ymm4[8],ymm2[8],ymm4[9],ymm2[9],ymm4[10],ymm2[10],ymm4[11],ymm2[11]
-; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm3, %zmm3, %zmm25
+; AVX512F-FAST-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[2],ymm3[2],ymm4[3],ymm3[3],ymm4[8],ymm3[8],ymm4[9],ymm3[9],ymm4[10],ymm3[10],ymm4[11],ymm3[11]
+; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm2, %zmm2, %zmm24
 ; AVX512F-FAST-NEXT:    vmovdqa 32(%rcx), %ymm5
 ; AVX512F-FAST-NEXT:    vmovdqa 32(%rdx), %ymm6
-; AVX512F-FAST-NEXT:    vpunpcklwd {{.*#+}} ymm3 = ymm6[0],ymm5[0],ymm6[1],ymm5[1],ymm6[2],ymm5[2],ymm6[3],ymm5[3],ymm6[8],ymm5[8],ymm6[9],ymm5[9],ymm6[10],ymm5[10],ymm6[11],ymm5[11]
-; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm3, %zmm3, %zmm22
+; AVX512F-FAST-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm6[0],ymm5[0],ymm6[1],ymm5[1],ymm6[2],ymm5[2],ymm6[3],ymm5[3],ymm6[8],ymm5[8],ymm6[9],ymm5[9],ymm6[10],ymm5[10],ymm6[11],ymm5[11]
+; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm2, %zmm2, %zmm21
 ; AVX512F-FAST-NEXT:    vmovdqa 32(%rsi), %ymm7
 ; AVX512F-FAST-NEXT:    vmovdqa 32(%rdi), %ymm8
-; AVX512F-FAST-NEXT:    vpunpcklwd {{.*#+}} ymm3 = ymm8[0],ymm7[0],ymm8[1],ymm7[1],ymm8[2],ymm7[2],ymm8[3],ymm7[3],ymm8[8],ymm7[8],ymm8[9],ymm7[9],ymm8[10],ymm7[10],ymm8[11],ymm7[11]
-; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm3, %zmm3, %zmm23
+; AVX512F-FAST-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm8[0],ymm7[0],ymm8[1],ymm7[1],ymm8[2],ymm7[2],ymm8[3],ymm7[3],ymm8[8],ymm7[8],ymm8[9],ymm7[9],ymm8[10],ymm7[10],ymm8[11],ymm7[11]
+; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm2, %zmm2, %zmm22
 ; AVX512F-FAST-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15]
 ; AVX512F-FAST-NEXT:    vmovdqa 32(%r10), %xmm0
 ; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm1, %zmm1, %zmm16
-; AVX512F-FAST-NEXT:    vmovdqa 32(%rax), %xmm3
-; AVX512F-FAST-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm4[4],ymm2[4],ymm4[5],ymm2[5],ymm4[6],ymm2[6],ymm4[7],ymm2[7],ymm4[12],ymm2[12],ymm4[13],ymm2[13],ymm4[14],ymm2[14],ymm4[15],ymm2[15]
+; AVX512F-FAST-NEXT:    vmovdqa 32(%rax), %xmm2
+; AVX512F-FAST-NEXT:    vpunpckhwd {{.*#+}} ymm1 = ymm4[4],ymm3[4],ymm4[5],ymm3[5],ymm4[6],ymm3[6],ymm4[7],ymm3[7],ymm4[12],ymm3[12],ymm4[13],ymm3[13],ymm4[14],ymm3[14],ymm4[15],ymm3[15]
+; AVX512F-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-FAST-NEXT:    vmovdqa 32(%r9), %xmm1
-; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm2, %zmm2, %zmm17
-; AVX512F-FAST-NEXT:    vmovdqa 32(%r8), %xmm4
-; AVX512F-FAST-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm6[4],ymm5[4],ymm6[5],ymm5[5],ymm6[6],ymm5[6],ymm6[7],ymm5[7],ymm6[12],ymm5[12],ymm6[13],ymm5[13],ymm6[14],ymm5[14],ymm6[15],ymm5[15]
-; AVX512F-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-FAST-NEXT:    vmovdqa 32(%rcx), %xmm2
-; AVX512F-FAST-NEXT:    vpunpckhwd {{.*#+}} ymm5 = ymm8[4],ymm7[4],ymm8[5],ymm7[5],ymm8[6],ymm7[6],ymm8[7],ymm7[7],ymm8[12],ymm7[12],ymm8[13],ymm7[13],ymm8[14],ymm7[14],ymm8[15],ymm7[15]
-; AVX512F-FAST-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-FAST-NEXT:    vpunpcklwd {{.*#+}} xmm11 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3]
-; AVX512F-FAST-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
-; AVX512F-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-FAST-NEXT:    vpunpcklwd {{.*#+}} xmm15 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
-; AVX512F-FAST-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm4[4],xmm1[4],xmm4[5],xmm1[5],xmm4[6],xmm1[6],xmm4[7],xmm1[7]
+; AVX512F-FAST-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm6[4],ymm5[4],ymm6[5],ymm5[5],ymm6[6],ymm5[6],ymm6[7],ymm5[7],ymm6[12],ymm5[12],ymm6[13],ymm5[13],ymm6[14],ymm5[14],ymm6[15],ymm5[15]
+; AVX512F-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-FAST-NEXT:    vmovdqa 32(%r8), %xmm3
+; AVX512F-FAST-NEXT:    vpunpckhwd {{.*#+}} ymm4 = ymm8[4],ymm7[4],ymm8[5],ymm7[5],ymm8[6],ymm7[6],ymm8[7],ymm7[7],ymm8[12],ymm7[12],ymm8[13],ymm7[13],ymm8[14],ymm7[14],ymm8[15],ymm7[15]
+; AVX512F-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-FAST-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
+; AVX512F-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-FAST-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; AVX512F-FAST-NEXT:    vmovdqu64 %zmm0, (%rsp) # 64-byte Spill
+; AVX512F-FAST-NEXT:    vpunpcklwd {{.*#+}} xmm12 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
+; AVX512F-FAST-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
 ; AVX512F-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-FAST-NEXT:    vmovdqa 32(%rdx), %xmm0
-; AVX512F-FAST-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
-; AVX512F-FAST-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; AVX512F-FAST-NEXT:    vmovdqa 32(%rcx), %xmm0
+; AVX512F-FAST-NEXT:    vmovdqa 32(%rdx), %xmm1
+; AVX512F-FAST-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; AVX512F-FAST-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
 ; AVX512F-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-FAST-NEXT:    vmovdqa 32(%rsi), %xmm0
 ; AVX512F-FAST-NEXT:    vmovdqa 32(%rdi), %xmm1
-; AVX512F-FAST-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
-; AVX512F-FAST-NEXT:    vpunpckhwd {{.*#+}} xmm14 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX512F-FAST-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; AVX512F-FAST-NEXT:    vpunpckhwd {{.*#+}} xmm15 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
 ; AVX512F-FAST-NEXT:    vmovdqa (%r10), %ymm0
 ; AVX512F-FAST-NEXT:    vmovdqa (%rax), %ymm1
-; AVX512F-FAST-NEXT:    vpunpcklwd {{.*#+}} ymm12 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11]
-; AVX512F-FAST-NEXT:    vpunpckhwd {{.*#+}} ymm10 = ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15]
+; AVX512F-FAST-NEXT:    vpunpcklwd {{.*#+}} ymm13 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11]
+; AVX512F-FAST-NEXT:    vpunpckhwd {{.*#+}} ymm11 = ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15]
 ; AVX512F-FAST-NEXT:    vmovdqa (%r9), %ymm0
 ; AVX512F-FAST-NEXT:    vmovdqa (%r8), %ymm1
-; AVX512F-FAST-NEXT:    vpunpcklwd {{.*#+}} ymm6 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11]
-; AVX512F-FAST-NEXT:    vpunpckhwd {{.*#+}} ymm7 = ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15]
+; AVX512F-FAST-NEXT:    vpunpcklwd {{.*#+}} ymm7 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11]
+; AVX512F-FAST-NEXT:    vpunpckhwd {{.*#+}} ymm8 = ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15]
 ; AVX512F-FAST-NEXT:    vmovdqa (%rcx), %ymm0
 ; AVX512F-FAST-NEXT:    vmovdqa (%rdx), %ymm1
-; AVX512F-FAST-NEXT:    vpunpcklwd {{.*#+}} ymm5 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11]
-; AVX512F-FAST-NEXT:    vpunpckhwd {{.*#+}} ymm4 = ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15]
+; AVX512F-FAST-NEXT:    vpunpcklwd {{.*#+}} ymm6 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11]
+; AVX512F-FAST-NEXT:    vpunpckhwd {{.*#+}} ymm5 = ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15]
 ; AVX512F-FAST-NEXT:    vmovdqa (%rsi), %ymm0
-; AVX512F-FAST-NEXT:    vmovdqa (%rdi), %ymm2
-; AVX512F-FAST-NEXT:    vpunpcklwd {{.*#+}} ymm3 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11]
-; AVX512F-FAST-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm2[4],ymm0[4],ymm2[5],ymm0[5],ymm2[6],ymm0[6],ymm2[7],ymm0[7],ymm2[12],ymm0[12],ymm2[13],ymm0[13],ymm2[14],ymm0[14],ymm2[15],ymm0[15]
-; AVX512F-FAST-NEXT:    vmovdqa64 %xmm26, %xmm0
-; AVX512F-FAST-NEXT:    vmovdqa64 %xmm27, %xmm1
+; AVX512F-FAST-NEXT:    vmovdqa (%rdi), %ymm1
+; AVX512F-FAST-NEXT:    vpunpcklwd {{.*#+}} ymm3 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11]
+; AVX512F-FAST-NEXT:    vpunpckhwd {{.*#+}} ymm4 = ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15]
+; AVX512F-FAST-NEXT:    vmovdqa64 %xmm25, %xmm0
+; AVX512F-FAST-NEXT:    vmovdqa64 %xmm26, %xmm1
+; AVX512F-FAST-NEXT:    vpunpcklwd {{.*#+}} xmm2 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX512F-FAST-NEXT:    vmovdqa64 %xmm27, %xmm0
+; AVX512F-FAST-NEXT:    vmovdqa64 %xmm28, %xmm1
 ; AVX512F-FAST-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
 ; AVX512F-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm25
+; AVX512F-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm26
 ; AVX512F-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm27
-; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm11, %zmm11, %zmm30
-; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm15, %zmm15, %zmm15
-; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm9, %zmm9, %zmm28
-; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm8, %zmm8, %zmm29
+; AVX512F-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm0, %zmm0, %zmm28
+; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm12, %zmm12, %zmm12
+; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm9, %zmm9, %zmm29
+; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm10, %zmm10, %zmm30
 ; AVX512F-FAST-NEXT:    vmovdqa64 %xmm31, %xmm0
-; AVX512F-FAST-NEXT:    vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm0, %xmm9 # 16-byte Folded Reload
-; AVX512F-FAST-NEXT:    # xmm9 = xmm0[0],mem[0],xmm0[1],mem[1],xmm0[2],mem[2],xmm0[3],mem[3]
-; AVX512F-FAST-NEXT:    vmovdqa64 %xmm20, %xmm0
-; AVX512F-FAST-NEXT:    vmovdqa64 %xmm21, %xmm8
-; AVX512F-FAST-NEXT:    vpunpcklwd {{.*#+}} xmm8 = xmm0[0],xmm8[0],xmm0[1],xmm8[1],xmm0[2],xmm8[2],xmm0[3],xmm8[3]
-; AVX512F-FAST-NEXT:    vmovdqa64 %xmm18, %xmm0
-; AVX512F-FAST-NEXT:    vmovdqa64 %xmm19, %xmm11
-; AVX512F-FAST-NEXT:    vpunpcklwd {{.*#+}} xmm11 = xmm0[0],xmm11[0],xmm0[1],xmm11[1],xmm0[2],xmm11[2],xmm0[3],xmm11[3]
+; AVX512F-FAST-NEXT:    vmovdqa64 %xmm19, %xmm9
+; AVX512F-FAST-NEXT:    vpunpcklwd {{.*#+}} xmm9 = xmm0[0],xmm9[0],xmm0[1],xmm9[1],xmm0[2],xmm9[2],xmm0[3],xmm9[3]
+; AVX512F-FAST-NEXT:    vmovdqa64 %xmm17, %xmm0
+; AVX512F-FAST-NEXT:    vmovdqa64 %xmm18, %xmm10
+; AVX512F-FAST-NEXT:    vpunpcklwd {{.*#+}} xmm10 = xmm0[0],xmm10[0],xmm0[1],xmm10[1],xmm0[2],xmm10[2],xmm0[3],xmm10[3]
 ; AVX512F-FAST-NEXT:    movw $-30584, %ax # imm = 0x8888
 ; AVX512F-FAST-NEXT:    kmovw %eax, %k1
 ; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm31 = [0,1,4,5,4,5,5,7,10,9,14,13,14,13,15,15]
-; AVX512F-FAST-NEXT:    vpermd %zmm25, %zmm31, %zmm25
+; AVX512F-FAST-NEXT:    vpermd %zmm24, %zmm31, %zmm24
 ; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [0,0,4,4,4,4,6,5,8,10,12,14,12,14,14,15]
-; AVX512F-FAST-NEXT:    vpermd %zmm24, %zmm0, %zmm25 {%k1}
-; AVX512F-FAST-NEXT:    vpermd %zmm17, %zmm31, %zmm17
-; AVX512F-FAST-NEXT:    vpermd %zmm16, %zmm0, %zmm17 {%k1}
-; AVX512F-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm13, %zmm13, %zmm16
-; AVX512F-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm13, %zmm13, %zmm24
-; AVX512F-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm13, %zmm13, %zmm13
+; AVX512F-FAST-NEXT:    vpermd %zmm23, %zmm0, %zmm24 {%k1}
+; AVX512F-FAST-NEXT:    vpermd %zmm25, %zmm31, %zmm23
+; AVX512F-FAST-NEXT:    vpermd %zmm16, %zmm0, %zmm23 {%k1}
+; AVX512F-FAST-NEXT:    vmovdqu64 (%rsp), %zmm14 # 64-byte Reload
+; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm14, %zmm14, %zmm16
+; AVX512F-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
+; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm14, %zmm14, %zmm25
+; AVX512F-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
 ; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm14, %zmm14, %zmm14
-; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm12, %zmm12, %zmm12
+; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm15, %zmm15, %zmm15
+; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm13, %zmm13, %zmm13
+; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm7, %zmm7, %zmm7
 ; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm6, %zmm6, %zmm6
-; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm5, %zmm5, %zmm5
 ; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm3, %zmm3, %zmm3
-; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm10, %zmm10, %zmm10
-; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm7, %zmm7, %zmm7
+; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm11, %zmm11, %zmm11
+; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm8, %zmm8, %zmm8
+; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm5, %zmm5, %zmm5
 ; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm4, %zmm4, %zmm4
-; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm2, %zmm2, %zmm2
+; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm2, %zmm2, %zmm2
 ; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm1, %zmm1, %zmm1
 ; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm9, %zmm9, %zmm9
-; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm8, %zmm8, %zmm8
-; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm11, %zmm11, %zmm11
-; AVX512F-FAST-NEXT:    vpermd %zmm6, %zmm31, %zmm6
-; AVX512F-FAST-NEXT:    vpermd %zmm12, %zmm0, %zmm6 {%k1}
+; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm10, %zmm10, %zmm10
 ; AVX512F-FAST-NEXT:    vpermd %zmm7, %zmm31, %zmm7
+; AVX512F-FAST-NEXT:    vpermd %zmm13, %zmm0, %zmm7 {%k1}
+; AVX512F-FAST-NEXT:    vpermd %zmm8, %zmm31, %zmm8
 ; AVX512F-FAST-NEXT:    movw $8738, %ax # imm = 0x2222
 ; AVX512F-FAST-NEXT:    kmovw %eax, %k2
-; AVX512F-FAST-NEXT:    vpermd %zmm10, %zmm0, %zmm7 {%k1}
+; AVX512F-FAST-NEXT:    vpermd %zmm11, %zmm0, %zmm8 {%k1}
 ; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [4,5,1,3,5,7,5,7,14,13,11,11,15,15,15,15]
-; AVX512F-FAST-NEXT:    vpermd %zmm23, %zmm0, %zmm10
-; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm12 = [4,4,2,1,6,5,6,5,12,14,10,11,14,15,14,15]
-; AVX512F-FAST-NEXT:    vpermd %zmm22, %zmm12, %zmm10 {%k2}
-; AVX512F-FAST-NEXT:    vpermd %zmm27, %zmm0, %zmm22
-; AVX512F-FAST-NEXT:    vpermd %zmm26, %zmm12, %zmm22 {%k2}
+; AVX512F-FAST-NEXT:    vpermd %zmm22, %zmm0, %zmm11
+; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm13 = [4,4,2,1,6,5,6,5,12,14,10,11,14,15,14,15]
+; AVX512F-FAST-NEXT:    vpermd %zmm21, %zmm13, %zmm11 {%k2}
+; AVX512F-FAST-NEXT:    vpermd %zmm27, %zmm0, %zmm21
+; AVX512F-FAST-NEXT:    vpermd %zmm26, %zmm13, %zmm21 {%k2}
 ; AVX512F-FAST-NEXT:    vpermd %zmm3, %zmm0, %zmm3
-; AVX512F-FAST-NEXT:    vpermd %zmm5, %zmm12, %zmm3 {%k2}
-; AVX512F-FAST-NEXT:    vpermd %zmm2, %zmm0, %zmm0
-; AVX512F-FAST-NEXT:    vpermd %zmm4, %zmm12, %zmm0 {%k2}
-; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm2 = <0,u,0,u,u,u,1,u,10,10,10,10,u,u,11,11>
-; AVX512F-FAST-NEXT:    vpermd {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm4 # 64-byte Folded Reload
-; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm5 = <0,0,0,0,u,u,1,1,10,10,10,10,u,u,11,11>
-; AVX512F-FAST-NEXT:    vpermd {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm4 {%k1} # 64-byte Folded Reload
-; AVX512F-FAST-NEXT:    vpermd %zmm15, %zmm2, %zmm12
-; AVX512F-FAST-NEXT:    vpermd %zmm30, %zmm5, %zmm12 {%k1}
-; AVX512F-FAST-NEXT:    vpermd %zmm24, %zmm2, %zmm15
-; AVX512F-FAST-NEXT:    vpermd %zmm16, %zmm5, %zmm15 {%k1}
-; AVX512F-FAST-NEXT:    vpermd %zmm9, %zmm2, %zmm2
-; AVX512F-FAST-NEXT:    vpermd %zmm1, %zmm5, %zmm2 {%k1}
-; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm1 = <0,u,1,u,1,u,u,u,10,u,11,u,11,u,u,u>
-; AVX512F-FAST-NEXT:    vpermd {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm1 # 64-byte Folded Reload
-; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm5 = <0,0,1,1,1,1,u,u,10,10,11,11,11,11,u,u>
-; AVX512F-FAST-NEXT:    vpermd {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm1 {%k2} # 64-byte Folded Reload
-; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm9 = <0,u,1,u,1,u,u,u,10,10,11,11,11,11,u,u>
-; AVX512F-FAST-NEXT:    vpermd %zmm29, %zmm9, %zmm16
-; AVX512F-FAST-NEXT:    vpermd %zmm28, %zmm5, %zmm16 {%k2}
-; AVX512F-FAST-NEXT:    vpermd %zmm14, %zmm9, %zmm14
-; AVX512F-FAST-NEXT:    vpermd %zmm13, %zmm5, %zmm14 {%k2}
-; AVX512F-FAST-NEXT:    vpermd %zmm11, %zmm9, %zmm9
-; AVX512F-FAST-NEXT:    vpermd %zmm8, %zmm5, %zmm9 {%k2}
+; AVX512F-FAST-NEXT:    vpermd %zmm6, %zmm13, %zmm3 {%k2}
+; AVX512F-FAST-NEXT:    vpermd %zmm4, %zmm0, %zmm0
+; AVX512F-FAST-NEXT:    vpermd %zmm5, %zmm13, %zmm0 {%k2}
+; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm4 = <0,0,0,0,u,u,1,1,10,10,10,10,u,u,11,11>
+; AVX512F-FAST-NEXT:    vpermd {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm5 # 64-byte Folded Reload
+; AVX512F-FAST-NEXT:    vpermd {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm5 {%k1} # 64-byte Folded Reload
+; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm6 = <0,u,0,u,u,u,1,u,10,10,10,10,u,u,11,11>
+; AVX512F-FAST-NEXT:    vpermd %zmm12, %zmm6, %zmm6
+; AVX512F-FAST-NEXT:    vpermd %zmm28, %zmm4, %zmm6 {%k1}
+; AVX512F-FAST-NEXT:    vpermd %zmm25, %zmm4, %zmm12
+; AVX512F-FAST-NEXT:    vpermd %zmm16, %zmm4, %zmm12 {%k1}
+; AVX512F-FAST-NEXT:    vpermd %zmm1, %zmm4, %zmm1
+; AVX512F-FAST-NEXT:    vpermd %zmm2, %zmm4, %zmm1 {%k1}
+; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm2 = <0,0,1,1,1,1,u,u,10,u,11,u,11,u,u,u>
+; AVX512F-FAST-NEXT:    vpermd %zmm20, %zmm2, %zmm2
+; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm4 = <0,0,1,1,1,1,u,u,10,10,11,11,11,11,u,u>
+; AVX512F-FAST-NEXT:    vpermd {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm2 {%k2} # 64-byte Folded Reload
+; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm13 = <0,u,1,u,1,u,u,u,10,10,11,11,11,11,u,u>
+; AVX512F-FAST-NEXT:    vpermd %zmm30, %zmm13, %zmm13
+; AVX512F-FAST-NEXT:    vpermd %zmm29, %zmm4, %zmm13 {%k2}
+; AVX512F-FAST-NEXT:    vpermd %zmm15, %zmm4, %zmm15
+; AVX512F-FAST-NEXT:    vpermd %zmm14, %zmm4, %zmm15 {%k2}
+; AVX512F-FAST-NEXT:    vpermd %zmm10, %zmm4, %zmm10
+; AVX512F-FAST-NEXT:    vpermd %zmm9, %zmm4, %zmm10 {%k2}
 ; AVX512F-FAST-NEXT:    movb $-86, %al
 ; AVX512F-FAST-NEXT:    kmovw %eax, %k1
-; AVX512F-FAST-NEXT:    vmovdqa64 %zmm4, %zmm1 {%k1}
-; AVX512F-FAST-NEXT:    vmovdqa64 %zmm25, %zmm10 {%k1}
-; AVX512F-FAST-NEXT:    vmovdqa64 %zmm17, %zmm22 {%k1}
-; AVX512F-FAST-NEXT:    vmovdqa64 %zmm12, %zmm16 {%k1}
-; AVX512F-FAST-NEXT:    vmovdqa64 %zmm15, %zmm14 {%k1}
-; AVX512F-FAST-NEXT:    vmovdqa64 %zmm6, %zmm3 {%k1}
-; AVX512F-FAST-NEXT:    vmovdqa64 %zmm7, %zmm0 {%k1}
-; AVX512F-FAST-NEXT:    vmovdqa64 %zmm2, %zmm9 {%k1}
+; AVX512F-FAST-NEXT:    vmovdqa64 %zmm5, %zmm2 {%k1}
+; AVX512F-FAST-NEXT:    vmovdqa64 %zmm24, %zmm11 {%k1}
+; AVX512F-FAST-NEXT:    vmovdqa64 %zmm23, %zmm21 {%k1}
+; AVX512F-FAST-NEXT:    vmovdqa64 %zmm6, %zmm13 {%k1}
+; AVX512F-FAST-NEXT:    vmovdqa64 %zmm12, %zmm15 {%k1}
+; AVX512F-FAST-NEXT:    vmovdqa64 %zmm7, %zmm3 {%k1}
+; AVX512F-FAST-NEXT:    vmovdqa64 %zmm8, %zmm0 {%k1}
+; AVX512F-FAST-NEXT:    vmovdqa64 %zmm1, %zmm10 {%k1}
 ; AVX512F-FAST-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512F-FAST-NEXT:    vmovdqa64 %zmm9, (%rax)
+; AVX512F-FAST-NEXT:    vmovdqa64 %zmm10, (%rax)
 ; AVX512F-FAST-NEXT:    vmovdqa64 %zmm0, 192(%rax)
 ; AVX512F-FAST-NEXT:    vmovdqa64 %zmm3, 128(%rax)
-; AVX512F-FAST-NEXT:    vmovdqa64 %zmm14, 320(%rax)
-; AVX512F-FAST-NEXT:    vmovdqa64 %zmm16, 256(%rax)
-; AVX512F-FAST-NEXT:    vmovdqa64 %zmm22, 448(%rax)
-; AVX512F-FAST-NEXT:    vmovdqa64 %zmm10, 384(%rax)
-; AVX512F-FAST-NEXT:    vmovdqa64 %zmm1, 64(%rax)
-; AVX512F-FAST-NEXT:    addq $472, %rsp # imm = 0x1D8
+; AVX512F-FAST-NEXT:    vmovdqa64 %zmm15, 320(%rax)
+; AVX512F-FAST-NEXT:    vmovdqa64 %zmm13, 256(%rax)
+; AVX512F-FAST-NEXT:    vmovdqa64 %zmm21, 448(%rax)
+; AVX512F-FAST-NEXT:    vmovdqa64 %zmm11, 384(%rax)
+; AVX512F-FAST-NEXT:    vmovdqa64 %zmm2, 64(%rax)
+; AVX512F-FAST-NEXT:    addq $520, %rsp # imm = 0x208
 ; AVX512F-FAST-NEXT:    vzeroupper
 ; AVX512F-FAST-NEXT:    retq
 ;
@@ -7248,11 +7251,11 @@ define void @store_i16_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512F-FAST-NEXT:    vmovdqa 32(%r10), %ymm0
 ; AVX512F-FAST-NEXT:    vmovdqa 32(%rax), %ymm2
 ; AVX512F-FAST-NEXT:    vpunpcklwd {{.*#+}} ymm3 = ymm2[0],ymm0[0],ymm2[1],ymm0[1],ymm2[2],ymm0[2],ymm2[3],ymm0[3],ymm2[8],ymm0[8],ymm2[9],ymm0[9],ymm2[10],ymm0[10],ymm2[11],ymm0[11]
-; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm3, %zmm3, %zmm27
+; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm3, %zmm3, %zmm17
 ; AVX512F-FAST-NEXT:    vmovdqa 32(%r9), %ymm3
 ; AVX512F-FAST-NEXT:    vmovdqa 32(%r8), %ymm4
 ; AVX512F-FAST-NEXT:    vpunpcklwd {{.*#+}} ymm5 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[2],ymm3[2],ymm4[3],ymm3[3],ymm4[8],ymm3[8],ymm4[9],ymm3[9],ymm4[10],ymm3[10],ymm4[11],ymm3[11]
-; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm5, %zmm5, %zmm17
+; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm5, %zmm5, %zmm15
 ; AVX512F-FAST-NEXT:    vmovdqa 32(%rcx), %ymm5
 ; AVX512F-FAST-NEXT:    vmovdqa 32(%rdx), %ymm6
 ; AVX512F-FAST-NEXT:    vpunpcklwd {{.*#+}} ymm7 = ymm6[0],ymm5[0],ymm6[1],ymm5[1],ymm6[2],ymm5[2],ymm6[3],ymm5[3],ymm6[8],ymm5[8],ymm6[9],ymm5[9],ymm6[10],ymm5[10],ymm6[11],ymm5[11]
@@ -7265,63 +7268,63 @@ define void @store_i16_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512F-FAST-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm4[4],ymm3[4],ymm4[5],ymm3[5],ymm4[6],ymm3[6],ymm4[7],ymm3[7],ymm4[12],ymm3[12],ymm4[13],ymm3[13],ymm4[14],ymm3[14],ymm4[15],ymm3[15]
 ; AVX512F-FAST-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm6[4],ymm5[4],ymm6[5],ymm5[5],ymm6[6],ymm5[6],ymm6[7],ymm5[7],ymm6[12],ymm5[12],ymm6[13],ymm5[13],ymm6[14],ymm5[14],ymm6[15],ymm5[15]
 ; AVX512F-FAST-NEXT:    vpunpckhwd {{.*#+}} ymm2 = ymm8[4],ymm7[4],ymm8[5],ymm7[5],ymm8[6],ymm7[6],ymm8[7],ymm7[7],ymm8[12],ymm7[12],ymm8[13],ymm7[13],ymm8[14],ymm7[14],ymm8[15],ymm7[15]
-; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm9, %zmm9, %zmm15
-; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm3, %zmm3, %zmm13
+; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm9, %zmm9, %zmm13
+; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm3, %zmm3, %zmm7
 ; AVX512F-FAST-NEXT:    vmovdqa 32(%r10), %xmm3
 ; AVX512F-FAST-NEXT:    vmovdqa 32(%rax), %xmm4
 ; AVX512F-FAST-NEXT:    vpunpcklwd {{.*#+}} xmm5 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
 ; AVX512F-FAST-NEXT:    vpunpckhwd {{.*#+}} xmm3 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
-; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm21
+; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm22
 ; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm2, %zmm2, %zmm9
 ; AVX512F-FAST-NEXT:    vmovdqa 32(%r9), %xmm0
 ; AVX512F-FAST-NEXT:    vmovdqa 32(%r8), %xmm2
 ; AVX512F-FAST-NEXT:    vpunpcklwd {{.*#+}} xmm4 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
 ; AVX512F-FAST-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
-; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm5, %zmm5, %zmm26
-; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm4, %zmm4, %zmm23
+; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm5, %zmm5, %zmm27
+; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm4, %zmm4, %zmm26
 ; AVX512F-FAST-NEXT:    vmovdqa 32(%rcx), %xmm2
 ; AVX512F-FAST-NEXT:    vmovdqa 32(%rdx), %xmm4
 ; AVX512F-FAST-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3]
 ; AVX512F-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-FAST-NEXT:    vpunpckhwd {{.*#+}} xmm1 = xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7]
 ; AVX512F-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm3, %zmm3, %zmm22
-; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm0, %zmm0, %zmm20
+; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm3, %zmm3, %zmm23
+; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm0, %zmm0, %zmm21
 ; AVX512F-FAST-NEXT:    vmovdqa 32(%rsi), %xmm0
 ; AVX512F-FAST-NEXT:    vmovdqa 32(%rdi), %xmm2
 ; AVX512F-FAST-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3]
 ; AVX512F-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-FAST-NEXT:    vpunpckhwd {{.*#+}} xmm0 = xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
 ; AVX512F-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-FAST-NEXT:    vmovdqa (%r10), %ymm6
+; AVX512F-FAST-NEXT:    vmovdqa (%r10), %ymm5
 ; AVX512F-FAST-NEXT:    vmovdqa (%rax), %ymm3
-; AVX512F-FAST-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm3[0],ymm6[0],ymm3[1],ymm6[1],ymm3[2],ymm6[2],ymm3[3],ymm6[3],ymm3[8],ymm6[8],ymm3[9],ymm6[9],ymm3[10],ymm6[10],ymm3[11],ymm6[11]
-; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm2, %zmm2, %zmm7
-; AVX512F-FAST-NEXT:    vmovdqa (%r9), %ymm5
+; AVX512F-FAST-NEXT:    vpunpcklwd {{.*#+}} ymm2 = ymm3[0],ymm5[0],ymm3[1],ymm5[1],ymm3[2],ymm5[2],ymm3[3],ymm5[3],ymm3[8],ymm5[8],ymm3[9],ymm5[9],ymm3[10],ymm5[10],ymm3[11],ymm5[11]
+; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm2, %zmm2, %zmm6
+; AVX512F-FAST-NEXT:    vmovdqa (%r9), %ymm8
 ; AVX512F-FAST-NEXT:    vmovdqa (%r8), %ymm10
-; AVX512F-FAST-NEXT:    vpunpcklwd {{.*#+}} ymm4 = ymm10[0],ymm5[0],ymm10[1],ymm5[1],ymm10[2],ymm5[2],ymm10[3],ymm5[3],ymm10[8],ymm5[8],ymm10[9],ymm5[9],ymm10[10],ymm5[10],ymm10[11],ymm5[11]
+; AVX512F-FAST-NEXT:    vpunpcklwd {{.*#+}} ymm4 = ymm10[0],ymm8[0],ymm10[1],ymm8[1],ymm10[2],ymm8[2],ymm10[3],ymm8[3],ymm10[8],ymm8[8],ymm10[9],ymm8[9],ymm10[10],ymm8[10],ymm10[11],ymm8[11]
 ; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm4, %zmm4, %zmm19
 ; AVX512F-FAST-NEXT:    vmovdqa (%rcx), %ymm11
 ; AVX512F-FAST-NEXT:    vmovdqa (%rdx), %ymm14
 ; AVX512F-FAST-NEXT:    vpunpcklwd {{.*#+}} ymm4 = ymm14[0],ymm11[0],ymm14[1],ymm11[1],ymm14[2],ymm11[2],ymm14[3],ymm11[3],ymm14[8],ymm11[8],ymm14[9],ymm11[9],ymm14[10],ymm11[10],ymm14[11],ymm11[11]
-; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm4, %zmm4, %zmm16
+; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm4, %zmm4, %zmm20
 ; AVX512F-FAST-NEXT:    vmovdqa (%rsi), %ymm1
 ; AVX512F-FAST-NEXT:    vmovdqa (%rdi), %ymm0
 ; AVX512F-FAST-NEXT:    vpunpcklwd {{.*#+}} ymm4 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11]
-; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm4, %zmm4, %zmm4
-; AVX512F-FAST-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm3[4],ymm6[4],ymm3[5],ymm6[5],ymm3[6],ymm6[6],ymm3[7],ymm6[7],ymm3[12],ymm6[12],ymm3[13],ymm6[13],ymm3[14],ymm6[14],ymm3[15],ymm6[15]
-; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm3, %zmm3, %zmm6
-; AVX512F-FAST-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm10[4],ymm5[4],ymm10[5],ymm5[5],ymm10[6],ymm5[6],ymm10[7],ymm5[7],ymm10[12],ymm5[12],ymm10[13],ymm5[13],ymm10[14],ymm5[14],ymm10[15],ymm5[15]
+; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm4, %zmm4, %zmm16
+; AVX512F-FAST-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm3[4],ymm5[4],ymm3[5],ymm5[5],ymm3[6],ymm5[6],ymm3[7],ymm5[7],ymm3[12],ymm5[12],ymm3[13],ymm5[13],ymm3[14],ymm5[14],ymm3[15],ymm5[15]
+; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm3, %zmm3, %zmm4
+; AVX512F-FAST-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm10[4],ymm8[4],ymm10[5],ymm8[5],ymm10[6],ymm8[6],ymm10[7],ymm8[7],ymm10[12],ymm8[12],ymm10[13],ymm8[13],ymm10[14],ymm8[14],ymm10[15],ymm8[15]
 ; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm3, %zmm3, %zmm2
 ; AVX512F-FAST-NEXT:    vpunpckhwd {{.*#+}} ymm3 = ymm14[4],ymm11[4],ymm14[5],ymm11[5],ymm14[6],ymm11[6],ymm14[7],ymm11[7],ymm14[12],ymm11[12],ymm14[13],ymm11[13],ymm14[14],ymm11[14],ymm14[15],ymm11[15]
 ; AVX512F-FAST-NEXT:    vpunpckhwd {{.*#+}} ymm0 = ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15]
 ; AVX512F-FAST-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
 ; AVX512F-FAST-NEXT:    vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm12[0],xmm1[1],xmm12[1],xmm1[2],xmm12[2],xmm1[3],xmm12[3]
 ; AVX512F-FAST-NEXT:    vmovdqa (%rsp), %xmm5 # 16-byte Reload
-; AVX512F-FAST-NEXT:    vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm5, %xmm5 # 16-byte Folded Reload
-; AVX512F-FAST-NEXT:    # xmm5 = xmm5[0],mem[0],xmm5[1],mem[1],xmm5[2],mem[2],xmm5[3],mem[3]
+; AVX512F-FAST-NEXT:    vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm5, %xmm8 # 16-byte Folded Reload
+; AVX512F-FAST-NEXT:    # xmm8 = xmm5[0],mem[0],xmm5[1],mem[1],xmm5[2],mem[2],xmm5[3],mem[3]
 ; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm3, %zmm3, %zmm10
-; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm8
+; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm0, %zmm0, %zmm5
 ; AVX512F-FAST-NEXT:    vmovdqa (%rcx), %xmm0
 ; AVX512F-FAST-NEXT:    vmovdqa (%rdx), %xmm3
 ; AVX512F-FAST-NEXT:    vpunpckhwd {{.*#+}} xmm11 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
@@ -7329,7 +7332,7 @@ define void @store_i16_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512F-FAST-NEXT:    vpunpcklwd {{.*#+}} xmm0 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3]
 ; AVX512F-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm1, %zmm1, %zmm14
-; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm5, %zmm5, %zmm12
+; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm8, %zmm8, %zmm12
 ; AVX512F-FAST-NEXT:    vmovdqa (%rsi), %xmm0
 ; AVX512F-FAST-NEXT:    vmovdqa (%rdi), %xmm1
 ; AVX512F-FAST-NEXT:    vpunpckhwd {{.*#+}} xmm11 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
@@ -7348,16 +7351,16 @@ define void @store_i16_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512F-FAST-NEXT:    vpermd %zmm25, %zmm1, %zmm24 {%k2}
 ; AVX512F-FAST-NEXT:    vpermd %zmm18, %zmm0, %zmm25
 ; AVX512F-FAST-NEXT:    vpermd %zmm30, %zmm1, %zmm25 {%k2}
-; AVX512F-FAST-NEXT:    vpermd %zmm17, %zmm0, %zmm5
-; AVX512F-FAST-NEXT:    vpermd %zmm27, %zmm1, %zmm5 {%k2}
-; AVX512F-FAST-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-FAST-NEXT:    vpermd %zmm13, %zmm0, %zmm27
-; AVX512F-FAST-NEXT:    vpermd %zmm15, %zmm1, %zmm27 {%k2}
-; AVX512F-FAST-NEXT:    vpermd %zmm19, %zmm0, %zmm5
-; AVX512F-FAST-NEXT:    vpermd %zmm7, %zmm1, %zmm5 {%k2}
-; AVX512F-FAST-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-FAST-NEXT:    vpermd %zmm15, %zmm0, %zmm8
+; AVX512F-FAST-NEXT:    vpermd %zmm17, %zmm1, %zmm8 {%k2}
+; AVX512F-FAST-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-FAST-NEXT:    vpermd %zmm7, %zmm0, %zmm30
+; AVX512F-FAST-NEXT:    vpermd %zmm13, %zmm1, %zmm30 {%k2}
+; AVX512F-FAST-NEXT:    vpermd %zmm19, %zmm0, %zmm7
+; AVX512F-FAST-NEXT:    vpermd %zmm6, %zmm1, %zmm7 {%k2}
+; AVX512F-FAST-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-FAST-NEXT:    vpermd %zmm2, %zmm0, %zmm19
-; AVX512F-FAST-NEXT:    vpermd %zmm6, %zmm1, %zmm19 {%k2}
+; AVX512F-FAST-NEXT:    vpermd %zmm4, %zmm1, %zmm19 {%k2}
 ; AVX512F-FAST-NEXT:    movw $8738, %ax # imm = 0x2222
 ; AVX512F-FAST-NEXT:    kmovw %eax, %k1
 ; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [4,5,1,3,5,7,5,7,14,13,11,11,15,15,15,15]
@@ -7373,114 +7376,114 @@ define void @store_i16_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512F-FAST-NEXT:    vpermd %zmm28, %zmm0, %zmm17
 ; AVX512F-FAST-NEXT:    vpermd %zmm29, %zmm1, %zmm17 {%k1}
 ; AVX512F-FAST-NEXT:    vpermd %zmm9, %zmm0, %zmm9
-; AVX512F-FAST-NEXT:    vpermd %zmm21, %zmm1, %zmm9 {%k1}
-; AVX512F-FAST-NEXT:    vpermd %zmm4, %zmm0, %zmm4
-; AVX512F-FAST-NEXT:    vpermd %zmm16, %zmm1, %zmm4 {%k1}
-; AVX512F-FAST-NEXT:    vpermd %zmm8, %zmm0, %zmm8
-; AVX512F-FAST-NEXT:    vpermd %zmm10, %zmm1, %zmm8 {%k1}
-; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm0 = <0,u,0,u,u,u,1,u,10,10,10,10,u,u,11,11>
+; AVX512F-FAST-NEXT:    vpermd %zmm22, %zmm1, %zmm9 {%k1}
+; AVX512F-FAST-NEXT:    vpermd %zmm16, %zmm0, %zmm4
+; AVX512F-FAST-NEXT:    vpermd %zmm20, %zmm1, %zmm4 {%k1}
+; AVX512F-FAST-NEXT:    vpermd %zmm5, %zmm0, %zmm5
+; AVX512F-FAST-NEXT:    vpermd %zmm10, %zmm1, %zmm5 {%k1}
+; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm0 = <0,0,0,0,u,u,1,1,10,10,10,10,u,u,11,11>
 ; AVX512F-FAST-NEXT:    vpermd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm1 # 64-byte Folded Reload
-; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm10 = <0,0,0,0,u,u,1,1,10,10,10,10,u,u,11,11>
-; AVX512F-FAST-NEXT:    vpermd {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm1 {%k2} # 64-byte Folded Reload
+; AVX512F-FAST-NEXT:    vpermd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm1 {%k2} # 64-byte Folded Reload
+; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm10 = <0,u,0,u,u,u,1,u,10,10,10,10,u,u,11,11>
+; AVX512F-FAST-NEXT:    vpermd {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm10 # 64-byte Folded Reload
+; AVX512F-FAST-NEXT:    vpermd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm10 {%k2} # 64-byte Folded Reload
 ; AVX512F-FAST-NEXT:    vpermd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm16 # 64-byte Folded Reload
-; AVX512F-FAST-NEXT:    vpermd {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm16 {%k2} # 64-byte Folded Reload
-; AVX512F-FAST-NEXT:    vpermd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm21 # 64-byte Folded Reload
-; AVX512F-FAST-NEXT:    vpermd {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm21 {%k2} # 64-byte Folded Reload
-; AVX512F-FAST-NEXT:    vpermd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm28 # 64-byte Folded Reload
-; AVX512F-FAST-NEXT:    vpermd {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm28 {%k2} # 64-byte Folded Reload
-; AVX512F-FAST-NEXT:    vpermd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm29 # 64-byte Folded Reload
-; AVX512F-FAST-NEXT:    vpermd {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm29 {%k2} # 64-byte Folded Reload
-; AVX512F-FAST-NEXT:    vpermd %zmm23, %zmm0, %zmm23
-; AVX512F-FAST-NEXT:    vpermd %zmm26, %zmm10, %zmm23 {%k2}
-; AVX512F-FAST-NEXT:    vpermd %zmm20, %zmm0, %zmm20
-; AVX512F-FAST-NEXT:    vpermd %zmm22, %zmm10, %zmm20 {%k2}
-; AVX512F-FAST-NEXT:    vpermd %zmm12, %zmm0, %zmm0
-; AVX512F-FAST-NEXT:    vpermd %zmm14, %zmm10, %zmm0 {%k2}
-; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm11, %zmm11, %zmm10
-; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm11 = <0,u,1,u,1,u,u,u,10,u,11,u,11,u,u,u>
-; AVX512F-FAST-NEXT:    vpermd %zmm10, %zmm11, %zmm12
-; AVX512F-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm2, %zmm2, %zmm10
+; AVX512F-FAST-NEXT:    vpermd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm16 {%k2} # 64-byte Folded Reload
+; AVX512F-FAST-NEXT:    vpermd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm20 # 64-byte Folded Reload
+; AVX512F-FAST-NEXT:    vpermd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm20 {%k2} # 64-byte Folded Reload
+; AVX512F-FAST-NEXT:    vpermd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm22 # 64-byte Folded Reload
+; AVX512F-FAST-NEXT:    vpermd {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm22 {%k2} # 64-byte Folded Reload
+; AVX512F-FAST-NEXT:    vpermd %zmm26, %zmm0, %zmm26
+; AVX512F-FAST-NEXT:    vpermd %zmm27, %zmm0, %zmm26 {%k2}
+; AVX512F-FAST-NEXT:    vpermd %zmm21, %zmm0, %zmm21
+; AVX512F-FAST-NEXT:    vpermd %zmm23, %zmm0, %zmm21 {%k2}
+; AVX512F-FAST-NEXT:    vpermd %zmm12, %zmm0, %zmm23
+; AVX512F-FAST-NEXT:    vpermd %zmm14, %zmm0, %zmm23 {%k2}
+; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm11, %zmm11, %zmm0
+; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm11 = <0,0,1,1,1,1,u,u,10,u,11,u,11,u,u,u>
+; AVX512F-FAST-NEXT:    vpermd %zmm0, %zmm11, %zmm12
+; AVX512F-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm0, %zmm0, %zmm0
 ; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm11 = <0,0,1,1,1,1,u,u,10,10,11,11,11,11,u,u>
-; AVX512F-FAST-NEXT:    vpermd %zmm10, %zmm11, %zmm12 {%k1}
-; AVX512F-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm5, %zmm5, %zmm10
-; AVX512F-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm5, %zmm5, %zmm14
-; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm22 = <0,u,1,u,1,u,u,u,10,10,11,11,11,11,u,u>
-; AVX512F-FAST-NEXT:    vpermd %zmm14, %zmm22, %zmm14
-; AVX512F-FAST-NEXT:    vpermd %zmm10, %zmm11, %zmm14 {%k1}
-; AVX512F-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm5, %zmm5, %zmm10
-; AVX512F-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm5, %zmm5, %zmm26
-; AVX512F-FAST-NEXT:    vpermd %zmm26, %zmm22, %zmm26
-; AVX512F-FAST-NEXT:    vpermd %zmm10, %zmm11, %zmm26 {%k1}
-; AVX512F-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm5, %zmm5, %zmm10
-; AVX512F-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm5, %zmm5, %zmm30
-; AVX512F-FAST-NEXT:    vpermd %zmm30, %zmm22, %zmm30
-; AVX512F-FAST-NEXT:    vpermd %zmm10, %zmm11, %zmm30 {%k1}
-; AVX512F-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm5, %zmm5, %zmm10
-; AVX512F-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm5, %zmm5, %zmm31
-; AVX512F-FAST-NEXT:    vpermd %zmm31, %zmm22, %zmm31
-; AVX512F-FAST-NEXT:    vpermd %zmm10, %zmm11, %zmm31 {%k1}
-; AVX512F-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm5, %zmm5, %zmm10
-; AVX512F-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm2, %zmm2, %zmm5
-; AVX512F-FAST-NEXT:    vpermd %zmm5, %zmm22, %zmm5
-; AVX512F-FAST-NEXT:    vpermd %zmm10, %zmm11, %zmm5 {%k1}
-; AVX512F-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
-; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm10, %zmm10, %zmm10
+; AVX512F-FAST-NEXT:    vpermd %zmm0, %zmm11, %zmm12 {%k1}
+; AVX512F-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm0, %zmm0, %zmm0
+; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm14 = <0,u,1,u,1,u,u,u,10,10,11,11,11,11,u,u>
+; AVX512F-FAST-NEXT:    vpermd %zmm0, %zmm14, %zmm0
+; AVX512F-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm8, %zmm8, %zmm14
+; AVX512F-FAST-NEXT:    vpermd %zmm14, %zmm11, %zmm0 {%k1}
+; AVX512F-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm8, %zmm8, %zmm14
+; AVX512F-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm8, %zmm8, %zmm27
+; AVX512F-FAST-NEXT:    vpermd %zmm27, %zmm11, %zmm27
+; AVX512F-FAST-NEXT:    vpermd %zmm14, %zmm11, %zmm27 {%k1}
+; AVX512F-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm8, %zmm8, %zmm14
+; AVX512F-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm8, %zmm8, %zmm28
+; AVX512F-FAST-NEXT:    vpermd %zmm28, %zmm11, %zmm28
+; AVX512F-FAST-NEXT:    vpermd %zmm14, %zmm11, %zmm28 {%k1}
+; AVX512F-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm8, %zmm8, %zmm14
+; AVX512F-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm8, %zmm8, %zmm29
+; AVX512F-FAST-NEXT:    vpermd %zmm29, %zmm11, %zmm29
+; AVX512F-FAST-NEXT:    vpermd %zmm14, %zmm11, %zmm29 {%k1}
+; AVX512F-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm8, %zmm8, %zmm14
+; AVX512F-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm8, %zmm8, %zmm31
+; AVX512F-FAST-NEXT:    vpermd %zmm31, %zmm11, %zmm31
+; AVX512F-FAST-NEXT:    vpermd %zmm14, %zmm11, %zmm31 {%k1}
+; AVX512F-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm8, %zmm8, %zmm14
+; AVX512F-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm8, %zmm8, %zmm8
+; AVX512F-FAST-NEXT:    vpermd %zmm8, %zmm11, %zmm8
+; AVX512F-FAST-NEXT:    vpermd %zmm14, %zmm11, %zmm8 {%k1}
+; AVX512F-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
+; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm14, %zmm14, %zmm14
 ; AVX512F-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
 ; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm2, %zmm2, %zmm18
-; AVX512F-FAST-NEXT:    vpermd %zmm18, %zmm22, %zmm18
-; AVX512F-FAST-NEXT:    vpermd %zmm10, %zmm11, %zmm18 {%k1}
-; AVX512F-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm2, %zmm2, %zmm10
-; AVX512F-FAST-NEXT:    vpermd %zmm10, %zmm22, %zmm10
-; AVX512F-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm2, %zmm2, %zmm22
-; AVX512F-FAST-NEXT:    vpermd %zmm22, %zmm11, %zmm10 {%k1}
+; AVX512F-FAST-NEXT:    vpermd %zmm18, %zmm11, %zmm18
+; AVX512F-FAST-NEXT:    vpermd %zmm14, %zmm11, %zmm18 {%k1}
 ; AVX512F-FAST-NEXT:    movb $-86, %al
 ; AVX512F-FAST-NEXT:    kmovw %eax, %k1
 ; AVX512F-FAST-NEXT:    vmovdqa64 %zmm1, %zmm12 {%k1}
 ; AVX512F-FAST-NEXT:    vmovdqu64 (%rsp), %zmm1 # 64-byte Reload
 ; AVX512F-FAST-NEXT:    vmovdqa64 %zmm1, %zmm6 {%k1}
 ; AVX512F-FAST-NEXT:    vmovdqa64 %zmm3, %zmm7 {%k1}
-; AVX512F-FAST-NEXT:    vmovdqa64 %zmm16, %zmm14 {%k1}
-; AVX512F-FAST-NEXT:    vmovdqa64 %zmm21, %zmm26 {%k1}
+; AVX512F-FAST-NEXT:    vmovdqa64 %zmm10, %zmm0 {%k1}
+; AVX512F-FAST-NEXT:    vmovdqa64 %zmm16, %zmm27 {%k1}
 ; AVX512F-FAST-NEXT:    vmovdqa64 %zmm24, %zmm13 {%k1}
 ; AVX512F-FAST-NEXT:    vmovdqa64 %zmm25, %zmm15 {%k1}
-; AVX512F-FAST-NEXT:    vmovdqa64 %zmm28, %zmm30 {%k1}
-; AVX512F-FAST-NEXT:    vmovdqa64 %zmm29, %zmm31 {%k1}
+; AVX512F-FAST-NEXT:    vmovdqa64 %zmm20, %zmm28 {%k1}
+; AVX512F-FAST-NEXT:    vmovdqa64 %zmm22, %zmm29 {%k1}
 ; AVX512F-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
 ; AVX512F-FAST-NEXT:    vmovdqa64 %zmm1, %zmm17 {%k1}
-; AVX512F-FAST-NEXT:    vmovdqa64 %zmm27, %zmm9 {%k1}
-; AVX512F-FAST-NEXT:    vmovdqa64 %zmm23, %zmm5 {%k1}
-; AVX512F-FAST-NEXT:    vmovdqa64 %zmm20, %zmm18 {%k1}
+; AVX512F-FAST-NEXT:    vmovdqa64 %zmm30, %zmm9 {%k1}
+; AVX512F-FAST-NEXT:    vmovdqa64 %zmm26, %zmm31 {%k1}
+; AVX512F-FAST-NEXT:    vmovdqa64 %zmm21, %zmm8 {%k1}
 ; AVX512F-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
 ; AVX512F-FAST-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
-; AVX512F-FAST-NEXT:    vmovdqa64 %zmm19, %zmm8 {%k1}
-; AVX512F-FAST-NEXT:    vmovdqa64 %zmm0, %zmm10 {%k1}
+; AVX512F-FAST-NEXT:    vmovdqa64 %zmm19, %zmm5 {%k1}
+; AVX512F-FAST-NEXT:    vmovdqa64 %zmm23, %zmm18 {%k1}
 ; AVX512F-FAST-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512F-FAST-NEXT:    vmovdqa64 %zmm10, (%rax)
-; AVX512F-FAST-NEXT:    vmovdqa64 %zmm8, 192(%rax)
+; AVX512F-FAST-NEXT:    vmovdqa64 %zmm18, (%rax)
+; AVX512F-FAST-NEXT:    vmovdqa64 %zmm5, 192(%rax)
 ; AVX512F-FAST-NEXT:    vmovdqa64 %zmm4, 128(%rax)
-; AVX512F-FAST-NEXT:    vmovdqa64 %zmm18, 320(%rax)
-; AVX512F-FAST-NEXT:    vmovdqa64 %zmm5, 256(%rax)
+; AVX512F-FAST-NEXT:    vmovdqa64 %zmm8, 320(%rax)
+; AVX512F-FAST-NEXT:    vmovdqa64 %zmm31, 256(%rax)
 ; AVX512F-FAST-NEXT:    vmovdqa64 %zmm9, 448(%rax)
 ; AVX512F-FAST-NEXT:    vmovdqa64 %zmm17, 384(%rax)
-; AVX512F-FAST-NEXT:    vmovdqa64 %zmm31, 576(%rax)
-; AVX512F-FAST-NEXT:    vmovdqa64 %zmm30, 512(%rax)
+; AVX512F-FAST-NEXT:    vmovdqa64 %zmm29, 576(%rax)
+; AVX512F-FAST-NEXT:    vmovdqa64 %zmm28, 512(%rax)
 ; AVX512F-FAST-NEXT:    vmovdqa64 %zmm15, 704(%rax)
 ; AVX512F-FAST-NEXT:    vmovdqa64 %zmm13, 640(%rax)
-; AVX512F-FAST-NEXT:    vmovdqa64 %zmm26, 832(%rax)
-; AVX512F-FAST-NEXT:    vmovdqa64 %zmm14, 768(%rax)
+; AVX512F-FAST-NEXT:    vmovdqa64 %zmm27, 832(%rax)
+; AVX512F-FAST-NEXT:    vmovdqa64 %zmm0, 768(%rax)
 ; AVX512F-FAST-NEXT:    vmovdqa64 %zmm7, 960(%rax)
 ; AVX512F-FAST-NEXT:    vmovdqa64 %zmm6, 896(%rax)
 ; AVX512F-FAST-NEXT:    vmovdqa64 %zmm12, 64(%rax)
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-6.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-6.ll
index 299d74fdf5f1771f58e068e2129698168e7d242a..a781ae5eb896758f1ce62b5fc93bccd8bab29543 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-6.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-6.ll
@@ -265,16 +265,14 @@ define void @store_i32_stride6_vf4(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX2-ONLY-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm6
 ; AVX2-ONLY-NEXT:    vinsertf128 $1, %xmm3, %ymm2, %ymm7
 ; AVX2-ONLY-NEXT:    vinsertf128 $1, %xmm5, %ymm4, %ymm8
-; AVX2-ONLY-NEXT:    vmovddup {{.*#+}} xmm9 = [0,4,0,4]
-; AVX2-ONLY-NEXT:    # xmm9 = mem[0,0]
-; AVX2-ONLY-NEXT:    vpermps %ymm7, %ymm9, %ymm9
-; AVX2-ONLY-NEXT:    vbroadcastf128 {{.*#+}} ymm10 = [0,4,1,5,0,4,1,5]
-; AVX2-ONLY-NEXT:    # ymm10 = mem[0,1,0,1]
-; AVX2-ONLY-NEXT:    vpermps %ymm6, %ymm10, %ymm10
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm9 = ymm10[0,1],ymm9[2,3],ymm10[4,5,6,7]
-; AVX2-ONLY-NEXT:    vbroadcastsd {{.*#+}} ymm10 = [0,4,0,4,0,4,0,4]
-; AVX2-ONLY-NEXT:    vpermps %ymm8, %ymm10, %ymm10
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm9 = ymm9[0,1,2,3],ymm10[4,5],ymm9[6,7]
+; AVX2-ONLY-NEXT:    vbroadcastsd {{.*#+}} ymm9 = [0,4,0,4,0,4,0,4]
+; AVX2-ONLY-NEXT:    vpermps %ymm7, %ymm9, %ymm10
+; AVX2-ONLY-NEXT:    vbroadcastf128 {{.*#+}} ymm11 = [0,4,1,5,0,4,1,5]
+; AVX2-ONLY-NEXT:    # ymm11 = mem[0,1,0,1]
+; AVX2-ONLY-NEXT:    vpermps %ymm6, %ymm11, %ymm11
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm10 = ymm11[0,1],ymm10[2,3],ymm11[4,5,6,7]
+; AVX2-ONLY-NEXT:    vpermps %ymm8, %ymm9, %ymm9
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm9 = ymm10[0,1,2,3],ymm9[4,5],ymm10[6,7]
 ; AVX2-ONLY-NEXT:    vbroadcastsd {{.*#+}} ymm10 = [2,6,2,6,2,6,2,6]
 ; AVX2-ONLY-NEXT:    vpermps %ymm6, %ymm10, %ymm6
 ; AVX2-ONLY-NEXT:    vbroadcastf128 {{.*#+}} ymm10 = [1,5,2,6,1,5,2,6]
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-7.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-7.ll
index d24abc59341a5cbee3730f203a064ad3b867357f..e5992acfaa2a7703d3a906bb90b857533f396c9e 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-7.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-7.ll
@@ -388,15 +388,13 @@ define void @store_i32_stride7_vf4(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX2-SLOW-NEXT:    vpermps %ymm4, %ymm10, %ymm10
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm5 = ymm10[0],ymm5[1,2],ymm10[3,4,5,6,7]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0,1,2],ymm9[3,4,5],ymm5[6,7]
-; AVX2-SLOW-NEXT:    vmovddup {{.*#+}} xmm9 = [0,4,0,4]
-; AVX2-SLOW-NEXT:    # xmm9 = mem[0,0]
+; AVX2-SLOW-NEXT:    vbroadcastsd {{.*#+}} ymm9 = [0,4,0,4,0,4,0,4]
 ; AVX2-SLOW-NEXT:    vpermps %ymm6, %ymm9, %ymm6
-; AVX2-SLOW-NEXT:    vbroadcastf128 {{.*#+}} ymm9 = [0,4,0,1,0,4,0,1]
-; AVX2-SLOW-NEXT:    # ymm9 = mem[0,1,0,1]
-; AVX2-SLOW-NEXT:    vpermps %ymm4, %ymm9, %ymm4
+; AVX2-SLOW-NEXT:    vbroadcastf128 {{.*#+}} ymm10 = [0,4,0,1,0,4,0,1]
+; AVX2-SLOW-NEXT:    # ymm10 = mem[0,1,0,1]
+; AVX2-SLOW-NEXT:    vpermps %ymm4, %ymm10, %ymm4
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1],ymm6[2,3],ymm4[4,5,6,7]
-; AVX2-SLOW-NEXT:    vbroadcastsd {{.*#+}} ymm6 = [0,4,0,4,0,4,0,4]
-; AVX2-SLOW-NEXT:    vpermps %ymm7, %ymm6, %ymm6
+; AVX2-SLOW-NEXT:    vpermps %ymm7, %ymm9, %ymm6
 ; AVX2-SLOW-NEXT:    vbroadcastss (%r10), %ymm7
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm6 = ymm6[0,1,2,3,4,5],ymm7[6,7]
 ; AVX2-SLOW-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm6[4,5,6],ymm4[7]
@@ -443,15 +441,13 @@ define void @store_i32_stride7_vf4(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX2-FAST-NEXT:    vunpckhps {{.*#+}} ymm4 = ymm4[2],ymm5[2],ymm4[3],ymm5[3],ymm4[6],ymm5[6],ymm4[7],ymm5[7]
 ; AVX2-FAST-NEXT:    vshufps {{.*#+}} ymm4 = ymm4[0,1,0,1,4,5,4,5]
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm4 = ymm8[0,1],ymm4[2,3,4],ymm8[5,6,7]
-; AVX2-FAST-NEXT:    vmovddup {{.*#+}} xmm5 = [0,4,0,4]
-; AVX2-FAST-NEXT:    # xmm5 = mem[0,0]
+; AVX2-FAST-NEXT:    vbroadcastsd {{.*#+}} ymm5 = [0,4,0,4,0,4,0,4]
 ; AVX2-FAST-NEXT:    vpermps %ymm3, %ymm5, %ymm3
-; AVX2-FAST-NEXT:    vbroadcastf128 {{.*#+}} ymm5 = [0,4,0,1,0,4,0,1]
-; AVX2-FAST-NEXT:    # ymm5 = mem[0,1,0,1]
-; AVX2-FAST-NEXT:    vpermps %ymm2, %ymm5, %ymm2
+; AVX2-FAST-NEXT:    vbroadcastf128 {{.*#+}} ymm8 = [0,4,0,1,0,4,0,1]
+; AVX2-FAST-NEXT:    # ymm8 = mem[0,1,0,1]
+; AVX2-FAST-NEXT:    vpermps %ymm2, %ymm8, %ymm2
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1],ymm3[2,3],ymm2[4,5,6,7]
-; AVX2-FAST-NEXT:    vbroadcastsd {{.*#+}} ymm3 = [0,4,0,4,0,4,0,4]
-; AVX2-FAST-NEXT:    vpermps %ymm6, %ymm3, %ymm3
+; AVX2-FAST-NEXT:    vpermps %ymm6, %ymm5, %ymm3
 ; AVX2-FAST-NEXT:    vbroadcastss (%r10), %ymm5
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5],ymm5[6,7]
 ; AVX2-FAST-NEXT:    vblendps {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5,6],ymm2[7]
@@ -499,15 +495,13 @@ define void @store_i32_stride7_vf4(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm4, %ymm10, %ymm10
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm5 = ymm10[0],ymm5[1,2],ymm10[3,4,5,6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0,1,2],ymm9[3,4,5],ymm5[6,7]
-; AVX2-FAST-PERLANE-NEXT:    vmovddup {{.*#+}} xmm9 = [0,4,0,4]
-; AVX2-FAST-PERLANE-NEXT:    # xmm9 = mem[0,0]
+; AVX2-FAST-PERLANE-NEXT:    vbroadcastsd {{.*#+}} ymm9 = [0,4,0,4,0,4,0,4]
 ; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm6, %ymm9, %ymm6
-; AVX2-FAST-PERLANE-NEXT:    vbroadcastf128 {{.*#+}} ymm9 = [0,4,0,1,0,4,0,1]
-; AVX2-FAST-PERLANE-NEXT:    # ymm9 = mem[0,1,0,1]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm4, %ymm9, %ymm4
+; AVX2-FAST-PERLANE-NEXT:    vbroadcastf128 {{.*#+}} ymm10 = [0,4,0,1,0,4,0,1]
+; AVX2-FAST-PERLANE-NEXT:    # ymm10 = mem[0,1,0,1]
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm4, %ymm10, %ymm4
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1],ymm6[2,3],ymm4[4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT:    vbroadcastsd {{.*#+}} ymm6 = [0,4,0,4,0,4,0,4]
-; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm7, %ymm6, %ymm6
+; AVX2-FAST-PERLANE-NEXT:    vpermps %ymm7, %ymm9, %ymm6
 ; AVX2-FAST-PERLANE-NEXT:    vbroadcastss (%r10), %ymm7
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm6 = ymm6[0,1,2,3,4,5],ymm7[6,7]
 ; AVX2-FAST-PERLANE-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm6[4,5,6],ymm4[7]
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-8.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-8.ll
index 85ffcdf71e3cbf3aa45835720605b60a418552cd..5c844cf04e1f509de5f3ca5e939e7f46e7b47884 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-8.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i32-stride-8.ll
@@ -256,49 +256,41 @@ define void @store_i32_stride8_vf4(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX2-ONLY-NEXT:    vmovaps (%r8), %xmm5
 ; AVX2-ONLY-NEXT:    vmovaps (%r11), %xmm6
 ; AVX2-ONLY-NEXT:    vinsertf128 $1, %xmm4, %ymm3, %ymm2
-; AVX2-ONLY-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm8
+; AVX2-ONLY-NEXT:    vinsertf128 $1, %xmm1, %ymm0, %ymm7
 ; AVX2-ONLY-NEXT:    vinsertf128 $1, (%r9), %ymm5, %ymm5
-; AVX2-ONLY-NEXT:    vinsertf128 $1, (%r10), %ymm6, %ymm7
+; AVX2-ONLY-NEXT:    vinsertf128 $1, (%r10), %ymm6, %ymm8
 ; AVX2-ONLY-NEXT:    vbroadcastsd {{.*#+}} ymm6 = [0,4,0,4,0,4,0,4]
-; AVX2-ONLY-NEXT:    vpermps %ymm7, %ymm6, %ymm9
-; AVX2-ONLY-NEXT:    vpermps %ymm5, %ymm6, %ymm6
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm6 = ymm6[0,1,2,3,4,5],ymm9[6,7]
-; AVX2-ONLY-NEXT:    vmovddup {{.*#+}} xmm9 = [0,4,0,4]
-; AVX2-ONLY-NEXT:    # xmm9 = mem[0,0]
-; AVX2-ONLY-NEXT:    vpermps %ymm8, %ymm9, %ymm9
+; AVX2-ONLY-NEXT:    vpermps %ymm8, %ymm6, %ymm9
+; AVX2-ONLY-NEXT:    vpermps %ymm5, %ymm6, %ymm10
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm9 = ymm10[0,1,2,3,4,5],ymm9[6,7]
 ; AVX2-ONLY-NEXT:    vunpcklps {{.*#+}} xmm10 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm9 = xmm10[0,1],xmm9[2,3]
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm6 = ymm9[0,1,2,3],ymm6[4,5,6,7]
+; AVX2-ONLY-NEXT:    vpermps %ymm7, %ymm6, %ymm6
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm6 = xmm10[0,1],xmm6[2,3]
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm6 = ymm6[0,1,2,3],ymm9[4,5,6,7]
 ; AVX2-ONLY-NEXT:    vbroadcastsd {{.*#+}} ymm9 = [1,5,1,5,1,5,1,5]
-; AVX2-ONLY-NEXT:    vpermps %ymm7, %ymm9, %ymm10
-; AVX2-ONLY-NEXT:    vpermps %ymm5, %ymm9, %ymm9
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm9 = ymm9[0,1,2,3,4,5],ymm10[6,7]
-; AVX2-ONLY-NEXT:    vmovddup {{.*#+}} xmm10 = [1,5,1,5]
-; AVX2-ONLY-NEXT:    # xmm10 = mem[0,0]
-; AVX2-ONLY-NEXT:    vpermps %ymm2, %ymm10, %ymm10
+; AVX2-ONLY-NEXT:    vpermps %ymm8, %ymm9, %ymm10
+; AVX2-ONLY-NEXT:    vpermps %ymm5, %ymm9, %ymm11
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm10 = ymm11[0,1,2,3,4,5],ymm10[6,7]
 ; AVX2-ONLY-NEXT:    vunpcklps {{.*#+}} xmm11 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm10 = xmm10[0,1],xmm11[2,3]
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm9 = ymm10[0,1,2,3],ymm9[4,5,6,7]
+; AVX2-ONLY-NEXT:    vpermps %ymm2, %ymm9, %ymm9
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm9 = xmm9[0,1],xmm11[2,3]
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm9 = ymm9[0,1,2,3],ymm10[4,5,6,7]
 ; AVX2-ONLY-NEXT:    vbroadcastsd {{.*#+}} ymm10 = [2,6,2,6,2,6,2,6]
-; AVX2-ONLY-NEXT:    vpermps %ymm7, %ymm10, %ymm11
-; AVX2-ONLY-NEXT:    vpermps %ymm5, %ymm10, %ymm10
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm10 = ymm10[0,1,2,3,4,5],ymm11[6,7]
-; AVX2-ONLY-NEXT:    vmovddup {{.*#+}} xmm11 = [2,6,2,6]
-; AVX2-ONLY-NEXT:    # xmm11 = mem[0,0]
-; AVX2-ONLY-NEXT:    vpermps %ymm8, %ymm11, %ymm8
+; AVX2-ONLY-NEXT:    vpermps %ymm8, %ymm10, %ymm11
+; AVX2-ONLY-NEXT:    vpermps %ymm5, %ymm10, %ymm12
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm11 = ymm12[0,1,2,3,4,5],ymm11[6,7]
 ; AVX2-ONLY-NEXT:    vunpckhps {{.*#+}} xmm3 = xmm3[2],xmm4[2],xmm3[3],xmm4[3]
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm3 = xmm3[0,1],xmm8[2,3]
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm10[4,5,6,7]
+; AVX2-ONLY-NEXT:    vpermps %ymm7, %ymm10, %ymm4
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm3 = xmm3[0,1],xmm4[2,3]
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm11[4,5,6,7]
 ; AVX2-ONLY-NEXT:    vbroadcastsd {{.*#+}} ymm4 = [3,7,3,7,3,7,3,7]
-; AVX2-ONLY-NEXT:    vpermps %ymm7, %ymm4, %ymm7
-; AVX2-ONLY-NEXT:    vpermps %ymm5, %ymm4, %ymm4
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm4 = ymm4[0,1,2,3,4,5],ymm7[6,7]
-; AVX2-ONLY-NEXT:    vmovddup {{.*#+}} xmm5 = [3,7,3,7]
-; AVX2-ONLY-NEXT:    # xmm5 = mem[0,0]
-; AVX2-ONLY-NEXT:    vpermps %ymm2, %ymm5, %ymm2
+; AVX2-ONLY-NEXT:    vpermps %ymm8, %ymm4, %ymm7
+; AVX2-ONLY-NEXT:    vpermps %ymm5, %ymm4, %ymm5
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm5 = ymm5[0,1,2,3,4,5],ymm7[6,7]
 ; AVX2-ONLY-NEXT:    vunpckhps {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3]
-; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm4[4,5,6,7]
+; AVX2-ONLY-NEXT:    vpermps %ymm2, %ymm4, %ymm1
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
+; AVX2-ONLY-NEXT:    vblendps {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5,6,7]
 ; AVX2-ONLY-NEXT:    vmovaps %ymm0, 96(%rax)
 ; AVX2-ONLY-NEXT:    vmovaps %ymm3, 64(%rax)
 ; AVX2-ONLY-NEXT:    vmovaps %ymm9, 32(%rax)
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i64-stride-6.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i64-stride-6.ll
index 6ef479b875419722b82345d14867e8d689b9ae55..ab51f0bf9135cc96c65c2c01e75ecb96e4171191 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i64-stride-6.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i64-stride-6.ll
@@ -599,174 +599,170 @@ define void @store_i64_stride6_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512F-LABEL: store_i64_stride6_vf8:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512F-NEXT:    vmovdqa64 (%rdi), %zmm4
-; AVX512F-NEXT:    vmovdqa64 (%rsi), %zmm6
-; AVX512F-NEXT:    vmovdqa64 (%rdx), %zmm2
-; AVX512F-NEXT:    vmovdqa64 (%rcx), %zmm3
+; AVX512F-NEXT:    vmovdqa64 (%rdi), %zmm2
+; AVX512F-NEXT:    vmovdqa64 (%rsi), %zmm3
+; AVX512F-NEXT:    vmovdqa64 (%rdx), %zmm5
+; AVX512F-NEXT:    vmovdqa64 (%rcx), %zmm6
 ; AVX512F-NEXT:    vmovdqa64 (%r8), %zmm1
 ; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [4,12,5,13,4,12,5,13]
 ; AVX512F-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermi2q %zmm6, %zmm4, %zmm0
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm5 = [4,12,4,12]
-; AVX512F-NEXT:    # ymm5 = mem[0,1,0,1]
-; AVX512F-NEXT:    vpermi2q %zmm3, %zmm2, %zmm5
+; AVX512F-NEXT:    vpermi2q %zmm3, %zmm2, %zmm0
+; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm4 = [4,12,4,12]
+; AVX512F-NEXT:    # ymm4 = mem[0,1,0,1]
+; AVX512F-NEXT:    vpermi2q %zmm6, %zmm5, %zmm4
 ; AVX512F-NEXT:    movb $12, %r10b
 ; AVX512F-NEXT:    kmovw %r10d, %k1
-; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm0 {%k1}
+; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm0 {%k1}
 ; AVX512F-NEXT:    movb $16, %r10b
 ; AVX512F-NEXT:    kmovw %r10d, %k2
 ; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm0 {%k2}
-; AVX512F-NEXT:    vmovdqa64 (%r9), %zmm5
+; AVX512F-NEXT:    vmovdqa64 (%r9), %zmm4
 ; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm7 = [2,10,2,10,2,10,2,10]
 ; AVX512F-NEXT:    # zmm7 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermi2q %zmm6, %zmm4, %zmm7
+; AVX512F-NEXT:    vpermi2q %zmm3, %zmm2, %zmm7
 ; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm8 = [1,9,2,10,1,9,2,10]
 ; AVX512F-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermi2q %zmm3, %zmm2, %zmm8
+; AVX512F-NEXT:    vpermi2q %zmm6, %zmm5, %zmm8
 ; AVX512F-NEXT:    movb $48, %r9b
 ; AVX512F-NEXT:    kmovw %r9d, %k2
 ; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm8 {%k2}
-; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm7 = <0,1,9,u,4,5,6,7>
-; AVX512F-NEXT:    vpermi2q %zmm1, %zmm8, %zmm7
-; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm8 = [0,1,2,9,4,5,6,7]
-; AVX512F-NEXT:    vpermi2q %zmm5, %zmm7, %zmm8
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm7 = [6,14,6,14,6,14,6,14]
-; AVX512F-NEXT:    # zmm7 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermi2q %zmm6, %zmm4, %zmm7
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm9 = <0,1,9,u,4,5,6,7>
+; AVX512F-NEXT:    vpermi2q %zmm1, %zmm8, %zmm9
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm7 = [0,1,2,9,4,5,6,7]
+; AVX512F-NEXT:    vpermi2q %zmm4, %zmm9, %zmm7
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm8 = [6,14,6,14,6,14,6,14]
+; AVX512F-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpermi2q %zmm3, %zmm2, %zmm8
 ; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm9 = [5,13,6,14,5,13,6,14]
 ; AVX512F-NEXT:    # zmm9 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermi2q %zmm3, %zmm2, %zmm9
-; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm9 {%k2}
-; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm7 = <0,1,13,u,4,5,6,7>
-; AVX512F-NEXT:    vpermi2q %zmm1, %zmm9, %zmm7
+; AVX512F-NEXT:    vpermi2q %zmm6, %zmm5, %zmm9
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm9 {%k2}
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm8 = <0,1,13,u,4,5,6,7>
+; AVX512F-NEXT:    vpermi2q %zmm1, %zmm9, %zmm8
 ; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm9 = [0,1,2,13,4,5,6,7]
-; AVX512F-NEXT:    vpermi2q %zmm5, %zmm7, %zmm9
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm7 = [0,8,1,9,0,8,1,9]
-; AVX512F-NEXT:    # zmm7 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermi2q %zmm6, %zmm4, %zmm7
-; AVX512F-NEXT:    vmovdqa (%rdx), %xmm10
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} xmm10 = xmm10[0],mem[0]
-; AVX512F-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm10, %zmm0, %zmm7 {%k1}
-; AVX512F-NEXT:    vinserti32x4 $2, (%r8), %zmm7, %zmm7
-; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm10 = [0,1,2,3,4,8,6,7]
-; AVX512F-NEXT:    vpermi2q %zmm5, %zmm7, %zmm10
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm7 = [7,15,7,15,7,15,7,15]
-; AVX512F-NEXT:    # zmm7 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermi2q %zmm3, %zmm2, %zmm7
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm11 = [7,15,7,15]
-; AVX512F-NEXT:    # ymm11 = mem[0,1,0,1]
-; AVX512F-NEXT:    vpermi2q %zmm6, %zmm4, %zmm11
-; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm4 = zmm11[0,1,2,3],zmm7[4,5,6,7]
+; AVX512F-NEXT:    vpermi2q %zmm4, %zmm8, %zmm9
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm8 = [7,15,7,15,7,15,7,15]
+; AVX512F-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm10 = [3,11,3,11,3,11,3,11]
+; AVX512F-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpermi2q %zmm6, %zmm5, %zmm10
+; AVX512F-NEXT:    vpermt2q %zmm6, %zmm8, %zmm5
+; AVX512F-NEXT:    vpermi2q %zmm3, %zmm2, %zmm8
+; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm5 = zmm8[0,1,2,3],zmm5[4,5,6,7]
 ; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm6 = <14,u,2,3,4,5,15,u>
-; AVX512F-NEXT:    vpermi2q %zmm1, %zmm4, %zmm6
-; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [0,14,2,3,4,5,6,15]
-; AVX512F-NEXT:    vpermi2q %zmm5, %zmm6, %zmm4
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm6 = [3,11,3,11,3,11,3,11]
-; AVX512F-NEXT:    # zmm6 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpermi2q %zmm1, %zmm5, %zmm6
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [0,14,2,3,4,5,6,15]
+; AVX512F-NEXT:    vpermi2q %zmm4, %zmm6, %zmm5
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm6 = [0,8,1,9,0,8,1,9]
+; AVX512F-NEXT:    # zmm6 = mem[0,1,2,3,0,1,2,3]
 ; AVX512F-NEXT:    vpermi2q %zmm3, %zmm2, %zmm6
+; AVX512F-NEXT:    vmovdqa (%rdx), %xmm2
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],mem[0]
+; AVX512F-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm0, %zmm6 {%k1}
+; AVX512F-NEXT:    vinserti32x4 $2, (%r8), %zmm6, %zmm2
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [0,1,2,3,4,8,6,7]
+; AVX512F-NEXT:    vpermi2q %zmm4, %zmm2, %zmm3
 ; AVX512F-NEXT:    vmovdqa (%rdi), %ymm2
 ; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm2 = ymm2[1],mem[1],ymm2[3],mem[3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm6, %zmm2
-; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm3 = <10,u,2,3,4,5,11,u>
-; AVX512F-NEXT:    vpermi2q %zmm1, %zmm2, %zmm3
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm10, %zmm2
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm6 = <10,u,2,3,4,5,11,u>
+; AVX512F-NEXT:    vpermi2q %zmm1, %zmm2, %zmm6
 ; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,10,2,3,4,5,6,11]
-; AVX512F-NEXT:    vpermi2q %zmm5, %zmm3, %zmm1
+; AVX512F-NEXT:    vpermi2q %zmm4, %zmm6, %zmm1
 ; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [0,1,2,3,4,12,6,7]
-; AVX512F-NEXT:    vpermi2q %zmm5, %zmm0, %zmm2
+; AVX512F-NEXT:    vpermi2q %zmm4, %zmm0, %zmm2
 ; AVX512F-NEXT:    vmovdqa64 %zmm2, 192(%rax)
 ; AVX512F-NEXT:    vmovdqa64 %zmm1, 128(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm4, 320(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm5, 320(%rax)
 ; AVX512F-NEXT:    vmovdqa64 %zmm9, 256(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm8, 64(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm10, (%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm7, 64(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm3, (%rax)
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: store_i64_stride6_vf8:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm4
-; AVX512BW-NEXT:    vmovdqa64 (%rsi), %zmm6
-; AVX512BW-NEXT:    vmovdqa64 (%rdx), %zmm2
-; AVX512BW-NEXT:    vmovdqa64 (%rcx), %zmm3
+; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm2
+; AVX512BW-NEXT:    vmovdqa64 (%rsi), %zmm3
+; AVX512BW-NEXT:    vmovdqa64 (%rdx), %zmm5
+; AVX512BW-NEXT:    vmovdqa64 (%rcx), %zmm6
 ; AVX512BW-NEXT:    vmovdqa64 (%r8), %zmm1
 ; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [4,12,5,13,4,12,5,13]
 ; AVX512BW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermi2q %zmm6, %zmm4, %zmm0
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm5 = [4,12,4,12]
-; AVX512BW-NEXT:    # ymm5 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vpermi2q %zmm3, %zmm2, %zmm5
+; AVX512BW-NEXT:    vpermi2q %zmm3, %zmm2, %zmm0
+; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm4 = [4,12,4,12]
+; AVX512BW-NEXT:    # ymm4 = mem[0,1,0,1]
+; AVX512BW-NEXT:    vpermi2q %zmm6, %zmm5, %zmm4
 ; AVX512BW-NEXT:    movb $12, %r10b
 ; AVX512BW-NEXT:    kmovd %r10d, %k1
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm0 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm0 {%k1}
 ; AVX512BW-NEXT:    movb $16, %r10b
 ; AVX512BW-NEXT:    kmovd %r10d, %k2
 ; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm0 {%k2}
-; AVX512BW-NEXT:    vmovdqa64 (%r9), %zmm5
+; AVX512BW-NEXT:    vmovdqa64 (%r9), %zmm4
 ; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm7 = [2,10,2,10,2,10,2,10]
 ; AVX512BW-NEXT:    # zmm7 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermi2q %zmm6, %zmm4, %zmm7
+; AVX512BW-NEXT:    vpermi2q %zmm3, %zmm2, %zmm7
 ; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm8 = [1,9,2,10,1,9,2,10]
 ; AVX512BW-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermi2q %zmm3, %zmm2, %zmm8
+; AVX512BW-NEXT:    vpermi2q %zmm6, %zmm5, %zmm8
 ; AVX512BW-NEXT:    movb $48, %r9b
 ; AVX512BW-NEXT:    kmovd %r9d, %k2
 ; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm8 {%k2}
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm7 = <0,1,9,u,4,5,6,7>
-; AVX512BW-NEXT:    vpermi2q %zmm1, %zmm8, %zmm7
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm8 = [0,1,2,9,4,5,6,7]
-; AVX512BW-NEXT:    vpermi2q %zmm5, %zmm7, %zmm8
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm7 = [6,14,6,14,6,14,6,14]
-; AVX512BW-NEXT:    # zmm7 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermi2q %zmm6, %zmm4, %zmm7
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm9 = <0,1,9,u,4,5,6,7>
+; AVX512BW-NEXT:    vpermi2q %zmm1, %zmm8, %zmm9
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm7 = [0,1,2,9,4,5,6,7]
+; AVX512BW-NEXT:    vpermi2q %zmm4, %zmm9, %zmm7
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm8 = [6,14,6,14,6,14,6,14]
+; AVX512BW-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpermi2q %zmm3, %zmm2, %zmm8
 ; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm9 = [5,13,6,14,5,13,6,14]
 ; AVX512BW-NEXT:    # zmm9 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermi2q %zmm3, %zmm2, %zmm9
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm9 {%k2}
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm7 = <0,1,13,u,4,5,6,7>
-; AVX512BW-NEXT:    vpermi2q %zmm1, %zmm9, %zmm7
+; AVX512BW-NEXT:    vpermi2q %zmm6, %zmm5, %zmm9
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm9 {%k2}
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm8 = <0,1,13,u,4,5,6,7>
+; AVX512BW-NEXT:    vpermi2q %zmm1, %zmm9, %zmm8
 ; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm9 = [0,1,2,13,4,5,6,7]
-; AVX512BW-NEXT:    vpermi2q %zmm5, %zmm7, %zmm9
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm7 = [0,8,1,9,0,8,1,9]
-; AVX512BW-NEXT:    # zmm7 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermi2q %zmm6, %zmm4, %zmm7
-; AVX512BW-NEXT:    vmovdqa (%rdx), %xmm10
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} xmm10 = xmm10[0],mem[0]
-; AVX512BW-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm10, %zmm0, %zmm7 {%k1}
-; AVX512BW-NEXT:    vinserti32x4 $2, (%r8), %zmm7, %zmm7
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm10 = [0,1,2,3,4,8,6,7]
-; AVX512BW-NEXT:    vpermi2q %zmm5, %zmm7, %zmm10
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm7 = [7,15,7,15,7,15,7,15]
-; AVX512BW-NEXT:    # zmm7 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermi2q %zmm3, %zmm2, %zmm7
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm11 = [7,15,7,15]
-; AVX512BW-NEXT:    # ymm11 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vpermi2q %zmm6, %zmm4, %zmm11
-; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm4 = zmm11[0,1,2,3],zmm7[4,5,6,7]
+; AVX512BW-NEXT:    vpermi2q %zmm4, %zmm8, %zmm9
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm8 = [7,15,7,15,7,15,7,15]
+; AVX512BW-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm10 = [3,11,3,11,3,11,3,11]
+; AVX512BW-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpermi2q %zmm6, %zmm5, %zmm10
+; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm8, %zmm5
+; AVX512BW-NEXT:    vpermi2q %zmm3, %zmm2, %zmm8
+; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm5 = zmm8[0,1,2,3],zmm5[4,5,6,7]
 ; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm6 = <14,u,2,3,4,5,15,u>
-; AVX512BW-NEXT:    vpermi2q %zmm1, %zmm4, %zmm6
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [0,14,2,3,4,5,6,15]
-; AVX512BW-NEXT:    vpermi2q %zmm5, %zmm6, %zmm4
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm6 = [3,11,3,11,3,11,3,11]
-; AVX512BW-NEXT:    # zmm6 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpermi2q %zmm1, %zmm5, %zmm6
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [0,14,2,3,4,5,6,15]
+; AVX512BW-NEXT:    vpermi2q %zmm4, %zmm6, %zmm5
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm6 = [0,8,1,9,0,8,1,9]
+; AVX512BW-NEXT:    # zmm6 = mem[0,1,2,3,0,1,2,3]
 ; AVX512BW-NEXT:    vpermi2q %zmm3, %zmm2, %zmm6
+; AVX512BW-NEXT:    vmovdqa (%rdx), %xmm2
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],mem[0]
+; AVX512BW-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm0, %zmm6 {%k1}
+; AVX512BW-NEXT:    vinserti32x4 $2, (%r8), %zmm6, %zmm2
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [0,1,2,3,4,8,6,7]
+; AVX512BW-NEXT:    vpermi2q %zmm4, %zmm2, %zmm3
 ; AVX512BW-NEXT:    vmovdqa (%rdi), %ymm2
 ; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm2 = ymm2[1],mem[1],ymm2[3],mem[3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm6, %zmm2
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm3 = <10,u,2,3,4,5,11,u>
-; AVX512BW-NEXT:    vpermi2q %zmm1, %zmm2, %zmm3
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm10, %zmm2
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm6 = <10,u,2,3,4,5,11,u>
+; AVX512BW-NEXT:    vpermi2q %zmm1, %zmm2, %zmm6
 ; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,10,2,3,4,5,6,11]
-; AVX512BW-NEXT:    vpermi2q %zmm5, %zmm3, %zmm1
+; AVX512BW-NEXT:    vpermi2q %zmm4, %zmm6, %zmm1
 ; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [0,1,2,3,4,12,6,7]
-; AVX512BW-NEXT:    vpermi2q %zmm5, %zmm0, %zmm2
+; AVX512BW-NEXT:    vpermi2q %zmm4, %zmm0, %zmm2
 ; AVX512BW-NEXT:    vmovdqa64 %zmm2, 192(%rax)
 ; AVX512BW-NEXT:    vmovdqa64 %zmm1, 128(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm4, 320(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, 320(%rax)
 ; AVX512BW-NEXT:    vmovdqa64 %zmm9, 256(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm8, 64(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, (%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, 64(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, (%rax)
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
   %in.vec0 = load <8 x i64>, ptr %in.vecptr0, align 64
@@ -1445,20 +1441,20 @@ define void @store_i64_stride6_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512F-LABEL: store_i64_stride6_vf16:
 ; AVX512F:       # %bb.0:
 ; AVX512F-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512F-NEXT:    vmovdqa64 64(%rdi), %zmm13
-; AVX512F-NEXT:    vmovdqa64 (%rdi), %zmm5
-; AVX512F-NEXT:    vmovdqa64 (%rsi), %zmm6
-; AVX512F-NEXT:    vmovdqa64 64(%rsi), %zmm14
-; AVX512F-NEXT:    vmovdqa64 64(%rdx), %zmm7
+; AVX512F-NEXT:    vmovdqa64 64(%rdi), %zmm8
+; AVX512F-NEXT:    vmovdqa64 (%rdi), %zmm12
+; AVX512F-NEXT:    vmovdqa64 (%rsi), %zmm15
+; AVX512F-NEXT:    vmovdqa64 64(%rsi), %zmm11
+; AVX512F-NEXT:    vmovdqa64 64(%rdx), %zmm5
 ; AVX512F-NEXT:    vmovdqa64 (%rdx), %zmm2
 ; AVX512F-NEXT:    vmovdqa64 (%rcx), %zmm4
-; AVX512F-NEXT:    vmovdqa64 64(%rcx), %zmm11
+; AVX512F-NEXT:    vmovdqa64 64(%rcx), %zmm7
 ; AVX512F-NEXT:    vmovdqa64 (%r8), %zmm3
-; AVX512F-NEXT:    vmovdqa64 64(%r8), %zmm8
+; AVX512F-NEXT:    vmovdqa64 64(%r8), %zmm6
 ; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [4,12,5,13,4,12,5,13]
 ; AVX512F-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm1
-; AVX512F-NEXT:    vpermt2q %zmm6, %zmm0, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm12, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm15, %zmm0, %zmm1
 ; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm9 = [4,12,4,12]
 ; AVX512F-NEXT:    # ymm9 = mem[0,1,0,1]
 ; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm10
@@ -1469,136 +1465,135 @@ define void @store_i64_stride6_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512F-NEXT:    movb $16, %r10b
 ; AVX512F-NEXT:    kmovw %r10d, %k2
 ; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm1 {%k2}
-; AVX512F-NEXT:    vpermi2q %zmm14, %zmm13, %zmm0
-; AVX512F-NEXT:    vpermi2q %zmm11, %zmm7, %zmm9
+; AVX512F-NEXT:    vpermi2q %zmm11, %zmm8, %zmm0
+; AVX512F-NEXT:    vpermi2q %zmm7, %zmm5, %zmm9
 ; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm0 {%k1}
-; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm0 {%k2}
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm0 {%k2}
 ; AVX512F-NEXT:    vmovdqa64 (%r9), %zmm10
-; AVX512F-NEXT:    vmovdqa64 64(%r9), %zmm16
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm18 = [6,14,6,14,6,14,6,14]
-; AVX512F-NEXT:    # zmm18 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm15
-; AVX512F-NEXT:    vpermt2q %zmm14, %zmm18, %zmm15
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm12 = [5,13,6,14,5,13,6,14]
-; AVX512F-NEXT:    # zmm12 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm9
-; AVX512F-NEXT:    vpermt2q %zmm11, %zmm12, %zmm9
+; AVX512F-NEXT:    vmovdqa64 64(%r9), %zmm14
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm19 = [7,15,7,15,7,15,7,15]
+; AVX512F-NEXT:    # zmm19 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm9
+; AVX512F-NEXT:    vpermt2q %zmm7, %zmm19, %zmm9
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm13
+; AVX512F-NEXT:    vpermt2q %zmm11, %zmm19, %zmm13
+; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm9 = zmm13[0,1,2,3],zmm9[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm20 = <14,u,2,3,4,5,15,u>
+; AVX512F-NEXT:    vpermt2q %zmm6, %zmm20, %zmm9
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm21 = [0,14,2,3,4,5,6,15]
+; AVX512F-NEXT:    vpermt2q %zmm14, %zmm21, %zmm9
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm22 = [6,14,6,14,6,14,6,14]
+; AVX512F-NEXT:    # zmm22 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm17
+; AVX512F-NEXT:    vpermt2q %zmm11, %zmm22, %zmm17
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm16 = [5,13,6,14,5,13,6,14]
+; AVX512F-NEXT:    # zmm16 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm13
+; AVX512F-NEXT:    vpermt2q %zmm7, %zmm16, %zmm13
 ; AVX512F-NEXT:    movb $48, %r9b
 ; AVX512F-NEXT:    kmovw %r9d, %k2
-; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm9 {%k2}
-; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm19 = <0,1,13,u,4,5,6,7>
-; AVX512F-NEXT:    vpermt2q %zmm8, %zmm19, %zmm9
-; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm20 = [0,1,2,13,4,5,6,7]
-; AVX512F-NEXT:    vpermt2q %zmm16, %zmm20, %zmm9
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm21 = [2,10,2,10,2,10,2,10]
-; AVX512F-NEXT:    # zmm21 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm22
-; AVX512F-NEXT:    vpermt2q %zmm14, %zmm21, %zmm22
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm17 = [1,9,2,10,1,9,2,10]
-; AVX512F-NEXT:    # zmm17 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm15
-; AVX512F-NEXT:    vpermt2q %zmm11, %zmm17, %zmm15
-; AVX512F-NEXT:    vmovdqa64 %zmm22, %zmm15 {%k2}
-; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm22 = <0,1,9,u,4,5,6,7>
-; AVX512F-NEXT:    vpermt2q %zmm8, %zmm22, %zmm15
-; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm23 = [0,1,2,9,4,5,6,7]
-; AVX512F-NEXT:    vpermt2q %zmm16, %zmm23, %zmm15
-; AVX512F-NEXT:    vpermi2q %zmm6, %zmm5, %zmm18
-; AVX512F-NEXT:    vpermi2q %zmm4, %zmm2, %zmm12
-; AVX512F-NEXT:    vmovdqa64 %zmm18, %zmm12 {%k2}
-; AVX512F-NEXT:    vpermt2q %zmm3, %zmm19, %zmm12
-; AVX512F-NEXT:    vpermt2q %zmm10, %zmm20, %zmm12
-; AVX512F-NEXT:    vpermi2q %zmm6, %zmm5, %zmm21
-; AVX512F-NEXT:    vpermi2q %zmm4, %zmm2, %zmm17
-; AVX512F-NEXT:    vmovdqa64 %zmm21, %zmm17 {%k2}
-; AVX512F-NEXT:    vpermt2q %zmm3, %zmm22, %zmm17
-; AVX512F-NEXT:    vpermt2q %zmm10, %zmm23, %zmm17
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm19 = [0,8,1,9,0,8,1,9]
-; AVX512F-NEXT:    # zmm19 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm18
-; AVX512F-NEXT:    vpermt2q %zmm6, %zmm19, %zmm18
-; AVX512F-NEXT:    vmovdqa64 (%rdx), %xmm20
+; AVX512F-NEXT:    vmovdqa64 %zmm17, %zmm13 {%k2}
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm23 = <0,1,13,u,4,5,6,7>
+; AVX512F-NEXT:    vpermt2q %zmm6, %zmm23, %zmm13
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm24 = [0,1,2,13,4,5,6,7]
+; AVX512F-NEXT:    vpermt2q %zmm14, %zmm24, %zmm13
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm25 = [2,10,2,10,2,10,2,10]
+; AVX512F-NEXT:    # zmm25 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm26
+; AVX512F-NEXT:    vpermt2q %zmm11, %zmm25, %zmm26
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm18 = [1,9,2,10,1,9,2,10]
+; AVX512F-NEXT:    # zmm18 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm17
+; AVX512F-NEXT:    vpermt2q %zmm7, %zmm18, %zmm17
+; AVX512F-NEXT:    vmovdqa64 %zmm26, %zmm17 {%k2}
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm26 = <0,1,9,u,4,5,6,7>
+; AVX512F-NEXT:    vpermt2q %zmm6, %zmm26, %zmm17
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm27 = [0,1,2,9,4,5,6,7]
+; AVX512F-NEXT:    vpermt2q %zmm14, %zmm27, %zmm17
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm28
+; AVX512F-NEXT:    vpermt2q %zmm4, %zmm19, %zmm28
+; AVX512F-NEXT:    vpermi2q %zmm15, %zmm12, %zmm19
+; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm19 = zmm19[0,1,2,3],zmm28[4,5,6,7]
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm20, %zmm19
+; AVX512F-NEXT:    vpermt2q %zmm10, %zmm21, %zmm19
+; AVX512F-NEXT:    vpermi2q %zmm15, %zmm12, %zmm22
+; AVX512F-NEXT:    vpermi2q %zmm4, %zmm2, %zmm16
+; AVX512F-NEXT:    vmovdqa64 %zmm22, %zmm16 {%k2}
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm23, %zmm16
+; AVX512F-NEXT:    vpermt2q %zmm10, %zmm24, %zmm16
+; AVX512F-NEXT:    vpermi2q %zmm15, %zmm12, %zmm25
+; AVX512F-NEXT:    vpermi2q %zmm4, %zmm2, %zmm18
+; AVX512F-NEXT:    vmovdqa64 %zmm25, %zmm18 {%k2}
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm26, %zmm18
+; AVX512F-NEXT:    vpermt2q %zmm10, %zmm27, %zmm18
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm20 = [0,8,1,9,0,8,1,9]
+; AVX512F-NEXT:    # zmm20 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpermt2q %zmm15, %zmm20, %zmm12
+; AVX512F-NEXT:    vmovdqa (%rdx), %xmm15
 ; AVX512F-NEXT:    vmovdqa64 64(%rdx), %xmm21
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} xmm20 = xmm20[0],mem[0]
-; AVX512F-NEXT:    vinserti32x4 $1, %xmm20, %ymm0, %ymm20
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm20, %zmm0, %zmm18 {%k1}
-; AVX512F-NEXT:    vinserti32x4 $2, (%r8), %zmm18, %zmm18
-; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm20 = [0,1,2,3,4,8,6,7]
-; AVX512F-NEXT:    vpermt2q %zmm10, %zmm20, %zmm18
-; AVX512F-NEXT:    vpermi2q %zmm14, %zmm13, %zmm19
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} xmm21 = xmm21[0],mem[0]
-; AVX512F-NEXT:    vinserti32x4 $1, %xmm21, %ymm0, %ymm21
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm21, %zmm0, %zmm19 {%k1}
-; AVX512F-NEXT:    vinserti32x4 $2, 64(%r8), %zmm19, %zmm19
-; AVX512F-NEXT:    vpermt2q %zmm16, %zmm20, %zmm19
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm20 = [7,15,7,15,7,15,7,15]
-; AVX512F-NEXT:    # zmm20 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm21
-; AVX512F-NEXT:    vpermt2q %zmm11, %zmm20, %zmm21
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} ymm22 = [7,15,7,15]
-; AVX512F-NEXT:    # ymm22 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermt2q %zmm14, %zmm22, %zmm13
-; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm13 = zmm13[0,1,2,3],zmm21[4,5,6,7]
-; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm14 = <14,u,2,3,4,5,15,u>
-; AVX512F-NEXT:    vpermt2q %zmm8, %zmm14, %zmm13
-; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm21 = [0,14,2,3,4,5,6,15]
-; AVX512F-NEXT:    vpermt2q %zmm16, %zmm21, %zmm13
-; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm23 = [0,1,2,3,4,12,6,7]
-; AVX512F-NEXT:    vpermt2q %zmm16, %zmm23, %zmm0
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm24 = [3,11,3,11,3,11,3,11]
-; AVX512F-NEXT:    # zmm24 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermt2q %zmm11, %zmm24, %zmm7
-; AVX512F-NEXT:    vmovdqa (%rdi), %ymm11
-; AVX512F-NEXT:    vmovdqa64 64(%rdi), %ymm25
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm25 = ymm25[1],mem[1],ymm25[3],mem[3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm25, %zmm7, %zmm7
-; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm25 = <10,u,2,3,4,5,11,u>
-; AVX512F-NEXT:    vpermt2q %zmm8, %zmm25, %zmm7
-; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm8 = [0,10,2,3,4,5,6,11]
-; AVX512F-NEXT:    vpermt2q %zmm16, %zmm8, %zmm7
-; AVX512F-NEXT:    vpermi2q %zmm4, %zmm2, %zmm20
-; AVX512F-NEXT:    vpermt2q %zmm6, %zmm22, %zmm5
-; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm5 = zmm5[0,1,2,3],zmm20[4,5,6,7]
-; AVX512F-NEXT:    vpermt2q %zmm3, %zmm14, %zmm5
-; AVX512F-NEXT:    vpermt2q %zmm10, %zmm21, %zmm5
-; AVX512F-NEXT:    vpermt2q %zmm10, %zmm23, %zmm1
-; AVX512F-NEXT:    vpermt2q %zmm4, %zmm24, %zmm2
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm4 = ymm11[1],mem[1],ymm11[3],mem[3]
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} xmm15 = xmm15[0],mem[0]
+; AVX512F-NEXT:    vinserti128 $1, %xmm15, %ymm0, %ymm15
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm15, %zmm0, %zmm12 {%k1}
+; AVX512F-NEXT:    vinserti32x4 $2, (%r8), %zmm12, %zmm12
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm15 = [0,1,2,3,4,8,6,7]
+; AVX512F-NEXT:    vpermt2q %zmm10, %zmm15, %zmm12
+; AVX512F-NEXT:    vpermt2q %zmm11, %zmm20, %zmm8
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} xmm11 = xmm21[0],mem[0]
+; AVX512F-NEXT:    vinserti128 $1, %xmm11, %ymm0, %ymm11
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm11, %zmm0, %zmm8 {%k1}
+; AVX512F-NEXT:    vinserti32x4 $2, 64(%r8), %zmm8, %zmm8
+; AVX512F-NEXT:    vpermt2q %zmm14, %zmm15, %zmm8
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm11 = [0,1,2,3,4,12,6,7]
+; AVX512F-NEXT:    vpermt2q %zmm14, %zmm11, %zmm0
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm15 = [3,11,3,11,3,11,3,11]
+; AVX512F-NEXT:    # zmm15 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpermt2q %zmm7, %zmm15, %zmm5
+; AVX512F-NEXT:    vmovdqa (%rdi), %ymm7
+; AVX512F-NEXT:    vmovdqa64 64(%rdi), %ymm20
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm20 = ymm20[1],mem[1],ymm20[3],mem[3]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm20, %zmm5, %zmm5
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm20 = <10,u,2,3,4,5,11,u>
+; AVX512F-NEXT:    vpermt2q %zmm6, %zmm20, %zmm5
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [0,10,2,3,4,5,6,11]
+; AVX512F-NEXT:    vpermt2q %zmm14, %zmm6, %zmm5
+; AVX512F-NEXT:    vpermt2q %zmm10, %zmm11, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm4, %zmm15, %zmm2
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm4 = ymm7[1],mem[1],ymm7[3],mem[3]
 ; AVX512F-NEXT:    vinserti64x4 $0, %ymm4, %zmm2, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm3, %zmm25, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm10, %zmm8, %zmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm17, 64(%rax)
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm20, %zmm2
+; AVX512F-NEXT:    vpermt2q %zmm10, %zmm6, %zmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm18, 64(%rax)
 ; AVX512F-NEXT:    vmovdqa64 %zmm2, 128(%rax)
 ; AVX512F-NEXT:    vmovdqa64 %zmm1, 192(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm12, 256(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm5, 320(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm15, 448(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm7, 512(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm16, 256(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm19, 320(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm17, 448(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm5, 512(%rax)
 ; AVX512F-NEXT:    vmovdqa64 %zmm0, 576(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm9, 640(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm13, 704(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm19, 384(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm18, (%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm13, 640(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm9, 704(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm8, 384(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm12, (%rax)
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: store_i64_stride6_vf16:
 ; AVX512BW:       # %bb.0:
 ; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512BW-NEXT:    vmovdqa64 64(%rdi), %zmm13
-; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm5
-; AVX512BW-NEXT:    vmovdqa64 (%rsi), %zmm6
-; AVX512BW-NEXT:    vmovdqa64 64(%rsi), %zmm14
-; AVX512BW-NEXT:    vmovdqa64 64(%rdx), %zmm7
+; AVX512BW-NEXT:    vmovdqa64 64(%rdi), %zmm8
+; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm12
+; AVX512BW-NEXT:    vmovdqa64 (%rsi), %zmm15
+; AVX512BW-NEXT:    vmovdqa64 64(%rsi), %zmm11
+; AVX512BW-NEXT:    vmovdqa64 64(%rdx), %zmm5
 ; AVX512BW-NEXT:    vmovdqa64 (%rdx), %zmm2
 ; AVX512BW-NEXT:    vmovdqa64 (%rcx), %zmm4
-; AVX512BW-NEXT:    vmovdqa64 64(%rcx), %zmm11
+; AVX512BW-NEXT:    vmovdqa64 64(%rcx), %zmm7
 ; AVX512BW-NEXT:    vmovdqa64 (%r8), %zmm3
-; AVX512BW-NEXT:    vmovdqa64 64(%r8), %zmm8
+; AVX512BW-NEXT:    vmovdqa64 64(%r8), %zmm6
 ; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [4,12,5,13,4,12,5,13]
 ; AVX512BW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm1
-; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm0, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm15, %zmm0, %zmm1
 ; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm9 = [4,12,4,12]
 ; AVX512BW-NEXT:    # ymm9 = mem[0,1,0,1]
 ; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm10
@@ -1609,116 +1604,115 @@ define void @store_i64_stride6_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512BW-NEXT:    movb $16, %r10b
 ; AVX512BW-NEXT:    kmovd %r10d, %k2
 ; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm1 {%k2}
-; AVX512BW-NEXT:    vpermi2q %zmm14, %zmm13, %zmm0
-; AVX512BW-NEXT:    vpermi2q %zmm11, %zmm7, %zmm9
+; AVX512BW-NEXT:    vpermi2q %zmm11, %zmm8, %zmm0
+; AVX512BW-NEXT:    vpermi2q %zmm7, %zmm5, %zmm9
 ; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm0 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm0 {%k2}
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm0 {%k2}
 ; AVX512BW-NEXT:    vmovdqa64 (%r9), %zmm10
-; AVX512BW-NEXT:    vmovdqa64 64(%r9), %zmm16
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm18 = [6,14,6,14,6,14,6,14]
-; AVX512BW-NEXT:    # zmm18 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm15
-; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm18, %zmm15
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm12 = [5,13,6,14,5,13,6,14]
-; AVX512BW-NEXT:    # zmm12 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm9
-; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm12, %zmm9
+; AVX512BW-NEXT:    vmovdqa64 64(%r9), %zmm14
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm19 = [7,15,7,15,7,15,7,15]
+; AVX512BW-NEXT:    # zmm19 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm9
+; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm19, %zmm9
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm13
+; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm19, %zmm13
+; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm9 = zmm13[0,1,2,3],zmm9[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm20 = <14,u,2,3,4,5,15,u>
+; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm20, %zmm9
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm21 = [0,14,2,3,4,5,6,15]
+; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm21, %zmm9
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm22 = [6,14,6,14,6,14,6,14]
+; AVX512BW-NEXT:    # zmm22 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm17
+; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm22, %zmm17
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm16 = [5,13,6,14,5,13,6,14]
+; AVX512BW-NEXT:    # zmm16 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm13
+; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm16, %zmm13
 ; AVX512BW-NEXT:    movb $48, %r9b
 ; AVX512BW-NEXT:    kmovd %r9d, %k2
-; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm9 {%k2}
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm19 = <0,1,13,u,4,5,6,7>
-; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm19, %zmm9
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm20 = [0,1,2,13,4,5,6,7]
-; AVX512BW-NEXT:    vpermt2q %zmm16, %zmm20, %zmm9
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm21 = [2,10,2,10,2,10,2,10]
-; AVX512BW-NEXT:    # zmm21 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm22
-; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm21, %zmm22
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm17 = [1,9,2,10,1,9,2,10]
-; AVX512BW-NEXT:    # zmm17 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm15
-; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm17, %zmm15
-; AVX512BW-NEXT:    vmovdqa64 %zmm22, %zmm15 {%k2}
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm22 = <0,1,9,u,4,5,6,7>
-; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm22, %zmm15
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm23 = [0,1,2,9,4,5,6,7]
-; AVX512BW-NEXT:    vpermt2q %zmm16, %zmm23, %zmm15
-; AVX512BW-NEXT:    vpermi2q %zmm6, %zmm5, %zmm18
-; AVX512BW-NEXT:    vpermi2q %zmm4, %zmm2, %zmm12
-; AVX512BW-NEXT:    vmovdqa64 %zmm18, %zmm12 {%k2}
-; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm19, %zmm12
-; AVX512BW-NEXT:    vpermt2q %zmm10, %zmm20, %zmm12
-; AVX512BW-NEXT:    vpermi2q %zmm6, %zmm5, %zmm21
-; AVX512BW-NEXT:    vpermi2q %zmm4, %zmm2, %zmm17
-; AVX512BW-NEXT:    vmovdqa64 %zmm21, %zmm17 {%k2}
-; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm22, %zmm17
-; AVX512BW-NEXT:    vpermt2q %zmm10, %zmm23, %zmm17
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm19 = [0,8,1,9,0,8,1,9]
-; AVX512BW-NEXT:    # zmm19 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm18
-; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm19, %zmm18
-; AVX512BW-NEXT:    vmovdqa64 (%rdx), %xmm20
+; AVX512BW-NEXT:    vmovdqa64 %zmm17, %zmm13 {%k2}
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm23 = <0,1,13,u,4,5,6,7>
+; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm23, %zmm13
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm24 = [0,1,2,13,4,5,6,7]
+; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm24, %zmm13
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm25 = [2,10,2,10,2,10,2,10]
+; AVX512BW-NEXT:    # zmm25 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm26
+; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm25, %zmm26
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm18 = [1,9,2,10,1,9,2,10]
+; AVX512BW-NEXT:    # zmm18 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm17
+; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm18, %zmm17
+; AVX512BW-NEXT:    vmovdqa64 %zmm26, %zmm17 {%k2}
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm26 = <0,1,9,u,4,5,6,7>
+; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm26, %zmm17
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm27 = [0,1,2,9,4,5,6,7]
+; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm27, %zmm17
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm28
+; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm19, %zmm28
+; AVX512BW-NEXT:    vpermi2q %zmm15, %zmm12, %zmm19
+; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm19 = zmm19[0,1,2,3],zmm28[4,5,6,7]
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm20, %zmm19
+; AVX512BW-NEXT:    vpermt2q %zmm10, %zmm21, %zmm19
+; AVX512BW-NEXT:    vpermi2q %zmm15, %zmm12, %zmm22
+; AVX512BW-NEXT:    vpermi2q %zmm4, %zmm2, %zmm16
+; AVX512BW-NEXT:    vmovdqa64 %zmm22, %zmm16 {%k2}
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm23, %zmm16
+; AVX512BW-NEXT:    vpermt2q %zmm10, %zmm24, %zmm16
+; AVX512BW-NEXT:    vpermi2q %zmm15, %zmm12, %zmm25
+; AVX512BW-NEXT:    vpermi2q %zmm4, %zmm2, %zmm18
+; AVX512BW-NEXT:    vmovdqa64 %zmm25, %zmm18 {%k2}
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm26, %zmm18
+; AVX512BW-NEXT:    vpermt2q %zmm10, %zmm27, %zmm18
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm20 = [0,8,1,9,0,8,1,9]
+; AVX512BW-NEXT:    # zmm20 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpermt2q %zmm15, %zmm20, %zmm12
+; AVX512BW-NEXT:    vmovdqa (%rdx), %xmm15
 ; AVX512BW-NEXT:    vmovdqa64 64(%rdx), %xmm21
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} xmm20 = xmm20[0],mem[0]
-; AVX512BW-NEXT:    vinserti32x4 $1, %xmm20, %ymm0, %ymm20
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm20, %zmm0, %zmm18 {%k1}
-; AVX512BW-NEXT:    vinserti32x4 $2, (%r8), %zmm18, %zmm18
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm20 = [0,1,2,3,4,8,6,7]
-; AVX512BW-NEXT:    vpermt2q %zmm10, %zmm20, %zmm18
-; AVX512BW-NEXT:    vpermi2q %zmm14, %zmm13, %zmm19
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} xmm21 = xmm21[0],mem[0]
-; AVX512BW-NEXT:    vinserti32x4 $1, %xmm21, %ymm0, %ymm21
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm21, %zmm0, %zmm19 {%k1}
-; AVX512BW-NEXT:    vinserti32x4 $2, 64(%r8), %zmm19, %zmm19
-; AVX512BW-NEXT:    vpermt2q %zmm16, %zmm20, %zmm19
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm20 = [7,15,7,15,7,15,7,15]
-; AVX512BW-NEXT:    # zmm20 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm21
-; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm20, %zmm21
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} ymm22 = [7,15,7,15]
-; AVX512BW-NEXT:    # ymm22 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm22, %zmm13
-; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm13 = zmm13[0,1,2,3],zmm21[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm14 = <14,u,2,3,4,5,15,u>
-; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm14, %zmm13
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm21 = [0,14,2,3,4,5,6,15]
-; AVX512BW-NEXT:    vpermt2q %zmm16, %zmm21, %zmm13
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm23 = [0,1,2,3,4,12,6,7]
-; AVX512BW-NEXT:    vpermt2q %zmm16, %zmm23, %zmm0
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm24 = [3,11,3,11,3,11,3,11]
-; AVX512BW-NEXT:    # zmm24 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm24, %zmm7
-; AVX512BW-NEXT:    vmovdqa (%rdi), %ymm11
-; AVX512BW-NEXT:    vmovdqa64 64(%rdi), %ymm25
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm25 = ymm25[1],mem[1],ymm25[3],mem[3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm25, %zmm7, %zmm7
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm25 = <10,u,2,3,4,5,11,u>
-; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm25, %zmm7
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm8 = [0,10,2,3,4,5,6,11]
-; AVX512BW-NEXT:    vpermt2q %zmm16, %zmm8, %zmm7
-; AVX512BW-NEXT:    vpermi2q %zmm4, %zmm2, %zmm20
-; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm22, %zmm5
-; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm5 = zmm5[0,1,2,3],zmm20[4,5,6,7]
-; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm14, %zmm5
-; AVX512BW-NEXT:    vpermt2q %zmm10, %zmm21, %zmm5
-; AVX512BW-NEXT:    vpermt2q %zmm10, %zmm23, %zmm1
-; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm24, %zmm2
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm4 = ymm11[1],mem[1],ymm11[3],mem[3]
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} xmm15 = xmm15[0],mem[0]
+; AVX512BW-NEXT:    vinserti128 $1, %xmm15, %ymm0, %ymm15
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm15, %zmm0, %zmm12 {%k1}
+; AVX512BW-NEXT:    vinserti32x4 $2, (%r8), %zmm12, %zmm12
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm15 = [0,1,2,3,4,8,6,7]
+; AVX512BW-NEXT:    vpermt2q %zmm10, %zmm15, %zmm12
+; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm20, %zmm8
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} xmm11 = xmm21[0],mem[0]
+; AVX512BW-NEXT:    vinserti128 $1, %xmm11, %ymm0, %ymm11
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm11, %zmm0, %zmm8 {%k1}
+; AVX512BW-NEXT:    vinserti32x4 $2, 64(%r8), %zmm8, %zmm8
+; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm15, %zmm8
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm11 = [0,1,2,3,4,12,6,7]
+; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm11, %zmm0
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm15 = [3,11,3,11,3,11,3,11]
+; AVX512BW-NEXT:    # zmm15 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm15, %zmm5
+; AVX512BW-NEXT:    vmovdqa (%rdi), %ymm7
+; AVX512BW-NEXT:    vmovdqa64 64(%rdi), %ymm20
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm20 = ymm20[1],mem[1],ymm20[3],mem[3]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm20, %zmm5, %zmm5
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm20 = <10,u,2,3,4,5,11,u>
+; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm20, %zmm5
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [0,10,2,3,4,5,6,11]
+; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm6, %zmm5
+; AVX512BW-NEXT:    vpermt2q %zmm10, %zmm11, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm15, %zmm2
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm4 = ymm7[1],mem[1],ymm7[3],mem[3]
 ; AVX512BW-NEXT:    vinserti64x4 $0, %ymm4, %zmm2, %zmm2
-; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm25, %zmm2
-; AVX512BW-NEXT:    vpermt2q %zmm10, %zmm8, %zmm2
-; AVX512BW-NEXT:    vmovdqa64 %zmm17, 64(%rax)
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm20, %zmm2
+; AVX512BW-NEXT:    vpermt2q %zmm10, %zmm6, %zmm2
+; AVX512BW-NEXT:    vmovdqa64 %zmm18, 64(%rax)
 ; AVX512BW-NEXT:    vmovdqa64 %zmm2, 128(%rax)
 ; AVX512BW-NEXT:    vmovdqa64 %zmm1, 192(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm12, 256(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, 320(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm15, 448(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, 512(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm16, 256(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm19, 320(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm17, 448(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, 512(%rax)
 ; AVX512BW-NEXT:    vmovdqa64 %zmm0, 576(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm9, 640(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm13, 704(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm19, 384(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm18, (%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm13, 640(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm9, 704(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, 384(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm12, (%rax)
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
   %in.vec0 = load <16 x i64>, ptr %in.vecptr0, align 64
@@ -3151,559 +3145,557 @@ define void @store_i64_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ;
 ; AVX512F-LABEL: store_i64_stride6_vf32:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    subq $712, %rsp # imm = 0x2C8
-; AVX512F-NEXT:    vmovdqa64 (%rdi), %zmm7
-; AVX512F-NEXT:    vmovdqa64 64(%rdi), %zmm4
-; AVX512F-NEXT:    vmovdqa64 128(%rdi), %zmm2
-; AVX512F-NEXT:    vmovdqa64 192(%rdi), %zmm18
-; AVX512F-NEXT:    vmovdqa64 (%rsi), %zmm22
-; AVX512F-NEXT:    vmovdqa64 64(%rsi), %zmm1
-; AVX512F-NEXT:    vmovdqa64 128(%rsi), %zmm19
-; AVX512F-NEXT:    vmovdqa64 192(%rsi), %zmm21
-; AVX512F-NEXT:    vmovdqa64 (%rdx), %zmm8
-; AVX512F-NEXT:    vmovdqa64 64(%rdx), %zmm6
-; AVX512F-NEXT:    vmovdqa64 128(%rdx), %zmm5
-; AVX512F-NEXT:    vmovdqa64 192(%rdx), %zmm12
-; AVX512F-NEXT:    vmovdqa64 (%rcx), %zmm29
-; AVX512F-NEXT:    vmovdqa64 64(%rcx), %zmm27
-; AVX512F-NEXT:    vmovdqa64 128(%rcx), %zmm26
-; AVX512F-NEXT:    vmovdqa64 192(%rcx), %zmm25
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm11 = [4,12,5,13,4,12,5,13]
-; AVX512F-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} ymm20 = [4,12,4,12]
-; AVX512F-NEXT:    # ymm20 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm12, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm25, %zmm20, %zmm0
+; AVX512F-NEXT:    subq $648, %rsp # imm = 0x288
+; AVX512F-NEXT:    vmovdqa64 (%rdi), %zmm11
+; AVX512F-NEXT:    vmovdqa64 64(%rdi), %zmm5
+; AVX512F-NEXT:    vmovdqa64 128(%rdi), %zmm1
+; AVX512F-NEXT:    vmovdqa64 192(%rdi), %zmm19
+; AVX512F-NEXT:    vmovdqa64 (%rsi), %zmm29
+; AVX512F-NEXT:    vmovdqa64 64(%rsi), %zmm25
+; AVX512F-NEXT:    vmovdqa64 128(%rsi), %zmm23
+; AVX512F-NEXT:    vmovdqa64 192(%rsi), %zmm20
+; AVX512F-NEXT:    vmovdqa64 (%rdx), %zmm24
+; AVX512F-NEXT:    vmovdqa64 64(%rdx), %zmm4
+; AVX512F-NEXT:    vmovdqa64 128(%rdx), %zmm7
+; AVX512F-NEXT:    vmovdqa64 192(%rdx), %zmm21
+; AVX512F-NEXT:    vmovdqa64 (%rcx), %zmm18
+; AVX512F-NEXT:    vmovdqa64 64(%rcx), %zmm13
+; AVX512F-NEXT:    vmovdqa64 128(%rcx), %zmm12
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm27 = [4,12,5,13,4,12,5,13]
+; AVX512F-NEXT:    # zmm27 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm19, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm20, %zmm27, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm23, %zmm27, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm26, %zmm20, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm25, %zmm27, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm27, %zmm20, %zmm0
+; AVX512F-NEXT:    vpermi2q %zmm29, %zmm11, %zmm27
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm14 = [2,10,2,10,2,10,2,10]
+; AVX512F-NEXT:    # zmm14 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm29, %zmm14, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermi2q %zmm29, %zmm8, %zmm20
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm10 = [1,9,2,10,1,9,2,10]
-; AVX512F-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm29, %zmm10, %zmm0
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm15 = [6,14,6,14,6,14,6,14]
+; AVX512F-NEXT:    # zmm15 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm29, %zmm15, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm13 = [5,13,6,14,5,13,6,14]
-; AVX512F-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm17
-; AVX512F-NEXT:    vpermt2q %zmm29, %zmm13, %zmm17
-; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm31
-; AVX512F-NEXT:    vpermt2q %zmm27, %zmm10, %zmm31
-; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm14
-; AVX512F-NEXT:    vpermt2q %zmm27, %zmm13, %zmm14
-; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm15
-; AVX512F-NEXT:    vpermt2q %zmm26, %zmm10, %zmm15
-; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm16
-; AVX512F-NEXT:    vpermt2q %zmm26, %zmm13, %zmm16
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm28 = [3,11,3,11,3,11,3,11]
-; AVX512F-NEXT:    # zmm28 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm29, %zmm28, %zmm0
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm16 = [7,15,7,15,7,15,7,15]
+; AVX512F-NEXT:    # zmm16 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm24, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm18, %zmm16, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [7,15,7,15,7,15,7,15]
-; AVX512F-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermt2q %zmm29, %zmm0, %zmm8
-; AVX512F-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm29
-; AVX512F-NEXT:    vpermt2q %zmm27, %zmm28, %zmm29
-; AVX512F-NEXT:    vpermt2q %zmm27, %zmm0, %zmm6
-; AVX512F-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm6
-; AVX512F-NEXT:    vpermt2q %zmm26, %zmm28, %zmm6
-; AVX512F-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm26, %zmm0, %zmm5
-; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm18, %zmm26
-; AVX512F-NEXT:    vpermt2q %zmm21, %zmm11, %zmm26
-; AVX512F-NEXT:    vpermi2q %zmm25, %zmm12, %zmm10
-; AVX512F-NEXT:    vpermi2q %zmm25, %zmm12, %zmm13
-; AVX512F-NEXT:    vpermi2q %zmm25, %zmm12, %zmm28
-; AVX512F-NEXT:    vpermt2q %zmm25, %zmm0, %zmm12
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm27
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm9
-; AVX512F-NEXT:    vmovdqa64 %zmm19, %zmm5
-; AVX512F-NEXT:    vpermt2q %zmm19, %zmm11, %zmm9
-; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm0
-; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm25
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm11, %zmm25
-; AVX512F-NEXT:    vpermi2q %zmm22, %zmm7, %zmm11
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm3 = [2,10,2,10,2,10,2,10]
-; AVX512F-NEXT:    # zmm3 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm23
-; AVX512F-NEXT:    vpermt2q %zmm22, %zmm3, %zmm23
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm6 = [6,14,6,14,6,14,6,14]
-; AVX512F-NEXT:    # zmm6 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm24
-; AVX512F-NEXT:    vpermt2q %zmm22, %zmm6, %zmm24
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [0,8,1,9,0,8,1,9]
-; AVX512F-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm30
-; AVX512F-NEXT:    vpermt2q %zmm22, %zmm2, %zmm30
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} ymm19 = [7,15,7,15]
-; AVX512F-NEXT:    # ymm19 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermt2q %zmm22, %zmm19, %zmm7
-; AVX512F-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm22
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm3, %zmm22
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm6, %zmm4
+; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm29, %zmm16, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm25, %zmm14, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm25, %zmm15, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm26
+; AVX512F-NEXT:    vpermt2q %zmm13, %zmm16, %zmm26
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm22
+; AVX512F-NEXT:    vpermt2q %zmm25, %zmm16, %zmm22
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm0
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm30
+; AVX512F-NEXT:    vpermt2q %zmm23, %zmm14, %zmm30
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm1 = [0,8,1,9,0,8,1,9]
+; AVX512F-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpermt2q %zmm29, %zmm1, %zmm11
 ; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm8
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm2, %zmm8
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm19, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, (%rsp) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm27, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm5, %zmm3, %zmm0
-; AVX512F-NEXT:    vmovdqa64 %zmm27, %zmm1
-; AVX512F-NEXT:    vpermt2q %zmm5, %zmm6, %zmm1
-; AVX512F-NEXT:    vmovdqa64 %zmm27, %zmm7
-; AVX512F-NEXT:    vpermt2q %zmm5, %zmm2, %zmm7
-; AVX512F-NEXT:    vpermt2q %zmm5, %zmm19, %zmm27
-; AVX512F-NEXT:    vmovdqu64 %zmm27, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermi2q %zmm21, %zmm18, %zmm3
-; AVX512F-NEXT:    vpermi2q %zmm21, %zmm18, %zmm6
-; AVX512F-NEXT:    vpermi2q %zmm21, %zmm18, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm21, %zmm19, %zmm18
-; AVX512F-NEXT:    vmovdqu64 %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm23, %zmm15, %zmm8
+; AVX512F-NEXT:    vpermt2q %zmm25, %zmm1, %zmm5
+; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm28
+; AVX512F-NEXT:    vpermt2q %zmm23, %zmm16, %zmm28
+; AVX512F-NEXT:    vpermt2q %zmm23, %zmm1, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm17
+; AVX512F-NEXT:    vpermt2q %zmm12, %zmm16, %zmm17
+; AVX512F-NEXT:    vpermi2q %zmm20, %zmm19, %zmm14
+; AVX512F-NEXT:    vpermi2q %zmm20, %zmm19, %zmm15
+; AVX512F-NEXT:    vmovdqa64 192(%rcx), %zmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm21, %zmm10
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm16, %zmm10
+; AVX512F-NEXT:    vpermi2q %zmm20, %zmm19, %zmm16
+; AVX512F-NEXT:    vpermt2q %zmm20, %zmm1, %zmm19
+; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [4,12,4,12]
+; AVX512F-NEXT:    # ymm1 = mem[0,1,0,1]
+; AVX512F-NEXT:    vmovdqa64 %zmm21, %zmm6
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm1, %zmm6
+; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm5
+; AVX512F-NEXT:    vpermt2q %zmm12, %zmm1, %zmm5
+; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm3
+; AVX512F-NEXT:    vpermt2q %zmm13, %zmm1, %zmm3
+; AVX512F-NEXT:    vpermi2q %zmm18, %zmm24, %zmm1
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm25 = [1,9,2,10,1,9,2,10]
+; AVX512F-NEXT:    # zmm25 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm24, %zmm20
+; AVX512F-NEXT:    vpermt2q %zmm18, %zmm25, %zmm20
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm29 = [5,13,6,14,5,13,6,14]
+; AVX512F-NEXT:    # zmm29 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm24, %zmm23
+; AVX512F-NEXT:    vmovdqa64 %zmm24, %zmm9
+; AVX512F-NEXT:    vpermt2q %zmm18, %zmm29, %zmm23
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [3,11,3,11,3,11,3,11]
+; AVX512F-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpermt2q %zmm18, %zmm0, %zmm9
+; AVX512F-NEXT:    vmovdqu64 %zmm9, (%rsp) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm18
+; AVX512F-NEXT:    vpermt2q %zmm13, %zmm25, %zmm18
+; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm31
+; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm24
+; AVX512F-NEXT:    vpermt2q %zmm13, %zmm29, %zmm31
+; AVX512F-NEXT:    vpermt2q %zmm13, %zmm0, %zmm24
+; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm13
+; AVX512F-NEXT:    vpermt2q %zmm12, %zmm25, %zmm13
+; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm4
+; AVX512F-NEXT:    vpermt2q %zmm12, %zmm29, %zmm4
+; AVX512F-NEXT:    vpermt2q %zmm12, %zmm0, %zmm7
+; AVX512F-NEXT:    vpermi2q %zmm2, %zmm21, %zmm25
+; AVX512F-NEXT:    vpermi2q %zmm2, %zmm21, %zmm29
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm0, %zmm21
 ; AVX512F-NEXT:    movb $12, %al
 ; AVX512F-NEXT:    kmovw %eax, %k1
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm26 {%k1}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm9 {%k1}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm25 {%k1}
-; AVX512F-NEXT:    vmovdqa64 %zmm20, %zmm11 {%k1}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm0 {%k1}
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm0 {%k1}
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm9 {%k1}
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm27 {%k1}
 ; AVX512F-NEXT:    movb $48, %al
 ; AVX512F-NEXT:    kmovw %eax, %k2
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm23, %zmm5 {%k2}
-; AVX512F-NEXT:    vmovdqa64 %zmm24, %zmm17 {%k2}
-; AVX512F-NEXT:    vmovdqa64 %zmm22, %zmm31 {%k2}
-; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm14 {%k2}
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm15 {%k2}
-; AVX512F-NEXT:    vmovdqa64 (%r8), %zmm23
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm16 {%k2}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm20 {%k2}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm23 {%k2}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm12 # 64-byte Folded Reload
+; AVX512F-NEXT:    # zmm12 = zmm0[0,1,2,3],mem[4,5,6,7]
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm18 {%k2}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm31 {%k2}
+; AVX512F-NEXT:    vmovdqa64 (%r8), %zmm0
+; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm22 = zmm22[0,1,2,3],zmm26[4,5,6,7]
 ; AVX512F-NEXT:    vmovdqa64 64(%r8), %zmm1
-; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm10 {%k2}
-; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm0 = <0,1,9,u,4,5,6,7>
-; AVX512F-NEXT:    vpermt2q %zmm23, %zmm0, %zmm5
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm0, %zmm31
-; AVX512F-NEXT:    vmovdqa64 128(%r8), %zmm3
-; AVX512F-NEXT:    vpermt2q %zmm3, %zmm0, %zmm15
-; AVX512F-NEXT:    vmovdqa64 192(%r8), %zmm22
-; AVX512F-NEXT:    vpermt2q %zmm22, %zmm0, %zmm10
-; AVX512F-NEXT:    vmovdqa64 (%r9), %zmm20
-; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [0,1,2,9,4,5,6,7]
-; AVX512F-NEXT:    vpermt2q %zmm20, %zmm0, %zmm5
-; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 64(%r9), %zmm21
-; AVX512F-NEXT:    vpermt2q %zmm21, %zmm0, %zmm31
-; AVX512F-NEXT:    vmovdqa64 128(%r9), %zmm19
-; AVX512F-NEXT:    vpermt2q %zmm19, %zmm0, %zmm15
-; AVX512F-NEXT:    vmovdqa64 192(%r9), %zmm18
-; AVX512F-NEXT:    vpermt2q %zmm18, %zmm0, %zmm10
-; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm13 {%k2}
-; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm0 = <0,1,13,u,4,5,6,7>
-; AVX512F-NEXT:    vmovdqa64 %zmm17, %zmm27
-; AVX512F-NEXT:    vpermt2q %zmm23, %zmm0, %zmm27
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm0, %zmm14
-; AVX512F-NEXT:    vpermt2q %zmm3, %zmm0, %zmm16
-; AVX512F-NEXT:    vpermt2q %zmm22, %zmm0, %zmm13
-; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [0,1,2,13,4,5,6,7]
-; AVX512F-NEXT:    vpermt2q %zmm20, %zmm0, %zmm27
-; AVX512F-NEXT:    vpermt2q %zmm21, %zmm0, %zmm14
-; AVX512F-NEXT:    vpermt2q %zmm19, %zmm0, %zmm16
-; AVX512F-NEXT:    vpermt2q %zmm18, %zmm0, %zmm13
-; AVX512F-NEXT:    vmovdqa (%rdx), %xmm0
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
-; AVX512F-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm30 {%k1}
-; AVX512F-NEXT:    vmovdqa 64(%rdx), %xmm0
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
-; AVX512F-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm8 {%k1}
-; AVX512F-NEXT:    vmovdqa 128(%rdx), %xmm0
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
-; AVX512F-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm7 {%k1}
-; AVX512F-NEXT:    vmovdqa 192(%rdx), %xmm0
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
-; AVX512F-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm2 {%k1}
-; AVX512F-NEXT:    vinserti32x4 $2, (%r8), %zmm30, %zmm24
-; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [0,1,2,3,4,8,6,7]
-; AVX512F-NEXT:    vpermt2q %zmm20, %zmm0, %zmm24
-; AVX512F-NEXT:    vinserti32x4 $2, 64(%r8), %zmm8, %zmm6
-; AVX512F-NEXT:    vpermt2q %zmm21, %zmm0, %zmm6
-; AVX512F-NEXT:    vinserti32x4 $2, 128(%r8), %zmm7, %zmm5
-; AVX512F-NEXT:    vpermt2q %zmm19, %zmm0, %zmm5
-; AVX512F-NEXT:    vinserti32x4 $2, 192(%r8), %zmm2, %zmm4
-; AVX512F-NEXT:    vpermt2q %zmm18, %zmm0, %zmm4
-; AVX512F-NEXT:    vmovdqa (%rdi), %ymm0
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm0 = ymm0[1],mem[1],ymm0[3],mem[3]
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm2, %zmm0
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm2 # 64-byte Folded Reload
-; AVX512F-NEXT:    # zmm2 = zmm2[0,1,2,3],mem[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 %zmm30, %zmm13 {%k2}
+; AVX512F-NEXT:    vmovdqa64 128(%r8), %zmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm4 {%k2}
+; AVX512F-NEXT:    vmovdqa64 192(%r8), %zmm26
+; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm3 = zmm28[0,1,2,3],zmm17[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 (%r9), %zmm8
+; AVX512F-NEXT:    vmovdqa64 %zmm14, %zmm25 {%k2}
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm5 = <0,1,9,u,4,5,6,7>
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm5, %zmm20
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm5, %zmm18
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm5, %zmm13
+; AVX512F-NEXT:    vpermt2q %zmm26, %zmm5, %zmm25
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [0,1,2,9,4,5,6,7]
+; AVX512F-NEXT:    vpermt2q %zmm8, %zmm5, %zmm20
+; AVX512F-NEXT:    vmovdqa64 64(%r9), %zmm17
+; AVX512F-NEXT:    vpermt2q %zmm17, %zmm5, %zmm18
+; AVX512F-NEXT:    vmovdqa64 128(%r9), %zmm30
+; AVX512F-NEXT:    vpermt2q %zmm30, %zmm5, %zmm13
+; AVX512F-NEXT:    vmovdqa64 192(%r9), %zmm14
+; AVX512F-NEXT:    vpermt2q %zmm14, %zmm5, %zmm25
+; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm29 {%k2}
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm5 = <0,1,13,u,4,5,6,7>
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm5, %zmm23
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm5, %zmm31
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm5, %zmm4
+; AVX512F-NEXT:    vpermt2q %zmm26, %zmm5, %zmm29
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [0,1,2,13,4,5,6,7]
+; AVX512F-NEXT:    vpermt2q %zmm8, %zmm5, %zmm23
+; AVX512F-NEXT:    vpermt2q %zmm17, %zmm5, %zmm31
+; AVX512F-NEXT:    vpermt2q %zmm30, %zmm5, %zmm4
+; AVX512F-NEXT:    vpermt2q %zmm14, %zmm5, %zmm29
+; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm15 = zmm16[0,1,2,3],zmm10[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm5 = <14,u,2,3,4,5,15,u>
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm5, %zmm12
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm5, %zmm22
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm5, %zmm3
+; AVX512F-NEXT:    vpermt2q %zmm26, %zmm5, %zmm15
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [0,14,2,3,4,5,6,15]
+; AVX512F-NEXT:    vpermt2q %zmm8, %zmm5, %zmm12
+; AVX512F-NEXT:    vpermt2q %zmm17, %zmm5, %zmm22
+; AVX512F-NEXT:    vpermt2q %zmm30, %zmm5, %zmm3
+; AVX512F-NEXT:    vpermt2q %zmm14, %zmm5, %zmm15
+; AVX512F-NEXT:    vmovdqa (%rdx), %xmm5
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} xmm5 = xmm5[0],mem[0]
+; AVX512F-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm5, %zmm0, %zmm11 {%k1}
+; AVX512F-NEXT:    vmovdqa 64(%rdx), %xmm5
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} xmm5 = xmm5[0],mem[0]
+; AVX512F-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm5, %zmm0, %zmm10 {%k1}
+; AVX512F-NEXT:    vmovdqa 128(%rdx), %xmm5
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} xmm5 = xmm5[0],mem[0]
+; AVX512F-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm5, %zmm0, %zmm16 {%k1}
+; AVX512F-NEXT:    vmovdqa 192(%rdx), %xmm5
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} xmm5 = xmm5[0],mem[0]
+; AVX512F-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm5, %zmm0, %zmm19 {%k1}
+; AVX512F-NEXT:    vinserti32x4 $2, (%r8), %zmm11, %zmm5
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [0,1,2,3,4,8,6,7]
+; AVX512F-NEXT:    vpermt2q %zmm8, %zmm6, %zmm5
+; AVX512F-NEXT:    vinserti32x4 $2, 64(%r8), %zmm10, %zmm10
+; AVX512F-NEXT:    vpermt2q %zmm17, %zmm6, %zmm10
+; AVX512F-NEXT:    vinserti32x4 $2, 128(%r8), %zmm16, %zmm11
+; AVX512F-NEXT:    vpermt2q %zmm30, %zmm6, %zmm11
+; AVX512F-NEXT:    vinserti32x4 $2, 192(%r8), %zmm19, %zmm16
+; AVX512F-NEXT:    vpermt2q %zmm14, %zmm6, %zmm16
+; AVX512F-NEXT:    vmovdqa (%rdi), %ymm6
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm6 = ymm6[1],mem[1],ymm6[3],mem[3]
+; AVX512F-NEXT:    vmovdqu64 (%rsp), %zmm19 # 64-byte Reload
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm6, %zmm19, %zmm6
 ; AVX512F-NEXT:    movb $16, %al
 ; AVX512F-NEXT:    kmovw %eax, %k1
-; AVX512F-NEXT:    vmovdqa64 %zmm23, %zmm11 {%k1}
-; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm7 = <10,u,2,3,4,5,11,u>
-; AVX512F-NEXT:    vpermt2q %zmm23, %zmm7, %zmm0
-; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm17 = <14,u,2,3,4,5,15,u>
-; AVX512F-NEXT:    vpermt2q %zmm23, %zmm17, %zmm2
-; AVX512F-NEXT:    vmovdqa64 64(%rdi), %ymm23
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm23 = ymm23[1],mem[1],ymm23[3],mem[3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm23, %zmm29, %zmm23
-; AVX512F-NEXT:    vmovdqu64 (%rsp), %zmm8 # 64-byte Reload
-; AVX512F-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm8, %zmm29 # 64-byte Folded Reload
-; AVX512F-NEXT:    # zmm29 = zmm8[0,1,2,3],mem[4,5,6,7]
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm25 {%k1}
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm7, %zmm23
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm17, %zmm29
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm27 {%k1}
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm19 = <10,u,2,3,4,5,11,u>
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm19, %zmm6
+; AVX512F-NEXT:    vmovdqa 64(%rdi), %ymm0
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm0 = ymm0[1],mem[1],ymm0[3],mem[3]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm24, %zmm0
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm9 {%k1}
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm19, %zmm0
 ; AVX512F-NEXT:    vmovdqa 128(%rdi), %ymm1
 ; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm1[1],mem[1],ymm1[3],mem[3]
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm8, %zmm1
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
-; AVX512F-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm8, %zmm30 # 64-byte Folded Reload
-; AVX512F-NEXT:    # zmm30 = zmm8[0,1,2,3],mem[4,5,6,7]
-; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm9 {%k1}
-; AVX512F-NEXT:    vpermt2q %zmm3, %zmm7, %zmm1
-; AVX512F-NEXT:    vpermt2q %zmm3, %zmm17, %zmm30
-; AVX512F-NEXT:    vmovdqa 192(%rdi), %ymm3
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm3 = ymm3[1],mem[1],ymm3[3],mem[3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm3, %zmm28, %zmm3
-; AVX512F-NEXT:    vpermt2q %zmm22, %zmm7, %zmm3
-; AVX512F-NEXT:    vmovdqa64 %zmm22, %zmm26 {%k1}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm7 = zmm7[0,1,2,3],zmm12[4,5,6,7]
-; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm28 = [0,10,2,3,4,5,6,11]
-; AVX512F-NEXT:    vpermt2q %zmm20, %zmm28, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm22, %zmm17, %zmm7
-; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm17 = [0,1,2,3,4,12,6,7]
-; AVX512F-NEXT:    vpermt2q %zmm20, %zmm17, %zmm11
-; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm22 = [0,14,2,3,4,5,6,15]
-; AVX512F-NEXT:    vpermt2q %zmm20, %zmm22, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm21, %zmm28, %zmm23
-; AVX512F-NEXT:    vpermt2q %zmm21, %zmm17, %zmm25
-; AVX512F-NEXT:    vpermt2q %zmm21, %zmm22, %zmm29
-; AVX512F-NEXT:    vpermt2q %zmm19, %zmm28, %zmm1
-; AVX512F-NEXT:    vpermt2q %zmm19, %zmm17, %zmm9
-; AVX512F-NEXT:    vpermt2q %zmm19, %zmm22, %zmm30
-; AVX512F-NEXT:    vpermt2q %zmm18, %zmm28, %zmm3
-; AVX512F-NEXT:    vpermt2q %zmm18, %zmm17, %zmm26
-; AVX512F-NEXT:    vpermt2q %zmm18, %zmm22, %zmm7
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm7, %zmm1
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm28 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm28 {%k1}
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm19, %zmm1
+; AVX512F-NEXT:    vmovdqa 192(%rdi), %ymm2
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm2 = ymm2[1],mem[1],ymm2[3],mem[3]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm21, %zmm2
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm24 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm26, %zmm24 {%k1}
+; AVX512F-NEXT:    vpermt2q %zmm26, %zmm19, %zmm2
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm19 = [0,10,2,3,4,5,6,11]
+; AVX512F-NEXT:    vpermt2q %zmm8, %zmm19, %zmm6
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm21 = [0,1,2,3,4,12,6,7]
+; AVX512F-NEXT:    vpermt2q %zmm8, %zmm21, %zmm27
+; AVX512F-NEXT:    vpermt2q %zmm17, %zmm19, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm17, %zmm21, %zmm9
+; AVX512F-NEXT:    vpermt2q %zmm30, %zmm19, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm30, %zmm21, %zmm28
+; AVX512F-NEXT:    vpermt2q %zmm14, %zmm19, %zmm2
+; AVX512F-NEXT:    vpermt2q %zmm14, %zmm21, %zmm24
 ; AVX512F-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512F-NEXT:    vmovdqa64 %zmm7, 1472(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm13, 1408(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm26, 1344(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm3, 1280(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm10, 1216(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm30, 1088(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm16, 1024(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm9, 960(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm15, 1472(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm29, 1408(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm24, 1344(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm2, 1280(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm25, 1216(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm3, 1088(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm4, 1024(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm28, 960(%rax)
 ; AVX512F-NEXT:    vmovdqa64 %zmm1, 896(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm15, 832(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm29, 704(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm14, 640(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm25, 576(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm23, 512(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm31, 448(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm2, 320(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm27, 256(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm11, 192(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm0, 128(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 64(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm4, 1152(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm5, 768(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm6, 384(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm24, (%rax)
-; AVX512F-NEXT:    addq $712, %rsp # imm = 0x2C8
+; AVX512F-NEXT:    vmovdqa64 %zmm13, 832(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm22, 704(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm31, 640(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm9, 576(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm0, 512(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm18, 448(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm12, 320(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm23, 256(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm27, 192(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm6, 128(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm20, 64(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm16, 1152(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm11, 768(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm10, 384(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm5, (%rax)
+; AVX512F-NEXT:    addq $648, %rsp # imm = 0x288
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: store_i64_stride6_vf32:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    subq $712, %rsp # imm = 0x2C8
-; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm7
-; AVX512BW-NEXT:    vmovdqa64 64(%rdi), %zmm4
-; AVX512BW-NEXT:    vmovdqa64 128(%rdi), %zmm2
-; AVX512BW-NEXT:    vmovdqa64 192(%rdi), %zmm18
-; AVX512BW-NEXT:    vmovdqa64 (%rsi), %zmm22
-; AVX512BW-NEXT:    vmovdqa64 64(%rsi), %zmm1
-; AVX512BW-NEXT:    vmovdqa64 128(%rsi), %zmm19
-; AVX512BW-NEXT:    vmovdqa64 192(%rsi), %zmm21
-; AVX512BW-NEXT:    vmovdqa64 (%rdx), %zmm8
-; AVX512BW-NEXT:    vmovdqa64 64(%rdx), %zmm6
-; AVX512BW-NEXT:    vmovdqa64 128(%rdx), %zmm5
-; AVX512BW-NEXT:    vmovdqa64 192(%rdx), %zmm12
-; AVX512BW-NEXT:    vmovdqa64 (%rcx), %zmm29
-; AVX512BW-NEXT:    vmovdqa64 64(%rcx), %zmm27
-; AVX512BW-NEXT:    vmovdqa64 128(%rcx), %zmm26
-; AVX512BW-NEXT:    vmovdqa64 192(%rcx), %zmm25
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm11 = [4,12,5,13,4,12,5,13]
-; AVX512BW-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} ymm20 = [4,12,4,12]
-; AVX512BW-NEXT:    # ymm20 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm25, %zmm20, %zmm0
+; AVX512BW-NEXT:    subq $648, %rsp # imm = 0x288
+; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm11
+; AVX512BW-NEXT:    vmovdqa64 64(%rdi), %zmm5
+; AVX512BW-NEXT:    vmovdqa64 128(%rdi), %zmm1
+; AVX512BW-NEXT:    vmovdqa64 192(%rdi), %zmm19
+; AVX512BW-NEXT:    vmovdqa64 (%rsi), %zmm29
+; AVX512BW-NEXT:    vmovdqa64 64(%rsi), %zmm25
+; AVX512BW-NEXT:    vmovdqa64 128(%rsi), %zmm23
+; AVX512BW-NEXT:    vmovdqa64 192(%rsi), %zmm20
+; AVX512BW-NEXT:    vmovdqa64 (%rdx), %zmm24
+; AVX512BW-NEXT:    vmovdqa64 64(%rdx), %zmm4
+; AVX512BW-NEXT:    vmovdqa64 128(%rdx), %zmm7
+; AVX512BW-NEXT:    vmovdqa64 192(%rdx), %zmm21
+; AVX512BW-NEXT:    vmovdqa64 (%rcx), %zmm18
+; AVX512BW-NEXT:    vmovdqa64 64(%rcx), %zmm13
+; AVX512BW-NEXT:    vmovdqa64 128(%rcx), %zmm12
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm27 = [4,12,5,13,4,12,5,13]
+; AVX512BW-NEXT:    # zmm27 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm19, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm27, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm23, %zmm27, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm26, %zmm20, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm25, %zmm27, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm27, %zmm20, %zmm0
+; AVX512BW-NEXT:    vpermi2q %zmm29, %zmm11, %zmm27
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm14 = [2,10,2,10,2,10,2,10]
+; AVX512BW-NEXT:    # zmm14 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm29, %zmm14, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermi2q %zmm29, %zmm8, %zmm20
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm10 = [1,9,2,10,1,9,2,10]
-; AVX512BW-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm29, %zmm10, %zmm0
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm15 = [6,14,6,14,6,14,6,14]
+; AVX512BW-NEXT:    # zmm15 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm29, %zmm15, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm13 = [5,13,6,14,5,13,6,14]
-; AVX512BW-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm17
-; AVX512BW-NEXT:    vpermt2q %zmm29, %zmm13, %zmm17
-; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm31
-; AVX512BW-NEXT:    vpermt2q %zmm27, %zmm10, %zmm31
-; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm14
-; AVX512BW-NEXT:    vpermt2q %zmm27, %zmm13, %zmm14
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm15
-; AVX512BW-NEXT:    vpermt2q %zmm26, %zmm10, %zmm15
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm16
-; AVX512BW-NEXT:    vpermt2q %zmm26, %zmm13, %zmm16
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm28 = [3,11,3,11,3,11,3,11]
-; AVX512BW-NEXT:    # zmm28 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm29, %zmm28, %zmm0
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm16 = [7,15,7,15,7,15,7,15]
+; AVX512BW-NEXT:    # zmm16 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm24, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm16, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [7,15,7,15,7,15,7,15]
-; AVX512BW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermt2q %zmm29, %zmm0, %zmm8
-; AVX512BW-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm29
-; AVX512BW-NEXT:    vpermt2q %zmm27, %zmm28, %zmm29
-; AVX512BW-NEXT:    vpermt2q %zmm27, %zmm0, %zmm6
-; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm6
-; AVX512BW-NEXT:    vpermt2q %zmm26, %zmm28, %zmm6
-; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm26, %zmm0, %zmm5
-; AVX512BW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm18, %zmm26
-; AVX512BW-NEXT:    vpermt2q %zmm21, %zmm11, %zmm26
-; AVX512BW-NEXT:    vpermi2q %zmm25, %zmm12, %zmm10
-; AVX512BW-NEXT:    vpermi2q %zmm25, %zmm12, %zmm13
-; AVX512BW-NEXT:    vpermi2q %zmm25, %zmm12, %zmm28
-; AVX512BW-NEXT:    vpermt2q %zmm25, %zmm0, %zmm12
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm27
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm9
-; AVX512BW-NEXT:    vmovdqa64 %zmm19, %zmm5
-; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm11, %zmm9
-; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm0
-; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm25
-; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm11, %zmm25
-; AVX512BW-NEXT:    vpermi2q %zmm22, %zmm7, %zmm11
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm3 = [2,10,2,10,2,10,2,10]
-; AVX512BW-NEXT:    # zmm3 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm23
-; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm3, %zmm23
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm6 = [6,14,6,14,6,14,6,14]
-; AVX512BW-NEXT:    # zmm6 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm24
-; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm6, %zmm24
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [0,8,1,9,0,8,1,9]
-; AVX512BW-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm30
-; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm2, %zmm30
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} ymm19 = [7,15,7,15]
-; AVX512BW-NEXT:    # ymm19 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm19, %zmm7
-; AVX512BW-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm22
-; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm3, %zmm22
-; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm6, %zmm4
+; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm29, %zmm16, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm25, %zmm14, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm25, %zmm15, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm26
+; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm16, %zmm26
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm22
+; AVX512BW-NEXT:    vpermt2q %zmm25, %zmm16, %zmm22
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm0
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm30
+; AVX512BW-NEXT:    vpermt2q %zmm23, %zmm14, %zmm30
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm1 = [0,8,1,9,0,8,1,9]
+; AVX512BW-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpermt2q %zmm29, %zmm1, %zmm11
 ; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm8
-; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm2, %zmm8
-; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm19, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, (%rsp) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm27, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm3, %zmm0
-; AVX512BW-NEXT:    vmovdqa64 %zmm27, %zmm1
-; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm6, %zmm1
-; AVX512BW-NEXT:    vmovdqa64 %zmm27, %zmm7
-; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm2, %zmm7
-; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm19, %zmm27
-; AVX512BW-NEXT:    vmovdqu64 %zmm27, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermi2q %zmm21, %zmm18, %zmm3
-; AVX512BW-NEXT:    vpermi2q %zmm21, %zmm18, %zmm6
-; AVX512BW-NEXT:    vpermi2q %zmm21, %zmm18, %zmm2
-; AVX512BW-NEXT:    vpermt2q %zmm21, %zmm19, %zmm18
-; AVX512BW-NEXT:    vmovdqu64 %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm23, %zmm15, %zmm8
+; AVX512BW-NEXT:    vpermt2q %zmm25, %zmm1, %zmm5
+; AVX512BW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm28
+; AVX512BW-NEXT:    vpermt2q %zmm23, %zmm16, %zmm28
+; AVX512BW-NEXT:    vpermt2q %zmm23, %zmm1, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm17
+; AVX512BW-NEXT:    vpermt2q %zmm12, %zmm16, %zmm17
+; AVX512BW-NEXT:    vpermi2q %zmm20, %zmm19, %zmm14
+; AVX512BW-NEXT:    vpermi2q %zmm20, %zmm19, %zmm15
+; AVX512BW-NEXT:    vmovdqa64 192(%rcx), %zmm2
+; AVX512BW-NEXT:    vmovdqa64 %zmm21, %zmm10
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm16, %zmm10
+; AVX512BW-NEXT:    vpermi2q %zmm20, %zmm19, %zmm16
+; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm1, %zmm19
+; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [4,12,4,12]
+; AVX512BW-NEXT:    # ymm1 = mem[0,1,0,1]
+; AVX512BW-NEXT:    vmovdqa64 %zmm21, %zmm6
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm1, %zmm6
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm5
+; AVX512BW-NEXT:    vpermt2q %zmm12, %zmm1, %zmm5
+; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm3
+; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm1, %zmm3
+; AVX512BW-NEXT:    vpermi2q %zmm18, %zmm24, %zmm1
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm25 = [1,9,2,10,1,9,2,10]
+; AVX512BW-NEXT:    # zmm25 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm24, %zmm20
+; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm25, %zmm20
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm29 = [5,13,6,14,5,13,6,14]
+; AVX512BW-NEXT:    # zmm29 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm24, %zmm23
+; AVX512BW-NEXT:    vmovdqa64 %zmm24, %zmm9
+; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm29, %zmm23
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [3,11,3,11,3,11,3,11]
+; AVX512BW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm0, %zmm9
+; AVX512BW-NEXT:    vmovdqu64 %zmm9, (%rsp) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm18
+; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm25, %zmm18
+; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm31
+; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm24
+; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm29, %zmm31
+; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm0, %zmm24
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm13
+; AVX512BW-NEXT:    vpermt2q %zmm12, %zmm25, %zmm13
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm4
+; AVX512BW-NEXT:    vpermt2q %zmm12, %zmm29, %zmm4
+; AVX512BW-NEXT:    vpermt2q %zmm12, %zmm0, %zmm7
+; AVX512BW-NEXT:    vpermi2q %zmm2, %zmm21, %zmm25
+; AVX512BW-NEXT:    vpermi2q %zmm2, %zmm21, %zmm29
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm0, %zmm21
 ; AVX512BW-NEXT:    movb $12, %al
 ; AVX512BW-NEXT:    kmovd %eax, %k1
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm26 {%k1}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm9 {%k1}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm25 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 %zmm20, %zmm11 {%k1}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm0 {%k1}
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm0 {%k1}
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm9 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm27 {%k1}
 ; AVX512BW-NEXT:    movb $48, %al
 ; AVX512BW-NEXT:    kmovd %eax, %k2
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm23, %zmm5 {%k2}
-; AVX512BW-NEXT:    vmovdqa64 %zmm24, %zmm17 {%k2}
-; AVX512BW-NEXT:    vmovdqa64 %zmm22, %zmm31 {%k2}
-; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm14 {%k2}
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm15 {%k2}
-; AVX512BW-NEXT:    vmovdqa64 (%r8), %zmm23
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm16 {%k2}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm20 {%k2}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm23 {%k2}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm12 # 64-byte Folded Reload
+; AVX512BW-NEXT:    # zmm12 = zmm0[0,1,2,3],mem[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm18 {%k2}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm31 {%k2}
+; AVX512BW-NEXT:    vmovdqa64 (%r8), %zmm0
+; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm22 = zmm22[0,1,2,3],zmm26[4,5,6,7]
 ; AVX512BW-NEXT:    vmovdqa64 64(%r8), %zmm1
-; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm10 {%k2}
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm0 = <0,1,9,u,4,5,6,7>
-; AVX512BW-NEXT:    vpermt2q %zmm23, %zmm0, %zmm5
-; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm0, %zmm31
-; AVX512BW-NEXT:    vmovdqa64 128(%r8), %zmm3
-; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm0, %zmm15
-; AVX512BW-NEXT:    vmovdqa64 192(%r8), %zmm22
-; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm0, %zmm10
-; AVX512BW-NEXT:    vmovdqa64 (%r9), %zmm20
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [0,1,2,9,4,5,6,7]
-; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm0, %zmm5
-; AVX512BW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 64(%r9), %zmm21
-; AVX512BW-NEXT:    vpermt2q %zmm21, %zmm0, %zmm31
-; AVX512BW-NEXT:    vmovdqa64 128(%r9), %zmm19
-; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm0, %zmm15
-; AVX512BW-NEXT:    vmovdqa64 192(%r9), %zmm18
-; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm0, %zmm10
-; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm13 {%k2}
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm0 = <0,1,13,u,4,5,6,7>
-; AVX512BW-NEXT:    vmovdqa64 %zmm17, %zmm27
-; AVX512BW-NEXT:    vpermt2q %zmm23, %zmm0, %zmm27
-; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm0, %zmm14
-; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm0, %zmm16
-; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm0, %zmm13
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [0,1,2,13,4,5,6,7]
-; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm0, %zmm27
-; AVX512BW-NEXT:    vpermt2q %zmm21, %zmm0, %zmm14
-; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm0, %zmm16
-; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm0, %zmm13
-; AVX512BW-NEXT:    vmovdqa (%rdx), %xmm0
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
-; AVX512BW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm30 {%k1}
-; AVX512BW-NEXT:    vmovdqa 64(%rdx), %xmm0
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
-; AVX512BW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm8 {%k1}
-; AVX512BW-NEXT:    vmovdqa 128(%rdx), %xmm0
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
-; AVX512BW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm7 {%k1}
-; AVX512BW-NEXT:    vmovdqa 192(%rdx), %xmm0
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
-; AVX512BW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm2 {%k1}
-; AVX512BW-NEXT:    vinserti32x4 $2, (%r8), %zmm30, %zmm24
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [0,1,2,3,4,8,6,7]
-; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm0, %zmm24
-; AVX512BW-NEXT:    vinserti32x4 $2, 64(%r8), %zmm8, %zmm6
-; AVX512BW-NEXT:    vpermt2q %zmm21, %zmm0, %zmm6
-; AVX512BW-NEXT:    vinserti32x4 $2, 128(%r8), %zmm7, %zmm5
-; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm0, %zmm5
-; AVX512BW-NEXT:    vinserti32x4 $2, 192(%r8), %zmm2, %zmm4
-; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm0, %zmm4
-; AVX512BW-NEXT:    vmovdqa (%rdi), %ymm0
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm0 = ymm0[1],mem[1],ymm0[3],mem[3]
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm2, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm2 # 64-byte Folded Reload
-; AVX512BW-NEXT:    # zmm2 = zmm2[0,1,2,3],mem[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm30, %zmm13 {%k2}
+; AVX512BW-NEXT:    vmovdqa64 128(%r8), %zmm2
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm4 {%k2}
+; AVX512BW-NEXT:    vmovdqa64 192(%r8), %zmm26
+; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm3 = zmm28[0,1,2,3],zmm17[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 (%r9), %zmm8
+; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm25 {%k2}
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm5 = <0,1,9,u,4,5,6,7>
+; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm5, %zmm20
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm5, %zmm18
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm5, %zmm13
+; AVX512BW-NEXT:    vpermt2q %zmm26, %zmm5, %zmm25
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [0,1,2,9,4,5,6,7]
+; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm5, %zmm20
+; AVX512BW-NEXT:    vmovdqa64 64(%r9), %zmm17
+; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm5, %zmm18
+; AVX512BW-NEXT:    vmovdqa64 128(%r9), %zmm30
+; AVX512BW-NEXT:    vpermt2q %zmm30, %zmm5, %zmm13
+; AVX512BW-NEXT:    vmovdqa64 192(%r9), %zmm14
+; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm5, %zmm25
+; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm29 {%k2}
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm5 = <0,1,13,u,4,5,6,7>
+; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm5, %zmm23
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm5, %zmm31
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm5, %zmm4
+; AVX512BW-NEXT:    vpermt2q %zmm26, %zmm5, %zmm29
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [0,1,2,13,4,5,6,7]
+; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm5, %zmm23
+; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm5, %zmm31
+; AVX512BW-NEXT:    vpermt2q %zmm30, %zmm5, %zmm4
+; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm5, %zmm29
+; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm15 = zmm16[0,1,2,3],zmm10[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm5 = <14,u,2,3,4,5,15,u>
+; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm5, %zmm12
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm5, %zmm22
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm5, %zmm3
+; AVX512BW-NEXT:    vpermt2q %zmm26, %zmm5, %zmm15
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [0,14,2,3,4,5,6,15]
+; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm5, %zmm12
+; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm5, %zmm22
+; AVX512BW-NEXT:    vpermt2q %zmm30, %zmm5, %zmm3
+; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm5, %zmm15
+; AVX512BW-NEXT:    vmovdqa (%rdx), %xmm5
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} xmm5 = xmm5[0],mem[0]
+; AVX512BW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm5, %zmm0, %zmm11 {%k1}
+; AVX512BW-NEXT:    vmovdqa 64(%rdx), %xmm5
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} xmm5 = xmm5[0],mem[0]
+; AVX512BW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm5, %zmm0, %zmm10 {%k1}
+; AVX512BW-NEXT:    vmovdqa 128(%rdx), %xmm5
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} xmm5 = xmm5[0],mem[0]
+; AVX512BW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm5, %zmm0, %zmm16 {%k1}
+; AVX512BW-NEXT:    vmovdqa 192(%rdx), %xmm5
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} xmm5 = xmm5[0],mem[0]
+; AVX512BW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm5, %zmm0, %zmm19 {%k1}
+; AVX512BW-NEXT:    vinserti32x4 $2, (%r8), %zmm11, %zmm5
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [0,1,2,3,4,8,6,7]
+; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm6, %zmm5
+; AVX512BW-NEXT:    vinserti32x4 $2, 64(%r8), %zmm10, %zmm10
+; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm6, %zmm10
+; AVX512BW-NEXT:    vinserti32x4 $2, 128(%r8), %zmm16, %zmm11
+; AVX512BW-NEXT:    vpermt2q %zmm30, %zmm6, %zmm11
+; AVX512BW-NEXT:    vinserti32x4 $2, 192(%r8), %zmm19, %zmm16
+; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm6, %zmm16
+; AVX512BW-NEXT:    vmovdqa (%rdi), %ymm6
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm6 = ymm6[1],mem[1],ymm6[3],mem[3]
+; AVX512BW-NEXT:    vmovdqu64 (%rsp), %zmm19 # 64-byte Reload
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm6, %zmm19, %zmm6
 ; AVX512BW-NEXT:    movb $16, %al
 ; AVX512BW-NEXT:    kmovd %eax, %k1
-; AVX512BW-NEXT:    vmovdqa64 %zmm23, %zmm11 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm7 = <10,u,2,3,4,5,11,u>
-; AVX512BW-NEXT:    vpermt2q %zmm23, %zmm7, %zmm0
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm17 = <14,u,2,3,4,5,15,u>
-; AVX512BW-NEXT:    vpermt2q %zmm23, %zmm17, %zmm2
-; AVX512BW-NEXT:    vmovdqa64 64(%rdi), %ymm23
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm23 = ymm23[1],mem[1],ymm23[3],mem[3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm23, %zmm29, %zmm23
-; AVX512BW-NEXT:    vmovdqu64 (%rsp), %zmm8 # 64-byte Reload
-; AVX512BW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm8, %zmm29 # 64-byte Folded Reload
-; AVX512BW-NEXT:    # zmm29 = zmm8[0,1,2,3],mem[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm25 {%k1}
-; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm7, %zmm23
-; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm17, %zmm29
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm27 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm19 = <10,u,2,3,4,5,11,u>
+; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm19, %zmm6
+; AVX512BW-NEXT:    vmovdqa 64(%rdi), %ymm0
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm0 = ymm0[1],mem[1],ymm0[3],mem[3]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm24, %zmm0
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm9 {%k1}
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm19, %zmm0
 ; AVX512BW-NEXT:    vmovdqa 128(%rdi), %ymm1
 ; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm1[1],mem[1],ymm1[3],mem[3]
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm8, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
-; AVX512BW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm8, %zmm30 # 64-byte Folded Reload
-; AVX512BW-NEXT:    # zmm30 = zmm8[0,1,2,3],mem[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm9 {%k1}
-; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm7, %zmm1
-; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm17, %zmm30
-; AVX512BW-NEXT:    vmovdqa 192(%rdi), %ymm3
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm3 = ymm3[1],mem[1],ymm3[3],mem[3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm3, %zmm28, %zmm3
-; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm7, %zmm3
-; AVX512BW-NEXT:    vmovdqa64 %zmm22, %zmm26 {%k1}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm7 = zmm7[0,1,2,3],zmm12[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm28 = [0,10,2,3,4,5,6,11]
-; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm28, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm17, %zmm7
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm17 = [0,1,2,3,4,12,6,7]
-; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm17, %zmm11
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm22 = [0,14,2,3,4,5,6,15]
-; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm22, %zmm2
-; AVX512BW-NEXT:    vpermt2q %zmm21, %zmm28, %zmm23
-; AVX512BW-NEXT:    vpermt2q %zmm21, %zmm17, %zmm25
-; AVX512BW-NEXT:    vpermt2q %zmm21, %zmm22, %zmm29
-; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm28, %zmm1
-; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm17, %zmm9
-; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm22, %zmm30
-; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm28, %zmm3
-; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm17, %zmm26
-; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm22, %zmm7
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm7, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm28 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm28 {%k1}
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm19, %zmm1
+; AVX512BW-NEXT:    vmovdqa 192(%rdi), %ymm2
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm2 = ymm2[1],mem[1],ymm2[3],mem[3]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm21, %zmm2
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm24 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm26, %zmm24 {%k1}
+; AVX512BW-NEXT:    vpermt2q %zmm26, %zmm19, %zmm2
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm19 = [0,10,2,3,4,5,6,11]
+; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm19, %zmm6
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm21 = [0,1,2,3,4,12,6,7]
+; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm21, %zmm27
+; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm19, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm21, %zmm9
+; AVX512BW-NEXT:    vpermt2q %zmm30, %zmm19, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm30, %zmm21, %zmm28
+; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm19, %zmm2
+; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm21, %zmm24
 ; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, 1472(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm13, 1408(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm26, 1344(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm3, 1280(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, 1216(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm30, 1088(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm16, 1024(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm9, 960(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm15, 1472(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm29, 1408(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm24, 1344(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, 1280(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm25, 1216(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, 1088(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm4, 1024(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm28, 960(%rax)
 ; AVX512BW-NEXT:    vmovdqa64 %zmm1, 896(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm15, 832(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm29, 704(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm14, 640(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm25, 576(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm23, 512(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm31, 448(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, 320(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm27, 256(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm11, 192(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, 128(%rax)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm0, 64(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm4, 1152(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, 768(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm6, 384(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm24, (%rax)
-; AVX512BW-NEXT:    addq $712, %rsp # imm = 0x2C8
+; AVX512BW-NEXT:    vmovdqa64 %zmm13, 832(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm22, 704(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm31, 640(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm9, 576(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, 512(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm18, 448(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm12, 320(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm23, 256(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm27, 192(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, 128(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm20, 64(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm16, 1152(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm11, 768(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm10, 384(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, (%rax)
+; AVX512BW-NEXT:    addq $648, %rsp # imm = 0x288
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
   %in.vec0 = load <32 x i64>, ptr %in.vecptr0, align 64
@@ -6680,1355 +6672,1393 @@ define void @store_i64_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ;
 ; AVX512F-LABEL: store_i64_stride6_vf64:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    subq $3400, %rsp # imm = 0xD48
-; AVX512F-NEXT:    vmovdqa64 (%rdx), %zmm13
-; AVX512F-NEXT:    vmovdqa64 64(%rdx), %zmm12
-; AVX512F-NEXT:    vmovdqa64 128(%rdx), %zmm11
-; AVX512F-NEXT:    vmovdqa64 192(%rdx), %zmm10
-; AVX512F-NEXT:    vmovdqa64 256(%rdx), %zmm9
-; AVX512F-NEXT:    vmovdqa64 320(%rdx), %zmm8
-; AVX512F-NEXT:    vmovdqa64 384(%rdx), %zmm7
-; AVX512F-NEXT:    vmovdqa64 448(%rdx), %zmm6
-; AVX512F-NEXT:    vmovdqa64 (%rcx), %zmm0
-; AVX512F-NEXT:    vmovdqa64 64(%rcx), %zmm1
-; AVX512F-NEXT:    vmovdqa64 128(%rcx), %zmm2
-; AVX512F-NEXT:    vmovdqa64 192(%rcx), %zmm30
-; AVX512F-NEXT:    vmovdqa64 256(%rcx), %zmm27
-; AVX512F-NEXT:    vmovdqa64 320(%rcx), %zmm24
-; AVX512F-NEXT:    vmovdqa64 384(%rcx), %zmm22
-; AVX512F-NEXT:    vmovdqa64 448(%rcx), %zmm21
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [4,12,4,12]
-; AVX512F-NEXT:    # ymm3 = mem[0,1,0,1]
-; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm4
-; AVX512F-NEXT:    vpermt2q %zmm21, %zmm3, %zmm4
-; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm4
-; AVX512F-NEXT:    vpermt2q %zmm22, %zmm3, %zmm4
-; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm4
-; AVX512F-NEXT:    vpermt2q %zmm24, %zmm3, %zmm4
-; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm4
-; AVX512F-NEXT:    vpermt2q %zmm27, %zmm3, %zmm4
-; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm4
-; AVX512F-NEXT:    vpermt2q %zmm30, %zmm3, %zmm4
-; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm4
-; AVX512F-NEXT:    vpermt2q %zmm2, %zmm3, %zmm4
-; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm12, %zmm4
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm3, %zmm4
-; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermi2q %zmm0, %zmm13, %zmm3
+; AVX512F-NEXT:    subq $3720, %rsp # imm = 0xE88
+; AVX512F-NEXT:    vmovdqa64 (%rdi), %zmm11
+; AVX512F-NEXT:    vmovdqa64 64(%rdi), %zmm10
+; AVX512F-NEXT:    vmovdqa64 128(%rdi), %zmm9
+; AVX512F-NEXT:    vmovdqa64 192(%rdi), %zmm8
+; AVX512F-NEXT:    vmovdqa64 256(%rdi), %zmm7
+; AVX512F-NEXT:    vmovdqa64 320(%rdi), %zmm6
+; AVX512F-NEXT:    vmovdqa64 384(%rdi), %zmm5
+; AVX512F-NEXT:    vmovdqa64 448(%rdi), %zmm4
+; AVX512F-NEXT:    vmovdqa64 (%rsi), %zmm30
+; AVX512F-NEXT:    vmovdqa64 64(%rsi), %zmm29
+; AVX512F-NEXT:    vmovdqa64 128(%rsi), %zmm28
+; AVX512F-NEXT:    vmovdqa64 192(%rsi), %zmm27
+; AVX512F-NEXT:    vmovdqa64 256(%rsi), %zmm1
+; AVX512F-NEXT:    vmovdqa64 320(%rsi), %zmm13
+; AVX512F-NEXT:    vmovdqa64 384(%rsi), %zmm24
+; AVX512F-NEXT:    vmovdqa64 448(%rsi), %zmm0
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [4,12,5,13,4,12,5,13]
+; AVX512F-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm22
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm2, %zmm22
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm3
+; AVX512F-NEXT:    vpermt2q %zmm24, %zmm2, %zmm3
 ; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm5 = [1,9,2,10,1,9,2,10]
-; AVX512F-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm3
-; AVX512F-NEXT:    vpermt2q %zmm0, %zmm5, %zmm3
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm3
+; AVX512F-NEXT:    vpermt2q %zmm13, %zmm2, %zmm3
 ; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm4
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm5 = [5,13,6,14,5,13,6,14]
-; AVX512F-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm3
-; AVX512F-NEXT:    vpermt2q %zmm0, %zmm5, %zmm3
+; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm3
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm2, %zmm3
 ; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm12, %zmm3
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm4, %zmm3
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm3
+; AVX512F-NEXT:    vpermt2q %zmm27, %zmm2, %zmm3
 ; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm14 = [3,11,3,11,3,11,3,11]
-; AVX512F-NEXT:    # zmm14 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm3
-; AVX512F-NEXT:    vpermt2q %zmm0, %zmm14, %zmm3
+; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm3
+; AVX512F-NEXT:    vpermt2q %zmm28, %zmm2, %zmm3
 ; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm3 = [7,15,7,15,7,15,7,15]
-; AVX512F-NEXT:    # zmm3 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermt2q %zmm0, %zmm3, %zmm13
-; AVX512F-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm12, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm5, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm12, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm14, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm3, %zmm12
-; AVX512F-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm2, %zmm4, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm2, %zmm5, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm2, %zmm14, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm2, %zmm3, %zmm11
+; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm3
+; AVX512F-NEXT:    vpermt2q %zmm29, %zmm2, %zmm3
+; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermi2q %zmm30, %zmm11, %zmm2
+; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm2 = [2,10,2,10,2,10,2,10]
+; AVX512F-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm3
+; AVX512F-NEXT:    vpermt2q %zmm30, %zmm2, %zmm3
+; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm12
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm2 = [6,14,6,14,6,14,6,14]
+; AVX512F-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm3
+; AVX512F-NEXT:    vpermt2q %zmm30, %zmm2, %zmm3
+; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm15 = [7,15,7,15,7,15,7,15]
+; AVX512F-NEXT:    # zmm15 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm3
+; AVX512F-NEXT:    vpermt2q %zmm30, %zmm15, %zmm3
+; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm3
+; AVX512F-NEXT:    vpermt2q %zmm29, %zmm12, %zmm3
+; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm3 = [0,8,1,9,0,8,1,9]
+; AVX512F-NEXT:    # zmm3 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpermt2q %zmm30, %zmm3, %zmm11
 ; AVX512F-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm30, %zmm4, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm30, %zmm5, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm30, %zmm14, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm30, %zmm3, %zmm10
+; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm11
+; AVX512F-NEXT:    vpermt2q %zmm29, %zmm2, %zmm11
+; AVX512F-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm11
+; AVX512F-NEXT:    vpermt2q %zmm29, %zmm15, %zmm11
+; AVX512F-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm29, %zmm3, %zmm10
 ; AVX512F-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm27, %zmm4, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm27, %zmm5, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm27, %zmm14, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm27, %zmm3, %zmm9
+; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm10
+; AVX512F-NEXT:    vpermt2q %zmm28, %zmm12, %zmm10
+; AVX512F-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm10
+; AVX512F-NEXT:    vpermt2q %zmm28, %zmm2, %zmm10
+; AVX512F-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm10
+; AVX512F-NEXT:    vpermt2q %zmm28, %zmm15, %zmm10
+; AVX512F-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm28, %zmm3, %zmm9
 ; AVX512F-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm24, %zmm4, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm24, %zmm5, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm24, %zmm14, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm24, %zmm3, %zmm8
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm9
+; AVX512F-NEXT:    vpermt2q %zmm27, %zmm12, %zmm9
+; AVX512F-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm9
+; AVX512F-NEXT:    vpermt2q %zmm27, %zmm2, %zmm9
+; AVX512F-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm9
+; AVX512F-NEXT:    vpermt2q %zmm27, %zmm15, %zmm9
+; AVX512F-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm27, %zmm3, %zmm8
+; AVX512F-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm8
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm12, %zmm8
+; AVX512F-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm8
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm2, %zmm8
+; AVX512F-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm8
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm15, %zmm8
+; AVX512F-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm3, %zmm7
+; AVX512F-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm13, %zmm12, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm13, %zmm2, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm13, %zmm15, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm13, %zmm3, %zmm6
+; AVX512F-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm24, %zmm12, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm24, %zmm2, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm8
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm24, %zmm15, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm24, %zmm3, %zmm5
+; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 (%rdx), %zmm5
+; AVX512F-NEXT:    vmovdqa64 (%rcx), %zmm14
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm29
+; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm21
+; AVX512F-NEXT:    vpermt2q %zmm14, %zmm15, %zmm29
+; AVX512F-NEXT:    vmovdqa64 64(%rdx), %zmm1
+; AVX512F-NEXT:    vmovdqa64 64(%rcx), %zmm11
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm20
+; AVX512F-NEXT:    vpermt2q %zmm11, %zmm15, %zmm2
+; AVX512F-NEXT:    vmovdqu64 %zmm2, (%rsp) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 128(%rdx), %zmm23
+; AVX512F-NEXT:    vmovdqa64 128(%rcx), %zmm9
+; AVX512F-NEXT:    vmovdqa64 %zmm23, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm9, %zmm15, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 192(%rdx), %zmm18
+; AVX512F-NEXT:    vmovdqa64 192(%rcx), %zmm6
+; AVX512F-NEXT:    vmovdqa64 %zmm18, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm6, %zmm15, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 256(%rdx), %zmm17
+; AVX512F-NEXT:    vmovdqa64 256(%rcx), %zmm7
+; AVX512F-NEXT:    vmovdqa64 %zmm17, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm7, %zmm15, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 320(%rdx), %zmm15
+; AVX512F-NEXT:    vmovdqa64 320(%rcx), %zmm10
+; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm10, %zmm21, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 384(%rdx), %zmm13
+; AVX512F-NEXT:    vmovdqa64 384(%rcx), %zmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm21, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm1
+; AVX512F-NEXT:    vpermi2q %zmm0, %zmm4, %zmm12
+; AVX512F-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermi2q %zmm0, %zmm4, %zmm8
+; AVX512F-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 448(%rdx), %zmm16
+; AVX512F-NEXT:    vmovdqa64 448(%rcx), %zmm4
+; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm8
+; AVX512F-NEXT:    vpermt2q %zmm4, %zmm21, %zmm8
 ; AVX512F-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm22, %zmm4, %zmm0
+; AVX512F-NEXT:    vpermi2q %zmm0, %zmm1, %zmm21
+; AVX512F-NEXT:    vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm3, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [4,12,4,12]
+; AVX512F-NEXT:    # ymm1 = mem[0,1,0,1]
+; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm24
+; AVX512F-NEXT:    vpermt2q %zmm4, %zmm1, %zmm24
+; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm19
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm1, %zmm19
+; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm10, %zmm1, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm22, %zmm5, %zmm0
+; AVX512F-NEXT:    vmovdqa64 %zmm17, %zmm31
+; AVX512F-NEXT:    vpermt2q %zmm7, %zmm1, %zmm31
+; AVX512F-NEXT:    vmovdqa64 %zmm18, %zmm21
+; AVX512F-NEXT:    vpermt2q %zmm6, %zmm1, %zmm21
+; AVX512F-NEXT:    vmovdqa64 %zmm23, %zmm12
+; AVX512F-NEXT:    vpermt2q %zmm9, %zmm1, %zmm12
+; AVX512F-NEXT:    vmovdqa64 %zmm20, %zmm3
+; AVX512F-NEXT:    vpermt2q %zmm11, %zmm1, %zmm3
+; AVX512F-NEXT:    vpermi2q %zmm14, %zmm5, %zmm1
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm27 = [1,9,2,10,1,9,2,10]
+; AVX512F-NEXT:    # zmm27 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm14, %zmm27, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm22, %zmm14, %zmm0
+; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm28 = [5,13,6,14,5,13,6,14]
+; AVX512F-NEXT:    # zmm28 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm14, %zmm28, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm22, %zmm3, %zmm7
-; AVX512F-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermi2q %zmm21, %zmm6, %zmm4
-; AVX512F-NEXT:    vmovdqu64 %zmm4, (%rsp) # 64-byte Spill
-; AVX512F-NEXT:    vpermi2q %zmm21, %zmm6, %zmm5
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [3,11,3,11,3,11,3,11]
+; AVX512F-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpermt2q %zmm14, %zmm0, %zmm5
+; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm20, %zmm5
+; AVX512F-NEXT:    vpermt2q %zmm11, %zmm27, %zmm5
 ; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermi2q %zmm21, %zmm6, %zmm14
+; AVX512F-NEXT:    vmovdqa64 %zmm20, %zmm5
+; AVX512F-NEXT:    vpermt2q %zmm11, %zmm28, %zmm5
+; AVX512F-NEXT:    vpermt2q %zmm11, %zmm0, %zmm20
+; AVX512F-NEXT:    vmovdqu64 %zmm20, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm23, %zmm8
+; AVX512F-NEXT:    vpermt2q %zmm9, %zmm27, %zmm8
+; AVX512F-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm23, %zmm11
+; AVX512F-NEXT:    vpermt2q %zmm9, %zmm28, %zmm11
+; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm30
+; AVX512F-NEXT:    vpermt2q %zmm9, %zmm0, %zmm23
+; AVX512F-NEXT:    vmovdqu64 %zmm23, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm18, %zmm9
+; AVX512F-NEXT:    vpermt2q %zmm6, %zmm27, %zmm9
+; AVX512F-NEXT:    vmovdqa64 %zmm18, %zmm8
+; AVX512F-NEXT:    vpermt2q %zmm6, %zmm28, %zmm8
+; AVX512F-NEXT:    vpermt2q %zmm6, %zmm0, %zmm18
+; AVX512F-NEXT:    vmovdqu64 %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm17, %zmm6
+; AVX512F-NEXT:    vpermt2q %zmm7, %zmm27, %zmm6
+; AVX512F-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm17, %zmm6
+; AVX512F-NEXT:    vmovdqa64 %zmm17, %zmm14
+; AVX512F-NEXT:    vpermt2q %zmm7, %zmm28, %zmm6
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm11
+; AVX512F-NEXT:    vpermt2q %zmm7, %zmm0, %zmm14
 ; AVX512F-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm21, %zmm3, %zmm6
+; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm6
+; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm7
+; AVX512F-NEXT:    vpermt2q %zmm10, %zmm27, %zmm7
+; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm26
+; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm7
+; AVX512F-NEXT:    vpermt2q %zmm10, %zmm28, %zmm7
+; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm25
+; AVX512F-NEXT:    vpermt2q %zmm10, %zmm0, %zmm6
 ; AVX512F-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 448(%rdi), %zmm4
-; AVX512F-NEXT:    vmovdqa64 448(%rsi), %zmm11
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm28 = [4,12,5,13,4,12,5,13]
-; AVX512F-NEXT:    # zmm28 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm11, %zmm28, %zmm0
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm8
-; AVX512F-NEXT:    vmovdqa64 384(%rdi), %zmm2
-; AVX512F-NEXT:    vmovdqa64 384(%rsi), %zmm13
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm1
-; AVX512F-NEXT:    vpermt2q %zmm13, %zmm28, %zmm1
-; AVX512F-NEXT:    vmovdqa64 320(%rdi), %zmm3
-; AVX512F-NEXT:    vmovdqa64 320(%rsi), %zmm15
-; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm15, %zmm28, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 256(%rdi), %zmm31
-; AVX512F-NEXT:    vmovdqa64 256(%rsi), %zmm17
-; AVX512F-NEXT:    vmovdqa64 %zmm31, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm17, %zmm28, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 192(%rdi), %zmm29
-; AVX512F-NEXT:    vmovdqa64 192(%rsi), %zmm18
-; AVX512F-NEXT:    vmovdqa64 %zmm29, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm18, %zmm28, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 128(%rdi), %zmm25
-; AVX512F-NEXT:    vmovdqa64 128(%rsi), %zmm19
-; AVX512F-NEXT:    vmovdqa64 %zmm25, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm19, %zmm28, %zmm0
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm5
-; AVX512F-NEXT:    vmovdqa64 64(%rdi), %zmm6
-; AVX512F-NEXT:    vmovdqa64 64(%rsi), %zmm20
-; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm20, %zmm28, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 (%rdi), %zmm7
-; AVX512F-NEXT:    vmovdqa64 (%rsi), %zmm22
-; AVX512F-NEXT:    vpermi2q %zmm22, %zmm7, %zmm28
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm12 = [2,10,2,10,2,10,2,10]
-; AVX512F-NEXT:    # zmm12 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm14
-; AVX512F-NEXT:    vpermt2q %zmm22, %zmm12, %zmm14
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm10 = [6,14,6,14,6,14,6,14]
-; AVX512F-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm16
-; AVX512F-NEXT:    vpermt2q %zmm22, %zmm10, %zmm16
-; AVX512F-NEXT:    vbroadcasti64x4 {{.*#+}} zmm21 = [0,8,1,9,0,8,1,9]
-; AVX512F-NEXT:    # zmm21 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm22, %zmm21, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} ymm23 = [7,15,7,15]
-; AVX512F-NEXT:    # ymm23 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermt2q %zmm22, %zmm23, %zmm7
-; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm22
-; AVX512F-NEXT:    vpermt2q %zmm20, %zmm12, %zmm22
-; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm24
-; AVX512F-NEXT:    vpermt2q %zmm20, %zmm10, %zmm24
-; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm20, %zmm21, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm20, %zmm23, %zmm6
-; AVX512F-NEXT:    vmovdqa64 %zmm25, %zmm20
-; AVX512F-NEXT:    vpermt2q %zmm19, %zmm12, %zmm20
-; AVX512F-NEXT:    vmovdqa64 %zmm25, %zmm26
-; AVX512F-NEXT:    vpermt2q %zmm19, %zmm10, %zmm26
-; AVX512F-NEXT:    vmovdqa64 %zmm25, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm19, %zmm21, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm19, %zmm23, %zmm25
-; AVX512F-NEXT:    vmovdqa64 %zmm29, %zmm19
-; AVX512F-NEXT:    vpermt2q %zmm18, %zmm12, %zmm19
-; AVX512F-NEXT:    vmovdqa64 %zmm29, %zmm27
-; AVX512F-NEXT:    vpermt2q %zmm18, %zmm10, %zmm27
-; AVX512F-NEXT:    vmovdqa64 %zmm29, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm18, %zmm21, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm18, %zmm23, %zmm29
-; AVX512F-NEXT:    vmovdqa64 %zmm31, %zmm18
-; AVX512F-NEXT:    vpermt2q %zmm17, %zmm12, %zmm18
-; AVX512F-NEXT:    vmovdqa64 %zmm31, %zmm9
-; AVX512F-NEXT:    vpermt2q %zmm17, %zmm10, %zmm9
-; AVX512F-NEXT:    vmovdqa64 %zmm31, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm17, %zmm21, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm17, %zmm23, %zmm31
-; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm17
-; AVX512F-NEXT:    vpermt2q %zmm15, %zmm12, %zmm17
-; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm30
-; AVX512F-NEXT:    vpermt2q %zmm15, %zmm10, %zmm30
-; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm15, %zmm21, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm15, %zmm23, %zmm3
-; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm15
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vpermt2q %zmm13, %zmm12, %zmm3
-; AVX512F-NEXT:    vpermt2q %zmm13, %zmm10, %zmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm13, %zmm21, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm13, %zmm23, %zmm15
-; AVX512F-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermi2q %zmm11, %zmm4, %zmm12
-; AVX512F-NEXT:    vpermi2q %zmm11, %zmm4, %zmm10
-; AVX512F-NEXT:    vpermi2q %zmm11, %zmm4, %zmm21
-; AVX512F-NEXT:    vpermt2q %zmm11, %zmm23, %zmm4
-; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm6
+; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm7
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm27, %zmm7
+; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm10
+; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm7
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm28, %zmm7
+; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm23
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm0, %zmm6
+; AVX512F-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermi2q %zmm4, %zmm16, %zmm27
+; AVX512F-NEXT:    vpermi2q %zmm4, %zmm16, %zmm28
+; AVX512F-NEXT:    vpermt2q %zmm4, %zmm0, %zmm16
+; AVX512F-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    movb $12, %al
 ; AVX512F-NEXT:    kmovw %eax, %k1
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm8 {%k1}
-; AVX512F-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm1 {%k1}
-; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm8 {%k1}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm24, %zmm22 {%k1}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm19, %zmm18 {%k1}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm0 {%k1}
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm4 {%k1}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm5 {%k1}
-; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm1 {%k1}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm28 {%k1}
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm19 {%k1}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm31, %zmm17 {%k1}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm21, %zmm16 {%k1}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm21 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm12, %zmm21 {%k1}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm31 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm31 {%k1}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm20 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm20 {%k1}
 ; AVX512F-NEXT:    movb $48, %al
 ; AVX512F-NEXT:    kmovw %eax, %k2
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm14, %zmm11 {%k2}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm23 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm23 {%k2}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm22, %zmm13 {%k2}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm22 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm24, %zmm22 {%k2}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm20, %zmm14 {%k2}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm20 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm26, %zmm20 {%k2}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm19, %zmm16 {%k2}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm27, %zmm19 {%k2}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm24 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm18, %zmm24 {%k2}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm18 {%k2}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm26 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm17, %zmm26 {%k2}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm30, %zmm17 {%k2}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm27 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm27 {%k2}
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm0 {%k2}
-; AVX512F-NEXT:    vmovdqu64 (%rsp), %zmm15 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm12, %zmm15 {%k2}
-; AVX512F-NEXT:    vmovdqa64 (%r8), %zmm5
-; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm2 = <0,1,9,u,4,5,6,7>
-; AVX512F-NEXT:    vpermt2q %zmm5, %zmm2, %zmm11
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k2}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm3 {%k2}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vshufi64x2 {{.*#+}} zmm14 = zmm0[0,1,2,3],zmm29[4,5,6,7]
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm4 {%k2}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm5 {%k2}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vshufi64x2 $228, (%rsp), %zmm0, %zmm0 # 64-byte Folded Reload
+; AVX512F-NEXT:    # zmm0 = zmm0[0,1,2,3],mem[4,5,6,7]
+; AVX512F-NEXT:    vmovdqu64 %zmm0, (%rsp) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm6 {%k2}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm30 {%k2}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm15 # 64-byte Folded Reload
+; AVX512F-NEXT:    # zmm15 = zmm0[0,1,2,3],mem[4,5,6,7]
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm9 {%k2}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm8 {%k2}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm13 # 64-byte Folded Reload
+; AVX512F-NEXT:    # zmm13 = zmm0[0,1,2,3],mem[4,5,6,7]
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm7 {%k2}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm11 {%k2}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm12 # 64-byte Folded Reload
+; AVX512F-NEXT:    # zmm12 = zmm0[0,1,2,3],mem[4,5,6,7]
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm26 {%k2}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm25 {%k2}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; AVX512F-NEXT:    # zmm0 = zmm0[0,1,2,3],mem[4,5,6,7]
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm10 {%k2}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm23 {%k2}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; AVX512F-NEXT:    # zmm0 = zmm0[0,1,2,3],mem[4,5,6,7]
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm27 {%k2}
+; AVX512F-NEXT:    vmovdqa64 (%r8), %zmm29
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm1 = <0,1,9,u,4,5,6,7>
+; AVX512F-NEXT:    vpermt2q %zmm29, %zmm1, %zmm2
+; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 64(%r8), %zmm2
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm1, %zmm4
+; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 128(%r8), %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm1, %zmm6
+; AVX512F-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 192(%r8), %zmm4
+; AVX512F-NEXT:    vpermt2q %zmm4, %zmm1, %zmm9
+; AVX512F-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 256(%r8), %zmm6
+; AVX512F-NEXT:    vpermt2q %zmm6, %zmm1, %zmm7
+; AVX512F-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 320(%r8), %zmm7
+; AVX512F-NEXT:    vpermt2q %zmm7, %zmm1, %zmm26
+; AVX512F-NEXT:    vmovdqu64 %zmm26, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 384(%r8), %zmm9
+; AVX512F-NEXT:    vpermt2q %zmm9, %zmm1, %zmm10
+; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm24
+; AVX512F-NEXT:    vmovdqa64 448(%r8), %zmm10
+; AVX512F-NEXT:    vpermt2q %zmm10, %zmm1, %zmm27
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm28 {%k2}
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm1 = <0,1,13,u,4,5,6,7>
+; AVX512F-NEXT:    vpermt2q %zmm29, %zmm1, %zmm3
+; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm1, %zmm5
+; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm1, %zmm30
+; AVX512F-NEXT:    vmovdqu64 %zmm30, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm4, %zmm1, %zmm8
+; AVX512F-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm6, %zmm1, %zmm11
 ; AVX512F-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 64(%r8), %zmm3
-; AVX512F-NEXT:    vpermt2q %zmm3, %zmm2, %zmm13
-; AVX512F-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 128(%r8), %zmm9
-; AVX512F-NEXT:    vpermt2q %zmm9, %zmm2, %zmm14
+; AVX512F-NEXT:    vpermt2q %zmm7, %zmm1, %zmm25
+; AVX512F-NEXT:    vmovdqu64 %zmm25, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm9, %zmm1, %zmm23
+; AVX512F-NEXT:    vpermt2q %zmm10, %zmm1, %zmm28
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm3 # 64-byte Folded Reload
+; AVX512F-NEXT:    # zmm3 = zmm1[0,1,2,3],mem[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm1 = <14,u,2,3,4,5,15,u>
+; AVX512F-NEXT:    vpermt2q %zmm29, %zmm1, %zmm14
 ; AVX512F-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 192(%r8), %zmm11
-; AVX512F-NEXT:    vpermt2q %zmm11, %zmm2, %zmm16
-; AVX512F-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 256(%r8), %zmm13
-; AVX512F-NEXT:    vpermt2q %zmm13, %zmm2, %zmm24
-; AVX512F-NEXT:    vmovdqu64 %zmm24, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 320(%r8), %zmm14
-; AVX512F-NEXT:    vpermt2q %zmm14, %zmm2, %zmm26
-; AVX512F-NEXT:    vmovdqu64 %zmm26, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 384(%r8), %zmm16
-; AVX512F-NEXT:    vpermt2q %zmm16, %zmm2, %zmm27
-; AVX512F-NEXT:    vmovdqu64 %zmm27, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 448(%r8), %zmm12
-; AVX512F-NEXT:    vpermt2q %zmm12, %zmm2, %zmm15
-; AVX512F-NEXT:    vmovdqu64 %zmm15, (%rsp) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 (%rsp), %zmm14 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm1, %zmm14
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm1, %zmm15
+; AVX512F-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm4, %zmm1, %zmm13
+; AVX512F-NEXT:    vpermt2q %zmm6, %zmm1, %zmm12
+; AVX512F-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm7, %zmm1, %zmm12
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm15 {%k2}
-; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm2 = <0,1,13,u,4,5,6,7>
-; AVX512F-NEXT:    vpermt2q %zmm5, %zmm2, %zmm23
-; AVX512F-NEXT:    vpermt2q %zmm3, %zmm2, %zmm22
-; AVX512F-NEXT:    vmovdqu64 %zmm22, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm9, %zmm2, %zmm20
-; AVX512F-NEXT:    vmovdqa64 %zmm20, %zmm24
-; AVX512F-NEXT:    vpermt2q %zmm11, %zmm2, %zmm19
-; AVX512F-NEXT:    vpermt2q %zmm13, %zmm2, %zmm18
-; AVX512F-NEXT:    vmovdqa64 %zmm18, %zmm26
-; AVX512F-NEXT:    vpermt2q %zmm14, %zmm2, %zmm17
+; AVX512F-NEXT:    vpermt2q %zmm9, %zmm1, %zmm15
+; AVX512F-NEXT:    vpermt2q %zmm10, %zmm1, %zmm3
+; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa (%rdi), %ymm1
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm1[1],mem[1],ymm1[3],mem[3]
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm3, %zmm3
+; AVX512F-NEXT:    vmovdqa 64(%rdi), %ymm1
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm1[1],mem[1],ymm1[3],mem[3]
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm5, %zmm5
+; AVX512F-NEXT:    movb $16, %al
+; AVX512F-NEXT:    kmovw %eax, %k2
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm31 {%k2}
+; AVX512F-NEXT:    vmovdqu64 %zmm31, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm1 = <10,u,2,3,4,5,11,u>
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm1, %zmm5
+; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa 128(%rdi), %ymm2
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm2 = ymm2[1],mem[1],ymm2[3],mem[3]
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm5, %zmm31
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm21 {%k2}
+; AVX512F-NEXT:    vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm1, %zmm31
+; AVX512F-NEXT:    vmovdqa 192(%rdi), %ymm2
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm2 = ymm2[1],mem[1],ymm2[3],mem[3]
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm0, %zmm30
+; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm16 {%k2}
+; AVX512F-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm4, %zmm1, %zmm30
+; AVX512F-NEXT:    vmovdqa 256(%rdi), %ymm2
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm2 = ymm2[1],mem[1],ymm2[3],mem[3]
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm0, %zmm26
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm17 {%k2}
 ; AVX512F-NEXT:    vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm16, %zmm2, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm12, %zmm2, %zmm15
-; AVX512F-NEXT:    vmovdqa (%rdi), %ymm2
+; AVX512F-NEXT:    vpermt2q %zmm6, %zmm1, %zmm26
+; AVX512F-NEXT:    vmovdqa 320(%rdi), %ymm2
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm2 = ymm2[1],mem[1],ymm2[3],mem[3]
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm0, %zmm25
+; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm19 {%k2}
+; AVX512F-NEXT:    vmovdqu64 %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm7, %zmm1, %zmm25
+; AVX512F-NEXT:    vmovdqa 384(%rdi), %ymm2
 ; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm2 = ymm2[1],mem[1],ymm2[3],mem[3]
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm0, %zmm10
-; AVX512F-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm7, %zmm7 # 64-byte Folded Reload
-; AVX512F-NEXT:    # zmm7 = zmm7[0,1,2,3],mem[4,5,6,7]
-; AVX512F-NEXT:    vmovdqa 64(%rdi), %ymm2
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm0, %zmm19
+; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm18 {%k2}
+; AVX512F-NEXT:    vmovdqu64 %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm9, %zmm1, %zmm19
+; AVX512F-NEXT:    vmovdqa 448(%rdi), %ymm2
 ; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm2 = ymm2[1],mem[1],ymm2[3],mem[3]
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm0, %zmm0
-; AVX512F-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm6, %zmm6 # 64-byte Folded Reload
-; AVX512F-NEXT:    # zmm6 = zmm6[0,1,2,3],mem[4,5,6,7]
-; AVX512F-NEXT:    movb $16, %al
-; AVX512F-NEXT:    kmovw %eax, %k2
-; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm1 {%k2}
-; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm1 = <10,u,2,3,4,5,11,u>
-; AVX512F-NEXT:    vpermt2q %zmm3, %zmm1, %zmm0
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm0, %zmm16
+; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm22 {%k2}
+; AVX512F-NEXT:    vmovdqu64 %zmm22, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm10, %zmm1, %zmm16
+; AVX512F-NEXT:    vpermt2q %zmm29, %zmm1, %zmm3
+; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm29, %zmm20 {%k2}
+; AVX512F-NEXT:    vmovdqu64 %zmm20, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 (%r9), %zmm29
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [0,1,2,9,4,5,6,7]
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm29, %zmm6, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm2 = <14,u,2,3,4,5,15,u>
-; AVX512F-NEXT:    vpermt2q %zmm3, %zmm2, %zmm6
-; AVX512F-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa 128(%rdi), %ymm3
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm3 = ymm3[1],mem[1],ymm3[3],mem[3]
+; AVX512F-NEXT:    vmovdqa64 64(%r9), %zmm3
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm3
-; AVX512F-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm25, %zmm6 # 64-byte Folded Reload
-; AVX512F-NEXT:    # zmm6 = zmm25[0,1,2,3],mem[4,5,6,7]
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm6, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 128(%r9), %zmm4
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm0 {%k2}
+; AVX512F-NEXT:    vpermt2q %zmm4, %zmm6, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm9, %zmm1, %zmm3
-; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm9, %zmm2, %zmm6
-; AVX512F-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa 192(%rdi), %ymm3
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm3 = ymm3[1],mem[1],ymm3[3],mem[3]
+; AVX512F-NEXT:    vmovdqa64 192(%r9), %zmm5
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm0
-; AVX512F-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm29, %zmm3 # 64-byte Folded Reload
-; AVX512F-NEXT:    # zmm3 = zmm29[0,1,2,3],mem[4,5,6,7]
-; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm4 {%k2}
-; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm11, %zmm1, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm5, %zmm6, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm11, %zmm2, %zmm3
-; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa 256(%rdi), %ymm0
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm0 = ymm0[1],mem[1],ymm0[3],mem[3]
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm3, %zmm3
-; AVX512F-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm31, %zmm4 # 64-byte Folded Reload
-; AVX512F-NEXT:    # zmm4 = zmm31[0,1,2,3],mem[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 256(%r9), %zmm11
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm0 {%k2}
+; AVX512F-NEXT:    vpermt2q %zmm11, %zmm6, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm13, %zmm1, %zmm3
-; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm13, %zmm2, %zmm4
-; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa 320(%rdi), %ymm0
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm0 = ymm0[1],mem[1],ymm0[3],mem[3]
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm3, %zmm27
+; AVX512F-NEXT:    vmovdqa64 320(%r9), %zmm8
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm25 # 64-byte Folded Reload
-; AVX512F-NEXT:    # zmm25 = zmm0[0,1,2,3],mem[4,5,6,7]
-; AVX512F-NEXT:    vmovdqa64 %zmm14, %zmm8 {%k2}
-; AVX512F-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm14, %zmm1, %zmm27
-; AVX512F-NEXT:    vpermt2q %zmm14, %zmm2, %zmm25
-; AVX512F-NEXT:    vmovdqa 384(%rdi), %ymm0
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm0 = ymm0[1],mem[1],ymm0[3],mem[3]
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm3, %zmm22
+; AVX512F-NEXT:    vpermt2q %zmm8, %zmm6, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 384(%r9), %zmm7
+; AVX512F-NEXT:    vpermt2q %zmm7, %zmm6, %zmm24
+; AVX512F-NEXT:    vmovdqu64 %zmm24, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 448(%r9), %zmm22
+; AVX512F-NEXT:    vpermt2q %zmm22, %zmm6, %zmm27
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [0,1,2,13,4,5,6,7]
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm20 # 64-byte Folded Reload
-; AVX512F-NEXT:    # zmm20 = zmm0[0,1,2,3],mem[4,5,6,7]
+; AVX512F-NEXT:    vpermt2q %zmm29, %zmm6, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm0 {%k2}
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm6, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm16, %zmm1, %zmm22
-; AVX512F-NEXT:    vpermt2q %zmm16, %zmm2, %zmm20
-; AVX512F-NEXT:    vmovdqa 448(%rdi), %ymm0
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm0 = ymm0[1],mem[1],ymm0[3],mem[3]
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm3, %zmm18
-; AVX512F-NEXT:    vpermt2q %zmm5, %zmm1, %zmm10
-; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm11
-; AVX512F-NEXT:    vpermt2q %zmm12, %zmm1, %zmm18
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm17 # 64-byte Folded Reload
-; AVX512F-NEXT:    # zmm17 = zmm0[0,1,2,3],mem[4,5,6,7]
+; AVX512F-NEXT:    vpermt2q %zmm4, %zmm6, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm12, %zmm0 {%k2}
+; AVX512F-NEXT:    vpermt2q %zmm5, %zmm6, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm12, %zmm2, %zmm17
-; AVX512F-NEXT:    vpermt2q %zmm5, %zmm2, %zmm7
-; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm13
-; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm28 {%k2}
-; AVX512F-NEXT:    vmovdqa64 (%r9), %zmm6
-; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm10 = [0,1,2,9,4,5,6,7]
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm6, %zmm10, %zmm1
-; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 64(%r9), %zmm7
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm7, %zmm10, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 128(%r9), %zmm2
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm2, %zmm10, %zmm3
-; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 192(%r9), %zmm3
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm3, %zmm10, %zmm4
-; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 256(%r9), %zmm4
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm4, %zmm10, %zmm5
-; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 320(%r9), %zmm5
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm5, %zmm10, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm11, %zmm6, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 384(%r9), %zmm8
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm8, %zmm10, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm8, %zmm6, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 448(%r9), %zmm9
-; AVX512F-NEXT:    vmovdqu64 (%rsp), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm9, %zmm10, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, (%rsp) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm10 = [0,1,2,13,4,5,6,7]
-; AVX512F-NEXT:    vpermt2q %zmm6, %zmm10, %zmm23
+; AVX512F-NEXT:    vpermt2q %zmm7, %zmm6, %zmm23
 ; AVX512F-NEXT:    vmovdqu64 %zmm23, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm7, %zmm10, %zmm12
-; AVX512F-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm2, %zmm10, %zmm24
-; AVX512F-NEXT:    vmovdqu64 %zmm24, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm3, %zmm10, %zmm19
-; AVX512F-NEXT:    vmovdqu64 %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm4, %zmm10, %zmm26
-; AVX512F-NEXT:    vmovdqu64 %zmm26, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm22, %zmm6, %zmm28
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [0,14,2,3,4,5,6,15]
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm29, %zmm6, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm6, %zmm14
+; AVX512F-NEXT:    vmovdqu64 %zmm14, (%rsp) # 64-byte Spill
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm5, %zmm10, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm4, %zmm6, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm5, %zmm6, %zmm13
+; AVX512F-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm8, %zmm10, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm11, %zmm6, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm9, %zmm10, %zmm15
+; AVX512F-NEXT:    vpermt2q %zmm8, %zmm6, %zmm12
+; AVX512F-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm7, %zmm6, %zmm15
 ; AVX512F-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa (%rdx), %xmm10
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} xmm10 = xmm10[0],mem[0]
-; AVX512F-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm10, %zmm0, %zmm12 {%k1}
-; AVX512F-NEXT:    vmovdqa 64(%rdx), %xmm10
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} xmm10 = xmm10[0],mem[0]
-; AVX512F-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm10, %zmm0, %zmm14 {%k1}
-; AVX512F-NEXT:    vmovdqa 128(%rdx), %xmm10
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} xmm10 = xmm10[0],mem[0]
-; AVX512F-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm10, %zmm0, %zmm16 {%k1}
-; AVX512F-NEXT:    vmovdqa 192(%rdx), %xmm10
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} xmm10 = xmm10[0],mem[0]
-; AVX512F-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm30 # 64-byte Reload
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm10, %zmm0, %zmm30 {%k1}
-; AVX512F-NEXT:    vmovdqa 256(%rdx), %xmm10
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} xmm10 = xmm10[0],mem[0]
-; AVX512F-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm31 # 64-byte Reload
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm10, %zmm0, %zmm31 {%k1}
-; AVX512F-NEXT:    vmovdqa 320(%rdx), %xmm10
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} xmm10 = xmm10[0],mem[0]
-; AVX512F-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm10, %zmm0, %zmm1 {%k1}
-; AVX512F-NEXT:    vmovdqa 384(%rdx), %xmm10
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} xmm10 = xmm10[0],mem[0]
-; AVX512F-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm10, %zmm0, %zmm0 {%k1}
-; AVX512F-NEXT:    vmovdqa 448(%rdx), %xmm10
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} xmm10 = xmm10[0],mem[0]
-; AVX512F-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm10, %zmm0, %zmm21 {%k1}
-; AVX512F-NEXT:    vinserti32x4 $2, (%r8), %zmm12, %zmm10
-; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm29 = [0,1,2,3,4,8,6,7]
-; AVX512F-NEXT:    vpermt2q %zmm6, %zmm29, %zmm10
-; AVX512F-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vinserti32x4 $2, 64(%r8), %zmm14, %zmm10
-; AVX512F-NEXT:    vpermt2q %zmm7, %zmm29, %zmm10
-; AVX512F-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vinserti32x4 $2, 128(%r8), %zmm16, %zmm10
-; AVX512F-NEXT:    vpermt2q %zmm2, %zmm29, %zmm10
-; AVX512F-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vinserti32x4 $2, 192(%r8), %zmm30, %zmm26
-; AVX512F-NEXT:    vpermt2q %zmm3, %zmm29, %zmm26
-; AVX512F-NEXT:    vinserti32x4 $2, 256(%r8), %zmm31, %zmm24
-; AVX512F-NEXT:    vpermt2q %zmm4, %zmm29, %zmm24
-; AVX512F-NEXT:    vinserti32x4 $2, 320(%r8), %zmm1, %zmm23
-; AVX512F-NEXT:    vpermt2q %zmm5, %zmm29, %zmm23
-; AVX512F-NEXT:    vinserti32x4 $2, 384(%r8), %zmm0, %zmm19
-; AVX512F-NEXT:    vpermt2q %zmm8, %zmm29, %zmm19
-; AVX512F-NEXT:    vinserti32x4 $2, 448(%r8), %zmm21, %zmm21
-; AVX512F-NEXT:    vpermt2q %zmm9, %zmm29, %zmm21
-; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm29 = [0,10,2,3,4,5,6,11]
-; AVX512F-NEXT:    vpermt2q %zmm6, %zmm29, %zmm11
-; AVX512F-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm30 = [0,1,2,3,4,12,6,7]
-; AVX512F-NEXT:    vpermt2q %zmm6, %zmm30, %zmm28
-; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm31 = [0,14,2,3,4,5,6,15]
-; AVX512F-NEXT:    vpermt2q %zmm6, %zmm31, %zmm13
-; AVX512F-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm7, %zmm29, %zmm16
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm7, %zmm30, %zmm15
+; AVX512F-NEXT:    vpermt2q %zmm22, %zmm6, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa (%rdx), %xmm6
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} xmm6 = xmm6[0],mem[0]
+; AVX512F-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm6, %zmm0, %zmm0 {%k1}
+; AVX512F-NEXT:    vmovdqa 64(%rdx), %xmm6
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} xmm6 = xmm6[0],mem[0]
+; AVX512F-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm7, %zmm31, %zmm14
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm2, %zmm29, %zmm13
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm2, %zmm30, %zmm12
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm2, %zmm31, %zmm11
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm6, %zmm0, %zmm14 {%k1}
+; AVX512F-NEXT:    vmovdqa 128(%rdx), %xmm6
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} xmm6 = xmm6[0],mem[0]
+; AVX512F-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm6, %zmm0, %zmm15 {%k1}
+; AVX512F-NEXT:    vmovdqa 192(%rdx), %xmm6
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} xmm6 = xmm6[0],mem[0]
+; AVX512F-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm3, %zmm29, %zmm10
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm3, %zmm30, %zmm7
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm6, %zmm0, %zmm10 {%k1}
+; AVX512F-NEXT:    vmovdqa 256(%rdx), %xmm6
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} xmm6 = xmm6[0],mem[0]
+; AVX512F-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm6, %zmm0, %zmm12 {%k1}
+; AVX512F-NEXT:    vmovdqa 320(%rdx), %xmm6
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} xmm6 = xmm6[0],mem[0]
+; AVX512F-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm3, %zmm31, %zmm2
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm6, %zmm0, %zmm2 {%k1}
+; AVX512F-NEXT:    vmovdqa 384(%rdx), %xmm6
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} xmm6 = xmm6[0],mem[0]
+; AVX512F-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm6, %zmm0, %zmm9 {%k1}
+; AVX512F-NEXT:    vmovdqa 448(%rdx), %xmm6
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} xmm6 = xmm6[0],mem[0]
+; AVX512F-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm4, %zmm29, %zmm1
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm6, %zmm0, %zmm1 {%k1}
+; AVX512F-NEXT:    vinserti32x4 $2, (%r8), %zmm0, %zmm24
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [0,1,2,3,4,8,6,7]
+; AVX512F-NEXT:    vpermt2q %zmm29, %zmm0, %zmm24
+; AVX512F-NEXT:    vinserti32x4 $2, 64(%r8), %zmm14, %zmm23
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm0, %zmm23
+; AVX512F-NEXT:    vinserti32x4 $2, 128(%r8), %zmm15, %zmm21
+; AVX512F-NEXT:    vpermt2q %zmm4, %zmm0, %zmm21
+; AVX512F-NEXT:    vinserti32x4 $2, 192(%r8), %zmm10, %zmm20
+; AVX512F-NEXT:    vpermt2q %zmm5, %zmm0, %zmm20
+; AVX512F-NEXT:    vinserti32x4 $2, 256(%r8), %zmm12, %zmm18
+; AVX512F-NEXT:    vpermt2q %zmm11, %zmm0, %zmm18
+; AVX512F-NEXT:    vinserti32x4 $2, 320(%r8), %zmm2, %zmm17
+; AVX512F-NEXT:    vpermt2q %zmm8, %zmm0, %zmm17
+; AVX512F-NEXT:    vinserti32x4 $2, 384(%r8), %zmm9, %zmm15
+; AVX512F-NEXT:    vpermt2q %zmm7, %zmm0, %zmm15
+; AVX512F-NEXT:    vinserti32x4 $2, 448(%r8), %zmm1, %zmm14
+; AVX512F-NEXT:    vpermt2q %zmm22, %zmm0, %zmm14
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [0,10,2,3,4,5,6,11]
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm29, %zmm0, %zmm13
+; AVX512F-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,1,2,3,4,12,6,7]
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm29, %zmm1, %zmm12
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm0, %zmm10
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm1, %zmm9
+; AVX512F-NEXT:    vpermt2q %zmm4, %zmm0, %zmm31
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm4, %zmm1, %zmm6
+; AVX512F-NEXT:    vpermt2q %zmm5, %zmm0, %zmm30
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm29 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm5, %zmm1, %zmm29
+; AVX512F-NEXT:    vpermt2q %zmm11, %zmm0, %zmm26
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm11, %zmm1, %zmm4
+; AVX512F-NEXT:    vpermt2q %zmm8, %zmm0, %zmm25
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm4, %zmm30, %zmm3
+; AVX512F-NEXT:    vpermt2q %zmm8, %zmm1, %zmm3
+; AVX512F-NEXT:    vpermt2q %zmm7, %zmm0, %zmm19
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm7, %zmm1, %zmm2
+; AVX512F-NEXT:    vpermt2q %zmm22, %zmm0, %zmm16
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm4, %zmm31, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm5, %zmm29, %zmm27
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm5, %zmm30, %zmm4
-; AVX512F-NEXT:    vpermt2q %zmm5, %zmm31, %zmm25
-; AVX512F-NEXT:    vpermt2q %zmm8, %zmm29, %zmm22
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm8, %zmm30, %zmm5
-; AVX512F-NEXT:    vpermt2q %zmm8, %zmm31, %zmm20
-; AVX512F-NEXT:    vpermt2q %zmm9, %zmm29, %zmm18
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm9, %zmm30, %zmm6
-; AVX512F-NEXT:    vpermt2q %zmm9, %zmm31, %zmm17
+; AVX512F-NEXT:    vpermt2q %zmm22, %zmm1, %zmm0
 ; AVX512F-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512F-NEXT:    vmovdqa64 %zmm17, 3008(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm8, 2944(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm6, 2880(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm18, 2816(%rax)
-; AVX512F-NEXT:    vmovups (%rsp), %zmm6 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm6, 2752(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm20, 2624(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm6, 2560(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm5, 2496(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm22, 2432(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm5, 2368(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm25, 2240(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm5, 2176(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm4, 2112(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm27, 2048(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm4, 1984(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm0, 1856(%rax)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm1, 3008(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm28, 2944(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm0, 2880(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm16, 2816(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm27, 2752(%rax)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm0, 2624(%rax)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm0, 2560(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm2, 2496(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm19, 2432(%rax)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm0, 2368(%rax)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm0, 2240(%rax)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm0, 2176(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm3, 2112(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm25, 2048(%rax)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm0, 1984(%rax)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm0, 1856(%rax)
 ; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-NEXT:    vmovaps %zmm0, 1792(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm3, 1728(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm1, 1664(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm3, 1600(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm2, 1472(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm4, 1728(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm26, 1664(%rax)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm0, 1600(%rax)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm0, 1472(%rax)
 ; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-NEXT:    vmovaps %zmm0, 1408(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm7, 1344(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm10, 1280(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm2, 1216(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm11, 1088(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm29, 1344(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm30, 1280(%rax)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm0, 1216(%rax)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm0, 1088(%rax)
 ; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-NEXT:    vmovaps %zmm0, 1024(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm12, 960(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm13, 896(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm1, 832(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm14, 704(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm1, 640(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm15, 576(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm16, 512(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm6, 960(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm31, 896(%rax)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm0, 832(%rax)
+; AVX512F-NEXT:    vmovups (%rsp), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm0, 704(%rax)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm0, 640(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm9, 576(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm10, 512(%rax)
 ; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-NEXT:    vmovaps %zmm0, 448(%rax)
 ; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-NEXT:    vmovaps %zmm0, 320(%rax)
 ; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-NEXT:    vmovaps %zmm0, 256(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm28, 192(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 128(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm12, 192(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm13, 128(%rax)
 ; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-NEXT:    vmovaps %zmm0, 64(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm21, 2688(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm19, 2304(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm23, 1920(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm24, 1536(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm26, 1152(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 768(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 384(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, (%rax)
-; AVX512F-NEXT:    addq $3400, %rsp # imm = 0xD48
+; AVX512F-NEXT:    vmovdqa64 %zmm14, 2688(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm15, 2304(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm17, 1920(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm18, 1536(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm20, 1152(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm21, 768(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm23, 384(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm24, (%rax)
+; AVX512F-NEXT:    addq $3720, %rsp # imm = 0xE88
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: store_i64_stride6_vf64:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    subq $3400, %rsp # imm = 0xD48
-; AVX512BW-NEXT:    vmovdqa64 (%rdx), %zmm13
-; AVX512BW-NEXT:    vmovdqa64 64(%rdx), %zmm12
-; AVX512BW-NEXT:    vmovdqa64 128(%rdx), %zmm11
-; AVX512BW-NEXT:    vmovdqa64 192(%rdx), %zmm10
-; AVX512BW-NEXT:    vmovdqa64 256(%rdx), %zmm9
-; AVX512BW-NEXT:    vmovdqa64 320(%rdx), %zmm8
-; AVX512BW-NEXT:    vmovdqa64 384(%rdx), %zmm7
-; AVX512BW-NEXT:    vmovdqa64 448(%rdx), %zmm6
-; AVX512BW-NEXT:    vmovdqa64 (%rcx), %zmm0
-; AVX512BW-NEXT:    vmovdqa64 64(%rcx), %zmm1
-; AVX512BW-NEXT:    vmovdqa64 128(%rcx), %zmm2
-; AVX512BW-NEXT:    vmovdqa64 192(%rcx), %zmm30
-; AVX512BW-NEXT:    vmovdqa64 256(%rcx), %zmm27
-; AVX512BW-NEXT:    vmovdqa64 320(%rcx), %zmm24
-; AVX512BW-NEXT:    vmovdqa64 384(%rcx), %zmm22
-; AVX512BW-NEXT:    vmovdqa64 448(%rcx), %zmm21
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm3 = [4,12,4,12]
-; AVX512BW-NEXT:    # ymm3 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm4
-; AVX512BW-NEXT:    vpermt2q %zmm21, %zmm3, %zmm4
-; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm4
-; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm3, %zmm4
-; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm4
-; AVX512BW-NEXT:    vpermt2q %zmm24, %zmm3, %zmm4
-; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm4
-; AVX512BW-NEXT:    vpermt2q %zmm27, %zmm3, %zmm4
-; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm4
-; AVX512BW-NEXT:    vpermt2q %zmm30, %zmm3, %zmm4
-; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm4
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm3, %zmm4
-; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm4
-; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm3, %zmm4
-; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermi2q %zmm0, %zmm13, %zmm3
+; AVX512BW-NEXT:    subq $3720, %rsp # imm = 0xE88
+; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm11
+; AVX512BW-NEXT:    vmovdqa64 64(%rdi), %zmm10
+; AVX512BW-NEXT:    vmovdqa64 128(%rdi), %zmm9
+; AVX512BW-NEXT:    vmovdqa64 192(%rdi), %zmm8
+; AVX512BW-NEXT:    vmovdqa64 256(%rdi), %zmm7
+; AVX512BW-NEXT:    vmovdqa64 320(%rdi), %zmm6
+; AVX512BW-NEXT:    vmovdqa64 384(%rdi), %zmm5
+; AVX512BW-NEXT:    vmovdqa64 448(%rdi), %zmm4
+; AVX512BW-NEXT:    vmovdqa64 (%rsi), %zmm30
+; AVX512BW-NEXT:    vmovdqa64 64(%rsi), %zmm29
+; AVX512BW-NEXT:    vmovdqa64 128(%rsi), %zmm28
+; AVX512BW-NEXT:    vmovdqa64 192(%rsi), %zmm27
+; AVX512BW-NEXT:    vmovdqa64 256(%rsi), %zmm1
+; AVX512BW-NEXT:    vmovdqa64 320(%rsi), %zmm13
+; AVX512BW-NEXT:    vmovdqa64 384(%rsi), %zmm24
+; AVX512BW-NEXT:    vmovdqa64 448(%rsi), %zmm0
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [4,12,5,13,4,12,5,13]
+; AVX512BW-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm22
+; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm2, %zmm22
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm3
+; AVX512BW-NEXT:    vpermt2q %zmm24, %zmm2, %zmm3
 ; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm5 = [1,9,2,10,1,9,2,10]
-; AVX512BW-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm3
-; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm5, %zmm3
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm3
+; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm2, %zmm3
 ; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm4
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm5 = [5,13,6,14,5,13,6,14]
-; AVX512BW-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm3
-; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm5, %zmm3
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm3
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm2, %zmm3
 ; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm3
-; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm4, %zmm3
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm3
+; AVX512BW-NEXT:    vpermt2q %zmm27, %zmm2, %zmm3
 ; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm14 = [3,11,3,11,3,11,3,11]
-; AVX512BW-NEXT:    # zmm14 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm3
-; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm14, %zmm3
+; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm3
+; AVX512BW-NEXT:    vpermt2q %zmm28, %zmm2, %zmm3
 ; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm3 = [7,15,7,15,7,15,7,15]
-; AVX512BW-NEXT:    # zmm3 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm3, %zmm13
-; AVX512BW-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm5, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm14, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm3, %zmm12
-; AVX512BW-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm4, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm5, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm14, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm3, %zmm11
+; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm3
+; AVX512BW-NEXT:    vpermt2q %zmm29, %zmm2, %zmm3
+; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermi2q %zmm30, %zmm11, %zmm2
+; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm2 = [2,10,2,10,2,10,2,10]
+; AVX512BW-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm3
+; AVX512BW-NEXT:    vpermt2q %zmm30, %zmm2, %zmm3
+; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm12
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm2 = [6,14,6,14,6,14,6,14]
+; AVX512BW-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm3
+; AVX512BW-NEXT:    vpermt2q %zmm30, %zmm2, %zmm3
+; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm15 = [7,15,7,15,7,15,7,15]
+; AVX512BW-NEXT:    # zmm15 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm3
+; AVX512BW-NEXT:    vpermt2q %zmm30, %zmm15, %zmm3
+; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm3
+; AVX512BW-NEXT:    vpermt2q %zmm29, %zmm12, %zmm3
+; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm3 = [0,8,1,9,0,8,1,9]
+; AVX512BW-NEXT:    # zmm3 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpermt2q %zmm30, %zmm3, %zmm11
 ; AVX512BW-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm30, %zmm4, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm30, %zmm5, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm30, %zmm14, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm30, %zmm3, %zmm10
+; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm11
+; AVX512BW-NEXT:    vpermt2q %zmm29, %zmm2, %zmm11
+; AVX512BW-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm11
+; AVX512BW-NEXT:    vpermt2q %zmm29, %zmm15, %zmm11
+; AVX512BW-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm29, %zmm3, %zmm10
 ; AVX512BW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm27, %zmm4, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm27, %zmm5, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm27, %zmm14, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm27, %zmm3, %zmm9
+; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm10
+; AVX512BW-NEXT:    vpermt2q %zmm28, %zmm12, %zmm10
+; AVX512BW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm10
+; AVX512BW-NEXT:    vpermt2q %zmm28, %zmm2, %zmm10
+; AVX512BW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm10
+; AVX512BW-NEXT:    vpermt2q %zmm28, %zmm15, %zmm10
+; AVX512BW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm28, %zmm3, %zmm9
 ; AVX512BW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm24, %zmm4, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm24, %zmm5, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm24, %zmm14, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm24, %zmm3, %zmm8
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm9
+; AVX512BW-NEXT:    vpermt2q %zmm27, %zmm12, %zmm9
+; AVX512BW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm9
+; AVX512BW-NEXT:    vpermt2q %zmm27, %zmm2, %zmm9
+; AVX512BW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm9
+; AVX512BW-NEXT:    vpermt2q %zmm27, %zmm15, %zmm9
+; AVX512BW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm27, %zmm3, %zmm8
+; AVX512BW-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm8
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm12, %zmm8
+; AVX512BW-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm8
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm2, %zmm8
 ; AVX512BW-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm4, %zmm0
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm8
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm15, %zmm8
+; AVX512BW-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm3, %zmm7
+; AVX512BW-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm12, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm2, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm15, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm3, %zmm6
+; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm24, %zmm12, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm24, %zmm2, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm8
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm24, %zmm15, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm24, %zmm3, %zmm5
+; AVX512BW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 (%rdx), %zmm5
+; AVX512BW-NEXT:    vmovdqa64 (%rcx), %zmm14
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm29
+; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm21
+; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm15, %zmm29
+; AVX512BW-NEXT:    vmovdqa64 64(%rdx), %zmm1
+; AVX512BW-NEXT:    vmovdqa64 64(%rcx), %zmm11
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm2
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm20
+; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm15, %zmm2
+; AVX512BW-NEXT:    vmovdqu64 %zmm2, (%rsp) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 128(%rdx), %zmm23
+; AVX512BW-NEXT:    vmovdqa64 128(%rcx), %zmm9
+; AVX512BW-NEXT:    vmovdqa64 %zmm23, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm15, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 192(%rdx), %zmm18
+; AVX512BW-NEXT:    vmovdqa64 192(%rcx), %zmm6
+; AVX512BW-NEXT:    vmovdqa64 %zmm18, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm15, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 256(%rdx), %zmm17
+; AVX512BW-NEXT:    vmovdqa64 256(%rcx), %zmm7
+; AVX512BW-NEXT:    vmovdqa64 %zmm17, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm15, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 320(%rdx), %zmm15
+; AVX512BW-NEXT:    vmovdqa64 320(%rcx), %zmm10
+; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm10, %zmm21, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 384(%rdx), %zmm13
+; AVX512BW-NEXT:    vmovdqa64 384(%rcx), %zmm2
+; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm21, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm1
+; AVX512BW-NEXT:    vpermi2q %zmm0, %zmm4, %zmm12
+; AVX512BW-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermi2q %zmm0, %zmm4, %zmm8
+; AVX512BW-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 448(%rdx), %zmm16
+; AVX512BW-NEXT:    vmovdqa64 448(%rcx), %zmm4
+; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm8
+; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm21, %zmm8
+; AVX512BW-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermi2q %zmm0, %zmm1, %zmm21
+; AVX512BW-NEXT:    vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm3, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [4,12,4,12]
+; AVX512BW-NEXT:    # ymm1 = mem[0,1,0,1]
+; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm24
+; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm1, %zmm24
+; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm19
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm1, %zmm19
+; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm10, %zmm1, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm5, %zmm0
+; AVX512BW-NEXT:    vmovdqa64 %zmm17, %zmm31
+; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm1, %zmm31
+; AVX512BW-NEXT:    vmovdqa64 %zmm18, %zmm21
+; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm1, %zmm21
+; AVX512BW-NEXT:    vmovdqa64 %zmm23, %zmm12
+; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm1, %zmm12
+; AVX512BW-NEXT:    vmovdqa64 %zmm20, %zmm3
+; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm1, %zmm3
+; AVX512BW-NEXT:    vpermi2q %zmm14, %zmm5, %zmm1
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm27 = [1,9,2,10,1,9,2,10]
+; AVX512BW-NEXT:    # zmm27 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm27, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm14, %zmm0
+; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm28 = [5,13,6,14,5,13,6,14]
+; AVX512BW-NEXT:    # zmm28 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm28, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm3, %zmm7
-; AVX512BW-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermi2q %zmm21, %zmm6, %zmm4
-; AVX512BW-NEXT:    vmovdqu64 %zmm4, (%rsp) # 64-byte Spill
-; AVX512BW-NEXT:    vpermi2q %zmm21, %zmm6, %zmm5
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [3,11,3,11,3,11,3,11]
+; AVX512BW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm0, %zmm5
+; AVX512BW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm20, %zmm5
+; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm27, %zmm5
 ; AVX512BW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermi2q %zmm21, %zmm6, %zmm14
+; AVX512BW-NEXT:    vmovdqa64 %zmm20, %zmm5
+; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm28, %zmm5
+; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm0, %zmm20
+; AVX512BW-NEXT:    vmovdqu64 %zmm20, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm23, %zmm8
+; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm27, %zmm8
+; AVX512BW-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm23, %zmm11
+; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm28, %zmm11
+; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm30
+; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm0, %zmm23
+; AVX512BW-NEXT:    vmovdqu64 %zmm23, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm18, %zmm9
+; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm27, %zmm9
+; AVX512BW-NEXT:    vmovdqa64 %zmm18, %zmm8
+; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm28, %zmm8
+; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm0, %zmm18
+; AVX512BW-NEXT:    vmovdqu64 %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm17, %zmm6
+; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm27, %zmm6
+; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm17, %zmm6
+; AVX512BW-NEXT:    vmovdqa64 %zmm17, %zmm14
+; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm28, %zmm6
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm11
+; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm0, %zmm14
 ; AVX512BW-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm21, %zmm3, %zmm6
+; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm6
+; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm7
+; AVX512BW-NEXT:    vpermt2q %zmm10, %zmm27, %zmm7
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm26
+; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm7
+; AVX512BW-NEXT:    vpermt2q %zmm10, %zmm28, %zmm7
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm25
+; AVX512BW-NEXT:    vpermt2q %zmm10, %zmm0, %zmm6
 ; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 448(%rdi), %zmm4
-; AVX512BW-NEXT:    vmovdqa64 448(%rsi), %zmm11
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm28 = [4,12,5,13,4,12,5,13]
-; AVX512BW-NEXT:    # zmm28 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm28, %zmm0
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm8
-; AVX512BW-NEXT:    vmovdqa64 384(%rdi), %zmm2
-; AVX512BW-NEXT:    vmovdqa64 384(%rsi), %zmm13
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm1
-; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm28, %zmm1
-; AVX512BW-NEXT:    vmovdqa64 320(%rdi), %zmm3
-; AVX512BW-NEXT:    vmovdqa64 320(%rsi), %zmm15
-; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm15, %zmm28, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 256(%rdi), %zmm31
-; AVX512BW-NEXT:    vmovdqa64 256(%rsi), %zmm17
-; AVX512BW-NEXT:    vmovdqa64 %zmm31, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm28, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 192(%rdi), %zmm29
-; AVX512BW-NEXT:    vmovdqa64 192(%rsi), %zmm18
-; AVX512BW-NEXT:    vmovdqa64 %zmm29, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm28, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 128(%rdi), %zmm25
-; AVX512BW-NEXT:    vmovdqa64 128(%rsi), %zmm19
-; AVX512BW-NEXT:    vmovdqa64 %zmm25, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm28, %zmm0
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm5
-; AVX512BW-NEXT:    vmovdqa64 64(%rdi), %zmm6
-; AVX512BW-NEXT:    vmovdqa64 64(%rsi), %zmm20
-; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm28, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm7
-; AVX512BW-NEXT:    vmovdqa64 (%rsi), %zmm22
-; AVX512BW-NEXT:    vpermi2q %zmm22, %zmm7, %zmm28
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm12 = [2,10,2,10,2,10,2,10]
-; AVX512BW-NEXT:    # zmm12 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm14
-; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm12, %zmm14
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm10 = [6,14,6,14,6,14,6,14]
-; AVX512BW-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm16
-; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm10, %zmm16
-; AVX512BW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm21 = [0,8,1,9,0,8,1,9]
-; AVX512BW-NEXT:    # zmm21 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm21, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} ymm23 = [7,15,7,15]
-; AVX512BW-NEXT:    # ymm23 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm23, %zmm7
-; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm22
-; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm12, %zmm22
-; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm24
-; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm10, %zmm24
-; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm21, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm23, %zmm6
-; AVX512BW-NEXT:    vmovdqa64 %zmm25, %zmm20
-; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm12, %zmm20
-; AVX512BW-NEXT:    vmovdqa64 %zmm25, %zmm26
-; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm10, %zmm26
-; AVX512BW-NEXT:    vmovdqa64 %zmm25, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm21, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm23, %zmm25
-; AVX512BW-NEXT:    vmovdqa64 %zmm29, %zmm19
-; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm12, %zmm19
-; AVX512BW-NEXT:    vmovdqa64 %zmm29, %zmm27
-; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm10, %zmm27
-; AVX512BW-NEXT:    vmovdqa64 %zmm29, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm21, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm23, %zmm29
-; AVX512BW-NEXT:    vmovdqa64 %zmm31, %zmm18
-; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm12, %zmm18
-; AVX512BW-NEXT:    vmovdqa64 %zmm31, %zmm9
-; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm10, %zmm9
-; AVX512BW-NEXT:    vmovdqa64 %zmm31, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm21, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm23, %zmm31
-; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm17
-; AVX512BW-NEXT:    vpermt2q %zmm15, %zmm12, %zmm17
-; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm30
-; AVX512BW-NEXT:    vpermt2q %zmm15, %zmm10, %zmm30
-; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm15, %zmm21, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm15, %zmm23, %zmm3
-; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm15
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm12, %zmm3
-; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm10, %zmm2
-; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm21, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm23, %zmm15
-; AVX512BW-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermi2q %zmm11, %zmm4, %zmm12
-; AVX512BW-NEXT:    vpermi2q %zmm11, %zmm4, %zmm10
-; AVX512BW-NEXT:    vpermi2q %zmm11, %zmm4, %zmm21
-; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm23, %zmm4
-; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm6
+; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm7
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm27, %zmm7
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm10
+; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm7
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm28, %zmm7
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm23
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm0, %zmm6
+; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermi2q %zmm4, %zmm16, %zmm27
+; AVX512BW-NEXT:    vpermi2q %zmm4, %zmm16, %zmm28
+; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm0, %zmm16
+; AVX512BW-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    movb $12, %al
 ; AVX512BW-NEXT:    kmovd %eax, %k1
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm8 {%k1}
-; AVX512BW-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm1 {%k1}
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm8 {%k1}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm24, %zmm22 {%k1}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm19, %zmm18 {%k1}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm0 {%k1}
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm4 {%k1}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm5 {%k1}
-; AVX512BW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm1 {%k1}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm28 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm19 {%k1}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm31, %zmm17 {%k1}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm21, %zmm16 {%k1}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm21 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm21 {%k1}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm31 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm31 {%k1}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm20 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm20 {%k1}
 ; AVX512BW-NEXT:    movb $48, %al
 ; AVX512BW-NEXT:    kmovd %eax, %k2
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm11 {%k2}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm23 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm23 {%k2}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm22, %zmm13 {%k2}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm22 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm24, %zmm22 {%k2}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm20, %zmm14 {%k2}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm20 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm26, %zmm20 {%k2}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm19, %zmm16 {%k2}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm27, %zmm19 {%k2}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm24 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm18, %zmm24 {%k2}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm18 {%k2}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm26 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm17, %zmm26 {%k2}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm30, %zmm17 {%k2}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm27 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm27 {%k2}
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm0 {%k2}
-; AVX512BW-NEXT:    vmovdqu64 (%rsp), %zmm15 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm15 {%k2}
-; AVX512BW-NEXT:    vmovdqa64 (%r8), %zmm5
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm2 = <0,1,9,u,4,5,6,7>
-; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm2, %zmm11
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k2}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm3 {%k2}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vshufi64x2 {{.*#+}} zmm14 = zmm0[0,1,2,3],zmm29[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm4 {%k2}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm5 {%k2}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vshufi64x2 $228, (%rsp), %zmm0, %zmm0 # 64-byte Folded Reload
+; AVX512BW-NEXT:    # zmm0 = zmm0[0,1,2,3],mem[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, (%rsp) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm6 {%k2}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm30 {%k2}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm15 # 64-byte Folded Reload
+; AVX512BW-NEXT:    # zmm15 = zmm0[0,1,2,3],mem[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm9 {%k2}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm8 {%k2}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm13 # 64-byte Folded Reload
+; AVX512BW-NEXT:    # zmm13 = zmm0[0,1,2,3],mem[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm7 {%k2}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm11 {%k2}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm12 # 64-byte Folded Reload
+; AVX512BW-NEXT:    # zmm12 = zmm0[0,1,2,3],mem[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm26 {%k2}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm25 {%k2}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; AVX512BW-NEXT:    # zmm0 = zmm0[0,1,2,3],mem[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm10 {%k2}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm23 {%k2}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm0 # 64-byte Folded Reload
+; AVX512BW-NEXT:    # zmm0 = zmm0[0,1,2,3],mem[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm27 {%k2}
+; AVX512BW-NEXT:    vmovdqa64 (%r8), %zmm29
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm1 = <0,1,9,u,4,5,6,7>
+; AVX512BW-NEXT:    vpermt2q %zmm29, %zmm1, %zmm2
+; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 64(%r8), %zmm2
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm1, %zmm4
+; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 128(%r8), %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm1, %zmm6
+; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 192(%r8), %zmm4
+; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm1, %zmm9
+; AVX512BW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 256(%r8), %zmm6
+; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm1, %zmm7
+; AVX512BW-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 320(%r8), %zmm7
+; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm1, %zmm26
+; AVX512BW-NEXT:    vmovdqu64 %zmm26, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 384(%r8), %zmm9
+; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm1, %zmm10
+; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm24
+; AVX512BW-NEXT:    vmovdqa64 448(%r8), %zmm10
+; AVX512BW-NEXT:    vpermt2q %zmm10, %zmm1, %zmm27
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm28 {%k2}
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm1 = <0,1,13,u,4,5,6,7>
+; AVX512BW-NEXT:    vpermt2q %zmm29, %zmm1, %zmm3
+; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm1, %zmm5
+; AVX512BW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm1, %zmm30
+; AVX512BW-NEXT:    vmovdqu64 %zmm30, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm1, %zmm8
+; AVX512BW-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm1, %zmm11
 ; AVX512BW-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 64(%r8), %zmm3
-; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm2, %zmm13
-; AVX512BW-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 128(%r8), %zmm9
-; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm2, %zmm14
+; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm1, %zmm25
+; AVX512BW-NEXT:    vmovdqu64 %zmm25, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm1, %zmm23
+; AVX512BW-NEXT:    vpermt2q %zmm10, %zmm1, %zmm28
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm3 # 64-byte Folded Reload
+; AVX512BW-NEXT:    # zmm3 = zmm1[0,1,2,3],mem[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm1 = <14,u,2,3,4,5,15,u>
+; AVX512BW-NEXT:    vpermt2q %zmm29, %zmm1, %zmm14
 ; AVX512BW-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 192(%r8), %zmm11
-; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm2, %zmm16
-; AVX512BW-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 256(%r8), %zmm13
-; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm2, %zmm24
-; AVX512BW-NEXT:    vmovdqu64 %zmm24, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 320(%r8), %zmm14
-; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm2, %zmm26
-; AVX512BW-NEXT:    vmovdqu64 %zmm26, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 384(%r8), %zmm16
-; AVX512BW-NEXT:    vpermt2q %zmm16, %zmm2, %zmm27
-; AVX512BW-NEXT:    vmovdqu64 %zmm27, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 448(%r8), %zmm12
-; AVX512BW-NEXT:    vpermt2q %zmm12, %zmm2, %zmm15
-; AVX512BW-NEXT:    vmovdqu64 %zmm15, (%rsp) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 (%rsp), %zmm14 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm1, %zmm14
+; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm1, %zmm15
+; AVX512BW-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm1, %zmm13
+; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm1, %zmm12
+; AVX512BW-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm1, %zmm12
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm15 {%k2}
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm2 = <0,1,13,u,4,5,6,7>
-; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm2, %zmm23
-; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm2, %zmm22
-; AVX512BW-NEXT:    vmovdqu64 %zmm22, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm2, %zmm20
-; AVX512BW-NEXT:    vmovdqa64 %zmm20, %zmm24
-; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm2, %zmm19
-; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm2, %zmm18
-; AVX512BW-NEXT:    vmovdqa64 %zmm18, %zmm26
-; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm2, %zmm17
+; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm1, %zmm15
+; AVX512BW-NEXT:    vpermt2q %zmm10, %zmm1, %zmm3
+; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa (%rdi), %ymm1
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm1[1],mem[1],ymm1[3],mem[3]
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm3, %zmm3
+; AVX512BW-NEXT:    vmovdqa 64(%rdi), %ymm1
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm1[1],mem[1],ymm1[3],mem[3]
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm5, %zmm5
+; AVX512BW-NEXT:    movb $16, %al
+; AVX512BW-NEXT:    kmovd %eax, %k2
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm31 {%k2}
+; AVX512BW-NEXT:    vmovdqu64 %zmm31, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm1 = <10,u,2,3,4,5,11,u>
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm1, %zmm5
+; AVX512BW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa 128(%rdi), %ymm2
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm2 = ymm2[1],mem[1],ymm2[3],mem[3]
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm5, %zmm31
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm21 {%k2}
+; AVX512BW-NEXT:    vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm1, %zmm31
+; AVX512BW-NEXT:    vmovdqa 192(%rdi), %ymm2
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm2 = ymm2[1],mem[1],ymm2[3],mem[3]
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm0, %zmm30
+; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm16 {%k2}
+; AVX512BW-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm1, %zmm30
+; AVX512BW-NEXT:    vmovdqa 256(%rdi), %ymm2
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm2 = ymm2[1],mem[1],ymm2[3],mem[3]
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm0, %zmm26
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm17 {%k2}
 ; AVX512BW-NEXT:    vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm16, %zmm2, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm12, %zmm2, %zmm15
-; AVX512BW-NEXT:    vmovdqa (%rdi), %ymm2
+; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm1, %zmm26
+; AVX512BW-NEXT:    vmovdqa 320(%rdi), %ymm2
 ; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm2 = ymm2[1],mem[1],ymm2[3],mem[3]
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm0, %zmm10
-; AVX512BW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm7, %zmm7 # 64-byte Folded Reload
-; AVX512BW-NEXT:    # zmm7 = zmm7[0,1,2,3],mem[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqa 64(%rdi), %ymm2
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm0, %zmm25
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm19 {%k2}
+; AVX512BW-NEXT:    vmovdqu64 %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm1, %zmm25
+; AVX512BW-NEXT:    vmovdqa 384(%rdi), %ymm2
 ; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm2 = ymm2[1],mem[1],ymm2[3],mem[3]
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm0, %zmm0
-; AVX512BW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm6, %zmm6 # 64-byte Folded Reload
-; AVX512BW-NEXT:    # zmm6 = zmm6[0,1,2,3],mem[4,5,6,7]
-; AVX512BW-NEXT:    movb $16, %al
-; AVX512BW-NEXT:    kmovd %eax, %k2
-; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm1 {%k2}
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm1 = <10,u,2,3,4,5,11,u>
-; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm1, %zmm0
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm0, %zmm19
+; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm18 {%k2}
+; AVX512BW-NEXT:    vmovdqu64 %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm1, %zmm19
+; AVX512BW-NEXT:    vmovdqa 448(%rdi), %ymm2
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm2 = ymm2[1],mem[1],ymm2[3],mem[3]
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm0, %zmm16
+; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm22 {%k2}
+; AVX512BW-NEXT:    vmovdqu64 %zmm22, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm10, %zmm1, %zmm16
+; AVX512BW-NEXT:    vpermt2q %zmm29, %zmm1, %zmm3
+; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm29, %zmm20 {%k2}
+; AVX512BW-NEXT:    vmovdqu64 %zmm20, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 (%r9), %zmm29
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [0,1,2,9,4,5,6,7]
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm29, %zmm6, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm2 = <14,u,2,3,4,5,15,u>
-; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm2, %zmm6
-; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa 128(%rdi), %ymm3
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm3 = ymm3[1],mem[1],ymm3[3],mem[3]
+; AVX512BW-NEXT:    vmovdqa64 64(%r9), %zmm3
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm3
-; AVX512BW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm25, %zmm6 # 64-byte Folded Reload
-; AVX512BW-NEXT:    # zmm6 = zmm25[0,1,2,3],mem[4,5,6,7]
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm6, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 128(%r9), %zmm4
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm0 {%k2}
+; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm6, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm1, %zmm3
-; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm2, %zmm6
-; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa 192(%rdi), %ymm3
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm3 = ymm3[1],mem[1],ymm3[3],mem[3]
+; AVX512BW-NEXT:    vmovdqa64 192(%r9), %zmm5
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm0
-; AVX512BW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm29, %zmm3 # 64-byte Folded Reload
-; AVX512BW-NEXT:    # zmm3 = zmm29[0,1,2,3],mem[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm4 {%k2}
-; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm1, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm6, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm2, %zmm3
-; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa 256(%rdi), %ymm0
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm0 = ymm0[1],mem[1],ymm0[3],mem[3]
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm3, %zmm3
-; AVX512BW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm31, %zmm4 # 64-byte Folded Reload
-; AVX512BW-NEXT:    # zmm4 = zmm31[0,1,2,3],mem[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 256(%r9), %zmm11
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm0 {%k2}
+; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm6, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm1, %zmm3
-; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm2, %zmm4
-; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa 320(%rdi), %ymm0
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm0 = ymm0[1],mem[1],ymm0[3],mem[3]
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm3, %zmm27
+; AVX512BW-NEXT:    vmovdqa64 320(%r9), %zmm8
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm25 # 64-byte Folded Reload
-; AVX512BW-NEXT:    # zmm25 = zmm0[0,1,2,3],mem[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm8 {%k2}
-; AVX512BW-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm1, %zmm27
-; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm2, %zmm25
-; AVX512BW-NEXT:    vmovdqa 384(%rdi), %ymm0
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm0 = ymm0[1],mem[1],ymm0[3],mem[3]
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm3, %zmm22
+; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm6, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 384(%r9), %zmm7
+; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm6, %zmm24
+; AVX512BW-NEXT:    vmovdqu64 %zmm24, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 448(%r9), %zmm22
+; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm6, %zmm27
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [0,1,2,13,4,5,6,7]
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm20 # 64-byte Folded Reload
-; AVX512BW-NEXT:    # zmm20 = zmm0[0,1,2,3],mem[4,5,6,7]
+; AVX512BW-NEXT:    vpermt2q %zmm29, %zmm6, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm0 {%k2}
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm6, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm16, %zmm1, %zmm22
-; AVX512BW-NEXT:    vpermt2q %zmm16, %zmm2, %zmm20
-; AVX512BW-NEXT:    vmovdqa 448(%rdi), %ymm0
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm0 = ymm0[1],mem[1],ymm0[3],mem[3]
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm3, %zmm18
-; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm1, %zmm10
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm11
-; AVX512BW-NEXT:    vpermt2q %zmm12, %zmm1, %zmm18
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm17 # 64-byte Folded Reload
-; AVX512BW-NEXT:    # zmm17 = zmm0[0,1,2,3],mem[4,5,6,7]
+; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm6, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm0 {%k2}
+; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm6, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm12, %zmm2, %zmm17
-; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm2, %zmm7
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm13
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm28 {%k2}
-; AVX512BW-NEXT:    vmovdqa64 (%r9), %zmm6
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm10 = [0,1,2,9,4,5,6,7]
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm10, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 64(%r9), %zmm7
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm10, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 128(%r9), %zmm2
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm10, %zmm3
-; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 192(%r9), %zmm3
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm10, %zmm4
-; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 256(%r9), %zmm4
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm10, %zmm5
-; AVX512BW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 320(%r9), %zmm5
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm10, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm6, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 384(%r9), %zmm8
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm10, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm6, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 448(%r9), %zmm9
-; AVX512BW-NEXT:    vmovdqu64 (%rsp), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm10, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, (%rsp) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm10 = [0,1,2,13,4,5,6,7]
-; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm10, %zmm23
+; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm6, %zmm23
 ; AVX512BW-NEXT:    vmovdqu64 %zmm23, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm10, %zmm12
-; AVX512BW-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm10, %zmm24
-; AVX512BW-NEXT:    vmovdqu64 %zmm24, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm10, %zmm19
-; AVX512BW-NEXT:    vmovdqu64 %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm10, %zmm26
-; AVX512BW-NEXT:    vmovdqu64 %zmm26, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm6, %zmm28
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [0,14,2,3,4,5,6,15]
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm10, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm29, %zmm6, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm6, %zmm14
+; AVX512BW-NEXT:    vmovdqu64 %zmm14, (%rsp) # 64-byte Spill
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm10, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm6, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm10, %zmm15
+; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm6, %zmm13
+; AVX512BW-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm6, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm6, %zmm12
+; AVX512BW-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm6, %zmm15
 ; AVX512BW-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa (%rdx), %xmm10
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} xmm10 = xmm10[0],mem[0]
-; AVX512BW-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm10, %zmm0, %zmm12 {%k1}
-; AVX512BW-NEXT:    vmovdqa 64(%rdx), %xmm10
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} xmm10 = xmm10[0],mem[0]
-; AVX512BW-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm10, %zmm0, %zmm14 {%k1}
-; AVX512BW-NEXT:    vmovdqa 128(%rdx), %xmm10
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} xmm10 = xmm10[0],mem[0]
-; AVX512BW-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm10, %zmm0, %zmm16 {%k1}
-; AVX512BW-NEXT:    vmovdqa 192(%rdx), %xmm10
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} xmm10 = xmm10[0],mem[0]
-; AVX512BW-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm30 # 64-byte Reload
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm10, %zmm0, %zmm30 {%k1}
-; AVX512BW-NEXT:    vmovdqa 256(%rdx), %xmm10
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} xmm10 = xmm10[0],mem[0]
-; AVX512BW-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm31 # 64-byte Reload
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm10, %zmm0, %zmm31 {%k1}
-; AVX512BW-NEXT:    vmovdqa 320(%rdx), %xmm10
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} xmm10 = xmm10[0],mem[0]
-; AVX512BW-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm10, %zmm0, %zmm1 {%k1}
-; AVX512BW-NEXT:    vmovdqa 384(%rdx), %xmm10
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} xmm10 = xmm10[0],mem[0]
-; AVX512BW-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm10, %zmm0, %zmm0 {%k1}
-; AVX512BW-NEXT:    vmovdqa 448(%rdx), %xmm10
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} xmm10 = xmm10[0],mem[0]
-; AVX512BW-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm10, %zmm0, %zmm21 {%k1}
-; AVX512BW-NEXT:    vinserti32x4 $2, (%r8), %zmm12, %zmm10
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm29 = [0,1,2,3,4,8,6,7]
-; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm29, %zmm10
-; AVX512BW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vinserti32x4 $2, 64(%r8), %zmm14, %zmm10
-; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm29, %zmm10
-; AVX512BW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vinserti32x4 $2, 128(%r8), %zmm16, %zmm10
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm29, %zmm10
-; AVX512BW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vinserti32x4 $2, 192(%r8), %zmm30, %zmm26
-; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm29, %zmm26
-; AVX512BW-NEXT:    vinserti32x4 $2, 256(%r8), %zmm31, %zmm24
-; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm29, %zmm24
-; AVX512BW-NEXT:    vinserti32x4 $2, 320(%r8), %zmm1, %zmm23
-; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm29, %zmm23
-; AVX512BW-NEXT:    vinserti32x4 $2, 384(%r8), %zmm0, %zmm19
-; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm29, %zmm19
-; AVX512BW-NEXT:    vinserti32x4 $2, 448(%r8), %zmm21, %zmm21
-; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm29, %zmm21
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm29 = [0,10,2,3,4,5,6,11]
-; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm29, %zmm11
-; AVX512BW-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm30 = [0,1,2,3,4,12,6,7]
-; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm30, %zmm28
-; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm31 = [0,14,2,3,4,5,6,15]
-; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm31, %zmm13
-; AVX512BW-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm29, %zmm16
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm30, %zmm15
+; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm6, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa (%rdx), %xmm6
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} xmm6 = xmm6[0],mem[0]
+; AVX512BW-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm6, %zmm0, %zmm0 {%k1}
+; AVX512BW-NEXT:    vmovdqa 64(%rdx), %xmm6
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} xmm6 = xmm6[0],mem[0]
+; AVX512BW-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm31, %zmm14
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm29, %zmm13
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm30, %zmm12
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm31, %zmm11
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm6, %zmm0, %zmm14 {%k1}
+; AVX512BW-NEXT:    vmovdqa 128(%rdx), %xmm6
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} xmm6 = xmm6[0],mem[0]
+; AVX512BW-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm6, %zmm0, %zmm15 {%k1}
+; AVX512BW-NEXT:    vmovdqa 192(%rdx), %xmm6
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} xmm6 = xmm6[0],mem[0]
+; AVX512BW-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm29, %zmm10
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm30, %zmm7
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm6, %zmm0, %zmm10 {%k1}
+; AVX512BW-NEXT:    vmovdqa 256(%rdx), %xmm6
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} xmm6 = xmm6[0],mem[0]
+; AVX512BW-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm6, %zmm0, %zmm12 {%k1}
+; AVX512BW-NEXT:    vmovdqa 320(%rdx), %xmm6
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} xmm6 = xmm6[0],mem[0]
+; AVX512BW-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm31, %zmm2
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm6, %zmm0, %zmm2 {%k1}
+; AVX512BW-NEXT:    vmovdqa 384(%rdx), %xmm6
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} xmm6 = xmm6[0],mem[0]
+; AVX512BW-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm6, %zmm0, %zmm9 {%k1}
+; AVX512BW-NEXT:    vmovdqa 448(%rdx), %xmm6
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} xmm6 = xmm6[0],mem[0]
+; AVX512BW-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm29, %zmm1
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm6, %zmm0, %zmm1 {%k1}
+; AVX512BW-NEXT:    vinserti32x4 $2, (%r8), %zmm0, %zmm24
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [0,1,2,3,4,8,6,7]
+; AVX512BW-NEXT:    vpermt2q %zmm29, %zmm0, %zmm24
+; AVX512BW-NEXT:    vinserti32x4 $2, 64(%r8), %zmm14, %zmm23
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm0, %zmm23
+; AVX512BW-NEXT:    vinserti32x4 $2, 128(%r8), %zmm15, %zmm21
+; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm0, %zmm21
+; AVX512BW-NEXT:    vinserti32x4 $2, 192(%r8), %zmm10, %zmm20
+; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm0, %zmm20
+; AVX512BW-NEXT:    vinserti32x4 $2, 256(%r8), %zmm12, %zmm18
+; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm0, %zmm18
+; AVX512BW-NEXT:    vinserti32x4 $2, 320(%r8), %zmm2, %zmm17
+; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm0, %zmm17
+; AVX512BW-NEXT:    vinserti32x4 $2, 384(%r8), %zmm9, %zmm15
+; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm0, %zmm15
+; AVX512BW-NEXT:    vinserti32x4 $2, 448(%r8), %zmm1, %zmm14
+; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm0, %zmm14
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [0,10,2,3,4,5,6,11]
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm29, %zmm0, %zmm13
+; AVX512BW-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,1,2,3,4,12,6,7]
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm29, %zmm1, %zmm12
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm0, %zmm10
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm1, %zmm9
+; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm0, %zmm31
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm1, %zmm6
+; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm0, %zmm30
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm29 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm1, %zmm29
+; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm0, %zmm26
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm1, %zmm4
+; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm0, %zmm25
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm30, %zmm3
+; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm1, %zmm3
+; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm0, %zmm19
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm1, %zmm2
+; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm0, %zmm16
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm31, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm29, %zmm27
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm30, %zmm4
-; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm31, %zmm25
-; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm29, %zmm22
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm30, %zmm5
-; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm31, %zmm20
-; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm29, %zmm18
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm30, %zmm6
-; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm31, %zmm17
+; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm1, %zmm0
 ; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512BW-NEXT:    vmovdqa64 %zmm17, 3008(%rax)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm8, 2944(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm6, 2880(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm18, 2816(%rax)
-; AVX512BW-NEXT:    vmovups (%rsp), %zmm6 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm6, 2752(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm20, 2624(%rax)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm6, 2560(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, 2496(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm22, 2432(%rax)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm5, 2368(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm25, 2240(%rax)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm5, 2176(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm4, 2112(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm27, 2048(%rax)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm4, 1984(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, 1856(%rax)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm1, 3008(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm28, 2944(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, 2880(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm16, 2816(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm27, 2752(%rax)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm0, 2624(%rax)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm0, 2560(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, 2496(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm19, 2432(%rax)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm0, 2368(%rax)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm0, 2240(%rax)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm0, 2176(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, 2112(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm25, 2048(%rax)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm0, 1984(%rax)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm0, 1856(%rax)
 ; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-NEXT:    vmovaps %zmm0, 1792(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm3, 1728(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, 1664(%rax)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm3, 1600(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, 1472(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm4, 1728(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm26, 1664(%rax)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm0, 1600(%rax)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm0, 1472(%rax)
 ; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-NEXT:    vmovaps %zmm0, 1408(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, 1344(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, 1280(%rax)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm2, 1216(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm11, 1088(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm29, 1344(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm30, 1280(%rax)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm0, 1216(%rax)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm0, 1088(%rax)
 ; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-NEXT:    vmovaps %zmm0, 1024(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm12, 960(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm13, 896(%rax)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm1, 832(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm14, 704(%rax)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm1, 640(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm15, 576(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm16, 512(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, 960(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm31, 896(%rax)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm0, 832(%rax)
+; AVX512BW-NEXT:    vmovups (%rsp), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm0, 704(%rax)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm0, 640(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm9, 576(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm10, 512(%rax)
 ; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-NEXT:    vmovaps %zmm0, 448(%rax)
 ; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-NEXT:    vmovaps %zmm0, 320(%rax)
 ; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-NEXT:    vmovaps %zmm0, 256(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm28, 192(%rax)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm0, 128(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm12, 192(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm13, 128(%rax)
 ; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-NEXT:    vmovaps %zmm0, 64(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm21, 2688(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm19, 2304(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm23, 1920(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm24, 1536(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm26, 1152(%rax)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm0, 768(%rax)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm0, 384(%rax)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm0, (%rax)
-; AVX512BW-NEXT:    addq $3400, %rsp # imm = 0xD48
+; AVX512BW-NEXT:    vmovdqa64 %zmm14, 2688(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm15, 2304(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm17, 1920(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm18, 1536(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm20, 1152(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm21, 768(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm23, 384(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm24, (%rax)
+; AVX512BW-NEXT:    addq $3720, %rsp # imm = 0xE88
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
   %in.vec0 = load <64 x i64>, ptr %in.vecptr0, align 64
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i64-stride-7.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i64-stride-7.ll
index 0de7beea9398af3e5ddc7d0fd5cb15381e1464f9..6fb5e43a2df68a9807d6c8160ffcf144b0ce5104 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i64-stride-7.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i64-stride-7.ll
@@ -750,123 +750,122 @@ define void @store_i64_stride7_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512F-ONLY-SLOW:       # %bb.0:
 ; AVX512F-ONLY-SLOW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512F-ONLY-SLOW-NEXT:    movq {{[0-9]+}}(%rsp), %r10
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 (%rdi), %zmm0
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 (%rsi), %zmm1
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 (%rdx), %zmm3
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 (%rcx), %zmm4
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 (%r8), %zmm6
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 (%r9), %zmm7
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 (%r10), %zmm2
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm5 = [15,7,15,7,15,7,15,7]
-; AVX512F-ONLY-SLOW-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm8 = [0,8,0,8,0,8,0,8]
-; AVX512F-ONLY-SLOW-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm7, %zmm6, %zmm8
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm9 = [5,0,14,6,5,0,14,6]
-; AVX512F-ONLY-SLOW-NEXT:    # zmm9 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm6, %zmm7, %zmm9
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm11 = [9,1,9,1,9,1,9,1]
-; AVX512F-ONLY-SLOW-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm7, %zmm6, %zmm11
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm10 = [4,12,0,5,4,12,0,5]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 (%rdi), %zmm5
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 (%rsi), %zmm6
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 (%rdx), %zmm8
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 (%rcx), %zmm7
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 (%r8), %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 (%r9), %zmm2
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 (%r10), %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm3 = [15,7,15,7,15,7,15,7]
+; AVX512F-ONLY-SLOW-NEXT:    # zmm3 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm9
+; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [2,10,0,3,2,10,0,3]
+; AVX512F-ONLY-SLOW-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm7, %zmm8, %zmm4
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm11
+; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm10 = [3,0,12,4,3,0,12,4]
 ; AVX512F-ONLY-SLOW-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm7, %zmm6, %zmm10
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm7, %zmm5, %zmm6
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm4, %zmm3, %zmm5
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti128 {{.*#+}} ymm7 = [15,7,15,7]
-; AVX512F-ONLY-SLOW-NEXT:    # ymm7 = mem[0,1,0,1]
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm1, %zmm0, %zmm7
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm8, %zmm7, %zmm10
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm13
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm7, %zmm3, %zmm13
+; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm8 = [0,8,0,1,0,8,0,1]
+; AVX512F-ONLY-SLOW-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm6, %zmm5, %zmm8
+; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm14 = [6,14,6,14,6,14,6,14]
+; AVX512F-ONLY-SLOW-NEXT:    # zmm14 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm7, %zmm14, %zmm9
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm6, %zmm5, %zmm14
+; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm15 = [11,3,11,3,11,3,11,3]
+; AVX512F-ONLY-SLOW-NEXT:    # zmm15 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm6, %zmm5, %zmm15
+; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm12 = [13,5,13,5,13,5,13,5]
+; AVX512F-ONLY-SLOW-NEXT:    # zmm12 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm7, %zmm12, %zmm11
+; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm7 = [1,0,10,2,1,0,10,2]
+; AVX512F-ONLY-SLOW-NEXT:    # zmm7 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm5, %zmm6, %zmm7
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm6, %zmm5, %zmm12
+; AVX512F-ONLY-SLOW-NEXT:    movb $48, %sil
+; AVX512F-ONLY-SLOW-NEXT:    kmovw %esi, %k1
+; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm10 {%k1} = zmm5[0],zmm6[0],zmm5[2],zmm6[2],zmm5[4],zmm6[4],zmm5[6],zmm6[6]
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm6, %zmm3, %zmm5
 ; AVX512F-ONLY-SLOW-NEXT:    movb $24, %sil
 ; AVX512F-ONLY-SLOW-NEXT:    kmovw %esi, %k1
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm7 {%k1}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm13, %zmm5 {%k1}
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm2, %zmm0, %zmm3
 ; AVX512F-ONLY-SLOW-NEXT:    movb $96, %sil
 ; AVX512F-ONLY-SLOW-NEXT:    kmovw %esi, %k1
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm7 {%k1}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [14,1,2,3,4,5,6,15]
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm2, %zmm7, %zmm5
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm6 = [0,8,0,1,0,8,0,1]
-; AVX512F-ONLY-SLOW-NEXT:    # zmm6 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm1, %zmm0, %zmm6
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa (%rdx), %xmm7
-; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm7 = xmm7[0],mem[0]
-; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm7, %ymm0, %ymm7
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm5 {%k1}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [14,1,2,3,4,5,6,15]
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm1, %zmm5, %zmm3
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa (%rdx), %xmm5
+; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm5 = xmm5[0],mem[0]
+; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
 ; AVX512F-ONLY-SLOW-NEXT:    movb $12, %sil
 ; AVX512F-ONLY-SLOW-NEXT:    kmovw %esi, %k2
-; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm7, %zmm0, %zmm6 {%k2}
-; AVX512F-ONLY-SLOW-NEXT:    vinserti32x4 $3, (%r10), %zmm8, %zmm7
+; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm5, %zmm0, %zmm8 {%k2}
+; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm5 = [0,8,0,8,0,8,0,8]
+; AVX512F-ONLY-SLOW-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm2, %zmm0, %zmm5
+; AVX512F-ONLY-SLOW-NEXT:    vinserti32x4 $3, (%r10), %zmm5, %zmm5
 ; AVX512F-ONLY-SLOW-NEXT:    movb $112, %sil
 ; AVX512F-ONLY-SLOW-NEXT:    kmovw %esi, %k2
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm6 {%k2}
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm7 = [0,13,6,7,0,13,6,7]
-; AVX512F-ONLY-SLOW-NEXT:    # zmm7 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm2, %zmm9, %zmm7
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm8 = [6,14,6,14,6,14,6,14]
-; AVX512F-ONLY-SLOW-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm4, %zmm3, %zmm8
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti128 {{.*#+}} ymm9 = [6,14,6,14]
-; AVX512F-ONLY-SLOW-NEXT:    # ymm9 = mem[0,1,0,1]
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm1, %zmm0, %zmm9
-; AVX512F-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm8 = zmm9[0,1,2,3],zmm8[4,5,6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm8 {%k2}
+; AVX512F-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm5 = zmm14[0,1,2,3],zmm9[4,5,6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm6 = [5,0,14,6,5,0,14,6]
+; AVX512F-ONLY-SLOW-NEXT:    # zmm6 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm0, %zmm2, %zmm6
+; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm9 = [0,13,6,7,0,13,6,7]
+; AVX512F-ONLY-SLOW-NEXT:    # zmm9 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm1, %zmm6, %zmm9
 ; AVX512F-ONLY-SLOW-NEXT:    movb $-61, %sil
 ; AVX512F-ONLY-SLOW-NEXT:    kmovw %esi, %k2
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm8 {%k2}
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm7 = [11,3,11,3,11,3,11,3]
-; AVX512F-ONLY-SLOW-NEXT:    # zmm7 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm1, %zmm0, %zmm7
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm9 = [2,10,0,3,2,10,0,3]
-; AVX512F-ONLY-SLOW-NEXT:    # zmm9 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm4, %zmm3, %zmm9
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm9 {%k1}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa (%r9), %ymm7
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa (%r8), %ymm12
-; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm13 = ymm12[0],ymm7[0],ymm12[2],ymm7[2]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm9, %zmm5 {%k2}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm15, %zmm4 {%k1}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa (%r9), %ymm6
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa (%r8), %ymm9
+; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm13 = ymm9[0],ymm6[0],ymm9[2],ymm6[2]
 ; AVX512F-ONLY-SLOW-NEXT:    movb $28, %sil
 ; AVX512F-ONLY-SLOW-NEXT:    kmovw %esi, %k2
-; AVX512F-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm9 {%k2} = zmm13[2,3,2,3],zmm2[2,3,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm13 = [1,0,10,2,1,0,10,2]
-; AVX512F-ONLY-SLOW-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm0, %zmm1, %zmm13
-; AVX512F-ONLY-SLOW-NEXT:    vpbroadcastq 8(%rcx), %ymm14
-; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm14 = mem[0,1,2,3],ymm14[4,5,6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm4 {%k2} = zmm13[2,3,2,3],zmm1[2,3,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vpbroadcastq 8(%rcx), %ymm13
+; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm13 = mem[0,1,2,3],ymm13[4,5,6,7]
 ; AVX512F-ONLY-SLOW-NEXT:    movb $6, %cl
 ; AVX512F-ONLY-SLOW-NEXT:    kmovw %ecx, %k2
-; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm14, %zmm0, %zmm13 {%k2}
+; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm13, %zmm0, %zmm7 {%k2}
+; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm13 = [9,1,9,1,9,1,9,1]
+; AVX512F-ONLY-SLOW-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm2, %zmm0, %zmm13
 ; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm14 = [4,9,0,3,4,9,0,3]
 ; AVX512F-ONLY-SLOW-NEXT:    # zmm14 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm2, %zmm11, %zmm14
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm1, %zmm13, %zmm14
 ; AVX512F-ONLY-SLOW-NEXT:    movb $56, %cl
 ; AVX512F-ONLY-SLOW-NEXT:    kmovw %ecx, %k2
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm13 {%k2}
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm11 = [13,5,13,5,13,5,13,5]
-; AVX512F-ONLY-SLOW-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm14 = [3,0,12,4,3,0,12,4]
-; AVX512F-ONLY-SLOW-NEXT:    # zmm14 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm3, %zmm4, %zmm14
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm4, %zmm11, %zmm3
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm1, %zmm0, %zmm11
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm11 {%k1}
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm3 = [0,1,12,7,0,1,12,7]
-; AVX512F-ONLY-SLOW-NEXT:    # zmm3 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm2, %zmm10, %zmm3
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm7 {%k2}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm12 {%k1}
+; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm11 = [4,12,0,5,4,12,0,5]
+; AVX512F-ONLY-SLOW-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm2, %zmm0, %zmm11
+; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [0,1,12,7,0,1,12,7]
+; AVX512F-ONLY-SLOW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm1, %zmm11, %zmm0
 ; AVX512F-ONLY-SLOW-NEXT:    movb $120, %cl
 ; AVX512F-ONLY-SLOW-NEXT:    kmovw %ecx, %k1
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm3 {%k1}
-; AVX512F-ONLY-SLOW-NEXT:    movb $48, %cl
-; AVX512F-ONLY-SLOW-NEXT:    kmovw %ecx, %k1
-; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm14 {%k1} = zmm0[0],zmm1[0],zmm0[2],zmm1[2],zmm0[4],zmm1[4],zmm0[6],zmm1[6]
-; AVX512F-ONLY-SLOW-NEXT:    vpunpckhqdq {{.*#+}} ymm0 = ymm12[1],ymm7[1],ymm12[3],ymm7[3]
-; AVX512F-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,3,3]
-; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],mem[6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm12, %zmm0 {%k1}
+; AVX512F-ONLY-SLOW-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm9[1],ymm6[1],ymm9[3],ymm6[3]
+; AVX512F-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,3,3]
+; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],mem[6,7]
 ; AVX512F-ONLY-SLOW-NEXT:    movb $14, %cl
 ; AVX512F-ONLY-SLOW-NEXT:    kmovw %ecx, %k1
-; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm14 {%k1}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, 256(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm13, 64(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm9, 128(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm8, 320(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, (%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm5, 384(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm14, 192(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm10 {%k1}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, 256(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm7, 64(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm4, 128(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm5, 320(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm8, (%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, 384(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm10, 192(%rax)
 ; AVX512F-ONLY-SLOW-NEXT:    vzeroupper
 ; AVX512F-ONLY-SLOW-NEXT:    retq
 ;
@@ -875,122 +874,121 @@ define void @store_i64_stride7_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512F-ONLY-FAST-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512F-ONLY-FAST-NEXT:    movq {{[0-9]+}}(%rsp), %r10
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 (%rdi), %zmm0
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 (%rsi), %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 (%rdx), %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 (%rcx), %zmm4
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 (%r8), %zmm6
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 (%r9), %zmm7
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 (%r10), %zmm3
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm5 = [15,7,15,7,15,7,15,7]
-; AVX512F-ONLY-FAST-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm8 = [0,8,0,8,0,8,0,8]
-; AVX512F-ONLY-FAST-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm7, %zmm6, %zmm8
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm9 = [5,0,14,6,5,0,14,6]
-; AVX512F-ONLY-FAST-NEXT:    # zmm9 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm6, %zmm7, %zmm9
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm10 = [9,1,9,1,9,1,9,1]
-; AVX512F-ONLY-FAST-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm7, %zmm6, %zmm10
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm11 = [4,12,0,5,4,12,0,5]
-; AVX512F-ONLY-FAST-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm7, %zmm6, %zmm11
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm7, %zmm5, %zmm6
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm4, %zmm2, %zmm5
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti128 {{.*#+}} ymm7 = [15,7,15,7]
-; AVX512F-ONLY-FAST-NEXT:    # ymm7 = mem[0,1,0,1]
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm1, %zmm0, %zmm7
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 (%rsi), %zmm5
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 (%rdx), %zmm4
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 (%rcx), %zmm7
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 (%r8), %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 (%r9), %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 (%r10), %zmm1
+; AVX512F-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm6 = [15,7,15,7,15,7,15,7]
+; AVX512F-ONLY-FAST-NEXT:    # zmm6 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm4, %zmm8
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm7, %zmm6, %zmm8
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm9
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm5, %zmm6, %zmm9
 ; AVX512F-ONLY-FAST-NEXT:    movb $24, %sil
 ; AVX512F-ONLY-FAST-NEXT:    kmovw %esi, %k1
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm5, %zmm7 {%k1}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm8, %zmm9 {%k1}
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm3, %zmm2, %zmm6
 ; AVX512F-ONLY-FAST-NEXT:    movb $96, %sil
 ; AVX512F-ONLY-FAST-NEXT:    kmovw %esi, %k1
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm6, %zmm7 {%k1}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [14,1,2,3,4,5,6,15]
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm3, %zmm7, %zmm5
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm6 = [0,8,0,1,0,8,0,1]
-; AVX512F-ONLY-FAST-NEXT:    # zmm6 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm1, %zmm0, %zmm6
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa (%rdx), %xmm7
-; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm7 = xmm7[0],mem[0]
-; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm7, %ymm0, %ymm7
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm6, %zmm9 {%k1}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [14,1,2,3,4,5,6,15]
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm1, %zmm9, %zmm6
+; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm8 = [0,8,0,1,0,8,0,1]
+; AVX512F-ONLY-FAST-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm5, %zmm0, %zmm8
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa (%rdx), %xmm9
+; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm9 = xmm9[0],mem[0]
+; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm9, %ymm0, %ymm9
 ; AVX512F-ONLY-FAST-NEXT:    movb $12, %sil
 ; AVX512F-ONLY-FAST-NEXT:    kmovw %esi, %k2
-; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm7, %zmm0, %zmm6 {%k2}
-; AVX512F-ONLY-FAST-NEXT:    vinserti32x4 $3, (%r10), %zmm8, %zmm7
+; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm9, %zmm0, %zmm8 {%k2}
+; AVX512F-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm9 = [0,8,0,8,0,8,0,8]
+; AVX512F-ONLY-FAST-NEXT:    # zmm9 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm3, %zmm2, %zmm9
+; AVX512F-ONLY-FAST-NEXT:    vinserti32x4 $3, (%r10), %zmm9, %zmm9
 ; AVX512F-ONLY-FAST-NEXT:    movb $112, %sil
 ; AVX512F-ONLY-FAST-NEXT:    kmovw %esi, %k2
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm7, %zmm6 {%k2}
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm7 = [0,13,6,7,0,13,6,7]
-; AVX512F-ONLY-FAST-NEXT:    # zmm7 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm3, %zmm9, %zmm7
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm8 = [6,14,6,14,6,14,6,14]
-; AVX512F-ONLY-FAST-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm4, %zmm2, %zmm8
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti128 {{.*#+}} ymm9 = [6,14,6,14]
-; AVX512F-ONLY-FAST-NEXT:    # ymm9 = mem[0,1,0,1]
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm1, %zmm0, %zmm9
-; AVX512F-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm8 = zmm9[0,1,2,3],zmm8[4,5,6,7]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm9, %zmm8 {%k2}
+; AVX512F-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm9 = [6,14,6,14,6,14,6,14]
+; AVX512F-ONLY-FAST-NEXT:    # zmm9 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm4, %zmm10
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm7, %zmm9, %zmm10
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm5, %zmm0, %zmm9
+; AVX512F-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm9 = zmm9[0,1,2,3],zmm10[4,5,6,7]
+; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm10 = [5,0,14,6,5,0,14,6]
+; AVX512F-ONLY-FAST-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm2, %zmm3, %zmm10
+; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm11 = [0,13,6,7,0,13,6,7]
+; AVX512F-ONLY-FAST-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm1, %zmm10, %zmm11
 ; AVX512F-ONLY-FAST-NEXT:    movb $-61, %sil
 ; AVX512F-ONLY-FAST-NEXT:    kmovw %esi, %k2
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm7, %zmm8 {%k2}
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm7 = [3,0,12,4,3,0,12,4]
-; AVX512F-ONLY-FAST-NEXT:    # zmm7 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm2, %zmm4, %zmm7
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm11, %zmm9 {%k2}
+; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm10 = [3,0,12,4,3,0,12,4]
+; AVX512F-ONLY-FAST-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm4, %zmm7, %zmm10
 ; AVX512F-ONLY-FAST-NEXT:    movb $48, %sil
 ; AVX512F-ONLY-FAST-NEXT:    kmovw %esi, %k2
-; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm7 {%k2} = zmm0[0],zmm1[0],zmm0[2],zmm1[2],zmm0[4],zmm1[4],zmm0[6],zmm1[6]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa (%r9), %ymm9
+; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm10 {%k2} = zmm0[0],zmm5[0],zmm0[2],zmm5[2],zmm0[4],zmm5[4],zmm0[6],zmm5[6]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa (%r9), %ymm11
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqa (%r8), %ymm12
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqa {{.*#+}} ymm13 = <1,3,7,u>
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %ymm9, %ymm12, %ymm13
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %ymm11, %ymm12, %ymm13
 ; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm13 = ymm13[0,1,2,3,4,5],mem[6,7]
 ; AVX512F-ONLY-FAST-NEXT:    movb $14, %sil
 ; AVX512F-ONLY-FAST-NEXT:    kmovw %esi, %k2
-; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm13, %zmm0, %zmm7 {%k2}
+; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm13, %zmm0, %zmm10 {%k2}
 ; AVX512F-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm13 = [11,3,11,3,11,3,11,3]
 ; AVX512F-ONLY-FAST-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm1, %zmm0, %zmm13
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm5, %zmm0, %zmm13
 ; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm14 = [2,10,0,3,2,10,0,3]
 ; AVX512F-ONLY-FAST-NEXT:    # zmm14 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm4, %zmm2, %zmm14
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm7, %zmm4, %zmm14
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm13, %zmm14 {%k1}
-; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm9 = ymm12[0],ymm9[0],ymm12[2],ymm9[2]
+; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm11 = ymm12[0],ymm11[0],ymm12[2],ymm11[2]
 ; AVX512F-ONLY-FAST-NEXT:    movb $28, %sil
 ; AVX512F-ONLY-FAST-NEXT:    kmovw %esi, %k2
-; AVX512F-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm14 {%k2} = zmm9[2,3,2,3],zmm3[2,3,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm9 = [1,0,10,2,1,0,10,2]
-; AVX512F-ONLY-FAST-NEXT:    # zmm9 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm0, %zmm1, %zmm9
+; AVX512F-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm14 {%k2} = zmm11[2,3,2,3],zmm1[2,3,2,3]
+; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm11 = [1,0,10,2,1,0,10,2]
+; AVX512F-ONLY-FAST-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm0, %zmm5, %zmm11
 ; AVX512F-ONLY-FAST-NEXT:    vpbroadcastq 8(%rcx), %ymm12
 ; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm12 = mem[0,1,2,3],ymm12[4,5,6,7]
 ; AVX512F-ONLY-FAST-NEXT:    movb $6, %cl
 ; AVX512F-ONLY-FAST-NEXT:    kmovw %ecx, %k2
-; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm12, %zmm0, %zmm9 {%k2}
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm12 = [4,9,0,3,4,9,0,3]
-; AVX512F-ONLY-FAST-NEXT:    # zmm12 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm3, %zmm10, %zmm12
+; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm12, %zmm0, %zmm11 {%k2}
+; AVX512F-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm12 = [9,1,9,1,9,1,9,1]
+; AVX512F-ONLY-FAST-NEXT:    # zmm12 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm3, %zmm2, %zmm12
+; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm13 = [4,9,0,3,4,9,0,3]
+; AVX512F-ONLY-FAST-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm1, %zmm12, %zmm13
 ; AVX512F-ONLY-FAST-NEXT:    movb $56, %cl
 ; AVX512F-ONLY-FAST-NEXT:    kmovw %ecx, %k2
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm12, %zmm9 {%k2}
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm10 = [13,5,13,5,13,5,13,5]
-; AVX512F-ONLY-FAST-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm4, %zmm10, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm1, %zmm10, %zmm0
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm0 {%k1}
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm1 = [0,1,12,7,0,1,12,7]
-; AVX512F-ONLY-FAST-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm3, %zmm11, %zmm1
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm13, %zmm11 {%k2}
+; AVX512F-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm12 = [13,5,13,5,13,5,13,5]
+; AVX512F-ONLY-FAST-NEXT:    # zmm12 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm7, %zmm12, %zmm4
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm5, %zmm12, %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm4, %zmm0 {%k1}
+; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [4,12,0,5,4,12,0,5]
+; AVX512F-ONLY-FAST-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm3, %zmm2, %zmm4
+; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [0,1,12,7,0,1,12,7]
+; AVX512F-ONLY-FAST-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm1, %zmm4, %zmm2
 ; AVX512F-ONLY-FAST-NEXT:    movb $120, %cl
 ; AVX512F-ONLY-FAST-NEXT:    kmovw %ecx, %k1
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, 256(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm9, 64(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k1}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, 256(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm11, 64(%rax)
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm14, 128(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm7, 192(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm8, 320(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm6, (%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm5, 384(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm10, 192(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm9, 320(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm8, (%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm6, 384(%rax)
 ; AVX512F-ONLY-FAST-NEXT:    vzeroupper
 ; AVX512F-ONLY-FAST-NEXT:    retq
 ;
@@ -998,122 +996,120 @@ define void @store_i64_stride7_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-SLOW:       # %bb.0:
 ; AVX512DQ-SLOW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512DQ-SLOW-NEXT:    movq {{[0-9]+}}(%rsp), %r10
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 (%rdi), %zmm0
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 (%rsi), %zmm1
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 (%rdx), %zmm4
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 (%rcx), %zmm5
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 (%r8), %zmm9
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 (%r9), %zmm10
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 (%r10), %zmm3
-; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [5,0,14,6,5,0,14,6]
-; AVX512DQ-SLOW-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm9, %zmm10, %zmm2
-; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm6 = [0,13,6,7,0,13,6,7]
-; AVX512DQ-SLOW-NEXT:    # zmm6 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm3, %zmm2, %zmm6
-; AVX512DQ-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm2 = [6,14,6,14,6,14,6,14]
-; AVX512DQ-SLOW-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm5, %zmm4, %zmm2
-; AVX512DQ-SLOW-NEXT:    vbroadcasti128 {{.*#+}} ymm7 = [6,14,6,14]
-; AVX512DQ-SLOW-NEXT:    # ymm7 = mem[0,1,0,1]
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm1, %zmm0, %zmm7
-; AVX512DQ-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm7[0,1,2,3],zmm2[4,5,6,7]
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 (%rdi), %zmm4
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 (%rsi), %zmm5
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 (%rdx), %zmm7
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 (%rcx), %zmm6
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 (%r8), %zmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 (%r9), %zmm2
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 (%r10), %zmm1
+; AVX512DQ-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm8 = [6,14,6,14,6,14,6,14]
+; AVX512DQ-SLOW-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm3 = [2,10,0,3,2,10,0,3]
+; AVX512DQ-SLOW-NEXT:    # zmm3 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm6, %zmm7, %zmm3
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm9
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm11
+; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm10 = [3,0,12,4,3,0,12,4]
+; AVX512DQ-SLOW-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm7, %zmm6, %zmm10
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm6, %zmm8, %zmm7
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm5, %zmm4, %zmm8
+; AVX512DQ-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm7 = zmm8[0,1,2,3],zmm7[4,5,6,7]
+; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm8 = [5,0,14,6,5,0,14,6]
+; AVX512DQ-SLOW-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm0, %zmm2, %zmm8
+; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm12 = [0,13,6,7,0,13,6,7]
+; AVX512DQ-SLOW-NEXT:    # zmm12 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm1, %zmm8, %zmm12
 ; AVX512DQ-SLOW-NEXT:    movb $-61, %sil
 ; AVX512DQ-SLOW-NEXT:    kmovw %esi, %k1
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm2 {%k1}
-; AVX512DQ-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm7 = [11,3,11,3,11,3,11,3]
-; AVX512DQ-SLOW-NEXT:    # zmm7 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm1, %zmm0, %zmm7
-; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm6 = [2,10,0,3,2,10,0,3]
-; AVX512DQ-SLOW-NEXT:    # zmm6 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm5, %zmm4, %zmm6
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm12, %zmm7 {%k1}
+; AVX512DQ-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm8 = [11,3,11,3,11,3,11,3]
+; AVX512DQ-SLOW-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm5, %zmm4, %zmm8
 ; AVX512DQ-SLOW-NEXT:    movb $96, %sil
 ; AVX512DQ-SLOW-NEXT:    kmovw %esi, %k1
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm6 {%k1}
-; AVX512DQ-SLOW-NEXT:    vmovdqa (%r9), %ymm7
-; AVX512DQ-SLOW-NEXT:    vmovdqa (%r8), %ymm8
-; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm11 = ymm8[0],ymm7[0],ymm8[2],ymm7[2]
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm3 {%k1}
+; AVX512DQ-SLOW-NEXT:    vmovdqa (%r9), %ymm8
+; AVX512DQ-SLOW-NEXT:    vmovdqa (%r8), %ymm12
+; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm13 = ymm12[0],ymm8[0],ymm12[2],ymm8[2]
 ; AVX512DQ-SLOW-NEXT:    movb $28, %sil
 ; AVX512DQ-SLOW-NEXT:    kmovw %esi, %k2
-; AVX512DQ-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm6 {%k2} = zmm11[2,3,2,3],zmm3[2,3,2,3]
-; AVX512DQ-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm11 = [15,7,15,7,15,7,15,7]
-; AVX512DQ-SLOW-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQ-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm12 = [9,1,9,1,9,1,9,1]
-; AVX512DQ-SLOW-NEXT:    # zmm12 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm10, %zmm9, %zmm12
-; AVX512DQ-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm13 = [0,8,0,8,0,8,0,8]
+; AVX512DQ-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm3 {%k2} = zmm13[2,3,2,3],zmm1[2,3,2,3]
+; AVX512DQ-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm13 = [15,7,15,7,15,7,15,7]
 ; AVX512DQ-SLOW-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm10, %zmm9, %zmm13
-; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm14 = [4,12,0,5,4,12,0,5]
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm6, %zmm13, %zmm9
+; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm14 = [1,0,10,2,1,0,10,2]
 ; AVX512DQ-SLOW-NEXT:    # zmm14 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm10, %zmm9, %zmm14
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm10, %zmm11, %zmm9
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm5, %zmm4, %zmm11
-; AVX512DQ-SLOW-NEXT:    vbroadcasti128 {{.*#+}} ymm10 = [15,7,15,7]
-; AVX512DQ-SLOW-NEXT:    # ymm10 = mem[0,1,0,1]
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm1, %zmm0, %zmm10
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm4, %zmm5, %zmm14
+; AVX512DQ-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm15 = [13,5,13,5,13,5,13,5]
+; AVX512DQ-SLOW-NEXT:    # zmm15 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm6, %zmm15, %zmm11
+; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm6 = [0,8,0,1,0,8,0,1]
+; AVX512DQ-SLOW-NEXT:    # zmm6 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm5, %zmm4, %zmm6
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm5, %zmm4, %zmm15
+; AVX512DQ-SLOW-NEXT:    movb $48, %sil
+; AVX512DQ-SLOW-NEXT:    kmovw %esi, %k2
+; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm10 {%k2} = zmm4[0],zmm5[0],zmm4[2],zmm5[2],zmm4[4],zmm5[4],zmm4[6],zmm5[6]
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm5, %zmm13, %zmm4
 ; AVX512DQ-SLOW-NEXT:    movb $24, %sil
 ; AVX512DQ-SLOW-NEXT:    kmovw %esi, %k2
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm10 {%k2}
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm9, %zmm10 {%k1}
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm9 = [14,1,2,3,4,5,6,15]
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm3, %zmm10, %zmm9
-; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm10 = [1,0,10,2,1,0,10,2]
-; AVX512DQ-SLOW-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm0, %zmm1, %zmm10
-; AVX512DQ-SLOW-NEXT:    vpbroadcastq 8(%rcx), %ymm11
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm11 = mem[0,1,2,3],ymm11[4,5,6,7]
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm9, %zmm4 {%k2}
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm2, %zmm0, %zmm13
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm13, %zmm4 {%k1}
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [14,1,2,3,4,5,6,15]
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm1, %zmm4, %zmm5
+; AVX512DQ-SLOW-NEXT:    vpbroadcastq 8(%rcx), %ymm4
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm4 = mem[0,1,2,3],ymm4[4,5,6,7]
 ; AVX512DQ-SLOW-NEXT:    movb $6, %sil
 ; AVX512DQ-SLOW-NEXT:    kmovw %esi, %k2
-; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm11, %zmm0, %zmm10 {%k2}
-; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm11 = [4,9,0,3,4,9,0,3]
-; AVX512DQ-SLOW-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm3, %zmm12, %zmm11
+; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm4, %zmm0, %zmm14 {%k2}
+; AVX512DQ-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm4 = [9,1,9,1,9,1,9,1]
+; AVX512DQ-SLOW-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm2, %zmm0, %zmm4
+; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm9 = [4,9,0,3,4,9,0,3]
+; AVX512DQ-SLOW-NEXT:    # zmm9 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm1, %zmm4, %zmm9
 ; AVX512DQ-SLOW-NEXT:    movb $56, %sil
 ; AVX512DQ-SLOW-NEXT:    kmovw %esi, %k2
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm10 {%k2}
-; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm11 = [0,8,0,1,0,8,0,1]
-; AVX512DQ-SLOW-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm1, %zmm0, %zmm11
-; AVX512DQ-SLOW-NEXT:    vmovdqa (%rdx), %xmm12
-; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm12 = xmm12[0],mem[0]
-; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm12, %ymm0, %ymm12
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm9, %zmm14 {%k2}
+; AVX512DQ-SLOW-NEXT:    vmovdqa (%rdx), %xmm4
+; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm4 = xmm4[0],mem[0]
+; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
 ; AVX512DQ-SLOW-NEXT:    movb $12, %cl
 ; AVX512DQ-SLOW-NEXT:    kmovw %ecx, %k2
-; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm12, %zmm0, %zmm11 {%k2}
+; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm4, %zmm0, %zmm6 {%k2}
+; AVX512DQ-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm4 = [0,8,0,8,0,8,0,8]
+; AVX512DQ-SLOW-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm2, %zmm0, %zmm4
 ; AVX512DQ-SLOW-NEXT:    movb $112, %cl
 ; AVX512DQ-SLOW-NEXT:    kmovw %ecx, %k2
-; AVX512DQ-SLOW-NEXT:    vinserti64x2 $3, (%r10), %zmm13, %zmm11 {%k2}
-; AVX512DQ-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm12 = [13,5,13,5,13,5,13,5]
-; AVX512DQ-SLOW-NEXT:    # zmm12 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm13 = [3,0,12,4,3,0,12,4]
-; AVX512DQ-SLOW-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm4, %zmm5, %zmm13
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm5, %zmm12, %zmm4
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm1, %zmm0, %zmm12
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm12 {%k1}
-; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [0,1,12,7,0,1,12,7]
+; AVX512DQ-SLOW-NEXT:    vinserti64x2 $3, (%r10), %zmm4, %zmm6 {%k2}
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm15 {%k1}
+; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [4,12,0,5,4,12,0,5]
 ; AVX512DQ-SLOW-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm3, %zmm14, %zmm4
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm2, %zmm0, %zmm4
+; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [0,1,12,7,0,1,12,7]
+; AVX512DQ-SLOW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm1, %zmm4, %zmm0
 ; AVX512DQ-SLOW-NEXT:    movb $120, %cl
 ; AVX512DQ-SLOW-NEXT:    kmovw %ecx, %k1
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm12, %zmm4 {%k1}
-; AVX512DQ-SLOW-NEXT:    movb $48, %cl
-; AVX512DQ-SLOW-NEXT:    kmovw %ecx, %k1
-; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm13 {%k1} = zmm0[0],zmm1[0],zmm0[2],zmm1[2],zmm0[4],zmm1[4],zmm0[6],zmm1[6]
-; AVX512DQ-SLOW-NEXT:    vpunpckhqdq {{.*#+}} ymm0 = ymm8[1],ymm7[1],ymm8[3],ymm7[3]
-; AVX512DQ-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,3,3]
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],mem[6,7]
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm15, %zmm0 {%k1}
+; AVX512DQ-SLOW-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm12[1],ymm8[1],ymm12[3],ymm8[3]
+; AVX512DQ-SLOW-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,3,3]
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],mem[6,7]
 ; AVX512DQ-SLOW-NEXT:    movb $14, %cl
 ; AVX512DQ-SLOW-NEXT:    kmovw %ecx, %k1
-; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm13 {%k1}
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm4, 256(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm11, (%rax)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm10, 64(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm9, 384(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm6, 128(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm13, 192(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm2, 320(%rax)
+; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm10 {%k1}
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, 256(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm6, (%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm14, 64(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm5, 384(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm3, 128(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm10, 192(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm7, 320(%rax)
 ; AVX512DQ-SLOW-NEXT:    vzeroupper
 ; AVX512DQ-SLOW-NEXT:    retq
 ;
@@ -1122,121 +1118,120 @@ define void @store_i64_stride7_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQ-FAST-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512DQ-FAST-NEXT:    movq {{[0-9]+}}(%rsp), %r10
 ; AVX512DQ-FAST-NEXT:    vmovdqa64 (%rdi), %zmm0
-; AVX512DQ-FAST-NEXT:    vmovdqa64 (%rsi), %zmm1
-; AVX512DQ-FAST-NEXT:    vmovdqa64 (%rdx), %zmm2
-; AVX512DQ-FAST-NEXT:    vmovdqa64 (%rcx), %zmm4
-; AVX512DQ-FAST-NEXT:    vmovdqa64 (%r8), %zmm8
-; AVX512DQ-FAST-NEXT:    vmovdqa64 (%r9), %zmm9
-; AVX512DQ-FAST-NEXT:    vmovdqa64 (%r10), %zmm3
-; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm5 = [5,0,14,6,5,0,14,6]
-; AVX512DQ-FAST-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm8, %zmm9, %zmm5
-; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm6 = [0,13,6,7,0,13,6,7]
-; AVX512DQ-FAST-NEXT:    # zmm6 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm3, %zmm5, %zmm6
-; AVX512DQ-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm5 = [6,14,6,14,6,14,6,14]
-; AVX512DQ-FAST-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm4, %zmm2, %zmm5
-; AVX512DQ-FAST-NEXT:    vbroadcasti128 {{.*#+}} ymm7 = [6,14,6,14]
-; AVX512DQ-FAST-NEXT:    # ymm7 = mem[0,1,0,1]
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm1, %zmm0, %zmm7
-; AVX512DQ-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm5 = zmm7[0,1,2,3],zmm5[4,5,6,7]
+; AVX512DQ-FAST-NEXT:    vmovdqa64 (%rsi), %zmm6
+; AVX512DQ-FAST-NEXT:    vmovdqa64 (%rdx), %zmm5
+; AVX512DQ-FAST-NEXT:    vmovdqa64 (%rcx), %zmm7
+; AVX512DQ-FAST-NEXT:    vmovdqa64 (%r8), %zmm3
+; AVX512DQ-FAST-NEXT:    vmovdqa64 (%r9), %zmm4
+; AVX512DQ-FAST-NEXT:    vmovdqa64 (%r10), %zmm2
+; AVX512DQ-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm1 = [6,14,6,14,6,14,6,14]
+; AVX512DQ-FAST-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm5, %zmm8
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm7, %zmm1, %zmm8
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm6, %zmm0, %zmm1
+; AVX512DQ-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm1[0,1,2,3],zmm8[4,5,6,7]
+; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm8 = [5,0,14,6,5,0,14,6]
+; AVX512DQ-FAST-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm3, %zmm4, %zmm8
+; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm9 = [0,13,6,7,0,13,6,7]
+; AVX512DQ-FAST-NEXT:    # zmm9 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm2, %zmm8, %zmm9
 ; AVX512DQ-FAST-NEXT:    movb $-61, %sil
 ; AVX512DQ-FAST-NEXT:    kmovw %esi, %k1
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm6, %zmm5 {%k1}
-; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm6 = [3,0,12,4,3,0,12,4]
-; AVX512DQ-FAST-NEXT:    # zmm6 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm2, %zmm4, %zmm6
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm9, %zmm1 {%k1}
+; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm8 = [3,0,12,4,3,0,12,4]
+; AVX512DQ-FAST-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm5, %zmm7, %zmm8
 ; AVX512DQ-FAST-NEXT:    movb $48, %sil
 ; AVX512DQ-FAST-NEXT:    kmovw %esi, %k1
-; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm6 {%k1} = zmm0[0],zmm1[0],zmm0[2],zmm1[2],zmm0[4],zmm1[4],zmm0[6],zmm1[6]
+; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm8 {%k1} = zmm0[0],zmm6[0],zmm0[2],zmm6[2],zmm0[4],zmm6[4],zmm0[6],zmm6[6]
 ; AVX512DQ-FAST-NEXT:    vmovdqa (%r9), %ymm10
 ; AVX512DQ-FAST-NEXT:    vmovdqa (%r8), %ymm11
-; AVX512DQ-FAST-NEXT:    vmovdqa {{.*#+}} ymm7 = <1,3,7,u>
-; AVX512DQ-FAST-NEXT:    vpermi2q %ymm10, %ymm11, %ymm7
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm7 = ymm7[0,1,2,3,4,5],mem[6,7]
+; AVX512DQ-FAST-NEXT:    vmovdqa {{.*#+}} ymm9 = <1,3,7,u>
+; AVX512DQ-FAST-NEXT:    vpermi2q %ymm10, %ymm11, %ymm9
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3,4,5],mem[6,7]
 ; AVX512DQ-FAST-NEXT:    movb $14, %sil
 ; AVX512DQ-FAST-NEXT:    kmovw %esi, %k1
-; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm7, %zmm0, %zmm6 {%k1}
+; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm9, %zmm0, %zmm8 {%k1}
 ; AVX512DQ-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm12 = [11,3,11,3,11,3,11,3]
 ; AVX512DQ-FAST-NEXT:    # zmm12 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm1, %zmm0, %zmm12
-; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm7 = [2,10,0,3,2,10,0,3]
-; AVX512DQ-FAST-NEXT:    # zmm7 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm4, %zmm2, %zmm7
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm6, %zmm0, %zmm12
+; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm9 = [2,10,0,3,2,10,0,3]
+; AVX512DQ-FAST-NEXT:    # zmm9 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm7, %zmm5, %zmm9
 ; AVX512DQ-FAST-NEXT:    movb $96, %sil
 ; AVX512DQ-FAST-NEXT:    kmovw %esi, %k1
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm12, %zmm7 {%k1}
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm12, %zmm9 {%k1}
 ; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm10 = ymm11[0],ymm10[0],ymm11[2],ymm10[2]
 ; AVX512DQ-FAST-NEXT:    movb $28, %sil
 ; AVX512DQ-FAST-NEXT:    kmovw %esi, %k2
-; AVX512DQ-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm7 {%k2} = zmm10[2,3,2,3],zmm3[2,3,2,3]
+; AVX512DQ-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm9 {%k2} = zmm10[2,3,2,3],zmm2[2,3,2,3]
 ; AVX512DQ-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm10 = [15,7,15,7,15,7,15,7]
 ; AVX512DQ-FAST-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQ-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm11 = [9,1,9,1,9,1,9,1]
-; AVX512DQ-FAST-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm9, %zmm8, %zmm11
-; AVX512DQ-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm12 = [0,8,0,8,0,8,0,8]
-; AVX512DQ-FAST-NEXT:    # zmm12 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm9, %zmm8, %zmm12
-; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm13 = [4,12,0,5,4,12,0,5]
-; AVX512DQ-FAST-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm9, %zmm8, %zmm13
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm9, %zmm10, %zmm8
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm4, %zmm2, %zmm10
-; AVX512DQ-FAST-NEXT:    vbroadcasti128 {{.*#+}} ymm9 = [15,7,15,7]
-; AVX512DQ-FAST-NEXT:    # ymm9 = mem[0,1,0,1]
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm1, %zmm0, %zmm9
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm5, %zmm11
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm7, %zmm10, %zmm11
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm12
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm6, %zmm10, %zmm12
 ; AVX512DQ-FAST-NEXT:    movb $24, %sil
 ; AVX512DQ-FAST-NEXT:    kmovw %esi, %k2
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm10, %zmm9 {%k2}
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm8, %zmm9 {%k1}
-; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm8 = [14,1,2,3,4,5,6,15]
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm3, %zmm9, %zmm8
-; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm9 = [1,0,10,2,1,0,10,2]
-; AVX512DQ-FAST-NEXT:    # zmm9 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm0, %zmm1, %zmm9
-; AVX512DQ-FAST-NEXT:    vpbroadcastq 8(%rcx), %ymm10
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm10 = mem[0,1,2,3],ymm10[4,5,6,7]
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm11, %zmm12 {%k2}
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm4, %zmm3, %zmm10
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm10, %zmm12 {%k1}
+; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm10 = [14,1,2,3,4,5,6,15]
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm2, %zmm12, %zmm10
+; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm11 = [1,0,10,2,1,0,10,2]
+; AVX512DQ-FAST-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm0, %zmm6, %zmm11
+; AVX512DQ-FAST-NEXT:    vpbroadcastq 8(%rcx), %ymm12
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm12 = mem[0,1,2,3],ymm12[4,5,6,7]
 ; AVX512DQ-FAST-NEXT:    movb $6, %sil
 ; AVX512DQ-FAST-NEXT:    kmovw %esi, %k2
-; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm10, %zmm0, %zmm9 {%k2}
-; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm10 = [4,9,0,3,4,9,0,3]
-; AVX512DQ-FAST-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm3, %zmm11, %zmm10
+; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm12, %zmm0, %zmm11 {%k2}
+; AVX512DQ-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm12 = [9,1,9,1,9,1,9,1]
+; AVX512DQ-FAST-NEXT:    # zmm12 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm4, %zmm3, %zmm12
+; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm13 = [4,9,0,3,4,9,0,3]
+; AVX512DQ-FAST-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm2, %zmm12, %zmm13
 ; AVX512DQ-FAST-NEXT:    movb $56, %sil
 ; AVX512DQ-FAST-NEXT:    kmovw %esi, %k2
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm10, %zmm9 {%k2}
-; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm10 = [0,8,0,1,0,8,0,1]
-; AVX512DQ-FAST-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm1, %zmm0, %zmm10
-; AVX512DQ-FAST-NEXT:    vmovdqa (%rdx), %xmm11
-; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm11 = xmm11[0],mem[0]
-; AVX512DQ-FAST-NEXT:    vinserti128 $1, %xmm11, %ymm0, %ymm11
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm13, %zmm11 {%k2}
+; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm12 = [0,8,0,1,0,8,0,1]
+; AVX512DQ-FAST-NEXT:    # zmm12 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm6, %zmm0, %zmm12
+; AVX512DQ-FAST-NEXT:    vmovdqa (%rdx), %xmm13
+; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm13 = xmm13[0],mem[0]
+; AVX512DQ-FAST-NEXT:    vinserti128 $1, %xmm13, %ymm0, %ymm13
 ; AVX512DQ-FAST-NEXT:    movb $12, %cl
 ; AVX512DQ-FAST-NEXT:    kmovw %ecx, %k2
-; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm11, %zmm0, %zmm10 {%k2}
+; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm13, %zmm0, %zmm12 {%k2}
+; AVX512DQ-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm13 = [0,8,0,8,0,8,0,8]
+; AVX512DQ-FAST-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm4, %zmm3, %zmm13
 ; AVX512DQ-FAST-NEXT:    movb $112, %cl
 ; AVX512DQ-FAST-NEXT:    kmovw %ecx, %k2
-; AVX512DQ-FAST-NEXT:    vinserti64x2 $3, (%r10), %zmm12, %zmm10 {%k2}
-; AVX512DQ-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm11 = [13,5,13,5,13,5,13,5]
-; AVX512DQ-FAST-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm4, %zmm11, %zmm2
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm1, %zmm11, %zmm0
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm2, %zmm0 {%k1}
-; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm1 = [0,1,12,7,0,1,12,7]
-; AVX512DQ-FAST-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm3, %zmm13, %zmm1
+; AVX512DQ-FAST-NEXT:    vinserti64x2 $3, (%r10), %zmm13, %zmm12 {%k2}
+; AVX512DQ-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm13 = [13,5,13,5,13,5,13,5]
+; AVX512DQ-FAST-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm7, %zmm13, %zmm5
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm6, %zmm13, %zmm0
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm5, %zmm0 {%k1}
+; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm5 = [4,12,0,5,4,12,0,5]
+; AVX512DQ-FAST-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm4, %zmm3, %zmm5
+; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm3 = [0,1,12,7,0,1,12,7]
+; AVX512DQ-FAST-NEXT:    # zmm3 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm2, %zmm5, %zmm3
 ; AVX512DQ-FAST-NEXT:    movb $120, %cl
 ; AVX512DQ-FAST-NEXT:    kmovw %ecx, %k1
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm1, 256(%rax)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm10, (%rax)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm9, 64(%rax)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm8, 384(%rax)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm7, 128(%rax)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm6, 192(%rax)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm5, 320(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm3 {%k1}
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm3, 256(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm12, (%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm11, 64(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm10, 384(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm9, 128(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm8, 192(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm1, 320(%rax)
 ; AVX512DQ-FAST-NEXT:    vzeroupper
 ; AVX512DQ-FAST-NEXT:    retq
 ;
@@ -1244,123 +1239,122 @@ define void @store_i64_stride7_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512BW-ONLY-SLOW:       # %bb.0:
 ; AVX512BW-ONLY-SLOW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512BW-ONLY-SLOW-NEXT:    movq {{[0-9]+}}(%rsp), %r10
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 (%rdi), %zmm0
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 (%rsi), %zmm1
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 (%rdx), %zmm3
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 (%rcx), %zmm4
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 (%r8), %zmm6
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 (%r9), %zmm7
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 (%r10), %zmm2
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm5 = [15,7,15,7,15,7,15,7]
-; AVX512BW-ONLY-SLOW-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm8 = [0,8,0,8,0,8,0,8]
-; AVX512BW-ONLY-SLOW-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm7, %zmm6, %zmm8
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm9 = [5,0,14,6,5,0,14,6]
-; AVX512BW-ONLY-SLOW-NEXT:    # zmm9 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm6, %zmm7, %zmm9
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm11 = [9,1,9,1,9,1,9,1]
-; AVX512BW-ONLY-SLOW-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm7, %zmm6, %zmm11
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm10 = [4,12,0,5,4,12,0,5]
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 (%rdi), %zmm5
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 (%rsi), %zmm6
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 (%rdx), %zmm8
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 (%rcx), %zmm7
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 (%r8), %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 (%r9), %zmm2
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 (%r10), %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm3 = [15,7,15,7,15,7,15,7]
+; AVX512BW-ONLY-SLOW-NEXT:    # zmm3 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm9
+; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [2,10,0,3,2,10,0,3]
+; AVX512BW-ONLY-SLOW-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm7, %zmm8, %zmm4
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm11
+; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm10 = [3,0,12,4,3,0,12,4]
 ; AVX512BW-ONLY-SLOW-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm7, %zmm6, %zmm10
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm7, %zmm5, %zmm6
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm4, %zmm3, %zmm5
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti128 {{.*#+}} ymm7 = [15,7,15,7]
-; AVX512BW-ONLY-SLOW-NEXT:    # ymm7 = mem[0,1,0,1]
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm1, %zmm0, %zmm7
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm8, %zmm7, %zmm10
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm13
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm7, %zmm3, %zmm13
+; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm8 = [0,8,0,1,0,8,0,1]
+; AVX512BW-ONLY-SLOW-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm6, %zmm5, %zmm8
+; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm14 = [6,14,6,14,6,14,6,14]
+; AVX512BW-ONLY-SLOW-NEXT:    # zmm14 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm7, %zmm14, %zmm9
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm6, %zmm5, %zmm14
+; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm15 = [11,3,11,3,11,3,11,3]
+; AVX512BW-ONLY-SLOW-NEXT:    # zmm15 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm6, %zmm5, %zmm15
+; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm12 = [13,5,13,5,13,5,13,5]
+; AVX512BW-ONLY-SLOW-NEXT:    # zmm12 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm7, %zmm12, %zmm11
+; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm7 = [1,0,10,2,1,0,10,2]
+; AVX512BW-ONLY-SLOW-NEXT:    # zmm7 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm5, %zmm6, %zmm7
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm6, %zmm5, %zmm12
+; AVX512BW-ONLY-SLOW-NEXT:    movb $48, %sil
+; AVX512BW-ONLY-SLOW-NEXT:    kmovd %esi, %k1
+; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm10 {%k1} = zmm5[0],zmm6[0],zmm5[2],zmm6[2],zmm5[4],zmm6[4],zmm5[6],zmm6[6]
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm6, %zmm3, %zmm5
 ; AVX512BW-ONLY-SLOW-NEXT:    movb $24, %sil
 ; AVX512BW-ONLY-SLOW-NEXT:    kmovd %esi, %k1
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm7 {%k1}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm13, %zmm5 {%k1}
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm2, %zmm0, %zmm3
 ; AVX512BW-ONLY-SLOW-NEXT:    movb $96, %sil
 ; AVX512BW-ONLY-SLOW-NEXT:    kmovd %esi, %k1
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm7 {%k1}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [14,1,2,3,4,5,6,15]
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm2, %zmm7, %zmm5
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm6 = [0,8,0,1,0,8,0,1]
-; AVX512BW-ONLY-SLOW-NEXT:    # zmm6 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm1, %zmm0, %zmm6
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa (%rdx), %xmm7
-; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm7 = xmm7[0],mem[0]
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm7, %ymm0, %ymm7
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm5 {%k1}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [14,1,2,3,4,5,6,15]
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm1, %zmm5, %zmm3
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa (%rdx), %xmm5
+; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm5 = xmm5[0],mem[0]
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
 ; AVX512BW-ONLY-SLOW-NEXT:    movb $12, %sil
 ; AVX512BW-ONLY-SLOW-NEXT:    kmovd %esi, %k2
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm7, %zmm0, %zmm6 {%k2}
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti32x4 $3, (%r10), %zmm8, %zmm7
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm5, %zmm0, %zmm8 {%k2}
+; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm5 = [0,8,0,8,0,8,0,8]
+; AVX512BW-ONLY-SLOW-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm2, %zmm0, %zmm5
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti32x4 $3, (%r10), %zmm5, %zmm5
 ; AVX512BW-ONLY-SLOW-NEXT:    movb $112, %sil
 ; AVX512BW-ONLY-SLOW-NEXT:    kmovd %esi, %k2
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm6 {%k2}
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm7 = [0,13,6,7,0,13,6,7]
-; AVX512BW-ONLY-SLOW-NEXT:    # zmm7 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm2, %zmm9, %zmm7
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm8 = [6,14,6,14,6,14,6,14]
-; AVX512BW-ONLY-SLOW-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm4, %zmm3, %zmm8
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti128 {{.*#+}} ymm9 = [6,14,6,14]
-; AVX512BW-ONLY-SLOW-NEXT:    # ymm9 = mem[0,1,0,1]
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm1, %zmm0, %zmm9
-; AVX512BW-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm8 = zmm9[0,1,2,3],zmm8[4,5,6,7]
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm8 {%k2}
+; AVX512BW-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm5 = zmm14[0,1,2,3],zmm9[4,5,6,7]
+; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm6 = [5,0,14,6,5,0,14,6]
+; AVX512BW-ONLY-SLOW-NEXT:    # zmm6 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm0, %zmm2, %zmm6
+; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm9 = [0,13,6,7,0,13,6,7]
+; AVX512BW-ONLY-SLOW-NEXT:    # zmm9 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm1, %zmm6, %zmm9
 ; AVX512BW-ONLY-SLOW-NEXT:    movb $-61, %sil
 ; AVX512BW-ONLY-SLOW-NEXT:    kmovd %esi, %k2
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm8 {%k2}
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm7 = [11,3,11,3,11,3,11,3]
-; AVX512BW-ONLY-SLOW-NEXT:    # zmm7 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm1, %zmm0, %zmm7
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm9 = [2,10,0,3,2,10,0,3]
-; AVX512BW-ONLY-SLOW-NEXT:    # zmm9 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm4, %zmm3, %zmm9
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm9 {%k1}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa (%r9), %ymm7
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa (%r8), %ymm12
-; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm13 = ymm12[0],ymm7[0],ymm12[2],ymm7[2]
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm9, %zmm5 {%k2}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm15, %zmm4 {%k1}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa (%r9), %ymm6
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa (%r8), %ymm9
+; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm13 = ymm9[0],ymm6[0],ymm9[2],ymm6[2]
 ; AVX512BW-ONLY-SLOW-NEXT:    movb $28, %sil
 ; AVX512BW-ONLY-SLOW-NEXT:    kmovd %esi, %k2
-; AVX512BW-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm9 {%k2} = zmm13[2,3,2,3],zmm2[2,3,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm13 = [1,0,10,2,1,0,10,2]
-; AVX512BW-ONLY-SLOW-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm0, %zmm1, %zmm13
-; AVX512BW-ONLY-SLOW-NEXT:    vpbroadcastq 8(%rcx), %ymm14
-; AVX512BW-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm14 = mem[0,1,2,3],ymm14[4,5,6,7]
+; AVX512BW-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm4 {%k2} = zmm13[2,3,2,3],zmm1[2,3,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vpbroadcastq 8(%rcx), %ymm13
+; AVX512BW-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm13 = mem[0,1,2,3],ymm13[4,5,6,7]
 ; AVX512BW-ONLY-SLOW-NEXT:    movb $6, %cl
 ; AVX512BW-ONLY-SLOW-NEXT:    kmovd %ecx, %k2
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm14, %zmm0, %zmm13 {%k2}
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm13, %zmm0, %zmm7 {%k2}
+; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm13 = [9,1,9,1,9,1,9,1]
+; AVX512BW-ONLY-SLOW-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm2, %zmm0, %zmm13
 ; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm14 = [4,9,0,3,4,9,0,3]
 ; AVX512BW-ONLY-SLOW-NEXT:    # zmm14 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm2, %zmm11, %zmm14
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm1, %zmm13, %zmm14
 ; AVX512BW-ONLY-SLOW-NEXT:    movb $56, %cl
 ; AVX512BW-ONLY-SLOW-NEXT:    kmovd %ecx, %k2
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm13 {%k2}
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm11 = [13,5,13,5,13,5,13,5]
-; AVX512BW-ONLY-SLOW-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm14 = [3,0,12,4,3,0,12,4]
-; AVX512BW-ONLY-SLOW-NEXT:    # zmm14 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm3, %zmm4, %zmm14
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm4, %zmm11, %zmm3
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm1, %zmm0, %zmm11
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm11 {%k1}
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm3 = [0,1,12,7,0,1,12,7]
-; AVX512BW-ONLY-SLOW-NEXT:    # zmm3 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm2, %zmm10, %zmm3
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm7 {%k2}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm12 {%k1}
+; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm11 = [4,12,0,5,4,12,0,5]
+; AVX512BW-ONLY-SLOW-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm2, %zmm0, %zmm11
+; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [0,1,12,7,0,1,12,7]
+; AVX512BW-ONLY-SLOW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm1, %zmm11, %zmm0
 ; AVX512BW-ONLY-SLOW-NEXT:    movb $120, %cl
 ; AVX512BW-ONLY-SLOW-NEXT:    kmovd %ecx, %k1
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm3 {%k1}
-; AVX512BW-ONLY-SLOW-NEXT:    movb $48, %cl
-; AVX512BW-ONLY-SLOW-NEXT:    kmovd %ecx, %k1
-; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm14 {%k1} = zmm0[0],zmm1[0],zmm0[2],zmm1[2],zmm0[4],zmm1[4],zmm0[6],zmm1[6]
-; AVX512BW-ONLY-SLOW-NEXT:    vpunpckhqdq {{.*#+}} ymm0 = ymm12[1],ymm7[1],ymm12[3],ymm7[3]
-; AVX512BW-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,3,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],mem[6,7]
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm12, %zmm0 {%k1}
+; AVX512BW-ONLY-SLOW-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm9[1],ymm6[1],ymm9[3],ymm6[3]
+; AVX512BW-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,3,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],mem[6,7]
 ; AVX512BW-ONLY-SLOW-NEXT:    movb $14, %cl
 ; AVX512BW-ONLY-SLOW-NEXT:    kmovd %ecx, %k1
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm14 {%k1}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, 256(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm13, 64(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm9, 128(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm8, 320(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, (%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm5, 384(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm14, 192(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm10 {%k1}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, 256(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm7, 64(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm4, 128(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm5, 320(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm8, (%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, 384(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm10, 192(%rax)
 ; AVX512BW-ONLY-SLOW-NEXT:    vzeroupper
 ; AVX512BW-ONLY-SLOW-NEXT:    retq
 ;
@@ -1369,122 +1363,121 @@ define void @store_i64_stride7_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512BW-ONLY-FAST-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512BW-ONLY-FAST-NEXT:    movq {{[0-9]+}}(%rsp), %r10
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 (%rdi), %zmm0
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 (%rsi), %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 (%rdx), %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 (%rcx), %zmm4
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 (%r8), %zmm6
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 (%r9), %zmm7
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 (%r10), %zmm3
-; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm5 = [15,7,15,7,15,7,15,7]
-; AVX512BW-ONLY-FAST-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm8 = [0,8,0,8,0,8,0,8]
-; AVX512BW-ONLY-FAST-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm7, %zmm6, %zmm8
-; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm9 = [5,0,14,6,5,0,14,6]
-; AVX512BW-ONLY-FAST-NEXT:    # zmm9 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm6, %zmm7, %zmm9
-; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm10 = [9,1,9,1,9,1,9,1]
-; AVX512BW-ONLY-FAST-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm7, %zmm6, %zmm10
-; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm11 = [4,12,0,5,4,12,0,5]
-; AVX512BW-ONLY-FAST-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm7, %zmm6, %zmm11
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm7, %zmm5, %zmm6
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm4, %zmm2, %zmm5
-; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti128 {{.*#+}} ymm7 = [15,7,15,7]
-; AVX512BW-ONLY-FAST-NEXT:    # ymm7 = mem[0,1,0,1]
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm1, %zmm0, %zmm7
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 (%rsi), %zmm5
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 (%rdx), %zmm4
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 (%rcx), %zmm7
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 (%r8), %zmm2
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 (%r9), %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 (%r10), %zmm1
+; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm6 = [15,7,15,7,15,7,15,7]
+; AVX512BW-ONLY-FAST-NEXT:    # zmm6 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm4, %zmm8
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm7, %zmm6, %zmm8
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm9
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm5, %zmm6, %zmm9
 ; AVX512BW-ONLY-FAST-NEXT:    movb $24, %sil
 ; AVX512BW-ONLY-FAST-NEXT:    kmovd %esi, %k1
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm5, %zmm7 {%k1}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm8, %zmm9 {%k1}
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm3, %zmm2, %zmm6
 ; AVX512BW-ONLY-FAST-NEXT:    movb $96, %sil
 ; AVX512BW-ONLY-FAST-NEXT:    kmovd %esi, %k1
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm6, %zmm7 {%k1}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [14,1,2,3,4,5,6,15]
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm3, %zmm7, %zmm5
-; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm6 = [0,8,0,1,0,8,0,1]
-; AVX512BW-ONLY-FAST-NEXT:    # zmm6 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm1, %zmm0, %zmm6
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa (%rdx), %xmm7
-; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm7 = xmm7[0],mem[0]
-; AVX512BW-ONLY-FAST-NEXT:    vinserti128 $1, %xmm7, %ymm0, %ymm7
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm6, %zmm9 {%k1}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [14,1,2,3,4,5,6,15]
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm1, %zmm9, %zmm6
+; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm8 = [0,8,0,1,0,8,0,1]
+; AVX512BW-ONLY-FAST-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm5, %zmm0, %zmm8
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa (%rdx), %xmm9
+; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm9 = xmm9[0],mem[0]
+; AVX512BW-ONLY-FAST-NEXT:    vinserti128 $1, %xmm9, %ymm0, %ymm9
 ; AVX512BW-ONLY-FAST-NEXT:    movb $12, %sil
 ; AVX512BW-ONLY-FAST-NEXT:    kmovd %esi, %k2
-; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm7, %zmm0, %zmm6 {%k2}
-; AVX512BW-ONLY-FAST-NEXT:    vinserti32x4 $3, (%r10), %zmm8, %zmm7
+; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm9, %zmm0, %zmm8 {%k2}
+; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm9 = [0,8,0,8,0,8,0,8]
+; AVX512BW-ONLY-FAST-NEXT:    # zmm9 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm3, %zmm2, %zmm9
+; AVX512BW-ONLY-FAST-NEXT:    vinserti32x4 $3, (%r10), %zmm9, %zmm9
 ; AVX512BW-ONLY-FAST-NEXT:    movb $112, %sil
 ; AVX512BW-ONLY-FAST-NEXT:    kmovd %esi, %k2
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm7, %zmm6 {%k2}
-; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm7 = [0,13,6,7,0,13,6,7]
-; AVX512BW-ONLY-FAST-NEXT:    # zmm7 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm3, %zmm9, %zmm7
-; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm8 = [6,14,6,14,6,14,6,14]
-; AVX512BW-ONLY-FAST-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm4, %zmm2, %zmm8
-; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti128 {{.*#+}} ymm9 = [6,14,6,14]
-; AVX512BW-ONLY-FAST-NEXT:    # ymm9 = mem[0,1,0,1]
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm1, %zmm0, %zmm9
-; AVX512BW-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm8 = zmm9[0,1,2,3],zmm8[4,5,6,7]
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm9, %zmm8 {%k2}
+; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm9 = [6,14,6,14,6,14,6,14]
+; AVX512BW-ONLY-FAST-NEXT:    # zmm9 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm4, %zmm10
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm7, %zmm9, %zmm10
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm5, %zmm0, %zmm9
+; AVX512BW-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm9 = zmm9[0,1,2,3],zmm10[4,5,6,7]
+; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm10 = [5,0,14,6,5,0,14,6]
+; AVX512BW-ONLY-FAST-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm2, %zmm3, %zmm10
+; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm11 = [0,13,6,7,0,13,6,7]
+; AVX512BW-ONLY-FAST-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm1, %zmm10, %zmm11
 ; AVX512BW-ONLY-FAST-NEXT:    movb $-61, %sil
 ; AVX512BW-ONLY-FAST-NEXT:    kmovd %esi, %k2
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm7, %zmm8 {%k2}
-; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm7 = [3,0,12,4,3,0,12,4]
-; AVX512BW-ONLY-FAST-NEXT:    # zmm7 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm2, %zmm4, %zmm7
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm11, %zmm9 {%k2}
+; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm10 = [3,0,12,4,3,0,12,4]
+; AVX512BW-ONLY-FAST-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm4, %zmm7, %zmm10
 ; AVX512BW-ONLY-FAST-NEXT:    movb $48, %sil
 ; AVX512BW-ONLY-FAST-NEXT:    kmovd %esi, %k2
-; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm7 {%k2} = zmm0[0],zmm1[0],zmm0[2],zmm1[2],zmm0[4],zmm1[4],zmm0[6],zmm1[6]
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa (%r9), %ymm9
+; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm10 {%k2} = zmm0[0],zmm5[0],zmm0[2],zmm5[2],zmm0[4],zmm5[4],zmm0[6],zmm5[6]
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa (%r9), %ymm11
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqa (%r8), %ymm12
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqa {{.*#+}} ymm13 = <1,3,7,u>
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %ymm9, %ymm12, %ymm13
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %ymm11, %ymm12, %ymm13
 ; AVX512BW-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm13 = ymm13[0,1,2,3,4,5],mem[6,7]
 ; AVX512BW-ONLY-FAST-NEXT:    movb $14, %sil
 ; AVX512BW-ONLY-FAST-NEXT:    kmovd %esi, %k2
-; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm13, %zmm0, %zmm7 {%k2}
+; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm13, %zmm0, %zmm10 {%k2}
 ; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm13 = [11,3,11,3,11,3,11,3]
 ; AVX512BW-ONLY-FAST-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm1, %zmm0, %zmm13
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm5, %zmm0, %zmm13
 ; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm14 = [2,10,0,3,2,10,0,3]
 ; AVX512BW-ONLY-FAST-NEXT:    # zmm14 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm4, %zmm2, %zmm14
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm7, %zmm4, %zmm14
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm13, %zmm14 {%k1}
-; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm9 = ymm12[0],ymm9[0],ymm12[2],ymm9[2]
+; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm11 = ymm12[0],ymm11[0],ymm12[2],ymm11[2]
 ; AVX512BW-ONLY-FAST-NEXT:    movb $28, %sil
 ; AVX512BW-ONLY-FAST-NEXT:    kmovd %esi, %k2
-; AVX512BW-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm14 {%k2} = zmm9[2,3,2,3],zmm3[2,3,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm9 = [1,0,10,2,1,0,10,2]
-; AVX512BW-ONLY-FAST-NEXT:    # zmm9 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm0, %zmm1, %zmm9
+; AVX512BW-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm14 {%k2} = zmm11[2,3,2,3],zmm1[2,3,2,3]
+; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm11 = [1,0,10,2,1,0,10,2]
+; AVX512BW-ONLY-FAST-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm0, %zmm5, %zmm11
 ; AVX512BW-ONLY-FAST-NEXT:    vpbroadcastq 8(%rcx), %ymm12
 ; AVX512BW-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm12 = mem[0,1,2,3],ymm12[4,5,6,7]
 ; AVX512BW-ONLY-FAST-NEXT:    movb $6, %cl
 ; AVX512BW-ONLY-FAST-NEXT:    kmovd %ecx, %k2
-; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm12, %zmm0, %zmm9 {%k2}
-; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm12 = [4,9,0,3,4,9,0,3]
-; AVX512BW-ONLY-FAST-NEXT:    # zmm12 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm3, %zmm10, %zmm12
+; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm12, %zmm0, %zmm11 {%k2}
+; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm12 = [9,1,9,1,9,1,9,1]
+; AVX512BW-ONLY-FAST-NEXT:    # zmm12 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm3, %zmm2, %zmm12
+; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm13 = [4,9,0,3,4,9,0,3]
+; AVX512BW-ONLY-FAST-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm1, %zmm12, %zmm13
 ; AVX512BW-ONLY-FAST-NEXT:    movb $56, %cl
 ; AVX512BW-ONLY-FAST-NEXT:    kmovd %ecx, %k2
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm12, %zmm9 {%k2}
-; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm10 = [13,5,13,5,13,5,13,5]
-; AVX512BW-ONLY-FAST-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm4, %zmm10, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm1, %zmm10, %zmm0
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm0 {%k1}
-; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm1 = [0,1,12,7,0,1,12,7]
-; AVX512BW-ONLY-FAST-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm3, %zmm11, %zmm1
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm13, %zmm11 {%k2}
+; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm12 = [13,5,13,5,13,5,13,5]
+; AVX512BW-ONLY-FAST-NEXT:    # zmm12 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm7, %zmm12, %zmm4
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm5, %zmm12, %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm4, %zmm0 {%k1}
+; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [4,12,0,5,4,12,0,5]
+; AVX512BW-ONLY-FAST-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm3, %zmm2, %zmm4
+; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [0,1,12,7,0,1,12,7]
+; AVX512BW-ONLY-FAST-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm1, %zmm4, %zmm2
 ; AVX512BW-ONLY-FAST-NEXT:    movb $120, %cl
 ; AVX512BW-ONLY-FAST-NEXT:    kmovd %ecx, %k1
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, 256(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm9, 64(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k1}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, 256(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm11, 64(%rax)
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm14, 128(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm7, 192(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm8, 320(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm6, (%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm5, 384(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm10, 192(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm9, 320(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm8, (%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm6, 384(%rax)
 ; AVX512BW-ONLY-FAST-NEXT:    vzeroupper
 ; AVX512BW-ONLY-FAST-NEXT:    retq
 ;
@@ -1492,122 +1485,120 @@ define void @store_i64_stride7_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQBW-SLOW:       # %bb.0:
 ; AVX512DQBW-SLOW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512DQBW-SLOW-NEXT:    movq {{[0-9]+}}(%rsp), %r10
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 (%rdi), %zmm0
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 (%rsi), %zmm1
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 (%rdx), %zmm4
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 (%rcx), %zmm5
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 (%r8), %zmm9
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 (%r9), %zmm10
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 (%r10), %zmm3
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [5,0,14,6,5,0,14,6]
-; AVX512DQBW-SLOW-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm9, %zmm10, %zmm2
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm6 = [0,13,6,7,0,13,6,7]
-; AVX512DQBW-SLOW-NEXT:    # zmm6 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm3, %zmm2, %zmm6
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm2 = [6,14,6,14,6,14,6,14]
-; AVX512DQBW-SLOW-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm5, %zmm4, %zmm2
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti128 {{.*#+}} ymm7 = [6,14,6,14]
-; AVX512DQBW-SLOW-NEXT:    # ymm7 = mem[0,1,0,1]
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm1, %zmm0, %zmm7
-; AVX512DQBW-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm7[0,1,2,3],zmm2[4,5,6,7]
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 (%rdi), %zmm4
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 (%rsi), %zmm5
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 (%rdx), %zmm7
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 (%rcx), %zmm6
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 (%r8), %zmm0
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 (%r9), %zmm2
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 (%r10), %zmm1
+; AVX512DQBW-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm8 = [6,14,6,14,6,14,6,14]
+; AVX512DQBW-SLOW-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm3 = [2,10,0,3,2,10,0,3]
+; AVX512DQBW-SLOW-NEXT:    # zmm3 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm6, %zmm7, %zmm3
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm9
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm11
+; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm10 = [3,0,12,4,3,0,12,4]
+; AVX512DQBW-SLOW-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm7, %zmm6, %zmm10
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm6, %zmm8, %zmm7
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm5, %zmm4, %zmm8
+; AVX512DQBW-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm7 = zmm8[0,1,2,3],zmm7[4,5,6,7]
+; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm8 = [5,0,14,6,5,0,14,6]
+; AVX512DQBW-SLOW-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm0, %zmm2, %zmm8
+; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm12 = [0,13,6,7,0,13,6,7]
+; AVX512DQBW-SLOW-NEXT:    # zmm12 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm1, %zmm8, %zmm12
 ; AVX512DQBW-SLOW-NEXT:    movb $-61, %sil
 ; AVX512DQBW-SLOW-NEXT:    kmovd %esi, %k1
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm2 {%k1}
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm7 = [11,3,11,3,11,3,11,3]
-; AVX512DQBW-SLOW-NEXT:    # zmm7 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm1, %zmm0, %zmm7
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm6 = [2,10,0,3,2,10,0,3]
-; AVX512DQBW-SLOW-NEXT:    # zmm6 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm5, %zmm4, %zmm6
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm12, %zmm7 {%k1}
+; AVX512DQBW-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm8 = [11,3,11,3,11,3,11,3]
+; AVX512DQBW-SLOW-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm5, %zmm4, %zmm8
 ; AVX512DQBW-SLOW-NEXT:    movb $96, %sil
 ; AVX512DQBW-SLOW-NEXT:    kmovd %esi, %k1
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm6 {%k1}
-; AVX512DQBW-SLOW-NEXT:    vmovdqa (%r9), %ymm7
-; AVX512DQBW-SLOW-NEXT:    vmovdqa (%r8), %ymm8
-; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm11 = ymm8[0],ymm7[0],ymm8[2],ymm7[2]
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm3 {%k1}
+; AVX512DQBW-SLOW-NEXT:    vmovdqa (%r9), %ymm8
+; AVX512DQBW-SLOW-NEXT:    vmovdqa (%r8), %ymm12
+; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm13 = ymm12[0],ymm8[0],ymm12[2],ymm8[2]
 ; AVX512DQBW-SLOW-NEXT:    movb $28, %sil
 ; AVX512DQBW-SLOW-NEXT:    kmovd %esi, %k2
-; AVX512DQBW-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm6 {%k2} = zmm11[2,3,2,3],zmm3[2,3,2,3]
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm11 = [15,7,15,7,15,7,15,7]
-; AVX512DQBW-SLOW-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm12 = [9,1,9,1,9,1,9,1]
-; AVX512DQBW-SLOW-NEXT:    # zmm12 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm10, %zmm9, %zmm12
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm13 = [0,8,0,8,0,8,0,8]
+; AVX512DQBW-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm3 {%k2} = zmm13[2,3,2,3],zmm1[2,3,2,3]
+; AVX512DQBW-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm13 = [15,7,15,7,15,7,15,7]
 ; AVX512DQBW-SLOW-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm10, %zmm9, %zmm13
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm14 = [4,12,0,5,4,12,0,5]
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm6, %zmm13, %zmm9
+; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm14 = [1,0,10,2,1,0,10,2]
 ; AVX512DQBW-SLOW-NEXT:    # zmm14 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm10, %zmm9, %zmm14
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm10, %zmm11, %zmm9
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm5, %zmm4, %zmm11
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti128 {{.*#+}} ymm10 = [15,7,15,7]
-; AVX512DQBW-SLOW-NEXT:    # ymm10 = mem[0,1,0,1]
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm1, %zmm0, %zmm10
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm4, %zmm5, %zmm14
+; AVX512DQBW-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm15 = [13,5,13,5,13,5,13,5]
+; AVX512DQBW-SLOW-NEXT:    # zmm15 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm6, %zmm15, %zmm11
+; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm6 = [0,8,0,1,0,8,0,1]
+; AVX512DQBW-SLOW-NEXT:    # zmm6 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm5, %zmm4, %zmm6
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm5, %zmm4, %zmm15
+; AVX512DQBW-SLOW-NEXT:    movb $48, %sil
+; AVX512DQBW-SLOW-NEXT:    kmovd %esi, %k2
+; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm10 {%k2} = zmm4[0],zmm5[0],zmm4[2],zmm5[2],zmm4[4],zmm5[4],zmm4[6],zmm5[6]
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm5, %zmm13, %zmm4
 ; AVX512DQBW-SLOW-NEXT:    movb $24, %sil
 ; AVX512DQBW-SLOW-NEXT:    kmovd %esi, %k2
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm10 {%k2}
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm9, %zmm10 {%k1}
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm9 = [14,1,2,3,4,5,6,15]
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm3, %zmm10, %zmm9
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm10 = [1,0,10,2,1,0,10,2]
-; AVX512DQBW-SLOW-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm0, %zmm1, %zmm10
-; AVX512DQBW-SLOW-NEXT:    vpbroadcastq 8(%rcx), %ymm11
-; AVX512DQBW-SLOW-NEXT:    vpblendd {{.*#+}} ymm11 = mem[0,1,2,3],ymm11[4,5,6,7]
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm9, %zmm4 {%k2}
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm2, %zmm0, %zmm13
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm13, %zmm4 {%k1}
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [14,1,2,3,4,5,6,15]
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm1, %zmm4, %zmm5
+; AVX512DQBW-SLOW-NEXT:    vpbroadcastq 8(%rcx), %ymm4
+; AVX512DQBW-SLOW-NEXT:    vpblendd {{.*#+}} ymm4 = mem[0,1,2,3],ymm4[4,5,6,7]
 ; AVX512DQBW-SLOW-NEXT:    movb $6, %sil
 ; AVX512DQBW-SLOW-NEXT:    kmovd %esi, %k2
-; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm11, %zmm0, %zmm10 {%k2}
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm11 = [4,9,0,3,4,9,0,3]
-; AVX512DQBW-SLOW-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm3, %zmm12, %zmm11
+; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm4, %zmm0, %zmm14 {%k2}
+; AVX512DQBW-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm4 = [9,1,9,1,9,1,9,1]
+; AVX512DQBW-SLOW-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm2, %zmm0, %zmm4
+; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm9 = [4,9,0,3,4,9,0,3]
+; AVX512DQBW-SLOW-NEXT:    # zmm9 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm1, %zmm4, %zmm9
 ; AVX512DQBW-SLOW-NEXT:    movb $56, %sil
 ; AVX512DQBW-SLOW-NEXT:    kmovd %esi, %k2
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm10 {%k2}
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm11 = [0,8,0,1,0,8,0,1]
-; AVX512DQBW-SLOW-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm1, %zmm0, %zmm11
-; AVX512DQBW-SLOW-NEXT:    vmovdqa (%rdx), %xmm12
-; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm12 = xmm12[0],mem[0]
-; AVX512DQBW-SLOW-NEXT:    vinserti128 $1, %xmm12, %ymm0, %ymm12
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm9, %zmm14 {%k2}
+; AVX512DQBW-SLOW-NEXT:    vmovdqa (%rdx), %xmm4
+; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm4 = xmm4[0],mem[0]
+; AVX512DQBW-SLOW-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
 ; AVX512DQBW-SLOW-NEXT:    movb $12, %cl
 ; AVX512DQBW-SLOW-NEXT:    kmovd %ecx, %k2
-; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm12, %zmm0, %zmm11 {%k2}
+; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm4, %zmm0, %zmm6 {%k2}
+; AVX512DQBW-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm4 = [0,8,0,8,0,8,0,8]
+; AVX512DQBW-SLOW-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm2, %zmm0, %zmm4
 ; AVX512DQBW-SLOW-NEXT:    movb $112, %cl
 ; AVX512DQBW-SLOW-NEXT:    kmovd %ecx, %k2
-; AVX512DQBW-SLOW-NEXT:    vinserti64x2 $3, (%r10), %zmm13, %zmm11 {%k2}
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm12 = [13,5,13,5,13,5,13,5]
-; AVX512DQBW-SLOW-NEXT:    # zmm12 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm13 = [3,0,12,4,3,0,12,4]
-; AVX512DQBW-SLOW-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm4, %zmm5, %zmm13
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm5, %zmm12, %zmm4
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm1, %zmm0, %zmm12
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm12 {%k1}
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [0,1,12,7,0,1,12,7]
+; AVX512DQBW-SLOW-NEXT:    vinserti64x2 $3, (%r10), %zmm4, %zmm6 {%k2}
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm15 {%k1}
+; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [4,12,0,5,4,12,0,5]
 ; AVX512DQBW-SLOW-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm3, %zmm14, %zmm4
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm2, %zmm0, %zmm4
+; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [0,1,12,7,0,1,12,7]
+; AVX512DQBW-SLOW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm1, %zmm4, %zmm0
 ; AVX512DQBW-SLOW-NEXT:    movb $120, %cl
 ; AVX512DQBW-SLOW-NEXT:    kmovd %ecx, %k1
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm12, %zmm4 {%k1}
-; AVX512DQBW-SLOW-NEXT:    movb $48, %cl
-; AVX512DQBW-SLOW-NEXT:    kmovd %ecx, %k1
-; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm13 {%k1} = zmm0[0],zmm1[0],zmm0[2],zmm1[2],zmm0[4],zmm1[4],zmm0[6],zmm1[6]
-; AVX512DQBW-SLOW-NEXT:    vpunpckhqdq {{.*#+}} ymm0 = ymm8[1],ymm7[1],ymm8[3],ymm7[3]
-; AVX512DQBW-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,3,3]
-; AVX512DQBW-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],mem[6,7]
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm15, %zmm0 {%k1}
+; AVX512DQBW-SLOW-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm12[1],ymm8[1],ymm12[3],ymm8[3]
+; AVX512DQBW-SLOW-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,3,3]
+; AVX512DQBW-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],mem[6,7]
 ; AVX512DQBW-SLOW-NEXT:    movb $14, %cl
 ; AVX512DQBW-SLOW-NEXT:    kmovd %ecx, %k1
-; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm13 {%k1}
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm4, 256(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm11, (%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm10, 64(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm9, 384(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm6, 128(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm13, 192(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm2, 320(%rax)
+; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm10 {%k1}
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, 256(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm6, (%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm14, 64(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm5, 384(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm3, 128(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm10, 192(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm7, 320(%rax)
 ; AVX512DQBW-SLOW-NEXT:    vzeroupper
 ; AVX512DQBW-SLOW-NEXT:    retq
 ;
@@ -1616,121 +1607,120 @@ define void @store_i64_stride7_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512DQBW-FAST-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512DQBW-FAST-NEXT:    movq {{[0-9]+}}(%rsp), %r10
 ; AVX512DQBW-FAST-NEXT:    vmovdqa64 (%rdi), %zmm0
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 (%rsi), %zmm1
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 (%rdx), %zmm2
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 (%rcx), %zmm4
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 (%r8), %zmm8
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 (%r9), %zmm9
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 (%r10), %zmm3
-; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm5 = [5,0,14,6,5,0,14,6]
-; AVX512DQBW-FAST-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm8, %zmm9, %zmm5
-; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm6 = [0,13,6,7,0,13,6,7]
-; AVX512DQBW-FAST-NEXT:    # zmm6 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm3, %zmm5, %zmm6
-; AVX512DQBW-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm5 = [6,14,6,14,6,14,6,14]
-; AVX512DQBW-FAST-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm4, %zmm2, %zmm5
-; AVX512DQBW-FAST-NEXT:    vbroadcasti128 {{.*#+}} ymm7 = [6,14,6,14]
-; AVX512DQBW-FAST-NEXT:    # ymm7 = mem[0,1,0,1]
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm1, %zmm0, %zmm7
-; AVX512DQBW-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm5 = zmm7[0,1,2,3],zmm5[4,5,6,7]
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 (%rsi), %zmm6
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 (%rdx), %zmm5
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 (%rcx), %zmm7
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 (%r8), %zmm3
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 (%r9), %zmm4
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 (%r10), %zmm2
+; AVX512DQBW-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm1 = [6,14,6,14,6,14,6,14]
+; AVX512DQBW-FAST-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm5, %zmm8
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm7, %zmm1, %zmm8
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm6, %zmm0, %zmm1
+; AVX512DQBW-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm1[0,1,2,3],zmm8[4,5,6,7]
+; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm8 = [5,0,14,6,5,0,14,6]
+; AVX512DQBW-FAST-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm3, %zmm4, %zmm8
+; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm9 = [0,13,6,7,0,13,6,7]
+; AVX512DQBW-FAST-NEXT:    # zmm9 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm2, %zmm8, %zmm9
 ; AVX512DQBW-FAST-NEXT:    movb $-61, %sil
 ; AVX512DQBW-FAST-NEXT:    kmovd %esi, %k1
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm6, %zmm5 {%k1}
-; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm6 = [3,0,12,4,3,0,12,4]
-; AVX512DQBW-FAST-NEXT:    # zmm6 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm2, %zmm4, %zmm6
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm9, %zmm1 {%k1}
+; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm8 = [3,0,12,4,3,0,12,4]
+; AVX512DQBW-FAST-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm5, %zmm7, %zmm8
 ; AVX512DQBW-FAST-NEXT:    movb $48, %sil
 ; AVX512DQBW-FAST-NEXT:    kmovd %esi, %k1
-; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm6 {%k1} = zmm0[0],zmm1[0],zmm0[2],zmm1[2],zmm0[4],zmm1[4],zmm0[6],zmm1[6]
+; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm8 {%k1} = zmm0[0],zmm6[0],zmm0[2],zmm6[2],zmm0[4],zmm6[4],zmm0[6],zmm6[6]
 ; AVX512DQBW-FAST-NEXT:    vmovdqa (%r9), %ymm10
 ; AVX512DQBW-FAST-NEXT:    vmovdqa (%r8), %ymm11
-; AVX512DQBW-FAST-NEXT:    vmovdqa {{.*#+}} ymm7 = <1,3,7,u>
-; AVX512DQBW-FAST-NEXT:    vpermi2q %ymm10, %ymm11, %ymm7
-; AVX512DQBW-FAST-NEXT:    vpblendd {{.*#+}} ymm7 = ymm7[0,1,2,3,4,5],mem[6,7]
+; AVX512DQBW-FAST-NEXT:    vmovdqa {{.*#+}} ymm9 = <1,3,7,u>
+; AVX512DQBW-FAST-NEXT:    vpermi2q %ymm10, %ymm11, %ymm9
+; AVX512DQBW-FAST-NEXT:    vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3,4,5],mem[6,7]
 ; AVX512DQBW-FAST-NEXT:    movb $14, %sil
 ; AVX512DQBW-FAST-NEXT:    kmovd %esi, %k1
-; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm7, %zmm0, %zmm6 {%k1}
+; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm9, %zmm0, %zmm8 {%k1}
 ; AVX512DQBW-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm12 = [11,3,11,3,11,3,11,3]
 ; AVX512DQBW-FAST-NEXT:    # zmm12 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm1, %zmm0, %zmm12
-; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm7 = [2,10,0,3,2,10,0,3]
-; AVX512DQBW-FAST-NEXT:    # zmm7 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm4, %zmm2, %zmm7
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm6, %zmm0, %zmm12
+; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm9 = [2,10,0,3,2,10,0,3]
+; AVX512DQBW-FAST-NEXT:    # zmm9 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm7, %zmm5, %zmm9
 ; AVX512DQBW-FAST-NEXT:    movb $96, %sil
 ; AVX512DQBW-FAST-NEXT:    kmovd %esi, %k1
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm12, %zmm7 {%k1}
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm12, %zmm9 {%k1}
 ; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm10 = ymm11[0],ymm10[0],ymm11[2],ymm10[2]
 ; AVX512DQBW-FAST-NEXT:    movb $28, %sil
 ; AVX512DQBW-FAST-NEXT:    kmovd %esi, %k2
-; AVX512DQBW-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm7 {%k2} = zmm10[2,3,2,3],zmm3[2,3,2,3]
+; AVX512DQBW-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm9 {%k2} = zmm10[2,3,2,3],zmm2[2,3,2,3]
 ; AVX512DQBW-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm10 = [15,7,15,7,15,7,15,7]
 ; AVX512DQBW-FAST-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQBW-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm11 = [9,1,9,1,9,1,9,1]
-; AVX512DQBW-FAST-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm9, %zmm8, %zmm11
-; AVX512DQBW-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm12 = [0,8,0,8,0,8,0,8]
-; AVX512DQBW-FAST-NEXT:    # zmm12 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm9, %zmm8, %zmm12
-; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm13 = [4,12,0,5,4,12,0,5]
-; AVX512DQBW-FAST-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm9, %zmm8, %zmm13
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm9, %zmm10, %zmm8
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm4, %zmm2, %zmm10
-; AVX512DQBW-FAST-NEXT:    vbroadcasti128 {{.*#+}} ymm9 = [15,7,15,7]
-; AVX512DQBW-FAST-NEXT:    # ymm9 = mem[0,1,0,1]
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm1, %zmm0, %zmm9
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm5, %zmm11
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm7, %zmm10, %zmm11
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm12
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm6, %zmm10, %zmm12
 ; AVX512DQBW-FAST-NEXT:    movb $24, %sil
 ; AVX512DQBW-FAST-NEXT:    kmovd %esi, %k2
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm10, %zmm9 {%k2}
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm8, %zmm9 {%k1}
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm8 = [14,1,2,3,4,5,6,15]
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm3, %zmm9, %zmm8
-; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm9 = [1,0,10,2,1,0,10,2]
-; AVX512DQBW-FAST-NEXT:    # zmm9 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm0, %zmm1, %zmm9
-; AVX512DQBW-FAST-NEXT:    vpbroadcastq 8(%rcx), %ymm10
-; AVX512DQBW-FAST-NEXT:    vpblendd {{.*#+}} ymm10 = mem[0,1,2,3],ymm10[4,5,6,7]
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm11, %zmm12 {%k2}
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm4, %zmm3, %zmm10
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm10, %zmm12 {%k1}
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm10 = [14,1,2,3,4,5,6,15]
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm2, %zmm12, %zmm10
+; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm11 = [1,0,10,2,1,0,10,2]
+; AVX512DQBW-FAST-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm0, %zmm6, %zmm11
+; AVX512DQBW-FAST-NEXT:    vpbroadcastq 8(%rcx), %ymm12
+; AVX512DQBW-FAST-NEXT:    vpblendd {{.*#+}} ymm12 = mem[0,1,2,3],ymm12[4,5,6,7]
 ; AVX512DQBW-FAST-NEXT:    movb $6, %sil
 ; AVX512DQBW-FAST-NEXT:    kmovd %esi, %k2
-; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm10, %zmm0, %zmm9 {%k2}
-; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm10 = [4,9,0,3,4,9,0,3]
-; AVX512DQBW-FAST-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm3, %zmm11, %zmm10
+; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm12, %zmm0, %zmm11 {%k2}
+; AVX512DQBW-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm12 = [9,1,9,1,9,1,9,1]
+; AVX512DQBW-FAST-NEXT:    # zmm12 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm4, %zmm3, %zmm12
+; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm13 = [4,9,0,3,4,9,0,3]
+; AVX512DQBW-FAST-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm2, %zmm12, %zmm13
 ; AVX512DQBW-FAST-NEXT:    movb $56, %sil
 ; AVX512DQBW-FAST-NEXT:    kmovd %esi, %k2
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm10, %zmm9 {%k2}
-; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm10 = [0,8,0,1,0,8,0,1]
-; AVX512DQBW-FAST-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm1, %zmm0, %zmm10
-; AVX512DQBW-FAST-NEXT:    vmovdqa (%rdx), %xmm11
-; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm11 = xmm11[0],mem[0]
-; AVX512DQBW-FAST-NEXT:    vinserti128 $1, %xmm11, %ymm0, %ymm11
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm13, %zmm11 {%k2}
+; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm12 = [0,8,0,1,0,8,0,1]
+; AVX512DQBW-FAST-NEXT:    # zmm12 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm6, %zmm0, %zmm12
+; AVX512DQBW-FAST-NEXT:    vmovdqa (%rdx), %xmm13
+; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm13 = xmm13[0],mem[0]
+; AVX512DQBW-FAST-NEXT:    vinserti128 $1, %xmm13, %ymm0, %ymm13
 ; AVX512DQBW-FAST-NEXT:    movb $12, %cl
 ; AVX512DQBW-FAST-NEXT:    kmovd %ecx, %k2
-; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm11, %zmm0, %zmm10 {%k2}
+; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm13, %zmm0, %zmm12 {%k2}
+; AVX512DQBW-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm13 = [0,8,0,8,0,8,0,8]
+; AVX512DQBW-FAST-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm4, %zmm3, %zmm13
 ; AVX512DQBW-FAST-NEXT:    movb $112, %cl
 ; AVX512DQBW-FAST-NEXT:    kmovd %ecx, %k2
-; AVX512DQBW-FAST-NEXT:    vinserti64x2 $3, (%r10), %zmm12, %zmm10 {%k2}
-; AVX512DQBW-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm11 = [13,5,13,5,13,5,13,5]
-; AVX512DQBW-FAST-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm4, %zmm11, %zmm2
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm1, %zmm11, %zmm0
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm2, %zmm0 {%k1}
-; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm1 = [0,1,12,7,0,1,12,7]
-; AVX512DQBW-FAST-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm3, %zmm13, %zmm1
+; AVX512DQBW-FAST-NEXT:    vinserti64x2 $3, (%r10), %zmm13, %zmm12 {%k2}
+; AVX512DQBW-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm13 = [13,5,13,5,13,5,13,5]
+; AVX512DQBW-FAST-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm7, %zmm13, %zmm5
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm6, %zmm13, %zmm0
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm5, %zmm0 {%k1}
+; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm5 = [4,12,0,5,4,12,0,5]
+; AVX512DQBW-FAST-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm4, %zmm3, %zmm5
+; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm3 = [0,1,12,7,0,1,12,7]
+; AVX512DQBW-FAST-NEXT:    # zmm3 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm2, %zmm5, %zmm3
 ; AVX512DQBW-FAST-NEXT:    movb $120, %cl
 ; AVX512DQBW-FAST-NEXT:    kmovd %ecx, %k1
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm1, 256(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm10, (%rax)
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm9, 64(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm8, 384(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm7, 128(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm6, 192(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm5, 320(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm3 {%k1}
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm3, 256(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm12, (%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm11, 64(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm10, 384(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm9, 128(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm8, 192(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm1, 320(%rax)
 ; AVX512DQBW-FAST-NEXT:    vzeroupper
 ; AVX512DQBW-FAST-NEXT:    retq
   %in.vec0 = load <8 x i64>, ptr %in.vecptr0, align 64
@@ -2509,841 +2499,823 @@ define void @store_i64_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512F-ONLY-SLOW-LABEL: store_i64_stride7_vf16:
 ; AVX512F-ONLY-SLOW:       # %bb.0:
 ; AVX512F-ONLY-SLOW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 (%rdi), %zmm4
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 64(%rdi), %zmm6
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 (%rsi), %zmm5
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 64(%rsi), %zmm7
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 (%rdx), %zmm8
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 64(%rdx), %zmm9
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 (%rcx), %zmm2
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 64(%rcx), %zmm3
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 64(%r8), %zmm11
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [6,14,6,14,6,14,6,14]
-; AVX512F-ONLY-SLOW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 (%rdi), %zmm6
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 64(%rdi), %zmm18
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 (%rsi), %zmm7
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 64(%rsi), %zmm9
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 (%rdx), %zmm11
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 64(%rdx), %zmm20
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 (%rcx), %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 64(%rcx), %zmm2
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 64(%r8), %zmm3
+; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm16 = [6,14,6,14,6,14,6,14]
+; AVX512F-ONLY-SLOW-NEXT:    # zmm16 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm20, %zmm19
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm2, %zmm16, %zmm19
+; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm8 = [15,7,15,7,15,7,15,7]
+; AVX512F-ONLY-SLOW-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm20, %zmm21
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm2, %zmm8, %zmm21
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm18, %zmm22
+; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm5 = [0,8,0,1,0,8,0,1]
+; AVX512F-ONLY-SLOW-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm4
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm7, %zmm5, %zmm4
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm9, %zmm18, %zmm5
+; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm10 = [13,5,13,5,13,5,13,5]
+; AVX512F-ONLY-SLOW-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm20, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm2, %zmm10, %zmm0
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm9, %zmm12
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm3, %zmm0, %zmm12
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = [6,14,6,14]
-; AVX512F-ONLY-SLOW-NEXT:    # ymm0 = mem[0,1,0,1]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm15
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} ymm16 = [15,7,15,7]
-; AVX512F-ONLY-SLOW-NEXT:    # ymm16 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm10
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm5, %zmm16, %zmm10
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm7, %zmm6, %zmm16
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm13
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm7, %zmm0, %zmm13
-; AVX512F-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm23 = zmm13[0,1,2,3],zmm12[4,5,6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm18, %zmm15
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm18, %zmm23
+; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm13 = [2,10,0,3,2,10,0,3]
+; AVX512F-ONLY-SLOW-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm20, %zmm12
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm2, %zmm13, %zmm12
+; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm17 = [1,0,10,2,1,0,10,2]
+; AVX512F-ONLY-SLOW-NEXT:    # zmm17 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm9, %zmm14
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm18, %zmm17, %zmm14
+; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm29 = [3,0,12,4,3,0,12,4]
+; AVX512F-ONLY-SLOW-NEXT:    # zmm29 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm20, %zmm29, %zmm2
+; AVX512F-ONLY-SLOW-NEXT:    movb $48, %sil
+; AVX512F-ONLY-SLOW-NEXT:    kmovw %esi, %k3
+; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm2 {%k3} = zmm18[0],zmm9[0],zmm18[2],zmm9[2],zmm18[4],zmm9[4],zmm18[6],zmm9[6]
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm9, %zmm16, %zmm18
+; AVX512F-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm19 = zmm18[0,1,2,3],zmm19[4,5,6,7]
 ; AVX512F-ONLY-SLOW-NEXT:    movb $64, %sil
 ; AVX512F-ONLY-SLOW-NEXT:    kmovw %esi, %k1
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm23 {%k1}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 (%r8), %zmm14
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 64(%r9), %zmm24
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 (%r9), %zmm25
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 (%rax), %zmm22
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 64(%rax), %zmm30
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm12 = [0,8,0,1,0,8,0,1]
-; AVX512F-ONLY-SLOW-NEXT:    # zmm12 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm20 = [0,8,0,8,0,8,0,8]
-; AVX512F-ONLY-SLOW-NEXT:    # zmm20 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm21
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm25, %zmm20, %zmm21
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm24, %zmm11, %zmm20
-; AVX512F-ONLY-SLOW-NEXT:    movb $96, %sil
-; AVX512F-ONLY-SLOW-NEXT:    kmovw %esi, %k1
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm13 = [4,12,0,5,4,12,0,5]
-; AVX512F-ONLY-SLOW-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm18
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm17 = [15,7,15,7,15,7,15,7]
-; AVX512F-ONLY-SLOW-NEXT:    # zmm17 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm26
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm25, %zmm17, %zmm26
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm19
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm2, %zmm17, %zmm19
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm19 {%k1}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 (%r8), %zmm20
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 (%r9), %zmm30
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 64(%r9), %zmm31
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 64(%rax), %zmm28
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm9, %zmm8, %zmm22
 ; AVX512F-ONLY-SLOW-NEXT:    movb $24, %sil
+; AVX512F-ONLY-SLOW-NEXT:    kmovw %esi, %k1
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm21, %zmm22 {%k1}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm18 = <u,1,2,3,4,15,u,u>
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm3, %zmm22, %zmm18
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm21 = <u,1,2,3,4,5,15,u>
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm31, %zmm18, %zmm21
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm18 = [14,1,2,3,4,5,6,15]
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm28, %zmm21, %zmm18
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm21 = <13,u,2,3,4,5,6,14>
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm31, %zmm19, %zmm21
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm19 = [0,13,2,3,4,5,6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm28, %zmm21, %zmm19
+; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm26 = [0,8,0,8,0,8,0,8]
+; AVX512F-ONLY-SLOW-NEXT:    # zmm26 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm20, %zmm27
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm30, %zmm26, %zmm27
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm31, %zmm3, %zmm26
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm9, %zmm10, %zmm15
+; AVX512F-ONLY-SLOW-NEXT:    movb $96, %sil
 ; AVX512F-ONLY-SLOW-NEXT:    kmovw %esi, %k2
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm3, %zmm9, %zmm17
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm17, %zmm16 {%k2}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm27 = <u,1,2,3,4,15,u,u>
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm11, %zmm16, %zmm27
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm24, %zmm13, %zmm11
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm29 = [0,1,12,7,0,1,12,7]
-; AVX512F-ONLY-SLOW-NEXT:    # zmm29 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm30, %zmm29, %zmm11
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm31 = [11,3,11,3,11,3,11,3]
-; AVX512F-ONLY-SLOW-NEXT:    # zmm31 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm7, %zmm31, %zmm15
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm17 = [2,10,0,3,2,10,0,3]
-; AVX512F-ONLY-SLOW-NEXT:    # zmm17 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm9, %zmm16
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm3, %zmm17, %zmm16
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm15, %zmm16 {%k1}
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm28 = [9,1,9,1,9,1,9,1]
-; AVX512F-ONLY-SLOW-NEXT:    # zmm28 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [5,0,14,6,5,0,14,6]
-; AVX512F-ONLY-SLOW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm14, %zmm25, %zmm0
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm25, %zmm14, %zmm13
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm25, %zmm28, %zmm14
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 64(%r9), %ymm15
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu %ymm15, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm24, %zmm28, %zmm18
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm25 = <u,1,2,3,4,5,15,u>
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm24, %zmm27, %zmm25
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm1 = <13,u,2,3,4,5,6,14>
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm24, %zmm23, %zmm1
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 64(%r8), %ymm24
-; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm23 = ymm24[0],ymm15[0],ymm24[2],ymm15[2]
-; AVX512F-ONLY-SLOW-NEXT:    movb $28, %sil
-; AVX512F-ONLY-SLOW-NEXT:    kmovw %esi, %k3
-; AVX512F-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm16 {%k3} = zmm23[2,3,2,3],zmm30[2,3,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm28 = [6,13,14,7,6,13,14,7]
-; AVX512F-ONLY-SLOW-NEXT:    # zmm28 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm26, %zmm22, %zmm28
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm27 = [0,13,6,7,0,13,6,7]
-; AVX512F-ONLY-SLOW-NEXT:    # zmm27 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm22, %zmm0, %zmm27
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa (%r9), %ymm15
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm22, %zmm29, %zmm13
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 (%r8), %ymm26
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm5, %zmm4, %zmm31
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm2, %zmm8, %zmm17
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm31, %zmm17 {%k1}
-; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm23 = ymm26[0],ymm15[0],ymm26[2],ymm15[2]
-; AVX512F-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm17 {%k3} = zmm23[2,3,2,3],zmm22[2,3,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm23 = [4,9,0,3,4,9,0,3]
-; AVX512F-ONLY-SLOW-NEXT:    # zmm23 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm22, %zmm23, %zmm14
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm29 = [13,5,13,5,13,5,13,5]
+; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm21 = [4,12,0,5,4,12,0,5]
+; AVX512F-ONLY-SLOW-NEXT:    # zmm21 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm25
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm8, %zmm25
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm24
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm16, %zmm24
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm22
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm10, %zmm22
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm1, %zmm11, %zmm13
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm11, %zmm29, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm11
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm31, %zmm21, %zmm11
+; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm29 = [11,3,11,3,11,3,11,3]
 ; AVX512F-ONLY-SLOW-NEXT:    # zmm29 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm30, %zmm23, %zmm18
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm22 = [14,1,2,3,4,5,6,15]
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm30, %zmm25, %zmm22
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm23 = [0,13,2,3,4,5,6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm30, %zmm1, %zmm23
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm9, %zmm1
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm3, %zmm29, %zmm1
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm2, %zmm8, %zmm0
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm25 = [3,0,12,4,3,0,12,4]
-; AVX512F-ONLY-SLOW-NEXT:    # zmm25 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm9, %zmm25, %zmm3
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm30
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm2, %zmm29, %zmm30
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm8, %zmm25, %zmm2
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm8
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm5, %zmm12, %zmm8
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm7, %zmm6, %zmm12
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm31 = [1,0,10,2,1,0,10,2]
-; AVX512F-ONLY-SLOW-NEXT:    # zmm31 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm9
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm6, %zmm31, %zmm9
-; AVX512F-ONLY-SLOW-NEXT:    movb $48, %sil
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm9, %zmm29, %zmm23
+; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm9 = [0,1,12,7,0,1,12,7]
+; AVX512F-ONLY-SLOW-NEXT:    # zmm9 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm28, %zmm9, %zmm11
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm23, %zmm12 {%k2}
+; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm23 = [9,1,9,1,9,1,9,1]
+; AVX512F-ONLY-SLOW-NEXT:    # zmm23 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm31, %zmm23, %zmm3
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 64(%r9), %ymm31
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm7, %zmm6, %zmm16
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm7, %zmm6, %zmm10
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm7, %zmm6, %zmm29
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm6, %zmm7, %zmm17
+; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm1 {%k3} = zmm6[0],zmm7[0],zmm6[2],zmm7[2],zmm6[4],zmm7[4],zmm6[6],zmm7[6]
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm7, %zmm8, %zmm6
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm30, %zmm20, %zmm8
+; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm7 = [5,0,14,6,5,0,14,6]
+; AVX512F-ONLY-SLOW-NEXT:    # zmm7 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm20, %zmm30, %zmm7
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm30, %zmm20, %zmm21
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm30, %zmm23, %zmm20
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 64(%r8), %ymm23
+; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm30 = ymm23[0],ymm31[0],ymm23[2],ymm31[2]
+; AVX512F-ONLY-SLOW-NEXT:    movb $28, %sil
 ; AVX512F-ONLY-SLOW-NEXT:    kmovw %esi, %k3
-; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm3 {%k3} = zmm6[0],zmm7[0],zmm6[2],zmm7[2],zmm6[4],zmm7[4],zmm6[6],zmm7[6]
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm7, %zmm29, %zmm6
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti128 {{.*#+}} ymm7 = [6,14,6,14]
-; AVX512F-ONLY-SLOW-NEXT:    # ymm7 = mem[0,1,0,1]
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm5, %zmm4, %zmm7
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm5, %zmm4, %zmm29
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm4, %zmm5, %zmm31
-; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm2 {%k3} = zmm4[0],zmm5[0],zmm4[2],zmm5[2],zmm4[4],zmm5[4],zmm4[6],zmm5[6]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa (%rdx), %xmm4
-; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm4 = xmm4[0],mem[0]
-; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512F-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm12 {%k3} = zmm30[2,3,2,3],zmm28[2,3,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm30 = [4,9,0,3,4,9,0,3]
+; AVX512F-ONLY-SLOW-NEXT:    # zmm30 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm28, %zmm30, %zmm3
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 (%rax), %zmm28
+; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [6,13,14,7,6,13,14,7]
+; AVX512F-ONLY-SLOW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm8, %zmm28, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm8 = [0,13,6,7,0,13,6,7]
+; AVX512F-ONLY-SLOW-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm28, %zmm7, %zmm8
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm28, %zmm9, %zmm21
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa (%r9), %ymm7
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm29, %zmm13 {%k2}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa (%r8), %ymm9
+; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm29 = ymm9[0],ymm7[0],ymm9[2],ymm7[2]
+; AVX512F-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm13 {%k3} = zmm29[2,3,2,3],zmm28[2,3,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm28, %zmm30, %zmm20
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 (%rdx), %xmm28
+; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm28 = xmm28[0],mem[0]
+; AVX512F-ONLY-SLOW-NEXT:    vinserti32x4 $1, %xmm28, %ymm0, %ymm28
 ; AVX512F-ONLY-SLOW-NEXT:    movb $12, %sil
 ; AVX512F-ONLY-SLOW-NEXT:    kmovw %esi, %k3
-; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm4, %zmm0, %zmm8 {%k3}
-; AVX512F-ONLY-SLOW-NEXT:    vinserti32x4 $3, (%rax), %zmm21, %zmm4
+; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm28, %zmm0, %zmm4 {%k3}
+; AVX512F-ONLY-SLOW-NEXT:    vinserti32x4 $3, (%rax), %zmm27, %zmm27
 ; AVX512F-ONLY-SLOW-NEXT:    movb $112, %sil
 ; AVX512F-ONLY-SLOW-NEXT:    kmovw %esi, %k4
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm8 {%k4}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 64(%rdx), %xmm4
-; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm4 = xmm4[0],mem[0]
-; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm4, %zmm0, %zmm12 {%k3}
-; AVX512F-ONLY-SLOW-NEXT:    vinserti32x4 $3, 64(%rax), %zmm20, %zmm4
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm12 {%k4}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm6 {%k1}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm27, %zmm4 {%k4}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 64(%rdx), %xmm27
+; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm27 = xmm27[0],mem[0]
+; AVX512F-ONLY-SLOW-NEXT:    vinserti32x4 $1, %xmm27, %ymm0, %ymm27
+; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm27, %zmm0, %zmm5 {%k3}
+; AVX512F-ONLY-SLOW-NEXT:    vinserti32x4 $3, 64(%rax), %zmm26, %zmm26
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm26, %zmm5 {%k4}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm26 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm26, %zmm15 {%k2}
 ; AVX512F-ONLY-SLOW-NEXT:    movb $120, %sil
 ; AVX512F-ONLY-SLOW-NEXT:    kmovw %esi, %k3
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm11 {%k3}
-; AVX512F-ONLY-SLOW-NEXT:    vpbroadcastq 72(%rcx), %ymm1
-; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = mem[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm15, %zmm11 {%k3}
+; AVX512F-ONLY-SLOW-NEXT:    vpbroadcastq 72(%rcx), %ymm15
+; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm15 = mem[0,1,2,3],ymm15[4,5,6,7]
 ; AVX512F-ONLY-SLOW-NEXT:    movb $6, %sil
 ; AVX512F-ONLY-SLOW-NEXT:    kmovw %esi, %k4
-; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm9 {%k4}
+; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm15, %zmm0, %zmm14 {%k4}
 ; AVX512F-ONLY-SLOW-NEXT:    movb $56, %sil
 ; AVX512F-ONLY-SLOW-NEXT:    kmovw %esi, %k5
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm18, %zmm9 {%k5}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm19, %zmm10 {%k2}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm14 {%k5}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm25, %zmm6 {%k1}
 ; AVX512F-ONLY-SLOW-NEXT:    movb $-31, %sil
-; AVX512F-ONLY-SLOW-NEXT:    kmovw %esi, %k2
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm28, %zmm10 {%k2}
-; AVX512F-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm7[0,1,2,3],zmm0[4,5,6,7]
+; AVX512F-ONLY-SLOW-NEXT:    kmovw %esi, %k1
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm6 {%k1}
+; AVX512F-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm16[0,1,2,3],zmm24[4,5,6,7]
 ; AVX512F-ONLY-SLOW-NEXT:    movb $-61, %sil
-; AVX512F-ONLY-SLOW-NEXT:    kmovw %esi, %k2
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm27, %zmm1 {%k2}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm30, %zmm29 {%k1}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm29, %zmm13 {%k3}
-; AVX512F-ONLY-SLOW-NEXT:    vpbroadcastq 8(%rcx), %ymm4
-; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm4 = mem[0,1,2,3],ymm4[4,5,6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm4, %zmm0, %zmm31 {%k4}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm31 {%k5}
-; AVX512F-ONLY-SLOW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm24, %ymm4 # 32-byte Folded Reload
-; AVX512F-ONLY-SLOW-NEXT:    # ymm4 = ymm24[1],mem[1],ymm24[3],mem[3]
-; AVX512F-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[0,2,3,3]
-; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3,4,5],mem[6,7]
+; AVX512F-ONLY-SLOW-NEXT:    kmovw %esi, %k1
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm0 {%k1}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm22, %zmm10 {%k2}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm10, %zmm21 {%k3}
+; AVX512F-ONLY-SLOW-NEXT:    vpbroadcastq 8(%rcx), %ymm3
+; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm3 = mem[0,1,2,3],ymm3[4,5,6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm17 {%k4}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm20, %zmm17 {%k5}
+; AVX512F-ONLY-SLOW-NEXT:    vpunpckhqdq {{.*#+}} ymm3 = ymm23[1],ymm31[1],ymm23[3],ymm31[3]
+; AVX512F-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,2,3,3]
+; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5],mem[6,7]
 ; AVX512F-ONLY-SLOW-NEXT:    movb $14, %cl
 ; AVX512F-ONLY-SLOW-NEXT:    kmovw %ecx, %k1
-; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm4, %zmm0, %zmm3 {%k1}
-; AVX512F-ONLY-SLOW-NEXT:    vpunpckhqdq {{.*#+}} ymm0 = ymm26[1],ymm15[1],ymm26[3],ymm15[3]
-; AVX512F-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,3,3]
-; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],mem[6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm2 {%k1}
+; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm2 {%k1}
+; AVX512F-ONLY-SLOW-NEXT:    vpunpckhqdq {{.*#+}} ymm3 = ymm9[1],ymm7[1],ymm9[3],ymm7[3]
+; AVX512F-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,2,3,3]
+; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5],mem[6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm1 {%k1}
 ; AVX512F-ONLY-SLOW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm31, 64(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm17, 128(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm13, 256(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, 320(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm10, 384(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm9, 512(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm16, 576(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm17, 64(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm13, 128(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm21, 256(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, 320(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, 384(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm14, 512(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm12, 576(%rax)
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm11, 704(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, 192(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm12, 448(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, 640(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm8, (%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm23, 768(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm22, 832(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, 192(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm5, 448(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, 640(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm4, (%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm19, 768(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm18, 832(%rax)
 ; AVX512F-ONLY-SLOW-NEXT:    vzeroupper
 ; AVX512F-ONLY-SLOW-NEXT:    retq
 ;
 ; AVX512F-ONLY-FAST-LABEL: store_i64_stride7_vf16:
 ; AVX512F-ONLY-FAST:       # %bb.0:
 ; AVX512F-ONLY-FAST-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 (%rdi), %zmm12
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 64(%rdi), %zmm4
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 (%rsi), %zmm0
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 64(%rsi), %zmm5
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 (%rdx), %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 64(%rdx), %zmm7
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 (%rcx), %zmm17
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 64(%rcx), %zmm9
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 64(%r8), %zmm30
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm20 = [6,14,6,14,6,14,6,14]
-; AVX512F-ONLY-FAST-NEXT:    # zmm20 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm7, %zmm18
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm9, %zmm20, %zmm18
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} ymm21 = [6,14,6,14]
-; AVX512F-ONLY-FAST-NEXT:    # ymm21 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm4, %zmm19
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm5, %zmm21, %zmm19
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 (%r8), %zmm11
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 64(%r9), %zmm10
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 (%r9), %zmm15
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 (%rax), %zmm14
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 64(%rax), %zmm31
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm3 = [0,8,0,1,0,8,0,1]
-; AVX512F-ONLY-FAST-NEXT:    # zmm3 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm12, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm3, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa (%rdx), %xmm13
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 64(%rdx), %xmm16
-; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm13 = xmm13[0],mem[0]
-; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm13, %ymm0, %ymm13
-; AVX512F-ONLY-FAST-NEXT:    movb $12, %sil
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 (%rdi), %zmm18
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 64(%rdi), %zmm26
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 (%rsi), %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 64(%rsi), %zmm19
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 (%rdx), %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 64(%rdx), %zmm1
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 (%rcx), %zmm15
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 64(%rcx), %zmm21
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 64(%r8), %zmm6
+; AVX512F-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm17 = [6,14,6,14,6,14,6,14]
+; AVX512F-ONLY-FAST-NEXT:    # zmm17 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm21, %zmm17, %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm26, %zmm4
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm19, %zmm17, %zmm4
+; AVX512F-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm8 = zmm4[0,1,2,3],zmm0[4,5,6,7]
+; AVX512F-ONLY-FAST-NEXT:    movb $64, %sil
 ; AVX512F-ONLY-FAST-NEXT:    kmovw %esi, %k1
-; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm13, %zmm0, %zmm2 {%k1}
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm13 = [0,8,0,8,0,8,0,8]
-; AVX512F-ONLY-FAST-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm11, %zmm22
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm15, %zmm13, %zmm22
-; AVX512F-ONLY-FAST-NEXT:    vinserti32x4 $3, (%rax), %zmm22, %zmm22
-; AVX512F-ONLY-FAST-NEXT:    movb $112, %sil
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm6, %zmm8 {%k1}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 (%r8), %zmm4
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 (%r9), %zmm5
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 64(%r9), %zmm23
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 64(%rax), %zmm12
+; AVX512F-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm11 = [15,7,15,7,15,7,15,7]
+; AVX512F-ONLY-FAST-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, %zmm7
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm21, %zmm11, %zmm7
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm26, %zmm9
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm19, %zmm11, %zmm9
+; AVX512F-ONLY-FAST-NEXT:    movb $24, %sil
+; AVX512F-ONLY-FAST-NEXT:    kmovw %esi, %k1
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm7, %zmm9 {%k1}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm7 = <u,1,2,3,4,15,u,u>
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm6, %zmm9, %zmm7
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm9 = <u,1,2,3,4,5,15,u>
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm23, %zmm7, %zmm9
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [14,1,2,3,4,5,6,15]
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm12, %zmm9, %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm9 = <13,u,2,3,4,5,6,14>
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm23, %zmm8, %zmm9
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm8 = [0,13,2,3,4,5,6,7]
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm12, %zmm9, %zmm8
+; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm10 = [0,8,0,1,0,8,0,1]
+; AVX512F-ONLY-FAST-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm18, %zmm9
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm2, %zmm10, %zmm9
+; AVX512F-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm7 = [0,8,0,8,0,8,0,8]
+; AVX512F-ONLY-FAST-NEXT:    # zmm7 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm4, %zmm22
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm5, %zmm7, %zmm22
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm19, %zmm26, %zmm10
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm23, %zmm6, %zmm7
+; AVX512F-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm14 = [13,5,13,5,13,5,13,5]
+; AVX512F-ONLY-FAST-NEXT:    # zmm14 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, %zmm25
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm21, %zmm14, %zmm25
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm26, %zmm24
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm19, %zmm14, %zmm24
+; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm13 = [4,12,0,5,4,12,0,5]
+; AVX512F-ONLY-FAST-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm20 = [3,0,12,4,3,0,12,4]
+; AVX512F-ONLY-FAST-NEXT:    # zmm20 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm21, %zmm16
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm1, %zmm20, %zmm16
+; AVX512F-ONLY-FAST-NEXT:    movb $48, %sil
 ; AVX512F-ONLY-FAST-NEXT:    kmovw %esi, %k2
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm22, %zmm2 {%k2}
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm5, %zmm4, %zmm3
-; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm16 = xmm16[0],mem[0]
-; AVX512F-ONLY-FAST-NEXT:    vinserti32x4 $1, %xmm16, %ymm0, %ymm16
-; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm16, %zmm0, %zmm3 {%k1}
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm10, %zmm30, %zmm13
-; AVX512F-ONLY-FAST-NEXT:    vinserti32x4 $3, 64(%rax), %zmm13, %zmm13
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm13, %zmm3 {%k2}
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm24 = [13,5,13,5,13,5,13,5]
-; AVX512F-ONLY-FAST-NEXT:    # zmm24 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm7, %zmm13
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm9, %zmm24, %zmm13
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm4, %zmm26
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm5, %zmm24, %zmm26
+; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm16 {%k2} = zmm26[0],zmm19[0],zmm26[2],zmm19[2],zmm26[4],zmm19[4],zmm26[6],zmm19[6]
+; AVX512F-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm27 = [11,3,11,3,11,3,11,3]
+; AVX512F-ONLY-FAST-NEXT:    # zmm27 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm26, %zmm29
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm19, %zmm27, %zmm29
+; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm31 = [1,0,10,2,1,0,10,2]
+; AVX512F-ONLY-FAST-NEXT:    # zmm31 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm26, %zmm31, %zmm19
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm18, %zmm26
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm2, %zmm11, %zmm26
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm28
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm15, %zmm17, %zmm28
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm2, %zmm18, %zmm17
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm30
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm15, %zmm14, %zmm30
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm2, %zmm18, %zmm14
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm3, %zmm15, %zmm20
+; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm20 {%k2} = zmm18[0],zmm2[0],zmm18[2],zmm2[2],zmm18[4],zmm2[4],zmm18[6],zmm2[6]
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm2, %zmm18, %zmm27
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm18, %zmm31, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm6, %zmm18
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm23, %zmm13, %zmm18
+; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm31 = [2,10,0,3,2,10,0,3]
+; AVX512F-ONLY-FAST-NEXT:    # zmm31 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm21, %zmm31, %zmm1
+; AVX512F-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [9,1,9,1,9,1,9,1]
+; AVX512F-ONLY-FAST-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm23, %zmm0, %zmm6
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm21
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm15, %zmm31, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 64(%r9), %ymm23
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm15, %zmm11, %zmm21
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm5, %zmm4, %zmm11
+; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm31 = [5,0,14,6,5,0,14,6]
+; AVX512F-ONLY-FAST-NEXT:    # zmm31 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm4, %zmm5, %zmm31
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm5, %zmm4, %zmm13
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm5, %zmm0, %zmm4
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa 64(%r8), %ymm0
+; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm5 = ymm0[0],ymm23[0],ymm0[2],ymm23[2]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa {{.*#+}} ymm15 = [1,3,7,7]
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %ymm23, %ymm15, %ymm0
 ; AVX512F-ONLY-FAST-NEXT:    movb $96, %sil
-; AVX512F-ONLY-FAST-NEXT:    kmovw %esi, %k1
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm13, %zmm26 {%k1}
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm16 = [4,12,0,5,4,12,0,5]
-; AVX512F-ONLY-FAST-NEXT:    # zmm16 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm30, %zmm13
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm10, %zmm16, %zmm13
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm27 = [0,1,12,7,0,1,12,7]
-; AVX512F-ONLY-FAST-NEXT:    # zmm27 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm31, %zmm27, %zmm13
-; AVX512F-ONLY-FAST-NEXT:    movb $120, %sil
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm23 = [15,7,15,7,15,7,15,7]
-; AVX512F-ONLY-FAST-NEXT:    # zmm23 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm11, %zmm22
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm15, %zmm23, %zmm22
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm28 = [6,13,14,7,6,13,14,7]
-; AVX512F-ONLY-FAST-NEXT:    # zmm28 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm22, %zmm14, %zmm28
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, %zmm29
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm17, %zmm23, %zmm29
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} ymm25 = [15,7,15,7]
-; AVX512F-ONLY-FAST-NEXT:    # ymm25 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm12, %zmm22
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm25, %zmm22
-; AVX512F-ONLY-FAST-NEXT:    movb $24, %dil
-; AVX512F-ONLY-FAST-NEXT:    kmovw %edi, %k2
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm29, %zmm22 {%k2}
-; AVX512F-ONLY-FAST-NEXT:    movb $-31, %dil
-; AVX512F-ONLY-FAST-NEXT:    kmovw %edi, %k3
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm28, %zmm22 {%k3}
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm28 = [5,0,14,6,5,0,14,6]
-; AVX512F-ONLY-FAST-NEXT:    # zmm28 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm11, %zmm15, %zmm28
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm29 = [0,13,6,7,0,13,6,7]
+; AVX512F-ONLY-FAST-NEXT:    kmovw %esi, %k2
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm29, %zmm1 {%k2}
+; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm23 = [0,1,12,7,0,1,12,7]
+; AVX512F-ONLY-FAST-NEXT:    # zmm23 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm12, %zmm23, %zmm18
+; AVX512F-ONLY-FAST-NEXT:    movb $28, %sil
+; AVX512F-ONLY-FAST-NEXT:    kmovw %esi, %k3
+; AVX512F-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm1 {%k3} = zmm5[2,3,2,3],zmm12[2,3,2,3]
+; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm5 = [4,9,0,3,4,9,0,3]
+; AVX512F-ONLY-FAST-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm12, %zmm5, %zmm6
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 (%rax), %zmm12
+; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm29 = [6,13,14,7,6,13,14,7]
 ; AVX512F-ONLY-FAST-NEXT:    # zmm29 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm14, %zmm28, %zmm29
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm17, %zmm1, %zmm20
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm0, %zmm12, %zmm21
-; AVX512F-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm20 = zmm21[0,1,2,3],zmm20[4,5,6,7]
-; AVX512F-ONLY-FAST-NEXT:    movb $-61, %dil
-; AVX512F-ONLY-FAST-NEXT:    kmovw %edi, %k3
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm29, %zmm20 {%k3}
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm11, %zmm12, %zmm29
+; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm11 = [0,13,6,7,0,13,6,7]
+; AVX512F-ONLY-FAST-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm12, %zmm31, %zmm11
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm12, %zmm23, %zmm13
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 (%r9), %ymm23
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm27, %zmm3 {%k2}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 (%r8), %ymm27
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %ymm23, %ymm27, %ymm15
+; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm23 = ymm27[0],ymm23[0],ymm27[2],ymm23[2]
+; AVX512F-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm3 {%k3} = zmm23[2,3,2,3],zmm12[2,3,2,3]
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm12, %zmm5, %zmm4
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa (%rdx), %xmm5
+; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm5 = xmm5[0],mem[0]
+; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512F-ONLY-FAST-NEXT:    movb $12, %sil
 ; AVX512F-ONLY-FAST-NEXT:    kmovw %esi, %k3
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm26, %zmm13 {%k3}
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm26 = [3,0,12,4,3,0,12,4]
-; AVX512F-ONLY-FAST-NEXT:    # zmm26 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm9, %zmm21
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm7, %zmm26, %zmm21
-; AVX512F-ONLY-FAST-NEXT:    movb $48, %sil
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, %zmm28
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm17, %zmm24, %zmm28
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm0, %zmm12, %zmm24
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm28, %zmm24 {%k1}
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm15, %zmm11, %zmm16
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm14, %zmm27, %zmm16
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm24, %zmm16 {%k3}
+; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm5, %zmm0, %zmm9 {%k3}
+; AVX512F-ONLY-FAST-NEXT:    vinserti32x4 $3, (%rax), %zmm22, %zmm5
+; AVX512F-ONLY-FAST-NEXT:    movb $112, %sil
+; AVX512F-ONLY-FAST-NEXT:    kmovw %esi, %k4
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm5, %zmm9 {%k4}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa 64(%rdx), %xmm5
+; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm5 = xmm5[0],mem[0]
+; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm5, %zmm0, %zmm10 {%k3}
+; AVX512F-ONLY-FAST-NEXT:    vinserti32x4 $3, 64(%rax), %zmm7, %zmm5
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm5, %zmm10 {%k4}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm25, %zmm24 {%k2}
+; AVX512F-ONLY-FAST-NEXT:    movb $120, %sil
 ; AVX512F-ONLY-FAST-NEXT:    kmovw %esi, %k3
-; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm21 {%k3} = zmm4[0],zmm5[0],zmm4[2],zmm5[2],zmm4[4],zmm5[4],zmm4[6],zmm5[6]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 (%r9), %ymm24
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 64(%r9), %ymm27
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 64(%r8), %ymm28
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa {{.*#+}} ymm6 = [1,3,7,7]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %ymm28, %ymm8
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %ymm27, %ymm6, %ymm8
-; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],mem[6,7]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm24, %zmm18 {%k3}
+; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],mem[6,7]
 ; AVX512F-ONLY-FAST-NEXT:    movb $14, %sil
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm1, %zmm17, %zmm26
-; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm26 {%k3} = zmm12[0],zmm0[0],zmm12[2],zmm0[2],zmm12[4],zmm0[4],zmm12[6],zmm0[6]
-; AVX512F-ONLY-FAST-NEXT:    kmovw %esi, %k3
-; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm8, %zmm0, %zmm21 {%k3}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa (%r8), %ymm8
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %ymm24, %ymm8, %ymm6
-; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3,4,5],mem[6,7]
-; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm6, %zmm0, %zmm26 {%k3}
-; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm6 = ymm28[0],ymm27[0],ymm28[2],ymm27[2]
-; AVX512F-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm18 = zmm19[0,1,2,3],zmm18[4,5,6,7]
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm19 = [11,3,11,3,11,3,11,3]
-; AVX512F-ONLY-FAST-NEXT:    # zmm19 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm27 = [2,10,0,3,2,10,0,3]
-; AVX512F-ONLY-FAST-NEXT:    # zmm27 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm17, %zmm27, %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm7, %zmm17
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm9, %zmm27, %zmm17
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm4, %zmm27
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm5, %zmm19, %zmm27
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm27, %zmm17 {%k1}
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm0, %zmm12, %zmm19
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm19, %zmm1 {%k1}
-; AVX512F-ONLY-FAST-NEXT:    movb $28, %al
-; AVX512F-ONLY-FAST-NEXT:    kmovw %eax, %k1
-; AVX512F-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm17 {%k1} = zmm6[2,3,2,3],zmm31[2,3,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm6 = [1,0,10,2,1,0,10,2]
-; AVX512F-ONLY-FAST-NEXT:    # zmm6 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm12, %zmm6, %zmm0
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm5, %zmm12
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm4, %zmm6, %zmm12
-; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm6 = ymm8[0],ymm24[0],ymm8[2],ymm24[2]
-; AVX512F-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm1 {%k1} = zmm6[2,3,2,3],zmm14[2,3,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vpbroadcastq 72(%rcx), %ymm6
-; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm6 = mem[0,1,2,3],ymm6[4,5,6,7]
-; AVX512F-ONLY-FAST-NEXT:    movb $6, %al
-; AVX512F-ONLY-FAST-NEXT:    kmovw %eax, %k1
-; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm6, %zmm0, %zmm12 {%k1}
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm6 = [9,1,9,1,9,1,9,1]
-; AVX512F-ONLY-FAST-NEXT:    # zmm6 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm15, %zmm6, %zmm11
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm30, %zmm8
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm10, %zmm6, %zmm8
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm6 = [4,9,0,3,4,9,0,3]
-; AVX512F-ONLY-FAST-NEXT:    # zmm6 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm14, %zmm6, %zmm11
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm31, %zmm6, %zmm8
-; AVX512F-ONLY-FAST-NEXT:    vpbroadcastq 8(%rcx), %ymm6
-; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm6 = mem[0,1,2,3],ymm6[4,5,6,7]
-; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm6, %zmm0, %zmm0 {%k1}
-; AVX512F-ONLY-FAST-NEXT:    movb $64, %al
-; AVX512F-ONLY-FAST-NEXT:    kmovw %eax, %k1
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm30, %zmm18 {%k1}
-; AVX512F-ONLY-FAST-NEXT:    movb $56, %al
-; AVX512F-ONLY-FAST-NEXT:    kmovw %eax, %k1
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm8, %zmm12 {%k1}
+; AVX512F-ONLY-FAST-NEXT:    kmovw %esi, %k4
+; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm16 {%k4}
+; AVX512F-ONLY-FAST-NEXT:    vpbroadcastq 72(%rcx), %ymm0
+; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
+; AVX512F-ONLY-FAST-NEXT:    movb $6, %sil
+; AVX512F-ONLY-FAST-NEXT:    kmovw %esi, %k5
+; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm19 {%k5}
+; AVX512F-ONLY-FAST-NEXT:    movb $56, %sil
+; AVX512F-ONLY-FAST-NEXT:    kmovw %esi, %k6
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm6, %zmm19 {%k6}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm21, %zmm26 {%k1}
+; AVX512F-ONLY-FAST-NEXT:    movb $-31, %sil
+; AVX512F-ONLY-FAST-NEXT:    kmovw %esi, %k1
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm29, %zmm26 {%k1}
+; AVX512F-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm17[0,1,2,3],zmm28[4,5,6,7]
+; AVX512F-ONLY-FAST-NEXT:    movb $-61, %sil
+; AVX512F-ONLY-FAST-NEXT:    kmovw %esi, %k1
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm11, %zmm0 {%k1}
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm9, %zmm23, %zmm7
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm5, %zmm25, %zmm4
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm7, %zmm4 {%k2}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm5 = <u,1,2,3,4,15,u,u>
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm30, %zmm4, %zmm5
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm4 = <u,1,2,3,4,5,15,u>
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm10, %zmm5, %zmm4
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm5 = <13,u,2,3,4,5,6,14>
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm10, %zmm18, %zmm5
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [14,1,2,3,4,5,6,15]
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm31, %zmm4, %zmm6
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [0,13,2,3,4,5,6,7]
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm31, %zmm5, %zmm4
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm30, %zmm14 {%k2}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm14, %zmm13 {%k3}
+; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm5 = ymm15[0,1,2,3,4,5],mem[6,7]
+; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm5, %zmm0, %zmm20 {%k4}
+; AVX512F-ONLY-FAST-NEXT:    vpbroadcastq 8(%rcx), %ymm5
+; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm5 = mem[0,1,2,3],ymm5[4,5,6,7]
+; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm5, %zmm0, %zmm2 {%k5}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm4, %zmm2 {%k6}
 ; AVX512F-ONLY-FAST-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, 64(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, 128(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm26, 192(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm16, 256(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm20, 320(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm22, 384(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm12, 512(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm17, 576(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm21, 640(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm13, 704(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, 448(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, (%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm4, 768(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm6, 832(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, 64(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, 128(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm20, 192(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm13, 256(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, 320(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm26, 384(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm19, 512(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, 576(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm16, 640(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm18, 704(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm10, 448(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm9, (%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm8, 768(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovaps %zmm0, 832(%rax)
 ; AVX512F-ONLY-FAST-NEXT:    vzeroupper
 ; AVX512F-ONLY-FAST-NEXT:    retq
 ;
 ; AVX512DQ-SLOW-LABEL: store_i64_stride7_vf16:
 ; AVX512DQ-SLOW:       # %bb.0:
 ; AVX512DQ-SLOW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 (%rdi), %zmm4
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 64(%rdi), %zmm6
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 (%rsi), %zmm13
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 64(%rsi), %zmm7
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 (%rdx), %zmm8
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 64(%rdx), %zmm9
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 (%rcx), %zmm2
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 64(%rcx), %zmm3
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 64(%r8), %zmm11
-; AVX512DQ-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [6,14,6,14,6,14,6,14]
-; AVX512DQ-SLOW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm9, %zmm12
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm3, %zmm0, %zmm12
-; AVX512DQ-SLOW-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [6,14,6,14]
-; AVX512DQ-SLOW-NEXT:    # ymm1 = mem[0,1,0,1]
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm0
-; AVX512DQ-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} ymm16 = [15,7,15,7]
-; AVX512DQ-SLOW-NEXT:    # ymm16 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm10
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm13, %zmm16, %zmm10
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm7, %zmm6, %zmm16
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm14
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm7, %zmm1, %zmm14
-; AVX512DQ-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm23 = zmm14[0,1,2,3],zmm12[4,5,6,7]
-; AVX512DQ-SLOW-NEXT:    movb $64, %sil
-; AVX512DQ-SLOW-NEXT:    kmovw %esi, %k1
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm23 {%k1}
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 (%r8), %zmm14
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 64(%r9), %zmm24
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 (%r9), %zmm25
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 (%rax), %zmm22
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 64(%rax), %zmm28
-; AVX512DQ-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm17 = [0,8,0,8,0,8,0,8]
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 (%rdi), %zmm6
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 64(%rdi), %zmm18
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 (%rsi), %zmm7
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 64(%rsi), %zmm12
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 (%rdx), %zmm13
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 64(%rdx), %zmm20
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 (%rcx), %zmm1
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 64(%rcx), %zmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 64(%r8), %zmm16
+; AVX512DQ-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm17 = [6,14,6,14,6,14,6,14]
 ; AVX512DQ-SLOW-NEXT:    # zmm17 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm21
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm25, %zmm17, %zmm21
-; AVX512DQ-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm15 = [13,5,13,5,13,5,13,5]
-; AVX512DQ-SLOW-NEXT:    # zmm15 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQ-SLOW-NEXT:    movb $96, %sil
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm20, %zmm19
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm0, %zmm17, %zmm19
+; AVX512DQ-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm9 = [15,7,15,7,15,7,15,7]
+; AVX512DQ-SLOW-NEXT:    # zmm9 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm20, %zmm21
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm0, %zmm9, %zmm21
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm18, %zmm22
+; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm5 = [0,8,0,1,0,8,0,1]
+; AVX512DQ-SLOW-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm4
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm7, %zmm5, %zmm4
+; AVX512DQ-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm10 = [13,5,13,5,13,5,13,5]
+; AVX512DQ-SLOW-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm20, %zmm2
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm0, %zmm10, %zmm2
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm18, %zmm2
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm18, %zmm23
+; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm11 = [2,10,0,3,2,10,0,3]
+; AVX512DQ-SLOW-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm20, %zmm8
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm0, %zmm11, %zmm8
+; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm15 = [1,0,10,2,1,0,10,2]
+; AVX512DQ-SLOW-NEXT:    # zmm15 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm12, %zmm14
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm18, %zmm15, %zmm14
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm12, %zmm18, %zmm5
+; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm29 = [3,0,12,4,3,0,12,4]
+; AVX512DQ-SLOW-NEXT:    # zmm29 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm20, %zmm29, %zmm0
+; AVX512DQ-SLOW-NEXT:    movb $48, %sil
+; AVX512DQ-SLOW-NEXT:    kmovw %esi, %k3
+; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm0 {%k3} = zmm18[0],zmm12[0],zmm18[2],zmm12[2],zmm18[4],zmm12[4],zmm18[6],zmm12[6]
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm12, %zmm17, %zmm18
+; AVX512DQ-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm19 = zmm18[0,1,2,3],zmm19[4,5,6,7]
+; AVX512DQ-SLOW-NEXT:    movb $64, %sil
 ; AVX512DQ-SLOW-NEXT:    kmovw %esi, %k1
-; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm12 = [4,12,0,5,4,12,0,5]
-; AVX512DQ-SLOW-NEXT:    # zmm12 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm20
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm24, %zmm11, %zmm17
-; AVX512DQ-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm18 = [15,7,15,7,15,7,15,7]
-; AVX512DQ-SLOW-NEXT:    # zmm18 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm26
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm25, %zmm18, %zmm26
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm19
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm2, %zmm18, %zmm19
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm16, %zmm19 {%k1}
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 (%r8), %zmm20
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 (%r9), %zmm28
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 64(%r9), %zmm30
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 64(%rax), %zmm27
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm12, %zmm9, %zmm22
 ; AVX512DQ-SLOW-NEXT:    movb $24, %sil
-; AVX512DQ-SLOW-NEXT:    kmovw %esi, %k2
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm3, %zmm9, %zmm18
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm18, %zmm16 {%k2}
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm27 = <u,1,2,3,4,15,u,u>
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm11, %zmm16, %zmm27
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm24, %zmm12, %zmm11
-; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm30 = [0,1,12,7,0,1,12,7]
-; AVX512DQ-SLOW-NEXT:    # zmm30 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm28, %zmm30, %zmm11
-; AVX512DQ-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm31 = [11,3,11,3,11,3,11,3]
+; AVX512DQ-SLOW-NEXT:    kmovw %esi, %k1
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm21, %zmm22 {%k1}
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm18 = <u,1,2,3,4,15,u,u>
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm16, %zmm22, %zmm18
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm21 = <u,1,2,3,4,5,15,u>
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm30, %zmm18, %zmm21
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm18 = [14,1,2,3,4,5,6,15]
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm27, %zmm21, %zmm18
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm21 = <13,u,2,3,4,5,6,14>
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm30, %zmm19, %zmm21
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm19 = [0,13,2,3,4,5,6,7]
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm27, %zmm21, %zmm19
+; AVX512DQ-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm31 = [0,8,0,8,0,8,0,8]
 ; AVX512DQ-SLOW-NEXT:    # zmm31 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm7, %zmm31, %zmm0
-; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm18 = [2,10,0,3,2,10,0,3]
-; AVX512DQ-SLOW-NEXT:    # zmm18 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm9, %zmm16
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm3, %zmm18, %zmm16
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm16 {%k1}
-; AVX512DQ-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm29 = [9,1,9,1,9,1,9,1]
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm20, %zmm26
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm28, %zmm31, %zmm26
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm12, %zmm10, %zmm2
+; AVX512DQ-SLOW-NEXT:    movb $96, %sil
+; AVX512DQ-SLOW-NEXT:    kmovw %esi, %k2
+; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm21 = [4,12,0,5,4,12,0,5]
+; AVX512DQ-SLOW-NEXT:    # zmm21 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm13, %zmm25
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm1, %zmm9, %zmm25
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm13, %zmm24
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm1, %zmm17, %zmm24
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm13, %zmm22
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm1, %zmm10, %zmm22
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm1, %zmm13, %zmm11
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm13, %zmm29, %zmm1
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm16, %zmm13
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm30, %zmm21, %zmm13
+; AVX512DQ-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm29 = [11,3,11,3,11,3,11,3]
 ; AVX512DQ-SLOW-NEXT:    # zmm29 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [5,0,14,6,5,0,14,6]
-; AVX512DQ-SLOW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm14, %zmm25, %zmm0
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm25, %zmm14, %zmm12
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm25, %zmm29, %zmm14
-; AVX512DQ-SLOW-NEXT:    vmovdqa 64(%r9), %ymm5
-; AVX512DQ-SLOW-NEXT:    vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm24, %zmm29, %zmm20
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm25 = <u,1,2,3,4,5,15,u>
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm24, %zmm27, %zmm25
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm1 = <13,u,2,3,4,5,6,14>
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm24, %zmm23, %zmm1
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 64(%r8), %ymm23
-; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm24 = ymm23[0],ymm5[0],ymm23[2],ymm5[2]
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm12, %zmm29, %zmm23
+; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm12 = [0,1,12,7,0,1,12,7]
+; AVX512DQ-SLOW-NEXT:    # zmm12 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm27, %zmm12, %zmm13
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm23, %zmm8 {%k2}
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm16, %zmm3
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm30, %zmm31, %zmm16
+; AVX512DQ-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm31 = [9,1,9,1,9,1,9,1]
+; AVX512DQ-SLOW-NEXT:    # zmm31 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm30, %zmm31, %zmm3
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 64(%r9), %ymm23
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm7, %zmm6, %zmm17
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm7, %zmm6, %zmm10
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm7, %zmm6, %zmm29
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm6, %zmm7, %zmm15
+; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm1 {%k3} = zmm6[0],zmm7[0],zmm6[2],zmm7[2],zmm6[4],zmm7[4],zmm6[6],zmm7[6]
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm7, %zmm9, %zmm6
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm28, %zmm20, %zmm9
+; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm7 = [5,0,14,6,5,0,14,6]
+; AVX512DQ-SLOW-NEXT:    # zmm7 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm20, %zmm28, %zmm7
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm28, %zmm20, %zmm21
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm28, %zmm31, %zmm20
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 64(%r8), %ymm28
+; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm30 = ymm28[0],ymm23[0],ymm28[2],ymm23[2]
 ; AVX512DQ-SLOW-NEXT:    movb $28, %sil
 ; AVX512DQ-SLOW-NEXT:    kmovw %esi, %k3
-; AVX512DQ-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm16 {%k3} = zmm24[2,3,2,3],zmm28[2,3,2,3]
-; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm29 = [6,13,14,7,6,13,14,7]
-; AVX512DQ-SLOW-NEXT:    # zmm29 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm26, %zmm22, %zmm29
-; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm27 = [0,13,6,7,0,13,6,7]
-; AVX512DQ-SLOW-NEXT:    # zmm27 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm22, %zmm0, %zmm27
-; AVX512DQ-SLOW-NEXT:    vmovdqa (%r9), %ymm5
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm22, %zmm30, %zmm12
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 (%r8), %ymm26
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm13, %zmm4, %zmm31
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm2, %zmm8, %zmm18
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm31, %zmm18 {%k1}
-; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm24 = ymm26[0],ymm5[0],ymm26[2],ymm5[2]
-; AVX512DQ-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm18 {%k3} = zmm24[2,3,2,3],zmm22[2,3,2,3]
-; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm24 = [4,9,0,3,4,9,0,3]
-; AVX512DQ-SLOW-NEXT:    # zmm24 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm22, %zmm24, %zmm14
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm9, %zmm0
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm3, %zmm15, %zmm0
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm28, %zmm24, %zmm20
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm22 = [14,1,2,3,4,5,6,15]
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm28, %zmm25, %zmm22
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm24 = [0,13,2,3,4,5,6,7]
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm28, %zmm1, %zmm24
-; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm28 = [1,0,10,2,1,0,10,2]
-; AVX512DQ-SLOW-NEXT:    # zmm28 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm25 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm2, %zmm8, %zmm25
-; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm1 = [3,0,12,4,3,0,12,4]
-; AVX512DQ-SLOW-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm9, %zmm1, %zmm3
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm31
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm2, %zmm15, %zmm31
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm8, %zmm1, %zmm2
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm9
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm6, %zmm28, %zmm9
-; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm30 = [0,8,0,1,0,8,0,1]
+; AVX512DQ-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm8 {%k3} = zmm30[2,3,2,3],zmm27[2,3,2,3]
+; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm30 = [4,9,0,3,4,9,0,3]
 ; AVX512DQ-SLOW-NEXT:    # zmm30 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm8
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm13, %zmm30, %zmm8
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm7, %zmm6, %zmm30
-; AVX512DQ-SLOW-NEXT:    movb $48, %sil
-; AVX512DQ-SLOW-NEXT:    kmovw %esi, %k3
-; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm3 {%k3} = zmm6[0],zmm7[0],zmm6[2],zmm7[2],zmm6[4],zmm7[4],zmm6[6],zmm7[6]
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm7, %zmm15, %zmm6
-; AVX512DQ-SLOW-NEXT:    vbroadcasti128 {{.*#+}} ymm7 = [6,14,6,14]
-; AVX512DQ-SLOW-NEXT:    # ymm7 = mem[0,1,0,1]
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm13, %zmm4, %zmm7
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm13, %zmm4, %zmm15
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm4, %zmm13, %zmm28
-; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm2 {%k3} = zmm4[0],zmm13[0],zmm4[2],zmm13[2],zmm4[4],zmm13[4],zmm4[6],zmm13[6]
-; AVX512DQ-SLOW-NEXT:    vmovdqa (%rdx), %xmm1
-; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
-; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm27, %zmm30, %zmm3
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 (%rax), %zmm27
+; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm31 = [6,13,14,7,6,13,14,7]
+; AVX512DQ-SLOW-NEXT:    # zmm31 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm9, %zmm27, %zmm31
+; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm9 = [0,13,6,7,0,13,6,7]
+; AVX512DQ-SLOW-NEXT:    # zmm9 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm27, %zmm7, %zmm9
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm27, %zmm12, %zmm21
+; AVX512DQ-SLOW-NEXT:    vmovdqa (%r9), %ymm7
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm29, %zmm11 {%k2}
+; AVX512DQ-SLOW-NEXT:    vmovdqa (%r8), %ymm12
+; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm29 = ymm12[0],ymm7[0],ymm12[2],ymm7[2]
+; AVX512DQ-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm11 {%k3} = zmm29[2,3,2,3],zmm27[2,3,2,3]
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm27, %zmm30, %zmm20
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 (%rdx), %xmm27
+; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm27 = xmm27[0],mem[0]
+; AVX512DQ-SLOW-NEXT:    vinserti32x4 $1, %xmm27, %ymm0, %ymm27
 ; AVX512DQ-SLOW-NEXT:    movb $12, %sil
 ; AVX512DQ-SLOW-NEXT:    kmovw %esi, %k5
-; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm8 {%k5}
+; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm27, %zmm0, %zmm4 {%k5}
 ; AVX512DQ-SLOW-NEXT:    movb $112, %sil
 ; AVX512DQ-SLOW-NEXT:    kmovw %esi, %k7
-; AVX512DQ-SLOW-NEXT:    vinserti64x2 $3, (%rax), %zmm21, %zmm8 {%k7}
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm6 {%k1}
+; AVX512DQ-SLOW-NEXT:    vinserti64x2 $3, (%rax), %zmm26, %zmm4 {%k7}
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm26 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm26, %zmm2 {%k2}
 ; AVX512DQ-SLOW-NEXT:    movb $120, %sil
 ; AVX512DQ-SLOW-NEXT:    kmovw %esi, %k3
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm11 {%k3}
-; AVX512DQ-SLOW-NEXT:    vpbroadcastq 72(%rcx), %ymm0
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm13 {%k3}
+; AVX512DQ-SLOW-NEXT:    vpbroadcastq 72(%rcx), %ymm2
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = mem[0,1,2,3],ymm2[4,5,6,7]
 ; AVX512DQ-SLOW-NEXT:    movb $6, %sil
 ; AVX512DQ-SLOW-NEXT:    kmovw %esi, %k4
-; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm9 {%k4}
+; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm2, %zmm0, %zmm14 {%k4}
 ; AVX512DQ-SLOW-NEXT:    movb $56, %sil
 ; AVX512DQ-SLOW-NEXT:    kmovw %esi, %k6
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm20, %zmm9 {%k6}
-; AVX512DQ-SLOW-NEXT:    vmovdqa 64(%rdx), %xmm0
-; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
-; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm30 {%k5}
-; AVX512DQ-SLOW-NEXT:    vinserti64x2 $3, 64(%rax), %zmm17, %zmm30 {%k7}
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm19, %zmm10 {%k2}
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm14 {%k6}
+; AVX512DQ-SLOW-NEXT:    vmovdqa 64(%rdx), %xmm2
+; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],mem[0]
+; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm2, %zmm0, %zmm5 {%k5}
+; AVX512DQ-SLOW-NEXT:    vinserti64x2 $3, 64(%rax), %zmm16, %zmm5 {%k7}
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm25, %zmm6 {%k1}
 ; AVX512DQ-SLOW-NEXT:    movb $-31, %sil
-; AVX512DQ-SLOW-NEXT:    kmovw %esi, %k2
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm29, %zmm10 {%k2}
-; AVX512DQ-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm7[0,1,2,3],zmm25[4,5,6,7]
+; AVX512DQ-SLOW-NEXT:    kmovw %esi, %k1
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm31, %zmm6 {%k1}
+; AVX512DQ-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm17[0,1,2,3],zmm24[4,5,6,7]
 ; AVX512DQ-SLOW-NEXT:    movb $-61, %sil
-; AVX512DQ-SLOW-NEXT:    kmovw %esi, %k2
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm27, %zmm0 {%k2}
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm31, %zmm15 {%k1}
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm15, %zmm12 {%k3}
-; AVX512DQ-SLOW-NEXT:    vpbroadcastq 8(%rcx), %ymm1
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = mem[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm28 {%k4}
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm28 {%k6}
-; AVX512DQ-SLOW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm23, %ymm1 # 32-byte Folded Reload
-; AVX512DQ-SLOW-NEXT:    # ymm1 = ymm23[1],mem[1],ymm23[3],mem[3]
-; AVX512DQ-SLOW-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,3,3]
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],mem[6,7]
+; AVX512DQ-SLOW-NEXT:    kmovw %esi, %k1
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm9, %zmm2 {%k1}
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm22, %zmm10 {%k2}
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm10, %zmm21 {%k3}
+; AVX512DQ-SLOW-NEXT:    vpbroadcastq 8(%rcx), %ymm3
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm3 = mem[0,1,2,3],ymm3[4,5,6,7]
+; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm15 {%k4}
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm20, %zmm15 {%k6}
+; AVX512DQ-SLOW-NEXT:    vpunpckhqdq {{.*#+}} ymm3 = ymm28[1],ymm23[1],ymm28[3],ymm23[3]
+; AVX512DQ-SLOW-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,2,3,3]
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5],mem[6,7]
 ; AVX512DQ-SLOW-NEXT:    movb $14, %cl
 ; AVX512DQ-SLOW-NEXT:    kmovw %ecx, %k1
-; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm3 {%k1}
-; AVX512DQ-SLOW-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm26[1],ymm5[1],ymm26[3],ymm5[3]
-; AVX512DQ-SLOW-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,3,3]
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],mem[6,7]
-; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm2 {%k1}
+; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm0 {%k1}
+; AVX512DQ-SLOW-NEXT:    vpunpckhqdq {{.*#+}} ymm3 = ymm12[1],ymm7[1],ymm12[3],ymm7[3]
+; AVX512DQ-SLOW-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,2,3,3]
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5],mem[6,7]
+; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm1 {%k1}
 ; AVX512DQ-SLOW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm28, 64(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm18, 128(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm2, 192(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm12, 256(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, 320(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm10, 384(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm30, 448(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm9, 512(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm16, 576(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm3, 640(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm11, 704(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm8, (%rax)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm24, 768(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm22, 832(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm15, 64(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm11, 128(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, 192(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm21, 256(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm2, 320(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm6, 384(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm5, 448(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm14, 512(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm8, 576(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, 640(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm13, 704(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm4, (%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm19, 768(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm18, 832(%rax)
 ; AVX512DQ-SLOW-NEXT:    vzeroupper
 ; AVX512DQ-SLOW-NEXT:    retq
 ;
 ; AVX512DQ-FAST-LABEL: store_i64_stride7_vf16:
 ; AVX512DQ-FAST:       # %bb.0:
 ; AVX512DQ-FAST-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-FAST-NEXT:    vmovdqa64 (%rdi), %zmm13
-; AVX512DQ-FAST-NEXT:    vmovdqa64 64(%rdi), %zmm3
-; AVX512DQ-FAST-NEXT:    vmovdqa64 (%rsi), %zmm0
-; AVX512DQ-FAST-NEXT:    vmovdqa64 64(%rsi), %zmm5
-; AVX512DQ-FAST-NEXT:    vmovdqa64 (%rdx), %zmm1
-; AVX512DQ-FAST-NEXT:    vmovdqa64 64(%rdx), %zmm7
-; AVX512DQ-FAST-NEXT:    vmovdqa64 (%rcx), %zmm16
-; AVX512DQ-FAST-NEXT:    vmovdqa64 64(%rcx), %zmm9
-; AVX512DQ-FAST-NEXT:    vmovdqa64 64(%r8), %zmm30
-; AVX512DQ-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm20 = [6,14,6,14,6,14,6,14]
+; AVX512DQ-FAST-NEXT:    vmovdqa64 (%rdi), %zmm10
+; AVX512DQ-FAST-NEXT:    vmovdqa64 64(%rdi), %zmm0
+; AVX512DQ-FAST-NEXT:    vmovdqa64 (%rsi), %zmm2
+; AVX512DQ-FAST-NEXT:    vmovdqa64 64(%rsi), %zmm13
+; AVX512DQ-FAST-NEXT:    vmovdqa64 (%rdx), %zmm3
+; AVX512DQ-FAST-NEXT:    vmovdqa64 64(%rdx), %zmm1
+; AVX512DQ-FAST-NEXT:    vmovdqa64 (%rcx), %zmm14
+; AVX512DQ-FAST-NEXT:    vmovdqa64 64(%rcx), %zmm15
+; AVX512DQ-FAST-NEXT:    vmovdqa64 64(%r8), %zmm21
+; AVX512DQ-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm18 = [6,14,6,14,6,14,6,14]
+; AVX512DQ-FAST-NEXT:    # zmm18 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm1, %zmm4
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm15, %zmm18, %zmm4
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm5
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm13, %zmm18, %zmm5
+; AVX512DQ-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm4 = zmm5[0,1,2,3],zmm4[4,5,6,7]
+; AVX512DQ-FAST-NEXT:    movb $64, %sil
+; AVX512DQ-FAST-NEXT:    kmovw %esi, %k1
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm21, %zmm4 {%k1}
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 (%r8), %zmm7
+; AVX512DQ-FAST-NEXT:    vmovdqa64 (%r9), %zmm12
+; AVX512DQ-FAST-NEXT:    vmovdqa64 64(%r9), %zmm6
+; AVX512DQ-FAST-NEXT:    vmovdqa64 (%rax), %zmm9
+; AVX512DQ-FAST-NEXT:    vmovdqa64 64(%rax), %zmm5
+; AVX512DQ-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm20 = [15,7,15,7,15,7,15,7]
 ; AVX512DQ-FAST-NEXT:    # zmm20 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm7, %zmm17
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm9, %zmm20, %zmm17
-; AVX512DQ-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} ymm21 = [6,14,6,14]
-; AVX512DQ-FAST-NEXT:    # ymm21 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm3, %zmm18
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm5, %zmm21, %zmm18
-; AVX512DQ-FAST-NEXT:    vmovdqa64 (%r8), %zmm11
-; AVX512DQ-FAST-NEXT:    vmovdqa64 64(%r9), %zmm10
-; AVX512DQ-FAST-NEXT:    vmovdqa64 (%r9), %zmm15
-; AVX512DQ-FAST-NEXT:    vmovdqa64 (%rax), %zmm14
-; AVX512DQ-FAST-NEXT:    vmovdqa64 64(%rax), %zmm31
-; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [0,8,0,1,0,8,0,1]
-; AVX512DQ-FAST-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm13, %zmm2
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm0, %zmm4, %zmm2
-; AVX512DQ-FAST-NEXT:    vmovdqa (%rdx), %xmm12
-; AVX512DQ-FAST-NEXT:    vmovdqa64 64(%rdx), %xmm22
-; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm12 = xmm12[0],mem[0]
-; AVX512DQ-FAST-NEXT:    vinserti128 $1, %xmm12, %ymm0, %ymm12
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm1, %zmm8
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm15, %zmm20, %zmm8
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm16
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm13, %zmm20, %zmm16
+; AVX512DQ-FAST-NEXT:    movb $24, %sil
+; AVX512DQ-FAST-NEXT:    kmovw %esi, %k1
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm8, %zmm16 {%k1}
+; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm31 = <u,1,2,3,4,15,u,u>
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm21, %zmm16, %zmm31
+; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm22 = [0,8,0,1,0,8,0,1]
+; AVX512DQ-FAST-NEXT:    # zmm22 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm10, %zmm8
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm2, %zmm22, %zmm8
+; AVX512DQ-FAST-NEXT:    vmovdqa64 (%rdx), %xmm16
+; AVX512DQ-FAST-NEXT:    vmovdqa64 64(%rdx), %xmm27
+; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm16 = xmm16[0],mem[0]
+; AVX512DQ-FAST-NEXT:    vinserti32x4 $1, %xmm16, %ymm0, %ymm16
 ; AVX512DQ-FAST-NEXT:    movb $12, %sil
-; AVX512DQ-FAST-NEXT:    kmovw %esi, %k2
-; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm12, %zmm0, %zmm2 {%k2}
-; AVX512DQ-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm24 = [0,8,0,8,0,8,0,8]
-; AVX512DQ-FAST-NEXT:    # zmm24 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm11, %zmm12
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm15, %zmm24, %zmm12
+; AVX512DQ-FAST-NEXT:    kmovw %esi, %k3
+; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm16, %zmm0, %zmm8 {%k3}
+; AVX512DQ-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm26 = [0,8,0,8,0,8,0,8]
+; AVX512DQ-FAST-NEXT:    # zmm26 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm7, %zmm28
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm12, %zmm26, %zmm28
 ; AVX512DQ-FAST-NEXT:    movb $112, %sil
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm24
+; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm17 = [3,0,12,4,3,0,12,4]
+; AVX512DQ-FAST-NEXT:    # zmm17 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm15, %zmm16
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm1, %zmm17, %zmm16
+; AVX512DQ-FAST-NEXT:    movb $48, %dil
+; AVX512DQ-FAST-NEXT:    kmovw %edi, %k2
+; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm16 {%k2} = zmm0[0],zmm13[0],zmm0[2],zmm13[2],zmm0[4],zmm13[4],zmm0[6],zmm13[6]
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm23
+; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm25 = [1,0,10,2,1,0,10,2]
+; AVX512DQ-FAST-NEXT:    # zmm25 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm13, %zmm19
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm0, %zmm25, %zmm19
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm13, %zmm22, %zmm0
+; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm22 = xmm27[0],mem[0]
+; AVX512DQ-FAST-NEXT:    vinserti32x4 $1, %xmm22, %ymm0, %ymm22
+; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm22, %zmm0, %zmm0 {%k3}
 ; AVX512DQ-FAST-NEXT:    kmovw %esi, %k3
-; AVX512DQ-FAST-NEXT:    vinserti64x2 $3, (%rax), %zmm12, %zmm2 {%k3}
-; AVX512DQ-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm23 = [13,5,13,5,13,5,13,5]
-; AVX512DQ-FAST-NEXT:    # zmm23 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm7, %zmm12
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm9, %zmm23, %zmm12
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm3, %zmm25
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm5, %zmm23, %zmm25
+; AVX512DQ-FAST-NEXT:    vinserti64x2 $3, (%rax), %zmm28, %zmm8 {%k3}
+; AVX512DQ-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm28 = [13,5,13,5,13,5,13,5]
+; AVX512DQ-FAST-NEXT:    # zmm28 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm1, %zmm29
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm15, %zmm28, %zmm29
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm13, %zmm28, %zmm24
 ; AVX512DQ-FAST-NEXT:    movb $96, %sil
-; AVX512DQ-FAST-NEXT:    kmovw %esi, %k1
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm12, %zmm25 {%k1}
-; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm12 = [4,12,0,5,4,12,0,5]
-; AVX512DQ-FAST-NEXT:    # zmm12 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm10, %zmm30, %zmm24
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm30, %zmm19
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm10, %zmm12, %zmm19
-; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm27 = [0,1,12,7,0,1,12,7]
-; AVX512DQ-FAST-NEXT:    # zmm27 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm31, %zmm27, %zmm19
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm21, %zmm22
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm21, %zmm27
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm6, %zmm26, %zmm21
+; AVX512DQ-FAST-NEXT:    vinserti64x2 $3, 64(%rax), %zmm21, %zmm0 {%k3}
+; AVX512DQ-FAST-NEXT:    kmovw %esi, %k3
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm29, %zmm24 {%k3}
+; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm21 = [4,12,0,5,4,12,0,5]
+; AVX512DQ-FAST-NEXT:    # zmm21 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm6, %zmm21, %zmm22
+; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm29 = [0,1,12,7,0,1,12,7]
+; AVX512DQ-FAST-NEXT:    # zmm29 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm5, %zmm29, %zmm22
 ; AVX512DQ-FAST-NEXT:    movb $120, %sil
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm5, %zmm3, %zmm4
-; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm22 = xmm22[0],mem[0]
-; AVX512DQ-FAST-NEXT:    vinserti32x4 $1, %xmm22, %ymm0, %ymm22
-; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm22, %zmm0, %zmm4 {%k2}
-; AVX512DQ-FAST-NEXT:    vinserti64x2 $3, 64(%rax), %zmm24, %zmm4 {%k3}
-; AVX512DQ-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm24 = [15,7,15,7,15,7,15,7]
-; AVX512DQ-FAST-NEXT:    # zmm24 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm11, %zmm22
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm15, %zmm24, %zmm22
-; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm28 = [6,13,14,7,6,13,14,7]
-; AVX512DQ-FAST-NEXT:    # zmm28 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm22, %zmm14, %zmm28
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm1, %zmm29
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm16, %zmm24, %zmm29
-; AVX512DQ-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} ymm26 = [15,7,15,7]
-; AVX512DQ-FAST-NEXT:    # ymm26 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm13, %zmm22
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm0, %zmm26, %zmm22
-; AVX512DQ-FAST-NEXT:    movb $24, %dil
-; AVX512DQ-FAST-NEXT:    kmovw %edi, %k2
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm29, %zmm22 {%k2}
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm3, %zmm30
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm14, %zmm20, %zmm30
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm10, %zmm26
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm2, %zmm20, %zmm26
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm30, %zmm26 {%k1}
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm12, %zmm7, %zmm20
+; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm30 = [6,13,14,7,6,13,14,7]
+; AVX512DQ-FAST-NEXT:    # zmm30 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm20, %zmm9, %zmm30
 ; AVX512DQ-FAST-NEXT:    movb $-31, %dil
-; AVX512DQ-FAST-NEXT:    kmovw %edi, %k3
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm28, %zmm22 {%k3}
-; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm28 = [5,0,14,6,5,0,14,6]
-; AVX512DQ-FAST-NEXT:    # zmm28 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm11, %zmm15, %zmm28
-; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm29 = [0,13,6,7,0,13,6,7]
-; AVX512DQ-FAST-NEXT:    # zmm29 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm14, %zmm28, %zmm29
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm16, %zmm1, %zmm20
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm0, %zmm13, %zmm21
-; AVX512DQ-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm20 = zmm21[0,1,2,3],zmm20[4,5,6,7]
+; AVX512DQ-FAST-NEXT:    kmovw %edi, %k1
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm30, %zmm26 {%k1}
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm3, %zmm20
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm14, %zmm18, %zmm20
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm2, %zmm10, %zmm18
+; AVX512DQ-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm18 = zmm18[0,1,2,3],zmm20[4,5,6,7]
+; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm20 = [5,0,14,6,5,0,14,6]
+; AVX512DQ-FAST-NEXT:    # zmm20 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm7, %zmm12, %zmm20
+; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm30 = [0,13,6,7,0,13,6,7]
+; AVX512DQ-FAST-NEXT:    # zmm30 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm9, %zmm20, %zmm30
 ; AVX512DQ-FAST-NEXT:    movb $-61, %dil
-; AVX512DQ-FAST-NEXT:    kmovw %edi, %k3
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm29, %zmm20 {%k3}
-; AVX512DQ-FAST-NEXT:    kmovw %esi, %k3
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm25, %zmm19 {%k3}
-; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm25 = [3,0,12,4,3,0,12,4]
-; AVX512DQ-FAST-NEXT:    # zmm25 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm9, %zmm21
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm7, %zmm25, %zmm21
-; AVX512DQ-FAST-NEXT:    movb $48, %sil
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm1, %zmm28
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm16, %zmm23, %zmm28
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm0, %zmm13, %zmm23
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm28, %zmm23 {%k1}
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm15, %zmm11, %zmm12
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm14, %zmm27, %zmm12
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm23, %zmm12 {%k3}
-; AVX512DQ-FAST-NEXT:    kmovw %esi, %k3
-; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm21 {%k3} = zmm3[0],zmm5[0],zmm3[2],zmm5[2],zmm3[4],zmm5[4],zmm3[6],zmm5[6]
-; AVX512DQ-FAST-NEXT:    vmovdqa64 (%r9), %ymm23
-; AVX512DQ-FAST-NEXT:    vmovdqa64 64(%r9), %ymm27
+; AVX512DQ-FAST-NEXT:    kmovw %edi, %k1
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm30, %zmm18 {%k1}
+; AVX512DQ-FAST-NEXT:    kmovw %esi, %k1
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm24, %zmm22 {%k1}
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm3, %zmm20
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm14, %zmm28, %zmm20
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm2, %zmm10, %zmm28
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm20, %zmm28 {%k3}
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm12, %zmm7, %zmm21
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm9, %zmm29, %zmm21
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm28, %zmm21 {%k1}
+; AVX512DQ-FAST-NEXT:    vmovdqa64 (%r9), %ymm20
+; AVX512DQ-FAST-NEXT:    vmovdqa64 64(%r9), %ymm24
 ; AVX512DQ-FAST-NEXT:    vmovdqa64 64(%r8), %ymm28
-; AVX512DQ-FAST-NEXT:    vmovdqa {{.*#+}} ymm6 = [1,3,7,7]
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %ymm28, %ymm8
-; AVX512DQ-FAST-NEXT:    vpermt2q %ymm27, %ymm6, %ymm8
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],mem[6,7]
+; AVX512DQ-FAST-NEXT:    vmovdqa {{.*#+}} ymm4 = [1,3,7,7]
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %ymm28, %ymm11
+; AVX512DQ-FAST-NEXT:    vpermt2q %ymm24, %ymm4, %ymm11
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm11 = ymm11[0,1,2,3,4,5],mem[6,7]
 ; AVX512DQ-FAST-NEXT:    movb $14, %sil
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm1, %zmm16, %zmm25
-; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm25 {%k3} = zmm13[0],zmm0[0],zmm13[2],zmm0[2],zmm13[4],zmm0[4],zmm13[6],zmm0[6]
-; AVX512DQ-FAST-NEXT:    kmovw %esi, %k3
-; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm8, %zmm0, %zmm21 {%k3}
-; AVX512DQ-FAST-NEXT:    vmovdqa (%r8), %ymm8
-; AVX512DQ-FAST-NEXT:    vpermi2q %ymm23, %ymm8, %ymm6
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3,4,5],mem[6,7]
-; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm6, %zmm0, %zmm25 {%k3}
-; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm6 = ymm28[0],ymm27[0],ymm28[2],ymm27[2]
-; AVX512DQ-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm17 = zmm18[0,1,2,3],zmm17[4,5,6,7]
-; AVX512DQ-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm18 = [11,3,11,3,11,3,11,3]
-; AVX512DQ-FAST-NEXT:    # zmm18 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm27 = [2,10,0,3,2,10,0,3]
-; AVX512DQ-FAST-NEXT:    # zmm27 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm16, %zmm27, %zmm1
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm7, %zmm16
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm9, %zmm27, %zmm16
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm3, %zmm27
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm5, %zmm18, %zmm27
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm27, %zmm16 {%k1}
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm0, %zmm13, %zmm18
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm18, %zmm1 {%k1}
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm3, %zmm14, %zmm17
+; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm17 {%k2} = zmm10[0],zmm2[0],zmm10[2],zmm2[2],zmm10[4],zmm2[4],zmm10[6],zmm2[6]
+; AVX512DQ-FAST-NEXT:    kmovw %esi, %k1
+; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm11, %zmm0, %zmm16 {%k1}
+; AVX512DQ-FAST-NEXT:    vmovdqa (%r8), %ymm11
+; AVX512DQ-FAST-NEXT:    vpermi2q %ymm20, %ymm11, %ymm4
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3,4,5],mem[6,7]
+; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm4, %zmm0, %zmm17 {%k1}
+; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [2,10,0,3,2,10,0,3]
+; AVX512DQ-FAST-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm15, %zmm4, %zmm1
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm14, %zmm4, %zmm3
+; AVX512DQ-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm4 = [11,3,11,3,11,3,11,3]
+; AVX512DQ-FAST-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm13, %zmm4, %zmm23
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm2, %zmm10, %zmm4
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm10, %zmm25, %zmm2
+; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm10 = ymm28[0],ymm24[0],ymm28[2],ymm24[2]
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm23, %zmm1 {%k3}
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm4, %zmm3 {%k3}
 ; AVX512DQ-FAST-NEXT:    movb $28, %al
 ; AVX512DQ-FAST-NEXT:    kmovw %eax, %k1
-; AVX512DQ-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm16 {%k1} = zmm6[2,3,2,3],zmm31[2,3,2,3]
-; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm6 = [1,0,10,2,1,0,10,2]
-; AVX512DQ-FAST-NEXT:    # zmm6 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm13, %zmm6, %zmm0
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm5, %zmm13
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm3, %zmm6, %zmm13
-; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm6 = ymm8[0],ymm23[0],ymm8[2],ymm23[2]
-; AVX512DQ-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm1 {%k1} = zmm6[2,3,2,3],zmm14[2,3,2,3]
-; AVX512DQ-FAST-NEXT:    vpbroadcastq 72(%rcx), %ymm6
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm6 = mem[0,1,2,3],ymm6[4,5,6,7]
+; AVX512DQ-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm1 {%k1} = zmm10[2,3,2,3],zmm5[2,3,2,3]
+; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm4 = ymm11[0],ymm20[0],ymm11[2],ymm20[2]
+; AVX512DQ-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm3 {%k1} = zmm4[2,3,2,3],zmm9[2,3,2,3]
+; AVX512DQ-FAST-NEXT:    vpbroadcastq 72(%rcx), %ymm4
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm4 = mem[0,1,2,3],ymm4[4,5,6,7]
 ; AVX512DQ-FAST-NEXT:    movb $6, %al
 ; AVX512DQ-FAST-NEXT:    kmovw %eax, %k1
-; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm6, %zmm0, %zmm13 {%k1}
-; AVX512DQ-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm6 = [9,1,9,1,9,1,9,1]
-; AVX512DQ-FAST-NEXT:    # zmm6 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm15, %zmm6, %zmm11
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm30, %zmm8
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm10, %zmm6, %zmm8
-; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm6 = [4,9,0,3,4,9,0,3]
-; AVX512DQ-FAST-NEXT:    # zmm6 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm14, %zmm6, %zmm11
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm31, %zmm6, %zmm8
-; AVX512DQ-FAST-NEXT:    vpbroadcastq 8(%rcx), %ymm6
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm6 = mem[0,1,2,3],ymm6[4,5,6,7]
-; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm6, %zmm0, %zmm0 {%k1}
-; AVX512DQ-FAST-NEXT:    movb $64, %al
-; AVX512DQ-FAST-NEXT:    kmovw %eax, %k1
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm30, %zmm17 {%k1}
+; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm4, %zmm0, %zmm19 {%k1}
+; AVX512DQ-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm4 = [9,1,9,1,9,1,9,1]
+; AVX512DQ-FAST-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm12, %zmm4, %zmm7
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm6, %zmm4, %zmm27
+; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [4,9,0,3,4,9,0,3]
+; AVX512DQ-FAST-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm9, %zmm4, %zmm7
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm5, %zmm4, %zmm27
+; AVX512DQ-FAST-NEXT:    vpbroadcastq 8(%rcx), %ymm4
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm4 = mem[0,1,2,3],ymm4[4,5,6,7]
+; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm4, %zmm0, %zmm2 {%k1}
 ; AVX512DQ-FAST-NEXT:    movb $56, %al
 ; AVX512DQ-FAST-NEXT:    kmovw %eax, %k1
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm8, %zmm13 {%k1}
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm11, %zmm0 {%k1}
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm9, %zmm24, %zmm7
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm5, %zmm26, %zmm3
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm7, %zmm3 {%k2}
-; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm5 = <u,1,2,3,4,15,u,u>
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm30, %zmm3, %zmm5
-; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm3 = <u,1,2,3,4,5,15,u>
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm10, %zmm5, %zmm3
-; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm5 = <13,u,2,3,4,5,6,14>
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm10, %zmm17, %zmm5
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm27, %zmm19 {%k1}
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm7, %zmm2 {%k1}
+; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm4 = <u,1,2,3,4,5,15,u>
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm6, %zmm31, %zmm4
+; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm7 = <13,u,2,3,4,5,6,14>
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm6, %zmm9, %zmm7
 ; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [14,1,2,3,4,5,6,15]
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm31, %zmm3, %zmm6
-; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [0,13,2,3,4,5,6,7]
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm31, %zmm5, %zmm3
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm5, %zmm4, %zmm6
+; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [0,13,2,3,4,5,6,7]
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm5, %zmm7, %zmm4
 ; AVX512DQ-FAST-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, 64(%rax)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm1, 128(%rax)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm25, 192(%rax)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm12, 256(%rax)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm20, 320(%rax)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm22, 384(%rax)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm4, 448(%rax)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm13, 512(%rax)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm16, 576(%rax)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm21, 640(%rax)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm19, 704(%rax)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm2, (%rax)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm3, 768(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm2, 64(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm3, 128(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm17, 192(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm21, 256(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm18, 320(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm26, 384(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, 448(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm19, 512(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm1, 576(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm16, 640(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm22, 704(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm8, (%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm4, 768(%rax)
 ; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm6, 832(%rax)
 ; AVX512DQ-FAST-NEXT:    vzeroupper
 ; AVX512DQ-FAST-NEXT:    retq
@@ -3351,841 +3323,823 @@ define void @store_i64_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512BW-ONLY-SLOW-LABEL: store_i64_stride7_vf16:
 ; AVX512BW-ONLY-SLOW:       # %bb.0:
 ; AVX512BW-ONLY-SLOW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 (%rdi), %zmm4
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 64(%rdi), %zmm6
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 (%rsi), %zmm5
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 64(%rsi), %zmm7
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 (%rdx), %zmm8
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 64(%rdx), %zmm9
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 (%rcx), %zmm2
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 64(%rcx), %zmm3
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 64(%r8), %zmm11
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [6,14,6,14,6,14,6,14]
-; AVX512BW-ONLY-SLOW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 (%rdi), %zmm6
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 64(%rdi), %zmm18
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 (%rsi), %zmm7
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 64(%rsi), %zmm9
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 (%rdx), %zmm11
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 64(%rdx), %zmm20
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 (%rcx), %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 64(%rcx), %zmm2
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 64(%r8), %zmm3
+; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm16 = [6,14,6,14,6,14,6,14]
+; AVX512BW-ONLY-SLOW-NEXT:    # zmm16 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm20, %zmm19
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm2, %zmm16, %zmm19
+; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm8 = [15,7,15,7,15,7,15,7]
+; AVX512BW-ONLY-SLOW-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm20, %zmm21
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm2, %zmm8, %zmm21
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm18, %zmm22
+; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm5 = [0,8,0,1,0,8,0,1]
+; AVX512BW-ONLY-SLOW-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm4
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm7, %zmm5, %zmm4
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm9, %zmm18, %zmm5
+; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm10 = [13,5,13,5,13,5,13,5]
+; AVX512BW-ONLY-SLOW-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm20, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm2, %zmm10, %zmm0
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm9, %zmm12
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm3, %zmm0, %zmm12
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = [6,14,6,14]
-; AVX512BW-ONLY-SLOW-NEXT:    # ymm0 = mem[0,1,0,1]
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm15
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} ymm16 = [15,7,15,7]
-; AVX512BW-ONLY-SLOW-NEXT:    # ymm16 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm10
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm5, %zmm16, %zmm10
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm7, %zmm6, %zmm16
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm13
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm7, %zmm0, %zmm13
-; AVX512BW-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm23 = zmm13[0,1,2,3],zmm12[4,5,6,7]
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm18, %zmm15
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm18, %zmm23
+; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm13 = [2,10,0,3,2,10,0,3]
+; AVX512BW-ONLY-SLOW-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm20, %zmm12
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm2, %zmm13, %zmm12
+; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm17 = [1,0,10,2,1,0,10,2]
+; AVX512BW-ONLY-SLOW-NEXT:    # zmm17 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm9, %zmm14
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm18, %zmm17, %zmm14
+; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm29 = [3,0,12,4,3,0,12,4]
+; AVX512BW-ONLY-SLOW-NEXT:    # zmm29 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm20, %zmm29, %zmm2
+; AVX512BW-ONLY-SLOW-NEXT:    movb $48, %sil
+; AVX512BW-ONLY-SLOW-NEXT:    kmovd %esi, %k3
+; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm2 {%k3} = zmm18[0],zmm9[0],zmm18[2],zmm9[2],zmm18[4],zmm9[4],zmm18[6],zmm9[6]
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm9, %zmm16, %zmm18
+; AVX512BW-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm19 = zmm18[0,1,2,3],zmm19[4,5,6,7]
 ; AVX512BW-ONLY-SLOW-NEXT:    movb $64, %sil
 ; AVX512BW-ONLY-SLOW-NEXT:    kmovd %esi, %k1
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm23 {%k1}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 (%r8), %zmm14
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 64(%r9), %zmm24
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 (%r9), %zmm25
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 (%rax), %zmm22
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 64(%rax), %zmm30
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm12 = [0,8,0,1,0,8,0,1]
-; AVX512BW-ONLY-SLOW-NEXT:    # zmm12 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm20 = [0,8,0,8,0,8,0,8]
-; AVX512BW-ONLY-SLOW-NEXT:    # zmm20 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm21
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm25, %zmm20, %zmm21
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm24, %zmm11, %zmm20
-; AVX512BW-ONLY-SLOW-NEXT:    movb $96, %sil
-; AVX512BW-ONLY-SLOW-NEXT:    kmovd %esi, %k1
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm13 = [4,12,0,5,4,12,0,5]
-; AVX512BW-ONLY-SLOW-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm18
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm17 = [15,7,15,7,15,7,15,7]
-; AVX512BW-ONLY-SLOW-NEXT:    # zmm17 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm26
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm25, %zmm17, %zmm26
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm19
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm2, %zmm17, %zmm19
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm19 {%k1}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 (%r8), %zmm20
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 (%r9), %zmm30
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 64(%r9), %zmm31
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 64(%rax), %zmm28
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm9, %zmm8, %zmm22
 ; AVX512BW-ONLY-SLOW-NEXT:    movb $24, %sil
+; AVX512BW-ONLY-SLOW-NEXT:    kmovd %esi, %k1
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm21, %zmm22 {%k1}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm18 = <u,1,2,3,4,15,u,u>
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm3, %zmm22, %zmm18
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm21 = <u,1,2,3,4,5,15,u>
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm31, %zmm18, %zmm21
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm18 = [14,1,2,3,4,5,6,15]
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm28, %zmm21, %zmm18
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm21 = <13,u,2,3,4,5,6,14>
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm31, %zmm19, %zmm21
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm19 = [0,13,2,3,4,5,6,7]
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm28, %zmm21, %zmm19
+; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm26 = [0,8,0,8,0,8,0,8]
+; AVX512BW-ONLY-SLOW-NEXT:    # zmm26 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm20, %zmm27
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm30, %zmm26, %zmm27
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm31, %zmm3, %zmm26
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm9, %zmm10, %zmm15
+; AVX512BW-ONLY-SLOW-NEXT:    movb $96, %sil
 ; AVX512BW-ONLY-SLOW-NEXT:    kmovd %esi, %k2
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm3, %zmm9, %zmm17
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm17, %zmm16 {%k2}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm27 = <u,1,2,3,4,15,u,u>
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm11, %zmm16, %zmm27
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm24, %zmm13, %zmm11
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm29 = [0,1,12,7,0,1,12,7]
-; AVX512BW-ONLY-SLOW-NEXT:    # zmm29 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm30, %zmm29, %zmm11
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm31 = [11,3,11,3,11,3,11,3]
-; AVX512BW-ONLY-SLOW-NEXT:    # zmm31 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm7, %zmm31, %zmm15
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm17 = [2,10,0,3,2,10,0,3]
-; AVX512BW-ONLY-SLOW-NEXT:    # zmm17 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm9, %zmm16
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm3, %zmm17, %zmm16
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm15, %zmm16 {%k1}
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm28 = [9,1,9,1,9,1,9,1]
-; AVX512BW-ONLY-SLOW-NEXT:    # zmm28 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [5,0,14,6,5,0,14,6]
-; AVX512BW-ONLY-SLOW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm14, %zmm25, %zmm0
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm25, %zmm14, %zmm13
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm25, %zmm28, %zmm14
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa 64(%r9), %ymm15
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu %ymm15, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm24, %zmm28, %zmm18
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm25 = <u,1,2,3,4,5,15,u>
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm24, %zmm27, %zmm25
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm1 = <13,u,2,3,4,5,6,14>
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm24, %zmm23, %zmm1
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 64(%r8), %ymm24
-; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm23 = ymm24[0],ymm15[0],ymm24[2],ymm15[2]
-; AVX512BW-ONLY-SLOW-NEXT:    movb $28, %sil
-; AVX512BW-ONLY-SLOW-NEXT:    kmovd %esi, %k3
-; AVX512BW-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm16 {%k3} = zmm23[2,3,2,3],zmm30[2,3,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm28 = [6,13,14,7,6,13,14,7]
-; AVX512BW-ONLY-SLOW-NEXT:    # zmm28 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm26, %zmm22, %zmm28
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm27 = [0,13,6,7,0,13,6,7]
-; AVX512BW-ONLY-SLOW-NEXT:    # zmm27 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm22, %zmm0, %zmm27
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa (%r9), %ymm15
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm22, %zmm29, %zmm13
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 (%r8), %ymm26
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm5, %zmm4, %zmm31
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm2, %zmm8, %zmm17
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm31, %zmm17 {%k1}
-; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm23 = ymm26[0],ymm15[0],ymm26[2],ymm15[2]
-; AVX512BW-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm17 {%k3} = zmm23[2,3,2,3],zmm22[2,3,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm23 = [4,9,0,3,4,9,0,3]
-; AVX512BW-ONLY-SLOW-NEXT:    # zmm23 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm22, %zmm23, %zmm14
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm29 = [13,5,13,5,13,5,13,5]
+; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm21 = [4,12,0,5,4,12,0,5]
+; AVX512BW-ONLY-SLOW-NEXT:    # zmm21 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm25
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm8, %zmm25
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm24
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm16, %zmm24
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm22
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm10, %zmm22
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm1, %zmm11, %zmm13
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm11, %zmm29, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm11
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm31, %zmm21, %zmm11
+; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm29 = [11,3,11,3,11,3,11,3]
 ; AVX512BW-ONLY-SLOW-NEXT:    # zmm29 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm30, %zmm23, %zmm18
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm22 = [14,1,2,3,4,5,6,15]
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm30, %zmm25, %zmm22
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm23 = [0,13,2,3,4,5,6,7]
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm30, %zmm1, %zmm23
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm9, %zmm1
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm3, %zmm29, %zmm1
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm2, %zmm8, %zmm0
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm25 = [3,0,12,4,3,0,12,4]
-; AVX512BW-ONLY-SLOW-NEXT:    # zmm25 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm9, %zmm25, %zmm3
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm30
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm2, %zmm29, %zmm30
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm8, %zmm25, %zmm2
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm8
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm5, %zmm12, %zmm8
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm7, %zmm6, %zmm12
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm31 = [1,0,10,2,1,0,10,2]
-; AVX512BW-ONLY-SLOW-NEXT:    # zmm31 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm9
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm6, %zmm31, %zmm9
-; AVX512BW-ONLY-SLOW-NEXT:    movb $48, %sil
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm9, %zmm29, %zmm23
+; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm9 = [0,1,12,7,0,1,12,7]
+; AVX512BW-ONLY-SLOW-NEXT:    # zmm9 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm28, %zmm9, %zmm11
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm23, %zmm12 {%k2}
+; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm23 = [9,1,9,1,9,1,9,1]
+; AVX512BW-ONLY-SLOW-NEXT:    # zmm23 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm31, %zmm23, %zmm3
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 64(%r9), %ymm31
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm7, %zmm6, %zmm16
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm7, %zmm6, %zmm10
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm7, %zmm6, %zmm29
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm6, %zmm7, %zmm17
+; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm1 {%k3} = zmm6[0],zmm7[0],zmm6[2],zmm7[2],zmm6[4],zmm7[4],zmm6[6],zmm7[6]
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm7, %zmm8, %zmm6
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm30, %zmm20, %zmm8
+; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm7 = [5,0,14,6,5,0,14,6]
+; AVX512BW-ONLY-SLOW-NEXT:    # zmm7 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm20, %zmm30, %zmm7
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm30, %zmm20, %zmm21
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm30, %zmm23, %zmm20
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 64(%r8), %ymm23
+; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm30 = ymm23[0],ymm31[0],ymm23[2],ymm31[2]
+; AVX512BW-ONLY-SLOW-NEXT:    movb $28, %sil
 ; AVX512BW-ONLY-SLOW-NEXT:    kmovd %esi, %k3
-; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm3 {%k3} = zmm6[0],zmm7[0],zmm6[2],zmm7[2],zmm6[4],zmm7[4],zmm6[6],zmm7[6]
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm7, %zmm29, %zmm6
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti128 {{.*#+}} ymm7 = [6,14,6,14]
-; AVX512BW-ONLY-SLOW-NEXT:    # ymm7 = mem[0,1,0,1]
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm5, %zmm4, %zmm7
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm5, %zmm4, %zmm29
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm4, %zmm5, %zmm31
-; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm2 {%k3} = zmm4[0],zmm5[0],zmm4[2],zmm5[2],zmm4[4],zmm5[4],zmm4[6],zmm5[6]
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa (%rdx), %xmm4
-; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm4 = xmm4[0],mem[0]
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512BW-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm12 {%k3} = zmm30[2,3,2,3],zmm28[2,3,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm30 = [4,9,0,3,4,9,0,3]
+; AVX512BW-ONLY-SLOW-NEXT:    # zmm30 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm28, %zmm30, %zmm3
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 (%rax), %zmm28
+; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [6,13,14,7,6,13,14,7]
+; AVX512BW-ONLY-SLOW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm8, %zmm28, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm8 = [0,13,6,7,0,13,6,7]
+; AVX512BW-ONLY-SLOW-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm28, %zmm7, %zmm8
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm28, %zmm9, %zmm21
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa (%r9), %ymm7
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm29, %zmm13 {%k2}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa (%r8), %ymm9
+; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm29 = ymm9[0],ymm7[0],ymm9[2],ymm7[2]
+; AVX512BW-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm13 {%k3} = zmm29[2,3,2,3],zmm28[2,3,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm28, %zmm30, %zmm20
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 (%rdx), %xmm28
+; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm28 = xmm28[0],mem[0]
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti32x4 $1, %xmm28, %ymm0, %ymm28
 ; AVX512BW-ONLY-SLOW-NEXT:    movb $12, %sil
 ; AVX512BW-ONLY-SLOW-NEXT:    kmovd %esi, %k3
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm4, %zmm0, %zmm8 {%k3}
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti32x4 $3, (%rax), %zmm21, %zmm4
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm28, %zmm0, %zmm4 {%k3}
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti32x4 $3, (%rax), %zmm27, %zmm27
 ; AVX512BW-ONLY-SLOW-NEXT:    movb $112, %sil
 ; AVX512BW-ONLY-SLOW-NEXT:    kmovd %esi, %k4
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm8 {%k4}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa 64(%rdx), %xmm4
-; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm4 = xmm4[0],mem[0]
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm4, %zmm0, %zmm12 {%k3}
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti32x4 $3, 64(%rax), %zmm20, %zmm4
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm12 {%k4}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm6 {%k1}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm27, %zmm4 {%k4}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 64(%rdx), %xmm27
+; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm27 = xmm27[0],mem[0]
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti32x4 $1, %xmm27, %ymm0, %ymm27
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm27, %zmm0, %zmm5 {%k3}
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti32x4 $3, 64(%rax), %zmm26, %zmm26
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm26, %zmm5 {%k4}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm26 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm26, %zmm15 {%k2}
 ; AVX512BW-ONLY-SLOW-NEXT:    movb $120, %sil
 ; AVX512BW-ONLY-SLOW-NEXT:    kmovd %esi, %k3
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm11 {%k3}
-; AVX512BW-ONLY-SLOW-NEXT:    vpbroadcastq 72(%rcx), %ymm1
-; AVX512BW-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = mem[0,1,2,3],ymm1[4,5,6,7]
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm15, %zmm11 {%k3}
+; AVX512BW-ONLY-SLOW-NEXT:    vpbroadcastq 72(%rcx), %ymm15
+; AVX512BW-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm15 = mem[0,1,2,3],ymm15[4,5,6,7]
 ; AVX512BW-ONLY-SLOW-NEXT:    movb $6, %sil
 ; AVX512BW-ONLY-SLOW-NEXT:    kmovd %esi, %k4
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm9 {%k4}
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm15, %zmm0, %zmm14 {%k4}
 ; AVX512BW-ONLY-SLOW-NEXT:    movb $56, %sil
 ; AVX512BW-ONLY-SLOW-NEXT:    kmovd %esi, %k5
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm18, %zmm9 {%k5}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm19, %zmm10 {%k2}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm14 {%k5}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm25, %zmm6 {%k1}
 ; AVX512BW-ONLY-SLOW-NEXT:    movb $-31, %sil
-; AVX512BW-ONLY-SLOW-NEXT:    kmovd %esi, %k2
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm28, %zmm10 {%k2}
-; AVX512BW-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm7[0,1,2,3],zmm0[4,5,6,7]
+; AVX512BW-ONLY-SLOW-NEXT:    kmovd %esi, %k1
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm6 {%k1}
+; AVX512BW-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm16[0,1,2,3],zmm24[4,5,6,7]
 ; AVX512BW-ONLY-SLOW-NEXT:    movb $-61, %sil
-; AVX512BW-ONLY-SLOW-NEXT:    kmovd %esi, %k2
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm27, %zmm1 {%k2}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm30, %zmm29 {%k1}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm29, %zmm13 {%k3}
-; AVX512BW-ONLY-SLOW-NEXT:    vpbroadcastq 8(%rcx), %ymm4
-; AVX512BW-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm4 = mem[0,1,2,3],ymm4[4,5,6,7]
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm4, %zmm0, %zmm31 {%k4}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm31 {%k5}
-; AVX512BW-ONLY-SLOW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm24, %ymm4 # 32-byte Folded Reload
-; AVX512BW-ONLY-SLOW-NEXT:    # ymm4 = ymm24[1],mem[1],ymm24[3],mem[3]
-; AVX512BW-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[0,2,3,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3,4,5],mem[6,7]
+; AVX512BW-ONLY-SLOW-NEXT:    kmovd %esi, %k1
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm0 {%k1}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm22, %zmm10 {%k2}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm10, %zmm21 {%k3}
+; AVX512BW-ONLY-SLOW-NEXT:    vpbroadcastq 8(%rcx), %ymm3
+; AVX512BW-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm3 = mem[0,1,2,3],ymm3[4,5,6,7]
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm17 {%k4}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm20, %zmm17 {%k5}
+; AVX512BW-ONLY-SLOW-NEXT:    vpunpckhqdq {{.*#+}} ymm3 = ymm23[1],ymm31[1],ymm23[3],ymm31[3]
+; AVX512BW-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,2,3,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5],mem[6,7]
 ; AVX512BW-ONLY-SLOW-NEXT:    movb $14, %cl
 ; AVX512BW-ONLY-SLOW-NEXT:    kmovd %ecx, %k1
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm4, %zmm0, %zmm3 {%k1}
-; AVX512BW-ONLY-SLOW-NEXT:    vpunpckhqdq {{.*#+}} ymm0 = ymm26[1],ymm15[1],ymm26[3],ymm15[3]
-; AVX512BW-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,3,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],mem[6,7]
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm2 {%k1}
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm2 {%k1}
+; AVX512BW-ONLY-SLOW-NEXT:    vpunpckhqdq {{.*#+}} ymm3 = ymm9[1],ymm7[1],ymm9[3],ymm7[3]
+; AVX512BW-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,2,3,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5],mem[6,7]
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm1 {%k1}
 ; AVX512BW-ONLY-SLOW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm31, 64(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm17, 128(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm13, 256(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, 320(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm10, 384(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm9, 512(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm16, 576(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm17, 64(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm13, 128(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm21, 256(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, 320(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, 384(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm14, 512(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm12, 576(%rax)
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm11, 704(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, 192(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm12, 448(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, 640(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm8, (%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm23, 768(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm22, 832(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, 192(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm5, 448(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, 640(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm4, (%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm19, 768(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm18, 832(%rax)
 ; AVX512BW-ONLY-SLOW-NEXT:    vzeroupper
 ; AVX512BW-ONLY-SLOW-NEXT:    retq
 ;
 ; AVX512BW-ONLY-FAST-LABEL: store_i64_stride7_vf16:
 ; AVX512BW-ONLY-FAST:       # %bb.0:
 ; AVX512BW-ONLY-FAST-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 (%rdi), %zmm12
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 64(%rdi), %zmm4
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 (%rsi), %zmm0
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 64(%rsi), %zmm5
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 (%rdx), %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 64(%rdx), %zmm7
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 (%rcx), %zmm17
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 64(%rcx), %zmm9
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 64(%r8), %zmm30
-; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm20 = [6,14,6,14,6,14,6,14]
-; AVX512BW-ONLY-FAST-NEXT:    # zmm20 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm7, %zmm18
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm9, %zmm20, %zmm18
-; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} ymm21 = [6,14,6,14]
-; AVX512BW-ONLY-FAST-NEXT:    # ymm21 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm4, %zmm19
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm5, %zmm21, %zmm19
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 (%r8), %zmm11
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 64(%r9), %zmm10
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 (%r9), %zmm15
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 (%rax), %zmm14
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 64(%rax), %zmm31
-; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm3 = [0,8,0,1,0,8,0,1]
-; AVX512BW-ONLY-FAST-NEXT:    # zmm3 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm12, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm3, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa (%rdx), %xmm13
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 64(%rdx), %xmm16
-; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm13 = xmm13[0],mem[0]
-; AVX512BW-ONLY-FAST-NEXT:    vinserti128 $1, %xmm13, %ymm0, %ymm13
-; AVX512BW-ONLY-FAST-NEXT:    movb $12, %sil
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 (%rdi), %zmm18
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 64(%rdi), %zmm26
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 (%rsi), %zmm2
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 64(%rsi), %zmm19
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 (%rdx), %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 64(%rdx), %zmm1
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 (%rcx), %zmm15
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 64(%rcx), %zmm21
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 64(%r8), %zmm6
+; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm17 = [6,14,6,14,6,14,6,14]
+; AVX512BW-ONLY-FAST-NEXT:    # zmm17 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm21, %zmm17, %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm26, %zmm4
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm19, %zmm17, %zmm4
+; AVX512BW-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm8 = zmm4[0,1,2,3],zmm0[4,5,6,7]
+; AVX512BW-ONLY-FAST-NEXT:    movb $64, %sil
 ; AVX512BW-ONLY-FAST-NEXT:    kmovd %esi, %k1
-; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm13, %zmm0, %zmm2 {%k1}
-; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm13 = [0,8,0,8,0,8,0,8]
-; AVX512BW-ONLY-FAST-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm11, %zmm22
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm15, %zmm13, %zmm22
-; AVX512BW-ONLY-FAST-NEXT:    vinserti32x4 $3, (%rax), %zmm22, %zmm22
-; AVX512BW-ONLY-FAST-NEXT:    movb $112, %sil
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm6, %zmm8 {%k1}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 (%r8), %zmm4
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 (%r9), %zmm5
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 64(%r9), %zmm23
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 64(%rax), %zmm12
+; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm11 = [15,7,15,7,15,7,15,7]
+; AVX512BW-ONLY-FAST-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, %zmm7
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm21, %zmm11, %zmm7
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm26, %zmm9
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm19, %zmm11, %zmm9
+; AVX512BW-ONLY-FAST-NEXT:    movb $24, %sil
+; AVX512BW-ONLY-FAST-NEXT:    kmovd %esi, %k1
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm7, %zmm9 {%k1}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm7 = <u,1,2,3,4,15,u,u>
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm6, %zmm9, %zmm7
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm9 = <u,1,2,3,4,5,15,u>
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm23, %zmm7, %zmm9
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [14,1,2,3,4,5,6,15]
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm12, %zmm9, %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm9 = <13,u,2,3,4,5,6,14>
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm23, %zmm8, %zmm9
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm8 = [0,13,2,3,4,5,6,7]
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm12, %zmm9, %zmm8
+; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm10 = [0,8,0,1,0,8,0,1]
+; AVX512BW-ONLY-FAST-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm18, %zmm9
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm2, %zmm10, %zmm9
+; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm7 = [0,8,0,8,0,8,0,8]
+; AVX512BW-ONLY-FAST-NEXT:    # zmm7 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm4, %zmm22
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm5, %zmm7, %zmm22
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm19, %zmm26, %zmm10
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm23, %zmm6, %zmm7
+; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm14 = [13,5,13,5,13,5,13,5]
+; AVX512BW-ONLY-FAST-NEXT:    # zmm14 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, %zmm25
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm21, %zmm14, %zmm25
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm26, %zmm24
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm19, %zmm14, %zmm24
+; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm13 = [4,12,0,5,4,12,0,5]
+; AVX512BW-ONLY-FAST-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm20 = [3,0,12,4,3,0,12,4]
+; AVX512BW-ONLY-FAST-NEXT:    # zmm20 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm21, %zmm16
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm1, %zmm20, %zmm16
+; AVX512BW-ONLY-FAST-NEXT:    movb $48, %sil
 ; AVX512BW-ONLY-FAST-NEXT:    kmovd %esi, %k2
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm22, %zmm2 {%k2}
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm5, %zmm4, %zmm3
-; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm16 = xmm16[0],mem[0]
-; AVX512BW-ONLY-FAST-NEXT:    vinserti32x4 $1, %xmm16, %ymm0, %ymm16
-; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm16, %zmm0, %zmm3 {%k1}
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm10, %zmm30, %zmm13
-; AVX512BW-ONLY-FAST-NEXT:    vinserti32x4 $3, 64(%rax), %zmm13, %zmm13
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm13, %zmm3 {%k2}
-; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm24 = [13,5,13,5,13,5,13,5]
-; AVX512BW-ONLY-FAST-NEXT:    # zmm24 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm7, %zmm13
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm9, %zmm24, %zmm13
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm4, %zmm26
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm5, %zmm24, %zmm26
+; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm16 {%k2} = zmm26[0],zmm19[0],zmm26[2],zmm19[2],zmm26[4],zmm19[4],zmm26[6],zmm19[6]
+; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm27 = [11,3,11,3,11,3,11,3]
+; AVX512BW-ONLY-FAST-NEXT:    # zmm27 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm26, %zmm29
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm19, %zmm27, %zmm29
+; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm31 = [1,0,10,2,1,0,10,2]
+; AVX512BW-ONLY-FAST-NEXT:    # zmm31 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm26, %zmm31, %zmm19
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm18, %zmm26
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm2, %zmm11, %zmm26
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm28
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm15, %zmm17, %zmm28
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm2, %zmm18, %zmm17
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm30
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm15, %zmm14, %zmm30
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm2, %zmm18, %zmm14
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm3, %zmm15, %zmm20
+; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm20 {%k2} = zmm18[0],zmm2[0],zmm18[2],zmm2[2],zmm18[4],zmm2[4],zmm18[6],zmm2[6]
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm2, %zmm18, %zmm27
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm18, %zmm31, %zmm2
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm6, %zmm18
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm23, %zmm13, %zmm18
+; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm31 = [2,10,0,3,2,10,0,3]
+; AVX512BW-ONLY-FAST-NEXT:    # zmm31 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm21, %zmm31, %zmm1
+; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [9,1,9,1,9,1,9,1]
+; AVX512BW-ONLY-FAST-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm23, %zmm0, %zmm6
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm21
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm15, %zmm31, %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 64(%r9), %ymm23
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm15, %zmm11, %zmm21
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm5, %zmm4, %zmm11
+; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm31 = [5,0,14,6,5,0,14,6]
+; AVX512BW-ONLY-FAST-NEXT:    # zmm31 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm4, %zmm5, %zmm31
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm5, %zmm4, %zmm13
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm5, %zmm0, %zmm4
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa 64(%r8), %ymm0
+; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm5 = ymm0[0],ymm23[0],ymm0[2],ymm23[2]
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa {{.*#+}} ymm15 = [1,3,7,7]
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %ymm23, %ymm15, %ymm0
 ; AVX512BW-ONLY-FAST-NEXT:    movb $96, %sil
-; AVX512BW-ONLY-FAST-NEXT:    kmovd %esi, %k1
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm13, %zmm26 {%k1}
-; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm16 = [4,12,0,5,4,12,0,5]
-; AVX512BW-ONLY-FAST-NEXT:    # zmm16 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm30, %zmm13
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm10, %zmm16, %zmm13
-; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm27 = [0,1,12,7,0,1,12,7]
-; AVX512BW-ONLY-FAST-NEXT:    # zmm27 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm31, %zmm27, %zmm13
-; AVX512BW-ONLY-FAST-NEXT:    movb $120, %sil
-; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm23 = [15,7,15,7,15,7,15,7]
-; AVX512BW-ONLY-FAST-NEXT:    # zmm23 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm11, %zmm22
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm15, %zmm23, %zmm22
-; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm28 = [6,13,14,7,6,13,14,7]
-; AVX512BW-ONLY-FAST-NEXT:    # zmm28 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm22, %zmm14, %zmm28
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, %zmm29
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm17, %zmm23, %zmm29
-; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} ymm25 = [15,7,15,7]
-; AVX512BW-ONLY-FAST-NEXT:    # ymm25 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm12, %zmm22
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm25, %zmm22
-; AVX512BW-ONLY-FAST-NEXT:    movb $24, %dil
-; AVX512BW-ONLY-FAST-NEXT:    kmovd %edi, %k2
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm29, %zmm22 {%k2}
-; AVX512BW-ONLY-FAST-NEXT:    movb $-31, %dil
-; AVX512BW-ONLY-FAST-NEXT:    kmovd %edi, %k3
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm28, %zmm22 {%k3}
-; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm28 = [5,0,14,6,5,0,14,6]
-; AVX512BW-ONLY-FAST-NEXT:    # zmm28 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm11, %zmm15, %zmm28
-; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm29 = [0,13,6,7,0,13,6,7]
+; AVX512BW-ONLY-FAST-NEXT:    kmovd %esi, %k2
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm29, %zmm1 {%k2}
+; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm23 = [0,1,12,7,0,1,12,7]
+; AVX512BW-ONLY-FAST-NEXT:    # zmm23 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm12, %zmm23, %zmm18
+; AVX512BW-ONLY-FAST-NEXT:    movb $28, %sil
+; AVX512BW-ONLY-FAST-NEXT:    kmovd %esi, %k3
+; AVX512BW-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm1 {%k3} = zmm5[2,3,2,3],zmm12[2,3,2,3]
+; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm5 = [4,9,0,3,4,9,0,3]
+; AVX512BW-ONLY-FAST-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm12, %zmm5, %zmm6
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 (%rax), %zmm12
+; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm29 = [6,13,14,7,6,13,14,7]
 ; AVX512BW-ONLY-FAST-NEXT:    # zmm29 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm14, %zmm28, %zmm29
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm17, %zmm1, %zmm20
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm0, %zmm12, %zmm21
-; AVX512BW-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm20 = zmm21[0,1,2,3],zmm20[4,5,6,7]
-; AVX512BW-ONLY-FAST-NEXT:    movb $-61, %dil
-; AVX512BW-ONLY-FAST-NEXT:    kmovd %edi, %k3
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm29, %zmm20 {%k3}
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm11, %zmm12, %zmm29
+; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm11 = [0,13,6,7,0,13,6,7]
+; AVX512BW-ONLY-FAST-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm12, %zmm31, %zmm11
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm12, %zmm23, %zmm13
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 (%r9), %ymm23
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm27, %zmm3 {%k2}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 (%r8), %ymm27
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %ymm23, %ymm27, %ymm15
+; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm23 = ymm27[0],ymm23[0],ymm27[2],ymm23[2]
+; AVX512BW-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm3 {%k3} = zmm23[2,3,2,3],zmm12[2,3,2,3]
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm12, %zmm5, %zmm4
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa (%rdx), %xmm5
+; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm5 = xmm5[0],mem[0]
+; AVX512BW-ONLY-FAST-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512BW-ONLY-FAST-NEXT:    movb $12, %sil
 ; AVX512BW-ONLY-FAST-NEXT:    kmovd %esi, %k3
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm26, %zmm13 {%k3}
-; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm26 = [3,0,12,4,3,0,12,4]
-; AVX512BW-ONLY-FAST-NEXT:    # zmm26 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm9, %zmm21
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm7, %zmm26, %zmm21
-; AVX512BW-ONLY-FAST-NEXT:    movb $48, %sil
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, %zmm28
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm17, %zmm24, %zmm28
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm0, %zmm12, %zmm24
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm28, %zmm24 {%k1}
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm15, %zmm11, %zmm16
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm14, %zmm27, %zmm16
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm24, %zmm16 {%k3}
+; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm5, %zmm0, %zmm9 {%k3}
+; AVX512BW-ONLY-FAST-NEXT:    vinserti32x4 $3, (%rax), %zmm22, %zmm5
+; AVX512BW-ONLY-FAST-NEXT:    movb $112, %sil
+; AVX512BW-ONLY-FAST-NEXT:    kmovd %esi, %k4
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm5, %zmm9 {%k4}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa 64(%rdx), %xmm5
+; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm5 = xmm5[0],mem[0]
+; AVX512BW-ONLY-FAST-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm5, %zmm0, %zmm10 {%k3}
+; AVX512BW-ONLY-FAST-NEXT:    vinserti32x4 $3, 64(%rax), %zmm7, %zmm5
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm5, %zmm10 {%k4}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm25, %zmm24 {%k2}
+; AVX512BW-ONLY-FAST-NEXT:    movb $120, %sil
 ; AVX512BW-ONLY-FAST-NEXT:    kmovd %esi, %k3
-; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm21 {%k3} = zmm4[0],zmm5[0],zmm4[2],zmm5[2],zmm4[4],zmm5[4],zmm4[6],zmm5[6]
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 (%r9), %ymm24
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 64(%r9), %ymm27
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 64(%r8), %ymm28
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa {{.*#+}} ymm6 = [1,3,7,7]
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %ymm28, %ymm8
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %ymm27, %ymm6, %ymm8
-; AVX512BW-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],mem[6,7]
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm24, %zmm18 {%k3}
+; AVX512BW-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],mem[6,7]
 ; AVX512BW-ONLY-FAST-NEXT:    movb $14, %sil
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm1, %zmm17, %zmm26
-; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm26 {%k3} = zmm12[0],zmm0[0],zmm12[2],zmm0[2],zmm12[4],zmm0[4],zmm12[6],zmm0[6]
-; AVX512BW-ONLY-FAST-NEXT:    kmovd %esi, %k3
-; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm8, %zmm0, %zmm21 {%k3}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa (%r8), %ymm8
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %ymm24, %ymm8, %ymm6
-; AVX512BW-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3,4,5],mem[6,7]
-; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm6, %zmm0, %zmm26 {%k3}
-; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm6 = ymm28[0],ymm27[0],ymm28[2],ymm27[2]
-; AVX512BW-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm18 = zmm19[0,1,2,3],zmm18[4,5,6,7]
-; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm19 = [11,3,11,3,11,3,11,3]
-; AVX512BW-ONLY-FAST-NEXT:    # zmm19 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm27 = [2,10,0,3,2,10,0,3]
-; AVX512BW-ONLY-FAST-NEXT:    # zmm27 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm17, %zmm27, %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm7, %zmm17
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm9, %zmm27, %zmm17
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm4, %zmm27
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm5, %zmm19, %zmm27
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm27, %zmm17 {%k1}
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm0, %zmm12, %zmm19
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm19, %zmm1 {%k1}
-; AVX512BW-ONLY-FAST-NEXT:    movb $28, %al
-; AVX512BW-ONLY-FAST-NEXT:    kmovd %eax, %k1
-; AVX512BW-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm17 {%k1} = zmm6[2,3,2,3],zmm31[2,3,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm6 = [1,0,10,2,1,0,10,2]
-; AVX512BW-ONLY-FAST-NEXT:    # zmm6 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm12, %zmm6, %zmm0
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm5, %zmm12
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm4, %zmm6, %zmm12
-; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm6 = ymm8[0],ymm24[0],ymm8[2],ymm24[2]
-; AVX512BW-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm1 {%k1} = zmm6[2,3,2,3],zmm14[2,3,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vpbroadcastq 72(%rcx), %ymm6
-; AVX512BW-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm6 = mem[0,1,2,3],ymm6[4,5,6,7]
-; AVX512BW-ONLY-FAST-NEXT:    movb $6, %al
-; AVX512BW-ONLY-FAST-NEXT:    kmovd %eax, %k1
-; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm6, %zmm0, %zmm12 {%k1}
-; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm6 = [9,1,9,1,9,1,9,1]
-; AVX512BW-ONLY-FAST-NEXT:    # zmm6 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm15, %zmm6, %zmm11
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm30, %zmm8
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm10, %zmm6, %zmm8
-; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm6 = [4,9,0,3,4,9,0,3]
-; AVX512BW-ONLY-FAST-NEXT:    # zmm6 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm14, %zmm6, %zmm11
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm31, %zmm6, %zmm8
-; AVX512BW-ONLY-FAST-NEXT:    vpbroadcastq 8(%rcx), %ymm6
-; AVX512BW-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm6 = mem[0,1,2,3],ymm6[4,5,6,7]
-; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm6, %zmm0, %zmm0 {%k1}
-; AVX512BW-ONLY-FAST-NEXT:    movb $64, %al
-; AVX512BW-ONLY-FAST-NEXT:    kmovd %eax, %k1
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm30, %zmm18 {%k1}
-; AVX512BW-ONLY-FAST-NEXT:    movb $56, %al
-; AVX512BW-ONLY-FAST-NEXT:    kmovd %eax, %k1
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm8, %zmm12 {%k1}
+; AVX512BW-ONLY-FAST-NEXT:    kmovd %esi, %k4
+; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm16 {%k4}
+; AVX512BW-ONLY-FAST-NEXT:    vpbroadcastq 72(%rcx), %ymm0
+; AVX512BW-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
+; AVX512BW-ONLY-FAST-NEXT:    movb $6, %sil
+; AVX512BW-ONLY-FAST-NEXT:    kmovd %esi, %k5
+; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm19 {%k5}
+; AVX512BW-ONLY-FAST-NEXT:    movb $56, %sil
+; AVX512BW-ONLY-FAST-NEXT:    kmovd %esi, %k6
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm6, %zmm19 {%k6}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm21, %zmm26 {%k1}
+; AVX512BW-ONLY-FAST-NEXT:    movb $-31, %sil
+; AVX512BW-ONLY-FAST-NEXT:    kmovd %esi, %k1
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm29, %zmm26 {%k1}
+; AVX512BW-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm17[0,1,2,3],zmm28[4,5,6,7]
+; AVX512BW-ONLY-FAST-NEXT:    movb $-61, %sil
+; AVX512BW-ONLY-FAST-NEXT:    kmovd %esi, %k1
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm11, %zmm0 {%k1}
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm9, %zmm23, %zmm7
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm5, %zmm25, %zmm4
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm7, %zmm4 {%k2}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm5 = <u,1,2,3,4,15,u,u>
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm30, %zmm4, %zmm5
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm4 = <u,1,2,3,4,5,15,u>
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm10, %zmm5, %zmm4
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm5 = <13,u,2,3,4,5,6,14>
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm10, %zmm18, %zmm5
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [14,1,2,3,4,5,6,15]
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm31, %zmm4, %zmm6
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [0,13,2,3,4,5,6,7]
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm31, %zmm5, %zmm4
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm30, %zmm14 {%k2}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm14, %zmm13 {%k3}
+; AVX512BW-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm5 = ymm15[0,1,2,3,4,5],mem[6,7]
+; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm5, %zmm0, %zmm20 {%k4}
+; AVX512BW-ONLY-FAST-NEXT:    vpbroadcastq 8(%rcx), %ymm5
+; AVX512BW-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm5 = mem[0,1,2,3],ymm5[4,5,6,7]
+; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm5, %zmm0, %zmm2 {%k5}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm4, %zmm2 {%k6}
 ; AVX512BW-ONLY-FAST-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, 64(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, 128(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm26, 192(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm16, 256(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm20, 320(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm22, 384(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm12, 512(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm17, 576(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm21, 640(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm13, 704(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, 448(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, (%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm4, 768(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm6, 832(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, 64(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, 128(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm20, 192(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm13, 256(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, 320(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm26, 384(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm19, 512(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, 576(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm16, 640(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm18, 704(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm10, 448(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm9, (%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm8, 768(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovaps %zmm0, 832(%rax)
 ; AVX512BW-ONLY-FAST-NEXT:    vzeroupper
 ; AVX512BW-ONLY-FAST-NEXT:    retq
 ;
 ; AVX512DQBW-SLOW-LABEL: store_i64_stride7_vf16:
 ; AVX512DQBW-SLOW:       # %bb.0:
 ; AVX512DQBW-SLOW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 (%rdi), %zmm4
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 64(%rdi), %zmm6
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 (%rsi), %zmm13
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 64(%rsi), %zmm7
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 (%rdx), %zmm8
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 64(%rdx), %zmm9
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 (%rcx), %zmm2
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 64(%rcx), %zmm3
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 64(%r8), %zmm11
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [6,14,6,14,6,14,6,14]
-; AVX512DQBW-SLOW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm9, %zmm12
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm3, %zmm0, %zmm12
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [6,14,6,14]
-; AVX512DQBW-SLOW-NEXT:    # ymm1 = mem[0,1,0,1]
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm0
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} ymm16 = [15,7,15,7]
-; AVX512DQBW-SLOW-NEXT:    # ymm16 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm10
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm13, %zmm16, %zmm10
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm7, %zmm6, %zmm16
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm14
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm7, %zmm1, %zmm14
-; AVX512DQBW-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm23 = zmm14[0,1,2,3],zmm12[4,5,6,7]
-; AVX512DQBW-SLOW-NEXT:    movb $64, %sil
-; AVX512DQBW-SLOW-NEXT:    kmovd %esi, %k1
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm23 {%k1}
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 (%r8), %zmm14
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 64(%r9), %zmm24
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 (%r9), %zmm25
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 (%rax), %zmm22
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 64(%rax), %zmm28
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm17 = [0,8,0,8,0,8,0,8]
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 (%rdi), %zmm6
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 64(%rdi), %zmm18
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 (%rsi), %zmm7
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 64(%rsi), %zmm12
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 (%rdx), %zmm13
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 64(%rdx), %zmm20
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 (%rcx), %zmm1
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 64(%rcx), %zmm0
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 64(%r8), %zmm16
+; AVX512DQBW-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm17 = [6,14,6,14,6,14,6,14]
 ; AVX512DQBW-SLOW-NEXT:    # zmm17 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm21
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm25, %zmm17, %zmm21
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm15 = [13,5,13,5,13,5,13,5]
-; AVX512DQBW-SLOW-NEXT:    # zmm15 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQBW-SLOW-NEXT:    movb $96, %sil
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm20, %zmm19
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm0, %zmm17, %zmm19
+; AVX512DQBW-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm9 = [15,7,15,7,15,7,15,7]
+; AVX512DQBW-SLOW-NEXT:    # zmm9 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm20, %zmm21
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm0, %zmm9, %zmm21
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm18, %zmm22
+; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm5 = [0,8,0,1,0,8,0,1]
+; AVX512DQBW-SLOW-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm4
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm7, %zmm5, %zmm4
+; AVX512DQBW-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm10 = [13,5,13,5,13,5,13,5]
+; AVX512DQBW-SLOW-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm20, %zmm2
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm0, %zmm10, %zmm2
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm18, %zmm2
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm18, %zmm23
+; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm11 = [2,10,0,3,2,10,0,3]
+; AVX512DQBW-SLOW-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm20, %zmm8
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm0, %zmm11, %zmm8
+; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm15 = [1,0,10,2,1,0,10,2]
+; AVX512DQBW-SLOW-NEXT:    # zmm15 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm12, %zmm14
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm18, %zmm15, %zmm14
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm12, %zmm18, %zmm5
+; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm29 = [3,0,12,4,3,0,12,4]
+; AVX512DQBW-SLOW-NEXT:    # zmm29 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm20, %zmm29, %zmm0
+; AVX512DQBW-SLOW-NEXT:    movb $48, %sil
+; AVX512DQBW-SLOW-NEXT:    kmovd %esi, %k3
+; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm0 {%k3} = zmm18[0],zmm12[0],zmm18[2],zmm12[2],zmm18[4],zmm12[4],zmm18[6],zmm12[6]
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm12, %zmm17, %zmm18
+; AVX512DQBW-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm19 = zmm18[0,1,2,3],zmm19[4,5,6,7]
+; AVX512DQBW-SLOW-NEXT:    movb $64, %sil
 ; AVX512DQBW-SLOW-NEXT:    kmovd %esi, %k1
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm12 = [4,12,0,5,4,12,0,5]
-; AVX512DQBW-SLOW-NEXT:    # zmm12 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm20
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm24, %zmm11, %zmm17
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm18 = [15,7,15,7,15,7,15,7]
-; AVX512DQBW-SLOW-NEXT:    # zmm18 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm26
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm25, %zmm18, %zmm26
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm19
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm2, %zmm18, %zmm19
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm16, %zmm19 {%k1}
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 (%r8), %zmm20
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 (%r9), %zmm28
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 64(%r9), %zmm30
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 64(%rax), %zmm27
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm12, %zmm9, %zmm22
 ; AVX512DQBW-SLOW-NEXT:    movb $24, %sil
-; AVX512DQBW-SLOW-NEXT:    kmovd %esi, %k2
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm3, %zmm9, %zmm18
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm18, %zmm16 {%k2}
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm27 = <u,1,2,3,4,15,u,u>
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm11, %zmm16, %zmm27
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm24, %zmm12, %zmm11
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm30 = [0,1,12,7,0,1,12,7]
-; AVX512DQBW-SLOW-NEXT:    # zmm30 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm28, %zmm30, %zmm11
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm31 = [11,3,11,3,11,3,11,3]
+; AVX512DQBW-SLOW-NEXT:    kmovd %esi, %k1
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm21, %zmm22 {%k1}
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm18 = <u,1,2,3,4,15,u,u>
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm16, %zmm22, %zmm18
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm21 = <u,1,2,3,4,5,15,u>
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm30, %zmm18, %zmm21
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm18 = [14,1,2,3,4,5,6,15]
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm27, %zmm21, %zmm18
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm21 = <13,u,2,3,4,5,6,14>
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm30, %zmm19, %zmm21
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm19 = [0,13,2,3,4,5,6,7]
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm27, %zmm21, %zmm19
+; AVX512DQBW-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm31 = [0,8,0,8,0,8,0,8]
 ; AVX512DQBW-SLOW-NEXT:    # zmm31 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm7, %zmm31, %zmm0
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm18 = [2,10,0,3,2,10,0,3]
-; AVX512DQBW-SLOW-NEXT:    # zmm18 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm9, %zmm16
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm3, %zmm18, %zmm16
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm16 {%k1}
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm29 = [9,1,9,1,9,1,9,1]
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm20, %zmm26
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm28, %zmm31, %zmm26
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm12, %zmm10, %zmm2
+; AVX512DQBW-SLOW-NEXT:    movb $96, %sil
+; AVX512DQBW-SLOW-NEXT:    kmovd %esi, %k2
+; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm21 = [4,12,0,5,4,12,0,5]
+; AVX512DQBW-SLOW-NEXT:    # zmm21 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm13, %zmm25
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm1, %zmm9, %zmm25
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm13, %zmm24
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm1, %zmm17, %zmm24
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm13, %zmm22
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm1, %zmm10, %zmm22
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm1, %zmm13, %zmm11
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm13, %zmm29, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm16, %zmm13
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm30, %zmm21, %zmm13
+; AVX512DQBW-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm29 = [11,3,11,3,11,3,11,3]
 ; AVX512DQBW-SLOW-NEXT:    # zmm29 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [5,0,14,6,5,0,14,6]
-; AVX512DQBW-SLOW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm14, %zmm25, %zmm0
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm25, %zmm14, %zmm12
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm25, %zmm29, %zmm14
-; AVX512DQBW-SLOW-NEXT:    vmovdqa 64(%r9), %ymm5
-; AVX512DQBW-SLOW-NEXT:    vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm24, %zmm29, %zmm20
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm25 = <u,1,2,3,4,5,15,u>
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm24, %zmm27, %zmm25
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm1 = <13,u,2,3,4,5,6,14>
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm24, %zmm23, %zmm1
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 64(%r8), %ymm23
-; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm24 = ymm23[0],ymm5[0],ymm23[2],ymm5[2]
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm12, %zmm29, %zmm23
+; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm12 = [0,1,12,7,0,1,12,7]
+; AVX512DQBW-SLOW-NEXT:    # zmm12 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm27, %zmm12, %zmm13
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm23, %zmm8 {%k2}
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm16, %zmm3
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm30, %zmm31, %zmm16
+; AVX512DQBW-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm31 = [9,1,9,1,9,1,9,1]
+; AVX512DQBW-SLOW-NEXT:    # zmm31 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm30, %zmm31, %zmm3
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 64(%r9), %ymm23
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm7, %zmm6, %zmm17
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm7, %zmm6, %zmm10
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm7, %zmm6, %zmm29
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm6, %zmm7, %zmm15
+; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm1 {%k3} = zmm6[0],zmm7[0],zmm6[2],zmm7[2],zmm6[4],zmm7[4],zmm6[6],zmm7[6]
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm7, %zmm9, %zmm6
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm28, %zmm20, %zmm9
+; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm7 = [5,0,14,6,5,0,14,6]
+; AVX512DQBW-SLOW-NEXT:    # zmm7 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm20, %zmm28, %zmm7
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm28, %zmm20, %zmm21
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm28, %zmm31, %zmm20
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 64(%r8), %ymm28
+; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm30 = ymm28[0],ymm23[0],ymm28[2],ymm23[2]
 ; AVX512DQBW-SLOW-NEXT:    movb $28, %sil
 ; AVX512DQBW-SLOW-NEXT:    kmovd %esi, %k3
-; AVX512DQBW-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm16 {%k3} = zmm24[2,3,2,3],zmm28[2,3,2,3]
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm29 = [6,13,14,7,6,13,14,7]
-; AVX512DQBW-SLOW-NEXT:    # zmm29 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm26, %zmm22, %zmm29
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm27 = [0,13,6,7,0,13,6,7]
-; AVX512DQBW-SLOW-NEXT:    # zmm27 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm22, %zmm0, %zmm27
-; AVX512DQBW-SLOW-NEXT:    vmovdqa (%r9), %ymm5
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm22, %zmm30, %zmm12
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 (%r8), %ymm26
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm13, %zmm4, %zmm31
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm2, %zmm8, %zmm18
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm31, %zmm18 {%k1}
-; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm24 = ymm26[0],ymm5[0],ymm26[2],ymm5[2]
-; AVX512DQBW-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm18 {%k3} = zmm24[2,3,2,3],zmm22[2,3,2,3]
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm24 = [4,9,0,3,4,9,0,3]
-; AVX512DQBW-SLOW-NEXT:    # zmm24 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm22, %zmm24, %zmm14
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm9, %zmm0
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm3, %zmm15, %zmm0
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm28, %zmm24, %zmm20
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm22 = [14,1,2,3,4,5,6,15]
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm28, %zmm25, %zmm22
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm24 = [0,13,2,3,4,5,6,7]
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm28, %zmm1, %zmm24
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm28 = [1,0,10,2,1,0,10,2]
-; AVX512DQBW-SLOW-NEXT:    # zmm28 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm25 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm2, %zmm8, %zmm25
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm1 = [3,0,12,4,3,0,12,4]
-; AVX512DQBW-SLOW-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm9, %zmm1, %zmm3
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm31
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm2, %zmm15, %zmm31
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm8, %zmm1, %zmm2
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm9
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm6, %zmm28, %zmm9
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm30 = [0,8,0,1,0,8,0,1]
+; AVX512DQBW-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm8 {%k3} = zmm30[2,3,2,3],zmm27[2,3,2,3]
+; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm30 = [4,9,0,3,4,9,0,3]
 ; AVX512DQBW-SLOW-NEXT:    # zmm30 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm8
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm13, %zmm30, %zmm8
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm7, %zmm6, %zmm30
-; AVX512DQBW-SLOW-NEXT:    movb $48, %sil
-; AVX512DQBW-SLOW-NEXT:    kmovd %esi, %k3
-; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm3 {%k3} = zmm6[0],zmm7[0],zmm6[2],zmm7[2],zmm6[4],zmm7[4],zmm6[6],zmm7[6]
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm7, %zmm15, %zmm6
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti128 {{.*#+}} ymm7 = [6,14,6,14]
-; AVX512DQBW-SLOW-NEXT:    # ymm7 = mem[0,1,0,1]
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm13, %zmm4, %zmm7
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm13, %zmm4, %zmm15
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm4, %zmm13, %zmm28
-; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm2 {%k3} = zmm4[0],zmm13[0],zmm4[2],zmm13[2],zmm4[4],zmm13[4],zmm4[6],zmm13[6]
-; AVX512DQBW-SLOW-NEXT:    vmovdqa (%rdx), %xmm1
-; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
-; AVX512DQBW-SLOW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm27, %zmm30, %zmm3
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 (%rax), %zmm27
+; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm31 = [6,13,14,7,6,13,14,7]
+; AVX512DQBW-SLOW-NEXT:    # zmm31 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm9, %zmm27, %zmm31
+; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm9 = [0,13,6,7,0,13,6,7]
+; AVX512DQBW-SLOW-NEXT:    # zmm9 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm27, %zmm7, %zmm9
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm27, %zmm12, %zmm21
+; AVX512DQBW-SLOW-NEXT:    vmovdqa (%r9), %ymm7
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm29, %zmm11 {%k2}
+; AVX512DQBW-SLOW-NEXT:    vmovdqa (%r8), %ymm12
+; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm29 = ymm12[0],ymm7[0],ymm12[2],ymm7[2]
+; AVX512DQBW-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm11 {%k3} = zmm29[2,3,2,3],zmm27[2,3,2,3]
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm27, %zmm30, %zmm20
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 (%rdx), %xmm27
+; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm27 = xmm27[0],mem[0]
+; AVX512DQBW-SLOW-NEXT:    vinserti32x4 $1, %xmm27, %ymm0, %ymm27
 ; AVX512DQBW-SLOW-NEXT:    movb $12, %sil
 ; AVX512DQBW-SLOW-NEXT:    kmovd %esi, %k5
-; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm8 {%k5}
+; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm27, %zmm0, %zmm4 {%k5}
 ; AVX512DQBW-SLOW-NEXT:    movb $112, %sil
 ; AVX512DQBW-SLOW-NEXT:    kmovd %esi, %k7
-; AVX512DQBW-SLOW-NEXT:    vinserti64x2 $3, (%rax), %zmm21, %zmm8 {%k7}
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm6 {%k1}
+; AVX512DQBW-SLOW-NEXT:    vinserti64x2 $3, (%rax), %zmm26, %zmm4 {%k7}
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm26 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm26, %zmm2 {%k2}
 ; AVX512DQBW-SLOW-NEXT:    movb $120, %sil
 ; AVX512DQBW-SLOW-NEXT:    kmovd %esi, %k3
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm11 {%k3}
-; AVX512DQBW-SLOW-NEXT:    vpbroadcastq 72(%rcx), %ymm0
-; AVX512DQBW-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm13 {%k3}
+; AVX512DQBW-SLOW-NEXT:    vpbroadcastq 72(%rcx), %ymm2
+; AVX512DQBW-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = mem[0,1,2,3],ymm2[4,5,6,7]
 ; AVX512DQBW-SLOW-NEXT:    movb $6, %sil
 ; AVX512DQBW-SLOW-NEXT:    kmovd %esi, %k4
-; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm9 {%k4}
+; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm2, %zmm0, %zmm14 {%k4}
 ; AVX512DQBW-SLOW-NEXT:    movb $56, %sil
 ; AVX512DQBW-SLOW-NEXT:    kmovd %esi, %k6
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm20, %zmm9 {%k6}
-; AVX512DQBW-SLOW-NEXT:    vmovdqa 64(%rdx), %xmm0
-; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
-; AVX512DQBW-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm30 {%k5}
-; AVX512DQBW-SLOW-NEXT:    vinserti64x2 $3, 64(%rax), %zmm17, %zmm30 {%k7}
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm19, %zmm10 {%k2}
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm14 {%k6}
+; AVX512DQBW-SLOW-NEXT:    vmovdqa 64(%rdx), %xmm2
+; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],mem[0]
+; AVX512DQBW-SLOW-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm2, %zmm0, %zmm5 {%k5}
+; AVX512DQBW-SLOW-NEXT:    vinserti64x2 $3, 64(%rax), %zmm16, %zmm5 {%k7}
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm25, %zmm6 {%k1}
 ; AVX512DQBW-SLOW-NEXT:    movb $-31, %sil
-; AVX512DQBW-SLOW-NEXT:    kmovd %esi, %k2
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm29, %zmm10 {%k2}
-; AVX512DQBW-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm7[0,1,2,3],zmm25[4,5,6,7]
+; AVX512DQBW-SLOW-NEXT:    kmovd %esi, %k1
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm31, %zmm6 {%k1}
+; AVX512DQBW-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm17[0,1,2,3],zmm24[4,5,6,7]
 ; AVX512DQBW-SLOW-NEXT:    movb $-61, %sil
-; AVX512DQBW-SLOW-NEXT:    kmovd %esi, %k2
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm27, %zmm0 {%k2}
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm31, %zmm15 {%k1}
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm15, %zmm12 {%k3}
-; AVX512DQBW-SLOW-NEXT:    vpbroadcastq 8(%rcx), %ymm1
-; AVX512DQBW-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = mem[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm28 {%k4}
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm28 {%k6}
-; AVX512DQBW-SLOW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm23, %ymm1 # 32-byte Folded Reload
-; AVX512DQBW-SLOW-NEXT:    # ymm1 = ymm23[1],mem[1],ymm23[3],mem[3]
-; AVX512DQBW-SLOW-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,3,3]
-; AVX512DQBW-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],mem[6,7]
+; AVX512DQBW-SLOW-NEXT:    kmovd %esi, %k1
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm9, %zmm2 {%k1}
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm22, %zmm10 {%k2}
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm10, %zmm21 {%k3}
+; AVX512DQBW-SLOW-NEXT:    vpbroadcastq 8(%rcx), %ymm3
+; AVX512DQBW-SLOW-NEXT:    vpblendd {{.*#+}} ymm3 = mem[0,1,2,3],ymm3[4,5,6,7]
+; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm15 {%k4}
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm20, %zmm15 {%k6}
+; AVX512DQBW-SLOW-NEXT:    vpunpckhqdq {{.*#+}} ymm3 = ymm28[1],ymm23[1],ymm28[3],ymm23[3]
+; AVX512DQBW-SLOW-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,2,3,3]
+; AVX512DQBW-SLOW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5],mem[6,7]
 ; AVX512DQBW-SLOW-NEXT:    movb $14, %cl
 ; AVX512DQBW-SLOW-NEXT:    kmovd %ecx, %k1
-; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm3 {%k1}
-; AVX512DQBW-SLOW-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm26[1],ymm5[1],ymm26[3],ymm5[3]
-; AVX512DQBW-SLOW-NEXT:    vpermq {{.*#+}} ymm1 = ymm1[0,2,3,3]
-; AVX512DQBW-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],mem[6,7]
-; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm2 {%k1}
+; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm0 {%k1}
+; AVX512DQBW-SLOW-NEXT:    vpunpckhqdq {{.*#+}} ymm3 = ymm12[1],ymm7[1],ymm12[3],ymm7[3]
+; AVX512DQBW-SLOW-NEXT:    vpermq {{.*#+}} ymm3 = ymm3[0,2,3,3]
+; AVX512DQBW-SLOW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5],mem[6,7]
+; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm1 {%k1}
 ; AVX512DQBW-SLOW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm28, 64(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm18, 128(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm2, 192(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm12, 256(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, 320(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm10, 384(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm30, 448(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm9, 512(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm16, 576(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm3, 640(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm11, 704(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm8, (%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm24, 768(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm22, 832(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm15, 64(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm11, 128(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, 192(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm21, 256(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm2, 320(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm6, 384(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm5, 448(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm14, 512(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm8, 576(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, 640(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm13, 704(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm4, (%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm19, 768(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm18, 832(%rax)
 ; AVX512DQBW-SLOW-NEXT:    vzeroupper
 ; AVX512DQBW-SLOW-NEXT:    retq
 ;
 ; AVX512DQBW-FAST-LABEL: store_i64_stride7_vf16:
 ; AVX512DQBW-FAST:       # %bb.0:
 ; AVX512DQBW-FAST-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 (%rdi), %zmm13
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 64(%rdi), %zmm3
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 (%rsi), %zmm0
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 64(%rsi), %zmm5
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 (%rdx), %zmm1
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 64(%rdx), %zmm7
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 (%rcx), %zmm16
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 64(%rcx), %zmm9
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 64(%r8), %zmm30
-; AVX512DQBW-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm20 = [6,14,6,14,6,14,6,14]
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 (%rdi), %zmm10
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 64(%rdi), %zmm0
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 (%rsi), %zmm2
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 64(%rsi), %zmm13
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 (%rdx), %zmm3
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 64(%rdx), %zmm1
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 (%rcx), %zmm14
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 64(%rcx), %zmm15
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 64(%r8), %zmm21
+; AVX512DQBW-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm18 = [6,14,6,14,6,14,6,14]
+; AVX512DQBW-FAST-NEXT:    # zmm18 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm1, %zmm4
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm15, %zmm18, %zmm4
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm5
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm13, %zmm18, %zmm5
+; AVX512DQBW-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm4 = zmm5[0,1,2,3],zmm4[4,5,6,7]
+; AVX512DQBW-FAST-NEXT:    movb $64, %sil
+; AVX512DQBW-FAST-NEXT:    kmovd %esi, %k1
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm21, %zmm4 {%k1}
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 (%r8), %zmm7
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 (%r9), %zmm12
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 64(%r9), %zmm6
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 (%rax), %zmm9
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 64(%rax), %zmm5
+; AVX512DQBW-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm20 = [15,7,15,7,15,7,15,7]
 ; AVX512DQBW-FAST-NEXT:    # zmm20 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm7, %zmm17
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm9, %zmm20, %zmm17
-; AVX512DQBW-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} ymm21 = [6,14,6,14]
-; AVX512DQBW-FAST-NEXT:    # ymm21 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm3, %zmm18
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm5, %zmm21, %zmm18
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 (%r8), %zmm11
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 64(%r9), %zmm10
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 (%r9), %zmm15
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 (%rax), %zmm14
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 64(%rax), %zmm31
-; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [0,8,0,1,0,8,0,1]
-; AVX512DQBW-FAST-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm13, %zmm2
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm0, %zmm4, %zmm2
-; AVX512DQBW-FAST-NEXT:    vmovdqa (%rdx), %xmm12
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 64(%rdx), %xmm22
-; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm12 = xmm12[0],mem[0]
-; AVX512DQBW-FAST-NEXT:    vinserti128 $1, %xmm12, %ymm0, %ymm12
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm1, %zmm8
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm15, %zmm20, %zmm8
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm16
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm13, %zmm20, %zmm16
+; AVX512DQBW-FAST-NEXT:    movb $24, %sil
+; AVX512DQBW-FAST-NEXT:    kmovd %esi, %k1
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm8, %zmm16 {%k1}
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm31 = <u,1,2,3,4,15,u,u>
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm21, %zmm16, %zmm31
+; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm22 = [0,8,0,1,0,8,0,1]
+; AVX512DQBW-FAST-NEXT:    # zmm22 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm10, %zmm8
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm2, %zmm22, %zmm8
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 (%rdx), %xmm16
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 64(%rdx), %xmm27
+; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm16 = xmm16[0],mem[0]
+; AVX512DQBW-FAST-NEXT:    vinserti32x4 $1, %xmm16, %ymm0, %ymm16
 ; AVX512DQBW-FAST-NEXT:    movb $12, %sil
-; AVX512DQBW-FAST-NEXT:    kmovd %esi, %k2
-; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm12, %zmm0, %zmm2 {%k2}
-; AVX512DQBW-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm24 = [0,8,0,8,0,8,0,8]
-; AVX512DQBW-FAST-NEXT:    # zmm24 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm11, %zmm12
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm15, %zmm24, %zmm12
+; AVX512DQBW-FAST-NEXT:    kmovd %esi, %k3
+; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm16, %zmm0, %zmm8 {%k3}
+; AVX512DQBW-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm26 = [0,8,0,8,0,8,0,8]
+; AVX512DQBW-FAST-NEXT:    # zmm26 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm7, %zmm28
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm12, %zmm26, %zmm28
 ; AVX512DQBW-FAST-NEXT:    movb $112, %sil
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm24
+; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm17 = [3,0,12,4,3,0,12,4]
+; AVX512DQBW-FAST-NEXT:    # zmm17 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm15, %zmm16
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm1, %zmm17, %zmm16
+; AVX512DQBW-FAST-NEXT:    movb $48, %dil
+; AVX512DQBW-FAST-NEXT:    kmovd %edi, %k2
+; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm16 {%k2} = zmm0[0],zmm13[0],zmm0[2],zmm13[2],zmm0[4],zmm13[4],zmm0[6],zmm13[6]
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm23
+; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm25 = [1,0,10,2,1,0,10,2]
+; AVX512DQBW-FAST-NEXT:    # zmm25 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm13, %zmm19
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm0, %zmm25, %zmm19
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm13, %zmm22, %zmm0
+; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm22 = xmm27[0],mem[0]
+; AVX512DQBW-FAST-NEXT:    vinserti32x4 $1, %xmm22, %ymm0, %ymm22
+; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm22, %zmm0, %zmm0 {%k3}
 ; AVX512DQBW-FAST-NEXT:    kmovd %esi, %k3
-; AVX512DQBW-FAST-NEXT:    vinserti64x2 $3, (%rax), %zmm12, %zmm2 {%k3}
-; AVX512DQBW-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm23 = [13,5,13,5,13,5,13,5]
-; AVX512DQBW-FAST-NEXT:    # zmm23 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm7, %zmm12
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm9, %zmm23, %zmm12
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm3, %zmm25
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm5, %zmm23, %zmm25
+; AVX512DQBW-FAST-NEXT:    vinserti64x2 $3, (%rax), %zmm28, %zmm8 {%k3}
+; AVX512DQBW-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm28 = [13,5,13,5,13,5,13,5]
+; AVX512DQBW-FAST-NEXT:    # zmm28 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm1, %zmm29
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm15, %zmm28, %zmm29
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm13, %zmm28, %zmm24
 ; AVX512DQBW-FAST-NEXT:    movb $96, %sil
-; AVX512DQBW-FAST-NEXT:    kmovd %esi, %k1
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm12, %zmm25 {%k1}
-; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm12 = [4,12,0,5,4,12,0,5]
-; AVX512DQBW-FAST-NEXT:    # zmm12 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm10, %zmm30, %zmm24
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm30, %zmm19
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm10, %zmm12, %zmm19
-; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm27 = [0,1,12,7,0,1,12,7]
-; AVX512DQBW-FAST-NEXT:    # zmm27 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm31, %zmm27, %zmm19
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm21, %zmm22
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm21, %zmm27
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm6, %zmm26, %zmm21
+; AVX512DQBW-FAST-NEXT:    vinserti64x2 $3, 64(%rax), %zmm21, %zmm0 {%k3}
+; AVX512DQBW-FAST-NEXT:    kmovd %esi, %k3
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm29, %zmm24 {%k3}
+; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm21 = [4,12,0,5,4,12,0,5]
+; AVX512DQBW-FAST-NEXT:    # zmm21 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm6, %zmm21, %zmm22
+; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm29 = [0,1,12,7,0,1,12,7]
+; AVX512DQBW-FAST-NEXT:    # zmm29 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm5, %zmm29, %zmm22
 ; AVX512DQBW-FAST-NEXT:    movb $120, %sil
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm5, %zmm3, %zmm4
-; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm22 = xmm22[0],mem[0]
-; AVX512DQBW-FAST-NEXT:    vinserti32x4 $1, %xmm22, %ymm0, %ymm22
-; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm22, %zmm0, %zmm4 {%k2}
-; AVX512DQBW-FAST-NEXT:    vinserti64x2 $3, 64(%rax), %zmm24, %zmm4 {%k3}
-; AVX512DQBW-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm24 = [15,7,15,7,15,7,15,7]
-; AVX512DQBW-FAST-NEXT:    # zmm24 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm11, %zmm22
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm15, %zmm24, %zmm22
-; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm28 = [6,13,14,7,6,13,14,7]
-; AVX512DQBW-FAST-NEXT:    # zmm28 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm22, %zmm14, %zmm28
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm1, %zmm29
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm16, %zmm24, %zmm29
-; AVX512DQBW-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} ymm26 = [15,7,15,7]
-; AVX512DQBW-FAST-NEXT:    # ymm26 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm13, %zmm22
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm0, %zmm26, %zmm22
-; AVX512DQBW-FAST-NEXT:    movb $24, %dil
-; AVX512DQBW-FAST-NEXT:    kmovd %edi, %k2
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm29, %zmm22 {%k2}
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm3, %zmm30
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm14, %zmm20, %zmm30
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm10, %zmm26
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm2, %zmm20, %zmm26
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm30, %zmm26 {%k1}
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm12, %zmm7, %zmm20
+; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm30 = [6,13,14,7,6,13,14,7]
+; AVX512DQBW-FAST-NEXT:    # zmm30 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm20, %zmm9, %zmm30
 ; AVX512DQBW-FAST-NEXT:    movb $-31, %dil
-; AVX512DQBW-FAST-NEXT:    kmovd %edi, %k3
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm28, %zmm22 {%k3}
-; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm28 = [5,0,14,6,5,0,14,6]
-; AVX512DQBW-FAST-NEXT:    # zmm28 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm11, %zmm15, %zmm28
-; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm29 = [0,13,6,7,0,13,6,7]
-; AVX512DQBW-FAST-NEXT:    # zmm29 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm14, %zmm28, %zmm29
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm16, %zmm1, %zmm20
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm0, %zmm13, %zmm21
-; AVX512DQBW-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm20 = zmm21[0,1,2,3],zmm20[4,5,6,7]
+; AVX512DQBW-FAST-NEXT:    kmovd %edi, %k1
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm30, %zmm26 {%k1}
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm3, %zmm20
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm14, %zmm18, %zmm20
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm2, %zmm10, %zmm18
+; AVX512DQBW-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm18 = zmm18[0,1,2,3],zmm20[4,5,6,7]
+; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm20 = [5,0,14,6,5,0,14,6]
+; AVX512DQBW-FAST-NEXT:    # zmm20 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm7, %zmm12, %zmm20
+; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm30 = [0,13,6,7,0,13,6,7]
+; AVX512DQBW-FAST-NEXT:    # zmm30 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm9, %zmm20, %zmm30
 ; AVX512DQBW-FAST-NEXT:    movb $-61, %dil
-; AVX512DQBW-FAST-NEXT:    kmovd %edi, %k3
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm29, %zmm20 {%k3}
-; AVX512DQBW-FAST-NEXT:    kmovd %esi, %k3
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm25, %zmm19 {%k3}
-; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm25 = [3,0,12,4,3,0,12,4]
-; AVX512DQBW-FAST-NEXT:    # zmm25 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm9, %zmm21
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm7, %zmm25, %zmm21
-; AVX512DQBW-FAST-NEXT:    movb $48, %sil
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm1, %zmm28
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm16, %zmm23, %zmm28
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm0, %zmm13, %zmm23
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm28, %zmm23 {%k1}
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm15, %zmm11, %zmm12
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm14, %zmm27, %zmm12
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm23, %zmm12 {%k3}
-; AVX512DQBW-FAST-NEXT:    kmovd %esi, %k3
-; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm21 {%k3} = zmm3[0],zmm5[0],zmm3[2],zmm5[2],zmm3[4],zmm5[4],zmm3[6],zmm5[6]
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 (%r9), %ymm23
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 64(%r9), %ymm27
+; AVX512DQBW-FAST-NEXT:    kmovd %edi, %k1
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm30, %zmm18 {%k1}
+; AVX512DQBW-FAST-NEXT:    kmovd %esi, %k1
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm24, %zmm22 {%k1}
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm3, %zmm20
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm14, %zmm28, %zmm20
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm2, %zmm10, %zmm28
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm20, %zmm28 {%k3}
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm12, %zmm7, %zmm21
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm9, %zmm29, %zmm21
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm28, %zmm21 {%k1}
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 (%r9), %ymm20
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 64(%r9), %ymm24
 ; AVX512DQBW-FAST-NEXT:    vmovdqa64 64(%r8), %ymm28
-; AVX512DQBW-FAST-NEXT:    vmovdqa {{.*#+}} ymm6 = [1,3,7,7]
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %ymm28, %ymm8
-; AVX512DQBW-FAST-NEXT:    vpermt2q %ymm27, %ymm6, %ymm8
-; AVX512DQBW-FAST-NEXT:    vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],mem[6,7]
+; AVX512DQBW-FAST-NEXT:    vmovdqa {{.*#+}} ymm4 = [1,3,7,7]
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %ymm28, %ymm11
+; AVX512DQBW-FAST-NEXT:    vpermt2q %ymm24, %ymm4, %ymm11
+; AVX512DQBW-FAST-NEXT:    vpblendd {{.*#+}} ymm11 = ymm11[0,1,2,3,4,5],mem[6,7]
 ; AVX512DQBW-FAST-NEXT:    movb $14, %sil
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm1, %zmm16, %zmm25
-; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm25 {%k3} = zmm13[0],zmm0[0],zmm13[2],zmm0[2],zmm13[4],zmm0[4],zmm13[6],zmm0[6]
-; AVX512DQBW-FAST-NEXT:    kmovd %esi, %k3
-; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm8, %zmm0, %zmm21 {%k3}
-; AVX512DQBW-FAST-NEXT:    vmovdqa (%r8), %ymm8
-; AVX512DQBW-FAST-NEXT:    vpermi2q %ymm23, %ymm8, %ymm6
-; AVX512DQBW-FAST-NEXT:    vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3,4,5],mem[6,7]
-; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm6, %zmm0, %zmm25 {%k3}
-; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm6 = ymm28[0],ymm27[0],ymm28[2],ymm27[2]
-; AVX512DQBW-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm17 = zmm18[0,1,2,3],zmm17[4,5,6,7]
-; AVX512DQBW-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm18 = [11,3,11,3,11,3,11,3]
-; AVX512DQBW-FAST-NEXT:    # zmm18 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm27 = [2,10,0,3,2,10,0,3]
-; AVX512DQBW-FAST-NEXT:    # zmm27 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm16, %zmm27, %zmm1
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm7, %zmm16
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm9, %zmm27, %zmm16
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm3, %zmm27
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm5, %zmm18, %zmm27
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm27, %zmm16 {%k1}
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm0, %zmm13, %zmm18
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm18, %zmm1 {%k1}
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm3, %zmm14, %zmm17
+; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm17 {%k2} = zmm10[0],zmm2[0],zmm10[2],zmm2[2],zmm10[4],zmm2[4],zmm10[6],zmm2[6]
+; AVX512DQBW-FAST-NEXT:    kmovd %esi, %k1
+; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm11, %zmm0, %zmm16 {%k1}
+; AVX512DQBW-FAST-NEXT:    vmovdqa (%r8), %ymm11
+; AVX512DQBW-FAST-NEXT:    vpermi2q %ymm20, %ymm11, %ymm4
+; AVX512DQBW-FAST-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3,4,5],mem[6,7]
+; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm4, %zmm0, %zmm17 {%k1}
+; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [2,10,0,3,2,10,0,3]
+; AVX512DQBW-FAST-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm15, %zmm4, %zmm1
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm14, %zmm4, %zmm3
+; AVX512DQBW-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm4 = [11,3,11,3,11,3,11,3]
+; AVX512DQBW-FAST-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm13, %zmm4, %zmm23
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm2, %zmm10, %zmm4
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm10, %zmm25, %zmm2
+; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm10 = ymm28[0],ymm24[0],ymm28[2],ymm24[2]
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm23, %zmm1 {%k3}
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm4, %zmm3 {%k3}
 ; AVX512DQBW-FAST-NEXT:    movb $28, %al
 ; AVX512DQBW-FAST-NEXT:    kmovd %eax, %k1
-; AVX512DQBW-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm16 {%k1} = zmm6[2,3,2,3],zmm31[2,3,2,3]
-; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm6 = [1,0,10,2,1,0,10,2]
-; AVX512DQBW-FAST-NEXT:    # zmm6 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm13, %zmm6, %zmm0
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm5, %zmm13
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm3, %zmm6, %zmm13
-; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm6 = ymm8[0],ymm23[0],ymm8[2],ymm23[2]
-; AVX512DQBW-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm1 {%k1} = zmm6[2,3,2,3],zmm14[2,3,2,3]
-; AVX512DQBW-FAST-NEXT:    vpbroadcastq 72(%rcx), %ymm6
-; AVX512DQBW-FAST-NEXT:    vpblendd {{.*#+}} ymm6 = mem[0,1,2,3],ymm6[4,5,6,7]
+; AVX512DQBW-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm1 {%k1} = zmm10[2,3,2,3],zmm5[2,3,2,3]
+; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm4 = ymm11[0],ymm20[0],ymm11[2],ymm20[2]
+; AVX512DQBW-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm3 {%k1} = zmm4[2,3,2,3],zmm9[2,3,2,3]
+; AVX512DQBW-FAST-NEXT:    vpbroadcastq 72(%rcx), %ymm4
+; AVX512DQBW-FAST-NEXT:    vpblendd {{.*#+}} ymm4 = mem[0,1,2,3],ymm4[4,5,6,7]
 ; AVX512DQBW-FAST-NEXT:    movb $6, %al
 ; AVX512DQBW-FAST-NEXT:    kmovd %eax, %k1
-; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm6, %zmm0, %zmm13 {%k1}
-; AVX512DQBW-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm6 = [9,1,9,1,9,1,9,1]
-; AVX512DQBW-FAST-NEXT:    # zmm6 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm15, %zmm6, %zmm11
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm30, %zmm8
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm10, %zmm6, %zmm8
-; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm6 = [4,9,0,3,4,9,0,3]
-; AVX512DQBW-FAST-NEXT:    # zmm6 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm14, %zmm6, %zmm11
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm31, %zmm6, %zmm8
-; AVX512DQBW-FAST-NEXT:    vpbroadcastq 8(%rcx), %ymm6
-; AVX512DQBW-FAST-NEXT:    vpblendd {{.*#+}} ymm6 = mem[0,1,2,3],ymm6[4,5,6,7]
-; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm6, %zmm0, %zmm0 {%k1}
-; AVX512DQBW-FAST-NEXT:    movb $64, %al
-; AVX512DQBW-FAST-NEXT:    kmovd %eax, %k1
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm30, %zmm17 {%k1}
+; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm4, %zmm0, %zmm19 {%k1}
+; AVX512DQBW-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm4 = [9,1,9,1,9,1,9,1]
+; AVX512DQBW-FAST-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm12, %zmm4, %zmm7
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm6, %zmm4, %zmm27
+; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [4,9,0,3,4,9,0,3]
+; AVX512DQBW-FAST-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm9, %zmm4, %zmm7
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm5, %zmm4, %zmm27
+; AVX512DQBW-FAST-NEXT:    vpbroadcastq 8(%rcx), %ymm4
+; AVX512DQBW-FAST-NEXT:    vpblendd {{.*#+}} ymm4 = mem[0,1,2,3],ymm4[4,5,6,7]
+; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm4, %zmm0, %zmm2 {%k1}
 ; AVX512DQBW-FAST-NEXT:    movb $56, %al
 ; AVX512DQBW-FAST-NEXT:    kmovd %eax, %k1
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm8, %zmm13 {%k1}
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm11, %zmm0 {%k1}
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm9, %zmm24, %zmm7
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm5, %zmm26, %zmm3
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm7, %zmm3 {%k2}
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm5 = <u,1,2,3,4,15,u,u>
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm30, %zmm3, %zmm5
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm3 = <u,1,2,3,4,5,15,u>
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm10, %zmm5, %zmm3
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm5 = <13,u,2,3,4,5,6,14>
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm10, %zmm17, %zmm5
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm27, %zmm19 {%k1}
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm7, %zmm2 {%k1}
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm4 = <u,1,2,3,4,5,15,u>
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm6, %zmm31, %zmm4
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm7 = <13,u,2,3,4,5,6,14>
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm6, %zmm9, %zmm7
 ; AVX512DQBW-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [14,1,2,3,4,5,6,15]
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm31, %zmm3, %zmm6
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [0,13,2,3,4,5,6,7]
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm31, %zmm5, %zmm3
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm5, %zmm4, %zmm6
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [0,13,2,3,4,5,6,7]
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm5, %zmm7, %zmm4
 ; AVX512DQBW-FAST-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, 64(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm1, 128(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm25, 192(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm12, 256(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm20, 320(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm22, 384(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm4, 448(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm13, 512(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm16, 576(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm21, 640(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm19, 704(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm2, (%rax)
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm3, 768(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm2, 64(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm3, 128(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm17, 192(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm21, 256(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm18, 320(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm26, 384(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, 448(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm19, 512(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm1, 576(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm16, 640(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm22, 704(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm8, (%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm4, 768(%rax)
 ; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm6, 832(%rax)
 ; AVX512DQBW-FAST-NEXT:    vzeroupper
 ; AVX512DQBW-FAST-NEXT:    retq
@@ -5754,3539 +5708,3517 @@ define void @store_i64_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ;
 ; AVX512F-ONLY-SLOW-LABEL: store_i64_stride7_vf32:
 ; AVX512F-ONLY-SLOW:       # %bb.0:
-; AVX512F-ONLY-SLOW-NEXT:    subq $2120, %rsp # imm = 0x848
+; AVX512F-ONLY-SLOW-NEXT:    subq $2184, %rsp # imm = 0x888
 ; AVX512F-ONLY-SLOW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 (%rdi), %zmm8
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 64(%rdi), %zmm11
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 (%rsi), %zmm27
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 64(%rsi), %zmm13
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 64(%rdx), %zmm18
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 (%rdx), %zmm21
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 (%rcx), %zmm7
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 64(%rcx), %zmm25
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 (%rdi), %zmm17
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 64(%rdi), %zmm14
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 (%rsi), %zmm16
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 64(%rsi), %zmm23
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 64(%rdx), %zmm6
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 (%rdx), %zmm7
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 (%rcx), %zmm8
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 64(%rcx), %zmm29
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 (%r8), %zmm0
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 (%r9), %zmm4
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 (%r9), %zmm3
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 (%rax), %zmm1
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 64(%rax), %zmm5
 ; AVX512F-ONLY-SLOW-NEXT:    movb $96, %r10b
 ; AVX512F-ONLY-SLOW-NEXT:    kmovw %r10d, %k1
+; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm19 = [15,7,15,7,15,7,15,7]
+; AVX512F-ONLY-SLOW-NEXT:    # zmm19 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm2 = [9,1,9,1,9,1,9,1]
 ; AVX512F-ONLY-SLOW-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm6
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm4, %zmm2, %zmm6
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm14
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm3 = [4,9,0,3,4,9,0,3]
-; AVX512F-ONLY-SLOW-NEXT:    # zmm3 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm10
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm3, %zmm2, %zmm10
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm11
+; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [4,9,0,3,4,9,0,3]
+; AVX512F-ONLY-SLOW-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm3, %zmm6
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm15
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm3 = [11,3,11,3,11,3,11,3]
-; AVX512F-ONLY-SLOW-NEXT:    # zmm3 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm1
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm27, %zmm3, %zmm1
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm17
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm6 = [2,10,0,3,2,10,0,3]
-; AVX512F-ONLY-SLOW-NEXT:    # zmm6 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm21, %zmm3
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm7, %zmm6, %zmm3
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm16
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm3 {%k1}
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm4, %zmm10
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm12
+; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm15 = [11,3,11,3,11,3,11,3]
+; AVX512F-ONLY-SLOW-NEXT:    # zmm15 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm17, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm16, %zmm15, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm10 = [2,10,0,3,2,10,0,3]
+; AVX512F-ONLY-SLOW-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm4
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm8, %zmm10, %zmm4
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm10, %zmm22
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqa (%r9), %ymm10
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu %ymm10, (%rsp) # 32-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 64(%r9), %ymm6
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 64(%r9), %ymm13
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu %ymm13, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqa (%r8), %ymm1
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 64(%r8), %ymm12
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 64(%r8), %ymm18
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %ymm18, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm1 = ymm1[0],ymm10[0],ymm1[2],ymm10[2]
 ; AVX512F-ONLY-SLOW-NEXT:    movb $28, %r10b
 ; AVX512F-ONLY-SLOW-NEXT:    kmovw %r10d, %k2
-; AVX512F-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm3 {%k2} = zmm1[2,3,2,3],zmm2[2,3,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm1 = [4,12,0,5,4,12,0,5]
-; AVX512F-ONLY-SLOW-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm3
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm4, %zmm1, %zmm3
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm10
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm1 = [0,1,12,7,0,1,12,7]
-; AVX512F-ONLY-SLOW-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm2, %zmm1, %zmm3
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm22 = [5,0,14,6,5,0,14,6]
-; AVX512F-ONLY-SLOW-NEXT:    # zmm22 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm19
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm20
+; AVX512F-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm4 {%k2} = zmm1[2,3,2,3],zmm2[2,3,2,3]
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm22, %zmm19
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm3 = [0,13,6,7,0,13,6,7]
-; AVX512F-ONLY-SLOW-NEXT:    # zmm3 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm2, %zmm3, %zmm19
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm19
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm4 = [15,7,15,7,15,7,15,7]
-; AVX512F-ONLY-SLOW-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm20, %zmm4, %zmm0
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [6,13,14,7,6,13,14,7]
+; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm10 = [4,12,0,5,4,12,0,5]
+; AVX512F-ONLY-SLOW-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm3, %zmm10, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [0,1,12,7,0,1,12,7]
+; AVX512F-ONLY-SLOW-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm2, %zmm4, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm20 = [5,0,14,6,5,0,14,6]
+; AVX512F-ONLY-SLOW-NEXT:    # zmm20 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm21
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm20, %zmm21
+; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [0,13,6,7,0,13,6,7]
 ; AVX512F-ONLY-SLOW-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm2, %zmm19
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm0
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm13, %zmm17, %zmm0
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm18, %zmm19
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm25, %zmm16, %zmm19
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm19 {%k1}
-; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm12[0],ymm6[0],ymm12[2],ymm6[2]
-; AVX512F-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm19 {%k2} = zmm0[2,3,2,3],zmm5[2,3,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm2, %zmm21
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm3, %zmm19, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm3 = [6,13,14,7,6,13,14,7]
+; AVX512F-ONLY-SLOW-NEXT:    # zmm3 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm3, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm23, %zmm15, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm22, %zmm21
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm22, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm29, %zmm22, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
+; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm18[0],ymm13[0],ymm18[2],ymm13[2]
+; AVX512F-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm1 {%k2} = zmm0[2,3,2,3],zmm5[2,3,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 64(%r8), %zmm0
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 64(%r9), %zmm12
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm6
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm12, %zmm14, %zmm6
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm5, %zmm15, %zmm6
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm6
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm12, %zmm10, %zmm6
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm5, %zmm1, %zmm6
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm12, %zmm6
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 64(%r9), %zmm13
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm13, %zmm11, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm22, %zmm6
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm5, %zmm3, %zmm6
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm5, %zmm12, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm13, %zmm10, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm5, %zmm4, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm13, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm20, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm5, %zmm2, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm12, %zmm4, %zmm0
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm2, %zmm5
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm13, %zmm19, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm3, %zmm5
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 128(%rdi), %zmm6
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 128(%rsi), %zmm19
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm0
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm19, %zmm17, %zmm0
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 128(%rdx), %zmm5
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 128(%rdi), %zmm18
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 128(%rsi), %zmm22
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm18, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm22, %zmm15, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 128(%rdx), %zmm0
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 128(%rcx), %zmm28
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm12
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm28, %zmm16, %zmm12
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm16, %zmm17
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm12 {%k1}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 128(%rax), %zmm29
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 128(%r9), %ymm16
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %ymm16, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 128(%r8), %ymm0
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm16[0],ymm0[2],ymm16[2]
-; AVX512F-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm12 {%k2} = zmm0[2,3,2,3],zmm29[2,3,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 128(%r8), %zmm0
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 128(%r9), %zmm20
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm20, %zmm0, %zmm10
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm29, %zmm1, %zmm10
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm5
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm28, %zmm21, %zmm5
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm5 {%k1}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 128(%rax), %zmm30
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 128(%r9), %ymm13
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu %ymm13, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 128(%r8), %ymm1
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm1 = ymm1[0],ymm13[0],ymm1[2],ymm13[2]
+; AVX512F-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm5 {%k2} = zmm1[2,3,2,3],zmm30[2,3,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 128(%r8), %zmm5
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 128(%r9), %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm1, %zmm5, %zmm10
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm30, %zmm4, %zmm10
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm0, %zmm20, %zmm22
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm29, %zmm3, %zmm22
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm22, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm1
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm20, %zmm14, %zmm1
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm29, %zmm15, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm4
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm5, %zmm1, %zmm20
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm30, %zmm2, %zmm20
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm20, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm4, %zmm11, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm30, %zmm12, %zmm1
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm20, %zmm4, %zmm0
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm2, %zmm29
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm4, %zmm19, %zmm5
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm5, %zmm3, %zmm30
 ; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm3 = [13,5,13,5,13,5,13,5]
 ; AVX512F-ONLY-SLOW-NEXT:    # zmm3 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm21, %zmm0
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm7, %zmm3, %zmm0
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm24 = [6,14,6,14,6,14,6,14]
-; AVX512F-ONLY-SLOW-NEXT:    # zmm24 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm21, %zmm0
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm7, %zmm24, %zmm0
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm21, %zmm0
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm7, %zmm4, %zmm0
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [3,0,12,4,3,0,12,4]
-; AVX512F-ONLY-SLOW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm21, %zmm0, %zmm7
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm18, %zmm1
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm25, %zmm3, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm8, %zmm3, %zmm1
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm18, %zmm1
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm25, %zmm24, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm12 = [6,14,6,14,6,14,6,14]
+; AVX512F-ONLY-SLOW-NEXT:    # zmm12 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm8, %zmm12, %zmm1
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm18, %zmm1
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm25, %zmm4, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm8, %zmm19, %zmm1
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm18, %zmm0, %zmm25
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 192(%rdx), %zmm18
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 192(%rcx), %zmm1
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm18, %zmm2
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm24, %zmm2
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm26
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm28, %zmm3, %zmm26
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm28, %zmm5, %zmm24
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm30
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm28, %zmm4, %zmm30
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm5, %zmm0, %zmm28
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm21
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm18, %zmm0, %zmm21
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm1, %zmm18, %zmm17
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm18, %zmm0
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm4, %zmm18
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm3, %zmm0
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    movb $48, %r10b
-; AVX512F-ONLY-SLOW-NEXT:    kmovw %r10d, %k3
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [1,0,10,2,1,0,10,2]
-; AVX512F-ONLY-SLOW-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm27, %zmm15
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm8, %zmm4, %zmm15
-; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm7 {%k3} = zmm8[0],zmm27[0],zmm8[2],zmm27[2],zmm8[4],zmm27[4],zmm8[6],zmm27[6]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm7
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm14
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm16
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm17
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm5 = [0,8,0,1,0,8,0,1]
-; AVX512F-ONLY-SLOW-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm27, %zmm5, %zmm17
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm27, %zmm3, %zmm7
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [6,14,6,14]
-; AVX512F-ONLY-SLOW-NEXT:    # ymm1 = mem[0,1,0,1]
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm27, %zmm1, %zmm14
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti128 {{.*#+}} ymm10 = [15,7,15,7]
-; AVX512F-ONLY-SLOW-NEXT:    # ymm10 = mem[0,1,0,1]
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm27, %zmm10, %zmm16
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm13, %zmm27
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm11, %zmm4, %zmm27
-; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm25 {%k3} = zmm11[0],zmm13[0],zmm11[2],zmm13[2],zmm11[4],zmm13[4],zmm11[6],zmm13[6]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm2
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm12
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm31
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm13, %zmm5, %zmm11
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm13, %zmm3, %zmm2
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm13, %zmm1, %zmm12
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm13, %zmm10, %zmm31
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 192(%rdi), %zmm22
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 192(%rsi), %zmm0
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm22, %zmm23
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm1, %zmm23
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm22, %zmm13
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm3, %zmm13
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm19, %zmm8
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm6, %zmm4, %zmm8
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm19, %zmm6, %zmm3
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm19, %zmm6, %zmm1
-; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm28 {%k3} = zmm6[0],zmm19[0],zmm6[2],zmm19[2],zmm6[4],zmm19[4],zmm6[6],zmm19[6]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm9
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm19, %zmm5, %zmm6
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm19, %zmm10, %zmm9
-; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm21 {%k3} = zmm22[0],zmm0[0],zmm22[2],zmm0[2],zmm22[4],zmm0[4],zmm22[6],zmm0[6]
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm0, %zmm22, %zmm5
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm22, %zmm0, %zmm4
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm0, %zmm22, %zmm19
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm10, %zmm22
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa (%rdx), %xmm0
-; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
-; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512F-ONLY-SLOW-NEXT:    movb $12, %sil
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm19, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [3,0,12,4,3,0,12,4]
+; AVX512F-ONLY-SLOW-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm7, %zmm2, %zmm8
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm31
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm4
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm29, %zmm3, %zmm4
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm4
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm29, %zmm12, %zmm4
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm4
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm29, %zmm19, %zmm4
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm6, %zmm2, %zmm29
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm4
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm28, %zmm3, %zmm4
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm4
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm28, %zmm12, %zmm4
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm4
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm28, %zmm19, %zmm4
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm2, %zmm28
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 192(%rdx), %zmm10
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 192(%rcx), %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm24
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm10, %zmm2, %zmm24
+; AVX512F-ONLY-SLOW-NEXT:    movb $48, %r10b
+; AVX512F-ONLY-SLOW-NEXT:    kmovw %r10d, %k3
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 192(%rdi), %zmm7
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 192(%rsi), %zmm2
+; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm24 {%k3} = zmm7[0],zmm2[0],zmm7[2],zmm2[2],zmm7[4],zmm2[4],zmm7[6],zmm2[6]
+; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm11 = [1,0,10,2,1,0,10,2]
+; AVX512F-ONLY-SLOW-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm16, %zmm26
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm17, %zmm11, %zmm26
+; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm13 = [0,8,0,1,0,8,0,1]
+; AVX512F-ONLY-SLOW-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm17, %zmm27
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm16, %zmm13, %zmm27
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm23, %zmm4
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm23, %zmm21
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm14, %zmm11, %zmm21
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm23
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm4, %zmm13, %zmm23
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm8
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm22, %zmm19
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm18, %zmm11, %zmm19
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm18, %zmm20
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm22, %zmm13, %zmm20
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm2, %zmm7, %zmm13
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm7, %zmm2, %zmm11
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm25
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm5
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm6
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm2, %zmm15, %zmm7
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm2, %zmm12, %zmm25
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm2, %zmm3, %zmm5
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm2, %zmm1, %zmm6
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm16, %zmm2
+; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm31 {%k3} = zmm17[0],zmm16[0],zmm17[2],zmm16[2],zmm17[4],zmm16[4],zmm17[6],zmm16[6]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm31, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm17, %zmm4
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm17, %zmm16
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm17, %zmm15
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm2, %zmm3, %zmm4
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm2, %zmm12, %zmm16
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm2, %zmm1, %zmm15
+; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm29 {%k3} = zmm14[0],zmm8[0],zmm14[2],zmm8[2],zmm14[4],zmm8[4],zmm14[6],zmm8[6]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm2
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm17
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm8, %zmm3, %zmm2
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm8, %zmm12, %zmm17
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm8, %zmm1, %zmm14
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm10, %zmm8
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm3, %zmm8
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm22, %zmm18, %zmm3
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm10, %zmm9
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm12, %zmm9
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm22, %zmm18, %zmm12
+; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm28 {%k3} = zmm18[0],zmm22[0],zmm18[2],zmm22[2],zmm18[4],zmm22[4],zmm18[6],zmm22[6]
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm22, %zmm1, %zmm18
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm10, %zmm22
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm1, %zmm22
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm1, %zmm10
+; AVX512F-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm31 = zmm25[0,1,2,3],zmm9[4,5,6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 192(%r8), %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm9 = <0,11,u,u,4,5,6,7>
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm0, %zmm24, %zmm9
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm5 {%k1}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm25 = <12,u,u,3,4,5,6,13>
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm0, %zmm5, %zmm25
+; AVX512F-ONLY-SLOW-NEXT:    movb $24, %sil
 ; AVX512F-ONLY-SLOW-NEXT:    kmovw %esi, %k3
-; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm17 {%k3}
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [0,8,0,8,0,8,0,8]
-; AVX512F-ONLY-SLOW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm10 # 64-byte Folded Reload
-; AVX512F-ONLY-SLOW-NEXT:    vinserti32x4 $3, (%rax), %zmm10, %zmm10
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm22, %zmm6 {%k3}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm5 = <u,1,2,3,4,15,u,u>
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm0, %zmm6, %zmm5
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa (%rdx), %xmm6
+; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm6 = xmm6[0],mem[0]
+; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
+; AVX512F-ONLY-SLOW-NEXT:    movb $12, %sil
+; AVX512F-ONLY-SLOW-NEXT:    kmovw %esi, %k4
+; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm6, %zmm0, %zmm27 {%k4}
+; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm6 = [0,8,0,8,0,8,0,8]
+; AVX512F-ONLY-SLOW-NEXT:    # zmm6 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm22 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm6, %zmm22 # 64-byte Folded Reload
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 64(%rdx), %xmm8
+; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm8 = xmm8[0],mem[0]
+; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm8, %ymm0, %ymm8
+; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm8, %zmm0, %zmm23 {%k4}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 128(%rdx), %xmm8
+; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm8 = xmm8[0],mem[0]
+; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm8, %ymm0, %ymm8
+; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm8, %zmm0, %zmm20 {%k4}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 192(%rdx), %xmm8
+; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm8 = xmm8[0],mem[0]
+; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm8, %ymm0, %ymm8
+; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm8, %zmm0, %zmm13 {%k4}
+; AVX512F-ONLY-SLOW-NEXT:    vinserti32x4 $3, (%rax), %zmm22, %zmm8
 ; AVX512F-ONLY-SLOW-NEXT:    movb $112, %sil
 ; AVX512F-ONLY-SLOW-NEXT:    kmovw %esi, %k4
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm10, %zmm17 {%k4}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 64(%rdx), %xmm10
-; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm10 = xmm10[0],mem[0]
-; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
-; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm10, %zmm0, %zmm11 {%k3}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm10 # 64-byte Folded Reload
-; AVX512F-ONLY-SLOW-NEXT:    vinserti32x4 $3, 64(%rax), %zmm10, %zmm10
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm10, %zmm11 {%k4}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 128(%rdx), %xmm10
-; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm10 = xmm10[0],mem[0]
-; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
-; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm10, %zmm0, %zmm6 {%k3}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm20, %zmm0, %zmm10
-; AVX512F-ONLY-SLOW-NEXT:    vinserti32x4 $3, 128(%rax), %zmm10, %zmm10
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm10, %zmm6 {%k4}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 192(%rdx), %xmm10
-; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm10 = xmm10[0],mem[0]
-; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
-; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm10, %zmm0, %zmm5 {%k3}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 192(%r8), %zmm19
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 192(%r9), %zmm20
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm20, %zmm19, %zmm0
-; AVX512F-ONLY-SLOW-NEXT:    vinserti32x4 $3, 192(%rax), %zmm0, %zmm0
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm5 {%k4}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm7 {%k1}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm27 {%k4}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm6, %zmm1 # 64-byte Folded Reload
+; AVX512F-ONLY-SLOW-NEXT:    vinserti32x4 $3, 64(%rax), %zmm1, %zmm8
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm23 {%k4}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm6, %zmm1 # 64-byte Folded Reload
+; AVX512F-ONLY-SLOW-NEXT:    vinserti32x4 $3, 128(%rax), %zmm1, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm20 {%k4}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 192(%r9), %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm1, %zmm0, %zmm6
+; AVX512F-ONLY-SLOW-NEXT:    vinserti32x4 $3, 192(%rax), %zmm6, %zmm6
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm13 {%k4}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm4 {%k1}
 ; AVX512F-ONLY-SLOW-NEXT:    movb $120, %sil
-; AVX512F-ONLY-SLOW-NEXT:    kmovw %esi, %k3
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm0 {%k3}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm14, %zmm7 # 64-byte Folded Reload
-; AVX512F-ONLY-SLOW-NEXT:    # zmm7 = zmm14[0,1,2,3],mem[4,5,6,7]
+; AVX512F-ONLY-SLOW-NEXT:    kmovw %esi, %k4
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm24 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm24 {%k4}
+; AVX512F-ONLY-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm16, %zmm22 # 64-byte Folded Reload
+; AVX512F-ONLY-SLOW-NEXT:    # zmm22 = zmm16[0,1,2,3],mem[4,5,6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm2 {%k1}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm16 {%k4}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm4 {%k4}
 ; AVX512F-ONLY-SLOW-NEXT:    movb $-61, %sil
 ; AVX512F-ONLY-SLOW-NEXT:    kmovw %esi, %k4
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm7 {%k4}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k1}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm0 {%k3}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm26, %zmm3 {%k1}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm26 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm26 {%k3}
-; AVX512F-ONLY-SLOW-NEXT:    movb $24, %sil
-; AVX512F-ONLY-SLOW-NEXT:    kmovw %esi, %k3
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm16 {%k3}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm22 {%k4}
+; AVX512F-ONLY-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm17, %zmm2 # 64-byte Folded Reload
+; AVX512F-ONLY-SLOW-NEXT:    # zmm2 = zmm17[0,1,2,3],mem[4,5,6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm2 {%k4}
 ; AVX512F-ONLY-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm12, %zmm3 # 64-byte Folded Reload
 ; AVX512F-ONLY-SLOW-NEXT:    # zmm3 = zmm12[0,1,2,3],mem[4,5,6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm3 {%k4}
-; AVX512F-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm1[0,1,2,3],zmm24[4,5,6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k4}
-; AVX512F-ONLY-SLOW-NEXT:    movb $-31, %sil
-; AVX512F-ONLY-SLOW-NEXT:    kmovw %esi, %k4
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm16 {%k4}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm31 {%k3}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm31 {%k4}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm30, %zmm9 {%k3}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm29, %zmm9 {%k4}
-; AVX512F-ONLY-SLOW-NEXT:    vpbroadcastq 8(%rcx), %ymm0
-; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm3 {%k4}
+; AVX512F-ONLY-SLOW-NEXT:    vpbroadcastq 8(%rcx), %ymm6
+; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm6 = mem[0,1,2,3],ymm6[4,5,6,7]
 ; AVX512F-ONLY-SLOW-NEXT:    movb $6, %sil
 ; AVX512F-ONLY-SLOW-NEXT:    kmovw %esi, %k4
-; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm15 {%k4}
-; AVX512F-ONLY-SLOW-NEXT:    vpbroadcastq 72(%rcx), %ymm0
-; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm27 {%k4}
-; AVX512F-ONLY-SLOW-NEXT:    vpbroadcastq 136(%rcx), %ymm0
-; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm8 {%k4}
-; AVX512F-ONLY-SLOW-NEXT:    vpbroadcastq 200(%rcx), %ymm0
-; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm4 {%k4}
-; AVX512F-ONLY-SLOW-NEXT:    movb $56, %cl
+; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm6, %zmm0, %zmm26 {%k4}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm15 {%k3}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm14 {%k3}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm18 {%k3}
+; AVX512F-ONLY-SLOW-NEXT:    movb $-31, %sil
+; AVX512F-ONLY-SLOW-NEXT:    kmovw %esi, %k3
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm15 {%k3}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm14 {%k3}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm30, %zmm18 {%k3}
+; AVX512F-ONLY-SLOW-NEXT:    movb $56, %sil
+; AVX512F-ONLY-SLOW-NEXT:    kmovw %esi, %k3
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm26 {%k3}
+; AVX512F-ONLY-SLOW-NEXT:    vpbroadcastq 72(%rcx), %ymm6
+; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm6 = mem[0,1,2,3],ymm6[4,5,6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm6, %zmm0, %zmm21 {%k4}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm21 {%k3}
+; AVX512F-ONLY-SLOW-NEXT:    vpbroadcastq 136(%rcx), %ymm6
+; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm6 = mem[0,1,2,3],ymm6[4,5,6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm6, %zmm0, %zmm19 {%k4}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm19 {%k3}
+; AVX512F-ONLY-SLOW-NEXT:    vpbroadcastq 200(%rcx), %ymm6
+; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm6 = mem[0,1,2,3],ymm6[4,5,6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm6, %zmm0, %zmm11 {%k4}
+; AVX512F-ONLY-SLOW-NEXT:    movb $64, %cl
 ; AVX512F-ONLY-SLOW-NEXT:    kmovw %ecx, %k4
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm15 {%k4}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm27 {%k4}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm8 {%k4}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm20, %zmm19, %zmm2
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 192(%rax), %zmm0
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm10, %zmm2
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm4 {%k4}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm13 {%k1}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm14 {%k1}
-; AVX512F-ONLY-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm23, %zmm10 # 64-byte Folded Reload
-; AVX512F-ONLY-SLOW-NEXT:    # zmm10 = zmm23[0,1,2,3],mem[4,5,6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm12 = <0,11,u,u,4,5,6,7>
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm19, %zmm21, %zmm12
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm2 = <0,1,11,u,4,5,6,7>
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm20, %zmm12, %zmm2
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 192(%r8), %ymm12
-; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm12 = ymm12[0],mem[0],ymm12[2],mem[2]
-; AVX512F-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm14 {%k2} = zmm12[2,3,2,3],zmm0[2,3,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm21
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm18, %zmm22 {%k3}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vpunpckhqdq (%rsp), %ymm12, %ymm12 # 32-byte Folded Reload
-; AVX512F-ONLY-SLOW-NEXT:    # ymm12 = ymm12[1],mem[1],ymm12[3],mem[3]
-; AVX512F-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm12 = ymm12[0,2,3,3]
-; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm12 = ymm12[0,1,2,3,4,5],mem[6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm31 {%k4}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm6, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 192(%rax), %zmm6
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm6, %zmm8, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm11 {%k3}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm0 = <0,1,11,u,4,5,6,7>
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm1, %zmm9, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm10 {%k1}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 192(%r8), %ymm7
+; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm7 = ymm7[0],mem[0],ymm7[2],mem[2]
+; AVX512F-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm10 {%k2} = zmm7[2,3,2,3],zmm6[2,3,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vpunpckhqdq (%rsp), %ymm7, %ymm7 # 32-byte Folded Reload
+; AVX512F-ONLY-SLOW-NEXT:    # ymm7 = ymm7[1],mem[1],ymm7[3],mem[3]
+; AVX512F-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[0,2,3,3]
+; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm7 = ymm7[0,1,2,3,4,5],mem[6,7]
 ; AVX512F-ONLY-SLOW-NEXT:    movb $14, %cl
 ; AVX512F-ONLY-SLOW-NEXT:    kmovw %ecx, %k1
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm12, %zmm0, %zmm18 {%k1}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm12, %ymm12 # 32-byte Folded Reload
-; AVX512F-ONLY-SLOW-NEXT:    # ymm12 = ymm12[1],mem[1],ymm12[3],mem[3]
-; AVX512F-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm12 = ymm12[0,2,3,3]
-; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm12 = ymm12[0,1,2,3,4,5],mem[6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm12, %zmm0, %zmm25 {%k1}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm12, %ymm12 # 32-byte Folded Reload
-; AVX512F-ONLY-SLOW-NEXT:    # ymm12 = ymm12[1],mem[1],ymm12[3],mem[3]
-; AVX512F-ONLY-SLOW-NEXT:    movb $64, %cl
-; AVX512F-ONLY-SLOW-NEXT:    kmovw %ecx, %k2
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm19, %zmm10 {%k2}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm7, %zmm0, %zmm12 {%k1}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm7 # 32-byte Folded Reload
+; AVX512F-ONLY-SLOW-NEXT:    # ymm7 = ymm7[1],mem[1],ymm7[3],mem[3]
+; AVX512F-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[0,2,3,3]
+; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm7 = ymm7[0,1,2,3,4,5],mem[6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm7, %zmm0, %zmm29 {%k1}
 ; AVX512F-ONLY-SLOW-NEXT:    movb $8, %cl
 ; AVX512F-ONLY-SLOW-NEXT:    kmovw %ecx, %k2
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k2}
-; AVX512F-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm12 = ymm12[0,2,3,3]
-; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm12 = ymm12[0,1,2,3,4,5],mem[6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm12, %zmm0, %zmm28 {%k1}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm12 = <12,u,u,3,4,5,6,13>
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm19, %zmm13, %zmm12
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm13 = <u,1,2,3,4,15,u,u>
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm19, %zmm22, %zmm13
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm14 = <0,12,u,3,4,5,6,7>
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm20, %zmm12, %zmm14
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm12 = <13,u,2,3,4,5,6,14>
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm20, %zmm10, %zmm12
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm10 = <u,1,2,3,4,5,15,u>
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm20, %zmm13, %zmm10
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm13 = [0,1,12,3,4,5,6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm0, %zmm14, %zmm13
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm14 = [0,13,2,3,4,5,6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm0, %zmm12, %zmm14
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm12 = [14,1,2,3,4,5,6,15]
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm0, %zmm10, %zmm12
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm0 {%k2}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm7 # 32-byte Folded Reload
+; AVX512F-ONLY-SLOW-NEXT:    # ymm7 = ymm7[1],mem[1],ymm7[3],mem[3]
+; AVX512F-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[0,2,3,3]
+; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm7 = ymm7[0,1,2,3,4,5],mem[6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm7, %zmm0, %zmm28 {%k1}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm7 = <0,12,u,3,4,5,6,7>
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm1, %zmm25, %zmm7
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm8 = <13,u,2,3,4,5,6,14>
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm1, %zmm31, %zmm8
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm9 = <u,1,2,3,4,5,15,u>
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm1, %zmm5, %zmm9
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,1,12,3,4,5,6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm6, %zmm7, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [0,13,2,3,4,5,6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm6, %zmm8, %zmm5
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm7 = [14,1,2,3,4,5,6,15]
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm6, %zmm9, %zmm7
 ; AVX512F-ONLY-SLOW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm21, 1472(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm4, 1408(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm9, 1280(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, 1216(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm26, 1152(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovaps %zmm0, 1024(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm8, 960(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm31, 832(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, 768(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovaps %zmm0, 704(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovaps %zmm0, 576(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm27, 512(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm16, 384(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm7, 320(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovaps %zmm0, 256(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovaps %zmm0, 128(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm15, 64(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm5, 1344(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm10, 1472(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm11, 1408(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm18, 1280(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, 1216(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm4, 1152(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vmovaps %zmm3, 1024(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm19, 960(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm14, 832(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, 768(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm16, 704(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vmovaps %zmm2, 576(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm21, 512(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm15, 384(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm22, 320(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm24, 256(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vmovaps %zmm2, 128(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm26, 64(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm13, 1344(%rax)
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm28, 1088(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, 896(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm25, 640(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm11, 448(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm18, 192(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm17, (%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm12, 1728(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm14, 1664(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm13, 1600(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, 1536(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    addq $2120, %rsp # imm = 0x848
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm20, 896(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm29, 640(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm23, 448(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm12, 192(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm27, (%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm7, 1728(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm5, 1664(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, 1600(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, 1536(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    addq $2184, %rsp # imm = 0x888
 ; AVX512F-ONLY-SLOW-NEXT:    vzeroupper
 ; AVX512F-ONLY-SLOW-NEXT:    retq
 ;
 ; AVX512F-ONLY-FAST-LABEL: store_i64_stride7_vf32:
 ; AVX512F-ONLY-FAST:       # %bb.0:
-; AVX512F-ONLY-FAST-NEXT:    subq $2024, %rsp # imm = 0x7E8
+; AVX512F-ONLY-FAST-NEXT:    subq $2152, %rsp # imm = 0x868
 ; AVX512F-ONLY-FAST-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 (%rdi), %zmm7
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 64(%rdi), %zmm30
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 (%rsi), %zmm17
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 64(%rsi), %zmm15
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 64(%rdx), %zmm13
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 (%rdx), %zmm8
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 (%rcx), %zmm27
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 64(%rcx), %zmm20
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 (%rdi), %zmm16
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 64(%rdi), %zmm17
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 (%rsi), %zmm29
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 64(%rsi), %zmm28
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 64(%rdx), %zmm19
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 (%rdx), %zmm23
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 (%rcx), %zmm22
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 64(%rcx), %zmm30
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 (%r8), %zmm0
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 (%r9), %zmm9
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 (%rax), %zmm4
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 (%r9), %zmm7
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 (%rax), %zmm3
 ; AVX512F-ONLY-FAST-NEXT:    movb $96, %r10b
 ; AVX512F-ONLY-FAST-NEXT:    kmovw %r10d, %k1
+; AVX512F-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm21 = [15,7,15,7,15,7,15,7]
+; AVX512F-ONLY-FAST-NEXT:    # zmm21 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm2 = [9,1,9,1,9,1,9,1]
 ; AVX512F-ONLY-FAST-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm3
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm9, %zmm2, %zmm3
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm10
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm4
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm7, %zmm2, %zmm4
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm8
 ; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [4,9,0,3,4,9,0,3]
 ; AVX512F-ONLY-FAST-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm4, %zmm2, %zmm3
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm11
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm2 = [11,3,11,3,11,3,11,3]
-; AVX512F-ONLY-FAST-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm7, %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm17, %zmm2, %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm14
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm3 = [2,10,0,3,2,10,0,3]
-; AVX512F-ONLY-FAST-NEXT:    # zmm3 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm8, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm27, %zmm3, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm16
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm3, %zmm2, %zmm4
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm9
+; AVX512F-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm25 = [11,3,11,3,11,3,11,3]
+; AVX512F-ONLY-FAST-NEXT:    # zmm25 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm16, %zmm1
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm29, %zmm25, %zmm1
+; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [2,10,0,3,2,10,0,3]
+; AVX512F-ONLY-FAST-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm23, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm22, %zmm4, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm4, %zmm15
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm5
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqa (%r9), %ymm1
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqa 64(%r9), %ymm6
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa 128(%r9), %ymm2
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 (%r8), %ymm19
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 64(%r8), %ymm23
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 128(%r8), %ymm22
-; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm3 = ymm19[0],ymm1[0],ymm19[2],ymm1[2]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa 128(%r9), %ymm4
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa (%r8), %ymm10
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa 64(%r8), %ymm12
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa 128(%r8), %ymm11
+; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm2 = ymm10[0],ymm1[0],ymm10[2],ymm1[2]
 ; AVX512F-ONLY-FAST-NEXT:    movb $28, %r10b
 ; AVX512F-ONLY-FAST-NEXT:    kmovw %r10d, %k2
-; AVX512F-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm5 {%k2} = zmm3[2,3,2,3],zmm4[2,3,2,3]
+; AVX512F-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm5 {%k2} = zmm2[2,3,2,3],zmm3[2,3,2,3]
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm4, %zmm21
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa {{.*#+}} ymm3 = [1,3,7,7]
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %ymm1, %ymm3, %ymm19
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %ymm19, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [4,12,0,5,4,12,0,5]
-; AVX512F-ONLY-FAST-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm9, %zmm4, %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm4, %zmm19
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm5 = [0,1,12,7,0,1,12,7]
-; AVX512F-ONLY-FAST-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm21, %zmm5, %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm25 = [5,0,14,6,5,0,14,6]
-; AVX512F-ONLY-FAST-NEXT:    # zmm25 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm9, %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm25, %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [0,13,6,7,0,13,6,7]
-; AVX512F-ONLY-FAST-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm21, %zmm4, %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm28 = [15,7,15,7,15,7,15,7]
-; AVX512F-ONLY-FAST-NEXT:    # zmm28 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm9, %zmm28, %zmm0
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm1 = [6,13,14,7,6,13,14,7]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa {{.*#+}} ymm5 = [1,3,7,7]
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %ymm1, %ymm5, %ymm10
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu %ymm10, (%rsp) # 32-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm1 = [4,12,0,5,4,12,0,5]
 ; AVX512F-ONLY-FAST-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm21
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm23[0],ymm6[0],ymm23[2],ymm6[2]
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %ymm6, %ymm3, %ymm23
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %ymm23, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm6 = ymm22[0],ymm2[0],ymm22[2],ymm2[2]
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %ymm2, %ymm3, %ymm22
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %ymm22, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm30, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm15, %zmm14, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm13, %zmm3
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm20, %zmm16, %zmm3
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 64(%rax), %zmm18
-; AVX512F-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm3 {%k2} = zmm0[2,3,2,3],zmm18[2,3,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 64(%r8), %zmm0
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 64(%r9), %zmm22
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm22, %zmm10, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm11, %zmm9
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm18, %zmm11, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm22, %zmm19, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm18, %zmm5, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm22, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm25, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm18, %zmm4, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm7, %zmm1, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, %zmm10
+; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm1 = [0,1,12,7,0,1,12,7]
+; AVX512F-ONLY-FAST-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm3, %zmm1, %zmm2
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm22, %zmm28, %zmm0
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm18
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 128(%rdi), %zmm26
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 128(%rsi), %zmm3
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm26, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm3, %zmm14, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 128(%rdx), %zmm24
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 128(%rcx), %zmm0
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm24, %zmm14
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm16, %zmm14
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm16, %zmm23
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm14 {%k1}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 128(%rax), %zmm11
-; AVX512F-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm14 {%k2} = zmm6[2,3,2,3],zmm11[2,3,2,3]
+; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm13 = [5,0,14,6,5,0,14,6]
+; AVX512F-ONLY-FAST-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm7, %zmm14
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm7, %zmm18
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm13, %zmm14
+; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [0,13,6,7,0,13,6,7]
+; AVX512F-ONLY-FAST-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm7
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm3, %zmm2, %zmm14
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 128(%r8), %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 128(%r9), %zmm14
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm14, %zmm2, %zmm19
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm11, %zmm5, %zmm19
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm2, %zmm14, %zmm25
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm11, %zmm4, %zmm25
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm25, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm4
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm14, %zmm10, %zmm4
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm18, %zmm21, %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm3 = [6,13,14,7,6,13,14,7]
+; AVX512F-ONLY-FAST-NEXT:    # zmm3 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm3, %zmm7
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm12[0],ymm6[0],ymm12[2],ymm6[2]
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %ymm6, %ymm5, %ymm12
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm7 = ymm11[0],ymm4[0],ymm11[2],ymm4[2]
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %ymm4, %ymm5, %ymm11
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm17, %zmm4
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm28, %zmm25, %zmm4
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm19, %zmm6
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm15, %zmm5
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm30, %zmm15, %zmm6
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm4, %zmm6 {%k1}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 64(%rax), %zmm11
+; AVX512F-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm6 {%k2} = zmm0[2,3,2,3],zmm11[2,3,2,3]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 64(%r8), %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 64(%r9), %zmm6
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm4
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm6, %zmm8, %zmm4
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm11, %zmm9, %zmm4
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm14, %zmm28, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm2, %zmm1, %zmm11
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm5 = [3,0,12,4,3,0,12,4]
-; AVX512F-ONLY-FAST-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm27, %zmm29
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm8, %zmm5, %zmm29
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm1 = [13,5,13,5,13,5,13,5]
-; AVX512F-ONLY-FAST-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm8, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm27, %zmm1, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm10 = [6,14,6,14,6,14,6,14]
-; AVX512F-ONLY-FAST-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm8, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm27, %zmm10, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm27, %zmm28, %zmm8
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm4
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm6, %zmm10, %zmm4
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm11, %zmm1, %zmm4
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm6, %zmm4
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm13, %zmm4
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm11, %zmm2, %zmm4
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm6, %zmm21, %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm3, %zmm11
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 128(%rdi), %zmm27
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 128(%rsi), %zmm18
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm27, %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm18, %zmm25, %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 128(%rdx), %zmm31
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 128(%rcx), %zmm15
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm31, %zmm4
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm15, %zmm5, %zmm4
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm4 {%k1}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 128(%rax), %zmm5
+; AVX512F-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm4 {%k2} = zmm7[2,3,2,3],zmm5[2,3,2,3]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 128(%r8), %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 128(%r9), %zmm4
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm4, %zmm0, %zmm10
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm5, %zmm1, %zmm10
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm0, %zmm4, %zmm13
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm5, %zmm2, %zmm13
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm1
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm4, %zmm8, %zmm1
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm5, %zmm9, %zmm1
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm4, %zmm21, %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm3, %zmm5
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 192(%rdx), %zmm7
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 192(%rcx), %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm13 = [3,0,12,4,3,0,12,4]
+; AVX512F-ONLY-FAST-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm24
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm7, %zmm13, %zmm24
 ; AVX512F-ONLY-FAST-NEXT:    movb $48, %r10b
 ; AVX512F-ONLY-FAST-NEXT:    kmovw %r10d, %k3
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm31 = [0,8,0,1,0,8,0,1]
-; AVX512F-ONLY-FAST-NEXT:    # zmm31 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm7, %zmm25
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm17, %zmm31, %zmm25
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [1,0,10,2,1,0,10,2]
-; AVX512F-ONLY-FAST-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm17, %zmm21
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm7, %zmm4, %zmm21
-; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm29 {%k3} = zmm7[0],zmm17[0],zmm7[2],zmm17[2],zmm7[4],zmm17[4],zmm7[6],zmm17[6]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm7, %zmm9
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm17, %zmm1, %zmm9
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti128 {{.*#+}} ymm8 = [6,14,6,14]
-; AVX512F-ONLY-FAST-NEXT:    # ymm8 = mem[0,1,0,1]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm7, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm17, %zmm8, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [15,7,15,7]
-; AVX512F-ONLY-FAST-NEXT:    # ymm2 = mem[0,1,0,1]
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm17, %zmm2, %zmm7
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm6
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm20, %zmm17
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm13, %zmm5, %zmm17
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm13, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm20, %zmm1, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm13, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm20, %zmm10, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 192(%rdi), %zmm12
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 192(%rsi), %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm24 {%k3} = zmm12[0],zmm3[0],zmm12[2],zmm3[2],zmm12[4],zmm3[4],zmm12[6],zmm3[6]
+; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm14 = [0,8,0,1,0,8,0,1]
+; AVX512F-ONLY-FAST-NEXT:    # zmm14 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm16, %zmm1
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm29, %zmm14, %zmm1
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm17, %zmm1
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm28, %zmm14, %zmm1
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm27, %zmm1
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm18, %zmm14, %zmm1
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm3, %zmm12, %zmm14
+; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm11 = [1,0,10,2,1,0,10,2]
+; AVX512F-ONLY-FAST-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm29, %zmm1
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm16, %zmm11, %zmm1
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm28, %zmm26
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm17, %zmm11, %zmm26
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm18, %zmm20
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm27, %zmm11, %zmm20
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm12, %zmm3, %zmm11
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm12, %zmm8
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm12, %zmm10
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm12, %zmm9
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm3, %zmm25, %zmm12
+; AVX512F-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm4 = [6,14,6,14,6,14,6,14]
+; AVX512F-ONLY-FAST-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm3, %zmm4, %zmm8
+; AVX512F-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm1 = [13,5,13,5,13,5,13,5]
+; AVX512F-ONLY-FAST-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm3, %zmm1, %zmm10
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm3, %zmm21, %zmm9
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm22, %zmm25
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm23, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm23, %zmm13, %zmm25
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm22, %zmm1, %zmm23
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm22, %zmm4, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm22, %zmm21, %zmm2
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm20, %zmm28, %zmm13
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm25 {%k3} = zmm16[0],zmm29[0],zmm16[2],zmm29[2],zmm16[4],zmm29[4],zmm16[6],zmm29[6]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm16, %zmm6
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm29, %zmm1, %zmm6
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm16, %zmm22
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm29, %zmm4, %zmm22
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm29, %zmm21, %zmm16
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm30, %zmm16
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm15, %zmm31, %zmm16
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm15, %zmm20
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm30, %zmm4, %zmm20
-; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm17 {%k3} = zmm30[0],zmm15[0],zmm30[2],zmm15[2],zmm30[4],zmm15[4],zmm30[6],zmm15[6]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm30, %zmm7
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm15, %zmm1, %zmm7
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm30, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm15, %zmm8, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm19, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm19, %zmm13, %zmm16
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm30, %zmm1, %zmm19
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm29
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm30, %zmm4, %zmm29
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm30, %zmm21, %zmm2
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm15, %zmm6, %zmm30
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm6, %zmm13
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 192(%rdx), %zmm15
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 192(%rcx), %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm15, %zmm6
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm2, %zmm10, %zmm6
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm12
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm15, %zmm5, %zmm12
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm15, %zmm6
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm2, %zmm1, %zmm6
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm2, %zmm15, %zmm23
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm23, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm2, %zmm28, %zmm15
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm24, %zmm0, %zmm5
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm0, %zmm24, %zmm10
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm24, %zmm27
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm28, %zmm24
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm27
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 192(%rdi), %zmm19
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 192(%rsi), %zmm0
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm19, %zmm23
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm8, %zmm23
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm19, %zmm6
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm6
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm26, %zmm28
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm3, %zmm31, %zmm28
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm26, %zmm4, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm5 {%k3} = zmm26[0],zmm3[0],zmm26[2],zmm3[2],zmm26[4],zmm3[4],zmm26[6],zmm3[6]
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm3, %zmm26, %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm3, %zmm26, %zmm8
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm3, %zmm13, %zmm26
-; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm12 {%k3} = zmm19[0],zmm0[0],zmm19[2],zmm0[2],zmm19[4],zmm0[4],zmm19[6],zmm0[6]
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm0, %zmm19, %zmm31
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm19, %zmm0, %zmm4
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm0, %zmm19, %zmm3
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm13, %zmm19
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm17, %zmm30
+; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm16 {%k3} = zmm17[0],zmm28[0],zmm17[2],zmm28[2],zmm17[4],zmm28[4],zmm17[6],zmm28[6]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm17, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm28, %zmm1, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm28, %zmm4, %zmm17
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm28, %zmm21, %zmm30
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm31, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm31, %zmm15, %zmm13
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm15, %zmm1, %zmm31
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm28
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm15, %zmm4, %zmm28
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm15, %zmm21, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm7, %zmm15
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm4, %zmm15
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm7, %zmm5
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm5
+; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm13 {%k3} = zmm27[0],zmm18[0],zmm27[2],zmm18[2],zmm27[4],zmm18[4],zmm27[6],zmm18[6]
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm18, %zmm27, %zmm1
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm18, %zmm27, %zmm4
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm18, %zmm21, %zmm27
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm7, %zmm18
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm21, %zmm18
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm2, %zmm7
+; AVX512F-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm8[0,1,2,3],zmm15[4,5,6,7]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 192(%r8), %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm0 = <0,11,u,u,4,5,6,7>
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm2, %zmm24, %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 192(%r9), %zmm8
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm15 = <0,1,11,u,4,5,6,7>
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm8, %zmm0, %zmm15
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm5, %zmm10 {%k1}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm0 = <12,u,u,3,4,5,6,13>
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm2, %zmm10, %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    movb $24, %sil
+; AVX512F-ONLY-FAST-NEXT:    kmovw %esi, %k3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm18, %zmm9 {%k3}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm24 = <u,1,2,3,4,15,u,u>
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm2, %zmm9, %zmm24
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqa (%rdx), %xmm0
 ; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
 ; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
 ; AVX512F-ONLY-FAST-NEXT:    movb $12, %sil
-; AVX512F-ONLY-FAST-NEXT:    kmovw %esi, %k3
-; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm25 {%k3}
+; AVX512F-ONLY-FAST-NEXT:    kmovw %esi, %k4
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm21 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm21 {%k4}
 ; AVX512F-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [0,8,0,8,0,8,0,8]
 ; AVX512F-ONLY-FAST-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm3 # 64-byte Folded Reload
-; AVX512F-ONLY-FAST-NEXT:    vinserti32x4 $3, (%rax), %zmm3, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm9 # 64-byte Folded Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa 64(%rdx), %xmm5
+; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm5 = xmm5[0],mem[0]
+; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm5, %zmm0, %zmm10 {%k4}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa 128(%rdx), %xmm5
+; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm5 = xmm5[0],mem[0]
+; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm5, %zmm0, %zmm18 {%k4}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa 192(%rdx), %xmm5
+; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm5 = xmm5[0],mem[0]
+; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm5, %zmm0, %zmm14 {%k4}
+; AVX512F-ONLY-FAST-NEXT:    vinserti32x4 $3, (%rax), %zmm9, %zmm5
 ; AVX512F-ONLY-FAST-NEXT:    movb $112, %sil
 ; AVX512F-ONLY-FAST-NEXT:    kmovw %esi, %k4
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm25 {%k4}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa 64(%rdx), %xmm3
-; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],mem[0]
-; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm16 {%k3}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm22, %zmm0, %zmm3
-; AVX512F-ONLY-FAST-NEXT:    vinserti32x4 $3, 64(%rax), %zmm3, %zmm3
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm16 {%k4}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa 128(%rdx), %xmm3
-; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],mem[0]
-; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm28 {%k3}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm14, %zmm0, %zmm3
-; AVX512F-ONLY-FAST-NEXT:    vinserti32x4 $3, 128(%rax), %zmm3, %zmm3
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm28 {%k4}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa 192(%rdx), %xmm3
-; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],mem[0]
-; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm31 {%k3}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 192(%r8), %zmm22
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 192(%r9), %zmm14
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm14, %zmm22, %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm5, %zmm21 {%k4}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm5 # 64-byte Folded Reload
+; AVX512F-ONLY-FAST-NEXT:    vinserti32x4 $3, 64(%rax), %zmm5, %zmm5
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm5, %zmm10 {%k4}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm5 # 64-byte Folded Reload
+; AVX512F-ONLY-FAST-NEXT:    vinserti32x4 $3, 128(%rax), %zmm5, %zmm5
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm5, %zmm18 {%k4}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm18, %zmm21
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm8, %zmm2, %zmm0
 ; AVX512F-ONLY-FAST-NEXT:    vinserti32x4 $3, 192(%rax), %zmm0, %zmm0
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm31 {%k4}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm14 {%k4}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu (%rsp), %ymm0 # 32-byte Reload
 ; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],mem[6,7]
 ; AVX512F-ONLY-FAST-NEXT:    movb $14, %sil
-; AVX512F-ONLY-FAST-NEXT:    kmovw %esi, %k3
-; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm29 {%k3}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm9 {%k1}
-; AVX512F-ONLY-FAST-NEXT:    movb $120, %sil
 ; AVX512F-ONLY-FAST-NEXT:    kmovw %esi, %k4
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm9, %zmm0 {%k4}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm13 # 64-byte Folded Reload
-; AVX512F-ONLY-FAST-NEXT:    # zmm13 = zmm0[0,1,2,3],mem[4,5,6,7]
+; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm25 {%k4}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm23, %zmm6 {%k1}
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],mem[6,7]
-; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm17 {%k3}
+; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm16 {%k4}
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],mem[6,7]
-; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm5 {%k3}
-; AVX512F-ONLY-FAST-NEXT:    movb $-61, %sil
-; AVX512F-ONLY-FAST-NEXT:    kmovw %esi, %k5
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm13 {%k5}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm7 {%k1}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm7, %zmm3 {%k4}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm27, %zmm1 {%k1}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm27 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, %zmm27 {%k4}
-; AVX512F-ONLY-FAST-NEXT:    movb $24, %sil
-; AVX512F-ONLY-FAST-NEXT:    kmovw %esi, %k3
+; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm13 {%k4}
+; AVX512F-ONLY-FAST-NEXT:    movb $120, %sil
+; AVX512F-ONLY-FAST-NEXT:    kmovw %esi, %k4
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm9 {%k3}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm1 # 64-byte Folded Reload
-; AVX512F-ONLY-FAST-NEXT:    # zmm1 = zmm0[0,1,2,3],mem[4,5,6,7]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k5}
-; AVX512F-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm7 = zmm8[0,1,2,3],zmm10[4,5,6,7]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm7 {%k5}
-; AVX512F-ONLY-FAST-NEXT:    movb $-31, %sil
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm6, %zmm9 {%k4}
+; AVX512F-ONLY-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm22, %zmm5 # 64-byte Folded Reload
+; AVX512F-ONLY-FAST-NEXT:    # zmm5 = zmm22[0,1,2,3],mem[4,5,6,7]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm19, %zmm3 {%k1}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm10 {%k4}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm31, %zmm1 {%k1}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, %zmm18 {%k4}
+; AVX512F-ONLY-FAST-NEXT:    movb $-61, %sil
 ; AVX512F-ONLY-FAST-NEXT:    kmovw %esi, %k4
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm9 {%k4}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm30 {%k3}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm18, %zmm30 {%k4}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm24, %zmm26 {%k3}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm11, %zmm26 {%k4}
-; AVX512F-ONLY-FAST-NEXT:    vpbroadcastq 8(%rcx), %ymm0
-; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm5 {%k4}
+; AVX512F-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm17[0,1,2,3],zmm29[4,5,6,7]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, %zmm0 {%k4}
+; AVX512F-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm4[0,1,2,3],zmm28[4,5,6,7]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm1 {%k4}
+; AVX512F-ONLY-FAST-NEXT:    vpbroadcastq 8(%rcx), %ymm3
+; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm3 = mem[0,1,2,3],ymm3[4,5,6,7]
 ; AVX512F-ONLY-FAST-NEXT:    movb $6, %sil
 ; AVX512F-ONLY-FAST-NEXT:    kmovw %esi, %k4
-; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm21 {%k4}
-; AVX512F-ONLY-FAST-NEXT:    vpbroadcastq 72(%rcx), %ymm0
-; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
-; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm20 {%k4}
-; AVX512F-ONLY-FAST-NEXT:    vpbroadcastq 136(%rcx), %ymm0
-; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
-; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm2 {%k4}
-; AVX512F-ONLY-FAST-NEXT:    vpbroadcastq 200(%rcx), %ymm0
-; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
-; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm4 {%k4}
-; AVX512F-ONLY-FAST-NEXT:    movb $56, %cl
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm4 {%k4}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm6 {%k3}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm30 {%k3}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm27 {%k3}
+; AVX512F-ONLY-FAST-NEXT:    movb $-31, %sil
+; AVX512F-ONLY-FAST-NEXT:    kmovw %esi, %k3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm6 {%k3}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm6, %zmm19
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm30 {%k3}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm27 {%k3}
+; AVX512F-ONLY-FAST-NEXT:    movb $56, %sil
+; AVX512F-ONLY-FAST-NEXT:    kmovw %esi, %k3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm4 {%k3}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm4, %zmm17
+; AVX512F-ONLY-FAST-NEXT:    vpbroadcastq 72(%rcx), %ymm3
+; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm3 = mem[0,1,2,3],ymm3[4,5,6,7]
+; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm26 {%k4}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm26 {%k3}
+; AVX512F-ONLY-FAST-NEXT:    vpbroadcastq 136(%rcx), %ymm3
+; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm3 = mem[0,1,2,3],ymm3[4,5,6,7]
+; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm20 {%k4}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm20 {%k3}
+; AVX512F-ONLY-FAST-NEXT:    vpbroadcastq 200(%rcx), %ymm3
+; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm3 = mem[0,1,2,3],ymm3[4,5,6,7]
+; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm11 {%k4}
+; AVX512F-ONLY-FAST-NEXT:    movb $64, %cl
 ; AVX512F-ONLY-FAST-NEXT:    kmovw %ecx, %k4
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm21 {%k4}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm20 {%k4}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k4}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm14, %zmm22, %zmm8
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 192(%rax), %zmm0
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm9, %zmm8
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm8, %zmm4 {%k4}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm8, %zmm6 {%k1}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm8, %zmm11 {%k1}
-; AVX512F-ONLY-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm23, %zmm8 # 64-byte Folded Reload
-; AVX512F-ONLY-FAST-NEXT:    # zmm8 = zmm23[0,1,2,3],mem[4,5,6,7]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm9 = <0,11,u,u,4,5,6,7>
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm22, %zmm12, %zmm9
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm10 = <0,1,11,u,4,5,6,7>
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm14, %zmm9, %zmm10
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa 192(%r8), %ymm9
-; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm9 = ymm9[0],mem[0],ymm9[2],mem[2]
-; AVX512F-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm11 {%k2} = zmm9[2,3,2,3],zmm0[2,3,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm11, %zmm12
-; AVX512F-ONLY-FAST-NEXT:    movb $64, %al
-; AVX512F-ONLY-FAST-NEXT:    kmovw %eax, %k1
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm22, %zmm8 {%k1}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm6 {%k4}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm8, %zmm3, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 192(%rax), %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm3, %zmm4, %zmm2
 ; AVX512F-ONLY-FAST-NEXT:    movb $8, %al
-; AVX512F-ONLY-FAST-NEXT:    kmovw %eax, %k1
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm10 {%k1}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm15, %zmm19 {%k3}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm9 = <12,u,u,3,4,5,6,13>
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm22, %zmm6, %zmm9
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm6 = <u,1,2,3,4,15,u,u>
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm22, %zmm19, %zmm6
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm11 = <0,12,u,3,4,5,6,7>
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm14, %zmm9, %zmm11
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm9 = <13,u,2,3,4,5,6,14>
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm14, %zmm8, %zmm9
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm8 = <u,1,2,3,4,5,15,u>
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm14, %zmm6, %zmm8
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [0,1,12,3,4,5,6,7]
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm0, %zmm11, %zmm6
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm11 = [0,13,2,3,4,5,6,7]
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm0, %zmm9, %zmm11
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm9 = [14,1,2,3,4,5,6,15]
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm0, %zmm8, %zmm9
+; AVX512F-ONLY-FAST-NEXT:    kmovw %eax, %k4
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm15 {%k4}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm11 {%k3}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm12, %zmm7 {%k1}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa 192(%r8), %ymm2
+; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm2 = ymm2[0],mem[0],ymm2[2],mem[2]
+; AVX512F-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm7 {%k2} = zmm2[2,3,2,3],zmm3[2,3,2,3]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm2 = <0,12,u,3,4,5,6,7>
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm8, %zmm4, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm4 = <13,u,2,3,4,5,6,14>
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm8, %zmm6, %zmm4
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm6 = <u,1,2,3,4,5,15,u>
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm8, %zmm24, %zmm6
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm8 = [0,1,12,3,4,5,6,7]
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm3, %zmm2, %zmm8
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [0,13,2,3,4,5,6,7]
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm3, %zmm4, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [14,1,2,3,4,5,6,15]
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm3, %zmm6, %zmm4
 ; AVX512F-ONLY-FAST-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm12, 1472(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm4, 1408(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm26, 1280(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm7, 1216(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm27, 1152(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm5, 1088(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovaps %zmm0, 1024(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, 960(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm7, 1472(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm11, 1408(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm27, 1280(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, 1216(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm18, 1152(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm13, 1088(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovaps %zmm1, 1024(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm20, 960(%rax)
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm30, 832(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, 768(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, 704(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm17, 640(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, 768(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm10, 704(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm16, 640(%rax)
 ; AVX512F-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-ONLY-FAST-NEXT:    vmovaps %zmm0, 576(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm20, 512(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm26, 512(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm19, 384(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm5, 320(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm9, 256(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm25, 192(%rax)
 ; AVX512F-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovaps %zmm0, 384(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm13, 320(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovaps %zmm0, 128(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm17, 64(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm14, 1344(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm21, 896(%rax)
 ; AVX512F-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovaps %zmm0, 256(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm29, 192(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovaps %zmm0, 448(%rax)
 ; AVX512F-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovaps %zmm0, 128(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm21, 64(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm31, 1344(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm28, 896(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm16, 448(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm25, (%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm9, 1728(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm11, 1664(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm6, 1600(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm10, 1536(%rax)
-; AVX512F-ONLY-FAST-NEXT:    addq $2024, %rsp # imm = 0x7E8
+; AVX512F-ONLY-FAST-NEXT:    vmovaps %zmm0, (%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm4, 1728(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, 1664(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm8, 1600(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm15, 1536(%rax)
+; AVX512F-ONLY-FAST-NEXT:    addq $2152, %rsp # imm = 0x868
 ; AVX512F-ONLY-FAST-NEXT:    vzeroupper
 ; AVX512F-ONLY-FAST-NEXT:    retq
 ;
 ; AVX512DQ-SLOW-LABEL: store_i64_stride7_vf32:
 ; AVX512DQ-SLOW:       # %bb.0:
-; AVX512DQ-SLOW-NEXT:    subq $2120, %rsp # imm = 0x848
+; AVX512DQ-SLOW-NEXT:    subq $2184, %rsp # imm = 0x888
 ; AVX512DQ-SLOW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 (%rdi), %zmm10
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 64(%rdi), %zmm8
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 (%rsi), %zmm15
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 64(%rsi), %zmm13
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 (%rdi), %zmm16
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 64(%rdi), %zmm9
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 (%rsi), %zmm23
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 64(%rsi), %zmm10
 ; AVX512DQ-SLOW-NEXT:    vmovdqa64 64(%rdx), %zmm20
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 (%rdx), %zmm21
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 (%rcx), %zmm5
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 64(%rcx), %zmm28
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 (%r8), %zmm3
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 (%r9), %zmm2
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 (%rax), %zmm4
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 64(%rax), %zmm7
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 (%rdx), %zmm18
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 (%rcx), %zmm6
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 64(%rcx), %zmm7
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 (%r8), %zmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 (%r9), %zmm3
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 (%rax), %zmm2
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 64(%rax), %zmm4
 ; AVX512DQ-SLOW-NEXT:    movb $96, %r10b
 ; AVX512DQ-SLOW-NEXT:    kmovw %r10d, %k1
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [9,1,9,1,9,1,9,1]
-; AVX512DQ-SLOW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm1
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm2, %zmm0, %zmm1
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm14
-; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [4,9,0,3,4,9,0,3]
-; AVX512DQ-SLOW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm4, %zmm0, %zmm1
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm16
-; AVX512DQ-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm1 = [11,3,11,3,11,3,11,3]
+; AVX512DQ-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm17 = [15,7,15,7,15,7,15,7]
+; AVX512DQ-SLOW-NEXT:    # zmm17 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm1 = [9,1,9,1,9,1,9,1]
 ; AVX512DQ-SLOW-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm10, %zmm0
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm15, %zmm1, %zmm0
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm18
-; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm9 = [2,10,0,3,2,10,0,3]
-; AVX512DQ-SLOW-NEXT:    # zmm9 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm21, %zmm1
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm5, %zmm9, %zmm1
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm9, %zmm17
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512DQ-SLOW-NEXT:    vmovdqa (%r9), %ymm9
-; AVX512DQ-SLOW-NEXT:    vmovdqu %ymm9, (%rsp) # 32-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa 64(%r9), %ymm11
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm5
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm3, %zmm1, %zmm5
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm12
+; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm1 = [4,9,0,3,4,9,0,3]
+; AVX512DQ-SLOW-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm2, %zmm1, %zmm5
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm13
+; AVX512DQ-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm25 = [11,3,11,3,11,3,11,3]
+; AVX512DQ-SLOW-NEXT:    # zmm25 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm16, %zmm1
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm23, %zmm25, %zmm1
+; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm8 = [2,10,0,3,2,10,0,3]
+; AVX512DQ-SLOW-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm18, %zmm5
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm6, %zmm8, %zmm5
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm15
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm5 {%k1}
+; AVX512DQ-SLOW-NEXT:    vmovdqa (%r9), %ymm8
+; AVX512DQ-SLOW-NEXT:    vmovdqu %ymm8, (%rsp) # 32-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa 64(%r9), %ymm14
+; AVX512DQ-SLOW-NEXT:    vmovdqu %ymm14, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa (%r8), %ymm1
+; AVX512DQ-SLOW-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa 64(%r8), %ymm11
 ; AVX512DQ-SLOW-NEXT:    vmovdqu %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa (%r8), %ymm0
-; AVX512DQ-SLOW-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa 64(%r8), %ymm12
-; AVX512DQ-SLOW-NEXT:    vmovdqu %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm9[0],ymm0[2],ymm9[2]
+; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm1 = ymm1[0],ymm8[0],ymm1[2],ymm8[2]
 ; AVX512DQ-SLOW-NEXT:    movb $28, %r10b
 ; AVX512DQ-SLOW-NEXT:    kmovw %r10d, %k2
-; AVX512DQ-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm1 {%k2} = zmm0[2,3,2,3],zmm4[2,3,2,3]
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [4,12,0,5,4,12,0,5]
-; AVX512DQ-SLOW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm1
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm2, %zmm0, %zmm1
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm9
-; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [0,1,12,7,0,1,12,7]
-; AVX512DQ-SLOW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm4, %zmm0, %zmm1
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm22 = [5,0,14,6,5,0,14,6]
-; AVX512DQ-SLOW-NEXT:    # zmm22 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm19
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm3, %zmm22, %zmm19
-; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm1 = [0,13,6,7,0,13,6,7]
+; AVX512DQ-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm5 {%k2} = zmm1[2,3,2,3],zmm2[2,3,2,3]
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm8 = [4,12,0,5,4,12,0,5]
+; AVX512DQ-SLOW-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm5
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm3, %zmm8, %zmm5
+; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm1 = [0,1,12,7,0,1,12,7]
 ; AVX512DQ-SLOW-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm4, %zmm1, %zmm19
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm19
-; AVX512DQ-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm4 = [15,7,15,7,15,7,15,7]
-; AVX512DQ-SLOW-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm2, %zmm4, %zmm3
-; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [6,13,14,7,6,13,14,7]
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm2, %zmm1, %zmm5
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm5
+; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm21 = [5,0,14,6,5,0,14,6]
+; AVX512DQ-SLOW-NEXT:    # zmm21 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm19
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm0, %zmm21, %zmm19
+; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [0,13,6,7,0,13,6,7]
 ; AVX512DQ-SLOW-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm3, %zmm2, %zmm19
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm3
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm13, %zmm18, %zmm3
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm20, %zmm19
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm28, %zmm17, %zmm19
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm17, %zmm23
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm19 {%k1}
-; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm3 = ymm12[0],ymm11[0],ymm12[2],ymm11[2]
-; AVX512DQ-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm19 {%k2} = zmm3[2,3,2,3],zmm7[2,3,2,3]
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm5, %zmm2, %zmm19
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 64(%r8), %zmm3
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 64(%r9), %zmm12
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm11
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm12, %zmm14, %zmm11
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm7, %zmm16, %zmm11
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm11
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm12, %zmm9, %zmm11
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm7, %zmm0, %zmm11
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm12, %zmm11
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm3, %zmm17, %zmm0
+; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm3 = [6,13,14,7,6,13,14,7]
+; AVX512DQ-SLOW-NEXT:    # zmm3 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm0, %zmm3, %zmm5
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm9, %zmm0
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm10, %zmm25, %zmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm20, %zmm5
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm7, %zmm15, %zmm5
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm5 {%k1}
+; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm11[0],ymm14[0],ymm11[2],ymm14[2]
+; AVX512DQ-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm5 {%k2} = zmm0[2,3,2,3],zmm4[2,3,2,3]
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 64(%r8), %zmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 64(%r9), %zmm11
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm5
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm11, %zmm12, %zmm5
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm3, %zmm22, %zmm11
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm7, %zmm1, %zmm11
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm4, %zmm13, %zmm5
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm5
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm4
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm11, %zmm8, %zmm4
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm5, %zmm1, %zmm4
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm4
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm12, %zmm4, %zmm3
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm3, %zmm2, %zmm7
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 128(%rdi), %zmm24
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 128(%rsi), %zmm31
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm24, %zmm3
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm31, %zmm18, %zmm3
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 128(%rdx), %zmm17
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 128(%rcx), %zmm30
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm17, %zmm7
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm30, %zmm23, %zmm7
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm7 {%k1}
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 128(%rax), %zmm29
-; AVX512DQ-SLOW-NEXT:    vmovdqa 128(%r9), %ymm11
-; AVX512DQ-SLOW-NEXT:    vmovdqu %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa 128(%r8), %ymm3
-; AVX512DQ-SLOW-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm3 = ymm3[0],ymm11[0],ymm3[2],ymm11[2]
-; AVX512DQ-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm7 {%k2} = zmm3[2,3,2,3],zmm29[2,3,2,3]
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 128(%r8), %zmm3
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 128(%r9), %zmm7
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm7, %zmm3, %zmm9
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm29, %zmm0, %zmm9
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm3, %zmm7, %zmm22
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm29, %zmm1, %zmm22
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm22, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm0
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm7, %zmm14, %zmm0
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm29, %zmm16, %zmm0
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm7, %zmm4, %zmm3
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm3, %zmm2, %zmm29
-; AVX512DQ-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm3 = [13,5,13,5,13,5,13,5]
-; AVX512DQ-SLOW-NEXT:    # zmm3 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm21, %zmm0
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm5, %zmm3, %zmm0
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm0, %zmm21, %zmm4
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm5, %zmm2, %zmm4
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm25 = [6,14,6,14,6,14,6,14]
-; AVX512DQ-SLOW-NEXT:    # zmm25 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm21, %zmm0
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm5, %zmm25, %zmm0
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm11, %zmm17, %zmm0
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm0, %zmm3, %zmm5
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 128(%rdi), %zmm11
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 128(%rsi), %zmm19
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm5
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm19, %zmm25, %zmm5
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 128(%rdx), %zmm4
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 128(%rcx), %zmm31
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm0
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm31, %zmm15, %zmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm0 {%k1}
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 128(%rax), %zmm15
+; AVX512DQ-SLOW-NEXT:    vmovdqa 128(%r9), %ymm14
+; AVX512DQ-SLOW-NEXT:    vmovdqu %ymm14, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa 128(%r8), %ymm5
+; AVX512DQ-SLOW-NEXT:    vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm5 = ymm5[0],ymm14[0],ymm5[2],ymm14[2]
+; AVX512DQ-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm0 {%k2} = zmm5[2,3,2,3],zmm15[2,3,2,3]
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm21, %zmm0
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm5, %zmm4, %zmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 128(%r8), %zmm5
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 128(%r9), %zmm0
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm0, %zmm5, %zmm8
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm15, %zmm1, %zmm8
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm5, %zmm0, %zmm21
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm15, %zmm2, %zmm21
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm1
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm0, %zmm12, %zmm1
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [3,0,12,4,3,0,12,4]
-; AVX512DQ-SLOW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm21, %zmm0, %zmm5
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm20, %zmm26
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm28, %zmm3, %zmm26
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm20, %zmm1
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm28, %zmm25, %zmm1
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm20, %zmm1
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm28, %zmm4, %zmm1
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm15, %zmm13, %zmm1
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm20, %zmm0, %zmm28
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 192(%rdx), %zmm12
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 192(%rcx), %zmm2
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm12, %zmm1
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm2, %zmm25, %zmm1
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm17, %zmm11
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm30, %zmm3, %zmm11
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm30, %zmm17, %zmm25
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm17, %zmm22
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm30, %zmm4, %zmm22
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm17, %zmm0, %zmm30
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm16
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm12, %zmm0, %zmm16
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm2, %zmm12, %zmm23
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm23, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm12, %zmm0
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm2, %zmm4, %zmm12
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm2, %zmm3, %zmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm0, %zmm17, %zmm5
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm5, %zmm3, %zmm15
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm1 = [13,5,13,5,13,5,13,5]
+; AVX512DQ-SLOW-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm18, %zmm2
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm0
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm6, %zmm1, %zmm2
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm6 = [6,14,6,14,6,14,6,14]
+; AVX512DQ-SLOW-NEXT:    # zmm6 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm18, %zmm2
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm0, %zmm6, %zmm2
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm18, %zmm2
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm0, %zmm17, %zmm2
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm17, %zmm15
+; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm3 = [3,0,12,4,3,0,12,4]
+; AVX512DQ-SLOW-NEXT:    # zmm3 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm18, %zmm3, %zmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm2
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm20, %zmm0
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm7, %zmm1, %zmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm20, %zmm0
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm7, %zmm6, %zmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm20, %zmm0
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm7, %zmm17, %zmm0
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm20, %zmm3, %zmm7
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm5
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm29
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm31, %zmm1, %zmm29
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm30
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm31, %zmm6, %zmm30
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm0
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm31, %zmm17, %zmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm4, %zmm3, %zmm31
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 192(%rdx), %zmm12
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 192(%rcx), %zmm21
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm21, %zmm22
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm12, %zmm3, %zmm22
 ; AVX512DQ-SLOW-NEXT:    movb $48, %r10b
 ; AVX512DQ-SLOW-NEXT:    kmovw %r10d, %k3
-; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [1,0,10,2,1,0,10,2]
-; AVX512DQ-SLOW-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm15, %zmm20
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm10, %zmm4, %zmm20
-; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm5 {%k3} = zmm10[0],zmm15[0],zmm10[2],zmm15[2],zmm10[4],zmm15[4],zmm10[6],zmm15[6]
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 192(%rdi), %zmm8
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 192(%rsi), %zmm7
+; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm22 {%k3} = zmm8[0],zmm7[0],zmm8[2],zmm7[2],zmm8[4],zmm7[4],zmm8[6],zmm7[6]
+; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm14 = [1,0,10,2,1,0,10,2]
+; AVX512DQ-SLOW-NEXT:    # zmm14 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm23, %zmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm23, %zmm28
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm16, %zmm14, %zmm28
+; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm13 = [0,8,0,1,0,8,0,1]
+; AVX512DQ-SLOW-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm16, %zmm27
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm23, %zmm13, %zmm27
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm10, %zmm26
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm9, %zmm14, %zmm26
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm9, %zmm23
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm10, %zmm13, %zmm23
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm19, %zmm20
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm11, %zmm14, %zmm20
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm18
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm19, %zmm13, %zmm18
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm7, %zmm8, %zmm13
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm8, %zmm7, %zmm14
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm24
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm3
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm4
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm7, %zmm25, %zmm8
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm7, %zmm6, %zmm24
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm7, %zmm1, %zmm3
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm7, %zmm17, %zmm4
+; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm2 {%k3} = zmm16[0],zmm0[0],zmm16[2],zmm0[2],zmm16[4],zmm0[4],zmm16[6],zmm0[6]
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm16, %zmm25
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm16, %zmm2
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm16, %zmm17
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm0, %zmm1, %zmm25
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm0, %zmm6, %zmm2
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm0, %zmm15, %zmm17
+; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm5 {%k3} = zmm9[0],zmm10[0],zmm9[2],zmm10[2],zmm9[4],zmm10[4],zmm9[6],zmm10[6]
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm10, %zmm18
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm10, %zmm7
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm10, %zmm21
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm10, %zmm14
-; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [0,8,0,1,0,8,0,1]
-; AVX512DQ-SLOW-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm15, %zmm2, %zmm14
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm15, %zmm3, %zmm18
-; AVX512DQ-SLOW-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [6,14,6,14]
-; AVX512DQ-SLOW-NEXT:    # ymm1 = mem[0,1,0,1]
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm15, %zmm1, %zmm7
-; AVX512DQ-SLOW-NEXT:    vbroadcasti128 {{.*#+}} ymm6 = [15,7,15,7]
-; AVX512DQ-SLOW-NEXT:    # ymm6 = mem[0,1,0,1]
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm15, %zmm6, %zmm21
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm13, %zmm15
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm8, %zmm4, %zmm15
-; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm28 {%k3} = zmm8[0],zmm13[0],zmm8[2],zmm13[2],zmm8[4],zmm13[4],zmm8[6],zmm13[6]
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm23
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm5
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm17
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm10
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm13, %zmm2, %zmm10
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm13, %zmm3, %zmm23
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm13, %zmm1, %zmm5
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm13, %zmm6, %zmm17
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 192(%rdi), %zmm19
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 192(%rsi), %zmm0
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm19, %zmm27
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm0, %zmm1, %zmm27
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm19, %zmm13
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm0, %zmm3, %zmm13
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm31, %zmm8
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm24, %zmm4, %zmm8
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm31, %zmm24, %zmm3
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm31, %zmm24, %zmm1
-; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm30 {%k3} = zmm24[0],zmm31[0],zmm24[2],zmm31[2],zmm24[4],zmm31[4],zmm24[6],zmm31[6]
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm24, %zmm9
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm31, %zmm2, %zmm24
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm31, %zmm6, %zmm9
-; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm16 {%k3} = zmm19[0],zmm0[0],zmm19[2],zmm0[2],zmm19[4],zmm0[4],zmm19[6],zmm0[6]
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm0, %zmm19, %zmm2
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm19, %zmm0, %zmm4
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm31 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm0, %zmm19, %zmm31
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm31, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm0, %zmm6, %zmm19
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm18 {%k1}
-; AVX512DQ-SLOW-NEXT:    movb $120, %sil
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm9, %zmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm9, %zmm5
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm9, %zmm16
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm10, %zmm1, %zmm0
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm10, %zmm6, %zmm5
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm10, %zmm15, %zmm16
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm12, %zmm9
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm21, %zmm1, %zmm9
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm19, %zmm11, %zmm1
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm12, %zmm10
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm21, %zmm6, %zmm10
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm19, %zmm11, %zmm6
+; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm31 {%k3} = zmm11[0],zmm19[0],zmm11[2],zmm19[2],zmm11[4],zmm19[4],zmm11[6],zmm19[6]
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm7
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm19, %zmm15, %zmm7
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm12, %zmm11
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm21, %zmm15, %zmm11
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm21, %zmm15, %zmm12
+; AVX512DQ-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm24 = zmm24[0,1,2,3],zmm10[4,5,6,7]
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 192(%r8), %zmm19
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm10 = <0,11,u,u,4,5,6,7>
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm19, %zmm22, %zmm10
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm9, %zmm3 {%k1}
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm22 = <12,u,u,3,4,5,6,13>
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm19, %zmm3, %zmm22
+; AVX512DQ-SLOW-NEXT:    movb $24, %sil
 ; AVX512DQ-SLOW-NEXT:    kmovw %esi, %k3
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm18, %zmm0 {%k3}
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm7, %zmm18 # 64-byte Folded Reload
-; AVX512DQ-SLOW-NEXT:    # zmm18 = zmm7[0,1,2,3],mem[4,5,6,7]
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm4 {%k3}
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm21 = <u,1,2,3,4,15,u,u>
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm19, %zmm4, %zmm21
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm25 {%k1}
+; AVX512DQ-SLOW-NEXT:    movb $120, %sil
+; AVX512DQ-SLOW-NEXT:    kmovw %esi, %k4
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm25, %zmm11 {%k4}
+; AVX512DQ-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm3 # 64-byte Folded Reload
+; AVX512DQ-SLOW-NEXT:    # zmm3 = zmm2[0,1,2,3],mem[4,5,6,7]
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm0 {%k1}
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm15 {%k4}
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm29, %zmm1 {%k1}
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm25 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm25 {%k4}
 ; AVX512DQ-SLOW-NEXT:    movb $-61, %sil
 ; AVX512DQ-SLOW-NEXT:    kmovw %esi, %k4
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm18 {%k4}
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm26, %zmm23 {%k1}
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm26 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm23, %zmm26 {%k3}
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm3 {%k1}
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm0 {%k3}
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    movb $24, %sil
-; AVX512DQ-SLOW-NEXT:    kmovw %esi, %k3
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm3 {%k4}
+; AVX512DQ-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm4 # 64-byte Folded Reload
+; AVX512DQ-SLOW-NEXT:    # zmm4 = zmm5[0,1,2,3],mem[4,5,6,7]
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm21 {%k3}
-; AVX512DQ-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm3 # 64-byte Folded Reload
-; AVX512DQ-SLOW-NEXT:    # zmm3 = zmm5[0,1,2,3],mem[4,5,6,7]
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm4 {%k4}
+; AVX512DQ-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm6[0,1,2,3],zmm30[4,5,6,7]
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm3 {%k4}
-; AVX512DQ-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm25[4,5,6,7]
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm0 {%k4}
-; AVX512DQ-SLOW-NEXT:    movb $-31, %sil
-; AVX512DQ-SLOW-NEXT:    kmovw %esi, %k4
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm21 {%k4}
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm17 {%k3}
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm17 {%k4}
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm22, %zmm9 {%k3}
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm29, %zmm9 {%k4}
-; AVX512DQ-SLOW-NEXT:    vmovdqa (%rdx), %xmm1
-; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
-; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k4}
+; AVX512DQ-SLOW-NEXT:    vmovdqa (%rdx), %xmm0
+; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
+; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
 ; AVX512DQ-SLOW-NEXT:    movb $12, %sil
 ; AVX512DQ-SLOW-NEXT:    kmovw %esi, %k4
-; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm14 {%k4}
-; AVX512DQ-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm1 = [0,8,0,8,0,8,0,8]
-; AVX512DQ-SLOW-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm6 # 64-byte Folded Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqa 64(%rdx), %xmm5
-; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm5 = xmm5[0],mem[0]
-; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm5, %zmm0, %zmm10 {%k4}
-; AVX512DQ-SLOW-NEXT:    vmovdqa 128(%rdx), %xmm5
-; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm5 = xmm5[0],mem[0]
-; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm5, %zmm0, %zmm24 {%k4}
-; AVX512DQ-SLOW-NEXT:    vmovdqa 192(%rdx), %xmm5
-; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm5 = xmm5[0],mem[0]
-; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm5, %zmm0, %zmm2 {%k4}
-; AVX512DQ-SLOW-NEXT:    movb $112, %sil
-; AVX512DQ-SLOW-NEXT:    kmovw %esi, %k4
-; AVX512DQ-SLOW-NEXT:    vinserti64x2 $3, (%rax), %zmm6, %zmm14 {%k4}
+; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm27 {%k4}
+; AVX512DQ-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [0,8,0,8,0,8,0,8]
+; AVX512DQ-SLOW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 # 64-byte Folded Reload
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm5 # 64-byte Folded Reload
-; AVX512DQ-SLOW-NEXT:    vinserti64x2 $3, 64(%rax), %zmm5, %zmm10 {%k4}
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm17 {%k3}
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm5 # 64-byte Folded Reload
-; AVX512DQ-SLOW-NEXT:    vinserti64x2 $3, 128(%rax), %zmm5, %zmm24 {%k4}
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 192(%r8), %zmm5
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 192(%r9), %zmm11
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm11, %zmm5, %zmm1
-; AVX512DQ-SLOW-NEXT:    vinserti64x2 $3, 192(%rax), %zmm1, %zmm2 {%k4}
-; AVX512DQ-SLOW-NEXT:    vpbroadcastq 8(%rcx), %ymm1
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = mem[0,1,2,3],ymm1[4,5,6,7]
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm16 {%k3}
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm7 {%k3}
+; AVX512DQ-SLOW-NEXT:    movb $-31, %sil
+; AVX512DQ-SLOW-NEXT:    kmovw %esi, %k3
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm17 {%k3}
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm16 {%k3}
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm7 {%k3}
+; AVX512DQ-SLOW-NEXT:    movb $112, %sil
+; AVX512DQ-SLOW-NEXT:    kmovw %esi, %k3
+; AVX512DQ-SLOW-NEXT:    vinserti64x2 $3, (%rax), %zmm2, %zmm27 {%k3}
+; AVX512DQ-SLOW-NEXT:    vmovdqa 64(%rdx), %xmm2
+; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],mem[0]
+; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm2, %zmm0, %zmm23 {%k4}
+; AVX512DQ-SLOW-NEXT:    vmovdqa 128(%rdx), %xmm2
+; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],mem[0]
+; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm2, %zmm0, %zmm18 {%k4}
+; AVX512DQ-SLOW-NEXT:    vmovdqa 192(%rdx), %xmm2
+; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],mem[0]
+; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm2, %zmm0, %zmm13 {%k4}
+; AVX512DQ-SLOW-NEXT:    vpbroadcastq 8(%rcx), %ymm2
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = mem[0,1,2,3],ymm2[4,5,6,7]
 ; AVX512DQ-SLOW-NEXT:    movb $6, %sil
 ; AVX512DQ-SLOW-NEXT:    kmovw %esi, %k4
-; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm20 {%k4}
-; AVX512DQ-SLOW-NEXT:    vpbroadcastq 72(%rcx), %ymm1
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = mem[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm15 {%k4}
-; AVX512DQ-SLOW-NEXT:    vpbroadcastq 136(%rcx), %ymm1
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = mem[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm8 {%k4}
-; AVX512DQ-SLOW-NEXT:    vpbroadcastq 200(%rcx), %ymm1
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = mem[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm4 {%k4}
-; AVX512DQ-SLOW-NEXT:    movb $56, %cl
+; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm2, %zmm0, %zmm28 {%k4}
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 # 64-byte Folded Reload
+; AVX512DQ-SLOW-NEXT:    vinserti64x2 $3, 64(%rax), %zmm2, %zmm23 {%k3}
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 # 64-byte Folded Reload
+; AVX512DQ-SLOW-NEXT:    vinserti64x2 $3, 128(%rax), %zmm2, %zmm18 {%k3}
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 192(%r9), %zmm2
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm2, %zmm19, %zmm0
+; AVX512DQ-SLOW-NEXT:    vinserti64x2 $3, 192(%rax), %zmm0, %zmm13 {%k3}
+; AVX512DQ-SLOW-NEXT:    movb $56, %sil
+; AVX512DQ-SLOW-NEXT:    kmovw %esi, %k3
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm28 {%k3}
+; AVX512DQ-SLOW-NEXT:    vpbroadcastq 72(%rcx), %ymm0
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
+; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm26 {%k4}
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm26 {%k3}
+; AVX512DQ-SLOW-NEXT:    vpbroadcastq 136(%rcx), %ymm0
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
+; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm20 {%k4}
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm20 {%k3}
+; AVX512DQ-SLOW-NEXT:    vpbroadcastq 200(%rcx), %ymm0
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
+; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm14 {%k4}
+; AVX512DQ-SLOW-NEXT:    movb $64, %cl
 ; AVX512DQ-SLOW-NEXT:    kmovw %ecx, %k4
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm20 {%k4}
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm15 {%k4}
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm8 {%k4}
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm11, %zmm5, %zmm6
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 192(%rax), %zmm1
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm1, %zmm7, %zmm6
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm4 {%k4}
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm13 {%k1}
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm23 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm23 {%k1}
-; AVX512DQ-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm27, %zmm7 # 64-byte Folded Reload
-; AVX512DQ-SLOW-NEXT:    # zmm7 = zmm27[0,1,2,3],mem[4,5,6,7]
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm22 = <0,11,u,u,4,5,6,7>
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm5, %zmm16, %zmm22
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm6 = <0,1,11,u,4,5,6,7>
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm11, %zmm22, %zmm6
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 192(%r8), %ymm16
-; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm16 = ymm16[0],mem[0],ymm16[2],mem[2]
-; AVX512DQ-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm23 {%k2} = zmm16[2,3,2,3],zmm1[2,3,2,3]
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm12, %zmm19 {%k3}
-; AVX512DQ-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload
-; AVX512DQ-SLOW-NEXT:    vpunpckhqdq (%rsp), %ymm12, %ymm12 # 32-byte Folded Reload
-; AVX512DQ-SLOW-NEXT:    # ymm12 = ymm12[1],mem[1],ymm12[3],mem[3]
-; AVX512DQ-SLOW-NEXT:    vpermq {{.*#+}} ymm12 = ymm12[0,2,3,3]
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm12 = ymm12[0,1,2,3,4,5],mem[6,7]
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm19, %zmm24 {%k4}
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm2, %zmm0, %zmm19
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 192(%rax), %zmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm0, %zmm5, %zmm19
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm19, %zmm14 {%k3}
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm5 = <0,1,11,u,4,5,6,7>
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm2, %zmm10, %zmm5
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm12 {%k1}
+; AVX512DQ-SLOW-NEXT:    vmovdqa 192(%r8), %ymm6
+; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm6 = ymm6[0],mem[0],ymm6[2],mem[2]
+; AVX512DQ-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm12 {%k2} = zmm6[2,3,2,3],zmm0[2,3,2,3]
+; AVX512DQ-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
+; AVX512DQ-SLOW-NEXT:    vpunpckhqdq (%rsp), %ymm6, %ymm6 # 32-byte Folded Reload
+; AVX512DQ-SLOW-NEXT:    # ymm6 = ymm6[1],mem[1],ymm6[3],mem[3]
+; AVX512DQ-SLOW-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[0,2,3,3]
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3,4,5],mem[6,7]
 ; AVX512DQ-SLOW-NEXT:    movb $14, %cl
 ; AVX512DQ-SLOW-NEXT:    kmovw %ecx, %k1
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm12, %zmm0, %zmm16 {%k1}
-; AVX512DQ-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload
-; AVX512DQ-SLOW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm12, %ymm12 # 32-byte Folded Reload
-; AVX512DQ-SLOW-NEXT:    # ymm12 = ymm12[1],mem[1],ymm12[3],mem[3]
-; AVX512DQ-SLOW-NEXT:    vpermq {{.*#+}} ymm12 = ymm12[0,2,3,3]
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm12 = ymm12[0,1,2,3,4,5],mem[6,7]
-; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm12, %zmm0, %zmm28 {%k1}
-; AVX512DQ-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload
-; AVX512DQ-SLOW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm12, %ymm12 # 32-byte Folded Reload
-; AVX512DQ-SLOW-NEXT:    # ymm12 = ymm12[1],mem[1],ymm12[3],mem[3]
-; AVX512DQ-SLOW-NEXT:    movb $64, %cl
-; AVX512DQ-SLOW-NEXT:    kmovw %ecx, %k2
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm7 {%k2}
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm6, %zmm0, %zmm10 {%k1}
+; AVX512DQ-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
+; AVX512DQ-SLOW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm6, %ymm6 # 32-byte Folded Reload
+; AVX512DQ-SLOW-NEXT:    # ymm6 = ymm6[1],mem[1],ymm6[3],mem[3]
+; AVX512DQ-SLOW-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[0,2,3,3]
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3,4,5],mem[6,7]
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm6, %zmm0, %zmm19 {%k1}
 ; AVX512DQ-SLOW-NEXT:    movb $8, %cl
 ; AVX512DQ-SLOW-NEXT:    kmovw %ecx, %k2
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm6 {%k2}
-; AVX512DQ-SLOW-NEXT:    vpermq {{.*#+}} ymm12 = ymm12[0,2,3,3]
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm12 = ymm12[0,1,2,3,4,5],mem[6,7]
-; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm12, %zmm0, %zmm30 {%k1}
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm12 = <12,u,u,3,4,5,6,13>
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm5, %zmm13, %zmm12
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm13 = <u,1,2,3,4,15,u,u>
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm5, %zmm19, %zmm13
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm5 = <0,12,u,3,4,5,6,7>
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm11, %zmm12, %zmm5
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm12 = <13,u,2,3,4,5,6,14>
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm11, %zmm7, %zmm12
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm7 = <u,1,2,3,4,5,15,u>
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm11, %zmm13, %zmm7
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm11 = [0,1,12,3,4,5,6,7]
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm1, %zmm5, %zmm11
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [0,13,2,3,4,5,6,7]
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm1, %zmm12, %zmm5
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm12 = [14,1,2,3,4,5,6,15]
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm1, %zmm7, %zmm12
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm5 {%k2}
+; AVX512DQ-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
+; AVX512DQ-SLOW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm6, %ymm6 # 32-byte Folded Reload
+; AVX512DQ-SLOW-NEXT:    # ymm6 = ymm6[1],mem[1],ymm6[3],mem[3]
+; AVX512DQ-SLOW-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[0,2,3,3]
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3,4,5],mem[6,7]
+; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm6, %zmm0, %zmm31 {%k1}
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm6 = <0,12,u,3,4,5,6,7>
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm2, %zmm22, %zmm6
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm8 = <13,u,2,3,4,5,6,14>
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm2, %zmm24, %zmm8
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm9 = <u,1,2,3,4,5,15,u>
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm2, %zmm21, %zmm9
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [0,1,12,3,4,5,6,7]
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm0, %zmm6, %zmm2
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [0,13,2,3,4,5,6,7]
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm0, %zmm8, %zmm6
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm8 = [14,1,2,3,4,5,6,15]
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm0, %zmm9, %zmm8
 ; AVX512DQ-SLOW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm23, 1472(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm4, 1408(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm2, 1344(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm9, 1280(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, 1216(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovaps %zmm0, 1152(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm30, 1088(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm12, 1472(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm14, 1408(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm13, 1344(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm7, 1280(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, 1216(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm25, 1152(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm31, 1088(%rax)
 ; AVX512DQ-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQ-SLOW-NEXT:    vmovaps %zmm0, 1024(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm8, 960(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm24, 896(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm17, 832(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm3, 768(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm26, 704(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm28, 640(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm20, 960(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm18, 896(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm16, 832(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm4, 768(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm15, 704(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm19, 640(%rax)
 ; AVX512DQ-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQ-SLOW-NEXT:    vmovaps %zmm0, 576(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm15, 512(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm10, 448(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm21, 384(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm18, 320(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovaps %zmm0, 256(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm16, 192(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm26, 512(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm23, 448(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm17, 384(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm3, 320(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm11, 256(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm10, 192(%rax)
 ; AVX512DQ-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQ-SLOW-NEXT:    vmovaps %zmm0, 128(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm20, 64(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm14, (%rax)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm12, 1728(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm5, 1664(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm11, 1600(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm6, 1536(%rax)
-; AVX512DQ-SLOW-NEXT:    addq $2120, %rsp # imm = 0x848
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm28, 64(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm27, (%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm8, 1728(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm6, 1664(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm2, 1600(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm5, 1536(%rax)
+; AVX512DQ-SLOW-NEXT:    addq $2184, %rsp # imm = 0x888
 ; AVX512DQ-SLOW-NEXT:    vzeroupper
 ; AVX512DQ-SLOW-NEXT:    retq
 ;
 ; AVX512DQ-FAST-LABEL: store_i64_stride7_vf32:
 ; AVX512DQ-FAST:       # %bb.0:
-; AVX512DQ-FAST-NEXT:    subq $2056, %rsp # imm = 0x808
+; AVX512DQ-FAST-NEXT:    subq $2088, %rsp # imm = 0x828
 ; AVX512DQ-FAST-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-FAST-NEXT:    vmovdqa64 (%rdi), %zmm10
-; AVX512DQ-FAST-NEXT:    vmovdqa64 64(%rdi), %zmm15
-; AVX512DQ-FAST-NEXT:    vmovdqa64 (%rsi), %zmm16
-; AVX512DQ-FAST-NEXT:    vmovdqa64 64(%rsi), %zmm18
-; AVX512DQ-FAST-NEXT:    vmovdqa64 64(%rdx), %zmm21
-; AVX512DQ-FAST-NEXT:    vmovdqa64 (%rdx), %zmm9
-; AVX512DQ-FAST-NEXT:    vmovdqa64 (%rcx), %zmm20
-; AVX512DQ-FAST-NEXT:    vmovdqa64 64(%rcx), %zmm17
-; AVX512DQ-FAST-NEXT:    vmovdqa64 (%r8), %zmm0
+; AVX512DQ-FAST-NEXT:    vmovdqa64 (%rdi), %zmm8
+; AVX512DQ-FAST-NEXT:    vmovdqa64 64(%rdi), %zmm9
+; AVX512DQ-FAST-NEXT:    vmovdqa64 (%rsi), %zmm26
+; AVX512DQ-FAST-NEXT:    vmovdqa64 64(%rsi), %zmm24
+; AVX512DQ-FAST-NEXT:    vmovdqa64 64(%rdx), %zmm14
+; AVX512DQ-FAST-NEXT:    vmovdqa64 (%rdx), %zmm11
+; AVX512DQ-FAST-NEXT:    vmovdqa64 (%rcx), %zmm22
+; AVX512DQ-FAST-NEXT:    vmovdqa64 64(%rcx), %zmm28
+; AVX512DQ-FAST-NEXT:    vmovdqa64 (%r8), %zmm3
 ; AVX512DQ-FAST-NEXT:    vmovdqa64 (%r9), %zmm6
-; AVX512DQ-FAST-NEXT:    vmovdqa64 (%rax), %zmm8
+; AVX512DQ-FAST-NEXT:    vmovdqa64 (%rax), %zmm5
 ; AVX512DQ-FAST-NEXT:    movb $96, %r10b
 ; AVX512DQ-FAST-NEXT:    kmovw %r10d, %k1
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm1 = [9,1,9,1,9,1,9,1]
-; AVX512DQ-FAST-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm2
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm6, %zmm1, %zmm2
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm1, %zmm11
-; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm1 = [4,9,0,3,4,9,0,3]
-; AVX512DQ-FAST-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm8, %zmm1, %zmm2
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm1, %zmm12
-; AVX512DQ-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm2 = [11,3,11,3,11,3,11,3]
-; AVX512DQ-FAST-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm10, %zmm1
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm16, %zmm2, %zmm1
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm2, %zmm14
-; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm3 = [2,10,0,3,2,10,0,3]
-; AVX512DQ-FAST-NEXT:    # zmm3 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm9, %zmm2
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm20, %zmm3, %zmm2
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm3, %zmm19
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm2, %zmm4
+; AVX512DQ-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm19 = [15,7,15,7,15,7,15,7]
+; AVX512DQ-FAST-NEXT:    # zmm19 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [9,1,9,1,9,1,9,1]
+; AVX512DQ-FAST-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm3, %zmm1
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm6, %zmm0, %zmm1
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm12
+; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [4,9,0,3,4,9,0,3]
+; AVX512DQ-FAST-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm5, %zmm0, %zmm1
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm13
+; AVX512DQ-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm31 = [11,3,11,3,11,3,11,3]
+; AVX512DQ-FAST-NEXT:    # zmm31 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm8, %zmm0
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm26, %zmm31, %zmm0
+; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [2,10,0,3,2,10,0,3]
+; AVX512DQ-FAST-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm11, %zmm1
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm22, %zmm2, %zmm1
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm2, %zmm18
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm1, %zmm10
 ; AVX512DQ-FAST-NEXT:    vmovdqa (%r9), %ymm1
-; AVX512DQ-FAST-NEXT:    vmovdqa 64(%r9), %ymm5
-; AVX512DQ-FAST-NEXT:    vmovdqa 128(%r9), %ymm3
+; AVX512DQ-FAST-NEXT:    vmovdqa 64(%r9), %ymm0
+; AVX512DQ-FAST-NEXT:    vmovdqa 128(%r9), %ymm4
 ; AVX512DQ-FAST-NEXT:    vmovdqa (%r8), %ymm7
-; AVX512DQ-FAST-NEXT:    vmovdqa64 64(%r8), %ymm23
-; AVX512DQ-FAST-NEXT:    vmovdqa64 128(%r8), %ymm22
+; AVX512DQ-FAST-NEXT:    vmovdqa64 64(%r8), %ymm16
+; AVX512DQ-FAST-NEXT:    vmovdqa 128(%r8), %ymm15
 ; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm2 = ymm7[0],ymm1[0],ymm7[2],ymm1[2]
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %ymm7, %ymm17
 ; AVX512DQ-FAST-NEXT:    movb $28, %r10b
 ; AVX512DQ-FAST-NEXT:    kmovw %r10d, %k2
-; AVX512DQ-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm4 {%k2} = zmm2[2,3,2,3],zmm8[2,3,2,3]
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa {{.*#+}} ymm4 = [1,3,7,7]
-; AVX512DQ-FAST-NEXT:    vpermt2q %ymm1, %ymm4, %ymm7
-; AVX512DQ-FAST-NEXT:    vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [4,12,0,5,4,12,0,5]
-; AVX512DQ-FAST-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm1
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm6, %zmm2, %zmm1
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm2, %zmm13
-; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm7 = [0,1,12,7,0,1,12,7]
-; AVX512DQ-FAST-NEXT:    # zmm7 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm8, %zmm7, %zmm1
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm24 = [5,0,14,6,5,0,14,6]
-; AVX512DQ-FAST-NEXT:    # zmm24 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm5, %zmm7
+; AVX512DQ-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm10 {%k2} = zmm2[2,3,2,3],zmm5[2,3,2,3]
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa {{.*#+}} ymm5 = [1,3,7,7]
+; AVX512DQ-FAST-NEXT:    vpermt2q %ymm1, %ymm5, %ymm17
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %ymm17, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm10 = [4,12,0,5,4,12,0,5]
+; AVX512DQ-FAST-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm3, %zmm2
 ; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm6, %zmm1
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm0, %zmm24, %zmm1
-; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [0,13,6,7,0,13,6,7]
-; AVX512DQ-FAST-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm8, %zmm2, %zmm1
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm6, %zmm10, %zmm2
+; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm6 = [0,1,12,7,0,1,12,7]
+; AVX512DQ-FAST-NEXT:    # zmm6 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm7, %zmm6, %zmm2
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm17 = [5,0,14,6,5,0,14,6]
+; AVX512DQ-FAST-NEXT:    # zmm17 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm1, %zmm20
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm1, %zmm2
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm26 = [15,7,15,7,15,7,15,7]
-; AVX512DQ-FAST-NEXT:    # zmm26 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm6, %zmm26, %zmm0
-; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm1 = [6,13,14,7,6,13,14,7]
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm3, %zmm17, %zmm20
+; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm1 = [0,13,6,7,0,13,6,7]
 ; AVX512DQ-FAST-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm8
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm23[0],ymm5[0],ymm23[2],ymm5[2]
-; AVX512DQ-FAST-NEXT:    vpermt2q %ymm5, %ymm4, %ymm23
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %ymm23, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm8 = ymm22[0],ymm3[0],ymm22[2],ymm3[2]
-; AVX512DQ-FAST-NEXT:    vpermt2q %ymm3, %ymm4, %ymm22
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %ymm22, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm15, %zmm23
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm15, %zmm3
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm7, %zmm1, %zmm20
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm20, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm2, %zmm19, %zmm3
+; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [6,13,14,7,6,13,14,7]
+; AVX512DQ-FAST-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm3, %zmm2, %zmm7
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm3 = ymm16[0],ymm0[0],ymm16[2],ymm0[2]
+; AVX512DQ-FAST-NEXT:    vpermt2q %ymm0, %ymm5, %ymm16
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %ymm16, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm15[0],ymm4[0],ymm15[2],ymm4[2]
+; AVX512DQ-FAST-NEXT:    vpermt2q %ymm4, %ymm5, %ymm15
+; AVX512DQ-FAST-NEXT:    vmovdqu %ymm15, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm9, %zmm4
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm24, %zmm31, %zmm4
 ; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm14, %zmm5
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm18, %zmm14, %zmm3
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm21, %zmm4
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm17, %zmm19, %zmm4
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm3, %zmm4 {%k1}
-; AVX512DQ-FAST-NEXT:    vmovdqa64 64(%rax), %zmm14
-; AVX512DQ-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm4 {%k2} = zmm0[2,3,2,3],zmm14[2,3,2,3]
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 64(%r8), %zmm0
-; AVX512DQ-FAST-NEXT:    vmovdqa64 64(%r9), %zmm22
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm22, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm4
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm22, %zmm11, %zmm4
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm12, %zmm3
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm28, %zmm18, %zmm5
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm4, %zmm5 {%k1}
+; AVX512DQ-FAST-NEXT:    vmovdqa64 64(%rax), %zmm7
+; AVX512DQ-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm5 {%k2} = zmm3[2,3,2,3],zmm7[2,3,2,3]
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 64(%r8), %zmm3
+; AVX512DQ-FAST-NEXT:    vmovdqa64 64(%r9), %zmm5
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm3, %zmm4
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm14, %zmm12, %zmm4
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm5, %zmm12, %zmm4
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm7, %zmm13, %zmm4
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm6
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm13, %zmm4
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm22, %zmm13, %zmm6
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm14, %zmm7, %zmm6
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm22, %zmm6
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm0, %zmm24, %zmm6
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm14, %zmm2, %zmm6
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm22, %zmm26, %zmm0
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm14
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm3, %zmm4
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm5, %zmm10, %zmm4
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm7, %zmm6, %zmm4
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm5, %zmm4
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm3, %zmm17, %zmm4
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm7, %zmm1, %zmm4
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm5, %zmm19, %zmm3
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm3, %zmm2, %zmm7
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-FAST-NEXT:    vmovdqa64 128(%rdi), %zmm27
-; AVX512DQ-FAST-NEXT:    vmovdqa64 128(%rsi), %zmm6
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm27, %zmm0
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm6, %zmm5, %zmm0
-; AVX512DQ-FAST-NEXT:    vmovdqa64 128(%rdx), %zmm13
-; AVX512DQ-FAST-NEXT:    vmovdqa64 128(%rcx), %zmm5
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm13, %zmm22
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm5, %zmm19, %zmm22
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm19, %zmm28
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm22 {%k1}
-; AVX512DQ-FAST-NEXT:    vmovdqa64 128(%rax), %zmm12
-; AVX512DQ-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm22 {%k2} = zmm8[2,3,2,3],zmm12[2,3,2,3]
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm22, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 128(%r8), %zmm0
-; AVX512DQ-FAST-NEXT:    vmovdqa64 128(%r9), %zmm8
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm8, %zmm0, %zmm4
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm12, %zmm7, %zmm4
+; AVX512DQ-FAST-NEXT:    vmovdqa64 128(%rsi), %zmm20
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm27, %zmm3
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm20, %zmm31, %zmm3
+; AVX512DQ-FAST-NEXT:    vmovdqa64 128(%rdx), %zmm29
+; AVX512DQ-FAST-NEXT:    vmovdqa64 128(%rcx), %zmm23
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm29, %zmm4
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm23, %zmm18, %zmm4
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm3, %zmm4 {%k1}
+; AVX512DQ-FAST-NEXT:    vmovdqa64 128(%rax), %zmm30
+; AVX512DQ-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm4 {%k2} = zmm0[2,3,2,3],zmm30[2,3,2,3]
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm0, %zmm8, %zmm24
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm12, %zmm2, %zmm24
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm24, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm2
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm8, %zmm11, %zmm2
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm12, %zmm3, %zmm2
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 128(%r8), %zmm0
+; AVX512DQ-FAST-NEXT:    vmovdqa64 128(%r9), %zmm3
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm3, %zmm0, %zmm10
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm30, %zmm6, %zmm10
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm0, %zmm3, %zmm17
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm30, %zmm1, %zmm17
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm1
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm3, %zmm12, %zmm1
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm30, %zmm13, %zmm1
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm3, %zmm19, %zmm0
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm0, %zmm2, %zmm30
+; AVX512DQ-FAST-NEXT:    vmovdqa64 192(%rdx), %zmm10
+; AVX512DQ-FAST-NEXT:    vmovdqa64 192(%rcx), %zmm1
+; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm25 = [3,0,12,4,3,0,12,4]
+; AVX512DQ-FAST-NEXT:    # zmm25 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm1, %zmm4
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm10, %zmm25, %zmm4
+; AVX512DQ-FAST-NEXT:    movb $48, %r10b
+; AVX512DQ-FAST-NEXT:    kmovw %r10d, %k3
+; AVX512DQ-FAST-NEXT:    vmovdqa64 192(%rdi), %zmm18
+; AVX512DQ-FAST-NEXT:    vmovdqa64 192(%rsi), %zmm7
+; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm4 {%k3} = zmm18[0],zmm7[0],zmm18[2],zmm7[2],zmm18[4],zmm7[4],zmm18[6],zmm7[6]
+; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm16 = [0,8,0,1,0,8,0,1]
+; AVX512DQ-FAST-NEXT:    # zmm16 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm8, %zmm0
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm26, %zmm16, %zmm0
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm8, %zmm26, %zmm0
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm12
-; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm15 = [3,0,12,4,3,0,12,4]
+; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm15 = [1,0,10,2,1,0,10,2]
 ; AVX512DQ-FAST-NEXT:    # zmm15 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm20, %zmm1
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm26, %zmm0
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm8, %zmm15, %zmm0
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm9, %zmm0
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm9, %zmm15, %zmm1
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm1, %zmm3
-; AVX512DQ-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm1 = [13,5,13,5,13,5,13,5]
-; AVX512DQ-FAST-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm9, %zmm2
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm20, %zmm1, %zmm2
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm9 = [6,14,6,14,6,14,6,14]
-; AVX512DQ-FAST-NEXT:    # zmm9 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm2
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm20, %zmm9, %zmm2
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm20, %zmm26, %zmm0
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm24, %zmm16, %zmm0
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    movb $48, %r10b
-; AVX512DQ-FAST-NEXT:    kmovw %r10d, %k3
-; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm11 = [0,8,0,1,0,8,0,1]
-; AVX512DQ-FAST-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm10, %zmm29
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm16, %zmm11, %zmm29
-; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm8 = [1,0,10,2,1,0,10,2]
-; AVX512DQ-FAST-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm16, %zmm31
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm10, %zmm8, %zmm31
-; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm3 {%k3} = zmm10[0],zmm16[0],zmm10[2],zmm16[2],zmm10[4],zmm16[4],zmm10[6],zmm16[6]
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm3, (%rsp) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm10, %zmm7
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm16, %zmm1, %zmm7
-; AVX512DQ-FAST-NEXT:    vbroadcasti128 {{.*#+}} ymm4 = [6,14,6,14]
-; AVX512DQ-FAST-NEXT:    # ymm4 = mem[0,1,0,1]
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm10, %zmm24
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm16, %zmm4, %zmm24
-; AVX512DQ-FAST-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [15,7,15,7]
-; AVX512DQ-FAST-NEXT:    # ymm2 = mem[0,1,0,1]
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm16, %zmm2, %zmm10
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm17, %zmm22
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm21, %zmm15, %zmm22
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm21, %zmm25
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm17, %zmm1, %zmm25
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm21, %zmm30
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm17, %zmm9, %zmm30
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm17, %zmm26, %zmm21
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm23, %zmm0
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm23, %zmm19
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm18, %zmm11, %zmm19
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm18, %zmm17
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm23, %zmm8, %zmm17
-; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm22 {%k3} = zmm23[0],zmm18[0],zmm23[2],zmm18[2],zmm23[4],zmm18[4],zmm23[6],zmm18[6]
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm23, %zmm3
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm18, %zmm1, %zmm3
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm18, %zmm4, %zmm23
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm18, %zmm2, %zmm0
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm24, %zmm0
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm9, %zmm15, %zmm0
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 192(%rdx), %zmm18
-; AVX512DQ-FAST-NEXT:    vmovdqa64 192(%rcx), %zmm0
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm18, %zmm10
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm0, %zmm9, %zmm10
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm16
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm18, %zmm15, %zmm16
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm18, %zmm10
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm10
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm0, %zmm18, %zmm28
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm28, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm0, %zmm26, %zmm18
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm13, %zmm5, %zmm15
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm5, %zmm13, %zmm9
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm13, %zmm28
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm5, %zmm26, %zmm13
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm5, %zmm1, %zmm28
-; AVX512DQ-FAST-NEXT:    vmovdqa64 192(%rdi), %zmm20
-; AVX512DQ-FAST-NEXT:    vmovdqa64 192(%rsi), %zmm0
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm20, %zmm21
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm0, %zmm4, %zmm21
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm20, %zmm10
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm10
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm27, %zmm26
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm6, %zmm11, %zmm26
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm6, %zmm5
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm27, %zmm8, %zmm5
-; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm15 {%k3} = zmm27[0],zmm6[0],zmm27[2],zmm6[2],zmm27[4],zmm6[4],zmm27[6],zmm6[6]
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm6, %zmm27, %zmm1
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm6, %zmm27, %zmm4
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm6, %zmm2, %zmm27
-; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm16 {%k3} = zmm20[0],zmm0[0],zmm20[2],zmm0[2],zmm20[4],zmm0[4],zmm20[6],zmm0[6]
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm0, %zmm20, %zmm11
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm20, %zmm0, %zmm8
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm0, %zmm20, %zmm6
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm0, %zmm2, %zmm20
-; AVX512DQ-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],mem[6,7]
-; AVX512DQ-FAST-NEXT:    movb $14, %sil
-; AVX512DQ-FAST-NEXT:    kmovw %esi, %k3
-; AVX512DQ-FAST-NEXT:    vmovdqu64 (%rsp), %zmm2 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm2 {%k3}
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm2, (%rsp) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm7 {%k1}
-; AVX512DQ-FAST-NEXT:    movb $120, %sil
-; AVX512DQ-FAST-NEXT:    kmovw %esi, %k4
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm7, %zmm0 {%k4}
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm27, %zmm0
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm20, %zmm16, %zmm0
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm24, %zmm2 # 64-byte Folded Reload
-; AVX512DQ-FAST-NEXT:    # zmm2 = zmm24[0,1,2,3],mem[4,5,6,7]
-; AVX512DQ-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],mem[6,7]
-; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm22 {%k3}
-; AVX512DQ-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],mem[6,7]
-; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm15 {%k3}
-; AVX512DQ-FAST-NEXT:    movb $-61, %sil
-; AVX512DQ-FAST-NEXT:    kmovw %esi, %k5
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k5}
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm20, %zmm0
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm27, %zmm15, %zmm0
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm7, %zmm18, %zmm16
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm18, %zmm7, %zmm15
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm18, %zmm17
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm18, %zmm6
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm18, %zmm21
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm7, %zmm31, %zmm18
+; AVX512DQ-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [6,14,6,14,6,14,6,14]
+; AVX512DQ-FAST-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm7, %zmm0, %zmm17
+; AVX512DQ-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm13 = [13,5,13,5,13,5,13,5]
+; AVX512DQ-FAST-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm7, %zmm13, %zmm6
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm7, %zmm19, %zmm21
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm22, %zmm3
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm22, %zmm31
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm11, %zmm2
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm11, %zmm25, %zmm31
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm22, %zmm13, %zmm11
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm2, %zmm22
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm3, %zmm0, %zmm22
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm3, %zmm19, %zmm2
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm25, %zmm3 {%k1}
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm3, %zmm7 {%k4}
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm28, %zmm1 {%k1}
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm24 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm1, %zmm24 {%k4}
+; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm31 {%k3} = zmm8[0],zmm26[0],zmm8[2],zmm26[2],zmm8[4],zmm26[4],zmm8[6],zmm26[6]
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm8, %zmm3
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm26, %zmm13, %zmm3
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm8, %zmm12
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm26, %zmm0, %zmm12
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm26, %zmm19, %zmm8
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm28, %zmm26
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm14, %zmm2
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm14, %zmm25, %zmm26
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm14, %zmm7
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm28, %zmm13, %zmm7
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm28, %zmm0, %zmm14
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm28, %zmm19, %zmm2
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm26 {%k3} = zmm9[0],zmm24[0],zmm9[2],zmm24[2],zmm9[4],zmm24[4],zmm9[6],zmm24[6]
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm9, %zmm2
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm24, %zmm13, %zmm2
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm9, %zmm5
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm24, %zmm0, %zmm5
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm24, %zmm19, %zmm9
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm29, %zmm23, %zmm25
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm29, %zmm28
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm23, %zmm13, %zmm28
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm29, %zmm24
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm23, %zmm0, %zmm24
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm23, %zmm19, %zmm29
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm10, %zmm23
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm1, %zmm0, %zmm23
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm10, %zmm8
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm1, %zmm13, %zmm8
+; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm25 {%k3} = zmm27[0],zmm20[0],zmm27[2],zmm20[2],zmm27[4],zmm20[4],zmm27[6],zmm20[6]
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm20, %zmm27, %zmm13
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm20, %zmm27, %zmm0
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm20, %zmm19, %zmm27
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm10, %zmm9
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm1, %zmm19, %zmm9
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm1, %zmm19, %zmm10
+; AVX512DQ-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm23 = zmm17[0,1,2,3],zmm23[4,5,6,7]
+; AVX512DQ-FAST-NEXT:    vmovdqa64 192(%r8), %zmm1
+; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm17 = <0,11,u,u,4,5,6,7>
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm1, %zmm4, %zmm17
+; AVX512DQ-FAST-NEXT:    vmovdqa64 192(%r9), %zmm4
+; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm20 = <0,1,11,u,4,5,6,7>
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm4, %zmm17, %zmm20
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm8, %zmm6 {%k1}
+; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm8 = <12,u,u,3,4,5,6,13>
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm1, %zmm6, %zmm8
 ; AVX512DQ-FAST-NEXT:    movb $24, %sil
 ; AVX512DQ-FAST-NEXT:    kmovw %esi, %k3
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm6 {%k3}
-; AVX512DQ-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm23[0,1,2,3],zmm30[4,5,6,7]
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k5}
-; AVX512DQ-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm4[0,1,2,3],zmm9[4,5,6,7]
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k5}
-; AVX512DQ-FAST-NEXT:    movb $-31, %sil
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm9, %zmm21 {%k3}
+; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm6 = <u,1,2,3,4,15,u,u>
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm1, %zmm21, %zmm6
+; AVX512DQ-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3,4,5],mem[6,7]
+; AVX512DQ-FAST-NEXT:    movb $14, %sil
 ; AVX512DQ-FAST-NEXT:    kmovw %esi, %k4
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm6 {%k4}
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm6, %zmm4
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm28 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm28 {%k3}
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm14, %zmm28 {%k4}
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm13, %zmm27 {%k3}
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm12, %zmm27 {%k4}
-; AVX512DQ-FAST-NEXT:    vmovdqa (%rdx), %xmm0
-; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
-; AVX512DQ-FAST-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm9, %zmm0, %zmm31 {%k4}
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm11, %zmm3 {%k1}
+; AVX512DQ-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3,4,5],mem[6,7]
+; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm9, %zmm0, %zmm26 {%k4}
+; AVX512DQ-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3,4,5],mem[6,7]
+; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm9, %zmm0, %zmm25 {%k4}
+; AVX512DQ-FAST-NEXT:    movb $120, %sil
+; AVX512DQ-FAST-NEXT:    kmovw %esi, %k4
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm3, %zmm9 {%k4}
+; AVX512DQ-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm21 = zmm12[0,1,2,3],zmm22[4,5,6,7]
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm7, %zmm2 {%k1}
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm2, %zmm7 {%k4}
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm28, %zmm13 {%k1}
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm13, %zmm11 {%k4}
+; AVX512DQ-FAST-NEXT:    movb $-61, %sil
+; AVX512DQ-FAST-NEXT:    kmovw %esi, %k4
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm2, %zmm21 {%k4}
+; AVX512DQ-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm13 = zmm5[0,1,2,3],zmm14[4,5,6,7]
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm2, %zmm13 {%k4}
+; AVX512DQ-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm24[4,5,6,7]
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm2, %zmm0 {%k4}
+; AVX512DQ-FAST-NEXT:    vmovdqa (%rdx), %xmm2
+; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],mem[0]
+; AVX512DQ-FAST-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
 ; AVX512DQ-FAST-NEXT:    movb $12, %sil
 ; AVX512DQ-FAST-NEXT:    kmovw %esi, %k4
-; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm29 {%k4}
-; AVX512DQ-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [0,8,0,8,0,8,0,8]
-; AVX512DQ-FAST-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm6 # 64-byte Folded Reload
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm2, %zmm0, %zmm12 {%k4}
+; AVX512DQ-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm2 = [0,8,0,8,0,8,0,8]
+; AVX512DQ-FAST-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm3 # 64-byte Folded Reload
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm28 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm5, %zmm28 {%k3}
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm5, %zmm14 {%k3}
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm29, %zmm27 {%k3}
+; AVX512DQ-FAST-NEXT:    movb $-31, %sil
+; AVX512DQ-FAST-NEXT:    kmovw %esi, %k3
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm5, %zmm28 {%k3}
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm5, %zmm14 {%k3}
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm14, %zmm29
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm30, %zmm27 {%k3}
+; AVX512DQ-FAST-NEXT:    movb $112, %sil
+; AVX512DQ-FAST-NEXT:    kmovw %esi, %k3
+; AVX512DQ-FAST-NEXT:    vinserti64x2 $3, (%rax), %zmm3, %zmm12 {%k3}
 ; AVX512DQ-FAST-NEXT:    vmovdqa 64(%rdx), %xmm3
 ; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],mem[0]
 ; AVX512DQ-FAST-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm19 {%k4}
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm14 {%k4}
 ; AVX512DQ-FAST-NEXT:    vmovdqa 128(%rdx), %xmm3
 ; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],mem[0]
 ; AVX512DQ-FAST-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm26 {%k4}
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm19 {%k4}
 ; AVX512DQ-FAST-NEXT:    vmovdqa 192(%rdx), %xmm3
 ; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],mem[0]
 ; AVX512DQ-FAST-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm11 {%k4}
-; AVX512DQ-FAST-NEXT:    movb $112, %sil
+; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm16 {%k4}
+; AVX512DQ-FAST-NEXT:    vpbroadcastq 8(%rcx), %ymm3
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm3 = mem[0,1,2,3],ymm3[4,5,6,7]
+; AVX512DQ-FAST-NEXT:    movb $6, %sil
 ; AVX512DQ-FAST-NEXT:    kmovw %esi, %k4
-; AVX512DQ-FAST-NEXT:    vinserti64x2 $3, (%rax), %zmm6, %zmm29 {%k4}
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm5 {%k4}
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm3 # 64-byte Folded Reload
-; AVX512DQ-FAST-NEXT:    vinserti64x2 $3, 64(%rax), %zmm3, %zmm19 {%k4}
+; AVX512DQ-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm3 # 64-byte Folded Reload
+; AVX512DQ-FAST-NEXT:    vinserti64x2 $3, 64(%rax), %zmm3, %zmm14 {%k3}
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm14, %zmm17
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm3 # 64-byte Folded Reload
-; AVX512DQ-FAST-NEXT:    vinserti64x2 $3, 128(%rax), %zmm3, %zmm26 {%k4}
-; AVX512DQ-FAST-NEXT:    vmovdqa64 192(%r8), %zmm3
-; AVX512DQ-FAST-NEXT:    vmovdqa64 192(%r9), %zmm6
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm6, %zmm3, %zmm0
-; AVX512DQ-FAST-NEXT:    vinserti64x2 $3, 192(%rax), %zmm0, %zmm11 {%k4}
-; AVX512DQ-FAST-NEXT:    vpbroadcastq 8(%rcx), %ymm0
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
-; AVX512DQ-FAST-NEXT:    movb $6, %sil
-; AVX512DQ-FAST-NEXT:    kmovw %esi, %k4
-; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm31 {%k4}
-; AVX512DQ-FAST-NEXT:    vpbroadcastq 72(%rcx), %ymm0
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
-; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm17 {%k4}
-; AVX512DQ-FAST-NEXT:    vpbroadcastq 136(%rcx), %ymm0
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
-; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm5 {%k4}
-; AVX512DQ-FAST-NEXT:    vpbroadcastq 200(%rcx), %ymm0
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
-; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm8 {%k4}
-; AVX512DQ-FAST-NEXT:    movb $56, %cl
+; AVX512DQ-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm3 # 64-byte Folded Reload
+; AVX512DQ-FAST-NEXT:    vinserti64x2 $3, 128(%rax), %zmm3, %zmm19 {%k3}
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm19, %zmm22
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm4, %zmm1, %zmm2
+; AVX512DQ-FAST-NEXT:    vinserti64x2 $3, 192(%rax), %zmm2, %zmm16 {%k3}
+; AVX512DQ-FAST-NEXT:    movb $56, %sil
+; AVX512DQ-FAST-NEXT:    kmovw %esi, %k3
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm2, %zmm5 {%k3}
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm5, %zmm14
+; AVX512DQ-FAST-NEXT:    vpbroadcastq 72(%rcx), %ymm2
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm2 = mem[0,1,2,3],ymm2[4,5,6,7]
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm2, %zmm0, %zmm3 {%k4}
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k3}
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm3, %zmm19
+; AVX512DQ-FAST-NEXT:    vpbroadcastq 136(%rcx), %ymm2
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm2 = mem[0,1,2,3],ymm2[4,5,6,7]
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm2, %zmm0, %zmm3 {%k4}
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k3}
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm3, %zmm24
+; AVX512DQ-FAST-NEXT:    vpbroadcastq 200(%rcx), %ymm2
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm2 = mem[0,1,2,3],ymm2[4,5,6,7]
+; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm2, %zmm0, %zmm15 {%k4}
+; AVX512DQ-FAST-NEXT:    movb $64, %cl
 ; AVX512DQ-FAST-NEXT:    kmovw %ecx, %k4
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm31 {%k4}
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm17 {%k4}
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm5 {%k4}
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm6, %zmm3, %zmm9
-; AVX512DQ-FAST-NEXT:    vmovdqa64 192(%rax), %zmm0
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm0, %zmm12, %zmm9
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm9, %zmm8 {%k4}
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm9, %zmm10 {%k1}
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm9, %zmm14 {%k1}
-; AVX512DQ-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm21, %zmm9 # 64-byte Folded Reload
-; AVX512DQ-FAST-NEXT:    # zmm9 = zmm21[0,1,2,3],mem[4,5,6,7]
-; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm12 = <0,11,u,u,4,5,6,7>
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm3, %zmm16, %zmm12
-; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm13 = <0,1,11,u,4,5,6,7>
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm6, %zmm12, %zmm13
-; AVX512DQ-FAST-NEXT:    vmovdqa 192(%r8), %ymm12
-; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm12 = ymm12[0],mem[0],ymm12[2],mem[2]
-; AVX512DQ-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm14 {%k2} = zmm12[2,3,2,3],zmm0[2,3,2,3]
-; AVX512DQ-FAST-NEXT:    movb $64, %al
-; AVX512DQ-FAST-NEXT:    kmovw %eax, %k1
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm3, %zmm9 {%k1}
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm1, %zmm23 {%k4}
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm4, %zmm2, %zmm1
+; AVX512DQ-FAST-NEXT:    vmovdqa64 192(%rax), %zmm2
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm2, %zmm3, %zmm1
 ; AVX512DQ-FAST-NEXT:    movb $8, %al
-; AVX512DQ-FAST-NEXT:    kmovw %eax, %k1
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm13 {%k1}
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm18, %zmm20 {%k3}
-; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm12 = <12,u,u,3,4,5,6,13>
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm3, %zmm10, %zmm12
-; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm10 = <u,1,2,3,4,15,u,u>
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm3, %zmm20, %zmm10
-; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm3 = <0,12,u,3,4,5,6,7>
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm6, %zmm12, %zmm3
-; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm12 = <13,u,2,3,4,5,6,14>
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm6, %zmm9, %zmm12
-; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm9 = <u,1,2,3,4,5,15,u>
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm6, %zmm10, %zmm9
-; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [0,1,12,3,4,5,6,7]
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm0, %zmm3, %zmm6
-; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [0,13,2,3,4,5,6,7]
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm0, %zmm12, %zmm3
-; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm10 = [14,1,2,3,4,5,6,15]
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm0, %zmm9, %zmm10
+; AVX512DQ-FAST-NEXT:    kmovw %eax, %k4
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm2, %zmm20 {%k4}
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm1, %zmm15 {%k3}
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm18, %zmm10 {%k1}
+; AVX512DQ-FAST-NEXT:    vmovdqa 192(%r8), %ymm1
+; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm1 = ymm1[0],mem[0],ymm1[2],mem[2]
+; AVX512DQ-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm10 {%k2} = zmm1[2,3,2,3],zmm2[2,3,2,3]
+; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm1 = <0,12,u,3,4,5,6,7>
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm4, %zmm8, %zmm1
+; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm3 = <13,u,2,3,4,5,6,14>
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm4, %zmm23, %zmm3
+; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm5 = <u,1,2,3,4,5,15,u>
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm4, %zmm6, %zmm5
+; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [0,1,12,3,4,5,6,7]
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm2, %zmm1, %zmm4
+; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,13,2,3,4,5,6,7]
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm2, %zmm3, %zmm1
+; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [14,1,2,3,4,5,6,15]
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm2, %zmm5, %zmm3
 ; AVX512DQ-FAST-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm14, 1472(%rax)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm8, 1408(%rax)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm11, 1344(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm10, 1472(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm15, 1408(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm16, 1344(%rax)
 ; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm27, 1280(%rax)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm2, 1216(%rax)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm24, 1152(%rax)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm15, 1088(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, 1216(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm11, 1152(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm25, 1088(%rax)
 ; AVX512DQ-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQ-FAST-NEXT:    vmovaps %zmm0, 1024(%rax)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm5, 960(%rax)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm26, 896(%rax)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm28, 832(%rax)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm1, 768(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm24, 960(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm22, 896(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm29, 832(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm13, 768(%rax)
 ; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm7, 704(%rax)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm22, 640(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm26, 640(%rax)
 ; AVX512DQ-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQ-FAST-NEXT:    vmovaps %zmm0, 576(%rax)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm17, 512(%rax)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm19, 448(%rax)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm4, 384(%rax)
-; AVX512DQ-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovaps %zmm0, 320(%rax)
-; AVX512DQ-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovaps %zmm0, 256(%rax)
-; AVX512DQ-FAST-NEXT:    vmovups (%rsp), %zmm0 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovaps %zmm0, 192(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm19, 512(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm17, 448(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm28, 384(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm21, 320(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm9, 256(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm31, 192(%rax)
 ; AVX512DQ-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQ-FAST-NEXT:    vmovaps %zmm0, 128(%rax)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm31, 64(%rax)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm29, (%rax)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm10, 1728(%rax)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm3, 1664(%rax)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm6, 1600(%rax)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm13, 1536(%rax)
-; AVX512DQ-FAST-NEXT:    addq $2056, %rsp # imm = 0x808
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm14, 64(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm12, (%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm3, 1728(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm1, 1664(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm4, 1600(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm20, 1536(%rax)
+; AVX512DQ-FAST-NEXT:    addq $2088, %rsp # imm = 0x828
 ; AVX512DQ-FAST-NEXT:    vzeroupper
 ; AVX512DQ-FAST-NEXT:    retq
 ;
 ; AVX512BW-ONLY-SLOW-LABEL: store_i64_stride7_vf32:
 ; AVX512BW-ONLY-SLOW:       # %bb.0:
-; AVX512BW-ONLY-SLOW-NEXT:    subq $2120, %rsp # imm = 0x848
+; AVX512BW-ONLY-SLOW-NEXT:    subq $2184, %rsp # imm = 0x888
 ; AVX512BW-ONLY-SLOW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 (%rdi), %zmm8
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 64(%rdi), %zmm11
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 (%rsi), %zmm27
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 64(%rsi), %zmm13
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 64(%rdx), %zmm18
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 (%rdx), %zmm21
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 (%rcx), %zmm7
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 64(%rcx), %zmm25
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 (%rdi), %zmm17
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 64(%rdi), %zmm14
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 (%rsi), %zmm16
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 64(%rsi), %zmm23
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 64(%rdx), %zmm6
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 (%rdx), %zmm7
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 (%rcx), %zmm8
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 64(%rcx), %zmm29
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 (%r8), %zmm0
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 (%r9), %zmm4
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 (%r9), %zmm3
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 (%rax), %zmm1
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 64(%rax), %zmm5
 ; AVX512BW-ONLY-SLOW-NEXT:    movb $96, %r10b
 ; AVX512BW-ONLY-SLOW-NEXT:    kmovd %r10d, %k1
+; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm19 = [15,7,15,7,15,7,15,7]
+; AVX512BW-ONLY-SLOW-NEXT:    # zmm19 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm2 = [9,1,9,1,9,1,9,1]
 ; AVX512BW-ONLY-SLOW-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm6
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm4, %zmm2, %zmm6
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm14
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm3 = [4,9,0,3,4,9,0,3]
-; AVX512BW-ONLY-SLOW-NEXT:    # zmm3 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm10
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm3, %zmm2, %zmm10
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm11
+; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [4,9,0,3,4,9,0,3]
+; AVX512BW-ONLY-SLOW-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm3, %zmm6
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm15
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm3 = [11,3,11,3,11,3,11,3]
-; AVX512BW-ONLY-SLOW-NEXT:    # zmm3 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm1
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm27, %zmm3, %zmm1
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm17
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm6 = [2,10,0,3,2,10,0,3]
-; AVX512BW-ONLY-SLOW-NEXT:    # zmm6 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm21, %zmm3
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm7, %zmm6, %zmm3
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm16
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm3 {%k1}
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm4, %zmm10
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm12
+; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm15 = [11,3,11,3,11,3,11,3]
+; AVX512BW-ONLY-SLOW-NEXT:    # zmm15 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm17, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm16, %zmm15, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm10 = [2,10,0,3,2,10,0,3]
+; AVX512BW-ONLY-SLOW-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm4
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm8, %zmm10, %zmm4
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm10, %zmm22
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa (%r9), %ymm10
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu %ymm10, (%rsp) # 32-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa 64(%r9), %ymm6
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa 64(%r9), %ymm13
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu %ymm13, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa (%r8), %ymm1
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa 64(%r8), %ymm12
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 64(%r8), %ymm18
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %ymm18, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm1 = ymm1[0],ymm10[0],ymm1[2],ymm10[2]
 ; AVX512BW-ONLY-SLOW-NEXT:    movb $28, %r10b
 ; AVX512BW-ONLY-SLOW-NEXT:    kmovd %r10d, %k2
-; AVX512BW-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm3 {%k2} = zmm1[2,3,2,3],zmm2[2,3,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm1 = [4,12,0,5,4,12,0,5]
-; AVX512BW-ONLY-SLOW-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm3
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm4, %zmm1, %zmm3
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm10
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm1 = [0,1,12,7,0,1,12,7]
-; AVX512BW-ONLY-SLOW-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm2, %zmm1, %zmm3
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm22 = [5,0,14,6,5,0,14,6]
-; AVX512BW-ONLY-SLOW-NEXT:    # zmm22 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm19
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm20
+; AVX512BW-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm4 {%k2} = zmm1[2,3,2,3],zmm2[2,3,2,3]
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm22, %zmm19
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm3 = [0,13,6,7,0,13,6,7]
-; AVX512BW-ONLY-SLOW-NEXT:    # zmm3 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm2, %zmm3, %zmm19
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm19
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm4 = [15,7,15,7,15,7,15,7]
-; AVX512BW-ONLY-SLOW-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm20, %zmm4, %zmm0
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [6,13,14,7,6,13,14,7]
-; AVX512BW-ONLY-SLOW-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm2, %zmm19
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm0
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm13, %zmm17, %zmm0
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm18, %zmm19
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm25, %zmm16, %zmm19
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm19 {%k1}
-; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm12[0],ymm6[0],ymm12[2],ymm6[2]
-; AVX512BW-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm19 {%k2} = zmm0[2,3,2,3],zmm5[2,3,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 64(%r8), %zmm0
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 64(%r9), %zmm12
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm6
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm12, %zmm14, %zmm6
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm5, %zmm15, %zmm6
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm6
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm12, %zmm10, %zmm6
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm5, %zmm1, %zmm6
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm12, %zmm6
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm22, %zmm6
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm5, %zmm3, %zmm6
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm12, %zmm4, %zmm0
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm2, %zmm5
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 128(%rdi), %zmm6
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 128(%rsi), %zmm19
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm0
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm19, %zmm17, %zmm0
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 128(%rdx), %zmm5
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 128(%rcx), %zmm28
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm12
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm28, %zmm16, %zmm12
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm16, %zmm17
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm12 {%k1}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 128(%rax), %zmm29
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 128(%r9), %ymm16
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %ymm16, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa 128(%r8), %ymm0
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm16[0],ymm0[2],ymm16[2]
-; AVX512BW-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm12 {%k2} = zmm0[2,3,2,3],zmm29[2,3,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 128(%r8), %zmm0
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 128(%r9), %zmm20
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm20, %zmm0, %zmm10
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm29, %zmm1, %zmm10
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm0, %zmm20, %zmm22
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm29, %zmm3, %zmm22
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm22, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm10 = [4,12,0,5,4,12,0,5]
+; AVX512BW-ONLY-SLOW-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3]
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm1
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm20, %zmm14, %zmm1
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm29, %zmm15, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm3, %zmm10, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [0,1,12,7,0,1,12,7]
+; AVX512BW-ONLY-SLOW-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm2, %zmm4, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm20 = [5,0,14,6,5,0,14,6]
+; AVX512BW-ONLY-SLOW-NEXT:    # zmm20 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm21
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm20, %zmm21
+; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [0,13,6,7,0,13,6,7]
+; AVX512BW-ONLY-SLOW-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm2, %zmm21
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm3, %zmm19, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm3 = [6,13,14,7,6,13,14,7]
+; AVX512BW-ONLY-SLOW-NEXT:    # zmm3 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm3, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm23, %zmm15, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm22, %zmm21
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm22, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm29, %zmm22, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
+; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm18[0],ymm13[0],ymm18[2],ymm13[2]
+; AVX512BW-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm1 {%k2} = zmm0[2,3,2,3],zmm5[2,3,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 64(%r8), %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 64(%r9), %zmm13
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm13, %zmm11, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm5, %zmm12, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm13, %zmm10, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm5, %zmm4, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm13, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm20, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm5, %zmm2, %zmm1
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm20, %zmm4, %zmm0
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm2, %zmm29
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm3 = [13,5,13,5,13,5,13,5]
-; AVX512BW-ONLY-SLOW-NEXT:    # zmm3 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm21, %zmm0
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm7, %zmm3, %zmm0
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm24 = [6,14,6,14,6,14,6,14]
-; AVX512BW-ONLY-SLOW-NEXT:    # zmm24 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm21, %zmm0
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm7, %zmm24, %zmm0
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm21, %zmm0
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm7, %zmm4, %zmm0
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [3,0,12,4,3,0,12,4]
-; AVX512BW-ONLY-SLOW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm21, %zmm0, %zmm7
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm13, %zmm19, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm3, %zmm5
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 128(%rdi), %zmm18
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 128(%rsi), %zmm22
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm18, %zmm1
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm25, %zmm3, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm22, %zmm15, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 128(%rdx), %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 128(%rcx), %zmm28
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm5
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm28, %zmm21, %zmm5
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm5 {%k1}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 128(%rax), %zmm30
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa 128(%r9), %ymm13
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu %ymm13, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa 128(%r8), %ymm1
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm1 = ymm1[0],ymm13[0],ymm1[2],ymm13[2]
+; AVX512BW-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm5 {%k2} = zmm1[2,3,2,3],zmm30[2,3,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 128(%r8), %zmm5
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 128(%r9), %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm1, %zmm5, %zmm10
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm30, %zmm4, %zmm10
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm4
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm5, %zmm1, %zmm20
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm30, %zmm2, %zmm20
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm20, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm4, %zmm11, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm30, %zmm12, %zmm1
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm18, %zmm1
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm25, %zmm24, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm4, %zmm19, %zmm5
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm5, %zmm3, %zmm30
+; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm3 = [13,5,13,5,13,5,13,5]
+; AVX512BW-ONLY-SLOW-NEXT:    # zmm3 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm8, %zmm3, %zmm1
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm18, %zmm1
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm25, %zmm4, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm12 = [6,14,6,14,6,14,6,14]
+; AVX512BW-ONLY-SLOW-NEXT:    # zmm12 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm8, %zmm12, %zmm1
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm18, %zmm0, %zmm25
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 192(%rdx), %zmm18
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 192(%rcx), %zmm1
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm18, %zmm2
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm24, %zmm2
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm26
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm28, %zmm3, %zmm26
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm28, %zmm5, %zmm24
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm30
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm28, %zmm4, %zmm30
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm5, %zmm0, %zmm28
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm21
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm18, %zmm0, %zmm21
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm1, %zmm18, %zmm17
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm18, %zmm0
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm4, %zmm18
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm3, %zmm0
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm8, %zmm19, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm19, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [3,0,12,4,3,0,12,4]
+; AVX512BW-ONLY-SLOW-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm7, %zmm2, %zmm8
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm31
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm4
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm29, %zmm3, %zmm4
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm4
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm29, %zmm12, %zmm4
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm4
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm29, %zmm19, %zmm4
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm6, %zmm2, %zmm29
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm4
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm28, %zmm3, %zmm4
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm4
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm28, %zmm12, %zmm4
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm4
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm28, %zmm19, %zmm4
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm2, %zmm28
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 192(%rdx), %zmm10
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 192(%rcx), %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm24
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm10, %zmm2, %zmm24
 ; AVX512BW-ONLY-SLOW-NEXT:    movb $48, %r10b
 ; AVX512BW-ONLY-SLOW-NEXT:    kmovd %r10d, %k3
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [1,0,10,2,1,0,10,2]
-; AVX512BW-ONLY-SLOW-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm27, %zmm15
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm8, %zmm4, %zmm15
-; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm7 {%k3} = zmm8[0],zmm27[0],zmm8[2],zmm27[2],zmm8[4],zmm27[4],zmm8[6],zmm27[6]
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm7
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm14
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm16
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm17
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm5 = [0,8,0,1,0,8,0,1]
-; AVX512BW-ONLY-SLOW-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm27, %zmm5, %zmm17
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm27, %zmm3, %zmm7
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [6,14,6,14]
-; AVX512BW-ONLY-SLOW-NEXT:    # ymm1 = mem[0,1,0,1]
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm27, %zmm1, %zmm14
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti128 {{.*#+}} ymm10 = [15,7,15,7]
-; AVX512BW-ONLY-SLOW-NEXT:    # ymm10 = mem[0,1,0,1]
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm27, %zmm10, %zmm16
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm13, %zmm27
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm11, %zmm4, %zmm27
-; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm25 {%k3} = zmm11[0],zmm13[0],zmm11[2],zmm13[2],zmm11[4],zmm13[4],zmm11[6],zmm13[6]
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm2
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm12
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm31
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm13, %zmm5, %zmm11
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm13, %zmm3, %zmm2
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm13, %zmm1, %zmm12
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm13, %zmm10, %zmm31
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 192(%rdi), %zmm22
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 192(%rsi), %zmm0
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm22, %zmm23
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm1, %zmm23
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm22, %zmm13
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm3, %zmm13
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm19, %zmm8
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm6, %zmm4, %zmm8
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm19, %zmm6, %zmm3
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm19, %zmm6, %zmm1
-; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm28 {%k3} = zmm6[0],zmm19[0],zmm6[2],zmm19[2],zmm6[4],zmm19[4],zmm6[6],zmm19[6]
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm9
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm19, %zmm5, %zmm6
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm19, %zmm10, %zmm9
-; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm21 {%k3} = zmm22[0],zmm0[0],zmm22[2],zmm0[2],zmm22[4],zmm0[4],zmm22[6],zmm0[6]
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm0, %zmm22, %zmm5
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm22, %zmm0, %zmm4
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm0, %zmm22, %zmm19
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm10, %zmm22
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa (%rdx), %xmm0
-; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512BW-ONLY-SLOW-NEXT:    movb $12, %sil
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 192(%rdi), %zmm7
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 192(%rsi), %zmm2
+; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm24 {%k3} = zmm7[0],zmm2[0],zmm7[2],zmm2[2],zmm7[4],zmm2[4],zmm7[6],zmm2[6]
+; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm11 = [1,0,10,2,1,0,10,2]
+; AVX512BW-ONLY-SLOW-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm16, %zmm26
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm17, %zmm11, %zmm26
+; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm13 = [0,8,0,1,0,8,0,1]
+; AVX512BW-ONLY-SLOW-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm17, %zmm27
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm16, %zmm13, %zmm27
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm23, %zmm4
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm23, %zmm21
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm14, %zmm11, %zmm21
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm23
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm4, %zmm13, %zmm23
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm8
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm22, %zmm19
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm18, %zmm11, %zmm19
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm18, %zmm20
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm22, %zmm13, %zmm20
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm2, %zmm7, %zmm13
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm7, %zmm2, %zmm11
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm25
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm5
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm6
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm2, %zmm15, %zmm7
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm2, %zmm12, %zmm25
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm2, %zmm3, %zmm5
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm2, %zmm1, %zmm6
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm16, %zmm2
+; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm31 {%k3} = zmm17[0],zmm16[0],zmm17[2],zmm16[2],zmm17[4],zmm16[4],zmm17[6],zmm16[6]
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm31, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm17, %zmm4
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm17, %zmm16
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm17, %zmm15
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm2, %zmm3, %zmm4
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm2, %zmm12, %zmm16
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm2, %zmm1, %zmm15
+; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm29 {%k3} = zmm14[0],zmm8[0],zmm14[2],zmm8[2],zmm14[4],zmm8[4],zmm14[6],zmm8[6]
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm2
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm17
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm8, %zmm3, %zmm2
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm8, %zmm12, %zmm17
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm8, %zmm1, %zmm14
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm10, %zmm8
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm3, %zmm8
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm22, %zmm18, %zmm3
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm10, %zmm9
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm12, %zmm9
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm22, %zmm18, %zmm12
+; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm28 {%k3} = zmm18[0],zmm22[0],zmm18[2],zmm22[2],zmm18[4],zmm22[4],zmm18[6],zmm22[6]
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm22, %zmm1, %zmm18
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm10, %zmm22
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm1, %zmm22
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm1, %zmm10
+; AVX512BW-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm31 = zmm25[0,1,2,3],zmm9[4,5,6,7]
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 192(%r8), %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm9 = <0,11,u,u,4,5,6,7>
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm0, %zmm24, %zmm9
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm5 {%k1}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm25 = <12,u,u,3,4,5,6,13>
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm0, %zmm5, %zmm25
+; AVX512BW-ONLY-SLOW-NEXT:    movb $24, %sil
 ; AVX512BW-ONLY-SLOW-NEXT:    kmovd %esi, %k3
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm17 {%k3}
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [0,8,0,8,0,8,0,8]
-; AVX512BW-ONLY-SLOW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm10 # 64-byte Folded Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti32x4 $3, (%rax), %zmm10, %zmm10
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm22, %zmm6 {%k3}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm5 = <u,1,2,3,4,15,u,u>
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm0, %zmm6, %zmm5
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa (%rdx), %xmm6
+; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm6 = xmm6[0],mem[0]
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
+; AVX512BW-ONLY-SLOW-NEXT:    movb $12, %sil
+; AVX512BW-ONLY-SLOW-NEXT:    kmovd %esi, %k4
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm6, %zmm0, %zmm27 {%k4}
+; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm6 = [0,8,0,8,0,8,0,8]
+; AVX512BW-ONLY-SLOW-NEXT:    # zmm6 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm22 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm6, %zmm22 # 64-byte Folded Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa 64(%rdx), %xmm8
+; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm8 = xmm8[0],mem[0]
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm8, %ymm0, %ymm8
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm8, %zmm0, %zmm23 {%k4}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa 128(%rdx), %xmm8
+; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm8 = xmm8[0],mem[0]
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm8, %ymm0, %ymm8
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm8, %zmm0, %zmm20 {%k4}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa 192(%rdx), %xmm8
+; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm8 = xmm8[0],mem[0]
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm8, %ymm0, %ymm8
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm8, %zmm0, %zmm13 {%k4}
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti32x4 $3, (%rax), %zmm22, %zmm8
 ; AVX512BW-ONLY-SLOW-NEXT:    movb $112, %sil
 ; AVX512BW-ONLY-SLOW-NEXT:    kmovd %esi, %k4
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm10, %zmm17 {%k4}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa 64(%rdx), %xmm10
-; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm10 = xmm10[0],mem[0]
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm10, %zmm0, %zmm11 {%k3}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm10 # 64-byte Folded Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti32x4 $3, 64(%rax), %zmm10, %zmm10
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm10, %zmm11 {%k4}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa 128(%rdx), %xmm10
-; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm10 = xmm10[0],mem[0]
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm10, %zmm0, %zmm6 {%k3}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm20, %zmm0, %zmm10
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti32x4 $3, 128(%rax), %zmm10, %zmm10
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm10, %zmm6 {%k4}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa 192(%rdx), %xmm10
-; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm10 = xmm10[0],mem[0]
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm10, %ymm0, %ymm10
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm10, %zmm0, %zmm5 {%k3}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 192(%r8), %zmm19
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 192(%r9), %zmm20
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm20, %zmm19, %zmm0
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti32x4 $3, 192(%rax), %zmm0, %zmm0
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm5 {%k4}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm7 {%k1}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm27 {%k4}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm6, %zmm1 # 64-byte Folded Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti32x4 $3, 64(%rax), %zmm1, %zmm8
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm23 {%k4}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm6, %zmm1 # 64-byte Folded Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti32x4 $3, 128(%rax), %zmm1, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm20 {%k4}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 192(%r9), %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm1, %zmm0, %zmm6
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti32x4 $3, 192(%rax), %zmm6, %zmm6
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm13 {%k4}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm4 {%k1}
 ; AVX512BW-ONLY-SLOW-NEXT:    movb $120, %sil
-; AVX512BW-ONLY-SLOW-NEXT:    kmovd %esi, %k3
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm0 {%k3}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm14, %zmm7 # 64-byte Folded Reload
-; AVX512BW-ONLY-SLOW-NEXT:    # zmm7 = zmm14[0,1,2,3],mem[4,5,6,7]
+; AVX512BW-ONLY-SLOW-NEXT:    kmovd %esi, %k4
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm24 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm24 {%k4}
+; AVX512BW-ONLY-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm16, %zmm22 # 64-byte Folded Reload
+; AVX512BW-ONLY-SLOW-NEXT:    # zmm22 = zmm16[0,1,2,3],mem[4,5,6,7]
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm2 {%k1}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm16 {%k4}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm4 {%k4}
 ; AVX512BW-ONLY-SLOW-NEXT:    movb $-61, %sil
 ; AVX512BW-ONLY-SLOW-NEXT:    kmovd %esi, %k4
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm7 {%k4}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k1}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm0 {%k3}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm26, %zmm3 {%k1}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm26 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm26 {%k3}
-; AVX512BW-ONLY-SLOW-NEXT:    movb $24, %sil
-; AVX512BW-ONLY-SLOW-NEXT:    kmovd %esi, %k3
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm16 {%k3}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm22 {%k4}
+; AVX512BW-ONLY-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm17, %zmm2 # 64-byte Folded Reload
+; AVX512BW-ONLY-SLOW-NEXT:    # zmm2 = zmm17[0,1,2,3],mem[4,5,6,7]
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm2 {%k4}
 ; AVX512BW-ONLY-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm12, %zmm3 # 64-byte Folded Reload
 ; AVX512BW-ONLY-SLOW-NEXT:    # zmm3 = zmm12[0,1,2,3],mem[4,5,6,7]
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm3 {%k4}
-; AVX512BW-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm1[0,1,2,3],zmm24[4,5,6,7]
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k4}
-; AVX512BW-ONLY-SLOW-NEXT:    movb $-31, %sil
-; AVX512BW-ONLY-SLOW-NEXT:    kmovd %esi, %k4
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm16 {%k4}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm31 {%k3}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm31 {%k4}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm30, %zmm9 {%k3}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm29, %zmm9 {%k4}
-; AVX512BW-ONLY-SLOW-NEXT:    vpbroadcastq 8(%rcx), %ymm0
-; AVX512BW-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm3 {%k4}
+; AVX512BW-ONLY-SLOW-NEXT:    vpbroadcastq 8(%rcx), %ymm6
+; AVX512BW-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm6 = mem[0,1,2,3],ymm6[4,5,6,7]
 ; AVX512BW-ONLY-SLOW-NEXT:    movb $6, %sil
 ; AVX512BW-ONLY-SLOW-NEXT:    kmovd %esi, %k4
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm15 {%k4}
-; AVX512BW-ONLY-SLOW-NEXT:    vpbroadcastq 72(%rcx), %ymm0
-; AVX512BW-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm27 {%k4}
-; AVX512BW-ONLY-SLOW-NEXT:    vpbroadcastq 136(%rcx), %ymm0
-; AVX512BW-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm8 {%k4}
-; AVX512BW-ONLY-SLOW-NEXT:    vpbroadcastq 200(%rcx), %ymm0
-; AVX512BW-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm4 {%k4}
-; AVX512BW-ONLY-SLOW-NEXT:    movb $56, %cl
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm6, %zmm0, %zmm26 {%k4}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm15 {%k3}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm14 {%k3}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm18 {%k3}
+; AVX512BW-ONLY-SLOW-NEXT:    movb $-31, %sil
+; AVX512BW-ONLY-SLOW-NEXT:    kmovd %esi, %k3
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm15 {%k3}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm14 {%k3}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm30, %zmm18 {%k3}
+; AVX512BW-ONLY-SLOW-NEXT:    movb $56, %sil
+; AVX512BW-ONLY-SLOW-NEXT:    kmovd %esi, %k3
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm26 {%k3}
+; AVX512BW-ONLY-SLOW-NEXT:    vpbroadcastq 72(%rcx), %ymm6
+; AVX512BW-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm6 = mem[0,1,2,3],ymm6[4,5,6,7]
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm6, %zmm0, %zmm21 {%k4}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm21 {%k3}
+; AVX512BW-ONLY-SLOW-NEXT:    vpbroadcastq 136(%rcx), %ymm6
+; AVX512BW-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm6 = mem[0,1,2,3],ymm6[4,5,6,7]
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm6, %zmm0, %zmm19 {%k4}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm19 {%k3}
+; AVX512BW-ONLY-SLOW-NEXT:    vpbroadcastq 200(%rcx), %ymm6
+; AVX512BW-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm6 = mem[0,1,2,3],ymm6[4,5,6,7]
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm6, %zmm0, %zmm11 {%k4}
+; AVX512BW-ONLY-SLOW-NEXT:    movb $64, %cl
 ; AVX512BW-ONLY-SLOW-NEXT:    kmovd %ecx, %k4
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm15 {%k4}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm27 {%k4}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm8 {%k4}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm20, %zmm19, %zmm2
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 192(%rax), %zmm0
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm10, %zmm2
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm4 {%k4}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm13 {%k1}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm14 {%k1}
-; AVX512BW-ONLY-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm23, %zmm10 # 64-byte Folded Reload
-; AVX512BW-ONLY-SLOW-NEXT:    # zmm10 = zmm23[0,1,2,3],mem[4,5,6,7]
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm12 = <0,11,u,u,4,5,6,7>
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm19, %zmm21, %zmm12
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm2 = <0,1,11,u,4,5,6,7>
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm20, %zmm12, %zmm2
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa 192(%r8), %ymm12
-; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm12 = ymm12[0],mem[0],ymm12[2],mem[2]
-; AVX512BW-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm14 {%k2} = zmm12[2,3,2,3],zmm0[2,3,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm21
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm18, %zmm22 {%k3}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vpunpckhqdq (%rsp), %ymm12, %ymm12 # 32-byte Folded Reload
-; AVX512BW-ONLY-SLOW-NEXT:    # ymm12 = ymm12[1],mem[1],ymm12[3],mem[3]
-; AVX512BW-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm12 = ymm12[0,2,3,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm12 = ymm12[0,1,2,3,4,5],mem[6,7]
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm31 {%k4}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm6, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 192(%rax), %zmm6
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm6, %zmm8, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm11 {%k3}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm0 = <0,1,11,u,4,5,6,7>
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm1, %zmm9, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm10 {%k1}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa 192(%r8), %ymm7
+; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm7 = ymm7[0],mem[0],ymm7[2],mem[2]
+; AVX512BW-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm10 {%k2} = zmm7[2,3,2,3],zmm6[2,3,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vpunpckhqdq (%rsp), %ymm7, %ymm7 # 32-byte Folded Reload
+; AVX512BW-ONLY-SLOW-NEXT:    # ymm7 = ymm7[1],mem[1],ymm7[3],mem[3]
+; AVX512BW-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[0,2,3,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm7 = ymm7[0,1,2,3,4,5],mem[6,7]
 ; AVX512BW-ONLY-SLOW-NEXT:    movb $14, %cl
 ; AVX512BW-ONLY-SLOW-NEXT:    kmovd %ecx, %k1
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm12, %zmm0, %zmm18 {%k1}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm12, %ymm12 # 32-byte Folded Reload
-; AVX512BW-ONLY-SLOW-NEXT:    # ymm12 = ymm12[1],mem[1],ymm12[3],mem[3]
-; AVX512BW-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm12 = ymm12[0,2,3,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm12 = ymm12[0,1,2,3,4,5],mem[6,7]
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm12, %zmm0, %zmm25 {%k1}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm12, %ymm12 # 32-byte Folded Reload
-; AVX512BW-ONLY-SLOW-NEXT:    # ymm12 = ymm12[1],mem[1],ymm12[3],mem[3]
-; AVX512BW-ONLY-SLOW-NEXT:    movb $64, %cl
-; AVX512BW-ONLY-SLOW-NEXT:    kmovd %ecx, %k2
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm19, %zmm10 {%k2}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm7, %zmm0, %zmm12 {%k1}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm7 # 32-byte Folded Reload
+; AVX512BW-ONLY-SLOW-NEXT:    # ymm7 = ymm7[1],mem[1],ymm7[3],mem[3]
+; AVX512BW-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[0,2,3,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm7 = ymm7[0,1,2,3,4,5],mem[6,7]
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm7, %zmm0, %zmm29 {%k1}
 ; AVX512BW-ONLY-SLOW-NEXT:    movb $8, %cl
 ; AVX512BW-ONLY-SLOW-NEXT:    kmovd %ecx, %k2
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k2}
-; AVX512BW-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm12 = ymm12[0,2,3,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm12 = ymm12[0,1,2,3,4,5],mem[6,7]
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm12, %zmm0, %zmm28 {%k1}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm12 = <12,u,u,3,4,5,6,13>
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm19, %zmm13, %zmm12
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm13 = <u,1,2,3,4,15,u,u>
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm19, %zmm22, %zmm13
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm14 = <0,12,u,3,4,5,6,7>
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm20, %zmm12, %zmm14
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm12 = <13,u,2,3,4,5,6,14>
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm20, %zmm10, %zmm12
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm10 = <u,1,2,3,4,5,15,u>
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm20, %zmm13, %zmm10
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm13 = [0,1,12,3,4,5,6,7]
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm0, %zmm14, %zmm13
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm14 = [0,13,2,3,4,5,6,7]
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm0, %zmm12, %zmm14
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm12 = [14,1,2,3,4,5,6,15]
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm0, %zmm10, %zmm12
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm0 {%k2}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm7 # 32-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm7, %ymm7 # 32-byte Folded Reload
+; AVX512BW-ONLY-SLOW-NEXT:    # ymm7 = ymm7[1],mem[1],ymm7[3],mem[3]
+; AVX512BW-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[0,2,3,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm7 = ymm7[0,1,2,3,4,5],mem[6,7]
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm7, %zmm0, %zmm28 {%k1}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm7 = <0,12,u,3,4,5,6,7>
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm1, %zmm25, %zmm7
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm8 = <13,u,2,3,4,5,6,14>
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm1, %zmm31, %zmm8
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm9 = <u,1,2,3,4,5,15,u>
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm1, %zmm5, %zmm9
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,1,12,3,4,5,6,7]
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm6, %zmm7, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [0,13,2,3,4,5,6,7]
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm6, %zmm8, %zmm5
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm7 = [14,1,2,3,4,5,6,15]
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm6, %zmm9, %zmm7
 ; AVX512BW-ONLY-SLOW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm21, 1472(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm4, 1408(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm9, 1280(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, 1216(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm26, 1152(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovaps %zmm0, 1024(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm8, 960(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm31, 832(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, 768(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovaps %zmm0, 704(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovaps %zmm0, 576(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm27, 512(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm16, 384(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm7, 320(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovaps %zmm0, 256(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovaps %zmm0, 128(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm15, 64(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm5, 1344(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm10, 1472(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm11, 1408(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm18, 1280(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, 1216(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm4, 1152(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vmovaps %zmm3, 1024(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm19, 960(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm14, 832(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, 768(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm16, 704(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vmovaps %zmm2, 576(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm21, 512(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm15, 384(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm22, 320(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm24, 256(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vmovaps %zmm2, 128(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm26, 64(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm13, 1344(%rax)
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm28, 1088(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, 896(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm25, 640(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm11, 448(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm18, 192(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm17, (%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm12, 1728(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm14, 1664(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm13, 1600(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, 1536(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    addq $2120, %rsp # imm = 0x848
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm20, 896(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm29, 640(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm23, 448(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm12, 192(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm27, (%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm7, 1728(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm5, 1664(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, 1600(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, 1536(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    addq $2184, %rsp # imm = 0x888
 ; AVX512BW-ONLY-SLOW-NEXT:    vzeroupper
 ; AVX512BW-ONLY-SLOW-NEXT:    retq
 ;
 ; AVX512BW-ONLY-FAST-LABEL: store_i64_stride7_vf32:
 ; AVX512BW-ONLY-FAST:       # %bb.0:
-; AVX512BW-ONLY-FAST-NEXT:    subq $2024, %rsp # imm = 0x7E8
+; AVX512BW-ONLY-FAST-NEXT:    subq $2152, %rsp # imm = 0x868
 ; AVX512BW-ONLY-FAST-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 (%rdi), %zmm7
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 64(%rdi), %zmm30
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 (%rsi), %zmm17
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 64(%rsi), %zmm15
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 64(%rdx), %zmm13
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 (%rdx), %zmm8
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 (%rcx), %zmm27
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 64(%rcx), %zmm20
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 (%rdi), %zmm16
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 64(%rdi), %zmm17
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 (%rsi), %zmm29
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 64(%rsi), %zmm28
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 64(%rdx), %zmm19
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 (%rdx), %zmm23
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 (%rcx), %zmm22
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 64(%rcx), %zmm30
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 (%r8), %zmm0
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 (%r9), %zmm9
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 (%rax), %zmm4
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 (%r9), %zmm7
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 (%rax), %zmm3
 ; AVX512BW-ONLY-FAST-NEXT:    movb $96, %r10b
 ; AVX512BW-ONLY-FAST-NEXT:    kmovd %r10d, %k1
+; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm21 = [15,7,15,7,15,7,15,7]
+; AVX512BW-ONLY-FAST-NEXT:    # zmm21 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm2 = [9,1,9,1,9,1,9,1]
 ; AVX512BW-ONLY-FAST-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm3
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm9, %zmm2, %zmm3
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm10
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm4
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm7, %zmm2, %zmm4
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm8
 ; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [4,9,0,3,4,9,0,3]
 ; AVX512BW-ONLY-FAST-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm4, %zmm2, %zmm3
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm11
-; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm2 = [11,3,11,3,11,3,11,3]
-; AVX512BW-ONLY-FAST-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm7, %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm17, %zmm2, %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm14
-; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm3 = [2,10,0,3,2,10,0,3]
-; AVX512BW-ONLY-FAST-NEXT:    # zmm3 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm8, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm27, %zmm3, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm16
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm3, %zmm2, %zmm4
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm9
+; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm25 = [11,3,11,3,11,3,11,3]
+; AVX512BW-ONLY-FAST-NEXT:    # zmm25 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm16, %zmm1
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm29, %zmm25, %zmm1
+; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [2,10,0,3,2,10,0,3]
+; AVX512BW-ONLY-FAST-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm23, %zmm2
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm22, %zmm4, %zmm2
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm4, %zmm15
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm5
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqa (%r9), %ymm1
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqa 64(%r9), %ymm6
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa 128(%r9), %ymm2
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 (%r8), %ymm19
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 64(%r8), %ymm23
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 128(%r8), %ymm22
-; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm3 = ymm19[0],ymm1[0],ymm19[2],ymm1[2]
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa 128(%r9), %ymm4
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa (%r8), %ymm10
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa 64(%r8), %ymm12
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa 128(%r8), %ymm11
+; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm2 = ymm10[0],ymm1[0],ymm10[2],ymm1[2]
 ; AVX512BW-ONLY-FAST-NEXT:    movb $28, %r10b
 ; AVX512BW-ONLY-FAST-NEXT:    kmovd %r10d, %k2
-; AVX512BW-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm5 {%k2} = zmm3[2,3,2,3],zmm4[2,3,2,3]
+; AVX512BW-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm5 {%k2} = zmm2[2,3,2,3],zmm3[2,3,2,3]
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm4, %zmm21
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa {{.*#+}} ymm3 = [1,3,7,7]
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %ymm1, %ymm3, %ymm19
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %ymm19, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [4,12,0,5,4,12,0,5]
-; AVX512BW-ONLY-FAST-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm9, %zmm4, %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm4, %zmm19
-; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm5 = [0,1,12,7,0,1,12,7]
-; AVX512BW-ONLY-FAST-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm21, %zmm5, %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm25 = [5,0,14,6,5,0,14,6]
-; AVX512BW-ONLY-FAST-NEXT:    # zmm25 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm9, %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm25, %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [0,13,6,7,0,13,6,7]
-; AVX512BW-ONLY-FAST-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm21, %zmm4, %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm28 = [15,7,15,7,15,7,15,7]
-; AVX512BW-ONLY-FAST-NEXT:    # zmm28 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm9, %zmm28, %zmm0
-; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm1 = [6,13,14,7,6,13,14,7]
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa {{.*#+}} ymm5 = [1,3,7,7]
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %ymm1, %ymm5, %ymm10
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu %ymm10, (%rsp) # 32-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm1 = [4,12,0,5,4,12,0,5]
 ; AVX512BW-ONLY-FAST-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm21
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm23[0],ymm6[0],ymm23[2],ymm6[2]
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %ymm6, %ymm3, %ymm23
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %ymm23, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm6 = ymm22[0],ymm2[0],ymm22[2],ymm2[2]
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %ymm2, %ymm3, %ymm22
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %ymm22, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm30, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm15, %zmm14, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm13, %zmm3
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm20, %zmm16, %zmm3
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 64(%rax), %zmm18
-; AVX512BW-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm3 {%k2} = zmm0[2,3,2,3],zmm18[2,3,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 64(%r8), %zmm0
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 64(%r9), %zmm22
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm22, %zmm10, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm11, %zmm9
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm18, %zmm11, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm22, %zmm19, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm18, %zmm5, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm22, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm25, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm18, %zmm4, %zmm2
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm7, %zmm1, %zmm2
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, %zmm10
+; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm1 = [0,1,12,7,0,1,12,7]
+; AVX512BW-ONLY-FAST-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm3, %zmm1, %zmm2
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm22, %zmm28, %zmm0
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm18
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 128(%rdi), %zmm26
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 128(%rsi), %zmm3
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm26, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm3, %zmm14, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 128(%rdx), %zmm24
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 128(%rcx), %zmm0
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm24, %zmm14
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm16, %zmm14
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm16, %zmm23
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm14 {%k1}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 128(%rax), %zmm11
-; AVX512BW-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm14 {%k2} = zmm6[2,3,2,3],zmm11[2,3,2,3]
+; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm13 = [5,0,14,6,5,0,14,6]
+; AVX512BW-ONLY-FAST-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm7, %zmm14
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm7, %zmm18
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm13, %zmm14
+; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [0,13,6,7,0,13,6,7]
+; AVX512BW-ONLY-FAST-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm7
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm3, %zmm2, %zmm14
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 128(%r8), %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 128(%r9), %zmm14
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm14, %zmm2, %zmm19
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm11, %zmm5, %zmm19
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm2, %zmm14, %zmm25
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm11, %zmm4, %zmm25
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm25, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm4
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm14, %zmm10, %zmm4
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm18, %zmm21, %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm3 = [6,13,14,7,6,13,14,7]
+; AVX512BW-ONLY-FAST-NEXT:    # zmm3 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm3, %zmm7
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm12[0],ymm6[0],ymm12[2],ymm6[2]
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %ymm6, %ymm5, %ymm12
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm7 = ymm11[0],ymm4[0],ymm11[2],ymm4[2]
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %ymm4, %ymm5, %ymm11
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm17, %zmm4
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm28, %zmm25, %zmm4
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm19, %zmm6
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm15, %zmm5
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm30, %zmm15, %zmm6
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm4, %zmm6 {%k1}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 64(%rax), %zmm11
+; AVX512BW-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm6 {%k2} = zmm0[2,3,2,3],zmm11[2,3,2,3]
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 64(%r8), %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 64(%r9), %zmm6
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm4
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm6, %zmm8, %zmm4
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm11, %zmm9, %zmm4
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm14, %zmm28, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm2, %zmm1, %zmm11
-; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm5 = [3,0,12,4,3,0,12,4]
-; AVX512BW-ONLY-FAST-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm27, %zmm29
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm8, %zmm5, %zmm29
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm4
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm6, %zmm10, %zmm4
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm11, %zmm1, %zmm4
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm6, %zmm4
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm13, %zmm4
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm11, %zmm2, %zmm4
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm6, %zmm21, %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm3, %zmm11
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 128(%rdi), %zmm27
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 128(%rsi), %zmm18
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm27, %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm18, %zmm25, %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 128(%rdx), %zmm31
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 128(%rcx), %zmm15
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm31, %zmm4
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm15, %zmm5, %zmm4
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm4 {%k1}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 128(%rax), %zmm5
+; AVX512BW-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm4 {%k2} = zmm7[2,3,2,3],zmm5[2,3,2,3]
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 128(%r8), %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 128(%r9), %zmm4
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm4, %zmm0, %zmm10
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm5, %zmm1, %zmm10
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm0, %zmm4, %zmm13
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm5, %zmm2, %zmm13
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm1
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm4, %zmm8, %zmm1
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm5, %zmm9, %zmm1
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm4, %zmm21, %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm3, %zmm5
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 192(%rdx), %zmm7
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 192(%rcx), %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm13 = [3,0,12,4,3,0,12,4]
+; AVX512BW-ONLY-FAST-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm24
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm7, %zmm13, %zmm24
+; AVX512BW-ONLY-FAST-NEXT:    movb $48, %r10b
+; AVX512BW-ONLY-FAST-NEXT:    kmovd %r10d, %k3
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 192(%rdi), %zmm12
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 192(%rsi), %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm24 {%k3} = zmm12[0],zmm3[0],zmm12[2],zmm3[2],zmm12[4],zmm3[4],zmm12[6],zmm3[6]
+; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm14 = [0,8,0,1,0,8,0,1]
+; AVX512BW-ONLY-FAST-NEXT:    # zmm14 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm16, %zmm1
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm29, %zmm14, %zmm1
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm17, %zmm1
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm28, %zmm14, %zmm1
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm27, %zmm1
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm18, %zmm14, %zmm1
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm3, %zmm12, %zmm14
+; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm11 = [1,0,10,2,1,0,10,2]
+; AVX512BW-ONLY-FAST-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm29, %zmm1
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm16, %zmm11, %zmm1
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm28, %zmm26
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm17, %zmm11, %zmm26
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm18, %zmm20
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm27, %zmm11, %zmm20
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm12, %zmm3, %zmm11
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm12, %zmm8
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm12, %zmm10
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm12, %zmm9
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm3, %zmm25, %zmm12
+; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm4 = [6,14,6,14,6,14,6,14]
+; AVX512BW-ONLY-FAST-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm3, %zmm4, %zmm8
 ; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm1 = [13,5,13,5,13,5,13,5]
 ; AVX512BW-ONLY-FAST-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm8, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm27, %zmm1, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm10 = [6,14,6,14,6,14,6,14]
-; AVX512BW-ONLY-FAST-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm8, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm27, %zmm10, %zmm2
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm3, %zmm1, %zmm10
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm3, %zmm21, %zmm9
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm22, %zmm25
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm23, %zmm2
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm23, %zmm13, %zmm25
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm22, %zmm1, %zmm23
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm22, %zmm4, %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm22, %zmm21, %zmm2
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm27, %zmm28, %zmm8
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    movb $48, %r10b
-; AVX512BW-ONLY-FAST-NEXT:    kmovd %r10d, %k3
-; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm31 = [0,8,0,1,0,8,0,1]
-; AVX512BW-ONLY-FAST-NEXT:    # zmm31 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm7, %zmm25
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm17, %zmm31, %zmm25
-; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [1,0,10,2,1,0,10,2]
-; AVX512BW-ONLY-FAST-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm17, %zmm21
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm7, %zmm4, %zmm21
-; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm29 {%k3} = zmm7[0],zmm17[0],zmm7[2],zmm17[2],zmm7[4],zmm17[4],zmm7[6],zmm17[6]
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm7, %zmm9
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm17, %zmm1, %zmm9
-; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti128 {{.*#+}} ymm8 = [6,14,6,14]
-; AVX512BW-ONLY-FAST-NEXT:    # ymm8 = mem[0,1,0,1]
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm7, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm17, %zmm8, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [15,7,15,7]
-; AVX512BW-ONLY-FAST-NEXT:    # ymm2 = mem[0,1,0,1]
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm17, %zmm2, %zmm7
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm6
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm20, %zmm17
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm13, %zmm5, %zmm17
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm13, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm20, %zmm1, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm13, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm20, %zmm10, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm20, %zmm28, %zmm13
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm25 {%k3} = zmm16[0],zmm29[0],zmm16[2],zmm29[2],zmm16[4],zmm29[4],zmm16[6],zmm29[6]
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm16, %zmm6
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm29, %zmm1, %zmm6
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm16, %zmm22
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm29, %zmm4, %zmm22
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm29, %zmm21, %zmm16
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm30, %zmm16
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm15, %zmm31, %zmm16
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm15, %zmm20
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm30, %zmm4, %zmm20
-; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm17 {%k3} = zmm30[0],zmm15[0],zmm30[2],zmm15[2],zmm30[4],zmm15[4],zmm30[6],zmm15[6]
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm30, %zmm7
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm15, %zmm1, %zmm7
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm30, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm15, %zmm8, %zmm2
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm19, %zmm2
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm19, %zmm13, %zmm16
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm30, %zmm1, %zmm19
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm29
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm30, %zmm4, %zmm29
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm30, %zmm21, %zmm2
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm15, %zmm6, %zmm30
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm6, %zmm13
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 192(%rdx), %zmm15
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 192(%rcx), %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm15, %zmm6
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm2, %zmm10, %zmm6
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm12
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm15, %zmm5, %zmm12
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm15, %zmm6
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm2, %zmm1, %zmm6
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm2, %zmm15, %zmm23
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm23, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm2, %zmm28, %zmm15
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm24, %zmm0, %zmm5
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm0, %zmm24, %zmm10
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm24, %zmm27
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm28, %zmm24
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm27
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 192(%rdi), %zmm19
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 192(%rsi), %zmm0
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm19, %zmm23
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm8, %zmm23
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm19, %zmm6
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm6
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm26, %zmm28
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm3, %zmm31, %zmm28
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm26, %zmm4, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm5 {%k3} = zmm26[0],zmm3[0],zmm26[2],zmm3[2],zmm26[4],zmm3[4],zmm26[6],zmm3[6]
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm3, %zmm26, %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm3, %zmm26, %zmm8
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm3, %zmm13, %zmm26
-; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm12 {%k3} = zmm19[0],zmm0[0],zmm19[2],zmm0[2],zmm19[4],zmm0[4],zmm19[6],zmm0[6]
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm0, %zmm19, %zmm31
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm19, %zmm0, %zmm4
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm0, %zmm19, %zmm3
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm13, %zmm19
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm17, %zmm30
+; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm16 {%k3} = zmm17[0],zmm28[0],zmm17[2],zmm28[2],zmm17[4],zmm28[4],zmm17[6],zmm28[6]
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm17, %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm28, %zmm1, %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm28, %zmm4, %zmm17
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm28, %zmm21, %zmm30
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm31, %zmm2
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm31, %zmm15, %zmm13
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm15, %zmm1, %zmm31
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm28
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm15, %zmm4, %zmm28
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm15, %zmm21, %zmm2
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm7, %zmm15
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm4, %zmm15
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm7, %zmm5
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm5
+; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm13 {%k3} = zmm27[0],zmm18[0],zmm27[2],zmm18[2],zmm27[4],zmm18[4],zmm27[6],zmm18[6]
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm18, %zmm27, %zmm1
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm18, %zmm27, %zmm4
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm18, %zmm21, %zmm27
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm7, %zmm18
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm21, %zmm18
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm2, %zmm7
+; AVX512BW-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm8[0,1,2,3],zmm15[4,5,6,7]
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 192(%r8), %zmm2
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm0 = <0,11,u,u,4,5,6,7>
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm2, %zmm24, %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 192(%r9), %zmm8
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm15 = <0,1,11,u,4,5,6,7>
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm8, %zmm0, %zmm15
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm5, %zmm10 {%k1}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm0 = <12,u,u,3,4,5,6,13>
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm2, %zmm10, %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    movb $24, %sil
+; AVX512BW-ONLY-FAST-NEXT:    kmovd %esi, %k3
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm18, %zmm9 {%k3}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm24 = <u,1,2,3,4,15,u,u>
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm2, %zmm9, %zmm24
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqa (%rdx), %xmm0
 ; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
 ; AVX512BW-ONLY-FAST-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
 ; AVX512BW-ONLY-FAST-NEXT:    movb $12, %sil
-; AVX512BW-ONLY-FAST-NEXT:    kmovd %esi, %k3
-; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm25 {%k3}
+; AVX512BW-ONLY-FAST-NEXT:    kmovd %esi, %k4
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm21 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm21 {%k4}
 ; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [0,8,0,8,0,8,0,8]
 ; AVX512BW-ONLY-FAST-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm3 # 64-byte Folded Reload
-; AVX512BW-ONLY-FAST-NEXT:    vinserti32x4 $3, (%rax), %zmm3, %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm9 # 64-byte Folded Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa 64(%rdx), %xmm5
+; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm5 = xmm5[0],mem[0]
+; AVX512BW-ONLY-FAST-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm5, %zmm0, %zmm10 {%k4}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa 128(%rdx), %xmm5
+; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm5 = xmm5[0],mem[0]
+; AVX512BW-ONLY-FAST-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm5, %zmm0, %zmm18 {%k4}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa 192(%rdx), %xmm5
+; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm5 = xmm5[0],mem[0]
+; AVX512BW-ONLY-FAST-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
+; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm5, %zmm0, %zmm14 {%k4}
+; AVX512BW-ONLY-FAST-NEXT:    vinserti32x4 $3, (%rax), %zmm9, %zmm5
 ; AVX512BW-ONLY-FAST-NEXT:    movb $112, %sil
 ; AVX512BW-ONLY-FAST-NEXT:    kmovd %esi, %k4
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm25 {%k4}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa 64(%rdx), %xmm3
-; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],mem[0]
-; AVX512BW-ONLY-FAST-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm16 {%k3}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm22, %zmm0, %zmm3
-; AVX512BW-ONLY-FAST-NEXT:    vinserti32x4 $3, 64(%rax), %zmm3, %zmm3
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm16 {%k4}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa 128(%rdx), %xmm3
-; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],mem[0]
-; AVX512BW-ONLY-FAST-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm28 {%k3}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm14, %zmm0, %zmm3
-; AVX512BW-ONLY-FAST-NEXT:    vinserti32x4 $3, 128(%rax), %zmm3, %zmm3
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm28 {%k4}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa 192(%rdx), %xmm3
-; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],mem[0]
-; AVX512BW-ONLY-FAST-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm31 {%k3}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 192(%r8), %zmm22
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 192(%r9), %zmm14
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm14, %zmm22, %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm5, %zmm21 {%k4}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm5 # 64-byte Folded Reload
+; AVX512BW-ONLY-FAST-NEXT:    vinserti32x4 $3, 64(%rax), %zmm5, %zmm5
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm5, %zmm10 {%k4}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm5 # 64-byte Folded Reload
+; AVX512BW-ONLY-FAST-NEXT:    vinserti32x4 $3, 128(%rax), %zmm5, %zmm5
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm5, %zmm18 {%k4}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm18, %zmm21
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm8, %zmm2, %zmm0
 ; AVX512BW-ONLY-FAST-NEXT:    vinserti32x4 $3, 192(%rax), %zmm0, %zmm0
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm31 {%k4}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm14 {%k4}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu (%rsp), %ymm0 # 32-byte Reload
 ; AVX512BW-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],mem[6,7]
 ; AVX512BW-ONLY-FAST-NEXT:    movb $14, %sil
-; AVX512BW-ONLY-FAST-NEXT:    kmovd %esi, %k3
-; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm29 {%k3}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm9 {%k1}
-; AVX512BW-ONLY-FAST-NEXT:    movb $120, %sil
 ; AVX512BW-ONLY-FAST-NEXT:    kmovd %esi, %k4
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm9, %zmm0 {%k4}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm13 # 64-byte Folded Reload
-; AVX512BW-ONLY-FAST-NEXT:    # zmm13 = zmm0[0,1,2,3],mem[4,5,6,7]
+; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm25 {%k4}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm23, %zmm6 {%k1}
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX512BW-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],mem[6,7]
-; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm17 {%k3}
+; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm16 {%k4}
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
 ; AVX512BW-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],mem[6,7]
-; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm5 {%k3}
-; AVX512BW-ONLY-FAST-NEXT:    movb $-61, %sil
-; AVX512BW-ONLY-FAST-NEXT:    kmovd %esi, %k5
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm13 {%k5}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm7 {%k1}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm7, %zmm3 {%k4}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm27, %zmm1 {%k1}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm27 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, %zmm27 {%k4}
-; AVX512BW-ONLY-FAST-NEXT:    movb $24, %sil
-; AVX512BW-ONLY-FAST-NEXT:    kmovd %esi, %k3
+; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm13 {%k4}
+; AVX512BW-ONLY-FAST-NEXT:    movb $120, %sil
+; AVX512BW-ONLY-FAST-NEXT:    kmovd %esi, %k4
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm9 {%k3}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm1 # 64-byte Folded Reload
-; AVX512BW-ONLY-FAST-NEXT:    # zmm1 = zmm0[0,1,2,3],mem[4,5,6,7]
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k5}
-; AVX512BW-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm7 = zmm8[0,1,2,3],zmm10[4,5,6,7]
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm7 {%k5}
-; AVX512BW-ONLY-FAST-NEXT:    movb $-31, %sil
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm6, %zmm9 {%k4}
+; AVX512BW-ONLY-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm22, %zmm5 # 64-byte Folded Reload
+; AVX512BW-ONLY-FAST-NEXT:    # zmm5 = zmm22[0,1,2,3],mem[4,5,6,7]
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm19, %zmm3 {%k1}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm10 {%k4}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm31, %zmm1 {%k1}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, %zmm18 {%k4}
+; AVX512BW-ONLY-FAST-NEXT:    movb $-61, %sil
 ; AVX512BW-ONLY-FAST-NEXT:    kmovd %esi, %k4
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm9 {%k4}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm30 {%k3}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm18, %zmm30 {%k4}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm24, %zmm26 {%k3}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm11, %zmm26 {%k4}
-; AVX512BW-ONLY-FAST-NEXT:    vpbroadcastq 8(%rcx), %ymm0
-; AVX512BW-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm5 {%k4}
+; AVX512BW-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm17[0,1,2,3],zmm29[4,5,6,7]
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, %zmm0 {%k4}
+; AVX512BW-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm4[0,1,2,3],zmm28[4,5,6,7]
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm1 {%k4}
+; AVX512BW-ONLY-FAST-NEXT:    vpbroadcastq 8(%rcx), %ymm3
+; AVX512BW-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm3 = mem[0,1,2,3],ymm3[4,5,6,7]
 ; AVX512BW-ONLY-FAST-NEXT:    movb $6, %sil
 ; AVX512BW-ONLY-FAST-NEXT:    kmovd %esi, %k4
-; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm21 {%k4}
-; AVX512BW-ONLY-FAST-NEXT:    vpbroadcastq 72(%rcx), %ymm0
-; AVX512BW-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
-; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm20 {%k4}
-; AVX512BW-ONLY-FAST-NEXT:    vpbroadcastq 136(%rcx), %ymm0
-; AVX512BW-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
-; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm2 {%k4}
-; AVX512BW-ONLY-FAST-NEXT:    vpbroadcastq 200(%rcx), %ymm0
-; AVX512BW-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
-; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm4 {%k4}
-; AVX512BW-ONLY-FAST-NEXT:    movb $56, %cl
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm4 {%k4}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm6 {%k3}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm30 {%k3}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm27 {%k3}
+; AVX512BW-ONLY-FAST-NEXT:    movb $-31, %sil
+; AVX512BW-ONLY-FAST-NEXT:    kmovd %esi, %k3
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm6 {%k3}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm6, %zmm19
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm30 {%k3}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm27 {%k3}
+; AVX512BW-ONLY-FAST-NEXT:    movb $56, %sil
+; AVX512BW-ONLY-FAST-NEXT:    kmovd %esi, %k3
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm4 {%k3}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm4, %zmm17
+; AVX512BW-ONLY-FAST-NEXT:    vpbroadcastq 72(%rcx), %ymm3
+; AVX512BW-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm3 = mem[0,1,2,3],ymm3[4,5,6,7]
+; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm26 {%k4}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm26 {%k3}
+; AVX512BW-ONLY-FAST-NEXT:    vpbroadcastq 136(%rcx), %ymm3
+; AVX512BW-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm3 = mem[0,1,2,3],ymm3[4,5,6,7]
+; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm20 {%k4}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm20 {%k3}
+; AVX512BW-ONLY-FAST-NEXT:    vpbroadcastq 200(%rcx), %ymm3
+; AVX512BW-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm3 = mem[0,1,2,3],ymm3[4,5,6,7]
+; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm11 {%k4}
+; AVX512BW-ONLY-FAST-NEXT:    movb $64, %cl
 ; AVX512BW-ONLY-FAST-NEXT:    kmovd %ecx, %k4
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm21 {%k4}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm20 {%k4}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k4}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm14, %zmm22, %zmm8
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 192(%rax), %zmm0
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm9, %zmm8
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm8, %zmm4 {%k4}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm8, %zmm6 {%k1}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm8, %zmm11 {%k1}
-; AVX512BW-ONLY-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm23, %zmm8 # 64-byte Folded Reload
-; AVX512BW-ONLY-FAST-NEXT:    # zmm8 = zmm23[0,1,2,3],mem[4,5,6,7]
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm9 = <0,11,u,u,4,5,6,7>
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm22, %zmm12, %zmm9
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm10 = <0,1,11,u,4,5,6,7>
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm14, %zmm9, %zmm10
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa 192(%r8), %ymm9
-; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm9 = ymm9[0],mem[0],ymm9[2],mem[2]
-; AVX512BW-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm11 {%k2} = zmm9[2,3,2,3],zmm0[2,3,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm11, %zmm12
-; AVX512BW-ONLY-FAST-NEXT:    movb $64, %al
-; AVX512BW-ONLY-FAST-NEXT:    kmovd %eax, %k1
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm22, %zmm8 {%k1}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm6 {%k4}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm8, %zmm3, %zmm2
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 192(%rax), %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm3, %zmm4, %zmm2
 ; AVX512BW-ONLY-FAST-NEXT:    movb $8, %al
-; AVX512BW-ONLY-FAST-NEXT:    kmovd %eax, %k1
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm10 {%k1}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm15, %zmm19 {%k3}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm9 = <12,u,u,3,4,5,6,13>
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm22, %zmm6, %zmm9
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm6 = <u,1,2,3,4,15,u,u>
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm22, %zmm19, %zmm6
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm11 = <0,12,u,3,4,5,6,7>
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm14, %zmm9, %zmm11
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm9 = <13,u,2,3,4,5,6,14>
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm14, %zmm8, %zmm9
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm8 = <u,1,2,3,4,5,15,u>
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm14, %zmm6, %zmm8
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [0,1,12,3,4,5,6,7]
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm0, %zmm11, %zmm6
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm11 = [0,13,2,3,4,5,6,7]
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm0, %zmm9, %zmm11
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm9 = [14,1,2,3,4,5,6,15]
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm0, %zmm8, %zmm9
+; AVX512BW-ONLY-FAST-NEXT:    kmovd %eax, %k4
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm15 {%k4}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm11 {%k3}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm12, %zmm7 {%k1}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa 192(%r8), %ymm2
+; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm2 = ymm2[0],mem[0],ymm2[2],mem[2]
+; AVX512BW-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm7 {%k2} = zmm2[2,3,2,3],zmm3[2,3,2,3]
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm2 = <0,12,u,3,4,5,6,7>
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm8, %zmm4, %zmm2
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm4 = <13,u,2,3,4,5,6,14>
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm8, %zmm6, %zmm4
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm6 = <u,1,2,3,4,5,15,u>
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm8, %zmm24, %zmm6
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm8 = [0,1,12,3,4,5,6,7]
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm3, %zmm2, %zmm8
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [0,13,2,3,4,5,6,7]
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm3, %zmm4, %zmm2
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [14,1,2,3,4,5,6,15]
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm3, %zmm6, %zmm4
 ; AVX512BW-ONLY-FAST-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm12, 1472(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm4, 1408(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm26, 1280(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm7, 1216(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm27, 1152(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm5, 1088(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovaps %zmm0, 1024(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, 960(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm7, 1472(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm11, 1408(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm27, 1280(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, 1216(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm18, 1152(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm13, 1088(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovaps %zmm1, 1024(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm20, 960(%rax)
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm30, 832(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, 768(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, 704(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm17, 640(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, 768(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm10, 704(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm16, 640(%rax)
 ; AVX512BW-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-ONLY-FAST-NEXT:    vmovaps %zmm0, 576(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm20, 512(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm26, 512(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm19, 384(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm5, 320(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm9, 256(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm25, 192(%rax)
 ; AVX512BW-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovaps %zmm0, 384(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm13, 320(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovaps %zmm0, 128(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm17, 64(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm14, 1344(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm21, 896(%rax)
 ; AVX512BW-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovaps %zmm0, 256(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm29, 192(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovaps %zmm0, 448(%rax)
 ; AVX512BW-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovaps %zmm0, 128(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm21, 64(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm31, 1344(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm28, 896(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm16, 448(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm25, (%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm9, 1728(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm11, 1664(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm6, 1600(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm10, 1536(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    addq $2024, %rsp # imm = 0x7E8
+; AVX512BW-ONLY-FAST-NEXT:    vmovaps %zmm0, (%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm4, 1728(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, 1664(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm8, 1600(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm15, 1536(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    addq $2152, %rsp # imm = 0x868
 ; AVX512BW-ONLY-FAST-NEXT:    vzeroupper
 ; AVX512BW-ONLY-FAST-NEXT:    retq
 ;
 ; AVX512DQBW-SLOW-LABEL: store_i64_stride7_vf32:
 ; AVX512DQBW-SLOW:       # %bb.0:
-; AVX512DQBW-SLOW-NEXT:    subq $2120, %rsp # imm = 0x848
+; AVX512DQBW-SLOW-NEXT:    subq $2184, %rsp # imm = 0x888
 ; AVX512DQBW-SLOW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 (%rdi), %zmm10
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 64(%rdi), %zmm8
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 (%rsi), %zmm15
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 64(%rsi), %zmm13
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 (%rdi), %zmm16
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 64(%rdi), %zmm9
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 (%rsi), %zmm23
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 64(%rsi), %zmm10
 ; AVX512DQBW-SLOW-NEXT:    vmovdqa64 64(%rdx), %zmm20
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 (%rdx), %zmm21
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 (%rcx), %zmm5
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 64(%rcx), %zmm28
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 (%r8), %zmm3
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 (%r9), %zmm2
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 (%rax), %zmm4
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 64(%rax), %zmm7
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 (%rdx), %zmm18
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 (%rcx), %zmm6
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 64(%rcx), %zmm7
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 (%r8), %zmm0
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 (%r9), %zmm3
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 (%rax), %zmm2
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 64(%rax), %zmm4
 ; AVX512DQBW-SLOW-NEXT:    movb $96, %r10b
 ; AVX512DQBW-SLOW-NEXT:    kmovd %r10d, %k1
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [9,1,9,1,9,1,9,1]
-; AVX512DQBW-SLOW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm1
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm2, %zmm0, %zmm1
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm14
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [4,9,0,3,4,9,0,3]
-; AVX512DQBW-SLOW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm4, %zmm0, %zmm1
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm16
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm1 = [11,3,11,3,11,3,11,3]
+; AVX512DQBW-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm17 = [15,7,15,7,15,7,15,7]
+; AVX512DQBW-SLOW-NEXT:    # zmm17 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm1 = [9,1,9,1,9,1,9,1]
 ; AVX512DQBW-SLOW-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm10, %zmm0
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm15, %zmm1, %zmm0
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm18
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm9 = [2,10,0,3,2,10,0,3]
-; AVX512DQBW-SLOW-NEXT:    # zmm9 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm21, %zmm1
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm5, %zmm9, %zmm1
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm9, %zmm17
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
-; AVX512DQBW-SLOW-NEXT:    vmovdqa (%r9), %ymm9
-; AVX512DQBW-SLOW-NEXT:    vmovdqu %ymm9, (%rsp) # 32-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa 64(%r9), %ymm11
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm5
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm3, %zmm1, %zmm5
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm12
+; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm1 = [4,9,0,3,4,9,0,3]
+; AVX512DQBW-SLOW-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm2, %zmm1, %zmm5
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm13
+; AVX512DQBW-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm25 = [11,3,11,3,11,3,11,3]
+; AVX512DQBW-SLOW-NEXT:    # zmm25 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm16, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm23, %zmm25, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm8 = [2,10,0,3,2,10,0,3]
+; AVX512DQBW-SLOW-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm18, %zmm5
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm6, %zmm8, %zmm5
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm15
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm5 {%k1}
+; AVX512DQBW-SLOW-NEXT:    vmovdqa (%r9), %ymm8
+; AVX512DQBW-SLOW-NEXT:    vmovdqu %ymm8, (%rsp) # 32-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa 64(%r9), %ymm14
+; AVX512DQBW-SLOW-NEXT:    vmovdqu %ymm14, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa (%r8), %ymm1
+; AVX512DQBW-SLOW-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa 64(%r8), %ymm11
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa (%r8), %ymm0
-; AVX512DQBW-SLOW-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa 64(%r8), %ymm12
-; AVX512DQBW-SLOW-NEXT:    vmovdqu %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm9[0],ymm0[2],ymm9[2]
+; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm1 = ymm1[0],ymm8[0],ymm1[2],ymm8[2]
 ; AVX512DQBW-SLOW-NEXT:    movb $28, %r10b
 ; AVX512DQBW-SLOW-NEXT:    kmovd %r10d, %k2
-; AVX512DQBW-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm1 {%k2} = zmm0[2,3,2,3],zmm4[2,3,2,3]
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [4,12,0,5,4,12,0,5]
-; AVX512DQBW-SLOW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm1
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm2, %zmm0, %zmm1
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm9
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [0,1,12,7,0,1,12,7]
-; AVX512DQBW-SLOW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm4, %zmm0, %zmm1
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm22 = [5,0,14,6,5,0,14,6]
-; AVX512DQBW-SLOW-NEXT:    # zmm22 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm19
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm3, %zmm22, %zmm19
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm1 = [0,13,6,7,0,13,6,7]
+; AVX512DQBW-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm5 {%k2} = zmm1[2,3,2,3],zmm2[2,3,2,3]
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm8 = [4,12,0,5,4,12,0,5]
+; AVX512DQBW-SLOW-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm5
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm3, %zmm8, %zmm5
+; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm1 = [0,1,12,7,0,1,12,7]
 ; AVX512DQBW-SLOW-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm4, %zmm1, %zmm19
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm19
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm4 = [15,7,15,7,15,7,15,7]
-; AVX512DQBW-SLOW-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm2, %zmm4, %zmm3
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [6,13,14,7,6,13,14,7]
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm2, %zmm1, %zmm5
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm5
+; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm21 = [5,0,14,6,5,0,14,6]
+; AVX512DQBW-SLOW-NEXT:    # zmm21 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm19
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm0, %zmm21, %zmm19
+; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [0,13,6,7,0,13,6,7]
 ; AVX512DQBW-SLOW-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm3, %zmm2, %zmm19
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm3
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm13, %zmm18, %zmm3
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm20, %zmm19
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm28, %zmm17, %zmm19
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm17, %zmm23
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm19 {%k1}
-; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm3 = ymm12[0],ymm11[0],ymm12[2],ymm11[2]
-; AVX512DQBW-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm19 {%k2} = zmm3[2,3,2,3],zmm7[2,3,2,3]
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm5, %zmm2, %zmm19
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 64(%r8), %zmm3
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 64(%r9), %zmm12
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm11
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm12, %zmm14, %zmm11
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm7, %zmm16, %zmm11
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm11
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm12, %zmm9, %zmm11
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm7, %zmm0, %zmm11
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm12, %zmm11
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm3, %zmm17, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm3 = [6,13,14,7,6,13,14,7]
+; AVX512DQBW-SLOW-NEXT:    # zmm3 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm0, %zmm3, %zmm5
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm9, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm10, %zmm25, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm20, %zmm5
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm7, %zmm15, %zmm5
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm5 {%k1}
+; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm11[0],ymm14[0],ymm11[2],ymm14[2]
+; AVX512DQBW-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm5 {%k2} = zmm0[2,3,2,3],zmm4[2,3,2,3]
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 64(%r8), %zmm0
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 64(%r9), %zmm11
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm5
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm11, %zmm12, %zmm5
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm3, %zmm22, %zmm11
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm7, %zmm1, %zmm11
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm4, %zmm13, %zmm5
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm5
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm4
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm11, %zmm8, %zmm4
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm5, %zmm1, %zmm4
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm4
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm12, %zmm4, %zmm3
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm3, %zmm2, %zmm7
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 128(%rdi), %zmm24
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 128(%rsi), %zmm31
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm24, %zmm3
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm31, %zmm18, %zmm3
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 128(%rdx), %zmm17
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 128(%rcx), %zmm30
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm17, %zmm7
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm30, %zmm23, %zmm7
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm7 {%k1}
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 128(%rax), %zmm29
-; AVX512DQBW-SLOW-NEXT:    vmovdqa 128(%r9), %ymm11
-; AVX512DQBW-SLOW-NEXT:    vmovdqu %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa 128(%r8), %ymm3
-; AVX512DQBW-SLOW-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm3 = ymm3[0],ymm11[0],ymm3[2],ymm11[2]
-; AVX512DQBW-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm7 {%k2} = zmm3[2,3,2,3],zmm29[2,3,2,3]
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 128(%r8), %zmm3
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 128(%r9), %zmm7
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm7, %zmm3, %zmm9
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm29, %zmm0, %zmm9
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm3, %zmm7, %zmm22
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm29, %zmm1, %zmm22
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm22, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm0
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm7, %zmm14, %zmm0
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm29, %zmm16, %zmm0
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm7, %zmm4, %zmm3
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm3, %zmm2, %zmm29
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm3 = [13,5,13,5,13,5,13,5]
-; AVX512DQBW-SLOW-NEXT:    # zmm3 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm21, %zmm0
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm5, %zmm3, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm0, %zmm21, %zmm4
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm5, %zmm2, %zmm4
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm25 = [6,14,6,14,6,14,6,14]
-; AVX512DQBW-SLOW-NEXT:    # zmm25 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm21, %zmm0
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm5, %zmm25, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm11, %zmm17, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm0, %zmm3, %zmm5
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 128(%rdi), %zmm11
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 128(%rsi), %zmm19
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm5
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm19, %zmm25, %zmm5
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 128(%rdx), %zmm4
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 128(%rcx), %zmm31
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm31, %zmm15, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm0 {%k1}
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 128(%rax), %zmm15
+; AVX512DQBW-SLOW-NEXT:    vmovdqa 128(%r9), %ymm14
+; AVX512DQBW-SLOW-NEXT:    vmovdqu %ymm14, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa 128(%r8), %ymm5
+; AVX512DQBW-SLOW-NEXT:    vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm5 = ymm5[0],ymm14[0],ymm5[2],ymm14[2]
+; AVX512DQBW-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm0 {%k2} = zmm5[2,3,2,3],zmm15[2,3,2,3]
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm21, %zmm0
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm5, %zmm4, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 128(%r8), %zmm5
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 128(%r9), %zmm0
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm0, %zmm5, %zmm8
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm15, %zmm1, %zmm8
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm5, %zmm0, %zmm21
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm15, %zmm2, %zmm21
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm0, %zmm12, %zmm1
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [3,0,12,4,3,0,12,4]
-; AVX512DQBW-SLOW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm21, %zmm0, %zmm5
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm20, %zmm26
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm28, %zmm3, %zmm26
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm20, %zmm1
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm28, %zmm25, %zmm1
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm20, %zmm1
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm28, %zmm4, %zmm1
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm20, %zmm0, %zmm28
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 192(%rdx), %zmm12
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 192(%rcx), %zmm2
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm12, %zmm1
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm2, %zmm25, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm15, %zmm13, %zmm1
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm17, %zmm11
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm30, %zmm3, %zmm11
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm30, %zmm17, %zmm25
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm17, %zmm22
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm30, %zmm4, %zmm22
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm17, %zmm0, %zmm30
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm16
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm12, %zmm0, %zmm16
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm2, %zmm12, %zmm23
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm23, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm12, %zmm0
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm2, %zmm4, %zmm12
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm2, %zmm3, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm0, %zmm17, %zmm5
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm5, %zmm3, %zmm15
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm1 = [13,5,13,5,13,5,13,5]
+; AVX512DQBW-SLOW-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm18, %zmm2
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm6, %zmm1, %zmm2
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm6 = [6,14,6,14,6,14,6,14]
+; AVX512DQBW-SLOW-NEXT:    # zmm6 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm18, %zmm2
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm0, %zmm6, %zmm2
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm18, %zmm2
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm0, %zmm17, %zmm2
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm17, %zmm15
+; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm3 = [3,0,12,4,3,0,12,4]
+; AVX512DQBW-SLOW-NEXT:    # zmm3 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm18, %zmm3, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm2
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm20, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm7, %zmm1, %zmm0
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm20, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm7, %zmm6, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm20, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm7, %zmm17, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm20, %zmm3, %zmm7
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm5
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm29
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm31, %zmm1, %zmm29
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm30
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm31, %zmm6, %zmm30
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm31, %zmm17, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm4, %zmm3, %zmm31
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 192(%rdx), %zmm12
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 192(%rcx), %zmm21
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm21, %zmm22
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm12, %zmm3, %zmm22
 ; AVX512DQBW-SLOW-NEXT:    movb $48, %r10b
 ; AVX512DQBW-SLOW-NEXT:    kmovd %r10d, %k3
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [1,0,10,2,1,0,10,2]
-; AVX512DQBW-SLOW-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm15, %zmm20
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm10, %zmm4, %zmm20
-; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm5 {%k3} = zmm10[0],zmm15[0],zmm10[2],zmm15[2],zmm10[4],zmm15[4],zmm10[6],zmm15[6]
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 192(%rdi), %zmm8
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 192(%rsi), %zmm7
+; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm22 {%k3} = zmm8[0],zmm7[0],zmm8[2],zmm7[2],zmm8[4],zmm7[4],zmm8[6],zmm7[6]
+; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm14 = [1,0,10,2,1,0,10,2]
+; AVX512DQBW-SLOW-NEXT:    # zmm14 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm23, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm23, %zmm28
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm16, %zmm14, %zmm28
+; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm13 = [0,8,0,1,0,8,0,1]
+; AVX512DQBW-SLOW-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm16, %zmm27
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm23, %zmm13, %zmm27
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm10, %zmm26
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm9, %zmm14, %zmm26
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm9, %zmm23
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm10, %zmm13, %zmm23
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm19, %zmm20
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm11, %zmm14, %zmm20
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm18
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm19, %zmm13, %zmm18
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm7, %zmm8, %zmm13
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm8, %zmm7, %zmm14
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm24
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm3
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm4
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm7, %zmm25, %zmm8
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm7, %zmm6, %zmm24
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm7, %zmm1, %zmm3
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm7, %zmm17, %zmm4
+; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm2 {%k3} = zmm16[0],zmm0[0],zmm16[2],zmm0[2],zmm16[4],zmm0[4],zmm16[6],zmm0[6]
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm16, %zmm25
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm16, %zmm2
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm16, %zmm17
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm0, %zmm1, %zmm25
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm0, %zmm6, %zmm2
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm0, %zmm15, %zmm17
+; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm5 {%k3} = zmm9[0],zmm10[0],zmm9[2],zmm10[2],zmm9[4],zmm10[4],zmm9[6],zmm10[6]
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm10, %zmm18
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm10, %zmm7
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm10, %zmm21
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm10, %zmm14
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [0,8,0,1,0,8,0,1]
-; AVX512DQBW-SLOW-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm15, %zmm2, %zmm14
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm15, %zmm3, %zmm18
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [6,14,6,14]
-; AVX512DQBW-SLOW-NEXT:    # ymm1 = mem[0,1,0,1]
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm15, %zmm1, %zmm7
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti128 {{.*#+}} ymm6 = [15,7,15,7]
-; AVX512DQBW-SLOW-NEXT:    # ymm6 = mem[0,1,0,1]
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm15, %zmm6, %zmm21
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm13, %zmm15
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm8, %zmm4, %zmm15
-; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm28 {%k3} = zmm8[0],zmm13[0],zmm8[2],zmm13[2],zmm8[4],zmm13[4],zmm8[6],zmm13[6]
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm23
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm5
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm17
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm10
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm13, %zmm2, %zmm10
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm13, %zmm3, %zmm23
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm13, %zmm1, %zmm5
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm13, %zmm6, %zmm17
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 192(%rdi), %zmm19
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 192(%rsi), %zmm0
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm19, %zmm27
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm0, %zmm1, %zmm27
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm19, %zmm13
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm0, %zmm3, %zmm13
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm31, %zmm8
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm24, %zmm4, %zmm8
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm31, %zmm24, %zmm3
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm31, %zmm24, %zmm1
-; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm30 {%k3} = zmm24[0],zmm31[0],zmm24[2],zmm31[2],zmm24[4],zmm31[4],zmm24[6],zmm31[6]
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm24, %zmm9
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm31, %zmm2, %zmm24
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm31, %zmm6, %zmm9
-; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm16 {%k3} = zmm19[0],zmm0[0],zmm19[2],zmm0[2],zmm19[4],zmm0[4],zmm19[6],zmm0[6]
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm0, %zmm19, %zmm2
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm19, %zmm0, %zmm4
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm31 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm0, %zmm19, %zmm31
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm31, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm0, %zmm6, %zmm19
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm18 {%k1}
-; AVX512DQBW-SLOW-NEXT:    movb $120, %sil
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm9, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm9, %zmm5
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm9, %zmm16
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm10, %zmm1, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm10, %zmm6, %zmm5
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm10, %zmm15, %zmm16
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm12, %zmm9
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm21, %zmm1, %zmm9
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm19, %zmm11, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm12, %zmm10
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm21, %zmm6, %zmm10
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm19, %zmm11, %zmm6
+; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm31 {%k3} = zmm11[0],zmm19[0],zmm11[2],zmm19[2],zmm11[4],zmm19[4],zmm11[6],zmm19[6]
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm7
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm19, %zmm15, %zmm7
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm12, %zmm11
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm21, %zmm15, %zmm11
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm21, %zmm15, %zmm12
+; AVX512DQBW-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm24 = zmm24[0,1,2,3],zmm10[4,5,6,7]
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 192(%r8), %zmm19
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm10 = <0,11,u,u,4,5,6,7>
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm19, %zmm22, %zmm10
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm9, %zmm3 {%k1}
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm22 = <12,u,u,3,4,5,6,13>
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm19, %zmm3, %zmm22
+; AVX512DQBW-SLOW-NEXT:    movb $24, %sil
 ; AVX512DQBW-SLOW-NEXT:    kmovd %esi, %k3
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm18, %zmm0 {%k3}
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm7, %zmm18 # 64-byte Folded Reload
-; AVX512DQBW-SLOW-NEXT:    # zmm18 = zmm7[0,1,2,3],mem[4,5,6,7]
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm4 {%k3}
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm21 = <u,1,2,3,4,15,u,u>
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm19, %zmm4, %zmm21
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm25 {%k1}
+; AVX512DQBW-SLOW-NEXT:    movb $120, %sil
+; AVX512DQBW-SLOW-NEXT:    kmovd %esi, %k4
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm25, %zmm11 {%k4}
+; AVX512DQBW-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm3 # 64-byte Folded Reload
+; AVX512DQBW-SLOW-NEXT:    # zmm3 = zmm2[0,1,2,3],mem[4,5,6,7]
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm0 {%k1}
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm15 {%k4}
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm29, %zmm1 {%k1}
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm25 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm25 {%k4}
 ; AVX512DQBW-SLOW-NEXT:    movb $-61, %sil
 ; AVX512DQBW-SLOW-NEXT:    kmovd %esi, %k4
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm18 {%k4}
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm26, %zmm23 {%k1}
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm26 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm23, %zmm26 {%k3}
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm3 {%k1}
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm3 {%k4}
+; AVX512DQBW-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm4 # 64-byte Folded Reload
+; AVX512DQBW-SLOW-NEXT:    # zmm4 = zmm5[0,1,2,3],mem[4,5,6,7]
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm0 {%k3}
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    movb $24, %sil
-; AVX512DQBW-SLOW-NEXT:    kmovd %esi, %k3
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm4 {%k4}
+; AVX512DQBW-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm6[0,1,2,3],zmm30[4,5,6,7]
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm21 {%k3}
-; AVX512DQBW-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm3 # 64-byte Folded Reload
-; AVX512DQBW-SLOW-NEXT:    # zmm3 = zmm5[0,1,2,3],mem[4,5,6,7]
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm3 {%k4}
-; AVX512DQBW-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm25[4,5,6,7]
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm0 {%k4}
-; AVX512DQBW-SLOW-NEXT:    movb $-31, %sil
-; AVX512DQBW-SLOW-NEXT:    kmovd %esi, %k4
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm21 {%k4}
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm17 {%k3}
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm17 {%k4}
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm22, %zmm9 {%k3}
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm29, %zmm9 {%k4}
-; AVX512DQBW-SLOW-NEXT:    vmovdqa (%rdx), %xmm1
-; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
-; AVX512DQBW-SLOW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k4}
+; AVX512DQBW-SLOW-NEXT:    vmovdqa (%rdx), %xmm0
+; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
+; AVX512DQBW-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
 ; AVX512DQBW-SLOW-NEXT:    movb $12, %sil
 ; AVX512DQBW-SLOW-NEXT:    kmovd %esi, %k4
-; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm14 {%k4}
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm1 = [0,8,0,8,0,8,0,8]
-; AVX512DQBW-SLOW-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm6 # 64-byte Folded Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqa 64(%rdx), %xmm5
-; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm5 = xmm5[0],mem[0]
-; AVX512DQBW-SLOW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm5, %zmm0, %zmm10 {%k4}
-; AVX512DQBW-SLOW-NEXT:    vmovdqa 128(%rdx), %xmm5
-; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm5 = xmm5[0],mem[0]
-; AVX512DQBW-SLOW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm5, %zmm0, %zmm24 {%k4}
-; AVX512DQBW-SLOW-NEXT:    vmovdqa 192(%rdx), %xmm5
-; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm5 = xmm5[0],mem[0]
-; AVX512DQBW-SLOW-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm5, %zmm0, %zmm2 {%k4}
-; AVX512DQBW-SLOW-NEXT:    movb $112, %sil
-; AVX512DQBW-SLOW-NEXT:    kmovd %esi, %k4
-; AVX512DQBW-SLOW-NEXT:    vinserti64x2 $3, (%rax), %zmm6, %zmm14 {%k4}
+; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm27 {%k4}
+; AVX512DQBW-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [0,8,0,8,0,8,0,8]
+; AVX512DQBW-SLOW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 # 64-byte Folded Reload
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm5 # 64-byte Folded Reload
-; AVX512DQBW-SLOW-NEXT:    vinserti64x2 $3, 64(%rax), %zmm5, %zmm10 {%k4}
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm17 {%k3}
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm5 # 64-byte Folded Reload
-; AVX512DQBW-SLOW-NEXT:    vinserti64x2 $3, 128(%rax), %zmm5, %zmm24 {%k4}
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 192(%r8), %zmm5
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 192(%r9), %zmm11
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm11, %zmm5, %zmm1
-; AVX512DQBW-SLOW-NEXT:    vinserti64x2 $3, 192(%rax), %zmm1, %zmm2 {%k4}
-; AVX512DQBW-SLOW-NEXT:    vpbroadcastq 8(%rcx), %ymm1
-; AVX512DQBW-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = mem[0,1,2,3],ymm1[4,5,6,7]
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm16 {%k3}
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm7 {%k3}
+; AVX512DQBW-SLOW-NEXT:    movb $-31, %sil
+; AVX512DQBW-SLOW-NEXT:    kmovd %esi, %k3
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm17 {%k3}
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm16 {%k3}
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm7 {%k3}
+; AVX512DQBW-SLOW-NEXT:    movb $112, %sil
+; AVX512DQBW-SLOW-NEXT:    kmovd %esi, %k3
+; AVX512DQBW-SLOW-NEXT:    vinserti64x2 $3, (%rax), %zmm2, %zmm27 {%k3}
+; AVX512DQBW-SLOW-NEXT:    vmovdqa 64(%rdx), %xmm2
+; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],mem[0]
+; AVX512DQBW-SLOW-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm2, %zmm0, %zmm23 {%k4}
+; AVX512DQBW-SLOW-NEXT:    vmovdqa 128(%rdx), %xmm2
+; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],mem[0]
+; AVX512DQBW-SLOW-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm2, %zmm0, %zmm18 {%k4}
+; AVX512DQBW-SLOW-NEXT:    vmovdqa 192(%rdx), %xmm2
+; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],mem[0]
+; AVX512DQBW-SLOW-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm2, %zmm0, %zmm13 {%k4}
+; AVX512DQBW-SLOW-NEXT:    vpbroadcastq 8(%rcx), %ymm2
+; AVX512DQBW-SLOW-NEXT:    vpblendd {{.*#+}} ymm2 = mem[0,1,2,3],ymm2[4,5,6,7]
 ; AVX512DQBW-SLOW-NEXT:    movb $6, %sil
 ; AVX512DQBW-SLOW-NEXT:    kmovd %esi, %k4
-; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm20 {%k4}
-; AVX512DQBW-SLOW-NEXT:    vpbroadcastq 72(%rcx), %ymm1
-; AVX512DQBW-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = mem[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm15 {%k4}
-; AVX512DQBW-SLOW-NEXT:    vpbroadcastq 136(%rcx), %ymm1
-; AVX512DQBW-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = mem[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm8 {%k4}
-; AVX512DQBW-SLOW-NEXT:    vpbroadcastq 200(%rcx), %ymm1
-; AVX512DQBW-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = mem[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm4 {%k4}
-; AVX512DQBW-SLOW-NEXT:    movb $56, %cl
+; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm2, %zmm0, %zmm28 {%k4}
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 # 64-byte Folded Reload
+; AVX512DQBW-SLOW-NEXT:    vinserti64x2 $3, 64(%rax), %zmm2, %zmm23 {%k3}
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 # 64-byte Folded Reload
+; AVX512DQBW-SLOW-NEXT:    vinserti64x2 $3, 128(%rax), %zmm2, %zmm18 {%k3}
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 192(%r9), %zmm2
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm2, %zmm19, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vinserti64x2 $3, 192(%rax), %zmm0, %zmm13 {%k3}
+; AVX512DQBW-SLOW-NEXT:    movb $56, %sil
+; AVX512DQBW-SLOW-NEXT:    kmovd %esi, %k3
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm28 {%k3}
+; AVX512DQBW-SLOW-NEXT:    vpbroadcastq 72(%rcx), %ymm0
+; AVX512DQBW-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
+; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm26 {%k4}
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm26 {%k3}
+; AVX512DQBW-SLOW-NEXT:    vpbroadcastq 136(%rcx), %ymm0
+; AVX512DQBW-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
+; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm20 {%k4}
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm20 {%k3}
+; AVX512DQBW-SLOW-NEXT:    vpbroadcastq 200(%rcx), %ymm0
+; AVX512DQBW-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
+; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm14 {%k4}
+; AVX512DQBW-SLOW-NEXT:    movb $64, %cl
 ; AVX512DQBW-SLOW-NEXT:    kmovd %ecx, %k4
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm20 {%k4}
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm15 {%k4}
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm8 {%k4}
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm11, %zmm5, %zmm6
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 192(%rax), %zmm1
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm1, %zmm7, %zmm6
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm4 {%k4}
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm13 {%k1}
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm23 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm23 {%k1}
-; AVX512DQBW-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm27, %zmm7 # 64-byte Folded Reload
-; AVX512DQBW-SLOW-NEXT:    # zmm7 = zmm27[0,1,2,3],mem[4,5,6,7]
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm22 = <0,11,u,u,4,5,6,7>
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm5, %zmm16, %zmm22
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm6 = <0,1,11,u,4,5,6,7>
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm11, %zmm22, %zmm6
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 192(%r8), %ymm16
-; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm16 = ymm16[0],mem[0],ymm16[2],mem[2]
-; AVX512DQBW-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm23 {%k2} = zmm16[2,3,2,3],zmm1[2,3,2,3]
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm12, %zmm19 {%k3}
-; AVX512DQBW-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vpunpckhqdq (%rsp), %ymm12, %ymm12 # 32-byte Folded Reload
-; AVX512DQBW-SLOW-NEXT:    # ymm12 = ymm12[1],mem[1],ymm12[3],mem[3]
-; AVX512DQBW-SLOW-NEXT:    vpermq {{.*#+}} ymm12 = ymm12[0,2,3,3]
-; AVX512DQBW-SLOW-NEXT:    vpblendd {{.*#+}} ymm12 = ymm12[0,1,2,3,4,5],mem[6,7]
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm19, %zmm24 {%k4}
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm2, %zmm0, %zmm19
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 192(%rax), %zmm0
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm0, %zmm5, %zmm19
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm19, %zmm14 {%k3}
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm5 = <0,1,11,u,4,5,6,7>
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm2, %zmm10, %zmm5
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm12 {%k1}
+; AVX512DQBW-SLOW-NEXT:    vmovdqa 192(%r8), %ymm6
+; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm6 = ymm6[0],mem[0],ymm6[2],mem[2]
+; AVX512DQBW-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm12 {%k2} = zmm6[2,3,2,3],zmm0[2,3,2,3]
+; AVX512DQBW-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vpunpckhqdq (%rsp), %ymm6, %ymm6 # 32-byte Folded Reload
+; AVX512DQBW-SLOW-NEXT:    # ymm6 = ymm6[1],mem[1],ymm6[3],mem[3]
+; AVX512DQBW-SLOW-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[0,2,3,3]
+; AVX512DQBW-SLOW-NEXT:    vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3,4,5],mem[6,7]
 ; AVX512DQBW-SLOW-NEXT:    movb $14, %cl
 ; AVX512DQBW-SLOW-NEXT:    kmovd %ecx, %k1
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm12, %zmm0, %zmm16 {%k1}
-; AVX512DQBW-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm12, %ymm12 # 32-byte Folded Reload
-; AVX512DQBW-SLOW-NEXT:    # ymm12 = ymm12[1],mem[1],ymm12[3],mem[3]
-; AVX512DQBW-SLOW-NEXT:    vpermq {{.*#+}} ymm12 = ymm12[0,2,3,3]
-; AVX512DQBW-SLOW-NEXT:    vpblendd {{.*#+}} ymm12 = ymm12[0,1,2,3,4,5],mem[6,7]
-; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm12, %zmm0, %zmm28 {%k1}
-; AVX512DQBW-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm12 # 32-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm12, %ymm12 # 32-byte Folded Reload
-; AVX512DQBW-SLOW-NEXT:    # ymm12 = ymm12[1],mem[1],ymm12[3],mem[3]
-; AVX512DQBW-SLOW-NEXT:    movb $64, %cl
-; AVX512DQBW-SLOW-NEXT:    kmovd %ecx, %k2
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm7 {%k2}
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm6, %zmm0, %zmm10 {%k1}
+; AVX512DQBW-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm6, %ymm6 # 32-byte Folded Reload
+; AVX512DQBW-SLOW-NEXT:    # ymm6 = ymm6[1],mem[1],ymm6[3],mem[3]
+; AVX512DQBW-SLOW-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[0,2,3,3]
+; AVX512DQBW-SLOW-NEXT:    vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3,4,5],mem[6,7]
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm6, %zmm0, %zmm19 {%k1}
 ; AVX512DQBW-SLOW-NEXT:    movb $8, %cl
 ; AVX512DQBW-SLOW-NEXT:    kmovd %ecx, %k2
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm6 {%k2}
-; AVX512DQBW-SLOW-NEXT:    vpermq {{.*#+}} ymm12 = ymm12[0,2,3,3]
-; AVX512DQBW-SLOW-NEXT:    vpblendd {{.*#+}} ymm12 = ymm12[0,1,2,3,4,5],mem[6,7]
-; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm12, %zmm0, %zmm30 {%k1}
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm12 = <12,u,u,3,4,5,6,13>
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm5, %zmm13, %zmm12
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm13 = <u,1,2,3,4,15,u,u>
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm5, %zmm19, %zmm13
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm5 = <0,12,u,3,4,5,6,7>
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm11, %zmm12, %zmm5
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm12 = <13,u,2,3,4,5,6,14>
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm11, %zmm7, %zmm12
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm7 = <u,1,2,3,4,5,15,u>
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm11, %zmm13, %zmm7
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm11 = [0,1,12,3,4,5,6,7]
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm1, %zmm5, %zmm11
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [0,13,2,3,4,5,6,7]
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm1, %zmm12, %zmm5
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm12 = [14,1,2,3,4,5,6,15]
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm1, %zmm7, %zmm12
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm5 {%k2}
+; AVX512DQBW-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm6, %ymm6 # 32-byte Folded Reload
+; AVX512DQBW-SLOW-NEXT:    # ymm6 = ymm6[1],mem[1],ymm6[3],mem[3]
+; AVX512DQBW-SLOW-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[0,2,3,3]
+; AVX512DQBW-SLOW-NEXT:    vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3,4,5],mem[6,7]
+; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm6, %zmm0, %zmm31 {%k1}
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm6 = <0,12,u,3,4,5,6,7>
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm2, %zmm22, %zmm6
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm8 = <13,u,2,3,4,5,6,14>
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm2, %zmm24, %zmm8
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm9 = <u,1,2,3,4,5,15,u>
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm2, %zmm21, %zmm9
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm2 = [0,1,12,3,4,5,6,7]
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm0, %zmm6, %zmm2
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [0,13,2,3,4,5,6,7]
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm0, %zmm8, %zmm6
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm8 = [14,1,2,3,4,5,6,15]
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm0, %zmm9, %zmm8
 ; AVX512DQBW-SLOW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm23, 1472(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm4, 1408(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm2, 1344(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm9, 1280(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, 1216(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovaps %zmm0, 1152(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm30, 1088(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm12, 1472(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm14, 1408(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm13, 1344(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm7, 1280(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, 1216(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm25, 1152(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm31, 1088(%rax)
 ; AVX512DQBW-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQBW-SLOW-NEXT:    vmovaps %zmm0, 1024(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm8, 960(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm24, 896(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm17, 832(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm3, 768(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm26, 704(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm28, 640(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm20, 960(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm18, 896(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm16, 832(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm4, 768(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm15, 704(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm19, 640(%rax)
 ; AVX512DQBW-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQBW-SLOW-NEXT:    vmovaps %zmm0, 576(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm15, 512(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm10, 448(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm21, 384(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm18, 320(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovaps %zmm0, 256(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm16, 192(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm26, 512(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm23, 448(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm17, 384(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm3, 320(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm11, 256(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm10, 192(%rax)
 ; AVX512DQBW-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQBW-SLOW-NEXT:    vmovaps %zmm0, 128(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm20, 64(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm14, (%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm12, 1728(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm5, 1664(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm11, 1600(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm6, 1536(%rax)
-; AVX512DQBW-SLOW-NEXT:    addq $2120, %rsp # imm = 0x848
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm28, 64(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm27, (%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm8, 1728(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm6, 1664(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm2, 1600(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm5, 1536(%rax)
+; AVX512DQBW-SLOW-NEXT:    addq $2184, %rsp # imm = 0x888
 ; AVX512DQBW-SLOW-NEXT:    vzeroupper
 ; AVX512DQBW-SLOW-NEXT:    retq
 ;
 ; AVX512DQBW-FAST-LABEL: store_i64_stride7_vf32:
 ; AVX512DQBW-FAST:       # %bb.0:
-; AVX512DQBW-FAST-NEXT:    subq $2056, %rsp # imm = 0x808
+; AVX512DQBW-FAST-NEXT:    subq $2088, %rsp # imm = 0x828
 ; AVX512DQBW-FAST-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 (%rdi), %zmm10
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 64(%rdi), %zmm15
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 (%rsi), %zmm16
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 64(%rsi), %zmm18
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 64(%rdx), %zmm21
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 (%rdx), %zmm9
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 (%rcx), %zmm20
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 64(%rcx), %zmm17
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 (%r8), %zmm0
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 (%rdi), %zmm8
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 64(%rdi), %zmm9
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 (%rsi), %zmm26
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 64(%rsi), %zmm24
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 64(%rdx), %zmm14
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 (%rdx), %zmm11
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 (%rcx), %zmm22
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 64(%rcx), %zmm28
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 (%r8), %zmm3
 ; AVX512DQBW-FAST-NEXT:    vmovdqa64 (%r9), %zmm6
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 (%rax), %zmm8
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 (%rax), %zmm5
 ; AVX512DQBW-FAST-NEXT:    movb $96, %r10b
 ; AVX512DQBW-FAST-NEXT:    kmovd %r10d, %k1
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm1 = [9,1,9,1,9,1,9,1]
-; AVX512DQBW-FAST-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm2
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm6, %zmm1, %zmm2
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm1, %zmm11
-; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm1 = [4,9,0,3,4,9,0,3]
-; AVX512DQBW-FAST-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm8, %zmm1, %zmm2
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm1, %zmm12
-; AVX512DQBW-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm2 = [11,3,11,3,11,3,11,3]
-; AVX512DQBW-FAST-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm10, %zmm1
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm16, %zmm2, %zmm1
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm2, %zmm14
-; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm3 = [2,10,0,3,2,10,0,3]
-; AVX512DQBW-FAST-NEXT:    # zmm3 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm9, %zmm2
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm20, %zmm3, %zmm2
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm3, %zmm19
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm2, %zmm4
+; AVX512DQBW-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm19 = [15,7,15,7,15,7,15,7]
+; AVX512DQBW-FAST-NEXT:    # zmm19 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [9,1,9,1,9,1,9,1]
+; AVX512DQBW-FAST-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm3, %zmm1
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm6, %zmm0, %zmm1
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm12
+; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [4,9,0,3,4,9,0,3]
+; AVX512DQBW-FAST-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm5, %zmm0, %zmm1
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm13
+; AVX512DQBW-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm31 = [11,3,11,3,11,3,11,3]
+; AVX512DQBW-FAST-NEXT:    # zmm31 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm8, %zmm0
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm26, %zmm31, %zmm0
+; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [2,10,0,3,2,10,0,3]
+; AVX512DQBW-FAST-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm11, %zmm1
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm22, %zmm2, %zmm1
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm2, %zmm18
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm1, %zmm10
 ; AVX512DQBW-FAST-NEXT:    vmovdqa (%r9), %ymm1
-; AVX512DQBW-FAST-NEXT:    vmovdqa 64(%r9), %ymm5
-; AVX512DQBW-FAST-NEXT:    vmovdqa 128(%r9), %ymm3
+; AVX512DQBW-FAST-NEXT:    vmovdqa 64(%r9), %ymm0
+; AVX512DQBW-FAST-NEXT:    vmovdqa 128(%r9), %ymm4
 ; AVX512DQBW-FAST-NEXT:    vmovdqa (%r8), %ymm7
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 64(%r8), %ymm23
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 128(%r8), %ymm22
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 64(%r8), %ymm16
+; AVX512DQBW-FAST-NEXT:    vmovdqa 128(%r8), %ymm15
 ; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm2 = ymm7[0],ymm1[0],ymm7[2],ymm1[2]
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %ymm7, %ymm17
 ; AVX512DQBW-FAST-NEXT:    movb $28, %r10b
 ; AVX512DQBW-FAST-NEXT:    kmovd %r10d, %k2
-; AVX512DQBW-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm4 {%k2} = zmm2[2,3,2,3],zmm8[2,3,2,3]
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa {{.*#+}} ymm4 = [1,3,7,7]
-; AVX512DQBW-FAST-NEXT:    vpermt2q %ymm1, %ymm4, %ymm7
-; AVX512DQBW-FAST-NEXT:    vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [4,12,0,5,4,12,0,5]
-; AVX512DQBW-FAST-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm1
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm6, %zmm2, %zmm1
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm2, %zmm13
-; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm7 = [0,1,12,7,0,1,12,7]
-; AVX512DQBW-FAST-NEXT:    # zmm7 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm8, %zmm7, %zmm1
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm24 = [5,0,14,6,5,0,14,6]
-; AVX512DQBW-FAST-NEXT:    # zmm24 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm5, %zmm7
+; AVX512DQBW-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm10 {%k2} = zmm2[2,3,2,3],zmm5[2,3,2,3]
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa {{.*#+}} ymm5 = [1,3,7,7]
+; AVX512DQBW-FAST-NEXT:    vpermt2q %ymm1, %ymm5, %ymm17
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %ymm17, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm10 = [4,12,0,5,4,12,0,5]
+; AVX512DQBW-FAST-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm3, %zmm2
 ; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm6, %zmm1
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm0, %zmm24, %zmm1
-; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [0,13,6,7,0,13,6,7]
-; AVX512DQBW-FAST-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm8, %zmm2, %zmm1
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm6, %zmm10, %zmm2
+; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm6 = [0,1,12,7,0,1,12,7]
+; AVX512DQBW-FAST-NEXT:    # zmm6 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm7, %zmm6, %zmm2
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm17 = [5,0,14,6,5,0,14,6]
+; AVX512DQBW-FAST-NEXT:    # zmm17 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm1, %zmm20
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm1, %zmm2
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm26 = [15,7,15,7,15,7,15,7]
-; AVX512DQBW-FAST-NEXT:    # zmm26 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm6, %zmm26, %zmm0
-; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm1 = [6,13,14,7,6,13,14,7]
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm3, %zmm17, %zmm20
+; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm1 = [0,13,6,7,0,13,6,7]
 ; AVX512DQBW-FAST-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm8
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm23[0],ymm5[0],ymm23[2],ymm5[2]
-; AVX512DQBW-FAST-NEXT:    vpermt2q %ymm5, %ymm4, %ymm23
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %ymm23, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm8 = ymm22[0],ymm3[0],ymm22[2],ymm3[2]
-; AVX512DQBW-FAST-NEXT:    vpermt2q %ymm3, %ymm4, %ymm22
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %ymm22, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm15, %zmm23
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm15, %zmm3
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm7, %zmm1, %zmm20
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm20, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm2, %zmm19, %zmm3
+; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [6,13,14,7,6,13,14,7]
+; AVX512DQBW-FAST-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm3, %zmm2, %zmm7
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm3 = ymm16[0],ymm0[0],ymm16[2],ymm0[2]
+; AVX512DQBW-FAST-NEXT:    vpermt2q %ymm0, %ymm5, %ymm16
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %ymm16, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm15[0],ymm4[0],ymm15[2],ymm4[2]
+; AVX512DQBW-FAST-NEXT:    vpermt2q %ymm4, %ymm5, %ymm15
+; AVX512DQBW-FAST-NEXT:    vmovdqu %ymm15, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm9, %zmm4
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm24, %zmm31, %zmm4
 ; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm14, %zmm5
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm18, %zmm14, %zmm3
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm21, %zmm4
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm17, %zmm19, %zmm4
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm3, %zmm4 {%k1}
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 64(%rax), %zmm14
-; AVX512DQBW-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm4 {%k2} = zmm0[2,3,2,3],zmm14[2,3,2,3]
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 64(%r8), %zmm0
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 64(%r9), %zmm22
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm22, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm4
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm22, %zmm11, %zmm4
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm12, %zmm3
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm28, %zmm18, %zmm5
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm4, %zmm5 {%k1}
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 64(%rax), %zmm7
+; AVX512DQBW-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm5 {%k2} = zmm3[2,3,2,3],zmm7[2,3,2,3]
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 64(%r8), %zmm3
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 64(%r9), %zmm5
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm3, %zmm4
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm14, %zmm12, %zmm4
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm5, %zmm12, %zmm4
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm7, %zmm13, %zmm4
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm6
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm13, %zmm4
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm22, %zmm13, %zmm6
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm14, %zmm7, %zmm6
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm22, %zmm6
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm0, %zmm24, %zmm6
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm14, %zmm2, %zmm6
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm22, %zmm26, %zmm0
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm14
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm3, %zmm4
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm5, %zmm10, %zmm4
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm7, %zmm6, %zmm4
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm5, %zmm4
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm3, %zmm17, %zmm4
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm7, %zmm1, %zmm4
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm5, %zmm19, %zmm3
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm3, %zmm2, %zmm7
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQBW-FAST-NEXT:    vmovdqa64 128(%rdi), %zmm27
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 128(%rsi), %zmm6
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm27, %zmm0
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm6, %zmm5, %zmm0
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 128(%rdx), %zmm13
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 128(%rcx), %zmm5
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm13, %zmm22
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm5, %zmm19, %zmm22
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm19, %zmm28
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm22 {%k1}
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 128(%rax), %zmm12
-; AVX512DQBW-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm22 {%k2} = zmm8[2,3,2,3],zmm12[2,3,2,3]
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm22, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 128(%r8), %zmm0
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 128(%r9), %zmm8
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm8, %zmm0, %zmm4
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm12, %zmm7, %zmm4
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 128(%rsi), %zmm20
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm27, %zmm3
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm20, %zmm31, %zmm3
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 128(%rdx), %zmm29
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 128(%rcx), %zmm23
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm29, %zmm4
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm23, %zmm18, %zmm4
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm3, %zmm4 {%k1}
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 128(%rax), %zmm30
+; AVX512DQBW-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm4 {%k2} = zmm0[2,3,2,3],zmm30[2,3,2,3]
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm0, %zmm8, %zmm24
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm12, %zmm2, %zmm24
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm24, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm2
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm8, %zmm11, %zmm2
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm12, %zmm3, %zmm2
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 128(%r8), %zmm0
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 128(%r9), %zmm3
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm3, %zmm0, %zmm10
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm30, %zmm6, %zmm10
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm0, %zmm3, %zmm17
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm30, %zmm1, %zmm17
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm1
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm3, %zmm12, %zmm1
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm30, %zmm13, %zmm1
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm3, %zmm19, %zmm0
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm0, %zmm2, %zmm30
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 192(%rdx), %zmm10
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 192(%rcx), %zmm1
+; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm25 = [3,0,12,4,3,0,12,4]
+; AVX512DQBW-FAST-NEXT:    # zmm25 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm1, %zmm4
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm10, %zmm25, %zmm4
+; AVX512DQBW-FAST-NEXT:    movb $48, %r10b
+; AVX512DQBW-FAST-NEXT:    kmovd %r10d, %k3
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 192(%rdi), %zmm18
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 192(%rsi), %zmm7
+; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm4 {%k3} = zmm18[0],zmm7[0],zmm18[2],zmm7[2],zmm18[4],zmm7[4],zmm18[6],zmm7[6]
+; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm16 = [0,8,0,1,0,8,0,1]
+; AVX512DQBW-FAST-NEXT:    # zmm16 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm8, %zmm0
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm26, %zmm16, %zmm0
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm8, %zmm26, %zmm0
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm12
-; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm15 = [3,0,12,4,3,0,12,4]
+; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm15 = [1,0,10,2,1,0,10,2]
 ; AVX512DQBW-FAST-NEXT:    # zmm15 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm20, %zmm1
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm26, %zmm0
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm8, %zmm15, %zmm0
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm9, %zmm0
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm9, %zmm15, %zmm1
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm1, %zmm3
-; AVX512DQBW-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm1 = [13,5,13,5,13,5,13,5]
-; AVX512DQBW-FAST-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm9, %zmm2
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm20, %zmm1, %zmm2
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm9 = [6,14,6,14,6,14,6,14]
-; AVX512DQBW-FAST-NEXT:    # zmm9 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm2
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm20, %zmm9, %zmm2
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm20, %zmm26, %zmm0
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm24, %zmm16, %zmm0
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    movb $48, %r10b
-; AVX512DQBW-FAST-NEXT:    kmovd %r10d, %k3
-; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm11 = [0,8,0,1,0,8,0,1]
-; AVX512DQBW-FAST-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm10, %zmm29
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm16, %zmm11, %zmm29
-; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm8 = [1,0,10,2,1,0,10,2]
-; AVX512DQBW-FAST-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm16, %zmm31
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm10, %zmm8, %zmm31
-; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm3 {%k3} = zmm10[0],zmm16[0],zmm10[2],zmm16[2],zmm10[4],zmm16[4],zmm10[6],zmm16[6]
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm3, (%rsp) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm10, %zmm7
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm16, %zmm1, %zmm7
-; AVX512DQBW-FAST-NEXT:    vbroadcasti128 {{.*#+}} ymm4 = [6,14,6,14]
-; AVX512DQBW-FAST-NEXT:    # ymm4 = mem[0,1,0,1]
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm10, %zmm24
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm16, %zmm4, %zmm24
-; AVX512DQBW-FAST-NEXT:    vbroadcasti128 {{.*#+}} ymm2 = [15,7,15,7]
-; AVX512DQBW-FAST-NEXT:    # ymm2 = mem[0,1,0,1]
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm16, %zmm2, %zmm10
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm17, %zmm22
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm21, %zmm15, %zmm22
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm21, %zmm25
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm17, %zmm1, %zmm25
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm21, %zmm30
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm17, %zmm9, %zmm30
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm17, %zmm26, %zmm21
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm23, %zmm0
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm23, %zmm19
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm18, %zmm11, %zmm19
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm18, %zmm17
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm23, %zmm8, %zmm17
-; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm22 {%k3} = zmm23[0],zmm18[0],zmm23[2],zmm18[2],zmm23[4],zmm18[4],zmm23[6],zmm18[6]
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm23, %zmm3
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm18, %zmm1, %zmm3
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm18, %zmm4, %zmm23
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm18, %zmm2, %zmm0
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm24, %zmm0
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm9, %zmm15, %zmm0
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 192(%rdx), %zmm18
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 192(%rcx), %zmm0
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm18, %zmm10
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm0, %zmm9, %zmm10
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm16
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm18, %zmm15, %zmm16
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm18, %zmm10
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm10
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm0, %zmm18, %zmm28
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm28, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm0, %zmm26, %zmm18
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm13, %zmm5, %zmm15
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm5, %zmm13, %zmm9
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm13, %zmm28
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm5, %zmm26, %zmm13
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm5, %zmm1, %zmm28
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 192(%rdi), %zmm20
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 192(%rsi), %zmm0
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm20, %zmm21
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm0, %zmm4, %zmm21
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm20, %zmm10
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm10
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm27, %zmm26
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm6, %zmm11, %zmm26
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm6, %zmm5
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm27, %zmm8, %zmm5
-; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm15 {%k3} = zmm27[0],zmm6[0],zmm27[2],zmm6[2],zmm27[4],zmm6[4],zmm27[6],zmm6[6]
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm6, %zmm27, %zmm1
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm6, %zmm27, %zmm4
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm6, %zmm2, %zmm27
-; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm16 {%k3} = zmm20[0],zmm0[0],zmm20[2],zmm0[2],zmm20[4],zmm0[4],zmm20[6],zmm0[6]
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm0, %zmm20, %zmm11
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm20, %zmm0, %zmm8
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm0, %zmm20, %zmm6
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm0, %zmm2, %zmm20
-; AVX512DQBW-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX512DQBW-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],mem[6,7]
-; AVX512DQBW-FAST-NEXT:    movb $14, %sil
-; AVX512DQBW-FAST-NEXT:    kmovd %esi, %k3
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 (%rsp), %zmm2 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm2 {%k3}
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm2, (%rsp) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm7 {%k1}
-; AVX512DQBW-FAST-NEXT:    movb $120, %sil
-; AVX512DQBW-FAST-NEXT:    kmovd %esi, %k4
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm7, %zmm0 {%k4}
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm27, %zmm0
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm20, %zmm16, %zmm0
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm24, %zmm2 # 64-byte Folded Reload
-; AVX512DQBW-FAST-NEXT:    # zmm2 = zmm24[0,1,2,3],mem[4,5,6,7]
-; AVX512DQBW-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX512DQBW-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],mem[6,7]
-; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm22 {%k3}
-; AVX512DQBW-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX512DQBW-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],mem[6,7]
-; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm15 {%k3}
-; AVX512DQBW-FAST-NEXT:    movb $-61, %sil
-; AVX512DQBW-FAST-NEXT:    kmovd %esi, %k5
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k5}
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm20, %zmm0
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm27, %zmm15, %zmm0
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm7, %zmm18, %zmm16
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm18, %zmm7, %zmm15
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm18, %zmm17
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm18, %zmm6
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm18, %zmm21
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm7, %zmm31, %zmm18
+; AVX512DQBW-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [6,14,6,14,6,14,6,14]
+; AVX512DQBW-FAST-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm7, %zmm0, %zmm17
+; AVX512DQBW-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm13 = [13,5,13,5,13,5,13,5]
+; AVX512DQBW-FAST-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm7, %zmm13, %zmm6
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm7, %zmm19, %zmm21
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm22, %zmm3
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm22, %zmm31
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm11, %zmm2
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm11, %zmm25, %zmm31
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm22, %zmm13, %zmm11
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm2, %zmm22
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm3, %zmm0, %zmm22
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm3, %zmm19, %zmm2
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm25, %zmm3 {%k1}
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm3, %zmm7 {%k4}
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm28, %zmm1 {%k1}
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm24 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm1, %zmm24 {%k4}
+; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm31 {%k3} = zmm8[0],zmm26[0],zmm8[2],zmm26[2],zmm8[4],zmm26[4],zmm8[6],zmm26[6]
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm8, %zmm3
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm26, %zmm13, %zmm3
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm8, %zmm12
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm26, %zmm0, %zmm12
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm26, %zmm19, %zmm8
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm28, %zmm26
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm14, %zmm2
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm14, %zmm25, %zmm26
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm14, %zmm7
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm28, %zmm13, %zmm7
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm28, %zmm0, %zmm14
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm28, %zmm19, %zmm2
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm26 {%k3} = zmm9[0],zmm24[0],zmm9[2],zmm24[2],zmm9[4],zmm24[4],zmm9[6],zmm24[6]
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm9, %zmm2
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm24, %zmm13, %zmm2
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm9, %zmm5
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm24, %zmm0, %zmm5
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm24, %zmm19, %zmm9
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm29, %zmm23, %zmm25
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm29, %zmm28
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm23, %zmm13, %zmm28
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm29, %zmm24
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm23, %zmm0, %zmm24
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm23, %zmm19, %zmm29
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm10, %zmm23
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm1, %zmm0, %zmm23
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm10, %zmm8
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm1, %zmm13, %zmm8
+; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm25 {%k3} = zmm27[0],zmm20[0],zmm27[2],zmm20[2],zmm27[4],zmm20[4],zmm27[6],zmm20[6]
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm20, %zmm27, %zmm13
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm20, %zmm27, %zmm0
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm20, %zmm19, %zmm27
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm10, %zmm9
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm1, %zmm19, %zmm9
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm1, %zmm19, %zmm10
+; AVX512DQBW-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm23 = zmm17[0,1,2,3],zmm23[4,5,6,7]
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 192(%r8), %zmm1
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm17 = <0,11,u,u,4,5,6,7>
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm1, %zmm4, %zmm17
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 192(%r9), %zmm4
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm20 = <0,1,11,u,4,5,6,7>
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm4, %zmm17, %zmm20
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm8, %zmm6 {%k1}
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm8 = <12,u,u,3,4,5,6,13>
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm1, %zmm6, %zmm8
 ; AVX512DQBW-FAST-NEXT:    movb $24, %sil
 ; AVX512DQBW-FAST-NEXT:    kmovd %esi, %k3
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm6 {%k3}
-; AVX512DQBW-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm1 = zmm23[0,1,2,3],zmm30[4,5,6,7]
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k5}
-; AVX512DQBW-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm4[0,1,2,3],zmm9[4,5,6,7]
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k5}
-; AVX512DQBW-FAST-NEXT:    movb $-31, %sil
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm9, %zmm21 {%k3}
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm6 = <u,1,2,3,4,15,u,u>
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm1, %zmm21, %zmm6
+; AVX512DQBW-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
+; AVX512DQBW-FAST-NEXT:    vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3,4,5],mem[6,7]
+; AVX512DQBW-FAST-NEXT:    movb $14, %sil
 ; AVX512DQBW-FAST-NEXT:    kmovd %esi, %k4
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm6 {%k4}
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm6, %zmm4
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm28 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm28 {%k3}
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm14, %zmm28 {%k4}
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm13, %zmm27 {%k3}
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm12, %zmm27 {%k4}
-; AVX512DQBW-FAST-NEXT:    vmovdqa (%rdx), %xmm0
-; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
-; AVX512DQBW-FAST-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm9, %zmm0, %zmm31 {%k4}
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm11, %zmm3 {%k1}
+; AVX512DQBW-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
+; AVX512DQBW-FAST-NEXT:    vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3,4,5],mem[6,7]
+; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm9, %zmm0, %zmm26 {%k4}
+; AVX512DQBW-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
+; AVX512DQBW-FAST-NEXT:    vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3,4,5],mem[6,7]
+; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm9, %zmm0, %zmm25 {%k4}
+; AVX512DQBW-FAST-NEXT:    movb $120, %sil
+; AVX512DQBW-FAST-NEXT:    kmovd %esi, %k4
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm3, %zmm9 {%k4}
+; AVX512DQBW-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm21 = zmm12[0,1,2,3],zmm22[4,5,6,7]
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm7, %zmm2 {%k1}
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm2, %zmm7 {%k4}
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm28, %zmm13 {%k1}
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm13, %zmm11 {%k4}
+; AVX512DQBW-FAST-NEXT:    movb $-61, %sil
+; AVX512DQBW-FAST-NEXT:    kmovd %esi, %k4
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm2, %zmm21 {%k4}
+; AVX512DQBW-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm13 = zmm5[0,1,2,3],zmm14[4,5,6,7]
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm2, %zmm13 {%k4}
+; AVX512DQBW-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm24[4,5,6,7]
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm2, %zmm0 {%k4}
+; AVX512DQBW-FAST-NEXT:    vmovdqa (%rdx), %xmm2
+; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],mem[0]
+; AVX512DQBW-FAST-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
 ; AVX512DQBW-FAST-NEXT:    movb $12, %sil
 ; AVX512DQBW-FAST-NEXT:    kmovd %esi, %k4
-; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm29 {%k4}
-; AVX512DQBW-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [0,8,0,8,0,8,0,8]
-; AVX512DQBW-FAST-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm6 # 64-byte Folded Reload
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm2, %zmm0, %zmm12 {%k4}
+; AVX512DQBW-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm2 = [0,8,0,8,0,8,0,8]
+; AVX512DQBW-FAST-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm3 # 64-byte Folded Reload
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm28 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm5, %zmm28 {%k3}
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm5, %zmm14 {%k3}
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm29, %zmm27 {%k3}
+; AVX512DQBW-FAST-NEXT:    movb $-31, %sil
+; AVX512DQBW-FAST-NEXT:    kmovd %esi, %k3
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm5, %zmm28 {%k3}
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm5, %zmm14 {%k3}
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm14, %zmm29
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm30, %zmm27 {%k3}
+; AVX512DQBW-FAST-NEXT:    movb $112, %sil
+; AVX512DQBW-FAST-NEXT:    kmovd %esi, %k3
+; AVX512DQBW-FAST-NEXT:    vinserti64x2 $3, (%rax), %zmm3, %zmm12 {%k3}
 ; AVX512DQBW-FAST-NEXT:    vmovdqa 64(%rdx), %xmm3
 ; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],mem[0]
 ; AVX512DQBW-FAST-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm19 {%k4}
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm14 {%k4}
 ; AVX512DQBW-FAST-NEXT:    vmovdqa 128(%rdx), %xmm3
 ; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],mem[0]
 ; AVX512DQBW-FAST-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm26 {%k4}
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm19 {%k4}
 ; AVX512DQBW-FAST-NEXT:    vmovdqa 192(%rdx), %xmm3
 ; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],mem[0]
 ; AVX512DQBW-FAST-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
-; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm11 {%k4}
-; AVX512DQBW-FAST-NEXT:    movb $112, %sil
+; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm16 {%k4}
+; AVX512DQBW-FAST-NEXT:    vpbroadcastq 8(%rcx), %ymm3
+; AVX512DQBW-FAST-NEXT:    vpblendd {{.*#+}} ymm3 = mem[0,1,2,3],ymm3[4,5,6,7]
+; AVX512DQBW-FAST-NEXT:    movb $6, %sil
 ; AVX512DQBW-FAST-NEXT:    kmovd %esi, %k4
-; AVX512DQBW-FAST-NEXT:    vinserti64x2 $3, (%rax), %zmm6, %zmm29 {%k4}
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm5 {%k4}
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm3 # 64-byte Folded Reload
-; AVX512DQBW-FAST-NEXT:    vinserti64x2 $3, 64(%rax), %zmm3, %zmm19 {%k4}
+; AVX512DQBW-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm3 # 64-byte Folded Reload
+; AVX512DQBW-FAST-NEXT:    vinserti64x2 $3, 64(%rax), %zmm3, %zmm14 {%k3}
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm14, %zmm17
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm3 # 64-byte Folded Reload
-; AVX512DQBW-FAST-NEXT:    vinserti64x2 $3, 128(%rax), %zmm3, %zmm26 {%k4}
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 192(%r8), %zmm3
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 192(%r9), %zmm6
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm6, %zmm3, %zmm0
-; AVX512DQBW-FAST-NEXT:    vinserti64x2 $3, 192(%rax), %zmm0, %zmm11 {%k4}
-; AVX512DQBW-FAST-NEXT:    vpbroadcastq 8(%rcx), %ymm0
-; AVX512DQBW-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
-; AVX512DQBW-FAST-NEXT:    movb $6, %sil
-; AVX512DQBW-FAST-NEXT:    kmovd %esi, %k4
-; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm31 {%k4}
-; AVX512DQBW-FAST-NEXT:    vpbroadcastq 72(%rcx), %ymm0
-; AVX512DQBW-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
-; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm17 {%k4}
-; AVX512DQBW-FAST-NEXT:    vpbroadcastq 136(%rcx), %ymm0
-; AVX512DQBW-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
-; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm5 {%k4}
-; AVX512DQBW-FAST-NEXT:    vpbroadcastq 200(%rcx), %ymm0
-; AVX512DQBW-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
-; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm8 {%k4}
-; AVX512DQBW-FAST-NEXT:    movb $56, %cl
+; AVX512DQBW-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm3 # 64-byte Folded Reload
+; AVX512DQBW-FAST-NEXT:    vinserti64x2 $3, 128(%rax), %zmm3, %zmm19 {%k3}
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm19, %zmm22
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm4, %zmm1, %zmm2
+; AVX512DQBW-FAST-NEXT:    vinserti64x2 $3, 192(%rax), %zmm2, %zmm16 {%k3}
+; AVX512DQBW-FAST-NEXT:    movb $56, %sil
+; AVX512DQBW-FAST-NEXT:    kmovd %esi, %k3
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm2, %zmm5 {%k3}
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm5, %zmm14
+; AVX512DQBW-FAST-NEXT:    vpbroadcastq 72(%rcx), %ymm2
+; AVX512DQBW-FAST-NEXT:    vpblendd {{.*#+}} ymm2 = mem[0,1,2,3],ymm2[4,5,6,7]
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm2, %zmm0, %zmm3 {%k4}
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k3}
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm3, %zmm19
+; AVX512DQBW-FAST-NEXT:    vpbroadcastq 136(%rcx), %ymm2
+; AVX512DQBW-FAST-NEXT:    vpblendd {{.*#+}} ymm2 = mem[0,1,2,3],ymm2[4,5,6,7]
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm2, %zmm0, %zmm3 {%k4}
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k3}
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm3, %zmm24
+; AVX512DQBW-FAST-NEXT:    vpbroadcastq 200(%rcx), %ymm2
+; AVX512DQBW-FAST-NEXT:    vpblendd {{.*#+}} ymm2 = mem[0,1,2,3],ymm2[4,5,6,7]
+; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm2, %zmm0, %zmm15 {%k4}
+; AVX512DQBW-FAST-NEXT:    movb $64, %cl
 ; AVX512DQBW-FAST-NEXT:    kmovd %ecx, %k4
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm31 {%k4}
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm17 {%k4}
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm5 {%k4}
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm6, %zmm3, %zmm9
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 192(%rax), %zmm0
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm0, %zmm12, %zmm9
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm9, %zmm8 {%k4}
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm9, %zmm10 {%k1}
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm9, %zmm14 {%k1}
-; AVX512DQBW-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm21, %zmm9 # 64-byte Folded Reload
-; AVX512DQBW-FAST-NEXT:    # zmm9 = zmm21[0,1,2,3],mem[4,5,6,7]
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm12 = <0,11,u,u,4,5,6,7>
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm3, %zmm16, %zmm12
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm13 = <0,1,11,u,4,5,6,7>
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm6, %zmm12, %zmm13
-; AVX512DQBW-FAST-NEXT:    vmovdqa 192(%r8), %ymm12
-; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm12 = ymm12[0],mem[0],ymm12[2],mem[2]
-; AVX512DQBW-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm14 {%k2} = zmm12[2,3,2,3],zmm0[2,3,2,3]
-; AVX512DQBW-FAST-NEXT:    movb $64, %al
-; AVX512DQBW-FAST-NEXT:    kmovd %eax, %k1
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm3, %zmm9 {%k1}
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm1, %zmm23 {%k4}
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm4, %zmm2, %zmm1
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 192(%rax), %zmm2
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm2, %zmm3, %zmm1
 ; AVX512DQBW-FAST-NEXT:    movb $8, %al
-; AVX512DQBW-FAST-NEXT:    kmovd %eax, %k1
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm13 {%k1}
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm18, %zmm20 {%k3}
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm12 = <12,u,u,3,4,5,6,13>
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm3, %zmm10, %zmm12
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm10 = <u,1,2,3,4,15,u,u>
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm3, %zmm20, %zmm10
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm3 = <0,12,u,3,4,5,6,7>
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm6, %zmm12, %zmm3
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm12 = <13,u,2,3,4,5,6,14>
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm6, %zmm9, %zmm12
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm9 = <u,1,2,3,4,5,15,u>
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm6, %zmm10, %zmm9
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm6 = [0,1,12,3,4,5,6,7]
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm0, %zmm3, %zmm6
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [0,13,2,3,4,5,6,7]
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm0, %zmm12, %zmm3
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm10 = [14,1,2,3,4,5,6,15]
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm0, %zmm9, %zmm10
+; AVX512DQBW-FAST-NEXT:    kmovd %eax, %k4
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm2, %zmm20 {%k4}
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm1, %zmm15 {%k3}
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm18, %zmm10 {%k1}
+; AVX512DQBW-FAST-NEXT:    vmovdqa 192(%r8), %ymm1
+; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm1 = ymm1[0],mem[0],ymm1[2],mem[2]
+; AVX512DQBW-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm10 {%k2} = zmm1[2,3,2,3],zmm2[2,3,2,3]
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm1 = <0,12,u,3,4,5,6,7>
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm4, %zmm8, %zmm1
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm3 = <13,u,2,3,4,5,6,14>
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm4, %zmm23, %zmm3
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm5 = <u,1,2,3,4,5,15,u>
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm4, %zmm6, %zmm5
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [0,1,12,3,4,5,6,7]
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm2, %zmm1, %zmm4
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,13,2,3,4,5,6,7]
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm2, %zmm3, %zmm1
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [14,1,2,3,4,5,6,15]
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm2, %zmm5, %zmm3
 ; AVX512DQBW-FAST-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm14, 1472(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm8, 1408(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm11, 1344(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm10, 1472(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm15, 1408(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm16, 1344(%rax)
 ; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm27, 1280(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm2, 1216(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm24, 1152(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm15, 1088(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, 1216(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm11, 1152(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm25, 1088(%rax)
 ; AVX512DQBW-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQBW-FAST-NEXT:    vmovaps %zmm0, 1024(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm5, 960(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm26, 896(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm28, 832(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm1, 768(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm24, 960(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm22, 896(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm29, 832(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm13, 768(%rax)
 ; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm7, 704(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm22, 640(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm26, 640(%rax)
 ; AVX512DQBW-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQBW-FAST-NEXT:    vmovaps %zmm0, 576(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm17, 512(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm19, 448(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm4, 384(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovaps %zmm0, 320(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovaps %zmm0, 256(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovups (%rsp), %zmm0 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovaps %zmm0, 192(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm19, 512(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm17, 448(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm28, 384(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm21, 320(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm9, 256(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm31, 192(%rax)
 ; AVX512DQBW-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQBW-FAST-NEXT:    vmovaps %zmm0, 128(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm31, 64(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm29, (%rax)
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm10, 1728(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm3, 1664(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm6, 1600(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm13, 1536(%rax)
-; AVX512DQBW-FAST-NEXT:    addq $2056, %rsp # imm = 0x808
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm14, 64(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm12, (%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm3, 1728(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm1, 1664(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm4, 1600(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm20, 1536(%rax)
+; AVX512DQBW-FAST-NEXT:    addq $2088, %rsp # imm = 0x828
 ; AVX512DQBW-FAST-NEXT:    vzeroupper
 ; AVX512DQBW-FAST-NEXT:    retq
   %in.vec0 = load <32 x i64>, ptr %in.vecptr0, align 64
@@ -12615,766 +12547,760 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ;
 ; AVX512F-ONLY-SLOW-LABEL: store_i64_stride7_vf64:
 ; AVX512F-ONLY-SLOW:       # %bb.0:
-; AVX512F-ONLY-SLOW-NEXT:    subq $6600, %rsp # imm = 0x19C8
+; AVX512F-ONLY-SLOW-NEXT:    subq $6248, %rsp # imm = 0x1868
 ; AVX512F-ONLY-SLOW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 (%rdi), %zmm3
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 (%rdi), %zmm7
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 64(%rdi), %zmm19
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 (%rsi), %zmm9
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 64(%rsi), %zmm16
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 64(%rdx), %zmm17
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm17, (%rsp) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 (%rdx), %zmm3
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 64(%rdi), %zmm29
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 (%rsi), %zmm4
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 (%rcx), %zmm4
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 64(%rsi), %zmm13
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm13, (%rsp) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 64(%rdx), %zmm20
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm20, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 (%rdx), %zmm5
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 (%rcx), %zmm8
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 64(%rcx), %zmm18
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm10 = [11,3,11,3,11,3,11,3]
-; AVX512F-ONLY-SLOW-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm28 = [2,10,0,3,2,10,0,3]
-; AVX512F-ONLY-SLOW-NEXT:    # zmm28 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm11 = [11,3,11,3,11,3,11,3]
+; AVX512F-ONLY-SLOW-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm31 = [2,10,0,3,2,10,0,3]
+; AVX512F-ONLY-SLOW-NEXT:    # zmm31 = mem[0,1,2,3,0,1,2,3]
 ; AVX512F-ONLY-SLOW-NEXT:    movb $96, %r10b
 ; AVX512F-ONLY-SLOW-NEXT:    kmovw %r10d, %k1
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 (%r8), %zmm1
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 (%r9), %zmm11
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 (%rax), %zmm6
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 64(%rax), %zmm9
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 (%r9), %zmm15
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 (%rax), %zmm5
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 64(%rax), %zmm6
+; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm25 = [15,7,15,7,15,7,15,7]
+; AVX512F-ONLY-SLOW-NEXT:    # zmm25 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [9,1,9,1,9,1,9,1]
 ; AVX512F-ONLY-SLOW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm11, %zmm0, %zmm2
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm14
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [4,9,0,3,4,9,0,3]
-; AVX512F-ONLY-SLOW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm6, %zmm0, %zmm2
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm12
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm15, %zmm0, %zmm2
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm8
+; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm24 = [4,9,0,3,4,9,0,3]
+; AVX512F-ONLY-SLOW-NEXT:    # zmm24 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm5, %zmm24, %zmm2
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm0
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm4, %zmm10, %zmm0
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm2
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm8, %zmm28, %zmm2
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm9, %zmm11, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm2
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm4, %zmm31, %zmm2
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k1}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa (%r9), %ymm5
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa (%r9), %ymm7
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 64(%r9), %ymm3
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqa (%r8), %ymm0
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 64(%r8), %ymm4
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm5[0],ymm0[2],ymm5[2]
+; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm7[0],ymm0[2],ymm7[2]
 ; AVX512F-ONLY-SLOW-NEXT:    movb $28, %r10b
 ; AVX512F-ONLY-SLOW-NEXT:    kmovw %r10d, %k2
-; AVX512F-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm2 {%k2} = zmm0[2,3,2,3],zmm6[2,3,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm2 {%k2} = zmm0[2,3,2,3],zmm5[2,3,2,3]
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [4,12,0,5,4,12,0,5]
 ; AVX512F-ONLY-SLOW-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3]
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm0
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm5
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm11, %zmm2, %zmm0
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm16
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm15, %zmm2, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm7
 ; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [0,1,12,7,0,1,12,7]
 ; AVX512F-ONLY-SLOW-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm6, %zmm2, %zmm0
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm11
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm5, %zmm2, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm14
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [5,0,14,6,5,0,14,6]
-; AVX512F-ONLY-SLOW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm2
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm17
+; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm13 = [5,0,14,6,5,0,14,6]
+; AVX512F-ONLY-SLOW-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm15, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm15, %zmm9
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm13, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [0,13,6,7,0,13,6,7]
+; AVX512F-ONLY-SLOW-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm5, %zmm2, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm15
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm9, %zmm25, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm10 = [6,13,14,7,6,13,14,7]
+; AVX512F-ONLY-SLOW-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm10, %zmm5
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm0, %zmm2
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm19
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [0,13,6,7,0,13,6,7]
-; AVX512F-ONLY-SLOW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm6, %zmm0, %zmm2
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm15
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm5 = [15,7,15,7,15,7,15,7]
-; AVX512F-ONLY-SLOW-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm17, %zmm5, %zmm1
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm7 = [6,13,14,7,6,13,14,7]
-; AVX512F-ONLY-SLOW-NEXT:    # zmm7 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm7, %zmm6
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm29, %zmm1
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm13, %zmm10, %zmm1
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm20, %zmm2
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm18, %zmm28, %zmm2
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm19, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm16, %zmm11, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm17, %zmm2
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm18, %zmm31, %zmm2
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}
 ; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm1 = ymm4[0],ymm3[0],ymm4[2],ymm3[2]
-; AVX512F-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm2 {%k2} = zmm1[2,3,2,3],zmm9[2,3,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm2 {%k2} = zmm1[2,3,2,3],zmm6[2,3,2,3]
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 64(%r8), %zmm1
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 64(%r9), %zmm3
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm18
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm3, %zmm14, %zmm2
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm12, %zmm4
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm9, %zmm12, %zmm2
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm3, %zmm16, %zmm2
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm9, %zmm11, %zmm2
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm2
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 64(%r9), %zmm16
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm16, %zmm8, %zmm3
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm6, %zmm24, %zmm3
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm19, %zmm2
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm9, %zmm0, %zmm2
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm3, %zmm5, %zmm1
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm7, %zmm9
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 128(%rdi), %zmm23
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 128(%rsi), %zmm24
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm23, %zmm1
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm24, %zmm10, %zmm1
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm10, %zmm30
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 128(%rdx), %zmm25
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 128(%rcx), %zmm12
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm25, %zmm2
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm12, %zmm28, %zmm2
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 128(%rax), %zmm9
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 128(%r9), %ymm6
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm16, %zmm7, %zmm3
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm6, %zmm14, %zmm3
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm16, %zmm3
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm13, %zmm3
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm6, %zmm15, %zmm3
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm29
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm16, %zmm25, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm10, %zmm6
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 128(%rdi), %zmm27
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 128(%rsi), %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm27, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm11, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm17
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 128(%rdx), %zmm21
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 128(%rcx), %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm21, %zmm3
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm31, %zmm3
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm3 {%k1}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 128(%rax), %zmm5
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 128(%r9), %ymm4
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 128(%r8), %ymm1
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm1 = ymm1[0],ymm6[0],ymm1[2],ymm6[2]
-; AVX512F-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm2 {%k2} = zmm1[2,3,2,3],zmm9[2,3,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm1 = ymm1[0],ymm4[0],ymm1[2],ymm4[2]
+; AVX512F-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm3 {%k2} = zmm1[2,3,2,3],zmm5[2,3,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 128(%r8), %zmm1
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 128(%r9), %zmm21
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm21, %zmm14, %zmm2
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm9, %zmm4, %zmm2
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm10
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm16, %zmm3
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm21, %zmm16, %zmm2
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm9, %zmm11, %zmm2
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm21, %zmm2
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm19, %zmm2
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm9, %zmm0, %zmm2
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm21, %zmm5, %zmm1
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm7, %zmm9
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 192(%rdi), %zmm27
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 192(%rsi), %zmm26
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm27, %zmm1
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm26, %zmm30, %zmm1
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 192(%rdx), %zmm17
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 192(%rcx), %zmm14
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm17, %zmm2
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm14, %zmm28, %zmm2
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 192(%rax), %zmm21
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 192(%r9), %ymm9
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 128(%r9), %zmm6
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm6, %zmm8, %zmm3
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm11
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm5, %zmm24, %zmm3
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm6, %zmm7, %zmm3
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm5, %zmm14, %zmm3
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm3
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm13, %zmm3
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm5, %zmm15, %zmm3
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm20
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm6, %zmm25, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm10, %zmm5
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 192(%rdi), %zmm26
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 192(%rsi), %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm26, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm17, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 192(%rdx), %zmm18
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 192(%rcx), %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm18, %zmm3
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm31, %zmm3
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm3 {%k1}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 192(%rax), %zmm2
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 192(%r9), %ymm5
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 192(%r8), %ymm1
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm1 = ymm1[0],ymm9[0],ymm1[2],ymm9[2]
-; AVX512F-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm2 {%k2} = zmm1[2,3,2,3],zmm21[2,3,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm1 = ymm1[0],ymm5[0],ymm1[2],ymm5[2]
+; AVX512F-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm3 {%k2} = zmm1[2,3,2,3],zmm2[2,3,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 192(%r8), %zmm1
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 192(%r9), %zmm0
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm18, %zmm2
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm21, %zmm4, %zmm2
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm16, %zmm2
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm21, %zmm11, %zmm2
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm2
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 192(%r9), %zmm8
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm8, %zmm11, %zmm3
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm2, %zmm24, %zmm3
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm8, %zmm7, %zmm3
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm28
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm2, %zmm14, %zmm3
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm30
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm5
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm13, %zmm5
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm15, %zmm3
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm2, %zmm15, %zmm5
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm9
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm8, %zmm25, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm10, %zmm0
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm19, %zmm2
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm21, %zmm15, %zmm2
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm15, %zmm31
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm5, %zmm1
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm7, %zmm21
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 256(%rdi), %zmm21
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 256(%rdi), %zmm15
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 256(%rsi), %zmm22
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm21, %zmm2
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm30, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm22, %zmm30, %zmm2
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 256(%rdx), %zmm16
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 256(%rcx), %zmm15
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm16, %zmm4
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm15, %zmm28, %zmm4
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm4 {%k1}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm15, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm17, %zmm14
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm22, %zmm17, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 256(%rdx), %zmm12
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 256(%rcx), %zmm23
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm12, %zmm5
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm23, %zmm31, %zmm5
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm5 {%k1}
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 256(%rax), %zmm0
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 256(%r9), %ymm6
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 256(%r8), %ymm2
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm2 = ymm2[0],ymm6[0],ymm2[2],ymm6[2]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm6
-; AVX512F-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm4 {%k2} = zmm2[2,3,2,3],zmm0[2,3,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 256(%r8), %zmm2
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 256(%r9), %zmm13
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm4
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm13, %zmm18, %zmm4
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm10, %zmm4
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm4
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm0
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm13, %zmm3, %zmm4
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm6, %zmm11, %zmm4
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 256(%r9), %ymm7
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 256(%r8), %ymm1
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm1 = ymm1[0],ymm7[0],ymm1[2],ymm7[2]
+; AVX512F-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm5 {%k2} = zmm1[2,3,2,3],zmm0[2,3,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 256(%r8), %zmm5
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 256(%r9), %zmm2
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm17
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm7
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm2, %zmm11, %zmm17
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm24, %zmm17
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm24, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm11
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm2, %zmm28, %zmm11
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm30, %zmm11
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm30, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm11
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm5, %zmm13, %zmm11
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm3, %zmm11
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm13, %zmm4
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm2, %zmm19, %zmm4
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm6, %zmm31, %zmm4
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm31, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm13, %zmm5, %zmm2
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm2, %zmm7, %zmm6
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 320(%rdi), %zmm6
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 320(%rsi), %zmm2
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm4
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm2, %zmm30, %zmm4
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 320(%rdx), %zmm2
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 320(%rcx), %zmm13
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm9
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm13, %zmm28, %zmm9
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm9 {%k1}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 320(%rax), %zmm3
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 320(%r9), %ymm30
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %ymm30, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 320(%r8), %ymm4
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm4 = ymm4[0],ymm30[0],ymm4[2],ymm30[2]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm30
-; AVX512F-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm9 {%k2} = zmm4[2,3,2,3],zmm3[2,3,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 320(%r8), %zmm4
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 320(%r9), %zmm9
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm3
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm9, %zmm18, %zmm3
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm30, %zmm10, %zmm3
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm3
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm9, %zmm0, %zmm3
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm30, %zmm11, %zmm3
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm9, %zmm0
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm4, %zmm19, %zmm0
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm30, %zmm31, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm4
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm2, %zmm25, %zmm5
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm5, %zmm10, %zmm0
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm9, %zmm5, %zmm4
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm4, %zmm7, %zmm30
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm30, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm11 = [13,5,13,5,13,5,13,5]
-; AVX512F-ONLY-SLOW-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm0
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm8, %zmm11, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 320(%rdi), %zmm19
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 320(%rsi), %zmm0
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [6,14,6,14,6,14,6,14]
-; AVX512F-ONLY-SLOW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm3
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm8, %zmm0, %zmm3
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm3
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm0
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm8, %zmm5, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm19, %zmm5
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm14, %zmm5
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 320(%rdx), %zmm11
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 320(%rcx), %zmm17
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm14
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm17, %zmm31, %zmm14
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm14 {%k1}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 320(%rax), %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 320(%r9), %ymm5
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 320(%r8), %ymm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm5 = ymm0[0],ymm5[0],ymm0[2],ymm5[2]
+; AVX512F-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm14 {%k2} = zmm5[2,3,2,3],zmm1[2,3,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 320(%r8), %zmm5
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 320(%r9), %zmm14
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm14, %zmm7, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm24, %zmm0
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [3,0,12,4,3,0,12,4]
-; AVX512F-ONLY-SLOW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm0, %zmm8
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm9
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm4
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm8, %zmm11, %zmm4
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm4
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm8, %zmm3, %zmm4
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm4
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm8, %zmm5, %zmm4
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm0, %zmm8
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm10
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm25, %zmm1
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm12, %zmm11, %zmm1
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm25, %zmm1
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm12, %zmm3, %zmm1
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm25, %zmm1
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm12, %zmm5, %zmm1
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm25, %zmm0, %zmm12
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm12, %zmm30
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm17, %zmm1
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm14, %zmm11, %zmm1
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm17, %zmm1
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm14, %zmm3, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm14, %zmm28, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm30, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm5, %zmm13, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm3, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm3
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm14, %zmm25, %zmm5
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm5, %zmm10, %zmm1
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm17, %zmm1
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm14, %zmm5, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 384(%rdx), %zmm5
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 384(%rcx), %zmm24
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm24, %zmm25, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 384(%rdi), %zmm10
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 384(%rsi), %zmm1
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm17, %zmm0, %zmm14
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm17
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm16, %zmm1
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm15, %zmm11, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm25, %zmm10
+; AVX512F-ONLY-SLOW-NEXT:    movb $24, %r10b
+; AVX512F-ONLY-SLOW-NEXT:    kmovw %r10d, %k3
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm10 {%k3}
+; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm1 = [0,8,0,8,0,8,0,8]
+; AVX512F-ONLY-SLOW-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 # 64-byte Folded Reload
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm16, %zmm1, %zmm29
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm29, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm6, %zmm1, %zmm20
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm20, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm8, %zmm1, %zmm9
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm2, %zmm1, %zmm4
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm14, %zmm1, %zmm3
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 384(%r8), %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm1 = <u,1,2,3,4,15,u,u>
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm1, %zmm10
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 384(%r9), %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm3 = <u,1,2,3,4,5,15,u>
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm3, %zmm10
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm1, %zmm0, %zmm2
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm1, %zmm0, %zmm7
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm1, %zmm0, %zmm28
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm28, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm13, %zmm1
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm16, %zmm1
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm15, %zmm3, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm7 = [13,5,13,5,13,5,13,5]
+; AVX512F-ONLY-SLOW-NEXT:    # zmm7 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm7, %zmm2
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm10 = [6,14,6,14,6,14,6,14]
+; AVX512F-ONLY-SLOW-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm10, %zmm2
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm25, %zmm2
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [3,0,12,4,3,0,12,4]
+; AVX512F-ONLY-SLOW-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm2, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm3
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 (%rsp), %zmm0 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm13, %zmm7, %zmm1
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm16, %zmm1
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm15, %zmm5, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm13, %zmm10, %zmm1
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm16, %zmm0, %zmm15
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm15, %zmm16
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm1
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm13, %zmm11, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm13, %zmm25, %zmm1
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm1
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm4
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm13, %zmm3, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm2, %zmm13
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm13, %zmm14
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm21, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm29 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm29, %zmm7, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm21, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm29, %zmm10, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm21, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm29, %zmm25, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm21, %zmm2, %zmm29
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm18, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm20 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm20, %zmm7, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm18, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm20, %zmm10, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm18, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm20, %zmm25, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm18, %zmm2, %zmm20
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm12, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm23, %zmm7, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm12, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm23, %zmm10, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm12, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm23, %zmm25, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm12, %zmm2, %zmm23
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm23, %zmm21
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm17, %zmm7, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm17, %zmm10, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm17, %zmm25, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm11, %zmm2, %zmm17
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm17, %zmm28
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 448(%rdx), %zmm8
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 448(%rcx), %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm6
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm31, %zmm6
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm24, %zmm11
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm24, %zmm5, %zmm31
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm31, (%rsp) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm24, %zmm7, %zmm1
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm1
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm13, %zmm5, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm24, %zmm10, %zmm1
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm2, %zmm0, %zmm13
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm13, %zmm18
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 448(%rdx), %zmm14
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 448(%rcx), %zmm1
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm12
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm28, %zmm12
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm19
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm3, %zmm19
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 384(%rdx), %zmm3
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 384(%rcx), %zmm20
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm20, %zmm3, %zmm28
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm28, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm2
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm20, %zmm11, %zmm2
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm20, %zmm3, %zmm4
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm25
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm20, %zmm5, %zmm25
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm3, %zmm0, %zmm20
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm7
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm14, %zmm0, %zmm7
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm15
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm5, %zmm14
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm11, %zmm15
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm5, %zmm2, %zmm11
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm9
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm8, %zmm2, %zmm9
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm12
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm10, %zmm12
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm13
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm7, %zmm13
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm25, %zmm8
 ; AVX512F-ONLY-SLOW-NEXT:    movb $48, %r10b
-; AVX512F-ONLY-SLOW-NEXT:    kmovw %r10d, %k3
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm1 = [1,0,10,2,1,0,10,2]
-; AVX512F-ONLY-SLOW-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm2
+; AVX512F-ONLY-SLOW-NEXT:    kmovw %r10d, %k4
+; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm5 = [1,0,10,2,1,0,10,2]
+; AVX512F-ONLY-SLOW-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm1, %zmm2
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm5, %zmm2
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm9 {%k3} = zmm0[0],zmm3[0],zmm0[2],zmm3[2],zmm0[4],zmm3[4],zmm0[6],zmm3[6]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm4
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm5
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm9
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [0,8,0,1,0,8,0,1]
-; AVX512F-ONLY-SLOW-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm3, %zmm2, %zmm0
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm3, %zmm11, %zmm4
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti128 {{.*#+}} ymm13 = [6,14,6,14]
-; AVX512F-ONLY-SLOW-NEXT:    # ymm13 = mem[0,1,0,1]
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm3, %zmm13, %zmm5
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti128 {{.*#+}} ymm8 = [15,7,15,7]
-; AVX512F-ONLY-SLOW-NEXT:    # ymm8 = mem[0,1,0,1]
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm3, %zmm8, %zmm9
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 (%rsp), %zmm0 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm3 {%k4} = zmm0[0],zmm1[0],zmm0[2],zmm1[2],zmm0[4],zmm1[4],zmm0[6],zmm1[6]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm2
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm3
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm29, %zmm1, %zmm3
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm16
+; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [0,8,0,1,0,8,0,1]
+; AVX512F-ONLY-SLOW-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm4, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm7, %zmm2
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm10, %zmm3
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm10 {%k3} = zmm29[0],zmm0[0],zmm29[2],zmm0[2],zmm29[4],zmm0[4],zmm29[6],zmm0[6]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm29, %zmm4
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm29, %zmm3
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm29, %zmm5
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm2, %zmm29
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm29, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm11, %zmm4
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm13, %zmm3
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm8, %zmm5
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm24, %zmm0
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm23, %zmm1, %zmm0
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, (%rsp) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm30 {%k3} = zmm23[0],zmm24[0],zmm23[2],zmm24[2],zmm23[4],zmm24[4],zmm23[6],zmm24[6]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm30, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm23, %zmm3
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm23, %zmm0
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm23, %zmm4
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm24, %zmm2, %zmm23
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm23, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm24, %zmm11, %zmm3
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm24, %zmm13, %zmm0
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm24, %zmm8, %zmm4
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm26, %zmm0
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm27, %zmm1, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm25, %zmm16
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm5, %zmm2
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm14 {%k4} = zmm0[0],zmm1[0],zmm0[2],zmm1[2],zmm0[4],zmm1[4],zmm0[6],zmm1[6]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm3
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm2
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm14
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm4, %zmm0
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm17 {%k3} = zmm27[0],zmm26[0],zmm27[2],zmm26[2],zmm27[4],zmm26[4],zmm27[6],zmm26[6]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm27, %zmm3
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm27, %zmm0
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm27, %zmm31
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm26, %zmm2, %zmm27
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm27, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm26, %zmm11, %zmm3
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm7, %zmm3
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm26, %zmm13, %zmm0
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm26, %zmm8, %zmm31
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm10, %zmm2
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm25, %zmm14
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm27, %zmm5, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm29 {%k4} = zmm27[0],zmm0[0],zmm27[2],zmm0[2],zmm27[4],zmm0[4],zmm27[6],zmm0[6]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm29, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm27, %zmm2
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm27, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm27, %zmm23
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm4, %zmm27
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm27, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm7, %zmm2
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm10, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm25, %zmm23
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm26, %zmm5, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm20 {%k4} = zmm26[0],zmm0[0],zmm26[2],zmm0[2],zmm26[4],zmm0[4],zmm26[6],zmm0[6]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm20, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm26, %zmm29
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm26, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm26, %zmm31
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm4, %zmm26
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm26, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm7, %zmm29
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm10, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm25, %zmm31
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm22, %zmm30
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm21, %zmm1, %zmm30
-; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm16 {%k3} = zmm21[0],zmm22[0],zmm21[2],zmm22[2],zmm21[4],zmm22[4],zmm21[6],zmm22[6]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm21, %zmm3
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm21, %zmm0
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm21, %zmm27
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm21, %zmm29
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm22, %zmm2, %zmm29
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm22, %zmm11, %zmm3
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm22, %zmm13, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm15, %zmm5, %zmm30
+; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm21 {%k4} = zmm15[0],zmm22[0],zmm15[2],zmm22[2],zmm15[4],zmm22[4],zmm15[6],zmm22[6]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm15, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm15, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm15, %zmm26
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm15, %zmm27
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm22, %zmm4, %zmm27
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm22, %zmm7, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm22, %zmm10, %zmm0
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm22, %zmm8, %zmm27
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm26
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm6, %zmm1, %zmm26
-; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm18 {%k3} = zmm6[0],zmm0[0],zmm6[2],zmm0[2],zmm6[4],zmm0[4],zmm6[6],zmm0[6]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm28
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm3
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm23
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm24
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm2, %zmm24
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm11, %zmm28
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm13, %zmm3
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm8, %zmm23
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 448(%rdi), %zmm9
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm22, %zmm25, %zmm26
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 448(%rdi), %zmm15
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 448(%rsi), %zmm0
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm9, %zmm6
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm5, %zmm6
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm9, %zmm3
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm13, %zmm3
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm9, %zmm4
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm11, %zmm4
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 384(%rdi), %zmm21
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 384(%rsi), %zmm17
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm17, %zmm22
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm21, %zmm1, %zmm22
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm17, %zmm21, %zmm5
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm17, %zmm21, %zmm11
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm17, %zmm21, %zmm13
-; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm20 {%k3} = zmm21[0],zmm17[0],zmm21[2],zmm17[2],zmm21[4],zmm17[4],zmm21[6],zmm17[6]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm21, %zmm18
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm17, %zmm2, %zmm21
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm17, %zmm8, %zmm18
-; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm7 {%k3} = zmm9[0],zmm0[0],zmm9[2],zmm0[2],zmm9[4],zmm0[4],zmm9[6],zmm0[6]
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm0, %zmm9, %zmm2
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm9, %zmm0, %zmm1
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm8, %zmm9
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm12 {%k1}
-; AVX512F-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm3 = zmm3[0,1,2,3],zmm19[4,5,6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 448(%r8), %zmm6
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm0 = <0,11,u,u,4,5,6,7>
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm6, %zmm7, %zmm0
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 448(%r9), %zmm7
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm8 = <0,1,11,u,4,5,6,7>
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm7, %zmm0, %zmm8
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm10
-; AVX512F-ONLY-SLOW-NEXT:    movb $4, %sil
-; AVX512F-ONLY-SLOW-NEXT:    kmovw %esi, %k3
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm12 {%k3}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm8 = <0,1,2,10,u,5,6,7>
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm7, %zmm12, %zmm8
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm15, %zmm4 {%k1}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm0 = <12,u,u,3,4,5,6,13>
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm6, %zmm4, %zmm0
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm4 = <0,12,u,3,4,5,6,7>
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm7, %zmm0, %zmm4
-; AVX512F-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [0,8,0,8,0,8,0,8]
-; AVX512F-ONLY-SLOW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm15, %zmm17
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm25, %zmm17
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm19, %zmm22
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm25, %zmm22
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm24
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm19, %zmm5, %zmm24
+; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm28 {%k4} = zmm19[0],zmm1[0],zmm19[2],zmm1[2],zmm19[4],zmm1[4],zmm19[6],zmm1[6]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm28, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm19, %zmm18
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm19, %zmm2
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm19, %zmm21
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm4, %zmm21
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm7, %zmm18
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm10, %zmm2
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm19 # 64-byte Folded Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm12 # 64-byte Folded Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm12 # 64-byte Folded Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm5 # 64-byte Folded Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm12 # 64-byte Folded Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm12 # 64-byte Folded Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    movb $24, %sil
-; AVX512F-ONLY-SLOW-NEXT:    kmovw %esi, %k5
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm25, %zmm18 {%k5}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm19, %zmm20
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm28 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm28, %zmm5, %zmm20
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm15, %zmm3
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm25 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm25, %zmm3
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm19, %zmm28, %zmm25
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm15, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm7, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm19, %zmm28, %zmm7
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm15, %zmm2
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm10, %zmm2
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm19, %zmm28, %zmm10
+; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm11 {%k4} = zmm28[0],zmm19[0],zmm28[2],zmm19[2],zmm28[4],zmm19[4],zmm28[6],zmm19[6]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm19, %zmm4, %zmm28
+; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm9 {%k4} = zmm15[0],zmm0[0],zmm15[2],zmm0[2],zmm15[4],zmm0[4],zmm15[6],zmm0[6]
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm0, %zmm15, %zmm4
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm15, %zmm5, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm2[0,1,2,3],zmm12[4,5,6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm6 {%k1}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 448(%r8), %zmm3
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm5 = <0,11,u,u,4,5,6,7>
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm3, %zmm9, %zmm5
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm13, %zmm1 {%k1}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm9 = <12,u,u,3,4,5,6,13>
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm3, %zmm1, %zmm9
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm17 {%k3}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm1 = <u,1,2,3,4,15,u,u>
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm3, %zmm1, %zmm17
 ; AVX512F-ONLY-SLOW-NEXT:    movb $6, %sil
-; AVX512F-ONLY-SLOW-NEXT:    kmovw %esi, %k3
-; AVX512F-ONLY-SLOW-NEXT:    vpbroadcastq 456(%rcx), %ymm12
-; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm12 = mem[0,1,2,3],ymm12[4,5,6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm12, %zmm0, %zmm1 {%k3}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm12 = <0,1,2,9,u,u,6,7>
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm6, %zmm1, %zmm12
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm9 {%k5}
-; AVX512F-ONLY-SLOW-NEXT:    movb $64, %sil
 ; AVX512F-ONLY-SLOW-NEXT:    kmovw %esi, %k4
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm3 {%k4}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm1 = <u,1,2,3,4,15,u,u>
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm6, %zmm1, %zmm9
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 384(%r8), %zmm6
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm6, %zmm1, %zmm18
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 384(%r9), %zmm1
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm1, %zmm6, %zmm0
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm1, %zmm6, %zmm14
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm1, %zmm6, %zmm15
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm6, %zmm1, %zmm16
+; AVX512F-ONLY-SLOW-NEXT:    vpbroadcastq 456(%rcx), %ymm1
+; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = mem[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm0 {%k4}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm8 = <0,1,2,9,u,u,6,7>
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm3, %zmm0, %zmm8
+; AVX512F-ONLY-SLOW-NEXT:    movb $64, %sil
+; AVX512F-ONLY-SLOW-NEXT:    kmovw %esi, %k5
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm2 {%k5}
+; AVX512F-ONLY-SLOW-NEXT:    movb $4, %sil
+; AVX512F-ONLY-SLOW-NEXT:    kmovw %esi, %k5
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm6 {%k5}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 448(%r9), %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm1 = <0,1,11,u,4,5,6,7>
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm0, %zmm5, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm11
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm3 = <0,1,2,10,u,5,6,7>
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm0, %zmm6, %zmm3
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm5 = <0,12,u,3,4,5,6,7>
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm0, %zmm9, %zmm5
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm6 = <13,u,2,3,4,5,6,14>
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm0, %zmm2, %zmm6
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm1 = <u,1,2,3,4,5,15,u>
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm1, %zmm17
 ; AVX512F-ONLY-SLOW-NEXT:    movb $12, %sil
-; AVX512F-ONLY-SLOW-NEXT:    kmovw %esi, %k4
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 448(%rdx), %xmm6
-; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm6 = xmm6[0],mem[0]
-; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
-; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm6, %zmm0, %zmm2 {%k4}
-; AVX512F-ONLY-SLOW-NEXT:    vinserti32x4 $2, 448(%r8), %zmm2, %zmm2
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm6 = <0,1,2,3,4,8,u,7>
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm7, %zmm2, %zmm6
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm2 = <u,1,2,3,4,5,15,u>
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm2, %zmm18
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm1 = <0,1,2,3,9,u,6,7>
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm7, %zmm12, %zmm1
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm12 = <13,u,2,3,4,5,6,14>
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm7, %zmm3, %zmm12
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm7, %zmm2, %zmm9
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 448(%rax), %zmm2
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [0,1,2,3,10,5,6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm2, %zmm8, %zmm3
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    kmovw %esi, %k5
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 448(%rdx), %xmm2
+; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],mem[0]
+; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm2, %zmm0, %zmm4 {%k5}
+; AVX512F-ONLY-SLOW-NEXT:    vinserti32x4 $2, 448(%r8), %zmm4, %zmm2
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm4 = <0,1,2,3,4,8,u,7>
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm0, %zmm2, %zmm4
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm2 = <0,1,2,3,9,u,6,7>
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm0, %zmm8, %zmm2
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 384(%rax), %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm1, %zmm9
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 (%rsp), %zmm1 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm25, %zmm1 {%k1}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 384(%r9), %ymm12
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 384(%r8), %ymm8
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm8 = ymm8[0],ymm12[0],ymm8[2],ymm12[2]
+; AVX512F-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm1 {%k2} = zmm8[2,3,2,3],zmm0[2,3,2,3]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, (%rsp) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm8, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm8, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm8 = [14,1,2,3,4,5,6,15]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm8, %zmm12
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 448(%rax), %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm12 = [0,1,2,3,10,5,6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm0, %zmm3, %zmm12
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [0,1,12,3,4,5,6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm2, %zmm4, %zmm3
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm0, %zmm5, %zmm3
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [0,13,2,3,4,5,6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm0, %zmm6, %zmm3
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm8, %zmm17
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [0,1,2,3,4,5,8,7]
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm0, %zmm4, %zmm3
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 384(%rax), %zmm3
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm3, %zmm4, %zmm14
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm7
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm8 {%k1}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 384(%r9), %ymm14
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu %ymm14, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 384(%r8), %ymm4
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm4 = ymm4[0],ymm14[0],ymm4[2],ymm14[2]
-; AVX512F-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm8 {%k2} = zmm4[2,3,2,3],zmm3[2,3,2,3]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm3, %zmm4, %zmm15
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm15, %zmm8
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm3, %zmm4, %zmm16
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [0,1,2,3,4,5,8,7]
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm2, %zmm6, %zmm4
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [14,1,2,3,4,5,6,15]
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm3, %zmm4, %zmm18
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [0,1,2,3,4,9,6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm2, %zmm1, %zmm3
+; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm0, %zmm2, %zmm3
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,13,2,3,4,5,6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vpermi2q %zmm2, %zmm12, %zmm1
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vpermt2q %zmm2, %zmm4, %zmm9
 ; AVX512F-ONLY-SLOW-NEXT:    movb $8, %sil
 ; AVX512F-ONLY-SLOW-NEXT:    kmovw %esi, %k2
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm10 {%k2}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k5}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm3 {%k5}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k5}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm31 {%k5}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm27 {%k5}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm23 {%k5}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm11 {%k2}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm16 {%k3}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm14 {%k3}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm23 {%k3}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm31 {%k3}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm26 {%k3}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm22 {%k3}
 ; AVX512F-ONLY-SLOW-NEXT:    movb $-31, %sil
 ; AVX512F-ONLY-SLOW-NEXT:    kmovw %esi, %k2
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k2}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm3 {%k2}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k2}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm31 {%k2}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm31, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm27 {%k2}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm23 {%k2}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa (%rdx), %xmm1
-; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
-; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm16 {%k2}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm14 {%k2}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm23 {%k2}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm23, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm31 {%k2}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm26 {%k2}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm22 {%k2}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa (%rdx), %xmm0
+; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
+; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm4 {%k5}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 64(%rdx), %xmm0
+; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
+; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm5 {%k5}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 128(%rdx), %xmm0
+; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
+; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm2 {%k4}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 64(%rdx), %xmm1
-; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
-; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
+; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm2 {%k5}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 192(%rdx), %xmm0
+; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
+; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm3 {%k4}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 128(%rdx), %xmm1
-; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
-; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm4 {%k4}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 192(%rdx), %xmm1
-; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
-; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm6 {%k4}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 256(%rdx), %xmm1
-; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
-; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm29 {%k4}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 320(%rdx), %xmm1
-; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
-; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm24 {%k4}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 384(%rdx), %xmm1
-; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
-; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm21 {%k4}
-; AVX512F-ONLY-SLOW-NEXT:    vinserti32x4 $3, (%rax), %zmm19, %zmm1
+; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm3 {%k5}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 256(%rdx), %xmm0
+; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
+; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm27 {%k5}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 320(%rdx), %xmm0
+; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
+; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm21 {%k5}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa 384(%rdx), %xmm0
+; AVX512F-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
+; AVX512F-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm28 {%k5}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vinserti32x4 $3, (%rax), %zmm0, %zmm0
 ; AVX512F-ONLY-SLOW-NEXT:    movb $112, %sil
 ; AVX512F-ONLY-SLOW-NEXT:    kmovw %esi, %k2
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k2}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm4 {%k2}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vinserti32x4 $3, 64(%rax), %zmm0, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm5 {%k2}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vinserti32x4 $3, 128(%rax), %zmm0, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k2}
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vinserti32x4 $3, 64(%rax), %zmm1, %zmm1
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm3 {%k2}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vinserti32x4 $3, 192(%rax), %zmm0, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm3 {%k2}
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vinserti32x4 $3, 128(%rax), %zmm1, %zmm1
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k2}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vinserti32x4 $3, 192(%rax), %zmm5, %zmm1
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm6 {%k2}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vinserti32x4 $3, 256(%rax), %zmm1, %zmm1
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm29 {%k2}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vinserti32x4 $3, 320(%rax), %zmm1, %zmm1
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm24 {%k2}
-; AVX512F-ONLY-SLOW-NEXT:    vinserti32x4 $3, 384(%rax), %zmm0, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vinserti32x4 $3, 256(%rax), %zmm0, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm27 {%k2}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vinserti32x4 $3, 320(%rax), %zmm0, %zmm0
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm21 {%k2}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vinserti32x4 $3, 384(%rax), %zmm0, %zmm0
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm28 {%k2}
 ; AVX512F-ONLY-SLOW-NEXT:    vpbroadcastq 8(%rcx), %ymm0
 ; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm1 {%k3}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm5 {%k4}
 ; AVX512F-ONLY-SLOW-NEXT:    vpbroadcastq 72(%rcx), %ymm0
 ; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm2 {%k3}
+; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm2 {%k4}
 ; AVX512F-ONLY-SLOW-NEXT:    vpbroadcastq 136(%rcx), %ymm0
 ; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 (%rsp), %zmm3 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm3 {%k3}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm3 {%k4}
 ; AVX512F-ONLY-SLOW-NEXT:    vpbroadcastq 200(%rcx), %ymm0
 ; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm4 {%k3}
+; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm4 {%k4}
 ; AVX512F-ONLY-SLOW-NEXT:    vpbroadcastq 264(%rcx), %ymm0
 ; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm30 {%k3}
+; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm30 {%k4}
 ; AVX512F-ONLY-SLOW-NEXT:    vpbroadcastq 328(%rcx), %ymm0
 ; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm26 {%k3}
+; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm24 {%k4}
 ; AVX512F-ONLY-SLOW-NEXT:    vpbroadcastq 392(%rcx), %ymm0
 ; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm22 {%k3}
+; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm20 {%k4}
 ; AVX512F-ONLY-SLOW-NEXT:    movb $56, %cl
 ; AVX512F-ONLY-SLOW-NEXT:    kmovw %ecx, %k2
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k2}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm5 {%k2}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k2}
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm3 {%k2}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, (%rsp) # 64-byte Spill
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm4 {%k2}
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm30 {%k2}
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm26 {%k2}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm22 {%k2}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm24 {%k2}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm9, %zmm20 {%k2}
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k1}
@@ -13385,64 +13311,65 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm4 {%k1}
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm29 {%k1}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm5 {%k1}
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm28 {%k1}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm18 {%k1}
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm11 {%k1}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm7 {%k1}
 ; AVX512F-ONLY-SLOW-NEXT:    movb $120, %cl
 ; AVX512F-ONLY-SLOW-NEXT:    kmovw %ecx, %k1
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm16 {%k1}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm15 # 64-byte Folded Reload
-; AVX512F-ONLY-SLOW-NEXT:    # zmm15 = zmm0[0,1,2,3],mem[4,5,6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm17 {%k1}
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm19 {%k1}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm19 {%k1}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm16 # 64-byte Folded Reload
+; AVX512F-ONLY-SLOW-NEXT:    # zmm16 = zmm0[0,1,2,3],mem[4,5,6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm14 {%k1}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm15 {%k1}
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm25 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm25 {%k1}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm10 {%k1}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm31 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm28, %zmm31 {%k1}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm5
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm5 {%k1}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm29, %zmm25 {%k1}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm29 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm29 {%k1}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm23 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm18, %zmm23 {%k1}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm12 {%k1}
 ; AVX512F-ONLY-SLOW-NEXT:    movb $-61, %cl
 ; AVX512F-ONLY-SLOW-NEXT:    kmovw %ecx, %k1
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm15 {%k1}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm14 # 64-byte Folded Reload
-; AVX512F-ONLY-SLOW-NEXT:    # zmm14 = zmm0[0,1,2,3],mem[4,5,6,7]
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm14 {%k1}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm16 {%k1}
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm12 # 64-byte Folded Reload
-; AVX512F-ONLY-SLOW-NEXT:    # zmm12 = zmm0[0,1,2,3],mem[4,5,6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm13 # 64-byte Folded Reload
+; AVX512F-ONLY-SLOW-NEXT:    # zmm13 = zmm0[0,1,2,3],mem[4,5,6,7]
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm12 {%k1}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm13 {%k1}
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm3 # 64-byte Folded Reload
-; AVX512F-ONLY-SLOW-NEXT:    # zmm3 = zmm0[0,1,2,3],mem[4,5,6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm11 # 64-byte Folded Reload
+; AVX512F-ONLY-SLOW-NEXT:    # zmm11 = zmm0[0,1,2,3],mem[4,5,6,7]
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm3 {%k1}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm11 {%k1}
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-ONLY-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm4 # 64-byte Folded Reload
 ; AVX512F-ONLY-SLOW-NEXT:    # zmm4 = zmm0[0,1,2,3],mem[4,5,6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm4 {%k1}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm5 # 64-byte Folded Reload
+; AVX512F-ONLY-SLOW-NEXT:    # zmm5 = zmm0[0,1,2,3],mem[4,5,6,7]
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm4 {%k1}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm5 {%k1}
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-ONLY-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm6 # 64-byte Folded Reload
 ; AVX512F-ONLY-SLOW-NEXT:    # zmm6 = zmm0[0,1,2,3],mem[4,5,6,7]
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm6 {%k1}
-; AVX512F-ONLY-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm13, %zmm7 # 64-byte Folded Reload
-; AVX512F-ONLY-SLOW-NEXT:    # zmm7 = zmm13[0,1,2,3],mem[4,5,6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm7 {%k1}
+; AVX512F-ONLY-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm7 # 64-byte Folded Reload
+; AVX512F-ONLY-SLOW-NEXT:    # zmm7 = zmm10[0,1,2,3],mem[4,5,6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm7 {%k1}
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
 ; AVX512F-ONLY-SLOW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm8 # 32-byte Folded Reload
 ; AVX512F-ONLY-SLOW-NEXT:    # ymm8 = ymm8[1],mem[1],ymm8[3],mem[3]
@@ -13450,22 +13377,29 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],mem[6,7]
 ; AVX512F-ONLY-SLOW-NEXT:    movb $14, %cl
 ; AVX512F-ONLY-SLOW-NEXT:    kmovw %ecx, %k1
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm8, %zmm0, %zmm13 {%k1}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm8, %zmm0, %zmm10 {%k1}
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
 ; AVX512F-ONLY-SLOW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm8 # 32-byte Folded Reload
 ; AVX512F-ONLY-SLOW-NEXT:    # ymm8 = ymm8[1],mem[1],ymm8[3],mem[3]
 ; AVX512F-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,2,3,3]
 ; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],mem[6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm8, %zmm0, %zmm11 {%k1}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm8, %zmm0, %zmm18 {%k1}
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
 ; AVX512F-ONLY-SLOW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm8 # 32-byte Folded Reload
 ; AVX512F-ONLY-SLOW-NEXT:    # ymm8 = ymm8[1],mem[1],ymm8[3],mem[3]
 ; AVX512F-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,2,3,3]
 ; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],mem[6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm28 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm8, %zmm0, %zmm28 {%k1}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm8, %zmm0, %zmm9 {%k1}
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm8 # 32-byte Folded Reload
+; AVX512F-ONLY-SLOW-NEXT:    # ymm8 = ymm8[1],mem[1],ymm8[3],mem[3]
+; AVX512F-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,2,3,3]
+; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],mem[6,7]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm8, %zmm0, %zmm3 {%k1}
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
 ; AVX512F-ONLY-SLOW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm8 # 32-byte Folded Reload
 ; AVX512F-ONLY-SLOW-NEXT:    # ymm8 = ymm8[1],mem[1],ymm8[3],mem[3]
@@ -13480,88 +13414,81 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],mem[6,7]
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
 ; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm8, %zmm0, %zmm1 {%k1}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm8 # 32-byte Folded Reload
-; AVX512F-ONLY-SLOW-NEXT:    # ymm8 = ymm8[1],mem[1],ymm8[3],mem[3]
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm8 # 32-byte Folded Reload
+; AVX512F-ONLY-SLOW-NEXT:    # ymm8 = ymm0[1],mem[1],ymm0[3],mem[3]
 ; AVX512F-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,2,3,3]
 ; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],mem[6,7]
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm8, %zmm0, %zmm0 {%k1}
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm8 # 32-byte Folded Reload
-; AVX512F-ONLY-SLOW-NEXT:    # ymm8 = ymm8[1],mem[1],ymm8[3],mem[3]
-; AVX512F-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,2,3,3]
-; AVX512F-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],mem[6,7]
-; AVX512F-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm8, %zmm0, %zmm20 {%k1}
 ; AVX512F-ONLY-SLOW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm7, 3008(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm5, 2944(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm20, 2880(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovaps %zmm7, 2816(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm22, 2752(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm21, 2688(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm23, 2624(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm12, 2944(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, 2880(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovups (%rsp), %zmm0 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vmovaps %zmm0, 2816(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm20, 2752(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm28, 2688(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm22, 2624(%rax)
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, 2560(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm31, 2496(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, 2432(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm23, 2496(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, 2432(%rax)
 ; AVX512F-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
 ; AVX512F-ONLY-SLOW-NEXT:    vmovaps %zmm6, 2368(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm26, 2304(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm24, 2240(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm27, 2176(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm4, 2112(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm10, 2048(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, 1984(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovaps %zmm4, 1920(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm24, 2304(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm21, 2240(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm26, 2176(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm5, 2112(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm29, 2048(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, 1984(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vmovaps %zmm5, 1920(%rax)
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm30, 1856(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm29, 1792(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovaps %zmm0, 1728(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, 1664(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm27, 1792(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm31, 1728(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm4, 1664(%rax)
 ; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm25, 1600(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, 1536(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovaps %zmm3, 1472(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, 1536(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vmovaps %zmm4, 1472(%rax)
 ; AVX512F-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-ONLY-SLOW-NEXT:    vmovaps %zmm0, 1408(%rax)
 ; AVX512F-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-ONLY-SLOW-NEXT:    vmovaps %zmm0, 1344(%rax)
 ; AVX512F-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-ONLY-SLOW-NEXT:    vmovaps %zmm0, 1280(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm12, 1216(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm19, 1152(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm28, 1088(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovaps %zmm2, 1024(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovups (%rsp), %zmm0 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm11, 1216(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm15, 1152(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm9, 1088(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vmovaps %zmm3, 1024(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-ONLY-SLOW-NEXT:    vmovaps %zmm0, 960(%rax)
 ; AVX512F-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-ONLY-SLOW-NEXT:    vmovaps %zmm0, 896(%rax)
 ; AVX512F-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-ONLY-SLOW-NEXT:    vmovaps %zmm0, 832(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm14, 768(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm17, 704(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm11, 640(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-ONLY-SLOW-NEXT:    vmovaps %zmm1, 576(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm13, 768(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm14, 704(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm18, 640(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vmovaps %zmm2, 576(%rax)
 ; AVX512F-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-ONLY-SLOW-NEXT:    vmovaps %zmm0, 512(%rax)
 ; AVX512F-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-ONLY-SLOW-NEXT:    vmovaps %zmm0, 448(%rax)
 ; AVX512F-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-ONLY-SLOW-NEXT:    vmovaps %zmm0, 384(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm15, 320(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm16, 256(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm13, 192(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm16, 320(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm19, 256(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm10, 192(%rax)
 ; AVX512F-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-ONLY-SLOW-NEXT:    vmovaps %zmm0, 128(%rax)
 ; AVX512F-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-ONLY-SLOW-NEXT:    vmovaps %zmm0, 64(%rax)
 ; AVX512F-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-ONLY-SLOW-NEXT:    vmovaps %zmm0, (%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm9, 3520(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm17, 3520(%rax)
 ; AVX512F-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-ONLY-SLOW-NEXT:    vmovaps %zmm0, 3456(%rax)
 ; AVX512F-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
@@ -13572,820 +13499,816 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512F-ONLY-SLOW-NEXT:    vmovaps %zmm0, 3264(%rax)
 ; AVX512F-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-ONLY-SLOW-NEXT:    vmovaps %zmm0, 3200(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    vmovdqa64 %zmm18, 3072(%rax)
+; AVX512F-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-ONLY-SLOW-NEXT:    vmovaps %zmm0, 3072(%rax)
 ; AVX512F-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-ONLY-SLOW-NEXT:    vmovaps %zmm0, 3136(%rax)
-; AVX512F-ONLY-SLOW-NEXT:    addq $6600, %rsp # imm = 0x19C8
+; AVX512F-ONLY-SLOW-NEXT:    addq $6248, %rsp # imm = 0x1868
 ; AVX512F-ONLY-SLOW-NEXT:    vzeroupper
 ; AVX512F-ONLY-SLOW-NEXT:    retq
 ;
 ; AVX512F-ONLY-FAST-LABEL: store_i64_stride7_vf64:
 ; AVX512F-ONLY-FAST:       # %bb.0:
-; AVX512F-ONLY-FAST-NEXT:    subq $6696, %rsp # imm = 0x1A28
+; AVX512F-ONLY-FAST-NEXT:    subq $6120, %rsp # imm = 0x17E8
 ; AVX512F-ONLY-FAST-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 (%rdi), %zmm5
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 64(%rdi), %zmm22
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm22, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 (%rsi), %zmm23
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 64(%rsi), %zmm17
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 64(%rdx), %zmm24
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm24, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 (%rdx), %zmm6
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 (%rcx), %zmm7
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 (%rdi), %zmm7
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 64(%rcx), %zmm9
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm10 = [11,3,11,3,11,3,11,3]
-; AVX512F-ONLY-FAST-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm12 = [2,10,0,3,2,10,0,3]
-; AVX512F-ONLY-FAST-NEXT:    # zmm12 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    movb $96, %r10b
-; AVX512F-ONLY-FAST-NEXT:    kmovw %r10d, %k1
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 64(%rdi), %zmm13
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 (%rsi), %zmm8
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 64(%rsi), %zmm20
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm20, (%rsp) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 64(%rdx), %zmm14
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 (%rdx), %zmm10
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 (%rcx), %zmm11
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 64(%rcx), %zmm19
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm9 = [11,3,11,3,11,3,11,3]
+; AVX512F-ONLY-FAST-NEXT:    # zmm9 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm3 = [2,10,0,3,2,10,0,3]
+; AVX512F-ONLY-FAST-NEXT:    # zmm3 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-FAST-NEXT:    movb $96, %r10b
+; AVX512F-ONLY-FAST-NEXT:    kmovw %r10d, %k1
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 (%r8), %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 (%r9), %zmm8
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 (%rax), %zmm3
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 64(%rax), %zmm4
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 (%r9), %zmm18
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 (%rax), %zmm4
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 64(%rax), %zmm5
+; AVX512F-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm23 = [15,7,15,7,15,7,15,7]
+; AVX512F-ONLY-FAST-NEXT:    # zmm23 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [9,1,9,1,9,1,9,1]
 ; AVX512F-ONLY-FAST-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm8, %zmm0, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm14
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm27 = [4,9,0,3,4,9,0,3]
-; AVX512F-ONLY-FAST-NEXT:    # zmm27 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm3, %zmm27, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm18, %zmm0, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm6
+; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [4,9,0,3,4,9,0,3]
+; AVX512F-ONLY-FAST-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm4, %zmm0, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm12
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm5, %zmm0
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm23, %zmm10, %zmm0
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm6, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm7, %zmm12, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm7, %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm8, %zmm9, %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm10, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm11, %zmm3, %zmm2
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k1}
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqa (%r9), %ymm0
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa 64(%r9), %ymm11
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa (%r8), %ymm6
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 64(%r8), %ymm26
-; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm6[0],ymm0[0],ymm6[2],ymm0[2]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa 64(%r9), %ymm10
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa (%r8), %ymm8
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa 64(%r8), %ymm7
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm8[0],ymm0[0],ymm8[2],ymm0[2]
 ; AVX512F-ONLY-FAST-NEXT:    movb $28, %r10b
 ; AVX512F-ONLY-FAST-NEXT:    kmovw %r10d, %k2
-; AVX512F-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm2 {%k2} = zmm0[2,3,2,3],zmm3[2,3,2,3]
+; AVX512F-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm2 {%k2} = zmm0[2,3,2,3],zmm4[2,3,2,3]
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [4,12,0,5,4,12,0,5]
 ; AVX512F-ONLY-FAST-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3]
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, %zmm0
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm8, %zmm6
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm8, %zmm2, %zmm0
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm7
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [0,1,12,7,0,1,12,7]
-; AVX512F-ONLY-FAST-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm3, %zmm2, %zmm0
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm18
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm8 = [5,0,14,6,5,0,14,6]
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm18, %zmm2, %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm8 = [0,1,12,7,0,1,12,7]
 ; AVX512F-ONLY-FAST-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm6, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm1, %zmm8, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [0,13,6,7,0,13,6,7]
-; AVX512F-ONLY-FAST-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm3, %zmm0, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm20
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm15 = [15,7,15,7,15,7,15,7]
-; AVX512F-ONLY-FAST-NEXT:    # zmm15 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm6, %zmm15, %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm16 = [6,13,14,7,6,13,14,7]
-; AVX512F-ONLY-FAST-NEXT:    # zmm16 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm1, %zmm16, %zmm3
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm22, %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm17, %zmm10, %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm24, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm9, %zmm12, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}
-; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm1 = ymm26[0],ymm11[0],ymm26[2],ymm11[2]
-; AVX512F-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm2 {%k2} = zmm1[2,3,2,3],zmm4[2,3,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm4, %zmm8, %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm22 = [5,0,14,6,5,0,14,6]
+; AVX512F-ONLY-FAST-NEXT:    # zmm22 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm18, %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm1, %zmm22, %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm15 = [0,13,6,7,0,13,6,7]
+; AVX512F-ONLY-FAST-NEXT:    # zmm15 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm4, %zmm15, %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm18, %zmm23, %zmm1
+; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm18 = [6,13,14,7,6,13,14,7]
+; AVX512F-ONLY-FAST-NEXT:    # zmm18 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm1, %zmm18, %zmm4
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm13, %zmm1
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm20, %zmm9, %zmm1
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm14, %zmm4
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm19, %zmm3, %zmm4
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
+; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm1 = ymm7[0],ymm10[0],ymm7[2],ymm10[2]
+; AVX512F-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm4 {%k2} = zmm1[2,3,2,3],zmm5[2,3,2,3]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 64(%r8), %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 64(%r9), %zmm3
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm3, %zmm14, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm4, %zmm27, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm3, %zmm7, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm4, %zmm18, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm1, %zmm8, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm4, %zmm0, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 64(%r9), %zmm30
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, %zmm4
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm30, %zmm6, %zmm4
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm5, %zmm12, %zmm4
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm5, %zmm7
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, %zmm4
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm5
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm30, %zmm2, %zmm4
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm7, %zmm8, %zmm4
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm30, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm1, %zmm22, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm7, %zmm15, %zmm2
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm3, %zmm15, %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm1, %zmm16, %zmm4
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm30, %zmm23, %zmm1
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm1, %zmm18, %zmm7
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 128(%rdi), %zmm1
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 128(%rsi), %zmm19
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm19, %zmm10, %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 128(%rdx), %zmm3
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 128(%rcx), %zmm24
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm24, %zmm12, %zmm3
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, %zmm3 {%k1}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 128(%rax), %zmm6
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 128(%rsi), %zmm26
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm26, %zmm9, %zmm1
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm9, %zmm29
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 128(%rdx), %zmm4
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 128(%rcx), %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm3, %zmm4
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm9
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 128(%rax), %zmm7
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqa 128(%r9), %ymm1
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa 128(%r8), %ymm4
-; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm2 = ymm4[0],ymm1[0],ymm4[2],ymm1[2]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %ymm4, %ymm25
-; AVX512F-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm3 {%k2} = zmm2[2,3,2,3],zmm6[2,3,2,3]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa 128(%r8), %ymm0
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm1 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
+; AVX512F-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm4 {%k2} = zmm1[2,3,2,3],zmm7[2,3,2,3]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 128(%r8), %zmm4
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 128(%r9), %zmm19
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm4, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm19, %zmm6, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm12, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm7, %zmm12, %zmm3
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 128(%r8), %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 128(%r9), %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm1, %zmm14, %zmm3
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm6, %zmm27, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm4, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm19, %zmm5, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm5, %zmm13
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm7, %zmm8, %zmm3
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm7, %zmm4
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm1, %zmm7, %zmm3
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm6, %zmm18, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm19, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm4, %zmm22, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm15, %zmm20
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm7, %zmm15, %zmm3
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, %zmm3
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm2, %zmm8, %zmm3
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm6, %zmm0, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm19, %zmm23, %zmm4
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm4, %zmm18, %zmm7
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 192(%rdi), %zmm4
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 192(%rsi), %zmm28
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm28, %zmm29, %zmm4
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 192(%rdx), %zmm3
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm1, %zmm15, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm2, %zmm16, %zmm6
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 192(%rdi), %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 192(%rsi), %zmm11
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm11, %zmm10, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm10, %zmm28
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 192(%rdx), %zmm10
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 192(%rcx), %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm1, %zmm12, %zmm10
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm12, %zmm5
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm10 {%k1}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 192(%rax), %zmm30
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 192(%r9), %ymm17
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa 192(%r8), %ymm12
-; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm3 = ymm12[0],ymm17[0],ymm12[2],ymm17[2]
-; AVX512F-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm10 {%k2} = zmm3[2,3,2,3],zmm30[2,3,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 192(%r8), %zmm3
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 192(%r9), %zmm0
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm10
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm14, %zmm10
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm14, %zmm7
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm30, %zmm27, %zmm10
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm13
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm4, %zmm13
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm4, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm18, %zmm6
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm30, %zmm18, %zmm13
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 192(%rcx), %zmm0
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm4
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm3, %zmm8, %zmm4
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm30, %zmm20, %zmm4
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm9, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm4, %zmm3 {%k1}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 192(%rax), %zmm14
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa 192(%r9), %ymm0
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa 192(%r8), %ymm5
+; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm4 = ymm5[0],ymm0[0],ymm5[2],ymm0[2]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %ymm5, %ymm24
+; AVX512F-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm3 {%k2} = zmm4[2,3,2,3],zmm14[2,3,2,3]
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm15, %zmm3
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm3, %zmm16, %zmm30
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm30, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 256(%rdi), %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 192(%r8), %zmm5
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 192(%r9), %zmm17
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm5, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm17, %zmm6, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm6, %zmm4
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm14, %zmm12, %zmm3
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 256(%rsi), %zmm13
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm13, %zmm28, %zmm3
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm28, %zmm29
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm28, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 256(%rdx), %zmm28
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 256(%rcx), %zmm18
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm28, %zmm14
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm18, %zmm5, %zmm14
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm5, %zmm30
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm14 {%k1}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 256(%rax), %zmm0
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa 256(%r9), %ymm9
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 256(%r8), %ymm22
-; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm4 = ymm22[0],ymm9[0],ymm22[2],ymm9[2]
-; AVX512F-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm14 {%k2} = zmm4[2,3,2,3],zmm0[2,3,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 256(%r8), %zmm4
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 256(%r9), %zmm3
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm4, %zmm14
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm3, %zmm7, %zmm14
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm5
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm27, %zmm14
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm27, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm4, %zmm14
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm5, %zmm2, %zmm14
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm6, %zmm14
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm5, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm17, %zmm13, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm14, %zmm8, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm8, %zmm7
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm17, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm5, %zmm22, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm14, %zmm15, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm5, %zmm21
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm17, %zmm23, %zmm5
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm5, %zmm18, %zmm14
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm5, %zmm1
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 256(%rdi), %zmm5
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm5, %zmm10
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm4, %zmm8, %zmm10
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm20, %zmm10
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm20, %zmm5
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 256(%rsi), %zmm15
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm29, %zmm1
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm29, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm15, %zmm29, %zmm5
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 256(%rdx), %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 256(%rcx), %zmm29
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm9, %zmm8
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm29, %zmm9, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm5, %zmm3 {%k1}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 256(%rax), %zmm16
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa 256(%r9), %ymm10
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa 256(%r8), %ymm5
+; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm6 = ymm5[0],ymm10[0],ymm5[2],ymm10[2]
+; AVX512F-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm3 {%k2} = zmm6[2,3,2,3],zmm16[2,3,2,3]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 256(%r8), %zmm6
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 256(%r9), %zmm9
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm6, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm9, %zmm4, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm16, %zmm12, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm16, %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm6, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm13, %zmm16
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm9, %zmm13, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm7, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm9, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm6, %zmm22, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm20, %zmm3
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm20, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm1, %zmm15, %zmm4
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm4, %zmm16, %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm6, %zmm11
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm9, %zmm23, %zmm6
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm6, %zmm18, %zmm0
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 320(%rdi), %zmm4
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 320(%rsi), %zmm20
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm20, %zmm29, %zmm4
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 320(%rdx), %zmm31
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 320(%rcx), %zmm14
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm31, %zmm21
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm14, %zmm30, %zmm21
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm4, %zmm21 {%k1}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 320(%rdi), %zmm27
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 320(%rsi), %zmm25
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm27, %zmm6
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm25, %zmm1, %zmm6
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 320(%rdx), %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 320(%rcx), %zmm12
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm12, %zmm8, %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm8, %zmm13
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm6, %zmm0 {%k1}
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 320(%rax), %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa 320(%r9), %ymm4
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa 320(%r8), %ymm2
-; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm10 = ymm2[0],ymm4[0],ymm2[2],ymm4[2]
-; AVX512F-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm21 {%k2} = zmm10[2,3,2,3],zmm1[2,3,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 320(%r8), %zmm10
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 320(%r9), %zmm21
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm10, %zmm0
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm21, %zmm7, %zmm0
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm1, %zmm27, %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa 320(%r9), %ymm6
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa 320(%r8), %ymm14
+; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm8 = ymm14[0],ymm6[0],ymm14[2],ymm6[2]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %ymm14, %ymm31
+; AVX512F-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm0 {%k2} = zmm8[2,3,2,3],zmm1[2,3,2,3]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 320(%r8), %zmm8
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 320(%r9), %zmm14
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm8, %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm14, %zmm4, %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm4, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm1, %zmm2, %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, %zmm2
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm10, %zmm0
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm21, %zmm3, %zmm0
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm1, %zmm6, %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm8, %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm14, %zmm16, %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm1, %zmm7, %zmm0
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm21, %zmm0
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm10, %zmm8, %zmm0
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm1, %zmm5, %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm14, %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm8, %zmm22, %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm1, %zmm20, %zmm0
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm21, %zmm15, %zmm10
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm10, %zmm16, %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm8, %zmm1
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm14, %zmm23, %zmm8
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm8, %zmm18, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqa {{.*#+}} ymm0 = [1,3,7,7]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 32-byte Folded Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm26 # 32-byte Folded Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %ymm26, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm25 # 32-byte Folded Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %ymm25, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %ymm17, %ymm0, %ymm12
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %ymm9, %ymm0, %ymm22
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %ymm22, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %ymm4, %ymm0, %ymm2
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm2 # 32-byte Folded Reload
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa 384(%r9), %ymm1
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa 384(%r8), %ymm2
-; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm3 = ymm2[0],ymm1[0],ymm2[2],ymm1[2]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %ymm1, %ymm0, %ymm2
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm8 # 32-byte Folded Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm2 # 32-byte Folded Reload
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm27 = [3,0,12,4,3,0,12,4]
-; AVX512F-ONLY-FAST-NEXT:    # zmm27 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm24 # 32-byte Folded Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %ymm24, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %ymm10, %ymm0, %ymm5
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %ymm6, %ymm0, %ymm31
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %ymm31, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa 384(%r9), %ymm2
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa 384(%r8), %ymm4
+; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm5 = ymm4[0],ymm2[0],ymm4[2],ymm2[2]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %ymm2, %ymm0, %ymm4
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 384(%rdx), %zmm31
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 384(%rcx), %zmm10
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm31, %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm10, %zmm23, %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 384(%rdi), %zmm24
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm24, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 384(%rsi), %zmm18
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm18, %zmm23, %zmm24
+; AVX512F-ONLY-FAST-NEXT:    movb $24, %r10b
+; AVX512F-ONLY-FAST-NEXT:    kmovw %r10d, %k3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm24 {%k3}
+; AVX512F-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm20 = [0,8,0,8,0,8,0,8]
+; AVX512F-ONLY-FAST-NEXT:    # zmm20 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm27, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm12 = [13,5,13,5,13,5,13,5]
-; AVX512F-ONLY-FAST-NEXT:    # zmm12 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm3
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm1, %zmm12, %zmm3
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm30 = [6,14,6,14,6,14,6,14]
-; AVX512F-ONLY-FAST-NEXT:    # zmm30 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm3
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm1, %zmm30, %zmm3
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm1, %zmm15, %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm20, %zmm0 # 64-byte Folded Reload
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    movb $48, %r10b
-; AVX512F-ONLY-FAST-NEXT:    kmovw %r10d, %k3
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm10 = [0,8,0,1,0,8,0,1]
-; AVX512F-ONLY-FAST-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3]
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm23, %zmm10, %zmm1
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm30, %zmm20, %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm19, %zmm20, %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm17, %zmm20, %zmm21
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm9, %zmm20, %zmm11
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm14, %zmm20, %zmm1
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm7 = [1,0,10,2,1,0,10,2]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 384(%r8), %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm1 = <u,1,2,3,4,15,u,u>
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm24
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 384(%r9), %zmm1
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm2 = <u,1,2,3,4,5,15,u>
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm1, %zmm2, %zmm24
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm1, %zmm0, %zmm20
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm1, %zmm0, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm1, %zmm0, %zmm16
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm22, %zmm1
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, %zmm9
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 448(%rdx), %zmm30
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 448(%rcx), %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm1 = [6,14,6,14,6,14,6,14]
+; AVX512F-ONLY-FAST-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm30, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm30, %zmm8
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm13, %zmm8
+; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm7 = [3,0,12,4,3,0,12,4]
 ; AVX512F-ONLY-FAST-NEXT:    # zmm7 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm23, %zmm9
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm7, %zmm9
-; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm2 {%k3} = zmm0[0],zmm23[0],zmm0[2],zmm23[2],zmm0[4],zmm23[4],zmm0[6],zmm23[6]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm11
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm30, %zmm7, %zmm11
+; AVX512F-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm5 = [13,5,13,5,13,5,13,5]
+; AVX512F-ONLY-FAST-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm30, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm5, %zmm2
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm23, %zmm12, %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} ymm29 = [6,14,6,14]
-; AVX512F-ONLY-FAST-NEXT:    # ymm29 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm23, %zmm29, %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} ymm25 = [15,7,15,7]
-; AVX512F-ONLY-FAST-NEXT:    # ymm25 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm23, %zmm25, %zmm0
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm1
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm23, %zmm30
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm17, %zmm0
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm3, %zmm27, %zmm1
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm3, %zmm7, %zmm0
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm12, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm17, %zmm5, %zmm2
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm30, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm17, %zmm1, %zmm2
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm15, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm17, %zmm23, %zmm3
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    movb $48, %r10b
+; AVX512F-ONLY-FAST-NEXT:    kmovw %r10d, %k4
+; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [0,8,0,1,0,8,0,1]
+; AVX512F-ONLY-FAST-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm10, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm3, %zmm7, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm19, %zmm4, %zmm2
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm1 {%k3} = zmm3[0],zmm0[0],zmm3[2],zmm0[2],zmm3[4],zmm0[4],zmm3[6],zmm0[6]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm12, %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm29, %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm25, %zmm3
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm24, %zmm0
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm2, %zmm27, %zmm0
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm24, %zmm12, %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm24, %zmm30, %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm24, %zmm15, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm6 = [1,0,10,2,1,0,10,2]
+; AVX512F-ONLY-FAST-NEXT:    # zmm6 = mem[0,1,2,3,0,1,2,3]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm19, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm3, %zmm6, %zmm2
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm19, %zmm10, %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm19, %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm2, %zmm7, %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm0 {%k3} = zmm2[0],zmm19[0],zmm2[2],zmm19[2],zmm2[4],zmm19[4],zmm2[6],zmm19[6]
+; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm0 {%k4} = zmm3[0],zmm19[0],zmm3[2],zmm19[2],zmm3[4],zmm19[4],zmm3[6],zmm19[6]
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm0
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm19, %zmm12, %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm19, %zmm5, %zmm0
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm0
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm19, %zmm29, %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm19, %zmm1, %zmm0
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm19, %zmm25, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm0
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm3, %zmm27, %zmm0
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm2, %zmm12, %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm2, %zmm30, %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm2, %zmm15, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm19, %zmm23, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm13, %zmm7, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm13, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm5, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm13, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm23, %zmm13
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm13, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 (%rsp), %zmm0 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm4, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm13, %zmm6, %zmm3
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm2 {%k4} = zmm13[0],zmm0[0],zmm13[2],zmm0[2],zmm13[4],zmm0[4],zmm13[6],zmm0[6]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm13, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm5, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm13, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm23, %zmm13
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm11, %zmm10, %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm11, %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm2, %zmm7, %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm0 {%k3} = zmm2[0],zmm11[0],zmm2[2],zmm11[2],zmm2[4],zmm11[4],zmm2[6],zmm11[6]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm7, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm14
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm2, %zmm5, %zmm14
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm14
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm2, %zmm1, %zmm14
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm2, %zmm23, %zmm0
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm0
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm11, %zmm12, %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm26, %zmm4, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm26, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm6, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, (%rsp) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm3 {%k4} = zmm0[0],zmm26[0],zmm0[2],zmm26[2],zmm0[4],zmm26[4],zmm0[6],zmm26[6]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm26, %zmm5, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm26, %zmm1, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm26, %zmm23, %zmm0
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm0
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm11, %zmm29, %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm13, %zmm7, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm13, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm5, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm13, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm23, %zmm13
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm28, %zmm4, %zmm0
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm11, %zmm25, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm28, %zmm22
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm3, %zmm6, %zmm22
+; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm2 {%k4} = zmm3[0],zmm28[0],zmm3[2],zmm28[2],zmm3[4],zmm28[4],zmm3[6],zmm28[6]
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm18, %zmm0
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm28, %zmm27, %zmm0
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm28, %zmm0
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm18, %zmm12, %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm28, %zmm5, %zmm0
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm28, %zmm0
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm18, %zmm30, %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm28, %zmm1, %zmm0
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm18, %zmm15, %zmm28
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm28, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm28, %zmm23, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm29, %zmm28
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm7, %zmm28
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm13, %zmm10, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm29, %zmm5, %zmm2
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm13, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm7, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm29, %zmm1, %zmm2
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm1 {%k3} = zmm0[0],zmm13[0],zmm0[2],zmm13[2],zmm0[4],zmm13[4],zmm0[6],zmm13[6]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm13, %zmm12, %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm13, %zmm29, %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm13, %zmm25, %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm29, %zmm23, %zmm0
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm14, %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm31, %zmm27, %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm31, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm14, %zmm12, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm31, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm14, %zmm30, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm14, %zmm15, %zmm31
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm31, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm29
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm15, %zmm4, %zmm29
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm15, %zmm21
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm6, %zmm21
+; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm28 {%k4} = zmm0[0],zmm15[0],zmm0[2],zmm15[2],zmm0[4],zmm15[4],zmm0[6],zmm15[6]
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm20, %zmm10, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm15, %zmm5, %zmm2
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm20, %zmm28
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm7, %zmm28
-; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm1 {%k3} = zmm0[0],zmm20[0],zmm0[2],zmm20[2],zmm0[4],zmm20[4],zmm0[6],zmm20[6]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm20, %zmm12, %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm20, %zmm29, %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm20, %zmm25, %zmm0
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm22
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 448(%rdx), %zmm13
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 448(%rcx), %zmm5
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm13, %zmm14
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm5, %zmm0, %zmm14
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm13, %zmm16
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm5, %zmm30, %zmm16
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm5, %zmm20
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm13, %zmm27, %zmm20
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm13, %zmm19
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm5, %zmm12, %zmm19
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm5, %zmm15, %zmm13
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 384(%rdx), %zmm4
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 384(%rcx), %zmm5
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm5, %zmm4, %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm15, %zmm1, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm15, %zmm23, %zmm0
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm4, %zmm5, %zmm27
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm5, %zmm4, %zmm30
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm4, %zmm0
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm5, %zmm15, %zmm4
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm5, %zmm12, %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm12, %zmm19
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm16, %zmm7, %zmm19
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm16, %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm12, %zmm5, %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm16, %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm12, %zmm1, %zmm0
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 448(%rdi), %zmm11
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm12, %zmm23, %zmm16
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 448(%rdi), %zmm12
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 448(%rsi), %zmm0
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm11, %zmm8
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm8
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm11, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm29, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm11, %zmm3
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm12, %zmm3
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 384(%rdi), %zmm6
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 384(%rsi), %zmm5
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm6, %zmm24
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm5, %zmm10, %zmm24
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm5, %zmm23
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm6, %zmm7, %zmm23
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm5, %zmm6, %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm27 {%k3} = zmm6[0],zmm5[0],zmm6[2],zmm5[2],zmm6[4],zmm5[4],zmm6[6],zmm5[6]
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm5, %zmm6, %zmm12
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm5, %zmm6, %zmm29
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm5, %zmm25, %zmm6
-; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm20 {%k3} = zmm11[0],zmm0[0],zmm11[2],zmm0[2],zmm11[4],zmm0[4],zmm11[6],zmm0[6]
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm0, %zmm11, %zmm10
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm11, %zmm0, %zmm7
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm25, %zmm11
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm8, %zmm14 {%k1}
-; AVX512F-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm2[0,1,2,3],zmm16[4,5,6,7]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 448(%r8), %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm5 = <0,11,u,u,4,5,6,7>
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm2, %zmm20, %zmm5
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 448(%r9), %zmm25
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm15 = <0,1,11,u,4,5,6,7>
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm25, %zmm5, %zmm15
-; AVX512F-ONLY-FAST-NEXT:    movb $4, %sil
-; AVX512F-ONLY-FAST-NEXT:    kmovw %esi, %k3
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm14 {%k3}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm16 = <0,1,2,10,u,5,6,7>
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm25, %zmm14, %zmm16
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm19, %zmm3 {%k1}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm5 = <12,u,u,3,4,5,6,13>
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm2, %zmm3, %zmm5
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm19 = <0,12,u,3,4,5,6,7>
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm25, %zmm5, %zmm19
-; AVX512F-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm20 = [0,8,0,8,0,8,0,8]
-; AVX512F-ONLY-FAST-NEXT:    # zmm20 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm20, %zmm1 # 64-byte Folded Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm20, %zmm1 # 64-byte Folded Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm20, %zmm1 # 64-byte Folded Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm20, %zmm1 # 64-byte Folded Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm20, %zmm17 # 64-byte Folded Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm12, %zmm15
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm23, %zmm15
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm25, %zmm17
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm27, %zmm13
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm27, %zmm6, %zmm17
+; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm19 {%k4} = zmm27[0],zmm25[0],zmm27[2],zmm25[2],zmm27[4],zmm25[4],zmm27[6],zmm25[6]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm27, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm27, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm25, %zmm23, %zmm13
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm25, %zmm4, %zmm27
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm25, %zmm5, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm25, %zmm1, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm26 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm21, %zmm20, %zmm26
-; AVX512F-ONLY-FAST-NEXT:    movb $24, %sil
-; AVX512F-ONLY-FAST-NEXT:    kmovw %esi, %k4
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm4, %zmm6 {%k4}
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm10, %zmm31, %zmm26
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm31, %zmm10, %zmm7
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm31, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm10, %zmm5, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm10, %zmm1, %zmm31
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm31, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm12, %zmm10
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm10
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm12, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm23 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm23, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm12, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm5, %zmm2
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm18, %zmm25
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm14, %zmm6, %zmm25
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm18, %zmm14, %zmm23
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm23, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm7 {%k4} = zmm14[0],zmm18[0],zmm14[2],zmm18[2],zmm14[4],zmm18[4],zmm14[6],zmm18[6]
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm18, %zmm14, %zmm5
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm14, %zmm23
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm18, %zmm1, %zmm14
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm18, %zmm4, %zmm23
+; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm11 {%k4} = zmm12[0],zmm0[0],zmm12[2],zmm0[2],zmm12[4],zmm0[4],zmm12[6],zmm0[6]
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm0, %zmm12, %zmm4
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm12, %zmm6, %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm1 # 64-byte Folded Reload
+; AVX512F-ONLY-FAST-NEXT:    # zmm1 = zmm10[0,1,2,3],mem[4,5,6,7]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm8 {%k1}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 448(%r8), %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm10 = <0,11,u,u,4,5,6,7>
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm3, %zmm11, %zmm10
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm6, %zmm2 {%k1}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm11 = <12,u,u,3,4,5,6,13>
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm3, %zmm2, %zmm11
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm30, %zmm15 {%k3}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm2 = <u,1,2,3,4,15,u,u>
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm3, %zmm2, %zmm15
 ; AVX512F-ONLY-FAST-NEXT:    movb $6, %sil
 ; AVX512F-ONLY-FAST-NEXT:    kmovw %esi, %k5
-; AVX512F-ONLY-FAST-NEXT:    vpbroadcastq 456(%rcx), %ymm1
-; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm1 = mem[0,1,2,3],ymm1[4,5,6,7]
-; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm7 {%k5}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm1 = <0,1,2,9,u,u,6,7>
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm2, %zmm7, %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm13, %zmm11 {%k4}
+; AVX512F-ONLY-FAST-NEXT:    vpbroadcastq 456(%rcx), %ymm2
+; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm2 = mem[0,1,2,3],ymm2[4,5,6,7]
+; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm2, %zmm0, %zmm0 {%k5}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm2 = <0,1,2,9,u,u,6,7>
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm3, %zmm0, %zmm2
 ; AVX512F-ONLY-FAST-NEXT:    movb $64, %sil
-; AVX512F-ONLY-FAST-NEXT:    kmovw %esi, %k3
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm0 {%k3}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm3 = <u,1,2,3,4,15,u,u>
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm2, %zmm3, %zmm11
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 384(%r8), %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm2, %zmm3, %zmm6
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 384(%r9), %zmm3
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm3, %zmm2, %zmm20
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm3, %zmm2, %zmm4
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm3, %zmm2, %zmm14
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm2, %zmm3, %zmm13
-; AVX512F-ONLY-FAST-NEXT:    movb $12, %sil
-; AVX512F-ONLY-FAST-NEXT:    kmovw %esi, %k3
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa 448(%rdx), %xmm2
-; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],mem[0]
-; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm2, %zmm0, %zmm10 {%k3}
-; AVX512F-ONLY-FAST-NEXT:    vinserti32x4 $2, 448(%r8), %zmm10, %zmm2
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm5 = <0,1,2,3,4,8,u,7>
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm25, %zmm2, %zmm5
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm2 = <u,1,2,3,4,5,15,u>
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm3, %zmm2, %zmm6
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm7 = <0,1,2,3,9,u,6,7>
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm25, %zmm1, %zmm7
+; AVX512F-ONLY-FAST-NEXT:    kmovw %esi, %k4
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm1 {%k4}
+; AVX512F-ONLY-FAST-NEXT:    movb $4, %sil
+; AVX512F-ONLY-FAST-NEXT:    kmovw %esi, %k4
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm8 {%k4}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 448(%r9), %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm31 = <0,1,11,u,4,5,6,7>
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm0, %zmm10, %zmm31
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm3 = <0,1,2,10,u,5,6,7>
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm0, %zmm8, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm8 = <0,12,u,3,4,5,6,7>
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm0, %zmm11, %zmm8
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm10 = <13,u,2,3,4,5,6,14>
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm25, %zmm0, %zmm10
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm25, %zmm2, %zmm11
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm0, %zmm1, %zmm10
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm1 = <u,1,2,3,4,5,15,u>
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm15
+; AVX512F-ONLY-FAST-NEXT:    movb $12, %sil
+; AVX512F-ONLY-FAST-NEXT:    kmovw %esi, %k4
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa 448(%rdx), %xmm1
+; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
+; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
+; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm4 {%k4}
+; AVX512F-ONLY-FAST-NEXT:    vinserti32x4 $2, 448(%r8), %zmm4, %zmm1
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm4 = <0,1,2,3,4,8,u,7>
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm0, %zmm1, %zmm4
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm11 = <0,1,2,3,9,u,6,7>
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm0, %zmm2, %zmm11
 ; AVX512F-ONLY-FAST-NEXT:    vpbroadcastq 8(%rcx), %ymm0
 ; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
-; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm9 {%k5}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm14 {%k5}
 ; AVX512F-ONLY-FAST-NEXT:    vpbroadcastq 72(%rcx), %ymm0
 ; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm21 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm21 {%k5}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm6 {%k5}
 ; AVX512F-ONLY-FAST-NEXT:    vpbroadcastq 136(%rcx), %ymm0
 ; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm8 {%k5}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 (%rsp), %zmm2 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm2 {%k5}
 ; AVX512F-ONLY-FAST-NEXT:    vpbroadcastq 200(%rcx), %ymm0
 ; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm18 {%k5}
+; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm22 {%k5}
 ; AVX512F-ONLY-FAST-NEXT:    vpbroadcastq 264(%rcx), %ymm0
 ; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm25 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm25 {%k5}
+; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm21 {%k5}
 ; AVX512F-ONLY-FAST-NEXT:    vpbroadcastq 328(%rcx), %ymm0
 ; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
-; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm28 {%k5}
+; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm17 {%k5}
 ; AVX512F-ONLY-FAST-NEXT:    vpbroadcastq 392(%rcx), %ymm0
 ; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
-; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm23 {%k5}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 448(%rax), %zmm0
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,1,2,3,10,5,6,7]
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm0, %zmm16, %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,1,12,3,4,5,6,7]
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm0, %zmm19, %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 384(%rax), %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm1, %zmm3, %zmm4
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm16 {%k1}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm16 {%k2} = zmm3[2,3,2,3],zmm1[2,3,2,3]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm1, %zmm3, %zmm14
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm1, %zmm3, %zmm13
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [0,1,2,3,4,5,8,7]
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm0, %zmm5, %zmm3
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [14,1,2,3,4,5,6,15]
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm1, %zmm5, %zmm6
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,1,2,3,4,9,6,7]
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm0, %zmm7, %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,13,2,3,4,5,6,7]
-; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm0, %zmm10, %zmm1
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm5, %zmm11
+; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm25 {%k5}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 384(%rax), %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm30 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm30
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, %zmm26 {%k1}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm26 {%k2} = zmm1[2,3,2,3],zmm0[2,3,2,3]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm26, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm26 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm26
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm9
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [14,1,2,3,4,5,6,15]
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm24
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 448(%rax), %zmm12
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [0,1,2,3,10,5,6,7]
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm12, %zmm3, %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [0,1,12,3,4,5,6,7]
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm12, %zmm8, %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [0,13,2,3,4,5,6,7]
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm12, %zmm10, %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vpermt2q %zmm12, %zmm1, %zmm15
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [0,1,2,3,4,5,8,7]
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm12, %zmm4, %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [0,1,2,3,4,9,6,7]
+; AVX512F-ONLY-FAST-NEXT:    vpermi2q %zmm12, %zmm11, %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-ONLY-FAST-NEXT:    movb $8, %sil
 ; AVX512F-ONLY-FAST-NEXT:    kmovw %esi, %k2
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm15 {%k2}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm12, %zmm31 {%k2}
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k4}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, %zmm0 {%k3}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm3 {%k4}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm5 {%k4}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm7 {%k4}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm10 {%k4}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm22 {%k4}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm1 {%k3}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm4 {%k3}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm8 {%k3}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm9 {%k3}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm16, %zmm13 {%k3}
 ; AVX512F-ONLY-FAST-NEXT:    movb $-31, %sil
 ; AVX512F-ONLY-FAST-NEXT:    kmovw %esi, %k2
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm0 {%k2}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k2}
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm3 {%k2}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm5 {%k2}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm4 {%k2}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm7 {%k2}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm8 {%k2}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm10 {%k2}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm9 {%k2}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm22 {%k2}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm22, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa (%rdx), %xmm0
-; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
-; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm13 {%k2}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa (%rdx), %xmm3
+; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],mem[0]
+; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm1 {%k3}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa 64(%rdx), %xmm0
-; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
-; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm3 {%k3}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa 128(%rdx), %xmm0
-; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
-; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm5 {%k3}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa 192(%rdx), %xmm0
-; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
-; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm7 {%k3}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa 256(%rdx), %xmm0
-; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
-; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm10 {%k3}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa 320(%rdx), %xmm0
-; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
-; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm13 {%k3}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa 384(%rdx), %xmm0
-; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
-; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm24 {%k3}
+; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm1 {%k4}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa 64(%rdx), %xmm3
+; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],mem[0]
+; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm4 {%k4}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa 128(%rdx), %xmm3
+; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],mem[0]
+; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm8 {%k4}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa 192(%rdx), %xmm3
+; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],mem[0]
+; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm9 {%k4}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa 256(%rdx), %xmm3
+; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],mem[0]
+; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm29 {%k4}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa 320(%rdx), %xmm3
+; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],mem[0]
+; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm27 {%k4}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa 384(%rdx), %xmm3
+; AVX512F-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],mem[0]
+; AVX512F-ONLY-FAST-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm23 {%k4}
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vinserti32x4 $3, (%rax), %zmm0, %zmm0
+; AVX512F-ONLY-FAST-NEXT:    vinserti32x4 $3, (%rax), %zmm0, %zmm3
 ; AVX512F-ONLY-FAST-NEXT:    movb $112, %cl
 ; AVX512F-ONLY-FAST-NEXT:    kmovw %ecx, %k2
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k2}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm1 {%k2}
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vinserti32x4 $3, 64(%rax), %zmm0, %zmm0
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm3 {%k2}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vinserti32x4 $3, 128(%rax), %zmm0, %zmm0
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm5 {%k2}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vinserti32x4 $3, 64(%rax), %zmm0, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm4 {%k2}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vinserti32x4 $3, 192(%rax), %zmm0, %zmm0
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm7 {%k2}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vinserti32x4 $3, 256(%rax), %zmm17, %zmm0
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm10 {%k2}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-ONLY-FAST-NEXT:    vinserti32x4 $3, 320(%rax), %zmm26, %zmm0
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm13 {%k2}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm13, %zmm26
-; AVX512F-ONLY-FAST-NEXT:    vinserti32x4 $3, 384(%rax), %zmm20, %zmm0
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm24 {%k2}
-; AVX512F-ONLY-FAST-NEXT:    movb $56, %cl
-; AVX512F-ONLY-FAST-NEXT:    kmovw %ecx, %k2
+; AVX512F-ONLY-FAST-NEXT:    vinserti32x4 $3, 128(%rax), %zmm0, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm8 {%k2}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm9 {%k2}
+; AVX512F-ONLY-FAST-NEXT:    vinserti32x4 $3, 192(%rax), %zmm0, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm9 {%k2}
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm21 {%k2}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vinserti32x4 $3, 256(%rax), %zmm0, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm29 {%k2}
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm8 {%k2}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vinserti32x4 $3, 320(%rax), %zmm0, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm27 {%k2}
+; AVX512F-ONLY-FAST-NEXT:    vinserti32x4 $3, 384(%rax), %zmm20, %zmm3
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm23 {%k2}
+; AVX512F-ONLY-FAST-NEXT:    movb $56, %cl
+; AVX512F-ONLY-FAST-NEXT:    kmovw %ecx, %k2
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm14 {%k2}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm6 {%k2}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm18 {%k2}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k2}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, (%rsp) # 64-byte Spill
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm22 {%k2}
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm25 {%k2}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm21 {%k2}
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm28 {%k2}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm4, %zmm23 {%k2}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm17 {%k2}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm30, %zmm25 {%k2}
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],mem[6,7]
+; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm3 = ymm0[0,1,2,3,4,5],mem[6,7]
 ; AVX512F-ONLY-FAST-NEXT:    movb $14, %cl
 ; AVX512F-ONLY-FAST-NEXT:    kmovw %ecx, %k2
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm22 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm22 {%k2}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],mem[6,7]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm21 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm21 {%k2}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm20 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm20 {%k2}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5],mem[6,7]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm14 {%k2}
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],mem[6,7]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm19 {%k2}
+; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm3 = ymm0[0,1,2,3,4,5],mem[6,7]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm13 {%k2}
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],mem[6,7]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm15 {%k2}
+; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm3 = ymm0[0,1,2,3,4,5],mem[6,7]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm6 {%k2}
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],mem[6,7]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm8 {%k2}
+; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm3 = ymm0[0,1,2,3,4,5],mem[6,7]
+; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm28 {%k2}
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],mem[6,7]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm3 {%k2}
+; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm3 = ymm0[0,1,2,3,4,5],mem[6,7]
+; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm19 {%k2}
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],mem[6,7]
-; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm27 {%k2}
+; AVX512F-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm3 = ymm0[0,1,2,3,4,5],mem[6,7]
+; AVX512F-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm7 {%k2}
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
@@ -14393,1089 +14316,1093 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k1}
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm5 {%k1}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm7 {%k1}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm4 {%k1}
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm9 {%k1}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm8 {%k1}
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm10 {%k1}
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm12 {%k1}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm11 {%k1}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm5 {%k1}
 ; AVX512F-ONLY-FAST-NEXT:    movb $120, %al
 ; AVX512F-ONLY-FAST-NEXT:    kmovw %eax, %k1
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, %zmm18 {%k1}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, %zmm16 {%k1}
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm16 # 64-byte Folded Reload
-; AVX512F-ONLY-FAST-NEXT:    # zmm16 = zmm0[0,1,2,3],mem[4,5,6,7]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm14 {%k1}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm5, %zmm4 {%k1}
+; AVX512F-ONLY-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm9 # 64-byte Folded Reload
+; AVX512F-ONLY-FAST-NEXT:    # zmm9 = zmm0[0,1,2,3],mem[4,5,6,7]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm30 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm30 {%k1}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm4, %zmm3 {%k1}
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm7, %zmm2 {%k1}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm8, %zmm2 {%k1}
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm9, %zmm1 {%k1}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm10, %zmm1 {%k1}
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm10, %zmm0 {%k1}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm20 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm12, %zmm20 {%k1}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm11, %zmm0 {%k1}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm5, %zmm26 {%k1}
 ; AVX512F-ONLY-FAST-NEXT:    movb $-61, %al
 ; AVX512F-ONLY-FAST-NEXT:    kmovw %eax, %k1
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm4, %zmm9 {%k1}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm4 # 64-byte Folded Reload
+; AVX512F-ONLY-FAST-NEXT:    # zmm4 = zmm4[0,1,2,3],mem[4,5,6,7]
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm5, %zmm16 {%k1}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm5, %zmm4 {%k1}
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
 ; AVX512F-ONLY-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm5 # 64-byte Folded Reload
 ; AVX512F-ONLY-FAST-NEXT:    # zmm5 = zmm5[0,1,2,3],mem[4,5,6,7]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm7, %zmm5 {%k1}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm7, %zmm7 # 64-byte Folded Reload
-; AVX512F-ONLY-FAST-NEXT:    # zmm7 = zmm7[0,1,2,3],mem[4,5,6,7]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm9, %zmm7 {%k1}
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm9, %zmm9 # 64-byte Folded Reload
-; AVX512F-ONLY-FAST-NEXT:    # zmm9 = zmm9[0,1,2,3],mem[4,5,6,7]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm8, %zmm5 {%k1}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm8, %zmm8 # 64-byte Folded Reload
+; AVX512F-ONLY-FAST-NEXT:    # zmm8 = zmm8[0,1,2,3],mem[4,5,6,7]
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm10, %zmm9 {%k1}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm10, %zmm8 {%k1}
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
 ; AVX512F-ONLY-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm10 # 64-byte Folded Reload
 ; AVX512F-ONLY-FAST-NEXT:    # zmm10 = zmm10[0,1,2,3],mem[4,5,6,7]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm13, %zmm10 {%k1}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm11, %zmm10 {%k1}
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm11, %zmm11 # 64-byte Folded Reload
+; AVX512F-ONLY-FAST-NEXT:    # zmm11 = zmm11[0,1,2,3],mem[4,5,6,7]
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm12, %zmm13 # 64-byte Folded Reload
-; AVX512F-ONLY-FAST-NEXT:    # zmm13 = zmm12[0,1,2,3],mem[4,5,6,7]
-; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm31 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm31, %zmm13 {%k1}
-; AVX512F-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm17 = zmm29[0,1,2,3],zmm30[4,5,6,7]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm12, %zmm11 {%k1}
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm12, %zmm17 {%k1}
+; AVX512F-ONLY-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm12, %zmm12 # 64-byte Folded Reload
+; AVX512F-ONLY-FAST-NEXT:    # zmm12 = zmm12[0,1,2,3],mem[4,5,6,7]
+; AVX512F-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm18, %zmm12 {%k1}
 ; AVX512F-ONLY-FAST-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm17, 3008(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm20, 2944(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm27, 2880(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovaps %zmm12, 2816(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm23, 2752(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm24, 2688(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovaps %zmm12, 2624(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm13, 2560(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm12, 3008(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm26, 2944(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm7, 2880(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovaps %zmm7, 2816(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm25, 2752(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm23, 2688(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovaps %zmm7, 2624(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm11, 2560(%rax)
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, 2496(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, 2432(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovaps %zmm12, 2368(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm28, 2304(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm26, 2240(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm19, 2432(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovaps %zmm7, 2368(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm17, 2304(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm27, 2240(%rax)
 ; AVX512F-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-ONLY-FAST-NEXT:    vmovaps %zmm0, 2176(%rax)
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm10, 2112(%rax)
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, 2048(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm8, 1984(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovaps %zmm10, 1920(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm25, 1856(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovaps %zmm0, 1792(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm28, 1984(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovaps %zmm7, 1920(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm21, 1856(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm29, 1792(%rax)
 ; AVX512F-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-ONLY-FAST-NEXT:    vmovaps %zmm0, 1728(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm9, 1664(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm8, 1664(%rax)
 ; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, 1600(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm15, 1536(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovaps %zmm9, 1472(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovaps %zmm0, 1408(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm6, 1536(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovaps %zmm7, 1472(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm22, 1408(%rax)
 ; AVX512F-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-ONLY-FAST-NEXT:    vmovaps %zmm0, 1344(%rax)
 ; AVX512F-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-ONLY-FAST-NEXT:    vmovaps %zmm0, 1280(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm7, 1216(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm4, 1152(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm19, 1088(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovaps %zmm7, 1024(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm5, 1216(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, 1152(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm13, 1088(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovaps %zmm5, 1024(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovups (%rsp), %zmm0 # 64-byte Reload
 ; AVX512F-ONLY-FAST-NEXT:    vmovaps %zmm0, 960(%rax)
 ; AVX512F-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-ONLY-FAST-NEXT:    vmovaps %zmm0, 896(%rax)
 ; AVX512F-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-ONLY-FAST-NEXT:    vmovaps %zmm0, 832(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm5, 768(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm14, 704(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm21, 640(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovaps %zmm5, 576(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm4, 768(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm30, 704(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm14, 640(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovaps %zmm4, 576(%rax)
 ; AVX512F-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-ONLY-FAST-NEXT:    vmovaps %zmm0, 512(%rax)
 ; AVX512F-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-ONLY-FAST-NEXT:    vmovaps %zmm0, 448(%rax)
 ; AVX512F-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-ONLY-FAST-NEXT:    vmovaps %zmm0, 384(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm16, 320(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm18, 256(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm22, 192(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovaps %zmm0, 128(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm9, 320(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm16, 256(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm20, 192(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512F-ONLY-FAST-NEXT:    vmovaps %zmm3, 128(%rax)
 ; AVX512F-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-ONLY-FAST-NEXT:    vmovaps %zmm0, 64(%rax)
 ; AVX512F-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-ONLY-FAST-NEXT:    vmovaps %zmm0, (%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm11, 3520(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm15, 3520(%rax)
 ; AVX512F-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-ONLY-FAST-NEXT:    vmovaps %zmm0, 3456(%rax)
 ; AVX512F-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-ONLY-FAST-NEXT:    vmovaps %zmm0, 3392(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-ONLY-FAST-NEXT:    vmovaps %zmm0, 3328(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm31, 3328(%rax)
 ; AVX512F-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-ONLY-FAST-NEXT:    vmovaps %zmm0, 3264(%rax)
 ; AVX512F-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-ONLY-FAST-NEXT:    vmovaps %zmm0, 3200(%rax)
-; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm6, 3072(%rax)
+; AVX512F-ONLY-FAST-NEXT:    vmovdqa64 %zmm24, 3072(%rax)
 ; AVX512F-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-ONLY-FAST-NEXT:    vmovaps %zmm0, 3136(%rax)
-; AVX512F-ONLY-FAST-NEXT:    addq $6696, %rsp # imm = 0x1A28
+; AVX512F-ONLY-FAST-NEXT:    addq $6120, %rsp # imm = 0x17E8
 ; AVX512F-ONLY-FAST-NEXT:    vzeroupper
 ; AVX512F-ONLY-FAST-NEXT:    retq
 ;
 ; AVX512DQ-SLOW-LABEL: store_i64_stride7_vf64:
 ; AVX512DQ-SLOW:       # %bb.0:
-; AVX512DQ-SLOW-NEXT:    subq $6472, %rsp # imm = 0x1948
+; AVX512DQ-SLOW-NEXT:    subq $6280, %rsp # imm = 0x1888
 ; AVX512DQ-SLOW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 (%rdi), %zmm3
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 64(%rdi), %zmm21
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 (%rsi), %zmm4
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 64(%rsi), %zmm18
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm18, (%rsp) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 64(%rdx), %zmm13
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 (%rdi), %zmm7
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 64(%rdi), %zmm16
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 (%rsi), %zmm9
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 64(%rsi), %zmm13
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 64(%rdx), %zmm18
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm18, (%rsp) # 64-byte Spill
 ; AVX512DQ-SLOW-NEXT:    vmovdqa64 (%rdx), %zmm5
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 (%rcx), %zmm12
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 64(%rcx), %zmm20
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm20, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm17 = [11,3,11,3,11,3,11,3]
-; AVX512DQ-SLOW-NEXT:    # zmm17 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm27 = [2,10,0,3,2,10,0,3]
-; AVX512DQ-SLOW-NEXT:    # zmm27 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 (%rcx), %zmm6
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 64(%rcx), %zmm15
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm10 = [11,3,11,3,11,3,11,3]
+; AVX512DQ-SLOW-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm31 = [2,10,0,3,2,10,0,3]
+; AVX512DQ-SLOW-NEXT:    # zmm31 = mem[0,1,2,3,0,1,2,3]
 ; AVX512DQ-SLOW-NEXT:    movb $96, %r10b
 ; AVX512DQ-SLOW-NEXT:    kmovw %r10d, %k1
 ; AVX512DQ-SLOW-NEXT:    vmovdqa64 (%r8), %zmm1
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 (%r9), %zmm9
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 (%rax), %zmm6
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 64(%rax), %zmm7
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 (%r9), %zmm14
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 (%rax), %zmm3
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 64(%rax), %zmm4
+; AVX512DQ-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm25 = [15,7,15,7,15,7,15,7]
+; AVX512DQ-SLOW-NEXT:    # zmm25 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [9,1,9,1,9,1,9,1]
 ; AVX512DQ-SLOW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm9, %zmm0, %zmm2
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm11
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm14, %zmm0, %zmm2
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm8
 ; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [4,9,0,3,4,9,0,3]
 ; AVX512DQ-SLOW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm6, %zmm0, %zmm2
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm10
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm3, %zmm0, %zmm2
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm11
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm0
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm4, %zmm17, %zmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm0
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm9, %zmm10, %zmm0
 ; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm2
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm12, %zmm27, %zmm2
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm6, %zmm31, %zmm2
 ; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k1}
-; AVX512DQ-SLOW-NEXT:    vmovdqa (%r9), %ymm5
+; AVX512DQ-SLOW-NEXT:    vmovdqa (%r9), %ymm7
+; AVX512DQ-SLOW-NEXT:    vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa 64(%r9), %ymm5
 ; AVX512DQ-SLOW-NEXT:    vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa 64(%r9), %ymm3
-; AVX512DQ-SLOW-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512DQ-SLOW-NEXT:    vmovdqa (%r8), %ymm0
 ; AVX512DQ-SLOW-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa 64(%r8), %ymm4
-; AVX512DQ-SLOW-NEXT:    vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm5[0],ymm0[2],ymm5[2]
+; AVX512DQ-SLOW-NEXT:    vmovdqa 64(%r8), %ymm6
+; AVX512DQ-SLOW-NEXT:    vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm7[0],ymm0[2],ymm7[2]
 ; AVX512DQ-SLOW-NEXT:    movb $28, %r10b
 ; AVX512DQ-SLOW-NEXT:    kmovw %r10d, %k2
-; AVX512DQ-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm2 {%k2} = zmm0[2,3,2,3],zmm6[2,3,2,3]
+; AVX512DQ-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm2 {%k2} = zmm0[2,3,2,3],zmm3[2,3,2,3]
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [4,12,0,5,4,12,0,5]
-; AVX512DQ-SLOW-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm28 = [4,12,0,5,4,12,0,5]
+; AVX512DQ-SLOW-NEXT:    # zmm28 = mem[0,1,2,3,0,1,2,3]
 ; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm0
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm9, %zmm5
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm9, %zmm2, %zmm0
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm16
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm14, %zmm28, %zmm0
 ; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [0,1,12,7,0,1,12,7]
 ; AVX512DQ-SLOW-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm6, %zmm2, %zmm0
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm9
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm3, %zmm2, %zmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm12
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm19 = [5,0,14,6,5,0,14,6]
-; AVX512DQ-SLOW-NEXT:    # zmm19 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm2
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm14
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm1, %zmm19, %zmm2
-; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm15 = [0,13,6,7,0,13,6,7]
-; AVX512DQ-SLOW-NEXT:    # zmm15 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm6, %zmm15, %zmm2
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm5 = [15,7,15,7,15,7,15,7]
-; AVX512DQ-SLOW-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm14, %zmm5, %zmm1
-; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm8 = [6,13,14,7,6,13,14,7]
-; AVX512DQ-SLOW-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm1, %zmm8, %zmm6
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm21, %zmm1
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm18, %zmm17, %zmm1
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm13, %zmm2
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm20, %zmm27, %zmm2
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}
-; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm1 = ymm4[0],ymm3[0],ymm4[2],ymm3[2]
-; AVX512DQ-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm2 {%k2} = zmm1[2,3,2,3],zmm7[2,3,2,3]
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 64(%r8), %zmm1
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 64(%r9), %zmm3
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm3, %zmm11, %zmm2
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm7, %zmm10, %zmm2
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm3, %zmm16, %zmm2
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm7, %zmm9, %zmm2
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm2
+; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm17 = [5,0,14,6,5,0,14,6]
+; AVX512DQ-SLOW-NEXT:    # zmm17 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm1, %zmm17, %zmm0
+; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm24 = [0,13,6,7,0,13,6,7]
+; AVX512DQ-SLOW-NEXT:    # zmm24 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm3, %zmm24, %zmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm14, %zmm25, %zmm1
+; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm27 = [6,13,14,7,6,13,14,7]
+; AVX512DQ-SLOW-NEXT:    # zmm27 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm1, %zmm27, %zmm3
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm1, %zmm19, %zmm2
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm7, %zmm15, %zmm2
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm3, %zmm5, %zmm1
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm1, %zmm8, %zmm7
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 128(%rdi), %zmm22
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 128(%rsi), %zmm23
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm22, %zmm1
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm23, %zmm17, %zmm1
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm17, %zmm29
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 128(%rdx), %zmm18
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 128(%rcx), %zmm14
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm16, %zmm1
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm13, %zmm10, %zmm1
 ; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm18, %zmm2
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm14, %zmm27, %zmm2
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm15, %zmm31, %zmm2
 ; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 128(%rax), %zmm7
-; AVX512DQ-SLOW-NEXT:    vmovdqa 128(%r9), %ymm6
-; AVX512DQ-SLOW-NEXT:    vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa 128(%r8), %ymm1
-; AVX512DQ-SLOW-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm1 = ymm1[0],ymm6[0],ymm1[2],ymm6[2]
-; AVX512DQ-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm2 {%k2} = zmm1[2,3,2,3],zmm7[2,3,2,3]
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 128(%r8), %zmm1
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 128(%r9), %zmm6
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm6, %zmm11, %zmm2
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm31
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm7, %zmm10, %zmm2
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm10, %zmm30
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm6, %zmm16, %zmm2
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm7, %zmm9, %zmm2
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm2
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm1, %zmm19, %zmm2
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm7, %zmm15, %zmm2
+; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm1 = ymm6[0],ymm5[0],ymm6[2],ymm5[2]
+; AVX512DQ-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm2 {%k2} = zmm1[2,3,2,3],zmm4[2,3,2,3]
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 64(%r8), %zmm1
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 64(%r9), %zmm20
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm6
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm20, %zmm8, %zmm3
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm4, %zmm11, %zmm3
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm20, %zmm28, %zmm3
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm4, %zmm12, %zmm3
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm20, %zmm3
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm1, %zmm17, %zmm3
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm4, %zmm24, %zmm3
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm30
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm20, %zmm25, %zmm1
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm1, %zmm27, %zmm4
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 128(%rdi), %zmm1
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm6, %zmm5, %zmm1
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm1, %zmm8, %zmm7
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 192(%rdi), %zmm28
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 192(%rsi), %zmm21
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm28, %zmm1
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm21, %zmm17, %zmm1
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 192(%rdx), %zmm13
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 192(%rcx), %zmm26
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm13, %zmm2
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm26, %zmm27, %zmm2
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 192(%rax), %zmm6
-; AVX512DQ-SLOW-NEXT:    vmovdqa 192(%r9), %ymm4
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 128(%rsi), %zmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm0, %zmm10, %zmm1
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 128(%rdx), %zmm23
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 128(%rcx), %zmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm23, %zmm3
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm0, %zmm31, %zmm3
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm3 {%k1}
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 128(%rax), %zmm5
+; AVX512DQ-SLOW-NEXT:    vmovdqa 128(%r9), %ymm4
 ; AVX512DQ-SLOW-NEXT:    vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa 192(%r8), %ymm1
+; AVX512DQ-SLOW-NEXT:    vmovdqa 128(%r8), %ymm1
 ; AVX512DQ-SLOW-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm1 = ymm1[0],ymm4[0],ymm1[2],ymm4[2]
-; AVX512DQ-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm2 {%k2} = zmm1[2,3,2,3],zmm6[2,3,2,3]
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 192(%r8), %zmm1
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 192(%r9), %zmm3
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm3, %zmm11, %zmm2
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm6, %zmm10, %zmm2
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm3, %zmm16, %zmm2
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm6, %zmm9, %zmm2
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm9, %zmm0
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm2
+; AVX512DQ-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm3 {%k2} = zmm1[2,3,2,3],zmm5[2,3,2,3]
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm1, %zmm19, %zmm2
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm6, %zmm15, %zmm2
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm3, %zmm5, %zmm1
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm1, %zmm8, %zmm6
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 256(%rdi), %zmm25
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 256(%rsi), %zmm17
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm25, %zmm2
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm29, %zmm9
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm29, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm17, %zmm29, %zmm2
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 256(%rdx), %zmm11
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 256(%rcx), %zmm29
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm4
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm29, %zmm27, %zmm4
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm4 {%k1}
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 256(%rax), %zmm1
-; AVX512DQ-SLOW-NEXT:    vmovdqa 256(%r9), %ymm7
-; AVX512DQ-SLOW-NEXT:    vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa 256(%r8), %ymm2
-; AVX512DQ-SLOW-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm2 = ymm2[0],ymm7[0],ymm2[2],ymm7[2]
-; AVX512DQ-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm4 {%k2} = zmm2[2,3,2,3],zmm1[2,3,2,3]
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 128(%r8), %zmm1
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 128(%r9), %zmm13
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm4
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm13, %zmm8, %zmm4
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm5, %zmm11, %zmm4
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 256(%r8), %zmm2
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 256(%r9), %zmm6
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm4
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm31, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm6, %zmm31, %zmm4
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm1, %zmm10, %zmm4
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm4
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm13, %zmm28, %zmm4
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm12, %zmm18
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm5, %zmm12, %zmm4
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm8
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm4
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm6, %zmm16, %zmm4
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm1, %zmm0, %zmm4
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm3
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm13, %zmm5
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm1, %zmm17, %zmm5
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm8, %zmm24, %zmm5
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm22
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm13, %zmm25, %zmm1
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm1, %zmm27, %zmm8
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 192(%rdi), %zmm26
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 192(%rsi), %zmm0
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm4
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm2, %zmm19, %zmm4
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm1, %zmm15, %zmm4
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm26, %zmm1
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm0, %zmm10, %zmm1
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm10, %zmm9
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 192(%rdx), %zmm21
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 192(%rcx), %zmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm21, %zmm5
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm0, %zmm31, %zmm5
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm5 {%k1}
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 192(%rax), %zmm2
+; AVX512DQ-SLOW-NEXT:    vmovdqa 192(%r9), %ymm8
+; AVX512DQ-SLOW-NEXT:    vmovdqu %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa 192(%r8), %ymm1
+; AVX512DQ-SLOW-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm1 = ymm1[0],ymm8[0],ymm1[2],ymm8[2]
+; AVX512DQ-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm5 {%k2} = zmm1[2,3,2,3],zmm2[2,3,2,3]
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 192(%r8), %zmm1
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 192(%r9), %zmm8
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm16
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm8, %zmm6, %zmm16
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm2, %zmm11, %zmm16
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm15
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm8, %zmm28, %zmm15
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm2, %zmm12, %zmm15
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm6, %zmm5, %zmm2
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm2, %zmm8, %zmm1
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm12
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm1, %zmm17, %zmm12
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm0
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm2, %zmm24, %zmm12
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm10
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm8, %zmm25, %zmm1
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm1, %zmm27, %zmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 256(%rdi), %zmm15
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 256(%rsi), %zmm19
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm15, %zmm4
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm19, %zmm9, %zmm4
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm9, %zmm29
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 256(%rdx), %zmm14
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 256(%rcx), %zmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm1
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm0, %zmm31, %zmm1
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm1 {%k1}
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 256(%rax), %zmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqa 256(%r9), %ymm12
+; AVX512DQ-SLOW-NEXT:    vmovdqu %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa 256(%r8), %ymm4
+; AVX512DQ-SLOW-NEXT:    vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm4 = ymm4[0],ymm12[0],ymm4[2],ymm12[2]
+; AVX512DQ-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm1 {%k2} = zmm4[2,3,2,3],zmm0[2,3,2,3]
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 320(%rdi), %zmm7
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 320(%rsi), %zmm10
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm4
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm10, %zmm9, %zmm4
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 320(%rdx), %zmm6
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 320(%rcx), %zmm24
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm9
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm24, %zmm27, %zmm9
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm9 {%k1}
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 320(%rax), %zmm2
-; AVX512DQ-SLOW-NEXT:    vmovdqa 320(%r9), %ymm1
-; AVX512DQ-SLOW-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 256(%r8), %zmm4
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 256(%r9), %zmm12
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm1
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm12, %zmm6, %zmm1
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm0, %zmm11, %zmm1
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm2
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm1
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm12, %zmm28, %zmm1
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm0, %zmm18, %zmm1
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm12, %zmm1
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm4, %zmm17, %zmm1
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm0, %zmm24, %zmm1
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm24, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm9
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm12, %zmm25, %zmm4
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm4, %zmm27, %zmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 320(%rdi), %zmm16
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 320(%rsi), %zmm3
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm16, %zmm7
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm3, %zmm29, %zmm7
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 320(%rdx), %zmm4
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 320(%rcx), %zmm29
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm5
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm29, %zmm31, %zmm5
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm5 {%k1}
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 320(%rax), %zmm1
+; AVX512DQ-SLOW-NEXT:    vmovdqa 320(%r9), %ymm7
+; AVX512DQ-SLOW-NEXT:    vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512DQ-SLOW-NEXT:    vmovdqa 320(%r8), %ymm0
 ; AVX512DQ-SLOW-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm4 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
-; AVX512DQ-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm9 {%k2} = zmm4[2,3,2,3],zmm2[2,3,2,3]
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 320(%r8), %zmm4
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 320(%r9), %zmm1
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm0
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm1, %zmm31, %zmm0
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm2, %zmm30, %zmm0
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm0
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm1, %zmm16, %zmm0
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm2, %zmm3, %zmm0
+; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm7 = ymm0[0],ymm7[0],ymm0[2],ymm7[2]
+; AVX512DQ-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm5 {%k2} = zmm7[2,3,2,3],zmm1[2,3,2,3]
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 320(%r8), %zmm7
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 320(%r9), %zmm11
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm0
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm11, %zmm6, %zmm0
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm1, %zmm2, %zmm0
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm0
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm4, %zmm19, %zmm0
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm2, %zmm15, %zmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm0
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm11, %zmm28, %zmm0
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm1, %zmm18, %zmm0
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm1, %zmm5, %zmm4
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm4, %zmm8, %zmm2
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm9 = [13,5,13,5,13,5,13,5]
-; AVX512DQ-SLOW-NEXT:    # zmm9 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm0
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm12, %zmm9, %zmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm0
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm7, %zmm17, %zmm0
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm1, %zmm24, %zmm0
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [6,14,6,14,6,14,6,14]
-; AVX512DQ-SLOW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm1
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm12, %zmm0, %zmm1
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm2
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm11, %zmm25, %zmm7
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm7, %zmm27, %zmm1
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm4
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm0
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm12, %zmm5, %zmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 384(%rdx), %zmm5
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 384(%rcx), %zmm27
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm0
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm27, %zmm25, %zmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 384(%rdi), %zmm7
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 384(%rsi), %zmm1
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm1, %zmm25, %zmm7
+; AVX512DQ-SLOW-NEXT:    movb $24, %r10b
+; AVX512DQ-SLOW-NEXT:    kmovw %r10d, %k3
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm7 {%k3}
+; AVX512DQ-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm1 = [0,8,0,8,0,8,0,8]
+; AVX512DQ-SLOW-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 # 64-byte Folded Reload
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [3,0,12,4,3,0,12,4]
-; AVX512DQ-SLOW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm2, %zmm0, %zmm12
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm12, %zmm8
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm12, %zmm9, %zmm2
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm12, %zmm4, %zmm2
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm20, %zmm1, %zmm30
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm30, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm13, %zmm1, %zmm22
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm22, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm8, %zmm1, %zmm10
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm12, %zmm1, %zmm9
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm11, %zmm1, %zmm2
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm12, %zmm5, %zmm2
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 384(%r8), %zmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm2 = <u,1,2,3,4,15,u,u>
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm0, %zmm2, %zmm7
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 384(%r9), %zmm2
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm8 = <u,1,2,3,4,5,15,u>
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm2, %zmm8, %zmm7
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm2, %zmm0, %zmm1
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm2, %zmm0, %zmm6
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm2, %zmm0, %zmm28
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm28, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm0, %zmm17, %zmm2
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm1, %zmm0, %zmm12
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm12, %zmm15
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm18, %zmm1
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm14, %zmm9, %zmm1
+; AVX512DQ-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm6 = [13,5,13,5,13,5,13,5]
+; AVX512DQ-SLOW-NEXT:    # zmm6 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm1
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm17, %zmm6, %zmm1
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm18, %zmm1
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm14, %zmm4, %zmm1
+; AVX512DQ-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm9 = [6,14,6,14,6,14,6,14]
+; AVX512DQ-SLOW-NEXT:    # zmm9 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm1
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm17, %zmm9, %zmm1
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm18, %zmm1
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm14, %zmm5, %zmm1
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm1
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm17, %zmm25, %zmm1
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm18, %zmm0, %zmm14
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm18
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm13, %zmm1
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm26, %zmm9, %zmm1
+; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [3,0,12,4,3,0,12,4]
+; AVX512DQ-SLOW-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm0, %zmm2, %zmm17
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 (%rsp), %zmm0 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm1
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm20 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm20, %zmm6, %zmm1
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm13, %zmm1
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm26, %zmm4, %zmm1
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm1
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm20, %zmm9, %zmm1
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm13, %zmm1
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm26, %zmm5, %zmm1
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm1
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm20, %zmm25, %zmm1
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm13, %zmm0, %zmm26
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm1
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm29, %zmm9, %zmm1
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm0, %zmm2, %zmm20
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm23, %zmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm30 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm30, %zmm6, %zmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm23, %zmm0
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm30, %zmm9, %zmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm23, %zmm0
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm30, %zmm25, %zmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm23, %zmm2, %zmm30
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm21, %zmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm22 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm22, %zmm6, %zmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm21, %zmm0
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm22, %zmm9, %zmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm21, %zmm0
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm22, %zmm25, %zmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm21, %zmm2, %zmm22
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm23 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm23, %zmm6, %zmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm0
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm23, %zmm9, %zmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm0
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm23, %zmm25, %zmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm14, %zmm2, %zmm23
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm0
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm29, %zmm6, %zmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm0
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm29, %zmm9, %zmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm0
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm29, %zmm25, %zmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm4, %zmm2, %zmm29
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm29, %zmm18
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 448(%rdx), %zmm10
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 448(%rcx), %zmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm10, %zmm7
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm0, %zmm31, %zmm7
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm27, %zmm5, %zmm31
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm31, (%rsp) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm1
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm27, %zmm6, %zmm1
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm1
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm29, %zmm4, %zmm1
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm1
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm27, %zmm9, %zmm1
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm1
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm29, %zmm5, %zmm1
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm5, %zmm2, %zmm27
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm27, %zmm28
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm8
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm10, %zmm2, %zmm8
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm10, %zmm12
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm0, %zmm9, %zmm12
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm10, %zmm11
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm0, %zmm6, %zmm11
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm0, %zmm25, %zmm10
+; AVX512DQ-SLOW-NEXT:    movb $48, %r10b
+; AVX512DQ-SLOW-NEXT:    kmovw %r10d, %k4
+; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm5 = [1,0,10,2,1,0,10,2]
+; AVX512DQ-SLOW-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm1
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm0, %zmm5, %zmm1
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm11, %zmm0, %zmm29
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm1
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm24, %zmm9, %zmm1
+; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm17 {%k4} = zmm0[0],zmm2[0],zmm0[2],zmm2[2],zmm0[4],zmm2[4],zmm0[6],zmm2[6]
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm1
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm13
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm14
+; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [0,8,0,1,0,8,0,1]
+; AVX512DQ-SLOW-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm2, %zmm4, %zmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm2, %zmm6, %zmm1
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm1
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm24, %zmm4, %zmm1
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm2, %zmm9, %zmm13
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm2, %zmm25, %zmm14
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm1
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm0, %zmm5, %zmm1
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm1
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm24, %zmm5, %zmm1
+; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm20 {%k4} = zmm0[0],zmm2[0],zmm0[2],zmm2[2],zmm0[4],zmm2[4],zmm0[6],zmm2[6]
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm20, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm13
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm1
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm17
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm2, %zmm4, %zmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm2, %zmm6, %zmm13
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm2, %zmm9, %zmm1
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm6, %zmm0, %zmm24
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm24, %zmm6
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 448(%rdx), %zmm14
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 448(%rcx), %zmm1
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm12
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm1, %zmm27, %zmm12
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm19
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm1, %zmm4, %zmm19
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 384(%rdx), %zmm3
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 384(%rcx), %zmm20
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm20, %zmm3, %zmm27
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm27, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm2
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm20, %zmm9, %zmm2
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm20, %zmm3, %zmm4
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm27
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm20, %zmm5, %zmm27
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm3, %zmm0, %zmm20
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm3
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm14, %zmm0, %zmm3
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm16
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm1, %zmm5, %zmm14
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm1, %zmm9, %zmm16
-; AVX512DQ-SLOW-NEXT:    movb $48, %r10b
-; AVX512DQ-SLOW-NEXT:    kmovw %r10d, %k3
-; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm1 = [1,0,10,2,1,0,10,2]
-; AVX512DQ-SLOW-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm2
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm2, %zmm25, %zmm17
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm0, %zmm1, %zmm2
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm2
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm1, %zmm5, %zmm2
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm8 {%k3} = zmm0[0],zmm4[0],zmm0[2],zmm4[2],zmm0[4],zmm4[4],zmm0[6],zmm4[6]
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm5
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm8
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm11
-; AVX512DQ-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [0,8,0,1,0,8,0,1]
-; AVX512DQ-SLOW-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm4, %zmm2, %zmm0
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm4, %zmm9, %zmm5
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vbroadcasti128 {{.*#+}} ymm13 = [6,14,6,14]
-; AVX512DQ-SLOW-NEXT:    # ymm13 = mem[0,1,0,1]
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm4, %zmm13, %zmm8
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vbroadcasti128 {{.*#+}} ymm8 = [15,7,15,7]
-; AVX512DQ-SLOW-NEXT:    # ymm8 = mem[0,1,0,1]
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm4, %zmm8, %zmm11
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 (%rsp), %zmm4 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm5
+; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm30 {%k4} = zmm1[0],zmm0[0],zmm1[2],zmm0[2],zmm1[4],zmm0[4],zmm1[6],zmm0[6]
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm30, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm13
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm29
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm0, %zmm4, %zmm1
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm0, %zmm6, %zmm13
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm0, %zmm9, %zmm2
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm0, %zmm25, %zmm29
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm0, %zmm1, %zmm5
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm15 {%k3} = zmm0[0],zmm4[0],zmm0[2],zmm4[2],zmm0[4],zmm4[4],zmm0[6],zmm4[6]
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm15
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm5
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm24
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm4, %zmm2, %zmm0
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm4, %zmm9, %zmm15
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm4, %zmm13, %zmm5
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm4, %zmm8, %zmm24
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm24, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm23, %zmm0
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm22, %zmm1, %zmm0
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm18 {%k3} = zmm22[0],zmm23[0],zmm22[2],zmm23[2],zmm22[4],zmm23[4],zmm22[6],zmm23[6]
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm22, %zmm4
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm22, %zmm0
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm22, %zmm5
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm23, %zmm2, %zmm22
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm1
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm26, %zmm5, %zmm1
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm22 {%k4} = zmm26[0],zmm0[0],zmm26[2],zmm0[2],zmm26[4],zmm0[4],zmm26[6],zmm0[6]
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm22, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm23, %zmm9, %zmm4
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm23, %zmm13, %zmm0
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm23, %zmm8, %zmm5
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm21, %zmm0
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm28, %zmm1, %zmm0
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm0, (%rsp) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm26 {%k3} = zmm28[0],zmm21[0],zmm28[2],zmm21[2],zmm28[4],zmm21[4],zmm28[6],zmm21[6]
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm26, %zmm2
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm26, %zmm1
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm26, %zmm31
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm0, %zmm4, %zmm26
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm26, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm28, %zmm4
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm28, %zmm0
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm28, %zmm23
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm21, %zmm2, %zmm28
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm28, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm21, %zmm9, %zmm4
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm21, %zmm13, %zmm0
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm21, %zmm8, %zmm23
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm17, %zmm0
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm25, %zmm1, %zmm0
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm29 {%k3} = zmm25[0],zmm17[0],zmm25[2],zmm17[2],zmm25[4],zmm17[4],zmm25[6],zmm17[6]
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm29, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm25, %zmm31
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm25, %zmm0
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm25, %zmm29
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm25, %zmm30
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm17, %zmm2, %zmm30
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm17, %zmm9, %zmm31
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm17, %zmm13, %zmm0
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm17, %zmm8, %zmm29
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm10, %zmm26
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm7, %zmm1, %zmm26
-; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm6 {%k3} = zmm7[0],zmm10[0],zmm7[2],zmm10[2],zmm7[4],zmm10[4],zmm7[6],zmm10[6]
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm28
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm0
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm24
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm25
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm10, %zmm2, %zmm25
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm10, %zmm9, %zmm28
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm10, %zmm13, %zmm0
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm0, %zmm6, %zmm2
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm0, %zmm9, %zmm1
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm0, %zmm25, %zmm31
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm19, %zmm30
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm15, %zmm5, %zmm30
+; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm23 {%k4} = zmm15[0],zmm19[0],zmm15[2],zmm19[2],zmm15[4],zmm19[4],zmm15[6],zmm19[6]
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm23, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm15, %zmm22
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm15, %zmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm15, %zmm26
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm15, %zmm27
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm19, %zmm4, %zmm27
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm19, %zmm6, %zmm22
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm19, %zmm9, %zmm0
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm10, %zmm8, %zmm24
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 448(%rdi), %zmm6
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm19, %zmm25, %zmm26
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 448(%rdi), %zmm15
 ; AVX512DQ-SLOW-NEXT:    vmovdqa64 448(%rsi), %zmm0
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm7
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm0, %zmm10, %zmm7
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm4
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm0, %zmm13, %zmm4
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm5
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm0, %zmm9, %zmm5
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 384(%rdi), %zmm21
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 384(%rsi), %zmm17
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm17, %zmm22
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm21, %zmm1, %zmm22
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm17, %zmm21, %zmm10
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm17, %zmm21, %zmm9
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm17, %zmm21, %zmm13
-; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm20 {%k3} = zmm21[0],zmm17[0],zmm21[2],zmm17[2],zmm21[4],zmm17[4],zmm21[6],zmm17[6]
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm21, %zmm18
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm17, %zmm2, %zmm21
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm17, %zmm8, %zmm18
-; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm3 {%k3} = zmm6[0],zmm0[0],zmm6[2],zmm0[2],zmm6[4],zmm0[4],zmm6[6],zmm0[6]
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm0, %zmm6, %zmm2
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm6, %zmm0, %zmm1
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm0, %zmm8, %zmm6
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm12 {%k1}
-; AVX512DQ-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm4[0,1,2,3],zmm19[4,5,6,7]
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 448(%r8), %zmm4
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm7 = <0,11,u,u,4,5,6,7>
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm4, %zmm3, %zmm7
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 448(%r9), %zmm8
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm3 = <0,1,11,u,4,5,6,7>
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm8, %zmm7, %zmm3
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm15
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm15, %zmm13
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm0, %zmm25, %zmm13
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm16, %zmm23
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm3, %zmm25, %zmm23
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm24
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm16, %zmm5, %zmm24
+; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm18 {%k4} = zmm16[0],zmm3[0],zmm16[2],zmm3[2],zmm16[4],zmm3[4],zmm16[6],zmm3[6]
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm16, %zmm17
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm16, %zmm1
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm16, %zmm21
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm3, %zmm4, %zmm21
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm3, %zmm6, %zmm17
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm3, %zmm9, %zmm1
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm18, %zmm20
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm16, %zmm5, %zmm20
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm15, %zmm3
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm25 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm0, %zmm25, %zmm3
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm18, %zmm16, %zmm25
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm15, %zmm1
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm0, %zmm6, %zmm1
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm18, %zmm16, %zmm6
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm15, %zmm2
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm0, %zmm9, %zmm2
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm18, %zmm16, %zmm9
+; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm28 {%k4} = zmm16[0],zmm18[0],zmm16[2],zmm18[2],zmm16[4],zmm18[4],zmm16[6],zmm18[6]
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm28, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm16, %zmm28
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm18, %zmm4, %zmm28
+; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm8 {%k4} = zmm15[0],zmm0[0],zmm15[2],zmm0[2],zmm15[4],zmm0[4],zmm15[6],zmm0[6]
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm0, %zmm15, %zmm4
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm15, %zmm5, %zmm0
+; AVX512DQ-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm2[0,1,2,3],zmm12[4,5,6,7]
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm7 {%k1}
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 448(%r8), %zmm3
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm5 = <0,11,u,u,4,5,6,7>
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm3, %zmm8, %zmm5
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm1 {%k1}
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm8 = <12,u,u,3,4,5,6,13>
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm3, %zmm1, %zmm8
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm10, %zmm13 {%k3}
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm1 = <u,1,2,3,4,15,u,u>
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm3, %zmm1, %zmm13
+; AVX512DQ-SLOW-NEXT:    movb $6, %sil
+; AVX512DQ-SLOW-NEXT:    kmovw %esi, %k4
+; AVX512DQ-SLOW-NEXT:    vpbroadcastq 456(%rcx), %ymm1
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = mem[0,1,2,3],ymm1[4,5,6,7]
+; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm0 {%k4}
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm10 = <0,1,2,9,u,u,6,7>
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm3, %zmm0, %zmm10
+; AVX512DQ-SLOW-NEXT:    movb $64, %sil
+; AVX512DQ-SLOW-NEXT:    kmovw %esi, %k5
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm2 {%k5}
 ; AVX512DQ-SLOW-NEXT:    movb $4, %sil
-; AVX512DQ-SLOW-NEXT:    kmovw %esi, %k3
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm12 {%k3}
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm7 = <0,1,2,10,u,5,6,7>
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm8, %zmm12, %zmm7
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm16, %zmm5 {%k1}
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm3 = <12,u,u,3,4,5,6,13>
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm4, %zmm5, %zmm3
+; AVX512DQ-SLOW-NEXT:    kmovw %esi, %k5
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm7 {%k5}
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 448(%r9), %zmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm1 = <0,1,11,u,4,5,6,7>
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm0, %zmm5, %zmm1
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm11
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm3 = <0,1,2,10,u,5,6,7>
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm0, %zmm7, %zmm3
 ; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm5 = <0,12,u,3,4,5,6,7>
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm8, %zmm3, %zmm5
-; AVX512DQ-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm3 = [0,8,0,8,0,8,0,8]
-; AVX512DQ-SLOW-NEXT:    # zmm3 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm19 # 64-byte Folded Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm12 # 64-byte Folded Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm0, %zmm8, %zmm5
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm7 = <13,u,2,3,4,5,6,14>
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm0, %zmm2, %zmm7
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm1 = <u,1,2,3,4,5,15,u>
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm0, %zmm1, %zmm13
+; AVX512DQ-SLOW-NEXT:    movb $12, %sil
+; AVX512DQ-SLOW-NEXT:    kmovw %esi, %k5
+; AVX512DQ-SLOW-NEXT:    vmovdqa 448(%rdx), %xmm2
+; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],mem[0]
+; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm2, %zmm0, %zmm4 {%k5}
+; AVX512DQ-SLOW-NEXT:    vinserti32x4 $2, 448(%r8), %zmm4, %zmm2
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm4 = <0,1,2,3,4,8,u,7>
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm0, %zmm2, %zmm4
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm2 = <0,1,2,3,9,u,6,7>
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm0, %zmm10, %zmm2
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 384(%rax), %zmm0
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm10 # 64-byte Folded Reload
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm0, %zmm8, %zmm10
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 (%rsp), %zmm1 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm25, %zmm1 {%k1}
+; AVX512DQ-SLOW-NEXT:    vmovdqa 384(%r9), %ymm12
+; AVX512DQ-SLOW-NEXT:    vmovdqu %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa 384(%r8), %ymm8
+; AVX512DQ-SLOW-NEXT:    vmovdqu %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm8 = ymm8[0],ymm12[0],ymm8[2],ymm12[2]
+; AVX512DQ-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm1 {%k2} = zmm8[2,3,2,3],zmm0[2,3,2,3]
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm1, (%rsp) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm12 # 64-byte Folded Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm0, %zmm12, %zmm8
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm12 # 64-byte Folded Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm0, %zmm12, %zmm8
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm8 = [14,1,2,3,4,5,6,15]
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm12 # 64-byte Folded Reload
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm0, %zmm8, %zmm12
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    movb $24, %sil
-; AVX512DQ-SLOW-NEXT:    kmovw %esi, %k5
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm27, %zmm18 {%k5}
-; AVX512DQ-SLOW-NEXT:    movb $6, %sil
-; AVX512DQ-SLOW-NEXT:    kmovw %esi, %k3
-; AVX512DQ-SLOW-NEXT:    vpbroadcastq 456(%rcx), %ymm12
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm12 = mem[0,1,2,3],ymm12[4,5,6,7]
-; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm12, %zmm0, %zmm1 {%k3}
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm12 = <0,1,2,9,u,u,6,7>
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm4, %zmm1, %zmm12
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm6 {%k5}
-; AVX512DQ-SLOW-NEXT:    movb $64, %sil
-; AVX512DQ-SLOW-NEXT:    kmovw %esi, %k4
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm0 {%k4}
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm1 = <u,1,2,3,4,15,u,u>
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm4, %zmm1, %zmm6
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 384(%r8), %zmm4
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm4, %zmm1, %zmm18
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 384(%r9), %zmm1
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm1, %zmm4, %zmm3
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm1, %zmm4, %zmm14
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm1, %zmm4, %zmm16
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm4, %zmm1, %zmm17
-; AVX512DQ-SLOW-NEXT:    movb $12, %sil
-; AVX512DQ-SLOW-NEXT:    kmovw %esi, %k4
-; AVX512DQ-SLOW-NEXT:    vmovdqa 448(%rdx), %xmm4
-; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm4 = xmm4[0],mem[0]
-; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm4, %zmm0, %zmm2 {%k4}
-; AVX512DQ-SLOW-NEXT:    vinserti32x4 $2, 448(%r8), %zmm2, %zmm2
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm4 = <0,1,2,3,4,8,u,7>
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm8, %zmm2, %zmm4
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm2 = <u,1,2,3,4,5,15,u>
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm1, %zmm2, %zmm18
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm1 = <0,1,2,3,9,u,6,7>
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm8, %zmm12, %zmm1
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm12 = <13,u,2,3,4,5,6,14>
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm8, %zmm0, %zmm12
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm8, %zmm2, %zmm6
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 448(%rax), %zmm2
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [0,1,2,3,10,5,6,7]
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm2, %zmm7, %zmm0
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [0,1,12,3,4,5,6,7]
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm2, %zmm5, %zmm0
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 384(%rax), %zmm5
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm5, %zmm0, %zmm14
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm8
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm14 {%k1}
-; AVX512DQ-SLOW-NEXT:    vmovdqa 384(%r9), %ymm7
-; AVX512DQ-SLOW-NEXT:    vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa 384(%r8), %ymm0
-; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm7 = ymm0[0],ymm7[0],ymm0[2],ymm7[2]
-; AVX512DQ-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm14 {%k2} = zmm7[2,3,2,3],zmm5[2,3,2,3]
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm5, %zmm7, %zmm16
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm16, %zmm14
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm5, %zmm7, %zmm17
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm7 = [0,1,2,3,4,5,8,7]
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm2, %zmm4, %zmm7
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [14,1,2,3,4,5,6,15]
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm5, %zmm4, %zmm18
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [0,1,2,3,4,9,6,7]
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm2, %zmm1, %zmm5
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 448(%rax), %zmm0
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,1,2,3,10,5,6,7]
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm0, %zmm3, %zmm1
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,1,12,3,4,5,6,7]
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm0, %zmm5, %zmm1
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,13,2,3,4,5,6,7]
-; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm2, %zmm12, %zmm1
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm0, %zmm7, %zmm1
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm0, %zmm8, %zmm13
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,1,2,3,4,5,8,7]
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm0, %zmm4, %zmm1
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,1,2,3,4,9,6,7]
+; AVX512DQ-SLOW-NEXT:    vpermi2q %zmm0, %zmm2, %zmm1
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vpermt2q %zmm2, %zmm4, %zmm6
 ; AVX512DQ-SLOW-NEXT:    movb $8, %sil
 ; AVX512DQ-SLOW-NEXT:    kmovw %esi, %k2
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm15 {%k2}
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm11 {%k5}
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k5}
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k5}
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm23 {%k5}
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm29 {%k5}
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm11 {%k2}
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm14 {%k3}
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm24 {%k5}
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k3}
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm29 {%k3}
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm31 {%k3}
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm26 {%k3}
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm23 {%k3}
 ; AVX512DQ-SLOW-NEXT:    movb $-31, %sil
 ; AVX512DQ-SLOW-NEXT:    kmovw %esi, %k2
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm14 {%k2}
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k2}
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm29 {%k2}
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm29, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm31 {%k2}
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm26 {%k2}
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm23 {%k2}
+; AVX512DQ-SLOW-NEXT:    vmovdqa (%rdx), %xmm0
+; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
+; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm11 {%k2}
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k2}
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k2}
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm23 {%k2}
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm23, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm29 {%k2}
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm24 {%k2}
-; AVX512DQ-SLOW-NEXT:    vmovdqa (%rdx), %xmm1
-; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
-; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm7 {%k4}
-; AVX512DQ-SLOW-NEXT:    vmovdqa 64(%rdx), %xmm1
-; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
-; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
+; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm1 {%k5}
+; AVX512DQ-SLOW-NEXT:    vmovdqa 64(%rdx), %xmm0
+; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
+; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm2 {%k4}
-; AVX512DQ-SLOW-NEXT:    vmovdqa 128(%rdx), %xmm1
-; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
-; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
+; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm2 {%k5}
+; AVX512DQ-SLOW-NEXT:    vmovdqa 128(%rdx), %xmm0
+; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
+; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm3 {%k5}
+; AVX512DQ-SLOW-NEXT:    vmovdqa 192(%rdx), %xmm0
+; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
+; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm4 {%k4}
-; AVX512DQ-SLOW-NEXT:    vmovdqa 192(%rdx), %xmm1
-; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
-; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm5 {%k4}
-; AVX512DQ-SLOW-NEXT:    vmovdqa 256(%rdx), %xmm1
-; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
-; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm30 {%k4}
-; AVX512DQ-SLOW-NEXT:    vmovdqa 320(%rdx), %xmm1
-; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
-; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm25 {%k4}
-; AVX512DQ-SLOW-NEXT:    vmovdqa 384(%rdx), %xmm1
-; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
-; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm21 {%k4}
+; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm4 {%k5}
+; AVX512DQ-SLOW-NEXT:    vmovdqa 256(%rdx), %xmm0
+; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
+; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm27 {%k5}
+; AVX512DQ-SLOW-NEXT:    vmovdqa 320(%rdx), %xmm0
+; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
+; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm21 {%k5}
+; AVX512DQ-SLOW-NEXT:    vmovdqa 384(%rdx), %xmm0
+; AVX512DQ-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
+; AVX512DQ-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm28 {%k5}
 ; AVX512DQ-SLOW-NEXT:    movb $112, %sil
 ; AVX512DQ-SLOW-NEXT:    kmovw %esi, %k2
-; AVX512DQ-SLOW-NEXT:    vinserti64x2 $3, (%rax), %zmm19, %zmm7 {%k2}
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vinserti64x2 $3, 64(%rax), %zmm1, %zmm2 {%k2}
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vinserti64x2 $3, (%rax), %zmm0, %zmm1 {%k2}
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vinserti64x2 $3, 64(%rax), %zmm0, %zmm2 {%k2}
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vinserti64x2 $3, 128(%rax), %zmm10, %zmm4 {%k2}
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vinserti64x2 $3, 128(%rax), %zmm0, %zmm3 {%k2}
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vinserti64x2 $3, 192(%rax), %zmm0, %zmm4 {%k2}
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vinserti64x2 $3, 256(%rax), %zmm0, %zmm27 {%k2}
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vinserti64x2 $3, 320(%rax), %zmm0, %zmm21 {%k2}
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vinserti64x2 $3, 384(%rax), %zmm0, %zmm28 {%k2}
+; AVX512DQ-SLOW-NEXT:    vpbroadcastq 8(%rcx), %ymm0
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vinserti64x2 $3, 192(%rax), %zmm1, %zmm5 {%k2}
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vinserti64x2 $3, 256(%rax), %zmm1, %zmm30 {%k2}
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vinserti64x2 $3, 320(%rax), %zmm1, %zmm25 {%k2}
-; AVX512DQ-SLOW-NEXT:    vinserti64x2 $3, 384(%rax), %zmm3, %zmm21 {%k2}
-; AVX512DQ-SLOW-NEXT:    vpbroadcastq 8(%rcx), %ymm1
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = mem[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm5 {%k3}
-; AVX512DQ-SLOW-NEXT:    vpbroadcastq 72(%rcx), %ymm1
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = mem[0,1,2,3],ymm1[4,5,6,7]
+; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm1 {%k4}
+; AVX512DQ-SLOW-NEXT:    vpbroadcastq 72(%rcx), %ymm0
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm2 {%k3}
-; AVX512DQ-SLOW-NEXT:    vpbroadcastq 136(%rcx), %ymm1
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = mem[0,1,2,3],ymm1[4,5,6,7]
+; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm2 {%k4}
+; AVX512DQ-SLOW-NEXT:    vpbroadcastq 136(%rcx), %ymm0
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm3 {%k3}
-; AVX512DQ-SLOW-NEXT:    vpbroadcastq 200(%rcx), %ymm1
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = mem[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 (%rsp), %zmm4 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm4 {%k3}
-; AVX512DQ-SLOW-NEXT:    vpbroadcastq 264(%rcx), %ymm1
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = mem[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm7 {%k3}
-; AVX512DQ-SLOW-NEXT:    vpbroadcastq 328(%rcx), %ymm1
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = mem[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm26 {%k3}
-; AVX512DQ-SLOW-NEXT:    vpbroadcastq 392(%rcx), %ymm1
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = mem[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm22 {%k3}
+; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm3 {%k4}
+; AVX512DQ-SLOW-NEXT:    vpbroadcastq 200(%rcx), %ymm0
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm4 {%k4}
+; AVX512DQ-SLOW-NEXT:    vpbroadcastq 264(%rcx), %ymm0
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
+; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm30 {%k4}
+; AVX512DQ-SLOW-NEXT:    vpbroadcastq 328(%rcx), %ymm0
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
+; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm24 {%k4}
+; AVX512DQ-SLOW-NEXT:    vpbroadcastq 392(%rcx), %ymm0
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
+; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm20 {%k4}
 ; AVX512DQ-SLOW-NEXT:    movb $56, %cl
 ; AVX512DQ-SLOW-NEXT:    kmovw %ecx, %k2
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm5 {%k2}
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k2}
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k2}
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k2}
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm3 {%k2}
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm3 {%k2}
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm4 {%k2}
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm30 {%k2}
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm24 {%k2}
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm10, %zmm20 {%k2}
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k2}
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm4, (%rsp) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm7 {%k2}
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm26 {%k2}
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm22 {%k2}
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k1}
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm3 {%k1}
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm3 {%k1}
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm5 {%k1}
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm31 {%k1}
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm28 {%k1}
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm9 {%k1}
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm4 {%k1}
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm22 {%k1}
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm17 {%k1}
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm6 {%k1}
 ; AVX512DQ-SLOW-NEXT:    movb $120, %cl
 ; AVX512DQ-SLOW-NEXT:    kmovw %ecx, %k1
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm16 {%k1}
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm15 # 64-byte Folded Reload
-; AVX512DQ-SLOW-NEXT:    # zmm15 = zmm1[0,1,2,3],mem[4,5,6,7]
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm19 {%k1}
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm27 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm27 {%k1}
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm19 {%k1}
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm16 # 64-byte Folded Reload
+; AVX512DQ-SLOW-NEXT:    # zmm16 = zmm0[0,1,2,3],mem[4,5,6,7]
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm14 {%k1}
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm15 {%k1}
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm25 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm25 {%k1}
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm29 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm22, %zmm29 {%k1}
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm22 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm17, %zmm22 {%k1}
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm12 {%k1}
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm31, %zmm10 {%k1}
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm31 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm28, %zmm31 {%k1}
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm9, %zmm14 {%k1}
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm23
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm12 {%k1}
 ; AVX512DQ-SLOW-NEXT:    movb $-61, %cl
 ; AVX512DQ-SLOW-NEXT:    kmovw %ecx, %k1
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm15 {%k1}
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm14 # 64-byte Folded Reload
-; AVX512DQ-SLOW-NEXT:    # zmm14 = zmm1[0,1,2,3],mem[4,5,6,7]
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm16 {%k1}
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm11 # 64-byte Folded Reload
+; AVX512DQ-SLOW-NEXT:    # zmm11 = zmm0[0,1,2,3],mem[4,5,6,7]
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm14 {%k1}
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm11 # 64-byte Folded Reload
-; AVX512DQ-SLOW-NEXT:    # zmm11 = zmm1[0,1,2,3],mem[4,5,6,7]
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm11 {%k1}
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm10 # 64-byte Folded Reload
+; AVX512DQ-SLOW-NEXT:    # zmm10 = zmm0[0,1,2,3],mem[4,5,6,7]
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm11 {%k1}
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm4 # 64-byte Folded Reload
-; AVX512DQ-SLOW-NEXT:    # zmm4 = zmm1[0,1,2,3],mem[4,5,6,7]
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm10 {%k1}
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm4 # 64-byte Folded Reload
+; AVX512DQ-SLOW-NEXT:    # zmm4 = zmm0[0,1,2,3],mem[4,5,6,7]
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
 ; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm4 {%k1}
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm5 # 64-byte Folded Reload
-; AVX512DQ-SLOW-NEXT:    # zmm5 = zmm1[0,1,2,3],mem[4,5,6,7]
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm5 # 64-byte Folded Reload
+; AVX512DQ-SLOW-NEXT:    # zmm5 = zmm0[0,1,2,3],mem[4,5,6,7]
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm5 {%k1}
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm6 # 64-byte Folded Reload
+; AVX512DQ-SLOW-NEXT:    # zmm6 = zmm0[0,1,2,3],mem[4,5,6,7]
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm5 {%k1}
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm7 # 64-byte Folded Reload
-; AVX512DQ-SLOW-NEXT:    # zmm7 = zmm1[0,1,2,3],mem[4,5,6,7]
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm6 {%k1}
+; AVX512DQ-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm9, %zmm7 # 64-byte Folded Reload
+; AVX512DQ-SLOW-NEXT:    # zmm7 = zmm9[0,1,2,3],mem[4,5,6,7]
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
 ; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm7 {%k1}
-; AVX512DQ-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm13, %zmm8 # 64-byte Folded Reload
-; AVX512DQ-SLOW-NEXT:    # zmm8 = zmm13[0,1,2,3],mem[4,5,6,7]
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm17, %zmm8 {%k1}
-; AVX512DQ-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
-; AVX512DQ-SLOW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm9 # 32-byte Folded Reload
-; AVX512DQ-SLOW-NEXT:    # ymm9 = ymm9[1],mem[1],ymm9[3],mem[3]
-; AVX512DQ-SLOW-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[0,2,3,3]
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3,4,5],mem[6,7]
+; AVX512DQ-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
+; AVX512DQ-SLOW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm8 # 32-byte Folded Reload
+; AVX512DQ-SLOW-NEXT:    # ymm8 = ymm8[1],mem[1],ymm8[3],mem[3]
+; AVX512DQ-SLOW-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,2,3,3]
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],mem[6,7]
 ; AVX512DQ-SLOW-NEXT:    movb $14, %cl
 ; AVX512DQ-SLOW-NEXT:    kmovw %ecx, %k1
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm9, %zmm0, %zmm13 {%k1}
-; AVX512DQ-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
-; AVX512DQ-SLOW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm9 # 32-byte Folded Reload
-; AVX512DQ-SLOW-NEXT:    # ymm9 = ymm9[1],mem[1],ymm9[3],mem[3]
-; AVX512DQ-SLOW-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[0,2,3,3]
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3,4,5],mem[6,7]
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm8, %zmm0, %zmm9 {%k1}
+; AVX512DQ-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
+; AVX512DQ-SLOW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm8 # 32-byte Folded Reload
+; AVX512DQ-SLOW-NEXT:    # ymm8 = ymm8[1],mem[1],ymm8[3],mem[3]
+; AVX512DQ-SLOW-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,2,3,3]
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],mem[6,7]
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm9, %zmm0, %zmm17 {%k1}
-; AVX512DQ-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
-; AVX512DQ-SLOW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm9 # 32-byte Folded Reload
-; AVX512DQ-SLOW-NEXT:    # ymm9 = ymm9[1],mem[1],ymm9[3],mem[3]
-; AVX512DQ-SLOW-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[0,2,3,3]
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3,4,5],mem[6,7]
-; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm28 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm9, %zmm0, %zmm28 {%k1}
-; AVX512DQ-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
-; AVX512DQ-SLOW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm9 # 32-byte Folded Reload
-; AVX512DQ-SLOW-NEXT:    # ymm9 = ymm9[1],mem[1],ymm9[3],mem[3]
-; AVX512DQ-SLOW-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[0,2,3,3]
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3,4,5],mem[6,7]
+; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm8, %zmm0, %zmm17 {%k1}
+; AVX512DQ-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
+; AVX512DQ-SLOW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm8 # 32-byte Folded Reload
+; AVX512DQ-SLOW-NEXT:    # ymm8 = ymm8[1],mem[1],ymm8[3],mem[3]
+; AVX512DQ-SLOW-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,2,3,3]
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],mem[6,7]
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm8, %zmm0, %zmm18 {%k1}
+; AVX512DQ-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
+; AVX512DQ-SLOW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm8 # 32-byte Folded Reload
+; AVX512DQ-SLOW-NEXT:    # ymm8 = ymm8[1],mem[1],ymm8[3],mem[3]
+; AVX512DQ-SLOW-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,2,3,3]
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],mem[6,7]
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm9, %zmm0, %zmm3 {%k1}
-; AVX512DQ-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
-; AVX512DQ-SLOW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm9 # 32-byte Folded Reload
-; AVX512DQ-SLOW-NEXT:    # ymm9 = ymm9[1],mem[1],ymm9[3],mem[3]
-; AVX512DQ-SLOW-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[0,2,3,3]
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3,4,5],mem[6,7]
+; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm8, %zmm0, %zmm3 {%k1}
+; AVX512DQ-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
+; AVX512DQ-SLOW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm8 # 32-byte Folded Reload
+; AVX512DQ-SLOW-NEXT:    # ymm8 = ymm8[1],mem[1],ymm8[3],mem[3]
+; AVX512DQ-SLOW-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,2,3,3]
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],mem[6,7]
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm9, %zmm0, %zmm2 {%k1}
-; AVX512DQ-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
-; AVX512DQ-SLOW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm9 # 32-byte Folded Reload
-; AVX512DQ-SLOW-NEXT:    # ymm9 = ymm9[1],mem[1],ymm9[3],mem[3]
-; AVX512DQ-SLOW-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[0,2,3,3]
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3,4,5],mem[6,7]
+; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm8, %zmm0, %zmm2 {%k1}
+; AVX512DQ-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
+; AVX512DQ-SLOW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm8 # 32-byte Folded Reload
+; AVX512DQ-SLOW-NEXT:    # ymm8 = ymm8[1],mem[1],ymm8[3],mem[3]
+; AVX512DQ-SLOW-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,2,3,3]
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],mem[6,7]
 ; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm9, %zmm0, %zmm1 {%k1}
-; AVX512DQ-SLOW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
-; AVX512DQ-SLOW-NEXT:    # ymm0 = ymm0[1],mem[1],ymm0[3],mem[3]
-; AVX512DQ-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,3,3]
-; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],mem[6,7]
-; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm20 {%k1}
+; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm8, %zmm0, %zmm1 {%k1}
+; AVX512DQ-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512DQ-SLOW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm8 # 32-byte Folded Reload
+; AVX512DQ-SLOW-NEXT:    # ymm8 = ymm0[1],mem[1],ymm0[3],mem[3]
+; AVX512DQ-SLOW-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,2,3,3]
+; AVX512DQ-SLOW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],mem[6,7]
+; AVX512DQ-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vinserti64x4 $0, %ymm8, %zmm0, %zmm0 {%k1}
 ; AVX512DQ-SLOW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm8, 3008(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm23, 2944(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm20, 2880(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm7, 3008(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm12, 2944(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm0, 2880(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovups (%rsp), %zmm0 # 64-byte Reload
 ; AVX512DQ-SLOW-NEXT:    vmovaps %zmm0, 2816(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm22, 2752(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm21, 2688(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm24, 2624(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm7, 2560(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm31, 2496(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm20, 2752(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm28, 2688(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm23, 2624(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm6, 2560(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm22, 2496(%rax)
 ; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm1, 2432(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovaps %zmm0, 2368(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm26, 2304(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm25, 2240(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm29, 2176(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vmovaps %zmm6, 2368(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm24, 2304(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm21, 2240(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm26, 2176(%rax)
 ; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm5, 2112(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm10, 2048(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm29, 2048(%rax)
 ; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm2, 1984(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovaps %zmm0, 1920(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovaps %zmm0, 1856(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm30, 1792(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovaps %zmm0, 1728(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vmovaps %zmm5, 1920(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm30, 1856(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm27, 1792(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm31, 1728(%rax)
 ; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm4, 1664(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm12, 1600(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm25, 1600(%rax)
 ; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm3, 1536(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vmovaps %zmm4, 1472(%rax)
 ; AVX512DQ-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovaps %zmm0, 1472(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovups (%rsp), %zmm0 # 64-byte Reload
 ; AVX512DQ-SLOW-NEXT:    vmovaps %zmm0, 1408(%rax)
 ; AVX512DQ-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQ-SLOW-NEXT:    vmovaps %zmm0, 1344(%rax)
 ; AVX512DQ-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQ-SLOW-NEXT:    vmovaps %zmm0, 1280(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm11, 1216(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm27, 1152(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm28, 1088(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovaps %zmm0, 1024(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm10, 1216(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm15, 1152(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm18, 1088(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vmovaps %zmm3, 1024(%rax)
 ; AVX512DQ-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQ-SLOW-NEXT:    vmovaps %zmm0, 960(%rax)
 ; AVX512DQ-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQ-SLOW-NEXT:    vmovaps %zmm0, 896(%rax)
 ; AVX512DQ-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQ-SLOW-NEXT:    vmovaps %zmm0, 832(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm14, 768(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm19, 704(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm11, 768(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm14, 704(%rax)
 ; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm17, 640(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-SLOW-NEXT:    vmovaps %zmm0, 576(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vmovaps %zmm2, 576(%rax)
 ; AVX512DQ-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQ-SLOW-NEXT:    vmovaps %zmm0, 512(%rax)
 ; AVX512DQ-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQ-SLOW-NEXT:    vmovaps %zmm0, 448(%rax)
 ; AVX512DQ-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQ-SLOW-NEXT:    vmovaps %zmm0, 384(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm15, 320(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm16, 256(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm13, 192(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm16, 320(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm19, 256(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm9, 192(%rax)
 ; AVX512DQ-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQ-SLOW-NEXT:    vmovaps %zmm0, 128(%rax)
 ; AVX512DQ-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQ-SLOW-NEXT:    vmovaps %zmm0, 64(%rax)
 ; AVX512DQ-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQ-SLOW-NEXT:    vmovaps %zmm0, (%rax)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm6, 3520(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm13, 3520(%rax)
 ; AVX512DQ-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQ-SLOW-NEXT:    vmovaps %zmm0, 3456(%rax)
 ; AVX512DQ-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
@@ -15486,815 +15413,809 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-SLOW-NEXT:    vmovaps %zmm0, 3264(%rax)
 ; AVX512DQ-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQ-SLOW-NEXT:    vmovaps %zmm0, 3200(%rax)
-; AVX512DQ-SLOW-NEXT:    vmovdqa64 %zmm18, 3072(%rax)
+; AVX512DQ-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-SLOW-NEXT:    vmovaps %zmm0, 3072(%rax)
 ; AVX512DQ-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQ-SLOW-NEXT:    vmovaps %zmm0, 3136(%rax)
-; AVX512DQ-SLOW-NEXT:    addq $6472, %rsp # imm = 0x1948
+; AVX512DQ-SLOW-NEXT:    addq $6280, %rsp # imm = 0x1888
 ; AVX512DQ-SLOW-NEXT:    vzeroupper
 ; AVX512DQ-SLOW-NEXT:    retq
 ;
 ; AVX512DQ-FAST-LABEL: store_i64_stride7_vf64:
 ; AVX512DQ-FAST:       # %bb.0:
-; AVX512DQ-FAST-NEXT:    subq $6568, %rsp # imm = 0x19A8
+; AVX512DQ-FAST-NEXT:    subq $6120, %rsp # imm = 0x17E8
 ; AVX512DQ-FAST-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-FAST-NEXT:    vmovdqa64 (%rdi), %zmm5
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 64(%rdi), %zmm11
+; AVX512DQ-FAST-NEXT:    vmovdqa64 (%rdi), %zmm6
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 64(%rdi), %zmm12
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 (%rsi), %zmm14
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 64(%rsi), %zmm16
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 64(%rdx), %zmm11
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 (%rsi), %zmm26
-; AVX512DQ-FAST-NEXT:    vmovdqa64 64(%rsi), %zmm22
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm22, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 64(%rdx), %zmm7
+; AVX512DQ-FAST-NEXT:    vmovdqa64 (%rdx), %zmm7
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 (%rdx), %zmm6
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 (%rcx), %zmm9
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 64(%rcx), %zmm19
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm10 = [11,3,11,3,11,3,11,3]
-; AVX512DQ-FAST-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm12 = [2,10,0,3,2,10,0,3]
-; AVX512DQ-FAST-NEXT:    # zmm12 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-FAST-NEXT:    vmovdqa64 (%rcx), %zmm13
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 64(%rcx), %zmm21
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm8 = [11,3,11,3,11,3,11,3]
+; AVX512DQ-FAST-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [2,10,0,3,2,10,0,3]
+; AVX512DQ-FAST-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
 ; AVX512DQ-FAST-NEXT:    movb $96, %r10b
 ; AVX512DQ-FAST-NEXT:    kmovw %r10d, %k1
 ; AVX512DQ-FAST-NEXT:    vmovdqa64 (%r8), %zmm1
-; AVX512DQ-FAST-NEXT:    vmovdqa64 (%r9), %zmm8
+; AVX512DQ-FAST-NEXT:    vmovdqa64 (%r9), %zmm22
 ; AVX512DQ-FAST-NEXT:    vmovdqa64 (%rax), %zmm3
-; AVX512DQ-FAST-NEXT:    vmovdqa64 64(%rax), %zmm4
+; AVX512DQ-FAST-NEXT:    vmovdqa64 64(%rax), %zmm5
+; AVX512DQ-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm23 = [15,7,15,7,15,7,15,7]
+; AVX512DQ-FAST-NEXT:    # zmm23 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [9,1,9,1,9,1,9,1]
 ; AVX512DQ-FAST-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm1, %zmm2
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm8, %zmm0, %zmm2
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm16
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm22, %zmm0, %zmm2
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm9
 ; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [4,9,0,3,4,9,0,3]
 ; AVX512DQ-FAST-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
 ; AVX512DQ-FAST-NEXT:    vpermt2q %zmm3, %zmm0, %zmm2
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm13
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm10
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm5, %zmm0
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm26, %zmm10, %zmm0
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm6, %zmm2
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm9, %zmm12, %zmm2
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm6, %zmm0
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm14, %zmm8, %zmm0
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm7, %zmm2
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm13, %zmm4, %zmm2
 ; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k1}
-; AVX512DQ-FAST-NEXT:    vmovdqa (%r9), %ymm0
+; AVX512DQ-FAST-NEXT:    vmovdqa (%r9), %ymm7
+; AVX512DQ-FAST-NEXT:    vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa 64(%r9), %ymm13
+; AVX512DQ-FAST-NEXT:    vmovdqu %ymm13, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa (%r8), %ymm0
 ; AVX512DQ-FAST-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa 64(%r9), %ymm9
-; AVX512DQ-FAST-NEXT:    vmovdqu %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa (%r8), %ymm6
+; AVX512DQ-FAST-NEXT:    vmovdqa 64(%r8), %ymm6
 ; AVX512DQ-FAST-NEXT:    vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 64(%r8), %ymm30
-; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm6[0],ymm0[0],ymm6[2],ymm0[2]
+; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm7[0],ymm0[2],ymm7[2]
 ; AVX512DQ-FAST-NEXT:    movb $28, %r10b
 ; AVX512DQ-FAST-NEXT:    kmovw %r10d, %k2
 ; AVX512DQ-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm2 {%k2} = zmm0[2,3,2,3],zmm3[2,3,2,3]
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [4,12,0,5,4,12,0,5]
-; AVX512DQ-FAST-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm1, %zmm0
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm8, %zmm6
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm8, %zmm2, %zmm0
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm2, %zmm20
-; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [0,1,12,7,0,1,12,7]
-; AVX512DQ-FAST-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm3, %zmm2, %zmm0
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm2, %zmm17
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [5,0,14,6,5,0,14,6]
+; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [4,12,0,5,4,12,0,5]
 ; AVX512DQ-FAST-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm8, %zmm2
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm1, %zmm0, %zmm2
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm21
-; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [0,13,6,7,0,13,6,7]
-; AVX512DQ-FAST-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm3, %zmm0, %zmm2
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm18
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm1, %zmm2
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm22, %zmm0, %zmm2
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm7
+; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm15 = [0,1,12,7,0,1,12,7]
+; AVX512DQ-FAST-NEXT:    # zmm15 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm3, %zmm15, %zmm2
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm8 = [15,7,15,7,15,7,15,7]
-; AVX512DQ-FAST-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm6, %zmm8, %zmm1
-; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm14 = [6,13,14,7,6,13,14,7]
-; AVX512DQ-FAST-NEXT:    # zmm14 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm1, %zmm14, %zmm3
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm11, %zmm1
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm22, %zmm10, %zmm1
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm7, %zmm2
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm12, %zmm7
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm19, %zmm12, %zmm2
+; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm20 = [5,0,14,6,5,0,14,6]
+; AVX512DQ-FAST-NEXT:    # zmm20 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm22, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm22, %zmm2
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm1, %zmm20, %zmm2
+; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm19 = [0,13,6,7,0,13,6,7]
+; AVX512DQ-FAST-NEXT:    # zmm19 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm3, %zmm19, %zmm2
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm3, %zmm2
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm22, %zmm23, %zmm1
+; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm18 = [6,13,14,7,6,13,14,7]
+; AVX512DQ-FAST-NEXT:    # zmm18 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm1, %zmm18, %zmm2
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm12, %zmm1
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm16, %zmm8, %zmm1
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm11, %zmm2
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm21, %zmm4, %zmm2
 ; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}
-; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm1 = ymm30[0],ymm9[0],ymm30[2],ymm9[2]
-; AVX512DQ-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm2 {%k2} = zmm1[2,3,2,3],zmm4[2,3,2,3]
+; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm1 = ymm6[0],ymm13[0],ymm6[2],ymm13[2]
+; AVX512DQ-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm2 {%k2} = zmm1[2,3,2,3],zmm5[2,3,2,3]
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-FAST-NEXT:    vmovdqa64 64(%r8), %zmm1
-; AVX512DQ-FAST-NEXT:    vmovdqa64 64(%r9), %zmm3
+; AVX512DQ-FAST-NEXT:    vmovdqa64 64(%r9), %zmm27
 ; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm1, %zmm2
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm3, %zmm16, %zmm2
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm4, %zmm13, %zmm2
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm27, %zmm9, %zmm2
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm5, %zmm10, %zmm2
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm1, %zmm2
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm3, %zmm20, %zmm2
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm4, %zmm17, %zmm2
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm27, %zmm0, %zmm2
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm5, %zmm15, %zmm2
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm3, %zmm2
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm1, %zmm21, %zmm2
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm4, %zmm0, %zmm2
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm27, %zmm2
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm1, %zmm20, %zmm2
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm5, %zmm19, %zmm2
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm3, %zmm8, %zmm1
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm1, %zmm14, %zmm4
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm27, %zmm23, %zmm1
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm1, %zmm18, %zmm5
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-FAST-NEXT:    vmovdqa64 128(%rdi), %zmm1
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 128(%rsi), %zmm19
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm19, %zmm10, %zmm1
-; AVX512DQ-FAST-NEXT:    vmovdqa64 128(%rdx), %zmm3
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 128(%rcx), %zmm27
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm27, %zmm12, %zmm3
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm1, %zmm3 {%k1}
-; AVX512DQ-FAST-NEXT:    vmovdqa64 128(%rax), %zmm5
+; AVX512DQ-FAST-NEXT:    vmovdqa64 128(%rsi), %zmm26
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm26, %zmm8, %zmm1
+; AVX512DQ-FAST-NEXT:    vmovdqa64 128(%rdx), %zmm2
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 128(%rcx), %zmm0
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm0, %zmm4, %zmm2
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}
+; AVX512DQ-FAST-NEXT:    vmovdqa64 128(%rax), %zmm6
 ; AVX512DQ-FAST-NEXT:    vmovdqa 128(%r9), %ymm0
 ; AVX512DQ-FAST-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 128(%r8), %ymm24
-; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm2 = ymm24[0],ymm0[0],ymm24[2],ymm0[2]
-; AVX512DQ-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm3 {%k2} = zmm2[2,3,2,3],zmm5[2,3,2,3]
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 128(%r8), %zmm2
-; AVX512DQ-FAST-NEXT:    vmovdqa64 128(%r9), %zmm0
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm16, %zmm4
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm0, %zmm16, %zmm3
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm5, %zmm13, %zmm3
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm0, %zmm20, %zmm3
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm5, %zmm17, %zmm3
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm3
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm2, %zmm21, %zmm3
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm5, %zmm18, %zmm3
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa 128(%r8), %ymm5
+; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm1 = ymm5[0],ymm0[0],ymm5[2],ymm0[2]
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %ymm5, %ymm30
+; AVX512DQ-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm2 {%k2} = zmm1[2,3,2,3],zmm6[2,3,2,3]
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm0, %zmm8, %zmm2
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm2, %zmm14, %zmm5
+; AVX512DQ-FAST-NEXT:    vmovdqa64 128(%r8), %zmm2
+; AVX512DQ-FAST-NEXT:    vmovdqa64 128(%r9), %zmm16
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm2, %zmm5
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm16, %zmm9, %zmm5
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm9, %zmm1
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm6, %zmm10, %zmm5
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm2, %zmm5
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm16, %zmm7, %zmm5
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm6, %zmm15, %zmm5
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm16, %zmm5
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm2, %zmm20, %zmm5
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm6, %zmm19, %zmm5
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm16, %zmm23, %zmm2
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm2, %zmm18, %zmm6
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-FAST-NEXT:    vmovdqa64 192(%rdi), %zmm2
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 192(%rsi), %zmm16
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm10, %zmm1
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm16, %zmm10, %zmm2
+; AVX512DQ-FAST-NEXT:    vmovdqa64 192(%rsi), %zmm28
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm28, %zmm8, %zmm2
 ; AVX512DQ-FAST-NEXT:    vmovdqa64 192(%rdx), %zmm5
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 192(%rcx), %zmm15
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm15, %zmm12, %zmm5
+; AVX512DQ-FAST-NEXT:    vmovdqa64 192(%rcx), %zmm0
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm0, %zmm4, %zmm5
 ; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm2, %zmm5 {%k1}
-; AVX512DQ-FAST-NEXT:    vmovdqa64 192(%rax), %zmm23
-; AVX512DQ-FAST-NEXT:    vmovdqa64 192(%r9), %ymm25
-; AVX512DQ-FAST-NEXT:    vmovdqa 192(%r8), %ymm11
-; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm3 = ymm11[0],ymm25[0],ymm11[2],ymm25[2]
-; AVX512DQ-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm5 {%k2} = zmm3[2,3,2,3],zmm23[2,3,2,3]
+; AVX512DQ-FAST-NEXT:    vmovdqa64 192(%rax), %zmm22
+; AVX512DQ-FAST-NEXT:    vmovdqa64 192(%r9), %ymm24
+; AVX512DQ-FAST-NEXT:    vmovdqa 192(%r8), %ymm9
+; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm2 = ymm9[0],ymm24[0],ymm9[2],ymm24[2]
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %ymm9, %ymm25
+; AVX512DQ-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm5 {%k2} = zmm2[2,3,2,3],zmm22[2,3,2,3]
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 192(%r8), %zmm3
-; AVX512DQ-FAST-NEXT:    vmovdqa64 192(%r9), %zmm6
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm3, %zmm5
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm6, %zmm4, %zmm5
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm4, %zmm2
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm23, %zmm13, %zmm5
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm13, %zmm0
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm3, %zmm4
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm6, %zmm5
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm6, %zmm20, %zmm4
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm20, %zmm6
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm17, %zmm10
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm23, %zmm17, %zmm4
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm5, %zmm4
+; AVX512DQ-FAST-NEXT:    vmovdqa64 192(%r8), %zmm5
+; AVX512DQ-FAST-NEXT:    vmovdqa64 192(%r9), %zmm13
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm5, %zmm9
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm13, %zmm1, %zmm9
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm22, %zmm10, %zmm9
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm10, %zmm0
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm5, %zmm12
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm7, %zmm6
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm13, %zmm7, %zmm12
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm15, %zmm9
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm22, %zmm15, %zmm12
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm13, %zmm7
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm5, %zmm20, %zmm7
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm22, %zmm19, %zmm7
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm5, %zmm17
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm13, %zmm23, %zmm5
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm5, %zmm18, %zmm22
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm22, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 256(%rdi), %zmm5
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm3, %zmm21, %zmm4
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm23, %zmm18, %zmm4
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 256(%rsi), %zmm3
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm5, %zmm8, %zmm3
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm3, %zmm14, %zmm23
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm23, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 256(%rdi), %zmm3
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 256(%rsi), %zmm31
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm31, %zmm1, %zmm3
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm1, %zmm22
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 256(%rdx), %zmm28
-; AVX512DQ-FAST-NEXT:    vmovdqa64 256(%rcx), %zmm12
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm28, %zmm13
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm12, %zmm7, %zmm13
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm3, %zmm8, %zmm5
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm8, %zmm2
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 256(%rdx), %zmm7
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm3, %zmm13 {%k1}
-; AVX512DQ-FAST-NEXT:    vmovdqa64 256(%rax), %zmm23
-; AVX512DQ-FAST-NEXT:    vmovdqa 256(%r9), %ymm9
-; AVX512DQ-FAST-NEXT:    vmovdqa 256(%r8), %ymm5
-; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm4 = ymm5[0],ymm9[0],ymm5[2],ymm9[2]
-; AVX512DQ-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm13 {%k2} = zmm4[2,3,2,3],zmm23[2,3,2,3]
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 256(%r8), %zmm4
-; AVX512DQ-FAST-NEXT:    vmovdqa64 256(%r9), %zmm1
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm4, %zmm13
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm1, %zmm2, %zmm13
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm23, %zmm0, %zmm13
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm17
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm4, %zmm0
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm20, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm1, %zmm2
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm1, %zmm20, %zmm0
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm23, %zmm10, %zmm0
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm10, %zmm1
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm2, %zmm10
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm4, %zmm21, %zmm10
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm21, %zmm29
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm23, %zmm18, %zmm10
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 256(%rcx), %zmm29
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm29, %zmm4, %zmm7
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm4, %zmm3
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm2, %zmm8, %zmm4
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm4, %zmm14, %zmm23
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm23, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 320(%rdi), %zmm23
-; AVX512DQ-FAST-NEXT:    vmovdqa64 320(%rsi), %zmm20
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm23, %zmm4
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm20, %zmm22, %zmm4
-; AVX512DQ-FAST-NEXT:    vmovdqa64 320(%rdx), %zmm0
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 320(%rcx), %zmm13
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm13, %zmm7, %zmm0
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm4, %zmm0 {%k1}
-; AVX512DQ-FAST-NEXT:    vmovdqa64 320(%rax), %zmm21
-; AVX512DQ-FAST-NEXT:    vmovdqa 320(%r9), %ymm4
-; AVX512DQ-FAST-NEXT:    vmovdqa 320(%r8), %ymm2
-; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm10 = ymm2[0],ymm4[0],ymm2[2],ymm4[2]
-; AVX512DQ-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm0 {%k2} = zmm10[2,3,2,3],zmm21[2,3,2,3]
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm5, %zmm7 {%k1}
+; AVX512DQ-FAST-NEXT:    vmovdqa64 256(%rax), %zmm22
+; AVX512DQ-FAST-NEXT:    vmovdqa 256(%r9), %ymm15
+; AVX512DQ-FAST-NEXT:    vmovdqa 256(%r8), %ymm11
+; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm5 = ymm11[0],ymm15[0],ymm11[2],ymm15[2]
+; AVX512DQ-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm7 {%k2} = zmm5[2,3,2,3],zmm22[2,3,2,3]
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 256(%r8), %zmm7
+; AVX512DQ-FAST-NEXT:    vmovdqa64 256(%r9), %zmm5
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm7, %zmm12
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm5, %zmm1, %zmm12
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm1, %zmm8
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm22, %zmm10, %zmm12
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm7, %zmm12
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm5, %zmm6, %zmm12
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm22, %zmm9, %zmm12
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm9, %zmm4
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm5, %zmm9
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm7, %zmm20, %zmm9
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm22, %zmm19, %zmm9
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm7, %zmm12
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm5, %zmm23, %zmm7
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm7, %zmm18, %zmm22
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm22, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 320(%rdi), %zmm31
+; AVX512DQ-FAST-NEXT:    vmovdqa64 320(%rsi), %zmm21
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm31, %zmm7
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm21, %zmm2, %zmm7
+; AVX512DQ-FAST-NEXT:    vmovdqa64 320(%rdx), %zmm1
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 320(%rcx), %zmm14
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm14, %zmm3, %zmm1
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm7, %zmm1 {%k1}
+; AVX512DQ-FAST-NEXT:    vmovdqa64 320(%rax), %zmm2
+; AVX512DQ-FAST-NEXT:    vmovdqa 320(%r9), %ymm9
+; AVX512DQ-FAST-NEXT:    vmovdqa 320(%r8), %ymm3
+; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm7 = ymm3[0],ymm9[0],ymm3[2],ymm9[2]
+; AVX512DQ-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm1 {%k2} = zmm7[2,3,2,3],zmm2[2,3,2,3]
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-FAST-NEXT:    vmovdqa64 320(%r8), %zmm10
-; AVX512DQ-FAST-NEXT:    vmovdqa64 320(%r9), %zmm22
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm10, %zmm0
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm22, %zmm3, %zmm0
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm21, %zmm17, %zmm0
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm10, %zmm0
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm22, %zmm6, %zmm0
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm21, %zmm1, %zmm0
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm22, %zmm0
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm10, %zmm29, %zmm0
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm21, %zmm18, %zmm0
+; AVX512DQ-FAST-NEXT:    vmovdqa64 320(%r9), %zmm7
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm10, %zmm1
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm7, %zmm8, %zmm1
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm8, %zmm22
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm2, %zmm0, %zmm1
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm10, %zmm1
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm7, %zmm6, %zmm1
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm2, %zmm4, %zmm1
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm7, %zmm0
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm10, %zmm20, %zmm0
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm2, %zmm19, %zmm0
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm22, %zmm8, %zmm10
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm10, %zmm14, %zmm21
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm10, %zmm8
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm7, %zmm23, %zmm10
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm10, %zmm18, %zmm2
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-FAST-NEXT:    vmovdqa {{.*#+}} ymm0 = [1,3,7,7]
+; AVX512DQ-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload
+; AVX512DQ-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm10 # 32-byte Folded Reload
+; AVX512DQ-FAST-NEXT:    vmovdqu %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512DQ-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
 ; AVX512DQ-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 32-byte Folded Reload
 ; AVX512DQ-FAST-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512DQ-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm30 # 32-byte Folded Reload
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 %ymm30, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm24 # 32-byte Folded Reload
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %ymm24, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FAST-NEXT:    vpermt2q %ymm25, %ymm0, %ymm11
+; AVX512DQ-FAST-NEXT:    vpermt2q %ymm24, %ymm0, %ymm25
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %ymm25, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FAST-NEXT:    vpermt2q %ymm15, %ymm0, %ymm11
 ; AVX512DQ-FAST-NEXT:    vmovdqu %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FAST-NEXT:    vpermt2q %ymm9, %ymm0, %ymm5
-; AVX512DQ-FAST-NEXT:    vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FAST-NEXT:    vpermt2q %ymm4, %ymm0, %ymm2
-; AVX512DQ-FAST-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa 384(%r9), %ymm1
-; AVX512DQ-FAST-NEXT:    vmovdqa 384(%r8), %ymm2
-; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm3 = ymm2[0],ymm1[0],ymm2[2],ymm1[2]
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vpermt2q %ymm1, %ymm0, %ymm2
-; AVX512DQ-FAST-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm29 = [3,0,12,4,3,0,12,4]
-; AVX512DQ-FAST-NEXT:    # zmm29 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm7, %zmm1
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm0, %zmm29, %zmm1
-; AVX512DQ-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm21 = [13,5,13,5,13,5,13,5]
-; AVX512DQ-FAST-NEXT:    # zmm21 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm2
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm7, %zmm21, %zmm2
+; AVX512DQ-FAST-NEXT:    vpermt2q %ymm9, %ymm0, %ymm3
+; AVX512DQ-FAST-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa 384(%r9), %ymm4
+; AVX512DQ-FAST-NEXT:    vmovdqa 384(%r8), %ymm1
+; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm4[0],ymm1[2],ymm4[2]
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm3 = [6,14,6,14,6,14,6,14]
-; AVX512DQ-FAST-NEXT:    # zmm3 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm2
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm7, %zmm3, %zmm2
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm7, %zmm8, %zmm0
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    movb $48, %r10b
+; AVX512DQ-FAST-NEXT:    vpermt2q %ymm4, %ymm0, %ymm1
+; AVX512DQ-FAST-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 384(%rdx), %zmm19
+; AVX512DQ-FAST-NEXT:    vmovdqa64 384(%rcx), %zmm15
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm19, %zmm0
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm15, %zmm23, %zmm0
+; AVX512DQ-FAST-NEXT:    vmovdqa64 384(%rdi), %zmm30
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm30, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 384(%rsi), %zmm9
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm9, %zmm23, %zmm30
+; AVX512DQ-FAST-NEXT:    movb $24, %r10b
 ; AVX512DQ-FAST-NEXT:    kmovw %r10d, %k3
-; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm11 = [0,8,0,1,0,8,0,1]
-; AVX512DQ-FAST-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm30 {%k3}
+; AVX512DQ-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm18 = [0,8,0,8,0,8,0,8]
+; AVX512DQ-FAST-NEXT:    # zmm18 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm2
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm26, %zmm11, %zmm2
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm7 = [1,0,10,2,1,0,10,2]
-; AVX512DQ-FAST-NEXT:    # zmm7 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm26, %zmm2
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm0, %zmm7, %zmm2
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm1 {%k3} = zmm0[0],zmm26[0],zmm0[2],zmm26[2],zmm0[4],zmm26[4],zmm0[6],zmm26[6]
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm1
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm26, %zmm21, %zmm1
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} ymm30 = [6,14,6,14]
-; AVX512DQ-FAST-NEXT:    # ymm30 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm1
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm26, %zmm30, %zmm1
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} ymm18 = [15,7,15,7]
-; AVX512DQ-FAST-NEXT:    # ymm18 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm26, %zmm18, %zmm0
+; AVX512DQ-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm18, %zmm0 # 64-byte Folded Reload
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm1
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm4, %zmm29, %zmm1
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm4, %zmm2
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm0, %zmm21, %zmm2
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm4, %zmm2
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm0, %zmm3, %zmm2
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm0, %zmm8, %zmm4
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm4, %zmm0
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm2, %zmm11, %zmm0
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm27, %zmm18, %zmm0
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm2, %zmm0
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm4, %zmm7, %zmm0
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm16, %zmm18, %zmm0
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm1 {%k3} = zmm4[0],zmm2[0],zmm4[2],zmm2[2],zmm4[4],zmm2[4],zmm4[6],zmm2[6]
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm13, %zmm18, %zmm17
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm5, %zmm18, %zmm12
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm7, %zmm18, %zmm8
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 384(%r8), %zmm0
+; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm1 = <u,1,2,3,4,15,u,u>
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm30
+; AVX512DQ-FAST-NEXT:    vmovdqa64 384(%r9), %zmm1
+; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm2 = <u,1,2,3,4,5,15,u>
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm1, %zmm2, %zmm30
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm1, %zmm0, %zmm18
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm1, %zmm0, %zmm22
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm22, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm1, %zmm0, %zmm6
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm0, %zmm20, %zmm1
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 448(%rdx), %zmm7
+; AVX512DQ-FAST-NEXT:    vmovdqa64 448(%rcx), %zmm0
+; AVX512DQ-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm1 = [6,14,6,14,6,14,6,14]
+; AVX512DQ-FAST-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm7, %zmm2
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm2
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm7, %zmm10
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm0, %zmm2, %zmm10
+; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm8 = [3,0,12,4,3,0,12,4]
+; AVX512DQ-FAST-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm13
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm7, %zmm8, %zmm13
+; AVX512DQ-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm5 = [13,5,13,5,13,5,13,5]
+; AVX512DQ-FAST-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm7, %zmm17
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm0, %zmm5, %zmm17
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm0, %zmm23, %zmm7
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm11, %zmm0
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm4, %zmm8, %zmm0
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm3
 ; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm4, %zmm0
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm2, %zmm21, %zmm0
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm11, %zmm5, %zmm0
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm4, %zmm0
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm2, %zmm30, %zmm0
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm11, %zmm1, %zmm0
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm2, %zmm18, %zmm4
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm11, %zmm23, %zmm4
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm27, %zmm0
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm2, %zmm29, %zmm0
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm2, %zmm1
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm27, %zmm21, %zmm1
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm2, %zmm1
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm27, %zmm3, %zmm1
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm27, %zmm8, %zmm2
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm2, %zmm1
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm19, %zmm11, %zmm1
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm19, %zmm1
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm2, %zmm7, %zmm1
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm0 {%k3} = zmm2[0],zmm19[0],zmm2[2],zmm19[2],zmm2[4],zmm19[4],zmm2[6],zmm19[6]
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm2, %zmm0
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm19, %zmm21, %zmm0
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm2, %zmm0
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm19, %zmm30, %zmm0
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm19, %zmm18, %zmm2
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm15, %zmm0
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm2, %zmm29, %zmm0
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm2, %zmm1
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm15, %zmm21, %zmm1
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm2, %zmm1
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm15, %zmm3, %zmm1
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm15, %zmm8, %zmm2
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm2, %zmm1
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm16, %zmm11, %zmm1
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm16, %zmm17
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm2, %zmm7, %zmm17
-; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm0 {%k3} = zmm2[0],zmm16[0],zmm2[2],zmm16[2],zmm2[4],zmm16[4],zmm2[6],zmm16[6]
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm2, %zmm0
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm16, %zmm21, %zmm0
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm2, %zmm0
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm16, %zmm30, %zmm0
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm16, %zmm18, %zmm2
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm12, %zmm0
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm28, %zmm29, %zmm0
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm1
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm28, %zmm0
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm12, %zmm21, %zmm0
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm28, %zmm0
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm12, %zmm3, %zmm0
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm12, %zmm8, %zmm28
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm28, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    movb $48, %r10b
+; AVX512DQ-FAST-NEXT:    kmovw %r10d, %k4
+; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [0,8,0,1,0,8,0,1]
+; AVX512DQ-FAST-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm11, %zmm6
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm2
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm31, %zmm11, %zmm2
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm31, %zmm2
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm0, %zmm7, %zmm2
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm2, %zmm19
-; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm1 {%k3} = zmm0[0],zmm31[0],zmm0[2],zmm31[2],zmm0[4],zmm31[4],zmm0[6],zmm31[6]
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm1
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm31, %zmm21, %zmm1
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm1
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm31, %zmm30, %zmm1
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm31, %zmm18, %zmm0
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm13, %zmm0
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm15, %zmm29, %zmm0
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm15, %zmm1
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm13, %zmm21, %zmm1
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm15, %zmm1
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm13, %zmm3, %zmm1
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm13, %zmm8, %zmm15
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm23, %zmm1
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm20, %zmm11, %zmm1
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm20, %zmm28
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm23, %zmm7, %zmm28
-; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm0 {%k3} = zmm23[0],zmm20[0],zmm23[2],zmm20[2],zmm23[4],zmm20[4],zmm23[6],zmm20[6]
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm23, %zmm0
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm20, %zmm21, %zmm0
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm0, %zmm4, %zmm6
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm6 = [1,0,10,2,1,0,10,2]
+; AVX512DQ-FAST-NEXT:    # zmm6 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm12
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm11, %zmm6, %zmm12
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm3 {%k4} = zmm11[0],zmm0[0],zmm11[2],zmm0[2],zmm11[4],zmm0[4],zmm11[6],zmm0[6]
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm11, %zmm3
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm0, %zmm5, %zmm3
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm11, %zmm3
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm3
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm0, %zmm23, %zmm11
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm16, %zmm0
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm11, %zmm8, %zmm0
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm3
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm11, %zmm0
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm16, %zmm5, %zmm0
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm23, %zmm0
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm20, %zmm30, %zmm0
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm11, %zmm0
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm16, %zmm1, %zmm0
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm20, %zmm18, %zmm23
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm23, %zmm26
-; AVX512DQ-FAST-NEXT:    vmovdqa64 448(%rdx), %zmm20
-; AVX512DQ-FAST-NEXT:    vmovdqa64 448(%rcx), %zmm5
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm20, %zmm13
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm16, %zmm23, %zmm11
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm12, %zmm11
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm5, %zmm0, %zmm13
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm20, %zmm10
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm5, %zmm3, %zmm10
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm5, %zmm9
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm20, %zmm29, %zmm9
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm20, %zmm16
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm5, %zmm21, %zmm16
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm5, %zmm8, %zmm20
-; AVX512DQ-FAST-NEXT:    vmovdqa64 384(%rdx), %zmm1
-; AVX512DQ-FAST-NEXT:    vmovdqa64 384(%rcx), %zmm4
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm4, %zmm1, %zmm0
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm0, %zmm4, %zmm11
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm11
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm12, %zmm6, %zmm11
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm3 {%k4} = zmm12[0],zmm0[0],zmm12[2],zmm0[2],zmm12[4],zmm0[4],zmm12[6],zmm0[6]
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm12, %zmm3
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm0, %zmm5, %zmm3
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm12, %zmm3
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm3
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm0, %zmm23, %zmm12
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm3
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm12, %zmm8, %zmm3
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm12, %zmm11
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm0, %zmm5, %zmm11
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm12, %zmm11
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm11
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm0, %zmm23, %zmm12
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm12, %zmm0
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm26, %zmm4, %zmm0
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm1, %zmm4, %zmm29
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm4, %zmm1, %zmm3
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm26, %zmm11
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm12, %zmm6, %zmm11
+; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm3 {%k4} = zmm12[0],zmm26[0],zmm12[2],zmm26[2],zmm12[4],zmm26[4],zmm12[6],zmm26[6]
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm1, %zmm0
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm4, %zmm8, %zmm1
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm4, %zmm21, %zmm0
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm12, %zmm0
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm26, %zmm5, %zmm0
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 448(%rdi), %zmm12
-; AVX512DQ-FAST-NEXT:    vmovdqa64 448(%rsi), %zmm4
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm12, %zmm8
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm12, %zmm0
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm26, %zmm1, %zmm0
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm26, %zmm23, %zmm12
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm3, %zmm12
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm4, %zmm0, %zmm8
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm12, %zmm3
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm4, %zmm30, %zmm3
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm12, %zmm2
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm4, %zmm21, %zmm2
-; AVX512DQ-FAST-NEXT:    vmovdqa64 384(%rdi), %zmm6
-; AVX512DQ-FAST-NEXT:    vmovdqa64 384(%rsi), %zmm5
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm6, %zmm25
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm5, %zmm11, %zmm25
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm5, %zmm24
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm6, %zmm7, %zmm24
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm5, %zmm6, %zmm0
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm0, %zmm8, %zmm12
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm16
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm3, %zmm5, %zmm16
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm16
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm3, %zmm1, %zmm16
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm3, %zmm23, %zmm0
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm29 {%k3} = zmm6[0],zmm5[0],zmm6[2],zmm5[2],zmm6[4],zmm5[4],zmm6[6],zmm5[6]
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm5, %zmm6, %zmm21
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm5, %zmm6, %zmm30
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm5, %zmm18, %zmm6
-; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm9 {%k3} = zmm12[0],zmm4[0],zmm12[2],zmm4[2],zmm12[4],zmm4[4],zmm12[6],zmm4[6]
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm4, %zmm12, %zmm11
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm12, %zmm4, %zmm7
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm4, %zmm18, %zmm12
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm8, %zmm13 {%k1}
-; AVX512DQ-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm3 = zmm3[0,1,2,3],zmm10[4,5,6,7]
-; AVX512DQ-FAST-NEXT:    vmovdqa64 448(%r8), %zmm5
-; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm4 = <0,11,u,u,4,5,6,7>
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm5, %zmm9, %zmm4
-; AVX512DQ-FAST-NEXT:    vmovdqa64 448(%r9), %zmm8
-; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm14 = <0,1,11,u,4,5,6,7>
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm8, %zmm4, %zmm14
-; AVX512DQ-FAST-NEXT:    movb $4, %sil
-; AVX512DQ-FAST-NEXT:    kmovw %esi, %k3
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm5, %zmm13 {%k3}
-; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm4 = <0,1,2,10,u,5,6,7>
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm8, %zmm13, %zmm4
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm16, %zmm2 {%k1}
-; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm10 = <12,u,u,3,4,5,6,13>
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm5, %zmm2, %zmm10
-; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm18 = <0,12,u,3,4,5,6,7>
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm8, %zmm10, %zmm18
-; AVX512DQ-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm10 = [0,8,0,8,0,8,0,8]
-; AVX512DQ-FAST-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm0 # 64-byte Folded Reload
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm3
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm28, %zmm4, %zmm3
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm28, %zmm3
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm0, %zmm6, %zmm3
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm3, %zmm24
+; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm12 {%k4} = zmm0[0],zmm28[0],zmm0[2],zmm28[2],zmm0[4],zmm28[4],zmm0[6],zmm28[6]
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm3
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm28, %zmm5, %zmm3
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm3
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm28, %zmm1, %zmm3
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm28, %zmm23, %zmm0
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm29, %zmm28
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm0 # 64-byte Folded Reload
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm0, %zmm8, %zmm28
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm3
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm29, %zmm5, %zmm3
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm3
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm29, %zmm1, %zmm3
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm29, %zmm23, %zmm0
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm0 # 64-byte Folded Reload
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm29
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm3, %zmm4, %zmm29
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm3, %zmm27
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm0, %zmm6, %zmm27
+; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm28 {%k4} = zmm0[0],zmm3[0],zmm0[2],zmm3[2],zmm0[4],zmm3[4],zmm0[6],zmm3[6]
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm12
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm3, %zmm5, %zmm12
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm12
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm3, %zmm1, %zmm12
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm3, %zmm23, %zmm0
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm14, %zmm26
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm0 # 64-byte Folded Reload
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm0, %zmm8, %zmm26
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm3
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm14, %zmm5, %zmm3
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm3
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm14, %zmm1, %zmm3
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm14, %zmm23, %zmm0
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm23 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm23 # 64-byte Folded Reload
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm27 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm22, %zmm10, %zmm27
-; AVX512DQ-FAST-NEXT:    movb $24, %sil
-; AVX512DQ-FAST-NEXT:    kmovw %esi, %k4
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm1, %zmm6 {%k4}
+; AVX512DQ-FAST-NEXT:    vmovdqa64 448(%rdi), %zmm12
+; AVX512DQ-FAST-NEXT:    vmovdqa64 448(%rsi), %zmm0
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm12, %zmm16
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm0, %zmm23, %zmm16
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm21, %zmm25
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm31, %zmm6, %zmm25
+; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm26 {%k4} = zmm31[0],zmm21[0],zmm31[2],zmm21[2],zmm31[4],zmm21[4],zmm31[6],zmm21[6]
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm31, %zmm22
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm31, %zmm14
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm31, %zmm3
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm21, %zmm23, %zmm31
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm21, %zmm4, %zmm22
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm21, %zmm5, %zmm14
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm21, %zmm1, %zmm3
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm15, %zmm19, %zmm2
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm19, %zmm15, %zmm8
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm19, %zmm2
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm15, %zmm5, %zmm2
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm15, %zmm1, %zmm19
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm12, %zmm19
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm19
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm12, %zmm3
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm0, %zmm15, %zmm3
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm12, %zmm2
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm0, %zmm5, %zmm2
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm9, %zmm20
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm14, %zmm6, %zmm20
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm9, %zmm14, %zmm15
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm8 {%k4} = zmm14[0],zmm9[0],zmm14[2],zmm9[2],zmm14[4],zmm9[4],zmm14[6],zmm9[6]
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm9, %zmm14, %zmm5
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm14, %zmm23
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm9, %zmm1, %zmm14
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm9, %zmm4, %zmm23
+; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm13 {%k4} = zmm12[0],zmm0[0],zmm12[2],zmm0[2],zmm12[4],zmm0[4],zmm12[6],zmm0[6]
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm0, %zmm12, %zmm4
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm12, %zmm6, %zmm0
+; AVX512DQ-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm19, %zmm1 # 64-byte Folded Reload
+; AVX512DQ-FAST-NEXT:    # zmm1 = zmm19[0,1,2,3],mem[4,5,6,7]
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm3, %zmm10 {%k1}
+; AVX512DQ-FAST-NEXT:    vmovdqa64 448(%r8), %zmm3
+; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm9 = <0,11,u,u,4,5,6,7>
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm3, %zmm13, %zmm9
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm17, %zmm2 {%k1}
+; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm12 = <12,u,u,3,4,5,6,13>
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm3, %zmm2, %zmm12
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm7, %zmm16 {%k3}
+; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm2 = <u,1,2,3,4,15,u,u>
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm3, %zmm2, %zmm16
 ; AVX512DQ-FAST-NEXT:    movb $6, %sil
 ; AVX512DQ-FAST-NEXT:    kmovw %esi, %k5
-; AVX512DQ-FAST-NEXT:    vpbroadcastq 456(%rcx), %ymm0
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
-; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm7 {%k5}
-; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm1 = <0,1,2,9,u,u,6,7>
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm5, %zmm7, %zmm1
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm20, %zmm12 {%k4}
+; AVX512DQ-FAST-NEXT:    vpbroadcastq 456(%rcx), %ymm2
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm2 = mem[0,1,2,3],ymm2[4,5,6,7]
+; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm2, %zmm0, %zmm0 {%k5}
+; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm2 = <0,1,2,9,u,u,6,7>
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm3, %zmm0, %zmm2
 ; AVX512DQ-FAST-NEXT:    movb $64, %sil
-; AVX512DQ-FAST-NEXT:    kmovw %esi, %k3
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm5, %zmm3 {%k3}
-; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm2 = <u,1,2,3,4,15,u,u>
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm5, %zmm2, %zmm12
-; AVX512DQ-FAST-NEXT:    vmovdqa64 384(%r8), %zmm5
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm5, %zmm2, %zmm6
-; AVX512DQ-FAST-NEXT:    vmovdqa64 384(%r9), %zmm2
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm2, %zmm5, %zmm10
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm2, %zmm5, %zmm16
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm2, %zmm5, %zmm13
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm5, %zmm2, %zmm0
+; AVX512DQ-FAST-NEXT:    kmovw %esi, %k4
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm3, %zmm1 {%k4}
+; AVX512DQ-FAST-NEXT:    movb $4, %sil
+; AVX512DQ-FAST-NEXT:    kmovw %esi, %k4
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm3, %zmm10 {%k4}
+; AVX512DQ-FAST-NEXT:    vmovdqa64 448(%r9), %zmm0
+; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm21 = <0,1,11,u,4,5,6,7>
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm0, %zmm9, %zmm21
+; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm3 = <0,1,2,10,u,5,6,7>
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm0, %zmm10, %zmm3
+; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm7 = <0,12,u,3,4,5,6,7>
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm0, %zmm12, %zmm7
+; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm9 = <13,u,2,3,4,5,6,14>
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm0, %zmm1, %zmm9
+; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm1 = <u,1,2,3,4,5,15,u>
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm16
 ; AVX512DQ-FAST-NEXT:    movb $12, %sil
-; AVX512DQ-FAST-NEXT:    kmovw %esi, %k3
-; AVX512DQ-FAST-NEXT:    vmovdqa 448(%rdx), %xmm5
-; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm5 = xmm5[0],mem[0]
-; AVX512DQ-FAST-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm5, %zmm0, %zmm11 {%k3}
-; AVX512DQ-FAST-NEXT:    vinserti32x4 $2, 448(%r8), %zmm11, %zmm5
-; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm7 = <0,1,2,3,4,8,u,7>
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm8, %zmm5, %zmm7
-; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm5 = <u,1,2,3,4,5,15,u>
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm2, %zmm5, %zmm6
-; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm11 = <0,1,2,3,9,u,6,7>
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm8, %zmm1, %zmm11
-; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm2 = <13,u,2,3,4,5,6,14>
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm8, %zmm3, %zmm2
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm8, %zmm5, %zmm12
-; AVX512DQ-FAST-NEXT:    vpbroadcastq 8(%rcx), %ymm1
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm1 = mem[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm9 {%k5}
-; AVX512DQ-FAST-NEXT:    vpbroadcastq 72(%rcx), %ymm1
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm1 = mem[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm31 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm31 {%k5}
-; AVX512DQ-FAST-NEXT:    vpbroadcastq 136(%rcx), %ymm1
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm1 = mem[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm8 {%k5}
-; AVX512DQ-FAST-NEXT:    vpbroadcastq 200(%rcx), %ymm1
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm1 = mem[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm17, %zmm3
-; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm3 {%k5}
-; AVX512DQ-FAST-NEXT:    vpbroadcastq 264(%rcx), %ymm1
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm1 = mem[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm19, %zmm20
-; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm20 {%k5}
-; AVX512DQ-FAST-NEXT:    vpbroadcastq 328(%rcx), %ymm1
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm1 = mem[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm28 {%k5}
-; AVX512DQ-FAST-NEXT:    vpbroadcastq 392(%rcx), %ymm1
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm1 = mem[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm24 {%k5}
-; AVX512DQ-FAST-NEXT:    vmovdqa64 448(%rax), %zmm5
-; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,1,2,3,10,5,6,7]
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm5, %zmm4, %zmm1
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,1,12,3,4,5,6,7]
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm5, %zmm18, %zmm1
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 384(%rax), %zmm1
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm1, %zmm4, %zmm16
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm16, %zmm4
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm17, %zmm16 {%k1}
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm16 {%k2} = zmm17[2,3,2,3],zmm1[2,3,2,3]
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm1, %zmm16, %zmm13
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm13, %zmm18
+; AVX512DQ-FAST-NEXT:    kmovw %esi, %k4
+; AVX512DQ-FAST-NEXT:    vmovdqa 448(%rdx), %xmm1
+; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
+; AVX512DQ-FAST-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
+; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm4 {%k4}
+; AVX512DQ-FAST-NEXT:    vinserti32x4 $2, 448(%r8), %zmm4, %zmm1
+; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm4 = <0,1,2,3,4,8,u,7>
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm0, %zmm1, %zmm4
+; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm10 = <0,1,2,3,9,u,6,7>
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm0, %zmm2, %zmm10
+; AVX512DQ-FAST-NEXT:    vpbroadcastq 8(%rcx), %ymm0
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm2 {%k5}
+; AVX512DQ-FAST-NEXT:    vpbroadcastq 72(%rcx), %ymm0
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm6 {%k5}
+; AVX512DQ-FAST-NEXT:    vpbroadcastq 136(%rcx), %ymm0
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm11, %zmm15
+; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm15 {%k5}
+; AVX512DQ-FAST-NEXT:    vpbroadcastq 200(%rcx), %ymm0
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm24, %zmm14
+; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm14 {%k5}
+; AVX512DQ-FAST-NEXT:    vpbroadcastq 264(%rcx), %ymm0
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
+; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm27 {%k5}
+; AVX512DQ-FAST-NEXT:    vpbroadcastq 328(%rcx), %ymm0
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
+; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm25 {%k5}
+; AVX512DQ-FAST-NEXT:    vpbroadcastq 392(%rcx), %ymm0
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
+; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm20 {%k5}
+; AVX512DQ-FAST-NEXT:    vmovdqa64 384(%rax), %zmm0
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm1, %zmm13, %zmm0
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm13
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm1, %zmm11 {%k1}
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm11 {%k2} = zmm1[2,3,2,3],zmm0[2,3,2,3]
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm24 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm24
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm11
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [14,1,2,3,4,5,6,15]
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm30
+; AVX512DQ-FAST-NEXT:    vmovdqa64 448(%rax), %zmm12
+; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [0,1,2,3,10,5,6,7]
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm12, %zmm3, %zmm0
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [0,1,12,3,4,5,6,7]
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm12, %zmm7, %zmm0
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [0,13,2,3,4,5,6,7]
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm12, %zmm9, %zmm0
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vpermt2q %zmm12, %zmm1, %zmm16
 ; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [0,1,2,3,4,5,8,7]
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm5, %zmm7, %zmm0
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm12, %zmm4, %zmm0
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm7 = [14,1,2,3,4,5,6,15]
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm1, %zmm7, %zmm6
 ; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [0,1,2,3,4,9,6,7]
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm5, %zmm11, %zmm0
+; AVX512DQ-FAST-NEXT:    vpermi2q %zmm12, %zmm10, %zmm0
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [0,13,2,3,4,5,6,7]
-; AVX512DQ-FAST-NEXT:    vpermi2q %zmm5, %zmm2, %zmm0
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vpermt2q %zmm5, %zmm7, %zmm12
 ; AVX512DQ-FAST-NEXT:    movb $8, %sil
 ; AVX512DQ-FAST-NEXT:    kmovw %esi, %k2
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm5, %zmm14 {%k2}
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm12, %zmm21 {%k2}
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k4}
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm5 {%k4}
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k3}
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm7 {%k4}
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm7 {%k3}
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k4}
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm3 {%k3}
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm11 {%k4}
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm15, %zmm26 {%k4}
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm9 {%k3}
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm4 {%k3}
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm31 {%k3}
 ; AVX512DQ-FAST-NEXT:    movb $-31, %sil
 ; AVX512DQ-FAST-NEXT:    kmovw %esi, %k2
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k2}
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm5 {%k2}
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm7 {%k2}
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k2}
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm3 {%k2}
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm11 {%k2}
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm9 {%k2}
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm26 {%k2}
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm26, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqa (%rdx), %xmm0
-; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
-; AVX512DQ-FAST-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm13 {%k3}
-; AVX512DQ-FAST-NEXT:    vmovdqa 64(%rdx), %xmm0
-; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
-; AVX512DQ-FAST-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm4 {%k2}
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm31 {%k2}
+; AVX512DQ-FAST-NEXT:    vmovdqa (%rdx), %xmm4
+; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm4 = xmm4[0],mem[0]
+; AVX512DQ-FAST-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm1 {%k3}
-; AVX512DQ-FAST-NEXT:    vmovdqa 128(%rdx), %xmm0
-; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
-; AVX512DQ-FAST-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm2 {%k3}
-; AVX512DQ-FAST-NEXT:    vmovdqa 192(%rdx), %xmm0
-; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
-; AVX512DQ-FAST-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm5 {%k3}
-; AVX512DQ-FAST-NEXT:    vmovdqa 256(%rdx), %xmm0
-; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
-; AVX512DQ-FAST-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm4, %zmm0, %zmm1 {%k4}
+; AVX512DQ-FAST-NEXT:    vmovdqa 64(%rdx), %xmm4
+; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm4 = xmm4[0],mem[0]
+; AVX512DQ-FAST-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm4, %zmm0, %zmm3 {%k4}
+; AVX512DQ-FAST-NEXT:    vmovdqa 128(%rdx), %xmm4
+; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm4 = xmm4[0],mem[0]
+; AVX512DQ-FAST-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm7 {%k3}
-; AVX512DQ-FAST-NEXT:    vmovdqa 320(%rdx), %xmm0
-; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
-; AVX512DQ-FAST-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm11 {%k3}
-; AVX512DQ-FAST-NEXT:    vmovdqa 384(%rdx), %xmm0
-; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
-; AVX512DQ-FAST-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm25 {%k3}
+; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm4, %zmm0, %zmm7 {%k4}
+; AVX512DQ-FAST-NEXT:    vmovdqa 192(%rdx), %xmm4
+; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm4 = xmm4[0],mem[0]
+; AVX512DQ-FAST-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm4, %zmm0, %zmm9 {%k4}
+; AVX512DQ-FAST-NEXT:    vmovdqa 256(%rdx), %xmm4
+; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm4 = xmm4[0],mem[0]
+; AVX512DQ-FAST-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm4, %zmm0, %zmm29 {%k4}
+; AVX512DQ-FAST-NEXT:    vmovdqa 320(%rdx), %xmm4
+; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm4 = xmm4[0],mem[0]
+; AVX512DQ-FAST-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm4, %zmm0, %zmm22 {%k4}
+; AVX512DQ-FAST-NEXT:    vmovdqa 384(%rdx), %xmm4
+; AVX512DQ-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm4 = xmm4[0],mem[0]
+; AVX512DQ-FAST-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm4, %zmm0, %zmm23 {%k4}
 ; AVX512DQ-FAST-NEXT:    movb $112, %cl
 ; AVX512DQ-FAST-NEXT:    kmovw %ecx, %k2
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vinserti64x2 $3, (%rax), %zmm0, %zmm13 {%k2}
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vinserti64x2 $3, 64(%rax), %zmm0, %zmm1 {%k2}
+; AVX512DQ-FAST-NEXT:    vinserti64x2 $3, (%rax), %zmm0, %zmm1 {%k2}
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vinserti64x2 $3, 128(%rax), %zmm0, %zmm2 {%k2}
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vinserti64x2 $3, 64(%rax), %zmm0, %zmm3 {%k2}
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vinserti64x2 $3, 192(%rax), %zmm0, %zmm5 {%k2}
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vinserti64x2 $3, 256(%rax), %zmm23, %zmm7 {%k2}
+; AVX512DQ-FAST-NEXT:    vinserti64x2 $3, 128(%rax), %zmm0, %zmm7 {%k2}
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FAST-NEXT:    vinserti64x2 $3, 320(%rax), %zmm27, %zmm11 {%k2}
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm11, %zmm27
-; AVX512DQ-FAST-NEXT:    vinserti64x2 $3, 384(%rax), %zmm10, %zmm25 {%k2}
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vinserti64x2 $3, 192(%rax), %zmm0, %zmm9 {%k2}
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vinserti64x2 $3, 256(%rax), %zmm0, %zmm29 {%k2}
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vinserti64x2 $3, 320(%rax), %zmm0, %zmm22 {%k2}
+; AVX512DQ-FAST-NEXT:    vinserti64x2 $3, 384(%rax), %zmm18, %zmm23 {%k2}
 ; AVX512DQ-FAST-NEXT:    movb $56, %cl
 ; AVX512DQ-FAST-NEXT:    kmovw %ecx, %k2
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm9 {%k2}
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k2}
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm31 {%k2}
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm6 {%k2}
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm8 {%k2}
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm15 {%k2}
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm3 {%k2}
-; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm14 {%k2}
+; AVX512DQ-FAST-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm20 {%k2}
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm20, %zmm26
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm27 {%k2}
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm28 {%k2}
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm4, %zmm24 {%k2}
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm25 {%k2}
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm13, %zmm20 {%k2}
 ; AVX512DQ-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],mem[6,7]
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm4 = ymm0[0,1,2,3,4,5],mem[6,7]
 ; AVX512DQ-FAST-NEXT:    movb $14, %cl
 ; AVX512DQ-FAST-NEXT:    kmovw %ecx, %k2
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm23 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm23 {%k2}
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm4, %zmm0, %zmm17 {%k2}
 ; AVX512DQ-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],mem[6,7]
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm22 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm22 {%k2}
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm4 = ymm0[0,1,2,3,4,5],mem[6,7]
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm4, %zmm0, %zmm15 {%k2}
 ; AVX512DQ-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],mem[6,7]
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm20 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm20 {%k2}
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm4 = ymm0[0,1,2,3,4,5],mem[6,7]
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm4, %zmm0, %zmm14 {%k2}
 ; AVX512DQ-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],mem[6,7]
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm16 {%k2}
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm4 = ymm0[0,1,2,3,4,5],mem[6,7]
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm4, %zmm0, %zmm7 {%k2}
 ; AVX512DQ-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],mem[6,7]
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm14 {%k2}
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm4 = ymm0[0,1,2,3,4,5],mem[6,7]
+; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm4, %zmm0, %zmm28 {%k2}
 ; AVX512DQ-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],mem[6,7]
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm4 {%k2}
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm4 = ymm0[0,1,2,3,4,5],mem[6,7]
+; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm4, %zmm0, %zmm26 {%k2}
 ; AVX512DQ-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],mem[6,7]
-; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm29 {%k2}
+; AVX512DQ-FAST-NEXT:    vpblendd {{.*#+}} ymm4 = ymm0[0,1,2,3,4,5],mem[6,7]
+; AVX512DQ-FAST-NEXT:    vinserti64x4 $0, %ymm4, %zmm0, %zmm8 {%k2}
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
 ; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
@@ -16302,920 +16223,913 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
 ; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k1}
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm3 {%k1}
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm4 {%k1}
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm5 {%k1}
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm10 {%k1}
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm7 {%k1}
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm12 {%k1}
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm8 {%k1}
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm13 {%k1}
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm21 {%k1}
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, %zmm5 {%k1}
 ; AVX512DQ-FAST-NEXT:    movb $120, %al
 ; AVX512DQ-FAST-NEXT:    kmovw %eax, %k1
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm1, %zmm19 {%k1}
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm1, %zmm11 {%k1}
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm17 # 64-byte Folded Reload
-; AVX512DQ-FAST-NEXT:    # zmm17 = zmm0[0,1,2,3],mem[4,5,6,7]
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm2, %zmm15 {%k1}
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm3, %zmm13 {%k1}
+; AVX512DQ-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm9 # 64-byte Folded Reload
+; AVX512DQ-FAST-NEXT:    # zmm9 = zmm0[0,1,2,3],mem[4,5,6,7]
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm2, %zmm6 {%k1}
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm5, %zmm3 {%k1}
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm4, %zmm3 {%k1}
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm10, %zmm2 {%k1}
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm7, %zmm1 {%k1}
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm12, %zmm1 {%k1}
 ; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm8, %zmm0 {%k1}
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm21, %zmm18 {%k1}
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm13, %zmm0 {%k1}
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm5, %zmm24 {%k1}
 ; AVX512DQ-FAST-NEXT:    movb $-61, %al
 ; AVX512DQ-FAST-NEXT:    kmovw %eax, %k1
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm2, %zmm17 {%k1}
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm5 # 64-byte Folded Reload
-; AVX512DQ-FAST-NEXT:    # zmm5 = zmm2[0,1,2,3],mem[4,5,6,7]
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm2, %zmm5 {%k1}
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm7 # 64-byte Folded Reload
-; AVX512DQ-FAST-NEXT:    # zmm7 = zmm2[0,1,2,3],mem[4,5,6,7]
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm2, %zmm7 {%k1}
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm8 # 64-byte Folded Reload
-; AVX512DQ-FAST-NEXT:    # zmm8 = zmm2[0,1,2,3],mem[4,5,6,7]
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm2, %zmm8 {%k1}
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm9 # 64-byte Folded Reload
-; AVX512DQ-FAST-NEXT:    # zmm9 = zmm2[0,1,2,3],mem[4,5,6,7]
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm2, %zmm9 {%k1}
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm10 # 64-byte Folded Reload
-; AVX512DQ-FAST-NEXT:    # zmm10 = zmm2[0,1,2,3],mem[4,5,6,7]
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm2, %zmm10 {%k1}
-; AVX512DQ-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm30, %zmm11 # 64-byte Folded Reload
-; AVX512DQ-FAST-NEXT:    # zmm11 = zmm30[0,1,2,3],mem[4,5,6,7]
-; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm2, %zmm11 {%k1}
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm4, %zmm9 {%k1}
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm4 # 64-byte Folded Reload
+; AVX512DQ-FAST-NEXT:    # zmm4 = zmm4[0,1,2,3],mem[4,5,6,7]
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm5, %zmm4 {%k1}
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm5 # 64-byte Folded Reload
+; AVX512DQ-FAST-NEXT:    # zmm5 = zmm5[0,1,2,3],mem[4,5,6,7]
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm10, %zmm5 {%k1}
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm10 # 64-byte Folded Reload
+; AVX512DQ-FAST-NEXT:    # zmm10 = zmm10[0,1,2,3],mem[4,5,6,7]
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm12, %zmm10 {%k1}
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm12, %zmm12 # 64-byte Folded Reload
+; AVX512DQ-FAST-NEXT:    # zmm12 = zmm12[0,1,2,3],mem[4,5,6,7]
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm13, %zmm12 {%k1}
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm13, %zmm13 # 64-byte Folded Reload
+; AVX512DQ-FAST-NEXT:    # zmm13 = zmm13[0,1,2,3],mem[4,5,6,7]
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm18, %zmm13 {%k1}
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm18, %zmm18 # 64-byte Folded Reload
+; AVX512DQ-FAST-NEXT:    # zmm18 = zmm18[0,1,2,3],mem[4,5,6,7]
+; AVX512DQ-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm19, %zmm18 {%k1}
 ; AVX512DQ-FAST-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm11, 3008(%rax)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm18, 2944(%rax)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm29, 2880(%rax)
-; AVX512DQ-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovaps %zmm2, 2816(%rax)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm24, 2752(%rax)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm25, 2688(%rax)
-; AVX512DQ-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovaps %zmm2, 2624(%rax)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm10, 2560(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm18, 3008(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm24, 2944(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm8, 2880(%rax)
+; AVX512DQ-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vmovaps %zmm8, 2816(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm20, 2752(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm23, 2688(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm31, 2624(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm13, 2560(%rax)
 ; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm0, 2496(%rax)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm4, 2432(%rax)
-; AVX512DQ-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovaps %zmm2, 2368(%rax)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm28, 2304(%rax)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm27, 2240(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm26, 2432(%rax)
+; AVX512DQ-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vmovaps %zmm8, 2368(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm25, 2304(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm22, 2240(%rax)
 ; AVX512DQ-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQ-FAST-NEXT:    vmovaps %zmm0, 2176(%rax)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm9, 2112(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm12, 2112(%rax)
 ; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm1, 2048(%rax)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm14, 1984(%rax)
-; AVX512DQ-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovaps %zmm2, 1920(%rax)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm26, 1856(%rax)
-; AVX512DQ-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovaps %zmm0, 1792(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm28, 1984(%rax)
+; AVX512DQ-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vmovaps %zmm8, 1920(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm27, 1856(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm29, 1792(%rax)
 ; AVX512DQ-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQ-FAST-NEXT:    vmovaps %zmm0, 1728(%rax)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm8, 1664(%rax)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm3, 1600(%rax)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm16, 1536(%rax)
-; AVX512DQ-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovaps %zmm2, 1472(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm10, 1664(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm2, 1600(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm7, 1536(%rax)
+; AVX512DQ-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vmovaps %zmm8, 1472(%rax)
 ; AVX512DQ-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQ-FAST-NEXT:    vmovaps %zmm0, 1408(%rax)
 ; AVX512DQ-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQ-FAST-NEXT:    vmovaps %zmm0, 1344(%rax)
 ; AVX512DQ-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQ-FAST-NEXT:    vmovaps %zmm0, 1280(%rax)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm7, 1216(%rax)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm13, 1152(%rax)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm20, 1088(%rax)
-; AVX512DQ-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovaps %zmm2, 1024(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm5, 1216(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm3, 1152(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm14, 1088(%rax)
+; AVX512DQ-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vmovaps %zmm5, 1024(%rax)
 ; AVX512DQ-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQ-FAST-NEXT:    vmovaps %zmm0, 960(%rax)
 ; AVX512DQ-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQ-FAST-NEXT:    vmovaps %zmm0, 896(%rax)
 ; AVX512DQ-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQ-FAST-NEXT:    vmovaps %zmm0, 832(%rax)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm5, 768(%rax)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm15, 704(%rax)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm22, 640(%rax)
-; AVX512DQ-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovaps %zmm2, 576(%rax)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm31, 512(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm4, 768(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm6, 704(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm15, 640(%rax)
+; AVX512DQ-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vmovaps %zmm4, 576(%rax)
+; AVX512DQ-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vmovaps %zmm0, 512(%rax)
 ; AVX512DQ-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQ-FAST-NEXT:    vmovaps %zmm0, 448(%rax)
 ; AVX512DQ-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQ-FAST-NEXT:    vmovaps %zmm0, 384(%rax)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm17, 320(%rax)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm19, 256(%rax)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm23, 192(%rax)
-; AVX512DQ-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovaps %zmm0, 128(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm9, 320(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm11, 256(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm17, 192(%rax)
+; AVX512DQ-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512DQ-FAST-NEXT:    vmovaps %zmm4, 128(%rax)
 ; AVX512DQ-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQ-FAST-NEXT:    vmovaps %zmm0, 64(%rax)
 ; AVX512DQ-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQ-FAST-NEXT:    vmovaps %zmm0, (%rax)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm12, 3520(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm16, 3520(%rax)
 ; AVX512DQ-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQ-FAST-NEXT:    vmovaps %zmm0, 3456(%rax)
 ; AVX512DQ-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQ-FAST-NEXT:    vmovaps %zmm0, 3392(%rax)
-; AVX512DQ-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQ-FAST-NEXT:    vmovaps %zmm0, 3328(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm21, 3328(%rax)
 ; AVX512DQ-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQ-FAST-NEXT:    vmovaps %zmm0, 3264(%rax)
 ; AVX512DQ-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQ-FAST-NEXT:    vmovaps %zmm0, 3200(%rax)
-; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm6, 3072(%rax)
+; AVX512DQ-FAST-NEXT:    vmovdqa64 %zmm30, 3072(%rax)
 ; AVX512DQ-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQ-FAST-NEXT:    vmovaps %zmm0, 3136(%rax)
-; AVX512DQ-FAST-NEXT:    addq $6568, %rsp # imm = 0x19A8
+; AVX512DQ-FAST-NEXT:    addq $6120, %rsp # imm = 0x17E8
 ; AVX512DQ-FAST-NEXT:    vzeroupper
 ; AVX512DQ-FAST-NEXT:    retq
 ;
 ; AVX512BW-ONLY-SLOW-LABEL: store_i64_stride7_vf64:
 ; AVX512BW-ONLY-SLOW:       # %bb.0:
-; AVX512BW-ONLY-SLOW-NEXT:    subq $6600, %rsp # imm = 0x19C8
+; AVX512BW-ONLY-SLOW-NEXT:    subq $6248, %rsp # imm = 0x1868
 ; AVX512BW-ONLY-SLOW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 (%rdi), %zmm3
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 (%rdi), %zmm7
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 64(%rdi), %zmm19
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 (%rsi), %zmm9
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 64(%rsi), %zmm16
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 64(%rdx), %zmm17
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm17, (%rsp) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 (%rdx), %zmm3
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 64(%rdi), %zmm29
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 (%rsi), %zmm4
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 (%rcx), %zmm4
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 64(%rsi), %zmm13
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm13, (%rsp) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 64(%rdx), %zmm20
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm20, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 (%rdx), %zmm5
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 (%rcx), %zmm8
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 64(%rcx), %zmm18
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm10 = [11,3,11,3,11,3,11,3]
-; AVX512BW-ONLY-SLOW-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm28 = [2,10,0,3,2,10,0,3]
-; AVX512BW-ONLY-SLOW-NEXT:    # zmm28 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm11 = [11,3,11,3,11,3,11,3]
+; AVX512BW-ONLY-SLOW-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm31 = [2,10,0,3,2,10,0,3]
+; AVX512BW-ONLY-SLOW-NEXT:    # zmm31 = mem[0,1,2,3,0,1,2,3]
 ; AVX512BW-ONLY-SLOW-NEXT:    movb $96, %r10b
 ; AVX512BW-ONLY-SLOW-NEXT:    kmovd %r10d, %k1
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 (%r8), %zmm1
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 (%r9), %zmm11
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 (%rax), %zmm6
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 64(%rax), %zmm9
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 (%r9), %zmm15
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 (%rax), %zmm5
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 64(%rax), %zmm6
+; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm25 = [15,7,15,7,15,7,15,7]
+; AVX512BW-ONLY-SLOW-NEXT:    # zmm25 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [9,1,9,1,9,1,9,1]
 ; AVX512BW-ONLY-SLOW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm11, %zmm0, %zmm2
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm14
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [4,9,0,3,4,9,0,3]
-; AVX512BW-ONLY-SLOW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm6, %zmm0, %zmm2
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm12
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm15, %zmm0, %zmm2
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm8
+; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm24 = [4,9,0,3,4,9,0,3]
+; AVX512BW-ONLY-SLOW-NEXT:    # zmm24 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm5, %zmm24, %zmm2
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm0
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm4, %zmm10, %zmm0
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm2
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm8, %zmm28, %zmm2
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm9, %zmm11, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm2
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm4, %zmm31, %zmm2
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k1}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa (%r9), %ymm5
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa (%r9), %ymm7
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa 64(%r9), %ymm3
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa (%r8), %ymm0
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa 64(%r8), %ymm4
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm5[0],ymm0[2],ymm5[2]
+; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm7[0],ymm0[2],ymm7[2]
 ; AVX512BW-ONLY-SLOW-NEXT:    movb $28, %r10b
 ; AVX512BW-ONLY-SLOW-NEXT:    kmovd %r10d, %k2
-; AVX512BW-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm2 {%k2} = zmm0[2,3,2,3],zmm6[2,3,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm2 {%k2} = zmm0[2,3,2,3],zmm5[2,3,2,3]
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [4,12,0,5,4,12,0,5]
 ; AVX512BW-ONLY-SLOW-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3]
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm0
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm5
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm11, %zmm2, %zmm0
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm16
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm15, %zmm2, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm7
 ; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [0,1,12,7,0,1,12,7]
 ; AVX512BW-ONLY-SLOW-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm6, %zmm2, %zmm0
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm11
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm5, %zmm2, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm14
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [5,0,14,6,5,0,14,6]
-; AVX512BW-ONLY-SLOW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm2
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm17
+; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm13 = [5,0,14,6,5,0,14,6]
+; AVX512BW-ONLY-SLOW-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm15, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm15, %zmm9
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm13, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [0,13,6,7,0,13,6,7]
+; AVX512BW-ONLY-SLOW-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm5, %zmm2, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm15
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm9, %zmm25, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm10 = [6,13,14,7,6,13,14,7]
+; AVX512BW-ONLY-SLOW-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm10, %zmm5
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm0, %zmm2
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm19
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [0,13,6,7,0,13,6,7]
-; AVX512BW-ONLY-SLOW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm6, %zmm0, %zmm2
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm15
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm5 = [15,7,15,7,15,7,15,7]
-; AVX512BW-ONLY-SLOW-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm17, %zmm5, %zmm1
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm7 = [6,13,14,7,6,13,14,7]
-; AVX512BW-ONLY-SLOW-NEXT:    # zmm7 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm7, %zmm6
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm29, %zmm1
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm13, %zmm10, %zmm1
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm20, %zmm2
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm18, %zmm28, %zmm2
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm19, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm16, %zmm11, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm17, %zmm2
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm18, %zmm31, %zmm2
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}
 ; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm1 = ymm4[0],ymm3[0],ymm4[2],ymm3[2]
-; AVX512BW-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm2 {%k2} = zmm1[2,3,2,3],zmm9[2,3,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm2 {%k2} = zmm1[2,3,2,3],zmm6[2,3,2,3]
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 64(%r8), %zmm1
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 64(%r9), %zmm3
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm18
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm3, %zmm14, %zmm2
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm12, %zmm4
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm9, %zmm12, %zmm2
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm3, %zmm16, %zmm2
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm9, %zmm11, %zmm2
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm2
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 64(%r9), %zmm16
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm16, %zmm8, %zmm3
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm6, %zmm24, %zmm3
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm19, %zmm2
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm9, %zmm0, %zmm2
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm3, %zmm5, %zmm1
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm7, %zmm9
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 128(%rdi), %zmm23
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 128(%rsi), %zmm24
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm23, %zmm1
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm24, %zmm10, %zmm1
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm10, %zmm30
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 128(%rdx), %zmm25
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 128(%rcx), %zmm12
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm25, %zmm2
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm12, %zmm28, %zmm2
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 128(%rax), %zmm9
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa 128(%r9), %ymm6
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm16, %zmm7, %zmm3
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm6, %zmm14, %zmm3
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm16, %zmm3
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm13, %zmm3
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm6, %zmm15, %zmm3
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm29
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm16, %zmm25, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm10, %zmm6
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 128(%rdi), %zmm27
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 128(%rsi), %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm27, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm11, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm17
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 128(%rdx), %zmm21
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 128(%rcx), %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm21, %zmm3
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm31, %zmm3
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm3 {%k1}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 128(%rax), %zmm5
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa 128(%r9), %ymm4
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa 128(%r8), %ymm1
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm1 = ymm1[0],ymm6[0],ymm1[2],ymm6[2]
-; AVX512BW-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm2 {%k2} = zmm1[2,3,2,3],zmm9[2,3,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm1 = ymm1[0],ymm4[0],ymm1[2],ymm4[2]
+; AVX512BW-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm3 {%k2} = zmm1[2,3,2,3],zmm5[2,3,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 128(%r8), %zmm1
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 128(%r9), %zmm21
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm21, %zmm14, %zmm2
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm9, %zmm4, %zmm2
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm10
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm16, %zmm3
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm21, %zmm16, %zmm2
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm9, %zmm11, %zmm2
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm21, %zmm2
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm19, %zmm2
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm9, %zmm0, %zmm2
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm21, %zmm5, %zmm1
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm7, %zmm9
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 192(%rdi), %zmm27
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 192(%rsi), %zmm26
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm27, %zmm1
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm26, %zmm30, %zmm1
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 192(%rdx), %zmm17
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 192(%rcx), %zmm14
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm17, %zmm2
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm14, %zmm28, %zmm2
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 192(%rax), %zmm21
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa 192(%r9), %ymm9
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 128(%r9), %zmm6
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm6, %zmm8, %zmm3
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm11
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm5, %zmm24, %zmm3
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm6, %zmm7, %zmm3
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm5, %zmm14, %zmm3
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm3
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm13, %zmm3
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm5, %zmm15, %zmm3
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm20
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm6, %zmm25, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm10, %zmm5
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 192(%rdi), %zmm26
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 192(%rsi), %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm26, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm17, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 192(%rdx), %zmm18
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 192(%rcx), %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm18, %zmm3
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm31, %zmm3
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm3 {%k1}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 192(%rax), %zmm2
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa 192(%r9), %ymm5
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa 192(%r8), %ymm1
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm1 = ymm1[0],ymm9[0],ymm1[2],ymm9[2]
-; AVX512BW-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm2 {%k2} = zmm1[2,3,2,3],zmm21[2,3,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm1 = ymm1[0],ymm5[0],ymm1[2],ymm5[2]
+; AVX512BW-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm3 {%k2} = zmm1[2,3,2,3],zmm2[2,3,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 192(%r8), %zmm1
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 192(%r9), %zmm0
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm18, %zmm2
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm21, %zmm4, %zmm2
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm16, %zmm2
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm21, %zmm11, %zmm2
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm2
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 192(%r9), %zmm8
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm8, %zmm11, %zmm3
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm2, %zmm24, %zmm3
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm8, %zmm7, %zmm3
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm28
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm2, %zmm14, %zmm3
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm30
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm5
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm13, %zmm5
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm15, %zmm3
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm2, %zmm15, %zmm5
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm9
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm8, %zmm25, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm10, %zmm0
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm19, %zmm2
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm21, %zmm15, %zmm2
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm15, %zmm31
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm5, %zmm1
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm7, %zmm21
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 256(%rdi), %zmm21
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 256(%rdi), %zmm15
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 256(%rsi), %zmm22
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm21, %zmm2
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm30, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm22, %zmm30, %zmm2
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 256(%rdx), %zmm16
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 256(%rcx), %zmm15
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm16, %zmm4
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm15, %zmm28, %zmm4
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm4 {%k1}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm15, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm17, %zmm14
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm22, %zmm17, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 256(%rdx), %zmm12
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 256(%rcx), %zmm23
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm12, %zmm5
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm23, %zmm31, %zmm5
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm5 {%k1}
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 256(%rax), %zmm0
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa 256(%r9), %ymm6
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa 256(%r8), %ymm2
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm2 = ymm2[0],ymm6[0],ymm2[2],ymm6[2]
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm6
-; AVX512BW-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm4 {%k2} = zmm2[2,3,2,3],zmm0[2,3,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 256(%r8), %zmm2
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 256(%r9), %zmm13
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm4
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm13, %zmm18, %zmm4
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm10, %zmm4
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm4
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm0
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm13, %zmm3, %zmm4
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm6, %zmm11, %zmm4
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa 256(%r9), %ymm7
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa 256(%r8), %ymm1
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm1 = ymm1[0],ymm7[0],ymm1[2],ymm7[2]
+; AVX512BW-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm5 {%k2} = zmm1[2,3,2,3],zmm0[2,3,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 256(%r8), %zmm5
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 256(%r9), %zmm2
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm17
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm7
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm2, %zmm11, %zmm17
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm24, %zmm17
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm24, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm11
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm2, %zmm28, %zmm11
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm30, %zmm11
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm30, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm11
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm5, %zmm13, %zmm11
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm3, %zmm11
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm13, %zmm4
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm2, %zmm19, %zmm4
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm6, %zmm31, %zmm4
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm31, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm13, %zmm5, %zmm2
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm2, %zmm7, %zmm6
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 320(%rdi), %zmm6
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 320(%rsi), %zmm2
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm4
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm2, %zmm30, %zmm4
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 320(%rdx), %zmm2
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 320(%rcx), %zmm13
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm9
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm13, %zmm28, %zmm9
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm9 {%k1}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 320(%rax), %zmm3
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 320(%r9), %ymm30
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %ymm30, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa 320(%r8), %ymm4
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm4 = ymm4[0],ymm30[0],ymm4[2],ymm30[2]
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm30
-; AVX512BW-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm9 {%k2} = zmm4[2,3,2,3],zmm3[2,3,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 320(%r8), %zmm4
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 320(%r9), %zmm9
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm3
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm9, %zmm18, %zmm3
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm30, %zmm10, %zmm3
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm3
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm9, %zmm0, %zmm3
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm30, %zmm11, %zmm3
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm9, %zmm0
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm4, %zmm19, %zmm0
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm30, %zmm31, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm4
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm2, %zmm25, %zmm5
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm5, %zmm10, %zmm0
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm9, %zmm5, %zmm4
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm4, %zmm7, %zmm30
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm30, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm11 = [13,5,13,5,13,5,13,5]
-; AVX512BW-ONLY-SLOW-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm0
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm8, %zmm11, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 320(%rdi), %zmm19
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 320(%rsi), %zmm0
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [6,14,6,14,6,14,6,14]
-; AVX512BW-ONLY-SLOW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm3
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm8, %zmm0, %zmm3
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm3
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm0
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm8, %zmm5, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm19, %zmm5
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm14, %zmm5
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 320(%rdx), %zmm11
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 320(%rcx), %zmm17
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm14
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm17, %zmm31, %zmm14
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm14 {%k1}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 320(%rax), %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa 320(%r9), %ymm5
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa 320(%r8), %ymm0
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm5 = ymm0[0],ymm5[0],ymm0[2],ymm5[2]
+; AVX512BW-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm14 {%k2} = zmm5[2,3,2,3],zmm1[2,3,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 320(%r8), %zmm5
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 320(%r9), %zmm14
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm14, %zmm7, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm24, %zmm0
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [3,0,12,4,3,0,12,4]
-; AVX512BW-ONLY-SLOW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm0, %zmm8
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm9
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm4
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm8, %zmm11, %zmm4
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm4
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm8, %zmm3, %zmm4
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm4
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm8, %zmm5, %zmm4
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm0, %zmm8
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm10
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm25, %zmm1
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm12, %zmm11, %zmm1
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm25, %zmm1
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm12, %zmm3, %zmm1
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm25, %zmm1
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm12, %zmm5, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm14, %zmm28, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm30, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm5, %zmm13, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm3, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm3
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm14, %zmm25, %zmm5
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm5, %zmm10, %zmm1
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm25, %zmm0, %zmm12
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm12, %zmm30
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm17, %zmm1
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm14, %zmm11, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 384(%rdx), %zmm5
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 384(%rcx), %zmm24
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm24, %zmm25, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 384(%rdi), %zmm10
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 384(%rsi), %zmm1
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm17, %zmm1
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm14, %zmm3, %zmm1
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm17, %zmm1
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm14, %zmm5, %zmm1
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm17, %zmm0, %zmm14
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm17
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm16, %zmm1
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm15, %zmm11, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm25, %zmm10
+; AVX512BW-ONLY-SLOW-NEXT:    movb $24, %r10b
+; AVX512BW-ONLY-SLOW-NEXT:    kmovd %r10d, %k3
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm10 {%k3}
+; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm1 = [0,8,0,8,0,8,0,8]
+; AVX512BW-ONLY-SLOW-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 # 64-byte Folded Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm16, %zmm1, %zmm29
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm29, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm6, %zmm1, %zmm20
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm20, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm8, %zmm1, %zmm9
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm2, %zmm1, %zmm4
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm14, %zmm1, %zmm3
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 384(%r8), %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm1 = <u,1,2,3,4,15,u,u>
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm1, %zmm10
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 384(%r9), %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm3 = <u,1,2,3,4,5,15,u>
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm3, %zmm10
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm1, %zmm0, %zmm2
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm1, %zmm0, %zmm7
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm1, %zmm0, %zmm28
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm28, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm13, %zmm1
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm16, %zmm1
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm15, %zmm3, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm7 = [13,5,13,5,13,5,13,5]
+; AVX512BW-ONLY-SLOW-NEXT:    # zmm7 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm7, %zmm2
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm10 = [6,14,6,14,6,14,6,14]
+; AVX512BW-ONLY-SLOW-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm10, %zmm2
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm25, %zmm2
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [3,0,12,4,3,0,12,4]
+; AVX512BW-ONLY-SLOW-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm2, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm3
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 (%rsp), %zmm0 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm13, %zmm7, %zmm1
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm16, %zmm1
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm15, %zmm5, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm13, %zmm10, %zmm1
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm16, %zmm0, %zmm15
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm15, %zmm16
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm1
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm13, %zmm11, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm13, %zmm25, %zmm1
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm1
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm4
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm13, %zmm3, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm2, %zmm13
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm13, %zmm14
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm21, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm29 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm29, %zmm7, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm21, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm29, %zmm10, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm21, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm29, %zmm25, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm21, %zmm2, %zmm29
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm18, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm20 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm20, %zmm7, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm18, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm20, %zmm10, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm18, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm20, %zmm25, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm18, %zmm2, %zmm20
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm12, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm23, %zmm7, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm12, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm23, %zmm10, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm12, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm23, %zmm25, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm12, %zmm2, %zmm23
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm23, %zmm21
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm17, %zmm7, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm17, %zmm10, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm17, %zmm25, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm11, %zmm2, %zmm17
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm17, %zmm28
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 448(%rdx), %zmm8
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 448(%rcx), %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm6
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm31, %zmm6
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm24, %zmm11
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm24, %zmm5, %zmm31
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm31, (%rsp) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm24, %zmm7, %zmm1
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm1
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm13, %zmm5, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm24, %zmm10, %zmm1
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm2, %zmm0, %zmm13
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm13, %zmm18
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 448(%rdx), %zmm14
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 448(%rcx), %zmm1
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm12
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm28, %zmm12
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm19
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm3, %zmm19
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 384(%rdx), %zmm3
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 384(%rcx), %zmm20
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm20, %zmm3, %zmm28
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm28, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm2
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm20, %zmm11, %zmm2
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm20, %zmm3, %zmm4
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm25
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm20, %zmm5, %zmm25
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm3, %zmm0, %zmm20
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm7
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm14, %zmm0, %zmm7
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm15
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm5, %zmm14
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm11, %zmm15
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm5, %zmm2, %zmm11
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm9
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm8, %zmm2, %zmm9
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm12
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm10, %zmm12
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm13
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm7, %zmm13
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm25, %zmm8
 ; AVX512BW-ONLY-SLOW-NEXT:    movb $48, %r10b
-; AVX512BW-ONLY-SLOW-NEXT:    kmovd %r10d, %k3
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm1 = [1,0,10,2,1,0,10,2]
-; AVX512BW-ONLY-SLOW-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm2
+; AVX512BW-ONLY-SLOW-NEXT:    kmovd %r10d, %k4
+; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm5 = [1,0,10,2,1,0,10,2]
+; AVX512BW-ONLY-SLOW-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm1, %zmm2
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm5, %zmm2
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm9 {%k3} = zmm0[0],zmm3[0],zmm0[2],zmm3[2],zmm0[4],zmm3[4],zmm0[6],zmm3[6]
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm4
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm5
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm9
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [0,8,0,1,0,8,0,1]
-; AVX512BW-ONLY-SLOW-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm3, %zmm2, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm3 {%k4} = zmm0[0],zmm1[0],zmm0[2],zmm1[2],zmm0[4],zmm1[4],zmm0[6],zmm1[6]
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm2
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm3
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm16
+; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [0,8,0,1,0,8,0,1]
+; AVX512BW-ONLY-SLOW-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm4, %zmm0
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm3, %zmm11, %zmm4
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti128 {{.*#+}} ymm13 = [6,14,6,14]
-; AVX512BW-ONLY-SLOW-NEXT:    # ymm13 = mem[0,1,0,1]
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm3, %zmm13, %zmm5
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti128 {{.*#+}} ymm8 = [15,7,15,7]
-; AVX512BW-ONLY-SLOW-NEXT:    # ymm8 = mem[0,1,0,1]
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm3, %zmm8, %zmm9
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 (%rsp), %zmm0 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm7, %zmm2
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm10, %zmm3
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm25, %zmm16
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm5, %zmm2
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm14 {%k4} = zmm0[0],zmm1[0],zmm0[2],zmm1[2],zmm0[4],zmm1[4],zmm0[6],zmm1[6]
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm3
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm29, %zmm1, %zmm3
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm2
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm14
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm4, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm7, %zmm3
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm10 {%k3} = zmm29[0],zmm0[0],zmm29[2],zmm0[2],zmm29[4],zmm0[4],zmm29[6],zmm0[6]
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm29, %zmm4
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm29, %zmm3
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm29, %zmm5
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm2, %zmm29
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm10, %zmm2
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm25, %zmm14
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm27, %zmm5, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm29 {%k4} = zmm27[0],zmm0[0],zmm27[2],zmm0[2],zmm27[4],zmm0[4],zmm27[6],zmm0[6]
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm29, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm11, %zmm4
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm13, %zmm3
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm8, %zmm5
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm24, %zmm0
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm23, %zmm1, %zmm0
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, (%rsp) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm30 {%k3} = zmm23[0],zmm24[0],zmm23[2],zmm24[2],zmm23[4],zmm24[4],zmm23[6],zmm24[6]
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm30, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm23, %zmm3
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm23, %zmm0
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm23, %zmm4
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm24, %zmm2, %zmm23
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm23, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm24, %zmm11, %zmm3
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm24, %zmm13, %zmm0
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm24, %zmm8, %zmm4
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm26, %zmm0
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm27, %zmm1, %zmm0
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm17 {%k3} = zmm27[0],zmm26[0],zmm27[2],zmm26[2],zmm27[4],zmm26[4],zmm27[6],zmm26[6]
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm27, %zmm3
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm27, %zmm0
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm27, %zmm31
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm26, %zmm2, %zmm27
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm27, %zmm2
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm27, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm27, %zmm23
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm4, %zmm27
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm27, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm26, %zmm11, %zmm3
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm26, %zmm13, %zmm0
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm26, %zmm8, %zmm31
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm7, %zmm2
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm10, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm25, %zmm23
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm26, %zmm5, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm20 {%k4} = zmm26[0],zmm0[0],zmm26[2],zmm0[2],zmm26[4],zmm0[4],zmm26[6],zmm0[6]
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm20, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm26, %zmm29
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm26, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm26, %zmm31
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm4, %zmm26
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm26, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm7, %zmm29
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm10, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm25, %zmm31
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm22, %zmm30
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm21, %zmm1, %zmm30
-; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm16 {%k3} = zmm21[0],zmm22[0],zmm21[2],zmm22[2],zmm21[4],zmm22[4],zmm21[6],zmm22[6]
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm21, %zmm3
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm21, %zmm0
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm21, %zmm27
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm21, %zmm29
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm22, %zmm2, %zmm29
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm22, %zmm11, %zmm3
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm22, %zmm13, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm15, %zmm5, %zmm30
+; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm21 {%k4} = zmm15[0],zmm22[0],zmm15[2],zmm22[2],zmm15[4],zmm22[4],zmm15[6],zmm22[6]
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm15, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm15, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm15, %zmm26
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm15, %zmm27
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm22, %zmm4, %zmm27
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm22, %zmm7, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm22, %zmm10, %zmm0
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm22, %zmm8, %zmm27
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm26
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm6, %zmm1, %zmm26
-; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm18 {%k3} = zmm6[0],zmm0[0],zmm6[2],zmm0[2],zmm6[4],zmm0[4],zmm6[6],zmm0[6]
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm28
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm3
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm23
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm24
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm2, %zmm24
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm11, %zmm28
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm13, %zmm3
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm8, %zmm23
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 448(%rdi), %zmm9
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm22, %zmm25, %zmm26
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 448(%rdi), %zmm15
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 448(%rsi), %zmm0
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm9, %zmm6
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm5, %zmm6
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm9, %zmm3
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm13, %zmm3
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm9, %zmm4
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm11, %zmm4
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 384(%rdi), %zmm21
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 384(%rsi), %zmm17
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm17, %zmm22
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm21, %zmm1, %zmm22
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm17, %zmm21, %zmm5
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm17, %zmm21, %zmm11
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm17, %zmm21, %zmm13
-; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm20 {%k3} = zmm21[0],zmm17[0],zmm21[2],zmm17[2],zmm21[4],zmm17[4],zmm21[6],zmm17[6]
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm21, %zmm18
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm17, %zmm2, %zmm21
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm17, %zmm8, %zmm18
-; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm7 {%k3} = zmm9[0],zmm0[0],zmm9[2],zmm0[2],zmm9[4],zmm0[4],zmm9[6],zmm0[6]
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm0, %zmm9, %zmm2
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm9, %zmm0, %zmm1
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm8, %zmm9
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm12 {%k1}
-; AVX512BW-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm3 = zmm3[0,1,2,3],zmm19[4,5,6,7]
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 448(%r8), %zmm6
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm0 = <0,11,u,u,4,5,6,7>
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm6, %zmm7, %zmm0
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 448(%r9), %zmm7
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm8 = <0,1,11,u,4,5,6,7>
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm7, %zmm0, %zmm8
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm10
-; AVX512BW-ONLY-SLOW-NEXT:    movb $4, %sil
-; AVX512BW-ONLY-SLOW-NEXT:    kmovd %esi, %k3
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm12 {%k3}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm8 = <0,1,2,10,u,5,6,7>
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm7, %zmm12, %zmm8
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm15, %zmm4 {%k1}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm0 = <12,u,u,3,4,5,6,13>
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm6, %zmm4, %zmm0
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm4 = <0,12,u,3,4,5,6,7>
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm7, %zmm0, %zmm4
-; AVX512BW-ONLY-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [0,8,0,8,0,8,0,8]
-; AVX512BW-ONLY-SLOW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm15, %zmm17
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm25, %zmm17
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm19, %zmm22
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm25, %zmm22
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm24
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm19, %zmm5, %zmm24
+; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm28 {%k4} = zmm19[0],zmm1[0],zmm19[2],zmm1[2],zmm19[4],zmm1[4],zmm19[6],zmm1[6]
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm28, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm19, %zmm18
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm19, %zmm2
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm19, %zmm21
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm4, %zmm21
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm7, %zmm18
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm10, %zmm2
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm19 # 64-byte Folded Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm12 # 64-byte Folded Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm12 # 64-byte Folded Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm5 # 64-byte Folded Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm12 # 64-byte Folded Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm12 # 64-byte Folded Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    movb $24, %sil
-; AVX512BW-ONLY-SLOW-NEXT:    kmovd %esi, %k5
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm25, %zmm18 {%k5}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm19, %zmm20
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm28 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm28, %zmm5, %zmm20
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm15, %zmm3
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm25 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm25, %zmm3
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm19, %zmm28, %zmm25
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm15, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm7, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm19, %zmm28, %zmm7
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm15, %zmm2
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm10, %zmm2
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm19, %zmm28, %zmm10
+; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm11 {%k4} = zmm28[0],zmm19[0],zmm28[2],zmm19[2],zmm28[4],zmm19[4],zmm28[6],zmm19[6]
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm19, %zmm4, %zmm28
+; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm9 {%k4} = zmm15[0],zmm0[0],zmm15[2],zmm0[2],zmm15[4],zmm0[4],zmm15[6],zmm0[6]
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm0, %zmm15, %zmm4
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm15, %zmm5, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm2[0,1,2,3],zmm12[4,5,6,7]
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm6 {%k1}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 448(%r8), %zmm3
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm5 = <0,11,u,u,4,5,6,7>
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm3, %zmm9, %zmm5
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm13, %zmm1 {%k1}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm9 = <12,u,u,3,4,5,6,13>
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm3, %zmm1, %zmm9
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm17 {%k3}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm1 = <u,1,2,3,4,15,u,u>
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm3, %zmm1, %zmm17
 ; AVX512BW-ONLY-SLOW-NEXT:    movb $6, %sil
-; AVX512BW-ONLY-SLOW-NEXT:    kmovd %esi, %k3
-; AVX512BW-ONLY-SLOW-NEXT:    vpbroadcastq 456(%rcx), %ymm12
-; AVX512BW-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm12 = mem[0,1,2,3],ymm12[4,5,6,7]
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm12, %zmm0, %zmm1 {%k3}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm12 = <0,1,2,9,u,u,6,7>
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm6, %zmm1, %zmm12
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm9 {%k5}
-; AVX512BW-ONLY-SLOW-NEXT:    movb $64, %sil
 ; AVX512BW-ONLY-SLOW-NEXT:    kmovd %esi, %k4
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm3 {%k4}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm1 = <u,1,2,3,4,15,u,u>
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm6, %zmm1, %zmm9
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 384(%r8), %zmm6
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm6, %zmm1, %zmm18
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 384(%r9), %zmm1
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm1, %zmm6, %zmm0
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm1, %zmm6, %zmm14
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm1, %zmm6, %zmm15
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm6, %zmm1, %zmm16
+; AVX512BW-ONLY-SLOW-NEXT:    vpbroadcastq 456(%rcx), %ymm1
+; AVX512BW-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = mem[0,1,2,3],ymm1[4,5,6,7]
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm0 {%k4}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm8 = <0,1,2,9,u,u,6,7>
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm3, %zmm0, %zmm8
+; AVX512BW-ONLY-SLOW-NEXT:    movb $64, %sil
+; AVX512BW-ONLY-SLOW-NEXT:    kmovd %esi, %k5
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm2 {%k5}
+; AVX512BW-ONLY-SLOW-NEXT:    movb $4, %sil
+; AVX512BW-ONLY-SLOW-NEXT:    kmovd %esi, %k5
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm6 {%k5}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 448(%r9), %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm1 = <0,1,11,u,4,5,6,7>
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm0, %zmm5, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm11
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm3 = <0,1,2,10,u,5,6,7>
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm0, %zmm6, %zmm3
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm5 = <0,12,u,3,4,5,6,7>
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm0, %zmm9, %zmm5
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm6 = <13,u,2,3,4,5,6,14>
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm0, %zmm2, %zmm6
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm1 = <u,1,2,3,4,5,15,u>
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm1, %zmm17
 ; AVX512BW-ONLY-SLOW-NEXT:    movb $12, %sil
-; AVX512BW-ONLY-SLOW-NEXT:    kmovd %esi, %k4
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa 448(%rdx), %xmm6
-; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm6 = xmm6[0],mem[0]
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm6, %ymm0, %ymm6
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm6, %zmm0, %zmm2 {%k4}
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti32x4 $2, 448(%r8), %zmm2, %zmm2
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm6 = <0,1,2,3,4,8,u,7>
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm7, %zmm2, %zmm6
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm2 = <u,1,2,3,4,5,15,u>
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm1, %zmm2, %zmm18
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm1 = <0,1,2,3,9,u,6,7>
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm7, %zmm12, %zmm1
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm12 = <13,u,2,3,4,5,6,14>
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm7, %zmm3, %zmm12
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm7, %zmm2, %zmm9
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 448(%rax), %zmm2
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [0,1,2,3,10,5,6,7]
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm2, %zmm8, %zmm3
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    kmovd %esi, %k5
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa 448(%rdx), %xmm2
+; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],mem[0]
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm2, %zmm0, %zmm4 {%k5}
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti32x4 $2, 448(%r8), %zmm4, %zmm2
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm4 = <0,1,2,3,4,8,u,7>
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm0, %zmm2, %zmm4
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm2 = <0,1,2,3,9,u,6,7>
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm0, %zmm8, %zmm2
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 384(%rax), %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm1, %zmm9
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 (%rsp), %zmm1 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm25, %zmm1 {%k1}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa 384(%r9), %ymm12
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa 384(%r8), %ymm8
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm8 = ymm8[0],ymm12[0],ymm8[2],ymm12[2]
+; AVX512BW-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm1 {%k2} = zmm8[2,3,2,3],zmm0[2,3,2,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, (%rsp) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm8, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm8, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm8 = [14,1,2,3,4,5,6,15]
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm8, %zmm12
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 448(%rax), %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm12 = [0,1,2,3,10,5,6,7]
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm0, %zmm3, %zmm12
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [0,1,12,3,4,5,6,7]
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm2, %zmm4, %zmm3
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm0, %zmm5, %zmm3
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [0,13,2,3,4,5,6,7]
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm0, %zmm6, %zmm3
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm0, %zmm8, %zmm17
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [0,1,2,3,4,5,8,7]
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm0, %zmm4, %zmm3
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 384(%rax), %zmm3
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm3, %zmm4, %zmm14
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm7
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm8 {%k1}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa 384(%r9), %ymm14
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu %ymm14, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa 384(%r8), %ymm4
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm4 = ymm4[0],ymm14[0],ymm4[2],ymm14[2]
-; AVX512BW-ONLY-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm8 {%k2} = zmm4[2,3,2,3],zmm3[2,3,2,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm3, %zmm4, %zmm15
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm15, %zmm8
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm3, %zmm4, %zmm16
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [0,1,2,3,4,5,8,7]
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm2, %zmm6, %zmm4
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [14,1,2,3,4,5,6,15]
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm3, %zmm4, %zmm18
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [0,1,2,3,4,9,6,7]
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm2, %zmm1, %zmm3
+; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm0, %zmm2, %zmm3
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,13,2,3,4,5,6,7]
-; AVX512BW-ONLY-SLOW-NEXT:    vpermi2q %zmm2, %zmm12, %zmm1
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vpermt2q %zmm2, %zmm4, %zmm9
 ; AVX512BW-ONLY-SLOW-NEXT:    movb $8, %sil
 ; AVX512BW-ONLY-SLOW-NEXT:    kmovd %esi, %k2
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm10 {%k2}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k5}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm3 {%k5}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k5}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm31 {%k5}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm27 {%k5}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm23 {%k5}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm11 {%k2}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm16 {%k3}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm14 {%k3}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm23 {%k3}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm31 {%k3}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm26 {%k3}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm22 {%k3}
 ; AVX512BW-ONLY-SLOW-NEXT:    movb $-31, %sil
 ; AVX512BW-ONLY-SLOW-NEXT:    kmovd %esi, %k2
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k2}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm3 {%k2}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k2}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm31 {%k2}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm31, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm27 {%k2}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm23 {%k2}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa (%rdx), %xmm1
-; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm16 {%k2}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm14 {%k2}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm23 {%k2}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm23, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm31 {%k2}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm26 {%k2}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm22 {%k2}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa (%rdx), %xmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm4 {%k5}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa 64(%rdx), %xmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm5 {%k5}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa 128(%rdx), %xmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm2 {%k4}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa 64(%rdx), %xmm1
-; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm2 {%k5}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa 192(%rdx), %xmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm3 {%k4}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa 128(%rdx), %xmm1
-; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm4 {%k4}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa 192(%rdx), %xmm1
-; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm6 {%k4}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa 256(%rdx), %xmm1
-; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm29 {%k4}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa 320(%rdx), %xmm1
-; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm24 {%k4}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa 384(%rdx), %xmm1
-; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm21 {%k4}
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti32x4 $3, (%rax), %zmm19, %zmm1
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm3 {%k5}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa 256(%rdx), %xmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm27 {%k5}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa 320(%rdx), %xmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm21 {%k5}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa 384(%rdx), %xmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm28 {%k5}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti32x4 $3, (%rax), %zmm0, %zmm0
 ; AVX512BW-ONLY-SLOW-NEXT:    movb $112, %sil
 ; AVX512BW-ONLY-SLOW-NEXT:    kmovd %esi, %k2
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k2}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm4 {%k2}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti32x4 $3, 64(%rax), %zmm0, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm5 {%k2}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti32x4 $3, 128(%rax), %zmm0, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k2}
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti32x4 $3, 64(%rax), %zmm1, %zmm1
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm3 {%k2}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti32x4 $3, 192(%rax), %zmm0, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm3 {%k2}
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti32x4 $3, 128(%rax), %zmm1, %zmm1
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k2}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti32x4 $3, 192(%rax), %zmm5, %zmm1
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm6 {%k2}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti32x4 $3, 256(%rax), %zmm1, %zmm1
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm29 {%k2}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti32x4 $3, 320(%rax), %zmm1, %zmm1
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm24 {%k2}
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti32x4 $3, 384(%rax), %zmm0, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti32x4 $3, 256(%rax), %zmm0, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm27 {%k2}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti32x4 $3, 320(%rax), %zmm0, %zmm0
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm21 {%k2}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti32x4 $3, 384(%rax), %zmm0, %zmm0
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm28 {%k2}
 ; AVX512BW-ONLY-SLOW-NEXT:    vpbroadcastq 8(%rcx), %ymm0
 ; AVX512BW-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm1 {%k3}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm5 {%k4}
 ; AVX512BW-ONLY-SLOW-NEXT:    vpbroadcastq 72(%rcx), %ymm0
 ; AVX512BW-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm2 {%k3}
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm2 {%k4}
 ; AVX512BW-ONLY-SLOW-NEXT:    vpbroadcastq 136(%rcx), %ymm0
 ; AVX512BW-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 (%rsp), %zmm3 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm3 {%k3}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm3 {%k4}
 ; AVX512BW-ONLY-SLOW-NEXT:    vpbroadcastq 200(%rcx), %ymm0
 ; AVX512BW-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm4 {%k3}
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm4 {%k4}
 ; AVX512BW-ONLY-SLOW-NEXT:    vpbroadcastq 264(%rcx), %ymm0
 ; AVX512BW-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm30 {%k3}
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm30 {%k4}
 ; AVX512BW-ONLY-SLOW-NEXT:    vpbroadcastq 328(%rcx), %ymm0
 ; AVX512BW-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm26 {%k3}
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm24 {%k4}
 ; AVX512BW-ONLY-SLOW-NEXT:    vpbroadcastq 392(%rcx), %ymm0
 ; AVX512BW-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm22 {%k3}
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm20 {%k4}
 ; AVX512BW-ONLY-SLOW-NEXT:    movb $56, %cl
 ; AVX512BW-ONLY-SLOW-NEXT:    kmovd %ecx, %k2
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k2}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm5 {%k2}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k2}
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm3 {%k2}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, (%rsp) # 64-byte Spill
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm4 {%k2}
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm30 {%k2}
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm26 {%k2}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm22 {%k2}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm24 {%k2}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm9, %zmm20 {%k2}
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k1}
@@ -17226,64 +17140,65 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm4 {%k1}
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm29 {%k1}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm5 {%k1}
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm28 {%k1}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm18 {%k1}
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm11 {%k1}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm7 {%k1}
 ; AVX512BW-ONLY-SLOW-NEXT:    movb $120, %cl
 ; AVX512BW-ONLY-SLOW-NEXT:    kmovd %ecx, %k1
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm16 {%k1}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm15 # 64-byte Folded Reload
-; AVX512BW-ONLY-SLOW-NEXT:    # zmm15 = zmm0[0,1,2,3],mem[4,5,6,7]
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm17 {%k1}
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm19 {%k1}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm19 {%k1}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm16 # 64-byte Folded Reload
+; AVX512BW-ONLY-SLOW-NEXT:    # zmm16 = zmm0[0,1,2,3],mem[4,5,6,7]
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm14 {%k1}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm15 {%k1}
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm25 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm25 {%k1}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm10 {%k1}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm31 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm28, %zmm31 {%k1}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm5
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm5 {%k1}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm29, %zmm25 {%k1}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm29 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm29 {%k1}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm23 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm18, %zmm23 {%k1}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm12 {%k1}
 ; AVX512BW-ONLY-SLOW-NEXT:    movb $-61, %cl
 ; AVX512BW-ONLY-SLOW-NEXT:    kmovd %ecx, %k1
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm15 {%k1}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm14 # 64-byte Folded Reload
-; AVX512BW-ONLY-SLOW-NEXT:    # zmm14 = zmm0[0,1,2,3],mem[4,5,6,7]
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm14 {%k1}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm16 {%k1}
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm12 # 64-byte Folded Reload
-; AVX512BW-ONLY-SLOW-NEXT:    # zmm12 = zmm0[0,1,2,3],mem[4,5,6,7]
+; AVX512BW-ONLY-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm13 # 64-byte Folded Reload
+; AVX512BW-ONLY-SLOW-NEXT:    # zmm13 = zmm0[0,1,2,3],mem[4,5,6,7]
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm12 {%k1}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm13 {%k1}
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm3 # 64-byte Folded Reload
-; AVX512BW-ONLY-SLOW-NEXT:    # zmm3 = zmm0[0,1,2,3],mem[4,5,6,7]
+; AVX512BW-ONLY-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm11 # 64-byte Folded Reload
+; AVX512BW-ONLY-SLOW-NEXT:    # zmm11 = zmm0[0,1,2,3],mem[4,5,6,7]
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm3 {%k1}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm11 {%k1}
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-ONLY-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm4 # 64-byte Folded Reload
 ; AVX512BW-ONLY-SLOW-NEXT:    # zmm4 = zmm0[0,1,2,3],mem[4,5,6,7]
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm4 {%k1}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm5 # 64-byte Folded Reload
+; AVX512BW-ONLY-SLOW-NEXT:    # zmm5 = zmm0[0,1,2,3],mem[4,5,6,7]
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm4 {%k1}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm5 {%k1}
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-ONLY-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm6 # 64-byte Folded Reload
 ; AVX512BW-ONLY-SLOW-NEXT:    # zmm6 = zmm0[0,1,2,3],mem[4,5,6,7]
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm6 {%k1}
-; AVX512BW-ONLY-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm13, %zmm7 # 64-byte Folded Reload
-; AVX512BW-ONLY-SLOW-NEXT:    # zmm7 = zmm13[0,1,2,3],mem[4,5,6,7]
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm7 {%k1}
+; AVX512BW-ONLY-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm7 # 64-byte Folded Reload
+; AVX512BW-ONLY-SLOW-NEXT:    # zmm7 = zmm10[0,1,2,3],mem[4,5,6,7]
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm7 {%k1}
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
 ; AVX512BW-ONLY-SLOW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm8 # 32-byte Folded Reload
 ; AVX512BW-ONLY-SLOW-NEXT:    # ymm8 = ymm8[1],mem[1],ymm8[3],mem[3]
@@ -17291,22 +17206,29 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512BW-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],mem[6,7]
 ; AVX512BW-ONLY-SLOW-NEXT:    movb $14, %cl
 ; AVX512BW-ONLY-SLOW-NEXT:    kmovd %ecx, %k1
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm8, %zmm0, %zmm13 {%k1}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm8, %zmm0, %zmm10 {%k1}
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
 ; AVX512BW-ONLY-SLOW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm8 # 32-byte Folded Reload
 ; AVX512BW-ONLY-SLOW-NEXT:    # ymm8 = ymm8[1],mem[1],ymm8[3],mem[3]
 ; AVX512BW-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,2,3,3]
 ; AVX512BW-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],mem[6,7]
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm8, %zmm0, %zmm11 {%k1}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm8, %zmm0, %zmm18 {%k1}
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
 ; AVX512BW-ONLY-SLOW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm8 # 32-byte Folded Reload
 ; AVX512BW-ONLY-SLOW-NEXT:    # ymm8 = ymm8[1],mem[1],ymm8[3],mem[3]
 ; AVX512BW-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,2,3,3]
 ; AVX512BW-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],mem[6,7]
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm28 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm8, %zmm0, %zmm28 {%k1}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm8, %zmm0, %zmm9 {%k1}
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm8 # 32-byte Folded Reload
+; AVX512BW-ONLY-SLOW-NEXT:    # ymm8 = ymm8[1],mem[1],ymm8[3],mem[3]
+; AVX512BW-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,2,3,3]
+; AVX512BW-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],mem[6,7]
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm8, %zmm0, %zmm3 {%k1}
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
 ; AVX512BW-ONLY-SLOW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm8 # 32-byte Folded Reload
 ; AVX512BW-ONLY-SLOW-NEXT:    # ymm8 = ymm8[1],mem[1],ymm8[3],mem[3]
@@ -17321,88 +17243,81 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512BW-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],mem[6,7]
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
 ; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm8, %zmm0, %zmm1 {%k1}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm8 # 32-byte Folded Reload
-; AVX512BW-ONLY-SLOW-NEXT:    # ymm8 = ymm8[1],mem[1],ymm8[3],mem[3]
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm8 # 32-byte Folded Reload
+; AVX512BW-ONLY-SLOW-NEXT:    # ymm8 = ymm0[1],mem[1],ymm0[3],mem[3]
 ; AVX512BW-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,2,3,3]
 ; AVX512BW-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],mem[6,7]
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm8, %zmm0, %zmm0 {%k1}
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm8 # 32-byte Folded Reload
-; AVX512BW-ONLY-SLOW-NEXT:    # ymm8 = ymm8[1],mem[1],ymm8[3],mem[3]
-; AVX512BW-ONLY-SLOW-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,2,3,3]
-; AVX512BW-ONLY-SLOW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],mem[6,7]
-; AVX512BW-ONLY-SLOW-NEXT:    vinserti64x4 $0, %ymm8, %zmm0, %zmm20 {%k1}
 ; AVX512BW-ONLY-SLOW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm7, 3008(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm5, 2944(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm20, 2880(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovaps %zmm7, 2816(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm22, 2752(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm21, 2688(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm23, 2624(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm12, 2944(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, 2880(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovups (%rsp), %zmm0 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vmovaps %zmm0, 2816(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm20, 2752(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm28, 2688(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm22, 2624(%rax)
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm6, 2560(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm31, 2496(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm0, 2432(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm23, 2496(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, 2432(%rax)
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovaps %zmm6, 2368(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm26, 2304(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm24, 2240(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm27, 2176(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm4, 2112(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm10, 2048(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm1, 1984(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovaps %zmm4, 1920(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm24, 2304(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm21, 2240(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm26, 2176(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm5, 2112(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm29, 2048(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, 1984(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vmovaps %zmm5, 1920(%rax)
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm30, 1856(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm29, 1792(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovaps %zmm0, 1728(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, 1664(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm27, 1792(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm31, 1728(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm4, 1664(%rax)
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm25, 1600(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm2, 1536(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovaps %zmm3, 1472(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm3, 1536(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vmovaps %zmm4, 1472(%rax)
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovaps %zmm0, 1408(%rax)
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovaps %zmm0, 1344(%rax)
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovaps %zmm0, 1280(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm12, 1216(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm19, 1152(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm28, 1088(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovaps %zmm2, 1024(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovups (%rsp), %zmm0 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm11, 1216(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm15, 1152(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm9, 1088(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vmovaps %zmm3, 1024(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovaps %zmm0, 960(%rax)
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovaps %zmm0, 896(%rax)
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovaps %zmm0, 832(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm14, 768(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm17, 704(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm11, 640(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-ONLY-SLOW-NEXT:    vmovaps %zmm1, 576(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm13, 768(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm14, 704(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm18, 640(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vmovaps %zmm2, 576(%rax)
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovaps %zmm0, 512(%rax)
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovaps %zmm0, 448(%rax)
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovaps %zmm0, 384(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm15, 320(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm16, 256(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm13, 192(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm16, 320(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm19, 256(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm10, 192(%rax)
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovaps %zmm0, 128(%rax)
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovaps %zmm0, 64(%rax)
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovaps %zmm0, (%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm9, 3520(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm17, 3520(%rax)
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovaps %zmm0, 3456(%rax)
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
@@ -17413,820 +17328,816 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovaps %zmm0, 3264(%rax)
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovaps %zmm0, 3200(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    vmovdqa64 %zmm18, 3072(%rax)
+; AVX512BW-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-ONLY-SLOW-NEXT:    vmovaps %zmm0, 3072(%rax)
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-ONLY-SLOW-NEXT:    vmovaps %zmm0, 3136(%rax)
-; AVX512BW-ONLY-SLOW-NEXT:    addq $6600, %rsp # imm = 0x19C8
+; AVX512BW-ONLY-SLOW-NEXT:    addq $6248, %rsp # imm = 0x1868
 ; AVX512BW-ONLY-SLOW-NEXT:    vzeroupper
 ; AVX512BW-ONLY-SLOW-NEXT:    retq
 ;
 ; AVX512BW-ONLY-FAST-LABEL: store_i64_stride7_vf64:
 ; AVX512BW-ONLY-FAST:       # %bb.0:
-; AVX512BW-ONLY-FAST-NEXT:    subq $6696, %rsp # imm = 0x1A28
+; AVX512BW-ONLY-FAST-NEXT:    subq $6120, %rsp # imm = 0x17E8
 ; AVX512BW-ONLY-FAST-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 (%rdi), %zmm5
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 64(%rdi), %zmm22
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm22, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 (%rsi), %zmm23
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 64(%rsi), %zmm17
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 64(%rdx), %zmm24
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm24, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 (%rdx), %zmm6
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 (%rcx), %zmm7
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 (%rdi), %zmm7
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 64(%rcx), %zmm9
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm10 = [11,3,11,3,11,3,11,3]
-; AVX512BW-ONLY-FAST-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm12 = [2,10,0,3,2,10,0,3]
-; AVX512BW-ONLY-FAST-NEXT:    # zmm12 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 64(%rdi), %zmm13
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 (%rsi), %zmm8
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 64(%rsi), %zmm20
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm20, (%rsp) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 64(%rdx), %zmm14
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 (%rdx), %zmm10
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 (%rcx), %zmm11
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 64(%rcx), %zmm19
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm9 = [11,3,11,3,11,3,11,3]
+; AVX512BW-ONLY-FAST-NEXT:    # zmm9 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm3 = [2,10,0,3,2,10,0,3]
+; AVX512BW-ONLY-FAST-NEXT:    # zmm3 = mem[0,1,2,3,0,1,2,3]
 ; AVX512BW-ONLY-FAST-NEXT:    movb $96, %r10b
 ; AVX512BW-ONLY-FAST-NEXT:    kmovd %r10d, %k1
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 (%r8), %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 (%r9), %zmm8
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 (%rax), %zmm3
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 64(%rax), %zmm4
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 (%r9), %zmm18
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 (%rax), %zmm4
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 64(%rax), %zmm5
+; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm23 = [15,7,15,7,15,7,15,7]
+; AVX512BW-ONLY-FAST-NEXT:    # zmm23 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [9,1,9,1,9,1,9,1]
 ; AVX512BW-ONLY-FAST-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm8, %zmm0, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm14
-; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm27 = [4,9,0,3,4,9,0,3]
-; AVX512BW-ONLY-FAST-NEXT:    # zmm27 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm3, %zmm27, %zmm2
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm18, %zmm0, %zmm2
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm6
+; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [4,9,0,3,4,9,0,3]
+; AVX512BW-ONLY-FAST-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm4, %zmm0, %zmm2
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm12
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm5, %zmm0
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm23, %zmm10, %zmm0
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm6, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm7, %zmm12, %zmm2
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm7, %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm8, %zmm9, %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm10, %zmm2
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm11, %zmm3, %zmm2
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k1}
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqa (%r9), %ymm0
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa 64(%r9), %ymm11
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa (%r8), %ymm6
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 64(%r8), %ymm26
-; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm6[0],ymm0[0],ymm6[2],ymm0[2]
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa 64(%r9), %ymm10
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa (%r8), %ymm8
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa 64(%r8), %ymm7
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm8[0],ymm0[0],ymm8[2],ymm0[2]
 ; AVX512BW-ONLY-FAST-NEXT:    movb $28, %r10b
 ; AVX512BW-ONLY-FAST-NEXT:    kmovd %r10d, %k2
-; AVX512BW-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm2 {%k2} = zmm0[2,3,2,3],zmm3[2,3,2,3]
+; AVX512BW-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm2 {%k2} = zmm0[2,3,2,3],zmm4[2,3,2,3]
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [4,12,0,5,4,12,0,5]
 ; AVX512BW-ONLY-FAST-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3]
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, %zmm0
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm8, %zmm6
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm8, %zmm2, %zmm0
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm7
-; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [0,1,12,7,0,1,12,7]
-; AVX512BW-ONLY-FAST-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm3, %zmm2, %zmm0
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm18
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm8 = [5,0,14,6,5,0,14,6]
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm18, %zmm2, %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm8 = [0,1,12,7,0,1,12,7]
 ; AVX512BW-ONLY-FAST-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm6, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm1, %zmm8, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [0,13,6,7,0,13,6,7]
-; AVX512BW-ONLY-FAST-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm3, %zmm0, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm20
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm15 = [15,7,15,7,15,7,15,7]
-; AVX512BW-ONLY-FAST-NEXT:    # zmm15 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm6, %zmm15, %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm16 = [6,13,14,7,6,13,14,7]
-; AVX512BW-ONLY-FAST-NEXT:    # zmm16 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm1, %zmm16, %zmm3
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm22, %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm17, %zmm10, %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm24, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm9, %zmm12, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}
-; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm1 = ymm26[0],ymm11[0],ymm26[2],ymm11[2]
-; AVX512BW-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm2 {%k2} = zmm1[2,3,2,3],zmm4[2,3,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm4, %zmm8, %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm22 = [5,0,14,6,5,0,14,6]
+; AVX512BW-ONLY-FAST-NEXT:    # zmm22 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm18, %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm1, %zmm22, %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm15 = [0,13,6,7,0,13,6,7]
+; AVX512BW-ONLY-FAST-NEXT:    # zmm15 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm4, %zmm15, %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm18, %zmm23, %zmm1
+; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm18 = [6,13,14,7,6,13,14,7]
+; AVX512BW-ONLY-FAST-NEXT:    # zmm18 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm1, %zmm18, %zmm4
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm13, %zmm1
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm20, %zmm9, %zmm1
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm14, %zmm4
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm19, %zmm3, %zmm4
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
+; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm1 = ymm7[0],ymm10[0],ymm7[2],ymm10[2]
+; AVX512BW-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm4 {%k2} = zmm1[2,3,2,3],zmm5[2,3,2,3]
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 64(%r8), %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 64(%r9), %zmm3
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm3, %zmm14, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm4, %zmm27, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm3, %zmm7, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm4, %zmm18, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm1, %zmm8, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm4, %zmm0, %zmm2
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 64(%r9), %zmm30
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, %zmm4
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm30, %zmm6, %zmm4
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm5, %zmm12, %zmm4
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm5, %zmm7
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, %zmm4
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm5
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm30, %zmm2, %zmm4
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm7, %zmm8, %zmm4
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm30, %zmm2
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm1, %zmm22, %zmm2
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm7, %zmm15, %zmm2
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm3, %zmm15, %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm1, %zmm16, %zmm4
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm30, %zmm23, %zmm1
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm1, %zmm18, %zmm7
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 128(%rdi), %zmm1
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 128(%rsi), %zmm19
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm19, %zmm10, %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 128(%rdx), %zmm3
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 128(%rcx), %zmm24
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm24, %zmm12, %zmm3
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, %zmm3 {%k1}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 128(%rax), %zmm6
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 128(%rsi), %zmm26
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm26, %zmm9, %zmm1
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm9, %zmm29
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 128(%rdx), %zmm4
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 128(%rcx), %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm3, %zmm4
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm9
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 128(%rax), %zmm7
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqa 128(%r9), %ymm1
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa 128(%r8), %ymm4
-; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm2 = ymm4[0],ymm1[0],ymm4[2],ymm1[2]
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %ymm4, %ymm25
-; AVX512BW-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm3 {%k2} = zmm2[2,3,2,3],zmm6[2,3,2,3]
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa 128(%r8), %ymm0
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm1 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
+; AVX512BW-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm4 {%k2} = zmm1[2,3,2,3],zmm7[2,3,2,3]
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 128(%r8), %zmm4
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 128(%r9), %zmm19
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm4, %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm19, %zmm6, %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm12, %zmm2
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm7, %zmm12, %zmm3
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 128(%r8), %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 128(%r9), %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm1, %zmm14, %zmm3
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm6, %zmm27, %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm4, %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm19, %zmm5, %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm5, %zmm13
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm7, %zmm8, %zmm3
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm7, %zmm4
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm1, %zmm7, %zmm3
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm6, %zmm18, %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm19, %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm4, %zmm22, %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm15, %zmm20
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm7, %zmm15, %zmm3
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, %zmm3
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm2, %zmm8, %zmm3
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm6, %zmm0, %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm19, %zmm23, %zmm4
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm4, %zmm18, %zmm7
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 192(%rdi), %zmm4
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 192(%rsi), %zmm28
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm28, %zmm29, %zmm4
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 192(%rdx), %zmm3
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm1, %zmm15, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm2, %zmm16, %zmm6
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 192(%rdi), %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 192(%rsi), %zmm11
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm11, %zmm10, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm10, %zmm28
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 192(%rdx), %zmm10
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 192(%rcx), %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm1, %zmm12, %zmm10
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm12, %zmm5
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm10 {%k1}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 192(%rax), %zmm30
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 192(%r9), %ymm17
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa 192(%r8), %ymm12
-; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm3 = ymm12[0],ymm17[0],ymm12[2],ymm17[2]
-; AVX512BW-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm10 {%k2} = zmm3[2,3,2,3],zmm30[2,3,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 192(%r8), %zmm3
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 192(%r9), %zmm0
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm10
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm14, %zmm10
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm14, %zmm7
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm30, %zmm27, %zmm10
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm13
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm4, %zmm13
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm4, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm18, %zmm6
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm30, %zmm18, %zmm13
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 192(%rcx), %zmm0
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm4
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm3, %zmm8, %zmm4
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm30, %zmm20, %zmm4
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm9, %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm4, %zmm3 {%k1}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 192(%rax), %zmm14
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa 192(%r9), %ymm0
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa 192(%r8), %ymm5
+; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm4 = ymm5[0],ymm0[0],ymm5[2],ymm0[2]
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %ymm5, %ymm24
+; AVX512BW-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm3 {%k2} = zmm4[2,3,2,3],zmm14[2,3,2,3]
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm15, %zmm3
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm3, %zmm16, %zmm30
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm30, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 256(%rdi), %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 192(%r8), %zmm5
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 192(%r9), %zmm17
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm5, %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm17, %zmm6, %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm6, %zmm4
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm14, %zmm12, %zmm3
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 256(%rsi), %zmm13
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm13, %zmm28, %zmm3
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm28, %zmm29
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm28, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 256(%rdx), %zmm28
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 256(%rcx), %zmm18
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm28, %zmm14
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm18, %zmm5, %zmm14
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm5, %zmm30
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm14 {%k1}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 256(%rax), %zmm0
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa 256(%r9), %ymm9
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 256(%r8), %ymm22
-; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm4 = ymm22[0],ymm9[0],ymm22[2],ymm9[2]
-; AVX512BW-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm14 {%k2} = zmm4[2,3,2,3],zmm0[2,3,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 256(%r8), %zmm4
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 256(%r9), %zmm3
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm4, %zmm14
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm3, %zmm7, %zmm14
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm5
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm27, %zmm14
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm27, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm4, %zmm14
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm5, %zmm2, %zmm14
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm6, %zmm14
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm5, %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm17, %zmm13, %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm14, %zmm8, %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm8, %zmm7
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm17, %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm5, %zmm22, %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm14, %zmm15, %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm5, %zmm21
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm17, %zmm23, %zmm5
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm5, %zmm18, %zmm14
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm5, %zmm1
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 256(%rdi), %zmm5
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm5, %zmm10
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm4, %zmm8, %zmm10
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm20, %zmm10
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm20, %zmm5
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 256(%rsi), %zmm15
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm29, %zmm1
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm29, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm15, %zmm29, %zmm5
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 256(%rdx), %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 256(%rcx), %zmm29
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm9, %zmm8
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm29, %zmm9, %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm5, %zmm3 {%k1}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 256(%rax), %zmm16
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa 256(%r9), %ymm10
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa 256(%r8), %ymm5
+; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm6 = ymm5[0],ymm10[0],ymm5[2],ymm10[2]
+; AVX512BW-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm3 {%k2} = zmm6[2,3,2,3],zmm16[2,3,2,3]
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 256(%r8), %zmm6
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 256(%r9), %zmm9
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm6, %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm9, %zmm4, %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm16, %zmm12, %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm16, %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm6, %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm13, %zmm16
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm9, %zmm13, %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm7, %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm9, %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm6, %zmm22, %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm20, %zmm3
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm20, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm1, %zmm15, %zmm4
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm4, %zmm16, %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm6, %zmm11
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm9, %zmm23, %zmm6
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm6, %zmm18, %zmm0
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 320(%rdi), %zmm4
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 320(%rsi), %zmm20
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm20, %zmm29, %zmm4
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 320(%rdx), %zmm31
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 320(%rcx), %zmm14
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm31, %zmm21
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm14, %zmm30, %zmm21
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm4, %zmm21 {%k1}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 320(%rdi), %zmm27
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 320(%rsi), %zmm25
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm27, %zmm6
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm25, %zmm1, %zmm6
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 320(%rdx), %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 320(%rcx), %zmm12
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm12, %zmm8, %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm8, %zmm13
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm6, %zmm0 {%k1}
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 320(%rax), %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa 320(%r9), %ymm4
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa 320(%r8), %ymm2
-; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm10 = ymm2[0],ymm4[0],ymm2[2],ymm4[2]
-; AVX512BW-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm21 {%k2} = zmm10[2,3,2,3],zmm1[2,3,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 320(%r8), %zmm10
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 320(%r9), %zmm21
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm10, %zmm0
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm21, %zmm7, %zmm0
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm1, %zmm27, %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa 320(%r9), %ymm6
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa 320(%r8), %ymm14
+; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm8 = ymm14[0],ymm6[0],ymm14[2],ymm6[2]
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %ymm14, %ymm31
+; AVX512BW-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm0 {%k2} = zmm8[2,3,2,3],zmm1[2,3,2,3]
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm10, %zmm0
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm21, %zmm3, %zmm0
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm1, %zmm6, %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 320(%r8), %zmm8
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 320(%r9), %zmm14
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm8, %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm14, %zmm4, %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm4, %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm1, %zmm2, %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, %zmm2
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm21, %zmm0
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm10, %zmm8, %zmm0
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm1, %zmm5, %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm8, %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm14, %zmm16, %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm1, %zmm7, %zmm0
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm21, %zmm15, %zmm10
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm10, %zmm16, %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm14, %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm8, %zmm22, %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm1, %zmm20, %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm8, %zmm1
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm14, %zmm23, %zmm8
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm8, %zmm18, %zmm2
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqa {{.*#+}} ymm0 = [1,3,7,7]
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 32-byte Folded Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm26 # 32-byte Folded Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %ymm26, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm25 # 32-byte Folded Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %ymm25, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %ymm17, %ymm0, %ymm12
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %ymm9, %ymm0, %ymm22
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %ymm22, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %ymm4, %ymm0, %ymm2
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm2 # 32-byte Folded Reload
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa 384(%r9), %ymm1
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa 384(%r8), %ymm2
-; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm3 = ymm2[0],ymm1[0],ymm2[2],ymm1[2]
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %ymm1, %ymm0, %ymm2
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm8 # 32-byte Folded Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm2 # 32-byte Folded Reload
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm27 = [3,0,12,4,3,0,12,4]
-; AVX512BW-ONLY-FAST-NEXT:    # zmm27 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm27, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm12 = [13,5,13,5,13,5,13,5]
-; AVX512BW-ONLY-FAST-NEXT:    # zmm12 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm3
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm1, %zmm12, %zmm3
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm30 = [6,14,6,14,6,14,6,14]
-; AVX512BW-ONLY-FAST-NEXT:    # zmm30 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm3
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm1, %zmm30, %zmm3
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm1, %zmm15, %zmm0
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    movb $48, %r10b
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm24 # 32-byte Folded Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %ymm24, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %ymm10, %ymm0, %ymm5
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %ymm6, %ymm0, %ymm31
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %ymm31, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa 384(%r9), %ymm2
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa 384(%r8), %ymm4
+; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm5 = ymm4[0],ymm2[0],ymm4[2],ymm2[2]
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %ymm2, %ymm0, %ymm4
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 384(%rdx), %zmm31
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 384(%rcx), %zmm10
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm31, %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm10, %zmm23, %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 384(%rdi), %zmm24
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm24, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 384(%rsi), %zmm18
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm18, %zmm23, %zmm24
+; AVX512BW-ONLY-FAST-NEXT:    movb $24, %r10b
 ; AVX512BW-ONLY-FAST-NEXT:    kmovd %r10d, %k3
-; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm10 = [0,8,0,1,0,8,0,1]
-; AVX512BW-ONLY-FAST-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm24 {%k3}
+; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm20 = [0,8,0,8,0,8,0,8]
+; AVX512BW-ONLY-FAST-NEXT:    # zmm20 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm23, %zmm10, %zmm1
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm20, %zmm0 # 64-byte Folded Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm30, %zmm20, %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm19, %zmm20, %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm17, %zmm20, %zmm21
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm9, %zmm20, %zmm11
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm14, %zmm20, %zmm1
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm7 = [1,0,10,2,1,0,10,2]
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 384(%r8), %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm1 = <u,1,2,3,4,15,u,u>
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm24
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 384(%r9), %zmm1
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm2 = <u,1,2,3,4,5,15,u>
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm1, %zmm2, %zmm24
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm1, %zmm0, %zmm20
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm1, %zmm0, %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm1, %zmm0, %zmm16
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm22, %zmm1
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, %zmm9
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 448(%rdx), %zmm30
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 448(%rcx), %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm1 = [6,14,6,14,6,14,6,14]
+; AVX512BW-ONLY-FAST-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm30, %zmm2
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm2
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm30, %zmm8
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm13, %zmm8
+; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm7 = [3,0,12,4,3,0,12,4]
 ; AVX512BW-ONLY-FAST-NEXT:    # zmm7 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm23, %zmm9
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm7, %zmm9
-; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm2 {%k3} = zmm0[0],zmm23[0],zmm0[2],zmm23[2],zmm0[4],zmm23[4],zmm0[6],zmm23[6]
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm11
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm30, %zmm7, %zmm11
+; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm5 = [13,5,13,5,13,5,13,5]
+; AVX512BW-ONLY-FAST-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm30, %zmm2
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm5, %zmm2
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm23, %zmm12, %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} ymm29 = [6,14,6,14]
-; AVX512BW-ONLY-FAST-NEXT:    # ymm29 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm23, %zmm29, %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} ymm25 = [15,7,15,7]
-; AVX512BW-ONLY-FAST-NEXT:    # ymm25 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm23, %zmm25, %zmm0
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm1
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm23, %zmm30
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm17, %zmm0
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm3, %zmm27, %zmm1
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm3, %zmm7, %zmm0
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm12, %zmm2
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm17, %zmm5, %zmm2
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm30, %zmm2
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm17, %zmm1, %zmm2
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm15, %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm17, %zmm23, %zmm3
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    movb $48, %r10b
+; AVX512BW-ONLY-FAST-NEXT:    kmovd %r10d, %k4
+; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [0,8,0,1,0,8,0,1]
+; AVX512BW-ONLY-FAST-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm10, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm3, %zmm7, %zmm2
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm19, %zmm4, %zmm2
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm1 {%k3} = zmm3[0],zmm0[0],zmm3[2],zmm0[2],zmm3[4],zmm0[4],zmm3[6],zmm0[6]
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm12, %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm29, %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm25, %zmm3
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm24, %zmm0
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm2, %zmm27, %zmm0
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm24, %zmm12, %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm24, %zmm30, %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm24, %zmm15, %zmm2
+; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm6 = [1,0,10,2,1,0,10,2]
+; AVX512BW-ONLY-FAST-NEXT:    # zmm6 = mem[0,1,2,3,0,1,2,3]
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm19, %zmm2
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm3, %zmm6, %zmm2
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm19, %zmm10, %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm19, %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm2, %zmm7, %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm0 {%k3} = zmm2[0],zmm19[0],zmm2[2],zmm19[2],zmm2[4],zmm19[4],zmm2[6],zmm19[6]
+; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm0 {%k4} = zmm3[0],zmm19[0],zmm3[2],zmm19[2],zmm3[4],zmm19[4],zmm3[6],zmm19[6]
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm0
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm19, %zmm12, %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm19, %zmm5, %zmm0
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm0
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm19, %zmm29, %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm19, %zmm1, %zmm0
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm19, %zmm25, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm0
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm3, %zmm27, %zmm0
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm2, %zmm12, %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm2, %zmm30, %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm2, %zmm15, %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm19, %zmm23, %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm2
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm13, %zmm7, %zmm2
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm13, %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm5, %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm13, %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm23, %zmm13
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm13, %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 (%rsp), %zmm0 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm4, %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm13, %zmm6, %zmm3
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm2 {%k4} = zmm13[0],zmm0[0],zmm13[2],zmm0[2],zmm13[4],zmm0[4],zmm13[6],zmm0[6]
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm13, %zmm2
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm5, %zmm2
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm13, %zmm2
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm2
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm23, %zmm13
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm11, %zmm10, %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm11, %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm2, %zmm7, %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm0 {%k3} = zmm2[0],zmm11[0],zmm2[2],zmm11[2],zmm2[4],zmm11[4],zmm2[6],zmm11[6]
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm7, %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm14
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm2, %zmm5, %zmm14
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm14
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm2, %zmm1, %zmm14
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm2, %zmm23, %zmm0
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm0
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm11, %zmm12, %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm2
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm26, %zmm4, %zmm2
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm26, %zmm2
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm6, %zmm2
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, (%rsp) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm3 {%k4} = zmm0[0],zmm26[0],zmm0[2],zmm26[2],zmm0[4],zmm26[4],zmm0[6],zmm26[6]
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm2
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm26, %zmm5, %zmm2
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm2
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm26, %zmm1, %zmm2
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm26, %zmm23, %zmm0
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm0
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm11, %zmm29, %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm2
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm13, %zmm7, %zmm2
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm13, %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm5, %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm13, %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm23, %zmm13
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm28, %zmm4, %zmm0
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm11, %zmm25, %zmm2
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm28, %zmm22
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm3, %zmm6, %zmm22
+; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm2 {%k4} = zmm3[0],zmm28[0],zmm3[2],zmm28[2],zmm3[4],zmm28[4],zmm3[6],zmm28[6]
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm18, %zmm0
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm28, %zmm27, %zmm0
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm28, %zmm0
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm18, %zmm12, %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm28, %zmm5, %zmm0
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm28, %zmm0
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm18, %zmm30, %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm28, %zmm1, %zmm0
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm18, %zmm15, %zmm28
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm28, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm28, %zmm23, %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm29, %zmm28
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm7, %zmm28
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm13, %zmm10, %zmm2
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm29, %zmm5, %zmm2
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm13, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm7, %zmm2
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm2
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm29, %zmm1, %zmm2
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm1 {%k3} = zmm0[0],zmm13[0],zmm0[2],zmm13[2],zmm0[4],zmm13[4],zmm0[6],zmm13[6]
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm13, %zmm12, %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm13, %zmm29, %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm13, %zmm25, %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm29, %zmm23, %zmm0
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm14, %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm31, %zmm27, %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm31, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm14, %zmm12, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm31, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm14, %zmm30, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm14, %zmm15, %zmm31
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm31, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm29
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm15, %zmm4, %zmm29
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm15, %zmm21
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm6, %zmm21
+; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm28 {%k4} = zmm0[0],zmm15[0],zmm0[2],zmm15[2],zmm0[4],zmm15[4],zmm0[6],zmm15[6]
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm20, %zmm10, %zmm2
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm15, %zmm5, %zmm2
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm20, %zmm28
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm7, %zmm28
-; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm1 {%k3} = zmm0[0],zmm20[0],zmm0[2],zmm20[2],zmm0[4],zmm20[4],zmm0[6],zmm20[6]
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm20, %zmm12, %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm20, %zmm29, %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm20, %zmm25, %zmm0
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm22
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 448(%rdx), %zmm13
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 448(%rcx), %zmm5
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm13, %zmm14
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm5, %zmm0, %zmm14
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm13, %zmm16
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm5, %zmm30, %zmm16
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm5, %zmm20
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm13, %zmm27, %zmm20
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm13, %zmm19
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm5, %zmm12, %zmm19
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm5, %zmm15, %zmm13
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 384(%rdx), %zmm4
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 384(%rcx), %zmm5
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm5, %zmm4, %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm2
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm15, %zmm1, %zmm2
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm15, %zmm23, %zmm0
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm4, %zmm5, %zmm27
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm5, %zmm4, %zmm30
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm4, %zmm0
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm5, %zmm15, %zmm4
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm5, %zmm12, %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm12, %zmm19
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm16, %zmm7, %zmm19
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm16, %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm12, %zmm5, %zmm0
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 448(%rdi), %zmm11
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm16, %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm12, %zmm1, %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm12, %zmm23, %zmm16
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 448(%rdi), %zmm12
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 448(%rsi), %zmm0
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm11, %zmm8
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm8
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm11, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm29, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm11, %zmm3
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm12, %zmm3
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 384(%rdi), %zmm6
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 384(%rsi), %zmm5
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm6, %zmm24
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm5, %zmm10, %zmm24
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm5, %zmm23
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm6, %zmm7, %zmm23
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm5, %zmm6, %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm27 {%k3} = zmm6[0],zmm5[0],zmm6[2],zmm5[2],zmm6[4],zmm5[4],zmm6[6],zmm5[6]
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm5, %zmm6, %zmm12
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm5, %zmm6, %zmm29
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm5, %zmm25, %zmm6
-; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm20 {%k3} = zmm11[0],zmm0[0],zmm11[2],zmm0[2],zmm11[4],zmm0[4],zmm11[6],zmm0[6]
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm0, %zmm11, %zmm10
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm11, %zmm0, %zmm7
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm25, %zmm11
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm8, %zmm14 {%k1}
-; AVX512BW-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm2[0,1,2,3],zmm16[4,5,6,7]
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 448(%r8), %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm5 = <0,11,u,u,4,5,6,7>
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm2, %zmm20, %zmm5
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 448(%r9), %zmm25
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm15 = <0,1,11,u,4,5,6,7>
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm25, %zmm5, %zmm15
-; AVX512BW-ONLY-FAST-NEXT:    movb $4, %sil
-; AVX512BW-ONLY-FAST-NEXT:    kmovd %esi, %k3
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm14 {%k3}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm16 = <0,1,2,10,u,5,6,7>
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm25, %zmm14, %zmm16
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm19, %zmm3 {%k1}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm5 = <12,u,u,3,4,5,6,13>
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm2, %zmm3, %zmm5
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm19 = <0,12,u,3,4,5,6,7>
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm25, %zmm5, %zmm19
-; AVX512BW-ONLY-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm20 = [0,8,0,8,0,8,0,8]
-; AVX512BW-ONLY-FAST-NEXT:    # zmm20 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm20, %zmm1 # 64-byte Folded Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm20, %zmm1 # 64-byte Folded Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm20, %zmm1 # 64-byte Folded Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm20, %zmm1 # 64-byte Folded Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm20, %zmm17 # 64-byte Folded Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm12, %zmm15
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm23, %zmm15
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm25, %zmm17
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm27, %zmm13
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm27, %zmm6, %zmm17
+; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm19 {%k4} = zmm27[0],zmm25[0],zmm27[2],zmm25[2],zmm27[4],zmm25[4],zmm27[6],zmm25[6]
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm27, %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm27, %zmm2
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm25, %zmm23, %zmm13
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm25, %zmm4, %zmm27
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm25, %zmm5, %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm25, %zmm1, %zmm2
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm26 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm21, %zmm20, %zmm26
-; AVX512BW-ONLY-FAST-NEXT:    movb $24, %sil
-; AVX512BW-ONLY-FAST-NEXT:    kmovd %esi, %k4
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm4, %zmm6 {%k4}
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm10, %zmm31, %zmm26
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm31, %zmm10, %zmm7
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm31, %zmm2
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm10, %zmm5, %zmm2
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm10, %zmm1, %zmm31
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm31, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm12, %zmm10
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm10
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm12, %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm23 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm23, %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm12, %zmm2
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm5, %zmm2
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm18, %zmm25
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm14, %zmm6, %zmm25
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm18, %zmm14, %zmm23
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm23, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm7 {%k4} = zmm14[0],zmm18[0],zmm14[2],zmm18[2],zmm14[4],zmm18[4],zmm14[6],zmm18[6]
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm18, %zmm14, %zmm5
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm14, %zmm23
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm18, %zmm1, %zmm14
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm18, %zmm4, %zmm23
+; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm11 {%k4} = zmm12[0],zmm0[0],zmm12[2],zmm0[2],zmm12[4],zmm0[4],zmm12[6],zmm0[6]
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm0, %zmm12, %zmm4
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm12, %zmm6, %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm1 # 64-byte Folded Reload
+; AVX512BW-ONLY-FAST-NEXT:    # zmm1 = zmm10[0,1,2,3],mem[4,5,6,7]
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm8 {%k1}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 448(%r8), %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm10 = <0,11,u,u,4,5,6,7>
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm3, %zmm11, %zmm10
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm6, %zmm2 {%k1}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm11 = <12,u,u,3,4,5,6,13>
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm3, %zmm2, %zmm11
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm30, %zmm15 {%k3}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm2 = <u,1,2,3,4,15,u,u>
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm3, %zmm2, %zmm15
 ; AVX512BW-ONLY-FAST-NEXT:    movb $6, %sil
 ; AVX512BW-ONLY-FAST-NEXT:    kmovd %esi, %k5
-; AVX512BW-ONLY-FAST-NEXT:    vpbroadcastq 456(%rcx), %ymm1
-; AVX512BW-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm1 = mem[0,1,2,3],ymm1[4,5,6,7]
-; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm7 {%k5}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm1 = <0,1,2,9,u,u,6,7>
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm2, %zmm7, %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm13, %zmm11 {%k4}
+; AVX512BW-ONLY-FAST-NEXT:    vpbroadcastq 456(%rcx), %ymm2
+; AVX512BW-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm2 = mem[0,1,2,3],ymm2[4,5,6,7]
+; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm2, %zmm0, %zmm0 {%k5}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm2 = <0,1,2,9,u,u,6,7>
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm3, %zmm0, %zmm2
 ; AVX512BW-ONLY-FAST-NEXT:    movb $64, %sil
-; AVX512BW-ONLY-FAST-NEXT:    kmovd %esi, %k3
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm0 {%k3}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm3 = <u,1,2,3,4,15,u,u>
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm2, %zmm3, %zmm11
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 384(%r8), %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm2, %zmm3, %zmm6
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 384(%r9), %zmm3
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm3, %zmm2, %zmm20
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm3, %zmm2, %zmm4
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm3, %zmm2, %zmm14
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm2, %zmm3, %zmm13
-; AVX512BW-ONLY-FAST-NEXT:    movb $12, %sil
-; AVX512BW-ONLY-FAST-NEXT:    kmovd %esi, %k3
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa 448(%rdx), %xmm2
-; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],mem[0]
-; AVX512BW-ONLY-FAST-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
-; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm2, %zmm0, %zmm10 {%k3}
-; AVX512BW-ONLY-FAST-NEXT:    vinserti32x4 $2, 448(%r8), %zmm10, %zmm2
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm5 = <0,1,2,3,4,8,u,7>
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm25, %zmm2, %zmm5
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm2 = <u,1,2,3,4,5,15,u>
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm3, %zmm2, %zmm6
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm7 = <0,1,2,3,9,u,6,7>
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm25, %zmm1, %zmm7
+; AVX512BW-ONLY-FAST-NEXT:    kmovd %esi, %k4
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm1 {%k4}
+; AVX512BW-ONLY-FAST-NEXT:    movb $4, %sil
+; AVX512BW-ONLY-FAST-NEXT:    kmovd %esi, %k4
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm8 {%k4}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 448(%r9), %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm31 = <0,1,11,u,4,5,6,7>
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm0, %zmm10, %zmm31
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm3 = <0,1,2,10,u,5,6,7>
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm0, %zmm8, %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm8 = <0,12,u,3,4,5,6,7>
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm0, %zmm11, %zmm8
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm10 = <13,u,2,3,4,5,6,14>
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm25, %zmm0, %zmm10
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm25, %zmm2, %zmm11
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm0, %zmm1, %zmm10
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm1 = <u,1,2,3,4,5,15,u>
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm15
+; AVX512BW-ONLY-FAST-NEXT:    movb $12, %sil
+; AVX512BW-ONLY-FAST-NEXT:    kmovd %esi, %k4
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa 448(%rdx), %xmm1
+; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
+; AVX512BW-ONLY-FAST-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
+; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm4 {%k4}
+; AVX512BW-ONLY-FAST-NEXT:    vinserti32x4 $2, 448(%r8), %zmm4, %zmm1
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm4 = <0,1,2,3,4,8,u,7>
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm0, %zmm1, %zmm4
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm11 = <0,1,2,3,9,u,6,7>
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm0, %zmm2, %zmm11
 ; AVX512BW-ONLY-FAST-NEXT:    vpbroadcastq 8(%rcx), %ymm0
 ; AVX512BW-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
-; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm9 {%k5}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm14 {%k5}
 ; AVX512BW-ONLY-FAST-NEXT:    vpbroadcastq 72(%rcx), %ymm0
 ; AVX512BW-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm21 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm21 {%k5}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm6 {%k5}
 ; AVX512BW-ONLY-FAST-NEXT:    vpbroadcastq 136(%rcx), %ymm0
 ; AVX512BW-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm8 {%k5}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 (%rsp), %zmm2 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm2 {%k5}
 ; AVX512BW-ONLY-FAST-NEXT:    vpbroadcastq 200(%rcx), %ymm0
 ; AVX512BW-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm18 {%k5}
+; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm22 {%k5}
 ; AVX512BW-ONLY-FAST-NEXT:    vpbroadcastq 264(%rcx), %ymm0
 ; AVX512BW-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm25 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm25 {%k5}
+; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm21 {%k5}
 ; AVX512BW-ONLY-FAST-NEXT:    vpbroadcastq 328(%rcx), %ymm0
 ; AVX512BW-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
-; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm28 {%k5}
+; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm17 {%k5}
 ; AVX512BW-ONLY-FAST-NEXT:    vpbroadcastq 392(%rcx), %ymm0
 ; AVX512BW-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
-; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm23 {%k5}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 448(%rax), %zmm0
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,1,2,3,10,5,6,7]
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm0, %zmm16, %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,1,12,3,4,5,6,7]
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm0, %zmm19, %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 384(%rax), %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm1, %zmm3, %zmm4
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm16 {%k1}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm16 {%k2} = zmm3[2,3,2,3],zmm1[2,3,2,3]
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm1, %zmm3, %zmm14
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm1, %zmm3, %zmm13
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm3 = [0,1,2,3,4,5,8,7]
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm0, %zmm5, %zmm3
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [14,1,2,3,4,5,6,15]
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm1, %zmm5, %zmm6
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,1,2,3,4,9,6,7]
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm0, %zmm7, %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,13,2,3,4,5,6,7]
-; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm0, %zmm10, %zmm1
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm5, %zmm11
+; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm25 {%k5}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 384(%rax), %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm30 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm30
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, %zmm26 {%k1}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm26 {%k2} = zmm1[2,3,2,3],zmm0[2,3,2,3]
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm26, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm26 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm26
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm9
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [14,1,2,3,4,5,6,15]
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm24
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 448(%rax), %zmm12
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [0,1,2,3,10,5,6,7]
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm12, %zmm3, %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [0,1,12,3,4,5,6,7]
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm12, %zmm8, %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [0,13,2,3,4,5,6,7]
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm12, %zmm10, %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vpermt2q %zmm12, %zmm1, %zmm15
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [0,1,2,3,4,5,8,7]
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm12, %zmm4, %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [0,1,2,3,4,9,6,7]
+; AVX512BW-ONLY-FAST-NEXT:    vpermi2q %zmm12, %zmm11, %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-ONLY-FAST-NEXT:    movb $8, %sil
 ; AVX512BW-ONLY-FAST-NEXT:    kmovd %esi, %k2
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm15 {%k2}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm12, %zmm31 {%k2}
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k4}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, %zmm0 {%k3}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm3 {%k4}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm5 {%k4}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm7 {%k4}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm10 {%k4}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm22 {%k4}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm1 {%k3}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm4 {%k3}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm8 {%k3}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm9 {%k3}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm16, %zmm13 {%k3}
 ; AVX512BW-ONLY-FAST-NEXT:    movb $-31, %sil
 ; AVX512BW-ONLY-FAST-NEXT:    kmovd %esi, %k2
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm0 {%k2}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k2}
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm3 {%k2}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm5 {%k2}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm7 {%k2}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm4 {%k2}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm10 {%k2}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm8 {%k2}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm22 {%k2}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm22, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa (%rdx), %xmm0
-; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
-; AVX512BW-ONLY-FAST-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm1 {%k3}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa 64(%rdx), %xmm0
-; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
-; AVX512BW-ONLY-FAST-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm3 {%k3}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa 128(%rdx), %xmm0
-; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
-; AVX512BW-ONLY-FAST-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm5 {%k3}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa 192(%rdx), %xmm0
-; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
-; AVX512BW-ONLY-FAST-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm7 {%k3}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa 256(%rdx), %xmm0
-; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
-; AVX512BW-ONLY-FAST-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm10 {%k3}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa 320(%rdx), %xmm0
-; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
-; AVX512BW-ONLY-FAST-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm13 {%k3}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa 384(%rdx), %xmm0
-; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
-; AVX512BW-ONLY-FAST-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm24 {%k3}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm9 {%k2}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm13 {%k2}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa (%rdx), %xmm3
+; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],mem[0]
+; AVX512BW-ONLY-FAST-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm1 {%k4}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa 64(%rdx), %xmm3
+; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],mem[0]
+; AVX512BW-ONLY-FAST-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm4 {%k4}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa 128(%rdx), %xmm3
+; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],mem[0]
+; AVX512BW-ONLY-FAST-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm8 {%k4}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa 192(%rdx), %xmm3
+; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],mem[0]
+; AVX512BW-ONLY-FAST-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm9 {%k4}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa 256(%rdx), %xmm3
+; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],mem[0]
+; AVX512BW-ONLY-FAST-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm29 {%k4}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa 320(%rdx), %xmm3
+; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],mem[0]
+; AVX512BW-ONLY-FAST-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm27 {%k4}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa 384(%rdx), %xmm3
+; AVX512BW-ONLY-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm3 = xmm3[0],mem[0]
+; AVX512BW-ONLY-FAST-NEXT:    vinserti128 $1, %xmm3, %ymm0, %ymm3
+; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm23 {%k4}
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vinserti32x4 $3, (%rax), %zmm0, %zmm0
+; AVX512BW-ONLY-FAST-NEXT:    vinserti32x4 $3, (%rax), %zmm0, %zmm3
 ; AVX512BW-ONLY-FAST-NEXT:    movb $112, %cl
 ; AVX512BW-ONLY-FAST-NEXT:    kmovd %ecx, %k2
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k2}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm1 {%k2}
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vinserti32x4 $3, 64(%rax), %zmm0, %zmm0
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm3 {%k2}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vinserti32x4 $3, 128(%rax), %zmm0, %zmm0
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm5 {%k2}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vinserti32x4 $3, 64(%rax), %zmm0, %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm4 {%k2}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vinserti32x4 $3, 192(%rax), %zmm0, %zmm0
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm7 {%k2}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vinserti32x4 $3, 256(%rax), %zmm17, %zmm0
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm10 {%k2}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-ONLY-FAST-NEXT:    vinserti32x4 $3, 320(%rax), %zmm26, %zmm0
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm13 {%k2}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm13, %zmm26
-; AVX512BW-ONLY-FAST-NEXT:    vinserti32x4 $3, 384(%rax), %zmm20, %zmm0
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm24 {%k2}
-; AVX512BW-ONLY-FAST-NEXT:    movb $56, %cl
-; AVX512BW-ONLY-FAST-NEXT:    kmovd %ecx, %k2
+; AVX512BW-ONLY-FAST-NEXT:    vinserti32x4 $3, 128(%rax), %zmm0, %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm8 {%k2}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm9 {%k2}
+; AVX512BW-ONLY-FAST-NEXT:    vinserti32x4 $3, 192(%rax), %zmm0, %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm9 {%k2}
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm21 {%k2}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vinserti32x4 $3, 256(%rax), %zmm0, %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm29 {%k2}
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm8 {%k2}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vinserti32x4 $3, 320(%rax), %zmm0, %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm27 {%k2}
+; AVX512BW-ONLY-FAST-NEXT:    vinserti32x4 $3, 384(%rax), %zmm20, %zmm3
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm23 {%k2}
+; AVX512BW-ONLY-FAST-NEXT:    movb $56, %cl
+; AVX512BW-ONLY-FAST-NEXT:    kmovd %ecx, %k2
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm14 {%k2}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, %zmm6 {%k2}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm18 {%k2}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k2}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 %zmm2, (%rsp) # 64-byte Spill
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm22 {%k2}
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm25 {%k2}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm21 {%k2}
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm28 {%k2}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm4, %zmm23 {%k2}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm17 {%k2}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm30, %zmm25 {%k2}
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],mem[6,7]
+; AVX512BW-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm3 = ymm0[0,1,2,3,4,5],mem[6,7]
 ; AVX512BW-ONLY-FAST-NEXT:    movb $14, %cl
 ; AVX512BW-ONLY-FAST-NEXT:    kmovd %ecx, %k2
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm22 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm22 {%k2}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],mem[6,7]
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm21 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm21 {%k2}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm20 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm20 {%k2}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5],mem[6,7]
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm14 {%k2}
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],mem[6,7]
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm19 {%k2}
+; AVX512BW-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm3 = ymm0[0,1,2,3,4,5],mem[6,7]
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm13 {%k2}
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],mem[6,7]
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm15 {%k2}
+; AVX512BW-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm3 = ymm0[0,1,2,3,4,5],mem[6,7]
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm6 {%k2}
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],mem[6,7]
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm8 {%k2}
+; AVX512BW-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm3 = ymm0[0,1,2,3,4,5],mem[6,7]
+; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm28 {%k2}
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],mem[6,7]
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm3 {%k2}
+; AVX512BW-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm3 = ymm0[0,1,2,3,4,5],mem[6,7]
+; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm19 {%k2}
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],mem[6,7]
-; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm27 {%k2}
+; AVX512BW-ONLY-FAST-NEXT:    vpblendd {{.*#+}} ymm3 = ymm0[0,1,2,3,4,5],mem[6,7]
+; AVX512BW-ONLY-FAST-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm7 {%k2}
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
@@ -18234,1089 +18145,1093 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k1}
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm5 {%k1}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm7 {%k1}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm4 {%k1}
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm9 {%k1}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm8 {%k1}
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm10 {%k1}
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm12 {%k1}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm11 {%k1}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, %zmm5 {%k1}
 ; AVX512BW-ONLY-FAST-NEXT:    movb $120, %al
 ; AVX512BW-ONLY-FAST-NEXT:    kmovd %eax, %k1
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, %zmm18 {%k1}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, %zmm16 {%k1}
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm16 # 64-byte Folded Reload
-; AVX512BW-ONLY-FAST-NEXT:    # zmm16 = zmm0[0,1,2,3],mem[4,5,6,7]
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm14 {%k1}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm5, %zmm4 {%k1}
+; AVX512BW-ONLY-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm9 # 64-byte Folded Reload
+; AVX512BW-ONLY-FAST-NEXT:    # zmm9 = zmm0[0,1,2,3],mem[4,5,6,7]
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm30 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, %zmm30 {%k1}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm4, %zmm3 {%k1}
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm7, %zmm2 {%k1}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm8, %zmm2 {%k1}
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm9, %zmm1 {%k1}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm10, %zmm1 {%k1}
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm10, %zmm0 {%k1}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm20 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm12, %zmm20 {%k1}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm11, %zmm0 {%k1}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm5, %zmm26 {%k1}
 ; AVX512BW-ONLY-FAST-NEXT:    movb $-61, %al
 ; AVX512BW-ONLY-FAST-NEXT:    kmovd %eax, %k1
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm4, %zmm9 {%k1}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm4 # 64-byte Folded Reload
+; AVX512BW-ONLY-FAST-NEXT:    # zmm4 = zmm4[0,1,2,3],mem[4,5,6,7]
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm5, %zmm16 {%k1}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm5, %zmm4 {%k1}
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
 ; AVX512BW-ONLY-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm5 # 64-byte Folded Reload
 ; AVX512BW-ONLY-FAST-NEXT:    # zmm5 = zmm5[0,1,2,3],mem[4,5,6,7]
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm7, %zmm5 {%k1}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm7, %zmm7 # 64-byte Folded Reload
-; AVX512BW-ONLY-FAST-NEXT:    # zmm7 = zmm7[0,1,2,3],mem[4,5,6,7]
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm9, %zmm7 {%k1}
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm9, %zmm9 # 64-byte Folded Reload
-; AVX512BW-ONLY-FAST-NEXT:    # zmm9 = zmm9[0,1,2,3],mem[4,5,6,7]
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm8, %zmm5 {%k1}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm8, %zmm8 # 64-byte Folded Reload
+; AVX512BW-ONLY-FAST-NEXT:    # zmm8 = zmm8[0,1,2,3],mem[4,5,6,7]
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm10, %zmm9 {%k1}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm10, %zmm8 {%k1}
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
 ; AVX512BW-ONLY-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm10 # 64-byte Folded Reload
 ; AVX512BW-ONLY-FAST-NEXT:    # zmm10 = zmm10[0,1,2,3],mem[4,5,6,7]
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm13, %zmm10 {%k1}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm11, %zmm10 {%k1}
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm11, %zmm11 # 64-byte Folded Reload
+; AVX512BW-ONLY-FAST-NEXT:    # zmm11 = zmm11[0,1,2,3],mem[4,5,6,7]
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm12, %zmm13 # 64-byte Folded Reload
-; AVX512BW-ONLY-FAST-NEXT:    # zmm13 = zmm12[0,1,2,3],mem[4,5,6,7]
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm31 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm31, %zmm13 {%k1}
-; AVX512BW-ONLY-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm17 = zmm29[0,1,2,3],zmm30[4,5,6,7]
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm12, %zmm11 {%k1}
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm12, %zmm17 {%k1}
+; AVX512BW-ONLY-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm12, %zmm12 # 64-byte Folded Reload
+; AVX512BW-ONLY-FAST-NEXT:    # zmm12 = zmm12[0,1,2,3],mem[4,5,6,7]
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm18, %zmm12 {%k1}
 ; AVX512BW-ONLY-FAST-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm17, 3008(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm20, 2944(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm27, 2880(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovaps %zmm12, 2816(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm23, 2752(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm24, 2688(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovaps %zmm12, 2624(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm13, 2560(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm12, 3008(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm26, 2944(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm7, 2880(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovaps %zmm7, 2816(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm25, 2752(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm23, 2688(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovaps %zmm7, 2624(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm11, 2560(%rax)
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm0, 2496(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, 2432(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovaps %zmm12, 2368(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm28, 2304(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm26, 2240(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm19, 2432(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovaps %zmm7, 2368(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm17, 2304(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm27, 2240(%rax)
 ; AVX512BW-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-ONLY-FAST-NEXT:    vmovaps %zmm0, 2176(%rax)
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm10, 2112(%rax)
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm1, 2048(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm8, 1984(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovaps %zmm10, 1920(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm25, 1856(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovaps %zmm0, 1792(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm28, 1984(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovaps %zmm7, 1920(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm21, 1856(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm29, 1792(%rax)
 ; AVX512BW-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-ONLY-FAST-NEXT:    vmovaps %zmm0, 1728(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm9, 1664(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm8, 1664(%rax)
 ; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm2, 1600(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm15, 1536(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovaps %zmm9, 1472(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovaps %zmm0, 1408(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm6, 1536(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovaps %zmm7, 1472(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm22, 1408(%rax)
 ; AVX512BW-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-ONLY-FAST-NEXT:    vmovaps %zmm0, 1344(%rax)
 ; AVX512BW-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-ONLY-FAST-NEXT:    vmovaps %zmm0, 1280(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm7, 1216(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm4, 1152(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm19, 1088(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovaps %zmm7, 1024(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm5, 1216(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm3, 1152(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm13, 1088(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovaps %zmm5, 1024(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovups (%rsp), %zmm0 # 64-byte Reload
 ; AVX512BW-ONLY-FAST-NEXT:    vmovaps %zmm0, 960(%rax)
 ; AVX512BW-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-ONLY-FAST-NEXT:    vmovaps %zmm0, 896(%rax)
 ; AVX512BW-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-ONLY-FAST-NEXT:    vmovaps %zmm0, 832(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm5, 768(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm14, 704(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm21, 640(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovaps %zmm5, 576(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm4, 768(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm30, 704(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm14, 640(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovaps %zmm4, 576(%rax)
 ; AVX512BW-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-ONLY-FAST-NEXT:    vmovaps %zmm0, 512(%rax)
 ; AVX512BW-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-ONLY-FAST-NEXT:    vmovaps %zmm0, 448(%rax)
 ; AVX512BW-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-ONLY-FAST-NEXT:    vmovaps %zmm0, 384(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm16, 320(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm18, 256(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm22, 192(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovaps %zmm0, 128(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm9, 320(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm16, 256(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm20, 192(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512BW-ONLY-FAST-NEXT:    vmovaps %zmm3, 128(%rax)
 ; AVX512BW-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-ONLY-FAST-NEXT:    vmovaps %zmm0, 64(%rax)
 ; AVX512BW-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-ONLY-FAST-NEXT:    vmovaps %zmm0, (%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm11, 3520(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm15, 3520(%rax)
 ; AVX512BW-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-ONLY-FAST-NEXT:    vmovaps %zmm0, 3456(%rax)
 ; AVX512BW-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-ONLY-FAST-NEXT:    vmovaps %zmm0, 3392(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-ONLY-FAST-NEXT:    vmovaps %zmm0, 3328(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm31, 3328(%rax)
 ; AVX512BW-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-ONLY-FAST-NEXT:    vmovaps %zmm0, 3264(%rax)
 ; AVX512BW-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-ONLY-FAST-NEXT:    vmovaps %zmm0, 3200(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm6, 3072(%rax)
+; AVX512BW-ONLY-FAST-NEXT:    vmovdqa64 %zmm24, 3072(%rax)
 ; AVX512BW-ONLY-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-ONLY-FAST-NEXT:    vmovaps %zmm0, 3136(%rax)
-; AVX512BW-ONLY-FAST-NEXT:    addq $6696, %rsp # imm = 0x1A28
+; AVX512BW-ONLY-FAST-NEXT:    addq $6120, %rsp # imm = 0x17E8
 ; AVX512BW-ONLY-FAST-NEXT:    vzeroupper
 ; AVX512BW-ONLY-FAST-NEXT:    retq
 ;
 ; AVX512DQBW-SLOW-LABEL: store_i64_stride7_vf64:
 ; AVX512DQBW-SLOW:       # %bb.0:
-; AVX512DQBW-SLOW-NEXT:    subq $6472, %rsp # imm = 0x1948
+; AVX512DQBW-SLOW-NEXT:    subq $6280, %rsp # imm = 0x1888
 ; AVX512DQBW-SLOW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 (%rdi), %zmm3
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 64(%rdi), %zmm21
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 (%rsi), %zmm4
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 64(%rsi), %zmm18
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm18, (%rsp) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 64(%rdx), %zmm13
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 (%rdi), %zmm7
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 64(%rdi), %zmm16
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 (%rsi), %zmm9
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 64(%rsi), %zmm13
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 64(%rdx), %zmm18
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm18, (%rsp) # 64-byte Spill
 ; AVX512DQBW-SLOW-NEXT:    vmovdqa64 (%rdx), %zmm5
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 (%rcx), %zmm12
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 64(%rcx), %zmm20
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm20, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm17 = [11,3,11,3,11,3,11,3]
-; AVX512DQBW-SLOW-NEXT:    # zmm17 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm27 = [2,10,0,3,2,10,0,3]
-; AVX512DQBW-SLOW-NEXT:    # zmm27 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 (%rcx), %zmm6
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 64(%rcx), %zmm15
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm10 = [11,3,11,3,11,3,11,3]
+; AVX512DQBW-SLOW-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm31 = [2,10,0,3,2,10,0,3]
+; AVX512DQBW-SLOW-NEXT:    # zmm31 = mem[0,1,2,3,0,1,2,3]
 ; AVX512DQBW-SLOW-NEXT:    movb $96, %r10b
 ; AVX512DQBW-SLOW-NEXT:    kmovd %r10d, %k1
 ; AVX512DQBW-SLOW-NEXT:    vmovdqa64 (%r8), %zmm1
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 (%r9), %zmm9
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 (%rax), %zmm6
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 64(%rax), %zmm7
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 (%r9), %zmm14
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 (%rax), %zmm3
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 64(%rax), %zmm4
+; AVX512DQBW-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm25 = [15,7,15,7,15,7,15,7]
+; AVX512DQBW-SLOW-NEXT:    # zmm25 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQBW-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [9,1,9,1,9,1,9,1]
 ; AVX512DQBW-SLOW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm9, %zmm0, %zmm2
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm11
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm14, %zmm0, %zmm2
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm8
 ; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [4,9,0,3,4,9,0,3]
 ; AVX512DQBW-SLOW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm6, %zmm0, %zmm2
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm10
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm3, %zmm0, %zmm2
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm11
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm0
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm4, %zmm17, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm9, %zmm10, %zmm0
 ; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm2
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm12, %zmm27, %zmm2
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm6, %zmm31, %zmm2
 ; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k1}
-; AVX512DQBW-SLOW-NEXT:    vmovdqa (%r9), %ymm5
+; AVX512DQBW-SLOW-NEXT:    vmovdqa (%r9), %ymm7
+; AVX512DQBW-SLOW-NEXT:    vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa 64(%r9), %ymm5
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa 64(%r9), %ymm3
-; AVX512DQBW-SLOW-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512DQBW-SLOW-NEXT:    vmovdqa (%r8), %ymm0
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa 64(%r8), %ymm4
-; AVX512DQBW-SLOW-NEXT:    vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm5[0],ymm0[2],ymm5[2]
+; AVX512DQBW-SLOW-NEXT:    vmovdqa 64(%r8), %ymm6
+; AVX512DQBW-SLOW-NEXT:    vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm7[0],ymm0[2],ymm7[2]
 ; AVX512DQBW-SLOW-NEXT:    movb $28, %r10b
 ; AVX512DQBW-SLOW-NEXT:    kmovd %r10d, %k2
-; AVX512DQBW-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm2 {%k2} = zmm0[2,3,2,3],zmm6[2,3,2,3]
+; AVX512DQBW-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm2 {%k2} = zmm0[2,3,2,3],zmm3[2,3,2,3]
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [4,12,0,5,4,12,0,5]
-; AVX512DQBW-SLOW-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm28 = [4,12,0,5,4,12,0,5]
+; AVX512DQBW-SLOW-NEXT:    # zmm28 = mem[0,1,2,3,0,1,2,3]
 ; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm0
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm9, %zmm5
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm9, %zmm2, %zmm0
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm16
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm14, %zmm28, %zmm0
 ; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [0,1,12,7,0,1,12,7]
 ; AVX512DQBW-SLOW-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm6, %zmm2, %zmm0
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm9
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm3, %zmm2, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm12
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm19 = [5,0,14,6,5,0,14,6]
-; AVX512DQBW-SLOW-NEXT:    # zmm19 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm2
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm14
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm1, %zmm19, %zmm2
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm15 = [0,13,6,7,0,13,6,7]
-; AVX512DQBW-SLOW-NEXT:    # zmm15 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm6, %zmm15, %zmm2
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm5 = [15,7,15,7,15,7,15,7]
-; AVX512DQBW-SLOW-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm14, %zmm5, %zmm1
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm8 = [6,13,14,7,6,13,14,7]
-; AVX512DQBW-SLOW-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm1, %zmm8, %zmm6
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm21, %zmm1
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm18, %zmm17, %zmm1
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm13, %zmm2
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm20, %zmm27, %zmm2
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}
-; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm1 = ymm4[0],ymm3[0],ymm4[2],ymm3[2]
-; AVX512DQBW-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm2 {%k2} = zmm1[2,3,2,3],zmm7[2,3,2,3]
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 64(%r8), %zmm1
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 64(%r9), %zmm3
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm3, %zmm11, %zmm2
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm7, %zmm10, %zmm2
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm3, %zmm16, %zmm2
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm7, %zmm9, %zmm2
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm2
+; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm17 = [5,0,14,6,5,0,14,6]
+; AVX512DQBW-SLOW-NEXT:    # zmm17 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm1, %zmm17, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm24 = [0,13,6,7,0,13,6,7]
+; AVX512DQBW-SLOW-NEXT:    # zmm24 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm3, %zmm24, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm14, %zmm25, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm27 = [6,13,14,7,6,13,14,7]
+; AVX512DQBW-SLOW-NEXT:    # zmm27 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm1, %zmm27, %zmm3
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm1, %zmm19, %zmm2
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm7, %zmm15, %zmm2
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm3, %zmm5, %zmm1
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm1, %zmm8, %zmm7
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 128(%rdi), %zmm22
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 128(%rsi), %zmm23
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm22, %zmm1
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm23, %zmm17, %zmm1
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm17, %zmm29
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 128(%rdx), %zmm18
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 128(%rcx), %zmm14
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm16, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm13, %zmm10, %zmm1
 ; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm18, %zmm2
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm14, %zmm27, %zmm2
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm15, %zmm31, %zmm2
 ; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 128(%rax), %zmm7
-; AVX512DQBW-SLOW-NEXT:    vmovdqa 128(%r9), %ymm6
-; AVX512DQBW-SLOW-NEXT:    vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa 128(%r8), %ymm1
-; AVX512DQBW-SLOW-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm1 = ymm1[0],ymm6[0],ymm1[2],ymm6[2]
-; AVX512DQBW-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm2 {%k2} = zmm1[2,3,2,3],zmm7[2,3,2,3]
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 128(%r8), %zmm1
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 128(%r9), %zmm6
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm6, %zmm11, %zmm2
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm31
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm7, %zmm10, %zmm2
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm10, %zmm30
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm6, %zmm16, %zmm2
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm7, %zmm9, %zmm2
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm2
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm1, %zmm19, %zmm2
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm7, %zmm15, %zmm2
+; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm1 = ymm6[0],ymm5[0],ymm6[2],ymm5[2]
+; AVX512DQBW-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm2 {%k2} = zmm1[2,3,2,3],zmm4[2,3,2,3]
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 64(%r8), %zmm1
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 64(%r9), %zmm20
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm6
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm20, %zmm8, %zmm3
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm4, %zmm11, %zmm3
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm20, %zmm28, %zmm3
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm4, %zmm12, %zmm3
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm20, %zmm3
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm1, %zmm17, %zmm3
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm4, %zmm24, %zmm3
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm30
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm20, %zmm25, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm1, %zmm27, %zmm4
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 128(%rdi), %zmm1
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm6, %zmm5, %zmm1
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm1, %zmm8, %zmm7
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 192(%rdi), %zmm28
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 192(%rsi), %zmm21
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm28, %zmm1
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm21, %zmm17, %zmm1
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 192(%rdx), %zmm13
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 192(%rcx), %zmm26
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm13, %zmm2
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm26, %zmm27, %zmm2
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 192(%rax), %zmm6
-; AVX512DQBW-SLOW-NEXT:    vmovdqa 192(%r9), %ymm4
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 128(%rsi), %zmm0
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm0, %zmm10, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 128(%rdx), %zmm23
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 128(%rcx), %zmm0
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm23, %zmm3
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm0, %zmm31, %zmm3
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm3 {%k1}
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 128(%rax), %zmm5
+; AVX512DQBW-SLOW-NEXT:    vmovdqa 128(%r9), %ymm4
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa 192(%r8), %ymm1
+; AVX512DQBW-SLOW-NEXT:    vmovdqa 128(%r8), %ymm1
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm1 = ymm1[0],ymm4[0],ymm1[2],ymm4[2]
-; AVX512DQBW-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm2 {%k2} = zmm1[2,3,2,3],zmm6[2,3,2,3]
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 192(%r8), %zmm1
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 192(%r9), %zmm3
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm3, %zmm11, %zmm2
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm6, %zmm10, %zmm2
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm3, %zmm16, %zmm2
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm6, %zmm9, %zmm2
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm9, %zmm0
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm2
+; AVX512DQBW-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm3 {%k2} = zmm1[2,3,2,3],zmm5[2,3,2,3]
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm1, %zmm19, %zmm2
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm6, %zmm15, %zmm2
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm3, %zmm5, %zmm1
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm1, %zmm8, %zmm6
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 256(%rdi), %zmm25
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 256(%rsi), %zmm17
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm25, %zmm2
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm29, %zmm9
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm29, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm17, %zmm29, %zmm2
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 256(%rdx), %zmm11
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 256(%rcx), %zmm29
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm4
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm29, %zmm27, %zmm4
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm4 {%k1}
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 256(%rax), %zmm1
-; AVX512DQBW-SLOW-NEXT:    vmovdqa 256(%r9), %ymm7
-; AVX512DQBW-SLOW-NEXT:    vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa 256(%r8), %ymm2
-; AVX512DQBW-SLOW-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm2 = ymm2[0],ymm7[0],ymm2[2],ymm7[2]
-; AVX512DQBW-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm4 {%k2} = zmm2[2,3,2,3],zmm1[2,3,2,3]
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 128(%r8), %zmm1
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 128(%r9), %zmm13
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm4
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm13, %zmm8, %zmm4
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm5, %zmm11, %zmm4
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 256(%r8), %zmm2
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 256(%r9), %zmm6
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm4
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm31, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm6, %zmm31, %zmm4
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm1, %zmm10, %zmm4
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm4
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm13, %zmm28, %zmm4
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm12, %zmm18
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm5, %zmm12, %zmm4
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm8
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm4
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm6, %zmm16, %zmm4
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm1, %zmm0, %zmm4
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm3
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm13, %zmm5
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm1, %zmm17, %zmm5
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm8, %zmm24, %zmm5
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm22
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm13, %zmm25, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm1, %zmm27, %zmm8
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 192(%rdi), %zmm26
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 192(%rsi), %zmm0
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm4
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm2, %zmm19, %zmm4
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm1, %zmm15, %zmm4
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm26, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm0, %zmm10, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm10, %zmm9
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 192(%rdx), %zmm21
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 192(%rcx), %zmm0
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm21, %zmm5
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm0, %zmm31, %zmm5
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm5 {%k1}
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 192(%rax), %zmm2
+; AVX512DQBW-SLOW-NEXT:    vmovdqa 192(%r9), %ymm8
+; AVX512DQBW-SLOW-NEXT:    vmovdqu %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa 192(%r8), %ymm1
+; AVX512DQBW-SLOW-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm1 = ymm1[0],ymm8[0],ymm1[2],ymm8[2]
+; AVX512DQBW-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm5 {%k2} = zmm1[2,3,2,3],zmm2[2,3,2,3]
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 192(%r8), %zmm1
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 192(%r9), %zmm8
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm16
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm8, %zmm6, %zmm16
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm2, %zmm11, %zmm16
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm15
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm8, %zmm28, %zmm15
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm2, %zmm12, %zmm15
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm6, %zmm5, %zmm2
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm2, %zmm8, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm12
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm1, %zmm17, %zmm12
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm2, %zmm24, %zmm12
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm10
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm8, %zmm25, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm1, %zmm27, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 256(%rdi), %zmm15
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 256(%rsi), %zmm19
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm15, %zmm4
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm19, %zmm9, %zmm4
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm9, %zmm29
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 256(%rdx), %zmm14
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 256(%rcx), %zmm0
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm0, %zmm31, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm1 {%k1}
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 256(%rax), %zmm0
+; AVX512DQBW-SLOW-NEXT:    vmovdqa 256(%r9), %ymm12
+; AVX512DQBW-SLOW-NEXT:    vmovdqu %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa 256(%r8), %ymm4
+; AVX512DQBW-SLOW-NEXT:    vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm4 = ymm4[0],ymm12[0],ymm4[2],ymm12[2]
+; AVX512DQBW-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm1 {%k2} = zmm4[2,3,2,3],zmm0[2,3,2,3]
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 320(%rdi), %zmm7
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 320(%rsi), %zmm10
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm4
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm10, %zmm9, %zmm4
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 320(%rdx), %zmm6
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 320(%rcx), %zmm24
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm9
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm24, %zmm27, %zmm9
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm9 {%k1}
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 320(%rax), %zmm2
-; AVX512DQBW-SLOW-NEXT:    vmovdqa 320(%r9), %ymm1
-; AVX512DQBW-SLOW-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 256(%r8), %zmm4
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 256(%r9), %zmm12
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm12, %zmm6, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm0, %zmm11, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm2
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm12, %zmm28, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm0, %zmm18, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm12, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm4, %zmm17, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm0, %zmm24, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm24, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm9
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm12, %zmm25, %zmm4
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm4, %zmm27, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 320(%rdi), %zmm16
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 320(%rsi), %zmm3
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm16, %zmm7
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm3, %zmm29, %zmm7
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 320(%rdx), %zmm4
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 320(%rcx), %zmm29
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm5
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm29, %zmm31, %zmm5
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm5 {%k1}
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 320(%rax), %zmm1
+; AVX512DQBW-SLOW-NEXT:    vmovdqa 320(%r9), %ymm7
+; AVX512DQBW-SLOW-NEXT:    vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512DQBW-SLOW-NEXT:    vmovdqa 320(%r8), %ymm0
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm4 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
-; AVX512DQBW-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm9 {%k2} = zmm4[2,3,2,3],zmm2[2,3,2,3]
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 320(%r8), %zmm4
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 320(%r9), %zmm1
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm0
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm1, %zmm31, %zmm0
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm2, %zmm30, %zmm0
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm0
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm1, %zmm16, %zmm0
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm2, %zmm3, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm7 = ymm0[0],ymm7[0],ymm0[2],ymm7[2]
+; AVX512DQBW-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm5 {%k2} = zmm7[2,3,2,3],zmm1[2,3,2,3]
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 320(%r8), %zmm7
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 320(%r9), %zmm11
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm11, %zmm6, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm1, %zmm2, %zmm0
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm0
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm4, %zmm19, %zmm0
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm2, %zmm15, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm11, %zmm28, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm1, %zmm18, %zmm0
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm1, %zmm5, %zmm4
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm4, %zmm8, %zmm2
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm9 = [13,5,13,5,13,5,13,5]
-; AVX512DQBW-SLOW-NEXT:    # zmm9 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm0
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm12, %zmm9, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm7, %zmm17, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm1, %zmm24, %zmm0
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [6,14,6,14,6,14,6,14]
-; AVX512DQBW-SLOW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm1
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm12, %zmm0, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm2
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm11, %zmm25, %zmm7
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm7, %zmm27, %zmm1
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm4
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm0
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm12, %zmm5, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 384(%rdx), %zmm5
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 384(%rcx), %zmm27
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm27, %zmm25, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 384(%rdi), %zmm7
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 384(%rsi), %zmm1
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm1, %zmm25, %zmm7
+; AVX512DQBW-SLOW-NEXT:    movb $24, %r10b
+; AVX512DQBW-SLOW-NEXT:    kmovd %r10d, %k3
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm7 {%k3}
+; AVX512DQBW-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm1 = [0,8,0,8,0,8,0,8]
+; AVX512DQBW-SLOW-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 # 64-byte Folded Reload
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [3,0,12,4,3,0,12,4]
-; AVX512DQBW-SLOW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm2, %zmm0, %zmm12
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm12, %zmm8
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm12, %zmm9, %zmm2
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm12, %zmm4, %zmm2
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm20, %zmm1, %zmm30
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm30, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm13, %zmm1, %zmm22
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm22, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm8, %zmm1, %zmm10
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm12, %zmm1, %zmm9
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm11, %zmm1, %zmm2
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm12, %zmm5, %zmm2
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 384(%r8), %zmm0
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm2 = <u,1,2,3,4,15,u,u>
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm0, %zmm2, %zmm7
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 384(%r9), %zmm2
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm8 = <u,1,2,3,4,5,15,u>
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm2, %zmm8, %zmm7
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm2, %zmm0, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm2, %zmm0, %zmm6
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm2, %zmm0, %zmm28
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm28, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm0, %zmm17, %zmm2
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm1, %zmm0, %zmm12
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm12, %zmm15
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm18, %zmm1
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm14, %zmm9, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm6 = [13,5,13,5,13,5,13,5]
+; AVX512DQBW-SLOW-NEXT:    # zmm6 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm17, %zmm6, %zmm1
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm18, %zmm1
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm14, %zmm4, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm9 = [6,14,6,14,6,14,6,14]
+; AVX512DQBW-SLOW-NEXT:    # zmm9 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm17, %zmm9, %zmm1
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm18, %zmm1
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm14, %zmm5, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm17, %zmm25, %zmm1
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm18, %zmm0, %zmm14
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm18
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm13, %zmm1
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm26, %zmm9, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [3,0,12,4,3,0,12,4]
+; AVX512DQBW-SLOW-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm0, %zmm2, %zmm17
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 (%rsp), %zmm0 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm20 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm20, %zmm6, %zmm1
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm13, %zmm1
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm26, %zmm4, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm20, %zmm9, %zmm1
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm13, %zmm1
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm26, %zmm5, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm20, %zmm25, %zmm1
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm13, %zmm0, %zmm26
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm1
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm29, %zmm9, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm0, %zmm2, %zmm20
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm23, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm30 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm30, %zmm6, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm23, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm30, %zmm9, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm23, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm30, %zmm25, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm23, %zmm2, %zmm30
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm21, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm22 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm22, %zmm6, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm21, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm22, %zmm9, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm21, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm22, %zmm25, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm21, %zmm2, %zmm22
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm23 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm23, %zmm6, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm23, %zmm9, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm23, %zmm25, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm14, %zmm2, %zmm23
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm29, %zmm6, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm29, %zmm9, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm29, %zmm25, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm4, %zmm2, %zmm29
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm29, %zmm18
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 448(%rdx), %zmm10
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 448(%rcx), %zmm0
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm10, %zmm7
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm0, %zmm31, %zmm7
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm27, %zmm5, %zmm31
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm31, (%rsp) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm27, %zmm6, %zmm1
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm1
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm29, %zmm4, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm27, %zmm9, %zmm1
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm1
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm29, %zmm5, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm5, %zmm2, %zmm27
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm27, %zmm28
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm8
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm10, %zmm2, %zmm8
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm10, %zmm12
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm0, %zmm9, %zmm12
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm10, %zmm11
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm0, %zmm6, %zmm11
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm0, %zmm25, %zmm10
+; AVX512DQBW-SLOW-NEXT:    movb $48, %r10b
+; AVX512DQBW-SLOW-NEXT:    kmovd %r10d, %k4
+; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm5 = [1,0,10,2,1,0,10,2]
+; AVX512DQBW-SLOW-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm0, %zmm5, %zmm1
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm11, %zmm0, %zmm29
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm1
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm24, %zmm9, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm17 {%k4} = zmm0[0],zmm2[0],zmm0[2],zmm2[2],zmm0[4],zmm2[4],zmm0[6],zmm2[6]
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm13
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm14
+; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [0,8,0,1,0,8,0,1]
+; AVX512DQBW-SLOW-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm2, %zmm4, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm2, %zmm6, %zmm1
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm1
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm24, %zmm4, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm2, %zmm9, %zmm13
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm2, %zmm25, %zmm14
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm0, %zmm5, %zmm1
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm1
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm24, %zmm5, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm20 {%k4} = zmm0[0],zmm2[0],zmm0[2],zmm2[2],zmm0[4],zmm2[4],zmm0[6],zmm2[6]
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm20, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm13
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm17
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm2, %zmm4, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm2, %zmm6, %zmm13
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm2, %zmm9, %zmm1
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm6, %zmm0, %zmm24
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm24, %zmm6
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 448(%rdx), %zmm14
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 448(%rcx), %zmm1
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm12
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm1, %zmm27, %zmm12
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm19
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm1, %zmm4, %zmm19
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 384(%rdx), %zmm3
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 384(%rcx), %zmm20
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm20, %zmm3, %zmm27
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm27, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm2
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm20, %zmm9, %zmm2
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm20, %zmm3, %zmm4
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm27
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm20, %zmm5, %zmm27
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm3, %zmm0, %zmm20
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm3
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm14, %zmm0, %zmm3
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm16
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm1, %zmm5, %zmm14
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm1, %zmm9, %zmm16
-; AVX512DQBW-SLOW-NEXT:    movb $48, %r10b
-; AVX512DQBW-SLOW-NEXT:    kmovd %r10d, %k3
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm1 = [1,0,10,2,1,0,10,2]
-; AVX512DQBW-SLOW-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm2
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm2, %zmm25, %zmm17
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm0, %zmm1, %zmm2
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm2
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm1, %zmm5, %zmm2
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm8 {%k3} = zmm0[0],zmm4[0],zmm0[2],zmm4[2],zmm0[4],zmm4[4],zmm0[6],zmm4[6]
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm5
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm8
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm11
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [0,8,0,1,0,8,0,1]
-; AVX512DQBW-SLOW-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm4, %zmm2, %zmm0
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm4, %zmm9, %zmm5
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti128 {{.*#+}} ymm13 = [6,14,6,14]
-; AVX512DQBW-SLOW-NEXT:    # ymm13 = mem[0,1,0,1]
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm4, %zmm13, %zmm8
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti128 {{.*#+}} ymm8 = [15,7,15,7]
-; AVX512DQBW-SLOW-NEXT:    # ymm8 = mem[0,1,0,1]
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm4, %zmm8, %zmm11
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 (%rsp), %zmm4 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm5
+; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm30 {%k4} = zmm1[0],zmm0[0],zmm1[2],zmm0[2],zmm1[4],zmm0[4],zmm1[6],zmm0[6]
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm30, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm13
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm29
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm0, %zmm4, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm0, %zmm6, %zmm13
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm0, %zmm9, %zmm2
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm0, %zmm25, %zmm29
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm0, %zmm1, %zmm5
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm15 {%k3} = zmm0[0],zmm4[0],zmm0[2],zmm4[2],zmm0[4],zmm4[4],zmm0[6],zmm4[6]
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm15
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm5
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm24
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm4, %zmm2, %zmm0
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm4, %zmm9, %zmm15
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm4, %zmm13, %zmm5
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm4, %zmm8, %zmm24
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm24, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm23, %zmm0
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm22, %zmm1, %zmm0
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm18 {%k3} = zmm22[0],zmm23[0],zmm22[2],zmm23[2],zmm22[4],zmm23[4],zmm22[6],zmm23[6]
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm22, %zmm4
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm22, %zmm0
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm22, %zmm5
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm23, %zmm2, %zmm22
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm26, %zmm5, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm22 {%k4} = zmm26[0],zmm0[0],zmm26[2],zmm0[2],zmm26[4],zmm0[4],zmm26[6],zmm0[6]
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm22, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm23, %zmm9, %zmm4
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm23, %zmm13, %zmm0
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm23, %zmm8, %zmm5
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm21, %zmm0
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm28, %zmm1, %zmm0
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm0, (%rsp) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm26 {%k3} = zmm28[0],zmm21[0],zmm28[2],zmm21[2],zmm28[4],zmm21[4],zmm28[6],zmm21[6]
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm26, %zmm2
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm26, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm26, %zmm31
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm0, %zmm4, %zmm26
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm26, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm28, %zmm4
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm28, %zmm0
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm28, %zmm23
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm21, %zmm2, %zmm28
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm28, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm21, %zmm9, %zmm4
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm21, %zmm13, %zmm0
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm21, %zmm8, %zmm23
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm17, %zmm0
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm25, %zmm1, %zmm0
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm29 {%k3} = zmm25[0],zmm17[0],zmm25[2],zmm17[2],zmm25[4],zmm17[4],zmm25[6],zmm17[6]
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm29, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm25, %zmm31
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm25, %zmm0
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm25, %zmm29
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm25, %zmm30
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm17, %zmm2, %zmm30
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm17, %zmm9, %zmm31
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm17, %zmm13, %zmm0
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm17, %zmm8, %zmm29
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm10, %zmm26
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm7, %zmm1, %zmm26
-; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm6 {%k3} = zmm7[0],zmm10[0],zmm7[2],zmm10[2],zmm7[4],zmm10[4],zmm7[6],zmm10[6]
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm28
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm0
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm24
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm25
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm10, %zmm2, %zmm25
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm10, %zmm9, %zmm28
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm10, %zmm13, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm0, %zmm6, %zmm2
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm0, %zmm9, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm0, %zmm25, %zmm31
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm19, %zmm30
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm15, %zmm5, %zmm30
+; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm23 {%k4} = zmm15[0],zmm19[0],zmm15[2],zmm19[2],zmm15[4],zmm19[4],zmm15[6],zmm19[6]
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm23, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm15, %zmm22
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm15, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm15, %zmm26
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm15, %zmm27
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm19, %zmm4, %zmm27
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm19, %zmm6, %zmm22
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm19, %zmm9, %zmm0
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm10, %zmm8, %zmm24
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 448(%rdi), %zmm6
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm19, %zmm25, %zmm26
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 448(%rdi), %zmm15
 ; AVX512DQBW-SLOW-NEXT:    vmovdqa64 448(%rsi), %zmm0
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm7
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm0, %zmm10, %zmm7
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm4
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm0, %zmm13, %zmm4
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm5
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm0, %zmm9, %zmm5
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 384(%rdi), %zmm21
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 384(%rsi), %zmm17
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm17, %zmm22
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm21, %zmm1, %zmm22
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm17, %zmm21, %zmm10
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm17, %zmm21, %zmm9
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm17, %zmm21, %zmm13
-; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm20 {%k3} = zmm21[0],zmm17[0],zmm21[2],zmm17[2],zmm21[4],zmm17[4],zmm21[6],zmm17[6]
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm21, %zmm18
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm17, %zmm2, %zmm21
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm17, %zmm8, %zmm18
-; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm3 {%k3} = zmm6[0],zmm0[0],zmm6[2],zmm0[2],zmm6[4],zmm0[4],zmm6[6],zmm0[6]
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm0, %zmm6, %zmm2
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm6, %zmm0, %zmm1
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm0, %zmm8, %zmm6
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm12 {%k1}
-; AVX512DQBW-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm0 = zmm4[0,1,2,3],zmm19[4,5,6,7]
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 448(%r8), %zmm4
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm7 = <0,11,u,u,4,5,6,7>
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm4, %zmm3, %zmm7
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 448(%r9), %zmm8
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm3 = <0,1,11,u,4,5,6,7>
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm8, %zmm7, %zmm3
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm15
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm15, %zmm13
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm0, %zmm25, %zmm13
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm16, %zmm23
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm3, %zmm25, %zmm23
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm24
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm16, %zmm5, %zmm24
+; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm18 {%k4} = zmm16[0],zmm3[0],zmm16[2],zmm3[2],zmm16[4],zmm3[4],zmm16[6],zmm3[6]
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm16, %zmm17
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm16, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm16, %zmm21
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm3, %zmm4, %zmm21
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm3, %zmm6, %zmm17
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm3, %zmm9, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm18, %zmm20
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm16, %zmm5, %zmm20
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm15, %zmm3
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm25 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm0, %zmm25, %zmm3
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm18, %zmm16, %zmm25
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm15, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm0, %zmm6, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm18, %zmm16, %zmm6
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm15, %zmm2
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm0, %zmm9, %zmm2
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm18, %zmm16, %zmm9
+; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm28 {%k4} = zmm16[0],zmm18[0],zmm16[2],zmm18[2],zmm16[4],zmm18[4],zmm16[6],zmm18[6]
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm28, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm16, %zmm28
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm18, %zmm4, %zmm28
+; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} zmm8 {%k4} = zmm15[0],zmm0[0],zmm15[2],zmm0[2],zmm15[4],zmm0[4],zmm15[6],zmm0[6]
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm0, %zmm15, %zmm4
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm15, %zmm5, %zmm0
+; AVX512DQBW-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm2 = zmm2[0,1,2,3],zmm12[4,5,6,7]
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm7 {%k1}
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 448(%r8), %zmm3
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm5 = <0,11,u,u,4,5,6,7>
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm3, %zmm8, %zmm5
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm11, %zmm1 {%k1}
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm8 = <12,u,u,3,4,5,6,13>
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm3, %zmm1, %zmm8
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm10, %zmm13 {%k3}
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm1 = <u,1,2,3,4,15,u,u>
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm3, %zmm1, %zmm13
+; AVX512DQBW-SLOW-NEXT:    movb $6, %sil
+; AVX512DQBW-SLOW-NEXT:    kmovd %esi, %k4
+; AVX512DQBW-SLOW-NEXT:    vpbroadcastq 456(%rcx), %ymm1
+; AVX512DQBW-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = mem[0,1,2,3],ymm1[4,5,6,7]
+; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm0 {%k4}
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm10 = <0,1,2,9,u,u,6,7>
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm3, %zmm0, %zmm10
+; AVX512DQBW-SLOW-NEXT:    movb $64, %sil
+; AVX512DQBW-SLOW-NEXT:    kmovd %esi, %k5
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm2 {%k5}
 ; AVX512DQBW-SLOW-NEXT:    movb $4, %sil
-; AVX512DQBW-SLOW-NEXT:    kmovd %esi, %k3
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm12 {%k3}
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm7 = <0,1,2,10,u,5,6,7>
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm8, %zmm12, %zmm7
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm16, %zmm5 {%k1}
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm3 = <12,u,u,3,4,5,6,13>
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm4, %zmm5, %zmm3
+; AVX512DQBW-SLOW-NEXT:    kmovd %esi, %k5
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm7 {%k5}
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 448(%r9), %zmm0
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm1 = <0,1,11,u,4,5,6,7>
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm0, %zmm5, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm11
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm3 = <0,1,2,10,u,5,6,7>
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm0, %zmm7, %zmm3
 ; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm5 = <0,12,u,3,4,5,6,7>
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm8, %zmm3, %zmm5
-; AVX512DQBW-SLOW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm3 = [0,8,0,8,0,8,0,8]
-; AVX512DQBW-SLOW-NEXT:    # zmm3 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm19 # 64-byte Folded Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm12 # 64-byte Folded Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm0, %zmm8, %zmm5
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm7 = <13,u,2,3,4,5,6,14>
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm0, %zmm2, %zmm7
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm1 = <u,1,2,3,4,5,15,u>
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm0, %zmm1, %zmm13
+; AVX512DQBW-SLOW-NEXT:    movb $12, %sil
+; AVX512DQBW-SLOW-NEXT:    kmovd %esi, %k5
+; AVX512DQBW-SLOW-NEXT:    vmovdqa 448(%rdx), %xmm2
+; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],mem[0]
+; AVX512DQBW-SLOW-NEXT:    vinserti128 $1, %xmm2, %ymm0, %ymm2
+; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm2, %zmm0, %zmm4 {%k5}
+; AVX512DQBW-SLOW-NEXT:    vinserti32x4 $2, 448(%r8), %zmm4, %zmm2
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm4 = <0,1,2,3,4,8,u,7>
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm0, %zmm2, %zmm4
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm2 = <0,1,2,3,9,u,6,7>
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm0, %zmm10, %zmm2
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 384(%rax), %zmm0
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm10 # 64-byte Folded Reload
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm0, %zmm8, %zmm10
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 (%rsp), %zmm1 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm25, %zmm1 {%k1}
+; AVX512DQBW-SLOW-NEXT:    vmovdqa 384(%r9), %ymm12
+; AVX512DQBW-SLOW-NEXT:    vmovdqu %ymm12, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa 384(%r8), %ymm8
+; AVX512DQBW-SLOW-NEXT:    vmovdqu %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm8 = ymm8[0],ymm12[0],ymm8[2],ymm12[2]
+; AVX512DQBW-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm1 {%k2} = zmm8[2,3,2,3],zmm0[2,3,2,3]
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm1, (%rsp) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm12 # 64-byte Folded Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm0, %zmm12, %zmm8
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm12 # 64-byte Folded Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm0, %zmm12, %zmm8
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm8 = [14,1,2,3,4,5,6,15]
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm12 # 64-byte Folded Reload
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm0, %zmm8, %zmm12
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    movb $24, %sil
-; AVX512DQBW-SLOW-NEXT:    kmovd %esi, %k5
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm27, %zmm18 {%k5}
-; AVX512DQBW-SLOW-NEXT:    movb $6, %sil
-; AVX512DQBW-SLOW-NEXT:    kmovd %esi, %k3
-; AVX512DQBW-SLOW-NEXT:    vpbroadcastq 456(%rcx), %ymm12
-; AVX512DQBW-SLOW-NEXT:    vpblendd {{.*#+}} ymm12 = mem[0,1,2,3],ymm12[4,5,6,7]
-; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm12, %zmm0, %zmm1 {%k3}
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm12 = <0,1,2,9,u,u,6,7>
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm4, %zmm1, %zmm12
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm6 {%k5}
-; AVX512DQBW-SLOW-NEXT:    movb $64, %sil
-; AVX512DQBW-SLOW-NEXT:    kmovd %esi, %k4
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm0 {%k4}
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm1 = <u,1,2,3,4,15,u,u>
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm4, %zmm1, %zmm6
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 384(%r8), %zmm4
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm4, %zmm1, %zmm18
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 384(%r9), %zmm1
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm1, %zmm4, %zmm3
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm1, %zmm4, %zmm14
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm1, %zmm4, %zmm16
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm4, %zmm1, %zmm17
-; AVX512DQBW-SLOW-NEXT:    movb $12, %sil
-; AVX512DQBW-SLOW-NEXT:    kmovd %esi, %k4
-; AVX512DQBW-SLOW-NEXT:    vmovdqa 448(%rdx), %xmm4
-; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm4 = xmm4[0],mem[0]
-; AVX512DQBW-SLOW-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
-; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm4, %zmm0, %zmm2 {%k4}
-; AVX512DQBW-SLOW-NEXT:    vinserti32x4 $2, 448(%r8), %zmm2, %zmm2
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm4 = <0,1,2,3,4,8,u,7>
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm8, %zmm2, %zmm4
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm2 = <u,1,2,3,4,5,15,u>
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm1, %zmm2, %zmm18
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm1 = <0,1,2,3,9,u,6,7>
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm8, %zmm12, %zmm1
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm12 = <13,u,2,3,4,5,6,14>
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm8, %zmm0, %zmm12
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm8, %zmm2, %zmm6
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 448(%rax), %zmm2
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [0,1,2,3,10,5,6,7]
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm2, %zmm7, %zmm0
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [0,1,12,3,4,5,6,7]
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm2, %zmm5, %zmm0
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 384(%rax), %zmm5
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm5, %zmm0, %zmm14
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm8
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm14 {%k1}
-; AVX512DQBW-SLOW-NEXT:    vmovdqa 384(%r9), %ymm7
-; AVX512DQBW-SLOW-NEXT:    vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa 384(%r8), %ymm0
-; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} ymm7 = ymm0[0],ymm7[0],ymm0[2],ymm7[2]
-; AVX512DQBW-SLOW-NEXT:    vshufi64x2 {{.*#+}} zmm14 {%k2} = zmm7[2,3,2,3],zmm5[2,3,2,3]
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm5, %zmm7, %zmm16
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm16, %zmm14
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm5, %zmm7, %zmm17
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm7 = [0,1,2,3,4,5,8,7]
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm2, %zmm4, %zmm7
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm4 = [14,1,2,3,4,5,6,15]
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm5, %zmm4, %zmm18
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm5 = [0,1,2,3,4,9,6,7]
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm2, %zmm1, %zmm5
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 448(%rax), %zmm0
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,1,2,3,10,5,6,7]
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm0, %zmm3, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,1,12,3,4,5,6,7]
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm0, %zmm5, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,13,2,3,4,5,6,7]
-; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm2, %zmm12, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm0, %zmm7, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm0, %zmm8, %zmm13
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,1,2,3,4,5,8,7]
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm0, %zmm4, %zmm1
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,1,2,3,4,9,6,7]
+; AVX512DQBW-SLOW-NEXT:    vpermi2q %zmm0, %zmm2, %zmm1
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vpermt2q %zmm2, %zmm4, %zmm6
 ; AVX512DQBW-SLOW-NEXT:    movb $8, %sil
 ; AVX512DQBW-SLOW-NEXT:    kmovd %esi, %k2
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm15 {%k2}
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm11 {%k5}
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k5}
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k5}
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm23 {%k5}
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm29 {%k5}
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm11 {%k2}
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm14 {%k3}
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm24 {%k5}
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k3}
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm29 {%k3}
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm31 {%k3}
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm26 {%k3}
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm23 {%k3}
 ; AVX512DQBW-SLOW-NEXT:    movb $-31, %sil
 ; AVX512DQBW-SLOW-NEXT:    kmovd %esi, %k2
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm14 {%k2}
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k2}
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm29 {%k2}
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm29, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm31 {%k2}
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm26 {%k2}
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm23 {%k2}
+; AVX512DQBW-SLOW-NEXT:    vmovdqa (%rdx), %xmm0
+; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
+; AVX512DQBW-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm11 {%k2}
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k2}
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k2}
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm23 {%k2}
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm23, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm29 {%k2}
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm24 {%k2}
-; AVX512DQBW-SLOW-NEXT:    vmovdqa (%rdx), %xmm1
-; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
-; AVX512DQBW-SLOW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm7 {%k4}
-; AVX512DQBW-SLOW-NEXT:    vmovdqa 64(%rdx), %xmm1
-; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
-; AVX512DQBW-SLOW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
+; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm1 {%k5}
+; AVX512DQBW-SLOW-NEXT:    vmovdqa 64(%rdx), %xmm0
+; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
+; AVX512DQBW-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm2 {%k4}
-; AVX512DQBW-SLOW-NEXT:    vmovdqa 128(%rdx), %xmm1
-; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
-; AVX512DQBW-SLOW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
+; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm2 {%k5}
+; AVX512DQBW-SLOW-NEXT:    vmovdqa 128(%rdx), %xmm0
+; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
+; AVX512DQBW-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm3 {%k5}
+; AVX512DQBW-SLOW-NEXT:    vmovdqa 192(%rdx), %xmm0
+; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
+; AVX512DQBW-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm4 {%k4}
-; AVX512DQBW-SLOW-NEXT:    vmovdqa 192(%rdx), %xmm1
-; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
-; AVX512DQBW-SLOW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm5 {%k4}
-; AVX512DQBW-SLOW-NEXT:    vmovdqa 256(%rdx), %xmm1
-; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
-; AVX512DQBW-SLOW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm30 {%k4}
-; AVX512DQBW-SLOW-NEXT:    vmovdqa 320(%rdx), %xmm1
-; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
-; AVX512DQBW-SLOW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm25 {%k4}
-; AVX512DQBW-SLOW-NEXT:    vmovdqa 384(%rdx), %xmm1
-; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
-; AVX512DQBW-SLOW-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm21 {%k4}
+; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm4 {%k5}
+; AVX512DQBW-SLOW-NEXT:    vmovdqa 256(%rdx), %xmm0
+; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
+; AVX512DQBW-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm27 {%k5}
+; AVX512DQBW-SLOW-NEXT:    vmovdqa 320(%rdx), %xmm0
+; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
+; AVX512DQBW-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm21 {%k5}
+; AVX512DQBW-SLOW-NEXT:    vmovdqa 384(%rdx), %xmm0
+; AVX512DQBW-SLOW-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
+; AVX512DQBW-SLOW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm28 {%k5}
 ; AVX512DQBW-SLOW-NEXT:    movb $112, %sil
 ; AVX512DQBW-SLOW-NEXT:    kmovd %esi, %k2
-; AVX512DQBW-SLOW-NEXT:    vinserti64x2 $3, (%rax), %zmm19, %zmm7 {%k2}
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vinserti64x2 $3, 64(%rax), %zmm1, %zmm2 {%k2}
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vinserti64x2 $3, (%rax), %zmm0, %zmm1 {%k2}
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vinserti64x2 $3, 64(%rax), %zmm0, %zmm2 {%k2}
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vinserti64x2 $3, 128(%rax), %zmm10, %zmm4 {%k2}
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vinserti64x2 $3, 128(%rax), %zmm0, %zmm3 {%k2}
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vinserti64x2 $3, 192(%rax), %zmm0, %zmm4 {%k2}
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vinserti64x2 $3, 256(%rax), %zmm0, %zmm27 {%k2}
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vinserti64x2 $3, 320(%rax), %zmm0, %zmm21 {%k2}
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vinserti64x2 $3, 384(%rax), %zmm0, %zmm28 {%k2}
+; AVX512DQBW-SLOW-NEXT:    vpbroadcastq 8(%rcx), %ymm0
+; AVX512DQBW-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vinserti64x2 $3, 192(%rax), %zmm1, %zmm5 {%k2}
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vinserti64x2 $3, 256(%rax), %zmm1, %zmm30 {%k2}
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vinserti64x2 $3, 320(%rax), %zmm1, %zmm25 {%k2}
-; AVX512DQBW-SLOW-NEXT:    vinserti64x2 $3, 384(%rax), %zmm3, %zmm21 {%k2}
-; AVX512DQBW-SLOW-NEXT:    vpbroadcastq 8(%rcx), %ymm1
-; AVX512DQBW-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = mem[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm5 {%k3}
-; AVX512DQBW-SLOW-NEXT:    vpbroadcastq 72(%rcx), %ymm1
-; AVX512DQBW-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = mem[0,1,2,3],ymm1[4,5,6,7]
+; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm1 {%k4}
+; AVX512DQBW-SLOW-NEXT:    vpbroadcastq 72(%rcx), %ymm0
+; AVX512DQBW-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm2 {%k3}
-; AVX512DQBW-SLOW-NEXT:    vpbroadcastq 136(%rcx), %ymm1
-; AVX512DQBW-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = mem[0,1,2,3],ymm1[4,5,6,7]
+; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm2 {%k4}
+; AVX512DQBW-SLOW-NEXT:    vpbroadcastq 136(%rcx), %ymm0
+; AVX512DQBW-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm3 {%k3}
-; AVX512DQBW-SLOW-NEXT:    vpbroadcastq 200(%rcx), %ymm1
-; AVX512DQBW-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = mem[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 (%rsp), %zmm4 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm4 {%k3}
-; AVX512DQBW-SLOW-NEXT:    vpbroadcastq 264(%rcx), %ymm1
-; AVX512DQBW-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = mem[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm7 {%k3}
-; AVX512DQBW-SLOW-NEXT:    vpbroadcastq 328(%rcx), %ymm1
-; AVX512DQBW-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = mem[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm26 {%k3}
-; AVX512DQBW-SLOW-NEXT:    vpbroadcastq 392(%rcx), %ymm1
-; AVX512DQBW-SLOW-NEXT:    vpblendd {{.*#+}} ymm1 = mem[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm22 {%k3}
+; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm3 {%k4}
+; AVX512DQBW-SLOW-NEXT:    vpbroadcastq 200(%rcx), %ymm0
+; AVX512DQBW-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm4 {%k4}
+; AVX512DQBW-SLOW-NEXT:    vpbroadcastq 264(%rcx), %ymm0
+; AVX512DQBW-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
+; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm30 {%k4}
+; AVX512DQBW-SLOW-NEXT:    vpbroadcastq 328(%rcx), %ymm0
+; AVX512DQBW-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
+; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm24 {%k4}
+; AVX512DQBW-SLOW-NEXT:    vpbroadcastq 392(%rcx), %ymm0
+; AVX512DQBW-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
+; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm20 {%k4}
 ; AVX512DQBW-SLOW-NEXT:    movb $56, %cl
 ; AVX512DQBW-SLOW-NEXT:    kmovd %ecx, %k2
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm5 {%k2}
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k2}
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k2}
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k2}
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm3 {%k2}
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm3 {%k2}
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm4 {%k2}
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm30 {%k2}
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm24 {%k2}
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm10, %zmm20 {%k2}
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k2}
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm4, (%rsp) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm7 {%k2}
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm26 {%k2}
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm22 {%k2}
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k1}
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm3 {%k1}
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm3 {%k1}
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm5 {%k1}
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm31 {%k1}
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm28 {%k1}
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm9 {%k1}
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm4 {%k1}
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm22 {%k1}
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm17 {%k1}
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, %zmm6 {%k1}
 ; AVX512DQBW-SLOW-NEXT:    movb $120, %cl
 ; AVX512DQBW-SLOW-NEXT:    kmovd %ecx, %k1
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm16 {%k1}
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm15 # 64-byte Folded Reload
-; AVX512DQBW-SLOW-NEXT:    # zmm15 = zmm1[0,1,2,3],mem[4,5,6,7]
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm19 {%k1}
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm27 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm27 {%k1}
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, %zmm19 {%k1}
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm16 # 64-byte Folded Reload
+; AVX512DQBW-SLOW-NEXT:    # zmm16 = zmm0[0,1,2,3],mem[4,5,6,7]
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm14 {%k1}
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm15 {%k1}
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm25 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm25 {%k1}
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm29 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm22, %zmm29 {%k1}
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm22 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm17, %zmm22 {%k1}
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm12 {%k1}
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm31, %zmm10 {%k1}
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm31 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm28, %zmm31 {%k1}
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm9, %zmm14 {%k1}
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm14, %zmm23
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm12 {%k1}
 ; AVX512DQBW-SLOW-NEXT:    movb $-61, %cl
 ; AVX512DQBW-SLOW-NEXT:    kmovd %ecx, %k1
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm15 {%k1}
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm14 # 64-byte Folded Reload
-; AVX512DQBW-SLOW-NEXT:    # zmm14 = zmm1[0,1,2,3],mem[4,5,6,7]
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm2, %zmm16 {%k1}
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm11 # 64-byte Folded Reload
+; AVX512DQBW-SLOW-NEXT:    # zmm11 = zmm0[0,1,2,3],mem[4,5,6,7]
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm14 {%k1}
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm11 # 64-byte Folded Reload
-; AVX512DQBW-SLOW-NEXT:    # zmm11 = zmm1[0,1,2,3],mem[4,5,6,7]
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm3, %zmm11 {%k1}
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm10 # 64-byte Folded Reload
+; AVX512DQBW-SLOW-NEXT:    # zmm10 = zmm0[0,1,2,3],mem[4,5,6,7]
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm11 {%k1}
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm4 # 64-byte Folded Reload
-; AVX512DQBW-SLOW-NEXT:    # zmm4 = zmm1[0,1,2,3],mem[4,5,6,7]
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm4, %zmm10 {%k1}
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm4 # 64-byte Folded Reload
+; AVX512DQBW-SLOW-NEXT:    # zmm4 = zmm0[0,1,2,3],mem[4,5,6,7]
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
 ; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm5, %zmm4 {%k1}
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm5 # 64-byte Folded Reload
-; AVX512DQBW-SLOW-NEXT:    # zmm5 = zmm1[0,1,2,3],mem[4,5,6,7]
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm5 # 64-byte Folded Reload
+; AVX512DQBW-SLOW-NEXT:    # zmm5 = zmm0[0,1,2,3],mem[4,5,6,7]
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm6, %zmm5 {%k1}
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm6 # 64-byte Folded Reload
+; AVX512DQBW-SLOW-NEXT:    # zmm6 = zmm0[0,1,2,3],mem[4,5,6,7]
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm5 {%k1}
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm7 # 64-byte Folded Reload
-; AVX512DQBW-SLOW-NEXT:    # zmm7 = zmm1[0,1,2,3],mem[4,5,6,7]
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm7, %zmm6 {%k1}
+; AVX512DQBW-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm9, %zmm7 # 64-byte Folded Reload
+; AVX512DQBW-SLOW-NEXT:    # zmm7 = zmm9[0,1,2,3],mem[4,5,6,7]
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
 ; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm8, %zmm7 {%k1}
-; AVX512DQBW-SLOW-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm13, %zmm8 # 64-byte Folded Reload
-; AVX512DQBW-SLOW-NEXT:    # zmm8 = zmm13[0,1,2,3],mem[4,5,6,7]
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm17, %zmm8 {%k1}
-; AVX512DQBW-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm9 # 32-byte Folded Reload
-; AVX512DQBW-SLOW-NEXT:    # ymm9 = ymm9[1],mem[1],ymm9[3],mem[3]
-; AVX512DQBW-SLOW-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[0,2,3,3]
-; AVX512DQBW-SLOW-NEXT:    vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3,4,5],mem[6,7]
+; AVX512DQBW-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm8 # 32-byte Folded Reload
+; AVX512DQBW-SLOW-NEXT:    # ymm8 = ymm8[1],mem[1],ymm8[3],mem[3]
+; AVX512DQBW-SLOW-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,2,3,3]
+; AVX512DQBW-SLOW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],mem[6,7]
 ; AVX512DQBW-SLOW-NEXT:    movb $14, %cl
 ; AVX512DQBW-SLOW-NEXT:    kmovd %ecx, %k1
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm9, %zmm0, %zmm13 {%k1}
-; AVX512DQBW-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm9 # 32-byte Folded Reload
-; AVX512DQBW-SLOW-NEXT:    # ymm9 = ymm9[1],mem[1],ymm9[3],mem[3]
-; AVX512DQBW-SLOW-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[0,2,3,3]
-; AVX512DQBW-SLOW-NEXT:    vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3,4,5],mem[6,7]
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm8, %zmm0, %zmm9 {%k1}
+; AVX512DQBW-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm8 # 32-byte Folded Reload
+; AVX512DQBW-SLOW-NEXT:    # ymm8 = ymm8[1],mem[1],ymm8[3],mem[3]
+; AVX512DQBW-SLOW-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,2,3,3]
+; AVX512DQBW-SLOW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],mem[6,7]
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm9, %zmm0, %zmm17 {%k1}
-; AVX512DQBW-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm9 # 32-byte Folded Reload
-; AVX512DQBW-SLOW-NEXT:    # ymm9 = ymm9[1],mem[1],ymm9[3],mem[3]
-; AVX512DQBW-SLOW-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[0,2,3,3]
-; AVX512DQBW-SLOW-NEXT:    vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3,4,5],mem[6,7]
-; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm28 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm9, %zmm0, %zmm28 {%k1}
-; AVX512DQBW-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm9 # 32-byte Folded Reload
-; AVX512DQBW-SLOW-NEXT:    # ymm9 = ymm9[1],mem[1],ymm9[3],mem[3]
-; AVX512DQBW-SLOW-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[0,2,3,3]
-; AVX512DQBW-SLOW-NEXT:    vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3,4,5],mem[6,7]
+; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm8, %zmm0, %zmm17 {%k1}
+; AVX512DQBW-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm8 # 32-byte Folded Reload
+; AVX512DQBW-SLOW-NEXT:    # ymm8 = ymm8[1],mem[1],ymm8[3],mem[3]
+; AVX512DQBW-SLOW-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,2,3,3]
+; AVX512DQBW-SLOW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],mem[6,7]
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm8, %zmm0, %zmm18 {%k1}
+; AVX512DQBW-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm8 # 32-byte Folded Reload
+; AVX512DQBW-SLOW-NEXT:    # ymm8 = ymm8[1],mem[1],ymm8[3],mem[3]
+; AVX512DQBW-SLOW-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,2,3,3]
+; AVX512DQBW-SLOW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],mem[6,7]
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm9, %zmm0, %zmm3 {%k1}
-; AVX512DQBW-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm9 # 32-byte Folded Reload
-; AVX512DQBW-SLOW-NEXT:    # ymm9 = ymm9[1],mem[1],ymm9[3],mem[3]
-; AVX512DQBW-SLOW-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[0,2,3,3]
-; AVX512DQBW-SLOW-NEXT:    vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3,4,5],mem[6,7]
+; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm8, %zmm0, %zmm3 {%k1}
+; AVX512DQBW-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm8 # 32-byte Folded Reload
+; AVX512DQBW-SLOW-NEXT:    # ymm8 = ymm8[1],mem[1],ymm8[3],mem[3]
+; AVX512DQBW-SLOW-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,2,3,3]
+; AVX512DQBW-SLOW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],mem[6,7]
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm9, %zmm0, %zmm2 {%k1}
-; AVX512DQBW-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm9 # 32-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm9, %ymm9 # 32-byte Folded Reload
-; AVX512DQBW-SLOW-NEXT:    # ymm9 = ymm9[1],mem[1],ymm9[3],mem[3]
-; AVX512DQBW-SLOW-NEXT:    vpermq {{.*#+}} ymm9 = ymm9[0,2,3,3]
-; AVX512DQBW-SLOW-NEXT:    vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3,4,5],mem[6,7]
+; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm8, %zmm0, %zmm2 {%k1}
+; AVX512DQBW-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm8, %ymm8 # 32-byte Folded Reload
+; AVX512DQBW-SLOW-NEXT:    # ymm8 = ymm8[1],mem[1],ymm8[3],mem[3]
+; AVX512DQBW-SLOW-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,2,3,3]
+; AVX512DQBW-SLOW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],mem[6,7]
 ; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm9, %zmm0, %zmm1 {%k1}
-; AVX512DQBW-SLOW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
-; AVX512DQBW-SLOW-NEXT:    # ymm0 = ymm0[1],mem[1],ymm0[3],mem[3]
-; AVX512DQBW-SLOW-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,2,3,3]
-; AVX512DQBW-SLOW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],mem[6,7]
-; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm20 {%k1}
+; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm8, %zmm0, %zmm1 {%k1}
+; AVX512DQBW-SLOW-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vpunpckhqdq {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm8 # 32-byte Folded Reload
+; AVX512DQBW-SLOW-NEXT:    # ymm8 = ymm0[1],mem[1],ymm0[3],mem[3]
+; AVX512DQBW-SLOW-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[0,2,3,3]
+; AVX512DQBW-SLOW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3,4,5],mem[6,7]
+; AVX512DQBW-SLOW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vinserti64x4 $0, %ymm8, %zmm0, %zmm0 {%k1}
 ; AVX512DQBW-SLOW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm8, 3008(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm23, 2944(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm20, 2880(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm7, 3008(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm12, 2944(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm0, 2880(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovups (%rsp), %zmm0 # 64-byte Reload
 ; AVX512DQBW-SLOW-NEXT:    vmovaps %zmm0, 2816(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm22, 2752(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm21, 2688(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm24, 2624(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm7, 2560(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm31, 2496(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm20, 2752(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm28, 2688(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm23, 2624(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm6, 2560(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm22, 2496(%rax)
 ; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm1, 2432(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovaps %zmm0, 2368(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm26, 2304(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm25, 2240(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm29, 2176(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vmovaps %zmm6, 2368(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm24, 2304(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm21, 2240(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm26, 2176(%rax)
 ; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm5, 2112(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm10, 2048(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm29, 2048(%rax)
 ; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm2, 1984(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovaps %zmm0, 1920(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovaps %zmm0, 1856(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm30, 1792(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovaps %zmm0, 1728(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vmovaps %zmm5, 1920(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm30, 1856(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm27, 1792(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm31, 1728(%rax)
 ; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm4, 1664(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm12, 1600(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm25, 1600(%rax)
 ; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm3, 1536(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vmovaps %zmm4, 1472(%rax)
 ; AVX512DQBW-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovaps %zmm0, 1472(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovups (%rsp), %zmm0 # 64-byte Reload
 ; AVX512DQBW-SLOW-NEXT:    vmovaps %zmm0, 1408(%rax)
 ; AVX512DQBW-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQBW-SLOW-NEXT:    vmovaps %zmm0, 1344(%rax)
 ; AVX512DQBW-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQBW-SLOW-NEXT:    vmovaps %zmm0, 1280(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm11, 1216(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm27, 1152(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm28, 1088(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovaps %zmm0, 1024(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm10, 1216(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm15, 1152(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm18, 1088(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vmovaps %zmm3, 1024(%rax)
 ; AVX512DQBW-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQBW-SLOW-NEXT:    vmovaps %zmm0, 960(%rax)
 ; AVX512DQBW-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQBW-SLOW-NEXT:    vmovaps %zmm0, 896(%rax)
 ; AVX512DQBW-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQBW-SLOW-NEXT:    vmovaps %zmm0, 832(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm14, 768(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm19, 704(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm11, 768(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm14, 704(%rax)
 ; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm17, 640(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-SLOW-NEXT:    vmovaps %zmm0, 576(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vmovaps %zmm2, 576(%rax)
 ; AVX512DQBW-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQBW-SLOW-NEXT:    vmovaps %zmm0, 512(%rax)
 ; AVX512DQBW-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQBW-SLOW-NEXT:    vmovaps %zmm0, 448(%rax)
 ; AVX512DQBW-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQBW-SLOW-NEXT:    vmovaps %zmm0, 384(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm15, 320(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm16, 256(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm13, 192(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm16, 320(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm19, 256(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm9, 192(%rax)
 ; AVX512DQBW-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQBW-SLOW-NEXT:    vmovaps %zmm0, 128(%rax)
 ; AVX512DQBW-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQBW-SLOW-NEXT:    vmovaps %zmm0, 64(%rax)
 ; AVX512DQBW-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQBW-SLOW-NEXT:    vmovaps %zmm0, (%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm6, 3520(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm13, 3520(%rax)
 ; AVX512DQBW-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQBW-SLOW-NEXT:    vmovaps %zmm0, 3456(%rax)
 ; AVX512DQBW-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
@@ -19327,815 +19242,809 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQBW-SLOW-NEXT:    vmovaps %zmm0, 3264(%rax)
 ; AVX512DQBW-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQBW-SLOW-NEXT:    vmovaps %zmm0, 3200(%rax)
-; AVX512DQBW-SLOW-NEXT:    vmovdqa64 %zmm18, 3072(%rax)
+; AVX512DQBW-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQBW-SLOW-NEXT:    vmovaps %zmm0, 3072(%rax)
 ; AVX512DQBW-SLOW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQBW-SLOW-NEXT:    vmovaps %zmm0, 3136(%rax)
-; AVX512DQBW-SLOW-NEXT:    addq $6472, %rsp # imm = 0x1948
+; AVX512DQBW-SLOW-NEXT:    addq $6280, %rsp # imm = 0x1888
 ; AVX512DQBW-SLOW-NEXT:    vzeroupper
 ; AVX512DQBW-SLOW-NEXT:    retq
 ;
 ; AVX512DQBW-FAST-LABEL: store_i64_stride7_vf64:
 ; AVX512DQBW-FAST:       # %bb.0:
-; AVX512DQBW-FAST-NEXT:    subq $6568, %rsp # imm = 0x19A8
+; AVX512DQBW-FAST-NEXT:    subq $6120, %rsp # imm = 0x17E8
 ; AVX512DQBW-FAST-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 (%rdi), %zmm5
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 64(%rdi), %zmm11
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 (%rdi), %zmm6
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 64(%rdi), %zmm12
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 (%rsi), %zmm14
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 64(%rsi), %zmm16
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 64(%rdx), %zmm11
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 (%rsi), %zmm26
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 64(%rsi), %zmm22
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm22, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 64(%rdx), %zmm7
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 (%rdx), %zmm7
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 (%rdx), %zmm6
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 (%rcx), %zmm9
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 64(%rcx), %zmm19
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm10 = [11,3,11,3,11,3,11,3]
-; AVX512DQBW-FAST-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm12 = [2,10,0,3,2,10,0,3]
-; AVX512DQBW-FAST-NEXT:    # zmm12 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 (%rcx), %zmm13
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 64(%rcx), %zmm21
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm8 = [11,3,11,3,11,3,11,3]
+; AVX512DQBW-FAST-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [2,10,0,3,2,10,0,3]
+; AVX512DQBW-FAST-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
 ; AVX512DQBW-FAST-NEXT:    movb $96, %r10b
 ; AVX512DQBW-FAST-NEXT:    kmovd %r10d, %k1
 ; AVX512DQBW-FAST-NEXT:    vmovdqa64 (%r8), %zmm1
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 (%r9), %zmm8
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 (%r9), %zmm22
 ; AVX512DQBW-FAST-NEXT:    vmovdqa64 (%rax), %zmm3
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 64(%rax), %zmm4
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 64(%rax), %zmm5
+; AVX512DQBW-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm23 = [15,7,15,7,15,7,15,7]
+; AVX512DQBW-FAST-NEXT:    # zmm23 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQBW-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [9,1,9,1,9,1,9,1]
 ; AVX512DQBW-FAST-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm1, %zmm2
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm8, %zmm0, %zmm2
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm16
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm22, %zmm0, %zmm2
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm9
 ; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [4,9,0,3,4,9,0,3]
 ; AVX512DQBW-FAST-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
 ; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm3, %zmm0, %zmm2
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm13
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm10
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm5, %zmm0
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm26, %zmm10, %zmm0
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm6, %zmm2
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm9, %zmm12, %zmm2
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm6, %zmm0
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm14, %zmm8, %zmm0
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm7, %zmm2
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm13, %zmm4, %zmm2
 ; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k1}
-; AVX512DQBW-FAST-NEXT:    vmovdqa (%r9), %ymm0
+; AVX512DQBW-FAST-NEXT:    vmovdqa (%r9), %ymm7
+; AVX512DQBW-FAST-NEXT:    vmovdqu %ymm7, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa 64(%r9), %ymm13
+; AVX512DQBW-FAST-NEXT:    vmovdqu %ymm13, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa (%r8), %ymm0
 ; AVX512DQBW-FAST-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa 64(%r9), %ymm9
-; AVX512DQBW-FAST-NEXT:    vmovdqu %ymm9, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa (%r8), %ymm6
+; AVX512DQBW-FAST-NEXT:    vmovdqa 64(%r8), %ymm6
 ; AVX512DQBW-FAST-NEXT:    vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 64(%r8), %ymm30
-; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm6[0],ymm0[0],ymm6[2],ymm0[2]
+; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm7[0],ymm0[2],ymm7[2]
 ; AVX512DQBW-FAST-NEXT:    movb $28, %r10b
 ; AVX512DQBW-FAST-NEXT:    kmovd %r10d, %k2
 ; AVX512DQBW-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm2 {%k2} = zmm0[2,3,2,3],zmm3[2,3,2,3]
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [4,12,0,5,4,12,0,5]
-; AVX512DQBW-FAST-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm1, %zmm0
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm8, %zmm6
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm8, %zmm2, %zmm0
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm2, %zmm20
-; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm2 = [0,1,12,7,0,1,12,7]
-; AVX512DQBW-FAST-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm3, %zmm2, %zmm0
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm2, %zmm17
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [5,0,14,6,5,0,14,6]
-; AVX512DQBW-FAST-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm8, %zmm2
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm1, %zmm0, %zmm2
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm21
-; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [0,13,6,7,0,13,6,7]
+; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm0 = [4,12,0,5,4,12,0,5]
 ; AVX512DQBW-FAST-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm3, %zmm0, %zmm2
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm18
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm1, %zmm2
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm22, %zmm0, %zmm2
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm7
+; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm15 = [0,1,12,7,0,1,12,7]
+; AVX512DQBW-FAST-NEXT:    # zmm15 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm3, %zmm15, %zmm2
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm8 = [15,7,15,7,15,7,15,7]
-; AVX512DQBW-FAST-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm6, %zmm8, %zmm1
-; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm14 = [6,13,14,7,6,13,14,7]
-; AVX512DQBW-FAST-NEXT:    # zmm14 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm1, %zmm14, %zmm3
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm11, %zmm1
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm22, %zmm10, %zmm1
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm7, %zmm2
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm12, %zmm7
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm19, %zmm12, %zmm2
+; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm20 = [5,0,14,6,5,0,14,6]
+; AVX512DQBW-FAST-NEXT:    # zmm20 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm22, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm22, %zmm2
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm1, %zmm20, %zmm2
+; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm19 = [0,13,6,7,0,13,6,7]
+; AVX512DQBW-FAST-NEXT:    # zmm19 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm3, %zmm19, %zmm2
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm3, %zmm2
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm22, %zmm23, %zmm1
+; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm18 = [6,13,14,7,6,13,14,7]
+; AVX512DQBW-FAST-NEXT:    # zmm18 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm1, %zmm18, %zmm2
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm12, %zmm1
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm16, %zmm8, %zmm1
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm11, %zmm2
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm21, %zmm4, %zmm2
 ; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}
-; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm1 = ymm30[0],ymm9[0],ymm30[2],ymm9[2]
-; AVX512DQBW-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm2 {%k2} = zmm1[2,3,2,3],zmm4[2,3,2,3]
+; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm1 = ymm6[0],ymm13[0],ymm6[2],ymm13[2]
+; AVX512DQBW-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm2 {%k2} = zmm1[2,3,2,3],zmm5[2,3,2,3]
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQBW-FAST-NEXT:    vmovdqa64 64(%r8), %zmm1
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 64(%r9), %zmm3
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 64(%r9), %zmm27
 ; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm1, %zmm2
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm3, %zmm16, %zmm2
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm4, %zmm13, %zmm2
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm27, %zmm9, %zmm2
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm5, %zmm10, %zmm2
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm1, %zmm2
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm3, %zmm20, %zmm2
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm4, %zmm17, %zmm2
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm27, %zmm0, %zmm2
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm5, %zmm15, %zmm2
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm3, %zmm2
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm1, %zmm21, %zmm2
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm4, %zmm0, %zmm2
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm27, %zmm2
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm1, %zmm20, %zmm2
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm5, %zmm19, %zmm2
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm3, %zmm8, %zmm1
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm1, %zmm14, %zmm4
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm27, %zmm23, %zmm1
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm1, %zmm18, %zmm5
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQBW-FAST-NEXT:    vmovdqa64 128(%rdi), %zmm1
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 128(%rsi), %zmm19
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm19, %zmm10, %zmm1
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 128(%rdx), %zmm3
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 128(%rcx), %zmm27
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm27, %zmm12, %zmm3
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm1, %zmm3 {%k1}
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 128(%rax), %zmm5
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 128(%rsi), %zmm26
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm26, %zmm8, %zmm1
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 128(%rdx), %zmm2
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 128(%rcx), %zmm0
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm0, %zmm4, %zmm2
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm1, %zmm2 {%k1}
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 128(%rax), %zmm6
 ; AVX512DQBW-FAST-NEXT:    vmovdqa 128(%r9), %ymm0
 ; AVX512DQBW-FAST-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 128(%r8), %ymm24
-; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm2 = ymm24[0],ymm0[0],ymm24[2],ymm0[2]
-; AVX512DQBW-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm3 {%k2} = zmm2[2,3,2,3],zmm5[2,3,2,3]
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 128(%r8), %zmm2
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 128(%r9), %zmm0
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm16, %zmm4
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm0, %zmm16, %zmm3
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm5, %zmm13, %zmm3
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm0, %zmm20, %zmm3
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm5, %zmm17, %zmm3
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm3
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm2, %zmm21, %zmm3
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm5, %zmm18, %zmm3
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa 128(%r8), %ymm5
+; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm1 = ymm5[0],ymm0[0],ymm5[2],ymm0[2]
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %ymm5, %ymm30
+; AVX512DQBW-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm2 {%k2} = zmm1[2,3,2,3],zmm6[2,3,2,3]
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm0, %zmm8, %zmm2
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm2, %zmm14, %zmm5
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 128(%r8), %zmm2
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 128(%r9), %zmm16
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm2, %zmm5
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm16, %zmm9, %zmm5
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm9, %zmm1
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm6, %zmm10, %zmm5
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm2, %zmm5
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm16, %zmm7, %zmm5
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm6, %zmm15, %zmm5
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm16, %zmm5
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm2, %zmm20, %zmm5
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm6, %zmm19, %zmm5
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm16, %zmm23, %zmm2
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm2, %zmm18, %zmm6
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQBW-FAST-NEXT:    vmovdqa64 192(%rdi), %zmm2
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 192(%rsi), %zmm16
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm10, %zmm1
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm16, %zmm10, %zmm2
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 192(%rsi), %zmm28
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm28, %zmm8, %zmm2
 ; AVX512DQBW-FAST-NEXT:    vmovdqa64 192(%rdx), %zmm5
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 192(%rcx), %zmm15
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm15, %zmm12, %zmm5
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 192(%rcx), %zmm0
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm0, %zmm4, %zmm5
 ; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm2, %zmm5 {%k1}
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 192(%rax), %zmm23
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 192(%r9), %ymm25
-; AVX512DQBW-FAST-NEXT:    vmovdqa 192(%r8), %ymm11
-; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm3 = ymm11[0],ymm25[0],ymm11[2],ymm25[2]
-; AVX512DQBW-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm5 {%k2} = zmm3[2,3,2,3],zmm23[2,3,2,3]
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 192(%r8), %zmm3
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 192(%r9), %zmm6
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm3, %zmm5
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm6, %zmm4, %zmm5
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm4, %zmm2
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm23, %zmm13, %zmm5
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 192(%rax), %zmm22
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 192(%r9), %ymm24
+; AVX512DQBW-FAST-NEXT:    vmovdqa 192(%r8), %ymm9
+; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm2 = ymm9[0],ymm24[0],ymm9[2],ymm24[2]
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %ymm9, %ymm25
+; AVX512DQBW-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm5 {%k2} = zmm2[2,3,2,3],zmm22[2,3,2,3]
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm13, %zmm0
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm3, %zmm4
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm6, %zmm5
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm6, %zmm20, %zmm4
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm20, %zmm6
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm17, %zmm10
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm23, %zmm17, %zmm4
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm5, %zmm4
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 192(%r8), %zmm5
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 192(%r9), %zmm13
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm5, %zmm9
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm13, %zmm1, %zmm9
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm22, %zmm10, %zmm9
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm10, %zmm0
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm5, %zmm12
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm7, %zmm6
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm13, %zmm7, %zmm12
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm15, %zmm9
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm22, %zmm15, %zmm12
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm13, %zmm7
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm5, %zmm20, %zmm7
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm22, %zmm19, %zmm7
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm5, %zmm17
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm13, %zmm23, %zmm5
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm5, %zmm18, %zmm22
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm22, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 256(%rdi), %zmm5
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm3, %zmm21, %zmm4
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm23, %zmm18, %zmm4
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm5, %zmm8, %zmm3
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm3, %zmm14, %zmm23
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm23, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 256(%rdi), %zmm3
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 256(%rsi), %zmm3
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 256(%rsi), %zmm31
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm31, %zmm1, %zmm3
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm1, %zmm22
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 256(%rdx), %zmm28
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 256(%rcx), %zmm12
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm28, %zmm13
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm12, %zmm7, %zmm13
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm3, %zmm8, %zmm5
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm8, %zmm2
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 256(%rdx), %zmm7
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm3, %zmm13 {%k1}
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 256(%rax), %zmm23
-; AVX512DQBW-FAST-NEXT:    vmovdqa 256(%r9), %ymm9
-; AVX512DQBW-FAST-NEXT:    vmovdqa 256(%r8), %ymm5
-; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm4 = ymm5[0],ymm9[0],ymm5[2],ymm9[2]
-; AVX512DQBW-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm13 {%k2} = zmm4[2,3,2,3],zmm23[2,3,2,3]
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 256(%r8), %zmm4
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 256(%r9), %zmm1
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm4, %zmm13
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm1, %zmm2, %zmm13
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm23, %zmm0, %zmm13
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm17
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm4, %zmm0
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm20, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm1, %zmm2
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm1, %zmm20, %zmm0
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm23, %zmm10, %zmm0
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm10, %zmm1
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm2, %zmm10
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm4, %zmm21, %zmm10
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm21, %zmm29
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm23, %zmm18, %zmm10
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 256(%rcx), %zmm29
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm29, %zmm4, %zmm7
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm4, %zmm3
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm2, %zmm8, %zmm4
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm4, %zmm14, %zmm23
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm23, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 320(%rdi), %zmm23
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 320(%rsi), %zmm20
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm23, %zmm4
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm20, %zmm22, %zmm4
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 320(%rdx), %zmm0
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 320(%rcx), %zmm13
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm13, %zmm7, %zmm0
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm4, %zmm0 {%k1}
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 320(%rax), %zmm21
-; AVX512DQBW-FAST-NEXT:    vmovdqa 320(%r9), %ymm4
-; AVX512DQBW-FAST-NEXT:    vmovdqa 320(%r8), %ymm2
-; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm10 = ymm2[0],ymm4[0],ymm2[2],ymm4[2]
-; AVX512DQBW-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm0 {%k2} = zmm10[2,3,2,3],zmm21[2,3,2,3]
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm5, %zmm7 {%k1}
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 256(%rax), %zmm22
+; AVX512DQBW-FAST-NEXT:    vmovdqa 256(%r9), %ymm15
+; AVX512DQBW-FAST-NEXT:    vmovdqa 256(%r8), %ymm11
+; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm5 = ymm11[0],ymm15[0],ymm11[2],ymm15[2]
+; AVX512DQBW-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm7 {%k2} = zmm5[2,3,2,3],zmm22[2,3,2,3]
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 256(%r8), %zmm7
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 256(%r9), %zmm5
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm7, %zmm12
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm5, %zmm1, %zmm12
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm1, %zmm8
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm22, %zmm10, %zmm12
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm7, %zmm12
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm5, %zmm6, %zmm12
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm22, %zmm9, %zmm12
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm9, %zmm4
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm5, %zmm9
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm7, %zmm20, %zmm9
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm22, %zmm19, %zmm9
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm7, %zmm12
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm5, %zmm23, %zmm7
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm7, %zmm18, %zmm22
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm22, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 320(%rdi), %zmm31
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 320(%rsi), %zmm21
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm31, %zmm7
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm21, %zmm2, %zmm7
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 320(%rdx), %zmm1
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 320(%rcx), %zmm14
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm14, %zmm3, %zmm1
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm7, %zmm1 {%k1}
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 320(%rax), %zmm2
+; AVX512DQBW-FAST-NEXT:    vmovdqa 320(%r9), %ymm9
+; AVX512DQBW-FAST-NEXT:    vmovdqa 320(%r8), %ymm3
+; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm7 = ymm3[0],ymm9[0],ymm3[2],ymm9[2]
+; AVX512DQBW-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm1 {%k2} = zmm7[2,3,2,3],zmm2[2,3,2,3]
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQBW-FAST-NEXT:    vmovdqa64 320(%r8), %zmm10
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 320(%r9), %zmm22
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm10, %zmm0
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm22, %zmm3, %zmm0
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm21, %zmm17, %zmm0
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm10, %zmm0
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm22, %zmm6, %zmm0
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm21, %zmm1, %zmm0
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm22, %zmm0
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm10, %zmm29, %zmm0
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm21, %zmm18, %zmm0
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 320(%r9), %zmm7
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm10, %zmm1
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm7, %zmm8, %zmm1
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm8, %zmm22
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm2, %zmm0, %zmm1
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm10, %zmm1
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm7, %zmm6, %zmm1
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm2, %zmm4, %zmm1
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm7, %zmm0
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm10, %zmm20, %zmm0
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm2, %zmm19, %zmm0
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm22, %zmm8, %zmm10
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm10, %zmm14, %zmm21
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm10, %zmm8
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm7, %zmm23, %zmm10
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm10, %zmm18, %zmm2
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQBW-FAST-NEXT:    vmovdqa {{.*#+}} ymm0 = [1,3,7,7]
+; AVX512DQBW-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm10 # 32-byte Reload
+; AVX512DQBW-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm10 # 32-byte Folded Reload
+; AVX512DQBW-FAST-NEXT:    vmovdqu %ymm10, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512DQBW-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
 ; AVX512DQBW-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 32-byte Folded Reload
 ; AVX512DQBW-FAST-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512DQBW-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm30 # 32-byte Folded Reload
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 %ymm30, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQBW-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm24 # 32-byte Folded Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %ymm24, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQBW-FAST-NEXT:    vpermt2q %ymm25, %ymm0, %ymm11
+; AVX512DQBW-FAST-NEXT:    vpermt2q %ymm24, %ymm0, %ymm25
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %ymm25, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQBW-FAST-NEXT:    vpermt2q %ymm15, %ymm0, %ymm11
 ; AVX512DQBW-FAST-NEXT:    vmovdqu %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQBW-FAST-NEXT:    vpermt2q %ymm9, %ymm0, %ymm5
-; AVX512DQBW-FAST-NEXT:    vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQBW-FAST-NEXT:    vpermt2q %ymm4, %ymm0, %ymm2
-; AVX512DQBW-FAST-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa 384(%r9), %ymm1
-; AVX512DQBW-FAST-NEXT:    vmovdqa 384(%r8), %ymm2
-; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm3 = ymm2[0],ymm1[0],ymm2[2],ymm1[2]
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vpermt2q %ymm1, %ymm0, %ymm2
-; AVX512DQBW-FAST-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm29 = [3,0,12,4,3,0,12,4]
-; AVX512DQBW-FAST-NEXT:    # zmm29 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm7, %zmm1
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm0, %zmm29, %zmm1
-; AVX512DQBW-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm21 = [13,5,13,5,13,5,13,5]
-; AVX512DQBW-FAST-NEXT:    # zmm21 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm2
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm7, %zmm21, %zmm2
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm3 = [6,14,6,14,6,14,6,14]
-; AVX512DQBW-FAST-NEXT:    # zmm3 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm2
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm7, %zmm3, %zmm2
+; AVX512DQBW-FAST-NEXT:    vpermt2q %ymm9, %ymm0, %ymm3
+; AVX512DQBW-FAST-NEXT:    vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa 384(%r9), %ymm4
+; AVX512DQBW-FAST-NEXT:    vmovdqa 384(%r8), %ymm1
+; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm4[0],ymm1[2],ymm4[2]
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm7, %zmm8, %zmm0
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    movb $48, %r10b
+; AVX512DQBW-FAST-NEXT:    vpermt2q %ymm4, %ymm0, %ymm1
+; AVX512DQBW-FAST-NEXT:    vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 384(%rdx), %zmm19
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 384(%rcx), %zmm15
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm19, %zmm0
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm15, %zmm23, %zmm0
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 384(%rdi), %zmm30
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm30, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 384(%rsi), %zmm9
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm9, %zmm23, %zmm30
+; AVX512DQBW-FAST-NEXT:    movb $24, %r10b
 ; AVX512DQBW-FAST-NEXT:    kmovd %r10d, %k3
-; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm11 = [0,8,0,1,0,8,0,1]
-; AVX512DQBW-FAST-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm30 {%k3}
+; AVX512DQBW-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm18 = [0,8,0,8,0,8,0,8]
+; AVX512DQBW-FAST-NEXT:    # zmm18 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm2
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm26, %zmm11, %zmm2
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm7 = [1,0,10,2,1,0,10,2]
-; AVX512DQBW-FAST-NEXT:    # zmm7 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm26, %zmm2
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm0, %zmm7, %zmm2
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm1 {%k3} = zmm0[0],zmm26[0],zmm0[2],zmm26[2],zmm0[4],zmm26[4],zmm0[6],zmm26[6]
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm1
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm26, %zmm21, %zmm1
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} ymm30 = [6,14,6,14]
-; AVX512DQBW-FAST-NEXT:    # ymm30 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm1
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm26, %zmm30, %zmm1
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} ymm18 = [15,7,15,7]
-; AVX512DQBW-FAST-NEXT:    # ymm18 = mem[0,1,2,3,0,1,2,3]
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm26, %zmm18, %zmm0
+; AVX512DQBW-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm18, %zmm0 # 64-byte Folded Reload
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm1
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm4, %zmm29, %zmm1
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm4, %zmm2
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm0, %zmm21, %zmm2
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm4, %zmm2
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm0, %zmm3, %zmm2
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm0, %zmm8, %zmm4
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm4, %zmm0
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm2, %zmm11, %zmm0
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm2, %zmm0
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm4, %zmm7, %zmm0
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm1 {%k3} = zmm4[0],zmm2[0],zmm4[2],zmm2[2],zmm4[4],zmm2[4],zmm4[6],zmm2[6]
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm4, %zmm0
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm2, %zmm21, %zmm0
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm4, %zmm0
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm2, %zmm30, %zmm0
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm2, %zmm18, %zmm4
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm27, %zmm0
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm2, %zmm29, %zmm0
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm2, %zmm1
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm27, %zmm21, %zmm1
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm2, %zmm1
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm27, %zmm3, %zmm1
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm27, %zmm8, %zmm2
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm2, %zmm1
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm19, %zmm11, %zmm1
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm19, %zmm1
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm2, %zmm7, %zmm1
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm0 {%k3} = zmm2[0],zmm19[0],zmm2[2],zmm19[2],zmm2[4],zmm19[4],zmm2[6],zmm19[6]
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm2, %zmm0
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm19, %zmm21, %zmm0
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm2, %zmm0
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm19, %zmm30, %zmm0
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm19, %zmm18, %zmm2
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm15, %zmm0
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm2, %zmm29, %zmm0
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm2, %zmm1
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm15, %zmm21, %zmm1
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm2, %zmm1
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm15, %zmm3, %zmm1
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm15, %zmm8, %zmm2
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm2, %zmm1
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm16, %zmm11, %zmm1
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm16, %zmm17
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm2, %zmm7, %zmm17
-; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm0 {%k3} = zmm2[0],zmm16[0],zmm2[2],zmm16[2],zmm2[4],zmm16[4],zmm2[6],zmm16[6]
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm2, %zmm0
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm16, %zmm21, %zmm0
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm2, %zmm0
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm16, %zmm30, %zmm0
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm16, %zmm18, %zmm2
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm12, %zmm0
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm28, %zmm29, %zmm0
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm1
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm28, %zmm0
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm12, %zmm21, %zmm0
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm27, %zmm18, %zmm0
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm28, %zmm0
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm12, %zmm3, %zmm0
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm12, %zmm8, %zmm28
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm28, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm2
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm31, %zmm11, %zmm2
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm31, %zmm2
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm0, %zmm7, %zmm2
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm2, %zmm19
-; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm1 {%k3} = zmm0[0],zmm31[0],zmm0[2],zmm31[2],zmm0[4],zmm31[4],zmm0[6],zmm31[6]
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm1
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm31, %zmm21, %zmm1
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm1
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm31, %zmm30, %zmm1
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm31, %zmm18, %zmm0
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm16, %zmm18, %zmm0
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm13, %zmm0
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm15, %zmm29, %zmm0
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm15, %zmm1
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm13, %zmm21, %zmm1
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm15, %zmm1
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm13, %zmm3, %zmm1
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm13, %zmm8, %zmm15
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm23, %zmm1
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm20, %zmm11, %zmm1
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm13, %zmm18, %zmm17
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm5, %zmm18, %zmm12
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm7, %zmm18, %zmm8
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 384(%r8), %zmm0
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm1 = <u,1,2,3,4,15,u,u>
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm30
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 384(%r9), %zmm1
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm2 = <u,1,2,3,4,5,15,u>
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm1, %zmm2, %zmm30
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm1, %zmm0, %zmm18
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm1, %zmm0, %zmm22
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm22, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm1, %zmm0, %zmm6
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm0, %zmm20, %zmm1
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm20, %zmm28
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm23, %zmm7, %zmm28
-; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm0 {%k3} = zmm23[0],zmm20[0],zmm23[2],zmm20[2],zmm23[4],zmm20[4],zmm23[6],zmm20[6]
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 448(%rdx), %zmm7
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 448(%rcx), %zmm0
+; AVX512DQBW-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm1 = [6,14,6,14,6,14,6,14]
+; AVX512DQBW-FAST-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm7, %zmm2
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm2
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm7, %zmm10
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm0, %zmm2, %zmm10
+; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm8 = [3,0,12,4,3,0,12,4]
+; AVX512DQBW-FAST-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm13
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm7, %zmm8, %zmm13
+; AVX512DQBW-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm5 = [13,5,13,5,13,5,13,5]
+; AVX512DQBW-FAST-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm7, %zmm17
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm0, %zmm5, %zmm17
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm0, %zmm23, %zmm7
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm11, %zmm0
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm4, %zmm8, %zmm0
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm3
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm4, %zmm0
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm11, %zmm5, %zmm0
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm23, %zmm0
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm20, %zmm21, %zmm0
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm4, %zmm0
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm11, %zmm1, %zmm0
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm11, %zmm23, %zmm4
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    movb $48, %r10b
+; AVX512DQBW-FAST-NEXT:    kmovd %r10d, %k4
+; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm4 = [0,8,0,1,0,8,0,1]
+; AVX512DQBW-FAST-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm11, %zmm6
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm0, %zmm4, %zmm6
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vbroadcasti64x4 {{.*#+}} zmm6 = [1,0,10,2,1,0,10,2]
+; AVX512DQBW-FAST-NEXT:    # zmm6 = mem[0,1,2,3,0,1,2,3]
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm12
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm11, %zmm6, %zmm12
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm3 {%k4} = zmm11[0],zmm0[0],zmm11[2],zmm0[2],zmm11[4],zmm0[4],zmm11[6],zmm0[6]
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm11, %zmm3
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm0, %zmm5, %zmm3
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm11, %zmm3
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm3
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm0, %zmm23, %zmm11
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm16, %zmm0
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm11, %zmm8, %zmm0
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm3
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm11, %zmm0
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm16, %zmm5, %zmm0
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm23, %zmm0
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm20, %zmm30, %zmm0
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm11, %zmm0
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm16, %zmm1, %zmm0
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm20, %zmm18, %zmm23
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm23, %zmm26
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 448(%rdx), %zmm20
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 448(%rcx), %zmm5
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm20, %zmm13
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm16, %zmm23, %zmm11
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm12, %zmm11
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm5, %zmm0, %zmm13
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm20, %zmm10
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm5, %zmm3, %zmm10
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm5, %zmm9
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm20, %zmm29, %zmm9
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm20, %zmm16
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm5, %zmm21, %zmm16
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm5, %zmm8, %zmm20
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 384(%rdx), %zmm1
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 384(%rcx), %zmm4
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm4, %zmm1, %zmm0
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm0, %zmm4, %zmm11
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm11
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm12, %zmm6, %zmm11
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm3 {%k4} = zmm12[0],zmm0[0],zmm12[2],zmm0[2],zmm12[4],zmm0[4],zmm12[6],zmm0[6]
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm12, %zmm3
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm0, %zmm5, %zmm3
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm12, %zmm3
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm3
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm0, %zmm23, %zmm12
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm3
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm12, %zmm8, %zmm3
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm12, %zmm11
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm0, %zmm5, %zmm11
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm12, %zmm11
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm11
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm0, %zmm23, %zmm12
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm12, %zmm0
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm26, %zmm4, %zmm0
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm1, %zmm4, %zmm29
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm4, %zmm1, %zmm3
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm26, %zmm11
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm12, %zmm6, %zmm11
+; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm3 {%k4} = zmm12[0],zmm26[0],zmm12[2],zmm26[2],zmm12[4],zmm26[4],zmm12[6],zmm26[6]
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm1, %zmm0
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm4, %zmm8, %zmm1
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm4, %zmm21, %zmm0
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm12, %zmm0
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm26, %zmm5, %zmm0
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 448(%rdi), %zmm12
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 448(%rsi), %zmm4
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm12, %zmm8
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm12, %zmm0
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm26, %zmm1, %zmm0
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm26, %zmm23, %zmm12
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm3, %zmm12
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm4, %zmm0, %zmm8
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm12, %zmm3
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm4, %zmm30, %zmm3
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm12, %zmm2
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm4, %zmm21, %zmm2
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 384(%rdi), %zmm6
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 384(%rsi), %zmm5
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm6, %zmm25
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm5, %zmm11, %zmm25
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm5, %zmm24
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm6, %zmm7, %zmm24
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm5, %zmm6, %zmm0
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm0, %zmm8, %zmm12
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm16
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm3, %zmm5, %zmm16
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm16
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm3, %zmm1, %zmm16
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm3, %zmm23, %zmm0
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm29 {%k3} = zmm6[0],zmm5[0],zmm6[2],zmm5[2],zmm6[4],zmm5[4],zmm6[6],zmm5[6]
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm5, %zmm6, %zmm21
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm5, %zmm6, %zmm30
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm5, %zmm18, %zmm6
-; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm9 {%k3} = zmm12[0],zmm4[0],zmm12[2],zmm4[2],zmm12[4],zmm4[4],zmm12[6],zmm4[6]
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm4, %zmm12, %zmm11
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm12, %zmm4, %zmm7
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm4, %zmm18, %zmm12
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm8, %zmm13 {%k1}
-; AVX512DQBW-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm3 = zmm3[0,1,2,3],zmm10[4,5,6,7]
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 448(%r8), %zmm5
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm4 = <0,11,u,u,4,5,6,7>
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm5, %zmm9, %zmm4
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 448(%r9), %zmm8
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm14 = <0,1,11,u,4,5,6,7>
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm8, %zmm4, %zmm14
-; AVX512DQBW-FAST-NEXT:    movb $4, %sil
-; AVX512DQBW-FAST-NEXT:    kmovd %esi, %k3
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm5, %zmm13 {%k3}
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm4 = <0,1,2,10,u,5,6,7>
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm8, %zmm13, %zmm4
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm16, %zmm2 {%k1}
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm10 = <12,u,u,3,4,5,6,13>
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm5, %zmm2, %zmm10
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm18 = <0,12,u,3,4,5,6,7>
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm8, %zmm10, %zmm18
-; AVX512DQBW-FAST-NEXT:    vbroadcasti32x4 {{.*#+}} zmm10 = [0,8,0,8,0,8,0,8]
-; AVX512DQBW-FAST-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm0 # 64-byte Folded Reload
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm3
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm28, %zmm4, %zmm3
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm28, %zmm3
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm0, %zmm6, %zmm3
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm3, %zmm24
+; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm12 {%k4} = zmm0[0],zmm28[0],zmm0[2],zmm28[2],zmm0[4],zmm28[4],zmm0[6],zmm28[6]
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm3
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm28, %zmm5, %zmm3
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm3
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm28, %zmm1, %zmm3
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm28, %zmm23, %zmm0
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm29, %zmm28
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm0 # 64-byte Folded Reload
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm0, %zmm8, %zmm28
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm3
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm29, %zmm5, %zmm3
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm3
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm29, %zmm1, %zmm3
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm29, %zmm23, %zmm0
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm0 # 64-byte Folded Reload
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm29
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm3, %zmm4, %zmm29
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm3, %zmm27
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm0, %zmm6, %zmm27
+; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm28 {%k4} = zmm0[0],zmm3[0],zmm0[2],zmm3[2],zmm0[4],zmm3[4],zmm0[6],zmm3[6]
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm12
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm3, %zmm5, %zmm12
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm12
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm3, %zmm1, %zmm12
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm3, %zmm23, %zmm0
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm14, %zmm26
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm0 # 64-byte Folded Reload
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm0, %zmm8, %zmm26
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm3
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm14, %zmm5, %zmm3
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm3
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm14, %zmm1, %zmm3
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm14, %zmm23, %zmm0
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm23 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vpermt2q {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm23 # 64-byte Folded Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm27 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm22, %zmm10, %zmm27
-; AVX512DQBW-FAST-NEXT:    movb $24, %sil
-; AVX512DQBW-FAST-NEXT:    kmovd %esi, %k4
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm1, %zmm6 {%k4}
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 448(%rdi), %zmm12
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 448(%rsi), %zmm0
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm12, %zmm16
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm0, %zmm23, %zmm16
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm21, %zmm25
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm31, %zmm6, %zmm25
+; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm26 {%k4} = zmm31[0],zmm21[0],zmm31[2],zmm21[2],zmm31[4],zmm21[4],zmm31[6],zmm21[6]
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm31, %zmm22
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm31, %zmm14
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm31, %zmm3
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm21, %zmm23, %zmm31
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm21, %zmm4, %zmm22
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm21, %zmm5, %zmm14
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm21, %zmm1, %zmm3
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm15, %zmm19, %zmm2
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm19, %zmm15, %zmm8
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm19, %zmm2
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm15, %zmm5, %zmm2
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm15, %zmm1, %zmm19
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm19, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm12, %zmm19
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm19
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm12, %zmm3
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm0, %zmm15, %zmm3
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm12, %zmm2
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm0, %zmm5, %zmm2
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm9, %zmm20
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm14, %zmm6, %zmm20
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm9, %zmm14, %zmm15
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm8 {%k4} = zmm14[0],zmm9[0],zmm14[2],zmm9[2],zmm14[4],zmm9[4],zmm14[6],zmm9[6]
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm9, %zmm14, %zmm5
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm14, %zmm23
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm9, %zmm1, %zmm14
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm9, %zmm4, %zmm23
+; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} zmm13 {%k4} = zmm12[0],zmm0[0],zmm12[2],zmm0[2],zmm12[4],zmm0[4],zmm12[6],zmm0[6]
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm0, %zmm12, %zmm4
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm12, %zmm6, %zmm0
+; AVX512DQBW-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm19, %zmm1 # 64-byte Folded Reload
+; AVX512DQBW-FAST-NEXT:    # zmm1 = zmm19[0,1,2,3],mem[4,5,6,7]
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm3, %zmm10 {%k1}
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 448(%r8), %zmm3
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm9 = <0,11,u,u,4,5,6,7>
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm3, %zmm13, %zmm9
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm17, %zmm2 {%k1}
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm12 = <12,u,u,3,4,5,6,13>
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm3, %zmm2, %zmm12
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm7, %zmm16 {%k3}
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm2 = <u,1,2,3,4,15,u,u>
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm3, %zmm2, %zmm16
 ; AVX512DQBW-FAST-NEXT:    movb $6, %sil
 ; AVX512DQBW-FAST-NEXT:    kmovd %esi, %k5
-; AVX512DQBW-FAST-NEXT:    vpbroadcastq 456(%rcx), %ymm0
-; AVX512DQBW-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
-; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm7 {%k5}
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm1 = <0,1,2,9,u,u,6,7>
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm5, %zmm7, %zmm1
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm20, %zmm12 {%k4}
+; AVX512DQBW-FAST-NEXT:    vpbroadcastq 456(%rcx), %ymm2
+; AVX512DQBW-FAST-NEXT:    vpblendd {{.*#+}} ymm2 = mem[0,1,2,3],ymm2[4,5,6,7]
+; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm2, %zmm0, %zmm0 {%k5}
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm2 = <0,1,2,9,u,u,6,7>
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm3, %zmm0, %zmm2
 ; AVX512DQBW-FAST-NEXT:    movb $64, %sil
-; AVX512DQBW-FAST-NEXT:    kmovd %esi, %k3
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm5, %zmm3 {%k3}
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm2 = <u,1,2,3,4,15,u,u>
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm5, %zmm2, %zmm12
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 384(%r8), %zmm5
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm5, %zmm2, %zmm6
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 384(%r9), %zmm2
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm2, %zmm5, %zmm10
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm2, %zmm5, %zmm16
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm2, %zmm5, %zmm13
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm5, %zmm2, %zmm0
+; AVX512DQBW-FAST-NEXT:    kmovd %esi, %k4
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm3, %zmm1 {%k4}
+; AVX512DQBW-FAST-NEXT:    movb $4, %sil
+; AVX512DQBW-FAST-NEXT:    kmovd %esi, %k4
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm3, %zmm10 {%k4}
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 448(%r9), %zmm0
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm21 = <0,1,11,u,4,5,6,7>
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm0, %zmm9, %zmm21
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm3 = <0,1,2,10,u,5,6,7>
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm0, %zmm10, %zmm3
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm7 = <0,12,u,3,4,5,6,7>
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm0, %zmm12, %zmm7
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm9 = <13,u,2,3,4,5,6,14>
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm0, %zmm1, %zmm9
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm1 = <u,1,2,3,4,5,15,u>
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm16
 ; AVX512DQBW-FAST-NEXT:    movb $12, %sil
-; AVX512DQBW-FAST-NEXT:    kmovd %esi, %k3
-; AVX512DQBW-FAST-NEXT:    vmovdqa 448(%rdx), %xmm5
-; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm5 = xmm5[0],mem[0]
-; AVX512DQBW-FAST-NEXT:    vinserti128 $1, %xmm5, %ymm0, %ymm5
-; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm5, %zmm0, %zmm11 {%k3}
-; AVX512DQBW-FAST-NEXT:    vinserti32x4 $2, 448(%r8), %zmm11, %zmm5
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm7 = <0,1,2,3,4,8,u,7>
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm8, %zmm5, %zmm7
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm5 = <u,1,2,3,4,5,15,u>
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm2, %zmm5, %zmm6
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm11 = <0,1,2,3,9,u,6,7>
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm8, %zmm1, %zmm11
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm2 = <13,u,2,3,4,5,6,14>
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm8, %zmm3, %zmm2
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm8, %zmm5, %zmm12
-; AVX512DQBW-FAST-NEXT:    vpbroadcastq 8(%rcx), %ymm1
-; AVX512DQBW-FAST-NEXT:    vpblendd {{.*#+}} ymm1 = mem[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm9 {%k5}
-; AVX512DQBW-FAST-NEXT:    vpbroadcastq 72(%rcx), %ymm1
-; AVX512DQBW-FAST-NEXT:    vpblendd {{.*#+}} ymm1 = mem[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm31 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm31 {%k5}
-; AVX512DQBW-FAST-NEXT:    vpbroadcastq 136(%rcx), %ymm1
-; AVX512DQBW-FAST-NEXT:    vpblendd {{.*#+}} ymm1 = mem[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm8 {%k5}
-; AVX512DQBW-FAST-NEXT:    vpbroadcastq 200(%rcx), %ymm1
-; AVX512DQBW-FAST-NEXT:    vpblendd {{.*#+}} ymm1 = mem[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm17, %zmm3
-; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm3 {%k5}
-; AVX512DQBW-FAST-NEXT:    vpbroadcastq 264(%rcx), %ymm1
-; AVX512DQBW-FAST-NEXT:    vpblendd {{.*#+}} ymm1 = mem[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm19, %zmm20
-; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm20 {%k5}
-; AVX512DQBW-FAST-NEXT:    vpbroadcastq 328(%rcx), %ymm1
-; AVX512DQBW-FAST-NEXT:    vpblendd {{.*#+}} ymm1 = mem[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm28 {%k5}
-; AVX512DQBW-FAST-NEXT:    vpbroadcastq 392(%rcx), %ymm1
-; AVX512DQBW-FAST-NEXT:    vpblendd {{.*#+}} ymm1 = mem[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm24 {%k5}
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 448(%rax), %zmm5
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,1,2,3,10,5,6,7]
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm5, %zmm4, %zmm1
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [0,1,12,3,4,5,6,7]
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm5, %zmm18, %zmm1
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 384(%rax), %zmm1
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm1, %zmm4, %zmm16
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm16, %zmm4
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm17, %zmm16 {%k1}
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm16 {%k2} = zmm17[2,3,2,3],zmm1[2,3,2,3]
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm1, %zmm16, %zmm13
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm13, %zmm18
+; AVX512DQBW-FAST-NEXT:    kmovd %esi, %k4
+; AVX512DQBW-FAST-NEXT:    vmovdqa 448(%rdx), %xmm1
+; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],mem[0]
+; AVX512DQBW-FAST-NEXT:    vinserti128 $1, %xmm1, %ymm0, %ymm1
+; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm4 {%k4}
+; AVX512DQBW-FAST-NEXT:    vinserti32x4 $2, 448(%r8), %zmm4, %zmm1
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm4 = <0,1,2,3,4,8,u,7>
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm0, %zmm1, %zmm4
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm10 = <0,1,2,3,9,u,6,7>
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm0, %zmm2, %zmm10
+; AVX512DQBW-FAST-NEXT:    vpbroadcastq 8(%rcx), %ymm0
+; AVX512DQBW-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm2 {%k5}
+; AVX512DQBW-FAST-NEXT:    vpbroadcastq 72(%rcx), %ymm0
+; AVX512DQBW-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm6 {%k5}
+; AVX512DQBW-FAST-NEXT:    vpbroadcastq 136(%rcx), %ymm0
+; AVX512DQBW-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm11, %zmm15
+; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm15 {%k5}
+; AVX512DQBW-FAST-NEXT:    vpbroadcastq 200(%rcx), %ymm0
+; AVX512DQBW-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm24, %zmm14
+; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm14 {%k5}
+; AVX512DQBW-FAST-NEXT:    vpbroadcastq 264(%rcx), %ymm0
+; AVX512DQBW-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
+; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm27 {%k5}
+; AVX512DQBW-FAST-NEXT:    vpbroadcastq 328(%rcx), %ymm0
+; AVX512DQBW-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
+; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm25 {%k5}
+; AVX512DQBW-FAST-NEXT:    vpbroadcastq 392(%rcx), %ymm0
+; AVX512DQBW-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = mem[0,1,2,3],ymm0[4,5,6,7]
+; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm20 {%k5}
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 384(%rax), %zmm0
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm1, %zmm13, %zmm0
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm13
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm1, %zmm11 {%k1}
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vshufi64x2 {{.*#+}} zmm11 {%k2} = zmm1[2,3,2,3],zmm0[2,3,2,3]
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm24 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm24
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm11
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm1 = [14,1,2,3,4,5,6,15]
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm0, %zmm1, %zmm30
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 448(%rax), %zmm12
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [0,1,2,3,10,5,6,7]
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm12, %zmm3, %zmm0
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [0,1,12,3,4,5,6,7]
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm12, %zmm7, %zmm0
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [0,13,2,3,4,5,6,7]
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm12, %zmm9, %zmm0
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm12, %zmm1, %zmm16
 ; AVX512DQBW-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [0,1,2,3,4,5,8,7]
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm5, %zmm7, %zmm0
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm12, %zmm4, %zmm0
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm7 = [14,1,2,3,4,5,6,15]
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm1, %zmm7, %zmm6
 ; AVX512DQBW-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [0,1,2,3,4,9,6,7]
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm5, %zmm11, %zmm0
+; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm12, %zmm10, %zmm0
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm0 = [0,13,2,3,4,5,6,7]
-; AVX512DQBW-FAST-NEXT:    vpermi2q %zmm5, %zmm2, %zmm0
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vpermt2q %zmm5, %zmm7, %zmm12
 ; AVX512DQBW-FAST-NEXT:    movb $8, %sil
 ; AVX512DQBW-FAST-NEXT:    kmovd %esi, %k2
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm5, %zmm14 {%k2}
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm12, %zmm21 {%k2}
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k4}
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm5 {%k4}
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k3}
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm7 {%k4}
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm7 {%k3}
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k4}
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm3 {%k3}
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm9 {%k3}
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm4 {%k3}
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm11 {%k4}
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm15, %zmm26 {%k4}
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm31 {%k3}
 ; AVX512DQBW-FAST-NEXT:    movb $-31, %sil
 ; AVX512DQBW-FAST-NEXT:    kmovd %esi, %k2
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k2}
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm5 {%k2}
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm7 {%k2}
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k2}
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm3 {%k2}
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm11 {%k2}
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm9 {%k2}
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm26 {%k2}
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm26, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqa (%rdx), %xmm0
-; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
-; AVX512DQBW-FAST-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm13 {%k3}
-; AVX512DQBW-FAST-NEXT:    vmovdqa 64(%rdx), %xmm0
-; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
-; AVX512DQBW-FAST-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm4 {%k2}
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm31 {%k2}
+; AVX512DQBW-FAST-NEXT:    vmovdqa (%rdx), %xmm4
+; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm4 = xmm4[0],mem[0]
+; AVX512DQBW-FAST-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm1 {%k3}
-; AVX512DQBW-FAST-NEXT:    vmovdqa 128(%rdx), %xmm0
-; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
-; AVX512DQBW-FAST-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm2 {%k3}
-; AVX512DQBW-FAST-NEXT:    vmovdqa 192(%rdx), %xmm0
-; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
-; AVX512DQBW-FAST-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm5 {%k3}
-; AVX512DQBW-FAST-NEXT:    vmovdqa 256(%rdx), %xmm0
-; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
-; AVX512DQBW-FAST-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
+; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm4, %zmm0, %zmm1 {%k4}
+; AVX512DQBW-FAST-NEXT:    vmovdqa 64(%rdx), %xmm4
+; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm4 = xmm4[0],mem[0]
+; AVX512DQBW-FAST-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm4, %zmm0, %zmm3 {%k4}
+; AVX512DQBW-FAST-NEXT:    vmovdqa 128(%rdx), %xmm4
+; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm4 = xmm4[0],mem[0]
+; AVX512DQBW-FAST-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm7 {%k3}
-; AVX512DQBW-FAST-NEXT:    vmovdqa 320(%rdx), %xmm0
-; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
-; AVX512DQBW-FAST-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm11 {%k3}
-; AVX512DQBW-FAST-NEXT:    vmovdqa 384(%rdx), %xmm0
-; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],mem[0]
-; AVX512DQBW-FAST-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm25 {%k3}
+; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm4, %zmm0, %zmm7 {%k4}
+; AVX512DQBW-FAST-NEXT:    vmovdqa 192(%rdx), %xmm4
+; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm4 = xmm4[0],mem[0]
+; AVX512DQBW-FAST-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm4, %zmm0, %zmm9 {%k4}
+; AVX512DQBW-FAST-NEXT:    vmovdqa 256(%rdx), %xmm4
+; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm4 = xmm4[0],mem[0]
+; AVX512DQBW-FAST-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm4, %zmm0, %zmm29 {%k4}
+; AVX512DQBW-FAST-NEXT:    vmovdqa 320(%rdx), %xmm4
+; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm4 = xmm4[0],mem[0]
+; AVX512DQBW-FAST-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm4, %zmm0, %zmm22 {%k4}
+; AVX512DQBW-FAST-NEXT:    vmovdqa 384(%rdx), %xmm4
+; AVX512DQBW-FAST-NEXT:    vpunpcklqdq {{.*#+}} xmm4 = xmm4[0],mem[0]
+; AVX512DQBW-FAST-NEXT:    vinserti128 $1, %xmm4, %ymm0, %ymm4
+; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm4, %zmm0, %zmm23 {%k4}
 ; AVX512DQBW-FAST-NEXT:    movb $112, %cl
 ; AVX512DQBW-FAST-NEXT:    kmovd %ecx, %k2
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vinserti64x2 $3, (%rax), %zmm0, %zmm13 {%k2}
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vinserti64x2 $3, 64(%rax), %zmm0, %zmm1 {%k2}
+; AVX512DQBW-FAST-NEXT:    vinserti64x2 $3, (%rax), %zmm0, %zmm1 {%k2}
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vinserti64x2 $3, 128(%rax), %zmm0, %zmm2 {%k2}
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vinserti64x2 $3, 64(%rax), %zmm0, %zmm3 {%k2}
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vinserti64x2 $3, 192(%rax), %zmm0, %zmm5 {%k2}
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vinserti64x2 $3, 256(%rax), %zmm23, %zmm7 {%k2}
+; AVX512DQBW-FAST-NEXT:    vinserti64x2 $3, 128(%rax), %zmm0, %zmm7 {%k2}
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQBW-FAST-NEXT:    vinserti64x2 $3, 320(%rax), %zmm27, %zmm11 {%k2}
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm11, %zmm27
-; AVX512DQBW-FAST-NEXT:    vinserti64x2 $3, 384(%rax), %zmm10, %zmm25 {%k2}
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vinserti64x2 $3, 192(%rax), %zmm0, %zmm9 {%k2}
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vinserti64x2 $3, 256(%rax), %zmm0, %zmm29 {%k2}
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vinserti64x2 $3, 320(%rax), %zmm0, %zmm22 {%k2}
+; AVX512DQBW-FAST-NEXT:    vinserti64x2 $3, 384(%rax), %zmm18, %zmm23 {%k2}
 ; AVX512DQBW-FAST-NEXT:    movb $56, %cl
 ; AVX512DQBW-FAST-NEXT:    kmovd %ecx, %k2
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm9 {%k2}
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k2}
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm31 {%k2}
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm6 {%k2}
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm8 {%k2}
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm15 {%k2}
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm3 {%k2}
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm14 {%k2}
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm20 {%k2}
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm20, %zmm26
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm27 {%k2}
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm28 {%k2}
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm4, %zmm24 {%k2}
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm25 {%k2}
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm13, %zmm20 {%k2}
 ; AVX512DQBW-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX512DQBW-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],mem[6,7]
+; AVX512DQBW-FAST-NEXT:    vpblendd {{.*#+}} ymm4 = ymm0[0,1,2,3,4,5],mem[6,7]
 ; AVX512DQBW-FAST-NEXT:    movb $14, %cl
 ; AVX512DQBW-FAST-NEXT:    kmovd %ecx, %k2
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm23 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm23 {%k2}
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm4, %zmm0, %zmm17 {%k2}
 ; AVX512DQBW-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX512DQBW-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],mem[6,7]
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm22 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm22 {%k2}
+; AVX512DQBW-FAST-NEXT:    vpblendd {{.*#+}} ymm4 = ymm0[0,1,2,3,4,5],mem[6,7]
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm4, %zmm0, %zmm15 {%k2}
 ; AVX512DQBW-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX512DQBW-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],mem[6,7]
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm20 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm20 {%k2}
+; AVX512DQBW-FAST-NEXT:    vpblendd {{.*#+}} ymm4 = ymm0[0,1,2,3,4,5],mem[6,7]
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm4, %zmm0, %zmm14 {%k2}
 ; AVX512DQBW-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX512DQBW-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],mem[6,7]
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm16 {%k2}
+; AVX512DQBW-FAST-NEXT:    vpblendd {{.*#+}} ymm4 = ymm0[0,1,2,3,4,5],mem[6,7]
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm4, %zmm0, %zmm7 {%k2}
 ; AVX512DQBW-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX512DQBW-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],mem[6,7]
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm14 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm14 {%k2}
+; AVX512DQBW-FAST-NEXT:    vpblendd {{.*#+}} ymm4 = ymm0[0,1,2,3,4,5],mem[6,7]
+; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm4, %zmm0, %zmm28 {%k2}
 ; AVX512DQBW-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX512DQBW-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],mem[6,7]
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm4 {%k2}
+; AVX512DQBW-FAST-NEXT:    vpblendd {{.*#+}} ymm4 = ymm0[0,1,2,3,4,5],mem[6,7]
+; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm4, %zmm0, %zmm26 {%k2}
 ; AVX512DQBW-FAST-NEXT:    vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX512DQBW-FAST-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],mem[6,7]
-; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm0, %zmm0, %zmm29 {%k2}
+; AVX512DQBW-FAST-NEXT:    vpblendd {{.*#+}} ymm4 = ymm0[0,1,2,3,4,5],mem[6,7]
+; AVX512DQBW-FAST-NEXT:    vinserti64x4 $0, %ymm4, %zmm0, %zmm8 {%k2}
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
 ; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm1 {%k1}
@@ -20143,155 +20052,154 @@ define void @store_i64_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
 ; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k1}
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm3 {%k1}
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm4 {%k1}
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm5 {%k1}
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm10 {%k1}
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm7 {%k1}
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm12 {%k1}
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm8 {%k1}
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm13 {%k1}
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm21 {%k1}
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, %zmm5 {%k1}
 ; AVX512DQBW-FAST-NEXT:    movb $120, %al
 ; AVX512DQBW-FAST-NEXT:    kmovd %eax, %k1
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm1, %zmm19 {%k1}
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm1, %zmm11 {%k1}
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm17 # 64-byte Folded Reload
-; AVX512DQBW-FAST-NEXT:    # zmm17 = zmm0[0,1,2,3],mem[4,5,6,7]
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm15 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm2, %zmm15 {%k1}
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm3, %zmm13 {%k1}
+; AVX512DQBW-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm9 # 64-byte Folded Reload
+; AVX512DQBW-FAST-NEXT:    # zmm9 = zmm0[0,1,2,3],mem[4,5,6,7]
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm2, %zmm6 {%k1}
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm5, %zmm3 {%k1}
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm4, %zmm3 {%k1}
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm10, %zmm2 {%k1}
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm7, %zmm1 {%k1}
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm12, %zmm1 {%k1}
 ; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm8, %zmm0 {%k1}
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm21, %zmm18 {%k1}
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm13, %zmm0 {%k1}
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm5, %zmm24 {%k1}
 ; AVX512DQBW-FAST-NEXT:    movb $-61, %al
 ; AVX512DQBW-FAST-NEXT:    kmovd %eax, %k1
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm2, %zmm17 {%k1}
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm5 # 64-byte Folded Reload
-; AVX512DQBW-FAST-NEXT:    # zmm5 = zmm2[0,1,2,3],mem[4,5,6,7]
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm2, %zmm5 {%k1}
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm7 # 64-byte Folded Reload
-; AVX512DQBW-FAST-NEXT:    # zmm7 = zmm2[0,1,2,3],mem[4,5,6,7]
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm2, %zmm7 {%k1}
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm8 # 64-byte Folded Reload
-; AVX512DQBW-FAST-NEXT:    # zmm8 = zmm2[0,1,2,3],mem[4,5,6,7]
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm2, %zmm8 {%k1}
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm9 # 64-byte Folded Reload
-; AVX512DQBW-FAST-NEXT:    # zmm9 = zmm2[0,1,2,3],mem[4,5,6,7]
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm2, %zmm9 {%k1}
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm10 # 64-byte Folded Reload
-; AVX512DQBW-FAST-NEXT:    # zmm10 = zmm2[0,1,2,3],mem[4,5,6,7]
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm2, %zmm10 {%k1}
-; AVX512DQBW-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm30, %zmm11 # 64-byte Folded Reload
-; AVX512DQBW-FAST-NEXT:    # zmm11 = zmm30[0,1,2,3],mem[4,5,6,7]
-; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm2, %zmm11 {%k1}
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm4, %zmm9 {%k1}
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm4, %zmm4 # 64-byte Folded Reload
+; AVX512DQBW-FAST-NEXT:    # zmm4 = zmm4[0,1,2,3],mem[4,5,6,7]
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm5, %zmm4 {%k1}
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm5 # 64-byte Folded Reload
+; AVX512DQBW-FAST-NEXT:    # zmm5 = zmm5[0,1,2,3],mem[4,5,6,7]
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm10, %zmm5 {%k1}
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm10 # 64-byte Folded Reload
+; AVX512DQBW-FAST-NEXT:    # zmm10 = zmm10[0,1,2,3],mem[4,5,6,7]
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm12, %zmm10 {%k1}
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm12, %zmm12 # 64-byte Folded Reload
+; AVX512DQBW-FAST-NEXT:    # zmm12 = zmm12[0,1,2,3],mem[4,5,6,7]
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm13, %zmm12 {%k1}
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm13, %zmm13 # 64-byte Folded Reload
+; AVX512DQBW-FAST-NEXT:    # zmm13 = zmm13[0,1,2,3],mem[4,5,6,7]
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm18, %zmm13 {%k1}
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vshufi64x2 $228, {{[-0-9]+}}(%r{{[sb]}}p), %zmm18, %zmm18 # 64-byte Folded Reload
+; AVX512DQBW-FAST-NEXT:    # zmm18 = zmm18[0,1,2,3],mem[4,5,6,7]
+; AVX512DQBW-FAST-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm19 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm19, %zmm18 {%k1}
 ; AVX512DQBW-FAST-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm11, 3008(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm18, 2944(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm29, 2880(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovaps %zmm2, 2816(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm24, 2752(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm25, 2688(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovaps %zmm2, 2624(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm10, 2560(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm18, 3008(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm24, 2944(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm8, 2880(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vmovaps %zmm8, 2816(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm20, 2752(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm23, 2688(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm31, 2624(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm13, 2560(%rax)
 ; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm0, 2496(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm4, 2432(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovaps %zmm2, 2368(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm28, 2304(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm27, 2240(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm26, 2432(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vmovaps %zmm8, 2368(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm25, 2304(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm22, 2240(%rax)
 ; AVX512DQBW-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQBW-FAST-NEXT:    vmovaps %zmm0, 2176(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm9, 2112(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm12, 2112(%rax)
 ; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm1, 2048(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm14, 1984(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovaps %zmm2, 1920(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm26, 1856(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovaps %zmm0, 1792(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm28, 1984(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vmovaps %zmm8, 1920(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm27, 1856(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm29, 1792(%rax)
 ; AVX512DQBW-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQBW-FAST-NEXT:    vmovaps %zmm0, 1728(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm8, 1664(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm3, 1600(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm16, 1536(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovaps %zmm2, 1472(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm10, 1664(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm2, 1600(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm7, 1536(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vmovaps %zmm8, 1472(%rax)
 ; AVX512DQBW-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQBW-FAST-NEXT:    vmovaps %zmm0, 1408(%rax)
 ; AVX512DQBW-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQBW-FAST-NEXT:    vmovaps %zmm0, 1344(%rax)
 ; AVX512DQBW-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQBW-FAST-NEXT:    vmovaps %zmm0, 1280(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm7, 1216(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm13, 1152(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm20, 1088(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovaps %zmm2, 1024(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm5, 1216(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm3, 1152(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm14, 1088(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vmovaps %zmm5, 1024(%rax)
 ; AVX512DQBW-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQBW-FAST-NEXT:    vmovaps %zmm0, 960(%rax)
 ; AVX512DQBW-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQBW-FAST-NEXT:    vmovaps %zmm0, 896(%rax)
 ; AVX512DQBW-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQBW-FAST-NEXT:    vmovaps %zmm0, 832(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm5, 768(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm15, 704(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm22, 640(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovaps %zmm2, 576(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm31, 512(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm4, 768(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm6, 704(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm15, 640(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vmovaps %zmm4, 576(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vmovaps %zmm0, 512(%rax)
 ; AVX512DQBW-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQBW-FAST-NEXT:    vmovaps %zmm0, 448(%rax)
 ; AVX512DQBW-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQBW-FAST-NEXT:    vmovaps %zmm0, 384(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm17, 320(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm19, 256(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm23, 192(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovaps %zmm0, 128(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm9, 320(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm11, 256(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm17, 192(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512DQBW-FAST-NEXT:    vmovaps %zmm4, 128(%rax)
 ; AVX512DQBW-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQBW-FAST-NEXT:    vmovaps %zmm0, 64(%rax)
 ; AVX512DQBW-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQBW-FAST-NEXT:    vmovaps %zmm0, (%rax)
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm12, 3520(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm16, 3520(%rax)
 ; AVX512DQBW-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQBW-FAST-NEXT:    vmovaps %zmm0, 3456(%rax)
 ; AVX512DQBW-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQBW-FAST-NEXT:    vmovaps %zmm0, 3392(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512DQBW-FAST-NEXT:    vmovaps %zmm0, 3328(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm21, 3328(%rax)
 ; AVX512DQBW-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQBW-FAST-NEXT:    vmovaps %zmm0, 3264(%rax)
 ; AVX512DQBW-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQBW-FAST-NEXT:    vmovaps %zmm0, 3200(%rax)
-; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm6, 3072(%rax)
+; AVX512DQBW-FAST-NEXT:    vmovdqa64 %zmm30, 3072(%rax)
 ; AVX512DQBW-FAST-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512DQBW-FAST-NEXT:    vmovaps %zmm0, 3136(%rax)
-; AVX512DQBW-FAST-NEXT:    addq $6568, %rsp # imm = 0x19A8
+; AVX512DQBW-FAST-NEXT:    addq $6120, %rsp # imm = 0x17E8
 ; AVX512DQBW-FAST-NEXT:    vzeroupper
 ; AVX512DQBW-FAST-NEXT:    retq
   %in.vec0 = load <64 x i64>, ptr %in.vecptr0, align 64
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i64-stride-8.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i64-stride-8.ll
index 8bad8e79ae36168ff04774f72bd4400e246bcbc6..87b5732cc1aa32d08312d04c5e16d88cfe70a889 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i64-stride-8.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i64-stride-8.ll
@@ -784,9 +784,9 @@ define void @store_i64_stride8_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512F-NEXT:    movq {{[0-9]+}}(%rsp), %r10
 ; AVX512F-NEXT:    movq {{[0-9]+}}(%rsp), %r11
 ; AVX512F-NEXT:    vmovdqa64 (%rdi), %zmm6
-; AVX512F-NEXT:    vmovdqa64 (%rsi), %zmm9
+; AVX512F-NEXT:    vmovdqa64 (%rsi), %zmm8
 ; AVX512F-NEXT:    vmovdqa64 (%rdx), %zmm7
-; AVX512F-NEXT:    vmovdqa64 (%rcx), %zmm8
+; AVX512F-NEXT:    vmovdqa64 (%rcx), %zmm11
 ; AVX512F-NEXT:    vmovdqa64 (%r8), %zmm0
 ; AVX512F-NEXT:    vmovdqa64 (%r9), %zmm2
 ; AVX512F-NEXT:    vmovdqa64 (%r11), %zmm1
@@ -804,53 +804,47 @@ define void @store_i64_stride8_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512F-NEXT:    vmovdqa (%rdi), %xmm5
 ; AVX512F-NEXT:    vinserti128 $1, (%rdx), %ymm5, %ymm12
 ; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm5 = ymm12[1],ymm10[1],ymm12[3],ymm10[3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm5, %zmm4, %zmm17
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm5, %zmm4, %zmm16
 ; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm5 = [6,14,6,14,6,14,6,14]
 ; AVX512F-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm11
-; AVX512F-NEXT:    vpermt2q %zmm2, %zmm5, %zmm11
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm11 {%k1} = zmm1[0],zmm3[0],zmm1[2],zmm3[2],zmm1[4],zmm3[4],zmm1[6],zmm3[6]
-; AVX512F-NEXT:    vpermi2q %zmm9, %zmm6, %zmm5
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm13 = [6,14,6,14]
-; AVX512F-NEXT:    # ymm13 = mem[0,1,0,1]
-; AVX512F-NEXT:    vpermi2q %zmm8, %zmm7, %zmm13
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm13[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm5, %zmm11, %zmm5
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm11 = [7,15,7,15,7,15,7,15]
-; AVX512F-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm13
-; AVX512F-NEXT:    vpermt2q %zmm2, %zmm11, %zmm13
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm13 {%k1} = zmm1[1],zmm3[1],zmm1[3],zmm3[3],zmm1[5],zmm3[5],zmm1[7],zmm3[7]
-; AVX512F-NEXT:    vpermi2q %zmm9, %zmm6, %zmm11
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm14 = [7,15,7,15]
-; AVX512F-NEXT:    # ymm14 = mem[0,1,0,1]
-; AVX512F-NEXT:    vpermi2q %zmm8, %zmm7, %zmm14
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm11 = ymm11[0,1,2,3],ymm14[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm11, %zmm13, %zmm11
+; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm9
+; AVX512F-NEXT:    vpermt2q %zmm11, %zmm5, %zmm9
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm13
+; AVX512F-NEXT:    vpermt2q %zmm8, %zmm5, %zmm13
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm9 = ymm13[0,1,2,3],ymm9[4,5,6,7]
+; AVX512F-NEXT:    vpermi2q %zmm2, %zmm0, %zmm5
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm5 {%k1} = zmm1[0],zmm3[0],zmm1[2],zmm3[2],zmm1[4],zmm3[4],zmm1[6],zmm3[6]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm9, %zmm5, %zmm5
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm9 = [7,15,7,15,7,15,7,15]
+; AVX512F-NEXT:    # zmm9 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm13
+; AVX512F-NEXT:    vpermt2q %zmm11, %zmm9, %zmm13
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm14
+; AVX512F-NEXT:    vpermt2q %zmm8, %zmm9, %zmm14
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm13 = ymm14[0,1,2,3],ymm13[4,5,6,7]
+; AVX512F-NEXT:    vpermi2q %zmm2, %zmm0, %zmm9
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm9 {%k1} = zmm1[1],zmm3[1],zmm1[3],zmm3[3],zmm1[5],zmm3[5],zmm1[7],zmm3[7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm13, %zmm9, %zmm9
 ; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm13 = [4,12,4,12,4,12,4,12]
 ; AVX512F-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm14
-; AVX512F-NEXT:    vpermt2q %zmm3, %zmm13, %zmm14
+; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm14
+; AVX512F-NEXT:    vpermt2q %zmm11, %zmm13, %zmm14
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm15
+; AVX512F-NEXT:    vpermt2q %zmm8, %zmm13, %zmm15
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm14 = ymm15[0,1,2,3],ymm14[4,5,6,7]
+; AVX512F-NEXT:    vpermi2q %zmm3, %zmm1, %zmm13
 ; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm15 = zmm0[0],zmm2[0],zmm0[2],zmm2[2],zmm0[4],zmm2[4],zmm0[6],zmm2[6]
-; AVX512F-NEXT:    vmovdqa64 %zmm14, %zmm15 {%k1}
-; AVX512F-NEXT:    vpermi2q %zmm9, %zmm6, %zmm13
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm14 = [4,12,4,12]
-; AVX512F-NEXT:    # ymm14 = mem[0,1,0,1]
-; AVX512F-NEXT:    vpermi2q %zmm8, %zmm7, %zmm14
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm13 = ymm13[0,1,2,3],ymm14[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm13, %zmm15, %zmm13
+; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm15 {%k1}
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm14, %zmm15, %zmm13
 ; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm14 = [5,13,5,13,5,13,5,13]
 ; AVX512F-NEXT:    # zmm14 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm15
-; AVX512F-NEXT:    vpermt2q %zmm3, %zmm14, %zmm15
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm16 = zmm0[1],zmm2[1],zmm0[3],zmm2[3],zmm0[5],zmm2[5],zmm0[7],zmm2[7]
-; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm16 {%k1}
-; AVX512F-NEXT:    vpermt2q %zmm9, %zmm14, %zmm6
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm9 = [5,13,5,13]
-; AVX512F-NEXT:    # ymm9 = mem[0,1,0,1]
-; AVX512F-NEXT:    vpermi2q %zmm8, %zmm7, %zmm9
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3],ymm9[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm6, %zmm16, %zmm6
+; AVX512F-NEXT:    vpermt2q %zmm11, %zmm14, %zmm7
+; AVX512F-NEXT:    vpermt2q %zmm8, %zmm14, %zmm6
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3],ymm7[4,5,6,7]
+; AVX512F-NEXT:    vpermi2q %zmm3, %zmm1, %zmm14
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm7 = zmm0[1],zmm2[1],zmm0[3],zmm2[3],zmm0[5],zmm2[5],zmm0[7],zmm2[7]
+; AVX512F-NEXT:    vmovdqa64 %zmm14, %zmm7 {%k1}
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm6, %zmm7, %zmm6
 ; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm7 = [0,8,0,8,0,8,0,8]
 ; AVX512F-NEXT:    # zmm7 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm8
@@ -861,13 +855,13 @@ define void @store_i64_stride8_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512F-NEXT:    vinserti64x4 $0, %ymm8, %zmm7, %zmm7
 ; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm8 = [2,10,2,10,2,10,2,10]
 ; AVX512F-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm9
-; AVX512F-NEXT:    vpermt2q %zmm3, %zmm8, %zmm9
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm10
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm8, %zmm10
 ; AVX512F-NEXT:    vpermi2q %zmm2, %zmm0, %zmm8
-; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm8 {%k1}
-; AVX512F-NEXT:    vmovdqa (%rcx), %ymm9
-; AVX512F-NEXT:    vmovdqa (%rdx), %ymm10
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm12 = ymm10[0],ymm9[0],ymm10[2],ymm9[2]
+; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm8 {%k1}
+; AVX512F-NEXT:    vmovdqa (%rcx), %ymm10
+; AVX512F-NEXT:    vmovdqa (%rdx), %ymm11
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm12 = ymm11[0],ymm10[0],ymm11[2],ymm10[2]
 ; AVX512F-NEXT:    vmovdqa (%rsi), %ymm14
 ; AVX512F-NEXT:    vmovdqa (%rdi), %ymm15
 ; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm4 = ymm15[0],ymm14[0],ymm15[2],ymm14[2]
@@ -878,7 +872,7 @@ define void @store_i64_stride8_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512F-NEXT:    vpermt2q %zmm3, %zmm8, %zmm1
 ; AVX512F-NEXT:    vpermt2q %zmm2, %zmm8, %zmm0
 ; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm0 {%k1}
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm10[1],ymm9[1],ymm10[3],ymm9[3]
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm11[1],ymm10[1],ymm11[3],ymm10[3]
 ; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm2 = ymm15[1],ymm14[1],ymm15[3],ymm14[3]
 ; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm1 = ymm2[2,3],ymm1[2,3]
 ; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm0
@@ -887,9 +881,9 @@ define void @store_i64_stride8_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512F-NEXT:    vmovdqa64 %zmm7, (%rax)
 ; AVX512F-NEXT:    vmovdqa64 %zmm6, 320(%rax)
 ; AVX512F-NEXT:    vmovdqa64 %zmm13, 256(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm11, 448(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm9, 448(%rax)
 ; AVX512F-NEXT:    vmovdqa64 %zmm5, 384(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm17, 64(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm16, 64(%rax)
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
@@ -899,9 +893,9 @@ define void @store_i64_stride8_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %r10
 ; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %r11
 ; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm6
-; AVX512BW-NEXT:    vmovdqa64 (%rsi), %zmm9
+; AVX512BW-NEXT:    vmovdqa64 (%rsi), %zmm8
 ; AVX512BW-NEXT:    vmovdqa64 (%rdx), %zmm7
-; AVX512BW-NEXT:    vmovdqa64 (%rcx), %zmm8
+; AVX512BW-NEXT:    vmovdqa64 (%rcx), %zmm11
 ; AVX512BW-NEXT:    vmovdqa64 (%r8), %zmm0
 ; AVX512BW-NEXT:    vmovdqa64 (%r9), %zmm2
 ; AVX512BW-NEXT:    vmovdqa64 (%r11), %zmm1
@@ -919,53 +913,47 @@ define void @store_i64_stride8_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm5
 ; AVX512BW-NEXT:    vinserti128 $1, (%rdx), %ymm5, %ymm12
 ; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm5 = ymm12[1],ymm10[1],ymm12[3],ymm10[3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm5, %zmm4, %zmm17
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm5, %zmm4, %zmm16
 ; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm5 = [6,14,6,14,6,14,6,14]
 ; AVX512BW-NEXT:    # zmm5 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm11
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm5, %zmm11
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm11 {%k1} = zmm1[0],zmm3[0],zmm1[2],zmm3[2],zmm1[4],zmm3[4],zmm1[6],zmm3[6]
-; AVX512BW-NEXT:    vpermi2q %zmm9, %zmm6, %zmm5
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm13 = [6,14,6,14]
-; AVX512BW-NEXT:    # ymm13 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vpermi2q %zmm8, %zmm7, %zmm13
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm13[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm5, %zmm11, %zmm5
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm11 = [7,15,7,15,7,15,7,15]
-; AVX512BW-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm13
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm11, %zmm13
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm13 {%k1} = zmm1[1],zmm3[1],zmm1[3],zmm3[3],zmm1[5],zmm3[5],zmm1[7],zmm3[7]
-; AVX512BW-NEXT:    vpermi2q %zmm9, %zmm6, %zmm11
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm14 = [7,15,7,15]
-; AVX512BW-NEXT:    # ymm14 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vpermi2q %zmm8, %zmm7, %zmm14
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm11 = ymm11[0,1,2,3],ymm14[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm11, %zmm13, %zmm11
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm9
+; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm5, %zmm9
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm13
+; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm5, %zmm13
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm9 = ymm13[0,1,2,3],ymm9[4,5,6,7]
+; AVX512BW-NEXT:    vpermi2q %zmm2, %zmm0, %zmm5
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm5 {%k1} = zmm1[0],zmm3[0],zmm1[2],zmm3[2],zmm1[4],zmm3[4],zmm1[6],zmm3[6]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm9, %zmm5, %zmm5
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm9 = [7,15,7,15,7,15,7,15]
+; AVX512BW-NEXT:    # zmm9 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm13
+; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm9, %zmm13
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm14
+; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm9, %zmm14
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm13 = ymm14[0,1,2,3],ymm13[4,5,6,7]
+; AVX512BW-NEXT:    vpermi2q %zmm2, %zmm0, %zmm9
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm9 {%k1} = zmm1[1],zmm3[1],zmm1[3],zmm3[3],zmm1[5],zmm3[5],zmm1[7],zmm3[7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm13, %zmm9, %zmm9
 ; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm13 = [4,12,4,12,4,12,4,12]
 ; AVX512BW-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm14
-; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm13, %zmm14
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm14
+; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm13, %zmm14
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm15
+; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm13, %zmm15
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm14 = ymm15[0,1,2,3],ymm14[4,5,6,7]
+; AVX512BW-NEXT:    vpermi2q %zmm3, %zmm1, %zmm13
 ; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm15 = zmm0[0],zmm2[0],zmm0[2],zmm2[2],zmm0[4],zmm2[4],zmm0[6],zmm2[6]
-; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm15 {%k1}
-; AVX512BW-NEXT:    vpermi2q %zmm9, %zmm6, %zmm13
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm14 = [4,12,4,12]
-; AVX512BW-NEXT:    # ymm14 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vpermi2q %zmm8, %zmm7, %zmm14
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm13 = ymm13[0,1,2,3],ymm14[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm13, %zmm15, %zmm13
+; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm15 {%k1}
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm14, %zmm15, %zmm13
 ; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm14 = [5,13,5,13,5,13,5,13]
 ; AVX512BW-NEXT:    # zmm14 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm15
-; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm14, %zmm15
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm16 = zmm0[1],zmm2[1],zmm0[3],zmm2[3],zmm0[5],zmm2[5],zmm0[7],zmm2[7]
-; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm16 {%k1}
-; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm14, %zmm6
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm9 = [5,13,5,13]
-; AVX512BW-NEXT:    # ymm9 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vpermi2q %zmm8, %zmm7, %zmm9
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3],ymm9[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm6, %zmm16, %zmm6
+; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm14, %zmm7
+; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm14, %zmm6
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3],ymm7[4,5,6,7]
+; AVX512BW-NEXT:    vpermi2q %zmm3, %zmm1, %zmm14
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm7 = zmm0[1],zmm2[1],zmm0[3],zmm2[3],zmm0[5],zmm2[5],zmm0[7],zmm2[7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm7 {%k1}
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm6, %zmm7, %zmm6
 ; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm7 = [0,8,0,8,0,8,0,8]
 ; AVX512BW-NEXT:    # zmm7 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm8
@@ -976,13 +964,13 @@ define void @store_i64_stride8_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512BW-NEXT:    vinserti64x4 $0, %ymm8, %zmm7, %zmm7
 ; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm8 = [2,10,2,10,2,10,2,10]
 ; AVX512BW-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm9
-; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm8, %zmm9
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm10
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm8, %zmm10
 ; AVX512BW-NEXT:    vpermi2q %zmm2, %zmm0, %zmm8
-; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm8 {%k1}
-; AVX512BW-NEXT:    vmovdqa (%rcx), %ymm9
-; AVX512BW-NEXT:    vmovdqa (%rdx), %ymm10
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm12 = ymm10[0],ymm9[0],ymm10[2],ymm9[2]
+; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm8 {%k1}
+; AVX512BW-NEXT:    vmovdqa (%rcx), %ymm10
+; AVX512BW-NEXT:    vmovdqa (%rdx), %ymm11
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm12 = ymm11[0],ymm10[0],ymm11[2],ymm10[2]
 ; AVX512BW-NEXT:    vmovdqa (%rsi), %ymm14
 ; AVX512BW-NEXT:    vmovdqa (%rdi), %ymm15
 ; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm4 = ymm15[0],ymm14[0],ymm15[2],ymm14[2]
@@ -993,7 +981,7 @@ define void @store_i64_stride8_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm8, %zmm1
 ; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm8, %zmm0
 ; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm0 {%k1}
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm10[1],ymm9[1],ymm10[3],ymm9[3]
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm11[1],ymm10[1],ymm11[3],ymm10[3]
 ; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm2 = ymm15[1],ymm14[1],ymm15[3],ymm14[3]
 ; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm1 = ymm2[2,3],ymm1[2,3]
 ; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm0
@@ -1002,9 +990,9 @@ define void @store_i64_stride8_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512BW-NEXT:    vmovdqa64 %zmm7, (%rax)
 ; AVX512BW-NEXT:    vmovdqa64 %zmm6, 320(%rax)
 ; AVX512BW-NEXT:    vmovdqa64 %zmm13, 256(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm11, 448(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm9, 448(%rax)
 ; AVX512BW-NEXT:    vmovdqa64 %zmm5, 384(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm17, 64(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm16, 64(%rax)
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
   %in.vec0 = load <8 x i64>, ptr %in.vecptr0, align 64
@@ -1894,204 +1882,196 @@ define void @store_i64_stride8_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512F-NEXT:    movq {{[0-9]+}}(%rsp), %r10
 ; AVX512F-NEXT:    movq {{[0-9]+}}(%rsp), %r11
 ; AVX512F-NEXT:    vmovdqa64 64(%rdi), %zmm11
-; AVX512F-NEXT:    vmovdqa64 (%rdi), %zmm3
-; AVX512F-NEXT:    vmovdqa64 64(%rsi), %zmm25
-; AVX512F-NEXT:    vmovdqa64 (%rsi), %zmm19
-; AVX512F-NEXT:    vmovdqa64 64(%rdx), %zmm12
-; AVX512F-NEXT:    vmovdqa64 (%rdx), %zmm0
-; AVX512F-NEXT:    vmovdqa64 64(%rcx), %zmm26
-; AVX512F-NEXT:    vmovdqa64 (%rcx), %zmm17
+; AVX512F-NEXT:    vmovdqa64 (%rdi), %zmm15
+; AVX512F-NEXT:    vmovdqa64 64(%rsi), %zmm27
+; AVX512F-NEXT:    vmovdqa64 (%rsi), %zmm17
+; AVX512F-NEXT:    vmovdqa64 64(%rdx), %zmm14
+; AVX512F-NEXT:    vmovdqa64 (%rdx), %zmm5
+; AVX512F-NEXT:    vmovdqa64 64(%rcx), %zmm28
+; AVX512F-NEXT:    vmovdqa64 (%rcx), %zmm26
 ; AVX512F-NEXT:    vmovdqa64 (%r8), %zmm6
-; AVX512F-NEXT:    vmovdqa64 64(%r8), %zmm16
-; AVX512F-NEXT:    vmovdqa64 (%r9), %zmm31
-; AVX512F-NEXT:    vmovdqa64 64(%r9), %zmm27
+; AVX512F-NEXT:    vmovdqa64 64(%r8), %zmm0
+; AVX512F-NEXT:    vmovdqa64 (%r9), %zmm7
+; AVX512F-NEXT:    vmovdqa64 64(%r9), %zmm1
 ; AVX512F-NEXT:    vmovdqa64 (%r11), %zmm8
-; AVX512F-NEXT:    vmovdqa64 64(%r11), %zmm30
+; AVX512F-NEXT:    vmovdqa64 64(%r11), %zmm3
 ; AVX512F-NEXT:    vmovdqa64 (%r10), %zmm9
-; AVX512F-NEXT:    vmovdqa64 64(%r10), %zmm29
+; AVX512F-NEXT:    vmovdqa64 64(%r10), %zmm4
 ; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm18 = [1,9,1,9,1,9,1,9]
 ; AVX512F-NEXT:    # zmm18 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm1
-; AVX512F-NEXT:    vpermt2q %zmm9, %zmm18, %zmm1
-; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm4
-; AVX512F-NEXT:    vpermt2q %zmm31, %zmm18, %zmm4
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm10
+; AVX512F-NEXT:    vpermt2q %zmm9, %zmm18, %zmm10
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm12
+; AVX512F-NEXT:    vpermt2q %zmm7, %zmm18, %zmm12
 ; AVX512F-NEXT:    movb $-64, %r8b
 ; AVX512F-NEXT:    kmovw %r8d, %k1
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
-; AVX512F-NEXT:    vmovdqa (%rsi), %xmm1
-; AVX512F-NEXT:    vinserti128 $1, (%rcx), %ymm1, %ymm1
-; AVX512F-NEXT:    vmovdqa (%rdi), %xmm10
-; AVX512F-NEXT:    vinserti128 $1, (%rdx), %ymm10, %ymm10
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm13 = ymm10[1],ymm1[1],ymm10[3],ymm1[3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm13, %zmm4, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm23 = [0,8,0,8,0,8,0,8]
+; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm12 {%k1}
+; AVX512F-NEXT:    vmovdqa (%rsi), %xmm10
+; AVX512F-NEXT:    vinserti128 $1, (%rcx), %ymm10, %ymm10
+; AVX512F-NEXT:    vmovdqa64 (%rdi), %xmm16
+; AVX512F-NEXT:    vinserti32x4 $1, (%rdx), %ymm16, %ymm16
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm19 = ymm16[1],ymm10[1],ymm16[3],ymm10[3]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm19, %zmm12, %zmm21
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm19 = [0,8,0,8,0,8,0,8]
+; AVX512F-NEXT:    # zmm19 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm12
+; AVX512F-NEXT:    vpermt2q %zmm9, %zmm19, %zmm12
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm20
+; AVX512F-NEXT:    vpermt2q %zmm7, %zmm19, %zmm20
+; AVX512F-NEXT:    vmovdqa64 %zmm12, %zmm20 {%k1}
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm10 = ymm16[0],ymm10[0],ymm16[2],ymm10[2]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm10, %zmm20, %zmm22
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm20 = [5,13,5,13,5,13,5,13]
+; AVX512F-NEXT:    # zmm20 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm10
+; AVX512F-NEXT:    vpermt2q %zmm26, %zmm20, %zmm10
+; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm12
+; AVX512F-NEXT:    vpermt2q %zmm17, %zmm20, %zmm12
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm10 = ymm12[0,1,2,3],ymm10[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm12
+; AVX512F-NEXT:    vpermt2q %zmm9, %zmm20, %zmm12
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm16 = zmm6[1],zmm7[1],zmm6[3],zmm7[3],zmm6[5],zmm7[5],zmm6[7],zmm7[7]
+; AVX512F-NEXT:    vmovdqa64 %zmm12, %zmm16 {%k1}
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm10, %zmm16, %zmm29
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm23 = [4,12,4,12,4,12,4,12]
 ; AVX512F-NEXT:    # zmm23 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm4
-; AVX512F-NEXT:    vpermt2q %zmm9, %zmm23, %zmm4
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm12
+; AVX512F-NEXT:    vpermt2q %zmm26, %zmm23, %zmm12
+; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm2
+; AVX512F-NEXT:    vpermt2q %zmm17, %zmm23, %zmm2
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm12[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm12
+; AVX512F-NEXT:    vpermt2q %zmm9, %zmm23, %zmm12
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm16 = zmm6[0],zmm7[0],zmm6[2],zmm7[2],zmm6[4],zmm7[4],zmm6[6],zmm7[6]
+; AVX512F-NEXT:    vmovdqa64 %zmm12, %zmm16 {%k1}
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm16, %zmm31
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm24 = [7,15,7,15,7,15,7,15]
+; AVX512F-NEXT:    # zmm24 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm2
+; AVX512F-NEXT:    vpermt2q %zmm26, %zmm24, %zmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm13
+; AVX512F-NEXT:    vpermt2q %zmm17, %zmm24, %zmm13
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm13[0,1,2,3],ymm2[4,5,6,7]
 ; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm13
-; AVX512F-NEXT:    vpermt2q %zmm31, %zmm23, %zmm13
-; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm13 {%k1}
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm1 = ymm10[0],ymm1[0],ymm10[2],ymm1[2]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm13, %zmm21
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm10 = [5,13,5,13,5,13,5,13]
-; AVX512F-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm1
-; AVX512F-NEXT:    vpermt2q %zmm9, %zmm10, %zmm1
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm4 = zmm6[1],zmm31[1],zmm6[3],zmm31[3],zmm6[5],zmm31[5],zmm6[7],zmm31[7]
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
-; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm1
-; AVX512F-NEXT:    vpermt2q %zmm19, %zmm10, %zmm1
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm14 = [5,13,5,13]
-; AVX512F-NEXT:    # ymm14 = mem[0,1,0,1]
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm13
-; AVX512F-NEXT:    vpermt2q %zmm17, %zmm14, %zmm13
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm13[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm4, %zmm22
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm15 = [4,12,4,12,4,12,4,12]
-; AVX512F-NEXT:    # zmm15 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm1
-; AVX512F-NEXT:    vpermt2q %zmm9, %zmm15, %zmm1
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm4 = zmm6[0],zmm31[0],zmm6[2],zmm31[2],zmm6[4],zmm31[4],zmm6[6],zmm31[6]
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
-; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm13
-; AVX512F-NEXT:    vpermt2q %zmm19, %zmm15, %zmm13
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [4,12,4,12]
-; AVX512F-NEXT:    # ymm1 = mem[0,1,0,1]
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm5
-; AVX512F-NEXT:    vpermt2q %zmm17, %zmm1, %zmm5
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm5 = ymm13[0,1,2,3],ymm5[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm5, %zmm4, %zmm24
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm13 = [7,15,7,15,7,15,7,15]
-; AVX512F-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpermt2q %zmm7, %zmm24, %zmm13
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm13 {%k1} = zmm8[1],zmm9[1],zmm8[3],zmm9[3],zmm8[5],zmm9[5],zmm8[7],zmm9[7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm13, %zmm16
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm25 = [6,14,6,14,6,14,6,14]
+; AVX512F-NEXT:    # zmm25 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpermt2q %zmm26, %zmm25, %zmm5
+; AVX512F-NEXT:    vpermt2q %zmm17, %zmm25, %zmm15
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm15[0,1,2,3],ymm5[4,5,6,7]
 ; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm5
-; AVX512F-NEXT:    vpermt2q %zmm31, %zmm13, %zmm5
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm5 {%k1} = zmm8[1],zmm9[1],zmm8[3],zmm9[3],zmm8[5],zmm9[5],zmm8[7],zmm9[7]
+; AVX512F-NEXT:    vpermt2q %zmm7, %zmm25, %zmm5
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm5 {%k1} = zmm8[0],zmm9[0],zmm8[2],zmm9[2],zmm8[4],zmm9[4],zmm8[6],zmm9[6]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm5, %zmm17
 ; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm19, %zmm13, %zmm2
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm4 = [7,15,7,15]
-; AVX512F-NEXT:    # ymm4 = mem[0,1,0,1]
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm7
-; AVX512F-NEXT:    vpermt2q %zmm17, %zmm4, %zmm7
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm7[4,5,6,7]
+; AVX512F-NEXT:    vpermt2q %zmm4, %zmm18, %zmm2
+; AVX512F-NEXT:    vpermi2q %zmm1, %zmm0, %zmm18
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm18 {%k1}
+; AVX512F-NEXT:    vmovdqa 64(%rsi), %xmm2
+; AVX512F-NEXT:    vinserti128 $1, 64(%rcx), %ymm2, %ymm2
+; AVX512F-NEXT:    vmovdqa 64(%rdi), %xmm5
+; AVX512F-NEXT:    vinserti128 $1, 64(%rdx), %ymm5, %ymm5
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm13 = ymm5[1],ymm2[1],ymm5[3],ymm2[3]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm13, %zmm18, %zmm18
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm13
+; AVX512F-NEXT:    vpermt2q %zmm4, %zmm19, %zmm13
+; AVX512F-NEXT:    vpermi2q %zmm1, %zmm0, %zmm19
+; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm19 {%k1}
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm2 = ymm5[0],ymm2[0],ymm5[2],ymm2[2]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm19, %zmm19
+; AVX512F-NEXT:    vmovdqa64 %zmm14, %zmm2
+; AVX512F-NEXT:    vpermt2q %zmm28, %zmm20, %zmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm5
+; AVX512F-NEXT:    vpermt2q %zmm27, %zmm20, %zmm5
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm5[0,1,2,3],ymm2[4,5,6,7]
+; AVX512F-NEXT:    vpermi2q %zmm4, %zmm3, %zmm20
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm5 = zmm0[1],zmm1[1],zmm0[3],zmm1[3],zmm0[5],zmm1[5],zmm0[7],zmm1[7]
+; AVX512F-NEXT:    vmovdqa64 %zmm20, %zmm5 {%k1}
 ; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm5, %zmm20
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm28 = [6,14,6,14,6,14,6,14]
-; AVX512F-NEXT:    # zmm28 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm31, %zmm28, %zmm2
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm2 {%k1} = zmm8[0],zmm9[0],zmm8[2],zmm9[2],zmm8[4],zmm9[4],zmm8[6],zmm9[6]
-; AVX512F-NEXT:    vpermt2q %zmm19, %zmm28, %zmm3
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm7 = [6,14,6,14]
-; AVX512F-NEXT:    # ymm7 = mem[0,1,0,1]
-; AVX512F-NEXT:    vpermt2q %zmm17, %zmm7, %zmm0
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3],ymm0[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm2, %zmm17
-; AVX512F-NEXT:    vmovdqa64 %zmm30, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm29, %zmm18, %zmm0
-; AVX512F-NEXT:    vpermi2q %zmm27, %zmm16, %zmm18
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm18 {%k1}
-; AVX512F-NEXT:    vmovdqa 64(%rsi), %xmm0
-; AVX512F-NEXT:    vinserti128 $1, 64(%rcx), %ymm0, %ymm0
-; AVX512F-NEXT:    vmovdqa 64(%rdi), %xmm2
-; AVX512F-NEXT:    vinserti128 $1, 64(%rdx), %ymm2, %ymm2
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm3 = ymm2[1],ymm0[1],ymm2[3],ymm0[3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm3, %zmm18, %zmm18
-; AVX512F-NEXT:    vmovdqa64 %zmm30, %zmm3
-; AVX512F-NEXT:    vpermt2q %zmm29, %zmm23, %zmm3
-; AVX512F-NEXT:    vpermi2q %zmm27, %zmm16, %zmm23
-; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm23 {%k1}
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm2[0],ymm0[0],ymm2[2],ymm0[2]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm23, %zmm19
-; AVX512F-NEXT:    vmovdqa64 %zmm30, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm29, %zmm10, %zmm0
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm2 = zmm16[1],zmm27[1],zmm16[3],zmm27[3],zmm16[5],zmm27[5],zmm16[7],zmm27[7]
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k1}
-; AVX512F-NEXT:    vpermi2q %zmm25, %zmm11, %zmm10
-; AVX512F-NEXT:    vpermi2q %zmm26, %zmm12, %zmm14
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm10[0,1,2,3],ymm14[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm2, %zmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm30, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm29, %zmm15, %zmm0
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm3 = zmm16[0],zmm27[0],zmm16[2],zmm27[2],zmm16[4],zmm27[4],zmm16[6],zmm27[6]
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm3 {%k1}
-; AVX512F-NEXT:    vpermi2q %zmm25, %zmm11, %zmm15
-; AVX512F-NEXT:    vpermi2q %zmm26, %zmm12, %zmm1
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm15[0,1,2,3],ymm1[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm3, %zmm1
-; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm27, %zmm13, %zmm0
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm0 {%k1} = zmm30[1],zmm29[1],zmm30[3],zmm29[3],zmm30[5],zmm29[5],zmm30[7],zmm29[7]
-; AVX512F-NEXT:    vpermi2q %zmm25, %zmm11, %zmm13
-; AVX512F-NEXT:    vpermi2q %zmm26, %zmm12, %zmm4
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm3 = ymm13[0,1,2,3],ymm4[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm4
-; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm27, %zmm28, %zmm0
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm0 {%k1} = zmm30[0],zmm29[0],zmm30[2],zmm29[2],zmm30[4],zmm29[4],zmm30[6],zmm29[6]
-; AVX512F-NEXT:    vpermt2q %zmm25, %zmm28, %zmm11
-; AVX512F-NEXT:    vpermt2q %zmm26, %zmm7, %zmm12
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm3 = ymm11[0,1,2,3],ymm12[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm0
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm3 = [3,11,3,11,3,11,3,11]
-; AVX512F-NEXT:    # zmm3 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm7
-; AVX512F-NEXT:    vpermt2q %zmm9, %zmm3, %zmm7
-; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm10
-; AVX512F-NEXT:    vpermt2q %zmm31, %zmm3, %zmm10
-; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm10 {%k1}
-; AVX512F-NEXT:    vmovdqa (%rcx), %ymm7
-; AVX512F-NEXT:    vmovdqa 64(%rcx), %ymm11
-; AVX512F-NEXT:    vmovdqa (%rdx), %ymm12
-; AVX512F-NEXT:    vmovdqa 64(%rdx), %ymm13
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm14 = ymm12[1],ymm7[1],ymm12[3],ymm7[3]
-; AVX512F-NEXT:    vmovdqa (%rsi), %ymm15
-; AVX512F-NEXT:    vmovdqa64 64(%rsi), %ymm23
-; AVX512F-NEXT:    vmovdqa64 (%rdi), %ymm25
-; AVX512F-NEXT:    vmovdqa64 64(%rdi), %ymm26
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm5 = ymm25[1],ymm15[1],ymm25[3],ymm15[3]
-; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm5 = ymm5[2,3],ymm14[2,3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm5, %zmm10, %zmm5
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm10 = [2,10,2,10,2,10,2,10]
-; AVX512F-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermt2q %zmm9, %zmm10, %zmm8
-; AVX512F-NEXT:    vpermt2q %zmm31, %zmm10, %zmm6
+; AVX512F-NEXT:    vmovdqa64 %zmm14, %zmm2
+; AVX512F-NEXT:    vpermt2q %zmm28, %zmm23, %zmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm5
+; AVX512F-NEXT:    vpermt2q %zmm27, %zmm23, %zmm5
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm5[0,1,2,3],ymm2[4,5,6,7]
+; AVX512F-NEXT:    vpermi2q %zmm4, %zmm3, %zmm23
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm5 = zmm0[0],zmm1[0],zmm0[2],zmm1[2],zmm0[4],zmm1[4],zmm0[6],zmm1[6]
+; AVX512F-NEXT:    vmovdqa64 %zmm23, %zmm5 {%k1}
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm5, %zmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm14, %zmm5
+; AVX512F-NEXT:    vpermt2q %zmm28, %zmm24, %zmm5
+; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm13
+; AVX512F-NEXT:    vpermt2q %zmm27, %zmm24, %zmm13
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm5 = ymm13[0,1,2,3],ymm5[4,5,6,7]
+; AVX512F-NEXT:    vpermi2q %zmm1, %zmm0, %zmm24
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm24 {%k1} = zmm3[1],zmm4[1],zmm3[3],zmm4[3],zmm3[5],zmm4[5],zmm3[7],zmm4[7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm5, %zmm24, %zmm5
+; AVX512F-NEXT:    vpermt2q %zmm28, %zmm25, %zmm14
+; AVX512F-NEXT:    vpermt2q %zmm27, %zmm25, %zmm11
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm11 = ymm11[0,1,2,3],ymm14[4,5,6,7]
+; AVX512F-NEXT:    vpermi2q %zmm1, %zmm0, %zmm25
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm25 {%k1} = zmm3[0],zmm4[0],zmm3[2],zmm4[2],zmm3[4],zmm4[4],zmm3[6],zmm4[6]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm11, %zmm25, %zmm11
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm13 = [3,11,3,11,3,11,3,11]
+; AVX512F-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm14
+; AVX512F-NEXT:    vpermt2q %zmm9, %zmm13, %zmm14
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm15
+; AVX512F-NEXT:    vpermt2q %zmm7, %zmm13, %zmm15
+; AVX512F-NEXT:    vmovdqa64 %zmm14, %zmm15 {%k1}
+; AVX512F-NEXT:    vmovdqa (%rcx), %ymm14
+; AVX512F-NEXT:    vmovdqa64 64(%rcx), %ymm23
+; AVX512F-NEXT:    vmovdqa64 (%rdx), %ymm24
+; AVX512F-NEXT:    vmovdqa64 64(%rdx), %ymm25
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm10 = ymm24[1],ymm14[1],ymm24[3],ymm14[3]
+; AVX512F-NEXT:    vmovdqa64 (%rsi), %ymm26
+; AVX512F-NEXT:    vmovdqa64 64(%rsi), %ymm27
+; AVX512F-NEXT:    vmovdqa64 (%rdi), %ymm28
+; AVX512F-NEXT:    vmovdqa64 64(%rdi), %ymm30
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm12 = ymm28[1],ymm26[1],ymm28[3],ymm26[3]
+; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm10 = ymm12[2,3],ymm10[2,3]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm10, %zmm15, %zmm10
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm12 = [2,10,2,10,2,10,2,10]
+; AVX512F-NEXT:    # zmm12 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpermt2q %zmm9, %zmm12, %zmm8
+; AVX512F-NEXT:    vpermt2q %zmm7, %zmm12, %zmm6
 ; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm6 {%k1}
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm7 = ymm12[0],ymm7[0],ymm12[2],ymm7[2]
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm8 = ymm25[0],ymm15[0],ymm25[2],ymm15[2]
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm7 = ymm24[0],ymm14[0],ymm24[2],ymm14[2]
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm8 = ymm28[0],ymm26[0],ymm28[2],ymm26[2]
 ; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm7 = ymm8[2,3],ymm7[2,3]
 ; AVX512F-NEXT:    vinserti64x4 $0, %ymm7, %zmm6, %zmm6
-; AVX512F-NEXT:    vmovdqa64 %zmm30, %zmm7
-; AVX512F-NEXT:    vpermt2q %zmm29, %zmm3, %zmm7
-; AVX512F-NEXT:    vpermi2q %zmm27, %zmm16, %zmm3
-; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm3 {%k1}
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm7 = ymm13[1],ymm11[1],ymm13[3],ymm11[3]
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm8 = ymm26[1],ymm23[1],ymm26[3],ymm23[3]
-; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm7 = ymm8[2,3],ymm7[2,3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm7, %zmm3, %zmm3
-; AVX512F-NEXT:    vpermt2q %zmm29, %zmm10, %zmm30
-; AVX512F-NEXT:    vpermt2q %zmm27, %zmm10, %zmm16
-; AVX512F-NEXT:    vmovdqa64 %zmm30, %zmm16 {%k1}
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm7 = ymm13[0],ymm11[0],ymm13[2],ymm11[2]
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm8 = ymm26[0],ymm23[0],ymm26[2],ymm23[2]
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm7
+; AVX512F-NEXT:    vpermt2q %zmm4, %zmm13, %zmm7
+; AVX512F-NEXT:    vpermi2q %zmm1, %zmm0, %zmm13
+; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm13 {%k1}
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm7 = ymm25[1],ymm23[1],ymm25[3],ymm23[3]
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm8 = ymm30[1],ymm27[1],ymm30[3],ymm27[3]
 ; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm7 = ymm8[2,3],ymm7[2,3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm7, %zmm16, %zmm7
-; AVX512F-NEXT:    vmovdqa64 %zmm7, 640(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm3, 704(%rax)
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm7, %zmm13, %zmm7
+; AVX512F-NEXT:    vpermt2q %zmm4, %zmm12, %zmm3
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm12, %zmm0
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm0 {%k1}
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm1 = ymm25[0],ymm23[0],ymm25[2],ymm23[2]
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm3 = ymm30[0],ymm27[0],ymm30[2],ymm27[2]
+; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm1 = ymm3[2,3],ymm1[2,3]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vmovdqa64 %zmm0, 640(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm7, 704(%rax)
 ; AVX512F-NEXT:    vmovdqa64 %zmm6, 128(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm5, 192(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm0, 896(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm4, 960(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm1, 768(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm2, 832(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm10, 192(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm11, 896(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm5, 960(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm2, 768(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm20, 832(%rax)
 ; AVX512F-NEXT:    vmovdqa64 %zmm19, 512(%rax)
 ; AVX512F-NEXT:    vmovdqa64 %zmm18, 576(%rax)
 ; AVX512F-NEXT:    vmovdqa64 %zmm17, 384(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm20, 448(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm24, 256(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm22, 320(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm21, (%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 64(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm16, 448(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm31, 256(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm29, 320(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm22, (%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm21, 64(%rax)
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
@@ -2101,204 +2081,196 @@ define void @store_i64_stride8_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %r10
 ; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %r11
 ; AVX512BW-NEXT:    vmovdqa64 64(%rdi), %zmm11
-; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm3
-; AVX512BW-NEXT:    vmovdqa64 64(%rsi), %zmm25
-; AVX512BW-NEXT:    vmovdqa64 (%rsi), %zmm19
-; AVX512BW-NEXT:    vmovdqa64 64(%rdx), %zmm12
-; AVX512BW-NEXT:    vmovdqa64 (%rdx), %zmm0
-; AVX512BW-NEXT:    vmovdqa64 64(%rcx), %zmm26
-; AVX512BW-NEXT:    vmovdqa64 (%rcx), %zmm17
+; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm15
+; AVX512BW-NEXT:    vmovdqa64 64(%rsi), %zmm27
+; AVX512BW-NEXT:    vmovdqa64 (%rsi), %zmm17
+; AVX512BW-NEXT:    vmovdqa64 64(%rdx), %zmm14
+; AVX512BW-NEXT:    vmovdqa64 (%rdx), %zmm5
+; AVX512BW-NEXT:    vmovdqa64 64(%rcx), %zmm28
+; AVX512BW-NEXT:    vmovdqa64 (%rcx), %zmm26
 ; AVX512BW-NEXT:    vmovdqa64 (%r8), %zmm6
-; AVX512BW-NEXT:    vmovdqa64 64(%r8), %zmm16
-; AVX512BW-NEXT:    vmovdqa64 (%r9), %zmm31
-; AVX512BW-NEXT:    vmovdqa64 64(%r9), %zmm27
+; AVX512BW-NEXT:    vmovdqa64 64(%r8), %zmm0
+; AVX512BW-NEXT:    vmovdqa64 (%r9), %zmm7
+; AVX512BW-NEXT:    vmovdqa64 64(%r9), %zmm1
 ; AVX512BW-NEXT:    vmovdqa64 (%r11), %zmm8
-; AVX512BW-NEXT:    vmovdqa64 64(%r11), %zmm30
+; AVX512BW-NEXT:    vmovdqa64 64(%r11), %zmm3
 ; AVX512BW-NEXT:    vmovdqa64 (%r10), %zmm9
-; AVX512BW-NEXT:    vmovdqa64 64(%r10), %zmm29
+; AVX512BW-NEXT:    vmovdqa64 64(%r10), %zmm4
 ; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm18 = [1,9,1,9,1,9,1,9]
 ; AVX512BW-NEXT:    # zmm18 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm1
-; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm18, %zmm1
-; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm4
-; AVX512BW-NEXT:    vpermt2q %zmm31, %zmm18, %zmm4
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm10
+; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm18, %zmm10
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm12
+; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm18, %zmm12
 ; AVX512BW-NEXT:    movb $-64, %r8b
 ; AVX512BW-NEXT:    kmovd %r8d, %k1
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
-; AVX512BW-NEXT:    vmovdqa (%rsi), %xmm1
-; AVX512BW-NEXT:    vinserti128 $1, (%rcx), %ymm1, %ymm1
-; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm10
-; AVX512BW-NEXT:    vinserti128 $1, (%rdx), %ymm10, %ymm10
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm13 = ymm10[1],ymm1[1],ymm10[3],ymm1[3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm13, %zmm4, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm23 = [0,8,0,8,0,8,0,8]
+; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm12 {%k1}
+; AVX512BW-NEXT:    vmovdqa (%rsi), %xmm10
+; AVX512BW-NEXT:    vinserti128 $1, (%rcx), %ymm10, %ymm10
+; AVX512BW-NEXT:    vmovdqa64 (%rdi), %xmm16
+; AVX512BW-NEXT:    vinserti32x4 $1, (%rdx), %ymm16, %ymm16
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm19 = ymm16[1],ymm10[1],ymm16[3],ymm10[3]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm19, %zmm12, %zmm21
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm19 = [0,8,0,8,0,8,0,8]
+; AVX512BW-NEXT:    # zmm19 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm12
+; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm19, %zmm12
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm20
+; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm19, %zmm20
+; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm20 {%k1}
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm10 = ymm16[0],ymm10[0],ymm16[2],ymm10[2]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm10, %zmm20, %zmm22
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm20 = [5,13,5,13,5,13,5,13]
+; AVX512BW-NEXT:    # zmm20 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm10
+; AVX512BW-NEXT:    vpermt2q %zmm26, %zmm20, %zmm10
+; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm12
+; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm20, %zmm12
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm10 = ymm12[0,1,2,3],ymm10[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm12
+; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm20, %zmm12
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm16 = zmm6[1],zmm7[1],zmm6[3],zmm7[3],zmm6[5],zmm7[5],zmm6[7],zmm7[7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm16 {%k1}
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm10, %zmm16, %zmm29
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm23 = [4,12,4,12,4,12,4,12]
 ; AVX512BW-NEXT:    # zmm23 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm4
-; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm23, %zmm4
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm12
+; AVX512BW-NEXT:    vpermt2q %zmm26, %zmm23, %zmm12
+; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm2
+; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm23, %zmm2
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm12[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm12
+; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm23, %zmm12
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm16 = zmm6[0],zmm7[0],zmm6[2],zmm7[2],zmm6[4],zmm7[4],zmm6[6],zmm7[6]
+; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm16 {%k1}
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm16, %zmm31
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm24 = [7,15,7,15,7,15,7,15]
+; AVX512BW-NEXT:    # zmm24 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm2
+; AVX512BW-NEXT:    vpermt2q %zmm26, %zmm24, %zmm2
+; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm13
+; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm24, %zmm13
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm13[0,1,2,3],ymm2[4,5,6,7]
 ; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm13
-; AVX512BW-NEXT:    vpermt2q %zmm31, %zmm23, %zmm13
-; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm13 {%k1}
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm1 = ymm10[0],ymm1[0],ymm10[2],ymm1[2]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm13, %zmm21
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm10 = [5,13,5,13,5,13,5,13]
-; AVX512BW-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm1
-; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm10, %zmm1
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm4 = zmm6[1],zmm31[1],zmm6[3],zmm31[3],zmm6[5],zmm31[5],zmm6[7],zmm31[7]
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm1
-; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm10, %zmm1
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm14 = [5,13,5,13]
-; AVX512BW-NEXT:    # ymm14 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm13
-; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm14, %zmm13
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm13[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm4, %zmm22
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm15 = [4,12,4,12,4,12,4,12]
-; AVX512BW-NEXT:    # zmm15 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm1
-; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm15, %zmm1
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm4 = zmm6[0],zmm31[0],zmm6[2],zmm31[2],zmm6[4],zmm31[4],zmm6[6],zmm31[6]
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm13
-; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm15, %zmm13
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [4,12,4,12]
-; AVX512BW-NEXT:    # ymm1 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm5
-; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm1, %zmm5
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm5 = ymm13[0,1,2,3],ymm5[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm5, %zmm4, %zmm24
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm13 = [7,15,7,15,7,15,7,15]
-; AVX512BW-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm24, %zmm13
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm13 {%k1} = zmm8[1],zmm9[1],zmm8[3],zmm9[3],zmm8[5],zmm9[5],zmm8[7],zmm9[7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm13, %zmm16
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm25 = [6,14,6,14,6,14,6,14]
+; AVX512BW-NEXT:    # zmm25 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpermt2q %zmm26, %zmm25, %zmm5
+; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm25, %zmm15
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm15[0,1,2,3],ymm5[4,5,6,7]
 ; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm5
-; AVX512BW-NEXT:    vpermt2q %zmm31, %zmm13, %zmm5
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm5 {%k1} = zmm8[1],zmm9[1],zmm8[3],zmm9[3],zmm8[5],zmm9[5],zmm8[7],zmm9[7]
+; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm25, %zmm5
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm5 {%k1} = zmm8[0],zmm9[0],zmm8[2],zmm9[2],zmm8[4],zmm9[4],zmm8[6],zmm9[6]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm5, %zmm17
 ; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm2
-; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm13, %zmm2
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm4 = [7,15,7,15]
-; AVX512BW-NEXT:    # ymm4 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm7
-; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm4, %zmm7
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm7[4,5,6,7]
+; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm18, %zmm2
+; AVX512BW-NEXT:    vpermi2q %zmm1, %zmm0, %zmm18
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm18 {%k1}
+; AVX512BW-NEXT:    vmovdqa 64(%rsi), %xmm2
+; AVX512BW-NEXT:    vinserti128 $1, 64(%rcx), %ymm2, %ymm2
+; AVX512BW-NEXT:    vmovdqa 64(%rdi), %xmm5
+; AVX512BW-NEXT:    vinserti128 $1, 64(%rdx), %ymm5, %ymm5
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm13 = ymm5[1],ymm2[1],ymm5[3],ymm2[3]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm13, %zmm18, %zmm18
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm13
+; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm19, %zmm13
+; AVX512BW-NEXT:    vpermi2q %zmm1, %zmm0, %zmm19
+; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm19 {%k1}
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm2 = ymm5[0],ymm2[0],ymm5[2],ymm2[2]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm19, %zmm19
+; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm2
+; AVX512BW-NEXT:    vpermt2q %zmm28, %zmm20, %zmm2
+; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm5
+; AVX512BW-NEXT:    vpermt2q %zmm27, %zmm20, %zmm5
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm5[0,1,2,3],ymm2[4,5,6,7]
+; AVX512BW-NEXT:    vpermi2q %zmm4, %zmm3, %zmm20
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm5 = zmm0[1],zmm1[1],zmm0[3],zmm1[3],zmm0[5],zmm1[5],zmm0[7],zmm1[7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm20, %zmm5 {%k1}
 ; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm5, %zmm20
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm28 = [6,14,6,14,6,14,6,14]
-; AVX512BW-NEXT:    # zmm28 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm2
-; AVX512BW-NEXT:    vpermt2q %zmm31, %zmm28, %zmm2
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm2 {%k1} = zmm8[0],zmm9[0],zmm8[2],zmm9[2],zmm8[4],zmm9[4],zmm8[6],zmm9[6]
-; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm28, %zmm3
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm7 = [6,14,6,14]
-; AVX512BW-NEXT:    # ymm7 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm7, %zmm0
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3],ymm0[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm2, %zmm17
-; AVX512BW-NEXT:    vmovdqa64 %zmm30, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm29, %zmm18, %zmm0
-; AVX512BW-NEXT:    vpermi2q %zmm27, %zmm16, %zmm18
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm18 {%k1}
-; AVX512BW-NEXT:    vmovdqa 64(%rsi), %xmm0
-; AVX512BW-NEXT:    vinserti128 $1, 64(%rcx), %ymm0, %ymm0
-; AVX512BW-NEXT:    vmovdqa 64(%rdi), %xmm2
-; AVX512BW-NEXT:    vinserti128 $1, 64(%rdx), %ymm2, %ymm2
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm3 = ymm2[1],ymm0[1],ymm2[3],ymm0[3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm3, %zmm18, %zmm18
-; AVX512BW-NEXT:    vmovdqa64 %zmm30, %zmm3
-; AVX512BW-NEXT:    vpermt2q %zmm29, %zmm23, %zmm3
-; AVX512BW-NEXT:    vpermi2q %zmm27, %zmm16, %zmm23
-; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm23 {%k1}
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm2[0],ymm0[0],ymm2[2],ymm0[2]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm23, %zmm19
-; AVX512BW-NEXT:    vmovdqa64 %zmm30, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm29, %zmm10, %zmm0
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm2 = zmm16[1],zmm27[1],zmm16[3],zmm27[3],zmm16[5],zmm27[5],zmm16[7],zmm27[7]
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k1}
-; AVX512BW-NEXT:    vpermi2q %zmm25, %zmm11, %zmm10
-; AVX512BW-NEXT:    vpermi2q %zmm26, %zmm12, %zmm14
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm10[0,1,2,3],ymm14[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm2, %zmm2
-; AVX512BW-NEXT:    vmovdqa64 %zmm30, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm29, %zmm15, %zmm0
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm3 = zmm16[0],zmm27[0],zmm16[2],zmm27[2],zmm16[4],zmm27[4],zmm16[6],zmm27[6]
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm3 {%k1}
-; AVX512BW-NEXT:    vpermi2q %zmm25, %zmm11, %zmm15
-; AVX512BW-NEXT:    vpermi2q %zmm26, %zmm12, %zmm1
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm15[0,1,2,3],ymm1[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm3, %zmm1
-; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm27, %zmm13, %zmm0
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm0 {%k1} = zmm30[1],zmm29[1],zmm30[3],zmm29[3],zmm30[5],zmm29[5],zmm30[7],zmm29[7]
-; AVX512BW-NEXT:    vpermi2q %zmm25, %zmm11, %zmm13
-; AVX512BW-NEXT:    vpermi2q %zmm26, %zmm12, %zmm4
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm13[0,1,2,3],ymm4[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm4
-; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm27, %zmm28, %zmm0
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm0 {%k1} = zmm30[0],zmm29[0],zmm30[2],zmm29[2],zmm30[4],zmm29[4],zmm30[6],zmm29[6]
-; AVX512BW-NEXT:    vpermt2q %zmm25, %zmm28, %zmm11
-; AVX512BW-NEXT:    vpermt2q %zmm26, %zmm7, %zmm12
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm3 = ymm11[0,1,2,3],ymm12[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm3, %zmm0, %zmm0
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm3 = [3,11,3,11,3,11,3,11]
-; AVX512BW-NEXT:    # zmm3 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm7
-; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm3, %zmm7
-; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm10
-; AVX512BW-NEXT:    vpermt2q %zmm31, %zmm3, %zmm10
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm10 {%k1}
-; AVX512BW-NEXT:    vmovdqa (%rcx), %ymm7
-; AVX512BW-NEXT:    vmovdqa 64(%rcx), %ymm11
-; AVX512BW-NEXT:    vmovdqa (%rdx), %ymm12
-; AVX512BW-NEXT:    vmovdqa 64(%rdx), %ymm13
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm14 = ymm12[1],ymm7[1],ymm12[3],ymm7[3]
-; AVX512BW-NEXT:    vmovdqa (%rsi), %ymm15
-; AVX512BW-NEXT:    vmovdqa64 64(%rsi), %ymm23
-; AVX512BW-NEXT:    vmovdqa64 (%rdi), %ymm25
-; AVX512BW-NEXT:    vmovdqa64 64(%rdi), %ymm26
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm5 = ymm25[1],ymm15[1],ymm25[3],ymm15[3]
-; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm5 = ymm5[2,3],ymm14[2,3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm5, %zmm10, %zmm5
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm10 = [2,10,2,10,2,10,2,10]
-; AVX512BW-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm10, %zmm8
-; AVX512BW-NEXT:    vpermt2q %zmm31, %zmm10, %zmm6
+; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm2
+; AVX512BW-NEXT:    vpermt2q %zmm28, %zmm23, %zmm2
+; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm5
+; AVX512BW-NEXT:    vpermt2q %zmm27, %zmm23, %zmm5
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm5[0,1,2,3],ymm2[4,5,6,7]
+; AVX512BW-NEXT:    vpermi2q %zmm4, %zmm3, %zmm23
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm5 = zmm0[0],zmm1[0],zmm0[2],zmm1[2],zmm0[4],zmm1[4],zmm0[6],zmm1[6]
+; AVX512BW-NEXT:    vmovdqa64 %zmm23, %zmm5 {%k1}
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm5, %zmm2
+; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm5
+; AVX512BW-NEXT:    vpermt2q %zmm28, %zmm24, %zmm5
+; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm13
+; AVX512BW-NEXT:    vpermt2q %zmm27, %zmm24, %zmm13
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm5 = ymm13[0,1,2,3],ymm5[4,5,6,7]
+; AVX512BW-NEXT:    vpermi2q %zmm1, %zmm0, %zmm24
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm24 {%k1} = zmm3[1],zmm4[1],zmm3[3],zmm4[3],zmm3[5],zmm4[5],zmm3[7],zmm4[7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm5, %zmm24, %zmm5
+; AVX512BW-NEXT:    vpermt2q %zmm28, %zmm25, %zmm14
+; AVX512BW-NEXT:    vpermt2q %zmm27, %zmm25, %zmm11
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm11 = ymm11[0,1,2,3],ymm14[4,5,6,7]
+; AVX512BW-NEXT:    vpermi2q %zmm1, %zmm0, %zmm25
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm25 {%k1} = zmm3[0],zmm4[0],zmm3[2],zmm4[2],zmm3[4],zmm4[4],zmm3[6],zmm4[6]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm11, %zmm25, %zmm11
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm13 = [3,11,3,11,3,11,3,11]
+; AVX512BW-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm14
+; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm13, %zmm14
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm15
+; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm13, %zmm15
+; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm15 {%k1}
+; AVX512BW-NEXT:    vmovdqa (%rcx), %ymm14
+; AVX512BW-NEXT:    vmovdqa64 64(%rcx), %ymm23
+; AVX512BW-NEXT:    vmovdqa64 (%rdx), %ymm24
+; AVX512BW-NEXT:    vmovdqa64 64(%rdx), %ymm25
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm10 = ymm24[1],ymm14[1],ymm24[3],ymm14[3]
+; AVX512BW-NEXT:    vmovdqa64 (%rsi), %ymm26
+; AVX512BW-NEXT:    vmovdqa64 64(%rsi), %ymm27
+; AVX512BW-NEXT:    vmovdqa64 (%rdi), %ymm28
+; AVX512BW-NEXT:    vmovdqa64 64(%rdi), %ymm30
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm12 = ymm28[1],ymm26[1],ymm28[3],ymm26[3]
+; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm10 = ymm12[2,3],ymm10[2,3]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm10, %zmm15, %zmm10
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm12 = [2,10,2,10,2,10,2,10]
+; AVX512BW-NEXT:    # zmm12 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm12, %zmm8
+; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm12, %zmm6
 ; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm6 {%k1}
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm7 = ymm12[0],ymm7[0],ymm12[2],ymm7[2]
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm8 = ymm25[0],ymm15[0],ymm25[2],ymm15[2]
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm7 = ymm24[0],ymm14[0],ymm24[2],ymm14[2]
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm8 = ymm28[0],ymm26[0],ymm28[2],ymm26[2]
 ; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm7 = ymm8[2,3],ymm7[2,3]
 ; AVX512BW-NEXT:    vinserti64x4 $0, %ymm7, %zmm6, %zmm6
-; AVX512BW-NEXT:    vmovdqa64 %zmm30, %zmm7
-; AVX512BW-NEXT:    vpermt2q %zmm29, %zmm3, %zmm7
-; AVX512BW-NEXT:    vpermi2q %zmm27, %zmm16, %zmm3
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm3 {%k1}
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm7 = ymm13[1],ymm11[1],ymm13[3],ymm11[3]
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm8 = ymm26[1],ymm23[1],ymm26[3],ymm23[3]
-; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm7 = ymm8[2,3],ymm7[2,3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm7, %zmm3, %zmm3
-; AVX512BW-NEXT:    vpermt2q %zmm29, %zmm10, %zmm30
-; AVX512BW-NEXT:    vpermt2q %zmm27, %zmm10, %zmm16
-; AVX512BW-NEXT:    vmovdqa64 %zmm30, %zmm16 {%k1}
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm7 = ymm13[0],ymm11[0],ymm13[2],ymm11[2]
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm8 = ymm26[0],ymm23[0],ymm26[2],ymm23[2]
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm7
+; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm13, %zmm7
+; AVX512BW-NEXT:    vpermi2q %zmm1, %zmm0, %zmm13
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm13 {%k1}
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm7 = ymm25[1],ymm23[1],ymm25[3],ymm23[3]
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm8 = ymm30[1],ymm27[1],ymm30[3],ymm27[3]
 ; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm7 = ymm8[2,3],ymm7[2,3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm7, %zmm16, %zmm7
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, 640(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm3, 704(%rax)
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm7, %zmm13, %zmm7
+; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm12, %zmm3
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm12, %zmm0
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm0 {%k1}
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm1 = ymm25[0],ymm23[0],ymm25[2],ymm23[2]
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm3 = ymm30[0],ymm27[0],ymm30[2],ymm27[2]
+; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm1 = ymm3[2,3],ymm1[2,3]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, 640(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, 704(%rax)
 ; AVX512BW-NEXT:    vmovdqa64 %zmm6, 128(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, 192(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, 896(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm4, 960(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, 768(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, 832(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm10, 192(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm11, 896(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, 960(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, 768(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm20, 832(%rax)
 ; AVX512BW-NEXT:    vmovdqa64 %zmm19, 512(%rax)
 ; AVX512BW-NEXT:    vmovdqa64 %zmm18, 576(%rax)
 ; AVX512BW-NEXT:    vmovdqa64 %zmm17, 384(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm20, 448(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm24, 256(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm22, 320(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm21, (%rax)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm0, 64(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm16, 448(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm31, 256(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm29, 320(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm22, (%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm21, 64(%rax)
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
   %in.vec0 = load <16 x i64>, ptr %in.vecptr0, align 64
@@ -4142,461 +4114,451 @@ define void @store_i64_stride8_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ;
 ; AVX512F-LABEL: store_i64_stride8_vf32:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    subq $2632, %rsp # imm = 0xA48
+; AVX512F-NEXT:    subq $2312, %rsp # imm = 0x908
 ; AVX512F-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512F-NEXT:    movq {{[0-9]+}}(%rsp), %r10
 ; AVX512F-NEXT:    vmovaps 128(%rdi), %zmm0
 ; AVX512F-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 64(%rdi), %zmm1
-; AVX512F-NEXT:    vmovdqa64 (%rdi), %zmm5
-; AVX512F-NEXT:    vmovdqa64 128(%rsi), %zmm25
-; AVX512F-NEXT:    vmovdqa64 64(%rsi), %zmm20
-; AVX512F-NEXT:    vmovdqa64 (%rsi), %zmm0
-; AVX512F-NEXT:    vmovaps 192(%rdx), %zmm2
-; AVX512F-NEXT:    vmovups %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 128(%rdx), %zmm13
-; AVX512F-NEXT:    vmovdqa64 64(%rdx), %zmm2
-; AVX512F-NEXT:    vmovdqa64 (%rdx), %zmm6
-; AVX512F-NEXT:    vmovdqa64 64(%rcx), %zmm21
-; AVX512F-NEXT:    vmovdqa64 (%rcx), %zmm7
-; AVX512F-NEXT:    vmovdqa64 (%r8), %zmm23
-; AVX512F-NEXT:    vmovdqa64 64(%r8), %zmm19
-; AVX512F-NEXT:    vmovdqa64 (%r9), %zmm18
-; AVX512F-NEXT:    vmovdqa64 64(%r9), %zmm28
-; AVX512F-NEXT:    vmovdqa64 (%r10), %zmm17
-; AVX512F-NEXT:    vmovdqa64 64(%r10), %zmm16
-; AVX512F-NEXT:    vmovdqa64 (%rax), %zmm24
-; AVX512F-NEXT:    vmovdqa64 64(%rax), %zmm22
+; AVX512F-NEXT:    vmovdqa64 64(%rdi), %zmm6
+; AVX512F-NEXT:    vmovdqa64 (%rdi), %zmm2
+; AVX512F-NEXT:    vmovdqa64 128(%rsi), %zmm0
+; AVX512F-NEXT:    vmovdqa64 64(%rsi), %zmm5
+; AVX512F-NEXT:    vmovdqa64 (%rsi), %zmm3
+; AVX512F-NEXT:    vmovaps 192(%rdx), %zmm1
+; AVX512F-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovaps 128(%rdx), %zmm1
+; AVX512F-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 64(%rdx), %zmm7
+; AVX512F-NEXT:    vmovdqa64 (%rdx), %zmm4
+; AVX512F-NEXT:    vmovdqa64 192(%rcx), %zmm23
+; AVX512F-NEXT:    vmovdqa64 128(%rcx), %zmm1
+; AVX512F-NEXT:    vmovdqa64 64(%rcx), %zmm9
+; AVX512F-NEXT:    vmovdqa64 (%rcx), %zmm8
+; AVX512F-NEXT:    vmovdqa64 (%r8), %zmm18
+; AVX512F-NEXT:    vmovdqa64 64(%r8), %zmm16
+; AVX512F-NEXT:    vmovdqa64 (%r9), %zmm12
+; AVX512F-NEXT:    vmovdqa64 64(%r9), %zmm19
+; AVX512F-NEXT:    vmovdqa64 (%r10), %zmm25
+; AVX512F-NEXT:    vmovdqa64 64(%r10), %zmm17
+; AVX512F-NEXT:    vmovdqa64 (%rax), %zmm15
+; AVX512F-NEXT:    vmovdqa64 64(%rax), %zmm20
 ; AVX512F-NEXT:    movb $-64, %r11b
 ; AVX512F-NEXT:    kmovw %r11d, %k1
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm15 = [4,12,4,12,4,12,4,12]
-; AVX512F-NEXT:    # zmm15 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm17, %zmm9
-; AVX512F-NEXT:    vpermt2q %zmm24, %zmm15, %zmm9
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm10 = zmm23[0],zmm18[0],zmm23[2],zmm18[2],zmm23[4],zmm18[4],zmm23[6],zmm18[6]
-; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm10 {%k1}
-; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm9
-; AVX512F-NEXT:    vpermt2q %zmm0, %zmm15, %zmm9
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm12 = [4,12,4,12]
-; AVX512F-NEXT:    # ymm12 = mem[0,1,0,1]
-; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm11
-; AVX512F-NEXT:    vpermt2q %zmm7, %zmm12, %zmm11
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3],ymm11[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm9, %zmm10, %zmm9
-; AVX512F-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm8 = [5,13,5,13,5,13,5,13]
-; AVX512F-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm17, %zmm9
-; AVX512F-NEXT:    vpermt2q %zmm24, %zmm8, %zmm9
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm10 = zmm23[1],zmm18[1],zmm23[3],zmm18[3],zmm23[5],zmm18[5],zmm23[7],zmm18[7]
-; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm10 {%k1}
-; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm9
-; AVX512F-NEXT:    vpermt2q %zmm0, %zmm8, %zmm9
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm14 = [5,13,5,13]
-; AVX512F-NEXT:    # ymm14 = mem[0,1,0,1]
-; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm11
-; AVX512F-NEXT:    vpermt2q %zmm7, %zmm14, %zmm11
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3],ymm11[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm9, %zmm10, %zmm9
-; AVX512F-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm3 = [6,14,6,14,6,14,6,14]
-; AVX512F-NEXT:    # zmm3 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm23, %zmm9
-; AVX512F-NEXT:    vpermt2q %zmm18, %zmm3, %zmm9
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm9 {%k1} = zmm17[0],zmm24[0],zmm17[2],zmm24[2],zmm17[4],zmm24[4],zmm17[6],zmm24[6]
-; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm10
-; AVX512F-NEXT:    vpermt2q %zmm0, %zmm3, %zmm10
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm4 = [6,14,6,14]
-; AVX512F-NEXT:    # ymm4 = mem[0,1,0,1]
-; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm11
-; AVX512F-NEXT:    vpermt2q %zmm7, %zmm4, %zmm11
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm10 = ymm10[0,1,2,3],ymm11[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm10, %zmm9, %zmm9
-; AVX512F-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm29 = [7,15,7,15,7,15,7,15]
-; AVX512F-NEXT:    # zmm29 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm23, %zmm9
-; AVX512F-NEXT:    vpermt2q %zmm18, %zmm29, %zmm9
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm9 {%k1} = zmm17[1],zmm24[1],zmm17[3],zmm24[3],zmm17[5],zmm24[5],zmm17[7],zmm24[7]
-; AVX512F-NEXT:    vpermt2q %zmm0, %zmm29, %zmm5
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = [7,15,7,15]
-; AVX512F-NEXT:    # ymm0 = mem[0,1,0,1]
-; AVX512F-NEXT:    vpermt2q %zmm7, %zmm0, %zmm6
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm6[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm5, %zmm9, %zmm5
-; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm5
-; AVX512F-NEXT:    vpermt2q %zmm22, %zmm15, %zmm5
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm6 = zmm19[0],zmm28[0],zmm19[2],zmm28[2],zmm19[4],zmm28[4],zmm19[6],zmm28[6]
-; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm6 {%k1}
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm5
-; AVX512F-NEXT:    vpermt2q %zmm20, %zmm15, %zmm5
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm7
-; AVX512F-NEXT:    vpermt2q %zmm21, %zmm12, %zmm7
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm7[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm5, %zmm6, %zmm5
-; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm5
-; AVX512F-NEXT:    vpermt2q %zmm22, %zmm8, %zmm5
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm6 = zmm19[1],zmm28[1],zmm19[3],zmm28[3],zmm19[5],zmm28[5],zmm19[7],zmm28[7]
-; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm6 {%k1}
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm5
-; AVX512F-NEXT:    vpermt2q %zmm20, %zmm8, %zmm5
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm7
-; AVX512F-NEXT:    vpermt2q %zmm21, %zmm14, %zmm7
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm7[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm5, %zmm6, %zmm5
-; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm5
-; AVX512F-NEXT:    vpermt2q %zmm20, %zmm3, %zmm5
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm6
-; AVX512F-NEXT:    vpermt2q %zmm21, %zmm4, %zmm6
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm6[4,5,6,7]
-; AVX512F-NEXT:    vmovdqa64 %zmm19, %zmm6
-; AVX512F-NEXT:    vpermt2q %zmm28, %zmm3, %zmm6
-; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm7
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm6 {%k1} = zmm16[0],zmm22[0],zmm16[2],zmm22[2],zmm16[4],zmm22[4],zmm16[6],zmm22[6]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm5, %zmm6, %zmm3
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm27 = [4,12,4,12,4,12,4,12]
+; AVX512F-NEXT:    # zmm27 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm10
+; AVX512F-NEXT:    vpermt2q %zmm8, %zmm27, %zmm10
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm11
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm27, %zmm11
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm10 = ymm11[0,1,2,3],ymm10[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 %zmm25, %zmm11
+; AVX512F-NEXT:    vpermt2q %zmm15, %zmm27, %zmm11
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm13 = zmm18[0],zmm12[0],zmm18[2],zmm12[2],zmm18[4],zmm12[4],zmm18[6],zmm12[6]
+; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm13 {%k1}
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm10, %zmm13, %zmm10
+; AVX512F-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm24 = [5,13,5,13,5,13,5,13]
+; AVX512F-NEXT:    # zmm24 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm10
+; AVX512F-NEXT:    vpermt2q %zmm8, %zmm24, %zmm10
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm11
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm24, %zmm11
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm10 = ymm11[0,1,2,3],ymm10[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 %zmm25, %zmm11
+; AVX512F-NEXT:    vpermt2q %zmm15, %zmm24, %zmm11
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm14 = zmm18[1],zmm12[1],zmm18[3],zmm12[3],zmm18[5],zmm12[5],zmm18[7],zmm12[7]
+; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm14 {%k1}
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm10, %zmm14, %zmm10
+; AVX512F-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm22 = [6,14,6,14,6,14,6,14]
+; AVX512F-NEXT:    # zmm22 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm10
+; AVX512F-NEXT:    vpermt2q %zmm8, %zmm22, %zmm10
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm11
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm22, %zmm11
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm10 = ymm11[0,1,2,3],ymm10[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 %zmm18, %zmm11
+; AVX512F-NEXT:    vpermt2q %zmm12, %zmm22, %zmm11
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm11 {%k1} = zmm25[0],zmm15[0],zmm25[2],zmm15[2],zmm25[4],zmm15[4],zmm25[6],zmm15[6]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm10, %zmm11, %zmm10
+; AVX512F-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm21 = [7,15,7,15,7,15,7,15]
+; AVX512F-NEXT:    # zmm21 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpermt2q %zmm8, %zmm21, %zmm4
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm21, %zmm2
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm4[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 %zmm18, %zmm3
+; AVX512F-NEXT:    vpermt2q %zmm12, %zmm21, %zmm3
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm3 {%k1} = zmm25[1],zmm15[1],zmm25[3],zmm15[3],zmm25[5],zmm15[5],zmm25[7],zmm15[7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm3, %zmm2
+; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm2
+; AVX512F-NEXT:    vpermt2q %zmm9, %zmm27, %zmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm3
+; AVX512F-NEXT:    vpermt2q %zmm5, %zmm27, %zmm3
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 %zmm17, %zmm3
+; AVX512F-NEXT:    vpermt2q %zmm20, %zmm27, %zmm3
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm4 = zmm16[0],zmm19[0],zmm16[2],zmm19[2],zmm16[4],zmm19[4],zmm16[6],zmm19[6]
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm4 {%k1}
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm4, %zmm2
+; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm2
+; AVX512F-NEXT:    vpermt2q %zmm9, %zmm24, %zmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm3
+; AVX512F-NEXT:    vpermt2q %zmm5, %zmm24, %zmm3
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 %zmm17, %zmm3
+; AVX512F-NEXT:    vpermt2q %zmm20, %zmm24, %zmm3
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm4 = zmm16[1],zmm19[1],zmm16[3],zmm19[3],zmm16[5],zmm19[5],zmm16[7],zmm19[7]
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm4 {%k1}
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm4, %zmm2
+; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm2
+; AVX512F-NEXT:    vpermt2q %zmm9, %zmm22, %zmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm3
+; AVX512F-NEXT:    vpermt2q %zmm5, %zmm22, %zmm3
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm3
+; AVX512F-NEXT:    vpermt2q %zmm19, %zmm22, %zmm3
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm3 {%k1} = zmm17[0],zmm20[0],zmm17[2],zmm20[2],zmm17[4],zmm20[4],zmm17[6],zmm20[6]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm3, %zmm2
+; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm9, %zmm21, %zmm7
+; AVX512F-NEXT:    vpermt2q %zmm5, %zmm21, %zmm6
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm6[0,1,2,3],ymm7[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm3
+; AVX512F-NEXT:    vpermt2q %zmm19, %zmm21, %zmm3
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm3 {%k1} = zmm17[1],zmm20[1],zmm17[3],zmm20[3],zmm17[5],zmm20[5],zmm17[7],zmm20[7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm3, %zmm2
+; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm2
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm27, %zmm2
+; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm4
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm27, %zmm4
+; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm4
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm24, %zmm4
+; AVX512F-NEXT:    vmovdqu64 %zmm4, (%rsp) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm4
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm24, %zmm4
+; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm4
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm22, %zmm4
+; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm4
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm22, %zmm4
+; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm21, %zmm3
 ; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 128(%rcx), %zmm5
-; AVX512F-NEXT:    vpermt2q %zmm20, %zmm29, %zmm1
-; AVX512F-NEXT:    vmovdqa64 %zmm19, %zmm3
-; AVX512F-NEXT:    vpermt2q %zmm28, %zmm29, %zmm3
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm3 {%k1} = zmm16[1],zmm22[1],zmm16[3],zmm22[3],zmm16[5],zmm22[5],zmm16[7],zmm22[7]
-; AVX512F-NEXT:    vpermt2q %zmm21, %zmm0, %zmm2
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm3, %zmm1
-; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm1
-; AVX512F-NEXT:    vpermt2q %zmm5, %zmm12, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm21, %zmm2
+; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm23, %zmm27, %zmm1
 ; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm1
-; AVX512F-NEXT:    vpermt2q %zmm5, %zmm14, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm23, %zmm24, %zmm1
 ; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm1
-; AVX512F-NEXT:    vpermt2q %zmm5, %zmm4, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm23, %zmm22, %zmm1
 ; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm5, %zmm0, %zmm13
-; AVX512F-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 192(%rcx), %zmm1
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512F-NEXT:    vpermi2q %zmm1, %zmm13, %zmm12
-; AVX512F-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermi2q %zmm1, %zmm13, %zmm14
-; AVX512F-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermi2q %zmm1, %zmm13, %zmm4
-; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm0, %zmm13
-; AVX512F-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm25, %zmm15, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm23, %zmm21, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm25, %zmm8, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm25, %zmm7, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm25, %zmm29, %zmm1
+; AVX512F-NEXT:    vmovdqa64 192(%rdi), %zmm1
+; AVX512F-NEXT:    vmovdqa64 192(%rsi), %zmm0
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm2
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm27, %zmm2
+; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm2
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm24, %zmm2
+; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm2
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm22, %zmm2
+; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm21, %zmm1
 ; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 128(%r10), %zmm30
-; AVX512F-NEXT:    vmovdqa64 128(%rax), %zmm6
-; AVX512F-NEXT:    vmovdqa64 %zmm30, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm6, %zmm15, %zmm0
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm30 = [0,8,0,8,0,8,0,8]
+; AVX512F-NEXT:    # zmm30 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm25, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm15, %zmm30, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm30, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm6, %zmm8, %zmm0
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm31 = [1,9,1,9,1,9,1,9]
+; AVX512F-NEXT:    # zmm31 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm25, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm15, %zmm31, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 128(%r8), %zmm21
-; AVX512F-NEXT:    vmovdqa64 128(%r9), %zmm9
-; AVX512F-NEXT:    vmovdqa64 %zmm21, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm9, %zmm7, %zmm0
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm10
-; AVX512F-NEXT:    vmovdqa64 192(%r10), %zmm14
-; AVX512F-NEXT:    vmovdqa64 192(%rax), %zmm4
-; AVX512F-NEXT:    vmovdqa64 %zmm14, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm4, %zmm15, %zmm0
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm23 = [2,10,2,10,2,10,2,10]
+; AVX512F-NEXT:    # zmm23 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm25, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm15, %zmm23, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 192(%rdi), %zmm1
-; AVX512F-NEXT:    vmovdqa64 192(%rsi), %zmm0
-; AVX512F-NEXT:    vpermi2q %zmm0, %zmm1, %zmm15
-; AVX512F-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm14, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm4, %zmm8, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermi2q %zmm0, %zmm1, %zmm8
-; AVX512F-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 192(%r8), %zmm31
-; AVX512F-NEXT:    vmovdqa64 192(%r9), %zmm3
-; AVX512F-NEXT:    vmovdqa64 %zmm31, %zmm27
-; AVX512F-NEXT:    vpermt2q %zmm3, %zmm7, %zmm27
-; AVX512F-NEXT:    vpermi2q %zmm0, %zmm1, %zmm7
-; AVX512F-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm0, %zmm29, %zmm1
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm26 = [3,11,3,11,3,11,3,11]
+; AVX512F-NEXT:    # zmm26 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpermt2q %zmm15, %zmm26, %zmm25
+; AVX512F-NEXT:    vmovdqu64 %zmm25, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm18, %zmm0
+; AVX512F-NEXT:    vmovdqa64 %zmm18, %zmm29
+; AVX512F-NEXT:    vpermt2q %zmm12, %zmm30, %zmm29
+; AVX512F-NEXT:    vpermt2q %zmm12, %zmm31, %zmm18
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm12, %zmm23, %zmm1
 ; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm21, %zmm20
-; AVX512F-NEXT:    vpermt2q %zmm9, %zmm29, %zmm20
-; AVX512F-NEXT:    vmovdqa64 %zmm31, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm3, %zmm29, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm12, %zmm26, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm2 = [0,8,0,8,0,8,0,8]
-; AVX512F-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; AVX512F-NEXT:    vmovdqa64 %zmm17, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm24, %zmm2, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm20, %zmm30, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm1 = [1,9,1,9,1,9,1,9]
-; AVX512F-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; AVX512F-NEXT:    vmovdqa64 %zmm17, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm24, %zmm1, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, (%rsp) # 64-byte Spill
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm29 = [2,10,2,10,2,10,2,10]
-; AVX512F-NEXT:    # zmm29 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpermt2q %zmm20, %zmm31, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vmovdqa64 %zmm17, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm24, %zmm29, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm20, %zmm23, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [3,11,3,11,3,11,3,11]
-; AVX512F-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermt2q %zmm24, %zmm0, %zmm17
+; AVX512F-NEXT:    vpermt2q %zmm20, %zmm26, %zmm17
 ; AVX512F-NEXT:    vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm23, %zmm13
-; AVX512F-NEXT:    vpermt2q %zmm18, %zmm2, %zmm13
-; AVX512F-NEXT:    vmovdqa64 %zmm23, %zmm12
-; AVX512F-NEXT:    vpermt2q %zmm18, %zmm1, %zmm12
-; AVX512F-NEXT:    vmovdqa64 %zmm23, %zmm24
-; AVX512F-NEXT:    vpermt2q %zmm18, %zmm29, %zmm24
-; AVX512F-NEXT:    vpermt2q %zmm18, %zmm0, %zmm23
-; AVX512F-NEXT:    vmovdqu64 %zmm23, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm26
-; AVX512F-NEXT:    vpermt2q %zmm22, %zmm2, %zmm26
-; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm23
-; AVX512F-NEXT:    vpermt2q %zmm22, %zmm1, %zmm23
-; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm5
-; AVX512F-NEXT:    vpermt2q %zmm22, %zmm29, %zmm5
-; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm22, %zmm0, %zmm16
+; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm14
+; AVX512F-NEXT:    vpermt2q %zmm19, %zmm30, %zmm14
+; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm11
+; AVX512F-NEXT:    vpermt2q %zmm19, %zmm31, %zmm11
+; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm25
+; AVX512F-NEXT:    vpermt2q %zmm19, %zmm23, %zmm25
+; AVX512F-NEXT:    vpermt2q %zmm19, %zmm26, %zmm16
 ; AVX512F-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm19, %zmm5
-; AVX512F-NEXT:    vpermt2q %zmm28, %zmm2, %zmm19
-; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm18
-; AVX512F-NEXT:    vpermt2q %zmm28, %zmm1, %zmm18
-; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm22
-; AVX512F-NEXT:    vpermt2q %zmm28, %zmm29, %zmm22
-; AVX512F-NEXT:    vpermt2q %zmm28, %zmm0, %zmm5
-; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm30, %zmm15
-; AVX512F-NEXT:    vpermt2q %zmm6, %zmm2, %zmm15
-; AVX512F-NEXT:    vmovdqa64 %zmm30, %zmm16
-; AVX512F-NEXT:    vpermt2q %zmm6, %zmm1, %zmm16
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm10 {%k1} = zmm30[0],zmm6[0],zmm30[2],zmm6[2],zmm30[4],zmm6[4],zmm30[6],zmm6[6]
-; AVX512F-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm20 {%k1} = zmm30[1],zmm6[1],zmm30[3],zmm6[3],zmm30[5],zmm6[5],zmm30[7],zmm6[7]
-; AVX512F-NEXT:    vmovdqa64 %zmm30, %zmm5
-; AVX512F-NEXT:    vpermt2q %zmm6, %zmm29, %zmm5
-; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm6, %zmm0, %zmm30
-; AVX512F-NEXT:    vmovdqa64 %zmm21, %zmm17
-; AVX512F-NEXT:    vpermt2q %zmm9, %zmm2, %zmm17
-; AVX512F-NEXT:    vmovdqa64 %zmm21, %zmm5
-; AVX512F-NEXT:    vpermt2q %zmm9, %zmm1, %zmm5
-; AVX512F-NEXT:    vmovdqa64 %zmm21, %zmm11
-; AVX512F-NEXT:    vpermt2q %zmm9, %zmm29, %zmm11
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm7 = zmm21[0],zmm9[0],zmm21[2],zmm9[2],zmm21[4],zmm9[4],zmm21[6],zmm9[6]
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm8 = zmm21[1],zmm9[1],zmm21[3],zmm9[3],zmm21[5],zmm9[5],zmm21[7],zmm9[7]
-; AVX512F-NEXT:    vpermt2q %zmm9, %zmm0, %zmm21
-; AVX512F-NEXT:    vmovdqa64 %zmm14, %zmm10
-; AVX512F-NEXT:    vpermt2q %zmm4, %zmm2, %zmm10
-; AVX512F-NEXT:    vmovdqa64 %zmm14, %zmm25
-; AVX512F-NEXT:    vpermt2q %zmm4, %zmm1, %zmm25
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm27 {%k1} = zmm14[0],zmm4[0],zmm14[2],zmm4[2],zmm14[4],zmm4[4],zmm14[6],zmm4[6]
-; AVX512F-NEXT:    vmovdqu64 %zmm27, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm9 {%k1} = zmm14[1],zmm4[1],zmm14[3],zmm4[3],zmm14[5],zmm4[5],zmm14[7],zmm4[7]
-; AVX512F-NEXT:    vmovdqa64 %zmm14, %zmm27
-; AVX512F-NEXT:    vpermt2q %zmm4, %zmm29, %zmm27
-; AVX512F-NEXT:    vpermt2q %zmm4, %zmm0, %zmm14
-; AVX512F-NEXT:    vpermi2q %zmm3, %zmm31, %zmm2
-; AVX512F-NEXT:    vpermi2q %zmm3, %zmm31, %zmm1
-; AVX512F-NEXT:    vpermi2q %zmm3, %zmm31, %zmm29
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm4 = zmm31[0],zmm3[0],zmm31[2],zmm3[2],zmm31[4],zmm3[4],zmm31[6],zmm3[6]
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm6 = zmm31[1],zmm3[1],zmm31[3],zmm3[3],zmm31[5],zmm3[5],zmm31[7],zmm3[7]
-; AVX512F-NEXT:    vpermt2q %zmm3, %zmm0, %zmm31
+; AVX512F-NEXT:    vmovdqa64 128(%r10), %zmm19
+; AVX512F-NEXT:    vmovdqa64 128(%rax), %zmm0
+; AVX512F-NEXT:    vmovdqa64 %zmm19, %zmm28
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm30, %zmm28
+; AVX512F-NEXT:    vmovdqa64 %zmm19, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm31, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm19, %zmm13
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm27, %zmm13
+; AVX512F-NEXT:    vmovdqa64 %zmm19, %zmm16
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm24, %zmm16
+; AVX512F-NEXT:    vmovdqa64 128(%r8), %zmm20
+; AVX512F-NEXT:    vmovdqa64 128(%r9), %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm20, %zmm3
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm22, %zmm3
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm3 {%k1} = zmm19[0],zmm0[0],zmm19[2],zmm0[2],zmm19[4],zmm0[4],zmm19[6],zmm0[6]
+; AVX512F-NEXT:    vmovdqa64 %zmm20, %zmm10
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm21, %zmm10
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm10 {%k1} = zmm19[1],zmm0[1],zmm19[3],zmm0[3],zmm19[5],zmm0[5],zmm19[7],zmm0[7]
+; AVX512F-NEXT:    vmovdqa64 %zmm19, %zmm2
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm23, %zmm2
+; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm26, %zmm19
+; AVX512F-NEXT:    vmovdqa64 %zmm20, %zmm4
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm30, %zmm4
+; AVX512F-NEXT:    vmovdqa64 %zmm20, %zmm6
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm31, %zmm6
+; AVX512F-NEXT:    vmovdqa64 %zmm20, %zmm17
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm23, %zmm17
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm5 = zmm20[0],zmm1[0],zmm20[2],zmm1[2],zmm20[4],zmm1[4],zmm20[6],zmm1[6]
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm7 = zmm20[1],zmm1[1],zmm20[3],zmm1[3],zmm20[5],zmm1[5],zmm20[7],zmm1[7]
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm26, %zmm20
+; AVX512F-NEXT:    vmovdqa64 192(%r10), %zmm8
+; AVX512F-NEXT:    vmovdqa64 192(%rax), %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm12
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm30, %zmm12
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm15
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm31, %zmm15
+; AVX512F-NEXT:    vpermi2q %zmm1, %zmm8, %zmm27
+; AVX512F-NEXT:    vpermi2q %zmm1, %zmm8, %zmm24
+; AVX512F-NEXT:    vmovdqa64 192(%r8), %zmm2
+; AVX512F-NEXT:    vmovdqa64 192(%r9), %zmm0
+; AVX512F-NEXT:    vpermi2q %zmm0, %zmm2, %zmm22
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm22 {%k1} = zmm8[0],zmm1[0],zmm8[2],zmm1[2],zmm8[4],zmm1[4],zmm8[6],zmm1[6]
+; AVX512F-NEXT:    vpermi2q %zmm0, %zmm2, %zmm21
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm21 {%k1} = zmm8[1],zmm1[1],zmm8[3],zmm1[3],zmm8[5],zmm1[5],zmm8[7],zmm1[7]
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm9
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm23, %zmm9
+; AVX512F-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm26, %zmm8
+; AVX512F-NEXT:    vpermi2q %zmm0, %zmm2, %zmm30
+; AVX512F-NEXT:    vpermi2q %zmm0, %zmm2, %zmm31
+; AVX512F-NEXT:    vpermi2q %zmm0, %zmm2, %zmm23
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm1 = zmm2[0],zmm0[0],zmm2[2],zmm0[2],zmm2[4],zmm0[4],zmm2[6],zmm0[6]
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm9 = zmm2[1],zmm0[1],zmm2[3],zmm0[3],zmm2[5],zmm0[5],zmm2[7],zmm0[7]
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm26, %zmm2
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm13 {%k1}
-; AVX512F-NEXT:    vmovdqa (%rsi), %xmm0
-; AVX512F-NEXT:    vinserti128 $1, (%rcx), %ymm0, %ymm0
-; AVX512F-NEXT:    vmovdqa (%rdi), %xmm3
-; AVX512F-NEXT:    vinserti128 $1, (%rdx), %ymm3, %ymm3
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm28 = ymm3[0],ymm0[0],ymm3[2],ymm0[2]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm28, %zmm13, %zmm13
-; AVX512F-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 (%rsp), %zmm13 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm12 {%k1}
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm0 = ymm3[1],ymm0[1],ymm3[3],ymm0[3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm12, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, (%rsp) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm26, %zmm19 {%k1}
-; AVX512F-NEXT:    vmovdqa 64(%rsi), %xmm0
-; AVX512F-NEXT:    vinserti128 $1, 64(%rcx), %ymm0, %ymm0
-; AVX512F-NEXT:    vmovdqa 64(%rdi), %xmm12
-; AVX512F-NEXT:    vinserti128 $1, 64(%rdx), %ymm12, %ymm12
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm13 = ymm12[0],ymm0[0],ymm12[2],ymm0[2]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm13, %zmm19, %zmm3
-; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm23, %zmm18 {%k1}
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm0 = ymm12[1],ymm0[1],ymm12[3],ymm0[3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm18, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm17 {%k1}
-; AVX512F-NEXT:    vmovdqa 128(%rsi), %xmm0
-; AVX512F-NEXT:    vinserti128 $1, 128(%rcx), %ymm0, %ymm0
-; AVX512F-NEXT:    vmovdqa 128(%rdi), %xmm12
-; AVX512F-NEXT:    vinserti128 $1, 128(%rdx), %ymm12, %ymm13
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm12 = ymm13[0],ymm0[0],ymm13[2],ymm0[2]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm12, %zmm17, %zmm3
-; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm5 {%k1}
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm0 = ymm13[1],ymm0[1],ymm13[3],ymm0[3]
+; AVX512F-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; AVX512F-NEXT:    # ymm0 = ymm0[0,1,2,3],mem[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm5 {%k1}
 ; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm5, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm7 {%k1}
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm5 # 32-byte Folded Reload
-; AVX512F-NEXT:    # ymm5 = ymm0[0,1,2,3],mem[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm5, %zmm7, %zmm28
+; AVX512F-NEXT:    vpblendd $240, (%rsp), %ymm0, %ymm0 # 32-byte Folded Reload
+; AVX512F-NEXT:    # ymm0 = ymm0[0,1,2,3],mem[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm7 {%k1}
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm7, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm8 {%k1}
+; AVX512F-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; AVX512F-NEXT:    # ymm0 = ymm0[0,1,2,3],mem[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm3, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm5 # 32-byte Folded Reload
-; AVX512F-NEXT:    # ymm5 = ymm0[0,1,2,3],mem[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm5, %zmm8, %zmm26
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm5 # 32-byte Folded Reload
-; AVX512F-NEXT:    # ymm5 = ymm0[0,1,2,3],mem[4,5,6,7]
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vinsertf64x4 $0, %ymm5, %zmm0, %zmm19
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm29 {%k1}
+; AVX512F-NEXT:    vmovdqa (%rsi), %xmm0
+; AVX512F-NEXT:    vinserti128 $1, (%rcx), %ymm0, %ymm3
+; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512F-NEXT:    vinserti128 $1, (%rdx), %ymm0, %ymm5
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm5[0],ymm3[0],ymm5[2],ymm3[2]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm29, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm5 # 32-byte Folded Reload
-; AVX512F-NEXT:    # ymm5 = ymm0[0,1,2,3],mem[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm5, %zmm20, %zmm18
-; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm2 {%k1}
-; AVX512F-NEXT:    vmovdqa 192(%rsi), %xmm7
-; AVX512F-NEXT:    vinserti128 $1, 192(%rcx), %ymm7, %ymm7
-; AVX512F-NEXT:    vmovdqa 192(%rdi), %xmm10
-; AVX512F-NEXT:    vinserti128 $1, 192(%rdx), %ymm10, %ymm10
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm15 = ymm10[0],ymm7[0],ymm10[2],ymm7[2]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm15, %zmm2, %zmm17
-; AVX512F-NEXT:    vmovdqa64 %zmm25, %zmm1 {%k1}
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm7 = ymm10[1],ymm7[1],ymm10[3],ymm7[3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm7, %zmm1, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm18 {%k1}
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm3 = ymm5[1],ymm3[1],ymm5[3],ymm3[3]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm3, %zmm18, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, (%rsp) # 64-byte Spill
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm4 {%k1}
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm14 {%k1}
+; AVX512F-NEXT:    vmovdqa 64(%rsi), %xmm3
+; AVX512F-NEXT:    vinserti128 $1, 64(%rcx), %ymm3, %ymm3
+; AVX512F-NEXT:    vmovdqa 64(%rdi), %xmm5
+; AVX512F-NEXT:    vinserti128 $1, 64(%rdx), %ymm5, %ymm5
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm7 = ymm5[0],ymm3[0],ymm5[2],ymm3[2]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm7, %zmm14, %zmm29
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm7 # 32-byte Folded Reload
-; AVX512F-NEXT:    # ymm7 = ymm0[0,1,2,3],mem[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm7, %zmm4, %zmm7
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm11 {%k1}
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm3 = ymm5[1],ymm3[1],ymm5[3],ymm3[3]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm3, %zmm11, %zmm26
+; AVX512F-NEXT:    vmovdqa64 %zmm28, %zmm4 {%k1}
+; AVX512F-NEXT:    vmovdqa 128(%rsi), %xmm3
+; AVX512F-NEXT:    vinserti128 $1, 128(%rcx), %ymm3, %ymm3
+; AVX512F-NEXT:    vmovdqa 128(%rdi), %xmm5
+; AVX512F-NEXT:    vinserti128 $1, 128(%rdx), %ymm5, %ymm5
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm7 = ymm5[0],ymm3[0],ymm5[2],ymm3[2]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm7, %zmm4, %zmm28
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm6 {%k1}
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm3 = ymm5[1],ymm3[1],ymm5[3],ymm3[3]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm3, %zmm6, %zmm18
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm4 # 32-byte Folded Reload
 ; AVX512F-NEXT:    # ymm4 = ymm0[0,1,2,3],mem[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm4, %zmm6, %zmm4
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm6 # 32-byte Folded Reload
-; AVX512F-NEXT:    # ymm6 = ymm0[0,1,2,3],mem[4,5,6,7]
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vinsertf64x4 $0, %ymm6, %zmm0, %zmm10
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm4, %zmm10, %zmm16
+; AVX512F-NEXT:    vmovdqa64 %zmm12, %zmm30 {%k1}
+; AVX512F-NEXT:    vmovdqa 192(%rsi), %xmm4
+; AVX512F-NEXT:    vinserti128 $1, 192(%rcx), %ymm4, %ymm5
+; AVX512F-NEXT:    vmovdqa 192(%rdi), %xmm4
+; AVX512F-NEXT:    vinserti128 $1, 192(%rdx), %ymm4, %ymm6
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm4 = ymm6[0],ymm5[0],ymm6[2],ymm5[2]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm4, %zmm30, %zmm30
+; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm31 {%k1}
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm5 = ymm6[1],ymm5[1],ymm6[3],ymm5[3]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm5, %zmm31, %zmm31
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm6 # 32-byte Folded Reload
 ; AVX512F-NEXT:    # ymm6 = ymm0[0,1,2,3],mem[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm6, %zmm9, %zmm15
+; AVX512F-NEXT:    vmovdqa64 %zmm27, %zmm1 {%k1}
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm6, %zmm1, %zmm6
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm24 {%k1}
-; AVX512F-NEXT:    vmovdqa (%rcx), %ymm6
-; AVX512F-NEXT:    vmovdqa64 (%rdx), %ymm16
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm3 = ymm16[0],ymm6[0],ymm16[2],ymm6[2]
-; AVX512F-NEXT:    vmovdqa64 (%rsi), %ymm20
-; AVX512F-NEXT:    vmovdqa64 (%rdi), %ymm23
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm12 = ymm23[0],ymm20[0],ymm23[2],ymm20[2]
-; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm3 = ymm12[2,3],ymm3[2,3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm3, %zmm24, %zmm3
+; AVX512F-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 32-byte Folded Reload
+; AVX512F-NEXT:    # ymm1 = ymm0[0,1,2,3],mem[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 %zmm24, %zmm9 {%k1}
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm9, %zmm7
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k1}
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm6 = ymm16[1],ymm6[1],ymm16[3],ymm6[3]
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm12 = ymm23[1],ymm20[1],ymm23[3],ymm20[3]
-; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm6 = ymm12[2,3],ymm6[2,3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm6, %zmm2, %zmm6
+; AVX512F-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 32-byte Folded Reload
+; AVX512F-NEXT:    # ymm1 = ymm0[0,1,2,3],mem[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm22, %zmm9
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm22 {%k1}
-; AVX512F-NEXT:    vmovdqa 64(%rcx), %ymm12
-; AVX512F-NEXT:    vmovdqa64 64(%rdx), %ymm16
-; AVX512F-NEXT:    vmovdqa64 64(%rsi), %ymm20
-; AVX512F-NEXT:    vmovdqa64 64(%rdi), %ymm23
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm16[0],ymm12[0],ymm16[2],ymm12[2]
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm13 = ymm23[0],ymm20[0],ymm23[2],ymm20[2]
-; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm13[2,3],ymm0[2,3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm22, %zmm0
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm5 {%k1}
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm12 = ymm16[1],ymm12[1],ymm16[3],ymm12[3]
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm13 = ymm23[1],ymm20[1],ymm23[3],ymm20[3]
-; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm12 = ymm13[2,3],ymm12[2,3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm12, %zmm5, %zmm12
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm11 {%k1}
-; AVX512F-NEXT:    vmovdqa 128(%rcx), %ymm13
-; AVX512F-NEXT:    vmovdqa64 128(%rdx), %ymm16
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm5 = ymm16[0],ymm13[0],ymm16[2],ymm13[2]
-; AVX512F-NEXT:    vmovdqa64 128(%rsi), %ymm20
+; AVX512F-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 32-byte Folded Reload
+; AVX512F-NEXT:    # ymm1 = ymm0[0,1,2,3],mem[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm21, %zmm12
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm3 {%k1}
+; AVX512F-NEXT:    vmovdqa (%rcx), %ymm1
+; AVX512F-NEXT:    vmovdqa (%rdx), %ymm15
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm15[0],ymm1[0],ymm15[2],ymm1[2]
+; AVX512F-NEXT:    vmovdqa64 (%rsi), %ymm21
+; AVX512F-NEXT:    vmovdqa64 (%rdi), %ymm22
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm14 = ymm22[0],ymm21[0],ymm22[2],ymm21[2]
+; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm14[2,3],ymm0[2,3]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm3, %zmm0
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm4 {%k1}
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm15[1],ymm1[1],ymm15[3],ymm1[3]
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm14 = ymm22[1],ymm21[1],ymm22[3],ymm21[3]
+; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm1 = ymm14[2,3],ymm1[2,3]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm4, %zmm1
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm25 {%k1}
+; AVX512F-NEXT:    vmovdqa 64(%rcx), %ymm14
+; AVX512F-NEXT:    vmovdqa 64(%rdx), %ymm15
+; AVX512F-NEXT:    vmovdqa64 64(%rsi), %ymm21
+; AVX512F-NEXT:    vmovdqa64 64(%rdi), %ymm22
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm13 = ymm15[0],ymm14[0],ymm15[2],ymm14[2]
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm11 = ymm22[0],ymm21[0],ymm22[2],ymm21[2]
+; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm11 = ymm11[2,3],ymm13[2,3]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm11, %zmm25, %zmm11
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm4 {%k1}
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm13 = ymm15[1],ymm14[1],ymm15[3],ymm14[3]
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm14 = ymm22[1],ymm21[1],ymm22[3],ymm21[3]
+; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm13 = ymm14[2,3],ymm13[2,3]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm13, %zmm4, %zmm13
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm17 {%k1}
+; AVX512F-NEXT:    vmovdqa 128(%rcx), %ymm14
+; AVX512F-NEXT:    vmovdqa 128(%rdx), %ymm15
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm4 = ymm15[0],ymm14[0],ymm15[2],ymm14[2]
+; AVX512F-NEXT:    vmovdqa64 128(%rsi), %ymm21
 ; AVX512F-NEXT:    vmovdqa64 128(%rdi), %ymm22
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm8 = ymm22[0],ymm20[0],ymm22[2],ymm20[2]
-; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm5 = ymm8[2,3],ymm5[2,3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm5, %zmm11, %zmm5
-; AVX512F-NEXT:    vmovdqa64 %zmm30, %zmm21 {%k1}
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm8 = ymm16[1],ymm13[1],ymm16[3],ymm13[3]
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm11 = ymm22[1],ymm20[1],ymm22[3],ymm20[3]
-; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm8 = ymm11[2,3],ymm8[2,3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm8, %zmm21, %zmm8
-; AVX512F-NEXT:    vmovdqa64 %zmm27, %zmm29 {%k1}
-; AVX512F-NEXT:    vmovdqa 192(%rcx), %ymm9
-; AVX512F-NEXT:    vmovdqa 192(%rdx), %ymm11
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm13 = ymm11[0],ymm9[0],ymm11[2],ymm9[2]
-; AVX512F-NEXT:    vmovdqa64 192(%rsi), %ymm16
-; AVX512F-NEXT:    vmovdqa64 192(%rdi), %ymm20
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm2 = ymm20[0],ymm16[0],ymm20[2],ymm16[2]
-; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm2 = ymm2[2,3],ymm13[2,3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm29, %zmm2
-; AVX512F-NEXT:    vmovdqa64 %zmm14, %zmm31 {%k1}
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm9 = ymm11[1],ymm9[1],ymm11[3],ymm9[3]
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm11 = ymm20[1],ymm16[1],ymm20[3],ymm16[3]
-; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm9 = ymm11[2,3],ymm9[2,3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm9, %zmm31, %zmm9
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm3 = ymm22[0],ymm21[0],ymm22[2],ymm21[2]
+; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm3 = ymm3[2,3],ymm4[2,3]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm3, %zmm17, %zmm3
+; AVX512F-NEXT:    vmovdqa64 %zmm19, %zmm20 {%k1}
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm4 = ymm15[1],ymm14[1],ymm15[3],ymm14[3]
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm14 = ymm22[1],ymm21[1],ymm22[3],ymm21[3]
+; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm4 = ymm14[2,3],ymm4[2,3]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm4, %zmm20, %zmm4
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm23 {%k1}
+; AVX512F-NEXT:    vmovdqa 192(%rcx), %ymm14
+; AVX512F-NEXT:    vmovdqa 192(%rdx), %ymm15
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm5 = ymm15[0],ymm14[0],ymm15[2],ymm14[2]
+; AVX512F-NEXT:    vmovdqa64 192(%rsi), %ymm17
+; AVX512F-NEXT:    vmovdqa64 192(%rdi), %ymm19
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm10 = ymm19[0],ymm17[0],ymm19[2],ymm17[2]
+; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm5 = ymm10[2,3],ymm5[2,3]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm5, %zmm23, %zmm5
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm2 {%k1}
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm8 = ymm15[1],ymm14[1],ymm15[3],ymm14[3]
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm10 = ymm19[1],ymm17[1],ymm19[3],ymm17[3]
+; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm8 = ymm10[2,3],ymm8[2,3]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm8, %zmm2, %zmm2
 ; AVX512F-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512F-NEXT:    vmovdqa64 %zmm9, 1728(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm2, 1664(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm8, 1216(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm5, 1152(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm12, 704(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm0, 640(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm6, 192(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm3, 128(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm15, 1984(%rax)
-; AVX512F-NEXT:    vmovaps %zmm10, 1920(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm4, 1856(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm7, 1792(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm1, 1600(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm17, 1536(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm18, 1472(%rax)
-; AVX512F-NEXT:    vmovaps %zmm19, 1408(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm26, 1344(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm28, 1280(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm2, 1728(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm5, 1664(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm4, 1216(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm3, 1152(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm13, 704(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm11, 640(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm1, 192(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm0, 128(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm12, 1984(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm9, 1920(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm7, 1856(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm6, 1792(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm31, 1600(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm30, 1536(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm16, 1472(%rax)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm0, 1408(%rax)
 ; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 1088(%rax)
+; AVX512F-NEXT:    vmovaps %zmm0, 1344(%rax)
 ; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 1024(%rax)
+; AVX512F-NEXT:    vmovaps %zmm0, 1280(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm18, 1088(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm28, 1024(%rax)
 ; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-NEXT:    vmovaps %zmm0, 960(%rax)
 ; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
@@ -4605,10 +4567,8 @@ define void @store_i64_stride8_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512F-NEXT:    vmovaps %zmm0, 832(%rax)
 ; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-NEXT:    vmovaps %zmm0, 768(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 576(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 512(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm26, 576(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm29, 512(%rax)
 ; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-NEXT:    vmovaps %zmm0, 448(%rax)
 ; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
@@ -4621,467 +4581,457 @@ define void @store_i64_stride8_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512F-NEXT:    vmovaps %zmm0, 64(%rax)
 ; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-NEXT:    vmovaps %zmm0, (%rax)
-; AVX512F-NEXT:    addq $2632, %rsp # imm = 0xA48
+; AVX512F-NEXT:    addq $2312, %rsp # imm = 0x908
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: store_i64_stride8_vf32:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    subq $2632, %rsp # imm = 0xA48
+; AVX512BW-NEXT:    subq $2312, %rsp # imm = 0x908
 ; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %r10
 ; AVX512BW-NEXT:    vmovaps 128(%rdi), %zmm0
 ; AVX512BW-NEXT:    vmovups %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 64(%rdi), %zmm1
-; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm5
-; AVX512BW-NEXT:    vmovdqa64 128(%rsi), %zmm25
-; AVX512BW-NEXT:    vmovdqa64 64(%rsi), %zmm20
-; AVX512BW-NEXT:    vmovdqa64 (%rsi), %zmm0
-; AVX512BW-NEXT:    vmovaps 192(%rdx), %zmm2
-; AVX512BW-NEXT:    vmovups %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 128(%rdx), %zmm13
-; AVX512BW-NEXT:    vmovdqa64 64(%rdx), %zmm2
-; AVX512BW-NEXT:    vmovdqa64 (%rdx), %zmm6
-; AVX512BW-NEXT:    vmovdqa64 64(%rcx), %zmm21
-; AVX512BW-NEXT:    vmovdqa64 (%rcx), %zmm7
-; AVX512BW-NEXT:    vmovdqa64 (%r8), %zmm23
-; AVX512BW-NEXT:    vmovdqa64 64(%r8), %zmm19
-; AVX512BW-NEXT:    vmovdqa64 (%r9), %zmm18
-; AVX512BW-NEXT:    vmovdqa64 64(%r9), %zmm28
-; AVX512BW-NEXT:    vmovdqa64 (%r10), %zmm17
-; AVX512BW-NEXT:    vmovdqa64 64(%r10), %zmm16
-; AVX512BW-NEXT:    vmovdqa64 (%rax), %zmm24
-; AVX512BW-NEXT:    vmovdqa64 64(%rax), %zmm22
+; AVX512BW-NEXT:    vmovdqa64 64(%rdi), %zmm6
+; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm2
+; AVX512BW-NEXT:    vmovdqa64 128(%rsi), %zmm0
+; AVX512BW-NEXT:    vmovdqa64 64(%rsi), %zmm5
+; AVX512BW-NEXT:    vmovdqa64 (%rsi), %zmm3
+; AVX512BW-NEXT:    vmovaps 192(%rdx), %zmm1
+; AVX512BW-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovaps 128(%rdx), %zmm1
+; AVX512BW-NEXT:    vmovups %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 64(%rdx), %zmm7
+; AVX512BW-NEXT:    vmovdqa64 (%rdx), %zmm4
+; AVX512BW-NEXT:    vmovdqa64 192(%rcx), %zmm23
+; AVX512BW-NEXT:    vmovdqa64 128(%rcx), %zmm1
+; AVX512BW-NEXT:    vmovdqa64 64(%rcx), %zmm9
+; AVX512BW-NEXT:    vmovdqa64 (%rcx), %zmm8
+; AVX512BW-NEXT:    vmovdqa64 (%r8), %zmm18
+; AVX512BW-NEXT:    vmovdqa64 64(%r8), %zmm16
+; AVX512BW-NEXT:    vmovdqa64 (%r9), %zmm12
+; AVX512BW-NEXT:    vmovdqa64 64(%r9), %zmm19
+; AVX512BW-NEXT:    vmovdqa64 (%r10), %zmm25
+; AVX512BW-NEXT:    vmovdqa64 64(%r10), %zmm17
+; AVX512BW-NEXT:    vmovdqa64 (%rax), %zmm15
+; AVX512BW-NEXT:    vmovdqa64 64(%rax), %zmm20
 ; AVX512BW-NEXT:    movb $-64, %r11b
 ; AVX512BW-NEXT:    kmovd %r11d, %k1
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm15 = [4,12,4,12,4,12,4,12]
-; AVX512BW-NEXT:    # zmm15 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm17, %zmm9
-; AVX512BW-NEXT:    vpermt2q %zmm24, %zmm15, %zmm9
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm10 = zmm23[0],zmm18[0],zmm23[2],zmm18[2],zmm23[4],zmm18[4],zmm23[6],zmm18[6]
-; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm10 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm9
-; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm15, %zmm9
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm12 = [4,12,4,12]
-; AVX512BW-NEXT:    # ymm12 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm11
-; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm12, %zmm11
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3],ymm11[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm9, %zmm10, %zmm9
-; AVX512BW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm8 = [5,13,5,13,5,13,5,13]
-; AVX512BW-NEXT:    # zmm8 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm17, %zmm9
-; AVX512BW-NEXT:    vpermt2q %zmm24, %zmm8, %zmm9
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm10 = zmm23[1],zmm18[1],zmm23[3],zmm18[3],zmm23[5],zmm18[5],zmm23[7],zmm18[7]
-; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm10 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm9
-; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm8, %zmm9
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm14 = [5,13,5,13]
-; AVX512BW-NEXT:    # ymm14 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm11
-; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm14, %zmm11
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm9 = ymm9[0,1,2,3],ymm11[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm9, %zmm10, %zmm9
-; AVX512BW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm3 = [6,14,6,14,6,14,6,14]
-; AVX512BW-NEXT:    # zmm3 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm23, %zmm9
-; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm3, %zmm9
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm9 {%k1} = zmm17[0],zmm24[0],zmm17[2],zmm24[2],zmm17[4],zmm24[4],zmm17[6],zmm24[6]
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm10
-; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm3, %zmm10
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm4 = [6,14,6,14]
-; AVX512BW-NEXT:    # ymm4 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm11
-; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm4, %zmm11
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm10 = ymm10[0,1,2,3],ymm11[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm10, %zmm9, %zmm9
-; AVX512BW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm29 = [7,15,7,15,7,15,7,15]
-; AVX512BW-NEXT:    # zmm29 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm23, %zmm9
-; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm29, %zmm9
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm9 {%k1} = zmm17[1],zmm24[1],zmm17[3],zmm24[3],zmm17[5],zmm24[5],zmm17[7],zmm24[7]
-; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm29, %zmm5
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = [7,15,7,15]
-; AVX512BW-NEXT:    # ymm0 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm0, %zmm6
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm6[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm5, %zmm9, %zmm5
-; AVX512BW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm5
-; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm15, %zmm5
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm6 = zmm19[0],zmm28[0],zmm19[2],zmm28[2],zmm19[4],zmm28[4],zmm19[6],zmm28[6]
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm6 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm5
-; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm15, %zmm5
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm7
-; AVX512BW-NEXT:    vpermt2q %zmm21, %zmm12, %zmm7
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm7[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm5, %zmm6, %zmm5
-; AVX512BW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm5
-; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm8, %zmm5
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm6 = zmm19[1],zmm28[1],zmm19[3],zmm28[3],zmm19[5],zmm28[5],zmm19[7],zmm28[7]
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm6 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm5
-; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm8, %zmm5
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm7
-; AVX512BW-NEXT:    vpermt2q %zmm21, %zmm14, %zmm7
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm7[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm5, %zmm6, %zmm5
-; AVX512BW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm5
-; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm3, %zmm5
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm6
-; AVX512BW-NEXT:    vpermt2q %zmm21, %zmm4, %zmm6
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm6[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqa64 %zmm19, %zmm6
-; AVX512BW-NEXT:    vpermt2q %zmm28, %zmm3, %zmm6
-; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm7
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm6 {%k1} = zmm16[0],zmm22[0],zmm16[2],zmm22[2],zmm16[4],zmm22[4],zmm16[6],zmm22[6]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm5, %zmm6, %zmm3
-; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 128(%rcx), %zmm5
-; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm29, %zmm1
-; AVX512BW-NEXT:    vmovdqa64 %zmm19, %zmm3
-; AVX512BW-NEXT:    vpermt2q %zmm28, %zmm29, %zmm3
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm3 {%k1} = zmm16[1],zmm22[1],zmm16[3],zmm22[3],zmm16[5],zmm22[5],zmm16[7],zmm22[7]
-; AVX512BW-NEXT:    vpermt2q %zmm21, %zmm0, %zmm2
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm3, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm1
-; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm12, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm1
-; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm14, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm1
-; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm4, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm0, %zmm13
-; AVX512BW-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 192(%rcx), %zmm1
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512BW-NEXT:    vpermi2q %zmm1, %zmm13, %zmm12
-; AVX512BW-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermi2q %zmm1, %zmm13, %zmm14
-; AVX512BW-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermi2q %zmm1, %zmm13, %zmm4
-; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm0, %zmm13
-; AVX512BW-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm25, %zmm15, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm25, %zmm8, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm25, %zmm7, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm25, %zmm29, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 128(%r10), %zmm30
-; AVX512BW-NEXT:    vmovdqa64 128(%rax), %zmm6
-; AVX512BW-NEXT:    vmovdqa64 %zmm30, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm15, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm30, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm8, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 128(%r8), %zmm21
-; AVX512BW-NEXT:    vmovdqa64 128(%r9), %zmm9
-; AVX512BW-NEXT:    vmovdqa64 %zmm21, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm7, %zmm0
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm10
-; AVX512BW-NEXT:    vmovdqa64 192(%r10), %zmm14
-; AVX512BW-NEXT:    vmovdqa64 192(%rax), %zmm4
-; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm15, %zmm0
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm27 = [4,12,4,12,4,12,4,12]
+; AVX512BW-NEXT:    # zmm27 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm10
+; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm27, %zmm10
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm11
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm27, %zmm11
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm10 = ymm11[0,1,2,3],ymm10[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm25, %zmm11
+; AVX512BW-NEXT:    vpermt2q %zmm15, %zmm27, %zmm11
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm13 = zmm18[0],zmm12[0],zmm18[2],zmm12[2],zmm18[4],zmm12[4],zmm18[6],zmm12[6]
+; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm13 {%k1}
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm10, %zmm13, %zmm10
+; AVX512BW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm24 = [5,13,5,13,5,13,5,13]
+; AVX512BW-NEXT:    # zmm24 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm10
+; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm24, %zmm10
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm11
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm24, %zmm11
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm10 = ymm11[0,1,2,3],ymm10[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm25, %zmm11
+; AVX512BW-NEXT:    vpermt2q %zmm15, %zmm24, %zmm11
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm14 = zmm18[1],zmm12[1],zmm18[3],zmm12[3],zmm18[5],zmm12[5],zmm18[7],zmm12[7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm14 {%k1}
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm10, %zmm14, %zmm10
+; AVX512BW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm22 = [6,14,6,14,6,14,6,14]
+; AVX512BW-NEXT:    # zmm22 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm10
+; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm22, %zmm10
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm11
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm22, %zmm11
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm10 = ymm11[0,1,2,3],ymm10[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm18, %zmm11
+; AVX512BW-NEXT:    vpermt2q %zmm12, %zmm22, %zmm11
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm11 {%k1} = zmm25[0],zmm15[0],zmm25[2],zmm15[2],zmm25[4],zmm15[4],zmm25[6],zmm15[6]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm10, %zmm11, %zmm10
+; AVX512BW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm21 = [7,15,7,15,7,15,7,15]
+; AVX512BW-NEXT:    # zmm21 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm21, %zmm4
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm21, %zmm2
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm4[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm18, %zmm3
+; AVX512BW-NEXT:    vpermt2q %zmm12, %zmm21, %zmm3
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm3 {%k1} = zmm25[1],zmm15[1],zmm25[3],zmm15[3],zmm25[5],zmm15[5],zmm25[7],zmm15[7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm3, %zmm2
+; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm2
+; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm27, %zmm2
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm3
+; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm27, %zmm3
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm17, %zmm3
+; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm27, %zmm3
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm4 = zmm16[0],zmm19[0],zmm16[2],zmm19[2],zmm16[4],zmm19[4],zmm16[6],zmm19[6]
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm4 {%k1}
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm4, %zmm2
+; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm2
+; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm24, %zmm2
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm3
+; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm24, %zmm3
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm17, %zmm3
+; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm24, %zmm3
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm4 = zmm16[1],zmm19[1],zmm16[3],zmm19[3],zmm16[5],zmm19[5],zmm16[7],zmm19[7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm4 {%k1}
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm4, %zmm2
+; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm2
+; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm22, %zmm2
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm3
+; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm22, %zmm3
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm3
+; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm22, %zmm3
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm3 {%k1} = zmm17[0],zmm20[0],zmm17[2],zmm20[2],zmm17[4],zmm20[4],zmm17[6],zmm20[6]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm3, %zmm2
+; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm21, %zmm7
+; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm21, %zmm6
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm6[0,1,2,3],ymm7[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm3
+; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm21, %zmm3
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm3 {%k1} = zmm17[1],zmm20[1],zmm17[3],zmm20[3],zmm17[5],zmm20[5],zmm17[7],zmm20[7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm3, %zmm2
+; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm2
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm27, %zmm2
+; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm4
+; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm27, %zmm4
+; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm4
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm24, %zmm4
+; AVX512BW-NEXT:    vmovdqu64 %zmm4, (%rsp) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm4
+; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm24, %zmm4
+; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm4
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm22, %zmm4
+; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm4
+; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm22, %zmm4
+; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm21, %zmm3
+; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm21, %zmm2
+; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm23, %zmm27, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm23, %zmm24, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm23, %zmm22, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm23, %zmm21, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vmovdqa64 192(%rdi), %zmm1
 ; AVX512BW-NEXT:    vmovdqa64 192(%rsi), %zmm0
-; AVX512BW-NEXT:    vpermi2q %zmm0, %zmm1, %zmm15
-; AVX512BW-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm2
-; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm8, %zmm2
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm2
+; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm27, %zmm2
 ; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermi2q %zmm0, %zmm1, %zmm8
-; AVX512BW-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 192(%r8), %zmm31
-; AVX512BW-NEXT:    vmovdqa64 192(%r9), %zmm3
-; AVX512BW-NEXT:    vmovdqa64 %zmm31, %zmm27
-; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm7, %zmm27
-; AVX512BW-NEXT:    vpermi2q %zmm0, %zmm1, %zmm7
-; AVX512BW-NEXT:    vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm29, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm2
+; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm24, %zmm2
+; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm2
+; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm22, %zmm2
+; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm21, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm30 = [0,8,0,8,0,8,0,8]
+; AVX512BW-NEXT:    # zmm30 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm25, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm15, %zmm30, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm31 = [1,9,1,9,1,9,1,9]
+; AVX512BW-NEXT:    # zmm31 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm25, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm15, %zmm31, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm23 = [2,10,2,10,2,10,2,10]
+; AVX512BW-NEXT:    # zmm23 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm25, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm15, %zmm23, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm26 = [3,11,3,11,3,11,3,11]
+; AVX512BW-NEXT:    # zmm26 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpermt2q %zmm15, %zmm26, %zmm25
+; AVX512BW-NEXT:    vmovdqu64 %zmm25, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm18, %zmm0
+; AVX512BW-NEXT:    vmovdqa64 %zmm18, %zmm29
+; AVX512BW-NEXT:    vpermt2q %zmm12, %zmm30, %zmm29
+; AVX512BW-NEXT:    vpermt2q %zmm12, %zmm31, %zmm18
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm12, %zmm23, %zmm1
 ; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm21, %zmm20
-; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm29, %zmm20
-; AVX512BW-NEXT:    vmovdqa64 %zmm31, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm29, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm12, %zmm26, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm2 = [0,8,0,8,0,8,0,8]
-; AVX512BW-NEXT:    # zmm2 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; AVX512BW-NEXT:    vmovdqa64 %zmm17, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm24, %zmm2, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm30, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm1 = [1,9,1,9,1,9,1,9]
-; AVX512BW-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
 ; AVX512BW-NEXT:    vmovdqa64 %zmm17, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm24, %zmm1, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, (%rsp) # 64-byte Spill
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm29 = [2,10,2,10,2,10,2,10]
-; AVX512BW-NEXT:    # zmm29 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm31, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vmovdqa64 %zmm17, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm24, %zmm29, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm23, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm0 = [3,11,3,11,3,11,3,11]
-; AVX512BW-NEXT:    # zmm0 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermt2q %zmm24, %zmm0, %zmm17
+; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm26, %zmm17
 ; AVX512BW-NEXT:    vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm23, %zmm13
-; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm2, %zmm13
-; AVX512BW-NEXT:    vmovdqa64 %zmm23, %zmm12
-; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm1, %zmm12
-; AVX512BW-NEXT:    vmovdqa64 %zmm23, %zmm24
-; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm29, %zmm24
-; AVX512BW-NEXT:    vpermt2q %zmm18, %zmm0, %zmm23
-; AVX512BW-NEXT:    vmovdqu64 %zmm23, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm26
-; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm2, %zmm26
-; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm23
-; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm1, %zmm23
-; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm5
-; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm29, %zmm5
-; AVX512BW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm0, %zmm16
+; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm14
+; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm30, %zmm14
+; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm11
+; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm31, %zmm11
+; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm25
+; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm23, %zmm25
+; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm26, %zmm16
 ; AVX512BW-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm19, %zmm5
-; AVX512BW-NEXT:    vpermt2q %zmm28, %zmm2, %zmm19
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm18
-; AVX512BW-NEXT:    vpermt2q %zmm28, %zmm1, %zmm18
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm22
-; AVX512BW-NEXT:    vpermt2q %zmm28, %zmm29, %zmm22
-; AVX512BW-NEXT:    vpermt2q %zmm28, %zmm0, %zmm5
-; AVX512BW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm30, %zmm15
-; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm2, %zmm15
-; AVX512BW-NEXT:    vmovdqa64 %zmm30, %zmm16
-; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm1, %zmm16
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm10 {%k1} = zmm30[0],zmm6[0],zmm30[2],zmm6[2],zmm30[4],zmm6[4],zmm30[6],zmm6[6]
-; AVX512BW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm20 {%k1} = zmm30[1],zmm6[1],zmm30[3],zmm6[3],zmm30[5],zmm6[5],zmm30[7],zmm6[7]
-; AVX512BW-NEXT:    vmovdqa64 %zmm30, %zmm5
-; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm29, %zmm5
-; AVX512BW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm0, %zmm30
-; AVX512BW-NEXT:    vmovdqa64 %zmm21, %zmm17
-; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm2, %zmm17
-; AVX512BW-NEXT:    vmovdqa64 %zmm21, %zmm5
-; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm1, %zmm5
-; AVX512BW-NEXT:    vmovdqa64 %zmm21, %zmm11
-; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm29, %zmm11
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm7 = zmm21[0],zmm9[0],zmm21[2],zmm9[2],zmm21[4],zmm9[4],zmm21[6],zmm9[6]
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm8 = zmm21[1],zmm9[1],zmm21[3],zmm9[3],zmm21[5],zmm9[5],zmm21[7],zmm9[7]
-; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm0, %zmm21
-; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm10
-; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm2, %zmm10
-; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm25
-; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm1, %zmm25
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm27 {%k1} = zmm14[0],zmm4[0],zmm14[2],zmm4[2],zmm14[4],zmm4[4],zmm14[6],zmm4[6]
-; AVX512BW-NEXT:    vmovdqu64 %zmm27, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm9 {%k1} = zmm14[1],zmm4[1],zmm14[3],zmm4[3],zmm14[5],zmm4[5],zmm14[7],zmm4[7]
-; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm27
-; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm29, %zmm27
-; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm0, %zmm14
-; AVX512BW-NEXT:    vpermi2q %zmm3, %zmm31, %zmm2
-; AVX512BW-NEXT:    vpermi2q %zmm3, %zmm31, %zmm1
-; AVX512BW-NEXT:    vpermi2q %zmm3, %zmm31, %zmm29
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm4 = zmm31[0],zmm3[0],zmm31[2],zmm3[2],zmm31[4],zmm3[4],zmm31[6],zmm3[6]
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm6 = zmm31[1],zmm3[1],zmm31[3],zmm3[3],zmm31[5],zmm3[5],zmm31[7],zmm3[7]
-; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm0, %zmm31
+; AVX512BW-NEXT:    vmovdqa64 128(%r10), %zmm19
+; AVX512BW-NEXT:    vmovdqa64 128(%rax), %zmm0
+; AVX512BW-NEXT:    vmovdqa64 %zmm19, %zmm28
+; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm30, %zmm28
+; AVX512BW-NEXT:    vmovdqa64 %zmm19, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm31, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm19, %zmm13
+; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm27, %zmm13
+; AVX512BW-NEXT:    vmovdqa64 %zmm19, %zmm16
+; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm24, %zmm16
+; AVX512BW-NEXT:    vmovdqa64 128(%r8), %zmm20
+; AVX512BW-NEXT:    vmovdqa64 128(%r9), %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm20, %zmm3
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm22, %zmm3
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm3 {%k1} = zmm19[0],zmm0[0],zmm19[2],zmm0[2],zmm19[4],zmm0[4],zmm19[6],zmm0[6]
+; AVX512BW-NEXT:    vmovdqa64 %zmm20, %zmm10
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm21, %zmm10
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm10 {%k1} = zmm19[1],zmm0[1],zmm19[3],zmm0[3],zmm19[5],zmm0[5],zmm19[7],zmm0[7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm19, %zmm2
+; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm23, %zmm2
+; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm26, %zmm19
+; AVX512BW-NEXT:    vmovdqa64 %zmm20, %zmm4
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm30, %zmm4
+; AVX512BW-NEXT:    vmovdqa64 %zmm20, %zmm6
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm31, %zmm6
+; AVX512BW-NEXT:    vmovdqa64 %zmm20, %zmm17
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm23, %zmm17
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm5 = zmm20[0],zmm1[0],zmm20[2],zmm1[2],zmm20[4],zmm1[4],zmm20[6],zmm1[6]
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm7 = zmm20[1],zmm1[1],zmm20[3],zmm1[3],zmm20[5],zmm1[5],zmm20[7],zmm1[7]
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm26, %zmm20
+; AVX512BW-NEXT:    vmovdqa64 192(%r10), %zmm8
+; AVX512BW-NEXT:    vmovdqa64 192(%rax), %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm12
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm30, %zmm12
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm15
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm31, %zmm15
+; AVX512BW-NEXT:    vpermi2q %zmm1, %zmm8, %zmm27
+; AVX512BW-NEXT:    vpermi2q %zmm1, %zmm8, %zmm24
+; AVX512BW-NEXT:    vmovdqa64 192(%r8), %zmm2
+; AVX512BW-NEXT:    vmovdqa64 192(%r9), %zmm0
+; AVX512BW-NEXT:    vpermi2q %zmm0, %zmm2, %zmm22
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm22 {%k1} = zmm8[0],zmm1[0],zmm8[2],zmm1[2],zmm8[4],zmm1[4],zmm8[6],zmm1[6]
+; AVX512BW-NEXT:    vpermi2q %zmm0, %zmm2, %zmm21
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm21 {%k1} = zmm8[1],zmm1[1],zmm8[3],zmm1[3],zmm8[5],zmm1[5],zmm8[7],zmm1[7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm9
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm23, %zmm9
+; AVX512BW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm26, %zmm8
+; AVX512BW-NEXT:    vpermi2q %zmm0, %zmm2, %zmm30
+; AVX512BW-NEXT:    vpermi2q %zmm0, %zmm2, %zmm31
+; AVX512BW-NEXT:    vpermi2q %zmm0, %zmm2, %zmm23
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm1 = zmm2[0],zmm0[0],zmm2[2],zmm0[2],zmm2[4],zmm0[4],zmm2[6],zmm0[6]
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm9 = zmm2[1],zmm0[1],zmm2[3],zmm0[3],zmm2[5],zmm0[5],zmm2[7],zmm0[7]
+; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm26, %zmm2
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm13 {%k1}
-; AVX512BW-NEXT:    vmovdqa (%rsi), %xmm0
-; AVX512BW-NEXT:    vinserti128 $1, (%rcx), %ymm0, %ymm0
-; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm3
-; AVX512BW-NEXT:    vinserti128 $1, (%rdx), %ymm3, %ymm3
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm28 = ymm3[0],ymm0[0],ymm3[2],ymm0[2]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm28, %zmm13, %zmm13
-; AVX512BW-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 (%rsp), %zmm13 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm12 {%k1}
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm0 = ymm3[1],ymm0[1],ymm3[3],ymm0[3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm12, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, (%rsp) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm26, %zmm19 {%k1}
-; AVX512BW-NEXT:    vmovdqa 64(%rsi), %xmm0
-; AVX512BW-NEXT:    vinserti128 $1, 64(%rcx), %ymm0, %ymm0
-; AVX512BW-NEXT:    vmovdqa 64(%rdi), %xmm12
-; AVX512BW-NEXT:    vinserti128 $1, 64(%rdx), %ymm12, %ymm12
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm13 = ymm12[0],ymm0[0],ymm12[2],ymm0[2]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm13, %zmm19, %zmm3
-; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm23, %zmm18 {%k1}
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm0 = ymm12[1],ymm0[1],ymm12[3],ymm0[3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm18, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm17 {%k1}
-; AVX512BW-NEXT:    vmovdqa 128(%rsi), %xmm0
-; AVX512BW-NEXT:    vinserti128 $1, 128(%rcx), %ymm0, %ymm0
-; AVX512BW-NEXT:    vmovdqa 128(%rdi), %xmm12
-; AVX512BW-NEXT:    vinserti128 $1, 128(%rdx), %ymm12, %ymm13
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm12 = ymm13[0],ymm0[0],ymm13[2],ymm0[2]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm12, %zmm17, %zmm3
-; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm5 {%k1}
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm0 = ymm13[1],ymm0[1],ymm13[3],ymm0[3]
+; AVX512BW-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; AVX512BW-NEXT:    # ymm0 = ymm0[0,1,2,3],mem[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm5 {%k1}
 ; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm5, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm7 {%k1}
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm5 # 32-byte Folded Reload
-; AVX512BW-NEXT:    # ymm5 = ymm0[0,1,2,3],mem[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm5, %zmm7, %zmm28
+; AVX512BW-NEXT:    vpblendd $240, (%rsp), %ymm0, %ymm0 # 32-byte Folded Reload
+; AVX512BW-NEXT:    # ymm0 = ymm0[0,1,2,3],mem[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm7 {%k1}
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm7, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm8 {%k1}
+; AVX512BW-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; AVX512BW-NEXT:    # ymm0 = ymm0[0,1,2,3],mem[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm3, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm5 # 32-byte Folded Reload
-; AVX512BW-NEXT:    # ymm5 = ymm0[0,1,2,3],mem[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm5, %zmm8, %zmm26
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm5 # 32-byte Folded Reload
-; AVX512BW-NEXT:    # ymm5 = ymm0[0,1,2,3],mem[4,5,6,7]
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vinsertf64x4 $0, %ymm5, %zmm0, %zmm19
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm29 {%k1}
+; AVX512BW-NEXT:    vmovdqa (%rsi), %xmm0
+; AVX512BW-NEXT:    vinserti128 $1, (%rcx), %ymm0, %ymm3
+; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512BW-NEXT:    vinserti128 $1, (%rdx), %ymm0, %ymm5
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm5[0],ymm3[0],ymm5[2],ymm3[2]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm29, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm5 # 32-byte Folded Reload
-; AVX512BW-NEXT:    # ymm5 = ymm0[0,1,2,3],mem[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm5, %zmm20, %zmm18
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm2 {%k1}
-; AVX512BW-NEXT:    vmovdqa 192(%rsi), %xmm7
-; AVX512BW-NEXT:    vinserti128 $1, 192(%rcx), %ymm7, %ymm7
-; AVX512BW-NEXT:    vmovdqa 192(%rdi), %xmm10
-; AVX512BW-NEXT:    vinserti128 $1, 192(%rdx), %ymm10, %ymm10
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm15 = ymm10[0],ymm7[0],ymm10[2],ymm7[2]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm15, %zmm2, %zmm17
-; AVX512BW-NEXT:    vmovdqa64 %zmm25, %zmm1 {%k1}
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm7 = ymm10[1],ymm7[1],ymm10[3],ymm7[3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm7, %zmm1, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm18 {%k1}
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm3 = ymm5[1],ymm3[1],ymm5[3],ymm3[3]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm3, %zmm18, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, (%rsp) # 64-byte Spill
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm4 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm14 {%k1}
+; AVX512BW-NEXT:    vmovdqa 64(%rsi), %xmm3
+; AVX512BW-NEXT:    vinserti128 $1, 64(%rcx), %ymm3, %ymm3
+; AVX512BW-NEXT:    vmovdqa 64(%rdi), %xmm5
+; AVX512BW-NEXT:    vinserti128 $1, 64(%rdx), %ymm5, %ymm5
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm7 = ymm5[0],ymm3[0],ymm5[2],ymm3[2]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm7, %zmm14, %zmm29
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm7 # 32-byte Folded Reload
-; AVX512BW-NEXT:    # ymm7 = ymm0[0,1,2,3],mem[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm7, %zmm4, %zmm7
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm11 {%k1}
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm3 = ymm5[1],ymm3[1],ymm5[3],ymm3[3]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm3, %zmm11, %zmm26
+; AVX512BW-NEXT:    vmovdqa64 %zmm28, %zmm4 {%k1}
+; AVX512BW-NEXT:    vmovdqa 128(%rsi), %xmm3
+; AVX512BW-NEXT:    vinserti128 $1, 128(%rcx), %ymm3, %ymm3
+; AVX512BW-NEXT:    vmovdqa 128(%rdi), %xmm5
+; AVX512BW-NEXT:    vinserti128 $1, 128(%rdx), %ymm5, %ymm5
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm7 = ymm5[0],ymm3[0],ymm5[2],ymm3[2]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm7, %zmm4, %zmm28
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm6 {%k1}
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm3 = ymm5[1],ymm3[1],ymm5[3],ymm3[3]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm3, %zmm6, %zmm18
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm4 # 32-byte Folded Reload
 ; AVX512BW-NEXT:    # ymm4 = ymm0[0,1,2,3],mem[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm4, %zmm6, %zmm4
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vblendps $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm6 # 32-byte Folded Reload
-; AVX512BW-NEXT:    # ymm6 = ymm0[0,1,2,3],mem[4,5,6,7]
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vinsertf64x4 $0, %ymm6, %zmm0, %zmm10
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm4, %zmm10, %zmm16
+; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm30 {%k1}
+; AVX512BW-NEXT:    vmovdqa 192(%rsi), %xmm4
+; AVX512BW-NEXT:    vinserti128 $1, 192(%rcx), %ymm4, %ymm5
+; AVX512BW-NEXT:    vmovdqa 192(%rdi), %xmm4
+; AVX512BW-NEXT:    vinserti128 $1, 192(%rdx), %ymm4, %ymm6
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm4 = ymm6[0],ymm5[0],ymm6[2],ymm5[2]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm4, %zmm30, %zmm30
+; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm31 {%k1}
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm5 = ymm6[1],ymm5[1],ymm6[3],ymm5[3]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm5, %zmm31, %zmm31
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm6 # 32-byte Folded Reload
 ; AVX512BW-NEXT:    # ymm6 = ymm0[0,1,2,3],mem[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm6, %zmm9, %zmm15
+; AVX512BW-NEXT:    vmovdqa64 %zmm27, %zmm1 {%k1}
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm6, %zmm1, %zmm6
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm24 {%k1}
-; AVX512BW-NEXT:    vmovdqa (%rcx), %ymm6
-; AVX512BW-NEXT:    vmovdqa64 (%rdx), %ymm16
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm3 = ymm16[0],ymm6[0],ymm16[2],ymm6[2]
-; AVX512BW-NEXT:    vmovdqa64 (%rsi), %ymm20
-; AVX512BW-NEXT:    vmovdqa64 (%rdi), %ymm23
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm12 = ymm23[0],ymm20[0],ymm23[2],ymm20[2]
-; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm3 = ymm12[2,3],ymm3[2,3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm3, %zmm24, %zmm3
+; AVX512BW-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 32-byte Folded Reload
+; AVX512BW-NEXT:    # ymm1 = ymm0[0,1,2,3],mem[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm24, %zmm9 {%k1}
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm9, %zmm7
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm2 {%k1}
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm6 = ymm16[1],ymm6[1],ymm16[3],ymm6[3]
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm12 = ymm23[1],ymm20[1],ymm23[3],ymm20[3]
-; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm6 = ymm12[2,3],ymm6[2,3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm6, %zmm2, %zmm6
+; AVX512BW-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 32-byte Folded Reload
+; AVX512BW-NEXT:    # ymm1 = ymm0[0,1,2,3],mem[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm22, %zmm9
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm22 {%k1}
-; AVX512BW-NEXT:    vmovdqa 64(%rcx), %ymm12
-; AVX512BW-NEXT:    vmovdqa64 64(%rdx), %ymm16
-; AVX512BW-NEXT:    vmovdqa64 64(%rsi), %ymm20
-; AVX512BW-NEXT:    vmovdqa64 64(%rdi), %ymm23
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm16[0],ymm12[0],ymm16[2],ymm12[2]
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm13 = ymm23[0],ymm20[0],ymm23[2],ymm20[2]
-; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm13[2,3],ymm0[2,3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm22, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm5 {%k1}
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm12 = ymm16[1],ymm12[1],ymm16[3],ymm12[3]
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm13 = ymm23[1],ymm20[1],ymm23[3],ymm20[3]
-; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm12 = ymm13[2,3],ymm12[2,3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm12, %zmm5, %zmm12
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm11 {%k1}
-; AVX512BW-NEXT:    vmovdqa 128(%rcx), %ymm13
-; AVX512BW-NEXT:    vmovdqa64 128(%rdx), %ymm16
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm5 = ymm16[0],ymm13[0],ymm16[2],ymm13[2]
-; AVX512BW-NEXT:    vmovdqa64 128(%rsi), %ymm20
+; AVX512BW-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 32-byte Folded Reload
+; AVX512BW-NEXT:    # ymm1 = ymm0[0,1,2,3],mem[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm21, %zmm12
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm3 {%k1}
+; AVX512BW-NEXT:    vmovdqa (%rcx), %ymm1
+; AVX512BW-NEXT:    vmovdqa (%rdx), %ymm15
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm15[0],ymm1[0],ymm15[2],ymm1[2]
+; AVX512BW-NEXT:    vmovdqa64 (%rsi), %ymm21
+; AVX512BW-NEXT:    vmovdqa64 (%rdi), %ymm22
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm14 = ymm22[0],ymm21[0],ymm22[2],ymm21[2]
+; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm14[2,3],ymm0[2,3]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm3, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm4 {%k1}
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm15[1],ymm1[1],ymm15[3],ymm1[3]
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm14 = ymm22[1],ymm21[1],ymm22[3],ymm21[3]
+; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm1 = ymm14[2,3],ymm1[2,3]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm4, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm25 {%k1}
+; AVX512BW-NEXT:    vmovdqa 64(%rcx), %ymm14
+; AVX512BW-NEXT:    vmovdqa 64(%rdx), %ymm15
+; AVX512BW-NEXT:    vmovdqa64 64(%rsi), %ymm21
+; AVX512BW-NEXT:    vmovdqa64 64(%rdi), %ymm22
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm13 = ymm15[0],ymm14[0],ymm15[2],ymm14[2]
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm11 = ymm22[0],ymm21[0],ymm22[2],ymm21[2]
+; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm11 = ymm11[2,3],ymm13[2,3]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm11, %zmm25, %zmm11
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm4 {%k1}
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm13 = ymm15[1],ymm14[1],ymm15[3],ymm14[3]
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm14 = ymm22[1],ymm21[1],ymm22[3],ymm21[3]
+; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm13 = ymm14[2,3],ymm13[2,3]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm13, %zmm4, %zmm13
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm17 {%k1}
+; AVX512BW-NEXT:    vmovdqa 128(%rcx), %ymm14
+; AVX512BW-NEXT:    vmovdqa 128(%rdx), %ymm15
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm4 = ymm15[0],ymm14[0],ymm15[2],ymm14[2]
+; AVX512BW-NEXT:    vmovdqa64 128(%rsi), %ymm21
 ; AVX512BW-NEXT:    vmovdqa64 128(%rdi), %ymm22
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm8 = ymm22[0],ymm20[0],ymm22[2],ymm20[2]
-; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm5 = ymm8[2,3],ymm5[2,3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm5, %zmm11, %zmm5
-; AVX512BW-NEXT:    vmovdqa64 %zmm30, %zmm21 {%k1}
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm8 = ymm16[1],ymm13[1],ymm16[3],ymm13[3]
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm11 = ymm22[1],ymm20[1],ymm22[3],ymm20[3]
-; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm8 = ymm11[2,3],ymm8[2,3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm8, %zmm21, %zmm8
-; AVX512BW-NEXT:    vmovdqa64 %zmm27, %zmm29 {%k1}
-; AVX512BW-NEXT:    vmovdqa 192(%rcx), %ymm9
-; AVX512BW-NEXT:    vmovdqa 192(%rdx), %ymm11
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm13 = ymm11[0],ymm9[0],ymm11[2],ymm9[2]
-; AVX512BW-NEXT:    vmovdqa64 192(%rsi), %ymm16
-; AVX512BW-NEXT:    vmovdqa64 192(%rdi), %ymm20
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm2 = ymm20[0],ymm16[0],ymm20[2],ymm16[2]
-; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm2 = ymm2[2,3],ymm13[2,3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm29, %zmm2
-; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm31 {%k1}
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm9 = ymm11[1],ymm9[1],ymm11[3],ymm9[3]
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm11 = ymm20[1],ymm16[1],ymm20[3],ymm16[3]
-; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm9 = ymm11[2,3],ymm9[2,3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm9, %zmm31, %zmm9
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm3 = ymm22[0],ymm21[0],ymm22[2],ymm21[2]
+; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm3 = ymm3[2,3],ymm4[2,3]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm3, %zmm17, %zmm3
+; AVX512BW-NEXT:    vmovdqa64 %zmm19, %zmm20 {%k1}
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm4 = ymm15[1],ymm14[1],ymm15[3],ymm14[3]
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm14 = ymm22[1],ymm21[1],ymm22[3],ymm21[3]
+; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm4 = ymm14[2,3],ymm4[2,3]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm4, %zmm20, %zmm4
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm23 {%k1}
+; AVX512BW-NEXT:    vmovdqa 192(%rcx), %ymm14
+; AVX512BW-NEXT:    vmovdqa 192(%rdx), %ymm15
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm5 = ymm15[0],ymm14[0],ymm15[2],ymm14[2]
+; AVX512BW-NEXT:    vmovdqa64 192(%rsi), %ymm17
+; AVX512BW-NEXT:    vmovdqa64 192(%rdi), %ymm19
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm10 = ymm19[0],ymm17[0],ymm19[2],ymm17[2]
+; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm5 = ymm10[2,3],ymm5[2,3]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm5, %zmm23, %zmm5
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm2 {%k1}
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm8 = ymm15[1],ymm14[1],ymm15[3],ymm14[3]
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm10 = ymm19[1],ymm17[1],ymm19[3],ymm17[3]
+; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm8 = ymm10[2,3],ymm8[2,3]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm8, %zmm2, %zmm2
 ; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512BW-NEXT:    vmovdqa64 %zmm9, 1728(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, 1664(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm8, 1216(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, 1152(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm12, 704(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, 640(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm6, 192(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm3, 128(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm15, 1984(%rax)
-; AVX512BW-NEXT:    vmovaps %zmm10, 1920(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm4, 1856(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, 1792(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, 1600(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm17, 1536(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm18, 1472(%rax)
-; AVX512BW-NEXT:    vmovaps %zmm19, 1408(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm26, 1344(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm28, 1280(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, 1728(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, 1664(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm4, 1216(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, 1152(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm13, 704(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm11, 640(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, 192(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, 128(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm12, 1984(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm9, 1920(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, 1856(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, 1792(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm31, 1600(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm30, 1536(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm16, 1472(%rax)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm0, 1408(%rax)
 ; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm0, 1088(%rax)
+; AVX512BW-NEXT:    vmovaps %zmm0, 1344(%rax)
 ; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm0, 1024(%rax)
+; AVX512BW-NEXT:    vmovaps %zmm0, 1280(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm18, 1088(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm28, 1024(%rax)
 ; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-NEXT:    vmovaps %zmm0, 960(%rax)
 ; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
@@ -5090,10 +5040,8 @@ define void @store_i64_stride8_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512BW-NEXT:    vmovaps %zmm0, 832(%rax)
 ; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-NEXT:    vmovaps %zmm0, 768(%rax)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm0, 576(%rax)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm0, 512(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm26, 576(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm29, 512(%rax)
 ; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-NEXT:    vmovaps %zmm0, 448(%rax)
 ; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
@@ -5106,7 +5054,7 @@ define void @store_i64_stride8_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ; AVX512BW-NEXT:    vmovaps %zmm0, 64(%rax)
 ; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-NEXT:    vmovaps %zmm0, (%rax)
-; AVX512BW-NEXT:    addq $2632, %rsp # imm = 0xA48
+; AVX512BW-NEXT:    addq $2312, %rsp # imm = 0x908
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
   %in.vec0 = load <32 x i64>, ptr %in.vecptr0, align 64
@@ -8878,1977 +8826,1895 @@ define void @store_i64_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
 ;
 ; AVX512F-LABEL: store_i64_stride8_vf64:
 ; AVX512F:       # %bb.0:
-; AVX512F-NEXT:    subq $5512, %rsp # imm = 0x1588
+; AVX512F-NEXT:    subq $5384, %rsp # imm = 0x1508
 ; AVX512F-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512F-NEXT:    movq {{[0-9]+}}(%rsp), %r10
-; AVX512F-NEXT:    vmovdqa64 128(%rdi), %zmm2
+; AVX512F-NEXT:    vmovdqa64 128(%rdi), %zmm0
 ; AVX512F-NEXT:    vmovdqa64 64(%rdi), %zmm4
-; AVX512F-NEXT:    vmovdqa64 (%rdi), %zmm8
-; AVX512F-NEXT:    vmovdqa64 128(%rsi), %zmm17
-; AVX512F-NEXT:    vmovdqa64 64(%rsi), %zmm19
-; AVX512F-NEXT:    vmovdqa64 (%rsi), %zmm22
-; AVX512F-NEXT:    vmovdqa64 64(%rdx), %zmm5
-; AVX512F-NEXT:    vmovdqa64 (%rdx), %zmm10
-; AVX512F-NEXT:    vmovdqa64 64(%rcx), %zmm20
-; AVX512F-NEXT:    vmovdqa64 (%rcx), %zmm11
-; AVX512F-NEXT:    vmovdqa64 (%r8), %zmm1
-; AVX512F-NEXT:    vmovdqa64 64(%r8), %zmm25
-; AVX512F-NEXT:    vmovdqa64 128(%r8), %zmm23
-; AVX512F-NEXT:    vmovdqa64 (%r9), %zmm28
-; AVX512F-NEXT:    vmovdqa64 64(%r9), %zmm26
-; AVX512F-NEXT:    vmovdqa64 128(%r9), %zmm24
-; AVX512F-NEXT:    vmovdqa64 (%r10), %zmm21
-; AVX512F-NEXT:    vmovdqa64 64(%r10), %zmm14
-; AVX512F-NEXT:    vmovdqa64 (%rax), %zmm27
-; AVX512F-NEXT:    vmovdqa64 64(%rax), %zmm16
+; AVX512F-NEXT:    vmovdqa64 (%rdi), %zmm14
+; AVX512F-NEXT:    vmovdqa64 128(%rsi), %zmm1
+; AVX512F-NEXT:    vmovdqa64 64(%rsi), %zmm5
+; AVX512F-NEXT:    vmovdqa64 (%rsi), %zmm10
+; AVX512F-NEXT:    vmovdqa64 128(%rdx), %zmm2
+; AVX512F-NEXT:    vmovdqa64 64(%rdx), %zmm7
+; AVX512F-NEXT:    vmovdqa64 (%rdx), %zmm12
+; AVX512F-NEXT:    vmovdqa64 128(%rcx), %zmm3
+; AVX512F-NEXT:    vmovdqa64 64(%rcx), %zmm9
+; AVX512F-NEXT:    vmovdqa64 (%rcx), %zmm15
+; AVX512F-NEXT:    vmovdqa64 (%r8), %zmm30
+; AVX512F-NEXT:    vmovdqa64 64(%r8), %zmm18
+; AVX512F-NEXT:    vmovdqa64 128(%r8), %zmm11
+; AVX512F-NEXT:    vmovdqa64 (%r9), %zmm24
+; AVX512F-NEXT:    vmovdqa64 64(%r9), %zmm28
+; AVX512F-NEXT:    vmovdqa64 128(%r9), %zmm22
+; AVX512F-NEXT:    vmovdqa64 (%r10), %zmm26
+; AVX512F-NEXT:    vmovdqa64 64(%r10), %zmm31
+; AVX512F-NEXT:    vmovdqa64 128(%r10), %zmm16
+; AVX512F-NEXT:    vmovdqa64 (%rax), %zmm17
+; AVX512F-NEXT:    vmovdqa64 64(%rax), %zmm27
+; AVX512F-NEXT:    vmovdqa64 128(%rax), %zmm13
 ; AVX512F-NEXT:    movb $-64, %r11b
 ; AVX512F-NEXT:    kmovw %r11d, %k1
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm3 = [4,12,4,12,4,12,4,12]
-; AVX512F-NEXT:    # zmm3 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm21, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm27, %zmm3, %zmm0
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm12 = zmm1[0],zmm28[0],zmm1[2],zmm28[2],zmm1[4],zmm28[4],zmm1[6],zmm28[6]
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm12 {%k1}
-; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm22, %zmm3, %zmm0
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm6 = [4,12,4,12]
-; AVX512F-NEXT:    # ymm6 = mem[0,1,0,1]
-; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm15
-; AVX512F-NEXT:    vpermt2q %zmm11, %zmm6, %zmm15
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm15[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm12, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm9 = [5,13,5,13,5,13,5,13]
-; AVX512F-NEXT:    # zmm9 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm21, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm27, %zmm9, %zmm0
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm12 = zmm1[1],zmm28[1],zmm1[3],zmm28[3],zmm1[5],zmm28[5],zmm1[7],zmm28[7]
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm12 {%k1}
-; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm22, %zmm9, %zmm0
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm7 = [5,13,5,13]
-; AVX512F-NEXT:    # ymm7 = mem[0,1,0,1]
-; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm15
-; AVX512F-NEXT:    vpermt2q %zmm11, %zmm7, %zmm15
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm15[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm12, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm13 = [6,14,6,14,6,14,6,14]
-; AVX512F-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm0
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm29
-; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm28, %zmm13, %zmm0
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm0 {%k1} = zmm21[0],zmm27[0],zmm21[2],zmm27[2],zmm21[4],zmm27[4],zmm21[6],zmm27[6]
-; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm12
-; AVX512F-NEXT:    vpermt2q %zmm22, %zmm13, %zmm12
-; AVX512F-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [6,14,6,14]
-; AVX512F-NEXT:    # ymm1 = mem[0,1,0,1]
-; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm15
-; AVX512F-NEXT:    vpermt2q %zmm11, %zmm1, %zmm15
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm12 = ymm12[0,1,2,3],ymm15[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm12, %zmm0, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm18 = [7,15,7,15,7,15,7,15]
-; AVX512F-NEXT:    # zmm18 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm29, %zmm12
-; AVX512F-NEXT:    vpermt2q %zmm28, %zmm18, %zmm12
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm12 {%k1} = zmm21[1],zmm27[1],zmm21[3],zmm27[3],zmm21[5],zmm27[5],zmm21[7],zmm27[7]
-; AVX512F-NEXT:    vpermt2q %zmm22, %zmm18, %zmm8
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} ymm30 = [7,15,7,15]
-; AVX512F-NEXT:    # ymm30 = mem[0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermt2q %zmm11, %zmm30, %zmm10
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3],ymm10[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm8, %zmm12, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm19 = [4,12,4,12,4,12,4,12]
+; AVX512F-NEXT:    # zmm19 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm12, %zmm8
+; AVX512F-NEXT:    vpermt2q %zmm15, %zmm19, %zmm8
+; AVX512F-NEXT:    vmovdqa64 %zmm14, %zmm6
+; AVX512F-NEXT:    vpermt2q %zmm10, %zmm19, %zmm6
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3],ymm8[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 %zmm26, %zmm8
+; AVX512F-NEXT:    vpermt2q %zmm17, %zmm19, %zmm8
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm20 = zmm30[0],zmm24[0],zmm30[2],zmm24[2],zmm30[4],zmm24[4],zmm30[6],zmm24[6]
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm20 {%k1}
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm6, %zmm20, %zmm6
+; AVX512F-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm21 = [5,13,5,13,5,13,5,13]
+; AVX512F-NEXT:    # zmm21 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm12, %zmm6
+; AVX512F-NEXT:    vpermt2q %zmm15, %zmm21, %zmm6
 ; AVX512F-NEXT:    vmovdqa64 %zmm14, %zmm8
-; AVX512F-NEXT:    vpermt2q %zmm16, %zmm3, %zmm8
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm10 = zmm25[0],zmm26[0],zmm25[2],zmm26[2],zmm25[4],zmm26[4],zmm25[6],zmm26[6]
-; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm10 {%k1}
-; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm8
-; AVX512F-NEXT:    vpermt2q %zmm19, %zmm3, %zmm8
-; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm11
-; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm15
-; AVX512F-NEXT:    vpermt2q %zmm20, %zmm6, %zmm11
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3],ymm11[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm8, %zmm10, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm10, %zmm21, %zmm8
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm6 = ymm8[0,1,2,3],ymm6[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 %zmm26, %zmm8
+; AVX512F-NEXT:    vpermt2q %zmm17, %zmm21, %zmm8
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm20 = zmm30[1],zmm24[1],zmm30[3],zmm24[3],zmm30[5],zmm24[5],zmm30[7],zmm24[7]
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm20 {%k1}
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm6, %zmm20, %zmm6
+; AVX512F-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm23 = [6,14,6,14,6,14,6,14]
+; AVX512F-NEXT:    # zmm23 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm12, %zmm6
+; AVX512F-NEXT:    vpermt2q %zmm15, %zmm23, %zmm6
 ; AVX512F-NEXT:    vmovdqa64 %zmm14, %zmm8
-; AVX512F-NEXT:    vpermt2q %zmm16, %zmm9, %zmm8
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm10 = zmm25[1],zmm26[1],zmm25[3],zmm26[3],zmm25[5],zmm26[5],zmm25[7],zmm26[7]
-; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm10 {%k1}
+; AVX512F-NEXT:    vpermt2q %zmm10, %zmm23, %zmm8
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm6 = ymm8[0,1,2,3],ymm6[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 %zmm30, %zmm8
+; AVX512F-NEXT:    vpermt2q %zmm24, %zmm23, %zmm8
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm8 {%k1} = zmm26[0],zmm17[0],zmm26[2],zmm17[2],zmm26[4],zmm17[4],zmm26[6],zmm17[6]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm6, %zmm8, %zmm6
+; AVX512F-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm25 = [7,15,7,15,7,15,7,15]
+; AVX512F-NEXT:    # zmm25 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpermt2q %zmm15, %zmm25, %zmm12
+; AVX512F-NEXT:    vpermt2q %zmm10, %zmm25, %zmm14
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm6 = ymm14[0,1,2,3],ymm12[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 %zmm30, %zmm8
+; AVX512F-NEXT:    vpermt2q %zmm24, %zmm25, %zmm8
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm8 {%k1} = zmm26[1],zmm17[1],zmm26[3],zmm17[3],zmm26[5],zmm17[5],zmm26[7],zmm17[7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm6, %zmm8, %zmm6
+; AVX512F-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm6
+; AVX512F-NEXT:    vpermt2q %zmm9, %zmm19, %zmm6
 ; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm8
-; AVX512F-NEXT:    vpermt2q %zmm19, %zmm9, %zmm8
-; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm11
-; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm12
-; AVX512F-NEXT:    vpermt2q %zmm20, %zmm7, %zmm11
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3],ymm11[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm8, %zmm10, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm5, %zmm19, %zmm8
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm6 = ymm8[0,1,2,3],ymm6[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 %zmm31, %zmm8
+; AVX512F-NEXT:    vpermt2q %zmm27, %zmm19, %zmm8
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm10 = zmm18[0],zmm28[0],zmm18[2],zmm28[2],zmm18[4],zmm28[4],zmm18[6],zmm28[6]
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm10 {%k1}
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm6, %zmm10, %zmm6
+; AVX512F-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm6
+; AVX512F-NEXT:    vpermt2q %zmm9, %zmm21, %zmm6
 ; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm8
-; AVX512F-NEXT:    vpermt2q %zmm19, %zmm13, %zmm8
-; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm10
-; AVX512F-NEXT:    vpermt2q %zmm20, %zmm1, %zmm10
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3],ymm10[4,5,6,7]
-; AVX512F-NEXT:    vmovdqu64 %zmm25, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm25, %zmm10
-; AVX512F-NEXT:    vmovdqu64 %zmm26, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm26, %zmm13, %zmm10
-; AVX512F-NEXT:    vmovdqa64 %zmm14, %zmm11
-; AVX512F-NEXT:    vmovdqu64 %zmm14, (%rsp) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm10 {%k1} = zmm14[0],zmm16[0],zmm14[2],zmm16[2],zmm14[4],zmm16[4],zmm14[6],zmm16[6]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm8, %zmm10, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 128(%r10), %zmm10
-; AVX512F-NEXT:    vpermt2q %zmm19, %zmm18, %zmm4
-; AVX512F-NEXT:    vmovdqa64 128(%rax), %zmm14
-; AVX512F-NEXT:    vpermt2q %zmm20, %zmm30, %zmm5
-; AVX512F-NEXT:    vmovdqa64 %zmm25, %zmm6
-; AVX512F-NEXT:    vpermt2q %zmm26, %zmm18, %zmm6
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm6 {%k1} = zmm11[1],zmm16[1],zmm11[3],zmm16[3],zmm11[5],zmm16[5],zmm11[7],zmm16[7]
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm5[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm4, %zmm6, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm4
-; AVX512F-NEXT:    vpermt2q %zmm14, %zmm3, %zmm4
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm5 = zmm23[0],zmm24[0],zmm23[2],zmm24[2],zmm23[4],zmm24[4],zmm23[6],zmm24[6]
-; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm5 {%k1}
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm6
-; AVX512F-NEXT:    vpermt2q %zmm17, %zmm3, %zmm6
-; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm11
-; AVX512F-NEXT:    vmovdqa64 128(%rdx), %zmm4
-; AVX512F-NEXT:    vmovdqa64 128(%rcx), %zmm7
+; AVX512F-NEXT:    vpermt2q %zmm5, %zmm21, %zmm8
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm6 = ymm8[0,1,2,3],ymm6[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 %zmm31, %zmm8
+; AVX512F-NEXT:    vpermt2q %zmm27, %zmm21, %zmm8
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm10 = zmm18[1],zmm28[1],zmm18[3],zmm28[3],zmm18[5],zmm28[5],zmm18[7],zmm28[7]
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm10 {%k1}
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm6, %zmm10, %zmm6
+; AVX512F-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm6
+; AVX512F-NEXT:    vpermt2q %zmm9, %zmm23, %zmm6
 ; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm8
-; AVX512F-NEXT:    vpermt2q %zmm7, %zmm15, %zmm8
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3],ymm8[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm6, %zmm5, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm5
-; AVX512F-NEXT:    vpermt2q %zmm14, %zmm9, %zmm5
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm6 = zmm23[1],zmm24[1],zmm23[3],zmm24[3],zmm23[5],zmm24[5],zmm23[7],zmm24[7]
+; AVX512F-NEXT:    vpermt2q %zmm5, %zmm23, %zmm8
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm6 = ymm8[0,1,2,3],ymm6[4,5,6,7]
+; AVX512F-NEXT:    vmovdqu64 %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm18, %zmm8
+; AVX512F-NEXT:    vpermt2q %zmm28, %zmm23, %zmm8
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm8 {%k1} = zmm31[0],zmm27[0],zmm31[2],zmm27[2],zmm31[4],zmm27[4],zmm31[6],zmm27[6]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm6, %zmm8, %zmm6
+; AVX512F-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm9, %zmm25, %zmm7
+; AVX512F-NEXT:    vpermt2q %zmm5, %zmm25, %zmm4
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm7[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 %zmm18, %zmm5
+; AVX512F-NEXT:    vpermt2q %zmm28, %zmm25, %zmm5
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm5 {%k1} = zmm31[1],zmm27[1],zmm31[3],zmm27[3],zmm31[5],zmm27[5],zmm31[7],zmm27[7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm4, %zmm5, %zmm4
+; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm4
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm19, %zmm4
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm5
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm19, %zmm5
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3],ymm4[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm5
+; AVX512F-NEXT:    vpermt2q %zmm13, %zmm19, %zmm5
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm6 = zmm11[0],zmm22[0],zmm11[2],zmm22[2],zmm11[4],zmm22[4],zmm11[6],zmm22[6]
 ; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm6 {%k1}
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm5
-; AVX512F-NEXT:    vpermt2q %zmm17, %zmm9, %zmm5
-; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm8
-; AVX512F-NEXT:    vpermt2q %zmm7, %zmm12, %zmm8
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm8[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm5, %zmm6, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm5
-; AVX512F-NEXT:    vpermt2q %zmm17, %zmm13, %zmm5
-; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm6
-; AVX512F-NEXT:    vpermt2q %zmm7, %zmm1, %zmm6
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm6[4,5,6,7]
-; AVX512F-NEXT:    vmovdqa64 %zmm23, %zmm3
-; AVX512F-NEXT:    vmovdqu64 %zmm23, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm23, %zmm6
-; AVX512F-NEXT:    vmovdqu64 %zmm24, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm24, %zmm13, %zmm6
-; AVX512F-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm6 {%k1} = zmm10[0],zmm14[0],zmm10[2],zmm14[2],zmm10[4],zmm14[4],zmm10[6],zmm14[6]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm5, %zmm6, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm17, %zmm18, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm7, %zmm30, %zmm4
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm4[4,5,6,7]
-; AVX512F-NEXT:    vpermt2q %zmm24, %zmm18, %zmm3
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm3 {%k1} = zmm10[1],zmm14[1],zmm10[3],zmm14[3],zmm10[5],zmm14[5],zmm10[7],zmm14[7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm3, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 192(%r10), %zmm10
-; AVX512F-NEXT:    vmovdqa64 192(%rax), %zmm26
-; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm26, %zmm11, %zmm2
-; AVX512F-NEXT:    vmovdqa64 192(%r8), %zmm23
-; AVX512F-NEXT:    vmovdqa64 192(%r9), %zmm29
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm4 = zmm23[0],zmm29[0],zmm23[2],zmm29[2],zmm23[4],zmm29[4],zmm23[6],zmm29[6]
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm4 {%k1}
-; AVX512F-NEXT:    vmovdqa64 192(%rdi), %zmm2
-; AVX512F-NEXT:    vmovdqa64 192(%rsi), %zmm5
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm6
-; AVX512F-NEXT:    vpermt2q %zmm5, %zmm11, %zmm6
-; AVX512F-NEXT:    vmovdqa64 192(%rdx), %zmm3
-; AVX512F-NEXT:    vmovdqa64 192(%rcx), %zmm7
-; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm8
-; AVX512F-NEXT:    vpermt2q %zmm7, %zmm15, %zmm8
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3],ymm8[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm6, %zmm4, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm4
-; AVX512F-NEXT:    vpermt2q %zmm26, %zmm9, %zmm4
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm6 = zmm23[1],zmm29[1],zmm23[3],zmm29[3],zmm23[5],zmm29[5],zmm23[7],zmm29[7]
-; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm6 {%k1}
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm4, %zmm6, %zmm4
+; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm4
-; AVX512F-NEXT:    vpermt2q %zmm5, %zmm9, %zmm4
-; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm8
-; AVX512F-NEXT:    vpermt2q %zmm7, %zmm12, %zmm8
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm8[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm4, %zmm6, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm21, %zmm4
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm5
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm21, %zmm5
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3],ymm4[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm5
+; AVX512F-NEXT:    vpermt2q %zmm13, %zmm21, %zmm5
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm6 = zmm11[1],zmm22[1],zmm11[3],zmm22[3],zmm11[5],zmm22[5],zmm11[7],zmm22[7]
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm6 {%k1}
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm4, %zmm6, %zmm4
+; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm4
-; AVX512F-NEXT:    vpermt2q %zmm5, %zmm13, %zmm4
-; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm6
-; AVX512F-NEXT:    vpermt2q %zmm7, %zmm1, %zmm6
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm6[4,5,6,7]
-; AVX512F-NEXT:    vmovdqa64 %zmm23, %zmm6
-; AVX512F-NEXT:    vpermt2q %zmm29, %zmm13, %zmm6
-; AVX512F-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm6 {%k1} = zmm10[0],zmm26[0],zmm10[2],zmm26[2],zmm10[4],zmm26[4],zmm10[6],zmm26[6]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm4, %zmm6, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm5, %zmm18, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm7, %zmm30, %zmm3
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5,6,7]
-; AVX512F-NEXT:    vmovdqa64 %zmm23, %zmm3
-; AVX512F-NEXT:    vpermt2q %zmm29, %zmm18, %zmm3
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm3 {%k1} = zmm10[1],zmm26[1],zmm10[3],zmm26[3],zmm10[5],zmm26[5],zmm10[7],zmm26[7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm3, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm23, %zmm4
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm5
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm23, %zmm5
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3],ymm4[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm5
+; AVX512F-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 %zmm22, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm22, %zmm23, %zmm5
+; AVX512F-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm5 {%k1} = zmm16[0],zmm13[0],zmm16[2],zmm13[2],zmm16[4],zmm13[4],zmm16[6],zmm13[6]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm4, %zmm5, %zmm4
+; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 192(%rdi), %zmm4
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm25, %zmm2
+; AVX512F-NEXT:    vmovdqa64 192(%rdx), %zmm3
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm25, %zmm0
+; AVX512F-NEXT:    vmovdqa64 192(%rcx), %zmm1
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm2
+; AVX512F-NEXT:    vpermt2q %zmm22, %zmm25, %zmm2
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm2 {%k1} = zmm16[1],zmm13[1],zmm16[3],zmm13[3],zmm16[5],zmm13[5],zmm16[7],zmm13[7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm2, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 256(%r10), %zmm10
-; AVX512F-NEXT:    vmovdqa64 256(%rax), %zmm22
-; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm22, %zmm11, %zmm2
-; AVX512F-NEXT:    vmovdqa64 256(%r8), %zmm0
-; AVX512F-NEXT:    vmovdqa64 256(%r9), %zmm24
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm4 = zmm0[0],zmm24[0],zmm0[2],zmm24[2],zmm0[4],zmm24[4],zmm0[6],zmm24[6]
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm17
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm4 {%k1}
-; AVX512F-NEXT:    vmovdqa64 256(%rdi), %zmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm19, %zmm0
+; AVX512F-NEXT:    vmovdqa64 192(%rsi), %zmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm5
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm19, %zmm5
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm0[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 192(%r10), %zmm9
+; AVX512F-NEXT:    vmovdqa64 192(%rax), %zmm20
+; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm6
+; AVX512F-NEXT:    vpermt2q %zmm20, %zmm19, %zmm6
+; AVX512F-NEXT:    vmovdqa64 192(%r8), %zmm8
+; AVX512F-NEXT:    vmovdqa64 192(%r9), %zmm0
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm7 = zmm8[0],zmm0[0],zmm8[2],zmm0[2],zmm8[4],zmm0[4],zmm8[6],zmm0[6]
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm7 {%k1}
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm5, %zmm7, %zmm5
+; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm5
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm21, %zmm5
+; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm6
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm21, %zmm6
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm5 = ymm6[0,1,2,3],ymm5[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm6
+; AVX512F-NEXT:    vpermt2q %zmm20, %zmm21, %zmm6
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm7 = zmm8[1],zmm0[1],zmm8[3],zmm0[3],zmm8[5],zmm0[5],zmm8[7],zmm0[7]
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm7 {%k1}
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm5, %zmm7, %zmm5
+; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm5
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm23, %zmm5
+; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm6
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm23, %zmm6
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm5 = ymm6[0,1,2,3],ymm5[4,5,6,7]
+; AVX512F-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm6
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm23, %zmm6
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm6 {%k1} = zmm9[0],zmm20[0],zmm9[2],zmm20[2],zmm9[4],zmm20[4],zmm9[6],zmm20[6]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm5, %zmm6, %zmm5
+; AVX512F-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm25, %zmm3
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm25, %zmm4
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3],ymm3[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 %zmm8, %zmm2
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm25, %zmm2
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm2 {%k1} = zmm9[1],zmm20[1],zmm9[3],zmm20[3],zmm9[5],zmm20[5],zmm9[7],zmm20[7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm2, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 256(%rdx), %zmm1
+; AVX512F-NEXT:    vmovdqa64 256(%rcx), %zmm3
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm2
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm19, %zmm2
+; AVX512F-NEXT:    vmovdqa64 256(%rdi), %zmm4
 ; AVX512F-NEXT:    vmovdqa64 256(%rsi), %zmm5
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm6
-; AVX512F-NEXT:    vpermt2q %zmm5, %zmm11, %zmm6
-; AVX512F-NEXT:    vmovdqa64 256(%rdx), %zmm3
-; AVX512F-NEXT:    vmovdqa64 256(%rcx), %zmm7
-; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm8
-; AVX512F-NEXT:    vpermt2q %zmm7, %zmm15, %zmm8
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3],ymm8[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm6, %zmm4, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm10, %zmm4
-; AVX512F-NEXT:    vpermt2q %zmm22, %zmm9, %zmm4
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm6 = zmm17[1],zmm24[1],zmm17[3],zmm24[3],zmm17[5],zmm24[5],zmm17[7],zmm24[7]
-; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm6 {%k1}
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm4
-; AVX512F-NEXT:    vpermt2q %zmm5, %zmm9, %zmm4
-; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm8
-; AVX512F-NEXT:    vpermt2q %zmm7, %zmm12, %zmm8
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm8[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm4, %zmm6, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm4
-; AVX512F-NEXT:    vpermt2q %zmm5, %zmm13, %zmm4
-; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm6
-; AVX512F-NEXT:    vpermt2q %zmm7, %zmm1, %zmm6
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm6[4,5,6,7]
-; AVX512F-NEXT:    vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm17, %zmm6
-; AVX512F-NEXT:    vpermt2q %zmm24, %zmm13, %zmm6
-; AVX512F-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm6 {%k1} = zmm10[0],zmm22[0],zmm10[2],zmm22[2],zmm10[4],zmm22[4],zmm10[6],zmm22[6]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm4, %zmm6, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm5, %zmm18, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm7, %zmm30, %zmm3
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5,6,7]
-; AVX512F-NEXT:    vmovdqa64 %zmm17, %zmm3
-; AVX512F-NEXT:    vpermt2q %zmm24, %zmm18, %zmm3
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm3 {%k1} = zmm10[1],zmm22[1],zmm10[3],zmm22[3],zmm10[5],zmm22[5],zmm10[7],zmm22[7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm3, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 320(%rdi), %zmm2
-; AVX512F-NEXT:    vmovdqa64 320(%rsi), %zmm3
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm5
-; AVX512F-NEXT:    vpermt2q %zmm3, %zmm11, %zmm5
-; AVX512F-NEXT:    vmovdqa64 320(%rdx), %zmm14
-; AVX512F-NEXT:    vmovdqa64 320(%rcx), %zmm6
-; AVX512F-NEXT:    vmovdqa64 %zmm14, %zmm7
-; AVX512F-NEXT:    vpermt2q %zmm6, %zmm15, %zmm7
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm7[4,5,6,7]
-; AVX512F-NEXT:    vmovdqa64 %zmm14, %zmm7
-; AVX512F-NEXT:    vpermt2q %zmm6, %zmm12, %zmm7
-; AVX512F-NEXT:    vmovdqa64 %zmm14, %zmm8
-; AVX512F-NEXT:    vpermt2q %zmm6, %zmm1, %zmm8
-; AVX512F-NEXT:    vpermt2q %zmm6, %zmm30, %zmm14
-; AVX512F-NEXT:    vmovdqa64 384(%rdx), %zmm0
-; AVX512F-NEXT:    vmovdqa64 384(%rcx), %zmm6
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm10
-; AVX512F-NEXT:    vpermt2q %zmm6, %zmm15, %zmm10
-; AVX512F-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm10
-; AVX512F-NEXT:    vpermt2q %zmm6, %zmm12, %zmm10
-; AVX512F-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm10
-; AVX512F-NEXT:    vpermt2q %zmm6, %zmm1, %zmm10
-; AVX512F-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm6, %zmm30, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 448(%rdx), %zmm0
-; AVX512F-NEXT:    vmovdqa64 448(%rcx), %zmm6
-; AVX512F-NEXT:    vpermi2q %zmm6, %zmm0, %zmm15
-; AVX512F-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermi2q %zmm6, %zmm0, %zmm12
-; AVX512F-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermi2q %zmm6, %zmm0, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm6
+; AVX512F-NEXT:    vpermt2q %zmm5, %zmm19, %zmm6
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3],ymm2[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 256(%r10), %zmm13
+; AVX512F-NEXT:    vmovdqa64 256(%rax), %zmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm7
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm19, %zmm7
+; AVX512F-NEXT:    vmovdqa64 256(%r8), %zmm18
+; AVX512F-NEXT:    vmovdqa64 256(%r9), %zmm15
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm8 = zmm18[0],zmm15[0],zmm18[2],zmm15[2],zmm18[4],zmm15[4],zmm18[6],zmm15[6]
+; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm8 {%k1}
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm6, %zmm8, %zmm6
+; AVX512F-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm6
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm21, %zmm6
+; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm7
+; AVX512F-NEXT:    vpermt2q %zmm5, %zmm21, %zmm7
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm6 = ymm7[0,1,2,3],ymm6[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm7
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm21, %zmm7
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm8 = zmm18[1],zmm15[1],zmm18[3],zmm15[3],zmm18[5],zmm15[5],zmm18[7],zmm15[7]
+; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm8 {%k1}
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm6, %zmm8, %zmm6
+; AVX512F-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm6
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm23, %zmm6
+; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm7
+; AVX512F-NEXT:    vpermt2q %zmm5, %zmm23, %zmm7
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm6 = ymm7[0,1,2,3],ymm6[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 %zmm18, %zmm7
+; AVX512F-NEXT:    vpermt2q %zmm15, %zmm23, %zmm7
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm7 {%k1} = zmm13[0],zmm2[0],zmm13[2],zmm2[2],zmm13[4],zmm2[4],zmm13[6],zmm2[6]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm6, %zmm7, %zmm6
+; AVX512F-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm3, %zmm25, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm5, %zmm25, %zmm4
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 %zmm18, %zmm3
+; AVX512F-NEXT:    vpermt2q %zmm15, %zmm25, %zmm3
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm3 {%k1} = zmm13[1],zmm2[1],zmm13[3],zmm2[3],zmm13[5],zmm2[5],zmm13[7],zmm2[7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm3, %zmm1
 ; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm6, %zmm30, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 320(%r10), %zmm31
-; AVX512F-NEXT:    vmovdqa64 320(%rax), %zmm12
-; AVX512F-NEXT:    vmovdqa64 %zmm31, %zmm1
-; AVX512F-NEXT:    vpermt2q %zmm12, %zmm11, %zmm1
-; AVX512F-NEXT:    vmovdqa64 320(%r8), %zmm17
-; AVX512F-NEXT:    vmovdqa64 320(%r9), %zmm4
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm10 = zmm17[0],zmm4[0],zmm17[2],zmm4[2],zmm17[4],zmm4[4],zmm17[6],zmm4[6]
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm10 {%k1}
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm5, %zmm10, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm31, %zmm1
-; AVX512F-NEXT:    vpermt2q %zmm12, %zmm9, %zmm1
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm5 = zmm17[1],zmm4[1],zmm17[3],zmm4[3],zmm17[5],zmm4[5],zmm17[7],zmm4[7]
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm5 {%k1}
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm1
-; AVX512F-NEXT:    vpermt2q %zmm3, %zmm9, %zmm1
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm7[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm5, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm1
-; AVX512F-NEXT:    vpermt2q %zmm3, %zmm13, %zmm1
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm8[4,5,6,7]
-; AVX512F-NEXT:    vmovdqa64 %zmm17, %zmm5
-; AVX512F-NEXT:    vpermt2q %zmm4, %zmm13, %zmm5
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm5 {%k1} = zmm31[0],zmm12[0],zmm31[2],zmm12[2],zmm31[4],zmm12[4],zmm31[6],zmm12[6]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm5, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm3, %zmm18, %zmm2
-; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm14[4,5,6,7]
-; AVX512F-NEXT:    vmovdqa64 %zmm17, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm4, %zmm18, %zmm2
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm2 {%k1} = zmm31[1],zmm12[1],zmm31[3],zmm12[3],zmm31[5],zmm12[5],zmm31[7],zmm12[7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm2, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 384(%rdi), %zmm0
-; AVX512F-NEXT:    vmovdqa64 384(%rsi), %zmm1
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm3
-; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm11, %zmm3
+; AVX512F-NEXT:    vmovdqa64 320(%rdx), %zmm1
+; AVX512F-NEXT:    vmovdqa64 320(%rcx), %zmm4
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512F-NEXT:    vpermt2q %zmm4, %zmm19, %zmm3
+; AVX512F-NEXT:    vmovdqa64 320(%rdi), %zmm6
+; AVX512F-NEXT:    vmovdqa64 320(%rsi), %zmm7
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm5
+; AVX512F-NEXT:    vpermt2q %zmm7, %zmm19, %zmm5
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm8 = ymm5[0,1,2,3],ymm3[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 320(%r10), %zmm16
+; AVX512F-NEXT:    vmovdqa64 320(%rax), %zmm22
+; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm12
+; AVX512F-NEXT:    vpermt2q %zmm22, %zmm19, %zmm12
+; AVX512F-NEXT:    vmovdqa64 320(%r8), %zmm29
+; AVX512F-NEXT:    vmovdqa64 320(%r9), %zmm5
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm14 = zmm29[0],zmm5[0],zmm29[2],zmm5[2],zmm29[4],zmm5[4],zmm29[6],zmm5[6]
+; AVX512F-NEXT:    vmovdqa64 %zmm12, %zmm14 {%k1}
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm8, %zmm14, %zmm3
 ; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm3
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm9, %zmm3
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm8
+; AVX512F-NEXT:    vpermt2q %zmm4, %zmm21, %zmm8
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm12
+; AVX512F-NEXT:    vpermt2q %zmm7, %zmm21, %zmm12
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm8 = ymm12[0,1,2,3],ymm8[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm12
+; AVX512F-NEXT:    vpermt2q %zmm22, %zmm21, %zmm12
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm14 = zmm29[1],zmm5[1],zmm29[3],zmm5[3],zmm29[5],zmm5[5],zmm29[7],zmm5[7]
+; AVX512F-NEXT:    vmovdqa64 %zmm12, %zmm14 {%k1}
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm8, %zmm14, %zmm3
 ; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm3
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm13, %zmm3
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm8
+; AVX512F-NEXT:    vpermt2q %zmm4, %zmm23, %zmm8
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm12
+; AVX512F-NEXT:    vpermt2q %zmm7, %zmm23, %zmm12
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm8 = ymm12[0,1,2,3],ymm8[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 %zmm29, %zmm12
+; AVX512F-NEXT:    vpermt2q %zmm5, %zmm23, %zmm12
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm12 {%k1} = zmm16[0],zmm22[0],zmm16[2],zmm22[2],zmm16[4],zmm22[4],zmm16[6],zmm22[6]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm8, %zmm12, %zmm3
 ; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm18, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 384(%r10), %zmm11
-; AVX512F-NEXT:    vmovdqa64 384(%rax), %zmm20
-; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm20, %zmm2, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm20, %zmm9, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 384(%r8), %zmm30
-; AVX512F-NEXT:    vmovdqa64 384(%r9), %zmm0
-; AVX512F-NEXT:    vmovdqa64 %zmm30, %zmm1
-; AVX512F-NEXT:    vpermt2q %zmm0, %zmm13, %zmm1
-; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 448(%r10), %zmm16
-; AVX512F-NEXT:    vmovdqa64 448(%rax), %zmm8
-; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm1
-; AVX512F-NEXT:    vpermt2q %zmm8, %zmm2, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm4, %zmm25, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm7, %zmm25, %zmm6
+; AVX512F-NEXT:    vpblendd {{.*#+}} ymm1 = ymm6[0,1,2,3],ymm1[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 %zmm29, %zmm4
+; AVX512F-NEXT:    vpermt2q %zmm5, %zmm25, %zmm4
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm4 {%k1} = zmm16[1],zmm22[1],zmm16[3],zmm22[3],zmm16[5],zmm22[5],zmm16[7],zmm22[7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm4, %zmm1
 ; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 384(%rdx), %zmm3
+; AVX512F-NEXT:    vmovdqa64 384(%rcx), %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm4
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm19, %zmm4
+; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm4
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm21, %zmm4
+; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm4
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm23, %zmm4
+; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm25, %zmm3
+; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 384(%rdi), %zmm3
+; AVX512F-NEXT:    vmovdqa64 384(%rsi), %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm4
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm19, %zmm4
+; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm4
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm21, %zmm4
+; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm4
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm23, %zmm4
+; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm25, %zmm3
+; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 448(%rdx), %zmm3
+; AVX512F-NEXT:    vmovdqa64 448(%rcx), %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm4
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm19, %zmm4
+; AVX512F-NEXT:    vmovdqu64 %zmm4, (%rsp) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm4
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm21, %zmm4
+; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm4
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm23, %zmm4
+; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm25, %zmm3
+; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vmovdqa64 448(%rdi), %zmm3
 ; AVX512F-NEXT:    vmovdqa64 448(%rsi), %zmm1
-; AVX512F-NEXT:    vpermi2q %zmm1, %zmm3, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm8, %zmm9, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermi2q %zmm1, %zmm3, %zmm9
-; AVX512F-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 448(%r8), %zmm19
-; AVX512F-NEXT:    vmovdqa64 448(%r9), %zmm7
-; AVX512F-NEXT:    vmovdqa64 %zmm19, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm7, %zmm13, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermi2q %zmm1, %zmm3, %zmm13
-; AVX512F-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm1, %zmm18, %zmm3
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm4
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm19, %zmm4
+; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm4
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm21, %zmm4
+; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm4
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm23, %zmm4
+; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm25, %zmm3
 ; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm11 = [0,8,0,8,0,8,0,8]
+; AVX512F-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm26, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm17, %zmm11, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm12 = [1,9,1,9,1,9,1,9]
+; AVX512F-NEXT:    # zmm12 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm26, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm17, %zmm12, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm10 = [2,10,2,10,2,10,2,10]
+; AVX512F-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vmovdqa64 %zmm26, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm17, %zmm10, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm4 = [3,11,3,11,3,11,3,11]
+; AVX512F-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512F-NEXT:    vpermt2q %zmm17, %zmm4, %zmm26
+; AVX512F-NEXT:    vmovdqu64 %zmm26, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vmovdqa64 %zmm30, %zmm1
-; AVX512F-NEXT:    vpermt2q %zmm0, %zmm18, %zmm1
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm5
-; AVX512F-NEXT:    vmovdqa64 %zmm19, %zmm1
-; AVX512F-NEXT:    vpermt2q %zmm7, %zmm18, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm24, %zmm11, %zmm1
 ; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm15 = [0,8,0,8,0,8,0,8]
-; AVX512F-NEXT:    # zmm15 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm21, %zmm1
-; AVX512F-NEXT:    vpermt2q %zmm27, %zmm15, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm30, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm24, %zmm12, %zmm1
 ; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm14 = [1,9,1,9,1,9,1,9]
-; AVX512F-NEXT:    # zmm14 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm21, %zmm1
-; AVX512F-NEXT:    vpermt2q %zmm27, %zmm14, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm30, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm24, %zmm10, %zmm1
 ; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm6 = [2,10,2,10,2,10,2,10]
-; AVX512F-NEXT:    # zmm6 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm21, %zmm1
-; AVX512F-NEXT:    vpermt2q %zmm27, %zmm6, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm24, %zmm4, %zmm30
+; AVX512F-NEXT:    vmovdqu64 %zmm30, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm31, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm27, %zmm11, %zmm1
 ; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vbroadcasti32x4 {{.*#+}} zmm1 = [3,11,3,11,3,11,3,11]
-; AVX512F-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512F-NEXT:    vpermt2q %zmm27, %zmm1, %zmm21
-; AVX512F-NEXT:    vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vpermt2q %zmm28, %zmm15, %zmm3
-; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vpermt2q %zmm28, %zmm14, %zmm3
-; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm25
-; AVX512F-NEXT:    vpermt2q %zmm28, %zmm6, %zmm25
-; AVX512F-NEXT:    vpermt2q %zmm28, %zmm1, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 (%rsp), %zmm27 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm27, %zmm2
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm3, %zmm15, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm27, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm3, %zmm14, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm27, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm3, %zmm6, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm3, %zmm1, %zmm27
-; AVX512F-NEXT:    vmovdqu64 %zmm27, (%rsp) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm3
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm2, %zmm15, %zmm3
-; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm3
-; AVX512F-NEXT:    vpermt2q %zmm2, %zmm14, %zmm3
+; AVX512F-NEXT:    vmovdqa64 %zmm31, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm27, %zmm12, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm31, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm27, %zmm10, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm27, %zmm4, %zmm31
+; AVX512F-NEXT:    vmovdqu64 %zmm31, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm28, %zmm11, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm28, %zmm12, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm31
+; AVX512F-NEXT:    vpermt2q %zmm28, %zmm10, %zmm31
+; AVX512F-NEXT:    vpermt2q %zmm28, %zmm4, %zmm3
 ; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm18
-; AVX512F-NEXT:    vpermt2q %zmm2, %zmm6, %zmm18
-; AVX512F-NEXT:    vpermt2q %zmm2, %zmm1, %zmm9
-; AVX512F-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm9, %zmm15, %zmm3
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm28 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm28, %zmm3
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm11, %zmm3
 ; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vpermt2q %zmm9, %zmm14, %zmm3
+; AVX512F-NEXT:    vmovdqa64 %zmm28, %zmm3
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm12, %zmm3
 ; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vpermt2q %zmm9, %zmm6, %zmm3
+; AVX512F-NEXT:    vmovdqa64 %zmm28, %zmm3
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm10, %zmm3
 ; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm9, %zmm1, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm3
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vpermt2q %zmm2, %zmm15, %zmm3
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm4, %zmm28
+; AVX512F-NEXT:    vmovdqu64 %zmm28, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm3
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm11, %zmm3
 ; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm3
-; AVX512F-NEXT:    vpermt2q %zmm2, %zmm14, %zmm3
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm3
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm12, %zmm3
 ; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm13
-; AVX512F-NEXT:    vpermt2q %zmm2, %zmm6, %zmm13
-; AVX512F-NEXT:    vpermt2q %zmm2, %zmm1, %zmm9
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm27
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm10, %zmm27
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm4, %zmm6
+; AVX512F-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm20, %zmm11, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm20, %zmm12, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm9, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm20, %zmm10, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm20, %zmm4, %zmm9
 ; AVX512F-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vpermt2q %zmm26, %zmm15, %zmm3
-; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vpermt2q %zmm26, %zmm14, %zmm3
-; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vpermt2q %zmm26, %zmm6, %zmm3
-; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm26, %zmm1, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm23, %zmm3
-; AVX512F-NEXT:    vpermt2q %zmm29, %zmm15, %zmm3
-; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm23, %zmm3
-; AVX512F-NEXT:    vpermt2q %zmm29, %zmm14, %zmm3
-; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm23, %zmm10
-; AVX512F-NEXT:    vpermt2q %zmm29, %zmm6, %zmm10
-; AVX512F-NEXT:    vpermt2q %zmm29, %zmm1, %zmm23
-; AVX512F-NEXT:    vmovdqu64 %zmm23, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm23 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm23, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm22, %zmm15, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm23, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm22, %zmm14, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm23, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm22, %zmm6, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm22, %zmm1, %zmm23
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vpermt2q %zmm24, %zmm15, %zmm3
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm11, %zmm3
 ; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512F-NEXT:    vpermt2q %zmm24, %zmm14, %zmm3
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm12, %zmm3
 ; AVX512F-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm26
-; AVX512F-NEXT:    vpermt2q %zmm24, %zmm6, %zmm26
-; AVX512F-NEXT:    vpermt2q %zmm24, %zmm1, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm31, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm12, %zmm15, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm31, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm12, %zmm14, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm31, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm12, %zmm6, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vpermt2q %zmm12, %zmm1, %zmm31
-; AVX512F-NEXT:    vmovdqa64 %zmm17, %zmm29
-; AVX512F-NEXT:    vpermt2q %zmm4, %zmm15, %zmm29
-; AVX512F-NEXT:    vmovdqa64 %zmm17, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm4, %zmm14, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm17, %zmm24
-; AVX512F-NEXT:    vpermt2q %zmm4, %zmm6, %zmm24
-; AVX512F-NEXT:    vpermt2q %zmm4, %zmm1, %zmm17
-; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm20, %zmm15, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm2
-; AVX512F-NEXT:    vpermt2q %zmm20, %zmm14, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm9 {%k1} = zmm11[0],zmm20[0],zmm11[2],zmm20[2],zmm11[4],zmm20[4],zmm11[6],zmm20[6]
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm5 {%k1} = zmm11[1],zmm20[1],zmm11[3],zmm20[3],zmm11[5],zmm20[5],zmm11[7],zmm20[7]
-; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm27
-; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm22
-; AVX512F-NEXT:    vpermt2q %zmm20, %zmm6, %zmm22
-; AVX512F-NEXT:    vpermt2q %zmm20, %zmm1, %zmm11
-; AVX512F-NEXT:    vmovdqa64 %zmm30, %zmm21
-; AVX512F-NEXT:    vpermt2q %zmm0, %zmm15, %zmm21
-; AVX512F-NEXT:    vmovdqa64 %zmm30, %zmm28
-; AVX512F-NEXT:    vpermt2q %zmm0, %zmm14, %zmm28
-; AVX512F-NEXT:    vmovdqa64 %zmm30, %zmm20
-; AVX512F-NEXT:    vpermt2q %zmm0, %zmm6, %zmm20
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm2 = zmm30[0],zmm0[0],zmm30[2],zmm0[2],zmm30[4],zmm0[4],zmm30[6],zmm0[6]
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm3 = zmm30[1],zmm0[1],zmm30[3],zmm0[3],zmm30[5],zmm0[5],zmm30[7],zmm0[7]
-; AVX512F-NEXT:    vpermt2q %zmm0, %zmm1, %zmm30
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm20
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm10, %zmm20
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm4, %zmm1
+; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm11, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm12, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm10, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm4, %zmm13
+; AVX512F-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm18, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm15, %zmm11, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm18, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm15, %zmm12, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm18, %zmm28
+; AVX512F-NEXT:    vpermt2q %zmm15, %zmm10, %zmm28
+; AVX512F-NEXT:    vpermt2q %zmm15, %zmm4, %zmm18
+; AVX512F-NEXT:    vmovdqu64 %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm8, %zmm15, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm22, %zmm11, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm0
-; AVX512F-NEXT:    vpermt2q %zmm8, %zmm14, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm22, %zmm12, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm4 {%k1} = zmm16[0],zmm8[0],zmm16[2],zmm8[2],zmm16[4],zmm8[4],zmm16[6],zmm8[6]
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm5 {%k1} = zmm16[1],zmm8[1],zmm16[3],zmm8[3],zmm16[5],zmm8[5],zmm16[7],zmm8[7]
-; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm12
-; AVX512F-NEXT:    vpermt2q %zmm8, %zmm6, %zmm12
-; AVX512F-NEXT:    vpermt2q %zmm8, %zmm1, %zmm16
-; AVX512F-NEXT:    vpermi2q %zmm7, %zmm19, %zmm15
-; AVX512F-NEXT:    vpermi2q %zmm7, %zmm19, %zmm14
-; AVX512F-NEXT:    vpermi2q %zmm7, %zmm19, %zmm6
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm8 = zmm19[0],zmm7[0],zmm19[2],zmm7[2],zmm19[4],zmm7[4],zmm19[6],zmm7[6]
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm0 = zmm19[1],zmm7[1],zmm19[3],zmm7[3],zmm19[5],zmm7[5],zmm19[7],zmm7[7]
-; AVX512F-NEXT:    vpermt2q %zmm7, %zmm1, %zmm19
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
-; AVX512F-NEXT:    # ymm1 = ymm1[0,1,2,3],mem[4,5,6,7]
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm2 {%k1}
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm2, %zmm1
-; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
-; AVX512F-NEXT:    # ymm1 = ymm1[0,1,2,3],mem[4,5,6,7]
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm3, %zmm1
-; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
-; AVX512F-NEXT:    # ymm1 = ymm1[0,1,2,3],mem[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm9, %zmm1
-; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
-; AVX512F-NEXT:    # ymm1 = ymm1[0,1,2,3],mem[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm27, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm22, %zmm10, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermt2q %zmm22, %zmm4, %zmm16
+; AVX512F-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm29, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm5, %zmm11, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm29, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm5, %zmm12, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm29, %zmm22
+; AVX512F-NEXT:    vpermt2q %zmm5, %zmm10, %zmm22
+; AVX512F-NEXT:    vpermt2q %zmm5, %zmm4, %zmm29
+; AVX512F-NEXT:    vmovdqa64 384(%r10), %zmm13
+; AVX512F-NEXT:    vmovdqa64 384(%rax), %zmm0
+; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm11, %zmm1
 ; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
-; AVX512F-NEXT:    # ymm1 = ymm1[0,1,2,3],mem[4,5,6,7]
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm8 {%k1}
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm8, %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm1
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm12, %zmm1
 ; AVX512F-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
-; AVX512F-NEXT:    # ymm1 = ymm1[0,1,2,3],mem[4,5,6,7]
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm0 {%k1}
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm0
+; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm14
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm19, %zmm14
+; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm15
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm21, %zmm15
+; AVX512F-NEXT:    vmovdqa64 384(%r8), %zmm5
+; AVX512F-NEXT:    vmovdqa64 384(%r9), %zmm2
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm8
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm23, %zmm8
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm8 {%k1} = zmm13[0],zmm0[0],zmm13[2],zmm0[2],zmm13[4],zmm0[4],zmm13[6],zmm0[6]
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm6
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm25, %zmm6
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm6 {%k1} = zmm13[1],zmm0[1],zmm13[3],zmm0[3],zmm13[5],zmm0[5],zmm13[7],zmm0[7]
+; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm30
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm10, %zmm30
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm4, %zmm13
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm24
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm11, %zmm24
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm26
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm12, %zmm26
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm17
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm10, %zmm17
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm7 = zmm5[0],zmm2[0],zmm5[2],zmm2[2],zmm5[4],zmm2[4],zmm5[6],zmm2[6]
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm9 = zmm5[1],zmm2[1],zmm5[3],zmm2[3],zmm5[5],zmm2[5],zmm5[7],zmm2[7]
+; AVX512F-NEXT:    vpermt2q %zmm2, %zmm4, %zmm5
+; AVX512F-NEXT:    vmovdqa64 448(%r10), %zmm16
+; AVX512F-NEXT:    vmovdqa64 448(%rax), %zmm1
+; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm11, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm0
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm12, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vpermi2q %zmm1, %zmm16, %zmm19
+; AVX512F-NEXT:    vpermi2q %zmm1, %zmm16, %zmm21
+; AVX512F-NEXT:    vmovdqa64 448(%r8), %zmm3
+; AVX512F-NEXT:    vmovdqa64 448(%r9), %zmm0
+; AVX512F-NEXT:    vpermi2q %zmm0, %zmm3, %zmm23
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm23 {%k1} = zmm16[0],zmm1[0],zmm16[2],zmm1[2],zmm16[4],zmm1[4],zmm16[6],zmm1[6]
+; AVX512F-NEXT:    vpermi2q %zmm0, %zmm3, %zmm25
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm25 {%k1} = zmm16[1],zmm1[1],zmm16[3],zmm1[3],zmm16[5],zmm1[5],zmm16[7],zmm1[7]
+; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm18
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm10, %zmm18
+; AVX512F-NEXT:    vpermt2q %zmm1, %zmm4, %zmm16
+; AVX512F-NEXT:    vpermi2q %zmm0, %zmm3, %zmm11
+; AVX512F-NEXT:    vpermi2q %zmm0, %zmm3, %zmm12
+; AVX512F-NEXT:    vpermi2q %zmm0, %zmm3, %zmm10
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} zmm1 = zmm3[0],zmm0[0],zmm3[2],zmm0[2],zmm3[4],zmm0[4],zmm3[6],zmm0[6]
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} zmm2 = zmm3[1],zmm0[1],zmm3[3],zmm0[3],zmm3[5],zmm0[5],zmm3[7],zmm0[7]
+; AVX512F-NEXT:    vpermt2q %zmm0, %zmm4, %zmm3
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
 ; AVX512F-NEXT:    # ymm0 = ymm0[0,1,2,3],mem[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm4, %zmm0
+; AVX512F-NEXT:    vmovdqa64 %zmm14, %zmm7 {%k1}
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm7, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512F-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
 ; AVX512F-NEXT:    # ymm0 = ymm0[0,1,2,3],mem[4,5,6,7]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm5, %zmm0
+; AVX512F-NEXT:    vmovdqa64 %zmm15, %zmm9 {%k1}
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm9, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; AVX512F-NEXT:    # ymm0 = ymm0[0,1,2,3],mem[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm8, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; AVX512F-NEXT:    # ymm0 = ymm0[0,1,2,3],mem[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm6, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vpblendd $240, (%rsp), %ymm0, %ymm0 # 32-byte Folded Reload
+; AVX512F-NEXT:    # ymm0 = ymm0[0,1,2,3],mem[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 %zmm19, %zmm1 {%k1}
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm1, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; AVX512F-NEXT:    # ymm0 = ymm0[0,1,2,3],mem[4,5,6,7]
+; AVX512F-NEXT:    vmovdqa64 %zmm21, %zmm2 {%k1}
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm2, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; AVX512F-NEXT:    # ymm0 = ymm0[0,1,2,3],mem[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm23, %zmm0
+; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; AVX512F-NEXT:    # ymm0 = ymm0[0,1,2,3],mem[4,5,6,7]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm25, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm25 {%k1}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm8 {%k1}
 ; AVX512F-NEXT:    vmovdqa (%rcx), %ymm0
 ; AVX512F-NEXT:    vmovdqa (%rdx), %ymm1
 ; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX512F-NEXT:    vmovdqa (%rsi), %ymm3
-; AVX512F-NEXT:    vmovdqa (%rdi), %ymm7
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm8 = ymm7[0],ymm3[0],ymm7[2],ymm3[2]
-; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm2 = ymm8[2,3],ymm2[2,3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm25, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa (%rsi), %ymm4
+; AVX512F-NEXT:    vmovdqa (%rdi), %ymm6
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm7 = ymm6[0],ymm4[0],ymm6[2],ymm4[2]
+; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm2 = ymm7[2,3],ymm2[2,3]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm8, %zmm2
+; AVX512F-NEXT:    vmovdqu64 %zmm2, (%rsp) # 64-byte Spill
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm4 {%k1}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm7 {%k1}
 ; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm0 = ymm1[1],ymm0[1],ymm1[3],ymm0[3]
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm7[1],ymm3[1],ymm7[3],ymm3[3]
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm6[1],ymm4[1],ymm6[3],ymm4[3]
 ; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm1[2,3],ymm0[2,3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm4, %zmm0
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm7, %zmm0
 ; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm18 {%k1}
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm31 {%k1}
 ; AVX512F-NEXT:    vmovdqa 64(%rcx), %ymm0
 ; AVX512F-NEXT:    vmovdqa 64(%rdx), %ymm1
 ; AVX512F-NEXT:    vmovdqa 64(%rsi), %ymm2
-; AVX512F-NEXT:    vmovdqa 64(%rdi), %ymm3
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm7 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm8 = ymm3[0],ymm2[0],ymm3[2],ymm2[2]
-; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm7 = ymm8[2,3],ymm7[2,3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm7, %zmm18, %zmm4
-; AVX512F-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512F-NEXT:    vmovdqu64 (%rsp), %zmm4 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm5 {%k1}
+; AVX512F-NEXT:    vmovdqa 64(%rdi), %ymm4
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm6 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm7 = ymm4[0],ymm2[0],ymm4[2],ymm2[2]
+; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm6 = ymm7[2,3],ymm6[2,3]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm6, %zmm31, %zmm6
+; AVX512F-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm6, %zmm7 {%k1}
 ; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm0 = ymm1[1],ymm0[1],ymm1[3],ymm0[3]
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm3[1],ymm2[1],ymm3[3],ymm2[3]
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm4[1],ymm2[1],ymm4[3],ymm2[3]
 ; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm1[2,3],ymm0[2,3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm5, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm7, %zmm8
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm13 {%k1}
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm27 {%k1}
 ; AVX512F-NEXT:    vmovdqa 128(%rcx), %ymm0
 ; AVX512F-NEXT:    vmovdqa 128(%rdx), %ymm1
 ; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX512F-NEXT:    vmovdqa 128(%rsi), %ymm3
-; AVX512F-NEXT:    vmovdqa 128(%rdi), %ymm7
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm8 = ymm7[0],ymm3[0],ymm7[2],ymm3[2]
-; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm2 = ymm8[2,3],ymm2[2,3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm13, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, (%rsp) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa 128(%rsi), %ymm4
+; AVX512F-NEXT:    vmovdqa 128(%rdi), %ymm6
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm7 = ymm6[0],ymm4[0],ymm6[2],ymm4[2]
+; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm2 = ymm7[2,3],ymm2[2,3]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm27, %zmm23
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm4 {%k1}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm7 {%k1}
 ; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm0 = ymm1[1],ymm0[1],ymm1[3],ymm0[3]
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm7[1],ymm3[1],ymm7[3],ymm3[3]
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm6[1],ymm4[1],ymm6[3],ymm4[3]
 ; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm1[2,3],ymm0[2,3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm4, %zmm0
-; AVX512F-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm7, %zmm25
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm10 {%k1}
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm20 {%k1}
 ; AVX512F-NEXT:    vmovdqa 192(%rcx), %ymm0
 ; AVX512F-NEXT:    vmovdqa 192(%rdx), %ymm1
 ; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX512F-NEXT:    vmovdqa 192(%rsi), %ymm3
-; AVX512F-NEXT:    vmovdqa 192(%rdi), %ymm7
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm8 = ymm7[0],ymm3[0],ymm7[2],ymm3[2]
-; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm2 = ymm8[2,3],ymm2[2,3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm10, %zmm2
-; AVX512F-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512F-NEXT:    vmovdqa 192(%rsi), %ymm4
+; AVX512F-NEXT:    vmovdqa 192(%rdi), %ymm6
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm7 = ymm6[0],ymm4[0],ymm6[2],ymm4[2]
+; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm2 = ymm7[2,3],ymm2[2,3]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm20, %zmm21
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm4 {%k1}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm7 {%k1}
 ; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm0 = ymm1[1],ymm0[1],ymm1[3],ymm0[3]
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm7[1],ymm3[1],ymm7[3],ymm3[3]
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm6[1],ymm4[1],ymm6[3],ymm4[3]
 ; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm1[2,3],ymm0[2,3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm4, %zmm9
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm7, %zmm19
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm26 {%k1}
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm28 {%k1}
 ; AVX512F-NEXT:    vmovdqa 256(%rcx), %ymm0
 ; AVX512F-NEXT:    vmovdqa 256(%rdx), %ymm1
 ; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX512F-NEXT:    vmovdqa 256(%rsi), %ymm3
-; AVX512F-NEXT:    vmovdqa 256(%rdi), %ymm7
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm8 = ymm7[0],ymm3[0],ymm7[2],ymm3[2]
-; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm2 = ymm8[2,3],ymm2[2,3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm26, %zmm26
+; AVX512F-NEXT:    vmovdqa 256(%rsi), %ymm4
+; AVX512F-NEXT:    vmovdqa 256(%rdi), %ymm6
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm7 = ymm6[0],ymm4[0],ymm6[2],ymm4[2]
+; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm2 = ymm7[2,3],ymm2[2,3]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm28, %zmm15
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm23, %zmm2 {%k1}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm7 {%k1}
 ; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm0 = ymm1[1],ymm0[1],ymm1[3],ymm0[3]
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm7[1],ymm3[1],ymm7[3],ymm3[3]
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm6[1],ymm4[1],ymm6[3],ymm4[3]
 ; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm1[2,3],ymm0[2,3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm2, %zmm7
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm7, %zmm28
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm24 {%k1}
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm22 {%k1}
 ; AVX512F-NEXT:    vmovdqa 320(%rcx), %ymm0
 ; AVX512F-NEXT:    vmovdqa 320(%rdx), %ymm1
 ; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX512F-NEXT:    vmovdqa 320(%rsi), %ymm3
-; AVX512F-NEXT:    vmovdqa64 320(%rdi), %ymm23
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm8 = ymm23[0],ymm3[0],ymm23[2],ymm3[2]
-; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm2 = ymm8[2,3],ymm2[2,3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm24, %zmm8
-; AVX512F-NEXT:    vmovdqa64 %zmm31, %zmm17 {%k1}
+; AVX512F-NEXT:    vmovdqa 320(%rsi), %ymm4
+; AVX512F-NEXT:    vmovdqa 320(%rdi), %ymm6
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm7 = ymm6[0],ymm4[0],ymm6[2],ymm4[2]
+; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm2 = ymm7[2,3],ymm2[2,3]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm22, %zmm22
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm2, %zmm29 {%k1}
 ; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm0 = ymm1[1],ymm0[1],ymm1[3],ymm0[3]
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm23[1],ymm3[1],ymm23[3],ymm3[3]
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm6[1],ymm4[1],ymm6[3],ymm4[3]
 ; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm1[2,3],ymm0[2,3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm17, %zmm17
-; AVX512F-NEXT:    vmovdqa64 %zmm22, %zmm20 {%k1}
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm29, %zmm27
+; AVX512F-NEXT:    vmovdqa64 %zmm30, %zmm17 {%k1}
 ; AVX512F-NEXT:    vmovdqa 384(%rcx), %ymm0
 ; AVX512F-NEXT:    vmovdqa 384(%rdx), %ymm1
 ; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX512F-NEXT:    vmovdqa 384(%rsi), %ymm3
-; AVX512F-NEXT:    vmovdqa64 384(%rdi), %ymm18
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm13 = ymm18[0],ymm3[0],ymm18[2],ymm3[2]
-; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm2 = ymm13[2,3],ymm2[2,3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm20, %zmm20
-; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm30 {%k1}
+; AVX512F-NEXT:    vmovdqa 384(%rsi), %ymm4
+; AVX512F-NEXT:    vmovdqa 384(%rdi), %ymm6
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm7 = ymm6[0],ymm4[0],ymm6[2],ymm4[2]
+; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm2 = ymm7[2,3],ymm2[2,3]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm17, %zmm14
+; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm5 {%k1}
 ; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm0 = ymm1[1],ymm0[1],ymm1[3],ymm0[3]
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm18[1],ymm3[1],ymm18[3],ymm3[3]
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm6[1],ymm4[1],ymm6[3],ymm4[3]
 ; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm1[2,3],ymm0[2,3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm30, %zmm24
-; AVX512F-NEXT:    vmovdqa64 %zmm12, %zmm6 {%k1}
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm5, %zmm13
+; AVX512F-NEXT:    vmovdqa64 %zmm18, %zmm10 {%k1}
 ; AVX512F-NEXT:    vmovdqa 448(%rcx), %ymm0
 ; AVX512F-NEXT:    vmovdqa 448(%rdx), %ymm1
 ; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX512F-NEXT:    vmovdqa 448(%rsi), %ymm3
-; AVX512F-NEXT:    vmovdqa 448(%rdi), %ymm10
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm12 = ymm10[0],ymm3[0],ymm10[2],ymm3[2]
-; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm2 = ymm12[2,3],ymm2[2,3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm6, %zmm31
+; AVX512F-NEXT:    vmovdqa 448(%rsi), %ymm4
+; AVX512F-NEXT:    vmovdqa 448(%rdi), %ymm5
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm6 = ymm5[0],ymm4[0],ymm5[2],ymm4[2]
+; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm2 = ymm6[2,3],ymm2[2,3]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm2, %zmm10, %zmm10
 ; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm0 = ymm1[1],ymm0[1],ymm1[3],ymm0[3]
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm10[1],ymm3[1],ymm10[3],ymm3[3]
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm5[1],ymm4[1],ymm5[3],ymm4[3]
 ; AVX512F-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm1[2,3],ymm0[2,3]
-; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm19 {%k1}
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm19, %zmm0
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
-; AVX512F-NEXT:    vmovdqa (%rsi), %xmm1
-; AVX512F-NEXT:    vinserti128 $1, (%rcx), %ymm1, %ymm1
-; AVX512F-NEXT:    vmovdqa (%rdi), %xmm2
-; AVX512F-NEXT:    vinserti128 $1, (%rdx), %ymm2, %ymm2
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm3 = ymm2[0],ymm1[0],ymm2[2],ymm1[2]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm3, %zmm4, %zmm3
+; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm3 {%k1}
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm3, %zmm9
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm0, %zmm3 {%k1}
+; AVX512F-NEXT:    vmovdqa (%rsi), %xmm0
+; AVX512F-NEXT:    vinserti128 $1, (%rcx), %ymm0, %ymm1
+; AVX512F-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512F-NEXT:    vinserti128 $1, (%rdx), %ymm0, %ymm2
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm2[0],ymm1[0],ymm2[2],ymm1[2]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm0, %zmm3, %zmm0
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm4, %zmm10 {%k1}
+; AVX512F-NEXT:    vmovdqa64 %zmm3, %zmm4 {%k1}
 ; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm2[1],ymm1[1],ymm2[3],ymm1[3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm10, %zmm2
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm4, %zmm2
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm11 {%k1}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm5 {%k1}
 ; AVX512F-NEXT:    vmovdqa 64(%rsi), %xmm1
 ; AVX512F-NEXT:    vinserti128 $1, 64(%rcx), %ymm1, %ymm1
-; AVX512F-NEXT:    vmovdqa 64(%rdi), %xmm4
-; AVX512F-NEXT:    vinserti128 $1, 64(%rdx), %ymm4, %ymm4
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm10 = ymm4[0],ymm1[0],ymm4[2],ymm1[2]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm10, %zmm11, %zmm10
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm11, %zmm12 {%k1}
+; AVX512F-NEXT:    vmovdqa 64(%rdi), %xmm3
+; AVX512F-NEXT:    vinserti128 $1, 64(%rdx), %ymm3, %ymm4
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm3 = ymm4[0],ymm1[0],ymm4[2],ymm1[2]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm3, %zmm5, %zmm3
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm6 {%k1}
 ; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm4[1],ymm1[1],ymm4[3],ymm1[3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm12, %zmm11
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm6, %zmm5
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm13 {%k1}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm7 {%k1}
 ; AVX512F-NEXT:    vmovdqa 128(%rsi), %xmm1
 ; AVX512F-NEXT:    vinserti128 $1, 128(%rcx), %ymm1, %ymm1
 ; AVX512F-NEXT:    vmovdqa 128(%rdi), %xmm4
-; AVX512F-NEXT:    vinserti128 $1, 128(%rdx), %ymm4, %ymm12
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm4 = ymm12[0],ymm1[0],ymm12[2],ymm1[2]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm4, %zmm13, %zmm4
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm18 {%k1}
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm12[1],ymm1[1],ymm12[3],ymm1[3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm18, %zmm19
+; AVX512F-NEXT:    vinserti128 $1, 128(%rdx), %ymm4, %ymm6
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm4 = ymm6[0],ymm1[0],ymm6[2],ymm1[2]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm4, %zmm7, %zmm4
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm7, %zmm16 {%k1}
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm6[1],ymm1[1],ymm6[3],ymm1[3]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm16, %zmm6
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm18 {%k1}
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm1, %zmm17 {%k1}
 ; AVX512F-NEXT:    vmovdqa 192(%rsi), %xmm1
 ; AVX512F-NEXT:    vinserti128 $1, 192(%rcx), %ymm1, %ymm1
-; AVX512F-NEXT:    vmovdqa 192(%rdi), %xmm12
-; AVX512F-NEXT:    vinserti128 $1, 192(%rdx), %ymm12, %ymm12
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm13 = ymm12[0],ymm1[0],ymm12[2],ymm1[2]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm13, %zmm18, %zmm30
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa 192(%rdi), %xmm7
+; AVX512F-NEXT:    vinserti32x4 $1, 192(%rdx), %ymm7, %ymm16
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm7 = ymm16[0],ymm1[0],ymm16[2],ymm1[2]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm7, %zmm17, %zmm7
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
 ; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm13, %zmm18 {%k1}
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm12[1],ymm1[1],ymm12[3],ymm1[3]
+; AVX512F-NEXT:    vmovdqa64 %zmm17, %zmm18 {%k1}
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm16[1],ymm1[1],ymm16[3],ymm1[3]
 ; AVX512F-NEXT:    vinserti64x4 $0, %ymm1, %zmm18, %zmm1
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm12, %zmm5 {%k1}
-; AVX512F-NEXT:    vmovdqa 256(%rsi), %xmm12
-; AVX512F-NEXT:    vinserti128 $1, 256(%rcx), %ymm12, %ymm13
-; AVX512F-NEXT:    vmovdqa 256(%rdi), %xmm12
-; AVX512F-NEXT:    vinserti32x4 $1, 256(%rdx), %ymm12, %ymm18
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm12 = ymm18[0],ymm13[0],ymm18[2],ymm13[2]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm12, %zmm5, %zmm12
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm23 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm23, %zmm5 {%k1}
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm13 = ymm18[1],ymm13[1],ymm18[3],ymm13[3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm13, %zmm5, %zmm23
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm29 {%k1}
-; AVX512F-NEXT:    vmovdqa 320(%rsi), %xmm13
-; AVX512F-NEXT:    vinserti128 $1, 320(%rcx), %ymm13, %ymm13
-; AVX512F-NEXT:    vmovdqa64 320(%rdi), %xmm18
-; AVX512F-NEXT:    vinserti32x4 $1, 320(%rdx), %ymm18, %ymm18
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm27 = ymm18[0],ymm13[0],ymm18[2],ymm13[2]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm27, %zmm29, %zmm22
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm6 {%k1}
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm13 = ymm18[1],ymm13[1],ymm18[3],ymm13[3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm13, %zmm6, %zmm13
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm21 {%k1}
-; AVX512F-NEXT:    vmovdqa64 384(%rsi), %xmm18
-; AVX512F-NEXT:    vinserti32x4 $1, 384(%rcx), %ymm18, %ymm18
-; AVX512F-NEXT:    vmovdqa64 384(%rdi), %xmm25
-; AVX512F-NEXT:    vinserti32x4 $1, 384(%rdx), %ymm25, %ymm25
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm27 = ymm25[0],ymm18[0],ymm25[2],ymm18[2]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm27, %zmm21, %zmm16
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm28 {%k1}
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm18 = ymm25[1],ymm18[1],ymm25[3],ymm18[3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm18, %zmm28, %zmm21
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm15 {%k1}
-; AVX512F-NEXT:    vmovdqa64 448(%rsi), %xmm18
-; AVX512F-NEXT:    vinserti32x4 $1, 448(%rcx), %ymm18, %ymm18
-; AVX512F-NEXT:    vmovdqa64 448(%rdi), %xmm25
-; AVX512F-NEXT:    vinserti32x4 $1, 448(%rdx), %ymm25, %ymm25
-; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm27 = ymm25[0],ymm18[0],ymm25[2],ymm18[2]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm27, %zmm15, %zmm6
-; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512F-NEXT:    vmovdqa64 %zmm5, %zmm14 {%k1}
-; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm18 = ymm25[1],ymm18[1],ymm25[3],ymm18[3]
-; AVX512F-NEXT:    vinserti64x4 $0, %ymm18, %zmm14, %zmm5
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm20 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm20 {%k1}
+; AVX512F-NEXT:    vmovdqa64 256(%rsi), %xmm16
+; AVX512F-NEXT:    vinserti32x4 $1, 256(%rcx), %ymm16, %ymm16
+; AVX512F-NEXT:    vmovdqa64 256(%rdi), %xmm17
+; AVX512F-NEXT:    vinserti32x4 $1, 256(%rdx), %ymm17, %ymm17
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm18 = ymm17[0],ymm16[0],ymm17[2],ymm16[2]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm18, %zmm20, %zmm29
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm20 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm18, %zmm20 {%k1}
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm16 = ymm17[1],ymm16[1],ymm17[3],ymm16[3]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm16, %zmm20, %zmm30
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm20 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm16, %zmm20 {%k1}
+; AVX512F-NEXT:    vmovdqa64 320(%rsi), %xmm16
+; AVX512F-NEXT:    vinserti32x4 $1, 320(%rcx), %ymm16, %ymm16
+; AVX512F-NEXT:    vmovdqa64 320(%rdi), %xmm17
+; AVX512F-NEXT:    vinserti32x4 $1, 320(%rdx), %ymm17, %ymm17
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm18 = ymm17[0],ymm16[0],ymm17[2],ymm16[2]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm18, %zmm20, %zmm18
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm20 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm31 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm20, %zmm31 {%k1}
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm16 = ymm17[1],ymm16[1],ymm17[3],ymm16[3]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm16, %zmm31, %zmm16
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm17, %zmm24 {%k1}
+; AVX512F-NEXT:    vmovdqa64 384(%rsi), %xmm17
+; AVX512F-NEXT:    vinserti32x4 $1, 384(%rcx), %ymm17, %ymm17
+; AVX512F-NEXT:    vmovdqa64 384(%rdi), %xmm20
+; AVX512F-NEXT:    vinserti32x4 $1, 384(%rdx), %ymm20, %ymm20
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm31 = ymm20[0],ymm17[0],ymm20[2],ymm17[2]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm31, %zmm24, %zmm24
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm31 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm31, %zmm26 {%k1}
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm17 = ymm20[1],ymm17[1],ymm20[3],ymm17[3]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm17, %zmm26, %zmm17
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm20 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm20, %zmm11 {%k1}
+; AVX512F-NEXT:    vmovdqa64 448(%rsi), %xmm20
+; AVX512F-NEXT:    vinserti32x4 $1, 448(%rcx), %ymm20, %ymm20
+; AVX512F-NEXT:    vmovdqa64 448(%rdi), %xmm26
+; AVX512F-NEXT:    vinserti32x4 $1, 448(%rdx), %ymm26, %ymm26
+; AVX512F-NEXT:    vpunpcklqdq {{.*#+}} ymm31 = ymm26[0],ymm20[0],ymm26[2],ymm20[2]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm31, %zmm11, %zmm11
+; AVX512F-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm31 # 64-byte Reload
+; AVX512F-NEXT:    vmovdqa64 %zmm31, %zmm12 {%k1}
+; AVX512F-NEXT:    vpunpckhqdq {{.*#+}} ymm20 = ymm26[1],ymm20[1],ymm26[3],ymm20[3]
+; AVX512F-NEXT:    vinserti64x4 $0, %ymm20, %zmm12, %zmm12
 ; AVX512F-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512F-NEXT:    vmovdqa64 %zmm0, 3776(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm31, 3712(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm24, 3264(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm20, 3200(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm17, 2752(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm8, 2688(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm7, 2240(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm26, 2176(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm9, 1728(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 1664(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 1216(%rax)
-; AVX512F-NEXT:    vmovups (%rsp), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 1152(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 704(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 640(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 192(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 128(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 4032(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 3968(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 3904(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 3840(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm5, 3648(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm6, 3584(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 3520(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 3456(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 3392(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 3328(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm21, 3136(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm16, 3072(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 3008(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 2944(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 2880(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 2816(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm13, 2624(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm22, 2560(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 2496(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 2432(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 2368(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 2304(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm23, 2112(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm12, 2048(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 1984(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 1920(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 1856(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 1792(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm9, 3776(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm10, 3712(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm13, 3264(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm14, 3200(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm27, 2752(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm22, 2688(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm28, 2240(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm15, 2176(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm19, 1728(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm21, 1664(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm25, 1216(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm23, 1152(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm8, 704(%rax)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm8, 640(%rax)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm8, 192(%rax)
+; AVX512F-NEXT:    vmovups (%rsp), %zmm8 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm8, 128(%rax)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm8, 4032(%rax)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm8, 3968(%rax)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm8, 3904(%rax)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm8, 3840(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm12, 3648(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm11, 3584(%rax)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm8, 3520(%rax)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm8, 3456(%rax)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm8, 3392(%rax)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm8, 3328(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm17, 3136(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm24, 3072(%rax)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm8, 3008(%rax)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm8, 2944(%rax)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm8, 2880(%rax)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm8, 2816(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm16, 2624(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm18, 2560(%rax)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm8, 2496(%rax)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm8, 2432(%rax)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm8, 2368(%rax)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm8, 2304(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm30, 2112(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm29, 2048(%rax)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm8, 1984(%rax)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm8, 1920(%rax)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm8, 1856(%rax)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm8, 1792(%rax)
 ; AVX512F-NEXT:    vmovdqa64 %zmm1, 1600(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm30, 1536(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 1472(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 1408(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 1344(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 1280(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm19, 1088(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm7, 1536(%rax)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm1, 1472(%rax)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm1, 1408(%rax)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm1, 1344(%rax)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm1, 1280(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm6, 1088(%rax)
 ; AVX512F-NEXT:    vmovdqa64 %zmm4, 1024(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 960(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 896(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 832(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 768(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm11, 576(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm10, 512(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 448(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 384(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 320(%rax)
-; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512F-NEXT:    vmovaps %zmm0, 256(%rax)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm1, 960(%rax)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm1, 896(%rax)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm1, 832(%rax)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm1, 768(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm5, 576(%rax)
+; AVX512F-NEXT:    vmovdqa64 %zmm3, 512(%rax)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm1, 448(%rax)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm1, 384(%rax)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm1, 320(%rax)
+; AVX512F-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512F-NEXT:    vmovaps %zmm1, 256(%rax)
 ; AVX512F-NEXT:    vmovdqa64 %zmm2, 64(%rax)
-; AVX512F-NEXT:    vmovdqa64 %zmm3, (%rax)
-; AVX512F-NEXT:    addq $5512, %rsp # imm = 0x1588
+; AVX512F-NEXT:    vmovdqa64 %zmm0, (%rax)
+; AVX512F-NEXT:    addq $5384, %rsp # imm = 0x1508
 ; AVX512F-NEXT:    vzeroupper
 ; AVX512F-NEXT:    retq
 ;
 ; AVX512BW-LABEL: store_i64_stride8_vf64:
 ; AVX512BW:       # %bb.0:
-; AVX512BW-NEXT:    subq $5512, %rsp # imm = 0x1588
+; AVX512BW-NEXT:    subq $5384, %rsp # imm = 0x1508
 ; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %r10
-; AVX512BW-NEXT:    vmovdqa64 128(%rdi), %zmm2
+; AVX512BW-NEXT:    vmovdqa64 128(%rdi), %zmm0
 ; AVX512BW-NEXT:    vmovdqa64 64(%rdi), %zmm4
-; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm8
-; AVX512BW-NEXT:    vmovdqa64 128(%rsi), %zmm17
-; AVX512BW-NEXT:    vmovdqa64 64(%rsi), %zmm19
-; AVX512BW-NEXT:    vmovdqa64 (%rsi), %zmm22
-; AVX512BW-NEXT:    vmovdqa64 64(%rdx), %zmm5
-; AVX512BW-NEXT:    vmovdqa64 (%rdx), %zmm10
-; AVX512BW-NEXT:    vmovdqa64 64(%rcx), %zmm20
-; AVX512BW-NEXT:    vmovdqa64 (%rcx), %zmm11
-; AVX512BW-NEXT:    vmovdqa64 (%r8), %zmm1
-; AVX512BW-NEXT:    vmovdqa64 64(%r8), %zmm25
-; AVX512BW-NEXT:    vmovdqa64 128(%r8), %zmm23
-; AVX512BW-NEXT:    vmovdqa64 (%r9), %zmm28
-; AVX512BW-NEXT:    vmovdqa64 64(%r9), %zmm26
-; AVX512BW-NEXT:    vmovdqa64 128(%r9), %zmm24
-; AVX512BW-NEXT:    vmovdqa64 (%r10), %zmm21
-; AVX512BW-NEXT:    vmovdqa64 64(%r10), %zmm14
-; AVX512BW-NEXT:    vmovdqa64 (%rax), %zmm27
-; AVX512BW-NEXT:    vmovdqa64 64(%rax), %zmm16
+; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm14
+; AVX512BW-NEXT:    vmovdqa64 128(%rsi), %zmm1
+; AVX512BW-NEXT:    vmovdqa64 64(%rsi), %zmm5
+; AVX512BW-NEXT:    vmovdqa64 (%rsi), %zmm10
+; AVX512BW-NEXT:    vmovdqa64 128(%rdx), %zmm2
+; AVX512BW-NEXT:    vmovdqa64 64(%rdx), %zmm7
+; AVX512BW-NEXT:    vmovdqa64 (%rdx), %zmm12
+; AVX512BW-NEXT:    vmovdqa64 128(%rcx), %zmm3
+; AVX512BW-NEXT:    vmovdqa64 64(%rcx), %zmm9
+; AVX512BW-NEXT:    vmovdqa64 (%rcx), %zmm15
+; AVX512BW-NEXT:    vmovdqa64 (%r8), %zmm30
+; AVX512BW-NEXT:    vmovdqa64 64(%r8), %zmm18
+; AVX512BW-NEXT:    vmovdqa64 128(%r8), %zmm11
+; AVX512BW-NEXT:    vmovdqa64 (%r9), %zmm24
+; AVX512BW-NEXT:    vmovdqa64 64(%r9), %zmm28
+; AVX512BW-NEXT:    vmovdqa64 128(%r9), %zmm22
+; AVX512BW-NEXT:    vmovdqa64 (%r10), %zmm26
+; AVX512BW-NEXT:    vmovdqa64 64(%r10), %zmm31
+; AVX512BW-NEXT:    vmovdqa64 128(%r10), %zmm16
+; AVX512BW-NEXT:    vmovdqa64 (%rax), %zmm17
+; AVX512BW-NEXT:    vmovdqa64 64(%rax), %zmm27
+; AVX512BW-NEXT:    vmovdqa64 128(%rax), %zmm13
 ; AVX512BW-NEXT:    movb $-64, %r11b
 ; AVX512BW-NEXT:    kmovd %r11d, %k1
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm3 = [4,12,4,12,4,12,4,12]
-; AVX512BW-NEXT:    # zmm3 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm21, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm27, %zmm3, %zmm0
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm12 = zmm1[0],zmm28[0],zmm1[2],zmm28[2],zmm1[4],zmm28[4],zmm1[6],zmm28[6]
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm12 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm3, %zmm0
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm6 = [4,12,4,12]
-; AVX512BW-NEXT:    # ymm6 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm15
-; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm6, %zmm15
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm15[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm12, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm9 = [5,13,5,13,5,13,5,13]
-; AVX512BW-NEXT:    # zmm9 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm21, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm27, %zmm9, %zmm0
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm12 = zmm1[1],zmm28[1],zmm1[3],zmm28[3],zmm1[5],zmm28[5],zmm1[7],zmm28[7]
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm12 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm9, %zmm0
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm7 = [5,13,5,13]
-; AVX512BW-NEXT:    # ymm7 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm15
-; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm7, %zmm15
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm15[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm12, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm13 = [6,14,6,14,6,14,6,14]
-; AVX512BW-NEXT:    # zmm13 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm0
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm29
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm28, %zmm13, %zmm0
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm0 {%k1} = zmm21[0],zmm27[0],zmm21[2],zmm27[2],zmm21[4],zmm27[4],zmm21[6],zmm27[6]
-; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm12
-; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm13, %zmm12
-; AVX512BW-NEXT:    vbroadcasti128 {{.*#+}} ymm1 = [6,14,6,14]
-; AVX512BW-NEXT:    # ymm1 = mem[0,1,0,1]
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm15
-; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm1, %zmm15
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm12 = ymm12[0,1,2,3],ymm15[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm12, %zmm0, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm18 = [7,15,7,15,7,15,7,15]
-; AVX512BW-NEXT:    # zmm18 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm29, %zmm12
-; AVX512BW-NEXT:    vpermt2q %zmm28, %zmm18, %zmm12
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm12 {%k1} = zmm21[1],zmm27[1],zmm21[3],zmm27[3],zmm21[5],zmm27[5],zmm21[7],zmm27[7]
-; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm18, %zmm8
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} ymm30 = [7,15,7,15]
-; AVX512BW-NEXT:    # ymm30 = mem[0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermt2q %zmm11, %zmm30, %zmm10
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3],ymm10[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm8, %zmm12, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm19 = [4,12,4,12,4,12,4,12]
+; AVX512BW-NEXT:    # zmm19 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm8
+; AVX512BW-NEXT:    vpermt2q %zmm15, %zmm19, %zmm8
+; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm6
+; AVX512BW-NEXT:    vpermt2q %zmm10, %zmm19, %zmm6
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3],ymm8[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm26, %zmm8
+; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm19, %zmm8
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm20 = zmm30[0],zmm24[0],zmm30[2],zmm24[2],zmm30[4],zmm24[4],zmm30[6],zmm24[6]
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm20 {%k1}
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm6, %zmm20, %zmm6
+; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm21 = [5,13,5,13,5,13,5,13]
+; AVX512BW-NEXT:    # zmm21 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm6
+; AVX512BW-NEXT:    vpermt2q %zmm15, %zmm21, %zmm6
 ; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm8
-; AVX512BW-NEXT:    vpermt2q %zmm16, %zmm3, %zmm8
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm10 = zmm25[0],zmm26[0],zmm25[2],zmm26[2],zmm25[4],zmm26[4],zmm25[6],zmm26[6]
-; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm10 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm8
-; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm3, %zmm8
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm11
-; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm15
-; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm6, %zmm11
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3],ymm11[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm8, %zmm10, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm10, %zmm21, %zmm8
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm6 = ymm8[0,1,2,3],ymm6[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm26, %zmm8
+; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm21, %zmm8
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm20 = zmm30[1],zmm24[1],zmm30[3],zmm24[3],zmm30[5],zmm24[5],zmm30[7],zmm24[7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm20 {%k1}
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm6, %zmm20, %zmm6
+; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm23 = [6,14,6,14,6,14,6,14]
+; AVX512BW-NEXT:    # zmm23 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm6
+; AVX512BW-NEXT:    vpermt2q %zmm15, %zmm23, %zmm6
 ; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm8
-; AVX512BW-NEXT:    vpermt2q %zmm16, %zmm9, %zmm8
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm10 = zmm25[1],zmm26[1],zmm25[3],zmm26[3],zmm25[5],zmm26[5],zmm25[7],zmm26[7]
-; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm10 {%k1}
+; AVX512BW-NEXT:    vpermt2q %zmm10, %zmm23, %zmm8
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm6 = ymm8[0,1,2,3],ymm6[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm30, %zmm8
+; AVX512BW-NEXT:    vpermt2q %zmm24, %zmm23, %zmm8
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm8 {%k1} = zmm26[0],zmm17[0],zmm26[2],zmm17[2],zmm26[4],zmm17[4],zmm26[6],zmm17[6]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm6, %zmm8, %zmm6
+; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm25 = [7,15,7,15,7,15,7,15]
+; AVX512BW-NEXT:    # zmm25 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpermt2q %zmm15, %zmm25, %zmm12
+; AVX512BW-NEXT:    vpermt2q %zmm10, %zmm25, %zmm14
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm6 = ymm14[0,1,2,3],ymm12[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm30, %zmm8
+; AVX512BW-NEXT:    vpermt2q %zmm24, %zmm25, %zmm8
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm8 {%k1} = zmm26[1],zmm17[1],zmm26[3],zmm17[3],zmm26[5],zmm17[5],zmm26[7],zmm17[7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm6, %zmm8, %zmm6
+; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm6
+; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm19, %zmm6
 ; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm8
-; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm9, %zmm8
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm11
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm12
-; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm7, %zmm11
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3],ymm11[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm8, %zmm10, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm19, %zmm8
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm6 = ymm8[0,1,2,3],ymm6[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm31, %zmm8
+; AVX512BW-NEXT:    vpermt2q %zmm27, %zmm19, %zmm8
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm10 = zmm18[0],zmm28[0],zmm18[2],zmm28[2],zmm18[4],zmm28[4],zmm18[6],zmm28[6]
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm10 {%k1}
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm6, %zmm10, %zmm6
+; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm6
+; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm21, %zmm6
 ; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm8
-; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm13, %zmm8
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm10
-; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm1, %zmm10
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm8[0,1,2,3],ymm10[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqu64 %zmm25, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm25, %zmm10
-; AVX512BW-NEXT:    vmovdqu64 %zmm26, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm26, %zmm13, %zmm10
-; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm11
-; AVX512BW-NEXT:    vmovdqu64 %zmm14, (%rsp) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm10 {%k1} = zmm14[0],zmm16[0],zmm14[2],zmm16[2],zmm14[4],zmm16[4],zmm14[6],zmm16[6]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm8, %zmm10, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 128(%r10), %zmm10
-; AVX512BW-NEXT:    vpermt2q %zmm19, %zmm18, %zmm4
-; AVX512BW-NEXT:    vmovdqa64 128(%rax), %zmm14
-; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm30, %zmm5
-; AVX512BW-NEXT:    vmovdqa64 %zmm25, %zmm6
-; AVX512BW-NEXT:    vpermt2q %zmm26, %zmm18, %zmm6
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm6 {%k1} = zmm11[1],zmm16[1],zmm11[3],zmm16[3],zmm11[5],zmm16[5],zmm11[7],zmm16[7]
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm5[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm4, %zmm6, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm4
-; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm3, %zmm4
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm5 = zmm23[0],zmm24[0],zmm23[2],zmm24[2],zmm23[4],zmm24[4],zmm23[6],zmm24[6]
-; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm5 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm6
-; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm3, %zmm6
-; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm11
-; AVX512BW-NEXT:    vmovdqa64 128(%rdx), %zmm4
-; AVX512BW-NEXT:    vmovdqa64 128(%rcx), %zmm7
+; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm21, %zmm8
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm6 = ymm8[0,1,2,3],ymm6[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm31, %zmm8
+; AVX512BW-NEXT:    vpermt2q %zmm27, %zmm21, %zmm8
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm10 = zmm18[1],zmm28[1],zmm18[3],zmm28[3],zmm18[5],zmm28[5],zmm18[7],zmm28[7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm10 {%k1}
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm6, %zmm10, %zmm6
+; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm6
+; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm23, %zmm6
 ; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm8
-; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm15, %zmm8
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3],ymm8[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm6, %zmm5, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm5
-; AVX512BW-NEXT:    vpermt2q %zmm14, %zmm9, %zmm5
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm6 = zmm23[1],zmm24[1],zmm23[3],zmm24[3],zmm23[5],zmm24[5],zmm23[7],zmm24[7]
+; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm23, %zmm8
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm6 = ymm8[0,1,2,3],ymm6[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqu64 %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm18, %zmm8
+; AVX512BW-NEXT:    vpermt2q %zmm28, %zmm23, %zmm8
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm8 {%k1} = zmm31[0],zmm27[0],zmm31[2],zmm27[2],zmm31[4],zmm27[4],zmm31[6],zmm27[6]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm6, %zmm8, %zmm6
+; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm25, %zmm7
+; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm25, %zmm4
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm7[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm18, %zmm5
+; AVX512BW-NEXT:    vpermt2q %zmm28, %zmm25, %zmm5
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm5 {%k1} = zmm31[1],zmm27[1],zmm31[3],zmm27[3],zmm31[5],zmm27[5],zmm31[7],zmm27[7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm4, %zmm5, %zmm4
+; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm4
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm19, %zmm4
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm5
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm19, %zmm5
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3],ymm4[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm5
+; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm19, %zmm5
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm6 = zmm11[0],zmm22[0],zmm11[2],zmm22[2],zmm11[4],zmm22[4],zmm11[6],zmm22[6]
 ; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm6 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm5
-; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm9, %zmm5
-; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm8
-; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm12, %zmm8
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm8[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm5, %zmm6, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm5
-; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm13, %zmm5
-; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm6
-; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm1, %zmm6
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm6[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqa64 %zmm23, %zmm3
-; AVX512BW-NEXT:    vmovdqu64 %zmm23, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm23, %zmm6
-; AVX512BW-NEXT:    vmovdqu64 %zmm24, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm24, %zmm13, %zmm6
-; AVX512BW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 %zmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm6 {%k1} = zmm10[0],zmm14[0],zmm10[2],zmm14[2],zmm10[4],zmm14[4],zmm10[6],zmm14[6]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm5, %zmm6, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm18, %zmm2
-; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm30, %zmm4
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm4[4,5,6,7]
-; AVX512BW-NEXT:    vpermt2q %zmm24, %zmm18, %zmm3
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm3 {%k1} = zmm10[1],zmm14[1],zmm10[3],zmm14[3],zmm10[5],zmm14[5],zmm10[7],zmm14[7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm3, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 192(%r10), %zmm10
-; AVX512BW-NEXT:    vmovdqa64 192(%rax), %zmm26
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm2
-; AVX512BW-NEXT:    vpermt2q %zmm26, %zmm11, %zmm2
-; AVX512BW-NEXT:    vmovdqa64 192(%r8), %zmm23
-; AVX512BW-NEXT:    vmovdqa64 192(%r9), %zmm29
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm4 = zmm23[0],zmm29[0],zmm23[2],zmm29[2],zmm23[4],zmm29[4],zmm23[6],zmm29[6]
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm4 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 192(%rdi), %zmm2
-; AVX512BW-NEXT:    vmovdqa64 192(%rsi), %zmm5
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm6
-; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm11, %zmm6
-; AVX512BW-NEXT:    vmovdqa64 192(%rdx), %zmm3
-; AVX512BW-NEXT:    vmovdqa64 192(%rcx), %zmm7
-; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm8
-; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm15, %zmm8
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3],ymm8[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm6, %zmm4, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm4
-; AVX512BW-NEXT:    vpermt2q %zmm26, %zmm9, %zmm4
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm6 = zmm23[1],zmm29[1],zmm23[3],zmm29[3],zmm23[5],zmm29[5],zmm23[7],zmm29[7]
-; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm6 {%k1}
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm4, %zmm6, %zmm4
+; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm4
-; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm9, %zmm4
-; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm8
-; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm12, %zmm8
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm8[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm4, %zmm6, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm21, %zmm4
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm5
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm21, %zmm5
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3],ymm4[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm5
+; AVX512BW-NEXT:    vpermt2q %zmm13, %zmm21, %zmm5
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm6 = zmm11[1],zmm22[1],zmm11[3],zmm22[3],zmm11[5],zmm22[5],zmm11[7],zmm22[7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm6 {%k1}
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm4, %zmm6, %zmm4
+; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm4
-; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm13, %zmm4
-; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm6
-; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm1, %zmm6
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm6[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqa64 %zmm23, %zmm6
-; AVX512BW-NEXT:    vpermt2q %zmm29, %zmm13, %zmm6
-; AVX512BW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm6 {%k1} = zmm10[0],zmm26[0],zmm10[2],zmm26[2],zmm10[4],zmm26[4],zmm10[6],zmm26[6]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm4, %zmm6, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm18, %zmm2
-; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm30, %zmm3
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqa64 %zmm23, %zmm3
-; AVX512BW-NEXT:    vpermt2q %zmm29, %zmm18, %zmm3
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm3 {%k1} = zmm10[1],zmm26[1],zmm10[3],zmm26[3],zmm10[5],zmm26[5],zmm10[7],zmm26[7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm3, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm23, %zmm4
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm5
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm23, %zmm5
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm5[0,1,2,3],ymm4[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm5
+; AVX512BW-NEXT:    vmovdqu64 %zmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 %zmm22, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm23, %zmm5
+; AVX512BW-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm5 {%k1} = zmm16[0],zmm13[0],zmm16[2],zmm13[2],zmm16[4],zmm13[4],zmm16[6],zmm13[6]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm4, %zmm5, %zmm4
+; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 192(%rdi), %zmm4
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm25, %zmm2
+; AVX512BW-NEXT:    vmovdqa64 192(%rdx), %zmm3
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm25, %zmm0
+; AVX512BW-NEXT:    vmovdqa64 192(%rcx), %zmm1
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm2
+; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm25, %zmm2
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm2 {%k1} = zmm16[1],zmm13[1],zmm16[3],zmm13[3],zmm16[5],zmm13[5],zmm16[7],zmm13[7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm2, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 256(%r10), %zmm10
-; AVX512BW-NEXT:    vmovdqa64 256(%rax), %zmm22
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm2
-; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm11, %zmm2
-; AVX512BW-NEXT:    vmovdqa64 256(%r8), %zmm0
-; AVX512BW-NEXT:    vmovdqa64 256(%r9), %zmm24
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm4 = zmm0[0],zmm24[0],zmm0[2],zmm24[2],zmm0[4],zmm24[4],zmm0[6],zmm24[6]
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm17
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm4 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 256(%rdi), %zmm2
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm19, %zmm0
+; AVX512BW-NEXT:    vmovdqa64 192(%rsi), %zmm2
+; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm5
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm19, %zmm5
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm0[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 192(%r10), %zmm9
+; AVX512BW-NEXT:    vmovdqa64 192(%rax), %zmm20
+; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm6
+; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm19, %zmm6
+; AVX512BW-NEXT:    vmovdqa64 192(%r8), %zmm8
+; AVX512BW-NEXT:    vmovdqa64 192(%r9), %zmm0
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm7 = zmm8[0],zmm0[0],zmm8[2],zmm0[2],zmm8[4],zmm0[4],zmm8[6],zmm0[6]
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm7 {%k1}
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm5, %zmm7, %zmm5
+; AVX512BW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm5
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm21, %zmm5
+; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm6
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm21, %zmm6
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm5 = ymm6[0,1,2,3],ymm5[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm6
+; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm21, %zmm6
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm7 = zmm8[1],zmm0[1],zmm8[3],zmm0[3],zmm8[5],zmm0[5],zmm8[7],zmm0[7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm7 {%k1}
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm5, %zmm7, %zmm5
+; AVX512BW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm5
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm23, %zmm5
+; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm6
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm23, %zmm6
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm5 = ymm6[0,1,2,3],ymm5[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqu64 %zmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm6
+; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm23, %zmm6
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm6 {%k1} = zmm9[0],zmm20[0],zmm9[2],zmm20[2],zmm9[4],zmm20[4],zmm9[6],zmm20[6]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm5, %zmm6, %zmm5
+; AVX512BW-NEXT:    vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm25, %zmm3
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm25, %zmm4
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3],ymm3[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, %zmm2
+; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm25, %zmm2
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm2 {%k1} = zmm9[1],zmm20[1],zmm9[3],zmm20[3],zmm9[5],zmm20[5],zmm9[7],zmm20[7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm2, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 256(%rdx), %zmm1
+; AVX512BW-NEXT:    vmovdqa64 256(%rcx), %zmm3
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm2
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm19, %zmm2
+; AVX512BW-NEXT:    vmovdqa64 256(%rdi), %zmm4
 ; AVX512BW-NEXT:    vmovdqa64 256(%rsi), %zmm5
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm6
-; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm11, %zmm6
-; AVX512BW-NEXT:    vmovdqa64 256(%rdx), %zmm3
-; AVX512BW-NEXT:    vmovdqa64 256(%rcx), %zmm7
-; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm8
-; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm15, %zmm8
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3],ymm8[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm6, %zmm4, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, %zmm4
-; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm9, %zmm4
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm6 = zmm17[1],zmm24[1],zmm17[3],zmm24[3],zmm17[5],zmm24[5],zmm17[7],zmm24[7]
-; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm6 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm4
-; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm9, %zmm4
-; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm8
-; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm12, %zmm8
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm8[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm4, %zmm6, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm4
-; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm13, %zmm4
-; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm6
-; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm1, %zmm6
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm6[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqu64 %zmm17, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm17, %zmm6
-; AVX512BW-NEXT:    vpermt2q %zmm24, %zmm13, %zmm6
-; AVX512BW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm6 {%k1} = zmm10[0],zmm22[0],zmm10[2],zmm22[2],zmm10[4],zmm22[4],zmm10[6],zmm22[6]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm4, %zmm6, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm18, %zmm2
-; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm30, %zmm3
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqa64 %zmm17, %zmm3
-; AVX512BW-NEXT:    vpermt2q %zmm24, %zmm18, %zmm3
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm3 {%k1} = zmm10[1],zmm22[1],zmm10[3],zmm22[3],zmm10[5],zmm22[5],zmm10[7],zmm22[7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm3, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 320(%rdi), %zmm2
-; AVX512BW-NEXT:    vmovdqa64 320(%rsi), %zmm3
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm5
-; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm11, %zmm5
-; AVX512BW-NEXT:    vmovdqa64 320(%rdx), %zmm14
-; AVX512BW-NEXT:    vmovdqa64 320(%rcx), %zmm6
-; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm7
-; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm15, %zmm7
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3],ymm7[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm7
-; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm12, %zmm7
-; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm8
-; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm1, %zmm8
-; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm30, %zmm14
-; AVX512BW-NEXT:    vmovdqa64 384(%rdx), %zmm0
-; AVX512BW-NEXT:    vmovdqa64 384(%rcx), %zmm6
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm10
-; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm15, %zmm10
-; AVX512BW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm10
-; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm12, %zmm10
-; AVX512BW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm10
-; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm1, %zmm10
-; AVX512BW-NEXT:    vmovdqu64 %zmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm30, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 448(%rdx), %zmm0
-; AVX512BW-NEXT:    vmovdqa64 448(%rcx), %zmm6
-; AVX512BW-NEXT:    vpermi2q %zmm6, %zmm0, %zmm15
-; AVX512BW-NEXT:    vmovdqu64 %zmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermi2q %zmm6, %zmm0, %zmm12
-; AVX512BW-NEXT:    vmovdqu64 %zmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermi2q %zmm6, %zmm0, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm6
+; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm19, %zmm6
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3],ymm2[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 256(%r10), %zmm13
+; AVX512BW-NEXT:    vmovdqa64 256(%rax), %zmm2
+; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm7
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm19, %zmm7
+; AVX512BW-NEXT:    vmovdqa64 256(%r8), %zmm18
+; AVX512BW-NEXT:    vmovdqa64 256(%r9), %zmm15
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm8 = zmm18[0],zmm15[0],zmm18[2],zmm15[2],zmm18[4],zmm15[4],zmm18[6],zmm15[6]
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm8 {%k1}
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm6, %zmm8, %zmm6
+; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm6
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm21, %zmm6
+; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm7
+; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm21, %zmm7
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm6 = ymm7[0,1,2,3],ymm6[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm7
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm21, %zmm7
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm8 = zmm18[1],zmm15[1],zmm18[3],zmm15[3],zmm18[5],zmm15[5],zmm18[7],zmm15[7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm8 {%k1}
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm6, %zmm8, %zmm6
+; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm6
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm23, %zmm6
+; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm7
+; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm23, %zmm7
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm6 = ymm7[0,1,2,3],ymm6[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm18, %zmm7
+; AVX512BW-NEXT:    vpermt2q %zmm15, %zmm23, %zmm7
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm7 {%k1} = zmm13[0],zmm2[0],zmm13[2],zmm2[2],zmm13[4],zmm2[4],zmm13[6],zmm2[6]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm6, %zmm7, %zmm6
+; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm25, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm25, %zmm4
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm4[0,1,2,3],ymm1[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm18, %zmm3
+; AVX512BW-NEXT:    vpermt2q %zmm15, %zmm25, %zmm3
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm3 {%k1} = zmm13[1],zmm2[1],zmm13[3],zmm2[3],zmm13[5],zmm2[5],zmm13[7],zmm2[7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm3, %zmm1
 ; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm6, %zmm30, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 320(%r10), %zmm31
-; AVX512BW-NEXT:    vmovdqa64 320(%rax), %zmm12
-; AVX512BW-NEXT:    vmovdqa64 %zmm31, %zmm1
-; AVX512BW-NEXT:    vpermt2q %zmm12, %zmm11, %zmm1
-; AVX512BW-NEXT:    vmovdqa64 320(%r8), %zmm17
-; AVX512BW-NEXT:    vmovdqa64 320(%r9), %zmm4
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm10 = zmm17[0],zmm4[0],zmm17[2],zmm4[2],zmm17[4],zmm4[4],zmm17[6],zmm4[6]
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm10 {%k1}
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm5, %zmm10, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm31, %zmm1
-; AVX512BW-NEXT:    vpermt2q %zmm12, %zmm9, %zmm1
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm5 = zmm17[1],zmm4[1],zmm17[3],zmm4[3],zmm17[5],zmm4[5],zmm17[7],zmm4[7]
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm5 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm1
-; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm9, %zmm1
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm7[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm5, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm1
-; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm13, %zmm1
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm8[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqa64 %zmm17, %zmm5
-; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm13, %zmm5
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm5 {%k1} = zmm31[0],zmm12[0],zmm31[2],zmm12[2],zmm31[4],zmm12[4],zmm31[6],zmm12[6]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm5, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm18, %zmm2
-; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm14[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqa64 %zmm17, %zmm2
-; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm18, %zmm2
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm2 {%k1} = zmm31[1],zmm12[1],zmm31[3],zmm12[3],zmm31[5],zmm12[5],zmm31[7],zmm12[7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm2, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 384(%rdi), %zmm0
-; AVX512BW-NEXT:    vmovdqa64 384(%rsi), %zmm1
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm3
-; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm2
-; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm11, %zmm3
+; AVX512BW-NEXT:    vmovdqa64 320(%rdx), %zmm1
+; AVX512BW-NEXT:    vmovdqa64 320(%rcx), %zmm4
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm19, %zmm3
+; AVX512BW-NEXT:    vmovdqa64 320(%rdi), %zmm6
+; AVX512BW-NEXT:    vmovdqa64 320(%rsi), %zmm7
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm5
+; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm19, %zmm5
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm5[0,1,2,3],ymm3[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 320(%r10), %zmm16
+; AVX512BW-NEXT:    vmovdqa64 320(%rax), %zmm22
+; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm12
+; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm19, %zmm12
+; AVX512BW-NEXT:    vmovdqa64 320(%r8), %zmm29
+; AVX512BW-NEXT:    vmovdqa64 320(%r9), %zmm5
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm14 = zmm29[0],zmm5[0],zmm29[2],zmm5[2],zmm29[4],zmm5[4],zmm29[6],zmm5[6]
+; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm14 {%k1}
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm8, %zmm14, %zmm3
 ; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm3
-; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm9, %zmm3
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm8
+; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm21, %zmm8
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm12
+; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm21, %zmm12
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm12[0,1,2,3],ymm8[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm12
+; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm21, %zmm12
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm14 = zmm29[1],zmm5[1],zmm29[3],zmm5[3],zmm29[5],zmm5[5],zmm29[7],zmm5[7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm14 {%k1}
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm8, %zmm14, %zmm3
 ; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm3
-; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm13, %zmm3
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm8
+; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm23, %zmm8
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm12
+; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm23, %zmm12
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm8 = ymm12[0,1,2,3],ymm8[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm29, %zmm12
+; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm23, %zmm12
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm12 {%k1} = zmm16[0],zmm22[0],zmm16[2],zmm22[2],zmm16[4],zmm22[4],zmm16[6],zmm22[6]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm8, %zmm12, %zmm3
 ; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm18, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 384(%r10), %zmm11
-; AVX512BW-NEXT:    vmovdqa64 384(%rax), %zmm20
-; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm2, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm9, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 384(%r8), %zmm30
-; AVX512BW-NEXT:    vmovdqa64 384(%r9), %zmm0
-; AVX512BW-NEXT:    vmovdqa64 %zmm30, %zmm1
-; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm13, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 448(%r10), %zmm16
-; AVX512BW-NEXT:    vmovdqa64 448(%rax), %zmm8
-; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm1
-; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm2, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm25, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm25, %zmm6
+; AVX512BW-NEXT:    vpblendd {{.*#+}} ymm1 = ymm6[0,1,2,3],ymm1[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm29, %zmm4
+; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm25, %zmm4
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm4 {%k1} = zmm16[1],zmm22[1],zmm16[3],zmm22[3],zmm16[5],zmm22[5],zmm16[7],zmm22[7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm4, %zmm1
 ; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 384(%rdx), %zmm3
+; AVX512BW-NEXT:    vmovdqa64 384(%rcx), %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm4
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm19, %zmm4
+; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm4
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm21, %zmm4
+; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm4
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm23, %zmm4
+; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm25, %zmm3
+; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 384(%rdi), %zmm3
+; AVX512BW-NEXT:    vmovdqa64 384(%rsi), %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm4
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm19, %zmm4
+; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm4
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm21, %zmm4
+; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm4
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm23, %zmm4
+; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm25, %zmm3
+; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 448(%rdx), %zmm3
+; AVX512BW-NEXT:    vmovdqa64 448(%rcx), %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm4
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm19, %zmm4
+; AVX512BW-NEXT:    vmovdqu64 %zmm4, (%rsp) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm4
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm21, %zmm4
+; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm4
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm23, %zmm4
+; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm25, %zmm3
+; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vmovdqa64 448(%rdi), %zmm3
 ; AVX512BW-NEXT:    vmovdqa64 448(%rsi), %zmm1
-; AVX512BW-NEXT:    vpermi2q %zmm1, %zmm3, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm2
-; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm9, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermi2q %zmm1, %zmm3, %zmm9
-; AVX512BW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 448(%r8), %zmm19
-; AVX512BW-NEXT:    vmovdqa64 448(%r9), %zmm7
-; AVX512BW-NEXT:    vmovdqa64 %zmm19, %zmm2
-; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm13, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermi2q %zmm1, %zmm3, %zmm13
-; AVX512BW-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm18, %zmm3
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm4
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm19, %zmm4
+; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm4
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm21, %zmm4
+; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm4
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm23, %zmm4
+; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm25, %zmm3
 ; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm11 = [0,8,0,8,0,8,0,8]
+; AVX512BW-NEXT:    # zmm11 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm26, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm11, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm12 = [1,9,1,9,1,9,1,9]
+; AVX512BW-NEXT:    # zmm12 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm26, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm12, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm10 = [2,10,2,10,2,10,2,10]
+; AVX512BW-NEXT:    # zmm10 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm26, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm10, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm4 = [3,11,3,11,3,11,3,11]
+; AVX512BW-NEXT:    # zmm4 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
+; AVX512BW-NEXT:    vpermt2q %zmm17, %zmm4, %zmm26
+; AVX512BW-NEXT:    vmovdqu64 %zmm26, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vmovdqa64 %zmm30, %zmm1
-; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm18, %zmm1
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm5
-; AVX512BW-NEXT:    vmovdqa64 %zmm19, %zmm1
-; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm18, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm24, %zmm11, %zmm1
 ; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm15 = [0,8,0,8,0,8,0,8]
-; AVX512BW-NEXT:    # zmm15 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm21, %zmm1
-; AVX512BW-NEXT:    vpermt2q %zmm27, %zmm15, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm30, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm24, %zmm12, %zmm1
 ; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm14 = [1,9,1,9,1,9,1,9]
-; AVX512BW-NEXT:    # zmm14 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm21, %zmm1
-; AVX512BW-NEXT:    vpermt2q %zmm27, %zmm14, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm30, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm24, %zmm10, %zmm1
 ; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm6 = [2,10,2,10,2,10,2,10]
-; AVX512BW-NEXT:    # zmm6 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm21, %zmm1
-; AVX512BW-NEXT:    vpermt2q %zmm27, %zmm6, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm24, %zmm4, %zmm30
+; AVX512BW-NEXT:    vmovdqu64 %zmm30, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm31, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm27, %zmm11, %zmm1
 ; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vbroadcasti32x4 {{.*#+}} zmm1 = [3,11,3,11,3,11,3,11]
-; AVX512BW-NEXT:    # zmm1 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
-; AVX512BW-NEXT:    vpermt2q %zmm27, %zmm1, %zmm21
-; AVX512BW-NEXT:    vmovdqu64 %zmm21, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512BW-NEXT:    vpermt2q %zmm28, %zmm15, %zmm3
-; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512BW-NEXT:    vpermt2q %zmm28, %zmm14, %zmm3
-; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm25
-; AVX512BW-NEXT:    vpermt2q %zmm28, %zmm6, %zmm25
-; AVX512BW-NEXT:    vpermt2q %zmm28, %zmm1, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 (%rsp), %zmm27 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm27, %zmm2
+; AVX512BW-NEXT:    vmovdqa64 %zmm31, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm27, %zmm12, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm31, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm27, %zmm10, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm27, %zmm4, %zmm31
+; AVX512BW-NEXT:    vmovdqu64 %zmm31, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm15, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm27, %zmm2
-; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm14, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm27, %zmm2
-; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm6, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm3, %zmm1, %zmm27
-; AVX512BW-NEXT:    vmovdqu64 %zmm27, (%rsp) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm3
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm15, %zmm3
-; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm3
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm14, %zmm3
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm28, %zmm11, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm28, %zmm12, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm31
+; AVX512BW-NEXT:    vpermt2q %zmm28, %zmm10, %zmm31
+; AVX512BW-NEXT:    vpermt2q %zmm28, %zmm4, %zmm3
 ; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm18
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm6, %zmm18
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm1, %zmm9
-; AVX512BW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm15, %zmm3
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm28 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm28, %zmm3
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm11, %zmm3
 ; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm14, %zmm3
+; AVX512BW-NEXT:    vmovdqa64 %zmm28, %zmm3
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm12, %zmm3
 ; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm6, %zmm3
+; AVX512BW-NEXT:    vmovdqa64 %zmm28, %zmm3
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm10, %zmm3
 ; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm9, %zmm1, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm3
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm15, %zmm3
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm4, %zmm28
+; AVX512BW-NEXT:    vmovdqu64 %zmm28, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm3
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm11, %zmm3
 ; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm3
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm14, %zmm3
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm3
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm12, %zmm3
 ; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm13
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm6, %zmm13
-; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm1, %zmm9
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm27
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm10, %zmm27
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm4, %zmm6
+; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm11, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm12, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm9, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm10, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm4, %zmm9
 ; AVX512BW-NEXT:    vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512BW-NEXT:    vpermt2q %zmm26, %zmm15, %zmm3
-; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512BW-NEXT:    vpermt2q %zmm26, %zmm14, %zmm3
-; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512BW-NEXT:    vpermt2q %zmm26, %zmm6, %zmm3
-; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm26, %zmm1, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm23, %zmm3
-; AVX512BW-NEXT:    vpermt2q %zmm29, %zmm15, %zmm3
-; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm23, %zmm3
-; AVX512BW-NEXT:    vpermt2q %zmm29, %zmm14, %zmm3
-; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm23, %zmm10
-; AVX512BW-NEXT:    vpermt2q %zmm29, %zmm6, %zmm10
-; AVX512BW-NEXT:    vpermt2q %zmm29, %zmm1, %zmm23
-; AVX512BW-NEXT:    vmovdqu64 %zmm23, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm23 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm23, %zmm2
-; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm15, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm23, %zmm2
-; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm14, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm23, %zmm2
-; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm6, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm1, %zmm23
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512BW-NEXT:    vpermt2q %zmm24, %zmm15, %zmm3
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm11, %zmm3
 ; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm3
-; AVX512BW-NEXT:    vpermt2q %zmm24, %zmm14, %zmm3
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm3
+; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm12, %zmm3
 ; AVX512BW-NEXT:    vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm26
-; AVX512BW-NEXT:    vpermt2q %zmm24, %zmm6, %zmm26
-; AVX512BW-NEXT:    vpermt2q %zmm24, %zmm1, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm31, %zmm2
-; AVX512BW-NEXT:    vpermt2q %zmm12, %zmm15, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm31, %zmm2
-; AVX512BW-NEXT:    vpermt2q %zmm12, %zmm14, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm31, %zmm2
-; AVX512BW-NEXT:    vpermt2q %zmm12, %zmm6, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vpermt2q %zmm12, %zmm1, %zmm31
-; AVX512BW-NEXT:    vmovdqa64 %zmm17, %zmm29
-; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm15, %zmm29
-; AVX512BW-NEXT:    vmovdqa64 %zmm17, %zmm2
-; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm14, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm17, %zmm24
-; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm6, %zmm24
-; AVX512BW-NEXT:    vpermt2q %zmm4, %zmm1, %zmm17
-; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm2
-; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm15, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm2
-; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm14, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm9 {%k1} = zmm11[0],zmm20[0],zmm11[2],zmm20[2],zmm11[4],zmm20[4],zmm11[6],zmm20[6]
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm5 {%k1} = zmm11[1],zmm20[1],zmm11[3],zmm20[3],zmm11[5],zmm20[5],zmm11[7],zmm20[7]
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm27
-; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm22
-; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm6, %zmm22
-; AVX512BW-NEXT:    vpermt2q %zmm20, %zmm1, %zmm11
-; AVX512BW-NEXT:    vmovdqa64 %zmm30, %zmm21
-; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm15, %zmm21
-; AVX512BW-NEXT:    vmovdqa64 %zmm30, %zmm28
-; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm14, %zmm28
-; AVX512BW-NEXT:    vmovdqa64 %zmm30, %zmm20
-; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm6, %zmm20
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm2 = zmm30[0],zmm0[0],zmm30[2],zmm0[2],zmm30[4],zmm0[4],zmm30[6],zmm0[6]
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm3 = zmm30[1],zmm0[1],zmm30[3],zmm0[3],zmm30[5],zmm0[5],zmm30[7],zmm0[7]
-; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm1, %zmm30
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm20
+; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm10, %zmm20
+; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm4, %zmm1
+; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm11, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm12, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm10, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm4, %zmm13
+; AVX512BW-NEXT:    vmovdqu64 %zmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm18, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm15, %zmm11, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm18, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm15, %zmm12, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm18, %zmm28
+; AVX512BW-NEXT:    vpermt2q %zmm15, %zmm10, %zmm28
+; AVX512BW-NEXT:    vpermt2q %zmm15, %zmm4, %zmm18
+; AVX512BW-NEXT:    vmovdqu64 %zmm18, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm15, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm11, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm0
-; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm14, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm12, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm4 {%k1} = zmm16[0],zmm8[0],zmm16[2],zmm8[2],zmm16[4],zmm8[4],zmm16[6],zmm8[6]
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm5 {%k1} = zmm16[1],zmm8[1],zmm16[3],zmm8[3],zmm16[5],zmm8[5],zmm16[7],zmm8[7]
-; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm12
-; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm6, %zmm12
-; AVX512BW-NEXT:    vpermt2q %zmm8, %zmm1, %zmm16
-; AVX512BW-NEXT:    vpermi2q %zmm7, %zmm19, %zmm15
-; AVX512BW-NEXT:    vpermi2q %zmm7, %zmm19, %zmm14
-; AVX512BW-NEXT:    vpermi2q %zmm7, %zmm19, %zmm6
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm8 = zmm19[0],zmm7[0],zmm19[2],zmm7[2],zmm19[4],zmm7[4],zmm19[6],zmm7[6]
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm0 = zmm19[1],zmm7[1],zmm19[3],zmm7[3],zmm19[5],zmm7[5],zmm19[7],zmm7[7]
-; AVX512BW-NEXT:    vpermt2q %zmm7, %zmm1, %zmm19
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
-; AVX512BW-NEXT:    # ymm1 = ymm1[0,1,2,3],mem[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm2 {%k1}
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm2, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
-; AVX512BW-NEXT:    # ymm1 = ymm1[0,1,2,3],mem[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm3 {%k1}
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm3, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
-; AVX512BW-NEXT:    # ymm1 = ymm1[0,1,2,3],mem[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm9, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
-; AVX512BW-NEXT:    # ymm1 = ymm1[0,1,2,3],mem[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm27, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm10, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermt2q %zmm22, %zmm4, %zmm16
+; AVX512BW-NEXT:    vmovdqu64 %zmm16, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm29, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm11, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm29, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm12, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm29, %zmm22
+; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm10, %zmm22
+; AVX512BW-NEXT:    vpermt2q %zmm5, %zmm4, %zmm29
+; AVX512BW-NEXT:    vmovdqa64 384(%r10), %zmm13
+; AVX512BW-NEXT:    vmovdqa64 384(%rax), %zmm0
+; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm11, %zmm1
 ; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
-; AVX512BW-NEXT:    # ymm1 = ymm1[0,1,2,3],mem[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm8 {%k1}
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm8, %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm1
+; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm12, %zmm1
 ; AVX512BW-NEXT:    vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1, %ymm1 # 32-byte Folded Reload
-; AVX512BW-NEXT:    # ymm1 = ymm1[0,1,2,3],mem[4,5,6,7]
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm0 {%k1}
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm0, %zmm0
+; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm14
+; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm19, %zmm14
+; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm15
+; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm21, %zmm15
+; AVX512BW-NEXT:    vmovdqa64 384(%r8), %zmm5
+; AVX512BW-NEXT:    vmovdqa64 384(%r9), %zmm2
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm8
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm23, %zmm8
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm8 {%k1} = zmm13[0],zmm0[0],zmm13[2],zmm0[2],zmm13[4],zmm0[4],zmm13[6],zmm0[6]
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm6
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm25, %zmm6
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm6 {%k1} = zmm13[1],zmm0[1],zmm13[3],zmm0[3],zmm13[5],zmm0[5],zmm13[7],zmm0[7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm30
+; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm10, %zmm30
+; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm4, %zmm13
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm24
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm11, %zmm24
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm26
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm12, %zmm26
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm17
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm10, %zmm17
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm7 = zmm5[0],zmm2[0],zmm5[2],zmm2[2],zmm5[4],zmm2[4],zmm5[6],zmm2[6]
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm9 = zmm5[1],zmm2[1],zmm5[3],zmm2[3],zmm5[5],zmm2[5],zmm5[7],zmm2[7]
+; AVX512BW-NEXT:    vpermt2q %zmm2, %zmm4, %zmm5
+; AVX512BW-NEXT:    vmovdqa64 448(%r10), %zmm16
+; AVX512BW-NEXT:    vmovdqa64 448(%rax), %zmm1
+; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm11, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm0
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm12, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vpermi2q %zmm1, %zmm16, %zmm19
+; AVX512BW-NEXT:    vpermi2q %zmm1, %zmm16, %zmm21
+; AVX512BW-NEXT:    vmovdqa64 448(%r8), %zmm3
+; AVX512BW-NEXT:    vmovdqa64 448(%r9), %zmm0
+; AVX512BW-NEXT:    vpermi2q %zmm0, %zmm3, %zmm23
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm23 {%k1} = zmm16[0],zmm1[0],zmm16[2],zmm1[2],zmm16[4],zmm1[4],zmm16[6],zmm1[6]
+; AVX512BW-NEXT:    vpermi2q %zmm0, %zmm3, %zmm25
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm25 {%k1} = zmm16[1],zmm1[1],zmm16[3],zmm1[3],zmm16[5],zmm1[5],zmm16[7],zmm1[7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm18
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm10, %zmm18
+; AVX512BW-NEXT:    vpermt2q %zmm1, %zmm4, %zmm16
+; AVX512BW-NEXT:    vpermi2q %zmm0, %zmm3, %zmm11
+; AVX512BW-NEXT:    vpermi2q %zmm0, %zmm3, %zmm12
+; AVX512BW-NEXT:    vpermi2q %zmm0, %zmm3, %zmm10
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} zmm1 = zmm3[0],zmm0[0],zmm3[2],zmm0[2],zmm3[4],zmm0[4],zmm3[6],zmm0[6]
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} zmm2 = zmm3[1],zmm0[1],zmm3[3],zmm0[3],zmm3[5],zmm0[5],zmm3[7],zmm0[7]
+; AVX512BW-NEXT:    vpermt2q %zmm0, %zmm4, %zmm3
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
 ; AVX512BW-NEXT:    # ymm0 = ymm0[0,1,2,3],mem[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm4, %zmm0
+; AVX512BW-NEXT:    vmovdqa64 %zmm14, %zmm7 {%k1}
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm7, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
 ; AVX512BW-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
 ; AVX512BW-NEXT:    # ymm0 = ymm0[0,1,2,3],mem[4,5,6,7]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm5, %zmm0
+; AVX512BW-NEXT:    vmovdqa64 %zmm15, %zmm9 {%k1}
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm9, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; AVX512BW-NEXT:    # ymm0 = ymm0[0,1,2,3],mem[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm8, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm25 {%k1}
+; AVX512BW-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; AVX512BW-NEXT:    # ymm0 = ymm0[0,1,2,3],mem[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm6, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vpblendd $240, (%rsp), %ymm0, %ymm0 # 32-byte Folded Reload
+; AVX512BW-NEXT:    # ymm0 = ymm0[0,1,2,3],mem[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm19, %zmm1 {%k1}
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm1, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; AVX512BW-NEXT:    # ymm0 = ymm0[0,1,2,3],mem[4,5,6,7]
+; AVX512BW-NEXT:    vmovdqa64 %zmm21, %zmm2 {%k1}
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm2, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; AVX512BW-NEXT:    # ymm0 = ymm0[0,1,2,3],mem[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm23, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vpblendd $240, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; AVX512BW-NEXT:    # ymm0 = ymm0[0,1,2,3],mem[4,5,6,7]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm25, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm8 {%k1}
 ; AVX512BW-NEXT:    vmovdqa (%rcx), %ymm0
 ; AVX512BW-NEXT:    vmovdqa (%rdx), %ymm1
 ; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX512BW-NEXT:    vmovdqa (%rsi), %ymm3
-; AVX512BW-NEXT:    vmovdqa (%rdi), %ymm7
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm8 = ymm7[0],ymm3[0],ymm7[2],ymm3[2]
-; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm2 = ymm8[2,3],ymm2[2,3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm25, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa (%rsi), %ymm4
+; AVX512BW-NEXT:    vmovdqa (%rdi), %ymm6
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm7 = ymm6[0],ymm4[0],ymm6[2],ymm4[2]
+; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm2 = ymm7[2,3],ymm2[2,3]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm8, %zmm2
+; AVX512BW-NEXT:    vmovdqu64 %zmm2, (%rsp) # 64-byte Spill
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm4 {%k1}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm7 {%k1}
 ; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm0 = ymm1[1],ymm0[1],ymm1[3],ymm0[3]
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm7[1],ymm3[1],ymm7[3],ymm3[3]
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm6[1],ymm4[1],ymm6[3],ymm4[3]
 ; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm1[2,3],ymm0[2,3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm4, %zmm0
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm7, %zmm0
 ; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm18 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm31 {%k1}
 ; AVX512BW-NEXT:    vmovdqa 64(%rcx), %ymm0
 ; AVX512BW-NEXT:    vmovdqa 64(%rdx), %ymm1
 ; AVX512BW-NEXT:    vmovdqa 64(%rsi), %ymm2
-; AVX512BW-NEXT:    vmovdqa 64(%rdi), %ymm3
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm7 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm8 = ymm3[0],ymm2[0],ymm3[2],ymm2[2]
-; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm7 = ymm8[2,3],ymm7[2,3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm7, %zmm18, %zmm4
-; AVX512BW-NEXT:    vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512BW-NEXT:    vmovdqu64 (%rsp), %zmm4 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm5 {%k1}
+; AVX512BW-NEXT:    vmovdqa 64(%rdi), %ymm4
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm6 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm7 = ymm4[0],ymm2[0],ymm4[2],ymm2[2]
+; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm6 = ymm7[2,3],ymm6[2,3]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm6, %zmm31, %zmm6
+; AVX512BW-NEXT:    vmovdqu64 %zmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, %zmm7 {%k1}
 ; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm0 = ymm1[1],ymm0[1],ymm1[3],ymm0[3]
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm3[1],ymm2[1],ymm3[3],ymm2[3]
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm4[1],ymm2[1],ymm4[3],ymm2[3]
 ; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm1[2,3],ymm0[2,3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm5, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm7, %zmm8
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm13 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm27 {%k1}
 ; AVX512BW-NEXT:    vmovdqa 128(%rcx), %ymm0
 ; AVX512BW-NEXT:    vmovdqa 128(%rdx), %ymm1
 ; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX512BW-NEXT:    vmovdqa 128(%rsi), %ymm3
-; AVX512BW-NEXT:    vmovdqa 128(%rdi), %ymm7
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm8 = ymm7[0],ymm3[0],ymm7[2],ymm3[2]
-; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm2 = ymm8[2,3],ymm2[2,3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm13, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, (%rsp) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa 128(%rsi), %ymm4
+; AVX512BW-NEXT:    vmovdqa 128(%rdi), %ymm6
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm7 = ymm6[0],ymm4[0],ymm6[2],ymm4[2]
+; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm2 = ymm7[2,3],ymm2[2,3]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm27, %zmm23
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm4 {%k1}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm7 {%k1}
 ; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm0 = ymm1[1],ymm0[1],ymm1[3],ymm0[3]
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm7[1],ymm3[1],ymm7[3],ymm3[3]
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm6[1],ymm4[1],ymm6[3],ymm4[3]
 ; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm1[2,3],ymm0[2,3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm4, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm7, %zmm25
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm10 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm20 {%k1}
 ; AVX512BW-NEXT:    vmovdqa 192(%rcx), %ymm0
 ; AVX512BW-NEXT:    vmovdqa 192(%rdx), %ymm1
 ; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX512BW-NEXT:    vmovdqa 192(%rsi), %ymm3
-; AVX512BW-NEXT:    vmovdqa 192(%rdi), %ymm7
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm8 = ymm7[0],ymm3[0],ymm7[2],ymm3[2]
-; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm2 = ymm8[2,3],ymm2[2,3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm10, %zmm2
-; AVX512BW-NEXT:    vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512BW-NEXT:    vmovdqa 192(%rsi), %ymm4
+; AVX512BW-NEXT:    vmovdqa 192(%rdi), %ymm6
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm7 = ymm6[0],ymm4[0],ymm6[2],ymm4[2]
+; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm2 = ymm7[2,3],ymm2[2,3]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm20, %zmm21
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm4 {%k1}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm7 {%k1}
 ; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm0 = ymm1[1],ymm0[1],ymm1[3],ymm0[3]
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm7[1],ymm3[1],ymm7[3],ymm3[3]
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm6[1],ymm4[1],ymm6[3],ymm4[3]
 ; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm1[2,3],ymm0[2,3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm4, %zmm9
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm7, %zmm19
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm26 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm28 {%k1}
 ; AVX512BW-NEXT:    vmovdqa 256(%rcx), %ymm0
 ; AVX512BW-NEXT:    vmovdqa 256(%rdx), %ymm1
 ; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX512BW-NEXT:    vmovdqa 256(%rsi), %ymm3
-; AVX512BW-NEXT:    vmovdqa 256(%rdi), %ymm7
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm8 = ymm7[0],ymm3[0],ymm7[2],ymm3[2]
-; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm2 = ymm8[2,3],ymm2[2,3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm26, %zmm26
+; AVX512BW-NEXT:    vmovdqa 256(%rsi), %ymm4
+; AVX512BW-NEXT:    vmovdqa 256(%rdi), %ymm6
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm7 = ymm6[0],ymm4[0],ymm6[2],ymm4[2]
+; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm2 = ymm7[2,3],ymm2[2,3]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm28, %zmm15
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm23, %zmm2 {%k1}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm7 {%k1}
 ; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm0 = ymm1[1],ymm0[1],ymm1[3],ymm0[3]
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm7[1],ymm3[1],ymm7[3],ymm3[3]
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm6[1],ymm4[1],ymm6[3],ymm4[3]
 ; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm1[2,3],ymm0[2,3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm2, %zmm7
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm7, %zmm28
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm24 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm22 {%k1}
 ; AVX512BW-NEXT:    vmovdqa 320(%rcx), %ymm0
 ; AVX512BW-NEXT:    vmovdqa 320(%rdx), %ymm1
 ; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX512BW-NEXT:    vmovdqa 320(%rsi), %ymm3
-; AVX512BW-NEXT:    vmovdqa64 320(%rdi), %ymm23
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm8 = ymm23[0],ymm3[0],ymm23[2],ymm3[2]
-; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm2 = ymm8[2,3],ymm2[2,3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm24, %zmm8
-; AVX512BW-NEXT:    vmovdqa64 %zmm31, %zmm17 {%k1}
+; AVX512BW-NEXT:    vmovdqa 320(%rsi), %ymm4
+; AVX512BW-NEXT:    vmovdqa 320(%rdi), %ymm6
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm7 = ymm6[0],ymm4[0],ymm6[2],ymm4[2]
+; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm2 = ymm7[2,3],ymm2[2,3]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm22, %zmm22
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm2, %zmm29 {%k1}
 ; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm0 = ymm1[1],ymm0[1],ymm1[3],ymm0[3]
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm23[1],ymm3[1],ymm23[3],ymm3[3]
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm6[1],ymm4[1],ymm6[3],ymm4[3]
 ; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm1[2,3],ymm0[2,3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm17, %zmm17
-; AVX512BW-NEXT:    vmovdqa64 %zmm22, %zmm20 {%k1}
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm29, %zmm27
+; AVX512BW-NEXT:    vmovdqa64 %zmm30, %zmm17 {%k1}
 ; AVX512BW-NEXT:    vmovdqa 384(%rcx), %ymm0
 ; AVX512BW-NEXT:    vmovdqa 384(%rdx), %ymm1
 ; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX512BW-NEXT:    vmovdqa 384(%rsi), %ymm3
-; AVX512BW-NEXT:    vmovdqa64 384(%rdi), %ymm18
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm13 = ymm18[0],ymm3[0],ymm18[2],ymm3[2]
-; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm2 = ymm13[2,3],ymm2[2,3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm20, %zmm20
-; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm30 {%k1}
+; AVX512BW-NEXT:    vmovdqa 384(%rsi), %ymm4
+; AVX512BW-NEXT:    vmovdqa 384(%rdi), %ymm6
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm7 = ymm6[0],ymm4[0],ymm6[2],ymm4[2]
+; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm2 = ymm7[2,3],ymm2[2,3]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm17, %zmm14
+; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm5 {%k1}
 ; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm0 = ymm1[1],ymm0[1],ymm1[3],ymm0[3]
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm18[1],ymm3[1],ymm18[3],ymm3[3]
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm6[1],ymm4[1],ymm6[3],ymm4[3]
 ; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm1[2,3],ymm0[2,3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm30, %zmm24
-; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm6 {%k1}
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm5, %zmm13
+; AVX512BW-NEXT:    vmovdqa64 %zmm18, %zmm10 {%k1}
 ; AVX512BW-NEXT:    vmovdqa 448(%rcx), %ymm0
 ; AVX512BW-NEXT:    vmovdqa 448(%rdx), %ymm1
 ; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
-; AVX512BW-NEXT:    vmovdqa 448(%rsi), %ymm3
-; AVX512BW-NEXT:    vmovdqa 448(%rdi), %ymm10
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm12 = ymm10[0],ymm3[0],ymm10[2],ymm3[2]
-; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm2 = ymm12[2,3],ymm2[2,3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm6, %zmm31
+; AVX512BW-NEXT:    vmovdqa 448(%rsi), %ymm4
+; AVX512BW-NEXT:    vmovdqa 448(%rdi), %ymm5
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm6 = ymm5[0],ymm4[0],ymm5[2],ymm4[2]
+; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm2 = ymm6[2,3],ymm2[2,3]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm2, %zmm10, %zmm10
 ; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm0 = ymm1[1],ymm0[1],ymm1[3],ymm0[3]
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm10[1],ymm3[1],ymm10[3],ymm3[3]
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm5[1],ymm4[1],ymm5[3],ymm4[3]
 ; AVX512BW-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm1[2,3],ymm0[2,3]
-; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm19 {%k1}
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm19, %zmm0
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm4 {%k1}
-; AVX512BW-NEXT:    vmovdqa (%rsi), %xmm1
-; AVX512BW-NEXT:    vinserti128 $1, (%rcx), %ymm1, %ymm1
-; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm2
-; AVX512BW-NEXT:    vinserti128 $1, (%rdx), %ymm2, %ymm2
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm3 = ymm2[0],ymm1[0],ymm2[2],ymm1[2]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm3, %zmm4, %zmm3
+; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm3 {%k1}
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm3, %zmm9
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, %zmm3 {%k1}
+; AVX512BW-NEXT:    vmovdqa (%rsi), %xmm0
+; AVX512BW-NEXT:    vinserti128 $1, (%rcx), %ymm0, %ymm1
+; AVX512BW-NEXT:    vmovdqa (%rdi), %xmm0
+; AVX512BW-NEXT:    vinserti128 $1, (%rdx), %ymm0, %ymm2
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm0 = ymm2[0],ymm1[0],ymm2[2],ymm1[2]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm0, %zmm3, %zmm0
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm4, %zmm10 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, %zmm4 {%k1}
 ; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm2[1],ymm1[1],ymm2[3],ymm1[3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm10, %zmm2
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm4, %zmm2
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm11 {%k1}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm5 {%k1}
 ; AVX512BW-NEXT:    vmovdqa 64(%rsi), %xmm1
 ; AVX512BW-NEXT:    vinserti128 $1, 64(%rcx), %ymm1, %ymm1
-; AVX512BW-NEXT:    vmovdqa 64(%rdi), %xmm4
-; AVX512BW-NEXT:    vinserti128 $1, 64(%rdx), %ymm4, %ymm4
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm10 = ymm4[0],ymm1[0],ymm4[2],ymm1[2]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm10, %zmm11, %zmm10
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm11, %zmm12 {%k1}
+; AVX512BW-NEXT:    vmovdqa 64(%rdi), %xmm3
+; AVX512BW-NEXT:    vinserti128 $1, 64(%rdx), %ymm3, %ymm4
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm3 = ymm4[0],ymm1[0],ymm4[2],ymm1[2]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm3, %zmm5, %zmm3
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm6 {%k1}
 ; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm4[1],ymm1[1],ymm4[3],ymm1[3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm12, %zmm11
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm6, %zmm5
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm13 {%k1}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm7 {%k1}
 ; AVX512BW-NEXT:    vmovdqa 128(%rsi), %xmm1
 ; AVX512BW-NEXT:    vinserti128 $1, 128(%rcx), %ymm1, %ymm1
 ; AVX512BW-NEXT:    vmovdqa 128(%rdi), %xmm4
-; AVX512BW-NEXT:    vinserti128 $1, 128(%rdx), %ymm4, %ymm12
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm4 = ymm12[0],ymm1[0],ymm12[2],ymm1[2]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm4, %zmm13, %zmm4
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm18 {%k1}
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm12[1],ymm1[1],ymm12[3],ymm1[3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm18, %zmm19
+; AVX512BW-NEXT:    vinserti128 $1, 128(%rdx), %ymm4, %ymm6
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm4 = ymm6[0],ymm1[0],ymm6[2],ymm1[2]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm4, %zmm7, %zmm4
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, %zmm16 {%k1}
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm6[1],ymm1[1],ymm6[3],ymm1[3]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm16, %zmm6
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm18 {%k1}
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm1, %zmm17 {%k1}
 ; AVX512BW-NEXT:    vmovdqa 192(%rsi), %xmm1
 ; AVX512BW-NEXT:    vinserti128 $1, 192(%rcx), %ymm1, %ymm1
-; AVX512BW-NEXT:    vmovdqa 192(%rdi), %xmm12
-; AVX512BW-NEXT:    vinserti128 $1, 192(%rdx), %ymm12, %ymm12
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm13 = ymm12[0],ymm1[0],ymm12[2],ymm1[2]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm13, %zmm18, %zmm30
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm13 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa 192(%rdi), %xmm7
+; AVX512BW-NEXT:    vinserti32x4 $1, 192(%rdx), %ymm7, %ymm16
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm7 = ymm16[0],ymm1[0],ymm16[2],ymm1[2]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm7, %zmm17, %zmm7
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
 ; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm13, %zmm18 {%k1}
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm12[1],ymm1[1],ymm12[3],ymm1[3]
+; AVX512BW-NEXT:    vmovdqa64 %zmm17, %zmm18 {%k1}
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm1 = ymm16[1],ymm1[1],ymm16[3],ymm1[3]
 ; AVX512BW-NEXT:    vinserti64x4 $0, %ymm1, %zmm18, %zmm1
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm12, %zmm5 {%k1}
-; AVX512BW-NEXT:    vmovdqa 256(%rsi), %xmm12
-; AVX512BW-NEXT:    vinserti128 $1, 256(%rcx), %ymm12, %ymm13
-; AVX512BW-NEXT:    vmovdqa 256(%rdi), %xmm12
-; AVX512BW-NEXT:    vinserti32x4 $1, 256(%rdx), %ymm12, %ymm18
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm12 = ymm18[0],ymm13[0],ymm18[2],ymm13[2]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm12, %zmm5, %zmm12
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm23 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm23, %zmm5 {%k1}
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm13 = ymm18[1],ymm13[1],ymm18[3],ymm13[3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm13, %zmm5, %zmm23
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm29 {%k1}
-; AVX512BW-NEXT:    vmovdqa 320(%rsi), %xmm13
-; AVX512BW-NEXT:    vinserti128 $1, 320(%rcx), %ymm13, %ymm13
-; AVX512BW-NEXT:    vmovdqa64 320(%rdi), %xmm18
-; AVX512BW-NEXT:    vinserti32x4 $1, 320(%rdx), %ymm18, %ymm18
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm27 = ymm18[0],ymm13[0],ymm18[2],ymm13[2]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm27, %zmm29, %zmm22
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm6 {%k1}
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm13 = ymm18[1],ymm13[1],ymm18[3],ymm13[3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm13, %zmm6, %zmm13
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm21 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 384(%rsi), %xmm18
-; AVX512BW-NEXT:    vinserti32x4 $1, 384(%rcx), %ymm18, %ymm18
-; AVX512BW-NEXT:    vmovdqa64 384(%rdi), %xmm25
-; AVX512BW-NEXT:    vinserti32x4 $1, 384(%rdx), %ymm25, %ymm25
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm27 = ymm25[0],ymm18[0],ymm25[2],ymm18[2]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm27, %zmm21, %zmm16
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm28 {%k1}
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm18 = ymm25[1],ymm18[1],ymm25[3],ymm18[3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm18, %zmm28, %zmm21
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm15 {%k1}
-; AVX512BW-NEXT:    vmovdqa64 448(%rsi), %xmm18
-; AVX512BW-NEXT:    vinserti32x4 $1, 448(%rcx), %ymm18, %ymm18
-; AVX512BW-NEXT:    vmovdqa64 448(%rdi), %xmm25
-; AVX512BW-NEXT:    vinserti32x4 $1, 448(%rdx), %ymm25, %ymm25
-; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm27 = ymm25[0],ymm18[0],ymm25[2],ymm18[2]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm27, %zmm15, %zmm6
-; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm5 # 64-byte Reload
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, %zmm14 {%k1}
-; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm18 = ymm25[1],ymm18[1],ymm25[3],ymm18[3]
-; AVX512BW-NEXT:    vinserti64x4 $0, %ymm18, %zmm14, %zmm5
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm20 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm20 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 256(%rsi), %xmm16
+; AVX512BW-NEXT:    vinserti32x4 $1, 256(%rcx), %ymm16, %ymm16
+; AVX512BW-NEXT:    vmovdqa64 256(%rdi), %xmm17
+; AVX512BW-NEXT:    vinserti32x4 $1, 256(%rdx), %ymm17, %ymm17
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm18 = ymm17[0],ymm16[0],ymm17[2],ymm16[2]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm18, %zmm20, %zmm29
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm18 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm20 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm18, %zmm20 {%k1}
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm16 = ymm17[1],ymm16[1],ymm17[3],ymm16[3]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm16, %zmm20, %zmm30
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm16 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm20 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm16, %zmm20 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 320(%rsi), %xmm16
+; AVX512BW-NEXT:    vinserti32x4 $1, 320(%rcx), %ymm16, %ymm16
+; AVX512BW-NEXT:    vmovdqa64 320(%rdi), %xmm17
+; AVX512BW-NEXT:    vinserti32x4 $1, 320(%rdx), %ymm17, %ymm17
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm18 = ymm17[0],ymm16[0],ymm17[2],ymm16[2]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm18, %zmm20, %zmm18
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm20 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm31 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm20, %zmm31 {%k1}
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm16 = ymm17[1],ymm16[1],ymm17[3],ymm16[3]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm16, %zmm31, %zmm16
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm17, %zmm24 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 384(%rsi), %xmm17
+; AVX512BW-NEXT:    vinserti32x4 $1, 384(%rcx), %ymm17, %ymm17
+; AVX512BW-NEXT:    vmovdqa64 384(%rdi), %xmm20
+; AVX512BW-NEXT:    vinserti32x4 $1, 384(%rdx), %ymm20, %ymm20
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm31 = ymm20[0],ymm17[0],ymm20[2],ymm17[2]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm31, %zmm24, %zmm24
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm31 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm31, %zmm26 {%k1}
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm17 = ymm20[1],ymm17[1],ymm20[3],ymm17[3]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm17, %zmm26, %zmm17
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm20 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm20, %zmm11 {%k1}
+; AVX512BW-NEXT:    vmovdqa64 448(%rsi), %xmm20
+; AVX512BW-NEXT:    vinserti32x4 $1, 448(%rcx), %ymm20, %ymm20
+; AVX512BW-NEXT:    vmovdqa64 448(%rdi), %xmm26
+; AVX512BW-NEXT:    vinserti32x4 $1, 448(%rdx), %ymm26, %ymm26
+; AVX512BW-NEXT:    vpunpcklqdq {{.*#+}} ymm31 = ymm26[0],ymm20[0],ymm26[2],ymm20[2]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm31, %zmm11, %zmm11
+; AVX512BW-NEXT:    vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm31 # 64-byte Reload
+; AVX512BW-NEXT:    vmovdqa64 %zmm31, %zmm12 {%k1}
+; AVX512BW-NEXT:    vpunpckhqdq {{.*#+}} ymm20 = ymm26[1],ymm20[1],ymm26[3],ymm20[3]
+; AVX512BW-NEXT:    vinserti64x4 $0, %ymm20, %zmm12, %zmm12
 ; AVX512BW-NEXT:    movq {{[0-9]+}}(%rsp), %rax
-; AVX512BW-NEXT:    vmovdqa64 %zmm0, 3776(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm31, 3712(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm24, 3264(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm20, 3200(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm17, 2752(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm8, 2688(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm7, 2240(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm26, 2176(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm9, 1728(%rax)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm0, 1664(%rax)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm0, 1216(%rax)
-; AVX512BW-NEXT:    vmovups (%rsp), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm0, 1152(%rax)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm0, 704(%rax)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm0, 640(%rax)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm0, 192(%rax)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm0, 128(%rax)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm0, 4032(%rax)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm0, 3968(%rax)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm0, 3904(%rax)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm0, 3840(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm5, 3648(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm6, 3584(%rax)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm0, 3520(%rax)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm0, 3456(%rax)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm0, 3392(%rax)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm0, 3328(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm21, 3136(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm16, 3072(%rax)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm0, 3008(%rax)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm0, 2944(%rax)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm0, 2880(%rax)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm0, 2816(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm13, 2624(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm22, 2560(%rax)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm0, 2496(%rax)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm0, 2432(%rax)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm0, 2368(%rax)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm0, 2304(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm23, 2112(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm12, 2048(%rax)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm0, 1984(%rax)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm0, 1920(%rax)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm0, 1856(%rax)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm0, 1792(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm9, 3776(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm10, 3712(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm13, 3264(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm14, 3200(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm27, 2752(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm22, 2688(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm28, 2240(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm15, 2176(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm19, 1728(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm21, 1664(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm25, 1216(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm23, 1152(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm8, 704(%rax)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm8, 640(%rax)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm8, 192(%rax)
+; AVX512BW-NEXT:    vmovups (%rsp), %zmm8 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm8, 128(%rax)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm8, 4032(%rax)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm8, 3968(%rax)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm8, 3904(%rax)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm8, 3840(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm12, 3648(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm11, 3584(%rax)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm8, 3520(%rax)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm8, 3456(%rax)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm8, 3392(%rax)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm8, 3328(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm17, 3136(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm24, 3072(%rax)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm8, 3008(%rax)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm8, 2944(%rax)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm8, 2880(%rax)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm8, 2816(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm16, 2624(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm18, 2560(%rax)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm8, 2496(%rax)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm8, 2432(%rax)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm8, 2368(%rax)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm8, 2304(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm30, 2112(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm29, 2048(%rax)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm8, 1984(%rax)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm8, 1920(%rax)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm8, 1856(%rax)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm8, 1792(%rax)
 ; AVX512BW-NEXT:    vmovdqa64 %zmm1, 1600(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm30, 1536(%rax)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm0, 1472(%rax)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm0, 1408(%rax)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm0, 1344(%rax)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm0, 1280(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm19, 1088(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm7, 1536(%rax)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm1, 1472(%rax)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm1, 1408(%rax)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm1, 1344(%rax)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm1, 1280(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm6, 1088(%rax)
 ; AVX512BW-NEXT:    vmovdqa64 %zmm4, 1024(%rax)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm0, 960(%rax)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm0, 896(%rax)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm0, 832(%rax)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm0, 768(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm11, 576(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm10, 512(%rax)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm0, 448(%rax)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm0, 384(%rax)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm0, 320(%rax)
-; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
-; AVX512BW-NEXT:    vmovaps %zmm0, 256(%rax)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm1, 960(%rax)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm1, 896(%rax)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm1, 832(%rax)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm1, 768(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm5, 576(%rax)
+; AVX512BW-NEXT:    vmovdqa64 %zmm3, 512(%rax)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm1, 448(%rax)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm1, 384(%rax)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm1, 320(%rax)
+; AVX512BW-NEXT:    vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload
+; AVX512BW-NEXT:    vmovaps %zmm1, 256(%rax)
 ; AVX512BW-NEXT:    vmovdqa64 %zmm2, 64(%rax)
-; AVX512BW-NEXT:    vmovdqa64 %zmm3, (%rax)
-; AVX512BW-NEXT:    addq $5512, %rsp # imm = 0x1588
+; AVX512BW-NEXT:    vmovdqa64 %zmm0, (%rax)
+; AVX512BW-NEXT:    addq $5384, %rsp # imm = 0x1508
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
   %in.vec0 = load <64 x i64>, ptr %in.vecptr0, align 64
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-5.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-5.ll
index 4174fde817472a9892cb2d32407505f85b6d6e25..9f664f8f16529085c53c5e6863880ed51325dba2 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-5.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-5.ll
@@ -4063,181 +4063,184 @@ define void @store_i8_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX512F-FAST-LABEL: store_i8_stride5_vf64:
 ; AVX512F-FAST:       # %bb.0:
 ; AVX512F-FAST-NEXT:    subq $24, %rsp
-; AVX512F-FAST-NEXT:    vmovdqa 32(%rsi), %ymm3
-; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} ymm8 = [128,128,13,128,128,128,128,14,128,128,128,128,15,128,128,128,128,16,128,128,128,128,17,128,128,128,128,18,128,128,128,128]
-; AVX512F-FAST-NEXT:    vpshufb %ymm8, %ymm3, %ymm0
+; AVX512F-FAST-NEXT:    vmovdqa 32(%rsi), %ymm9
+; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} ymm6 = [128,128,13,128,128,128,128,14,128,128,128,128,15,128,128,128,128,16,128,128,128,128,17,128,128,128,128,18,128,128,128,128]
+; AVX512F-FAST-NEXT:    vpshufb %ymm6, %ymm9, %ymm0
 ; AVX512F-FAST-NEXT:    vmovdqa 32(%rdi), %ymm1
-; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} ymm11 = <12,13,128,15,12,13,14,128,12,13,14,15,128,u,u,u,16,128,18,19,16,17,128,19,16,17,18,128,16,17,18,19>
-; AVX512F-FAST-NEXT:    vpshufb %ymm11, %ymm1, %ymm2
-; AVX512F-FAST-NEXT:    vpor %ymm0, %ymm2, %ymm0
+; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} ymm2 = <12,13,128,15,12,13,14,128,12,13,14,15,128,u,u,u,16,128,18,19,16,17,128,19,16,17,18,128,16,17,18,19>
+; AVX512F-FAST-NEXT:    vpshufb %ymm2, %ymm1, %ymm3
+; AVX512F-FAST-NEXT:    vpor %ymm0, %ymm3, %ymm0
 ; AVX512F-FAST-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
 ; AVX512F-FAST-NEXT:    vmovdqa 32(%rdi), %xmm0
 ; AVX512F-FAST-NEXT:    vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} xmm5 = <8,128,u,7,128,9,128,u,128,u,10,128,12,128,u,11>
-; AVX512F-FAST-NEXT:    vpshufb %xmm5, %xmm0, %xmm0
+; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} xmm4 = <8,128,u,7,128,9,128,u,128,u,10,128,12,128,u,11>
+; AVX512F-FAST-NEXT:    vpshufb %xmm4, %xmm0, %xmm0
+; AVX512F-FAST-NEXT:    vmovdqa64 %xmm4, %xmm25
 ; AVX512F-FAST-NEXT:    vmovdqa 32(%rsi), %xmm4
-; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} xmm13 = <128,8,u,128,7,128,9,u,11,u,128,10,128,12,u,128>
-; AVX512F-FAST-NEXT:    vpshufb %xmm13, %xmm4, %xmm2
-; AVX512F-FAST-NEXT:    vmovdqa64 %xmm4, %xmm19
-; AVX512F-FAST-NEXT:    vpor %xmm0, %xmm2, %xmm0
+; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} xmm5 = <128,8,u,128,7,128,9,u,11,u,128,10,128,12,u,128>
+; AVX512F-FAST-NEXT:    vpshufb %xmm5, %xmm4, %xmm3
+; AVX512F-FAST-NEXT:    vmovdqa64 %xmm5, %xmm26
+; AVX512F-FAST-NEXT:    vmovdqa64 %xmm4, %xmm20
+; AVX512F-FAST-NEXT:    vpor %xmm0, %xmm3, %xmm0
 ; AVX512F-FAST-NEXT:    vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512F-FAST-NEXT:    vmovdqa 32(%rcx), %ymm7
+; AVX512F-FAST-NEXT:    vmovdqa 32(%rcx), %ymm10
 ; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} ymm0 = [128,128,128,128,13,128,128,128,128,14,128,128,128,128,15,128,128,128,128,16,128,128,128,128,17,128,128,128,128,18,128,128]
-; AVX512F-FAST-NEXT:    vpshufb %ymm0, %ymm7, %ymm2
-; AVX512F-FAST-NEXT:    vmovdqa 32(%rdx), %ymm9
-; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} ymm15 = <u,u,12,13,128,u,u,u,14,128,u,u,14,15,128,u,u,u,16,128,u,u,16,17,128,u,u,u,18,128,u,u>
-; AVX512F-FAST-NEXT:    vpshufb %ymm15, %ymm9, %ymm4
-; AVX512F-FAST-NEXT:    vpor %ymm2, %ymm4, %ymm2
-; AVX512F-FAST-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512F-FAST-NEXT:    vmovdqa 32(%rcx), %xmm12
-; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} xmm4 = <128,6,128,8,u,128,7,128,9,128,11,u,128,10,128,12>
-; AVX512F-FAST-NEXT:    vpshufb %xmm4, %xmm12, %xmm2
-; AVX512F-FAST-NEXT:    vmovdqa64 %xmm4, %xmm25
-; AVX512F-FAST-NEXT:    vmovdqa 32(%rdx), %xmm10
-; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} xmm6 = <6,128,8,128,u,7,128,9,128,11,128,u,10,128,12,128>
-; AVX512F-FAST-NEXT:    vpshufb %xmm6, %xmm10, %xmm4
-; AVX512F-FAST-NEXT:    vmovdqa64 %xmm6, %xmm26
-; AVX512F-FAST-NEXT:    vpor %xmm2, %xmm4, %xmm2
-; AVX512F-FAST-NEXT:    vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} ymm2 = ymm3[11,u,u,10,u,12,u,u,u,u,13,u,15,u,u,14,27,u,u,26,u,28,u,u,u,u,29,u,31,u,u,30]
-; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} ymm3 = ymm3[3,u,5,u,u,4,u,6,u,8,u,u,7,u,9,u,19,u,21,u,u,20,u,22,u,24,u,u,23,u,25,u]
-; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm2, %zmm3, %zmm21
-; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} ymm2 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm1[26],zero,ymm1[28],zero,zero,ymm1[27],zero,ymm1[29],zero,ymm1[31],zero,zero,ymm1[30],zero
+; AVX512F-FAST-NEXT:    vpshufb %ymm0, %ymm10, %ymm4
+; AVX512F-FAST-NEXT:    vmovdqa 32(%rdx), %ymm11
+; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} ymm3 = <u,u,12,13,128,u,u,u,14,128,u,u,14,15,128,u,u,u,16,128,u,u,16,17,128,u,u,u,18,128,u,u>
+; AVX512F-FAST-NEXT:    vpshufb %ymm3, %ymm11, %ymm5
+; AVX512F-FAST-NEXT:    vpor %ymm4, %ymm5, %ymm4
+; AVX512F-FAST-NEXT:    vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512F-FAST-NEXT:    vmovdqa 32(%rcx), %xmm14
+; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} xmm5 = <128,6,128,8,u,128,7,128,9,128,11,u,128,10,128,12>
+; AVX512F-FAST-NEXT:    vpshufb %xmm5, %xmm14, %xmm4
+; AVX512F-FAST-NEXT:    vmovdqa64 %xmm5, %xmm27
+; AVX512F-FAST-NEXT:    vmovdqa 32(%rdx), %xmm13
+; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} xmm15 = <6,128,8,128,u,7,128,9,128,11,128,u,10,128,12,128>
+; AVX512F-FAST-NEXT:    vpshufb %xmm15, %xmm13, %xmm5
+; AVX512F-FAST-NEXT:    vpor %xmm4, %xmm5, %xmm4
+; AVX512F-FAST-NEXT:    vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} ymm4 = ymm9[11,u,u,10,u,12,u,u,u,u,13,u,15,u,u,14,27,u,u,26,u,28,u,u,u,u,29,u,31,u,u,30]
+; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} ymm5 = ymm9[3,u,5,u,u,4,u,6,u,8,u,u,7,u,9,u,19,u,21,u,u,20,u,22,u,24,u,u,23,u,25,u]
+; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm4, %zmm5, %zmm22
+; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} ymm4 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm1[26],zero,ymm1[28],zero,zero,ymm1[27],zero,ymm1[29],zero,ymm1[31],zero,zero,ymm1[30],zero
 ; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} ymm1 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm1[21],zero,zero,ymm1[20],zero,ymm1[22],zero,ymm1[24],zero,zero,ymm1[23],zero,ymm1[25],zero,zero
-; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm2, %zmm1, %zmm22
-; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} ymm1 = ymm9[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm9[27],zero,zero,ymm9[26],zero,ymm9[28],zero,ymm9[30],zero,zero,ymm9[29],zero,ymm9[31],zero,zero
-; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} ymm2 = ymm7[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm7[19],zero,ymm7[21],zero,zero,ymm7[20],zero,ymm7[22],zero,ymm7[24],zero,zero,ymm7[23],zero
-; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm1, %zmm2, %zmm23
-; AVX512F-FAST-NEXT:    vmovdqa (%rcx), %ymm4
-; AVX512F-FAST-NEXT:    vpshufb %ymm0, %ymm4, %ymm0
-; AVX512F-FAST-NEXT:    vmovdqa (%rdx), %ymm14
-; AVX512F-FAST-NEXT:    vpshufb %ymm15, %ymm14, %ymm1
-; AVX512F-FAST-NEXT:    vporq %ymm0, %ymm1, %ymm24
-; AVX512F-FAST-NEXT:    vmovdqa (%rsi), %ymm15
-; AVX512F-FAST-NEXT:    vpshufb %ymm8, %ymm15, %ymm0
-; AVX512F-FAST-NEXT:    vmovdqa (%rdi), %ymm6
-; AVX512F-FAST-NEXT:    vpshufb %ymm11, %ymm6, %ymm1
-; AVX512F-FAST-NEXT:    vporq %ymm0, %ymm1, %ymm20
-; AVX512F-FAST-NEXT:    vmovdqa (%rdi), %xmm0
-; AVX512F-FAST-NEXT:    vpshufb %xmm5, %xmm0, %xmm1
-; AVX512F-FAST-NEXT:    vmovdqa64 %xmm0, %xmm16
+; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm4, %zmm1, %zmm23
+; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} ymm1 = ymm11[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm11[27],zero,zero,ymm11[26],zero,ymm11[28],zero,ymm11[30],zero,zero,ymm11[29],zero,ymm11[31],zero,zero
+; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} ymm4 = ymm10[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm10[19],zero,ymm10[21],zero,zero,ymm10[20],zero,ymm10[22],zero,ymm10[24],zero,zero,ymm10[23],zero
+; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm1, %zmm4, %zmm24
+; AVX512F-FAST-NEXT:    vmovdqa (%rcx), %ymm12
+; AVX512F-FAST-NEXT:    vpshufb %ymm0, %ymm12, %ymm0
+; AVX512F-FAST-NEXT:    vmovdqa (%rdx), %ymm7
+; AVX512F-FAST-NEXT:    vpshufb %ymm3, %ymm7, %ymm1
+; AVX512F-FAST-NEXT:    vporq %ymm0, %ymm1, %ymm19
+; AVX512F-FAST-NEXT:    vmovdqa (%rsi), %ymm5
+; AVX512F-FAST-NEXT:    vpshufb %ymm6, %ymm5, %ymm0
+; AVX512F-FAST-NEXT:    vmovdqa (%rdi), %ymm8
+; AVX512F-FAST-NEXT:    vpshufb %ymm2, %ymm8, %ymm1
+; AVX512F-FAST-NEXT:    vporq %ymm0, %ymm1, %ymm21
+; AVX512F-FAST-NEXT:    vmovdqa (%rdi), %xmm1
+; AVX512F-FAST-NEXT:    vmovdqa64 %xmm25, %xmm0
+; AVX512F-FAST-NEXT:    vpshufb %xmm0, %xmm1, %xmm0
+; AVX512F-FAST-NEXT:    vmovdqa64 %xmm1, %xmm16
 ; AVX512F-FAST-NEXT:    vmovdqa (%rsi), %xmm3
-; AVX512F-FAST-NEXT:    vpshufb %xmm13, %xmm3, %xmm2
-; AVX512F-FAST-NEXT:    vporq %xmm1, %xmm2, %xmm28
+; AVX512F-FAST-NEXT:    vmovdqa64 %xmm26, %xmm1
+; AVX512F-FAST-NEXT:    vpshufb %xmm1, %xmm3, %xmm2
+; AVX512F-FAST-NEXT:    vporq %xmm0, %xmm2, %xmm28
 ; AVX512F-FAST-NEXT:    vmovdqa (%rcx), %xmm1
-; AVX512F-FAST-NEXT:    vmovdqa64 %xmm25, %xmm0
-; AVX512F-FAST-NEXT:    vpshufb %xmm0, %xmm1, %xmm8
+; AVX512F-FAST-NEXT:    vmovdqa64 %xmm27, %xmm0
+; AVX512F-FAST-NEXT:    vpshufb %xmm0, %xmm1, %xmm0
 ; AVX512F-FAST-NEXT:    vmovdqa64 %xmm1, %xmm18
 ; AVX512F-FAST-NEXT:    vmovdqa (%rdx), %xmm1
-; AVX512F-FAST-NEXT:    vmovdqa64 %xmm26, %xmm0
-; AVX512F-FAST-NEXT:    vpshufb %xmm0, %xmm1, %xmm11
+; AVX512F-FAST-NEXT:    vpshufb %xmm15, %xmm1, %xmm15
 ; AVX512F-FAST-NEXT:    vmovdqa64 %xmm1, %xmm17
-; AVX512F-FAST-NEXT:    vporq %xmm8, %xmm11, %xmm29
-; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} ymm11 = [1,1,2,2,2,2,2,2]
-; AVX512F-FAST-NEXT:    vmovdqa 32(%r8), %ymm8
-; AVX512F-FAST-NEXT:    vpermd %ymm8, %ymm11, %ymm11
-; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm25 = [255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255]
-; AVX512F-FAST-NEXT:    vpandnq %ymm11, %ymm25, %ymm11
-; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} ymm0 = [12,128,128,128,128,13,128,128,128,128,14,128,128,128,128,15,128,128,128,128,16,128,128,128,128,17,128,128,128,128,18,128]
-; AVX512F-FAST-NEXT:    vpshufb %ymm0, %ymm8, %ymm13
-; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm13, %zmm11, %zmm26
-; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm31 = <4,u,5,5,5,5,u,6,30,30,30,u,31,31,31,31>
-; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm30 = [255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0]
-; AVX512F-FAST-NEXT:    vmovdqa (%r8), %ymm11
-; AVX512F-FAST-NEXT:    vpermd %ymm11, %ymm31, %ymm27
-; AVX512F-FAST-NEXT:    vpandnq %ymm27, %ymm30, %ymm27
-; AVX512F-FAST-NEXT:    vpshufb %ymm0, %ymm11, %ymm0
-; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm27, %zmm0, %zmm27
-; AVX512F-FAST-NEXT:    vpbroadcastq {{.*#+}} ymm0 = [9,14,11,0,13,10,15,12,9,14,11,0,13,10,15,12,9,14,11,0,13,10,15,12,9,14,11,0,13,10,15,12]
-; AVX512F-FAST-NEXT:    vpshufb %ymm0, %ymm7, %ymm7
-; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} ymm13 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm4[19],zero,ymm4[21],zero,zero,ymm4[20],zero,ymm4[22],zero,ymm4[24],zero,zero,ymm4[23],zero
-; AVX512F-FAST-NEXT:    vpshufb %ymm0, %ymm4, %ymm5
-; AVX512F-FAST-NEXT:    vbroadcasti128 {{.*#+}} ymm0 = [18,19,128,21,128,21,20,128,22,128,24,128,22,23,128,25,18,19,128,21,128,21,20,128,22,128,24,128,22,23,128,25]
-; AVX512F-FAST-NEXT:    # ymm0 = mem[0,1,0,1]
-; AVX512F-FAST-NEXT:    vpshufb %ymm0, %ymm9, %ymm9
-; AVX512F-FAST-NEXT:    vpshufb %ymm0, %ymm14, %ymm4
-; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} ymm14 = ymm14[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm14[27],zero,zero,ymm14[26],zero,ymm14[28],zero,ymm14[30],zero,zero,ymm14[29],zero,ymm14[31],zero,zero
-; AVX512F-FAST-NEXT:    vpunpcklbw {{.*#+}} xmm10 = xmm12[0],xmm10[0],xmm12[1],xmm10[1],xmm12[2],xmm10[2],xmm12[3],xmm10[3],xmm12[4],xmm10[4],xmm12[5],xmm10[5],xmm12[6],xmm10[6],xmm12[7],xmm10[7]
-; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} ymm12 = ymm15[3,u,5,u,u,4,u,6,u,8,u,u,7,u,9,u,19,u,21,u,u,20,u,22,u,24,u,u,23,u,25,u]
-; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} ymm15 = ymm15[11,u,u,10,u,12,u,u,u,u,13,u,15,u,u,14,27,u,u,26,u,28,u,u,u,u,29,u,31,u,u,30]
-; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} ymm2 = ymm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm6[21],zero,zero,ymm6[20],zero,ymm6[22],zero,ymm6[24],zero,zero,ymm6[23],zero,ymm6[25],zero,zero
-; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} ymm6 = ymm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm6[26],zero,ymm6[28],zero,zero,ymm6[27],zero,ymm6[29],zero,ymm6[31],zero,zero,ymm6[30],zero
-; AVX512F-FAST-NEXT:    vmovdqa64 %xmm19, %xmm1
+; AVX512F-FAST-NEXT:    vporq %xmm0, %xmm15, %xmm29
+; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} ymm15 = [1,1,2,2,2,2,2,2]
+; AVX512F-FAST-NEXT:    vmovdqa 32(%r8), %ymm0
+; AVX512F-FAST-NEXT:    vpermd %ymm0, %ymm15, %ymm15
+; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm30 = [255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255]
+; AVX512F-FAST-NEXT:    vpandnq %ymm15, %ymm30, %ymm15
+; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} ymm2 = [12,128,128,128,128,13,128,128,128,128,14,128,128,128,128,15,128,128,128,128,16,128,128,128,128,17,128,128,128,128,18,128]
+; AVX512F-FAST-NEXT:    vpshufb %ymm2, %ymm0, %ymm9
+; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm9, %zmm15, %zmm27
+; AVX512F-FAST-NEXT:    vmovdqa64 (%r8), %zmm25
+; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm15 = <4,u,5,5,5,5,u,6,30,30,30,u,31,31,31,31>
+; AVX512F-FAST-NEXT:    vpermi2d %zmm25, %zmm0, %zmm15
+; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} ymm9 = [4,6,5,5,5,5,4,6]
+; AVX512F-FAST-NEXT:    vmovdqa (%r8), %ymm1
+; AVX512F-FAST-NEXT:    vpermd %ymm1, %ymm9, %ymm9
+; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm31 = [255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0]
+; AVX512F-FAST-NEXT:    vpandnq %ymm9, %ymm31, %ymm9
+; AVX512F-FAST-NEXT:    vpshufb %ymm2, %ymm1, %ymm2
+; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm9, %zmm2, %zmm26
+; AVX512F-FAST-NEXT:    vpbroadcastq {{.*#+}} ymm2 = [9,14,11,0,13,10,15,12,9,14,11,0,13,10,15,12,9,14,11,0,13,10,15,12,9,14,11,0,13,10,15,12]
+; AVX512F-FAST-NEXT:    vpshufb %ymm2, %ymm10, %ymm9
+; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} ymm10 = ymm12[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm12[19],zero,ymm12[21],zero,zero,ymm12[20],zero,ymm12[22],zero,ymm12[24],zero,zero,ymm12[23],zero
+; AVX512F-FAST-NEXT:    vpshufb %ymm2, %ymm12, %ymm6
+; AVX512F-FAST-NEXT:    vbroadcasti128 {{.*#+}} ymm12 = [18,19,128,21,128,21,20,128,22,128,24,128,22,23,128,25,18,19,128,21,128,21,20,128,22,128,24,128,22,23,128,25]
+; AVX512F-FAST-NEXT:    # ymm12 = mem[0,1,0,1]
+; AVX512F-FAST-NEXT:    vpshufb %ymm12, %ymm11, %ymm11
+; AVX512F-FAST-NEXT:    vpshufb %ymm12, %ymm7, %ymm12
+; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} ymm7 = ymm7[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm7[27],zero,zero,ymm7[26],zero,ymm7[28],zero,ymm7[30],zero,zero,ymm7[29],zero,ymm7[31],zero,zero
+; AVX512F-FAST-NEXT:    vpunpcklbw {{.*#+}} xmm13 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3],xmm14[4],xmm13[4],xmm14[5],xmm13[5],xmm14[6],xmm13[6],xmm14[7],xmm13[7]
+; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} ymm14 = ymm5[3,u,5,u,u,4,u,6,u,8,u,u,7,u,9,u,19,u,21,u,u,20,u,22,u,24,u,u,23,u,25,u]
+; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} ymm5 = ymm5[11,u,u,10,u,12,u,u,u,u,13,u,15,u,u,14,27,u,u,26,u,28,u,u,u,u,29,u,31,u,u,30]
+; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} ymm4 = ymm8[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm8[21],zero,zero,ymm8[20],zero,ymm8[22],zero,ymm8[24],zero,zero,ymm8[23],zero,ymm8[25],zero,zero
+; AVX512F-FAST-NEXT:    vpshufb {{.*#+}} ymm8 = ymm8[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm8[26],zero,ymm8[28],zero,zero,ymm8[27],zero,ymm8[29],zero,ymm8[31],zero,zero,ymm8[30],zero
+; AVX512F-FAST-NEXT:    vmovdqa64 %xmm20, %xmm2
 ; AVX512F-FAST-NEXT:    vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; AVX512F-FAST-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
-; AVX512F-FAST-NEXT:    vmovdqa64 %xmm16, %xmm1
-; AVX512F-FAST-NEXT:    vpunpcklbw {{.*#+}} xmm3 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3],xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7]
-; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} xmm1 = <0,1,4,5,u,2,3,6,7,10,11,u,8,9,12,13>
-; AVX512F-FAST-NEXT:    vpshufb %xmm1, %xmm0, %xmm0
-; AVX512F-FAST-NEXT:    vpshufb %xmm1, %xmm3, %xmm1
-; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm28, %zmm1, %zmm28
-; AVX512F-FAST-NEXT:    vpermq {{.*#+}} ymm1 = ymm7[2,2,3,3]
-; AVX512F-FAST-NEXT:    vpermq {{.*#+}} ymm7 = ymm9[2,2,3,3]
-; AVX512F-FAST-NEXT:    vpermq {{.*#+}} ymm9 = ymm13[2,2,3,3]
+; AVX512F-FAST-NEXT:    vpunpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; AVX512F-FAST-NEXT:    vmovdqa64 %xmm16, %xmm2
+; AVX512F-FAST-NEXT:    vpunpcklbw {{.*#+}} xmm3 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3],xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]
+; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} xmm2 = <0,1,4,5,u,2,3,6,7,10,11,u,8,9,12,13>
+; AVX512F-FAST-NEXT:    vpshufb %xmm2, %xmm0, %xmm0
+; AVX512F-FAST-NEXT:    vpshufb %xmm2, %xmm3, %xmm2
+; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm28, %zmm2, %zmm28
+; AVX512F-FAST-NEXT:    vpermq {{.*#+}} ymm3 = ymm9[2,2,3,3]
+; AVX512F-FAST-NEXT:    vpermq {{.*#+}} ymm9 = ymm11[2,2,3,3]
+; AVX512F-FAST-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[2,2,3,3]
+; AVX512F-FAST-NEXT:    vpermq {{.*#+}} ymm11 = ymm12[2,2,3,3]
+; AVX512F-FAST-NEXT:    vpermq {{.*#+}} ymm12 = ymm14[2,2,3,3]
 ; AVX512F-FAST-NEXT:    vpermq {{.*#+}} ymm4 = ymm4[2,2,3,3]
-; AVX512F-FAST-NEXT:    vpermq {{.*#+}} ymm12 = ymm12[2,2,3,3]
-; AVX512F-FAST-NEXT:    vpermq {{.*#+}} ymm2 = ymm2[2,2,3,3]
-; AVX512F-FAST-NEXT:    vpermq {{.*#+}} ymm13 = ymm5[2,2,3,3]
-; AVX512F-FAST-NEXT:    vpermq {{.*#+}} ymm14 = ymm14[2,2,3,3]
-; AVX512F-FAST-NEXT:    vmovdqa64 %xmm18, %xmm3
-; AVX512F-FAST-NEXT:    vmovdqa64 %xmm17, %xmm5
-; AVX512F-FAST-NEXT:    vpunpcklbw {{.*#+}} xmm5 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3],xmm3[4],xmm5[4],xmm3[5],xmm5[5],xmm3[6],xmm5[6],xmm3[7],xmm5[7]
-; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} xmm3 = <2,u,1,0,5,4,u,3,u,7,6,11,10,u,9,8>
-; AVX512F-FAST-NEXT:    vpshufb %xmm3, %xmm10, %xmm10
-; AVX512F-FAST-NEXT:    vpermq {{.*#+}} ymm10 = ymm10[0,0,1,1]
-; AVX512F-FAST-NEXT:    vpermq {{.*#+}} ymm15 = ymm15[2,2,3,3]
 ; AVX512F-FAST-NEXT:    vpermq {{.*#+}} ymm6 = ymm6[2,2,3,3]
+; AVX512F-FAST-NEXT:    vpermq {{.*#+}} ymm7 = ymm7[2,2,3,3]
+; AVX512F-FAST-NEXT:    vmovdqa64 %xmm18, %xmm2
+; AVX512F-FAST-NEXT:    vmovdqa64 %xmm17, %xmm14
+; AVX512F-FAST-NEXT:    vpunpcklbw {{.*#+}} xmm14 = xmm2[0],xmm14[0],xmm2[1],xmm14[1],xmm2[2],xmm14[2],xmm2[3],xmm14[3],xmm2[4],xmm14[4],xmm2[5],xmm14[5],xmm2[6],xmm14[6],xmm2[7],xmm14[7]
+; AVX512F-FAST-NEXT:    vmovdqa {{.*#+}} xmm2 = <2,u,1,0,5,4,u,3,u,7,6,11,10,u,9,8>
+; AVX512F-FAST-NEXT:    vpshufb %xmm2, %xmm13, %xmm13
+; AVX512F-FAST-NEXT:    vpermq {{.*#+}} ymm13 = ymm13[0,0,1,1]
+; AVX512F-FAST-NEXT:    vpermq {{.*#+}} ymm5 = ymm5[2,2,3,3]
+; AVX512F-FAST-NEXT:    vpermq {{.*#+}} ymm8 = ymm8[2,2,3,3]
 ; AVX512F-FAST-NEXT:    vpermq {{.*#+}} ymm0 = ymm0[0,0,1,1]
-; AVX512F-FAST-NEXT:    vpshufb %xmm3, %xmm5, %xmm3
-; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm29, %zmm3, %zmm3
-; AVX512F-FAST-NEXT:    vmovdqa64 (%r8), %zmm5
-; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm11, %zmm5, %zmm11
-; AVX512F-FAST-NEXT:    vpermq {{.*#+}} zmm21 = zmm21[2,2,3,3,6,6,7,7]
-; AVX512F-FAST-NEXT:    vpermq {{.*#+}} zmm22 = zmm22[2,2,3,3,6,6,7,7]
-; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm29 = [18374966859431608575,18374966859431608575,18446463693966278400,18446463693966278400,72056498804555775,72056498804555775,18374967950370078975,18374967950370078975]
-; AVX512F-FAST-NEXT:    vpternlogq $248, %zmm29, %zmm21, %zmm22
-; AVX512F-FAST-NEXT:    vpandq %ymm29, %ymm1, %ymm1
-; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm1, %zmm7, %zmm1
-; AVX512F-FAST-NEXT:    vpermq {{.*#+}} zmm7 = zmm23[2,2,3,3,6,6,7,7]
-; AVX512F-FAST-NEXT:    vporq %zmm7, %zmm1, %zmm1
-; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm7 = [0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255]
-; AVX512F-FAST-NEXT:    vpternlogq $226, %zmm22, %zmm7, %zmm1
-; AVX512F-FAST-NEXT:    vpermt2d %zmm5, %zmm31, %zmm8
-; AVX512F-FAST-NEXT:    vpternlogd $184, %zmm1, %zmm30, %zmm8
-; AVX512F-FAST-NEXT:    vmovdqa64 %zmm8, 256(%r9)
-; AVX512F-FAST-NEXT:    vpermq $80, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Folded Reload
-; AVX512F-FAST-NEXT:    # ymm1 = mem[0,0,1,1]
-; AVX512F-FAST-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm1 # 32-byte Folded Reload
-; AVX512F-FAST-NEXT:    vpermq $80, {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Folded Reload
-; AVX512F-FAST-NEXT:    # ymm8 = mem[0,0,1,1]
-; AVX512F-FAST-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm8, %zmm8 # 32-byte Folded Reload
-; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm16 = [255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0]
-; AVX512F-FAST-NEXT:    vpternlogq $226, %zmm1, %zmm16, %zmm8
-; AVX512F-FAST-NEXT:    vpternlogq $248, %zmm25, %zmm8, %zmm26
-; AVX512F-FAST-NEXT:    vpor %ymm4, %ymm9, %ymm1
-; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm1, %zmm24, %zmm1
-; AVX512F-FAST-NEXT:    vpternlogq $248, %ymm29, %ymm12, %ymm2
-; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm2, %zmm20, %zmm2
-; AVX512F-FAST-NEXT:    vpternlogq $226, %zmm1, %zmm16, %zmm2
-; AVX512F-FAST-NEXT:    vpternlogq $248, %ymm29, %ymm13, %ymm14
-; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm10, %zmm14, %zmm1
-; AVX512F-FAST-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm15, %ymm6
-; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm0, %zmm6, %zmm0
-; AVX512F-FAST-NEXT:    vpternlogq $226, %zmm1, %zmm7, %zmm0
-; AVX512F-FAST-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm27
-; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm1 = <6,6,6,u,7,7,7,7,u,8,8,8,8,u,9,9>
-; AVX512F-FAST-NEXT:    vpermd %zmm5, %zmm1, %zmm1
-; AVX512F-FAST-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm1
+; AVX512F-FAST-NEXT:    vpshufb %xmm2, %xmm14, %xmm2
+; AVX512F-FAST-NEXT:    vinserti32x4 $2, %xmm29, %zmm2, %zmm2
+; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm1, %zmm25, %zmm1
+; AVX512F-FAST-NEXT:    vpermq $80, {{[-0-9]+}}(%r{{[sb]}}p), %ymm14 # 32-byte Folded Reload
+; AVX512F-FAST-NEXT:    # ymm14 = mem[0,0,1,1]
+; AVX512F-FAST-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm14, %zmm14 # 32-byte Folded Reload
+; AVX512F-FAST-NEXT:    vpermq $80, {{[-0-9]+}}(%r{{[sb]}}p), %ymm16 # 32-byte Folded Reload
+; AVX512F-FAST-NEXT:    # ymm16 = mem[0,0,1,1]
+; AVX512F-FAST-NEXT:    vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm16, %zmm16 # 32-byte Folded Reload
+; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm17 = [255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0]
+; AVX512F-FAST-NEXT:    vpternlogq $226, %zmm14, %zmm17, %zmm16
+; AVX512F-FAST-NEXT:    vpternlogq $248, %zmm30, %zmm16, %zmm27
+; AVX512F-FAST-NEXT:    vpermq {{.*#+}} zmm14 = zmm22[2,2,3,3,6,6,7,7]
+; AVX512F-FAST-NEXT:    vpermq {{.*#+}} zmm16 = zmm23[2,2,3,3,6,6,7,7]
+; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm18 = [18374966859431608575,18374966859431608575,18446463693966278400,18446463693966278400,72056498804555775,72056498804555775,18374967950370078975,18374967950370078975]
+; AVX512F-FAST-NEXT:    vpternlogq $248, %zmm18, %zmm14, %zmm16
+; AVX512F-FAST-NEXT:    vpandq %ymm18, %ymm3, %ymm3
+; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm3, %zmm9, %zmm3
+; AVX512F-FAST-NEXT:    vpermq {{.*#+}} zmm9 = zmm24[2,2,3,3,6,6,7,7]
+; AVX512F-FAST-NEXT:    vporq %zmm9, %zmm3, %zmm3
+; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm9 = [0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255]
+; AVX512F-FAST-NEXT:    vpternlogq $226, %zmm16, %zmm9, %zmm3
+; AVX512F-FAST-NEXT:    vpternlogd $184, %zmm3, %zmm31, %zmm15
+; AVX512F-FAST-NEXT:    vpor %ymm10, %ymm11, %ymm3
+; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm3, %zmm19, %zmm3
+; AVX512F-FAST-NEXT:    vpternlogq $248, %ymm18, %ymm12, %ymm4
+; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm4, %zmm21, %zmm4
+; AVX512F-FAST-NEXT:    vpternlogq $226, %zmm3, %zmm17, %zmm4
+; AVX512F-FAST-NEXT:    vpternlogq $248, %ymm18, %ymm6, %ymm7
+; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm13, %zmm7, %zmm3
+; AVX512F-FAST-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm5, %ymm8
+; AVX512F-FAST-NEXT:    vinserti64x4 $1, %ymm0, %zmm8, %zmm0
+; AVX512F-FAST-NEXT:    vpternlogq $226, %zmm3, %zmm9, %zmm0
+; AVX512F-FAST-NEXT:    vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm4, %zmm26
+; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm3 = <6,6,6,u,7,7,7,7,u,8,8,8,8,u,9,9>
+; AVX512F-FAST-NEXT:    vpermd %zmm25, %zmm3, %zmm3
+; AVX512F-FAST-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm3
 ; AVX512F-FAST-NEXT:    vpermq {{.*#+}} zmm0 = zmm28[0,0,1,1,4,4,5,5]
-; AVX512F-FAST-NEXT:    vpermq {{.*#+}} zmm2 = zmm3[0,0,1,1,4,4,5,5]
+; AVX512F-FAST-NEXT:    vpermq {{.*#+}} zmm2 = zmm2[0,0,1,1,4,4,5,5]
 ; AVX512F-FAST-NEXT:    vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm2
 ; AVX512F-FAST-NEXT:    vmovdqa64 {{.*#+}} zmm0 = <u,0,0,0,0,u,1,1,9,9,u,10,10,10,10,u>
-; AVX512F-FAST-NEXT:    vpermd %zmm11, %zmm0, %zmm0
+; AVX512F-FAST-NEXT:    vpermd %zmm1, %zmm0, %zmm0
 ; AVX512F-FAST-NEXT:    vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm0
-; AVX512F-FAST-NEXT:    vmovdqa64 %zmm27, 64(%r9)
+; AVX512F-FAST-NEXT:    vmovdqa64 %zmm26, 64(%r9)
 ; AVX512F-FAST-NEXT:    vmovdqa64 %zmm0, (%r9)
-; AVX512F-FAST-NEXT:    vmovdqa64 %zmm1, 128(%r9)
-; AVX512F-FAST-NEXT:    vmovdqa64 %zmm26, 192(%r9)
+; AVX512F-FAST-NEXT:    vmovdqa64 %zmm3, 128(%r9)
+; AVX512F-FAST-NEXT:    vmovdqa64 %zmm15, 256(%r9)
+; AVX512F-FAST-NEXT:    vmovdqa64 %zmm27, 192(%r9)
 ; AVX512F-FAST-NEXT:    addq $24, %rsp
 ; AVX512F-FAST-NEXT:    vzeroupper
 ; AVX512F-FAST-NEXT:    retq
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-7.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-7.ll
index 04c1bd53225206c0565b7604e97a62e2625bdee7..a00cfe5d3d021c02050a5a0fcbf5b4d7c0005614 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-7.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-7.ll
@@ -1162,83 +1162,82 @@ define void @store_i8_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX1-ONLY:       # %bb.0:
 ; AVX1-ONLY-NEXT:    movq {{[0-9]+}}(%rsp), %rax
 ; AVX1-ONLY-NEXT:    movq {{[0-9]+}}(%rsp), %r10
-; AVX1-ONLY-NEXT:    vmovdqa (%rdi), %xmm4
+; AVX1-ONLY-NEXT:    vmovdqa (%rdi), %xmm3
 ; AVX1-ONLY-NEXT:    vmovdqa (%rsi), %xmm5
 ; AVX1-ONLY-NEXT:    vmovdqa (%rdx), %xmm6
 ; AVX1-ONLY-NEXT:    vmovdqa (%rcx), %xmm7
 ; AVX1-ONLY-NEXT:    vmovdqa (%r8), %xmm0
 ; AVX1-ONLY-NEXT:    vmovdqa (%r9), %xmm2
 ; AVX1-ONLY-NEXT:    vmovdqa (%r10), %xmm1
-; AVX1-ONLY-NEXT:    vpunpcklbw {{.*#+}} xmm9 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3],xmm4[4],xmm5[4],xmm4[5],xmm5[5],xmm4[6],xmm5[6],xmm4[7],xmm5[7]
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm3 = zero,zero,zero,zero,zero,xmm9[6,7],zero,zero,zero,zero,zero,xmm9[8,9],zero,zero
-; AVX1-ONLY-NEXT:    vpunpcklbw {{.*#+}} xmm10 = xmm6[0],xmm7[0],xmm6[1],xmm7[1],xmm6[2],xmm7[2],xmm6[3],xmm7[3],xmm6[4],xmm7[4],xmm6[5],xmm7[5],xmm6[6],xmm7[6],xmm6[7],xmm7[7]
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm8 = xmm10[4,5],zero,zero,zero,zero,zero,xmm10[6,7],zero,zero,zero,zero,zero,xmm10[8,9]
-; AVX1-ONLY-NEXT:    vpor %xmm3, %xmm8, %xmm3
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm8 = zero,zero,xmm10[0,1],zero,zero,zero,zero,zero,xmm10[2,3],zero,zero,zero,zero,zero
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm11 = xmm9[0,1],zero,zero,zero,zero,zero,xmm9[2,3],zero,zero,zero,zero,zero,xmm9[4,5]
-; AVX1-ONLY-NEXT:    vpor %xmm8, %xmm11, %xmm8
-; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm3, %ymm8, %ymm3
-; AVX1-ONLY-NEXT:    vpunpcklbw {{.*#+}} xmm8 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm11 = xmm8[u,u,4,5],zero,xmm8[u,u,u,u,6,7],zero,xmm8[u,u,u,u]
+; AVX1-ONLY-NEXT:    vpunpcklbw {{.*#+}} xmm10 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3],xmm3[4],xmm5[4],xmm3[5],xmm5[5],xmm3[6],xmm5[6],xmm3[7],xmm5[7]
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm4 = zero,zero,zero,zero,zero,xmm10[6,7],zero,zero,zero,zero,zero,xmm10[8,9],zero,zero
+; AVX1-ONLY-NEXT:    vpunpcklbw {{.*#+}} xmm11 = xmm6[0],xmm7[0],xmm6[1],xmm7[1],xmm6[2],xmm7[2],xmm6[3],xmm7[3],xmm6[4],xmm7[4],xmm6[5],xmm7[5],xmm6[6],xmm7[6],xmm6[7],xmm7[7]
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm8 = xmm11[4,5],zero,zero,zero,zero,zero,xmm11[6,7],zero,zero,zero,zero,zero,xmm11[8,9]
+; AVX1-ONLY-NEXT:    vpor %xmm4, %xmm8, %xmm4
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm8 = zero,zero,xmm11[0,1],zero,zero,zero,zero,zero,xmm11[2,3],zero,zero,zero,zero,zero
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm9 = xmm10[0,1],zero,zero,zero,zero,zero,xmm10[2,3],zero,zero,zero,zero,zero,xmm10[4,5]
+; AVX1-ONLY-NEXT:    vpor %xmm8, %xmm9, %xmm8
+; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm4, %ymm8, %ymm4
+; AVX1-ONLY-NEXT:    vpunpcklbw {{.*#+}} xmm9 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm8 = xmm9[u,u,4,5],zero,xmm9[u,u,u,u,6,7],zero,xmm9[u,u,u,u]
 ; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm12 = xmm1[u,u],zero,zero,xmm1[2,u,u,u,u],zero,zero,xmm1[3,u,u,u,u]
-; AVX1-ONLY-NEXT:    vpor %xmm12, %xmm11, %xmm11
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm12 = xmm8[u,u,u,u,0,1,u,u,u,u,u,2,3,u,u,u]
+; AVX1-ONLY-NEXT:    vpor %xmm12, %xmm8, %xmm8
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm12 = xmm9[u,u,u,u,0,1,u,u,u,u,u,2,3,u,u,u]
 ; AVX1-ONLY-NEXT:    vpalignr {{.*#+}} xmm12 = xmm12[4,5,6,7,8,9,10,11,12,13,14,15],xmm1[0,1,2,3]
 ; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm12 = xmm12[u,u,u,u,0,1,12,u,u,u,u,7,8,13,u,u]
-; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm11, %ymm12, %ymm11
+; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm8, %ymm12, %ymm8
 ; AVX1-ONLY-NEXT:    vmovaps {{.*#+}} ymm12 = [255,255,255,255,0,0,0,255,255,255,255,0,0,0,255,255,255,255,0,0,0,255,255,255,255,0,0,0,255,255,255,255]
-; AVX1-ONLY-NEXT:    vandnps %ymm11, %ymm12, %ymm11
-; AVX1-ONLY-NEXT:    vorps %ymm3, %ymm11, %ymm3
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm11 = xmm5[u],zero,xmm5[7,u,u,u,u,u],zero,xmm5[8,u,u,u,u,u],zero
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm12 = xmm4[u,7],zero,xmm4[u,u,u,u,u,8],zero,xmm4[u,u,u,u,u,9]
-; AVX1-ONLY-NEXT:    vpor %xmm11, %xmm12, %xmm11
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm12 = xmm7[u,u,u],zero,xmm7[7,u,u,u,u,u],zero,xmm7[8,u,u,u,u]
+; AVX1-ONLY-NEXT:    vandnps %ymm8, %ymm12, %ymm8
+; AVX1-ONLY-NEXT:    vorps %ymm4, %ymm8, %ymm4
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm8 = xmm5[u],zero,xmm5[7,u,u,u,u,u],zero,xmm5[8,u,u,u,u,u],zero
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm12 = xmm3[u,7],zero,xmm3[u,u,u,u,u,8],zero,xmm3[u,u,u,u,u,9]
+; AVX1-ONLY-NEXT:    vpor %xmm8, %xmm12, %xmm12
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm8 = xmm7[u,u,u],zero,xmm7[7,u,u,u,u,u],zero,xmm7[8,u,u,u,u]
 ; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm13 = xmm6[u,u,u,7],zero,xmm6[u,u,u,u,u,8],zero,xmm6[u,u,u,u]
+; AVX1-ONLY-NEXT:    vpor %xmm8, %xmm13, %xmm13
+; AVX1-ONLY-NEXT:    vmovdqa {{.*#+}} ymm8 = [255,255,255,0,0,0,255,255,255,255,0,0,0,255,255,255,255,0,0,0,255,255,255,255,0,0,0,255,255,255,255,0]
+; AVX1-ONLY-NEXT:    vpblendvb %xmm8, %xmm12, %xmm13, %xmm12
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm11 = zero,zero,zero,zero,zero,xmm11[10,11],zero,zero,zero,zero,zero,xmm11[12,13],zero,zero
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm10 = zero,zero,zero,xmm10[10,11],zero,zero,zero,zero,zero,xmm10[12,13],zero,zero,zero,zero
+; AVX1-ONLY-NEXT:    vpor %xmm11, %xmm10, %xmm10
+; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm12, %ymm10, %ymm10
+; AVX1-ONLY-NEXT:    vmovaps {{.*#+}} ymm11 = [0,0,0,255,255,255,255,0,0,0,255,255,255,255,0,0,0,255,255,255,255,0,0,0,255,255,255,255,0,0,0,255]
+; AVX1-ONLY-NEXT:    vandps %ymm11, %ymm10, %ymm10
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm12 = xmm2[u,u,u,u,u],zero,xmm2[7,u,u,u,u,u],zero,xmm2[8,u,u]
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm13 = xmm0[u,u,u,u,u,7],zero,xmm0[u,u,u,u,u,8],zero,xmm0[u,u]
 ; AVX1-ONLY-NEXT:    vpor %xmm12, %xmm13, %xmm12
-; AVX1-ONLY-NEXT:    vmovdqa {{.*#+}} xmm13 = <u,255,255,0,0,u,u,u,255,255,0,0,u,u,u,255>
-; AVX1-ONLY-NEXT:    vpblendvb %xmm13, %xmm11, %xmm12, %xmm11
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm10 = zero,zero,zero,zero,zero,xmm10[10,11],zero,zero,zero,zero,zero,xmm10[12,13],zero,zero
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm9 = zero,zero,zero,xmm9[10,11],zero,zero,zero,zero,zero,xmm9[12,13],zero,zero,zero,zero
-; AVX1-ONLY-NEXT:    vpor %xmm10, %xmm9, %xmm9
-; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm11, %ymm9, %ymm9
-; AVX1-ONLY-NEXT:    vmovaps {{.*#+}} ymm10 = [0,0,0,255,255,255,255,0,0,0,255,255,255,255,0,0,0,255,255,255,255,0,0,0,255,255,255,255,0,0,0,255]
-; AVX1-ONLY-NEXT:    vandps %ymm10, %ymm9, %ymm9
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm11 = xmm2[u,u,u,u,u],zero,xmm2[7,u,u,u,u,u],zero,xmm2[8,u,u]
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm12 = xmm0[u,u,u,u,u,7],zero,xmm0[u,u,u,u,u,8],zero,xmm0[u,u]
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm12 = zero,xmm12[u,u,u,u,5,6],zero,xmm12[u,u,u,u,12,13],zero,xmm12[u]
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm13 = xmm1[6,u,u,u,u],zero,zero,xmm1[7,u,u,u,u],zero,zero,xmm1[8,u]
+; AVX1-ONLY-NEXT:    vpor %xmm13, %xmm12, %xmm12
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm9 = xmm9[8,9],zero,xmm9[u,u,u,u,10,11],zero,xmm9[u,u,u,u,12,13]
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm13 = zero,zero,xmm1[4,u,u,u,u],zero,zero,xmm1[5,u,u,u,u],zero,zero
+; AVX1-ONLY-NEXT:    vpor %xmm13, %xmm9, %xmm9
+; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm12, %ymm9, %ymm9
+; AVX1-ONLY-NEXT:    vandnps %ymm9, %ymm11, %ymm9
+; AVX1-ONLY-NEXT:    vorps %ymm9, %ymm10, %ymm9
+; AVX1-ONLY-NEXT:    vpunpckhbw {{.*#+}} xmm10 = xmm3[8],xmm5[8],xmm3[9],xmm5[9],xmm3[10],xmm5[10],xmm3[11],xmm5[11],xmm3[12],xmm5[12],xmm3[13],xmm5[13],xmm3[14],xmm5[14],xmm3[15],xmm5[15]
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm11 = zero,zero,zero,zero,xmm10[8,9],zero,zero,zero,zero,zero,xmm10[10,11],zero,zero,zero
+; AVX1-ONLY-NEXT:    vpunpckhbw {{.*#+}} xmm12 = xmm7[8],xmm6[8],xmm7[9],xmm6[9],xmm7[10],xmm6[10],xmm7[11],xmm6[11],xmm7[12],xmm6[12],xmm7[13],xmm6[13],xmm7[14],xmm6[14],xmm7[15],xmm6[15]
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm12 = xmm12[6],zero,zero,zero,zero,zero,xmm12[9,8],zero,zero,zero,zero,zero,xmm12[11,10],zero
 ; AVX1-ONLY-NEXT:    vpor %xmm11, %xmm12, %xmm11
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm11 = zero,xmm11[u,u,u,u,5,6],zero,xmm11[u,u,u,u,12,13],zero,xmm11[u]
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm12 = xmm1[6,u,u,u,u],zero,zero,xmm1[7,u,u,u,u],zero,zero,xmm1[8,u]
-; AVX1-ONLY-NEXT:    vpor %xmm12, %xmm11, %xmm11
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm8 = xmm8[8,9],zero,xmm8[u,u,u,u,10,11],zero,xmm8[u,u,u,u,12,13]
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm12 = zero,zero,xmm1[4,u,u,u,u],zero,zero,xmm1[5,u,u,u,u],zero,zero
-; AVX1-ONLY-NEXT:    vpor %xmm12, %xmm8, %xmm8
-; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm11, %ymm8, %ymm8
-; AVX1-ONLY-NEXT:    vandnps %ymm8, %ymm10, %ymm8
-; AVX1-ONLY-NEXT:    vorps %ymm8, %ymm9, %ymm8
-; AVX1-ONLY-NEXT:    vpunpckhbw {{.*#+}} xmm9 = xmm4[8],xmm5[8],xmm4[9],xmm5[9],xmm4[10],xmm5[10],xmm4[11],xmm5[11],xmm4[12],xmm5[12],xmm4[13],xmm5[13],xmm4[14],xmm5[14],xmm4[15],xmm5[15]
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm10 = zero,zero,zero,zero,xmm9[8,9],zero,zero,zero,zero,zero,xmm9[10,11],zero,zero,zero
-; AVX1-ONLY-NEXT:    vpunpckhbw {{.*#+}} xmm11 = xmm7[8],xmm6[8],xmm7[9],xmm6[9],xmm7[10],xmm6[10],xmm7[11],xmm6[11],xmm7[12],xmm6[12],xmm7[13],xmm6[13],xmm7[14],xmm6[14],xmm7[15],xmm6[15]
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm11 = xmm11[6],zero,zero,zero,zero,zero,xmm11[9,8],zero,zero,zero,zero,zero,xmm11[11,10],zero
-; AVX1-ONLY-NEXT:    vpor %xmm10, %xmm11, %xmm10
 ; AVX1-ONLY-NEXT:    vpunpckhbw {{.*#+}} xmm6 = xmm6[8],xmm7[8],xmm6[9],xmm7[9],xmm6[10],xmm7[10],xmm6[11],xmm7[11],xmm6[12],xmm7[12],xmm6[13],xmm7[13],xmm6[14],xmm7[14],xmm6[15],xmm7[15]
 ; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm7 = zero,xmm6[2,3],zero,zero,zero,zero,zero,xmm6[4,5],zero,zero,zero,zero,zero,xmm6[6]
-; AVX1-ONLY-NEXT:    vpunpckhbw {{.*#+}} xmm4 = xmm5[8],xmm4[8],xmm5[9],xmm4[9],xmm5[10],xmm4[10],xmm5[11],xmm4[11],xmm5[12],xmm4[12],xmm5[13],xmm4[13],xmm5[14],xmm4[14],xmm5[15],xmm4[15]
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm4 = xmm4[2],zero,zero,zero,zero,zero,xmm4[5,4],zero,zero,zero,zero,zero,xmm4[7,6],zero
-; AVX1-ONLY-NEXT:    vpor %xmm7, %xmm4, %xmm4
-; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm10, %ymm4, %ymm4
+; AVX1-ONLY-NEXT:    vpunpckhbw {{.*#+}} xmm3 = xmm5[8],xmm3[8],xmm5[9],xmm3[9],xmm5[10],xmm3[10],xmm5[11],xmm3[11],xmm5[12],xmm3[12],xmm5[13],xmm3[13],xmm5[14],xmm3[14],xmm5[15],xmm3[15]
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm3 = xmm3[2],zero,zero,zero,zero,zero,xmm3[5,4],zero,zero,zero,zero,zero,xmm3[7,6],zero
+; AVX1-ONLY-NEXT:    vpor %xmm7, %xmm3, %xmm3
+; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm11, %ymm3, %ymm3
 ; AVX1-ONLY-NEXT:    vpunpckhbw {{.*#+}} xmm5 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15]
 ; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm7 = xmm5[u,6,7],zero,xmm5[u,u,u,u,8,9],zero,xmm5[u,u,u,u,10]
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm10 = xmm1[u],zero,zero,xmm1[11,u,u,u,u],zero,zero,xmm1[12,u,u,u,u],zero
-; AVX1-ONLY-NEXT:    vpor %xmm7, %xmm10, %xmm7
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm11 = xmm1[u],zero,zero,xmm1[11,u,u,u,u],zero,zero,xmm1[12,u,u,u,u],zero
+; AVX1-ONLY-NEXT:    vpor %xmm7, %xmm11, %xmm7
 ; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm5 = xmm5[u,u,u,2,3],zero,xmm5[u,u,u,u,4,5],zero,xmm5[u,u,u]
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm10 = xmm1[u,u,u],zero,zero,xmm1[9,u,u,u,u],zero,zero,xmm1[10,u,u,u]
-; AVX1-ONLY-NEXT:    vpor %xmm5, %xmm10, %xmm5
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm11 = xmm1[u,u,u],zero,zero,xmm1[9,u,u,u,u],zero,zero,xmm1[10,u,u,u]
+; AVX1-ONLY-NEXT:    vpor %xmm5, %xmm11, %xmm5
 ; AVX1-ONLY-NEXT:    vinsertf128 $1, %xmm7, %ymm5, %ymm5
-; AVX1-ONLY-NEXT:    vmovaps {{.*#+}} ymm7 = [255,255,255,0,0,0,255,255,255,255,0,0,0,255,255,255,255,0,0,0,255,255,255,255,0,0,0,255,255,255,255,0]
-; AVX1-ONLY-NEXT:    vandnps %ymm5, %ymm7, %ymm5
-; AVX1-ONLY-NEXT:    vorps %ymm5, %ymm4, %ymm4
+; AVX1-ONLY-NEXT:    vandnps %ymm5, %ymm8, %ymm5
+; AVX1-ONLY-NEXT:    vorps %ymm5, %ymm3, %ymm3
 ; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm5 = zero,zero,zero,zero,xmm6[12,13],zero,zero,zero,zero,zero,xmm6[14,15],zero,zero,zero
-; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm6 = zero,zero,xmm9[12,13],zero,zero,zero,zero,zero,xmm9[14,15],zero,zero,zero,zero,zero
+; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm6 = zero,zero,xmm10[12,13],zero,zero,zero,zero,zero,xmm10[14,15],zero,zero,zero,zero,zero
 ; AVX1-ONLY-NEXT:    vpor %xmm5, %xmm6, %xmm5
 ; AVX1-ONLY-NEXT:    vpunpckhbw {{.*#+}} xmm0 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15]
 ; AVX1-ONLY-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[10],zero,xmm0[u,u,u,u,13,12],zero,xmm0[u,u,u,u,15,14],zero
@@ -1247,9 +1246,9 @@ define void @store_i8_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec
 ; AVX1-ONLY-NEXT:    vmovdqa {{.*#+}} xmm1 = [0,0,255,255,255,255,0,0,0,255,255,255,255,0,0,0]
 ; AVX1-ONLY-NEXT:    vpblendvb %xmm1, %xmm5, %xmm0, %xmm0
 ; AVX1-ONLY-NEXT:    vmovdqa %xmm0, 96(%rax)
-; AVX1-ONLY-NEXT:    vmovaps %ymm4, 64(%rax)
-; AVX1-ONLY-NEXT:    vmovaps %ymm8, 32(%rax)
-; AVX1-ONLY-NEXT:    vmovaps %ymm3, (%rax)
+; AVX1-ONLY-NEXT:    vmovaps %ymm3, 64(%rax)
+; AVX1-ONLY-NEXT:    vmovaps %ymm9, 32(%rax)
+; AVX1-ONLY-NEXT:    vmovaps %ymm4, (%rax)
 ; AVX1-ONLY-NEXT:    vzeroupper
 ; AVX1-ONLY-NEXT:    retq
 ;
diff --git a/llvm/test/MC/AMDGPU/ds-err.s b/llvm/test/MC/AMDGPU/ds-err.s
index bc2a2fff354ffe0620be90ba8df47841e4b21860..2d25fdf5e302d1b8df9b7eacb8e4276605008756 100644
--- a/llvm/test/MC/AMDGPU/ds-err.s
+++ b/llvm/test/MC/AMDGPU/ds-err.s
@@ -2,11 +2,11 @@
 // RUN: not llvm-mc -triple=amdgcn -mcpu=tahiti %s 2>&1 | FileCheck --implicit-check-not=error: %s
 
 // offset too big
-// CHECK: :[[@LINE+1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK: :[[@LINE+1]]:{{[0-9]+}}: error: expected a 16-bit unsigned offset
 ds_add_u32 v2, v4 offset:1000000000
 
 // offset too big
-// CHECK: :[[@LINE+1]]:{{[0-9]+}}: error: invalid operand for instruction
+// CHECK: :[[@LINE+1]]:{{[0-9]+}}: error: expected a 16-bit unsigned offset
 ds_add_u32 v2, v4 offset:0x10000
 
 // offset0 twice
diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_err.s b/llvm/test/MC/AMDGPU/gfx11_asm_err.s
index 5e508b466e830b43d73da0fbb5e7f65f18b4df0a..9726dea77f98ef7a2dfee9254504fbcdb911b40a 100644
--- a/llvm/test/MC/AMDGPU/gfx11_asm_err.s
+++ b/llvm/test/MC/AMDGPU/gfx11_asm_err.s
@@ -152,3 +152,9 @@ v_fmac_f32_e64_dpp v5, v2, 0x1234 quad_perm:[3,2,1,0]
 
 s_load_dword s1, s[2:3], s0 0x1
 // GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+scratch_store_b128 off, v[2:5], s0 offset:8000000
+// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: expected a 13-bit signed offset
+
+flat_atomic_add_f32 v1, v[0:1], v2 offset:-1
+// GFX11: :[[@LINE-1]]:{{[0-9]+}}: error: expected a 12-bit unsigned offset
diff --git a/llvm/test/MC/AMDGPU/gfx11_unsupported.s b/llvm/test/MC/AMDGPU/gfx11_unsupported.s
index 3b1b7e47b2143cb2ee31c859a7f46b8b51daecc5..c6ec918147014bbc2801627784191d624f78cd07 100644
--- a/llvm/test/MC/AMDGPU/gfx11_unsupported.s
+++ b/llvm/test/MC/AMDGPU/gfx11_unsupported.s
@@ -1983,3 +1983,27 @@ s_cmp_nlt_f16 s1, s2
 
 s_singleuse_vdst 0x1234
 // CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 offset:0 glc
+// CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+buffer_atomic_max_num_f32 v5, off, s[8:11], s3 offset:4095
+// CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+buffer_atomic_min_num_f32 v5, off, s[8:11], s3 offset:4095
+// CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+global_atomic_sub_clamp_u32 v5, v[1:2], v2, off glc
+// CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+flat_atomic_csub_u32 v1, v[0:1], v2 offset:64 th:TH_ATOMIC_RETURN
+// CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+flat_atomic_sub_clamp_u32 v1, v[0:1], v2 offset:64 th:TH_ATOMIC_RETURN
+// CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+ds_sub_clamp_rtn_u32 v5, v1, v2
+// CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+ds_sub_clamp_u32 v1, v2
+// CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_ds.s b/llvm/test/MC/AMDGPU/gfx12_asm_ds.s
new file mode 100644
index 0000000000000000000000000000000000000000..ba32dc8820eaada3fead319f11d4de5c4e9d4138
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_ds.s
@@ -0,0 +1,1819 @@
+// RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -show-encoding %s | FileCheck --check-prefix=GFX12 %s
+
+ds_nop
+// GFX12: [0x00,0x00,0x50,0xd8,0x00,0x00,0x00,0x00]
+
+ds_add_f32 v1, v2
+// GFX12: [0x00,0x00,0x54,0xd8,0x01,0x02,0x00,0x00]
+
+ds_add_f32 v1, v2 offset:65535
+// GFX12: [0xff,0xff,0x54,0xd8,0x01,0x02,0x00,0x00]
+
+ds_add_f32 v1, v2 offset:0
+// GFX12: [0x00,0x00,0x54,0xd8,0x01,0x02,0x00,0x00]
+
+ds_add_f32 v255, v255 offset:4
+// GFX12: [0x04,0x00,0x54,0xd8,0xff,0xff,0x00,0x00]
+
+ds_add_rtn_f32 v5, v1, v2
+// GFX12: [0x00,0x00,0xe4,0xd9,0x01,0x02,0x00,0x05]
+
+ds_add_rtn_f32 v5, v1, v2 offset:65535
+// GFX12: [0xff,0xff,0xe4,0xd9,0x01,0x02,0x00,0x05]
+
+ds_add_rtn_f32 v5, v1, v2 offset:0
+// GFX12: [0x00,0x00,0xe4,0xd9,0x01,0x02,0x00,0x05]
+
+ds_add_rtn_f32 v255, v255, v255 offset:4
+// GFX12: [0x04,0x00,0xe4,0xd9,0xff,0xff,0x00,0xff]
+
+ds_add_rtn_u32 v5, v1, v2
+// GFX12: [0x00,0x00,0x80,0xd8,0x01,0x02,0x00,0x05]
+
+ds_add_rtn_u32 v5, v1, v2 offset:65535
+// GFX12: [0xff,0xff,0x80,0xd8,0x01,0x02,0x00,0x05]
+
+ds_add_rtn_u32 v5, v1, v2 offset:0
+// GFX12: [0x00,0x00,0x80,0xd8,0x01,0x02,0x00,0x05]
+
+ds_add_rtn_u32 v255, v255, v255 offset:4
+// GFX12: [0x04,0x00,0x80,0xd8,0xff,0xff,0x00,0xff]
+
+ds_add_rtn_u64 v[5:6], v1, v[2:3]
+// GFX12: [0x00,0x00,0x80,0xd9,0x01,0x02,0x00,0x05]
+
+ds_add_rtn_u64 v[5:6], v1, v[2:3] offset:65535
+// GFX12: [0xff,0xff,0x80,0xd9,0x01,0x02,0x00,0x05]
+
+ds_add_rtn_u64 v[5:6], v1, v[2:3] offset:0
+// GFX12: [0x00,0x00,0x80,0xd9,0x01,0x02,0x00,0x05]
+
+ds_add_rtn_u64 v[254:255], v255, v[254:255] offset:4
+// GFX12: [0x04,0x00,0x80,0xd9,0xff,0xfe,0x00,0xfe]
+
+ds_add_u32 v1, v2
+// GFX12: [0x00,0x00,0x00,0xd8,0x01,0x02,0x00,0x00]
+
+ds_add_u32 v1, v2 offset:65535
+// GFX12: [0xff,0xff,0x00,0xd8,0x01,0x02,0x00,0x00]
+
+ds_add_u32 v1, v2 offset:0
+// GFX12: [0x00,0x00,0x00,0xd8,0x01,0x02,0x00,0x00]
+
+ds_add_u32 v255, v255 offset:4
+// GFX12: [0x04,0x00,0x00,0xd8,0xff,0xff,0x00,0x00]
+
+ds_add_u64 v1, v[2:3]
+// GFX12: [0x00,0x00,0x00,0xd9,0x01,0x02,0x00,0x00]
+
+ds_add_u64 v1, v[2:3] offset:65535
+// GFX12: [0xff,0xff,0x00,0xd9,0x01,0x02,0x00,0x00]
+
+ds_add_u64 v1, v[2:3] offset:0
+// GFX12: [0x00,0x00,0x00,0xd9,0x01,0x02,0x00,0x00]
+
+ds_add_u64 v255, v[254:255] offset:4
+// GFX12: [0x04,0x00,0x00,0xd9,0xff,0xfe,0x00,0x00]
+
+ds_and_b32 v1, v2
+// GFX12: [0x00,0x00,0x24,0xd8,0x01,0x02,0x00,0x00]
+
+ds_and_b32 v1, v2 offset:65535
+// GFX12: [0xff,0xff,0x24,0xd8,0x01,0x02,0x00,0x00]
+
+ds_and_b32 v1, v2 offset:0
+// GFX12: [0x00,0x00,0x24,0xd8,0x01,0x02,0x00,0x00]
+
+ds_and_b32 v255, v255 offset:4
+// GFX12: [0x04,0x00,0x24,0xd8,0xff,0xff,0x00,0x00]
+
+ds_and_b64 v1, v[2:3]
+// GFX12: [0x00,0x00,0x24,0xd9,0x01,0x02,0x00,0x00]
+
+ds_and_b64 v1, v[2:3] offset:65535
+// GFX12: [0xff,0xff,0x24,0xd9,0x01,0x02,0x00,0x00]
+
+ds_and_b64 v1, v[2:3] offset:0
+// GFX12: [0x00,0x00,0x24,0xd9,0x01,0x02,0x00,0x00]
+
+ds_and_b64 v255, v[254:255] offset:4
+// GFX12: [0x04,0x00,0x24,0xd9,0xff,0xfe,0x00,0x00]
+
+ds_and_rtn_b32 v5, v1, v2
+// GFX12: [0x00,0x00,0xa4,0xd8,0x01,0x02,0x00,0x05]
+
+ds_and_rtn_b32 v5, v1, v2 offset:65535
+// GFX12: [0xff,0xff,0xa4,0xd8,0x01,0x02,0x00,0x05]
+
+ds_and_rtn_b32 v5, v1, v2 offset:0
+// GFX12: [0x00,0x00,0xa4,0xd8,0x01,0x02,0x00,0x05]
+
+ds_and_rtn_b32 v255, v255, v255 offset:4
+// GFX12: [0x04,0x00,0xa4,0xd8,0xff,0xff,0x00,0xff]
+
+ds_and_rtn_b64 v[5:6], v1, v[2:3]
+// GFX12: [0x00,0x00,0xa4,0xd9,0x01,0x02,0x00,0x05]
+
+ds_and_rtn_b64 v[5:6], v1, v[2:3] offset:65535
+// GFX12: [0xff,0xff,0xa4,0xd9,0x01,0x02,0x00,0x05]
+
+ds_and_rtn_b64 v[5:6], v1, v[2:3] offset:0
+// GFX12: [0x00,0x00,0xa4,0xd9,0x01,0x02,0x00,0x05]
+
+ds_and_rtn_b64 v[254:255], v255, v[254:255] offset:4
+// GFX12: [0x04,0x00,0xa4,0xd9,0xff,0xfe,0x00,0xfe]
+
+ds_append v5
+// GFX12: [0x00,0x00,0xf8,0xd8,0x00,0x00,0x00,0x05]
+
+ds_append v5 offset:65535
+// GFX12: [0xff,0xff,0xf8,0xd8,0x00,0x00,0x00,0x05]
+
+ds_append v5 offset:0
+// GFX12: [0x00,0x00,0xf8,0xd8,0x00,0x00,0x00,0x05]
+
+ds_append v255 offset:4
+// GFX12: [0x04,0x00,0xf8,0xd8,0x00,0x00,0x00,0xff]
+
+ds_bpermute_b32 v5, v1, v2
+// GFX12: [0x00,0x00,0xcc,0xda,0x01,0x02,0x00,0x05]
+
+ds_bpermute_b32 v5, v1, v2 offset:65535
+// GFX12: [0xff,0xff,0xcc,0xda,0x01,0x02,0x00,0x05]
+
+ds_bpermute_b32 v5, v1, v2 offset:0
+// GFX12: [0x00,0x00,0xcc,0xda,0x01,0x02,0x00,0x05]
+
+ds_bpermute_b32 v255, v255, v255 offset:4
+// GFX12: [0x04,0x00,0xcc,0xda,0xff,0xff,0x00,0xff]
+
+ds_cmpstore_b32 v1, v2, v3
+// GFX12: [0x00,0x00,0x40,0xd8,0x01,0x02,0x03,0x00]
+
+ds_cmpstore_b32 v1, v2, v3 offset:65535
+// GFX12: [0xff,0xff,0x40,0xd8,0x01,0x02,0x03,0x00]
+
+ds_cmpstore_b32 v1, v2, v3 offset:0
+// GFX12: [0x00,0x00,0x40,0xd8,0x01,0x02,0x03,0x00]
+
+ds_cmpstore_b32 v255, v255, v255 offset:4
+// GFX12: [0x04,0x00,0x40,0xd8,0xff,0xff,0xff,0x00]
+
+ds_cmpstore_b64 v1, v[2:3], v[3:4]
+// GFX12: [0x00,0x00,0x40,0xd9,0x01,0x02,0x03,0x00]
+
+ds_cmpstore_b64 v1, v[2:3], v[3:4] offset:65535
+// GFX12: [0xff,0xff,0x40,0xd9,0x01,0x02,0x03,0x00]
+
+ds_cmpstore_b64 v1, v[2:3], v[3:4] offset:0
+// GFX12: [0x00,0x00,0x40,0xd9,0x01,0x02,0x03,0x00]
+
+ds_cmpstore_b64 v255, v[254:255], v[254:255] offset:4
+// GFX12: [0x04,0x00,0x40,0xd9,0xff,0xfe,0xfe,0x00]
+
+ds_cmpstore_rtn_b32 v5, v1, v2, v3
+// GFX12: [0x00,0x00,0xc0,0xd8,0x01,0x02,0x03,0x05]
+
+ds_cmpstore_rtn_b32 v5, v1, v2, v3 offset:65535
+// GFX12: [0xff,0xff,0xc0,0xd8,0x01,0x02,0x03,0x05]
+
+ds_cmpstore_rtn_b32 v5, v1, v2, v3 offset:0
+// GFX12: [0x00,0x00,0xc0,0xd8,0x01,0x02,0x03,0x05]
+
+ds_cmpstore_rtn_b32 v255, v255, v255, v255 offset:4
+// GFX12: [0x04,0x00,0xc0,0xd8,0xff,0xff,0xff,0xff]
+
+ds_cmpstore_rtn_b64 v[5:6], v1, v[2:3], v[3:4]
+// GFX12: [0x00,0x00,0xc0,0xd9,0x01,0x02,0x03,0x05]
+
+ds_cmpstore_rtn_b64 v[5:6], v1, v[2:3], v[3:4] offset:65535
+// GFX12: [0xff,0xff,0xc0,0xd9,0x01,0x02,0x03,0x05]
+
+ds_cmpstore_rtn_b64 v[5:6], v1, v[2:3], v[3:4] offset:0
+// GFX12: [0x00,0x00,0xc0,0xd9,0x01,0x02,0x03,0x05]
+
+ds_cmpstore_rtn_b64 v[254:255], v255, v[254:255], v[254:255] offset:4
+// GFX12: [0x04,0x00,0xc0,0xd9,0xff,0xfe,0xfe,0xfe]
+
+ds_condxchg32_rtn_b64 v[5:6], v1, v[2:3]
+// GFX12: [0x00,0x00,0xf8,0xd9,0x01,0x02,0x00,0x05]
+
+ds_condxchg32_rtn_b64 v[5:6], v1, v[2:3] offset:65535
+// GFX12: [0xff,0xff,0xf8,0xd9,0x01,0x02,0x00,0x05]
+
+ds_condxchg32_rtn_b64 v[5:6], v1, v[2:3] offset:0
+// GFX12: [0x00,0x00,0xf8,0xd9,0x01,0x02,0x00,0x05]
+
+ds_condxchg32_rtn_b64 v[254:255], v255, v[254:255] offset:4
+// GFX12: [0x04,0x00,0xf8,0xd9,0xff,0xfe,0x00,0xfe]
+
+ds_consume v5
+// GFX12: [0x00,0x00,0xf4,0xd8,0x00,0x00,0x00,0x05]
+
+ds_consume v5 offset:65535
+// GFX12: [0xff,0xff,0xf4,0xd8,0x00,0x00,0x00,0x05]
+
+ds_consume v5 offset:0
+// GFX12: [0x00,0x00,0xf4,0xd8,0x00,0x00,0x00,0x05]
+
+ds_consume v255 offset:4
+// GFX12: [0x04,0x00,0xf4,0xd8,0x00,0x00,0x00,0xff]
+
+ds_dec_rtn_u32 v5, v1, v2
+// GFX12: [0x00,0x00,0x90,0xd8,0x01,0x02,0x00,0x05]
+
+ds_dec_rtn_u32 v5, v1, v2 offset:65535
+// GFX12: [0xff,0xff,0x90,0xd8,0x01,0x02,0x00,0x05]
+
+ds_dec_rtn_u32 v5, v1, v2 offset:0
+// GFX12: [0x00,0x00,0x90,0xd8,0x01,0x02,0x00,0x05]
+
+ds_dec_rtn_u32 v255, v255, v255 offset:4
+// GFX12: [0x04,0x00,0x90,0xd8,0xff,0xff,0x00,0xff]
+
+ds_dec_rtn_u64 v[5:6], v1, v[2:3]
+// GFX12: [0x00,0x00,0x90,0xd9,0x01,0x02,0x00,0x05]
+
+ds_dec_rtn_u64 v[5:6], v1, v[2:3] offset:65535
+// GFX12: [0xff,0xff,0x90,0xd9,0x01,0x02,0x00,0x05]
+
+ds_dec_rtn_u64 v[5:6], v1, v[2:3] offset:0
+// GFX12: [0x00,0x00,0x90,0xd9,0x01,0x02,0x00,0x05]
+
+ds_dec_rtn_u64 v[254:255], v255, v[254:255] offset:4
+// GFX12: [0x04,0x00,0x90,0xd9,0xff,0xfe,0x00,0xfe]
+
+ds_dec_u32 v1, v2
+// GFX12: [0x00,0x00,0x10,0xd8,0x01,0x02,0x00,0x00]
+
+ds_dec_u32 v1, v2 offset:65535
+// GFX12: [0xff,0xff,0x10,0xd8,0x01,0x02,0x00,0x00]
+
+ds_dec_u32 v1, v2 offset:0
+// GFX12: [0x00,0x00,0x10,0xd8,0x01,0x02,0x00,0x00]
+
+ds_dec_u32 v255, v255 offset:4
+// GFX12: [0x04,0x00,0x10,0xd8,0xff,0xff,0x00,0x00]
+
+ds_dec_u64 v1, v[2:3]
+// GFX12: [0x00,0x00,0x10,0xd9,0x01,0x02,0x00,0x00]
+
+ds_dec_u64 v1, v[2:3] offset:65535
+// GFX12: [0xff,0xff,0x10,0xd9,0x01,0x02,0x00,0x00]
+
+ds_dec_u64 v1, v[2:3] offset:0
+// GFX12: [0x00,0x00,0x10,0xd9,0x01,0x02,0x00,0x00]
+
+ds_dec_u64 v255, v[254:255] offset:4
+// GFX12: [0x04,0x00,0x10,0xd9,0xff,0xfe,0x00,0x00]
+
+ds_inc_rtn_u32 v5, v1, v2
+// GFX12: [0x00,0x00,0x8c,0xd8,0x01,0x02,0x00,0x05]
+
+ds_inc_rtn_u32 v5, v1, v2 offset:65535
+// GFX12: [0xff,0xff,0x8c,0xd8,0x01,0x02,0x00,0x05]
+
+ds_inc_rtn_u32 v5, v1, v2 offset:0
+// GFX12: [0x00,0x00,0x8c,0xd8,0x01,0x02,0x00,0x05]
+
+ds_inc_rtn_u32 v255, v255, v255 offset:4
+// GFX12: [0x04,0x00,0x8c,0xd8,0xff,0xff,0x00,0xff]
+
+ds_inc_rtn_u64 v[5:6], v1, v[2:3]
+// GFX12: [0x00,0x00,0x8c,0xd9,0x01,0x02,0x00,0x05]
+
+ds_inc_rtn_u64 v[5:6], v1, v[2:3] offset:65535
+// GFX12: [0xff,0xff,0x8c,0xd9,0x01,0x02,0x00,0x05]
+
+ds_inc_rtn_u64 v[5:6], v1, v[2:3] offset:0
+// GFX12: [0x00,0x00,0x8c,0xd9,0x01,0x02,0x00,0x05]
+
+ds_inc_rtn_u64 v[254:255], v255, v[254:255] offset:4
+// GFX12: [0x04,0x00,0x8c,0xd9,0xff,0xfe,0x00,0xfe]
+
+ds_inc_u32 v1, v2
+// GFX12: [0x00,0x00,0x0c,0xd8,0x01,0x02,0x00,0x00]
+
+ds_inc_u32 v1, v2 offset:65535
+// GFX12: [0xff,0xff,0x0c,0xd8,0x01,0x02,0x00,0x00]
+
+ds_inc_u32 v1, v2 offset:0
+// GFX12: [0x00,0x00,0x0c,0xd8,0x01,0x02,0x00,0x00]
+
+ds_inc_u32 v255, v255 offset:4
+// GFX12: [0x04,0x00,0x0c,0xd8,0xff,0xff,0x00,0x00]
+
+ds_inc_u64 v1, v[2:3]
+// GFX12: [0x00,0x00,0x0c,0xd9,0x01,0x02,0x00,0x00]
+
+ds_inc_u64 v1, v[2:3] offset:65535
+// GFX12: [0xff,0xff,0x0c,0xd9,0x01,0x02,0x00,0x00]
+
+ds_inc_u64 v1, v[2:3] offset:0
+// GFX12: [0x00,0x00,0x0c,0xd9,0x01,0x02,0x00,0x00]
+
+ds_inc_u64 v255, v[254:255] offset:4
+// GFX12: [0x04,0x00,0x0c,0xd9,0xff,0xfe,0x00,0x00]
+
+ds_load_2addr_b32 v[5:6], v1
+// GFX12: [0x00,0x00,0xdc,0xd8,0x01,0x00,0x00,0x05]
+
+ds_load_2addr_b32 v[5:6], v1 offset0:127 offset1:255
+// GFX12: [0x7f,0xff,0xdc,0xd8,0x01,0x00,0x00,0x05]
+
+ds_load_2addr_b32 v[5:6], v1 offset0:0 offset1:0
+// GFX12: [0x00,0x00,0xdc,0xd8,0x01,0x00,0x00,0x05]
+
+ds_load_2addr_b32 v[254:255], v255 offset0:16 offset1:1
+// GFX12: [0x10,0x01,0xdc,0xd8,0xff,0x00,0x00,0xfe]
+
+ds_load_2addr_b64 v[5:8], v1
+// GFX12: [0x00,0x00,0xdc,0xd9,0x01,0x00,0x00,0x05]
+
+ds_load_2addr_b64 v[5:8], v1 offset0:127 offset1:255
+// GFX12: [0x7f,0xff,0xdc,0xd9,0x01,0x00,0x00,0x05]
+
+ds_load_2addr_b64 v[5:8], v1 offset0:0 offset1:0
+// GFX12: [0x00,0x00,0xdc,0xd9,0x01,0x00,0x00,0x05]
+
+ds_load_2addr_b64 v[252:255], v255 offset0:16 offset1:1
+// GFX12: [0x10,0x01,0xdc,0xd9,0xff,0x00,0x00,0xfc]
+
+ds_load_2addr_stride64_b32 v[5:6], v1
+// GFX12: [0x00,0x00,0xe0,0xd8,0x01,0x00,0x00,0x05]
+
+ds_load_2addr_stride64_b32 v[5:6], v1 offset0:127 offset1:255
+// GFX12: [0x7f,0xff,0xe0,0xd8,0x01,0x00,0x00,0x05]
+
+ds_load_2addr_stride64_b32 v[5:6], v1 offset0:0 offset1:0
+// GFX12: [0x00,0x00,0xe0,0xd8,0x01,0x00,0x00,0x05]
+
+ds_load_2addr_stride64_b32 v[254:255], v255 offset0:16 offset1:1
+// GFX12: [0x10,0x01,0xe0,0xd8,0xff,0x00,0x00,0xfe]
+
+ds_load_2addr_stride64_b64 v[5:8], v1
+// GFX12: [0x00,0x00,0xe0,0xd9,0x01,0x00,0x00,0x05]
+
+ds_load_2addr_stride64_b64 v[5:8], v1 offset0:127 offset1:255
+// GFX12: [0x7f,0xff,0xe0,0xd9,0x01,0x00,0x00,0x05]
+
+ds_load_2addr_stride64_b64 v[5:8], v1 offset0:0 offset1:0
+// GFX12: [0x00,0x00,0xe0,0xd9,0x01,0x00,0x00,0x05]
+
+ds_load_2addr_stride64_b64 v[252:255], v255 offset0:16 offset1:1
+// GFX12: [0x10,0x01,0xe0,0xd9,0xff,0x00,0x00,0xfc]
+
+ds_load_addtid_b32 v5
+// GFX12: [0x00,0x00,0xc4,0xda,0x00,0x00,0x00,0x05]
+
+ds_load_addtid_b32 v5 offset:65535
+// GFX12: [0xff,0xff,0xc4,0xda,0x00,0x00,0x00,0x05]
+
+ds_load_addtid_b32 v5 offset:0
+// GFX12: [0x00,0x00,0xc4,0xda,0x00,0x00,0x00,0x05]
+
+ds_load_addtid_b32 v255 offset:4
+// GFX12: [0x04,0x00,0xc4,0xda,0x00,0x00,0x00,0xff]
+
+ds_load_b128 v[5:8], v1
+// GFX12: [0x00,0x00,0xfc,0xdb,0x01,0x00,0x00,0x05]
+
+ds_load_b128 v[5:8], v1 offset:65535
+// GFX12: [0xff,0xff,0xfc,0xdb,0x01,0x00,0x00,0x05]
+
+ds_load_b128 v[5:8], v1 offset:0
+// GFX12: [0x00,0x00,0xfc,0xdb,0x01,0x00,0x00,0x05]
+
+ds_load_b128 v[252:255], v255 offset:4
+// GFX12: [0x04,0x00,0xfc,0xdb,0xff,0x00,0x00,0xfc]
+
+ds_load_b32 v5, v1
+// GFX12: [0x00,0x00,0xd8,0xd8,0x01,0x00,0x00,0x05]
+
+ds_load_b32 v5, v1 offset:65535
+// GFX12: [0xff,0xff,0xd8,0xd8,0x01,0x00,0x00,0x05]
+
+ds_load_b32 v5, v1 offset:0
+// GFX12: [0x00,0x00,0xd8,0xd8,0x01,0x00,0x00,0x05]
+
+ds_load_b32 v255, v255 offset:4
+// GFX12: [0x04,0x00,0xd8,0xd8,0xff,0x00,0x00,0xff]
+
+ds_load_b64 v[5:6], v1
+// GFX12: [0x00,0x00,0xd8,0xd9,0x01,0x00,0x00,0x05]
+
+ds_load_b64 v[5:6], v1 offset:65535
+// GFX12: [0xff,0xff,0xd8,0xd9,0x01,0x00,0x00,0x05]
+
+ds_load_b64 v[5:6], v1 offset:0
+// GFX12: [0x00,0x00,0xd8,0xd9,0x01,0x00,0x00,0x05]
+
+ds_load_b64 v[254:255], v255 offset:4
+// GFX12: [0x04,0x00,0xd8,0xd9,0xff,0x00,0x00,0xfe]
+
+ds_load_b96 v[5:7], v1
+// GFX12: [0x00,0x00,0xf8,0xdb,0x01,0x00,0x00,0x05]
+
+ds_load_b96 v[5:7], v1 offset:65535
+// GFX12: [0xff,0xff,0xf8,0xdb,0x01,0x00,0x00,0x05]
+
+ds_load_b96 v[5:7], v1 offset:0
+// GFX12: [0x00,0x00,0xf8,0xdb,0x01,0x00,0x00,0x05]
+
+ds_load_b96 v[253:255], v255 offset:4
+// GFX12: [0x04,0x00,0xf8,0xdb,0xff,0x00,0x00,0xfd]
+
+ds_load_i16 v5, v1
+// GFX12: [0x00,0x00,0xec,0xd8,0x01,0x00,0x00,0x05]
+
+ds_load_i16 v5, v1 offset:65535
+// GFX12: [0xff,0xff,0xec,0xd8,0x01,0x00,0x00,0x05]
+
+ds_load_i16 v5, v1 offset:0
+// GFX12: [0x00,0x00,0xec,0xd8,0x01,0x00,0x00,0x05]
+
+ds_load_i16 v255, v255 offset:4
+// GFX12: [0x04,0x00,0xec,0xd8,0xff,0x00,0x00,0xff]
+
+ds_load_i8 v5, v1
+// GFX12: [0x00,0x00,0xe4,0xd8,0x01,0x00,0x00,0x05]
+
+ds_load_i8 v5, v1 offset:65535
+// GFX12: [0xff,0xff,0xe4,0xd8,0x01,0x00,0x00,0x05]
+
+ds_load_i8 v5, v1 offset:0
+// GFX12: [0x00,0x00,0xe4,0xd8,0x01,0x00,0x00,0x05]
+
+ds_load_i8 v255, v255 offset:4
+// GFX12: [0x04,0x00,0xe4,0xd8,0xff,0x00,0x00,0xff]
+
+ds_load_i8_d16 v5, v1
+// GFX12: [0x00,0x00,0x90,0xda,0x01,0x00,0x00,0x05]
+
+ds_load_i8_d16 v5, v1 offset:65535
+// GFX12: [0xff,0xff,0x90,0xda,0x01,0x00,0x00,0x05]
+
+ds_load_i8_d16 v5, v1 offset:0
+// GFX12: [0x00,0x00,0x90,0xda,0x01,0x00,0x00,0x05]
+
+ds_load_i8_d16 v255, v255 offset:4
+// GFX12: [0x04,0x00,0x90,0xda,0xff,0x00,0x00,0xff]
+
+ds_load_i8_d16_hi v5, v1
+// GFX12: [0x00,0x00,0x94,0xda,0x01,0x00,0x00,0x05]
+
+ds_load_i8_d16_hi v5, v1 offset:65535
+// GFX12: [0xff,0xff,0x94,0xda,0x01,0x00,0x00,0x05]
+
+ds_load_i8_d16_hi v5, v1 offset:0
+// GFX12: [0x00,0x00,0x94,0xda,0x01,0x00,0x00,0x05]
+
+ds_load_i8_d16_hi v255, v255 offset:4
+// GFX12: [0x04,0x00,0x94,0xda,0xff,0x00,0x00,0xff]
+
+ds_load_u16 v5, v1
+// GFX12: [0x00,0x00,0xf0,0xd8,0x01,0x00,0x00,0x05]
+
+ds_load_u16 v5, v1 offset:65535
+// GFX12: [0xff,0xff,0xf0,0xd8,0x01,0x00,0x00,0x05]
+
+ds_load_u16 v5, v1 offset:0
+// GFX12: [0x00,0x00,0xf0,0xd8,0x01,0x00,0x00,0x05]
+
+ds_load_u16 v255, v255 offset:4
+// GFX12: [0x04,0x00,0xf0,0xd8,0xff,0x00,0x00,0xff]
+
+ds_load_u16_d16 v5, v1
+// GFX12: [0x00,0x00,0x98,0xda,0x01,0x00,0x00,0x05]
+
+ds_load_u16_d16 v5, v1 offset:65535
+// GFX12: [0xff,0xff,0x98,0xda,0x01,0x00,0x00,0x05]
+
+ds_load_u16_d16 v5, v1 offset:0
+// GFX12: [0x00,0x00,0x98,0xda,0x01,0x00,0x00,0x05]
+
+ds_load_u16_d16 v255, v255 offset:4
+// GFX12: [0x04,0x00,0x98,0xda,0xff,0x00,0x00,0xff]
+
+ds_load_u16_d16_hi v5, v1
+// GFX12: [0x00,0x00,0x9c,0xda,0x01,0x00,0x00,0x05]
+
+ds_load_u16_d16_hi v5, v1 offset:65535
+// GFX12: [0xff,0xff,0x9c,0xda,0x01,0x00,0x00,0x05]
+
+ds_load_u16_d16_hi v5, v1 offset:0
+// GFX12: [0x00,0x00,0x9c,0xda,0x01,0x00,0x00,0x05]
+
+ds_load_u16_d16_hi v255, v255 offset:4
+// GFX12: [0x04,0x00,0x9c,0xda,0xff,0x00,0x00,0xff]
+
+ds_load_u8 v5, v1
+// GFX12: [0x00,0x00,0xe8,0xd8,0x01,0x00,0x00,0x05]
+
+ds_load_u8 v5, v1 offset:65535
+// GFX12: [0xff,0xff,0xe8,0xd8,0x01,0x00,0x00,0x05]
+
+ds_load_u8 v5, v1 offset:0
+// GFX12: [0x00,0x00,0xe8,0xd8,0x01,0x00,0x00,0x05]
+
+ds_load_u8 v255, v255 offset:4
+// GFX12: [0x04,0x00,0xe8,0xd8,0xff,0x00,0x00,0xff]
+
+ds_load_u8_d16 v5, v1
+// GFX12: [0x00,0x00,0x88,0xda,0x01,0x00,0x00,0x05]
+
+ds_load_u8_d16 v5, v1 offset:65535
+// GFX12: [0xff,0xff,0x88,0xda,0x01,0x00,0x00,0x05]
+
+ds_load_u8_d16 v5, v1 offset:0
+// GFX12: [0x00,0x00,0x88,0xda,0x01,0x00,0x00,0x05]
+
+ds_load_u8_d16 v255, v255 offset:4
+// GFX12: [0x04,0x00,0x88,0xda,0xff,0x00,0x00,0xff]
+
+ds_load_u8_d16_hi v5, v1
+// GFX12: [0x00,0x00,0x8c,0xda,0x01,0x00,0x00,0x05]
+
+ds_load_u8_d16_hi v5, v1 offset:65535
+// GFX12: [0xff,0xff,0x8c,0xda,0x01,0x00,0x00,0x05]
+
+ds_load_u8_d16_hi v5, v1 offset:0
+// GFX12: [0x00,0x00,0x8c,0xda,0x01,0x00,0x00,0x05]
+
+ds_load_u8_d16_hi v255, v255 offset:4
+// GFX12: [0x04,0x00,0x8c,0xda,0xff,0x00,0x00,0xff]
+
+ds_max_num_f32 v1, v2
+// GFX12: [0x00,0x00,0x4c,0xd8,0x01,0x02,0x00,0x00]
+
+ds_max_num_f32 v1, v2 offset:65535
+// GFX12: [0xff,0xff,0x4c,0xd8,0x01,0x02,0x00,0x00]
+
+ds_max_num_f32 v1, v2 offset:0
+// GFX12: [0x00,0x00,0x4c,0xd8,0x01,0x02,0x00,0x00]
+
+ds_max_num_f32 v255, v255 offset:4
+// GFX12: [0x04,0x00,0x4c,0xd8,0xff,0xff,0x00,0x00]
+
+ds_max_num_f64 v1, v[2:3]
+// GFX12: [0x00,0x00,0x4c,0xd9,0x01,0x02,0x00,0x00]
+
+ds_max_num_f64 v1, v[2:3] offset:65535
+// GFX12: [0xff,0xff,0x4c,0xd9,0x01,0x02,0x00,0x00]
+
+ds_max_num_f64 v1, v[2:3] offset:0
+// GFX12: [0x00,0x00,0x4c,0xd9,0x01,0x02,0x00,0x00]
+
+ds_max_num_f64 v255, v[254:255] offset:4
+// GFX12: [0x04,0x00,0x4c,0xd9,0xff,0xfe,0x00,0x00]
+
+ds_max_i32 v1, v2
+// GFX12: [0x00,0x00,0x18,0xd8,0x01,0x02,0x00,0x00]
+
+ds_max_i32 v1, v2 offset:65535
+// GFX12: [0xff,0xff,0x18,0xd8,0x01,0x02,0x00,0x00]
+
+ds_max_i32 v1, v2 offset:0
+// GFX12: [0x00,0x00,0x18,0xd8,0x01,0x02,0x00,0x00]
+
+ds_max_i32 v255, v255 offset:4
+// GFX12: [0x04,0x00,0x18,0xd8,0xff,0xff,0x00,0x00]
+
+ds_max_i64 v1, v[2:3]
+// GFX12: [0x00,0x00,0x18,0xd9,0x01,0x02,0x00,0x00]
+
+ds_max_i64 v1, v[2:3] offset:65535
+// GFX12: [0xff,0xff,0x18,0xd9,0x01,0x02,0x00,0x00]
+
+ds_max_i64 v1, v[2:3] offset:0
+// GFX12: [0x00,0x00,0x18,0xd9,0x01,0x02,0x00,0x00]
+
+ds_max_i64 v255, v[254:255] offset:4
+// GFX12: [0x04,0x00,0x18,0xd9,0xff,0xfe,0x00,0x00]
+
+ds_max_num_rtn_f32 v5, v1, v2
+// GFX12: [0x00,0x00,0xcc,0xd8,0x01,0x02,0x00,0x05]
+
+ds_max_num_rtn_f32 v5, v1, v2 offset:65535
+// GFX12: [0xff,0xff,0xcc,0xd8,0x01,0x02,0x00,0x05]
+
+ds_max_num_rtn_f32 v5, v1, v2 offset:0
+// GFX12: [0x00,0x00,0xcc,0xd8,0x01,0x02,0x00,0x05]
+
+ds_max_num_rtn_f32 v255, v255, v255 offset:4
+// GFX12: [0x04,0x00,0xcc,0xd8,0xff,0xff,0x00,0xff]
+
+ds_max_num_rtn_f64 v[5:6], v1, v[2:3]
+// GFX12: [0x00,0x00,0xcc,0xd9,0x01,0x02,0x00,0x05]
+
+ds_max_num_rtn_f64 v[5:6], v1, v[2:3] offset:65535
+// GFX12: [0xff,0xff,0xcc,0xd9,0x01,0x02,0x00,0x05]
+
+ds_max_num_rtn_f64 v[5:6], v1, v[2:3] offset:0
+// GFX12: [0x00,0x00,0xcc,0xd9,0x01,0x02,0x00,0x05]
+
+ds_max_num_rtn_f64 v[254:255], v255, v[254:255] offset:4
+// GFX12: [0x04,0x00,0xcc,0xd9,0xff,0xfe,0x00,0xfe]
+
+ds_max_rtn_i32 v5, v1, v2
+// GFX12: [0x00,0x00,0x98,0xd8,0x01,0x02,0x00,0x05]
+
+ds_max_rtn_i32 v5, v1, v2 offset:65535
+// GFX12: [0xff,0xff,0x98,0xd8,0x01,0x02,0x00,0x05]
+
+ds_max_rtn_i32 v5, v1, v2 offset:0
+// GFX12: [0x00,0x00,0x98,0xd8,0x01,0x02,0x00,0x05]
+
+ds_max_rtn_i32 v255, v255, v255 offset:4
+// GFX12: [0x04,0x00,0x98,0xd8,0xff,0xff,0x00,0xff]
+
+ds_max_rtn_i64 v[5:6], v1, v[2:3]
+// GFX12: [0x00,0x00,0x98,0xd9,0x01,0x02,0x00,0x05]
+
+ds_max_rtn_i64 v[5:6], v1, v[2:3] offset:65535
+// GFX12: [0xff,0xff,0x98,0xd9,0x01,0x02,0x00,0x05]
+
+ds_max_rtn_i64 v[5:6], v1, v[2:3] offset:0
+// GFX12: [0x00,0x00,0x98,0xd9,0x01,0x02,0x00,0x05]
+
+ds_max_rtn_i64 v[254:255], v255, v[254:255] offset:4
+// GFX12: [0x04,0x00,0x98,0xd9,0xff,0xfe,0x00,0xfe]
+
+ds_max_rtn_u32 v5, v1, v2
+// GFX12: [0x00,0x00,0xa0,0xd8,0x01,0x02,0x00,0x05]
+
+ds_max_rtn_u32 v5, v1, v2 offset:65535
+// GFX12: [0xff,0xff,0xa0,0xd8,0x01,0x02,0x00,0x05]
+
+ds_max_rtn_u32 v5, v1, v2 offset:0
+// GFX12: [0x00,0x00,0xa0,0xd8,0x01,0x02,0x00,0x05]
+
+ds_max_rtn_u32 v255, v255, v255 offset:4
+// GFX12: [0x04,0x00,0xa0,0xd8,0xff,0xff,0x00,0xff]
+
+ds_max_rtn_u64 v[5:6], v1, v[2:3]
+// GFX12: [0x00,0x00,0xa0,0xd9,0x01,0x02,0x00,0x05]
+
+ds_max_rtn_u64 v[5:6], v1, v[2:3] offset:65535
+// GFX12: [0xff,0xff,0xa0,0xd9,0x01,0x02,0x00,0x05]
+
+ds_max_rtn_u64 v[5:6], v1, v[2:3] offset:0
+// GFX12: [0x00,0x00,0xa0,0xd9,0x01,0x02,0x00,0x05]
+
+ds_max_rtn_u64 v[254:255], v255, v[254:255] offset:4
+// GFX12: [0x04,0x00,0xa0,0xd9,0xff,0xfe,0x00,0xfe]
+
+ds_max_u32 v1, v2
+// GFX12: [0x00,0x00,0x20,0xd8,0x01,0x02,0x00,0x00]
+
+ds_max_u32 v1, v2 offset:65535
+// GFX12: [0xff,0xff,0x20,0xd8,0x01,0x02,0x00,0x00]
+
+ds_max_u32 v1, v2 offset:0
+// GFX12: [0x00,0x00,0x20,0xd8,0x01,0x02,0x00,0x00]
+
+ds_max_u32 v255, v255 offset:4
+// GFX12: [0x04,0x00,0x20,0xd8,0xff,0xff,0x00,0x00]
+
+ds_max_u64 v1, v[2:3]
+// GFX12: [0x00,0x00,0x20,0xd9,0x01,0x02,0x00,0x00]
+
+ds_max_u64 v1, v[2:3] offset:65535
+// GFX12: [0xff,0xff,0x20,0xd9,0x01,0x02,0x00,0x00]
+
+ds_max_u64 v1, v[2:3] offset:0
+// GFX12: [0x00,0x00,0x20,0xd9,0x01,0x02,0x00,0x00]
+
+ds_max_u64 v255, v[254:255] offset:4
+// GFX12: [0x04,0x00,0x20,0xd9,0xff,0xfe,0x00,0x00]
+
+ds_min_num_f32 v1, v2
+// GFX12: [0x00,0x00,0x48,0xd8,0x01,0x02,0x00,0x00]
+
+ds_min_num_f32 v1, v2 offset:65535
+// GFX12: [0xff,0xff,0x48,0xd8,0x01,0x02,0x00,0x00]
+
+ds_min_num_f32 v1, v2 offset:0
+// GFX12: [0x00,0x00,0x48,0xd8,0x01,0x02,0x00,0x00]
+
+ds_min_num_f32 v255, v255 offset:4
+// GFX12: [0x04,0x00,0x48,0xd8,0xff,0xff,0x00,0x00]
+
+ds_min_num_f64 v1, v[2:3]
+// GFX12: [0x00,0x00,0x48,0xd9,0x01,0x02,0x00,0x00]
+
+ds_min_num_f64 v1, v[2:3] offset:65535
+// GFX12: [0xff,0xff,0x48,0xd9,0x01,0x02,0x00,0x00]
+
+ds_min_num_f64 v1, v[2:3] offset:0
+// GFX12: [0x00,0x00,0x48,0xd9,0x01,0x02,0x00,0x00]
+
+ds_min_num_f64 v255, v[254:255] offset:4
+// GFX12: [0x04,0x00,0x48,0xd9,0xff,0xfe,0x00,0x00]
+
+ds_min_i32 v1, v2
+// GFX12: [0x00,0x00,0x14,0xd8,0x01,0x02,0x00,0x00]
+
+ds_min_i32 v1, v2 offset:65535
+// GFX12: [0xff,0xff,0x14,0xd8,0x01,0x02,0x00,0x00]
+
+ds_min_i32 v1, v2 offset:0
+// GFX12: [0x00,0x00,0x14,0xd8,0x01,0x02,0x00,0x00]
+
+ds_min_i32 v255, v255 offset:4
+// GFX12: [0x04,0x00,0x14,0xd8,0xff,0xff,0x00,0x00]
+
+ds_min_i64 v1, v[2:3]
+// GFX12: [0x00,0x00,0x14,0xd9,0x01,0x02,0x00,0x00]
+
+ds_min_i64 v1, v[2:3] offset:65535
+// GFX12: [0xff,0xff,0x14,0xd9,0x01,0x02,0x00,0x00]
+
+ds_min_i64 v1, v[2:3] offset:0
+// GFX12: [0x00,0x00,0x14,0xd9,0x01,0x02,0x00,0x00]
+
+ds_min_i64 v255, v[254:255] offset:4
+// GFX12: [0x04,0x00,0x14,0xd9,0xff,0xfe,0x00,0x00]
+
+ds_min_num_rtn_f32 v5, v1, v2
+// GFX12: [0x00,0x00,0xc8,0xd8,0x01,0x02,0x00,0x05]
+
+ds_min_num_rtn_f32 v5, v1, v2 offset:65535
+// GFX12: [0xff,0xff,0xc8,0xd8,0x01,0x02,0x00,0x05]
+
+ds_min_num_rtn_f32 v5, v1, v2 offset:0
+// GFX12: [0x00,0x00,0xc8,0xd8,0x01,0x02,0x00,0x05]
+
+ds_min_num_rtn_f32 v255, v255, v255 offset:4
+// GFX12: [0x04,0x00,0xc8,0xd8,0xff,0xff,0x00,0xff]
+
+ds_min_num_rtn_f64 v[5:6], v1, v[2:3]
+// GFX12: [0x00,0x00,0xc8,0xd9,0x01,0x02,0x00,0x05]
+
+ds_min_num_rtn_f64 v[5:6], v1, v[2:3] offset:65535
+// GFX12: [0xff,0xff,0xc8,0xd9,0x01,0x02,0x00,0x05]
+
+ds_min_num_rtn_f64 v[5:6], v1, v[2:3] offset:0
+// GFX12: [0x00,0x00,0xc8,0xd9,0x01,0x02,0x00,0x05]
+
+ds_min_num_rtn_f64 v[254:255], v255, v[254:255] offset:4
+// GFX12: [0x04,0x00,0xc8,0xd9,0xff,0xfe,0x00,0xfe]
+
+ds_min_rtn_i32 v5, v1, v2
+// GFX12: [0x00,0x00,0x94,0xd8,0x01,0x02,0x00,0x05]
+
+ds_min_rtn_i32 v5, v1, v2 offset:65535
+// GFX12: [0xff,0xff,0x94,0xd8,0x01,0x02,0x00,0x05]
+
+ds_min_rtn_i32 v5, v1, v2 offset:0
+// GFX12: [0x00,0x00,0x94,0xd8,0x01,0x02,0x00,0x05]
+
+ds_min_rtn_i32 v255, v255, v255 offset:4
+// GFX12: [0x04,0x00,0x94,0xd8,0xff,0xff,0x00,0xff]
+
+ds_min_rtn_i64 v[5:6], v1, v[2:3]
+// GFX12: [0x00,0x00,0x94,0xd9,0x01,0x02,0x00,0x05]
+
+ds_min_rtn_i64 v[5:6], v1, v[2:3] offset:65535
+// GFX12: [0xff,0xff,0x94,0xd9,0x01,0x02,0x00,0x05]
+
+ds_min_rtn_i64 v[5:6], v1, v[2:3] offset:0
+// GFX12: [0x00,0x00,0x94,0xd9,0x01,0x02,0x00,0x05]
+
+ds_min_rtn_i64 v[254:255], v255, v[254:255] offset:4
+// GFX12: [0x04,0x00,0x94,0xd9,0xff,0xfe,0x00,0xfe]
+
+ds_min_rtn_u32 v5, v1, v2
+// GFX12: [0x00,0x00,0x9c,0xd8,0x01,0x02,0x00,0x05]
+
+ds_min_rtn_u32 v5, v1, v2 offset:65535
+// GFX12: [0xff,0xff,0x9c,0xd8,0x01,0x02,0x00,0x05]
+
+ds_min_rtn_u32 v5, v1, v2 offset:0
+// GFX12: [0x00,0x00,0x9c,0xd8,0x01,0x02,0x00,0x05]
+
+ds_min_rtn_u32 v255, v255, v255 offset:4
+// GFX12: [0x04,0x00,0x9c,0xd8,0xff,0xff,0x00,0xff]
+
+ds_min_rtn_u64 v[5:6], v1, v[2:3]
+// GFX12: [0x00,0x00,0x9c,0xd9,0x01,0x02,0x00,0x05]
+
+ds_min_rtn_u64 v[5:6], v1, v[2:3] offset:65535
+// GFX12: [0xff,0xff,0x9c,0xd9,0x01,0x02,0x00,0x05]
+
+ds_min_rtn_u64 v[5:6], v1, v[2:3] offset:0
+// GFX12: [0x00,0x00,0x9c,0xd9,0x01,0x02,0x00,0x05]
+
+ds_min_rtn_u64 v[254:255], v255, v[254:255] offset:4
+// GFX12: [0x04,0x00,0x9c,0xd9,0xff,0xfe,0x00,0xfe]
+
+ds_min_u32 v1, v2
+// GFX12: [0x00,0x00,0x1c,0xd8,0x01,0x02,0x00,0x00]
+
+ds_min_u32 v1, v2 offset:65535
+// GFX12: [0xff,0xff,0x1c,0xd8,0x01,0x02,0x00,0x00]
+
+ds_min_u32 v1, v2 offset:0
+// GFX12: [0x00,0x00,0x1c,0xd8,0x01,0x02,0x00,0x00]
+
+ds_min_u32 v255, v255 offset:4
+// GFX12: [0x04,0x00,0x1c,0xd8,0xff,0xff,0x00,0x00]
+
+ds_min_u64 v1, v[2:3]
+// GFX12: [0x00,0x00,0x1c,0xd9,0x01,0x02,0x00,0x00]
+
+ds_min_u64 v1, v[2:3] offset:65535
+// GFX12: [0xff,0xff,0x1c,0xd9,0x01,0x02,0x00,0x00]
+
+ds_min_u64 v1, v[2:3] offset:0
+// GFX12: [0x00,0x00,0x1c,0xd9,0x01,0x02,0x00,0x00]
+
+ds_min_u64 v255, v[254:255] offset:4
+// GFX12: [0x04,0x00,0x1c,0xd9,0xff,0xfe,0x00,0x00]
+
+ds_mskor_b32 v1, v2, v3
+// GFX12: [0x00,0x00,0x30,0xd8,0x01,0x02,0x03,0x00]
+
+ds_mskor_b32 v1, v2, v3 offset:65535
+// GFX12: [0xff,0xff,0x30,0xd8,0x01,0x02,0x03,0x00]
+
+ds_mskor_b32 v1, v2, v3 offset:0
+// GFX12: [0x00,0x00,0x30,0xd8,0x01,0x02,0x03,0x00]
+
+ds_mskor_b32 v255, v255, v255 offset:4
+// GFX12: [0x04,0x00,0x30,0xd8,0xff,0xff,0xff,0x00]
+
+ds_mskor_b64 v1, v[2:3], v[3:4]
+// GFX12: [0x00,0x00,0x30,0xd9,0x01,0x02,0x03,0x00]
+
+ds_mskor_b64 v1, v[2:3], v[3:4] offset:65535
+// GFX12: [0xff,0xff,0x30,0xd9,0x01,0x02,0x03,0x00]
+
+ds_mskor_b64 v1, v[2:3], v[3:4] offset:0
+// GFX12: [0x00,0x00,0x30,0xd9,0x01,0x02,0x03,0x00]
+
+ds_mskor_b64 v255, v[254:255], v[254:255] offset:4
+// GFX12: [0x04,0x00,0x30,0xd9,0xff,0xfe,0xfe,0x00]
+
+ds_mskor_rtn_b32 v5, v1, v2, v3
+// GFX12: [0x00,0x00,0xb0,0xd8,0x01,0x02,0x03,0x05]
+
+ds_mskor_rtn_b32 v5, v1, v2, v3 offset:65535
+// GFX12: [0xff,0xff,0xb0,0xd8,0x01,0x02,0x03,0x05]
+
+ds_mskor_rtn_b32 v5, v1, v2, v3 offset:0
+// GFX12: [0x00,0x00,0xb0,0xd8,0x01,0x02,0x03,0x05]
+
+ds_mskor_rtn_b32 v255, v255, v255, v255 offset:4
+// GFX12: [0x04,0x00,0xb0,0xd8,0xff,0xff,0xff,0xff]
+
+ds_mskor_rtn_b64 v[5:6], v1, v[2:3], v[3:4]
+// GFX12: [0x00,0x00,0xb0,0xd9,0x01,0x02,0x03,0x05]
+
+ds_mskor_rtn_b64 v[5:6], v1, v[2:3], v[3:4] offset:65535
+// GFX12: [0xff,0xff,0xb0,0xd9,0x01,0x02,0x03,0x05]
+
+ds_mskor_rtn_b64 v[5:6], v1, v[2:3], v[3:4] offset:0
+// GFX12: [0x00,0x00,0xb0,0xd9,0x01,0x02,0x03,0x05]
+
+ds_mskor_rtn_b64 v[254:255], v255, v[254:255], v[254:255] offset:4
+// GFX12: [0x04,0x00,0xb0,0xd9,0xff,0xfe,0xfe,0xfe]
+
+ds_or_b32 v1, v2
+// GFX12: [0x00,0x00,0x28,0xd8,0x01,0x02,0x00,0x00]
+
+ds_or_b32 v1, v2 offset:65535
+// GFX12: [0xff,0xff,0x28,0xd8,0x01,0x02,0x00,0x00]
+
+ds_or_b32 v1, v2 offset:0
+// GFX12: [0x00,0x00,0x28,0xd8,0x01,0x02,0x00,0x00]
+
+ds_or_b32 v255, v255 offset:4
+// GFX12: [0x04,0x00,0x28,0xd8,0xff,0xff,0x00,0x00]
+
+ds_or_b64 v1, v[2:3]
+// GFX12: [0x00,0x00,0x28,0xd9,0x01,0x02,0x00,0x00]
+
+ds_or_b64 v1, v[2:3] offset:65535
+// GFX12: [0xff,0xff,0x28,0xd9,0x01,0x02,0x00,0x00]
+
+ds_or_b64 v1, v[2:3] offset:0
+// GFX12: [0x00,0x00,0x28,0xd9,0x01,0x02,0x00,0x00]
+
+ds_or_b64 v255, v[254:255] offset:4
+// GFX12: [0x04,0x00,0x28,0xd9,0xff,0xfe,0x00,0x00]
+
+ds_or_rtn_b32 v5, v1, v2
+// GFX12: [0x00,0x00,0xa8,0xd8,0x01,0x02,0x00,0x05]
+
+ds_or_rtn_b32 v5, v1, v2 offset:65535
+// GFX12: [0xff,0xff,0xa8,0xd8,0x01,0x02,0x00,0x05]
+
+ds_or_rtn_b32 v5, v1, v2 offset:0
+// GFX12: [0x00,0x00,0xa8,0xd8,0x01,0x02,0x00,0x05]
+
+ds_or_rtn_b32 v255, v255, v255 offset:4
+// GFX12: [0x04,0x00,0xa8,0xd8,0xff,0xff,0x00,0xff]
+
+ds_or_rtn_b64 v[5:6], v1, v[2:3]
+// GFX12: [0x00,0x00,0xa8,0xd9,0x01,0x02,0x00,0x05]
+
+ds_or_rtn_b64 v[5:6], v1, v[2:3] offset:65535
+// GFX12: [0xff,0xff,0xa8,0xd9,0x01,0x02,0x00,0x05]
+
+ds_or_rtn_b64 v[5:6], v1, v[2:3] offset:0
+// GFX12: [0x00,0x00,0xa8,0xd9,0x01,0x02,0x00,0x05]
+
+ds_or_rtn_b64 v[254:255], v255, v[254:255] offset:4
+// GFX12: [0x04,0x00,0xa8,0xd9,0xff,0xfe,0x00,0xfe]
+
+ds_permute_b32 v5, v1, v2
+// GFX12: [0x00,0x00,0xc8,0xda,0x01,0x02,0x00,0x05]
+
+ds_permute_b32 v5, v1, v2 offset:65535
+// GFX12: [0xff,0xff,0xc8,0xda,0x01,0x02,0x00,0x05]
+
+ds_permute_b32 v5, v1, v2 offset:0
+// GFX12: [0x00,0x00,0xc8,0xda,0x01,0x02,0x00,0x05]
+
+ds_permute_b32 v255, v255, v255 offset:4
+// GFX12: [0x04,0x00,0xc8,0xda,0xff,0xff,0x00,0xff]
+
+ds_read2_b32 v[5:6], v1
+// GFX12: [0x00,0x00,0xdc,0xd8,0x01,0x00,0x00,0x05]
+
+ds_read2_b32 v[5:6], v1 offset0:127 offset1:255
+// GFX12: [0x7f,0xff,0xdc,0xd8,0x01,0x00,0x00,0x05]
+
+ds_read2_b32 v[5:6], v1 offset0:0 offset1:0
+// GFX12: [0x00,0x00,0xdc,0xd8,0x01,0x00,0x00,0x05]
+
+ds_read2_b32 v[254:255], v255 offset0:16 offset1:1
+// GFX12: [0x10,0x01,0xdc,0xd8,0xff,0x00,0x00,0xfe]
+
+ds_read2_b64 v[5:8], v1
+// GFX12: [0x00,0x00,0xdc,0xd9,0x01,0x00,0x00,0x05]
+
+ds_read2_b64 v[5:8], v1 offset0:127 offset1:255
+// GFX12: [0x7f,0xff,0xdc,0xd9,0x01,0x00,0x00,0x05]
+
+ds_read2_b64 v[5:8], v1 offset0:0 offset1:0
+// GFX12: [0x00,0x00,0xdc,0xd9,0x01,0x00,0x00,0x05]
+
+ds_read2_b64 v[252:255], v255 offset0:16 offset1:1
+// GFX12: [0x10,0x01,0xdc,0xd9,0xff,0x00,0x00,0xfc]
+
+ds_read2st64_b32 v[5:6], v1
+// GFX12: [0x00,0x00,0xe0,0xd8,0x01,0x00,0x00,0x05]
+
+ds_read2st64_b32 v[5:6], v1 offset0:127 offset1:255
+// GFX12: [0x7f,0xff,0xe0,0xd8,0x01,0x00,0x00,0x05]
+
+ds_read2st64_b32 v[5:6], v1 offset0:0 offset1:0
+// GFX12: [0x00,0x00,0xe0,0xd8,0x01,0x00,0x00,0x05]
+
+ds_read2st64_b32 v[254:255], v255 offset0:16 offset1:1
+// GFX12: [0x10,0x01,0xe0,0xd8,0xff,0x00,0x00,0xfe]
+
+ds_read2st64_b64 v[5:8], v1
+// GFX12: [0x00,0x00,0xe0,0xd9,0x01,0x00,0x00,0x05]
+
+ds_read2st64_b64 v[5:8], v1 offset0:127 offset1:255
+// GFX12: [0x7f,0xff,0xe0,0xd9,0x01,0x00,0x00,0x05]
+
+ds_read2st64_b64 v[5:8], v1 offset0:0 offset1:0
+// GFX12: [0x00,0x00,0xe0,0xd9,0x01,0x00,0x00,0x05]
+
+ds_read2st64_b64 v[252:255], v255 offset0:16 offset1:1
+// GFX12: [0x10,0x01,0xe0,0xd9,0xff,0x00,0x00,0xfc]
+
+ds_read_addtid_b32 v5
+// GFX12: [0x00,0x00,0xc4,0xda,0x00,0x00,0x00,0x05]
+
+ds_read_addtid_b32 v5 offset:65535
+// GFX12: [0xff,0xff,0xc4,0xda,0x00,0x00,0x00,0x05]
+
+ds_read_addtid_b32 v5 offset:0
+// GFX12: [0x00,0x00,0xc4,0xda,0x00,0x00,0x00,0x05]
+
+ds_read_addtid_b32 v255 offset:4
+// GFX12: [0x04,0x00,0xc4,0xda,0x00,0x00,0x00,0xff]
+
+ds_read_b128 v[5:8], v1
+// GFX12: [0x00,0x00,0xfc,0xdb,0x01,0x00,0x00,0x05]
+
+ds_read_b128 v[5:8], v1 offset:65535
+// GFX12: [0xff,0xff,0xfc,0xdb,0x01,0x00,0x00,0x05]
+
+ds_read_b128 v[5:8], v1 offset:0
+// GFX12: [0x00,0x00,0xfc,0xdb,0x01,0x00,0x00,0x05]
+
+ds_read_b128 v[252:255], v255 offset:4
+// GFX12: [0x04,0x00,0xfc,0xdb,0xff,0x00,0x00,0xfc]
+
+ds_read_b32 v5, v1
+// GFX12: [0x00,0x00,0xd8,0xd8,0x01,0x00,0x00,0x05]
+
+ds_read_b32 v5, v1 offset:65535
+// GFX12: [0xff,0xff,0xd8,0xd8,0x01,0x00,0x00,0x05]
+
+ds_read_b32 v5, v1 offset:0
+// GFX12: [0x00,0x00,0xd8,0xd8,0x01,0x00,0x00,0x05]
+
+ds_read_b32 v255, v255 offset:4
+// GFX12: [0x04,0x00,0xd8,0xd8,0xff,0x00,0x00,0xff]
+
+ds_read_b64 v[5:6], v1
+// GFX12: [0x00,0x00,0xd8,0xd9,0x01,0x00,0x00,0x05]
+
+ds_read_b64 v[5:6], v1 offset:65535
+// GFX12: [0xff,0xff,0xd8,0xd9,0x01,0x00,0x00,0x05]
+
+ds_read_b64 v[5:6], v1 offset:0
+// GFX12: [0x00,0x00,0xd8,0xd9,0x01,0x00,0x00,0x05]
+
+ds_read_b64 v[254:255], v255 offset:4
+// GFX12: [0x04,0x00,0xd8,0xd9,0xff,0x00,0x00,0xfe]
+
+ds_read_b96 v[5:7], v1
+// GFX12: [0x00,0x00,0xf8,0xdb,0x01,0x00,0x00,0x05]
+
+ds_read_b96 v[5:7], v1 offset:65535
+// GFX12: [0xff,0xff,0xf8,0xdb,0x01,0x00,0x00,0x05]
+
+ds_read_b96 v[5:7], v1 offset:0
+// GFX12: [0x00,0x00,0xf8,0xdb,0x01,0x00,0x00,0x05]
+
+ds_read_b96 v[253:255], v255 offset:4
+// GFX12: [0x04,0x00,0xf8,0xdb,0xff,0x00,0x00,0xfd]
+
+ds_read_i16 v5, v1
+// GFX12: [0x00,0x00,0xec,0xd8,0x01,0x00,0x00,0x05]
+
+ds_read_i16 v5, v1 offset:65535
+// GFX12: [0xff,0xff,0xec,0xd8,0x01,0x00,0x00,0x05]
+
+ds_read_i16 v5, v1 offset:0
+// GFX12: [0x00,0x00,0xec,0xd8,0x01,0x00,0x00,0x05]
+
+ds_read_i16 v255, v255 offset:4
+// GFX12: [0x04,0x00,0xec,0xd8,0xff,0x00,0x00,0xff]
+
+ds_read_i8 v5, v1
+// GFX12: [0x00,0x00,0xe4,0xd8,0x01,0x00,0x00,0x05]
+
+ds_read_i8 v5, v1 offset:65535
+// GFX12: [0xff,0xff,0xe4,0xd8,0x01,0x00,0x00,0x05]
+
+ds_read_i8 v5, v1 offset:0
+// GFX12: [0x00,0x00,0xe4,0xd8,0x01,0x00,0x00,0x05]
+
+ds_read_i8 v255, v255 offset:4
+// GFX12: [0x04,0x00,0xe4,0xd8,0xff,0x00,0x00,0xff]
+
+ds_read_i8_d16 v5, v1
+// GFX12: [0x00,0x00,0x90,0xda,0x01,0x00,0x00,0x05]
+
+ds_read_i8_d16 v5, v1 offset:65535
+// GFX12: [0xff,0xff,0x90,0xda,0x01,0x00,0x00,0x05]
+
+ds_read_i8_d16 v5, v1 offset:0
+// GFX12: [0x00,0x00,0x90,0xda,0x01,0x00,0x00,0x05]
+
+ds_read_i8_d16 v255, v255 offset:4
+// GFX12: [0x04,0x00,0x90,0xda,0xff,0x00,0x00,0xff]
+
+ds_read_i8_d16_hi v5, v1
+// GFX12: [0x00,0x00,0x94,0xda,0x01,0x00,0x00,0x05]
+
+ds_read_i8_d16_hi v5, v1 offset:65535
+// GFX12: [0xff,0xff,0x94,0xda,0x01,0x00,0x00,0x05]
+
+ds_read_i8_d16_hi v5, v1 offset:0
+// GFX12: [0x00,0x00,0x94,0xda,0x01,0x00,0x00,0x05]
+
+ds_read_i8_d16_hi v255, v255 offset:4
+// GFX12: [0x04,0x00,0x94,0xda,0xff,0x00,0x00,0xff]
+
+ds_read_u16 v5, v1
+// GFX12: [0x00,0x00,0xf0,0xd8,0x01,0x00,0x00,0x05]
+
+ds_read_u16 v5, v1 offset:65535
+// GFX12: [0xff,0xff,0xf0,0xd8,0x01,0x00,0x00,0x05]
+
+ds_read_u16 v5, v1 offset:0
+// GFX12: [0x00,0x00,0xf0,0xd8,0x01,0x00,0x00,0x05]
+
+ds_read_u16 v255, v255 offset:4
+// GFX12: [0x04,0x00,0xf0,0xd8,0xff,0x00,0x00,0xff]
+
+ds_read_u16_d16 v5, v1
+// GFX12: [0x00,0x00,0x98,0xda,0x01,0x00,0x00,0x05]
+
+ds_read_u16_d16 v5, v1 offset:65535
+// GFX12: [0xff,0xff,0x98,0xda,0x01,0x00,0x00,0x05]
+
+ds_read_u16_d16 v5, v1 offset:0
+// GFX12: [0x00,0x00,0x98,0xda,0x01,0x00,0x00,0x05]
+
+ds_read_u16_d16 v255, v255 offset:4
+// GFX12: [0x04,0x00,0x98,0xda,0xff,0x00,0x00,0xff]
+
+ds_read_u16_d16_hi v5, v1
+// GFX12: [0x00,0x00,0x9c,0xda,0x01,0x00,0x00,0x05]
+
+ds_read_u16_d16_hi v5, v1 offset:65535
+// GFX12: [0xff,0xff,0x9c,0xda,0x01,0x00,0x00,0x05]
+
+ds_read_u16_d16_hi v5, v1 offset:0
+// GFX12: [0x00,0x00,0x9c,0xda,0x01,0x00,0x00,0x05]
+
+ds_read_u16_d16_hi v255, v255 offset:4
+// GFX12: [0x04,0x00,0x9c,0xda,0xff,0x00,0x00,0xff]
+
+ds_read_u8 v5, v1
+// GFX12: [0x00,0x00,0xe8,0xd8,0x01,0x00,0x00,0x05]
+
+ds_read_u8 v5, v1 offset:65535
+// GFX12: [0xff,0xff,0xe8,0xd8,0x01,0x00,0x00,0x05]
+
+ds_read_u8 v5, v1 offset:0
+// GFX12: [0x00,0x00,0xe8,0xd8,0x01,0x00,0x00,0x05]
+
+ds_read_u8 v255, v255 offset:4
+// GFX12: [0x04,0x00,0xe8,0xd8,0xff,0x00,0x00,0xff]
+
+ds_read_u8_d16 v5, v1
+// GFX12: [0x00,0x00,0x88,0xda,0x01,0x00,0x00,0x05]
+
+ds_read_u8_d16 v5, v1 offset:65535
+// GFX12: [0xff,0xff,0x88,0xda,0x01,0x00,0x00,0x05]
+
+ds_read_u8_d16 v5, v1 offset:0
+// GFX12: [0x00,0x00,0x88,0xda,0x01,0x00,0x00,0x05]
+
+ds_read_u8_d16 v255, v255 offset:4
+// GFX12: [0x04,0x00,0x88,0xda,0xff,0x00,0x00,0xff]
+
+ds_read_u8_d16_hi v5, v1
+// GFX12: [0x00,0x00,0x8c,0xda,0x01,0x00,0x00,0x05]
+
+ds_read_u8_d16_hi v5, v1 offset:65535
+// GFX12: [0xff,0xff,0x8c,0xda,0x01,0x00,0x00,0x05]
+
+ds_read_u8_d16_hi v5, v1 offset:0
+// GFX12: [0x00,0x00,0x8c,0xda,0x01,0x00,0x00,0x05]
+
+ds_read_u8_d16_hi v255, v255 offset:4
+// GFX12: [0x04,0x00,0x8c,0xda,0xff,0x00,0x00,0xff]
+
+ds_rsub_rtn_u32 v5, v1, v2
+// GFX12: [0x00,0x00,0x88,0xd8,0x01,0x02,0x00,0x05]
+
+ds_rsub_rtn_u32 v5, v1, v2 offset:65535
+// GFX12: [0xff,0xff,0x88,0xd8,0x01,0x02,0x00,0x05]
+
+ds_rsub_rtn_u32 v5, v1, v2 offset:0
+// GFX12: [0x00,0x00,0x88,0xd8,0x01,0x02,0x00,0x05]
+
+ds_rsub_rtn_u32 v255, v255, v255 offset:4
+// GFX12: [0x04,0x00,0x88,0xd8,0xff,0xff,0x00,0xff]
+
+ds_rsub_rtn_u64 v[5:6], v1, v[2:3]
+// GFX12: [0x00,0x00,0x88,0xd9,0x01,0x02,0x00,0x05]
+
+ds_rsub_rtn_u64 v[5:6], v1, v[2:3] offset:65535
+// GFX12: [0xff,0xff,0x88,0xd9,0x01,0x02,0x00,0x05]
+
+ds_rsub_rtn_u64 v[5:6], v1, v[2:3] offset:0
+// GFX12: [0x00,0x00,0x88,0xd9,0x01,0x02,0x00,0x05]
+
+ds_rsub_rtn_u64 v[254:255], v255, v[254:255] offset:4
+// GFX12: [0x04,0x00,0x88,0xd9,0xff,0xfe,0x00,0xfe]
+
+ds_rsub_u32 v1, v2
+// GFX12: [0x00,0x00,0x08,0xd8,0x01,0x02,0x00,0x00]
+
+ds_rsub_u32 v1, v2 offset:65535
+// GFX12: [0xff,0xff,0x08,0xd8,0x01,0x02,0x00,0x00]
+
+ds_rsub_u32 v1, v2 offset:0
+// GFX12: [0x00,0x00,0x08,0xd8,0x01,0x02,0x00,0x00]
+
+ds_rsub_u32 v255, v255 offset:4
+// GFX12: [0x04,0x00,0x08,0xd8,0xff,0xff,0x00,0x00]
+
+ds_rsub_u64 v1, v[2:3]
+// GFX12: [0x00,0x00,0x08,0xd9,0x01,0x02,0x00,0x00]
+
+ds_rsub_u64 v1, v[2:3] offset:65535
+// GFX12: [0xff,0xff,0x08,0xd9,0x01,0x02,0x00,0x00]
+
+ds_rsub_u64 v1, v[2:3] offset:0
+// GFX12: [0x00,0x00,0x08,0xd9,0x01,0x02,0x00,0x00]
+
+ds_rsub_u64 v255, v[254:255] offset:4
+// GFX12: [0x04,0x00,0x08,0xd9,0xff,0xfe,0x00,0x00]
+
+ds_store_2addr_b32 v1, v2, v3
+// GFX12: [0x00,0x00,0x38,0xd8,0x01,0x02,0x03,0x00]
+
+ds_store_2addr_b32 v1, v2, v3 offset0:127 offset1:255
+// GFX12: [0x7f,0xff,0x38,0xd8,0x01,0x02,0x03,0x00]
+
+ds_store_2addr_b32 v1, v2, v3 offset0:0 offset1:0
+// GFX12: [0x00,0x00,0x38,0xd8,0x01,0x02,0x03,0x00]
+
+ds_store_2addr_b32 v255, v255, v255 offset0:16 offset1:1
+// GFX12: [0x10,0x01,0x38,0xd8,0xff,0xff,0xff,0x00]
+
+ds_store_2addr_b64 v1, v[2:3], v[3:4]
+// GFX12: [0x00,0x00,0x38,0xd9,0x01,0x02,0x03,0x00]
+
+ds_store_2addr_b64 v1, v[2:3], v[3:4] offset0:127 offset1:255
+// GFX12: [0x7f,0xff,0x38,0xd9,0x01,0x02,0x03,0x00]
+
+ds_store_2addr_b64 v1, v[2:3], v[3:4] offset0:0 offset1:0
+// GFX12: [0x00,0x00,0x38,0xd9,0x01,0x02,0x03,0x00]
+
+ds_store_2addr_b64 v255, v[254:255], v[254:255] offset0:16 offset1:1
+// GFX12: [0x10,0x01,0x38,0xd9,0xff,0xfe,0xfe,0x00]
+
+ds_store_2addr_stride64_b32 v1, v2, v3
+// GFX12: [0x00,0x00,0x3c,0xd8,0x01,0x02,0x03,0x00]
+
+ds_store_2addr_stride64_b32 v1, v2, v3 offset0:127 offset1:255
+// GFX12: [0x7f,0xff,0x3c,0xd8,0x01,0x02,0x03,0x00]
+
+ds_store_2addr_stride64_b32 v1, v2, v3 offset0:0 offset1:0
+// GFX12: [0x00,0x00,0x3c,0xd8,0x01,0x02,0x03,0x00]
+
+ds_store_2addr_stride64_b32 v255, v255, v255 offset0:16 offset1:1
+// GFX12: [0x10,0x01,0x3c,0xd8,0xff,0xff,0xff,0x00]
+
+ds_store_2addr_stride64_b64 v1, v[2:3], v[3:4]
+// GFX12: [0x00,0x00,0x3c,0xd9,0x01,0x02,0x03,0x00]
+
+ds_store_2addr_stride64_b64 v1, v[2:3], v[3:4] offset0:127 offset1:255
+// GFX12: [0x7f,0xff,0x3c,0xd9,0x01,0x02,0x03,0x00]
+
+ds_store_2addr_stride64_b64 v1, v[2:3], v[3:4] offset0:0 offset1:0
+// GFX12: [0x00,0x00,0x3c,0xd9,0x01,0x02,0x03,0x00]
+
+ds_store_2addr_stride64_b64 v255, v[254:255], v[254:255] offset0:16 offset1:1
+// GFX12: [0x10,0x01,0x3c,0xd9,0xff,0xfe,0xfe,0x00]
+
+ds_store_addtid_b32 v1
+// GFX12: [0x00,0x00,0xc0,0xda,0x00,0x01,0x00,0x00]
+
+ds_store_addtid_b32 v1 offset:65535
+// GFX12: [0xff,0xff,0xc0,0xda,0x00,0x01,0x00,0x00]
+
+ds_store_addtid_b32 v1 offset:0
+// GFX12: [0x00,0x00,0xc0,0xda,0x00,0x01,0x00,0x00]
+
+ds_store_addtid_b32 v255 offset:4
+// GFX12: [0x04,0x00,0xc0,0xda,0x00,0xff,0x00,0x00]
+
+ds_store_b128 v1, v[2:5]
+// GFX12: [0x00,0x00,0x7c,0xdb,0x01,0x02,0x00,0x00]
+
+ds_store_b128 v1, v[2:5] offset:65535
+// GFX12: [0xff,0xff,0x7c,0xdb,0x01,0x02,0x00,0x00]
+
+ds_store_b128 v1, v[2:5] offset:0
+// GFX12: [0x00,0x00,0x7c,0xdb,0x01,0x02,0x00,0x00]
+
+ds_store_b128 v255, v[252:255] offset:4
+// GFX12: [0x04,0x00,0x7c,0xdb,0xff,0xfc,0x00,0x00]
+
+ds_store_b16 v1, v2
+// GFX12: [0x00,0x00,0x7c,0xd8,0x01,0x02,0x00,0x00]
+
+ds_store_b16 v1, v2 offset:65535
+// GFX12: [0xff,0xff,0x7c,0xd8,0x01,0x02,0x00,0x00]
+
+ds_store_b16 v1, v2 offset:0
+// GFX12: [0x00,0x00,0x7c,0xd8,0x01,0x02,0x00,0x00]
+
+ds_store_b16 v255, v255 offset:4
+// GFX12: [0x04,0x00,0x7c,0xd8,0xff,0xff,0x00,0x00]
+
+ds_store_b16_d16_hi v1, v2
+// GFX12: [0x00,0x00,0x84,0xda,0x01,0x02,0x00,0x00]
+
+ds_store_b16_d16_hi v1, v2 offset:65535
+// GFX12: [0xff,0xff,0x84,0xda,0x01,0x02,0x00,0x00]
+
+ds_store_b16_d16_hi v1, v2 offset:0
+// GFX12: [0x00,0x00,0x84,0xda,0x01,0x02,0x00,0x00]
+
+ds_store_b16_d16_hi v255, v255 offset:4
+// GFX12: [0x04,0x00,0x84,0xda,0xff,0xff,0x00,0x00]
+
+ds_store_b32 v1, v2
+// GFX12: [0x00,0x00,0x34,0xd8,0x01,0x02,0x00,0x00]
+
+ds_store_b32 v1, v2 offset:65535
+// GFX12: [0xff,0xff,0x34,0xd8,0x01,0x02,0x00,0x00]
+
+ds_store_b32 v1, v2 offset:0
+// GFX12: [0x00,0x00,0x34,0xd8,0x01,0x02,0x00,0x00]
+
+ds_store_b32 v255, v255 offset:4
+// GFX12: [0x04,0x00,0x34,0xd8,0xff,0xff,0x00,0x00]
+
+ds_store_b64 v1, v[2:3]
+// GFX12: [0x00,0x00,0x34,0xd9,0x01,0x02,0x00,0x00]
+
+ds_store_b64 v1, v[2:3] offset:65535
+// GFX12: [0xff,0xff,0x34,0xd9,0x01,0x02,0x00,0x00]
+
+ds_store_b64 v1, v[2:3] offset:0
+// GFX12: [0x00,0x00,0x34,0xd9,0x01,0x02,0x00,0x00]
+
+ds_store_b64 v255, v[254:255] offset:4
+// GFX12: [0x04,0x00,0x34,0xd9,0xff,0xfe,0x00,0x00]
+
+ds_store_b8 v1, v2
+// GFX12: [0x00,0x00,0x78,0xd8,0x01,0x02,0x00,0x00]
+
+ds_store_b8 v1, v2 offset:65535
+// GFX12: [0xff,0xff,0x78,0xd8,0x01,0x02,0x00,0x00]
+
+ds_store_b8 v1, v2 offset:0
+// GFX12: [0x00,0x00,0x78,0xd8,0x01,0x02,0x00,0x00]
+
+ds_store_b8 v255, v255 offset:4
+// GFX12: [0x04,0x00,0x78,0xd8,0xff,0xff,0x00,0x00]
+
+ds_store_b8_d16_hi v1, v2
+// GFX12: [0x00,0x00,0x80,0xda,0x01,0x02,0x00,0x00]
+
+ds_store_b8_d16_hi v1, v2 offset:65535
+// GFX12: [0xff,0xff,0x80,0xda,0x01,0x02,0x00,0x00]
+
+ds_store_b8_d16_hi v1, v2 offset:0
+// GFX12: [0x00,0x00,0x80,0xda,0x01,0x02,0x00,0x00]
+
+ds_store_b8_d16_hi v255, v255 offset:4
+// GFX12: [0x04,0x00,0x80,0xda,0xff,0xff,0x00,0x00]
+
+ds_store_b96 v1, v[2:4]
+// GFX12: [0x00,0x00,0x78,0xdb,0x01,0x02,0x00,0x00]
+
+ds_store_b96 v1, v[2:4] offset:65535
+// GFX12: [0xff,0xff,0x78,0xdb,0x01,0x02,0x00,0x00]
+
+ds_store_b96 v1, v[2:4] offset:0
+// GFX12: [0x00,0x00,0x78,0xdb,0x01,0x02,0x00,0x00]
+
+ds_store_b96 v255, v[253:255] offset:4
+// GFX12: [0x04,0x00,0x78,0xdb,0xff,0xfd,0x00,0x00]
+
+ds_storexchg_2addr_rtn_b32 v[5:6], v1, v2, v3
+// GFX12: [0x00,0x00,0xb8,0xd8,0x01,0x02,0x03,0x05]
+
+ds_storexchg_2addr_rtn_b32 v[5:6], v1, v2, v3 offset0:127 offset1:255
+// GFX12: [0x7f,0xff,0xb8,0xd8,0x01,0x02,0x03,0x05]
+
+ds_storexchg_2addr_rtn_b32 v[5:6], v1, v2, v3 offset0:0 offset1:0
+// GFX12: [0x00,0x00,0xb8,0xd8,0x01,0x02,0x03,0x05]
+
+ds_storexchg_2addr_rtn_b32 v[254:255], v255, v255, v255 offset0:16 offset1:1
+// GFX12: [0x10,0x01,0xb8,0xd8,0xff,0xff,0xff,0xfe]
+
+ds_storexchg_2addr_rtn_b64 v[5:8], v1, v[2:3], v[3:4]
+// GFX12: [0x00,0x00,0xb8,0xd9,0x01,0x02,0x03,0x05]
+
+ds_storexchg_2addr_rtn_b64 v[5:8], v1, v[2:3], v[3:4] offset0:127 offset1:255
+// GFX12: [0x7f,0xff,0xb8,0xd9,0x01,0x02,0x03,0x05]
+
+ds_storexchg_2addr_rtn_b64 v[5:8], v1, v[2:3], v[3:4] offset0:0 offset1:0
+// GFX12: [0x00,0x00,0xb8,0xd9,0x01,0x02,0x03,0x05]
+
+ds_storexchg_2addr_rtn_b64 v[252:255], v255, v[254:255], v[254:255] offset0:16 offset1:1
+// GFX12: [0x10,0x01,0xb8,0xd9,0xff,0xfe,0xfe,0xfc]
+
+ds_storexchg_2addr_stride64_rtn_b32 v[5:6], v1, v2, v3
+// GFX12: [0x00,0x00,0xbc,0xd8,0x01,0x02,0x03,0x05]
+
+ds_storexchg_2addr_stride64_rtn_b32 v[5:6], v1, v2, v3 offset0:127 offset1:255
+// GFX12: [0x7f,0xff,0xbc,0xd8,0x01,0x02,0x03,0x05]
+
+ds_storexchg_2addr_stride64_rtn_b32 v[5:6], v1, v2, v3 offset0:0 offset1:0
+// GFX12: [0x00,0x00,0xbc,0xd8,0x01,0x02,0x03,0x05]
+
+ds_storexchg_2addr_stride64_rtn_b32 v[254:255], v255, v255, v255 offset0:16 offset1:1
+// GFX12: [0x10,0x01,0xbc,0xd8,0xff,0xff,0xff,0xfe]
+
+ds_storexchg_2addr_stride64_rtn_b64 v[5:8], v1, v[2:3], v[3:4]
+// GFX12: [0x00,0x00,0xbc,0xd9,0x01,0x02,0x03,0x05]
+
+ds_storexchg_2addr_stride64_rtn_b64 v[5:8], v1, v[2:3], v[3:4] offset0:127 offset1:255
+// GFX12: [0x7f,0xff,0xbc,0xd9,0x01,0x02,0x03,0x05]
+
+ds_storexchg_2addr_stride64_rtn_b64 v[5:8], v1, v[2:3], v[3:4] offset0:0 offset1:0
+// GFX12: [0x00,0x00,0xbc,0xd9,0x01,0x02,0x03,0x05]
+
+ds_storexchg_2addr_stride64_rtn_b64 v[252:255], v255, v[254:255], v[254:255] offset0:16 offset1:1
+// GFX12: [0x10,0x01,0xbc,0xd9,0xff,0xfe,0xfe,0xfc]
+
+ds_storexchg_rtn_b32 v5, v1, v2
+// GFX12: [0x00,0x00,0xb4,0xd8,0x01,0x02,0x00,0x05]
+
+ds_storexchg_rtn_b32 v5, v1, v2 offset:65535
+// GFX12: [0xff,0xff,0xb4,0xd8,0x01,0x02,0x00,0x05]
+
+ds_storexchg_rtn_b32 v5, v1, v2 offset:0
+// GFX12: [0x00,0x00,0xb4,0xd8,0x01,0x02,0x00,0x05]
+
+ds_storexchg_rtn_b32 v255, v255, v255 offset:4
+// GFX12: [0x04,0x00,0xb4,0xd8,0xff,0xff,0x00,0xff]
+
+ds_storexchg_rtn_b64 v[5:6], v1, v[2:3]
+// GFX12: [0x00,0x00,0xb4,0xd9,0x01,0x02,0x00,0x05]
+
+ds_storexchg_rtn_b64 v[5:6], v1, v[2:3] offset:65535
+// GFX12: [0xff,0xff,0xb4,0xd9,0x01,0x02,0x00,0x05]
+
+ds_storexchg_rtn_b64 v[5:6], v1, v[2:3] offset:0
+// GFX12: [0x00,0x00,0xb4,0xd9,0x01,0x02,0x00,0x05]
+
+ds_storexchg_rtn_b64 v[254:255], v255, v[254:255] offset:4
+// GFX12: [0x04,0x00,0xb4,0xd9,0xff,0xfe,0x00,0xfe]
+
+ds_sub_clamp_rtn_u32 v5, v1, v2
+// GFX12: [0x00,0x00,0xa4,0xda,0x01,0x02,0x00,0x05]
+
+ds_sub_clamp_rtn_u32 v5, v1, v2 offset:65535
+// GFX12: [0xff,0xff,0xa4,0xda,0x01,0x02,0x00,0x05]
+
+ds_sub_clamp_rtn_u32 v5, v1, v2 offset:0
+// GFX12: [0x00,0x00,0xa4,0xda,0x01,0x02,0x00,0x05]
+
+ds_sub_clamp_rtn_u32 v255, v255, v255 offset:4
+// GFX12: [0x04,0x00,0xa4,0xda,0xff,0xff,0x00,0xff]
+
+ds_sub_clamp_u32 v1, v2
+// GFX12: [0x00,0x00,0x64,0xda,0x01,0x02,0x00,0x00]
+
+ds_sub_clamp_u32 v1, v2 offset:65535
+// GFX12: [0xff,0xff,0x64,0xda,0x01,0x02,0x00,0x00]
+
+ds_sub_clamp_u32 v1, v2 offset:0
+// GFX12: [0x00,0x00,0x64,0xda,0x01,0x02,0x00,0x00]
+
+ds_sub_clamp_u32 v255, v255 offset:4
+// GFX12: [0x04,0x00,0x64,0xda,0xff,0xff,0x00,0x00]
+
+ds_sub_rtn_u32 v5, v1, v2
+// GFX12: [0x00,0x00,0x84,0xd8,0x01,0x02,0x00,0x05]
+
+ds_sub_rtn_u32 v5, v1, v2 offset:65535
+// GFX12: [0xff,0xff,0x84,0xd8,0x01,0x02,0x00,0x05]
+
+ds_sub_rtn_u32 v5, v1, v2 offset:0
+// GFX12: [0x00,0x00,0x84,0xd8,0x01,0x02,0x00,0x05]
+
+ds_sub_rtn_u32 v255, v255, v255 offset:4
+// GFX12: [0x04,0x00,0x84,0xd8,0xff,0xff,0x00,0xff]
+
+ds_sub_rtn_u64 v[5:6], v1, v[2:3]
+// GFX12: [0x00,0x00,0x84,0xd9,0x01,0x02,0x00,0x05]
+
+ds_sub_rtn_u64 v[5:6], v1, v[2:3] offset:65535
+// GFX12: [0xff,0xff,0x84,0xd9,0x01,0x02,0x00,0x05]
+
+ds_sub_rtn_u64 v[5:6], v1, v[2:3] offset:0
+// GFX12: [0x00,0x00,0x84,0xd9,0x01,0x02,0x00,0x05]
+
+ds_sub_rtn_u64 v[254:255], v255, v[254:255] offset:4
+// GFX12: [0x04,0x00,0x84,0xd9,0xff,0xfe,0x00,0xfe]
+
+ds_sub_u32 v1, v2
+// GFX12: [0x00,0x00,0x04,0xd8,0x01,0x02,0x00,0x00]
+
+ds_sub_u32 v1, v2 offset:65535
+// GFX12: [0xff,0xff,0x04,0xd8,0x01,0x02,0x00,0x00]
+
+ds_sub_u32 v1, v2 offset:0
+// GFX12: [0x00,0x00,0x04,0xd8,0x01,0x02,0x00,0x00]
+
+ds_sub_u32 v255, v255 offset:4
+// GFX12: [0x04,0x00,0x04,0xd8,0xff,0xff,0x00,0x00]
+
+ds_sub_u64 v1, v[2:3]
+// GFX12: [0x00,0x00,0x04,0xd9,0x01,0x02,0x00,0x00]
+
+ds_sub_u64 v1, v[2:3] offset:65535
+// GFX12: [0xff,0xff,0x04,0xd9,0x01,0x02,0x00,0x00]
+
+ds_sub_u64 v1, v[2:3] offset:0
+// GFX12: [0x00,0x00,0x04,0xd9,0x01,0x02,0x00,0x00]
+
+ds_sub_u64 v255, v[254:255] offset:4
+// GFX12: [0x04,0x00,0x04,0xd9,0xff,0xfe,0x00,0x00]
+
+ds_write2_b32 v1, v2, v3
+// GFX12: [0x00,0x00,0x38,0xd8,0x01,0x02,0x03,0x00]
+
+ds_write2_b32 v1, v2, v3 offset0:127 offset1:255
+// GFX12: [0x7f,0xff,0x38,0xd8,0x01,0x02,0x03,0x00]
+
+ds_write2_b32 v1, v2, v3 offset0:0 offset1:0
+// GFX12: [0x00,0x00,0x38,0xd8,0x01,0x02,0x03,0x00]
+
+ds_write2_b32 v255, v255, v255 offset0:16 offset1:1
+// GFX12: [0x10,0x01,0x38,0xd8,0xff,0xff,0xff,0x00]
+
+ds_write2_b64 v1, v[2:3], v[3:4]
+// GFX12: [0x00,0x00,0x38,0xd9,0x01,0x02,0x03,0x00]
+
+ds_write2_b64 v1, v[2:3], v[3:4] offset0:127 offset1:255
+// GFX12: [0x7f,0xff,0x38,0xd9,0x01,0x02,0x03,0x00]
+
+ds_write2_b64 v1, v[2:3], v[3:4] offset0:0 offset1:0
+// GFX12: [0x00,0x00,0x38,0xd9,0x01,0x02,0x03,0x00]
+
+ds_write2_b64 v255, v[254:255], v[254:255] offset0:16 offset1:1
+// GFX12: [0x10,0x01,0x38,0xd9,0xff,0xfe,0xfe,0x00]
+
+ds_write2st64_b32 v1, v2, v3
+// GFX12: [0x00,0x00,0x3c,0xd8,0x01,0x02,0x03,0x00]
+
+ds_write2st64_b32 v1, v2, v3 offset0:127 offset1:255
+// GFX12: [0x7f,0xff,0x3c,0xd8,0x01,0x02,0x03,0x00]
+
+ds_write2st64_b32 v1, v2, v3 offset0:0 offset1:0
+// GFX12: [0x00,0x00,0x3c,0xd8,0x01,0x02,0x03,0x00]
+
+ds_write2st64_b32 v255, v255, v255 offset0:16 offset1:1
+// GFX12: [0x10,0x01,0x3c,0xd8,0xff,0xff,0xff,0x00]
+
+ds_write2st64_b64 v1, v[2:3], v[3:4]
+// GFX12: [0x00,0x00,0x3c,0xd9,0x01,0x02,0x03,0x00]
+
+ds_write2st64_b64 v1, v[2:3], v[3:4] offset0:127 offset1:255
+// GFX12: [0x7f,0xff,0x3c,0xd9,0x01,0x02,0x03,0x00]
+
+ds_write2st64_b64 v1, v[2:3], v[3:4] offset0:0 offset1:0
+// GFX12: [0x00,0x00,0x3c,0xd9,0x01,0x02,0x03,0x00]
+
+ds_write2st64_b64 v255, v[254:255], v[254:255] offset0:16 offset1:1
+// GFX12: [0x10,0x01,0x3c,0xd9,0xff,0xfe,0xfe,0x00]
+
+ds_write_addtid_b32 v1
+// GFX12: [0x00,0x00,0xc0,0xda,0x00,0x01,0x00,0x00]
+
+ds_write_addtid_b32 v1 offset:65535
+// GFX12: [0xff,0xff,0xc0,0xda,0x00,0x01,0x00,0x00]
+
+ds_write_addtid_b32 v1 offset:0
+// GFX12: [0x00,0x00,0xc0,0xda,0x00,0x01,0x00,0x00]
+
+ds_write_addtid_b32 v255 offset:4
+// GFX12: [0x04,0x00,0xc0,0xda,0x00,0xff,0x00,0x00]
+
+ds_write_b128 v1, v[2:5]
+// GFX12: [0x00,0x00,0x7c,0xdb,0x01,0x02,0x00,0x00]
+
+ds_write_b128 v1, v[2:5] offset:65535
+// GFX12: [0xff,0xff,0x7c,0xdb,0x01,0x02,0x00,0x00]
+
+ds_write_b128 v1, v[2:5] offset:0
+// GFX12: [0x00,0x00,0x7c,0xdb,0x01,0x02,0x00,0x00]
+
+ds_write_b128 v255, v[252:255] offset:4
+// GFX12: [0x04,0x00,0x7c,0xdb,0xff,0xfc,0x00,0x00]
+
+ds_write_b16 v1, v2
+// GFX12: [0x00,0x00,0x7c,0xd8,0x01,0x02,0x00,0x00]
+
+ds_write_b16 v1, v2 offset:65535
+// GFX12: [0xff,0xff,0x7c,0xd8,0x01,0x02,0x00,0x00]
+
+ds_write_b16 v1, v2 offset:0
+// GFX12: [0x00,0x00,0x7c,0xd8,0x01,0x02,0x00,0x00]
+
+ds_write_b16 v255, v255 offset:4
+// GFX12: [0x04,0x00,0x7c,0xd8,0xff,0xff,0x00,0x00]
+
+ds_write_b16_d16_hi v1, v2
+// GFX12: [0x00,0x00,0x84,0xda,0x01,0x02,0x00,0x00]
+
+ds_write_b16_d16_hi v1, v2 offset:65535
+// GFX12: [0xff,0xff,0x84,0xda,0x01,0x02,0x00,0x00]
+
+ds_write_b16_d16_hi v1, v2 offset:0
+// GFX12: [0x00,0x00,0x84,0xda,0x01,0x02,0x00,0x00]
+
+ds_write_b16_d16_hi v255, v255 offset:4
+// GFX12: [0x04,0x00,0x84,0xda,0xff,0xff,0x00,0x00]
+
+ds_write_b32 v1, v2
+// GFX12: [0x00,0x00,0x34,0xd8,0x01,0x02,0x00,0x00]
+
+ds_write_b32 v1, v2 offset:65535
+// GFX12: [0xff,0xff,0x34,0xd8,0x01,0x02,0x00,0x00]
+
+ds_write_b32 v1, v2 offset:0
+// GFX12: [0x00,0x00,0x34,0xd8,0x01,0x02,0x00,0x00]
+
+ds_write_b32 v255, v255 offset:4
+// GFX12: [0x04,0x00,0x34,0xd8,0xff,0xff,0x00,0x00]
+
+ds_write_b64 v1, v[2:3]
+// GFX12: [0x00,0x00,0x34,0xd9,0x01,0x02,0x00,0x00]
+
+ds_write_b64 v1, v[2:3] offset:65535
+// GFX12: [0xff,0xff,0x34,0xd9,0x01,0x02,0x00,0x00]
+
+ds_write_b64 v1, v[2:3] offset:0
+// GFX12: [0x00,0x00,0x34,0xd9,0x01,0x02,0x00,0x00]
+
+ds_write_b64 v255, v[254:255] offset:4
+// GFX12: [0x04,0x00,0x34,0xd9,0xff,0xfe,0x00,0x00]
+
+ds_write_b8 v1, v2
+// GFX12: [0x00,0x00,0x78,0xd8,0x01,0x02,0x00,0x00]
+
+ds_write_b8 v1, v2 offset:65535
+// GFX12: [0xff,0xff,0x78,0xd8,0x01,0x02,0x00,0x00]
+
+ds_write_b8 v1, v2 offset:0
+// GFX12: [0x00,0x00,0x78,0xd8,0x01,0x02,0x00,0x00]
+
+ds_write_b8 v255, v255 offset:4
+// GFX12: [0x04,0x00,0x78,0xd8,0xff,0xff,0x00,0x00]
+
+ds_write_b8_d16_hi v1, v2
+// GFX12: [0x00,0x00,0x80,0xda,0x01,0x02,0x00,0x00]
+
+ds_write_b8_d16_hi v1, v2 offset:65535
+// GFX12: [0xff,0xff,0x80,0xda,0x01,0x02,0x00,0x00]
+
+ds_write_b8_d16_hi v1, v2 offset:0
+// GFX12: [0x00,0x00,0x80,0xda,0x01,0x02,0x00,0x00]
+
+ds_write_b8_d16_hi v255, v255 offset:4
+// GFX12: [0x04,0x00,0x80,0xda,0xff,0xff,0x00,0x00]
+
+ds_write_b96 v1, v[2:4]
+// GFX12: [0x00,0x00,0x78,0xdb,0x01,0x02,0x00,0x00]
+
+ds_write_b96 v1, v[2:4] offset:65535
+// GFX12: [0xff,0xff,0x78,0xdb,0x01,0x02,0x00,0x00]
+
+ds_write_b96 v1, v[2:4] offset:0
+// GFX12: [0x00,0x00,0x78,0xdb,0x01,0x02,0x00,0x00]
+
+ds_write_b96 v255, v[253:255] offset:4
+// GFX12: [0x04,0x00,0x78,0xdb,0xff,0xfd,0x00,0x00]
+
+ds_wrxchg2_rtn_b32 v[5:6], v1, v2, v3
+// GFX12: [0x00,0x00,0xb8,0xd8,0x01,0x02,0x03,0x05]
+
+ds_wrxchg2_rtn_b32 v[5:6], v1, v2, v3 offset0:127 offset1:255
+// GFX12: [0x7f,0xff,0xb8,0xd8,0x01,0x02,0x03,0x05]
+
+ds_wrxchg2_rtn_b32 v[5:6], v1, v2, v3 offset0:0 offset1:0
+// GFX12: [0x00,0x00,0xb8,0xd8,0x01,0x02,0x03,0x05]
+
+ds_wrxchg2_rtn_b32 v[254:255], v255, v255, v255 offset0:16 offset1:1
+// GFX12: [0x10,0x01,0xb8,0xd8,0xff,0xff,0xff,0xfe]
+
+ds_wrxchg2_rtn_b64 v[5:8], v1, v[2:3], v[3:4]
+// GFX12: [0x00,0x00,0xb8,0xd9,0x01,0x02,0x03,0x05]
+
+ds_wrxchg2_rtn_b64 v[5:8], v1, v[2:3], v[3:4] offset0:127 offset1:255
+// GFX12: [0x7f,0xff,0xb8,0xd9,0x01,0x02,0x03,0x05]
+
+ds_wrxchg2_rtn_b64 v[5:8], v1, v[2:3], v[3:4] offset0:0 offset1:0
+// GFX12: [0x00,0x00,0xb8,0xd9,0x01,0x02,0x03,0x05]
+
+ds_wrxchg2_rtn_b64 v[252:255], v255, v[254:255], v[254:255] offset0:16 offset1:1
+// GFX12: [0x10,0x01,0xb8,0xd9,0xff,0xfe,0xfe,0xfc]
+
+ds_wrxchg2st64_rtn_b32 v[5:6], v1, v2, v3
+// GFX12: [0x00,0x00,0xbc,0xd8,0x01,0x02,0x03,0x05]
+
+ds_wrxchg2st64_rtn_b32 v[5:6], v1, v2, v3 offset0:127 offset1:255
+// GFX12: [0x7f,0xff,0xbc,0xd8,0x01,0x02,0x03,0x05]
+
+ds_wrxchg2st64_rtn_b32 v[5:6], v1, v2, v3 offset0:0 offset1:0
+// GFX12: [0x00,0x00,0xbc,0xd8,0x01,0x02,0x03,0x05]
+
+ds_wrxchg2st64_rtn_b32 v[254:255], v255, v255, v255 offset0:16 offset1:1
+// GFX12: [0x10,0x01,0xbc,0xd8,0xff,0xff,0xff,0xfe]
+
+ds_wrxchg2st64_rtn_b64 v[5:8], v1, v[2:3], v[3:4]
+// GFX12: [0x00,0x00,0xbc,0xd9,0x01,0x02,0x03,0x05]
+
+ds_wrxchg2st64_rtn_b64 v[5:8], v1, v[2:3], v[3:4] offset0:127 offset1:255
+// GFX12: [0x7f,0xff,0xbc,0xd9,0x01,0x02,0x03,0x05]
+
+ds_wrxchg2st64_rtn_b64 v[5:8], v1, v[2:3], v[3:4] offset0:0 offset1:0
+// GFX12: [0x00,0x00,0xbc,0xd9,0x01,0x02,0x03,0x05]
+
+ds_wrxchg2st64_rtn_b64 v[252:255], v255, v[254:255], v[254:255] offset0:16 offset1:1
+// GFX12: [0x10,0x01,0xbc,0xd9,0xff,0xfe,0xfe,0xfc]
+
+ds_wrxchg_rtn_b32 v5, v1, v2
+// GFX12: [0x00,0x00,0xb4,0xd8,0x01,0x02,0x00,0x05]
+
+ds_wrxchg_rtn_b32 v5, v1, v2 offset:65535
+// GFX12: [0xff,0xff,0xb4,0xd8,0x01,0x02,0x00,0x05]
+
+ds_wrxchg_rtn_b32 v5, v1, v2 offset:0
+// GFX12: [0x00,0x00,0xb4,0xd8,0x01,0x02,0x00,0x05]
+
+ds_wrxchg_rtn_b32 v255, v255, v255 offset:4
+// GFX12: [0x04,0x00,0xb4,0xd8,0xff,0xff,0x00,0xff]
+
+ds_wrxchg_rtn_b64 v[5:6], v1, v[2:3]
+// GFX12: [0x00,0x00,0xb4,0xd9,0x01,0x02,0x00,0x05]
+
+ds_wrxchg_rtn_b64 v[5:6], v1, v[2:3] offset:65535
+// GFX12: [0xff,0xff,0xb4,0xd9,0x01,0x02,0x00,0x05]
+
+ds_wrxchg_rtn_b64 v[5:6], v1, v[2:3] offset:0
+// GFX12: [0x00,0x00,0xb4,0xd9,0x01,0x02,0x00,0x05]
+
+ds_wrxchg_rtn_b64 v[254:255], v255, v[254:255] offset:4
+// GFX12: [0x04,0x00,0xb4,0xd9,0xff,0xfe,0x00,0xfe]
+
+ds_xor_b32 v1, v2
+// GFX12: [0x00,0x00,0x2c,0xd8,0x01,0x02,0x00,0x00]
+
+ds_xor_b32 v1, v2 offset:65535
+// GFX12: [0xff,0xff,0x2c,0xd8,0x01,0x02,0x00,0x00]
+
+ds_xor_b32 v1, v2 offset:0
+// GFX12: [0x00,0x00,0x2c,0xd8,0x01,0x02,0x00,0x00]
+
+ds_xor_b32 v255, v255 offset:4
+// GFX12: [0x04,0x00,0x2c,0xd8,0xff,0xff,0x00,0x00]
+
+ds_xor_b64 v1, v[2:3]
+// GFX12: [0x00,0x00,0x2c,0xd9,0x01,0x02,0x00,0x00]
+
+ds_xor_b64 v1, v[2:3] offset:65535
+// GFX12: [0xff,0xff,0x2c,0xd9,0x01,0x02,0x00,0x00]
+
+ds_xor_b64 v1, v[2:3] offset:0
+// GFX12: [0x00,0x00,0x2c,0xd9,0x01,0x02,0x00,0x00]
+
+ds_xor_b64 v255, v[254:255] offset:4
+// GFX12: [0x04,0x00,0x2c,0xd9,0xff,0xfe,0x00,0x00]
+
+ds_xor_rtn_b32 v5, v1, v2
+// GFX12: [0x00,0x00,0xac,0xd8,0x01,0x02,0x00,0x05]
+
+ds_xor_rtn_b32 v5, v1, v2 offset:65535
+// GFX12: [0xff,0xff,0xac,0xd8,0x01,0x02,0x00,0x05]
+
+ds_xor_rtn_b32 v5, v1, v2 offset:0
+// GFX12: [0x00,0x00,0xac,0xd8,0x01,0x02,0x00,0x05]
+
+ds_xor_rtn_b32 v255, v255, v255 offset:4
+// GFX12: [0x04,0x00,0xac,0xd8,0xff,0xff,0x00,0xff]
+
+ds_xor_rtn_b64 v[5:6], v1, v[2:3]
+// GFX12: [0x00,0x00,0xac,0xd9,0x01,0x02,0x00,0x05]
+
+ds_xor_rtn_b64 v[5:6], v1, v[2:3] offset:65535
+// GFX12: [0xff,0xff,0xac,0xd9,0x01,0x02,0x00,0x05]
+
+ds_xor_rtn_b64 v[5:6], v1, v[2:3] offset:0
+// GFX12: [0x00,0x00,0xac,0xd9,0x01,0x02,0x00,0x05]
+
+ds_xor_rtn_b64 v[254:255], v255, v[254:255] offset:4
+// GFX12: [0x04,0x00,0xac,0xd9,0xff,0xfe,0x00,0xfe]
+
+ds_swizzle_b32 v8, v2
+// GFX12: [0x00,0x00,0xd4,0xd8,0x02,0x00,0x00,0x08]
+
+ds_swizzle_b32 v8, v2 offset:0
+// GFX12: [0x00,0x00,0xd4,0xd8,0x02,0x00,0x00,0x08]
+
+ds_swizzle_b32 v8, v2 offset:0xFFFF
+// GFX12: [0xff,0xff,0xd4,0xd8,0x02,0x00,0x00,0x08]
+
+ds_swizzle_b32 v8, v2 offset:swizzle(QUAD_PERM, 0, 1, 2, 3)
+// GFX12: [0xe4,0x80,0xd4,0xd8,0x02,0x00,0x00,0x08]
+
+ds_swizzle_b32 v8, v2 offset:swizzle(SWAP,16)
+// GFX12: [0x1f,0x40,0xd4,0xd8,0x02,0x00,0x00,0x08]
+
+ds_swizzle_b32 v8, v2 offset:swizzle(REVERSE,8)
+// GFX12: [0x1f,0x1c,0xd4,0xd8,0x02,0x00,0x00,0x08]
+
+ds_swizzle_b32 v8, v2 offset:swizzle(BROADCAST,4,1)
+// GFX12: [0x3c,0x00,0xd4,0xd8,0x02,0x00,0x00,0x08]
+
+ds_swizzle_b32 v8, v2 offset:swizzle(BROADCAST,8,7)
+// GFX12: [0xf8,0x00,0xd4,0xd8,0x02,0x00,0x00,0x08]
+
+ds_swizzle_b32 v8, v2 offset:swizzle(BITMASK_PERM, "01pip")
+// GFX12: [0x07,0x09,0xd4,0xd8,0x02,0x00,0x00,0x08]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_ds_alias.s b/llvm/test/MC/AMDGPU/gfx12_asm_ds_alias.s
new file mode 100644
index 0000000000000000000000000000000000000000..aa4b028160175ca86451fd160136b6cc51648b90
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_ds_alias.s
@@ -0,0 +1,25 @@
+// RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -show-encoding %s | FileCheck --check-prefix=GFX12 %s
+
+ds_max_f32 v1, v2
+// GFX12: [0x00,0x00,0x4c,0xd8,0x01,0x02,0x00,0x00]
+
+ds_max_f64 v1, v[2:3]
+// GFX12: [0x00,0x00,0x4c,0xd9,0x01,0x02,0x00,0x00]
+
+ds_max_rtn_f32 v5, v1, v2
+// GFX12: [0x00,0x00,0xcc,0xd8,0x01,0x02,0x00,0x05]
+
+ds_max_rtn_f64 v[5:6], v1, v[2:3]
+// GFX12: [0x00,0x00,0xcc,0xd9,0x01,0x02,0x00,0x05]
+
+ds_min_f32 v1, v2
+// GFX12: [0x00,0x00,0x48,0xd8,0x01,0x02,0x00,0x00]
+
+ds_min_f64 v1, v[2:3]
+// GFX12: [0x00,0x00,0x48,0xd9,0x01,0x02,0x00,0x00]
+
+ds_min_rtn_f32 v5, v1, v2
+// GFX12: [0x00,0x00,0xc8,0xd8,0x01,0x02,0x00,0x05]
+
+ds_min_rtn_f64 v[5:6], v1, v[2:3]
+// GFX12: [0x00,0x00,0xc8,0xd9,0x01,0x02,0x00,0x05]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_sop1.s b/llvm/test/MC/AMDGPU/gfx12_asm_sop1.s
index db166e8ffc104f7dc059bffcfcf748c9f567b78a..8f2944586ed290f2758c8b668701f037393f2862 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_sop1.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_sop1.s
@@ -684,6 +684,51 @@ s_rndne_f16 s5, 0xfe0b
 s_rndne_f16 s5, 0x3456
 // GFX12: encoding: [0xff,0x6e,0x85,0xbe,0x56,0x34,0x00,0x00]
 
+s_barrier_signal -2
+// GFX12: encoding: [0xc2,0x4e,0x80,0xbe]
+
+s_barrier_signal -1
+// GFX12: encoding: [0xc1,0x4e,0x80,0xbe]
+
+s_barrier_signal m0
+// GFX12: encoding: [0x7d,0x4e,0x80,0xbe]
+
+s_barrier_signal_isfirst -2
+// GFX12: encoding: [0xc2,0x4f,0x80,0xbe]
+
+s_barrier_signal_isfirst -1
+// GFX12: encoding: [0xc1,0x4f,0x80,0xbe]
+
+s_barrier_signal_isfirst m0
+// GFX12: encoding: [0x7d,0x4f,0x80,0xbe]
+
+s_barrier_init -1
+// GFX12: encoding: [0xc1,0x51,0x80,0xbe]
+
+s_barrier_init -2
+// GFX12: encoding: [0xc2,0x51,0x80,0xbe]
+
+s_barrier_init m0
+// GFX12: encoding: [0x7d,0x51,0x80,0xbe]
+
+s_barrier_join -1
+// GFX12: encoding: [0xc1,0x52,0x80,0xbe]
+
+s_barrier_join -2
+// GFX12: encoding: [0xc2,0x52,0x80,0xbe]
+
+s_barrier_join m0
+// GFX12: encoding: [0x7d,0x52,0x80,0xbe]
+
+s_wakeup_barrier 1
+// GFX12: encoding: [0x81,0x57,0x80,0xbe]
+
+s_wakeup_barrier -1
+// GFX12: encoding: [0xc1,0x57,0x80,0xbe]
+
+s_wakeup_barrier m0
+// GFX12: encoding: [0x7d,0x57,0x80,0xbe]
+
 s_mov_b32 s0, s1
 // GFX12: encoding: [0x01,0x00,0x80,0xbe]
 
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_sop2.s b/llvm/test/MC/AMDGPU/gfx12_asm_sop2.s
index 1a898bebde57fa4f83cc14fd5eb69b514ad695b7..d1d9208724181b5980c51f8cb4dbb46a3865c680 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_sop2.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_sop2.s
@@ -483,6 +483,108 @@ s_max_num_f32 s5, 0x3f717273, s2
 s_max_num_f32 s5, s1, s105
 // GFX12: encoding: [0x01,0x69,0x85,0xa1]
 
+s_minimum_f32 s5, s1, s2
+// GFX12: encoding: [0x01,0x02,0x85,0xa7]
+
+s_minimum_f32 s105, s1, s2
+// GFX12: encoding: [0x01,0x02,0xe9,0xa7]
+
+s_minimum_f32 s5, s105, s2
+// GFX12: encoding: [0x69,0x02,0x85,0xa7]
+
+s_minimum_f32 s5, s103, s2
+// GFX12: encoding: [0x67,0x02,0x85,0xa7]
+
+s_minimum_f32 s5, vcc_lo, s2
+// GFX12: encoding: [0x6a,0x02,0x85,0xa7]
+
+s_minimum_f32 s5, vcc_hi, s2
+// GFX12: encoding: [0x6b,0x02,0x85,0xa7]
+
+s_minimum_f32 s5, ttmp11, s2
+// GFX12: encoding: [0x77,0x02,0x85,0xa7]
+
+s_minimum_f32 s5, m0, s2
+// GFX12: encoding: [0x7d,0x02,0x85,0xa7]
+
+s_minimum_f32 s5, exec_lo, s2
+// GFX12: encoding: [0x7e,0x02,0x85,0xa7]
+
+s_minimum_f32 s5, exec_hi, s2
+// GFX12: encoding: [0x7f,0x02,0x85,0xa7]
+
+s_minimum_f32 s5, 0, s2
+// GFX12: encoding: [0x80,0x02,0x85,0xa7]
+
+s_minimum_f32 s5, -1, s2
+// GFX12: encoding: [0xc1,0x02,0x85,0xa7]
+
+s_minimum_f32 s5, 0.5, s2
+// GFX12: encoding: [0xf0,0x02,0x85,0xa7]
+
+s_minimum_f32 s5, -4.0, s2
+// GFX12: encoding: [0xf7,0x02,0x85,0xa7]
+
+s_minimum_f32 s5, 0xaf123456, s2
+// GFX12: encoding: [0xff,0x02,0x85,0xa7,0x56,0x34,0x12,0xaf]
+
+s_minimum_f32 s5, 0x3f717273, s2
+// GFX12: encoding: [0xff,0x02,0x85,0xa7,0x73,0x72,0x71,0x3f]
+
+s_minimum_f32 s5, s1, s105
+// GFX12: encoding: [0x01,0x69,0x85,0xa7]
+
+s_maximum_f32 s5, s1, s2
+// GFX12: encoding: [0x01,0x02,0x05,0xa8]
+
+s_maximum_f32 s105, s1, s2
+// GFX12: encoding: [0x01,0x02,0x69,0xa8]
+
+s_maximum_f32 s5, s105, s2
+// GFX12: encoding: [0x69,0x02,0x05,0xa8]
+
+s_maximum_f32 s5, s103, s2
+// GFX12: encoding: [0x67,0x02,0x05,0xa8]
+
+s_maximum_f32 s5, vcc_lo, s2
+// GFX12: encoding: [0x6a,0x02,0x05,0xa8]
+
+s_maximum_f32 s5, vcc_hi, s2
+// GFX12: encoding: [0x6b,0x02,0x05,0xa8]
+
+s_maximum_f32 s5, ttmp11, s2
+// GFX12: encoding: [0x77,0x02,0x05,0xa8]
+
+s_maximum_f32 s5, m0, s2
+// GFX12: encoding: [0x7d,0x02,0x05,0xa8]
+
+s_maximum_f32 s5, exec_lo, s2
+// GFX12: encoding: [0x7e,0x02,0x05,0xa8]
+
+s_maximum_f32 s5, exec_hi, s2
+// GFX12: encoding: [0x7f,0x02,0x05,0xa8]
+
+s_maximum_f32 s5, 0, s2
+// GFX12: encoding: [0x80,0x02,0x05,0xa8]
+
+s_maximum_f32 s5, -1, s2
+// GFX12: encoding: [0xc1,0x02,0x05,0xa8]
+
+s_maximum_f32 s5, 0.5, s2
+// GFX12: encoding: [0xf0,0x02,0x05,0xa8]
+
+s_maximum_f32 s5, -4.0, s2
+// GFX12: encoding: [0xf7,0x02,0x05,0xa8]
+
+s_maximum_f32 s5, 0xaf123456, s2
+// GFX12: encoding: [0xff,0x02,0x05,0xa8,0x56,0x34,0x12,0xaf]
+
+s_maximum_f32 s5, 0x3f717273, s2
+// GFX12: encoding: [0xff,0x02,0x05,0xa8,0x73,0x72,0x71,0x3f]
+
+s_maximum_f32 s5, s1, s105
+// GFX12: encoding: [0x01,0x69,0x05,0xa8]
+
 s_fmac_f32 s5, s1, s2
 // GFX12: encoding: [0x01,0x02,0x85,0xa3]
 
@@ -942,6 +1044,102 @@ s_min_num_f16 s5, 0x3456, s2
 s_min_num_f16 s5, s1, s105
 // GFX12: encoding: [0x01,0x69,0x85,0xa5]
 
+s_maximum_f16 s5, s1, s2
+// GFX12: encoding: [0x01,0x02,0x05,0xa9]
+
+s_maximum_f16 s105, s1, s2
+// GFX12: encoding: [0x01,0x02,0x69,0xa9]
+
+s_maximum_f16 s5, s105, s2
+// GFX12: encoding: [0x69,0x02,0x05,0xa9]
+
+s_maximum_f16 s5, s101, s2
+// GFX12: encoding: [0x65,0x02,0x05,0xa9]
+
+s_maximum_f16 s5, vcc_lo, s2
+// GFX12: encoding: [0x6a,0x02,0x05,0xa9]
+
+s_maximum_f16 s5, vcc_hi, s2
+// GFX12: encoding: [0x6b,0x02,0x05,0xa9]
+
+s_maximum_f16 s5, m0, s2
+// GFX12: encoding: [0x7d,0x02,0x05,0xa9]
+
+s_maximum_f16 s5, exec_lo, s2
+// GFX12: encoding: [0x7e,0x02,0x05,0xa9]
+
+s_maximum_f16 s5, exec_hi, s2
+// GFX12: encoding: [0x7f,0x02,0x05,0xa9]
+
+s_maximum_f16 s5, 0, s2
+// GFX12: encoding: [0x80,0x02,0x05,0xa9]
+
+s_maximum_f16 s5, -1, s2
+// GFX12: encoding: [0xc1,0x02,0x05,0xa9]
+
+s_maximum_f16 s5, 0.5, s2
+// GFX12: encoding: [0xf0,0x02,0x05,0xa9]
+
+s_maximum_f16 s5, -4.0, s2
+// GFX12: encoding: [0xf7,0x02,0x05,0xa9]
+
+s_maximum_f16 s5, 0xfe0b, s2
+// GFX12: encoding: [0xff,0x02,0x05,0xa9,0x0b,0xfe,0x00,0x00]
+
+s_maximum_f16 s5, 0x3456, s2
+// GFX12: encoding: [0xff,0x02,0x05,0xa9,0x56,0x34,0x00,0x00]
+
+s_maximum_f16 s5, s1, s105
+// GFX12: encoding: [0x01,0x69,0x05,0xa9]
+
+s_minimum_f16 s5, s1, s2
+// GFX12: encoding: [0x01,0x02,0x85,0xa8]
+
+s_minimum_f16 s105, s1, s2
+// GFX12: encoding: [0x01,0x02,0xe9,0xa8]
+
+s_minimum_f16 s5, s105, s2
+// GFX12: encoding: [0x69,0x02,0x85,0xa8]
+
+s_minimum_f16 s5, s101, s2
+// GFX12: encoding: [0x65,0x02,0x85,0xa8]
+
+s_minimum_f16 s5, vcc_lo, s2
+// GFX12: encoding: [0x6a,0x02,0x85,0xa8]
+
+s_minimum_f16 s5, vcc_hi, s2
+// GFX12: encoding: [0x6b,0x02,0x85,0xa8]
+
+s_minimum_f16 s5, m0, s2
+// GFX12: encoding: [0x7d,0x02,0x85,0xa8]
+
+s_minimum_f16 s5, exec_lo, s2
+// GFX12: encoding: [0x7e,0x02,0x85,0xa8]
+
+s_minimum_f16 s5, exec_hi, s2
+// GFX12: encoding: [0x7f,0x02,0x85,0xa8]
+
+s_minimum_f16 s5, 0, s2
+// GFX12: encoding: [0x80,0x02,0x85,0xa8]
+
+s_minimum_f16 s5, -1, s2
+// GFX12: encoding: [0xc1,0x02,0x85,0xa8]
+
+s_minimum_f16 s5, 0.5, s2
+// GFX12: encoding: [0xf0,0x02,0x85,0xa8]
+
+s_minimum_f16 s5, -4.0, s2
+// GFX12: encoding: [0xf7,0x02,0x85,0xa8]
+
+s_minimum_f16 s5, 0xfe0b, s2
+// GFX12: encoding: [0xff,0x02,0x85,0xa8,0x0b,0xfe,0x00,0x00]
+
+s_minimum_f16 s5, 0x3456, s2
+// GFX12: encoding: [0xff,0x02,0x85,0xa8,0x56,0x34,0x00,0x00]
+
+s_minimum_f16 s5, s1, s105
+// GFX12: encoding: [0x01,0x69,0x85,0xa8]
+
 s_add_co_u32 s0, s1, s2
 // GFX12: encoding: [0x01,0x02,0x00,0x80]
 
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_sopp.s b/llvm/test/MC/AMDGPU/gfx12_asm_sopp.s
index 2e9df11d6f5a4560a31cd1d1b8856fce1f3f1bcf..cf78b87a476183bc4b50bd517a55fcc1954e67fc 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_sopp.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_sopp.s
@@ -33,6 +33,15 @@ s_singleuse_vdst 0xffff
 s_singleuse_vdst 0x1234
 // GFX12: encoding: [0x34,0x12,0x93,0xbf]
 
+s_barrier_wait 0xffff
+// GFX12: encoding: [0xff,0xff,0x94,0xbf]
+
+s_barrier_wait 1
+// GFX12: encoding: [0x01,0x00,0x94,0xbf]
+
+s_barrier_leave
+// GFX12: encoding: [0x00,0x00,0x95,0xbf]
+
 //===----------------------------------------------------------------------===//
 // s_waitcnt
 //===----------------------------------------------------------------------===//
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vbuffer_mtbuf.s b/llvm/test/MC/AMDGPU/gfx12_asm_vbuffer_mtbuf.s
new file mode 100644
index 0000000000000000000000000000000000000000..d2cd09ad1d40fb89979ba3762770bd13061fb294
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vbuffer_mtbuf.s
@@ -0,0 +1,907 @@
+// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -show-encoding %s | FileCheck --check-prefix=GFX12 %s
+// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -show-encoding %s 2>&1 | FileCheck --check-prefixes=GFX12-ERR --implicit-check-not=error: %s
+
+tbuffer_load_d16_format_x v4, off, s[8:11], s3 format:[BUF_FMT_8_UNORM] offset:8388607
+// GFX12: encoding: [0x03,0x00,0x22,0xc4,0x04,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_d16_format_x v255, off, s[8:11], s3 format:1 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x22,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_d16_format_x v4, off, s[12:15], s3 format:[BUF_DATA_FORMAT_8, BUF_NUM_FORMAT_UNORM] offset:8388607
+// GFX12: encoding: [0x03,0x00,0x22,0xc4,0x04,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_d16_format_x v4, off, s[12:15], s101 format:[BUF_FMT_8_SNORM] offset:8388607
+// GFX12: encoding: [0x65,0x00,0x22,0xc4,0x04,0x18,0x00,0x01,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_d16_format_x v4, off, s[12:15], m0 format:2 offset:8388607
+// GFX12: encoding: [0x7d,0x00,0x22,0xc4,0x04,0x18,0x00,0x01,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_d16_format_x v4, off, s[8:11], s0 format:[BUF_DATA_FORMAT_8, BUF_NUM_FORMAT_SNORM] offset:8388607
+// GFX12: encoding: [0x00,0x00,0x22,0xc4,0x04,0x10,0x00,0x01,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_d16_format_x v4, off, s[8:11], s61 format:[BUF_FMT_8_USCALED] offset:8388607
+// GFX12: encoding: [0x3d,0x00,0x22,0xc4,0x04,0x10,0x80,0x01,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_d16_format_x v4, off, ttmp[4:7], s61 format:3 offset:8388607
+// GFX12: encoding: [0x3d,0x00,0x22,0xc4,0x04,0xe0,0x80,0x01,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_d16_format_x v4, v1, s[8:11], s3 format:[BUF_DATA_FORMAT_8, BUF_NUM_FORMAT_USCALED] offen offset:52
+// GFX12: encoding: [0x03,0x00,0x22,0xc4,0x04,0x10,0x80,0x41,0x01,0x34,0x00,0x00]
+
+tbuffer_load_d16_format_x v4, v1, s[8:11], s3 format:[BUF_FMT_8_SSCALED] idxen offset:52
+// GFX12: encoding: [0x03,0x00,0x22,0xc4,0x04,0x10,0x00,0x82,0x01,0x34,0x00,0x00]
+
+tbuffer_load_d16_format_x v4, v[1:2], s[8:11], s0 format:4 idxen offen offset:52
+// GFX12: encoding: [0x00,0x00,0x22,0xc4,0x04,0x10,0x00,0xc2,0x01,0x34,0x00,0x00]
+
+tbuffer_load_d16_format_x v4, off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_8, BUF_NUM_FORMAT_SSCALED] offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0x00,0x22,0xc4,0x04,0xe0,0x68,0x02,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_d16_format_x v4, off, ttmp[4:7], s3 format:[BUF_FMT_8_UINT] offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS
+// GFX12: encoding: [0x03,0x00,0x22,0xc4,0x04,0xe0,0xbc,0x02,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_d16_format_x v4, off, ttmp[4:7], s3 format:5 offset:8388607 scope:SCOPE_SE
+// GFX12: encoding: [0x03,0x00,0x22,0xc4,0x04,0xe0,0x84,0x02,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_d16_format_x v4, off, ttmp[4:7], 0 format:[BUF_DATA_FORMAT_8, BUF_NUM_FORMAT_UINT] offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+tbuffer_load_d16_format_x v4, off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_8, BUF_NUM_FORMAT_UINT] offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_load_d16_format_x v4, off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_8, BUF_NUM_FORMAT_UINT] offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_load_d16_format_x v4, off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_8, BUF_NUM_FORMAT_UINT] offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_load_d16_format_xy v4, off, s[8:11], s3 format:[BUF_FMT_8_SINT] offset:8388607
+// GFX12: encoding: [0x03,0x40,0x22,0xc4,0x04,0x10,0x00,0x03,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_d16_format_xy v255, off, s[8:11], s3 format:6 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x22,0xc4,0xff,0x10,0x00,0x03,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_d16_format_xy v4, off, s[12:15], s3 format:[BUF_DATA_FORMAT_8, BUF_NUM_FORMAT_SINT] offset:8388607
+// GFX12: encoding: [0x03,0x40,0x22,0xc4,0x04,0x18,0x00,0x03,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_d16_format_xy v4, off, s[12:15], s101 format:[BUF_FMT_16_UNORM] offset:8388607
+// GFX12: encoding: [0x65,0x40,0x22,0xc4,0x04,0x18,0x80,0x03,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_d16_format_xy v4, off, s[12:15], m0 format:7 offset:8388607
+// GFX12: encoding: [0x7d,0x40,0x22,0xc4,0x04,0x18,0x80,0x03,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_d16_format_xy v4, off, s[8:11], s0 format:[BUF_DATA_FORMAT_16, BUF_NUM_FORMAT_UNORM] offset:8388607
+// GFX12: encoding: [0x00,0x40,0x22,0xc4,0x04,0x10,0x80,0x03,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_d16_format_xy v4, off, s[8:11], s61 format:[BUF_FMT_16_SNORM] offset:8388607
+// GFX12: encoding: [0x3d,0x40,0x22,0xc4,0x04,0x10,0x00,0x04,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_d16_format_xy v4, off, ttmp[4:7], s61 format:8 offset:8388607
+// GFX12: encoding: [0x3d,0x40,0x22,0xc4,0x04,0xe0,0x00,0x04,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_d16_format_xy v4, v1, s[8:11], s3 format:[BUF_DATA_FORMAT_16, BUF_NUM_FORMAT_SNORM] offen offset:52
+// GFX12: encoding: [0x03,0x40,0x22,0xc4,0x04,0x10,0x00,0x44,0x01,0x34,0x00,0x00]
+
+tbuffer_load_d16_format_xy v4, v1, s[8:11], s3 format:[BUF_FMT_16_USCALED] idxen offset:52
+// GFX12: encoding: [0x03,0x40,0x22,0xc4,0x04,0x10,0x80,0x84,0x01,0x34,0x00,0x00]
+
+tbuffer_load_d16_format_xy v4, v[1:2], s[8:11], s0 format:9 idxen offen offset:52
+// GFX12: encoding: [0x00,0x40,0x22,0xc4,0x04,0x10,0x80,0xc4,0x01,0x34,0x00,0x00]
+
+tbuffer_load_d16_format_xy v4, off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_16, BUF_NUM_FORMAT_USCALED] offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0x40,0x22,0xc4,0x04,0xe0,0xe8,0x04,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_d16_format_xy v4, off, ttmp[4:7], s3 format:[BUF_FMT_16_SSCALED] offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS
+// GFX12: encoding: [0x03,0x40,0x22,0xc4,0x04,0xe0,0x3c,0x05,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_d16_format_xy v4, off, ttmp[4:7], s3 format:10 offset:8388607 scope:SCOPE_SE
+// GFX12: encoding: [0x03,0x40,0x22,0xc4,0x04,0xe0,0x04,0x05,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_d16_format_xy v4, off, ttmp[4:7], 0 format:[BUF_DATA_FORMAT_16, BUF_NUM_FORMAT_SSCALED] offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+tbuffer_load_d16_format_xy v4, off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_16, BUF_NUM_FORMAT_SSCALED] offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_load_d16_format_xy v4, off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_16, BUF_NUM_FORMAT_SSCALED] offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_load_d16_format_xy v4, off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_16, BUF_NUM_FORMAT_SSCALED] offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_load_d16_format_xyz v[4:5], off, s[8:11], s3 format:[BUF_FMT_16_UINT] offset:8388607
+// GFX12: encoding: [0x03,0x80,0x22,0xc4,0x04,0x10,0x80,0x05,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_d16_format_xyz v[254:255], off, s[8:11], s3 format:11 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x22,0xc4,0xfe,0x10,0x80,0x05,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_d16_format_xyz v[4:5], off, s[12:15], s3 format:[BUF_DATA_FORMAT_16, BUF_NUM_FORMAT_UINT] offset:8388607
+// GFX12: encoding: [0x03,0x80,0x22,0xc4,0x04,0x18,0x80,0x05,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_d16_format_xyz v[4:5], off, s[12:15], s101 format:[BUF_FMT_16_SINT] offset:8388607
+// GFX12: encoding: [0x65,0x80,0x22,0xc4,0x04,0x18,0x00,0x06,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_d16_format_xyz v[4:5], off, s[12:15], m0 format:12 offset:8388607
+// GFX12: encoding: [0x7d,0x80,0x22,0xc4,0x04,0x18,0x00,0x06,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_d16_format_xyz v[4:5], off, s[8:11], s0 format:[BUF_DATA_FORMAT_16, BUF_NUM_FORMAT_SINT] offset:8388607
+// GFX12: encoding: [0x00,0x80,0x22,0xc4,0x04,0x10,0x00,0x06,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_d16_format_xyz v[4:5], off, s[8:11], s61 format:[BUF_FMT_16_FLOAT] offset:8388607
+// GFX12: encoding: [0x3d,0x80,0x22,0xc4,0x04,0x10,0x80,0x06,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_d16_format_xyz v[4:5], off, ttmp[4:7], s61 format:13 offset:8388607
+// GFX12: encoding: [0x3d,0x80,0x22,0xc4,0x04,0xe0,0x80,0x06,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_d16_format_xyz v[4:5], v1, s[8:11], s3 format:[BUF_DATA_FORMAT_16, BUF_NUM_FORMAT_FLOAT] offen offset:52
+// GFX12: encoding: [0x03,0x80,0x22,0xc4,0x04,0x10,0x80,0x46,0x01,0x34,0x00,0x00]
+
+tbuffer_load_d16_format_xyz v[4:5], v1, s[8:11], s3 format:[BUF_FMT_8_8_UNORM] idxen offset:52
+// GFX12: encoding: [0x03,0x80,0x22,0xc4,0x04,0x10,0x00,0x87,0x01,0x34,0x00,0x00]
+
+tbuffer_load_d16_format_xyz v[4:5], v[1:2], s[8:11], s0 format:14 idxen offen offset:52
+// GFX12: encoding: [0x00,0x80,0x22,0xc4,0x04,0x10,0x00,0xc7,0x01,0x34,0x00,0x00]
+
+tbuffer_load_d16_format_xyz v[4:5], off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_8_8, BUF_NUM_FORMAT_UNORM] offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0x80,0x22,0xc4,0x04,0xe0,0x68,0x07,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_d16_format_xyz v[4:5], off, ttmp[4:7], s3 format:[BUF_FMT_8_8_SNORM] offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS
+// GFX12: encoding: [0x03,0x80,0x22,0xc4,0x04,0xe0,0xbc,0x07,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_d16_format_xyz v[4:5], off, ttmp[4:7], s3 format:15 offset:8388607 scope:SCOPE_SE
+// GFX12: encoding: [0x03,0x80,0x22,0xc4,0x04,0xe0,0x84,0x07,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_d16_format_xyz v[4:5], off, ttmp[4:7], 0 format:[BUF_DATA_FORMAT_8_8, BUF_NUM_FORMAT_SNORM] offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+tbuffer_load_d16_format_xyz v[4:5], off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_8_8, BUF_NUM_FORMAT_SNORM] offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_load_d16_format_xyz v[4:5], off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_8_8, BUF_NUM_FORMAT_SNORM] offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_load_d16_format_xyz v[4:5], off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_8_8, BUF_NUM_FORMAT_SNORM] offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_load_d16_format_xyzw v[4:5], off, s[8:11], s3 format:[BUF_FMT_8_8_USCALED] offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x22,0xc4,0x04,0x10,0x00,0x08,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_d16_format_xyzw v[254:255], off, s[8:11], s3 format:16 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x22,0xc4,0xfe,0x10,0x00,0x08,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_d16_format_xyzw v[4:5], off, s[12:15], s3 format:[BUF_DATA_FORMAT_8_8, BUF_NUM_FORMAT_USCALED] offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x22,0xc4,0x04,0x18,0x00,0x08,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_d16_format_xyzw v[4:5], off, s[12:15], s101 format:[BUF_FMT_8_8_SSCALED] offset:8388607
+// GFX12: encoding: [0x65,0xc0,0x22,0xc4,0x04,0x18,0x80,0x08,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_d16_format_xyzw v[4:5], off, s[12:15], m0 format:17 offset:8388607
+// GFX12: encoding: [0x7d,0xc0,0x22,0xc4,0x04,0x18,0x80,0x08,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_d16_format_xyzw v[4:5], off, s[8:11], s0 format:[BUF_DATA_FORMAT_8_8, BUF_NUM_FORMAT_SSCALED] offset:8388607
+// GFX12: encoding: [0x00,0xc0,0x22,0xc4,0x04,0x10,0x80,0x08,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_d16_format_xyzw v[4:5], off, s[8:11], s61 format:[BUF_FMT_8_8_UINT] offset:8388607
+// GFX12: encoding: [0x3d,0xc0,0x22,0xc4,0x04,0x10,0x00,0x09,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_d16_format_xyzw v[4:5], off, ttmp[4:7], s61 format:18 offset:8388607
+// GFX12: encoding: [0x3d,0xc0,0x22,0xc4,0x04,0xe0,0x00,0x09,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_d16_format_xyzw v[4:5], v1, s[8:11], s3 format:[BUF_DATA_FORMAT_8_8, BUF_NUM_FORMAT_UINT] offen offset:52
+// GFX12: encoding: [0x03,0xc0,0x22,0xc4,0x04,0x10,0x00,0x49,0x01,0x34,0x00,0x00]
+
+tbuffer_load_d16_format_xyzw v[4:5], v1, s[8:11], s3 format:[BUF_FMT_8_8_SINT] idxen offset:52
+// GFX12: encoding: [0x03,0xc0,0x22,0xc4,0x04,0x10,0x80,0x89,0x01,0x34,0x00,0x00]
+
+tbuffer_load_d16_format_xyzw v[4:5], v[1:2], s[8:11], s0 format:19 idxen offen offset:52
+// GFX12: encoding: [0x00,0xc0,0x22,0xc4,0x04,0x10,0x80,0xc9,0x01,0x34,0x00,0x00]
+
+tbuffer_load_d16_format_xyzw v[4:5], off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_8_8, BUF_NUM_FORMAT_SINT] offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0xc0,0x22,0xc4,0x04,0xe0,0xe8,0x09,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_d16_format_xyzw v[4:5], off, ttmp[4:7], s3 format:[BUF_FMT_32_UINT] offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS
+// GFX12: encoding: [0x03,0xc0,0x22,0xc4,0x04,0xe0,0x3c,0x0a,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_d16_format_xyzw v[4:5], off, ttmp[4:7], s3 format:20 offset:8388607 scope:SCOPE_SE
+// GFX12: encoding: [0x03,0xc0,0x22,0xc4,0x04,0xe0,0x04,0x0a,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_d16_format_xyzw v[4:5], off, ttmp[4:7], 0 format:[BUF_DATA_FORMAT_32, BUF_NUM_FORMAT_UINT] offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+tbuffer_load_d16_format_xyzw v[4:5], off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_32, BUF_NUM_FORMAT_UINT] offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_load_d16_format_xyzw v[4:5], off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_32, BUF_NUM_FORMAT_UINT] offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_load_d16_format_xyzw v[4:5], off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_32, BUF_NUM_FORMAT_UINT] offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_load_format_x v4, off, s[8:11], s3 format:[BUF_FMT_32_SINT] offset:8388607
+// GFX12: encoding: [0x03,0x00,0x20,0xc4,0x04,0x10,0x80,0x0a,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_x v255, off, s[8:11], s3 format:21 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x20,0xc4,0xff,0x10,0x80,0x0a,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_x v4, off, s[12:15], s3 format:[BUF_DATA_FORMAT_32, BUF_NUM_FORMAT_SINT] offset:8388607
+// GFX12: encoding: [0x03,0x00,0x20,0xc4,0x04,0x18,0x80,0x0a,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_x v4, off, s[12:15], s101 format:[BUF_FMT_32_FLOAT] offset:8388607
+// GFX12: encoding: [0x65,0x00,0x20,0xc4,0x04,0x18,0x00,0x0b,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_x v4, off, s[12:15], m0 format:22 offset:8388607
+// GFX12: encoding: [0x7d,0x00,0x20,0xc4,0x04,0x18,0x00,0x0b,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_x v4, off, s[8:11], s0 format:[BUF_DATA_FORMAT_32, BUF_NUM_FORMAT_FLOAT] offset:8388607
+// GFX12: encoding: [0x00,0x00,0x20,0xc4,0x04,0x10,0x00,0x0b,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_x v4, off, s[8:11], s61 format:[BUF_FMT_16_16_UNORM] offset:8388607
+// GFX12: encoding: [0x3d,0x00,0x20,0xc4,0x04,0x10,0x80,0x0b,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_x v4, off, ttmp[4:7], s61 format:23 offset:8388607
+// GFX12: encoding: [0x3d,0x00,0x20,0xc4,0x04,0xe0,0x80,0x0b,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_x v4, v1, s[8:11], s3 format:[BUF_DATA_FORMAT_16_16, BUF_NUM_FORMAT_UNORM] offen offset:52
+// GFX12: encoding: [0x03,0x00,0x20,0xc4,0x04,0x10,0x80,0x4b,0x01,0x34,0x00,0x00]
+
+tbuffer_load_format_x v4, v1, s[8:11], s3 format:[BUF_FMT_16_16_SNORM] idxen offset:52
+// GFX12: encoding: [0x03,0x00,0x20,0xc4,0x04,0x10,0x00,0x8c,0x01,0x34,0x00,0x00]
+
+tbuffer_load_format_x v4, v[1:2], s[8:11], s0 format:24 idxen offen offset:52
+// GFX12: encoding: [0x00,0x00,0x20,0xc4,0x04,0x10,0x00,0xcc,0x01,0x34,0x00,0x00]
+
+tbuffer_load_format_x v4, off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_16_16, BUF_NUM_FORMAT_SNORM] offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0x00,0x20,0xc4,0x04,0xe0,0x68,0x0c,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_x v4, off, ttmp[4:7], s3 format:[BUF_FMT_16_16_USCALED] offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS
+// GFX12: encoding: [0x03,0x00,0x20,0xc4,0x04,0xe0,0xbc,0x0c,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_x v4, off, ttmp[4:7], s3 format:25 offset:8388607 scope:SCOPE_SE
+// GFX12: encoding: [0x03,0x00,0x20,0xc4,0x04,0xe0,0x84,0x0c,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_x v4, off, ttmp[4:7], 0 format:[BUF_DATA_FORMAT_16_16, BUF_NUM_FORMAT_USCALED] offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+tbuffer_load_format_x v4, off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_16_16, BUF_NUM_FORMAT_USCALED] offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_load_format_x v4, off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_16_16, BUF_NUM_FORMAT_USCALED] offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_load_format_x v4, off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_16_16, BUF_NUM_FORMAT_USCALED] offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_load_format_xy v[4:5], off, s[8:11], s3 format:[BUF_FMT_16_16_SSCALED] offset:8388607
+// GFX12: encoding: [0x03,0x40,0x20,0xc4,0x04,0x10,0x00,0x0d,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_xy v[254:255], off, s[8:11], s3 format:26 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x20,0xc4,0xfe,0x10,0x00,0x0d,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_xy v[4:5], off, s[12:15], s3 format:[BUF_DATA_FORMAT_16_16, BUF_NUM_FORMAT_SSCALED] offset:8388607
+// GFX12: encoding: [0x03,0x40,0x20,0xc4,0x04,0x18,0x00,0x0d,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_xy v[4:5], off, s[12:15], s101 format:[BUF_FMT_16_16_UINT] offset:8388607
+// GFX12: encoding: [0x65,0x40,0x20,0xc4,0x04,0x18,0x80,0x0d,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_xy v[4:5], off, s[12:15], m0 format:27 offset:8388607
+// GFX12: encoding: [0x7d,0x40,0x20,0xc4,0x04,0x18,0x80,0x0d,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_xy v[4:5], off, s[8:11], s0 format:[BUF_DATA_FORMAT_16_16, BUF_NUM_FORMAT_UINT] offset:8388607
+// GFX12: encoding: [0x00,0x40,0x20,0xc4,0x04,0x10,0x80,0x0d,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_xy v[4:5], off, s[8:11], s61 format:[BUF_FMT_16_16_SINT] offset:8388607
+// GFX12: encoding: [0x3d,0x40,0x20,0xc4,0x04,0x10,0x00,0x0e,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_xy v[4:5], off, ttmp[4:7], s61 format:28 offset:8388607
+// GFX12: encoding: [0x3d,0x40,0x20,0xc4,0x04,0xe0,0x00,0x0e,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_xy v[4:5], v1, s[8:11], s3 format:[BUF_DATA_FORMAT_16_16, BUF_NUM_FORMAT_SINT] offen offset:52
+// GFX12: encoding: [0x03,0x40,0x20,0xc4,0x04,0x10,0x00,0x4e,0x01,0x34,0x00,0x00]
+
+tbuffer_load_format_xy v[4:5], v1, s[8:11], s3 format:[BUF_FMT_16_16_FLOAT] idxen offset:52
+// GFX12: encoding: [0x03,0x40,0x20,0xc4,0x04,0x10,0x80,0x8e,0x01,0x34,0x00,0x00]
+
+tbuffer_load_format_xy v[4:5], v[1:2], s[8:11], s0 format:29 idxen offen offset:52
+// GFX12: encoding: [0x00,0x40,0x20,0xc4,0x04,0x10,0x80,0xce,0x01,0x34,0x00,0x00]
+
+tbuffer_load_format_xy v[4:5], off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_16_16, BUF_NUM_FORMAT_FLOAT] offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0x40,0x20,0xc4,0x04,0xe0,0xe8,0x0e,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_xy v[4:5], off, ttmp[4:7], s3 format:[BUF_FMT_10_11_11_FLOAT] offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS
+// GFX12: encoding: [0x03,0x40,0x20,0xc4,0x04,0xe0,0x3c,0x0f,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_xy v[4:5], off, ttmp[4:7], s3 format:30 offset:8388607 scope:SCOPE_SE
+// GFX12: encoding: [0x03,0x40,0x20,0xc4,0x04,0xe0,0x04,0x0f,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_xy v[4:5], off, ttmp[4:7], 0 format:[BUF_DATA_FORMAT_10_11_11, BUF_NUM_FORMAT_FLOAT] offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+tbuffer_load_format_xy v[4:5], off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_10_11_11, BUF_NUM_FORMAT_FLOAT] offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_load_format_xy v[4:5], off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_10_11_11, BUF_NUM_FORMAT_FLOAT] offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_load_format_xy v[4:5], off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_10_11_11, BUF_NUM_FORMAT_FLOAT] offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_load_format_xyz v[4:6], off, s[8:11], s3 format:[BUF_FMT_11_11_10_FLOAT] offset:8388607
+// GFX12: encoding: [0x03,0x80,0x20,0xc4,0x04,0x10,0x80,0x0f,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_xyz v[253:255], off, s[8:11], s3 format:31 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x20,0xc4,0xfd,0x10,0x80,0x0f,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_xyz v[4:6], off, s[12:15], s3 format:[BUF_DATA_FORMAT_11_11_10, BUF_NUM_FORMAT_FLOAT] offset:8388607
+// GFX12: encoding: [0x03,0x80,0x20,0xc4,0x04,0x18,0x80,0x0f,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_xyz v[4:6], off, s[12:15], s101 format:[BUF_FMT_10_10_10_2_UNORM] offset:8388607
+// GFX12: encoding: [0x65,0x80,0x20,0xc4,0x04,0x18,0x00,0x10,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_xyz v[4:6], off, s[12:15], m0 format:32 offset:8388607
+// GFX12: encoding: [0x7d,0x80,0x20,0xc4,0x04,0x18,0x00,0x10,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_xyz v[4:6], off, s[8:11], s0 format:[BUF_DATA_FORMAT_10_10_10_2, BUF_NUM_FORMAT_UNORM] offset:8388607
+// GFX12: encoding: [0x00,0x80,0x20,0xc4,0x04,0x10,0x00,0x10,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_xyz v[4:6], off, s[8:11], s61 format:[BUF_FMT_10_10_10_2_SNORM] offset:8388607
+// GFX12: encoding: [0x3d,0x80,0x20,0xc4,0x04,0x10,0x80,0x10,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_xyz v[4:6], off, ttmp[4:7], s61 format:33 offset:8388607
+// GFX12: encoding: [0x3d,0x80,0x20,0xc4,0x04,0xe0,0x80,0x10,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_xyz v[4:6], v1, s[8:11], s3 format:[BUF_DATA_FORMAT_10_10_10_2, BUF_NUM_FORMAT_SNORM] offen offset:52
+// GFX12: encoding: [0x03,0x80,0x20,0xc4,0x04,0x10,0x80,0x50,0x01,0x34,0x00,0x00]
+
+tbuffer_load_format_xyz v[4:6], v1, s[8:11], s3 format:[BUF_FMT_10_10_10_2_UINT] idxen offset:52
+// GFX12: encoding: [0x03,0x80,0x20,0xc4,0x04,0x10,0x00,0x91,0x01,0x34,0x00,0x00]
+
+tbuffer_load_format_xyz v[4:6], v[1:2], s[8:11], s0 format:34 idxen offen offset:52
+// GFX12: encoding: [0x00,0x80,0x20,0xc4,0x04,0x10,0x00,0xd1,0x01,0x34,0x00,0x00]
+
+tbuffer_load_format_xyz v[4:6], off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_10_10_10_2, BUF_NUM_FORMAT_UINT] offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0x80,0x20,0xc4,0x04,0xe0,0x68,0x11,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_xyz v[4:6], off, ttmp[4:7], s3 format:[BUF_FMT_10_10_10_2_SINT] offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS
+// GFX12: encoding: [0x03,0x80,0x20,0xc4,0x04,0xe0,0xbc,0x11,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_xyz v[4:6], off, ttmp[4:7], s3 format:35 offset:8388607 scope:SCOPE_SE
+// GFX12: encoding: [0x03,0x80,0x20,0xc4,0x04,0xe0,0x84,0x11,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_xyz v[4:6], off, ttmp[4:7], 0 format:[BUF_DATA_FORMAT_10_10_10_2, BUF_NUM_FORMAT_SINT] offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+tbuffer_load_format_xyz v[4:6], off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_10_10_10_2, BUF_NUM_FORMAT_SINT] offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_load_format_xyz v[4:6], off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_10_10_10_2, BUF_NUM_FORMAT_SINT] offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_load_format_xyz v[4:6], off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_10_10_10_2, BUF_NUM_FORMAT_SINT] offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_load_format_xyzw v[4:7], off, s[8:11], s3 format:[BUF_FMT_2_10_10_10_UNORM] offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x20,0xc4,0x04,0x10,0x00,0x12,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_xyzw v[252:255], off, s[8:11], s3 format:36 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x20,0xc4,0xfc,0x10,0x00,0x12,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_xyzw v[4:7], off, s[12:15], s3 format:[BUF_DATA_FORMAT_2_10_10_10, BUF_NUM_FORMAT_UNORM] offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x20,0xc4,0x04,0x18,0x00,0x12,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_xyzw v[4:7], off, s[12:15], s101 format:[BUF_FMT_2_10_10_10_SNORM] offset:8388607
+// GFX12: encoding: [0x65,0xc0,0x20,0xc4,0x04,0x18,0x80,0x12,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_xyzw v[4:7], off, s[12:15], m0 format:37 offset:8388607
+// GFX12: encoding: [0x7d,0xc0,0x20,0xc4,0x04,0x18,0x80,0x12,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_xyzw v[4:7], off, s[8:11], s0 format:[BUF_DATA_FORMAT_2_10_10_10, BUF_NUM_FORMAT_SNORM] offset:8388607
+// GFX12: encoding: [0x00,0xc0,0x20,0xc4,0x04,0x10,0x80,0x12,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_xyzw v[4:7], off, s[8:11], s61 format:[BUF_FMT_2_10_10_10_USCALED] offset:8388607
+// GFX12: encoding: [0x3d,0xc0,0x20,0xc4,0x04,0x10,0x00,0x13,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_xyzw v[4:7], off, ttmp[4:7], s61 format:38 offset:8388607
+// GFX12: encoding: [0x3d,0xc0,0x20,0xc4,0x04,0xe0,0x00,0x13,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_xyzw v[4:7], v1, s[8:11], s3 format:[BUF_DATA_FORMAT_2_10_10_10, BUF_NUM_FORMAT_USCALED] offen offset:52
+// GFX12: encoding: [0x03,0xc0,0x20,0xc4,0x04,0x10,0x00,0x53,0x01,0x34,0x00,0x00]
+
+tbuffer_load_format_xyzw v[4:7], v1, s[8:11], s3 format:[BUF_FMT_2_10_10_10_SSCALED] idxen offset:52
+// GFX12: encoding: [0x03,0xc0,0x20,0xc4,0x04,0x10,0x80,0x93,0x01,0x34,0x00,0x00]
+
+tbuffer_load_format_xyzw v[4:7], v[1:2], s[8:11], s0 format:39 idxen offen offset:52
+// GFX12: encoding: [0x00,0xc0,0x20,0xc4,0x04,0x10,0x80,0xd3,0x01,0x34,0x00,0x00]
+
+tbuffer_load_format_xyzw v[4:7], off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_2_10_10_10, BUF_NUM_FORMAT_SSCALED] offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0xc0,0x20,0xc4,0x04,0xe0,0xe8,0x13,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_xyzw v[4:7], off, ttmp[4:7], s3 format:[BUF_FMT_2_10_10_10_UINT] offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS
+// GFX12: encoding: [0x03,0xc0,0x20,0xc4,0x04,0xe0,0x3c,0x14,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_xyzw v[4:7], off, ttmp[4:7], s3 format:40 offset:8388607 scope:SCOPE_SE
+// GFX12: encoding: [0x03,0xc0,0x20,0xc4,0x04,0xe0,0x04,0x14,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_xyzw v[4:7], off, ttmp[4:7], 0 format:[BUF_DATA_FORMAT_2_10_10_10, BUF_NUM_FORMAT_UINT] offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+tbuffer_load_format_xyzw v[4:7], off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_2_10_10_10, BUF_NUM_FORMAT_UINT] offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_load_format_xyzw v[4:7], off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_2_10_10_10, BUF_NUM_FORMAT_UINT] offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_load_format_xyzw v[4:7], off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_2_10_10_10, BUF_NUM_FORMAT_UINT] offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_store_d16_format_x v4, off, s[8:11], s3 format:[BUF_FMT_2_10_10_10_SINT] offset:8388607
+// GFX12: encoding: [0x03,0x00,0x23,0xc4,0x04,0x10,0x80,0x14,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_d16_format_x v255, off, s[8:11], s3 format:41 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x23,0xc4,0xff,0x10,0x80,0x14,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_d16_format_x v4, off, s[12:15], s3 format:[BUF_DATA_FORMAT_2_10_10_10, BUF_NUM_FORMAT_SINT] offset:8388607
+// GFX12: encoding: [0x03,0x00,0x23,0xc4,0x04,0x18,0x80,0x14,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_d16_format_x v4, off, s[12:15], s101 format:[BUF_FMT_8_8_8_8_UNORM] offset:8388607
+// GFX12: encoding: [0x65,0x00,0x23,0xc4,0x04,0x18,0x00,0x15,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_d16_format_x v4, off, s[12:15], m0 format:42 offset:8388607
+// GFX12: encoding: [0x7d,0x00,0x23,0xc4,0x04,0x18,0x00,0x15,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_d16_format_x v4, off, s[8:11], s0 format:[BUF_DATA_FORMAT_8_8_8_8, BUF_NUM_FORMAT_UNORM] offset:8388607
+// GFX12: encoding: [0x00,0x00,0x23,0xc4,0x04,0x10,0x00,0x15,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_d16_format_x v4, off, s[8:11], s61 format:[BUF_FMT_8_8_8_8_SNORM] offset:8388607
+// GFX12: encoding: [0x3d,0x00,0x23,0xc4,0x04,0x10,0x80,0x15,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_d16_format_x v4, off, ttmp[4:7], s61 format:43 offset:8388607
+// GFX12: encoding: [0x3d,0x00,0x23,0xc4,0x04,0xe0,0x80,0x15,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_d16_format_x v4, v1, s[8:11], s3 format:[BUF_DATA_FORMAT_8_8_8_8, BUF_NUM_FORMAT_SNORM] offen offset:52
+// GFX12: encoding: [0x03,0x00,0x23,0xc4,0x04,0x10,0x80,0x55,0x01,0x34,0x00,0x00]
+
+tbuffer_store_d16_format_x v4, v1, s[8:11], s3 format:[BUF_FMT_8_8_8_8_USCALED] idxen offset:52
+// GFX12: encoding: [0x03,0x00,0x23,0xc4,0x04,0x10,0x00,0x96,0x01,0x34,0x00,0x00]
+
+tbuffer_store_d16_format_x v4, v[1:2], s[8:11], s0 format:44 idxen offen offset:52
+// GFX12: encoding: [0x00,0x00,0x23,0xc4,0x04,0x10,0x00,0xd6,0x01,0x34,0x00,0x00]
+
+tbuffer_store_d16_format_x v4, off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_8_8_8_8, BUF_NUM_FORMAT_USCALED] offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0x00,0x23,0xc4,0x04,0xe0,0x68,0x16,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_d16_format_x v4, off, ttmp[4:7], s3 format:[BUF_FMT_8_8_8_8_SSCALED] offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS
+// GFX12: encoding: [0x03,0x00,0x23,0xc4,0x04,0xe0,0xbc,0x16,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_d16_format_x v4, off, ttmp[4:7], s3 format:45 offset:8388607 scope:SCOPE_SE
+// GFX12: encoding: [0x03,0x00,0x23,0xc4,0x04,0xe0,0x84,0x16,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_d16_format_x v4, off, ttmp[4:7], 0 format:[BUF_DATA_FORMAT_8_8_8_8, BUF_NUM_FORMAT_SSCALED] offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+tbuffer_store_d16_format_x v4, off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_8_8_8_8, BUF_NUM_FORMAT_SSCALED] offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_store_d16_format_x v4, off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_8_8_8_8, BUF_NUM_FORMAT_SSCALED] offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_store_d16_format_x v4, off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_8_8_8_8, BUF_NUM_FORMAT_SSCALED] offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_store_d16_format_xy v4, off, s[8:11], s3 format:[BUF_FMT_8_8_8_8_UINT] offset:8388607
+// GFX12: encoding: [0x03,0x40,0x23,0xc4,0x04,0x10,0x00,0x17,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_d16_format_xy v255, off, s[8:11], s3 format:46 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x23,0xc4,0xff,0x10,0x00,0x17,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_d16_format_xy v4, off, s[12:15], s3 format:[BUF_DATA_FORMAT_8_8_8_8, BUF_NUM_FORMAT_UINT] offset:8388607
+// GFX12: encoding: [0x03,0x40,0x23,0xc4,0x04,0x18,0x00,0x17,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_d16_format_xy v4, off, s[12:15], s101 format:[BUF_FMT_8_8_8_8_SINT] offset:8388607
+// GFX12: encoding: [0x65,0x40,0x23,0xc4,0x04,0x18,0x80,0x17,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_d16_format_xy v4, off, s[12:15], m0 format:47 offset:8388607
+// GFX12: encoding: [0x7d,0x40,0x23,0xc4,0x04,0x18,0x80,0x17,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_d16_format_xy v4, off, s[8:11], s0 format:[BUF_DATA_FORMAT_8_8_8_8, BUF_NUM_FORMAT_SINT] offset:8388607
+// GFX12: encoding: [0x00,0x40,0x23,0xc4,0x04,0x10,0x80,0x17,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_d16_format_xy v4, off, s[8:11], s61 format:[BUF_FMT_32_32_UINT] offset:8388607
+// GFX12: encoding: [0x3d,0x40,0x23,0xc4,0x04,0x10,0x00,0x18,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_d16_format_xy v4, off, ttmp[4:7], s61 format:48 offset:8388607
+// GFX12: encoding: [0x3d,0x40,0x23,0xc4,0x04,0xe0,0x00,0x18,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_d16_format_xy v4, v1, s[8:11], s3 format:[BUF_DATA_FORMAT_32_32, BUF_NUM_FORMAT_UINT] offen offset:52
+// GFX12: encoding: [0x03,0x40,0x23,0xc4,0x04,0x10,0x00,0x58,0x01,0x34,0x00,0x00]
+
+tbuffer_store_d16_format_xy v4, v1, s[8:11], s3 format:[BUF_FMT_32_32_SINT] idxen offset:52
+// GFX12: encoding: [0x03,0x40,0x23,0xc4,0x04,0x10,0x80,0x98,0x01,0x34,0x00,0x00]
+
+tbuffer_store_d16_format_xy v4, v[1:2], s[8:11], s0 format:49 idxen offen offset:52
+// GFX12: encoding: [0x00,0x40,0x23,0xc4,0x04,0x10,0x80,0xd8,0x01,0x34,0x00,0x00]
+
+tbuffer_store_d16_format_xy v4, off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_32_32, BUF_NUM_FORMAT_SINT] offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0x40,0x23,0xc4,0x04,0xe0,0xe8,0x18,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_d16_format_xy v4, off, ttmp[4:7], s3 format:[BUF_FMT_32_32_FLOAT] offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS
+// GFX12: encoding: [0x03,0x40,0x23,0xc4,0x04,0xe0,0x3c,0x19,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_d16_format_xy v4, off, ttmp[4:7], s3 format:50 offset:8388607 scope:SCOPE_SE
+// GFX12: encoding: [0x03,0x40,0x23,0xc4,0x04,0xe0,0x04,0x19,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_d16_format_xy v4, off, ttmp[4:7], 0 format:[BUF_DATA_FORMAT_32_32, BUF_NUM_FORMAT_FLOAT] offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+tbuffer_store_d16_format_xy v4, off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_32_32, BUF_NUM_FORMAT_FLOAT] offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_store_d16_format_xy v4, off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_32_32, BUF_NUM_FORMAT_FLOAT] offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_store_d16_format_xy v4, off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_32_32, BUF_NUM_FORMAT_FLOAT] offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_store_d16_format_xyz v[4:5], off, s[8:11], s3 format:[BUF_FMT_16_16_16_16_UNORM] offset:8388607
+// GFX12: encoding: [0x03,0x80,0x23,0xc4,0x04,0x10,0x80,0x19,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_d16_format_xyz v[254:255], off, s[8:11], s3 format:51 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x23,0xc4,0xfe,0x10,0x80,0x19,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_d16_format_xyz v[4:5], off, s[12:15], s3 format:[BUF_DATA_FORMAT_16_16_16_16, BUF_NUM_FORMAT_UNORM] offset:8388607
+// GFX12: encoding: [0x03,0x80,0x23,0xc4,0x04,0x18,0x80,0x19,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_d16_format_xyz v[4:5], off, s[12:15], s101 format:[BUF_FMT_16_16_16_16_SNORM] offset:8388607
+// GFX12: encoding: [0x65,0x80,0x23,0xc4,0x04,0x18,0x00,0x1a,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_d16_format_xyz v[4:5], off, s[12:15], m0 format:52 offset:8388607
+// GFX12: encoding: [0x7d,0x80,0x23,0xc4,0x04,0x18,0x00,0x1a,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_d16_format_xyz v[4:5], off, s[8:11], s0 format:[BUF_DATA_FORMAT_16_16_16_16, BUF_NUM_FORMAT_SNORM] offset:8388607
+// GFX12: encoding: [0x00,0x80,0x23,0xc4,0x04,0x10,0x00,0x1a,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_d16_format_xyz v[4:5], off, s[8:11], s61 format:[BUF_FMT_16_16_16_16_USCALED] offset:8388607
+// GFX12: encoding: [0x3d,0x80,0x23,0xc4,0x04,0x10,0x80,0x1a,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_d16_format_xyz v[4:5], off, ttmp[4:7], s61 format:53 offset:8388607
+// GFX12: encoding: [0x3d,0x80,0x23,0xc4,0x04,0xe0,0x80,0x1a,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_d16_format_xyz v[4:5], v1, s[8:11], s3 format:[BUF_DATA_FORMAT_16_16_16_16, BUF_NUM_FORMAT_USCALED] offen offset:52
+// GFX12: encoding: [0x03,0x80,0x23,0xc4,0x04,0x10,0x80,0x5a,0x01,0x34,0x00,0x00]
+
+tbuffer_store_d16_format_xyz v[4:5], v1, s[8:11], s3 format:[BUF_FMT_16_16_16_16_SSCALED] idxen offset:52
+// GFX12: encoding: [0x03,0x80,0x23,0xc4,0x04,0x10,0x00,0x9b,0x01,0x34,0x00,0x00]
+
+tbuffer_store_d16_format_xyz v[4:5], v[1:2], s[8:11], s0 format:54 idxen offen offset:52
+// GFX12: encoding: [0x00,0x80,0x23,0xc4,0x04,0x10,0x00,0xdb,0x01,0x34,0x00,0x00]
+
+tbuffer_store_d16_format_xyz v[4:5], off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_16_16_16_16, BUF_NUM_FORMAT_SSCALED] offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0x80,0x23,0xc4,0x04,0xe0,0x68,0x1b,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_d16_format_xyz v[4:5], off, ttmp[4:7], s3 format:[BUF_FMT_16_16_16_16_UINT] offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS
+// GFX12: encoding: [0x03,0x80,0x23,0xc4,0x04,0xe0,0xbc,0x1b,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_d16_format_xyz v[4:5], off, ttmp[4:7], s3 format:55 offset:8388607 scope:SCOPE_SE
+// GFX12: encoding: [0x03,0x80,0x23,0xc4,0x04,0xe0,0x84,0x1b,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_d16_format_xyz v[4:5], off, ttmp[4:7], 0 format:[BUF_DATA_FORMAT_16_16_16_16, BUF_NUM_FORMAT_UINT] offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+tbuffer_store_d16_format_xyz v[4:5], off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_16_16_16_16, BUF_NUM_FORMAT_UINT] offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_store_d16_format_xyz v[4:5], off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_16_16_16_16, BUF_NUM_FORMAT_UINT] offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_store_d16_format_xyz v[4:5], off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_16_16_16_16, BUF_NUM_FORMAT_UINT] offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_store_d16_format_xyzw v[4:5], off, s[8:11], s3 format:[BUF_FMT_16_16_16_16_SINT] offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x23,0xc4,0x04,0x10,0x00,0x1c,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_d16_format_xyzw v[254:255], off, s[8:11], s3 format:56 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x23,0xc4,0xfe,0x10,0x00,0x1c,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_d16_format_xyzw v[4:5], off, s[12:15], s3 format:[BUF_DATA_FORMAT_16_16_16_16, BUF_NUM_FORMAT_SINT] offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x23,0xc4,0x04,0x18,0x00,0x1c,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_d16_format_xyzw v[4:5], off, s[12:15], s101 format:[BUF_FMT_16_16_16_16_FLOAT] offset:8388607
+// GFX12: encoding: [0x65,0xc0,0x23,0xc4,0x04,0x18,0x80,0x1c,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_d16_format_xyzw v[4:5], off, s[12:15], m0 format:57 offset:8388607
+// GFX12: encoding: [0x7d,0xc0,0x23,0xc4,0x04,0x18,0x80,0x1c,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_d16_format_xyzw v[4:5], off, s[8:11], s0 format:[BUF_DATA_FORMAT_16_16_16_16, BUF_NUM_FORMAT_FLOAT] offset:8388607
+// GFX12: encoding: [0x00,0xc0,0x23,0xc4,0x04,0x10,0x80,0x1c,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_d16_format_xyzw v[4:5], off, s[8:11], s61 format:[BUF_FMT_32_32_32_UINT] offset:8388607
+// GFX12: encoding: [0x3d,0xc0,0x23,0xc4,0x04,0x10,0x00,0x1d,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_d16_format_xyzw v[4:5], off, ttmp[4:7], s61 format:58 offset:8388607
+// GFX12: encoding: [0x3d,0xc0,0x23,0xc4,0x04,0xe0,0x00,0x1d,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_d16_format_xyzw v[4:5], v1, s[8:11], s3 format:[BUF_DATA_FORMAT_32_32_32, BUF_NUM_FORMAT_UINT] offen offset:52
+// GFX12: encoding: [0x03,0xc0,0x23,0xc4,0x04,0x10,0x00,0x5d,0x01,0x34,0x00,0x00]
+
+tbuffer_store_d16_format_xyzw v[4:5], v1, s[8:11], s3 format:[BUF_FMT_32_32_32_SINT] idxen offset:52
+// GFX12: encoding: [0x03,0xc0,0x23,0xc4,0x04,0x10,0x80,0x9d,0x01,0x34,0x00,0x00]
+
+tbuffer_store_d16_format_xyzw v[4:5], v[1:2], s[8:11], s0 format:59 idxen offen offset:52
+// GFX12: encoding: [0x00,0xc0,0x23,0xc4,0x04,0x10,0x80,0xdd,0x01,0x34,0x00,0x00]
+
+tbuffer_store_d16_format_xyzw v[4:5], off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_32_32_32, BUF_NUM_FORMAT_SINT] offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0xc0,0x23,0xc4,0x04,0xe0,0xe8,0x1d,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_d16_format_xyzw v[4:5], off, ttmp[4:7], s3 format:[BUF_FMT_32_32_32_FLOAT] offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS
+// GFX12: encoding: [0x03,0xc0,0x23,0xc4,0x04,0xe0,0x3c,0x1e,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_d16_format_xyzw v[4:5], off, ttmp[4:7], s3 format:60 offset:8388607 scope:SCOPE_SE
+// GFX12: encoding: [0x03,0xc0,0x23,0xc4,0x04,0xe0,0x04,0x1e,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_d16_format_xyzw v[4:5], off, ttmp[4:7], 0 format:[BUF_DATA_FORMAT_32_32_32, BUF_NUM_FORMAT_FLOAT] offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+tbuffer_store_d16_format_xyzw v[4:5], off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_32_32_32, BUF_NUM_FORMAT_FLOAT] offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_store_d16_format_xyzw v[4:5], off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_32_32_32, BUF_NUM_FORMAT_FLOAT] offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_store_d16_format_xyzw v[4:5], off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_32_32_32, BUF_NUM_FORMAT_FLOAT] offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_store_format_x v4, off, s[8:11], s3 format:[BUF_FMT_32_32_32_32_UINT] offset:8388607
+// GFX12: encoding: [0x03,0x00,0x21,0xc4,0x04,0x10,0x80,0x1e,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_x v255, off, s[8:11], s3 format:61 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x21,0xc4,0xff,0x10,0x80,0x1e,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_x v4, off, s[12:15], s3 format:[BUF_DATA_FORMAT_32_32_32_32, BUF_NUM_FORMAT_UINT] offset:8388607
+// GFX12: encoding: [0x03,0x00,0x21,0xc4,0x04,0x18,0x80,0x1e,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_x v4, off, s[12:15], s101 format:[BUF_FMT_32_32_32_32_SINT] offset:8388607
+// GFX12: encoding: [0x65,0x00,0x21,0xc4,0x04,0x18,0x00,0x1f,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_x v4, off, s[12:15], m0 format:62 offset:8388607
+// GFX12: encoding: [0x7d,0x00,0x21,0xc4,0x04,0x18,0x00,0x1f,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_x v4, off, s[8:11], s0 format:[BUF_DATA_FORMAT_32_32_32_32, BUF_NUM_FORMAT_SINT] offset:8388607
+// GFX12: encoding: [0x00,0x00,0x21,0xc4,0x04,0x10,0x00,0x1f,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_x v4, off, s[8:11], s61 format:[BUF_FMT_32_32_32_32_FLOAT] offset:8388607
+// GFX12: encoding: [0x3d,0x00,0x21,0xc4,0x04,0x10,0x80,0x1f,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_x v4, off, ttmp[4:7], s61 format:63 offset:8388607
+// GFX12: encoding: [0x3d,0x00,0x21,0xc4,0x04,0xe0,0x80,0x1f,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_x v4, v1, s[8:11], s3 format:[BUF_DATA_FORMAT_32_32_32_32, BUF_NUM_FORMAT_FLOAT] offen offset:52
+// GFX12: encoding: [0x03,0x00,0x21,0xc4,0x04,0x10,0x80,0x5f,0x01,0x34,0x00,0x00]
+
+tbuffer_store_format_x v4, v1, s[8:11], s3 format:[BUF_FMT_8_UNORM] idxen offset:52
+// GFX12: encoding: [0x03,0x00,0x21,0xc4,0x04,0x10,0x80,0x80,0x01,0x34,0x00,0x00]
+
+tbuffer_store_format_x v4, v[1:2], s[8:11], s0 format:[BUF_FMT_8_SNORM] idxen offen offset:52
+// GFX12: encoding: [0x00,0x00,0x21,0xc4,0x04,0x10,0x00,0xc1,0x01,0x34,0x00,0x00]
+
+tbuffer_store_format_x v4, off, ttmp[4:7], s3 format:[BUF_FMT_8_USCALED] offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0x00,0x21,0xc4,0x04,0xe0,0xe8,0x01,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_x v4, off, ttmp[4:7], s3 format:[BUF_FMT_8_SSCALED] offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS
+// GFX12: encoding: [0x03,0x00,0x21,0xc4,0x04,0xe0,0x3c,0x02,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_x v4, off, ttmp[4:7], s3 format:[BUF_FMT_8_UINT] offset:8388607 scope:SCOPE_SE
+// GFX12: encoding: [0x03,0x00,0x21,0xc4,0x04,0xe0,0x84,0x02,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_x v4, off, ttmp[4:7], 0 format:[BUF_FMT_8_SINT] offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+tbuffer_store_format_x v4, off, ttmp[4:7], s3 format:[BUF_FMT_8_SINT] offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_store_format_x v4, off, ttmp[4:7], s3 format:[BUF_FMT_8_SINT] offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_store_format_x v4, off, ttmp[4:7], s3 format:[BUF_FMT_8_SINT] offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_store_format_xy v[4:5], off, s[8:11], s3 format:[BUF_FMT_16_UNORM] offset:8388607
+// GFX12: encoding: [0x03,0x40,0x21,0xc4,0x04,0x10,0x80,0x03,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_xy v[254:255], off, s[8:11], s3 format:[BUF_FMT_16_SNORM] offset:8388607
+// GFX12: encoding: [0x03,0x40,0x21,0xc4,0xfe,0x10,0x00,0x04,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_xy v[4:5], off, s[12:15], s3 format:[BUF_FMT_16_USCALED] offset:8388607
+// GFX12: encoding: [0x03,0x40,0x21,0xc4,0x04,0x18,0x80,0x04,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_xy v[4:5], off, s[12:15], s101 format:[BUF_FMT_16_SSCALED] offset:8388607
+// GFX12: encoding: [0x65,0x40,0x21,0xc4,0x04,0x18,0x00,0x05,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_xy v[4:5], off, s[12:15], m0 format:[BUF_FMT_16_UINT] offset:8388607
+// GFX12: encoding: [0x7d,0x40,0x21,0xc4,0x04,0x18,0x80,0x05,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_xy v[4:5], off, s[8:11], s0 format:[BUF_FMT_16_SINT] offset:8388607
+// GFX12: encoding: [0x00,0x40,0x21,0xc4,0x04,0x10,0x00,0x06,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_xy v[4:5], off, s[8:11], s61 format:[BUF_FMT_16_FLOAT] offset:8388607
+// GFX12: encoding: [0x3d,0x40,0x21,0xc4,0x04,0x10,0x80,0x06,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_xy v[4:5], off, ttmp[4:7], s61 format:[BUF_FMT_8_8_UNORM] offset:8388607
+// GFX12: encoding: [0x3d,0x40,0x21,0xc4,0x04,0xe0,0x00,0x07,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_xy v[4:5], v1, s[8:11], s3 format:[BUF_FMT_8_8_SNORM] offen offset:52
+// GFX12: encoding: [0x03,0x40,0x21,0xc4,0x04,0x10,0x80,0x47,0x01,0x34,0x00,0x00]
+
+tbuffer_store_format_xy v[4:5], v1, s[8:11], s3 format:[BUF_FMT_8_8_USCALED] idxen offset:52
+// GFX12: encoding: [0x03,0x40,0x21,0xc4,0x04,0x10,0x00,0x88,0x01,0x34,0x00,0x00]
+
+tbuffer_store_format_xy v[4:5], v[1:2], s[8:11], s0 format:[BUF_FMT_8_8_SSCALED] idxen offen offset:52
+// GFX12: encoding: [0x00,0x40,0x21,0xc4,0x04,0x10,0x80,0xc8,0x01,0x34,0x00,0x00]
+
+tbuffer_store_format_xy v[4:5], off, ttmp[4:7], s3 format:[BUF_FMT_8_8_UINT] offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0x40,0x21,0xc4,0x04,0xe0,0x68,0x09,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_xy v[4:5], off, ttmp[4:7], s3 format:[BUF_FMT_8_8_SINT] offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS
+// GFX12: encoding: [0x03,0x40,0x21,0xc4,0x04,0xe0,0xbc,0x09,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_xy v[4:5], off, ttmp[4:7], s3 format:[BUF_FMT_32_UINT] offset:8388607 scope:SCOPE_SE
+// GFX12: encoding: [0x03,0x40,0x21,0xc4,0x04,0xe0,0x04,0x0a,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_xy v[4:5], off, ttmp[4:7], 0 format:[BUF_FMT_32_SINT] offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+tbuffer_store_format_xy v[4:5], off, ttmp[4:7], s3 format:[BUF_FMT_32_SINT] offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_store_format_xy v[4:5], off, ttmp[4:7], s3 format:[BUF_FMT_32_SINT] offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_store_format_xy v[4:5], off, ttmp[4:7], s3 format:[BUF_FMT_32_SINT] offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_store_format_xyz v[4:6], off, s[8:11], s3 format:[BUF_FMT_32_FLOAT] offset:8388607
+// GFX12: encoding: [0x03,0x80,0x21,0xc4,0x04,0x10,0x00,0x0b,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_xyz v[253:255], off, s[8:11], s3 format:[BUF_FMT_16_16_UNORM] offset:8388607
+// GFX12: encoding: [0x03,0x80,0x21,0xc4,0xfd,0x10,0x80,0x0b,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_xyz v[4:6], off, s[12:15], s3 format:[BUF_FMT_16_16_SNORM] offset:8388607
+// GFX12: encoding: [0x03,0x80,0x21,0xc4,0x04,0x18,0x00,0x0c,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_xyz v[4:6], off, s[12:15], s101 format:[BUF_FMT_16_16_USCALED] offset:8388607
+// GFX12: encoding: [0x65,0x80,0x21,0xc4,0x04,0x18,0x80,0x0c,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_xyz v[4:6], off, s[12:15], m0 format:[BUF_FMT_16_16_SSCALED] offset:8388607
+// GFX12: encoding: [0x7d,0x80,0x21,0xc4,0x04,0x18,0x00,0x0d,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_xyz v[4:6], off, s[8:11], s0 format:[BUF_FMT_16_16_UINT] offset:8388607
+// GFX12: encoding: [0x00,0x80,0x21,0xc4,0x04,0x10,0x80,0x0d,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_xyz v[4:6], off, s[8:11], s61 format:[BUF_FMT_16_16_SINT] offset:8388607
+// GFX12: encoding: [0x3d,0x80,0x21,0xc4,0x04,0x10,0x00,0x0e,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_xyz v[4:6], off, ttmp[4:7], s61 format:[BUF_FMT_16_16_FLOAT] offset:8388607
+// GFX12: encoding: [0x3d,0x80,0x21,0xc4,0x04,0xe0,0x80,0x0e,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_xyz v[4:6], v1, s[8:11], s3 format:[BUF_FMT_10_11_11_FLOAT] offen offset:52
+// GFX12: encoding: [0x03,0x80,0x21,0xc4,0x04,0x10,0x00,0x4f,0x01,0x34,0x00,0x00]
+
+tbuffer_store_format_xyz v[4:6], v1, s[8:11], s3 format:[BUF_FMT_11_11_10_FLOAT] idxen offset:52
+// GFX12: encoding: [0x03,0x80,0x21,0xc4,0x04,0x10,0x80,0x8f,0x01,0x34,0x00,0x00]
+
+tbuffer_store_format_xyz v[4:6], v[1:2], s[8:11], s0 format:[BUF_FMT_10_10_10_2_UNORM] idxen offen offset:52
+// GFX12: encoding: [0x00,0x80,0x21,0xc4,0x04,0x10,0x00,0xd0,0x01,0x34,0x00,0x00]
+
+tbuffer_store_format_xyz v[4:6], off, ttmp[4:7], s3 format:[BUF_FMT_10_10_10_2_SNORM] offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0x80,0x21,0xc4,0x04,0xe0,0xe8,0x10,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_xyz v[4:6], off, ttmp[4:7], s3 format:[BUF_FMT_10_10_10_2_UINT] offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS
+// GFX12: encoding: [0x03,0x80,0x21,0xc4,0x04,0xe0,0x3c,0x11,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_xyz v[4:6], off, ttmp[4:7], s3 format:[BUF_FMT_10_10_10_2_SINT]  offset:8388607 scope:SCOPE_SE
+// GFX12: encoding: [0x03,0x80,0x21,0xc4,0x04,0xe0,0x84,0x11,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_xyz v[4:6], off, ttmp[4:7], 0 format:[BUF_FMT_2_10_10_10_UNORM] offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+tbuffer_store_format_xyz v[4:6], off, ttmp[4:7], s3 format:[BUF_FMT_2_10_10_10_UNORM] offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_store_format_xyz v[4:6], off, ttmp[4:7], s3 format:[BUF_FMT_2_10_10_10_UNORM] offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_store_format_xyz v[4:6], off, ttmp[4:7], s3 format:[BUF_FMT_2_10_10_10_UNORM] offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_store_format_xyzw v[4:7], off, s[8:11], s3 format:[BUF_FMT_2_10_10_10_SNORM] offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x21,0xc4,0x04,0x10,0x80,0x12,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_xyzw v[252:255], off, s[8:11], s3 format:[BUF_FMT_2_10_10_10_USCALED] offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x21,0xc4,0xfc,0x10,0x00,0x13,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_xyzw v[4:7], off, s[12:15], s3 format:[BUF_FMT_2_10_10_10_SSCALED] offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x21,0xc4,0x04,0x18,0x80,0x13,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_xyzw v[4:7], off, s[12:15], s101 format:[BUF_FMT_2_10_10_10_UINT] offset:8388607
+// GFX12: encoding: [0x65,0xc0,0x21,0xc4,0x04,0x18,0x00,0x14,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_xyzw v[4:7], off, s[12:15], m0 format:[BUF_FMT_2_10_10_10_SINT] offset:8388607
+// GFX12: encoding: [0x7d,0xc0,0x21,0xc4,0x04,0x18,0x80,0x14,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_xyzw v[4:7], off, s[8:11], s0 format:[BUF_FMT_8_8_8_8_UNORM] offset:8388607
+// GFX12: encoding: [0x00,0xc0,0x21,0xc4,0x04,0x10,0x00,0x15,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_xyzw v[4:7], off, s[8:11], s61 format:[BUF_FMT_8_8_8_8_SNORM] offset:8388607
+// GFX12: encoding: [0x3d,0xc0,0x21,0xc4,0x04,0x10,0x80,0x15,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_xyzw v[4:7], off, ttmp[4:7], s61 format:[BUF_FMT_8_8_8_8_USCALED] offset:8388607
+// GFX12: encoding: [0x3d,0xc0,0x21,0xc4,0x04,0xe0,0x00,0x16,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_xyzw v[4:7], v1, s[8:11], s3 format:[BUF_FMT_8_8_8_8_SSCALED] offen offset:52
+// GFX12: encoding: [0x03,0xc0,0x21,0xc4,0x04,0x10,0x80,0x56,0x01,0x34,0x00,0x00]
+
+tbuffer_store_format_xyzw v[4:7], v1, s[8:11], s3 format:[BUF_FMT_8_8_8_8_UINT] idxen offset:52
+// GFX12: encoding: [0x03,0xc0,0x21,0xc4,0x04,0x10,0x00,0x97,0x01,0x34,0x00,0x00]
+
+tbuffer_store_format_xyzw v[4:7], v[1:2], s[8:11], s0 format:[BUF_FMT_8_8_8_8_SINT] idxen offen offset:52
+// GFX12: encoding: [0x00,0xc0,0x21,0xc4,0x04,0x10,0x80,0xd7,0x01,0x34,0x00,0x00]
+
+tbuffer_store_format_xyzw v[4:7], off, ttmp[4:7], s3 format:[BUF_FMT_32_32_UINT] offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0xc0,0x21,0xc4,0x04,0xe0,0x68,0x18,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_xyzw v[4:7], off, ttmp[4:7], s3 format:[BUF_FMT_32_32_SINT] offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS
+// GFX12: encoding: [0x03,0xc0,0x21,0xc4,0x04,0xe0,0xbc,0x18,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_xyzw v[4:7], off, ttmp[4:7], s3 format:[BUF_FMT_32_32_FLOAT] offset:8388607 scope:SCOPE_SE
+// GFX12: encoding: [0x03,0xc0,0x21,0xc4,0x04,0xe0,0x04,0x19,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_xyzw v[4:7], off, ttmp[4:7], 0 format:[BUF_FMT_16_16_16_16_UNORM] offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+tbuffer_store_format_xyzw v[4:7], off, ttmp[4:7], s3 format:[BUF_FMT_16_16_16_16_UNORM] offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_store_format_xyzw v[4:7], off, ttmp[4:7], s3 format:[BUF_FMT_16_16_16_16_UNORM] offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_store_format_xyzw v[4:7], off, ttmp[4:7], s3 format:[BUF_FMT_16_16_16_16_UNORM] offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+//Removed formats (compared to gfx10)
+
+tbuffer_load_format_x v4, off, s[8:11], s3 format:[BUF_FMT_10_11_11_UNORM] offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: unsupported format
+
+tbuffer_load_format_x v4, off, s[8:11], s3 format:[BUF_FMT_10_11_11_SNORM] offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: unsupported format
+
+tbuffer_load_format_x v4, off, s[8:11], s3 format:[BUF_FMT_10_11_11_USCALED] offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: unsupported format
+
+tbuffer_load_format_x v4, off, s[8:11], s3 format:[BUF_FMT_10_11_11_SSCALED] offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: unsupported format
+
+tbuffer_load_format_x v4, off, s[8:11], s3 format:[BUF_FMT_10_11_11_UINT] offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: unsupported format
+
+tbuffer_load_format_x v4, off, s[8:11], s3 format:[BUF_FMT_10_11_11_SINT] offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: unsupported format
+
+tbuffer_load_format_x v4, off, s[8:11], s3 format:[BUF_FMT_11_11_10_UNORM] offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: unsupported format
+
+tbuffer_load_format_x v4, off, s[8:11], s3 format:[BUF_FMT_11_11_10_SNORM] offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: unsupported format
+
+tbuffer_load_format_x v4, off, s[8:11], s3 format:[BUF_FMT_11_11_10_USCALED] offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: unsupported format
+
+tbuffer_load_format_x v4, off, s[8:11], s3 format:[BUF_FMT_11_11_10_SSCALED] offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: unsupported format
+
+tbuffer_load_format_x v4, off, s[8:11], s3 format:[BUF_FMT_11_11_10_UINT] offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: unsupported format
+
+tbuffer_load_format_x v4, off, s[8:11], s3 format:[BUF_FMT_10_10_10_2_USCALED] offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: unsupported format
+
+tbuffer_load_format_x v4, off, s[8:11], s3 format:[BUF_FMT_10_10_10_2_SSCALED] offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: unsupported format
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vbuffer_mtbuf_alias.s b/llvm/test/MC/AMDGPU/gfx12_asm_vbuffer_mtbuf_alias.s
new file mode 100644
index 0000000000000000000000000000000000000000..15d6269b6f5956804e977771ef8d2af1a3ac7956
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vbuffer_mtbuf_alias.s
@@ -0,0 +1,434 @@
+// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -show-encoding %s | FileCheck --check-prefix=GFX12 %s
+// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -show-encoding %s 2>&1 | FileCheck --check-prefixes=GFX12-ERR --implicit-check-not=error: %s
+
+tbuffer_load_format_d16_x v4, off, s[8:11], s3 format:[BUF_FMT_8_UNORM] offset:8388607
+// GFX12: encoding: [0x03,0x00,0x22,0xc4,0x04,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_d16_x v255, off, s[8:11], s3 format:1 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x22,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_d16_x v4, off, s[12:15], s3 format:[BUF_DATA_FORMAT_8, BUF_NUM_FORMAT_UNORM] offset:8388607
+// GFX12: encoding: [0x03,0x00,0x22,0xc4,0x04,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_d16_x v4, off, s[12:15], s101 format:[BUF_FMT_8_SNORM] offset:8388607
+// GFX12: encoding: [0x65,0x00,0x22,0xc4,0x04,0x18,0x00,0x01,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_d16_x v4, off, s[12:15], m0 format:2 offset:8388607
+// GFX12: encoding: [0x7d,0x00,0x22,0xc4,0x04,0x18,0x00,0x01,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_d16_x v4, off, s[8:11], s0 format:[BUF_DATA_FORMAT_8, BUF_NUM_FORMAT_SNORM] offset:8388607
+// GFX12: encoding: [0x00,0x00,0x22,0xc4,0x04,0x10,0x00,0x01,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_d16_x v4, off, s[8:11], s61 format:[BUF_FMT_8_USCALED] offset:8388607
+// GFX12: encoding: [0x3d,0x00,0x22,0xc4,0x04,0x10,0x80,0x01,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_d16_x v4, off, ttmp[4:7], s61 format:3 offset:8388607
+// GFX12: encoding: [0x3d,0x00,0x22,0xc4,0x04,0xe0,0x80,0x01,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_d16_x v4, v1, s[8:11], s3 format:[BUF_DATA_FORMAT_8, BUF_NUM_FORMAT_USCALED] offen offset:52
+// GFX12: encoding: [0x03,0x00,0x22,0xc4,0x04,0x10,0x80,0x41,0x01,0x34,0x00,0x00]
+
+tbuffer_load_format_d16_x v4, v1, s[8:11], s3 format:[BUF_FMT_8_SSCALED] idxen offset:52
+// GFX12: encoding: [0x03,0x00,0x22,0xc4,0x04,0x10,0x00,0x82,0x01,0x34,0x00,0x00]
+
+tbuffer_load_format_d16_x v4, v[1:2], s[8:11], s0 format:4 idxen offen offset:52
+// GFX12: encoding: [0x00,0x00,0x22,0xc4,0x04,0x10,0x00,0xc2,0x01,0x34,0x00,0x00]
+
+tbuffer_load_format_d16_x v4, off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_8, BUF_NUM_FORMAT_SSCALED] offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0x00,0x22,0xc4,0x04,0xe0,0x68,0x02,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_d16_x v4, off, ttmp[4:7], s3 format:[BUF_FMT_8_UINT] offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS
+// GFX12: encoding: [0x03,0x00,0x22,0xc4,0x04,0xe0,0xbc,0x02,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_d16_x v4, off, ttmp[4:7], s3 format:5 offset:8388607 scope:SCOPE_SE
+// GFX12: encoding: [0x03,0x00,0x22,0xc4,0x04,0xe0,0x84,0x02,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_d16_x v4, off, ttmp[4:7], 0 format:[BUF_DATA_FORMAT_8, BUF_NUM_FORMAT_UINT] offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+tbuffer_load_format_d16_x v4, off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_8, BUF_NUM_FORMAT_UINT] offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_load_format_d16_x v4, off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_8, BUF_NUM_FORMAT_UINT] offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_load_format_d16_x v4, off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_8, BUF_NUM_FORMAT_UINT] offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_load_format_d16_xy v4, off, s[8:11], s3 format:[BUF_FMT_8_SINT] offset:8388607
+// GFX12: encoding: [0x03,0x40,0x22,0xc4,0x04,0x10,0x00,0x03,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_d16_xy v255, off, s[8:11], s3 format:6 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x22,0xc4,0xff,0x10,0x00,0x03,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_d16_xy v4, off, s[12:15], s3 format:[BUF_DATA_FORMAT_8, BUF_NUM_FORMAT_SINT] offset:8388607
+// GFX12: encoding: [0x03,0x40,0x22,0xc4,0x04,0x18,0x00,0x03,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_d16_xy v4, off, s[12:15], s101 format:[BUF_FMT_16_UNORM] offset:8388607
+// GFX12: encoding: [0x65,0x40,0x22,0xc4,0x04,0x18,0x80,0x03,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_d16_xy v4, off, s[12:15], m0 format:7 offset:8388607
+// GFX12: encoding: [0x7d,0x40,0x22,0xc4,0x04,0x18,0x80,0x03,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_d16_xy v4, off, s[8:11], s0 format:[BUF_DATA_FORMAT_16, BUF_NUM_FORMAT_UNORM] offset:8388607
+// GFX12: encoding: [0x00,0x40,0x22,0xc4,0x04,0x10,0x80,0x03,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_d16_xy v4, off, s[8:11], s61 format:[BUF_FMT_16_SNORM] offset:8388607
+// GFX12: encoding: [0x3d,0x40,0x22,0xc4,0x04,0x10,0x00,0x04,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_d16_xy v4, off, ttmp[4:7], s61 format:8 offset:8388607
+// GFX12: encoding: [0x3d,0x40,0x22,0xc4,0x04,0xe0,0x00,0x04,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_d16_xy v4, v1, s[8:11], s3 format:[BUF_DATA_FORMAT_16, BUF_NUM_FORMAT_SNORM] offen offset:52
+// GFX12: encoding: [0x03,0x40,0x22,0xc4,0x04,0x10,0x00,0x44,0x01,0x34,0x00,0x00]
+
+tbuffer_load_format_d16_xy v4, v1, s[8:11], s3 format:[BUF_FMT_16_USCALED] idxen offset:52
+// GFX12: encoding: [0x03,0x40,0x22,0xc4,0x04,0x10,0x80,0x84,0x01,0x34,0x00,0x00]
+
+tbuffer_load_format_d16_xy v4, v[1:2], s[8:11], s0 format:9 idxen offen offset:52
+// GFX12: encoding: [0x00,0x40,0x22,0xc4,0x04,0x10,0x80,0xc4,0x01,0x34,0x00,0x00]
+
+tbuffer_load_format_d16_xy v4, off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_16, BUF_NUM_FORMAT_USCALED] offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0x40,0x22,0xc4,0x04,0xe0,0xe8,0x04,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_d16_xy v4, off, ttmp[4:7], s3 format:[BUF_FMT_16_SSCALED] offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS
+// GFX12: encoding: [0x03,0x40,0x22,0xc4,0x04,0xe0,0x3c,0x05,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_d16_xy v4, off, ttmp[4:7], s3 format:10 offset:8388607 scope:SCOPE_SE
+// GFX12: encoding: [0x03,0x40,0x22,0xc4,0x04,0xe0,0x04,0x05,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_d16_xy v4, off, ttmp[4:7], 0 format:[BUF_DATA_FORMAT_16, BUF_NUM_FORMAT_SSCALED] offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+tbuffer_load_format_d16_xy v4, off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_16, BUF_NUM_FORMAT_SSCALED] offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_load_format_d16_xy v4, off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_16, BUF_NUM_FORMAT_SSCALED] offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_load_format_d16_xy v4, off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_16, BUF_NUM_FORMAT_SSCALED] offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_load_format_d16_xyz v[4:5], off, s[8:11], s3 format:[BUF_FMT_16_UINT] offset:8388607
+// GFX12: encoding: [0x03,0x80,0x22,0xc4,0x04,0x10,0x80,0x05,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_d16_xyz v[254:255], off, s[8:11], s3 format:11 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x22,0xc4,0xfe,0x10,0x80,0x05,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_d16_xyz v[4:5], off, s[12:15], s3 format:[BUF_DATA_FORMAT_16, BUF_NUM_FORMAT_UINT] offset:8388607
+// GFX12: encoding: [0x03,0x80,0x22,0xc4,0x04,0x18,0x80,0x05,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_d16_xyz v[4:5], off, s[12:15], s101 format:[BUF_FMT_16_SINT] offset:8388607
+// GFX12: encoding: [0x65,0x80,0x22,0xc4,0x04,0x18,0x00,0x06,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_d16_xyz v[4:5], off, s[12:15], m0 format:12 offset:8388607
+// GFX12: encoding: [0x7d,0x80,0x22,0xc4,0x04,0x18,0x00,0x06,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_d16_xyz v[4:5], off, s[8:11], s0 format:[BUF_DATA_FORMAT_16, BUF_NUM_FORMAT_SINT] offset:8388607
+// GFX12: encoding: [0x00,0x80,0x22,0xc4,0x04,0x10,0x00,0x06,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_d16_xyz v[4:5], off, s[8:11], s61 format:[BUF_FMT_16_FLOAT] offset:8388607
+// GFX12: encoding: [0x3d,0x80,0x22,0xc4,0x04,0x10,0x80,0x06,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_d16_xyz v[4:5], off, ttmp[4:7], s61 format:13 offset:8388607
+// GFX12: encoding: [0x3d,0x80,0x22,0xc4,0x04,0xe0,0x80,0x06,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_d16_xyz v[4:5], v1, s[8:11], s3 format:[BUF_DATA_FORMAT_16, BUF_NUM_FORMAT_FLOAT] offen offset:52
+// GFX12: encoding: [0x03,0x80,0x22,0xc4,0x04,0x10,0x80,0x46,0x01,0x34,0x00,0x00]
+
+tbuffer_load_format_d16_xyz v[4:5], v1, s[8:11], s3 format:[BUF_FMT_8_8_UNORM] idxen offset:52
+// GFX12: encoding: [0x03,0x80,0x22,0xc4,0x04,0x10,0x00,0x87,0x01,0x34,0x00,0x00]
+
+tbuffer_load_format_d16_xyz v[4:5], v[1:2], s[8:11], s0 format:14 idxen offen offset:52
+// GFX12: encoding: [0x00,0x80,0x22,0xc4,0x04,0x10,0x00,0xc7,0x01,0x34,0x00,0x00]
+
+tbuffer_load_format_d16_xyz v[4:5], off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_8_8, BUF_NUM_FORMAT_UNORM] offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0x80,0x22,0xc4,0x04,0xe0,0x68,0x07,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_d16_xyz v[4:5], off, ttmp[4:7], s3 format:[BUF_FMT_8_8_SNORM] offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS
+// GFX12: encoding: [0x03,0x80,0x22,0xc4,0x04,0xe0,0xbc,0x07,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_d16_xyz v[4:5], off, ttmp[4:7], s3 format:15 offset:8388607 scope:SCOPE_SE
+// GFX12: encoding: [0x03,0x80,0x22,0xc4,0x04,0xe0,0x84,0x07,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_d16_xyz v[4:5], off, ttmp[4:7], 0 format:[BUF_DATA_FORMAT_8_8, BUF_NUM_FORMAT_SNORM] offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+tbuffer_load_format_d16_xyz v[4:5], off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_8_8, BUF_NUM_FORMAT_SNORM] offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_load_format_d16_xyz v[4:5], off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_8_8, BUF_NUM_FORMAT_SNORM] offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_load_format_d16_xyz v[4:5], off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_8_8, BUF_NUM_FORMAT_SNORM] offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_load_format_d16_xyzw v[4:5], off, s[8:11], s3 format:[BUF_FMT_8_8_USCALED] offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x22,0xc4,0x04,0x10,0x00,0x08,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_d16_xyzw v[254:255], off, s[8:11], s3 format:16 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x22,0xc4,0xfe,0x10,0x00,0x08,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_d16_xyzw v[4:5], off, s[12:15], s3 format:[BUF_DATA_FORMAT_8_8, BUF_NUM_FORMAT_USCALED] offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x22,0xc4,0x04,0x18,0x00,0x08,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_d16_xyzw v[4:5], off, s[12:15], s101 format:[BUF_FMT_8_8_SSCALED] offset:8388607
+// GFX12: encoding: [0x65,0xc0,0x22,0xc4,0x04,0x18,0x80,0x08,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_d16_xyzw v[4:5], off, s[12:15], m0 format:17 offset:8388607
+// GFX12: encoding: [0x7d,0xc0,0x22,0xc4,0x04,0x18,0x80,0x08,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_d16_xyzw v[4:5], off, s[8:11], s0 format:[BUF_DATA_FORMAT_8_8, BUF_NUM_FORMAT_SSCALED] offset:8388607
+// GFX12: encoding: [0x00,0xc0,0x22,0xc4,0x04,0x10,0x80,0x08,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_d16_xyzw v[4:5], off, s[8:11], s61 format:[BUF_FMT_8_8_UINT] offset:8388607
+// GFX12: encoding: [0x3d,0xc0,0x22,0xc4,0x04,0x10,0x00,0x09,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_d16_xyzw v[4:5], off, ttmp[4:7], s61 format:18 offset:8388607
+// GFX12: encoding: [0x3d,0xc0,0x22,0xc4,0x04,0xe0,0x00,0x09,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_d16_xyzw v[4:5], v1, s[8:11], s3 format:[BUF_DATA_FORMAT_8_8, BUF_NUM_FORMAT_UINT] offen offset:52
+// GFX12: encoding: [0x03,0xc0,0x22,0xc4,0x04,0x10,0x00,0x49,0x01,0x34,0x00,0x00]
+
+tbuffer_load_format_d16_xyzw v[4:5], v1, s[8:11], s3 format:[BUF_FMT_8_8_SINT] idxen offset:52
+// GFX12: encoding: [0x03,0xc0,0x22,0xc4,0x04,0x10,0x80,0x89,0x01,0x34,0x00,0x00]
+
+tbuffer_load_format_d16_xyzw v[4:5], v[1:2], s[8:11], s0 format:19 idxen offen offset:52
+// GFX12: encoding: [0x00,0xc0,0x22,0xc4,0x04,0x10,0x80,0xc9,0x01,0x34,0x00,0x00]
+
+tbuffer_load_format_d16_xyzw v[4:5], off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_8_8, BUF_NUM_FORMAT_SINT] offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0xc0,0x22,0xc4,0x04,0xe0,0xe8,0x09,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_d16_xyzw v[4:5], off, ttmp[4:7], s3 format:[BUF_FMT_32_UINT] offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS
+// GFX12: encoding: [0x03,0xc0,0x22,0xc4,0x04,0xe0,0x3c,0x0a,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_d16_xyzw v[4:5], off, ttmp[4:7], s3 format:20 offset:8388607 scope:SCOPE_SE
+// GFX12: encoding: [0x03,0xc0,0x22,0xc4,0x04,0xe0,0x04,0x0a,0x00,0xff,0xff,0x7f]
+
+tbuffer_load_format_d16_xyzw v[4:5], off, ttmp[4:7], 0 format:[BUF_DATA_FORMAT_32, BUF_NUM_FORMAT_UINT] offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+tbuffer_load_format_d16_xyzw v[4:5], off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_32, BUF_NUM_FORMAT_UINT] offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_load_format_d16_xyzw v[4:5], off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_32, BUF_NUM_FORMAT_UINT] offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_load_format_d16_xyzw v[4:5], off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_32, BUF_NUM_FORMAT_UINT] offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_store_format_d16_x v4, off, s[8:11], s3 format:[BUF_FMT_2_10_10_10_SINT] offset:8388607
+// GFX12: encoding: [0x03,0x00,0x23,0xc4,0x04,0x10,0x80,0x14,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_d16_x v255, off, s[8:11], s3 format:41 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x23,0xc4,0xff,0x10,0x80,0x14,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_d16_x v4, off, s[12:15], s3 format:[BUF_DATA_FORMAT_2_10_10_10, BUF_NUM_FORMAT_SINT] offset:8388607
+// GFX12: encoding: [0x03,0x00,0x23,0xc4,0x04,0x18,0x80,0x14,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_d16_x v4, off, s[12:15], s101 format:[BUF_FMT_8_8_8_8_UNORM] offset:8388607
+// GFX12: encoding: [0x65,0x00,0x23,0xc4,0x04,0x18,0x00,0x15,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_d16_x v4, off, s[12:15], m0 format:42 offset:8388607
+// GFX12: encoding: [0x7d,0x00,0x23,0xc4,0x04,0x18,0x00,0x15,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_d16_x v4, off, s[8:11], s0 format:[BUF_DATA_FORMAT_8_8_8_8, BUF_NUM_FORMAT_UNORM] offset:8388607
+// GFX12: encoding: [0x00,0x00,0x23,0xc4,0x04,0x10,0x00,0x15,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_d16_x v4, off, s[8:11], s61 format:[BUF_FMT_8_8_8_8_SNORM] offset:8388607
+// GFX12: encoding: [0x3d,0x00,0x23,0xc4,0x04,0x10,0x80,0x15,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_d16_x v4, off, ttmp[4:7], s61 format:43 offset:8388607
+// GFX12: encoding: [0x3d,0x00,0x23,0xc4,0x04,0xe0,0x80,0x15,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_d16_x v4, v1, s[8:11], s3 format:[BUF_DATA_FORMAT_8_8_8_8, BUF_NUM_FORMAT_SNORM] offen offset:52
+// GFX12: encoding: [0x03,0x00,0x23,0xc4,0x04,0x10,0x80,0x55,0x01,0x34,0x00,0x00]
+
+tbuffer_store_format_d16_x v4, v1, s[8:11], s3 format:[BUF_FMT_8_8_8_8_USCALED] idxen offset:52
+// GFX12: encoding: [0x03,0x00,0x23,0xc4,0x04,0x10,0x00,0x96,0x01,0x34,0x00,0x00]
+
+tbuffer_store_format_d16_x v4, v[1:2], s[8:11], s0 format:44 idxen offen offset:52
+// GFX12: encoding: [0x00,0x00,0x23,0xc4,0x04,0x10,0x00,0xd6,0x01,0x34,0x00,0x00]
+
+tbuffer_store_format_d16_x v4, off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_8_8_8_8, BUF_NUM_FORMAT_USCALED] offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0x00,0x23,0xc4,0x04,0xe0,0x68,0x16,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_d16_x v4, off, ttmp[4:7], s3 format:[BUF_FMT_8_8_8_8_SSCALED] offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS
+// GFX12: encoding: [0x03,0x00,0x23,0xc4,0x04,0xe0,0xbc,0x16,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_d16_x v4, off, ttmp[4:7], s3 format:45 offset:8388607 scope:SCOPE_SE
+// GFX12: encoding: [0x03,0x00,0x23,0xc4,0x04,0xe0,0x84,0x16,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_d16_x v4, off, ttmp[4:7], 0 format:[BUF_DATA_FORMAT_8_8_8_8, BUF_NUM_FORMAT_SSCALED] offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+tbuffer_store_format_d16_x v4, off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_8_8_8_8, BUF_NUM_FORMAT_SSCALED] offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_store_format_d16_x v4, off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_8_8_8_8, BUF_NUM_FORMAT_SSCALED] offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_store_format_d16_x v4, off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_8_8_8_8, BUF_NUM_FORMAT_SSCALED] offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_store_format_d16_xy v4, off, s[8:11], s3 format:[BUF_FMT_8_8_8_8_UINT] offset:8388607
+// GFX12: encoding: [0x03,0x40,0x23,0xc4,0x04,0x10,0x00,0x17,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_d16_xy v255, off, s[8:11], s3 format:46 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x23,0xc4,0xff,0x10,0x00,0x17,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_d16_xy v4, off, s[12:15], s3 format:[BUF_DATA_FORMAT_8_8_8_8, BUF_NUM_FORMAT_UINT] offset:8388607
+// GFX12: encoding: [0x03,0x40,0x23,0xc4,0x04,0x18,0x00,0x17,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_d16_xy v4, off, s[12:15], s101 format:[BUF_FMT_8_8_8_8_SINT] offset:8388607
+// GFX12: encoding: [0x65,0x40,0x23,0xc4,0x04,0x18,0x80,0x17,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_d16_xy v4, off, s[12:15], m0 format:47 offset:8388607
+// GFX12: encoding: [0x7d,0x40,0x23,0xc4,0x04,0x18,0x80,0x17,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_d16_xy v4, off, s[8:11], s0 format:[BUF_DATA_FORMAT_8_8_8_8, BUF_NUM_FORMAT_SINT] offset:8388607
+// GFX12: encoding: [0x00,0x40,0x23,0xc4,0x04,0x10,0x80,0x17,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_d16_xy v4, off, s[8:11], s61 format:[BUF_FMT_32_32_UINT] offset:8388607
+// GFX12: encoding: [0x3d,0x40,0x23,0xc4,0x04,0x10,0x00,0x18,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_d16_xy v4, off, ttmp[4:7], s61 format:48 offset:8388607
+// GFX12: encoding: [0x3d,0x40,0x23,0xc4,0x04,0xe0,0x00,0x18,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_d16_xy v4, v1, s[8:11], s3 format:[BUF_DATA_FORMAT_32_32, BUF_NUM_FORMAT_UINT] offen offset:52
+// GFX12: encoding: [0x03,0x40,0x23,0xc4,0x04,0x10,0x00,0x58,0x01,0x34,0x00,0x00]
+
+tbuffer_store_format_d16_xy v4, v1, s[8:11], s3 format:[BUF_FMT_32_32_SINT] idxen offset:52
+// GFX12: encoding: [0x03,0x40,0x23,0xc4,0x04,0x10,0x80,0x98,0x01,0x34,0x00,0x00]
+
+tbuffer_store_format_d16_xy v4, v[1:2], s[8:11], s0 format:49 idxen offen offset:52
+// GFX12: encoding: [0x00,0x40,0x23,0xc4,0x04,0x10,0x80,0xd8,0x01,0x34,0x00,0x00]
+
+tbuffer_store_format_d16_xy v4, off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_32_32, BUF_NUM_FORMAT_SINT] offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0x40,0x23,0xc4,0x04,0xe0,0xe8,0x18,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_d16_xy v4, off, ttmp[4:7], s3 format:[BUF_FMT_32_32_FLOAT] offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS
+// GFX12: encoding: [0x03,0x40,0x23,0xc4,0x04,0xe0,0x3c,0x19,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_d16_xy v4, off, ttmp[4:7], s3 format:50 offset:8388607 scope:SCOPE_SE
+// GFX12: encoding: [0x03,0x40,0x23,0xc4,0x04,0xe0,0x04,0x19,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_d16_xy v4, off, ttmp[4:7], 0 format:[BUF_DATA_FORMAT_32_32, BUF_NUM_FORMAT_FLOAT] offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+tbuffer_store_format_d16_xy v4, off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_32_32, BUF_NUM_FORMAT_FLOAT] offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_store_format_d16_xy v4, off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_32_32, BUF_NUM_FORMAT_FLOAT] offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_store_format_d16_xy v4, off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_32_32, BUF_NUM_FORMAT_FLOAT] offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_store_format_d16_xyz v[4:5], off, s[8:11], s3 format:[BUF_FMT_16_16_16_16_UNORM] offset:8388607
+// GFX12: encoding: [0x03,0x80,0x23,0xc4,0x04,0x10,0x80,0x19,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_d16_xyz v[254:255], off, s[8:11], s3 format:51 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x23,0xc4,0xfe,0x10,0x80,0x19,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_d16_xyz v[4:5], off, s[12:15], s3 format:[BUF_DATA_FORMAT_16_16_16_16, BUF_NUM_FORMAT_UNORM] offset:8388607
+// GFX12: encoding: [0x03,0x80,0x23,0xc4,0x04,0x18,0x80,0x19,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_d16_xyz v[4:5], off, s[12:15], s101 format:[BUF_FMT_16_16_16_16_SNORM] offset:8388607
+// GFX12: encoding: [0x65,0x80,0x23,0xc4,0x04,0x18,0x00,0x1a,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_d16_xyz v[4:5], off, s[12:15], m0 format:52 offset:8388607
+// GFX12: encoding: [0x7d,0x80,0x23,0xc4,0x04,0x18,0x00,0x1a,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_d16_xyz v[4:5], off, s[8:11], s0 format:[BUF_DATA_FORMAT_16_16_16_16, BUF_NUM_FORMAT_SNORM] offset:8388607
+// GFX12: encoding: [0x00,0x80,0x23,0xc4,0x04,0x10,0x00,0x1a,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_d16_xyz v[4:5], off, s[8:11], s61 format:[BUF_FMT_16_16_16_16_USCALED] offset:8388607
+// GFX12: encoding: [0x3d,0x80,0x23,0xc4,0x04,0x10,0x80,0x1a,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_d16_xyz v[4:5], off, ttmp[4:7], s61 format:53 offset:8388607
+// GFX12: encoding: [0x3d,0x80,0x23,0xc4,0x04,0xe0,0x80,0x1a,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_d16_xyz v[4:5], v1, s[8:11], s3 format:[BUF_DATA_FORMAT_16_16_16_16, BUF_NUM_FORMAT_USCALED] offen offset:52
+// GFX12: encoding: [0x03,0x80,0x23,0xc4,0x04,0x10,0x80,0x5a,0x01,0x34,0x00,0x00]
+
+tbuffer_store_format_d16_xyz v[4:5], v1, s[8:11], s3 format:[BUF_FMT_16_16_16_16_SSCALED] idxen offset:52
+// GFX12: encoding: [0x03,0x80,0x23,0xc4,0x04,0x10,0x00,0x9b,0x01,0x34,0x00,0x00]
+
+tbuffer_store_format_d16_xyz v[4:5], v[1:2], s[8:11], s0 format:54 idxen offen offset:52
+// GFX12: encoding: [0x00,0x80,0x23,0xc4,0x04,0x10,0x00,0xdb,0x01,0x34,0x00,0x00]
+
+tbuffer_store_format_d16_xyz v[4:5], off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_16_16_16_16, BUF_NUM_FORMAT_SSCALED] offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0x80,0x23,0xc4,0x04,0xe0,0x68,0x1b,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_d16_xyz v[4:5], off, ttmp[4:7], s3 format:[BUF_FMT_16_16_16_16_UINT] offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS
+// GFX12: encoding: [0x03,0x80,0x23,0xc4,0x04,0xe0,0xbc,0x1b,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_d16_xyz v[4:5], off, ttmp[4:7], s3 format:55 offset:8388607 scope:SCOPE_SE
+// GFX12: encoding: [0x03,0x80,0x23,0xc4,0x04,0xe0,0x84,0x1b,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_d16_xyz v[4:5], off, ttmp[4:7], 0 format:[BUF_DATA_FORMAT_16_16_16_16, BUF_NUM_FORMAT_UINT] offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+tbuffer_store_format_d16_xyz v[4:5], off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_16_16_16_16, BUF_NUM_FORMAT_UINT] offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_store_format_d16_xyz v[4:5], off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_16_16_16_16, BUF_NUM_FORMAT_UINT] offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_store_format_d16_xyz v[4:5], off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_16_16_16_16, BUF_NUM_FORMAT_UINT] offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_store_format_d16_xyzw v[4:5], off, s[8:11], s3 format:[BUF_FMT_16_16_16_16_SINT] offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x23,0xc4,0x04,0x10,0x00,0x1c,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_d16_xyzw v[254:255], off, s[8:11], s3 format:56 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x23,0xc4,0xfe,0x10,0x00,0x1c,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_d16_xyzw v[4:5], off, s[12:15], s3 format:[BUF_DATA_FORMAT_16_16_16_16, BUF_NUM_FORMAT_SINT] offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x23,0xc4,0x04,0x18,0x00,0x1c,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_d16_xyzw v[4:5], off, s[12:15], s101 format:[BUF_FMT_16_16_16_16_FLOAT] offset:8388607
+// GFX12: encoding: [0x65,0xc0,0x23,0xc4,0x04,0x18,0x80,0x1c,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_d16_xyzw v[4:5], off, s[12:15], m0 format:57 offset:8388607
+// GFX12: encoding: [0x7d,0xc0,0x23,0xc4,0x04,0x18,0x80,0x1c,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_d16_xyzw v[4:5], off, s[8:11], s0 format:[BUF_DATA_FORMAT_16_16_16_16, BUF_NUM_FORMAT_FLOAT] offset:8388607
+// GFX12: encoding: [0x00,0xc0,0x23,0xc4,0x04,0x10,0x80,0x1c,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_d16_xyzw v[4:5], off, s[8:11], s61 format:[BUF_FMT_32_32_32_UINT] offset:8388607
+// GFX12: encoding: [0x3d,0xc0,0x23,0xc4,0x04,0x10,0x00,0x1d,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_d16_xyzw v[4:5], off, ttmp[4:7], s61 format:58 offset:8388607
+// GFX12: encoding: [0x3d,0xc0,0x23,0xc4,0x04,0xe0,0x00,0x1d,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_d16_xyzw v[4:5], v1, s[8:11], s3 format:[BUF_DATA_FORMAT_32_32_32, BUF_NUM_FORMAT_UINT] offen offset:52
+// GFX12: encoding: [0x03,0xc0,0x23,0xc4,0x04,0x10,0x00,0x5d,0x01,0x34,0x00,0x00]
+
+tbuffer_store_format_d16_xyzw v[4:5], v1, s[8:11], s3 format:[BUF_FMT_32_32_32_SINT] idxen offset:52
+// GFX12: encoding: [0x03,0xc0,0x23,0xc4,0x04,0x10,0x80,0x9d,0x01,0x34,0x00,0x00]
+
+tbuffer_store_format_d16_xyzw v[4:5], v[1:2], s[8:11], s0 format:59 idxen offen offset:52
+// GFX12: encoding: [0x00,0xc0,0x23,0xc4,0x04,0x10,0x80,0xdd,0x01,0x34,0x00,0x00]
+
+tbuffer_store_format_d16_xyzw v[4:5], off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_32_32_32, BUF_NUM_FORMAT_SINT] offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0xc0,0x23,0xc4,0x04,0xe0,0xe8,0x1d,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_d16_xyzw v[4:5], off, ttmp[4:7], s3 format:[BUF_FMT_32_32_32_FLOAT] offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS
+// GFX12: encoding: [0x03,0xc0,0x23,0xc4,0x04,0xe0,0x3c,0x1e,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_d16_xyzw v[4:5], off, ttmp[4:7], s3 format:60 offset:8388607 scope:SCOPE_SE
+// GFX12: encoding: [0x03,0xc0,0x23,0xc4,0x04,0xe0,0x04,0x1e,0x00,0xff,0xff,0x7f]
+
+tbuffer_store_format_d16_xyzw v[4:5], off, ttmp[4:7], 0 format:[BUF_DATA_FORMAT_32_32_32, BUF_NUM_FORMAT_FLOAT] offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+tbuffer_store_format_d16_xyzw v[4:5], off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_32_32_32, BUF_NUM_FORMAT_FLOAT] offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_store_format_d16_xyzw v[4:5], off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_32_32_32, BUF_NUM_FORMAT_FLOAT] offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+tbuffer_store_format_d16_xyzw v[4:5], off, ttmp[4:7], s3 format:[BUF_DATA_FORMAT_32_32_32, BUF_NUM_FORMAT_FLOAT] offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vbuffer_mubuf.s b/llvm/test/MC/AMDGPU/gfx12_asm_vbuffer_mubuf.s
new file mode 100644
index 0000000000000000000000000000000000000000..a7a256cfd2b8fef3036b9b67c34acaf91bedcd05
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vbuffer_mubuf.s
@@ -0,0 +1,4631 @@
+// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -show-encoding %s | FileCheck --check-prefix=GFX12 %s
+// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -show-encoding %s 2>&1 | FileCheck --check-prefixes=GFX12-ERR --implicit-check-not=error: %s
+
+buffer_load_b32 v5, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b32 v255, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x05,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b32 v5, off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x05,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b32 v5, off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x05,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b32 v5, off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x00,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b32 v5, off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x00,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b32 v5, off, s[8:11], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_b32 v5, off, s[8:11], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_b32 v5, off, s[8:11], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_b32 v5, off, s[8:11], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_b32 v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x00,0x05,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_load_b32 v5, v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x00,0x05,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_load_b32 v5, off, s[8:11], s3
+// GFX12: encoding: [0x03,0x00,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_b32 v5, off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x00,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_b32 v5, off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x00,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_load_b32 v5, off, s[8:11], s3 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_b32 v5, off, s[8:11], s3 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_b32 v5, off, s[8:11], s3 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_b32 v5, off, s[8:11], s3 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0x00,0x05,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS
+// GFX12: encoding: [0x03,0x00,0x05,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b32 v5, off, s[8:11], s3 offset:8388607 lds
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_b64 v[5:6], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b64 v[254:255], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x05,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b64 v[5:6], off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x05,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b64 v[5:6], off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x05,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b64 v[5:6], off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x40,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b64 v[5:6], off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x40,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b64 v[5:6], off, s[8:11], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_b64 v[5:6], off, s[8:11], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_b64 v[5:6], off, s[8:11], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_b64 v[5:6], off, s[8:11], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_b64 v[5:6], v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x40,0x05,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_load_b64 v[5:6], v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x40,0x05,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_load_b64 v[5:6], off, s[8:11], s3
+// GFX12: encoding: [0x03,0x40,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_b64 v[5:6], off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x40,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_b64 v[5:6], off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x40,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_load_b64 v[5:6], off, s[8:11], s3 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_b64 v[5:6], off, s[8:11], s3 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_b64 v[5:6], off, s[8:11], s3 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_b64 v[5:6], off, s[8:11], s3 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_b64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0x40,0x05,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS
+// GFX12: encoding: [0x03,0x40,0x05,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b96 v[5:7], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b96 v[253:255], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x05,0xc4,0xfd,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b96 v[5:7], off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x05,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b96 v[5:7], off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x05,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b96 v[5:7], off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x80,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b96 v[5:7], off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x80,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b96 v[5:7], off, s[8:11], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_b96 v[5:7], off, s[8:11], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_b96 v[5:7], off, s[8:11], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_b96 v[5:7], off, s[8:11], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_b96 v[5:7], v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x80,0x05,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_load_b96 v[5:7], v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x80,0x05,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_load_b96 v[5:7], off, s[8:11], s3
+// GFX12: encoding: [0x03,0x80,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_b96 v[5:7], off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x80,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_b96 v[5:7], off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x80,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_load_b96 v[5:7], off, s[8:11], s3 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_b96 v[5:7], off, s[8:11], s3 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_b96 v[5:7], off, s[8:11], s3 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_b96 v[5:7], off, s[8:11], s3 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_b96 v[5:7], off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0x80,0x05,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b96 v[5:7], off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS
+// GFX12: encoding: [0x03,0x80,0x05,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b128 v[5:8], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b128 v[252:255], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x05,0xc4,0xfc,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b128 v[5:8], off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x05,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b128 v[5:8], off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x05,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b128 v[5:8], off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0xc0,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b128 v[5:8], off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0xc0,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b128 v[5:8], off, s[8:11], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_b128 v[5:8], off, s[8:11], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_b128 v[5:8], off, s[8:11], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_b128 v[5:8], off, s[8:11], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_b128 v[5:8], v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x05,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_load_b128 v[5:8], v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x05,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_load_b128 v[5:8], off, s[8:11], s3
+// GFX12: encoding: [0x03,0xc0,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_b128 v[5:8], off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0xc0,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_b128 v[5:8], off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0xc0,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_load_b128 v[5:8], off, s[8:11], s3 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_b128 v[5:8], off, s[8:11], s3 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_b128 v[5:8], off, s[8:11], s3 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_b128 v[5:8], off, s[8:11], s3 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_b128 v[5:8], off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0xc0,0x05,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_b128 v[5:8], off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS
+// GFX12: encoding: [0x03,0xc0,0x05,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_b16 v5, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_b16 v255, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x08,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_b16 v5, off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x08,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_b16 v5, off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x08,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_b16 v5, off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x00,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_b16 v5, off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x00,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_b16 v5, off, s[8:11], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_d16_b16 v5, off, s[8:11], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_d16_b16 v5, off, s[8:11], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_d16_b16 v5, off, s[8:11], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_d16_b16 v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x00,0x08,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_b16 v5, v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x00,0x08,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_b16 v5, off, s[8:11], s3
+// GFX12: encoding: [0x03,0x00,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_d16_b16 v5, off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x00,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_d16_b16 v5, off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x00,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_load_d16_b16 v5, off, s[8:11], s3 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_d16_b16 v5, off, s[8:11], s3 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_d16_b16 v5, off, s[8:11], s3 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_d16_b16 v5, off, s[8:11], s3 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_d16_b16 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0x00,0x08,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_b16 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS
+// GFX12: encoding: [0x03,0x00,0x08,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_format_x v5, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_format_x v255, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x02,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_format_x v5, off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x02,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_format_x v5, off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x02,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_format_x v5, off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x00,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_format_x v5, off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x00,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_format_x v5, off, s[8:11], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_d16_format_x v5, off, s[8:11], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_d16_format_x v5, off, s[8:11], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_d16_format_x v5, off, s[8:11], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_d16_format_x v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x00,0x02,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_format_x v5, v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x00,0x02,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_format_x v5, off, s[8:11], s3
+// GFX12: encoding: [0x03,0x00,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_d16_format_x v5, off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x00,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_d16_format_x v5, off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x00,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_load_d16_format_x v5, off, s[8:11], s3 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_d16_format_x v5, off, s[8:11], s3 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_d16_format_x v5, off, s[8:11], s3 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_d16_format_x v5, off, s[8:11], s3 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_d16_format_x v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0x00,0x02,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_format_x v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS
+// GFX12: encoding: [0x03,0x00,0x02,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_format_xy v5, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_format_xy v255, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x02,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_format_xy v5, off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x02,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_format_xy v5, off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x02,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_format_xy v5, off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x40,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_format_xy v5, off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x40,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_format_xy v5, off, s[8:11], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_d16_format_xy v5, off, s[8:11], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_d16_format_xy v5, off, s[8:11], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_d16_format_xy v5, off, s[8:11], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_d16_format_xy v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x40,0x02,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_format_xy v5, v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x40,0x02,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_format_xy v5, off, s[8:11], s3
+// GFX12: encoding: [0x03,0x40,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_d16_format_xy v5, off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x40,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_d16_format_xy v5, off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x40,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_load_d16_format_xy v5, off, s[8:11], s3 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_d16_format_xy v5, off, s[8:11], s3 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_d16_format_xy v5, off, s[8:11], s3 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_d16_format_xy v5, off, s[8:11], s3 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_d16_format_xy v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0x40,0x02,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_format_xy v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS
+// GFX12: encoding: [0x03,0x40,0x02,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_format_xyz v[5:6], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_format_xyz v[254:255], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x02,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_format_xyz v[5:6], off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x02,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_format_xyz v[5:6], off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x02,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_format_xyz v[5:6], off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x80,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_format_xyz v[5:6], off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x80,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_format_xyz v[5:6], off, s[8:11], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_d16_format_xyz v[5:6], off, s[8:11], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_d16_format_xyz v[5:6], off, s[8:11], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_d16_format_xyz v[5:6], off, s[8:11], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_d16_format_xyz v[5:6], v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x80,0x02,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_format_xyz v[5:6], v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x80,0x02,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_format_xyz v[5:6], off, s[8:11], s3
+// GFX12: encoding: [0x03,0x80,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_d16_format_xyz v[5:6], off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x80,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_d16_format_xyz v[5:6], off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x80,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_load_d16_format_xyz v[5:6], off, s[8:11], s3 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_d16_format_xyz v[5:6], off, s[8:11], s3 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_d16_format_xyz v[5:6], off, s[8:11], s3 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_d16_format_xyz v[5:6], off, s[8:11], s3 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_d16_format_xyz v[5:6], off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0x80,0x02,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_format_xyz v[5:6], off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS
+// GFX12: encoding: [0x03,0x80,0x02,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_format_xyzw v[5:6], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_format_xyzw v[254:255], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x02,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_format_xyzw v[5:6], off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x02,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_format_xyzw v[5:6], off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x02,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_format_xyzw v[5:6], off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0xc0,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_format_xyzw v[5:6], off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0xc0,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_format_xyzw v[5:6], off, s[8:11], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_d16_format_xyzw v[5:6], off, s[8:11], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_d16_format_xyzw v[5:6], off, s[8:11], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_d16_format_xyzw v[5:6], off, s[8:11], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_d16_format_xyzw v[5:6], v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x02,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_format_xyzw v[5:6], v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x02,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_format_xyzw v[5:6], off, s[8:11], s3
+// GFX12: encoding: [0x03,0xc0,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_d16_format_xyzw v[5:6], off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0xc0,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_d16_format_xyzw v[5:6], off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0xc0,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_load_d16_format_xyzw v[5:6], off, s[8:11], s3 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_d16_format_xyzw v[5:6], off, s[8:11], s3 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_d16_format_xyzw v[5:6], off, s[8:11], s3 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_d16_format_xyzw v[5:6], off, s[8:11], s3 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_d16_format_xyzw v[5:6], off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0xc0,0x02,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_format_xyzw v[5:6], off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS
+// GFX12: encoding: [0x03,0xc0,0x02,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_b16 v5, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_b16 v255, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x08,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_b16 v5, off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x08,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_b16 v5, off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x08,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_b16 v5, off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0xc0,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_b16 v5, off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0xc0,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_b16 v5, off, s[8:11], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_d16_hi_b16 v5, off, s[8:11], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_d16_hi_b16 v5, off, s[8:11], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_d16_hi_b16 v5, off, s[8:11], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_d16_hi_b16 v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x08,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_b16 v5, v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x08,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_b16 v5, off, s[8:11], s3
+// GFX12: encoding: [0x03,0xc0,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_d16_hi_b16 v5, off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0xc0,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_d16_hi_b16 v5, off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0xc0,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_load_d16_hi_b16 v5, off, s[8:11], s3 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_d16_hi_b16 v5, off, s[8:11], s3 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_d16_hi_b16 v5, off, s[8:11], s3 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_d16_hi_b16 v5, off, s[8:11], s3 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_d16_hi_b16 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0xc0,0x08,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_b16 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS
+// GFX12: encoding: [0x03,0xc0,0x08,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_format_x v5, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x09,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_format_x v255, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x09,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_format_x v5, off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x09,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_format_x v5, off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x09,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_format_x v5, off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x80,0x09,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_format_x v5, off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x80,0x09,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_format_x v5, off, s[8:11], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_d16_hi_format_x v5, off, s[8:11], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_d16_hi_format_x v5, off, s[8:11], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_d16_hi_format_x v5, off, s[8:11], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_d16_hi_format_x v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x80,0x09,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_format_x v5, v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x80,0x09,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_format_x v5, off, s[8:11], s3
+// GFX12: encoding: [0x03,0x80,0x09,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_d16_hi_format_x v5, off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x80,0x09,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_d16_hi_format_x v5, off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x80,0x09,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_load_d16_hi_format_x v5, off, s[8:11], s3 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_d16_hi_format_x v5, off, s[8:11], s3 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_d16_hi_format_x v5, off, s[8:11], s3 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_d16_hi_format_x v5, off, s[8:11], s3 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_d16_hi_format_x v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0x80,0x09,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_format_x v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS
+// GFX12: encoding: [0x03,0x80,0x09,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_i8 v5, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_i8 v255, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x08,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_i8 v5, off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x08,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_i8 v5, off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x08,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_i8 v5, off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x80,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_i8 v5, off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x80,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_i8 v5, off, s[8:11], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_d16_hi_i8 v5, off, s[8:11], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_d16_hi_i8 v5, off, s[8:11], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_d16_hi_i8 v5, off, s[8:11], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_d16_hi_i8 v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x80,0x08,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_i8 v5, v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x80,0x08,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_i8 v5, off, s[8:11], s3
+// GFX12: encoding: [0x03,0x80,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_d16_hi_i8 v5, off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x80,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_d16_hi_i8 v5, off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x80,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_load_d16_hi_i8 v5, off, s[8:11], s3 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_d16_hi_i8 v5, off, s[8:11], s3 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_d16_hi_i8 v5, off, s[8:11], s3 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_d16_hi_i8 v5, off, s[8:11], s3 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_d16_hi_i8 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0x80,0x08,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_i8 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS
+// GFX12: encoding: [0x03,0x80,0x08,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_u8 v5, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_u8 v255, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x08,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_u8 v5, off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x08,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_u8 v5, off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x08,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_u8 v5, off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x40,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_u8 v5, off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x40,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_u8 v5, off, s[8:11], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_d16_hi_u8 v5, off, s[8:11], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_d16_hi_u8 v5, off, s[8:11], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_d16_hi_u8 v5, off, s[8:11], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_d16_hi_u8 v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x40,0x08,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_u8 v5, v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x40,0x08,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_u8 v5, off, s[8:11], s3
+// GFX12: encoding: [0x03,0x40,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_d16_hi_u8 v5, off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x40,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_d16_hi_u8 v5, off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x40,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_load_d16_hi_u8 v5, off, s[8:11], s3 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_d16_hi_u8 v5, off, s[8:11], s3 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_d16_hi_u8 v5, off, s[8:11], s3 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_d16_hi_u8 v5, off, s[8:11], s3 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_d16_hi_u8 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0x40,0x08,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_hi_u8 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS
+// GFX12: encoding: [0x03,0x40,0x08,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_i8 v5, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_i8 v255, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x07,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_i8 v5, off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x07,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_i8 v5, off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x07,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_i8 v5, off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0xc0,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_i8 v5, off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0xc0,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_i8 v5, off, s[8:11], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_d16_i8 v5, off, s[8:11], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_d16_i8 v5, off, s[8:11], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_d16_i8 v5, off, s[8:11], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_d16_i8 v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x07,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_i8 v5, v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x07,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_i8 v5, off, s[8:11], s3
+// GFX12: encoding: [0x03,0xc0,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_d16_i8 v5, off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0xc0,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_d16_i8 v5, off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0xc0,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_load_d16_i8 v5, off, s[8:11], s3 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_d16_i8 v5, off, s[8:11], s3 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_d16_i8 v5, off, s[8:11], s3 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_d16_i8 v5, off, s[8:11], s3 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_d16_i8 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0xc0,0x07,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_i8 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS
+// GFX12: encoding: [0x03,0xc0,0x07,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_u8 v5, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_u8 v255, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x07,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_u8 v5, off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x07,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_u8 v5, off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x07,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_u8 v5, off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x80,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_u8 v5, off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x80,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_u8 v5, off, s[8:11], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_d16_u8 v5, off, s[8:11], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_d16_u8 v5, off, s[8:11], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_d16_u8 v5, off, s[8:11], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_d16_u8 v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x80,0x07,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_u8 v5, v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x80,0x07,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_u8 v5, off, s[8:11], s3
+// GFX12: encoding: [0x03,0x80,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_d16_u8 v5, off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x80,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_d16_u8 v5, off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x80,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_load_d16_u8 v5, off, s[8:11], s3 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_d16_u8 v5, off, s[8:11], s3 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_d16_u8 v5, off, s[8:11], s3 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_d16_u8 v5, off, s[8:11], s3 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_d16_u8 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0x80,0x07,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_d16_u8 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS
+// GFX12: encoding: [0x03,0x80,0x07,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_x v5, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_x v255, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x00,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_x v5, off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x00,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_x v5, off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x00,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_x v5, off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x00,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_x v5, off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x00,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_x v5, off, s[8:11], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_format_x v5, off, s[8:11], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_format_x v5, off, s[8:11], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_format_x v5, off, s[8:11], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_format_x v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x00,0x00,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_x v5, v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x00,0x00,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_x v5, off, s[8:11], s3
+// GFX12: encoding: [0x03,0x00,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_format_x v5, off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x00,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_format_x v5, off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x00,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_load_format_x v5, off, s[8:11], s3 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_format_x v5, off, s[8:11], s3 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_format_x v5, off, s[8:11], s3 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_format_x v5, off, s[8:11], s3 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_format_x v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0x00,0x00,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_x v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS
+// GFX12: encoding: [0x03,0x00,0x00,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_xy v[5:6], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_xy v[254:255], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x00,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_xy v[5:6], off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x00,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_xy v[5:6], off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x00,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_xy v[5:6], off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x40,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_xy v[5:6], off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x40,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_xy v[5:6], off, s[8:11], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_format_xy v[5:6], off, s[8:11], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_format_xy v[5:6], off, s[8:11], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_format_xy v[5:6], off, s[8:11], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_format_xy v[5:6], v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x40,0x00,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_xy v[5:6], v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x40,0x00,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_xy v[5:6], off, s[8:11], s3
+// GFX12: encoding: [0x03,0x40,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_format_xy v[5:6], off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x40,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_format_xy v[5:6], off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x40,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_load_format_xy v[5:6], off, s[8:11], s3 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_format_xy v[5:6], off, s[8:11], s3 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_format_xy v[5:6], off, s[8:11], s3 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_format_xy v[5:6], off, s[8:11], s3 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_format_xy v[5:6], off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0x40,0x00,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_xy v[5:6], off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS
+// GFX12: encoding: [0x03,0x40,0x00,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_xy v[5:7], off, s[8:11], s3 offset:8388607 glc slc dlc tfe
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_format_xyz v[5:7], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_xyz v[253:255], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x00,0xc4,0xfd,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_xyz v[5:7], off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x00,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_xyz v[5:7], off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x00,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_xyz v[5:7], off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x80,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_xyz v[5:7], off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x80,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_xyz v[5:7], off, s[8:11], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_format_xyz v[5:7], off, s[8:11], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_format_xyz v[5:7], off, s[8:11], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_format_xyz v[5:7], off, s[8:11], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_format_xyz v[5:7], v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x80,0x00,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_xyz v[5:7], v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x80,0x00,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_xyz v[5:7], off, s[8:11], s3
+// GFX12: encoding: [0x03,0x80,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_format_xyz v[5:7], off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x80,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_format_xyz v[5:7], off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x80,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_load_format_xyz v[5:7], off, s[8:11], s3 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_format_xyz v[5:7], off, s[8:11], s3 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_format_xyz v[5:7], off, s[8:11], s3 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_format_xyz v[5:7], off, s[8:11], s3 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_format_xyz v[5:7], off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0x80,0x00,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_xyz v[5:7], off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS
+// GFX12: encoding: [0x03,0x80,0x00,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_xyzw v[5:8], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_xyzw v[252:255], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x00,0xc4,0xfc,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_xyzw v[5:8], off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x00,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_xyzw v[5:8], off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x00,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_xyzw v[5:8], off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0xc0,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_xyzw v[5:8], off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0xc0,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_xyzw v[5:8], off, s[8:11], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_format_xyzw v[5:8], off, s[8:11], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_format_xyzw v[5:8], off, s[8:11], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_format_xyzw v[5:8], off, s[8:11], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_format_xyzw v[5:8], v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x00,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_xyzw v[5:8], v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x00,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_xyzw v[5:8], off, s[8:11], s3
+// GFX12: encoding: [0x03,0xc0,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_format_xyzw v[5:8], off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0xc0,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_format_xyzw v[5:8], off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0xc0,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_load_format_xyzw v[5:8], off, s[8:11], s3 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_format_xyzw v[5:8], off, s[8:11], s3 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_format_xyzw v[5:8], off, s[8:11], s3 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_format_xyzw v[5:8], off, s[8:11], s3 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_format_xyzw v[5:8], off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0xc0,0x00,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_xyzw v[5:8], off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS
+// GFX12: encoding: [0x03,0xc0,0x00,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_i8 v5, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_i8 v255, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x04,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_i8 v5, off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x04,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_i8 v5, off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x04,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_i8 v5, off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x40,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_i8 v5, off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x40,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_i8 v5, off, s[8:11], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_i8 v5, off, s[8:11], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_i8 v5, off, s[8:11], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_i8 v5, off, s[8:11], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_i8 v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x40,0x04,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_load_i8 v5, v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x40,0x04,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_load_i8 v5, off, s[8:11], s3
+// GFX12: encoding: [0x03,0x40,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_i8 v5, off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x40,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_i8 v5, off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x40,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_load_i8 v5, off, s[8:11], s3 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_i8 v5, off, s[8:11], s3 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_i8 v5, off, s[8:11], s3 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_i8 v5, off, s[8:11], s3 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_i8 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0x40,0x04,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_i8 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS
+// GFX12: encoding: [0x03,0x40,0x04,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_i16 v5, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_i16 v255, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x04,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_i16 v5, off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x04,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_i16 v5, off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x04,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_i16 v5, off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0xc0,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_i16 v5, off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0xc0,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_i16 v5, off, s[8:11], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_i16 v5, off, s[8:11], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_i16 v5, off, s[8:11], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_i16 v5, off, s[8:11], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_i16 v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x04,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_load_i16 v5, v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x04,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_load_i16 v5, off, s[8:11], s3
+// GFX12: encoding: [0x03,0xc0,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_i16 v5, off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0xc0,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_i16 v5, off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0xc0,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_load_i16 v5, off, s[8:11], s3 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_i16 v5, off, s[8:11], s3 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_i16 v5, off, s[8:11], s3 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_i16 v5, off, s[8:11], s3 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_i16 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0xc0,0x04,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_i16 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS
+// GFX12: encoding: [0x03,0xc0,0x04,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_u8 v5, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_u8 v255, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x04,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_u8 v5, off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x04,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_u8 v5, off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x04,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_u8 v5, off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x00,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_u8 v5, off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x00,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_u8 v5, off, s[8:11], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_u8 v5, off, s[8:11], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_u8 v5, off, s[8:11], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_u8 v5, off, s[8:11], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_u8 v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x00,0x04,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_load_u8 v5, v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x00,0x04,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_load_u8 v5, off, s[8:11], s3
+// GFX12: encoding: [0x03,0x00,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_u8 v5, off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x00,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_u8 v5, off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x00,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_load_u8 v5, off, s[8:11], s3 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_u8 v5, off, s[8:11], s3 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_u8 v5, off, s[8:11], s3 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_u8 v5, off, s[8:11], s3 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_u8 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0x00,0x04,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_u8 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS
+// GFX12: encoding: [0x03,0x00,0x04,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_u16 v5, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_u16 v255, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x04,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_u16 v5, off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x04,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_u16 v5, off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x04,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_u16 v5, off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x80,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_u16 v5, off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x80,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_u16 v5, off, s[8:11], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_u16 v5, off, s[8:11], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_u16 v5, off, s[8:11], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_u16 v5, off, s[8:11], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_load_u16 v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x80,0x04,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_load_u16 v5, v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x80,0x04,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_load_u16 v5, off, s[8:11], s3
+// GFX12: encoding: [0x03,0x80,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_u16 v5, off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x80,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_u16 v5, off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x80,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_load_u16 v5, off, s[8:11], s3 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_u16 v5, off, s[8:11], s3 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_u16 v5, off, s[8:11], s3 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_u16 v5, off, s[8:11], s3 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_u16 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0x80,0x04,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_u16 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS
+// GFX12: encoding: [0x03,0x80,0x04,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b8 v1, off, s[12:15], s4 offset:8388607
+// GFX12: encoding: [0x04,0x00,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b8 v255, off, s[12:15], s4 offset:8388607
+// GFX12: encoding: [0x04,0x00,0x06,0xc4,0xff,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b8 v1, off, s[16:19], s4 offset:8388607
+// GFX12: encoding: [0x04,0x00,0x06,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b8 v1, off, s[96:99], s4 offset:8388607
+// GFX12: encoding: [0x04,0x00,0x06,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b8 v1, off, s[12:15], s101 offset:8388607
+// GFX12: encoding: [0x65,0x00,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b8 v1, off, s[12:15], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x00,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b8 v1, off, s[12:15], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_b8 v1, off, s[12:15], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_b8 v1, off, s[12:15], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_b8 v1, off, s[12:15], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_b8 v1, v0, s[12:15], s4 idxen offset:8388607
+// GFX12: encoding: [0x04,0x00,0x06,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_store_b8 v1, v0, s[12:15], s4 offen offset:8388607
+// GFX12: encoding: [0x04,0x00,0x06,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_store_b8 v1, off, s[12:15], s4
+// GFX12: encoding: [0x04,0x00,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_b8 v1, off, s[12:15], s4 offset:0
+// GFX12: encoding: [0x04,0x00,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_b8 v1, off, s[12:15], s4 offset:7
+// GFX12: encoding: [0x04,0x00,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_store_b8 v1, off, s[12:15], s4 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_b8 v1, off, s[12:15], s4 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_b8 v1, off, s[12:15], s4 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_b8 v1, off, s[12:15], s4 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_b8 v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV
+// GFX12: encoding: [0x04,0x00,0x06,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b8 v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS
+// GFX12: encoding: [0x04,0x00,0x06,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b16 v1, off, s[12:15], s4 offset:8388607
+// GFX12: encoding: [0x04,0x40,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b16 v255, off, s[12:15], s4 offset:8388607
+// GFX12: encoding: [0x04,0x40,0x06,0xc4,0xff,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b16 v1, off, s[16:19], s4 offset:8388607
+// GFX12: encoding: [0x04,0x40,0x06,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b16 v1, off, s[96:99], s4 offset:8388607
+// GFX12: encoding: [0x04,0x40,0x06,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b16 v1, off, s[12:15], s101 offset:8388607
+// GFX12: encoding: [0x65,0x40,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b16 v1, off, s[12:15], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x40,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b16 v1, off, s[12:15], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_b16 v1, off, s[12:15], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_b16 v1, off, s[12:15], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_b16 v1, off, s[12:15], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_b16 v1, v0, s[12:15], s4 idxen offset:8388607
+// GFX12: encoding: [0x04,0x40,0x06,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_store_b16 v1, v0, s[12:15], s4 offen offset:8388607
+// GFX12: encoding: [0x04,0x40,0x06,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_store_b16 v1, off, s[12:15], s4
+// GFX12: encoding: [0x04,0x40,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_b16 v1, off, s[12:15], s4 offset:0
+// GFX12: encoding: [0x04,0x40,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_b16 v1, off, s[12:15], s4 offset:7
+// GFX12: encoding: [0x04,0x40,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_store_b16 v1, off, s[12:15], s4 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_b16 v1, off, s[12:15], s4 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_b16 v1, off, s[12:15], s4 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_b16 v1, off, s[12:15], s4 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_b16 v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV
+// GFX12: encoding: [0x04,0x40,0x06,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b16 v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS
+// GFX12: encoding: [0x04,0x40,0x06,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b32 v1, off, s[12:15], s4 offset:8388607
+// GFX12: encoding: [0x04,0x80,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b32 v255, off, s[12:15], s4 offset:8388607
+// GFX12: encoding: [0x04,0x80,0x06,0xc4,0xff,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b32 v1, off, s[16:19], s4 offset:8388607
+// GFX12: encoding: [0x04,0x80,0x06,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b32 v1, off, s[96:99], s4 offset:8388607
+// GFX12: encoding: [0x04,0x80,0x06,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b32 v1, off, s[12:15], s101 offset:8388607
+// GFX12: encoding: [0x65,0x80,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b32 v1, off, s[12:15], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x80,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b32 v1, off, s[12:15], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_b32 v1, off, s[12:15], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_b32 v1, off, s[12:15], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_b32 v1, off, s[12:15], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_b32 v1, v0, s[12:15], s4 idxen offset:8388607
+// GFX12: encoding: [0x04,0x80,0x06,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_store_b32 v1, v0, s[12:15], s4 offen offset:8388607
+// GFX12: encoding: [0x04,0x80,0x06,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_store_b32 v1, off, s[12:15], s4
+// GFX12: encoding: [0x04,0x80,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_b32 v1, off, s[12:15], s4 offset:0
+// GFX12: encoding: [0x04,0x80,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_b32 v1, off, s[12:15], s4 offset:7
+// GFX12: encoding: [0x04,0x80,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_store_b32 v1, off, s[12:15], s4 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_b32 v1, off, s[12:15], s4 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_b32 v1, off, s[12:15], s4 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_b32 v1, off, s[12:15], s4 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_b32 v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV
+// GFX12: encoding: [0x04,0x80,0x06,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b32 v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS
+// GFX12: encoding: [0x04,0x80,0x06,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b64 v[1:2], off, s[12:15], s4 offset:8388607
+// GFX12: encoding: [0x04,0xc0,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b64 v[254:255], off, s[12:15], s4 offset:8388607
+// GFX12: encoding: [0x04,0xc0,0x06,0xc4,0xfe,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b64 v[1:2], off, s[16:19], s4 offset:8388607
+// GFX12: encoding: [0x04,0xc0,0x06,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b64 v[1:2], off, s[96:99], s4 offset:8388607
+// GFX12: encoding: [0x04,0xc0,0x06,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b64 v[1:2], off, s[12:15], s101 offset:8388607
+// GFX12: encoding: [0x65,0xc0,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b64 v[1:2], off, s[12:15], m0 offset:8388607
+// GFX12: encoding: [0x7d,0xc0,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b64 v[1:2], off, s[12:15], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_b64 v[1:2], off, s[12:15], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_b64 v[1:2], off, s[12:15], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_b64 v[1:2], off, s[12:15], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_b64 v[1:2], v0, s[12:15], s4 idxen offset:8388607
+// GFX12: encoding: [0x04,0xc0,0x06,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_store_b64 v[1:2], v0, s[12:15], s4 offen offset:8388607
+// GFX12: encoding: [0x04,0xc0,0x06,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_store_b64 v[1:2], off, s[12:15], s4
+// GFX12: encoding: [0x04,0xc0,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_b64 v[1:2], off, s[12:15], s4 offset:0
+// GFX12: encoding: [0x04,0xc0,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_b64 v[1:2], off, s[12:15], s4 offset:7
+// GFX12: encoding: [0x04,0xc0,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_store_b64 v[1:2], off, s[12:15], s4 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_b64 v[1:2], off, s[12:15], s4 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_b64 v[1:2], off, s[12:15], s4 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_b64 v[1:2], off, s[12:15], s4 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_b64 v[1:2], off, s[12:15], s4 offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV
+// GFX12: encoding: [0x04,0xc0,0x06,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b64 v[1:2], off, s[12:15], s4 offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS
+// GFX12: encoding: [0x04,0xc0,0x06,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b96 v[1:3], off, s[12:15], s4 offset:8388607
+// GFX12: encoding: [0x04,0x00,0x07,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b96 v[253:255], off, s[12:15], s4 offset:8388607
+// GFX12: encoding: [0x04,0x00,0x07,0xc4,0xfd,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b96 v[1:3], off, s[16:19], s4 offset:8388607
+// GFX12: encoding: [0x04,0x00,0x07,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b96 v[1:3], off, s[96:99], s4 offset:8388607
+// GFX12: encoding: [0x04,0x00,0x07,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b96 v[1:3], off, s[12:15], s101 offset:8388607
+// GFX12: encoding: [0x65,0x00,0x07,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b96 v[1:3], off, s[12:15], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x00,0x07,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b96 v[1:3], off, s[12:15], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_b96 v[1:3], off, s[12:15], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_b96 v[1:3], off, s[12:15], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_b96 v[1:3], off, s[12:15], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_b96 v[1:3], v0, s[12:15], s4 idxen offset:8388607
+// GFX12: encoding: [0x04,0x00,0x07,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_store_b96 v[1:3], v0, s[12:15], s4 offen offset:8388607
+// GFX12: encoding: [0x04,0x00,0x07,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_store_b96 v[1:3], off, s[12:15], s4
+// GFX12: encoding: [0x04,0x00,0x07,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_b96 v[1:3], off, s[12:15], s4 offset:0
+// GFX12: encoding: [0x04,0x00,0x07,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_b96 v[1:3], off, s[12:15], s4 offset:7
+// GFX12: encoding: [0x04,0x00,0x07,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_store_b96 v[1:3], off, s[12:15], s4 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_b96 v[1:3], off, s[12:15], s4 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_b96 v[1:3], off, s[12:15], s4 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_b96 v[1:3], off, s[12:15], s4 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_b96 v[1:3], off, s[12:15], s4 offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV
+// GFX12: encoding: [0x04,0x00,0x07,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b96 v[1:3], off, s[12:15], s4 offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS
+// GFX12: encoding: [0x04,0x00,0x07,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b128 v[1:4], off, s[12:15], s4 offset:8388607
+// GFX12: encoding: [0x04,0x40,0x07,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b128 v[252:255], off, s[12:15], s4 offset:8388607
+// GFX12: encoding: [0x04,0x40,0x07,0xc4,0xfc,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b128 v[1:4], off, s[16:19], s4 offset:8388607
+// GFX12: encoding: [0x04,0x40,0x07,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b128 v[1:4], off, s[96:99], s4 offset:8388607
+// GFX12: encoding: [0x04,0x40,0x07,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b128 v[1:4], off, s[12:15], s101 offset:8388607
+// GFX12: encoding: [0x65,0x40,0x07,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b128 v[1:4], off, s[12:15], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x40,0x07,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b128 v[1:4], off, s[12:15], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_b128 v[1:4], off, s[12:15], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_b128 v[1:4], off, s[12:15], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_b128 v[1:4], off, s[12:15], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_b128 v[1:4], v0, s[12:15], s4 idxen offset:8388607
+// GFX12: encoding: [0x04,0x40,0x07,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_store_b128 v[1:4], v0, s[12:15], s4 offen offset:8388607
+// GFX12: encoding: [0x04,0x40,0x07,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_store_b128 v[1:4], off, s[12:15], s4
+// GFX12: encoding: [0x04,0x40,0x07,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_b128 v[1:4], off, s[12:15], s4 offset:0
+// GFX12: encoding: [0x04,0x40,0x07,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_b128 v[1:4], off, s[12:15], s4 offset:7
+// GFX12: encoding: [0x04,0x40,0x07,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_store_b128 v[1:4], off, s[12:15], s4 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_b128 v[1:4], off, s[12:15], s4 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_b128 v[1:4], off, s[12:15], s4 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_b128 v[1:4], off, s[12:15], s4 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_b128 v[1:4], off, s[12:15], s4 offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV
+// GFX12: encoding: [0x04,0x40,0x07,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_b128 v[1:4], off, s[12:15], s4 offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS
+// GFX12: encoding: [0x04,0x40,0x07,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_format_x v1, off, s[12:15], s4 offset:8388607
+// GFX12: encoding: [0x04,0x00,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_format_x v255, off, s[12:15], s4 offset:8388607
+// GFX12: encoding: [0x04,0x00,0x03,0xc4,0xff,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_format_x v1, off, s[16:19], s4 offset:8388607
+// GFX12: encoding: [0x04,0x00,0x03,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_format_x v1, off, s[96:99], s4 offset:8388607
+// GFX12: encoding: [0x04,0x00,0x03,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_format_x v1, off, s[12:15], s101 offset:8388607
+// GFX12: encoding: [0x65,0x00,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_format_x v1, off, s[12:15], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x00,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_format_x v1, off, s[12:15], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_d16_format_x v1, off, s[12:15], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_d16_format_x v1, off, s[12:15], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_d16_format_x v1, off, s[12:15], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_d16_format_x v1, v0, s[12:15], s4 idxen offset:8388607
+// GFX12: encoding: [0x04,0x00,0x03,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_format_x v1, v0, s[12:15], s4 offen offset:8388607
+// GFX12: encoding: [0x04,0x00,0x03,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_format_x v1, off, s[12:15], s4
+// GFX12: encoding: [0x04,0x00,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_d16_format_x v1, off, s[12:15], s4 offset:0
+// GFX12: encoding: [0x04,0x00,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_d16_format_x v1, off, s[12:15], s4 offset:7
+// GFX12: encoding: [0x04,0x00,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_store_d16_format_x v1, off, s[12:15], s4 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_d16_format_x v1, off, s[12:15], s4 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_d16_format_x v1, off, s[12:15], s4 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_d16_format_x v1, off, s[12:15], s4 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_d16_format_x v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV
+// GFX12: encoding: [0x04,0x00,0x03,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_format_x v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS
+// GFX12: encoding: [0x04,0x00,0x03,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_format_xy v1, off, s[12:15], s4 offset:8388607
+// GFX12: encoding: [0x04,0x40,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_format_xy v255, off, s[12:15], s4 offset:8388607
+// GFX12: encoding: [0x04,0x40,0x03,0xc4,0xff,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_format_xy v1, off, s[16:19], s4 offset:8388607
+// GFX12: encoding: [0x04,0x40,0x03,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_format_xy v1, off, s[96:99], s4 offset:8388607
+// GFX12: encoding: [0x04,0x40,0x03,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_format_xy v1, off, s[12:15], s101 offset:8388607
+// GFX12: encoding: [0x65,0x40,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_format_xy v1, off, s[12:15], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x40,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_format_xy v1, off, s[12:15], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_d16_format_xy v1, off, s[12:15], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_d16_format_xy v1, off, s[12:15], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_d16_format_xy v1, off, s[12:15], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_d16_format_xy v1, v0, s[12:15], s4 idxen offset:8388607
+// GFX12: encoding: [0x04,0x40,0x03,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_format_xy v1, v0, s[12:15], s4 offen offset:8388607
+// GFX12: encoding: [0x04,0x40,0x03,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_format_xy v1, off, s[12:15], s4
+// GFX12: encoding: [0x04,0x40,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_d16_format_xy v1, off, s[12:15], s4 offset:0
+// GFX12: encoding: [0x04,0x40,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_d16_format_xy v1, off, s[12:15], s4 offset:7
+// GFX12: encoding: [0x04,0x40,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_store_d16_format_xy v1, off, s[12:15], s4 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_d16_format_xy v1, off, s[12:15], s4 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_d16_format_xy v1, off, s[12:15], s4 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_d16_format_xy v1, off, s[12:15], s4 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_d16_format_xy v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV
+// GFX12: encoding: [0x04,0x40,0x03,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_format_xy v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS
+// GFX12: encoding: [0x04,0x40,0x03,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_format_xyz v[1:2], off, s[12:15], s4 offset:8388607
+// GFX12: encoding: [0x04,0x80,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_format_xyz v[254:255], off, s[12:15], s4 offset:8388607
+// GFX12: encoding: [0x04,0x80,0x03,0xc4,0xfe,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_format_xyz v[1:2], off, s[16:19], s4 offset:8388607
+// GFX12: encoding: [0x04,0x80,0x03,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_format_xyz v[1:2], off, s[96:99], s4 offset:8388607
+// GFX12: encoding: [0x04,0x80,0x03,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_format_xyz v[1:2], off, s[12:15], s101 offset:8388607
+// GFX12: encoding: [0x65,0x80,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_format_xyz v[1:2], off, s[12:15], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x80,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_format_xyz v[1:2], off, s[12:15], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_d16_format_xyz v[1:2], off, s[12:15], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_d16_format_xyz v[1:2], off, s[12:15], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_d16_format_xyz v[1:2], off, s[12:15], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_d16_format_xyz v[1:2], v0, s[12:15], s4 idxen offset:8388607
+// GFX12: encoding: [0x04,0x80,0x03,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_format_xyz v[1:2], v0, s[12:15], s4 offen offset:8388607
+// GFX12: encoding: [0x04,0x80,0x03,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_format_xyz v[1:2], off, s[12:15], s4
+// GFX12: encoding: [0x04,0x80,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_d16_format_xyz v[1:2], off, s[12:15], s4 offset:0
+// GFX12: encoding: [0x04,0x80,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_d16_format_xyz v[1:2], off, s[12:15], s4 offset:7
+// GFX12: encoding: [0x04,0x80,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_store_d16_format_xyz v[1:2], off, s[12:15], s4 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_d16_format_xyz v[1:2], off, s[12:15], s4 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_d16_format_xyz v[1:2], off, s[12:15], s4 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_d16_format_xyz v[1:2], off, s[12:15], s4 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_d16_format_xyz v[1:2], off, s[12:15], s4 offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV
+// GFX12: encoding: [0x04,0x80,0x03,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_format_xyz v[1:2], off, s[12:15], s4 offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS
+// GFX12: encoding: [0x04,0x80,0x03,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_format_xyzw v[1:2], off, s[12:15], s4 offset:8388607
+// GFX12: encoding: [0x04,0xc0,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_format_xyzw v[254:255], off, s[12:15], s4 offset:8388607
+// GFX12: encoding: [0x04,0xc0,0x03,0xc4,0xfe,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_format_xyzw v[1:2], off, s[16:19], s4 offset:8388607
+// GFX12: encoding: [0x04,0xc0,0x03,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_format_xyzw v[1:2], off, s[96:99], s4 offset:8388607
+// GFX12: encoding: [0x04,0xc0,0x03,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_format_xyzw v[1:2], off, s[12:15], s101 offset:8388607
+// GFX12: encoding: [0x65,0xc0,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_format_xyzw v[1:2], off, s[12:15], m0 offset:8388607
+// GFX12: encoding: [0x7d,0xc0,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_format_xyzw v[1:2], off, s[12:15], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_d16_format_xyzw v[1:2], off, s[12:15], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_d16_format_xyzw v[1:2], off, s[12:15], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_d16_format_xyzw v[1:2], off, s[12:15], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_d16_format_xyzw v[1:2], v0, s[12:15], s4 idxen offset:8388607
+// GFX12: encoding: [0x04,0xc0,0x03,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_format_xyzw v[1:2], v0, s[12:15], s4 offen offset:8388607
+// GFX12: encoding: [0x04,0xc0,0x03,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_format_xyzw v[1:2], off, s[12:15], s4
+// GFX12: encoding: [0x04,0xc0,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_d16_format_xyzw v[1:2], off, s[12:15], s4 offset:0
+// GFX12: encoding: [0x04,0xc0,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_d16_format_xyzw v[1:2], off, s[12:15], s4 offset:7
+// GFX12: encoding: [0x04,0xc0,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_store_d16_format_xyzw v[1:2], off, s[12:15], s4 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_d16_format_xyzw v[1:2], off, s[12:15], s4 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_d16_format_xyzw v[1:2], off, s[12:15], s4 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_d16_format_xyzw v[1:2], off, s[12:15], s4 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_d16_format_xyzw v[1:2], off, s[12:15], s4 offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV
+// GFX12: encoding: [0x04,0xc0,0x03,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_format_xyzw v[1:2], off, s[12:15], s4 offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS
+// GFX12: encoding: [0x04,0xc0,0x03,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_hi_b8 v1, off, s[12:15], s4 offset:8388607
+// GFX12: encoding: [0x04,0x00,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_hi_b8 v255, off, s[12:15], s4 offset:8388607
+// GFX12: encoding: [0x04,0x00,0x09,0xc4,0xff,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_hi_b8 v1, off, s[16:19], s4 offset:8388607
+// GFX12: encoding: [0x04,0x00,0x09,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_hi_b8 v1, off, s[96:99], s4 offset:8388607
+// GFX12: encoding: [0x04,0x00,0x09,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_hi_b8 v1, off, s[12:15], s101 offset:8388607
+// GFX12: encoding: [0x65,0x00,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_hi_b8 v1, off, s[12:15], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x00,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_hi_b8 v1, off, s[12:15], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_d16_hi_b8 v1, off, s[12:15], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_d16_hi_b8 v1, off, s[12:15], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_d16_hi_b8 v1, off, s[12:15], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_d16_hi_b8 v1, v0, s[12:15], s4 idxen offset:8388607
+// GFX12: encoding: [0x04,0x00,0x09,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_hi_b8 v1, v0, s[12:15], s4 offen offset:8388607
+// GFX12: encoding: [0x04,0x00,0x09,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_hi_b8 v1, off, s[12:15], s4
+// GFX12: encoding: [0x04,0x00,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_d16_hi_b8 v1, off, s[12:15], s4 offset:0
+// GFX12: encoding: [0x04,0x00,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_d16_hi_b8 v1, off, s[12:15], s4 offset:7
+// GFX12: encoding: [0x04,0x00,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_store_d16_hi_b8 v1, off, s[12:15], s4 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_d16_hi_b8 v1, off, s[12:15], s4 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_d16_hi_b8 v1, off, s[12:15], s4 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_d16_hi_b8 v1, off, s[12:15], s4 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_d16_hi_b8 v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV
+// GFX12: encoding: [0x04,0x00,0x09,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_hi_b8 v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS
+// GFX12: encoding: [0x04,0x00,0x09,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_hi_b16 v1, off, s[12:15], s4 offset:8388607
+// GFX12: encoding: [0x04,0x40,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_hi_b16 v255, off, s[12:15], s4 offset:8388607
+// GFX12: encoding: [0x04,0x40,0x09,0xc4,0xff,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_hi_b16 v1, off, s[16:19], s4 offset:8388607
+// GFX12: encoding: [0x04,0x40,0x09,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_hi_b16 v1, off, s[96:99], s4 offset:8388607
+// GFX12: encoding: [0x04,0x40,0x09,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_hi_b16 v1, off, s[12:15], s101 offset:8388607
+// GFX12: encoding: [0x65,0x40,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_hi_b16 v1, off, s[12:15], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x40,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_hi_b16 v1, off, s[12:15], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_d16_hi_b16 v1, off, s[12:15], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_d16_hi_b16 v1, off, s[12:15], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_d16_hi_b16 v1, off, s[12:15], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_d16_hi_b16 v1, v0, s[12:15], s4 idxen offset:8388607
+// GFX12: encoding: [0x04,0x40,0x09,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_hi_b16 v1, v0, s[12:15], s4 offen offset:8388607
+// GFX12: encoding: [0x04,0x40,0x09,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_hi_b16 v1, off, s[12:15], s4
+// GFX12: encoding: [0x04,0x40,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_d16_hi_b16 v1, off, s[12:15], s4 offset:0
+// GFX12: encoding: [0x04,0x40,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_d16_hi_b16 v1, off, s[12:15], s4 offset:7
+// GFX12: encoding: [0x04,0x40,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_store_d16_hi_b16 v1, off, s[12:15], s4 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_d16_hi_b16 v1, off, s[12:15], s4 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_d16_hi_b16 v1, off, s[12:15], s4 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_d16_hi_b16 v1, off, s[12:15], s4 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_d16_hi_b16 v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV
+// GFX12: encoding: [0x04,0x40,0x09,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_hi_b16 v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS
+// GFX12: encoding: [0x04,0x40,0x09,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_hi_format_x v1, off, s[12:15], s4 offset:8388607
+// GFX12: encoding: [0x04,0xc0,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_hi_format_x v255, off, s[12:15], s4 offset:8388607
+// GFX12: encoding: [0x04,0xc0,0x09,0xc4,0xff,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_hi_format_x v1, off, s[16:19], s4 offset:8388607
+// GFX12: encoding: [0x04,0xc0,0x09,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_hi_format_x v1, off, s[96:99], s4 offset:8388607
+// GFX12: encoding: [0x04,0xc0,0x09,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_hi_format_x v1, off, s[12:15], s101 offset:8388607
+// GFX12: encoding: [0x65,0xc0,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_hi_format_x v1, off, s[12:15], m0 offset:8388607
+// GFX12: encoding: [0x7d,0xc0,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_hi_format_x v1, off, s[12:15], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_d16_hi_format_x v1, off, s[12:15], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_d16_hi_format_x v1, off, s[12:15], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_d16_hi_format_x v1, off, s[12:15], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_d16_hi_format_x v1, v0, s[12:15], s4 idxen offset:8388607
+// GFX12: encoding: [0x04,0xc0,0x09,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_hi_format_x v1, v0, s[12:15], s4 offen offset:8388607
+// GFX12: encoding: [0x04,0xc0,0x09,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_hi_format_x v1, off, s[12:15], s4
+// GFX12: encoding: [0x04,0xc0,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_d16_hi_format_x v1, off, s[12:15], s4 offset:0
+// GFX12: encoding: [0x04,0xc0,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_d16_hi_format_x v1, off, s[12:15], s4 offset:7
+// GFX12: encoding: [0x04,0xc0,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_store_d16_hi_format_x v1, off, s[12:15], s4 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_d16_hi_format_x v1, off, s[12:15], s4 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_d16_hi_format_x v1, off, s[12:15], s4 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_d16_hi_format_x v1, off, s[12:15], s4 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_d16_hi_format_x v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV
+// GFX12: encoding: [0x04,0xc0,0x09,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_d16_hi_format_x v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS
+// GFX12: encoding: [0x04,0xc0,0x09,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_x v1, off, s[12:15], s4 offset:8388607
+// GFX12: encoding: [0x04,0x00,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_x v255, off, s[12:15], s4 offset:8388607
+// GFX12: encoding: [0x04,0x00,0x01,0xc4,0xff,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_x v1, off, s[16:19], s4 offset:8388607
+// GFX12: encoding: [0x04,0x00,0x01,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_x v1, off, s[96:99], s4 offset:8388607
+// GFX12: encoding: [0x04,0x00,0x01,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_x v1, off, s[12:15], s101 offset:8388607
+// GFX12: encoding: [0x65,0x00,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_x v1, off, s[12:15], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x00,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_x v1, off, s[12:15], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_format_x v1, off, s[12:15], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_format_x v1, off, s[12:15], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_format_x v1, off, s[12:15], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_format_x v1, v0, s[12:15], s4 idxen offset:8388607
+// GFX12: encoding: [0x04,0x00,0x01,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_x v1, v0, s[12:15], s4 offen offset:8388607
+// GFX12: encoding: [0x04,0x00,0x01,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_x v1, off, s[12:15], s4
+// GFX12: encoding: [0x04,0x00,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_format_x v1, off, s[12:15], s4 offset:0
+// GFX12: encoding: [0x04,0x00,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_format_x v1, off, s[12:15], s4 offset:7
+// GFX12: encoding: [0x04,0x00,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_store_format_x v1, off, s[12:15], s4 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_format_x v1, off, s[12:15], s4 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_format_x v1, off, s[12:15], s4 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_format_x v1, off, s[12:15], s4 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_format_x v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV
+// GFX12: encoding: [0x04,0x00,0x01,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_x v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS
+// GFX12: encoding: [0x04,0x00,0x01,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_xy v[1:2], off, s[12:15], s4 offset:8388607
+// GFX12: encoding: [0x04,0x40,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_xy v[254:255], off, s[12:15], s4 offset:8388607
+// GFX12: encoding: [0x04,0x40,0x01,0xc4,0xfe,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_xy v[1:2], off, s[16:19], s4 offset:8388607
+// GFX12: encoding: [0x04,0x40,0x01,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_xy v[1:2], off, s[96:99], s4 offset:8388607
+// GFX12: encoding: [0x04,0x40,0x01,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_xy v[1:2], off, s[12:15], s101 offset:8388607
+// GFX12: encoding: [0x65,0x40,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_xy v[1:2], off, s[12:15], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x40,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_xy v[1:2], off, s[12:15], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_format_xy v[1:2], off, s[12:15], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_format_xy v[1:2], off, s[12:15], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_format_xy v[1:2], off, s[12:15], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_format_xy v[1:2], v0, s[12:15], s4 idxen offset:8388607
+// GFX12: encoding: [0x04,0x40,0x01,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_xy v[1:2], v0, s[12:15], s4 offen offset:8388607
+// GFX12: encoding: [0x04,0x40,0x01,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_xy v[1:2], off, s[12:15], s4
+// GFX12: encoding: [0x04,0x40,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_format_xy v[1:2], off, s[12:15], s4 offset:0
+// GFX12: encoding: [0x04,0x40,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_format_xy v[1:2], off, s[12:15], s4 offset:7
+// GFX12: encoding: [0x04,0x40,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_store_format_xy v[1:2], off, s[12:15], s4 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_format_xy v[1:2], off, s[12:15], s4 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_format_xy v[1:2], off, s[12:15], s4 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_format_xy v[1:2], off, s[12:15], s4 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_format_xy v[1:2], off, s[12:15], s4 offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV
+// GFX12: encoding: [0x04,0x40,0x01,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_xy v[1:2], off, s[12:15], s4 offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS
+// GFX12: encoding: [0x04,0x40,0x01,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_xyz v[1:3], off, s[12:15], s4 offset:8388607
+// GFX12: encoding: [0x04,0x80,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_xyz v[253:255], off, s[12:15], s4 offset:8388607
+// GFX12: encoding: [0x04,0x80,0x01,0xc4,0xfd,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_xyz v[1:3], off, s[16:19], s4 offset:8388607
+// GFX12: encoding: [0x04,0x80,0x01,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_xyz v[1:3], off, s[96:99], s4 offset:8388607
+// GFX12: encoding: [0x04,0x80,0x01,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_xyz v[1:3], off, s[12:15], s101 offset:8388607
+// GFX12: encoding: [0x65,0x80,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_xyz v[1:3], off, s[12:15], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x80,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_xyz v[1:3], off, s[12:15], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_format_xyz v[1:3], off, s[12:15], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_format_xyz v[1:3], off, s[12:15], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_format_xyz v[1:3], off, s[12:15], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_format_xyz v[1:3], v0, s[12:15], s4 idxen offset:8388607
+// GFX12: encoding: [0x04,0x80,0x01,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_xyz v[1:3], v0, s[12:15], s4 offen offset:8388607
+// GFX12: encoding: [0x04,0x80,0x01,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_xyz v[1:3], off, s[12:15], s4
+// GFX12: encoding: [0x04,0x80,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_format_xyz v[1:3], off, s[12:15], s4 offset:0
+// GFX12: encoding: [0x04,0x80,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_format_xyz v[1:3], off, s[12:15], s4 offset:7
+// GFX12: encoding: [0x04,0x80,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_store_format_xyz v[1:3], off, s[12:15], s4 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_format_xyz v[1:3], off, s[12:15], s4 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_format_xyz v[1:3], off, s[12:15], s4 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_format_xyz v[1:3], off, s[12:15], s4 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_format_xyz v[1:3], off, s[12:15], s4 offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV
+// GFX12: encoding: [0x04,0x80,0x01,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_xyz v[1:3], off, s[12:15], s4 offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS
+// GFX12: encoding: [0x04,0x80,0x01,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_xyzw v[1:4], off, s[12:15], s4 offset:8388607
+// GFX12: encoding: [0x04,0xc0,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_xyzw v[252:255], off, s[12:15], s4 offset:8388607
+// GFX12: encoding: [0x04,0xc0,0x01,0xc4,0xfc,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_xyzw v[1:4], off, s[16:19], s4 offset:8388607
+// GFX12: encoding: [0x04,0xc0,0x01,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_xyzw v[1:4], off, s[96:99], s4 offset:8388607
+// GFX12: encoding: [0x04,0xc0,0x01,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_xyzw v[1:4], off, s[12:15], s101 offset:8388607
+// GFX12: encoding: [0x65,0xc0,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_xyzw v[1:4], off, s[12:15], m0 offset:8388607
+// GFX12: encoding: [0x7d,0xc0,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_xyzw v[1:4], off, s[12:15], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_format_xyzw v[1:4], off, s[12:15], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_format_xyzw v[1:4], off, s[12:15], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_format_xyzw v[1:4], off, s[12:15], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_store_format_xyzw v[1:4], v0, s[12:15], s4 idxen offset:8388607
+// GFX12: encoding: [0x04,0xc0,0x01,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_xyzw v[1:4], v0, s[12:15], s4 offen offset:8388607
+// GFX12: encoding: [0x04,0xc0,0x01,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_xyzw v[1:4], off, s[12:15], s4
+// GFX12: encoding: [0x04,0xc0,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_format_xyzw v[1:4], off, s[12:15], s4 offset:0
+// GFX12: encoding: [0x04,0xc0,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_format_xyzw v[1:4], off, s[12:15], s4 offset:7
+// GFX12: encoding: [0x04,0xc0,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_store_format_xyzw v[1:4], off, s[12:15], s4 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_format_xyzw v[1:4], off, s[12:15], s4 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_format_xyzw v[1:4], off, s[12:15], s4 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_format_xyzw v[1:4], off, s[12:15], s4 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_store_format_xyzw v[1:4], off, s[12:15], s4 offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV
+// GFX12: encoding: [0x04,0xc0,0x01,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_xyzw v[1:4], off, s[12:15], s4 offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS
+// GFX12: encoding: [0x04,0xc0,0x01,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_f32 v5, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x15,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_f32 v255, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x15,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_f32 v5, off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x15,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_f32 v5, off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x15,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_f32 v5, off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x80,0x15,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_f32 v5, off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x80,0x15,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_f32 v5, off, s[8:11], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_add_f32 v5, off, s[8:11], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_add_f32 v5, off, s[8:11], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_add_f32 v5, off, s[8:11], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_add_f32 v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x80,0x15,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_f32 v5, v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x80,0x15,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_f32 v5, off, s[8:11], s3
+// GFX12: encoding: [0x03,0x80,0x15,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_add_f32 v5, off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x80,0x15,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_add_f32 v5, off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x80,0x15,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_add_f32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0x80,0x15,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_f32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX12: encoding: [0x03,0x80,0x15,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_f32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0x80,0x15,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_f32 v5, off, s[8:11], s3 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_add_f32 v5, off, s[8:11], s3 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_add_f32 v5, off, s[8:11], s3 offset:8388607 glc slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_add_u32 v5, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_u32 v255, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x0d,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_u32 v5, off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x0d,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_u32 v5, off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x0d,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_u32 v5, off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x40,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_u32 v5, off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x40,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_u32 v5, off, s[8:11], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_add_u32 v5, off, s[8:11], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_add_u32 v5, off, s[8:11], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_add_u32 v5, off, s[8:11], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_add_u32 v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x40,0x0d,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_u32 v5, v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x40,0x0d,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_u32 v5, off, s[8:11], s3
+// GFX12: encoding: [0x03,0x40,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_add_u32 v5, off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x40,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_add_u32 v5, off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x40,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_add_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0x40,0x0d,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX12: encoding: [0x03,0x40,0x0d,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0x40,0x0d,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_u32 v5, off, s[8:11], s3 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_add_u32 v5, off, s[8:11], s3 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_add_u32 v5, off, s[8:11], s3 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_add_u32 v5, off, s[8:11], s3 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_add_u64 v[5:6], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_u64 v[254:255], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x10,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_u64 v[5:6], off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x10,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_u64 v[5:6], off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x10,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_u64 v[5:6], off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0xc0,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_u64 v[5:6], off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0xc0,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_u64 v[5:6], off, s[8:11], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_add_u64 v[5:6], off, s[8:11], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_add_u64 v[5:6], off, s[8:11], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_add_u64 v[5:6], off, s[8:11], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_add_u64 v[5:6], v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x10,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_u64 v[5:6], v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x10,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_u64 v[5:6], off, s[8:11], s3
+// GFX12: encoding: [0x03,0xc0,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_add_u64 v[5:6], off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0xc0,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_add_u64 v[5:6], off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0xc0,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_add_u64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0xc0,0x10,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_u64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX12: encoding: [0x03,0xc0,0x10,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_u64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0xc0,0x10,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_u64 v[5:6], off, s[8:11], s3 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_add_u64 v[5:6], off, s[8:11], s3 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_add_u64 v[5:6], off, s[8:11], s3 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_add_u64 v[5:6], off, s[8:11], s3 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_and_b32 v5, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_and_b32 v255, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x0f,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_and_b32 v5, off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x0f,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_and_b32 v5, off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x0f,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_and_b32 v5, off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x00,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_and_b32 v5, off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x00,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_and_b32 v5, off, s[8:11], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_and_b32 v5, off, s[8:11], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_and_b32 v5, off, s[8:11], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_and_b32 v5, off, s[8:11], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_and_b32 v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x00,0x0f,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_and_b32 v5, v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x00,0x0f,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_and_b32 v5, off, s[8:11], s3
+// GFX12: encoding: [0x03,0x00,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_and_b32 v5, off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x00,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_and_b32 v5, off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x00,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_and_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0x00,0x0f,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_and_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX12: encoding: [0x03,0x00,0x0f,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_and_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0x00,0x0f,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_and_b32 v5, off, s[8:11], s3 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_and_b32 v5, off, s[8:11], s3 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_and_b32 v5, off, s[8:11], s3 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_and_b32 v5, off, s[8:11], s3 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_and_b64 v[5:6], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_and_b64 v[254:255], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x12,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_and_b64 v[5:6], off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x12,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_and_b64 v[5:6], off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x12,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_and_b64 v[5:6], off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x40,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_and_b64 v[5:6], off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x40,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_and_b64 v[5:6], off, s[8:11], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_and_b64 v[5:6], off, s[8:11], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_and_b64 v[5:6], off, s[8:11], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_and_b64 v[5:6], off, s[8:11], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_and_b64 v[5:6], v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x40,0x12,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_and_b64 v[5:6], v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x40,0x12,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_and_b64 v[5:6], off, s[8:11], s3
+// GFX12: encoding: [0x03,0x40,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_and_b64 v[5:6], off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x40,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_and_b64 v[5:6], off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x40,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_and_b64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0x40,0x12,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_and_b64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX12: encoding: [0x03,0x40,0x12,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_and_b64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0x40,0x12,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_and_b64 v[5:6], off, s[8:11], s3 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_and_b64 v[5:6], off, s[8:11], s3 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_and_b64 v[5:6], off, s[8:11], s3 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_and_b64 v[5:6], off, s[8:11], s3 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_cmpswap_b32 v[5:6], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cmpswap_b32 v[254:255], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x0d,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cmpswap_b32 v[5:6], off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x0d,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cmpswap_b32 v[5:6], off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x0d,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cmpswap_b32 v[5:6], off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x00,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cmpswap_b32 v[5:6], off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x00,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cmpswap_b32 v[5:6], off, s[8:11], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_cmpswap_b32 v[5:6], off, s[8:11], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_cmpswap_b32 v[5:6], off, s[8:11], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_cmpswap_b32 v[5:6], off, s[8:11], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_cmpswap_b32 v[5:6], v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x00,0x0d,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cmpswap_b32 v[5:6], v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x00,0x0d,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cmpswap_b32 v[5:6], off, s[8:11], s3
+// GFX12: encoding: [0x03,0x00,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_cmpswap_b32 v[5:6], off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x00,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_cmpswap_b32 v[5:6], off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x00,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_cmpswap_b32 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0x00,0x0d,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cmpswap_b32 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX12: encoding: [0x03,0x00,0x0d,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cmpswap_b32 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0x00,0x0d,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cmpswap_b32 v[5:6], off, s[8:11], s3 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_cmpswap_b32 v[5:6], off, s[8:11], s3 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_cmpswap_b32 v[5:6], off, s[8:11], s3 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_cmpswap_b32 v[5:6], off, s[8:11], s3 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_cmpswap_b64 v[5:8], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cmpswap_b64 v[252:255], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x10,0xc4,0xfc,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cmpswap_b64 v[5:8], off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x10,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cmpswap_b64 v[5:8], off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x10,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cmpswap_b64 v[5:8], off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x80,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cmpswap_b64 v[5:8], off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x80,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cmpswap_b64 v[5:8], off, s[8:11], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_cmpswap_b64 v[5:8], off, s[8:11], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_cmpswap_b64 v[5:8], off, s[8:11], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_cmpswap_b64 v[5:8], off, s[8:11], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_cmpswap_b64 v[5:8], v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x80,0x10,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cmpswap_b64 v[5:8], v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x80,0x10,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cmpswap_b64 v[5:8], off, s[8:11], s3
+// GFX12: encoding: [0x03,0x80,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_cmpswap_b64 v[5:8], off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x80,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_cmpswap_b64 v[5:8], off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x80,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_cmpswap_b64 v[5:8], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0x80,0x10,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cmpswap_b64 v[5:8], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX12: encoding: [0x03,0x80,0x10,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cmpswap_b64 v[5:8], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0x80,0x10,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cmpswap_b64 v[5:8], off, s[8:11], s3 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_cmpswap_b64 v[5:8], off, s[8:11], s3 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_cmpswap_b64 v[5:8], off, s[8:11], s3 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_cmpswap_b64 v[5:8], off, s[8:11], s3 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_clamp_u32 v255, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0xc0,0x0d,0xc4,0xff,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_clamp_u32 v255, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX12: encoding: [0x03,0xc0,0x0d,0xc4,0xff,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_clamp_u32 v255, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0xc0,0x0d,0xc4,0xff,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_clamp_u32 v255, off, s[8:11], s3 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_sub_clamp_u32 v5, off, s[12:15], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x18,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_clamp_u32 v5, off, s[12:15], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX12: encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x18,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_clamp_u32 v5, off, s[12:15], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_clamp_u32 v5, off, s[12:15], s3 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_sub_clamp_u32 v5, off, s[96:99], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0xc0,0x0d,0xc4,0x05,0xc0,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_clamp_u32 v5, off, s[96:99], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX12: encoding: [0x03,0xc0,0x0d,0xc4,0x05,0xc0,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_clamp_u32 v5, off, s[96:99], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0xc0,0x0d,0xc4,0x05,0xc0,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_clamp_u32 v5, off, s[96:99], s3 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s101 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x65,0xc0,0x0d,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s101 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX12: encoding: [0x65,0xc0,0x0d,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s101 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX12: encoding: [0x65,0xc0,0x0d,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s101 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_sub_clamp_u32 v5, off, s[8:11], m0 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7d,0xc0,0x0d,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_clamp_u32 v5, off, s[8:11], m0 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX12: encoding: [0x7d,0xc0,0x0d,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_clamp_u32 v5, off, s[8:11], m0 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX12: encoding: [0x7d,0xc0,0x0d,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_clamp_u32 v5, off, s[8:11], m0 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_sub_clamp_u32 v5, off, s[8:11], 0 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_sub_clamp_u32 v5, off, s[8:11], 0 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_sub_clamp_u32 v5, off, s[8:11], 0 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_sub_clamp_u32 v5, off, s[8:11], 0 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_sub_clamp_u32 v5, off, s[8:11], -1 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_sub_clamp_u32 v5, off, s[8:11], -1 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_sub_clamp_u32 v5, off, s[8:11], -1 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_sub_clamp_u32 v5, off, s[8:11], -1 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_sub_clamp_u32 v5, off, s[8:11], 0.5 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_sub_clamp_u32 v5, off, s[8:11], 0.5 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_sub_clamp_u32 v5, off, s[8:11], 0.5 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_sub_clamp_u32 v5, off, s[8:11], 0.5 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_sub_clamp_u32 v5, off, s[8:11], -4.0 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_sub_clamp_u32 v5, off, s[8:11], -4.0 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_sub_clamp_u32 v5, off, s[8:11], -4.0 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_sub_clamp_u32 v5, off, s[8:11], -4.0 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_sub_clamp_u32 v5, v0, s[8:11], s3 idxen offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x90,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_clamp_u32 v5, v0, s[8:11], s3 idxen offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX12: encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x94,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_clamp_u32 v5, v0, s[8:11], s3 idxen offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0xe8,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_clamp_u32 v5, v0, s[8:11], s3 idxen offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_sub_clamp_u32 v5, v0, s[8:11], s3 offen offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x90,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_clamp_u32 v5, v0, s[8:11], s3 offen offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX12: encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x94,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_clamp_u32 v5, v0, s[8:11], s3 offen offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0xe8,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_clamp_u32 v5, v0, s[8:11], s3 offen offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x90,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX12: encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x94,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0xe8,0x00,0x00,0x00,0x00,0x00
+
+buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 offset:0 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x90,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 offset:0 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX12: encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x94,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 offset:0 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0xe8,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 offset:0 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 offset:7 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x90,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 offset:7 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX12: encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x94,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 offset:7 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0xe8,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 offset:7 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX12: encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 offset:8388607 glc slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 offset:8388607 glc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_dec_u32 v5, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_dec_u32 v255, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x10,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_dec_u32 v5, off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x10,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_dec_u32 v5, off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x10,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_dec_u32 v5, off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x00,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_dec_u32 v5, off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x00,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_dec_u32 v5, off, s[8:11], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_dec_u32 v5, off, s[8:11], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_dec_u32 v5, off, s[8:11], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_dec_u32 v5, off, s[8:11], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_dec_u32 v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x00,0x10,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_dec_u32 v5, v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x00,0x10,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_dec_u32 v5, off, s[8:11], s3
+// GFX12: encoding: [0x03,0x00,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_dec_u32 v5, off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x00,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_dec_u32 v5, off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x00,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_dec_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0x00,0x10,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_dec_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX12: encoding: [0x03,0x00,0x10,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_dec_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0x00,0x10,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_dec_u32 v5, off, s[8:11], s3 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_dec_u32 v5, off, s[8:11], s3 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_dec_u32 v5, off, s[8:11], s3 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_dec_u32 v5, off, s[8:11], s3 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_dec_u64 v[5:6], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x13,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_dec_u64 v[254:255], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x13,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_dec_u64 v[5:6], off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x13,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_dec_u64 v[5:6], off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x13,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_dec_u64 v[5:6], off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x40,0x13,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_dec_u64 v[5:6], off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x40,0x13,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_dec_u64 v[5:6], off, s[8:11], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_dec_u64 v[5:6], off, s[8:11], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_dec_u64 v[5:6], off, s[8:11], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_dec_u64 v[5:6], off, s[8:11], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_dec_u64 v[5:6], v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x40,0x13,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_dec_u64 v[5:6], v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x40,0x13,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_dec_u64 v[5:6], off, s[8:11], s3
+// GFX12: encoding: [0x03,0x40,0x13,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_dec_u64 v[5:6], off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x40,0x13,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_dec_u64 v[5:6], off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x40,0x13,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_dec_u64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0x40,0x13,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_dec_u64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX12: encoding: [0x03,0x40,0x13,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_dec_u64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0x40,0x13,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_dec_u64 v[5:6], off, s[8:11], s3 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_dec_u64 v[5:6], off, s[8:11], s3 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_dec_u64 v[5:6], off, s[8:11], s3 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_dec_u64 v[5:6], off, s[8:11], s3 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_inc_u32 v5, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_inc_u32 v255, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x0f,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_inc_u32 v5, off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x0f,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_inc_u32 v5, off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x0f,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_inc_u32 v5, off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0xc0,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_inc_u32 v5, off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0xc0,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_inc_u32 v5, off, s[8:11], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_inc_u32 v5, off, s[8:11], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_inc_u32 v5, off, s[8:11], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_inc_u32 v5, off, s[8:11], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_inc_u32 v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x0f,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_inc_u32 v5, v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x0f,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_inc_u32 v5, off, s[8:11], s3
+// GFX12: encoding: [0x03,0xc0,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_inc_u32 v5, off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0xc0,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_inc_u32 v5, off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0xc0,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_inc_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0xc0,0x0f,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_inc_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX12: encoding: [0x03,0xc0,0x0f,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_inc_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0xc0,0x0f,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_inc_u32 v5, off, s[8:11], s3 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_inc_u32 v5, off, s[8:11], s3 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_inc_u32 v5, off, s[8:11], s3 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_inc_u32 v5, off, s[8:11], s3 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_inc_u64 v[5:6], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x13,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_inc_u64 v[254:255], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x13,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_inc_u64 v[5:6], off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x13,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_inc_u64 v[5:6], off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x13,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_inc_u64 v[5:6], off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x00,0x13,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_inc_u64 v[5:6], off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x00,0x13,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_inc_u64 v[5:6], off, s[8:11], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_inc_u64 v[5:6], off, s[8:11], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_inc_u64 v[5:6], off, s[8:11], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_inc_u64 v[5:6], off, s[8:11], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_inc_u64 v[5:6], v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x00,0x13,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_inc_u64 v[5:6], v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x00,0x13,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_inc_u64 v[5:6], off, s[8:11], s3
+// GFX12: encoding: [0x03,0x00,0x13,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_inc_u64 v[5:6], off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x00,0x13,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_inc_u64 v[5:6], off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x00,0x13,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_inc_u64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0x00,0x13,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_inc_u64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX12: encoding: [0x03,0x00,0x13,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_inc_u64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0x00,0x13,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_inc_u64 v[5:6], off, s[8:11], s3 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_inc_u64 v[5:6], off, s[8:11], s3 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_inc_u64 v[5:6], off, s[8:11], s3 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_inc_u64 v[5:6], off, s[8:11], s3 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_max_num_f32 v5, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_num_f32 v255, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x14,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_num_f32 v5, off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x14,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_num_f32 v5, off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x14,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_num_f32 v5, off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x80,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_num_f32 v5, off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x80,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_num_f32 v5, off, s[8:11], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_max_num_f32 v5, off, s[8:11], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_max_num_f32 v5, off, s[8:11], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_max_num_f32 v5, off, s[8:11], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_max_num_f32 v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x80,0x14,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_num_f32 v5, v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x80,0x14,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_num_f32 v5, off, s[8:11], s3
+// GFX12: encoding: [0x03,0x80,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_max_num_f32 v5, off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x80,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_max_num_f32 v5, off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x80,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_max_num_f32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0x80,0x14,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_num_f32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX12: encoding: [0x03,0x80,0x14,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_num_f32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0x80,0x14,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_num_f32 v5, off, s[8:11], s3 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_max_num_f32 v5, off, s[8:11], s3 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_max_num_f32 v5, off, s[8:11], s3 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_max_num_f32 v5, off, s[8:11], s3 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_max_i32 v5, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_i32 v255, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x0e,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_i32 v5, off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x0e,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_i32 v5, off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x0e,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_i32 v5, off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x80,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_i32 v5, off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x80,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_i32 v5, off, s[8:11], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_max_i32 v5, off, s[8:11], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_max_i32 v5, off, s[8:11], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_max_i32 v5, off, s[8:11], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_max_i32 v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x80,0x0e,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_i32 v5, v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x80,0x0e,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_i32 v5, off, s[8:11], s3
+// GFX12: encoding: [0x03,0x80,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_max_i32 v5, off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x80,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_max_i32 v5, off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x80,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_max_i32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0x80,0x0e,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_i32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX12: encoding: [0x03,0x80,0x0e,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_i32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0x80,0x0e,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_i32 v5, off, s[8:11], s3 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_max_i32 v5, off, s[8:11], s3 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_max_i32 v5, off, s[8:11], s3 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_max_i32 v5, off, s[8:11], s3 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_max_i64 v[5:6], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_i64 v[254:255], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x11,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_i64 v[5:6], off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x11,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_i64 v[5:6], off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x11,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_i64 v[5:6], off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0xc0,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_i64 v[5:6], off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0xc0,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_i64 v[5:6], off, s[8:11], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_max_i64 v[5:6], off, s[8:11], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_max_i64 v[5:6], off, s[8:11], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_max_i64 v[5:6], off, s[8:11], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_max_i64 v[5:6], v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x11,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_i64 v[5:6], v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x11,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_i64 v[5:6], off, s[8:11], s3
+// GFX12: encoding: [0x03,0xc0,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_max_i64 v[5:6], off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0xc0,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_max_i64 v[5:6], off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0xc0,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_max_i64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0xc0,0x11,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_i64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX12: encoding: [0x03,0xc0,0x11,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_i64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0xc0,0x11,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_i64 v[5:6], off, s[8:11], s3 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_max_i64 v[5:6], off, s[8:11], s3 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_max_i64 v[5:6], off, s[8:11], s3 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_max_i64 v[5:6], off, s[8:11], s3 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_max_u32 v5, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_u32 v255, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x0e,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_u32 v5, off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x0e,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_u32 v5, off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x0e,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_u32 v5, off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0xc0,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_u32 v5, off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0xc0,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_u32 v5, off, s[8:11], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_max_u32 v5, off, s[8:11], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_max_u32 v5, off, s[8:11], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_max_u32 v5, off, s[8:11], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_max_u32 v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x0e,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_u32 v5, v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x0e,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_u32 v5, off, s[8:11], s3
+// GFX12: encoding: [0x03,0xc0,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_max_u32 v5, off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0xc0,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_max_u32 v5, off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0xc0,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_max_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0xc0,0x0e,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX12: encoding: [0x03,0xc0,0x0e,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0xc0,0x0e,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_u32 v5, off, s[8:11], s3 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_max_u32 v5, off, s[8:11], s3 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_max_u32 v5, off, s[8:11], s3 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_max_u32 v5, off, s[8:11], s3 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_max_u64 v[5:6], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_u64 v[254:255], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x12,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_u64 v[5:6], off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x12,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_u64 v[5:6], off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x12,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_u64 v[5:6], off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x00,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_u64 v[5:6], off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x00,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_u64 v[5:6], off, s[8:11], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_max_u64 v[5:6], off, s[8:11], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_max_u64 v[5:6], off, s[8:11], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_max_u64 v[5:6], off, s[8:11], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_max_u64 v[5:6], v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x00,0x12,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_u64 v[5:6], v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x00,0x12,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_u64 v[5:6], off, s[8:11], s3
+// GFX12: encoding: [0x03,0x00,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_max_u64 v[5:6], off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x00,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_max_u64 v[5:6], off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x00,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_max_u64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0x00,0x12,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_u64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX12: encoding: [0x03,0x00,0x12,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_u64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0x00,0x12,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_u64 v[5:6], off, s[8:11], s3 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_max_u64 v[5:6], off, s[8:11], s3 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_max_u64 v[5:6], off, s[8:11], s3 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_max_u64 v[5:6], off, s[8:11], s3 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_min_f32 v5, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_f32 v255, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x14,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_f32 v5, off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x14,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_f32 v5, off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x14,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_f32 v5, off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x40,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_f32 v5, off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x40,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_f32 v5, off, s[8:11], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_min_f32 v5, off, s[8:11], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_min_f32 v5, off, s[8:11], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_min_f32 v5, off, s[8:11], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_min_f32 v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x40,0x14,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_f32 v5, v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x40,0x14,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_f32 v5, off, s[8:11], s3
+// GFX12: encoding: [0x03,0x40,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_min_f32 v5, off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x40,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_min_f32 v5, off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x40,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_min_f32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0x40,0x14,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_f32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX12: encoding: [0x03,0x40,0x14,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_f32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0x40,0x14,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_f32 v5, off, s[8:11], s3 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_min_f32 v5, off, s[8:11], s3 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_min_f32 v5, off, s[8:11], s3 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_min_f32 v5, off, s[8:11], s3 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_min_i32 v5, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_i32 v255, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x0e,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_i32 v5, off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x0e,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_i32 v5, off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x0e,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_i32 v5, off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x00,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_i32 v5, off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x00,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_i32 v5, off, s[8:11], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_min_i32 v5, off, s[8:11], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_min_i32 v5, off, s[8:11], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_min_i32 v5, off, s[8:11], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_min_i32 v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x00,0x0e,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_i32 v5, v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x00,0x0e,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_i32 v5, off, s[8:11], s3
+// GFX12: encoding: [0x03,0x00,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_min_i32 v5, off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x00,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_min_i32 v5, off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x00,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_min_i32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0x00,0x0e,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_i32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX12: encoding: [0x03,0x00,0x0e,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_i32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0x00,0x0e,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_i32 v5, off, s[8:11], s3 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_min_i32 v5, off, s[8:11], s3 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_min_i32 v5, off, s[8:11], s3 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_min_i32 v5, off, s[8:11], s3 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_min_i64 v[5:6], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_i64 v[254:255], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x11,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_i64 v[5:6], off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x11,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_i64 v[5:6], off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x11,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_i64 v[5:6], off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x40,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_i64 v[5:6], off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x40,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_i64 v[5:6], off, s[8:11], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_min_i64 v[5:6], off, s[8:11], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_min_i64 v[5:6], off, s[8:11], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_min_i64 v[5:6], off, s[8:11], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_min_i64 v[5:6], v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x40,0x11,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_i64 v[5:6], v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x40,0x11,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_i64 v[5:6], off, s[8:11], s3
+// GFX12: encoding: [0x03,0x40,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_min_i64 v[5:6], off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x40,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_min_i64 v[5:6], off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x40,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_min_i64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0x40,0x11,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_i64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX12: encoding: [0x03,0x40,0x11,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_i64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0x40,0x11,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_i64 v[5:6], off, s[8:11], s3 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_min_i64 v[5:6], off, s[8:11], s3 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_min_i64 v[5:6], off, s[8:11], s3 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_min_i64 v[5:6], off, s[8:11], s3 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_min_u32 v5, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_u32 v255, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x0e,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_u32 v5, off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x0e,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_u32 v5, off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x0e,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_u32 v5, off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x40,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_u32 v5, off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x40,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_u32 v5, off, s[8:11], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_min_u32 v5, off, s[8:11], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_min_u32 v5, off, s[8:11], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_min_u32 v5, off, s[8:11], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_min_u32 v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x40,0x0e,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_u32 v5, v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x40,0x0e,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_u32 v5, off, s[8:11], s3
+// GFX12: encoding: [0x03,0x40,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_min_u32 v5, off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x40,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_min_u32 v5, off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x40,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_min_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0x40,0x0e,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX12: encoding: [0x03,0x40,0x0e,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0x40,0x0e,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_u32 v5, off, s[8:11], s3 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_min_u32 v5, off, s[8:11], s3 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_min_u32 v5, off, s[8:11], s3 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_min_u32 v5, off, s[8:11], s3 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_min_u64 v[5:6], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_u64 v[254:255], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x11,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_u64 v[5:6], off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x11,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_u64 v[5:6], off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x11,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_u64 v[5:6], off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x80,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_u64 v[5:6], off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x80,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_u64 v[5:6], off, s[8:11], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_min_u64 v[5:6], off, s[8:11], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_min_u64 v[5:6], off, s[8:11], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_min_u64 v[5:6], off, s[8:11], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_min_u64 v[5:6], v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x80,0x11,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_u64 v[5:6], v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x80,0x11,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_u64 v[5:6], off, s[8:11], s3
+// GFX12: encoding: [0x03,0x80,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_min_u64 v[5:6], off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x80,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_min_u64 v[5:6], off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x80,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_min_u64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0x80,0x11,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_u64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX12: encoding: [0x03,0x80,0x11,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_u64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0x80,0x11,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_u64 v[5:6], off, s[8:11], s3 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_min_u64 v[5:6], off, s[8:11], s3 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_min_u64 v[5:6], off, s[8:11], s3 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_min_u64 v[5:6], off, s[8:11], s3 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_or_b32 v5, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_or_b32 v255, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x0f,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_or_b32 v5, off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x0f,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_or_b32 v5, off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x0f,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_or_b32 v5, off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x40,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_or_b32 v5, off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x40,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_or_b32 v5, off, s[8:11], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_or_b32 v5, off, s[8:11], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_or_b32 v5, off, s[8:11], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_or_b32 v5, off, s[8:11], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_or_b32 v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x40,0x0f,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_or_b32 v5, v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x40,0x0f,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_or_b32 v5, off, s[8:11], s3
+// GFX12: encoding: [0x03,0x40,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_or_b32 v5, off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x40,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_or_b32 v5, off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x40,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_or_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0x40,0x0f,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_or_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX12: encoding: [0x03,0x40,0x0f,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_or_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0x40,0x0f,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_or_b32 v5, off, s[8:11], s3 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_or_b32 v5, off, s[8:11], s3 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_or_b32 v5, off, s[8:11], s3 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_or_b32 v5, off, s[8:11], s3 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_or_b64 v[5:6], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_or_b64 v[254:255], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x12,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_or_b64 v[5:6], off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x12,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_or_b64 v[5:6], off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x12,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_or_b64 v[5:6], off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x80,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_or_b64 v[5:6], off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x80,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_or_b64 v[5:6], off, s[8:11], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_or_b64 v[5:6], off, s[8:11], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_or_b64 v[5:6], off, s[8:11], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_or_b64 v[5:6], off, s[8:11], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_or_b64 v[5:6], v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x80,0x12,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_or_b64 v[5:6], v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x80,0x12,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_or_b64 v[5:6], off, s[8:11], s3
+// GFX12: encoding: [0x03,0x80,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_or_b64 v[5:6], off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x80,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_or_b64 v[5:6], off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x80,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_or_b64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0x80,0x12,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_or_b64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX12: encoding: [0x03,0x80,0x12,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_or_b64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0x80,0x12,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_or_b64 v[5:6], off, s[8:11], s3 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_or_b64 v[5:6], off, s[8:11], s3 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_or_b64 v[5:6], off, s[8:11], s3 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_or_b64 v[5:6], off, s[8:11], s3 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_sub_u32 v5, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_u32 v255, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x0d,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_u32 v5, off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x0d,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_u32 v5, off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x0d,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_u32 v5, off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x80,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_u32 v5, off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x80,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_u32 v5, off, s[8:11], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_sub_u32 v5, off, s[8:11], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_sub_u32 v5, off, s[8:11], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_sub_u32 v5, off, s[8:11], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_sub_u32 v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x80,0x0d,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_u32 v5, v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x80,0x0d,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_u32 v5, off, s[8:11], s3
+// GFX12: encoding: [0x03,0x80,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_sub_u32 v5, off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x80,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_sub_u32 v5, off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x80,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_sub_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0x80,0x0d,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX12: encoding: [0x03,0x80,0x0d,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0x80,0x0d,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_u32 v5, off, s[8:11], s3 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_sub_u32 v5, off, s[8:11], s3 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_sub_u32 v5, off, s[8:11], s3 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_sub_u32 v5, off, s[8:11], s3 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_sub_u64 v[5:6], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_u64 v[254:255], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x11,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_u64 v[5:6], off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x11,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_u64 v[5:6], off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x11,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_u64 v[5:6], off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x00,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_u64 v[5:6], off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x00,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_u64 v[5:6], off, s[8:11], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_sub_u64 v[5:6], off, s[8:11], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_sub_u64 v[5:6], off, s[8:11], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_sub_u64 v[5:6], off, s[8:11], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_sub_u64 v[5:6], v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x00,0x11,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_u64 v[5:6], v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x00,0x11,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_u64 v[5:6], off, s[8:11], s3
+// GFX12: encoding: [0x03,0x00,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_sub_u64 v[5:6], off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x00,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_sub_u64 v[5:6], off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x00,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_sub_u64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0x00,0x11,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_u64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX12: encoding: [0x03,0x00,0x11,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_u64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0x00,0x11,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_u64 v[5:6], off, s[8:11], s3 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_sub_u64 v[5:6], off, s[8:11], s3 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_sub_u64 v[5:6], off, s[8:11], s3 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_sub_u64 v[5:6], off, s[8:11], s3 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_swap_b32 v5, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x0c,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_swap_b32 v255, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x0c,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_swap_b32 v5, off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x0c,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_swap_b32 v5, off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x0c,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_swap_b32 v5, off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0xc0,0x0c,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_swap_b32 v5, off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0xc0,0x0c,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_swap_b32 v5, off, s[8:11], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_swap_b32 v5, off, s[8:11], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_swap_b32 v5, off, s[8:11], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_swap_b32 v5, off, s[8:11], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_swap_b32 v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x0c,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_swap_b32 v5, v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x0c,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_swap_b32 v5, off, s[8:11], s3
+// GFX12: encoding: [0x03,0xc0,0x0c,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_swap_b32 v5, off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0xc0,0x0c,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_swap_b32 v5, off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0xc0,0x0c,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_swap_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0xc0,0x0c,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_swap_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX12: encoding: [0x03,0xc0,0x0c,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_swap_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0xc0,0x0c,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_swap_b32 v5, off, s[8:11], s3 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_swap_b32 v5, off, s[8:11], s3 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_swap_b32 v5, off, s[8:11], s3 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_swap_b32 v5, off, s[8:11], s3 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_swap_b64 v[5:6], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_swap_b64 v[254:255], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x10,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_swap_b64 v[5:6], off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x10,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_swap_b64 v[5:6], off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x10,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_swap_b64 v[5:6], off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x40,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_swap_b64 v[5:6], off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x40,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_swap_b64 v[5:6], off, s[8:11], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_swap_b64 v[5:6], off, s[8:11], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_swap_b64 v[5:6], off, s[8:11], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_swap_b64 v[5:6], off, s[8:11], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_swap_b64 v[5:6], v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x40,0x10,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_swap_b64 v[5:6], v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x40,0x10,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_swap_b64 v[5:6], off, s[8:11], s3
+// GFX12: encoding: [0x03,0x40,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_swap_b64 v[5:6], off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x40,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_swap_b64 v[5:6], off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x40,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_swap_b64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0x40,0x10,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_swap_b64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX12: encoding: [0x03,0x40,0x10,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_swap_b64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0x40,0x10,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_swap_b64 v[5:6], off, s[8:11], s3 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_swap_b64 v[5:6], off, s[8:11], s3 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_swap_b64 v[5:6], off, s[8:11], s3 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_swap_b64 v[5:6], off, s[8:11], s3 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_xor_b32 v5, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_xor_b32 v255, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x0f,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_xor_b32 v5, off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x0f,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_xor_b32 v5, off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x0f,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_xor_b32 v5, off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x80,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_xor_b32 v5, off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x80,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_xor_b32 v5, off, s[8:11], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_xor_b32 v5, off, s[8:11], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_xor_b32 v5, off, s[8:11], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_xor_b32 v5, off, s[8:11], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_xor_b32 v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x80,0x0f,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_xor_b32 v5, v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x80,0x0f,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_xor_b32 v5, off, s[8:11], s3
+// GFX12: encoding: [0x03,0x80,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_xor_b32 v5, off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x80,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_xor_b32 v5, off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x80,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_xor_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0x80,0x0f,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_xor_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX12: encoding: [0x03,0x80,0x0f,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_xor_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0x80,0x0f,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_xor_b32 v5, off, s[8:11], s3 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_xor_b32 v5, off, s[8:11], s3 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_xor_b32 v5, off, s[8:11], s3 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_xor_b32 v5, off, s[8:11], s3 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_xor_b64 v[5:6], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_xor_b64 v[254:255], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x12,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_xor_b64 v[5:6], off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x12,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_xor_b64 v[5:6], off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x12,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_xor_b64 v[5:6], off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0xc0,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_xor_b64 v[5:6], off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0xc0,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_xor_b64 v[5:6], off, s[8:11], 0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_xor_b64 v[5:6], off, s[8:11], -1 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_xor_b64 v[5:6], off, s[8:11], 0.5 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_xor_b64 v[5:6], off, s[8:11], -4.0 offset:8388607
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: operands are not valid for this GPU or mode
+
+buffer_atomic_xor_b64 v[5:6], v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x12,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_xor_b64 v[5:6], v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x12,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_xor_b64 v[5:6], off, s[8:11], s3
+// GFX12: encoding: [0x03,0xc0,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_xor_b64 v[5:6], off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0xc0,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_xor_b64 v[5:6], off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0xc0,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_xor_b64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0xc0,0x12,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_xor_b64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RT_RETURN scope:SCOPE_SE
+// GFX12: encoding: [0x03,0xc0,0x12,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_xor_b64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV
+// GFX12: encoding: [0x03,0xc0,0x12,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_xor_b64 v[5:6], off, s[8:11], s3 offset:8388607 glc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_xor_b64 v[5:6], off, s[8:11], s3 offset:8388607 slc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_xor_b64 v[5:6], off, s[8:11], s3 offset:8388607 dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_atomic_xor_b64 v[5:6], off, s[8:11], s3 offset:8388607 glc slc dlc
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vbuffer_mubuf_alias.s b/llvm/test/MC/AMDGPU/gfx12_asm_vbuffer_mubuf_alias.s
new file mode 100644
index 0000000000000000000000000000000000000000..6a8a4ef567b96566778778ee268b4995bc3c2cd0
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vbuffer_mubuf_alias.s
@@ -0,0 +1,2214 @@
+// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -show-encoding %s | FileCheck --check-prefix=GFX12 %s
+// RUN: not llvm-mc -arch=amdgcn -mcpu=gfx1200 -show-encoding %s 2>&1 | FileCheck --check-prefixes=GFX12-ERR --implicit-check-not=error: %s
+
+buffer_load_dword v5, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_dword v255, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x05,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_dword v5, off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x05,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_dword v5, off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x05,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_dword v5, off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x00,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_dword v5, off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x00,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_dword v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x00,0x05,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_load_dword v5, v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x00,0x05,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_load_dword v5, off, s[8:11], s3
+// GFX12: encoding: [0x03,0x00,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_dword v5, off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x00,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_dword v5, off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x00,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_load_dword v5, off, s[8:11], s3 offset:8388607 lds
+// GFX12-ERR: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction
+
+buffer_load_dwordx2 v[5:6], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_dwordx2 v[254:255], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x05,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_dwordx2 v[5:6], off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x05,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_dwordx2 v[5:6], off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x05,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_dwordx2 v[5:6], off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x40,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_dwordx2 v[5:6], off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x40,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_dwordx2 v[5:6], v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x40,0x05,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_load_dwordx2 v[5:6], v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x40,0x05,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_load_dwordx2 v[5:6], off, s[8:11], s3
+// GFX12: encoding: [0x03,0x40,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_dwordx2 v[5:6], off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x40,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_dwordx2 v[5:6], off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x40,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_load_dwordx3 v[5:7], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_dwordx3 v[253:255], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x05,0xc4,0xfd,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_dwordx3 v[5:7], off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x05,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_dwordx3 v[5:7], off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x05,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_dwordx3 v[5:7], off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x80,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_dwordx3 v[5:7], off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x80,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_dwordx3 v[5:7], v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x80,0x05,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_load_dwordx3 v[5:7], v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x80,0x05,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_load_dwordx3 v[5:7], off, s[8:11], s3
+// GFX12: encoding: [0x03,0x80,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_dwordx3 v[5:7], off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x80,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_dwordx3 v[5:7], off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x80,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_load_dwordx4 v[5:8], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_dwordx4 v[252:255], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x05,0xc4,0xfc,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_dwordx4 v[5:8], off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x05,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_dwordx4 v[5:8], off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x05,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_dwordx4 v[5:8], off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0xc0,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_dwordx4 v[5:8], off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0xc0,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_dwordx4 v[5:8], v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x05,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_load_dwordx4 v[5:8], v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x05,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_load_dwordx4 v[5:8], off, s[8:11], s3
+// GFX12: encoding: [0x03,0xc0,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_dwordx4 v[5:8], off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0xc0,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_dwordx4 v[5:8], off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0xc0,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_load_short_d16 v5, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_short_d16 v255, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x08,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_short_d16 v5, off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x08,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_short_d16 v5, off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x08,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_short_d16 v5, off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x00,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_short_d16 v5, off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x00,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_short_d16 v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x00,0x08,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_load_short_d16 v5, v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x00,0x08,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_load_short_d16 v5, off, s[8:11], s3
+// GFX12: encoding: [0x03,0x00,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_short_d16 v5, off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x00,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_short_d16 v5, off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x00,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_load_format_d16_x v5, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_d16_x v255, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x02,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_d16_x v5, off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x02,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_d16_x v5, off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x02,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_d16_x v5, off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x00,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_d16_x v5, off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x00,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_d16_x v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x00,0x02,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_d16_x v5, v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x00,0x02,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_d16_x v5, off, s[8:11], s3
+// GFX12: encoding: [0x03,0x00,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_format_d16_x v5, off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x00,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_format_d16_x v5, off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x00,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_load_format_d16_xy v5, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_d16_xy v255, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x02,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_d16_xy v5, off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x02,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_d16_xy v5, off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x02,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_d16_xy v5, off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x40,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_d16_xy v5, off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x40,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_d16_xy v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x40,0x02,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_d16_xy v5, v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x40,0x02,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_d16_xy v5, off, s[8:11], s3
+// GFX12: encoding: [0x03,0x40,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_format_d16_xy v5, off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x40,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_format_d16_xy v5, off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x40,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_load_format_d16_xyz v[5:6], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_d16_xyz v[254:255], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x02,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_d16_xyz v[5:6], off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x02,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_d16_xyz v[5:6], off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x02,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_d16_xyz v[5:6], off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x80,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_d16_xyz v[5:6], off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x80,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_d16_xyz v[5:6], v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x80,0x02,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_d16_xyz v[5:6], v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x80,0x02,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_d16_xyz v[5:6], off, s[8:11], s3
+// GFX12: encoding: [0x03,0x80,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_format_d16_xyz v[5:6], off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x80,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_format_d16_xyz v[5:6], off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x80,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_load_format_d16_xyzw v[5:6], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_d16_xyzw v[254:255], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x02,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_d16_xyzw v[5:6], off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x02,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_d16_xyzw v[5:6], off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x02,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_d16_xyzw v[5:6], off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0xc0,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_d16_xyzw v[5:6], off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0xc0,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_d16_xyzw v[5:6], v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x02,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_d16_xyzw v[5:6], v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x02,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_d16_xyzw v[5:6], off, s[8:11], s3
+// GFX12: encoding: [0x03,0xc0,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_format_d16_xyzw v[5:6], off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0xc0,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_format_d16_xyzw v[5:6], off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0xc0,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_load_short_d16_hi v5, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_short_d16_hi v255, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x08,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_short_d16_hi v5, off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x08,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_short_d16_hi v5, off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x08,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_short_d16_hi v5, off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0xc0,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_short_d16_hi v5, off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0xc0,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_short_d16_hi v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x08,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_load_short_d16_hi v5, v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x08,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_load_short_d16_hi v5, off, s[8:11], s3
+// GFX12: encoding: [0x03,0xc0,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_short_d16_hi v5, off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0xc0,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_short_d16_hi v5, off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0xc0,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_load_format_d16_hi_x v5, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x09,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_d16_hi_x v255, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x09,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_d16_hi_x v5, off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x09,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_d16_hi_x v5, off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x09,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_d16_hi_x v5, off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x80,0x09,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_d16_hi_x v5, off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x80,0x09,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_d16_hi_x v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x80,0x09,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_d16_hi_x v5, v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x80,0x09,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_load_format_d16_hi_x v5, off, s[8:11], s3
+// GFX12: encoding: [0x03,0x80,0x09,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_format_d16_hi_x v5, off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x80,0x09,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_format_d16_hi_x v5, off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x80,0x09,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_load_sbyte_d16_hi v5, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_sbyte_d16_hi v255, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x08,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_sbyte_d16_hi v5, off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x08,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_sbyte_d16_hi v5, off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x08,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_sbyte_d16_hi v5, off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x80,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_sbyte_d16_hi v5, off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x80,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_sbyte_d16_hi v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x80,0x08,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_load_sbyte_d16_hi v5, v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x80,0x08,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_load_sbyte_d16_hi v5, off, s[8:11], s3
+// GFX12: encoding: [0x03,0x80,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_sbyte_d16_hi v5, off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x80,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_sbyte_d16_hi v5, off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x80,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_load_ubyte_d16_hi v5, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_ubyte_d16_hi v255, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x08,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_ubyte_d16_hi v5, off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x08,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_ubyte_d16_hi v5, off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x08,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_ubyte_d16_hi v5, off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x40,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_ubyte_d16_hi v5, off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x40,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_ubyte_d16_hi v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x40,0x08,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_load_ubyte_d16_hi v5, v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x40,0x08,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_load_ubyte_d16_hi v5, off, s[8:11], s3
+// GFX12: encoding: [0x03,0x40,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_ubyte_d16_hi v5, off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x40,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_ubyte_d16_hi v5, off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x40,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_load_sbyte_d16 v5, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_sbyte_d16 v255, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x07,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_sbyte_d16 v5, off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x07,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_sbyte_d16 v5, off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x07,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_sbyte_d16 v5, off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0xc0,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_sbyte_d16 v5, off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0xc0,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_sbyte_d16 v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x07,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_load_sbyte_d16 v5, v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x07,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_load_sbyte_d16 v5, off, s[8:11], s3
+// GFX12: encoding: [0x03,0xc0,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_sbyte_d16 v5, off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0xc0,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_sbyte_d16 v5, off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0xc0,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_load_ubyte_d16 v5, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_ubyte_d16 v255, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x07,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_ubyte_d16 v5, off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x07,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_ubyte_d16 v5, off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x07,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_ubyte_d16 v5, off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x80,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_ubyte_d16 v5, off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x80,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_ubyte_d16 v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x80,0x07,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_load_ubyte_d16 v5, v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x80,0x07,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_load_ubyte_d16 v5, off, s[8:11], s3
+// GFX12: encoding: [0x03,0x80,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_ubyte_d16 v5, off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x80,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_ubyte_d16 v5, off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x80,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_load_sbyte v5, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_sbyte v255, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x04,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_sbyte v5, off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x04,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_sbyte v5, off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x04,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_sbyte v5, off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x40,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_sbyte v5, off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x40,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_sbyte v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x40,0x04,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_load_sbyte v5, v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x40,0x04,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_load_sbyte v5, off, s[8:11], s3
+// GFX12: encoding: [0x03,0x40,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_sbyte v5, off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x40,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_sbyte v5, off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x40,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_load_sshort v5, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_sshort v255, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x04,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_sshort v5, off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x04,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_sshort v5, off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x04,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_sshort v5, off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0xc0,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_sshort v5, off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0xc0,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_sshort v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x04,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_load_sshort v5, v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x04,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_load_sshort v5, off, s[8:11], s3
+// GFX12: encoding: [0x03,0xc0,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_sshort v5, off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0xc0,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_sshort v5, off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0xc0,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_load_ubyte v5, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_ubyte v255, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x04,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_ubyte v5, off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x04,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_ubyte v5, off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x04,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_ubyte v5, off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x00,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_ubyte v5, off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x00,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_ubyte v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x00,0x04,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_load_ubyte v5, v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x00,0x04,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_load_ubyte v5, off, s[8:11], s3
+// GFX12: encoding: [0x03,0x00,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_ubyte v5, off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x00,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_ubyte v5, off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x00,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_load_ushort v5, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_ushort v255, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x04,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_ushort v5, off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x04,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_ushort v5, off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x04,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_ushort v5, off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x80,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_ushort v5, off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x80,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_load_ushort v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x80,0x04,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_load_ushort v5, v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x80,0x04,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_load_ushort v5, off, s[8:11], s3
+// GFX12: encoding: [0x03,0x80,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_ushort v5, off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x80,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_load_ushort v5, off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x80,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+
+buffer_store_byte v1, off, s[12:15], s4 offset:8388607
+// GFX12: encoding: [0x04,0x00,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_byte v255, off, s[12:15], s4 offset:8388607
+// GFX12: encoding: [0x04,0x00,0x06,0xc4,0xff,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_byte v1, off, s[16:19], s4 offset:8388607
+// GFX12: encoding: [0x04,0x00,0x06,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_byte v1, off, s[96:99], s4 offset:8388607
+// GFX12: encoding: [0x04,0x00,0x06,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_byte v1, off, s[12:15], s101 offset:8388607
+// GFX12: encoding: [0x65,0x00,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_byte v1, off, s[12:15], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x00,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_byte v1, v0, s[12:15], s4 idxen offset:8388607
+// GFX12: encoding: [0x04,0x00,0x06,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_store_byte v1, v0, s[12:15], s4 offen offset:8388607
+// GFX12: encoding: [0x04,0x00,0x06,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_store_byte v1, off, s[12:15], s4
+// GFX12: encoding: [0x04,0x00,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_byte v1, off, s[12:15], s4 offset:0
+// GFX12: encoding: [0x04,0x00,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_byte v1, off, s[12:15], s4 offset:7
+// GFX12: encoding: [0x04,0x00,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_store_short v1, off, s[12:15], s4 offset:8388607
+// GFX12: encoding: [0x04,0x40,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_short v255, off, s[12:15], s4 offset:8388607
+// GFX12: encoding: [0x04,0x40,0x06,0xc4,0xff,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_short v1, off, s[16:19], s4 offset:8388607
+// GFX12: encoding: [0x04,0x40,0x06,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_short v1, off, s[96:99], s4 offset:8388607
+// GFX12: encoding: [0x04,0x40,0x06,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_short v1, off, s[12:15], s101 offset:8388607
+// GFX12: encoding: [0x65,0x40,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_short v1, off, s[12:15], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x40,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_short v1, v0, s[12:15], s4 idxen offset:8388607
+// GFX12: encoding: [0x04,0x40,0x06,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_store_short v1, v0, s[12:15], s4 offen offset:8388607
+// GFX12: encoding: [0x04,0x40,0x06,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_store_short v1, off, s[12:15], s4
+// GFX12: encoding: [0x04,0x40,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_short v1, off, s[12:15], s4 offset:0
+// GFX12: encoding: [0x04,0x40,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_short v1, off, s[12:15], s4 offset:7
+// GFX12: encoding: [0x04,0x40,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_store_dword v1, off, s[12:15], s4 offset:8388607
+// GFX12: encoding: [0x04,0x80,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_dword v255, off, s[12:15], s4 offset:8388607
+// GFX12: encoding: [0x04,0x80,0x06,0xc4,0xff,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_dword v1, off, s[16:19], s4 offset:8388607
+// GFX12: encoding: [0x04,0x80,0x06,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_dword v1, off, s[96:99], s4 offset:8388607
+// GFX12: encoding: [0x04,0x80,0x06,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_dword v1, off, s[12:15], s101 offset:8388607
+// GFX12: encoding: [0x65,0x80,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_dword v1, off, s[12:15], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x80,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_dword v1, v0, s[12:15], s4 idxen offset:8388607
+// GFX12: encoding: [0x04,0x80,0x06,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_store_dword v1, v0, s[12:15], s4 offen offset:8388607
+// GFX12: encoding: [0x04,0x80,0x06,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_store_dword v1, off, s[12:15], s4
+// GFX12: encoding: [0x04,0x80,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_dword v1, off, s[12:15], s4 offset:0
+// GFX12: encoding: [0x04,0x80,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_dword v1, off, s[12:15], s4 offset:7
+// GFX12: encoding: [0x04,0x80,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_store_dwordx2 v[1:2], off, s[12:15], s4 offset:8388607
+// GFX12: encoding: [0x04,0xc0,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_dwordx2 v[254:255], off, s[12:15], s4 offset:8388607
+// GFX12: encoding: [0x04,0xc0,0x06,0xc4,0xfe,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_dwordx2 v[1:2], off, s[16:19], s4 offset:8388607
+// GFX12: encoding: [0x04,0xc0,0x06,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_dwordx2 v[1:2], off, s[96:99], s4 offset:8388607
+// GFX12: encoding: [0x04,0xc0,0x06,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_dwordx2 v[1:2], off, s[12:15], s101 offset:8388607
+// GFX12: encoding: [0x65,0xc0,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_dwordx2 v[1:2], off, s[12:15], m0 offset:8388607
+// GFX12: encoding: [0x7d,0xc0,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_dwordx2 v[1:2], v0, s[12:15], s4 idxen offset:8388607
+// GFX12: encoding: [0x04,0xc0,0x06,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_store_dwordx2 v[1:2], v0, s[12:15], s4 offen offset:8388607
+// GFX12: encoding: [0x04,0xc0,0x06,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_store_dwordx2 v[1:2], off, s[12:15], s4
+// GFX12: encoding: [0x04,0xc0,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_dwordx2 v[1:2], off, s[12:15], s4 offset:0
+// GFX12: encoding: [0x04,0xc0,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_dwordx2 v[1:2], off, s[12:15], s4 offset:7
+// GFX12: encoding: [0x04,0xc0,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_store_dwordx3 v[1:3], off, s[12:15], s4 offset:8388607
+// GFX12: encoding: [0x04,0x00,0x07,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_dwordx3 v[253:255], off, s[12:15], s4 offset:8388607
+// GFX12: encoding: [0x04,0x00,0x07,0xc4,0xfd,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_dwordx3 v[1:3], off, s[16:19], s4 offset:8388607
+// GFX12: encoding: [0x04,0x00,0x07,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_dwordx3 v[1:3], off, s[96:99], s4 offset:8388607
+// GFX12: encoding: [0x04,0x00,0x07,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_dwordx3 v[1:3], off, s[12:15], s101 offset:8388607
+// GFX12: encoding: [0x65,0x00,0x07,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_dwordx3 v[1:3], off, s[12:15], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x00,0x07,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_dwordx3 v[1:3], v0, s[12:15], s4 idxen offset:8388607
+// GFX12: encoding: [0x04,0x00,0x07,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_store_dwordx3 v[1:3], v0, s[12:15], s4 offen offset:8388607
+// GFX12: encoding: [0x04,0x00,0x07,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_store_dwordx3 v[1:3], off, s[12:15], s4
+// GFX12: encoding: [0x04,0x00,0x07,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_dwordx3 v[1:3], off, s[12:15], s4 offset:0
+// GFX12: encoding: [0x04,0x00,0x07,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_dwordx3 v[1:3], off, s[12:15], s4 offset:7
+// GFX12: encoding: [0x04,0x00,0x07,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_store_dwordx4 v[1:4], off, s[12:15], s4 offset:8388607
+// GFX12: encoding: [0x04,0x40,0x07,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_dwordx4 v[252:255], off, s[12:15], s4 offset:8388607
+// GFX12: encoding: [0x04,0x40,0x07,0xc4,0xfc,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_dwordx4 v[1:4], off, s[16:19], s4 offset:8388607
+// GFX12: encoding: [0x04,0x40,0x07,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_dwordx4 v[1:4], off, s[96:99], s4 offset:8388607
+// GFX12: encoding: [0x04,0x40,0x07,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_dwordx4 v[1:4], off, s[12:15], s101 offset:8388607
+// GFX12: encoding: [0x65,0x40,0x07,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_dwordx4 v[1:4], off, s[12:15], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x40,0x07,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_dwordx4 v[1:4], v0, s[12:15], s4 idxen offset:8388607
+// GFX12: encoding: [0x04,0x40,0x07,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_store_dwordx4 v[1:4], v0, s[12:15], s4 offen offset:8388607
+// GFX12: encoding: [0x04,0x40,0x07,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_store_dwordx4 v[1:4], off, s[12:15], s4
+// GFX12: encoding: [0x04,0x40,0x07,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_dwordx4 v[1:4], off, s[12:15], s4 offset:0
+// GFX12: encoding: [0x04,0x40,0x07,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_dwordx4 v[1:4], off, s[12:15], s4 offset:7
+// GFX12: encoding: [0x04,0x40,0x07,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_store_format_d16_x v1, off, s[12:15], s4 offset:8388607
+// GFX12: encoding: [0x04,0x00,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_d16_x v255, off, s[12:15], s4 offset:8388607
+// GFX12: encoding: [0x04,0x00,0x03,0xc4,0xff,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_d16_x v1, off, s[16:19], s4 offset:8388607
+// GFX12: encoding: [0x04,0x00,0x03,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_d16_x v1, off, s[96:99], s4 offset:8388607
+// GFX12: encoding: [0x04,0x00,0x03,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_d16_x v1, off, s[12:15], s101 offset:8388607
+// GFX12: encoding: [0x65,0x00,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_d16_x v1, off, s[12:15], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x00,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_d16_x v1, v0, s[12:15], s4 idxen offset:8388607
+// GFX12: encoding: [0x04,0x00,0x03,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_d16_x v1, v0, s[12:15], s4 offen offset:8388607
+// GFX12: encoding: [0x04,0x00,0x03,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_d16_x v1, off, s[12:15], s4
+// GFX12: encoding: [0x04,0x00,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_format_d16_x v1, off, s[12:15], s4 offset:0
+// GFX12: encoding: [0x04,0x00,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_format_d16_x v1, off, s[12:15], s4 offset:7
+// GFX12: encoding: [0x04,0x00,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_store_format_d16_xy v1, off, s[12:15], s4 offset:8388607
+// GFX12: encoding: [0x04,0x40,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_d16_xy v255, off, s[12:15], s4 offset:8388607
+// GFX12: encoding: [0x04,0x40,0x03,0xc4,0xff,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_d16_xy v1, off, s[16:19], s4 offset:8388607
+// GFX12: encoding: [0x04,0x40,0x03,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_d16_xy v1, off, s[96:99], s4 offset:8388607
+// GFX12: encoding: [0x04,0x40,0x03,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_d16_xy v1, off, s[12:15], s101 offset:8388607
+// GFX12: encoding: [0x65,0x40,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_d16_xy v1, off, s[12:15], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x40,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_d16_xy v1, v0, s[12:15], s4 idxen offset:8388607
+// GFX12: encoding: [0x04,0x40,0x03,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_d16_xy v1, v0, s[12:15], s4 offen offset:8388607
+// GFX12: encoding: [0x04,0x40,0x03,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_d16_xy v1, off, s[12:15], s4
+// GFX12: encoding: [0x04,0x40,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_format_d16_xy v1, off, s[12:15], s4 offset:0
+// GFX12: encoding: [0x04,0x40,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_format_d16_xy v1, off, s[12:15], s4 offset:7
+// GFX12: encoding: [0x04,0x40,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_store_format_d16_xyz v[1:2], off, s[12:15], s4 offset:8388607
+// GFX12: encoding: [0x04,0x80,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_d16_xyz v[254:255], off, s[12:15], s4 offset:8388607
+// GFX12: encoding: [0x04,0x80,0x03,0xc4,0xfe,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_d16_xyz v[1:2], off, s[16:19], s4 offset:8388607
+// GFX12: encoding: [0x04,0x80,0x03,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_d16_xyz v[1:2], off, s[96:99], s4 offset:8388607
+// GFX12: encoding: [0x04,0x80,0x03,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_d16_xyz v[1:2], off, s[12:15], s101 offset:8388607
+// GFX12: encoding: [0x65,0x80,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_d16_xyz v[1:2], off, s[12:15], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x80,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_d16_xyz v[1:2], v0, s[12:15], s4 idxen offset:8388607
+// GFX12: encoding: [0x04,0x80,0x03,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_d16_xyz v[1:2], v0, s[12:15], s4 offen offset:8388607
+// GFX12: encoding: [0x04,0x80,0x03,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_d16_xyz v[1:2], off, s[12:15], s4
+// GFX12: encoding: [0x04,0x80,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_format_d16_xyz v[1:2], off, s[12:15], s4 offset:0
+// GFX12: encoding: [0x04,0x80,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_format_d16_xyz v[1:2], off, s[12:15], s4 offset:7
+// GFX12: encoding: [0x04,0x80,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_store_format_d16_xyzw v[1:2], off, s[12:15], s4 offset:8388607
+// GFX12: encoding: [0x04,0xc0,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_d16_xyzw v[254:255], off, s[12:15], s4 offset:8388607
+// GFX12: encoding: [0x04,0xc0,0x03,0xc4,0xfe,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_d16_xyzw v[1:2], off, s[16:19], s4 offset:8388607
+// GFX12: encoding: [0x04,0xc0,0x03,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_d16_xyzw v[1:2], off, s[96:99], s4 offset:8388607
+// GFX12: encoding: [0x04,0xc0,0x03,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_d16_xyzw v[1:2], off, s[12:15], s101 offset:8388607
+// GFX12: encoding: [0x65,0xc0,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_d16_xyzw v[1:2], off, s[12:15], m0 offset:8388607
+// GFX12: encoding: [0x7d,0xc0,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_d16_xyzw v[1:2], v0, s[12:15], s4 idxen offset:8388607
+// GFX12: encoding: [0x04,0xc0,0x03,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_d16_xyzw v[1:2], v0, s[12:15], s4 offen offset:8388607
+// GFX12: encoding: [0x04,0xc0,0x03,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_d16_xyzw v[1:2], off, s[12:15], s4
+// GFX12: encoding: [0x04,0xc0,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_format_d16_xyzw v[1:2], off, s[12:15], s4 offset:0
+// GFX12: encoding: [0x04,0xc0,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_format_d16_xyzw v[1:2], off, s[12:15], s4 offset:7
+// GFX12: encoding: [0x04,0xc0,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_store_byte_d16_hi v1, off, s[12:15], s4 offset:8388607
+// GFX12: encoding: [0x04,0x00,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_byte_d16_hi v255, off, s[12:15], s4 offset:8388607
+// GFX12: encoding: [0x04,0x00,0x09,0xc4,0xff,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_byte_d16_hi v1, off, s[16:19], s4 offset:8388607
+// GFX12: encoding: [0x04,0x00,0x09,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_byte_d16_hi v1, off, s[96:99], s4 offset:8388607
+// GFX12: encoding: [0x04,0x00,0x09,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_byte_d16_hi v1, off, s[12:15], s101 offset:8388607
+// GFX12: encoding: [0x65,0x00,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_byte_d16_hi v1, off, s[12:15], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x00,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_byte_d16_hi v1, v0, s[12:15], s4 idxen offset:8388607
+// GFX12: encoding: [0x04,0x00,0x09,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_store_byte_d16_hi v1, v0, s[12:15], s4 offen offset:8388607
+// GFX12: encoding: [0x04,0x00,0x09,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_store_byte_d16_hi v1, off, s[12:15], s4
+// GFX12: encoding: [0x04,0x00,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_byte_d16_hi v1, off, s[12:15], s4 offset:0
+// GFX12: encoding: [0x04,0x00,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_byte_d16_hi v1, off, s[12:15], s4 offset:7
+// GFX12: encoding: [0x04,0x00,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_store_short_d16_hi v1, off, s[12:15], s4 offset:8388607
+// GFX12: encoding: [0x04,0x40,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_short_d16_hi v255, off, s[12:15], s4 offset:8388607
+// GFX12: encoding: [0x04,0x40,0x09,0xc4,0xff,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_short_d16_hi v1, off, s[16:19], s4 offset:8388607
+// GFX12: encoding: [0x04,0x40,0x09,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_short_d16_hi v1, off, s[96:99], s4 offset:8388607
+// GFX12: encoding: [0x04,0x40,0x09,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_short_d16_hi v1, off, s[12:15], s101 offset:8388607
+// GFX12: encoding: [0x65,0x40,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_short_d16_hi v1, off, s[12:15], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x40,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_short_d16_hi v1, v0, s[12:15], s4 idxen offset:8388607
+// GFX12: encoding: [0x04,0x40,0x09,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_store_short_d16_hi v1, v0, s[12:15], s4 offen offset:8388607
+// GFX12: encoding: [0x04,0x40,0x09,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_store_short_d16_hi v1, off, s[12:15], s4
+// GFX12: encoding: [0x04,0x40,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_short_d16_hi v1, off, s[12:15], s4 offset:0
+// GFX12: encoding: [0x04,0x40,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_short_d16_hi v1, off, s[12:15], s4 offset:7
+// GFX12: encoding: [0x04,0x40,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_store_format_d16_hi_x v1, off, s[12:15], s4 offset:8388607
+// GFX12: encoding: [0x04,0xc0,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_d16_hi_x v255, off, s[12:15], s4 offset:8388607
+// GFX12: encoding: [0x04,0xc0,0x09,0xc4,0xff,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_d16_hi_x v1, off, s[16:19], s4 offset:8388607
+// GFX12: encoding: [0x04,0xc0,0x09,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_d16_hi_x v1, off, s[96:99], s4 offset:8388607
+// GFX12: encoding: [0x04,0xc0,0x09,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_d16_hi_x v1, off, s[12:15], s101 offset:8388607
+// GFX12: encoding: [0x65,0xc0,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_d16_hi_x v1, off, s[12:15], m0 offset:8388607
+// GFX12: encoding: [0x7d,0xc0,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_d16_hi_x v1, v0, s[12:15], s4 idxen offset:8388607
+// GFX12: encoding: [0x04,0xc0,0x09,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_d16_hi_x v1, v0, s[12:15], s4 offen offset:8388607
+// GFX12: encoding: [0x04,0xc0,0x09,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_store_format_d16_hi_x v1, off, s[12:15], s4
+// GFX12: encoding: [0x04,0xc0,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_format_d16_hi_x v1, off, s[12:15], s4 offset:0
+// GFX12: encoding: [0x04,0xc0,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_store_format_d16_hi_x v1, off, s[12:15], s4 offset:7
+// GFX12: encoding: [0x04,0xc0,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_add v5, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add v255, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x0d,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add v5, off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x0d,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add v5, off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x0d,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add v5, off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x40,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add v5, off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x40,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x40,0x0d,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add v5, v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x40,0x0d,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add v5, off, s[8:11], s3
+// GFX12: encoding: [0x03,0x40,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_add v5, off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x40,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_add v5, off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x40,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_add v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0x40,0x0d,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_x2 v[5:6], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_x2 v[254:255], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x10,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_x2 v[5:6], off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x10,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_x2 v[5:6], off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x10,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_x2 v[5:6], off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0xc0,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_x2 v[5:6], off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0xc0,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_x2 v[5:6], v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x10,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_x2 v[5:6], v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x10,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_add_x2 v[5:6], off, s[8:11], s3
+// GFX12: encoding: [0x03,0xc0,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_add_x2 v[5:6], off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0xc0,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_add_x2 v[5:6], off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0xc0,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_add_x2 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0xc0,0x10,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_and v5, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_and v255, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x0f,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_and v5, off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x0f,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_and v5, off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x0f,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_and v5, off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x00,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_and v5, off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x00,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_and v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x00,0x0f,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_and v5, v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x00,0x0f,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_and v5, off, s[8:11], s3
+// GFX12: encoding: [0x03,0x00,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_and v5, off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x00,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_and v5, off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x00,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_and v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0x00,0x0f,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_and_x2 v[5:6], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_and_x2 v[254:255], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x12,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_and_x2 v[5:6], off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x12,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_and_x2 v[5:6], off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x12,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_and_x2 v[5:6], off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x40,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_and_x2 v[5:6], off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x40,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_and_x2 v[5:6], v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x40,0x12,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_and_x2 v[5:6], v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x40,0x12,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_and_x2 v[5:6], off, s[8:11], s3
+// GFX12: encoding: [0x03,0x40,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_and_x2 v[5:6], off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x40,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_and_x2 v[5:6], off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x40,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_and_x2 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0x40,0x12,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cmpswap v[5:6], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cmpswap v[254:255], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x0d,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cmpswap v[5:6], off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x0d,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cmpswap v[5:6], off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x0d,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cmpswap v[5:6], off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x00,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cmpswap v[5:6], off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x00,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cmpswap v[5:6], v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x00,0x0d,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cmpswap v[5:6], v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x00,0x0d,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cmpswap v[5:6], off, s[8:11], s3
+// GFX12: encoding: [0x03,0x00,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_cmpswap v[5:6], off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x00,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_cmpswap v[5:6], off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x00,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_cmpswap v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0x00,0x0d,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cmpswap_x2 v[5:8], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cmpswap_x2 v[252:255], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x10,0xc4,0xfc,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cmpswap_x2 v[5:8], off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x10,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cmpswap_x2 v[5:8], off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x10,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cmpswap_x2 v[5:8], off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x80,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cmpswap_x2 v[5:8], off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x80,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cmpswap_x2 v[5:8], v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x80,0x10,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cmpswap_x2 v[5:8], v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x80,0x10,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_cmpswap_x2 v[5:8], off, s[8:11], s3
+// GFX12: encoding: [0x03,0x80,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_cmpswap_x2 v[5:8], off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x80,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_cmpswap_x2 v[5:8], off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x80,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_cmpswap_x2 v[5:8], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0x80,0x10,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_csub v5, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_csub v255, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0xc0,0x0d,0xc4,0xff,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_csub v5, off, s[12:15], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x18,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_csub v5, off, s[96:99], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0xc0,0x0d,0xc4,0x05,0xc0,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_csub v5, off, s[8:11], s101 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x65,0xc0,0x0d,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_csub v5, off, s[8:11], m0 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7d,0xc0,0x0d,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_csub v5, v0, s[8:11], s3 idxen offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x90,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_csub v5, v0, s[8:11], s3 offen offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x90,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_csub v5, off, s[8:11], s3 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x90,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_csub v5, off, s[8:11], s3 offset:0 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x90,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_csub v5, off, s[8:11], s3 offset:7 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x90,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_csub v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_csub_u32 v5, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_csub_u32 v255, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0xc0,0x0d,0xc4,0xff,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_csub_u32 v5, off, s[12:15], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x18,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_csub_u32 v5, off, s[96:99], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0xc0,0x0d,0xc4,0x05,0xc0,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_csub_u32 v5, off, s[8:11], s101 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x65,0xc0,0x0d,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_csub_u32 v5, off, s[8:11], m0 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7d,0xc0,0x0d,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_csub_u32 v5, v0, s[8:11], s3 idxen offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x90,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_csub_u32 v5, v0, s[8:11], s3 offen offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x90,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_csub_u32 v5, off, s[8:11], s3 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x90,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_csub_u32 v5, off, s[8:11], s3 offset:0 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x90,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_csub_u32 v5, off, s[8:11], s3 offset:7 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x90,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_csub_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_dec v5, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_dec v255, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x10,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_dec v5, off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x10,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_dec v5, off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x10,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_dec v5, off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x00,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_dec v5, off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x00,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_dec v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x00,0x10,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_dec v5, v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x00,0x10,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_dec v5, off, s[8:11], s3
+// GFX12: encoding: [0x03,0x00,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_dec v5, off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x00,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_dec v5, off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x00,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_dec v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0x00,0x10,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_dec_x2 v[5:6], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x13,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_dec_x2 v[254:255], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x13,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_dec_x2 v[5:6], off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x13,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_dec_x2 v[5:6], off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x13,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_dec_x2 v[5:6], off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x40,0x13,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_dec_x2 v[5:6], off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x40,0x13,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_dec_x2 v[5:6], v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x40,0x13,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_dec_x2 v[5:6], v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x40,0x13,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_dec_x2 v[5:6], off, s[8:11], s3
+// GFX12: encoding: [0x03,0x40,0x13,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_dec_x2 v[5:6], off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x40,0x13,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_dec_x2 v[5:6], off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x40,0x13,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_dec_x2 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0x40,0x13,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_inc v5, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_inc v255, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x0f,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_inc v5, off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x0f,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_inc v5, off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x0f,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_inc v5, off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0xc0,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_inc v5, off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0xc0,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_inc v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x0f,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_inc v5, v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x0f,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_inc v5, off, s[8:11], s3
+// GFX12: encoding: [0x03,0xc0,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_inc v5, off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0xc0,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_inc v5, off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0xc0,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_inc v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0xc0,0x0f,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_inc_x2 v[5:6], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x13,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_inc_x2 v[254:255], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x13,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_inc_x2 v[5:6], off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x13,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_inc_x2 v[5:6], off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x13,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_inc_x2 v[5:6], off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x00,0x13,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_inc_x2 v[5:6], off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x00,0x13,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_inc_x2 v[5:6], v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x00,0x13,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_inc_x2 v[5:6], v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x00,0x13,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_inc_x2 v[5:6], off, s[8:11], s3
+// GFX12: encoding: [0x03,0x00,0x13,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_inc_x2 v[5:6], off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x00,0x13,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_inc_x2 v[5:6], off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x00,0x13,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_inc_x2 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0x00,0x13,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_fmax v5, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_fmax v255, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x14,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_fmax v5, off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x14,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_fmax v5, off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x14,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_fmax v5, off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x80,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_fmax v5, off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x80,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_fmax v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x80,0x14,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_fmax v5, v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x80,0x14,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_fmax v5, off, s[8:11], s3
+// GFX12: encoding: [0x03,0x80,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_fmax v5, off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x80,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_fmax v5, off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x80,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_fmax v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0x80,0x14,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_f32 v5, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_f32 v255, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x14,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_f32 v5, off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x14,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_f32 v5, off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x14,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_f32 v5, off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x80,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_f32 v5, off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x80,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_f32 v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x80,0x14,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_f32 v5, v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x80,0x14,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_max_f32 v5, off, s[8:11], s3
+// GFX12: encoding: [0x03,0x80,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_max_f32 v5, off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x80,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_max_f32 v5, off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x80,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_max_f32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0x80,0x14,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_smax v5, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_smax v255, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x0e,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_smax v5, off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x0e,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_smax v5, off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x0e,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_smax v5, off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x80,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_smax v5, off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x80,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_smax v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x80,0x0e,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_smax v5, v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x80,0x0e,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_smax v5, off, s[8:11], s3
+// GFX12: encoding: [0x03,0x80,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_smax v5, off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x80,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_smax v5, off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x80,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_smax v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0x80,0x0e,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_smax_x2 v[5:6], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_smax_x2 v[254:255], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x11,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_smax_x2 v[5:6], off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x11,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_smax_x2 v[5:6], off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x11,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_smax_x2 v[5:6], off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0xc0,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_smax_x2 v[5:6], off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0xc0,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_smax_x2 v[5:6], v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x11,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_smax_x2 v[5:6], v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x11,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_smax_x2 v[5:6], off, s[8:11], s3
+// GFX12: encoding: [0x03,0xc0,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_smax_x2 v[5:6], off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0xc0,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_smax_x2 v[5:6], off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0xc0,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_smax_x2 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0xc0,0x11,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_umax v5, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_umax v255, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x0e,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_umax v5, off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x0e,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_umax v5, off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x0e,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_umax v5, off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0xc0,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_umax v5, off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0xc0,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_umax v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x0e,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_umax v5, v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x0e,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_umax v5, off, s[8:11], s3
+// GFX12: encoding: [0x03,0xc0,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_umax v5, off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0xc0,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_umax v5, off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0xc0,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_umax v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0xc0,0x0e,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_umax_x2 v[5:6], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_umax_x2 v[254:255], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x12,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_umax_x2 v[5:6], off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x12,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_umax_x2 v[5:6], off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x12,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_umax_x2 v[5:6], off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x00,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_umax_x2 v[5:6], off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x00,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_umax_x2 v[5:6], v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x00,0x12,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_umax_x2 v[5:6], v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x00,0x12,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_umax_x2 v[5:6], off, s[8:11], s3
+// GFX12: encoding: [0x03,0x00,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_umax_x2 v[5:6], off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x00,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_umax_x2 v[5:6], off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x00,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_umax_x2 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0x00,0x12,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_fmin v5, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_fmin v255, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x14,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_fmin v5, off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x14,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_fmin v5, off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x14,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_fmin v5, off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x40,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_fmin v5, off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x40,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_fmin v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x40,0x14,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_fmin v5, v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x40,0x14,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_fmin v5, off, s[8:11], s3
+// GFX12: encoding: [0x03,0x40,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_fmin v5, off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x40,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_fmin v5, off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x40,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_fmin v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0x40,0x14,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_f32 v5, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_f32 v255, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x14,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_f32 v5, off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x14,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_f32 v5, off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x14,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_f32 v5, off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x40,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_f32 v5, off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x40,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_f32 v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x40,0x14,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_f32 v5, v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x40,0x14,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_min_f32 v5, off, s[8:11], s3
+// GFX12: encoding: [0x03,0x40,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_min_f32 v5, off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x40,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_min_f32 v5, off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x40,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_min_f32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0x40,0x14,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_smin v5, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_smin v255, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x0e,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_smin v5, off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x0e,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_smin v5, off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x0e,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_smin v5, off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x00,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_smin v5, off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x00,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_smin v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x00,0x0e,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_smin v5, v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x00,0x0e,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_smin v5, off, s[8:11], s3
+// GFX12: encoding: [0x03,0x00,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_smin v5, off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x00,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_smin v5, off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x00,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_smin v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0x00,0x0e,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_smin_x2 v[5:6], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_smin_x2 v[254:255], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x11,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_smin_x2 v[5:6], off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x11,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_smin_x2 v[5:6], off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x11,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_smin_x2 v[5:6], off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x40,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_smin_x2 v[5:6], off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x40,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_smin_x2 v[5:6], v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x40,0x11,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_smin_x2 v[5:6], v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x40,0x11,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_smin_x2 v[5:6], off, s[8:11], s3
+// GFX12: encoding: [0x03,0x40,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_smin_x2 v[5:6], off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x40,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_smin_x2 v[5:6], off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x40,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_smin_x2 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0x40,0x11,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_umin v5, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_umin v255, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x0e,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_umin v5, off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x0e,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_umin v5, off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x0e,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_umin v5, off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x40,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_umin v5, off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x40,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_umin v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x40,0x0e,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_umin v5, v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x40,0x0e,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_umin v5, off, s[8:11], s3
+// GFX12: encoding: [0x03,0x40,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_umin v5, off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x40,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_umin v5, off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x40,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_umin v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0x40,0x0e,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_umin_x2 v[5:6], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_umin_x2 v[254:255], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x11,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_umin_x2 v[5:6], off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x11,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_umin_x2 v[5:6], off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x11,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_umin_x2 v[5:6], off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x80,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_umin_x2 v[5:6], off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x80,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_umin_x2 v[5:6], v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x80,0x11,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_umin_x2 v[5:6], v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x80,0x11,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_umin_x2 v[5:6], off, s[8:11], s3
+// GFX12: encoding: [0x03,0x80,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_umin_x2 v[5:6], off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x80,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_umin_x2 v[5:6], off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x80,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_umin_x2 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0x80,0x11,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_or v5, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_or v255, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x0f,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_or v5, off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x0f,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_or v5, off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x0f,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_or v5, off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x40,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_or v5, off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x40,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_or v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x40,0x0f,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_or v5, v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x40,0x0f,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_or v5, off, s[8:11], s3
+// GFX12: encoding: [0x03,0x40,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_or v5, off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x40,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_or v5, off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x40,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_or v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0x40,0x0f,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_or_x2 v[5:6], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_or_x2 v[254:255], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x12,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_or_x2 v[5:6], off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x12,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_or_x2 v[5:6], off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x12,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_or_x2 v[5:6], off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x80,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_or_x2 v[5:6], off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x80,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_or_x2 v[5:6], v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x80,0x12,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_or_x2 v[5:6], v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x80,0x12,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_or_x2 v[5:6], off, s[8:11], s3
+// GFX12: encoding: [0x03,0x80,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_or_x2 v[5:6], off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x80,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_or_x2 v[5:6], off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x80,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_or_x2 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0x80,0x12,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub v5, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub v255, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x0d,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub v5, off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x0d,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub v5, off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x0d,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub v5, off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x80,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub v5, off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x80,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x80,0x0d,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub v5, v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x80,0x0d,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub v5, off, s[8:11], s3
+// GFX12: encoding: [0x03,0x80,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_sub v5, off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x80,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_sub v5, off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x80,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_sub v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0x80,0x0d,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_x2 v[5:6], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_x2 v[254:255], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x11,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_x2 v[5:6], off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x11,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_x2 v[5:6], off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x00,0x11,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_x2 v[5:6], off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x00,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_x2 v[5:6], off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x00,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_x2 v[5:6], v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x00,0x11,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_x2 v[5:6], v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x00,0x11,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_sub_x2 v[5:6], off, s[8:11], s3
+// GFX12: encoding: [0x03,0x00,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_sub_x2 v[5:6], off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x00,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_sub_x2 v[5:6], off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x00,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_sub_x2 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0x00,0x11,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_swap v5, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x0c,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_swap v255, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x0c,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_swap v5, off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x0c,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_swap v5, off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x0c,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_swap v5, off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0xc0,0x0c,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_swap v5, off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0xc0,0x0c,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_swap v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x0c,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_swap v5, v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x0c,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_swap v5, off, s[8:11], s3
+// GFX12: encoding: [0x03,0xc0,0x0c,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_swap v5, off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0xc0,0x0c,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_swap v5, off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0xc0,0x0c,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_swap v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0xc0,0x0c,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_swap_x2 v[5:6], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_swap_x2 v[254:255], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x10,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_swap_x2 v[5:6], off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x10,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_swap_x2 v[5:6], off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x40,0x10,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_swap_x2 v[5:6], off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x40,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_swap_x2 v[5:6], off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x40,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_swap_x2 v[5:6], v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x40,0x10,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_swap_x2 v[5:6], v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x40,0x10,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_swap_x2 v[5:6], off, s[8:11], s3
+// GFX12: encoding: [0x03,0x40,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_swap_x2 v[5:6], off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x40,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_swap_x2 v[5:6], off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x40,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_swap_x2 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0x40,0x10,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_xor v5, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_xor v255, off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x0f,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_xor v5, off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x0f,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_xor v5, off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0x80,0x0f,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_xor v5, off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0x80,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_xor v5, off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0x80,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_xor v5, v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0x80,0x0f,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_xor v5, v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0x80,0x0f,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_xor v5, off, s[8:11], s3
+// GFX12: encoding: [0x03,0x80,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_xor v5, off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0x80,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_xor v5, off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0x80,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_xor v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0x80,0x0f,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_xor_x2 v[5:6], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_xor_x2 v[254:255], off, s[8:11], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x12,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_xor_x2 v[5:6], off, s[12:15], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x12,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_xor_x2 v[5:6], off, s[96:99], s3 offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x12,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_xor_x2 v[5:6], off, s[8:11], s101 offset:8388607
+// GFX12: encoding: [0x65,0xc0,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_xor_x2 v[5:6], off, s[8:11], m0 offset:8388607
+// GFX12: encoding: [0x7d,0xc0,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_xor_x2 v[5:6], v0, s[8:11], s3 idxen offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x12,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_xor_x2 v[5:6], v0, s[8:11], s3 offen offset:8388607
+// GFX12: encoding: [0x03,0xc0,0x12,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+
+buffer_atomic_xor_x2 v[5:6], off, s[8:11], s3
+// GFX12: encoding: [0x03,0xc0,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_xor_x2 v[5:6], off, s[8:11], s3 offset:0
+// GFX12: encoding: [0x03,0xc0,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+
+buffer_atomic_xor_x2 v[5:6], off, s[8:11], s3 offset:7
+// GFX12: encoding: [0x03,0xc0,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+
+buffer_atomic_xor_x2 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x03,0xc0,0x12,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vflat.s b/llvm/test/MC/AMDGPU/gfx12_asm_vflat.s
new file mode 100644
index 0000000000000000000000000000000000000000..c0ffc5247d90e8375e06a79e48e92ed96c489bb2
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vflat.s
@@ -0,0 +1,2545 @@
+; RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -show-encoding %s | FileCheck --check-prefix=GFX12 %s
+
+flat_atomic_add_f32 v1, v[0:1], v2 offset:-8000000 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x80,0x15,0xec,0x01,0x00,0x10,0x01,0x00,0x00,0xee,0x85]
+
+flat_atomic_add_f32 v1, v[0:1], v2 offset:8000000 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x80,0x15,0xec,0x01,0x00,0x10,0x01,0x00,0x00,0x12,0x7a]
+
+flat_atomic_add_f32 v1, v[0:1], v2 offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x80,0x15,0xec,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+flat_atomic_add_f32 v1, v[0:1], v2 offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x80,0x15,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+flat_atomic_add_f32 v[0:1], v2 offset:-64
+// GFX12: encoding: [0x7c,0x80,0x15,0xec,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+flat_atomic_add_f32 v[0:1], v2 offset:64
+// GFX12: encoding: [0x7c,0x80,0x15,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+flat_atomic_add_u32 v1, v[0:1], v2 offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x40,0x0d,0xec,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+flat_atomic_add_u32 v1, v[0:1], v2 offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x40,0x0d,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+flat_atomic_add_u32 v[0:1], v2 offset:-64
+// GFX12: encoding: [0x7c,0x40,0x0d,0xec,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+flat_atomic_add_u32 v[0:1], v2 offset:64
+// GFX12: encoding: [0x7c,0x40,0x0d,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+flat_atomic_add_u32 v[3:4], v5
+// GFX12: encoding: [0x7c,0x40,0x0d,0xec,0x00,0x00,0x80,0x02,0x03,0x00,0x00,0x00]
+
+flat_atomic_add_u64 v[0:1], v[2:3] offset:-64
+// GFX12: encoding: [0x7c,0xc0,0x10,0xec,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+flat_atomic_add_u64 v[0:1], v[2:3] offset:64
+// GFX12: encoding: [0x7c,0xc0,0x10,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+flat_atomic_add_u64 v[1:2], v[0:1], v[2:3] offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0xc0,0x10,0xec,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+flat_atomic_add_u64 v[1:2], v[0:1], v[2:3] offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0xc0,0x10,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+flat_atomic_and_b32 v1, v[0:1], v2 offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x00,0x0f,0xec,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+flat_atomic_and_b32 v1, v[0:1], v2 offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x00,0x0f,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+flat_atomic_and_b32 v[0:1], v2 offset:-64
+// GFX12: encoding: [0x7c,0x00,0x0f,0xec,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+flat_atomic_and_b32 v[0:1], v2 offset:64
+// GFX12: encoding: [0x7c,0x00,0x0f,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+flat_atomic_and_b64 v[0:1], v[2:3] offset:-64
+// GFX12: encoding: [0x7c,0x40,0x12,0xec,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+flat_atomic_and_b64 v[0:1], v[2:3] offset:64
+// GFX12: encoding: [0x7c,0x40,0x12,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+flat_atomic_and_b64 v[1:2], v[0:1], v[2:3] offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x40,0x12,0xec,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+flat_atomic_and_b64 v[1:2], v[0:1], v[2:3] offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x40,0x12,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+flat_atomic_cmpswap_b32 v0, v[1:2], v[3:4] th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x00,0x0d,0xec,0x00,0x00,0x90,0x01,0x01,0x00,0x00,0x00]
+
+flat_atomic_cmpswap_b32 v0, v[1:2], v[3:4] th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x00,0x0d,0xec,0x00,0x00,0x90,0x01,0x01,0x00,0x00,0x00]
+
+flat_atomic_cmpswap_b32 v0, v[1:2], v[3:4] offset:2047 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x00,0x0d,0xec,0x00,0x00,0x90,0x01,0x01,0xff,0x07,0x00]
+
+flat_atomic_cmpswap_b32 v0, v[1:2], v[3:4] offset:2047 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x00,0x0d,0xec,0x00,0x00,0x90,0x01,0x01,0xff,0x07,0x00]
+
+flat_atomic_cmpswap_b32 v1, v[0:1], v[2:3] offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x00,0x0d,0xec,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+flat_atomic_cmpswap_b32 v1, v[0:1], v[2:3] offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x00,0x0d,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:-64
+// GFX12: encoding: [0x7c,0x00,0x0d,0xec,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:64
+// GFX12: encoding: [0x7c,0x00,0x0d,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+flat_atomic_cmpswap_b64 v[0:1], v[2:5] offset:-64
+// GFX12: encoding: [0x7c,0x80,0x10,0xec,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+flat_atomic_cmpswap_b64 v[0:1], v[2:5] offset:64
+// GFX12: encoding: [0x7c,0x80,0x10,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+flat_atomic_cmpswap_b64 v[1:2], v[0:1], v[2:5] offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x80,0x10,0xec,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+flat_atomic_cmpswap_b64 v[1:2], v[0:1], v[2:5] offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x80,0x10,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+flat_atomic_cmpswap_b64 v[1:2], v[3:4], v[5:8] th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x80,0x10,0xec,0x01,0x00,0x90,0x02,0x03,0x00,0x00,0x00]
+
+flat_atomic_cmpswap_b64 v[1:2], v[3:4], v[5:8] th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x80,0x10,0xec,0x01,0x00,0x90,0x02,0x03,0x00,0x00,0x00]
+
+flat_atomic_cmpswap_b64 v[1:2], v[3:4], v[5:8] offset:2047 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x80,0x10,0xec,0x01,0x00,0x90,0x02,0x03,0xff,0x07,0x00]
+
+flat_atomic_cmpswap_b64 v[1:2], v[3:4], v[5:8] offset:2047 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x80,0x10,0xec,0x01,0x00,0x90,0x02,0x03,0xff,0x07,0x00]
+
+flat_atomic_dec_u32 v1, v[0:1], v2 offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x00,0x10,0xec,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+flat_atomic_dec_u32 v1, v[0:1], v2 offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x00,0x10,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+flat_atomic_dec_u32 v[0:1], v2 offset:-64
+// GFX12: encoding: [0x7c,0x00,0x10,0xec,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+flat_atomic_dec_u32 v[0:1], v2 offset:64
+// GFX12: encoding: [0x7c,0x00,0x10,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+flat_atomic_dec_u64 v[0:1], v[2:3] offset:-64
+// GFX12: encoding: [0x7c,0x40,0x13,0xec,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+flat_atomic_dec_u64 v[0:1], v[2:3] offset:64
+// GFX12: encoding: [0x7c,0x40,0x13,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+flat_atomic_dec_u64 v[1:2], v[0:1], v[2:3] offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x40,0x13,0xec,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+flat_atomic_dec_u64 v[1:2], v[0:1], v[2:3] offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x40,0x13,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+flat_atomic_inc_u32 v1, v[0:1], v2 offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0xc0,0x0f,0xec,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+flat_atomic_inc_u32 v1, v[0:1], v2 offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0xc0,0x0f,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+flat_atomic_inc_u32 v[0:1], v2 offset:-64
+// GFX12: encoding: [0x7c,0xc0,0x0f,0xec,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+flat_atomic_inc_u32 v[0:1], v2 offset:64
+// GFX12: encoding: [0x7c,0xc0,0x0f,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+flat_atomic_inc_u64 v[0:1], v[2:3] offset:-64
+// GFX12: encoding: [0x7c,0x00,0x13,0xec,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+flat_atomic_inc_u64 v[0:1], v[2:3] offset:64
+// GFX12: encoding: [0x7c,0x00,0x13,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+flat_atomic_inc_u64 v[1:2], v[0:1], v[2:3] offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x00,0x13,0xec,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+flat_atomic_inc_u64 v[1:2], v[0:1], v[2:3] offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x00,0x13,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+flat_atomic_max_num_f32 v1, v[0:1], v2 offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x80,0x14,0xec,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+flat_atomic_max_num_f32 v1, v[0:1], v2 offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x80,0x14,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+flat_atomic_max_num_f32 v[0:1], v2 offset:-64
+// GFX12: encoding: [0x7c,0x80,0x14,0xec,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+flat_atomic_max_num_f32 v[0:1], v2 offset:64
+// GFX12: encoding: [0x7c,0x80,0x14,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+flat_atomic_max_i32 v1, v[0:1], v2 offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x80,0x0e,0xec,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+flat_atomic_max_i32 v1, v[0:1], v2 offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x80,0x0e,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+flat_atomic_max_i32 v[0:1], v2 offset:-64
+// GFX12: encoding: [0x7c,0x80,0x0e,0xec,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+flat_atomic_max_i32 v[0:1], v2 offset:64
+// GFX12: encoding: [0x7c,0x80,0x0e,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+flat_atomic_max_i64 v[0:1], v[2:3] offset:-64
+// GFX12: encoding: [0x7c,0xc0,0x11,0xec,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+flat_atomic_max_i64 v[0:1], v[2:3] offset:64
+// GFX12: encoding: [0x7c,0xc0,0x11,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+flat_atomic_max_i64 v[1:2], v[0:1], v[2:3] offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0xc0,0x11,0xec,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+flat_atomic_max_i64 v[1:2], v[0:1], v[2:3] offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0xc0,0x11,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+flat_atomic_max_u32 v1, v[0:1], v2 offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0xc0,0x0e,0xec,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+flat_atomic_max_u32 v1, v[0:1], v2 offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0xc0,0x0e,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+flat_atomic_max_u32 v[0:1], v2 offset:-64
+// GFX12: encoding: [0x7c,0xc0,0x0e,0xec,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+flat_atomic_max_u32 v[0:1], v2 offset:64
+// GFX12: encoding: [0x7c,0xc0,0x0e,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+flat_atomic_max_u64 v[0:1], v[2:3] offset:-64
+// GFX12: encoding: [0x7c,0x00,0x12,0xec,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+flat_atomic_max_u64 v[0:1], v[2:3] offset:64
+// GFX12: encoding: [0x7c,0x00,0x12,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+flat_atomic_max_u64 v[1:2], v[0:1], v[2:3] offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x00,0x12,0xec,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+flat_atomic_max_u64 v[1:2], v[0:1], v[2:3] offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x00,0x12,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+flat_atomic_min_num_f32 v1, v[0:1], v2 offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x40,0x14,0xec,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+flat_atomic_min_num_f32 v1, v[0:1], v2 offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x40,0x14,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+flat_atomic_min_num_f32 v[0:1], v2 offset:-64
+// GFX12: encoding: [0x7c,0x40,0x14,0xec,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+flat_atomic_min_num_f32 v[0:1], v2 offset:64
+// GFX12: encoding: [0x7c,0x40,0x14,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+flat_atomic_min_i32 v1, v[0:1], v2 offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x00,0x0e,0xec,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+flat_atomic_min_i32 v1, v[0:1], v2 offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x00,0x0e,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+flat_atomic_min_i32 v[0:1], v2 offset:-64
+// GFX12: encoding: [0x7c,0x00,0x0e,0xec,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+flat_atomic_min_i32 v[0:1], v2 offset:64
+// GFX12: encoding: [0x7c,0x00,0x0e,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+flat_atomic_min_i64 v[0:1], v[2:3] offset:-64
+// GFX12: encoding: [0x7c,0x40,0x11,0xec,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+flat_atomic_min_i64 v[0:1], v[2:3] offset:64
+// GFX12: encoding: [0x7c,0x40,0x11,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+flat_atomic_min_i64 v[1:2], v[0:1], v[2:3] offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x40,0x11,0xec,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+flat_atomic_min_i64 v[1:2], v[0:1], v[2:3] offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x40,0x11,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+flat_atomic_min_u32 v1, v[0:1], v2 offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x40,0x0e,0xec,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+flat_atomic_min_u32 v1, v[0:1], v2 offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x40,0x0e,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+flat_atomic_min_u32 v[0:1], v2 offset:-64
+// GFX12: encoding: [0x7c,0x40,0x0e,0xec,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+flat_atomic_min_u32 v[0:1], v2 offset:64
+// GFX12: encoding: [0x7c,0x40,0x0e,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+flat_atomic_min_u64 v[0:1], v[2:3] offset:-64
+// GFX12: encoding: [0x7c,0x80,0x11,0xec,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+flat_atomic_min_u64 v[0:1], v[2:3] offset:64
+// GFX12: encoding: [0x7c,0x80,0x11,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+flat_atomic_min_u64 v[1:2], v[0:1], v[2:3] offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x80,0x11,0xec,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+flat_atomic_min_u64 v[1:2], v[0:1], v[2:3] offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x80,0x11,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+flat_atomic_or_b32 v1, v[0:1], v2 offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x40,0x0f,0xec,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+flat_atomic_or_b32 v1, v[0:1], v2 offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x40,0x0f,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+flat_atomic_or_b32 v[0:1], v2 offset:-64
+// GFX12: encoding: [0x7c,0x40,0x0f,0xec,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+flat_atomic_or_b32 v[0:1], v2 offset:64
+// GFX12: encoding: [0x7c,0x40,0x0f,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+flat_atomic_or_b64 v[0:1], v[2:3] offset:-64
+// GFX12: encoding: [0x7c,0x80,0x12,0xec,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+flat_atomic_or_b64 v[0:1], v[2:3] offset:64
+// GFX12: encoding: [0x7c,0x80,0x12,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+flat_atomic_or_b64 v[1:2], v[0:1], v[2:3] offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x80,0x12,0xec,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+flat_atomic_or_b64 v[1:2], v[0:1], v[2:3] offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x80,0x12,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+flat_atomic_sub_clamp_u32 v1, v[0:1], v2 offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0xc0,0x0d,0xec,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+flat_atomic_sub_clamp_u32 v1, v[0:1], v2 offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0xc0,0x0d,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+flat_atomic_sub_clamp_u32 v[0:1], v2 offset:64
+// GFX12: encoding: [0x7c,0xc0,0x0d,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+flat_atomic_sub_u32 v1, v[0:1], v2 offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x80,0x0d,0xec,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+flat_atomic_sub_u32 v1, v[0:1], v2 offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x80,0x0d,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+flat_atomic_sub_u32 v[0:1], v2 offset:-64
+// GFX12: encoding: [0x7c,0x80,0x0d,0xec,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+flat_atomic_sub_u32 v[0:1], v2 offset:64
+// GFX12: encoding: [0x7c,0x80,0x0d,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+flat_atomic_sub_u64 v[0:1], v[2:3] offset:-64
+// GFX12: encoding: [0x7c,0x00,0x11,0xec,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+flat_atomic_sub_u64 v[0:1], v[2:3] offset:64
+// GFX12: encoding: [0x7c,0x00,0x11,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+flat_atomic_sub_u64 v[1:2], v[0:1], v[2:3] offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x00,0x11,0xec,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+flat_atomic_sub_u64 v[1:2], v[0:1], v[2:3] offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x00,0x11,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+flat_atomic_swap_b32 v0, v[1:2], v3 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0xc0,0x0c,0xec,0x00,0x00,0x90,0x01,0x01,0x00,0x00,0x00]
+
+flat_atomic_swap_b32 v0, v[1:2], v3 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0xc0,0x0c,0xec,0x00,0x00,0x90,0x01,0x01,0x00,0x00,0x00]
+
+flat_atomic_swap_b32 v0, v[1:2], v3 offset:2047 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0xc0,0x0c,0xec,0x00,0x00,0x90,0x01,0x01,0xff,0x07,0x00]
+
+flat_atomic_swap_b32 v0, v[1:2], v3 offset:2047 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0xc0,0x0c,0xec,0x00,0x00,0x90,0x01,0x01,0xff,0x07,0x00]
+
+flat_atomic_swap_b32 v0, v[1:2], v3 offset:-2048 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0xc0,0x0c,0xec,0x00,0x00,0x90,0x01,0x01,0x00,0xf8,0xff]
+
+flat_atomic_swap_b32 v0, v[1:2], v3 offset:2048 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0xc0,0x0c,0xec,0x00,0x00,0x90,0x01,0x01,0x00,0x08,0x00]
+
+flat_atomic_swap_b32 v1, v[0:1], v2 offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0xc0,0x0c,0xec,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+flat_atomic_swap_b32 v1, v[0:1], v2 offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0xc0,0x0c,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+flat_atomic_swap_b32 v[0:1], v2 offset:-64
+// GFX12: encoding: [0x7c,0xc0,0x0c,0xec,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+flat_atomic_swap_b32 v[0:1], v2 offset:64
+// GFX12: encoding: [0x7c,0xc0,0x0c,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+flat_atomic_swap_b64 v[0:1], v[2:3] offset:-64
+// GFX12: encoding: [0x7c,0x40,0x10,0xec,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+flat_atomic_swap_b64 v[0:1], v[2:3] offset:64
+// GFX12: encoding: [0x7c,0x40,0x10,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+flat_atomic_swap_b64 v[1:2], v[0:1], v[2:3] offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x40,0x10,0xec,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+flat_atomic_swap_b64 v[1:2], v[0:1], v[2:3] offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x40,0x10,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+flat_atomic_swap_b64 v[1:2], v[3:4], v[5:6] th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x40,0x10,0xec,0x01,0x00,0x90,0x02,0x03,0x00,0x00,0x00]
+
+flat_atomic_swap_b64 v[1:2], v[3:4], v[5:6] th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x40,0x10,0xec,0x01,0x00,0x90,0x02,0x03,0x00,0x00,0x00]
+
+flat_atomic_swap_b64 v[1:2], v[3:4], v[5:6] offset:2047 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x40,0x10,0xec,0x01,0x00,0x90,0x02,0x03,0xff,0x07,0x00]
+
+flat_atomic_swap_b64 v[1:2], v[3:4], v[5:6] offset:2047 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x40,0x10,0xec,0x01,0x00,0x90,0x02,0x03,0xff,0x07,0x00]
+
+flat_atomic_xor_b32 v1, v[0:1], v2 offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x80,0x0f,0xec,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+flat_atomic_xor_b32 v1, v[0:1], v2 offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x80,0x0f,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+flat_atomic_xor_b32 v[0:1], v2 offset:-64
+// GFX12: encoding: [0x7c,0x80,0x0f,0xec,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+flat_atomic_xor_b32 v[0:1], v2 offset:64
+// GFX12: encoding: [0x7c,0x80,0x0f,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+flat_atomic_xor_b64 v[0:1], v[2:3] offset:-64
+// GFX12: encoding: [0x7c,0xc0,0x12,0xec,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+flat_atomic_xor_b64 v[0:1], v[2:3] offset:64
+// GFX12: encoding: [0x7c,0xc0,0x12,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+flat_atomic_xor_b64 v[1:2], v[0:1], v[2:3] offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0xc0,0x12,0xec,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+flat_atomic_xor_b64 v[1:2], v[0:1], v[2:3] offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0xc0,0x12,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+flat_load_b128 v[1:4], v[0:1] offset:-64
+// GFX12: encoding: [0x7c,0xc0,0x05,0xec,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+flat_load_b128 v[1:4], v[0:1] offset:64
+// GFX12: encoding: [0x7c,0xc0,0x05,0xec,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+flat_load_b128 v[1:4], v[5:6]
+// GFX12: encoding: [0x7c,0xc0,0x05,0xec,0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00]
+
+flat_load_b32 v1, v[0:1] offset:-64
+// GFX12: encoding: [0x7c,0x00,0x05,0xec,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+flat_load_b32 v1, v[0:1] offset:64
+// GFX12: encoding: [0x7c,0x00,0x05,0xec,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+flat_load_b32 v1, v[3:4] offset:2047
+// GFX12: encoding: [0x7c,0x00,0x05,0xec,0x01,0x00,0x00,0x00,0x03,0xff,0x07,0x00]
+
+flat_load_b32 v1, v[3:4] offset:-2048
+// GFX12: encoding: [0x7c,0x00,0x05,0xec,0x01,0x00,0x00,0x00,0x03,0x00,0xf8,0xff]
+
+flat_load_b32 v1, v[3:4] offset:2048
+// GFX12: encoding: [0x7c,0x00,0x05,0xec,0x01,0x00,0x00,0x00,0x03,0x00,0x08,0x00]
+
+flat_load_b32 v1, v[3:4] offset:-4
+// GFX12: encoding: [0x7c,0x00,0x05,0xec,0x01,0x00,0x00,0x00,0x03,0xfc,0xff,0xff]
+
+flat_load_b32 v1, v[3:4] offset:4
+// GFX12: encoding: [0x7c,0x00,0x05,0xec,0x01,0x00,0x00,0x00,0x03,0x04,0x00,0x00]
+
+flat_load_b32 v1, v[3:4] offset:-4
+// GFX12: encoding: [0x7c,0x00,0x05,0xec,0x01,0x00,0x00,0x00,0x03,0xfc,0xff,0xff]
+
+flat_load_b32 v1, v[3:4] offset:4
+// GFX12: encoding: [0x7c,0x00,0x05,0xec,0x01,0x00,0x00,0x00,0x03,0x04,0x00,0x00]
+
+flat_load_b32 v1, v[3:4] offset:-4
+// GFX12: encoding: [0x7c,0x00,0x05,0xec,0x01,0x00,0x00,0x00,0x03,0xfc,0xff,0xff]
+
+flat_load_b32 v1, v[3:4] offset:4
+// GFX12: encoding: [0x7c,0x00,0x05,0xec,0x01,0x00,0x00,0x00,0x03,0x04,0x00,0x00]
+
+flat_load_b32 v1, v[3:4]
+// GFX12: encoding: [0x7c,0x00,0x05,0xec,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+
+flat_load_b64 v[1:2], v[0:1] offset:-64
+// GFX12: encoding: [0x7c,0x40,0x05,0xec,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+flat_load_b64 v[1:2], v[0:1] offset:64
+// GFX12: encoding: [0x7c,0x40,0x05,0xec,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+flat_load_b64 v[1:2], v[3:4]
+// GFX12: encoding: [0x7c,0x40,0x05,0xec,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+
+flat_load_b96 v[1:3], v[0:1] offset:-64
+// GFX12: encoding: [0x7c,0x80,0x05,0xec,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+flat_load_b96 v[1:3], v[0:1] offset:64
+// GFX12: encoding: [0x7c,0x80,0x05,0xec,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+flat_load_b96 v[1:3], v[5:6]
+// GFX12: encoding: [0x7c,0x80,0x05,0xec,0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00]
+
+flat_load_d16_b16 v1, v[0:1] offset:-64
+// GFX12: encoding: [0x7c,0x00,0x08,0xec,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+flat_load_d16_b16 v1, v[0:1] offset:64
+// GFX12: encoding: [0x7c,0x00,0x08,0xec,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+flat_load_d16_b16 v1, v[3:4]
+// GFX12: encoding: [0x7c,0x00,0x08,0xec,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+
+flat_load_d16_hi_b16 v1, v[0:1] offset:-64
+// GFX12: encoding: [0x7c,0xc0,0x08,0xec,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+flat_load_d16_hi_b16 v1, v[0:1] offset:64
+// GFX12: encoding: [0x7c,0xc0,0x08,0xec,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+flat_load_d16_hi_b16 v1, v[3:4]
+// GFX12: encoding: [0x7c,0xc0,0x08,0xec,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+
+flat_load_d16_hi_i8 v1, v[0:1] offset:-64
+// GFX12: encoding: [0x7c,0x80,0x08,0xec,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+flat_load_d16_hi_i8 v1, v[0:1] offset:64
+// GFX12: encoding: [0x7c,0x80,0x08,0xec,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+flat_load_d16_hi_i8 v1, v[3:4]
+// GFX12: encoding: [0x7c,0x80,0x08,0xec,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+
+flat_load_d16_hi_u8 v1, v[0:1] offset:-64
+// GFX12: encoding: [0x7c,0x40,0x08,0xec,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+flat_load_d16_hi_u8 v1, v[0:1] offset:64
+// GFX12: encoding: [0x7c,0x40,0x08,0xec,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+flat_load_d16_hi_u8 v1, v[3:4]
+// GFX12: encoding: [0x7c,0x40,0x08,0xec,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+
+flat_load_d16_i8 v1, v[0:1] offset:-64
+// GFX12: encoding: [0x7c,0xc0,0x07,0xec,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+flat_load_d16_i8 v1, v[0:1] offset:64
+// GFX12: encoding: [0x7c,0xc0,0x07,0xec,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+flat_load_d16_i8 v1, v[3:4]
+// GFX12: encoding: [0x7c,0xc0,0x07,0xec,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+
+flat_load_d16_u8 v1, v[0:1] offset:-64
+// GFX12: encoding: [0x7c,0x80,0x07,0xec,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+flat_load_d16_u8 v1, v[0:1] offset:64
+// GFX12: encoding: [0x7c,0x80,0x07,0xec,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+flat_load_d16_u8 v1, v[3:4]
+// GFX12: encoding: [0x7c,0x80,0x07,0xec,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+
+flat_load_i16 v1, v[0:1] offset:-64
+// GFX12: encoding: [0x7c,0xc0,0x04,0xec,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+flat_load_i16 v1, v[0:1] offset:64
+// GFX12: encoding: [0x7c,0xc0,0x04,0xec,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+flat_load_i16 v1, v[3:4]
+// GFX12: encoding: [0x7c,0xc0,0x04,0xec,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+
+flat_load_i8 v1, v[0:1] offset:-64
+// GFX12: encoding: [0x7c,0x40,0x04,0xec,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+flat_load_i8 v1, v[0:1] offset:64
+// GFX12: encoding: [0x7c,0x40,0x04,0xec,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+flat_load_i8 v1, v[3:4]
+// GFX12: encoding: [0x7c,0x40,0x04,0xec,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+
+flat_load_u16 v1, v[0:1] offset:-64
+// GFX12: encoding: [0x7c,0x80,0x04,0xec,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+flat_load_u16 v1, v[0:1] offset:64
+// GFX12: encoding: [0x7c,0x80,0x04,0xec,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+flat_load_u16 v1, v[3:4]
+// GFX12: encoding: [0x7c,0x80,0x04,0xec,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+
+flat_load_u8 v1, v[0:1] offset:-64
+// GFX12: encoding: [0x7c,0x00,0x04,0xec,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+flat_load_u8 v1, v[0:1] offset:64
+// GFX12: encoding: [0x7c,0x00,0x04,0xec,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+flat_load_u8 v1, v[3:4]
+// GFX12: encoding: [0x7c,0x00,0x04,0xec,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+
+flat_store_b128 v[0:1], v[2:5] offset:-64
+// GFX12: encoding: [0x7c,0x40,0x07,0xec,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+flat_store_b128 v[0:1], v[2:5] offset:64
+// GFX12: encoding: [0x7c,0x40,0x07,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+flat_store_b128 v[1:2], v[3:6]
+// GFX12: encoding: [0x7c,0x40,0x07,0xec,0x00,0x00,0x80,0x01,0x01,0x00,0x00,0x00]
+
+flat_store_b16 v[0:1], v2 offset:-64
+// GFX12: encoding: [0x7c,0x40,0x06,0xec,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+flat_store_b16 v[0:1], v2 offset:64
+// GFX12: encoding: [0x7c,0x40,0x06,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+flat_store_b16 v[3:4], v1
+// GFX12: encoding: [0x7c,0x40,0x06,0xec,0x00,0x00,0x80,0x00,0x03,0x00,0x00,0x00]
+
+flat_store_b32 v[0:1], v2 offset:-64
+// GFX12: encoding: [0x7c,0x80,0x06,0xec,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+flat_store_b32 v[0:1], v2 offset:64
+// GFX12: encoding: [0x7c,0x80,0x06,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+flat_store_b32 v[3:4], v1 offset:-16
+// GFX12: encoding: [0x7c,0x80,0x06,0xec,0x00,0x00,0x80,0x00,0x03,0xf0,0xff,0xff]
+
+flat_store_b32 v[3:4], v1 offset:16
+// GFX12: encoding: [0x7c,0x80,0x06,0xec,0x00,0x00,0x80,0x00,0x03,0x10,0x00,0x00]
+
+flat_store_b64 v[0:1], v[2:3] offset:-64
+// GFX12: encoding: [0x7c,0xc0,0x06,0xec,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+flat_store_b64 v[0:1], v[2:3] offset:64
+// GFX12: encoding: [0x7c,0xc0,0x06,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+flat_store_b64 v[1:2], v[3:4]
+// GFX12: encoding: [0x7c,0xc0,0x06,0xec,0x00,0x00,0x80,0x01,0x01,0x00,0x00,0x00]
+
+flat_store_b8 v[0:1], v2 offset:-64
+// GFX12: encoding: [0x7c,0x00,0x06,0xec,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+flat_store_b8 v[0:1], v2 offset:64
+// GFX12: encoding: [0x7c,0x00,0x06,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+flat_store_b8 v[3:4], v1
+// GFX12: encoding: [0x7c,0x00,0x06,0xec,0x00,0x00,0x80,0x00,0x03,0x00,0x00,0x00]
+
+flat_store_b96 v[0:1], v[2:4] offset:-64
+// GFX12: encoding: [0x7c,0x00,0x07,0xec,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+flat_store_b96 v[0:1], v[2:4] offset:64
+// GFX12: encoding: [0x7c,0x00,0x07,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+flat_store_b96 v[1:2], v[3:5]
+// GFX12: encoding: [0x7c,0x00,0x07,0xec,0x00,0x00,0x80,0x01,0x01,0x00,0x00,0x00]
+
+flat_store_d16_hi_b16 v[0:1], v2 offset:-64
+// GFX12: encoding: [0x7c,0x40,0x09,0xec,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+flat_store_d16_hi_b16 v[0:1], v2 offset:64
+// GFX12: encoding: [0x7c,0x40,0x09,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+flat_store_d16_hi_b16 v[3:4], v1
+// GFX12: encoding: [0x7c,0x40,0x09,0xec,0x00,0x00,0x80,0x00,0x03,0x00,0x00,0x00]
+
+flat_store_d16_hi_b8 v[0:1], v2 offset:-64
+// GFX12: encoding: [0x7c,0x00,0x09,0xec,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+flat_store_d16_hi_b8 v[0:1], v2 offset:64
+// GFX12: encoding: [0x7c,0x00,0x09,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+flat_store_d16_hi_b8 v[3:4], v1
+// GFX12: encoding: [0x7c,0x00,0x09,0xec,0x00,0x00,0x80,0x00,0x03,0x00,0x00,0x00]
+
+global_atomic_add_f32 v0, v2, s[0:1] offset:-64
+// GFX12: encoding: [0x00,0x80,0x15,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_add_f32 v0, v2, s[0:1] offset:64
+// GFX12: encoding: [0x00,0x80,0x15,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_add_f32 v1, v0, v2, s[0:1] offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x00,0x80,0x15,0xee,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_add_f32 v1, v0, v2, s[0:1] offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x00,0x80,0x15,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_add_f32 v1, v[0:1], v2, off offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x80,0x15,0xee,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_add_f32 v1, v[0:1], v2, off offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x80,0x15,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_add_f32 v[0:1], v2, off offset:-64
+// GFX12: encoding: [0x7c,0x80,0x15,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_add_f32 v[0:1], v2, off offset:64
+// GFX12: encoding: [0x7c,0x80,0x15,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_add_u32 v0, v2, s[0:1] offset:-64
+// GFX12: encoding: [0x00,0x40,0x0d,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_add_u32 v0, v2, s[0:1] offset:64
+// GFX12: encoding: [0x00,0x40,0x0d,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_add_u32 v1, v0, v2, s[0:1] offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x00,0x40,0x0d,0xee,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_add_u32 v1, v0, v2, s[0:1] offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x00,0x40,0x0d,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_add_u32 v1, v[0:1], v2, off offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x40,0x0d,0xee,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_add_u32 v1, v[0:1], v2, off offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x40,0x0d,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_add_u32 v[0:1], v2, off offset:-64
+// GFX12: encoding: [0x7c,0x40,0x0d,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_add_u32 v[0:1], v2, off offset:64
+// GFX12: encoding: [0x7c,0x40,0x0d,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_add_u64 v0, v[2:3], s[0:1] offset:-64
+// GFX12: encoding: [0x00,0xc0,0x10,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_add_u64 v0, v[2:3], s[0:1] offset:64
+// GFX12: encoding: [0x00,0xc0,0x10,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_add_u64 v[0:1], v[2:3], off offset:-64
+// GFX12: encoding: [0x7c,0xc0,0x10,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_add_u64 v[0:1], v[2:3], off offset:64
+// GFX12: encoding: [0x7c,0xc0,0x10,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_add_u64 v[1:2], v0, v[2:3], s[0:1] offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x00,0xc0,0x10,0xee,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_add_u64 v[1:2], v0, v[2:3], s[0:1] offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x00,0xc0,0x10,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_add_u64 v[1:2], v[0:1], v[2:3], off offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0xc0,0x10,0xee,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_add_u64 v[1:2], v[0:1], v[2:3], off offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0xc0,0x10,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_and_b32 v0, v2, s[0:1] offset:-64
+// GFX12: encoding: [0x00,0x00,0x0f,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_and_b32 v0, v2, s[0:1] offset:64
+// GFX12: encoding: [0x00,0x00,0x0f,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_and_b32 v1, v0, v2, s[0:1] offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x00,0x00,0x0f,0xee,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_and_b32 v1, v0, v2, s[0:1] offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x00,0x00,0x0f,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_and_b32 v1, v[0:1], v2, off offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x00,0x0f,0xee,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_and_b32 v1, v[0:1], v2, off offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x00,0x0f,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_and_b32 v[0:1], v2, off offset:-64
+// GFX12: encoding: [0x7c,0x00,0x0f,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_and_b32 v[0:1], v2, off offset:64
+// GFX12: encoding: [0x7c,0x00,0x0f,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_and_b64 v0, v[2:3], s[0:1] offset:-64
+// GFX12: encoding: [0x00,0x40,0x12,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_and_b64 v0, v[2:3], s[0:1] offset:64
+// GFX12: encoding: [0x00,0x40,0x12,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_and_b64 v[0:1], v[2:3], off offset:-64
+// GFX12: encoding: [0x7c,0x40,0x12,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_and_b64 v[0:1], v[2:3], off offset:64
+// GFX12: encoding: [0x7c,0x40,0x12,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_and_b64 v[1:2], v0, v[2:3], s[0:1] offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x00,0x40,0x12,0xee,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_and_b64 v[1:2], v0, v[2:3], s[0:1] offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x00,0x40,0x12,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_and_b64 v[1:2], v[0:1], v[2:3], off offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x40,0x12,0xee,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_and_b64 v[1:2], v[0:1], v[2:3], off offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x40,0x12,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_cmpswap_b32 v0, v1, v[2:3], s[2:3] th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x02,0x00,0x0d,0xee,0x00,0x00,0x10,0x01,0x01,0x00,0x00,0x00]
+
+global_atomic_cmpswap_b32 v0, v1, v[2:3], s[2:3] th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x02,0x00,0x0d,0xee,0x00,0x00,0x10,0x01,0x01,0x00,0x00,0x00]
+
+global_atomic_cmpswap_b32 v0, v1, v[2:3], s[2:3] offset:2047 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x02,0x00,0x0d,0xee,0x00,0x00,0x10,0x01,0x01,0xff,0x07,0x00]
+
+global_atomic_cmpswap_b32 v0, v1, v[2:3], s[2:3] offset:2047 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x02,0x00,0x0d,0xee,0x00,0x00,0x10,0x01,0x01,0xff,0x07,0x00]
+
+global_atomic_cmpswap_b32 v0, v[1:2], v[3:4], off th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x00,0x0d,0xee,0x00,0x00,0x90,0x01,0x01,0x00,0x00,0x00]
+
+global_atomic_cmpswap_b32 v0, v[1:2], v[3:4], off th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x00,0x0d,0xee,0x00,0x00,0x90,0x01,0x01,0x00,0x00,0x00]
+
+global_atomic_cmpswap_b32 v0, v[1:2], v[3:4], off offset:2047 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x00,0x0d,0xee,0x00,0x00,0x90,0x01,0x01,0xff,0x07,0x00]
+
+global_atomic_cmpswap_b32 v0, v[1:2], v[3:4], off offset:2047 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x00,0x0d,0xee,0x00,0x00,0x90,0x01,0x01,0xff,0x07,0x00]
+
+global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:-64
+// GFX12: encoding: [0x00,0x00,0x0d,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:64
+// GFX12: encoding: [0x00,0x00,0x0d,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x00,0x00,0x0d,0xee,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x00,0x00,0x0d,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_cmpswap_b32 v1, v[0:1], v[2:3], off offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x00,0x0d,0xee,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_cmpswap_b32 v1, v[0:1], v[2:3], off offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x00,0x0d,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_cmpswap_b32 v[0:1], v[2:3], off offset:-64
+// GFX12: encoding: [0x7c,0x00,0x0d,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_cmpswap_b32 v[0:1], v[2:3], off offset:64
+// GFX12: encoding: [0x7c,0x00,0x0d,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_cmpswap_b64 v0, v[2:5], s[0:1] offset:-64
+// GFX12: encoding: [0x00,0x80,0x10,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_cmpswap_b64 v0, v[2:5], s[0:1] offset:64
+// GFX12: encoding: [0x00,0x80,0x10,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_cmpswap_b64 v[0:1], v[2:5], off offset:-64
+// GFX12: encoding: [0x7c,0x80,0x10,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_cmpswap_b64 v[0:1], v[2:5], off offset:64
+// GFX12: encoding: [0x7c,0x80,0x10,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_cmpswap_b64 v[1:2], v0, v[2:5], s[0:1] offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x00,0x80,0x10,0xee,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xf
+
+global_atomic_cmpswap_b64 v[1:2], v0, v[2:5], s[0:1] offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x00,0x80,0x10,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+global_atomic_cmpswap_b64 v[1:2], v3, v[5:8], s[2:3] th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x02,0x80,0x10,0xee,0x01,0x00,0x90,0x02,0x03,0x00,0x00,0x00]
+
+global_atomic_cmpswap_b64 v[1:2], v3, v[5:8], s[2:3] th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x02,0x80,0x10,0xee,0x01,0x00,0x90,0x02,0x03,0x00,0x00,0x00]
+
+global_atomic_cmpswap_b64 v[1:2], v3, v[5:8], s[2:3] offset:2047 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x02,0x80,0x10,0xee,0x01,0x00,0x90,0x02,0x03,0xff,0x07,0x
+
+global_atomic_cmpswap_b64 v[1:2], v3, v[5:8], s[2:3] offset:2047 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x02,0x80,0x10,0xee,0x01,0x00,0x90,0x02,0x03,0xff,0x07,0x
+
+global_atomic_cmpswap_b64 v[1:2], v[0:1], v[2:5], off offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x80,0x10,0xee,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0x
+
+global_atomic_cmpswap_b64 v[1:2], v[0:1], v[2:5], off offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x80,0x10,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x0
+
+global_atomic_cmpswap_b64 v[1:2], v[3:4], v[5:8], off th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x80,0x10,0xee,0x01,0x00,0x90,0x02,0x03,0x00,0x00,0x00]
+
+global_atomic_cmpswap_b64 v[1:2], v[3:4], v[5:8], off th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x80,0x10,0xee,0x01,0x00,0x90,0x02,0x03,0x00,0x00,0x00]
+
+global_atomic_cmpswap_b64 v[1:2], v[3:4], v[5:8], off offset:2047 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x80,0x10,0xee,0x01,0x00,0x90,0x02,0x03,0xff,0x07,0
+
+global_atomic_cmpswap_b64 v[1:2], v[3:4], v[5:8], off offset:2047 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x80,0x10,0xee,0x01,0x00,0x90,0x02,0x03,0xff,0x07,0
+
+global_atomic_sub_clamp_u32 v1, v0, v2, s[0:1] offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x00,0xc0,0x0d,0xee,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_sub_clamp_u32 v1, v0, v2, s[0:1] offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x00,0xc0,0x0d,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_sub_clamp_u32 v0, v2, s[0:1] offset:64
+// GFX12: encoding: [0x00,0xc0,0x0d,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_sub_clamp_u32 v1, v[0:1], v2, off offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0xc0,0x0d,0xee,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_sub_clamp_u32 v1, v[0:1], v2, off offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0xc0,0x0d,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_sub_clamp_u32 v[0:1], v2, off offset:64
+// GFX12: encoding: [0x7c,0xc0,0x0d,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_dec_u32 v0, v2, s[0:1] offset:-64
+// GFX12: encoding: [0x00,0x00,0x10,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_dec_u32 v0, v2, s[0:1] offset:64
+// GFX12: encoding: [0x00,0x00,0x10,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_dec_u32 v1, v0, v2, s[0:1] offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x00,0x00,0x10,0xee,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_dec_u32 v1, v0, v2, s[0:1] offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x00,0x00,0x10,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_dec_u32 v1, v[0:1], v2, off offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x00,0x10,0xee,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_dec_u32 v1, v[0:1], v2, off offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x00,0x10,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_dec_u32 v[0:1], v2, off offset:-64
+// GFX12: encoding: [0x7c,0x00,0x10,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_dec_u32 v[0:1], v2, off offset:64
+// GFX12: encoding: [0x7c,0x00,0x10,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_dec_u64 v0, v[2:3], s[0:1] offset:-64
+// GFX12: encoding: [0x00,0x40,0x13,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_dec_u64 v0, v[2:3], s[0:1] offset:64
+// GFX12: encoding: [0x00,0x40,0x13,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_dec_u64 v[0:1], v[2:3], off offset:-64
+// GFX12: encoding: [0x7c,0x40,0x13,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_dec_u64 v[0:1], v[2:3], off offset:64
+// GFX12: encoding: [0x7c,0x40,0x13,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_dec_u64 v[1:2], v0, v[2:3], s[0:1] offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x00,0x40,0x13,0xee,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_dec_u64 v[1:2], v0, v[2:3], s[0:1] offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x00,0x40,0x13,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_dec_u64 v[1:2], v[0:1], v[2:3], off offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x40,0x13,0xee,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_dec_u64 v[1:2], v[0:1], v[2:3], off offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x40,0x13,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_inc_u32 v0, v2, s[0:1] offset:-64
+// GFX12: encoding: [0x00,0xc0,0x0f,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_inc_u32 v0, v2, s[0:1] offset:64
+// GFX12: encoding: [0x00,0xc0,0x0f,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_inc_u32 v1, v0, v2, s[0:1] offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x00,0xc0,0x0f,0xee,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_inc_u32 v1, v0, v2, s[0:1] offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x00,0xc0,0x0f,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_inc_u32 v1, v[0:1], v2, off offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0xc0,0x0f,0xee,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_inc_u32 v1, v[0:1], v2, off offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0xc0,0x0f,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_inc_u32 v[0:1], v2, off offset:-64
+// GFX12: encoding: [0x7c,0xc0,0x0f,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_inc_u32 v[0:1], v2, off offset:64
+// GFX12: encoding: [0x7c,0xc0,0x0f,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_inc_u64 v0, v[2:3], s[0:1] offset:-64
+// GFX12: encoding: [0x00,0x00,0x13,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_inc_u64 v0, v[2:3], s[0:1] offset:64
+// GFX12: encoding: [0x00,0x00,0x13,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_inc_u64 v[0:1], v[2:3], off offset:-64
+// GFX12: encoding: [0x7c,0x00,0x13,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_inc_u64 v[0:1], v[2:3], off offset:64
+// GFX12: encoding: [0x7c,0x00,0x13,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_inc_u64 v[1:2], v0, v[2:3], s[0:1] offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x00,0x00,0x13,0xee,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_inc_u64 v[1:2], v0, v[2:3], s[0:1] offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x00,0x00,0x13,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_inc_u64 v[1:2], v[0:1], v[2:3], off offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x00,0x13,0xee,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_inc_u64 v[1:2], v[0:1], v[2:3], off offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x00,0x13,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_max_num_f32 v0, v2, s[0:1] offset:-64
+// GFX12: encoding: [0x00,0x80,0x14,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_max_num_f32 v0, v2, s[0:1] offset:64
+// GFX12: encoding: [0x00,0x80,0x14,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_max_num_f32 v1, v0, v2, s[0:1] offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x00,0x80,0x14,0xee,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_max_num_f32 v1, v0, v2, s[0:1] offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x00,0x80,0x14,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_max_num_f32 v1, v[0:1], v2, off offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x80,0x14,0xee,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_max_num_f32 v1, v[0:1], v2, off offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x80,0x14,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_max_num_f32 v[0:1], v2, off offset:-64
+// GFX12: encoding: [0x7c,0x80,0x14,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_max_num_f32 v[0:1], v2, off offset:64
+// GFX12: encoding: [0x7c,0x80,0x14,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_max_i32 v0, v2, s[0:1] offset:-64
+// GFX12: encoding: [0x00,0x80,0x0e,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_max_i32 v0, v2, s[0:1] offset:64
+// GFX12: encoding: [0x00,0x80,0x0e,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_max_i32 v1, v0, v2, s[0:1] offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x00,0x80,0x0e,0xee,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_max_i32 v1, v0, v2, s[0:1] offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x00,0x80,0x0e,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_max_i32 v1, v[0:1], v2, off offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x80,0x0e,0xee,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_max_i32 v1, v[0:1], v2, off offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x80,0x0e,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_max_i32 v[0:1], v2, off offset:-64
+// GFX12: encoding: [0x7c,0x80,0x0e,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_max_i32 v[0:1], v2, off offset:64
+// GFX12: encoding: [0x7c,0x80,0x0e,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_max_i64 v0, v[2:3], s[0:1] offset:-64
+// GFX12: encoding: [0x00,0xc0,0x11,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_max_i64 v0, v[2:3], s[0:1] offset:64
+// GFX12: encoding: [0x00,0xc0,0x11,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_max_i64 v[0:1], v[2:3], off offset:-64
+// GFX12: encoding: [0x7c,0xc0,0x11,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_max_i64 v[0:1], v[2:3], off offset:64
+// GFX12: encoding: [0x7c,0xc0,0x11,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_max_i64 v[1:2], v0, v[2:3], s[0:1] offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x00,0xc0,0x11,0xee,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_max_i64 v[1:2], v0, v[2:3], s[0:1] offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x00,0xc0,0x11,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_max_i64 v[1:2], v[0:1], v[2:3], off offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0xc0,0x11,0xee,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_max_i64 v[1:2], v[0:1], v[2:3], off offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0xc0,0x11,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_max_u32 v0, v2, s[0:1] offset:-64
+// GFX12: encoding: [0x00,0xc0,0x0e,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_max_u32 v0, v2, s[0:1] offset:64
+// GFX12: encoding: [0x00,0xc0,0x0e,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_max_u32 v1, v0, v2, s[0:1] offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x00,0xc0,0x0e,0xee,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_max_u32 v1, v0, v2, s[0:1] offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x00,0xc0,0x0e,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_max_u32 v1, v[0:1], v2, off offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0xc0,0x0e,0xee,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_max_u32 v1, v[0:1], v2, off offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0xc0,0x0e,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_max_u32 v[0:1], v2, off offset:-64
+// GFX12: encoding: [0x7c,0xc0,0x0e,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_max_u32 v[0:1], v2, off offset:64
+// GFX12: encoding: [0x7c,0xc0,0x0e,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_max_u64 v0, v[2:3], s[0:1] offset:-64
+// GFX12: encoding: [0x00,0x00,0x12,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_max_u64 v0, v[2:3], s[0:1] offset:64
+// GFX12: encoding: [0x00,0x00,0x12,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_max_u64 v[0:1], v[2:3], off offset:-64
+// GFX12: encoding: [0x7c,0x00,0x12,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_max_u64 v[0:1], v[2:3], off offset:64
+// GFX12: encoding: [0x7c,0x00,0x12,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_max_u64 v[1:2], v0, v[2:3], s[0:1] offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x00,0x00,0x12,0xee,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_max_u64 v[1:2], v0, v[2:3], s[0:1] offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x00,0x00,0x12,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_max_u64 v[1:2], v[0:1], v[2:3], off offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x00,0x12,0xee,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_max_u64 v[1:2], v[0:1], v[2:3], off offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x00,0x12,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_min_num_f32 v0, v2, s[0:1] offset:-64
+// GFX12: encoding: [0x00,0x40,0x14,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_min_num_f32 v0, v2, s[0:1] offset:64
+// GFX12: encoding: [0x00,0x40,0x14,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_min_num_f32 v1, v0, v2, s[0:1] offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x00,0x40,0x14,0xee,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_min_num_f32 v1, v0, v2, s[0:1] offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x00,0x40,0x14,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_min_num_f32 v1, v[0:1], v2, off offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x40,0x14,0xee,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_min_num_f32 v1, v[0:1], v2, off offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x40,0x14,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_min_num_f32 v[0:1], v2, off offset:-64
+// GFX12: encoding: [0x7c,0x40,0x14,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_min_num_f32 v[0:1], v2, off offset:64
+// GFX12: encoding: [0x7c,0x40,0x14,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_min_i32 v0, v2, s[0:1] offset:-64
+// GFX12: encoding: [0x00,0x00,0x0e,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_min_i32 v0, v2, s[0:1] offset:64
+// GFX12: encoding: [0x00,0x00,0x0e,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_min_i32 v1, v0, v2, s[0:1] offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x00,0x00,0x0e,0xee,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_min_i32 v1, v0, v2, s[0:1] offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x00,0x00,0x0e,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_min_i32 v1, v[0:1], v2, off offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x00,0x0e,0xee,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_min_i32 v1, v[0:1], v2, off offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x00,0x0e,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_min_i32 v[0:1], v2, off offset:-64
+// GFX12: encoding: [0x7c,0x00,0x0e,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_min_i32 v[0:1], v2, off offset:64
+// GFX12: encoding: [0x7c,0x00,0x0e,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_min_i64 v0, v[2:3], s[0:1] offset:-64
+// GFX12: encoding: [0x00,0x40,0x11,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_min_i64 v0, v[2:3], s[0:1] offset:64
+// GFX12: encoding: [0x00,0x40,0x11,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_min_i64 v[0:1], v[2:3], off offset:-64
+// GFX12: encoding: [0x7c,0x40,0x11,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_min_i64 v[0:1], v[2:3], off offset:64
+// GFX12: encoding: [0x7c,0x40,0x11,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_min_i64 v[1:2], v0, v[2:3], s[0:1] offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x00,0x40,0x11,0xee,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_min_i64 v[1:2], v0, v[2:3], s[0:1] offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x00,0x40,0x11,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_min_i64 v[1:2], v[0:1], v[2:3], off offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x40,0x11,0xee,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_min_i64 v[1:2], v[0:1], v[2:3], off offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x40,0x11,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_min_u32 v0, v2, s[0:1] offset:-64
+// GFX12: encoding: [0x00,0x40,0x0e,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_min_u32 v0, v2, s[0:1] offset:64
+// GFX12: encoding: [0x00,0x40,0x0e,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_min_u32 v1, v0, v2, s[0:1] offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x00,0x40,0x0e,0xee,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_min_u32 v1, v0, v2, s[0:1] offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x00,0x40,0x0e,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_min_u32 v1, v[0:1], v2, off offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x40,0x0e,0xee,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_min_u32 v1, v[0:1], v2, off offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x40,0x0e,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_min_u32 v[0:1], v2, off offset:-64
+// GFX12: encoding: [0x7c,0x40,0x0e,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_min_u32 v[0:1], v2, off offset:64
+// GFX12: encoding: [0x7c,0x40,0x0e,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_min_u64 v0, v[2:3], s[0:1] offset:-64
+// GFX12: encoding: [0x00,0x80,0x11,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_min_u64 v0, v[2:3], s[0:1] offset:64
+// GFX12: encoding: [0x00,0x80,0x11,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_min_u64 v[0:1], v[2:3], off offset:-64
+// GFX12: encoding: [0x7c,0x80,0x11,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_min_u64 v[0:1], v[2:3], off offset:64
+// GFX12: encoding: [0x7c,0x80,0x11,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_min_u64 v[1:2], v0, v[2:3], s[0:1] offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x00,0x80,0x11,0xee,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_min_u64 v[1:2], v0, v[2:3], s[0:1] offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x00,0x80,0x11,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_min_u64 v[1:2], v[0:1], v[2:3], off offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x80,0x11,0xee,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_min_u64 v[1:2], v[0:1], v[2:3], off offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x80,0x11,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_or_b32 v0, v2, s[0:1] offset:-64
+// GFX12: encoding: [0x00,0x40,0x0f,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_or_b32 v0, v2, s[0:1] offset:64
+// GFX12: encoding: [0x00,0x40,0x0f,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_or_b32 v1, v0, v2, s[0:1] offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x00,0x40,0x0f,0xee,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_or_b32 v1, v0, v2, s[0:1] offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x00,0x40,0x0f,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_or_b32 v1, v[0:1], v2, off offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x40,0x0f,0xee,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_or_b32 v1, v[0:1], v2, off offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x40,0x0f,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_or_b32 v[0:1], v2, off offset:-64
+// GFX12: encoding: [0x7c,0x40,0x0f,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_or_b32 v[0:1], v2, off offset:64
+// GFX12: encoding: [0x7c,0x40,0x0f,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_or_b64 v0, v[2:3], s[0:1] offset:-64
+// GFX12: encoding: [0x00,0x80,0x12,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_or_b64 v0, v[2:3], s[0:1] offset:64
+// GFX12: encoding: [0x00,0x80,0x12,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_or_b64 v[0:1], v[2:3], off offset:-64
+// GFX12: encoding: [0x7c,0x80,0x12,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_or_b64 v[0:1], v[2:3], off offset:64
+// GFX12: encoding: [0x7c,0x80,0x12,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_or_b64 v[1:2], v0, v[2:3], s[0:1] offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x00,0x80,0x12,0xee,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_or_b64 v[1:2], v0, v[2:3], s[0:1] offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x00,0x80,0x12,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_or_b64 v[1:2], v[0:1], v[2:3], off offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x80,0x12,0xee,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_or_b64 v[1:2], v[0:1], v[2:3], off offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x80,0x12,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_sub_u32 v0, v2, s[0:1] offset:-64
+// GFX12: encoding: [0x00,0x80,0x0d,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_sub_u32 v0, v2, s[0:1] offset:64
+// GFX12: encoding: [0x00,0x80,0x0d,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_sub_u32 v1, v0, v2, s[0:1] offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x00,0x80,0x0d,0xee,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_sub_u32 v1, v0, v2, s[0:1] offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x00,0x80,0x0d,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_sub_u32 v1, v[0:1], v2, off offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x80,0x0d,0xee,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_sub_u32 v1, v[0:1], v2, off offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x80,0x0d,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_sub_u32 v[0:1], v2, off offset:-64
+// GFX12: encoding: [0x7c,0x80,0x0d,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_sub_u32 v[0:1], v2, off offset:64
+// GFX12: encoding: [0x7c,0x80,0x0d,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_sub_u64 v0, v[2:3], s[0:1] offset:-64
+// GFX12: encoding: [0x00,0x00,0x11,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_sub_u64 v0, v[2:3], s[0:1] offset:64
+// GFX12: encoding: [0x00,0x00,0x11,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_sub_u64 v[0:1], v[2:3], off offset:-64
+// GFX12: encoding: [0x7c,0x00,0x11,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_sub_u64 v[0:1], v[2:3], off offset:64
+// GFX12: encoding: [0x7c,0x00,0x11,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_sub_u64 v[1:2], v0, v[2:3], s[0:1] offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x00,0x00,0x11,0xee,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_sub_u64 v[1:2], v0, v[2:3], s[0:1] offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x00,0x00,0x11,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_sub_u64 v[1:2], v[0:1], v[2:3], off offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x00,0x11,0xee,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_sub_u64 v[1:2], v[0:1], v[2:3], off offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x00,0x11,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_swap_b32 v0, v1, v3, s[2:3] th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x02,0xc0,0x0c,0xee,0x00,0x00,0x90,0x01,0x01,0x00,0x00,0x00]
+
+global_atomic_swap_b32 v0, v1, v3, s[2:3] th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x02,0xc0,0x0c,0xee,0x00,0x00,0x90,0x01,0x01,0x00,0x00,0x00]
+
+global_atomic_swap_b32 v0, v1, v3, s[2:3] offset:2047 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x02,0xc0,0x0c,0xee,0x00,0x00,0x90,0x01,0x01,0xff,0x07,0x00]
+
+global_atomic_swap_b32 v0, v1, v3, s[2:3] offset:2047 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x02,0xc0,0x0c,0xee,0x00,0x00,0x90,0x01,0x01,0xff,0x07,0x00]
+
+global_atomic_swap_b32 v0, v2, s[0:1] offset:-64
+// GFX12: encoding: [0x00,0xc0,0x0c,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_swap_b32 v0, v2, s[0:1] offset:64
+// GFX12: encoding: [0x00,0xc0,0x0c,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_swap_b32 v0, v[1:2], v3, off th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0xc0,0x0c,0xee,0x00,0x00,0x90,0x01,0x01,0x00,0x00,0x00]
+
+global_atomic_swap_b32 v0, v[1:2], v3, off th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0xc0,0x0c,0xee,0x00,0x00,0x90,0x01,0x01,0x00,0x00,0x00]
+
+global_atomic_swap_b32 v0, v[1:2], v3, off offset:2047 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0xc0,0x0c,0xee,0x00,0x00,0x90,0x01,0x01,0xff,0x07,0x00]
+
+global_atomic_swap_b32 v0, v[1:2], v3, off offset:2047 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0xc0,0x0c,0xee,0x00,0x00,0x90,0x01,0x01,0xff,0x07,0x00]
+
+global_atomic_swap_b32 v1, v0, v2, s[0:1] offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x00,0xc0,0x0c,0xee,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_swap_b32 v1, v0, v2, s[0:1] offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x00,0xc0,0x0c,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_swap_b32 v1, v[0:1], v2, off offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0xc0,0x0c,0xee,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_swap_b32 v1, v[0:1], v2, off offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0xc0,0x0c,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_swap_b32 v[0:1], v2, off offset:-64
+// GFX12: encoding: [0x7c,0xc0,0x0c,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_swap_b32 v[0:1], v2, off offset:64
+// GFX12: encoding: [0x7c,0xc0,0x0c,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_swap_b64 v0, v[2:3], s[0:1] offset:-64
+// GFX12: encoding: [0x00,0x40,0x10,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_swap_b64 v0, v[2:3], s[0:1] offset:64
+// GFX12: encoding: [0x00,0x40,0x10,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_swap_b64 v[0:1], v[2:3], off offset:-64
+// GFX12: encoding: [0x7c,0x40,0x10,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_swap_b64 v[0:1], v[2:3], off offset:64
+// GFX12: encoding: [0x7c,0x40,0x10,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_swap_b64 v[1:2], v0, v[2:3], s[0:1] offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x00,0x40,0x10,0xee,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_swap_b64 v[1:2], v0, v[2:3], s[0:1] offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x00,0x40,0x10,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_swap_b64 v[1:2], v3, v[5:6], s[2:3] th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x02,0x40,0x10,0xee,0x01,0x00,0x90,0x02,0x03,0x00,0x00,0x00]
+
+global_atomic_swap_b64 v[1:2], v3, v[5:6], s[2:3] th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x02,0x40,0x10,0xee,0x01,0x00,0x90,0x02,0x03,0x00,0x00,0x00]
+
+global_atomic_swap_b64 v[1:2], v3, v[5:6], s[2:3] offset:2047 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x02,0x40,0x10,0xee,0x01,0x00,0x90,0x02,0x03,0xff,0x07,0x00]
+
+global_atomic_swap_b64 v[1:2], v3, v[5:6], s[2:3] offset:2047 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x02,0x40,0x10,0xee,0x01,0x00,0x90,0x02,0x03,0xff,0x07,0x00]
+
+global_atomic_swap_b64 v[1:2], v[0:1], v[2:3], off offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x40,0x10,0xee,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_swap_b64 v[1:2], v[0:1], v[2:3], off offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x40,0x10,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_swap_b64 v[1:2], v[3:4], v[5:6], off th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x40,0x10,0xee,0x01,0x00,0x90,0x02,0x03,0x00,0x00,0x00]
+
+global_atomic_swap_b64 v[1:2], v[3:4], v[5:6], off th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x40,0x10,0xee,0x01,0x00,0x90,0x02,0x03,0x00,0x00,0x00]
+
+global_atomic_swap_b64 v[1:2], v[3:4], v[5:6], off offset:2047 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x40,0x10,0xee,0x01,0x00,0x90,0x02,0x03,0xff,0x07,0x00
+
+global_atomic_swap_b64 v[1:2], v[3:4], v[5:6], off offset:2047 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x40,0x10,0xee,0x01,0x00,0x90,0x02,0x03,0xff,0x07,0x00
+
+global_atomic_xor_b32 v0, v2, s[0:1] offset:-64
+// GFX12: encoding: [0x00,0x80,0x0f,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_xor_b32 v0, v2, s[0:1] offset:64
+// GFX12: encoding: [0x00,0x80,0x0f,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_xor_b32 v1, v0, v2, s[0:1] offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x00,0x80,0x0f,0xee,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_xor_b32 v1, v0, v2, s[0:1] offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x00,0x80,0x0f,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_xor_b32 v1, v[0:1], v2, off offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x80,0x0f,0xee,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_xor_b32 v1, v[0:1], v2, off offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0x80,0x0f,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_xor_b32 v[0:1], v2, off offset:-64
+// GFX12: encoding: [0x7c,0x80,0x0f,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_xor_b32 v[0:1], v2, off offset:64
+// GFX12: encoding: [0x7c,0x80,0x0f,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_xor_b64 v0, v[2:3], s[0:1] offset:-64
+// GFX12: encoding: [0x00,0xc0,0x12,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_xor_b64 v0, v[2:3], s[0:1] offset:64
+// GFX12: encoding: [0x00,0xc0,0x12,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_xor_b64 v[0:1], v[2:3], off offset:-64
+// GFX12: encoding: [0x7c,0xc0,0x12,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_xor_b64 v[0:1], v[2:3], off offset:64
+// GFX12: encoding: [0x7c,0xc0,0x12,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_xor_b64 v[1:2], v0, v[2:3], s[0:1] offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x00,0xc0,0x12,0xee,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_xor_b64 v[1:2], v0, v[2:3], s[0:1] offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x00,0xc0,0x12,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_xor_b64 v[1:2], v[0:1], v[2:3], off offset:-64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0xc0,0x12,0xee,0x01,0x00,0x10,0x01,0x00,0xc0,0xff,0xff]
+
+global_atomic_xor_b64 v[1:2], v[0:1], v[2:3], off offset:64 th:TH_ATOMIC_RETURN
+// GFX12: encoding: [0x7c,0xc0,0x12,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+global_load_addtid_b32 v1, off offset:-64
+// GFX12: encoding: [0x7c,0x00,0x0a,0xee,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+global_load_addtid_b32 v1, off offset:64
+// GFX12: encoding: [0x7c,0x00,0x0a,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+global_load_addtid_b32 v1, off
+// GFX12: encoding: [0x7c,0x00,0x0a,0xee,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00]
+
+global_load_addtid_b32 v1, s[0:1] offset:-64
+// GFX12: encoding: [0x00,0x00,0x0a,0xee,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+global_load_addtid_b32 v1, s[0:1] offset:64
+// GFX12: encoding: [0x00,0x00,0x0a,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+global_load_addtid_b32 v1, s[2:3]
+// GFX12: encoding: [0x02,0x00,0x0a,0xee,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00]
+
+global_load_b128 v[1:4], v0, s[0:1] offset:-64
+// GFX12: encoding: [0x00,0xc0,0x05,0xee,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+global_load_b128 v[1:4], v0, s[0:1] offset:64
+// GFX12: encoding: [0x00,0xc0,0x05,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+global_load_b128 v[1:4], v5, s[2:3]
+// GFX12: encoding: [0x02,0xc0,0x05,0xee,0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00]
+
+global_load_b128 v[1:4], v[0:1], off offset:-64
+// GFX12: encoding: [0x7c,0xc0,0x05,0xee,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+global_load_b128 v[1:4], v[0:1], off offset:64
+// GFX12: encoding: [0x7c,0xc0,0x05,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+global_load_b128 v[1:4], v[5:6], off
+// GFX12: encoding: [0x7c,0xc0,0x05,0xee,0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00]
+
+global_load_b32 v1, v0, s[0:1] offset:-64
+// GFX12: encoding: [0x00,0x00,0x05,0xee,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+global_load_b32 v1, v0, s[0:1] offset:64
+// GFX12: encoding: [0x00,0x00,0x05,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+global_load_b32 v1, v3, s[2:3] offset:2047
+// GFX12: encoding: [0x02,0x00,0x05,0xee,0x01,0x00,0x00,0x00,0x03,0xff,0x07,0x00]
+
+global_load_b32 v1, v3, s[2:3]
+// GFX12: encoding: [0x02,0x00,0x05,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+
+global_load_b32 v1, v[0:1], off offset:-64
+// GFX12: encoding: [0x7c,0x00,0x05,0xee,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+global_load_b32 v1, v[0:1], off offset:64
+// GFX12: encoding: [0x7c,0x00,0x05,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+global_load_b32 v1, v[3:4], off offset:2047
+// GFX12: encoding: [0x7c,0x00,0x05,0xee,0x01,0x00,0x00,0x00,0x03,0xff,0x07,0x00]
+
+global_load_b32 v1, v[3:4], off
+// GFX12: encoding: [0x7c,0x00,0x05,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+
+global_load_b64 v[1:2], v0, s[0:1] offset:-64
+// GFX12: encoding: [0x00,0x40,0x05,0xee,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+global_load_b64 v[1:2], v0, s[0:1] offset:64
+// GFX12: encoding: [0x00,0x40,0x05,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+global_load_b64 v[1:2], v3, s[2:3]
+// GFX12: encoding: [0x02,0x40,0x05,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+
+global_load_b64 v[1:2], v[0:1], off offset:-64
+// GFX12: encoding: [0x7c,0x40,0x05,0xee,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+global_load_b64 v[1:2], v[0:1], off offset:64
+// GFX12: encoding: [0x7c,0x40,0x05,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+global_load_b64 v[1:2], v[3:4], off
+// GFX12: encoding: [0x7c,0x40,0x05,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+
+global_load_b96 v[1:3], v0, s[0:1] offset:-64
+// GFX12: encoding: [0x00,0x80,0x05,0xee,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+global_load_b96 v[1:3], v0, s[0:1] offset:64
+// GFX12: encoding: [0x00,0x80,0x05,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+global_load_b96 v[1:3], v5, s[2:3]
+// GFX12: encoding: [0x02,0x80,0x05,0xee,0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00]
+
+global_load_b96 v[1:3], v[0:1], off offset:-64
+// GFX12: encoding: [0x7c,0x80,0x05,0xee,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+global_load_b96 v[1:3], v[0:1], off offset:64
+// GFX12: encoding: [0x7c,0x80,0x05,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+global_load_b96 v[1:3], v[5:6], off
+// GFX12: encoding: [0x7c,0x80,0x05,0xee,0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00]
+
+global_load_d16_b16 v1, v0, s[0:1] offset:-64
+// GFX12: encoding: [0x00,0x00,0x08,0xee,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+global_load_d16_b16 v1, v0, s[0:1] offset:64
+// GFX12: encoding: [0x00,0x00,0x08,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+global_load_d16_b16 v1, v3, s[2:3]
+// GFX12: encoding: [0x02,0x00,0x08,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+
+global_load_d16_b16 v1, v[0:1], off offset:-64
+// GFX12: encoding: [0x7c,0x00,0x08,0xee,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+global_load_d16_b16 v1, v[0:1], off offset:64
+// GFX12: encoding: [0x7c,0x00,0x08,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+global_load_d16_b16 v1, v[3:4], off
+// GFX12: encoding: [0x7c,0x00,0x08,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+
+global_load_d16_hi_b16 v1, v0, s[0:1] offset:-64
+// GFX12: encoding: [0x00,0xc0,0x08,0xee,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+global_load_d16_hi_b16 v1, v0, s[0:1] offset:64
+// GFX12: encoding: [0x00,0xc0,0x08,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+global_load_d16_hi_b16 v1, v3, s[2:3]
+// GFX12: encoding: [0x02,0xc0,0x08,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+
+global_load_d16_hi_b16 v1, v[0:1], off offset:-64
+// GFX12: encoding: [0x7c,0xc0,0x08,0xee,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+global_load_d16_hi_b16 v1, v[0:1], off offset:64
+// GFX12: encoding: [0x7c,0xc0,0x08,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+global_load_d16_hi_b16 v1, v[3:4], off
+// GFX12: encoding: [0x7c,0xc0,0x08,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+
+global_load_d16_hi_i8 v1, v0, s[0:1] offset:-64
+// GFX12: encoding: [0x00,0x80,0x08,0xee,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+global_load_d16_hi_i8 v1, v0, s[0:1] offset:64
+// GFX12: encoding: [0x00,0x80,0x08,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+global_load_d16_hi_i8 v1, v3, s[2:3]
+// GFX12: encoding: [0x02,0x80,0x08,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+
+global_load_d16_hi_i8 v1, v[0:1], off offset:-64
+// GFX12: encoding: [0x7c,0x80,0x08,0xee,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+global_load_d16_hi_i8 v1, v[0:1], off offset:64
+// GFX12: encoding: [0x7c,0x80,0x08,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+global_load_d16_hi_i8 v1, v[3:4], off
+// GFX12: encoding: [0x7c,0x80,0x08,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+
+global_load_d16_hi_u8 v1, v0, s[0:1] offset:-64
+// GFX12: encoding: [0x00,0x40,0x08,0xee,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+global_load_d16_hi_u8 v1, v0, s[0:1] offset:64
+// GFX12: encoding: [0x00,0x40,0x08,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+global_load_d16_hi_u8 v1, v3, s[2:3]
+// GFX12: encoding: [0x02,0x40,0x08,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+
+global_load_d16_hi_u8 v1, v[0:1], off offset:-64
+// GFX12: encoding: [0x7c,0x40,0x08,0xee,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+global_load_d16_hi_u8 v1, v[0:1], off offset:64
+// GFX12: encoding: [0x7c,0x40,0x08,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+global_load_d16_hi_u8 v1, v[3:4], off
+// GFX12: encoding: [0x7c,0x40,0x08,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+
+global_load_d16_i8 v1, v0, s[0:1] offset:-64
+// GFX12: encoding: [0x00,0xc0,0x07,0xee,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+global_load_d16_i8 v1, v0, s[0:1] offset:64
+// GFX12: encoding: [0x00,0xc0,0x07,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+global_load_d16_i8 v1, v3, s[2:3]
+// GFX12: encoding: [0x02,0xc0,0x07,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+
+global_load_d16_i8 v1, v[0:1], off offset:-64
+// GFX12: encoding: [0x7c,0xc0,0x07,0xee,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+global_load_d16_i8 v1, v[0:1], off offset:64
+// GFX12: encoding: [0x7c,0xc0,0x07,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+global_load_d16_i8 v1, v[3:4], off
+// GFX12: encoding: [0x7c,0xc0,0x07,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+
+global_load_d16_u8 v1, v0, s[0:1] offset:-64
+// GFX12: encoding: [0x00,0x80,0x07,0xee,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+global_load_d16_u8 v1, v0, s[0:1] offset:64
+// GFX12: encoding: [0x00,0x80,0x07,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+global_load_d16_u8 v1, v3, s[2:3]
+// GFX12: encoding: [0x02,0x80,0x07,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+
+global_load_d16_u8 v1, v[0:1], off offset:-64
+// GFX12: encoding: [0x7c,0x80,0x07,0xee,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+global_load_d16_u8 v1, v[0:1], off offset:64
+// GFX12: encoding: [0x7c,0x80,0x07,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+global_load_d16_u8 v1, v[3:4], off
+// GFX12: encoding: [0x7c,0x80,0x07,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+
+global_load_i16 v1, v0, s[0:1] offset:-64
+// GFX12: encoding: [0x00,0xc0,0x04,0xee,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+global_load_i16 v1, v0, s[0:1] offset:64
+// GFX12: encoding: [0x00,0xc0,0x04,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+global_load_i16 v1, v3, s[2:3]
+// GFX12: encoding: [0x02,0xc0,0x04,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+
+global_load_i16 v1, v[0:1], off offset:-64
+// GFX12: encoding: [0x7c,0xc0,0x04,0xee,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+global_load_i16 v1, v[0:1], off offset:64
+// GFX12: encoding: [0x7c,0xc0,0x04,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+global_load_i16 v1, v[3:4], off
+// GFX12: encoding: [0x7c,0xc0,0x04,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+
+global_load_i8 v1, v0, s[0:1] offset:-64
+// GFX12: encoding: [0x00,0x40,0x04,0xee,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+global_load_i8 v1, v0, s[0:1] offset:64
+// GFX12: encoding: [0x00,0x40,0x04,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+global_load_i8 v1, v3, s[2:3]
+// GFX12: encoding: [0x02,0x40,0x04,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+
+global_load_i8 v1, v[0:1], off offset:-64
+// GFX12: encoding: [0x7c,0x40,0x04,0xee,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+global_load_i8 v1, v[0:1], off offset:64
+// GFX12: encoding: [0x7c,0x40,0x04,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+global_load_i8 v1, v[3:4], off
+// GFX12: encoding: [0x7c,0x40,0x04,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+
+global_load_u16 v1, v0, s[0:1] offset:-64
+// GFX12: encoding: [0x00,0x80,0x04,0xee,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+global_load_u16 v1, v0, s[0:1] offset:64
+// GFX12: encoding: [0x00,0x80,0x04,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+global_load_u16 v1, v3, s[2:3]
+// GFX12: encoding: [0x02,0x80,0x04,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+
+global_load_u16 v1, v[0:1], off offset:-64
+// GFX12: encoding: [0x7c,0x80,0x04,0xee,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+global_load_u16 v1, v[0:1], off offset:64
+// GFX12: encoding: [0x7c,0x80,0x04,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+global_load_u16 v1, v[3:4], off
+// GFX12: encoding: [0x7c,0x80,0x04,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+
+global_load_u8 v1, v0, s[0:1] offset:-64
+// GFX12: encoding: [0x00,0x00,0x04,0xee,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+global_load_u8 v1, v0, s[0:1] offset:64
+// GFX12: encoding: [0x00,0x00,0x04,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+global_load_u8 v1, v3, s[2:3]
+// GFX12: encoding: [0x02,0x00,0x04,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+
+global_load_u8 v1, v[0:1], off offset:-64
+// GFX12: encoding: [0x7c,0x00,0x04,0xee,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+global_load_u8 v1, v[0:1], off offset:64
+// GFX12: encoding: [0x7c,0x00,0x04,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+global_load_u8 v1, v[3:4], off
+// GFX12: encoding: [0x7c,0x00,0x04,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+
+global_store_addtid_b32 v2, off offset:-64
+// GFX12: encoding: [0x7c,0x40,0x0a,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_store_addtid_b32 v2, off offset:64
+// GFX12: encoding: [0x7c,0x40,0x0a,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_store_addtid_b32 v2, s[0:1] offset:-64
+// GFX12: encoding: [0x00,0x40,0x0a,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_store_addtid_b32 v2, s[0:1] offset:64
+// GFX12: encoding: [0x00,0x40,0x0a,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_store_b128 v0, v[2:5], s[0:1] offset:-64
+// GFX12: encoding: [0x00,0x40,0x07,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_store_b128 v0, v[2:5], s[0:1] offset:64
+// GFX12: encoding: [0x00,0x40,0x07,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_store_b128 v1, v[3:6], s[2:3]
+// GFX12: encoding: [0x02,0x40,0x07,0xee,0x00,0x00,0x80,0x01,0x01,0x00,0x00,0x00]
+
+global_store_b128 v[0:1], v[2:5], off offset:-64
+// GFX12: encoding: [0x7c,0x40,0x07,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_store_b128 v[0:1], v[2:5], off offset:64
+// GFX12: encoding: [0x7c,0x40,0x07,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_store_b128 v[1:2], v[3:6], off
+// GFX12: encoding: [0x7c,0x40,0x07,0xee,0x00,0x00,0x80,0x01,0x01,0x00,0x00,0x00]
+
+global_store_b16 v0, v2, s[0:1] offset:-64
+// GFX12: encoding: [0x00,0x40,0x06,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_store_b16 v0, v2, s[0:1] offset:64
+// GFX12: encoding: [0x00,0x40,0x06,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_store_b16 v3, v1, s[2:3]
+// GFX12: encoding: [0x02,0x40,0x06,0xee,0x00,0x00,0x80,0x00,0x03,0x00,0x00,0x00]
+
+global_store_b16 v[0:1], v2, off offset:-64
+// GFX12: encoding: [0x7c,0x40,0x06,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_store_b16 v[0:1], v2, off offset:64
+// GFX12: encoding: [0x7c,0x40,0x06,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_store_b16 v[3:4], v1, off
+// GFX12: encoding: [0x7c,0x40,0x06,0xee,0x00,0x00,0x80,0x00,0x03,0x00,0x00,0x00]
+
+global_store_b32 v0, v2, s[0:1] offset:-64
+// GFX12: encoding: [0x00,0x80,0x06,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_store_b32 v0, v2, s[0:1] offset:64
+// GFX12: encoding: [0x00,0x80,0x06,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_store_b32 v3, v1, s[2:3] offset:-16
+// GFX12: encoding: [0x02,0x80,0x06,0xee,0x00,0x00,0x80,0x00,0x03,0xf0,0xff,0xff]
+
+global_store_b32 v3, v1, s[2:3] offset:16
+// GFX12: encoding: [0x02,0x80,0x06,0xee,0x00,0x00,0x80,0x00,0x03,0x10,0x00,0x00]
+
+global_store_b32 v[0:1], v2, off offset:-64
+// GFX12: encoding: [0x7c,0x80,0x06,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_store_b32 v[0:1], v2, off offset:64
+// GFX12: encoding: [0x7c,0x80,0x06,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_store_b32 v[3:4], v1, off offset:-16
+// GFX12: encoding: [0x7c,0x80,0x06,0xee,0x00,0x00,0x80,0x00,0x03,0xf0,0xff,0xff]
+
+global_store_b32 v[3:4], v1, off offset:16
+// GFX12: encoding: [0x7c,0x80,0x06,0xee,0x00,0x00,0x80,0x00,0x03,0x10,0x00,0x00]
+
+global_store_b64 v0, v[2:3], s[0:1] offset:-64
+// GFX12: encoding: [0x00,0xc0,0x06,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_store_b64 v0, v[2:3], s[0:1] offset:64
+// GFX12: encoding: [0x00,0xc0,0x06,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_store_b64 v1, v[2:3], s[2:3]
+// GFX12: encoding: [0x02,0xc0,0x06,0xee,0x00,0x00,0x00,0x01,0x01,0x00,0x00,0x00]
+
+global_store_b64 v[0:1], v[2:3], off offset:-64
+// GFX12: encoding: [0x7c,0xc0,0x06,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_store_b64 v[0:1], v[2:3], off offset:64
+// GFX12: encoding: [0x7c,0xc0,0x06,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_store_b64 v[1:2], v[3:4], off
+// GFX12: encoding: [0x7c,0xc0,0x06,0xee,0x00,0x00,0x80,0x01,0x01,0x00,0x00,0x00]
+
+global_store_b8 v0, v2, s[0:1] offset:-64
+// GFX12: encoding: [0x00,0x00,0x06,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_store_b8 v0, v2, s[0:1] offset:64
+// GFX12: encoding: [0x00,0x00,0x06,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_store_b8 v3, v1, s[2:3]
+// GFX12: encoding: [0x02,0x00,0x06,0xee,0x00,0x00,0x80,0x00,0x03,0x00,0x00,0x00]
+
+global_store_b8 v[0:1], v2, off offset:-64
+// GFX12: encoding: [0x7c,0x00,0x06,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_store_b8 v[0:1], v2, off offset:64
+// GFX12: encoding: [0x7c,0x00,0x06,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_store_b8 v[3:4], v1, off
+// GFX12: encoding: [0x7c,0x00,0x06,0xee,0x00,0x00,0x80,0x00,0x03,0x00,0x00,0x00]
+
+global_store_b96 v0, v[2:4], s[0:1] offset:-64
+// GFX12: encoding: [0x00,0x00,0x07,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_store_b96 v0, v[2:4], s[0:1] offset:64
+// GFX12: encoding: [0x00,0x00,0x07,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_store_b96 v1, v[3:5], s[2:3]
+// GFX12: encoding: [0x02,0x00,0x07,0xee,0x00,0x00,0x80,0x01,0x01,0x00,0x00,0x00]
+
+global_store_b96 v[0:1], v[2:4], off offset:-64
+// GFX12: encoding: [0x7c,0x00,0x07,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_store_b96 v[0:1], v[2:4], off offset:64
+// GFX12: encoding: [0x7c,0x00,0x07,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_store_b96 v[1:2], v[3:5], off
+// GFX12: encoding: [0x7c,0x00,0x07,0xee,0x00,0x00,0x80,0x01,0x01,0x00,0x00,0x00]
+
+global_store_d16_hi_b16 v0, v2, s[0:1] offset:-64
+// GFX12: encoding: [0x00,0x40,0x09,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_store_d16_hi_b16 v0, v2, s[0:1] offset:64
+// GFX12: encoding: [0x00,0x40,0x09,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_store_d16_hi_b16 v3, v1, s[2:3]
+// GFX12: encoding: [0x02,0x40,0x09,0xee,0x00,0x00,0x80,0x00,0x03,0x00,0x00,0x00]
+
+global_store_d16_hi_b16 v[0:1], v2, off offset:-64
+// GFX12: encoding: [0x7c,0x40,0x09,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_store_d16_hi_b16 v[0:1], v2, off offset:64
+// GFX12: encoding: [0x7c,0x40,0x09,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_store_d16_hi_b16 v[3:4], v1, off
+// GFX12: encoding: [0x7c,0x40,0x09,0xee,0x00,0x00,0x80,0x00,0x03,0x00,0x00,0x00]
+
+global_store_d16_hi_b8 v0, v2, s[0:1] offset:-64
+// GFX12: encoding: [0x00,0x00,0x09,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_store_d16_hi_b8 v0, v2, s[0:1] offset:64
+// GFX12: encoding: [0x00,0x00,0x09,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_store_d16_hi_b8 v3, v1, s[2:3]
+// GFX12: encoding: [0x02,0x00,0x09,0xee,0x00,0x00,0x80,0x00,0x03,0x00,0x00,0x00]
+
+global_store_d16_hi_b8 v[0:1], v2, off offset:-64
+// GFX12: encoding: [0x7c,0x00,0x09,0xee,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+global_store_d16_hi_b8 v[0:1], v2, off offset:64
+// GFX12: encoding: [0x7c,0x00,0x09,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_store_d16_hi_b8 v[3:4], v1, off
+// GFX12: encoding: [0x7c,0x00,0x09,0xee,0x00,0x00,0x80,0x00,0x03,0x00,0x00,0x00]
+
+scratch_load_b128 v[1:4], off, off offset:-64
+// GFX12: encoding: [0x7c,0xc0,0x05,0xed,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+scratch_load_b128 v[1:4], off, off offset:64
+// GFX12: encoding: [0x7c,0xc0,0x05,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+scratch_load_b128 v[1:4], off, s0 offset:-64
+// GFX12: encoding: [0x00,0xc0,0x05,0xed,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+scratch_load_b128 v[1:4], off, s0 offset:64
+// GFX12: encoding: [0x00,0xc0,0x05,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+scratch_load_b128 v[1:4], v0, off offset:-64
+// GFX12: encoding: [0x7c,0xc0,0x05,0xed,0x01,0x00,0x02,0x00,0x00,0xc0,0xff,0xff]
+
+scratch_load_b128 v[1:4], v0, off offset:64
+// GFX12: encoding: [0x7c,0xc0,0x05,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00]
+
+scratch_load_b128 v[1:4], v0, s0 offset:-64
+// GFX12: encoding: [0x00,0xc0,0x05,0xed,0x01,0x00,0x02,0x00,0x00,0xc0,0xff,0xff]
+
+scratch_load_b128 v[1:4], v0, s0 offset:64
+// GFX12: encoding: [0x00,0xc0,0x05,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00]
+
+scratch_load_b128 v[1:4], v2, s1
+// GFX12: encoding: [0x01,0xc0,0x05,0xed,0x01,0x00,0x02,0x00,0x02,0x00,0x00,0x00]
+
+scratch_load_b32 v1, off, off offset:2047
+// GFX12: encoding: [0x7c,0x00,0x05,0xed,0x01,0x00,0x00,0x00,0x00,0xff,0x07,0x00]
+
+scratch_load_b32 v1, off, off offset:-64
+// GFX12: encoding: [0x7c,0x00,0x05,0xed,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+scratch_load_b32 v1, off, off offset:64
+// GFX12: encoding: [0x7c,0x00,0x05,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+scratch_load_b32 v1, off, off
+// GFX12: encoding: [0x7c,0x00,0x05,0xed,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00]
+
+scratch_load_b32 v1, off, s0 offset:-64
+// GFX12: encoding: [0x00,0x00,0x05,0xed,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+scratch_load_b32 v1, off, s0 offset:64
+// GFX12: encoding: [0x00,0x00,0x05,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+scratch_load_b32 v1, off, s1 offset:2047
+// GFX12: encoding: [0x01,0x00,0x05,0xed,0x01,0x00,0x00,0x00,0x00,0xff,0x07,0x00]
+
+scratch_load_b32 v1, v0, off offset:-64
+// GFX12: encoding: [0x7c,0x00,0x05,0xed,0x01,0x00,0x02,0x00,0x00,0xc0,0xff,0xff]
+
+scratch_load_b32 v1, v0, off offset:64
+// GFX12: encoding: [0x7c,0x00,0x05,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00]
+
+scratch_load_b32 v1, v0, s0 offset:-64
+// GFX12: encoding: [0x00,0x00,0x05,0xed,0x01,0x00,0x02,0x00,0x00,0xc0,0xff,0xff]
+
+scratch_load_b32 v1, v0, s0 offset:64
+// GFX12: encoding: [0x00,0x00,0x05,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00]
+
+scratch_load_b32 v1, v2, off offset:2047
+// GFX12: encoding: [0x7c,0x00,0x05,0xed,0x01,0x00,0x02,0x00,0x02,0xff,0x07,0x00]
+
+scratch_load_b32 v1, v2, s1 offset:-61440
+// GFX12: encoding: [0x01,0x00,0x05,0xed,0x01,0x00,0x02,0x00,0x02,0x00,0x10,0xff]
+
+scratch_load_b32 v1, v2, s1 offset:61440
+// GFX12: encoding: [0x01,0x00,0x05,0xed,0x01,0x00,0x02,0x00,0x02,0x00,0xf0,0x00]
+
+scratch_load_b32 v1, v2, s1 offset:2047
+// GFX12: encoding: [0x01,0x00,0x05,0xed,0x01,0x00,0x02,0x00,0x02,0xff,0x07,0x00]
+
+scratch_load_b32 v1, v2, s1 offset:-4095
+// GFX12: encoding: [0x01,0x00,0x05,0xed,0x01,0x00,0x02,0x00,0x02,0x01,0xf0,0xff]
+
+scratch_load_b32 v1, v2, s1 offset:4095
+// GFX12: encoding: [0x01,0x00,0x05,0xed,0x01,0x00,0x02,0x00,0x02,0xff,0x0f,0x00]
+
+scratch_load_b32 v1, v2, s1
+// GFX12: encoding: [0x01,0x00,0x05,0xed,0x01,0x00,0x02,0x00,0x02,0x00,0x00,0x00]
+
+scratch_load_b64 v[1:2], off, off offset:-64
+// GFX12: encoding: [0x7c,0x40,0x05,0xed,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+scratch_load_b64 v[1:2], off, off offset:64
+// GFX12: encoding: [0x7c,0x40,0x05,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+scratch_load_b64 v[1:2], off, s0 offset:-64
+// GFX12: encoding: [0x00,0x40,0x05,0xed,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+scratch_load_b64 v[1:2], off, s0 offset:64
+// GFX12: encoding: [0x00,0x40,0x05,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+scratch_load_b64 v[1:2], v0, off offset:-64
+// GFX12: encoding: [0x7c,0x40,0x05,0xed,0x01,0x00,0x02,0x00,0x00,0xc0,0xff,0xff]
+
+scratch_load_b64 v[1:2], v0, off offset:64
+// GFX12: encoding: [0x7c,0x40,0x05,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00]
+
+scratch_load_b64 v[1:2], v0, s0 offset:-64
+// GFX12: encoding: [0x00,0x40,0x05,0xed,0x01,0x00,0x02,0x00,0x00,0xc0,0xff,0xff]
+
+scratch_load_b64 v[1:2], v0, s0 offset:64
+// GFX12: encoding: [0x00,0x40,0x05,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00]
+
+scratch_load_b64 v[1:2], v2, s1
+// GFX12: encoding: [0x01,0x40,0x05,0xed,0x01,0x00,0x02,0x00,0x02,0x00,0x00,0x00]
+
+scratch_load_b96 v[1:3], off, off offset:-64
+// GFX12: encoding: [0x7c,0x80,0x05,0xed,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+scratch_load_b96 v[1:3], off, off offset:64
+// GFX12: encoding: [0x7c,0x80,0x05,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+scratch_load_b96 v[1:3], off, s0 offset:-64
+// GFX12: encoding: [0x00,0x80,0x05,0xed,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+scratch_load_b96 v[1:3], off, s0 offset:64
+// GFX12: encoding: [0x00,0x80,0x05,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+scratch_load_b96 v[1:3], v0, off offset:-64
+// GFX12: encoding: [0x7c,0x80,0x05,0xed,0x01,0x00,0x02,0x00,0x00,0xc0,0xff,0xff]
+
+scratch_load_b96 v[1:3], v0, off offset:64
+// GFX12: encoding: [0x7c,0x80,0x05,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00]
+
+scratch_load_b96 v[1:3], v0, s0 offset:-64
+// GFX12: encoding: [0x00,0x80,0x05,0xed,0x01,0x00,0x02,0x00,0x00,0xc0,0xff,0xff]
+
+scratch_load_b96 v[1:3], v0, s0 offset:64
+// GFX12: encoding: [0x00,0x80,0x05,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00]
+
+scratch_load_b96 v[1:3], v2, s1
+// GFX12: encoding: [0x01,0x80,0x05,0xed,0x01,0x00,0x02,0x00,0x02,0x00,0x00,0x00]
+
+scratch_load_d16_b16 v1, off, off offset:-64
+// GFX12: encoding: [0x7c,0x00,0x08,0xed,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+scratch_load_d16_b16 v1, off, off offset:64
+// GFX12: encoding: [0x7c,0x00,0x08,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+scratch_load_d16_b16 v1, off, s0 offset:-64
+// GFX12: encoding: [0x00,0x00,0x08,0xed,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+scratch_load_d16_b16 v1, off, s0 offset:64
+// GFX12: encoding: [0x00,0x00,0x08,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+scratch_load_d16_b16 v1, v0, off offset:-64
+// GFX12: encoding: [0x7c,0x00,0x08,0xed,0x01,0x00,0x02,0x00,0x00,0xc0,0xff,0xff]
+
+scratch_load_d16_b16 v1, v0, off offset:64
+// GFX12: encoding: [0x7c,0x00,0x08,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00]
+
+scratch_load_d16_b16 v1, v0, s0 offset:-64
+// GFX12: encoding: [0x00,0x00,0x08,0xed,0x01,0x00,0x02,0x00,0x00,0xc0,0xff,0xff]
+
+scratch_load_d16_b16 v1, v0, s0 offset:64
+// GFX12: encoding: [0x00,0x00,0x08,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00]
+
+scratch_load_d16_b16 v1, v2, s1
+// GFX12: encoding: [0x01,0x00,0x08,0xed,0x01,0x00,0x02,0x00,0x02,0x00,0x00,0x00]
+
+scratch_load_d16_hi_b16 v1, off, off offset:-64
+// GFX12: encoding: [0x7c,0xc0,0x08,0xed,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+scratch_load_d16_hi_b16 v1, off, off offset:64
+// GFX12: encoding: [0x7c,0xc0,0x08,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+scratch_load_d16_hi_b16 v1, off, s0 offset:-64
+// GFX12: encoding: [0x00,0xc0,0x08,0xed,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+scratch_load_d16_hi_b16 v1, off, s0 offset:64
+// GFX12: encoding: [0x00,0xc0,0x08,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+scratch_load_d16_hi_b16 v1, v0, off offset:-64
+// GFX12: encoding: [0x7c,0xc0,0x08,0xed,0x01,0x00,0x02,0x00,0x00,0xc0,0xff,0xff]
+
+scratch_load_d16_hi_b16 v1, v0, off offset:64
+// GFX12: encoding: [0x7c,0xc0,0x08,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00]
+
+scratch_load_d16_hi_b16 v1, v0, s0 offset:-64
+// GFX12: encoding: [0x00,0xc0,0x08,0xed,0x01,0x00,0x02,0x00,0x00,0xc0,0xff,0xff]
+
+scratch_load_d16_hi_b16 v1, v0, s0 offset:64
+// GFX12: encoding: [0x00,0xc0,0x08,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00]
+
+scratch_load_d16_hi_b16 v1, v2, s1
+// GFX12: encoding: [0x01,0xc0,0x08,0xed,0x01,0x00,0x02,0x00,0x02,0x00,0x00,0x00]
+
+scratch_load_d16_hi_i8 v1, off, off offset:-64
+// GFX12: encoding: [0x7c,0x80,0x08,0xed,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+scratch_load_d16_hi_i8 v1, off, off offset:64
+// GFX12: encoding: [0x7c,0x80,0x08,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+scratch_load_d16_hi_i8 v1, off, s0 offset:-64
+// GFX12: encoding: [0x00,0x80,0x08,0xed,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+scratch_load_d16_hi_i8 v1, off, s0 offset:64
+// GFX12: encoding: [0x00,0x80,0x08,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+scratch_load_d16_hi_i8 v1, v0, off offset:-64
+// GFX12: encoding: [0x7c,0x80,0x08,0xed,0x01,0x00,0x02,0x00,0x00,0xc0,0xff,0xff]
+
+scratch_load_d16_hi_i8 v1, v0, off offset:64
+// GFX12: encoding: [0x7c,0x80,0x08,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00]
+
+scratch_load_d16_hi_i8 v1, v0, s0 offset:-64
+// GFX12: encoding: [0x00,0x80,0x08,0xed,0x01,0x00,0x02,0x00,0x00,0xc0,0xff,0xff]
+
+scratch_load_d16_hi_i8 v1, v0, s0 offset:64
+// GFX12: encoding: [0x00,0x80,0x08,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00]
+
+scratch_load_d16_hi_i8 v1, v2, s1
+// GFX12: encoding: [0x01,0x80,0x08,0xed,0x01,0x00,0x02,0x00,0x02,0x00,0x00,0x00]
+
+scratch_load_d16_hi_u8 v1, off, off offset:-64
+// GFX12: encoding: [0x7c,0x40,0x08,0xed,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+scratch_load_d16_hi_u8 v1, off, off offset:64
+// GFX12: encoding: [0x7c,0x40,0x08,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+scratch_load_d16_hi_u8 v1, off, s0 offset:-64
+// GFX12: encoding: [0x00,0x40,0x08,0xed,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+scratch_load_d16_hi_u8 v1, off, s0 offset:64
+// GFX12: encoding: [0x00,0x40,0x08,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+scratch_load_d16_hi_u8 v1, v0, off offset:-64
+// GFX12: encoding: [0x7c,0x40,0x08,0xed,0x01,0x00,0x02,0x00,0x00,0xc0,0xff,0xff]
+
+scratch_load_d16_hi_u8 v1, v0, off offset:64
+// GFX12: encoding: [0x7c,0x40,0x08,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00]
+
+scratch_load_d16_hi_u8 v1, v0, s0 offset:-64
+// GFX12: encoding: [0x00,0x40,0x08,0xed,0x01,0x00,0x02,0x00,0x00,0xc0,0xff,0xff]
+
+scratch_load_d16_hi_u8 v1, v0, s0 offset:64
+// GFX12: encoding: [0x00,0x40,0x08,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00]
+
+scratch_load_d16_hi_u8 v1, v2, s1
+// GFX12: encoding: [0x01,0x40,0x08,0xed,0x01,0x00,0x02,0x00,0x02,0x00,0x00,0x00]
+
+scratch_load_d16_i8 v1, off, off offset:-64
+// GFX12: encoding: [0x7c,0xc0,0x07,0xed,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+scratch_load_d16_i8 v1, off, off offset:64
+// GFX12: encoding: [0x7c,0xc0,0x07,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+scratch_load_d16_i8 v1, off, s0 offset:-64
+// GFX12: encoding: [0x00,0xc0,0x07,0xed,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+scratch_load_d16_i8 v1, off, s0 offset:64
+// GFX12: encoding: [0x00,0xc0,0x07,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+scratch_load_d16_i8 v1, v0, off offset:-64
+// GFX12: encoding: [0x7c,0xc0,0x07,0xed,0x01,0x00,0x02,0x00,0x00,0xc0,0xff,0xff]
+
+scratch_load_d16_i8 v1, v0, off offset:64
+// GFX12: encoding: [0x7c,0xc0,0x07,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00]
+
+scratch_load_d16_i8 v1, v0, s0 offset:-64
+// GFX12: encoding: [0x00,0xc0,0x07,0xed,0x01,0x00,0x02,0x00,0x00,0xc0,0xff,0xff]
+
+scratch_load_d16_i8 v1, v0, s0 offset:64
+// GFX12: encoding: [0x00,0xc0,0x07,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00]
+
+scratch_load_d16_i8 v1, v2, s1
+// GFX12: encoding: [0x01,0xc0,0x07,0xed,0x01,0x00,0x02,0x00,0x02,0x00,0x00,0x00]
+
+scratch_load_d16_u8 v1, off, off offset:-64
+// GFX12: encoding: [0x7c,0x80,0x07,0xed,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+scratch_load_d16_u8 v1, off, off offset:64
+// GFX12: encoding: [0x7c,0x80,0x07,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+scratch_load_d16_u8 v1, off, s0 offset:-64
+// GFX12: encoding: [0x00,0x80,0x07,0xed,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+scratch_load_d16_u8 v1, off, s0 offset:64
+// GFX12: encoding: [0x00,0x80,0x07,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+scratch_load_d16_u8 v1, v0, off offset:-64
+// GFX12: encoding: [0x7c,0x80,0x07,0xed,0x01,0x00,0x02,0x00,0x00,0xc0,0xff,0xff]
+
+scratch_load_d16_u8 v1, v0, off offset:64
+// GFX12: encoding: [0x7c,0x80,0x07,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00]
+
+scratch_load_d16_u8 v1, v0, s0 offset:-64
+// GFX12: encoding: [0x00,0x80,0x07,0xed,0x01,0x00,0x02,0x00,0x00,0xc0,0xff,0xff]
+
+scratch_load_d16_u8 v1, v0, s0 offset:64
+// GFX12: encoding: [0x00,0x80,0x07,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00]
+
+scratch_load_d16_u8 v1, v2, s1
+// GFX12: encoding: [0x01,0x80,0x07,0xed,0x01,0x00,0x02,0x00,0x02,0x00,0x00,0x00]
+
+scratch_load_i16 v1, off, off offset:-64
+// GFX12: encoding: [0x7c,0xc0,0x04,0xed,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+scratch_load_i16 v1, off, off offset:64
+// GFX12: encoding: [0x7c,0xc0,0x04,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+scratch_load_i16 v1, off, s0 offset:-64
+// GFX12: encoding: [0x00,0xc0,0x04,0xed,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+scratch_load_i16 v1, off, s0 offset:64
+// GFX12: encoding: [0x00,0xc0,0x04,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+scratch_load_i16 v1, v0, off offset:-64
+// GFX12: encoding: [0x7c,0xc0,0x04,0xed,0x01,0x00,0x02,0x00,0x00,0xc0,0xff,0xff]
+
+scratch_load_i16 v1, v0, off offset:64
+// GFX12: encoding: [0x7c,0xc0,0x04,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00]
+
+scratch_load_i16 v1, v0, s0 offset:-64
+// GFX12: encoding: [0x00,0xc0,0x04,0xed,0x01,0x00,0x02,0x00,0x00,0xc0,0xff,0xff]
+
+scratch_load_i16 v1, v0, s0 offset:64
+// GFX12: encoding: [0x00,0xc0,0x04,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00]
+
+scratch_load_i16 v1, v2, s1
+// GFX12: encoding: [0x01,0xc0,0x04,0xed,0x01,0x00,0x02,0x00,0x02,0x00,0x00,0x00]
+
+scratch_load_i8 v1, off, off offset:-64
+// GFX12: encoding: [0x7c,0x40,0x04,0xed,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+scratch_load_i8 v1, off, off offset:64
+// GFX12: encoding: [0x7c,0x40,0x04,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+scratch_load_i8 v1, off, s0 offset:-64
+// GFX12: encoding: [0x00,0x40,0x04,0xed,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+scratch_load_i8 v1, off, s0 offset:64
+// GFX12: encoding: [0x00,0x40,0x04,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+scratch_load_i8 v1, v0, off offset:-64
+// GFX12: encoding: [0x7c,0x40,0x04,0xed,0x01,0x00,0x02,0x00,0x00,0xc0,0xff,0xff]
+
+scratch_load_i8 v1, v0, off offset:64
+// GFX12: encoding: [0x7c,0x40,0x04,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00]
+
+scratch_load_i8 v1, v0, s0 offset:-64
+// GFX12: encoding: [0x00,0x40,0x04,0xed,0x01,0x00,0x02,0x00,0x00,0xc0,0xff,0xff]
+
+scratch_load_i8 v1, v0, s0 offset:64
+// GFX12: encoding: [0x00,0x40,0x04,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00]
+
+scratch_load_i8 v1, v2, s1
+// GFX12: encoding: [0x01,0x40,0x04,0xed,0x01,0x00,0x02,0x00,0x02,0x00,0x00,0x00]
+
+scratch_load_u16 v1, off, off offset:-64
+// GFX12: encoding: [0x7c,0x80,0x04,0xed,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+scratch_load_u16 v1, off, off offset:64
+// GFX12: encoding: [0x7c,0x80,0x04,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+scratch_load_u16 v1, off, s0 offset:-64
+// GFX12: encoding: [0x00,0x80,0x04,0xed,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+scratch_load_u16 v1, off, s0 offset:64
+// GFX12: encoding: [0x00,0x80,0x04,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+scratch_load_u16 v1, v0, off offset:-64
+// GFX12: encoding: [0x7c,0x80,0x04,0xed,0x01,0x00,0x02,0x00,0x00,0xc0,0xff,0xff]
+
+scratch_load_u16 v1, v0, off offset:64
+// GFX12: encoding: [0x7c,0x80,0x04,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00]
+
+scratch_load_u16 v1, v0, s0 offset:-64
+// GFX12: encoding: [0x00,0x80,0x04,0xed,0x01,0x00,0x02,0x00,0x00,0xc0,0xff,0xff]
+
+scratch_load_u16 v1, v0, s0 offset:64
+// GFX12: encoding: [0x00,0x80,0x04,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00]
+
+scratch_load_u16 v1, v2, s1
+// GFX12: encoding: [0x01,0x80,0x04,0xed,0x01,0x00,0x02,0x00,0x02,0x00,0x00,0x00]
+
+scratch_load_u8 v1, off, off offset:-64
+// GFX12: encoding: [0x7c,0x00,0x04,0xed,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+scratch_load_u8 v1, off, off offset:64
+// GFX12: encoding: [0x7c,0x00,0x04,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+scratch_load_u8 v1, off, s0 offset:-64
+// GFX12: encoding: [0x00,0x00,0x04,0xed,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+
+scratch_load_u8 v1, off, s0 offset:64
+// GFX12: encoding: [0x00,0x00,0x04,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+
+scratch_load_u8 v1, v0, off offset:-64
+// GFX12: encoding: [0x7c,0x00,0x04,0xed,0x01,0x00,0x02,0x00,0x00,0xc0,0xff,0xff]
+
+scratch_load_u8 v1, v0, off offset:64
+// GFX12: encoding: [0x7c,0x00,0x04,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00]
+
+scratch_load_u8 v1, v0, s0 offset:-64
+// GFX12: encoding: [0x00,0x00,0x04,0xed,0x01,0x00,0x02,0x00,0x00,0xc0,0xff,0xff]
+
+scratch_load_u8 v1, v0, s0 offset:64
+// GFX12: encoding: [0x00,0x00,0x04,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00]
+
+scratch_load_u8 v1, v2, s1
+// GFX12: encoding: [0x01,0x00,0x04,0xed,0x01,0x00,0x02,0x00,0x02,0x00,0x00,0x00]
+
+scratch_store_b128 off, v[2:5], off offset:-64
+// GFX12: encoding: [0x7c,0x40,0x07,0xed,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+scratch_store_b128 off, v[2:5], off offset:64
+// GFX12: encoding: [0x7c,0x40,0x07,0xed,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+scratch_store_b128 off, v[2:5], s0 offset:-64
+// GFX12: encoding: [0x00,0x40,0x07,0xed,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+scratch_store_b128 off, v[2:5], s0 offset:64
+// GFX12: encoding: [0x00,0x40,0x07,0xed,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+scratch_store_b128 v0, v[2:5], off offset:-64
+// GFX12: encoding: [0x7c,0x40,0x07,0xed,0x00,0x00,0x02,0x01,0x00,0xc0,0xff,0xff]
+
+scratch_store_b128 v0, v[2:5], off offset:64
+// GFX12: encoding: [0x7c,0x40,0x07,0xed,0x00,0x00,0x02,0x01,0x00,0x40,0x00,0x00]
+
+scratch_store_b128 v0, v[2:5], s0 offset:-64
+// GFX12: encoding: [0x00,0x40,0x07,0xed,0x00,0x00,0x02,0x01,0x00,0xc0,0xff,0xff]
+
+scratch_store_b128 v0, v[2:5], s0 offset:64
+// GFX12: encoding: [0x00,0x40,0x07,0xed,0x00,0x00,0x02,0x01,0x00,0x40,0x00,0x00]
+
+scratch_store_b128 v1, v[2:5], s3
+// GFX12: encoding: [0x03,0x40,0x07,0xed,0x00,0x00,0x02,0x01,0x01,0x00,0x00,0x00]
+
+scratch_store_b16 off, v2, off offset:-64
+// GFX12: encoding: [0x7c,0x40,0x06,0xed,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+scratch_store_b16 off, v2, off offset:64
+// GFX12: encoding: [0x7c,0x40,0x06,0xed,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+scratch_store_b16 off, v2, s0 offset:-64
+// GFX12: encoding: [0x00,0x40,0x06,0xed,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+scratch_store_b16 off, v2, s0 offset:64
+// GFX12: encoding: [0x00,0x40,0x06,0xed,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+scratch_store_b16 v0, v2, off offset:-64
+// GFX12: encoding: [0x7c,0x40,0x06,0xed,0x00,0x00,0x02,0x01,0x00,0xc0,0xff,0xff]
+
+scratch_store_b16 v0, v2, off offset:64
+// GFX12: encoding: [0x7c,0x40,0x06,0xed,0x00,0x00,0x02,0x01,0x00,0x40,0x00,0x00]
+
+scratch_store_b16 v0, v2, s0 offset:-64
+// GFX12: encoding: [0x00,0x40,0x06,0xed,0x00,0x00,0x02,0x01,0x00,0xc0,0xff,0xff]
+
+scratch_store_b16 v0, v2, s0 offset:64
+// GFX12: encoding: [0x00,0x40,0x06,0xed,0x00,0x00,0x02,0x01,0x00,0x40,0x00,0x00]
+
+scratch_store_b16 v1, v2, s3
+// GFX12: encoding: [0x03,0x40,0x06,0xed,0x00,0x00,0x02,0x01,0x01,0x00,0x00,0x00]
+
+scratch_store_b32 off, v2, off offset:2047
+// GFX12: encoding: [0x7c,0x80,0x06,0xed,0x00,0x00,0x00,0x01,0x00,0xff,0x07,0x00]
+
+scratch_store_b32 off, v2, off offset:-64
+// GFX12: encoding: [0x7c,0x80,0x06,0xed,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+scratch_store_b32 off, v2, off offset:64
+// GFX12: encoding: [0x7c,0x80,0x06,0xed,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+scratch_store_b32 off, v2, off
+// GFX12: encoding: [0x7c,0x80,0x06,0xed,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x00]
+
+scratch_store_b32 off, v2, s0 offset:-64
+// GFX12: encoding: [0x00,0x80,0x06,0xed,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+scratch_store_b32 off, v2, s0 offset:64
+// GFX12: encoding: [0x00,0x80,0x06,0xed,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+scratch_store_b32 off, v2, s3 offset:2047
+// GFX12: encoding: [0x03,0x80,0x06,0xed,0x00,0x00,0x00,0x01,0x00,0xff,0x07,0x00]
+
+scratch_store_b32 v0, v2, off offset:-64
+// GFX12: encoding: [0x7c,0x80,0x06,0xed,0x00,0x00,0x02,0x01,0x00,0xc0,0xff,0xff]
+
+scratch_store_b32 v0, v2, off offset:64
+// GFX12: encoding: [0x7c,0x80,0x06,0xed,0x00,0x00,0x02,0x01,0x00,0x40,0x00,0x00]
+
+scratch_store_b32 v0, v2, s0 offset:-64
+// GFX12: encoding: [0x00,0x80,0x06,0xed,0x00,0x00,0x02,0x01,0x00,0xc0,0xff,0xff]
+
+scratch_store_b32 v0, v2, s0 offset:64
+// GFX12: encoding: [0x00,0x80,0x06,0xed,0x00,0x00,0x02,0x01,0x00,0x40,0x00,0x00]
+
+scratch_store_b32 v1, v2, off offset:2047
+// GFX12: encoding: [0x7c,0x80,0x06,0xed,0x00,0x00,0x02,0x01,0x01,0xff,0x07,0x00]
+
+scratch_store_b32 v1, v2, s1 offset:-61440
+// GFX12: encoding: [0x01,0x80,0x06,0xed,0x00,0x00,0x02,0x01,0x01,0x00,0x10,0xff]
+
+scratch_store_b32 v1, v2, s1 offset:61440
+// GFX12: encoding: [0x01,0x80,0x06,0xed,0x00,0x00,0x02,0x01,0x01,0x00,0xf0,0x00]
+
+scratch_store_b32 v1, v2, s1 offset:-4095
+// GFX12: encoding: [0x01,0x80,0x06,0xed,0x00,0x00,0x02,0x01,0x01,0x01,0xf0,0xff]
+
+scratch_store_b32 v1, v2, s1 offset:4095
+// GFX12: encoding: [0x01,0x80,0x06,0xed,0x00,0x00,0x02,0x01,0x01,0xff,0x0f,0x00]
+
+scratch_store_b32 v1, v2, s3 offset:2047
+// GFX12: encoding: [0x03,0x80,0x06,0xed,0x00,0x00,0x02,0x01,0x01,0xff,0x07,0x00]
+
+scratch_store_b32 v1, v2, s3
+// GFX12: encoding: [0x03,0x80,0x06,0xed,0x00,0x00,0x02,0x01,0x01,0x00,0x00,0x00]
+
+scratch_store_b64 off, v[2:3], off offset:-64
+// GFX12: encoding: [0x7c,0xc0,0x06,0xed,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+scratch_store_b64 off, v[2:3], off offset:64
+// GFX12: encoding: [0x7c,0xc0,0x06,0xed,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+scratch_store_b64 off, v[2:3], s0 offset:-64
+// GFX12: encoding: [0x00,0xc0,0x06,0xed,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+scratch_store_b64 off, v[2:3], s0 offset:64
+// GFX12: encoding: [0x00,0xc0,0x06,0xed,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+scratch_store_b64 v0, v[2:3], off offset:-64
+// GFX12: encoding: [0x7c,0xc0,0x06,0xed,0x00,0x00,0x02,0x01,0x00,0xc0,0xff,0xff]
+
+scratch_store_b64 v0, v[2:3], off offset:64
+// GFX12: encoding: [0x7c,0xc0,0x06,0xed,0x00,0x00,0x02,0x01,0x00,0x40,0x00,0x00]
+
+scratch_store_b64 v0, v[2:3], s0 offset:-64
+// GFX12: encoding: [0x00,0xc0,0x06,0xed,0x00,0x00,0x02,0x01,0x00,0xc0,0xff,0xff]
+
+scratch_store_b64 v0, v[2:3], s0 offset:64
+// GFX12: encoding: [0x00,0xc0,0x06,0xed,0x00,0x00,0x02,0x01,0x00,0x40,0x00,0x00]
+
+scratch_store_b64 v1, v[2:3], s3
+// GFX12: encoding: [0x03,0xc0,0x06,0xed,0x00,0x00,0x02,0x01,0x01,0x00,0x00,0x00]
+
+scratch_store_b8 off, v2, off offset:-64
+// GFX12: encoding: [0x7c,0x00,0x06,0xed,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+scratch_store_b8 off, v2, off offset:64
+// GFX12: encoding: [0x7c,0x00,0x06,0xed,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+scratch_store_b8 off, v2, s0 offset:-64
+// GFX12: encoding: [0x00,0x00,0x06,0xed,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+scratch_store_b8 off, v2, s0 offset:64
+// GFX12: encoding: [0x00,0x00,0x06,0xed,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+scratch_store_b8 v0, v2, off offset:-64
+// GFX12: encoding: [0x7c,0x00,0x06,0xed,0x00,0x00,0x02,0x01,0x00,0xc0,0xff,0xff]
+
+scratch_store_b8 v0, v2, off offset:64
+// GFX12: encoding: [0x7c,0x00,0x06,0xed,0x00,0x00,0x02,0x01,0x00,0x40,0x00,0x00]
+
+scratch_store_b8 v0, v2, s0 offset:-64
+// GFX12: encoding: [0x00,0x00,0x06,0xed,0x00,0x00,0x02,0x01,0x00,0xc0,0xff,0xff]
+
+scratch_store_b8 v0, v2, s0 offset:64
+// GFX12: encoding: [0x00,0x00,0x06,0xed,0x00,0x00,0x02,0x01,0x00,0x40,0x00,0x00]
+
+scratch_store_b8 v1, v2, s3
+// GFX12: encoding: [0x03,0x00,0x06,0xed,0x00,0x00,0x02,0x01,0x01,0x00,0x00,0x00]
+
+scratch_store_b96 off, v[2:4], off offset:-64
+// GFX12: encoding: [0x7c,0x00,0x07,0xed,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+scratch_store_b96 off, v[2:4], off offset:64
+// GFX12: encoding: [0x7c,0x00,0x07,0xed,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+scratch_store_b96 off, v[2:4], s0 offset:-64
+// GFX12: encoding: [0x00,0x00,0x07,0xed,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+scratch_store_b96 off, v[2:4], s0 offset:64
+// GFX12: encoding: [0x00,0x00,0x07,0xed,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+scratch_store_b96 v0, v[2:4], off offset:-64
+// GFX12: encoding: [0x7c,0x00,0x07,0xed,0x00,0x00,0x02,0x01,0x00,0xc0,0xff,0xff]
+
+scratch_store_b96 v0, v[2:4], off offset:64
+// GFX12: encoding: [0x7c,0x00,0x07,0xed,0x00,0x00,0x02,0x01,0x00,0x40,0x00,0x00]
+
+scratch_store_b96 v0, v[2:4], s0 offset:-64
+// GFX12: encoding: [0x00,0x00,0x07,0xed,0x00,0x00,0x02,0x01,0x00,0xc0,0xff,0xff]
+
+scratch_store_b96 v0, v[2:4], s0 offset:64
+// GFX12: encoding: [0x00,0x00,0x07,0xed,0x00,0x00,0x02,0x01,0x00,0x40,0x00,0x00]
+
+scratch_store_b96 v1, v[2:4], s3
+// GFX12: encoding: [0x03,0x00,0x07,0xed,0x00,0x00,0x02,0x01,0x01,0x00,0x00,0x00]
+
+scratch_store_d16_hi_b16 off, v2, off offset:-64
+// GFX12: encoding: [0x7c,0x40,0x09,0xed,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+scratch_store_d16_hi_b16 off, v2, off offset:64
+// GFX12: encoding: [0x7c,0x40,0x09,0xed,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+scratch_store_d16_hi_b16 off, v2, s0 offset:-64
+// GFX12: encoding: [0x00,0x40,0x09,0xed,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+scratch_store_d16_hi_b16 off, v2, s0 offset:64
+// GFX12: encoding: [0x00,0x40,0x09,0xed,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+scratch_store_d16_hi_b16 v0, v2, off offset:-64
+// GFX12: encoding: [0x7c,0x40,0x09,0xed,0x00,0x00,0x02,0x01,0x00,0xc0,0xff,0xff]
+
+scratch_store_d16_hi_b16 v0, v2, off offset:64
+// GFX12: encoding: [0x7c,0x40,0x09,0xed,0x00,0x00,0x02,0x01,0x00,0x40,0x00,0x00]
+
+scratch_store_d16_hi_b16 v0, v2, s0 offset:-64
+// GFX12: encoding: [0x00,0x40,0x09,0xed,0x00,0x00,0x02,0x01,0x00,0xc0,0xff,0xff]
+
+scratch_store_d16_hi_b16 v0, v2, s0 offset:64
+// GFX12: encoding: [0x00,0x40,0x09,0xed,0x00,0x00,0x02,0x01,0x00,0x40,0x00,0x00]
+
+scratch_store_d16_hi_b16 v1, v2, s3
+// GFX12: encoding: [0x03,0x40,0x09,0xed,0x00,0x00,0x02,0x01,0x01,0x00,0x00,0x00]
+
+scratch_store_d16_hi_b8 off, v2, off offset:-64
+// GFX12: encoding: [0x7c,0x00,0x09,0xed,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+scratch_store_d16_hi_b8 off, v2, off offset:64
+// GFX12: encoding: [0x7c,0x00,0x09,0xed,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+scratch_store_d16_hi_b8 off, v2, s0 offset:-64
+// GFX12: encoding: [0x00,0x00,0x09,0xed,0x00,0x00,0x00,0x01,0x00,0xc0,0xff,0xff]
+
+scratch_store_d16_hi_b8 off, v2, s0 offset:64
+// GFX12: encoding: [0x00,0x00,0x09,0xed,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+scratch_store_d16_hi_b8 v0, v2, off offset:-64
+// GFX12: encoding: [0x7c,0x00,0x09,0xed,0x00,0x00,0x02,0x01,0x00,0xc0,0xff,0xff]
+
+scratch_store_d16_hi_b8 v0, v2, off offset:64
+// GFX12: encoding: [0x7c,0x00,0x09,0xed,0x00,0x00,0x02,0x01,0x00,0x40,0x00,0x00]
+
+scratch_store_d16_hi_b8 v0, v2, s0 offset:-64
+// GFX12: encoding: [0x00,0x00,0x09,0xed,0x00,0x00,0x02,0x01,0x00,0xc0,0xff,0xff]
+
+scratch_store_d16_hi_b8 v0, v2, s0 offset:64
+// GFX12: encoding: [0x00,0x00,0x09,0xed,0x00,0x00,0x02,0x01,0x00,0x40,0x00,0x00]
+
+scratch_store_d16_hi_b8 v1, v2, s3
+// GFX12: encoding: [0x03,0x00,0x09,0xed,0x00,0x00,0x02,0x01,0x01,0x00,0x00,0x00]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vflat_alias.s b/llvm/test/MC/AMDGPU/gfx12_asm_vflat_alias.s
new file mode 100644
index 0000000000000000000000000000000000000000..3633f8815c00a355e987162bbf00b4c3f815c962
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vflat_alias.s
@@ -0,0 +1,43 @@
+; RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -show-encoding %s | FileCheck -check-prefix=GFX12 %s
+
+global_atomic_csub v1, v0, v2, s[0:1] offset:64 th:TH_ATOMIC_RETURN
+// GFX12: global_atomic_sub_clamp_u32 {{.*}} encoding: [0x00,0xc0,0x0d,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_csub v0, v2, s[0:1] offset:64
+// GFX12: global_atomic_sub_clamp_u32 {{.*}} encoding: [0x00,0xc0,0x0d,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_csub_u32 v1, v0, v2, s[0:1] offset:64 th:TH_ATOMIC_RETURN
+// GFX12: global_atomic_sub_clamp_u32 {{.*}} encoding: [0x00,0xc0,0x0d,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_csub_u32 v0, v2, s[0:1] offset:64
+// GFX12: global_atomic_sub_clamp_u32 {{.*}} encoding: [0x00,0xc0,0x0d,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+flat_atomic_csub_u32 v1, v[0:1], v2 offset:64 th:TH_ATOMIC_RETURN
+// GFX12: flat_atomic_sub_clamp_u32 {{.*}} encoding: [0x7c,0xc0,0x0d,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+
+flat_atomic_csub_u32 v[0:1], v2 offset:64
+// GFX12: flat_atomic_sub_clamp_u32 {{.*}} encoding: [0x7c,0xc0,0x0d,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+flat_atomic_fmax v[0:1], v2 offset:64
+// GFX12: flat_atomic_max_num_f32 {{.*}} encoding: [0x7c,0x80,0x14,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+flat_atomic_max_f32 v[0:1], v2 offset:64
+// GFX12: flat_atomic_max_num_f32 {{.*}} encoding: [0x7c,0x80,0x14,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+flat_atomic_fmin v[0:1], v2 offset:64
+// GFX12: flat_atomic_min_num_f32 {{.*}} encoding: [0x7c,0x40,0x14,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+flat_atomic_min_f32 v[0:1], v2 offset:64
+// GFX12: flat_atomic_min_num_f32 {{.*}} encoding: [0x7c,0x40,0x14,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_fmax v0, v2, s[0:1] offset:64
+// GFX12: global_atomic_max_num_f32 {{.*}} encoding: [0x00,0x80,0x14,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_max_f32 v0, v2, s[0:1] offset:64
+// GFX12: global_atomic_max_num_f32 {{.*}} encoding: [0x00,0x80,0x14,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_fmin v0, v2, s[0:1] offset:64
+// GFX12: global_atomic_min_num_f32 {{.*}} encoding: [0x00,0x40,0x14,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+
+global_atomic_min_f32 v0, v2, s[0:1] offset:64
+// GFX12: global_atomic_min_num_f32 {{.*}} encoding: [0x00,0x40,0x14,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3.s
index 71b2c442460f331468aed56bbd8b0a4dd8b15781..e97e4b8c7241c56d7b478d70ba4e01bb28d4d18e 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vop3.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3.s
@@ -5981,3 +5981,633 @@ v_xor_b16 v5, src_scc, vcc_lo
 
 v_xor_b16 v255, 0xfe0b, vcc_hi
 // GFX12: encoding: [0xff,0x00,0x64,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
+
+v_minimum_f32 v5, v1, v2
+// GFX12: encoding: [0x05,0x00,0x65,0xd7,0x01,0x05,0x02,0x00]
+
+v_minimum_f32 v5, v255, v255
+// GFX12: encoding: [0x05,0x00,0x65,0xd7,0xff,0xff,0x03,0x00]
+
+v_minimum_f32 v5, s1, s2
+// GFX12: encoding: [0x05,0x00,0x65,0xd7,0x01,0x04,0x00,0x00]
+
+v_minimum_f32 v5, s105, s105
+// GFX12: encoding: [0x05,0x00,0x65,0xd7,0x69,0xd2,0x00,0x00]
+
+v_minimum_f32 v5, vcc_lo, ttmp15
+// GFX12: encoding: [0x05,0x00,0x65,0xd7,0x6a,0xf6,0x00,0x00]
+
+v_minimum_f32 v5, vcc_hi, 0xaf123456
+// GFX12: encoding: [0x05,0x00,0x65,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_minimum_f32 v5, ttmp15, src_scc
+// GFX12: encoding: [0x05,0x00,0x65,0xd7,0x7b,0xfa,0x01,0x00]
+
+v_minimum_f32 v5, m0, 0.5
+// GFX12: encoding: [0x05,0x00,0x65,0xd7,0x7d,0xe0,0x01,0x00]
+
+v_minimum_f32 v5, exec_lo, -1
+// GFX12: encoding: [0x05,0x00,0x65,0xd7,0x7e,0x82,0x01,0x00]
+
+v_minimum_f32 v5, |exec_hi|, null
+// GFX12: encoding: [0x05,0x01,0x65,0xd7,0x7f,0xf8,0x00,0x00]
+
+v_minimum_f32 v5, null, exec_lo
+// GFX12: encoding: [0x05,0x00,0x65,0xd7,0x7c,0xfc,0x00,0x00]
+
+v_minimum_f32 v5, -1, exec_hi
+// GFX12: encoding: [0x05,0x00,0x65,0xd7,0xc1,0xfe,0x00,0x00]
+
+v_minimum_f32 v5, 0.5, -m0
+// GFX12: encoding: [0x05,0x00,0x65,0xd7,0xf0,0xfa,0x00,0x40]
+
+v_minimum_f32 v5, -src_scc, |vcc_lo|
+// GFX12: encoding: [0x05,0x02,0x65,0xd7,0xfd,0xd4,0x00,0x20]
+
+v_minimum_f32 v255, -|0xaf123456|, -|vcc_hi|
+// GFX12: encoding: [0xff,0x03,0x65,0xd7,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_maximum_f32 v5, v1, v2
+// GFX12: encoding: [0x05,0x00,0x66,0xd7,0x01,0x05,0x02,0x00]
+
+v_maximum_f32 v5, v255, v255
+// GFX12: encoding: [0x05,0x00,0x66,0xd7,0xff,0xff,0x03,0x00]
+
+v_maximum_f32 v5, s1, s2
+// GFX12: encoding: [0x05,0x00,0x66,0xd7,0x01,0x04,0x00,0x00]
+
+v_maximum_f32 v5, s105, s105
+// GFX12: encoding: [0x05,0x00,0x66,0xd7,0x69,0xd2,0x00,0x00]
+
+v_maximum_f32 v5, vcc_lo, ttmp15
+// GFX12: encoding: [0x05,0x00,0x66,0xd7,0x6a,0xf6,0x00,0x00]
+
+v_maximum_f32 v5, vcc_hi, 0xaf123456
+// GFX12: encoding: [0x05,0x00,0x66,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+
+v_maximum_f32 v5, ttmp15, src_scc
+// GFX12: encoding: [0x05,0x00,0x66,0xd7,0x7b,0xfa,0x01,0x00]
+
+v_maximum_f32 v5, m0, 0.5
+// GFX12: encoding: [0x05,0x00,0x66,0xd7,0x7d,0xe0,0x01,0x00]
+
+v_maximum_f32 v5, exec_lo, -1
+// GFX12: encoding: [0x05,0x00,0x66,0xd7,0x7e,0x82,0x01,0x00]
+
+v_maximum_f32 v5, |exec_hi|, null
+// GFX12: encoding: [0x05,0x01,0x66,0xd7,0x7f,0xf8,0x00,0x00]
+
+v_maximum_f32 v5, null, exec_lo
+// GFX12: encoding: [0x05,0x00,0x66,0xd7,0x7c,0xfc,0x00,0x00]
+
+v_maximum_f32 v5, -1, exec_hi
+// GFX12: encoding: [0x05,0x00,0x66,0xd7,0xc1,0xfe,0x00,0x00]
+
+v_maximum_f32 v5, 0.5, -m0
+// GFX12: encoding: [0x05,0x00,0x66,0xd7,0xf0,0xfa,0x00,0x40]
+
+v_maximum_f32 v5, -src_scc, |vcc_lo|
+// GFX12: encoding: [0x05,0x02,0x66,0xd7,0xfd,0xd4,0x00,0x20]
+
+v_maximum_f32 v255, -|0xaf123456|, -|vcc_hi|
+// GFX12: encoding: [0xff,0x03,0x66,0xd7,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+
+v_minimum_f16 v5, v1, v2
+// GFX12: encoding: [0x05,0x00,0x67,0xd7,0x01,0x05,0x02,0x00]
+
+v_minimum_f16 v5, v255, v255
+// GFX12: encoding: [0x05,0x00,0x67,0xd7,0xff,0xff,0x03,0x00]
+
+v_minimum_f16 v5, s1, s2
+// GFX12: encoding: [0x05,0x00,0x67,0xd7,0x01,0x04,0x00,0x00]
+
+v_minimum_f16 v5, s105, s105
+// GFX12: encoding: [0x05,0x00,0x67,0xd7,0x69,0xd2,0x00,0x00]
+
+v_minimum_f16 v5, vcc_lo, ttmp15
+// GFX12: encoding: [0x05,0x00,0x67,0xd7,0x6a,0xf6,0x00,0x00]
+
+v_minimum_f16 v5, vcc_hi, 0xaf12
+// GFX12: encoding: [0x05,0x00,0x67,0xd7,0x6b,0xfe,0x01,0x00,0x12,0xaf,0x00,0x00]
+
+v_minimum_f16 v5, ttmp15, src_scc
+// GFX12: encoding: [0x05,0x00,0x67,0xd7,0x7b,0xfa,0x01,0x00]
+
+v_minimum_f16 v5, m0, 0.5
+// GFX12: encoding: [0x05,0x00,0x67,0xd7,0x7d,0xe0,0x01,0x00]
+
+v_minimum_f16 v5, exec_lo, -1
+// GFX12: encoding: [0x05,0x00,0x67,0xd7,0x7e,0x82,0x01,0x00]
+
+v_minimum_f16 v5, |exec_hi|, null
+// GFX12: encoding: [0x05,0x01,0x67,0xd7,0x7f,0xf8,0x00,0x00]
+
+v_minimum_f16 v5, null, exec_lo
+// GFX12: encoding: [0x05,0x00,0x67,0xd7,0x7c,0xfc,0x00,0x00]
+
+v_minimum_f16 v5, -1, exec_hi
+// GFX12: encoding: [0x05,0x00,0x67,0xd7,0xc1,0xfe,0x00,0x00]
+
+v_minimum_f16 v5, 0.5, -m0
+// GFX12: encoding: [0x05,0x00,0x67,0xd7,0xf0,0xfa,0x00,0x40]
+
+v_minimum_f16 v5, -src_scc, |vcc_lo|
+// GFX12: encoding: [0x05,0x02,0x67,0xd7,0xfd,0xd4,0x00,0x20]
+
+v_minimum_f16 v255, -|0xaf12|, -|vcc_hi|
+// GFX12: encoding: [0xff,0x03,0x67,0xd7,0xff,0xd6,0x00,0x60,0x12,0xaf,0x00,0x00]
+
+v_minimum_f16 v205, v201, v200
+// GFX12: encoding: [0xcd,0x00,0x67,0xd7,0xc9,0x91,0x03,0x00]
+
+v_maximum_f16 v5, v1, v2
+// GFX12: encoding: [0x05,0x00,0x68,0xd7,0x01,0x05,0x02,0x00]
+
+v_maximum_f16 v5, v255, v255
+// GFX12: encoding: [0x05,0x00,0x68,0xd7,0xff,0xff,0x03,0x00]
+
+v_maximum_f16 v5, s1, s2
+// GFX12: encoding: [0x05,0x00,0x68,0xd7,0x01,0x04,0x00,0x00]
+
+v_maximum_f16 v5, s105, s105
+// GFX12: encoding: [0x05,0x00,0x68,0xd7,0x69,0xd2,0x00,0x00]
+
+v_maximum_f16 v5, vcc_lo, ttmp15
+// GFX12: encoding: [0x05,0x00,0x68,0xd7,0x6a,0xf6,0x00,0x00]
+
+v_maximum_f16 v5, vcc_hi, 0xaf12
+// GFX12: encoding: [0x05,0x00,0x68,0xd7,0x6b,0xfe,0x01,0x00,0x12,0xaf,0x00,0x00]
+
+v_maximum_f16 v5, ttmp15, src_scc
+// GFX12: encoding: [0x05,0x00,0x68,0xd7,0x7b,0xfa,0x01,0x00]
+
+v_maximum_f16 v5, m0, 0.5
+// GFX12: encoding: [0x05,0x00,0x68,0xd7,0x7d,0xe0,0x01,0x00]
+
+v_maximum_f16 v5, exec_lo, -1
+// GFX12: encoding: [0x05,0x00,0x68,0xd7,0x7e,0x82,0x01,0x00]
+
+v_maximum_f16 v5, |exec_hi|, null
+// GFX12: encoding: [0x05,0x01,0x68,0xd7,0x7f,0xf8,0x00,0x00]
+
+v_maximum_f16 v5, null, exec_lo
+// GFX12: encoding: [0x05,0x00,0x68,0xd7,0x7c,0xfc,0x00,0x00]
+
+v_maximum_f16 v5, -1, exec_hi
+// GFX12: encoding: [0x05,0x00,0x68,0xd7,0xc1,0xfe,0x00,0x00]
+
+v_maximum_f16 v5, 0.5, -m0
+// GFX12: encoding: [0x05,0x00,0x68,0xd7,0xf0,0xfa,0x00,0x40]
+
+v_maximum_f16 v5, -src_scc, |vcc_lo|
+// GFX12: encoding: [0x05,0x02,0x68,0xd7,0xfd,0xd4,0x00,0x20]
+
+v_maximum_f16 v255, -|0xaf12|, -|vcc_hi|
+// GFX12: encoding: [0xff,0x03,0x68,0xd7,0xff,0xd6,0x00,0x60,0x12,0xaf,0x00,0x00]
+
+v_maximum_f16 v205, v201, v200
+// GFX12: encoding: [0xcd,0x00,0x68,0xd7,0xc9,0x91,0x03,0x00]
+
+v_minimum_f64 v[5:6], v[1:2], v[3:4]
+// GFX12: encoding: [0x05,0x00,0x41,0xd7,0x01,0x07,0x02,0x00]
+
+v_minimum_f64 v[5:6], v[254:255], v[254:255]
+// GFX12: encoding: [0x05,0x00,0x41,0xd7,0xfe,0xfd,0x03,0x00]
+
+v_minimum_f64 v[5:6], s[6:7], s[4:5]
+// GFX12: encoding: [0x05,0x00,0x41,0xd7,0x06,0x08,0x00,0x00]
+
+v_minimum_f64 v[5:6], s[104:105], s[104:105]
+// GFX12: encoding: [0x05,0x00,0x41,0xd7,0x68,0xd0,0x00,0x00]
+
+v_minimum_f64 v[5:6], vcc, ttmp[14:15]
+// GFX12: encoding: [0x05,0x00,0x41,0xd7,0x6a,0xf4,0x00,0x00]
+
+v_minimum_f64 v[5:6], vcc, 0xaf121234
+// GFX12: encoding: [0x05,0x00,0x41,0xd7,0x6a,0xfe,0x01,0x00,0x34,0x12,0x12,0xaf]
+
+v_minimum_f64 v[5:6], ttmp[14:15], src_scc
+// GFX12: encoding: [0x05,0x00,0x41,0xd7,0x7a,0xfa,0x01,0x00]
+
+v_minimum_f64 v[5:6], vcc, 0.5
+// GFX12: encoding: [0x05,0x00,0x41,0xd7,0x6a,0xe0,0x01,0x00]
+
+v_minimum_f64 v[5:6], exec, -1
+// GFX12: encoding: [0x05,0x00,0x41,0xd7,0x7e,0x82,0x01,0x00]
+
+v_minimum_f64 v[5:6], |exec|, null
+// GFX12: encoding: [0x05,0x01,0x41,0xd7,0x7e,0xf8,0x00,0x00]
+
+v_minimum_f64 v[5:6], null, exec
+// GFX12: encoding: [0x05,0x00,0x41,0xd7,0x7c,0xfc,0x00,0x00]
+
+v_minimum_f64 v[5:6], -1, exec
+// GFX12: encoding: [0x05,0x00,0x41,0xd7,0xc1,0xfc,0x00,0x00]
+
+v_minimum_f64 v[5:6], 0.5, -vcc
+// GFX12: encoding: [0x05,0x00,0x41,0xd7,0xf0,0xd4,0x00,0x40]
+
+v_minimum_f64 v[5:6], -src_scc, |vcc|
+// GFX12: encoding: [0x05,0x02,0x41,0xd7,0xfd,0xd4,0x00,0x20]
+
+v_minimum_f64 v[254:255], -|2|, -|vcc|
+// GFX12: encoding: [0xfe,0x03,0x41,0xd7,0x82,0xd4,0x00,0x60]
+
+v_maximum_f64 v[5:6], v[1:2], v[3:4]
+// GFX12: encoding: [0x05,0x00,0x42,0xd7,0x01,0x07,0x02,0x00]
+
+v_maximum_f64 v[5:6], v[254:255], v[254:255]
+// GFX12: encoding: [0x05,0x00,0x42,0xd7,0xfe,0xfd,0x03,0x00]
+
+v_maximum_f64 v[5:6], s[6:7], s[4:5]
+// GFX12: encoding: [0x05,0x00,0x42,0xd7,0x06,0x08,0x00,0x00]
+
+v_maximum_f64 v[5:6], s[104:105], s[104:105]
+// GFX12: encoding: [0x05,0x00,0x42,0xd7,0x68,0xd0,0x00,0x00]
+
+v_maximum_f64 v[5:6], vcc, ttmp[14:15]
+// GFX12: encoding: [0x05,0x00,0x42,0xd7,0x6a,0xf4,0x00,0x00]
+
+v_maximum_f64 v[5:6], vcc, 0xaf121234
+// GFX12: encoding: [0x05,0x00,0x42,0xd7,0x6a,0xfe,0x01,0x00,0x34,0x12,0x12,0xaf]
+
+v_maximum_f64 v[5:6], ttmp[14:15], src_scc
+// GFX12: encoding: [0x05,0x00,0x42,0xd7,0x7a,0xfa,0x01,0x00]
+
+v_maximum_f64 v[5:6], vcc, 0.5
+// GFX12: encoding: [0x05,0x00,0x42,0xd7,0x6a,0xe0,0x01,0x00]
+
+v_maximum_f64 v[5:6], exec, -1
+// GFX12: encoding: [0x05,0x00,0x42,0xd7,0x7e,0x82,0x01,0x00]
+
+v_maximum_f64 v[5:6], |exec|, null
+// GFX12: encoding: [0x05,0x01,0x42,0xd7,0x7e,0xf8,0x00,0x00]
+
+v_maximum_f64 v[5:6], null, exec
+// GFX12: encoding: [0x05,0x00,0x42,0xd7,0x7c,0xfc,0x00,0x00]
+
+v_maximum_f64 v[5:6], -1, exec
+// GFX12: encoding: [0x05,0x00,0x42,0xd7,0xc1,0xfc,0x00,0x00]
+
+v_maximum_f64 v[5:6], 0.5, -vcc
+// GFX12: encoding: [0x05,0x00,0x42,0xd7,0xf0,0xd4,0x00,0x40]
+
+v_maximum_f64 v[5:6], -src_scc, |vcc|
+// GFX12: encoding: [0x05,0x02,0x42,0xd7,0xfd,0xd4,0x00,0x20]
+
+v_maximum_f64 v[254:255], -|2|, -|vcc|
+// GFX12: encoding: [0xfe,0x03,0x42,0xd7,0x82,0xd4,0x00,0x60]
+
+v_minimum3_f32 v5, v1, v2, s3
+// GFX12: encoding: [0x05,0x00,0x2d,0xd6,0x01,0x05,0x0e,0x00]
+
+v_minimum3_f32 v5, v255, s2, s105
+// GFX12: encoding: [0x05,0x00,0x2d,0xd6,0xff,0x05,0xa4,0x01]
+
+v_minimum3_f32 v5, s1, v255, exec_hi
+// GFX12: encoding: [0x05,0x00,0x2d,0xd6,0x01,0xfe,0xff,0x01]
+
+v_minimum3_f32 v5, s105, s105, exec_lo
+// GFX12: encoding: [0x05,0x00,0x2d,0xd6,0x69,0xd2,0xf8,0x01]
+
+v_minimum3_f32 v5, vcc_lo, ttmp15, v3
+// GFX12: encoding: [0x05,0x00,0x2d,0xd6,0x6a,0xf6,0x0c,0x04]
+
+v_minimum3_f32 v5, vcc_hi, 0xaf123456, v255
+// GFX12: encoding: [0x05,0x00,0x2d,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
+
+v_minimum3_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15|
+// GFX12: encoding: [0x05,0x07,0x2d,0xd6,0x7b,0xfa,0xed,0xe1]
+
+v_minimum3_f32 v5, m0, 0.5, m0
+// GFX12: encoding: [0x05,0x00,0x2d,0xd6,0x7d,0xe0,0xf5,0x01]
+
+v_minimum3_f32 v5, |exec_lo|, -1, vcc_hi
+// GFX12: encoding: [0x05,0x01,0x2d,0xd6,0x7e,0x82,0xad,0x01]
+
+v_minimum3_f32 v5, -|exec_hi|, null, -|vcc_lo|
+// GFX12: encoding: [0x05,0x05,0x2d,0xd6,0x7f,0xf8,0xa8,0xa1]
+
+v_minimum3_f32 v5, null, exec_lo, -|0xaf123456|
+// GFX12: encoding: [0x05,0x04,0x2d,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
+
+v_minimum3_f32 v5, -1, -|exec_hi|, -|src_scc|
+// GFX12: encoding: [0x05,0x06,0x2d,0xd6,0xc1,0xfe,0xf4,0xc3]
+
+v_minimum3_f32 v5, 0.5, -m0, 0.5 mul:2
+// GFX12: encoding: [0x05,0x00,0x2d,0xd6,0xf0,0xfa,0xc0,0x4b]
+
+v_minimum3_f32 v5, -src_scc, |vcc_lo|, -1 mul:4
+// GFX12: encoding: [0x05,0x02,0x2d,0xd6,0xfd,0xd4,0x04,0x33]
+
+v_minimum3_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2
+// GFX12: encoding: [0xff,0x83,0x2d,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
+
+v_maximum3_f32 v5, v1, v2, s3
+// GFX12: encoding: [0x05,0x00,0x2e,0xd6,0x01,0x05,0x0e,0x00]
+
+v_maximum3_f32 v5, v255, s2, s105
+// GFX12: encoding: [0x05,0x00,0x2e,0xd6,0xff,0x05,0xa4,0x01]
+
+v_maximum3_f32 v5, s1, v255, exec_hi
+// GFX12: encoding: [0x05,0x00,0x2e,0xd6,0x01,0xfe,0xff,0x01]
+
+v_maximum3_f32 v5, s105, s105, exec_lo
+// GFX12: encoding: [0x05,0x00,0x2e,0xd6,0x69,0xd2,0xf8,0x01]
+
+v_maximum3_f32 v5, vcc_lo, ttmp15, v3
+// GFX12: encoding: [0x05,0x00,0x2e,0xd6,0x6a,0xf6,0x0c,0x04]
+
+v_maximum3_f32 v5, vcc_hi, 0xaf123456, v255
+// GFX12: encoding: [0x05,0x00,0x2e,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
+
+v_maximum3_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15|
+// GFX12: encoding: [0x05,0x07,0x2e,0xd6,0x7b,0xfa,0xed,0xe1]
+
+v_maximum3_f32 v5, m0, 0.5, m0
+// GFX12: encoding: [0x05,0x00,0x2e,0xd6,0x7d,0xe0,0xf5,0x01]
+
+v_maximum3_f32 v5, |exec_lo|, -1, vcc_hi
+// GFX12: encoding: [0x05,0x01,0x2e,0xd6,0x7e,0x82,0xad,0x01]
+
+v_maximum3_f32 v5, -|exec_hi|, null, -|vcc_lo|
+// GFX12: encoding: [0x05,0x05,0x2e,0xd6,0x7f,0xf8,0xa8,0xa1]
+
+v_maximum3_f32 v5, null, exec_lo, -|0xaf123456|
+// GFX12: encoding: [0x05,0x04,0x2e,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
+
+v_maximum3_f32 v5, -1, -|exec_hi|, -|src_scc|
+// GFX12: encoding: [0x05,0x06,0x2e,0xd6,0xc1,0xfe,0xf4,0xc3]
+
+v_maximum3_f32 v5, 0.5, -m0, 0.5 mul:2
+// GFX12: encoding: [0x05,0x00,0x2e,0xd6,0xf0,0xfa,0xc0,0x4b]
+
+v_maximum3_f32 v5, -src_scc, |vcc_lo|, -1 mul:4
+// GFX12: encoding: [0x05,0x02,0x2e,0xd6,0xfd,0xd4,0x04,0x33]
+
+v_maximum3_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2
+// GFX12: encoding: [0xff,0x83,0x2e,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
+
+v_minimum3_f16 v5, v1, v2, s3
+// GFX12: encoding: [0x05,0x00,0x2f,0xd6,0x01,0x05,0x0e,0x00]
+
+v_minimum3_f16 v5, v255, s2, s105
+// GFX12: encoding: [0x05,0x00,0x2f,0xd6,0xff,0x05,0xa4,0x01]
+
+v_minimum3_f16 v5, s1, v255, exec_hi
+// GFX12: encoding: [0x05,0x00,0x2f,0xd6,0x01,0xfe,0xff,0x01]
+
+v_minimum3_f16 v5, s105, s105, exec_lo
+// GFX12: encoding: [0x05,0x00,0x2f,0xd6,0x69,0xd2,0xf8,0x01]
+
+v_minimum3_f16 v5, vcc_lo, ttmp15, v3
+// GFX12: encoding: [0x05,0x00,0x2f,0xd6,0x6a,0xf6,0x0c,0x04]
+
+v_minimum3_f16 v5, vcc_hi, 0xaf12, v255
+// GFX12: encoding: [0x05,0x00,0x2f,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00]
+
+v_minimum3_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15|
+// GFX12: encoding: [0x05,0x07,0x2f,0xd6,0x7b,0xfa,0xed,0xe1]
+
+v_minimum3_f16 v5, m0, 0.5, m0
+// GFX12: encoding: [0x05,0x00,0x2f,0xd6,0x7d,0xe0,0xf5,0x01]
+
+v_minimum3_f16 v5, |exec_lo|, -1, vcc_hi
+// GFX12: encoding: [0x05,0x01,0x2f,0xd6,0x7e,0x82,0xad,0x01]
+
+v_minimum3_f16 v5, -|exec_hi|, null, -|vcc_lo|
+// GFX12: encoding: [0x05,0x05,0x2f,0xd6,0x7f,0xf8,0xa8,0xa1]
+
+v_minimum3_f16 v5, null, exec_lo, -|0xaf12|
+// GFX12: encoding: [0x05,0x04,0x2f,0xd6,0x7c,0xfc,0xfc,0x83,0x12,0xaf,0x00,0x00]
+
+v_minimum3_f16 v5, -1, -|exec_hi|, -|src_scc|
+// GFX12: encoding: [0x05,0x06,0x2f,0xd6,0xc1,0xfe,0xf4,0xc3]
+
+v_minimum3_f16 v5, 0.5, -m0, 0.5
+// GFX12: encoding: [0x05,0x00,0x2f,0xd6,0xf0,0xfa,0xc0,0x43]
+
+v_minimum3_f16 v5, -src_scc, |vcc_lo|, -1
+// GFX12: encoding: [0x05,0x02,0x2f,0xd6,0xfd,0xd4,0x04,0x23]
+
+v_minimum3_f16 v255, -|0xaf12|, -|vcc_hi|, null clamp
+// GFX12: encoding: [0xff,0x83,0x2f,0xd6,0xff,0xd6,0xf0,0x61,0x12,0xaf,0x00,0x00]
+
+v_maximum3_f16 v5, v1, v2, s3
+// GFX12: encoding: [0x05,0x00,0x30,0xd6,0x01,0x05,0x0e,0x00]
+
+v_maximum3_f16 v5, v255, s2, s105
+// GFX12: encoding: [0x05,0x00,0x30,0xd6,0xff,0x05,0xa4,0x01]
+
+v_maximum3_f16 v5, s1, v255, exec_hi
+// GFX12: encoding: [0x05,0x00,0x30,0xd6,0x01,0xfe,0xff,0x01]
+
+v_maximum3_f16 v5, s105, s105, exec_lo
+// GFX12: encoding: [0x05,0x00,0x30,0xd6,0x69,0xd2,0xf8,0x01]
+
+v_maximum3_f16 v5, vcc_lo, ttmp15, v3
+// GFX12: encoding: [0x05,0x00,0x30,0xd6,0x6a,0xf6,0x0c,0x04]
+
+v_maximum3_f16 v5, vcc_hi, 0xaf12, v255
+// GFX12: encoding: [0x05,0x00,0x30,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00]
+
+v_maximum3_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15|
+// GFX12: encoding: [0x05,0x07,0x30,0xd6,0x7b,0xfa,0xed,0xe1]
+
+v_maximum3_f16 v5, m0, 0.5, m0
+// GFX12: encoding: [0x05,0x00,0x30,0xd6,0x7d,0xe0,0xf5,0x01]
+
+v_maximum3_f16 v5, |exec_lo|, -1, vcc_hi
+// GFX12: encoding: [0x05,0x01,0x30,0xd6,0x7e,0x82,0xad,0x01]
+
+v_maximum3_f16 v5, -|exec_hi|, null, -|vcc_lo|
+// GFX12: encoding: [0x05,0x05,0x30,0xd6,0x7f,0xf8,0xa8,0xa1]
+
+v_maximum3_f16 v5, null, exec_lo, -|0xaf12|
+// GFX12: encoding: [0x05,0x04,0x30,0xd6,0x7c,0xfc,0xfc,0x83,0x12,0xaf,0x00,0x00]
+
+v_maximum3_f16 v5, -1, -|exec_hi|, -|src_scc|
+// GFX12: encoding: [0x05,0x06,0x30,0xd6,0xc1,0xfe,0xf4,0xc3]
+
+v_maximum3_f16 v5, 0.5, -m0, 0.5
+// GFX12: encoding: [0x05,0x00,0x30,0xd6,0xf0,0xfa,0xc0,0x43]
+
+v_maximum3_f16 v5, -src_scc, |vcc_lo|, -1
+// GFX12: encoding: [0x05,0x02,0x30,0xd6,0xfd,0xd4,0x04,0x23]
+
+v_maximumminimum_f32 v5, v1, v2, s3
+// GFX12: encoding: [0x05,0x00,0x6d,0xd6,0x01,0x05,0x0e,0x00]
+
+v_maximumminimum_f32 v5, v255, s2, s105
+// GFX12: encoding: [0x05,0x00,0x6d,0xd6,0xff,0x05,0xa4,0x01]
+
+v_maximumminimum_f32 v5, s1, v255, exec_hi
+// GFX12: encoding: [0x05,0x00,0x6d,0xd6,0x01,0xfe,0xff,0x01]
+
+v_maximumminimum_f32 v5, s105, s105, exec_lo
+// GFX12: encoding: [0x05,0x00,0x6d,0xd6,0x69,0xd2,0xf8,0x01]
+
+v_maximumminimum_f32 v5, vcc_lo, ttmp15, v3
+// GFX12: encoding: [0x05,0x00,0x6d,0xd6,0x6a,0xf6,0x0c,0x04]
+
+v_maximumminimum_f32 v5, vcc_hi, 0xaf123456, v255
+// GFX12: encoding: [0x05,0x00,0x6d,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
+
+v_maximumminimum_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15|
+// GFX12: encoding: [0x05,0x07,0x6d,0xd6,0x7b,0xfa,0xed,0xe1]
+
+v_maximumminimum_f32 v5, m0, 0.5, m0
+// GFX12: encoding: [0x05,0x00,0x6d,0xd6,0x7d,0xe0,0xf5,0x01]
+
+v_maximumminimum_f32 v5, |exec_lo|, -1, vcc_hi
+// GFX12: encoding: [0x05,0x01,0x6d,0xd6,0x7e,0x82,0xad,0x01]
+
+v_maximumminimum_f32 v5, -|exec_hi|, null, -|vcc_lo|
+// GFX12: encoding: [0x05,0x05,0x6d,0xd6,0x7f,0xf8,0xa8,0xa1]
+
+v_maximumminimum_f32 v5, null, exec_lo, -|0xaf123456|
+// GFX12: encoding: [0x05,0x04,0x6d,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
+
+v_maximumminimum_f32 v5, -1, -|exec_hi|, -|src_scc|
+// GFX12: encoding: [0x05,0x06,0x6d,0xd6,0xc1,0xfe,0xf4,0xc3]
+
+v_maximumminimum_f32 v5, 0.5, -m0, 0.5 mul:2
+// GFX12: encoding: [0x05,0x00,0x6d,0xd6,0xf0,0xfa,0xc0,0x4b]
+
+v_maximumminimum_f32 v5, -src_scc, |vcc_lo|, -1 mul:4
+// GFX12: encoding: [0x05,0x02,0x6d,0xd6,0xfd,0xd4,0x04,0x33]
+
+v_maximumminimum_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2
+// GFX12: encoding: [0xff,0x83,0x6d,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
+
+v_minimummaximum_f32 v5, v1, v2, s3
+// GFX12: encoding: [0x05,0x00,0x6c,0xd6,0x01,0x05,0x0e,0x00]
+
+v_minimummaximum_f32 v5, v255, s2, s105
+// GFX12: encoding: [0x05,0x00,0x6c,0xd6,0xff,0x05,0xa4,0x01]
+
+v_minimummaximum_f32 v5, s1, v255, exec_hi
+// GFX12: encoding: [0x05,0x00,0x6c,0xd6,0x01,0xfe,0xff,0x01]
+
+v_minimummaximum_f32 v5, s105, s105, exec_lo
+// GFX12: encoding: [0x05,0x00,0x6c,0xd6,0x69,0xd2,0xf8,0x01]
+
+v_minimummaximum_f32 v5, vcc_lo, ttmp15, v3
+// GFX12: encoding: [0x05,0x00,0x6c,0xd6,0x6a,0xf6,0x0c,0x04]
+
+v_minimummaximum_f32 v5, vcc_hi, 0xaf123456, v255
+// GFX12: encoding: [0x05,0x00,0x6c,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
+
+v_minimummaximum_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15|
+// GFX12: encoding: [0x05,0x07,0x6c,0xd6,0x7b,0xfa,0xed,0xe1]
+
+v_minimummaximum_f32 v5, m0, 0.5, m0
+// GFX12: encoding: [0x05,0x00,0x6c,0xd6,0x7d,0xe0,0xf5,0x01]
+
+v_minimummaximum_f32 v5, |exec_lo|, -1, vcc_hi
+// GFX12: encoding: [0x05,0x01,0x6c,0xd6,0x7e,0x82,0xad,0x01]
+
+v_minimummaximum_f32 v5, -|exec_hi|, null, -|vcc_lo|
+// GFX12: encoding: [0x05,0x05,0x6c,0xd6,0x7f,0xf8,0xa8,0xa1]
+
+v_minimummaximum_f32 v5, null, exec_lo, -|0xaf123456|
+// GFX12: encoding: [0x05,0x04,0x6c,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
+
+v_minimummaximum_f32 v5, -1, -|exec_hi|, -|src_scc|
+// GFX12: encoding: [0x05,0x06,0x6c,0xd6,0xc1,0xfe,0xf4,0xc3]
+
+v_minimummaximum_f32 v5, 0.5, -m0, 0.5 mul:2
+// GFX12: encoding: [0x05,0x00,0x6c,0xd6,0xf0,0xfa,0xc0,0x4b]
+
+v_minimummaximum_f32 v5, -src_scc, |vcc_lo|, -1 mul:4
+// GFX12: encoding: [0x05,0x02,0x6c,0xd6,0xfd,0xd4,0x04,0x33]
+
+v_minimummaximum_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2
+// GFX12: encoding: [0xff,0x83,0x6c,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
+
+v_maximumminimum_f16 v5, v1, v2, s3
+// GFX12: encoding: [0x05,0x00,0x6f,0xd6,0x01,0x05,0x0e,0x00]
+
+v_maximumminimum_f16 v5, v255, s2, s105
+// GFX12: encoding: [0x05,0x00,0x6f,0xd6,0xff,0x05,0xa4,0x01]
+
+v_maximumminimum_f16 v5, s1, v255, exec_hi
+// GFX12: encoding: [0x05,0x00,0x6f,0xd6,0x01,0xfe,0xff,0x01]
+
+v_maximumminimum_f16 v5, s105, s105, exec_lo
+// GFX12: encoding: [0x05,0x00,0x6f,0xd6,0x69,0xd2,0xf8,0x01]
+
+v_maximumminimum_f16 v5, vcc_lo, ttmp15, v3
+// GFX12: encoding: [0x05,0x00,0x6f,0xd6,0x6a,0xf6,0x0c,0x04]
+
+v_maximumminimum_f16 v5, vcc_hi, 0xaf12, v255
+// GFX12: encoding: [0x05,0x00,0x6f,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00]
+
+v_maximumminimum_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15|
+// GFX12: encoding: [0x05,0x07,0x6f,0xd6,0x7b,0xfa,0xed,0xe1]
+
+v_maximumminimum_f16 v5, m0, 0.5, m0
+// GFX12: encoding: [0x05,0x00,0x6f,0xd6,0x7d,0xe0,0xf5,0x01]
+
+v_maximumminimum_f16 v5, |exec_lo|, -1, vcc_hi
+// GFX12: encoding: [0x05,0x01,0x6f,0xd6,0x7e,0x82,0xad,0x01]
+
+v_maximumminimum_f16 v5, -|exec_hi|, null, -|vcc_lo|
+// GFX12: encoding: [0x05,0x05,0x6f,0xd6,0x7f,0xf8,0xa8,0xa1]
+
+v_maximumminimum_f16 v5, null, exec_lo, -|0xaf12|
+// GFX12: encoding: [0x05,0x04,0x6f,0xd6,0x7c,0xfc,0xfc,0x83,0x12,0xaf,0x00,0x00]
+
+v_maximumminimum_f16 v5, -1, -|exec_hi|, -|src_scc|
+// GFX12: encoding: [0x05,0x06,0x6f,0xd6,0xc1,0xfe,0xf4,0xc3]
+
+v_maximumminimum_f16 v5, 0.5, -m0, 0.5
+// GFX12: encoding: [0x05,0x00,0x6f,0xd6,0xf0,0xfa,0xc0,0x43]
+
+v_maximumminimum_f16 v5, -src_scc, |vcc_lo|, -1
+// GFX12: encoding: [0x05,0x02,0x6f,0xd6,0xfd,0xd4,0x04,0x23]
+
+v_maximumminimum_f16 v255, -|0xaf12|, -|vcc_hi|, null clamp
+// GFX12: encoding: [0xff,0x83,0x6f,0xd6,0xff,0xd6,0xf0,0x61,0x12,0xaf,0x00,0x00]
+
+v_minimummaximum_f16 v5, v1, v2, s3
+// GFX12: encoding: [0x05,0x00,0x6e,0xd6,0x01,0x05,0x0e,0x00]
+
+v_minimummaximum_f16 v5, v255, s2, s105
+// GFX12: encoding: [0x05,0x00,0x6e,0xd6,0xff,0x05,0xa4,0x01]
+
+v_minimummaximum_f16 v5, s1, v255, exec_hi
+// GFX12: encoding: [0x05,0x00,0x6e,0xd6,0x01,0xfe,0xff,0x01]
+
+v_minimummaximum_f16 v5, s105, s105, exec_lo
+// GFX12: encoding: [0x05,0x00,0x6e,0xd6,0x69,0xd2,0xf8,0x01]
+
+v_minimummaximum_f16 v5, vcc_lo, ttmp15, v3
+// GFX12: encoding: [0x05,0x00,0x6e,0xd6,0x6a,0xf6,0x0c,0x04]
+
+v_minimummaximum_f16 v5, vcc_hi, 0xaf12, v255
+// GFX12: encoding: [0x05,0x00,0x6e,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00]
+
+v_minimummaximum_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15|
+// GFX12: encoding: [0x05,0x07,0x6e,0xd6,0x7b,0xfa,0xed,0xe1]
+
+v_minimummaximum_f16 v5, m0, 0.5, m0
+// GFX12: encoding: [0x05,0x00,0x6e,0xd6,0x7d,0xe0,0xf5,0x01]
+
+v_minimummaximum_f16 v5, |exec_lo|, -1, vcc_hi
+// GFX12: encoding: [0x05,0x01,0x6e,0xd6,0x7e,0x82,0xad,0x01]
+
+v_minimummaximum_f16 v5, -|exec_hi|, null, -|vcc_lo|
+// GFX12: encoding: [0x05,0x05,0x6e,0xd6,0x7f,0xf8,0xa8,0xa1]
+
+v_minimummaximum_f16 v5, null, exec_lo, -|0xaf12|
+// GFX12: encoding: [0x05,0x04,0x6e,0xd6,0x7c,0xfc,0xfc,0x83,0x12,0xaf,0x00,0x00]
+
+v_minimummaximum_f16 v5, -1, -|exec_hi|, -|src_scc|
+// GFX12: encoding: [0x05,0x06,0x6e,0xd6,0xc1,0xfe,0xf4,0xc3]
+
+v_minimummaximum_f16 v5, 0.5, -m0, 0.5
+// GFX12: encoding: [0x05,0x00,0x6e,0xd6,0xf0,0xfa,0xc0,0x43]
+
+v_minimummaximum_f16 v5, -src_scc, |vcc_lo|, -1
+// GFX12: encoding: [0x05,0x02,0x6e,0xd6,0xfd,0xd4,0x04,0x23]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3_dpp16.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_dpp16.s
index 63087442c46488d9e99562ba96d05887390e62aa..26f63102df9508130ce7dabfee73f4ec2ec56898 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vop3_dpp16.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_dpp16.s
@@ -4693,3 +4693,504 @@ v_dot2_bf16_bf16_e64_dpp v0, |v1|, -v2, -|s3| op_sel:[0,0,1,1] quad_perm:[0,1,2,
 
 v_dot2_bf16_bf16_e64_dpp v5, v1, v2, 0 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf
 // GFX12: [0x05,0x00,0x67,0xd6,0xfa,0x04,0x02,0x02,0x01,0x1b,0x00,0xff]
+
+v_minimum_f32 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: [0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_minimum_f32 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: [0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_minimum_f32 v5, v1, v2 row_mirror
+// GFX12: [0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_minimum_f32 v5, v1, v2 row_half_mirror
+// GFX12: [0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_minimum_f32 v5, v1, v2 row_shl:1
+// GFX12: [0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_minimum_f32 v5, v1, v2 row_shl:15
+// GFX12: [0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_minimum_f32 v5, v1, v2 row_shr:1
+// GFX12: [0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_minimum_f32 v5, v1, v2 row_shr:15
+// GFX12: [0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_minimum_f32 v5, v1, v2 row_ror:1
+// GFX12: [0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_minimum_f32 v5, v1, v2 row_ror:15
+// GFX12: [0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_minimum_f32 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: [0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_minimum_f32 v5, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: [0x05,0x01,0x65,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+v_minimum_f32 v5, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: [0x05,0x02,0x65,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+
+v_minimum_f32 v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: [0xff,0x03,0x65,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_maximum_f32 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: [0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_maximum_f32 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: [0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_maximum_f32 v5, v1, v2 row_mirror
+// GFX12: [0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_maximum_f32 v5, v1, v2 row_half_mirror
+// GFX12: [0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_maximum_f32 v5, v1, v2 row_shl:1
+// GFX12: [0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_maximum_f32 v5, v1, v2 row_shl:15
+// GFX12: [0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_maximum_f32 v5, v1, v2 row_shr:1
+// GFX12: [0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_maximum_f32 v5, v1, v2 row_shr:15
+// GFX12: [0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_maximum_f32 v5, v1, v2 row_ror:1
+// GFX12: [0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_maximum_f32 v5, v1, v2 row_ror:15
+// GFX12: [0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_maximum_f32 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: [0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_maximum_f32 v5, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: [0x05,0x01,0x66,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+v_maximum_f32 v5, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: [0x05,0x02,0x66,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+
+v_maximum_f32 v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: [0xff,0x03,0x66,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_minimum_f16 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_minimum_f16 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_minimum_f16 v5, v1, v2 row_mirror
+// GFX12: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_minimum_f16 v5, v1, v2 row_half_mirror
+// GFX12: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_minimum_f16 v5, v1, v2 row_shl:1
+// GFX12: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_minimum_f16 v5, v1, v2 row_shl:15
+// GFX12: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_minimum_f16 v5, v1, v2 row_shr:1
+// GFX12: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_minimum_f16 v5, v1, v2 row_shr:15
+// GFX12: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_minimum_f16 v5, v1, v2 row_ror:1
+// GFX12: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_minimum_f16 v5, v1, v2 row_ror:15
+// GFX12: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_minimum_f16 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_minimum_f16 v5, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: [0x05,0x01,0x67,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+v_minimum_f16 v5, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: [0x05,0x02,0x67,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+
+v_minimum_f16 v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: [0xff,0x03,0x67,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_maximum_f16 v5, v1, v2 quad_perm:[3,2,1,0]
+// GFX12: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+
+v_maximum_f16 v5, v1, v2 quad_perm:[0,1,2,3]
+// GFX12: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+
+v_maximum_f16 v5, v1, v2 row_mirror
+// GFX12: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+
+v_maximum_f16 v5, v1, v2 row_half_mirror
+// GFX12: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+
+v_maximum_f16 v5, v1, v2 row_shl:1
+// GFX12: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+
+v_maximum_f16 v5, v1, v2 row_shl:15
+// GFX12: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+
+v_maximum_f16 v5, v1, v2 row_shr:1
+// GFX12: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+
+v_maximum_f16 v5, v1, v2 row_shr:15
+// GFX12: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+
+v_maximum_f16 v5, v1, v2 row_ror:1
+// GFX12: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+
+v_maximum_f16 v5, v1, v2 row_ror:15
+// GFX12: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+
+v_maximum_f16 v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+
+v_maximum_f16 v5, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: [0x05,0x01,0x68,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+
+v_maximum_f16 v5, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: [0x05,0x02,0x68,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+
+v_maximum_f16 v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: [0xff,0x03,0x68,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x05,0x30]
+
+v_minimum3_f32 v5, v1, v2, v3 quad_perm:[3,2,1,0]
+// GFX12: [0x05,0x00,0x2d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+
+v_minimum3_f32 v5, v1, v2, v3 quad_perm:[0,1,2,3]
+// GFX12: [0x05,0x00,0x2d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+
+v_minimum3_f32 v5, v1, v2, v3 row_mirror
+// GFX12: [0x05,0x00,0x2d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
+
+v_minimum3_f32 v5, v1, v2, v255 row_half_mirror
+// GFX12: [0x05,0x00,0x2d,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
+
+v_minimum3_f32 v5, v1, v2, s105 row_shl:1
+// GFX12: [0x05,0x00,0x2d,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
+
+v_minimum3_f32 v5, v1, v2, vcc_hi row_shl:15
+// GFX12: [0x05,0x00,0x2d,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
+
+v_minimum3_f32 v5, v1, v2, vcc_lo row_shr:1
+// GFX12: [0x05,0x00,0x2d,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
+
+v_minimum3_f32 v5, |v1|, v2, -ttmp15 row_shr:15
+// GFX12: [0x05,0x01,0x2d,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
+
+v_minimum3_f32 v5, v1, -|v2|, exec_hi row_ror:1
+// GFX12: [0x05,0x02,0x2d,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
+
+v_minimum3_f32 v5, -v1, v2, |exec_lo| row_ror:15
+// GFX12: [0x05,0x04,0x2d,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
+
+v_minimum3_f32 v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: [0x05,0x03,0x2d,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
+
+v_minimum3_f32 v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: [0x05,0x05,0x2d,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
+
+v_minimum3_f32 v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: [0x05,0x06,0x2d,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x09,0x13]
+
+v_minimum3_f32 v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: [0xff,0x87,0x2d,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x05,0x30]
+
+v_maximum3_f32 v5, v1, v2, v3 quad_perm:[3,2,1,0]
+// GFX12: [0x05,0x00,0x2e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+
+v_maximum3_f32 v5, v1, v2, v3 quad_perm:[0,1,2,3]
+// GFX12: [0x05,0x00,0x2e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+
+v_maximum3_f32 v5, v1, v2, v3 row_mirror
+// GFX12: [0x05,0x00,0x2e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
+
+v_maximum3_f32 v5, v1, v2, v255 row_half_mirror
+// GFX12: [0x05,0x00,0x2e,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
+
+v_maximum3_f32 v5, v1, v2, s105 row_shl:1
+// GFX12: [0x05,0x00,0x2e,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
+
+v_maximum3_f32 v5, v1, v2, vcc_hi row_shl:15
+// GFX12: [0x05,0x00,0x2e,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
+
+v_maximum3_f32 v5, v1, v2, vcc_lo row_shr:1
+// GFX12: [0x05,0x00,0x2e,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
+
+v_maximum3_f32 v5, |v1|, v2, -ttmp15 row_shr:15
+// GFX12: [0x05,0x01,0x2e,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
+
+v_maximum3_f32 v5, v1, -|v2|, exec_hi row_ror:1
+// GFX12: [0x05,0x02,0x2e,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
+
+v_maximum3_f32 v5, -v1, v2, |exec_lo| row_ror:15
+// GFX12: [0x05,0x04,0x2e,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
+
+v_maximum3_f32 v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: [0x05,0x03,0x2e,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
+
+v_maximum3_f32 v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: [0x05,0x05,0x2e,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
+
+v_maximum3_f32 v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: [0x05,0x06,0x2e,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x09,0x13]
+
+v_maximum3_f32 v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: [0xff,0x87,0x2e,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x05,0x30]
+
+v_minimum3_f16 v5, v1, v2, v3 quad_perm:[3,2,1,0]
+// GFX12: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+
+v_minimum3_f16 v5, v1, v2, v3 quad_perm:[0,1,2,3]
+// GFX12: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+
+v_minimum3_f16 v5, v1, v2, v3 row_mirror
+// GFX12: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
+
+v_minimum3_f16 v5, v1, v2, v255 row_half_mirror
+// GFX12: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
+
+v_minimum3_f16 v5, v1, v2, s105 row_shl:1
+// GFX12: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
+
+v_minimum3_f16 v5, v1, v2, vcc_hi row_shl:15
+// GFX12: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
+
+v_minimum3_f16 v5, v1, v2, vcc_lo row_shr:1
+// GFX12: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
+
+v_minimum3_f16 v5, |v1|, v2, -ttmp15 row_shr:15
+// GFX12: [0x05,0x01,0x2f,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
+
+v_minimum3_f16 v5, v1, -|v2|, exec_hi row_ror:1
+// GFX12: [0x05,0x02,0x2f,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
+
+v_minimum3_f16 v5, -v1, v2, |exec_lo| row_ror:15
+// GFX12: [0x05,0x04,0x2f,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
+
+v_minimum3_f16 v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: [0x05,0x03,0x2f,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
+
+v_minimum3_f16 v5, -|v1|, v2, -|-1| row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: [0x05,0x05,0x2f,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01]
+
+v_minimum3_f16 v5, v1, -|v2|, -|0.5| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: [0x05,0x06,0x2f,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x09,0x13]
+
+v_minimum3_f16 v255, -|v255|, -|v255|, -|src_scc| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: [0xff,0x87,0x2f,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x05,0x30]
+
+v_maximum3_f16 v5, v1, v2, v3 quad_perm:[3,2,1,0]
+// GFX12: [0x05,0x00,0x30,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+
+v_maximum3_f16 v5, v1, v2, v3 quad_perm:[0,1,2,3]
+// GFX12: [0x05,0x00,0x30,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+
+v_maximum3_f16 v5, v1, v2, v3 row_mirror
+// GFX12: [0x05,0x00,0x30,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
+
+v_maximum3_f16 v5, v1, v2, v255 row_half_mirror
+// GFX12: [0x05,0x00,0x30,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
+
+v_maximum3_f16 v5, v1, v2, s105 row_shl:1
+// GFX12: [0x05,0x00,0x30,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
+
+v_maximum3_f16 v5, v1, v2, vcc_hi row_shl:15
+// GFX12: [0x05,0x00,0x30,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
+
+v_maximum3_f16 v5, v1, v2, vcc_lo row_shr:1
+// GFX12: [0x05,0x00,0x30,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
+
+v_maximum3_f16 v5, |v1|, v2, -ttmp15 row_shr:15
+// GFX12: [0x05,0x01,0x30,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
+
+v_maximum3_f16 v5, v1, -|v2|, exec_hi row_ror:1
+// GFX12: [0x05,0x02,0x30,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
+
+v_maximum3_f16 v5, -v1, v2, |exec_lo| row_ror:15
+// GFX12: [0x05,0x04,0x30,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
+
+v_maximum3_f16 v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: [0x05,0x03,0x30,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
+
+v_maximum3_f16 v5, -|v1|, v2, -|-1| row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: [0x05,0x05,0x30,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01]
+
+v_maximum3_f16 v5, v1, -|v2|, -|0.5| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: [0x05,0x06,0x30,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x09,0x13]
+
+v_maximum3_f16 v255, -|v255|, -|v255|, -|src_scc| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: [0xff,0x87,0x30,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x05,0x30]
+
+v_maximumminimum_f32 v5, v1, v2, v3 quad_perm:[0,1,2,3]
+// GFX12: [0x05,0x00,0x6d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+
+v_maximumminimum_f32 v5, v1, v2, v3 row_mirror
+// GFX12: [0x05,0x00,0x6d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
+
+v_maximumminimum_f32 v5, v1, v2, v255 row_half_mirror
+// GFX12: [0x05,0x00,0x6d,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
+
+v_maximumminimum_f32 v5, v1, v2, s105 row_shl:1
+// GFX12: [0x05,0x00,0x6d,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
+
+v_maximumminimum_f32 v5, v1, v2, vcc_hi row_shl:15
+// GFX12: [0x05,0x00,0x6d,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
+
+v_maximumminimum_f32 v5, v1, v2, vcc_lo row_shr:1
+// GFX12: [0x05,0x00,0x6d,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
+
+v_maximumminimum_f32 v5, |v1|, v2, -ttmp15 row_shr:15
+// GFX12: [0x05,0x01,0x6d,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
+
+v_maximumminimum_f32 v5, v1, -|v2|, exec_hi row_ror:1
+// GFX12: [0x05,0x02,0x6d,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
+
+v_maximumminimum_f32 v5, -v1, v2, |exec_lo| row_ror:15
+// GFX12: [0x05,0x04,0x6d,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
+
+v_maximumminimum_f32 v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: [0x05,0x03,0x6d,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
+
+v_maximumminimum_f32 v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: [0x05,0x05,0x6d,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
+
+v_maximumminimum_f32 v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: [0x05,0x06,0x6d,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x09,0x13]
+
+v_maximumminimum_f32 v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: [0xff,0x87,0x6d,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x05,0x30]
+
+v_minimummaximum_f32 v5, v1, v2, v3 quad_perm:[3,2,1,0]
+// GFX12: [0x05,0x00,0x6c,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+
+v_minimummaximum_f32 v5, v1, v2, v3 quad_perm:[0,1,2,3]
+// GFX12: [0x05,0x00,0x6c,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+
+v_minimummaximum_f32 v5, v1, v2, v3 row_mirror
+// GFX12: [0x05,0x00,0x6c,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
+
+v_minimummaximum_f32 v5, v1, v2, v255 row_half_mirror
+// GFX12: [0x05,0x00,0x6c,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
+
+v_minimummaximum_f32 v5, v1, v2, s105 row_shl:1
+// GFX12: [0x05,0x00,0x6c,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
+
+v_minimummaximum_f32 v5, v1, v2, vcc_hi row_shl:15
+// GFX12: [0x05,0x00,0x6c,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
+
+v_minimummaximum_f32 v5, v1, v2, vcc_lo row_shr:1
+// GFX12: [0x05,0x00,0x6c,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
+
+v_minimummaximum_f32 v5, |v1|, v2, -ttmp15 row_shr:15
+// GFX12: [0x05,0x01,0x6c,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
+
+v_minimummaximum_f32 v5, v1, -|v2|, exec_hi row_ror:1
+// GFX12: [0x05,0x02,0x6c,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
+
+v_minimummaximum_f32 v5, -v1, v2, |exec_lo| row_ror:15
+// GFX12: [0x05,0x04,0x6c,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
+
+v_minimummaximum_f32 v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: [0x05,0x03,0x6c,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
+
+v_minimummaximum_f32 v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: [0x05,0x05,0x6c,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
+
+v_minimummaximum_f32 v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: [0x05,0x06,0x6c,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x09,0x13]
+
+v_minimummaximum_f32 v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: [0xff,0x87,0x6c,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x05,0x30]
+
+v_maximumminimum_f16 v5, v1, v2, v3 quad_perm:[3,2,1,0]
+// GFX12: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+
+v_maximumminimum_f16 v5, v1, v2, v3 quad_perm:[0,1,2,3]
+// GFX12: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+
+v_maximumminimum_f16 v5, v1, v2, v3 row_mirror
+// GFX12: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
+
+v_maximumminimum_f16 v5, v1, v2, v255 row_half_mirror
+// GFX12: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
+
+v_maximumminimum_f16 v5, v1, v2, s105 row_shl:1
+// GFX12: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
+
+v_maximumminimum_f16 v5, v1, v2, vcc_hi row_shl:15
+// GFX12: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
+
+v_maximumminimum_f16 v5, v1, v2, vcc_lo row_shr:1
+// GFX12: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
+
+v_maximumminimum_f16 v5, |v1|, v2, -ttmp15 row_shr:15
+// GFX12: [0x05,0x01,0x6f,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
+
+v_maximumminimum_f16 v5, v1, -|v2|, exec_hi row_ror:1
+// GFX12: [0x05,0x02,0x6f,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
+
+v_maximumminimum_f16 v5, -v1, v2, |exec_lo| row_ror:15
+// GFX12: [0x05,0x04,0x6f,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
+
+v_maximumminimum_f16 v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: [0x05,0x03,0x6f,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
+
+v_maximumminimum_f16 v5, -|v1|, v2, -|-1| row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: [0x05,0x05,0x6f,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01]
+
+v_maximumminimum_f16 v5, v1, -|v2|, -|0.5| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: [0x05,0x06,0x6f,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x09,0x13]
+
+v_maximumminimum_f16 v255, -|v255|, -|v255|, -|src_scc| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: [0xff,0x87,0x6f,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x05,0x30]
+
+v_minimummaximum_f16 v5, v1, v2, v3 quad_perm:[3,2,1,0]
+// GFX12: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+
+v_minimummaximum_f16 v5, v1, v2, v3 quad_perm:[0,1,2,3]
+// GFX12: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+
+v_minimummaximum_f16 v5, v1, v2, v3 row_mirror
+// GFX12: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
+
+v_minimummaximum_f16 v5, v1, v2, v255 row_half_mirror
+// GFX12: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
+
+v_minimummaximum_f16 v5, v1, v2, s105 row_shl:1
+// GFX12: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
+
+v_minimummaximum_f16 v5, v1, v2, vcc_hi row_shl:15
+// GFX12: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
+
+v_minimummaximum_f16 v5, v1, v2, vcc_lo row_shr:1
+// GFX12: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
+
+v_minimummaximum_f16 v5, |v1|, v2, -ttmp15 row_shr:15
+// GFX12: [0x05,0x01,0x6e,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
+
+v_minimummaximum_f16 v5, v1, -|v2|, exec_hi row_ror:1
+// GFX12: [0x05,0x02,0x6e,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
+
+v_minimummaximum_f16 v5, -v1, v2, |exec_lo| row_ror:15
+// GFX12: [0x05,0x04,0x6e,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
+
+v_minimummaximum_f16 v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf
+// GFX12: [0x05,0x03,0x6e,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
+
+v_minimummaximum_f16 v5, -|v1|, v2, -|-1| row_share:15 row_mask:0x0 bank_mask:0x1
+// GFX12: [0x05,0x05,0x6e,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01]
+
+v_minimummaximum_f16 v5, v1, -|v2|, -|0.5| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 fi:0
+// GFX12: [0x05,0x06,0x6e,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x09,0x13]
+
+v_minimummaximum_f16 v255, -|v255|, -|v255|, -|src_scc| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:0 fi:1
+// GFX12: [0xff,0x87,0x6e,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x05,0x30]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3_dpp8.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_dpp8.s
index 82807aca0e7bfb176fcdc5a6a38a9465861502ed..de294b1ff2a22a4f5f32d789bc10b3a58fd343bd 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vop3_dpp8.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3_dpp8.s
@@ -2966,3 +2966,339 @@ v_dot2_bf16_bf16_e64_dpp v0, |v1|, -v2, -|s3| op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4
 
 v_dot2_bf16_bf16_e64_dpp v5, v1, v2, 0 dpp8:[7,6,5,4,3,2,1,0]
 // GFX12: [0x05,0x00,0x67,0xd6,0xe9,0x04,0x02,0x02,0x01,0x77,0x39,0x05]
+
+v_minimum_f32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x00,0x65,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_minimum_f32 v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x01,0x65,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_minimum_f32 v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: [0x05,0x02,0x65,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_minimum_f32 v255, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: [0xff,0x03,0x65,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_maximum_f32 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x00,0x66,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_maximum_f32 v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x01,0x66,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_maximum_f32 v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: [0x05,0x02,0x66,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_maximum_f32 v255, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: [0xff,0x03,0x66,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_minimum_f16 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x00,0x67,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_minimum_f16 v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x01,0x67,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_minimum_f16 v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: [0x05,0x02,0x67,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_minimum_f16 v255, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: [0xff,0x03,0x67,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_maximum_f16 v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x00,0x68,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+
+v_maximum_f16 v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x01,0x68,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+
+v_maximum_f16 v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: [0x05,0x02,0x68,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+
+v_maximum_f16 v255, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: [0xff,0x03,0x68,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+
+v_minimum3_f32 v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x00,0x2d,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+
+v_minimum3_f32 v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x00,0x2d,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+
+v_minimum3_f32 v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x00,0x2d,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+
+v_minimum3_f32 v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x00,0x2d,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+
+v_minimum3_f32 v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x00,0x2d,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+
+v_minimum3_f32 v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x01,0x2d,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
+
+v_minimum3_f32 v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x02,0x2d,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
+
+v_minimum3_f32 v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x04,0x2d,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
+
+v_minimum3_f32 v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x03,0x2d,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
+
+v_minimum3_f32 v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x05,0x2d,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
+
+v_minimum3_f32 v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: [0x05,0x06,0x2d,0xd6,0xea,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
+
+v_minimum3_f32 v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: [0xff,0x87,0x2d,0xd6,0xe9,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
+
+v_maximum3_f32 v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x00,0x2e,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+
+v_maximum3_f32 v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x00,0x2e,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+
+v_maximum3_f32 v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x00,0x2e,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+
+v_maximum3_f32 v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x00,0x2e,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+
+v_maximum3_f32 v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x00,0x2e,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+
+v_maximum3_f32 v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x01,0x2e,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
+
+v_maximum3_f32 v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x02,0x2e,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
+
+v_maximum3_f32 v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x04,0x2e,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
+
+v_maximum3_f32 v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x03,0x2e,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
+
+v_maximum3_f32 v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x05,0x2e,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
+
+v_maximum3_f32 v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: [0x05,0x06,0x2e,0xd6,0xea,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
+
+v_maximum3_f32 v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: [0xff,0x87,0x2e,0xd6,0xe9,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
+
+v_minimum3_f16 v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x00,0x2f,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+
+v_minimum3_f16 v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x00,0x2f,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+
+v_minimum3_f16 v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x00,0x2f,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+
+v_minimum3_f16 v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x00,0x2f,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+
+v_minimum3_f16 v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x00,0x2f,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+
+v_minimum3_f16 v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x01,0x2f,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
+
+v_minimum3_f16 v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x02,0x2f,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
+
+v_minimum3_f16 v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x04,0x2f,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
+
+v_minimum3_f16 v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x03,0x2f,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
+
+v_minimum3_f16 v5, -|v1|, v2, -|-1| dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x05,0x2f,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05]
+
+v_minimum3_f16 v5, v1, -|v2|, -|0.5| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: [0x05,0x06,0x2f,0xd6,0xea,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05]
+
+v_minimum3_f16 v255, -|v255|, -|v255|, -|src_scc| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: [0xff,0x87,0x2f,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+
+v_maximum3_f16 v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x00,0x30,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+
+v_maximum3_f16 v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x00,0x30,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+
+v_maximum3_f16 v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x00,0x30,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+
+v_maximum3_f16 v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x00,0x30,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+
+v_maximum3_f16 v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x00,0x30,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+
+v_maximum3_f16 v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x01,0x30,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
+
+v_maximum3_f16 v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x02,0x30,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
+
+v_maximum3_f16 v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x04,0x30,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
+
+v_maximum3_f16 v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x03,0x30,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
+
+v_maximum3_f16 v5, -|v1|, v2, -|-1| dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x05,0x30,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05]
+
+v_maximum3_f16 v5, v1, -|v2|, -|0.5| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: [0x05,0x06,0x30,0xd6,0xea,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05]
+
+v_maximum3_f16 v255, -|v255|, -|v255|, -|src_scc| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: [0xff,0x87,0x30,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+
+v_maximumminimum_f32 v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x00,0x6d,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+
+v_maximumminimum_f32 v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x00,0x6d,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+
+v_maximumminimum_f32 v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x00,0x6d,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+
+v_maximumminimum_f32 v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x00,0x6d,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+
+v_maximumminimum_f32 v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x00,0x6d,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+
+v_maximumminimum_f32 v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x01,0x6d,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
+
+v_maximumminimum_f32 v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x02,0x6d,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
+
+v_maximumminimum_f32 v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x04,0x6d,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
+
+v_maximumminimum_f32 v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x03,0x6d,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
+
+v_maximumminimum_f32 v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x05,0x6d,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
+
+v_maximumminimum_f32 v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: [0x05,0x06,0x6d,0xd6,0xea,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
+
+v_maximumminimum_f32 v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: [0xff,0x87,0x6d,0xd6,0xe9,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
+
+v_minimummaximum_f32 v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x00,0x6c,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+
+v_minimummaximum_f32 v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x00,0x6c,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+
+v_minimummaximum_f32 v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x00,0x6c,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+
+v_minimummaximum_f32 v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x00,0x6c,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+
+v_minimummaximum_f32 v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x00,0x6c,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+
+v_minimummaximum_f32 v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x01,0x6c,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
+
+v_minimummaximum_f32 v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x02,0x6c,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
+
+v_minimummaximum_f32 v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x04,0x6c,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
+
+v_minimummaximum_f32 v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x03,0x6c,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
+
+v_minimummaximum_f32 v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x05,0x6c,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
+
+v_minimummaximum_f32 v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: [0x05,0x06,0x6c,0xd6,0xea,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
+
+v_minimummaximum_f32 v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: [0xff,0x87,0x6c,0xd6,0xe9,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
+
+v_maximumminimum_f16 v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x00,0x6f,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+
+v_maximumminimum_f16 v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x00,0x6f,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+
+v_maximumminimum_f16 v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x00,0x6f,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+
+v_maximumminimum_f16 v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x00,0x6f,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+
+v_maximumminimum_f16 v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x00,0x6f,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+
+v_maximumminimum_f16 v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x01,0x6f,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
+
+v_maximumminimum_f16 v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x02,0x6f,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
+
+v_maximumminimum_f16 v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x04,0x6f,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
+
+v_maximumminimum_f16 v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x03,0x6f,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
+
+v_maximumminimum_f16 v5, -|v1|, v2, -|-1| dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x05,0x6f,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05]
+
+v_maximumminimum_f16 v5, v1, -|v2|, -|0.5| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: [0x05,0x06,0x6f,0xd6,0xea,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05]
+
+v_maximumminimum_f16 v255, -|v255|, -|v255|, -|src_scc| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: [0xff,0x87,0x6f,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+
+v_minimummaximum_f16 v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x00,0x6e,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+
+v_minimummaximum_f16 v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x00,0x6e,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+
+v_minimummaximum_f16 v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x00,0x6e,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+
+v_minimummaximum_f16 v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x00,0x6e,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+
+v_minimummaximum_f16 v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x00,0x6e,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+
+v_minimummaximum_f16 v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x01,0x6e,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
+
+v_minimummaximum_f16 v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x02,0x6e,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
+
+v_minimummaximum_f16 v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x04,0x6e,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
+
+v_minimummaximum_f16 v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x03,0x6e,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
+
+v_minimummaximum_f16 v5, -|v1|, v2, -|-1| dpp8:[7,6,5,4,3,2,1,0]
+// GFX12: [0x05,0x05,0x6e,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05]
+
+v_minimummaximum_f16 v5, v1, -|v2|, -|0.5| dpp8:[7,6,5,4,3,2,1,0] fi:1
+// GFX12: [0x05,0x06,0x6e,0xd6,0xea,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05]
+
+v_minimummaximum_f16 v255, -|v255|, -|v255|, -|src_scc| clamp dpp8:[0,0,0,0,0,0,0,0] fi:0
+// GFX12: [0xff,0x87,0x6e,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_vop3p.s b/llvm/test/MC/AMDGPU/gfx12_asm_vop3p.s
index 7a4687b34f8f8fde2e79e9b5f637dd06f7cfa864..9a21f7a2eb5606c0b3e0995ab6f7188d15ea06ce 100644
--- a/llvm/test/MC/AMDGPU/gfx12_asm_vop3p.s
+++ b/llvm/test/MC/AMDGPU/gfx12_asm_vop3p.s
@@ -1250,3 +1250,93 @@ v_pk_sub_u16 v5, src_scc, vcc_lo op_sel:[1,0] op_sel_hi:[0,1]
 
 v_pk_sub_u16 v255, 0xfe0b, vcc_hi op_sel:[0,1] op_sel_hi:[1,0] clamp
 // GFX12: [0xff,0xd0,0x0b,0xcc,0xff,0xd6,0x00,0x08,0x0b,0xfe,0x00,0x00]
+
+v_pk_minimum_f16 v5, v1, v2
+// GFX12: [0x05,0x40,0x1d,0xcc,0x01,0x05,0x02,0x18]
+
+v_pk_minimum_f16 v5, v255, v255
+// GFX12: [0x05,0x40,0x1d,0xcc,0xff,0xff,0x03,0x18]
+
+v_pk_minimum_f16 v5, s1, s2
+// GFX12: [0x05,0x40,0x1d,0xcc,0x01,0x04,0x00,0x18]
+
+v_pk_minimum_f16 v5, s105, s105
+// GFX12: [0x05,0x40,0x1d,0xcc,0x69,0xd2,0x00,0x18]
+
+v_pk_minimum_f16 v5, vcc_lo, ttmp15
+// GFX12: [0x05,0x40,0x1d,0xcc,0x6a,0xf6,0x00,0x18]
+
+v_pk_minimum_f16 v5, vcc_hi, 0xfe0b
+// GFX12: [0x05,0x40,0x1d,0xcc,0x6b,0xfe,0x01,0x18,0x0b,0xfe,0x00,0x00]
+
+v_pk_minimum_f16 v5, ttmp15, src_scc
+// GFX12: [0x05,0x40,0x1d,0xcc,0x7b,0xfa,0x01,0x18]
+
+v_pk_minimum_f16 v5, m0, 0.5
+// GFX12: [0x05,0x40,0x1d,0xcc,0x7d,0xe0,0x01,0x18]
+
+v_pk_minimum_f16 v5, exec_lo, -1
+// GFX12: [0x05,0x40,0x1d,0xcc,0x7e,0x82,0x01,0x18]
+
+v_pk_minimum_f16 v5, exec_hi, null
+// GFX12: [0x05,0x40,0x1d,0xcc,0x7f,0xf8,0x00,0x18]
+
+v_pk_minimum_f16 v5, null, exec_lo
+// GFX12: [0x05,0x40,0x1d,0xcc,0x7c,0xfc,0x00,0x18]
+
+v_pk_minimum_f16 v5, -1, exec_hi op_sel:[1,1] op_sel_hi:[0,0] neg_lo:[1,0] neg_hi:[1,0]
+// GFX12: [0x05,0x59,0x1d,0xcc,0xc1,0xfe,0x00,0x20]
+
+v_pk_minimum_f16 v5, 0.5, m0 op_sel:[0,0] op_sel_hi:[1,1] neg_lo:[0,1] neg_hi:[0,1]
+// GFX12: [0x05,0x42,0x1d,0xcc,0xf0,0xfa,0x00,0x58]
+
+v_pk_minimum_f16 v5, src_scc, vcc_lo op_sel:[1,0] op_sel_hi:[0,1] neg_lo:[0,0] neg_hi:[0,0]
+// GFX12: [0x05,0x48,0x1d,0xcc,0xfd,0xd4,0x00,0x10]
+
+v_pk_minimum_f16 v255, 0xfe0b, vcc_hi op_sel:[0,1] op_sel_hi:[1,0] neg_lo:[1,1] neg_hi:[1,1] clamp
+// GFX12: [0xff,0xd3,0x1d,0xcc,0xff,0xd6,0x00,0x68,0x0b,0xfe,0x00,0x00]
+
+v_pk_maximum_f16 v5, v1, v2
+// GFX12: [0x05,0x40,0x1e,0xcc,0x01,0x05,0x02,0x18]
+
+v_pk_maximum_f16 v5, v255, v255
+// GFX12: [0x05,0x40,0x1e,0xcc,0xff,0xff,0x03,0x18]
+
+v_pk_maximum_f16 v5, s1, s2
+// GFX12: [0x05,0x40,0x1e,0xcc,0x01,0x04,0x00,0x18]
+
+v_pk_maximum_f16 v5, s105, s105
+// GFX12: [0x05,0x40,0x1e,0xcc,0x69,0xd2,0x00,0x18]
+
+v_pk_maximum_f16 v5, vcc_lo, ttmp15
+// GFX12: [0x05,0x40,0x1e,0xcc,0x6a,0xf6,0x00,0x18]
+
+v_pk_maximum_f16 v5, vcc_hi, 0xfe0b
+// GFX12: [0x05,0x40,0x1e,0xcc,0x6b,0xfe,0x01,0x18,0x0b,0xfe,0x00,0x00]
+
+v_pk_maximum_f16 v5, ttmp15, src_scc
+// GFX12: [0x05,0x40,0x1e,0xcc,0x7b,0xfa,0x01,0x18]
+
+v_pk_maximum_f16 v5, m0, 0.5
+// GFX12: [0x05,0x40,0x1e,0xcc,0x7d,0xe0,0x01,0x18]
+
+v_pk_maximum_f16 v5, exec_lo, -1
+// GFX12: [0x05,0x40,0x1e,0xcc,0x7e,0x82,0x01,0x18]
+
+v_pk_maximum_f16 v5, exec_hi, null
+// GFX12: [0x05,0x40,0x1e,0xcc,0x7f,0xf8,0x00,0x18]
+
+v_pk_maximum_f16 v5, null, exec_lo
+// GFX12: [0x05,0x40,0x1e,0xcc,0x7c,0xfc,0x00,0x18]
+
+v_pk_maximum_f16 v5, -1, exec_hi op_sel:[1,1] op_sel_hi:[0,0] neg_lo:[1,0] neg_hi:[1,0]
+// GFX12: [0x05,0x59,0x1e,0xcc,0xc1,0xfe,0x00,0x20]
+
+v_pk_maximum_f16 v5, 0.5, m0 op_sel:[0,0] op_sel_hi:[1,1] neg_lo:[0,1] neg_hi:[0,1]
+// GFX12: [0x05,0x42,0x1e,0xcc,0xf0,0xfa,0x00,0x58]
+
+v_pk_maximum_f16 v5, src_scc, vcc_lo op_sel:[1,0] op_sel_hi:[0,1] neg_lo:[0,0] neg_hi:[0,0]
+// GFX12: [0x05,0x48,0x1e,0xcc,0xfd,0xd4,0x00,0x10]
+
+v_pk_maximum_f16 v255, 0xfe0b, vcc_hi op_sel:[0,1] op_sel_hi:[1,0] neg_lo:[1,1] neg_hi:[1,1] clamp
+// GFX12: [0xff,0xd3,0x1e,0xcc,0xff,0xd6,0x00,0x68,0x0b,0xfe,0x00,0x00]
diff --git a/llvm/test/MC/AMDGPU/gfx12_unsupported.s b/llvm/test/MC/AMDGPU/gfx12_unsupported.s
index c9eb29c18a1a607124e792409d6bbcb12a7a9bb0..e835710a69fe851a6cbbd310dc1bbe7bb4e703c4 100644
--- a/llvm/test/MC/AMDGPU/gfx12_unsupported.s
+++ b/llvm/test/MC/AMDGPU/gfx12_unsupported.s
@@ -9,3 +9,75 @@ s_subvector_loop_begin s0, 0x1234
 
 s_subvector_loop_end s0, 0x1234
 // CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+ds_cmpstore_f32 v0, v1, v2
+// CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+ds_cmpstore_rtn_f32 v0, v1, v2, v3
+// CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+ds_cmpstore_f64 v0, v[1:2], v[3:4]
+// CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+ds_cmpstore_rtn_f64 v[0:1], v2, v[3:4], v[5:6]
+// CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+ds_add_gs_reg_rtn v[0:1], v2 gds
+// CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+ds_sub_gs_reg_rtn v[0:1], v2 gds
+// CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+ds_wrap_rtn_b32 v0, v1, v2, v3
+// CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+ds_gws_sema_release_all gds
+// CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+ds_gws_init v0 gds
+// CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+ds_gws_sema_v gds
+// CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+ds_gws_sema_br v0 gds
+// CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+ds_gws_sema_p gds
+// CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+ds_gws_barrier v0 gds
+// CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+ds_ordered_count v0, v1 gds
+// CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+buffer_atomic_cmpswap_f32 v[5:6], off, s[96:99], s3
+// CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+buffer_gl0_inv
+// CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+buffer_gl1_inv
+// CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+buffer_wbinvl1
+// CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+buffer_load_lds_b32 off, s[8:11], s3
+// CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+buffer_load_lds_format_x off, s[8:11], s3
+// CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+buffer_load_lds_i8 off, s[8:11], s3
+// CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+buffer_load_lds_i16 off, s[8:11], s3
+// CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+buffer_load_lds_u8 off, s[8:11], s3
+// CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
+
+buffer_load_lds_u16 off, s[8:11], s3
+// CHECK: :[[@LINE-1]]:{{[0-9]+}}: error: instruction not supported on this GPU
diff --git a/llvm/test/MC/AMDGPU/hsa-gfx12-v4.s b/llvm/test/MC/AMDGPU/hsa-gfx12-v4.s
new file mode 100644
index 0000000000000000000000000000000000000000..efbcec21f586b96c621a4127ba23c9e538976dbb
--- /dev/null
+++ b/llvm/test/MC/AMDGPU/hsa-gfx12-v4.s
@@ -0,0 +1,294 @@
+// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx1200 --amdhsa-code-object-version=4 < %s | FileCheck --check-prefix=ASM %s
+// RUN: llvm-mc -triple amdgcn-amd-amdhsa -mcpu=gfx1200 --amdhsa-code-object-version=4 -filetype=obj < %s > %t
+// RUN: llvm-readelf -S -r -s %t | FileCheck --check-prefix=READOBJ %s
+// RUN: llvm-objdump -s -j .rodata %t | FileCheck --check-prefix=OBJDUMP %s
+
+// READOBJ: Section Headers
+// READOBJ: .text   PROGBITS {{[0-9a-f]+}} {{[0-9a-f]+}} {{[0-9a-f]+}} {{[0-9]+}} AX {{[0-9]+}} {{[0-9]+}} 256
+// READOBJ: .rodata PROGBITS {{[0-9a-f]+}} {{[0-9a-f]+}}        000100 {{[0-9]+}}  A {{[0-9]+}} {{[0-9]+}} 64
+
+// READOBJ: Relocation section '.rela.rodata' at offset
+// READOBJ: 0000000000000010 {{[0-9a-f]+}}00000005 R_AMDGPU_REL64 0000000000000000 .text + 10
+// READOBJ: 0000000000000050 {{[0-9a-f]+}}00000005 R_AMDGPU_REL64 0000000000000000 .text + 110
+// READOBJ: 0000000000000090 {{[0-9a-f]+}}00000005 R_AMDGPU_REL64 0000000000000000 .text + 210
+// READOBJ: 00000000000000d0 {{[0-9a-f]+}}00000005 R_AMDGPU_REL64 0000000000000000 .text + 310
+
+// READOBJ: Symbol table '.symtab' contains {{[0-9]+}} entries:
+// READOBJ:      0000000000000000  0 FUNC    LOCAL  PROTECTED 2 minimal
+// READOBJ-NEXT: 0000000000000100  0 FUNC    LOCAL  PROTECTED 2 complete
+// READOBJ-NEXT: 0000000000000200  0 FUNC    LOCAL  PROTECTED 2 special_sgpr
+// READOBJ-NEXT: 0000000000000300  0 FUNC    LOCAL  PROTECTED 2 disabled_user_sgpr
+// READOBJ-NEXT: 0000000000000000 64 OBJECT  LOCAL  DEFAULT   3 minimal.kd
+// READOBJ-NEXT: 0000000000000040 64 OBJECT  LOCAL  DEFAULT   3 complete.kd
+// READOBJ-NEXT: 0000000000000080 64 OBJECT  LOCAL  DEFAULT   3 special_sgpr.kd
+// READOBJ-NEXT: 00000000000000c0 64 OBJECT  LOCAL  DEFAULT   3 disabled_user_sgpr.kd
+
+// OBJDUMP: Contents of section .rodata
+// Note, relocation for KERNEL_CODE_ENTRY_BYTE_OFFSET is not resolved here.
+// minimal
+// OBJDUMP-NEXT: 0000 00000000 00000000 00000000 00000000
+// OBJDUMP-NEXT: 0010 00000000 00000000 00000000 00000000
+// OBJDUMP-NEXT: 0020 00000000 00000000 00000000 00000000
+// OBJDUMP-NEXT: 0030 00000c60 80000000 00000000 00000000
+// complete
+// OBJDUMP-NEXT: 0040 01000000 01000000 08000000 00000000
+// OBJDUMP-NEXT: 0050 00000000 00000000 00000000 00000000
+// OBJDUMP-NEXT: 0060 00000000 00000000 00000000 00000000
+// OBJDUMP-NEXT: 0070 015021e4 1f0f007f 5e040000 00000000
+// special_sgpr
+// OBJDUMP-NEXT: 0080 00000000 00000000 00000000 00000000
+// OBJDUMP-NEXT: 0090 00000000 00000000 00000000 00000000
+// OBJDUMP-NEXT: 00a0 00000000 00000000 00000000 00000000
+// OBJDUMP-NEXT: 00b0 00000060 80000000 00000000 00000000
+// disabled_user_sgpr
+// OBJDUMP-NEXT: 00c0 00000000 00000000 00000000 00000000
+// OBJDUMP-NEXT: 00d0 00000000 00000000 00000000 00000000
+// OBJDUMP-NEXT: 00e0 00000000 00000000 00000000 00000000
+// OBJDUMP-NEXT: 00f0 00000c60 80000000 00000000 00000000
+
+.text
+// ASM: .text
+
+.amdgcn_target "amdgcn-amd-amdhsa--gfx1200"
+// ASM: .amdgcn_target "amdgcn-amd-amdhsa--gfx1200"
+
+.p2align 8
+.type minimal,@function
+minimal:
+  s_endpgm
+
+.p2align 8
+.type complete,@function
+complete:
+  s_endpgm
+
+.p2align 8
+.type special_sgpr,@function
+special_sgpr:
+  s_endpgm
+
+.p2align 8
+.type disabled_user_sgpr,@function
+disabled_user_sgpr:
+  s_endpgm
+
+.rodata
+// ASM: .rodata
+
+// Test that only specifying required directives is allowed, and that defaulted
+// values are omitted.
+.p2align 6
+.amdhsa_kernel minimal
+  .amdhsa_next_free_vgpr 0
+  .amdhsa_next_free_sgpr 0
+.end_amdhsa_kernel
+
+// ASM: .amdhsa_kernel minimal
+// ASM: .amdhsa_next_free_vgpr 0
+// ASM-NEXT: .amdhsa_next_free_sgpr 0
+// ASM: .end_amdhsa_kernel
+
+// Test that we can specify all available directives with non-default values.
+.p2align 6
+.amdhsa_kernel complete
+  .amdhsa_group_segment_fixed_size 1
+  .amdhsa_private_segment_fixed_size 1
+  .amdhsa_kernarg_size 8
+  .amdhsa_user_sgpr_count 15
+  .amdhsa_user_sgpr_dispatch_ptr 1
+  .amdhsa_user_sgpr_queue_ptr 1
+  .amdhsa_user_sgpr_kernarg_segment_ptr 1
+  .amdhsa_user_sgpr_dispatch_id 1
+  .amdhsa_user_sgpr_private_segment_size 1
+  .amdhsa_wavefront_size32 1
+  .amdhsa_enable_private_segment 1
+  .amdhsa_system_sgpr_workgroup_id_x 0
+  .amdhsa_system_sgpr_workgroup_id_y 1
+  .amdhsa_system_sgpr_workgroup_id_z 1
+  .amdhsa_system_sgpr_workgroup_info 1
+  .amdhsa_system_vgpr_workitem_id 1
+  .amdhsa_next_free_vgpr 9
+  .amdhsa_next_free_sgpr 27
+  .amdhsa_reserve_vcc 0
+  .amdhsa_float_round_mode_32 1
+  .amdhsa_float_round_mode_16_64 1
+  .amdhsa_float_denorm_mode_32 1
+  .amdhsa_float_denorm_mode_16_64 0
+  .amdhsa_fp16_overflow 1
+  .amdhsa_workgroup_processor_mode 1
+  .amdhsa_memory_ordered 1
+  .amdhsa_forward_progress 1
+  .amdhsa_shared_vgpr_count 0
+  .amdhsa_round_robin_scheduling 1
+  .amdhsa_exception_fp_ieee_invalid_op 1
+  .amdhsa_exception_fp_denorm_src 1
+  .amdhsa_exception_fp_ieee_div_zero 1
+  .amdhsa_exception_fp_ieee_overflow 1
+  .amdhsa_exception_fp_ieee_underflow 1
+  .amdhsa_exception_fp_ieee_inexact 1
+  .amdhsa_exception_int_div_zero 1
+.end_amdhsa_kernel
+
+// ASM: .amdhsa_kernel complete
+// ASM-NEXT: .amdhsa_group_segment_fixed_size 1
+// ASM-NEXT: .amdhsa_private_segment_fixed_size 1
+// ASM-NEXT: .amdhsa_kernarg_size 8
+// ASM-NEXT: .amdhsa_user_sgpr_count 15
+// ASM-NEXT: .amdhsa_user_sgpr_dispatch_ptr 1
+// ASM-NEXT: .amdhsa_user_sgpr_queue_ptr 1
+// ASM-NEXT: .amdhsa_user_sgpr_kernarg_segment_ptr 1
+// ASM-NEXT: .amdhsa_user_sgpr_dispatch_id 1
+// ASM-NEXT: .amdhsa_user_sgpr_private_segment_size 1
+// ASM-NEXT: .amdhsa_wavefront_size32 1
+// ASM-NEXT: .amdhsa_enable_private_segment 1
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_x 0
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_y 1
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_id_z 1
+// ASM-NEXT: .amdhsa_system_sgpr_workgroup_info 1
+// ASM-NEXT: .amdhsa_system_vgpr_workitem_id 1
+// ASM-NEXT: .amdhsa_next_free_vgpr 9
+// ASM-NEXT: .amdhsa_next_free_sgpr 27
+// ASM-NEXT: .amdhsa_reserve_vcc 0
+// ASM-NEXT: .amdhsa_float_round_mode_32 1
+// ASM-NEXT: .amdhsa_float_round_mode_16_64 1
+// ASM-NEXT: .amdhsa_float_denorm_mode_32 1
+// ASM-NEXT: .amdhsa_float_denorm_mode_16_64 0
+// ASM-NEXT: .amdhsa_fp16_overflow 1
+// ASM-NEXT: .amdhsa_workgroup_processor_mode 1
+// ASM-NEXT: .amdhsa_memory_ordered 1
+// ASM-NEXT: .amdhsa_forward_progress 1
+// ASM-NEXT: .amdhsa_shared_vgpr_count 0
+// ASM-NEXT: .amdhsa_round_robin_scheduling 1
+// ASM-NEXT: .amdhsa_exception_fp_ieee_invalid_op 1
+// ASM-NEXT: .amdhsa_exception_fp_denorm_src 1
+// ASM-NEXT: .amdhsa_exception_fp_ieee_div_zero 1
+// ASM-NEXT: .amdhsa_exception_fp_ieee_overflow 1
+// ASM-NEXT: .amdhsa_exception_fp_ieee_underflow 1
+// ASM-NEXT: .amdhsa_exception_fp_ieee_inexact 1
+// ASM-NEXT: .amdhsa_exception_int_div_zero 1
+// ASM-NEXT: .end_amdhsa_kernel
+
+// Test that we are including special SGPR usage in the granulated count.
+.p2align 6
+.amdhsa_kernel special_sgpr
+  .amdhsa_next_free_sgpr 27
+
+  .amdhsa_reserve_vcc 0
+
+  .amdhsa_float_denorm_mode_16_64 0
+  .amdhsa_next_free_vgpr 0
+.end_amdhsa_kernel
+
+// ASM: .amdhsa_kernel special_sgpr
+// ASM: .amdhsa_next_free_vgpr 0
+// ASM-NEXT: .amdhsa_next_free_sgpr 27
+// ASM-NEXT: .amdhsa_reserve_vcc 0
+// ASM: .amdhsa_float_denorm_mode_16_64 0
+// ASM: .end_amdhsa_kernel
+
+// Test that explicitly disabling user_sgpr's does not affect the user_sgpr
+// count, i.e. this should produce the same descriptor as minimal.
+.p2align 6
+.amdhsa_kernel disabled_user_sgpr
+  .amdhsa_next_free_vgpr 0
+  .amdhsa_next_free_sgpr 0
+.end_amdhsa_kernel
+
+// ASM: .amdhsa_kernel disabled_user_sgpr
+// ASM: .amdhsa_next_free_vgpr 0
+// ASM-NEXT: .amdhsa_next_free_sgpr 0
+// ASM: .end_amdhsa_kernel
+
+.section .foo
+
+.byte .amdgcn.gfx_generation_number
+// ASM: .byte 12
+
+.byte .amdgcn.gfx_generation_minor
+// ASM: .byte 0
+
+.byte .amdgcn.gfx_generation_stepping
+// ASM: .byte 0
+
+.byte .amdgcn.next_free_vgpr
+// ASM: .byte 0
+.byte .amdgcn.next_free_sgpr
+// ASM: .byte 0
+
+v_mov_b32_e32 v7, s10
+
+.byte .amdgcn.next_free_vgpr
+// ASM: .byte 8
+.byte .amdgcn.next_free_sgpr
+// ASM: .byte 11
+
+.set .amdgcn.next_free_vgpr, 0
+.set .amdgcn.next_free_sgpr, 0
+
+.byte .amdgcn.next_free_vgpr
+// ASM: .byte 0
+.byte .amdgcn.next_free_sgpr
+// ASM: .byte 0
+
+v_mov_b32_e32 v16, s3
+
+.byte .amdgcn.next_free_vgpr
+// ASM: .byte 17
+.byte .amdgcn.next_free_sgpr
+// ASM: .byte 4
+
+// Metadata
+
+.amdgpu_metadata
+  amdhsa.version:
+    - 3
+    - 0
+  amdhsa.kernels:
+    - .name:       amd_kernel_code_t_test_all
+      .symbol: amd_kernel_code_t_test_all@kd
+      .kernarg_segment_size: 8
+      .group_segment_fixed_size: 16
+      .private_segment_fixed_size: 32
+      .kernarg_segment_align: 64
+      .wavefront_size: 128
+      .sgpr_count: 14
+      .vgpr_count: 40
+      .max_flat_workgroup_size: 256
+    - .name:       amd_kernel_code_t_minimal
+      .symbol: amd_kernel_code_t_minimal@kd
+      .kernarg_segment_size: 8
+      .group_segment_fixed_size: 16
+      .private_segment_fixed_size: 32
+      .kernarg_segment_align: 64
+      .wavefront_size: 128
+      .sgpr_count: 14
+      .vgpr_count: 40
+      .max_flat_workgroup_size: 256
+.end_amdgpu_metadata
+
+// ASM:      	.amdgpu_metadata
+// ASM:      amdhsa.kernels:
+// ASM:        - .group_segment_fixed_size: 16
+// ASM:          .kernarg_segment_align: 64
+// ASM:          .kernarg_segment_size: 8
+// ASM:          .max_flat_workgroup_size: 256
+// ASM:          .name:           amd_kernel_code_t_test_all
+// ASM:          .private_segment_fixed_size: 32
+// ASM:          .sgpr_count:     14
+// ASM:          .symbol:         'amd_kernel_code_t_test_all@kd'
+// ASM:          .vgpr_count:     40
+// ASM:          .wavefront_size: 128
+// ASM:        - .group_segment_fixed_size: 16
+// ASM:          .kernarg_segment_align: 64
+// ASM:          .kernarg_segment_size: 8
+// ASM:          .max_flat_workgroup_size: 256
+// ASM:          .name:           amd_kernel_code_t_minimal
+// ASM:          .private_segment_fixed_size: 32
+// ASM:          .sgpr_count:     14
+// ASM:          .symbol:         'amd_kernel_code_t_minimal@kd'
+// ASM:          .vgpr_count:     40
+// ASM:          .wavefront_size: 128
+// ASM:      amdhsa.version:
+// ASM-NEXT:   - 3
+// ASM-NEXT:   - 0
+// ASM:      	.end_amdgpu_metadata
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_ds.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_ds.txt
new file mode 100644
index 0000000000000000000000000000000000000000..d3c0e714949907e62e8c959ae83c0b6e4e074f92
--- /dev/null
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_ds.txt
@@ -0,0 +1,3070 @@
+# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -strict-whitespace -check-prefix=GFX12 %s
+
+# GFX12: ds_add_f32 v0, v1                       ; encoding: [0x00,0x00,0x54,0xd8,0x00,0x01,0x00,0x00]
+0x00,0x00,0x54,0xd8,0x00,0x01,0x00,0x00
+
+# GFX12: ds_add_f32 v0, v1 offset:4660           ; encoding: [0x34,0x12,0x54,0xd8,0x00,0x01,0x00,0x00]
+0x34,0x12,0x54,0xd8,0x00,0x01,0x00,0x00
+
+# GFX12: ds_add_f32 v0, v1 offset:65535          ; encoding: [0xff,0xff,0x54,0xd8,0x00,0x01,0x00,0x00]
+0xff,0xff,0x54,0xd8,0x00,0x01,0x00,0x00
+
+# GFX12: ds_add_f32 v0, v254                     ; encoding: [0x00,0x00,0x54,0xd8,0x00,0xfe,0x00,0x00]
+0x00,0x00,0x54,0xd8,0x00,0xfe,0x00,0x00
+
+# GFX12: ds_add_f32 v0, v254 offset:4660         ; encoding: [0x34,0x12,0x54,0xd8,0x00,0xfe,0x00,0x00]
+0x34,0x12,0x54,0xd8,0x00,0xfe,0x00,0x00
+
+# GFX12: ds_add_f32 v0, v254 offset:65535        ; encoding: [0xff,0xff,0x54,0xd8,0x00,0xfe,0x00,0x00]
+0xff,0xff,0x54,0xd8,0x00,0xfe,0x00,0x00
+
+# GFX12: ds_add_f32 v255, v1                     ; encoding: [0x00,0x00,0x54,0xd8,0xff,0x01,0x00,0x00]
+0x00,0x00,0x54,0xd8,0xff,0x01,0x00,0x00
+
+# GFX12: ds_add_f32 v255, v1 offset:4660         ; encoding: [0x34,0x12,0x54,0xd8,0xff,0x01,0x00,0x00]
+0x34,0x12,0x54,0xd8,0xff,0x01,0x00,0x00
+
+# GFX12: ds_add_f32 v255, v1 offset:65535        ; encoding: [0xff,0xff,0x54,0xd8,0xff,0x01,0x00,0x00]
+0xff,0xff,0x54,0xd8,0xff,0x01,0x00,0x00
+
+# GFX12: ds_add_f32 v255, v254                   ; encoding: [0x00,0x00,0x54,0xd8,0xff,0xfe,0x00,0x00]
+0x00,0x00,0x54,0xd8,0xff,0xfe,0x00,0x00
+
+# GFX12: ds_add_f32 v255, v254 offset:4660       ; encoding: [0x34,0x12,0x54,0xd8,0xff,0xfe,0x00,0x00]
+0x34,0x12,0x54,0xd8,0xff,0xfe,0x00,0x00
+
+# GFX12: ds_add_f32 v255, v254 offset:65535      ; encoding: [0xff,0xff,0x54,0xd8,0xff,0xfe,0x00,0x00]
+0xff,0xff,0x54,0xd8,0xff,0xfe,0x00,0x00
+
+# GFX12: ds_add_rtn_f32 v0, v1, v2               ; encoding: [0x00,0x00,0xe4,0xd9,0x01,0x02,0x00,0x00]
+0x00,0x00,0xe4,0xd9,0x01,0x02,0x00,0x00
+
+# GFX12: ds_add_rtn_f32 v0, v1, v2 offset:4660   ; encoding: [0x34,0x12,0xe4,0xd9,0x01,0x02,0x00,0x00]
+0x34,0x12,0xe4,0xd9,0x01,0x02,0x00,0x00
+
+# GFX12: ds_add_rtn_f32 v0, v1, v2 offset:65535  ; encoding: [0xff,0xff,0xe4,0xd9,0x01,0x02,0x00,0x00]
+0xff,0xff,0xe4,0xd9,0x01,0x02,0x00,0x00
+
+# GFX12: ds_add_rtn_f32 v0, v254, v253           ; encoding: [0x00,0x00,0xe4,0xd9,0xfe,0xfd,0x00,0x00]
+0x00,0x00,0xe4,0xd9,0xfe,0xfd,0x00,0x00
+
+# GFX12: ds_add_rtn_f32 v0, v254, v253 offset:4660 ; encoding: [0x34,0x12,0xe4,0xd9,0xfe,0xfd,0x00,0x00]
+0x34,0x12,0xe4,0xd9,0xfe,0xfd,0x00,0x00
+
+# GFX12: ds_add_rtn_f32 v0, v254, v253 offset:65535 ; encoding: [0xff,0xff,0xe4,0xd9,0xfe,0xfd,0x00,0x00]
+0xff,0xff,0xe4,0xd9,0xfe,0xfd,0x00,0x00
+
+# GFX12: ds_add_rtn_f32 v255, v1, v253           ; encoding: [0x00,0x00,0xe4,0xd9,0x01,0xfd,0x00,0xff]
+0x00,0x00,0xe4,0xd9,0x01,0xfd,0x00,0xff
+
+# GFX12: ds_add_rtn_f32 v255, v1, v253 offset:4660 ; encoding: [0x34,0x12,0xe4,0xd9,0x01,0xfd,0x00,0xff]
+0x34,0x12,0xe4,0xd9,0x01,0xfd,0x00,0xff
+
+# GFX12: ds_add_rtn_f32 v255, v1, v253 offset:65535 ; encoding: [0xff,0xff,0xe4,0xd9,0x01,0xfd,0x00,0xff]
+0xff,0xff,0xe4,0xd9,0x01,0xfd,0x00,0xff
+
+# GFX12: ds_add_rtn_f32 v255, v254, v2           ; encoding: [0x00,0x00,0xe4,0xd9,0xfe,0x02,0x00,0xff]
+0x00,0x00,0xe4,0xd9,0xfe,0x02,0x00,0xff
+
+# GFX12: ds_add_rtn_f32 v255, v254, v2 offset:4660 ; encoding: [0x34,0x12,0xe4,0xd9,0xfe,0x02,0x00,0xff]
+0x34,0x12,0xe4,0xd9,0xfe,0x02,0x00,0xff
+
+# GFX12: ds_add_rtn_f32 v255, v254, v2 offset:65535 ; encoding: [0xff,0xff,0xe4,0xd9,0xfe,0x02,0x00,0xff]
+0xff,0xff,0xe4,0xd9,0xfe,0x02,0x00,0xff
+
+# GFX12: ds_add_rtn_f32 v255, v254, v253         ; encoding: [0x00,0x00,0xe4,0xd9,0xfe,0xfd,0x00,0xff]
+0x00,0x00,0xe4,0xd9,0xfe,0xfd,0x00,0xff
+
+# GFX12: ds_add_rtn_f32 v255, v254, v253 offset:4660 ; encoding: [0x34,0x12,0xe4,0xd9,0xfe,0xfd,0x00,0xff]
+0x34,0x12,0xe4,0xd9,0xfe,0xfd,0x00,0xff
+
+# GFX12: ds_add_rtn_f32 v255, v254, v253 offset:65535 ; encoding: [0xff,0xff,0xe4,0xd9,0xfe,0xfd,0x00,0xff]
+0xff,0xff,0xe4,0xd9,0xfe,0xfd,0x00,0xff
+
+# GFX12: ds_add_rtn_u32 v0, v1, v2               ; encoding: [0x00,0x00,0x80,0xd8,0x01,0x02,0x00,0x00]
+0x00,0x00,0x80,0xd8,0x01,0x02,0x00,0x00
+
+# GFX12: ds_add_rtn_u32 v0, v1, v2 offset:4660   ; encoding: [0x34,0x12,0x80,0xd8,0x01,0x02,0x00,0x00]
+0x34,0x12,0x80,0xd8,0x01,0x02,0x00,0x00
+
+# GFX12: ds_add_rtn_u32 v0, v1, v2 offset:65535  ; encoding: [0xff,0xff,0x80,0xd8,0x01,0x02,0x00,0x00]
+0xff,0xff,0x80,0xd8,0x01,0x02,0x00,0x00
+
+# GFX12: ds_add_rtn_u32 v0, v254, v253           ; encoding: [0x00,0x00,0x80,0xd8,0xfe,0xfd,0x00,0x00]
+0x00,0x00,0x80,0xd8,0xfe,0xfd,0x00,0x00
+
+# GFX12: ds_add_rtn_u32 v0, v254, v253 offset:4660 ; encoding: [0x34,0x12,0x80,0xd8,0xfe,0xfd,0x00,0x00]
+0x34,0x12,0x80,0xd8,0xfe,0xfd,0x00,0x00
+
+# GFX12: ds_add_rtn_u32 v0, v254, v253 offset:65535 ; encoding: [0xff,0xff,0x80,0xd8,0xfe,0xfd,0x00,0x00]
+0xff,0xff,0x80,0xd8,0xfe,0xfd,0x00,0x00
+
+# GFX12: ds_add_rtn_u32 v255, v1, v253           ; encoding: [0x00,0x00,0x80,0xd8,0x01,0xfd,0x00,0xff]
+0x00,0x00,0x80,0xd8,0x01,0xfd,0x00,0xff
+
+# GFX12: ds_add_rtn_u32 v255, v1, v253 offset:4660 ; encoding: [0x34,0x12,0x80,0xd8,0x01,0xfd,0x00,0xff]
+0x34,0x12,0x80,0xd8,0x01,0xfd,0x00,0xff
+
+# GFX12: ds_add_rtn_u32 v255, v1, v253 offset:65535 ; encoding: [0xff,0xff,0x80,0xd8,0x01,0xfd,0x00,0xff]
+0xff,0xff,0x80,0xd8,0x01,0xfd,0x00,0xff
+
+# GFX12: ds_add_rtn_u32 v255, v254, v2           ; encoding: [0x00,0x00,0x80,0xd8,0xfe,0x02,0x00,0xff]
+0x00,0x00,0x80,0xd8,0xfe,0x02,0x00,0xff
+
+# GFX12: ds_add_rtn_u32 v255, v254, v2 offset:4660 ; encoding: [0x34,0x12,0x80,0xd8,0xfe,0x02,0x00,0xff]
+0x34,0x12,0x80,0xd8,0xfe,0x02,0x00,0xff
+
+# GFX12: ds_add_rtn_u32 v255, v254, v2 offset:65535 ; encoding: [0xff,0xff,0x80,0xd8,0xfe,0x02,0x00,0xff]
+0xff,0xff,0x80,0xd8,0xfe,0x02,0x00,0xff
+
+# GFX12: ds_add_rtn_u32 v255, v254, v253         ; encoding: [0x00,0x00,0x80,0xd8,0xfe,0xfd,0x00,0xff]
+0x00,0x00,0x80,0xd8,0xfe,0xfd,0x00,0xff
+
+# GFX12: ds_add_rtn_u32 v255, v254, v253 offset:4660 ; encoding: [0x34,0x12,0x80,0xd8,0xfe,0xfd,0x00,0xff]
+0x34,0x12,0x80,0xd8,0xfe,0xfd,0x00,0xff
+
+# GFX12: ds_add_rtn_u32 v255, v254, v253 offset:65535 ; encoding: [0xff,0xff,0x80,0xd8,0xfe,0xfd,0x00,0xff]
+0xff,0xff,0x80,0xd8,0xfe,0xfd,0x00,0xff
+
+# GFX12: ds_add_rtn_u64 v[254:255], v1, v[2:3] offset:65535 ; encoding: [0xff,0xff,0x80,0xd9,0x01,0x02,0x00,0xfe]
+0xff,0xff,0x80,0xd9,0x01,0x02,0x00,0xfe
+
+# GFX12: ds_add_rtn_u64 v[5:6], v1, v[254:255] offset:65535 ; encoding: [0xff,0xff,0x80,0xd9,0x01,0xfe,0x00,0x05]
+0xff,0xff,0x80,0xd9,0x01,0xfe,0x00,0x05
+
+# GFX12: ds_add_rtn_u64 v[5:6], v1, v[2:3]       ; encoding: [0x00,0x00,0x80,0xd9,0x01,0x02,0x00,0x05]
+0x00,0x00,0x80,0xd9,0x01,0x02,0x00,0x05
+
+# GFX12: ds_add_rtn_u64 v[5:6], v1, v[2:3] offset:4 ; encoding: [0x04,0x00,0x80,0xd9,0x01,0x02,0x00,0x05]
+0x04,0x00,0x80,0xd9,0x01,0x02,0x00,0x05
+
+# GFX12: ds_add_rtn_u64 v[5:6], v1, v[2:3] offset:65535 ; encoding: [0xff,0xff,0x80,0xd9,0x01,0x02,0x00,0x05]
+0xff,0xff,0x80,0xd9,0x01,0x02,0x00,0x05
+
+# GFX12: ds_add_rtn_u64 v[5:6], v255, v[2:3] offset:65535 ; encoding: [0xff,0xff,0x80,0xd9,0xff,0x02,0x00,0x05]
+0xff,0xff,0x80,0xd9,0xff,0x02,0x00,0x05
+
+# GFX12: ds_add_u32 v0, v1                       ; encoding: [0x00,0x00,0x00,0xd8,0x00,0x01,0x00,0x00]
+0x00,0x00,0x00,0xd8,0x00,0x01,0x00,0x00
+
+# GFX12: ds_add_u32 v0, v1 offset:4660           ; encoding: [0x34,0x12,0x00,0xd8,0x00,0x01,0x00,0x00]
+0x34,0x12,0x00,0xd8,0x00,0x01,0x00,0x00
+
+# GFX12: ds_add_u32 v0, v1 offset:65535          ; encoding: [0xff,0xff,0x00,0xd8,0x00,0x01,0x00,0x00]
+0xff,0xff,0x00,0xd8,0x00,0x01,0x00,0x00
+
+# GFX12: ds_add_u32 v0, v254                     ; encoding: [0x00,0x00,0x00,0xd8,0x00,0xfe,0x00,0x00]
+0x00,0x00,0x00,0xd8,0x00,0xfe,0x00,0x00
+
+# GFX12: ds_add_u32 v0, v254 offset:4660         ; encoding: [0x34,0x12,0x00,0xd8,0x00,0xfe,0x00,0x00]
+0x34,0x12,0x00,0xd8,0x00,0xfe,0x00,0x00
+
+# GFX12: ds_add_u32 v0, v254 offset:65535        ; encoding: [0xff,0xff,0x00,0xd8,0x00,0xfe,0x00,0x00]
+0xff,0xff,0x00,0xd8,0x00,0xfe,0x00,0x00
+
+# GFX12: ds_add_u32 v255, v1                     ; encoding: [0x00,0x00,0x00,0xd8,0xff,0x01,0x00,0x00]
+0x00,0x00,0x00,0xd8,0xff,0x01,0x00,0x00
+
+# GFX12: ds_add_u32 v255, v1 offset:4660         ; encoding: [0x34,0x12,0x00,0xd8,0xff,0x01,0x00,0x00]
+0x34,0x12,0x00,0xd8,0xff,0x01,0x00,0x00
+
+# GFX12: ds_add_u32 v255, v1 offset:65535        ; encoding: [0xff,0xff,0x00,0xd8,0xff,0x01,0x00,0x00]
+0xff,0xff,0x00,0xd8,0xff,0x01,0x00,0x00
+
+# GFX12: ds_add_u32 v255, v254                   ; encoding: [0x00,0x00,0x00,0xd8,0xff,0xfe,0x00,0x00]
+0x00,0x00,0x00,0xd8,0xff,0xfe,0x00,0x00
+
+# GFX12: ds_add_u32 v255, v254 offset:4660       ; encoding: [0x34,0x12,0x00,0xd8,0xff,0xfe,0x00,0x00]
+0x34,0x12,0x00,0xd8,0xff,0xfe,0x00,0x00
+
+# GFX12: ds_add_u32 v255, v254 offset:65535      ; encoding: [0xff,0xff,0x00,0xd8,0xff,0xfe,0x00,0x00]
+0xff,0xff,0x00,0xd8,0xff,0xfe,0x00,0x00
+
+# GFX12: ds_add_u64 v1, v[254:255] offset:65535  ; encoding: [0xff,0xff,0x00,0xd9,0x01,0xfe,0x00,0x00]
+0xff,0xff,0x00,0xd9,0x01,0xfe,0x00,0x00
+
+# GFX12: ds_add_u64 v1, v[2:3]                   ; encoding: [0x00,0x00,0x00,0xd9,0x01,0x02,0x00,0x00]
+0x00,0x00,0x00,0xd9,0x01,0x02,0x00,0x00
+
+# GFX12: ds_add_u64 v1, v[2:3] offset:4          ; encoding: [0x04,0x00,0x00,0xd9,0x01,0x02,0x00,0x00]
+0x04,0x00,0x00,0xd9,0x01,0x02,0x00,0x00
+
+# GFX12: ds_add_u64 v1, v[2:3] offset:65535      ; encoding: [0xff,0xff,0x00,0xd9,0x01,0x02,0x00,0x00]
+0xff,0xff,0x00,0xd9,0x01,0x02,0x00,0x00
+
+# GFX12: ds_add_u64 v255, v[2:3] offset:65535    ; encoding: [0xff,0xff,0x00,0xd9,0xff,0x02,0x00,0x00]
+0xff,0xff,0x00,0xd9,0xff,0x02,0x00,0x00
+
+# GFX12: ds_and_b32 v0, v1                       ; encoding: [0x00,0x00,0x24,0xd8,0x00,0x01,0x00,0x00]
+0x00,0x00,0x24,0xd8,0x00,0x01,0x00,0x00
+
+# GFX12: ds_and_b32 v0, v1 offset:4660           ; encoding: [0x34,0x12,0x24,0xd8,0x00,0x01,0x00,0x00]
+0x34,0x12,0x24,0xd8,0x00,0x01,0x00,0x00
+
+# GFX12: ds_and_b32 v0, v1 offset:65535          ; encoding: [0xff,0xff,0x24,0xd8,0x00,0x01,0x00,0x00]
+0xff,0xff,0x24,0xd8,0x00,0x01,0x00,0x00
+
+# GFX12: ds_and_b32 v0, v254                     ; encoding: [0x00,0x00,0x24,0xd8,0x00,0xfe,0x00,0x00]
+0x00,0x00,0x24,0xd8,0x00,0xfe,0x00,0x00
+
+# GFX12: ds_and_b32 v0, v254 offset:4660         ; encoding: [0x34,0x12,0x24,0xd8,0x00,0xfe,0x00,0x00]
+0x34,0x12,0x24,0xd8,0x00,0xfe,0x00,0x00
+
+# GFX12: ds_and_b32 v0, v254 offset:65535        ; encoding: [0xff,0xff,0x24,0xd8,0x00,0xfe,0x00,0x00]
+0xff,0xff,0x24,0xd8,0x00,0xfe,0x00,0x00
+
+# GFX12: ds_and_b32 v255, v1                     ; encoding: [0x00,0x00,0x24,0xd8,0xff,0x01,0x00,0x00]
+0x00,0x00,0x24,0xd8,0xff,0x01,0x00,0x00
+
+# GFX12: ds_and_b32 v255, v1 offset:4660         ; encoding: [0x34,0x12,0x24,0xd8,0xff,0x01,0x00,0x00]
+0x34,0x12,0x24,0xd8,0xff,0x01,0x00,0x00
+
+# GFX12: ds_and_b32 v255, v1 offset:65535        ; encoding: [0xff,0xff,0x24,0xd8,0xff,0x01,0x00,0x00]
+0xff,0xff,0x24,0xd8,0xff,0x01,0x00,0x00
+
+# GFX12: ds_and_b32 v255, v254                   ; encoding: [0x00,0x00,0x24,0xd8,0xff,0xfe,0x00,0x00]
+0x00,0x00,0x24,0xd8,0xff,0xfe,0x00,0x00
+
+# GFX12: ds_and_b32 v255, v254 offset:4660       ; encoding: [0x34,0x12,0x24,0xd8,0xff,0xfe,0x00,0x00]
+0x34,0x12,0x24,0xd8,0xff,0xfe,0x00,0x00
+
+# GFX12: ds_and_b32 v255, v254 offset:65535      ; encoding: [0xff,0xff,0x24,0xd8,0xff,0xfe,0x00,0x00]
+0xff,0xff,0x24,0xd8,0xff,0xfe,0x00,0x00
+
+# GFX12: ds_and_b64 v1, v[254:255] offset:65535  ; encoding: [0xff,0xff,0x24,0xd9,0x01,0xfe,0x00,0x00]
+0xff,0xff,0x24,0xd9,0x01,0xfe,0x00,0x00
+
+# GFX12: ds_and_b64 v1, v[2:3]                   ; encoding: [0x00,0x00,0x24,0xd9,0x01,0x02,0x00,0x00]
+0x00,0x00,0x24,0xd9,0x01,0x02,0x00,0x00
+
+# GFX12: ds_and_b64 v1, v[2:3] offset:4          ; encoding: [0x04,0x00,0x24,0xd9,0x01,0x02,0x00,0x00]
+0x04,0x00,0x24,0xd9,0x01,0x02,0x00,0x00
+
+# GFX12: ds_and_b64 v1, v[2:3] offset:65535      ; encoding: [0xff,0xff,0x24,0xd9,0x01,0x02,0x00,0x00]
+0xff,0xff,0x24,0xd9,0x01,0x02,0x00,0x00
+
+# GFX12: ds_and_b64 v255, v[2:3] offset:65535    ; encoding: [0xff,0xff,0x24,0xd9,0xff,0x02,0x00,0x00]
+0xff,0xff,0x24,0xd9,0xff,0x02,0x00,0x00
+
+# GFX12: ds_and_rtn_b32 v0, v1, v2               ; encoding: [0x00,0x00,0xa4,0xd8,0x01,0x02,0x00,0x00]
+0x00,0x00,0xa4,0xd8,0x01,0x02,0x00,0x00
+
+# GFX12: ds_and_rtn_b32 v0, v1, v2 offset:4660   ; encoding: [0x34,0x12,0xa4,0xd8,0x01,0x02,0x00,0x00]
+0x34,0x12,0xa4,0xd8,0x01,0x02,0x00,0x00
+
+# GFX12: ds_and_rtn_b32 v0, v1, v2 offset:65535  ; encoding: [0xff,0xff,0xa4,0xd8,0x01,0x02,0x00,0x00]
+0xff,0xff,0xa4,0xd8,0x01,0x02,0x00,0x00
+
+# GFX12: ds_and_rtn_b32 v0, v254, v253           ; encoding: [0x00,0x00,0xa4,0xd8,0xfe,0xfd,0x00,0x00]
+0x00,0x00,0xa4,0xd8,0xfe,0xfd,0x00,0x00
+
+# GFX12: ds_and_rtn_b32 v0, v254, v253 offset:4660 ; encoding: [0x34,0x12,0xa4,0xd8,0xfe,0xfd,0x00,0x00]
+0x34,0x12,0xa4,0xd8,0xfe,0xfd,0x00,0x00
+
+# GFX12: ds_and_rtn_b32 v0, v254, v253 offset:65535 ; encoding: [0xff,0xff,0xa4,0xd8,0xfe,0xfd,0x00,0x00]
+0xff,0xff,0xa4,0xd8,0xfe,0xfd,0x00,0x00
+
+# GFX12: ds_and_rtn_b32 v255, v1, v253           ; encoding: [0x00,0x00,0xa4,0xd8,0x01,0xfd,0x00,0xff]
+0x00,0x00,0xa4,0xd8,0x01,0xfd,0x00,0xff
+
+# GFX12: ds_and_rtn_b32 v255, v1, v253 offset:4660 ; encoding: [0x34,0x12,0xa4,0xd8,0x01,0xfd,0x00,0xff]
+0x34,0x12,0xa4,0xd8,0x01,0xfd,0x00,0xff
+
+# GFX12: ds_and_rtn_b32 v255, v1, v253 offset:65535 ; encoding: [0xff,0xff,0xa4,0xd8,0x01,0xfd,0x00,0xff]
+0xff,0xff,0xa4,0xd8,0x01,0xfd,0x00,0xff
+
+# GFX12: ds_and_rtn_b32 v255, v254, v2           ; encoding: [0x00,0x00,0xa4,0xd8,0xfe,0x02,0x00,0xff]
+0x00,0x00,0xa4,0xd8,0xfe,0x02,0x00,0xff
+
+# GFX12: ds_and_rtn_b32 v255, v254, v2 offset:4660 ; encoding: [0x34,0x12,0xa4,0xd8,0xfe,0x02,0x00,0xff]
+0x34,0x12,0xa4,0xd8,0xfe,0x02,0x00,0xff
+
+# GFX12: ds_and_rtn_b32 v255, v254, v2 offset:65535 ; encoding: [0xff,0xff,0xa4,0xd8,0xfe,0x02,0x00,0xff]
+0xff,0xff,0xa4,0xd8,0xfe,0x02,0x00,0xff
+
+# GFX12: ds_and_rtn_b32 v255, v254, v253         ; encoding: [0x00,0x00,0xa4,0xd8,0xfe,0xfd,0x00,0xff]
+0x00,0x00,0xa4,0xd8,0xfe,0xfd,0x00,0xff
+
+# GFX12: ds_and_rtn_b32 v255, v254, v253 offset:4660 ; encoding: [0x34,0x12,0xa4,0xd8,0xfe,0xfd,0x00,0xff]
+0x34,0x12,0xa4,0xd8,0xfe,0xfd,0x00,0xff
+
+# GFX12: ds_and_rtn_b32 v255, v254, v253 offset:65535 ; encoding: [0xff,0xff,0xa4,0xd8,0xfe,0xfd,0x00,0xff]
+0xff,0xff,0xa4,0xd8,0xfe,0xfd,0x00,0xff
+
+# GFX12: ds_and_rtn_b64 v[254:255], v1, v[2:3] offset:65535 ; encoding: [0xff,0xff,0xa4,0xd9,0x01,0x02,0x00,0xfe]
+0xff,0xff,0xa4,0xd9,0x01,0x02,0x00,0xfe
+
+# GFX12: ds_and_rtn_b64 v[5:6], v1, v[254:255] offset:65535 ; encoding: [0xff,0xff,0xa4,0xd9,0x01,0xfe,0x00,0x05]
+0xff,0xff,0xa4,0xd9,0x01,0xfe,0x00,0x05
+
+# GFX12: ds_and_rtn_b64 v[5:6], v1, v[2:3]       ; encoding: [0x00,0x00,0xa4,0xd9,0x01,0x02,0x00,0x05]
+0x00,0x00,0xa4,0xd9,0x01,0x02,0x00,0x05
+
+# GFX12: ds_and_rtn_b64 v[5:6], v1, v[2:3] offset:4 ; encoding: [0x04,0x00,0xa4,0xd9,0x01,0x02,0x00,0x05]
+0x04,0x00,0xa4,0xd9,0x01,0x02,0x00,0x05
+
+# GFX12: ds_and_rtn_b64 v[5:6], v1, v[2:3] offset:65535 ; encoding: [0xff,0xff,0xa4,0xd9,0x01,0x02,0x00,0x05]
+0xff,0xff,0xa4,0xd9,0x01,0x02,0x00,0x05
+
+# GFX12: ds_and_rtn_b64 v[5:6], v255, v[2:3] offset:65535 ; encoding: [0xff,0xff,0xa4,0xd9,0xff,0x02,0x00,0x05]
+0xff,0xff,0xa4,0xd9,0xff,0x02,0x00,0x05
+
+# GFX12: ds_append v255 offset:65535             ; encoding: [0xff,0xff,0xf8,0xd8,0x00,0x00,0x00,0xff]
+0xff,0xff,0xf8,0xd8,0x00,0x00,0x00,0xff
+
+# GFX12: ds_append v5                            ; encoding: [0x00,0x00,0xf8,0xd8,0x00,0x00,0x00,0x05]
+0x00,0x00,0xf8,0xd8,0x00,0x00,0x00,0x05
+
+# GFX12: ds_append v5 offset:4                   ; encoding: [0x04,0x00,0xf8,0xd8,0x00,0x00,0x00,0x05]
+0x04,0x00,0xf8,0xd8,0x00,0x00,0x00,0x05
+
+# GFX12: ds_append v5 offset:65535               ; encoding: [0xff,0xff,0xf8,0xd8,0x00,0x00,0x00,0x05]
+0xff,0xff,0xf8,0xd8,0x00,0x00,0x00,0x05
+
+# GFX12: ds_bpermute_b32 v0, v1, v2              ; encoding: [0x00,0x00,0xcc,0xda,0x01,0x02,0x00,0x00]
+0x00,0x00,0xcc,0xda,0x01,0x02,0x00,0x00
+
+# GFX12: ds_bpermute_b32 v0, v1, v2 offset:4660  ; encoding: [0x34,0x12,0xcc,0xda,0x01,0x02,0x00,0x00]
+0x34,0x12,0xcc,0xda,0x01,0x02,0x00,0x00
+
+# GFX12: ds_bpermute_b32 v0, v1, v2 offset:65535 ; encoding: [0xff,0xff,0xcc,0xda,0x01,0x02,0x00,0x00]
+0xff,0xff,0xcc,0xda,0x01,0x02,0x00,0x00
+
+# GFX12: ds_bpermute_b32 v0, v254, v253          ; encoding: [0x00,0x00,0xcc,0xda,0xfe,0xfd,0x00,0x00]
+0x00,0x00,0xcc,0xda,0xfe,0xfd,0x00,0x00
+
+# GFX12: ds_bpermute_b32 v0, v254, v253 offset:4660 ; encoding: [0x34,0x12,0xcc,0xda,0xfe,0xfd,0x00,0x00]
+0x34,0x12,0xcc,0xda,0xfe,0xfd,0x00,0x00
+
+# GFX12: ds_bpermute_b32 v0, v254, v253 offset:65535 ; encoding: [0xff,0xff,0xcc,0xda,0xfe,0xfd,0x00,0x00]
+0xff,0xff,0xcc,0xda,0xfe,0xfd,0x00,0x00
+
+# GFX12: ds_bpermute_b32 v255, v1, v253          ; encoding: [0x00,0x00,0xcc,0xda,0x01,0xfd,0x00,0xff]
+0x00,0x00,0xcc,0xda,0x01,0xfd,0x00,0xff
+
+# GFX12: ds_bpermute_b32 v255, v1, v253 offset:4660 ; encoding: [0x34,0x12,0xcc,0xda,0x01,0xfd,0x00,0xff]
+0x34,0x12,0xcc,0xda,0x01,0xfd,0x00,0xff
+
+# GFX12: ds_bpermute_b32 v255, v1, v253 offset:65535 ; encoding: [0xff,0xff,0xcc,0xda,0x01,0xfd,0x00,0xff]
+0xff,0xff,0xcc,0xda,0x01,0xfd,0x00,0xff
+
+# GFX12: ds_bpermute_b32 v255, v254, v2          ; encoding: [0x00,0x00,0xcc,0xda,0xfe,0x02,0x00,0xff]
+0x00,0x00,0xcc,0xda,0xfe,0x02,0x00,0xff
+
+# GFX12: ds_bpermute_b32 v255, v254, v2 offset:4660 ; encoding: [0x34,0x12,0xcc,0xda,0xfe,0x02,0x00,0xff]
+0x34,0x12,0xcc,0xda,0xfe,0x02,0x00,0xff
+
+# GFX12: ds_bpermute_b32 v255, v254, v2 offset:65535 ; encoding: [0xff,0xff,0xcc,0xda,0xfe,0x02,0x00,0xff]
+0xff,0xff,0xcc,0xda,0xfe,0x02,0x00,0xff
+
+# GFX12: ds_bpermute_b32 v255, v254, v253        ; encoding: [0x00,0x00,0xcc,0xda,0xfe,0xfd,0x00,0xff]
+0x00,0x00,0xcc,0xda,0xfe,0xfd,0x00,0xff
+
+# GFX12: ds_bpermute_b32 v255, v254, v253 offset:4660 ; encoding: [0x34,0x12,0xcc,0xda,0xfe,0xfd,0x00,0xff]
+0x34,0x12,0xcc,0xda,0xfe,0xfd,0x00,0xff
+
+# GFX12: ds_bpermute_b32 v255, v254, v253 offset:65535 ; encoding: [0xff,0xff,0xcc,0xda,0xfe,0xfd,0x00,0xff]
+0xff,0xff,0xcc,0xda,0xfe,0xfd,0x00,0xff
+
+# GFX12: ds_cmpstore_b32 v0, v1, v2              ; encoding: [0x00,0x00,0x40,0xd8,0x00,0x01,0x02,0x00]
+0x00,0x00,0x40,0xd8,0x00,0x01,0x02,0x00
+
+# GFX12: ds_cmpstore_b32 v0, v1, v2 offset:4660  ; encoding: [0x34,0x12,0x40,0xd8,0x00,0x01,0x02,0x00]
+0x34,0x12,0x40,0xd8,0x00,0x01,0x02,0x00
+
+# GFX12: ds_cmpstore_b32 v0, v1, v2 offset:65535 ; encoding: [0xff,0xff,0x40,0xd8,0x00,0x01,0x02,0x00]
+0xff,0xff,0x40,0xd8,0x00,0x01,0x02,0x00
+
+# GFX12: ds_cmpstore_b32 v0, v254, v253          ; encoding: [0x00,0x00,0x40,0xd8,0x00,0xfe,0xfd,0x00]
+0x00,0x00,0x40,0xd8,0x00,0xfe,0xfd,0x00
+
+# GFX12: ds_cmpstore_b32 v0, v254, v253 offset:4660 ; encoding: [0x34,0x12,0x40,0xd8,0x00,0xfe,0xfd,0x00]
+0x34,0x12,0x40,0xd8,0x00,0xfe,0xfd,0x00
+
+# GFX12: ds_cmpstore_b32 v0, v254, v253 offset:65535 ; encoding: [0xff,0xff,0x40,0xd8,0x00,0xfe,0xfd,0x00]
+0xff,0xff,0x40,0xd8,0x00,0xfe,0xfd,0x00
+
+# GFX12: ds_cmpstore_b32 v255, v1, v253          ; encoding: [0x00,0x00,0x40,0xd8,0xff,0x01,0xfd,0x00]
+0x00,0x00,0x40,0xd8,0xff,0x01,0xfd,0x00
+
+# GFX12: ds_cmpstore_b32 v255, v1, v253 offset:4660 ; encoding: [0x34,0x12,0x40,0xd8,0xff,0x01,0xfd,0x00]
+0x34,0x12,0x40,0xd8,0xff,0x01,0xfd,0x00
+
+# GFX12: ds_cmpstore_b32 v255, v1, v253 offset:65535 ; encoding: [0xff,0xff,0x40,0xd8,0xff,0x01,0xfd,0x00]
+0xff,0xff,0x40,0xd8,0xff,0x01,0xfd,0x00
+
+# GFX12: ds_cmpstore_b32 v255, v254, v2          ; encoding: [0x00,0x00,0x40,0xd8,0xff,0xfe,0x02,0x00]
+0x00,0x00,0x40,0xd8,0xff,0xfe,0x02,0x00
+
+# GFX12: ds_cmpstore_b32 v255, v254, v2 offset:4660 ; encoding: [0x34,0x12,0x40,0xd8,0xff,0xfe,0x02,0x00]
+0x34,0x12,0x40,0xd8,0xff,0xfe,0x02,0x00
+
+# GFX12: ds_cmpstore_b32 v255, v254, v2 offset:65535 ; encoding: [0xff,0xff,0x40,0xd8,0xff,0xfe,0x02,0x00]
+0xff,0xff,0x40,0xd8,0xff,0xfe,0x02,0x00
+
+# GFX12: ds_cmpstore_b32 v255, v254, v253        ; encoding: [0x00,0x00,0x40,0xd8,0xff,0xfe,0xfd,0x00]
+0x00,0x00,0x40,0xd8,0xff,0xfe,0xfd,0x00
+
+# GFX12: ds_cmpstore_b32 v255, v254, v253 offset:4660 ; encoding: [0x34,0x12,0x40,0xd8,0xff,0xfe,0xfd,0x00]
+0x34,0x12,0x40,0xd8,0xff,0xfe,0xfd,0x00
+
+# GFX12: ds_cmpstore_b32 v255, v254, v253 offset:65535 ; encoding: [0xff,0xff,0x40,0xd8,0xff,0xfe,0xfd,0x00]
+0xff,0xff,0x40,0xd8,0xff,0xfe,0xfd,0x00
+
+# GFX12: ds_cmpstore_b64 v1, v[254:255], v[3:4] offset:65535 ; encoding: [0xff,0xff,0x40,0xd9,0x01,0xfe,0x03,0x00]
+0xff,0xff,0x40,0xd9,0x01,0xfe,0x03,0x00
+
+# GFX12: ds_cmpstore_b64 v1, v[2:3], v[254:255] offset:65535 ; encoding: [0xff,0xff,0x40,0xd9,0x01,0x02,0xfe,0x00]
+0xff,0xff,0x40,0xd9,0x01,0x02,0xfe,0x00
+
+# GFX12: ds_cmpstore_b64 v1, v[2:3], v[3:4]      ; encoding: [0x00,0x00,0x40,0xd9,0x01,0x02,0x03,0x00]
+0x00,0x00,0x40,0xd9,0x01,0x02,0x03,0x00
+
+# GFX12: ds_cmpstore_b64 v1, v[2:3], v[3:4] offset:4 ; encoding: [0x04,0x00,0x40,0xd9,0x01,0x02,0x03,0x00]
+0x04,0x00,0x40,0xd9,0x01,0x02,0x03,0x00
+
+# GFX12: ds_cmpstore_b64 v1, v[2:3], v[3:4] offset:65535 ; encoding: [0xff,0xff,0x40,0xd9,0x01,0x02,0x03,0x00]
+0xff,0xff,0x40,0xd9,0x01,0x02,0x03,0x00
+
+# GFX12: ds_cmpstore_b64 v255, v[2:3], v[3:4] offset:65535 ; encoding: [0xff,0xff,0x40,0xd9,0xff,0x02,0x03,0x00]
+0xff,0xff,0x40,0xd9,0xff,0x02,0x03,0x00
+
+# GFX12: ds_cmpstore_rtn_b32 v255, v1, v2, v3 offset:65535 ; encoding: [0xff,0xff,0xc0,0xd8,0x01,0x02,0x03,0xff]
+0xff,0xff,0xc0,0xd8,0x01,0x02,0x03,0xff
+
+# GFX12: ds_cmpstore_rtn_b32 v5, v1, v2, v255 offset:65535 ; encoding: [0xff,0xff,0xc0,0xd8,0x01,0x02,0xff,0x05]
+0xff,0xff,0xc0,0xd8,0x01,0x02,0xff,0x05
+
+# GFX12: ds_cmpstore_rtn_b32 v5, v1, v2, v3      ; encoding: [0x00,0x00,0xc0,0xd8,0x01,0x02,0x03,0x05]
+0x00,0x00,0xc0,0xd8,0x01,0x02,0x03,0x05
+
+# GFX12: ds_cmpstore_rtn_b32 v5, v1, v2, v3 offset:4 ; encoding: [0x04,0x00,0xc0,0xd8,0x01,0x02,0x03,0x05]
+0x04,0x00,0xc0,0xd8,0x01,0x02,0x03,0x05
+
+# GFX12: ds_cmpstore_rtn_b32 v5, v1, v2, v3 offset:65535 ; encoding: [0xff,0xff,0xc0,0xd8,0x01,0x02,0x03,0x05]
+0xff,0xff,0xc0,0xd8,0x01,0x02,0x03,0x05
+
+# GFX12: ds_cmpstore_rtn_b32 v5, v1, v255, v3 offset:65535 ; encoding: [0xff,0xff,0xc0,0xd8,0x01,0xff,0x03,0x05]
+0xff,0xff,0xc0,0xd8,0x01,0xff,0x03,0x05
+
+# GFX12: ds_cmpstore_rtn_b32 v5, v255, v2, v3 offset:65535 ; encoding: [0xff,0xff,0xc0,0xd8,0xff,0x02,0x03,0x05]
+0xff,0xff,0xc0,0xd8,0xff,0x02,0x03,0x05
+
+# GFX12: ds_cmpstore_rtn_b64 v[254:255], v1, v[2:3], v[3:4] offset:65535 ; encoding: [0xff,0xff,0xc0,0xd9,0x01,0x02,0x03,0xfe]
+0xff,0xff,0xc0,0xd9,0x01,0x02,0x03,0xfe
+
+# GFX12: ds_cmpstore_rtn_b64 v[5:6], v1, v[254:255], v[3:4] offset:65535 ; encoding: [0xff,0xff,0xc0,0xd9,0x01,0xfe,0x03,0x05]
+0xff,0xff,0xc0,0xd9,0x01,0xfe,0x03,0x05
+
+# GFX12: ds_cmpstore_rtn_b64 v[5:6], v1, v[2:3], v[254:255] offset:65535 ; encoding: [0xff,0xff,0xc0,0xd9,0x01,0x02,0xfe,0x05]
+0xff,0xff,0xc0,0xd9,0x01,0x02,0xfe,0x05
+
+# GFX12: ds_cmpstore_rtn_b64 v[5:6], v1, v[2:3], v[3:4] ; encoding: [0x00,0x00,0xc0,0xd9,0x01,0x02,0x03,0x05]
+0x00,0x00,0xc0,0xd9,0x01,0x02,0x03,0x05
+
+# GFX12: ds_cmpstore_rtn_b64 v[5:6], v1, v[2:3], v[3:4] offset:4 ; encoding: [0x04,0x00,0xc0,0xd9,0x01,0x02,0x03,0x05]
+0x04,0x00,0xc0,0xd9,0x01,0x02,0x03,0x05
+
+# GFX12: ds_cmpstore_rtn_b64 v[5:6], v1, v[2:3], v[3:4] offset:65535 ; encoding: [0xff,0xff,0xc0,0xd9,0x01,0x02,0x03,0x05]
+0xff,0xff,0xc0,0xd9,0x01,0x02,0x03,0x05
+
+# GFX12: ds_cmpstore_rtn_b64 v[5:6], v255, v[2:3], v[3:4] offset:65535 ; encoding: [0xff,0xff,0xc0,0xd9,0xff,0x02,0x03,0x05]
+0xff,0xff,0xc0,0xd9,0xff,0x02,0x03,0x05
+
+# GFX12: ds_condxchg32_rtn_b64 v[254:255], v1, v[2:3] offset:65535 ; encoding: [0xff,0xff,0xf8,0xd9,0x01,0x02,0x00,0xfe]
+0xff,0xff,0xf8,0xd9,0x01,0x02,0x00,0xfe
+
+# GFX12: ds_condxchg32_rtn_b64 v[5:6], v1, v[254:255] offset:65535 ; encoding: [0xff,0xff,0xf8,0xd9,0x01,0xfe,0x00,0x05]
+0xff,0xff,0xf8,0xd9,0x01,0xfe,0x00,0x05
+
+# GFX12: ds_condxchg32_rtn_b64 v[5:6], v1, v[2:3] ; encoding: [0x00,0x00,0xf8,0xd9,0x01,0x02,0x00,0x05]
+0x00,0x00,0xf8,0xd9,0x01,0x02,0x00,0x05
+
+# GFX12: ds_condxchg32_rtn_b64 v[5:6], v1, v[2:3] offset:4 ; encoding: [0x04,0x00,0xf8,0xd9,0x01,0x02,0x00,0x05]
+0x04,0x00,0xf8,0xd9,0x01,0x02,0x00,0x05
+
+# GFX12: ds_condxchg32_rtn_b64 v[5:6], v1, v[2:3] offset:65535 ; encoding: [0xff,0xff,0xf8,0xd9,0x01,0x02,0x00,0x05]
+0xff,0xff,0xf8,0xd9,0x01,0x02,0x00,0x05
+
+# GFX12: ds_condxchg32_rtn_b64 v[5:6], v255, v[2:3] offset:65535 ; encoding: [0xff,0xff,0xf8,0xd9,0xff,0x02,0x00,0x05]
+0xff,0xff,0xf8,0xd9,0xff,0x02,0x00,0x05
+
+# GFX12: ds_consume v255 offset:65535            ; encoding: [0xff,0xff,0xf4,0xd8,0x00,0x00,0x00,0xff]
+0xff,0xff,0xf4,0xd8,0x00,0x00,0x00,0xff
+
+# GFX12: ds_consume v5                           ; encoding: [0x00,0x00,0xf4,0xd8,0x00,0x00,0x00,0x05]
+0x00,0x00,0xf4,0xd8,0x00,0x00,0x00,0x05
+
+# GFX12: ds_consume v5 offset:4                  ; encoding: [0x04,0x00,0xf4,0xd8,0x00,0x00,0x00,0x05]
+0x04,0x00,0xf4,0xd8,0x00,0x00,0x00,0x05
+
+# GFX12: ds_consume v5 offset:65535              ; encoding: [0xff,0xff,0xf4,0xd8,0x00,0x00,0x00,0x05]
+0xff,0xff,0xf4,0xd8,0x00,0x00,0x00,0x05
+
+# GFX12: ds_dec_rtn_u32 v0, v1, v2               ; encoding: [0x00,0x00,0x90,0xd8,0x01,0x02,0x00,0x00]
+0x00,0x00,0x90,0xd8,0x01,0x02,0x00,0x00
+
+# GFX12: ds_dec_rtn_u32 v0, v1, v2 offset:4660   ; encoding: [0x34,0x12,0x90,0xd8,0x01,0x02,0x00,0x00]
+0x34,0x12,0x90,0xd8,0x01,0x02,0x00,0x00
+
+# GFX12: ds_dec_rtn_u32 v0, v1, v2 offset:65535  ; encoding: [0xff,0xff,0x90,0xd8,0x01,0x02,0x00,0x00]
+0xff,0xff,0x90,0xd8,0x01,0x02,0x00,0x00
+
+# GFX12: ds_dec_rtn_u32 v0, v254, v253           ; encoding: [0x00,0x00,0x90,0xd8,0xfe,0xfd,0x00,0x00]
+0x00,0x00,0x90,0xd8,0xfe,0xfd,0x00,0x00
+
+# GFX12: ds_dec_rtn_u32 v0, v254, v253 offset:4660 ; encoding: [0x34,0x12,0x90,0xd8,0xfe,0xfd,0x00,0x00]
+0x34,0x12,0x90,0xd8,0xfe,0xfd,0x00,0x00
+
+# GFX12: ds_dec_rtn_u32 v0, v254, v253 offset:65535 ; encoding: [0xff,0xff,0x90,0xd8,0xfe,0xfd,0x00,0x00]
+0xff,0xff,0x90,0xd8,0xfe,0xfd,0x00,0x00
+
+# GFX12: ds_dec_rtn_u32 v255, v1, v253           ; encoding: [0x00,0x00,0x90,0xd8,0x01,0xfd,0x00,0xff]
+0x00,0x00,0x90,0xd8,0x01,0xfd,0x00,0xff
+
+# GFX12: ds_dec_rtn_u32 v255, v1, v253 offset:4660 ; encoding: [0x34,0x12,0x90,0xd8,0x01,0xfd,0x00,0xff]
+0x34,0x12,0x90,0xd8,0x01,0xfd,0x00,0xff
+
+# GFX12: ds_dec_rtn_u32 v255, v1, v253 offset:65535 ; encoding: [0xff,0xff,0x90,0xd8,0x01,0xfd,0x00,0xff]
+0xff,0xff,0x90,0xd8,0x01,0xfd,0x00,0xff
+
+# GFX12: ds_dec_rtn_u32 v255, v254, v2           ; encoding: [0x00,0x00,0x90,0xd8,0xfe,0x02,0x00,0xff]
+0x00,0x00,0x90,0xd8,0xfe,0x02,0x00,0xff
+
+# GFX12: ds_dec_rtn_u32 v255, v254, v2 offset:4660 ; encoding: [0x34,0x12,0x90,0xd8,0xfe,0x02,0x00,0xff]
+0x34,0x12,0x90,0xd8,0xfe,0x02,0x00,0xff
+
+# GFX12: ds_dec_rtn_u32 v255, v254, v2 offset:65535 ; encoding: [0xff,0xff,0x90,0xd8,0xfe,0x02,0x00,0xff]
+0xff,0xff,0x90,0xd8,0xfe,0x02,0x00,0xff
+
+# GFX12: ds_dec_rtn_u32 v255, v254, v253         ; encoding: [0x00,0x00,0x90,0xd8,0xfe,0xfd,0x00,0xff]
+0x00,0x00,0x90,0xd8,0xfe,0xfd,0x00,0xff
+
+# GFX12: ds_dec_rtn_u32 v255, v254, v253 offset:4660 ; encoding: [0x34,0x12,0x90,0xd8,0xfe,0xfd,0x00,0xff]
+0x34,0x12,0x90,0xd8,0xfe,0xfd,0x00,0xff
+
+# GFX12: ds_dec_rtn_u32 v255, v254, v253 offset:65535 ; encoding: [0xff,0xff,0x90,0xd8,0xfe,0xfd,0x00,0xff]
+0xff,0xff,0x90,0xd8,0xfe,0xfd,0x00,0xff
+
+# GFX12: ds_dec_rtn_u64 v[254:255], v1, v[2:3] offset:65535 ; encoding: [0xff,0xff,0x90,0xd9,0x01,0x02,0x00,0xfe]
+0xff,0xff,0x90,0xd9,0x01,0x02,0x00,0xfe
+
+# GFX12: ds_dec_rtn_u64 v[5:6], v1, v[254:255] offset:65535 ; encoding: [0xff,0xff,0x90,0xd9,0x01,0xfe,0x00,0x05]
+0xff,0xff,0x90,0xd9,0x01,0xfe,0x00,0x05
+
+# GFX12: ds_dec_rtn_u64 v[5:6], v1, v[2:3]       ; encoding: [0x00,0x00,0x90,0xd9,0x01,0x02,0x00,0x05]
+0x00,0x00,0x90,0xd9,0x01,0x02,0x00,0x05
+
+# GFX12: ds_dec_rtn_u64 v[5:6], v1, v[2:3] offset:4 ; encoding: [0x04,0x00,0x90,0xd9,0x01,0x02,0x00,0x05]
+0x04,0x00,0x90,0xd9,0x01,0x02,0x00,0x05
+
+# GFX12: ds_dec_rtn_u64 v[5:6], v1, v[2:3] offset:65535 ; encoding: [0xff,0xff,0x90,0xd9,0x01,0x02,0x00,0x05]
+0xff,0xff,0x90,0xd9,0x01,0x02,0x00,0x05
+
+# GFX12: ds_dec_rtn_u64 v[5:6], v255, v[2:3] offset:65535 ; encoding: [0xff,0xff,0x90,0xd9,0xff,0x02,0x00,0x05]
+0xff,0xff,0x90,0xd9,0xff,0x02,0x00,0x05
+
+# GFX12: ds_dec_u32 v0, v1                       ; encoding: [0x00,0x00,0x10,0xd8,0x00,0x01,0x00,0x00]
+0x00,0x00,0x10,0xd8,0x00,0x01,0x00,0x00
+
+# GFX12: ds_dec_u32 v0, v1 offset:4660           ; encoding: [0x34,0x12,0x10,0xd8,0x00,0x01,0x00,0x00]
+0x34,0x12,0x10,0xd8,0x00,0x01,0x00,0x00
+
+# GFX12: ds_dec_u32 v0, v1 offset:65535          ; encoding: [0xff,0xff,0x10,0xd8,0x00,0x01,0x00,0x00]
+0xff,0xff,0x10,0xd8,0x00,0x01,0x00,0x00
+
+# GFX12: ds_dec_u32 v0, v254                     ; encoding: [0x00,0x00,0x10,0xd8,0x00,0xfe,0x00,0x00]
+0x00,0x00,0x10,0xd8,0x00,0xfe,0x00,0x00
+
+# GFX12: ds_dec_u32 v0, v254 offset:4660         ; encoding: [0x34,0x12,0x10,0xd8,0x00,0xfe,0x00,0x00]
+0x34,0x12,0x10,0xd8,0x00,0xfe,0x00,0x00
+
+# GFX12: ds_dec_u32 v0, v254 offset:65535        ; encoding: [0xff,0xff,0x10,0xd8,0x00,0xfe,0x00,0x00]
+0xff,0xff,0x10,0xd8,0x00,0xfe,0x00,0x00
+
+# GFX12: ds_dec_u32 v255, v1                     ; encoding: [0x00,0x00,0x10,0xd8,0xff,0x01,0x00,0x00]
+0x00,0x00,0x10,0xd8,0xff,0x01,0x00,0x00
+
+# GFX12: ds_dec_u32 v255, v1 offset:4660         ; encoding: [0x34,0x12,0x10,0xd8,0xff,0x01,0x00,0x00]
+0x34,0x12,0x10,0xd8,0xff,0x01,0x00,0x00
+
+# GFX12: ds_dec_u32 v255, v1 offset:65535        ; encoding: [0xff,0xff,0x10,0xd8,0xff,0x01,0x00,0x00]
+0xff,0xff,0x10,0xd8,0xff,0x01,0x00,0x00
+
+# GFX12: ds_dec_u32 v255, v254                   ; encoding: [0x00,0x00,0x10,0xd8,0xff,0xfe,0x00,0x00]
+0x00,0x00,0x10,0xd8,0xff,0xfe,0x00,0x00
+
+# GFX12: ds_dec_u32 v255, v254 offset:4660       ; encoding: [0x34,0x12,0x10,0xd8,0xff,0xfe,0x00,0x00]
+0x34,0x12,0x10,0xd8,0xff,0xfe,0x00,0x00
+
+# GFX12: ds_dec_u32 v255, v254 offset:65535      ; encoding: [0xff,0xff,0x10,0xd8,0xff,0xfe,0x00,0x00]
+0xff,0xff,0x10,0xd8,0xff,0xfe,0x00,0x00
+
+# GFX12: ds_dec_u64 v1, v[254:255] offset:65535  ; encoding: [0xff,0xff,0x10,0xd9,0x01,0xfe,0x00,0x00]
+0xff,0xff,0x10,0xd9,0x01,0xfe,0x00,0x00
+
+# GFX12: ds_dec_u64 v1, v[2:3]                   ; encoding: [0x00,0x00,0x10,0xd9,0x01,0x02,0x00,0x00]
+0x00,0x00,0x10,0xd9,0x01,0x02,0x00,0x00
+
+# GFX12: ds_dec_u64 v1, v[2:3] offset:4          ; encoding: [0x04,0x00,0x10,0xd9,0x01,0x02,0x00,0x00]
+0x04,0x00,0x10,0xd9,0x01,0x02,0x00,0x00
+
+# GFX12: ds_dec_u64 v1, v[2:3] offset:65535      ; encoding: [0xff,0xff,0x10,0xd9,0x01,0x02,0x00,0x00]
+0xff,0xff,0x10,0xd9,0x01,0x02,0x00,0x00
+
+# GFX12: ds_dec_u64 v255, v[2:3] offset:65535    ; encoding: [0xff,0xff,0x10,0xd9,0xff,0x02,0x00,0x00]
+0xff,0xff,0x10,0xd9,0xff,0x02,0x00,0x00
+
+# GFX12: ds_inc_rtn_u32 v0, v1, v2               ; encoding: [0x00,0x00,0x8c,0xd8,0x01,0x02,0x00,0x00]
+0x00,0x00,0x8c,0xd8,0x01,0x02,0x00,0x00
+
+# GFX12: ds_inc_rtn_u32 v0, v1, v2 offset:4660   ; encoding: [0x34,0x12,0x8c,0xd8,0x01,0x02,0x00,0x00]
+0x34,0x12,0x8c,0xd8,0x01,0x02,0x00,0x00
+
+# GFX12: ds_inc_rtn_u32 v0, v1, v2 offset:65535  ; encoding: [0xff,0xff,0x8c,0xd8,0x01,0x02,0x00,0x00]
+0xff,0xff,0x8c,0xd8,0x01,0x02,0x00,0x00
+
+# GFX12: ds_inc_rtn_u32 v0, v254, v253           ; encoding: [0x00,0x00,0x8c,0xd8,0xfe,0xfd,0x00,0x00]
+0x00,0x00,0x8c,0xd8,0xfe,0xfd,0x00,0x00
+
+# GFX12: ds_inc_rtn_u32 v0, v254, v253 offset:4660 ; encoding: [0x34,0x12,0x8c,0xd8,0xfe,0xfd,0x00,0x00]
+0x34,0x12,0x8c,0xd8,0xfe,0xfd,0x00,0x00
+
+# GFX12: ds_inc_rtn_u32 v0, v254, v253 offset:65535 ; encoding: [0xff,0xff,0x8c,0xd8,0xfe,0xfd,0x00,0x00]
+0xff,0xff,0x8c,0xd8,0xfe,0xfd,0x00,0x00
+
+# GFX12: ds_inc_rtn_u32 v255, v1, v253           ; encoding: [0x00,0x00,0x8c,0xd8,0x01,0xfd,0x00,0xff]
+0x00,0x00,0x8c,0xd8,0x01,0xfd,0x00,0xff
+
+# GFX12: ds_inc_rtn_u32 v255, v1, v253 offset:4660 ; encoding: [0x34,0x12,0x8c,0xd8,0x01,0xfd,0x00,0xff]
+0x34,0x12,0x8c,0xd8,0x01,0xfd,0x00,0xff
+
+# GFX12: ds_inc_rtn_u32 v255, v1, v253 offset:65535 ; encoding: [0xff,0xff,0x8c,0xd8,0x01,0xfd,0x00,0xff]
+0xff,0xff,0x8c,0xd8,0x01,0xfd,0x00,0xff
+
+# GFX12: ds_inc_rtn_u32 v255, v254, v2           ; encoding: [0x00,0x00,0x8c,0xd8,0xfe,0x02,0x00,0xff]
+0x00,0x00,0x8c,0xd8,0xfe,0x02,0x00,0xff
+
+# GFX12: ds_inc_rtn_u32 v255, v254, v2 offset:4660 ; encoding: [0x34,0x12,0x8c,0xd8,0xfe,0x02,0x00,0xff]
+0x34,0x12,0x8c,0xd8,0xfe,0x02,0x00,0xff
+
+# GFX12: ds_inc_rtn_u32 v255, v254, v2 offset:65535 ; encoding: [0xff,0xff,0x8c,0xd8,0xfe,0x02,0x00,0xff]
+0xff,0xff,0x8c,0xd8,0xfe,0x02,0x00,0xff
+
+# GFX12: ds_inc_rtn_u32 v255, v254, v253         ; encoding: [0x00,0x00,0x8c,0xd8,0xfe,0xfd,0x00,0xff]
+0x00,0x00,0x8c,0xd8,0xfe,0xfd,0x00,0xff
+
+# GFX12: ds_inc_rtn_u32 v255, v254, v253 offset:4660 ; encoding: [0x34,0x12,0x8c,0xd8,0xfe,0xfd,0x00,0xff]
+0x34,0x12,0x8c,0xd8,0xfe,0xfd,0x00,0xff
+
+# GFX12: ds_inc_rtn_u32 v255, v254, v253 offset:65535 ; encoding: [0xff,0xff,0x8c,0xd8,0xfe,0xfd,0x00,0xff]
+0xff,0xff,0x8c,0xd8,0xfe,0xfd,0x00,0xff
+
+# GFX12: ds_inc_rtn_u64 v[254:255], v1, v[2:3] offset:65535 ; encoding: [0xff,0xff,0x8c,0xd9,0x01,0x02,0x00,0xfe]
+0xff,0xff,0x8c,0xd9,0x01,0x02,0x00,0xfe
+
+# GFX12: ds_inc_rtn_u64 v[5:6], v1, v[254:255] offset:65535 ; encoding: [0xff,0xff,0x8c,0xd9,0x01,0xfe,0x00,0x05]
+0xff,0xff,0x8c,0xd9,0x01,0xfe,0x00,0x05
+
+# GFX12: ds_inc_rtn_u64 v[5:6], v1, v[2:3]       ; encoding: [0x00,0x00,0x8c,0xd9,0x01,0x02,0x00,0x05]
+0x00,0x00,0x8c,0xd9,0x01,0x02,0x00,0x05
+
+# GFX12: ds_inc_rtn_u64 v[5:6], v1, v[2:3] offset:4 ; encoding: [0x04,0x00,0x8c,0xd9,0x01,0x02,0x00,0x05]
+0x04,0x00,0x8c,0xd9,0x01,0x02,0x00,0x05
+
+# GFX12: ds_inc_rtn_u64 v[5:6], v1, v[2:3] offset:65535 ; encoding: [0xff,0xff,0x8c,0xd9,0x01,0x02,0x00,0x05]
+0xff,0xff,0x8c,0xd9,0x01,0x02,0x00,0x05
+
+# GFX12: ds_inc_rtn_u64 v[5:6], v255, v[2:3] offset:65535 ; encoding: [0xff,0xff,0x8c,0xd9,0xff,0x02,0x00,0x05]
+0xff,0xff,0x8c,0xd9,0xff,0x02,0x00,0x05
+
+# GFX12: ds_inc_u32 v0, v1                       ; encoding: [0x00,0x00,0x0c,0xd8,0x00,0x01,0x00,0x00]
+0x00,0x00,0x0c,0xd8,0x00,0x01,0x00,0x00
+
+# GFX12: ds_inc_u32 v0, v1 offset:4660           ; encoding: [0x34,0x12,0x0c,0xd8,0x00,0x01,0x00,0x00]
+0x34,0x12,0x0c,0xd8,0x00,0x01,0x00,0x00
+
+# GFX12: ds_inc_u32 v0, v1 offset:65535          ; encoding: [0xff,0xff,0x0c,0xd8,0x00,0x01,0x00,0x00]
+0xff,0xff,0x0c,0xd8,0x00,0x01,0x00,0x00
+
+# GFX12: ds_inc_u32 v0, v254                     ; encoding: [0x00,0x00,0x0c,0xd8,0x00,0xfe,0x00,0x00]
+0x00,0x00,0x0c,0xd8,0x00,0xfe,0x00,0x00
+
+# GFX12: ds_inc_u32 v0, v254 offset:4660         ; encoding: [0x34,0x12,0x0c,0xd8,0x00,0xfe,0x00,0x00]
+0x34,0x12,0x0c,0xd8,0x00,0xfe,0x00,0x00
+
+# GFX12: ds_inc_u32 v0, v254 offset:65535        ; encoding: [0xff,0xff,0x0c,0xd8,0x00,0xfe,0x00,0x00]
+0xff,0xff,0x0c,0xd8,0x00,0xfe,0x00,0x00
+
+# GFX12: ds_inc_u32 v255, v1                     ; encoding: [0x00,0x00,0x0c,0xd8,0xff,0x01,0x00,0x00]
+0x00,0x00,0x0c,0xd8,0xff,0x01,0x00,0x00
+
+# GFX12: ds_inc_u32 v255, v1 offset:4660         ; encoding: [0x34,0x12,0x0c,0xd8,0xff,0x01,0x00,0x00]
+0x34,0x12,0x0c,0xd8,0xff,0x01,0x00,0x00
+
+# GFX12: ds_inc_u32 v255, v1 offset:65535        ; encoding: [0xff,0xff,0x0c,0xd8,0xff,0x01,0x00,0x00]
+0xff,0xff,0x0c,0xd8,0xff,0x01,0x00,0x00
+
+# GFX12: ds_inc_u32 v255, v254                   ; encoding: [0x00,0x00,0x0c,0xd8,0xff,0xfe,0x00,0x00]
+0x00,0x00,0x0c,0xd8,0xff,0xfe,0x00,0x00
+
+# GFX12: ds_inc_u32 v255, v254 offset:4660       ; encoding: [0x34,0x12,0x0c,0xd8,0xff,0xfe,0x00,0x00]
+0x34,0x12,0x0c,0xd8,0xff,0xfe,0x00,0x00
+
+# GFX12: ds_inc_u32 v255, v254 offset:65535      ; encoding: [0xff,0xff,0x0c,0xd8,0xff,0xfe,0x00,0x00]
+0xff,0xff,0x0c,0xd8,0xff,0xfe,0x00,0x00
+
+# GFX12: ds_inc_u64 v1, v[254:255] offset:65535  ; encoding: [0xff,0xff,0x0c,0xd9,0x01,0xfe,0x00,0x00]
+0xff,0xff,0x0c,0xd9,0x01,0xfe,0x00,0x00
+
+# GFX12: ds_inc_u64 v1, v[2:3]                   ; encoding: [0x00,0x00,0x0c,0xd9,0x01,0x02,0x00,0x00]
+0x00,0x00,0x0c,0xd9,0x01,0x02,0x00,0x00
+
+# GFX12: ds_inc_u64 v1, v[2:3] offset:4          ; encoding: [0x04,0x00,0x0c,0xd9,0x01,0x02,0x00,0x00]
+0x04,0x00,0x0c,0xd9,0x01,0x02,0x00,0x00
+
+# GFX12: ds_inc_u64 v1, v[2:3] offset:65535      ; encoding: [0xff,0xff,0x0c,0xd9,0x01,0x02,0x00,0x00]
+0xff,0xff,0x0c,0xd9,0x01,0x02,0x00,0x00
+
+# GFX12: ds_inc_u64 v255, v[2:3] offset:65535    ; encoding: [0xff,0xff,0x0c,0xd9,0xff,0x02,0x00,0x00]
+0xff,0xff,0x0c,0xd9,0xff,0x02,0x00,0x00
+
+# GFX12: ds_max_num_f32 v0, v1                   ; encoding: [0x00,0x00,0x4c,0xd8,0x00,0x01,0x00,0x00]
+0x00,0x00,0x4c,0xd8,0x00,0x01,0x00,0x00
+
+# GFX12: ds_max_num_f32 v0, v1 offset:4660       ; encoding: [0x34,0x12,0x4c,0xd8,0x00,0x01,0x00,0x00]
+0x34,0x12,0x4c,0xd8,0x00,0x01,0x00,0x00
+
+# GFX12: ds_max_num_f32 v0, v1 offset:65535      ; encoding: [0xff,0xff,0x4c,0xd8,0x00,0x01,0x00,0x00]
+0xff,0xff,0x4c,0xd8,0x00,0x01,0x00,0x00
+
+# GFX12: ds_max_num_f32 v0, v254                 ; encoding: [0x00,0x00,0x4c,0xd8,0x00,0xfe,0x00,0x00]
+0x00,0x00,0x4c,0xd8,0x00,0xfe,0x00,0x00
+
+# GFX12: ds_max_num_f32 v0, v254 offset:4660     ; encoding: [0x34,0x12,0x4c,0xd8,0x00,0xfe,0x00,0x00]
+0x34,0x12,0x4c,0xd8,0x00,0xfe,0x00,0x00
+
+# GFX12: ds_max_num_f32 v0, v254 offset:65535    ; encoding: [0xff,0xff,0x4c,0xd8,0x00,0xfe,0x00,0x00]
+0xff,0xff,0x4c,0xd8,0x00,0xfe,0x00,0x00
+
+# GFX12: ds_max_num_f32 v255, v1                 ; encoding: [0x00,0x00,0x4c,0xd8,0xff,0x01,0x00,0x00]
+0x00,0x00,0x4c,0xd8,0xff,0x01,0x00,0x00
+
+# GFX12: ds_max_num_f32 v255, v1 offset:4660     ; encoding: [0x34,0x12,0x4c,0xd8,0xff,0x01,0x00,0x00]
+0x34,0x12,0x4c,0xd8,0xff,0x01,0x00,0x00
+
+# GFX12: ds_max_num_f32 v255, v1 offset:65535    ; encoding: [0xff,0xff,0x4c,0xd8,0xff,0x01,0x00,0x00]
+0xff,0xff,0x4c,0xd8,0xff,0x01,0x00,0x00
+
+# GFX12: ds_max_num_f32 v255, v254               ; encoding: [0x00,0x00,0x4c,0xd8,0xff,0xfe,0x00,0x00]
+0x00,0x00,0x4c,0xd8,0xff,0xfe,0x00,0x00
+
+# GFX12: ds_max_num_f32 v255, v254 offset:4660   ; encoding: [0x34,0x12,0x4c,0xd8,0xff,0xfe,0x00,0x00]
+0x34,0x12,0x4c,0xd8,0xff,0xfe,0x00,0x00
+
+# GFX12: ds_max_num_f32 v255, v254 offset:65535  ; encoding: [0xff,0xff,0x4c,0xd8,0xff,0xfe,0x00,0x00]
+0xff,0xff,0x4c,0xd8,0xff,0xfe,0x00,0x00
+
+# GFX12: ds_max_num_f64 v1, v[254:255] offset:65535 ; encoding: [0xff,0xff,0x4c,0xd9,0x01,0xfe,0x00,0x00]
+0xff,0xff,0x4c,0xd9,0x01,0xfe,0x00,0x00
+
+# GFX12: ds_max_num_f64 v1, v[2:3]               ; encoding: [0x00,0x00,0x4c,0xd9,0x01,0x02,0x00,0x00]
+0x00,0x00,0x4c,0xd9,0x01,0x02,0x00,0x00
+
+# GFX12: ds_max_num_f64 v1, v[2:3] offset:4      ; encoding: [0x04,0x00,0x4c,0xd9,0x01,0x02,0x00,0x00]
+0x04,0x00,0x4c,0xd9,0x01,0x02,0x00,0x00
+
+# GFX12: ds_max_num_f64 v1, v[2:3] offset:65535  ; encoding: [0xff,0xff,0x4c,0xd9,0x01,0x02,0x00,0x00]
+0xff,0xff,0x4c,0xd9,0x01,0x02,0x00,0x00
+
+# GFX12: ds_max_num_f64 v255, v[2:3] offset:65535 ; encoding: [0xff,0xff,0x4c,0xd9,0xff,0x02,0x00,0x00]
+0xff,0xff,0x4c,0xd9,0xff,0x02,0x00,0x00
+
+# GFX12: ds_max_i32 v0, v1                       ; encoding: [0x00,0x00,0x18,0xd8,0x00,0x01,0x00,0x00]
+0x00,0x00,0x18,0xd8,0x00,0x01,0x00,0x00
+
+# GFX12: ds_max_i32 v0, v1 offset:4660           ; encoding: [0x34,0x12,0x18,0xd8,0x00,0x01,0x00,0x00]
+0x34,0x12,0x18,0xd8,0x00,0x01,0x00,0x00
+
+# GFX12: ds_max_i32 v0, v1 offset:65535          ; encoding: [0xff,0xff,0x18,0xd8,0x00,0x01,0x00,0x00]
+0xff,0xff,0x18,0xd8,0x00,0x01,0x00,0x00
+
+# GFX12: ds_max_i32 v0, v254                     ; encoding: [0x00,0x00,0x18,0xd8,0x00,0xfe,0x00,0x00]
+0x00,0x00,0x18,0xd8,0x00,0xfe,0x00,0x00
+
+# GFX12: ds_max_i32 v0, v254 offset:4660         ; encoding: [0x34,0x12,0x18,0xd8,0x00,0xfe,0x00,0x00]
+0x34,0x12,0x18,0xd8,0x00,0xfe,0x00,0x00
+
+# GFX12: ds_max_i32 v0, v254 offset:65535        ; encoding: [0xff,0xff,0x18,0xd8,0x00,0xfe,0x00,0x00]
+0xff,0xff,0x18,0xd8,0x00,0xfe,0x00,0x00
+
+# GFX12: ds_max_i32 v255, v1                     ; encoding: [0x00,0x00,0x18,0xd8,0xff,0x01,0x00,0x00]
+0x00,0x00,0x18,0xd8,0xff,0x01,0x00,0x00
+
+# GFX12: ds_max_i32 v255, v1 offset:4660         ; encoding: [0x34,0x12,0x18,0xd8,0xff,0x01,0x00,0x00]
+0x34,0x12,0x18,0xd8,0xff,0x01,0x00,0x00
+
+# GFX12: ds_max_i32 v255, v1 offset:65535        ; encoding: [0xff,0xff,0x18,0xd8,0xff,0x01,0x00,0x00]
+0xff,0xff,0x18,0xd8,0xff,0x01,0x00,0x00
+
+# GFX12: ds_max_i32 v255, v254                   ; encoding: [0x00,0x00,0x18,0xd8,0xff,0xfe,0x00,0x00]
+0x00,0x00,0x18,0xd8,0xff,0xfe,0x00,0x00
+
+# GFX12: ds_max_i32 v255, v254 offset:4660       ; encoding: [0x34,0x12,0x18,0xd8,0xff,0xfe,0x00,0x00]
+0x34,0x12,0x18,0xd8,0xff,0xfe,0x00,0x00
+
+# GFX12: ds_max_i32 v255, v254 offset:65535      ; encoding: [0xff,0xff,0x18,0xd8,0xff,0xfe,0x00,0x00]
+0xff,0xff,0x18,0xd8,0xff,0xfe,0x00,0x00
+
+# GFX12: ds_max_i64 v1, v[254:255] offset:65535  ; encoding: [0xff,0xff,0x18,0xd9,0x01,0xfe,0x00,0x00]
+0xff,0xff,0x18,0xd9,0x01,0xfe,0x00,0x00
+
+# GFX12: ds_max_i64 v1, v[2:3]                   ; encoding: [0x00,0x00,0x18,0xd9,0x01,0x02,0x00,0x00]
+0x00,0x00,0x18,0xd9,0x01,0x02,0x00,0x00
+
+# GFX12: ds_max_i64 v1, v[2:3] offset:4          ; encoding: [0x04,0x00,0x18,0xd9,0x01,0x02,0x00,0x00]
+0x04,0x00,0x18,0xd9,0x01,0x02,0x00,0x00
+
+# GFX12: ds_max_i64 v1, v[2:3] offset:65535      ; encoding: [0xff,0xff,0x18,0xd9,0x01,0x02,0x00,0x00]
+0xff,0xff,0x18,0xd9,0x01,0x02,0x00,0x00
+
+# GFX12: ds_max_i64 v255, v[2:3] offset:65535    ; encoding: [0xff,0xff,0x18,0xd9,0xff,0x02,0x00,0x00]
+0xff,0xff,0x18,0xd9,0xff,0x02,0x00,0x00
+
+# GFX12: ds_max_num_rtn_f32 v255, v1, v2 offset:65535 ; encoding: [0xff,0xff,0xcc,0xd8,0x01,0x02,0x00,0xff]
+0xff,0xff,0xcc,0xd8,0x01,0x02,0x00,0xff
+
+# GFX12: ds_max_num_rtn_f32 v5, v1, v2           ; encoding: [0x00,0x00,0xcc,0xd8,0x01,0x02,0x00,0x05]
+0x00,0x00,0xcc,0xd8,0x01,0x02,0x00,0x05
+
+# GFX12: ds_max_num_rtn_f32 v5, v1, v2 offset:4  ; encoding: [0x04,0x00,0xcc,0xd8,0x01,0x02,0x00,0x05]
+0x04,0x00,0xcc,0xd8,0x01,0x02,0x00,0x05
+
+# GFX12: ds_max_num_rtn_f32 v5, v1, v2 offset:65535 ; encoding: [0xff,0xff,0xcc,0xd8,0x01,0x02,0x00,0x05]
+0xff,0xff,0xcc,0xd8,0x01,0x02,0x00,0x05
+
+# GFX12: ds_max_num_rtn_f32 v5, v1, v255 offset:65535 ; encoding: [0xff,0xff,0xcc,0xd8,0x01,0xff,0x00,0x05]
+0xff,0xff,0xcc,0xd8,0x01,0xff,0x00,0x05
+
+# GFX12: ds_max_num_rtn_f32 v5, v255, v2 offset:65535 ; encoding: [0xff,0xff,0xcc,0xd8,0xff,0x02,0x00,0x05]
+0xff,0xff,0xcc,0xd8,0xff,0x02,0x00,0x05
+
+# GFX12: ds_max_num_rtn_f64 v[254:255], v1, v[2:3] offset:65535 ; encoding: [0xff,0xff,0xcc,0xd9,0x01,0x02,0x00,0xfe]
+0xff,0xff,0xcc,0xd9,0x01,0x02,0x00,0xfe
+
+# GFX12: ds_max_num_rtn_f64 v[5:6], v1, v[254:255] offset:65535 ; encoding: [0xff,0xff,0xcc,0xd9,0x01,0xfe,0x00,0x05]
+0xff,0xff,0xcc,0xd9,0x01,0xfe,0x00,0x05
+
+# GFX12: ds_max_num_rtn_f64 v[5:6], v1, v[2:3]   ; encoding: [0x00,0x00,0xcc,0xd9,0x01,0x02,0x00,0x05]
+0x00,0x00,0xcc,0xd9,0x01,0x02,0x00,0x05
+
+# GFX12: ds_max_num_rtn_f64 v[5:6], v1, v[2:3] offset:4 ; encoding: [0x04,0x00,0xcc,0xd9,0x01,0x02,0x00,0x05]
+0x04,0x00,0xcc,0xd9,0x01,0x02,0x00,0x05
+
+# GFX12: ds_max_num_rtn_f64 v[5:6], v1, v[2:3] offset:65535 ; encoding: [0xff,0xff,0xcc,0xd9,0x01,0x02,0x00,0x05]
+0xff,0xff,0xcc,0xd9,0x01,0x02,0x00,0x05
+
+# GFX12: ds_max_num_rtn_f64 v[5:6], v255, v[2:3] offset:65535 ; encoding: [0xff,0xff,0xcc,0xd9,0xff,0x02,0x00,0x05]
+0xff,0xff,0xcc,0xd9,0xff,0x02,0x00,0x05
+
+# GFX12: ds_max_rtn_i32 v0, v1, v2               ; encoding: [0x00,0x00,0x98,0xd8,0x01,0x02,0x00,0x00]
+0x00,0x00,0x98,0xd8,0x01,0x02,0x00,0x00
+
+# GFX12: ds_max_rtn_i32 v0, v1, v2 offset:4660   ; encoding: [0x34,0x12,0x98,0xd8,0x01,0x02,0x00,0x00]
+0x34,0x12,0x98,0xd8,0x01,0x02,0x00,0x00
+
+# GFX12: ds_max_rtn_i32 v0, v1, v2 offset:65535  ; encoding: [0xff,0xff,0x98,0xd8,0x01,0x02,0x00,0x00]
+0xff,0xff,0x98,0xd8,0x01,0x02,0x00,0x00
+
+# GFX12: ds_max_rtn_i32 v0, v254, v253           ; encoding: [0x00,0x00,0x98,0xd8,0xfe,0xfd,0x00,0x00]
+0x00,0x00,0x98,0xd8,0xfe,0xfd,0x00,0x00
+
+# GFX12: ds_max_rtn_i32 v0, v254, v253 offset:4660 ; encoding: [0x34,0x12,0x98,0xd8,0xfe,0xfd,0x00,0x00]
+0x34,0x12,0x98,0xd8,0xfe,0xfd,0x00,0x00
+
+# GFX12: ds_max_rtn_i32 v0, v254, v253 offset:65535 ; encoding: [0xff,0xff,0x98,0xd8,0xfe,0xfd,0x00,0x00]
+0xff,0xff,0x98,0xd8,0xfe,0xfd,0x00,0x00
+
+# GFX12: ds_max_rtn_i32 v255, v1, v253           ; encoding: [0x00,0x00,0x98,0xd8,0x01,0xfd,0x00,0xff]
+0x00,0x00,0x98,0xd8,0x01,0xfd,0x00,0xff
+
+# GFX12: ds_max_rtn_i32 v255, v1, v253 offset:4660 ; encoding: [0x34,0x12,0x98,0xd8,0x01,0xfd,0x00,0xff]
+0x34,0x12,0x98,0xd8,0x01,0xfd,0x00,0xff
+
+# GFX12: ds_max_rtn_i32 v255, v1, v253 offset:65535 ; encoding: [0xff,0xff,0x98,0xd8,0x01,0xfd,0x00,0xff]
+0xff,0xff,0x98,0xd8,0x01,0xfd,0x00,0xff
+
+# GFX12: ds_max_rtn_i32 v255, v254, v2           ; encoding: [0x00,0x00,0x98,0xd8,0xfe,0x02,0x00,0xff]
+0x00,0x00,0x98,0xd8,0xfe,0x02,0x00,0xff
+
+# GFX12: ds_max_rtn_i32 v255, v254, v2 offset:4660 ; encoding: [0x34,0x12,0x98,0xd8,0xfe,0x02,0x00,0xff]
+0x34,0x12,0x98,0xd8,0xfe,0x02,0x00,0xff
+
+# GFX12: ds_max_rtn_i32 v255, v254, v2 offset:65535 ; encoding: [0xff,0xff,0x98,0xd8,0xfe,0x02,0x00,0xff]
+0xff,0xff,0x98,0xd8,0xfe,0x02,0x00,0xff
+
+# GFX12: ds_max_rtn_i32 v255, v254, v253         ; encoding: [0x00,0x00,0x98,0xd8,0xfe,0xfd,0x00,0xff]
+0x00,0x00,0x98,0xd8,0xfe,0xfd,0x00,0xff
+
+# GFX12: ds_max_rtn_i32 v255, v254, v253 offset:4660 ; encoding: [0x34,0x12,0x98,0xd8,0xfe,0xfd,0x00,0xff]
+0x34,0x12,0x98,0xd8,0xfe,0xfd,0x00,0xff
+
+# GFX12: ds_max_rtn_i32 v255, v254, v253 offset:65535 ; encoding: [0xff,0xff,0x98,0xd8,0xfe,0xfd,0x00,0xff]
+0xff,0xff,0x98,0xd8,0xfe,0xfd,0x00,0xff
+
+# GFX12: ds_max_rtn_i64 v[254:255], v1, v[2:3] offset:65535 ; encoding: [0xff,0xff,0x98,0xd9,0x01,0x02,0x00,0xfe]
+0xff,0xff,0x98,0xd9,0x01,0x02,0x00,0xfe
+
+# GFX12: ds_max_rtn_i64 v[5:6], v1, v[254:255] offset:65535 ; encoding: [0xff,0xff,0x98,0xd9,0x01,0xfe,0x00,0x05]
+0xff,0xff,0x98,0xd9,0x01,0xfe,0x00,0x05
+
+# GFX12: ds_max_rtn_i64 v[5:6], v1, v[2:3]       ; encoding: [0x00,0x00,0x98,0xd9,0x01,0x02,0x00,0x05]
+0x00,0x00,0x98,0xd9,0x01,0x02,0x00,0x05
+
+# GFX12: ds_max_rtn_i64 v[5:6], v1, v[2:3] offset:4 ; encoding: [0x04,0x00,0x98,0xd9,0x01,0x02,0x00,0x05]
+0x04,0x00,0x98,0xd9,0x01,0x02,0x00,0x05
+
+# GFX12: ds_max_rtn_i64 v[5:6], v1, v[2:3] offset:65535 ; encoding: [0xff,0xff,0x98,0xd9,0x01,0x02,0x00,0x05]
+0xff,0xff,0x98,0xd9,0x01,0x02,0x00,0x05
+
+# GFX12: ds_max_rtn_i64 v[5:6], v255, v[2:3] offset:65535 ; encoding: [0xff,0xff,0x98,0xd9,0xff,0x02,0x00,0x05]
+0xff,0xff,0x98,0xd9,0xff,0x02,0x00,0x05
+
+# GFX12: ds_max_rtn_u32 v0, v1, v2               ; encoding: [0x00,0x00,0xa0,0xd8,0x01,0x02,0x00,0x00]
+0x00,0x00,0xa0,0xd8,0x01,0x02,0x00,0x00
+
+# GFX12: ds_max_rtn_u32 v0, v1, v2 offset:4660   ; encoding: [0x34,0x12,0xa0,0xd8,0x01,0x02,0x00,0x00]
+0x34,0x12,0xa0,0xd8,0x01,0x02,0x00,0x00
+
+# GFX12: ds_max_rtn_u32 v0, v1, v2 offset:65535  ; encoding: [0xff,0xff,0xa0,0xd8,0x01,0x02,0x00,0x00]
+0xff,0xff,0xa0,0xd8,0x01,0x02,0x00,0x00
+
+# GFX12: ds_max_rtn_u32 v0, v254, v253           ; encoding: [0x00,0x00,0xa0,0xd8,0xfe,0xfd,0x00,0x00]
+0x00,0x00,0xa0,0xd8,0xfe,0xfd,0x00,0x00
+
+# GFX12: ds_max_rtn_u32 v0, v254, v253 offset:4660 ; encoding: [0x34,0x12,0xa0,0xd8,0xfe,0xfd,0x00,0x00]
+0x34,0x12,0xa0,0xd8,0xfe,0xfd,0x00,0x00
+
+# GFX12: ds_max_rtn_u32 v0, v254, v253 offset:65535 ; encoding: [0xff,0xff,0xa0,0xd8,0xfe,0xfd,0x00,0x00]
+0xff,0xff,0xa0,0xd8,0xfe,0xfd,0x00,0x00
+
+# GFX12: ds_max_rtn_u32 v255, v1, v253           ; encoding: [0x00,0x00,0xa0,0xd8,0x01,0xfd,0x00,0xff]
+0x00,0x00,0xa0,0xd8,0x01,0xfd,0x00,0xff
+
+# GFX12: ds_max_rtn_u32 v255, v1, v253 offset:4660 ; encoding: [0x34,0x12,0xa0,0xd8,0x01,0xfd,0x00,0xff]
+0x34,0x12,0xa0,0xd8,0x01,0xfd,0x00,0xff
+
+# GFX12: ds_max_rtn_u32 v255, v1, v253 offset:65535 ; encoding: [0xff,0xff,0xa0,0xd8,0x01,0xfd,0x00,0xff]
+0xff,0xff,0xa0,0xd8,0x01,0xfd,0x00,0xff
+
+# GFX12: ds_max_rtn_u32 v255, v254, v2           ; encoding: [0x00,0x00,0xa0,0xd8,0xfe,0x02,0x00,0xff]
+0x00,0x00,0xa0,0xd8,0xfe,0x02,0x00,0xff
+
+# GFX12: ds_max_rtn_u32 v255, v254, v2 offset:4660 ; encoding: [0x34,0x12,0xa0,0xd8,0xfe,0x02,0x00,0xff]
+0x34,0x12,0xa0,0xd8,0xfe,0x02,0x00,0xff
+
+# GFX12: ds_max_rtn_u32 v255, v254, v2 offset:65535 ; encoding: [0xff,0xff,0xa0,0xd8,0xfe,0x02,0x00,0xff]
+0xff,0xff,0xa0,0xd8,0xfe,0x02,0x00,0xff
+
+# GFX12: ds_max_rtn_u32 v255, v254, v253         ; encoding: [0x00,0x00,0xa0,0xd8,0xfe,0xfd,0x00,0xff]
+0x00,0x00,0xa0,0xd8,0xfe,0xfd,0x00,0xff
+
+# GFX12: ds_max_rtn_u32 v255, v254, v253 offset:4660 ; encoding: [0x34,0x12,0xa0,0xd8,0xfe,0xfd,0x00,0xff]
+0x34,0x12,0xa0,0xd8,0xfe,0xfd,0x00,0xff
+
+# GFX12: ds_max_rtn_u32 v255, v254, v253 offset:65535 ; encoding: [0xff,0xff,0xa0,0xd8,0xfe,0xfd,0x00,0xff]
+0xff,0xff,0xa0,0xd8,0xfe,0xfd,0x00,0xff
+
+# GFX12: ds_max_rtn_u64 v[254:255], v1, v[2:3] offset:65535 ; encoding: [0xff,0xff,0xa0,0xd9,0x01,0x02,0x00,0xfe]
+0xff,0xff,0xa0,0xd9,0x01,0x02,0x00,0xfe
+
+# GFX12: ds_max_rtn_u64 v[5:6], v1, v[254:255] offset:65535 ; encoding: [0xff,0xff,0xa0,0xd9,0x01,0xfe,0x00,0x05]
+0xff,0xff,0xa0,0xd9,0x01,0xfe,0x00,0x05
+
+# GFX12: ds_max_rtn_u64 v[5:6], v1, v[2:3]       ; encoding: [0x00,0x00,0xa0,0xd9,0x01,0x02,0x00,0x05]
+0x00,0x00,0xa0,0xd9,0x01,0x02,0x00,0x05
+
+# GFX12: ds_max_rtn_u64 v[5:6], v1, v[2:3] offset:4 ; encoding: [0x04,0x00,0xa0,0xd9,0x01,0x02,0x00,0x05]
+0x04,0x00,0xa0,0xd9,0x01,0x02,0x00,0x05
+
+# GFX12: ds_max_rtn_u64 v[5:6], v1, v[2:3] offset:65535 ; encoding: [0xff,0xff,0xa0,0xd9,0x01,0x02,0x00,0x05]
+0xff,0xff,0xa0,0xd9,0x01,0x02,0x00,0x05
+
+# GFX12: ds_max_rtn_u64 v[5:6], v255, v[2:3] offset:65535 ; encoding: [0xff,0xff,0xa0,0xd9,0xff,0x02,0x00,0x05]
+0xff,0xff,0xa0,0xd9,0xff,0x02,0x00,0x05
+
+# GFX12: ds_max_u32 v0, v1                       ; encoding: [0x00,0x00,0x20,0xd8,0x00,0x01,0x00,0x00]
+0x00,0x00,0x20,0xd8,0x00,0x01,0x00,0x00
+
+# GFX12: ds_max_u32 v0, v1 offset:4660           ; encoding: [0x34,0x12,0x20,0xd8,0x00,0x01,0x00,0x00]
+0x34,0x12,0x20,0xd8,0x00,0x01,0x00,0x00
+
+# GFX12: ds_max_u32 v0, v1 offset:65535          ; encoding: [0xff,0xff,0x20,0xd8,0x00,0x01,0x00,0x00]
+0xff,0xff,0x20,0xd8,0x00,0x01,0x00,0x00
+
+# GFX12: ds_max_u32 v0, v254                     ; encoding: [0x00,0x00,0x20,0xd8,0x00,0xfe,0x00,0x00]
+0x00,0x00,0x20,0xd8,0x00,0xfe,0x00,0x00
+
+# GFX12: ds_max_u32 v0, v254 offset:4660         ; encoding: [0x34,0x12,0x20,0xd8,0x00,0xfe,0x00,0x00]
+0x34,0x12,0x20,0xd8,0x00,0xfe,0x00,0x00
+
+# GFX12: ds_max_u32 v0, v254 offset:65535        ; encoding: [0xff,0xff,0x20,0xd8,0x00,0xfe,0x00,0x00]
+0xff,0xff,0x20,0xd8,0x00,0xfe,0x00,0x00
+
+# GFX12: ds_max_u32 v255, v1                     ; encoding: [0x00,0x00,0x20,0xd8,0xff,0x01,0x00,0x00]
+0x00,0x00,0x20,0xd8,0xff,0x01,0x00,0x00
+
+# GFX12: ds_max_u32 v255, v1 offset:4660         ; encoding: [0x34,0x12,0x20,0xd8,0xff,0x01,0x00,0x00]
+0x34,0x12,0x20,0xd8,0xff,0x01,0x00,0x00
+
+# GFX12: ds_max_u32 v255, v1 offset:65535        ; encoding: [0xff,0xff,0x20,0xd8,0xff,0x01,0x00,0x00]
+0xff,0xff,0x20,0xd8,0xff,0x01,0x00,0x00
+
+# GFX12: ds_max_u32 v255, v254                   ; encoding: [0x00,0x00,0x20,0xd8,0xff,0xfe,0x00,0x00]
+0x00,0x00,0x20,0xd8,0xff,0xfe,0x00,0x00
+
+# GFX12: ds_max_u32 v255, v254 offset:4660       ; encoding: [0x34,0x12,0x20,0xd8,0xff,0xfe,0x00,0x00]
+0x34,0x12,0x20,0xd8,0xff,0xfe,0x00,0x00
+
+# GFX12: ds_max_u32 v255, v254 offset:65535      ; encoding: [0xff,0xff,0x20,0xd8,0xff,0xfe,0x00,0x00]
+0xff,0xff,0x20,0xd8,0xff,0xfe,0x00,0x00
+
+# GFX12: ds_max_u64 v1, v[254:255] offset:65535  ; encoding: [0xff,0xff,0x20,0xd9,0x01,0xfe,0x00,0x00]
+0xff,0xff,0x20,0xd9,0x01,0xfe,0x00,0x00
+
+# GFX12: ds_max_u64 v1, v[2:3]                   ; encoding: [0x00,0x00,0x20,0xd9,0x01,0x02,0x00,0x00]
+0x00,0x00,0x20,0xd9,0x01,0x02,0x00,0x00
+
+# GFX12: ds_max_u64 v1, v[2:3] offset:4          ; encoding: [0x04,0x00,0x20,0xd9,0x01,0x02,0x00,0x00]
+0x04,0x00,0x20,0xd9,0x01,0x02,0x00,0x00
+
+# GFX12: ds_max_u64 v1, v[2:3] offset:65535      ; encoding: [0xff,0xff,0x20,0xd9,0x01,0x02,0x00,0x00]
+0xff,0xff,0x20,0xd9,0x01,0x02,0x00,0x00
+
+# GFX12: ds_max_u64 v255, v[2:3] offset:65535    ; encoding: [0xff,0xff,0x20,0xd9,0xff,0x02,0x00,0x00]
+0xff,0xff,0x20,0xd9,0xff,0x02,0x00,0x00
+
+# GFX12: ds_min_num_f32 v0, v1                   ; encoding: [0x00,0x00,0x48,0xd8,0x00,0x01,0x00,0x00]
+0x00,0x00,0x48,0xd8,0x00,0x01,0x00,0x00
+
+# GFX12: ds_min_num_f32 v0, v1 offset:4660       ; encoding: [0x34,0x12,0x48,0xd8,0x00,0x01,0x00,0x00]
+0x34,0x12,0x48,0xd8,0x00,0x01,0x00,0x00
+
+# GFX12: ds_min_num_f32 v0, v1 offset:65535      ; encoding: [0xff,0xff,0x48,0xd8,0x00,0x01,0x00,0x00]
+0xff,0xff,0x48,0xd8,0x00,0x01,0x00,0x00
+
+# GFX12: ds_min_num_f32 v0, v254                 ; encoding: [0x00,0x00,0x48,0xd8,0x00,0xfe,0x00,0x00]
+0x00,0x00,0x48,0xd8,0x00,0xfe,0x00,0x00
+
+# GFX12: ds_min_num_f32 v0, v254 offset:4660     ; encoding: [0x34,0x12,0x48,0xd8,0x00,0xfe,0x00,0x00]
+0x34,0x12,0x48,0xd8,0x00,0xfe,0x00,0x00
+
+# GFX12: ds_min_num_f32 v0, v254 offset:65535    ; encoding: [0xff,0xff,0x48,0xd8,0x00,0xfe,0x00,0x00]
+0xff,0xff,0x48,0xd8,0x00,0xfe,0x00,0x00
+
+# GFX12: ds_min_num_f32 v255, v1                 ; encoding: [0x00,0x00,0x48,0xd8,0xff,0x01,0x00,0x00]
+0x00,0x00,0x48,0xd8,0xff,0x01,0x00,0x00
+
+# GFX12: ds_min_num_f32 v255, v1 offset:4660     ; encoding: [0x34,0x12,0x48,0xd8,0xff,0x01,0x00,0x00]
+0x34,0x12,0x48,0xd8,0xff,0x01,0x00,0x00
+
+# GFX12: ds_min_num_f32 v255, v1 offset:65535    ; encoding: [0xff,0xff,0x48,0xd8,0xff,0x01,0x00,0x00]
+0xff,0xff,0x48,0xd8,0xff,0x01,0x00,0x00
+
+# GFX12: ds_min_num_f32 v255, v254               ; encoding: [0x00,0x00,0x48,0xd8,0xff,0xfe,0x00,0x00]
+0x00,0x00,0x48,0xd8,0xff,0xfe,0x00,0x00
+
+# GFX12: ds_min_num_f32 v255, v254 offset:4660   ; encoding: [0x34,0x12,0x48,0xd8,0xff,0xfe,0x00,0x00]
+0x34,0x12,0x48,0xd8,0xff,0xfe,0x00,0x00
+
+# GFX12: ds_min_num_f32 v255, v254 offset:65535  ; encoding: [0xff,0xff,0x48,0xd8,0xff,0xfe,0x00,0x00]
+0xff,0xff,0x48,0xd8,0xff,0xfe,0x00,0x00
+
+# GFX12: ds_min_num_f64 v1, v[254:255] offset:65535 ; encoding: [0xff,0xff,0x48,0xd9,0x01,0xfe,0x00,0x00]
+0xff,0xff,0x48,0xd9,0x01,0xfe,0x00,0x00
+
+# GFX12: ds_min_num_f64 v1, v[2:3]               ; encoding: [0x00,0x00,0x48,0xd9,0x01,0x02,0x00,0x00]
+0x00,0x00,0x48,0xd9,0x01,0x02,0x00,0x00
+
+# GFX12: ds_min_num_f64 v1, v[2:3] offset:4      ; encoding: [0x04,0x00,0x48,0xd9,0x01,0x02,0x00,0x00]
+0x04,0x00,0x48,0xd9,0x01,0x02,0x00,0x00
+
+# GFX12: ds_min_num_f64 v1, v[2:3] offset:65535  ; encoding: [0xff,0xff,0x48,0xd9,0x01,0x02,0x00,0x00]
+0xff,0xff,0x48,0xd9,0x01,0x02,0x00,0x00
+
+# GFX12: ds_min_num_f64 v255, v[2:3] offset:65535 ; encoding: [0xff,0xff,0x48,0xd9,0xff,0x02,0x00,0x00]
+0xff,0xff,0x48,0xd9,0xff,0x02,0x00,0x00
+
+# GFX12: ds_min_i32 v0, v1                       ; encoding: [0x00,0x00,0x14,0xd8,0x00,0x01,0x00,0x00]
+0x00,0x00,0x14,0xd8,0x00,0x01,0x00,0x00
+
+# GFX12: ds_min_i32 v0, v1 offset:4660           ; encoding: [0x34,0x12,0x14,0xd8,0x00,0x01,0x00,0x00]
+0x34,0x12,0x14,0xd8,0x00,0x01,0x00,0x00
+
+# GFX12: ds_min_i32 v0, v1 offset:65535          ; encoding: [0xff,0xff,0x14,0xd8,0x00,0x01,0x00,0x00]
+0xff,0xff,0x14,0xd8,0x00,0x01,0x00,0x00
+
+# GFX12: ds_min_i32 v0, v254                     ; encoding: [0x00,0x00,0x14,0xd8,0x00,0xfe,0x00,0x00]
+0x00,0x00,0x14,0xd8,0x00,0xfe,0x00,0x00
+
+# GFX12: ds_min_i32 v0, v254 offset:4660         ; encoding: [0x34,0x12,0x14,0xd8,0x00,0xfe,0x00,0x00]
+0x34,0x12,0x14,0xd8,0x00,0xfe,0x00,0x00
+
+# GFX12: ds_min_i32 v0, v254 offset:65535        ; encoding: [0xff,0xff,0x14,0xd8,0x00,0xfe,0x00,0x00]
+0xff,0xff,0x14,0xd8,0x00,0xfe,0x00,0x00
+
+# GFX12: ds_min_i32 v255, v1                     ; encoding: [0x00,0x00,0x14,0xd8,0xff,0x01,0x00,0x00]
+0x00,0x00,0x14,0xd8,0xff,0x01,0x00,0x00
+
+# GFX12: ds_min_i32 v255, v1 offset:4660         ; encoding: [0x34,0x12,0x14,0xd8,0xff,0x01,0x00,0x00]
+0x34,0x12,0x14,0xd8,0xff,0x01,0x00,0x00
+
+# GFX12: ds_min_i32 v255, v1 offset:65535        ; encoding: [0xff,0xff,0x14,0xd8,0xff,0x01,0x00,0x00]
+0xff,0xff,0x14,0xd8,0xff,0x01,0x00,0x00
+
+# GFX12: ds_min_i32 v255, v254                   ; encoding: [0x00,0x00,0x14,0xd8,0xff,0xfe,0x00,0x00]
+0x00,0x00,0x14,0xd8,0xff,0xfe,0x00,0x00
+
+# GFX12: ds_min_i32 v255, v254 offset:4660       ; encoding: [0x34,0x12,0x14,0xd8,0xff,0xfe,0x00,0x00]
+0x34,0x12,0x14,0xd8,0xff,0xfe,0x00,0x00
+
+# GFX12: ds_min_i32 v255, v254 offset:65535      ; encoding: [0xff,0xff,0x14,0xd8,0xff,0xfe,0x00,0x00]
+0xff,0xff,0x14,0xd8,0xff,0xfe,0x00,0x00
+
+# GFX12: ds_min_i64 v1, v[254:255] offset:65535  ; encoding: [0xff,0xff,0x14,0xd9,0x01,0xfe,0x00,0x00]
+0xff,0xff,0x14,0xd9,0x01,0xfe,0x00,0x00
+
+# GFX12: ds_min_i64 v1, v[2:3]                   ; encoding: [0x00,0x00,0x14,0xd9,0x01,0x02,0x00,0x00]
+0x00,0x00,0x14,0xd9,0x01,0x02,0x00,0x00
+
+# GFX12: ds_min_i64 v1, v[2:3] offset:4          ; encoding: [0x04,0x00,0x14,0xd9,0x01,0x02,0x00,0x00]
+0x04,0x00,0x14,0xd9,0x01,0x02,0x00,0x00
+
+# GFX12: ds_min_i64 v1, v[2:3] offset:65535      ; encoding: [0xff,0xff,0x14,0xd9,0x01,0x02,0x00,0x00]
+0xff,0xff,0x14,0xd9,0x01,0x02,0x00,0x00
+
+# GFX12: ds_min_i64 v255, v[2:3] offset:65535    ; encoding: [0xff,0xff,0x14,0xd9,0xff,0x02,0x00,0x00]
+0xff,0xff,0x14,0xd9,0xff,0x02,0x00,0x00
+
+# GFX12: ds_min_num_rtn_f32 v255, v1, v2 offset:65535 ; encoding: [0xff,0xff,0xc8,0xd8,0x01,0x02,0x00,0xff]
+0xff,0xff,0xc8,0xd8,0x01,0x02,0x00,0xff
+
+# GFX12: ds_min_num_rtn_f32 v5, v1, v2           ; encoding: [0x00,0x00,0xc8,0xd8,0x01,0x02,0x00,0x05]
+0x00,0x00,0xc8,0xd8,0x01,0x02,0x00,0x05
+
+# GFX12: ds_min_num_rtn_f32 v5, v1, v2 offset:4  ; encoding: [0x04,0x00,0xc8,0xd8,0x01,0x02,0x00,0x05]
+0x04,0x00,0xc8,0xd8,0x01,0x02,0x00,0x05
+
+# GFX12: ds_min_num_rtn_f32 v5, v1, v2 offset:65535 ; encoding: [0xff,0xff,0xc8,0xd8,0x01,0x02,0x00,0x05]
+0xff,0xff,0xc8,0xd8,0x01,0x02,0x00,0x05
+
+# GFX12: ds_min_num_rtn_f32 v5, v1, v255 offset:65535 ; encoding: [0xff,0xff,0xc8,0xd8,0x01,0xff,0x00,0x05]
+0xff,0xff,0xc8,0xd8,0x01,0xff,0x00,0x05
+
+# GFX12: ds_min_num_rtn_f32 v5, v255, v2 offset:65535 ; encoding: [0xff,0xff,0xc8,0xd8,0xff,0x02,0x00,0x05]
+0xff,0xff,0xc8,0xd8,0xff,0x02,0x00,0x05
+
+# GFX12: ds_min_num_rtn_f64 v[254:255], v1, v[2:3] offset:65535 ; encoding: [0xff,0xff,0xc8,0xd9,0x01,0x02,0x00,0xfe]
+0xff,0xff,0xc8,0xd9,0x01,0x02,0x00,0xfe
+
+# GFX12: ds_min_num_rtn_f64 v[5:6], v1, v[254:255] offset:65535 ; encoding: [0xff,0xff,0xc8,0xd9,0x01,0xfe,0x00,0x05]
+0xff,0xff,0xc8,0xd9,0x01,0xfe,0x00,0x05
+
+# GFX12: ds_min_num_rtn_f64 v[5:6], v1, v[2:3]   ; encoding: [0x00,0x00,0xc8,0xd9,0x01,0x02,0x00,0x05]
+0x00,0x00,0xc8,0xd9,0x01,0x02,0x00,0x05
+
+# GFX12: ds_min_num_rtn_f64 v[5:6], v1, v[2:3] offset:4 ; encoding: [0x04,0x00,0xc8,0xd9,0x01,0x02,0x00,0x05]
+0x04,0x00,0xc8,0xd9,0x01,0x02,0x00,0x05
+
+# GFX12: ds_min_num_rtn_f64 v[5:6], v1, v[2:3] offset:65535 ; encoding: [0xff,0xff,0xc8,0xd9,0x01,0x02,0x00,0x05]
+0xff,0xff,0xc8,0xd9,0x01,0x02,0x00,0x05
+
+# GFX12: ds_min_num_rtn_f64 v[5:6], v255, v[2:3] offset:65535 ; encoding: [0xff,0xff,0xc8,0xd9,0xff,0x02,0x00,0x05]
+0xff,0xff,0xc8,0xd9,0xff,0x02,0x00,0x05
+
+# GFX12: ds_min_rtn_i32 v0, v1, v2               ; encoding: [0x00,0x00,0x94,0xd8,0x01,0x02,0x00,0x00]
+0x00,0x00,0x94,0xd8,0x01,0x02,0x00,0x00
+
+# GFX12: ds_min_rtn_i32 v0, v1, v2 offset:4660   ; encoding: [0x34,0x12,0x94,0xd8,0x01,0x02,0x00,0x00]
+0x34,0x12,0x94,0xd8,0x01,0x02,0x00,0x00
+
+# GFX12: ds_min_rtn_i32 v0, v1, v2 offset:65535  ; encoding: [0xff,0xff,0x94,0xd8,0x01,0x02,0x00,0x00]
+0xff,0xff,0x94,0xd8,0x01,0x02,0x00,0x00
+
+# GFX12: ds_min_rtn_i32 v0, v254, v253           ; encoding: [0x00,0x00,0x94,0xd8,0xfe,0xfd,0x00,0x00]
+0x00,0x00,0x94,0xd8,0xfe,0xfd,0x00,0x00
+
+# GFX12: ds_min_rtn_i32 v0, v254, v253 offset:4660 ; encoding: [0x34,0x12,0x94,0xd8,0xfe,0xfd,0x00,0x00]
+0x34,0x12,0x94,0xd8,0xfe,0xfd,0x00,0x00
+
+# GFX12: ds_min_rtn_i32 v0, v254, v253 offset:65535 ; encoding: [0xff,0xff,0x94,0xd8,0xfe,0xfd,0x00,0x00]
+0xff,0xff,0x94,0xd8,0xfe,0xfd,0x00,0x00
+
+# GFX12: ds_min_rtn_i32 v255, v1, v253           ; encoding: [0x00,0x00,0x94,0xd8,0x01,0xfd,0x00,0xff]
+0x00,0x00,0x94,0xd8,0x01,0xfd,0x00,0xff
+
+# GFX12: ds_min_rtn_i32 v255, v1, v253 offset:4660 ; encoding: [0x34,0x12,0x94,0xd8,0x01,0xfd,0x00,0xff]
+0x34,0x12,0x94,0xd8,0x01,0xfd,0x00,0xff
+
+# GFX12: ds_min_rtn_i32 v255, v1, v253 offset:65535 ; encoding: [0xff,0xff,0x94,0xd8,0x01,0xfd,0x00,0xff]
+0xff,0xff,0x94,0xd8,0x01,0xfd,0x00,0xff
+
+# GFX12: ds_min_rtn_i32 v255, v254, v2           ; encoding: [0x00,0x00,0x94,0xd8,0xfe,0x02,0x00,0xff]
+0x00,0x00,0x94,0xd8,0xfe,0x02,0x00,0xff
+
+# GFX12: ds_min_rtn_i32 v255, v254, v2 offset:4660 ; encoding: [0x34,0x12,0x94,0xd8,0xfe,0x02,0x00,0xff]
+0x34,0x12,0x94,0xd8,0xfe,0x02,0x00,0xff
+
+# GFX12: ds_min_rtn_i32 v255, v254, v2 offset:65535 ; encoding: [0xff,0xff,0x94,0xd8,0xfe,0x02,0x00,0xff]
+0xff,0xff,0x94,0xd8,0xfe,0x02,0x00,0xff
+
+# GFX12: ds_min_rtn_i32 v255, v254, v253         ; encoding: [0x00,0x00,0x94,0xd8,0xfe,0xfd,0x00,0xff]
+0x00,0x00,0x94,0xd8,0xfe,0xfd,0x00,0xff
+
+# GFX12: ds_min_rtn_i32 v255, v254, v253 offset:4660 ; encoding: [0x34,0x12,0x94,0xd8,0xfe,0xfd,0x00,0xff]
+0x34,0x12,0x94,0xd8,0xfe,0xfd,0x00,0xff
+
+# GFX12: ds_min_rtn_i32 v255, v254, v253 offset:65535 ; encoding: [0xff,0xff,0x94,0xd8,0xfe,0xfd,0x00,0xff]
+0xff,0xff,0x94,0xd8,0xfe,0xfd,0x00,0xff
+
+# GFX12: ds_min_rtn_i64 v[254:255], v1, v[2:3] offset:65535 ; encoding: [0xff,0xff,0x94,0xd9,0x01,0x02,0x00,0xfe]
+0xff,0xff,0x94,0xd9,0x01,0x02,0x00,0xfe
+
+# GFX12: ds_min_rtn_i64 v[5:6], v1, v[254:255] offset:65535 ; encoding: [0xff,0xff,0x94,0xd9,0x01,0xfe,0x00,0x05]
+0xff,0xff,0x94,0xd9,0x01,0xfe,0x00,0x05
+
+# GFX12: ds_min_rtn_i64 v[5:6], v1, v[2:3]       ; encoding: [0x00,0x00,0x94,0xd9,0x01,0x02,0x00,0x05]
+0x00,0x00,0x94,0xd9,0x01,0x02,0x00,0x05
+
+# GFX12: ds_min_rtn_i64 v[5:6], v1, v[2:3] offset:4 ; encoding: [0x04,0x00,0x94,0xd9,0x01,0x02,0x00,0x05]
+0x04,0x00,0x94,0xd9,0x01,0x02,0x00,0x05
+
+# GFX12: ds_min_rtn_i64 v[5:6], v1, v[2:3] offset:65535 ; encoding: [0xff,0xff,0x94,0xd9,0x01,0x02,0x00,0x05]
+0xff,0xff,0x94,0xd9,0x01,0x02,0x00,0x05
+
+# GFX12: ds_min_rtn_i64 v[5:6], v255, v[2:3] offset:65535 ; encoding: [0xff,0xff,0x94,0xd9,0xff,0x02,0x00,0x05]
+0xff,0xff,0x94,0xd9,0xff,0x02,0x00,0x05
+
+# GFX12: ds_min_rtn_u32 v0, v1, v2               ; encoding: [0x00,0x00,0x9c,0xd8,0x01,0x02,0x00,0x00]
+0x00,0x00,0x9c,0xd8,0x01,0x02,0x00,0x00
+
+# GFX12: ds_min_rtn_u32 v0, v1, v2 offset:4660   ; encoding: [0x34,0x12,0x9c,0xd8,0x01,0x02,0x00,0x00]
+0x34,0x12,0x9c,0xd8,0x01,0x02,0x00,0x00
+
+# GFX12: ds_min_rtn_u32 v0, v1, v2 offset:65535  ; encoding: [0xff,0xff,0x9c,0xd8,0x01,0x02,0x00,0x00]
+0xff,0xff,0x9c,0xd8,0x01,0x02,0x00,0x00
+
+# GFX12: ds_min_rtn_u32 v0, v254, v253           ; encoding: [0x00,0x00,0x9c,0xd8,0xfe,0xfd,0x00,0x00]
+0x00,0x00,0x9c,0xd8,0xfe,0xfd,0x00,0x00
+
+# GFX12: ds_min_rtn_u32 v0, v254, v253 offset:4660 ; encoding: [0x34,0x12,0x9c,0xd8,0xfe,0xfd,0x00,0x00]
+0x34,0x12,0x9c,0xd8,0xfe,0xfd,0x00,0x00
+
+# GFX12: ds_min_rtn_u32 v0, v254, v253 offset:65535 ; encoding: [0xff,0xff,0x9c,0xd8,0xfe,0xfd,0x00,0x00]
+0xff,0xff,0x9c,0xd8,0xfe,0xfd,0x00,0x00
+
+# GFX12: ds_min_rtn_u32 v255, v1, v253           ; encoding: [0x00,0x00,0x9c,0xd8,0x01,0xfd,0x00,0xff]
+0x00,0x00,0x9c,0xd8,0x01,0xfd,0x00,0xff
+
+# GFX12: ds_min_rtn_u32 v255, v1, v253 offset:4660 ; encoding: [0x34,0x12,0x9c,0xd8,0x01,0xfd,0x00,0xff]
+0x34,0x12,0x9c,0xd8,0x01,0xfd,0x00,0xff
+
+# GFX12: ds_min_rtn_u32 v255, v1, v253 offset:65535 ; encoding: [0xff,0xff,0x9c,0xd8,0x01,0xfd,0x00,0xff]
+0xff,0xff,0x9c,0xd8,0x01,0xfd,0x00,0xff
+
+# GFX12: ds_min_rtn_u32 v255, v254, v2           ; encoding: [0x00,0x00,0x9c,0xd8,0xfe,0x02,0x00,0xff]
+0x00,0x00,0x9c,0xd8,0xfe,0x02,0x00,0xff
+
+# GFX12: ds_min_rtn_u32 v255, v254, v2 offset:4660 ; encoding: [0x34,0x12,0x9c,0xd8,0xfe,0x02,0x00,0xff]
+0x34,0x12,0x9c,0xd8,0xfe,0x02,0x00,0xff
+
+# GFX12: ds_min_rtn_u32 v255, v254, v2 offset:65535 ; encoding: [0xff,0xff,0x9c,0xd8,0xfe,0x02,0x00,0xff]
+0xff,0xff,0x9c,0xd8,0xfe,0x02,0x00,0xff
+
+# GFX12: ds_min_rtn_u32 v255, v254, v253         ; encoding: [0x00,0x00,0x9c,0xd8,0xfe,0xfd,0x00,0xff]
+0x00,0x00,0x9c,0xd8,0xfe,0xfd,0x00,0xff
+
+# GFX12: ds_min_rtn_u32 v255, v254, v253 offset:4660 ; encoding: [0x34,0x12,0x9c,0xd8,0xfe,0xfd,0x00,0xff]
+0x34,0x12,0x9c,0xd8,0xfe,0xfd,0x00,0xff
+
+# GFX12: ds_min_rtn_u32 v255, v254, v253 offset:65535 ; encoding: [0xff,0xff,0x9c,0xd8,0xfe,0xfd,0x00,0xff]
+0xff,0xff,0x9c,0xd8,0xfe,0xfd,0x00,0xff
+
+# GFX12: ds_min_rtn_u64 v[254:255], v1, v[2:3] offset:65535 ; encoding: [0xff,0xff,0x9c,0xd9,0x01,0x02,0x00,0xfe]
+0xff,0xff,0x9c,0xd9,0x01,0x02,0x00,0xfe
+
+# GFX12: ds_min_rtn_u64 v[5:6], v1, v[254:255] offset:65535 ; encoding: [0xff,0xff,0x9c,0xd9,0x01,0xfe,0x00,0x05]
+0xff,0xff,0x9c,0xd9,0x01,0xfe,0x00,0x05
+
+# GFX12: ds_min_rtn_u64 v[5:6], v1, v[2:3]       ; encoding: [0x00,0x00,0x9c,0xd9,0x01,0x02,0x00,0x05]
+0x00,0x00,0x9c,0xd9,0x01,0x02,0x00,0x05
+
+# GFX12: ds_min_rtn_u64 v[5:6], v1, v[2:3] offset:4 ; encoding: [0x04,0x00,0x9c,0xd9,0x01,0x02,0x00,0x05]
+0x04,0x00,0x9c,0xd9,0x01,0x02,0x00,0x05
+
+# GFX12: ds_min_rtn_u64 v[5:6], v1, v[2:3] offset:65535 ; encoding: [0xff,0xff,0x9c,0xd9,0x01,0x02,0x00,0x05]
+0xff,0xff,0x9c,0xd9,0x01,0x02,0x00,0x05
+
+# GFX12: ds_min_rtn_u64 v[5:6], v255, v[2:3] offset:65535 ; encoding: [0xff,0xff,0x9c,0xd9,0xff,0x02,0x00,0x05]
+0xff,0xff,0x9c,0xd9,0xff,0x02,0x00,0x05
+
+# GFX12: ds_min_u32 v0, v1                       ; encoding: [0x00,0x00,0x1c,0xd8,0x00,0x01,0x00,0x00]
+0x00,0x00,0x1c,0xd8,0x00,0x01,0x00,0x00
+
+# GFX12: ds_min_u32 v0, v1 offset:4660           ; encoding: [0x34,0x12,0x1c,0xd8,0x00,0x01,0x00,0x00]
+0x34,0x12,0x1c,0xd8,0x00,0x01,0x00,0x00
+
+# GFX12: ds_min_u32 v0, v1 offset:65535          ; encoding: [0xff,0xff,0x1c,0xd8,0x00,0x01,0x00,0x00]
+0xff,0xff,0x1c,0xd8,0x00,0x01,0x00,0x00
+
+# GFX12: ds_min_u32 v0, v254                     ; encoding: [0x00,0x00,0x1c,0xd8,0x00,0xfe,0x00,0x00]
+0x00,0x00,0x1c,0xd8,0x00,0xfe,0x00,0x00
+
+# GFX12: ds_min_u32 v0, v254 offset:4660         ; encoding: [0x34,0x12,0x1c,0xd8,0x00,0xfe,0x00,0x00]
+0x34,0x12,0x1c,0xd8,0x00,0xfe,0x00,0x00
+
+# GFX12: ds_min_u32 v0, v254 offset:65535        ; encoding: [0xff,0xff,0x1c,0xd8,0x00,0xfe,0x00,0x00]
+0xff,0xff,0x1c,0xd8,0x00,0xfe,0x00,0x00
+
+# GFX12: ds_min_u32 v255, v1                     ; encoding: [0x00,0x00,0x1c,0xd8,0xff,0x01,0x00,0x00]
+0x00,0x00,0x1c,0xd8,0xff,0x01,0x00,0x00
+
+# GFX12: ds_min_u32 v255, v1 offset:4660         ; encoding: [0x34,0x12,0x1c,0xd8,0xff,0x01,0x00,0x00]
+0x34,0x12,0x1c,0xd8,0xff,0x01,0x00,0x00
+
+# GFX12: ds_min_u32 v255, v1 offset:65535        ; encoding: [0xff,0xff,0x1c,0xd8,0xff,0x01,0x00,0x00]
+0xff,0xff,0x1c,0xd8,0xff,0x01,0x00,0x00
+
+# GFX12: ds_min_u32 v255, v254                   ; encoding: [0x00,0x00,0x1c,0xd8,0xff,0xfe,0x00,0x00]
+0x00,0x00,0x1c,0xd8,0xff,0xfe,0x00,0x00
+
+# GFX12: ds_min_u32 v255, v254 offset:4660       ; encoding: [0x34,0x12,0x1c,0xd8,0xff,0xfe,0x00,0x00]
+0x34,0x12,0x1c,0xd8,0xff,0xfe,0x00,0x00
+
+# GFX12: ds_min_u32 v255, v254 offset:65535      ; encoding: [0xff,0xff,0x1c,0xd8,0xff,0xfe,0x00,0x00]
+0xff,0xff,0x1c,0xd8,0xff,0xfe,0x00,0x00
+
+# GFX12: ds_min_u64 v1, v[254:255] offset:65535  ; encoding: [0xff,0xff,0x1c,0xd9,0x01,0xfe,0x00,0x00]
+0xff,0xff,0x1c,0xd9,0x01,0xfe,0x00,0x00
+
+# GFX12: ds_min_u64 v1, v[2:3]                   ; encoding: [0x00,0x00,0x1c,0xd9,0x01,0x02,0x00,0x00]
+0x00,0x00,0x1c,0xd9,0x01,0x02,0x00,0x00
+
+# GFX12: ds_min_u64 v1, v[2:3] offset:4          ; encoding: [0x04,0x00,0x1c,0xd9,0x01,0x02,0x00,0x00]
+0x04,0x00,0x1c,0xd9,0x01,0x02,0x00,0x00
+
+# GFX12: ds_min_u64 v1, v[2:3] offset:65535      ; encoding: [0xff,0xff,0x1c,0xd9,0x01,0x02,0x00,0x00]
+0xff,0xff,0x1c,0xd9,0x01,0x02,0x00,0x00
+
+# GFX12: ds_min_u64 v255, v[2:3] offset:65535    ; encoding: [0xff,0xff,0x1c,0xd9,0xff,0x02,0x00,0x00]
+0xff,0xff,0x1c,0xd9,0xff,0x02,0x00,0x00
+
+# GFX12: ds_mskor_b32 v0, v1, v2                 ; encoding: [0x00,0x00,0x30,0xd8,0x00,0x01,0x02,0x00]
+0x00,0x00,0x30,0xd8,0x00,0x01,0x02,0x00
+
+# GFX12: ds_mskor_b32 v0, v1, v2 offset:4660     ; encoding: [0x34,0x12,0x30,0xd8,0x00,0x01,0x02,0x00]
+0x34,0x12,0x30,0xd8,0x00,0x01,0x02,0x00
+
+# GFX12: ds_mskor_b32 v0, v1, v2 offset:65535    ; encoding: [0xff,0xff,0x30,0xd8,0x00,0x01,0x02,0x00]
+0xff,0xff,0x30,0xd8,0x00,0x01,0x02,0x00
+
+# GFX12: ds_mskor_b32 v0, v254, v253             ; encoding: [0x00,0x00,0x30,0xd8,0x00,0xfe,0xfd,0x00]
+0x00,0x00,0x30,0xd8,0x00,0xfe,0xfd,0x00
+
+# GFX12: ds_mskor_b32 v0, v254, v253 offset:4660 ; encoding: [0x34,0x12,0x30,0xd8,0x00,0xfe,0xfd,0x00]
+0x34,0x12,0x30,0xd8,0x00,0xfe,0xfd,0x00
+
+# GFX12: ds_mskor_b32 v0, v254, v253 offset:65535 ; encoding: [0xff,0xff,0x30,0xd8,0x00,0xfe,0xfd,0x00]
+0xff,0xff,0x30,0xd8,0x00,0xfe,0xfd,0x00
+
+# GFX12: ds_mskor_b32 v255, v1, v253             ; encoding: [0x00,0x00,0x30,0xd8,0xff,0x01,0xfd,0x00]
+0x00,0x00,0x30,0xd8,0xff,0x01,0xfd,0x00
+
+# GFX12: ds_mskor_b32 v255, v1, v253 offset:4660 ; encoding: [0x34,0x12,0x30,0xd8,0xff,0x01,0xfd,0x00]
+0x34,0x12,0x30,0xd8,0xff,0x01,0xfd,0x00
+
+# GFX12: ds_mskor_b32 v255, v1, v253 offset:65535 ; encoding: [0xff,0xff,0x30,0xd8,0xff,0x01,0xfd,0x00]
+0xff,0xff,0x30,0xd8,0xff,0x01,0xfd,0x00
+
+# GFX12: ds_mskor_b32 v255, v254, v2             ; encoding: [0x00,0x00,0x30,0xd8,0xff,0xfe,0x02,0x00]
+0x00,0x00,0x30,0xd8,0xff,0xfe,0x02,0x00
+
+# GFX12: ds_mskor_b32 v255, v254, v2 offset:4660 ; encoding: [0x34,0x12,0x30,0xd8,0xff,0xfe,0x02,0x00]
+0x34,0x12,0x30,0xd8,0xff,0xfe,0x02,0x00
+
+# GFX12: ds_mskor_b32 v255, v254, v2 offset:65535 ; encoding: [0xff,0xff,0x30,0xd8,0xff,0xfe,0x02,0x00]
+0xff,0xff,0x30,0xd8,0xff,0xfe,0x02,0x00
+
+# GFX12: ds_mskor_b32 v255, v254, v253           ; encoding: [0x00,0x00,0x30,0xd8,0xff,0xfe,0xfd,0x00]
+0x00,0x00,0x30,0xd8,0xff,0xfe,0xfd,0x00
+
+# GFX12: ds_mskor_b32 v255, v254, v253 offset:4660 ; encoding: [0x34,0x12,0x30,0xd8,0xff,0xfe,0xfd,0x00]
+0x34,0x12,0x30,0xd8,0xff,0xfe,0xfd,0x00
+
+# GFX12: ds_mskor_b32 v255, v254, v253 offset:65535 ; encoding: [0xff,0xff,0x30,0xd8,0xff,0xfe,0xfd,0x00]
+0xff,0xff,0x30,0xd8,0xff,0xfe,0xfd,0x00
+
+# GFX12: ds_mskor_b64 v1, v[254:255], v[3:4] offset:65535 ; encoding: [0xff,0xff,0x30,0xd9,0x01,0xfe,0x03,0x00]
+0xff,0xff,0x30,0xd9,0x01,0xfe,0x03,0x00
+
+# GFX12: ds_mskor_b64 v1, v[2:3], v[254:255] offset:65535 ; encoding: [0xff,0xff,0x30,0xd9,0x01,0x02,0xfe,0x00]
+0xff,0xff,0x30,0xd9,0x01,0x02,0xfe,0x00
+
+# GFX12: ds_mskor_b64 v1, v[2:3], v[3:4]         ; encoding: [0x00,0x00,0x30,0xd9,0x01,0x02,0x03,0x00]
+0x00,0x00,0x30,0xd9,0x01,0x02,0x03,0x00
+
+# GFX12: ds_mskor_b64 v1, v[2:3], v[3:4] offset:4 ; encoding: [0x04,0x00,0x30,0xd9,0x01,0x02,0x03,0x00]
+0x04,0x00,0x30,0xd9,0x01,0x02,0x03,0x00
+
+# GFX12: ds_mskor_b64 v1, v[2:3], v[3:4] offset:65535 ; encoding: [0xff,0xff,0x30,0xd9,0x01,0x02,0x03,0x00]
+0xff,0xff,0x30,0xd9,0x01,0x02,0x03,0x00
+
+# GFX12: ds_mskor_b64 v255, v[2:3], v[3:4] offset:65535 ; encoding: [0xff,0xff,0x30,0xd9,0xff,0x02,0x03,0x00]
+0xff,0xff,0x30,0xd9,0xff,0x02,0x03,0x00
+
+# GFX12: ds_mskor_rtn_b32 v0, v1, v2, v3         ; encoding: [0x00,0x00,0xb0,0xd8,0x01,0x02,0x03,0x00]
+0x00,0x00,0xb0,0xd8,0x01,0x02,0x03,0x00
+
+# GFX12: ds_mskor_rtn_b32 v0, v1, v2, v3 offset:4660 ; encoding: [0x34,0x12,0xb0,0xd8,0x01,0x02,0x03,0x00]
+0x34,0x12,0xb0,0xd8,0x01,0x02,0x03,0x00
+
+# GFX12: ds_mskor_rtn_b32 v0, v1, v2, v3 offset:65535 ; encoding: [0xff,0xff,0xb0,0xd8,0x01,0x02,0x03,0x00]
+0xff,0xff,0xb0,0xd8,0x01,0x02,0x03,0x00
+
+# GFX12: ds_mskor_rtn_b32 v0, v254, v253, v252   ; encoding: [0x00,0x00,0xb0,0xd8,0xfe,0xfd,0xfc,0x00]
+0x00,0x00,0xb0,0xd8,0xfe,0xfd,0xfc,0x00
+
+# GFX12: ds_mskor_rtn_b32 v0, v254, v253, v252 offset:4660 ; encoding: [0x34,0x12,0xb0,0xd8,0xfe,0xfd,0xfc,0x00]
+0x34,0x12,0xb0,0xd8,0xfe,0xfd,0xfc,0x00
+
+# GFX12: ds_mskor_rtn_b32 v0, v254, v253, v252 offset:65535 ; encoding: [0xff,0xff,0xb0,0xd8,0xfe,0xfd,0xfc,0x00]
+0xff,0xff,0xb0,0xd8,0xfe,0xfd,0xfc,0x00
+
+# GFX12: ds_mskor_rtn_b32 v255, v1, v253, v252   ; encoding: [0x00,0x00,0xb0,0xd8,0x01,0xfd,0xfc,0xff]
+0x00,0x00,0xb0,0xd8,0x01,0xfd,0xfc,0xff
+
+# GFX12: ds_mskor_rtn_b32 v255, v1, v253, v252 offset:4660 ; encoding: [0x34,0x12,0xb0,0xd8,0x01,0xfd,0xfc,0xff]
+0x34,0x12,0xb0,0xd8,0x01,0xfd,0xfc,0xff
+
+# GFX12: ds_mskor_rtn_b32 v255, v1, v253, v252 offset:65535 ; encoding: [0xff,0xff,0xb0,0xd8,0x01,0xfd,0xfc,0xff]
+0xff,0xff,0xb0,0xd8,0x01,0xfd,0xfc,0xff
+
+# GFX12: ds_mskor_rtn_b32 v255, v254, v2, v252   ; encoding: [0x00,0x00,0xb0,0xd8,0xfe,0x02,0xfc,0xff]
+0x00,0x00,0xb0,0xd8,0xfe,0x02,0xfc,0xff
+
+# GFX12: ds_mskor_rtn_b32 v255, v254, v2, v252 offset:4660 ; encoding: [0x34,0x12,0xb0,0xd8,0xfe,0x02,0xfc,0xff]
+0x34,0x12,0xb0,0xd8,0xfe,0x02,0xfc,0xff
+
+# GFX12: ds_mskor_rtn_b32 v255, v254, v2, v252 offset:65535 ; encoding: [0xff,0xff,0xb0,0xd8,0xfe,0x02,0xfc,0xff]
+0xff,0xff,0xb0,0xd8,0xfe,0x02,0xfc,0xff
+
+# GFX12: ds_mskor_rtn_b32 v255, v254, v253, v252 ; encoding: [0x00,0x00,0xb0,0xd8,0xfe,0xfd,0xfc,0xff]
+0x00,0x00,0xb0,0xd8,0xfe,0xfd,0xfc,0xff
+
+# GFX12: ds_mskor_rtn_b32 v255, v254, v253, v252 offset:4660 ; encoding: [0x34,0x12,0xb0,0xd8,0xfe,0xfd,0xfc,0xff]
+0x34,0x12,0xb0,0xd8,0xfe,0xfd,0xfc,0xff
+
+# GFX12: ds_mskor_rtn_b32 v255, v254, v253, v252 offset:65535 ; encoding: [0xff,0xff,0xb0,0xd8,0xfe,0xfd,0xfc,0xff]
+0xff,0xff,0xb0,0xd8,0xfe,0xfd,0xfc,0xff
+
+# GFX12: ds_mskor_rtn_b32 v255, v254, v253, v3   ; encoding: [0x00,0x00,0xb0,0xd8,0xfe,0xfd,0x03,0xff]
+0x00,0x00,0xb0,0xd8,0xfe,0xfd,0x03,0xff
+
+# GFX12: ds_mskor_rtn_b32 v255, v254, v253, v3 offset:4660 ; encoding: [0x34,0x12,0xb0,0xd8,0xfe,0xfd,0x03,0xff]
+0x34,0x12,0xb0,0xd8,0xfe,0xfd,0x03,0xff
+
+# GFX12: ds_mskor_rtn_b32 v255, v254, v253, v3 offset:65535 ; encoding: [0xff,0xff,0xb0,0xd8,0xfe,0xfd,0x03,0xff]
+0xff,0xff,0xb0,0xd8,0xfe,0xfd,0x03,0xff
+
+# GFX12: ds_mskor_rtn_b64 v[254:255], v1, v[2:3], v[3:4] offset:65535 ; encoding: [0xff,0xff,0xb0,0xd9,0x01,0x02,0x03,0xfe]
+0xff,0xff,0xb0,0xd9,0x01,0x02,0x03,0xfe
+
+# GFX12: ds_mskor_rtn_b64 v[5:6], v1, v[254:255], v[3:4] offset:65535 ; encoding: [0xff,0xff,0xb0,0xd9,0x01,0xfe,0x03,0x05]
+0xff,0xff,0xb0,0xd9,0x01,0xfe,0x03,0x05
+
+# GFX12: ds_mskor_rtn_b64 v[5:6], v1, v[2:3], v[254:255] offset:65535 ; encoding: [0xff,0xff,0xb0,0xd9,0x01,0x02,0xfe,0x05]
+0xff,0xff,0xb0,0xd9,0x01,0x02,0xfe,0x05
+
+# GFX12: ds_mskor_rtn_b64 v[5:6], v1, v[2:3], v[3:4] ; encoding: [0x00,0x00,0xb0,0xd9,0x01,0x02,0x03,0x05]
+0x00,0x00,0xb0,0xd9,0x01,0x02,0x03,0x05
+
+# GFX12: ds_mskor_rtn_b64 v[5:6], v1, v[2:3], v[3:4] offset:4 ; encoding: [0x04,0x00,0xb0,0xd9,0x01,0x02,0x03,0x05]
+0x04,0x00,0xb0,0xd9,0x01,0x02,0x03,0x05
+
+# GFX12: ds_mskor_rtn_b64 v[5:6], v1, v[2:3], v[3:4] offset:65535 ; encoding: [0xff,0xff,0xb0,0xd9,0x01,0x02,0x03,0x05]
+0xff,0xff,0xb0,0xd9,0x01,0x02,0x03,0x05
+
+# GFX12: ds_mskor_rtn_b64 v[5:6], v255, v[2:3], v[3:4] offset:65535 ; encoding: [0xff,0xff,0xb0,0xd9,0xff,0x02,0x03,0x05]
+0xff,0xff,0xb0,0xd9,0xff,0x02,0x03,0x05
+
+# GFX12: ds_nop                                  ; encoding: [0x00,0x00,0x50,0xd8,0x00,0x00,0x00,0x00]
+0x00,0x00,0x50,0xd8,0x00,0x00,0x00,0x00
+
+# GFX12: ds_or_b32 v0, v1                        ; encoding: [0x00,0x00,0x28,0xd8,0x00,0x01,0x00,0x00]
+0x00,0x00,0x28,0xd8,0x00,0x01,0x00,0x00
+
+# GFX12: ds_or_b32 v0, v1 offset:4660            ; encoding: [0x34,0x12,0x28,0xd8,0x00,0x01,0x00,0x00]
+0x34,0x12,0x28,0xd8,0x00,0x01,0x00,0x00
+
+# GFX12: ds_or_b32 v0, v1 offset:65535           ; encoding: [0xff,0xff,0x28,0xd8,0x00,0x01,0x00,0x00]
+0xff,0xff,0x28,0xd8,0x00,0x01,0x00,0x00
+
+# GFX12: ds_or_b32 v0, v254                      ; encoding: [0x00,0x00,0x28,0xd8,0x00,0xfe,0x00,0x00]
+0x00,0x00,0x28,0xd8,0x00,0xfe,0x00,0x00
+
+# GFX12: ds_or_b32 v0, v254 offset:4660          ; encoding: [0x34,0x12,0x28,0xd8,0x00,0xfe,0x00,0x00]
+0x34,0x12,0x28,0xd8,0x00,0xfe,0x00,0x00
+
+# GFX12: ds_or_b32 v0, v254 offset:65535         ; encoding: [0xff,0xff,0x28,0xd8,0x00,0xfe,0x00,0x00]
+0xff,0xff,0x28,0xd8,0x00,0xfe,0x00,0x00
+
+# GFX12: ds_or_b32 v255, v1                      ; encoding: [0x00,0x00,0x28,0xd8,0xff,0x01,0x00,0x00]
+0x00,0x00,0x28,0xd8,0xff,0x01,0x00,0x00
+
+# GFX12: ds_or_b32 v255, v1 offset:4660          ; encoding: [0x34,0x12,0x28,0xd8,0xff,0x01,0x00,0x00]
+0x34,0x12,0x28,0xd8,0xff,0x01,0x00,0x00
+
+# GFX12: ds_or_b32 v255, v1 offset:65535         ; encoding: [0xff,0xff,0x28,0xd8,0xff,0x01,0x00,0x00]
+0xff,0xff,0x28,0xd8,0xff,0x01,0x00,0x00
+
+# GFX12: ds_or_b32 v255, v254                    ; encoding: [0x00,0x00,0x28,0xd8,0xff,0xfe,0x00,0x00]
+0x00,0x00,0x28,0xd8,0xff,0xfe,0x00,0x00
+
+# GFX12: ds_or_b32 v255, v254 offset:4660        ; encoding: [0x34,0x12,0x28,0xd8,0xff,0xfe,0x00,0x00]
+0x34,0x12,0x28,0xd8,0xff,0xfe,0x00,0x00
+
+# GFX12: ds_or_b32 v255, v254 offset:65535       ; encoding: [0xff,0xff,0x28,0xd8,0xff,0xfe,0x00,0x00]
+0xff,0xff,0x28,0xd8,0xff,0xfe,0x00,0x00
+
+# GFX12: ds_or_b64 v1, v[254:255] offset:65535   ; encoding: [0xff,0xff,0x28,0xd9,0x01,0xfe,0x00,0x00]
+0xff,0xff,0x28,0xd9,0x01,0xfe,0x00,0x00
+
+# GFX12: ds_or_b64 v1, v[2:3]                    ; encoding: [0x00,0x00,0x28,0xd9,0x01,0x02,0x00,0x00]
+0x00,0x00,0x28,0xd9,0x01,0x02,0x00,0x00
+
+# GFX12: ds_or_b64 v1, v[2:3] offset:4           ; encoding: [0x04,0x00,0x28,0xd9,0x01,0x02,0x00,0x00]
+0x04,0x00,0x28,0xd9,0x01,0x02,0x00,0x00
+
+# GFX12: ds_or_b64 v1, v[2:3] offset:65535       ; encoding: [0xff,0xff,0x28,0xd9,0x01,0x02,0x00,0x00]
+0xff,0xff,0x28,0xd9,0x01,0x02,0x00,0x00
+
+# GFX12: ds_or_b64 v255, v[2:3] offset:65535     ; encoding: [0xff,0xff,0x28,0xd9,0xff,0x02,0x00,0x00]
+0xff,0xff,0x28,0xd9,0xff,0x02,0x00,0x00
+
+# GFX12: ds_or_rtn_b32 v0, v1, v2                ; encoding: [0x00,0x00,0xa8,0xd8,0x01,0x02,0x00,0x00]
+0x00,0x00,0xa8,0xd8,0x01,0x02,0x00,0x00
+
+# GFX12: ds_or_rtn_b32 v0, v1, v2 offset:4660    ; encoding: [0x34,0x12,0xa8,0xd8,0x01,0x02,0x00,0x00]
+0x34,0x12,0xa8,0xd8,0x01,0x02,0x00,0x00
+
+# GFX12: ds_or_rtn_b32 v0, v1, v2 offset:65535   ; encoding: [0xff,0xff,0xa8,0xd8,0x01,0x02,0x00,0x00]
+0xff,0xff,0xa8,0xd8,0x01,0x02,0x00,0x00
+
+# GFX12: ds_or_rtn_b32 v0, v254, v253            ; encoding: [0x00,0x00,0xa8,0xd8,0xfe,0xfd,0x00,0x00]
+0x00,0x00,0xa8,0xd8,0xfe,0xfd,0x00,0x00
+
+# GFX12: ds_or_rtn_b32 v0, v254, v253 offset:4660 ; encoding: [0x34,0x12,0xa8,0xd8,0xfe,0xfd,0x00,0x00]
+0x34,0x12,0xa8,0xd8,0xfe,0xfd,0x00,0x00
+
+# GFX12: ds_or_rtn_b32 v0, v254, v253 offset:65535 ; encoding: [0xff,0xff,0xa8,0xd8,0xfe,0xfd,0x00,0x00]
+0xff,0xff,0xa8,0xd8,0xfe,0xfd,0x00,0x00
+
+# GFX12: ds_or_rtn_b32 v255, v1, v253            ; encoding: [0x00,0x00,0xa8,0xd8,0x01,0xfd,0x00,0xff]
+0x00,0x00,0xa8,0xd8,0x01,0xfd,0x00,0xff
+
+# GFX12: ds_or_rtn_b32 v255, v1, v253 offset:4660 ; encoding: [0x34,0x12,0xa8,0xd8,0x01,0xfd,0x00,0xff]
+0x34,0x12,0xa8,0xd8,0x01,0xfd,0x00,0xff
+
+# GFX12: ds_or_rtn_b32 v255, v1, v253 offset:65535 ; encoding: [0xff,0xff,0xa8,0xd8,0x01,0xfd,0x00,0xff]
+0xff,0xff,0xa8,0xd8,0x01,0xfd,0x00,0xff
+
+# GFX12: ds_or_rtn_b32 v255, v254, v2            ; encoding: [0x00,0x00,0xa8,0xd8,0xfe,0x02,0x00,0xff]
+0x00,0x00,0xa8,0xd8,0xfe,0x02,0x00,0xff
+
+# GFX12: ds_or_rtn_b32 v255, v254, v2 offset:4660 ; encoding: [0x34,0x12,0xa8,0xd8,0xfe,0x02,0x00,0xff]
+0x34,0x12,0xa8,0xd8,0xfe,0x02,0x00,0xff
+
+# GFX12: ds_or_rtn_b32 v255, v254, v2 offset:65535 ; encoding: [0xff,0xff,0xa8,0xd8,0xfe,0x02,0x00,0xff]
+0xff,0xff,0xa8,0xd8,0xfe,0x02,0x00,0xff
+
+# GFX12: ds_or_rtn_b32 v255, v254, v253          ; encoding: [0x00,0x00,0xa8,0xd8,0xfe,0xfd,0x00,0xff]
+0x00,0x00,0xa8,0xd8,0xfe,0xfd,0x00,0xff
+
+# GFX12: ds_or_rtn_b32 v255, v254, v253 offset:4660 ; encoding: [0x34,0x12,0xa8,0xd8,0xfe,0xfd,0x00,0xff]
+0x34,0x12,0xa8,0xd8,0xfe,0xfd,0x00,0xff
+
+# GFX12: ds_or_rtn_b32 v255, v254, v253 offset:65535 ; encoding: [0xff,0xff,0xa8,0xd8,0xfe,0xfd,0x00,0xff]
+0xff,0xff,0xa8,0xd8,0xfe,0xfd,0x00,0xff
+
+# GFX12: ds_or_rtn_b64 v[254:255], v1, v[2:3] offset:65535 ; encoding: [0xff,0xff,0xa8,0xd9,0x01,0x02,0x00,0xfe]
+0xff,0xff,0xa8,0xd9,0x01,0x02,0x00,0xfe
+
+# GFX12: ds_or_rtn_b64 v[5:6], v1, v[254:255] offset:65535 ; encoding: [0xff,0xff,0xa8,0xd9,0x01,0xfe,0x00,0x05]
+0xff,0xff,0xa8,0xd9,0x01,0xfe,0x00,0x05
+
+# GFX12: ds_or_rtn_b64 v[5:6], v1, v[2:3]        ; encoding: [0x00,0x00,0xa8,0xd9,0x01,0x02,0x00,0x05]
+0x00,0x00,0xa8,0xd9,0x01,0x02,0x00,0x05
+
+# GFX12: ds_or_rtn_b64 v[5:6], v1, v[2:3] offset:4 ; encoding: [0x04,0x00,0xa8,0xd9,0x01,0x02,0x00,0x05]
+0x04,0x00,0xa8,0xd9,0x01,0x02,0x00,0x05
+
+# GFX12: ds_or_rtn_b64 v[5:6], v1, v[2:3] offset:65535 ; encoding: [0xff,0xff,0xa8,0xd9,0x01,0x02,0x00,0x05]
+0xff,0xff,0xa8,0xd9,0x01,0x02,0x00,0x05
+
+# GFX12: ds_or_rtn_b64 v[5:6], v255, v[2:3] offset:65535 ; encoding: [0xff,0xff,0xa8,0xd9,0xff,0x02,0x00,0x05]
+0xff,0xff,0xa8,0xd9,0xff,0x02,0x00,0x05
+
+# GFX12: ds_permute_b32 v0, v1, v2               ; encoding: [0x00,0x00,0xc8,0xda,0x01,0x02,0x00,0x00]
+0x00,0x00,0xc8,0xda,0x01,0x02,0x00,0x00
+
+# GFX12: ds_permute_b32 v0, v1, v2 offset:4660   ; encoding: [0x34,0x12,0xc8,0xda,0x01,0x02,0x00,0x00]
+0x34,0x12,0xc8,0xda,0x01,0x02,0x00,0x00
+
+# GFX12: ds_permute_b32 v0, v1, v2 offset:65535  ; encoding: [0xff,0xff,0xc8,0xda,0x01,0x02,0x00,0x00]
+0xff,0xff,0xc8,0xda,0x01,0x02,0x00,0x00
+
+# GFX12: ds_permute_b32 v0, v254, v253           ; encoding: [0x00,0x00,0xc8,0xda,0xfe,0xfd,0x00,0x00]
+0x00,0x00,0xc8,0xda,0xfe,0xfd,0x00,0x00
+
+# GFX12: ds_permute_b32 v0, v254, v253 offset:4660 ; encoding: [0x34,0x12,0xc8,0xda,0xfe,0xfd,0x00,0x00]
+0x34,0x12,0xc8,0xda,0xfe,0xfd,0x00,0x00
+
+# GFX12: ds_permute_b32 v0, v254, v253 offset:65535 ; encoding: [0xff,0xff,0xc8,0xda,0xfe,0xfd,0x00,0x00]
+0xff,0xff,0xc8,0xda,0xfe,0xfd,0x00,0x00
+
+# GFX12: ds_permute_b32 v255, v1, v253           ; encoding: [0x00,0x00,0xc8,0xda,0x01,0xfd,0x00,0xff]
+0x00,0x00,0xc8,0xda,0x01,0xfd,0x00,0xff
+
+# GFX12: ds_permute_b32 v255, v1, v253 offset:4660 ; encoding: [0x34,0x12,0xc8,0xda,0x01,0xfd,0x00,0xff]
+0x34,0x12,0xc8,0xda,0x01,0xfd,0x00,0xff
+
+# GFX12: ds_permute_b32 v255, v1, v253 offset:65535 ; encoding: [0xff,0xff,0xc8,0xda,0x01,0xfd,0x00,0xff]
+0xff,0xff,0xc8,0xda,0x01,0xfd,0x00,0xff
+
+# GFX12: ds_permute_b32 v255, v254, v2           ; encoding: [0x00,0x00,0xc8,0xda,0xfe,0x02,0x00,0xff]
+0x00,0x00,0xc8,0xda,0xfe,0x02,0x00,0xff
+
+# GFX12: ds_permute_b32 v255, v254, v2 offset:4660 ; encoding: [0x34,0x12,0xc8,0xda,0xfe,0x02,0x00,0xff]
+0x34,0x12,0xc8,0xda,0xfe,0x02,0x00,0xff
+
+# GFX12: ds_permute_b32 v255, v254, v2 offset:65535 ; encoding: [0xff,0xff,0xc8,0xda,0xfe,0x02,0x00,0xff]
+0xff,0xff,0xc8,0xda,0xfe,0x02,0x00,0xff
+
+# GFX12: ds_permute_b32 v255, v254, v253         ; encoding: [0x00,0x00,0xc8,0xda,0xfe,0xfd,0x00,0xff]
+0x00,0x00,0xc8,0xda,0xfe,0xfd,0x00,0xff
+
+# GFX12: ds_permute_b32 v255, v254, v253 offset:4660 ; encoding: [0x34,0x12,0xc8,0xda,0xfe,0xfd,0x00,0xff]
+0x34,0x12,0xc8,0xda,0xfe,0xfd,0x00,0xff
+
+# GFX12: ds_permute_b32 v255, v254, v253 offset:65535 ; encoding: [0xff,0xff,0xc8,0xda,0xfe,0xfd,0x00,0xff]
+0xff,0xff,0xc8,0xda,0xfe,0xfd,0x00,0xff
+
+# GFX12: ds_load_2addr_b32 v[254:255], v1 offset0:127 offset1:255 ; encoding: [0x7f,0xff,0xdc,0xd8,0x01,0x00,0x00,0xfe]
+0x7f,0xff,0xdc,0xd8,0x01,0x00,0x00,0xfe
+
+# GFX12: ds_load_2addr_b32 v[5:6], v1 offset0:127 ; encoding: [0x7f,0x00,0xdc,0xd8,0x01,0x00,0x00,0x05]
+0x7f,0x00,0xdc,0xd8,0x01,0x00,0x00,0x05
+
+# GFX12: ds_load_2addr_b32 v[5:6], v1 offset0:127 offset1:1 ; encoding: [0x7f,0x01,0xdc,0xd8,0x01,0x00,0x00,0x05]
+0x7f,0x01,0xdc,0xd8,0x01,0x00,0x00,0x05
+
+# GFX12: ds_load_2addr_b32 v[5:6], v1 offset0:127 offset1:255 ; encoding: [0x7f,0xff,0xdc,0xd8,0x01,0x00,0x00,0x05]
+0x7f,0xff,0xdc,0xd8,0x01,0x00,0x00,0x05
+
+# GFX12: ds_load_2addr_b32 v[5:6], v1 offset0:16 offset1:255 ; encoding: [0x10,0xff,0xdc,0xd8,0x01,0x00,0x00,0x05]
+0x10,0xff,0xdc,0xd8,0x01,0x00,0x00,0x05
+
+# GFX12: ds_load_2addr_b32 v[5:6], v1 offset1:255 ; encoding: [0x00,0xff,0xdc,0xd8,0x01,0x00,0x00,0x05]
+0x00,0xff,0xdc,0xd8,0x01,0x00,0x00,0x05
+
+# GFX12: ds_load_2addr_b32 v[5:6], v255 offset0:127 offset1:255 ; encoding: [0x7f,0xff,0xdc,0xd8,0xff,0x00,0x00,0x05]
+0x7f,0xff,0xdc,0xd8,0xff,0x00,0x00,0x05
+
+# GFX12: ds_load_2addr_b64 v[252:255], v1 offset0:127 offset1:255 ; encoding: [0x7f,0xff,0xdc,0xd9,0x01,0x00,0x00,0xfc]
+0x7f,0xff,0xdc,0xd9,0x01,0x00,0x00,0xfc
+
+# GFX12: ds_load_2addr_b64 v[5:8], v1 offset0:127 ; encoding: [0x7f,0x00,0xdc,0xd9,0x01,0x00,0x00,0x05]
+0x7f,0x00,0xdc,0xd9,0x01,0x00,0x00,0x05
+
+# GFX12: ds_load_2addr_b64 v[5:8], v1 offset0:127 offset1:1 ; encoding: [0x7f,0x01,0xdc,0xd9,0x01,0x00,0x00,0x05]
+0x7f,0x01,0xdc,0xd9,0x01,0x00,0x00,0x05
+
+# GFX12: ds_load_2addr_b64 v[5:8], v1 offset0:127 offset1:255 ; encoding: [0x7f,0xff,0xdc,0xd9,0x01,0x00,0x00,0x05]
+0x7f,0xff,0xdc,0xd9,0x01,0x00,0x00,0x05
+
+# GFX12: ds_load_2addr_b64 v[5:8], v1 offset0:16 offset1:255 ; encoding: [0x10,0xff,0xdc,0xd9,0x01,0x00,0x00,0x05]
+0x10,0xff,0xdc,0xd9,0x01,0x00,0x00,0x05
+
+# GFX12: ds_load_2addr_b64 v[5:8], v1 offset1:255 ; encoding: [0x00,0xff,0xdc,0xd9,0x01,0x00,0x00,0x05]
+0x00,0xff,0xdc,0xd9,0x01,0x00,0x00,0x05
+
+# GFX12: ds_load_2addr_b64 v[5:8], v255 offset0:127 offset1:255 ; encoding: [0x7f,0xff,0xdc,0xd9,0xff,0x00,0x00,0x05]
+0x7f,0xff,0xdc,0xd9,0xff,0x00,0x00,0x05
+
+# GFX12: ds_load_2addr_stride64_b32 v[254:255], v1 offset0:127 offset1:255 ; encoding: [0x7f,0xff,0xe0,0xd8,0x01,0x00,0x00,0xfe]
+0x7f,0xff,0xe0,0xd8,0x01,0x00,0x00,0xfe
+
+# GFX12: ds_load_2addr_stride64_b32 v[5:6], v1 offset0:127 ; encoding: [0x7f,0x00,0xe0,0xd8,0x01,0x00,0x00,0x05]
+0x7f,0x00,0xe0,0xd8,0x01,0x00,0x00,0x05
+
+# GFX12: ds_load_2addr_stride64_b32 v[5:6], v1 offset0:127 offset1:1 ; encoding: [0x7f,0x01,0xe0,0xd8,0x01,0x00,0x00,0x05]
+0x7f,0x01,0xe0,0xd8,0x01,0x00,0x00,0x05
+
+# GFX12: ds_load_2addr_stride64_b32 v[5:6], v1 offset0:127 offset1:255 ; encoding: [0x7f,0xff,0xe0,0xd8,0x01,0x00,0x00,0x05]
+0x7f,0xff,0xe0,0xd8,0x01,0x00,0x00,0x05
+
+# GFX12: ds_load_2addr_stride64_b32 v[5:6], v1 offset0:16 offset1:255 ; encoding: [0x10,0xff,0xe0,0xd8,0x01,0x00,0x00,0x05]
+0x10,0xff,0xe0,0xd8,0x01,0x00,0x00,0x05
+
+# GFX12: ds_load_2addr_stride64_b32 v[5:6], v1 offset1:255 ; encoding: [0x00,0xff,0xe0,0xd8,0x01,0x00,0x00,0x05]
+0x00,0xff,0xe0,0xd8,0x01,0x00,0x00,0x05
+
+# GFX12: ds_load_2addr_stride64_b32 v[5:6], v255 offset0:127 offset1:255 ; encoding: [0x7f,0xff,0xe0,0xd8,0xff,0x00,0x00,0x05]
+0x7f,0xff,0xe0,0xd8,0xff,0x00,0x00,0x05
+
+# GFX12: ds_load_2addr_stride64_b64 v[252:255], v1 offset0:127 offset1:255 ; encoding: [0x7f,0xff,0xe0,0xd9,0x01,0x00,0x00,0xfc]
+0x7f,0xff,0xe0,0xd9,0x01,0x00,0x00,0xfc
+
+# GFX12: ds_load_2addr_stride64_b64 v[5:8], v1 offset0:127 ; encoding: [0x7f,0x00,0xe0,0xd9,0x01,0x00,0x00,0x05]
+0x7f,0x00,0xe0,0xd9,0x01,0x00,0x00,0x05
+
+# GFX12: ds_load_2addr_stride64_b64 v[5:8], v1 offset0:127 offset1:1 ; encoding: [0x7f,0x01,0xe0,0xd9,0x01,0x00,0x00,0x05]
+0x7f,0x01,0xe0,0xd9,0x01,0x00,0x00,0x05
+
+# GFX12: ds_load_2addr_stride64_b64 v[5:8], v1 offset0:127 offset1:255 ; encoding: [0x7f,0xff,0xe0,0xd9,0x01,0x00,0x00,0x05]
+0x7f,0xff,0xe0,0xd9,0x01,0x00,0x00,0x05
+
+# GFX12: ds_load_2addr_stride64_b64 v[5:8], v1 offset0:16 offset1:255 ; encoding: [0x10,0xff,0xe0,0xd9,0x01,0x00,0x00,0x05]
+0x10,0xff,0xe0,0xd9,0x01,0x00,0x00,0x05
+
+# GFX12: ds_load_2addr_stride64_b64 v[5:8], v1 offset1:255 ; encoding: [0x00,0xff,0xe0,0xd9,0x01,0x00,0x00,0x05]
+0x00,0xff,0xe0,0xd9,0x01,0x00,0x00,0x05
+
+# GFX12: ds_load_2addr_stride64_b64 v[5:8], v255 offset0:127 offset1:255 ; encoding: [0x7f,0xff,0xe0,0xd9,0xff,0x00,0x00,0x05]
+0x7f,0xff,0xe0,0xd9,0xff,0x00,0x00,0x05
+
+# GFX12: ds_load_addtid_b32 v255 offset:65535    ; encoding: [0xff,0xff,0xc4,0xda,0x00,0x00,0x00,0xff]
+0xff 0xff 0xc4 0xda 0x00 0x00 0x00 0xff
+
+# GFX12: ds_load_addtid_b32 v5                   ; encoding: [0x00,0x00,0xc4,0xda,0x00,0x00,0x00,0x05]
+0x00 0x00 0xc4 0xda 0x00 0x00 0x00 0x05
+
+# GFX12: ds_load_addtid_b32 v5 offset:4          ; encoding: [0x04,0x00,0xc4,0xda,0x00,0x00,0x00,0x05]
+0x04 0x00 0xc4 0xda 0x00 0x00 0x00 0x05
+
+# GFX12: ds_load_addtid_b32 v5 offset:65535      ; encoding: [0xff,0xff,0xc4,0xda,0x00,0x00,0x00,0x05]
+0xff 0xff 0xc4 0xda 0x00 0x00 0x00 0x05
+
+# GFX12: ds_load_b128 v[252:255], v1 offset:65535 ; encoding: [0xff,0xff,0xfc,0xdb,0x01,0x00,0x00,0xfc]
+0xff,0xff,0xfc,0xdb,0x01,0x00,0x00,0xfc
+
+# GFX12: ds_load_b128 v[5:8], v1                 ; encoding: [0x00,0x00,0xfc,0xdb,0x01,0x00,0x00,0x05]
+0x00,0x00,0xfc,0xdb,0x01,0x00,0x00,0x05
+
+# GFX12: ds_load_b128 v[5:8], v1 offset:4        ; encoding: [0x04,0x00,0xfc,0xdb,0x01,0x00,0x00,0x05]
+0x04,0x00,0xfc,0xdb,0x01,0x00,0x00,0x05
+
+# GFX12: ds_load_b128 v[5:8], v1 offset:65535    ; encoding: [0xff,0xff,0xfc,0xdb,0x01,0x00,0x00,0x05]
+0xff,0xff,0xfc,0xdb,0x01,0x00,0x00,0x05
+
+# GFX12: ds_load_b128 v[5:8], v255 offset:65535  ; encoding: [0xff,0xff,0xfc,0xdb,0xff,0x00,0x00,0x05]
+0xff,0xff,0xfc,0xdb,0xff,0x00,0x00,0x05
+
+# GFX12: ds_load_b32 v255, v1 offset:65535       ; encoding: [0xff,0xff,0xd8,0xd8,0x01,0x00,0x00,0xff]
+0xff,0xff,0xd8,0xd8,0x01,0x00,0x00,0xff
+
+# GFX12: ds_load_b32 v5, v1                      ; encoding: [0x00,0x00,0xd8,0xd8,0x01,0x00,0x00,0x05]
+0x00,0x00,0xd8,0xd8,0x01,0x00,0x00,0x05
+
+# GFX12: ds_load_b32 v5, v1 offset:4             ; encoding: [0x04,0x00,0xd8,0xd8,0x01,0x00,0x00,0x05]
+0x04,0x00,0xd8,0xd8,0x01,0x00,0x00,0x05
+
+# GFX12: ds_load_b32 v5, v1 offset:65535         ; encoding: [0xff,0xff,0xd8,0xd8,0x01,0x00,0x00,0x05]
+0xff,0xff,0xd8,0xd8,0x01,0x00,0x00,0x05
+
+# GFX12: ds_load_b32 v5, v255 offset:65535       ; encoding: [0xff,0xff,0xd8,0xd8,0xff,0x00,0x00,0x05]
+0xff,0xff,0xd8,0xd8,0xff,0x00,0x00,0x05
+
+# GFX12: ds_load_b64 v[254:255], v1 offset:65535 ; encoding: [0xff,0xff,0xd8,0xd9,0x01,0x00,0x00,0xfe]
+0xff,0xff,0xd8,0xd9,0x01,0x00,0x00,0xfe
+
+# GFX12: ds_load_b64 v[5:6], v1                  ; encoding: [0x00,0x00,0xd8,0xd9,0x01,0x00,0x00,0x05]
+0x00,0x00,0xd8,0xd9,0x01,0x00,0x00,0x05
+
+# GFX12: ds_load_b64 v[5:6], v1 offset:4         ; encoding: [0x04,0x00,0xd8,0xd9,0x01,0x00,0x00,0x05]
+0x04,0x00,0xd8,0xd9,0x01,0x00,0x00,0x05
+
+# GFX12: ds_load_b64 v[5:6], v1 offset:65535     ; encoding: [0xff,0xff,0xd8,0xd9,0x01,0x00,0x00,0x05]
+0xff,0xff,0xd8,0xd9,0x01,0x00,0x00,0x05
+
+# GFX12: ds_load_b64 v[5:6], v255 offset:65535   ; encoding: [0xff,0xff,0xd8,0xd9,0xff,0x00,0x00,0x05]
+0xff,0xff,0xd8,0xd9,0xff,0x00,0x00,0x05
+
+# GFX12: ds_load_b96 v[253:255], v1 offset:65535 ; encoding: [0xff,0xff,0xf8,0xdb,0x01,0x00,0x00,0xfd]
+0xff,0xff,0xf8,0xdb,0x01,0x00,0x00,0xfd
+
+# GFX12: ds_load_b96 v[5:7], v1                  ; encoding: [0x00,0x00,0xf8,0xdb,0x01,0x00,0x00,0x05]
+0x00,0x00,0xf8,0xdb,0x01,0x00,0x00,0x05
+
+# GFX12: ds_load_b96 v[5:7], v1 offset:4         ; encoding: [0x04,0x00,0xf8,0xdb,0x01,0x00,0x00,0x05]
+0x04,0x00,0xf8,0xdb,0x01,0x00,0x00,0x05
+
+# GFX12: ds_load_b96 v[5:7], v1 offset:65535     ; encoding: [0xff,0xff,0xf8,0xdb,0x01,0x00,0x00,0x05]
+0xff,0xff,0xf8,0xdb,0x01,0x00,0x00,0x05
+
+# GFX12: ds_load_b96 v[5:7], v255 offset:65535   ; encoding: [0xff,0xff,0xf8,0xdb,0xff,0x00,0x00,0x05]
+0xff,0xff,0xf8,0xdb,0xff,0x00,0x00,0x05
+
+# GFX12: ds_load_i16 v255, v1 offset:65535       ; encoding: [0xff,0xff,0xec,0xd8,0x01,0x00,0x00,0xff]
+0xff,0xff,0xec,0xd8,0x01,0x00,0x00,0xff
+
+# GFX12: ds_load_i16 v5, v1                      ; encoding: [0x00,0x00,0xec,0xd8,0x01,0x00,0x00,0x05]
+0x00,0x00,0xec,0xd8,0x01,0x00,0x00,0x05
+
+# GFX12: ds_load_i16 v5, v1 offset:4             ; encoding: [0x04,0x00,0xec,0xd8,0x01,0x00,0x00,0x05]
+0x04,0x00,0xec,0xd8,0x01,0x00,0x00,0x05
+
+# GFX12: ds_load_i16 v5, v1 offset:65535         ; encoding: [0xff,0xff,0xec,0xd8,0x01,0x00,0x00,0x05]
+0xff,0xff,0xec,0xd8,0x01,0x00,0x00,0x05
+
+# GFX12: ds_load_i16 v5, v255 offset:65535       ; encoding: [0xff,0xff,0xec,0xd8,0xff,0x00,0x00,0x05]
+0xff,0xff,0xec,0xd8,0xff,0x00,0x00,0x05
+
+# GFX12: ds_load_i8 v255, v1 offset:65535        ; encoding: [0xff,0xff,0xe4,0xd8,0x01,0x00,0x00,0xff]
+0xff,0xff,0xe4,0xd8,0x01,0x00,0x00,0xff
+
+# GFX12: ds_load_i8 v5, v1                       ; encoding: [0x00,0x00,0xe4,0xd8,0x01,0x00,0x00,0x05]
+0x00,0x00,0xe4,0xd8,0x01,0x00,0x00,0x05
+
+# GFX12: ds_load_i8 v5, v1 offset:4              ; encoding: [0x04,0x00,0xe4,0xd8,0x01,0x00,0x00,0x05]
+0x04,0x00,0xe4,0xd8,0x01,0x00,0x00,0x05
+
+# GFX12: ds_load_i8 v5, v1 offset:65535          ; encoding: [0xff,0xff,0xe4,0xd8,0x01,0x00,0x00,0x05]
+0xff,0xff,0xe4,0xd8,0x01,0x00,0x00,0x05
+
+# GFX12: ds_load_i8 v5, v255 offset:65535        ; encoding: [0xff,0xff,0xe4,0xd8,0xff,0x00,0x00,0x05]
+0xff,0xff,0xe4,0xd8,0xff,0x00,0x00,0x05
+
+# GFX12: ds_load_i8_d16 v255, v1 offset:65535    ; encoding: [0xff,0xff,0x90,0xda,0x01,0x00,0x00,0xff]
+0xff,0xff,0x90,0xda,0x01,0x00,0x00,0xff
+
+# GFX12: ds_load_i8_d16 v5, v1                   ; encoding: [0x00,0x00,0x90,0xda,0x01,0x00,0x00,0x05]
+0x00,0x00,0x90,0xda,0x01,0x00,0x00,0x05
+
+# GFX12: ds_load_i8_d16 v5, v1 offset:4          ; encoding: [0x04,0x00,0x90,0xda,0x01,0x00,0x00,0x05]
+0x04,0x00,0x90,0xda,0x01,0x00,0x00,0x05
+
+# GFX12: ds_load_i8_d16 v5, v1 offset:65535      ; encoding: [0xff,0xff,0x90,0xda,0x01,0x00,0x00,0x05]
+0xff,0xff,0x90,0xda,0x01,0x00,0x00,0x05
+
+# GFX12: ds_load_i8_d16 v5, v255 offset:65535    ; encoding: [0xff,0xff,0x90,0xda,0xff,0x00,0x00,0x05]
+0xff,0xff,0x90,0xda,0xff,0x00,0x00,0x05
+
+# GFX12: ds_load_i8_d16_hi v255, v1 offset:65535 ; encoding: [0xff,0xff,0x94,0xda,0x01,0x00,0x00,0xff]
+0xff,0xff,0x94,0xda,0x01,0x00,0x00,0xff
+
+# GFX12: ds_load_i8_d16_hi v5, v1                ; encoding: [0x00,0x00,0x94,0xda,0x01,0x00,0x00,0x05]
+0x00,0x00,0x94,0xda,0x01,0x00,0x00,0x05
+
+# GFX12: ds_load_i8_d16_hi v5, v1 offset:4       ; encoding: [0x04,0x00,0x94,0xda,0x01,0x00,0x00,0x05]
+0x04,0x00,0x94,0xda,0x01,0x00,0x00,0x05
+
+# GFX12: ds_load_i8_d16_hi v5, v1 offset:65535   ; encoding: [0xff,0xff,0x94,0xda,0x01,0x00,0x00,0x05]
+0xff,0xff,0x94,0xda,0x01,0x00,0x00,0x05
+
+# GFX12: ds_load_i8_d16_hi v5, v255 offset:65535 ; encoding: [0xff,0xff,0x94,0xda,0xff,0x00,0x00,0x05]
+0xff,0xff,0x94,0xda,0xff,0x00,0x00,0x05
+
+# GFX12: ds_load_u16 v255, v1 offset:65535       ; encoding: [0xff,0xff,0xf0,0xd8,0x01,0x00,0x00,0xff]
+0xff,0xff,0xf0,0xd8,0x01,0x00,0x00,0xff
+
+# GFX12: ds_load_u16 v5, v1                      ; encoding: [0x00,0x00,0xf0,0xd8,0x01,0x00,0x00,0x05]
+0x00,0x00,0xf0,0xd8,0x01,0x00,0x00,0x05
+
+# GFX12: ds_load_u16 v5, v1 offset:4             ; encoding: [0x04,0x00,0xf0,0xd8,0x01,0x00,0x00,0x05]
+0x04,0x00,0xf0,0xd8,0x01,0x00,0x00,0x05
+
+# GFX12: ds_load_u16 v5, v1 offset:65535         ; encoding: [0xff,0xff,0xf0,0xd8,0x01,0x00,0x00,0x05]
+0xff,0xff,0xf0,0xd8,0x01,0x00,0x00,0x05
+
+# GFX12: ds_load_u16 v5, v255 offset:65535       ; encoding: [0xff,0xff,0xf0,0xd8,0xff,0x00,0x00,0x05]
+0xff,0xff,0xf0,0xd8,0xff,0x00,0x00,0x05
+
+# GFX12: ds_load_u16_d16 v255, v1 offset:65535   ; encoding: [0xff,0xff,0x98,0xda,0x01,0x00,0x00,0xff]
+0xff,0xff,0x98,0xda,0x01,0x00,0x00,0xff
+
+# GFX12: ds_load_u16_d16 v5, v1                  ; encoding: [0x00,0x00,0x98,0xda,0x01,0x00,0x00,0x05]
+0x00,0x00,0x98,0xda,0x01,0x00,0x00,0x05
+
+# GFX12: ds_load_u16_d16 v5, v1 offset:4         ; encoding: [0x04,0x00,0x98,0xda,0x01,0x00,0x00,0x05]
+0x04,0x00,0x98,0xda,0x01,0x00,0x00,0x05
+
+# GFX12: ds_load_u16_d16 v5, v1 offset:65535     ; encoding: [0xff,0xff,0x98,0xda,0x01,0x00,0x00,0x05]
+0xff,0xff,0x98,0xda,0x01,0x00,0x00,0x05
+
+# GFX12: ds_load_u16_d16 v5, v255 offset:65535   ; encoding: [0xff,0xff,0x98,0xda,0xff,0x00,0x00,0x05]
+0xff,0xff,0x98,0xda,0xff,0x00,0x00,0x05
+
+# GFX12: ds_load_u16_d16_hi v255, v1 offset:65535 ; encoding: [0xff,0xff,0x9c,0xda,0x01,0x00,0x00,0xff]
+0xff,0xff,0x9c,0xda,0x01,0x00,0x00,0xff
+
+# GFX12: ds_load_u16_d16_hi v5, v1               ; encoding: [0x00,0x00,0x9c,0xda,0x01,0x00,0x00,0x05]
+0x00,0x00,0x9c,0xda,0x01,0x00,0x00,0x05
+
+# GFX12: ds_load_u16_d16_hi v5, v1 offset:4      ; encoding: [0x04,0x00,0x9c,0xda,0x01,0x00,0x00,0x05]
+0x04,0x00,0x9c,0xda,0x01,0x00,0x00,0x05
+
+# GFX12: ds_load_u16_d16_hi v5, v1 offset:65535  ; encoding: [0xff,0xff,0x9c,0xda,0x01,0x00,0x00,0x05]
+0xff,0xff,0x9c,0xda,0x01,0x00,0x00,0x05
+
+# GFX12: ds_load_u16_d16_hi v5, v255 offset:65535 ; encoding: [0xff,0xff,0x9c,0xda,0xff,0x00,0x00,0x05]
+0xff,0xff,0x9c,0xda,0xff,0x00,0x00,0x05
+
+# GFX12: ds_load_u8 v255, v1 offset:65535        ; encoding: [0xff,0xff,0xe8,0xd8,0x01,0x00,0x00,0xff]
+0xff,0xff,0xe8,0xd8,0x01,0x00,0x00,0xff
+
+# GFX12: ds_load_u8 v5, v1                       ; encoding: [0x00,0x00,0xe8,0xd8,0x01,0x00,0x00,0x05]
+0x00,0x00,0xe8,0xd8,0x01,0x00,0x00,0x05
+
+# GFX12: ds_load_u8 v5, v1 offset:4              ; encoding: [0x04,0x00,0xe8,0xd8,0x01,0x00,0x00,0x05]
+0x04,0x00,0xe8,0xd8,0x01,0x00,0x00,0x05
+
+# GFX12: ds_load_u8 v5, v1 offset:65535          ; encoding: [0xff,0xff,0xe8,0xd8,0x01,0x00,0x00,0x05]
+0xff,0xff,0xe8,0xd8,0x01,0x00,0x00,0x05
+
+# GFX12: ds_load_u8 v5, v255 offset:65535        ; encoding: [0xff,0xff,0xe8,0xd8,0xff,0x00,0x00,0x05]
+0xff,0xff,0xe8,0xd8,0xff,0x00,0x00,0x05
+
+# GFX12: ds_load_u8_d16 v255, v1 offset:65535    ; encoding: [0xff,0xff,0x88,0xda,0x01,0x00,0x00,0xff]
+0xff,0xff,0x88,0xda,0x01,0x00,0x00,0xff
+
+# GFX12: ds_load_u8_d16 v5, v1                   ; encoding: [0x00,0x00,0x88,0xda,0x01,0x00,0x00,0x05]
+0x00,0x00,0x88,0xda,0x01,0x00,0x00,0x05
+
+# GFX12: ds_load_u8_d16 v5, v1 offset:4          ; encoding: [0x04,0x00,0x88,0xda,0x01,0x00,0x00,0x05]
+0x04,0x00,0x88,0xda,0x01,0x00,0x00,0x05
+
+# GFX12: ds_load_u8_d16 v5, v1 offset:65535      ; encoding: [0xff,0xff,0x88,0xda,0x01,0x00,0x00,0x05]
+0xff,0xff,0x88,0xda,0x01,0x00,0x00,0x05
+
+# GFX12: ds_load_u8_d16 v5, v255 offset:65535    ; encoding: [0xff,0xff,0x88,0xda,0xff,0x00,0x00,0x05]
+0xff,0xff,0x88,0xda,0xff,0x00,0x00,0x05
+
+# GFX12: ds_load_u8_d16_hi v255, v1 offset:65535 ; encoding: [0xff,0xff,0x8c,0xda,0x01,0x00,0x00,0xff]
+0xff,0xff,0x8c,0xda,0x01,0x00,0x00,0xff
+
+# GFX12: ds_load_u8_d16_hi v5, v1                ; encoding: [0x00,0x00,0x8c,0xda,0x01,0x00,0x00,0x05]
+0x00,0x00,0x8c,0xda,0x01,0x00,0x00,0x05
+
+# GFX12: ds_load_u8_d16_hi v5, v1 offset:4       ; encoding: [0x04,0x00,0x8c,0xda,0x01,0x00,0x00,0x05]
+0x04,0x00,0x8c,0xda,0x01,0x00,0x00,0x05
+
+# GFX12: ds_load_u8_d16_hi v5, v1 offset:65535   ; encoding: [0xff,0xff,0x8c,0xda,0x01,0x00,0x00,0x05]
+0xff,0xff,0x8c,0xda,0x01,0x00,0x00,0x05
+
+# GFX12: ds_load_u8_d16_hi v5, v255 offset:65535 ; encoding: [0xff,0xff,0x8c,0xda,0xff,0x00,0x00,0x05]
+0xff,0xff,0x8c,0xda,0xff,0x00,0x00,0x05
+
+# GFX12: ds_rsub_rtn_u32 v0, v1, v2              ; encoding: [0x00,0x00,0x88,0xd8,0x01,0x02,0x00,0x00]
+0x00,0x00,0x88,0xd8,0x01,0x02,0x00,0x00
+
+# GFX12: ds_rsub_rtn_u32 v0, v1, v2 offset:4660  ; encoding: [0x34,0x12,0x88,0xd8,0x01,0x02,0x00,0x00]
+0x34,0x12,0x88,0xd8,0x01,0x02,0x00,0x00
+
+# GFX12: ds_rsub_rtn_u32 v0, v1, v2 offset:65535 ; encoding: [0xff,0xff,0x88,0xd8,0x01,0x02,0x00,0x00]
+0xff,0xff,0x88,0xd8,0x01,0x02,0x00,0x00
+
+# GFX12: ds_rsub_rtn_u32 v0, v254, v253          ; encoding: [0x00,0x00,0x88,0xd8,0xfe,0xfd,0x00,0x00]
+0x00,0x00,0x88,0xd8,0xfe,0xfd,0x00,0x00
+
+# GFX12: ds_rsub_rtn_u32 v0, v254, v253 offset:4660 ; encoding: [0x34,0x12,0x88,0xd8,0xfe,0xfd,0x00,0x00]
+0x34,0x12,0x88,0xd8,0xfe,0xfd,0x00,0x00
+
+# GFX12: ds_rsub_rtn_u32 v0, v254, v253 offset:65535 ; encoding: [0xff,0xff,0x88,0xd8,0xfe,0xfd,0x00,0x00]
+0xff,0xff,0x88,0xd8,0xfe,0xfd,0x00,0x00
+
+# GFX12: ds_rsub_rtn_u32 v255, v1, v253          ; encoding: [0x00,0x00,0x88,0xd8,0x01,0xfd,0x00,0xff]
+0x00,0x00,0x88,0xd8,0x01,0xfd,0x00,0xff
+
+# GFX12: ds_rsub_rtn_u32 v255, v1, v253 offset:4660 ; encoding: [0x34,0x12,0x88,0xd8,0x01,0xfd,0x00,0xff]
+0x34,0x12,0x88,0xd8,0x01,0xfd,0x00,0xff
+
+# GFX12: ds_rsub_rtn_u32 v255, v1, v253 offset:65535 ; encoding: [0xff,0xff,0x88,0xd8,0x01,0xfd,0x00,0xff]
+0xff,0xff,0x88,0xd8,0x01,0xfd,0x00,0xff
+
+# GFX12: ds_rsub_rtn_u32 v255, v254, v2          ; encoding: [0x00,0x00,0x88,0xd8,0xfe,0x02,0x00,0xff]
+0x00,0x00,0x88,0xd8,0xfe,0x02,0x00,0xff
+
+# GFX12: ds_rsub_rtn_u32 v255, v254, v2 offset:4660 ; encoding: [0x34,0x12,0x88,0xd8,0xfe,0x02,0x00,0xff]
+0x34,0x12,0x88,0xd8,0xfe,0x02,0x00,0xff
+
+# GFX12: ds_rsub_rtn_u32 v255, v254, v2 offset:65535 ; encoding: [0xff,0xff,0x88,0xd8,0xfe,0x02,0x00,0xff]
+0xff,0xff,0x88,0xd8,0xfe,0x02,0x00,0xff
+
+# GFX12: ds_rsub_rtn_u32 v255, v254, v253        ; encoding: [0x00,0x00,0x88,0xd8,0xfe,0xfd,0x00,0xff]
+0x00,0x00,0x88,0xd8,0xfe,0xfd,0x00,0xff
+
+# GFX12: ds_rsub_rtn_u32 v255, v254, v253 offset:4660 ; encoding: [0x34,0x12,0x88,0xd8,0xfe,0xfd,0x00,0xff]
+0x34,0x12,0x88,0xd8,0xfe,0xfd,0x00,0xff
+
+# GFX12: ds_rsub_rtn_u32 v255, v254, v253 offset:65535 ; encoding: [0xff,0xff,0x88,0xd8,0xfe,0xfd,0x00,0xff]
+0xff,0xff,0x88,0xd8,0xfe,0xfd,0x00,0xff
+
+# GFX12: ds_rsub_rtn_u64 v[254:255], v1, v[2:3] offset:65535 ; encoding: [0xff,0xff,0x88,0xd9,0x01,0x02,0x00,0xfe]
+0xff,0xff,0x88,0xd9,0x01,0x02,0x00,0xfe
+
+# GFX12: ds_rsub_rtn_u64 v[5:6], v1, v[254:255] offset:65535 ; encoding: [0xff,0xff,0x88,0xd9,0x01,0xfe,0x00,0x05]
+0xff,0xff,0x88,0xd9,0x01,0xfe,0x00,0x05
+
+# GFX12: ds_rsub_rtn_u64 v[5:6], v1, v[2:3]      ; encoding: [0x00,0x00,0x88,0xd9,0x01,0x02,0x00,0x05]
+0x00,0x00,0x88,0xd9,0x01,0x02,0x00,0x05
+
+# GFX12: ds_rsub_rtn_u64 v[5:6], v1, v[2:3] offset:4 ; encoding: [0x04,0x00,0x88,0xd9,0x01,0x02,0x00,0x05]
+0x04,0x00,0x88,0xd9,0x01,0x02,0x00,0x05
+
+# GFX12: ds_rsub_rtn_u64 v[5:6], v1, v[2:3] offset:65535 ; encoding: [0xff,0xff,0x88,0xd9,0x01,0x02,0x00,0x05]
+0xff,0xff,0x88,0xd9,0x01,0x02,0x00,0x05
+
+# GFX12: ds_rsub_rtn_u64 v[5:6], v255, v[2:3] offset:65535 ; encoding: [0xff,0xff,0x88,0xd9,0xff,0x02,0x00,0x05]
+0xff,0xff,0x88,0xd9,0xff,0x02,0x00,0x05
+
+# GFX12: ds_rsub_u32 v0, v1                      ; encoding: [0x00,0x00,0x08,0xd8,0x00,0x01,0x00,0x00]
+0x00,0x00,0x08,0xd8,0x00,0x01,0x00,0x00
+
+# GFX12: ds_rsub_u32 v0, v1 offset:4660          ; encoding: [0x34,0x12,0x08,0xd8,0x00,0x01,0x00,0x00]
+0x34,0x12,0x08,0xd8,0x00,0x01,0x00,0x00
+
+# GFX12: ds_rsub_u32 v0, v1 offset:65535         ; encoding: [0xff,0xff,0x08,0xd8,0x00,0x01,0x00,0x00]
+0xff,0xff,0x08,0xd8,0x00,0x01,0x00,0x00
+
+# GFX12: ds_rsub_u32 v0, v254                    ; encoding: [0x00,0x00,0x08,0xd8,0x00,0xfe,0x00,0x00]
+0x00,0x00,0x08,0xd8,0x00,0xfe,0x00,0x00
+
+# GFX12: ds_rsub_u32 v0, v254 offset:4660        ; encoding: [0x34,0x12,0x08,0xd8,0x00,0xfe,0x00,0x00]
+0x34,0x12,0x08,0xd8,0x00,0xfe,0x00,0x00
+
+# GFX12: ds_rsub_u32 v0, v254 offset:65535       ; encoding: [0xff,0xff,0x08,0xd8,0x00,0xfe,0x00,0x00]
+0xff,0xff,0x08,0xd8,0x00,0xfe,0x00,0x00
+
+# GFX12: ds_rsub_u32 v255, v1                    ; encoding: [0x00,0x00,0x08,0xd8,0xff,0x01,0x00,0x00]
+0x00,0x00,0x08,0xd8,0xff,0x01,0x00,0x00
+
+# GFX12: ds_rsub_u32 v255, v1 offset:4660        ; encoding: [0x34,0x12,0x08,0xd8,0xff,0x01,0x00,0x00]
+0x34,0x12,0x08,0xd8,0xff,0x01,0x00,0x00
+
+# GFX12: ds_rsub_u32 v255, v1 offset:65535       ; encoding: [0xff,0xff,0x08,0xd8,0xff,0x01,0x00,0x00]
+0xff,0xff,0x08,0xd8,0xff,0x01,0x00,0x00
+
+# GFX12: ds_rsub_u32 v255, v254                  ; encoding: [0x00,0x00,0x08,0xd8,0xff,0xfe,0x00,0x00]
+0x00,0x00,0x08,0xd8,0xff,0xfe,0x00,0x00
+
+# GFX12: ds_rsub_u32 v255, v254 offset:4660      ; encoding: [0x34,0x12,0x08,0xd8,0xff,0xfe,0x00,0x00]
+0x34,0x12,0x08,0xd8,0xff,0xfe,0x00,0x00
+
+# GFX12: ds_rsub_u32 v255, v254 offset:65535     ; encoding: [0xff,0xff,0x08,0xd8,0xff,0xfe,0x00,0x00]
+0xff,0xff,0x08,0xd8,0xff,0xfe,0x00,0x00
+
+# GFX12: ds_rsub_u64 v1, v[254:255] offset:65535 ; encoding: [0xff,0xff,0x08,0xd9,0x01,0xfe,0x00,0x00]
+0xff,0xff,0x08,0xd9,0x01,0xfe,0x00,0x00
+
+# GFX12: ds_rsub_u64 v1, v[2:3]                  ; encoding: [0x00,0x00,0x08,0xd9,0x01,0x02,0x00,0x00]
+0x00,0x00,0x08,0xd9,0x01,0x02,0x00,0x00
+
+# GFX12: ds_rsub_u64 v1, v[2:3] offset:4         ; encoding: [0x04,0x00,0x08,0xd9,0x01,0x02,0x00,0x00]
+0x04,0x00,0x08,0xd9,0x01,0x02,0x00,0x00
+
+# GFX12: ds_rsub_u64 v1, v[2:3] offset:65535     ; encoding: [0xff,0xff,0x08,0xd9,0x01,0x02,0x00,0x00]
+0xff,0xff,0x08,0xd9,0x01,0x02,0x00,0x00
+
+# GFX12: ds_rsub_u64 v255, v[2:3] offset:65535   ; encoding: [0xff,0xff,0x08,0xd9,0xff,0x02,0x00,0x00]
+0xff,0xff,0x08,0xd9,0xff,0x02,0x00,0x00
+
+# GFX12: ds_sub_clamp_rtn_u32 v0, v1, v2         ; encoding: [0x00,0x00,0xa4,0xda,0x01,0x02,0x00,0x00]
+0x00,0x00,0xa4,0xda,0x01,0x02,0x00,0x00
+
+# GFX12: ds_sub_clamp_rtn_u32 v0, v1, v2 offset:4660 ; encoding: [0x34,0x12,0xa4,0xda,0x01,0x02,0x00,0x00]
+0x34,0x12,0xa4,0xda,0x01,0x02,0x00,0x00
+
+# GFX12: ds_sub_clamp_rtn_u32 v0, v1, v2 offset:65535 ; encoding: [0xff,0xff,0xa4,0xda,0x01,0x02,0x00,0x00]
+0xff,0xff,0xa4,0xda,0x01,0x02,0x00,0x00
+
+# GFX12: ds_sub_clamp_rtn_u32 v0, v254, v253     ; encoding: [0x00,0x00,0xa4,0xda,0xfe,0xfd,0x00,0x00]
+0x00,0x00,0xa4,0xda,0xfe,0xfd,0x00,0x00
+
+# GFX12: ds_sub_clamp_rtn_u32 v0, v254, v253 offset:4660 ; encoding: [0x34,0x12,0xa4,0xda,0xfe,0xfd,0x00,0x00]
+0x34,0x12,0xa4,0xda,0xfe,0xfd,0x00,0x00
+
+# GFX12: ds_sub_clamp_rtn_u32 v0, v254, v253 offset:65535 ; encoding: [0xff,0xff,0xa4,0xda,0xfe,0xfd,0x00,0x00]
+0xff,0xff,0xa4,0xda,0xfe,0xfd,0x00,0x00
+
+# GFX12: ds_sub_clamp_rtn_u32 v255, v1, v253     ; encoding: [0x00,0x00,0xa4,0xda,0x01,0xfd,0x00,0xff]
+0x00,0x00,0xa4,0xda,0x01,0xfd,0x00,0xff
+
+# GFX12: ds_sub_clamp_rtn_u32 v255, v1, v253 offset:4660 ; encoding: [0x34,0x12,0xa4,0xda,0x01,0xfd,0x00,0xff]
+0x34,0x12,0xa4,0xda,0x01,0xfd,0x00,0xff
+
+# GFX12: ds_sub_clamp_rtn_u32 v255, v1, v253 offset:65535 ; encoding: [0xff,0xff,0xa4,0xda,0x01,0xfd,0x00,0xff]
+0xff,0xff,0xa4,0xda,0x01,0xfd,0x00,0xff
+
+# GFX12: ds_sub_clamp_rtn_u32 v255, v254, v2     ; encoding: [0x00,0x00,0xa4,0xda,0xfe,0x02,0x00,0xff]
+0x00,0x00,0xa4,0xda,0xfe,0x02,0x00,0xff
+
+# GFX12: ds_sub_clamp_rtn_u32 v255, v254, v2 offset:4660 ; encoding: [0x34,0x12,0xa4,0xda,0xfe,0x02,0x00,0xff]
+0x34,0x12,0xa4,0xda,0xfe,0x02,0x00,0xff
+
+# GFX12: ds_sub_clamp_rtn_u32 v255, v254, v2 offset:65535 ; encoding: [0xff,0xff,0xa4,0xda,0xfe,0x02,0x00,0xff]
+0xff,0xff,0xa4,0xda,0xfe,0x02,0x00,0xff
+
+# GFX12: ds_sub_clamp_rtn_u32 v255, v254, v253   ; encoding: [0x00,0x00,0xa4,0xda,0xfe,0xfd,0x00,0xff]
+0x00,0x00,0xa4,0xda,0xfe,0xfd,0x00,0xff
+
+# GFX12: ds_sub_clamp_rtn_u32 v255, v254, v253 offset:4660 ; encoding: [0x34,0x12,0xa4,0xda,0xfe,0xfd,0x00,0xff]
+0x34,0x12,0xa4,0xda,0xfe,0xfd,0x00,0xff
+
+# GFX12: ds_sub_clamp_rtn_u32 v255, v254, v253 offset:65535 ; encoding: [0xff,0xff,0xa4,0xda,0xfe,0xfd,0x00,0xff]
+0xff,0xff,0xa4,0xda,0xfe,0xfd,0x00,0xff
+
+# GFX12: ds_sub_clamp_u32 v0, v1                 ; encoding: [0x00,0x00,0x64,0xda,0x00,0x01,0x00,0x00]
+0x00,0x00,0x64,0xda,0x00,0x01,0x00,0x00
+
+# GFX12: ds_sub_clamp_u32 v0, v1 offset:4660     ; encoding: [0x34,0x12,0x64,0xda,0x00,0x01,0x00,0x00]
+0x34,0x12,0x64,0xda,0x00,0x01,0x00,0x00
+
+# GFX12: ds_sub_clamp_u32 v0, v1 offset:65535    ; encoding: [0xff,0xff,0x64,0xda,0x00,0x01,0x00,0x00]
+0xff,0xff,0x64,0xda,0x00,0x01,0x00,0x00
+
+# GFX12: ds_sub_clamp_u32 v0, v254               ; encoding: [0x00,0x00,0x64,0xda,0x00,0xfe,0x00,0x00]
+0x00,0x00,0x64,0xda,0x00,0xfe,0x00,0x00
+
+# GFX12: ds_sub_clamp_u32 v0, v254 offset:4660   ; encoding: [0x34,0x12,0x64,0xda,0x00,0xfe,0x00,0x00]
+0x34,0x12,0x64,0xda,0x00,0xfe,0x00,0x00
+
+# GFX12: ds_sub_clamp_u32 v0, v254 offset:65535  ; encoding: [0xff,0xff,0x64,0xda,0x00,0xfe,0x00,0x00]
+0xff,0xff,0x64,0xda,0x00,0xfe,0x00,0x00
+
+# GFX12: ds_sub_clamp_u32 v255, v1               ; encoding: [0x00,0x00,0x64,0xda,0xff,0x01,0x00,0x00]
+0x00,0x00,0x64,0xda,0xff,0x01,0x00,0x00
+
+# GFX12: ds_sub_clamp_u32 v255, v1 offset:4660   ; encoding: [0x34,0x12,0x64,0xda,0xff,0x01,0x00,0x00]
+0x34,0x12,0x64,0xda,0xff,0x01,0x00,0x00
+
+# GFX12: ds_sub_clamp_u32 v255, v1 offset:65535  ; encoding: [0xff,0xff,0x64,0xda,0xff,0x01,0x00,0x00]
+0xff,0xff,0x64,0xda,0xff,0x01,0x00,0x00
+
+# GFX12: ds_sub_clamp_u32 v255, v254             ; encoding: [0x00,0x00,0x64,0xda,0xff,0xfe,0x00,0x00]
+0x00,0x00,0x64,0xda,0xff,0xfe,0x00,0x00
+
+# GFX12: ds_sub_clamp_u32 v255, v254 offset:4660 ; encoding: [0x34,0x12,0x64,0xda,0xff,0xfe,0x00,0x00]
+0x34,0x12,0x64,0xda,0xff,0xfe,0x00,0x00
+
+# GFX12: ds_sub_clamp_u32 v255, v254 offset:65535 ; encoding: [0xff,0xff,0x64,0xda,0xff,0xfe,0x00,0x00]
+0xff,0xff,0x64,0xda,0xff,0xfe,0x00,0x00
+
+# GFX12: ds_sub_rtn_u32 v0, v1, v2               ; encoding: [0x00,0x00,0x84,0xd8,0x01,0x02,0x00,0x00]
+0x00,0x00,0x84,0xd8,0x01,0x02,0x00,0x00
+
+# GFX12: ds_sub_rtn_u32 v0, v1, v2 offset:4660   ; encoding: [0x34,0x12,0x84,0xd8,0x01,0x02,0x00,0x00]
+0x34,0x12,0x84,0xd8,0x01,0x02,0x00,0x00
+
+# GFX12: ds_sub_rtn_u32 v0, v1, v2 offset:65535  ; encoding: [0xff,0xff,0x84,0xd8,0x01,0x02,0x00,0x00]
+0xff,0xff,0x84,0xd8,0x01,0x02,0x00,0x00
+
+# GFX12: ds_sub_rtn_u32 v0, v254, v253           ; encoding: [0x00,0x00,0x84,0xd8,0xfe,0xfd,0x00,0x00]
+0x00,0x00,0x84,0xd8,0xfe,0xfd,0x00,0x00
+
+# GFX12: ds_sub_rtn_u32 v0, v254, v253 offset:4660 ; encoding: [0x34,0x12,0x84,0xd8,0xfe,0xfd,0x00,0x00]
+0x34,0x12,0x84,0xd8,0xfe,0xfd,0x00,0x00
+
+# GFX12: ds_sub_rtn_u32 v0, v254, v253 offset:65535 ; encoding: [0xff,0xff,0x84,0xd8,0xfe,0xfd,0x00,0x00]
+0xff,0xff,0x84,0xd8,0xfe,0xfd,0x00,0x00
+
+# GFX12: ds_sub_rtn_u32 v255, v1, v253           ; encoding: [0x00,0x00,0x84,0xd8,0x01,0xfd,0x00,0xff]
+0x00,0x00,0x84,0xd8,0x01,0xfd,0x00,0xff
+
+# GFX12: ds_sub_rtn_u32 v255, v1, v253 offset:4660 ; encoding: [0x34,0x12,0x84,0xd8,0x01,0xfd,0x00,0xff]
+0x34,0x12,0x84,0xd8,0x01,0xfd,0x00,0xff
+
+# GFX12: ds_sub_rtn_u32 v255, v1, v253 offset:65535 ; encoding: [0xff,0xff,0x84,0xd8,0x01,0xfd,0x00,0xff]
+0xff,0xff,0x84,0xd8,0x01,0xfd,0x00,0xff
+
+# GFX12: ds_sub_rtn_u32 v255, v254, v2           ; encoding: [0x00,0x00,0x84,0xd8,0xfe,0x02,0x00,0xff]
+0x00,0x00,0x84,0xd8,0xfe,0x02,0x00,0xff
+
+# GFX12: ds_sub_rtn_u32 v255, v254, v2 offset:4660 ; encoding: [0x34,0x12,0x84,0xd8,0xfe,0x02,0x00,0xff]
+0x34,0x12,0x84,0xd8,0xfe,0x02,0x00,0xff
+
+# GFX12: ds_sub_rtn_u32 v255, v254, v2 offset:65535 ; encoding: [0xff,0xff,0x84,0xd8,0xfe,0x02,0x00,0xff]
+0xff,0xff,0x84,0xd8,0xfe,0x02,0x00,0xff
+
+# GFX12: ds_sub_rtn_u32 v255, v254, v253         ; encoding: [0x00,0x00,0x84,0xd8,0xfe,0xfd,0x00,0xff]
+0x00,0x00,0x84,0xd8,0xfe,0xfd,0x00,0xff
+
+# GFX12: ds_sub_rtn_u32 v255, v254, v253 offset:4660 ; encoding: [0x34,0x12,0x84,0xd8,0xfe,0xfd,0x00,0xff]
+0x34,0x12,0x84,0xd8,0xfe,0xfd,0x00,0xff
+
+# GFX12: ds_sub_rtn_u32 v255, v254, v253 offset:65535 ; encoding: [0xff,0xff,0x84,0xd8,0xfe,0xfd,0x00,0xff]
+0xff,0xff,0x84,0xd8,0xfe,0xfd,0x00,0xff
+
+# GFX12: ds_sub_rtn_u64 v[254:255], v1, v[2:3] offset:65535 ; encoding: [0xff,0xff,0x84,0xd9,0x01,0x02,0x00,0xfe]
+0xff,0xff,0x84,0xd9,0x01,0x02,0x00,0xfe
+
+# GFX12: ds_sub_rtn_u64 v[5:6], v1, v[254:255] offset:65535 ; encoding: [0xff,0xff,0x84,0xd9,0x01,0xfe,0x00,0x05]
+0xff,0xff,0x84,0xd9,0x01,0xfe,0x00,0x05
+
+# GFX12: ds_sub_rtn_u64 v[5:6], v1, v[2:3]       ; encoding: [0x00,0x00,0x84,0xd9,0x01,0x02,0x00,0x05]
+0x00,0x00,0x84,0xd9,0x01,0x02,0x00,0x05
+
+# GFX12: ds_sub_rtn_u64 v[5:6], v1, v[2:3] offset:4 ; encoding: [0x04,0x00,0x84,0xd9,0x01,0x02,0x00,0x05]
+0x04,0x00,0x84,0xd9,0x01,0x02,0x00,0x05
+
+# GFX12: ds_sub_rtn_u64 v[5:6], v1, v[2:3] offset:65535 ; encoding: [0xff,0xff,0x84,0xd9,0x01,0x02,0x00,0x05]
+0xff,0xff,0x84,0xd9,0x01,0x02,0x00,0x05
+
+# GFX12: ds_sub_rtn_u64 v[5:6], v255, v[2:3] offset:65535 ; encoding: [0xff,0xff,0x84,0xd9,0xff,0x02,0x00,0x05]
+0xff,0xff,0x84,0xd9,0xff,0x02,0x00,0x05
+
+# GFX12: ds_sub_u32 v0, v1                       ; encoding: [0x00,0x00,0x04,0xd8,0x00,0x01,0x00,0x00]
+0x00,0x00,0x04,0xd8,0x00,0x01,0x00,0x00
+
+# GFX12: ds_sub_u32 v0, v1 offset:4660           ; encoding: [0x34,0x12,0x04,0xd8,0x00,0x01,0x00,0x00]
+0x34,0x12,0x04,0xd8,0x00,0x01,0x00,0x00
+
+# GFX12: ds_sub_u32 v0, v1 offset:65535          ; encoding: [0xff,0xff,0x04,0xd8,0x00,0x01,0x00,0x00]
+0xff,0xff,0x04,0xd8,0x00,0x01,0x00,0x00
+
+# GFX12: ds_sub_u32 v0, v254                     ; encoding: [0x00,0x00,0x04,0xd8,0x00,0xfe,0x00,0x00]
+0x00,0x00,0x04,0xd8,0x00,0xfe,0x00,0x00
+
+# GFX12: ds_sub_u32 v0, v254 offset:4660         ; encoding: [0x34,0x12,0x04,0xd8,0x00,0xfe,0x00,0x00]
+0x34,0x12,0x04,0xd8,0x00,0xfe,0x00,0x00
+
+# GFX12: ds_sub_u32 v0, v254 offset:65535        ; encoding: [0xff,0xff,0x04,0xd8,0x00,0xfe,0x00,0x00]
+0xff,0xff,0x04,0xd8,0x00,0xfe,0x00,0x00
+
+# GFX12: ds_sub_u32 v255, v1                     ; encoding: [0x00,0x00,0x04,0xd8,0xff,0x01,0x00,0x00]
+0x00,0x00,0x04,0xd8,0xff,0x01,0x00,0x00
+
+# GFX12: ds_sub_u32 v255, v1 offset:4660         ; encoding: [0x34,0x12,0x04,0xd8,0xff,0x01,0x00,0x00]
+0x34,0x12,0x04,0xd8,0xff,0x01,0x00,0x00
+
+# GFX12: ds_sub_u32 v255, v1 offset:65535        ; encoding: [0xff,0xff,0x04,0xd8,0xff,0x01,0x00,0x00]
+0xff,0xff,0x04,0xd8,0xff,0x01,0x00,0x00
+
+# GFX12: ds_sub_u32 v255, v254                   ; encoding: [0x00,0x00,0x04,0xd8,0xff,0xfe,0x00,0x00]
+0x00,0x00,0x04,0xd8,0xff,0xfe,0x00,0x00
+
+# GFX12: ds_sub_u32 v255, v254 offset:4660       ; encoding: [0x34,0x12,0x04,0xd8,0xff,0xfe,0x00,0x00]
+0x34,0x12,0x04,0xd8,0xff,0xfe,0x00,0x00
+
+# GFX12: ds_sub_u32 v255, v254 offset:65535      ; encoding: [0xff,0xff,0x04,0xd8,0xff,0xfe,0x00,0x00]
+0xff,0xff,0x04,0xd8,0xff,0xfe,0x00,0x00
+
+# GFX12: ds_sub_u64 v1, v[254:255] offset:65535  ; encoding: [0xff,0xff,0x04,0xd9,0x01,0xfe,0x00,0x00]
+0xff,0xff,0x04,0xd9,0x01,0xfe,0x00,0x00
+
+# GFX12: ds_sub_u64 v1, v[2:3]                   ; encoding: [0x00,0x00,0x04,0xd9,0x01,0x02,0x00,0x00]
+0x00,0x00,0x04,0xd9,0x01,0x02,0x00,0x00
+
+# GFX12: ds_sub_u64 v1, v[2:3] offset:4          ; encoding: [0x04,0x00,0x04,0xd9,0x01,0x02,0x00,0x00]
+0x04,0x00,0x04,0xd9,0x01,0x02,0x00,0x00
+
+# GFX12: ds_sub_u64 v1, v[2:3] offset:65535      ; encoding: [0xff,0xff,0x04,0xd9,0x01,0x02,0x00,0x00]
+0xff,0xff,0x04,0xd9,0x01,0x02,0x00,0x00
+
+# GFX12: ds_sub_u64 v255, v[2:3] offset:65535    ; encoding: [0xff,0xff,0x04,0xd9,0xff,0x02,0x00,0x00]
+0xff,0xff,0x04,0xd9,0xff,0x02,0x00,0x00
+
+# GFX12: ds_swizzle_b32 v255, v1 offset:65535    ; encoding: [0xff,0xff,0xd4,0xd8,0x01,0x00,0x00,0xff]
+0xff,0xff,0xd4,0xd8,0x01,0x00,0x00,0xff
+
+# GFX12: ds_swizzle_b32 v5, v1                   ; encoding: [0x00,0x00,0xd4,0xd8,0x01,0x00,0x00,0x05]
+0x00,0x00,0xd4,0xd8,0x01,0x00,0x00,0x05
+
+# GFX12: ds_swizzle_b32 v5, v1 offset:65535      ; encoding: [0xff,0xff,0xd4,0xd8,0x01,0x00,0x00,0x05]
+0xff,0xff,0xd4,0xd8,0x01,0x00,0x00,0x05
+
+# GFX12: ds_swizzle_b32 v5, v1 offset:swizzle(BITMASK_PERM,"00p00") ; encoding: [0x04,0x00,0xd4,0xd8,0x01,0x00,0x00,0x05]
+0x04,0x00,0xd4,0xd8,0x01,0x00,0x00,0x05
+
+# GFX12: ds_swizzle_b32 v8, v2 offset:swizzle(QUAD_PERM,0,1,2,3) ; encoding: [0xe4,0x80,0xd4,0xd8,0x02,0x00,0x00,0x08]
+0xe4,0x80,0xd4,0xd8,0x02,0x00,0x00,0x08
+
+# GFX12: ds_swizzle_b32 v8, v2 offset:swizzle(SWAP,4) ; encoding: [0x1f,0x10,0xd4,0xd8,0x02,0x00,0x00,0x08]
+0x1f,0x10,0xd4,0xd8,0x02,0x00,0x00,0x08
+
+# GFX12: ds_swizzle_b32 v8, v2 offset:swizzle(REVERSE,16) ; encoding: [0x1f,0x3c,0xd4,0xd8,0x02,0x00,0x00,0x08]
+0x1f,0x3c,0xd4,0xd8,0x02,0x00,0x00,0x08
+
+# GFX12: ds_swizzle_b32 v8, v2 offset:swizzle(BROADCAST,32,1) ; encoding: [0x20,0x00,0xd4,0xd8,0x02,0x00,0x00,0x08]
+0x20,0x00,0xd4,0xd8,0x02,0x00,0x00,0x08
+
+# GFX12: ds_swizzle_b32 v5, v255 offset:65535    ; encoding: [0xff,0xff,0xd4,0xd8,0xff,0x00,0x00,0x05]
+0xff,0xff,0xd4,0xd8,0xff,0x00,0x00,0x05
+
+# GFX12: ds_store_2addr_b32 v0, v1, v2           ; encoding: [0x00,0x00,0x38,0xd8,0x00,0x01,0x02,0x00]
+0x00,0x00,0x38,0xd8,0x00,0x01,0x02,0x00
+
+# GFX12: ds_store_2addr_b32 v0, v1, v2 offset0:123 ; encoding: [0x7b,0x00,0x38,0xd8,0x00,0x01,0x02,0x00]
+0x7b,0x00,0x38,0xd8,0x00,0x01,0x02,0x00
+
+# GFX12: ds_store_2addr_b32 v0, v1, v2 offset0:123 offset1:64 ; encoding: [0x7b,0x40,0x38,0xd8,0x00,0x01,0x02,0x00]
+0x7b,0x40,0x38,0xd8,0x00,0x01,0x02,0x00
+
+# GFX12: ds_store_2addr_b32 v0, v1, v2 offset0:255 offset1:255 ; encoding: [0xff,0xff,0x38,0xd8,0x00,0x01,0x02,0x00]
+0xff,0xff,0x38,0xd8,0x00,0x01,0x02,0x00
+
+# GFX12: ds_store_2addr_b32 v0, v1, v2 offset0:255 offset1:64 ; encoding: [0xff,0x40,0x38,0xd8,0x00,0x01,0x02,0x00]
+0xff,0x40,0x38,0xd8,0x00,0x01,0x02,0x00
+
+# GFX12: ds_store_2addr_b32 v0, v1, v2 offset0:64 offset1:123 ; encoding: [0x40,0x7b,0x38,0xd8,0x00,0x01,0x02,0x00]
+0x40,0x7b,0x38,0xd8,0x00,0x01,0x02,0x00
+
+# GFX12: ds_store_2addr_b32 v0, v1, v2 offset0:64 offset1:255 ; encoding: [0x40,0xff,0x38,0xd8,0x00,0x01,0x02,0x00]
+0x40,0xff,0x38,0xd8,0x00,0x01,0x02,0x00
+
+# GFX12: ds_store_2addr_b32 v0, v1, v2 offset1:123 ; encoding: [0x00,0x7b,0x38,0xd8,0x00,0x01,0x02,0x00]
+0x00,0x7b,0x38,0xd8,0x00,0x01,0x02,0x00
+
+# GFX12: ds_store_2addr_b32 v0, v254, v253       ; encoding: [0x00,0x00,0x38,0xd8,0x00,0xfe,0xfd,0x00]
+0x00,0x00,0x38,0xd8,0x00,0xfe,0xfd,0x00
+
+# GFX12: ds_store_2addr_b32 v0, v254, v253 offset0:123 ; encoding: [0x7b,0x00,0x38,0xd8,0x00,0xfe,0xfd,0x00]
+0x7b,0x00,0x38,0xd8,0x00,0xfe,0xfd,0x00
+
+# GFX12: ds_store_2addr_b32 v0, v254, v253 offset0:123 offset1:64 ; encoding: [0x7b,0x40,0x38,0xd8,0x00,0xfe,0xfd,0x00]
+0x7b,0x40,0x38,0xd8,0x00,0xfe,0xfd,0x00
+
+# GFX12: ds_store_2addr_b32 v0, v254, v253 offset0:255 offset1:255 ; encoding: [0xff,0xff,0x38,0xd8,0x00,0xfe,0xfd,0x00]
+0xff,0xff,0x38,0xd8,0x00,0xfe,0xfd,0x00
+
+# GFX12: ds_store_2addr_b32 v0, v254, v253 offset0:255 offset1:64 ; encoding: [0xff,0x40,0x38,0xd8,0x00,0xfe,0xfd,0x00]
+0xff,0x40,0x38,0xd8,0x00,0xfe,0xfd,0x00
+
+# GFX12: ds_store_2addr_b32 v0, v254, v253 offset0:64 offset1:123 ; encoding: [0x40,0x7b,0x38,0xd8,0x00,0xfe,0xfd,0x00]
+0x40,0x7b,0x38,0xd8,0x00,0xfe,0xfd,0x00
+
+# GFX12: ds_store_2addr_b32 v0, v254, v253 offset0:64 offset1:255 ; encoding: [0x40,0xff,0x38,0xd8,0x00,0xfe,0xfd,0x00]
+0x40,0xff,0x38,0xd8,0x00,0xfe,0xfd,0x00
+
+# GFX12: ds_store_2addr_b32 v0, v254, v253 offset1:123 ; encoding: [0x00,0x7b,0x38,0xd8,0x00,0xfe,0xfd,0x00]
+0x00,0x7b,0x38,0xd8,0x00,0xfe,0xfd,0x00
+
+# GFX12: ds_store_2addr_b32 v255, v1, v253       ; encoding: [0x00,0x00,0x38,0xd8,0xff,0x01,0xfd,0x00]
+0x00,0x00,0x38,0xd8,0xff,0x01,0xfd,0x00
+
+# GFX12: ds_store_2addr_b32 v255, v1, v253 offset0:123 ; encoding: [0x7b,0x00,0x38,0xd8,0xff,0x01,0xfd,0x00]
+0x7b,0x00,0x38,0xd8,0xff,0x01,0xfd,0x00
+
+# GFX12: ds_store_2addr_b32 v255, v1, v253 offset0:123 offset1:64 ; encoding: [0x7b,0x40,0x38,0xd8,0xff,0x01,0xfd,0x00]
+0x7b,0x40,0x38,0xd8,0xff,0x01,0xfd,0x00
+
+# GFX12: ds_store_2addr_b32 v255, v1, v253 offset0:255 offset1:255 ; encoding: [0xff,0xff,0x38,0xd8,0xff,0x01,0xfd,0x00]
+0xff,0xff,0x38,0xd8,0xff,0x01,0xfd,0x00
+
+# GFX12: ds_store_2addr_b32 v255, v1, v253 offset0:255 offset1:64 ; encoding: [0xff,0x40,0x38,0xd8,0xff,0x01,0xfd,0x00]
+0xff,0x40,0x38,0xd8,0xff,0x01,0xfd,0x00
+
+# GFX12: ds_store_2addr_b32 v255, v1, v253 offset0:64 offset1:123 ; encoding: [0x40,0x7b,0x38,0xd8,0xff,0x01,0xfd,0x00]
+0x40,0x7b,0x38,0xd8,0xff,0x01,0xfd,0x00
+
+# GFX12: ds_store_2addr_b32 v255, v1, v253 offset0:64 offset1:255 ; encoding: [0x40,0xff,0x38,0xd8,0xff,0x01,0xfd,0x00]
+0x40,0xff,0x38,0xd8,0xff,0x01,0xfd,0x00
+
+# GFX12: ds_store_2addr_b32 v255, v1, v253 offset1:123 ; encoding: [0x00,0x7b,0x38,0xd8,0xff,0x01,0xfd,0x00]
+0x00,0x7b,0x38,0xd8,0xff,0x01,0xfd,0x00
+
+# GFX12: ds_store_2addr_b32 v255, v254, v2       ; encoding: [0x00,0x00,0x38,0xd8,0xff,0xfe,0x02,0x00]
+0x00,0x00,0x38,0xd8,0xff,0xfe,0x02,0x00
+
+# GFX12: ds_store_2addr_b32 v255, v254, v2 offset0:123 ; encoding: [0x7b,0x00,0x38,0xd8,0xff,0xfe,0x02,0x00]
+0x7b,0x00,0x38,0xd8,0xff,0xfe,0x02,0x00
+
+# GFX12: ds_store_2addr_b32 v255, v254, v2 offset0:123 offset1:64 ; encoding: [0x7b,0x40,0x38,0xd8,0xff,0xfe,0x02,0x00]
+0x7b,0x40,0x38,0xd8,0xff,0xfe,0x02,0x00
+
+# GFX12: ds_store_2addr_b32 v255, v254, v2 offset0:255 offset1:255 ; encoding: [0xff,0xff,0x38,0xd8,0xff,0xfe,0x02,0x00]
+0xff,0xff,0x38,0xd8,0xff,0xfe,0x02,0x00
+
+# GFX12: ds_store_2addr_b32 v255, v254, v2 offset0:255 offset1:64 ; encoding: [0xff,0x40,0x38,0xd8,0xff,0xfe,0x02,0x00]
+0xff,0x40,0x38,0xd8,0xff,0xfe,0x02,0x00
+
+# GFX12: ds_store_2addr_b32 v255, v254, v2 offset0:64 offset1:123 ; encoding: [0x40,0x7b,0x38,0xd8,0xff,0xfe,0x02,0x00]
+0x40,0x7b,0x38,0xd8,0xff,0xfe,0x02,0x00
+
+# GFX12: ds_store_2addr_b32 v255, v254, v2 offset0:64 offset1:255 ; encoding: [0x40,0xff,0x38,0xd8,0xff,0xfe,0x02,0x00]
+0x40,0xff,0x38,0xd8,0xff,0xfe,0x02,0x00
+
+# GFX12: ds_store_2addr_b32 v255, v254, v2 offset1:123 ; encoding: [0x00,0x7b,0x38,0xd8,0xff,0xfe,0x02,0x00]
+0x00,0x7b,0x38,0xd8,0xff,0xfe,0x02,0x00
+
+# GFX12: ds_store_2addr_b32 v255, v254, v253     ; encoding: [0x00,0x00,0x38,0xd8,0xff,0xfe,0xfd,0x00]
+0x00,0x00,0x38,0xd8,0xff,0xfe,0xfd,0x00
+
+# GFX12: ds_store_2addr_b32 v255, v254, v253 offset0:123 ; encoding: [0x7b,0x00,0x38,0xd8,0xff,0xfe,0xfd,0x00]
+0x7b,0x00,0x38,0xd8,0xff,0xfe,0xfd,0x00
+
+# GFX12: ds_store_2addr_b32 v255, v254, v253 offset0:123 offset1:64 ; encoding: [0x7b,0x40,0x38,0xd8,0xff,0xfe,0xfd,0x00]
+0x7b,0x40,0x38,0xd8,0xff,0xfe,0xfd,0x00
+
+# GFX12: ds_store_2addr_b32 v255, v254, v253 offset0:255 offset1:255 ; encoding: [0xff,0xff,0x38,0xd8,0xff,0xfe,0xfd,0x00]
+0xff,0xff,0x38,0xd8,0xff,0xfe,0xfd,0x00
+
+# GFX12: ds_store_2addr_b32 v255, v254, v253 offset0:255 offset1:64 ; encoding: [0xff,0x40,0x38,0xd8,0xff,0xfe,0xfd,0x00]
+0xff,0x40,0x38,0xd8,0xff,0xfe,0xfd,0x00
+
+# GFX12: ds_store_2addr_b32 v255, v254, v253 offset0:64 offset1:123 ; encoding: [0x40,0x7b,0x38,0xd8,0xff,0xfe,0xfd,0x00]
+0x40,0x7b,0x38,0xd8,0xff,0xfe,0xfd,0x00
+
+# GFX12: ds_store_2addr_b32 v255, v254, v253 offset0:64 offset1:255 ; encoding: [0x40,0xff,0x38,0xd8,0xff,0xfe,0xfd,0x00]
+0x40,0xff,0x38,0xd8,0xff,0xfe,0xfd,0x00
+
+# GFX12: ds_store_2addr_b32 v255, v254, v253 offset1:123 ; encoding: [0x00,0x7b,0x38,0xd8,0xff,0xfe,0xfd,0x00]
+0x00,0x7b,0x38,0xd8,0xff,0xfe,0xfd,0x00
+
+# GFX12: ds_store_2addr_b64 v1, v[254:255], v[3:4] offset0:127 offset1:255 ; encoding: [0x7f,0xff,0x38,0xd9,0x01,0xfe,0x03,0x00]
+0x7f,0xff,0x38,0xd9,0x01,0xfe,0x03,0x00
+
+# GFX12: ds_store_2addr_b64 v1, v[2:3], v[254:255] offset0:127 offset1:255 ; encoding: [0x7f,0xff,0x38,0xd9,0x01,0x02,0xfe,0x00]
+0x7f,0xff,0x38,0xd9,0x01,0x02,0xfe,0x00
+
+# GFX12: ds_store_2addr_b64 v1, v[2:3], v[3:4] offset0:127 ; encoding: [0x7f,0x00,0x38,0xd9,0x01,0x02,0x03,0x00]
+0x7f,0x00,0x38,0xd9,0x01,0x02,0x03,0x00
+
+# GFX12: ds_store_2addr_b64 v1, v[2:3], v[3:4] offset0:127 offset1:1 ; encoding: [0x7f,0x01,0x38,0xd9,0x01,0x02,0x03,0x00]
+0x7f,0x01,0x38,0xd9,0x01,0x02,0x03,0x00
+
+# GFX12: ds_store_2addr_b64 v1, v[2:3], v[3:4] offset0:127 offset1:255 ; encoding: [0x7f,0xff,0x38,0xd9,0x01,0x02,0x03,0x00]
+0x7f,0xff,0x38,0xd9,0x01,0x02,0x03,0x00
+
+# GFX12: ds_store_2addr_b64 v1, v[2:3], v[3:4] offset0:16 offset1:255 ; encoding: [0x10,0xff,0x38,0xd9,0x01,0x02,0x03,0x00]
+0x10,0xff,0x38,0xd9,0x01,0x02,0x03,0x00
+
+# GFX12: ds_store_2addr_b64 v1, v[2:3], v[3:4] offset1:255 ; encoding: [0x00,0xff,0x38,0xd9,0x01,0x02,0x03,0x00]
+0x00,0xff,0x38,0xd9,0x01,0x02,0x03,0x00
+
+# GFX12: ds_store_2addr_b64 v255, v[2:3], v[3:4] offset0:127 offset1:255 ; encoding: [0x7f,0xff,0x38,0xd9,0xff,0x02,0x03,0x00]
+0x7f,0xff,0x38,0xd9,0xff,0x02,0x03,0x00
+
+# GFX12: ds_store_2addr_stride64_b32 v0, v1, v2  ; encoding: [0x00,0x00,0x3c,0xd8,0x00,0x01,0x02,0x00]
+0x00,0x00,0x3c,0xd8,0x00,0x01,0x02,0x00
+
+# GFX12: ds_store_2addr_stride64_b32 v0, v1, v2 offset0:123 ; encoding: [0x7b,0x00,0x3c,0xd8,0x00,0x01,0x02,0x00]
+0x7b,0x00,0x3c,0xd8,0x00,0x01,0x02,0x00
+
+# GFX12: ds_store_2addr_stride64_b32 v0, v1, v2 offset0:123 offset1:64 ; encoding: [0x7b,0x40,0x3c,0xd8,0x00,0x01,0x02,0x00]
+0x7b,0x40,0x3c,0xd8,0x00,0x01,0x02,0x00
+
+# GFX12: ds_store_2addr_stride64_b32 v0, v1, v2 offset0:255 offset1:255 ; encoding: [0xff,0xff,0x3c,0xd8,0x00,0x01,0x02,0x00]
+0xff,0xff,0x3c,0xd8,0x00,0x01,0x02,0x00
+
+# GFX12: ds_store_2addr_stride64_b32 v0, v1, v2 offset0:255 offset1:64 ; encoding: [0xff,0x40,0x3c,0xd8,0x00,0x01,0x02,0x00]
+0xff,0x40,0x3c,0xd8,0x00,0x01,0x02,0x00
+
+# GFX12: ds_store_2addr_stride64_b32 v0, v1, v2 offset0:64 offset1:123 ; encoding: [0x40,0x7b,0x3c,0xd8,0x00,0x01,0x02,0x00]
+0x40,0x7b,0x3c,0xd8,0x00,0x01,0x02,0x00
+
+# GFX12: ds_store_2addr_stride64_b32 v0, v1, v2 offset0:64 offset1:255 ; encoding: [0x40,0xff,0x3c,0xd8,0x00,0x01,0x02,0x00]
+0x40,0xff,0x3c,0xd8,0x00,0x01,0x02,0x00
+
+# GFX12: ds_store_2addr_stride64_b32 v0, v1, v2 offset1:123 ; encoding: [0x00,0x7b,0x3c,0xd8,0x00,0x01,0x02,0x00]
+0x00,0x7b,0x3c,0xd8,0x00,0x01,0x02,0x00
+
+# GFX12: ds_store_2addr_stride64_b32 v0, v254, v253 ; encoding: [0x00,0x00,0x3c,0xd8,0x00,0xfe,0xfd,0x00]
+0x00,0x00,0x3c,0xd8,0x00,0xfe,0xfd,0x00
+
+# GFX12: ds_store_2addr_stride64_b32 v0, v254, v253 offset0:123 ; encoding: [0x7b,0x00,0x3c,0xd8,0x00,0xfe,0xfd,0x00]
+0x7b,0x00,0x3c,0xd8,0x00,0xfe,0xfd,0x00
+
+# GFX12: ds_store_2addr_stride64_b32 v0, v254, v253 offset0:123 offset1:64 ; encoding: [0x7b,0x40,0x3c,0xd8,0x00,0xfe,0xfd,0x00]
+0x7b,0x40,0x3c,0xd8,0x00,0xfe,0xfd,0x00
+
+# GFX12: ds_store_2addr_stride64_b32 v0, v254, v253 offset0:255 offset1:255 ; encoding: [0xff,0xff,0x3c,0xd8,0x00,0xfe,0xfd,0x00]
+0xff,0xff,0x3c,0xd8,0x00,0xfe,0xfd,0x00
+
+# GFX12: ds_store_2addr_stride64_b32 v0, v254, v253 offset0:255 offset1:64 ; encoding: [0xff,0x40,0x3c,0xd8,0x00,0xfe,0xfd,0x00]
+0xff,0x40,0x3c,0xd8,0x00,0xfe,0xfd,0x00
+
+# GFX12: ds_store_2addr_stride64_b32 v0, v254, v253 offset0:64 offset1:123 ; encoding: [0x40,0x7b,0x3c,0xd8,0x00,0xfe,0xfd,0x00]
+0x40,0x7b,0x3c,0xd8,0x00,0xfe,0xfd,0x00
+
+# GFX12: ds_store_2addr_stride64_b32 v0, v254, v253 offset0:64 offset1:255 ; encoding: [0x40,0xff,0x3c,0xd8,0x00,0xfe,0xfd,0x00]
+0x40,0xff,0x3c,0xd8,0x00,0xfe,0xfd,0x00
+
+# GFX12: ds_store_2addr_stride64_b32 v0, v254, v253 offset1:123 ; encoding: [0x00,0x7b,0x3c,0xd8,0x00,0xfe,0xfd,0x00]
+0x00,0x7b,0x3c,0xd8,0x00,0xfe,0xfd,0x00
+
+# GFX12: ds_store_2addr_stride64_b32 v255, v1, v253 ; encoding: [0x00,0x00,0x3c,0xd8,0xff,0x01,0xfd,0x00]
+0x00,0x00,0x3c,0xd8,0xff,0x01,0xfd,0x00
+
+# GFX12: ds_store_2addr_stride64_b32 v255, v1, v253 offset0:123 ; encoding: [0x7b,0x00,0x3c,0xd8,0xff,0x01,0xfd,0x00]
+0x7b,0x00,0x3c,0xd8,0xff,0x01,0xfd,0x00
+
+# GFX12: ds_store_2addr_stride64_b32 v255, v1, v253 offset0:123 offset1:64 ; encoding: [0x7b,0x40,0x3c,0xd8,0xff,0x01,0xfd,0x00]
+0x7b,0x40,0x3c,0xd8,0xff,0x01,0xfd,0x00
+
+# GFX12: ds_store_2addr_stride64_b32 v255, v1, v253 offset0:255 offset1:255 ; encoding: [0xff,0xff,0x3c,0xd8,0xff,0x01,0xfd,0x00]
+0xff,0xff,0x3c,0xd8,0xff,0x01,0xfd,0x00
+
+# GFX12: ds_store_2addr_stride64_b32 v255, v1, v253 offset0:255 offset1:64 ; encoding: [0xff,0x40,0x3c,0xd8,0xff,0x01,0xfd,0x00]
+0xff,0x40,0x3c,0xd8,0xff,0x01,0xfd,0x00
+
+# GFX12: ds_store_2addr_stride64_b32 v255, v1, v253 offset0:64 offset1:123 ; encoding: [0x40,0x7b,0x3c,0xd8,0xff,0x01,0xfd,0x00]
+0x40,0x7b,0x3c,0xd8,0xff,0x01,0xfd,0x00
+
+# GFX12: ds_store_2addr_stride64_b32 v255, v1, v253 offset0:64 offset1:255 ; encoding: [0x40,0xff,0x3c,0xd8,0xff,0x01,0xfd,0x00]
+0x40,0xff,0x3c,0xd8,0xff,0x01,0xfd,0x00
+
+# GFX12: ds_store_2addr_stride64_b32 v255, v1, v253 offset1:123 ; encoding: [0x00,0x7b,0x3c,0xd8,0xff,0x01,0xfd,0x00]
+0x00,0x7b,0x3c,0xd8,0xff,0x01,0xfd,0x00
+
+# GFX12: ds_store_2addr_stride64_b32 v255, v254, v2 ; encoding: [0x00,0x00,0x3c,0xd8,0xff,0xfe,0x02,0x00]
+0x00,0x00,0x3c,0xd8,0xff,0xfe,0x02,0x00
+
+# GFX12: ds_store_2addr_stride64_b32 v255, v254, v2 offset0:123 ; encoding: [0x7b,0x00,0x3c,0xd8,0xff,0xfe,0x02,0x00]
+0x7b,0x00,0x3c,0xd8,0xff,0xfe,0x02,0x00
+
+# GFX12: ds_store_2addr_stride64_b32 v255, v254, v2 offset0:123 offset1:64 ; encoding: [0x7b,0x40,0x3c,0xd8,0xff,0xfe,0x02,0x00]
+0x7b,0x40,0x3c,0xd8,0xff,0xfe,0x02,0x00
+
+# GFX12: ds_store_2addr_stride64_b32 v255, v254, v2 offset0:255 offset1:255 ; encoding: [0xff,0xff,0x3c,0xd8,0xff,0xfe,0x02,0x00]
+0xff,0xff,0x3c,0xd8,0xff,0xfe,0x02,0x00
+
+# GFX12: ds_store_2addr_stride64_b32 v255, v254, v2 offset0:255 offset1:64 ; encoding: [0xff,0x40,0x3c,0xd8,0xff,0xfe,0x02,0x00]
+0xff,0x40,0x3c,0xd8,0xff,0xfe,0x02,0x00
+
+# GFX12: ds_store_2addr_stride64_b32 v255, v254, v2 offset0:64 offset1:123 ; encoding: [0x40,0x7b,0x3c,0xd8,0xff,0xfe,0x02,0x00]
+0x40,0x7b,0x3c,0xd8,0xff,0xfe,0x02,0x00
+
+# GFX12: ds_store_2addr_stride64_b32 v255, v254, v2 offset0:64 offset1:255 ; encoding: [0x40,0xff,0x3c,0xd8,0xff,0xfe,0x02,0x00]
+0x40,0xff,0x3c,0xd8,0xff,0xfe,0x02,0x00
+
+# GFX12: ds_store_2addr_stride64_b32 v255, v254, v2 offset1:123 ; encoding: [0x00,0x7b,0x3c,0xd8,0xff,0xfe,0x02,0x00]
+0x00,0x7b,0x3c,0xd8,0xff,0xfe,0x02,0x00
+
+# GFX12: ds_store_2addr_stride64_b32 v255, v254, v253 ; encoding: [0x00,0x00,0x3c,0xd8,0xff,0xfe,0xfd,0x00]
+0x00,0x00,0x3c,0xd8,0xff,0xfe,0xfd,0x00
+
+# GFX12: ds_store_2addr_stride64_b32 v255, v254, v253 offset0:123 ; encoding: [0x7b,0x00,0x3c,0xd8,0xff,0xfe,0xfd,0x00]
+0x7b,0x00,0x3c,0xd8,0xff,0xfe,0xfd,0x00
+
+# GFX12: ds_store_2addr_stride64_b32 v255, v254, v253 offset0:123 offset1:64 ; encoding: [0x7b,0x40,0x3c,0xd8,0xff,0xfe,0xfd,0x00]
+0x7b,0x40,0x3c,0xd8,0xff,0xfe,0xfd,0x00
+
+# GFX12: ds_store_2addr_stride64_b32 v255, v254, v253 offset0:255 offset1:255 ; encoding: [0xff,0xff,0x3c,0xd8,0xff,0xfe,0xfd,0x00]
+0xff,0xff,0x3c,0xd8,0xff,0xfe,0xfd,0x00
+
+# GFX12: ds_store_2addr_stride64_b32 v255, v254, v253 offset0:255 offset1:64 ; encoding: [0xff,0x40,0x3c,0xd8,0xff,0xfe,0xfd,0x00]
+0xff,0x40,0x3c,0xd8,0xff,0xfe,0xfd,0x00
+
+# GFX12: ds_store_2addr_stride64_b32 v255, v254, v253 offset0:64 offset1:123 ; encoding: [0x40,0x7b,0x3c,0xd8,0xff,0xfe,0xfd,0x00]
+0x40,0x7b,0x3c,0xd8,0xff,0xfe,0xfd,0x00
+
+# GFX12: ds_store_2addr_stride64_b32 v255, v254, v253 offset0:64 offset1:255 ; encoding: [0x40,0xff,0x3c,0xd8,0xff,0xfe,0xfd,0x00]
+0x40,0xff,0x3c,0xd8,0xff,0xfe,0xfd,0x00
+
+# GFX12: ds_store_2addr_stride64_b32 v255, v254, v253 offset1:123 ; encoding: [0x00,0x7b,0x3c,0xd8,0xff,0xfe,0xfd,0x00]
+0x00,0x7b,0x3c,0xd8,0xff,0xfe,0xfd,0x00
+
+# GFX12: ds_store_2addr_stride64_b64 v1, v[254:255], v[3:4] offset0:127 offset1:255 ; encoding: [0x7f,0xff,0x3c,0xd9,0x01,0xfe,0x03,0x00]
+0x7f,0xff,0x3c,0xd9,0x01,0xfe,0x03,0x00
+
+# GFX12: ds_store_2addr_stride64_b64 v1, v[2:3], v[254:255] offset0:127 offset1:255 ; encoding: [0x7f,0xff,0x3c,0xd9,0x01,0x02,0xfe,0x00]
+0x7f,0xff,0x3c,0xd9,0x01,0x02,0xfe,0x00
+
+# GFX12: ds_store_2addr_stride64_b64 v1, v[2:3], v[3:4] offset0:127 ; encoding: [0x7f,0x00,0x3c,0xd9,0x01,0x02,0x03,0x00]
+0x7f,0x00,0x3c,0xd9,0x01,0x02,0x03,0x00
+
+# GFX12: ds_store_2addr_stride64_b64 v1, v[2:3], v[3:4] offset0:127 offset1:1 ; encoding: [0x7f,0x01,0x3c,0xd9,0x01,0x02,0x03,0x00]
+0x7f,0x01,0x3c,0xd9,0x01,0x02,0x03,0x00
+
+# GFX12: ds_store_2addr_stride64_b64 v1, v[2:3], v[3:4] offset0:127 offset1:255 ; encoding: [0x7f,0xff,0x3c,0xd9,0x01,0x02,0x03,0x00]
+0x7f,0xff,0x3c,0xd9,0x01,0x02,0x03,0x00
+
+# GFX12: ds_store_2addr_stride64_b64 v1, v[2:3], v[3:4] offset0:16 offset1:255 ; encoding: [0x10,0xff,0x3c,0xd9,0x01,0x02,0x03,0x00]
+0x10,0xff,0x3c,0xd9,0x01,0x02,0x03,0x00
+
+# GFX12: ds_store_2addr_stride64_b64 v1, v[2:3], v[3:4] offset1:255 ; encoding: [0x00,0xff,0x3c,0xd9,0x01,0x02,0x03,0x00]
+0x00,0xff,0x3c,0xd9,0x01,0x02,0x03,0x00
+
+# GFX12: ds_store_2addr_stride64_b64 v255, v[2:3], v[3:4] offset0:127 offset1:255 ; encoding: [0x7f,0xff,0x3c,0xd9,0xff,0x02,0x03,0x00]
+0x7f,0xff,0x3c,0xd9,0xff,0x02,0x03,0x00
+
+# GFX12: ds_store_addtid_b32 v255 offset:65535   ; encoding: [0xff,0xff,0xc0,0xda,0x00,0xff,0x00,0x00]
+0xff 0xff 0xc0 0xda 0x00 0xff 0x00 0x00
+
+# GFX12: ds_store_addtid_b32 v5                  ; encoding: [0x00,0x00,0xc0,0xda,0x00,0x05,0x00,0x00]
+0x00 0x00 0xc0 0xda 0x00 0x05 0x00 0x00
+
+# GFX12: ds_store_addtid_b32 v5 offset:4         ; encoding: [0x04,0x00,0xc0,0xda,0x00,0x05,0x00,0x00]
+0x04 0x00 0xc0 0xda 0x00 0x05 0x00 0x00
+
+# GFX12: ds_store_addtid_b32 v5 offset:65535     ; encoding: [0xff,0xff,0xc0,0xda,0x00,0x05,0x00,0x00]
+0xff 0xff 0xc0 0xda 0x00 0x05 0x00 0x00
+
+# GFX12: ds_store_b128 v1, v[252:255] offset:65535 ; encoding: [0xff,0xff,0x7c,0xdb,0x01,0xfc,0x00,0x00]
+0xff,0xff,0x7c,0xdb,0x01,0xfc,0x00,0x00
+
+# GFX12: ds_store_b128 v1, v[2:5]                ; encoding: [0x00,0x00,0x7c,0xdb,0x01,0x02,0x00,0x00]
+0x00,0x00,0x7c,0xdb,0x01,0x02,0x00,0x00
+
+# GFX12: ds_store_b128 v1, v[2:5] offset:4       ; encoding: [0x04,0x00,0x7c,0xdb,0x01,0x02,0x00,0x00]
+0x04,0x00,0x7c,0xdb,0x01,0x02,0x00,0x00
+
+# GFX12: ds_store_b128 v1, v[2:5] offset:65535   ; encoding: [0xff,0xff,0x7c,0xdb,0x01,0x02,0x00,0x00]
+0xff,0xff,0x7c,0xdb,0x01,0x02,0x00,0x00
+
+# GFX12: ds_store_b128 v255, v[2:5] offset:65535 ; encoding: [0xff,0xff,0x7c,0xdb,0xff,0x02,0x00,0x00]
+0xff,0xff,0x7c,0xdb,0xff,0x02,0x00,0x00
+
+# GFX12: ds_store_b16 v0, v1                     ; encoding: [0x00,0x00,0x7c,0xd8,0x00,0x01,0x00,0x00]
+0x00,0x00,0x7c,0xd8,0x00,0x01,0x00,0x00
+
+# GFX12: ds_store_b16 v0, v1 offset:4660         ; encoding: [0x34,0x12,0x7c,0xd8,0x00,0x01,0x00,0x00]
+0x34,0x12,0x7c,0xd8,0x00,0x01,0x00,0x00
+
+# GFX12: ds_store_b16 v0, v1 offset:65535        ; encoding: [0xff,0xff,0x7c,0xd8,0x00,0x01,0x00,0x00]
+0xff,0xff,0x7c,0xd8,0x00,0x01,0x00,0x00
+
+# GFX12: ds_store_b16 v0, v254                   ; encoding: [0x00,0x00,0x7c,0xd8,0x00,0xfe,0x00,0x00]
+0x00,0x00,0x7c,0xd8,0x00,0xfe,0x00,0x00
+
+# GFX12: ds_store_b16 v0, v254 offset:4660       ; encoding: [0x34,0x12,0x7c,0xd8,0x00,0xfe,0x00,0x00]
+0x34,0x12,0x7c,0xd8,0x00,0xfe,0x00,0x00
+
+# GFX12: ds_store_b16 v0, v254 offset:65535      ; encoding: [0xff,0xff,0x7c,0xd8,0x00,0xfe,0x00,0x00]
+0xff,0xff,0x7c,0xd8,0x00,0xfe,0x00,0x00
+
+# GFX12: ds_store_b16 v255, v1                   ; encoding: [0x00,0x00,0x7c,0xd8,0xff,0x01,0x00,0x00]
+0x00,0x00,0x7c,0xd8,0xff,0x01,0x00,0x00
+
+# GFX12: ds_store_b16 v255, v1 offset:4660       ; encoding: [0x34,0x12,0x7c,0xd8,0xff,0x01,0x00,0x00]
+0x34,0x12,0x7c,0xd8,0xff,0x01,0x00,0x00
+
+# GFX12: ds_store_b16 v255, v1 offset:65535      ; encoding: [0xff,0xff,0x7c,0xd8,0xff,0x01,0x00,0x00]
+0xff,0xff,0x7c,0xd8,0xff,0x01,0x00,0x00
+
+# GFX12: ds_store_b16 v255, v254                 ; encoding: [0x00,0x00,0x7c,0xd8,0xff,0xfe,0x00,0x00]
+0x00,0x00,0x7c,0xd8,0xff,0xfe,0x00,0x00
+
+# GFX12: ds_store_b16 v255, v254 offset:4660     ; encoding: [0x34,0x12,0x7c,0xd8,0xff,0xfe,0x00,0x00]
+0x34,0x12,0x7c,0xd8,0xff,0xfe,0x00,0x00
+
+# GFX12: ds_store_b16 v255, v254 offset:65535    ; encoding: [0xff,0xff,0x7c,0xd8,0xff,0xfe,0x00,0x00]
+0xff,0xff,0x7c,0xd8,0xff,0xfe,0x00,0x00
+
+# GFX12: ds_store_b16_d16_hi v1, v2              ; encoding: [0x00,0x00,0x84,0xda,0x01,0x02,0x00,0x00]
+0x00,0x00,0x84,0xda,0x01,0x02,0x00,0x00
+
+# GFX12: ds_store_b16_d16_hi v1, v2 offset:4     ; encoding: [0x04,0x00,0x84,0xda,0x01,0x02,0x00,0x00]
+0x04,0x00,0x84,0xda,0x01,0x02,0x00,0x00
+
+# GFX12: ds_store_b16_d16_hi v1, v2 offset:65535 ; encoding: [0xff,0xff,0x84,0xda,0x01,0x02,0x00,0x00]
+0xff,0xff,0x84,0xda,0x01,0x02,0x00,0x00
+
+# GFX12: ds_store_b16_d16_hi v1, v255 offset:65535 ; encoding: [0xff,0xff,0x84,0xda,0x01,0xff,0x00,0x00]
+0xff,0xff,0x84,0xda,0x01,0xff,0x00,0x00
+
+# GFX12: ds_store_b16_d16_hi v255, v2 offset:65535 ; encoding: [0xff,0xff,0x84,0xda,0xff,0x02,0x00,0x00]
+0xff,0xff,0x84,0xda,0xff,0x02,0x00,0x00
+
+# GFX12: ds_store_b32 v0, v1                     ; encoding: [0x00,0x00,0x34,0xd8,0x00,0x01,0x00,0x00]
+0x00,0x00,0x34,0xd8,0x00,0x01,0x00,0x00
+
+# GFX12: ds_store_b32 v0, v1 offset:4660         ; encoding: [0x34,0x12,0x34,0xd8,0x00,0x01,0x00,0x00]
+0x34,0x12,0x34,0xd8,0x00,0x01,0x00,0x00
+
+# GFX12: ds_store_b32 v0, v1 offset:65535        ; encoding: [0xff,0xff,0x34,0xd8,0x00,0x01,0x00,0x00]
+0xff,0xff,0x34,0xd8,0x00,0x01,0x00,0x00
+
+# GFX12: ds_store_b32 v0, v254                   ; encoding: [0x00,0x00,0x34,0xd8,0x00,0xfe,0x00,0x00]
+0x00,0x00,0x34,0xd8,0x00,0xfe,0x00,0x00
+
+# GFX12: ds_store_b32 v0, v254 offset:4660       ; encoding: [0x34,0x12,0x34,0xd8,0x00,0xfe,0x00,0x00]
+0x34,0x12,0x34,0xd8,0x00,0xfe,0x00,0x00
+
+# GFX12: ds_store_b32 v0, v254 offset:65535      ; encoding: [0xff,0xff,0x34,0xd8,0x00,0xfe,0x00,0x00]
+0xff,0xff,0x34,0xd8,0x00,0xfe,0x00,0x00
+
+# GFX12: ds_store_b32 v255, v1                   ; encoding: [0x00,0x00,0x34,0xd8,0xff,0x01,0x00,0x00]
+0x00,0x00,0x34,0xd8,0xff,0x01,0x00,0x00
+
+# GFX12: ds_store_b32 v255, v1 offset:4660       ; encoding: [0x34,0x12,0x34,0xd8,0xff,0x01,0x00,0x00]
+0x34,0x12,0x34,0xd8,0xff,0x01,0x00,0x00
+
+# GFX12: ds_store_b32 v255, v1 offset:65535      ; encoding: [0xff,0xff,0x34,0xd8,0xff,0x01,0x00,0x00]
+0xff,0xff,0x34,0xd8,0xff,0x01,0x00,0x00
+
+# GFX12: ds_store_b32 v255, v254                 ; encoding: [0x00,0x00,0x34,0xd8,0xff,0xfe,0x00,0x00]
+0x00,0x00,0x34,0xd8,0xff,0xfe,0x00,0x00
+
+# GFX12: ds_store_b32 v255, v254 offset:4660     ; encoding: [0x34,0x12,0x34,0xd8,0xff,0xfe,0x00,0x00]
+0x34,0x12,0x34,0xd8,0xff,0xfe,0x00,0x00
+
+# GFX12: ds_store_b32 v255, v254 offset:65535    ; encoding: [0xff,0xff,0x34,0xd8,0xff,0xfe,0x00,0x00]
+0xff,0xff,0x34,0xd8,0xff,0xfe,0x00,0x00
+
+# GFX12: ds_store_b64 v1, v[254:255] offset:65535 ; encoding: [0xff,0xff,0x34,0xd9,0x01,0xfe,0x00,0x00]
+0xff,0xff,0x34,0xd9,0x01,0xfe,0x00,0x00
+
+# GFX12: ds_store_b64 v1, v[2:3]                 ; encoding: [0x00,0x00,0x34,0xd9,0x01,0x02,0x00,0x00]
+0x00,0x00,0x34,0xd9,0x01,0x02,0x00,0x00
+
+# GFX12: ds_store_b64 v1, v[2:3] offset:4        ; encoding: [0x04,0x00,0x34,0xd9,0x01,0x02,0x00,0x00]
+0x04,0x00,0x34,0xd9,0x01,0x02,0x00,0x00
+
+# GFX12: ds_store_b64 v1, v[2:3] offset:65535    ; encoding: [0xff,0xff,0x34,0xd9,0x01,0x02,0x00,0x00]
+0xff,0xff,0x34,0xd9,0x01,0x02,0x00,0x00
+
+# GFX12: ds_store_b64 v255, v[2:3] offset:65535  ; encoding: [0xff,0xff,0x34,0xd9,0xff,0x02,0x00,0x00]
+0xff,0xff,0x34,0xd9,0xff,0x02,0x00,0x00
+
+# GFX12: ds_store_b8 v0, v1                      ; encoding: [0x00,0x00,0x78,0xd8,0x00,0x01,0x00,0x00]
+0x00,0x00,0x78,0xd8,0x00,0x01,0x00,0x00
+
+# GFX12: ds_store_b8 v0, v1 offset:4660          ; encoding: [0x34,0x12,0x78,0xd8,0x00,0x01,0x00,0x00]
+0x34,0x12,0x78,0xd8,0x00,0x01,0x00,0x00
+
+# GFX12: ds_store_b8 v0, v1 offset:65535         ; encoding: [0xff,0xff,0x78,0xd8,0x00,0x01,0x00,0x00]
+0xff,0xff,0x78,0xd8,0x00,0x01,0x00,0x00
+
+# GFX12: ds_store_b8 v0, v254                    ; encoding: [0x00,0x00,0x78,0xd8,0x00,0xfe,0x00,0x00]
+0x00,0x00,0x78,0xd8,0x00,0xfe,0x00,0x00
+
+# GFX12: ds_store_b8 v0, v254 offset:4660        ; encoding: [0x34,0x12,0x78,0xd8,0x00,0xfe,0x00,0x00]
+0x34,0x12,0x78,0xd8,0x00,0xfe,0x00,0x00
+
+# GFX12: ds_store_b8 v0, v254 offset:65535       ; encoding: [0xff,0xff,0x78,0xd8,0x00,0xfe,0x00,0x00]
+0xff,0xff,0x78,0xd8,0x00,0xfe,0x00,0x00
+
+# GFX12: ds_store_b8 v255, v1                    ; encoding: [0x00,0x00,0x78,0xd8,0xff,0x01,0x00,0x00]
+0x00,0x00,0x78,0xd8,0xff,0x01,0x00,0x00
+
+# GFX12: ds_store_b8 v255, v1 offset:4660        ; encoding: [0x34,0x12,0x78,0xd8,0xff,0x01,0x00,0x00]
+0x34,0x12,0x78,0xd8,0xff,0x01,0x00,0x00
+
+# GFX12: ds_store_b8 v255, v1 offset:65535       ; encoding: [0xff,0xff,0x78,0xd8,0xff,0x01,0x00,0x00]
+0xff,0xff,0x78,0xd8,0xff,0x01,0x00,0x00
+
+# GFX12: ds_store_b8 v255, v254                  ; encoding: [0x00,0x00,0x78,0xd8,0xff,0xfe,0x00,0x00]
+0x00,0x00,0x78,0xd8,0xff,0xfe,0x00,0x00
+
+# GFX12: ds_store_b8 v255, v254 offset:4660      ; encoding: [0x34,0x12,0x78,0xd8,0xff,0xfe,0x00,0x00]
+0x34,0x12,0x78,0xd8,0xff,0xfe,0x00,0x00
+
+# GFX12: ds_store_b8 v255, v254 offset:65535     ; encoding: [0xff,0xff,0x78,0xd8,0xff,0xfe,0x00,0x00]
+0xff,0xff,0x78,0xd8,0xff,0xfe,0x00,0x00
+
+# GFX12: ds_store_b8_d16_hi v1, v2               ; encoding: [0x00,0x00,0x80,0xda,0x01,0x02,0x00,0x00]
+0x00,0x00,0x80,0xda,0x01,0x02,0x00,0x00
+
+# GFX12: ds_store_b8_d16_hi v1, v2 offset:4      ; encoding: [0x04,0x00,0x80,0xda,0x01,0x02,0x00,0x00]
+0x04,0x00,0x80,0xda,0x01,0x02,0x00,0x00
+
+# GFX12: ds_store_b8_d16_hi v1, v2 offset:65535  ; encoding: [0xff,0xff,0x80,0xda,0x01,0x02,0x00,0x00]
+0xff,0xff,0x80,0xda,0x01,0x02,0x00,0x00
+
+# GFX12: ds_store_b8_d16_hi v1, v255 offset:65535 ; encoding: [0xff,0xff,0x80,0xda,0x01,0xff,0x00,0x00]
+0xff,0xff,0x80,0xda,0x01,0xff,0x00,0x00
+
+# GFX12: ds_store_b8_d16_hi v255, v2 offset:65535 ; encoding: [0xff,0xff,0x80,0xda,0xff,0x02,0x00,0x00]
+0xff,0xff,0x80,0xda,0xff,0x02,0x00,0x00
+
+# GFX12: ds_store_b96 v1, v[253:255] offset:65535 ; encoding: [0xff,0xff,0x78,0xdb,0x01,0xfd,0x00,0x00]
+0xff,0xff,0x78,0xdb,0x01,0xfd,0x00,0x00
+
+# GFX12: ds_store_b96 v1, v[2:4]                 ; encoding: [0x00,0x00,0x78,0xdb,0x01,0x02,0x00,0x00]
+0x00,0x00,0x78,0xdb,0x01,0x02,0x00,0x00
+
+# GFX12: ds_store_b96 v1, v[2:4] offset:4        ; encoding: [0x04,0x00,0x78,0xdb,0x01,0x02,0x00,0x00]
+0x04,0x00,0x78,0xdb,0x01,0x02,0x00,0x00
+
+# GFX12: ds_store_b96 v1, v[2:4] offset:65535    ; encoding: [0xff,0xff,0x78,0xdb,0x01,0x02,0x00,0x00]
+0xff,0xff,0x78,0xdb,0x01,0x02,0x00,0x00
+
+# GFX12: ds_store_b96 v255, v[2:4] offset:65535  ; encoding: [0xff,0xff,0x78,0xdb,0xff,0x02,0x00,0x00]
+0xff,0xff,0x78,0xdb,0xff,0x02,0x00,0x00
+
+# GFX12: ds_storexchg_2addr_rtn_b32 v[254:255], v1, v2, v3 offset0:127 offset1:255 ; encoding: [0x7f,0xff,0xb8,0xd8,0x01,0x02,0x03,0xfe]
+0x7f,0xff,0xb8,0xd8,0x01,0x02,0x03,0xfe
+
+# GFX12: ds_storexchg_2addr_rtn_b32 v[5:6], v1, v2, v255 offset0:127 offset1:255 ; encoding: [0x7f,0xff,0xb8,0xd8,0x01,0x02,0xff,0x05]
+0x7f,0xff,0xb8,0xd8,0x01,0x02,0xff,0x05
+
+# GFX12: ds_storexchg_2addr_rtn_b32 v[5:6], v1, v2, v3 offset0:127 ; encoding: [0x7f,0x00,0xb8,0xd8,0x01,0x02,0x03,0x05]
+0x7f,0x00,0xb8,0xd8,0x01,0x02,0x03,0x05
+
+# GFX12: ds_storexchg_2addr_rtn_b32 v[5:6], v1, v2, v3 offset0:127 offset1:1 ; encoding: [0x7f,0x01,0xb8,0xd8,0x01,0x02,0x03,0x05]
+0x7f,0x01,0xb8,0xd8,0x01,0x02,0x03,0x05
+
+# GFX12: ds_storexchg_2addr_rtn_b32 v[5:6], v1, v2, v3 offset0:127 offset1:255 ; encoding: [0x7f,0xff,0xb8,0xd8,0x01,0x02,0x03,0x05]
+0x7f,0xff,0xb8,0xd8,0x01,0x02,0x03,0x05
+
+# GFX12: ds_storexchg_2addr_rtn_b32 v[5:6], v1, v2, v3 offset0:16 offset1:255 ; encoding: [0x10,0xff,0xb8,0xd8,0x01,0x02,0x03,0x05]
+0x10,0xff,0xb8,0xd8,0x01,0x02,0x03,0x05
+
+# GFX12: ds_storexchg_2addr_rtn_b32 v[5:6], v1, v2, v3 offset1:255 ; encoding: [0x00,0xff,0xb8,0xd8,0x01,0x02,0x03,0x05]
+0x00,0xff,0xb8,0xd8,0x01,0x02,0x03,0x05
+
+# GFX12: ds_storexchg_2addr_rtn_b32 v[5:6], v1, v255, v3 offset0:127 offset1:255 ; encoding: [0x7f,0xff,0xb8,0xd8,0x01,0xff,0x03,0x05]
+0x7f,0xff,0xb8,0xd8,0x01,0xff,0x03,0x05
+
+# GFX12: ds_storexchg_2addr_rtn_b32 v[5:6], v255, v2, v3 offset0:127 offset1:255 ; encoding: [0x7f,0xff,0xb8,0xd8,0xff,0x02,0x03,0x05]
+0x7f,0xff,0xb8,0xd8,0xff,0x02,0x03,0x05
+
+# GFX12: ds_storexchg_2addr_rtn_b64 v[252:255], v1, v[2:3], v[3:4] offset0:127 offset1:255 ; encoding: [0x7f,0xff,0xb8,0xd9,0x01,0x02,0x03,0xfc]
+0x7f,0xff,0xb8,0xd9,0x01,0x02,0x03,0xfc
+
+# GFX12: ds_storexchg_2addr_rtn_b64 v[5:8], v1, v[254:255], v[3:4] offset0:127 offset1:255 ; encoding: [0x7f,0xff,0xb8,0xd9,0x01,0xfe,0x03,0x05]
+0x7f,0xff,0xb8,0xd9,0x01,0xfe,0x03,0x05
+
+# GFX12: ds_storexchg_2addr_rtn_b64 v[5:8], v1, v[2:3], v[254:255] offset0:127 offset1:255 ; encoding: [0x7f,0xff,0xb8,0xd9,0x01,0x02,0xfe,0x05]
+0x7f,0xff,0xb8,0xd9,0x01,0x02,0xfe,0x05
+
+# GFX12: ds_storexchg_2addr_rtn_b64 v[5:8], v1, v[2:3], v[3:4] offset0:127 ; encoding: [0x7f,0x00,0xb8,0xd9,0x01,0x02,0x03,0x05]
+0x7f,0x00,0xb8,0xd9,0x01,0x02,0x03,0x05
+
+# GFX12: ds_storexchg_2addr_rtn_b64 v[5:8], v1, v[2:3], v[3:4] offset0:127 offset1:1 ; encoding: [0x7f,0x01,0xb8,0xd9,0x01,0x02,0x03,0x05]
+0x7f,0x01,0xb8,0xd9,0x01,0x02,0x03,0x05
+
+# GFX12: ds_storexchg_2addr_rtn_b64 v[5:8], v1, v[2:3], v[3:4] offset0:127 offset1:255 ; encoding: [0x7f,0xff,0xb8,0xd9,0x01,0x02,0x03,0x05]
+0x7f,0xff,0xb8,0xd9,0x01,0x02,0x03,0x05
+
+# GFX12: ds_storexchg_2addr_rtn_b64 v[5:8], v1, v[2:3], v[3:4] offset0:16 offset1:255 ; encoding: [0x10,0xff,0xb8,0xd9,0x01,0x02,0x03,0x05]
+0x10,0xff,0xb8,0xd9,0x01,0x02,0x03,0x05
+
+# GFX12: ds_storexchg_2addr_rtn_b64 v[5:8], v1, v[2:3], v[3:4] offset1:255 ; encoding: [0x00,0xff,0xb8,0xd9,0x01,0x02,0x03,0x05]
+0x00,0xff,0xb8,0xd9,0x01,0x02,0x03,0x05
+
+# GFX12: ds_storexchg_2addr_rtn_b64 v[5:8], v255, v[2:3], v[3:4] offset0:127 offset1:255 ; encoding: [0x7f,0xff,0xb8,0xd9,0xff,0x02,0x03,0x05]
+0x7f,0xff,0xb8,0xd9,0xff,0x02,0x03,0x05
+
+# GFX12: ds_storexchg_2addr_stride64_rtn_b32 v[254:255], v1, v2, v3 offset0:127 offset1:255 ; encoding: [0x7f,0xff,0xbc,0xd8,0x01,0x02,0x03,0xfe]
+0x7f,0xff,0xbc,0xd8,0x01,0x02,0x03,0xfe
+
+# GFX12: ds_storexchg_2addr_stride64_rtn_b32 v[5:6], v1, v2, v255 offset0:127 offset1:255 ; encoding: [0x7f,0xff,0xbc,0xd8,0x01,0x02,0xff,0x05]
+0x7f,0xff,0xbc,0xd8,0x01,0x02,0xff,0x05
+
+# GFX12: ds_storexchg_2addr_stride64_rtn_b32 v[5:6], v1, v2, v3 offset0:127 ; encoding: [0x7f,0x00,0xbc,0xd8,0x01,0x02,0x03,0x05]
+0x7f,0x00,0xbc,0xd8,0x01,0x02,0x03,0x05
+
+# GFX12: ds_storexchg_2addr_stride64_rtn_b32 v[5:6], v1, v2, v3 offset0:127 offset1:1 ; encoding: [0x7f,0x01,0xbc,0xd8,0x01,0x02,0x03,0x05]
+0x7f,0x01,0xbc,0xd8,0x01,0x02,0x03,0x05
+
+# GFX12: ds_storexchg_2addr_stride64_rtn_b32 v[5:6], v1, v2, v3 offset0:127 offset1:255 ; encoding: [0x7f,0xff,0xbc,0xd8,0x01,0x02,0x03,0x05]
+0x7f,0xff,0xbc,0xd8,0x01,0x02,0x03,0x05
+
+# GFX12: ds_storexchg_2addr_stride64_rtn_b32 v[5:6], v1, v2, v3 offset0:16 offset1:255 ; encoding: [0x10,0xff,0xbc,0xd8,0x01,0x02,0x03,0x05]
+0x10,0xff,0xbc,0xd8,0x01,0x02,0x03,0x05
+
+# GFX12: ds_storexchg_2addr_stride64_rtn_b32 v[5:6], v1, v2, v3 offset1:255 ; encoding: [0x00,0xff,0xbc,0xd8,0x01,0x02,0x03,0x05]
+0x00,0xff,0xbc,0xd8,0x01,0x02,0x03,0x05
+
+# GFX12: ds_storexchg_2addr_stride64_rtn_b32 v[5:6], v1, v255, v3 offset0:127 offset1:255 ; encoding: [0x7f,0xff,0xbc,0xd8,0x01,0xff,0x03,0x05]
+0x7f,0xff,0xbc,0xd8,0x01,0xff,0x03,0x05
+
+# GFX12: ds_storexchg_2addr_stride64_rtn_b32 v[5:6], v255, v2, v3 offset0:127 offset1:255 ; encoding: [0x7f,0xff,0xbc,0xd8,0xff,0x02,0x03,0x05]
+0x7f,0xff,0xbc,0xd8,0xff,0x02,0x03,0x05
+
+# GFX12: ds_storexchg_2addr_stride64_rtn_b64 v[252:255], v1, v[2:3], v[3:4] offset0:127 offset1:255 ; encoding: [0x7f,0xff,0xbc,0xd9,0x01,0x02,0x03,0xfc]
+0x7f,0xff,0xbc,0xd9,0x01,0x02,0x03,0xfc
+
+# GFX12: ds_storexchg_2addr_stride64_rtn_b64 v[5:8], v1, v[254:255], v[3:4] offset0:127 offset1:255 ; encoding: [0x7f,0xff,0xbc,0xd9,0x01,0xfe,0x03,0x05]
+0x7f,0xff,0xbc,0xd9,0x01,0xfe,0x03,0x05
+
+# GFX12: ds_storexchg_2addr_stride64_rtn_b64 v[5:8], v1, v[2:3], v[254:255] offset0:127 offset1:255 ; encoding: [0x7f,0xff,0xbc,0xd9,0x01,0x02,0xfe,0x05]
+0x7f,0xff,0xbc,0xd9,0x01,0x02,0xfe,0x05
+
+# GFX12: ds_storexchg_2addr_stride64_rtn_b64 v[5:8], v1, v[2:3], v[3:4] offset0:127 ; encoding: [0x7f,0x00,0xbc,0xd9,0x01,0x02,0x03,0x05]
+0x7f,0x00,0xbc,0xd9,0x01,0x02,0x03,0x05
+
+# GFX12: ds_storexchg_2addr_stride64_rtn_b64 v[5:8], v1, v[2:3], v[3:4] offset0:127 offset1:1 ; encoding: [0x7f,0x01,0xbc,0xd9,0x01,0x02,0x03,0x05]
+0x7f,0x01,0xbc,0xd9,0x01,0x02,0x03,0x05
+
+# GFX12: ds_storexchg_2addr_stride64_rtn_b64 v[5:8], v1, v[2:3], v[3:4] offset0:127 offset1:255 ; encoding: [0x7f,0xff,0xbc,0xd9,0x01,0x02,0x03,0x05]
+0x7f,0xff,0xbc,0xd9,0x01,0x02,0x03,0x05
+
+# GFX12: ds_storexchg_2addr_stride64_rtn_b64 v[5:8], v1, v[2:3], v[3:4] offset0:16 offset1:255 ; encoding: [0x10,0xff,0xbc,0xd9,0x01,0x02,0x03,0x05]
+0x10,0xff,0xbc,0xd9,0x01,0x02,0x03,0x05
+
+# GFX12: ds_storexchg_2addr_stride64_rtn_b64 v[5:8], v1, v[2:3], v[3:4] offset1:255 ; encoding: [0x00,0xff,0xbc,0xd9,0x01,0x02,0x03,0x05]
+0x00,0xff,0xbc,0xd9,0x01,0x02,0x03,0x05
+
+# GFX12: ds_storexchg_2addr_stride64_rtn_b64 v[5:8], v255, v[2:3], v[3:4] offset0:127 offset1:255 ; encoding: [0x7f,0xff,0xbc,0xd9,0xff,0x02,0x03,0x05]
+0x7f,0xff,0xbc,0xd9,0xff,0x02,0x03,0x05
+
+# GFX12: ds_storexchg_rtn_b32 v0, v1, v2         ; encoding: [0x00,0x00,0xb4,0xd8,0x01,0x02,0x00,0x00]
+0x00,0x00,0xb4,0xd8,0x01,0x02,0x00,0x00
+
+# GFX12: ds_storexchg_rtn_b32 v0, v1, v2 offset:4660 ; encoding: [0x34,0x12,0xb4,0xd8,0x01,0x02,0x00,0x00]
+0x34,0x12,0xb4,0xd8,0x01,0x02,0x00,0x00
+
+# GFX12: ds_storexchg_rtn_b32 v0, v1, v2 offset:65535 ; encoding: [0xff,0xff,0xb4,0xd8,0x01,0x02,0x00,0x00]
+0xff,0xff,0xb4,0xd8,0x01,0x02,0x00,0x00
+
+# GFX12: ds_storexchg_rtn_b32 v0, v254, v253     ; encoding: [0x00,0x00,0xb4,0xd8,0xfe,0xfd,0x00,0x00]
+0x00,0x00,0xb4,0xd8,0xfe,0xfd,0x00,0x00
+
+# GFX12: ds_storexchg_rtn_b32 v0, v254, v253 offset:4660 ; encoding: [0x34,0x12,0xb4,0xd8,0xfe,0xfd,0x00,0x00]
+0x34,0x12,0xb4,0xd8,0xfe,0xfd,0x00,0x00
+
+# GFX12: ds_storexchg_rtn_b32 v0, v254, v253 offset:65535 ; encoding: [0xff,0xff,0xb4,0xd8,0xfe,0xfd,0x00,0x00]
+0xff,0xff,0xb4,0xd8,0xfe,0xfd,0x00,0x00
+
+# GFX12: ds_storexchg_rtn_b32 v255, v1, v253     ; encoding: [0x00,0x00,0xb4,0xd8,0x01,0xfd,0x00,0xff]
+0x00,0x00,0xb4,0xd8,0x01,0xfd,0x00,0xff
+
+# GFX12: ds_storexchg_rtn_b32 v255, v1, v253 offset:4660 ; encoding: [0x34,0x12,0xb4,0xd8,0x01,0xfd,0x00,0xff]
+0x34,0x12,0xb4,0xd8,0x01,0xfd,0x00,0xff
+
+# GFX12: ds_storexchg_rtn_b32 v255, v1, v253 offset:65535 ; encoding: [0xff,0xff,0xb4,0xd8,0x01,0xfd,0x00,0xff]
+0xff,0xff,0xb4,0xd8,0x01,0xfd,0x00,0xff
+
+# GFX12: ds_storexchg_rtn_b32 v255, v254, v2     ; encoding: [0x00,0x00,0xb4,0xd8,0xfe,0x02,0x00,0xff]
+0x00,0x00,0xb4,0xd8,0xfe,0x02,0x00,0xff
+
+# GFX12: ds_storexchg_rtn_b32 v255, v254, v2 offset:4660 ; encoding: [0x34,0x12,0xb4,0xd8,0xfe,0x02,0x00,0xff]
+0x34,0x12,0xb4,0xd8,0xfe,0x02,0x00,0xff
+
+# GFX12: ds_storexchg_rtn_b32 v255, v254, v2 offset:65535 ; encoding: [0xff,0xff,0xb4,0xd8,0xfe,0x02,0x00,0xff]
+0xff,0xff,0xb4,0xd8,0xfe,0x02,0x00,0xff
+
+# GFX12: ds_storexchg_rtn_b32 v255, v254, v253   ; encoding: [0x00,0x00,0xb4,0xd8,0xfe,0xfd,0x00,0xff]
+0x00,0x00,0xb4,0xd8,0xfe,0xfd,0x00,0xff
+
+# GFX12: ds_storexchg_rtn_b32 v255, v254, v253 offset:4660 ; encoding: [0x34,0x12,0xb4,0xd8,0xfe,0xfd,0x00,0xff]
+0x34,0x12,0xb4,0xd8,0xfe,0xfd,0x00,0xff
+
+# GFX12: ds_storexchg_rtn_b32 v255, v254, v253 offset:65535 ; encoding: [0xff,0xff,0xb4,0xd8,0xfe,0xfd,0x00,0xff]
+0xff,0xff,0xb4,0xd8,0xfe,0xfd,0x00,0xff
+
+# GFX12: ds_storexchg_rtn_b64 v[254:255], v1, v[2:3] offset:65535 ; encoding: [0xff,0xff,0xb4,0xd9,0x01,0x02,0x00,0xfe]
+0xff,0xff,0xb4,0xd9,0x01,0x02,0x00,0xfe
+
+# GFX12: ds_storexchg_rtn_b64 v[5:6], v1, v[254:255] offset:65535 ; encoding: [0xff,0xff,0xb4,0xd9,0x01,0xfe,0x00,0x05]
+0xff,0xff,0xb4,0xd9,0x01,0xfe,0x00,0x05
+
+# GFX12: ds_storexchg_rtn_b64 v[5:6], v1, v[2:3] ; encoding: [0x00,0x00,0xb4,0xd9,0x01,0x02,0x00,0x05]
+0x00,0x00,0xb4,0xd9,0x01,0x02,0x00,0x05
+
+# GFX12: ds_storexchg_rtn_b64 v[5:6], v1, v[2:3] offset:4 ; encoding: [0x04,0x00,0xb4,0xd9,0x01,0x02,0x00,0x05]
+0x04,0x00,0xb4,0xd9,0x01,0x02,0x00,0x05
+
+# GFX12: ds_storexchg_rtn_b64 v[5:6], v1, v[2:3] offset:65535 ; encoding: [0xff,0xff,0xb4,0xd9,0x01,0x02,0x00,0x05]
+0xff,0xff,0xb4,0xd9,0x01,0x02,0x00,0x05
+
+# GFX12: ds_storexchg_rtn_b64 v[5:6], v255, v[2:3] offset:65535 ; encoding: [0xff,0xff,0xb4,0xd9,0xff,0x02,0x00,0x05]
+0xff,0xff,0xb4,0xd9,0xff,0x02,0x00,0x05
+
+# GFX12: ds_xor_b32 v0, v1                       ; encoding: [0x00,0x00,0x2c,0xd8,0x00,0x01,0x00,0x00]
+0x00,0x00,0x2c,0xd8,0x00,0x01,0x00,0x00
+
+# GFX12: ds_xor_b32 v0, v1 offset:4660           ; encoding: [0x34,0x12,0x2c,0xd8,0x00,0x01,0x00,0x00]
+0x34,0x12,0x2c,0xd8,0x00,0x01,0x00,0x00
+
+# GFX12: ds_xor_b32 v0, v1 offset:65535          ; encoding: [0xff,0xff,0x2c,0xd8,0x00,0x01,0x00,0x00]
+0xff,0xff,0x2c,0xd8,0x00,0x01,0x00,0x00
+
+# GFX12: ds_xor_b32 v0, v254                     ; encoding: [0x00,0x00,0x2c,0xd8,0x00,0xfe,0x00,0x00]
+0x00,0x00,0x2c,0xd8,0x00,0xfe,0x00,0x00
+
+# GFX12: ds_xor_b32 v0, v254 offset:4660         ; encoding: [0x34,0x12,0x2c,0xd8,0x00,0xfe,0x00,0x00]
+0x34,0x12,0x2c,0xd8,0x00,0xfe,0x00,0x00
+
+# GFX12: ds_xor_b32 v0, v254 offset:65535        ; encoding: [0xff,0xff,0x2c,0xd8,0x00,0xfe,0x00,0x00]
+0xff,0xff,0x2c,0xd8,0x00,0xfe,0x00,0x00
+
+# GFX12: ds_xor_b32 v255, v1                     ; encoding: [0x00,0x00,0x2c,0xd8,0xff,0x01,0x00,0x00]
+0x00,0x00,0x2c,0xd8,0xff,0x01,0x00,0x00
+
+# GFX12: ds_xor_b32 v255, v1 offset:4660         ; encoding: [0x34,0x12,0x2c,0xd8,0xff,0x01,0x00,0x00]
+0x34,0x12,0x2c,0xd8,0xff,0x01,0x00,0x00
+
+# GFX12: ds_xor_b32 v255, v1 offset:65535        ; encoding: [0xff,0xff,0x2c,0xd8,0xff,0x01,0x00,0x00]
+0xff,0xff,0x2c,0xd8,0xff,0x01,0x00,0x00
+
+# GFX12: ds_xor_b32 v255, v254                   ; encoding: [0x00,0x00,0x2c,0xd8,0xff,0xfe,0x00,0x00]
+0x00,0x00,0x2c,0xd8,0xff,0xfe,0x00,0x00
+
+# GFX12: ds_xor_b32 v255, v254 offset:4660       ; encoding: [0x34,0x12,0x2c,0xd8,0xff,0xfe,0x00,0x00]
+0x34,0x12,0x2c,0xd8,0xff,0xfe,0x00,0x00
+
+# GFX12: ds_xor_b32 v255, v254 offset:65535      ; encoding: [0xff,0xff,0x2c,0xd8,0xff,0xfe,0x00,0x00]
+0xff,0xff,0x2c,0xd8,0xff,0xfe,0x00,0x00
+
+# GFX12: ds_xor_b64 v1, v[254:255] offset:65535  ; encoding: [0xff,0xff,0x2c,0xd9,0x01,0xfe,0x00,0x00]
+0xff,0xff,0x2c,0xd9,0x01,0xfe,0x00,0x00
+
+# GFX12: ds_xor_b64 v1, v[2:3]                   ; encoding: [0x00,0x00,0x2c,0xd9,0x01,0x02,0x00,0x00]
+0x00,0x00,0x2c,0xd9,0x01,0x02,0x00,0x00
+
+# GFX12: ds_xor_b64 v1, v[2:3] offset:4          ; encoding: [0x04,0x00,0x2c,0xd9,0x01,0x02,0x00,0x00]
+0x04,0x00,0x2c,0xd9,0x01,0x02,0x00,0x00
+
+# GFX12: ds_xor_b64 v1, v[2:3] offset:65535      ; encoding: [0xff,0xff,0x2c,0xd9,0x01,0x02,0x00,0x00]
+0xff,0xff,0x2c,0xd9,0x01,0x02,0x00,0x00
+
+# GFX12: ds_xor_b64 v255, v[2:3] offset:65535    ; encoding: [0xff,0xff,0x2c,0xd9,0xff,0x02,0x00,0x00]
+0xff,0xff,0x2c,0xd9,0xff,0x02,0x00,0x00
+
+# GFX12: ds_xor_rtn_b32 v0, v1, v2               ; encoding: [0x00,0x00,0xac,0xd8,0x01,0x02,0x00,0x00]
+0x00,0x00,0xac,0xd8,0x01,0x02,0x00,0x00
+
+# GFX12: ds_xor_rtn_b32 v0, v1, v2 offset:4660   ; encoding: [0x34,0x12,0xac,0xd8,0x01,0x02,0x00,0x00]
+0x34,0x12,0xac,0xd8,0x01,0x02,0x00,0x00
+
+# GFX12: ds_xor_rtn_b32 v0, v1, v2 offset:65535  ; encoding: [0xff,0xff,0xac,0xd8,0x01,0x02,0x00,0x00]
+0xff,0xff,0xac,0xd8,0x01,0x02,0x00,0x00
+
+# GFX12: ds_xor_rtn_b32 v0, v254, v253           ; encoding: [0x00,0x00,0xac,0xd8,0xfe,0xfd,0x00,0x00]
+0x00,0x00,0xac,0xd8,0xfe,0xfd,0x00,0x00
+
+# GFX12: ds_xor_rtn_b32 v0, v254, v253 offset:4660 ; encoding: [0x34,0x12,0xac,0xd8,0xfe,0xfd,0x00,0x00]
+0x34,0x12,0xac,0xd8,0xfe,0xfd,0x00,0x00
+
+# GFX12: ds_xor_rtn_b32 v0, v254, v253 offset:65535 ; encoding: [0xff,0xff,0xac,0xd8,0xfe,0xfd,0x00,0x00]
+0xff,0xff,0xac,0xd8,0xfe,0xfd,0x00,0x00
+
+# GFX12: ds_xor_rtn_b32 v255, v1, v253           ; encoding: [0x00,0x00,0xac,0xd8,0x01,0xfd,0x00,0xff]
+0x00,0x00,0xac,0xd8,0x01,0xfd,0x00,0xff
+
+# GFX12: ds_xor_rtn_b32 v255, v1, v253 offset:4660 ; encoding: [0x34,0x12,0xac,0xd8,0x01,0xfd,0x00,0xff]
+0x34,0x12,0xac,0xd8,0x01,0xfd,0x00,0xff
+
+# GFX12: ds_xor_rtn_b32 v255, v1, v253 offset:65535 ; encoding: [0xff,0xff,0xac,0xd8,0x01,0xfd,0x00,0xff]
+0xff,0xff,0xac,0xd8,0x01,0xfd,0x00,0xff
+
+# GFX12: ds_xor_rtn_b32 v255, v254, v2           ; encoding: [0x00,0x00,0xac,0xd8,0xfe,0x02,0x00,0xff]
+0x00,0x00,0xac,0xd8,0xfe,0x02,0x00,0xff
+
+# GFX12: ds_xor_rtn_b32 v255, v254, v2 offset:4660 ; encoding: [0x34,0x12,0xac,0xd8,0xfe,0x02,0x00,0xff]
+0x34,0x12,0xac,0xd8,0xfe,0x02,0x00,0xff
+
+# GFX12: ds_xor_rtn_b32 v255, v254, v2 offset:65535 ; encoding: [0xff,0xff,0xac,0xd8,0xfe,0x02,0x00,0xff]
+0xff,0xff,0xac,0xd8,0xfe,0x02,0x00,0xff
+
+# GFX12: ds_xor_rtn_b32 v255, v254, v253         ; encoding: [0x00,0x00,0xac,0xd8,0xfe,0xfd,0x00,0xff]
+0x00,0x00,0xac,0xd8,0xfe,0xfd,0x00,0xff
+
+# GFX12: ds_xor_rtn_b32 v255, v254, v253 offset:4660 ; encoding: [0x34,0x12,0xac,0xd8,0xfe,0xfd,0x00,0xff]
+0x34,0x12,0xac,0xd8,0xfe,0xfd,0x00,0xff
+
+# GFX12: ds_xor_rtn_b32 v255, v254, v253 offset:65535 ; encoding: [0xff,0xff,0xac,0xd8,0xfe,0xfd,0x00,0xff]
+0xff,0xff,0xac,0xd8,0xfe,0xfd,0x00,0xff
+
+# GFX12: ds_xor_rtn_b64 v[254:255], v1, v[2:3] offset:65535 ; encoding: [0xff,0xff,0xac,0xd9,0x01,0x02,0x00,0xfe]
+0xff,0xff,0xac,0xd9,0x01,0x02,0x00,0xfe
+
+# GFX12: ds_xor_rtn_b64 v[5:6], v1, v[254:255] offset:65535 ; encoding: [0xff,0xff,0xac,0xd9,0x01,0xfe,0x00,0x05]
+0xff,0xff,0xac,0xd9,0x01,0xfe,0x00,0x05
+
+# GFX12: ds_xor_rtn_b64 v[5:6], v1, v[2:3]       ; encoding: [0x00,0x00,0xac,0xd9,0x01,0x02,0x00,0x05]
+0x00,0x00,0xac,0xd9,0x01,0x02,0x00,0x05
+
+# GFX12: ds_xor_rtn_b64 v[5:6], v1, v[2:3] offset:4 ; encoding: [0x04,0x00,0xac,0xd9,0x01,0x02,0x00,0x05]
+0x04,0x00,0xac,0xd9,0x01,0x02,0x00,0x05
+
+# GFX12: ds_xor_rtn_b64 v[5:6], v1, v[2:3] offset:65535 ; encoding: [0xff,0xff,0xac,0xd9,0x01,0x02,0x00,0x05]
+0xff,0xff,0xac,0xd9,0x01,0x02,0x00,0x05
+
+# GFX12: ds_xor_rtn_b64 v[5:6], v255, v[2:3] offset:65535 ; encoding: [0xff,0xff,0xac,0xd9,0xff,0x02,0x00,0x05]
+0xff,0xff,0xac,0xd9,0xff,0x02,0x00,0x05
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sop1.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sop1.txt
index c061462339b66c1cad651b3fbc49bd56029d6e4c..1c31ee1e5dd7fbf07379881c28ae452ffd3c02f6 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sop1.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sop1.txt
@@ -684,6 +684,60 @@
 # GFX12: s_rndne_f16 s5, 0x3456                  ; encoding: [0xff,0x6e,0x85,0xbe,0x56,0x34,0x00,0x00]
 0xff,0x6e,0x85,0xbe,0x56,0x34,0x00,0x00
 
+# GFX12: s_barrier_signal -2                      ; encoding: [0xc2,0x4e,0x80,0xbe]
+0xc2,0x4e,0x80,0xbe
+
+# GFX12: s_barrier_signal -1                      ; encoding: [0xc1,0x4e,0x80,0xbe]
+0xc1,0x4e,0x80,0xbe
+
+# GFX12: s_barrier_signal m0                      ; encoding: [0x7d,0x4e,0x80,0xbe]
+0x7d,0x4e,0x80,0xbe
+
+# GFX12: s_barrier_signal_isfirst -2              ; encoding: [0xc2,0x4f,0x80,0xbe]
+0xc2,0x4f,0x80,0xbe
+
+# GFX12: s_barrier_signal_isfirst -1              ; encoding: [0xc1,0x4f,0x80,0xbe]
+0xc1,0x4f,0x80,0xbe
+
+# GFX12: s_barrier_signal_isfirst m0              ; encoding: [0x7d,0x4f,0x80,0xbe]
+0x7d,0x4f,0x80,0xbe
+
+# GFX12: s_barrier_init -1                        ; encoding: [0xc1,0x51,0x80,0xbe]
+0xc1,0x51,0x80,0xbe
+
+# GFX12: s_barrier_init -2                        ; encoding: [0xc2,0x51,0x80,0xbe]
+0xc2,0x51,0x80,0xbe
+
+# GFX12: s_barrier_init m0                        ; encoding: [0x7d,0x51,0x80,0xbe]
+0x7d,0x51,0x80,0xbe
+
+# GFX12: s_barrier_join -1                        ; encoding: [0xc1,0x52,0x80,0xbe]
+0xc1,0x52,0x80,0xbe
+
+# GFX12: s_barrier_join -2                        ; encoding: [0xc2,0x52,0x80,0xbe]
+0xc2,0x52,0x80,0xbe
+
+# GFX12: s_barrier_join m0                        ; encoding: [0x7d,0x52,0x80,0xbe]
+0x7d,0x52,0x80,0xbe
+
+# GFX12: s_wakeup_barrier 1                       ; encoding: [0x81,0x57,0x80,0xbe]
+0x81,0x57,0x80,0xbe
+
+# GFX12: s_wakeup_barrier -1                      ; encoding: [0xc1,0x57,0x80,0xbe]
+0xc1,0x57,0x80,0xbe
+
+# GFX12: s_wakeup_barrier m0                      ; encoding: [0x7d,0x57,0x80,0xbe]
+0x7d,0x57,0x80,0xbe
+
+# GFX12: s_get_barrier_state s3, -1               ; encoding: [0xc1,0x50,0x83,0xbe]
+0xc1,0x50,0x83,0xbe
+
+# GFX12: s_get_barrier_state s3, -2               ; encoding: [0xc2,0x50,0x83,0xbe]
+0xc2,0x50,0x83,0xbe
+
+# GFX12: s_get_barrier_state s3, m0               ; encoding: [0x7d,0x50,0x83,0xbe]
+0x7d,0x50,0x83,0xbe
+
 # GFX12: s_abs_i32 exec_hi, s1                   ; encoding: [0x01,0x15,0xff,0xbe]
 0x01,0x15,0xff,0xbe
 
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sop2.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sop2.txt
index c177af7a6e50bc6cf0734a16a904fa19e9594096..297b6b083550cc8fe3906b7f5478c9e9a74bbd8a 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sop2.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sop2.txt
@@ -484,6 +484,108 @@
 # GFX12: s_max_num_f32 s5, s1, s105              ; encoding: [0x01,0x69,0x85,0xa1]
 0x01,0x69,0x85,0xa1
 
+# GFX12: s_minimum_f32 s5, s1, s2                ; encoding: [0x01,0x02,0x85,0xa7]
+0x01,0x02,0x85,0xa7
+
+# GFX12: s_minimum_f32 s105, s1, s2              ; encoding: [0x01,0x02,0xe9,0xa7]
+0x01,0x02,0xe9,0xa7
+
+# GFX12: s_minimum_f32 s5, s105, s2              ; encoding: [0x69,0x02,0x85,0xa7]
+0x69,0x02,0x85,0xa7
+
+# GFX12: s_minimum_f32 s5, s103, s2              ; encoding: [0x67,0x02,0x85,0xa7]
+0x67,0x02,0x85,0xa7
+
+# GFX12: s_minimum_f32 s5, vcc_lo, s2            ; encoding: [0x6a,0x02,0x85,0xa7]
+0x6a,0x02,0x85,0xa7
+
+# GFX12: s_minimum_f32 s5, vcc_hi, s2            ; encoding: [0x6b,0x02,0x85,0xa7]
+0x6b,0x02,0x85,0xa7
+
+# GFX12: s_minimum_f32 s5, ttmp11, s2            ; encoding: [0x77,0x02,0x85,0xa7]
+0x77,0x02,0x85,0xa7
+
+# GFX12: s_minimum_f32 s5, m0, s2                ; encoding: [0x7d,0x02,0x85,0xa7]
+0x7d,0x02,0x85,0xa7
+
+# GFX12: s_minimum_f32 s5, exec_lo, s2           ; encoding: [0x7e,0x02,0x85,0xa7]
+0x7e,0x02,0x85,0xa7
+
+# GFX12: s_minimum_f32 s5, exec_hi, s2           ; encoding: [0x7f,0x02,0x85,0xa7]
+0x7f,0x02,0x85,0xa7
+
+# GFX12: s_minimum_f32 s5, 0, s2                 ; encoding: [0x80,0x02,0x85,0xa7]
+0x80,0x02,0x85,0xa7
+
+# GFX12: s_minimum_f32 s5, -1, s2                ; encoding: [0xc1,0x02,0x85,0xa7]
+0xc1,0x02,0x85,0xa7
+
+# GFX12: s_minimum_f32 s5, 0.5, s2               ; encoding: [0xf0,0x02,0x85,0xa7]
+0xf0,0x02,0x85,0xa7
+
+# GFX12: s_minimum_f32 s5, -4.0, s2              ; encoding: [0xf7,0x02,0x85,0xa7]
+0xf7,0x02,0x85,0xa7
+
+# GFX12: s_minimum_f32 s5, 0xaf123456, s2        ; encoding: [0xff,0x02,0x85,0xa7,0x56,0x34,0x12,0xaf]
+0xff,0x02,0x85,0xa7,0x56,0x34,0x12,0xaf
+
+# GFX12: s_minimum_f32 s5, 0x3f717273, s2        ; encoding: [0xff,0x02,0x85,0xa7,0x73,0x72,0x71,0x3f]
+0xff,0x02,0x85,0xa7,0x73,0x72,0x71,0x3f
+
+# GFX12: s_minimum_f32 s5, s1, s105              ; encoding: [0x01,0x69,0x85,0xa7]
+0x01,0x69,0x85,0xa7
+
+# GFX12: s_maximum_f32 s5, s1, s2                ; encoding: [0x01,0x02,0x05,0xa8]
+0x01,0x02,0x05,0xa8
+
+# GFX12: s_maximum_f32 s105, s1, s2              ; encoding: [0x01,0x02,0x69,0xa8]
+0x01,0x02,0x69,0xa8
+
+# GFX12: s_maximum_f32 s5, s105, s2              ; encoding: [0x69,0x02,0x05,0xa8]
+0x69,0x02,0x05,0xa8
+
+# GFX12: s_maximum_f32 s5, s103, s2              ; encoding: [0x67,0x02,0x05,0xa8]
+0x67,0x02,0x05,0xa8
+
+# GFX12: s_maximum_f32 s5, vcc_lo, s2            ; encoding: [0x6a,0x02,0x05,0xa8]
+0x6a,0x02,0x05,0xa8
+
+# GFX12: s_maximum_f32 s5, vcc_hi, s2            ; encoding: [0x6b,0x02,0x05,0xa8]
+0x6b,0x02,0x05,0xa8
+
+# GFX12: s_maximum_f32 s5, ttmp11, s2            ; encoding: [0x77,0x02,0x05,0xa8]
+0x77,0x02,0x05,0xa8
+
+# GFX12: s_maximum_f32 s5, m0, s2                ; encoding: [0x7d,0x02,0x05,0xa8]
+0x7d,0x02,0x05,0xa8
+
+# GFX12: s_maximum_f32 s5, exec_lo, s2           ; encoding: [0x7e,0x02,0x05,0xa8]
+0x7e,0x02,0x05,0xa8
+
+# GFX12: s_maximum_f32 s5, exec_hi, s2           ; encoding: [0x7f,0x02,0x05,0xa8]
+0x7f,0x02,0x05,0xa8
+
+# GFX12: s_maximum_f32 s5, 0, s2                 ; encoding: [0x80,0x02,0x05,0xa8]
+0x80,0x02,0x05,0xa8
+
+# GFX12: s_maximum_f32 s5, -1, s2                ; encoding: [0xc1,0x02,0x05,0xa8]
+0xc1,0x02,0x05,0xa8
+
+# GFX12: s_maximum_f32 s5, 0.5, s2               ; encoding: [0xf0,0x02,0x05,0xa8]
+0xf0,0x02,0x05,0xa8
+
+# GFX12: s_maximum_f32 s5, -4.0, s2              ; encoding: [0xf7,0x02,0x05,0xa8]
+0xf7,0x02,0x05,0xa8
+
+# GFX12: s_maximum_f32 s5, 0xaf123456, s2        ; encoding: [0xff,0x02,0x05,0xa8,0x56,0x34,0x12,0xaf]
+0xff,0x02,0x05,0xa8,0x56,0x34,0x12,0xaf
+
+# GFX12: s_maximum_f32 s5, 0x3f717273, s2        ; encoding: [0xff,0x02,0x05,0xa8,0x73,0x72,0x71,0x3f]
+0xff,0x02,0x05,0xa8,0x73,0x72,0x71,0x3f
+
+# GFX12: s_maximum_f32 s5, s1, s105              ; encoding: [0x01,0x69,0x05,0xa8]
+0x01,0x69,0x05,0xa8
+
 # GFX12: s_fmac_f32 s5, s1, s2                   ; encoding: [0x01,0x02,0x85,0xa3]
 0x01,0x02,0x85,0xa3
 
@@ -943,6 +1045,102 @@
 # GFX12: s_min_num_f16 s5, s1, s105              ; encoding: [0x01,0x69,0x85,0xa5]
 0x01,0x69,0x85,0xa5
 
+# GFX12: s_maximum_f16 s5, s1, s2                ; encoding: [0x01,0x02,0x05,0xa9]
+0x01,0x02,0x05,0xa9
+
+# GFX12: s_maximum_f16 s105, s1, s2              ; encoding: [0x01,0x02,0x69,0xa9]
+0x01,0x02,0x69,0xa9
+
+# GFX12: s_maximum_f16 s5, s105, s2              ; encoding: [0x69,0x02,0x05,0xa9]
+0x69,0x02,0x05,0xa9
+
+# GFX12: s_maximum_f16 s5, s101, s2              ; encoding: [0x65,0x02,0x05,0xa9]
+0x65,0x02,0x05,0xa9
+
+# GFX12: s_maximum_f16 s5, vcc_lo, s2            ; encoding: [0x6a,0x02,0x05,0xa9]
+0x6a,0x02,0x05,0xa9
+
+# GFX12: s_maximum_f16 s5, vcc_hi, s2            ; encoding: [0x6b,0x02,0x05,0xa9]
+0x6b,0x02,0x05,0xa9
+
+# GFX12: s_maximum_f16 s5, m0, s2                ; encoding: [0x7d,0x02,0x05,0xa9]
+0x7d,0x02,0x05,0xa9
+
+# GFX12: s_maximum_f16 s5, exec_lo, s2           ; encoding: [0x7e,0x02,0x05,0xa9]
+0x7e,0x02,0x05,0xa9
+
+# GFX12: s_maximum_f16 s5, exec_hi, s2           ; encoding: [0x7f,0x02,0x05,0xa9]
+0x7f,0x02,0x05,0xa9
+
+# GFX12: s_maximum_f16 s5, 0, s2                 ; encoding: [0x80,0x02,0x05,0xa9]
+0x80,0x02,0x05,0xa9
+
+# GFX12: s_maximum_f16 s5, -1, s2                ; encoding: [0xc1,0x02,0x05,0xa9]
+0xc1,0x02,0x05,0xa9
+
+# GFX12: s_maximum_f16 s5, 0.5, s2               ; encoding: [0xf0,0x02,0x05,0xa9]
+0xf0,0x02,0x05,0xa9
+
+# GFX12: s_maximum_f16 s5, -4.0, s2              ; encoding: [0xf7,0x02,0x05,0xa9]
+0xf7,0x02,0x05,0xa9
+
+# GFX12: s_maximum_f16 s5, 0xfe0b, s2            ; encoding: [0xff,0x02,0x05,0xa9,0x0b,0xfe,0x00,0x00]
+0xff,0x02,0x05,0xa9,0x0b,0xfe,0x00,0x00
+
+# GFX12: s_maximum_f16 s5, 0x3456, s2            ; encoding: [0xff,0x02,0x05,0xa9,0x56,0x34,0x00,0x00]
+0xff,0x02,0x05,0xa9,0x56,0x34,0x00,0x00
+
+# GFX12: s_maximum_f16 s5, s1, s105              ; encoding: [0x01,0x69,0x05,0xa9]
+0x01,0x69,0x05,0xa9
+
+# GFX12: s_minimum_f16 s5, s1, s2                ; encoding: [0x01,0x02,0x85,0xa8]
+0x01,0x02,0x85,0xa8
+
+# GFX12: s_minimum_f16 s105, s1, s2              ; encoding: [0x01,0x02,0xe9,0xa8]
+0x01,0x02,0xe9,0xa8
+
+# GFX12: s_minimum_f16 s5, s105, s2              ; encoding: [0x69,0x02,0x85,0xa8]
+0x69,0x02,0x85,0xa8
+
+# GFX12: s_minimum_f16 s5, s101, s2              ; encoding: [0x65,0x02,0x85,0xa8]
+0x65,0x02,0x85,0xa8
+
+# GFX12: s_minimum_f16 s5, vcc_lo, s2            ; encoding: [0x6a,0x02,0x85,0xa8]
+0x6a,0x02,0x85,0xa8
+
+# GFX12: s_minimum_f16 s5, vcc_hi, s2            ; encoding: [0x6b,0x02,0x85,0xa8]
+0x6b,0x02,0x85,0xa8
+
+# GFX12: s_minimum_f16 s5, m0, s2                ; encoding: [0x7d,0x02,0x85,0xa8]
+0x7d,0x02,0x85,0xa8
+
+# GFX12: s_minimum_f16 s5, exec_lo, s2           ; encoding: [0x7e,0x02,0x85,0xa8]
+0x7e,0x02,0x85,0xa8
+
+# GFX12: s_minimum_f16 s5, exec_hi, s2           ; encoding: [0x7f,0x02,0x85,0xa8]
+0x7f,0x02,0x85,0xa8
+
+# GFX12: s_minimum_f16 s5, 0, s2                 ; encoding: [0x80,0x02,0x85,0xa8]
+0x80,0x02,0x85,0xa8
+
+# GFX12: s_minimum_f16 s5, -1, s2                ; encoding: [0xc1,0x02,0x85,0xa8]
+0xc1,0x02,0x85,0xa8
+
+# GFX12: s_minimum_f16 s5, 0.5, s2               ; encoding: [0xf0,0x02,0x85,0xa8]
+0xf0,0x02,0x85,0xa8
+
+# GFX12: s_minimum_f16 s5, -4.0, s2              ; encoding: [0xf7,0x02,0x85,0xa8]
+0xf7,0x02,0x85,0xa8
+
+# GFX12: s_minimum_f16 s5, 0xfe0b, s2            ; encoding: [0xff,0x02,0x85,0xa8,0x0b,0xfe,0x00,0x00]
+0xff,0x02,0x85,0xa8,0x0b,0xfe,0x00,0x00
+
+# GFX12: s_minimum_f16 s5, 0x3456, s2            ; encoding: [0xff,0x02,0x85,0xa8,0x56,0x34,0x00,0x00]
+0xff,0x02,0x85,0xa8,0x56,0x34,0x00,0x00
+
+# GFX12: s_minimum_f16 s5, s1, s105              ; encoding: [0x01,0x69,0x85,0xa8]
+0x01,0x69,0x85,0xa8
+
 # GFX12: s_absdiff_i32 exec_hi, s1, s2           ; encoding: [0x01,0x02,0x7f,0x83]
 0x01,0x02,0x7f,0x83
 
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sopp.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sopp.txt
index fe74ff08a8e5c9583b6cb1a17fff9d5fc6ef123c..13ded15998fb22b254ab02ea3114b6feeb17a6c6 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sopp.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sopp.txt
@@ -15,6 +15,18 @@
 # GFX12: s_singleuse_vdst 0x1234                 ; encoding: [0x34,0x12,0x93,0xbf]
 0x34,0x12,0x93,0xbf
 
+# GFX12: s_barrier_wait 0xffff                   ; encoding: [0xff,0xff,0x94,0xbf]
+0xff,0xff,0x94,0xbf
+
+# GFX12: s_barrier_wait 1                        ; encoding: [0x01,0x00,0x94,0xbf]
+0x01,0x00,0x94,0xbf
+
+# GFX12: s_barrier_leave                         ; encoding: [0x00,0x00,0x95,0xbf]
+0x00,0x00,0x95,0xbf
+
+# GFX12: s_barrier                               ; encoding: [0x00,0x00,0xbd,0xbf]
+0x00,0x00,0xbd,0xbf
+
 # GFX12: s_branch 0                              ; encoding: [0x00,0x00,0xa0,0xbf]
 0x00,0x00,0xa0,0xbf
 
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vbuffer_mtbuf.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vbuffer_mtbuf.txt
new file mode 100644
index 0000000000000000000000000000000000000000..affea2e081c1ba23b6a40cd96e07dd4d50f9469c
--- /dev/null
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vbuffer_mtbuf.txt
@@ -0,0 +1,673 @@
+# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12 %s
+
+# GFX12: tbuffer_load_d16_format_x v4, off, s[8:11], s3 offset:8388607 		; encoding:  [0x03,0x00,0x22,0xc4,0x04,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x22,0xc4,0x04,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_d16_format_x v255, off, s[8:11], s3 offset:8388607 		; encoding:  [0x03,0x00,0x22,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x22,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_d16_format_x v4, off, s[12:15], s3 offset:8388607 		; encoding:  [0x03,0x00,0x22,0xc4,0x04,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x22,0xc4,0x04,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_d16_format_x v4, off, s[12:15], s101 format:[BUF_FMT_8_SNORM] offset:8388607 		; encoding:  [0x65,0x00,0x22,0xc4,0x04,0x18,0x00,0x01,0x00,0xff,0xff,0x7f]
+0x65,0x00,0x22,0xc4,0x04,0x18,0x00,0x01,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_d16_format_x v4, off, s[12:15], m0 format:[BUF_FMT_8_SNORM] offset:8388607 		; encoding:  [0x7d,0x00,0x22,0xc4,0x04,0x18,0x00,0x01,0x00,0xff,0xff,0x7f]
+0x7d,0x00,0x22,0xc4,0x04,0x18,0x00,0x01,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_d16_format_x v4, off, s[8:11], s0 format:[BUF_FMT_8_SNORM] offset:8388607 		; encoding:  [0x00,0x00,0x22,0xc4,0x04,0x10,0x00,0x01,0x00,0xff,0xff,0x7f]
+0x00,0x00,0x22,0xc4,0x04,0x10,0x00,0x01,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_d16_format_x v4, off, s[8:11], s61 format:[BUF_FMT_8_USCALED] offset:8388607 		; encoding:  [0x3d,0x00,0x22,0xc4,0x04,0x10,0x80,0x01,0x00,0xff,0xff,0x7f]
+0x3d,0x00,0x22,0xc4,0x04,0x10,0x80,0x01,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_d16_format_x v4, off, ttmp[4:7], s61 format:[BUF_FMT_8_USCALED] offset:8388607 		; encoding:  [0x3d,0x00,0x22,0xc4,0x04,0xe0,0x80,0x01,0x00,0xff,0xff,0x7f]
+0x3d,0x00,0x22,0xc4,0x04,0xe0,0x80,0x01,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_d16_format_x v4, v1, s[8:11], s3 format:[BUF_FMT_8_USCALED] offen offset:52 		; encoding:  [0x03,0x00,0x22,0xc4,0x04,0x10,0x80,0x41,0x01,0x34,0x00,0x00]
+0x03,0x00,0x22,0xc4,0x04,0x10,0x80,0x41,0x01,0x34,0x00,0x00
+
+# GFX12: tbuffer_load_d16_format_x v4, v1, s[8:11], s3 format:[BUF_FMT_8_SSCALED] idxen offset:52 		; encoding:  [0x03,0x00,0x22,0xc4,0x04,0x10,0x00,0x82,0x01,0x34,0x00,0x00]
+0x03,0x00,0x22,0xc4,0x04,0x10,0x00,0x82,0x01,0x34,0x00,0x00
+
+# GFX12: tbuffer_load_d16_format_x v4, v[1:2], s[8:11], s0 format:[BUF_FMT_8_SSCALED] idxen offen offset:52 		; encoding:  [0x00,0x00,0x22,0xc4,0x04,0x10,0x00,0xc2,0x01,0x34,0x00,0x00]
+0x00,0x00,0x22,0xc4,0x04,0x10,0x00,0xc2,0x01,0x34,0x00,0x00
+
+# GFX12: tbuffer_load_d16_format_x v4, off, ttmp[4:7], s3 format:[BUF_FMT_8_SSCALED] offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV 		; encoding:  [0x03,0x00,0x22,0xc4,0x04,0xe0,0x68,0x02,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x22,0xc4,0x04,0xe0,0x68,0x02,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_d16_format_x v4, off, ttmp[4:7], s3 format:[BUF_FMT_8_UINT] offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS 		; encoding:  [0x03,0x00,0x22,0xc4,0x04,0xe0,0xbc,0x02,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x22,0xc4,0x04,0xe0,0xbc,0x02,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_d16_format_x v4, off, ttmp[4:7], s3 format:[BUF_FMT_8_UINT] offset:8388607 scope:SCOPE_SE 		; encoding:  [0x03,0x00,0x22,0xc4,0x04,0xe0,0x84,0x02,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x22,0xc4,0x04,0xe0,0x84,0x02,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_d16_format_xy v4, off, s[8:11], s3 format:[BUF_FMT_8_SINT] offset:8388607 		; encoding:  [0x03,0x40,0x22,0xc4,0x04,0x10,0x00,0x03,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x22,0xc4,0x04,0x10,0x00,0x03,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_d16_format_xy v255, off, s[8:11], s3 format:[BUF_FMT_8_SINT] offset:8388607 		; encoding:  [0x03,0x40,0x22,0xc4,0xff,0x10,0x00,0x03,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x22,0xc4,0xff,0x10,0x00,0x03,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_d16_format_xy v4, off, s[12:15], s3 format:[BUF_FMT_8_SINT] offset:8388607 		; encoding:  [0x03,0x40,0x22,0xc4,0x04,0x18,0x00,0x03,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x22,0xc4,0x04,0x18,0x00,0x03,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_d16_format_xy v4, off, s[12:15], s101 format:[BUF_FMT_16_UNORM] offset:8388607 		; encoding:  [0x65,0x40,0x22,0xc4,0x04,0x18,0x80,0x03,0x00,0xff,0xff,0x7f]
+0x65,0x40,0x22,0xc4,0x04,0x18,0x80,0x03,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_d16_format_xy v4, off, s[12:15], m0 format:[BUF_FMT_16_UNORM] offset:8388607 		; encoding:  [0x7d,0x40,0x22,0xc4,0x04,0x18,0x80,0x03,0x00,0xff,0xff,0x7f]
+0x7d,0x40,0x22,0xc4,0x04,0x18,0x80,0x03,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_d16_format_xy v4, off, s[8:11], s0 format:[BUF_FMT_16_UNORM] offset:8388607 		; encoding:  [0x00,0x40,0x22,0xc4,0x04,0x10,0x80,0x03,0x00,0xff,0xff,0x7f]
+0x00,0x40,0x22,0xc4,0x04,0x10,0x80,0x03,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_d16_format_xy v4, off, s[8:11], s61 format:[BUF_FMT_16_SNORM] offset:8388607 		; encoding:  [0x3d,0x40,0x22,0xc4,0x04,0x10,0x00,0x04,0x00,0xff,0xff,0x7f]
+0x3d,0x40,0x22,0xc4,0x04,0x10,0x00,0x04,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_d16_format_xy v4, off, ttmp[4:7], s61 format:[BUF_FMT_16_SNORM] offset:8388607 		; encoding:  [0x3d,0x40,0x22,0xc4,0x04,0xe0,0x00,0x04,0x00,0xff,0xff,0x7f]
+0x3d,0x40,0x22,0xc4,0x04,0xe0,0x00,0x04,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_d16_format_xy v4, v1, s[8:11], s3 format:[BUF_FMT_16_SNORM] offen offset:52 		; encoding:  [0x03,0x40,0x22,0xc4,0x04,0x10,0x00,0x44,0x01,0x34,0x00,0x00]
+0x03,0x40,0x22,0xc4,0x04,0x10,0x00,0x44,0x01,0x34,0x00,0x00
+
+# GFX12: tbuffer_load_d16_format_xy v4, v1, s[8:11], s3 format:[BUF_FMT_16_USCALED] idxen offset:52 		; encoding:  [0x03,0x40,0x22,0xc4,0x04,0x10,0x80,0x84,0x01,0x34,0x00,0x00]
+0x03,0x40,0x22,0xc4,0x04,0x10,0x80,0x84,0x01,0x34,0x00,0x00
+
+# GFX12: tbuffer_load_d16_format_xy v4, v[1:2], s[8:11], s0 format:[BUF_FMT_16_USCALED] idxen offen offset:52 		; encoding:  [0x00,0x40,0x22,0xc4,0x04,0x10,0x80,0xc4,0x01,0x34,0x00,0x00]
+0x00,0x40,0x22,0xc4,0x04,0x10,0x80,0xc4,0x01,0x34,0x00,0x00
+
+# GFX12: tbuffer_load_d16_format_xy v4, off, ttmp[4:7], s3 format:[BUF_FMT_16_USCALED] offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV 		; encoding:  [0x03,0x40,0x22,0xc4,0x04,0xe0,0xe8,0x04,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x22,0xc4,0x04,0xe0,0xe8,0x04,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_d16_format_xy v4, off, ttmp[4:7], s3 format:[BUF_FMT_16_SSCALED] offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS 		; encoding:  [0x03,0x40,0x22,0xc4,0x04,0xe0,0x3c,0x05,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x22,0xc4,0x04,0xe0,0x3c,0x05,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_d16_format_xy v4, off, ttmp[4:7], s3 format:[BUF_FMT_16_SSCALED] offset:8388607 scope:SCOPE_SE 		; encoding:  [0x03,0x40,0x22,0xc4,0x04,0xe0,0x04,0x05,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x22,0xc4,0x04,0xe0,0x04,0x05,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_d16_format_xyz v[4:5], off, s[8:11], s3 format:[BUF_FMT_16_UINT] offset:8388607 		; encoding:  [0x03,0x80,0x22,0xc4,0x04,0x10,0x80,0x05,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x22,0xc4,0x04,0x10,0x80,0x05,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_d16_format_xyz v[254:255], off, s[8:11], s3 format:[BUF_FMT_16_UINT] offset:8388607 		; encoding:  [0x03,0x80,0x22,0xc4,0xfe,0x10,0x80,0x05,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x22,0xc4,0xfe,0x10,0x80,0x05,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_d16_format_xyz v[4:5], off, s[12:15], s3 format:[BUF_FMT_16_UINT] offset:8388607 		; encoding:  [0x03,0x80,0x22,0xc4,0x04,0x18,0x80,0x05,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x22,0xc4,0x04,0x18,0x80,0x05,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_d16_format_xyz v[4:5], off, s[12:15], s101 format:[BUF_FMT_16_SINT] offset:8388607 		; encoding:  [0x65,0x80,0x22,0xc4,0x04,0x18,0x00,0x06,0x00,0xff,0xff,0x7f]
+0x65,0x80,0x22,0xc4,0x04,0x18,0x00,0x06,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_d16_format_xyz v[4:5], off, s[12:15], m0 format:[BUF_FMT_16_SINT] offset:8388607 		; encoding:  [0x7d,0x80,0x22,0xc4,0x04,0x18,0x00,0x06,0x00,0xff,0xff,0x7f]
+0x7d,0x80,0x22,0xc4,0x04,0x18,0x00,0x06,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_d16_format_xyz v[4:5], off, s[8:11], s0 format:[BUF_FMT_16_SINT] offset:8388607 		; encoding:  [0x00,0x80,0x22,0xc4,0x04,0x10,0x00,0x06,0x00,0xff,0xff,0x7f]
+0x00,0x80,0x22,0xc4,0x04,0x10,0x00,0x06,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_d16_format_xyz v[4:5], off, s[8:11], s61 format:[BUF_FMT_16_FLOAT] offset:8388607 		; encoding:  [0x3d,0x80,0x22,0xc4,0x04,0x10,0x80,0x06,0x00,0xff,0xff,0x7f]
+0x3d,0x80,0x22,0xc4,0x04,0x10,0x80,0x06,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_d16_format_xyz v[4:5], off, ttmp[4:7], s61 format:[BUF_FMT_16_FLOAT] offset:8388607 		; encoding:  [0x3d,0x80,0x22,0xc4,0x04,0xe0,0x80,0x06,0x00,0xff,0xff,0x7f]
+0x3d,0x80,0x22,0xc4,0x04,0xe0,0x80,0x06,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_d16_format_xyz v[4:5], v1, s[8:11], s3 format:[BUF_FMT_16_FLOAT] offen offset:52 		; encoding:  [0x03,0x80,0x22,0xc4,0x04,0x10,0x80,0x46,0x01,0x34,0x00,0x00]
+0x03,0x80,0x22,0xc4,0x04,0x10,0x80,0x46,0x01,0x34,0x00,0x00
+
+# GFX12: tbuffer_load_d16_format_xyz v[4:5], v1, s[8:11], s3 format:[BUF_FMT_8_8_UNORM] idxen offset:52 		; encoding:  [0x03,0x80,0x22,0xc4,0x04,0x10,0x00,0x87,0x01,0x34,0x00,0x00]
+0x03,0x80,0x22,0xc4,0x04,0x10,0x00,0x87,0x01,0x34,0x00,0x00
+
+# GFX12: tbuffer_load_d16_format_xyz v[4:5], v[1:2], s[8:11], s0 format:[BUF_FMT_8_8_UNORM] idxen offen offset:52 		; encoding:  [0x00,0x80,0x22,0xc4,0x04,0x10,0x00,0xc7,0x01,0x34,0x00,0x00]
+0x00,0x80,0x22,0xc4,0x04,0x10,0x00,0xc7,0x01,0x34,0x00,0x00
+
+# GFX12: tbuffer_load_d16_format_xyz v[4:5], off, ttmp[4:7], s3 format:[BUF_FMT_8_8_UNORM] offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV 		; encoding:  [0x03,0x80,0x22,0xc4,0x04,0xe0,0x68,0x07,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x22,0xc4,0x04,0xe0,0x68,0x07,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_d16_format_xyz v[4:5], off, ttmp[4:7], s3 format:[BUF_FMT_8_8_SNORM] offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS 		; encoding:  [0x03,0x80,0x22,0xc4,0x04,0xe0,0xbc,0x07,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x22,0xc4,0x04,0xe0,0xbc,0x07,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_d16_format_xyz v[4:5], off, ttmp[4:7], s3 format:[BUF_FMT_8_8_SNORM] offset:8388607 scope:SCOPE_SE 		; encoding:  [0x03,0x80,0x22,0xc4,0x04,0xe0,0x84,0x07,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x22,0xc4,0x04,0xe0,0x84,0x07,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_d16_format_xyzw v[4:5], off, s[8:11], s3 format:[BUF_FMT_8_8_USCALED] offset:8388607 		; encoding:  [0x03,0xc0,0x22,0xc4,0x04,0x10,0x00,0x08,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x22,0xc4,0x04,0x10,0x00,0x08,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_d16_format_xyzw v[254:255], off, s[8:11], s3 format:[BUF_FMT_8_8_USCALED] offset:8388607 		; encoding:  [0x03,0xc0,0x22,0xc4,0xfe,0x10,0x00,0x08,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x22,0xc4,0xfe,0x10,0x00,0x08,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_d16_format_xyzw v[4:5], off, s[12:15], s3 format:[BUF_FMT_8_8_USCALED] offset:8388607 		; encoding:  [0x03,0xc0,0x22,0xc4,0x04,0x18,0x00,0x08,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x22,0xc4,0x04,0x18,0x00,0x08,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_d16_format_xyzw v[4:5], off, s[12:15], s101 format:[BUF_FMT_8_8_SSCALED] offset:8388607 		; encoding:  [0x65,0xc0,0x22,0xc4,0x04,0x18,0x80,0x08,0x00,0xff,0xff,0x7f]
+0x65,0xc0,0x22,0xc4,0x04,0x18,0x80,0x08,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_d16_format_xyzw v[4:5], off, s[12:15], m0 format:[BUF_FMT_8_8_SSCALED] offset:8388607 		; encoding:  [0x7d,0xc0,0x22,0xc4,0x04,0x18,0x80,0x08,0x00,0xff,0xff,0x7f]
+0x7d,0xc0,0x22,0xc4,0x04,0x18,0x80,0x08,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_d16_format_xyzw v[4:5], off, s[8:11], s0 format:[BUF_FMT_8_8_SSCALED] offset:8388607 		; encoding:  [0x00,0xc0,0x22,0xc4,0x04,0x10,0x80,0x08,0x00,0xff,0xff,0x7f]
+0x00,0xc0,0x22,0xc4,0x04,0x10,0x80,0x08,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_d16_format_xyzw v[4:5], off, s[8:11], s61 format:[BUF_FMT_8_8_UINT] offset:8388607 		; encoding:  [0x3d,0xc0,0x22,0xc4,0x04,0x10,0x00,0x09,0x00,0xff,0xff,0x7f]
+0x3d,0xc0,0x22,0xc4,0x04,0x10,0x00,0x09,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_d16_format_xyzw v[4:5], off, ttmp[4:7], s61 format:[BUF_FMT_8_8_UINT] offset:8388607 		; encoding:  [0x3d,0xc0,0x22,0xc4,0x04,0xe0,0x00,0x09,0x00,0xff,0xff,0x7f]
+0x3d,0xc0,0x22,0xc4,0x04,0xe0,0x00,0x09,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_d16_format_xyzw v[4:5], v1, s[8:11], s3 format:[BUF_FMT_8_8_UINT] offen offset:52 		; encoding:  [0x03,0xc0,0x22,0xc4,0x04,0x10,0x00,0x49,0x01,0x34,0x00,0x00]
+0x03,0xc0,0x22,0xc4,0x04,0x10,0x00,0x49,0x01,0x34,0x00,0x00
+
+# GFX12: tbuffer_load_d16_format_xyzw v[4:5], v1, s[8:11], s3 format:[BUF_FMT_8_8_SINT] idxen offset:52 		; encoding:  [0x03,0xc0,0x22,0xc4,0x04,0x10,0x80,0x89,0x01,0x34,0x00,0x00]
+0x03,0xc0,0x22,0xc4,0x04,0x10,0x80,0x89,0x01,0x34,0x00,0x00
+
+# GFX12: tbuffer_load_d16_format_xyzw v[4:5], v[1:2], s[8:11], s0 format:[BUF_FMT_8_8_SINT] idxen offen offset:52 		; encoding:  [0x00,0xc0,0x22,0xc4,0x04,0x10,0x80,0xc9,0x01,0x34,0x00,0x00]
+0x00,0xc0,0x22,0xc4,0x04,0x10,0x80,0xc9,0x01,0x34,0x00,0x00
+
+# GFX12: tbuffer_load_d16_format_xyzw v[4:5], off, ttmp[4:7], s3 format:[BUF_FMT_8_8_SINT] offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV 		; encoding:  [0x03,0xc0,0x22,0xc4,0x04,0xe0,0xe8,0x09,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x22,0xc4,0x04,0xe0,0xe8,0x09,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_d16_format_xyzw v[4:5], off, ttmp[4:7], s3 format:[BUF_FMT_32_UINT] offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS 		; encoding:  [0x03,0xc0,0x22,0xc4,0x04,0xe0,0x3c,0x0a,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x22,0xc4,0x04,0xe0,0x3c,0x0a,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_d16_format_xyzw v[4:5], off, ttmp[4:7], s3 format:[BUF_FMT_32_UINT] offset:8388607 scope:SCOPE_SE 		; encoding:  [0x03,0xc0,0x22,0xc4,0x04,0xe0,0x04,0x0a,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x22,0xc4,0x04,0xe0,0x04,0x0a,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_format_x v4, off, s[8:11], s3 format:[BUF_FMT_32_SINT] offset:8388607 		; encoding:  [0x03,0x00,0x20,0xc4,0x04,0x10,0x80,0x0a,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x20,0xc4,0x04,0x10,0x80,0x0a,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_format_x v255, off, s[8:11], s3 format:[BUF_FMT_32_SINT] offset:8388607 		; encoding:  [0x03,0x00,0x20,0xc4,0xff,0x10,0x80,0x0a,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x20,0xc4,0xff,0x10,0x80,0x0a,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_format_x v4, off, s[12:15], s3 format:[BUF_FMT_32_SINT] offset:8388607 		; encoding:  [0x03,0x00,0x20,0xc4,0x04,0x18,0x80,0x0a,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x20,0xc4,0x04,0x18,0x80,0x0a,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_format_x v4, off, s[12:15], s101 format:[BUF_FMT_32_FLOAT] offset:8388607 		; encoding:  [0x65,0x00,0x20,0xc4,0x04,0x18,0x00,0x0b,0x00,0xff,0xff,0x7f]
+0x65,0x00,0x20,0xc4,0x04,0x18,0x00,0x0b,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_format_x v4, off, s[12:15], m0 format:[BUF_FMT_32_FLOAT] offset:8388607 		; encoding:  [0x7d,0x00,0x20,0xc4,0x04,0x18,0x00,0x0b,0x00,0xff,0xff,0x7f]
+0x7d,0x00,0x20,0xc4,0x04,0x18,0x00,0x0b,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_format_x v4, off, s[8:11], s0 format:[BUF_FMT_32_FLOAT] offset:8388607 		; encoding:  [0x00,0x00,0x20,0xc4,0x04,0x10,0x00,0x0b,0x00,0xff,0xff,0x7f]
+0x00,0x00,0x20,0xc4,0x04,0x10,0x00,0x0b,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_format_x v4, off, s[8:11], s61 format:[BUF_FMT_16_16_UNORM] offset:8388607 		; encoding:  [0x3d,0x00,0x20,0xc4,0x04,0x10,0x80,0x0b,0x00,0xff,0xff,0x7f]
+0x3d,0x00,0x20,0xc4,0x04,0x10,0x80,0x0b,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_format_x v4, off, ttmp[4:7], s61 format:[BUF_FMT_16_16_UNORM] offset:8388607 		; encoding:  [0x3d,0x00,0x20,0xc4,0x04,0xe0,0x80,0x0b,0x00,0xff,0xff,0x7f]
+0x3d,0x00,0x20,0xc4,0x04,0xe0,0x80,0x0b,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_format_x v4, v1, s[8:11], s3 format:[BUF_FMT_16_16_UNORM] offen offset:52 		; encoding:  [0x03,0x00,0x20,0xc4,0x04,0x10,0x80,0x4b,0x01,0x34,0x00,0x00]
+0x03,0x00,0x20,0xc4,0x04,0x10,0x80,0x4b,0x01,0x34,0x00,0x00
+
+# GFX12: tbuffer_load_format_x v4, v1, s[8:11], s3 format:[BUF_FMT_16_16_SNORM] idxen offset:52 		; encoding:  [0x03,0x00,0x20,0xc4,0x04,0x10,0x00,0x8c,0x01,0x34,0x00,0x00]
+0x03,0x00,0x20,0xc4,0x04,0x10,0x00,0x8c,0x01,0x34,0x00,0x00
+
+# GFX12: tbuffer_load_format_x v4, v[1:2], s[8:11], s0 format:[BUF_FMT_16_16_SNORM] idxen offen offset:52 		; encoding:  [0x00,0x00,0x20,0xc4,0x04,0x10,0x00,0xcc,0x01,0x34,0x00,0x00]
+0x00,0x00,0x20,0xc4,0x04,0x10,0x00,0xcc,0x01,0x34,0x00,0x00
+
+# GFX12: tbuffer_load_format_x v4, off, ttmp[4:7], s3 format:[BUF_FMT_16_16_SNORM] offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV 		; encoding:  [0x03,0x00,0x20,0xc4,0x04,0xe0,0x68,0x0c,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x20,0xc4,0x04,0xe0,0x68,0x0c,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_format_x v4, off, ttmp[4:7], s3 format:[BUF_FMT_16_16_USCALED] offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS 		; encoding:  [0x03,0x00,0x20,0xc4,0x04,0xe0,0xbc,0x0c,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x20,0xc4,0x04,0xe0,0xbc,0x0c,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_format_x v4, off, ttmp[4:7], s3 format:[BUF_FMT_16_16_USCALED] offset:8388607 scope:SCOPE_SE 		; encoding:  [0x03,0x00,0x20,0xc4,0x04,0xe0,0x84,0x0c,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x20,0xc4,0x04,0xe0,0x84,0x0c,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_format_xy v[4:5], off, s[8:11], s3 format:[BUF_FMT_16_16_SSCALED] offset:8388607 		; encoding:  [0x03,0x40,0x20,0xc4,0x04,0x10,0x00,0x0d,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x20,0xc4,0x04,0x10,0x00,0x0d,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_format_xy v[254:255], off, s[8:11], s3 format:[BUF_FMT_16_16_SSCALED] offset:8388607 		; encoding:  [0x03,0x40,0x20,0xc4,0xfe,0x10,0x00,0x0d,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x20,0xc4,0xfe,0x10,0x00,0x0d,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_format_xy v[4:5], off, s[12:15], s3 format:[BUF_FMT_16_16_SSCALED] offset:8388607 		; encoding:  [0x03,0x40,0x20,0xc4,0x04,0x18,0x00,0x0d,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x20,0xc4,0x04,0x18,0x00,0x0d,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_format_xy v[4:5], off, s[12:15], s101 format:[BUF_FMT_16_16_UINT] offset:8388607 		; encoding:  [0x65,0x40,0x20,0xc4,0x04,0x18,0x80,0x0d,0x00,0xff,0xff,0x7f]
+0x65,0x40,0x20,0xc4,0x04,0x18,0x80,0x0d,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_format_xy v[4:5], off, s[12:15], m0 format:[BUF_FMT_16_16_UINT] offset:8388607 		; encoding:  [0x7d,0x40,0x20,0xc4,0x04,0x18,0x80,0x0d,0x00,0xff,0xff,0x7f]
+0x7d,0x40,0x20,0xc4,0x04,0x18,0x80,0x0d,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_format_xy v[4:5], off, s[8:11], s0 format:[BUF_FMT_16_16_UINT] offset:8388607 		; encoding:  [0x00,0x40,0x20,0xc4,0x04,0x10,0x80,0x0d,0x00,0xff,0xff,0x7f]
+0x00,0x40,0x20,0xc4,0x04,0x10,0x80,0x0d,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_format_xy v[4:5], off, s[8:11], s61 format:[BUF_FMT_16_16_SINT] offset:8388607 		; encoding:  [0x3d,0x40,0x20,0xc4,0x04,0x10,0x00,0x0e,0x00,0xff,0xff,0x7f]
+0x3d,0x40,0x20,0xc4,0x04,0x10,0x00,0x0e,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_format_xy v[4:5], off, ttmp[4:7], s61 format:[BUF_FMT_16_16_SINT] offset:8388607 		; encoding:  [0x3d,0x40,0x20,0xc4,0x04,0xe0,0x00,0x0e,0x00,0xff,0xff,0x7f]
+0x3d,0x40,0x20,0xc4,0x04,0xe0,0x00,0x0e,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_format_xy v[4:5], v1, s[8:11], s3 format:[BUF_FMT_16_16_SINT] offen offset:52 		; encoding:  [0x03,0x40,0x20,0xc4,0x04,0x10,0x00,0x4e,0x01,0x34,0x00,0x00]
+0x03,0x40,0x20,0xc4,0x04,0x10,0x00,0x4e,0x01,0x34,0x00,0x00
+
+# GFX12: tbuffer_load_format_xy v[4:5], v1, s[8:11], s3 format:[BUF_FMT_16_16_FLOAT] idxen offset:52 		; encoding:  [0x03,0x40,0x20,0xc4,0x04,0x10,0x80,0x8e,0x01,0x34,0x00,0x00]
+0x03,0x40,0x20,0xc4,0x04,0x10,0x80,0x8e,0x01,0x34,0x00,0x00
+
+# GFX12: tbuffer_load_format_xy v[4:5], v[1:2], s[8:11], s0 format:[BUF_FMT_16_16_FLOAT] idxen offen offset:52 		; encoding:  [0x00,0x40,0x20,0xc4,0x04,0x10,0x80,0xce,0x01,0x34,0x00,0x00]
+0x00,0x40,0x20,0xc4,0x04,0x10,0x80,0xce,0x01,0x34,0x00,0x00
+
+# GFX12: tbuffer_load_format_xy v[4:5], off, ttmp[4:7], s3 format:[BUF_FMT_16_16_FLOAT] offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV 		; encoding:  [0x03,0x40,0x20,0xc4,0x04,0xe0,0xe8,0x0e,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x20,0xc4,0x04,0xe0,0xe8,0x0e,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_format_xy v[4:5], off, ttmp[4:7], s3 format:[BUF_FMT_10_11_11_FLOAT] offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS 		; encoding:  [0x03,0x40,0x20,0xc4,0x04,0xe0,0x3c,0x0f,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x20,0xc4,0x04,0xe0,0x3c,0x0f,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_format_xy v[4:5], off, ttmp[4:7], s3 format:[BUF_FMT_10_11_11_FLOAT] offset:8388607 scope:SCOPE_SE 		; encoding:  [0x03,0x40,0x20,0xc4,0x04,0xe0,0x04,0x0f,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x20,0xc4,0x04,0xe0,0x04,0x0f,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_format_xyz v[4:6], off, s[8:11], s3 format:[BUF_FMT_11_11_10_FLOAT] offset:8388607 		; encoding:  [0x03,0x80,0x20,0xc4,0x04,0x10,0x80,0x0f,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x20,0xc4,0x04,0x10,0x80,0x0f,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_format_xyz v[253:255], off, s[8:11], s3 format:[BUF_FMT_11_11_10_FLOAT] offset:8388607 		; encoding:  [0x03,0x80,0x20,0xc4,0xfd,0x10,0x80,0x0f,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x20,0xc4,0xfd,0x10,0x80,0x0f,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_format_xyz v[4:6], off, s[12:15], s3 format:[BUF_FMT_11_11_10_FLOAT] offset:8388607 		; encoding:  [0x03,0x80,0x20,0xc4,0x04,0x18,0x80,0x0f,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x20,0xc4,0x04,0x18,0x80,0x0f,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_format_xyz v[4:6], off, s[12:15], s101 format:[BUF_FMT_10_10_10_2_UNORM] offset:8388607 		; encoding:  [0x65,0x80,0x20,0xc4,0x04,0x18,0x00,0x10,0x00,0xff,0xff,0x7f]
+0x65,0x80,0x20,0xc4,0x04,0x18,0x00,0x10,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_format_xyz v[4:6], off, s[12:15], m0 format:[BUF_FMT_10_10_10_2_UNORM] offset:8388607 		; encoding:  [0x7d,0x80,0x20,0xc4,0x04,0x18,0x00,0x10,0x00,0xff,0xff,0x7f]
+0x7d,0x80,0x20,0xc4,0x04,0x18,0x00,0x10,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_format_xyz v[4:6], off, s[8:11], s0 format:[BUF_FMT_10_10_10_2_UNORM] offset:8388607 		; encoding:  [0x00,0x80,0x20,0xc4,0x04,0x10,0x00,0x10,0x00,0xff,0xff,0x7f]
+0x00,0x80,0x20,0xc4,0x04,0x10,0x00,0x10,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_format_xyz v[4:6], off, s[8:11], s61 format:[BUF_FMT_10_10_10_2_SNORM] offset:8388607 		; encoding:  [0x3d,0x80,0x20,0xc4,0x04,0x10,0x80,0x10,0x00,0xff,0xff,0x7f]
+0x3d,0x80,0x20,0xc4,0x04,0x10,0x80,0x10,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_format_xyz v[4:6], off, ttmp[4:7], s61 format:[BUF_FMT_10_10_10_2_SNORM] offset:8388607 		; encoding:  [0x3d,0x80,0x20,0xc4,0x04,0xe0,0x80,0x10,0x00,0xff,0xff,0x7f]
+0x3d,0x80,0x20,0xc4,0x04,0xe0,0x80,0x10,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_format_xyz v[4:6], v1, s[8:11], s3 format:[BUF_FMT_10_10_10_2_SNORM] offen offset:52 		; encoding:  [0x03,0x80,0x20,0xc4,0x04,0x10,0x80,0x50,0x01,0x34,0x00,0x00]
+0x03,0x80,0x20,0xc4,0x04,0x10,0x80,0x50,0x01,0x34,0x00,0x00
+
+# GFX12: tbuffer_load_format_xyz v[4:6], v1, s[8:11], s3 format:[BUF_FMT_10_10_10_2_UINT] idxen offset:52 		; encoding:  [0x03,0x80,0x20,0xc4,0x04,0x10,0x00,0x91,0x01,0x34,0x00,0x00]
+0x03,0x80,0x20,0xc4,0x04,0x10,0x00,0x91,0x01,0x34,0x00,0x00
+
+# GFX12: tbuffer_load_format_xyz v[4:6], v[1:2], s[8:11], s0 format:[BUF_FMT_10_10_10_2_UINT] idxen offen offset:52 		; encoding:  [0x00,0x80,0x20,0xc4,0x04,0x10,0x00,0xd1,0x01,0x34,0x00,0x00]
+0x00,0x80,0x20,0xc4,0x04,0x10,0x00,0xd1,0x01,0x34,0x00,0x00
+
+# GFX12: tbuffer_load_format_xyz v[4:6], off, ttmp[4:7], s3 format:[BUF_FMT_10_10_10_2_UINT] offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV 		; encoding:  [0x03,0x80,0x20,0xc4,0x04,0xe0,0x68,0x11,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x20,0xc4,0x04,0xe0,0x68,0x11,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_format_xyz v[4:6], off, ttmp[4:7], s3 format:[BUF_FMT_10_10_10_2_SINT] offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS 		; encoding:  [0x03,0x80,0x20,0xc4,0x04,0xe0,0xbc,0x11,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x20,0xc4,0x04,0xe0,0xbc,0x11,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_format_xyz v[4:6], off, ttmp[4:7], s3 format:[BUF_FMT_10_10_10_2_SINT] offset:8388607 scope:SCOPE_SE 		; encoding:  [0x03,0x80,0x20,0xc4,0x04,0xe0,0x84,0x11,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x20,0xc4,0x04,0xe0,0x84,0x11,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_format_xyzw v[4:7], off, s[8:11], s3 format:[BUF_FMT_2_10_10_10_UNORM] offset:8388607 		; encoding:  [0x03,0xc0,0x20,0xc4,0x04,0x10,0x00,0x12,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x20,0xc4,0x04,0x10,0x00,0x12,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_format_xyzw v[252:255], off, s[8:11], s3 format:[BUF_FMT_2_10_10_10_UNORM] offset:8388607 		; encoding:  [0x03,0xc0,0x20,0xc4,0xfc,0x10,0x00,0x12,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x20,0xc4,0xfc,0x10,0x00,0x12,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_format_xyzw v[4:7], off, s[12:15], s3 format:[BUF_FMT_2_10_10_10_UNORM] offset:8388607 		; encoding:  [0x03,0xc0,0x20,0xc4,0x04,0x18,0x00,0x12,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x20,0xc4,0x04,0x18,0x00,0x12,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_format_xyzw v[4:7], off, s[12:15], s101 format:[BUF_FMT_2_10_10_10_SNORM] offset:8388607 		; encoding:  [0x65,0xc0,0x20,0xc4,0x04,0x18,0x80,0x12,0x00,0xff,0xff,0x7f]
+0x65,0xc0,0x20,0xc4,0x04,0x18,0x80,0x12,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_format_xyzw v[4:7], off, s[12:15], m0 format:[BUF_FMT_2_10_10_10_SNORM] offset:8388607 		; encoding:  [0x7d,0xc0,0x20,0xc4,0x04,0x18,0x80,0x12,0x00,0xff,0xff,0x7f]
+0x7d,0xc0,0x20,0xc4,0x04,0x18,0x80,0x12,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_format_xyzw v[4:7], off, s[8:11], s0 format:[BUF_FMT_2_10_10_10_SNORM] offset:8388607 		; encoding:  [0x00,0xc0,0x20,0xc4,0x04,0x10,0x80,0x12,0x00,0xff,0xff,0x7f]
+0x00,0xc0,0x20,0xc4,0x04,0x10,0x80,0x12,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_format_xyzw v[4:7], off, s[8:11], s61 format:[BUF_FMT_2_10_10_10_USCALED] offset:8388607 		; encoding:  [0x3d,0xc0,0x20,0xc4,0x04,0x10,0x00,0x13,0x00,0xff,0xff,0x7f]
+0x3d,0xc0,0x20,0xc4,0x04,0x10,0x00,0x13,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_format_xyzw v[4:7], off, ttmp[4:7], s61 format:[BUF_FMT_2_10_10_10_USCALED] offset:8388607 		; encoding:  [0x3d,0xc0,0x20,0xc4,0x04,0xe0,0x00,0x13,0x00,0xff,0xff,0x7f]
+0x3d,0xc0,0x20,0xc4,0x04,0xe0,0x00,0x13,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_format_xyzw v[4:7], v1, s[8:11], s3 format:[BUF_FMT_2_10_10_10_USCALED] offen offset:52 		; encoding:  [0x03,0xc0,0x20,0xc4,0x04,0x10,0x00,0x53,0x01,0x34,0x00,0x00]
+0x03,0xc0,0x20,0xc4,0x04,0x10,0x00,0x53,0x01,0x34,0x00,0x00
+
+# GFX12: tbuffer_load_format_xyzw v[4:7], v1, s[8:11], s3 format:[BUF_FMT_2_10_10_10_SSCALED] idxen offset:52 		; encoding:  [0x03,0xc0,0x20,0xc4,0x04,0x10,0x80,0x93,0x01,0x34,0x00,0x00]
+0x03,0xc0,0x20,0xc4,0x04,0x10,0x80,0x93,0x01,0x34,0x00,0x00
+
+# GFX12: tbuffer_load_format_xyzw v[4:7], v[1:2], s[8:11], s0 format:[BUF_FMT_2_10_10_10_SSCALED] idxen offen offset:52 		; encoding:  [0x00,0xc0,0x20,0xc4,0x04,0x10,0x80,0xd3,0x01,0x34,0x00,0x00]
+0x00,0xc0,0x20,0xc4,0x04,0x10,0x80,0xd3,0x01,0x34,0x00,0x00
+
+# GFX12: tbuffer_load_format_xyzw v[4:7], off, ttmp[4:7], s3 format:[BUF_FMT_2_10_10_10_SSCALED] offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV 		; encoding:  [0x03,0xc0,0x20,0xc4,0x04,0xe0,0xe8,0x13,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x20,0xc4,0x04,0xe0,0xe8,0x13,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_format_xyzw v[4:7], off, ttmp[4:7], s3 format:[BUF_FMT_2_10_10_10_UINT] offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS 		; encoding:  [0x03,0xc0,0x20,0xc4,0x04,0xe0,0x3c,0x14,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x20,0xc4,0x04,0xe0,0x3c,0x14,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_load_format_xyzw v[4:7], off, ttmp[4:7], s3 format:[BUF_FMT_2_10_10_10_UINT] offset:8388607 scope:SCOPE_SE 		; encoding:  [0x03,0xc0,0x20,0xc4,0x04,0xe0,0x04,0x14,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x20,0xc4,0x04,0xe0,0x04,0x14,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_d16_format_x v4, off, s[8:11], s3 format:[BUF_FMT_2_10_10_10_SINT] offset:8388607 		; encoding:  [0x03,0x00,0x23,0xc4,0x04,0x10,0x80,0x14,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x23,0xc4,0x04,0x10,0x80,0x14,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_d16_format_x v255, off, s[8:11], s3 format:[BUF_FMT_2_10_10_10_SINT] offset:8388607 		; encoding:  [0x03,0x00,0x23,0xc4,0xff,0x10,0x80,0x14,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x23,0xc4,0xff,0x10,0x80,0x14,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_d16_format_x v4, off, s[12:15], s3 format:[BUF_FMT_2_10_10_10_SINT] offset:8388607 		; encoding:  [0x03,0x00,0x23,0xc4,0x04,0x18,0x80,0x14,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x23,0xc4,0x04,0x18,0x80,0x14,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_d16_format_x v4, off, s[12:15], s101 format:[BUF_FMT_8_8_8_8_UNORM] offset:8388607 		; encoding:  [0x65,0x00,0x23,0xc4,0x04,0x18,0x00,0x15,0x00,0xff,0xff,0x7f]
+0x65,0x00,0x23,0xc4,0x04,0x18,0x00,0x15,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_d16_format_x v4, off, s[12:15], m0 format:[BUF_FMT_8_8_8_8_UNORM] offset:8388607 		; encoding:  [0x7d,0x00,0x23,0xc4,0x04,0x18,0x00,0x15,0x00,0xff,0xff,0x7f]
+0x7d,0x00,0x23,0xc4,0x04,0x18,0x00,0x15,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_d16_format_x v4, off, s[8:11], s0 format:[BUF_FMT_8_8_8_8_UNORM] offset:8388607 		; encoding:  [0x00,0x00,0x23,0xc4,0x04,0x10,0x00,0x15,0x00,0xff,0xff,0x7f]
+0x00,0x00,0x23,0xc4,0x04,0x10,0x00,0x15,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_d16_format_x v4, off, s[8:11], s61 format:[BUF_FMT_8_8_8_8_SNORM] offset:8388607 		; encoding:  [0x3d,0x00,0x23,0xc4,0x04,0x10,0x80,0x15,0x00,0xff,0xff,0x7f]
+0x3d,0x00,0x23,0xc4,0x04,0x10,0x80,0x15,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_d16_format_x v4, off, ttmp[4:7], s61 format:[BUF_FMT_8_8_8_8_SNORM] offset:8388607 		; encoding:  [0x3d,0x00,0x23,0xc4,0x04,0xe0,0x80,0x15,0x00,0xff,0xff,0x7f]
+0x3d,0x00,0x23,0xc4,0x04,0xe0,0x80,0x15,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_d16_format_x v4, v1, s[8:11], s3 format:[BUF_FMT_8_8_8_8_SNORM] offen offset:52 		; encoding:  [0x03,0x00,0x23,0xc4,0x04,0x10,0x80,0x55,0x01,0x34,0x00,0x00]
+0x03,0x00,0x23,0xc4,0x04,0x10,0x80,0x55,0x01,0x34,0x00,0x00
+
+# GFX12: tbuffer_store_d16_format_x v4, v1, s[8:11], s3 format:[BUF_FMT_8_8_8_8_USCALED] idxen offset:52 		; encoding:  [0x03,0x00,0x23,0xc4,0x04,0x10,0x00,0x96,0x01,0x34,0x00,0x00]
+0x03,0x00,0x23,0xc4,0x04,0x10,0x00,0x96,0x01,0x34,0x00,0x00
+
+# GFX12: tbuffer_store_d16_format_x v4, v[1:2], s[8:11], s0 format:[BUF_FMT_8_8_8_8_USCALED] idxen offen offset:52 		; encoding:  [0x00,0x00,0x23,0xc4,0x04,0x10,0x00,0xd6,0x01,0x34,0x00,0x00]
+0x00,0x00,0x23,0xc4,0x04,0x10,0x00,0xd6,0x01,0x34,0x00,0x00
+
+# GFX12: tbuffer_store_d16_format_x v4, off, ttmp[4:7], s3 format:[BUF_FMT_8_8_8_8_USCALED] offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV 		; encoding:  [0x03,0x00,0x23,0xc4,0x04,0xe0,0x68,0x16,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x23,0xc4,0x04,0xe0,0x68,0x16,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_d16_format_x v4, off, ttmp[4:7], s3 format:[BUF_FMT_8_8_8_8_SSCALED] offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS 		; encoding:  [0x03,0x00,0x23,0xc4,0x04,0xe0,0xbc,0x16,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x23,0xc4,0x04,0xe0,0xbc,0x16,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_d16_format_x v4, off, ttmp[4:7], s3 format:[BUF_FMT_8_8_8_8_SSCALED] offset:8388607 scope:SCOPE_SE 		; encoding:  [0x03,0x00,0x23,0xc4,0x04,0xe0,0x84,0x16,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x23,0xc4,0x04,0xe0,0x84,0x16,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_d16_format_xy v4, off, s[8:11], s3 format:[BUF_FMT_8_8_8_8_UINT] offset:8388607 		; encoding:  [0x03,0x40,0x23,0xc4,0x04,0x10,0x00,0x17,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x23,0xc4,0x04,0x10,0x00,0x17,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_d16_format_xy v255, off, s[8:11], s3 format:[BUF_FMT_8_8_8_8_UINT] offset:8388607 		; encoding:  [0x03,0x40,0x23,0xc4,0xff,0x10,0x00,0x17,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x23,0xc4,0xff,0x10,0x00,0x17,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_d16_format_xy v4, off, s[12:15], s3 format:[BUF_FMT_8_8_8_8_UINT] offset:8388607 		; encoding:  [0x03,0x40,0x23,0xc4,0x04,0x18,0x00,0x17,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x23,0xc4,0x04,0x18,0x00,0x17,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_d16_format_xy v4, off, s[12:15], s101 format:[BUF_FMT_8_8_8_8_SINT] offset:8388607 		; encoding:  [0x65,0x40,0x23,0xc4,0x04,0x18,0x80,0x17,0x00,0xff,0xff,0x7f]
+0x65,0x40,0x23,0xc4,0x04,0x18,0x80,0x17,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_d16_format_xy v4, off, s[12:15], m0 format:[BUF_FMT_8_8_8_8_SINT] offset:8388607 		; encoding:  [0x7d,0x40,0x23,0xc4,0x04,0x18,0x80,0x17,0x00,0xff,0xff,0x7f]
+0x7d,0x40,0x23,0xc4,0x04,0x18,0x80,0x17,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_d16_format_xy v4, off, s[8:11], s0 format:[BUF_FMT_8_8_8_8_SINT] offset:8388607 		; encoding:  [0x00,0x40,0x23,0xc4,0x04,0x10,0x80,0x17,0x00,0xff,0xff,0x7f]
+0x00,0x40,0x23,0xc4,0x04,0x10,0x80,0x17,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_d16_format_xy v4, off, s[8:11], s61 format:[BUF_FMT_32_32_UINT] offset:8388607 		; encoding:  [0x3d,0x40,0x23,0xc4,0x04,0x10,0x00,0x18,0x00,0xff,0xff,0x7f]
+0x3d,0x40,0x23,0xc4,0x04,0x10,0x00,0x18,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_d16_format_xy v4, off, ttmp[4:7], s61 format:[BUF_FMT_32_32_UINT] offset:8388607 		; encoding:  [0x3d,0x40,0x23,0xc4,0x04,0xe0,0x00,0x18,0x00,0xff,0xff,0x7f]
+0x3d,0x40,0x23,0xc4,0x04,0xe0,0x00,0x18,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_d16_format_xy v4, v1, s[8:11], s3 format:[BUF_FMT_32_32_UINT] offen offset:52 		; encoding:  [0x03,0x40,0x23,0xc4,0x04,0x10,0x00,0x58,0x01,0x34,0x00,0x00]
+0x03,0x40,0x23,0xc4,0x04,0x10,0x00,0x58,0x01,0x34,0x00,0x00
+
+# GFX12: tbuffer_store_d16_format_xy v4, v1, s[8:11], s3 format:[BUF_FMT_32_32_SINT] idxen offset:52 		; encoding:  [0x03,0x40,0x23,0xc4,0x04,0x10,0x80,0x98,0x01,0x34,0x00,0x00]
+0x03,0x40,0x23,0xc4,0x04,0x10,0x80,0x98,0x01,0x34,0x00,0x00
+
+# GFX12: tbuffer_store_d16_format_xy v4, v[1:2], s[8:11], s0 format:[BUF_FMT_32_32_SINT] idxen offen offset:52 		; encoding:  [0x00,0x40,0x23,0xc4,0x04,0x10,0x80,0xd8,0x01,0x34,0x00,0x00]
+0x00,0x40,0x23,0xc4,0x04,0x10,0x80,0xd8,0x01,0x34,0x00,0x00
+
+# GFX12: tbuffer_store_d16_format_xy v4, off, ttmp[4:7], s3 format:[BUF_FMT_32_32_SINT] offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV 		; encoding:  [0x03,0x40,0x23,0xc4,0x04,0xe0,0xe8,0x18,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x23,0xc4,0x04,0xe0,0xe8,0x18,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_d16_format_xy v4, off, ttmp[4:7], s3 format:[BUF_FMT_32_32_FLOAT] offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS 		; encoding:  [0x03,0x40,0x23,0xc4,0x04,0xe0,0x3c,0x19,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x23,0xc4,0x04,0xe0,0x3c,0x19,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_d16_format_xy v4, off, ttmp[4:7], s3 format:[BUF_FMT_32_32_FLOAT] offset:8388607 scope:SCOPE_SE 		; encoding:  [0x03,0x40,0x23,0xc4,0x04,0xe0,0x04,0x19,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x23,0xc4,0x04,0xe0,0x04,0x19,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_d16_format_xyz v[4:5], off, s[8:11], s3 format:[BUF_FMT_16_16_16_16_UNORM] offset:8388607 		; encoding:  [0x03,0x80,0x23,0xc4,0x04,0x10,0x80,0x19,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x23,0xc4,0x04,0x10,0x80,0x19,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_d16_format_xyz v[254:255], off, s[8:11], s3 format:[BUF_FMT_16_16_16_16_UNORM] offset:8388607 		; encoding:  [0x03,0x80,0x23,0xc4,0xfe,0x10,0x80,0x19,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x23,0xc4,0xfe,0x10,0x80,0x19,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_d16_format_xyz v[4:5], off, s[12:15], s3 format:[BUF_FMT_16_16_16_16_UNORM] offset:8388607 		; encoding:  [0x03,0x80,0x23,0xc4,0x04,0x18,0x80,0x19,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x23,0xc4,0x04,0x18,0x80,0x19,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_d16_format_xyz v[4:5], off, s[12:15], s101 format:[BUF_FMT_16_16_16_16_SNORM] offset:8388607 		; encoding:  [0x65,0x80,0x23,0xc4,0x04,0x18,0x00,0x1a,0x00,0xff,0xff,0x7f]
+0x65,0x80,0x23,0xc4,0x04,0x18,0x00,0x1a,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_d16_format_xyz v[4:5], off, s[12:15], m0 format:[BUF_FMT_16_16_16_16_SNORM] offset:8388607 		; encoding:  [0x7d,0x80,0x23,0xc4,0x04,0x18,0x00,0x1a,0x00,0xff,0xff,0x7f]
+0x7d,0x80,0x23,0xc4,0x04,0x18,0x00,0x1a,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_d16_format_xyz v[4:5], off, s[8:11], s0 format:[BUF_FMT_16_16_16_16_SNORM] offset:8388607 		; encoding:  [0x00,0x80,0x23,0xc4,0x04,0x10,0x00,0x1a,0x00,0xff,0xff,0x7f]
+0x00,0x80,0x23,0xc4,0x04,0x10,0x00,0x1a,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_d16_format_xyz v[4:5], off, s[8:11], s61 format:[BUF_FMT_16_16_16_16_USCALED] offset:8388607 		; encoding:  [0x3d,0x80,0x23,0xc4,0x04,0x10,0x80,0x1a,0x00,0xff,0xff,0x7f]
+0x3d,0x80,0x23,0xc4,0x04,0x10,0x80,0x1a,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_d16_format_xyz v[4:5], off, ttmp[4:7], s61 format:[BUF_FMT_16_16_16_16_USCALED] offset:8388607 		; encoding:  [0x3d,0x80,0x23,0xc4,0x04,0xe0,0x80,0x1a,0x00,0xff,0xff,0x7f]
+0x3d,0x80,0x23,0xc4,0x04,0xe0,0x80,0x1a,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_d16_format_xyz v[4:5], v1, s[8:11], s3 format:[BUF_FMT_16_16_16_16_USCALED] offen offset:52 		; encoding:  [0x03,0x80,0x23,0xc4,0x04,0x10,0x80,0x5a,0x01,0x34,0x00,0x00]
+0x03,0x80,0x23,0xc4,0x04,0x10,0x80,0x5a,0x01,0x34,0x00,0x00
+
+# GFX12: tbuffer_store_d16_format_xyz v[4:5], v1, s[8:11], s3 format:[BUF_FMT_16_16_16_16_SSCALED] idxen offset:52 		; encoding:  [0x03,0x80,0x23,0xc4,0x04,0x10,0x00,0x9b,0x01,0x34,0x00,0x00]
+0x03,0x80,0x23,0xc4,0x04,0x10,0x00,0x9b,0x01,0x34,0x00,0x00
+
+# GFX12: tbuffer_store_d16_format_xyz v[4:5], v[1:2], s[8:11], s0 format:[BUF_FMT_16_16_16_16_SSCALED] idxen offen offset:52 		; encoding:  [0x00,0x80,0x23,0xc4,0x04,0x10,0x00,0xdb,0x01,0x34,0x00,0x00]
+0x00,0x80,0x23,0xc4,0x04,0x10,0x00,0xdb,0x01,0x34,0x00,0x00
+
+# GFX12: tbuffer_store_d16_format_xyz v[4:5], off, ttmp[4:7], s3 format:[BUF_FMT_16_16_16_16_SSCALED] offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV 		; encoding:  [0x03,0x80,0x23,0xc4,0x04,0xe0,0x68,0x1b,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x23,0xc4,0x04,0xe0,0x68,0x1b,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_d16_format_xyz v[4:5], off, ttmp[4:7], s3 format:[BUF_FMT_16_16_16_16_UINT] offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS 		; encoding:  [0x03,0x80,0x23,0xc4,0x04,0xe0,0xbc,0x1b,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x23,0xc4,0x04,0xe0,0xbc,0x1b,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_d16_format_xyz v[4:5], off, ttmp[4:7], s3 format:[BUF_FMT_16_16_16_16_UINT] offset:8388607 scope:SCOPE_SE 		; encoding:  [0x03,0x80,0x23,0xc4,0x04,0xe0,0x84,0x1b,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x23,0xc4,0x04,0xe0,0x84,0x1b,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_d16_format_xyzw v[4:5], off, s[8:11], s3 format:[BUF_FMT_16_16_16_16_SINT] offset:8388607 		; encoding:  [0x03,0xc0,0x23,0xc4,0x04,0x10,0x00,0x1c,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x23,0xc4,0x04,0x10,0x00,0x1c,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_d16_format_xyzw v[254:255], off, s[8:11], s3 format:[BUF_FMT_16_16_16_16_SINT] offset:8388607 		; encoding:  [0x03,0xc0,0x23,0xc4,0xfe,0x10,0x00,0x1c,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x23,0xc4,0xfe,0x10,0x00,0x1c,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_d16_format_xyzw v[4:5], off, s[12:15], s3 format:[BUF_FMT_16_16_16_16_SINT] offset:8388607 		; encoding:  [0x03,0xc0,0x23,0xc4,0x04,0x18,0x00,0x1c,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x23,0xc4,0x04,0x18,0x00,0x1c,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_d16_format_xyzw v[4:5], off, s[12:15], s101 format:[BUF_FMT_16_16_16_16_FLOAT] offset:8388607 		; encoding:  [0x65,0xc0,0x23,0xc4,0x04,0x18,0x80,0x1c,0x00,0xff,0xff,0x7f]
+0x65,0xc0,0x23,0xc4,0x04,0x18,0x80,0x1c,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_d16_format_xyzw v[4:5], off, s[12:15], m0 format:[BUF_FMT_16_16_16_16_FLOAT] offset:8388607 		; encoding:  [0x7d,0xc0,0x23,0xc4,0x04,0x18,0x80,0x1c,0x00,0xff,0xff,0x7f]
+0x7d,0xc0,0x23,0xc4,0x04,0x18,0x80,0x1c,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_d16_format_xyzw v[4:5], off, s[8:11], s0 format:[BUF_FMT_16_16_16_16_FLOAT] offset:8388607 		; encoding:  [0x00,0xc0,0x23,0xc4,0x04,0x10,0x80,0x1c,0x00,0xff,0xff,0x7f]
+0x00,0xc0,0x23,0xc4,0x04,0x10,0x80,0x1c,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_d16_format_xyzw v[4:5], off, s[8:11], s61 format:[BUF_FMT_32_32_32_UINT] offset:8388607 		; encoding:  [0x3d,0xc0,0x23,0xc4,0x04,0x10,0x00,0x1d,0x00,0xff,0xff,0x7f]
+0x3d,0xc0,0x23,0xc4,0x04,0x10,0x00,0x1d,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_d16_format_xyzw v[4:5], off, ttmp[4:7], s61 format:[BUF_FMT_32_32_32_UINT] offset:8388607 		; encoding:  [0x3d,0xc0,0x23,0xc4,0x04,0xe0,0x00,0x1d,0x00,0xff,0xff,0x7f]
+0x3d,0xc0,0x23,0xc4,0x04,0xe0,0x00,0x1d,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_d16_format_xyzw v[4:5], v1, s[8:11], s3 format:[BUF_FMT_32_32_32_UINT] offen offset:52 		; encoding:  [0x03,0xc0,0x23,0xc4,0x04,0x10,0x00,0x5d,0x01,0x34,0x00,0x00]
+0x03,0xc0,0x23,0xc4,0x04,0x10,0x00,0x5d,0x01,0x34,0x00,0x00
+
+# GFX12: tbuffer_store_d16_format_xyzw v[4:5], v1, s[8:11], s3 format:[BUF_FMT_32_32_32_SINT] idxen offset:52 		; encoding:  [0x03,0xc0,0x23,0xc4,0x04,0x10,0x80,0x9d,0x01,0x34,0x00,0x00]
+0x03,0xc0,0x23,0xc4,0x04,0x10,0x80,0x9d,0x01,0x34,0x00,0x00
+
+# GFX12: tbuffer_store_d16_format_xyzw v[4:5], v[1:2], s[8:11], s0 format:[BUF_FMT_32_32_32_SINT] idxen offen offset:52 		; encoding:  [0x00,0xc0,0x23,0xc4,0x04,0x10,0x80,0xdd,0x01,0x34,0x00,0x00]
+0x00,0xc0,0x23,0xc4,0x04,0x10,0x80,0xdd,0x01,0x34,0x00,0x00
+
+# GFX12: tbuffer_store_d16_format_xyzw v[4:5], off, ttmp[4:7], s3 format:[BUF_FMT_32_32_32_SINT] offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV 		; encoding:  [0x03,0xc0,0x23,0xc4,0x04,0xe0,0xe8,0x1d,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x23,0xc4,0x04,0xe0,0xe8,0x1d,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_d16_format_xyzw v[4:5], off, ttmp[4:7], s3 format:[BUF_FMT_32_32_32_FLOAT] offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS 		; encoding:  [0x03,0xc0,0x23,0xc4,0x04,0xe0,0x3c,0x1e,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x23,0xc4,0x04,0xe0,0x3c,0x1e,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_d16_format_xyzw v[4:5], off, ttmp[4:7], s3 format:[BUF_FMT_32_32_32_FLOAT] offset:8388607 scope:SCOPE_SE 		; encoding:  [0x03,0xc0,0x23,0xc4,0x04,0xe0,0x04,0x1e,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x23,0xc4,0x04,0xe0,0x04,0x1e,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_format_x v4, off, s[8:11], s3 format:[BUF_FMT_32_32_32_32_UINT] offset:8388607 		; encoding:  [0x03,0x00,0x21,0xc4,0x04,0x10,0x80,0x1e,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x21,0xc4,0x04,0x10,0x80,0x1e,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_format_x v255, off, s[8:11], s3 format:[BUF_FMT_32_32_32_32_UINT] offset:8388607 		; encoding:  [0x03,0x00,0x21,0xc4,0xff,0x10,0x80,0x1e,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x21,0xc4,0xff,0x10,0x80,0x1e,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_format_x v4, off, s[12:15], s3 format:[BUF_FMT_32_32_32_32_UINT] offset:8388607 		; encoding:  [0x03,0x00,0x21,0xc4,0x04,0x18,0x80,0x1e,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x21,0xc4,0x04,0x18,0x80,0x1e,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_format_x v4, off, s[12:15], s101 format:[BUF_FMT_32_32_32_32_SINT] offset:8388607 		; encoding:  [0x65,0x00,0x21,0xc4,0x04,0x18,0x00,0x1f,0x00,0xff,0xff,0x7f]
+0x65,0x00,0x21,0xc4,0x04,0x18,0x00,0x1f,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_format_x v4, off, s[12:15], m0 format:[BUF_FMT_32_32_32_32_SINT] offset:8388607 		; encoding:  [0x7d,0x00,0x21,0xc4,0x04,0x18,0x00,0x1f,0x00,0xff,0xff,0x7f]
+0x7d,0x00,0x21,0xc4,0x04,0x18,0x00,0x1f,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_format_x v4, off, s[8:11], s0 format:[BUF_FMT_32_32_32_32_SINT] offset:8388607 		; encoding:  [0x00,0x00,0x21,0xc4,0x04,0x10,0x00,0x1f,0x00,0xff,0xff,0x7f]
+0x00,0x00,0x21,0xc4,0x04,0x10,0x00,0x1f,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_format_x v4, off, s[8:11], s61 format:[BUF_FMT_32_32_32_32_FLOAT] offset:8388607 		; encoding:  [0x3d,0x00,0x21,0xc4,0x04,0x10,0x80,0x1f,0x00,0xff,0xff,0x7f]
+0x3d,0x00,0x21,0xc4,0x04,0x10,0x80,0x1f,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_format_x v4, off, ttmp[4:7], s61 format:[BUF_FMT_32_32_32_32_FLOAT] offset:8388607 		; encoding:  [0x3d,0x00,0x21,0xc4,0x04,0xe0,0x80,0x1f,0x00,0xff,0xff,0x7f]
+0x3d,0x00,0x21,0xc4,0x04,0xe0,0x80,0x1f,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_format_x v4, v1, s[8:11], s3 format:[BUF_FMT_32_32_32_32_FLOAT] offen offset:52 		; encoding:  [0x03,0x00,0x21,0xc4,0x04,0x10,0x80,0x5f,0x01,0x34,0x00,0x00]
+0x03,0x00,0x21,0xc4,0x04,0x10,0x80,0x5f,0x01,0x34,0x00,0x00
+
+# GFX12: tbuffer_store_format_x v4, v1, s[8:11], s3 idxen offset:52 		; encoding:  [0x03,0x00,0x21,0xc4,0x04,0x10,0x80,0x80,0x01,0x34,0x00,0x00]
+0x03,0x00,0x21,0xc4,0x04,0x10,0x80,0x80,0x01,0x34,0x00,0x00
+
+# GFX12: tbuffer_store_format_x v4, v[1:2], s[8:11], s0 format:[BUF_FMT_8_SNORM] idxen offen offset:52 		; encoding:  [0x00,0x00,0x21,0xc4,0x04,0x10,0x00,0xc1,0x01,0x34,0x00,0x00]
+0x00,0x00,0x21,0xc4,0x04,0x10,0x00,0xc1,0x01,0x34,0x00,0x00
+
+# GFX12: tbuffer_store_format_x v4, off, ttmp[4:7], s3 format:[BUF_FMT_8_USCALED] offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV 		; encoding:  [0x03,0x00,0x21,0xc4,0x04,0xe0,0xe8,0x01,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x21,0xc4,0x04,0xe0,0xe8,0x01,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_format_x v4, off, ttmp[4:7], s3 format:[BUF_FMT_8_SSCALED] offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS 		; encoding:  [0x03,0x00,0x21,0xc4,0x04,0xe0,0x3c,0x02,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x21,0xc4,0x04,0xe0,0x3c,0x02,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_format_x v4, off, ttmp[4:7], s3 format:[BUF_FMT_8_UINT] offset:8388607 scope:SCOPE_SE 		; encoding:  [0x03,0x00,0x21,0xc4,0x04,0xe0,0x84,0x02,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x21,0xc4,0x04,0xe0,0x84,0x02,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_format_xy v[4:5], off, s[8:11], s3 format:[BUF_FMT_16_UNORM] offset:8388607 		; encoding:  [0x03,0x40,0x21,0xc4,0x04,0x10,0x80,0x03,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x21,0xc4,0x04,0x10,0x80,0x03,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_format_xy v[254:255], off, s[8:11], s3 format:[BUF_FMT_16_SNORM] offset:8388607 		; encoding:  [0x03,0x40,0x21,0xc4,0xfe,0x10,0x00,0x04,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x21,0xc4,0xfe,0x10,0x00,0x04,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_format_xy v[4:5], off, s[12:15], s3 format:[BUF_FMT_16_USCALED] offset:8388607 		; encoding:  [0x03,0x40,0x21,0xc4,0x04,0x18,0x80,0x04,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x21,0xc4,0x04,0x18,0x80,0x04,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_format_xy v[4:5], off, s[12:15], s101 format:[BUF_FMT_16_SSCALED] offset:8388607 		; encoding:  [0x65,0x40,0x21,0xc4,0x04,0x18,0x00,0x05,0x00,0xff,0xff,0x7f]
+0x65,0x40,0x21,0xc4,0x04,0x18,0x00,0x05,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_format_xy v[4:5], off, s[12:15], m0 format:[BUF_FMT_16_UINT] offset:8388607 		; encoding:  [0x7d,0x40,0x21,0xc4,0x04,0x18,0x80,0x05,0x00,0xff,0xff,0x7f]
+0x7d,0x40,0x21,0xc4,0x04,0x18,0x80,0x05,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_format_xy v[4:5], off, s[8:11], s0 format:[BUF_FMT_16_SINT] offset:8388607 		; encoding:  [0x00,0x40,0x21,0xc4,0x04,0x10,0x00,0x06,0x00,0xff,0xff,0x7f]
+0x00,0x40,0x21,0xc4,0x04,0x10,0x00,0x06,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_format_xy v[4:5], off, s[8:11], s61 format:[BUF_FMT_16_FLOAT] offset:8388607 		; encoding:  [0x3d,0x40,0x21,0xc4,0x04,0x10,0x80,0x06,0x00,0xff,0xff,0x7f]
+0x3d,0x40,0x21,0xc4,0x04,0x10,0x80,0x06,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_format_xy v[4:5], off, ttmp[4:7], s61 format:[BUF_FMT_8_8_UNORM] offset:8388607 		; encoding:  [0x3d,0x40,0x21,0xc4,0x04,0xe0,0x00,0x07,0x00,0xff,0xff,0x7f]
+0x3d,0x40,0x21,0xc4,0x04,0xe0,0x00,0x07,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_format_xy v[4:5], v1, s[8:11], s3 format:[BUF_FMT_8_8_SNORM] offen offset:52 		; encoding:  [0x03,0x40,0x21,0xc4,0x04,0x10,0x80,0x47,0x01,0x34,0x00,0x00]
+0x03,0x40,0x21,0xc4,0x04,0x10,0x80,0x47,0x01,0x34,0x00,0x00
+
+# GFX12: tbuffer_store_format_xy v[4:5], v1, s[8:11], s3 format:[BUF_FMT_8_8_USCALED] idxen offset:52 		; encoding:  [0x03,0x40,0x21,0xc4,0x04,0x10,0x00,0x88,0x01,0x34,0x00,0x00]
+0x03,0x40,0x21,0xc4,0x04,0x10,0x00,0x88,0x01,0x34,0x00,0x00
+
+# GFX12: tbuffer_store_format_xy v[4:5], v[1:2], s[8:11], s0 format:[BUF_FMT_8_8_SSCALED] idxen offen offset:52 		; encoding:  [0x00,0x40,0x21,0xc4,0x04,0x10,0x80,0xc8,0x01,0x34,0x00,0x00]
+0x00,0x40,0x21,0xc4,0x04,0x10,0x80,0xc8,0x01,0x34,0x00,0x00
+
+# GFX12: tbuffer_store_format_xy v[4:5], off, ttmp[4:7], s3 format:[BUF_FMT_8_8_UINT] offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV 		; encoding:  [0x03,0x40,0x21,0xc4,0x04,0xe0,0x68,0x09,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x21,0xc4,0x04,0xe0,0x68,0x09,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_format_xy v[4:5], off, ttmp[4:7], s3 format:[BUF_FMT_8_8_SINT] offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS 		; encoding:  [0x03,0x40,0x21,0xc4,0x04,0xe0,0xbc,0x09,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x21,0xc4,0x04,0xe0,0xbc,0x09,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_format_xy v[4:5], off, ttmp[4:7], s3 format:[BUF_FMT_32_UINT] offset:8388607 scope:SCOPE_SE 		; encoding:  [0x03,0x40,0x21,0xc4,0x04,0xe0,0x04,0x0a,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x21,0xc4,0x04,0xe0,0x04,0x0a,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_format_xyz v[4:6], off, s[8:11], s3 format:[BUF_FMT_32_FLOAT] offset:8388607 		; encoding:  [0x03,0x80,0x21,0xc4,0x04,0x10,0x00,0x0b,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x21,0xc4,0x04,0x10,0x00,0x0b,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_format_xyz v[253:255], off, s[8:11], s3 format:[BUF_FMT_16_16_UNORM] offset:8388607 		; encoding:  [0x03,0x80,0x21,0xc4,0xfd,0x10,0x80,0x0b,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x21,0xc4,0xfd,0x10,0x80,0x0b,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_format_xyz v[4:6], off, s[12:15], s3 format:[BUF_FMT_16_16_SNORM] offset:8388607 		; encoding:  [0x03,0x80,0x21,0xc4,0x04,0x18,0x00,0x0c,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x21,0xc4,0x04,0x18,0x00,0x0c,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_format_xyz v[4:6], off, s[12:15], s101 format:[BUF_FMT_16_16_USCALED] offset:8388607 		; encoding:  [0x65,0x80,0x21,0xc4,0x04,0x18,0x80,0x0c,0x00,0xff,0xff,0x7f]
+0x65,0x80,0x21,0xc4,0x04,0x18,0x80,0x0c,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_format_xyz v[4:6], off, s[12:15], m0 format:[BUF_FMT_16_16_SSCALED] offset:8388607 		; encoding:  [0x7d,0x80,0x21,0xc4,0x04,0x18,0x00,0x0d,0x00,0xff,0xff,0x7f]
+0x7d,0x80,0x21,0xc4,0x04,0x18,0x00,0x0d,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_format_xyz v[4:6], off, s[8:11], s0 format:[BUF_FMT_16_16_UINT] offset:8388607 		; encoding:  [0x00,0x80,0x21,0xc4,0x04,0x10,0x80,0x0d,0x00,0xff,0xff,0x7f]
+0x00,0x80,0x21,0xc4,0x04,0x10,0x80,0x0d,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_format_xyz v[4:6], off, s[8:11], s61 format:[BUF_FMT_16_16_SINT] offset:8388607 		; encoding:  [0x3d,0x80,0x21,0xc4,0x04,0x10,0x00,0x0e,0x00,0xff,0xff,0x7f]
+0x3d,0x80,0x21,0xc4,0x04,0x10,0x00,0x0e,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_format_xyz v[4:6], off, ttmp[4:7], s61 format:[BUF_FMT_16_16_FLOAT] offset:8388607 		; encoding:  [0x3d,0x80,0x21,0xc4,0x04,0xe0,0x80,0x0e,0x00,0xff,0xff,0x7f]
+0x3d,0x80,0x21,0xc4,0x04,0xe0,0x80,0x0e,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_format_xyz v[4:6], v1, s[8:11], s3 format:[BUF_FMT_10_11_11_FLOAT] offen offset:52 		; encoding:  [0x03,0x80,0x21,0xc4,0x04,0x10,0x00,0x4f,0x01,0x34,0x00,0x00]
+0x03,0x80,0x21,0xc4,0x04,0x10,0x00,0x4f,0x01,0x34,0x00,0x00
+
+# GFX12: tbuffer_store_format_xyz v[4:6], v1, s[8:11], s3 format:[BUF_FMT_11_11_10_FLOAT] idxen offset:52 		; encoding:  [0x03,0x80,0x21,0xc4,0x04,0x10,0x80,0x8f,0x01,0x34,0x00,0x00]
+0x03,0x80,0x21,0xc4,0x04,0x10,0x80,0x8f,0x01,0x34,0x00,0x00
+
+# GFX12: tbuffer_store_format_xyz v[4:6], v[1:2], s[8:11], s0 format:[BUF_FMT_10_10_10_2_UNORM] idxen offen offset:52 		; encoding:  [0x00,0x80,0x21,0xc4,0x04,0x10,0x00,0xd0,0x01,0x34,0x00,0x00]
+0x00,0x80,0x21,0xc4,0x04,0x10,0x00,0xd0,0x01,0x34,0x00,0x00
+
+# GFX12: tbuffer_store_format_xyz v[4:6], off, ttmp[4:7], s3 format:[BUF_FMT_10_10_10_2_SNORM] offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV 		; encoding:  [0x03,0x80,0x21,0xc4,0x04,0xe0,0xe8,0x10,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x21,0xc4,0x04,0xe0,0xe8,0x10,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_format_xyz v[4:6], off, ttmp[4:7], s3 format:[BUF_FMT_10_10_10_2_UINT] offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS 		; encoding:  [0x03,0x80,0x21,0xc4,0x04,0xe0,0x3c,0x11,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x21,0xc4,0x04,0xe0,0x3c,0x11,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_format_xyz v[4:6], off, ttmp[4:7], s3 format:[BUF_FMT_10_10_10_2_SINT] offset:8388607 scope:SCOPE_SE 		; encoding:  [0x03,0x80,0x21,0xc4,0x04,0xe0,0x84,0x11,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x21,0xc4,0x04,0xe0,0x84,0x11,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_format_xyzw v[4:7], off, s[8:11], s3 format:[BUF_FMT_2_10_10_10_SNORM] offset:8388607 		; encoding:  [0x03,0xc0,0x21,0xc4,0x04,0x10,0x80,0x12,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x21,0xc4,0x04,0x10,0x80,0x12,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_format_xyzw v[252:255], off, s[8:11], s3 format:[BUF_FMT_2_10_10_10_USCALED] offset:8388607 		; encoding:  [0x03,0xc0,0x21,0xc4,0xfc,0x10,0x00,0x13,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x21,0xc4,0xfc,0x10,0x00,0x13,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_format_xyzw v[4:7], off, s[12:15], s3 format:[BUF_FMT_2_10_10_10_SSCALED] offset:8388607 		; encoding:  [0x03,0xc0,0x21,0xc4,0x04,0x18,0x80,0x13,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x21,0xc4,0x04,0x18,0x80,0x13,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_format_xyzw v[4:7], off, s[12:15], s101 format:[BUF_FMT_2_10_10_10_UINT] offset:8388607 		; encoding:  [0x65,0xc0,0x21,0xc4,0x04,0x18,0x00,0x14,0x00,0xff,0xff,0x7f]
+0x65,0xc0,0x21,0xc4,0x04,0x18,0x00,0x14,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_format_xyzw v[4:7], off, s[12:15], m0 format:[BUF_FMT_2_10_10_10_SINT] offset:8388607 		; encoding:  [0x7d,0xc0,0x21,0xc4,0x04,0x18,0x80,0x14,0x00,0xff,0xff,0x7f]
+0x7d,0xc0,0x21,0xc4,0x04,0x18,0x80,0x14,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_format_xyzw v[4:7], off, s[8:11], s0 format:[BUF_FMT_8_8_8_8_UNORM] offset:8388607 		; encoding:  [0x00,0xc0,0x21,0xc4,0x04,0x10,0x00,0x15,0x00,0xff,0xff,0x7f]
+0x00,0xc0,0x21,0xc4,0x04,0x10,0x00,0x15,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_format_xyzw v[4:7], off, s[8:11], s61 format:[BUF_FMT_8_8_8_8_SNORM] offset:8388607 		; encoding:  [0x3d,0xc0,0x21,0xc4,0x04,0x10,0x80,0x15,0x00,0xff,0xff,0x7f]
+0x3d,0xc0,0x21,0xc4,0x04,0x10,0x80,0x15,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_format_xyzw v[4:7], off, ttmp[4:7], s61 format:[BUF_FMT_8_8_8_8_USCALED] offset:8388607 		; encoding:  [0x3d,0xc0,0x21,0xc4,0x04,0xe0,0x00,0x16,0x00,0xff,0xff,0x7f]
+0x3d,0xc0,0x21,0xc4,0x04,0xe0,0x00,0x16,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_format_xyzw v[4:7], v1, s[8:11], s3 format:[BUF_FMT_8_8_8_8_SSCALED] offen offset:52 		; encoding:  [0x03,0xc0,0x21,0xc4,0x04,0x10,0x80,0x56,0x01,0x34,0x00,0x00]
+0x03,0xc0,0x21,0xc4,0x04,0x10,0x80,0x56,0x01,0x34,0x00,0x00
+
+# GFX12: tbuffer_store_format_xyzw v[4:7], v1, s[8:11], s3 format:[BUF_FMT_8_8_8_8_UINT] idxen offset:52 		; encoding:  [0x03,0xc0,0x21,0xc4,0x04,0x10,0x00,0x97,0x01,0x34,0x00,0x00]
+0x03,0xc0,0x21,0xc4,0x04,0x10,0x00,0x97,0x01,0x34,0x00,0x00
+
+# GFX12: tbuffer_store_format_xyzw v[4:7], v[1:2], s[8:11], s0 format:[BUF_FMT_8_8_8_8_SINT] idxen offen offset:52 		; encoding:  [0x00,0xc0,0x21,0xc4,0x04,0x10,0x80,0xd7,0x01,0x34,0x00,0x00]
+0x00,0xc0,0x21,0xc4,0x04,0x10,0x80,0xd7,0x01,0x34,0x00,0x00
+
+# GFX12: tbuffer_store_format_xyzw v[4:7], off, ttmp[4:7], s3 format:[BUF_FMT_32_32_UINT] offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV 		; encoding:  [0x03,0xc0,0x21,0xc4,0x04,0xe0,0x68,0x18,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x21,0xc4,0x04,0xe0,0x68,0x18,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_format_xyzw v[4:7], off, ttmp[4:7], s3 format:[BUF_FMT_32_32_SINT] offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS 		; encoding:  [0x03,0xc0,0x21,0xc4,0x04,0xe0,0xbc,0x18,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x21,0xc4,0x04,0xe0,0xbc,0x18,0x00,0xff,0xff,0x7f
+
+# GFX12: tbuffer_store_format_xyzw v[4:7], off, ttmp[4:7], s3 format:[BUF_FMT_32_32_FLOAT] offset:8388607 scope:SCOPE_SE 		; encoding:  [0x03,0xc0,0x21,0xc4,0x04,0xe0,0x04,0x19,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x21,0xc4,0x04,0xe0,0x04,0x19,0x00,0xff,0xff,0x7f
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vbuffer_mubuf.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vbuffer_mubuf.txt
new file mode 100644
index 0000000000000000000000000000000000000000..ff8437155e12edd66c51de0cea1c131ee4b4cbec
--- /dev/null
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vbuffer_mubuf.txt
@@ -0,0 +1,2911 @@
+# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck -check-prefixes=GFX12 %s
+
+# GFX12: buffer_load_b32 v5, off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x00,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_b32 v255, off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x00,0x05,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x05,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_b32 v5, off, s[12:15], s3 offset:8388607 ; encoding:  [0x03,0x00,0x05,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x05,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_b32 v5, off, s[96:99], s3 offset:8388607 ; encoding:  [0x03,0x00,0x05,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x05,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_b32 v5, off, s[8:11], s101 offset:8388607 ; encoding:  [0x65,0x00,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x00,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_b32 v5, off, s[8:11], m0 offset:8388607 ; encoding:  [0x7d,0x00,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x00,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_b32 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding:  [0x03,0x00,0x05,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x05,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_b32 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding:  [0x03,0x00,0x05,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x05,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_b32 v5, off, s[8:11], s3 ; encoding:  [0x03,0x00,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x00,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_load_b32 v5, off, s[8:11], s3 ; encoding:  [0x03,0x00,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x00,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_load_b32 v5, off, s[8:11], s3 offset:7 ; encoding:  [0x03,0x00,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x00,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_load_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV ; encoding:  [0x03,0x00,0x05,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x05,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS ; encoding:  [0x03,0x00,0x05,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x05,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_b64 v[5:6], off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x40,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_b64 v[254:255], off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x40,0x05,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x05,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_b64 v[5:6], off, s[12:15], s3 offset:8388607 ; encoding:  [0x03,0x40,0x05,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x05,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_b64 v[5:6], off, s[96:99], s3 offset:8388607 ; encoding:  [0x03,0x40,0x05,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x05,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_b64 v[5:6], off, s[8:11], s101 offset:8388607 ; encoding:  [0x65,0x40,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x40,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_b64 v[5:6], off, s[8:11], m0 offset:8388607 ; encoding:  [0x7d,0x40,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x40,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_b64 v[5:6], v0, s[8:11], s3 idxen offset:8388607 ; encoding:  [0x03,0x40,0x05,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x05,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_b64 v[5:6], v0, s[8:11], s3 offen offset:8388607 ; encoding:  [0x03,0x40,0x05,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x05,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_b64 v[5:6], off, s[8:11], s3 ; encoding:  [0x03,0x40,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x40,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_load_b64 v[5:6], off, s[8:11], s3 ; encoding:  [0x03,0x40,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x40,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_load_b64 v[5:6], off, s[8:11], s3 offset:7 ; encoding:  [0x03,0x40,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x40,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_load_b64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV ; encoding:  [0x03,0x40,0x05,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x05,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_b64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS ; encoding:  [0x03,0x40,0x05,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x05,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_b96 v[5:7], off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x80,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_b96 v[253:255], off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x80,0x05,0xc4,0xfd,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x05,0xc4,0xfd,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_b96 v[5:7], off, s[12:15], s3 offset:8388607 ; encoding:  [0x03,0x80,0x05,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x05,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_b96 v[5:7], off, s[96:99], s3 offset:8388607 ; encoding:  [0x03,0x80,0x05,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x05,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_b96 v[5:7], off, s[8:11], s101 offset:8388607 ; encoding:  [0x65,0x80,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x80,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_b96 v[5:7], off, s[8:11], m0 offset:8388607 ; encoding:  [0x7d,0x80,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x80,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_b96 v[5:7], v0, s[8:11], s3 idxen offset:8388607 ; encoding:  [0x03,0x80,0x05,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x05,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_b96 v[5:7], v0, s[8:11], s3 offen offset:8388607 ; encoding:  [0x03,0x80,0x05,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x05,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_b96 v[5:7], off, s[8:11], s3 ; encoding:  [0x03,0x80,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x80,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_load_b96 v[5:7], off, s[8:11], s3 ; encoding:  [0x03,0x80,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x80,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_load_b96 v[5:7], off, s[8:11], s3 offset:7 ; encoding:  [0x03,0x80,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x80,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_load_b96 v[5:7], off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV ; encoding:  [0x03,0x80,0x05,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x05,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_b96 v[5:7], off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS ; encoding:  [0x03,0x80,0x05,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x05,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_b128 v[5:8], off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0xc0,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_b128 v[252:255], off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0xc0,0x05,0xc4,0xfc,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x05,0xc4,0xfc,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_b128 v[5:8], off, s[12:15], s3 offset:8388607 ; encoding:  [0x03,0xc0,0x05,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x05,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_b128 v[5:8], off, s[96:99], s3 offset:8388607 ; encoding:  [0x03,0xc0,0x05,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x05,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_b128 v[5:8], off, s[8:11], s101 offset:8388607 ; encoding:  [0x65,0xc0,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0xc0,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_b128 v[5:8], off, s[8:11], m0 offset:8388607 ; encoding:  [0x7d,0xc0,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0xc0,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_b128 v[5:8], v0, s[8:11], s3 idxen offset:8388607 ; encoding:  [0x03,0xc0,0x05,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x05,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_b128 v[5:8], v0, s[8:11], s3 offen offset:8388607 ; encoding:  [0x03,0xc0,0x05,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x05,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_b128 v[5:8], off, s[8:11], s3 ; encoding:  [0x03,0xc0,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0xc0,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_load_b128 v[5:8], off, s[8:11], s3 ; encoding:  [0x03,0xc0,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0xc0,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_load_b128 v[5:8], off, s[8:11], s3 offset:7 ; encoding:  [0x03,0xc0,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0xc0,0x05,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_load_b128 v[5:8], off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV ; encoding:  [0x03,0xc0,0x05,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x05,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_b128 v[5:8], off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS ; encoding:  [0x03,0xc0,0x05,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x05,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_b16 v5, off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x00,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_b16 v255, off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x00,0x08,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x08,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_b16 v5, off, s[12:15], s3 offset:8388607 ; encoding:  [0x03,0x00,0x08,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x08,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_b16 v5, off, s[96:99], s3 offset:8388607 ; encoding:  [0x03,0x00,0x08,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x08,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_b16 v5, off, s[8:11], s101 offset:8388607 ; encoding:  [0x65,0x00,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x00,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_b16 v5, off, s[8:11], m0 offset:8388607 ; encoding:  [0x7d,0x00,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x00,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_b16 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding:  [0x03,0x00,0x08,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x08,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_b16 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding:  [0x03,0x00,0x08,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x08,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_b16 v5, off, s[8:11], s3 ; encoding:  [0x03,0x00,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x00,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_load_d16_b16 v5, off, s[8:11], s3 ; encoding:  [0x03,0x00,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x00,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_load_d16_b16 v5, off, s[8:11], s3 offset:7 ; encoding:  [0x03,0x00,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x00,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_load_d16_b16 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV ; encoding:  [0x03,0x00,0x08,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x08,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_b16 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS ; encoding:  [0x03,0x00,0x08,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x08,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_format_x v5, off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x00,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_format_x v255, off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x00,0x02,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x02,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_format_x v5, off, s[12:15], s3 offset:8388607 ; encoding:  [0x03,0x00,0x02,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x02,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_format_x v5, off, s[96:99], s3 offset:8388607 ; encoding:  [0x03,0x00,0x02,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x02,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_format_x v5, off, s[8:11], s101 offset:8388607 ; encoding:  [0x65,0x00,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x00,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_format_x v5, off, s[8:11], m0 offset:8388607 ; encoding:  [0x7d,0x00,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x00,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_format_x v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding:  [0x03,0x00,0x02,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x02,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_format_x v5, v0, s[8:11], s3 offen offset:8388607 ; encoding:  [0x03,0x00,0x02,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x02,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_format_x v5, off, s[8:11], s3 ; encoding:  [0x03,0x00,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x00,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_load_d16_format_x v5, off, s[8:11], s3 ; encoding:  [0x03,0x00,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x00,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_load_d16_format_x v5, off, s[8:11], s3 offset:7 ; encoding:  [0x03,0x00,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x00,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_load_d16_format_x v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV ; encoding:  [0x03,0x00,0x02,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x02,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_format_x v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS ; encoding:  [0x03,0x00,0x02,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x02,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_format_xy v5, off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x40,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_format_xy v255, off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x40,0x02,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x02,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_format_xy v5, off, s[12:15], s3 offset:8388607 ; encoding:  [0x03,0x40,0x02,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x02,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_format_xy v5, off, s[96:99], s3 offset:8388607 ; encoding:  [0x03,0x40,0x02,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x02,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_format_xy v5, off, s[8:11], s101 offset:8388607 ; encoding:  [0x65,0x40,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x40,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_format_xy v5, off, s[8:11], m0 offset:8388607 ; encoding:  [0x7d,0x40,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x40,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_format_xy v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding:  [0x03,0x40,0x02,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x02,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_format_xy v5, v0, s[8:11], s3 offen offset:8388607 ; encoding:  [0x03,0x40,0x02,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x02,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_format_xy v5, off, s[8:11], s3 ; encoding:  [0x03,0x40,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x40,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_load_d16_format_xy v5, off, s[8:11], s3 ; encoding:  [0x03,0x40,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x40,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_load_d16_format_xy v5, off, s[8:11], s3 offset:7 ; encoding:  [0x03,0x40,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x40,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_load_d16_format_xy v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV ; encoding:  [0x03,0x40,0x02,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x02,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_format_xy v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS ; encoding:  [0x03,0x40,0x02,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x02,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_format_xyz v[5:6], off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x80,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_format_xyz v[254:255], off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x80,0x02,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x02,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_format_xyz v[5:6], off, s[12:15], s3 offset:8388607 ; encoding:  [0x03,0x80,0x02,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x02,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_format_xyz v[5:6], off, s[96:99], s3 offset:8388607 ; encoding:  [0x03,0x80,0x02,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x02,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_format_xyz v[5:6], off, s[8:11], s101 offset:8388607 ; encoding:  [0x65,0x80,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x80,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_format_xyz v[5:6], off, s[8:11], m0 offset:8388607 ; encoding:  [0x7d,0x80,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x80,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_format_xyz v[5:6], v0, s[8:11], s3 idxen offset:8388607 ; encoding:  [0x03,0x80,0x02,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x02,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_format_xyz v[5:6], v0, s[8:11], s3 offen offset:8388607 ; encoding:  [0x03,0x80,0x02,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x02,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_format_xyz v[5:6], off, s[8:11], s3 ; encoding:  [0x03,0x80,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x80,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_load_d16_format_xyz v[5:6], off, s[8:11], s3 ; encoding:  [0x03,0x80,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x80,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_load_d16_format_xyz v[5:6], off, s[8:11], s3 offset:7 ; encoding:  [0x03,0x80,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x80,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_load_d16_format_xyz v[5:6], off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV ; encoding:  [0x03,0x80,0x02,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x02,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_format_xyz v[5:6], off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS ; encoding:  [0x03,0x80,0x02,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x02,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_format_xyzw v[5:6], off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0xc0,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_format_xyzw v[254:255], off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0xc0,0x02,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x02,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_format_xyzw v[5:6], off, s[12:15], s3 offset:8388607 ; encoding:  [0x03,0xc0,0x02,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x02,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_format_xyzw v[5:6], off, s[96:99], s3 offset:8388607 ; encoding:  [0x03,0xc0,0x02,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x02,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_format_xyzw v[5:6], off, s[8:11], s101 offset:8388607 ; encoding:  [0x65,0xc0,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0xc0,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_format_xyzw v[5:6], off, s[8:11], m0 offset:8388607 ; encoding:  [0x7d,0xc0,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0xc0,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_format_xyzw v[5:6], v0, s[8:11], s3 idxen offset:8388607 ; encoding:  [0x03,0xc0,0x02,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x02,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_format_xyzw v[5:6], v0, s[8:11], s3 offen offset:8388607 ; encoding:  [0x03,0xc0,0x02,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x02,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_format_xyzw v[5:6], off, s[8:11], s3 ; encoding:  [0x03,0xc0,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0xc0,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_load_d16_format_xyzw v[5:6], off, s[8:11], s3 ; encoding:  [0x03,0xc0,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0xc0,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_load_d16_format_xyzw v[5:6], off, s[8:11], s3 offset:7 ; encoding:  [0x03,0xc0,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0xc0,0x02,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_load_d16_format_xyzw v[5:6], off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV ; encoding:  [0x03,0xc0,0x02,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x02,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_format_xyzw v[5:6], off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS ; encoding:  [0x03,0xc0,0x02,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x02,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_hi_b16 v5, off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0xc0,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_hi_b16 v255, off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0xc0,0x08,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x08,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_hi_b16 v5, off, s[12:15], s3 offset:8388607 ; encoding:  [0x03,0xc0,0x08,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x08,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_hi_b16 v5, off, s[96:99], s3 offset:8388607 ; encoding:  [0x03,0xc0,0x08,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x08,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_hi_b16 v5, off, s[8:11], s101 offset:8388607 ; encoding:  [0x65,0xc0,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0xc0,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_hi_b16 v5, off, s[8:11], m0 offset:8388607 ; encoding:  [0x7d,0xc0,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0xc0,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_hi_b16 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding:  [0x03,0xc0,0x08,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x08,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_hi_b16 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding:  [0x03,0xc0,0x08,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x08,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_hi_b16 v5, off, s[8:11], s3 ; encoding:  [0x03,0xc0,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0xc0,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_load_d16_hi_b16 v5, off, s[8:11], s3 ; encoding:  [0x03,0xc0,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0xc0,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_load_d16_hi_b16 v5, off, s[8:11], s3 offset:7 ; encoding:  [0x03,0xc0,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0xc0,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_load_d16_hi_b16 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV ; encoding:  [0x03,0xc0,0x08,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x08,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_hi_b16 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS ; encoding:  [0x03,0xc0,0x08,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x08,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_hi_format_x v5, off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x80,0x09,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x09,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_hi_format_x v255, off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x80,0x09,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x09,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_hi_format_x v5, off, s[12:15], s3 offset:8388607 ; encoding:  [0x03,0x80,0x09,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x09,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_hi_format_x v5, off, s[96:99], s3 offset:8388607 ; encoding:  [0x03,0x80,0x09,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x09,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_hi_format_x v5, off, s[8:11], s101 offset:8388607 ; encoding:  [0x65,0x80,0x09,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x80,0x09,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_hi_format_x v5, off, s[8:11], m0 offset:8388607 ; encoding:  [0x7d,0x80,0x09,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x80,0x09,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_hi_format_x v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding:  [0x03,0x80,0x09,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x09,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_hi_format_x v5, v0, s[8:11], s3 offen offset:8388607 ; encoding:  [0x03,0x80,0x09,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x09,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_hi_format_x v5, off, s[8:11], s3 ; encoding:  [0x03,0x80,0x09,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x80,0x09,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_load_d16_hi_format_x v5, off, s[8:11], s3 ; encoding:  [0x03,0x80,0x09,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x80,0x09,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_load_d16_hi_format_x v5, off, s[8:11], s3 offset:7 ; encoding:  [0x03,0x80,0x09,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x80,0x09,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_load_d16_hi_format_x v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV ; encoding:  [0x03,0x80,0x09,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x09,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_hi_format_x v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS ; encoding:  [0x03,0x80,0x09,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x09,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_hi_i8 v5, off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x80,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_hi_i8 v255, off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x80,0x08,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x08,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_hi_i8 v5, off, s[12:15], s3 offset:8388607 ; encoding:  [0x03,0x80,0x08,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x08,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_hi_i8 v5, off, s[96:99], s3 offset:8388607 ; encoding:  [0x03,0x80,0x08,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x08,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_hi_i8 v5, off, s[8:11], s101 offset:8388607 ; encoding:  [0x65,0x80,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x80,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_hi_i8 v5, off, s[8:11], m0 offset:8388607 ; encoding:  [0x7d,0x80,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x80,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_hi_i8 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding:  [0x03,0x80,0x08,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x08,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_hi_i8 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding:  [0x03,0x80,0x08,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x08,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_hi_i8 v5, off, s[8:11], s3 ; encoding:  [0x03,0x80,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x80,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_load_d16_hi_i8 v5, off, s[8:11], s3 ; encoding:  [0x03,0x80,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x80,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_load_d16_hi_i8 v5, off, s[8:11], s3 offset:7 ; encoding:  [0x03,0x80,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x80,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_load_d16_hi_i8 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV ; encoding:  [0x03,0x80,0x08,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x08,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_hi_i8 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS ; encoding:  [0x03,0x80,0x08,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x08,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_hi_u8 v5, off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x40,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_hi_u8 v255, off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x40,0x08,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x08,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_hi_u8 v5, off, s[12:15], s3 offset:8388607 ; encoding:  [0x03,0x40,0x08,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x08,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_hi_u8 v5, off, s[96:99], s3 offset:8388607 ; encoding:  [0x03,0x40,0x08,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x08,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_hi_u8 v5, off, s[8:11], s101 offset:8388607 ; encoding:  [0x65,0x40,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x40,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_hi_u8 v5, off, s[8:11], m0 offset:8388607 ; encoding:  [0x7d,0x40,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x40,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_hi_u8 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding:  [0x03,0x40,0x08,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x08,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_hi_u8 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding:  [0x03,0x40,0x08,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x08,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_hi_u8 v5, off, s[8:11], s3 ; encoding:  [0x03,0x40,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x40,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_load_d16_hi_u8 v5, off, s[8:11], s3 ; encoding:  [0x03,0x40,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x40,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_load_d16_hi_u8 v5, off, s[8:11], s3 offset:7 ; encoding:  [0x03,0x40,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x40,0x08,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_load_d16_hi_u8 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV ; encoding:  [0x03,0x40,0x08,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x08,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_hi_u8 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS ; encoding:  [0x03,0x40,0x08,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x08,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_i8 v5, off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0xc0,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_i8 v255, off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0xc0,0x07,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x07,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_i8 v5, off, s[12:15], s3 offset:8388607 ; encoding:  [0x03,0xc0,0x07,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x07,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_i8 v5, off, s[96:99], s3 offset:8388607 ; encoding:  [0x03,0xc0,0x07,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x07,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_i8 v5, off, s[8:11], s101 offset:8388607 ; encoding:  [0x65,0xc0,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0xc0,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_i8 v5, off, s[8:11], m0 offset:8388607 ; encoding:  [0x7d,0xc0,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0xc0,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_i8 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding:  [0x03,0xc0,0x07,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x07,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_i8 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding:  [0x03,0xc0,0x07,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x07,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_i8 v5, off, s[8:11], s3 ; encoding:  [0x03,0xc0,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0xc0,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_load_d16_i8 v5, off, s[8:11], s3 ; encoding:  [0x03,0xc0,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0xc0,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_load_d16_i8 v5, off, s[8:11], s3 offset:7 ; encoding:  [0x03,0xc0,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0xc0,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_load_d16_i8 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV ; encoding:  [0x03,0xc0,0x07,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x07,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_i8 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS ; encoding:  [0x03,0xc0,0x07,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x07,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_u8 v5, off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x80,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_u8 v255, off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x80,0x07,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x07,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_u8 v5, off, s[12:15], s3 offset:8388607 ; encoding:  [0x03,0x80,0x07,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x07,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_u8 v5, off, s[96:99], s3 offset:8388607 ; encoding:  [0x03,0x80,0x07,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x07,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_u8 v5, off, s[8:11], s101 offset:8388607 ; encoding:  [0x65,0x80,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x80,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_u8 v5, off, s[8:11], m0 offset:8388607 ; encoding:  [0x7d,0x80,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x80,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_u8 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding:  [0x03,0x80,0x07,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x07,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_u8 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding:  [0x03,0x80,0x07,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x07,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_u8 v5, off, s[8:11], s3 ; encoding:  [0x03,0x80,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x80,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_load_d16_u8 v5, off, s[8:11], s3 ; encoding:  [0x03,0x80,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x80,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_load_d16_u8 v5, off, s[8:11], s3 offset:7 ; encoding:  [0x03,0x80,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x80,0x07,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_load_d16_u8 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV ; encoding:  [0x03,0x80,0x07,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x07,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_d16_u8 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS ; encoding:  [0x03,0x80,0x07,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x07,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_format_x v5, off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x00,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_format_x v255, off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x00,0x00,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x00,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_format_x v5, off, s[12:15], s3 offset:8388607 ; encoding:  [0x03,0x00,0x00,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x00,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_format_x v5, off, s[96:99], s3 offset:8388607 ; encoding:  [0x03,0x00,0x00,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x00,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_format_x v5, off, s[8:11], s101 offset:8388607 ; encoding:  [0x65,0x00,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x00,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_format_x v5, off, s[8:11], m0 offset:8388607 ; encoding:  [0x7d,0x00,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x00,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_format_x v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding:  [0x03,0x00,0x00,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x00,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_format_x v5, v0, s[8:11], s3 offen offset:8388607 ; encoding:  [0x03,0x00,0x00,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x00,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_format_x v5, off, s[8:11], s3 ; encoding:  [0x03,0x00,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x00,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_load_format_x v5, off, s[8:11], s3 ; encoding:  [0x03,0x00,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x00,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_load_format_x v5, off, s[8:11], s3 offset:7 ; encoding:  [0x03,0x00,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x00,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_load_format_x v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV ; encoding:  [0x03,0x00,0x00,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x00,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_format_x v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS ; encoding:  [0x03,0x00,0x00,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x00,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_format_xy v[5:6], off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x40,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_format_xy v[254:255], off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x40,0x00,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x00,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_format_xy v[5:6], off, s[12:15], s3 offset:8388607 ; encoding:  [0x03,0x40,0x00,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x00,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_format_xy v[5:6], off, s[96:99], s3 offset:8388607 ; encoding:  [0x03,0x40,0x00,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x00,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_format_xy v[5:6], off, s[8:11], s101 offset:8388607 ; encoding:  [0x65,0x40,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x40,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_format_xy v[5:6], off, s[8:11], m0 offset:8388607 ; encoding:  [0x7d,0x40,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x40,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_format_xy v[5:6], v0, s[8:11], s3 idxen offset:8388607 ; encoding:  [0x03,0x40,0x00,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x00,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_format_xy v[5:6], v0, s[8:11], s3 offen offset:8388607 ; encoding:  [0x03,0x40,0x00,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x00,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_format_xy v[5:6], off, s[8:11], s3 ; encoding:  [0x03,0x40,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x40,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_load_format_xy v[5:6], off, s[8:11], s3 ; encoding:  [0x03,0x40,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x40,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_load_format_xy v[5:6], off, s[8:11], s3 offset:7 ; encoding:  [0x03,0x40,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x40,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_load_format_xy v[5:6], off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV ; encoding:  [0x03,0x40,0x00,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x00,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_format_xy v[5:6], off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS ; encoding:  [0x03,0x40,0x00,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x00,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_format_xyz v[5:7], off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x80,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_format_xyz v[253:255], off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x80,0x00,0xc4,0xfd,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x00,0xc4,0xfd,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_format_xyz v[5:7], off, s[12:15], s3 offset:8388607 ; encoding:  [0x03,0x80,0x00,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x00,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_format_xyz v[5:7], off, s[96:99], s3 offset:8388607 ; encoding:  [0x03,0x80,0x00,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x00,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_format_xyz v[5:7], off, s[8:11], s101 offset:8388607 ; encoding:  [0x65,0x80,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x80,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_format_xyz v[5:7], off, s[8:11], m0 offset:8388607 ; encoding:  [0x7d,0x80,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x80,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_format_xyz v[5:7], v0, s[8:11], s3 idxen offset:8388607 ; encoding:  [0x03,0x80,0x00,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x00,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_format_xyz v[5:7], v0, s[8:11], s3 offen offset:8388607 ; encoding:  [0x03,0x80,0x00,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x00,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_format_xyz v[5:7], off, s[8:11], s3 ; encoding:  [0x03,0x80,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x80,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_load_format_xyz v[5:7], off, s[8:11], s3 ; encoding:  [0x03,0x80,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x80,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_load_format_xyz v[5:7], off, s[8:11], s3 offset:7 ; encoding:  [0x03,0x80,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x80,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_load_format_xyz v[5:7], off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV ; encoding:  [0x03,0x80,0x00,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x00,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_format_xyz v[5:7], off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS ; encoding:  [0x03,0x80,0x00,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x00,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_format_xyzw v[5:8], off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0xc0,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_format_xyzw v[252:255], off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0xc0,0x00,0xc4,0xfc,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x00,0xc4,0xfc,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_format_xyzw v[5:8], off, s[12:15], s3 offset:8388607 ; encoding:  [0x03,0xc0,0x00,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x00,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_format_xyzw v[5:8], off, s[96:99], s3 offset:8388607 ; encoding:  [0x03,0xc0,0x00,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x00,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_format_xyzw v[5:8], off, s[8:11], s101 offset:8388607 ; encoding:  [0x65,0xc0,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0xc0,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_format_xyzw v[5:8], off, s[8:11], m0 offset:8388607 ; encoding:  [0x7d,0xc0,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0xc0,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_format_xyzw v[5:8], v0, s[8:11], s3 idxen offset:8388607 ; encoding:  [0x03,0xc0,0x00,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x00,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_format_xyzw v[5:8], v0, s[8:11], s3 offen offset:8388607 ; encoding:  [0x03,0xc0,0x00,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x00,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_format_xyzw v[5:8], off, s[8:11], s3 ; encoding:  [0x03,0xc0,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0xc0,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_load_format_xyzw v[5:8], off, s[8:11], s3 ; encoding:  [0x03,0xc0,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0xc0,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_load_format_xyzw v[5:8], off, s[8:11], s3 offset:7 ; encoding:  [0x03,0xc0,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0xc0,0x00,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_load_format_xyzw v[5:8], off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV ; encoding:  [0x03,0xc0,0x00,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x00,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_format_xyzw v[5:8], off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS ; encoding:  [0x03,0xc0,0x00,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x00,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_i8 v5, off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x40,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_i8 v255, off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x40,0x04,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x04,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_i8 v5, off, s[12:15], s3 offset:8388607 ; encoding:  [0x03,0x40,0x04,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x04,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_i8 v5, off, s[96:99], s3 offset:8388607 ; encoding:  [0x03,0x40,0x04,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x04,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_i8 v5, off, s[8:11], s101 offset:8388607 ; encoding:  [0x65,0x40,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x40,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_i8 v5, off, s[8:11], m0 offset:8388607 ; encoding:  [0x7d,0x40,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x40,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_i8 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding:  [0x03,0x40,0x04,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x04,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_i8 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding:  [0x03,0x40,0x04,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x04,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_i8 v5, off, s[8:11], s3 ; encoding:  [0x03,0x40,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x40,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_load_i8 v5, off, s[8:11], s3 ; encoding:  [0x03,0x40,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x40,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_load_i8 v5, off, s[8:11], s3 offset:7 ; encoding:  [0x03,0x40,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x40,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_load_i8 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV ; encoding:  [0x03,0x40,0x04,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x04,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_i8 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS ; encoding:  [0x03,0x40,0x04,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x04,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_i16 v5, off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0xc0,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_i16 v255, off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0xc0,0x04,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x04,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_i16 v5, off, s[12:15], s3 offset:8388607 ; encoding:  [0x03,0xc0,0x04,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x04,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_i16 v5, off, s[96:99], s3 offset:8388607 ; encoding:  [0x03,0xc0,0x04,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x04,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_i16 v5, off, s[8:11], s101 offset:8388607 ; encoding:  [0x65,0xc0,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0xc0,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_i16 v5, off, s[8:11], m0 offset:8388607 ; encoding:  [0x7d,0xc0,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0xc0,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_i16 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding:  [0x03,0xc0,0x04,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x04,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_i16 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding:  [0x03,0xc0,0x04,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x04,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_i16 v5, off, s[8:11], s3 ; encoding:  [0x03,0xc0,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0xc0,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_load_i16 v5, off, s[8:11], s3 ; encoding:  [0x03,0xc0,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0xc0,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_load_i16 v5, off, s[8:11], s3 offset:7 ; encoding:  [0x03,0xc0,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0xc0,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_load_i16 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV ; encoding:  [0x03,0xc0,0x04,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x04,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_i16 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS ; encoding:  [0x03,0xc0,0x04,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x04,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_u8 v5, off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x00,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_u8 v255, off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x00,0x04,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x04,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_u8 v5, off, s[12:15], s3 offset:8388607 ; encoding:  [0x03,0x00,0x04,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x04,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_u8 v5, off, s[96:99], s3 offset:8388607 ; encoding:  [0x03,0x00,0x04,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x04,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_u8 v5, off, s[8:11], s101 offset:8388607 ; encoding:  [0x65,0x00,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x00,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_u8 v5, off, s[8:11], m0 offset:8388607 ; encoding:  [0x7d,0x00,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x00,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_u8 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding:  [0x03,0x00,0x04,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x04,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_u8 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding:  [0x03,0x00,0x04,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x04,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_u8 v5, off, s[8:11], s3 ; encoding:  [0x03,0x00,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x00,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_load_u8 v5, off, s[8:11], s3 ; encoding:  [0x03,0x00,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x00,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_load_u8 v5, off, s[8:11], s3 offset:7 ; encoding:  [0x03,0x00,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x00,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_load_u8 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV ; encoding:  [0x03,0x00,0x04,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x04,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_u8 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS ; encoding:  [0x03,0x00,0x04,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x04,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_u16 v5, off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x80,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_u16 v255, off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x80,0x04,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x04,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_u16 v5, off, s[12:15], s3 offset:8388607 ; encoding:  [0x03,0x80,0x04,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x04,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_u16 v5, off, s[96:99], s3 offset:8388607 ; encoding:  [0x03,0x80,0x04,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x04,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_u16 v5, off, s[8:11], s101 offset:8388607 ; encoding:  [0x65,0x80,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x80,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_u16 v5, off, s[8:11], m0 offset:8388607 ; encoding:  [0x7d,0x80,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x80,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_u16 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding:  [0x03,0x80,0x04,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x04,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_u16 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding:  [0x03,0x80,0x04,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x04,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_u16 v5, off, s[8:11], s3 ; encoding:  [0x03,0x80,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x80,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_load_u16 v5, off, s[8:11], s3 ; encoding:  [0x03,0x80,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x80,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_load_u16 v5, off, s[8:11], s3 offset:7 ; encoding:  [0x03,0x80,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x80,0x04,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_load_u16 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_NT_HT scope:SCOPE_DEV ; encoding:  [0x03,0x80,0x04,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x04,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_load_u16 v5, off, s[8:11], s3 offset:8388607 th:TH_LOAD_BYPASS scope:SCOPE_SYS ; encoding:  [0x03,0x80,0x04,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x04,0xc4,0x05,0x10,0xbc,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_b8 v1, off, s[12:15], s4 offset:8388607 ; encoding:  [0x04,0x00,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x00,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_b8 v255, off, s[12:15], s4 offset:8388607 ; encoding:  [0x04,0x00,0x06,0xc4,0xff,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x00,0x06,0xc4,0xff,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_b8 v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV ; encoding:  [0x04,0x00,0x06,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x00,0x06,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_b8 v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS ; encoding:  [0x04,0x00,0x06,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x00,0x06,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_b8 v1, off, s[16:19], s4 offset:8388607 ; encoding:  [0x04,0x00,0x06,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x00,0x06,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_b8 v1, off, s[96:99], s4 offset:8388607 ; encoding:  [0x04,0x00,0x06,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x00,0x06,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_b8 v1, off, s[12:15], s101 offset:8388607 ; encoding:  [0x65,0x00,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x00,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_b8 v1, off, s[12:15], m0 offset:8388607 ; encoding:  [0x7d,0x00,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x00,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_b8 v1, v0, s[12:15], s4 idxen offset:8388607 ; encoding:  [0x04,0x00,0x06,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x04,0x00,0x06,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_b8 v1, v0, s[12:15], s4 offen offset:8388607 ; encoding:  [0x04,0x00,0x06,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x04,0x00,0x06,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_b8 v1, off, s[12:15], s4 ; encoding:  [0x04,0x00,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+0x04,0x00,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_store_b8 v1, off, s[12:15], s4 ; encoding:  [0x04,0x00,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+0x04,0x00,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_store_b8 v1, off, s[12:15], s4 offset:7 ; encoding:  [0x04,0x00,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00]
+0x04,0x00,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_store_b16 v1, off, s[12:15], s4 offset:8388607 ; encoding:  [0x04,0x40,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x40,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_b16 v255, off, s[12:15], s4 offset:8388607 ; encoding:  [0x04,0x40,0x06,0xc4,0xff,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x40,0x06,0xc4,0xff,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_b16 v1, off, s[16:19], s4 offset:8388607 ; encoding:  [0x04,0x40,0x06,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x40,0x06,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_b16 v1, off, s[96:99], s4 offset:8388607 ; encoding:  [0x04,0x40,0x06,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x40,0x06,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_b16 v1, off, s[12:15], s101 offset:8388607 ; encoding:  [0x65,0x40,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x40,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_b16 v1, off, s[12:15], m0 offset:8388607 ; encoding:  [0x7d,0x40,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x40,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_b16 v1, v0, s[12:15], s4 idxen offset:8388607 ; encoding:  [0x04,0x40,0x06,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x04,0x40,0x06,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_b16 v1, v0, s[12:15], s4 offen offset:8388607 ; encoding:  [0x04,0x40,0x06,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x04,0x40,0x06,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_b16 v1, off, s[12:15], s4 ; encoding:  [0x04,0x40,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+0x04,0x40,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_store_b16 v1, off, s[12:15], s4 ; encoding:  [0x04,0x40,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+0x04,0x40,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_store_b16 v1, off, s[12:15], s4 offset:7 ; encoding:  [0x04,0x40,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00]
+0x04,0x40,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_store_b16 v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV ; encoding:  [0x04,0x40,0x06,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x40,0x06,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_b16 v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS ; encoding:  [0x04,0x40,0x06,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x40,0x06,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_b32 v1, off, s[12:15], s4 offset:8388607 ; encoding:  [0x04,0x80,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x80,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_b32 v255, off, s[12:15], s4 offset:8388607 ; encoding:  [0x04,0x80,0x06,0xc4,0xff,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x80,0x06,0xc4,0xff,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_b32 v1, off, s[16:19], s4 offset:8388607 ; encoding:  [0x04,0x80,0x06,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x80,0x06,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_b32 v1, off, s[96:99], s4 offset:8388607 ; encoding:  [0x04,0x80,0x06,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x80,0x06,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_b32 v1, off, s[12:15], s101 offset:8388607 ; encoding:  [0x65,0x80,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x80,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_b32 v1, off, s[12:15], m0 offset:8388607 ; encoding:  [0x7d,0x80,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x80,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_b32 v1, v0, s[12:15], s4 idxen offset:8388607 ; encoding:  [0x04,0x80,0x06,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x04,0x80,0x06,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_b32 v1, v0, s[12:15], s4 offen offset:8388607 ; encoding:  [0x04,0x80,0x06,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x04,0x80,0x06,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_b32 v1, off, s[12:15], s4 ; encoding:  [0x04,0x80,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+0x04,0x80,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_store_b32 v1, off, s[12:15], s4 ; encoding:  [0x04,0x80,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+0x04,0x80,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_store_b32 v1, off, s[12:15], s4 offset:7 ; encoding:  [0x04,0x80,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00]
+0x04,0x80,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_store_b32 v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV ; encoding:  [0x04,0x80,0x06,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x80,0x06,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_b32 v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS ; encoding:  [0x04,0x80,0x06,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x80,0x06,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_b64 v[1:2], off, s[12:15], s4 offset:8388607 ; encoding:  [0x04,0xc0,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0xc0,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_b64 v[254:255], off, s[12:15], s4 offset:8388607 ; encoding:  [0x04,0xc0,0x06,0xc4,0xfe,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0xc0,0x06,0xc4,0xfe,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_b64 v[1:2], off, s[16:19], s4 offset:8388607 ; encoding:  [0x04,0xc0,0x06,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0xc0,0x06,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_b64 v[1:2], off, s[96:99], s4 offset:8388607 ; encoding:  [0x04,0xc0,0x06,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0xc0,0x06,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_b64 v[1:2], off, s[12:15], s101 offset:8388607 ; encoding:  [0x65,0xc0,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0xc0,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_b64 v[1:2], off, s[12:15], m0 offset:8388607 ; encoding:  [0x7d,0xc0,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0xc0,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_b64 v[1:2], v0, s[12:15], s4 idxen offset:8388607 ; encoding:  [0x04,0xc0,0x06,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x04,0xc0,0x06,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_b64 v[1:2], v0, s[12:15], s4 offen offset:8388607 ; encoding:  [0x04,0xc0,0x06,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x04,0xc0,0x06,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_b64 v[1:2], off, s[12:15], s4 ; encoding:  [0x04,0xc0,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+0x04,0xc0,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_store_b64 v[1:2], off, s[12:15], s4 ; encoding:  [0x04,0xc0,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+0x04,0xc0,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_store_b64 v[1:2], off, s[12:15], s4 offset:7 ; encoding:  [0x04,0xc0,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00]
+0x04,0xc0,0x06,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_store_b64 v[1:2], off, s[12:15], s4 offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV ; encoding:  [0x04,0xc0,0x06,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x04,0xc0,0x06,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_b64 v[1:2], off, s[12:15], s4 offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS ; encoding:  [0x04,0xc0,0x06,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f]
+0x04,0xc0,0x06,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_b96 v[1:3], off, s[12:15], s4 offset:8388607 ; encoding:  [0x04,0x00,0x07,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x00,0x07,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_b96 v[253:255], off, s[12:15], s4 offset:8388607 ; encoding:  [0x04,0x00,0x07,0xc4,0xfd,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x00,0x07,0xc4,0xfd,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_b96 v[1:3], off, s[16:19], s4 offset:8388607 ; encoding:  [0x04,0x00,0x07,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x00,0x07,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_b96 v[1:3], off, s[96:99], s4 offset:8388607 ; encoding:  [0x04,0x00,0x07,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x00,0x07,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_b96 v[1:3], off, s[12:15], s101 offset:8388607 ; encoding:  [0x65,0x00,0x07,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x00,0x07,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_b96 v[1:3], off, s[12:15], m0 offset:8388607 ; encoding:  [0x7d,0x00,0x07,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x00,0x07,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_b96 v[1:3], v0, s[12:15], s4 idxen offset:8388607 ; encoding:  [0x04,0x00,0x07,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x04,0x00,0x07,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_b96 v[1:3], v0, s[12:15], s4 offen offset:8388607 ; encoding:  [0x04,0x00,0x07,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x04,0x00,0x07,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_b96 v[1:3], off, s[12:15], s4 ; encoding:  [0x04,0x00,0x07,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+0x04,0x00,0x07,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_store_b96 v[1:3], off, s[12:15], s4 ; encoding:  [0x04,0x00,0x07,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+0x04,0x00,0x07,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_store_b96 v[1:3], off, s[12:15], s4 offset:7 ; encoding:  [0x04,0x00,0x07,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00]
+0x04,0x00,0x07,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_store_b96 v[1:3], off, s[12:15], s4 offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV ; encoding:  [0x04,0x00,0x07,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x00,0x07,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_b96 v[1:3], off, s[12:15], s4 offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS ; encoding:  [0x04,0x00,0x07,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x00,0x07,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_b128 v[1:4], off, s[12:15], s4 offset:8388607 ; encoding:  [0x04,0x40,0x07,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x40,0x07,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_b128 v[252:255], off, s[12:15], s4 offset:8388607 ; encoding:  [0x04,0x40,0x07,0xc4,0xfc,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x40,0x07,0xc4,0xfc,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_b128 v[1:4], off, s[16:19], s4 offset:8388607 ; encoding:  [0x04,0x40,0x07,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x40,0x07,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_b128 v[1:4], off, s[96:99], s4 offset:8388607 ; encoding:  [0x04,0x40,0x07,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x40,0x07,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_b128 v[1:4], off, s[12:15], s101 offset:8388607 ; encoding:  [0x65,0x40,0x07,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x40,0x07,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_b128 v[1:4], off, s[12:15], m0 offset:8388607 ; encoding:  [0x7d,0x40,0x07,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x40,0x07,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_b128 v[1:4], v0, s[12:15], s4 idxen offset:8388607 ; encoding:  [0x04,0x40,0x07,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x04,0x40,0x07,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_b128 v[1:4], v0, s[12:15], s4 offen offset:8388607 ; encoding:  [0x04,0x40,0x07,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x04,0x40,0x07,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_b128 v[1:4], off, s[12:15], s4 ; encoding:  [0x04,0x40,0x07,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+0x04,0x40,0x07,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_store_b128 v[1:4], off, s[12:15], s4 ; encoding:  [0x04,0x40,0x07,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+0x04,0x40,0x07,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_store_b128 v[1:4], off, s[12:15], s4 offset:7 ; encoding:  [0x04,0x40,0x07,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00]
+0x04,0x40,0x07,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_store_b128 v[1:4], off, s[12:15], s4 offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV ; encoding:  [0x04,0x40,0x07,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x40,0x07,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_b128 v[1:4], off, s[12:15], s4 offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS ; encoding:  [0x04,0x40,0x07,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x40,0x07,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_format_x v1, off, s[12:15], s4 offset:8388607 ; encoding:  [0x04,0x00,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x00,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_format_x v255, off, s[12:15], s4 offset:8388607 ; encoding:  [0x04,0x00,0x03,0xc4,0xff,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x00,0x03,0xc4,0xff,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_format_x v1, off, s[16:19], s4 offset:8388607 ; encoding:  [0x04,0x00,0x03,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x00,0x03,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_format_x v1, off, s[96:99], s4 offset:8388607 ; encoding:  [0x04,0x00,0x03,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x00,0x03,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_format_x v1, off, s[12:15], s101 offset:8388607 ; encoding:  [0x65,0x00,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x00,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_format_x v1, off, s[12:15], m0 offset:8388607 ; encoding:  [0x7d,0x00,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x00,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_format_x v1, v0, s[12:15], s4 idxen offset:8388607 ; encoding:  [0x04,0x00,0x03,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x04,0x00,0x03,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_format_x v1, v0, s[12:15], s4 offen offset:8388607 ; encoding:  [0x04,0x00,0x03,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x04,0x00,0x03,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_format_x v1, off, s[12:15], s4 ; encoding:  [0x04,0x00,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+0x04,0x00,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_store_d16_format_x v1, off, s[12:15], s4 ; encoding:  [0x04,0x00,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+0x04,0x00,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_store_d16_format_x v1, off, s[12:15], s4 offset:7 ; encoding:  [0x04,0x00,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00]
+0x04,0x00,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_store_d16_format_x v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV ; encoding:  [0x04,0x00,0x03,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x00,0x03,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_format_x v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS ; encoding:  [0x04,0x00,0x03,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x00,0x03,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_format_xy v1, off, s[12:15], s4 offset:8388607 ; encoding:  [0x04,0x40,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x40,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_format_xy v255, off, s[12:15], s4 offset:8388607 ; encoding:  [0x04,0x40,0x03,0xc4,0xff,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x40,0x03,0xc4,0xff,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_format_xy v1, off, s[16:19], s4 offset:8388607 ; encoding:  [0x04,0x40,0x03,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x40,0x03,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_format_xy v1, off, s[96:99], s4 offset:8388607 ; encoding:  [0x04,0x40,0x03,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x40,0x03,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_format_xy v1, off, s[12:15], s101 offset:8388607 ; encoding:  [0x65,0x40,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x40,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_format_xy v1, off, s[12:15], m0 offset:8388607 ; encoding:  [0x7d,0x40,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x40,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_format_xy v1, v0, s[12:15], s4 idxen offset:8388607 ; encoding:  [0x04,0x40,0x03,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x04,0x40,0x03,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_format_xy v1, v0, s[12:15], s4 offen offset:8388607 ; encoding:  [0x04,0x40,0x03,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x04,0x40,0x03,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_format_xy v1, off, s[12:15], s4 ; encoding:  [0x04,0x40,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+0x04,0x40,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_store_d16_format_xy v1, off, s[12:15], s4 ; encoding:  [0x04,0x40,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+0x04,0x40,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_store_d16_format_xy v1, off, s[12:15], s4 offset:7 ; encoding:  [0x04,0x40,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00]
+0x04,0x40,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_store_d16_format_xy v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV ; encoding:  [0x04,0x40,0x03,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x40,0x03,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_format_xy v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS ; encoding:  [0x04,0x40,0x03,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x40,0x03,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_format_xyz v[1:2], off, s[12:15], s4 offset:8388607 ; encoding:  [0x04,0x80,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x80,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_format_xyz v[254:255], off, s[12:15], s4 offset:8388607 ; encoding:  [0x04,0x80,0x03,0xc4,0xfe,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x80,0x03,0xc4,0xfe,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_format_xyz v[1:2], off, s[16:19], s4 offset:8388607 ; encoding:  [0x04,0x80,0x03,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x80,0x03,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_format_xyz v[1:2], off, s[96:99], s4 offset:8388607 ; encoding:  [0x04,0x80,0x03,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x80,0x03,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_format_xyz v[1:2], off, s[12:15], s101 offset:8388607 ; encoding:  [0x65,0x80,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x80,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_format_xyz v[1:2], off, s[12:15], m0 offset:8388607 ; encoding:  [0x7d,0x80,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x80,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_format_xyz v[1:2], v0, s[12:15], s4 idxen offset:8388607 ; encoding:  [0x04,0x80,0x03,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x04,0x80,0x03,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_format_xyz v[1:2], v0, s[12:15], s4 offen offset:8388607 ; encoding:  [0x04,0x80,0x03,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x04,0x80,0x03,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_format_xyz v[1:2], off, s[12:15], s4 ; encoding:  [0x04,0x80,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+0x04,0x80,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_store_d16_format_xyz v[1:2], off, s[12:15], s4 ; encoding:  [0x04,0x80,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+0x04,0x80,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_store_d16_format_xyz v[1:2], off, s[12:15], s4 offset:7 ; encoding:  [0x04,0x80,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00]
+0x04,0x80,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_store_d16_format_xyz v[1:2], off, s[12:15], s4 offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV ; encoding:  [0x04,0x80,0x03,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x80,0x03,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_format_xyz v[1:2], off, s[12:15], s4 offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS ; encoding:  [0x04,0x80,0x03,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x80,0x03,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_format_xyzw v[1:2], off, s[12:15], s4 offset:8388607 ; encoding:  [0x04,0xc0,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0xc0,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_format_xyzw v[254:255], off, s[12:15], s4 offset:8388607 ; encoding:  [0x04,0xc0,0x03,0xc4,0xfe,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0xc0,0x03,0xc4,0xfe,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_format_xyzw v[1:2], off, s[16:19], s4 offset:8388607 ; encoding:  [0x04,0xc0,0x03,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0xc0,0x03,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_format_xyzw v[1:2], off, s[96:99], s4 offset:8388607 ; encoding:  [0x04,0xc0,0x03,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0xc0,0x03,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_format_xyzw v[1:2], off, s[12:15], s101 offset:8388607 ; encoding:  [0x65,0xc0,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0xc0,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_format_xyzw v[1:2], off, s[12:15], m0 offset:8388607 ; encoding:  [0x7d,0xc0,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0xc0,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_format_xyzw v[1:2], v0, s[12:15], s4 idxen offset:8388607 ; encoding:  [0x04,0xc0,0x03,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x04,0xc0,0x03,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_format_xyzw v[1:2], v0, s[12:15], s4 offen offset:8388607 ; encoding:  [0x04,0xc0,0x03,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x04,0xc0,0x03,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_format_xyzw v[1:2], off, s[12:15], s4 ; encoding:  [0x04,0xc0,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+0x04,0xc0,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_store_d16_format_xyzw v[1:2], off, s[12:15], s4 ; encoding:  [0x04,0xc0,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+0x04,0xc0,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_store_d16_format_xyzw v[1:2], off, s[12:15], s4 offset:7 ; encoding:  [0x04,0xc0,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00]
+0x04,0xc0,0x03,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_store_d16_format_xyzw v[1:2], off, s[12:15], s4 offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV ; encoding:  [0x04,0xc0,0x03,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x04,0xc0,0x03,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_format_xyzw v[1:2], off, s[12:15], s4 offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS ; encoding:  [0x04,0xc0,0x03,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f]
+0x04,0xc0,0x03,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_hi_b8 v1, off, s[12:15], s4 offset:8388607 ; encoding:  [0x04,0x00,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x00,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_hi_b8 v255, off, s[12:15], s4 offset:8388607 ; encoding:  [0x04,0x00,0x09,0xc4,0xff,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x00,0x09,0xc4,0xff,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_hi_b8 v1, off, s[16:19], s4 offset:8388607 ; encoding:  [0x04,0x00,0x09,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x00,0x09,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_hi_b8 v1, off, s[96:99], s4 offset:8388607 ; encoding:  [0x04,0x00,0x09,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x00,0x09,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_hi_b8 v1, off, s[12:15], s101 offset:8388607 ; encoding:  [0x65,0x00,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x00,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_hi_b8 v1, off, s[12:15], m0 offset:8388607 ; encoding:  [0x7d,0x00,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x00,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_hi_b8 v1, v0, s[12:15], s4 idxen offset:8388607 ; encoding:  [0x04,0x00,0x09,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x04,0x00,0x09,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_hi_b8 v1, v0, s[12:15], s4 offen offset:8388607 ; encoding:  [0x04,0x00,0x09,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x04,0x00,0x09,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_hi_b8 v1, off, s[12:15], s4 ; encoding:  [0x04,0x00,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+0x04,0x00,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_store_d16_hi_b8 v1, off, s[12:15], s4 ; encoding:  [0x04,0x00,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+0x04,0x00,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_store_d16_hi_b8 v1, off, s[12:15], s4 offset:7 ; encoding:  [0x04,0x00,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00]
+0x04,0x00,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_store_d16_hi_b8 v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV ; encoding:  [0x04,0x00,0x09,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x00,0x09,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_hi_b8 v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS ; encoding:  [0x04,0x00,0x09,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x00,0x09,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_hi_b16 v1, off, s[12:15], s4 offset:8388607 ; encoding:  [0x04,0x40,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x40,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_hi_b16 v255, off, s[12:15], s4 offset:8388607 ; encoding:  [0x04,0x40,0x09,0xc4,0xff,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x40,0x09,0xc4,0xff,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_hi_b16 v1, off, s[16:19], s4 offset:8388607 ; encoding:  [0x04,0x40,0x09,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x40,0x09,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_hi_b16 v1, off, s[96:99], s4 offset:8388607 ; encoding:  [0x04,0x40,0x09,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x40,0x09,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_hi_b16 v1, off, s[12:15], s101 offset:8388607 ; encoding:  [0x65,0x40,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x40,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_hi_b16 v1, off, s[12:15], m0 offset:8388607 ; encoding:  [0x7d,0x40,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x40,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_hi_b16 v1, v0, s[12:15], s4 idxen offset:8388607 ; encoding:  [0x04,0x40,0x09,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x04,0x40,0x09,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_hi_b16 v1, v0, s[12:15], s4 offen offset:8388607 ; encoding:  [0x04,0x40,0x09,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x04,0x40,0x09,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_hi_b16 v1, off, s[12:15], s4 ; encoding:  [0x04,0x40,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+0x04,0x40,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_store_d16_hi_b16 v1, off, s[12:15], s4 ; encoding:  [0x04,0x40,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+0x04,0x40,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_store_d16_hi_b16 v1, off, s[12:15], s4 offset:7 ; encoding:  [0x04,0x40,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00]
+0x04,0x40,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_store_d16_hi_b16 v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV ; encoding:  [0x04,0x40,0x09,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x40,0x09,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_hi_b16 v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS ; encoding:  [0x04,0x40,0x09,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x40,0x09,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_hi_format_x v1, off, s[12:15], s4 offset:8388607 ; encoding:  [0x04,0xc0,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0xc0,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_hi_format_x v255, off, s[12:15], s4 offset:8388607 ; encoding:  [0x04,0xc0,0x09,0xc4,0xff,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0xc0,0x09,0xc4,0xff,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_hi_format_x v1, off, s[16:19], s4 offset:8388607 ; encoding:  [0x04,0xc0,0x09,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0xc0,0x09,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_hi_format_x v1, off, s[96:99], s4 offset:8388607 ; encoding:  [0x04,0xc0,0x09,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0xc0,0x09,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_hi_format_x v1, off, s[12:15], s101 offset:8388607 ; encoding:  [0x65,0xc0,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0xc0,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_hi_format_x v1, off, s[12:15], m0 offset:8388607 ; encoding:  [0x7d,0xc0,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0xc0,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_hi_format_x v1, v0, s[12:15], s4 idxen offset:8388607 ; encoding:  [0x04,0xc0,0x09,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x04,0xc0,0x09,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_hi_format_x v1, v0, s[12:15], s4 offen offset:8388607 ; encoding:  [0x04,0xc0,0x09,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x04,0xc0,0x09,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_hi_format_x v1, off, s[12:15], s4 ; encoding:  [0x04,0xc0,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+0x04,0xc0,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_store_d16_hi_format_x v1, off, s[12:15], s4 ; encoding:  [0x04,0xc0,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+0x04,0xc0,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_store_d16_hi_format_x v1, off, s[12:15], s4 offset:7 ; encoding:  [0x04,0xc0,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00]
+0x04,0xc0,0x09,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_store_d16_hi_format_x v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV ; encoding:  [0x04,0xc0,0x09,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x04,0xc0,0x09,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_d16_hi_format_x v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS ; encoding:  [0x04,0xc0,0x09,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f]
+0x04,0xc0,0x09,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_format_x v1, off, s[12:15], s4 offset:8388607 ; encoding:  [0x04,0x00,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x00,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_format_x v255, off, s[12:15], s4 offset:8388607 ; encoding:  [0x04,0x00,0x01,0xc4,0xff,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x00,0x01,0xc4,0xff,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_format_x v1, off, s[16:19], s4 offset:8388607 ; encoding:  [0x04,0x00,0x01,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x00,0x01,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_format_x v1, off, s[96:99], s4 offset:8388607 ; encoding:  [0x04,0x00,0x01,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x00,0x01,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_format_x v1, off, s[12:15], s101 offset:8388607 ; encoding:  [0x65,0x00,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x00,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_format_x v1, off, s[12:15], m0 offset:8388607 ; encoding:  [0x7d,0x00,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x00,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_format_x v1, v0, s[12:15], s4 idxen offset:8388607 ; encoding:  [0x04,0x00,0x01,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x04,0x00,0x01,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_format_x v1, v0, s[12:15], s4 offen offset:8388607 ; encoding:  [0x04,0x00,0x01,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x04,0x00,0x01,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_format_x v1, off, s[12:15], s4 ; encoding:  [0x04,0x00,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+0x04,0x00,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_store_format_x v1, off, s[12:15], s4 ; encoding:  [0x04,0x00,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+0x04,0x00,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_store_format_x v1, off, s[12:15], s4 offset:7 ; encoding:  [0x04,0x00,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00]
+0x04,0x00,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_store_format_x v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV ; encoding:  [0x04,0x00,0x01,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x00,0x01,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_format_x v1, off, s[12:15], s4 offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS ; encoding:  [0x04,0x00,0x01,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x00,0x01,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_format_xy v[1:2], off, s[12:15], s4 offset:8388607 ; encoding:  [0x04,0x40,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x40,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_format_xy v[254:255], off, s[12:15], s4 offset:8388607 ; encoding:  [0x04,0x40,0x01,0xc4,0xfe,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x40,0x01,0xc4,0xfe,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_format_xy v[1:2], off, s[16:19], s4 offset:8388607 ; encoding:  [0x04,0x40,0x01,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x40,0x01,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_format_xy v[1:2], off, s[96:99], s4 offset:8388607 ; encoding:  [0x04,0x40,0x01,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x40,0x01,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_format_xy v[1:2], off, s[12:15], s101 offset:8388607 ; encoding:  [0x65,0x40,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x40,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_format_xy v[1:2], off, s[12:15], m0 offset:8388607 ; encoding:  [0x7d,0x40,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x40,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_format_xy v[1:2], v0, s[12:15], s4 idxen offset:8388607 ; encoding:  [0x04,0x40,0x01,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x04,0x40,0x01,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_format_xy v[1:2], v0, s[12:15], s4 offen offset:8388607 ; encoding:  [0x04,0x40,0x01,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x04,0x40,0x01,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_format_xy v[1:2], off, s[12:15], s4 ; encoding:  [0x04,0x40,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+0x04,0x40,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_store_format_xy v[1:2], off, s[12:15], s4 ; encoding:  [0x04,0x40,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+0x04,0x40,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_store_format_xy v[1:2], off, s[12:15], s4 offset:7 ; encoding:  [0x04,0x40,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00]
+0x04,0x40,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_store_format_xy v[1:2], off, s[12:15], s4 offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV ; encoding:  [0x04,0x40,0x01,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x40,0x01,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_format_xy v[1:2], off, s[12:15], s4 offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS ; encoding:  [0x04,0x40,0x01,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x40,0x01,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_format_xyz v[1:3], off, s[12:15], s4 offset:8388607 ; encoding:  [0x04,0x80,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x80,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_format_xyz v[253:255], off, s[12:15], s4 offset:8388607 ; encoding:  [0x04,0x80,0x01,0xc4,0xfd,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x80,0x01,0xc4,0xfd,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_format_xyz v[1:3], off, s[16:19], s4 offset:8388607 ; encoding:  [0x04,0x80,0x01,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x80,0x01,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_format_xyz v[1:3], off, s[96:99], s4 offset:8388607 ; encoding:  [0x04,0x80,0x01,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x80,0x01,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_format_xyz v[1:3], off, s[12:15], s101 offset:8388607 ; encoding:  [0x65,0x80,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x80,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_format_xyz v[1:3], off, s[12:15], m0 offset:8388607 ; encoding:  [0x7d,0x80,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x80,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_format_xyz v[1:3], v0, s[12:15], s4 idxen offset:8388607 ; encoding:  [0x04,0x80,0x01,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x04,0x80,0x01,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_format_xyz v[1:3], v0, s[12:15], s4 offen offset:8388607 ; encoding:  [0x04,0x80,0x01,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x04,0x80,0x01,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_format_xyz v[1:3], off, s[12:15], s4 ; encoding:  [0x04,0x80,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+0x04,0x80,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_store_format_xyz v[1:3], off, s[12:15], s4 ; encoding:  [0x04,0x80,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+0x04,0x80,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_store_format_xyz v[1:3], off, s[12:15], s4 offset:7 ; encoding:  [0x04,0x80,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00]
+0x04,0x80,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_store_format_xyz v[1:3], off, s[12:15], s4 offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV ; encoding:  [0x04,0x80,0x01,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x80,0x01,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_format_xyz v[1:3], off, s[12:15], s4 offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS ; encoding:  [0x04,0x80,0x01,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f]
+0x04,0x80,0x01,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_format_xyzw v[1:4], off, s[12:15], s4 offset:8388607 ; encoding:  [0x04,0xc0,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0xc0,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_format_xyzw v[252:255], off, s[12:15], s4 offset:8388607 ; encoding:  [0x04,0xc0,0x01,0xc4,0xfc,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0xc0,0x01,0xc4,0xfc,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_format_xyzw v[1:4], off, s[16:19], s4 offset:8388607 ; encoding:  [0x04,0xc0,0x01,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0xc0,0x01,0xc4,0x01,0x20,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_format_xyzw v[1:4], off, s[96:99], s4 offset:8388607 ; encoding:  [0x04,0xc0,0x01,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x04,0xc0,0x01,0xc4,0x01,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_format_xyzw v[1:4], off, s[12:15], s101 offset:8388607 ; encoding:  [0x65,0xc0,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0xc0,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_format_xyzw v[1:4], off, s[12:15], m0 offset:8388607 ; encoding:  [0x7d,0xc0,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0xc0,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_format_xyzw v[1:4], v0, s[12:15], s4 idxen offset:8388607 ; encoding:  [0x04,0xc0,0x01,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x04,0xc0,0x01,0xc4,0x01,0x18,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_format_xyzw v[1:4], v0, s[12:15], s4 offen offset:8388607 ; encoding:  [0x04,0xc0,0x01,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x04,0xc0,0x01,0xc4,0x01,0x18,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_format_xyzw v[1:4], off, s[12:15], s4 ; encoding:  [0x04,0xc0,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+0x04,0xc0,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_store_format_xyzw v[1:4], off, s[12:15], s4 ; encoding:  [0x04,0xc0,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00]
+0x04,0xc0,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_store_format_xyzw v[1:4], off, s[12:15], s4 offset:7 ; encoding:  [0x04,0xc0,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00]
+0x04,0xc0,0x01,0xc4,0x01,0x18,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_store_format_xyzw v[1:4], off, s[12:15], s4 offset:8388607 th:TH_STORE_NT_HT scope:SCOPE_DEV ; encoding:  [0x04,0xc0,0x01,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x04,0xc0,0x01,0xc4,0x01,0x18,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_store_format_xyzw v[1:4], off, s[12:15], s4 offset:8388607 th:TH_STORE_BYPASS scope:SCOPE_SYS ; encoding:  [0x04,0xc0,0x01,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f]
+0x04,0xc0,0x01,0xc4,0x01,0x18,0xbc,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_add_f32 v5, off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x80,0x15,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x15,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_add_f32 v255, off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x80,0x15,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x15,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_add_f32 v5, off, s[12:15], s3 offset:8388607 ; encoding:  [0x03,0x80,0x15,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x15,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_add_f32 v5, off, s[96:99], s3 offset:8388607 ; encoding:  [0x03,0x80,0x15,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x15,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_add_f32 v5, off, s[8:11], s101 offset:8388607 ; encoding:  [0x65,0x80,0x15,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x80,0x15,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_add_f32 v5, off, s[8:11], m0 offset:8388607 ; encoding:  [0x7d,0x80,0x15,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x80,0x15,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_add_f32 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding:  [0x03,0x80,0x15,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x15,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_add_f32 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding:  [0x03,0x80,0x15,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x15,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_add_f32 v5, off, s[8:11], s3 ; encoding:  [0x03,0x80,0x15,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x80,0x15,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_add_f32 v5, off, s[8:11], s3 ; encoding:  [0x03,0x80,0x15,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x80,0x15,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_add_f32 v5, off, s[8:11], s3 offset:7 ; encoding:  [0x03,0x80,0x15,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x80,0x15,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_atomic_add_f32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding:  [0x03,0x80,0x15,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x15,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_add_f32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding:  [0x03,0x80,0x15,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x15,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_add_f32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding:  [0x03,0x80,0x15,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x15,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_add_u32 v5, off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x40,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_add_u32 v255, off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x40,0x0d,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x0d,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_add_u32 v5, off, s[12:15], s3 offset:8388607 ; encoding:  [0x03,0x40,0x0d,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x0d,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_add_u32 v5, off, s[96:99], s3 offset:8388607 ; encoding:  [0x03,0x40,0x0d,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x0d,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_add_u32 v5, off, s[8:11], s101 offset:8388607 ; encoding:  [0x65,0x40,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x40,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_add_u32 v5, off, s[8:11], m0 offset:8388607 ; encoding:  [0x7d,0x40,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x40,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_add_u32 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding:  [0x03,0x40,0x0d,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x0d,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_add_u32 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding:  [0x03,0x40,0x0d,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x0d,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_add_u32 v5, off, s[8:11], s3 ; encoding:  [0x03,0x40,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x40,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_add_u32 v5, off, s[8:11], s3 ; encoding:  [0x03,0x40,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x40,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_add_u32 v5, off, s[8:11], s3 offset:7 ; encoding:  [0x03,0x40,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x40,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_atomic_add_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding:  [0x03,0x40,0x0d,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x0d,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_add_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding:  [0x03,0x40,0x0d,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x0d,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_add_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding:  [0x03,0x40,0x0d,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x0d,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_add_u64 v[5:6], off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0xc0,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_add_u64 v[254:255], off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0xc0,0x10,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x10,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_add_u64 v[5:6], off, s[12:15], s3 offset:8388607 ; encoding:  [0x03,0xc0,0x10,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x10,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_add_u64 v[5:6], off, s[96:99], s3 offset:8388607 ; encoding:  [0x03,0xc0,0x10,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x10,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_add_u64 v[5:6], off, s[8:11], s101 offset:8388607 ; encoding:  [0x65,0xc0,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0xc0,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_add_u64 v[5:6], off, s[8:11], m0 offset:8388607 ; encoding:  [0x7d,0xc0,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0xc0,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_add_u64 v[5:6], v0, s[8:11], s3 idxen offset:8388607 ; encoding:  [0x03,0xc0,0x10,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x10,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_add_u64 v[5:6], v0, s[8:11], s3 offen offset:8388607 ; encoding:  [0x03,0xc0,0x10,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x10,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_add_u64 v[5:6], off, s[8:11], s3 ; encoding:  [0x03,0xc0,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0xc0,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_add_u64 v[5:6], off, s[8:11], s3 ; encoding:  [0x03,0xc0,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0xc0,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_add_u64 v[5:6], off, s[8:11], s3 offset:7 ; encoding:  [0x03,0xc0,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0xc0,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_atomic_add_u64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding:  [0x03,0xc0,0x10,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x10,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_add_u64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding:  [0x03,0xc0,0x10,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x10,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_add_u64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding:  [0x03,0xc0,0x10,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x10,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_and_b32 v5, off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x00,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_and_b32 v255, off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x00,0x0f,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x0f,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_and_b32 v5, off, s[12:15], s3 offset:8388607 ; encoding:  [0x03,0x00,0x0f,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x0f,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_and_b32 v5, off, s[96:99], s3 offset:8388607 ; encoding:  [0x03,0x00,0x0f,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x0f,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_and_b32 v5, off, s[8:11], s101 offset:8388607 ; encoding:  [0x65,0x00,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x00,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_and_b32 v5, off, s[8:11], m0 offset:8388607 ; encoding:  [0x7d,0x00,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x00,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_and_b32 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding:  [0x03,0x00,0x0f,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x0f,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_and_b32 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding:  [0x03,0x00,0x0f,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x0f,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_and_b32 v5, off, s[8:11], s3 ; encoding:  [0x03,0x00,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x00,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_and_b32 v5, off, s[8:11], s3 ; encoding:  [0x03,0x00,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x00,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_and_b32 v5, off, s[8:11], s3 offset:7 ; encoding:  [0x03,0x00,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x00,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_atomic_and_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding:  [0x03,0x00,0x0f,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x0f,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_and_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding:  [0x03,0x00,0x0f,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x0f,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_and_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding:  [0x03,0x00,0x0f,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x0f,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_and_b64 v[5:6], off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x40,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_and_b64 v[254:255], off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x40,0x12,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x12,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_and_b64 v[5:6], off, s[12:15], s3 offset:8388607 ; encoding:  [0x03,0x40,0x12,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x12,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_and_b64 v[5:6], off, s[96:99], s3 offset:8388607 ; encoding:  [0x03,0x40,0x12,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x12,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_and_b64 v[5:6], off, s[8:11], s101 offset:8388607 ; encoding:  [0x65,0x40,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x40,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_and_b64 v[5:6], off, s[8:11], m0 offset:8388607 ; encoding:  [0x7d,0x40,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x40,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_and_b64 v[5:6], v0, s[8:11], s3 idxen offset:8388607 ; encoding:  [0x03,0x40,0x12,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x12,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_and_b64 v[5:6], v0, s[8:11], s3 offen offset:8388607 ; encoding:  [0x03,0x40,0x12,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x12,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_and_b64 v[5:6], off, s[8:11], s3 ; encoding:  [0x03,0x40,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x40,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_and_b64 v[5:6], off, s[8:11], s3 ; encoding:  [0x03,0x40,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x40,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_and_b64 v[5:6], off, s[8:11], s3 offset:7 ; encoding:  [0x03,0x40,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x40,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_atomic_and_b64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding:  [0x03,0x40,0x12,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x12,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_and_b64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding:  [0x03,0x40,0x12,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x12,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_and_b64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding:  [0x03,0x40,0x12,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x12,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_cmpswap_b32 v[5:6], off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x00,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_cmpswap_b32 v[254:255], off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x00,0x0d,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x0d,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_cmpswap_b32 v[5:6], off, s[12:15], s3 offset:8388607 ; encoding:  [0x03,0x00,0x0d,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x0d,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_cmpswap_b32 v[5:6], off, s[96:99], s3 offset:8388607 ; encoding:  [0x03,0x00,0x0d,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x0d,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_cmpswap_b32 v[5:6], off, s[8:11], s101 offset:8388607 ; encoding:  [0x65,0x00,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x00,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_cmpswap_b32 v[5:6], off, s[8:11], m0 offset:8388607 ; encoding:  [0x7d,0x00,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x00,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_cmpswap_b32 v[5:6], v0, s[8:11], s3 idxen offset:8388607 ; encoding:  [0x03,0x00,0x0d,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x0d,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_cmpswap_b32 v[5:6], v0, s[8:11], s3 offen offset:8388607 ; encoding:  [0x03,0x00,0x0d,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x0d,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_cmpswap_b32 v[5:6], off, s[8:11], s3 ; encoding:  [0x03,0x00,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x00,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_cmpswap_b32 v[5:6], off, s[8:11], s3 ; encoding:  [0x03,0x00,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x00,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_cmpswap_b32 v[5:6], off, s[8:11], s3 offset:7 ; encoding:  [0x03,0x00,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x00,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_atomic_cmpswap_b32 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding:  [0x03,0x00,0x0d,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x0d,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_cmpswap_b32 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding:  [0x03,0x00,0x0d,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x0d,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_cmpswap_b32 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding:  [0x03,0x00,0x0d,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x0d,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_cmpswap_b64 v[5:8], off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x80,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_cmpswap_b64 v[252:255], off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x80,0x10,0xc4,0xfc,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x10,0xc4,0xfc,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_cmpswap_b64 v[5:8], off, s[12:15], s3 offset:8388607 ; encoding:  [0x03,0x80,0x10,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x10,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_cmpswap_b64 v[5:8], off, s[96:99], s3 offset:8388607 ; encoding:  [0x03,0x80,0x10,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x10,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_cmpswap_b64 v[5:8], off, s[8:11], s101 offset:8388607 ; encoding:  [0x65,0x80,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x80,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_cmpswap_b64 v[5:8], off, s[8:11], m0 offset:8388607 ; encoding:  [0x7d,0x80,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x80,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_cmpswap_b64 v[5:8], v0, s[8:11], s3 idxen offset:8388607 ; encoding:  [0x03,0x80,0x10,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x10,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_cmpswap_b64 v[5:8], v0, s[8:11], s3 offen offset:8388607 ; encoding:  [0x03,0x80,0x10,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x10,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_cmpswap_b64 v[5:8], off, s[8:11], s3 ; encoding:  [0x03,0x80,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x80,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_cmpswap_b64 v[5:8], off, s[8:11], s3 ; encoding:  [0x03,0x80,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x80,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_cmpswap_b64 v[5:8], off, s[8:11], s3 offset:7 ; encoding:  [0x03,0x80,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x80,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_atomic_cmpswap_b64 v[5:8], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding:  [0x03,0x80,0x10,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x10,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_cmpswap_b64 v[5:8], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding:  [0x03,0x80,0x10,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x10,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_cmpswap_b64 v[5:8], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding:  [0x03,0x80,0x10,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x10,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_sub_clamp_u32 v255, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding:  [0x03,0xc0,0x0d,0xc4,0xff,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0d,0xc4,0xff,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_sub_clamp_u32 v255, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding:  [0x03,0xc0,0x0d,0xc4,0xff,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0d,0xc4,0xff,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_sub_clamp_u32 v5, off, s[12:15], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding:  [0x03,0xc0,0x0d,0xc4,0x05,0x18,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0d,0xc4,0x05,0x18,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_sub_clamp_u32 v5, off, s[12:15], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding:  [0x03,0xc0,0x0d,0xc4,0x05,0x18,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0d,0xc4,0x05,0x18,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_sub_clamp_u32 v5, off, s[96:99], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding:  [0x03,0xc0,0x0d,0xc4,0x05,0xc0,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0d,0xc4,0x05,0xc0,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_sub_clamp_u32 v5, off, s[96:99], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding:  [0x03,0xc0,0x0d,0xc4,0x05,0xc0,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0d,0xc4,0x05,0xc0,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s101 offset:8388607 th:TH_ATOMIC_RETURN ; encoding:  [0x65,0xc0,0x0d,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x65,0xc0,0x0d,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s101 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding:  [0x65,0xc0,0x0d,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x65,0xc0,0x0d,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_sub_clamp_u32 v5, off, s[8:11], m0 offset:8388607 th:TH_ATOMIC_RETURN ; encoding:  [0x7d,0xc0,0x0d,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0xc0,0x0d,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_sub_clamp_u32 v5, off, s[8:11], m0 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding:  [0x7d,0xc0,0x0d,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0xc0,0x0d,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_sub_clamp_u32 v5, v0, s[8:11], s3 idxen offset:8388607 th:TH_ATOMIC_RETURN ; encoding:  [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x90,0x80,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0d,0xc4,0x05,0x10,0x90,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_sub_clamp_u32 v5, v0, s[8:11], s3 idxen offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding:  [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x94,0x80,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0d,0xc4,0x05,0x10,0x94,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_sub_clamp_u32 v5, v0, s[8:11], s3 offen offset:8388607 th:TH_ATOMIC_RETURN ; encoding:  [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x90,0x40,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0d,0xc4,0x05,0x10,0x90,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_sub_clamp_u32 v5, v0, s[8:11], s3 offen offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding:  [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x94,0x40,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0d,0xc4,0x05,0x10,0x94,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 th:TH_ATOMIC_RETURN ; encoding:  [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x90,0x00,0x00,0x00,0x00,0x00]
+0x03,0xc0,0x0d,0xc4,0x05,0x10,0x90,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding:  [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x94,0x00,0x00,0x00,0x00,0x00]
+0x03,0xc0,0x0d,0xc4,0x05,0x10,0x94,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 th:TH_ATOMIC_RETURN ; encoding:  [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x90,0x00,0x00,0x00,0x00,0x00]
+0x03,0xc0,0x0d,0xc4,0x05,0x10,0x90,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding:  [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x94,0x00,0x00,0x00,0x00,0x00]
+0x03,0xc0,0x0d,0xc4,0x05,0x10,0x94,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 offset:7 th:TH_ATOMIC_RETURN ; encoding:  [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x90,0x00,0x00,0x07,0x00,0x00]
+0x03,0xc0,0x0d,0xc4,0x05,0x10,0x90,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 offset:7 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding:  [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x94,0x00,0x00,0x07,0x00,0x00]
+0x03,0xc0,0x0d,0xc4,0x05,0x10,0x94,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding:  [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0d,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding:  [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0d,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_sub_clamp_u32 v255, off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0xc0,0x0d,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0d,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_sub_clamp_u32 v255, off, s[8:11], s3 offset:8388607 scope:SCOPE_SE ; encoding:  [0x03,0xc0,0x0d,0xc4,0xff,0x10,0x84,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0d,0xc4,0xff,0x10,0x84,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_sub_clamp_u32 v5, off, s[12:15], s3 offset:8388607 ; encoding:  [0x03,0xc0,0x0d,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0d,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_sub_clamp_u32 v5, off, s[12:15], s3 offset:8388607 scope:SCOPE_SE ; encoding:  [0x03,0xc0,0x0d,0xc4,0x05,0x18,0x84,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0d,0xc4,0x05,0x18,0x84,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_sub_clamp_u32 v5, off, s[96:99], s3 offset:8388607 ; encoding:  [0x03,0xc0,0x0d,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0d,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_sub_clamp_u32 v5, off, s[96:99], s3 offset:8388607 scope:SCOPE_SE ; encoding:  [0x03,0xc0,0x0d,0xc4,0x05,0xc0,0x84,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0d,0xc4,0x05,0xc0,0x84,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s101 offset:8388607 ; encoding:  [0x65,0xc0,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0xc0,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s101 offset:8388607 scope:SCOPE_SE ; encoding:  [0x65,0xc0,0x0d,0xc4,0x05,0x10,0x84,0x00,0x00,0xff,0xff,0x7f]
+0x65,0xc0,0x0d,0xc4,0x05,0x10,0x84,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_sub_clamp_u32 v5, off, s[8:11], m0 offset:8388607 ; encoding:  [0x7d,0xc0,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0xc0,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_sub_clamp_u32 v5, off, s[8:11], m0 offset:8388607 scope:SCOPE_SE ; encoding:  [0x7d,0xc0,0x0d,0xc4,0x05,0x10,0x84,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0xc0,0x0d,0xc4,0x05,0x10,0x84,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_sub_clamp_u32 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding:  [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0d,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_sub_clamp_u32 v5, v0, s[8:11], s3 idxen offset:8388607 scope:SCOPE_SE ; encoding:  [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x84,0x80,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0d,0xc4,0x05,0x10,0x84,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_sub_clamp_u32 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding:  [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0d,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_sub_clamp_u32 v5, v0, s[8:11], s3 offen offset:8388607 scope:SCOPE_SE ; encoding:  [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x84,0x40,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0d,0xc4,0x05,0x10,0x84,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 ; encoding:  [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0xc0,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 scope:SCOPE_SE ; encoding:  [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x84,0x00,0x00,0x00,0x00,0x00]
+0x03,0xc0,0x0d,0xc4,0x05,0x10,0x84,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 ; encoding:  [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0xc0,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 scope:SCOPE_SE ; encoding:  [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x84,0x00,0x00,0x00,0x00,0x00]
+0x03,0xc0,0x0d,0xc4,0x05,0x10,0x84,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 offset:7 ; encoding:  [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0xc0,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 offset:7 scope:SCOPE_SE ; encoding:  [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x84,0x00,0x00,0x07,0x00,0x00]
+0x03,0xc0,0x0d,0xc4,0x05,0x10,0x84,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_sub_clamp_u32 v5, off, s[8:11], s3 offset:8388607 scope:SCOPE_SE ; encoding:  [0x03,0xc0,0x0d,0xc4,0x05,0x10,0x84,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0d,0xc4,0x05,0x10,0x84,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_dec_u32 v5, off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x00,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_dec_u32 v255, off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x00,0x10,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x10,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_dec_u32 v5, off, s[12:15], s3 offset:8388607 ; encoding:  [0x03,0x00,0x10,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x10,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_dec_u32 v5, off, s[96:99], s3 offset:8388607 ; encoding:  [0x03,0x00,0x10,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x10,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_dec_u32 v5, off, s[8:11], s101 offset:8388607 ; encoding:  [0x65,0x00,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x00,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_dec_u32 v5, off, s[8:11], m0 offset:8388607 ; encoding:  [0x7d,0x00,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x00,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_dec_u32 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding:  [0x03,0x00,0x10,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x10,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_dec_u32 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding:  [0x03,0x00,0x10,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x10,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_dec_u32 v5, off, s[8:11], s3 ; encoding:  [0x03,0x00,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x00,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_dec_u32 v5, off, s[8:11], s3 ; encoding:  [0x03,0x00,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x00,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_dec_u32 v5, off, s[8:11], s3 offset:7 ; encoding:  [0x03,0x00,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x00,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_atomic_dec_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding:  [0x03,0x00,0x10,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x10,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_dec_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding:  [0x03,0x00,0x10,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x10,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_dec_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding:  [0x03,0x00,0x10,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x10,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_dec_u64 v[5:6], off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x40,0x13,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x13,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_dec_u64 v[254:255], off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x40,0x13,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x13,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_dec_u64 v[5:6], off, s[12:15], s3 offset:8388607 ; encoding:  [0x03,0x40,0x13,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x13,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_dec_u64 v[5:6], off, s[96:99], s3 offset:8388607 ; encoding:  [0x03,0x40,0x13,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x13,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_dec_u64 v[5:6], off, s[8:11], s101 offset:8388607 ; encoding:  [0x65,0x40,0x13,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x40,0x13,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_dec_u64 v[5:6], off, s[8:11], m0 offset:8388607 ; encoding:  [0x7d,0x40,0x13,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x40,0x13,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_dec_u64 v[5:6], v0, s[8:11], s3 idxen offset:8388607 ; encoding:  [0x03,0x40,0x13,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x13,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_dec_u64 v[5:6], v0, s[8:11], s3 offen offset:8388607 ; encoding:  [0x03,0x40,0x13,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x13,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_dec_u64 v[5:6], off, s[8:11], s3 ; encoding:  [0x03,0x40,0x13,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x40,0x13,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_dec_u64 v[5:6], off, s[8:11], s3 ; encoding:  [0x03,0x40,0x13,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x40,0x13,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_dec_u64 v[5:6], off, s[8:11], s3 offset:7 ; encoding:  [0x03,0x40,0x13,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x40,0x13,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_atomic_dec_u64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding:  [0x03,0x40,0x13,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x13,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_dec_u64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding:  [0x03,0x40,0x13,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x13,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_dec_u64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding:  [0x03,0x40,0x13,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x13,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_inc_u32 v5, off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0xc0,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_inc_u32 v255, off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0xc0,0x0f,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0f,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_inc_u32 v5, off, s[12:15], s3 offset:8388607 ; encoding:  [0x03,0xc0,0x0f,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0f,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_inc_u32 v5, off, s[96:99], s3 offset:8388607 ; encoding:  [0x03,0xc0,0x0f,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0f,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_inc_u32 v5, off, s[8:11], s101 offset:8388607 ; encoding:  [0x65,0xc0,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0xc0,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_inc_u32 v5, off, s[8:11], m0 offset:8388607 ; encoding:  [0x7d,0xc0,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0xc0,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_inc_u32 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding:  [0x03,0xc0,0x0f,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0f,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_inc_u32 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding:  [0x03,0xc0,0x0f,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0f,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_inc_u32 v5, off, s[8:11], s3 ; encoding:  [0x03,0xc0,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0xc0,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_inc_u32 v5, off, s[8:11], s3 ; encoding:  [0x03,0xc0,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0xc0,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_inc_u32 v5, off, s[8:11], s3 offset:7 ; encoding:  [0x03,0xc0,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0xc0,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_atomic_inc_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding:  [0x03,0xc0,0x0f,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0f,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_inc_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding:  [0x03,0xc0,0x0f,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0f,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_inc_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding:  [0x03,0xc0,0x0f,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0f,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_inc_u64 v[5:6], off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x00,0x13,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x13,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_inc_u64 v[254:255], off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x00,0x13,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x13,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_inc_u64 v[5:6], off, s[12:15], s3 offset:8388607 ; encoding:  [0x03,0x00,0x13,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x13,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_inc_u64 v[5:6], off, s[96:99], s3 offset:8388607 ; encoding:  [0x03,0x00,0x13,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x13,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_inc_u64 v[5:6], off, s[8:11], s101 offset:8388607 ; encoding:  [0x65,0x00,0x13,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x00,0x13,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_inc_u64 v[5:6], off, s[8:11], m0 offset:8388607 ; encoding:  [0x7d,0x00,0x13,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x00,0x13,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_inc_u64 v[5:6], v0, s[8:11], s3 idxen offset:8388607 ; encoding:  [0x03,0x00,0x13,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x13,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_inc_u64 v[5:6], v0, s[8:11], s3 offen offset:8388607 ; encoding:  [0x03,0x00,0x13,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x13,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_inc_u64 v[5:6], off, s[8:11], s3 ; encoding:  [0x03,0x00,0x13,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x00,0x13,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_inc_u64 v[5:6], off, s[8:11], s3 ; encoding:  [0x03,0x00,0x13,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x00,0x13,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_inc_u64 v[5:6], off, s[8:11], s3 offset:7 ; encoding:  [0x03,0x00,0x13,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x00,0x13,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_atomic_inc_u64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding:  [0x03,0x00,0x13,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x13,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_inc_u64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding:  [0x03,0x00,0x13,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x13,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_inc_u64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding:  [0x03,0x00,0x13,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x13,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_max_num_f32 v5, off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x80,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_max_num_f32 v255, off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x80,0x14,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x14,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_max_num_f32 v5, off, s[12:15], s3 offset:8388607 ; encoding:  [0x03,0x80,0x14,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x14,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_max_num_f32 v5, off, s[96:99], s3 offset:8388607 ; encoding:  [0x03,0x80,0x14,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x14,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_max_num_f32 v5, off, s[8:11], s101 offset:8388607 ; encoding:  [0x65,0x80,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x80,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_max_num_f32 v5, off, s[8:11], m0 offset:8388607 ; encoding:  [0x7d,0x80,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x80,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_max_num_f32 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding:  [0x03,0x80,0x14,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x14,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_max_num_f32 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding:  [0x03,0x80,0x14,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x14,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_max_num_f32 v5, off, s[8:11], s3 ; encoding:  [0x03,0x80,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x80,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_max_num_f32 v5, off, s[8:11], s3 ; encoding:  [0x03,0x80,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x80,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_max_num_f32 v5, off, s[8:11], s3 offset:7 ; encoding:  [0x03,0x80,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x80,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_atomic_max_num_f32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding:  [0x03,0x80,0x14,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x14,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_max_num_f32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding:  [0x03,0x80,0x14,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x14,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_max_num_f32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding:  [0x03,0x80,0x14,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x14,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_max_i32 v5, off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x80,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_max_i32 v255, off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x80,0x0e,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x0e,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_max_i32 v5, off, s[12:15], s3 offset:8388607 ; encoding:  [0x03,0x80,0x0e,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x0e,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_max_i32 v5, off, s[96:99], s3 offset:8388607 ; encoding:  [0x03,0x80,0x0e,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x0e,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_max_i32 v5, off, s[8:11], s101 offset:8388607 ; encoding:  [0x65,0x80,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x80,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_max_i32 v5, off, s[8:11], m0 offset:8388607 ; encoding:  [0x7d,0x80,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x80,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_max_i32 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding:  [0x03,0x80,0x0e,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x0e,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_max_i32 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding:  [0x03,0x80,0x0e,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x0e,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_max_i32 v5, off, s[8:11], s3 ; encoding:  [0x03,0x80,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x80,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_max_i32 v5, off, s[8:11], s3 ; encoding:  [0x03,0x80,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x80,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_max_i32 v5, off, s[8:11], s3 offset:7 ; encoding:  [0x03,0x80,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x80,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_atomic_max_i32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding:  [0x03,0x80,0x0e,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x0e,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_max_i32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding:  [0x03,0x80,0x0e,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x0e,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_max_i32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding:  [0x03,0x80,0x0e,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x0e,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_max_i64 v[5:6], off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0xc0,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_max_i64 v[254:255], off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0xc0,0x11,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x11,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_max_i64 v[5:6], off, s[12:15], s3 offset:8388607 ; encoding:  [0x03,0xc0,0x11,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x11,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_max_i64 v[5:6], off, s[96:99], s3 offset:8388607 ; encoding:  [0x03,0xc0,0x11,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x11,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_max_i64 v[5:6], off, s[8:11], s101 offset:8388607 ; encoding:  [0x65,0xc0,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0xc0,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_max_i64 v[5:6], off, s[8:11], m0 offset:8388607 ; encoding:  [0x7d,0xc0,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0xc0,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_max_i64 v[5:6], v0, s[8:11], s3 idxen offset:8388607 ; encoding:  [0x03,0xc0,0x11,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x11,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_max_i64 v[5:6], v0, s[8:11], s3 offen offset:8388607 ; encoding:  [0x03,0xc0,0x11,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x11,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_max_i64 v[5:6], off, s[8:11], s3 ; encoding:  [0x03,0xc0,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0xc0,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_max_i64 v[5:6], off, s[8:11], s3 ; encoding:  [0x03,0xc0,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0xc0,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_max_i64 v[5:6], off, s[8:11], s3 offset:7 ; encoding:  [0x03,0xc0,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0xc0,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_atomic_max_i64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding:  [0x03,0xc0,0x11,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x11,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_max_i64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding:  [0x03,0xc0,0x11,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x11,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_max_i64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding:  [0x03,0xc0,0x11,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x11,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_max_u32 v5, off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0xc0,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_max_u32 v255, off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0xc0,0x0e,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0e,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_max_u32 v5, off, s[12:15], s3 offset:8388607 ; encoding:  [0x03,0xc0,0x0e,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0e,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_max_u32 v5, off, s[96:99], s3 offset:8388607 ; encoding:  [0x03,0xc0,0x0e,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0e,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_max_u32 v5, off, s[8:11], s101 offset:8388607 ; encoding:  [0x65,0xc0,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0xc0,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_max_u32 v5, off, s[8:11], m0 offset:8388607 ; encoding:  [0x7d,0xc0,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0xc0,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_max_u32 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding:  [0x03,0xc0,0x0e,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0e,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_max_u32 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding:  [0x03,0xc0,0x0e,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0e,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_max_u32 v5, off, s[8:11], s3 ; encoding:  [0x03,0xc0,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0xc0,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_max_u32 v5, off, s[8:11], s3 ; encoding:  [0x03,0xc0,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0xc0,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_max_u32 v5, off, s[8:11], s3 offset:7 ; encoding:  [0x03,0xc0,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0xc0,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_atomic_max_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding:  [0x03,0xc0,0x0e,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0e,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_max_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding:  [0x03,0xc0,0x0e,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0e,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_max_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding:  [0x03,0xc0,0x0e,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0e,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_max_u64 v[5:6], off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x00,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_max_u64 v[254:255], off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x00,0x12,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x12,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_max_u64 v[5:6], off, s[12:15], s3 offset:8388607 ; encoding:  [0x03,0x00,0x12,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x12,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_max_u64 v[5:6], off, s[96:99], s3 offset:8388607 ; encoding:  [0x03,0x00,0x12,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x12,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_max_u64 v[5:6], off, s[8:11], s101 offset:8388607 ; encoding:  [0x65,0x00,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x00,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_max_u64 v[5:6], off, s[8:11], m0 offset:8388607 ; encoding:  [0x7d,0x00,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x00,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_max_u64 v[5:6], v0, s[8:11], s3 idxen offset:8388607 ; encoding:  [0x03,0x00,0x12,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x12,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_max_u64 v[5:6], v0, s[8:11], s3 offen offset:8388607 ; encoding:  [0x03,0x00,0x12,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x12,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_max_u64 v[5:6], off, s[8:11], s3 ; encoding:  [0x03,0x00,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x00,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_max_u64 v[5:6], off, s[8:11], s3 ; encoding:  [0x03,0x00,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x00,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_max_u64 v[5:6], off, s[8:11], s3 offset:7 ; encoding:  [0x03,0x00,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x00,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_atomic_max_u64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding:  [0x03,0x00,0x12,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x12,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_max_u64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding:  [0x03,0x00,0x12,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x12,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_max_u64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding:  [0x03,0x00,0x12,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x12,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_min_num_f32 v5, off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x40,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_min_num_f32 v255, off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x40,0x14,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x14,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_min_num_f32 v5, off, s[12:15], s3 offset:8388607 ; encoding:  [0x03,0x40,0x14,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x14,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_min_num_f32 v5, off, s[96:99], s3 offset:8388607 ; encoding:  [0x03,0x40,0x14,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x14,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_min_num_f32 v5, off, s[8:11], s101 offset:8388607 ; encoding:  [0x65,0x40,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x40,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_min_num_f32 v5, off, s[8:11], m0 offset:8388607 ; encoding:  [0x7d,0x40,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x40,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_min_num_f32 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding:  [0x03,0x40,0x14,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x14,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_min_num_f32 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding:  [0x03,0x40,0x14,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x14,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_min_num_f32 v5, off, s[8:11], s3 ; encoding:  [0x03,0x40,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x40,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_min_num_f32 v5, off, s[8:11], s3 ; encoding:  [0x03,0x40,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x40,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_min_num_f32 v5, off, s[8:11], s3 offset:7 ; encoding:  [0x03,0x40,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x40,0x14,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_atomic_min_num_f32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding:  [0x03,0x40,0x14,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x14,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_min_num_f32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding:  [0x03,0x40,0x14,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x14,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_min_num_f32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding:  [0x03,0x40,0x14,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x14,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_min_i32 v5, off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x00,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_min_i32 v255, off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x00,0x0e,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x0e,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_min_i32 v5, off, s[12:15], s3 offset:8388607 ; encoding:  [0x03,0x00,0x0e,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x0e,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_min_i32 v5, off, s[96:99], s3 offset:8388607 ; encoding:  [0x03,0x00,0x0e,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x0e,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_min_i32 v5, off, s[8:11], s101 offset:8388607 ; encoding:  [0x65,0x00,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x00,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_min_i32 v5, off, s[8:11], m0 offset:8388607 ; encoding:  [0x7d,0x00,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x00,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_min_i32 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding:  [0x03,0x00,0x0e,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x0e,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_min_i32 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding:  [0x03,0x00,0x0e,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x0e,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_min_i32 v5, off, s[8:11], s3 ; encoding:  [0x03,0x00,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x00,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_min_i32 v5, off, s[8:11], s3 ; encoding:  [0x03,0x00,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x00,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_min_i32 v5, off, s[8:11], s3 offset:7 ; encoding:  [0x03,0x00,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x00,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_atomic_min_i32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding:  [0x03,0x00,0x0e,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x0e,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_min_i32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding:  [0x03,0x00,0x0e,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x0e,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_min_i32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding:  [0x03,0x00,0x0e,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x0e,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_min_i64 v[5:6], off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x40,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_min_i64 v[254:255], off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x40,0x11,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x11,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_min_i64 v[5:6], off, s[12:15], s3 offset:8388607 ; encoding:  [0x03,0x40,0x11,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x11,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_min_i64 v[5:6], off, s[96:99], s3 offset:8388607 ; encoding:  [0x03,0x40,0x11,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x11,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_min_i64 v[5:6], off, s[8:11], s101 offset:8388607 ; encoding:  [0x65,0x40,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x40,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_min_i64 v[5:6], off, s[8:11], m0 offset:8388607 ; encoding:  [0x7d,0x40,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x40,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_min_i64 v[5:6], v0, s[8:11], s3 idxen offset:8388607 ; encoding:  [0x03,0x40,0x11,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x11,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_min_i64 v[5:6], v0, s[8:11], s3 offen offset:8388607 ; encoding:  [0x03,0x40,0x11,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x11,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_min_i64 v[5:6], off, s[8:11], s3 ; encoding:  [0x03,0x40,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x40,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_min_i64 v[5:6], off, s[8:11], s3 ; encoding:  [0x03,0x40,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x40,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_min_i64 v[5:6], off, s[8:11], s3 offset:7 ; encoding:  [0x03,0x40,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x40,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_atomic_min_i64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding:  [0x03,0x40,0x11,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x11,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_min_i64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding:  [0x03,0x40,0x11,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x11,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_min_i64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding:  [0x03,0x40,0x11,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x11,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_min_u32 v5, off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x40,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_min_u32 v255, off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x40,0x0e,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x0e,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_min_u32 v5, off, s[12:15], s3 offset:8388607 ; encoding:  [0x03,0x40,0x0e,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x0e,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_min_u32 v5, off, s[96:99], s3 offset:8388607 ; encoding:  [0x03,0x40,0x0e,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x0e,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_min_u32 v5, off, s[8:11], s101 offset:8388607 ; encoding:  [0x65,0x40,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x40,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_min_u32 v5, off, s[8:11], m0 offset:8388607 ; encoding:  [0x7d,0x40,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x40,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_min_u32 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding:  [0x03,0x40,0x0e,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x0e,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_min_u32 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding:  [0x03,0x40,0x0e,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x0e,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_min_u32 v5, off, s[8:11], s3 ; encoding:  [0x03,0x40,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x40,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_min_u32 v5, off, s[8:11], s3 ; encoding:  [0x03,0x40,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x40,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_min_u32 v5, off, s[8:11], s3 offset:7 ; encoding:  [0x03,0x40,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x40,0x0e,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_atomic_min_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding:  [0x03,0x40,0x0e,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x0e,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_min_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding:  [0x03,0x40,0x0e,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x0e,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_min_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding:  [0x03,0x40,0x0e,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x0e,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_min_u64 v[5:6], off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x80,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_min_u64 v[254:255], off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x80,0x11,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x11,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_min_u64 v[5:6], off, s[12:15], s3 offset:8388607 ; encoding:  [0x03,0x80,0x11,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x11,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_min_u64 v[5:6], off, s[96:99], s3 offset:8388607 ; encoding:  [0x03,0x80,0x11,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x11,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_min_u64 v[5:6], off, s[8:11], s101 offset:8388607 ; encoding:  [0x65,0x80,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x80,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_min_u64 v[5:6], off, s[8:11], m0 offset:8388607 ; encoding:  [0x7d,0x80,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x80,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_min_u64 v[5:6], v0, s[8:11], s3 idxen offset:8388607 ; encoding:  [0x03,0x80,0x11,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x11,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_min_u64 v[5:6], v0, s[8:11], s3 offen offset:8388607 ; encoding:  [0x03,0x80,0x11,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x11,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_min_u64 v[5:6], off, s[8:11], s3 ; encoding:  [0x03,0x80,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x80,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_min_u64 v[5:6], off, s[8:11], s3 ; encoding:  [0x03,0x80,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x80,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_min_u64 v[5:6], off, s[8:11], s3 offset:7 ; encoding:  [0x03,0x80,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x80,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_atomic_min_u64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding:  [0x03,0x80,0x11,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x11,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_min_u64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding:  [0x03,0x80,0x11,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x11,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_min_u64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding:  [0x03,0x80,0x11,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x11,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_or_b32 v5, off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x40,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_or_b32 v255, off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x40,0x0f,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x0f,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_or_b32 v5, off, s[12:15], s3 offset:8388607 ; encoding:  [0x03,0x40,0x0f,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x0f,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_or_b32 v5, off, s[96:99], s3 offset:8388607 ; encoding:  [0x03,0x40,0x0f,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x0f,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_or_b32 v5, off, s[8:11], s101 offset:8388607 ; encoding:  [0x65,0x40,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x40,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_or_b32 v5, off, s[8:11], m0 offset:8388607 ; encoding:  [0x7d,0x40,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x40,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_or_b32 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding:  [0x03,0x40,0x0f,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x0f,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_or_b32 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding:  [0x03,0x40,0x0f,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x0f,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_or_b32 v5, off, s[8:11], s3 ; encoding:  [0x03,0x40,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x40,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_or_b32 v5, off, s[8:11], s3 ; encoding:  [0x03,0x40,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x40,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_or_b32 v5, off, s[8:11], s3 offset:7 ; encoding:  [0x03,0x40,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x40,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_atomic_or_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding:  [0x03,0x40,0x0f,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x0f,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_or_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding:  [0x03,0x40,0x0f,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x0f,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_or_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding:  [0x03,0x40,0x0f,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x0f,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_or_b64 v[5:6], off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x80,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_or_b64 v[254:255], off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x80,0x12,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x12,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_or_b64 v[5:6], off, s[12:15], s3 offset:8388607 ; encoding:  [0x03,0x80,0x12,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x12,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_or_b64 v[5:6], off, s[96:99], s3 offset:8388607 ; encoding:  [0x03,0x80,0x12,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x12,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_or_b64 v[5:6], off, s[8:11], s101 offset:8388607 ; encoding:  [0x65,0x80,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x80,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_or_b64 v[5:6], off, s[8:11], m0 offset:8388607 ; encoding:  [0x7d,0x80,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x80,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_or_b64 v[5:6], v0, s[8:11], s3 idxen offset:8388607 ; encoding:  [0x03,0x80,0x12,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x12,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_or_b64 v[5:6], v0, s[8:11], s3 offen offset:8388607 ; encoding:  [0x03,0x80,0x12,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x12,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_or_b64 v[5:6], off, s[8:11], s3 ; encoding:  [0x03,0x80,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x80,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_or_b64 v[5:6], off, s[8:11], s3 ; encoding:  [0x03,0x80,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x80,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_or_b64 v[5:6], off, s[8:11], s3 offset:7 ; encoding:  [0x03,0x80,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x80,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_atomic_or_b64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding:  [0x03,0x80,0x12,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x12,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_or_b64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding:  [0x03,0x80,0x12,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x12,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_or_b64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding:  [0x03,0x80,0x12,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x12,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_sub_u32 v5, off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x80,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_sub_u32 v255, off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x80,0x0d,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x0d,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_sub_u32 v5, off, s[12:15], s3 offset:8388607 ; encoding:  [0x03,0x80,0x0d,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x0d,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_sub_u32 v5, off, s[96:99], s3 offset:8388607 ; encoding:  [0x03,0x80,0x0d,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x0d,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_sub_u32 v5, off, s[8:11], s101 offset:8388607 ; encoding:  [0x65,0x80,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x80,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_sub_u32 v5, off, s[8:11], m0 offset:8388607 ; encoding:  [0x7d,0x80,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x80,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_sub_u32 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding:  [0x03,0x80,0x0d,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x0d,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_sub_u32 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding:  [0x03,0x80,0x0d,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x0d,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_sub_u32 v5, off, s[8:11], s3 ; encoding:  [0x03,0x80,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x80,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_sub_u32 v5, off, s[8:11], s3 ; encoding:  [0x03,0x80,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x80,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_sub_u32 v5, off, s[8:11], s3 offset:7 ; encoding:  [0x03,0x80,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x80,0x0d,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_atomic_sub_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding:  [0x03,0x80,0x0d,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x0d,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_sub_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding:  [0x03,0x80,0x0d,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x0d,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_sub_u32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding:  [0x03,0x80,0x0d,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x0d,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_sub_u64 v[5:6], off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x00,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_sub_u64 v[254:255], off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x00,0x11,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x11,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_sub_u64 v[5:6], off, s[12:15], s3 offset:8388607 ; encoding:  [0x03,0x00,0x11,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x11,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_sub_u64 v[5:6], off, s[96:99], s3 offset:8388607 ; encoding:  [0x03,0x00,0x11,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x11,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_sub_u64 v[5:6], off, s[8:11], s101 offset:8388607 ; encoding:  [0x65,0x00,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x00,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_sub_u64 v[5:6], off, s[8:11], m0 offset:8388607 ; encoding:  [0x7d,0x00,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x00,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_sub_u64 v[5:6], v0, s[8:11], s3 idxen offset:8388607 ; encoding:  [0x03,0x00,0x11,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x11,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_sub_u64 v[5:6], v0, s[8:11], s3 offen offset:8388607 ; encoding:  [0x03,0x00,0x11,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x11,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_sub_u64 v[5:6], off, s[8:11], s3 ; encoding:  [0x03,0x00,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x00,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_sub_u64 v[5:6], off, s[8:11], s3 ; encoding:  [0x03,0x00,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x00,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_sub_u64 v[5:6], off, s[8:11], s3 offset:7 ; encoding:  [0x03,0x00,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x00,0x11,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_atomic_sub_u64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding:  [0x03,0x00,0x11,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x11,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_sub_u64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding:  [0x03,0x00,0x11,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x11,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_sub_u64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding:  [0x03,0x00,0x11,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x00,0x11,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_swap_b32 v5, off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0xc0,0x0c,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0c,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_swap_b32 v255, off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0xc0,0x0c,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0c,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_swap_b32 v5, off, s[12:15], s3 offset:8388607 ; encoding:  [0x03,0xc0,0x0c,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0c,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_swap_b32 v5, off, s[96:99], s3 offset:8388607 ; encoding:  [0x03,0xc0,0x0c,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0c,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_swap_b32 v5, off, s[8:11], s101 offset:8388607 ; encoding:  [0x65,0xc0,0x0c,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0xc0,0x0c,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_swap_b32 v5, off, s[8:11], m0 offset:8388607 ; encoding:  [0x7d,0xc0,0x0c,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0xc0,0x0c,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_swap_b32 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding:  [0x03,0xc0,0x0c,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0c,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_swap_b32 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding:  [0x03,0xc0,0x0c,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0c,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_swap_b32 v5, off, s[8:11], s3 ; encoding:  [0x03,0xc0,0x0c,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0xc0,0x0c,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_swap_b32 v5, off, s[8:11], s3 ; encoding:  [0x03,0xc0,0x0c,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0xc0,0x0c,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_swap_b32 v5, off, s[8:11], s3 offset:7 ; encoding:  [0x03,0xc0,0x0c,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0xc0,0x0c,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_atomic_swap_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding:  [0x03,0xc0,0x0c,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0c,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_swap_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding:  [0x03,0xc0,0x0c,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0c,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_swap_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding:  [0x03,0xc0,0x0c,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x0c,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_swap_b64 v[5:6], off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x40,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_swap_b64 v[254:255], off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x40,0x10,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x10,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_swap_b64 v[5:6], off, s[12:15], s3 offset:8388607 ; encoding:  [0x03,0x40,0x10,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x10,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_swap_b64 v[5:6], off, s[96:99], s3 offset:8388607 ; encoding:  [0x03,0x40,0x10,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x10,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_swap_b64 v[5:6], off, s[8:11], s101 offset:8388607 ; encoding:  [0x65,0x40,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x40,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_swap_b64 v[5:6], off, s[8:11], m0 offset:8388607 ; encoding:  [0x7d,0x40,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x40,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_swap_b64 v[5:6], v0, s[8:11], s3 idxen offset:8388607 ; encoding:  [0x03,0x40,0x10,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x10,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_swap_b64 v[5:6], v0, s[8:11], s3 offen offset:8388607 ; encoding:  [0x03,0x40,0x10,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x10,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_swap_b64 v[5:6], off, s[8:11], s3 ; encoding:  [0x03,0x40,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x40,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_swap_b64 v[5:6], off, s[8:11], s3 ; encoding:  [0x03,0x40,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x40,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_swap_b64 v[5:6], off, s[8:11], s3 offset:7 ; encoding:  [0x03,0x40,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x40,0x10,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_atomic_swap_b64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding:  [0x03,0x40,0x10,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x10,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_swap_b64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding:  [0x03,0x40,0x10,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x10,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_swap_b64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding:  [0x03,0x40,0x10,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x40,0x10,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_xor_b32 v5, off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x80,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_xor_b32 v255, off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0x80,0x0f,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x0f,0xc4,0xff,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_xor_b32 v5, off, s[12:15], s3 offset:8388607 ; encoding:  [0x03,0x80,0x0f,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x0f,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_xor_b32 v5, off, s[96:99], s3 offset:8388607 ; encoding:  [0x03,0x80,0x0f,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x0f,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_xor_b32 v5, off, s[8:11], s101 offset:8388607 ; encoding:  [0x65,0x80,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0x80,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_xor_b32 v5, off, s[8:11], m0 offset:8388607 ; encoding:  [0x7d,0x80,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0x80,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_xor_b32 v5, v0, s[8:11], s3 idxen offset:8388607 ; encoding:  [0x03,0x80,0x0f,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x0f,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_xor_b32 v5, v0, s[8:11], s3 offen offset:8388607 ; encoding:  [0x03,0x80,0x0f,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x0f,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_xor_b32 v5, off, s[8:11], s3 ; encoding:  [0x03,0x80,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x80,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_xor_b32 v5, off, s[8:11], s3 ; encoding:  [0x03,0x80,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0x80,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_xor_b32 v5, off, s[8:11], s3 offset:7 ; encoding:  [0x03,0x80,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0x80,0x0f,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_atomic_xor_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding:  [0x03,0x80,0x0f,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x0f,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_xor_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding:  [0x03,0x80,0x0f,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x0f,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_xor_b32 v5, off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding:  [0x03,0x80,0x0f,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0x80,0x0f,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_xor_b64 v[5:6], off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0xc0,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_xor_b64 v[254:255], off, s[8:11], s3 offset:8388607 ; encoding:  [0x03,0xc0,0x12,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x12,0xc4,0xfe,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_xor_b64 v[5:6], off, s[12:15], s3 offset:8388607 ; encoding:  [0x03,0xc0,0x12,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x12,0xc4,0x05,0x18,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_xor_b64 v[5:6], off, s[96:99], s3 offset:8388607 ; encoding:  [0x03,0xc0,0x12,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x12,0xc4,0x05,0xc0,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_xor_b64 v[5:6], off, s[8:11], s101 offset:8388607 ; encoding:  [0x65,0xc0,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x65,0xc0,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_xor_b64 v[5:6], off, s[8:11], m0 offset:8388607 ; encoding:  [0x7d,0xc0,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f]
+0x7d,0xc0,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_xor_b64 v[5:6], v0, s[8:11], s3 idxen offset:8388607 ; encoding:  [0x03,0xc0,0x12,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x12,0xc4,0x05,0x10,0x80,0x80,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_xor_b64 v[5:6], v0, s[8:11], s3 offen offset:8388607 ; encoding:  [0x03,0xc0,0x12,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x12,0xc4,0x05,0x10,0x80,0x40,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_xor_b64 v[5:6], off, s[8:11], s3 ; encoding:  [0x03,0xc0,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0xc0,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_xor_b64 v[5:6], off, s[8:11], s3 ; encoding:  [0x03,0xc0,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00]
+0x03,0xc0,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: buffer_atomic_xor_b64 v[5:6], off, s[8:11], s3 offset:7 ; encoding:  [0x03,0xc0,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00]
+0x03,0xc0,0x12,0xc4,0x05,0x10,0x80,0x00,0x00,0x07,0x00,0x00
+
+# GFX12: buffer_atomic_xor_b64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN ; encoding:  [0x03,0xc0,0x12,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x12,0xc4,0x05,0x10,0x90,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_xor_b64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_RETURN scope:SCOPE_SE ; encoding:  [0x03,0xc0,0x12,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x12,0xc4,0x05,0x10,0x94,0x00,0x00,0xff,0xff,0x7f
+
+# GFX12: buffer_atomic_xor_b64 v[5:6], off, s[8:11], s3 offset:8388607 th:TH_ATOMIC_CASCADE_NT scope:SCOPE_DEV ; encoding:  [0x03,0xc0,0x12,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f]
+0x03,0xc0,0x12,0xc4,0x05,0x10,0xe8,0x00,0x00,0xff,0xff,0x7f
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vflat.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vflat.txt
new file mode 100644
index 0000000000000000000000000000000000000000..d7f9daf295845a69f879b6e8c192dfb16753c9a6
--- /dev/null
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vflat.txt
@@ -0,0 +1,1501 @@
+# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck %s -check-prefix=GFX12
+
+# GFX12: flat_atomic_add_f32 v1, v[0:1], v2 offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x80,0x15,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x80,0x15,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_atomic_add_f32 v[0:1], v2 offset:64 ; encoding: [0x7c,0x80,0x15,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x80,0x15,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_atomic_add_u32 v1, v[0:1], v2 offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x40,0x0d,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x40,0x0d,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_atomic_add_u32 v[0:1], v2 offset:64 ; encoding: [0x7c,0x40,0x0d,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x40,0x0d,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_atomic_add_u32 v[3:4], v5          ; encoding: [0x7c,0x40,0x0d,0xec,0x00,0x00,0x80,0x02,0x03,0x00,0x00,0x00]
+0x7c,0x40,0x0d,0xec,0x00,0x00,0x80,0x02,0x03,0x00,0x00,0x00
+
+# GFX12: flat_atomic_add_u64 v[0:1], v[2:3] offset:64 ; encoding: [0x7c,0xc0,0x10,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0xc0,0x10,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_atomic_add_u64 v[1:2], v[0:1], v[2:3] offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0xc0,0x10,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x7c,0xc0,0x10,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_atomic_and_b32 v1, v[0:1], v2 offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x00,0x0f,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x00,0x0f,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_atomic_and_b32 v[0:1], v2 offset:64 ; encoding: [0x7c,0x00,0x0f,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x00,0x0f,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_atomic_and_b64 v[0:1], v[2:3] offset:64 ; encoding: [0x7c,0x40,0x12,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x40,0x12,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_atomic_and_b64 v[1:2], v[0:1], v[2:3] offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x40,0x12,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x40,0x12,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_atomic_cmpswap_b32 v0, v[1:2], v[3:4] th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x00,0x0d,0xec,0x00,0x00,0x90,0x01,0x01,0x00,0x00,0x00]
+0x7c,0x00,0x0d,0xec,0x00,0x00,0x90,0x01,0x01,0x00,0x00,0x00
+
+# GFX12: flat_atomic_cmpswap_b32 v0, v[1:2], v[3:4] th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x00,0x0d,0xec,0x00,0x00,0x90,0x01,0x01,0x00,0x00,0x00]
+0x7c,0x00,0x0d,0xec,0x00,0x00,0x90,0x01,0x01,0x00,0x00,0x00
+
+# GFX12: flat_atomic_cmpswap_b32 v0, v[1:2], v[3:4] offset:2047 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x00,0x0d,0xec,0x00,0x00,0x90,0x01,0x01,0xff,0x07,0x00]
+0x7c,0x00,0x0d,0xec,0x00,0x00,0x90,0x01,0x01,0xff,0x07,0x00
+
+# GFX12: flat_atomic_cmpswap_b32 v0, v[1:2], v[3:4] offset:2047 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x00,0x0d,0xec,0x00,0x00,0x90,0x01,0x01,0xff,0x07,0x00]
+0x7c,0x00,0x0d,0xec,0x00,0x00,0x90,0x01,0x01,0xff,0x07,0x00
+
+# GFX12: flat_atomic_cmpswap_b32 v1, v[0:1], v[2:3] offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x00,0x0d,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x00,0x0d,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_atomic_cmpswap_b32 v[0:1], v[2:3] offset:64 ; encoding: [0x7c,0x00,0x0d,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x00,0x0d,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_atomic_cmpswap_b64 v[0:1], v[2:5] offset:64 ; encoding: [0x7c,0x80,0x10,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x80,0x10,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_atomic_cmpswap_b64 v[1:2], v[0:1], v[2:5] offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x80,0x10,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x80,0x10,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_atomic_cmpswap_b64 v[1:2], v[3:4], v[5:8] th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x80,0x10,0xec,0x01,0x00,0x90,0x02,0x03,0x00,0x00,0x00]
+0x7c,0x80,0x10,0xec,0x01,0x00,0x90,0x02,0x03,0x00,0x00,0x00
+
+# GFX12: flat_atomic_cmpswap_b64 v[1:2], v[3:4], v[5:8] th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x80,0x10,0xec,0x01,0x00,0x90,0x02,0x03,0x00,0x00,0x00]
+0x7c,0x80,0x10,0xec,0x01,0x00,0x90,0x02,0x03,0x00,0x00,0x00
+
+# GFX12: flat_atomic_cmpswap_b64 v[1:2], v[3:4], v[5:8] offset:2047 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x80,0x10,0xec,0x01,0x00,0x90,0x02,0x03,0xff,0x07,0x00]
+0x7c,0x80,0x10,0xec,0x01,0x00,0x90,0x02,0x03,0xff,0x07,0x00
+
+# GFX12: flat_atomic_cmpswap_b64 v[1:2], v[3:4], v[5:8] offset:2047 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x80,0x10,0xec,0x01,0x00,0x90,0x02,0x03,0xff,0x07,0x00]
+0x7c,0x80,0x10,0xec,0x01,0x00,0x90,0x02,0x03,0xff,0x07,0x00
+
+# GFX12: flat_atomic_dec_u32 v1, v[0:1], v2 offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x00,0x10,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x00,0x10,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_atomic_dec_u32 v[0:1], v2 offset:64 ; encoding: [0x7c,0x00,0x10,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x00,0x10,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_atomic_dec_u64 v[0:1], v[2:3] offset:64 ; encoding: [0x7c,0x40,0x13,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x40,0x13,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_atomic_dec_u64 v[1:2], v[0:1], v[2:3] offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x40,0x13,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x40,0x13,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_atomic_inc_u32 v1, v[0:1], v2 offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0xc0,0x0f,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x7c,0xc0,0x0f,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_atomic_inc_u32 v[0:1], v2 offset:64 ; encoding: [0x7c,0xc0,0x0f,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0xc0,0x0f,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_atomic_inc_u64 v[0:1], v[2:3] offset:64 ; encoding: [0x7c,0x00,0x13,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x00,0x13,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_atomic_inc_u64 v[1:2], v[0:1], v[2:3] offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x00,0x13,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x00,0x13,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_atomic_max_num_f32 v1, v[0:1], v2 offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x80,0x14,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x80,0x14,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_atomic_max_num_f32 v[0:1], v2 offset:64 ; encoding: [0x7c,0x80,0x14,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x80,0x14,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_atomic_max_i32 v1, v[0:1], v2 offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x80,0x0e,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x80,0x0e,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_atomic_max_i32 v[0:1], v2 offset:64 ; encoding: [0x7c,0x80,0x0e,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x80,0x0e,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_atomic_max_i64 v[0:1], v[2:3] offset:64 ; encoding: [0x7c,0xc0,0x11,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0xc0,0x11,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_atomic_max_i64 v[1:2], v[0:1], v[2:3] offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0xc0,0x11,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x7c,0xc0,0x11,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_atomic_max_u32 v1, v[0:1], v2 offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0xc0,0x0e,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x7c,0xc0,0x0e,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_atomic_max_u32 v[0:1], v2 offset:64 ; encoding: [0x7c,0xc0,0x0e,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0xc0,0x0e,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_atomic_max_u64 v[0:1], v[2:3] offset:64 ; encoding: [0x7c,0x00,0x12,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x00,0x12,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_atomic_max_u64 v[1:2], v[0:1], v[2:3] offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x00,0x12,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x00,0x12,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_atomic_min_num_f32 v1, v[0:1], v2 offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x40,0x14,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x40,0x14,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_atomic_min_num_f32 v[0:1], v2 offset:64 ; encoding: [0x7c,0x40,0x14,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x40,0x14,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_atomic_min_i32 v1, v[0:1], v2 offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x00,0x0e,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x00,0x0e,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_atomic_min_i32 v[0:1], v2 offset:64 ; encoding: [0x7c,0x00,0x0e,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x00,0x0e,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_atomic_min_i64 v[0:1], v[2:3] offset:64 ; encoding: [0x7c,0x40,0x11,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x40,0x11,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_atomic_min_i64 v[1:2], v[0:1], v[2:3] offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x40,0x11,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x40,0x11,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_atomic_min_u32 v1, v[0:1], v2 offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x40,0x0e,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x40,0x0e,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_atomic_min_u32 v[0:1], v2 offset:64 ; encoding: [0x7c,0x40,0x0e,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x40,0x0e,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_atomic_min_u64 v[0:1], v[2:3] offset:64 ; encoding: [0x7c,0x80,0x11,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x80,0x11,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_atomic_min_u64 v[1:2], v[0:1], v[2:3] offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x80,0x11,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x80,0x11,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_atomic_or_b32 v1, v[0:1], v2 offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x40,0x0f,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x40,0x0f,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_atomic_or_b32 v[0:1], v2 offset:64 ; encoding: [0x7c,0x40,0x0f,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x40,0x0f,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_atomic_or_b64 v[0:1], v[2:3] offset:64 ; encoding: [0x7c,0x80,0x12,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x80,0x12,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_atomic_or_b64 v[1:2], v[0:1], v[2:3] offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x80,0x12,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x80,0x12,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_atomic_sub_clamp_u32 v1, v[0:1], v2 offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0xc0,0x0d,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x7c,0xc0,0x0d,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_atomic_sub_clamp_u32 v[0:1], v2 offset:64 ; encoding: [0x7c,0xc0,0x0d,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0xc0,0x0d,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_atomic_sub_u32 v1, v[0:1], v2 offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x80,0x0d,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x80,0x0d,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_atomic_sub_u32 v[0:1], v2 offset:64 ; encoding: [0x7c,0x80,0x0d,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x80,0x0d,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_atomic_sub_u64 v[0:1], v[2:3] offset:64 ; encoding: [0x7c,0x00,0x11,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x00,0x11,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_atomic_sub_u64 v[1:2], v[0:1], v[2:3] offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x00,0x11,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x00,0x11,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_atomic_swap_b32 v0, v[1:2], v3 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0xc0,0x0c,0xec,0x00,0x00,0x90,0x01,0x01,0x00,0x00,0x00]
+0x7c,0xc0,0x0c,0xec,0x00,0x00,0x90,0x01,0x01,0x00,0x00,0x00
+
+# GFX12: flat_atomic_swap_b32 v0, v[1:2], v3 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0xc0,0x0c,0xec,0x00,0x00,0x90,0x01,0x01,0x00,0x00,0x00]
+0x7c,0xc0,0x0c,0xec,0x00,0x00,0x90,0x01,0x01,0x00,0x00,0x00
+
+# GFX12: flat_atomic_swap_b32 v0, v[1:2], v3 offset:2047 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0xc0,0x0c,0xec,0x00,0x00,0x90,0x01,0x01,0xff,0x07,0x00]
+0x7c,0xc0,0x0c,0xec,0x00,0x00,0x90,0x01,0x01,0xff,0x07,0x00
+
+# GFX12: flat_atomic_swap_b32 v0, v[1:2], v3 offset:2047 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0xc0,0x0c,0xec,0x00,0x00,0x90,0x01,0x01,0xff,0x07,0x00]
+0x7c,0xc0,0x0c,0xec,0x00,0x00,0x90,0x01,0x01,0xff,0x07,0x00
+
+# GFX12: flat_atomic_swap_b32 v0, v[1:2], v3 offset:2048 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0xc0,0x0c,0xec,0x00,0x00,0x90,0x01,0x01,0x00,0x08,0x00]
+0x7c,0xc0,0x0c,0xec,0x00,0x00,0x90,0x01,0x01,0x00,0x08,0x00
+
+# GFX12: flat_atomic_swap_b32 v1, v[0:1], v2 offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0xc0,0x0c,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x7c,0xc0,0x0c,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_atomic_swap_b32 v[0:1], v2 offset:64 ; encoding: [0x7c,0xc0,0x0c,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0xc0,0x0c,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_atomic_swap_b64 v[0:1], v[2:3] offset:64 ; encoding: [0x7c,0x40,0x10,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x40,0x10,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_atomic_swap_b64 v[1:2], v[0:1], v[2:3] offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x40,0x10,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x40,0x10,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_atomic_swap_b64 v[1:2], v[3:4], v[5:6] th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x40,0x10,0xec,0x01,0x00,0x90,0x02,0x03,0x00,0x00,0x00]
+0x7c,0x40,0x10,0xec,0x01,0x00,0x90,0x02,0x03,0x00,0x00,0x00
+
+# GFX12: flat_atomic_swap_b64 v[1:2], v[3:4], v[5:6] th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x40,0x10,0xec,0x01,0x00,0x90,0x02,0x03,0x00,0x00,0x00]
+0x7c,0x40,0x10,0xec,0x01,0x00,0x90,0x02,0x03,0x00,0x00,0x00
+
+# GFX12: flat_atomic_swap_b64 v[1:2], v[3:4], v[5:6] offset:2047 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x40,0x10,0xec,0x01,0x00,0x90,0x02,0x03,0xff,0x07,0x00]
+0x7c,0x40,0x10,0xec,0x01,0x00,0x90,0x02,0x03,0xff,0x07,0x00
+
+# GFX12: flat_atomic_swap_b64 v[1:2], v[3:4], v[5:6] offset:2047 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x40,0x10,0xec,0x01,0x00,0x90,0x02,0x03,0xff,0x07,0x00]
+0x7c,0x40,0x10,0xec,0x01,0x00,0x90,0x02,0x03,0xff,0x07,0x00
+
+# GFX12: flat_atomic_xor_b32 v1, v[0:1], v2 offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x80,0x0f,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x80,0x0f,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_atomic_xor_b32 v[0:1], v2 offset:64 ; encoding: [0x7c,0x80,0x0f,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x80,0x0f,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_atomic_xor_b64 v[0:1], v[2:3] offset:64 ; encoding: [0x7c,0xc0,0x12,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0xc0,0x12,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_atomic_xor_b64 v[1:2], v[0:1], v[2:3] offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0xc0,0x12,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x7c,0xc0,0x12,0xec,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_load_b128 v[1:4], v[0:1] offset:64 ; encoding: [0x7c,0xc0,0x05,0xec,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x7c,0xc0,0x05,0xec,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: flat_load_b128 v[1:4], v[5:6]           ; encoding: [0x7c,0xc0,0x05,0xec,0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00]
+0x7c,0xc0,0x05,0xec,0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00
+
+# GFX12: flat_load_b32 v1, v[0:1] offset:64     ; encoding: [0x7c,0x00,0x05,0xec,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x7c,0x00,0x05,0xec,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: flat_load_b32 v1, v[3:4] offset:2047   ; encoding: [0x7c,0x00,0x05,0xec,0x01,0x00,0x00,0x00,0x03,0xff,0x07,0x00]
+0x7c,0x00,0x05,0xec,0x01,0x00,0x00,0x00,0x03,0xff,0x07,0x00
+
+# GFX12: flat_load_b32 v1, v[3:4] offset:2048   ; encoding: [0x7c,0x00,0x05,0xec,0x01,0x00,0x00,0x00,0x03,0x00,0x08,0x00]
+0x7c,0x00,0x05,0xec,0x01,0x00,0x00,0x00,0x03,0x00,0x08,0x00
+
+# GFX12: flat_load_b32 v1, v[3:4] offset:4      ; encoding: [0x7c,0x00,0x05,0xec,0x01,0x00,0x00,0x00,0x03,0x04,0x00,0x00]
+0x7c,0x00,0x05,0xec,0x01,0x00,0x00,0x00,0x03,0x04,0x00,0x00
+
+# GFX12: flat_load_b32 v1, v[3:4] offset:4      ; encoding: [0x7c,0x00,0x05,0xec,0x01,0x00,0x00,0x00,0x03,0x04,0x00,0x00]
+0x7c,0x00,0x05,0xec,0x01,0x00,0x00,0x00,0x03,0x04,0x00,0x00
+
+# GFX12: flat_load_b32 v1, v[3:4] offset:4      ; encoding: [0x7c,0x00,0x05,0xec,0x01,0x00,0x00,0x00,0x03,0x04,0x00,0x00]
+0x7c,0x00,0x05,0xec,0x01,0x00,0x00,0x00,0x03,0x04,0x00,0x00
+
+# GFX12: flat_load_b32 v1, v[3:4]                ; encoding: [0x7c,0x00,0x05,0xec,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+0x7c,0x00,0x05,0xec,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00
+
+# GFX12: flat_load_b64 v[1:2], v[0:1] offset:64 ; encoding: [0x7c,0x40,0x05,0xec,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x7c,0x40,0x05,0xec,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: flat_load_b64 v[1:2], v[3:4]            ; encoding: [0x7c,0x40,0x05,0xec,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+0x7c,0x40,0x05,0xec,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00
+
+# GFX12: flat_load_b96 v[1:3], v[0:1] offset:64 ; encoding: [0x7c,0x80,0x05,0xec,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x7c,0x80,0x05,0xec,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: flat_load_b96 v[1:3], v[5:6]            ; encoding: [0x7c,0x80,0x05,0xec,0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00]
+0x7c,0x80,0x05,0xec,0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00
+
+# GFX12: flat_load_d16_b16 v1, v[0:1] offset:64 ; encoding: [0x7c,0x00,0x08,0xec,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x7c,0x00,0x08,0xec,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: flat_load_d16_b16 v1, v[3:4]            ; encoding: [0x7c,0x00,0x08,0xec,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+0x7c,0x00,0x08,0xec,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00
+
+# GFX12: flat_load_d16_hi_b16 v1, v[0:1] offset:64 ; encoding: [0x7c,0xc0,0x08,0xec,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x7c,0xc0,0x08,0xec,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: flat_load_d16_hi_b16 v1, v[3:4]         ; encoding: [0x7c,0xc0,0x08,0xec,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+0x7c,0xc0,0x08,0xec,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00
+
+# GFX12: flat_load_d16_hi_i8 v1, v[0:1] offset:64 ; encoding: [0x7c,0x80,0x08,0xec,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x7c,0x80,0x08,0xec,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: flat_load_d16_hi_i8 v1, v[3:4]          ; encoding: [0x7c,0x80,0x08,0xec,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+0x7c,0x80,0x08,0xec,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00
+
+# GFX12: flat_load_d16_hi_u8 v1, v[0:1] offset:64 ; encoding: [0x7c,0x40,0x08,0xec,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x7c,0x40,0x08,0xec,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: flat_load_d16_hi_u8 v1, v[3:4]          ; encoding: [0x7c,0x40,0x08,0xec,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+0x7c,0x40,0x08,0xec,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00
+
+# GFX12: flat_load_d16_i8 v1, v[0:1] offset:64  ; encoding: [0x7c,0xc0,0x07,0xec,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x7c,0xc0,0x07,0xec,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: flat_load_d16_i8 v1, v[3:4]             ; encoding: [0x7c,0xc0,0x07,0xec,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+0x7c,0xc0,0x07,0xec,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00
+
+# GFX12: flat_load_d16_u8 v1, v[0:1] offset:64  ; encoding: [0x7c,0x80,0x07,0xec,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x7c,0x80,0x07,0xec,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: flat_load_d16_u8 v1, v[3:4]             ; encoding: [0x7c,0x80,0x07,0xec,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+0x7c,0x80,0x07,0xec,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00
+
+# GFX12: flat_load_i16 v1, v[0:1] offset:64     ; encoding: [0x7c,0xc0,0x04,0xec,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x7c,0xc0,0x04,0xec,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: flat_load_i16 v1, v[3:4]                ; encoding: [0x7c,0xc0,0x04,0xec,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+0x7c,0xc0,0x04,0xec,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00
+
+# GFX12: flat_load_i8 v1, v[0:1] offset:64      ; encoding: [0x7c,0x40,0x04,0xec,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x7c,0x40,0x04,0xec,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: flat_load_i8 v1, v[3:4]                 ; encoding: [0x7c,0x40,0x04,0xec,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+0x7c,0x40,0x04,0xec,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00
+
+# GFX12: flat_load_u16 v1, v[0:1] offset:64     ; encoding: [0x7c,0x80,0x04,0xec,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x7c,0x80,0x04,0xec,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: flat_load_u16 v1, v[3:4]                ; encoding: [0x7c,0x80,0x04,0xec,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+0x7c,0x80,0x04,0xec,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00
+
+# GFX12: flat_load_u8 v1, v[0:1] offset:64      ; encoding: [0x7c,0x00,0x04,0xec,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x7c,0x00,0x04,0xec,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: flat_load_u8 v1, v[3:4]                 ; encoding: [0x7c,0x00,0x04,0xec,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+0x7c,0x00,0x04,0xec,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00
+
+# GFX12: flat_store_b128 v[0:1], v[2:5] offset:64 ; encoding: [0x7c,0x40,0x07,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x40,0x07,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_store_b128 v[1:2], v[3:6]          ; encoding: [0x7c,0x40,0x07,0xec,0x00,0x00,0x80,0x01,0x01,0x00,0x00,0x00]
+0x7c,0x40,0x07,0xec,0x00,0x00,0x80,0x01,0x01,0x00,0x00,0x00
+
+# GFX12: flat_store_b16 v[0:1], v2 offset:64    ; encoding: [0x7c,0x40,0x06,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x40,0x06,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_store_b16 v[3:4], v1               ; encoding: [0x7c,0x40,0x06,0xec,0x00,0x00,0x80,0x00,0x03,0x00,0x00,0x00]
+0x7c,0x40,0x06,0xec,0x00,0x00,0x80,0x00,0x03,0x00,0x00,0x00
+
+# GFX12: flat_store_b32 v[0:1], v2 offset:64    ; encoding: [0x7c,0x80,0x06,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x80,0x06,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_store_b32 v[3:4], v1 offset:16    ; encoding: [0x7c,0x80,0x06,0xec,0x00,0x00,0x80,0x00,0x03,0x10,0x00,0x00]
+0x7c,0x80,0x06,0xec,0x00,0x00,0x80,0x00,0x03,0x10,0x00,0x00
+
+# GFX12: flat_store_b64 v[0:1], v[2:3] offset:64 ; encoding: [0x7c,0xc0,0x06,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0xc0,0x06,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_store_b64 v[1:2], v[3:4]           ; encoding: [0x7c,0xc0,0x06,0xec,0x00,0x00,0x80,0x01,0x01,0x00,0x00,0x00]
+0x7c,0xc0,0x06,0xec,0x00,0x00,0x80,0x01,0x01,0x00,0x00,0x00
+
+# GFX12: flat_store_b8 v[0:1], v2 offset:64     ; encoding: [0x7c,0x00,0x06,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x00,0x06,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_store_b8 v[3:4], v1                ; encoding: [0x7c,0x00,0x06,0xec,0x00,0x00,0x80,0x00,0x03,0x00,0x00,0x00]
+0x7c,0x00,0x06,0xec,0x00,0x00,0x80,0x00,0x03,0x00,0x00,0x00
+
+# GFX12: flat_store_b96 v[0:1], v[2:4] offset:64 ; encoding: [0x7c,0x00,0x07,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x00,0x07,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_store_b96 v[1:2], v[3:5]           ; encoding: [0x7c,0x00,0x07,0xec,0x00,0x00,0x80,0x01,0x01,0x00,0x00,0x00]
+0x7c,0x00,0x07,0xec,0x00,0x00,0x80,0x01,0x01,0x00,0x00,0x00
+
+# GFX12: flat_store_d16_hi_b16 v[0:1], v2 offset:64 ; encoding: [0x7c,0x40,0x09,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x40,0x09,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_store_d16_hi_b16 v[3:4], v1        ; encoding: [0x7c,0x40,0x09,0xec,0x00,0x00,0x80,0x00,0x03,0x00,0x00,0x00]
+0x7c,0x40,0x09,0xec,0x00,0x00,0x80,0x00,0x03,0x00,0x00,0x00
+
+# GFX12: flat_store_d16_hi_b8 v[0:1], v2 offset:64 ; encoding: [0x7c,0x00,0x09,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x00,0x09,0xec,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: flat_store_d16_hi_b8 v[3:4], v1         ; encoding: [0x7c,0x00,0x09,0xec,0x00,0x00,0x80,0x00,0x03,0x00,0x00,0x00]
+0x7c,0x00,0x09,0xec,0x00,0x00,0x80,0x00,0x03,0x00,0x00,0x00
+
+# GFX12: global_atomic_add_f32 v0, v2, s[0:1] offset:64 ; encoding: [0x00,0x80,0x15,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x00,0x80,0x15,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_add_f32 v1, v0, v2, s[0:1] offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x00,0x80,0x15,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x00,0x80,0x15,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_add_f32 v1, v[0:1], v2, off offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x80,0x15,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x80,0x15,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_add_f32 v[0:1], v2, off offset:64 ; encoding: [0x7c,0x80,0x15,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x80,0x15,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_add_u32 v0, v2, s[0:1] offset:64 ; encoding: [0x00,0x40,0x0d,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x00,0x40,0x0d,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_add_u32 v1, v0, v2, s[0:1] offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x00,0x40,0x0d,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x00,0x40,0x0d,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_add_u32 v1, v[0:1], v2, off offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x40,0x0d,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x40,0x0d,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_add_u32 v[0:1], v2, off offset:64 ; encoding: [0x7c,0x40,0x0d,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x40,0x0d,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_add_u64 v0, v[2:3], s[0:1] offset:64 ; encoding: [0x00,0xc0,0x10,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x00,0xc0,0x10,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_add_u64 v[0:1], v[2:3], off offset:64 ; encoding: [0x7c,0xc0,0x10,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0xc0,0x10,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_add_u64 v[1:2], v0, v[2:3], s[0:1] offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x00,0xc0,0x10,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x00,0xc0,0x10,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_add_u64 v[1:2], v[0:1], v[2:3], off offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0xc0,0x10,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x7c,0xc0,0x10,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_and_b32 v0, v2, s[0:1] offset:64 ; encoding: [0x00,0x00,0x0f,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x00,0x00,0x0f,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_and_b32 v1, v0, v2, s[0:1] offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x00,0x00,0x0f,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x00,0x00,0x0f,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_and_b32 v1, v[0:1], v2, off offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x00,0x0f,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x00,0x0f,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_and_b32 v[0:1], v2, off offset:64 ; encoding: [0x7c,0x00,0x0f,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x00,0x0f,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_and_b64 v0, v[2:3], s[0:1] offset:64 ; encoding: [0x00,0x40,0x12,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x00,0x40,0x12,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_and_b64 v[0:1], v[2:3], off offset:64 ; encoding: [0x7c,0x40,0x12,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x40,0x12,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_and_b64 v[1:2], v0, v[2:3], s[0:1] offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x00,0x40,0x12,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x00,0x40,0x12,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_and_b64 v[1:2], v[0:1], v[2:3], off offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x40,0x12,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x40,0x12,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_cmpswap_b32 v0, v1, v[2:3], s[2:3] th:TH_ATOMIC_RETURN ; encoding: [0x02,0x00,0x0d,0xee,0x00,0x00,0x10,0x01,0x01,0x00,0x00,0x00]
+0x02,0x00,0x0d,0xee,0x00,0x00,0x10,0x01,0x01,0x00,0x00,0x00
+
+# GFX12: global_atomic_cmpswap_b32 v0, v1, v[2:3], s[2:3] th:TH_ATOMIC_RETURN ; encoding: [0x02,0x00,0x0d,0xee,0x00,0x00,0x10,0x01,0x01,0x00,0x00,0x00]
+0x02,0x00,0x0d,0xee,0x00,0x00,0x10,0x01,0x01,0x00,0x00,0x00
+
+# GFX12: global_atomic_cmpswap_b32 v0, v1, v[2:3], s[2:3] offset:2047 th:TH_ATOMIC_RETURN ; encoding: [0x02,0x00,0x0d,0xee,0x00,0x00,0x10,0x01,0x01,0xff,0x07,0x00]
+0x02,0x00,0x0d,0xee,0x00,0x00,0x10,0x01,0x01,0xff,0x07,0x00
+
+# GFX12: global_atomic_cmpswap_b32 v0, v1, v[2:3], s[2:3] offset:2047 th:TH_ATOMIC_RETURN ; encoding: [0x02,0x00,0x0d,0xee,0x00,0x00,0x10,0x01,0x01,0xff,0x07,0x00]
+0x02,0x00,0x0d,0xee,0x00,0x00,0x10,0x01,0x01,0xff,0x07,0x00
+
+# GFX12: global_atomic_cmpswap_b32 v0, v[1:2], v[3:4], off th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x00,0x0d,0xee,0x00,0x00,0x90,0x01,0x01,0x00,0x00,0x00]
+0x7c,0x00,0x0d,0xee,0x00,0x00,0x90,0x01,0x01,0x00,0x00,0x00
+
+# GFX12: global_atomic_cmpswap_b32 v0, v[1:2], v[3:4], off th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x00,0x0d,0xee,0x00,0x00,0x90,0x01,0x01,0x00,0x00,0x00]
+0x7c,0x00,0x0d,0xee,0x00,0x00,0x90,0x01,0x01,0x00,0x00,0x00
+
+# GFX12: global_atomic_cmpswap_b32 v0, v[1:2], v[3:4], off offset:2047 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x00,0x0d,0xee,0x00,0x00,0x90,0x01,0x01,0xff,0x07,0x00]
+0x7c,0x00,0x0d,0xee,0x00,0x00,0x90,0x01,0x01,0xff,0x07,0x00
+
+# GFX12: global_atomic_cmpswap_b32 v0, v[1:2], v[3:4], off offset:2047 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x00,0x0d,0xee,0x00,0x00,0x90,0x01,0x01,0xff,0x07,0x00]
+0x7c,0x00,0x0d,0xee,0x00,0x00,0x90,0x01,0x01,0xff,0x07,0x00
+
+# GFX12: global_atomic_cmpswap_b32 v0, v[2:3], s[0:1] offset:64 ; encoding: [0x00,0x00,0x0d,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x00,0x00,0x0d,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_cmpswap_b32 v1, v0, v[2:3], s[0:1] offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x00,0x00,0x0d,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x00,0x00,0x0d,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_cmpswap_b32 v1, v[0:1], v[2:3], off offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x00,0x0d,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x00,0x0d,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_cmpswap_b32 v[0:1], v[2:3], off offset:64 ; encoding: [0x7c,0x00,0x0d,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x00,0x0d,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_cmpswap_b64 v0, v[2:5], s[0:1] offset:64 ; encoding: [0x00,0x80,0x10,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x00,0x80,0x10,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_cmpswap_b64 v[0:1], v[2:5], off offset:64 ; encoding: [0x7c,0x80,0x10,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x80,0x10,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_cmpswap_b64 v[1:2], v0, v[2:5], s[0:1] offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x00,0x80,0x10,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x00,0x80,0x10,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_cmpswap_b64 v[1:2], v3, v[5:8], s[2:3] th:TH_ATOMIC_RETURN ; encoding: [0x02,0x80,0x10,0xee,0x01,0x00,0x90,0x02,0x03,0x00,0x00,0x00]
+0x02,0x80,0x10,0xee,0x01,0x00,0x90,0x02,0x03,0x00,0x00,0x00
+
+# GFX12: global_atomic_cmpswap_b64 v[1:2], v3, v[5:8], s[2:3] th:TH_ATOMIC_RETURN ; encoding: [0x02,0x80,0x10,0xee,0x01,0x00,0x90,0x02,0x03,0x00,0x00,0x00]
+0x02,0x80,0x10,0xee,0x01,0x00,0x90,0x02,0x03,0x00,0x00,0x00
+
+# GFX12: global_atomic_cmpswap_b64 v[1:2], v3, v[5:8], s[2:3] offset:2047 th:TH_ATOMIC_RETURN ; encoding: [0x02,0x80,0x10,0xee,0x01,0x00,0x90,0x02,0x03,0xff,0x07,0x00]
+0x02,0x80,0x10,0xee,0x01,0x00,0x90,0x02,0x03,0xff,0x07,0x00
+
+# GFX12: global_atomic_cmpswap_b64 v[1:2], v3, v[5:8], s[2:3] offset:2047 th:TH_ATOMIC_RETURN ; encoding: [0x02,0x80,0x10,0xee,0x01,0x00,0x90,0x02,0x03,0xff,0x07,0x00]
+0x02,0x80,0x10,0xee,0x01,0x00,0x90,0x02,0x03,0xff,0x07,0x00
+
+# GFX12: global_atomic_cmpswap_b64 v[1:2], v[0:1], v[2:5], off offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x80,0x10,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x80,0x10,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_cmpswap_b64 v[1:2], v[3:4], v[5:8], off th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x80,0x10,0xee,0x01,0x00,0x90,0x02,0x03,0x00,0x00,0x00]
+0x7c,0x80,0x10,0xee,0x01,0x00,0x90,0x02,0x03,0x00,0x00,0x00
+
+# GFX12: global_atomic_cmpswap_b64 v[1:2], v[3:4], v[5:8], off th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x80,0x10,0xee,0x01,0x00,0x90,0x02,0x03,0x00,0x00,0x00]
+0x7c,0x80,0x10,0xee,0x01,0x00,0x90,0x02,0x03,0x00,0x00,0x00
+
+# GFX12: global_atomic_cmpswap_b64 v[1:2], v[3:4], v[5:8], off offset:2047 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x80,0x10,0xee,0x01,0x00,0x90,0x02,0x03,0xff,0x07,0x00]
+0x7c,0x80,0x10,0xee,0x01,0x00,0x90,0x02,0x03,0xff,0x07,0x00
+
+# GFX12: global_atomic_cmpswap_b64 v[1:2], v[3:4], v[5:8], off offset:2047 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x80,0x10,0xee,0x01,0x00,0x90,0x02,0x03,0xff,0x07,0x00]
+0x7c,0x80,0x10,0xee,0x01,0x00,0x90,0x02,0x03,0xff,0x07,0x00
+
+# GFX12: global_atomic_sub_clamp_u32 v1, v0, v2, s[0:1] offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x00,0xc0,0x0d,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x00,0xc0,0x0d,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_sub_clamp_u32 v0, v2, s[0:1] offset:64 ; encoding: [0x00,0xc0,0x0d,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x00,0xc0,0x0d,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_sub_clamp_u32 v1, v[0:1], v2, off offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0xc0,0x0d,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x7c,0xc0,0x0d,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_sub_clamp_u32 v[0:1], v2, off offset:64 ; encoding: [0x7c,0xc0,0x0d,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0xc0,0x0d,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_dec_u32 v0, v2, s[0:1] offset:64 ; encoding: [0x00,0x00,0x10,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x00,0x00,0x10,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_dec_u32 v1, v0, v2, s[0:1] offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x00,0x00,0x10,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x00,0x00,0x10,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_dec_u32 v1, v[0:1], v2, off offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x00,0x10,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x00,0x10,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_dec_u32 v[0:1], v2, off offset:64 ; encoding: [0x7c,0x00,0x10,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x00,0x10,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_dec_u64 v0, v[2:3], s[0:1] offset:64 ; encoding: [0x00,0x40,0x13,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x00,0x40,0x13,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_dec_u64 v[0:1], v[2:3], off offset:64 ; encoding: [0x7c,0x40,0x13,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x40,0x13,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_dec_u64 v[1:2], v0, v[2:3], s[0:1] offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x00,0x40,0x13,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x00,0x40,0x13,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_dec_u64 v[1:2], v[0:1], v[2:3], off offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x40,0x13,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x40,0x13,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_inc_u32 v0, v2, s[0:1] offset:64 ; encoding: [0x00,0xc0,0x0f,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x00,0xc0,0x0f,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_inc_u32 v1, v0, v2, s[0:1] offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x00,0xc0,0x0f,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x00,0xc0,0x0f,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_inc_u32 v1, v[0:1], v2, off offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0xc0,0x0f,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x7c,0xc0,0x0f,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_inc_u32 v[0:1], v2, off offset:64 ; encoding: [0x7c,0xc0,0x0f,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0xc0,0x0f,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_inc_u64 v0, v[2:3], s[0:1] offset:64 ; encoding: [0x00,0x00,0x13,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x00,0x00,0x13,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_inc_u64 v[0:1], v[2:3], off offset:64 ; encoding: [0x7c,0x00,0x13,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x00,0x13,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_inc_u64 v[1:2], v0, v[2:3], s[0:1] offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x00,0x00,0x13,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x00,0x00,0x13,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_inc_u64 v[1:2], v[0:1], v[2:3], off offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x00,0x13,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x00,0x13,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_max_num_f32 v0, v2, s[0:1] offset:64 ; encoding: [0x00,0x80,0x14,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x00,0x80,0x14,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_max_num_f32 v1, v0, v2, s[0:1] offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x00,0x80,0x14,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x00,0x80,0x14,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_max_num_f32 v1, v[0:1], v2, off offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x80,0x14,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x80,0x14,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_max_num_f32 v[0:1], v2, off offset:64 ; encoding: [0x7c,0x80,0x14,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x80,0x14,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_max_i32 v0, v2, s[0:1] offset:64 ; encoding: [0x00,0x80,0x0e,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x00,0x80,0x0e,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_max_i32 v1, v0, v2, s[0:1] offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x00,0x80,0x0e,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x00,0x80,0x0e,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_max_i32 v1, v[0:1], v2, off offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x80,0x0e,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x80,0x0e,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_max_i32 v[0:1], v2, off offset:64 ; encoding: [0x7c,0x80,0x0e,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x80,0x0e,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_max_i64 v0, v[2:3], s[0:1] offset:64 ; encoding: [0x00,0xc0,0x11,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x00,0xc0,0x11,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_max_i64 v[0:1], v[2:3], off offset:64 ; encoding: [0x7c,0xc0,0x11,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0xc0,0x11,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_max_i64 v[1:2], v0, v[2:3], s[0:1] offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x00,0xc0,0x11,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x00,0xc0,0x11,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_max_i64 v[1:2], v[0:1], v[2:3], off offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0xc0,0x11,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x7c,0xc0,0x11,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_max_u32 v0, v2, s[0:1] offset:64 ; encoding: [0x00,0xc0,0x0e,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x00,0xc0,0x0e,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_max_u32 v1, v0, v2, s[0:1] offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x00,0xc0,0x0e,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x00,0xc0,0x0e,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_max_u32 v1, v[0:1], v2, off offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0xc0,0x0e,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x7c,0xc0,0x0e,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_max_u32 v[0:1], v2, off offset:64 ; encoding: [0x7c,0xc0,0x0e,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0xc0,0x0e,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_max_u64 v0, v[2:3], s[0:1] offset:64 ; encoding: [0x00,0x00,0x12,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x00,0x00,0x12,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_max_u64 v[0:1], v[2:3], off offset:64 ; encoding: [0x7c,0x00,0x12,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x00,0x12,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_max_u64 v[1:2], v0, v[2:3], s[0:1] offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x00,0x00,0x12,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x00,0x00,0x12,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_max_u64 v[1:2], v[0:1], v[2:3], off offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x00,0x12,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x00,0x12,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_min_num_f32 v0, v2, s[0:1] offset:64 ; encoding: [0x00,0x40,0x14,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x00,0x40,0x14,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_min_num_f32 v1, v0, v2, s[0:1] offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x00,0x40,0x14,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x00,0x40,0x14,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_min_num_f32 v1, v[0:1], v2, off offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x40,0x14,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x40,0x14,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_min_num_f32 v[0:1], v2, off offset:64 ; encoding: [0x7c,0x40,0x14,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x40,0x14,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_min_i32 v0, v2, s[0:1] offset:64 ; encoding: [0x00,0x00,0x0e,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x00,0x00,0x0e,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_min_i32 v1, v0, v2, s[0:1] offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x00,0x00,0x0e,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x00,0x00,0x0e,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_min_i32 v1, v[0:1], v2, off offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x00,0x0e,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x00,0x0e,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_min_i32 v[0:1], v2, off offset:64 ; encoding: [0x7c,0x00,0x0e,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x00,0x0e,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_min_i64 v0, v[2:3], s[0:1] offset:64 ; encoding: [0x00,0x40,0x11,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x00,0x40,0x11,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_min_i64 v[0:1], v[2:3], off offset:64 ; encoding: [0x7c,0x40,0x11,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x40,0x11,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_min_i64 v[1:2], v0, v[2:3], s[0:1] offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x00,0x40,0x11,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x00,0x40,0x11,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_min_i64 v[1:2], v[0:1], v[2:3], off offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x40,0x11,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x40,0x11,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_min_u32 v0, v2, s[0:1] offset:64 ; encoding: [0x00,0x40,0x0e,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x00,0x40,0x0e,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_min_u32 v1, v0, v2, s[0:1] offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x00,0x40,0x0e,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x00,0x40,0x0e,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_min_u32 v1, v[0:1], v2, off offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x40,0x0e,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x40,0x0e,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_min_u32 v[0:1], v2, off offset:64 ; encoding: [0x7c,0x40,0x0e,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x40,0x0e,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_min_u64 v0, v[2:3], s[0:1] offset:64 ; encoding: [0x00,0x80,0x11,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x00,0x80,0x11,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_min_u64 v[0:1], v[2:3], off offset:64 ; encoding: [0x7c,0x80,0x11,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x80,0x11,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_min_u64 v[1:2], v0, v[2:3], s[0:1] offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x00,0x80,0x11,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x00,0x80,0x11,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_min_u64 v[1:2], v[0:1], v[2:3], off offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x80,0x11,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x80,0x11,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_or_b32 v0, v2, s[0:1] offset:64 ; encoding: [0x00,0x40,0x0f,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x00,0x40,0x0f,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_or_b32 v1, v0, v2, s[0:1] offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x00,0x40,0x0f,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x00,0x40,0x0f,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_or_b32 v1, v[0:1], v2, off offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x40,0x0f,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x40,0x0f,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_or_b32 v[0:1], v2, off offset:64 ; encoding: [0x7c,0x40,0x0f,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x40,0x0f,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_or_b64 v0, v[2:3], s[0:1] offset:64 ; encoding: [0x00,0x80,0x12,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x00,0x80,0x12,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_or_b64 v[0:1], v[2:3], off offset:64 ; encoding: [0x7c,0x80,0x12,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x80,0x12,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_or_b64 v[1:2], v0, v[2:3], s[0:1] offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x00,0x80,0x12,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x00,0x80,0x12,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_or_b64 v[1:2], v[0:1], v[2:3], off offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x80,0x12,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x80,0x12,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_sub_u32 v0, v2, s[0:1] offset:64 ; encoding: [0x00,0x80,0x0d,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x00,0x80,0x0d,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_sub_u32 v1, v0, v2, s[0:1] offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x00,0x80,0x0d,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x00,0x80,0x0d,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_sub_u32 v1, v[0:1], v2, off offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x80,0x0d,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x80,0x0d,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_sub_u32 v[0:1], v2, off offset:64 ; encoding: [0x7c,0x80,0x0d,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x80,0x0d,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_sub_u64 v0, v[2:3], s[0:1] offset:64 ; encoding: [0x00,0x00,0x11,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x00,0x00,0x11,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_sub_u64 v[0:1], v[2:3], off offset:64 ; encoding: [0x7c,0x00,0x11,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x00,0x11,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_sub_u64 v[1:2], v0, v[2:3], s[0:1] offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x00,0x00,0x11,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x00,0x00,0x11,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_sub_u64 v[1:2], v[0:1], v[2:3], off offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x00,0x11,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x00,0x11,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_swap_b32 v0, v1, v3, s[2:3] th:TH_ATOMIC_RETURN ; encoding: [0x02,0xc0,0x0c,0xee,0x00,0x00,0x90,0x01,0x01,0x00,0x00,0x00]
+0x02,0xc0,0x0c,0xee,0x00,0x00,0x90,0x01,0x01,0x00,0x00,0x00
+
+# GFX12: global_atomic_swap_b32 v0, v1, v3, s[2:3] th:TH_ATOMIC_RETURN ; encoding: [0x02,0xc0,0x0c,0xee,0x00,0x00,0x90,0x01,0x01,0x00,0x00,0x00]
+0x02,0xc0,0x0c,0xee,0x00,0x00,0x90,0x01,0x01,0x00,0x00,0x00
+
+# GFX12: global_atomic_swap_b32 v0, v1, v3, s[2:3] offset:2047 th:TH_ATOMIC_RETURN ; encoding: [0x02,0xc0,0x0c,0xee,0x00,0x00,0x90,0x01,0x01,0xff,0x07,0x00]
+0x02,0xc0,0x0c,0xee,0x00,0x00,0x90,0x01,0x01,0xff,0x07,0x00
+
+# GFX12: global_atomic_swap_b32 v0, v1, v3, s[2:3] offset:2047 th:TH_ATOMIC_RETURN ; encoding: [0x02,0xc0,0x0c,0xee,0x00,0x00,0x90,0x01,0x01,0xff,0x07,0x00]
+0x02,0xc0,0x0c,0xee,0x00,0x00,0x90,0x01,0x01,0xff,0x07,0x00
+
+# GFX12: global_atomic_swap_b32 v0, v2, s[0:1] offset:64 ; encoding: [0x00,0xc0,0x0c,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x00,0xc0,0x0c,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_swap_b32 v0, v[1:2], v3, off th:TH_ATOMIC_RETURN ; encoding: [0x7c,0xc0,0x0c,0xee,0x00,0x00,0x90,0x01,0x01,0x00,0x00,0x00]
+0x7c,0xc0,0x0c,0xee,0x00,0x00,0x90,0x01,0x01,0x00,0x00,0x00
+
+# GFX12: global_atomic_swap_b32 v0, v[1:2], v3, off th:TH_ATOMIC_RETURN ; encoding: [0x7c,0xc0,0x0c,0xee,0x00,0x00,0x90,0x01,0x01,0x00,0x00,0x00]
+0x7c,0xc0,0x0c,0xee,0x00,0x00,0x90,0x01,0x01,0x00,0x00,0x00
+
+# GFX12: global_atomic_swap_b32 v0, v[1:2], v3, off offset:2047 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0xc0,0x0c,0xee,0x00,0x00,0x90,0x01,0x01,0xff,0x07,0x00]
+0x7c,0xc0,0x0c,0xee,0x00,0x00,0x90,0x01,0x01,0xff,0x07,0x00
+
+# GFX12: global_atomic_swap_b32 v0, v[1:2], v3, off offset:2047 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0xc0,0x0c,0xee,0x00,0x00,0x90,0x01,0x01,0xff,0x07,0x00]
+0x7c,0xc0,0x0c,0xee,0x00,0x00,0x90,0x01,0x01,0xff,0x07,0x00
+
+# GFX12: global_atomic_swap_b32 v1, v0, v2, s[0:1] offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x00,0xc0,0x0c,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x00,0xc0,0x0c,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_swap_b32 v1, v[0:1], v2, off offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0xc0,0x0c,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x7c,0xc0,0x0c,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_swap_b32 v[0:1], v2, off offset:64 ; encoding: [0x7c,0xc0,0x0c,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0xc0,0x0c,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_swap_b64 v0, v[2:3], s[0:1] offset:64 ; encoding: [0x00,0x40,0x10,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x00,0x40,0x10,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_swap_b64 v[0:1], v[2:3], off offset:64 ; encoding: [0x7c,0x40,0x10,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x40,0x10,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_swap_b64 v[1:2], v0, v[2:3], s[0:1] offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x00,0x40,0x10,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x00,0x40,0x10,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_swap_b64 v[1:2], v3, v[5:6], s[2:3] th:TH_ATOMIC_RETURN ; encoding: [0x02,0x40,0x10,0xee,0x01,0x00,0x90,0x02,0x03,0x00,0x00,0x00]
+0x02,0x40,0x10,0xee,0x01,0x00,0x90,0x02,0x03,0x00,0x00,0x00
+
+# GFX12: global_atomic_swap_b64 v[1:2], v3, v[5:6], s[2:3] th:TH_ATOMIC_RETURN ; encoding: [0x02,0x40,0x10,0xee,0x01,0x00,0x90,0x02,0x03,0x00,0x00,0x00]
+0x02,0x40,0x10,0xee,0x01,0x00,0x90,0x02,0x03,0x00,0x00,0x00
+
+# GFX12: global_atomic_swap_b64 v[1:2], v3, v[5:6], s[2:3] offset:2047 th:TH_ATOMIC_RETURN ; encoding: [0x02,0x40,0x10,0xee,0x01,0x00,0x90,0x02,0x03,0xff,0x07,0x00]
+0x02,0x40,0x10,0xee,0x01,0x00,0x90,0x02,0x03,0xff,0x07,0x00
+
+# GFX12: global_atomic_swap_b64 v[1:2], v3, v[5:6], s[2:3] offset:2047 th:TH_ATOMIC_RETURN ; encoding: [0x02,0x40,0x10,0xee,0x01,0x00,0x90,0x02,0x03,0xff,0x07,0x00]
+0x02,0x40,0x10,0xee,0x01,0x00,0x90,0x02,0x03,0xff,0x07,0x00
+
+# GFX12: global_atomic_swap_b64 v[1:2], v[0:1], v[2:3], off offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x40,0x10,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x40,0x10,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_swap_b64 v[1:2], v[3:4], v[5:6], off th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x40,0x10,0xee,0x01,0x00,0x90,0x02,0x03,0x00,0x00,0x00]
+0x7c,0x40,0x10,0xee,0x01,0x00,0x90,0x02,0x03,0x00,0x00,0x00
+
+# GFX12: global_atomic_swap_b64 v[1:2], v[3:4], v[5:6], off th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x40,0x10,0xee,0x01,0x00,0x90,0x02,0x03,0x00,0x00,0x00]
+0x7c,0x40,0x10,0xee,0x01,0x00,0x90,0x02,0x03,0x00,0x00,0x00
+
+# GFX12: global_atomic_swap_b64 v[1:2], v[3:4], v[5:6], off offset:2047 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x40,0x10,0xee,0x01,0x00,0x90,0x02,0x03,0xff,0x07,0x00]
+0x7c,0x40,0x10,0xee,0x01,0x00,0x90,0x02,0x03,0xff,0x07,0x00
+
+# GFX12: global_atomic_swap_b64 v[1:2], v[3:4], v[5:6], off offset:2047 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x40,0x10,0xee,0x01,0x00,0x90,0x02,0x03,0xff,0x07,0x00]
+0x7c,0x40,0x10,0xee,0x01,0x00,0x90,0x02,0x03,0xff,0x07,0x00
+
+# GFX12: global_atomic_xor_b32 v0, v2, s[0:1] offset:64 ; encoding: [0x00,0x80,0x0f,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x00,0x80,0x0f,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_xor_b32 v1, v0, v2, s[0:1] offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x00,0x80,0x0f,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x00,0x80,0x0f,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_xor_b32 v1, v[0:1], v2, off offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0x80,0x0f,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x80,0x0f,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_xor_b32 v[0:1], v2, off offset:64 ; encoding: [0x7c,0x80,0x0f,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x80,0x0f,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_xor_b64 v0, v[2:3], s[0:1] offset:64 ; encoding: [0x00,0xc0,0x12,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x00,0xc0,0x12,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_xor_b64 v[0:1], v[2:3], off offset:64 ; encoding: [0x7c,0xc0,0x12,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0xc0,0x12,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_xor_b64 v[1:2], v0, v[2:3], s[0:1] offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x00,0xc0,0x12,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x00,0xc0,0x12,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_atomic_xor_b64 v[1:2], v[0:1], v[2:3], off offset:64 th:TH_ATOMIC_RETURN ; encoding: [0x7c,0xc0,0x12,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00]
+0x7c,0xc0,0x12,0xee,0x01,0x00,0x10,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_load_addtid_b32 v1, off offset:64 ; encoding: [0x7c,0x00,0x0a,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x7c,0x00,0x0a,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: global_load_addtid_b32 v1, off          ; encoding: [0x7c,0x00,0x0a,0xee,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00]
+0x7c,0x00,0x0a,0xee,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: global_load_addtid_b32 v1, s[0:1] offset:64 ; encoding: [0x00,0x00,0x0a,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x00,0x00,0x0a,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: global_load_addtid_b32 v1, s[2:3]       ; encoding: [0x02,0x00,0x0a,0xee,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00]
+0x02,0x00,0x0a,0xee,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: global_load_b128 v[1:4], v0, s[0:1] offset:64 ; encoding: [0x00,0xc0,0x05,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x00,0xc0,0x05,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: global_load_b128 v[1:4], v5, s[2:3]     ; encoding: [0x02,0xc0,0x05,0xee,0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00]
+0x02,0xc0,0x05,0xee,0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00
+
+# GFX12: global_load_b128 v[1:4], v[0:1], off offset:64 ; encoding: [0x7c,0xc0,0x05,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x7c,0xc0,0x05,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: global_load_b128 v[1:4], v[5:6], off    ; encoding: [0x7c,0xc0,0x05,0xee,0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00]
+0x7c,0xc0,0x05,0xee,0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00
+
+# GFX12: global_load_b32 v1, v0, s[0:1] offset:64 ; encoding: [0x00,0x00,0x05,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x00,0x00,0x05,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: global_load_b32 v1, v3, s[2:3] offset:2047 ; encoding: [0x02,0x00,0x05,0xee,0x01,0x00,0x00,0x00,0x03,0xff,0x07,0x00]
+0x02,0x00,0x05,0xee,0x01,0x00,0x00,0x00,0x03,0xff,0x07,0x00
+
+# GFX12: global_load_b32 v1, v3, s[2:3]          ; encoding: [0x02,0x00,0x05,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+0x02,0x00,0x05,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00
+
+# GFX12: global_load_b32 v1, v[0:1], off offset:64 ; encoding: [0x7c,0x00,0x05,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x7c,0x00,0x05,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: global_load_b32 v1, v[3:4], off offset:2047 ; encoding: [0x7c,0x00,0x05,0xee,0x01,0x00,0x00,0x00,0x03,0xff,0x07,0x00]
+0x7c,0x00,0x05,0xee,0x01,0x00,0x00,0x00,0x03,0xff,0x07,0x00
+
+# GFX12: global_load_b32 v1, v[3:4], off         ; encoding: [0x7c,0x00,0x05,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+0x7c,0x00,0x05,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00
+
+# GFX12: global_load_b64 v[1:2], v0, s[0:1] offset:64 ; encoding: [0x00,0x40,0x05,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x00,0x40,0x05,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: global_load_b64 v[1:2], v3, s[2:3]      ; encoding: [0x02,0x40,0x05,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+0x02,0x40,0x05,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00
+
+# GFX12: global_load_b64 v[1:2], v[0:1], off offset:64 ; encoding: [0x7c,0x40,0x05,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x7c,0x40,0x05,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: global_load_b64 v[1:2], v[3:4], off     ; encoding: [0x7c,0x40,0x05,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+0x7c,0x40,0x05,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00
+
+# GFX12: global_load_b96 v[1:3], v0, s[0:1] offset:64 ; encoding: [0x00,0x80,0x05,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x00,0x80,0x05,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: global_load_b96 v[1:3], v5, s[2:3]      ; encoding: [0x02,0x80,0x05,0xee,0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00]
+0x02,0x80,0x05,0xee,0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00
+
+# GFX12: global_load_b96 v[1:3], v[0:1], off offset:64 ; encoding: [0x7c,0x80,0x05,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x7c,0x80,0x05,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: global_load_b96 v[1:3], v[5:6], off     ; encoding: [0x7c,0x80,0x05,0xee,0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00]
+0x7c,0x80,0x05,0xee,0x01,0x00,0x00,0x00,0x05,0x00,0x00,0x00
+
+# GFX12: global_load_d16_b16 v1, v0, s[0:1] offset:64 ; encoding: [0x00,0x00,0x08,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x00,0x00,0x08,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: global_load_d16_b16 v1, v3, s[2:3]      ; encoding: [0x02,0x00,0x08,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+0x02,0x00,0x08,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00
+
+# GFX12: global_load_d16_b16 v1, v[0:1], off offset:64 ; encoding: [0x7c,0x00,0x08,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x7c,0x00,0x08,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: global_load_d16_b16 v1, v[3:4], off     ; encoding: [0x7c,0x00,0x08,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+0x7c,0x00,0x08,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00
+
+# GFX12: global_load_d16_hi_b16 v1, v0, s[0:1] offset:64 ; encoding: [0x00,0xc0,0x08,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x00,0xc0,0x08,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: global_load_d16_hi_b16 v1, v3, s[2:3]   ; encoding: [0x02,0xc0,0x08,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+0x02,0xc0,0x08,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00
+
+# GFX12: global_load_d16_hi_b16 v1, v[0:1], off offset:64 ; encoding: [0x7c,0xc0,0x08,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x7c,0xc0,0x08,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: global_load_d16_hi_b16 v1, v[3:4], off  ; encoding: [0x7c,0xc0,0x08,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+0x7c,0xc0,0x08,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00
+
+# GFX12: global_load_d16_hi_i8 v1, v0, s[0:1] offset:64 ; encoding: [0x00,0x80,0x08,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x00,0x80,0x08,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: global_load_d16_hi_i8 v1, v3, s[2:3]    ; encoding: [0x02,0x80,0x08,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+0x02,0x80,0x08,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00
+
+# GFX12: global_load_d16_hi_i8 v1, v[0:1], off offset:64 ; encoding: [0x7c,0x80,0x08,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x7c,0x80,0x08,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: global_load_d16_hi_i8 v1, v[3:4], off   ; encoding: [0x7c,0x80,0x08,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+0x7c,0x80,0x08,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00
+
+# GFX12: global_load_d16_hi_u8 v1, v0, s[0:1] offset:64 ; encoding: [0x00,0x40,0x08,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x00,0x40,0x08,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: global_load_d16_hi_u8 v1, v3, s[2:3]    ; encoding: [0x02,0x40,0x08,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+0x02,0x40,0x08,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00
+
+# GFX12: global_load_d16_hi_u8 v1, v[0:1], off offset:64 ; encoding: [0x7c,0x40,0x08,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x7c,0x40,0x08,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: global_load_d16_hi_u8 v1, v[3:4], off   ; encoding: [0x7c,0x40,0x08,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+0x7c,0x40,0x08,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00
+
+# GFX12: global_load_d16_i8 v1, v0, s[0:1] offset:64 ; encoding: [0x00,0xc0,0x07,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x00,0xc0,0x07,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: global_load_d16_i8 v1, v3, s[2:3]       ; encoding: [0x02,0xc0,0x07,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+0x02,0xc0,0x07,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00
+
+# GFX12: global_load_d16_i8 v1, v[0:1], off offset:64 ; encoding: [0x7c,0xc0,0x07,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x7c,0xc0,0x07,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: global_load_d16_i8 v1, v[3:4], off      ; encoding: [0x7c,0xc0,0x07,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+0x7c,0xc0,0x07,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00
+
+# GFX12: global_load_d16_u8 v1, v0, s[0:1] offset:64 ; encoding: [0x00,0x80,0x07,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x00,0x80,0x07,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: global_load_d16_u8 v1, v3, s[2:3]       ; encoding: [0x02,0x80,0x07,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+0x02,0x80,0x07,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00
+
+# GFX12: global_load_d16_u8 v1, v[0:1], off offset:64 ; encoding: [0x7c,0x80,0x07,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x7c,0x80,0x07,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: global_load_d16_u8 v1, v[3:4], off      ; encoding: [0x7c,0x80,0x07,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+0x7c,0x80,0x07,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00
+
+# GFX12: global_load_i16 v1, v0, s[0:1] offset:64 ; encoding: [0x00,0xc0,0x04,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x00,0xc0,0x04,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: global_load_i16 v1, v3, s[2:3]          ; encoding: [0x02,0xc0,0x04,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+0x02,0xc0,0x04,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00
+
+# GFX12: global_load_i16 v1, v[0:1], off offset:64 ; encoding: [0x7c,0xc0,0x04,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x7c,0xc0,0x04,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: global_load_i16 v1, v[3:4], off         ; encoding: [0x7c,0xc0,0x04,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+0x7c,0xc0,0x04,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00
+
+# GFX12: global_load_i8 v1, v0, s[0:1] offset:64 ; encoding: [0x00,0x40,0x04,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x00,0x40,0x04,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: global_load_i8 v1, v3, s[2:3]           ; encoding: [0x02,0x40,0x04,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+0x02,0x40,0x04,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00
+
+# GFX12: global_load_i8 v1, v[0:1], off offset:64 ; encoding: [0x7c,0x40,0x04,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x7c,0x40,0x04,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: global_load_i8 v1, v[3:4], off          ; encoding: [0x7c,0x40,0x04,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+0x7c,0x40,0x04,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00
+
+# GFX12: global_load_u16 v1, v0, s[0:1] offset:64 ; encoding: [0x00,0x80,0x04,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x00,0x80,0x04,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: global_load_u16 v1, v3, s[2:3]          ; encoding: [0x02,0x80,0x04,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+0x02,0x80,0x04,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00
+
+# GFX12: global_load_u16 v1, v[0:1], off offset:64 ; encoding: [0x7c,0x80,0x04,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x7c,0x80,0x04,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: global_load_u16 v1, v[3:4], off         ; encoding: [0x7c,0x80,0x04,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+0x7c,0x80,0x04,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00
+
+# GFX12: global_load_u8 v1, v0, s[0:1] offset:64 ; encoding: [0x00,0x00,0x04,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x00,0x00,0x04,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: global_load_u8 v1, v3, s[2:3]           ; encoding: [0x02,0x00,0x04,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+0x02,0x00,0x04,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00
+
+# GFX12: global_load_u8 v1, v[0:1], off offset:64 ; encoding: [0x7c,0x00,0x04,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x7c,0x00,0x04,0xee,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: global_load_u8 v1, v[3:4], off          ; encoding: [0x7c,0x00,0x04,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00]
+0x7c,0x00,0x04,0xee,0x01,0x00,0x00,0x00,0x03,0x00,0x00,0x00
+
+# GFX12: global_store_addtid_b32 v2, off offset:64 ; encoding: [0x7c,0x40,0x0a,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x40,0x0a,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_store_addtid_b32 v2, s[0:1] offset:64 ; encoding: [0x00,0x40,0x0a,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x00,0x40,0x0a,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_store_b128 v0, v[2:5], s[0:1] offset:64 ; encoding: [0x00,0x40,0x07,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x00,0x40,0x07,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_store_b128 v1, v[3:6], s[2:3]    ; encoding: [0x02,0x40,0x07,0xee,0x00,0x00,0x80,0x01,0x01,0x00,0x00,0x00]
+0x02,0x40,0x07,0xee,0x00,0x00,0x80,0x01,0x01,0x00,0x00,0x00
+
+# GFX12: global_store_b128 v[0:1], v[2:5], off offset:64 ; encoding: [0x7c,0x40,0x07,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x40,0x07,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_store_b128 v[1:2], v[3:6], off   ; encoding: [0x7c,0x40,0x07,0xee,0x00,0x00,0x80,0x01,0x01,0x00,0x00,0x00]
+0x7c,0x40,0x07,0xee,0x00,0x00,0x80,0x01,0x01,0x00,0x00,0x00
+
+# GFX12: global_store_b16 v0, v2, s[0:1] offset:64 ; encoding: [0x00,0x40,0x06,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x00,0x40,0x06,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_store_b16 v3, v1, s[2:3]         ; encoding: [0x02,0x40,0x06,0xee,0x00,0x00,0x80,0x00,0x03,0x00,0x00,0x00]
+0x02,0x40,0x06,0xee,0x00,0x00,0x80,0x00,0x03,0x00,0x00,0x00
+
+# GFX12: global_store_b16 v[0:1], v2, off offset:64 ; encoding: [0x7c,0x40,0x06,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x40,0x06,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_store_b16 v[3:4], v1, off        ; encoding: [0x7c,0x40,0x06,0xee,0x00,0x00,0x80,0x00,0x03,0x00,0x00,0x00]
+0x7c,0x40,0x06,0xee,0x00,0x00,0x80,0x00,0x03,0x00,0x00,0x00
+
+# GFX12: global_store_b32 v0, v2, s[0:1] offset:64 ; encoding: [0x00,0x80,0x06,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x00,0x80,0x06,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_store_b32 v3, v1, s[2:3] offset:16 ; encoding: [0x02,0x80,0x06,0xee,0x00,0x00,0x80,0x00,0x03,0x10,0x00,0x00]
+0x02,0x80,0x06,0xee,0x00,0x00,0x80,0x00,0x03,0x10,0x00,0x00
+
+# GFX12: global_store_b32 v[0:1], v2, off offset:64 ; encoding: [0x7c,0x80,0x06,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x80,0x06,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_store_b32 v[3:4], v1, off offset:16 ; encoding: [0x7c,0x80,0x06,0xee,0x00,0x00,0x80,0x00,0x03,0x10,0x00,0x00]
+0x7c,0x80,0x06,0xee,0x00,0x00,0x80,0x00,0x03,0x10,0x00,0x00
+
+# GFX12: global_store_b64 v0, v[2:3], s[0:1] offset:64 ; encoding: [0x00,0xc0,0x06,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x00,0xc0,0x06,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_store_b64 v1, v[2:3], s[2:3]     ; encoding: [0x02,0xc0,0x06,0xee,0x00,0x00,0x00,0x01,0x01,0x00,0x00,0x00]
+0x02,0xc0,0x06,0xee,0x00,0x00,0x00,0x01,0x01,0x00,0x00,0x00
+
+# GFX12: global_store_b64 v[0:1], v[2:3], off offset:64 ; encoding: [0x7c,0xc0,0x06,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0xc0,0x06,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_store_b64 v[1:2], v[3:4], off    ; encoding: [0x7c,0xc0,0x06,0xee,0x00,0x00,0x80,0x01,0x01,0x00,0x00,0x00]
+0x7c,0xc0,0x06,0xee,0x00,0x00,0x80,0x01,0x01,0x00,0x00,0x00
+
+# GFX12: global_store_b8 v0, v2, s[0:1] offset:64 ; encoding: [0x00,0x00,0x06,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x00,0x00,0x06,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_store_b8 v3, v1, s[2:3]          ; encoding: [0x02,0x00,0x06,0xee,0x00,0x00,0x80,0x00,0x03,0x00,0x00,0x00]
+0x02,0x00,0x06,0xee,0x00,0x00,0x80,0x00,0x03,0x00,0x00,0x00
+
+# GFX12: global_store_b8 v[0:1], v2, off offset:64 ; encoding: [0x7c,0x00,0x06,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x00,0x06,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_store_b8 v[3:4], v1, off         ; encoding: [0x7c,0x00,0x06,0xee,0x00,0x00,0x80,0x00,0x03,0x00,0x00,0x00]
+0x7c,0x00,0x06,0xee,0x00,0x00,0x80,0x00,0x03,0x00,0x00,0x00
+
+# GFX12: global_store_b96 v0, v[2:4], s[0:1] offset:64 ; encoding: [0x00,0x00,0x07,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x00,0x00,0x07,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_store_b96 v1, v[3:5], s[2:3]     ; encoding: [0x02,0x00,0x07,0xee,0x00,0x00,0x80,0x01,0x01,0x00,0x00,0x00]
+0x02,0x00,0x07,0xee,0x00,0x00,0x80,0x01,0x01,0x00,0x00,0x00
+
+# GFX12: global_store_b96 v[0:1], v[2:4], off offset:64 ; encoding: [0x7c,0x00,0x07,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x00,0x07,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_store_b96 v[1:2], v[3:5], off    ; encoding: [0x7c,0x00,0x07,0xee,0x00,0x00,0x80,0x01,0x01,0x00,0x00,0x00]
+0x7c,0x00,0x07,0xee,0x00,0x00,0x80,0x01,0x01,0x00,0x00,0x00
+
+# GFX12: global_store_d16_hi_b16 v0, v2, s[0:1] offset:64 ; encoding: [0x00,0x40,0x09,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x00,0x40,0x09,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_store_d16_hi_b16 v3, v1, s[2:3]  ; encoding: [0x02,0x40,0x09,0xee,0x00,0x00,0x80,0x00,0x03,0x00,0x00,0x00]
+0x02,0x40,0x09,0xee,0x00,0x00,0x80,0x00,0x03,0x00,0x00,0x00
+
+# GFX12: global_store_d16_hi_b16 v[0:1], v2, off offset:64 ; encoding: [0x7c,0x40,0x09,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x40,0x09,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_store_d16_hi_b16 v[3:4], v1, off ; encoding: [0x7c,0x40,0x09,0xee,0x00,0x00,0x80,0x00,0x03,0x00,0x00,0x00]
+0x7c,0x40,0x09,0xee,0x00,0x00,0x80,0x00,0x03,0x00,0x00,0x00
+
+# GFX12: global_store_d16_hi_b8 v0, v2, s[0:1] offset:64 ; encoding: [0x00,0x00,0x09,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x00,0x00,0x09,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_store_d16_hi_b8 v3, v1, s[2:3]   ; encoding: [0x02,0x00,0x09,0xee,0x00,0x00,0x80,0x00,0x03,0x00,0x00,0x00]
+0x02,0x00,0x09,0xee,0x00,0x00,0x80,0x00,0x03,0x00,0x00,0x00
+
+# GFX12: global_store_d16_hi_b8 v[0:1], v2, off offset:64 ; encoding: [0x7c,0x00,0x09,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x00,0x09,0xee,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: global_store_d16_hi_b8 v[3:4], v1, off  ; encoding: [0x7c,0x00,0x09,0xee,0x00,0x00,0x80,0x00,0x03,0x00,0x00,0x00]
+0x7c,0x00,0x09,0xee,0x00,0x00,0x80,0x00,0x03,0x00,0x00,0x00
+
+# GFX12: scratch_load_b128 v[1:4], off, off offset:-64 ; encoding:  [0x7c,0xc0,0x05,0xed,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff]
+0x7c,0xc0,0x05,0xed,0x01,0x00,0x00,0x00,0x00,0xc0,0xff,0xff
+
+# GFX12: scratch_load_b128 v[1:4], off, off offset:64 ; encoding: [0x7c,0xc0,0x05,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x7c,0xc0,0x05,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_load_b128 v[1:4], off, s0 offset:64 ; encoding: [0x00,0xc0,0x05,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x00,0xc0,0x05,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_load_b128 v[1:4], v0, off offset:64 ; encoding: [0x7c,0xc0,0x05,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00]
+0x7c,0xc0,0x05,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_load_b128 v[1:4], v0, s0 offset:64 ; encoding: [0x00,0xc0,0x05,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00]
+0x00,0xc0,0x05,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_load_b128 v[1:4], v2, s1        ; encoding: [0x01,0xc0,0x05,0xed,0x01,0x00,0x02,0x00,0x02,0x00,0x00,0x00]
+0x01,0xc0,0x05,0xed,0x01,0x00,0x02,0x00,0x02,0x00,0x00,0x00
+
+# GFX12: scratch_load_b32 v1, off, off offset:2047 ; encoding: [0x7c,0x00,0x05,0xed,0x01,0x00,0x00,0x00,0x00,0xff,0x07,0x00]
+0x7c,0x00,0x05,0xed,0x01,0x00,0x00,0x00,0x00,0xff,0x07,0x00
+
+# GFX12: scratch_load_b32 v1, off, off offset:64 ; encoding: [0x7c,0x00,0x05,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x7c,0x00,0x05,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_load_b32 v1, off, off           ; encoding: [0x7c,0x00,0x05,0xed,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00]
+0x7c,0x00,0x05,0xed,0x01,0x00,0x00,0x00,0x00,0x00,0x00,0x00
+
+# GFX12: scratch_load_b32 v1, off, s0 offset:64 ; encoding: [0x00,0x00,0x05,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x00,0x00,0x05,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_load_b32 v1, off, s1 offset:2047 ; encoding: [0x01,0x00,0x05,0xed,0x01,0x00,0x00,0x00,0x00,0xff,0x07,0x00]
+0x01,0x00,0x05,0xed,0x01,0x00,0x00,0x00,0x00,0xff,0x07,0x00
+
+# GFX12: scratch_load_b32 v1, v0, off offset:64 ; encoding: [0x7c,0x00,0x05,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00]
+0x7c,0x00,0x05,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_load_b32 v1, v0, s0 offset:64  ; encoding: [0x00,0x00,0x05,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00]
+0x00,0x00,0x05,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_load_b32 v1, v2, off offset:2047 ; encoding: [0x7c,0x00,0x05,0xed,0x01,0x00,0x02,0x00,0x02,0xff,0x07,0x00]
+0x7c,0x00,0x05,0xed,0x01,0x00,0x02,0x00,0x02,0xff,0x07,0x00
+
+# GFX12: scratch_load_b32 v1, v2, s1 offset:61440 ; encoding: [0x01,0x00,0x05,0xed,0x01,0x00,0x02,0x00,0x02,0x00,0xf0,0x00]
+0x01,0x00,0x05,0xed,0x01,0x00,0x02,0x00,0x02,0x00,0xf0,0x00
+
+# GFX12: scratch_load_b32 v1, v2, s1 offset:2047 ; encoding: [0x01,0x00,0x05,0xed,0x01,0x00,0x02,0x00,0x02,0xff,0x07,0x00]
+0x01,0x00,0x05,0xed,0x01,0x00,0x02,0x00,0x02,0xff,0x07,0x00
+
+# GFX12: scratch_load_b32 v1, v2, s1 offset:4095 ; encoding: [0x01,0x00,0x05,0xed,0x01,0x00,0x02,0x00,0x02,0xff,0x0f,0x00]
+0x01,0x00,0x05,0xed,0x01,0x00,0x02,0x00,0x02,0xff,0x0f,0x00
+
+# GFX12: scratch_load_b32 v1, v2, s1             ; encoding: [0x01,0x00,0x05,0xed,0x01,0x00,0x02,0x00,0x02,0x00,0x00,0x00]
+0x01,0x00,0x05,0xed,0x01,0x00,0x02,0x00,0x02,0x00,0x00,0x00
+
+# GFX12: scratch_load_b64 v[1:2], off, off offset:64 ; encoding: [0x7c,0x40,0x05,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x7c,0x40,0x05,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_load_b64 v[1:2], off, s0 offset:64 ; encoding: [0x00,0x40,0x05,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x00,0x40,0x05,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_load_b64 v[1:2], v0, off offset:64 ; encoding: [0x7c,0x40,0x05,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00]
+0x7c,0x40,0x05,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_load_b64 v[1:2], v0, s0 offset:64 ; encoding: [0x00,0x40,0x05,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00]
+0x00,0x40,0x05,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_load_b64 v[1:2], v2, s1         ; encoding: [0x01,0x40,0x05,0xed,0x01,0x00,0x02,0x00,0x02,0x00,0x00,0x00]
+0x01,0x40,0x05,0xed,0x01,0x00,0x02,0x00,0x02,0x00,0x00,0x00
+
+# GFX12: scratch_load_b96 v[1:3], off, off offset:64 ; encoding: [0x7c,0x80,0x05,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x7c,0x80,0x05,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_load_b96 v[1:3], off, s0 offset:64 ; encoding: [0x00,0x80,0x05,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x00,0x80,0x05,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_load_b96 v[1:3], v0, off offset:64 ; encoding: [0x7c,0x80,0x05,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00]
+0x7c,0x80,0x05,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_load_b96 v[1:3], v0, s0 offset:64 ; encoding: [0x00,0x80,0x05,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00]
+0x00,0x80,0x05,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_load_b96 v[1:3], v2, s1         ; encoding: [0x01,0x80,0x05,0xed,0x01,0x00,0x02,0x00,0x02,0x00,0x00,0x00]
+0x01,0x80,0x05,0xed,0x01,0x00,0x02,0x00,0x02,0x00,0x00,0x00
+
+# GFX12: scratch_load_d16_b16 v1, off, off offset:64 ; encoding: [0x7c,0x00,0x08,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x7c,0x00,0x08,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_load_d16_b16 v1, off, s0 offset:64 ; encoding: [0x00,0x00,0x08,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x00,0x00,0x08,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_load_d16_b16 v1, v0, off offset:64 ; encoding: [0x7c,0x00,0x08,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00]
+0x7c,0x00,0x08,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_load_d16_b16 v1, v0, s0 offset:64 ; encoding: [0x00,0x00,0x08,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00]
+0x00,0x00,0x08,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_load_d16_b16 v1, v2, s1         ; encoding: [0x01,0x00,0x08,0xed,0x01,0x00,0x02,0x00,0x02,0x00,0x00,0x00]
+0x01,0x00,0x08,0xed,0x01,0x00,0x02,0x00,0x02,0x00,0x00,0x00
+
+# GFX12: scratch_load_d16_hi_b16 v1, off, off offset:64 ; encoding: [0x7c,0xc0,0x08,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x7c,0xc0,0x08,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_load_d16_hi_b16 v1, off, s0 offset:64 ; encoding: [0x00,0xc0,0x08,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x00,0xc0,0x08,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_load_d16_hi_b16 v1, v0, off offset:64 ; encoding: [0x7c,0xc0,0x08,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00]
+0x7c,0xc0,0x08,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_load_d16_hi_b16 v1, v0, s0 offset:64 ; encoding: [0x00,0xc0,0x08,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00]
+0x00,0xc0,0x08,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_load_d16_hi_b16 v1, v2, s1      ; encoding: [0x01,0xc0,0x08,0xed,0x01,0x00,0x02,0x00,0x02,0x00,0x00,0x00]
+0x01,0xc0,0x08,0xed,0x01,0x00,0x02,0x00,0x02,0x00,0x00,0x00
+
+# GFX12: scratch_load_d16_hi_i8 v1, off, off offset:64 ; encoding: [0x7c,0x80,0x08,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x7c,0x80,0x08,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_load_d16_hi_i8 v1, off, s0 offset:64 ; encoding: [0x00,0x80,0x08,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x00,0x80,0x08,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_load_d16_hi_i8 v1, v0, off offset:64 ; encoding: [0x7c,0x80,0x08,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00]
+0x7c,0x80,0x08,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_load_d16_hi_i8 v1, v0, s0 offset:64 ; encoding: [0x00,0x80,0x08,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00]
+0x00,0x80,0x08,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_load_d16_hi_i8 v1, v2, s1       ; encoding: [0x01,0x80,0x08,0xed,0x01,0x00,0x02,0x00,0x02,0x00,0x00,0x00]
+0x01,0x80,0x08,0xed,0x01,0x00,0x02,0x00,0x02,0x00,0x00,0x00
+
+# GFX12: scratch_load_d16_hi_u8 v1, off, off offset:64 ; encoding: [0x7c,0x40,0x08,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x7c,0x40,0x08,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_load_d16_hi_u8 v1, off, s0 offset:64 ; encoding: [0x00,0x40,0x08,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x00,0x40,0x08,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_load_d16_hi_u8 v1, v0, off offset:64 ; encoding: [0x7c,0x40,0x08,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00]
+0x7c,0x40,0x08,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_load_d16_hi_u8 v1, v0, s0 offset:64 ; encoding: [0x00,0x40,0x08,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00]
+0x00,0x40,0x08,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_load_d16_hi_u8 v1, v2, s1       ; encoding: [0x01,0x40,0x08,0xed,0x01,0x00,0x02,0x00,0x02,0x00,0x00,0x00]
+0x01,0x40,0x08,0xed,0x01,0x00,0x02,0x00,0x02,0x00,0x00,0x00
+
+# GFX12: scratch_load_d16_i8 v1, off, off offset:64 ; encoding: [0x7c,0xc0,0x07,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x7c,0xc0,0x07,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_load_d16_i8 v1, off, s0 offset:64 ; encoding: [0x00,0xc0,0x07,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x00,0xc0,0x07,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_load_d16_i8 v1, v0, off offset:64 ; encoding: [0x7c,0xc0,0x07,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00]
+0x7c,0xc0,0x07,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_load_d16_i8 v1, v0, s0 offset:64 ; encoding: [0x00,0xc0,0x07,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00]
+0x00,0xc0,0x07,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_load_d16_i8 v1, v2, s1          ; encoding: [0x01,0xc0,0x07,0xed,0x01,0x00,0x02,0x00,0x02,0x00,0x00,0x00]
+0x01,0xc0,0x07,0xed,0x01,0x00,0x02,0x00,0x02,0x00,0x00,0x00
+
+# GFX12: scratch_load_d16_u8 v1, off, off offset:64 ; encoding: [0x7c,0x80,0x07,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x7c,0x80,0x07,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_load_d16_u8 v1, off, s0 offset:64 ; encoding: [0x00,0x80,0x07,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x00,0x80,0x07,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_load_d16_u8 v1, v0, off offset:64 ; encoding: [0x7c,0x80,0x07,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00]
+0x7c,0x80,0x07,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_load_d16_u8 v1, v0, s0 offset:64 ; encoding: [0x00,0x80,0x07,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00]
+0x00,0x80,0x07,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_load_d16_u8 v1, v2, s1          ; encoding: [0x01,0x80,0x07,0xed,0x01,0x00,0x02,0x00,0x02,0x00,0x00,0x00]
+0x01,0x80,0x07,0xed,0x01,0x00,0x02,0x00,0x02,0x00,0x00,0x00
+
+# GFX12: scratch_load_i16 v1, off, off offset:64 ; encoding: [0x7c,0xc0,0x04,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x7c,0xc0,0x04,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_load_i16 v1, off, s0 offset:64 ; encoding: [0x00,0xc0,0x04,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x00,0xc0,0x04,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_load_i16 v1, v0, off offset:64 ; encoding: [0x7c,0xc0,0x04,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00]
+0x7c,0xc0,0x04,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_load_i16 v1, v0, s0 offset:64  ; encoding: [0x00,0xc0,0x04,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00]
+0x00,0xc0,0x04,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_load_i16 v1, v2, s1             ; encoding: [0x01,0xc0,0x04,0xed,0x01,0x00,0x02,0x00,0x02,0x00,0x00,0x00]
+0x01,0xc0,0x04,0xed,0x01,0x00,0x02,0x00,0x02,0x00,0x00,0x00
+
+# GFX12: scratch_load_i8 v1, off, off offset:64 ; encoding: [0x7c,0x40,0x04,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x7c,0x40,0x04,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_load_i8 v1, off, s0 offset:64  ; encoding: [0x00,0x40,0x04,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x00,0x40,0x04,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_load_i8 v1, v0, off offset:64  ; encoding: [0x7c,0x40,0x04,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00]
+0x7c,0x40,0x04,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_load_i8 v1, v0, s0 offset:64   ; encoding: [0x00,0x40,0x04,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00]
+0x00,0x40,0x04,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_load_i8 v1, v2, s1              ; encoding: [0x01,0x40,0x04,0xed,0x01,0x00,0x02,0x00,0x02,0x00,0x00,0x00]
+0x01,0x40,0x04,0xed,0x01,0x00,0x02,0x00,0x02,0x00,0x00,0x00
+
+# GFX12: scratch_load_u16 v1, off, off offset:64 ; encoding: [0x7c,0x80,0x04,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x7c,0x80,0x04,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_load_u16 v1, off, s0 offset:64 ; encoding: [0x00,0x80,0x04,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x00,0x80,0x04,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_load_u16 v1, v0, off offset:64 ; encoding: [0x7c,0x80,0x04,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00]
+0x7c,0x80,0x04,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_load_u16 v1, v0, s0 offset:64  ; encoding: [0x00,0x80,0x04,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00]
+0x00,0x80,0x04,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_load_u16 v1, v2, s1             ; encoding: [0x01,0x80,0x04,0xed,0x01,0x00,0x02,0x00,0x02,0x00,0x00,0x00]
+0x01,0x80,0x04,0xed,0x01,0x00,0x02,0x00,0x02,0x00,0x00,0x00
+
+# GFX12: scratch_load_u8 v1, off, off offset:64 ; encoding: [0x7c,0x00,0x04,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x7c,0x00,0x04,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_load_u8 v1, off, s0 offset:64  ; encoding: [0x00,0x00,0x04,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00]
+0x00,0x00,0x04,0xed,0x01,0x00,0x00,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_load_u8 v1, v0, off offset:64  ; encoding: [0x7c,0x00,0x04,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00]
+0x7c,0x00,0x04,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_load_u8 v1, v0, s0 offset:64   ; encoding: [0x00,0x00,0x04,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00]
+0x00,0x00,0x04,0xed,0x01,0x00,0x02,0x00,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_load_u8 v1, v2, s1              ; encoding: [0x01,0x00,0x04,0xed,0x01,0x00,0x02,0x00,0x02,0x00,0x00,0x00]
+0x01,0x00,0x04,0xed,0x01,0x00,0x02,0x00,0x02,0x00,0x00,0x00
+
+# GFX12: scratch_store_b128 off, v[2:5], off offset:64 ; encoding: [0x7c,0x40,0x07,0xed,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x40,0x07,0xed,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_store_b128 off, v[2:5], s0 offset:64 ; encoding: [0x00,0x40,0x07,0xed,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x00,0x40,0x07,0xed,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_store_b128 v0, v[2:5], off offset:64 ; encoding: [0x7c,0x40,0x07,0xed,0x00,0x00,0x02,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x40,0x07,0xed,0x00,0x00,0x02,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_store_b128 v0, v[2:5], s0 offset:64 ; encoding: [0x00,0x40,0x07,0xed,0x00,0x00,0x02,0x01,0x00,0x40,0x00,0x00]
+0x00,0x40,0x07,0xed,0x00,0x00,0x02,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_store_b128 v1, v[2:5], s3       ; encoding: [0x03,0x40,0x07,0xed,0x00,0x00,0x02,0x01,0x01,0x00,0x00,0x00]
+0x03,0x40,0x07,0xed,0x00,0x00,0x02,0x01,0x01,0x00,0x00,0x00
+
+# GFX12: scratch_store_b16 off, v2, off offset:64 ; encoding: [0x7c,0x40,0x06,0xed,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x40,0x06,0xed,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_store_b16 off, v2, s0 offset:64 ; encoding: [0x00,0x40,0x06,0xed,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x00,0x40,0x06,0xed,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_store_b16 v0, v2, off offset:64 ; encoding: [0x7c,0x40,0x06,0xed,0x00,0x00,0x02,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x40,0x06,0xed,0x00,0x00,0x02,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_store_b16 v0, v2, s0 offset:64 ; encoding: [0x00,0x40,0x06,0xed,0x00,0x00,0x02,0x01,0x00,0x40,0x00,0x00]
+0x00,0x40,0x06,0xed,0x00,0x00,0x02,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_store_b16 v1, v2, s3            ; encoding: [0x03,0x40,0x06,0xed,0x00,0x00,0x02,0x01,0x01,0x00,0x00,0x00]
+0x03,0x40,0x06,0xed,0x00,0x00,0x02,0x01,0x01,0x00,0x00,0x00
+
+# GFX12: scratch_store_b32 off, v2, off offset:2047 ; encoding: [0x7c,0x80,0x06,0xed,0x00,0x00,0x00,0x01,0x00,0xff,0x07,0x00]
+0x7c,0x80,0x06,0xed,0x00,0x00,0x00,0x01,0x00,0xff,0x07,0x00
+
+# GFX12: scratch_store_b32 off, v2, off offset:64 ; encoding: [0x7c,0x80,0x06,0xed,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x80,0x06,0xed,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_store_b32 off, v2, off          ; encoding: [0x7c,0x80,0x06,0xed,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x00]
+0x7c,0x80,0x06,0xed,0x00,0x00,0x00,0x01,0x00,0x00,0x00,0x00
+
+# GFX12: scratch_store_b32 off, v2, s0 offset:64 ; encoding: [0x00,0x80,0x06,0xed,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x00,0x80,0x06,0xed,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_store_b32 off, v2, s3 offset:2047 ; encoding: [0x03,0x80,0x06,0xed,0x00,0x00,0x00,0x01,0x00,0xff,0x07,0x00]
+0x03,0x80,0x06,0xed,0x00,0x00,0x00,0x01,0x00,0xff,0x07,0x00
+
+# GFX12: scratch_store_b32 v0, v2, off offset:64 ; encoding: [0x7c,0x80,0x06,0xed,0x00,0x00,0x02,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x80,0x06,0xed,0x00,0x00,0x02,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_store_b32 v0, v2, s0 offset:64 ; encoding: [0x00,0x80,0x06,0xed,0x00,0x00,0x02,0x01,0x00,0x40,0x00,0x00]
+0x00,0x80,0x06,0xed,0x00,0x00,0x02,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_store_b32 v1, v2, off offset:2047 ; encoding: [0x7c,0x80,0x06,0xed,0x00,0x00,0x02,0x01,0x01,0xff,0x07,0x00]
+0x7c,0x80,0x06,0xed,0x00,0x00,0x02,0x01,0x01,0xff,0x07,0x00
+
+# GFX12: scratch_store_b32 v1, v2, s1 offset:61440 ; encoding: [0x01,0x80,0x06,0xed,0x00,0x00,0x02,0x01,0x01,0x00,0xf0,0x00]
+0x01,0x80,0x06,0xed,0x00,0x00,0x02,0x01,0x01,0x00,0xf0,0x00
+
+# GFX12: scratch_store_b32 v1, v2, s1 offset:4095 ; encoding: [0x01,0x80,0x06,0xed,0x00,0x00,0x02,0x01,0x01,0xff,0x0f,0x00]
+0x01,0x80,0x06,0xed,0x00,0x00,0x02,0x01,0x01,0xff,0x0f,0x00
+
+# GFX12: scratch_store_b32 v1, v2, s3 offset:2047 ; encoding: [0x03,0x80,0x06,0xed,0x00,0x00,0x02,0x01,0x01,0xff,0x07,0x00]
+0x03,0x80,0x06,0xed,0x00,0x00,0x02,0x01,0x01,0xff,0x07,0x00
+
+# GFX12: scratch_store_b32 v1, v2, s3            ; encoding: [0x03,0x80,0x06,0xed,0x00,0x00,0x02,0x01,0x01,0x00,0x00,0x00]
+0x03,0x80,0x06,0xed,0x00,0x00,0x02,0x01,0x01,0x00,0x00,0x00
+
+# GFX12: scratch_store_b64 off, v[2:3], off offset:64 ; encoding: [0x7c,0xc0,0x06,0xed,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0xc0,0x06,0xed,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_store_b64 off, v[2:3], s0 offset:64 ; encoding: [0x00,0xc0,0x06,0xed,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x00,0xc0,0x06,0xed,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_store_b64 v0, v[2:3], off offset:64 ; encoding: [0x7c,0xc0,0x06,0xed,0x00,0x00,0x02,0x01,0x00,0x40,0x00,0x00]
+0x7c,0xc0,0x06,0xed,0x00,0x00,0x02,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_store_b64 v0, v[2:3], s0 offset:64 ; encoding: [0x00,0xc0,0x06,0xed,0x00,0x00,0x02,0x01,0x00,0x40,0x00,0x00]
+0x00,0xc0,0x06,0xed,0x00,0x00,0x02,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_store_b64 v1, v[2:3], s3        ; encoding: [0x03,0xc0,0x06,0xed,0x00,0x00,0x02,0x01,0x01,0x00,0x00,0x00]
+0x03,0xc0,0x06,0xed,0x00,0x00,0x02,0x01,0x01,0x00,0x00,0x00
+
+# GFX12: scratch_store_b8 off, v2, off offset:64 ; encoding: [0x7c,0x00,0x06,0xed,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x00,0x06,0xed,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_store_b8 off, v2, s0 offset:64 ; encoding: [0x00,0x00,0x06,0xed,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x00,0x00,0x06,0xed,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_store_b8 v0, v2, off offset:64 ; encoding: [0x7c,0x00,0x06,0xed,0x00,0x00,0x02,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x00,0x06,0xed,0x00,0x00,0x02,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_store_b8 v0, v2, s0 offset:64  ; encoding: [0x00,0x00,0x06,0xed,0x00,0x00,0x02,0x01,0x00,0x40,0x00,0x00]
+0x00,0x00,0x06,0xed,0x00,0x00,0x02,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_store_b8 v1, v2, s3             ; encoding: [0x03,0x00,0x06,0xed,0x00,0x00,0x02,0x01,0x01,0x00,0x00,0x00]
+0x03,0x00,0x06,0xed,0x00,0x00,0x02,0x01,0x01,0x00,0x00,0x00
+
+# GFX12: scratch_store_b96 off, v[2:4], off offset:64 ; encoding: [0x7c,0x00,0x07,0xed,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x00,0x07,0xed,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_store_b96 off, v[2:4], s0 offset:64 ; encoding: [0x00,0x00,0x07,0xed,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x00,0x00,0x07,0xed,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_store_b96 v0, v[2:4], off offset:64 ; encoding: [0x7c,0x00,0x07,0xed,0x00,0x00,0x02,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x00,0x07,0xed,0x00,0x00,0x02,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_store_b96 v0, v[2:4], s0 offset:64 ; encoding: [0x00,0x00,0x07,0xed,0x00,0x00,0x02,0x01,0x00,0x40,0x00,0x00]
+0x00,0x00,0x07,0xed,0x00,0x00,0x02,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_store_b96 v1, v[2:4], s3        ; encoding: [0x03,0x00,0x07,0xed,0x00,0x00,0x02,0x01,0x01,0x00,0x00,0x00]
+0x03,0x00,0x07,0xed,0x00,0x00,0x02,0x01,0x01,0x00,0x00,0x00
+
+# GFX12: scratch_store_d16_hi_b16 off, v2, off offset:64 ; encoding: [0x7c,0x40,0x09,0xed,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x40,0x09,0xed,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_store_d16_hi_b16 off, v2, s0 offset:64 ; encoding: [0x00,0x40,0x09,0xed,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x00,0x40,0x09,0xed,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_store_d16_hi_b16 v0, v2, off offset:64 ; encoding: [0x7c,0x40,0x09,0xed,0x00,0x00,0x02,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x40,0x09,0xed,0x00,0x00,0x02,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_store_d16_hi_b16 v0, v2, s0 offset:64 ; encoding: [0x00,0x40,0x09,0xed,0x00,0x00,0x02,0x01,0x00,0x40,0x00,0x00]
+0x00,0x40,0x09,0xed,0x00,0x00,0x02,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_store_d16_hi_b16 v1, v2, s3     ; encoding: [0x03,0x40,0x09,0xed,0x00,0x00,0x02,0x01,0x01,0x00,0x00,0x00]
+0x03,0x40,0x09,0xed,0x00,0x00,0x02,0x01,0x01,0x00,0x00,0x00
+
+# GFX12: scratch_store_d16_hi_b8 off, v2, off offset:64 ; encoding: [0x7c,0x00,0x09,0xed,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x00,0x09,0xed,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_store_d16_hi_b8 off, v2, s0 offset:64 ; encoding: [0x00,0x00,0x09,0xed,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00]
+0x00,0x00,0x09,0xed,0x00,0x00,0x00,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_store_d16_hi_b8 v0, v2, off offset:64 ; encoding: [0x7c,0x00,0x09,0xed,0x00,0x00,0x02,0x01,0x00,0x40,0x00,0x00]
+0x7c,0x00,0x09,0xed,0x00,0x00,0x02,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_store_d16_hi_b8 v0, v2, s0 offset:64 ; encoding: [0x00,0x00,0x09,0xed,0x00,0x00,0x02,0x01,0x00,0x40,0x00,0x00]
+0x00,0x00,0x09,0xed,0x00,0x00,0x02,0x01,0x00,0x40,0x00,0x00
+
+# GFX12: scratch_store_d16_hi_b8 v1, v2, s3      ; encoding: [0x03,0x00,0x09,0xed,0x00,0x00,0x02,0x01,0x01,0x00,0x00,0x00]
+0x03,0x00,0x09,0xed,0x00,0x00,0x02,0x01,0x01,0x00,0x00,0x00
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3.txt
index 7f4f142d5bc1cd2959e691801b9267ea61fe2d34..8ff8c2c4c4f6a1eafb013785ca8544fadfdf95b7 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3.txt
@@ -5495,3 +5495,632 @@
 # GFX12: v_xor_b16 v255, 0xfe0b, vcc_hi          ; encoding: [0xff,0x00,0x64,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00]
 0xff,0x00,0x64,0xd7,0xff,0xd6,0x00,0x00,0x0b,0xfe,0x00,0x00
 
+# GFX12: v_minimum_f32 v255, -|0xaf123456|, -|vcc_hi| ; encoding: [0xff,0x03,0x65,0xd7,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+0xff,0x03,0x65,0xd7,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf
+
+# GFX12: v_minimum_f32 v5, -1, exec_hi           ; encoding: [0x05,0x00,0x65,0xd7,0xc1,0xfe,0x00,0x00]
+0x05,0x00,0x65,0xd7,0xc1,0xfe,0x00,0x00
+
+# GFX12: v_minimum_f32 v5, -src_scc, |vcc_lo|    ; encoding: [0x05,0x02,0x65,0xd7,0xfd,0xd4,0x00,0x20]
+0x05,0x02,0x65,0xd7,0xfd,0xd4,0x00,0x20
+
+# GFX12: v_minimum_f32 v5, 0.5, -m0              ; encoding: [0x05,0x00,0x65,0xd7,0xf0,0xfa,0x00,0x40]
+0x05,0x00,0x65,0xd7,0xf0,0xfa,0x00,0x40
+
+# GFX12: v_minimum_f32 v5, exec_lo, -1           ; encoding: [0x05,0x00,0x65,0xd7,0x7e,0x82,0x01,0x00]
+0x05,0x00,0x65,0xd7,0x7e,0x82,0x01,0x00
+
+# GFX12: v_minimum_f32 v5, m0, 0.5               ; encoding: [0x05,0x00,0x65,0xd7,0x7d,0xe0,0x01,0x00]
+0x05,0x00,0x65,0xd7,0x7d,0xe0,0x01,0x00
+
+# GFX12: v_minimum_f32 v5, null, exec_lo         ; encoding: [0x05,0x00,0x65,0xd7,0x7c,0xfc,0x00,0x00]
+0x05,0x00,0x65,0xd7,0x7c,0xfc,0x00,0x00
+
+# GFX12: v_minimum_f32 v5, s1, s2                ; encoding: [0x05,0x00,0x65,0xd7,0x01,0x04,0x00,0x00]
+0x05,0x00,0x65,0xd7,0x01,0x04,0x00,0x00
+
+# GFX12: v_minimum_f32 v5, s105, s105            ; encoding: [0x05,0x00,0x65,0xd7,0x69,0xd2,0x00,0x00]
+0x05,0x00,0x65,0xd7,0x69,0xd2,0x00,0x00
+
+# GFX12: v_minimum_f32 v5, ttmp15, src_scc       ; encoding: [0x05,0x00,0x65,0xd7,0x7b,0xfa,0x01,0x00]
+0x05,0x00,0x65,0xd7,0x7b,0xfa,0x01,0x00
+
+# GFX12: v_minimum_f32 v5, v1, v2                ; encoding: [0x05,0x00,0x65,0xd7,0x01,0x05,0x02,0x00]
+0x05,0x00,0x65,0xd7,0x01,0x05,0x02,0x00
+
+# GFX12: v_minimum_f32 v5, v255, v255            ; encoding: [0x05,0x00,0x65,0xd7,0xff,0xff,0x03,0x00]
+0x05,0x00,0x65,0xd7,0xff,0xff,0x03,0x00
+
+# GFX12: v_minimum_f32 v5, vcc_hi, 0xaf123456    ; encoding: [0x05,0x00,0x65,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+0x05,0x00,0x65,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+
+# GFX12: v_minimum_f32 v5, vcc_lo, ttmp15        ; encoding: [0x05,0x00,0x65,0xd7,0x6a,0xf6,0x00,0x00]
+0x05,0x00,0x65,0xd7,0x6a,0xf6,0x00,0x00
+
+# GFX12: v_minimum_f32 v5, |exec_hi|, null       ; encoding: [0x05,0x01,0x65,0xd7,0x7f,0xf8,0x00,0x00]
+0x05,0x01,0x65,0xd7,0x7f,0xf8,0x00,0x00
+
+# GFX12: v_maximum_f32 v255, -|0xaf123456|, -|vcc_hi| ; encoding: [0xff,0x03,0x66,0xd7,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf]
+0xff,0x03,0x66,0xd7,0xff,0xd6,0x00,0x60,0x56,0x34,0x12,0xaf
+
+# GFX12: v_maximum_f32 v5, -1, exec_hi           ; encoding: [0x05,0x00,0x66,0xd7,0xc1,0xfe,0x00,0x00]
+0x05,0x00,0x66,0xd7,0xc1,0xfe,0x00,0x00
+
+# GFX12: v_maximum_f32 v5, -src_scc, |vcc_lo|    ; encoding: [0x05,0x02,0x66,0xd7,0xfd,0xd4,0x00,0x20]
+0x05,0x02,0x66,0xd7,0xfd,0xd4,0x00,0x20
+
+# GFX12: v_maximum_f32 v5, 0.5, -m0              ; encoding: [0x05,0x00,0x66,0xd7,0xf0,0xfa,0x00,0x40]
+0x05,0x00,0x66,0xd7,0xf0,0xfa,0x00,0x40
+
+# GFX12: v_maximum_f32 v5, exec_lo, -1           ; encoding: [0x05,0x00,0x66,0xd7,0x7e,0x82,0x01,0x00]
+0x05,0x00,0x66,0xd7,0x7e,0x82,0x01,0x00
+
+# GFX12: v_maximum_f32 v5, m0, 0.5               ; encoding: [0x05,0x00,0x66,0xd7,0x7d,0xe0,0x01,0x00]
+0x05,0x00,0x66,0xd7,0x7d,0xe0,0x01,0x00
+
+# GFX12: v_maximum_f32 v5, null, exec_lo         ; encoding: [0x05,0x00,0x66,0xd7,0x7c,0xfc,0x00,0x00]
+0x05,0x00,0x66,0xd7,0x7c,0xfc,0x00,0x00
+
+# GFX12: v_maximum_f32 v5, s1, s2                ; encoding: [0x05,0x00,0x66,0xd7,0x01,0x04,0x00,0x00]
+0x05,0x00,0x66,0xd7,0x01,0x04,0x00,0x00
+
+# GFX12: v_maximum_f32 v5, s105, s105            ; encoding: [0x05,0x00,0x66,0xd7,0x69,0xd2,0x00,0x00]
+0x05,0x00,0x66,0xd7,0x69,0xd2,0x00,0x00
+
+# GFX12: v_maximum_f32 v5, ttmp15, src_scc       ; encoding: [0x05,0x00,0x66,0xd7,0x7b,0xfa,0x01,0x00]
+0x05,0x00,0x66,0xd7,0x7b,0xfa,0x01,0x00
+
+# GFX12: v_maximum_f32 v5, v1, v2                ; encoding: [0x05,0x00,0x66,0xd7,0x01,0x05,0x02,0x00]
+0x05,0x00,0x66,0xd7,0x01,0x05,0x02,0x00
+
+# GFX12: v_maximum_f32 v5, v255, v255            ; encoding: [0x05,0x00,0x66,0xd7,0xff,0xff,0x03,0x00]
+0x05,0x00,0x66,0xd7,0xff,0xff,0x03,0x00
+
+# GFX12: v_maximum_f32 v5, vcc_hi, 0xaf123456    ; encoding: [0x05,0x00,0x66,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf]
+0x05,0x00,0x66,0xd7,0x6b,0xfe,0x01,0x00,0x56,0x34,0x12,0xaf
+
+# GFX12: v_maximum_f32 v5, vcc_lo, ttmp15        ; encoding: [0x05,0x00,0x66,0xd7,0x6a,0xf6,0x00,0x00]
+0x05,0x00,0x66,0xd7,0x6a,0xf6,0x00,0x00
+
+# GFX12: v_maximum_f32 v5, |exec_hi|, null       ; encoding: [0x05,0x01,0x66,0xd7,0x7f,0xf8,0x00,0x00]
+0x05,0x01,0x66,0xd7,0x7f,0xf8,0x00,0x00
+
+# GFX12: v_minimum_f16 v205, v201, v200          ; encoding: [0xcd,0x00,0x67,0xd7,0xc9,0x91,0x03,0x00]
+0xcd,0x00,0x67,0xd7,0xc9,0x91,0x03,0x00
+
+# GFX12: v_minimum_f16 v255, -|0xaf12|, -|vcc_hi| ; encoding: [0xff,0x03,0x67,0xd7,0xff,0xd6,0x00,0x60,0x12,0xaf,0x00,0x00]
+0xff,0x03,0x67,0xd7,0xff,0xd6,0x00,0x60,0x12,0xaf,0x00,0x00
+
+# GFX12: v_minimum_f16 v5, -1, exec_hi           ; encoding: [0x05,0x00,0x67,0xd7,0xc1,0xfe,0x00,0x00]
+0x05,0x00,0x67,0xd7,0xc1,0xfe,0x00,0x00
+
+# GFX12: v_minimum_f16 v5, -src_scc, |vcc_lo|    ; encoding: [0x05,0x02,0x67,0xd7,0xfd,0xd4,0x00,0x20]
+0x05,0x02,0x67,0xd7,0xfd,0xd4,0x00,0x20
+
+# GFX12: v_minimum_f16 v5, 0.5, -m0              ; encoding: [0x05,0x00,0x67,0xd7,0xf0,0xfa,0x00,0x40]
+0x05,0x00,0x67,0xd7,0xf0,0xfa,0x00,0x40
+
+# GFX12: v_minimum_f16 v5, exec_lo, -1           ; encoding: [0x05,0x00,0x67,0xd7,0x7e,0x82,0x01,0x00]
+0x05,0x00,0x67,0xd7,0x7e,0x82,0x01,0x00
+
+# GFX12: v_minimum_f16 v5, m0, 0.5               ; encoding: [0x05,0x00,0x67,0xd7,0x7d,0xe0,0x01,0x00]
+0x05,0x00,0x67,0xd7,0x7d,0xe0,0x01,0x00
+
+# GFX12: v_minimum_f16 v5, null, exec_lo         ; encoding: [0x05,0x00,0x67,0xd7,0x7c,0xfc,0x00,0x00]
+0x05,0x00,0x67,0xd7,0x7c,0xfc,0x00,0x00
+
+# GFX12: v_minimum_f16 v5, s1, s2                ; encoding: [0x05,0x00,0x67,0xd7,0x01,0x04,0x00,0x00]
+0x05,0x00,0x67,0xd7,0x01,0x04,0x00,0x00
+
+# GFX12: v_minimum_f16 v5, s105, s105            ; encoding: [0x05,0x00,0x67,0xd7,0x69,0xd2,0x00,0x00]
+0x05,0x00,0x67,0xd7,0x69,0xd2,0x00,0x00
+
+# GFX12: v_minimum_f16 v5, ttmp15, src_scc       ; encoding: [0x05,0x00,0x67,0xd7,0x7b,0xfa,0x01,0x00]
+0x05,0x00,0x67,0xd7,0x7b,0xfa,0x01,0x00
+
+# GFX12: v_minimum_f16 v5, v1, v2                ; encoding: [0x05,0x00,0x67,0xd7,0x01,0x05,0x02,0x00]
+0x05,0x00,0x67,0xd7,0x01,0x05,0x02,0x00
+
+# GFX12: v_minimum_f16 v5, v255, v255            ; encoding: [0x05,0x00,0x67,0xd7,0xff,0xff,0x03,0x00]
+0x05,0x00,0x67,0xd7,0xff,0xff,0x03,0x00
+
+# GFX12: v_minimum_f16 v5, vcc_hi, 0xaf12        ; encoding: [0x05,0x00,0x67,0xd7,0x6b,0xfe,0x01,0x00,0x12,0xaf,0x00,0x00]
+0x05,0x00,0x67,0xd7,0x6b,0xfe,0x01,0x00,0x12,0xaf,0x00,0x00
+
+# GFX12: v_minimum_f16 v5, vcc_lo, ttmp15        ; encoding: [0x05,0x00,0x67,0xd7,0x6a,0xf6,0x00,0x00]
+0x05,0x00,0x67,0xd7,0x6a,0xf6,0x00,0x00
+
+# GFX12: v_minimum_f16 v5, |exec_hi|, null       ; encoding: [0x05,0x01,0x67,0xd7,0x7f,0xf8,0x00,0x00]
+0x05,0x01,0x67,0xd7,0x7f,0xf8,0x00,0x00
+
+# GFX12: v_maximum_f16 v205, v201, v200          ; encoding: [0xcd,0x00,0x68,0xd7,0xc9,0x91,0x03,0x00]
+0xcd,0x00,0x68,0xd7,0xc9,0x91,0x03,0x00
+
+# GFX12: v_maximum_f16 v255, -|0xaf12|, -|vcc_hi| ; encoding: [0xff,0x03,0x68,0xd7,0xff,0xd6,0x00,0x60,0x12,0xaf,0x00,0x00]
+0xff,0x03,0x68,0xd7,0xff,0xd6,0x00,0x60,0x12,0xaf,0x00,0x00
+
+# GFX12: v_maximum_f16 v5, -1, exec_hi           ; encoding: [0x05,0x00,0x68,0xd7,0xc1,0xfe,0x00,0x00]
+0x05,0x00,0x68,0xd7,0xc1,0xfe,0x00,0x00
+
+# GFX12: v_maximum_f16 v5, -src_scc, |vcc_lo|    ; encoding: [0x05,0x02,0x68,0xd7,0xfd,0xd4,0x00,0x20]
+0x05,0x02,0x68,0xd7,0xfd,0xd4,0x00,0x20
+
+# GFX12: v_maximum_f16 v5, 0.5, -m0              ; encoding: [0x05,0x00,0x68,0xd7,0xf0,0xfa,0x00,0x40]
+0x05,0x00,0x68,0xd7,0xf0,0xfa,0x00,0x40
+
+# GFX12: v_maximum_f16 v5, exec_lo, -1           ; encoding: [0x05,0x00,0x68,0xd7,0x7e,0x82,0x01,0x00]
+0x05,0x00,0x68,0xd7,0x7e,0x82,0x01,0x00
+
+# GFX12: v_maximum_f16 v5, m0, 0.5               ; encoding: [0x05,0x00,0x68,0xd7,0x7d,0xe0,0x01,0x00]
+0x05,0x00,0x68,0xd7,0x7d,0xe0,0x01,0x00
+
+# GFX12: v_maximum_f16 v5, null, exec_lo         ; encoding: [0x05,0x00,0x68,0xd7,0x7c,0xfc,0x00,0x00]
+0x05,0x00,0x68,0xd7,0x7c,0xfc,0x00,0x00
+
+# GFX12: v_maximum_f16 v5, s1, s2                ; encoding: [0x05,0x00,0x68,0xd7,0x01,0x04,0x00,0x00]
+0x05,0x00,0x68,0xd7,0x01,0x04,0x00,0x00
+
+# GFX12: v_maximum_f16 v5, s105, s105            ; encoding: [0x05,0x00,0x68,0xd7,0x69,0xd2,0x00,0x00]
+0x05,0x00,0x68,0xd7,0x69,0xd2,0x00,0x00
+
+# GFX12: v_maximum_f16 v5, ttmp15, src_scc       ; encoding: [0x05,0x00,0x68,0xd7,0x7b,0xfa,0x01,0x00]
+0x05,0x00,0x68,0xd7,0x7b,0xfa,0x01,0x00
+
+# GFX12: v_maximum_f16 v5, v1, v2                ; encoding: [0x05,0x00,0x68,0xd7,0x01,0x05,0x02,0x00]
+0x05,0x00,0x68,0xd7,0x01,0x05,0x02,0x00
+
+# GFX12: v_maximum_f16 v5, v255, v255            ; encoding: [0x05,0x00,0x68,0xd7,0xff,0xff,0x03,0x00]
+0x05,0x00,0x68,0xd7,0xff,0xff,0x03,0x00
+
+# GFX12: v_maximum_f16 v5, vcc_hi, 0xaf12        ; encoding: [0x05,0x00,0x68,0xd7,0x6b,0xfe,0x01,0x00,0x12,0xaf,0x00,0x00]
+0x05,0x00,0x68,0xd7,0x6b,0xfe,0x01,0x00,0x12,0xaf,0x00,0x00
+
+# GFX12: v_maximum_f16 v5, vcc_lo, ttmp15        ; encoding: [0x05,0x00,0x68,0xd7,0x6a,0xf6,0x00,0x00]
+0x05,0x00,0x68,0xd7,0x6a,0xf6,0x00,0x00
+
+# GFX12: v_maximum_f16 v5, |exec_hi|, null       ; encoding: [0x05,0x01,0x68,0xd7,0x7f,0xf8,0x00,0x00]
+0x05,0x01,0x68,0xd7,0x7f,0xf8,0x00,0x00
+
+# GFX12: v_minimum_f64 v[254:255], -|2|, -|vcc|  ; encoding: [0xfe,0x03,0x41,0xd7,0x82,0xd4,0x00,0x60]
+0xfe,0x03,0x41,0xd7,0x82,0xd4,0x00,0x60
+
+# GFX12: v_minimum_f64 v[5:6], -1, exec          ; encoding: [0x05,0x00,0x41,0xd7,0xc1,0xfc,0x00,0x00]
+0x05,0x00,0x41,0xd7,0xc1,0xfc,0x00,0x00
+
+# GFX12: v_minimum_f64 v[5:6], -src_scc, |vcc|   ; encoding: [0x05,0x02,0x41,0xd7,0xfd,0xd4,0x00,0x20]
+0x05,0x02,0x41,0xd7,0xfd,0xd4,0x00,0x20
+
+# GFX12: v_minimum_f64 v[5:6], 0.5, -vcc         ; encoding: [0x05,0x00,0x41,0xd7,0xf0,0xd4,0x00,0x40]
+0x05,0x00,0x41,0xd7,0xf0,0xd4,0x00,0x40
+
+# GFX12: v_minimum_f64 v[5:6], exec, -1          ; encoding: [0x05,0x00,0x41,0xd7,0x7e,0x82,0x01,0x00]
+0x05,0x00,0x41,0xd7,0x7e,0x82,0x01,0x00
+
+# GFX12: v_minimum_f64 v[5:6], null, exec        ; encoding: [0x05,0x00,0x41,0xd7,0x7c,0xfc,0x00,0x00]
+0x05,0x00,0x41,0xd7,0x7c,0xfc,0x00,0x00
+
+# GFX12: v_minimum_f64 v[5:6], s[104:105], s[104:105] ; encoding: [0x05,0x00,0x41,0xd7,0x68,0xd0,0x00,0x00]
+0x05,0x00,0x41,0xd7,0x68,0xd0,0x00,0x00
+
+# GFX12: v_minimum_f64 v[5:6], s[6:7], s[4:5]    ; encoding: [0x05,0x00,0x41,0xd7,0x06,0x08,0x00,0x00]
+0x05,0x00,0x41,0xd7,0x06,0x08,0x00,0x00
+
+# GFX12: v_minimum_f64 v[5:6], ttmp[14:15], src_scc ; encoding: [0x05,0x00,0x41,0xd7,0x7a,0xfa,0x01,0x00]
+0x05,0x00,0x41,0xd7,0x7a,0xfa,0x01,0x00
+
+# GFX12: v_minimum_f64 v[5:6], v[1:2], v[3:4]    ; encoding: [0x05,0x00,0x41,0xd7,0x01,0x07,0x02,0x00]
+0x05,0x00,0x41,0xd7,0x01,0x07,0x02,0x00
+
+# GFX12: v_minimum_f64 v[5:6], v[254:255], v[254:255] ; encoding: [0x05,0x00,0x41,0xd7,0xfe,0xfd,0x03,0x00]
+0x05,0x00,0x41,0xd7,0xfe,0xfd,0x03,0x00
+
+# GFX12: v_minimum_f64 v[5:6], vcc, 0.5          ; encoding: [0x05,0x00,0x41,0xd7,0x6a,0xe0,0x01,0x00]
+0x05,0x00,0x41,0xd7,0x6a,0xe0,0x01,0x00
+
+# GFX12: v_minimum_f64 v[5:6], vcc, 0xaf121234   ; encoding: [0x05,0x00,0x41,0xd7,0x6a,0xfe,0x01,0x00,0x34,0x12,0x12,0xaf]
+0x05,0x00,0x41,0xd7,0x6a,0xfe,0x01,0x00,0x34,0x12,0x12,0xaf
+
+# GFX12: v_minimum_f64 v[5:6], vcc, ttmp[14:15]  ; encoding: [0x05,0x00,0x41,0xd7,0x6a,0xf4,0x00,0x00]
+0x05,0x00,0x41,0xd7,0x6a,0xf4,0x00,0x00
+
+# GFX12: v_minimum_f64 v[5:6], |exec|, null      ; encoding: [0x05,0x01,0x41,0xd7,0x7e,0xf8,0x00,0x00]
+0x05,0x01,0x41,0xd7,0x7e,0xf8,0x00,0x00
+
+# GFX12: v_maximum_f64 v[254:255], -|2|, -|vcc|  ; encoding: [0xfe,0x03,0x42,0xd7,0x82,0xd4,0x00,0x60]
+0xfe,0x03,0x42,0xd7,0x82,0xd4,0x00,0x60
+
+# GFX12: v_maximum_f64 v[5:6], -1, exec          ; encoding: [0x05,0x00,0x42,0xd7,0xc1,0xfc,0x00,0x00]
+0x05,0x00,0x42,0xd7,0xc1,0xfc,0x00,0x00
+
+# GFX12: v_maximum_f64 v[5:6], -src_scc, |vcc|   ; encoding: [0x05,0x02,0x42,0xd7,0xfd,0xd4,0x00,0x20]
+0x05,0x02,0x42,0xd7,0xfd,0xd4,0x00,0x20
+
+# GFX12: v_maximum_f64 v[5:6], 0.5, -vcc         ; encoding: [0x05,0x00,0x42,0xd7,0xf0,0xd4,0x00,0x40]
+0x05,0x00,0x42,0xd7,0xf0,0xd4,0x00,0x40
+
+# GFX12: v_maximum_f64 v[5:6], exec, -1          ; encoding: [0x05,0x00,0x42,0xd7,0x7e,0x82,0x01,0x00]
+0x05,0x00,0x42,0xd7,0x7e,0x82,0x01,0x00
+
+# GFX12: v_maximum_f64 v[5:6], null, exec        ; encoding: [0x05,0x00,0x42,0xd7,0x7c,0xfc,0x00,0x00]
+0x05,0x00,0x42,0xd7,0x7c,0xfc,0x00,0x00
+
+# GFX12: v_maximum_f64 v[5:6], s[104:105], s[104:105] ; encoding: [0x05,0x00,0x42,0xd7,0x68,0xd0,0x00,0x00]
+0x05,0x00,0x42,0xd7,0x68,0xd0,0x00,0x00
+
+# GFX12: v_maximum_f64 v[5:6], s[6:7], s[4:5]    ; encoding: [0x05,0x00,0x42,0xd7,0x06,0x08,0x00,0x00]
+0x05,0x00,0x42,0xd7,0x06,0x08,0x00,0x00
+
+# GFX12: v_maximum_f64 v[5:6], ttmp[14:15], src_scc ; encoding: [0x05,0x00,0x42,0xd7,0x7a,0xfa,0x01,0x00]
+0x05,0x00,0x42,0xd7,0x7a,0xfa,0x01,0x00
+
+# GFX12: v_maximum_f64 v[5:6], v[1:2], v[3:4]    ; encoding: [0x05,0x00,0x42,0xd7,0x01,0x07,0x02,0x00]
+0x05,0x00,0x42,0xd7,0x01,0x07,0x02,0x00
+
+# GFX12: v_maximum_f64 v[5:6], v[254:255], v[254:255] ; encoding: [0x05,0x00,0x42,0xd7,0xfe,0xfd,0x03,0x00]
+0x05,0x00,0x42,0xd7,0xfe,0xfd,0x03,0x00
+
+# GFX12: v_maximum_f64 v[5:6], vcc, 0.5          ; encoding: [0x05,0x00,0x42,0xd7,0x6a,0xe0,0x01,0x00]
+0x05,0x00,0x42,0xd7,0x6a,0xe0,0x01,0x00
+
+# GFX12: v_maximum_f64 v[5:6], vcc, 0xaf121234   ; encoding: [0x05,0x00,0x42,0xd7,0x6a,0xfe,0x01,0x00,0x34,0x12,0x12,0xaf]
+0x05,0x00,0x42,0xd7,0x6a,0xfe,0x01,0x00,0x34,0x12,0x12,0xaf
+
+# GFX12: v_maximum_f64 v[5:6], vcc, ttmp[14:15]  ; encoding: [0x05,0x00,0x42,0xd7,0x6a,0xf4,0x00,0x00]
+0x05,0x00,0x42,0xd7,0x6a,0xf4,0x00,0x00
+
+# GFX12: v_maximum_f64 v[5:6], |exec|, null      ; encoding: [0x05,0x01,0x42,0xd7,0x7e,0xf8,0x00,0x00]
+0x05,0x01,0x42,0xd7,0x7e,0xf8,0x00,0x00
+
+# GFX12: v_maximum3_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x2e,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
+0xff,0x83,0x2e,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf
+
+# GFX12: v_maximum3_f32 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x2e,0xd6,0xc1,0xfe,0xf4,0xc3]
+0x05,0x06,0x2e,0xd6,0xc1,0xfe,0xf4,0xc3
+
+# GFX12: v_maximum3_f32 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x2e,0xd6,0xfd,0xd4,0x04,0x33]
+0x05,0x02,0x2e,0xd6,0xfd,0xd4,0x04,0x33
+
+# GFX12: v_maximum3_f32 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x2e,0xd6,0x7f,0xf8,0xa8,0xa1]
+0x05,0x05,0x2e,0xd6,0x7f,0xf8,0xa8,0xa1
+
+# GFX12: v_maximum3_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x2e,0xd6,0x7b,0xfa,0xed,0xe1]
+0x05,0x07,0x2e,0xd6,0x7b,0xfa,0xed,0xe1
+
+# GFX12: v_maximum3_f32 v5, 0.5, -m0, 0.5 mul:2  ; encoding: [0x05,0x00,0x2e,0xd6,0xf0,0xfa,0xc0,0x4b]
+0x05,0x00,0x2e,0xd6,0xf0,0xfa,0xc0,0x4b
+
+# GFX12: v_maximum3_f32 v5, m0, 0.5, m0          ; encoding: [0x05,0x00,0x2e,0xd6,0x7d,0xe0,0xf5,0x01]
+0x05,0x00,0x2e,0xd6,0x7d,0xe0,0xf5,0x01
+
+# GFX12: v_maximum3_f32 v5, null, exec_lo, -|0xaf123456| ; encoding: [0x05,0x04,0x2e,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
+0x05,0x04,0x2e,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf
+
+# GFX12: v_maximum3_f32 v5, s1, v255, exec_hi    ; encoding: [0x05,0x00,0x2e,0xd6,0x01,0xfe,0xff,0x01]
+0x05,0x00,0x2e,0xd6,0x01,0xfe,0xff,0x01
+
+# GFX12: v_maximum3_f32 v5, s105, s105, exec_lo  ; encoding: [0x05,0x00,0x2e,0xd6,0x69,0xd2,0xf8,0x01]
+0x05,0x00,0x2e,0xd6,0x69,0xd2,0xf8,0x01
+
+# GFX12: v_maximum3_f32 v5, v1, v2, s3           ; encoding: [0x05,0x00,0x2e,0xd6,0x01,0x05,0x0e,0x00]
+0x05,0x00,0x2e,0xd6,0x01,0x05,0x0e,0x00
+
+# GFX12: v_maximum3_f32 v5, v255, s2, s105       ; encoding: [0x05,0x00,0x2e,0xd6,0xff,0x05,0xa4,0x01]
+0x05,0x00,0x2e,0xd6,0xff,0x05,0xa4,0x01
+
+# GFX12: v_maximum3_f32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x2e,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
+0x05,0x00,0x2e,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+
+# GFX12: v_maximum3_f32 v5, vcc_lo, ttmp15, v3   ; encoding: [0x05,0x00,0x2e,0xd6,0x6a,0xf6,0x0c,0x04]
+0x05,0x00,0x2e,0xd6,0x6a,0xf6,0x0c,0x04
+
+# GFX12: v_maximum3_f32 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x2e,0xd6,0x7e,0x82,0xad,0x01]
+0x05,0x01,0x2e,0xd6,0x7e,0x82,0xad,0x01
+
+# GFX12: v_minimum3_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x2d,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
+0xff,0x83,0x2d,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf
+
+# GFX12: v_minimum3_f32 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x2d,0xd6,0xc1,0xfe,0xf4,0xc3]
+0x05,0x06,0x2d,0xd6,0xc1,0xfe,0xf4,0xc3
+
+# GFX12: v_minimum3_f32 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x2d,0xd6,0xfd,0xd4,0x04,0x33]
+0x05,0x02,0x2d,0xd6,0xfd,0xd4,0x04,0x33
+
+# GFX12: v_minimum3_f32 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x2d,0xd6,0x7f,0xf8,0xa8,0xa1]
+0x05,0x05,0x2d,0xd6,0x7f,0xf8,0xa8,0xa1
+
+# GFX12: v_minimum3_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x2d,0xd6,0x7b,0xfa,0xed,0xe1]
+0x05,0x07,0x2d,0xd6,0x7b,0xfa,0xed,0xe1
+
+# GFX12: v_minimum3_f32 v5, 0.5, -m0, 0.5 mul:2  ; encoding: [0x05,0x00,0x2d,0xd6,0xf0,0xfa,0xc0,0x4b]
+0x05,0x00,0x2d,0xd6,0xf0,0xfa,0xc0,0x4b
+
+# GFX12: v_minimum3_f32 v5, m0, 0.5, m0          ; encoding: [0x05,0x00,0x2d,0xd6,0x7d,0xe0,0xf5,0x01]
+0x05,0x00,0x2d,0xd6,0x7d,0xe0,0xf5,0x01
+
+# GFX12: v_minimum3_f32 v5, null, exec_lo, -|0xaf123456| ; encoding: [0x05,0x04,0x2d,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
+0x05,0x04,0x2d,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf
+
+# GFX12: v_minimum3_f32 v5, s1, v255, exec_hi    ; encoding: [0x05,0x00,0x2d,0xd6,0x01,0xfe,0xff,0x01]
+0x05,0x00,0x2d,0xd6,0x01,0xfe,0xff,0x01
+
+# GFX12: v_minimum3_f32 v5, s105, s105, exec_lo  ; encoding: [0x05,0x00,0x2d,0xd6,0x69,0xd2,0xf8,0x01]
+0x05,0x00,0x2d,0xd6,0x69,0xd2,0xf8,0x01
+
+# GFX12: v_minimum3_f32 v5, v1, v2, s3           ; encoding: [0x05,0x00,0x2d,0xd6,0x01,0x05,0x0e,0x00]
+0x05,0x00,0x2d,0xd6,0x01,0x05,0x0e,0x00
+
+# GFX12: v_minimum3_f32 v5, v255, s2, s105       ; encoding: [0x05,0x00,0x2d,0xd6,0xff,0x05,0xa4,0x01]
+0x05,0x00,0x2d,0xd6,0xff,0x05,0xa4,0x01
+
+# GFX12: v_minimum3_f32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x2d,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
+0x05,0x00,0x2d,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+
+# GFX12: v_minimum3_f32 v5, vcc_lo, ttmp15, v3   ; encoding: [0x05,0x00,0x2d,0xd6,0x6a,0xf6,0x0c,0x04]
+0x05,0x00,0x2d,0xd6,0x6a,0xf6,0x0c,0x04
+
+# GFX12: v_minimum3_f32 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x2d,0xd6,0x7e,0x82,0xad,0x01]
+0x05,0x01,0x2d,0xd6,0x7e,0x82,0xad,0x01
+
+# GFX12: v_maximum3_f16 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x30,0xd6,0xc1,0xfe,0xf4,0xc3]
+0x05,0x06,0x30,0xd6,0xc1,0xfe,0xf4,0xc3
+
+# GFX12: v_maximum3_f16 v5, -src_scc, |vcc_lo|, -1 ; encoding: [0x05,0x02,0x30,0xd6,0xfd,0xd4,0x04,0x23]
+0x05,0x02,0x30,0xd6,0xfd,0xd4,0x04,0x23
+
+# GFX12: v_maximum3_f16 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x30,0xd6,0x7f,0xf8,0xa8,0xa1]
+0x05,0x05,0x30,0xd6,0x7f,0xf8,0xa8,0xa1
+
+# GFX12: v_maximum3_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x30,0xd6,0x7b,0xfa,0xed,0xe1]
+0x05,0x07,0x30,0xd6,0x7b,0xfa,0xed,0xe1
+
+# GFX12: v_maximum3_f16 v5, 0.5, -m0, 0.5        ; encoding: [0x05,0x00,0x30,0xd6,0xf0,0xfa,0xc0,0x43]
+0x05,0x00,0x30,0xd6,0xf0,0xfa,0xc0,0x43
+
+# GFX12: v_maximum3_f16 v5, m0, 0.5, m0          ; encoding: [0x05,0x00,0x30,0xd6,0x7d,0xe0,0xf5,0x01]
+0x05,0x00,0x30,0xd6,0x7d,0xe0,0xf5,0x01
+
+# GFX12: v_maximum3_f16 v5, null, exec_lo, -|0xaf12| ; encoding: [0x05,0x04,0x30,0xd6,0x7c,0xfc,0xfc,0x83,0x12,0xaf,0x00,0x00]
+0x05,0x04,0x30,0xd6,0x7c,0xfc,0xfc,0x83,0x12,0xaf,0x00,0x00
+
+# GFX12: v_maximum3_f16 v5, s1, v255, exec_hi    ; encoding: [0x05,0x00,0x30,0xd6,0x01,0xfe,0xff,0x01]
+0x05,0x00,0x30,0xd6,0x01,0xfe,0xff,0x01
+
+# GFX12: v_maximum3_f16 v5, s105, s105, exec_lo  ; encoding: [0x05,0x00,0x30,0xd6,0x69,0xd2,0xf8,0x01]
+0x05,0x00,0x30,0xd6,0x69,0xd2,0xf8,0x01
+
+# GFX12: v_maximum3_f16 v5, v1, v2, s3           ; encoding: [0x05,0x00,0x30,0xd6,0x01,0x05,0x0e,0x00]
+0x05,0x00,0x30,0xd6,0x01,0x05,0x0e,0x00
+
+# GFX12: v_maximum3_f16 v5, v255, s2, s105       ; encoding: [0x05,0x00,0x30,0xd6,0xff,0x05,0xa4,0x01]
+0x05,0x00,0x30,0xd6,0xff,0x05,0xa4,0x01
+
+# GFX12: v_maximum3_f16 v5, vcc_hi, 0xaf12, v255 ; encoding: [0x05,0x00,0x30,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00]
+0x05,0x00,0x30,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00
+
+# GFX12: v_maximum3_f16 v5, vcc_lo, ttmp15, v3   ; encoding: [0x05,0x00,0x30,0xd6,0x6a,0xf6,0x0c,0x04]
+0x05,0x00,0x30,0xd6,0x6a,0xf6,0x0c,0x04
+
+# GFX12: v_maximum3_f16 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x30,0xd6,0x7e,0x82,0xad,0x01]
+0x05,0x01,0x30,0xd6,0x7e,0x82,0xad,0x01
+
+# GFX12: v_minimum3_f16 v255, -|0xaf12|, -|vcc_hi|, null clamp ; encoding: [0xff,0x83,0x2f,0xd6,0xff,0xd6,0xf0,0x61,0x12,0xaf,0x00,0x00]
+0xff,0x83,0x2f,0xd6,0xff,0xd6,0xf0,0x61,0x12,0xaf,0x00,0x00
+
+# GFX12: v_minimum3_f16 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x2f,0xd6,0xc1,0xfe,0xf4,0xc3]
+0x05,0x06,0x2f,0xd6,0xc1,0xfe,0xf4,0xc3
+
+# GFX12: v_minimum3_f16 v5, -src_scc, |vcc_lo|, -1 ; encoding: [0x05,0x02,0x2f,0xd6,0xfd,0xd4,0x04,0x23]
+0x05,0x02,0x2f,0xd6,0xfd,0xd4,0x04,0x23
+
+# GFX12: v_minimum3_f16 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x2f,0xd6,0x7f,0xf8,0xa8,0xa1]
+0x05,0x05,0x2f,0xd6,0x7f,0xf8,0xa8,0xa1
+
+# GFX12: v_minimum3_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x2f,0xd6,0x7b,0xfa,0xed,0xe1]
+0x05,0x07,0x2f,0xd6,0x7b,0xfa,0xed,0xe1
+
+# GFX12: v_minimum3_f16 v5, 0.5, -m0, 0.5        ; encoding: [0x05,0x00,0x2f,0xd6,0xf0,0xfa,0xc0,0x43]
+0x05,0x00,0x2f,0xd6,0xf0,0xfa,0xc0,0x43
+
+# GFX12: v_minimum3_f16 v5, m0, 0.5, m0          ; encoding: [0x05,0x00,0x2f,0xd6,0x7d,0xe0,0xf5,0x01]
+0x05,0x00,0x2f,0xd6,0x7d,0xe0,0xf5,0x01
+
+# GFX12: v_minimum3_f16 v5, null, exec_lo, -|0xaf12| ; encoding: [0x05,0x04,0x2f,0xd6,0x7c,0xfc,0xfc,0x83,0x12,0xaf,0x00,0x00]
+0x05,0x04,0x2f,0xd6,0x7c,0xfc,0xfc,0x83,0x12,0xaf,0x00,0x00
+
+# GFX12: v_minimum3_f16 v5, s1, v255, exec_hi    ; encoding: [0x05,0x00,0x2f,0xd6,0x01,0xfe,0xff,0x01]
+0x05,0x00,0x2f,0xd6,0x01,0xfe,0xff,0x01
+
+# GFX12: v_minimum3_f16 v5, s105, s105, exec_lo  ; encoding: [0x05,0x00,0x2f,0xd6,0x69,0xd2,0xf8,0x01]
+0x05,0x00,0x2f,0xd6,0x69,0xd2,0xf8,0x01
+
+# GFX12: v_minimum3_f16 v5, v1, v2, s3           ; encoding: [0x05,0x00,0x2f,0xd6,0x01,0x05,0x0e,0x00]
+0x05,0x00,0x2f,0xd6,0x01,0x05,0x0e,0x00
+
+# GFX12: v_minimum3_f16 v5, v255, s2, s105       ; encoding: [0x05,0x00,0x2f,0xd6,0xff,0x05,0xa4,0x01]
+0x05,0x00,0x2f,0xd6,0xff,0x05,0xa4,0x01
+
+# GFX12: v_minimum3_f16 v5, vcc_hi, 0xaf12, v255 ; encoding: [0x05,0x00,0x2f,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00]
+0x05,0x00,0x2f,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00
+
+# GFX12: v_minimum3_f16 v5, vcc_lo, ttmp15, v3   ; encoding: [0x05,0x00,0x2f,0xd6,0x6a,0xf6,0x0c,0x04]
+0x05,0x00,0x2f,0xd6,0x6a,0xf6,0x0c,0x04
+
+# GFX12: v_minimum3_f16 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x2f,0xd6,0x7e,0x82,0xad,0x01]
+0x05,0x01,0x2f,0xd6,0x7e,0x82,0xad,0x01
+
+# GFX12: v_maximumminimum_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x6d,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
+0xff,0x83,0x6d,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf
+
+# GFX12: v_maximumminimum_f32 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x6d,0xd6,0xc1,0xfe,0xf4,0xc3]
+0x05,0x06,0x6d,0xd6,0xc1,0xfe,0xf4,0xc3
+
+# GFX12: v_maximumminimum_f32 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x6d,0xd6,0xfd,0xd4,0x04,0x33]
+0x05,0x02,0x6d,0xd6,0xfd,0xd4,0x04,0x33
+
+# GFX12: v_maximumminimum_f32 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x6d,0xd6,0x7f,0xf8,0xa8,0xa1]
+0x05,0x05,0x6d,0xd6,0x7f,0xf8,0xa8,0xa1
+
+# GFX12: v_maximumminimum_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x6d,0xd6,0x7b,0xfa,0xed,0xe1]
+0x05,0x07,0x6d,0xd6,0x7b,0xfa,0xed,0xe1
+
+# GFX12: v_maximumminimum_f32 v5, 0.5, -m0, 0.5 mul:2 ; encoding: [0x05,0x00,0x6d,0xd6,0xf0,0xfa,0xc0,0x4b]
+0x05,0x00,0x6d,0xd6,0xf0,0xfa,0xc0,0x4b
+
+# GFX12: v_maximumminimum_f32 v5, m0, 0.5, m0    ; encoding: [0x05,0x00,0x6d,0xd6,0x7d,0xe0,0xf5,0x01]
+0x05,0x00,0x6d,0xd6,0x7d,0xe0,0xf5,0x01
+
+# GFX12: v_maximumminimum_f32 v5, null, exec_lo, -|0xaf123456| ; encoding: [0x05,0x04,0x6d,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
+0x05,0x04,0x6d,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf
+
+# GFX12: v_maximumminimum_f32 v5, s1, v255, exec_hi ; encoding: [0x05,0x00,0x6d,0xd6,0x01,0xfe,0xff,0x01]
+0x05,0x00,0x6d,0xd6,0x01,0xfe,0xff,0x01
+
+# GFX12: v_maximumminimum_f32 v5, s105, s105, exec_lo ; encoding: [0x05,0x00,0x6d,0xd6,0x69,0xd2,0xf8,0x01]
+0x05,0x00,0x6d,0xd6,0x69,0xd2,0xf8,0x01
+
+# GFX12: v_maximumminimum_f32 v5, v1, v2, s3     ; encoding: [0x05,0x00,0x6d,0xd6,0x01,0x05,0x0e,0x00]
+0x05,0x00,0x6d,0xd6,0x01,0x05,0x0e,0x00
+
+# GFX12: v_maximumminimum_f32 v5, v255, s2, s105 ; encoding: [0x05,0x00,0x6d,0xd6,0xff,0x05,0xa4,0x01]
+0x05,0x00,0x6d,0xd6,0xff,0x05,0xa4,0x01
+
+# GFX12: v_maximumminimum_f32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x6d,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
+0x05,0x00,0x6d,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+
+# GFX12: v_maximumminimum_f32 v5, vcc_lo, ttmp15, v3 ; encoding: [0x05,0x00,0x6d,0xd6,0x6a,0xf6,0x0c,0x04]
+0x05,0x00,0x6d,0xd6,0x6a,0xf6,0x0c,0x04
+
+# GFX12: v_maximumminimum_f32 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x6d,0xd6,0x7e,0x82,0xad,0x01]
+0x05,0x01,0x6d,0xd6,0x7e,0x82,0xad,0x01
+
+# GFX12: v_minimummaximum_f32 v255, -|0xaf123456|, -|vcc_hi|, null clamp div:2 ; encoding: [0xff,0x83,0x6c,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf]
+0xff,0x83,0x6c,0xd6,0xff,0xd6,0xf0,0x79,0x56,0x34,0x12,0xaf
+
+# GFX12: v_minimummaximum_f32 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x6c,0xd6,0xc1,0xfe,0xf4,0xc3]
+0x05,0x06,0x6c,0xd6,0xc1,0xfe,0xf4,0xc3
+
+# GFX12: v_minimummaximum_f32 v5, -src_scc, |vcc_lo|, -1 mul:4 ; encoding: [0x05,0x02,0x6c,0xd6,0xfd,0xd4,0x04,0x33]
+0x05,0x02,0x6c,0xd6,0xfd,0xd4,0x04,0x33
+
+# GFX12: v_minimummaximum_f32 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x6c,0xd6,0x7f,0xf8,0xa8,0xa1]
+0x05,0x05,0x6c,0xd6,0x7f,0xf8,0xa8,0xa1
+
+# GFX12: v_minimummaximum_f32 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x6c,0xd6,0x7b,0xfa,0xed,0xe1]
+0x05,0x07,0x6c,0xd6,0x7b,0xfa,0xed,0xe1
+
+# GFX12: v_minimummaximum_f32 v5, 0.5, -m0, 0.5 mul:2 ; encoding: [0x05,0x00,0x6c,0xd6,0xf0,0xfa,0xc0,0x4b]
+0x05,0x00,0x6c,0xd6,0xf0,0xfa,0xc0,0x4b
+
+# GFX12: v_minimummaximum_f32 v5, m0, 0.5, m0    ; encoding: [0x05,0x00,0x6c,0xd6,0x7d,0xe0,0xf5,0x01]
+0x05,0x00,0x6c,0xd6,0x7d,0xe0,0xf5,0x01
+
+# GFX12: v_minimummaximum_f32 v5, null, exec_lo, -|0xaf123456| ; encoding: [0x05,0x04,0x6c,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf]
+0x05,0x04,0x6c,0xd6,0x7c,0xfc,0xfc,0x83,0x56,0x34,0x12,0xaf
+
+# GFX12: v_minimummaximum_f32 v5, s1, v255, exec_hi ; encoding: [0x05,0x00,0x6c,0xd6,0x01,0xfe,0xff,0x01]
+0x05,0x00,0x6c,0xd6,0x01,0xfe,0xff,0x01
+
+# GFX12: v_minimummaximum_f32 v5, s105, s105, exec_lo ; encoding: [0x05,0x00,0x6c,0xd6,0x69,0xd2,0xf8,0x01]
+0x05,0x00,0x6c,0xd6,0x69,0xd2,0xf8,0x01
+
+# GFX12: v_minimummaximum_f32 v5, v1, v2, s3     ; encoding: [0x05,0x00,0x6c,0xd6,0x01,0x05,0x0e,0x00]
+0x05,0x00,0x6c,0xd6,0x01,0x05,0x0e,0x00
+
+# GFX12: v_minimummaximum_f32 v5, v255, s2, s105 ; encoding: [0x05,0x00,0x6c,0xd6,0xff,0x05,0xa4,0x01]
+0x05,0x00,0x6c,0xd6,0xff,0x05,0xa4,0x01
+
+# GFX12: v_minimummaximum_f32 v5, vcc_hi, 0xaf123456, v255 ; encoding: [0x05,0x00,0x6c,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf]
+0x05,0x00,0x6c,0xd6,0x6b,0xfe,0xfd,0x07,0x56,0x34,0x12,0xaf
+
+# GFX12: v_minimummaximum_f32 v5, vcc_lo, ttmp15, v3 ; encoding: [0x05,0x00,0x6c,0xd6,0x6a,0xf6,0x0c,0x04]
+0x05,0x00,0x6c,0xd6,0x6a,0xf6,0x0c,0x04
+
+# GFX12: v_minimummaximum_f32 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x6c,0xd6,0x7e,0x82,0xad,0x01]
+0x05,0x01,0x6c,0xd6,0x7e,0x82,0xad,0x01
+
+# GFX12: v_maximumminimum_f16 v255, -|0xaf12|, -|vcc_hi|, null clamp ; encoding: [0xff,0x83,0x6f,0xd6,0xff,0xd6,0xf0,0x61,0x12,0xaf,0x00,0x00]
+0xff,0x83,0x6f,0xd6,0xff,0xd6,0xf0,0x61,0x12,0xaf,0x00,0x00
+
+# GFX12: v_maximumminimum_f16 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x6f,0xd6,0xc1,0xfe,0xf4,0xc3]
+0x05,0x06,0x6f,0xd6,0xc1,0xfe,0xf4,0xc3
+
+# GFX12: v_maximumminimum_f16 v5, -src_scc, |vcc_lo|, -1 ; encoding: [0x05,0x02,0x6f,0xd6,0xfd,0xd4,0x04,0x23]
+0x05,0x02,0x6f,0xd6,0xfd,0xd4,0x04,0x23
+
+# GFX12: v_maximumminimum_f16 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x6f,0xd6,0x7f,0xf8,0xa8,0xa1]
+0x05,0x05,0x6f,0xd6,0x7f,0xf8,0xa8,0xa1
+
+# GFX12: v_maximumminimum_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x6f,0xd6,0x7b,0xfa,0xed,0xe1]
+0x05,0x07,0x6f,0xd6,0x7b,0xfa,0xed,0xe1
+
+# GFX12: v_maximumminimum_f16 v5, 0.5, -m0, 0.5  ; encoding: [0x05,0x00,0x6f,0xd6,0xf0,0xfa,0xc0,0x43]
+0x05,0x00,0x6f,0xd6,0xf0,0xfa,0xc0,0x43
+
+# GFX12: v_maximumminimum_f16 v5, m0, 0.5, m0    ; encoding: [0x05,0x00,0x6f,0xd6,0x7d,0xe0,0xf5,0x01]
+0x05,0x00,0x6f,0xd6,0x7d,0xe0,0xf5,0x01
+
+# GFX12: v_maximumminimum_f16 v5, null, exec_lo, -|0xaf12| ; encoding: [0x05,0x04,0x6f,0xd6,0x7c,0xfc,0xfc,0x83,0x12,0xaf,0x00,0x00]
+0x05,0x04,0x6f,0xd6,0x7c,0xfc,0xfc,0x83,0x12,0xaf,0x00,0x00
+
+# GFX12: v_maximumminimum_f16 v5, s1, v255, exec_hi ; encoding: [0x05,0x00,0x6f,0xd6,0x01,0xfe,0xff,0x01]
+0x05,0x00,0x6f,0xd6,0x01,0xfe,0xff,0x01
+
+# GFX12: v_maximumminimum_f16 v5, s105, s105, exec_lo ; encoding: [0x05,0x00,0x6f,0xd6,0x69,0xd2,0xf8,0x01]
+0x05,0x00,0x6f,0xd6,0x69,0xd2,0xf8,0x01
+
+# GFX12: v_maximumminimum_f16 v5, v1, v2, s3     ; encoding: [0x05,0x00,0x6f,0xd6,0x01,0x05,0x0e,0x00]
+0x05,0x00,0x6f,0xd6,0x01,0x05,0x0e,0x00
+
+# GFX12: v_maximumminimum_f16 v5, v255, s2, s105 ; encoding: [0x05,0x00,0x6f,0xd6,0xff,0x05,0xa4,0x01]
+0x05,0x00,0x6f,0xd6,0xff,0x05,0xa4,0x01
+
+# GFX12: v_maximumminimum_f16 v5, vcc_hi, 0xaf12, v255 ; encoding: [0x05,0x00,0x6f,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00]
+0x05,0x00,0x6f,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00
+
+# GFX12: v_maximumminimum_f16 v5, vcc_lo, ttmp15, v3 ; encoding: [0x05,0x00,0x6f,0xd6,0x6a,0xf6,0x0c,0x04]
+0x05,0x00,0x6f,0xd6,0x6a,0xf6,0x0c,0x04
+
+# GFX12: v_maximumminimum_f16 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x6f,0xd6,0x7e,0x82,0xad,0x01]
+0x05,0x01,0x6f,0xd6,0x7e,0x82,0xad,0x01
+
+# GFX12: v_minimummaximum_f16 v5, -1, -|exec_hi|, -|src_scc| ; encoding: [0x05,0x06,0x6e,0xd6,0xc1,0xfe,0xf4,0xc3]
+0x05,0x06,0x6e,0xd6,0xc1,0xfe,0xf4,0xc3
+
+# GFX12: v_minimummaximum_f16 v5, -src_scc, |vcc_lo|, -1 ; encoding: [0x05,0x02,0x6e,0xd6,0xfd,0xd4,0x04,0x23]
+0x05,0x02,0x6e,0xd6,0xfd,0xd4,0x04,0x23
+
+# GFX12: v_minimummaximum_f16 v5, -|exec_hi|, null, -|vcc_lo| ; encoding: [0x05,0x05,0x6e,0xd6,0x7f,0xf8,0xa8,0xa1]
+0x05,0x05,0x6e,0xd6,0x7f,0xf8,0xa8,0xa1
+
+# GFX12: v_minimummaximum_f16 v5, -|ttmp15|, -|src_scc|, -|ttmp15| ; encoding: [0x05,0x07,0x6e,0xd6,0x7b,0xfa,0xed,0xe1]
+0x05,0x07,0x6e,0xd6,0x7b,0xfa,0xed,0xe1
+
+# GFX12: v_minimummaximum_f16 v5, 0.5, -m0, 0.5  ; encoding: [0x05,0x00,0x6e,0xd6,0xf0,0xfa,0xc0,0x43]
+0x05,0x00,0x6e,0xd6,0xf0,0xfa,0xc0,0x43
+
+# GFX12: v_minimummaximum_f16 v5, m0, 0.5, m0    ; encoding: [0x05,0x00,0x6e,0xd6,0x7d,0xe0,0xf5,0x01]
+0x05,0x00,0x6e,0xd6,0x7d,0xe0,0xf5,0x01
+
+# GFX12: v_minimummaximum_f16 v5, null, exec_lo, -|0xaf12| ; encoding: [0x05,0x04,0x6e,0xd6,0x7c,0xfc,0xfc,0x83,0x12,0xaf,0x00,0x00]
+0x05,0x04,0x6e,0xd6,0x7c,0xfc,0xfc,0x83,0x12,0xaf,0x00,0x00
+
+# GFX12: v_minimummaximum_f16 v5, s1, v255, exec_hi ; encoding: [0x05,0x00,0x6e,0xd6,0x01,0xfe,0xff,0x01]
+0x05,0x00,0x6e,0xd6,0x01,0xfe,0xff,0x01
+
+# GFX12: v_minimummaximum_f16 v5, s105, s105, exec_lo ; encoding: [0x05,0x00,0x6e,0xd6,0x69,0xd2,0xf8,0x01]
+0x05,0x00,0x6e,0xd6,0x69,0xd2,0xf8,0x01
+
+# GFX12: v_minimummaximum_f16 v5, v1, v2, s3     ; encoding: [0x05,0x00,0x6e,0xd6,0x01,0x05,0x0e,0x00]
+0x05,0x00,0x6e,0xd6,0x01,0x05,0x0e,0x00
+
+# GFX12: v_minimummaximum_f16 v5, v255, s2, s105 ; encoding: [0x05,0x00,0x6e,0xd6,0xff,0x05,0xa4,0x01]
+0x05,0x00,0x6e,0xd6,0xff,0x05,0xa4,0x01
+
+# GFX12: v_minimummaximum_f16 v5, vcc_hi, 0xaf12, v255 ; encoding: [0x05,0x00,0x6e,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00]
+0x05,0x00,0x6e,0xd6,0x6b,0xfe,0xfd,0x07,0x12,0xaf,0x00,0x00
+
+# GFX12: v_minimummaximum_f16 v5, vcc_lo, ttmp15, v3 ; encoding: [0x05,0x00,0x6e,0xd6,0x6a,0xf6,0x0c,0x04]
+0x05,0x00,0x6e,0xd6,0x6a,0xf6,0x0c,0x04
+
+# GFX12: v_minimummaximum_f16 v5, |exec_lo|, -1, vcc_hi ; encoding: [0x05,0x01,0x6e,0xd6,0x7e,0x82,0xad,0x01]
+0x05,0x01,0x6e,0xd6,0x7e,0x82,0xad,0x01
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_dpp16.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_dpp16.txt
index 40204c0dfdf4dfcac43161991f64f621fc852b4f..69f61c7eb8030f7a7ece02a8c11c8b5d8de58f22 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_dpp16.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_dpp16.txt
@@ -4110,3 +4110,507 @@
 
 # GFX12: v_dot2_bf16_bf16_e64_dpp v0, |v1|, -v2, -|s3| op_sel:[0,0,1,1] quad_perm:[0,1,2,3] row_mask:0x0 bank_mask:0x0 fi:1 ; encoding: [0x00,0x65,0x67,0xd6,0xfa,0x04,0x0e,0xc0,0x01,0xe4,0x04,0x00]
 0x00,0x65,0x67,0xd6,0xfa,0x04,0x0e,0xc0,0x01,0xe4,0x04,0x00
+
+# GFX12: v_minimum_f32_e64_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x03,0x65,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+0xff,0x03,0x65,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
+
+# GFX12: v_minimum_f32_e64_dpp v5, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x65,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+0x05,0x02,0x65,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13
+
+# GFX12: v_minimum_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
+
+# GFX12: v_minimum_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
+
+# GFX12: v_minimum_f32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
+
+# GFX12: v_minimum_f32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
+
+# GFX12: v_minimum_f32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
+
+# GFX12: v_minimum_f32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
+
+# GFX12: v_minimum_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
+
+# GFX12: v_minimum_f32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
+
+# GFX12: v_minimum_f32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
+
+# GFX12: v_minimum_f32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
+
+# GFX12: v_minimum_f32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+0x05,0x00,0x65,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
+
+# GFX12: v_minimum_f32_e64_dpp v5, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x65,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+0x05,0x01,0x65,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
+
+# GFX12: v_maximum_f32_e64_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x03,0x66,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+0xff,0x03,0x66,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
+
+# GFX12: v_maximum_f32_e64_dpp v5, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x66,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+0x05,0x02,0x66,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13
+
+# GFX12: v_maximum_f32_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
+
+# GFX12: v_maximum_f32_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
+
+# GFX12: v_maximum_f32_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
+
+# GFX12: v_maximum_f32_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
+
+# GFX12: v_maximum_f32_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
+
+# GFX12: v_maximum_f32_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
+
+# GFX12: v_maximum_f32_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
+
+# GFX12: v_maximum_f32_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
+
+# GFX12: v_maximum_f32_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
+
+# GFX12: v_maximum_f32_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
+
+# GFX12: v_maximum_f32_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+0x05,0x00,0x66,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
+
+# GFX12: v_maximum_f32_e64_dpp v5, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x66,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+0x05,0x01,0x66,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
+
+# GFX12: v_minimum_f16_e64_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x03,0x67,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+0xff,0x03,0x67,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
+
+# GFX12: v_minimum_f16_e64_dpp v5, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x67,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+0x05,0x02,0x67,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13
+
+# GFX12: v_minimum_f16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
+
+# GFX12: v_minimum_f16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
+
+# GFX12: v_minimum_f16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
+
+# GFX12: v_minimum_f16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
+
+# GFX12: v_minimum_f16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
+
+# GFX12: v_minimum_f16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
+
+# GFX12: v_minimum_f16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
+
+# GFX12: v_minimum_f16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
+
+# GFX12: v_minimum_f16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
+
+# GFX12: v_minimum_f16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
+
+# GFX12: v_minimum_f16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+0x05,0x00,0x67,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
+
+# GFX12: v_minimum_f16_e64_dpp v5, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x67,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+0x05,0x01,0x67,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
+
+# GFX12: v_maximum_f16_e64_dpp v255, -|v255|, -|v255| row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x03,0x68,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30]
+0xff,0x03,0x68,0xd7,0xfa,0xfe,0x03,0x60,0xff,0x6f,0x0d,0x30
+
+# GFX12: v_maximum_f16_e64_dpp v5, -v1, |v2| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x02,0x68,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13]
+0x05,0x02,0x68,0xd7,0xfa,0x04,0x02,0x20,0x01,0x60,0x09,0x13
+
+# GFX12: v_maximum_f16_e64_dpp v5, v1, v2 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff]
+0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0xe4,0x00,0xff
+
+# GFX12: v_maximum_f16_e64_dpp v5, v1, v2 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff]
+0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1b,0x00,0xff
+
+# GFX12: v_maximum_f16_e64_dpp v5, v1, v2 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff]
+0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x41,0x01,0xff
+
+# GFX12: v_maximum_f16_e64_dpp v5, v1, v2 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff]
+0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x40,0x01,0xff
+
+# GFX12: v_maximum_f16_e64_dpp v5, v1, v2 row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff]
+0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x21,0x01,0xff
+
+# GFX12: v_maximum_f16_e64_dpp v5, v1, v2 row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff]
+0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x2f,0x01,0xff
+
+# GFX12: v_maximum_f16_e64_dpp v5, v1, v2 row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff]
+0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x50,0x01,0xff
+
+# GFX12: v_maximum_f16_e64_dpp v5, v1, v2 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff]
+0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x01,0x01,0xff
+
+# GFX12: v_maximum_f16_e64_dpp v5, v1, v2 row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff]
+0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x0f,0x01,0xff
+
+# GFX12: v_maximum_f16_e64_dpp v5, v1, v2 row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff]
+0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x11,0x01,0xff
+
+# GFX12: v_maximum_f16_e64_dpp v5, v1, v2 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff]
+0x05,0x00,0x68,0xd7,0xfa,0x04,0x02,0x00,0x01,0x1f,0x01,0xff
+
+# GFX12: v_maximum_f16_e64_dpp v5, |v1|, -v2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x01,0x68,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01]
+0x05,0x01,0x68,0xd7,0xfa,0x04,0x02,0x40,0x01,0x5f,0x01,0x01
+
+# GFX12: v_maximum3_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x2e,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
+0xff,0x87,0x2e,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30
+
+# GFX12: v_maximum3_f32_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x2e,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
+0x05,0x04,0x2e,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff
+
+# GFX12: v_maximum3_f32_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x2e,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
+0x05,0x03,0x2e,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff
+
+# GFX12: v_maximum3_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x2e,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
+0x05,0x05,0x2e,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01
+
+# GFX12: v_maximum3_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x06,0x2e,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x09,0x13]
+0x05,0x06,0x2e,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x09,0x13
+
+# GFX12: v_maximum3_f32_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x2e,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
+0x05,0x02,0x2e,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff
+
+# GFX12: v_maximum3_f32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2e,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
+0x05,0x00,0x2e,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+
+# GFX12: v_maximum3_f32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2e,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
+0x05,0x00,0x2e,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+
+# GFX12: v_maximum3_f32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+0x05,0x00,0x2e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+
+# GFX12: v_maximum3_f32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+0x05,0x00,0x2e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+
+# GFX12: v_maximum3_f32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
+0x05,0x00,0x2e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+
+# GFX12: v_maximum3_f32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2e,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
+0x05,0x00,0x2e,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+
+# GFX12: v_maximum3_f32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2e,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
+0x05,0x00,0x2e,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+
+# GFX12: v_maximum3_f32_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x2e,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
+0x05,0x01,0x2e,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff
+
+# GFX12: v_minimum3_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x2d,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
+0xff,0x87,0x2d,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30
+
+# GFX12: v_minimum3_f32_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x2d,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
+0x05,0x04,0x2d,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff
+
+# GFX12: v_minimum3_f32_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x2d,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
+0x05,0x03,0x2d,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff
+
+# GFX12: v_minimum3_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x2d,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
+0x05,0x05,0x2d,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01
+
+# GFX12: v_minimum3_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x06,0x2d,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x09,0x13]
+0x05,0x06,0x2d,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x09,0x13
+
+# GFX12: v_minimum3_f32_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x2d,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
+0x05,0x02,0x2d,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff
+
+# GFX12: v_minimum3_f32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2d,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
+0x05,0x00,0x2d,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+
+# GFX12: v_minimum3_f32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2d,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
+0x05,0x00,0x2d,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+
+# GFX12: v_minimum3_f32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+0x05,0x00,0x2d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+
+# GFX12: v_minimum3_f32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+0x05,0x00,0x2d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+
+# GFX12: v_minimum3_f32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
+0x05,0x00,0x2d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+
+# GFX12: v_minimum3_f32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2d,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
+0x05,0x00,0x2d,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+
+# GFX12: v_minimum3_f32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2d,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
+0x05,0x00,0x2d,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+
+# GFX12: v_minimum3_f32_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x2d,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
+0x05,0x01,0x2d,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff
+
+# GFX12: v_maximum3_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x30,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
+0xff,0x87,0x30,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30
+
+# GFX12: v_maximum3_f16_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x30,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
+0x05,0x04,0x30,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff
+
+# GFX12: v_maximum3_f16_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x30,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
+0x05,0x03,0x30,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff
+
+# GFX12: v_maximum3_f16_e64_dpp v5, -|v1|, v2, -|-1| row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x30,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01]
+0x05,0x05,0x30,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01
+
+# GFX12: v_maximum3_f16_e64_dpp v5, v1, -|v2|, -|0.5| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x06,0x30,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x09,0x13]
+0x05,0x06,0x30,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x09,0x13
+
+# GFX12: v_maximum3_f16_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x30,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
+0x05,0x02,0x30,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff
+
+# GFX12: v_maximum3_f16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
+0x05,0x00,0x30,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+
+# GFX12: v_maximum3_f16_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
+0x05,0x00,0x30,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+
+# GFX12: v_maximum3_f16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+0x05,0x00,0x30,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+
+# GFX12: v_maximum3_f16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+0x05,0x00,0x30,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+
+# GFX12: v_maximum3_f16_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
+0x05,0x00,0x30,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+
+# GFX12: v_maximum3_f16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
+0x05,0x00,0x30,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+
+# GFX12: v_maximum3_f16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x30,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
+0x05,0x00,0x30,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+
+# GFX12: v_maximum3_f16_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x30,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
+0x05,0x01,0x30,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff
+
+# GFX12: v_minimum3_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x2f,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
+0xff,0x87,0x2f,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30
+
+# GFX12: v_minimum3_f16_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x2f,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
+0x05,0x04,0x2f,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff
+
+# GFX12: v_minimum3_f16_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x2f,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
+0x05,0x03,0x2f,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff
+
+# GFX12: v_minimum3_f16_e64_dpp v5, -|v1|, v2, -|-1| row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x2f,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01]
+0x05,0x05,0x2f,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01
+
+# GFX12: v_minimum3_f16_e64_dpp v5, v1, -|v2|, -|0.5| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x06,0x2f,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x09,0x13]
+0x05,0x06,0x2f,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x09,0x13
+
+# GFX12: v_minimum3_f16_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x2f,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
+0x05,0x02,0x2f,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff
+
+# GFX12: v_minimum3_f16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
+0x05,0x00,0x2f,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+
+# GFX12: v_minimum3_f16_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
+0x05,0x00,0x2f,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+
+# GFX12: v_minimum3_f16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+0x05,0x00,0x2f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+
+# GFX12: v_minimum3_f16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+0x05,0x00,0x2f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+
+# GFX12: v_minimum3_f16_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
+0x05,0x00,0x2f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+
+# GFX12: v_minimum3_f16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
+0x05,0x00,0x2f,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+
+# GFX12: v_minimum3_f16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x2f,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
+0x05,0x00,0x2f,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+
+# GFX12: v_minimum3_f16_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x2f,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
+0x05,0x01,0x2f,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff
+
+# GFX12: v_maximumminimum_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x6d,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
+0xff,0x87,0x6d,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30
+
+# GFX12: v_maximumminimum_f32_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x6d,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
+0x05,0x04,0x6d,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff
+
+# GFX12: v_maximumminimum_f32_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x6d,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
+0x05,0x03,0x6d,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff
+
+# GFX12: v_maximumminimum_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x6d,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
+0x05,0x05,0x6d,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01
+
+# GFX12: v_maximumminimum_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x06,0x6d,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x09,0x13]
+0x05,0x06,0x6d,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x09,0x13
+
+# GFX12: v_maximumminimum_f32_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x6d,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
+0x05,0x02,0x6d,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff
+
+# GFX12: v_maximumminimum_f32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6d,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
+0x05,0x00,0x6d,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+
+# GFX12: v_maximumminimum_f32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6d,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
+0x05,0x00,0x6d,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+
+# GFX12: v_maximumminimum_f32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+0x05,0x00,0x6d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+
+# GFX12: v_maximumminimum_f32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+0x05,0x00,0x6d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+
+# GFX12: v_maximumminimum_f32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
+0x05,0x00,0x6d,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+
+# GFX12: v_maximumminimum_f32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6d,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
+0x05,0x00,0x6d,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+
+# GFX12: v_maximumminimum_f32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6d,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
+0x05,0x00,0x6d,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+
+# GFX12: v_maximumminimum_f32_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x6d,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
+0x05,0x01,0x6d,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff
+
+# GFX12: v_minimummaximum_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x6c,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30]
+0xff,0x87,0x6c,0xd6,0xfa,0xfe,0xf7,0xfb,0xff,0x6f,0x0d,0x30
+
+# GFX12: v_minimummaximum_f32_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x6c,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
+0x05,0x04,0x6c,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff
+
+# GFX12: v_minimummaximum_f32_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x6c,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
+0x05,0x03,0x6c,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff
+
+# GFX12: v_minimummaximum_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x6c,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01]
+0x05,0x05,0x6c,0xd6,0xfa,0x04,0x06,0xab,0x01,0x5f,0x01,0x01
+
+# GFX12: v_minimummaximum_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x06,0x6c,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x09,0x13]
+0x05,0x06,0x6c,0xd6,0xfa,0x04,0xc2,0xd3,0x01,0x60,0x09,0x13
+
+# GFX12: v_minimummaximum_f32_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x6c,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
+0x05,0x02,0x6c,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff
+
+# GFX12: v_minimummaximum_f32_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6c,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
+0x05,0x00,0x6c,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+
+# GFX12: v_minimummaximum_f32_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6c,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
+0x05,0x00,0x6c,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+
+# GFX12: v_minimummaximum_f32_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6c,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+0x05,0x00,0x6c,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+
+# GFX12: v_minimummaximum_f32_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6c,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+0x05,0x00,0x6c,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+
+# GFX12: v_minimummaximum_f32_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6c,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
+0x05,0x00,0x6c,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+
+# GFX12: v_minimummaximum_f32_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6c,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
+0x05,0x00,0x6c,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+
+# GFX12: v_minimummaximum_f32_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6c,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
+0x05,0x00,0x6c,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+
+# GFX12: v_minimummaximum_f32_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x6c,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
+0x05,0x01,0x6c,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff
+
+# GFX12: v_maximumminimum_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x6f,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
+0xff,0x87,0x6f,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30
+
+# GFX12: v_maximumminimum_f16_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x6f,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
+0x05,0x04,0x6f,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff
+
+# GFX12: v_maximumminimum_f16_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x6f,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
+0x05,0x03,0x6f,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff
+
+# GFX12: v_maximumminimum_f16_e64_dpp v5, -|v1|, v2, -|-1| row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x6f,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01]
+0x05,0x05,0x6f,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01
+
+# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, -|v2|, -|0.5| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x06,0x6f,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x09,0x13]
+0x05,0x06,0x6f,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x09,0x13
+
+# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x6f,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
+0x05,0x02,0x6f,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff
+
+# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
+0x05,0x00,0x6f,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+
+# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
+0x05,0x00,0x6f,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+
+# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+0x05,0x00,0x6f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+
+# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+0x05,0x00,0x6f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+
+# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
+0x05,0x00,0x6f,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+
+# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
+0x05,0x00,0x6f,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+
+# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6f,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
+0x05,0x00,0x6f,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+
+# GFX12: v_maximumminimum_f16_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x6f,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
+0x05,0x01,0x6f,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff
+
+# GFX12: v_minimummaximum_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp row_xmask:15 row_mask:0x3 bank_mask:0x0 bound_ctrl:1 fi:1 ; encoding: [0xff,0x87,0x6e,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30]
+0xff,0x87,0x6e,0xd6,0xfa,0xfe,0xf7,0xe3,0xff,0x6f,0x0d,0x30
+
+# GFX12: v_minimummaximum_f16_e64_dpp v5, -v1, v2, |exec_lo| row_ror:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x04,0x6e,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff]
+0x05,0x04,0x6e,0xd6,0xfa,0x04,0xfa,0x21,0x01,0x2f,0x01,0xff
+
+# GFX12: v_minimummaximum_f16_e64_dpp v5, -|v1|, -|v2|, null row_share:0 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x03,0x6e,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff]
+0x05,0x03,0x6e,0xd6,0xfa,0x04,0xf2,0x61,0x01,0x50,0x01,0xff
+
+# GFX12: v_minimummaximum_f16_e64_dpp v5, -|v1|, v2, -|-1| row_share:15 row_mask:0x0 bank_mask:0x1 ; encoding: [0x05,0x05,0x6e,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01]
+0x05,0x05,0x6e,0xd6,0xfa,0x04,0x06,0xa3,0x01,0x5f,0x01,0x01
+
+# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, -|v2|, -|0.5| row_xmask:0 row_mask:0x1 bank_mask:0x3 bound_ctrl:1 ; encoding: [0x05,0x06,0x6e,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x09,0x13]
+0x05,0x06,0x6e,0xd6,0xfa,0x04,0xc2,0xc3,0x01,0x60,0x09,0x13
+
+# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, -|v2|, exec_hi row_ror:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x02,0x6e,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff]
+0x05,0x02,0x6e,0xd6,0xfa,0x04,0xfe,0x41,0x01,0x21,0x01,0xff
+
+# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, v2, s105 row_shl:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff]
+0x05,0x00,0x6e,0xd6,0xfa,0x04,0xa6,0x01,0x01,0x01,0x01,0xff
+
+# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, v2, v255 row_half_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff]
+0x05,0x00,0x6e,0xd6,0xfa,0x04,0xfe,0x07,0x01,0x41,0x01,0xff
+
+# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, v2, v3 quad_perm:[0,1,2,3] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff]
+0x05,0x00,0x6e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0xe4,0x00,0xff
+
+# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, v2, v3 quad_perm:[3,2,1,0] row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff]
+0x05,0x00,0x6e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x1b,0x00,0xff
+
+# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, v2, v3 row_mirror row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff]
+0x05,0x00,0x6e,0xd6,0xfa,0x04,0x0e,0x04,0x01,0x40,0x01,0xff
+
+# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, v2, vcc_hi row_shl:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff]
+0x05,0x00,0x6e,0xd6,0xfa,0x04,0xae,0x01,0x01,0x0f,0x01,0xff
+
+# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, v2, vcc_lo row_shr:1 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x00,0x6e,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff]
+0x05,0x00,0x6e,0xd6,0xfa,0x04,0xaa,0x01,0x01,0x11,0x01,0xff
+
+# GFX12: v_minimummaximum_f16_e64_dpp v5, |v1|, v2, -ttmp15 row_shr:15 row_mask:0xf bank_mask:0xf ; encoding: [0x05,0x01,0x6e,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff]
+0x05,0x01,0x6e,0xd6,0xfa,0x04,0xee,0x81,0x01,0x1f,0x01,0xff
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_dpp8.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_dpp8.txt
index 904ec7c1347f250781b5b8bc48df77e82e7e6c2c..a7f0183016147f0afe0dfec1efa345d3a4c8ed9f 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_dpp8.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3_dpp8.txt
@@ -2629,3 +2629,338 @@
 # GFX12: v_dot2_bf16_bf16_e64_dpp v0, |v1|, -v2, -|s3| op_sel:[0,0,1,1] dpp8:[0,1,2,3,4,4,4,4] ; encoding: [0x00,0x65,0x67,0xd6,0xe9,0x04,0x0e,0xc0,0x01,0x88,0x46,0x92]
 0x00,0x65,0x67,0xd6,0xe9,0x04,0x0e,0xc0,0x01,0x88,0x46,0x92
 
+# GFX12: v_maximum_f32_e64_dpp v255, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x03,0x66,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+0xff,0x03,0x66,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
+
+# GFX12: v_maximum_f32_e64_dpp v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x66,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+0x05,0x02,0x66,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05
+
+# GFX12: v_maximum_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x66,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+0x05,0x00,0x66,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
+
+# GFX12: v_maximum_f32_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x66,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+0x05,0x01,0x66,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
+
+# GFX12: v_minimum_f32_e64_dpp v255, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x03,0x65,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+0xff,0x03,0x65,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
+
+# GFX12: v_minimum_f32_e64_dpp v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x65,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+0x05,0x02,0x65,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05
+
+# GFX12: v_minimum_f32_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x65,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+0x05,0x00,0x65,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
+
+# GFX12: v_minimum_f32_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x65,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+0x05,0x01,0x65,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
+
+# GFX12: v_maximum_f16_e64_dpp v255, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x03,0x68,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+0xff,0x03,0x68,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
+
+# GFX12: v_maximum_f16_e64_dpp v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x68,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+0x05,0x02,0x68,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05
+
+# GFX12: v_maximum_f16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x68,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+0x05,0x00,0x68,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
+
+# GFX12: v_maximum_f16_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x68,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+0x05,0x01,0x68,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
+
+# GFX12: v_minimum_f16_e64_dpp v255, -|v255|, -|v255| dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x03,0x67,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00]
+0xff,0x03,0x67,0xd7,0xe9,0xfe,0x03,0x60,0xff,0x00,0x00,0x00
+
+# GFX12: v_minimum_f16_e64_dpp v5, -v1, |v2| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x02,0x67,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05]
+0x05,0x02,0x67,0xd7,0xea,0x04,0x02,0x20,0x01,0x77,0x39,0x05
+
+# GFX12: v_minimum_f16_e64_dpp v5, v1, v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x67,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05]
+0x05,0x00,0x67,0xd7,0xe9,0x04,0x02,0x00,0x01,0x77,0x39,0x05
+
+# GFX12: v_minimum_f16_e64_dpp v5, |v1|, -v2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x67,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05]
+0x05,0x01,0x67,0xd7,0xe9,0x04,0x02,0x40,0x01,0x77,0x39,0x05
+
+# GFX12: v_maximum3_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x87,0x2e,0xd6,0xe9,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
+0xff,0x87,0x2e,0xd6,0xe9,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00
+
+# GFX12: v_maximum3_f32_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x2e,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
+0x05,0x04,0x2e,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05
+
+# GFX12: v_maximum3_f32_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x2e,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
+0x05,0x03,0x2e,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05
+
+# GFX12: v_maximum3_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x2e,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
+0x05,0x05,0x2e,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05
+
+# GFX12: v_maximum3_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x06,0x2e,0xd6,0xea,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
+0x05,0x06,0x2e,0xd6,0xea,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05
+
+# GFX12: v_maximum3_f32_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x2e,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
+0x05,0x02,0x2e,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05
+
+# GFX12: v_maximum3_f32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2e,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+0x05,0x00,0x2e,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+
+# GFX12: v_maximum3_f32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2e,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+0x05,0x00,0x2e,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+
+# GFX12: v_maximum3_f32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2e,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+0x05,0x00,0x2e,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+
+# GFX12: v_maximum3_f32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2e,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+0x05,0x00,0x2e,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+
+# GFX12: v_maximum3_f32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2e,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+0x05,0x00,0x2e,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+
+# GFX12: v_maximum3_f32_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x2e,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
+0x05,0x01,0x2e,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05
+
+# GFX12: v_minimum3_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x87,0x2d,0xd6,0xe9,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
+0xff,0x87,0x2d,0xd6,0xe9,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00
+
+# GFX12: v_minimum3_f32_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x2d,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
+0x05,0x04,0x2d,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05
+
+# GFX12: v_minimum3_f32_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x2d,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
+0x05,0x03,0x2d,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05
+
+# GFX12: v_minimum3_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x2d,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
+0x05,0x05,0x2d,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05
+
+# GFX12: v_minimum3_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x06,0x2d,0xd6,0xea,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
+0x05,0x06,0x2d,0xd6,0xea,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05
+
+# GFX12: v_minimum3_f32_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x2d,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
+0x05,0x02,0x2d,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05
+
+# GFX12: v_minimum3_f32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2d,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+0x05,0x00,0x2d,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+
+# GFX12: v_minimum3_f32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2d,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+0x05,0x00,0x2d,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+
+# GFX12: v_minimum3_f32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2d,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+0x05,0x00,0x2d,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+
+# GFX12: v_minimum3_f32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2d,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+0x05,0x00,0x2d,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+
+# GFX12: v_minimum3_f32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2d,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+0x05,0x00,0x2d,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+
+# GFX12: v_minimum3_f32_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x2d,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
+0x05,0x01,0x2d,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05
+
+# GFX12: v_maximum3_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x87,0x30,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+0xff,0x87,0x30,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00
+
+# GFX12: v_maximum3_f16_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x30,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
+0x05,0x04,0x30,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05
+
+# GFX12: v_maximum3_f16_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x30,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
+0x05,0x03,0x30,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05
+
+# GFX12: v_maximum3_f16_e64_dpp v5, -|v1|, v2, -|-1| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x30,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05]
+0x05,0x05,0x30,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05
+
+# GFX12: v_maximum3_f16_e64_dpp v5, v1, -|v2|, -|0.5| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x06,0x30,0xd6,0xea,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05]
+0x05,0x06,0x30,0xd6,0xea,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05
+
+# GFX12: v_maximum3_f16_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x30,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
+0x05,0x02,0x30,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05
+
+# GFX12: v_maximum3_f16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+0x05,0x00,0x30,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+
+# GFX12: v_maximum3_f16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+0x05,0x00,0x30,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+
+# GFX12: v_maximum3_f16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+0x05,0x00,0x30,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+
+# GFX12: v_maximum3_f16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+0x05,0x00,0x30,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+
+# GFX12: v_maximum3_f16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x30,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+0x05,0x00,0x30,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+
+# GFX12: v_maximum3_f16_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x30,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
+0x05,0x01,0x30,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05
+
+# GFX12: v_minimum3_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x87,0x2f,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+0xff,0x87,0x2f,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00
+
+# GFX12: v_minimum3_f16_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x2f,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
+0x05,0x04,0x2f,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05
+
+# GFX12: v_minimum3_f16_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x2f,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
+0x05,0x03,0x2f,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05
+
+# GFX12: v_minimum3_f16_e64_dpp v5, -|v1|, v2, -|-1| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x2f,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05]
+0x05,0x05,0x2f,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05
+
+# GFX12: v_minimum3_f16_e64_dpp v5, v1, -|v2|, -|0.5| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x06,0x2f,0xd6,0xea,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05]
+0x05,0x06,0x2f,0xd6,0xea,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05
+
+# GFX12: v_minimum3_f16_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x2f,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
+0x05,0x02,0x2f,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05
+
+# GFX12: v_minimum3_f16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+0x05,0x00,0x2f,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+
+# GFX12: v_minimum3_f16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+0x05,0x00,0x2f,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+
+# GFX12: v_minimum3_f16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+0x05,0x00,0x2f,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+
+# GFX12: v_minimum3_f16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+0x05,0x00,0x2f,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+
+# GFX12: v_minimum3_f16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x2f,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+0x05,0x00,0x2f,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+
+# GFX12: v_minimum3_f16_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x2f,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
+0x05,0x01,0x2f,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05
+
+# GFX12: v_maximumminimum_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x87,0x6d,0xd6,0xe9,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
+0xff,0x87,0x6d,0xd6,0xe9,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00
+
+# GFX12: v_maximumminimum_f32_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x6d,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
+0x05,0x04,0x6d,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05
+
+# GFX12: v_maximumminimum_f32_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x6d,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
+0x05,0x03,0x6d,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05
+
+# GFX12: v_maximumminimum_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x6d,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
+0x05,0x05,0x6d,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05
+
+# GFX12: v_maximumminimum_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x06,0x6d,0xd6,0xea,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
+0x05,0x06,0x6d,0xd6,0xea,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05
+
+# GFX12: v_maximumminimum_f32_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x6d,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
+0x05,0x02,0x6d,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05
+
+# GFX12: v_maximumminimum_f32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6d,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+0x05,0x00,0x6d,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+
+# GFX12: v_maximumminimum_f32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6d,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+0x05,0x00,0x6d,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+
+# GFX12: v_maximumminimum_f32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6d,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+0x05,0x00,0x6d,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+
+# GFX12: v_maximumminimum_f32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6d,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+0x05,0x00,0x6d,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+
+# GFX12: v_maximumminimum_f32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6d,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+0x05,0x00,0x6d,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+
+# GFX12: v_maximumminimum_f32_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x6d,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
+0x05,0x01,0x6d,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05
+
+# GFX12: v_minimummaximum_f32_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp div:2 dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x87,0x6c,0xd6,0xe9,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00]
+0xff,0x87,0x6c,0xd6,0xe9,0xfe,0xf7,0xfb,0xff,0x00,0x00,0x00
+
+# GFX12: v_minimummaximum_f32_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x6c,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
+0x05,0x04,0x6c,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05
+
+# GFX12: v_minimummaximum_f32_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x6c,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
+0x05,0x03,0x6c,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05
+
+# GFX12: v_minimummaximum_f32_e64_dpp v5, -|v1|, v2, -|-1| mul:2 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x6c,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05]
+0x05,0x05,0x6c,0xd6,0xe9,0x04,0x06,0xab,0x01,0x77,0x39,0x05
+
+# GFX12: v_minimummaximum_f32_e64_dpp v5, v1, -|v2|, -|0.5| mul:4 dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x06,0x6c,0xd6,0xea,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05]
+0x05,0x06,0x6c,0xd6,0xea,0x04,0xc2,0xd3,0x01,0x77,0x39,0x05
+
+# GFX12: v_minimummaximum_f32_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x6c,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
+0x05,0x02,0x6c,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05
+
+# GFX12: v_minimummaximum_f32_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6c,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+0x05,0x00,0x6c,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+
+# GFX12: v_minimummaximum_f32_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6c,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+0x05,0x00,0x6c,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+
+# GFX12: v_minimummaximum_f32_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6c,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+0x05,0x00,0x6c,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+
+# GFX12: v_minimummaximum_f32_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6c,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+0x05,0x00,0x6c,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+
+# GFX12: v_minimummaximum_f32_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6c,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+0x05,0x00,0x6c,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+
+# GFX12: v_minimummaximum_f32_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x6c,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
+0x05,0x01,0x6c,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05
+
+# GFX12: v_maximumminimum_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x87,0x6f,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+0xff,0x87,0x6f,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00
+
+# GFX12: v_maximumminimum_f16_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x6f,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
+0x05,0x04,0x6f,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05
+
+# GFX12: v_maximumminimum_f16_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x6f,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
+0x05,0x03,0x6f,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05
+
+# GFX12: v_maximumminimum_f16_e64_dpp v5, -|v1|, v2, -|-1| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x6f,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05]
+0x05,0x05,0x6f,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05
+
+# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, -|v2|, -|0.5| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x06,0x6f,0xd6,0xea,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05]
+0x05,0x06,0x6f,0xd6,0xea,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05
+
+# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x6f,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
+0x05,0x02,0x6f,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05
+
+# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6f,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+0x05,0x00,0x6f,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+
+# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6f,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+0x05,0x00,0x6f,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+
+# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6f,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+0x05,0x00,0x6f,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+
+# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6f,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+0x05,0x00,0x6f,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+
+# GFX12: v_maximumminimum_f16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6f,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+0x05,0x00,0x6f,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+
+# GFX12: v_maximumminimum_f16_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x6f,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
+0x05,0x01,0x6f,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05
+
+# GFX12: v_minimummaximum_f16_e64_dpp v255, -|v255|, -|v255|, -|src_scc| clamp dpp8:[0,0,0,0,0,0,0,0] ; encoding: [0xff,0x87,0x6e,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00]
+0xff,0x87,0x6e,0xd6,0xe9,0xfe,0xf7,0xe3,0xff,0x00,0x00,0x00
+
+# GFX12: v_minimummaximum_f16_e64_dpp v5, -v1, v2, |exec_lo| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x04,0x6e,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05]
+0x05,0x04,0x6e,0xd6,0xe9,0x04,0xfa,0x21,0x01,0x77,0x39,0x05
+
+# GFX12: v_minimummaximum_f16_e64_dpp v5, -|v1|, -|v2|, null dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x03,0x6e,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05]
+0x05,0x03,0x6e,0xd6,0xe9,0x04,0xf2,0x61,0x01,0x77,0x39,0x05
+
+# GFX12: v_minimummaximum_f16_e64_dpp v5, -|v1|, v2, -|-1| dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x05,0x6e,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05]
+0x05,0x05,0x6e,0xd6,0xe9,0x04,0x06,0xa3,0x01,0x77,0x39,0x05
+
+# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, -|v2|, -|0.5| dpp8:[7,6,5,4,3,2,1,0] fi:1 ; encoding: [0x05,0x06,0x6e,0xd6,0xea,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05]
+0x05,0x06,0x6e,0xd6,0xea,0x04,0xc2,0xc3,0x01,0x77,0x39,0x05
+
+# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, -|v2|, exec_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x02,0x6e,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05]
+0x05,0x02,0x6e,0xd6,0xe9,0x04,0xfe,0x41,0x01,0x77,0x39,0x05
+
+# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, v2, s105 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6e,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05]
+0x05,0x00,0x6e,0xd6,0xe9,0x04,0xa6,0x01,0x01,0x77,0x39,0x05
+
+# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, v2, v255 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6e,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05]
+0x05,0x00,0x6e,0xd6,0xe9,0x04,0xfe,0x07,0x01,0x77,0x39,0x05
+
+# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, v2, v3 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6e,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05]
+0x05,0x00,0x6e,0xd6,0xe9,0x04,0x0e,0x04,0x01,0x77,0x39,0x05
+
+# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, v2, vcc_hi dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6e,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05]
+0x05,0x00,0x6e,0xd6,0xe9,0x04,0xae,0x01,0x01,0x77,0x39,0x05
+
+# GFX12: v_minimummaximum_f16_e64_dpp v5, v1, v2, vcc_lo dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x00,0x6e,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05]
+0x05,0x00,0x6e,0xd6,0xe9,0x04,0xaa,0x01,0x01,0x77,0x39,0x05
+
+# GFX12: v_minimummaximum_f16_e64_dpp v5, |v1|, v2, -ttmp15 dpp8:[7,6,5,4,3,2,1,0] ; encoding: [0x05,0x01,0x6e,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05]
+0x05,0x01,0x6e,0xd6,0xe9,0x04,0xee,0x81,0x01,0x77,0x39,0x05
diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3p.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3p.txt
index d07516b21cc61f9d266f285fa91617f64bf5fe1f..373cd71261449d86b4fc41aa14703ff3904a5a0c 100644
--- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3p.txt
+++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vop3p.txt
@@ -1251,3 +1251,93 @@
 
 # GFX12: v_pk_sub_u16 v255, 0xfe0b, vcc_hi op_sel:[0,1] op_sel_hi:[1,0] clamp ; encoding: [0xff,0xd0,0x0b,0xcc,0xff,0xd6,0x00,0x08,0x0b,0xfe,0x00,0x00]
 0xff,0xd0,0x0b,0xcc,0xff,0xd6,0x00,0x08,0x0b,0xfe,0x00,0x00
+
+# GFX12: v_pk_maximum_f16 v255, 0xfe0b, vcc_hi op_sel:[0,1] op_sel_hi:[1,0] neg_lo:[1,1] neg_hi:[1,1] clamp ; encoding: [0xff,0xd3,0x1e,0xcc,0xff,0xd6,0x00,0x68,0x0b,0xfe,0x00,0x00]
+0xff,0xd3,0x1e,0xcc,0xff,0xd6,0x00,0x68,0x0b,0xfe,0x00,0x00
+
+# GFX12: v_pk_maximum_f16 v5, -1, exec_hi op_sel:[1,1] op_sel_hi:[0,0] neg_lo:[1,0] neg_hi:[1,0] ; encoding: [0x05,0x59,0x1e,0xcc,0xc1,0xfe,0x00,0x20]
+0x05,0x59,0x1e,0xcc,0xc1,0xfe,0x00,0x20
+
+# GFX12: v_pk_maximum_f16 v5, 0.5, m0 neg_lo:[0,1] neg_hi:[0,1] ; encoding: [0x05,0x42,0x1e,0xcc,0xf0,0xfa,0x00,0x58]
+0x05,0x42,0x1e,0xcc,0xf0,0xfa,0x00,0x58
+
+# GFX12: v_pk_maximum_f16 v5, exec_hi, null      ; encoding: [0x05,0x40,0x1e,0xcc,0x7f,0xf8,0x00,0x18]
+0x05,0x40,0x1e,0xcc,0x7f,0xf8,0x00,0x18
+
+# GFX12: v_pk_maximum_f16 v5, exec_lo, -1        ; encoding: [0x05,0x40,0x1e,0xcc,0x7e,0x82,0x01,0x18]
+0x05,0x40,0x1e,0xcc,0x7e,0x82,0x01,0x18
+
+# GFX12: v_pk_maximum_f16 v5, m0, 0.5            ; encoding: [0x05,0x40,0x1e,0xcc,0x7d,0xe0,0x01,0x18]
+0x05,0x40,0x1e,0xcc,0x7d,0xe0,0x01,0x18
+
+# GFX12: v_pk_maximum_f16 v5, null, exec_lo      ; encoding: [0x05,0x40,0x1e,0xcc,0x7c,0xfc,0x00,0x18]
+0x05,0x40,0x1e,0xcc,0x7c,0xfc,0x00,0x18
+
+# GFX12: v_pk_maximum_f16 v5, s1, s2             ; encoding: [0x05,0x40,0x1e,0xcc,0x01,0x04,0x00,0x18]
+0x05,0x40,0x1e,0xcc,0x01,0x04,0x00,0x18
+
+# GFX12: v_pk_maximum_f16 v5, s105, s105         ; encoding: [0x05,0x40,0x1e,0xcc,0x69,0xd2,0x00,0x18]
+0x05,0x40,0x1e,0xcc,0x69,0xd2,0x00,0x18
+
+# GFX12: v_pk_maximum_f16 v5, src_scc, vcc_lo op_sel:[1,0] op_sel_hi:[0,1] ; encoding: [0x05,0x48,0x1e,0xcc,0xfd,0xd4,0x00,0x10]
+0x05,0x48,0x1e,0xcc,0xfd,0xd4,0x00,0x10
+
+# GFX12: v_pk_maximum_f16 v5, ttmp15, src_scc    ; encoding: [0x05,0x40,0x1e,0xcc,0x7b,0xfa,0x01,0x18]
+0x05,0x40,0x1e,0xcc,0x7b,0xfa,0x01,0x18
+
+# GFX12: v_pk_maximum_f16 v5, v1, v2             ; encoding: [0x05,0x40,0x1e,0xcc,0x01,0x05,0x02,0x18]
+0x05,0x40,0x1e,0xcc,0x01,0x05,0x02,0x18
+
+# GFX12: v_pk_maximum_f16 v5, v255, v255         ; encoding: [0x05,0x40,0x1e,0xcc,0xff,0xff,0x03,0x18]
+0x05,0x40,0x1e,0xcc,0xff,0xff,0x03,0x18
+
+# GFX12: v_pk_maximum_f16 v5, vcc_hi, 0xfe0b     ; encoding: [0x05,0x40,0x1e,0xcc,0x6b,0xfe,0x01,0x18,0x0b,0xfe,0x00,0x00]
+0x05,0x40,0x1e,0xcc,0x6b,0xfe,0x01,0x18,0x0b,0xfe,0x00,0x00
+
+# GFX12: v_pk_maximum_f16 v5, vcc_lo, ttmp15     ; encoding: [0x05,0x40,0x1e,0xcc,0x6a,0xf6,0x00,0x18]
+0x05,0x40,0x1e,0xcc,0x6a,0xf6,0x00,0x18
+
+# GFX12: v_pk_minimum_f16 v255, 0xfe0b, vcc_hi op_sel:[0,1] op_sel_hi:[1,0] neg_lo:[1,1] neg_hi:[1,1] clamp ; encoding: [0xff,0xd3,0x1d,0xcc,0xff,0xd6,0x00,0x68,0x0b,0xfe,0x00,0x00]
+0xff,0xd3,0x1d,0xcc,0xff,0xd6,0x00,0x68,0x0b,0xfe,0x00,0x00
+
+# GFX12: v_pk_minimum_f16 v5, -1, exec_hi op_sel:[1,1] op_sel_hi:[0,0] neg_lo:[1,0] neg_hi:[1,0] ; encoding: [0x05,0x59,0x1d,0xcc,0xc1,0xfe,0x00,0x20]
+0x05,0x59,0x1d,0xcc,0xc1,0xfe,0x00,0x20
+
+# GFX12: v_pk_minimum_f16 v5, 0.5, m0 neg_lo:[0,1] neg_hi:[0,1] ; encoding: [0x05,0x42,0x1d,0xcc,0xf0,0xfa,0x00,0x58]
+0x05,0x42,0x1d,0xcc,0xf0,0xfa,0x00,0x58
+
+# GFX12: v_pk_minimum_f16 v5, exec_hi, null      ; encoding: [0x05,0x40,0x1d,0xcc,0x7f,0xf8,0x00,0x18]
+0x05,0x40,0x1d,0xcc,0x7f,0xf8,0x00,0x18
+
+# GFX12: v_pk_minimum_f16 v5, exec_lo, -1        ; encoding: [0x05,0x40,0x1d,0xcc,0x7e,0x82,0x01,0x18]
+0x05,0x40,0x1d,0xcc,0x7e,0x82,0x01,0x18
+
+# GFX12: v_pk_minimum_f16 v5, m0, 0.5            ; encoding: [0x05,0x40,0x1d,0xcc,0x7d,0xe0,0x01,0x18]
+0x05,0x40,0x1d,0xcc,0x7d,0xe0,0x01,0x18
+
+# GFX12: v_pk_minimum_f16 v5, null, exec_lo      ; encoding: [0x05,0x40,0x1d,0xcc,0x7c,0xfc,0x00,0x18]
+0x05,0x40,0x1d,0xcc,0x7c,0xfc,0x00,0x18
+
+# GFX12: v_pk_minimum_f16 v5, s1, s2             ; encoding: [0x05,0x40,0x1d,0xcc,0x01,0x04,0x00,0x18]
+0x05,0x40,0x1d,0xcc,0x01,0x04,0x00,0x18
+
+# GFX12: v_pk_minimum_f16 v5, s105, s105         ; encoding: [0x05,0x40,0x1d,0xcc,0x69,0xd2,0x00,0x18]
+0x05,0x40,0x1d,0xcc,0x69,0xd2,0x00,0x18
+
+# GFX12: v_pk_minimum_f16 v5, src_scc, vcc_lo op_sel:[1,0] op_sel_hi:[0,1] ; encoding: [0x05,0x48,0x1d,0xcc,0xfd,0xd4,0x00,0x10]
+0x05,0x48,0x1d,0xcc,0xfd,0xd4,0x00,0x10
+
+# GFX12: v_pk_minimum_f16 v5, ttmp15, src_scc    ; encoding: [0x05,0x40,0x1d,0xcc,0x7b,0xfa,0x01,0x18]
+0x05,0x40,0x1d,0xcc,0x7b,0xfa,0x01,0x18
+
+# GFX12: v_pk_minimum_f16 v5, v1, v2             ; encoding: [0x05,0x40,0x1d,0xcc,0x01,0x05,0x02,0x18]
+0x05,0x40,0x1d,0xcc,0x01,0x05,0x02,0x18
+
+# GFX12: v_pk_minimum_f16 v5, v255, v255         ; encoding: [0x05,0x40,0x1d,0xcc,0xff,0xff,0x03,0x18]
+0x05,0x40,0x1d,0xcc,0xff,0xff,0x03,0x18
+
+# GFX12: v_pk_minimum_f16 v5, vcc_hi, 0xfe0b     ; encoding: [0x05,0x40,0x1d,0xcc,0x6b,0xfe,0x01,0x18,0x0b,0xfe,0x00,0x00]
+0x05,0x40,0x1d,0xcc,0x6b,0xfe,0x01,0x18,0x0b,0xfe,0x00,0x00
+
+# GFX12: v_pk_minimum_f16 v5, vcc_lo, ttmp15     ; encoding: [0x05,0x40,0x1d,0xcc,0x6a,0xf6,0x00,0x18]
+0x05,0x40,0x1d,0xcc,0x6a,0xf6,0x00,0x18
diff --git a/llvm/test/MC/Disassembler/PowerPC/ppc64-operands.txt b/llvm/test/MC/Disassembler/PowerPC/ppc64-operands.txt
index 3fcba735584b93144e6cb9a5638d0382065bfb03..8d63b153b2d5ef2ef86a9fd5150abd549819ec3a 100644
--- a/llvm/test/MC/Disassembler/PowerPC/ppc64-operands.txt
+++ b/llvm/test/MC/Disassembler/PowerPC/ppc64-operands.txt
@@ -81,7 +81,7 @@
 # CHECK: b .+1024                        
 0x48 0x00 0x04 0x00
 
-# CHECK: ba 1024                         
+# CHECK: ba 0x400
 0x48 0x00 0x04 0x02
 
 # FIXME: decode as beq 0, .+1024
@@ -89,6 +89,6 @@
 0x41 0x82 0x04 0x00
 
 # FIXME: decode as beqa 0, 1024
-# CHECK: bta 2, 1024
+# CHECK: bta 2, 0x400
 0x41 0x82 0x04 0x02
 
diff --git a/llvm/test/MC/Disassembler/X86/avx512_bf16-32.txt b/llvm/test/MC/Disassembler/X86/avx512_bf16-32.txt
new file mode 100644
index 0000000000000000000000000000000000000000..42f0c75fe367d4101773d33d69248ea119b40445
--- /dev/null
+++ b/llvm/test/MC/Disassembler/X86/avx512_bf16-32.txt
@@ -0,0 +1,90 @@
+# RUN: llvm-mc -triple i686-unknown-unknown -disassemble %s | FileCheck %s --check-prefix=ATT
+# RUN: llvm-mc -triple i686-unknown-unknown -disassemble -output-asm-variant=1 %s | FileCheck %s --check-prefix=INTEL
+
+# ATT:   vcvtne2ps2bf16	%zmm4, %zmm5, %zmm6
+# INTEL: vcvtne2ps2bf16	zmm6, zmm5, zmm4
+0x62,0xf2,0x57,0x48,0x72,0xf4
+
+# ATT:   vcvtne2ps2bf16	%zmm4, %zmm5, %zmm6 {%k7}
+# INTEL: vcvtne2ps2bf16	zmm6 {k7}, zmm5, zmm4
+0x62,0xf2,0x57,0x4f,0x72,0xf4
+
+# ATT:   vcvtne2ps2bf16	%zmm4, %zmm5, %zmm6 {%k7} {z}
+# INTEL: vcvtne2ps2bf16	zmm6 {k7} {z}, zmm5, zmm4
+0x62,0xf2,0x57,0xcf,0x72,0xf4
+
+# ATT:   vcvtne2ps2bf16	(%ecx), %zmm5, %zmm6
+# INTEL: vcvtne2ps2bf16	zmm6, zmm5, zmmword ptr [ecx]
+0x62,0xf2,0x57,0x48,0x72,0x31
+
+# ATT:   vcvtne2ps2bf16	291(%esp,%esi,8), %zmm5, %zmm6
+# INTEL: vcvtne2ps2bf16	zmm6, zmm5, zmmword ptr [esp + 8*esi + 291]
+0x62,0xf2,0x57,0x48,0x72,0xb4,0xf4,0x23,0x01,0x00,0x00
+
+# ATT:   vcvtne2ps2bf16	268435456(%esp,%esi,8), %zmm5, %zmm6
+# INTEL: vcvtne2ps2bf16	zmm6, zmm5, zmmword ptr [esp + 8*esi + 268435456]
+0x62,0xf2,0x57,0x48,0x72,0xb4,0xf4,0x00,0x00,0x00,0x10
+
+# ATT:   vcvtne2ps2bf16	-64(%esp), %zmm5, %zmm6
+# INTEL: vcvtne2ps2bf16	zmm6, zmm5, zmmword ptr [esp - 64]
+0x62,0xf2,0x57,0x48,0x72,0x74,0x24,0xff
+
+# ATT:   vcvtne2ps2bf16	(%eax){1to16}, %zmm5, %zmm6
+# INTEL: vcvtne2ps2bf16	zmm6, zmm5, dword ptr [eax]{1to16}
+0x62,0xf2,0x57,0x58,0x72,0x30
+
+# ATT:   vcvtne2ps2bf16	8128(%edx), %zmm5, %zmm6
+# INTEL: vcvtne2ps2bf16	zmm6, zmm5, zmmword ptr [edx + 8128]
+0x62,0xf2,0x57,0x48,0x72,0x72,0x7f
+
+# ATT:   vcvtne2ps2bf16	-8192(%edx), %zmm5, %zmm6
+# INTEL: vcvtne2ps2bf16	zmm6, zmm5, zmmword ptr [edx - 8192]
+0x62,0xf2,0x57,0x48,0x72,0x72,0x80
+
+# ATT:   vcvtne2ps2bf16	508(%edx){1to16}, %zmm5, %zmm6
+# INTEL: vcvtne2ps2bf16	zmm6, zmm5, dword ptr [edx + 508]{1to16}
+0x62,0xf2,0x57,0x58,0x72,0x72,0x7f
+
+# ATT:   vcvtne2ps2bf16	-512(%edx){1to16}, %zmm5, %zmm6
+# INTEL: vcvtne2ps2bf16	zmm6, zmm5, dword ptr [edx - 512]{1to16}
+0x62,0xf2,0x57,0x58,0x72,0x72,0x80
+
+# ATT:   vcvtneps2bf16	%zmm5, %ymm6
+# INTEL: vcvtneps2bf16	ymm6, zmm5
+0x62,0xf2,0x7e,0x48,0x72,0xf5
+
+# ATT:   vcvtneps2bf16	268435456(%esp,%esi,8), %ymm6 {%k7}
+# INTEL: vcvtneps2bf16	ymm6 {k7}, zmmword ptr [esp + 8*esi + 268435456]
+0x62,0xf2,0x7e,0x4f,0x72,0xb4,0xf4,0x00,0x00,0x00,0x10
+
+# ATT:   vcvtneps2bf16	(%ecx){1to16}, %ymm6
+# INTEL: vcvtneps2bf16	ymm6, dword ptr [ecx]{1to16}
+0x62,0xf2,0x7e,0x58,0x72,0x31
+
+# ATT:   vcvtneps2bf16	8128(%ecx), %ymm6
+# INTEL: vcvtneps2bf16	ymm6, zmmword ptr [ecx + 8128]
+0x62,0xf2,0x7e,0x48,0x72,0x71,0x7f
+
+# ATT:   vcvtneps2bf16	-512(%edx){1to16}, %ymm6 {%k7} {z}
+# INTEL: vcvtneps2bf16	ymm6 {k7} {z}, dword ptr [edx - 512]{1to16}
+0x62,0xf2,0x7e,0xdf,0x72,0x72,0x80
+
+# ATT:   vdpbf16ps	%zmm4, %zmm5, %zmm6
+# INTEL: vdpbf16ps	zmm6, zmm5, zmm4
+0x62,0xf2,0x56,0x48,0x52,0xf4
+
+# ATT:   vdpbf16ps	268435456(%esp,%esi,8), %zmm5, %zmm6 {%k7}
+# INTEL: vdpbf16ps	zmm6 {k7}, zmm5, zmmword ptr [esp + 8*esi + 268435456]
+0x62,0xf2,0x56,0x4f,0x52,0xb4,0xf4,0x00,0x00,0x00,0x10
+
+# ATT:   vdpbf16ps	(%ecx){1to16}, %zmm5, %zmm6
+# INTEL: vdpbf16ps	zmm6, zmm5, dword ptr [ecx]{1to16}
+0x62,0xf2,0x56,0x58,0x52,0x31
+
+# ATT:   vdpbf16ps	8128(%ecx), %zmm5, %zmm6
+# INTEL: vdpbf16ps	zmm6, zmm5, zmmword ptr [ecx + 8128]
+0x62,0xf2,0x56,0x48,0x52,0x71,0x7f
+
+# ATT:   vdpbf16ps	-512(%edx){1to16}, %zmm5, %zmm6 {%k7} {z}
+# INTEL: vdpbf16ps	zmm6 {k7} {z}, zmm5, dword ptr [edx - 512]{1to16}
+0x62,0xf2,0x56,0xdf,0x52,0x72,0x80
diff --git a/llvm/test/MC/Disassembler/X86/avx512_bf16-64.txt b/llvm/test/MC/Disassembler/X86/avx512_bf16-64.txt
new file mode 100644
index 0000000000000000000000000000000000000000..7296bd4ed3d9713e3e095d9793f6debfa2c61c67
--- /dev/null
+++ b/llvm/test/MC/Disassembler/X86/avx512_bf16-64.txt
@@ -0,0 +1,90 @@
+# RUN: llvm-mc -triple x86_64 -disassemble %s | FileCheck %s --check-prefix=ATT
+# RUN: llvm-mc -triple x86_64 -disassemble -output-asm-variant=1 %s | FileCheck %s --check-prefix=INTEL
+
+# ATT:   vcvtne2ps2bf16	%zmm28, %zmm29, %zmm30
+# INTEL: vcvtne2ps2bf16	zmm30, zmm29, zmm28
+0x62,0x02,0x17,0x40,0x72,0xf4
+
+# ATT:   vcvtne2ps2bf16	%zmm28, %zmm29, %zmm30 {%k7}
+# INTEL: vcvtne2ps2bf16	zmm30 {k7}, zmm29, zmm28
+0x62,0x02,0x17,0x47,0x72,0xf4
+
+# ATT:   vcvtne2ps2bf16	%zmm28, %zmm29, %zmm30 {%k7} {z}
+# INTEL: vcvtne2ps2bf16	zmm30 {k7} {z}, zmm29, zmm28
+0x62,0x02,0x17,0xc7,0x72,0xf4
+
+# ATT:   vcvtne2ps2bf16	(%rcx), %zmm29, %zmm30
+# INTEL: vcvtne2ps2bf16	zmm30, zmm29, zmmword ptr [rcx]
+0x62,0x62,0x17,0x40,0x72,0x31
+
+# ATT:   vcvtne2ps2bf16	291(%rax,%r14,8), %zmm29, %zmm30
+# INTEL: vcvtne2ps2bf16	zmm30, zmm29, zmmword ptr [rax + 8*r14 + 291]
+0x62,0x22,0x17,0x40,0x72,0xb4,0xf0,0x23,0x01,0x00,0x00
+
+# ATT:   vcvtne2ps2bf16	268435456(%rax,%r14,8), %zmm29, %zmm30
+# INTEL: vcvtne2ps2bf16	zmm30, zmm29, zmmword ptr [rax + 8*r14 + 268435456]
+0x62,0x22,0x17,0x40,0x72,0xb4,0xf0,0x00,0x00,0x00,0x10
+
+# ATT:   vcvtne2ps2bf16	-64(%rsp), %zmm29, %zmm30
+# INTEL: vcvtne2ps2bf16	zmm30, zmm29, zmmword ptr [rsp - 64]
+0x62,0x62,0x17,0x40,0x72,0x74,0x24,0xff
+
+# ATT:   vcvtne2ps2bf16	(%rcx){1to16}, %zmm29, %zmm30
+# INTEL: vcvtne2ps2bf16	zmm30, zmm29, dword ptr [rcx]{1to16}
+0x62,0x62,0x17,0x50,0x72,0x31
+
+# ATT:   vcvtne2ps2bf16	8128(%rdx), %zmm29, %zmm30
+# INTEL: vcvtne2ps2bf16	zmm30, zmm29, zmmword ptr [rdx + 8128]
+0x62,0x62,0x17,0x40,0x72,0x72,0x7f
+
+# ATT:   vcvtne2ps2bf16	-8192(%rdx), %zmm29, %zmm30
+# INTEL: vcvtne2ps2bf16	zmm30, zmm29, zmmword ptr [rdx - 8192]
+0x62,0x62,0x17,0x40,0x72,0x72,0x80
+
+# ATT:   vcvtne2ps2bf16	508(%rdx){1to16}, %zmm29, %zmm30
+# INTEL: vcvtne2ps2bf16	zmm30, zmm29, dword ptr [rdx + 508]{1to16}
+0x62,0x62,0x17,0x50,0x72,0x72,0x7f
+
+# ATT:   vcvtne2ps2bf16	-512(%rdx){1to16}, %zmm29, %zmm30
+# INTEL: vcvtne2ps2bf16	zmm30, zmm29, dword ptr [rdx - 512]{1to16}
+0x62,0x62,0x17,0x50,0x72,0x72,0x80
+
+# ATT:   vcvtneps2bf16	%zmm29, %ymm30
+# INTEL: vcvtneps2bf16	ymm30, zmm29
+0x62,0x02,0x7e,0x48,0x72,0xf5
+
+# ATT:   vcvtneps2bf16	268435456(%rbp,%r14,8), %ymm30 {%k7}
+# INTEL: vcvtneps2bf16	ymm30 {k7}, zmmword ptr [rbp + 8*r14 + 268435456]
+0x62,0x22,0x7e,0x4f,0x72,0xb4,0xf5,0x00,0x00,0x00,0x10
+
+# ATT:   vcvtneps2bf16	(%r9){1to16}, %ymm30
+# INTEL: vcvtneps2bf16	ymm30, dword ptr [r9]{1to16}
+0x62,0x42,0x7e,0x58,0x72,0x31
+
+# ATT:   vcvtneps2bf16	8128(%rcx), %ymm30
+# INTEL: vcvtneps2bf16	ymm30, zmmword ptr [rcx + 8128]
+0x62,0x62,0x7e,0x48,0x72,0x71,0x7f
+
+# ATT:   vcvtneps2bf16	-512(%rdx){1to16}, %ymm30 {%k7} {z}
+# INTEL: vcvtneps2bf16	ymm30 {k7} {z}, dword ptr [rdx - 512]{1to16}
+0x62,0x62,0x7e,0xdf,0x72,0x72,0x80
+
+# ATT:   vdpbf16ps	%zmm28, %zmm29, %zmm30
+# INTEL: vdpbf16ps	zmm30, zmm29, zmm28
+0x62,0x02,0x16,0x40,0x52,0xf4
+
+# ATT:   vdpbf16ps	268435456(%rbp,%r14,8), %zmm29, %zmm30 {%k7}
+# INTEL: vdpbf16ps	zmm30 {k7}, zmm29, zmmword ptr [rbp + 8*r14 + 268435456]
+0x62,0x22,0x16,0x47,0x52,0xb4,0xf5,0x00,0x00,0x00,0x10
+
+# ATT:   vdpbf16ps	(%r9){1to16}, %zmm29, %zmm30
+# INTEL: vdpbf16ps	zmm30, zmm29, dword ptr [r9]{1to16}
+0x62,0x42,0x16,0x50,0x52,0x31
+
+# ATT:   vdpbf16ps	8128(%rcx), %zmm29, %zmm30
+# INTEL: vdpbf16ps	zmm30, zmm29, zmmword ptr [rcx + 8128]
+0x62,0x62,0x16,0x40,0x52,0x71,0x7f
+
+# ATT:   vdpbf16ps	-512(%rdx){1to16}, %zmm29, %zmm30 {%k7} {z}
+# INTEL: vdpbf16ps	zmm30 {k7} {z}, zmm29, dword ptr [rdx - 512]{1to16}
+0x62,0x62,0x16,0xd7,0x52,0x72,0x80
diff --git a/llvm/test/MC/Disassembler/X86/avx512_bf16_vl-32.txt b/llvm/test/MC/Disassembler/X86/avx512_bf16_vl-32.txt
new file mode 100644
index 0000000000000000000000000000000000000000..65ff8f73410f637332049cf028143975a9db2f3b
--- /dev/null
+++ b/llvm/test/MC/Disassembler/X86/avx512_bf16_vl-32.txt
@@ -0,0 +1,170 @@
+# RUN: llvm-mc -triple i686-unknown-unknown -disassemble %s | FileCheck %s --check-prefix=ATT
+# RUN: llvm-mc -triple i686-unknown-unknown -disassemble -output-asm-variant=1 %s | FileCheck %s --check-prefix=INTEL
+
+# ATT:   vcvtne2ps2bf16	%xmm4, %xmm5, %xmm6 {%k7}
+# INTEL: vcvtne2ps2bf16	xmm6 {k7}, xmm5, xmm4
+0x62,0xf2,0x57,0x0f,0x72,0xf4
+
+# ATT:   vcvtne2ps2bf16	%xmm4, %xmm5, %xmm6 {%k7} {z}
+# INTEL: vcvtne2ps2bf16	xmm6 {k7} {z}, xmm5, xmm4
+0x62,0xf2,0x57,0x8f,0x72,0xf4
+
+# ATT:   vcvtne2ps2bf16	(%ecx), %xmm5, %xmm6 {%k7}
+# INTEL: vcvtne2ps2bf16	xmm6 {k7}, xmm5, xmmword ptr [ecx]
+0x62,0xf2,0x57,0x0f,0x72,0x31
+
+# ATT:   vcvtne2ps2bf16	291(%esp,%esi,8), %xmm5, %xmm6 {%k7}
+# INTEL: vcvtne2ps2bf16	xmm6 {k7}, xmm5, xmmword ptr [esp + 8*esi + 291]
+0x62,0xf2,0x57,0x0f,0x72,0xb4,0xf4,0x23,0x01,0x00,0x00
+
+# ATT:   vcvtne2ps2bf16	268435456(%esp,%esi,8), %xmm5, %xmm6 {%k7}
+# INTEL: vcvtne2ps2bf16	xmm6 {k7}, xmm5, xmmword ptr [esp + 8*esi + 268435456]
+0x62,0xf2,0x57,0x0f,0x72,0xb4,0xf4,0x00,0x00,0x00,0x10
+
+# ATT:   vcvtne2ps2bf16	-16(%esp), %xmm5, %xmm6 {%k7}
+# INTEL: vcvtne2ps2bf16	xmm6 {k7}, xmm5, xmmword ptr [esp - 16]
+0x62,0xf2,0x57,0x0f,0x72,0x74,0x24,0xff
+
+# ATT:   vcvtne2ps2bf16	(%eax){1to4}, %xmm5, %xmm6 {%k7}
+# INTEL: vcvtne2ps2bf16	xmm6 {k7}, xmm5, dword ptr [eax]{1to4}
+0x62,0xf2,0x57,0x1f,0x72,0x30
+
+# ATT:   vcvtne2ps2bf16	2032(%edx), %xmm5, %xmm6 {%k7}
+# INTEL: vcvtne2ps2bf16	xmm6 {k7}, xmm5, xmmword ptr [edx + 2032]
+0x62,0xf2,0x57,0x0f,0x72,0x72,0x7f
+
+# ATT:   vcvtne2ps2bf16	-2048(%edx), %xmm5, %xmm6 {%k7}
+# INTEL: vcvtne2ps2bf16	xmm6 {k7}, xmm5, xmmword ptr [edx - 2048]
+0x62,0xf2,0x57,0x0f,0x72,0x72,0x80
+
+# ATT:   vcvtne2ps2bf16	508(%edx){1to4}, %xmm5, %xmm6 {%k7}
+# INTEL: vcvtne2ps2bf16	xmm6 {k7}, xmm5, dword ptr [edx + 508]{1to4}
+0x62,0xf2,0x57,0x1f,0x72,0x72,0x7f
+
+# ATT:   vcvtne2ps2bf16	-512(%edx){1to4}, %xmm5, %xmm6 {%k7}
+# INTEL: vcvtne2ps2bf16	xmm6 {k7}, xmm5, dword ptr [edx - 512]{1to4}
+0x62,0xf2,0x57,0x1f,0x72,0x72,0x80
+
+# ATT:   vcvtne2ps2bf16	%ymm4, %ymm5, %ymm6 {%k7}
+# INTEL: vcvtne2ps2bf16	ymm6 {k7}, ymm5, ymm4
+0x62,0xf2,0x57,0x2f,0x72,0xf4
+
+# ATT:   vcvtne2ps2bf16	%ymm4, %ymm5, %ymm6 {%k7} {z}
+# INTEL: vcvtne2ps2bf16	ymm6 {k7} {z}, ymm5, ymm4
+0x62,0xf2,0x57,0xaf,0x72,0xf4
+
+# ATT:   vcvtne2ps2bf16	(%ecx), %ymm5, %ymm6 {%k7}
+# INTEL: vcvtne2ps2bf16	ymm6 {k7}, ymm5, ymmword ptr [ecx]
+0x62,0xf2,0x57,0x2f,0x72,0x31
+
+# ATT:   vcvtne2ps2bf16	291(%esp,%esi,8), %ymm5, %ymm6 {%k7}
+# INTEL: vcvtne2ps2bf16	ymm6 {k7}, ymm5, ymmword ptr [esp + 8*esi + 291]
+0x62,0xf2,0x57,0x2f,0x72,0xb4,0xf4,0x23,0x01,0x00,0x00
+
+# ATT:   vcvtne2ps2bf16	268435456(%esp,%esi,8), %ymm5, %ymm6 {%k7}
+# INTEL: vcvtne2ps2bf16	ymm6 {k7}, ymm5, ymmword ptr [esp + 8*esi + 268435456]
+0x62,0xf2,0x57,0x2f,0x72,0xb4,0xf4,0x00,0x00,0x00,0x10
+
+# ATT:   vcvtne2ps2bf16	-32(%esp), %ymm5, %ymm6 {%k7}
+# INTEL: vcvtne2ps2bf16	ymm6 {k7}, ymm5, ymmword ptr [esp - 32]
+0x62,0xf2,0x57,0x2f,0x72,0x74,0x24,0xff
+
+# ATT:   vcvtne2ps2bf16	(%eax){1to8}, %ymm5, %ymm6 {%k7}
+# INTEL: vcvtne2ps2bf16	ymm6 {k7}, ymm5, dword ptr [eax]{1to8}
+0x62,0xf2,0x57,0x3f,0x72,0x30
+
+# ATT:   vcvtne2ps2bf16	4064(%edx), %ymm5, %ymm6 {%k7}
+# INTEL: vcvtne2ps2bf16	ymm6 {k7}, ymm5, ymmword ptr [edx + 4064]
+0x62,0xf2,0x57,0x2f,0x72,0x72,0x7f
+
+# ATT:   vcvtne2ps2bf16	-4096(%edx), %ymm5, %ymm6 {%k7}
+# INTEL: vcvtne2ps2bf16	ymm6 {k7}, ymm5, ymmword ptr [edx - 4096]
+0x62,0xf2,0x57,0x2f,0x72,0x72,0x80
+
+# ATT:   vcvtne2ps2bf16	508(%edx){1to8}, %ymm5, %ymm6 {%k7}
+# INTEL: vcvtne2ps2bf16	ymm6 {k7}, ymm5, dword ptr [edx + 508]{1to8}
+0x62,0xf2,0x57,0x3f,0x72,0x72,0x7f
+
+# ATT:   vcvtne2ps2bf16	-512(%edx){1to8}, %ymm5, %ymm6 {%k7}
+# INTEL: vcvtne2ps2bf16	ymm6 {k7}, ymm5, dword ptr [edx - 512]{1to8}
+0x62,0xf2,0x57,0x3f,0x72,0x72,0x80
+
+# ATT:   vcvtneps2bf16	%xmm5, %xmm6
+# INTEL: vcvtneps2bf16	xmm6, xmm5
+0x62,0xf2,0x7e,0x08,0x72,0xf5
+
+# ATT:   vcvtneps2bf16x	268435456(%esp,%esi,8), %xmm6 {%k7}
+# INTEL: vcvtneps2bf16	xmm6 {k7}, xmmword ptr [esp + 8*esi + 268435456]
+0x62,0xf2,0x7e,0x0f,0x72,0xb4,0xf4,0x00,0x00,0x00,0x10
+
+# ATT:   vcvtneps2bf16	(%ecx){1to4}, %xmm6
+# INTEL: vcvtneps2bf16	xmm6, dword ptr [ecx]{1to4}
+0x62,0xf2,0x7e,0x18,0x72,0x31
+
+# ATT:   vcvtneps2bf16x	2032(%ecx), %xmm6
+# INTEL: vcvtneps2bf16	xmm6, xmmword ptr [ecx + 2032]
+0x62,0xf2,0x7e,0x08,0x72,0x71,0x7f
+
+# ATT:   vcvtneps2bf16	-512(%edx){1to4}, %xmm6 {%k7} {z}
+# INTEL: vcvtneps2bf16	xmm6 {k7} {z}, dword ptr [edx - 512]{1to4}
+0x62,0xf2,0x7e,0x9f,0x72,0x72,0x80
+
+# ATT:   vcvtneps2bf16	%ymm5, %xmm6
+# INTEL: vcvtneps2bf16	xmm6, ymm5
+0x62,0xf2,0x7e,0x28,0x72,0xf5
+
+# ATT:   vcvtneps2bf16y	268435456(%esp,%esi,8), %xmm6 {%k7}
+# INTEL: vcvtneps2bf16	xmm6 {k7}, ymmword ptr [esp + 8*esi + 268435456]
+0x62,0xf2,0x7e,0x2f,0x72,0xb4,0xf4,0x00,0x00,0x00,0x10
+
+# ATT:   vcvtneps2bf16	(%ecx){1to8}, %xmm6
+# INTEL: vcvtneps2bf16	xmm6, dword ptr [ecx]{1to8}
+0x62,0xf2,0x7e,0x38,0x72,0x31
+
+# ATT:   vcvtneps2bf16y	4064(%ecx), %xmm6
+# INTEL: vcvtneps2bf16	xmm6, ymmword ptr [ecx + 4064]
+0x62,0xf2,0x7e,0x28,0x72,0x71,0x7f
+
+# ATT:   vcvtneps2bf16	-512(%edx){1to8}, %xmm6 {%k7} {z}
+# INTEL: vcvtneps2bf16	xmm6 {k7} {z}, dword ptr [edx - 512]{1to8}
+0x62,0xf2,0x7e,0xbf,0x72,0x72,0x80
+
+# ATT:   vdpbf16ps	%ymm4, %ymm5, %ymm6
+# INTEL: vdpbf16ps	ymm6, ymm5, ymm4
+0x62,0xf2,0x56,0x28,0x52,0xf4
+
+# ATT:   vdpbf16ps	268435456(%esp,%esi,8), %ymm5, %ymm6 {%k7}
+# INTEL: vdpbf16ps	ymm6 {k7}, ymm5, ymmword ptr [esp + 8*esi + 268435456]
+0x62,0xf2,0x56,0x2f,0x52,0xb4,0xf4,0x00,0x00,0x00,0x10
+
+# ATT:   vdpbf16ps	(%ecx){1to8}, %ymm5, %ymm6
+# INTEL: vdpbf16ps	ymm6, ymm5, dword ptr [ecx]{1to8}
+0x62,0xf2,0x56,0x38,0x52,0x31
+
+# ATT:   vdpbf16ps	4064(%ecx), %ymm5, %ymm6
+# INTEL: vdpbf16ps	ymm6, ymm5, ymmword ptr [ecx + 4064]
+0x62,0xf2,0x56,0x28,0x52,0x71,0x7f
+
+# ATT:   vdpbf16ps	-512(%edx){1to8}, %ymm5, %ymm6 {%k7} {z}
+# INTEL: vdpbf16ps	ymm6 {k7} {z}, ymm5, dword ptr [edx - 512]{1to8}
+0x62,0xf2,0x56,0xbf,0x52,0x72,0x80
+
+# ATT:   vdpbf16ps	%xmm4, %xmm5, %xmm6
+# INTEL: vdpbf16ps	xmm6, xmm5, xmm4
+0x62,0xf2,0x56,0x08,0x52,0xf4
+
+# ATT:   vdpbf16ps	268435456(%esp,%esi,8), %xmm5, %xmm6 {%k7}
+# INTEL: vdpbf16ps	xmm6 {k7}, xmm5, xmmword ptr [esp + 8*esi + 268435456]
+0x62,0xf2,0x56,0x0f,0x52,0xb4,0xf4,0x00,0x00,0x00,0x10
+
+# ATT:   vdpbf16ps	(%ecx){1to4}, %xmm5, %xmm6
+# INTEL: vdpbf16ps	xmm6, xmm5, dword ptr [ecx]{1to4}
+0x62,0xf2,0x56,0x18,0x52,0x31
+
+# ATT:   vdpbf16ps	2032(%ecx), %xmm5, %xmm6
+# INTEL: vdpbf16ps	xmm6, xmm5, xmmword ptr [ecx + 2032]
+0x62,0xf2,0x56,0x08,0x52,0x71,0x7f
+
+# ATT:   vdpbf16ps	-512(%edx){1to4}, %xmm5, %xmm6 {%k7} {z}
+# INTEL: vdpbf16ps	xmm6 {k7} {z}, xmm5, dword ptr [edx - 512]{1to4}
+0x62,0xf2,0x56,0x9f,0x52,0x72,0x80
diff --git a/llvm/test/MC/Disassembler/X86/avx512_bf16_vl-64.txt b/llvm/test/MC/Disassembler/X86/avx512_bf16_vl-64.txt
new file mode 100644
index 0000000000000000000000000000000000000000..d7fe10270cd3e9a634df53be9083fc99ab7f91f3
--- /dev/null
+++ b/llvm/test/MC/Disassembler/X86/avx512_bf16_vl-64.txt
@@ -0,0 +1,178 @@
+# RUN: llvm-mc -triple x86_64 -disassemble %s | FileCheck %s --check-prefix=ATT
+# RUN: llvm-mc -triple x86_64 -disassemble -output-asm-variant=1 %s | FileCheck %s --check-prefix=INTEL
+
+# ATT:   vcvtne2ps2bf16	%xmm28, %xmm29, %xmm30
+# INTEL: vcvtne2ps2bf16	xmm30, xmm29, xmm28
+0x62,0x02,0x17,0x00,0x72,0xf4
+
+# ATT:   vcvtne2ps2bf16	%xmm28, %xmm29, %xmm30 {%k7}
+# INTEL: vcvtne2ps2bf16	xmm30 {k7}, xmm29, xmm28
+0x62,0x02,0x17,0x07,0x72,0xf4
+
+# ATT:   vcvtne2ps2bf16	%xmm28, %xmm29, %xmm30 {%k7} {z}
+# INTEL: vcvtne2ps2bf16	xmm30 {k7} {z}, xmm29, xmm28
+0x62,0x02,0x17,0x87,0x72,0xf4
+
+# ATT:   vcvtne2ps2bf16	(%rcx), %xmm29, %xmm30
+# INTEL: vcvtne2ps2bf16	xmm30, xmm29, xmmword ptr [rcx]
+0x62,0x62,0x17,0x00,0x72,0x31
+
+# ATT:   vcvtne2ps2bf16	291(%rax,%r14,8), %xmm29, %xmm30
+# INTEL: vcvtne2ps2bf16	xmm30, xmm29, xmmword ptr [rax + 8*r14 + 291]
+0x62,0x22,0x17,0x00,0x72,0xb4,0xf0,0x23,0x01,0x00,0x00
+
+# ATT:   vcvtne2ps2bf16	268435456(%rax,%r14,8), %xmm29, %xmm30
+# INTEL: vcvtne2ps2bf16	xmm30, xmm29, xmmword ptr [rax + 8*r14 + 268435456]
+0x62,0x22,0x17,0x00,0x72,0xb4,0xf0,0x00,0x00,0x00,0x10
+
+# ATT:   vcvtne2ps2bf16	-16(%rsp), %xmm29, %xmm30
+# INTEL: vcvtne2ps2bf16	xmm30, xmm29, xmmword ptr [rsp - 16]
+0x62,0x62,0x17,0x00,0x72,0x74,0x24,0xff
+
+# ATT:   vcvtne2ps2bf16	(%rcx){1to4}, %xmm29, %xmm30
+# INTEL: vcvtne2ps2bf16	xmm30, xmm29, dword ptr [rcx]{1to4}
+0x62,0x62,0x17,0x10,0x72,0x31
+
+# ATT:   vcvtne2ps2bf16	2032(%rdx), %xmm29, %xmm30
+# INTEL: vcvtne2ps2bf16	xmm30, xmm29, xmmword ptr [rdx + 2032]
+0x62,0x62,0x17,0x00,0x72,0x72,0x7f
+
+# ATT:   vcvtne2ps2bf16	-2048(%rdx), %xmm29, %xmm30
+# INTEL: vcvtne2ps2bf16	xmm30, xmm29, xmmword ptr [rdx - 2048]
+0x62,0x62,0x17,0x00,0x72,0x72,0x80
+
+# ATT:   vcvtne2ps2bf16	508(%rdx){1to4}, %xmm29, %xmm30
+# INTEL: vcvtne2ps2bf16	xmm30, xmm29, dword ptr [rdx + 508]{1to4}
+0x62,0x62,0x17,0x10,0x72,0x72,0x7f
+
+# ATT:   vcvtne2ps2bf16	-512(%rdx){1to4}, %xmm29, %xmm30
+# INTEL: vcvtne2ps2bf16	xmm30, xmm29, dword ptr [rdx - 512]{1to4}
+0x62,0x62,0x17,0x10,0x72,0x72,0x80
+
+# ATT:   vcvtne2ps2bf16	%ymm28, %ymm29, %ymm30
+# INTEL: vcvtne2ps2bf16	ymm30, ymm29, ymm28
+0x62,0x02,0x17,0x20,0x72,0xf4
+
+# ATT:   vcvtne2ps2bf16	%ymm28, %ymm29, %ymm30 {%k7}
+# INTEL: vcvtne2ps2bf16	ymm30 {k7}, ymm29, ymm28
+0x62,0x02,0x17,0x27,0x72,0xf4
+
+# ATT:   vcvtne2ps2bf16	%ymm28, %ymm29, %ymm30 {%k7} {z}
+# INTEL: vcvtne2ps2bf16	ymm30 {k7} {z}, ymm29, ymm28
+0x62,0x02,0x17,0xa7,0x72,0xf4
+
+# ATT:   vcvtne2ps2bf16	(%rcx), %ymm29, %ymm30
+# INTEL: vcvtne2ps2bf16	ymm30, ymm29, ymmword ptr [rcx]
+0x62,0x62,0x17,0x20,0x72,0x31
+
+# ATT:   vcvtne2ps2bf16	291(%rax,%r14,8), %ymm29, %ymm30
+# INTEL: vcvtne2ps2bf16	ymm30, ymm29, ymmword ptr [rax + 8*r14 + 291]
+0x62,0x22,0x17,0x20,0x72,0xb4,0xf0,0x23,0x01,0x00,0x00
+
+# ATT:   vcvtne2ps2bf16	268435456(%rax,%r14,8), %ymm29, %ymm30
+# INTEL: vcvtne2ps2bf16	ymm30, ymm29, ymmword ptr [rax + 8*r14 + 268435456]
+0x62,0x22,0x17,0x20,0x72,0xb4,0xf0,0x00,0x00,0x00,0x10
+
+# ATT:   vcvtne2ps2bf16	-32(%rsp), %ymm29, %ymm30
+# INTEL: vcvtne2ps2bf16	ymm30, ymm29, ymmword ptr [rsp - 32]
+0x62,0x62,0x17,0x20,0x72,0x74,0x24,0xff
+
+# ATT:   vcvtne2ps2bf16	(%rcx){1to8}, %ymm29, %ymm30
+# INTEL: vcvtne2ps2bf16	ymm30, ymm29, dword ptr [rcx]{1to8}
+0x62,0x62,0x17,0x30,0x72,0x31
+
+# ATT:   vcvtne2ps2bf16	4064(%rdx), %ymm29, %ymm30
+# INTEL: vcvtne2ps2bf16	ymm30, ymm29, ymmword ptr [rdx + 4064]
+0x62,0x62,0x17,0x20,0x72,0x72,0x7f
+
+# ATT:   vcvtne2ps2bf16	-4096(%rdx), %ymm29, %ymm30
+# INTEL: vcvtne2ps2bf16	ymm30, ymm29, ymmword ptr [rdx - 4096]
+0x62,0x62,0x17,0x20,0x72,0x72,0x80
+
+# ATT:   vcvtne2ps2bf16	508(%rdx){1to8}, %ymm29, %ymm30
+# INTEL: vcvtne2ps2bf16	ymm30, ymm29, dword ptr [rdx + 508]{1to8}
+0x62,0x62,0x17,0x30,0x72,0x72,0x7f
+
+# ATT:   vcvtne2ps2bf16	-512(%rdx){1to8}, %ymm29, %ymm30
+# INTEL: vcvtne2ps2bf16	ymm30, ymm29, dword ptr [rdx - 512]{1to8}
+0x62,0x62,0x17,0x30,0x72,0x72,0x80
+
+# ATT:   vcvtneps2bf16	%xmm29, %xmm30
+# INTEL: vcvtneps2bf16	xmm30, xmm29
+0x62,0x02,0x7e,0x08,0x72,0xf5
+
+# ATT:   vcvtneps2bf16x	268435456(%rbp,%r14,8), %xmm30 {%k7}
+# INTEL: vcvtneps2bf16	xmm30 {k7}, xmmword ptr [rbp + 8*r14 + 268435456]
+0x62,0x22,0x7e,0x0f,0x72,0xb4,0xf5,0x00,0x00,0x00,0x10
+
+# ATT:   vcvtneps2bf16	(%r9){1to4}, %xmm30
+# INTEL: vcvtneps2bf16	xmm30, dword ptr [r9]{1to4}
+0x62,0x42,0x7e,0x18,0x72,0x31
+
+# ATT:   vcvtneps2bf16x	2032(%rcx), %xmm30
+# INTEL: vcvtneps2bf16	xmm30, xmmword ptr [rcx + 2032]
+0x62,0x62,0x7e,0x08,0x72,0x71,0x7f
+
+# ATT:   vcvtneps2bf16	-512(%rdx){1to4}, %xmm30 {%k7} {z}
+# INTEL: vcvtneps2bf16	xmm30 {k7} {z}, dword ptr [rdx - 512]{1to4}
+0x62,0x62,0x7e,0x9f,0x72,0x72,0x80
+
+# ATT:   vcvtneps2bf16	%ymm29, %xmm30
+# INTEL: vcvtneps2bf16	xmm30, ymm29
+0x62,0x02,0x7e,0x28,0x72,0xf5
+
+# ATT:   vcvtneps2bf16y	268435456(%rbp,%r14,8), %xmm30 {%k7}
+# INTEL: vcvtneps2bf16	xmm30 {k7}, ymmword ptr [rbp + 8*r14 + 268435456]
+0x62,0x22,0x7e,0x2f,0x72,0xb4,0xf5,0x00,0x00,0x00,0x10
+
+# ATT:   vcvtneps2bf16	(%r9){1to8}, %xmm30
+# INTEL: vcvtneps2bf16	xmm30, dword ptr [r9]{1to8}
+0x62,0x42,0x7e,0x38,0x72,0x31
+
+# ATT:   vcvtneps2bf16y	4064(%rcx), %xmm30
+# INTEL: vcvtneps2bf16	xmm30, ymmword ptr [rcx + 4064]
+0x62,0x62,0x7e,0x28,0x72,0x71,0x7f
+
+# ATT:   vcvtneps2bf16	-512(%rdx){1to8}, %xmm30 {%k7} {z}
+# INTEL: vcvtneps2bf16	xmm30 {k7} {z}, dword ptr [rdx - 512]{1to8}
+0x62,0x62,0x7e,0xbf,0x72,0x72,0x80
+
+# ATT:   vdpbf16ps	%ymm28, %ymm29, %ymm30
+# INTEL: vdpbf16ps	ymm30, ymm29, ymm28
+0x62,0x02,0x16,0x20,0x52,0xf4
+
+# ATT:   vdpbf16ps	268435456(%rbp,%r14,8), %ymm29, %ymm30 {%k7}
+# INTEL: vdpbf16ps	ymm30 {k7}, ymm29, ymmword ptr [rbp + 8*r14 + 268435456]
+0x62,0x22,0x16,0x27,0x52,0xb4,0xf5,0x00,0x00,0x00,0x10
+
+# ATT:   vdpbf16ps	(%r9){1to8}, %ymm29, %ymm30
+# INTEL: vdpbf16ps	ymm30, ymm29, dword ptr [r9]{1to8}
+0x62,0x42,0x16,0x30,0x52,0x31
+
+# ATT:   vdpbf16ps	4064(%rcx), %ymm29, %ymm30
+# INTEL: vdpbf16ps	ymm30, ymm29, ymmword ptr [rcx + 4064]
+0x62,0x62,0x16,0x20,0x52,0x71,0x7f
+
+# ATT:   vdpbf16ps	-512(%rdx){1to8}, %ymm29, %ymm30 {%k7} {z}
+# INTEL: vdpbf16ps	ymm30 {k7} {z}, ymm29, dword ptr [rdx - 512]{1to8}
+0x62,0x62,0x16,0xb7,0x52,0x72,0x80
+
+# ATT:   vdpbf16ps	%xmm28, %xmm29, %xmm30
+# INTEL: vdpbf16ps	xmm30, xmm29, xmm28
+0x62,0x02,0x16,0x00,0x52,0xf4
+
+# ATT:   vdpbf16ps	268435456(%rbp,%r14,8), %xmm29, %xmm30 {%k7}
+# INTEL: vdpbf16ps	xmm30 {k7}, xmm29, xmmword ptr [rbp + 8*r14 + 268435456]
+0x62,0x22,0x16,0x07,0x52,0xb4,0xf5,0x00,0x00,0x00,0x10
+
+# ATT:   vdpbf16ps	(%r9){1to4}, %xmm29, %xmm30
+# INTEL: vdpbf16ps	xmm30, xmm29, dword ptr [r9]{1to4}
+0x62,0x42,0x16,0x10,0x52,0x31
+
+# ATT:   vdpbf16ps	2032(%rcx), %xmm29, %xmm30
+# INTEL: vdpbf16ps	xmm30, xmm29, xmmword ptr [rcx + 2032]
+0x62,0x62,0x16,0x00,0x52,0x71,0x7f
+
+# ATT:   vdpbf16ps	-512(%rdx){1to4}, %xmm29, %xmm30 {%k7} {z}
+# INTEL: vdpbf16ps	xmm30 {k7} {z}, xmm29, dword ptr [rdx - 512]{1to4}
+0x62,0x62,0x16,0x97,0x52,0x72,0x80
diff --git a/llvm/test/MC/Disassembler/X86/avx512bf16-att.txt b/llvm/test/MC/Disassembler/X86/avx512bf16-att.txt
deleted file mode 100644
index a6dc4b684c5153ee7b962aad3e97622aab47f0f4..0000000000000000000000000000000000000000
--- a/llvm/test/MC/Disassembler/X86/avx512bf16-att.txt
+++ /dev/null
@@ -1,82 +0,0 @@
-# RUN: llvm-mc --disassemble %s -triple=i686-apple-darwin9 | FileCheck %s
-
-# CHECK: vcvtne2ps2bf16 %zmm4, %zmm3, %zmm2
-0x62,0xf2,0x67,0x48,0x72,0xd4
-
-# CHECK: vcvtne2ps2bf16 %zmm4, %zmm3, %zmm2 {%k7}
-0x62,0xf2,0x67,0x4f,0x72,0xd4
-
-# CHECK: vcvtne2ps2bf16 %zmm4, %zmm3, %zmm2 {%k7} {z}
-0x62,0xf2,0x67,0xcf,0x72,0xd4
-
-# CHECK: vcvtne2ps2bf16  268435456(%esp,%esi,8), %zmm3, %zmm2
-0x62,0xf2,0x67,0x48,0x72,0x94,0xf4,0x00,0x00,0x00,0x10
-
-# CHECK: vcvtne2ps2bf16  291(%edi,%eax,4), %zmm3, %zmm2 {%k7}
-0x62,0xf2,0x67,0x4f,0x72,0x94,0x87,0x23,0x01,0x00,0x00
-
-# CHECK: vcvtne2ps2bf16  (%eax){1to16}, %zmm3, %zmm2
-0x62,0xf2,0x67,0x58,0x72,0x10
-
-# CHECK: vcvtne2ps2bf16  -2048(,%ebp,2), %zmm3, %zmm2
-0x62,0xf2,0x67,0x48,0x72,0x14,0x6d,0x00,0xf8,0xff,0xff
-
-# CHECK: vcvtne2ps2bf16  8128(%ecx), %zmm3, %zmm2 {%k7} {z}
-0x62,0xf2,0x67,0xcf,0x72,0x51,0x7f
-
-# CHECK: vcvtne2ps2bf16  -512(%edx){1to16}, %zmm3, %zmm2 {%k7} {z}
-0x62,0xf2,0x67,0xdf,0x72,0x52,0x80
-
-# CHECK: vcvtneps2bf16 %zmm3, %ymm2
-0x62,0xf2,0x7e,0x48,0x72,0xd3
-
-# CHECK: vcvtneps2bf16 %zmm3, %ymm2 {%k7}
-0x62,0xf2,0x7e,0x4f,0x72,0xd3
-
-# CHECK: vcvtneps2bf16 %zmm3, %ymm2 {%k7} {z}
-0x62,0xf2,0x7e,0xcf,0x72,0xd3
-
-# CHECK: vcvtneps2bf16  268435456(%esp,%esi,8), %ymm2
-0x62,0xf2,0x7e,0x48,0x72,0x94,0xf4,0x00,0x00,0x00,0x10
-
-# CHECK: vcvtneps2bf16  291(%edi,%eax,4), %ymm2 {%k7}
-0x62,0xf2,0x7e,0x4f,0x72,0x94,0x87,0x23,0x01,0x00,0x00
-
-# CHECK: vcvtneps2bf16  (%eax){1to16}, %ymm2
-0x62,0xf2,0x7e,0x58,0x72,0x10
-
-# CHECK: vcvtneps2bf16  -2048(,%ebp,2), %ymm2
-0x62,0xf2,0x7e,0x48,0x72,0x14,0x6d,0x00,0xf8,0xff,0xff
-
-# CHECK: vcvtneps2bf16  8128(%ecx), %ymm2 {%k7} {z}
-0x62,0xf2,0x7e,0xcf,0x72,0x51,0x7f
-
-# CHECK: vcvtneps2bf16  -512(%edx){1to16}, %ymm2 {%k7} {z}
-0x62,0xf2,0x7e,0xdf,0x72,0x52,0x80
-
-# CHECK: vdpbf16ps %zmm4, %zmm3, %zmm2
-0x62,0xf2,0x66,0x48,0x52,0xd4
-
-# CHECK: vdpbf16ps %zmm4, %zmm3, %zmm2 {%k7}
-0x62,0xf2,0x66,0x4f,0x52,0xd4
-
-# CHECK: vdpbf16ps %zmm4, %zmm3, %zmm2 {%k7} {z}
-0x62,0xf2,0x66,0xcf,0x52,0xd4
-
-# CHECK: vdpbf16ps  268435456(%esp,%esi,8), %zmm3, %zmm2
-0x62,0xf2,0x66,0x48,0x52,0x94,0xf4,0x00,0x00,0x00,0x10
-
-# CHECK: vdpbf16ps  291(%edi,%eax,4), %zmm3, %zmm2 {%k7}
-0x62,0xf2,0x66,0x4f,0x52,0x94,0x87,0x23,0x01,0x00,0x00
-
-# CHECK: vdpbf16ps  (%eax){1to16}, %zmm3, %zmm2
-0x62,0xf2,0x66,0x58,0x52,0x10
-
-# CHECK: vdpbf16ps  -2048(,%ebp,2), %zmm3, %zmm2
-0x62,0xf2,0x66,0x48,0x52,0x14,0x6d,0x00,0xf8,0xff,0xff
-
-# CHECK: vdpbf16ps  8128(%ecx), %zmm3, %zmm2 {%k7} {z}
-0x62,0xf2,0x66,0xcf,0x52,0x51,0x7f
-
-# CHECK: vdpbf16ps  -512(%edx){1to16}, %zmm3, %zmm2 {%k7} {z}
-0x62,0xf2,0x66,0xdf,0x52,0x52,0x80
diff --git a/llvm/test/MC/Disassembler/X86/avx512bf16-intel.txt b/llvm/test/MC/Disassembler/X86/avx512bf16-intel.txt
deleted file mode 100644
index 53d44fa7fb7a2fb326444f327535314fc3b34b66..0000000000000000000000000000000000000000
--- a/llvm/test/MC/Disassembler/X86/avx512bf16-intel.txt
+++ /dev/null
@@ -1,82 +0,0 @@
-# RUN: llvm-mc --disassemble %s -triple=i686 -x86-asm-syntax=intel --output-asm-variant=1 | FileCheck %s
-
-# CHECK: vcvtne2ps2bf16 zmm2, zmm3, zmm4
-0x62,0xf2,0x67,0x48,0x72,0xd4
-
-# CHECK: vcvtne2ps2bf16 zmm2 {k7}, zmm3, zmm4
-0x62,0xf2,0x67,0x4f,0x72,0xd4
-
-# CHECK: vcvtne2ps2bf16 zmm2 {k7} {z}, zmm3, zmm4
-0x62,0xf2,0x67,0xcf,0x72,0xd4
-
-# CHECK: vcvtne2ps2bf16 zmm2, zmm3, zmmword ptr [esp + 8*esi + 268435456]
-0x62,0xf2,0x67,0x48,0x72,0x94,0xf4,0x00,0x00,0x00,0x10
-
-# CHECK: vcvtne2ps2bf16 zmm2 {k7}, zmm3, zmmword ptr [edi + 4*eax + 291]
-0x62,0xf2,0x67,0x4f,0x72,0x94,0x87,0x23,0x01,0x00,0x00
-
-# CHECK: vcvtne2ps2bf16 zmm2, zmm3, dword ptr [eax]{1to16}
-0x62,0xf2,0x67,0x58,0x72,0x10
-
-# CHECK: vcvtne2ps2bf16 zmm2, zmm3, zmmword ptr [2*ebp - 2048]
-0x62,0xf2,0x67,0x48,0x72,0x14,0x6d,0x00,0xf8,0xff,0xff
-
-# CHECK: vcvtne2ps2bf16 zmm2 {k7} {z}, zmm3, zmmword ptr [ecx + 8128]
-0x62,0xf2,0x67,0xcf,0x72,0x51,0x7f
-
-# CHECK: vcvtne2ps2bf16 zmm2 {k7} {z}, zmm3, dword ptr [edx - 512]{1to16}
-0x62,0xf2,0x67,0xdf,0x72,0x52,0x80
-
-# CHECK: vcvtneps2bf16 ymm2, zmm3
-0x62,0xf2,0x7e,0x48,0x72,0xd3
-
-# CHECK: vcvtneps2bf16 ymm2 {k7}, zmm3
-0x62,0xf2,0x7e,0x4f,0x72,0xd3
-
-# CHECK: vcvtneps2bf16 ymm2 {k7} {z}, zmm3
-0x62,0xf2,0x7e,0xcf,0x72,0xd3
-
-# CHECK: vcvtneps2bf16 ymm2, zmmword ptr [esp + 8*esi + 268435456]
-0x62,0xf2,0x7e,0x48,0x72,0x94,0xf4,0x00,0x00,0x00,0x10
-
-# CHECK: vcvtneps2bf16 ymm2 {k7}, zmmword ptr [edi + 4*eax + 291]
-0x62,0xf2,0x7e,0x4f,0x72,0x94,0x87,0x23,0x01,0x00,0x00
-
-# CHECK: vcvtneps2bf16 ymm2, dword ptr [eax]{1to16}
-0x62,0xf2,0x7e,0x58,0x72,0x10
-
-# CHECK: vcvtneps2bf16 ymm2, zmmword ptr [2*ebp - 2048]
-0x62,0xf2,0x7e,0x48,0x72,0x14,0x6d,0x00,0xf8,0xff,0xff
-
-# CHECK: vcvtneps2bf16 ymm2 {k7} {z}, zmmword ptr [ecx + 8128]
-0x62,0xf2,0x7e,0xcf,0x72,0x51,0x7f
-
-# CHECK: vcvtneps2bf16 ymm2 {k7} {z}, dword ptr [edx - 512]{1to16}
-0x62,0xf2,0x7e,0xdf,0x72,0x52,0x80
-
-# CHECK: vdpbf16ps zmm2, zmm3, zmm4
-0x62,0xf2,0x66,0x48,0x52,0xd4
-
-# CHECK: vdpbf16ps zmm2 {k7}, zmm3, zmm4
-0x62,0xf2,0x66,0x4f,0x52,0xd4
-
-# CHECK: vdpbf16ps zmm2 {k7} {z}, zmm3, zmm4
-0x62,0xf2,0x66,0xcf,0x52,0xd4
-
-# CHECK: vdpbf16ps zmm2, zmm3, zmmword ptr [esp + 8*esi + 268435456]
-0x62,0xf2,0x66,0x48,0x52,0x94,0xf4,0x00,0x00,0x00,0x10
-
-# CHECK: vdpbf16ps zmm2 {k7}, zmm3, zmmword ptr [edi + 4*eax + 291]
-0x62,0xf2,0x66,0x4f,0x52,0x94,0x87,0x23,0x01,0x00,0x00
-
-# CHECK: vdpbf16ps zmm2, zmm3, dword ptr [eax]{1to16}
-0x62,0xf2,0x66,0x58,0x52,0x10
-
-# CHECK: vdpbf16ps zmm2, zmm3, zmmword ptr [2*ebp - 2048]
-0x62,0xf2,0x66,0x48,0x52,0x14,0x6d,0x00,0xf8,0xff,0xff
-
-# CHECK: vdpbf16ps zmm2 {k7} {z}, zmm3, zmmword ptr [ecx + 8128]
-0x62,0xf2,0x66,0xcf,0x52,0x51,0x7f
-
-# CHECK: vdpbf16ps zmm2 {k7} {z}, zmm3, dword ptr [edx - 512]{1to16}
-0x62,0xf2,0x66,0xdf,0x52,0x52,0x80
diff --git a/llvm/test/MC/Disassembler/X86/avx512bf16vl-att.txt b/llvm/test/MC/Disassembler/X86/avx512bf16vl-att.txt
deleted file mode 100644
index ea5e84ffab46c35337fd7fd46e5951d3c5e5a577..0000000000000000000000000000000000000000
--- a/llvm/test/MC/Disassembler/X86/avx512bf16vl-att.txt
+++ /dev/null
@@ -1,157 +0,0 @@
-# RUN: llvm-mc --disassemble %s -triple=i686-apple-darwin9 | FileCheck %s
-
-# CHECK: vcvtne2ps2bf16 %ymm4, %ymm3, %ymm2
-0x62,0xf2,0x67,0x28,0x72,0xd4
-
-# CHECK: vcvtne2ps2bf16 %ymm4, %ymm3, %ymm2 {%k7}
-0x62,0xf2,0x67,0x2f,0x72,0xd4
-
-# CHECK: vcvtne2ps2bf16 %ymm4, %ymm3, %ymm2 {%k7} {z}
-0x62,0xf2,0x67,0xaf,0x72,0xd4
-
-# CHECK: vcvtne2ps2bf16 %xmm4, %xmm3, %xmm2
-0x62,0xf2,0x67,0x08,0x72,0xd4
-
-# CHECK: vcvtne2ps2bf16 %xmm4, %xmm3, %xmm2 {%k7}
-0x62,0xf2,0x67,0x0f,0x72,0xd4
-
-# CHECK: vcvtne2ps2bf16 %xmm4, %xmm3, %xmm2 {%k7} {z}
-0x62,0xf2,0x67,0x8f,0x72,0xd4
-
-# CHECK: vcvtne2ps2bf16  268435456(%esp,%esi,8), %ymm3, %ymm2
-0x62,0xf2,0x67,0x28,0x72,0x94,0xf4,0x00,0x00,0x00,0x10
-
-# CHECK: vcvtne2ps2bf16  291(%edi,%eax,4), %ymm3, %ymm2 {%k7}
-0x62,0xf2,0x67,0x2f,0x72,0x94,0x87,0x23,0x01,0x00,0x00
-
-# CHECK: vcvtne2ps2bf16  (%eax){1to8}, %ymm3, %ymm2
-0x62,0xf2,0x67,0x38,0x72,0x10
-
-# CHECK: vcvtne2ps2bf16  -1024(,%ebp,2), %ymm3, %ymm2
-0x62,0xf2,0x67,0x28,0x72,0x14,0x6d,0x00,0xfc,0xff,0xff
-
-# CHECK: vcvtne2ps2bf16  4064(%ecx), %ymm3, %ymm2 {%k7} {z}
-0x62,0xf2,0x67,0xaf,0x72,0x51,0x7f
-
-# CHECK: vcvtne2ps2bf16  -512(%edx){1to8}, %ymm3, %ymm2 {%k7} {z}
-0x62,0xf2,0x67,0xbf,0x72,0x52,0x80
-
-# CHECK: vcvtne2ps2bf16  268435456(%esp,%esi,8), %xmm3, %xmm2
-0x62,0xf2,0x67,0x08,0x72,0x94,0xf4,0x00,0x00,0x00,0x10
-
-# CHECK: vcvtne2ps2bf16  291(%edi,%eax,4), %xmm3, %xmm2 {%k7}
-0x62,0xf2,0x67,0x0f,0x72,0x94,0x87,0x23,0x01,0x00,0x00
-
-# CHECK: vcvtne2ps2bf16  (%eax){1to4}, %xmm3, %xmm2
-0x62,0xf2,0x67,0x18,0x72,0x10
-
-# CHECK: vcvtne2ps2bf16  -512(,%ebp,2), %xmm3, %xmm2
-0x62,0xf2,0x67,0x08,0x72,0x14,0x6d,0x00,0xfe,0xff,0xff
-
-# CHECK: vcvtne2ps2bf16  2032(%ecx), %xmm3, %xmm2 {%k7} {z}
-0x62,0xf2,0x67,0x8f,0x72,0x51,0x7f
-
-# CHECK: vcvtne2ps2bf16  -512(%edx){1to4}, %xmm3, %xmm2 {%k7} {z}
-0x62,0xf2,0x67,0x9f,0x72,0x52,0x80
-
-# CHECK: vcvtneps2bf16 %xmm3, %xmm2
-0x62,0xf2,0x7e,0x08,0x72,0xd3
-
-# CHECK: vcvtneps2bf16 %xmm3, %xmm2 {%k7}
-0x62,0xf2,0x7e,0x0f,0x72,0xd3
-
-# CHECK: vcvtneps2bf16 %xmm3, %xmm2 {%k7} {z}
-0x62,0xf2,0x7e,0x8f,0x72,0xd3
-
-# CHECK: vcvtneps2bf16 %ymm3, %xmm2
-0x62,0xf2,0x7e,0x28,0x72,0xd3
-
-# CHECK: vcvtneps2bf16 %ymm3, %xmm2 {%k7}
-0x62,0xf2,0x7e,0x2f,0x72,0xd3
-
-# CHECK: vcvtneps2bf16 %ymm3, %xmm2 {%k7} {z}
-0x62,0xf2,0x7e,0xaf,0x72,0xd3
-
-# CHECK: vcvtneps2bf16x  268435456(%esp,%esi,8), %xmm2
-0x62,0xf2,0x7e,0x08,0x72,0x94,0xf4,0x00,0x00,0x00,0x10
-
-# CHECK: vcvtneps2bf16x  291(%edi,%eax,4), %xmm2 {%k7}
-0x62,0xf2,0x7e,0x0f,0x72,0x94,0x87,0x23,0x01,0x00,0x00
-
-# CHECK: vcvtneps2bf16  (%eax){1to4}, %xmm2
-0x62,0xf2,0x7e,0x18,0x72,0x10
-
-# CHECK: vcvtneps2bf16x  -512(,%ebp,2), %xmm2
-0x62,0xf2,0x7e,0x08,0x72,0x14,0x6d,0x00,0xfe,0xff,0xff
-
-# CHECK: vcvtneps2bf16x  2032(%ecx), %xmm2 {%k7} {z}
-0x62,0xf2,0x7e,0x8f,0x72,0x51,0x7f
-
-# CHECK: vcvtneps2bf16  -512(%edx){1to4}, %xmm2 {%k7} {z}
-0x62,0xf2,0x7e,0x9f,0x72,0x52,0x80
-
-# CHECK: vcvtneps2bf16  (%eax){1to8}, %xmm2
-0x62,0xf2,0x7e,0x38,0x72,0x10
-
-# CHECK: vcvtneps2bf16y  -1024(,%ebp,2), %xmm2
-0x62,0xf2,0x7e,0x28,0x72,0x14,0x6d,0x00,0xfc,0xff,0xff
-
-# CHECK: vcvtneps2bf16y  4064(%ecx), %xmm2 {%k7} {z}
-0x62,0xf2,0x7e,0xaf,0x72,0x51,0x7f
-
-# CHECK: vcvtneps2bf16  -512(%edx){1to8}, %xmm2 {%k7} {z}
-0x62,0xf2,0x7e,0xbf,0x72,0x52,0x80
-
-# CHECK: vdpbf16ps %ymm4, %ymm3, %ymm2
-0x62,0xf2,0x66,0x28,0x52,0xd4
-
-# CHECK: vdpbf16ps %ymm4, %ymm3, %ymm2 {%k7}
-0x62,0xf2,0x66,0x2f,0x52,0xd4
-
-# CHECK: vdpbf16ps %ymm4, %ymm3, %ymm2 {%k7} {z}
-0x62,0xf2,0x66,0xaf,0x52,0xd4
-
-# CHECK: vdpbf16ps %xmm4, %xmm3, %xmm2
-0x62,0xf2,0x66,0x08,0x52,0xd4
-
-# CHECK: vdpbf16ps %xmm4, %xmm3, %xmm2 {%k7}
-0x62,0xf2,0x66,0x0f,0x52,0xd4
-
-# CHECK: vdpbf16ps %xmm4, %xmm3, %xmm2 {%k7} {z}
-0x62,0xf2,0x66,0x8f,0x52,0xd4
-
-# CHECK: vdpbf16ps  268435456(%esp,%esi,8), %ymm3, %ymm2
-0x62,0xf2,0x66,0x28,0x52,0x94,0xf4,0x00,0x00,0x00,0x10
-
-# CHECK: vdpbf16ps  291(%edi,%eax,4), %ymm3, %ymm2 {%k7}
-0x62,0xf2,0x66,0x2f,0x52,0x94,0x87,0x23,0x01,0x00,0x00
-
-# CHECK: vdpbf16ps  (%eax){1to8}, %ymm3, %ymm2
-0x62,0xf2,0x66,0x38,0x52,0x10
-
-# CHECK: vdpbf16ps  -1024(,%ebp,2), %ymm3, %ymm2
-0x62,0xf2,0x66,0x28,0x52,0x14,0x6d,0x00,0xfc,0xff,0xff
-
-# CHECK: vdpbf16ps  4064(%ecx), %ymm3, %ymm2 {%k7} {z}
-0x62,0xf2,0x66,0xaf,0x52,0x51,0x7f
-
-# CHECK: vdpbf16ps  -512(%edx){1to8}, %ymm3, %ymm2 {%k7} {z}
-0x62,0xf2,0x66,0xbf,0x52,0x52,0x80
-
-# CHECK: vdpbf16ps  268435456(%esp,%esi,8), %xmm3, %xmm2
-0x62,0xf2,0x66,0x08,0x52,0x94,0xf4,0x00,0x00,0x00,0x10
-
-# CHECK: vdpbf16ps  291(%edi,%eax,4), %xmm3, %xmm2 {%k7}
-0x62,0xf2,0x66,0x0f,0x52,0x94,0x87,0x23,0x01,0x00,0x00
-
-# CHECK: vdpbf16ps  (%eax){1to4}, %xmm3, %xmm2
-0x62,0xf2,0x66,0x18,0x52,0x10
-
-# CHECK: vdpbf16ps  -512(,%ebp,2), %xmm3, %xmm2
-0x62,0xf2,0x66,0x08,0x52,0x14,0x6d,0x00,0xfe,0xff,0xff
-
-# CHECK: vdpbf16ps  2032(%ecx), %xmm3, %xmm2 {%k7} {z}
-0x62,0xf2,0x66,0x8f,0x52,0x51,0x7f
-
-# CHECK: vdpbf16ps  -512(%edx){1to4}, %xmm3, %xmm2 {%k7} {z}
-0x62,0xf2,0x66,0x9f,0x52,0x52,0x80
diff --git a/llvm/test/MC/Disassembler/X86/avx512bf16vl-intel.txt b/llvm/test/MC/Disassembler/X86/avx512bf16vl-intel.txt
deleted file mode 100644
index 3d9dcb1879d4459cef943b94ddcde763a1ef0fa5..0000000000000000000000000000000000000000
--- a/llvm/test/MC/Disassembler/X86/avx512bf16vl-intel.txt
+++ /dev/null
@@ -1,157 +0,0 @@
-# RUN: llvm-mc --disassemble %s -triple=i686 -x86-asm-syntax=intel --output-asm-variant=1 | FileCheck %s
-
-# CHECK: vcvtne2ps2bf16 ymm2, ymm3, ymm4
-0x62,0xf2,0x67,0x28,0x72,0xd4
-
-# CHECK: vcvtne2ps2bf16 ymm2 {k7}, ymm3, ymm4
-0x62,0xf2,0x67,0x2f,0x72,0xd4
-
-# CHECK: vcvtne2ps2bf16 ymm2 {k7} {z}, ymm3, ymm4
-0x62,0xf2,0x67,0xaf,0x72,0xd4
-
-# CHECK: vcvtne2ps2bf16 xmm2, xmm3, xmm4
-0x62,0xf2,0x67,0x08,0x72,0xd4
-
-# CHECK: vcvtne2ps2bf16 xmm2 {k7}, xmm3, xmm4
-0x62,0xf2,0x67,0x0f,0x72,0xd4
-
-# CHECK: vcvtne2ps2bf16 xmm2 {k7} {z}, xmm3, xmm4
-0x62,0xf2,0x67,0x8f,0x72,0xd4
-
-# CHECK: vcvtne2ps2bf16 ymm2, ymm3, ymmword ptr [esp + 8*esi + 268435456]
-0x62,0xf2,0x67,0x28,0x72,0x94,0xf4,0x00,0x00,0x00,0x10
-
-# CHECK: vcvtne2ps2bf16 ymm2 {k7}, ymm3, ymmword ptr [edi + 4*eax + 291]
-0x62,0xf2,0x67,0x2f,0x72,0x94,0x87,0x23,0x01,0x00,0x00
-
-# CHECK: vcvtne2ps2bf16 ymm2, ymm3, dword ptr [eax]{1to8}
-0x62,0xf2,0x67,0x38,0x72,0x10
-
-# CHECK: vcvtne2ps2bf16 ymm2, ymm3, ymmword ptr [2*ebp - 1024]
-0x62,0xf2,0x67,0x28,0x72,0x14,0x6d,0x00,0xfc,0xff,0xff
-
-# CHECK: vcvtne2ps2bf16 ymm2 {k7} {z}, ymm3, ymmword ptr [ecx + 4064]
-0x62,0xf2,0x67,0xaf,0x72,0x51,0x7f
-
-# CHECK: vcvtne2ps2bf16 ymm2 {k7} {z}, ymm3, dword ptr [edx - 512]{1to8}
-0x62,0xf2,0x67,0xbf,0x72,0x52,0x80
-
-# CHECK: vcvtne2ps2bf16 xmm2, xmm3, xmmword ptr [esp + 8*esi + 268435456]
-0x62,0xf2,0x67,0x08,0x72,0x94,0xf4,0x00,0x00,0x00,0x10
-
-# CHECK: vcvtne2ps2bf16 xmm2 {k7}, xmm3, xmmword ptr [edi + 4*eax + 291]
-0x62,0xf2,0x67,0x0f,0x72,0x94,0x87,0x23,0x01,0x00,0x00
-
-# CHECK: vcvtne2ps2bf16 xmm2, xmm3, dword ptr [eax]{1to4}
-0x62,0xf2,0x67,0x18,0x72,0x10
-
-# CHECK: vcvtne2ps2bf16 xmm2, xmm3, xmmword ptr [2*ebp - 512]
-0x62,0xf2,0x67,0x08,0x72,0x14,0x6d,0x00,0xfe,0xff,0xff
-
-# CHECK: vcvtne2ps2bf16 xmm2 {k7} {z}, xmm3, xmmword ptr [ecx + 2032]
-0x62,0xf2,0x67,0x8f,0x72,0x51,0x7f
-
-# CHECK: vcvtne2ps2bf16 xmm2 {k7} {z}, xmm3, dword ptr [edx - 512]{1to4}
-0x62,0xf2,0x67,0x9f,0x72,0x52,0x80
-
-# CHECK: vcvtneps2bf16 xmm2, xmm3
-0x62,0xf2,0x7e,0x08,0x72,0xd3
-
-# CHECK: vcvtneps2bf16 xmm2 {k7}, xmm3
-0x62,0xf2,0x7e,0x0f,0x72,0xd3
-
-# CHECK: vcvtneps2bf16 xmm2 {k7} {z}, xmm3
-0x62,0xf2,0x7e,0x8f,0x72,0xd3
-
-# CHECK: vcvtneps2bf16 xmm2, ymm3
-0x62,0xf2,0x7e,0x28,0x72,0xd3
-
-# CHECK: vcvtneps2bf16 xmm2 {k7}, ymm3
-0x62,0xf2,0x7e,0x2f,0x72,0xd3
-
-# CHECK: vcvtneps2bf16 xmm2 {k7} {z}, ymm3
-0x62,0xf2,0x7e,0xaf,0x72,0xd3
-
-# CHECK: vcvtneps2bf16 xmm2, xmmword ptr [esp + 8*esi + 268435456]
-0x62,0xf2,0x7e,0x08,0x72,0x94,0xf4,0x00,0x00,0x00,0x10
-
-# CHECK: vcvtneps2bf16 xmm2 {k7}, xmmword ptr [edi + 4*eax + 291]
-0x62,0xf2,0x7e,0x0f,0x72,0x94,0x87,0x23,0x01,0x00,0x00
-
-# CHECK: vcvtneps2bf16 xmm2, dword ptr [eax]{1to4}
-0x62,0xf2,0x7e,0x18,0x72,0x10
-
-# CHECK: vcvtneps2bf16 xmm2, xmmword ptr [2*ebp - 512]
-0x62,0xf2,0x7e,0x08,0x72,0x14,0x6d,0x00,0xfe,0xff,0xff
-
-# CHECK: vcvtneps2bf16 xmm2 {k7} {z}, xmmword ptr [ecx + 2032]
-0x62,0xf2,0x7e,0x8f,0x72,0x51,0x7f
-
-# CHECK: vcvtneps2bf16 xmm2 {k7} {z}, dword ptr [edx - 512]{1to4}
-0x62,0xf2,0x7e,0x9f,0x72,0x52,0x80
-
-# CHECK: vcvtneps2bf16 xmm2, dword ptr [eax]{1to8}
-0x62,0xf2,0x7e,0x38,0x72,0x10
-
-# CHECK: vcvtneps2bf16 xmm2, ymmword ptr [2*ebp - 1024]
-0x62,0xf2,0x7e,0x28,0x72,0x14,0x6d,0x00,0xfc,0xff,0xff
-
-# CHECK: vcvtneps2bf16 xmm2 {k7} {z}, ymmword ptr [ecx + 4064]
-0x62,0xf2,0x7e,0xaf,0x72,0x51,0x7f
-
-# CHECK: vcvtneps2bf16 xmm2 {k7} {z}, dword ptr [edx - 512]{1to8}
-0x62,0xf2,0x7e,0xbf,0x72,0x52,0x80
-
-# CHECK: vdpbf16ps ymm2, ymm3, ymm4
-0x62,0xf2,0x66,0x28,0x52,0xd4
-
-# CHECK: vdpbf16ps ymm2 {k7}, ymm3, ymm4
-0x62,0xf2,0x66,0x2f,0x52,0xd4
-
-# CHECK: vdpbf16ps ymm2 {k7} {z}, ymm3, ymm4
-0x62,0xf2,0x66,0xaf,0x52,0xd4
-
-# CHECK: vdpbf16ps xmm2, xmm3, xmm4
-0x62,0xf2,0x66,0x08,0x52,0xd4
-
-# CHECK: vdpbf16ps xmm2 {k7}, xmm3, xmm4
-0x62,0xf2,0x66,0x0f,0x52,0xd4
-
-# CHECK: vdpbf16ps xmm2 {k7} {z}, xmm3, xmm4
-0x62,0xf2,0x66,0x8f,0x52,0xd4
-
-# CHECK: vdpbf16ps ymm2, ymm3, ymmword ptr [esp + 8*esi + 268435456]
-0x62,0xf2,0x66,0x28,0x52,0x94,0xf4,0x00,0x00,0x00,0x10
-
-# CHECK: vdpbf16ps ymm2 {k7}, ymm3, ymmword ptr [edi + 4*eax + 291]
-0x62,0xf2,0x66,0x2f,0x52,0x94,0x87,0x23,0x01,0x00,0x00
-
-# CHECK: vdpbf16ps ymm2, ymm3, dword ptr [eax]{1to8}
-0x62,0xf2,0x66,0x38,0x52,0x10
-
-# CHECK: vdpbf16ps ymm2, ymm3, ymmword ptr [2*ebp - 1024]
-0x62,0xf2,0x66,0x28,0x52,0x14,0x6d,0x00,0xfc,0xff,0xff
-
-# CHECK: vdpbf16ps ymm2 {k7} {z}, ymm3, ymmword ptr [ecx + 4064]
-0x62,0xf2,0x66,0xaf,0x52,0x51,0x7f
-
-# CHECK: vdpbf16ps ymm2 {k7} {z}, ymm3, dword ptr [edx - 512]{1to8}
-0x62,0xf2,0x66,0xbf,0x52,0x52,0x80
-
-# CHECK: vdpbf16ps xmm2, xmm3, xmmword ptr [esp + 8*esi + 268435456]
-0x62,0xf2,0x66,0x08,0x52,0x94,0xf4,0x00,0x00,0x00,0x10
-
-# CHECK: vdpbf16ps xmm2 {k7}, xmm3, xmmword ptr [edi + 4*eax + 291]
-0x62,0xf2,0x66,0x0f,0x52,0x94,0x87,0x23,0x01,0x00,0x00
-
-# CHECK: vdpbf16ps xmm2, xmm3, dword ptr [eax]{1to4}
-0x62,0xf2,0x66,0x18,0x52,0x10
-
-# CHECK: vdpbf16ps xmm2, xmm3, xmmword ptr [2*ebp - 512]
-0x62,0xf2,0x66,0x08,0x52,0x14,0x6d,0x00,0xfe,0xff,0xff
-
-# CHECK: vdpbf16ps xmm2 {k7} {z}, xmm3, xmmword ptr [ecx + 2032]
-0x62,0xf2,0x66,0x8f,0x52,0x51,0x7f
-
-# CHECK: vdpbf16ps xmm2 {k7} {z}, xmm3, dword ptr [edx - 512]{1to4}
-0x62,0xf2,0x66,0x9f,0x52,0x52,0x80
diff --git a/llvm/test/MC/Disassembler/X86/avx512bitalg.txt b/llvm/test/MC/Disassembler/X86/avx512bitalg.txt
new file mode 100644
index 0000000000000000000000000000000000000000..13ab5d6dfa840263db75c39fcf5671849b391152
--- /dev/null
+++ b/llvm/test/MC/Disassembler/X86/avx512bitalg.txt
@@ -0,0 +1,170 @@
+# RUN: llvm-mc -triple x86_64 -disassemble %s | FileCheck %s --check-prefix=ATT
+# RUN: llvm-mc -triple x86_64 -disassemble -output-asm-variant=1 %s | FileCheck %s --check-prefix=INTEL
+
+# ATT:   vpopcntb	%zmm23, %zmm21
+# INTEL: vpopcntb	zmm21, zmm23
+0x62,0xa2,0x7d,0x48,0x54,0xef
+
+# ATT:   vpopcntw	%zmm23, %zmm21
+# INTEL: vpopcntw	zmm21, zmm23
+0x62,0xa2,0xfd,0x48,0x54,0xef
+
+# ATT:   vpopcntb	%zmm3, %zmm1 {%k2}
+# INTEL: vpopcntb	zmm1 {k2}, zmm3
+0x62,0xf2,0x7d,0x4a,0x54,0xcb
+
+# ATT:   vpopcntw	%zmm3, %zmm1 {%k2}
+# INTEL: vpopcntw	zmm1 {k2}, zmm3
+0x62,0xf2,0xfd,0x4a,0x54,0xcb
+
+# ATT:   vpopcntb	(%rcx), %zmm1
+# INTEL: vpopcntb	zmm1, zmmword ptr [rcx]
+0x62,0xf2,0x7d,0x48,0x54,0x09
+
+# ATT:   vpopcntb	-256(%rsp), %zmm1
+# INTEL: vpopcntb	zmm1, zmmword ptr [rsp - 256]
+0x62,0xf2,0x7d,0x48,0x54,0x4c,0x24,0xfc
+
+# ATT:   vpopcntb	256(%rsp), %zmm1
+# INTEL: vpopcntb	zmm1, zmmword ptr [rsp + 256]
+0x62,0xf2,0x7d,0x48,0x54,0x4c,0x24,0x04
+
+# ATT:   vpopcntb	268435456(%rcx,%r14,8), %zmm1
+# INTEL: vpopcntb	zmm1, zmmword ptr [rcx + 8*r14 + 268435456]
+0x62,0xb2,0x7d,0x48,0x54,0x8c,0xf1,0x00,0x00,0x00,0x10
+
+# ATT:   vpopcntb	-536870912(%rcx,%r14,8), %zmm1
+# INTEL: vpopcntb	zmm1, zmmword ptr [rcx + 8*r14 - 536870912]
+0x62,0xb2,0x7d,0x48,0x54,0x8c,0xf1,0x00,0x00,0x00,0xe0
+
+# ATT:   vpopcntb	-536870910(%rcx,%r14,8), %zmm1
+# INTEL: vpopcntb	zmm1, zmmword ptr [rcx + 8*r14 - 536870910]
+0x62,0xb2,0x7d,0x48,0x54,0x8c,0xf1,0x02,0x00,0x00,0xe0
+
+# ATT:   vpopcntw	(%rcx), %zmm1
+# INTEL: vpopcntw	zmm1, zmmword ptr [rcx]
+0x62,0xf2,0xfd,0x48,0x54,0x09
+
+# ATT:   vpopcntw	-256(%rsp), %zmm1
+# INTEL: vpopcntw	zmm1, zmmword ptr [rsp - 256]
+0x62,0xf2,0xfd,0x48,0x54,0x4c,0x24,0xfc
+
+# ATT:   vpopcntw	256(%rsp), %zmm1
+# INTEL: vpopcntw	zmm1, zmmword ptr [rsp + 256]
+0x62,0xf2,0xfd,0x48,0x54,0x4c,0x24,0x04
+
+# ATT:   vpopcntw	268435456(%rcx,%r14,8), %zmm1
+# INTEL: vpopcntw	zmm1, zmmword ptr [rcx + 8*r14 + 268435456]
+0x62,0xb2,0xfd,0x48,0x54,0x8c,0xf1,0x00,0x00,0x00,0x10
+
+# ATT:   vpopcntw	-536870912(%rcx,%r14,8), %zmm1
+# INTEL: vpopcntw	zmm1, zmmword ptr [rcx + 8*r14 - 536870912]
+0x62,0xb2,0xfd,0x48,0x54,0x8c,0xf1,0x00,0x00,0x00,0xe0
+
+# ATT:   vpopcntw	-536870910(%rcx,%r14,8), %zmm1
+# INTEL: vpopcntw	zmm1, zmmword ptr [rcx + 8*r14 - 536870910]
+0x62,0xb2,0xfd,0x48,0x54,0x8c,0xf1,0x02,0x00,0x00,0xe0
+
+# ATT:   vpopcntb	(%rcx), %zmm21 {%k2}
+# INTEL: vpopcntb	zmm21 {k2}, zmmword ptr [rcx]
+0x62,0xe2,0x7d,0x4a,0x54,0x29
+
+# ATT:   vpopcntb	-256(%rsp), %zmm21 {%k2}
+# INTEL: vpopcntb	zmm21 {k2}, zmmword ptr [rsp - 256]
+0x62,0xe2,0x7d,0x4a,0x54,0x6c,0x24,0xfc
+
+# ATT:   vpopcntb	256(%rsp), %zmm21 {%k2}
+# INTEL: vpopcntb	zmm21 {k2}, zmmword ptr [rsp + 256]
+0x62,0xe2,0x7d,0x4a,0x54,0x6c,0x24,0x04
+
+# ATT:   vpopcntb	268435456(%rcx,%r14,8), %zmm21 {%k2}
+# INTEL: vpopcntb	zmm21 {k2}, zmmword ptr [rcx + 8*r14 + 268435456]
+0x62,0xa2,0x7d,0x4a,0x54,0xac,0xf1,0x00,0x00,0x00,0x10
+
+# ATT:   vpopcntb	-536870912(%rcx,%r14,8), %zmm21 {%k2}
+# INTEL: vpopcntb	zmm21 {k2}, zmmword ptr [rcx + 8*r14 - 536870912]
+0x62,0xa2,0x7d,0x4a,0x54,0xac,0xf1,0x00,0x00,0x00,0xe0
+
+# ATT:   vpopcntb	-536870910(%rcx,%r14,8), %zmm21 {%k2}
+# INTEL: vpopcntb	zmm21 {k2}, zmmword ptr [rcx + 8*r14 - 536870910]
+0x62,0xa2,0x7d,0x4a,0x54,0xac,0xf1,0x02,0x00,0x00,0xe0
+
+# ATT:   vpopcntw	(%rcx), %zmm21 {%k2}
+# INTEL: vpopcntw	zmm21 {k2}, zmmword ptr [rcx]
+0x62,0xe2,0xfd,0x4a,0x54,0x29
+
+# ATT:   vpopcntw	-256(%rsp), %zmm21 {%k2}
+# INTEL: vpopcntw	zmm21 {k2}, zmmword ptr [rsp - 256]
+0x62,0xe2,0xfd,0x4a,0x54,0x6c,0x24,0xfc
+
+# ATT:   vpopcntw	256(%rsp), %zmm21 {%k2}
+# INTEL: vpopcntw	zmm21 {k2}, zmmword ptr [rsp + 256]
+0x62,0xe2,0xfd,0x4a,0x54,0x6c,0x24,0x04
+
+# ATT:   vpopcntw	268435456(%rcx,%r14,8), %zmm21 {%k2}
+# INTEL: vpopcntw	zmm21 {k2}, zmmword ptr [rcx + 8*r14 + 268435456]
+0x62,0xa2,0xfd,0x4a,0x54,0xac,0xf1,0x00,0x00,0x00,0x10
+
+# ATT:   vpopcntw	-536870912(%rcx,%r14,8), %zmm21 {%k2}
+# INTEL: vpopcntw	zmm21 {k2}, zmmword ptr [rcx + 8*r14 - 536870912]
+0x62,0xa2,0xfd,0x4a,0x54,0xac,0xf1,0x00,0x00,0x00,0xe0
+
+# ATT:   vpopcntw	-536870910(%rcx,%r14,8), %zmm21 {%k2}
+# INTEL: vpopcntw	zmm21 {k2}, zmmword ptr [rcx + 8*r14 - 536870910]
+0x62,0xa2,0xfd,0x4a,0x54,0xac,0xf1,0x02,0x00,0x00,0xe0
+
+# ATT:   vpshufbitqmb	%zmm2, %zmm23, %k1
+# INTEL: vpshufbitqmb	k1, zmm23, zmm2
+0x62,0xf2,0x45,0x40,0x8f,0xca
+
+# ATT:   vpshufbitqmb	%zmm2, %zmm23, %k1 {%k2}
+# INTEL: vpshufbitqmb	k1 {k2}, zmm23, zmm2
+0x62,0xf2,0x45,0x42,0x8f,0xca
+
+# ATT:   vpshufbitqmb	(%rcx), %zmm23, %k1
+# INTEL: vpshufbitqmb	k1, zmm23, zmmword ptr [rcx]
+0x62,0xf2,0x45,0x40,0x8f,0x09
+
+# ATT:   vpshufbitqmb	-256(%rsp), %zmm23, %k1
+# INTEL: vpshufbitqmb	k1, zmm23, zmmword ptr [rsp - 256]
+0x62,0xf2,0x45,0x40,0x8f,0x4c,0x24,0xfc
+
+# ATT:   vpshufbitqmb	256(%rsp), %zmm23, %k1
+# INTEL: vpshufbitqmb	k1, zmm23, zmmword ptr [rsp + 256]
+0x62,0xf2,0x45,0x40,0x8f,0x4c,0x24,0x04
+
+# ATT:   vpshufbitqmb	268435456(%rcx,%r14,8), %zmm23, %k1
+# INTEL: vpshufbitqmb	k1, zmm23, zmmword ptr [rcx + 8*r14 + 268435456]
+0x62,0xb2,0x45,0x40,0x8f,0x8c,0xf1,0x00,0x00,0x00,0x10
+
+# ATT:   vpshufbitqmb	-536870912(%rcx,%r14,8), %zmm23, %k1
+# INTEL: vpshufbitqmb	k1, zmm23, zmmword ptr [rcx + 8*r14 - 536870912]
+0x62,0xb2,0x45,0x40,0x8f,0x8c,0xf1,0x00,0x00,0x00,0xe0
+
+# ATT:   vpshufbitqmb	-536870910(%rcx,%r14,8), %zmm23, %k1
+# INTEL: vpshufbitqmb	k1, zmm23, zmmword ptr [rcx + 8*r14 - 536870910]
+0x62,0xb2,0x45,0x40,0x8f,0x8c,0xf1,0x02,0x00,0x00,0xe0
+
+# ATT:   vpshufbitqmb	(%rcx), %zmm23, %k1 {%k2}
+# INTEL: vpshufbitqmb	k1 {k2}, zmm23, zmmword ptr [rcx]
+0x62,0xf2,0x45,0x42,0x8f,0x09
+
+# ATT:   vpshufbitqmb	-256(%rsp), %zmm23, %k1 {%k2}
+# INTEL: vpshufbitqmb	k1 {k2}, zmm23, zmmword ptr [rsp - 256]
+0x62,0xf2,0x45,0x42,0x8f,0x4c,0x24,0xfc
+
+# ATT:   vpshufbitqmb	256(%rsp), %zmm23, %k1 {%k2}
+# INTEL: vpshufbitqmb	k1 {k2}, zmm23, zmmword ptr [rsp + 256]
+0x62,0xf2,0x45,0x42,0x8f,0x4c,0x24,0x04
+
+# ATT:   vpshufbitqmb	268435456(%rcx,%r14,8), %zmm23, %k1 {%k2}
+# INTEL: vpshufbitqmb	k1 {k2}, zmm23, zmmword ptr [rcx + 8*r14 + 268435456]
+0x62,0xb2,0x45,0x42,0x8f,0x8c,0xf1,0x00,0x00,0x00,0x10
+
+# ATT:   vpshufbitqmb	-536870912(%rcx,%r14,8), %zmm23, %k1 {%k2}
+# INTEL: vpshufbitqmb	k1 {k2}, zmm23, zmmword ptr [rcx + 8*r14 - 536870912]
+0x62,0xb2,0x45,0x42,0x8f,0x8c,0xf1,0x00,0x00,0x00,0xe0
+
+# ATT:   vpshufbitqmb	-536870910(%rcx,%r14,8), %zmm23, %k1 {%k2}
+# INTEL: vpshufbitqmb	k1 {k2}, zmm23, zmmword ptr [rcx + 8*r14 - 536870910]
+0x62,0xb2,0x45,0x42,0x8f,0x8c,0xf1,0x02,0x00,0x00,0xe0
diff --git a/llvm/test/MC/Disassembler/X86/avx512dq_vl.txt b/llvm/test/MC/Disassembler/X86/avx512dq_vl.txt
new file mode 100644
index 0000000000000000000000000000000000000000..8cd9ed8e46e2bdd79e14f94e5be39ca8cba2eaf6
--- /dev/null
+++ b/llvm/test/MC/Disassembler/X86/avx512dq_vl.txt
@@ -0,0 +1,198 @@
+# RUN: llvm-mc -triple x86_64 -disassemble %s | FileCheck %s --check-prefix=ATT
+# RUN: llvm-mc -triple x86_64 -disassemble -output-asm-variant=1 %s | FileCheck %s --check-prefix=INTEL
+
+# ATT:   vcvtps2qq	128(%rcx), %xmm2 {%k2} {z}
+# INTEL: vcvtps2qq	xmm2 {k2} {z}, qword ptr [rcx + 128]
+0x62,0xf1,0x7d,0x8a,0x7b,0x51,0x10
+
+# ATT:   vcvtps2qq	128(%rcx), %xmm2 {%k2}
+# INTEL: vcvtps2qq	xmm2 {k2}, qword ptr [rcx + 128]
+0x62,0xf1,0x7d,0x0a,0x7b,0x51,0x10
+
+# ATT:   vcvtps2qq	128(%rcx), %xmm2
+# INTEL: vcvtps2qq	xmm2, qword ptr [rcx + 128]
+0x62,0xf1,0x7d,0x08,0x7b,0x51,0x10
+
+# ATT:   vcvttps2qq	128(%rcx), %xmm1 {%k2} {z}
+# INTEL: vcvttps2qq	xmm1 {k2} {z}, qword ptr [rcx + 128]
+0x62,0xf1,0x7d,0x8a,0x7a,0x49,0x10
+
+# ATT:   vcvttps2qq	128(%rcx), %xmm1 {%k2}
+# INTEL: vcvttps2qq	xmm1 {k2}, qword ptr [rcx + 128]
+0x62,0xf1,0x7d,0x0a,0x7a,0x49,0x10
+
+# ATT:   vcvttps2qq	128(%rcx), %xmm1
+# INTEL: vcvttps2qq	xmm1, qword ptr [rcx + 128]
+0x62,0xf1,0x7d,0x08,0x7a,0x49,0x10
+
+# ATT:   vcvtps2uqq	128(%rcx), %xmm1 {%k2} {z}
+# INTEL: vcvtps2uqq	xmm1 {k2} {z}, qword ptr [rcx + 128]
+0x62,0xf1,0x7d,0x8a,0x79,0x49,0x10
+
+# ATT:   vcvtps2uqq	128(%rcx), %xmm1 {%k2}
+# INTEL: vcvtps2uqq	xmm1 {k2}, qword ptr [rcx + 128]
+0x62,0xf1,0x7d,0x0a,0x79,0x49,0x10
+
+# ATT:   vcvtps2uqq	128(%rcx), %xmm1
+# INTEL: vcvtps2uqq	xmm1, qword ptr [rcx + 128]
+0x62,0xf1,0x7d,0x08,0x79,0x49,0x10
+
+# ATT:   vcvttps2uqq	128(%rcx), %xmm1 {%k2} {z}
+# INTEL: vcvttps2uqq	xmm1 {k2} {z}, qword ptr [rcx + 128]
+0x62,0xf1,0x7d,0x8a,0x78,0x49,0x10
+
+# ATT:   vcvttps2uqq	128(%rcx), %xmm1 {%k2}
+# INTEL: vcvttps2uqq	xmm1 {k2}, qword ptr [rcx + 128]
+0x62,0xf1,0x7d,0x0a,0x78,0x49,0x10
+
+# ATT:   vcvttps2uqq	128(%rcx), %xmm1
+# INTEL: vcvttps2uqq	xmm1, qword ptr [rcx + 128]
+0x62,0xf1,0x7d,0x08,0x78,0x49,0x10
+
+# ATT:   vcvtps2qq	128(%rcx), %xmm2 {%k2} {z}
+# INTEL: vcvtps2qq	xmm2 {k2} {z}, qword ptr [rcx + 128]
+0x62,0xf1,0x7d,0x8a,0x7b,0x51,0x10
+
+# ATT:   vcvtps2qq	128(%rcx), %xmm2 {%k2}
+# INTEL: vcvtps2qq	xmm2 {k2}, qword ptr [rcx + 128]
+0x62,0xf1,0x7d,0x0a,0x7b,0x51,0x10
+
+# ATT:   vcvtps2qq	128(%rcx), %xmm2
+# INTEL: vcvtps2qq	xmm2, qword ptr [rcx + 128]
+0x62,0xf1,0x7d,0x08,0x7b,0x51,0x10
+
+# ATT:   vcvttps2qq	128(%rcx), %xmm1 {%k2} {z}
+# INTEL: vcvttps2qq	xmm1 {k2} {z}, qword ptr [rcx + 128]
+0x62,0xf1,0x7d,0x8a,0x7a,0x49,0x10
+
+# ATT:   vcvttps2qq	128(%rcx), %xmm1 {%k2}
+# INTEL: vcvttps2qq	xmm1 {k2}, qword ptr [rcx + 128]
+0x62,0xf1,0x7d,0x0a,0x7a,0x49,0x10
+
+# ATT:   vcvttps2qq	128(%rcx), %xmm1
+# INTEL: vcvttps2qq	xmm1, qword ptr [rcx + 128]
+0x62,0xf1,0x7d,0x08,0x7a,0x49,0x10
+
+# ATT:   vcvtps2uqq	128(%rcx), %xmm1 {%k2} {z}
+# INTEL: vcvtps2uqq	xmm1 {k2} {z}, qword ptr [rcx + 128]
+0x62,0xf1,0x7d,0x8a,0x79,0x49,0x10
+
+# ATT:   vcvtps2uqq	128(%rcx), %xmm1 {%k2}
+# INTEL: vcvtps2uqq	xmm1 {k2}, qword ptr [rcx + 128]
+0x62,0xf1,0x7d,0x0a,0x79,0x49,0x10
+
+# ATT:   vcvtps2uqq	128(%rcx), %xmm1
+# INTEL: vcvtps2uqq	xmm1, qword ptr [rcx + 128]
+0x62,0xf1,0x7d,0x08,0x79,0x49,0x10
+
+# ATT:   vcvttps2uqq	128(%rcx), %xmm1 {%k2} {z}
+# INTEL: vcvttps2uqq	xmm1 {k2} {z}, qword ptr [rcx + 128]
+0x62,0xf1,0x7d,0x8a,0x78,0x49,0x10
+
+# ATT:   vcvttps2uqq	128(%rcx), %xmm1 {%k2}
+# INTEL: vcvttps2uqq	xmm1 {k2}, qword ptr [rcx + 128]
+0x62,0xf1,0x7d,0x0a,0x78,0x49,0x10
+
+# ATT:   vcvttps2uqq	128(%rcx), %xmm1
+# INTEL: vcvttps2uqq	xmm1, qword ptr [rcx + 128]
+0x62,0xf1,0x7d,0x08,0x78,0x49,0x10
+
+# ATT:   vfpclasspd	$171, %xmm18, %k2
+# INTEL: vfpclasspd	k2, xmm18, 171
+0x62,0xb3,0xfd,0x08,0x66,0xd2,0xab
+
+# ATT:   vfpclasspd	$171, %xmm18, %k2 {%k7}
+# INTEL: vfpclasspd	k2 {k7}, xmm18, 171
+0x62,0xb3,0xfd,0x0f,0x66,0xd2,0xab
+
+# ATT:   vfpclasspdx	$123, (%rcx), %k2
+# INTEL: vfpclasspd	k2, xmmword ptr [rcx], 123
+0x62,0xf3,0xfd,0x08,0x66,0x11,0x7b
+
+# ATT:   vfpclasspdx	$123, (%rcx), %k2 {%k7}
+# INTEL: vfpclasspd	k2 {k7}, xmmword ptr [rcx], 123
+0x62,0xf3,0xfd,0x0f,0x66,0x11,0x7b
+
+# ATT:   vfpclasspd	$123, (%rcx){1to2}, %k2
+# INTEL: vfpclasspd	k2, qword ptr [rcx]{1to2}, 123
+0x62,0xf3,0xfd,0x18,0x66,0x11,0x7b
+
+# ATT:   vfpclasspd	$123, (%rcx){1to2}, %k2 {%k7}
+# INTEL: vfpclasspd	k2 {k7}, qword ptr [rcx]{1to2}, 123
+0x62,0xf3,0xfd,0x1f,0x66,0x11,0x7b
+
+# ATT:   vfpclassps	$171, %xmm18, %k2
+# INTEL: vfpclassps	k2, xmm18, 171
+0x62,0xb3,0x7d,0x08,0x66,0xd2,0xab
+
+# ATT:   vfpclassps	$171, %xmm18, %k2 {%k7}
+# INTEL: vfpclassps	k2 {k7}, xmm18, 171
+0x62,0xb3,0x7d,0x0f,0x66,0xd2,0xab
+
+# ATT:   vfpclasspsx	$123, (%rcx), %k2
+# INTEL: vfpclassps	k2, xmmword ptr [rcx], 123
+0x62,0xf3,0x7d,0x08,0x66,0x11,0x7b
+
+# ATT:   vfpclasspsx	$123, (%rcx), %k2 {%k7}
+# INTEL: vfpclassps	k2 {k7}, xmmword ptr [rcx], 123
+0x62,0xf3,0x7d,0x0f,0x66,0x11,0x7b
+
+# ATT:   vfpclassps	$123, (%rcx){1to4}, %k2
+# INTEL: vfpclassps	k2, dword ptr [rcx]{1to4}, 123
+0x62,0xf3,0x7d,0x18,0x66,0x11,0x7b
+
+# ATT:   vfpclassps	$123, (%rcx){1to4}, %k2 {%k7}
+# INTEL: vfpclassps	k2 {k7}, dword ptr [rcx]{1to4}, 123
+0x62,0xf3,0x7d,0x1f,0x66,0x11,0x7b
+
+# ATT:   vfpclasspd	$171, %ymm18, %k2
+# INTEL: vfpclasspd	k2, ymm18, 171
+0x62,0xb3,0xfd,0x28,0x66,0xd2,0xab
+
+# ATT:   vfpclasspd	$171, %ymm18, %k2 {%k7}
+# INTEL: vfpclasspd	k2 {k7}, ymm18, 171
+0x62,0xb3,0xfd,0x2f,0x66,0xd2,0xab
+
+# ATT:   vfpclasspdy	$123, (%rcx), %k2
+# INTEL: vfpclasspd	k2, ymmword ptr [rcx], 123
+0x62,0xf3,0xfd,0x28,0x66,0x11,0x7b
+
+# ATT:   vfpclasspdy	$123, (%rcx), %k2 {%k7}
+# INTEL: vfpclasspd	k2 {k7}, ymmword ptr [rcx], 123
+0x62,0xf3,0xfd,0x2f,0x66,0x11,0x7b
+
+# ATT:   vfpclasspd	$123, (%rcx){1to4}, %k2
+# INTEL: vfpclasspd	k2, qword ptr [rcx]{1to4}, 123
+0x62,0xf3,0xfd,0x38,0x66,0x11,0x7b
+
+# ATT:   vfpclasspd	$123, (%rcx){1to4}, %k2 {%k7}
+# INTEL: vfpclasspd	k2 {k7}, qword ptr [rcx]{1to4}, 123
+0x62,0xf3,0xfd,0x3f,0x66,0x11,0x7b
+
+# ATT:   vfpclassps	$171, %ymm18, %k2
+# INTEL: vfpclassps	k2, ymm18, 171
+0x62,0xb3,0x7d,0x28,0x66,0xd2,0xab
+
+# ATT:   vfpclassps	$171, %ymm18, %k2 {%k7}
+# INTEL: vfpclassps	k2 {k7}, ymm18, 171
+0x62,0xb3,0x7d,0x2f,0x66,0xd2,0xab
+
+# ATT:   vfpclasspsy	$123, (%rcx), %k2
+# INTEL: vfpclassps	k2, ymmword ptr [rcx], 123
+0x62,0xf3,0x7d,0x28,0x66,0x11,0x7b
+
+# ATT:   vfpclasspsy	$123, (%rcx), %k2 {%k7}
+# INTEL: vfpclassps	k2 {k7}, ymmword ptr [rcx], 123
+0x62,0xf3,0x7d,0x2f,0x66,0x11,0x7b
+
+# ATT:   vfpclassps	$123, (%rcx){1to8}, %k2
+# INTEL: vfpclassps	k2, dword ptr [rcx]{1to8}, 123
+0x62,0xf3,0x7d,0x38,0x66,0x11,0x7b
+
+# ATT:   vfpclassps	$123, (%rcx){1to8}, %k2 {%k7}
+# INTEL: vfpclassps	k2 {k7}, dword ptr [rcx]{1to8}, 123
+0x62,0xf3,0x7d,0x3f,0x66,0x11,0x7b
+
+# ATT: vcvttps2uqq     128(%ecx), %xmm1 {%k2}
+# INTEL: vcvttps2uqq     xmm1 {k2}, qword ptr [ecx + 128]
+0x67,0x62,0xf1,0x7d,0x0a,0x78,0x49,0x10
diff --git a/llvm/test/MC/Disassembler/X86/avx512vbmi.txt b/llvm/test/MC/Disassembler/X86/avx512vbmi.txt
new file mode 100644
index 0000000000000000000000000000000000000000..5bec7c639377b8d8d4f991746348fe30e0c1a0c1
--- /dev/null
+++ b/llvm/test/MC/Disassembler/X86/avx512vbmi.txt
@@ -0,0 +1,542 @@
+# RUN: llvm-mc -triple x86_64 -disassemble %s | FileCheck %s --check-prefix=ATT
+# RUN: llvm-mc -triple x86_64 -disassemble -output-asm-variant=1 %s | FileCheck %s --check-prefix=INTEL
+
+# ATT:   vpermb	%xmm28, %xmm29, %xmm30 {%k7}
+# INTEL: vpermb	xmm30 {k7}, xmm29, xmm28
+0x62,0x02,0x15,0x07,0x8d,0xf4
+
+# ATT:   vpermb	%xmm28, %xmm29, %xmm30 {%k7} {z}
+# INTEL: vpermb	xmm30 {k7} {z}, xmm29, xmm28
+0x62,0x02,0x15,0x87,0x8d,0xf4
+
+# ATT:   vpermb	(%rcx), %xmm29, %xmm30
+# INTEL: vpermb	xmm30, xmm29, xmmword ptr [rcx]
+0x62,0x62,0x15,0x00,0x8d,0x31
+
+# ATT:   vpermb	291(%rax,%r14,8), %xmm29, %xmm30
+# INTEL: vpermb	xmm30, xmm29, xmmword ptr [rax + 8*r14 + 291]
+0x62,0x22,0x15,0x00,0x8d,0xb4,0xf0,0x23,0x01,0x00,0x00
+
+# ATT:   vpermb	2032(%rdx), %xmm29, %xmm30
+# INTEL: vpermb	xmm30, xmm29, xmmword ptr [rdx + 2032]
+0x62,0x62,0x15,0x00,0x8d,0x72,0x7f
+
+# ATT:   vpermb	2048(%rdx), %xmm29, %xmm30
+# INTEL: vpermb	xmm30, xmm29, xmmword ptr [rdx + 2048]
+0x62,0x62,0x15,0x00,0x8d,0xb2,0x00,0x08,0x00,0x00
+
+# ATT:   vpermb	-2048(%rdx), %xmm29, %xmm30
+# INTEL: vpermb	xmm30, xmm29, xmmword ptr [rdx - 2048]
+0x62,0x62,0x15,0x00,0x8d,0x72,0x80
+
+# ATT:   vpermb	-2064(%rdx), %xmm29, %xmm30
+# INTEL: vpermb	xmm30, xmm29, xmmword ptr [rdx - 2064]
+0x62,0x62,0x15,0x00,0x8d,0xb2,0xf0,0xf7,0xff,0xff
+
+# ATT:   vpermb	%ymm28, %ymm29, %ymm30
+# INTEL: vpermb	ymm30, ymm29, ymm28
+0x62,0x02,0x15,0x20,0x8d,0xf4
+
+# ATT:   vpermb	%ymm28, %ymm29, %ymm30 {%k7}
+# INTEL: vpermb	ymm30 {k7}, ymm29, ymm28
+0x62,0x02,0x15,0x27,0x8d,0xf4
+
+# ATT:   vpermb	%ymm28, %ymm29, %ymm30 {%k7} {z}
+# INTEL: vpermb	ymm30 {k7} {z}, ymm29, ymm28
+0x62,0x02,0x15,0xa7,0x8d,0xf4
+
+# ATT:   vpermb	(%rcx), %ymm29, %ymm30
+# INTEL: vpermb	ymm30, ymm29, ymmword ptr [rcx]
+0x62,0x62,0x15,0x20,0x8d,0x31
+
+# ATT:   vpermb	291(%rax,%r14,8), %ymm29, %ymm30
+# INTEL: vpermb	ymm30, ymm29, ymmword ptr [rax + 8*r14 + 291]
+0x62,0x22,0x15,0x20,0x8d,0xb4,0xf0,0x23,0x01,0x00,0x00
+
+# ATT:   vpermb	4064(%rdx), %ymm29, %ymm30
+# INTEL: vpermb	ymm30, ymm29, ymmword ptr [rdx + 4064]
+0x62,0x62,0x15,0x20,0x8d,0x72,0x7f
+
+# ATT:   vpermb	4096(%rdx), %ymm29, %ymm30
+# INTEL: vpermb	ymm30, ymm29, ymmword ptr [rdx + 4096]
+0x62,0x62,0x15,0x20,0x8d,0xb2,0x00,0x10,0x00,0x00
+
+# ATT:   vpermb	-4096(%rdx), %ymm29, %ymm30
+# INTEL: vpermb	ymm30, ymm29, ymmword ptr [rdx - 4096]
+0x62,0x62,0x15,0x20,0x8d,0x72,0x80
+
+# ATT:   vpermb	-4128(%rdx), %ymm29, %ymm30
+# INTEL: vpermb	ymm30, ymm29, ymmword ptr [rdx - 4128]
+0x62,0x62,0x15,0x20,0x8d,0xb2,0xe0,0xef,0xff,0xff
+
+# ATT:   vpermb	%xmm28, %xmm29, %xmm30
+# INTEL: vpermb	xmm30, xmm29, xmm28
+0x62,0x02,0x15,0x00,0x8d,0xf4
+
+# ATT:   vpermb	4660(%rax,%r14,8), %xmm29, %xmm30
+# INTEL: vpermb	xmm30, xmm29, xmmword ptr [rax + 8*r14 + 4660]
+0x62,0x22,0x15,0x00,0x8d,0xb4,0xf0,0x34,0x12,0x00,0x00
+
+# ATT:   vpermb	4660(%rax,%r14,8), %ymm29, %ymm30
+# INTEL: vpermb	ymm30, ymm29, ymmword ptr [rax + 8*r14 + 4660]
+0x62,0x22,0x15,0x20,0x8d,0xb4,0xf0,0x34,0x12,0x00,0x00
+
+# ATT:   vpermb	%zmm28, %zmm29, %zmm30
+# INTEL: vpermb	zmm30, zmm29, zmm28
+0x62,0x02,0x15,0x40,0x8d,0xf4
+
+# ATT:   vpermb	%zmm28, %zmm29, %zmm30 {%k7}
+# INTEL: vpermb	zmm30 {k7}, zmm29, zmm28
+0x62,0x02,0x15,0x47,0x8d,0xf4
+
+# ATT:   vpermb	%zmm28, %zmm29, %zmm30 {%k7} {z}
+# INTEL: vpermb	zmm30 {k7} {z}, zmm29, zmm28
+0x62,0x02,0x15,0xc7,0x8d,0xf4
+
+# ATT:   vpermb	(%rcx), %zmm29, %zmm30
+# INTEL: vpermb	zmm30, zmm29, zmmword ptr [rcx]
+0x62,0x62,0x15,0x40,0x8d,0x31
+
+# ATT:   vpermb	291(%rax,%r14,8), %zmm29, %zmm30
+# INTEL: vpermb	zmm30, zmm29, zmmword ptr [rax + 8*r14 + 291]
+0x62,0x22,0x15,0x40,0x8d,0xb4,0xf0,0x23,0x01,0x00,0x00
+
+# ATT:   vpermb	8128(%rdx), %zmm29, %zmm30
+# INTEL: vpermb	zmm30, zmm29, zmmword ptr [rdx + 8128]
+0x62,0x62,0x15,0x40,0x8d,0x72,0x7f
+
+# ATT:   vpermb	8192(%rdx), %zmm29, %zmm30
+# INTEL: vpermb	zmm30, zmm29, zmmword ptr [rdx + 8192]
+0x62,0x62,0x15,0x40,0x8d,0xb2,0x00,0x20,0x00,0x00
+
+# ATT:   vpermb	-8192(%rdx), %zmm29, %zmm30
+# INTEL: vpermb	zmm30, zmm29, zmmword ptr [rdx - 8192]
+0x62,0x62,0x15,0x40,0x8d,0x72,0x80
+
+# ATT:   vpermb	-8256(%rdx), %zmm29, %zmm30
+# INTEL: vpermb	zmm30, zmm29, zmmword ptr [rdx - 8256]
+0x62,0x62,0x15,0x40,0x8d,0xb2,0xc0,0xdf,0xff,0xff
+
+# ATT:   vpermb	4660(%rax,%r14,8), %zmm29, %zmm30
+# INTEL: vpermb	zmm30, zmm29, zmmword ptr [rax + 8*r14 + 4660]
+0x62,0x22,0x15,0x40,0x8d,0xb4,0xf0,0x34,0x12,0x00,0x00
+
+# ATT:   vpermt2b	%xmm28, %xmm29, %xmm30
+# INTEL: vpermt2b	xmm30, xmm29, xmm28
+0x62,0x02,0x15,0x00,0x7d,0xf4
+
+# ATT:   vpermt2b	%xmm28, %xmm29, %xmm30 {%k7}
+# INTEL: vpermt2b	xmm30 {k7}, xmm29, xmm28
+0x62,0x02,0x15,0x07,0x7d,0xf4
+
+# ATT:   vpermt2b	%xmm28, %xmm29, %xmm30 {%k7} {z}
+# INTEL: vpermt2b	xmm30 {k7} {z}, xmm29, xmm28
+0x62,0x02,0x15,0x87,0x7d,0xf4
+
+# ATT:   vpermt2b	(%rcx), %xmm29, %xmm30
+# INTEL: vpermt2b	xmm30, xmm29, xmmword ptr [rcx]
+0x62,0x62,0x15,0x00,0x7d,0x31
+
+# ATT:   vpermt2b	291(%rax,%r14,8), %xmm29, %xmm30
+# INTEL: vpermt2b	xmm30, xmm29, xmmword ptr [rax + 8*r14 + 291]
+0x62,0x22,0x15,0x00,0x7d,0xb4,0xf0,0x23,0x01,0x00,0x00
+
+# ATT:   vpermt2b	2032(%rdx), %xmm29, %xmm30
+# INTEL: vpermt2b	xmm30, xmm29, xmmword ptr [rdx + 2032]
+0x62,0x62,0x15,0x00,0x7d,0x72,0x7f
+
+# ATT:   vpermt2b	2048(%rdx), %xmm29, %xmm30
+# INTEL: vpermt2b	xmm30, xmm29, xmmword ptr [rdx + 2048]
+0x62,0x62,0x15,0x00,0x7d,0xb2,0x00,0x08,0x00,0x00
+
+# ATT:   vpermt2b	-2048(%rdx), %xmm29, %xmm30
+# INTEL: vpermt2b	xmm30, xmm29, xmmword ptr [rdx - 2048]
+0x62,0x62,0x15,0x00,0x7d,0x72,0x80
+
+# ATT:   vpermt2b	-2064(%rdx), %xmm29, %xmm30
+# INTEL: vpermt2b	xmm30, xmm29, xmmword ptr [rdx - 2064]
+0x62,0x62,0x15,0x00,0x7d,0xb2,0xf0,0xf7,0xff,0xff
+
+# ATT:   vpermt2b	%ymm28, %ymm29, %ymm30
+# INTEL: vpermt2b	ymm30, ymm29, ymm28
+0x62,0x02,0x15,0x20,0x7d,0xf4
+
+# ATT:   vpermt2b	%ymm28, %ymm29, %ymm30 {%k7}
+# INTEL: vpermt2b	ymm30 {k7}, ymm29, ymm28
+0x62,0x02,0x15,0x27,0x7d,0xf4
+
+# ATT:   vpermt2b	%ymm28, %ymm29, %ymm30 {%k7} {z}
+# INTEL: vpermt2b	ymm30 {k7} {z}, ymm29, ymm28
+0x62,0x02,0x15,0xa7,0x7d,0xf4
+
+# ATT:   vpermt2b	(%rcx), %ymm29, %ymm30
+# INTEL: vpermt2b	ymm30, ymm29, ymmword ptr [rcx]
+0x62,0x62,0x15,0x20,0x7d,0x31
+
+# ATT:   vpermt2b	291(%rax,%r14,8), %ymm29, %ymm30
+# INTEL: vpermt2b	ymm30, ymm29, ymmword ptr [rax + 8*r14 + 291]
+0x62,0x22,0x15,0x20,0x7d,0xb4,0xf0,0x23,0x01,0x00,0x00
+
+# ATT:   vpermt2b	4064(%rdx), %ymm29, %ymm30
+# INTEL: vpermt2b	ymm30, ymm29, ymmword ptr [rdx + 4064]
+0x62,0x62,0x15,0x20,0x7d,0x72,0x7f
+
+# ATT:   vpermt2b	4096(%rdx), %ymm29, %ymm30
+# INTEL: vpermt2b	ymm30, ymm29, ymmword ptr [rdx + 4096]
+0x62,0x62,0x15,0x20,0x7d,0xb2,0x00,0x10,0x00,0x00
+
+# ATT:   vpermt2b	-4096(%rdx), %ymm29, %ymm30
+# INTEL: vpermt2b	ymm30, ymm29, ymmword ptr [rdx - 4096]
+0x62,0x62,0x15,0x20,0x7d,0x72,0x80
+
+# ATT:   vpermt2b	-4128(%rdx), %ymm29, %ymm30
+# INTEL: vpermt2b	ymm30, ymm29, ymmword ptr [rdx - 4128]
+0x62,0x62,0x15,0x20,0x7d,0xb2,0xe0,0xef,0xff,0xff
+
+# ATT:   vpermt2b	4660(%rax,%r14,8), %xmm29, %xmm30
+# INTEL: vpermt2b	xmm30, xmm29, xmmword ptr [rax + 8*r14 + 4660]
+0x62,0x22,0x15,0x00,0x7d,0xb4,0xf0,0x34,0x12,0x00,0x00
+
+# ATT:   vpermt2b	4660(%rax,%r14,8), %ymm29, %ymm30
+# INTEL: vpermt2b	ymm30, ymm29, ymmword ptr [rax + 8*r14 + 4660]
+0x62,0x22,0x15,0x20,0x7d,0xb4,0xf0,0x34,0x12,0x00,0x00
+
+# ATT:   vpermt2b	%zmm28, %zmm29, %zmm30
+# INTEL: vpermt2b	zmm30, zmm29, zmm28
+0x62,0x02,0x15,0x40,0x7d,0xf4
+
+# ATT:   vpermt2b	%zmm28, %zmm29, %zmm30 {%k7}
+# INTEL: vpermt2b	zmm30 {k7}, zmm29, zmm28
+0x62,0x02,0x15,0x47,0x7d,0xf4
+
+# ATT:   vpermt2b	%zmm28, %zmm29, %zmm30 {%k7} {z}
+# INTEL: vpermt2b	zmm30 {k7} {z}, zmm29, zmm28
+0x62,0x02,0x15,0xc7,0x7d,0xf4
+
+# ATT:   vpermt2b	(%rcx), %zmm29, %zmm30
+# INTEL: vpermt2b	zmm30, zmm29, zmmword ptr [rcx]
+0x62,0x62,0x15,0x40,0x7d,0x31
+
+# ATT:   vpermt2b	291(%rax,%r14,8), %zmm29, %zmm30
+# INTEL: vpermt2b	zmm30, zmm29, zmmword ptr [rax + 8*r14 + 291]
+0x62,0x22,0x15,0x40,0x7d,0xb4,0xf0,0x23,0x01,0x00,0x00
+
+# ATT:   vpermt2b	8128(%rdx), %zmm29, %zmm30
+# INTEL: vpermt2b	zmm30, zmm29, zmmword ptr [rdx + 8128]
+0x62,0x62,0x15,0x40,0x7d,0x72,0x7f
+
+# ATT:   vpermt2b	8192(%rdx), %zmm29, %zmm30
+# INTEL: vpermt2b	zmm30, zmm29, zmmword ptr [rdx + 8192]
+0x62,0x62,0x15,0x40,0x7d,0xb2,0x00,0x20,0x00,0x00
+
+# ATT:   vpermt2b	-8192(%rdx), %zmm29, %zmm30
+# INTEL: vpermt2b	zmm30, zmm29, zmmword ptr [rdx - 8192]
+0x62,0x62,0x15,0x40,0x7d,0x72,0x80
+
+# ATT:   vpermt2b	-8256(%rdx), %zmm29, %zmm30
+# INTEL: vpermt2b	zmm30, zmm29, zmmword ptr [rdx - 8256]
+0x62,0x62,0x15,0x40,0x7d,0xb2,0xc0,0xdf,0xff,0xff
+
+# ATT:   vpermt2b	4660(%rax,%r14,8), %zmm29, %zmm30
+# INTEL: vpermt2b	zmm30, zmm29, zmmword ptr [rax + 8*r14 + 4660]
+0x62,0x22,0x15,0x40,0x7d,0xb4,0xf0,0x34,0x12,0x00,0x00
+
+# ATT:   vpermi2b	%xmm28, %xmm29, %xmm30
+# INTEL: vpermi2b	xmm30, xmm29, xmm28
+0x62,0x02,0x15,0x00,0x75,0xf4
+
+# ATT:   vpermi2b	%xmm28, %xmm29, %xmm30 {%k7}
+# INTEL: vpermi2b	xmm30 {k7}, xmm29, xmm28
+0x62,0x02,0x15,0x07,0x75,0xf4
+
+# ATT:   vpermi2b	%xmm28, %xmm29, %xmm30 {%k7} {z}
+# INTEL: vpermi2b	xmm30 {k7} {z}, xmm29, xmm28
+0x62,0x02,0x15,0x87,0x75,0xf4
+
+# ATT:   vpermi2b	(%rcx), %xmm29, %xmm30
+# INTEL: vpermi2b	xmm30, xmm29, xmmword ptr [rcx]
+0x62,0x62,0x15,0x00,0x75,0x31
+
+# ATT:   vpermi2b	291(%rax,%r14,8), %xmm29, %xmm30
+# INTEL: vpermi2b	xmm30, xmm29, xmmword ptr [rax + 8*r14 + 291]
+0x62,0x22,0x15,0x00,0x75,0xb4,0xf0,0x23,0x01,0x00,0x00
+
+# ATT:   vpermi2b	2032(%rdx), %xmm29, %xmm30
+# INTEL: vpermi2b	xmm30, xmm29, xmmword ptr [rdx + 2032]
+0x62,0x62,0x15,0x00,0x75,0x72,0x7f
+
+# ATT:   vpermi2b	2048(%rdx), %xmm29, %xmm30
+# INTEL: vpermi2b	xmm30, xmm29, xmmword ptr [rdx + 2048]
+0x62,0x62,0x15,0x00,0x75,0xb2,0x00,0x08,0x00,0x00
+
+# ATT:   vpermi2b	-2048(%rdx), %xmm29, %xmm30
+# INTEL: vpermi2b	xmm30, xmm29, xmmword ptr [rdx - 2048]
+0x62,0x62,0x15,0x00,0x75,0x72,0x80
+
+# ATT:   vpermi2b	-2064(%rdx), %xmm29, %xmm30
+# INTEL: vpermi2b	xmm30, xmm29, xmmword ptr [rdx - 2064]
+0x62,0x62,0x15,0x00,0x75,0xb2,0xf0,0xf7,0xff,0xff
+
+# ATT:   vpermi2b	%ymm28, %ymm29, %ymm30
+# INTEL: vpermi2b	ymm30, ymm29, ymm28
+0x62,0x02,0x15,0x20,0x75,0xf4
+
+# ATT:   vpermi2b	%ymm28, %ymm29, %ymm30 {%k7}
+# INTEL: vpermi2b	ymm30 {k7}, ymm29, ymm28
+0x62,0x02,0x15,0x27,0x75,0xf4
+
+# ATT:   vpermi2b	%ymm28, %ymm29, %ymm30 {%k7} {z}
+# INTEL: vpermi2b	ymm30 {k7} {z}, ymm29, ymm28
+0x62,0x02,0x15,0xa7,0x75,0xf4
+
+# ATT:   vpermi2b	(%rcx), %ymm29, %ymm30
+# INTEL: vpermi2b	ymm30, ymm29, ymmword ptr [rcx]
+0x62,0x62,0x15,0x20,0x75,0x31
+
+# ATT:   vpermi2b	291(%rax,%r14,8), %ymm29, %ymm30
+# INTEL: vpermi2b	ymm30, ymm29, ymmword ptr [rax + 8*r14 + 291]
+0x62,0x22,0x15,0x20,0x75,0xb4,0xf0,0x23,0x01,0x00,0x00
+
+# ATT:   vpermi2b	4064(%rdx), %ymm29, %ymm30
+# INTEL: vpermi2b	ymm30, ymm29, ymmword ptr [rdx + 4064]
+0x62,0x62,0x15,0x20,0x75,0x72,0x7f
+
+# ATT:   vpermi2b	4096(%rdx), %ymm29, %ymm30
+# INTEL: vpermi2b	ymm30, ymm29, ymmword ptr [rdx + 4096]
+0x62,0x62,0x15,0x20,0x75,0xb2,0x00,0x10,0x00,0x00
+
+# ATT:   vpermi2b	-4096(%rdx), %ymm29, %ymm30
+# INTEL: vpermi2b	ymm30, ymm29, ymmword ptr [rdx - 4096]
+0x62,0x62,0x15,0x20,0x75,0x72,0x80
+
+# ATT:   vpermi2b	-4128(%rdx), %ymm29, %ymm30
+# INTEL: vpermi2b	ymm30, ymm29, ymmword ptr [rdx - 4128]
+0x62,0x62,0x15,0x20,0x75,0xb2,0xe0,0xef,0xff,0xff
+
+# ATT:   vpermi2b	4660(%rax,%r14,8), %xmm29, %xmm30
+# INTEL: vpermi2b	xmm30, xmm29, xmmword ptr [rax + 8*r14 + 4660]
+0x62,0x22,0x15,0x00,0x75,0xb4,0xf0,0x34,0x12,0x00,0x00
+
+# ATT:   vpermi2b	4660(%rax,%r14,8), %ymm29, %ymm30
+# INTEL: vpermi2b	ymm30, ymm29, ymmword ptr [rax + 8*r14 + 4660]
+0x62,0x22,0x15,0x20,0x75,0xb4,0xf0,0x34,0x12,0x00,0x00
+
+# ATT:   vpermi2b	%zmm28, %zmm29, %zmm30
+# INTEL: vpermi2b	zmm30, zmm29, zmm28
+0x62,0x02,0x15,0x40,0x75,0xf4
+
+# ATT:   vpermi2b	%zmm28, %zmm29, %zmm30 {%k7}
+# INTEL: vpermi2b	zmm30 {k7}, zmm29, zmm28
+0x62,0x02,0x15,0x47,0x75,0xf4
+
+# ATT:   vpermi2b	%zmm28, %zmm29, %zmm30 {%k7} {z}
+# INTEL: vpermi2b	zmm30 {k7} {z}, zmm29, zmm28
+0x62,0x02,0x15,0xc7,0x75,0xf4
+
+# ATT:   vpermi2b	(%rcx), %zmm29, %zmm30
+# INTEL: vpermi2b	zmm30, zmm29, zmmword ptr [rcx]
+0x62,0x62,0x15,0x40,0x75,0x31
+
+# ATT:   vpermi2b	291(%rax,%r14,8), %zmm29, %zmm30
+# INTEL: vpermi2b	zmm30, zmm29, zmmword ptr [rax + 8*r14 + 291]
+0x62,0x22,0x15,0x40,0x75,0xb4,0xf0,0x23,0x01,0x00,0x00
+
+# ATT:   vpermi2b	8128(%rdx), %zmm29, %zmm30
+# INTEL: vpermi2b	zmm30, zmm29, zmmword ptr [rdx + 8128]
+0x62,0x62,0x15,0x40,0x75,0x72,0x7f
+
+# ATT:   vpermi2b	8192(%rdx), %zmm29, %zmm30
+# INTEL: vpermi2b	zmm30, zmm29, zmmword ptr [rdx + 8192]
+0x62,0x62,0x15,0x40,0x75,0xb2,0x00,0x20,0x00,0x00
+
+# ATT:   vpermi2b	-8192(%rdx), %zmm29, %zmm30
+# INTEL: vpermi2b	zmm30, zmm29, zmmword ptr [rdx - 8192]
+0x62,0x62,0x15,0x40,0x75,0x72,0x80
+
+# ATT:   vpermi2b	-8256(%rdx), %zmm29, %zmm30
+# INTEL: vpermi2b	zmm30, zmm29, zmmword ptr [rdx - 8256]
+0x62,0x62,0x15,0x40,0x75,0xb2,0xc0,0xdf,0xff,0xff
+
+# ATT:   vpermi2b	4660(%rax,%r14,8), %zmm29, %zmm30
+# INTEL: vpermi2b	zmm30, zmm29, zmmword ptr [rax + 8*r14 + 4660]
+0x62,0x22,0x15,0x40,0x75,0xb4,0xf0,0x34,0x12,0x00,0x00
+
+# ATT:   vpmultishiftqb	%xmm28, %xmm29, %xmm30
+# INTEL: vpmultishiftqb	xmm30, xmm29, xmm28
+0x62,0x02,0x95,0x00,0x83,0xf4
+
+# ATT:   vpmultishiftqb	%xmm28, %xmm29, %xmm30 {%k7}
+# INTEL: vpmultishiftqb	xmm30 {k7}, xmm29, xmm28
+0x62,0x02,0x95,0x07,0x83,0xf4
+
+# ATT:   vpmultishiftqb	%xmm28, %xmm29, %xmm30 {%k7} {z}
+# INTEL: vpmultishiftqb	xmm30 {k7} {z}, xmm29, xmm28
+0x62,0x02,0x95,0x87,0x83,0xf4
+
+# ATT:   vpmultishiftqb	(%rcx), %xmm29, %xmm30
+# INTEL: vpmultishiftqb	xmm30, xmm29, xmmword ptr [rcx]
+0x62,0x62,0x95,0x00,0x83,0x31
+
+# ATT:   vpmultishiftqb	291(%rax,%r14,8), %xmm29, %xmm30
+# INTEL: vpmultishiftqb	xmm30, xmm29, xmmword ptr [rax + 8*r14 + 291]
+0x62,0x22,0x95,0x00,0x83,0xb4,0xf0,0x23,0x01,0x00,0x00
+
+# ATT:   vpmultishiftqb	(%rcx){1to2}, %xmm29, %xmm30
+# INTEL: vpmultishiftqb	xmm30, xmm29, qword ptr [rcx]{1to2}
+0x62,0x62,0x95,0x10,0x83,0x31
+
+# ATT:   vpmultishiftqb	2032(%rdx), %xmm29, %xmm30
+# INTEL: vpmultishiftqb	xmm30, xmm29, xmmword ptr [rdx + 2032]
+0x62,0x62,0x95,0x00,0x83,0x72,0x7f
+
+# ATT:   vpmultishiftqb	2048(%rdx), %xmm29, %xmm30
+# INTEL: vpmultishiftqb	xmm30, xmm29, xmmword ptr [rdx + 2048]
+0x62,0x62,0x95,0x00,0x83,0xb2,0x00,0x08,0x00,0x00
+
+# ATT:   vpmultishiftqb	-2048(%rdx), %xmm29, %xmm30
+# INTEL: vpmultishiftqb	xmm30, xmm29, xmmword ptr [rdx - 2048]
+0x62,0x62,0x95,0x00,0x83,0x72,0x80
+
+# ATT:   vpmultishiftqb	-2064(%rdx), %xmm29, %xmm30
+# INTEL: vpmultishiftqb	xmm30, xmm29, xmmword ptr [rdx - 2064]
+0x62,0x62,0x95,0x00,0x83,0xb2,0xf0,0xf7,0xff,0xff
+
+# ATT:   vpmultishiftqb	1016(%rdx){1to2}, %xmm29, %xmm30
+# INTEL: vpmultishiftqb	xmm30, xmm29, qword ptr [rdx + 1016]{1to2}
+0x62,0x62,0x95,0x10,0x83,0x72,0x7f
+
+# ATT:   vpmultishiftqb	1024(%rdx){1to2}, %xmm29, %xmm30
+# INTEL: vpmultishiftqb	xmm30, xmm29, qword ptr [rdx + 1024]{1to2}
+0x62,0x62,0x95,0x10,0x83,0xb2,0x00,0x04,0x00,0x00
+
+# ATT:   vpmultishiftqb	-1024(%rdx){1to2}, %xmm29, %xmm30
+# INTEL: vpmultishiftqb	xmm30, xmm29, qword ptr [rdx - 1024]{1to2}
+0x62,0x62,0x95,0x10,0x83,0x72,0x80
+
+# ATT:   vpmultishiftqb	-1032(%rdx){1to2}, %xmm29, %xmm30
+# INTEL: vpmultishiftqb	xmm30, xmm29, qword ptr [rdx - 1032]{1to2}
+0x62,0x62,0x95,0x10,0x83,0xb2,0xf8,0xfb,0xff,0xff
+
+# ATT:   vpmultishiftqb	%ymm28, %ymm29, %ymm30
+# INTEL: vpmultishiftqb	ymm30, ymm29, ymm28
+0x62,0x02,0x95,0x20,0x83,0xf4
+
+# ATT:   vpmultishiftqb	%ymm28, %ymm29, %ymm30 {%k7}
+# INTEL: vpmultishiftqb	ymm30 {k7}, ymm29, ymm28
+0x62,0x02,0x95,0x27,0x83,0xf4
+
+# ATT:   vpmultishiftqb	%ymm28, %ymm29, %ymm30 {%k7} {z}
+# INTEL: vpmultishiftqb	ymm30 {k7} {z}, ymm29, ymm28
+0x62,0x02,0x95,0xa7,0x83,0xf4
+
+# ATT:   vpmultishiftqb	(%rcx), %ymm29, %ymm30
+# INTEL: vpmultishiftqb	ymm30, ymm29, ymmword ptr [rcx]
+0x62,0x62,0x95,0x20,0x83,0x31
+
+# ATT:   vpmultishiftqb	291(%rax,%r14,8), %ymm29, %ymm30
+# INTEL: vpmultishiftqb	ymm30, ymm29, ymmword ptr [rax + 8*r14 + 291]
+0x62,0x22,0x95,0x20,0x83,0xb4,0xf0,0x23,0x01,0x00,0x00
+
+# ATT:   vpmultishiftqb	(%rcx){1to4}, %ymm29, %ymm30
+# INTEL: vpmultishiftqb	ymm30, ymm29, qword ptr [rcx]{1to4}
+0x62,0x62,0x95,0x30,0x83,0x31
+
+# ATT:   vpmultishiftqb	4064(%rdx), %ymm29, %ymm30
+# INTEL: vpmultishiftqb	ymm30, ymm29, ymmword ptr [rdx + 4064]
+0x62,0x62,0x95,0x20,0x83,0x72,0x7f
+
+# ATT:   vpmultishiftqb	4096(%rdx), %ymm29, %ymm30
+# INTEL: vpmultishiftqb	ymm30, ymm29, ymmword ptr [rdx + 4096]
+0x62,0x62,0x95,0x20,0x83,0xb2,0x00,0x10,0x00,0x00
+
+# ATT:   vpmultishiftqb	-4096(%rdx), %ymm29, %ymm30
+# INTEL: vpmultishiftqb	ymm30, ymm29, ymmword ptr [rdx - 4096]
+0x62,0x62,0x95,0x20,0x83,0x72,0x80
+
+# ATT:   vpmultishiftqb	-4128(%rdx), %ymm29, %ymm30
+# INTEL: vpmultishiftqb	ymm30, ymm29, ymmword ptr [rdx - 4128]
+0x62,0x62,0x95,0x20,0x83,0xb2,0xe0,0xef,0xff,0xff
+
+# ATT:   vpmultishiftqb	1016(%rdx){1to4}, %ymm29, %ymm30
+# INTEL: vpmultishiftqb	ymm30, ymm29, qword ptr [rdx + 1016]{1to4}
+0x62,0x62,0x95,0x30,0x83,0x72,0x7f
+
+# ATT:   vpmultishiftqb	1024(%rdx){1to4}, %ymm29, %ymm30
+# INTEL: vpmultishiftqb	ymm30, ymm29, qword ptr [rdx + 1024]{1to4}
+0x62,0x62,0x95,0x30,0x83,0xb2,0x00,0x04,0x00,0x00
+
+# ATT:   vpmultishiftqb	-1024(%rdx){1to4}, %ymm29, %ymm30
+# INTEL: vpmultishiftqb	ymm30, ymm29, qword ptr [rdx - 1024]{1to4}
+0x62,0x62,0x95,0x30,0x83,0x72,0x80
+
+# ATT:   vpmultishiftqb	-1032(%rdx){1to4}, %ymm29, %ymm30
+# INTEL: vpmultishiftqb	ymm30, ymm29, qword ptr [rdx - 1032]{1to4}
+0x62,0x62,0x95,0x30,0x83,0xb2,0xf8,0xfb,0xff,0xff
+
+# ATT:   vpmultishiftqb	4660(%rax,%r14,8), %xmm29, %xmm30
+# INTEL: vpmultishiftqb	xmm30, xmm29, xmmword ptr [rax + 8*r14 + 4660]
+0x62,0x22,0x95,0x00,0x83,0xb4,0xf0,0x34,0x12,0x00,0x00
+
+# ATT:   vpmultishiftqb	4660(%rax,%r14,8), %ymm29, %ymm30
+# INTEL: vpmultishiftqb	ymm30, ymm29, ymmword ptr [rax + 8*r14 + 4660]
+0x62,0x22,0x95,0x20,0x83,0xb4,0xf0,0x34,0x12,0x00,0x00
+
+# ATT:   vpmultishiftqb	%zmm28, %zmm29, %zmm30
+# INTEL: vpmultishiftqb	zmm30, zmm29, zmm28
+0x62,0x02,0x95,0x40,0x83,0xf4
+
+# ATT:   vpmultishiftqb	%zmm28, %zmm29, %zmm30 {%k7}
+# INTEL: vpmultishiftqb	zmm30 {k7}, zmm29, zmm28
+0x62,0x02,0x95,0x47,0x83,0xf4
+
+# ATT:   vpmultishiftqb	%zmm28, %zmm29, %zmm30 {%k7} {z}
+# INTEL: vpmultishiftqb	zmm30 {k7} {z}, zmm29, zmm28
+0x62,0x02,0x95,0xc7,0x83,0xf4
+
+# ATT:   vpmultishiftqb	(%rcx), %zmm29, %zmm30
+# INTEL: vpmultishiftqb	zmm30, zmm29, zmmword ptr [rcx]
+0x62,0x62,0x95,0x40,0x83,0x31
+
+# ATT:   vpmultishiftqb	291(%rax,%r14,8), %zmm29, %zmm30
+# INTEL: vpmultishiftqb	zmm30, zmm29, zmmword ptr [rax + 8*r14 + 291]
+0x62,0x22,0x95,0x40,0x83,0xb4,0xf0,0x23,0x01,0x00,0x00
+
+# ATT:   vpmultishiftqb	(%rcx){1to8}, %zmm29, %zmm30
+# INTEL: vpmultishiftqb	zmm30, zmm29, qword ptr [rcx]{1to8}
+0x62,0x62,0x95,0x50,0x83,0x31
+
+# ATT:   vpmultishiftqb	8128(%rdx), %zmm29, %zmm30
+# INTEL: vpmultishiftqb	zmm30, zmm29, zmmword ptr [rdx + 8128]
+0x62,0x62,0x95,0x40,0x83,0x72,0x7f
+
+# ATT:   vpmultishiftqb	8192(%rdx), %zmm29, %zmm30
+# INTEL: vpmultishiftqb	zmm30, zmm29, zmmword ptr [rdx + 8192]
+0x62,0x62,0x95,0x40,0x83,0xb2,0x00,0x20,0x00,0x00
+
+# ATT:   vpmultishiftqb	-8192(%rdx), %zmm29, %zmm30
+# INTEL: vpmultishiftqb	zmm30, zmm29, zmmword ptr [rdx - 8192]
+0x62,0x62,0x95,0x40,0x83,0x72,0x80
+
+# ATT:   vpmultishiftqb	-8256(%rdx), %zmm29, %zmm30
+# INTEL: vpmultishiftqb	zmm30, zmm29, zmmword ptr [rdx - 8256]
+0x62,0x62,0x95,0x40,0x83,0xb2,0xc0,0xdf,0xff,0xff
+
+# ATT:   vpmultishiftqb	1016(%rdx){1to8}, %zmm29, %zmm30
+# INTEL: vpmultishiftqb	zmm30, zmm29, qword ptr [rdx + 1016]{1to8}
+0x62,0x62,0x95,0x50,0x83,0x72,0x7f
+
+# ATT:   vpmultishiftqb	1024(%rdx){1to8}, %zmm29, %zmm30
+# INTEL: vpmultishiftqb	zmm30, zmm29, qword ptr [rdx + 1024]{1to8}
+0x62,0x62,0x95,0x50,0x83,0xb2,0x00,0x04,0x00,0x00
+
+# ATT:   vpmultishiftqb	-1024(%rdx){1to8}, %zmm29, %zmm30
+# INTEL: vpmultishiftqb	zmm30, zmm29, qword ptr [rdx - 1024]{1to8}
+0x62,0x62,0x95,0x50,0x83,0x72,0x80
+
+# ATT:   vpmultishiftqb	-1032(%rdx){1to8}, %zmm29, %zmm30
+# INTEL: vpmultishiftqb	zmm30, zmm29, qword ptr [rdx - 1032]{1to8}
+0x62,0x62,0x95,0x50,0x83,0xb2,0xf8,0xfb,0xff,0xff
+
+# ATT:   vpmultishiftqb	4660(%rax,%r14,8), %zmm29, %zmm30
+# INTEL: vpmultishiftqb	zmm30, zmm29, zmmword ptr [rax + 8*r14 + 4660]
+0x62,0x22,0x95,0x40,0x83,0xb4,0xf0,0x34,0x12,0x00,0x00
diff --git a/llvm/test/MC/Disassembler/X86/avx512vp2intersect-32.txt b/llvm/test/MC/Disassembler/X86/avx512vp2intersect-32.txt
new file mode 100644
index 0000000000000000000000000000000000000000..cc1e3104e6ceed2127dee269de6d4d3b72efdd78
--- /dev/null
+++ b/llvm/test/MC/Disassembler/X86/avx512vp2intersect-32.txt
@@ -0,0 +1,226 @@
+# RUN: llvm-mc -triple i386 -disassemble %s | FileCheck %s --check-prefix=ATT
+# RUN: llvm-mc -triple i386 -disassemble -output-asm-variant=1 %s | FileCheck %s --check-prefix=INTEL
+
+# ATT:   vp2intersectq	%zmm2, %zmm1, %k0
+# INTEL: vp2intersectq	k0, zmm1, zmm2
+0x62,0xf2,0xf7,0x48,0x68,0xc2
+
+# ATT:   vp2intersectq	(%edi), %zmm1, %k0
+# INTEL: vp2intersectq	k0, zmm1, zmmword ptr [edi]
+0x62,0xf2,0xf7,0x48,0x68,0x07
+
+# ATT:   vp2intersectq	(%edi){1to8}, %zmm1, %k0
+# INTEL: vp2intersectq	k0, zmm1, qword ptr [edi]{1to8}
+0x62,0xf2,0xf7,0x58,0x68,0x07
+
+# ATT:   vp2intersectq	%zmm2, %zmm1, %k0
+# INTEL: vp2intersectq	k0, zmm1, zmm2
+0x62,0xf2,0xf7,0x48,0x68,0xc2
+
+# ATT:   vp2intersectq	(%edi), %zmm1, %k0
+# INTEL: vp2intersectq	k0, zmm1, zmmword ptr [edi]
+0x62,0xf2,0xf7,0x48,0x68,0x07
+
+# ATT:   vp2intersectq	(%edi){1to8}, %zmm1, %k0
+# INTEL: vp2intersectq	k0, zmm1, qword ptr [edi]{1to8}
+0x62,0xf2,0xf7,0x58,0x68,0x07
+
+# ATT:   vp2intersectq	%zmm7, %zmm4, %k6
+# INTEL: vp2intersectq	k6, zmm4, zmm7
+0x62,0xf2,0xdf,0x48,0x68,0xf7
+
+# ATT:   vp2intersectq	(%esi), %zmm4, %k6
+# INTEL: vp2intersectq	k6, zmm4, zmmword ptr [esi]
+0x62,0xf2,0xdf,0x48,0x68,0x36
+
+# ATT:   vp2intersectq	(%esi){1to8}, %zmm4, %k6
+# INTEL: vp2intersectq	k6, zmm4, qword ptr [esi]{1to8}
+0x62,0xf2,0xdf,0x58,0x68,0x36
+
+# ATT:   vp2intersectq	%zmm7, %zmm4, %k6
+# INTEL: vp2intersectq	k6, zmm4, zmm7
+0x62,0xf2,0xdf,0x48,0x68,0xf7
+
+# ATT:   vp2intersectq	(%esi), %zmm4, %k6
+# INTEL: vp2intersectq	k6, zmm4, zmmword ptr [esi]
+0x62,0xf2,0xdf,0x48,0x68,0x36
+
+# ATT:   vp2intersectq	(%esi){1to8}, %zmm4, %k6
+# INTEL: vp2intersectq	k6, zmm4, qword ptr [esi]{1to8}
+0x62,0xf2,0xdf,0x58,0x68,0x36
+
+# ATT:   vp2intersectq	%ymm2, %ymm1, %k0
+# INTEL: vp2intersectq	k0, ymm1, ymm2
+0x62,0xf2,0xf7,0x28,0x68,0xc2
+
+# ATT:   vp2intersectq	(%edi), %ymm1, %k0
+# INTEL: vp2intersectq	k0, ymm1, ymmword ptr [edi]
+0x62,0xf2,0xf7,0x28,0x68,0x07
+
+# ATT:   vp2intersectq	(%edi){1to4}, %ymm1, %k0
+# INTEL: vp2intersectq	k0, ymm1, qword ptr [edi]{1to4}
+0x62,0xf2,0xf7,0x38,0x68,0x07
+
+# ATT:   vp2intersectq	%ymm2, %ymm1, %k0
+# INTEL: vp2intersectq	k0, ymm1, ymm2
+0x62,0xf2,0xf7,0x28,0x68,0xc2
+
+# ATT:   vp2intersectq	(%edi), %ymm1, %k0
+# INTEL: vp2intersectq	k0, ymm1, ymmword ptr [edi]
+0x62,0xf2,0xf7,0x28,0x68,0x07
+
+# ATT:   vp2intersectq	(%edi){1to4}, %ymm1, %k0
+# INTEL: vp2intersectq	k0, ymm1, qword ptr [edi]{1to4}
+0x62,0xf2,0xf7,0x38,0x68,0x07
+
+# ATT:   vp2intersectq	%ymm7, %ymm4, %k6
+# INTEL: vp2intersectq	k6, ymm4, ymm7
+0x62,0xf2,0xdf,0x28,0x68,0xf7
+
+# ATT:   vp2intersectq	(%esi), %ymm4, %k6
+# INTEL: vp2intersectq	k6, ymm4, ymmword ptr [esi]
+0x62,0xf2,0xdf,0x28,0x68,0x36
+
+# ATT:   vp2intersectq	(%esi){1to4}, %ymm4, %k6
+# INTEL: vp2intersectq	k6, ymm4, qword ptr [esi]{1to4}
+0x62,0xf2,0xdf,0x38,0x68,0x36
+
+# ATT:   vp2intersectq	%ymm7, %ymm4, %k6
+# INTEL: vp2intersectq	k6, ymm4, ymm7
+0x62,0xf2,0xdf,0x28,0x68,0xf7
+
+# ATT:   vp2intersectq	(%esi), %ymm4, %k6
+# INTEL: vp2intersectq	k6, ymm4, ymmword ptr [esi]
+0x62,0xf2,0xdf,0x28,0x68,0x36
+
+# ATT:   vp2intersectq	%xmm2, %xmm1, %k0
+# INTEL: vp2intersectq	k0, xmm1, xmm2
+0x62,0xf2,0xf7,0x08,0x68,0xc2
+
+# ATT:   vp2intersectq	(%edi), %xmm1, %k0
+# INTEL: vp2intersectq	k0, xmm1, xmmword ptr [edi]
+0x62,0xf2,0xf7,0x08,0x68,0x07
+
+# ATT:   vp2intersectq	(%edi){1to2}, %xmm1, %k0
+# INTEL: vp2intersectq	k0, xmm1, qword ptr [edi]{1to2}
+0x62,0xf2,0xf7,0x18,0x68,0x07
+
+# ATT:   vp2intersectq	%xmm2, %xmm1, %k0
+# INTEL: vp2intersectq	k0, xmm1, xmm2
+0x62,0xf2,0xf7,0x08,0x68,0xc2
+
+# ATT:   vp2intersectq	(%edi), %xmm1, %k0
+# INTEL: vp2intersectq	k0, xmm1, xmmword ptr [edi]
+0x62,0xf2,0xf7,0x08,0x68,0x07
+
+# ATT:   vp2intersectq	%xmm7, %xmm4, %k6
+# INTEL: vp2intersectq	k6, xmm4, xmm7
+0x62,0xf2,0xdf,0x08,0x68,0xf7
+
+# ATT:   vp2intersectq	(%esi), %xmm4, %k6
+# INTEL: vp2intersectq	k6, xmm4, xmmword ptr [esi]
+0x62,0xf2,0xdf,0x08,0x68,0x36
+
+# ATT:   vp2intersectq	%xmm7, %xmm4, %k6
+# INTEL: vp2intersectq	k6, xmm4, xmm7
+0x62,0xf2,0xdf,0x08,0x68,0xf7
+
+# ATT:   vp2intersectq	(%esi), %xmm4, %k6
+# INTEL: vp2intersectq	k6, xmm4, xmmword ptr [esi]
+0x62,0xf2,0xdf,0x08,0x68,0x36
+
+# ATT:   vp2intersectd	%zmm2, %zmm1, %k0
+# INTEL: vp2intersectd	k0, zmm1, zmm2
+0x62,0xf2,0x77,0x48,0x68,0xc2
+
+# ATT:   vp2intersectd	(%edi), %zmm1, %k0
+# INTEL: vp2intersectd	k0, zmm1, zmmword ptr [edi]
+0x62,0xf2,0x77,0x48,0x68,0x07
+
+# ATT:   vp2intersectd	%zmm2, %zmm1, %k0
+# INTEL: vp2intersectd	k0, zmm1, zmm2
+0x62,0xf2,0x77,0x48,0x68,0xc2
+
+# ATT:   vp2intersectd	(%edi), %zmm1, %k0
+# INTEL: vp2intersectd	k0, zmm1, zmmword ptr [edi]
+0x62,0xf2,0x77,0x48,0x68,0x07
+
+# ATT:   vp2intersectd	%zmm7, %zmm4, %k6
+# INTEL: vp2intersectd	k6, zmm4, zmm7
+0x62,0xf2,0x5f,0x48,0x68,0xf7
+
+# ATT:   vp2intersectd	(%esi), %zmm4, %k6
+# INTEL: vp2intersectd	k6, zmm4, zmmword ptr [esi]
+0x62,0xf2,0x5f,0x48,0x68,0x36
+
+# ATT:   vp2intersectd	%zmm7, %zmm4, %k6
+# INTEL: vp2intersectd	k6, zmm4, zmm7
+0x62,0xf2,0x5f,0x48,0x68,0xf7
+
+# ATT:   vp2intersectd	(%esi), %zmm4, %k6
+# INTEL: vp2intersectd	k6, zmm4, zmmword ptr [esi]
+0x62,0xf2,0x5f,0x48,0x68,0x36
+
+# ATT:   vp2intersectd	%ymm2, %ymm1, %k0
+# INTEL: vp2intersectd	k0, ymm1, ymm2
+0x62,0xf2,0x77,0x28,0x68,0xc2
+
+# ATT:   vp2intersectd	(%edi), %ymm1, %k0
+# INTEL: vp2intersectd	k0, ymm1, ymmword ptr [edi]
+0x62,0xf2,0x77,0x28,0x68,0x07
+
+# ATT:   vp2intersectd	%ymm2, %ymm1, %k0
+# INTEL: vp2intersectd	k0, ymm1, ymm2
+0x62,0xf2,0x77,0x28,0x68,0xc2
+
+# ATT:   vp2intersectd	(%edi), %ymm1, %k0
+# INTEL: vp2intersectd	k0, ymm1, ymmword ptr [edi]
+0x62,0xf2,0x77,0x28,0x68,0x07
+
+# ATT:   vp2intersectd	%ymm7, %ymm4, %k6
+# INTEL: vp2intersectd	k6, ymm4, ymm7
+0x62,0xf2,0x5f,0x28,0x68,0xf7
+
+# ATT:   vp2intersectd	(%esi), %ymm4, %k6
+# INTEL: vp2intersectd	k6, ymm4, ymmword ptr [esi]
+0x62,0xf2,0x5f,0x28,0x68,0x36
+
+# ATT:   vp2intersectd	%ymm7, %ymm4, %k6
+# INTEL: vp2intersectd	k6, ymm4, ymm7
+0x62,0xf2,0x5f,0x28,0x68,0xf7
+
+# ATT:   vp2intersectd	(%esi), %ymm4, %k6
+# INTEL: vp2intersectd	k6, ymm4, ymmword ptr [esi]
+0x62,0xf2,0x5f,0x28,0x68,0x36
+
+# ATT:   vp2intersectd	%xmm2, %xmm1, %k0
+# INTEL: vp2intersectd	k0, xmm1, xmm2
+0x62,0xf2,0x77,0x08,0x68,0xc2
+
+# ATT:   vp2intersectd	(%edi), %xmm1, %k0
+# INTEL: vp2intersectd	k0, xmm1, xmmword ptr [edi]
+0x62,0xf2,0x77,0x08,0x68,0x07
+
+# ATT:   vp2intersectd	%xmm2, %xmm1, %k0
+# INTEL: vp2intersectd	k0, xmm1, xmm2
+0x62,0xf2,0x77,0x08,0x68,0xc2
+
+# ATT:   vp2intersectd	(%edi), %xmm1, %k0
+# INTEL: vp2intersectd	k0, xmm1, xmmword ptr [edi]
+0x62,0xf2,0x77,0x08,0x68,0x07
+
+# ATT:   vp2intersectd	%xmm7, %xmm4, %k6
+# INTEL: vp2intersectd	k6, xmm4, xmm7
+0x62,0xf2,0x5f,0x08,0x68,0xf7
+
+# ATT:   vp2intersectd	(%esi), %xmm4, %k6
+# INTEL: vp2intersectd	k6, xmm4, xmmword ptr [esi]
+0x62,0xf2,0x5f,0x08,0x68,0x36
+
+# ATT:   vp2intersectd	%xmm7, %xmm4, %k6
+# INTEL: vp2intersectd	k6, xmm4, xmm7
+0x62,0xf2,0x5f,0x08,0x68,0xf7
+
+# ATT:   vp2intersectd	(%esi), %xmm4, %k6
+# INTEL: vp2intersectd	k6, xmm4, xmmword ptr [esi]
+0x62,0xf2,0x5f,0x08,0x68,0x36
diff --git a/llvm/test/MC/Disassembler/X86/avx512vp2intersect-64.txt b/llvm/test/MC/Disassembler/X86/avx512vp2intersect-64.txt
new file mode 100644
index 0000000000000000000000000000000000000000..e528140586ef8520ff76a96113078ebec4124f0b
--- /dev/null
+++ b/llvm/test/MC/Disassembler/X86/avx512vp2intersect-64.txt
@@ -0,0 +1,226 @@
+# RUN: llvm-mc -triple x86_64 -disassemble %s | FileCheck %s --check-prefix=ATT
+# RUN: llvm-mc -triple x86_64 -disassemble -output-asm-variant=1 %s | FileCheck %s --check-prefix=INTEL
+
+# ATT:   vp2intersectq	%zmm2, %zmm1, %k0
+# INTEL: vp2intersectq	k0, zmm1, zmm2
+0x62,0xf2,0xf7,0x48,0x68,0xc2
+
+# ATT:   vp2intersectq	(%rdi), %zmm1, %k0
+# INTEL: vp2intersectq	k0, zmm1, zmmword ptr [rdi]
+0x62,0xf2,0xf7,0x48,0x68,0x07
+
+# ATT:   vp2intersectq	(%rdi){1to8}, %zmm1, %k0
+# INTEL: vp2intersectq	k0, zmm1, qword ptr [rdi]{1to8}
+0x62,0xf2,0xf7,0x58,0x68,0x07
+
+# ATT:   vp2intersectq	%zmm2, %zmm1, %k0
+# INTEL: vp2intersectq	k0, zmm1, zmm2
+0x62,0xf2,0xf7,0x48,0x68,0xc2
+
+# ATT:   vp2intersectq	(%rdi), %zmm1, %k0
+# INTEL: vp2intersectq	k0, zmm1, zmmword ptr [rdi]
+0x62,0xf2,0xf7,0x48,0x68,0x07
+
+# ATT:   vp2intersectq	(%rdi){1to8}, %zmm1, %k0
+# INTEL: vp2intersectq	k0, zmm1, qword ptr [rdi]{1to8}
+0x62,0xf2,0xf7,0x58,0x68,0x07
+
+# ATT:   vp2intersectq	%zmm7, %zmm9, %k6
+# INTEL: vp2intersectq	k6, zmm9, zmm7
+0x62,0xf2,0xb7,0x48,0x68,0xf7
+
+# ATT:   vp2intersectq	(%rsi), %zmm9, %k6
+# INTEL: vp2intersectq	k6, zmm9, zmmword ptr [rsi]
+0x62,0xf2,0xb7,0x48,0x68,0x36
+
+# ATT:   vp2intersectq	(%rsi){1to8}, %zmm9, %k6
+# INTEL: vp2intersectq	k6, zmm9, qword ptr [rsi]{1to8}
+0x62,0xf2,0xb7,0x58,0x68,0x36
+
+# ATT:   vp2intersectq	%zmm7, %zmm9, %k6
+# INTEL: vp2intersectq	k6, zmm9, zmm7
+0x62,0xf2,0xb7,0x48,0x68,0xf7
+
+# ATT:   vp2intersectq	(%rsi), %zmm9, %k6
+# INTEL: vp2intersectq	k6, zmm9, zmmword ptr [rsi]
+0x62,0xf2,0xb7,0x48,0x68,0x36
+
+# ATT:   vp2intersectq	(%rsi){1to8}, %zmm9, %k6
+# INTEL: vp2intersectq	k6, zmm9, qword ptr [rsi]{1to8}
+0x62,0xf2,0xb7,0x58,0x68,0x36
+
+# ATT:   vp2intersectq	%ymm2, %ymm1, %k0
+# INTEL: vp2intersectq	k0, ymm1, ymm2
+0x62,0xf2,0xf7,0x28,0x68,0xc2
+
+# ATT:   vp2intersectq	(%rdi), %ymm1, %k0
+# INTEL: vp2intersectq	k0, ymm1, ymmword ptr [rdi]
+0x62,0xf2,0xf7,0x28,0x68,0x07
+
+# ATT:   vp2intersectq	(%rdi){1to4}, %ymm1, %k0
+# INTEL: vp2intersectq	k0, ymm1, qword ptr [rdi]{1to4}
+0x62,0xf2,0xf7,0x38,0x68,0x07
+
+# ATT:   vp2intersectq	%ymm2, %ymm1, %k0
+# INTEL: vp2intersectq	k0, ymm1, ymm2
+0x62,0xf2,0xf7,0x28,0x68,0xc2
+
+# ATT:   vp2intersectq	(%rdi), %ymm1, %k0
+# INTEL: vp2intersectq	k0, ymm1, ymmword ptr [rdi]
+0x62,0xf2,0xf7,0x28,0x68,0x07
+
+# ATT:   vp2intersectq	(%rdi){1to4}, %ymm1, %k0
+# INTEL: vp2intersectq	k0, ymm1, qword ptr [rdi]{1to4}
+0x62,0xf2,0xf7,0x38,0x68,0x07
+
+# ATT:   vp2intersectq	%ymm7, %ymm9, %k6
+# INTEL: vp2intersectq	k6, ymm9, ymm7
+0x62,0xf2,0xb7,0x28,0x68,0xf7
+
+# ATT:   vp2intersectq	(%rsi), %ymm9, %k6
+# INTEL: vp2intersectq	k6, ymm9, ymmword ptr [rsi]
+0x62,0xf2,0xb7,0x28,0x68,0x36
+
+# ATT:   vp2intersectq	(%rsi){1to4}, %ymm9, %k6
+# INTEL: vp2intersectq	k6, ymm9, qword ptr [rsi]{1to4}
+0x62,0xf2,0xb7,0x38,0x68,0x36
+
+# ATT:   vp2intersectq	%ymm7, %ymm9, %k6
+# INTEL: vp2intersectq	k6, ymm9, ymm7
+0x62,0xf2,0xb7,0x28,0x68,0xf7
+
+# ATT:   vp2intersectq	(%rsi), %ymm9, %k6
+# INTEL: vp2intersectq	k6, ymm9, ymmword ptr [rsi]
+0x62,0xf2,0xb7,0x28,0x68,0x36
+
+# ATT:   vp2intersectq	%xmm2, %xmm1, %k0
+# INTEL: vp2intersectq	k0, xmm1, xmm2
+0x62,0xf2,0xf7,0x08,0x68,0xc2
+
+# ATT:   vp2intersectq	(%rdi), %xmm1, %k0
+# INTEL: vp2intersectq	k0, xmm1, xmmword ptr [rdi]
+0x62,0xf2,0xf7,0x08,0x68,0x07
+
+# ATT:   vp2intersectq	(%rdi){1to2}, %xmm1, %k0
+# INTEL: vp2intersectq	k0, xmm1, qword ptr [rdi]{1to2}
+0x62,0xf2,0xf7,0x18,0x68,0x07
+
+# ATT:   vp2intersectq	%xmm2, %xmm1, %k0
+# INTEL: vp2intersectq	k0, xmm1, xmm2
+0x62,0xf2,0xf7,0x08,0x68,0xc2
+
+# ATT:   vp2intersectq	(%rdi), %xmm1, %k0
+# INTEL: vp2intersectq	k0, xmm1, xmmword ptr [rdi]
+0x62,0xf2,0xf7,0x08,0x68,0x07
+
+# ATT:   vp2intersectq	%xmm7, %xmm9, %k6
+# INTEL: vp2intersectq	k6, xmm9, xmm7
+0x62,0xf2,0xb7,0x08,0x68,0xf7
+
+# ATT:   vp2intersectq	(%rsi), %xmm9, %k6
+# INTEL: vp2intersectq	k6, xmm9, xmmword ptr [rsi]
+0x62,0xf2,0xb7,0x08,0x68,0x36
+
+# ATT:   vp2intersectq	%xmm7, %xmm9, %k6
+# INTEL: vp2intersectq	k6, xmm9, xmm7
+0x62,0xf2,0xb7,0x08,0x68,0xf7
+
+# ATT:   vp2intersectq	(%rsi), %xmm9, %k6
+# INTEL: vp2intersectq	k6, xmm9, xmmword ptr [rsi]
+0x62,0xf2,0xb7,0x08,0x68,0x36
+
+# ATT:   vp2intersectd	%zmm2, %zmm1, %k0
+# INTEL: vp2intersectd	k0, zmm1, zmm2
+0x62,0xf2,0x77,0x48,0x68,0xc2
+
+# ATT:   vp2intersectd	(%rdi), %zmm1, %k0
+# INTEL: vp2intersectd	k0, zmm1, zmmword ptr [rdi]
+0x62,0xf2,0x77,0x48,0x68,0x07
+
+# ATT:   vp2intersectd	%zmm2, %zmm1, %k0
+# INTEL: vp2intersectd	k0, zmm1, zmm2
+0x62,0xf2,0x77,0x48,0x68,0xc2
+
+# ATT:   vp2intersectd	(%rdi), %zmm1, %k0
+# INTEL: vp2intersectd	k0, zmm1, zmmword ptr [rdi]
+0x62,0xf2,0x77,0x48,0x68,0x07
+
+# ATT:   vp2intersectd	%zmm7, %zmm9, %k6
+# INTEL: vp2intersectd	k6, zmm9, zmm7
+0x62,0xf2,0x37,0x48,0x68,0xf7
+
+# ATT:   vp2intersectd	(%rsi), %zmm9, %k6
+# INTEL: vp2intersectd	k6, zmm9, zmmword ptr [rsi]
+0x62,0xf2,0x37,0x48,0x68,0x36
+
+# ATT:   vp2intersectd	%zmm7, %zmm9, %k6
+# INTEL: vp2intersectd	k6, zmm9, zmm7
+0x62,0xf2,0x37,0x48,0x68,0xf7
+
+# ATT:   vp2intersectd	(%rsi), %zmm9, %k6
+# INTEL: vp2intersectd	k6, zmm9, zmmword ptr [rsi]
+0x62,0xf2,0x37,0x48,0x68,0x36
+
+# ATT:   vp2intersectd	%ymm2, %ymm1, %k0
+# INTEL: vp2intersectd	k0, ymm1, ymm2
+0x62,0xf2,0x77,0x28,0x68,0xc2
+
+# ATT:   vp2intersectd	(%rdi), %ymm1, %k0
+# INTEL: vp2intersectd	k0, ymm1, ymmword ptr [rdi]
+0x62,0xf2,0x77,0x28,0x68,0x07
+
+# ATT:   vp2intersectd	%ymm2, %ymm1, %k0
+# INTEL: vp2intersectd	k0, ymm1, ymm2
+0x62,0xf2,0x77,0x28,0x68,0xc2
+
+# ATT:   vp2intersectd	(%rdi), %ymm1, %k0
+# INTEL: vp2intersectd	k0, ymm1, ymmword ptr [rdi]
+0x62,0xf2,0x77,0x28,0x68,0x07
+
+# ATT:   vp2intersectd	%ymm7, %ymm9, %k6
+# INTEL: vp2intersectd	k6, ymm9, ymm7
+0x62,0xf2,0x37,0x28,0x68,0xf7
+
+# ATT:   vp2intersectd	(%rsi), %ymm9, %k6
+# INTEL: vp2intersectd	k6, ymm9, ymmword ptr [rsi]
+0x62,0xf2,0x37,0x28,0x68,0x36
+
+# ATT:   vp2intersectd	%ymm7, %ymm9, %k6
+# INTEL: vp2intersectd	k6, ymm9, ymm7
+0x62,0xf2,0x37,0x28,0x68,0xf7
+
+# ATT:   vp2intersectd	(%rsi), %ymm9, %k6
+# INTEL: vp2intersectd	k6, ymm9, ymmword ptr [rsi]
+0x62,0xf2,0x37,0x28,0x68,0x36
+
+# ATT:   vp2intersectd	%xmm2, %xmm1, %k0
+# INTEL: vp2intersectd	k0, xmm1, xmm2
+0x62,0xf2,0x77,0x08,0x68,0xc2
+
+# ATT:   vp2intersectd	(%rdi), %xmm1, %k0
+# INTEL: vp2intersectd	k0, xmm1, xmmword ptr [rdi]
+0x62,0xf2,0x77,0x08,0x68,0x07
+
+# ATT:   vp2intersectd	%xmm2, %xmm1, %k0
+# INTEL: vp2intersectd	k0, xmm1, xmm2
+0x62,0xf2,0x77,0x08,0x68,0xc2
+
+# ATT:   vp2intersectd	(%rdi), %xmm1, %k0
+# INTEL: vp2intersectd	k0, xmm1, xmmword ptr [rdi]
+0x62,0xf2,0x77,0x08,0x68,0x07
+
+# ATT:   vp2intersectd	%xmm7, %xmm9, %k6
+# INTEL: vp2intersectd	k6, xmm9, xmm7
+0x62,0xf2,0x37,0x08,0x68,0xf7
+
+# ATT:   vp2intersectd	(%rsi), %xmm9, %k6
+# INTEL: vp2intersectd	k6, xmm9, xmmword ptr [rsi]
+0x62,0xf2,0x37,0x08,0x68,0x36
+
+# ATT:   vp2intersectd	%xmm7, %xmm9, %k6
+# INTEL: vp2intersectd	k6, xmm9, xmm7
+0x62,0xf2,0x37,0x08,0x68,0xf7
+
+# ATT:   vp2intersectd	(%rsi), %xmm9, %k6
+# INTEL: vp2intersectd	k6, xmm9, xmmword ptr [rsi]
+0x62,0xf2,0x37,0x08,0x68,0x36
diff --git a/llvm/test/MC/Disassembler/X86/avx512vp2intersectvl-32.txt b/llvm/test/MC/Disassembler/X86/avx512vp2intersectvl-32.txt
new file mode 100644
index 0000000000000000000000000000000000000000..814a81ad729b167efff2db7643ec9ffb1d3f8a24
--- /dev/null
+++ b/llvm/test/MC/Disassembler/X86/avx512vp2intersectvl-32.txt
@@ -0,0 +1,114 @@
+# RUN: llvm-mc -triple i386 -disassemble %s | FileCheck %s --check-prefix=ATT
+# RUN: llvm-mc -triple i386 -disassemble -output-asm-variant=1 %s | FileCheck %s --check-prefix=INTEL
+
+# ATT:   vp2intersectd	%ymm4, %ymm3, %k6
+# INTEL: vp2intersectd	k6, ymm3, ymm4
+0x62,0xf2,0x67,0x28,0x68,0xf4
+
+# ATT:   vp2intersectd	%xmm4, %xmm3, %k6
+# INTEL: vp2intersectd	k6, xmm3, xmm4
+0x62,0xf2,0x67,0x08,0x68,0xf4
+
+# ATT:   vp2intersectd	268435456(%esp,%esi,8), %ymm3, %k6
+# INTEL: vp2intersectd	k6, ymm3, ymmword ptr [esp + 8*esi + 268435456]
+0x62,0xf2,0x67,0x28,0x68,0xb4,0xf4,0x00,0x00,0x00,0x10
+
+# ATT:   vp2intersectd	291(%edi,%eax,4), %ymm3, %k6
+# INTEL: vp2intersectd	k6, ymm3, ymmword ptr [edi + 4*eax + 291]
+0x62,0xf2,0x67,0x28,0x68,0xb4,0x87,0x23,0x01,0x00,0x00
+
+# ATT:   vp2intersectd	(%eax){1to8}, %ymm3, %k6
+# INTEL: vp2intersectd	k6, ymm3, dword ptr [eax]{1to8}
+0x62,0xf2,0x67,0x38,0x68,0x30
+
+# ATT:   vp2intersectd	-1024(,%ebp,2), %ymm3, %k6
+# INTEL: vp2intersectd	k6, ymm3, ymmword ptr [2*ebp - 1024]
+0x62,0xf2,0x67,0x28,0x68,0x34,0x6d,0x00,0xfc,0xff,0xff
+
+# ATT:   vp2intersectd	4064(%ecx), %ymm3, %k6
+# INTEL: vp2intersectd	k6, ymm3, ymmword ptr [ecx + 4064]
+0x62,0xf2,0x67,0x28,0x68,0x71,0x7f
+
+# ATT:   vp2intersectd	-512(%edx){1to8}, %ymm3, %k6
+# INTEL: vp2intersectd	k6, ymm3, dword ptr [edx - 512]{1to8}
+0x62,0xf2,0x67,0x38,0x68,0x72,0x80
+
+# ATT:   vp2intersectd	268435456(%esp,%esi,8), %xmm3, %k6
+# INTEL: vp2intersectd	k6, xmm3, xmmword ptr [esp + 8*esi + 268435456]
+0x62,0xf2,0x67,0x08,0x68,0xb4,0xf4,0x00,0x00,0x00,0x10
+
+# ATT:   vp2intersectd	291(%edi,%eax,4), %xmm3, %k6
+# INTEL: vp2intersectd	k6, xmm3, xmmword ptr [edi + 4*eax + 291]
+0x62,0xf2,0x67,0x08,0x68,0xb4,0x87,0x23,0x01,0x00,0x00
+
+# ATT:   vp2intersectd	(%eax){1to4}, %xmm3, %k6
+# INTEL: vp2intersectd	k6, xmm3, dword ptr [eax]{1to4}
+0x62,0xf2,0x67,0x18,0x68,0x30
+
+# ATT:   vp2intersectd	-512(,%ebp,2), %xmm3, %k6
+# INTEL: vp2intersectd	k6, xmm3, xmmword ptr [2*ebp - 512]
+0x62,0xf2,0x67,0x08,0x68,0x34,0x6d,0x00,0xfe,0xff,0xff
+
+# ATT:   vp2intersectd	2032(%ecx), %xmm3, %k6
+# INTEL: vp2intersectd	k6, xmm3, xmmword ptr [ecx + 2032]
+0x62,0xf2,0x67,0x08,0x68,0x71,0x7f
+
+# ATT:   vp2intersectd	-512(%edx){1to4}, %xmm3, %k6
+# INTEL: vp2intersectd	k6, xmm3, dword ptr [edx - 512]{1to4}
+0x62,0xf2,0x67,0x18,0x68,0x72,0x80
+
+# ATT:   vp2intersectq	%ymm4, %ymm3, %k6
+# INTEL: vp2intersectq	k6, ymm3, ymm4
+0x62,0xf2,0xe7,0x28,0x68,0xf4
+
+# ATT:   vp2intersectq	%xmm4, %xmm3, %k6
+# INTEL: vp2intersectq	k6, xmm3, xmm4
+0x62,0xf2,0xe7,0x08,0x68,0xf4
+
+# ATT:   vp2intersectq	268435456(%esp,%esi,8), %ymm3, %k6
+# INTEL: vp2intersectq	k6, ymm3, ymmword ptr [esp + 8*esi + 268435456]
+0x62,0xf2,0xe7,0x28,0x68,0xb4,0xf4,0x00,0x00,0x00,0x10
+
+# ATT:   vp2intersectq	291(%edi,%eax,4), %ymm3, %k6
+# INTEL: vp2intersectq	k6, ymm3, ymmword ptr [edi + 4*eax + 291]
+0x62,0xf2,0xe7,0x28,0x68,0xb4,0x87,0x23,0x01,0x00,0x00
+
+# ATT:   vp2intersectq	(%eax){1to4}, %ymm3, %k6
+# INTEL: vp2intersectq	k6, ymm3, qword ptr [eax]{1to4}
+0x62,0xf2,0xe7,0x38,0x68,0x30
+
+# ATT:   vp2intersectq	-1024(,%ebp,2), %ymm3, %k6
+# INTEL: vp2intersectq	k6, ymm3, ymmword ptr [2*ebp - 1024]
+0x62,0xf2,0xe7,0x28,0x68,0x34,0x6d,0x00,0xfc,0xff,0xff
+
+# ATT:   vp2intersectq	4064(%ecx), %ymm3, %k6
+# INTEL: vp2intersectq	k6, ymm3, ymmword ptr [ecx + 4064]
+0x62,0xf2,0xe7,0x28,0x68,0x71,0x7f
+
+# ATT:   vp2intersectq	-1024(%edx){1to4}, %ymm3, %k6
+# INTEL: vp2intersectq	k6, ymm3, qword ptr [edx - 1024]{1to4}
+0x62,0xf2,0xe7,0x38,0x68,0x72,0x80
+
+# ATT:   vp2intersectq	268435456(%esp,%esi,8), %xmm3, %k6
+# INTEL: vp2intersectq	k6, xmm3, xmmword ptr [esp + 8*esi + 268435456]
+0x62,0xf2,0xe7,0x08,0x68,0xb4,0xf4,0x00,0x00,0x00,0x10
+
+# ATT:   vp2intersectq	291(%edi,%eax,4), %xmm3, %k6
+# INTEL: vp2intersectq	k6, xmm3, xmmword ptr [edi + 4*eax + 291]
+0x62,0xf2,0xe7,0x08,0x68,0xb4,0x87,0x23,0x01,0x00,0x00
+
+# ATT:   vp2intersectq	(%eax){1to2}, %xmm3, %k6
+# INTEL: vp2intersectq	k6, xmm3, qword ptr [eax]{1to2}
+0x62,0xf2,0xe7,0x18,0x68,0x30
+
+# ATT:   vp2intersectq	-512(,%ebp,2), %xmm3, %k6
+# INTEL: vp2intersectq	k6, xmm3, xmmword ptr [2*ebp - 512]
+0x62,0xf2,0xe7,0x08,0x68,0x34,0x6d,0x00,0xfe,0xff,0xff
+
+# ATT:   vp2intersectq	2032(%ecx), %xmm3, %k6
+# INTEL: vp2intersectq	k6, xmm3, xmmword ptr [ecx + 2032]
+0x62,0xf2,0xe7,0x08,0x68,0x71,0x7f
+
+# ATT:   vp2intersectq	-1024(%edx){1to2}, %xmm3, %k6
+# INTEL: vp2intersectq	k6, xmm3, qword ptr [edx - 1024]{1to2}
+0x62,0xf2,0xe7,0x18,0x68,0x72,0x80
diff --git a/llvm/test/MC/Disassembler/X86/avx512vp2intersectvl-64.txt b/llvm/test/MC/Disassembler/X86/avx512vp2intersectvl-64.txt
new file mode 100644
index 0000000000000000000000000000000000000000..2d4cbeb94b7cda9414abfc2664dad2f6709246d6
--- /dev/null
+++ b/llvm/test/MC/Disassembler/X86/avx512vp2intersectvl-64.txt
@@ -0,0 +1,114 @@
+# RUN: llvm-mc -triple x86_64 -disassemble %s | FileCheck %s --check-prefix=ATT
+# RUN: llvm-mc -triple x86_64 -disassemble -output-asm-variant=1 %s | FileCheck %s --check-prefix=INTEL
+
+# ATT:   vp2intersectd	%ymm24, %ymm23, %k6
+# INTEL: vp2intersectd	k6, ymm23, ymm24
+0x62,0x92,0x47,0x20,0x68,0xf0
+
+# ATT:   vp2intersectd	%xmm24, %xmm23, %k6
+# INTEL: vp2intersectd	k6, xmm23, xmm24
+0x62,0x92,0x47,0x00,0x68,0xf0
+
+# ATT:   vp2intersectd	268435456(%rbp,%r14,8), %ymm23, %k6
+# INTEL: vp2intersectd	k6, ymm23, ymmword ptr [rbp + 8*r14 + 268435456]
+0x62,0xb2,0x47,0x20,0x68,0xb4,0xf5,0x00,0x00,0x00,0x10
+
+# ATT:   vp2intersectd	291(%r8,%rax,4), %ymm23, %k6
+# INTEL: vp2intersectd	k6, ymm23, ymmword ptr [r8 + 4*rax + 291]
+0x62,0xd2,0x47,0x20,0x68,0xb4,0x80,0x23,0x01,0x00,0x00
+
+# ATT:   vp2intersectd	(%rip){1to8}, %ymm23, %k6
+# INTEL: vp2intersectd	k6, ymm23, dword ptr [rip]{1to8}
+0x62,0xf2,0x47,0x30,0x68,0x35,0x00,0x00,0x00,0x00
+
+# ATT:   vp2intersectd	-1024(,%rbp,2), %ymm23, %k6
+# INTEL: vp2intersectd	k6, ymm23, ymmword ptr [2*rbp - 1024]
+0x62,0xf2,0x47,0x20,0x68,0x34,0x6d,0x00,0xfc,0xff,0xff
+
+# ATT:   vp2intersectd	4064(%rcx), %ymm23, %k6
+# INTEL: vp2intersectd	k6, ymm23, ymmword ptr [rcx + 4064]
+0x62,0xf2,0x47,0x20,0x68,0x71,0x7f
+
+# ATT:   vp2intersectd	-512(%rdx){1to8}, %ymm23, %k6
+# INTEL: vp2intersectd	k6, ymm23, dword ptr [rdx - 512]{1to8}
+0x62,0xf2,0x47,0x30,0x68,0x72,0x80
+
+# ATT:   vp2intersectd	268435456(%rbp,%r14,8), %xmm23, %k6
+# INTEL: vp2intersectd	k6, xmm23, xmmword ptr [rbp + 8*r14 + 268435456]
+0x62,0xb2,0x47,0x00,0x68,0xb4,0xf5,0x00,0x00,0x00,0x10
+
+# ATT:   vp2intersectd	291(%r8,%rax,4), %xmm23, %k6
+# INTEL: vp2intersectd	k6, xmm23, xmmword ptr [r8 + 4*rax + 291]
+0x62,0xd2,0x47,0x00,0x68,0xb4,0x80,0x23,0x01,0x00,0x00
+
+# ATT:   vp2intersectd	(%rip){1to4}, %xmm23, %k6
+# INTEL: vp2intersectd	k6, xmm23, dword ptr [rip]{1to4}
+0x62,0xf2,0x47,0x10,0x68,0x35,0x00,0x00,0x00,0x00
+
+# ATT:   vp2intersectd	-512(,%rbp,2), %xmm23, %k6
+# INTEL: vp2intersectd	k6, xmm23, xmmword ptr [2*rbp - 512]
+0x62,0xf2,0x47,0x00,0x68,0x34,0x6d,0x00,0xfe,0xff,0xff
+
+# ATT:   vp2intersectd	2032(%rcx), %xmm23, %k6
+# INTEL: vp2intersectd	k6, xmm23, xmmword ptr [rcx + 2032]
+0x62,0xf2,0x47,0x00,0x68,0x71,0x7f
+
+# ATT:   vp2intersectd	-512(%rdx){1to4}, %xmm23, %k6
+# INTEL: vp2intersectd	k6, xmm23, dword ptr [rdx - 512]{1to4}
+0x62,0xf2,0x47,0x10,0x68,0x72,0x80
+
+# ATT:   vp2intersectq	%ymm24, %ymm23, %k6
+# INTEL: vp2intersectq	k6, ymm23, ymm24
+0x62,0x92,0xc7,0x20,0x68,0xf0
+
+# ATT:   vp2intersectq	%xmm24, %xmm23, %k6
+# INTEL: vp2intersectq	k6, xmm23, xmm24
+0x62,0x92,0xc7,0x00,0x68,0xf0
+
+# ATT:   vp2intersectq	268435456(%rbp,%r14,8), %ymm23, %k6
+# INTEL: vp2intersectq	k6, ymm23, ymmword ptr [rbp + 8*r14 + 268435456]
+0x62,0xb2,0xc7,0x20,0x68,0xb4,0xf5,0x00,0x00,0x00,0x10
+
+# ATT:   vp2intersectq	291(%r8,%rax,4), %ymm23, %k6
+# INTEL: vp2intersectq	k6, ymm23, ymmword ptr [r8 + 4*rax + 291]
+0x62,0xd2,0xc7,0x20,0x68,0xb4,0x80,0x23,0x01,0x00,0x00
+
+# ATT:   vp2intersectq	(%rip){1to4}, %ymm23, %k6
+# INTEL: vp2intersectq	k6, ymm23, qword ptr [rip]{1to4}
+0x62,0xf2,0xc7,0x30,0x68,0x35,0x00,0x00,0x00,0x00
+
+# ATT:   vp2intersectq	-1024(,%rbp,2), %ymm23, %k6
+# INTEL: vp2intersectq	k6, ymm23, ymmword ptr [2*rbp - 1024]
+0x62,0xf2,0xc7,0x20,0x68,0x34,0x6d,0x00,0xfc,0xff,0xff
+
+# ATT:   vp2intersectq	4064(%rcx), %ymm23, %k6
+# INTEL: vp2intersectq	k6, ymm23, ymmword ptr [rcx + 4064]
+0x62,0xf2,0xc7,0x20,0x68,0x71,0x7f
+
+# ATT:   vp2intersectq	-1024(%rdx){1to4}, %ymm23, %k6
+# INTEL: vp2intersectq	k6, ymm23, qword ptr [rdx - 1024]{1to4}
+0x62,0xf2,0xc7,0x30,0x68,0x72,0x80
+
+# ATT:   vp2intersectq	268435456(%rbp,%r14,8), %xmm23, %k6
+# INTEL: vp2intersectq	k6, xmm23, xmmword ptr [rbp + 8*r14 + 268435456]
+0x62,0xb2,0xc7,0x00,0x68,0xb4,0xf5,0x00,0x00,0x00,0x10
+
+# ATT:   vp2intersectq	291(%r8,%rax,4), %xmm23, %k6
+# INTEL: vp2intersectq	k6, xmm23, xmmword ptr [r8 + 4*rax + 291]
+0x62,0xd2,0xc7,0x00,0x68,0xb4,0x80,0x23,0x01,0x00,0x00
+
+# ATT:   vp2intersectq	(%rip){1to2}, %xmm23, %k6
+# INTEL: vp2intersectq	k6, xmm23, qword ptr [rip]{1to2}
+0x62,0xf2,0xc7,0x10,0x68,0x35,0x00,0x00,0x00,0x00
+
+# ATT:   vp2intersectq	-512(,%rbp,2), %xmm23, %k6
+# INTEL: vp2intersectq	k6, xmm23, xmmword ptr [2*rbp - 512]
+0x62,0xf2,0xc7,0x00,0x68,0x34,0x6d,0x00,0xfe,0xff,0xff
+
+# ATT:   vp2intersectq	2032(%rcx), %xmm23, %k6
+# INTEL: vp2intersectq	k6, xmm23, xmmword ptr [rcx + 2032]
+0x62,0xf2,0xc7,0x00,0x68,0x71,0x7f
+
+# ATT:   vp2intersectq	-1024(%rdx){1to2}, %xmm23, %k6
+# INTEL: vp2intersectq	k6, xmm23, qword ptr [rdx - 1024]{1to2}
+0x62,0xf2,0xc7,0x10,0x68,0x72,0x80
diff --git a/llvm/test/MC/Disassembler/X86/avx512vp2intersectvl-att.txt b/llvm/test/MC/Disassembler/X86/avx512vp2intersectvl-att.txt
deleted file mode 100644
index 9cb68e659c5231de50b05b207f5fc727f9a60a33..0000000000000000000000000000000000000000
--- a/llvm/test/MC/Disassembler/X86/avx512vp2intersectvl-att.txt
+++ /dev/null
@@ -1,86 +0,0 @@
-# RUN: llvm-mc --disassemble %s -triple=i686-apple-darwin9 | FileCheck %s
-
-# CHECK: vp2intersectd %ymm4, %ymm3, %k6
-0x62,0xf2,0x67,0x28,0x68,0xf4
-
-# CHECK: vp2intersectd %xmm4, %xmm3, %k6
-0x62,0xf2,0x67,0x08,0x68,0xf4
-
-# CHECK: vp2intersectd  268435456(%esp,%esi,8), %ymm3, %k6
-0x62,0xf2,0x67,0x28,0x68,0xb4,0xf4,0x00,0x00,0x00,0x10
-
-# CHECK: vp2intersectd  291(%edi,%eax,4), %ymm3, %k6
-0x62,0xf2,0x67,0x28,0x68,0xb4,0x87,0x23,0x01,0x00,0x00
-
-# CHECK: vp2intersectd  (%eax){1to8}, %ymm3, %k6
-0x62,0xf2,0x67,0x38,0x68,0x30
-
-# CHECK: vp2intersectd  -1024(,%ebp,2), %ymm3, %k6
-0x62,0xf2,0x67,0x28,0x68,0x34,0x6d,0x00,0xfc,0xff,0xff
-
-# CHECK: vp2intersectd  4064(%ecx), %ymm3, %k6
-0x62,0xf2,0x67,0x28,0x68,0x71,0x7f
-
-# CHECK: vp2intersectd  -512(%edx){1to8}, %ymm3, %k6
-0x62,0xf2,0x67,0x38,0x68,0x72,0x80
-
-# CHECK: vp2intersectd  268435456(%esp,%esi,8), %xmm3, %k6
-0x62,0xf2,0x67,0x08,0x68,0xb4,0xf4,0x00,0x00,0x00,0x10
-
-# CHECK: vp2intersectd  291(%edi,%eax,4), %xmm3, %k6
-0x62,0xf2,0x67,0x08,0x68,0xb4,0x87,0x23,0x01,0x00,0x00
-
-# CHECK: vp2intersectd  (%eax){1to4}, %xmm3, %k6
-0x62,0xf2,0x67,0x18,0x68,0x30
-
-# CHECK: vp2intersectd  -512(,%ebp,2), %xmm3, %k6
-0x62,0xf2,0x67,0x08,0x68,0x34,0x6d,0x00,0xfe,0xff,0xff
-
-# CHECK: vp2intersectd  2032(%ecx), %xmm3, %k6
-0x62,0xf2,0x67,0x08,0x68,0x71,0x7f
-
-# CHECK: vp2intersectd  -512(%edx){1to4}, %xmm3, %k6
-0x62,0xf2,0x67,0x18,0x68,0x72,0x80
-
-# CHECK: vp2intersectq %ymm4, %ymm3, %k6
-0x62,0xf2,0xe7,0x28,0x68,0xf4
-
-# CHECK: vp2intersectq %xmm4, %xmm3, %k6
-0x62,0xf2,0xe7,0x08,0x68,0xf4
-
-# CHECK: vp2intersectq  268435456(%esp,%esi,8), %ymm3, %k6
-0x62,0xf2,0xe7,0x28,0x68,0xb4,0xf4,0x00,0x00,0x00,0x10
-
-# CHECK: vp2intersectq  291(%edi,%eax,4), %ymm3, %k6
-0x62,0xf2,0xe7,0x28,0x68,0xb4,0x87,0x23,0x01,0x00,0x00
-
-# CHECK: vp2intersectq  (%eax){1to4}, %ymm3, %k6
-0x62,0xf2,0xe7,0x38,0x68,0x30
-
-# CHECK: vp2intersectq  -1024(,%ebp,2), %ymm3, %k6
-0x62,0xf2,0xe7,0x28,0x68,0x34,0x6d,0x00,0xfc,0xff,0xff
-
-# CHECK: vp2intersectq  4064(%ecx), %ymm3, %k6
-0x62,0xf2,0xe7,0x28,0x68,0x71,0x7f
-
-# CHECK: vp2intersectq  -1024(%edx){1to4}, %ymm3, %k6
-0x62,0xf2,0xe7,0x38,0x68,0x72,0x80
-
-# CHECK: vp2intersectq  268435456(%esp,%esi,8), %xmm3, %k6
-0x62,0xf2,0xe7,0x08,0x68,0xb4,0xf4,0x00,0x00,0x00,0x10
-
-# CHECK: vp2intersectq  291(%edi,%eax,4), %xmm3, %k6
-0x62,0xf2,0xe7,0x08,0x68,0xb4,0x87,0x23,0x01,0x00,0x00
-
-# CHECK: vp2intersectq  (%eax){1to2}, %xmm3, %k6
-0x62,0xf2,0xe7,0x18,0x68,0x30
-
-# CHECK: vp2intersectq  -512(,%ebp,2), %xmm3, %k6
-0x62,0xf2,0xe7,0x08,0x68,0x34,0x6d,0x00,0xfe,0xff,0xff
-
-# CHECK: vp2intersectq  2032(%ecx), %xmm3, %k6
-0x62,0xf2,0xe7,0x08,0x68,0x71,0x7f
-
-# CHECK: vp2intersectq  -1024(%edx){1to2}, %xmm3, %k6
-0x62,0xf2,0xe7,0x18,0x68,0x72,0x80
-
diff --git a/llvm/test/MC/Disassembler/X86/avx512vp2intersectvl-intel.txt b/llvm/test/MC/Disassembler/X86/avx512vp2intersectvl-intel.txt
deleted file mode 100644
index 0c34a6ec016c34eabc5c82e214d837e7b27bde94..0000000000000000000000000000000000000000
--- a/llvm/test/MC/Disassembler/X86/avx512vp2intersectvl-intel.txt
+++ /dev/null
@@ -1,85 +0,0 @@
-# RUN: llvm-mc --disassemble %s -triple=i386 -x86-asm-syntax=intel --output-asm-variant=1 | FileCheck %s
-
-# CHECK: vp2intersectd k6, ymm3, ymm4
-0x62,0xf2,0x67,0x28,0x68,0xf4
-
-# CHECK: vp2intersectd k6, xmm3, xmm4
-0x62,0xf2,0x67,0x08,0x68,0xf4
-
-# CHECK: vp2intersectd k6, ymm3, ymmword ptr [esp + 8*esi + 268435456]
-0x62,0xf2,0x67,0x28,0x68,0xb4,0xf4,0x00,0x00,0x00,0x10
-
-# CHECK: vp2intersectd k6, ymm3, ymmword ptr [edi + 4*eax + 291]
-0x62,0xf2,0x67,0x28,0x68,0xb4,0x87,0x23,0x01,0x00,0x00
-
-# CHECK: vp2intersectd k6, ymm3, dword ptr [eax]{1to8}
-0x62,0xf2,0x67,0x38,0x68,0x30
-
-# CHECK: vp2intersectd k6, ymm3, ymmword ptr [2*ebp - 1024]
-0x62,0xf2,0x67,0x28,0x68,0x34,0x6d,0x00,0xfc,0xff,0xff
-
-# CHECK: vp2intersectd k6, ymm3, ymmword ptr [ecx + 4064]
-0x62,0xf2,0x67,0x28,0x68,0x71,0x7f
-
-# CHECK: vp2intersectd k6, ymm3, dword ptr [edx - 512]{1to8}
-0x62,0xf2,0x67,0x38,0x68,0x72,0x80
-
-# CHECK: vp2intersectd k6, xmm3, xmmword ptr [esp + 8*esi + 268435456]
-0x62,0xf2,0x67,0x08,0x68,0xb4,0xf4,0x00,0x00,0x00,0x10
-
-# CHECK: vp2intersectd k6, xmm3, xmmword ptr [edi + 4*eax + 291]
-0x62,0xf2,0x67,0x08,0x68,0xb4,0x87,0x23,0x01,0x00,0x00
-
-# CHECK: vp2intersectd k6, xmm3, dword ptr [eax]{1to4}
-0x62,0xf2,0x67,0x18,0x68,0x30
-
-# CHECK: vp2intersectd k6, xmm3, xmmword ptr [2*ebp - 512]
-0x62,0xf2,0x67,0x08,0x68,0x34,0x6d,0x00,0xfe,0xff,0xff
-
-# CHECK: vp2intersectd k6, xmm3, xmmword ptr [ecx + 2032]
-0x62,0xf2,0x67,0x08,0x68,0x71,0x7f
-
-# CHECK: vp2intersectd k6, xmm3, dword ptr [edx - 512]{1to4}
-0x62,0xf2,0x67,0x18,0x68,0x72,0x80
-
-# CHECK: vp2intersectq k6, ymm3, ymm4
-0x62,0xf2,0xe7,0x28,0x68,0xf4
-
-# CHECK: vp2intersectq k6, xmm3, xmm4
-0x62,0xf2,0xe7,0x08,0x68,0xf4
-
-# CHECK: vp2intersectq k6, ymm3, ymmword ptr [esp + 8*esi + 268435456]
-0x62,0xf2,0xe7,0x28,0x68,0xb4,0xf4,0x00,0x00,0x00,0x10
-
-# CHECK: vp2intersectq k6, ymm3, ymmword ptr [edi + 4*eax + 291]
-0x62,0xf2,0xe7,0x28,0x68,0xb4,0x87,0x23,0x01,0x00,0x00
-
-# CHECK: vp2intersectq k6, ymm3, qword ptr [eax]{1to4}
-0x62,0xf2,0xe7,0x38,0x68,0x30
-
-# CHECK: vp2intersectq k6, ymm3, ymmword ptr [2*ebp - 1024]
-0x62,0xf2,0xe7,0x28,0x68,0x34,0x6d,0x00,0xfc,0xff,0xff
-
-# CHECK: vp2intersectq k6, ymm3, ymmword ptr [ecx + 4064]
-0x62,0xf2,0xe7,0x28,0x68,0x71,0x7f
-
-# CHECK: vp2intersectq k6, ymm3, qword ptr [edx - 1024]{1to4}
-0x62,0xf2,0xe7,0x38,0x68,0x72,0x80
-
-# CHECK: vp2intersectq k6, xmm3, xmmword ptr [esp + 8*esi + 268435456]
-0x62,0xf2,0xe7,0x08,0x68,0xb4,0xf4,0x00,0x00,0x00,0x10
-
-# CHECK: vp2intersectq k6, xmm3, xmmword ptr [edi + 4*eax + 291]
-0x62,0xf2,0xe7,0x08,0x68,0xb4,0x87,0x23,0x01,0x00,0x00
-
-# CHECK: vp2intersectq k6, xmm3, qword ptr [eax]{1to2}
-0x62,0xf2,0xe7,0x18,0x68,0x30
-
-# CHECK: vp2intersectq k6, xmm3, xmmword ptr [2*ebp - 512]
-0x62,0xf2,0xe7,0x08,0x68,0x34,0x6d,0x00,0xfe,0xff,0xff
-
-# CHECK: vp2intersectq k6, xmm3, xmmword ptr [ecx + 2032]
-0x62,0xf2,0xe7,0x08,0x68,0x71,0x7f
-
-# CHECK: vp2intersectq k6, xmm3, qword ptr [edx - 1024]{1to2}
-0x62,0xf2,0xe7,0x18,0x68,0x72,0x80
diff --git a/llvm/test/MC/Disassembler/X86/x86-64-avx512bf16-att.txt b/llvm/test/MC/Disassembler/X86/x86-64-avx512bf16-att.txt
deleted file mode 100644
index 2b633a9a7cf859f08d3fcbbe59be75c844f1999e..0000000000000000000000000000000000000000
--- a/llvm/test/MC/Disassembler/X86/x86-64-avx512bf16-att.txt
+++ /dev/null
@@ -1,82 +0,0 @@
-# RUN: llvm-mc --disassemble %s -triple=x86_64-apple-darwin9 | FileCheck %s
-
-# CHECK: vcvtne2ps2bf16 %zmm24, %zmm23, %zmm22
-0x62,0x82,0x47,0x40,0x72,0xf0
-
-# CHECK: vcvtne2ps2bf16 %zmm24, %zmm23, %zmm22 {%k7}
-0x62,0x82,0x47,0x47,0x72,0xf0
-
-# CHECK: vcvtne2ps2bf16 %zmm24, %zmm23, %zmm22 {%k7} {z}
-0x62,0x82,0x47,0xc7,0x72,0xf0
-
-# CHECK: vcvtne2ps2bf16  268435456(%rbp,%r14,8), %zmm23, %zmm22
-0x62,0xa2,0x47,0x40,0x72,0xb4,0xf5,0x00,0x00,0x00,0x10
-
-# CHECK: vcvtne2ps2bf16  291(%r8,%rax,4), %zmm23, %zmm22 {%k7}
-0x62,0xc2,0x47,0x47,0x72,0xb4,0x80,0x23,0x01,0x00,0x00
-
-# CHECK: vcvtne2ps2bf16  (%rip){1to16}, %zmm23, %zmm22
-0x62,0xe2,0x47,0x50,0x72,0x35,0x00,0x00,0x00,0x00
-
-# CHECK: vcvtne2ps2bf16  -2048(,%rbp,2), %zmm23, %zmm22
-0x62,0xe2,0x47,0x40,0x72,0x34,0x6d,0x00,0xf8,0xff,0xff
-
-# CHECK: vcvtne2ps2bf16  8128(%rcx), %zmm23, %zmm22 {%k7} {z}
-0x62,0xe2,0x47,0xc7,0x72,0x71,0x7f
-
-# CHECK: vcvtne2ps2bf16  -512(%rdx){1to16}, %zmm23, %zmm22 {%k7} {z}
-0x62,0xe2,0x47,0xd7,0x72,0x72,0x80
-
-# CHECK: vcvtneps2bf16 %zmm23, %ymm22
-0x62,0xa2,0x7e,0x48,0x72,0xf7
-
-# CHECK: vcvtneps2bf16 %zmm23, %ymm22 {%k7}
-0x62,0xa2,0x7e,0x4f,0x72,0xf7
-
-# CHECK: vcvtneps2bf16 %zmm23, %ymm22 {%k7} {z}
-0x62,0xa2,0x7e,0xcf,0x72,0xf7
-
-# CHECK: vcvtneps2bf16  268435456(%rbp,%r14,8), %ymm22
-0x62,0xa2,0x7e,0x48,0x72,0xb4,0xf5,0x00,0x00,0x00,0x10
-
-# CHECK: vcvtneps2bf16  291(%r8,%rax,4), %ymm22 {%k7}
-0x62,0xc2,0x7e,0x4f,0x72,0xb4,0x80,0x23,0x01,0x00,0x00
-
-# CHECK: vcvtneps2bf16  (%rip){1to16}, %ymm22
-0x62,0xe2,0x7e,0x58,0x72,0x35,0x00,0x00,0x00,0x00
-
-# CHECK: vcvtneps2bf16  -2048(,%rbp,2), %ymm22
-0x62,0xe2,0x7e,0x48,0x72,0x34,0x6d,0x00,0xf8,0xff,0xff
-
-# CHECK: vcvtneps2bf16  8128(%rcx), %ymm22 {%k7} {z}
-0x62,0xe2,0x7e,0xcf,0x72,0x71,0x7f
-
-# CHECK: vcvtneps2bf16  -512(%rdx){1to16}, %ymm22 {%k7} {z}
-0x62,0xe2,0x7e,0xdf,0x72,0x72,0x80
-
-# CHECK: vdpbf16ps %zmm24, %zmm23, %zmm22
-0x62,0x82,0x46,0x40,0x52,0xf0
-
-# CHECK: vdpbf16ps %zmm24, %zmm23, %zmm22 {%k7}
-0x62,0x82,0x46,0x47,0x52,0xf0
-
-# CHECK: vdpbf16ps %zmm24, %zmm23, %zmm22 {%k7} {z}
-0x62,0x82,0x46,0xc7,0x52,0xf0
-
-# CHECK: vdpbf16ps  268435456(%rbp,%r14,8), %zmm23, %zmm22
-0x62,0xa2,0x46,0x40,0x52,0xb4,0xf5,0x00,0x00,0x00,0x10
-
-# CHECK: vdpbf16ps  291(%r8,%rax,4), %zmm23, %zmm22 {%k7}
-0x62,0xc2,0x46,0x47,0x52,0xb4,0x80,0x23,0x01,0x00,0x00
-
-# CHECK: vdpbf16ps  (%rip){1to16}, %zmm23, %zmm22
-0x62,0xe2,0x46,0x50,0x52,0x35,0x00,0x00,0x00,0x00
-
-# CHECK: vdpbf16ps  -2048(,%rbp,2), %zmm23, %zmm22
-0x62,0xe2,0x46,0x40,0x52,0x34,0x6d,0x00,0xf8,0xff,0xff
-
-# CHECK: vdpbf16ps  8128(%rcx), %zmm23, %zmm22 {%k7} {z}
-0x62,0xe2,0x46,0xc7,0x52,0x71,0x7f
-
-# CHECK: vdpbf16ps  -512(%rdx){1to16}, %zmm23, %zmm22 {%k7} {z}
-0x62,0xe2,0x46,0xd7,0x52,0x72,0x80
diff --git a/llvm/test/MC/Disassembler/X86/x86-64-avx512bf16-intel.txt b/llvm/test/MC/Disassembler/X86/x86-64-avx512bf16-intel.txt
deleted file mode 100644
index 8bb3be91aed2ae71a346cb1323bb381749756388..0000000000000000000000000000000000000000
--- a/llvm/test/MC/Disassembler/X86/x86-64-avx512bf16-intel.txt
+++ /dev/null
@@ -1,83 +0,0 @@
-# RUN: llvm-mc --disassemble %s -triple=x86_64 -x86-asm-syntax=intel --output-asm-variant=1 | FileCheck %s
-
-# CHECK: vcvtne2ps2bf16 zmm22, zmm23, zmm24
-0x62,0x82,0x47,0x40,0x72,0xf0
-
-# CHECK: vcvtne2ps2bf16 zmm22 {k7}, zmm23, zmm24
-0x62,0x82,0x47,0x47,0x72,0xf0
-
-# CHECK: vcvtne2ps2bf16 zmm22 {k7} {z}, zmm23, zmm24
-0x62,0x82,0x47,0xc7,0x72,0xf0
-
-# CHECK: vcvtne2ps2bf16 zmm22, zmm23, zmmword ptr [rbp + 8*r14 + 268435456]
-0x62,0xa2,0x47,0x40,0x72,0xb4,0xf5,0x00,0x00,0x00,0x10
-
-# CHECK: vcvtne2ps2bf16 zmm22 {k7}, zmm23, zmmword ptr [r8 + 4*rax + 291]
-0x62,0xc2,0x47,0x47,0x72,0xb4,0x80,0x23,0x01,0x00,0x00
-
-# CHECK: vcvtne2ps2bf16 zmm22, zmm23, dword ptr [rip]{1to16}
-0x62,0xe2,0x47,0x50,0x72,0x35,0x00,0x00,0x00,0x00
-
-# CHECK: vcvtne2ps2bf16 zmm22, zmm23, zmmword ptr [2*rbp - 2048]
-0x62,0xe2,0x47,0x40,0x72,0x34,0x6d,0x00,0xf8,0xff,0xff
-
-# CHECK: vcvtne2ps2bf16 zmm22 {k7} {z}, zmm23, zmmword ptr [rcx + 8128]
-0x62,0xe2,0x47,0xc7,0x72,0x71,0x7f
-
-# CHECK: vcvtne2ps2bf16 zmm22 {k7} {z}, zmm23, dword ptr [rdx - 512]{1to16}
-0x62,0xe2,0x47,0xd7,0x72,0x72,0x80
-
-# CHECK: vcvtneps2bf16 ymm22, zmm23
-0x62,0xa2,0x7e,0x48,0x72,0xf7
-
-# CHECK: vcvtneps2bf16 ymm22 {k7}, zmm23
-0x62,0xa2,0x7e,0x4f,0x72,0xf7
-
-# CHECK: vcvtneps2bf16 ymm22 {k7} {z}, zmm23
-0x62,0xa2,0x7e,0xcf,0x72,0xf7
-
-# CHECK: vcvtneps2bf16 ymm22, zmmword ptr [rbp + 8*r14 + 268435456]
-0x62,0xa2,0x7e,0x48,0x72,0xb4,0xf5,0x00,0x00,0x00,0x10
-
-# CHECK: vcvtneps2bf16 ymm22 {k7}, zmmword ptr [r8 + 4*rax + 291]
-0x62,0xc2,0x7e,0x4f,0x72,0xb4,0x80,0x23,0x01,0x00,0x00
-
-# CHECK: vcvtneps2bf16 ymm22, dword ptr [rip]{1to16}
-0x62,0xe2,0x7e,0x58,0x72,0x35,0x00,0x00,0x00,0x00
-
-# CHECK: vcvtneps2bf16 ymm22, zmmword ptr [2*rbp - 2048]
-0x62,0xe2,0x7e,0x48,0x72,0x34,0x6d,0x00,0xf8,0xff,0xff
-
-# CHECK: vcvtneps2bf16 ymm22 {k7} {z}, zmmword ptr [rcx + 8128]
-0x62,0xe2,0x7e,0xcf,0x72,0x71,0x7f
-
-# CHECK: vcvtneps2bf16 ymm22 {k7} {z}, dword ptr [rdx - 512]{1to16}
-0x62,0xe2,0x7e,0xdf,0x72,0x72,0x80
-
-# CHECK: vdpbf16ps zmm22, zmm23, zmm24
-0x62,0x82,0x46,0x40,0x52,0xf0
-
-# CHECK: vdpbf16ps zmm22 {k7}, zmm23, zmm24
-0x62,0x82,0x46,0x47,0x52,0xf0
-
-# CHECK: vdpbf16ps zmm22 {k7} {z}, zmm23, zmm24
-0x62,0x82,0x46,0xc7,0x52,0xf0
-
-# CHECK: vdpbf16ps zmm22, zmm23, zmmword ptr [rbp + 8*r14 + 268435456]
-0x62,0xa2,0x46,0x40,0x52,0xb4,0xf5,0x00,0x00,0x00,0x10
-
-# CHECK: vdpbf16ps zmm22 {k7}, zmm23, zmmword ptr [r8 + 4*rax + 291]
-0x62,0xc2,0x46,0x47,0x52,0xb4,0x80,0x23,0x01,0x00,0x00
-
-# CHECK: vdpbf16ps zmm22, zmm23, dword ptr [rip]{1to16}
-0x62,0xe2,0x46,0x50,0x52,0x35,0x00,0x00,0x00,0x00
-
-# CHECK: vdpbf16ps zmm22, zmm23, zmmword ptr [2*rbp - 2048]
-0x62,0xe2,0x46,0x40,0x52,0x34,0x6d,0x00,0xf8,0xff,0xff
-
-# CHECK: vdpbf16ps zmm22 {k7} {z}, zmm23, zmmword ptr [rcx + 8128]
-0x62,0xe2,0x46,0xc7,0x52,0x71,0x7f
-
-# CHECK: vdpbf16ps zmm22 {k7} {z}, zmm23, dword ptr [rdx - 512]{1to16}
-0x62,0xe2,0x46,0xd7,0x52,0x72,0x80
-
diff --git a/llvm/test/MC/Disassembler/X86/x86-64-avx512bf16vl-att.txt b/llvm/test/MC/Disassembler/X86/x86-64-avx512bf16vl-att.txt
deleted file mode 100644
index e2bfc98b13a1073a91f840a5ef4fd43bb2c788fe..0000000000000000000000000000000000000000
--- a/llvm/test/MC/Disassembler/X86/x86-64-avx512bf16vl-att.txt
+++ /dev/null
@@ -1,158 +0,0 @@
-# RUN: llvm-mc --disassemble %s -triple=x86_64-apple-darwin9 | FileCheck %s
-
-# CHECK: vcvtne2ps2bf16 %ymm24, %ymm23, %ymm22
-0x62,0x82,0x47,0x20,0x72,0xf0
-
-# CHECK: vcvtne2ps2bf16 %ymm24, %ymm23, %ymm22 {%k7}
-0x62,0x82,0x47,0x27,0x72,0xf0
-
-# CHECK: vcvtne2ps2bf16 %ymm24, %ymm23, %ymm22 {%k7} {z}
-0x62,0x82,0x47,0xa7,0x72,0xf0
-
-# CHECK: vcvtne2ps2bf16 %xmm24, %xmm23, %xmm22
-0x62,0x82,0x47,0x00,0x72,0xf0
-
-# CHECK: vcvtne2ps2bf16 %xmm24, %xmm23, %xmm22 {%k7}
-0x62,0x82,0x47,0x07,0x72,0xf0
-
-# CHECK: vcvtne2ps2bf16 %xmm24, %xmm23, %xmm22 {%k7} {z}
-0x62,0x82,0x47,0x87,0x72,0xf0
-
-# CHECK: vcvtne2ps2bf16  268435456(%rbp,%r14,8), %ymm23, %ymm22
-0x62,0xa2,0x47,0x20,0x72,0xb4,0xf5,0x00,0x00,0x00,0x10
-
-# CHECK: vcvtne2ps2bf16  291(%r8,%rax,4), %ymm23, %ymm22 {%k7}
-0x62,0xc2,0x47,0x27,0x72,0xb4,0x80,0x23,0x01,0x00,0x00
-
-# CHECK: vcvtne2ps2bf16  (%rip){1to8}, %ymm23, %ymm22
-0x62,0xe2,0x47,0x30,0x72,0x35,0x00,0x00,0x00,0x00
-
-# CHECK: vcvtne2ps2bf16  -1024(,%rbp,2), %ymm23, %ymm22
-0x62,0xe2,0x47,0x20,0x72,0x34,0x6d,0x00,0xfc,0xff,0xff
-
-# CHECK: vcvtne2ps2bf16  4064(%rcx), %ymm23, %ymm22 {%k7} {z}
-0x62,0xe2,0x47,0xa7,0x72,0x71,0x7f
-
-# CHECK: vcvtne2ps2bf16  -512(%rdx){1to8}, %ymm23, %ymm22 {%k7} {z}
-0x62,0xe2,0x47,0xb7,0x72,0x72,0x80
-
-# CHECK: vcvtne2ps2bf16  268435456(%rbp,%r14,8), %xmm23, %xmm22
-0x62,0xa2,0x47,0x00,0x72,0xb4,0xf5,0x00,0x00,0x00,0x10
-
-# CHECK: vcvtne2ps2bf16  291(%r8,%rax,4), %xmm23, %xmm22 {%k7}
-0x62,0xc2,0x47,0x07,0x72,0xb4,0x80,0x23,0x01,0x00,0x00
-
-# CHECK: vcvtne2ps2bf16  (%rip){1to4}, %xmm23, %xmm22
-0x62,0xe2,0x47,0x10,0x72,0x35,0x00,0x00,0x00,0x00
-
-# CHECK: vcvtne2ps2bf16  -512(,%rbp,2), %xmm23, %xmm22
-0x62,0xe2,0x47,0x00,0x72,0x34,0x6d,0x00,0xfe,0xff,0xff
-
-# CHECK: vcvtne2ps2bf16  2032(%rcx), %xmm23, %xmm22 {%k7} {z}
-0x62,0xe2,0x47,0x87,0x72,0x71,0x7f
-
-# CHECK: vcvtne2ps2bf16  -512(%rdx){1to4}, %xmm23, %xmm22 {%k7} {z}
-0x62,0xe2,0x47,0x97,0x72,0x72,0x80
-
-# CHECK: vcvtneps2bf16 %xmm23, %xmm22
-0x62,0xa2,0x7e,0x08,0x72,0xf7
-
-# CHECK: vcvtneps2bf16 %xmm23, %xmm22 {%k7}
-0x62,0xa2,0x7e,0x0f,0x72,0xf7
-
-# CHECK: vcvtneps2bf16 %xmm23, %xmm22 {%k7} {z}
-0x62,0xa2,0x7e,0x8f,0x72,0xf7
-
-# CHECK: vcvtneps2bf16 %ymm23, %xmm22
-0x62,0xa2,0x7e,0x28,0x72,0xf7
-
-# CHECK: vcvtneps2bf16 %ymm23, %xmm22 {%k7}
-0x62,0xa2,0x7e,0x2f,0x72,0xf7
-
-# CHECK: vcvtneps2bf16 %ymm23, %xmm22 {%k7} {z}
-0x62,0xa2,0x7e,0xaf,0x72,0xf7
-
-# CHECK: vcvtneps2bf16x  268435456(%rbp,%r14,8), %xmm22
-0x62,0xa2,0x7e,0x08,0x72,0xb4,0xf5,0x00,0x00,0x00,0x10
-
-# CHECK: vcvtneps2bf16x  291(%r8,%rax,4), %xmm22 {%k7}
-0x62,0xc2,0x7e,0x0f,0x72,0xb4,0x80,0x23,0x01,0x00,0x00
-
-# CHECK: vcvtneps2bf16  (%rip){1to4}, %xmm22
-0x62,0xe2,0x7e,0x18,0x72,0x35,0x00,0x00,0x00,0x00
-
-# CHECK: vcvtneps2bf16x  -512(,%rbp,2), %xmm22
-0x62,0xe2,0x7e,0x08,0x72,0x34,0x6d,0x00,0xfe,0xff,0xff
-
-# CHECK: vcvtneps2bf16x  2032(%rcx), %xmm22 {%k7} {z}
-0x62,0xe2,0x7e,0x8f,0x72,0x71,0x7f
-
-# CHECK: vcvtneps2bf16  -512(%rdx){1to4}, %xmm22 {%k7} {z}
-0x62,0xe2,0x7e,0x9f,0x72,0x72,0x80
-
-# CHECK: vcvtneps2bf16  (%rip){1to8}, %xmm22
-0x62,0xe2,0x7e,0x38,0x72,0x35,0x00,0x00,0x00,0x00
-
-# CHECK: vcvtneps2bf16y  -1024(,%rbp,2), %xmm22
-0x62,0xe2,0x7e,0x28,0x72,0x34,0x6d,0x00,0xfc,0xff,0xff
-
-# CHECK: vcvtneps2bf16y  4064(%rcx), %xmm22 {%k7} {z}
-0x62,0xe2,0x7e,0xaf,0x72,0x71,0x7f
-
-# CHECK: vcvtneps2bf16  -512(%rdx){1to8}, %xmm22 {%k7} {z}
-0x62,0xe2,0x7e,0xbf,0x72,0x72,0x80
-
-# CHECK: vdpbf16ps %ymm24, %ymm23, %ymm22
-0x62,0x82,0x46,0x20,0x52,0xf0
-
-# CHECK: vdpbf16ps %ymm24, %ymm23, %ymm22 {%k7}
-0x62,0x82,0x46,0x27,0x52,0xf0
-
-# CHECK: vdpbf16ps %ymm24, %ymm23, %ymm22 {%k7} {z}
-0x62,0x82,0x46,0xa7,0x52,0xf0
-
-# CHECK: vdpbf16ps %xmm24, %xmm23, %xmm22
-0x62,0x82,0x46,0x00,0x52,0xf0
-
-# CHECK: vdpbf16ps %xmm24, %xmm23, %xmm22 {%k7}
-0x62,0x82,0x46,0x07,0x52,0xf0
-
-# CHECK: vdpbf16ps %xmm24, %xmm23, %xmm22 {%k7} {z}
-0x62,0x82,0x46,0x87,0x52,0xf0
-
-# CHECK: vdpbf16ps  268435456(%rbp,%r14,8), %ymm23, %ymm22
-0x62,0xa2,0x46,0x20,0x52,0xb4,0xf5,0x00,0x00,0x00,0x10
-
-# CHECK: vdpbf16ps  291(%r8,%rax,4), %ymm23, %ymm22 {%k7}
-0x62,0xc2,0x46,0x27,0x52,0xb4,0x80,0x23,0x01,0x00,0x00
-
-# CHECK: vdpbf16ps  (%rip){1to8}, %ymm23, %ymm22
-0x62,0xe2,0x46,0x30,0x52,0x35,0x00,0x00,0x00,0x00
-
-# CHECK: vdpbf16ps  -1024(,%rbp,2), %ymm23, %ymm22
-0x62,0xe2,0x46,0x20,0x52,0x34,0x6d,0x00,0xfc,0xff,0xff
-
-# CHECK: vdpbf16ps  4064(%rcx), %ymm23, %ymm22 {%k7} {z}
-0x62,0xe2,0x46,0xa7,0x52,0x71,0x7f
-
-# CHECK: vdpbf16ps  -512(%rdx){1to8}, %ymm23, %ymm22 {%k7} {z}
-0x62,0xe2,0x46,0xb7,0x52,0x72,0x80
-
-# CHECK: vdpbf16ps  268435456(%rbp,%r14,8), %xmm23, %xmm22
-0x62,0xa2,0x46,0x00,0x52,0xb4,0xf5,0x00,0x00,0x00,0x10
-
-# CHECK: vdpbf16ps  291(%r8,%rax,4), %xmm23, %xmm22 {%k7}
-0x62,0xc2,0x46,0x07,0x52,0xb4,0x80,0x23,0x01,0x00,0x00
-
-# CHECK: vdpbf16ps  (%rip){1to4}, %xmm23, %xmm22
-0x62,0xe2,0x46,0x10,0x52,0x35,0x00,0x00,0x00,0x00
-
-# CHECK: vdpbf16ps  -512(,%rbp,2), %xmm23, %xmm22
-0x62,0xe2,0x46,0x00,0x52,0x34,0x6d,0x00,0xfe,0xff,0xff
-
-# CHECK: vdpbf16ps  2032(%rcx), %xmm23, %xmm22 {%k7} {z}
-0x62,0xe2,0x46,0x87,0x52,0x71,0x7f
-
-# CHECK: vdpbf16ps  -512(%rdx){1to4}, %xmm23, %xmm22 {%k7} {z}
-0x62,0xe2,0x46,0x97,0x52,0x72,0x80
-
diff --git a/llvm/test/MC/Disassembler/X86/x86-64-avx512bf16vl-intel.txt b/llvm/test/MC/Disassembler/X86/x86-64-avx512bf16vl-intel.txt
deleted file mode 100644
index 4def95ed899273cae82d51f3bd0e632dffc7ffb0..0000000000000000000000000000000000000000
--- a/llvm/test/MC/Disassembler/X86/x86-64-avx512bf16vl-intel.txt
+++ /dev/null
@@ -1,158 +0,0 @@
-# RUN: llvm-mc --disassemble %s -triple=x86_64 -x86-asm-syntax=intel --output-asm-variant=1 | FileCheck %s
-
-# CHECK: vcvtne2ps2bf16 ymm22, ymm23, ymm24
-0x62,0x82,0x47,0x20,0x72,0xf0
-
-# CHECK: vcvtne2ps2bf16 ymm22 {k7}, ymm23, ymm24
-0x62,0x82,0x47,0x27,0x72,0xf0
-
-# CHECK: vcvtne2ps2bf16 ymm22 {k7} {z}, ymm23, ymm24
-0x62,0x82,0x47,0xa7,0x72,0xf0
-
-# CHECK: vcvtne2ps2bf16 xmm22, xmm23, xmm24
-0x62,0x82,0x47,0x00,0x72,0xf0
-
-# CHECK: vcvtne2ps2bf16 xmm22 {k7}, xmm23, xmm24
-0x62,0x82,0x47,0x07,0x72,0xf0
-
-# CHECK: vcvtne2ps2bf16 xmm22 {k7} {z}, xmm23, xmm24
-0x62,0x82,0x47,0x87,0x72,0xf0
-
-# CHECK: vcvtne2ps2bf16 ymm22, ymm23, ymmword ptr [rbp + 8*r14 + 268435456]
-0x62,0xa2,0x47,0x20,0x72,0xb4,0xf5,0x00,0x00,0x00,0x10
-
-# CHECK: vcvtne2ps2bf16 ymm22 {k7}, ymm23, ymmword ptr [r8 + 4*rax + 291]
-0x62,0xc2,0x47,0x27,0x72,0xb4,0x80,0x23,0x01,0x00,0x00
-
-# CHECK: vcvtne2ps2bf16 ymm22, ymm23, dword ptr [rip]{1to8}
-0x62,0xe2,0x47,0x30,0x72,0x35,0x00,0x00,0x00,0x00
-
-# CHECK: vcvtne2ps2bf16 ymm22, ymm23, ymmword ptr [2*rbp - 1024]
-0x62,0xe2,0x47,0x20,0x72,0x34,0x6d,0x00,0xfc,0xff,0xff
-
-# CHECK: vcvtne2ps2bf16 ymm22 {k7} {z}, ymm23, ymmword ptr [rcx + 4064]
-0x62,0xe2,0x47,0xa7,0x72,0x71,0x7f
-
-# CHECK: vcvtne2ps2bf16 ymm22 {k7} {z}, ymm23, dword ptr [rdx - 512]{1to8}
-0x62,0xe2,0x47,0xb7,0x72,0x72,0x80
-
-# CHECK: vcvtne2ps2bf16 xmm22, xmm23, xmmword ptr [rbp + 8*r14 + 268435456]
-0x62,0xa2,0x47,0x00,0x72,0xb4,0xf5,0x00,0x00,0x00,0x10
-
-# CHECK: vcvtne2ps2bf16 xmm22 {k7}, xmm23, xmmword ptr [r8 + 4*rax + 291]
-0x62,0xc2,0x47,0x07,0x72,0xb4,0x80,0x23,0x01,0x00,0x00
-
-# CHECK: vcvtne2ps2bf16 xmm22, xmm23, dword ptr [rip]{1to4}
-0x62,0xe2,0x47,0x10,0x72,0x35,0x00,0x00,0x00,0x00
-
-# CHECK: vcvtne2ps2bf16 xmm22, xmm23, xmmword ptr [2*rbp - 512]
-0x62,0xe2,0x47,0x00,0x72,0x34,0x6d,0x00,0xfe,0xff,0xff
-
-# CHECK: vcvtne2ps2bf16 xmm22 {k7} {z}, xmm23, xmmword ptr [rcx + 2032]
-0x62,0xe2,0x47,0x87,0x72,0x71,0x7f
-
-# CHECK: vcvtne2ps2bf16 xmm22 {k7} {z}, xmm23, dword ptr [rdx - 512]{1to4}
-0x62,0xe2,0x47,0x97,0x72,0x72,0x80
-
-# CHECK: vcvtneps2bf16 xmm22, xmm23
-0x62,0xa2,0x7e,0x08,0x72,0xf7
-
-# CHECK: vcvtneps2bf16 xmm22 {k7}, xmm23
-0x62,0xa2,0x7e,0x0f,0x72,0xf7
-
-# CHECK: vcvtneps2bf16 xmm22 {k7} {z}, xmm23
-0x62,0xa2,0x7e,0x8f,0x72,0xf7
-
-# CHECK: vcvtneps2bf16 xmm22, ymm23
-0x62,0xa2,0x7e,0x28,0x72,0xf7
-
-# CHECK: vcvtneps2bf16 xmm22 {k7}, ymm23
-0x62,0xa2,0x7e,0x2f,0x72,0xf7
-
-# CHECK: vcvtneps2bf16 xmm22 {k7} {z}, ymm23
-0x62,0xa2,0x7e,0xaf,0x72,0xf7
-
-# CHECK: vcvtneps2bf16 xmm22, xmmword ptr [rbp + 8*r14 + 268435456]
-0x62,0xa2,0x7e,0x08,0x72,0xb4,0xf5,0x00,0x00,0x00,0x10
-
-# CHECK: vcvtneps2bf16 xmm22 {k7}, xmmword ptr [r8 + 4*rax + 291]
-0x62,0xc2,0x7e,0x0f,0x72,0xb4,0x80,0x23,0x01,0x00,0x00
-
-# CHECK: vcvtneps2bf16 xmm22, dword ptr [rip]{1to4}
-0x62,0xe2,0x7e,0x18,0x72,0x35,0x00,0x00,0x00,0x00
-
-# CHECK: vcvtneps2bf16 xmm22, xmmword ptr [2*rbp - 512]
-0x62,0xe2,0x7e,0x08,0x72,0x34,0x6d,0x00,0xfe,0xff,0xff
-
-# CHECK: vcvtneps2bf16 xmm22 {k7} {z}, xmmword ptr [rcx + 2032]
-0x62,0xe2,0x7e,0x8f,0x72,0x71,0x7f
-
-# CHECK: vcvtneps2bf16 xmm22 {k7} {z}, dword ptr [rdx - 512]{1to4}
-0x62,0xe2,0x7e,0x9f,0x72,0x72,0x80
-
-# CHECK: vcvtneps2bf16 xmm22, dword ptr [rip]{1to8}
-0x62,0xe2,0x7e,0x38,0x72,0x35,0x00,0x00,0x00,0x00
-
-# CHECK: vcvtneps2bf16 xmm22, ymmword ptr [2*rbp - 1024]
-0x62,0xe2,0x7e,0x28,0x72,0x34,0x6d,0x00,0xfc,0xff,0xff
-
-# CHECK: vcvtneps2bf16 xmm22 {k7} {z}, ymmword ptr [rcx + 4064]
-0x62,0xe2,0x7e,0xaf,0x72,0x71,0x7f
-
-# CHECK: vcvtneps2bf16 xmm22 {k7} {z}, dword ptr [rdx - 512]{1to8}
-0x62,0xe2,0x7e,0xbf,0x72,0x72,0x80
-
-# CHECK: vdpbf16ps ymm22, ymm23, ymm24
-0x62,0x82,0x46,0x20,0x52,0xf0
-
-# CHECK: vdpbf16ps ymm22 {k7}, ymm23, ymm24
-0x62,0x82,0x46,0x27,0x52,0xf0
-
-# CHECK: vdpbf16ps ymm22 {k7} {z}, ymm23, ymm24
-0x62,0x82,0x46,0xa7,0x52,0xf0
-
-# CHECK: vdpbf16ps xmm22, xmm23, xmm24
-0x62,0x82,0x46,0x00,0x52,0xf0
-
-# CHECK: vdpbf16ps xmm22 {k7}, xmm23, xmm24
-0x62,0x82,0x46,0x07,0x52,0xf0
-
-# CHECK: vdpbf16ps xmm22 {k7} {z}, xmm23, xmm24
-0x62,0x82,0x46,0x87,0x52,0xf0
-
-# CHECK: vdpbf16ps ymm22, ymm23, ymmword ptr [rbp + 8*r14 + 268435456]
-0x62,0xa2,0x46,0x20,0x52,0xb4,0xf5,0x00,0x00,0x00,0x10
-
-# CHECK: vdpbf16ps ymm22 {k7}, ymm23, ymmword ptr [r8 + 4*rax + 291]
-0x62,0xc2,0x46,0x27,0x52,0xb4,0x80,0x23,0x01,0x00,0x00
-
-# CHECK: vdpbf16ps ymm22, ymm23, dword ptr [rip]{1to8}
-0x62,0xe2,0x46,0x30,0x52,0x35,0x00,0x00,0x00,0x00
-
-# CHECK: vdpbf16ps ymm22, ymm23, ymmword ptr [2*rbp - 1024]
-0x62,0xe2,0x46,0x20,0x52,0x34,0x6d,0x00,0xfc,0xff,0xff
-
-# CHECK: vdpbf16ps ymm22 {k7} {z}, ymm23, ymmword ptr [rcx + 4064]
-0x62,0xe2,0x46,0xa7,0x52,0x71,0x7f
-
-# CHECK: vdpbf16ps ymm22 {k7} {z}, ymm23, dword ptr [rdx - 512]{1to8}
-0x62,0xe2,0x46,0xb7,0x52,0x72,0x80
-
-# CHECK: vdpbf16ps xmm22, xmm23, xmmword ptr [rbp + 8*r14 + 268435456]
-0x62,0xa2,0x46,0x00,0x52,0xb4,0xf5,0x00,0x00,0x00,0x10
-
-# CHECK: vdpbf16ps xmm22 {k7}, xmm23, xmmword ptr [r8 + 4*rax + 291]
-0x62,0xc2,0x46,0x07,0x52,0xb4,0x80,0x23,0x01,0x00,0x00
-
-# CHECK: vdpbf16ps xmm22, xmm23, dword ptr [rip]{1to4}
-0x62,0xe2,0x46,0x10,0x52,0x35,0x00,0x00,0x00,0x00
-
-# CHECK: vdpbf16ps xmm22, xmm23, xmmword ptr [2*rbp - 512]
-0x62,0xe2,0x46,0x00,0x52,0x34,0x6d,0x00,0xfe,0xff,0xff
-
-# CHECK: vdpbf16ps xmm22 {k7} {z}, xmm23, xmmword ptr [rcx + 2032]
-0x62,0xe2,0x46,0x87,0x52,0x71,0x7f
-
-# CHECK: vdpbf16ps xmm22 {k7} {z}, xmm23, dword ptr [rdx - 512]{1to4}
-0x62,0xe2,0x46,0x97,0x52,0x72,0x80
-
diff --git a/llvm/test/MC/Disassembler/X86/x86-64-avx512vp2intersectvl-att.txt b/llvm/test/MC/Disassembler/X86/x86-64-avx512vp2intersectvl-att.txt
deleted file mode 100644
index b166ccbf6ee079ed1e86e2b55b87e994975a03b5..0000000000000000000000000000000000000000
--- a/llvm/test/MC/Disassembler/X86/x86-64-avx512vp2intersectvl-att.txt
+++ /dev/null
@@ -1,85 +0,0 @@
-# RUN: llvm-mc --disassemble %s -triple=x86_64 | FileCheck %s
-
-# CHECK: vp2intersectd %ymm24, %ymm23, %k6
-0x62,0x92,0x47,0x20,0x68,0xf0
-
-# CHECK: vp2intersectd %xmm24, %xmm23, %k6
-0x62,0x92,0x47,0x00,0x68,0xf0
-
-# CHECK: vp2intersectd  268435456(%rbp,%r14,8), %ymm23, %k6
-0x62,0xb2,0x47,0x20,0x68,0xb4,0xf5,0x00,0x00,0x00,0x10
-
-# CHECK: vp2intersectd  291(%r8,%rax,4), %ymm23, %k6
-0x62,0xd2,0x47,0x20,0x68,0xb4,0x80,0x23,0x01,0x00,0x00
-
-# CHECK: vp2intersectd  (%rip){1to8}, %ymm23, %k6
-0x62,0xf2,0x47,0x30,0x68,0x35,0x00,0x00,0x00,0x00
-
-# CHECK: vp2intersectd  -1024(,%rbp,2), %ymm23, %k6
-0x62,0xf2,0x47,0x20,0x68,0x34,0x6d,0x00,0xfc,0xff,0xff
-
-# CHECK: vp2intersectd  4064(%rcx), %ymm23, %k6
-0x62,0xf2,0x47,0x20,0x68,0x71,0x7f
-
-# CHECK: vp2intersectd  -512(%rdx){1to8}, %ymm23, %k6
-0x62,0xf2,0x47,0x30,0x68,0x72,0x80
-
-# CHECK: vp2intersectd  268435456(%rbp,%r14,8), %xmm23, %k6
-0x62,0xb2,0x47,0x00,0x68,0xb4,0xf5,0x00,0x00,0x00,0x10
-
-# CHECK: vp2intersectd  291(%r8,%rax,4), %xmm23, %k6
-0x62,0xd2,0x47,0x00,0x68,0xb4,0x80,0x23,0x01,0x00,0x00
-
-# CHECK: vp2intersectd  (%rip){1to4}, %xmm23, %k6
-0x62,0xf2,0x47,0x10,0x68,0x35,0x00,0x00,0x00,0x00
-
-# CHECK: vp2intersectd  -512(,%rbp,2), %xmm23, %k6
-0x62,0xf2,0x47,0x00,0x68,0x34,0x6d,0x00,0xfe,0xff,0xff
-
-# CHECK: vp2intersectd  2032(%rcx), %xmm23, %k6
-0x62,0xf2,0x47,0x00,0x68,0x71,0x7f
-
-# CHECK: vp2intersectd  -512(%rdx){1to4}, %xmm23, %k6
-0x62,0xf2,0x47,0x10,0x68,0x72,0x80
-
-# CHECK: vp2intersectq %ymm24, %ymm23, %k6
-0x62,0x92,0xc7,0x20,0x68,0xf0
-
-# CHECK: vp2intersectq %xmm24, %xmm23, %k6
-0x62,0x92,0xc7,0x00,0x68,0xf0
-
-# CHECK: vp2intersectq  268435456(%rbp,%r14,8), %ymm23, %k6
-0x62,0xb2,0xc7,0x20,0x68,0xb4,0xf5,0x00,0x00,0x00,0x10
-
-# CHECK: vp2intersectq  291(%r8,%rax,4), %ymm23, %k6
-0x62,0xd2,0xc7,0x20,0x68,0xb4,0x80,0x23,0x01,0x00,0x00
-
-# CHECK: vp2intersectq  (%rip){1to4}, %ymm23, %k6
-0x62,0xf2,0xc7,0x30,0x68,0x35,0x00,0x00,0x00,0x00
-
-# CHECK: vp2intersectq  -1024(,%rbp,2), %ymm23, %k6
-0x62,0xf2,0xc7,0x20,0x68,0x34,0x6d,0x00,0xfc,0xff,0xff
-
-# CHECK: vp2intersectq  4064(%rcx), %ymm23, %k6
-0x62,0xf2,0xc7,0x20,0x68,0x71,0x7f
-
-# CHECK: vp2intersectq  -1024(%rdx){1to4}, %ymm23, %k6
-0x62,0xf2,0xc7,0x30,0x68,0x72,0x80
-
-# CHECK: vp2intersectq  268435456(%rbp,%r14,8), %xmm23, %k6
-0x62,0xb2,0xc7,0x00,0x68,0xb4,0xf5,0x00,0x00,0x00,0x10
-
-# CHECK: vp2intersectq  291(%r8,%rax,4), %xmm23, %k6
-0x62,0xd2,0xc7,0x00,0x68,0xb4,0x80,0x23,0x01,0x00,0x00
-
-# CHECK: vp2intersectq  (%rip){1to2}, %xmm23, %k6
-0x62,0xf2,0xc7,0x10,0x68,0x35,0x00,0x00,0x00,0x00
-
-# CHECK: vp2intersectq  -512(,%rbp,2), %xmm23, %k6
-0x62,0xf2,0xc7,0x00,0x68,0x34,0x6d,0x00,0xfe,0xff,0xff
-
-# CHECK: vp2intersectq  2032(%rcx), %xmm23, %k6
-0x62,0xf2,0xc7,0x00,0x68,0x71,0x7f
-
-# CHECK: vp2intersectq  -1024(%rdx){1to2}, %xmm23, %k6
-0x62,0xf2,0xc7,0x10,0x68,0x72,0x80
diff --git a/llvm/test/MC/Disassembler/X86/x86-64-avx512vp2intersectvl-intel.txt b/llvm/test/MC/Disassembler/X86/x86-64-avx512vp2intersectvl-intel.txt
deleted file mode 100644
index b952b668454c5c48cf344b7428191d15a166ea63..0000000000000000000000000000000000000000
--- a/llvm/test/MC/Disassembler/X86/x86-64-avx512vp2intersectvl-intel.txt
+++ /dev/null
@@ -1,85 +0,0 @@
-# RUN: llvm-mc --disassemble %s -triple=x86_64 -x86-asm-syntax=intel --output-asm-variant=1 | FileCheck %s
-
-# CHECK: vp2intersectd k6, ymm23, ymm24
-0x62,0x92,0x47,0x20,0x68,0xf0
-
-# CHECK: vp2intersectd k6, xmm23, xmm24
-0x62,0x92,0x47,0x00,0x68,0xf0
-
-# CHECK: vp2intersectd k6, ymm23, ymmword ptr [rbp + 8*r14 + 268435456]
-0x62,0xb2,0x47,0x20,0x68,0xb4,0xf5,0x00,0x00,0x00,0x10
-
-# CHECK: vp2intersectd k6, ymm23, ymmword ptr [r8 + 4*rax + 291]
-0x62,0xd2,0x47,0x20,0x68,0xb4,0x80,0x23,0x01,0x00,0x00
-
-# CHECK: vp2intersectd k6, ymm23, dword ptr [rip]{1to8}
-0x62,0xf2,0x47,0x30,0x68,0x35,0x00,0x00,0x00,0x00
-
-# CHECK: vp2intersectd k6, ymm23, ymmword ptr [2*rbp - 1024]
-0x62,0xf2,0x47,0x20,0x68,0x34,0x6d,0x00,0xfc,0xff,0xff
-
-# CHECK: vp2intersectd k6, ymm23, ymmword ptr [rcx + 4064]
-0x62,0xf2,0x47,0x20,0x68,0x71,0x7f
-
-# CHECK: vp2intersectd k6, ymm23, dword ptr [rdx - 512]{1to8}
-0x62,0xf2,0x47,0x30,0x68,0x72,0x80
-
-# CHECK: vp2intersectd k6, xmm23, xmmword ptr [rbp + 8*r14 + 268435456]
-0x62,0xb2,0x47,0x00,0x68,0xb4,0xf5,0x00,0x00,0x00,0x10
-
-# CHECK: vp2intersectd k6, xmm23, xmmword ptr [r8 + 4*rax + 291]
-0x62,0xd2,0x47,0x00,0x68,0xb4,0x80,0x23,0x01,0x00,0x00
-
-# CHECK: vp2intersectd k6, xmm23, dword ptr [rip]{1to4}
-0x62,0xf2,0x47,0x10,0x68,0x35,0x00,0x00,0x00,0x00
-
-# CHECK: vp2intersectd k6, xmm23, xmmword ptr [2*rbp - 512]
-0x62,0xf2,0x47,0x00,0x68,0x34,0x6d,0x00,0xfe,0xff,0xff
-
-# CHECK: vp2intersectd k6, xmm23, xmmword ptr [rcx + 2032]
-0x62,0xf2,0x47,0x00,0x68,0x71,0x7f
-
-# CHECK: vp2intersectd k6, xmm23, dword ptr [rdx - 512]{1to4}
-0x62,0xf2,0x47,0x10,0x68,0x72,0x80
-
-# CHECK: vp2intersectq k6, ymm23, ymm24
-0x62,0x92,0xc7,0x20,0x68,0xf0
-
-# CHECK: vp2intersectq k6, xmm23, xmm24
-0x62,0x92,0xc7,0x00,0x68,0xf0
-
-# CHECK: vp2intersectq k6, ymm23, ymmword ptr [rbp + 8*r14 + 268435456]
-0x62,0xb2,0xc7,0x20,0x68,0xb4,0xf5,0x00,0x00,0x00,0x10
-
-# CHECK: vp2intersectq k6, ymm23, ymmword ptr [r8 + 4*rax + 291]
-0x62,0xd2,0xc7,0x20,0x68,0xb4,0x80,0x23,0x01,0x00,0x00
-
-# CHECK: vp2intersectq k6, ymm23, qword ptr [rip]{1to4}
-0x62,0xf2,0xc7,0x30,0x68,0x35,0x00,0x00,0x00,0x00
-
-# CHECK: vp2intersectq k6, ymm23, ymmword ptr [2*rbp - 1024]
-0x62,0xf2,0xc7,0x20,0x68,0x34,0x6d,0x00,0xfc,0xff,0xff
-
-# CHECK: vp2intersectq k6, ymm23, ymmword ptr [rcx + 4064]
-0x62,0xf2,0xc7,0x20,0x68,0x71,0x7f
-
-# CHECK: vp2intersectq k6, ymm23, qword ptr [rdx - 1024]{1to4}
-0x62,0xf2,0xc7,0x30,0x68,0x72,0x80
-
-# CHECK: vp2intersectq k6, xmm23, xmmword ptr [rbp + 8*r14 + 268435456]
-0x62,0xb2,0xc7,0x00,0x68,0xb4,0xf5,0x00,0x00,0x00,0x10
-
-# CHECK: vp2intersectq k6, xmm23, xmmword ptr [r8 + 4*rax + 291]
-0x62,0xd2,0xc7,0x00,0x68,0xb4,0x80,0x23,0x01,0x00,0x00
-
-# CHECK: vp2intersectq k6, xmm23, qword ptr [rip]{1to2}
-0x62,0xf2,0xc7,0x10,0x68,0x35,0x00,0x00,0x00,0x00
-
-# CHECK: vp2intersectq k6, xmm23, xmmword ptr [2*rbp - 512]
-0x62,0xf2,0xc7,0x00,0x68,0x34,0x6d,0x00,0xfe,0xff,0xff
-
-# CHECK: vp2intersectq k6, xmm23, xmmword ptr [rcx + 2032]
-0x62,0xf2,0xc7,0x00,0x68,0x71,0x7f
-
-# CHECK: vp2intersectq k6, xmm23, qword ptr [rdx - 1024]{1to2}
-0x62,0xf2,0xc7,0x10,0x68,0x72,0x80
diff --git a/llvm/test/MC/PowerPC/ppc32-ba.s b/llvm/test/MC/PowerPC/ppc32-ba.s
index 133423b4e8c31231477c7a3f7767be27c258a748..3b7814815534795b039465bf3d1d2d636f99636b 100644
--- a/llvm/test/MC/PowerPC/ppc32-ba.s
+++ b/llvm/test/MC/PowerPC/ppc32-ba.s
@@ -1,6 +1,10 @@
 # RUN: llvm-mc -triple powerpc-unknown-unknown --show-encoding %s | FileCheck %s
 
-# Check that large immediates in 32bit mode are accepted.
+# Check that large and/or negative immediates in 32-bit mode are accepted.
 
-# CHECK: ba -33554432 # encoding: [0x4a,0x00,0x00,0x02]
+# CHECK:      ba 0xfe000000  # encoding: [0x4a,0x00,0x00,0x02]
+# CHECK-NEXT: ba 0xfe000000  # encoding: [0x4a,0x00,0x00,0x02]
+# CHECK-NEXT: ba 0xfffffc00  # encoding: [0x4b,0xff,0xfc,0x02]
          ba 0xfe000000
+         ba (-33554432)
+	 ba (-1024)
diff --git a/llvm/test/MC/PowerPC/ppc64-operands.s b/llvm/test/MC/PowerPC/ppc64-operands.s
index 9cd94bea6f815c11fd319bdaba92f95ee19b4df9..366e1faf31357c563e04678e8b50edfd95e97b3d 100644
--- a/llvm/test/MC/PowerPC/ppc64-operands.s
+++ b/llvm/test/MC/PowerPC/ppc64-operands.s
@@ -128,16 +128,16 @@
 # CHECK-LE: b .+1024                        # encoding: [0x00,0x04,0x00,0x48]
             b 1024
 
-# CHECK-BE: ba 1024                         # encoding: [0x48,0x00,0x04,0x02]
-# CHECK-LE: ba 1024                         # encoding: [0x02,0x04,0x00,0x48]
+# CHECK-BE: ba 0x400                        # encoding: [0x48,0x00,0x04,0x02]
+# CHECK-LE: ba 0x400                        # encoding: [0x02,0x04,0x00,0x48]
             ba 1024
 
 # CHECK-BE: beq 0, .+1024                   # encoding: [0x41,0x82,0x04,0x00]
 # CHECK-LE: beq 0, .+1024                   # encoding: [0x00,0x04,0x82,0x41]
             beq 1024
 
-# CHECK-BE: beqa 0, 1024                    # encoding: [0x41,0x82,0x04,0x02]
-# CHECK-LE: beqa 0, 1024                    # encoding: [0x02,0x04,0x82,0x41]
+# CHECK-BE: beqa 0, 0x400                   # encoding: [0x41,0x82,0x04,0x02]
+# CHECK-LE: beqa 0, 0x400                   # encoding: [0x02,0x04,0x82,0x41]
             beqa 1024
 
 # CHECK-BE:                                 # encoding: [0x42,0x9f,A,0bAAAAAA01]
diff --git a/llvm/test/MC/RISCV/attribute-arch.s b/llvm/test/MC/RISCV/attribute-arch.s
index aa919f266592f4506f8e1d8a04c81e11fbd87138..3ed48401e43fc8db4c4db698b51455e9091e249d 100644
--- a/llvm/test/MC/RISCV/attribute-arch.s
+++ b/llvm/test/MC/RISCV/attribute-arch.s
@@ -309,5 +309,5 @@
 .attribute arch, "rv32i_xcvbi"
 # CHECK: attribute      5, "rv32i2p1_xcvbi1p0"
 
-.attribute arch, "rv32i_zicfilp0p2"
-# CHECK: attribute      5, "rv32i2p1_zicfilp0p2"
+.attribute arch, "rv32i_zicfilp0p4"
+# CHECK: attribute      5, "rv32i2p1_zicfilp0p4"
diff --git a/llvm/test/MC/X86/x86-32-avx.s b/llvm/test/MC/X86/avx-32-att.s
similarity index 100%
rename from llvm/test/MC/X86/x86-32-avx.s
rename to llvm/test/MC/X86/avx-32-att.s
diff --git a/llvm/test/MC/X86/x86_64-avx-encoding.s b/llvm/test/MC/X86/avx-64-att.s
similarity index 100%
rename from llvm/test/MC/X86/x86_64-avx-encoding.s
rename to llvm/test/MC/X86/avx-64-att.s
diff --git a/llvm/test/MC/X86/intel-syntax-x86-64-avx.s b/llvm/test/MC/X86/avx-64-intel.s
similarity index 100%
rename from llvm/test/MC/X86/intel-syntax-x86-64-avx.s
rename to llvm/test/MC/X86/avx-64-intel.s
diff --git a/llvm/test/MC/X86/avx512-err.s b/llvm/test/MC/X86/avx512-att-errors.s
similarity index 100%
rename from llvm/test/MC/X86/avx512-err.s
rename to llvm/test/MC/X86/avx512-att-errors.s
diff --git a/llvm/test/MC/X86/avx512-encodings.s b/llvm/test/MC/X86/avx512-att.s
similarity index 100%
rename from llvm/test/MC/X86/avx512-encodings.s
rename to llvm/test/MC/X86/avx512-att.s
diff --git a/llvm/test/MC/X86/intel-syntax-avx512-error.s b/llvm/test/MC/X86/avx512-intel-errors.s
similarity index 100%
rename from llvm/test/MC/X86/intel-syntax-avx512-error.s
rename to llvm/test/MC/X86/avx512-intel-errors.s
diff --git a/llvm/test/MC/X86/intel-syntax-avx512.s b/llvm/test/MC/X86/avx512-intel.s
similarity index 100%
rename from llvm/test/MC/X86/intel-syntax-avx512.s
rename to llvm/test/MC/X86/avx512-intel.s
diff --git a/llvm/test/MC/X86/avx5124fmaps-encoding.s b/llvm/test/MC/X86/avx5124fmaps-att.s
similarity index 100%
rename from llvm/test/MC/X86/avx5124fmaps-encoding.s
rename to llvm/test/MC/X86/avx5124fmaps-att.s
diff --git a/llvm/test/MC/X86/avx5124vnniw-encoding.s b/llvm/test/MC/X86/avx5124vnniw-att.s
similarity index 100%
rename from llvm/test/MC/X86/avx5124vnniw-encoding.s
rename to llvm/test/MC/X86/avx5124vnniw-att.s
diff --git a/llvm/test/MC/X86/avx512_bf16-encoding.s b/llvm/test/MC/X86/avx512_bf16-32-att.s
similarity index 100%
rename from llvm/test/MC/X86/avx512_bf16-encoding.s
rename to llvm/test/MC/X86/avx512_bf16-32-att.s
diff --git a/llvm/test/MC/X86/intel-syntax-avx512_bf16.s b/llvm/test/MC/X86/avx512_bf16-32-intel.s
similarity index 100%
rename from llvm/test/MC/X86/intel-syntax-avx512_bf16.s
rename to llvm/test/MC/X86/avx512_bf16-32-intel.s
diff --git a/llvm/test/MC/X86/x86-64-avx512_bf16-encoding.s b/llvm/test/MC/X86/avx512_bf16-64-att.s
similarity index 100%
rename from llvm/test/MC/X86/x86-64-avx512_bf16-encoding.s
rename to llvm/test/MC/X86/avx512_bf16-64-att.s
diff --git a/llvm/test/MC/X86/intel-syntax-x86-64-avx512_bf16.s b/llvm/test/MC/X86/avx512_bf16-64-intel.s
similarity index 100%
rename from llvm/test/MC/X86/intel-syntax-x86-64-avx512_bf16.s
rename to llvm/test/MC/X86/avx512_bf16-64-intel.s
diff --git a/llvm/test/MC/X86/avx512_bf16_vl-encoding.s b/llvm/test/MC/X86/avx512_bf16_vl-32-att.s
similarity index 100%
rename from llvm/test/MC/X86/avx512_bf16_vl-encoding.s
rename to llvm/test/MC/X86/avx512_bf16_vl-32-att.s
diff --git a/llvm/test/MC/X86/intel-syntax-avx512_bf16_vl.s b/llvm/test/MC/X86/avx512_bf16_vl-32-intel.s
similarity index 100%
rename from llvm/test/MC/X86/intel-syntax-avx512_bf16_vl.s
rename to llvm/test/MC/X86/avx512_bf16_vl-32-intel.s
diff --git a/llvm/test/MC/X86/x86-64-avx512_bf16_vl-encoding.s b/llvm/test/MC/X86/avx512_bf16_vl-64-att.s
similarity index 100%
rename from llvm/test/MC/X86/x86-64-avx512_bf16_vl-encoding.s
rename to llvm/test/MC/X86/avx512_bf16_vl-64-att.s
diff --git a/llvm/test/MC/X86/intel-syntax-x86-64-avx512_bf16_vl.s b/llvm/test/MC/X86/avx512_bf16_vl-64-intel.s
similarity index 100%
rename from llvm/test/MC/X86/intel-syntax-x86-64-avx512_bf16_vl.s
rename to llvm/test/MC/X86/avx512_bf16_vl-64-intel.s
diff --git a/llvm/test/MC/X86/avx512bitalg-att.s b/llvm/test/MC/X86/avx512bitalg-att.s
new file mode 100644
index 0000000000000000000000000000000000000000..3bf11ff8254e1175ba9de585ea85867d2894862a
--- /dev/null
+++ b/llvm/test/MC/X86/avx512bitalg-att.s
@@ -0,0 +1,169 @@
+# RUN: llvm-mc -triple x86_64 -show-encoding %s | FileCheck %s
+
+# CHECK: vpopcntb	%zmm23, %zmm21
+# CHECK: encoding: [0x62,0xa2,0x7d,0x48,0x54,0xef]
+         vpopcntb	%zmm23, %zmm21
+
+# CHECK: vpopcntw	%zmm23, %zmm21
+# CHECK: encoding: [0x62,0xa2,0xfd,0x48,0x54,0xef]
+         vpopcntw	%zmm23, %zmm21
+
+# CHECK: vpopcntb	%zmm3, %zmm1 {%k2}
+# CHECK: encoding: [0x62,0xf2,0x7d,0x4a,0x54,0xcb]
+         vpopcntb	%zmm3, %zmm1 {%k2}
+
+# CHECK: vpopcntw	%zmm3, %zmm1 {%k2}
+# CHECK: encoding: [0x62,0xf2,0xfd,0x4a,0x54,0xcb]
+         vpopcntw	%zmm3, %zmm1 {%k2}
+
+# CHECK: vpopcntb	(%rcx), %zmm1
+# CHECK: encoding: [0x62,0xf2,0x7d,0x48,0x54,0x09]
+         vpopcntb	(%rcx), %zmm1
+
+# CHECK: vpopcntb	-256(%rsp), %zmm1
+# CHECK: encoding: [0x62,0xf2,0x7d,0x48,0x54,0x4c,0x24,0xfc]
+         vpopcntb	-256(%rsp), %zmm1
+
+# CHECK: vpopcntb	256(%rsp), %zmm1
+# CHECK: encoding: [0x62,0xf2,0x7d,0x48,0x54,0x4c,0x24,0x04]
+         vpopcntb	256(%rsp), %zmm1
+
+# CHECK: vpopcntb	268435456(%rcx,%r14,8), %zmm1
+# CHECK: encoding: [0x62,0xb2,0x7d,0x48,0x54,0x8c,0xf1,0x00,0x00,0x00,0x10]
+         vpopcntb	268435456(%rcx,%r14,8), %zmm1
+
+# CHECK: vpopcntb	-536870912(%rcx,%r14,8), %zmm1
+# CHECK: encoding: [0x62,0xb2,0x7d,0x48,0x54,0x8c,0xf1,0x00,0x00,0x00,0xe0]
+         vpopcntb	-536870912(%rcx,%r14,8), %zmm1
+
+# CHECK: vpopcntb	-536870910(%rcx,%r14,8), %zmm1
+# CHECK: encoding: [0x62,0xb2,0x7d,0x48,0x54,0x8c,0xf1,0x02,0x00,0x00,0xe0]
+         vpopcntb	-536870910(%rcx,%r14,8), %zmm1
+
+# CHECK: vpopcntw	(%rcx), %zmm1
+# CHECK: encoding: [0x62,0xf2,0xfd,0x48,0x54,0x09]
+         vpopcntw	(%rcx), %zmm1
+
+# CHECK: vpopcntw	-256(%rsp), %zmm1
+# CHECK: encoding: [0x62,0xf2,0xfd,0x48,0x54,0x4c,0x24,0xfc]
+         vpopcntw	-256(%rsp), %zmm1
+
+# CHECK: vpopcntw	256(%rsp), %zmm1
+# CHECK: encoding: [0x62,0xf2,0xfd,0x48,0x54,0x4c,0x24,0x04]
+         vpopcntw	256(%rsp), %zmm1
+
+# CHECK: vpopcntw	268435456(%rcx,%r14,8), %zmm1
+# CHECK: encoding: [0x62,0xb2,0xfd,0x48,0x54,0x8c,0xf1,0x00,0x00,0x00,0x10]
+         vpopcntw	268435456(%rcx,%r14,8), %zmm1
+
+# CHECK: vpopcntw	-536870912(%rcx,%r14,8), %zmm1
+# CHECK: encoding: [0x62,0xb2,0xfd,0x48,0x54,0x8c,0xf1,0x00,0x00,0x00,0xe0]
+         vpopcntw	-536870912(%rcx,%r14,8), %zmm1
+
+# CHECK: vpopcntw	-536870910(%rcx,%r14,8), %zmm1
+# CHECK: encoding: [0x62,0xb2,0xfd,0x48,0x54,0x8c,0xf1,0x02,0x00,0x00,0xe0]
+         vpopcntw	-536870910(%rcx,%r14,8), %zmm1
+
+# CHECK: vpopcntb	(%rcx), %zmm21 {%k2}
+# CHECK: encoding: [0x62,0xe2,0x7d,0x4a,0x54,0x29]
+         vpopcntb	(%rcx), %zmm21 {%k2}
+
+# CHECK: vpopcntb	-256(%rsp), %zmm21 {%k2}
+# CHECK: encoding: [0x62,0xe2,0x7d,0x4a,0x54,0x6c,0x24,0xfc]
+         vpopcntb	-256(%rsp), %zmm21 {%k2}
+
+# CHECK: vpopcntb	256(%rsp), %zmm21 {%k2}
+# CHECK: encoding: [0x62,0xe2,0x7d,0x4a,0x54,0x6c,0x24,0x04]
+         vpopcntb	256(%rsp), %zmm21 {%k2}
+
+# CHECK: vpopcntb	268435456(%rcx,%r14,8), %zmm21 {%k2}
+# CHECK: encoding: [0x62,0xa2,0x7d,0x4a,0x54,0xac,0xf1,0x00,0x00,0x00,0x10]
+         vpopcntb	268435456(%rcx,%r14,8), %zmm21 {%k2}
+
+# CHECK: vpopcntb	-536870912(%rcx,%r14,8), %zmm21 {%k2}
+# CHECK: encoding: [0x62,0xa2,0x7d,0x4a,0x54,0xac,0xf1,0x00,0x00,0x00,0xe0]
+         vpopcntb	-536870912(%rcx,%r14,8), %zmm21 {%k2}
+
+# CHECK: vpopcntb	-536870910(%rcx,%r14,8), %zmm21 {%k2}
+# CHECK: encoding: [0x62,0xa2,0x7d,0x4a,0x54,0xac,0xf1,0x02,0x00,0x00,0xe0]
+         vpopcntb	-536870910(%rcx,%r14,8), %zmm21 {%k2}
+
+# CHECK: vpopcntw	(%rcx), %zmm21 {%k2}
+# CHECK: encoding: [0x62,0xe2,0xfd,0x4a,0x54,0x29]
+         vpopcntw	(%rcx), %zmm21 {%k2}
+
+# CHECK: vpopcntw	-256(%rsp), %zmm21 {%k2}
+# CHECK: encoding: [0x62,0xe2,0xfd,0x4a,0x54,0x6c,0x24,0xfc]
+         vpopcntw	-256(%rsp), %zmm21 {%k2}
+
+# CHECK: vpopcntw	256(%rsp), %zmm21 {%k2}
+# CHECK: encoding: [0x62,0xe2,0xfd,0x4a,0x54,0x6c,0x24,0x04]
+         vpopcntw	256(%rsp), %zmm21 {%k2}
+
+# CHECK: vpopcntw	268435456(%rcx,%r14,8), %zmm21 {%k2}
+# CHECK: encoding: [0x62,0xa2,0xfd,0x4a,0x54,0xac,0xf1,0x00,0x00,0x00,0x10]
+         vpopcntw	268435456(%rcx,%r14,8), %zmm21 {%k2}
+
+# CHECK: vpopcntw	-536870912(%rcx,%r14,8), %zmm21 {%k2}
+# CHECK: encoding: [0x62,0xa2,0xfd,0x4a,0x54,0xac,0xf1,0x00,0x00,0x00,0xe0]
+         vpopcntw	-536870912(%rcx,%r14,8), %zmm21 {%k2}
+
+# CHECK: vpopcntw	-536870910(%rcx,%r14,8), %zmm21 {%k2}
+# CHECK: encoding: [0x62,0xa2,0xfd,0x4a,0x54,0xac,0xf1,0x02,0x00,0x00,0xe0]
+         vpopcntw	-536870910(%rcx,%r14,8), %zmm21 {%k2}
+
+# CHECK: vpshufbitqmb	%zmm2, %zmm23, %k1
+# CHECK: encoding: [0x62,0xf2,0x45,0x40,0x8f,0xca]
+         vpshufbitqmb	%zmm2, %zmm23, %k1
+
+# CHECK: vpshufbitqmb	%zmm2, %zmm23, %k1 {%k2}
+# CHECK: encoding: [0x62,0xf2,0x45,0x42,0x8f,0xca]
+         vpshufbitqmb	%zmm2, %zmm23, %k1 {%k2}
+
+# CHECK: vpshufbitqmb	(%rcx), %zmm23, %k1
+# CHECK: encoding: [0x62,0xf2,0x45,0x40,0x8f,0x09]
+         vpshufbitqmb	(%rcx), %zmm23, %k1
+
+# CHECK: vpshufbitqmb	-256(%rsp), %zmm23, %k1
+# CHECK: encoding: [0x62,0xf2,0x45,0x40,0x8f,0x4c,0x24,0xfc]
+         vpshufbitqmb	-256(%rsp), %zmm23, %k1
+
+# CHECK: vpshufbitqmb	256(%rsp), %zmm23, %k1
+# CHECK: encoding: [0x62,0xf2,0x45,0x40,0x8f,0x4c,0x24,0x04]
+         vpshufbitqmb	256(%rsp), %zmm23, %k1
+
+# CHECK: vpshufbitqmb	268435456(%rcx,%r14,8), %zmm23, %k1
+# CHECK: encoding: [0x62,0xb2,0x45,0x40,0x8f,0x8c,0xf1,0x00,0x00,0x00,0x10]
+         vpshufbitqmb	268435456(%rcx,%r14,8), %zmm23, %k1
+
+# CHECK: vpshufbitqmb	-536870912(%rcx,%r14,8), %zmm23, %k1
+# CHECK: encoding: [0x62,0xb2,0x45,0x40,0x8f,0x8c,0xf1,0x00,0x00,0x00,0xe0]
+         vpshufbitqmb	-536870912(%rcx,%r14,8), %zmm23, %k1
+
+# CHECK: vpshufbitqmb	-536870910(%rcx,%r14,8), %zmm23, %k1
+# CHECK: encoding: [0x62,0xb2,0x45,0x40,0x8f,0x8c,0xf1,0x02,0x00,0x00,0xe0]
+         vpshufbitqmb	-536870910(%rcx,%r14,8), %zmm23, %k1
+
+# CHECK: vpshufbitqmb	(%rcx), %zmm23, %k1 {%k2}
+# CHECK: encoding: [0x62,0xf2,0x45,0x42,0x8f,0x09]
+         vpshufbitqmb	(%rcx), %zmm23, %k1 {%k2}
+
+# CHECK: vpshufbitqmb	-256(%rsp), %zmm23, %k1 {%k2}
+# CHECK: encoding: [0x62,0xf2,0x45,0x42,0x8f,0x4c,0x24,0xfc]
+         vpshufbitqmb	-256(%rsp), %zmm23, %k1 {%k2}
+
+# CHECK: vpshufbitqmb	256(%rsp), %zmm23, %k1 {%k2}
+# CHECK: encoding: [0x62,0xf2,0x45,0x42,0x8f,0x4c,0x24,0x04]
+         vpshufbitqmb	256(%rsp), %zmm23, %k1 {%k2}
+
+# CHECK: vpshufbitqmb	268435456(%rcx,%r14,8), %zmm23, %k1 {%k2}
+# CHECK: encoding: [0x62,0xb2,0x45,0x42,0x8f,0x8c,0xf1,0x00,0x00,0x00,0x10]
+         vpshufbitqmb	268435456(%rcx,%r14,8), %zmm23, %k1 {%k2}
+
+# CHECK: vpshufbitqmb	-536870912(%rcx,%r14,8), %zmm23, %k1 {%k2}
+# CHECK: encoding: [0x62,0xb2,0x45,0x42,0x8f,0x8c,0xf1,0x00,0x00,0x00,0xe0]
+         vpshufbitqmb	-536870912(%rcx,%r14,8), %zmm23, %k1 {%k2}
+
+# CHECK: vpshufbitqmb	-536870910(%rcx,%r14,8), %zmm23, %k1 {%k2}
+# CHECK: encoding: [0x62,0xb2,0x45,0x42,0x8f,0x8c,0xf1,0x02,0x00,0x00,0xe0]
+         vpshufbitqmb	-536870910(%rcx,%r14,8), %zmm23, %k1 {%k2}
diff --git a/llvm/test/MC/X86/avx512bitalg-encoding.s b/llvm/test/MC/X86/avx512bitalg-encoding.s
deleted file mode 100644
index f8b507adfc0912684b512dc97ce8d316eaf0a703..0000000000000000000000000000000000000000
--- a/llvm/test/MC/X86/avx512bitalg-encoding.s
+++ /dev/null
@@ -1,170 +0,0 @@
-// RUN: llvm-mc -triple x86_64-unknown-unknown --show-encoding < %s | FileCheck %s
-
-// CHECK: vpopcntb %zmm23, %zmm21
-// CHECK: encoding: [0x62,0xa2,0x7d,0x48,0x54,0xef]
-          vpopcntb %zmm23, %zmm21
-
-// CHECK: vpopcntw %zmm23, %zmm21
-// CHECK: encoding: [0x62,0xa2,0xfd,0x48,0x54,0xef]
-          vpopcntw %zmm23, %zmm21
-
-// CHECK: vpopcntb %zmm3, %zmm1 {%k2}
-// CHECK: encoding: [0x62,0xf2,0x7d,0x4a,0x54,0xcb]
-          vpopcntb %zmm3, %zmm1 {%k2}
-
-// CHECK: vpopcntw %zmm3, %zmm1 {%k2}
-// CHECK: encoding: [0x62,0xf2,0xfd,0x4a,0x54,0xcb]
-          vpopcntw %zmm3, %zmm1 {%k2}
-
-// CHECK: vpopcntb  (%rcx), %zmm1
-// CHECK: encoding: [0x62,0xf2,0x7d,0x48,0x54,0x09]
-          vpopcntb  (%rcx), %zmm1
-
-// CHECK: vpopcntb  -256(%rsp), %zmm1
-// CHECK: encoding: [0x62,0xf2,0x7d,0x48,0x54,0x4c,0x24,0xfc]
-          vpopcntb  -256(%rsp), %zmm1
-
-// CHECK: vpopcntb  256(%rsp), %zmm1
-// CHECK: encoding: [0x62,0xf2,0x7d,0x48,0x54,0x4c,0x24,0x04]
-          vpopcntb  256(%rsp), %zmm1
-
-// CHECK: vpopcntb  268435456(%rcx,%r14,8), %zmm1
-// CHECK: encoding: [0x62,0xb2,0x7d,0x48,0x54,0x8c,0xf1,0x00,0x00,0x00,0x10]
-          vpopcntb  268435456(%rcx,%r14,8), %zmm1
-
-// CHECK: vpopcntb  -536870912(%rcx,%r14,8), %zmm1
-// CHECK: encoding: [0x62,0xb2,0x7d,0x48,0x54,0x8c,0xf1,0x00,0x00,0x00,0xe0]
-          vpopcntb  -536870912(%rcx,%r14,8), %zmm1
-
-// CHECK: vpopcntb  -536870910(%rcx,%r14,8), %zmm1
-// CHECK: encoding: [0x62,0xb2,0x7d,0x48,0x54,0x8c,0xf1,0x02,0x00,0x00,0xe0]
-          vpopcntb  -536870910(%rcx,%r14,8), %zmm1
-
-// CHECK: vpopcntw  (%rcx), %zmm1
-// CHECK: encoding: [0x62,0xf2,0xfd,0x48,0x54,0x09]
-          vpopcntw  (%rcx), %zmm1
-
-// CHECK: vpopcntw  -256(%rsp), %zmm1
-// CHECK: encoding: [0x62,0xf2,0xfd,0x48,0x54,0x4c,0x24,0xfc]
-          vpopcntw  -256(%rsp), %zmm1
-
-// CHECK: vpopcntw  256(%rsp), %zmm1
-// CHECK: encoding: [0x62,0xf2,0xfd,0x48,0x54,0x4c,0x24,0x04]
-          vpopcntw  256(%rsp), %zmm1
-
-// CHECK: vpopcntw  268435456(%rcx,%r14,8), %zmm1
-// CHECK: encoding: [0x62,0xb2,0xfd,0x48,0x54,0x8c,0xf1,0x00,0x00,0x00,0x10]
-          vpopcntw  268435456(%rcx,%r14,8), %zmm1
-
-// CHECK: vpopcntw  -536870912(%rcx,%r14,8), %zmm1
-// CHECK: encoding: [0x62,0xb2,0xfd,0x48,0x54,0x8c,0xf1,0x00,0x00,0x00,0xe0]
-          vpopcntw  -536870912(%rcx,%r14,8), %zmm1
-
-// CHECK: vpopcntw  -536870910(%rcx,%r14,8), %zmm1
-// CHECK: encoding: [0x62,0xb2,0xfd,0x48,0x54,0x8c,0xf1,0x02,0x00,0x00,0xe0]
-          vpopcntw  -536870910(%rcx,%r14,8), %zmm1
-
-// CHECK: vpopcntb  (%rcx), %zmm21 {%k2}
-// CHECK: encoding: [0x62,0xe2,0x7d,0x4a,0x54,0x29]
-          vpopcntb  (%rcx), %zmm21 {%k2}
-
-// CHECK: vpopcntb  -256(%rsp), %zmm21 {%k2}
-// CHECK: encoding: [0x62,0xe2,0x7d,0x4a,0x54,0x6c,0x24,0xfc]
-          vpopcntb  -256(%rsp), %zmm21 {%k2}
-
-// CHECK: vpopcntb  256(%rsp), %zmm21 {%k2}
-// CHECK: encoding: [0x62,0xe2,0x7d,0x4a,0x54,0x6c,0x24,0x04]
-          vpopcntb  256(%rsp), %zmm21 {%k2}
-
-// CHECK: vpopcntb  268435456(%rcx,%r14,8), %zmm21 {%k2}
-// CHECK: encoding: [0x62,0xa2,0x7d,0x4a,0x54,0xac,0xf1,0x00,0x00,0x00,0x10]
-          vpopcntb  268435456(%rcx,%r14,8), %zmm21 {%k2}
-
-// CHECK: vpopcntb  -536870912(%rcx,%r14,8), %zmm21 {%k2}
-// CHECK: encoding: [0x62,0xa2,0x7d,0x4a,0x54,0xac,0xf1,0x00,0x00,0x00,0xe0]
-          vpopcntb  -536870912(%rcx,%r14,8), %zmm21 {%k2}
-
-// CHECK: vpopcntb  -536870910(%rcx,%r14,8), %zmm21 {%k2}
-// CHECK: encoding: [0x62,0xa2,0x7d,0x4a,0x54,0xac,0xf1,0x02,0x00,0x00,0xe0]
-          vpopcntb  -536870910(%rcx,%r14,8), %zmm21 {%k2}
-
-// CHECK: vpopcntw  (%rcx), %zmm21 {%k2}
-// CHECK: encoding: [0x62,0xe2,0xfd,0x4a,0x54,0x29]
-          vpopcntw  (%rcx), %zmm21 {%k2}
-
-// CHECK: vpopcntw  -256(%rsp), %zmm21 {%k2}
-// CHECK: encoding: [0x62,0xe2,0xfd,0x4a,0x54,0x6c,0x24,0xfc]
-          vpopcntw  -256(%rsp), %zmm21 {%k2}
-
-// CHECK: vpopcntw  256(%rsp), %zmm21 {%k2}
-// CHECK: encoding: [0x62,0xe2,0xfd,0x4a,0x54,0x6c,0x24,0x04]
-          vpopcntw  256(%rsp), %zmm21 {%k2}
-
-// CHECK: vpopcntw  268435456(%rcx,%r14,8), %zmm21 {%k2}
-// CHECK: encoding: [0x62,0xa2,0xfd,0x4a,0x54,0xac,0xf1,0x00,0x00,0x00,0x10]
-          vpopcntw  268435456(%rcx,%r14,8), %zmm21 {%k2}
-
-// CHECK: vpopcntw  -536870912(%rcx,%r14,8), %zmm21 {%k2}
-// CHECK: encoding: [0x62,0xa2,0xfd,0x4a,0x54,0xac,0xf1,0x00,0x00,0x00,0xe0]
-          vpopcntw  -536870912(%rcx,%r14,8), %zmm21 {%k2}
-
-// CHECK: vpopcntw  -536870910(%rcx,%r14,8), %zmm21 {%k2}
-// CHECK: encoding: [0x62,0xa2,0xfd,0x4a,0x54,0xac,0xf1,0x02,0x00,0x00,0xe0]
-          vpopcntw  -536870910(%rcx,%r14,8), %zmm21 {%k2}
-
-// CHECK: vpshufbitqmb %zmm2, %zmm23, %k1
-// CHECK: encoding: [0x62,0xf2,0x45,0x40,0x8f,0xca]
-          vpshufbitqmb %zmm2, %zmm23, %k1
-
-// CHECK: vpshufbitqmb %zmm2, %zmm23, %k1 {%k2}
-// CHECK: encoding: [0x62,0xf2,0x45,0x42,0x8f,0xca]
-          vpshufbitqmb %zmm2, %zmm23, %k1 {%k2}
-
-// CHECK: vpshufbitqmb  (%rcx), %zmm23, %k1
-// CHECK: encoding: [0x62,0xf2,0x45,0x40,0x8f,0x09]
-          vpshufbitqmb  (%rcx), %zmm23, %k1
-
-// CHECK: vpshufbitqmb  -256(%rsp), %zmm23, %k1
-// CHECK: encoding: [0x62,0xf2,0x45,0x40,0x8f,0x4c,0x24,0xfc]
-          vpshufbitqmb  -256(%rsp), %zmm23, %k1
-
-// CHECK: vpshufbitqmb  256(%rsp), %zmm23, %k1
-// CHECK: encoding: [0x62,0xf2,0x45,0x40,0x8f,0x4c,0x24,0x04]
-          vpshufbitqmb  256(%rsp), %zmm23, %k1
-
-// CHECK: vpshufbitqmb  268435456(%rcx,%r14,8), %zmm23, %k1
-// CHECK: encoding: [0x62,0xb2,0x45,0x40,0x8f,0x8c,0xf1,0x00,0x00,0x00,0x10]
-          vpshufbitqmb  268435456(%rcx,%r14,8), %zmm23, %k1
-
-// CHECK: vpshufbitqmb  -536870912(%rcx,%r14,8), %zmm23, %k1
-// CHECK: encoding: [0x62,0xb2,0x45,0x40,0x8f,0x8c,0xf1,0x00,0x00,0x00,0xe0]
-          vpshufbitqmb  -536870912(%rcx,%r14,8), %zmm23, %k1
-
-// CHECK: vpshufbitqmb  -536870910(%rcx,%r14,8), %zmm23, %k1
-// CHECK: encoding: [0x62,0xb2,0x45,0x40,0x8f,0x8c,0xf1,0x02,0x00,0x00,0xe0]
-          vpshufbitqmb  -536870910(%rcx,%r14,8), %zmm23, %k1
-
-// CHECK: vpshufbitqmb  (%rcx), %zmm23, %k1 {%k2}
-// CHECK: encoding: [0x62,0xf2,0x45,0x42,0x8f,0x09]
-          vpshufbitqmb  (%rcx), %zmm23, %k1 {%k2}
-
-// CHECK: vpshufbitqmb  -256(%rsp), %zmm23, %k1 {%k2}
-// CHECK: encoding: [0x62,0xf2,0x45,0x42,0x8f,0x4c,0x24,0xfc]
-          vpshufbitqmb  -256(%rsp), %zmm23, %k1 {%k2}
-
-// CHECK: vpshufbitqmb  256(%rsp), %zmm23, %k1 {%k2}
-// CHECK: encoding: [0x62,0xf2,0x45,0x42,0x8f,0x4c,0x24,0x04]
-          vpshufbitqmb  256(%rsp), %zmm23, %k1 {%k2}
-
-// CHECK: vpshufbitqmb  268435456(%rcx,%r14,8), %zmm23, %k1 {%k2}
-// CHECK: encoding: [0x62,0xb2,0x45,0x42,0x8f,0x8c,0xf1,0x00,0x00,0x00,0x10]
-          vpshufbitqmb  268435456(%rcx,%r14,8), %zmm23, %k1 {%k2}
-
-// CHECK: vpshufbitqmb  -536870912(%rcx,%r14,8), %zmm23, %k1 {%k2}
-// CHECK: encoding: [0x62,0xb2,0x45,0x42,0x8f,0x8c,0xf1,0x00,0x00,0x00,0xe0]
-          vpshufbitqmb  -536870912(%rcx,%r14,8), %zmm23, %k1 {%k2}
-
-// CHECK: vpshufbitqmb  -536870910(%rcx,%r14,8), %zmm23, %k1 {%k2}
-// CHECK: encoding: [0x62,0xb2,0x45,0x42,0x8f,0x8c,0xf1,0x02,0x00,0x00,0xe0]
-          vpshufbitqmb  -536870910(%rcx,%r14,8), %zmm23, %k1 {%k2}
-
diff --git a/llvm/test/MC/X86/avx512bitalg-intel.s b/llvm/test/MC/X86/avx512bitalg-intel.s
new file mode 100644
index 0000000000000000000000000000000000000000..ab5e3ff0c1b7a3fd818c733d5c1826307e405064
--- /dev/null
+++ b/llvm/test/MC/X86/avx512bitalg-intel.s
@@ -0,0 +1,169 @@
+# RUN: llvm-mc -triple x86_64 -show-encoding -x86-asm-syntax=intel -output-asm-variant=1 %s | FileCheck %s
+
+# CHECK: vpopcntb	zmm21, zmm23
+# CHECK: encoding: [0x62,0xa2,0x7d,0x48,0x54,0xef]
+         vpopcntb	zmm21, zmm23
+
+# CHECK: vpopcntw	zmm21, zmm23
+# CHECK: encoding: [0x62,0xa2,0xfd,0x48,0x54,0xef]
+         vpopcntw	zmm21, zmm23
+
+# CHECK: vpopcntb	zmm1 {k2}, zmm3
+# CHECK: encoding: [0x62,0xf2,0x7d,0x4a,0x54,0xcb]
+         vpopcntb	zmm1 {k2}, zmm3
+
+# CHECK: vpopcntw	zmm1 {k2}, zmm3
+# CHECK: encoding: [0x62,0xf2,0xfd,0x4a,0x54,0xcb]
+         vpopcntw	zmm1 {k2}, zmm3
+
+# CHECK: vpopcntb	zmm1, zmmword ptr [rcx]
+# CHECK: encoding: [0x62,0xf2,0x7d,0x48,0x54,0x09]
+         vpopcntb	zmm1, zmmword ptr [rcx]
+
+# CHECK: vpopcntb	zmm1, zmmword ptr [rsp - 256]
+# CHECK: encoding: [0x62,0xf2,0x7d,0x48,0x54,0x4c,0x24,0xfc]
+         vpopcntb	zmm1, zmmword ptr [rsp - 256]
+
+# CHECK: vpopcntb	zmm1, zmmword ptr [rsp + 256]
+# CHECK: encoding: [0x62,0xf2,0x7d,0x48,0x54,0x4c,0x24,0x04]
+         vpopcntb	zmm1, zmmword ptr [rsp + 256]
+
+# CHECK: vpopcntb	zmm1, zmmword ptr [rcx + 8*r14 + 268435456]
+# CHECK: encoding: [0x62,0xb2,0x7d,0x48,0x54,0x8c,0xf1,0x00,0x00,0x00,0x10]
+         vpopcntb	zmm1, zmmword ptr [rcx + 8*r14 + 268435456]
+
+# CHECK: vpopcntb	zmm1, zmmword ptr [rcx + 8*r14 - 536870912]
+# CHECK: encoding: [0x62,0xb2,0x7d,0x48,0x54,0x8c,0xf1,0x00,0x00,0x00,0xe0]
+         vpopcntb	zmm1, zmmword ptr [rcx + 8*r14 - 536870912]
+
+# CHECK: vpopcntb	zmm1, zmmword ptr [rcx + 8*r14 - 536870910]
+# CHECK: encoding: [0x62,0xb2,0x7d,0x48,0x54,0x8c,0xf1,0x02,0x00,0x00,0xe0]
+         vpopcntb	zmm1, zmmword ptr [rcx + 8*r14 - 536870910]
+
+# CHECK: vpopcntw	zmm1, zmmword ptr [rcx]
+# CHECK: encoding: [0x62,0xf2,0xfd,0x48,0x54,0x09]
+         vpopcntw	zmm1, zmmword ptr [rcx]
+
+# CHECK: vpopcntw	zmm1, zmmword ptr [rsp - 256]
+# CHECK: encoding: [0x62,0xf2,0xfd,0x48,0x54,0x4c,0x24,0xfc]
+         vpopcntw	zmm1, zmmword ptr [rsp - 256]
+
+# CHECK: vpopcntw	zmm1, zmmword ptr [rsp + 256]
+# CHECK: encoding: [0x62,0xf2,0xfd,0x48,0x54,0x4c,0x24,0x04]
+         vpopcntw	zmm1, zmmword ptr [rsp + 256]
+
+# CHECK: vpopcntw	zmm1, zmmword ptr [rcx + 8*r14 + 268435456]
+# CHECK: encoding: [0x62,0xb2,0xfd,0x48,0x54,0x8c,0xf1,0x00,0x00,0x00,0x10]
+         vpopcntw	zmm1, zmmword ptr [rcx + 8*r14 + 268435456]
+
+# CHECK: vpopcntw	zmm1, zmmword ptr [rcx + 8*r14 - 536870912]
+# CHECK: encoding: [0x62,0xb2,0xfd,0x48,0x54,0x8c,0xf1,0x00,0x00,0x00,0xe0]
+         vpopcntw	zmm1, zmmword ptr [rcx + 8*r14 - 536870912]
+
+# CHECK: vpopcntw	zmm1, zmmword ptr [rcx + 8*r14 - 536870910]
+# CHECK: encoding: [0x62,0xb2,0xfd,0x48,0x54,0x8c,0xf1,0x02,0x00,0x00,0xe0]
+         vpopcntw	zmm1, zmmword ptr [rcx + 8*r14 - 536870910]
+
+# CHECK: vpopcntb	zmm21 {k2}, zmmword ptr [rcx]
+# CHECK: encoding: [0x62,0xe2,0x7d,0x4a,0x54,0x29]
+         vpopcntb	zmm21 {k2}, zmmword ptr [rcx]
+
+# CHECK: vpopcntb	zmm21 {k2}, zmmword ptr [rsp - 256]
+# CHECK: encoding: [0x62,0xe2,0x7d,0x4a,0x54,0x6c,0x24,0xfc]
+         vpopcntb	zmm21 {k2}, zmmword ptr [rsp - 256]
+
+# CHECK: vpopcntb	zmm21 {k2}, zmmword ptr [rsp + 256]
+# CHECK: encoding: [0x62,0xe2,0x7d,0x4a,0x54,0x6c,0x24,0x04]
+         vpopcntb	zmm21 {k2}, zmmword ptr [rsp + 256]
+
+# CHECK: vpopcntb	zmm21 {k2}, zmmword ptr [rcx + 8*r14 + 268435456]
+# CHECK: encoding: [0x62,0xa2,0x7d,0x4a,0x54,0xac,0xf1,0x00,0x00,0x00,0x10]
+         vpopcntb	zmm21 {k2}, zmmword ptr [rcx + 8*r14 + 268435456]
+
+# CHECK: vpopcntb	zmm21 {k2}, zmmword ptr [rcx + 8*r14 - 536870912]
+# CHECK: encoding: [0x62,0xa2,0x7d,0x4a,0x54,0xac,0xf1,0x00,0x00,0x00,0xe0]
+         vpopcntb	zmm21 {k2}, zmmword ptr [rcx + 8*r14 - 536870912]
+
+# CHECK: vpopcntb	zmm21 {k2}, zmmword ptr [rcx + 8*r14 - 536870910]
+# CHECK: encoding: [0x62,0xa2,0x7d,0x4a,0x54,0xac,0xf1,0x02,0x00,0x00,0xe0]
+         vpopcntb	zmm21 {k2}, zmmword ptr [rcx + 8*r14 - 536870910]
+
+# CHECK: vpopcntw	zmm21 {k2}, zmmword ptr [rcx]
+# CHECK: encoding: [0x62,0xe2,0xfd,0x4a,0x54,0x29]
+         vpopcntw	zmm21 {k2}, zmmword ptr [rcx]
+
+# CHECK: vpopcntw	zmm21 {k2}, zmmword ptr [rsp - 256]
+# CHECK: encoding: [0x62,0xe2,0xfd,0x4a,0x54,0x6c,0x24,0xfc]
+         vpopcntw	zmm21 {k2}, zmmword ptr [rsp - 256]
+
+# CHECK: vpopcntw	zmm21 {k2}, zmmword ptr [rsp + 256]
+# CHECK: encoding: [0x62,0xe2,0xfd,0x4a,0x54,0x6c,0x24,0x04]
+         vpopcntw	zmm21 {k2}, zmmword ptr [rsp + 256]
+
+# CHECK: vpopcntw	zmm21 {k2}, zmmword ptr [rcx + 8*r14 + 268435456]
+# CHECK: encoding: [0x62,0xa2,0xfd,0x4a,0x54,0xac,0xf1,0x00,0x00,0x00,0x10]
+         vpopcntw	zmm21 {k2}, zmmword ptr [rcx + 8*r14 + 268435456]
+
+# CHECK: vpopcntw	zmm21 {k2}, zmmword ptr [rcx + 8*r14 - 536870912]
+# CHECK: encoding: [0x62,0xa2,0xfd,0x4a,0x54,0xac,0xf1,0x00,0x00,0x00,0xe0]
+         vpopcntw	zmm21 {k2}, zmmword ptr [rcx + 8*r14 - 536870912]
+
+# CHECK: vpopcntw	zmm21 {k2}, zmmword ptr [rcx + 8*r14 - 536870910]
+# CHECK: encoding: [0x62,0xa2,0xfd,0x4a,0x54,0xac,0xf1,0x02,0x00,0x00,0xe0]
+         vpopcntw	zmm21 {k2}, zmmword ptr [rcx + 8*r14 - 536870910]
+
+# CHECK: vpshufbitqmb	k1, zmm23, zmm2
+# CHECK: encoding: [0x62,0xf2,0x45,0x40,0x8f,0xca]
+         vpshufbitqmb	k1, zmm23, zmm2
+
+# CHECK: vpshufbitqmb	k1 {k2}, zmm23, zmm2
+# CHECK: encoding: [0x62,0xf2,0x45,0x42,0x8f,0xca]
+         vpshufbitqmb	k1 {k2}, zmm23, zmm2
+
+# CHECK: vpshufbitqmb	k1, zmm23, zmmword ptr [rcx]
+# CHECK: encoding: [0x62,0xf2,0x45,0x40,0x8f,0x09]
+         vpshufbitqmb	k1, zmm23, zmmword ptr [rcx]
+
+# CHECK: vpshufbitqmb	k1, zmm23, zmmword ptr [rsp - 256]
+# CHECK: encoding: [0x62,0xf2,0x45,0x40,0x8f,0x4c,0x24,0xfc]
+         vpshufbitqmb	k1, zmm23, zmmword ptr [rsp - 256]
+
+# CHECK: vpshufbitqmb	k1, zmm23, zmmword ptr [rsp + 256]
+# CHECK: encoding: [0x62,0xf2,0x45,0x40,0x8f,0x4c,0x24,0x04]
+         vpshufbitqmb	k1, zmm23, zmmword ptr [rsp + 256]
+
+# CHECK: vpshufbitqmb	k1, zmm23, zmmword ptr [rcx + 8*r14 + 268435456]
+# CHECK: encoding: [0x62,0xb2,0x45,0x40,0x8f,0x8c,0xf1,0x00,0x00,0x00,0x10]
+         vpshufbitqmb	k1, zmm23, zmmword ptr [rcx + 8*r14 + 268435456]
+
+# CHECK: vpshufbitqmb	k1, zmm23, zmmword ptr [rcx + 8*r14 - 536870912]
+# CHECK: encoding: [0x62,0xb2,0x45,0x40,0x8f,0x8c,0xf1,0x00,0x00,0x00,0xe0]
+         vpshufbitqmb	k1, zmm23, zmmword ptr [rcx + 8*r14 - 536870912]
+
+# CHECK: vpshufbitqmb	k1, zmm23, zmmword ptr [rcx + 8*r14 - 536870910]
+# CHECK: encoding: [0x62,0xb2,0x45,0x40,0x8f,0x8c,0xf1,0x02,0x00,0x00,0xe0]
+         vpshufbitqmb	k1, zmm23, zmmword ptr [rcx + 8*r14 - 536870910]
+
+# CHECK: vpshufbitqmb	k1 {k2}, zmm23, zmmword ptr [rcx]
+# CHECK: encoding: [0x62,0xf2,0x45,0x42,0x8f,0x09]
+         vpshufbitqmb	k1 {k2}, zmm23, zmmword ptr [rcx]
+
+# CHECK: vpshufbitqmb	k1 {k2}, zmm23, zmmword ptr [rsp - 256]
+# CHECK: encoding: [0x62,0xf2,0x45,0x42,0x8f,0x4c,0x24,0xfc]
+         vpshufbitqmb	k1 {k2}, zmm23, zmmword ptr [rsp - 256]
+
+# CHECK: vpshufbitqmb	k1 {k2}, zmm23, zmmword ptr [rsp + 256]
+# CHECK: encoding: [0x62,0xf2,0x45,0x42,0x8f,0x4c,0x24,0x04]
+         vpshufbitqmb	k1 {k2}, zmm23, zmmword ptr [rsp + 256]
+
+# CHECK: vpshufbitqmb	k1 {k2}, zmm23, zmmword ptr [rcx + 8*r14 + 268435456]
+# CHECK: encoding: [0x62,0xb2,0x45,0x42,0x8f,0x8c,0xf1,0x00,0x00,0x00,0x10]
+         vpshufbitqmb	k1 {k2}, zmm23, zmmword ptr [rcx + 8*r14 + 268435456]
+
+# CHECK: vpshufbitqmb	k1 {k2}, zmm23, zmmword ptr [rcx + 8*r14 - 536870912]
+# CHECK: encoding: [0x62,0xb2,0x45,0x42,0x8f,0x8c,0xf1,0x00,0x00,0x00,0xe0]
+         vpshufbitqmb	k1 {k2}, zmm23, zmmword ptr [rcx + 8*r14 - 536870912]
+
+# CHECK: vpshufbitqmb	k1 {k2}, zmm23, zmmword ptr [rcx + 8*r14 - 536870910]
+# CHECK: encoding: [0x62,0xb2,0x45,0x42,0x8f,0x8c,0xf1,0x02,0x00,0x00,0xe0]
+         vpshufbitqmb	k1 {k2}, zmm23, zmmword ptr [rcx + 8*r14 - 536870910]
diff --git a/llvm/test/MC/X86/x86-64-avx512bw.s b/llvm/test/MC/X86/avx512bw-64-att.s
similarity index 100%
rename from llvm/test/MC/X86/x86-64-avx512bw.s
rename to llvm/test/MC/X86/avx512bw-64-att.s
diff --git a/llvm/test/MC/X86/avx512bw-encoding.s b/llvm/test/MC/X86/avx512bw-att.s
similarity index 100%
rename from llvm/test/MC/X86/avx512bw-encoding.s
rename to llvm/test/MC/X86/avx512bw-att.s
diff --git a/llvm/test/MC/X86/x86-64-avx512bw_vl.s b/llvm/test/MC/X86/avx512bw_vl-64-att.s
similarity index 100%
rename from llvm/test/MC/X86/x86-64-avx512bw_vl.s
rename to llvm/test/MC/X86/avx512bw_vl-64-att.s
diff --git a/llvm/test/MC/X86/x86-64-avx512cd.s b/llvm/test/MC/X86/avx512cd-att.s
similarity index 100%
rename from llvm/test/MC/X86/x86-64-avx512cd.s
rename to llvm/test/MC/X86/avx512cd-att.s
diff --git a/llvm/test/MC/X86/x86-64-avx512cd_vl.s b/llvm/test/MC/X86/avx512cd_vl-att.s
similarity index 100%
rename from llvm/test/MC/X86/x86-64-avx512cd_vl.s
rename to llvm/test/MC/X86/avx512cd_vl-att.s
diff --git a/llvm/test/MC/X86/x86-64-avx512dq.s b/llvm/test/MC/X86/avx512dq-att.s
similarity index 100%
rename from llvm/test/MC/X86/x86-64-avx512dq.s
rename to llvm/test/MC/X86/avx512dq-att.s
diff --git a/llvm/test/MC/X86/avx512dq_vl-att.s b/llvm/test/MC/X86/avx512dq_vl-att.s
new file mode 100644
index 0000000000000000000000000000000000000000..debddebc06894278a1a41c4a064b1b7462da8cac
--- /dev/null
+++ b/llvm/test/MC/X86/avx512dq_vl-att.s
@@ -0,0 +1,4911 @@
+# RUN: llvm-mc -triple x86_64 -show-encoding %s | FileCheck %s
+
+# CHECK: vpmullq	%xmm22, %xmm17, %xmm26
+# CHECK: encoding: [0x62,0x22,0xf5,0x00,0x40,0xd6]
+         vpmullq	%xmm22, %xmm17, %xmm26
+# CHECK: vpmullq	%xmm22, %xmm17, %xmm26 {%k6}
+# CHECK: encoding: [0x62,0x22,0xf5,0x06,0x40,0xd6]
+         vpmullq	%xmm22, %xmm17, %xmm26 {%k6}
+# CHECK: vpmullq	%xmm22, %xmm17, %xmm26 {%k6} {z}
+# CHECK: encoding: [0x62,0x22,0xf5,0x86,0x40,0xd6]
+         vpmullq	%xmm22, %xmm17, %xmm26 {%k6} {z}
+# CHECK: vpmullq	(%rcx), %xmm17, %xmm26
+# CHECK: encoding: [0x62,0x62,0xf5,0x00,0x40,0x11]
+         vpmullq	(%rcx), %xmm17, %xmm26
+# CHECK: vpmullq	291(%rax,%r14,8), %xmm17, %xmm26
+# CHECK: encoding: [0x62,0x22,0xf5,0x00,0x40,0x94,0xf0,0x23,0x01,0x00,0x00]
+         vpmullq	291(%rax,%r14,8), %xmm17, %xmm26
+# CHECK: vpmullq	(%rcx){1to2}, %xmm17, %xmm26
+# CHECK: encoding: [0x62,0x62,0xf5,0x10,0x40,0x11]
+         vpmullq	(%rcx){1to2}, %xmm17, %xmm26
+# CHECK: vpmullq	2032(%rdx), %xmm17, %xmm26
+# CHECK: encoding: [0x62,0x62,0xf5,0x00,0x40,0x52,0x7f]
+         vpmullq	2032(%rdx), %xmm17, %xmm26
+# CHECK: vpmullq	2048(%rdx), %xmm17, %xmm26
+# CHECK: encoding: [0x62,0x62,0xf5,0x00,0x40,0x92,0x00,0x08,0x00,0x00]
+         vpmullq	2048(%rdx), %xmm17, %xmm26
+# CHECK: vpmullq	-2048(%rdx), %xmm17, %xmm26
+# CHECK: encoding: [0x62,0x62,0xf5,0x00,0x40,0x52,0x80]
+         vpmullq	-2048(%rdx), %xmm17, %xmm26
+# CHECK: vpmullq	-2064(%rdx), %xmm17, %xmm26
+# CHECK: encoding: [0x62,0x62,0xf5,0x00,0x40,0x92,0xf0,0xf7,0xff,0xff]
+         vpmullq	-2064(%rdx), %xmm17, %xmm26
+# CHECK: vpmullq	1016(%rdx){1to2}, %xmm17, %xmm26
+# CHECK: encoding: [0x62,0x62,0xf5,0x10,0x40,0x52,0x7f]
+         vpmullq	1016(%rdx){1to2}, %xmm17, %xmm26
+# CHECK: vpmullq	1024(%rdx){1to2}, %xmm17, %xmm26
+# CHECK: encoding: [0x62,0x62,0xf5,0x10,0x40,0x92,0x00,0x04,0x00,0x00]
+         vpmullq	1024(%rdx){1to2}, %xmm17, %xmm26
+# CHECK: vpmullq	-1024(%rdx){1to2}, %xmm17, %xmm26
+# CHECK: encoding: [0x62,0x62,0xf5,0x10,0x40,0x52,0x80]
+         vpmullq	-1024(%rdx){1to2}, %xmm17, %xmm26
+# CHECK: vpmullq	-1032(%rdx){1to2}, %xmm17, %xmm26
+# CHECK: encoding: [0x62,0x62,0xf5,0x10,0x40,0x92,0xf8,0xfb,0xff,0xff]
+         vpmullq	-1032(%rdx){1to2}, %xmm17, %xmm26
+# CHECK: vpmullq	%ymm25, %ymm25, %ymm25
+# CHECK: encoding: [0x62,0x02,0xb5,0x20,0x40,0xc9]
+         vpmullq	%ymm25, %ymm25, %ymm25
+# CHECK: vpmullq	%ymm25, %ymm25, %ymm25 {%k3}
+# CHECK: encoding: [0x62,0x02,0xb5,0x23,0x40,0xc9]
+         vpmullq	%ymm25, %ymm25, %ymm25 {%k3}
+# CHECK: vpmullq	%ymm25, %ymm25, %ymm25 {%k3} {z}
+# CHECK: encoding: [0x62,0x02,0xb5,0xa3,0x40,0xc9]
+         vpmullq	%ymm25, %ymm25, %ymm25 {%k3} {z}
+# CHECK: vpmullq	(%rcx), %ymm25, %ymm25
+# CHECK: encoding: [0x62,0x62,0xb5,0x20,0x40,0x09]
+         vpmullq	(%rcx), %ymm25, %ymm25
+# CHECK: vpmullq	291(%rax,%r14,8), %ymm25, %ymm25
+# CHECK: encoding: [0x62,0x22,0xb5,0x20,0x40,0x8c,0xf0,0x23,0x01,0x00,0x00]
+         vpmullq	291(%rax,%r14,8), %ymm25, %ymm25
+# CHECK: vpmullq	(%rcx){1to4}, %ymm25, %ymm25
+# CHECK: encoding: [0x62,0x62,0xb5,0x30,0x40,0x09]
+         vpmullq	(%rcx){1to4}, %ymm25, %ymm25
+# CHECK: vpmullq	4064(%rdx), %ymm25, %ymm25
+# CHECK: encoding: [0x62,0x62,0xb5,0x20,0x40,0x4a,0x7f]
+         vpmullq	4064(%rdx), %ymm25, %ymm25
+# CHECK: vpmullq	4096(%rdx), %ymm25, %ymm25
+# CHECK: encoding: [0x62,0x62,0xb5,0x20,0x40,0x8a,0x00,0x10,0x00,0x00]
+         vpmullq	4096(%rdx), %ymm25, %ymm25
+# CHECK: vpmullq	-4096(%rdx), %ymm25, %ymm25
+# CHECK: encoding: [0x62,0x62,0xb5,0x20,0x40,0x4a,0x80]
+         vpmullq	-4096(%rdx), %ymm25, %ymm25
+# CHECK: vpmullq	-4128(%rdx), %ymm25, %ymm25
+# CHECK: encoding: [0x62,0x62,0xb5,0x20,0x40,0x8a,0xe0,0xef,0xff,0xff]
+         vpmullq	-4128(%rdx), %ymm25, %ymm25
+# CHECK: vpmullq	1016(%rdx){1to4}, %ymm25, %ymm25
+# CHECK: encoding: [0x62,0x62,0xb5,0x30,0x40,0x4a,0x7f]
+         vpmullq	1016(%rdx){1to4}, %ymm25, %ymm25
+# CHECK: vpmullq	1024(%rdx){1to4}, %ymm25, %ymm25
+# CHECK: encoding: [0x62,0x62,0xb5,0x30,0x40,0x8a,0x00,0x04,0x00,0x00]
+         vpmullq	1024(%rdx){1to4}, %ymm25, %ymm25
+# CHECK: vpmullq	-1024(%rdx){1to4}, %ymm25, %ymm25
+# CHECK: encoding: [0x62,0x62,0xb5,0x30,0x40,0x4a,0x80]
+         vpmullq	-1024(%rdx){1to4}, %ymm25, %ymm25
+# CHECK: vpmullq	-1032(%rdx){1to4}, %ymm25, %ymm25
+# CHECK: encoding: [0x62,0x62,0xb5,0x30,0x40,0x8a,0xf8,0xfb,0xff,0xff]
+         vpmullq	-1032(%rdx){1to4}, %ymm25, %ymm25
+# CHECK: vandpd	%xmm20, %xmm29, %xmm21
+# CHECK: encoding: [0x62,0xa1,0x95,0x00,0x54,0xec]
+         vandpd	%xmm20, %xmm29, %xmm21
+# CHECK: vandpd	%xmm20, %xmm29, %xmm21 {%k6}
+# CHECK: encoding: [0x62,0xa1,0x95,0x06,0x54,0xec]
+         vandpd	%xmm20, %xmm29, %xmm21 {%k6}
+# CHECK: vandpd	%xmm20, %xmm29, %xmm21 {%k6} {z}
+# CHECK: encoding: [0x62,0xa1,0x95,0x86,0x54,0xec]
+         vandpd	%xmm20, %xmm29, %xmm21 {%k6} {z}
+# CHECK: vandpd	(%rcx), %xmm29, %xmm21
+# CHECK: encoding: [0x62,0xe1,0x95,0x00,0x54,0x29]
+         vandpd	(%rcx), %xmm29, %xmm21
+# CHECK: vandpd	291(%rax,%r14,8), %xmm29, %xmm21
+# CHECK: encoding: [0x62,0xa1,0x95,0x00,0x54,0xac,0xf0,0x23,0x01,0x00,0x00]
+         vandpd	291(%rax,%r14,8), %xmm29, %xmm21
+# CHECK: vandpd	(%rcx){1to2}, %xmm29, %xmm21
+# CHECK: encoding: [0x62,0xe1,0x95,0x10,0x54,0x29]
+         vandpd	(%rcx){1to2}, %xmm29, %xmm21
+# CHECK: vandpd	2032(%rdx), %xmm29, %xmm21
+# CHECK: encoding: [0x62,0xe1,0x95,0x00,0x54,0x6a,0x7f]
+         vandpd	2032(%rdx), %xmm29, %xmm21
+# CHECK: vandpd	2048(%rdx), %xmm29, %xmm21
+# CHECK: encoding: [0x62,0xe1,0x95,0x00,0x54,0xaa,0x00,0x08,0x00,0x00]
+         vandpd	2048(%rdx), %xmm29, %xmm21
+# CHECK: vandpd	-2048(%rdx), %xmm29, %xmm21
+# CHECK: encoding: [0x62,0xe1,0x95,0x00,0x54,0x6a,0x80]
+         vandpd	-2048(%rdx), %xmm29, %xmm21
+# CHECK: vandpd	-2064(%rdx), %xmm29, %xmm21
+# CHECK: encoding: [0x62,0xe1,0x95,0x00,0x54,0xaa,0xf0,0xf7,0xff,0xff]
+         vandpd	-2064(%rdx), %xmm29, %xmm21
+# CHECK: vandpd	1016(%rdx){1to2}, %xmm29, %xmm21
+# CHECK: encoding: [0x62,0xe1,0x95,0x10,0x54,0x6a,0x7f]
+         vandpd	1016(%rdx){1to2}, %xmm29, %xmm21
+# CHECK: vandpd	1024(%rdx){1to2}, %xmm29, %xmm21
+# CHECK: encoding: [0x62,0xe1,0x95,0x10,0x54,0xaa,0x00,0x04,0x00,0x00]
+         vandpd	1024(%rdx){1to2}, %xmm29, %xmm21
+# CHECK: vandpd	-1024(%rdx){1to2}, %xmm29, %xmm21
+# CHECK: encoding: [0x62,0xe1,0x95,0x10,0x54,0x6a,0x80]
+         vandpd	-1024(%rdx){1to2}, %xmm29, %xmm21
+# CHECK: vandpd	-1032(%rdx){1to2}, %xmm29, %xmm21
+# CHECK: encoding: [0x62,0xe1,0x95,0x10,0x54,0xaa,0xf8,0xfb,0xff,0xff]
+         vandpd	-1032(%rdx){1to2}, %xmm29, %xmm21
+# CHECK: vandpd	%ymm28, %ymm21, %ymm28
+# CHECK: encoding: [0x62,0x01,0xd5,0x20,0x54,0xe4]
+         vandpd	%ymm28, %ymm21, %ymm28
+# CHECK: vandpd	%ymm28, %ymm21, %ymm28 {%k4}
+# CHECK: encoding: [0x62,0x01,0xd5,0x24,0x54,0xe4]
+         vandpd	%ymm28, %ymm21, %ymm28 {%k4}
+# CHECK: vandpd	%ymm28, %ymm21, %ymm28 {%k4} {z}
+# CHECK: encoding: [0x62,0x01,0xd5,0xa4,0x54,0xe4]
+         vandpd	%ymm28, %ymm21, %ymm28 {%k4} {z}
+# CHECK: vandpd	(%rcx), %ymm21, %ymm28
+# CHECK: encoding: [0x62,0x61,0xd5,0x20,0x54,0x21]
+         vandpd	(%rcx), %ymm21, %ymm28
+# CHECK: vandpd	291(%rax,%r14,8), %ymm21, %ymm28
+# CHECK: encoding: [0x62,0x21,0xd5,0x20,0x54,0xa4,0xf0,0x23,0x01,0x00,0x00]
+         vandpd	291(%rax,%r14,8), %ymm21, %ymm28
+# CHECK: vandpd	(%rcx){1to4}, %ymm21, %ymm28
+# CHECK: encoding: [0x62,0x61,0xd5,0x30,0x54,0x21]
+         vandpd	(%rcx){1to4}, %ymm21, %ymm28
+# CHECK: vandpd	4064(%rdx), %ymm21, %ymm28
+# CHECK: encoding: [0x62,0x61,0xd5,0x20,0x54,0x62,0x7f]
+         vandpd	4064(%rdx), %ymm21, %ymm28
+# CHECK: vandpd	4096(%rdx), %ymm21, %ymm28
+# CHECK: encoding: [0x62,0x61,0xd5,0x20,0x54,0xa2,0x00,0x10,0x00,0x00]
+         vandpd	4096(%rdx), %ymm21, %ymm28
+# CHECK: vandpd	-4096(%rdx), %ymm21, %ymm28
+# CHECK: encoding: [0x62,0x61,0xd5,0x20,0x54,0x62,0x80]
+         vandpd	-4096(%rdx), %ymm21, %ymm28
+# CHECK: vandpd	-4128(%rdx), %ymm21, %ymm28
+# CHECK: encoding: [0x62,0x61,0xd5,0x20,0x54,0xa2,0xe0,0xef,0xff,0xff]
+         vandpd	-4128(%rdx), %ymm21, %ymm28
+# CHECK: vandpd	1016(%rdx){1to4}, %ymm21, %ymm28
+# CHECK: encoding: [0x62,0x61,0xd5,0x30,0x54,0x62,0x7f]
+         vandpd	1016(%rdx){1to4}, %ymm21, %ymm28
+# CHECK: vandpd	1024(%rdx){1to4}, %ymm21, %ymm28
+# CHECK: encoding: [0x62,0x61,0xd5,0x30,0x54,0xa2,0x00,0x04,0x00,0x00]
+         vandpd	1024(%rdx){1to4}, %ymm21, %ymm28
+# CHECK: vandpd	-1024(%rdx){1to4}, %ymm21, %ymm28
+# CHECK: encoding: [0x62,0x61,0xd5,0x30,0x54,0x62,0x80]
+         vandpd	-1024(%rdx){1to4}, %ymm21, %ymm28
+# CHECK: vandpd	-1032(%rdx){1to4}, %ymm21, %ymm28
+# CHECK: encoding: [0x62,0x61,0xd5,0x30,0x54,0xa2,0xf8,0xfb,0xff,0xff]
+         vandpd	-1032(%rdx){1to4}, %ymm21, %ymm28
+# CHECK: vandps	%xmm24, %xmm21, %xmm23
+# CHECK: encoding: [0x62,0x81,0x54,0x00,0x54,0xf8]
+         vandps	%xmm24, %xmm21, %xmm23
+# CHECK: vandps	%xmm24, %xmm21, %xmm23 {%k5}
+# CHECK: encoding: [0x62,0x81,0x54,0x05,0x54,0xf8]
+         vandps	%xmm24, %xmm21, %xmm23 {%k5}
+# CHECK: vandps	%xmm24, %xmm21, %xmm23 {%k5} {z}
+# CHECK: encoding: [0x62,0x81,0x54,0x85,0x54,0xf8]
+         vandps	%xmm24, %xmm21, %xmm23 {%k5} {z}
+# CHECK: vandps	(%rcx), %xmm21, %xmm23
+# CHECK: encoding: [0x62,0xe1,0x54,0x00,0x54,0x39]
+         vandps	(%rcx), %xmm21, %xmm23
+# CHECK: vandps	291(%rax,%r14,8), %xmm21, %xmm23
+# CHECK: encoding: [0x62,0xa1,0x54,0x00,0x54,0xbc,0xf0,0x23,0x01,0x00,0x00]
+         vandps	291(%rax,%r14,8), %xmm21, %xmm23
+# CHECK: vandps	(%rcx){1to4}, %xmm21, %xmm23
+# CHECK: encoding: [0x62,0xe1,0x54,0x10,0x54,0x39]
+         vandps	(%rcx){1to4}, %xmm21, %xmm23
+# CHECK: vandps	2032(%rdx), %xmm21, %xmm23
+# CHECK: encoding: [0x62,0xe1,0x54,0x00,0x54,0x7a,0x7f]
+         vandps	2032(%rdx), %xmm21, %xmm23
+# CHECK: vandps	2048(%rdx), %xmm21, %xmm23
+# CHECK: encoding: [0x62,0xe1,0x54,0x00,0x54,0xba,0x00,0x08,0x00,0x00]
+         vandps	2048(%rdx), %xmm21, %xmm23
+# CHECK: vandps	-2048(%rdx), %xmm21, %xmm23
+# CHECK: encoding: [0x62,0xe1,0x54,0x00,0x54,0x7a,0x80]
+         vandps	-2048(%rdx), %xmm21, %xmm23
+# CHECK: vandps	-2064(%rdx), %xmm21, %xmm23
+# CHECK: encoding: [0x62,0xe1,0x54,0x00,0x54,0xba,0xf0,0xf7,0xff,0xff]
+         vandps	-2064(%rdx), %xmm21, %xmm23
+# CHECK: vandps	508(%rdx){1to4}, %xmm21, %xmm23
+# CHECK: encoding: [0x62,0xe1,0x54,0x10,0x54,0x7a,0x7f]
+         vandps	508(%rdx){1to4}, %xmm21, %xmm23
+# CHECK: vandps	512(%rdx){1to4}, %xmm21, %xmm23
+# CHECK: encoding: [0x62,0xe1,0x54,0x10,0x54,0xba,0x00,0x02,0x00,0x00]
+         vandps	512(%rdx){1to4}, %xmm21, %xmm23
+# CHECK: vandps	-512(%rdx){1to4}, %xmm21, %xmm23
+# CHECK: encoding: [0x62,0xe1,0x54,0x10,0x54,0x7a,0x80]
+         vandps	-512(%rdx){1to4}, %xmm21, %xmm23
+# CHECK: vandps	-516(%rdx){1to4}, %xmm21, %xmm23
+# CHECK: encoding: [0x62,0xe1,0x54,0x10,0x54,0xba,0xfc,0xfd,0xff,0xff]
+         vandps	-516(%rdx){1to4}, %xmm21, %xmm23
+# CHECK: vandps	%ymm23, %ymm18, %ymm26
+# CHECK: encoding: [0x62,0x21,0x6c,0x20,0x54,0xd7]
+         vandps	%ymm23, %ymm18, %ymm26
+# CHECK: vandps	%ymm23, %ymm18, %ymm26 {%k6}
+# CHECK: encoding: [0x62,0x21,0x6c,0x26,0x54,0xd7]
+         vandps	%ymm23, %ymm18, %ymm26 {%k6}
+# CHECK: vandps	%ymm23, %ymm18, %ymm26 {%k6} {z}
+# CHECK: encoding: [0x62,0x21,0x6c,0xa6,0x54,0xd7]
+         vandps	%ymm23, %ymm18, %ymm26 {%k6} {z}
+# CHECK: vandps	(%rcx), %ymm18, %ymm26
+# CHECK: encoding: [0x62,0x61,0x6c,0x20,0x54,0x11]
+         vandps	(%rcx), %ymm18, %ymm26
+# CHECK: vandps	291(%rax,%r14,8), %ymm18, %ymm26
+# CHECK: encoding: [0x62,0x21,0x6c,0x20,0x54,0x94,0xf0,0x23,0x01,0x00,0x00]
+         vandps	291(%rax,%r14,8), %ymm18, %ymm26
+# CHECK: vandps	(%rcx){1to8}, %ymm18, %ymm26
+# CHECK: encoding: [0x62,0x61,0x6c,0x30,0x54,0x11]
+         vandps	(%rcx){1to8}, %ymm18, %ymm26
+# CHECK: vandps	4064(%rdx), %ymm18, %ymm26
+# CHECK: encoding: [0x62,0x61,0x6c,0x20,0x54,0x52,0x7f]
+         vandps	4064(%rdx), %ymm18, %ymm26
+# CHECK: vandps	4096(%rdx), %ymm18, %ymm26
+# CHECK: encoding: [0x62,0x61,0x6c,0x20,0x54,0x92,0x00,0x10,0x00,0x00]
+         vandps	4096(%rdx), %ymm18, %ymm26
+# CHECK: vandps	-4096(%rdx), %ymm18, %ymm26
+# CHECK: encoding: [0x62,0x61,0x6c,0x20,0x54,0x52,0x80]
+         vandps	-4096(%rdx), %ymm18, %ymm26
+# CHECK: vandps	-4128(%rdx), %ymm18, %ymm26
+# CHECK: encoding: [0x62,0x61,0x6c,0x20,0x54,0x92,0xe0,0xef,0xff,0xff]
+         vandps	-4128(%rdx), %ymm18, %ymm26
+# CHECK: vandps	508(%rdx){1to8}, %ymm18, %ymm26
+# CHECK: encoding: [0x62,0x61,0x6c,0x30,0x54,0x52,0x7f]
+         vandps	508(%rdx){1to8}, %ymm18, %ymm26
+# CHECK: vandps	512(%rdx){1to8}, %ymm18, %ymm26
+# CHECK: encoding: [0x62,0x61,0x6c,0x30,0x54,0x92,0x00,0x02,0x00,0x00]
+         vandps	512(%rdx){1to8}, %ymm18, %ymm26
+# CHECK: vandps	-512(%rdx){1to8}, %ymm18, %ymm26
+# CHECK: encoding: [0x62,0x61,0x6c,0x30,0x54,0x52,0x80]
+         vandps	-512(%rdx){1to8}, %ymm18, %ymm26
+# CHECK: vandps	-516(%rdx){1to8}, %ymm18, %ymm26
+# CHECK: encoding: [0x62,0x61,0x6c,0x30,0x54,0x92,0xfc,0xfd,0xff,0xff]
+         vandps	-516(%rdx){1to8}, %ymm18, %ymm26
+# CHECK: vandnpd	%xmm25, %xmm27, %xmm25
+# CHECK: encoding: [0x62,0x01,0xa5,0x00,0x55,0xc9]
+         vandnpd	%xmm25, %xmm27, %xmm25
+# CHECK: vandnpd	%xmm25, %xmm27, %xmm25 {%k5}
+# CHECK: encoding: [0x62,0x01,0xa5,0x05,0x55,0xc9]
+         vandnpd	%xmm25, %xmm27, %xmm25 {%k5}
+# CHECK: vandnpd	%xmm25, %xmm27, %xmm25 {%k5} {z}
+# CHECK: encoding: [0x62,0x01,0xa5,0x85,0x55,0xc9]
+         vandnpd	%xmm25, %xmm27, %xmm25 {%k5} {z}
+# CHECK: vandnpd	(%rcx), %xmm27, %xmm25
+# CHECK: encoding: [0x62,0x61,0xa5,0x00,0x55,0x09]
+         vandnpd	(%rcx), %xmm27, %xmm25
+# CHECK: vandnpd	291(%rax,%r14,8), %xmm27, %xmm25
+# CHECK: encoding: [0x62,0x21,0xa5,0x00,0x55,0x8c,0xf0,0x23,0x01,0x00,0x00]
+         vandnpd	291(%rax,%r14,8), %xmm27, %xmm25
+# CHECK: vandnpd	(%rcx){1to2}, %xmm27, %xmm25
+# CHECK: encoding: [0x62,0x61,0xa5,0x10,0x55,0x09]
+         vandnpd	(%rcx){1to2}, %xmm27, %xmm25
+# CHECK: vandnpd	2032(%rdx), %xmm27, %xmm25
+# CHECK: encoding: [0x62,0x61,0xa5,0x00,0x55,0x4a,0x7f]
+         vandnpd	2032(%rdx), %xmm27, %xmm25
+# CHECK: vandnpd	2048(%rdx), %xmm27, %xmm25
+# CHECK: encoding: [0x62,0x61,0xa5,0x00,0x55,0x8a,0x00,0x08,0x00,0x00]
+         vandnpd	2048(%rdx), %xmm27, %xmm25
+# CHECK: vandnpd	-2048(%rdx), %xmm27, %xmm25
+# CHECK: encoding: [0x62,0x61,0xa5,0x00,0x55,0x4a,0x80]
+         vandnpd	-2048(%rdx), %xmm27, %xmm25
+# CHECK: vandnpd	-2064(%rdx), %xmm27, %xmm25
+# CHECK: encoding: [0x62,0x61,0xa5,0x00,0x55,0x8a,0xf0,0xf7,0xff,0xff]
+         vandnpd	-2064(%rdx), %xmm27, %xmm25
+# CHECK: vandnpd	1016(%rdx){1to2}, %xmm27, %xmm25
+# CHECK: encoding: [0x62,0x61,0xa5,0x10,0x55,0x4a,0x7f]
+         vandnpd	1016(%rdx){1to2}, %xmm27, %xmm25
+# CHECK: vandnpd	1024(%rdx){1to2}, %xmm27, %xmm25
+# CHECK: encoding: [0x62,0x61,0xa5,0x10,0x55,0x8a,0x00,0x04,0x00,0x00]
+         vandnpd	1024(%rdx){1to2}, %xmm27, %xmm25
+# CHECK: vandnpd	-1024(%rdx){1to2}, %xmm27, %xmm25
+# CHECK: encoding: [0x62,0x61,0xa5,0x10,0x55,0x4a,0x80]
+         vandnpd	-1024(%rdx){1to2}, %xmm27, %xmm25
+# CHECK: vandnpd	-1032(%rdx){1to2}, %xmm27, %xmm25
+# CHECK: encoding: [0x62,0x61,0xa5,0x10,0x55,0x8a,0xf8,0xfb,0xff,0xff]
+         vandnpd	-1032(%rdx){1to2}, %xmm27, %xmm25
+# CHECK: vandnpd	%ymm22, %ymm18, %ymm22
+# CHECK: encoding: [0x62,0xa1,0xed,0x20,0x55,0xf6]
+         vandnpd	%ymm22, %ymm18, %ymm22
+# CHECK: vandnpd	%ymm22, %ymm18, %ymm22 {%k7}
+# CHECK: encoding: [0x62,0xa1,0xed,0x27,0x55,0xf6]
+         vandnpd	%ymm22, %ymm18, %ymm22 {%k7}
+# CHECK: vandnpd	%ymm22, %ymm18, %ymm22 {%k7} {z}
+# CHECK: encoding: [0x62,0xa1,0xed,0xa7,0x55,0xf6]
+         vandnpd	%ymm22, %ymm18, %ymm22 {%k7} {z}
+# CHECK: vandnpd	(%rcx), %ymm18, %ymm22
+# CHECK: encoding: [0x62,0xe1,0xed,0x20,0x55,0x31]
+         vandnpd	(%rcx), %ymm18, %ymm22
+# CHECK: vandnpd	291(%rax,%r14,8), %ymm18, %ymm22
+# CHECK: encoding: [0x62,0xa1,0xed,0x20,0x55,0xb4,0xf0,0x23,0x01,0x00,0x00]
+         vandnpd	291(%rax,%r14,8), %ymm18, %ymm22
+# CHECK: vandnpd	(%rcx){1to4}, %ymm18, %ymm22
+# CHECK: encoding: [0x62,0xe1,0xed,0x30,0x55,0x31]
+         vandnpd	(%rcx){1to4}, %ymm18, %ymm22
+# CHECK: vandnpd	4064(%rdx), %ymm18, %ymm22
+# CHECK: encoding: [0x62,0xe1,0xed,0x20,0x55,0x72,0x7f]
+         vandnpd	4064(%rdx), %ymm18, %ymm22
+# CHECK: vandnpd	4096(%rdx), %ymm18, %ymm22
+# CHECK: encoding: [0x62,0xe1,0xed,0x20,0x55,0xb2,0x00,0x10,0x00,0x00]
+         vandnpd	4096(%rdx), %ymm18, %ymm22
+# CHECK: vandnpd	-4096(%rdx), %ymm18, %ymm22
+# CHECK: encoding: [0x62,0xe1,0xed,0x20,0x55,0x72,0x80]
+         vandnpd	-4096(%rdx), %ymm18, %ymm22
+# CHECK: vandnpd	-4128(%rdx), %ymm18, %ymm22
+# CHECK: encoding: [0x62,0xe1,0xed,0x20,0x55,0xb2,0xe0,0xef,0xff,0xff]
+         vandnpd	-4128(%rdx), %ymm18, %ymm22
+# CHECK: vandnpd	1016(%rdx){1to4}, %ymm18, %ymm22
+# CHECK: encoding: [0x62,0xe1,0xed,0x30,0x55,0x72,0x7f]
+         vandnpd	1016(%rdx){1to4}, %ymm18, %ymm22
+# CHECK: vandnpd	1024(%rdx){1to4}, %ymm18, %ymm22
+# CHECK: encoding: [0x62,0xe1,0xed,0x30,0x55,0xb2,0x00,0x04,0x00,0x00]
+         vandnpd	1024(%rdx){1to4}, %ymm18, %ymm22
+# CHECK: vandnpd	-1024(%rdx){1to4}, %ymm18, %ymm22
+# CHECK: encoding: [0x62,0xe1,0xed,0x30,0x55,0x72,0x80]
+         vandnpd	-1024(%rdx){1to4}, %ymm18, %ymm22
+# CHECK: vandnpd	-1032(%rdx){1to4}, %ymm18, %ymm22
+# CHECK: encoding: [0x62,0xe1,0xed,0x30,0x55,0xb2,0xf8,0xfb,0xff,0xff]
+         vandnpd	-1032(%rdx){1to4}, %ymm18, %ymm22
+# CHECK: vandnps	%xmm27, %xmm21, %xmm21
+# CHECK: encoding: [0x62,0x81,0x54,0x00,0x55,0xeb]
+         vandnps	%xmm27, %xmm21, %xmm21
+# CHECK: vandnps	%xmm27, %xmm21, %xmm21 {%k2}
+# CHECK: encoding: [0x62,0x81,0x54,0x02,0x55,0xeb]
+         vandnps	%xmm27, %xmm21, %xmm21 {%k2}
+# CHECK: vandnps	%xmm27, %xmm21, %xmm21 {%k2} {z}
+# CHECK: encoding: [0x62,0x81,0x54,0x82,0x55,0xeb]
+         vandnps	%xmm27, %xmm21, %xmm21 {%k2} {z}
+# CHECK: vandnps	(%rcx), %xmm21, %xmm21
+# CHECK: encoding: [0x62,0xe1,0x54,0x00,0x55,0x29]
+         vandnps	(%rcx), %xmm21, %xmm21
+# CHECK: vandnps	291(%rax,%r14,8), %xmm21, %xmm21
+# CHECK: encoding: [0x62,0xa1,0x54,0x00,0x55,0xac,0xf0,0x23,0x01,0x00,0x00]
+         vandnps	291(%rax,%r14,8), %xmm21, %xmm21
+# CHECK: vandnps	(%rcx){1to4}, %xmm21, %xmm21
+# CHECK: encoding: [0x62,0xe1,0x54,0x10,0x55,0x29]
+         vandnps	(%rcx){1to4}, %xmm21, %xmm21
+# CHECK: vandnps	2032(%rdx), %xmm21, %xmm21
+# CHECK: encoding: [0x62,0xe1,0x54,0x00,0x55,0x6a,0x7f]
+         vandnps	2032(%rdx), %xmm21, %xmm21
+# CHECK: vandnps	2048(%rdx), %xmm21, %xmm21
+# CHECK: encoding: [0x62,0xe1,0x54,0x00,0x55,0xaa,0x00,0x08,0x00,0x00]
+         vandnps	2048(%rdx), %xmm21, %xmm21
+# CHECK: vandnps	-2048(%rdx), %xmm21, %xmm21
+# CHECK: encoding: [0x62,0xe1,0x54,0x00,0x55,0x6a,0x80]
+         vandnps	-2048(%rdx), %xmm21, %xmm21
+# CHECK: vandnps	-2064(%rdx), %xmm21, %xmm21
+# CHECK: encoding: [0x62,0xe1,0x54,0x00,0x55,0xaa,0xf0,0xf7,0xff,0xff]
+         vandnps	-2064(%rdx), %xmm21, %xmm21
+# CHECK: vandnps	508(%rdx){1to4}, %xmm21, %xmm21
+# CHECK: encoding: [0x62,0xe1,0x54,0x10,0x55,0x6a,0x7f]
+         vandnps	508(%rdx){1to4}, %xmm21, %xmm21
+# CHECK: vandnps	512(%rdx){1to4}, %xmm21, %xmm21
+# CHECK: encoding: [0x62,0xe1,0x54,0x10,0x55,0xaa,0x00,0x02,0x00,0x00]
+         vandnps	512(%rdx){1to4}, %xmm21, %xmm21
+# CHECK: vandnps	-512(%rdx){1to4}, %xmm21, %xmm21
+# CHECK: encoding: [0x62,0xe1,0x54,0x10,0x55,0x6a,0x80]
+         vandnps	-512(%rdx){1to4}, %xmm21, %xmm21
+# CHECK: vandnps	-516(%rdx){1to4}, %xmm21, %xmm21
+# CHECK: encoding: [0x62,0xe1,0x54,0x10,0x55,0xaa,0xfc,0xfd,0xff,0xff]
+         vandnps	-516(%rdx){1to4}, %xmm21, %xmm21
+# CHECK: vandnps	%ymm25, %ymm23, %ymm19
+# CHECK: encoding: [0x62,0x81,0x44,0x20,0x55,0xd9]
+         vandnps	%ymm25, %ymm23, %ymm19
+# CHECK: vandnps	%ymm25, %ymm23, %ymm19 {%k1}
+# CHECK: encoding: [0x62,0x81,0x44,0x21,0x55,0xd9]
+         vandnps	%ymm25, %ymm23, %ymm19 {%k1}
+# CHECK: vandnps	%ymm25, %ymm23, %ymm19 {%k1} {z}
+# CHECK: encoding: [0x62,0x81,0x44,0xa1,0x55,0xd9]
+         vandnps	%ymm25, %ymm23, %ymm19 {%k1} {z}
+# CHECK: vandnps	(%rcx), %ymm23, %ymm19
+# CHECK: encoding: [0x62,0xe1,0x44,0x20,0x55,0x19]
+         vandnps	(%rcx), %ymm23, %ymm19
+# CHECK: vandnps	291(%rax,%r14,8), %ymm23, %ymm19
+# CHECK: encoding: [0x62,0xa1,0x44,0x20,0x55,0x9c,0xf0,0x23,0x01,0x00,0x00]
+         vandnps	291(%rax,%r14,8), %ymm23, %ymm19
+# CHECK: vandnps	(%rcx){1to8}, %ymm23, %ymm19
+# CHECK: encoding: [0x62,0xe1,0x44,0x30,0x55,0x19]
+         vandnps	(%rcx){1to8}, %ymm23, %ymm19
+# CHECK: vandnps	4064(%rdx), %ymm23, %ymm19
+# CHECK: encoding: [0x62,0xe1,0x44,0x20,0x55,0x5a,0x7f]
+         vandnps	4064(%rdx), %ymm23, %ymm19
+# CHECK: vandnps	4096(%rdx), %ymm23, %ymm19
+# CHECK: encoding: [0x62,0xe1,0x44,0x20,0x55,0x9a,0x00,0x10,0x00,0x00]
+         vandnps	4096(%rdx), %ymm23, %ymm19
+# CHECK: vandnps	-4096(%rdx), %ymm23, %ymm19
+# CHECK: encoding: [0x62,0xe1,0x44,0x20,0x55,0x5a,0x80]
+         vandnps	-4096(%rdx), %ymm23, %ymm19
+# CHECK: vandnps	-4128(%rdx), %ymm23, %ymm19
+# CHECK: encoding: [0x62,0xe1,0x44,0x20,0x55,0x9a,0xe0,0xef,0xff,0xff]
+         vandnps	-4128(%rdx), %ymm23, %ymm19
+# CHECK: vandnps	508(%rdx){1to8}, %ymm23, %ymm19
+# CHECK: encoding: [0x62,0xe1,0x44,0x30,0x55,0x5a,0x7f]
+         vandnps	508(%rdx){1to8}, %ymm23, %ymm19
+# CHECK: vandnps	512(%rdx){1to8}, %ymm23, %ymm19
+# CHECK: encoding: [0x62,0xe1,0x44,0x30,0x55,0x9a,0x00,0x02,0x00,0x00]
+         vandnps	512(%rdx){1to8}, %ymm23, %ymm19
+# CHECK: vandnps	-512(%rdx){1to8}, %ymm23, %ymm19
+# CHECK: encoding: [0x62,0xe1,0x44,0x30,0x55,0x5a,0x80]
+         vandnps	-512(%rdx){1to8}, %ymm23, %ymm19
+# CHECK: vandnps	-516(%rdx){1to8}, %ymm23, %ymm19
+# CHECK: encoding: [0x62,0xe1,0x44,0x30,0x55,0x9a,0xfc,0xfd,0xff,0xff]
+         vandnps	-516(%rdx){1to8}, %ymm23, %ymm19
+# CHECK: vorpd	%xmm18, %xmm27, %xmm23
+# CHECK: encoding: [0x62,0xa1,0xa5,0x00,0x56,0xfa]
+         vorpd	%xmm18, %xmm27, %xmm23
+# CHECK: vorpd	%xmm18, %xmm27, %xmm23 {%k1}
+# CHECK: encoding: [0x62,0xa1,0xa5,0x01,0x56,0xfa]
+         vorpd	%xmm18, %xmm27, %xmm23 {%k1}
+# CHECK: vorpd	%xmm18, %xmm27, %xmm23 {%k1} {z}
+# CHECK: encoding: [0x62,0xa1,0xa5,0x81,0x56,0xfa]
+         vorpd	%xmm18, %xmm27, %xmm23 {%k1} {z}
+# CHECK: vorpd	(%rcx), %xmm27, %xmm23
+# CHECK: encoding: [0x62,0xe1,0xa5,0x00,0x56,0x39]
+         vorpd	(%rcx), %xmm27, %xmm23
+# CHECK: vorpd	291(%rax,%r14,8), %xmm27, %xmm23
+# CHECK: encoding: [0x62,0xa1,0xa5,0x00,0x56,0xbc,0xf0,0x23,0x01,0x00,0x00]
+         vorpd	291(%rax,%r14,8), %xmm27, %xmm23
+# CHECK: vorpd	(%rcx){1to2}, %xmm27, %xmm23
+# CHECK: encoding: [0x62,0xe1,0xa5,0x10,0x56,0x39]
+         vorpd	(%rcx){1to2}, %xmm27, %xmm23
+# CHECK: vorpd	2032(%rdx), %xmm27, %xmm23
+# CHECK: encoding: [0x62,0xe1,0xa5,0x00,0x56,0x7a,0x7f]
+         vorpd	2032(%rdx), %xmm27, %xmm23
+# CHECK: vorpd	2048(%rdx), %xmm27, %xmm23
+# CHECK: encoding: [0x62,0xe1,0xa5,0x00,0x56,0xba,0x00,0x08,0x00,0x00]
+         vorpd	2048(%rdx), %xmm27, %xmm23
+# CHECK: vorpd	-2048(%rdx), %xmm27, %xmm23
+# CHECK: encoding: [0x62,0xe1,0xa5,0x00,0x56,0x7a,0x80]
+         vorpd	-2048(%rdx), %xmm27, %xmm23
+# CHECK: vorpd	-2064(%rdx), %xmm27, %xmm23
+# CHECK: encoding: [0x62,0xe1,0xa5,0x00,0x56,0xba,0xf0,0xf7,0xff,0xff]
+         vorpd	-2064(%rdx), %xmm27, %xmm23
+# CHECK: vorpd	1016(%rdx){1to2}, %xmm27, %xmm23
+# CHECK: encoding: [0x62,0xe1,0xa5,0x10,0x56,0x7a,0x7f]
+         vorpd	1016(%rdx){1to2}, %xmm27, %xmm23
+# CHECK: vorpd	1024(%rdx){1to2}, %xmm27, %xmm23
+# CHECK: encoding: [0x62,0xe1,0xa5,0x10,0x56,0xba,0x00,0x04,0x00,0x00]
+         vorpd	1024(%rdx){1to2}, %xmm27, %xmm23
+# CHECK: vorpd	-1024(%rdx){1to2}, %xmm27, %xmm23
+# CHECK: encoding: [0x62,0xe1,0xa5,0x10,0x56,0x7a,0x80]
+         vorpd	-1024(%rdx){1to2}, %xmm27, %xmm23
+# CHECK: vorpd	-1032(%rdx){1to2}, %xmm27, %xmm23
+# CHECK: encoding: [0x62,0xe1,0xa5,0x10,0x56,0xba,0xf8,0xfb,0xff,0xff]
+         vorpd	-1032(%rdx){1to2}, %xmm27, %xmm23
+# CHECK: vorpd	%ymm20, %ymm29, %ymm26
+# CHECK: encoding: [0x62,0x21,0x95,0x20,0x56,0xd4]
+         vorpd	%ymm20, %ymm29, %ymm26
+# CHECK: vorpd	%ymm20, %ymm29, %ymm26 {%k5}
+# CHECK: encoding: [0x62,0x21,0x95,0x25,0x56,0xd4]
+         vorpd	%ymm20, %ymm29, %ymm26 {%k5}
+# CHECK: vorpd	%ymm20, %ymm29, %ymm26 {%k5} {z}
+# CHECK: encoding: [0x62,0x21,0x95,0xa5,0x56,0xd4]
+         vorpd	%ymm20, %ymm29, %ymm26 {%k5} {z}
+# CHECK: vorpd	(%rcx), %ymm29, %ymm26
+# CHECK: encoding: [0x62,0x61,0x95,0x20,0x56,0x11]
+         vorpd	(%rcx), %ymm29, %ymm26
+# CHECK: vorpd	291(%rax,%r14,8), %ymm29, %ymm26
+# CHECK: encoding: [0x62,0x21,0x95,0x20,0x56,0x94,0xf0,0x23,0x01,0x00,0x00]
+         vorpd	291(%rax,%r14,8), %ymm29, %ymm26
+# CHECK: vorpd	(%rcx){1to4}, %ymm29, %ymm26
+# CHECK: encoding: [0x62,0x61,0x95,0x30,0x56,0x11]
+         vorpd	(%rcx){1to4}, %ymm29, %ymm26
+# CHECK: vorpd	4064(%rdx), %ymm29, %ymm26
+# CHECK: encoding: [0x62,0x61,0x95,0x20,0x56,0x52,0x7f]
+         vorpd	4064(%rdx), %ymm29, %ymm26
+# CHECK: vorpd	4096(%rdx), %ymm29, %ymm26
+# CHECK: encoding: [0x62,0x61,0x95,0x20,0x56,0x92,0x00,0x10,0x00,0x00]
+         vorpd	4096(%rdx), %ymm29, %ymm26
+# CHECK: vorpd	-4096(%rdx), %ymm29, %ymm26
+# CHECK: encoding: [0x62,0x61,0x95,0x20,0x56,0x52,0x80]
+         vorpd	-4096(%rdx), %ymm29, %ymm26
+# CHECK: vorpd	-4128(%rdx), %ymm29, %ymm26
+# CHECK: encoding: [0x62,0x61,0x95,0x20,0x56,0x92,0xe0,0xef,0xff,0xff]
+         vorpd	-4128(%rdx), %ymm29, %ymm26
+# CHECK: vorpd	1016(%rdx){1to4}, %ymm29, %ymm26
+# CHECK: encoding: [0x62,0x61,0x95,0x30,0x56,0x52,0x7f]
+         vorpd	1016(%rdx){1to4}, %ymm29, %ymm26
+# CHECK: vorpd	1024(%rdx){1to4}, %ymm29, %ymm26
+# CHECK: encoding: [0x62,0x61,0x95,0x30,0x56,0x92,0x00,0x04,0x00,0x00]
+         vorpd	1024(%rdx){1to4}, %ymm29, %ymm26
+# CHECK: vorpd	-1024(%rdx){1to4}, %ymm29, %ymm26
+# CHECK: encoding: [0x62,0x61,0x95,0x30,0x56,0x52,0x80]
+         vorpd	-1024(%rdx){1to4}, %ymm29, %ymm26
+# CHECK: vorpd	-1032(%rdx){1to4}, %ymm29, %ymm26
+# CHECK: encoding: [0x62,0x61,0x95,0x30,0x56,0x92,0xf8,0xfb,0xff,0xff]
+         vorpd	-1032(%rdx){1to4}, %ymm29, %ymm26
+# CHECK: vorps	%xmm27, %xmm28, %xmm19
+# CHECK: encoding: [0x62,0x81,0x1c,0x00,0x56,0xdb]
+         vorps	%xmm27, %xmm28, %xmm19
+# CHECK: vorps	%xmm27, %xmm28, %xmm19 {%k4}
+# CHECK: encoding: [0x62,0x81,0x1c,0x04,0x56,0xdb]
+         vorps	%xmm27, %xmm28, %xmm19 {%k4}
+# CHECK: vorps	%xmm27, %xmm28, %xmm19 {%k4} {z}
+# CHECK: encoding: [0x62,0x81,0x1c,0x84,0x56,0xdb]
+         vorps	%xmm27, %xmm28, %xmm19 {%k4} {z}
+# CHECK: vorps	(%rcx), %xmm28, %xmm19
+# CHECK: encoding: [0x62,0xe1,0x1c,0x00,0x56,0x19]
+         vorps	(%rcx), %xmm28, %xmm19
+# CHECK: vorps	291(%rax,%r14,8), %xmm28, %xmm19
+# CHECK: encoding: [0x62,0xa1,0x1c,0x00,0x56,0x9c,0xf0,0x23,0x01,0x00,0x00]
+         vorps	291(%rax,%r14,8), %xmm28, %xmm19
+# CHECK: vorps	(%rcx){1to4}, %xmm28, %xmm19
+# CHECK: encoding: [0x62,0xe1,0x1c,0x10,0x56,0x19]
+         vorps	(%rcx){1to4}, %xmm28, %xmm19
+# CHECK: vorps	2032(%rdx), %xmm28, %xmm19
+# CHECK: encoding: [0x62,0xe1,0x1c,0x00,0x56,0x5a,0x7f]
+         vorps	2032(%rdx), %xmm28, %xmm19
+# CHECK: vorps	2048(%rdx), %xmm28, %xmm19
+# CHECK: encoding: [0x62,0xe1,0x1c,0x00,0x56,0x9a,0x00,0x08,0x00,0x00]
+         vorps	2048(%rdx), %xmm28, %xmm19
+# CHECK: vorps	-2048(%rdx), %xmm28, %xmm19
+# CHECK: encoding: [0x62,0xe1,0x1c,0x00,0x56,0x5a,0x80]
+         vorps	-2048(%rdx), %xmm28, %xmm19
+# CHECK: vorps	-2064(%rdx), %xmm28, %xmm19
+# CHECK: encoding: [0x62,0xe1,0x1c,0x00,0x56,0x9a,0xf0,0xf7,0xff,0xff]
+         vorps	-2064(%rdx), %xmm28, %xmm19
+# CHECK: vorps	508(%rdx){1to4}, %xmm28, %xmm19
+# CHECK: encoding: [0x62,0xe1,0x1c,0x10,0x56,0x5a,0x7f]
+         vorps	508(%rdx){1to4}, %xmm28, %xmm19
+# CHECK: vorps	512(%rdx){1to4}, %xmm28, %xmm19
+# CHECK: encoding: [0x62,0xe1,0x1c,0x10,0x56,0x9a,0x00,0x02,0x00,0x00]
+         vorps	512(%rdx){1to4}, %xmm28, %xmm19
+# CHECK: vorps	-512(%rdx){1to4}, %xmm28, %xmm19
+# CHECK: encoding: [0x62,0xe1,0x1c,0x10,0x56,0x5a,0x80]
+         vorps	-512(%rdx){1to4}, %xmm28, %xmm19
+# CHECK: vorps	-516(%rdx){1to4}, %xmm28, %xmm19
+# CHECK: encoding: [0x62,0xe1,0x1c,0x10,0x56,0x9a,0xfc,0xfd,0xff,0xff]
+         vorps	-516(%rdx){1to4}, %xmm28, %xmm19
+# CHECK: vorps	%ymm26, %ymm26, %ymm27
+# CHECK: encoding: [0x62,0x01,0x2c,0x20,0x56,0xda]
+         vorps	%ymm26, %ymm26, %ymm27
+# CHECK: vorps	%ymm26, %ymm26, %ymm27 {%k1}
+# CHECK: encoding: [0x62,0x01,0x2c,0x21,0x56,0xda]
+         vorps	%ymm26, %ymm26, %ymm27 {%k1}
+# CHECK: vorps	%ymm26, %ymm26, %ymm27 {%k1} {z}
+# CHECK: encoding: [0x62,0x01,0x2c,0xa1,0x56,0xda]
+         vorps	%ymm26, %ymm26, %ymm27 {%k1} {z}
+# CHECK: vorps	(%rcx), %ymm26, %ymm27
+# CHECK: encoding: [0x62,0x61,0x2c,0x20,0x56,0x19]
+         vorps	(%rcx), %ymm26, %ymm27
+# CHECK: vorps	291(%rax,%r14,8), %ymm26, %ymm27
+# CHECK: encoding: [0x62,0x21,0x2c,0x20,0x56,0x9c,0xf0,0x23,0x01,0x00,0x00]
+         vorps	291(%rax,%r14,8), %ymm26, %ymm27
+# CHECK: vorps	(%rcx){1to8}, %ymm26, %ymm27
+# CHECK: encoding: [0x62,0x61,0x2c,0x30,0x56,0x19]
+         vorps	(%rcx){1to8}, %ymm26, %ymm27
+# CHECK: vorps	4064(%rdx), %ymm26, %ymm27
+# CHECK: encoding: [0x62,0x61,0x2c,0x20,0x56,0x5a,0x7f]
+         vorps	4064(%rdx), %ymm26, %ymm27
+# CHECK: vorps	4096(%rdx), %ymm26, %ymm27
+# CHECK: encoding: [0x62,0x61,0x2c,0x20,0x56,0x9a,0x00,0x10,0x00,0x00]
+         vorps	4096(%rdx), %ymm26, %ymm27
+# CHECK: vorps	-4096(%rdx), %ymm26, %ymm27
+# CHECK: encoding: [0x62,0x61,0x2c,0x20,0x56,0x5a,0x80]
+         vorps	-4096(%rdx), %ymm26, %ymm27
+# CHECK: vorps	-4128(%rdx), %ymm26, %ymm27
+# CHECK: encoding: [0x62,0x61,0x2c,0x20,0x56,0x9a,0xe0,0xef,0xff,0xff]
+         vorps	-4128(%rdx), %ymm26, %ymm27
+# CHECK: vorps	508(%rdx){1to8}, %ymm26, %ymm27
+# CHECK: encoding: [0x62,0x61,0x2c,0x30,0x56,0x5a,0x7f]
+         vorps	508(%rdx){1to8}, %ymm26, %ymm27
+# CHECK: vorps	512(%rdx){1to8}, %ymm26, %ymm27
+# CHECK: encoding: [0x62,0x61,0x2c,0x30,0x56,0x9a,0x00,0x02,0x00,0x00]
+         vorps	512(%rdx){1to8}, %ymm26, %ymm27
+# CHECK: vorps	-512(%rdx){1to8}, %ymm26, %ymm27
+# CHECK: encoding: [0x62,0x61,0x2c,0x30,0x56,0x5a,0x80]
+         vorps	-512(%rdx){1to8}, %ymm26, %ymm27
+# CHECK: vorps	-516(%rdx){1to8}, %ymm26, %ymm27
+# CHECK: encoding: [0x62,0x61,0x2c,0x30,0x56,0x9a,0xfc,0xfd,0xff,0xff]
+         vorps	-516(%rdx){1to8}, %ymm26, %ymm27
+# CHECK: vxorpd	%xmm23, %xmm21, %xmm18
+# CHECK: encoding: [0x62,0xa1,0xd5,0x00,0x57,0xd7]
+         vxorpd	%xmm23, %xmm21, %xmm18
+# CHECK: vxorpd	%xmm23, %xmm21, %xmm18 {%k2}
+# CHECK: encoding: [0x62,0xa1,0xd5,0x02,0x57,0xd7]
+         vxorpd	%xmm23, %xmm21, %xmm18 {%k2}
+# CHECK: vxorpd	%xmm23, %xmm21, %xmm18 {%k2} {z}
+# CHECK: encoding: [0x62,0xa1,0xd5,0x82,0x57,0xd7]
+         vxorpd	%xmm23, %xmm21, %xmm18 {%k2} {z}
+# CHECK: vxorpd	(%rcx), %xmm21, %xmm18
+# CHECK: encoding: [0x62,0xe1,0xd5,0x00,0x57,0x11]
+         vxorpd	(%rcx), %xmm21, %xmm18
+# CHECK: vxorpd	291(%rax,%r14,8), %xmm21, %xmm18
+# CHECK: encoding: [0x62,0xa1,0xd5,0x00,0x57,0x94,0xf0,0x23,0x01,0x00,0x00]
+         vxorpd	291(%rax,%r14,8), %xmm21, %xmm18
+# CHECK: vxorpd	(%rcx){1to2}, %xmm21, %xmm18
+# CHECK: encoding: [0x62,0xe1,0xd5,0x10,0x57,0x11]
+         vxorpd	(%rcx){1to2}, %xmm21, %xmm18
+# CHECK: vxorpd	2032(%rdx), %xmm21, %xmm18
+# CHECK: encoding: [0x62,0xe1,0xd5,0x00,0x57,0x52,0x7f]
+         vxorpd	2032(%rdx), %xmm21, %xmm18
+# CHECK: vxorpd	2048(%rdx), %xmm21, %xmm18
+# CHECK: encoding: [0x62,0xe1,0xd5,0x00,0x57,0x92,0x00,0x08,0x00,0x00]
+         vxorpd	2048(%rdx), %xmm21, %xmm18
+# CHECK: vxorpd	-2048(%rdx), %xmm21, %xmm18
+# CHECK: encoding: [0x62,0xe1,0xd5,0x00,0x57,0x52,0x80]
+         vxorpd	-2048(%rdx), %xmm21, %xmm18
+# CHECK: vxorpd	-2064(%rdx), %xmm21, %xmm18
+# CHECK: encoding: [0x62,0xe1,0xd5,0x00,0x57,0x92,0xf0,0xf7,0xff,0xff]
+         vxorpd	-2064(%rdx), %xmm21, %xmm18
+# CHECK: vxorpd	1016(%rdx){1to2}, %xmm21, %xmm18
+# CHECK: encoding: [0x62,0xe1,0xd5,0x10,0x57,0x52,0x7f]
+         vxorpd	1016(%rdx){1to2}, %xmm21, %xmm18
+# CHECK: vxorpd	1024(%rdx){1to2}, %xmm21, %xmm18
+# CHECK: encoding: [0x62,0xe1,0xd5,0x10,0x57,0x92,0x00,0x04,0x00,0x00]
+         vxorpd	1024(%rdx){1to2}, %xmm21, %xmm18
+# CHECK: vxorpd	-1024(%rdx){1to2}, %xmm21, %xmm18
+# CHECK: encoding: [0x62,0xe1,0xd5,0x10,0x57,0x52,0x80]
+         vxorpd	-1024(%rdx){1to2}, %xmm21, %xmm18
+# CHECK: vxorpd	-1032(%rdx){1to2}, %xmm21, %xmm18
+# CHECK: encoding: [0x62,0xe1,0xd5,0x10,0x57,0x92,0xf8,0xfb,0xff,0xff]
+         vxorpd	-1032(%rdx){1to2}, %xmm21, %xmm18
+# CHECK: vxorpd	%ymm19, %ymm29, %ymm24
+# CHECK: encoding: [0x62,0x21,0x95,0x20,0x57,0xc3]
+         vxorpd	%ymm19, %ymm29, %ymm24
+# CHECK: vxorpd	%ymm19, %ymm29, %ymm24 {%k7}
+# CHECK: encoding: [0x62,0x21,0x95,0x27,0x57,0xc3]
+         vxorpd	%ymm19, %ymm29, %ymm24 {%k7}
+# CHECK: vxorpd	%ymm19, %ymm29, %ymm24 {%k7} {z}
+# CHECK: encoding: [0x62,0x21,0x95,0xa7,0x57,0xc3]
+         vxorpd	%ymm19, %ymm29, %ymm24 {%k7} {z}
+# CHECK: vxorpd	(%rcx), %ymm29, %ymm24
+# CHECK: encoding: [0x62,0x61,0x95,0x20,0x57,0x01]
+         vxorpd	(%rcx), %ymm29, %ymm24
+# CHECK: vxorpd	291(%rax,%r14,8), %ymm29, %ymm24
+# CHECK: encoding: [0x62,0x21,0x95,0x20,0x57,0x84,0xf0,0x23,0x01,0x00,0x00]
+         vxorpd	291(%rax,%r14,8), %ymm29, %ymm24
+# CHECK: vxorpd	(%rcx){1to4}, %ymm29, %ymm24
+# CHECK: encoding: [0x62,0x61,0x95,0x30,0x57,0x01]
+         vxorpd	(%rcx){1to4}, %ymm29, %ymm24
+# CHECK: vxorpd	4064(%rdx), %ymm29, %ymm24
+# CHECK: encoding: [0x62,0x61,0x95,0x20,0x57,0x42,0x7f]
+         vxorpd	4064(%rdx), %ymm29, %ymm24
+# CHECK: vxorpd	4096(%rdx), %ymm29, %ymm24
+# CHECK: encoding: [0x62,0x61,0x95,0x20,0x57,0x82,0x00,0x10,0x00,0x00]
+         vxorpd	4096(%rdx), %ymm29, %ymm24
+# CHECK: vxorpd	-4096(%rdx), %ymm29, %ymm24
+# CHECK: encoding: [0x62,0x61,0x95,0x20,0x57,0x42,0x80]
+         vxorpd	-4096(%rdx), %ymm29, %ymm24
+# CHECK: vxorpd	-4128(%rdx), %ymm29, %ymm24
+# CHECK: encoding: [0x62,0x61,0x95,0x20,0x57,0x82,0xe0,0xef,0xff,0xff]
+         vxorpd	-4128(%rdx), %ymm29, %ymm24
+# CHECK: vxorpd	1016(%rdx){1to4}, %ymm29, %ymm24
+# CHECK: encoding: [0x62,0x61,0x95,0x30,0x57,0x42,0x7f]
+         vxorpd	1016(%rdx){1to4}, %ymm29, %ymm24
+# CHECK: vxorpd	1024(%rdx){1to4}, %ymm29, %ymm24
+# CHECK: encoding: [0x62,0x61,0x95,0x30,0x57,0x82,0x00,0x04,0x00,0x00]
+         vxorpd	1024(%rdx){1to4}, %ymm29, %ymm24
+# CHECK: vxorpd	-1024(%rdx){1to4}, %ymm29, %ymm24
+# CHECK: encoding: [0x62,0x61,0x95,0x30,0x57,0x42,0x80]
+         vxorpd	-1024(%rdx){1to4}, %ymm29, %ymm24
+# CHECK: vxorpd	-1032(%rdx){1to4}, %ymm29, %ymm24
+# CHECK: encoding: [0x62,0x61,0x95,0x30,0x57,0x82,0xf8,0xfb,0xff,0xff]
+         vxorpd	-1032(%rdx){1to4}, %ymm29, %ymm24
+# CHECK: vxorps	%xmm19, %xmm18, %xmm20
+# CHECK: encoding: [0x62,0xa1,0x6c,0x00,0x57,0xe3]
+         vxorps	%xmm19, %xmm18, %xmm20
+# CHECK: vxorps	%xmm19, %xmm18, %xmm20 {%k1}
+# CHECK: encoding: [0x62,0xa1,0x6c,0x01,0x57,0xe3]
+         vxorps	%xmm19, %xmm18, %xmm20 {%k1}
+# CHECK: vxorps	%xmm19, %xmm18, %xmm20 {%k1} {z}
+# CHECK: encoding: [0x62,0xa1,0x6c,0x81,0x57,0xe3]
+         vxorps	%xmm19, %xmm18, %xmm20 {%k1} {z}
+# CHECK: vxorps	(%rcx), %xmm18, %xmm20
+# CHECK: encoding: [0x62,0xe1,0x6c,0x00,0x57,0x21]
+         vxorps	(%rcx), %xmm18, %xmm20
+# CHECK: vxorps	291(%rax,%r14,8), %xmm18, %xmm20
+# CHECK: encoding: [0x62,0xa1,0x6c,0x00,0x57,0xa4,0xf0,0x23,0x01,0x00,0x00]
+         vxorps	291(%rax,%r14,8), %xmm18, %xmm20
+# CHECK: vxorps	(%rcx){1to4}, %xmm18, %xmm20
+# CHECK: encoding: [0x62,0xe1,0x6c,0x10,0x57,0x21]
+         vxorps	(%rcx){1to4}, %xmm18, %xmm20
+# CHECK: vxorps	2032(%rdx), %xmm18, %xmm20
+# CHECK: encoding: [0x62,0xe1,0x6c,0x00,0x57,0x62,0x7f]
+         vxorps	2032(%rdx), %xmm18, %xmm20
+# CHECK: vxorps	2048(%rdx), %xmm18, %xmm20
+# CHECK: encoding: [0x62,0xe1,0x6c,0x00,0x57,0xa2,0x00,0x08,0x00,0x00]
+         vxorps	2048(%rdx), %xmm18, %xmm20
+# CHECK: vxorps	-2048(%rdx), %xmm18, %xmm20
+# CHECK: encoding: [0x62,0xe1,0x6c,0x00,0x57,0x62,0x80]
+         vxorps	-2048(%rdx), %xmm18, %xmm20
+# CHECK: vxorps	-2064(%rdx), %xmm18, %xmm20
+# CHECK: encoding: [0x62,0xe1,0x6c,0x00,0x57,0xa2,0xf0,0xf7,0xff,0xff]
+         vxorps	-2064(%rdx), %xmm18, %xmm20
+# CHECK: vxorps	508(%rdx){1to4}, %xmm18, %xmm20
+# CHECK: encoding: [0x62,0xe1,0x6c,0x10,0x57,0x62,0x7f]
+         vxorps	508(%rdx){1to4}, %xmm18, %xmm20
+# CHECK: vxorps	512(%rdx){1to4}, %xmm18, %xmm20
+# CHECK: encoding: [0x62,0xe1,0x6c,0x10,0x57,0xa2,0x00,0x02,0x00,0x00]
+         vxorps	512(%rdx){1to4}, %xmm18, %xmm20
+# CHECK: vxorps	-512(%rdx){1to4}, %xmm18, %xmm20
+# CHECK: encoding: [0x62,0xe1,0x6c,0x10,0x57,0x62,0x80]
+         vxorps	-512(%rdx){1to4}, %xmm18, %xmm20
+# CHECK: vxorps	-516(%rdx){1to4}, %xmm18, %xmm20
+# CHECK: encoding: [0x62,0xe1,0x6c,0x10,0x57,0xa2,0xfc,0xfd,0xff,0xff]
+         vxorps	-516(%rdx){1to4}, %xmm18, %xmm20
+# CHECK: vxorps	%ymm24, %ymm20, %ymm27
+# CHECK: encoding: [0x62,0x01,0x5c,0x20,0x57,0xd8]
+         vxorps	%ymm24, %ymm20, %ymm27
+# CHECK: vxorps	%ymm24, %ymm20, %ymm27 {%k2}
+# CHECK: encoding: [0x62,0x01,0x5c,0x22,0x57,0xd8]
+         vxorps	%ymm24, %ymm20, %ymm27 {%k2}
+# CHECK: vxorps	%ymm24, %ymm20, %ymm27 {%k2} {z}
+# CHECK: encoding: [0x62,0x01,0x5c,0xa2,0x57,0xd8]
+         vxorps	%ymm24, %ymm20, %ymm27 {%k2} {z}
+# CHECK: vxorps	(%rcx), %ymm20, %ymm27
+# CHECK: encoding: [0x62,0x61,0x5c,0x20,0x57,0x19]
+         vxorps	(%rcx), %ymm20, %ymm27
+# CHECK: vxorps	291(%rax,%r14,8), %ymm20, %ymm27
+# CHECK: encoding: [0x62,0x21,0x5c,0x20,0x57,0x9c,0xf0,0x23,0x01,0x00,0x00]
+         vxorps	291(%rax,%r14,8), %ymm20, %ymm27
+# CHECK: vxorps	(%rcx){1to8}, %ymm20, %ymm27
+# CHECK: encoding: [0x62,0x61,0x5c,0x30,0x57,0x19]
+         vxorps	(%rcx){1to8}, %ymm20, %ymm27
+# CHECK: vxorps	4064(%rdx), %ymm20, %ymm27
+# CHECK: encoding: [0x62,0x61,0x5c,0x20,0x57,0x5a,0x7f]
+         vxorps	4064(%rdx), %ymm20, %ymm27
+# CHECK: vxorps	4096(%rdx), %ymm20, %ymm27
+# CHECK: encoding: [0x62,0x61,0x5c,0x20,0x57,0x9a,0x00,0x10,0x00,0x00]
+         vxorps	4096(%rdx), %ymm20, %ymm27
+# CHECK: vxorps	-4096(%rdx), %ymm20, %ymm27
+# CHECK: encoding: [0x62,0x61,0x5c,0x20,0x57,0x5a,0x80]
+         vxorps	-4096(%rdx), %ymm20, %ymm27
+# CHECK: vxorps	-4128(%rdx), %ymm20, %ymm27
+# CHECK: encoding: [0x62,0x61,0x5c,0x20,0x57,0x9a,0xe0,0xef,0xff,0xff]
+         vxorps	-4128(%rdx), %ymm20, %ymm27
+# CHECK: vxorps	508(%rdx){1to8}, %ymm20, %ymm27
+# CHECK: encoding: [0x62,0x61,0x5c,0x30,0x57,0x5a,0x7f]
+         vxorps	508(%rdx){1to8}, %ymm20, %ymm27
+# CHECK: vxorps	512(%rdx){1to8}, %ymm20, %ymm27
+# CHECK: encoding: [0x62,0x61,0x5c,0x30,0x57,0x9a,0x00,0x02,0x00,0x00]
+         vxorps	512(%rdx){1to8}, %ymm20, %ymm27
+# CHECK: vxorps	-512(%rdx){1to8}, %ymm20, %ymm27
+# CHECK: encoding: [0x62,0x61,0x5c,0x30,0x57,0x5a,0x80]
+         vxorps	-512(%rdx){1to8}, %ymm20, %ymm27
+# CHECK: vxorps	-516(%rdx){1to8}, %ymm20, %ymm27
+# CHECK: encoding: [0x62,0x61,0x5c,0x30,0x57,0x9a,0xfc,0xfd,0xff,0xff]
+         vxorps	-516(%rdx){1to8}, %ymm20, %ymm27
+# CHECK: vandpd	%xmm27, %xmm25, %xmm19
+# CHECK: encoding: [0x62,0x81,0xb5,0x00,0x54,0xdb]
+         vandpd	%xmm27, %xmm25, %xmm19
+# CHECK: vandpd	%xmm27, %xmm25, %xmm19 {%k6}
+# CHECK: encoding: [0x62,0x81,0xb5,0x06,0x54,0xdb]
+         vandpd	%xmm27, %xmm25, %xmm19 {%k6}
+# CHECK: vandpd	%xmm27, %xmm25, %xmm19 {%k6} {z}
+# CHECK: encoding: [0x62,0x81,0xb5,0x86,0x54,0xdb]
+         vandpd	%xmm27, %xmm25, %xmm19 {%k6} {z}
+# CHECK: vandpd	(%rcx), %xmm25, %xmm19
+# CHECK: encoding: [0x62,0xe1,0xb5,0x00,0x54,0x19]
+         vandpd	(%rcx), %xmm25, %xmm19
+# CHECK: vandpd	4660(%rax,%r14,8), %xmm25, %xmm19
+# CHECK: encoding: [0x62,0xa1,0xb5,0x00,0x54,0x9c,0xf0,0x34,0x12,0x00,0x00]
+         vandpd	4660(%rax,%r14,8), %xmm25, %xmm19
+# CHECK: vandpd	(%rcx){1to2}, %xmm25, %xmm19
+# CHECK: encoding: [0x62,0xe1,0xb5,0x10,0x54,0x19]
+         vandpd	(%rcx){1to2}, %xmm25, %xmm19
+# CHECK: vandpd	2032(%rdx), %xmm25, %xmm19
+# CHECK: encoding: [0x62,0xe1,0xb5,0x00,0x54,0x5a,0x7f]
+         vandpd	2032(%rdx), %xmm25, %xmm19
+# CHECK: vandpd	2048(%rdx), %xmm25, %xmm19
+# CHECK: encoding: [0x62,0xe1,0xb5,0x00,0x54,0x9a,0x00,0x08,0x00,0x00]
+         vandpd	2048(%rdx), %xmm25, %xmm19
+# CHECK: vandpd	-2048(%rdx), %xmm25, %xmm19
+# CHECK: encoding: [0x62,0xe1,0xb5,0x00,0x54,0x5a,0x80]
+         vandpd	-2048(%rdx), %xmm25, %xmm19
+# CHECK: vandpd	-2064(%rdx), %xmm25, %xmm19
+# CHECK: encoding: [0x62,0xe1,0xb5,0x00,0x54,0x9a,0xf0,0xf7,0xff,0xff]
+         vandpd	-2064(%rdx), %xmm25, %xmm19
+# CHECK: vandpd	1016(%rdx){1to2}, %xmm25, %xmm19
+# CHECK: encoding: [0x62,0xe1,0xb5,0x10,0x54,0x5a,0x7f]
+         vandpd	1016(%rdx){1to2}, %xmm25, %xmm19
+# CHECK: vandpd	1024(%rdx){1to2}, %xmm25, %xmm19
+# CHECK: encoding: [0x62,0xe1,0xb5,0x10,0x54,0x9a,0x00,0x04,0x00,0x00]
+         vandpd	1024(%rdx){1to2}, %xmm25, %xmm19
+# CHECK: vandpd	-1024(%rdx){1to2}, %xmm25, %xmm19
+# CHECK: encoding: [0x62,0xe1,0xb5,0x10,0x54,0x5a,0x80]
+         vandpd	-1024(%rdx){1to2}, %xmm25, %xmm19
+# CHECK: vandpd	-1032(%rdx){1to2}, %xmm25, %xmm19
+# CHECK: encoding: [0x62,0xe1,0xb5,0x10,0x54,0x9a,0xf8,0xfb,0xff,0xff]
+         vandpd	-1032(%rdx){1to2}, %xmm25, %xmm19
+# CHECK: vandpd	%ymm21, %ymm29, %ymm24
+# CHECK: encoding: [0x62,0x21,0x95,0x20,0x54,0xc5]
+         vandpd	%ymm21, %ymm29, %ymm24
+# CHECK: vandpd	%ymm21, %ymm29, %ymm24 {%k2}
+# CHECK: encoding: [0x62,0x21,0x95,0x22,0x54,0xc5]
+         vandpd	%ymm21, %ymm29, %ymm24 {%k2}
+# CHECK: vandpd	%ymm21, %ymm29, %ymm24 {%k2} {z}
+# CHECK: encoding: [0x62,0x21,0x95,0xa2,0x54,0xc5]
+         vandpd	%ymm21, %ymm29, %ymm24 {%k2} {z}
+# CHECK: vandpd	(%rcx), %ymm29, %ymm24
+# CHECK: encoding: [0x62,0x61,0x95,0x20,0x54,0x01]
+         vandpd	(%rcx), %ymm29, %ymm24
+# CHECK: vandpd	4660(%rax,%r14,8), %ymm29, %ymm24
+# CHECK: encoding: [0x62,0x21,0x95,0x20,0x54,0x84,0xf0,0x34,0x12,0x00,0x00]
+         vandpd	4660(%rax,%r14,8), %ymm29, %ymm24
+# CHECK: vandpd	(%rcx){1to4}, %ymm29, %ymm24
+# CHECK: encoding: [0x62,0x61,0x95,0x30,0x54,0x01]
+         vandpd	(%rcx){1to4}, %ymm29, %ymm24
+# CHECK: vandpd	4064(%rdx), %ymm29, %ymm24
+# CHECK: encoding: [0x62,0x61,0x95,0x20,0x54,0x42,0x7f]
+         vandpd	4064(%rdx), %ymm29, %ymm24
+# CHECK: vandpd	4096(%rdx), %ymm29, %ymm24
+# CHECK: encoding: [0x62,0x61,0x95,0x20,0x54,0x82,0x00,0x10,0x00,0x00]
+         vandpd	4096(%rdx), %ymm29, %ymm24
+# CHECK: vandpd	-4096(%rdx), %ymm29, %ymm24
+# CHECK: encoding: [0x62,0x61,0x95,0x20,0x54,0x42,0x80]
+         vandpd	-4096(%rdx), %ymm29, %ymm24
+# CHECK: vandpd	-4128(%rdx), %ymm29, %ymm24
+# CHECK: encoding: [0x62,0x61,0x95,0x20,0x54,0x82,0xe0,0xef,0xff,0xff]
+         vandpd	-4128(%rdx), %ymm29, %ymm24
+# CHECK: vandpd	1016(%rdx){1to4}, %ymm29, %ymm24
+# CHECK: encoding: [0x62,0x61,0x95,0x30,0x54,0x42,0x7f]
+         vandpd	1016(%rdx){1to4}, %ymm29, %ymm24
+# CHECK: vandpd	1024(%rdx){1to4}, %ymm29, %ymm24
+# CHECK: encoding: [0x62,0x61,0x95,0x30,0x54,0x82,0x00,0x04,0x00,0x00]
+         vandpd	1024(%rdx){1to4}, %ymm29, %ymm24
+# CHECK: vandpd	-1024(%rdx){1to4}, %ymm29, %ymm24
+# CHECK: encoding: [0x62,0x61,0x95,0x30,0x54,0x42,0x80]
+         vandpd	-1024(%rdx){1to4}, %ymm29, %ymm24
+# CHECK: vandpd	-1032(%rdx){1to4}, %ymm29, %ymm24
+# CHECK: encoding: [0x62,0x61,0x95,0x30,0x54,0x82,0xf8,0xfb,0xff,0xff]
+         vandpd	-1032(%rdx){1to4}, %ymm29, %ymm24
+# CHECK: vandps	%xmm17, %xmm25, %xmm22
+# CHECK: encoding: [0x62,0xa1,0x34,0x00,0x54,0xf1]
+         vandps	%xmm17, %xmm25, %xmm22
+# CHECK: vandps	%xmm17, %xmm25, %xmm22 {%k3}
+# CHECK: encoding: [0x62,0xa1,0x34,0x03,0x54,0xf1]
+         vandps	%xmm17, %xmm25, %xmm22 {%k3}
+# CHECK: vandps	%xmm17, %xmm25, %xmm22 {%k3} {z}
+# CHECK: encoding: [0x62,0xa1,0x34,0x83,0x54,0xf1]
+         vandps	%xmm17, %xmm25, %xmm22 {%k3} {z}
+# CHECK: vandps	(%rcx), %xmm25, %xmm22
+# CHECK: encoding: [0x62,0xe1,0x34,0x00,0x54,0x31]
+         vandps	(%rcx), %xmm25, %xmm22
+# CHECK: vandps	4660(%rax,%r14,8), %xmm25, %xmm22
+# CHECK: encoding: [0x62,0xa1,0x34,0x00,0x54,0xb4,0xf0,0x34,0x12,0x00,0x00]
+         vandps	4660(%rax,%r14,8), %xmm25, %xmm22
+# CHECK: vandps	(%rcx){1to4}, %xmm25, %xmm22
+# CHECK: encoding: [0x62,0xe1,0x34,0x10,0x54,0x31]
+         vandps	(%rcx){1to4}, %xmm25, %xmm22
+# CHECK: vandps	2032(%rdx), %xmm25, %xmm22
+# CHECK: encoding: [0x62,0xe1,0x34,0x00,0x54,0x72,0x7f]
+         vandps	2032(%rdx), %xmm25, %xmm22
+# CHECK: vandps	2048(%rdx), %xmm25, %xmm22
+# CHECK: encoding: [0x62,0xe1,0x34,0x00,0x54,0xb2,0x00,0x08,0x00,0x00]
+         vandps	2048(%rdx), %xmm25, %xmm22
+# CHECK: vandps	-2048(%rdx), %xmm25, %xmm22
+# CHECK: encoding: [0x62,0xe1,0x34,0x00,0x54,0x72,0x80]
+         vandps	-2048(%rdx), %xmm25, %xmm22
+# CHECK: vandps	-2064(%rdx), %xmm25, %xmm22
+# CHECK: encoding: [0x62,0xe1,0x34,0x00,0x54,0xb2,0xf0,0xf7,0xff,0xff]
+         vandps	-2064(%rdx), %xmm25, %xmm22
+# CHECK: vandps	508(%rdx){1to4}, %xmm25, %xmm22
+# CHECK: encoding: [0x62,0xe1,0x34,0x10,0x54,0x72,0x7f]
+         vandps	508(%rdx){1to4}, %xmm25, %xmm22
+# CHECK: vandps	512(%rdx){1to4}, %xmm25, %xmm22
+# CHECK: encoding: [0x62,0xe1,0x34,0x10,0x54,0xb2,0x00,0x02,0x00,0x00]
+         vandps	512(%rdx){1to4}, %xmm25, %xmm22
+# CHECK: vandps	-512(%rdx){1to4}, %xmm25, %xmm22
+# CHECK: encoding: [0x62,0xe1,0x34,0x10,0x54,0x72,0x80]
+         vandps	-512(%rdx){1to4}, %xmm25, %xmm22
+# CHECK: vandps	-516(%rdx){1to4}, %xmm25, %xmm22
+# CHECK: encoding: [0x62,0xe1,0x34,0x10,0x54,0xb2,0xfc,0xfd,0xff,0xff]
+         vandps	-516(%rdx){1to4}, %xmm25, %xmm22
+# CHECK: vandps	%ymm18, %ymm25, %ymm22
+# CHECK: encoding: [0x62,0xa1,0x34,0x20,0x54,0xf2]
+         vandps	%ymm18, %ymm25, %ymm22
+# CHECK: vandps	%ymm18, %ymm25, %ymm22 {%k1}
+# CHECK: encoding: [0x62,0xa1,0x34,0x21,0x54,0xf2]
+         vandps	%ymm18, %ymm25, %ymm22 {%k1}
+# CHECK: vandps	%ymm18, %ymm25, %ymm22 {%k1} {z}
+# CHECK: encoding: [0x62,0xa1,0x34,0xa1,0x54,0xf2]
+         vandps	%ymm18, %ymm25, %ymm22 {%k1} {z}
+# CHECK: vandps	(%rcx), %ymm25, %ymm22
+# CHECK: encoding: [0x62,0xe1,0x34,0x20,0x54,0x31]
+         vandps	(%rcx), %ymm25, %ymm22
+# CHECK: vandps	4660(%rax,%r14,8), %ymm25, %ymm22
+# CHECK: encoding: [0x62,0xa1,0x34,0x20,0x54,0xb4,0xf0,0x34,0x12,0x00,0x00]
+         vandps	4660(%rax,%r14,8), %ymm25, %ymm22
+# CHECK: vandps	(%rcx){1to8}, %ymm25, %ymm22
+# CHECK: encoding: [0x62,0xe1,0x34,0x30,0x54,0x31]
+         vandps	(%rcx){1to8}, %ymm25, %ymm22
+# CHECK: vandps	4064(%rdx), %ymm25, %ymm22
+# CHECK: encoding: [0x62,0xe1,0x34,0x20,0x54,0x72,0x7f]
+         vandps	4064(%rdx), %ymm25, %ymm22
+# CHECK: vandps	4096(%rdx), %ymm25, %ymm22
+# CHECK: encoding: [0x62,0xe1,0x34,0x20,0x54,0xb2,0x00,0x10,0x00,0x00]
+         vandps	4096(%rdx), %ymm25, %ymm22
+# CHECK: vandps	-4096(%rdx), %ymm25, %ymm22
+# CHECK: encoding: [0x62,0xe1,0x34,0x20,0x54,0x72,0x80]
+         vandps	-4096(%rdx), %ymm25, %ymm22
+# CHECK: vandps	-4128(%rdx), %ymm25, %ymm22
+# CHECK: encoding: [0x62,0xe1,0x34,0x20,0x54,0xb2,0xe0,0xef,0xff,0xff]
+         vandps	-4128(%rdx), %ymm25, %ymm22
+# CHECK: vandps	508(%rdx){1to8}, %ymm25, %ymm22
+# CHECK: encoding: [0x62,0xe1,0x34,0x30,0x54,0x72,0x7f]
+         vandps	508(%rdx){1to8}, %ymm25, %ymm22
+# CHECK: vandps	512(%rdx){1to8}, %ymm25, %ymm22
+# CHECK: encoding: [0x62,0xe1,0x34,0x30,0x54,0xb2,0x00,0x02,0x00,0x00]
+         vandps	512(%rdx){1to8}, %ymm25, %ymm22
+# CHECK: vandps	-512(%rdx){1to8}, %ymm25, %ymm22
+# CHECK: encoding: [0x62,0xe1,0x34,0x30,0x54,0x72,0x80]
+         vandps	-512(%rdx){1to8}, %ymm25, %ymm22
+# CHECK: vandps	-516(%rdx){1to8}, %ymm25, %ymm22
+# CHECK: encoding: [0x62,0xe1,0x34,0x30,0x54,0xb2,0xfc,0xfd,0xff,0xff]
+         vandps	-516(%rdx){1to8}, %ymm25, %ymm22
+# CHECK: vandnpd	%xmm23, %xmm18, %xmm19
+# CHECK: encoding: [0x62,0xa1,0xed,0x00,0x55,0xdf]
+         vandnpd	%xmm23, %xmm18, %xmm19
+# CHECK: vandnpd	%xmm23, %xmm18, %xmm19 {%k1}
+# CHECK: encoding: [0x62,0xa1,0xed,0x01,0x55,0xdf]
+         vandnpd	%xmm23, %xmm18, %xmm19 {%k1}
+# CHECK: vandnpd	%xmm23, %xmm18, %xmm19 {%k1} {z}
+# CHECK: encoding: [0x62,0xa1,0xed,0x81,0x55,0xdf]
+         vandnpd	%xmm23, %xmm18, %xmm19 {%k1} {z}
+# CHECK: vandnpd	(%rcx), %xmm18, %xmm19
+# CHECK: encoding: [0x62,0xe1,0xed,0x00,0x55,0x19]
+         vandnpd	(%rcx), %xmm18, %xmm19
+# CHECK: vandnpd	4660(%rax,%r14,8), %xmm18, %xmm19
+# CHECK: encoding: [0x62,0xa1,0xed,0x00,0x55,0x9c,0xf0,0x34,0x12,0x00,0x00]
+         vandnpd	4660(%rax,%r14,8), %xmm18, %xmm19
+# CHECK: vandnpd	(%rcx){1to2}, %xmm18, %xmm19
+# CHECK: encoding: [0x62,0xe1,0xed,0x10,0x55,0x19]
+         vandnpd	(%rcx){1to2}, %xmm18, %xmm19
+# CHECK: vandnpd	2032(%rdx), %xmm18, %xmm19
+# CHECK: encoding: [0x62,0xe1,0xed,0x00,0x55,0x5a,0x7f]
+         vandnpd	2032(%rdx), %xmm18, %xmm19
+# CHECK: vandnpd	2048(%rdx), %xmm18, %xmm19
+# CHECK: encoding: [0x62,0xe1,0xed,0x00,0x55,0x9a,0x00,0x08,0x00,0x00]
+         vandnpd	2048(%rdx), %xmm18, %xmm19
+# CHECK: vandnpd	-2048(%rdx), %xmm18, %xmm19
+# CHECK: encoding: [0x62,0xe1,0xed,0x00,0x55,0x5a,0x80]
+         vandnpd	-2048(%rdx), %xmm18, %xmm19
+# CHECK: vandnpd	-2064(%rdx), %xmm18, %xmm19
+# CHECK: encoding: [0x62,0xe1,0xed,0x00,0x55,0x9a,0xf0,0xf7,0xff,0xff]
+         vandnpd	-2064(%rdx), %xmm18, %xmm19
+# CHECK: vandnpd	1016(%rdx){1to2}, %xmm18, %xmm19
+# CHECK: encoding: [0x62,0xe1,0xed,0x10,0x55,0x5a,0x7f]
+         vandnpd	1016(%rdx){1to2}, %xmm18, %xmm19
+# CHECK: vandnpd	1024(%rdx){1to2}, %xmm18, %xmm19
+# CHECK: encoding: [0x62,0xe1,0xed,0x10,0x55,0x9a,0x00,0x04,0x00,0x00]
+         vandnpd	1024(%rdx){1to2}, %xmm18, %xmm19
+# CHECK: vandnpd	-1024(%rdx){1to2}, %xmm18, %xmm19
+# CHECK: encoding: [0x62,0xe1,0xed,0x10,0x55,0x5a,0x80]
+         vandnpd	-1024(%rdx){1to2}, %xmm18, %xmm19
+# CHECK: vandnpd	-1032(%rdx){1to2}, %xmm18, %xmm19
+# CHECK: encoding: [0x62,0xe1,0xed,0x10,0x55,0x9a,0xf8,0xfb,0xff,0xff]
+         vandnpd	-1032(%rdx){1to2}, %xmm18, %xmm19
+# CHECK: vandnpd	%ymm28, %ymm29, %ymm25
+# CHECK: encoding: [0x62,0x01,0x95,0x20,0x55,0xcc]
+         vandnpd	%ymm28, %ymm29, %ymm25
+# CHECK: vandnpd	%ymm28, %ymm29, %ymm25 {%k7}
+# CHECK: encoding: [0x62,0x01,0x95,0x27,0x55,0xcc]
+         vandnpd	%ymm28, %ymm29, %ymm25 {%k7}
+# CHECK: vandnpd	%ymm28, %ymm29, %ymm25 {%k7} {z}
+# CHECK: encoding: [0x62,0x01,0x95,0xa7,0x55,0xcc]
+         vandnpd	%ymm28, %ymm29, %ymm25 {%k7} {z}
+# CHECK: vandnpd	(%rcx), %ymm29, %ymm25
+# CHECK: encoding: [0x62,0x61,0x95,0x20,0x55,0x09]
+         vandnpd	(%rcx), %ymm29, %ymm25
+# CHECK: vandnpd	4660(%rax,%r14,8), %ymm29, %ymm25
+# CHECK: encoding: [0x62,0x21,0x95,0x20,0x55,0x8c,0xf0,0x34,0x12,0x00,0x00]
+         vandnpd	4660(%rax,%r14,8), %ymm29, %ymm25
+# CHECK: vandnpd	(%rcx){1to4}, %ymm29, %ymm25
+# CHECK: encoding: [0x62,0x61,0x95,0x30,0x55,0x09]
+         vandnpd	(%rcx){1to4}, %ymm29, %ymm25
+# CHECK: vandnpd	4064(%rdx), %ymm29, %ymm25
+# CHECK: encoding: [0x62,0x61,0x95,0x20,0x55,0x4a,0x7f]
+         vandnpd	4064(%rdx), %ymm29, %ymm25
+# CHECK: vandnpd	4096(%rdx), %ymm29, %ymm25
+# CHECK: encoding: [0x62,0x61,0x95,0x20,0x55,0x8a,0x00,0x10,0x00,0x00]
+         vandnpd	4096(%rdx), %ymm29, %ymm25
+# CHECK: vandnpd	-4096(%rdx), %ymm29, %ymm25
+# CHECK: encoding: [0x62,0x61,0x95,0x20,0x55,0x4a,0x80]
+         vandnpd	-4096(%rdx), %ymm29, %ymm25
+# CHECK: vandnpd	-4128(%rdx), %ymm29, %ymm25
+# CHECK: encoding: [0x62,0x61,0x95,0x20,0x55,0x8a,0xe0,0xef,0xff,0xff]
+         vandnpd	-4128(%rdx), %ymm29, %ymm25
+# CHECK: vandnpd	1016(%rdx){1to4}, %ymm29, %ymm25
+# CHECK: encoding: [0x62,0x61,0x95,0x30,0x55,0x4a,0x7f]
+         vandnpd	1016(%rdx){1to4}, %ymm29, %ymm25
+# CHECK: vandnpd	1024(%rdx){1to4}, %ymm29, %ymm25
+# CHECK: encoding: [0x62,0x61,0x95,0x30,0x55,0x8a,0x00,0x04,0x00,0x00]
+         vandnpd	1024(%rdx){1to4}, %ymm29, %ymm25
+# CHECK: vandnpd	-1024(%rdx){1to4}, %ymm29, %ymm25
+# CHECK: encoding: [0x62,0x61,0x95,0x30,0x55,0x4a,0x80]
+         vandnpd	-1024(%rdx){1to4}, %ymm29, %ymm25
+# CHECK: vandnpd	-1032(%rdx){1to4}, %ymm29, %ymm25
+# CHECK: encoding: [0x62,0x61,0x95,0x30,0x55,0x8a,0xf8,0xfb,0xff,0xff]
+         vandnpd	-1032(%rdx){1to4}, %ymm29, %ymm25
+# CHECK: vandnps	%xmm23, %xmm17, %xmm25
+# CHECK: encoding: [0x62,0x21,0x74,0x00,0x55,0xcf]
+         vandnps	%xmm23, %xmm17, %xmm25
+# CHECK: vandnps	%xmm23, %xmm17, %xmm25 {%k5}
+# CHECK: encoding: [0x62,0x21,0x74,0x05,0x55,0xcf]
+         vandnps	%xmm23, %xmm17, %xmm25 {%k5}
+# CHECK: vandnps	%xmm23, %xmm17, %xmm25 {%k5} {z}
+# CHECK: encoding: [0x62,0x21,0x74,0x85,0x55,0xcf]
+         vandnps	%xmm23, %xmm17, %xmm25 {%k5} {z}
+# CHECK: vandnps	(%rcx), %xmm17, %xmm25
+# CHECK: encoding: [0x62,0x61,0x74,0x00,0x55,0x09]
+         vandnps	(%rcx), %xmm17, %xmm25
+# CHECK: vandnps	4660(%rax,%r14,8), %xmm17, %xmm25
+# CHECK: encoding: [0x62,0x21,0x74,0x00,0x55,0x8c,0xf0,0x34,0x12,0x00,0x00]
+         vandnps	4660(%rax,%r14,8), %xmm17, %xmm25
+# CHECK: vandnps	(%rcx){1to4}, %xmm17, %xmm25
+# CHECK: encoding: [0x62,0x61,0x74,0x10,0x55,0x09]
+         vandnps	(%rcx){1to4}, %xmm17, %xmm25
+# CHECK: vandnps	2032(%rdx), %xmm17, %xmm25
+# CHECK: encoding: [0x62,0x61,0x74,0x00,0x55,0x4a,0x7f]
+         vandnps	2032(%rdx), %xmm17, %xmm25
+# CHECK: vandnps	2048(%rdx), %xmm17, %xmm25
+# CHECK: encoding: [0x62,0x61,0x74,0x00,0x55,0x8a,0x00,0x08,0x00,0x00]
+         vandnps	2048(%rdx), %xmm17, %xmm25
+# CHECK: vandnps	-2048(%rdx), %xmm17, %xmm25
+# CHECK: encoding: [0x62,0x61,0x74,0x00,0x55,0x4a,0x80]
+         vandnps	-2048(%rdx), %xmm17, %xmm25
+# CHECK: vandnps	-2064(%rdx), %xmm17, %xmm25
+# CHECK: encoding: [0x62,0x61,0x74,0x00,0x55,0x8a,0xf0,0xf7,0xff,0xff]
+         vandnps	-2064(%rdx), %xmm17, %xmm25
+# CHECK: vandnps	508(%rdx){1to4}, %xmm17, %xmm25
+# CHECK: encoding: [0x62,0x61,0x74,0x10,0x55,0x4a,0x7f]
+         vandnps	508(%rdx){1to4}, %xmm17, %xmm25
+# CHECK: vandnps	512(%rdx){1to4}, %xmm17, %xmm25
+# CHECK: encoding: [0x62,0x61,0x74,0x10,0x55,0x8a,0x00,0x02,0x00,0x00]
+         vandnps	512(%rdx){1to4}, %xmm17, %xmm25
+# CHECK: vandnps	-512(%rdx){1to4}, %xmm17, %xmm25
+# CHECK: encoding: [0x62,0x61,0x74,0x10,0x55,0x4a,0x80]
+         vandnps	-512(%rdx){1to4}, %xmm17, %xmm25
+# CHECK: vandnps	-516(%rdx){1to4}, %xmm17, %xmm25
+# CHECK: encoding: [0x62,0x61,0x74,0x10,0x55,0x8a,0xfc,0xfd,0xff,0xff]
+         vandnps	-516(%rdx){1to4}, %xmm17, %xmm25
+# CHECK: vandnps	%ymm23, %ymm19, %ymm18
+# CHECK: encoding: [0x62,0xa1,0x64,0x20,0x55,0xd7]
+         vandnps	%ymm23, %ymm19, %ymm18
+# CHECK: vandnps	%ymm23, %ymm19, %ymm18 {%k6}
+# CHECK: encoding: [0x62,0xa1,0x64,0x26,0x55,0xd7]
+         vandnps	%ymm23, %ymm19, %ymm18 {%k6}
+# CHECK: vandnps	%ymm23, %ymm19, %ymm18 {%k6} {z}
+# CHECK: encoding: [0x62,0xa1,0x64,0xa6,0x55,0xd7]
+         vandnps	%ymm23, %ymm19, %ymm18 {%k6} {z}
+# CHECK: vandnps	(%rcx), %ymm19, %ymm18
+# CHECK: encoding: [0x62,0xe1,0x64,0x20,0x55,0x11]
+         vandnps	(%rcx), %ymm19, %ymm18
+# CHECK: vandnps	4660(%rax,%r14,8), %ymm19, %ymm18
+# CHECK: encoding: [0x62,0xa1,0x64,0x20,0x55,0x94,0xf0,0x34,0x12,0x00,0x00]
+         vandnps	4660(%rax,%r14,8), %ymm19, %ymm18
+# CHECK: vandnps	(%rcx){1to8}, %ymm19, %ymm18
+# CHECK: encoding: [0x62,0xe1,0x64,0x30,0x55,0x11]
+         vandnps	(%rcx){1to8}, %ymm19, %ymm18
+# CHECK: vandnps	4064(%rdx), %ymm19, %ymm18
+# CHECK: encoding: [0x62,0xe1,0x64,0x20,0x55,0x52,0x7f]
+         vandnps	4064(%rdx), %ymm19, %ymm18
+# CHECK: vandnps	4096(%rdx), %ymm19, %ymm18
+# CHECK: encoding: [0x62,0xe1,0x64,0x20,0x55,0x92,0x00,0x10,0x00,0x00]
+         vandnps	4096(%rdx), %ymm19, %ymm18
+# CHECK: vandnps	-4096(%rdx), %ymm19, %ymm18
+# CHECK: encoding: [0x62,0xe1,0x64,0x20,0x55,0x52,0x80]
+         vandnps	-4096(%rdx), %ymm19, %ymm18
+# CHECK: vandnps	-4128(%rdx), %ymm19, %ymm18
+# CHECK: encoding: [0x62,0xe1,0x64,0x20,0x55,0x92,0xe0,0xef,0xff,0xff]
+         vandnps	-4128(%rdx), %ymm19, %ymm18
+# CHECK: vandnps	508(%rdx){1to8}, %ymm19, %ymm18
+# CHECK: encoding: [0x62,0xe1,0x64,0x30,0x55,0x52,0x7f]
+         vandnps	508(%rdx){1to8}, %ymm19, %ymm18
+# CHECK: vandnps	512(%rdx){1to8}, %ymm19, %ymm18
+# CHECK: encoding: [0x62,0xe1,0x64,0x30,0x55,0x92,0x00,0x02,0x00,0x00]
+         vandnps	512(%rdx){1to8}, %ymm19, %ymm18
+# CHECK: vandnps	-512(%rdx){1to8}, %ymm19, %ymm18
+# CHECK: encoding: [0x62,0xe1,0x64,0x30,0x55,0x52,0x80]
+         vandnps	-512(%rdx){1to8}, %ymm19, %ymm18
+# CHECK: vandnps	-516(%rdx){1to8}, %ymm19, %ymm18
+# CHECK: encoding: [0x62,0xe1,0x64,0x30,0x55,0x92,0xfc,0xfd,0xff,0xff]
+         vandnps	-516(%rdx){1to8}, %ymm19, %ymm18
+# CHECK: vorpd	%xmm18, %xmm29, %xmm26
+# CHECK: encoding: [0x62,0x21,0x95,0x00,0x56,0xd2]
+         vorpd	%xmm18, %xmm29, %xmm26
+# CHECK: vorpd	%xmm18, %xmm29, %xmm26 {%k2}
+# CHECK: encoding: [0x62,0x21,0x95,0x02,0x56,0xd2]
+         vorpd	%xmm18, %xmm29, %xmm26 {%k2}
+# CHECK: vorpd	%xmm18, %xmm29, %xmm26 {%k2} {z}
+# CHECK: encoding: [0x62,0x21,0x95,0x82,0x56,0xd2]
+         vorpd	%xmm18, %xmm29, %xmm26 {%k2} {z}
+# CHECK: vorpd	(%rcx), %xmm29, %xmm26
+# CHECK: encoding: [0x62,0x61,0x95,0x00,0x56,0x11]
+         vorpd	(%rcx), %xmm29, %xmm26
+# CHECK: vorpd	4660(%rax,%r14,8), %xmm29, %xmm26
+# CHECK: encoding: [0x62,0x21,0x95,0x00,0x56,0x94,0xf0,0x34,0x12,0x00,0x00]
+         vorpd	4660(%rax,%r14,8), %xmm29, %xmm26
+# CHECK: vorpd	(%rcx){1to2}, %xmm29, %xmm26
+# CHECK: encoding: [0x62,0x61,0x95,0x10,0x56,0x11]
+         vorpd	(%rcx){1to2}, %xmm29, %xmm26
+# CHECK: vorpd	2032(%rdx), %xmm29, %xmm26
+# CHECK: encoding: [0x62,0x61,0x95,0x00,0x56,0x52,0x7f]
+         vorpd	2032(%rdx), %xmm29, %xmm26
+# CHECK: vorpd	2048(%rdx), %xmm29, %xmm26
+# CHECK: encoding: [0x62,0x61,0x95,0x00,0x56,0x92,0x00,0x08,0x00,0x00]
+         vorpd	2048(%rdx), %xmm29, %xmm26
+# CHECK: vorpd	-2048(%rdx), %xmm29, %xmm26
+# CHECK: encoding: [0x62,0x61,0x95,0x00,0x56,0x52,0x80]
+         vorpd	-2048(%rdx), %xmm29, %xmm26
+# CHECK: vorpd	-2064(%rdx), %xmm29, %xmm26
+# CHECK: encoding: [0x62,0x61,0x95,0x00,0x56,0x92,0xf0,0xf7,0xff,0xff]
+         vorpd	-2064(%rdx), %xmm29, %xmm26
+# CHECK: vorpd	1016(%rdx){1to2}, %xmm29, %xmm26
+# CHECK: encoding: [0x62,0x61,0x95,0x10,0x56,0x52,0x7f]
+         vorpd	1016(%rdx){1to2}, %xmm29, %xmm26
+# CHECK: vorpd	1024(%rdx){1to2}, %xmm29, %xmm26
+# CHECK: encoding: [0x62,0x61,0x95,0x10,0x56,0x92,0x00,0x04,0x00,0x00]
+         vorpd	1024(%rdx){1to2}, %xmm29, %xmm26
+# CHECK: vorpd	-1024(%rdx){1to2}, %xmm29, %xmm26
+# CHECK: encoding: [0x62,0x61,0x95,0x10,0x56,0x52,0x80]
+         vorpd	-1024(%rdx){1to2}, %xmm29, %xmm26
+# CHECK: vorpd	-1032(%rdx){1to2}, %xmm29, %xmm26
+# CHECK: encoding: [0x62,0x61,0x95,0x10,0x56,0x92,0xf8,0xfb,0xff,0xff]
+         vorpd	-1032(%rdx){1to2}, %xmm29, %xmm26
+# CHECK: vorpd	%ymm22, %ymm19, %ymm28
+# CHECK: encoding: [0x62,0x21,0xe5,0x20,0x56,0xe6]
+         vorpd	%ymm22, %ymm19, %ymm28
+# CHECK: vorpd	%ymm22, %ymm19, %ymm28 {%k1}
+# CHECK: encoding: [0x62,0x21,0xe5,0x21,0x56,0xe6]
+         vorpd	%ymm22, %ymm19, %ymm28 {%k1}
+# CHECK: vorpd	%ymm22, %ymm19, %ymm28 {%k1} {z}
+# CHECK: encoding: [0x62,0x21,0xe5,0xa1,0x56,0xe6]
+         vorpd	%ymm22, %ymm19, %ymm28 {%k1} {z}
+# CHECK: vorpd	(%rcx), %ymm19, %ymm28
+# CHECK: encoding: [0x62,0x61,0xe5,0x20,0x56,0x21]
+         vorpd	(%rcx), %ymm19, %ymm28
+# CHECK: vorpd	4660(%rax,%r14,8), %ymm19, %ymm28
+# CHECK: encoding: [0x62,0x21,0xe5,0x20,0x56,0xa4,0xf0,0x34,0x12,0x00,0x00]
+         vorpd	4660(%rax,%r14,8), %ymm19, %ymm28
+# CHECK: vorpd	(%rcx){1to4}, %ymm19, %ymm28
+# CHECK: encoding: [0x62,0x61,0xe5,0x30,0x56,0x21]
+         vorpd	(%rcx){1to4}, %ymm19, %ymm28
+# CHECK: vorpd	4064(%rdx), %ymm19, %ymm28
+# CHECK: encoding: [0x62,0x61,0xe5,0x20,0x56,0x62,0x7f]
+         vorpd	4064(%rdx), %ymm19, %ymm28
+# CHECK: vorpd	4096(%rdx), %ymm19, %ymm28
+# CHECK: encoding: [0x62,0x61,0xe5,0x20,0x56,0xa2,0x00,0x10,0x00,0x00]
+         vorpd	4096(%rdx), %ymm19, %ymm28
+# CHECK: vorpd	-4096(%rdx), %ymm19, %ymm28
+# CHECK: encoding: [0x62,0x61,0xe5,0x20,0x56,0x62,0x80]
+         vorpd	-4096(%rdx), %ymm19, %ymm28
+# CHECK: vorpd	-4128(%rdx), %ymm19, %ymm28
+# CHECK: encoding: [0x62,0x61,0xe5,0x20,0x56,0xa2,0xe0,0xef,0xff,0xff]
+         vorpd	-4128(%rdx), %ymm19, %ymm28
+# CHECK: vorpd	1016(%rdx){1to4}, %ymm19, %ymm28
+# CHECK: encoding: [0x62,0x61,0xe5,0x30,0x56,0x62,0x7f]
+         vorpd	1016(%rdx){1to4}, %ymm19, %ymm28
+# CHECK: vorpd	1024(%rdx){1to4}, %ymm19, %ymm28
+# CHECK: encoding: [0x62,0x61,0xe5,0x30,0x56,0xa2,0x00,0x04,0x00,0x00]
+         vorpd	1024(%rdx){1to4}, %ymm19, %ymm28
+# CHECK: vorpd	-1024(%rdx){1to4}, %ymm19, %ymm28
+# CHECK: encoding: [0x62,0x61,0xe5,0x30,0x56,0x62,0x80]
+         vorpd	-1024(%rdx){1to4}, %ymm19, %ymm28
+# CHECK: vorpd	-1032(%rdx){1to4}, %ymm19, %ymm28
+# CHECK: encoding: [0x62,0x61,0xe5,0x30,0x56,0xa2,0xf8,0xfb,0xff,0xff]
+         vorpd	-1032(%rdx){1to4}, %ymm19, %ymm28
+# CHECK: vorps	%xmm24, %xmm22, %xmm28
+# CHECK: encoding: [0x62,0x01,0x4c,0x00,0x56,0xe0]
+         vorps	%xmm24, %xmm22, %xmm28
+# CHECK: vorps	%xmm24, %xmm22, %xmm28 {%k6}
+# CHECK: encoding: [0x62,0x01,0x4c,0x06,0x56,0xe0]
+         vorps	%xmm24, %xmm22, %xmm28 {%k6}
+# CHECK: vorps	%xmm24, %xmm22, %xmm28 {%k6} {z}
+# CHECK: encoding: [0x62,0x01,0x4c,0x86,0x56,0xe0]
+         vorps	%xmm24, %xmm22, %xmm28 {%k6} {z}
+# CHECK: vorps	(%rcx), %xmm22, %xmm28
+# CHECK: encoding: [0x62,0x61,0x4c,0x00,0x56,0x21]
+         vorps	(%rcx), %xmm22, %xmm28
+# CHECK: vorps	4660(%rax,%r14,8), %xmm22, %xmm28
+# CHECK: encoding: [0x62,0x21,0x4c,0x00,0x56,0xa4,0xf0,0x34,0x12,0x00,0x00]
+         vorps	4660(%rax,%r14,8), %xmm22, %xmm28
+# CHECK: vorps	(%rcx){1to4}, %xmm22, %xmm28
+# CHECK: encoding: [0x62,0x61,0x4c,0x10,0x56,0x21]
+         vorps	(%rcx){1to4}, %xmm22, %xmm28
+# CHECK: vorps	2032(%rdx), %xmm22, %xmm28
+# CHECK: encoding: [0x62,0x61,0x4c,0x00,0x56,0x62,0x7f]
+         vorps	2032(%rdx), %xmm22, %xmm28
+# CHECK: vorps	2048(%rdx), %xmm22, %xmm28
+# CHECK: encoding: [0x62,0x61,0x4c,0x00,0x56,0xa2,0x00,0x08,0x00,0x00]
+         vorps	2048(%rdx), %xmm22, %xmm28
+# CHECK: vorps	-2048(%rdx), %xmm22, %xmm28
+# CHECK: encoding: [0x62,0x61,0x4c,0x00,0x56,0x62,0x80]
+         vorps	-2048(%rdx), %xmm22, %xmm28
+# CHECK: vorps	-2064(%rdx), %xmm22, %xmm28
+# CHECK: encoding: [0x62,0x61,0x4c,0x00,0x56,0xa2,0xf0,0xf7,0xff,0xff]
+         vorps	-2064(%rdx), %xmm22, %xmm28
+# CHECK: vorps	508(%rdx){1to4}, %xmm22, %xmm28
+# CHECK: encoding: [0x62,0x61,0x4c,0x10,0x56,0x62,0x7f]
+         vorps	508(%rdx){1to4}, %xmm22, %xmm28
+# CHECK: vorps	512(%rdx){1to4}, %xmm22, %xmm28
+# CHECK: encoding: [0x62,0x61,0x4c,0x10,0x56,0xa2,0x00,0x02,0x00,0x00]
+         vorps	512(%rdx){1to4}, %xmm22, %xmm28
+# CHECK: vorps	-512(%rdx){1to4}, %xmm22, %xmm28
+# CHECK: encoding: [0x62,0x61,0x4c,0x10,0x56,0x62,0x80]
+         vorps	-512(%rdx){1to4}, %xmm22, %xmm28
+# CHECK: vorps	-516(%rdx){1to4}, %xmm22, %xmm28
+# CHECK: encoding: [0x62,0x61,0x4c,0x10,0x56,0xa2,0xfc,0xfd,0xff,0xff]
+         vorps	-516(%rdx){1to4}, %xmm22, %xmm28
+# CHECK: vorps	%ymm25, %ymm24, %ymm20
+# CHECK: encoding: [0x62,0x81,0x3c,0x20,0x56,0xe1]
+         vorps	%ymm25, %ymm24, %ymm20
+# CHECK: vorps	%ymm25, %ymm24, %ymm20 {%k1}
+# CHECK: encoding: [0x62,0x81,0x3c,0x21,0x56,0xe1]
+         vorps	%ymm25, %ymm24, %ymm20 {%k1}
+# CHECK: vorps	%ymm25, %ymm24, %ymm20 {%k1} {z}
+# CHECK: encoding: [0x62,0x81,0x3c,0xa1,0x56,0xe1]
+         vorps	%ymm25, %ymm24, %ymm20 {%k1} {z}
+# CHECK: vorps	(%rcx), %ymm24, %ymm20
+# CHECK: encoding: [0x62,0xe1,0x3c,0x20,0x56,0x21]
+         vorps	(%rcx), %ymm24, %ymm20
+# CHECK: vorps	4660(%rax,%r14,8), %ymm24, %ymm20
+# CHECK: encoding: [0x62,0xa1,0x3c,0x20,0x56,0xa4,0xf0,0x34,0x12,0x00,0x00]
+         vorps	4660(%rax,%r14,8), %ymm24, %ymm20
+# CHECK: vorps	(%rcx){1to8}, %ymm24, %ymm20
+# CHECK: encoding: [0x62,0xe1,0x3c,0x30,0x56,0x21]
+         vorps	(%rcx){1to8}, %ymm24, %ymm20
+# CHECK: vorps	4064(%rdx), %ymm24, %ymm20
+# CHECK: encoding: [0x62,0xe1,0x3c,0x20,0x56,0x62,0x7f]
+         vorps	4064(%rdx), %ymm24, %ymm20
+# CHECK: vorps	4096(%rdx), %ymm24, %ymm20
+# CHECK: encoding: [0x62,0xe1,0x3c,0x20,0x56,0xa2,0x00,0x10,0x00,0x00]
+         vorps	4096(%rdx), %ymm24, %ymm20
+# CHECK: vorps	-4096(%rdx), %ymm24, %ymm20
+# CHECK: encoding: [0x62,0xe1,0x3c,0x20,0x56,0x62,0x80]
+         vorps	-4096(%rdx), %ymm24, %ymm20
+# CHECK: vorps	-4128(%rdx), %ymm24, %ymm20
+# CHECK: encoding: [0x62,0xe1,0x3c,0x20,0x56,0xa2,0xe0,0xef,0xff,0xff]
+         vorps	-4128(%rdx), %ymm24, %ymm20
+# CHECK: vorps	508(%rdx){1to8}, %ymm24, %ymm20
+# CHECK: encoding: [0x62,0xe1,0x3c,0x30,0x56,0x62,0x7f]
+         vorps	508(%rdx){1to8}, %ymm24, %ymm20
+# CHECK: vorps	512(%rdx){1to8}, %ymm24, %ymm20
+# CHECK: encoding: [0x62,0xe1,0x3c,0x30,0x56,0xa2,0x00,0x02,0x00,0x00]
+         vorps	512(%rdx){1to8}, %ymm24, %ymm20
+# CHECK: vorps	-512(%rdx){1to8}, %ymm24, %ymm20
+# CHECK: encoding: [0x62,0xe1,0x3c,0x30,0x56,0x62,0x80]
+         vorps	-512(%rdx){1to8}, %ymm24, %ymm20
+# CHECK: vorps	-516(%rdx){1to8}, %ymm24, %ymm20
+# CHECK: encoding: [0x62,0xe1,0x3c,0x30,0x56,0xa2,0xfc,0xfd,0xff,0xff]
+         vorps	-516(%rdx){1to8}, %ymm24, %ymm20
+# CHECK: vxorpd	%xmm18, %xmm21, %xmm22
+# CHECK: encoding: [0x62,0xa1,0xd5,0x00,0x57,0xf2]
+         vxorpd	%xmm18, %xmm21, %xmm22
+# CHECK: vxorpd	%xmm18, %xmm21, %xmm22 {%k3}
+# CHECK: encoding: [0x62,0xa1,0xd5,0x03,0x57,0xf2]
+         vxorpd	%xmm18, %xmm21, %xmm22 {%k3}
+# CHECK: vxorpd	%xmm18, %xmm21, %xmm22 {%k3} {z}
+# CHECK: encoding: [0x62,0xa1,0xd5,0x83,0x57,0xf2]
+         vxorpd	%xmm18, %xmm21, %xmm22 {%k3} {z}
+# CHECK: vxorpd	(%rcx), %xmm21, %xmm22
+# CHECK: encoding: [0x62,0xe1,0xd5,0x00,0x57,0x31]
+         vxorpd	(%rcx), %xmm21, %xmm22
+# CHECK: vxorpd	4660(%rax,%r14,8), %xmm21, %xmm22
+# CHECK: encoding: [0x62,0xa1,0xd5,0x00,0x57,0xb4,0xf0,0x34,0x12,0x00,0x00]
+         vxorpd	4660(%rax,%r14,8), %xmm21, %xmm22
+# CHECK: vxorpd	(%rcx){1to2}, %xmm21, %xmm22
+# CHECK: encoding: [0x62,0xe1,0xd5,0x10,0x57,0x31]
+         vxorpd	(%rcx){1to2}, %xmm21, %xmm22
+# CHECK: vxorpd	2032(%rdx), %xmm21, %xmm22
+# CHECK: encoding: [0x62,0xe1,0xd5,0x00,0x57,0x72,0x7f]
+         vxorpd	2032(%rdx), %xmm21, %xmm22
+# CHECK: vxorpd	2048(%rdx), %xmm21, %xmm22
+# CHECK: encoding: [0x62,0xe1,0xd5,0x00,0x57,0xb2,0x00,0x08,0x00,0x00]
+         vxorpd	2048(%rdx), %xmm21, %xmm22
+# CHECK: vxorpd	-2048(%rdx), %xmm21, %xmm22
+# CHECK: encoding: [0x62,0xe1,0xd5,0x00,0x57,0x72,0x80]
+         vxorpd	-2048(%rdx), %xmm21, %xmm22
+# CHECK: vxorpd	-2064(%rdx), %xmm21, %xmm22
+# CHECK: encoding: [0x62,0xe1,0xd5,0x00,0x57,0xb2,0xf0,0xf7,0xff,0xff]
+         vxorpd	-2064(%rdx), %xmm21, %xmm22
+# CHECK: vxorpd	1016(%rdx){1to2}, %xmm21, %xmm22
+# CHECK: encoding: [0x62,0xe1,0xd5,0x10,0x57,0x72,0x7f]
+         vxorpd	1016(%rdx){1to2}, %xmm21, %xmm22
+# CHECK: vxorpd	1024(%rdx){1to2}, %xmm21, %xmm22
+# CHECK: encoding: [0x62,0xe1,0xd5,0x10,0x57,0xb2,0x00,0x04,0x00,0x00]
+         vxorpd	1024(%rdx){1to2}, %xmm21, %xmm22
+# CHECK: vxorpd	-1024(%rdx){1to2}, %xmm21, %xmm22
+# CHECK: encoding: [0x62,0xe1,0xd5,0x10,0x57,0x72,0x80]
+         vxorpd	-1024(%rdx){1to2}, %xmm21, %xmm22
+# CHECK: vxorpd	-1032(%rdx){1to2}, %xmm21, %xmm22
+# CHECK: encoding: [0x62,0xe1,0xd5,0x10,0x57,0xb2,0xf8,0xfb,0xff,0xff]
+         vxorpd	-1032(%rdx){1to2}, %xmm21, %xmm22
+# CHECK: vxorpd	%ymm27, %ymm21, %ymm25
+# CHECK: encoding: [0x62,0x01,0xd5,0x20,0x57,0xcb]
+         vxorpd	%ymm27, %ymm21, %ymm25
+# CHECK: vxorpd	%ymm27, %ymm21, %ymm25 {%k7}
+# CHECK: encoding: [0x62,0x01,0xd5,0x27,0x57,0xcb]
+         vxorpd	%ymm27, %ymm21, %ymm25 {%k7}
+# CHECK: vxorpd	%ymm27, %ymm21, %ymm25 {%k7} {z}
+# CHECK: encoding: [0x62,0x01,0xd5,0xa7,0x57,0xcb]
+         vxorpd	%ymm27, %ymm21, %ymm25 {%k7} {z}
+# CHECK: vxorpd	(%rcx), %ymm21, %ymm25
+# CHECK: encoding: [0x62,0x61,0xd5,0x20,0x57,0x09]
+         vxorpd	(%rcx), %ymm21, %ymm25
+# CHECK: vxorpd	4660(%rax,%r14,8), %ymm21, %ymm25
+# CHECK: encoding: [0x62,0x21,0xd5,0x20,0x57,0x8c,0xf0,0x34,0x12,0x00,0x00]
+         vxorpd	4660(%rax,%r14,8), %ymm21, %ymm25
+# CHECK: vxorpd	(%rcx){1to4}, %ymm21, %ymm25
+# CHECK: encoding: [0x62,0x61,0xd5,0x30,0x57,0x09]
+         vxorpd	(%rcx){1to4}, %ymm21, %ymm25
+# CHECK: vxorpd	4064(%rdx), %ymm21, %ymm25
+# CHECK: encoding: [0x62,0x61,0xd5,0x20,0x57,0x4a,0x7f]
+         vxorpd	4064(%rdx), %ymm21, %ymm25
+# CHECK: vxorpd	4096(%rdx), %ymm21, %ymm25
+# CHECK: encoding: [0x62,0x61,0xd5,0x20,0x57,0x8a,0x00,0x10,0x00,0x00]
+         vxorpd	4096(%rdx), %ymm21, %ymm25
+# CHECK: vxorpd	-4096(%rdx), %ymm21, %ymm25
+# CHECK: encoding: [0x62,0x61,0xd5,0x20,0x57,0x4a,0x80]
+         vxorpd	-4096(%rdx), %ymm21, %ymm25
+# CHECK: vxorpd	-4128(%rdx), %ymm21, %ymm25
+# CHECK: encoding: [0x62,0x61,0xd5,0x20,0x57,0x8a,0xe0,0xef,0xff,0xff]
+         vxorpd	-4128(%rdx), %ymm21, %ymm25
+# CHECK: vxorpd	1016(%rdx){1to4}, %ymm21, %ymm25
+# CHECK: encoding: [0x62,0x61,0xd5,0x30,0x57,0x4a,0x7f]
+         vxorpd	1016(%rdx){1to4}, %ymm21, %ymm25
+# CHECK: vxorpd	1024(%rdx){1to4}, %ymm21, %ymm25
+# CHECK: encoding: [0x62,0x61,0xd5,0x30,0x57,0x8a,0x00,0x04,0x00,0x00]
+         vxorpd	1024(%rdx){1to4}, %ymm21, %ymm25
+# CHECK: vxorpd	-1024(%rdx){1to4}, %ymm21, %ymm25
+# CHECK: encoding: [0x62,0x61,0xd5,0x30,0x57,0x4a,0x80]
+         vxorpd	-1024(%rdx){1to4}, %ymm21, %ymm25
+# CHECK: vxorpd	-1032(%rdx){1to4}, %ymm21, %ymm25
+# CHECK: encoding: [0x62,0x61,0xd5,0x30,0x57,0x8a,0xf8,0xfb,0xff,0xff]
+         vxorpd	-1032(%rdx){1to4}, %ymm21, %ymm25
+# CHECK: vxorps	%xmm21, %xmm21, %xmm17
+# CHECK: encoding: [0x62,0xa1,0x54,0x00,0x57,0xcd]
+         vxorps	%xmm21, %xmm21, %xmm17
+# CHECK: vxorps	%xmm21, %xmm21, %xmm17 {%k5}
+# CHECK: encoding: [0x62,0xa1,0x54,0x05,0x57,0xcd]
+         vxorps	%xmm21, %xmm21, %xmm17 {%k5}
+# CHECK: vxorps	%xmm21, %xmm21, %xmm17 {%k5} {z}
+# CHECK: encoding: [0x62,0xa1,0x54,0x85,0x57,0xcd]
+         vxorps	%xmm21, %xmm21, %xmm17 {%k5} {z}
+# CHECK: vxorps	(%rcx), %xmm21, %xmm17
+# CHECK: encoding: [0x62,0xe1,0x54,0x00,0x57,0x09]
+         vxorps	(%rcx), %xmm21, %xmm17
+# CHECK: vxorps	4660(%rax,%r14,8), %xmm21, %xmm17
+# CHECK: encoding: [0x62,0xa1,0x54,0x00,0x57,0x8c,0xf0,0x34,0x12,0x00,0x00]
+         vxorps	4660(%rax,%r14,8), %xmm21, %xmm17
+# CHECK: vxorps	(%rcx){1to4}, %xmm21, %xmm17
+# CHECK: encoding: [0x62,0xe1,0x54,0x10,0x57,0x09]
+         vxorps	(%rcx){1to4}, %xmm21, %xmm17
+# CHECK: vxorps	2032(%rdx), %xmm21, %xmm17
+# CHECK: encoding: [0x62,0xe1,0x54,0x00,0x57,0x4a,0x7f]
+         vxorps	2032(%rdx), %xmm21, %xmm17
+# CHECK: vxorps	2048(%rdx), %xmm21, %xmm17
+# CHECK: encoding: [0x62,0xe1,0x54,0x00,0x57,0x8a,0x00,0x08,0x00,0x00]
+         vxorps	2048(%rdx), %xmm21, %xmm17
+# CHECK: vxorps	-2048(%rdx), %xmm21, %xmm17
+# CHECK: encoding: [0x62,0xe1,0x54,0x00,0x57,0x4a,0x80]
+         vxorps	-2048(%rdx), %xmm21, %xmm17
+# CHECK: vxorps	-2064(%rdx), %xmm21, %xmm17
+# CHECK: encoding: [0x62,0xe1,0x54,0x00,0x57,0x8a,0xf0,0xf7,0xff,0xff]
+         vxorps	-2064(%rdx), %xmm21, %xmm17
+# CHECK: vxorps	508(%rdx){1to4}, %xmm21, %xmm17
+# CHECK: encoding: [0x62,0xe1,0x54,0x10,0x57,0x4a,0x7f]
+         vxorps	508(%rdx){1to4}, %xmm21, %xmm17
+# CHECK: vxorps	512(%rdx){1to4}, %xmm21, %xmm17
+# CHECK: encoding: [0x62,0xe1,0x54,0x10,0x57,0x8a,0x00,0x02,0x00,0x00]
+         vxorps	512(%rdx){1to4}, %xmm21, %xmm17
+# CHECK: vxorps	-512(%rdx){1to4}, %xmm21, %xmm17
+# CHECK: encoding: [0x62,0xe1,0x54,0x10,0x57,0x4a,0x80]
+         vxorps	-512(%rdx){1to4}, %xmm21, %xmm17
+# CHECK: vxorps	-516(%rdx){1to4}, %xmm21, %xmm17
+# CHECK: encoding: [0x62,0xe1,0x54,0x10,0x57,0x8a,0xfc,0xfd,0xff,0xff]
+         vxorps	-516(%rdx){1to4}, %xmm21, %xmm17
+# CHECK: vxorps	%ymm22, %ymm25, %ymm28
+# CHECK: encoding: [0x62,0x21,0x34,0x20,0x57,0xe6]
+         vxorps	%ymm22, %ymm25, %ymm28
+# CHECK: vxorps	%ymm22, %ymm25, %ymm28 {%k3}
+# CHECK: encoding: [0x62,0x21,0x34,0x23,0x57,0xe6]
+         vxorps	%ymm22, %ymm25, %ymm28 {%k3}
+# CHECK: vxorps	%ymm22, %ymm25, %ymm28 {%k3} {z}
+# CHECK: encoding: [0x62,0x21,0x34,0xa3,0x57,0xe6]
+         vxorps	%ymm22, %ymm25, %ymm28 {%k3} {z}
+# CHECK: vxorps	(%rcx), %ymm25, %ymm28
+# CHECK: encoding: [0x62,0x61,0x34,0x20,0x57,0x21]
+         vxorps	(%rcx), %ymm25, %ymm28
+# CHECK: vxorps	4660(%rax,%r14,8), %ymm25, %ymm28
+# CHECK: encoding: [0x62,0x21,0x34,0x20,0x57,0xa4,0xf0,0x34,0x12,0x00,0x00]
+         vxorps	4660(%rax,%r14,8), %ymm25, %ymm28
+# CHECK: vxorps	(%rcx){1to8}, %ymm25, %ymm28
+# CHECK: encoding: [0x62,0x61,0x34,0x30,0x57,0x21]
+         vxorps	(%rcx){1to8}, %ymm25, %ymm28
+# CHECK: vxorps	4064(%rdx), %ymm25, %ymm28
+# CHECK: encoding: [0x62,0x61,0x34,0x20,0x57,0x62,0x7f]
+         vxorps	4064(%rdx), %ymm25, %ymm28
+# CHECK: vxorps	4096(%rdx), %ymm25, %ymm28
+# CHECK: encoding: [0x62,0x61,0x34,0x20,0x57,0xa2,0x00,0x10,0x00,0x00]
+         vxorps	4096(%rdx), %ymm25, %ymm28
+# CHECK: vxorps	-4096(%rdx), %ymm25, %ymm28
+# CHECK: encoding: [0x62,0x61,0x34,0x20,0x57,0x62,0x80]
+         vxorps	-4096(%rdx), %ymm25, %ymm28
+# CHECK: vxorps	-4128(%rdx), %ymm25, %ymm28
+# CHECK: encoding: [0x62,0x61,0x34,0x20,0x57,0xa2,0xe0,0xef,0xff,0xff]
+         vxorps	-4128(%rdx), %ymm25, %ymm28
+# CHECK: vxorps	508(%rdx){1to8}, %ymm25, %ymm28
+# CHECK: encoding: [0x62,0x61,0x34,0x30,0x57,0x62,0x7f]
+         vxorps	508(%rdx){1to8}, %ymm25, %ymm28
+# CHECK: vxorps	512(%rdx){1to8}, %ymm25, %ymm28
+# CHECK: encoding: [0x62,0x61,0x34,0x30,0x57,0xa2,0x00,0x02,0x00,0x00]
+         vxorps	512(%rdx){1to8}, %ymm25, %ymm28
+# CHECK: vxorps	-512(%rdx){1to8}, %ymm25, %ymm28
+# CHECK: encoding: [0x62,0x61,0x34,0x30,0x57,0x62,0x80]
+         vxorps	-512(%rdx){1to8}, %ymm25, %ymm28
+# CHECK: vxorps	-516(%rdx){1to8}, %ymm25, %ymm28
+# CHECK: encoding: [0x62,0x61,0x34,0x30,0x57,0xa2,0xfc,0xfd,0xff,0xff]
+         vxorps	-516(%rdx){1to8}, %ymm25, %ymm28
+# CHECK: vbroadcastf64x2	(%rcx), %ymm27
+# CHECK: encoding: [0x62,0x62,0xfd,0x28,0x1a,0x19]
+         vbroadcastf64x2	(%rcx), %ymm27
+# CHECK: vbroadcastf64x2	(%rcx), %ymm27 {%k5}
+# CHECK: encoding: [0x62,0x62,0xfd,0x2d,0x1a,0x19]
+         vbroadcastf64x2	(%rcx), %ymm27 {%k5}
+# CHECK: vbroadcastf64x2	(%rcx), %ymm27 {%k5} {z}
+# CHECK: encoding: [0x62,0x62,0xfd,0xad,0x1a,0x19]
+         vbroadcastf64x2	(%rcx), %ymm27 {%k5} {z}
+# CHECK: vbroadcastf64x2	291(%rax,%r14,8), %ymm27
+# CHECK: encoding: [0x62,0x22,0xfd,0x28,0x1a,0x9c,0xf0,0x23,0x01,0x00,0x00]
+         vbroadcastf64x2	291(%rax,%r14,8), %ymm27
+# CHECK: vbroadcastf64x2	2032(%rdx), %ymm27
+# CHECK: encoding: [0x62,0x62,0xfd,0x28,0x1a,0x5a,0x7f]
+         vbroadcastf64x2	2032(%rdx), %ymm27
+# CHECK: vbroadcastf64x2	2048(%rdx), %ymm27
+# CHECK: encoding: [0x62,0x62,0xfd,0x28,0x1a,0x9a,0x00,0x08,0x00,0x00]
+         vbroadcastf64x2	2048(%rdx), %ymm27
+# CHECK: vbroadcastf64x2	-2048(%rdx), %ymm27
+# CHECK: encoding: [0x62,0x62,0xfd,0x28,0x1a,0x5a,0x80]
+         vbroadcastf64x2	-2048(%rdx), %ymm27
+# CHECK: vbroadcastf64x2	-2064(%rdx), %ymm27
+# CHECK: encoding: [0x62,0x62,0xfd,0x28,0x1a,0x9a,0xf0,0xf7,0xff,0xff]
+         vbroadcastf64x2	-2064(%rdx), %ymm27
+# CHECK: vbroadcasti64x2	(%rcx), %ymm18
+# CHECK: encoding: [0x62,0xe2,0xfd,0x28,0x5a,0x11]
+         vbroadcasti64x2	(%rcx), %ymm18
+# CHECK: vbroadcasti64x2	(%rcx), %ymm18 {%k1}
+# CHECK: encoding: [0x62,0xe2,0xfd,0x29,0x5a,0x11]
+         vbroadcasti64x2	(%rcx), %ymm18 {%k1}
+# CHECK: vbroadcasti64x2	(%rcx), %ymm18 {%k1} {z}
+# CHECK: encoding: [0x62,0xe2,0xfd,0xa9,0x5a,0x11]
+         vbroadcasti64x2	(%rcx), %ymm18 {%k1} {z}
+# CHECK: vbroadcasti64x2	291(%rax,%r14,8), %ymm18
+# CHECK: encoding: [0x62,0xa2,0xfd,0x28,0x5a,0x94,0xf0,0x23,0x01,0x00,0x00]
+         vbroadcasti64x2	291(%rax,%r14,8), %ymm18
+# CHECK: vbroadcasti64x2	2032(%rdx), %ymm18
+# CHECK: encoding: [0x62,0xe2,0xfd,0x28,0x5a,0x52,0x7f]
+         vbroadcasti64x2	2032(%rdx), %ymm18
+# CHECK: vbroadcasti64x2	2048(%rdx), %ymm18
+# CHECK: encoding: [0x62,0xe2,0xfd,0x28,0x5a,0x92,0x00,0x08,0x00,0x00]
+         vbroadcasti64x2	2048(%rdx), %ymm18
+# CHECK: vbroadcasti64x2	-2048(%rdx), %ymm18
+# CHECK: encoding: [0x62,0xe2,0xfd,0x28,0x5a,0x52,0x80]
+         vbroadcasti64x2	-2048(%rdx), %ymm18
+# CHECK: vbroadcasti64x2	-2064(%rdx), %ymm18
+# CHECK: encoding: [0x62,0xe2,0xfd,0x28,0x5a,0x92,0xf0,0xf7,0xff,0xff]
+         vbroadcasti64x2	-2064(%rdx), %ymm18
+# CHECK: vrangepd	$171, %xmm26, %xmm27, %xmm25
+# CHECK: encoding: [0x62,0x03,0xa5,0x00,0x50,0xca,0xab]
+         vrangepd	$171, %xmm26, %xmm27, %xmm25
+# CHECK: vrangepd	$171, %xmm26, %xmm27, %xmm25 {%k6}
+# CHECK: encoding: [0x62,0x03,0xa5,0x06,0x50,0xca,0xab]
+         vrangepd	$171, %xmm26, %xmm27, %xmm25 {%k6}
+# CHECK: vrangepd	$171, %xmm26, %xmm27, %xmm25 {%k6} {z}
+# CHECK: encoding: [0x62,0x03,0xa5,0x86,0x50,0xca,0xab]
+         vrangepd	$171, %xmm26, %xmm27, %xmm25 {%k6} {z}
+# CHECK: vrangepd	$123, %xmm26, %xmm27, %xmm25
+# CHECK: encoding: [0x62,0x03,0xa5,0x00,0x50,0xca,0x7b]
+         vrangepd	$123, %xmm26, %xmm27, %xmm25
+# CHECK: vrangepd	$123, (%rcx), %xmm27, %xmm25
+# CHECK: encoding: [0x62,0x63,0xa5,0x00,0x50,0x09,0x7b]
+         vrangepd	$123, (%rcx), %xmm27, %xmm25
+# CHECK: vrangepd	$123, 291(%rax,%r14,8), %xmm27, %xmm25
+# CHECK: encoding: [0x62,0x23,0xa5,0x00,0x50,0x8c,0xf0,0x23,0x01,0x00,0x00,0x7b]
+         vrangepd	$123, 291(%rax,%r14,8), %xmm27, %xmm25
+# CHECK: vrangepd	$123, (%rcx){1to2}, %xmm27, %xmm25
+# CHECK: encoding: [0x62,0x63,0xa5,0x10,0x50,0x09,0x7b]
+         vrangepd	$123, (%rcx){1to2}, %xmm27, %xmm25
+# CHECK: vrangepd	$123, 2032(%rdx), %xmm27, %xmm25
+# CHECK: encoding: [0x62,0x63,0xa5,0x00,0x50,0x4a,0x7f,0x7b]
+         vrangepd	$123, 2032(%rdx), %xmm27, %xmm25
+# CHECK: vrangepd	$123, 2048(%rdx), %xmm27, %xmm25
+# CHECK: encoding: [0x62,0x63,0xa5,0x00,0x50,0x8a,0x00,0x08,0x00,0x00,0x7b]
+         vrangepd	$123, 2048(%rdx), %xmm27, %xmm25
+# CHECK: vrangepd	$123, -2048(%rdx), %xmm27, %xmm25
+# CHECK: encoding: [0x62,0x63,0xa5,0x00,0x50,0x4a,0x80,0x7b]
+         vrangepd	$123, -2048(%rdx), %xmm27, %xmm25
+# CHECK: vrangepd	$123, -2064(%rdx), %xmm27, %xmm25
+# CHECK: encoding: [0x62,0x63,0xa5,0x00,0x50,0x8a,0xf0,0xf7,0xff,0xff,0x7b]
+         vrangepd	$123, -2064(%rdx), %xmm27, %xmm25
+# CHECK: vrangepd	$123, 1016(%rdx){1to2}, %xmm27, %xmm25
+# CHECK: encoding: [0x62,0x63,0xa5,0x10,0x50,0x4a,0x7f,0x7b]
+         vrangepd	$123, 1016(%rdx){1to2}, %xmm27, %xmm25
+# CHECK: vrangepd	$123, 1024(%rdx){1to2}, %xmm27, %xmm25
+# CHECK: encoding: [0x62,0x63,0xa5,0x10,0x50,0x8a,0x00,0x04,0x00,0x00,0x7b]
+         vrangepd	$123, 1024(%rdx){1to2}, %xmm27, %xmm25
+# CHECK: vrangepd	$123, -1024(%rdx){1to2}, %xmm27, %xmm25
+# CHECK: encoding: [0x62,0x63,0xa5,0x10,0x50,0x4a,0x80,0x7b]
+         vrangepd	$123, -1024(%rdx){1to2}, %xmm27, %xmm25
+# CHECK: vrangepd	$123, -1032(%rdx){1to2}, %xmm27, %xmm25
+# CHECK: encoding: [0x62,0x63,0xa5,0x10,0x50,0x8a,0xf8,0xfb,0xff,0xff,0x7b]
+         vrangepd	$123, -1032(%rdx){1to2}, %xmm27, %xmm25
+# CHECK: vrangepd	$171, %ymm28, %ymm27, %ymm29
+# CHECK: encoding: [0x62,0x03,0xa5,0x20,0x50,0xec,0xab]
+         vrangepd	$171, %ymm28, %ymm27, %ymm29
+# CHECK: vrangepd	$171, %ymm28, %ymm27, %ymm29 {%k7}
+# CHECK: encoding: [0x62,0x03,0xa5,0x27,0x50,0xec,0xab]
+         vrangepd	$171, %ymm28, %ymm27, %ymm29 {%k7}
+# CHECK: vrangepd	$171, %ymm28, %ymm27, %ymm29 {%k7} {z}
+# CHECK: encoding: [0x62,0x03,0xa5,0xa7,0x50,0xec,0xab]
+         vrangepd	$171, %ymm28, %ymm27, %ymm29 {%k7} {z}
+# CHECK: vrangepd	$123, %ymm28, %ymm27, %ymm29
+# CHECK: encoding: [0x62,0x03,0xa5,0x20,0x50,0xec,0x7b]
+         vrangepd	$123, %ymm28, %ymm27, %ymm29
+# CHECK: vrangepd	$123, (%rcx), %ymm27, %ymm29
+# CHECK: encoding: [0x62,0x63,0xa5,0x20,0x50,0x29,0x7b]
+         vrangepd	$123, (%rcx), %ymm27, %ymm29
+# CHECK: vrangepd	$123, 291(%rax,%r14,8), %ymm27, %ymm29
+# CHECK: encoding: [0x62,0x23,0xa5,0x20,0x50,0xac,0xf0,0x23,0x01,0x00,0x00,0x7b]
+         vrangepd	$123, 291(%rax,%r14,8), %ymm27, %ymm29
+# CHECK: vrangepd	$123, (%rcx){1to4}, %ymm27, %ymm29
+# CHECK: encoding: [0x62,0x63,0xa5,0x30,0x50,0x29,0x7b]
+         vrangepd	$123, (%rcx){1to4}, %ymm27, %ymm29
+# CHECK: vrangepd	$123, 4064(%rdx), %ymm27, %ymm29
+# CHECK: encoding: [0x62,0x63,0xa5,0x20,0x50,0x6a,0x7f,0x7b]
+         vrangepd	$123, 4064(%rdx), %ymm27, %ymm29
+# CHECK: vrangepd	$123, 4096(%rdx), %ymm27, %ymm29
+# CHECK: encoding: [0x62,0x63,0xa5,0x20,0x50,0xaa,0x00,0x10,0x00,0x00,0x7b]
+         vrangepd	$123, 4096(%rdx), %ymm27, %ymm29
+# CHECK: vrangepd	$123, -4096(%rdx), %ymm27, %ymm29
+# CHECK: encoding: [0x62,0x63,0xa5,0x20,0x50,0x6a,0x80,0x7b]
+         vrangepd	$123, -4096(%rdx), %ymm27, %ymm29
+# CHECK: vrangepd	$123, -4128(%rdx), %ymm27, %ymm29
+# CHECK: encoding: [0x62,0x63,0xa5,0x20,0x50,0xaa,0xe0,0xef,0xff,0xff,0x7b]
+         vrangepd	$123, -4128(%rdx), %ymm27, %ymm29
+# CHECK: vrangepd	$123, 1016(%rdx){1to4}, %ymm27, %ymm29
+# CHECK: encoding: [0x62,0x63,0xa5,0x30,0x50,0x6a,0x7f,0x7b]
+         vrangepd	$123, 1016(%rdx){1to4}, %ymm27, %ymm29
+# CHECK: vrangepd	$123, 1024(%rdx){1to4}, %ymm27, %ymm29
+# CHECK: encoding: [0x62,0x63,0xa5,0x30,0x50,0xaa,0x00,0x04,0x00,0x00,0x7b]
+         vrangepd	$123, 1024(%rdx){1to4}, %ymm27, %ymm29
+# CHECK: vrangepd	$123, -1024(%rdx){1to4}, %ymm27, %ymm29
+# CHECK: encoding: [0x62,0x63,0xa5,0x30,0x50,0x6a,0x80,0x7b]
+         vrangepd	$123, -1024(%rdx){1to4}, %ymm27, %ymm29
+# CHECK: vrangepd	$123, -1032(%rdx){1to4}, %ymm27, %ymm29
+# CHECK: encoding: [0x62,0x63,0xa5,0x30,0x50,0xaa,0xf8,0xfb,0xff,0xff,0x7b]
+         vrangepd	$123, -1032(%rdx){1to4}, %ymm27, %ymm29
+# CHECK: vrangeps	$171, %xmm24, %xmm23, %xmm27
+# CHECK: encoding: [0x62,0x03,0x45,0x00,0x50,0xd8,0xab]
+         vrangeps	$171, %xmm24, %xmm23, %xmm27
+# CHECK: vrangeps	$171, %xmm24, %xmm23, %xmm27 {%k6}
+# CHECK: encoding: [0x62,0x03,0x45,0x06,0x50,0xd8,0xab]
+         vrangeps	$171, %xmm24, %xmm23, %xmm27 {%k6}
+# CHECK: vrangeps	$171, %xmm24, %xmm23, %xmm27 {%k6} {z}
+# CHECK: encoding: [0x62,0x03,0x45,0x86,0x50,0xd8,0xab]
+         vrangeps	$171, %xmm24, %xmm23, %xmm27 {%k6} {z}
+# CHECK: vrangeps	$123, %xmm24, %xmm23, %xmm27
+# CHECK: encoding: [0x62,0x03,0x45,0x00,0x50,0xd8,0x7b]
+         vrangeps	$123, %xmm24, %xmm23, %xmm27
+# CHECK: vrangeps	$123, (%rcx), %xmm23, %xmm27
+# CHECK: encoding: [0x62,0x63,0x45,0x00,0x50,0x19,0x7b]
+         vrangeps	$123, (%rcx), %xmm23, %xmm27
+# CHECK: vrangeps	$123, 291(%rax,%r14,8), %xmm23, %xmm27
+# CHECK: encoding: [0x62,0x23,0x45,0x00,0x50,0x9c,0xf0,0x23,0x01,0x00,0x00,0x7b]
+         vrangeps	$123, 291(%rax,%r14,8), %xmm23, %xmm27
+# CHECK: vrangeps	$123, (%rcx){1to4}, %xmm23, %xmm27
+# CHECK: encoding: [0x62,0x63,0x45,0x10,0x50,0x19,0x7b]
+         vrangeps	$123, (%rcx){1to4}, %xmm23, %xmm27
+# CHECK: vrangeps	$123, 2032(%rdx), %xmm23, %xmm27
+# CHECK: encoding: [0x62,0x63,0x45,0x00,0x50,0x5a,0x7f,0x7b]
+         vrangeps	$123, 2032(%rdx), %xmm23, %xmm27
+# CHECK: vrangeps	$123, 2048(%rdx), %xmm23, %xmm27
+# CHECK: encoding: [0x62,0x63,0x45,0x00,0x50,0x9a,0x00,0x08,0x00,0x00,0x7b]
+         vrangeps	$123, 2048(%rdx), %xmm23, %xmm27
+# CHECK: vrangeps	$123, -2048(%rdx), %xmm23, %xmm27
+# CHECK: encoding: [0x62,0x63,0x45,0x00,0x50,0x5a,0x80,0x7b]
+         vrangeps	$123, -2048(%rdx), %xmm23, %xmm27
+# CHECK: vrangeps	$123, -2064(%rdx), %xmm23, %xmm27
+# CHECK: encoding: [0x62,0x63,0x45,0x00,0x50,0x9a,0xf0,0xf7,0xff,0xff,0x7b]
+         vrangeps	$123, -2064(%rdx), %xmm23, %xmm27
+# CHECK: vrangeps	$123, 508(%rdx){1to4}, %xmm23, %xmm27
+# CHECK: encoding: [0x62,0x63,0x45,0x10,0x50,0x5a,0x7f,0x7b]
+         vrangeps	$123, 508(%rdx){1to4}, %xmm23, %xmm27
+# CHECK: vrangeps	$123, 512(%rdx){1to4}, %xmm23, %xmm27
+# CHECK: encoding: [0x62,0x63,0x45,0x10,0x50,0x9a,0x00,0x02,0x00,0x00,0x7b]
+         vrangeps	$123, 512(%rdx){1to4}, %xmm23, %xmm27
+# CHECK: vrangeps	$123, -512(%rdx){1to4}, %xmm23, %xmm27
+# CHECK: encoding: [0x62,0x63,0x45,0x10,0x50,0x5a,0x80,0x7b]
+         vrangeps	$123, -512(%rdx){1to4}, %xmm23, %xmm27
+# CHECK: vrangeps	$123, -516(%rdx){1to4}, %xmm23, %xmm27
+# CHECK: encoding: [0x62,0x63,0x45,0x10,0x50,0x9a,0xfc,0xfd,0xff,0xff,0x7b]
+         vrangeps	$123, -516(%rdx){1to4}, %xmm23, %xmm27
+# CHECK: vrangeps	$171, %ymm21, %ymm23, %ymm24
+# CHECK: encoding: [0x62,0x23,0x45,0x20,0x50,0xc5,0xab]
+         vrangeps	$171, %ymm21, %ymm23, %ymm24
+# CHECK: vrangeps	$171, %ymm21, %ymm23, %ymm24 {%k7}
+# CHECK: encoding: [0x62,0x23,0x45,0x27,0x50,0xc5,0xab]
+         vrangeps	$171, %ymm21, %ymm23, %ymm24 {%k7}
+# CHECK: vrangeps	$171, %ymm21, %ymm23, %ymm24 {%k7} {z}
+# CHECK: encoding: [0x62,0x23,0x45,0xa7,0x50,0xc5,0xab]
+         vrangeps	$171, %ymm21, %ymm23, %ymm24 {%k7} {z}
+# CHECK: vrangeps	$123, %ymm21, %ymm23, %ymm24
+# CHECK: encoding: [0x62,0x23,0x45,0x20,0x50,0xc5,0x7b]
+         vrangeps	$123, %ymm21, %ymm23, %ymm24
+# CHECK: vrangeps	$123, (%rcx), %ymm23, %ymm24
+# CHECK: encoding: [0x62,0x63,0x45,0x20,0x50,0x01,0x7b]
+         vrangeps	$123, (%rcx), %ymm23, %ymm24
+# CHECK: vrangeps	$123, 291(%rax,%r14,8), %ymm23, %ymm24
+# CHECK: encoding: [0x62,0x23,0x45,0x20,0x50,0x84,0xf0,0x23,0x01,0x00,0x00,0x7b]
+         vrangeps	$123, 291(%rax,%r14,8), %ymm23, %ymm24
+# CHECK: vrangeps	$123, (%rcx){1to8}, %ymm23, %ymm24
+# CHECK: encoding: [0x62,0x63,0x45,0x30,0x50,0x01,0x7b]
+         vrangeps	$123, (%rcx){1to8}, %ymm23, %ymm24
+# CHECK: vrangeps	$123, 4064(%rdx), %ymm23, %ymm24
+# CHECK: encoding: [0x62,0x63,0x45,0x20,0x50,0x42,0x7f,0x7b]
+         vrangeps	$123, 4064(%rdx), %ymm23, %ymm24
+# CHECK: vrangeps	$123, 4096(%rdx), %ymm23, %ymm24
+# CHECK: encoding: [0x62,0x63,0x45,0x20,0x50,0x82,0x00,0x10,0x00,0x00,0x7b]
+         vrangeps	$123, 4096(%rdx), %ymm23, %ymm24
+# CHECK: vrangeps	$123, -4096(%rdx), %ymm23, %ymm24
+# CHECK: encoding: [0x62,0x63,0x45,0x20,0x50,0x42,0x80,0x7b]
+         vrangeps	$123, -4096(%rdx), %ymm23, %ymm24
+# CHECK: vrangeps	$123, -4128(%rdx), %ymm23, %ymm24
+# CHECK: encoding: [0x62,0x63,0x45,0x20,0x50,0x82,0xe0,0xef,0xff,0xff,0x7b]
+         vrangeps	$123, -4128(%rdx), %ymm23, %ymm24
+# CHECK: vrangeps	$123, 508(%rdx){1to8}, %ymm23, %ymm24
+# CHECK: encoding: [0x62,0x63,0x45,0x30,0x50,0x42,0x7f,0x7b]
+         vrangeps	$123, 508(%rdx){1to8}, %ymm23, %ymm24
+# CHECK: vrangeps	$123, 512(%rdx){1to8}, %ymm23, %ymm24
+# CHECK: encoding: [0x62,0x63,0x45,0x30,0x50,0x82,0x00,0x02,0x00,0x00,0x7b]
+         vrangeps	$123, 512(%rdx){1to8}, %ymm23, %ymm24
+# CHECK: vrangeps	$123, -512(%rdx){1to8}, %ymm23, %ymm24
+# CHECK: encoding: [0x62,0x63,0x45,0x30,0x50,0x42,0x80,0x7b]
+         vrangeps	$123, -512(%rdx){1to8}, %ymm23, %ymm24
+# CHECK: vrangeps	$123, -516(%rdx){1to8}, %ymm23, %ymm24
+# CHECK: encoding: [0x62,0x63,0x45,0x30,0x50,0x82,0xfc,0xfd,0xff,0xff,0x7b]
+         vrangeps	$123, -516(%rdx){1to8}, %ymm23, %ymm24
+# CHECK: vreducepd	$171, %xmm17, %xmm18
+# CHECK: encoding: [0x62,0xa3,0xfd,0x08,0x56,0xd1,0xab]
+         vreducepd	$171, %xmm17, %xmm18
+# CHECK: vreducepd	$171, %xmm17, %xmm18 {%k3}
+# CHECK: encoding: [0x62,0xa3,0xfd,0x0b,0x56,0xd1,0xab]
+         vreducepd	$171, %xmm17, %xmm18 {%k3}
+# CHECK: vreducepd	$171, %xmm17, %xmm18 {%k3} {z}
+# CHECK: encoding: [0x62,0xa3,0xfd,0x8b,0x56,0xd1,0xab]
+         vreducepd	$171, %xmm17, %xmm18 {%k3} {z}
+# CHECK: vreducepd	$123, %xmm17, %xmm18
+# CHECK: encoding: [0x62,0xa3,0xfd,0x08,0x56,0xd1,0x7b]
+         vreducepd	$123, %xmm17, %xmm18
+# CHECK: vreducepd	$123, (%rcx), %xmm18
+# CHECK: encoding: [0x62,0xe3,0xfd,0x08,0x56,0x11,0x7b]
+         vreducepd	$123, (%rcx), %xmm18
+# CHECK: vreducepd	$123, 291(%rax,%r14,8), %xmm18
+# CHECK: encoding: [0x62,0xa3,0xfd,0x08,0x56,0x94,0xf0,0x23,0x01,0x00,0x00,0x7b]
+         vreducepd	$123, 291(%rax,%r14,8), %xmm18
+# CHECK: vreducepd	$171, %xmm28, %xmm25
+# CHECK: encoding: [0x62,0x03,0xfd,0x08,0x56,0xcc,0xab]
+         vreducepd	$171, %xmm28, %xmm25
+# CHECK: vreducepd	$171, %xmm28, %xmm25 {%k4}
+# CHECK: encoding: [0x62,0x03,0xfd,0x0c,0x56,0xcc,0xab]
+         vreducepd	$171, %xmm28, %xmm25 {%k4}
+# CHECK: vreducepd	$171, %xmm28, %xmm25 {%k4} {z}
+# CHECK: encoding: [0x62,0x03,0xfd,0x8c,0x56,0xcc,0xab]
+         vreducepd	$171, %xmm28, %xmm25 {%k4} {z}
+# CHECK: vreducepd	$123, %xmm28, %xmm25
+# CHECK: encoding: [0x62,0x03,0xfd,0x08,0x56,0xcc,0x7b]
+         vreducepd	$123, %xmm28, %xmm25
+# CHECK: vreducepd	$123, (%rcx), %xmm25
+# CHECK: encoding: [0x62,0x63,0xfd,0x08,0x56,0x09,0x7b]
+         vreducepd	$123, (%rcx), %xmm25
+# CHECK: vreducepd	$123, 4660(%rax,%r14,8), %xmm25
+# CHECK: encoding: [0x62,0x23,0xfd,0x08,0x56,0x8c,0xf0,0x34,0x12,0x00,0x00,0x7b]
+         vreducepd	$123, 4660(%rax,%r14,8), %xmm25
+# CHECK: vreducepd	$123, (%rcx){1to2}, %xmm25
+# CHECK: encoding: [0x62,0x63,0xfd,0x18,0x56,0x09,0x7b]
+         vreducepd	$123, (%rcx){1to2}, %xmm25
+# CHECK: vreducepd	$123, 2032(%rdx), %xmm25
+# CHECK: encoding: [0x62,0x63,0xfd,0x08,0x56,0x4a,0x7f,0x7b]
+         vreducepd	$123, 2032(%rdx), %xmm25
+# CHECK: vreducepd	$123, 2048(%rdx), %xmm25
+# CHECK: encoding: [0x62,0x63,0xfd,0x08,0x56,0x8a,0x00,0x08,0x00,0x00,0x7b]
+         vreducepd	$123, 2048(%rdx), %xmm25
+# CHECK: vreducepd	$123, -2048(%rdx), %xmm25
+# CHECK: encoding: [0x62,0x63,0xfd,0x08,0x56,0x4a,0x80,0x7b]
+         vreducepd	$123, -2048(%rdx), %xmm25
+# CHECK: vreducepd	$123, -2064(%rdx), %xmm25
+# CHECK: encoding: [0x62,0x63,0xfd,0x08,0x56,0x8a,0xf0,0xf7,0xff,0xff,0x7b]
+         vreducepd	$123, -2064(%rdx), %xmm25
+# CHECK: vreducepd	$123, 1016(%rdx){1to2}, %xmm25
+# CHECK: encoding: [0x62,0x63,0xfd,0x18,0x56,0x4a,0x7f,0x7b]
+         vreducepd	$123, 1016(%rdx){1to2}, %xmm25
+# CHECK: vreducepd	$123, 1024(%rdx){1to2}, %xmm25
+# CHECK: encoding: [0x62,0x63,0xfd,0x18,0x56,0x8a,0x00,0x04,0x00,0x00,0x7b]
+         vreducepd	$123, 1024(%rdx){1to2}, %xmm25
+# CHECK: vreducepd	$123, -1024(%rdx){1to2}, %xmm25
+# CHECK: encoding: [0x62,0x63,0xfd,0x18,0x56,0x4a,0x80,0x7b]
+         vreducepd	$123, -1024(%rdx){1to2}, %xmm25
+# CHECK: vreducepd	$123, -1032(%rdx){1to2}, %xmm25
+# CHECK: encoding: [0x62,0x63,0xfd,0x18,0x56,0x8a,0xf8,0xfb,0xff,0xff,0x7b]
+         vreducepd	$123, -1032(%rdx){1to2}, %xmm25
+# CHECK: vreducepd	$171, %ymm17, %ymm28
+# CHECK: encoding: [0x62,0x23,0xfd,0x28,0x56,0xe1,0xab]
+         vreducepd	$171, %ymm17, %ymm28
+# CHECK: vreducepd	$171, %ymm17, %ymm28 {%k4}
+# CHECK: encoding: [0x62,0x23,0xfd,0x2c,0x56,0xe1,0xab]
+         vreducepd	$171, %ymm17, %ymm28 {%k4}
+# CHECK: vreducepd	$171, %ymm17, %ymm28 {%k4} {z}
+# CHECK: encoding: [0x62,0x23,0xfd,0xac,0x56,0xe1,0xab]
+         vreducepd	$171, %ymm17, %ymm28 {%k4} {z}
+# CHECK: vreducepd	$123, %ymm17, %ymm28
+# CHECK: encoding: [0x62,0x23,0xfd,0x28,0x56,0xe1,0x7b]
+         vreducepd	$123, %ymm17, %ymm28
+# CHECK: vreducepd	$123, (%rcx), %ymm28
+# CHECK: encoding: [0x62,0x63,0xfd,0x28,0x56,0x21,0x7b]
+         vreducepd	$123, (%rcx), %ymm28
+# CHECK: vreducepd	$123, 4660(%rax,%r14,8), %ymm28
+# CHECK: encoding: [0x62,0x23,0xfd,0x28,0x56,0xa4,0xf0,0x34,0x12,0x00,0x00,0x7b]
+         vreducepd	$123, 4660(%rax,%r14,8), %ymm28
+# CHECK: vreducepd	$123, (%rcx){1to4}, %ymm28
+# CHECK: encoding: [0x62,0x63,0xfd,0x38,0x56,0x21,0x7b]
+         vreducepd	$123, (%rcx){1to4}, %ymm28
+# CHECK: vreducepd	$123, 4064(%rdx), %ymm28
+# CHECK: encoding: [0x62,0x63,0xfd,0x28,0x56,0x62,0x7f,0x7b]
+         vreducepd	$123, 4064(%rdx), %ymm28
+# CHECK: vreducepd	$123, 4096(%rdx), %ymm28
+# CHECK: encoding: [0x62,0x63,0xfd,0x28,0x56,0xa2,0x00,0x10,0x00,0x00,0x7b]
+         vreducepd	$123, 4096(%rdx), %ymm28
+# CHECK: vreducepd	$123, -4096(%rdx), %ymm28
+# CHECK: encoding: [0x62,0x63,0xfd,0x28,0x56,0x62,0x80,0x7b]
+         vreducepd	$123, -4096(%rdx), %ymm28
+# CHECK: vreducepd	$123, -4128(%rdx), %ymm28
+# CHECK: encoding: [0x62,0x63,0xfd,0x28,0x56,0xa2,0xe0,0xef,0xff,0xff,0x7b]
+         vreducepd	$123, -4128(%rdx), %ymm28
+# CHECK: vreducepd	$123, 1016(%rdx){1to4}, %ymm28
+# CHECK: encoding: [0x62,0x63,0xfd,0x38,0x56,0x62,0x7f,0x7b]
+         vreducepd	$123, 1016(%rdx){1to4}, %ymm28
+# CHECK: vreducepd	$123, 1024(%rdx){1to4}, %ymm28
+# CHECK: encoding: [0x62,0x63,0xfd,0x38,0x56,0xa2,0x00,0x04,0x00,0x00,0x7b]
+         vreducepd	$123, 1024(%rdx){1to4}, %ymm28
+# CHECK: vreducepd	$123, -1024(%rdx){1to4}, %ymm28
+# CHECK: encoding: [0x62,0x63,0xfd,0x38,0x56,0x62,0x80,0x7b]
+         vreducepd	$123, -1024(%rdx){1to4}, %ymm28
+# CHECK: vreducepd	$123, -1032(%rdx){1to4}, %ymm28
+# CHECK: encoding: [0x62,0x63,0xfd,0x38,0x56,0xa2,0xf8,0xfb,0xff,0xff,0x7b]
+         vreducepd	$123, -1032(%rdx){1to4}, %ymm28
+# CHECK: vreduceps	$171, %xmm21, %xmm29
+# CHECK: encoding: [0x62,0x23,0x7d,0x08,0x56,0xed,0xab]
+         vreduceps	$171, %xmm21, %xmm29
+# CHECK: vreduceps	$171, %xmm21, %xmm29 {%k7}
+# CHECK: encoding: [0x62,0x23,0x7d,0x0f,0x56,0xed,0xab]
+         vreduceps	$171, %xmm21, %xmm29 {%k7}
+# CHECK: vreduceps	$171, %xmm21, %xmm29 {%k7} {z}
+# CHECK: encoding: [0x62,0x23,0x7d,0x8f,0x56,0xed,0xab]
+         vreduceps	$171, %xmm21, %xmm29 {%k7} {z}
+# CHECK: vreduceps	$123, %xmm21, %xmm29
+# CHECK: encoding: [0x62,0x23,0x7d,0x08,0x56,0xed,0x7b]
+         vreduceps	$123, %xmm21, %xmm29
+# CHECK: vreduceps	$123, (%rcx), %xmm29
+# CHECK: encoding: [0x62,0x63,0x7d,0x08,0x56,0x29,0x7b]
+         vreduceps	$123, (%rcx), %xmm29
+# CHECK: vreduceps	$123, 4660(%rax,%r14,8), %xmm29
+# CHECK: encoding: [0x62,0x23,0x7d,0x08,0x56,0xac,0xf0,0x34,0x12,0x00,0x00,0x7b]
+         vreduceps	$123, 4660(%rax,%r14,8), %xmm29
+# CHECK: vreduceps	$123, (%rcx){1to4}, %xmm29
+# CHECK: encoding: [0x62,0x63,0x7d,0x18,0x56,0x29,0x7b]
+         vreduceps	$123, (%rcx){1to4}, %xmm29
+# CHECK: vreduceps	$123, 2032(%rdx), %xmm29
+# CHECK: encoding: [0x62,0x63,0x7d,0x08,0x56,0x6a,0x7f,0x7b]
+         vreduceps	$123, 2032(%rdx), %xmm29
+# CHECK: vreduceps	$123, 2048(%rdx), %xmm29
+# CHECK: encoding: [0x62,0x63,0x7d,0x08,0x56,0xaa,0x00,0x08,0x00,0x00,0x7b]
+         vreduceps	$123, 2048(%rdx), %xmm29
+# CHECK: vreduceps	$123, -2048(%rdx), %xmm29
+# CHECK: encoding: [0x62,0x63,0x7d,0x08,0x56,0x6a,0x80,0x7b]
+         vreduceps	$123, -2048(%rdx), %xmm29
+# CHECK: vreduceps	$123, -2064(%rdx), %xmm29
+# CHECK: encoding: [0x62,0x63,0x7d,0x08,0x56,0xaa,0xf0,0xf7,0xff,0xff,0x7b]
+         vreduceps	$123, -2064(%rdx), %xmm29
+# CHECK: vreduceps	$123, 508(%rdx){1to4}, %xmm29
+# CHECK: encoding: [0x62,0x63,0x7d,0x18,0x56,0x6a,0x7f,0x7b]
+         vreduceps	$123, 508(%rdx){1to4}, %xmm29
+# CHECK: vreduceps	$123, 512(%rdx){1to4}, %xmm29
+# CHECK: encoding: [0x62,0x63,0x7d,0x18,0x56,0xaa,0x00,0x02,0x00,0x00,0x7b]
+         vreduceps	$123, 512(%rdx){1to4}, %xmm29
+# CHECK: vreduceps	$123, -512(%rdx){1to4}, %xmm29
+# CHECK: encoding: [0x62,0x63,0x7d,0x18,0x56,0x6a,0x80,0x7b]
+         vreduceps	$123, -512(%rdx){1to4}, %xmm29
+# CHECK: vreduceps	$123, -516(%rdx){1to4}, %xmm29
+# CHECK: encoding: [0x62,0x63,0x7d,0x18,0x56,0xaa,0xfc,0xfd,0xff,0xff,0x7b]
+         vreduceps	$123, -516(%rdx){1to4}, %xmm29
+# CHECK: vreduceps	$171, %ymm23, %ymm25
+# CHECK: encoding: [0x62,0x23,0x7d,0x28,0x56,0xcf,0xab]
+         vreduceps	$171, %ymm23, %ymm25
+# CHECK: vreduceps	$171, %ymm23, %ymm25 {%k3}
+# CHECK: encoding: [0x62,0x23,0x7d,0x2b,0x56,0xcf,0xab]
+         vreduceps	$171, %ymm23, %ymm25 {%k3}
+# CHECK: vreduceps	$171, %ymm23, %ymm25 {%k3} {z}
+# CHECK: encoding: [0x62,0x23,0x7d,0xab,0x56,0xcf,0xab]
+         vreduceps	$171, %ymm23, %ymm25 {%k3} {z}
+# CHECK: vreduceps	$123, %ymm23, %ymm25
+# CHECK: encoding: [0x62,0x23,0x7d,0x28,0x56,0xcf,0x7b]
+         vreduceps	$123, %ymm23, %ymm25
+# CHECK: vreduceps	$123, (%rcx), %ymm25
+# CHECK: encoding: [0x62,0x63,0x7d,0x28,0x56,0x09,0x7b]
+         vreduceps	$123, (%rcx), %ymm25
+# CHECK: vreduceps	$123, 4660(%rax,%r14,8), %ymm25
+# CHECK: encoding: [0x62,0x23,0x7d,0x28,0x56,0x8c,0xf0,0x34,0x12,0x00,0x00,0x7b]
+         vreduceps	$123, 4660(%rax,%r14,8), %ymm25
+# CHECK: vreduceps	$123, (%rcx){1to8}, %ymm25
+# CHECK: encoding: [0x62,0x63,0x7d,0x38,0x56,0x09,0x7b]
+         vreduceps	$123, (%rcx){1to8}, %ymm25
+# CHECK: vreduceps	$123, 4064(%rdx), %ymm25
+# CHECK: encoding: [0x62,0x63,0x7d,0x28,0x56,0x4a,0x7f,0x7b]
+         vreduceps	$123, 4064(%rdx), %ymm25
+# CHECK: vreduceps	$123, 4096(%rdx), %ymm25
+# CHECK: encoding: [0x62,0x63,0x7d,0x28,0x56,0x8a,0x00,0x10,0x00,0x00,0x7b]
+         vreduceps	$123, 4096(%rdx), %ymm25
+# CHECK: vreduceps	$123, -4096(%rdx), %ymm25
+# CHECK: encoding: [0x62,0x63,0x7d,0x28,0x56,0x4a,0x80,0x7b]
+         vreduceps	$123, -4096(%rdx), %ymm25
+# CHECK: vreduceps	$123, -4128(%rdx), %ymm25
+# CHECK: encoding: [0x62,0x63,0x7d,0x28,0x56,0x8a,0xe0,0xef,0xff,0xff,0x7b]
+         vreduceps	$123, -4128(%rdx), %ymm25
+# CHECK: vreduceps	$123, 508(%rdx){1to8}, %ymm25
+# CHECK: encoding: [0x62,0x63,0x7d,0x38,0x56,0x4a,0x7f,0x7b]
+         vreduceps	$123, 508(%rdx){1to8}, %ymm25
+# CHECK: vreduceps	$123, 512(%rdx){1to8}, %ymm25
+# CHECK: encoding: [0x62,0x63,0x7d,0x38,0x56,0x8a,0x00,0x02,0x00,0x00,0x7b]
+         vreduceps	$123, 512(%rdx){1to8}, %ymm25
+# CHECK: vreduceps	$123, -512(%rdx){1to8}, %ymm25
+# CHECK: encoding: [0x62,0x63,0x7d,0x38,0x56,0x4a,0x80,0x7b]
+         vreduceps	$123, -512(%rdx){1to8}, %ymm25
+# CHECK: vreduceps	$123, -516(%rdx){1to8}, %ymm25
+# CHECK: encoding: [0x62,0x63,0x7d,0x38,0x56,0x8a,0xfc,0xfd,0xff,0xff,0x7b]
+         vreduceps	$123, -516(%rdx){1to8}, %ymm25
+# CHECK: vreducepd	$123, (%rcx){1to2}, %xmm18
+# CHECK: encoding: [0x62,0xe3,0xfd,0x18,0x56,0x11,0x7b]
+         vreducepd	$123, (%rcx){1to2}, %xmm18
+# CHECK: vreducepd	$123, 2032(%rdx), %xmm18
+# CHECK: encoding: [0x62,0xe3,0xfd,0x08,0x56,0x52,0x7f,0x7b]
+         vreducepd	$123, 2032(%rdx), %xmm18
+# CHECK: vreducepd	$123, 2048(%rdx), %xmm18
+# CHECK: encoding: [0x62,0xe3,0xfd,0x08,0x56,0x92,0x00,0x08,0x00,0x00,0x7b]
+         vreducepd	$123, 2048(%rdx), %xmm18
+# CHECK: vreducepd	$123, -2048(%rdx), %xmm18
+# CHECK: encoding: [0x62,0xe3,0xfd,0x08,0x56,0x52,0x80,0x7b]
+         vreducepd	$123, -2048(%rdx), %xmm18
+# CHECK: vreducepd	$123, -2064(%rdx), %xmm18
+# CHECK: encoding: [0x62,0xe3,0xfd,0x08,0x56,0x92,0xf0,0xf7,0xff,0xff,0x7b]
+         vreducepd	$123, -2064(%rdx), %xmm18
+# CHECK: vreducepd	$123, 1016(%rdx){1to2}, %xmm18
+# CHECK: encoding: [0x62,0xe3,0xfd,0x18,0x56,0x52,0x7f,0x7b]
+         vreducepd	$123, 1016(%rdx){1to2}, %xmm18
+# CHECK: vreducepd	$123, 1024(%rdx){1to2}, %xmm18
+# CHECK: encoding: [0x62,0xe3,0xfd,0x18,0x56,0x92,0x00,0x04,0x00,0x00,0x7b]
+         vreducepd	$123, 1024(%rdx){1to2}, %xmm18
+# CHECK: vreducepd	$123, -1024(%rdx){1to2}, %xmm18
+# CHECK: encoding: [0x62,0xe3,0xfd,0x18,0x56,0x52,0x80,0x7b]
+         vreducepd	$123, -1024(%rdx){1to2}, %xmm18
+# CHECK: vreducepd	$123, -1032(%rdx){1to2}, %xmm18
+# CHECK: encoding: [0x62,0xe3,0xfd,0x18,0x56,0x92,0xf8,0xfb,0xff,0xff,0x7b]
+         vreducepd	$123, -1032(%rdx){1to2}, %xmm18
+# CHECK: vreducepd	$171, %ymm29, %ymm25
+# CHECK: encoding: [0x62,0x03,0xfd,0x28,0x56,0xcd,0xab]
+         vreducepd	$171, %ymm29, %ymm25
+# CHECK: vreducepd	$171, %ymm29, %ymm25 {%k1}
+# CHECK: encoding: [0x62,0x03,0xfd,0x29,0x56,0xcd,0xab]
+         vreducepd	$171, %ymm29, %ymm25 {%k1}
+# CHECK: vreducepd	$171, %ymm29, %ymm25 {%k1} {z}
+# CHECK: encoding: [0x62,0x03,0xfd,0xa9,0x56,0xcd,0xab]
+         vreducepd	$171, %ymm29, %ymm25 {%k1} {z}
+# CHECK: vreducepd	$123, %ymm29, %ymm25
+# CHECK: encoding: [0x62,0x03,0xfd,0x28,0x56,0xcd,0x7b]
+         vreducepd	$123, %ymm29, %ymm25
+# CHECK: vreducepd	$123, (%rcx), %ymm25
+# CHECK: encoding: [0x62,0x63,0xfd,0x28,0x56,0x09,0x7b]
+         vreducepd	$123, (%rcx), %ymm25
+# CHECK: vreducepd	$123, 291(%rax,%r14,8), %ymm25
+# CHECK: encoding: [0x62,0x23,0xfd,0x28,0x56,0x8c,0xf0,0x23,0x01,0x00,0x00,0x7b]
+         vreducepd	$123, 291(%rax,%r14,8), %ymm25
+# CHECK: vreducepd	$123, (%rcx){1to4}, %ymm25
+# CHECK: encoding: [0x62,0x63,0xfd,0x38,0x56,0x09,0x7b]
+         vreducepd	$123, (%rcx){1to4}, %ymm25
+# CHECK: vreducepd	$123, 4064(%rdx), %ymm25
+# CHECK: encoding: [0x62,0x63,0xfd,0x28,0x56,0x4a,0x7f,0x7b]
+         vreducepd	$123, 4064(%rdx), %ymm25
+# CHECK: vreducepd	$123, 4096(%rdx), %ymm25
+# CHECK: encoding: [0x62,0x63,0xfd,0x28,0x56,0x8a,0x00,0x10,0x00,0x00,0x7b]
+         vreducepd	$123, 4096(%rdx), %ymm25
+# CHECK: vreducepd	$123, -4096(%rdx), %ymm25
+# CHECK: encoding: [0x62,0x63,0xfd,0x28,0x56,0x4a,0x80,0x7b]
+         vreducepd	$123, -4096(%rdx), %ymm25
+# CHECK: vreducepd	$123, -4128(%rdx), %ymm25
+# CHECK: encoding: [0x62,0x63,0xfd,0x28,0x56,0x8a,0xe0,0xef,0xff,0xff,0x7b]
+         vreducepd	$123, -4128(%rdx), %ymm25
+# CHECK: vreducepd	$123, 1016(%rdx){1to4}, %ymm25
+# CHECK: encoding: [0x62,0x63,0xfd,0x38,0x56,0x4a,0x7f,0x7b]
+         vreducepd	$123, 1016(%rdx){1to4}, %ymm25
+# CHECK: vreducepd	$123, 1024(%rdx){1to4}, %ymm25
+# CHECK: encoding: [0x62,0x63,0xfd,0x38,0x56,0x8a,0x00,0x04,0x00,0x00,0x7b]
+         vreducepd	$123, 1024(%rdx){1to4}, %ymm25
+# CHECK: vreducepd	$123, -1024(%rdx){1to4}, %ymm25
+# CHECK: encoding: [0x62,0x63,0xfd,0x38,0x56,0x4a,0x80,0x7b]
+         vreducepd	$123, -1024(%rdx){1to4}, %ymm25
+# CHECK: vreducepd	$123, -1032(%rdx){1to4}, %ymm25
+# CHECK: encoding: [0x62,0x63,0xfd,0x38,0x56,0x8a,0xf8,0xfb,0xff,0xff,0x7b]
+         vreducepd	$123, -1032(%rdx){1to4}, %ymm25
+# CHECK: vreduceps	$171, %xmm23, %xmm20
+# CHECK: encoding: [0x62,0xa3,0x7d,0x08,0x56,0xe7,0xab]
+         vreduceps	$171, %xmm23, %xmm20
+# CHECK: vreduceps	$171, %xmm23, %xmm20 {%k7}
+# CHECK: encoding: [0x62,0xa3,0x7d,0x0f,0x56,0xe7,0xab]
+         vreduceps	$171, %xmm23, %xmm20 {%k7}
+# CHECK: vreduceps	$171, %xmm23, %xmm20 {%k7} {z}
+# CHECK: encoding: [0x62,0xa3,0x7d,0x8f,0x56,0xe7,0xab]
+         vreduceps	$171, %xmm23, %xmm20 {%k7} {z}
+# CHECK: vreduceps	$123, %xmm23, %xmm20
+# CHECK: encoding: [0x62,0xa3,0x7d,0x08,0x56,0xe7,0x7b]
+         vreduceps	$123, %xmm23, %xmm20
+# CHECK: vreduceps	$123, (%rcx), %xmm20
+# CHECK: encoding: [0x62,0xe3,0x7d,0x08,0x56,0x21,0x7b]
+         vreduceps	$123, (%rcx), %xmm20
+# CHECK: vreduceps	$123, 291(%rax,%r14,8), %xmm20
+# CHECK: encoding: [0x62,0xa3,0x7d,0x08,0x56,0xa4,0xf0,0x23,0x01,0x00,0x00,0x7b]
+         vreduceps	$123, 291(%rax,%r14,8), %xmm20
+# CHECK: vreduceps	$123, (%rcx){1to4}, %xmm20
+# CHECK: encoding: [0x62,0xe3,0x7d,0x18,0x56,0x21,0x7b]
+         vreduceps	$123, (%rcx){1to4}, %xmm20
+# CHECK: vreduceps	$123, 2032(%rdx), %xmm20
+# CHECK: encoding: [0x62,0xe3,0x7d,0x08,0x56,0x62,0x7f,0x7b]
+         vreduceps	$123, 2032(%rdx), %xmm20
+# CHECK: vreduceps	$123, 2048(%rdx), %xmm20
+# CHECK: encoding: [0x62,0xe3,0x7d,0x08,0x56,0xa2,0x00,0x08,0x00,0x00,0x7b]
+         vreduceps	$123, 2048(%rdx), %xmm20
+# CHECK: vreduceps	$123, -2048(%rdx), %xmm20
+# CHECK: encoding: [0x62,0xe3,0x7d,0x08,0x56,0x62,0x80,0x7b]
+         vreduceps	$123, -2048(%rdx), %xmm20
+# CHECK: vreduceps	$123, -2064(%rdx), %xmm20
+# CHECK: encoding: [0x62,0xe3,0x7d,0x08,0x56,0xa2,0xf0,0xf7,0xff,0xff,0x7b]
+         vreduceps	$123, -2064(%rdx), %xmm20
+# CHECK: vreduceps	$123, 508(%rdx){1to4}, %xmm20
+# CHECK: encoding: [0x62,0xe3,0x7d,0x18,0x56,0x62,0x7f,0x7b]
+         vreduceps	$123, 508(%rdx){1to4}, %xmm20
+# CHECK: vreduceps	$123, 512(%rdx){1to4}, %xmm20
+# CHECK: encoding: [0x62,0xe3,0x7d,0x18,0x56,0xa2,0x00,0x02,0x00,0x00,0x7b]
+         vreduceps	$123, 512(%rdx){1to4}, %xmm20
+# CHECK: vreduceps	$123, -512(%rdx){1to4}, %xmm20
+# CHECK: encoding: [0x62,0xe3,0x7d,0x18,0x56,0x62,0x80,0x7b]
+         vreduceps	$123, -512(%rdx){1to4}, %xmm20
+# CHECK: vreduceps	$123, -516(%rdx){1to4}, %xmm20
+# CHECK: encoding: [0x62,0xe3,0x7d,0x18,0x56,0xa2,0xfc,0xfd,0xff,0xff,0x7b]
+         vreduceps	$123, -516(%rdx){1to4}, %xmm20
+# CHECK: vreduceps	$171, %ymm22, %ymm26
+# CHECK: encoding: [0x62,0x23,0x7d,0x28,0x56,0xd6,0xab]
+         vreduceps	$171, %ymm22, %ymm26
+# CHECK: vreduceps	$171, %ymm22, %ymm26 {%k6}
+# CHECK: encoding: [0x62,0x23,0x7d,0x2e,0x56,0xd6,0xab]
+         vreduceps	$171, %ymm22, %ymm26 {%k6}
+# CHECK: vreduceps	$171, %ymm22, %ymm26 {%k6} {z}
+# CHECK: encoding: [0x62,0x23,0x7d,0xae,0x56,0xd6,0xab]
+         vreduceps	$171, %ymm22, %ymm26 {%k6} {z}
+# CHECK: vreduceps	$123, %ymm22, %ymm26
+# CHECK: encoding: [0x62,0x23,0x7d,0x28,0x56,0xd6,0x7b]
+         vreduceps	$123, %ymm22, %ymm26
+# CHECK: vreduceps	$123, (%rcx), %ymm26
+# CHECK: encoding: [0x62,0x63,0x7d,0x28,0x56,0x11,0x7b]
+         vreduceps	$123, (%rcx), %ymm26
+# CHECK: vreduceps	$123, 291(%rax,%r14,8), %ymm26
+# CHECK: encoding: [0x62,0x23,0x7d,0x28,0x56,0x94,0xf0,0x23,0x01,0x00,0x00,0x7b]
+         vreduceps	$123, 291(%rax,%r14,8), %ymm26
+# CHECK: vreduceps	$123, (%rcx){1to8}, %ymm26
+# CHECK: encoding: [0x62,0x63,0x7d,0x38,0x56,0x11,0x7b]
+         vreduceps	$123, (%rcx){1to8}, %ymm26
+# CHECK: vreduceps	$123, 4064(%rdx), %ymm26
+# CHECK: encoding: [0x62,0x63,0x7d,0x28,0x56,0x52,0x7f,0x7b]
+         vreduceps	$123, 4064(%rdx), %ymm26
+# CHECK: vreduceps	$123, 4096(%rdx), %ymm26
+# CHECK: encoding: [0x62,0x63,0x7d,0x28,0x56,0x92,0x00,0x10,0x00,0x00,0x7b]
+         vreduceps	$123, 4096(%rdx), %ymm26
+# CHECK: vreduceps	$123, -4096(%rdx), %ymm26
+# CHECK: encoding: [0x62,0x63,0x7d,0x28,0x56,0x52,0x80,0x7b]
+         vreduceps	$123, -4096(%rdx), %ymm26
+# CHECK: vreduceps	$123, -4128(%rdx), %ymm26
+# CHECK: encoding: [0x62,0x63,0x7d,0x28,0x56,0x92,0xe0,0xef,0xff,0xff,0x7b]
+         vreduceps	$123, -4128(%rdx), %ymm26
+# CHECK: vreduceps	$123, 508(%rdx){1to8}, %ymm26
+# CHECK: encoding: [0x62,0x63,0x7d,0x38,0x56,0x52,0x7f,0x7b]
+         vreduceps	$123, 508(%rdx){1to8}, %ymm26
+# CHECK: vreduceps	$123, 512(%rdx){1to8}, %ymm26
+# CHECK: encoding: [0x62,0x63,0x7d,0x38,0x56,0x92,0x00,0x02,0x00,0x00,0x7b]
+         vreduceps	$123, 512(%rdx){1to8}, %ymm26
+# CHECK: vreduceps	$123, -512(%rdx){1to8}, %ymm26
+# CHECK: encoding: [0x62,0x63,0x7d,0x38,0x56,0x52,0x80,0x7b]
+         vreduceps	$123, -512(%rdx){1to8}, %ymm26
+# CHECK: vreduceps	$123, -516(%rdx){1to8}, %ymm26
+# CHECK: encoding: [0x62,0x63,0x7d,0x38,0x56,0x92,0xfc,0xfd,0xff,0xff,0x7b]
+         vreduceps	$123, -516(%rdx){1to8}, %ymm26
+# CHECK: vcvtpd2qq	%xmm22, %xmm24
+# CHECK: encoding: [0x62,0x21,0xfd,0x08,0x7b,0xc6]
+         vcvtpd2qq	%xmm22, %xmm24
+# CHECK: vcvtpd2qq	%xmm22, %xmm24 {%k6}
+# CHECK: encoding: [0x62,0x21,0xfd,0x0e,0x7b,0xc6]
+         vcvtpd2qq	%xmm22, %xmm24 {%k6}
+# CHECK: vcvtpd2qq	%xmm22, %xmm24 {%k6} {z}
+# CHECK: encoding: [0x62,0x21,0xfd,0x8e,0x7b,0xc6]
+         vcvtpd2qq	%xmm22, %xmm24 {%k6} {z}
+# CHECK: vcvtpd2qq	(%rcx), %xmm24
+# CHECK: encoding: [0x62,0x61,0xfd,0x08,0x7b,0x01]
+         vcvtpd2qq	(%rcx), %xmm24
+# CHECK: vcvtpd2qq	291(%rax,%r14,8), %xmm24
+# CHECK: encoding: [0x62,0x21,0xfd,0x08,0x7b,0x84,0xf0,0x23,0x01,0x00,0x00]
+         vcvtpd2qq	291(%rax,%r14,8), %xmm24
+# CHECK: vcvtpd2qq	(%rcx){1to2}, %xmm24
+# CHECK: encoding: [0x62,0x61,0xfd,0x18,0x7b,0x01]
+         vcvtpd2qq	(%rcx){1to2}, %xmm24
+# CHECK: vcvtpd2qq	2032(%rdx), %xmm24
+# CHECK: encoding: [0x62,0x61,0xfd,0x08,0x7b,0x42,0x7f]
+         vcvtpd2qq	2032(%rdx), %xmm24
+# CHECK: vcvtpd2qq	2048(%rdx), %xmm24
+# CHECK: encoding: [0x62,0x61,0xfd,0x08,0x7b,0x82,0x00,0x08,0x00,0x00]
+         vcvtpd2qq	2048(%rdx), %xmm24
+# CHECK: vcvtpd2qq	-2048(%rdx), %xmm24
+# CHECK: encoding: [0x62,0x61,0xfd,0x08,0x7b,0x42,0x80]
+         vcvtpd2qq	-2048(%rdx), %xmm24
+# CHECK: vcvtpd2qq	-2064(%rdx), %xmm24
+# CHECK: encoding: [0x62,0x61,0xfd,0x08,0x7b,0x82,0xf0,0xf7,0xff,0xff]
+         vcvtpd2qq	-2064(%rdx), %xmm24
+# CHECK: vcvtpd2qq	1016(%rdx){1to2}, %xmm24
+# CHECK: encoding: [0x62,0x61,0xfd,0x18,0x7b,0x42,0x7f]
+         vcvtpd2qq	1016(%rdx){1to2}, %xmm24
+# CHECK: vcvtpd2qq	1024(%rdx){1to2}, %xmm24
+# CHECK: encoding: [0x62,0x61,0xfd,0x18,0x7b,0x82,0x00,0x04,0x00,0x00]
+         vcvtpd2qq	1024(%rdx){1to2}, %xmm24
+# CHECK: vcvtpd2qq	-1024(%rdx){1to2}, %xmm24
+# CHECK: encoding: [0x62,0x61,0xfd,0x18,0x7b,0x42,0x80]
+         vcvtpd2qq	-1024(%rdx){1to2}, %xmm24
+# CHECK: vcvtpd2qq	-1032(%rdx){1to2}, %xmm24
+# CHECK: encoding: [0x62,0x61,0xfd,0x18,0x7b,0x82,0xf8,0xfb,0xff,0xff]
+         vcvtpd2qq	-1032(%rdx){1to2}, %xmm24
+# CHECK: vcvtpd2qq	%ymm27, %ymm24
+# CHECK: encoding: [0x62,0x01,0xfd,0x28,0x7b,0xc3]
+         vcvtpd2qq	%ymm27, %ymm24
+# CHECK: vcvtpd2qq	%ymm27, %ymm24 {%k7}
+# CHECK: encoding: [0x62,0x01,0xfd,0x2f,0x7b,0xc3]
+         vcvtpd2qq	%ymm27, %ymm24 {%k7}
+# CHECK: vcvtpd2qq	%ymm27, %ymm24 {%k7} {z}
+# CHECK: encoding: [0x62,0x01,0xfd,0xaf,0x7b,0xc3]
+         vcvtpd2qq	%ymm27, %ymm24 {%k7} {z}
+# CHECK: vcvtpd2qq	(%rcx), %ymm24
+# CHECK: encoding: [0x62,0x61,0xfd,0x28,0x7b,0x01]
+         vcvtpd2qq	(%rcx), %ymm24
+# CHECK: vcvtpd2qq	291(%rax,%r14,8), %ymm24
+# CHECK: encoding: [0x62,0x21,0xfd,0x28,0x7b,0x84,0xf0,0x23,0x01,0x00,0x00]
+         vcvtpd2qq	291(%rax,%r14,8), %ymm24
+# CHECK: vcvtpd2qq	(%rcx){1to4}, %ymm24
+# CHECK: encoding: [0x62,0x61,0xfd,0x38,0x7b,0x01]
+         vcvtpd2qq	(%rcx){1to4}, %ymm24
+# CHECK: vcvtpd2qq	4064(%rdx), %ymm24
+# CHECK: encoding: [0x62,0x61,0xfd,0x28,0x7b,0x42,0x7f]
+         vcvtpd2qq	4064(%rdx), %ymm24
+# CHECK: vcvtpd2qq	4096(%rdx), %ymm24
+# CHECK: encoding: [0x62,0x61,0xfd,0x28,0x7b,0x82,0x00,0x10,0x00,0x00]
+         vcvtpd2qq	4096(%rdx), %ymm24
+# CHECK: vcvtpd2qq	-4096(%rdx), %ymm24
+# CHECK: encoding: [0x62,0x61,0xfd,0x28,0x7b,0x42,0x80]
+         vcvtpd2qq	-4096(%rdx), %ymm24
+# CHECK: vcvtpd2qq	-4128(%rdx), %ymm24
+# CHECK: encoding: [0x62,0x61,0xfd,0x28,0x7b,0x82,0xe0,0xef,0xff,0xff]
+         vcvtpd2qq	-4128(%rdx), %ymm24
+# CHECK: vcvtpd2qq	1016(%rdx){1to4}, %ymm24
+# CHECK: encoding: [0x62,0x61,0xfd,0x38,0x7b,0x42,0x7f]
+         vcvtpd2qq	1016(%rdx){1to4}, %ymm24
+# CHECK: vcvtpd2qq	1024(%rdx){1to4}, %ymm24
+# CHECK: encoding: [0x62,0x61,0xfd,0x38,0x7b,0x82,0x00,0x04,0x00,0x00]
+         vcvtpd2qq	1024(%rdx){1to4}, %ymm24
+# CHECK: vcvtpd2qq	-1024(%rdx){1to4}, %ymm24
+# CHECK: encoding: [0x62,0x61,0xfd,0x38,0x7b,0x42,0x80]
+         vcvtpd2qq	-1024(%rdx){1to4}, %ymm24
+# CHECK: vcvtpd2qq	-1032(%rdx){1to4}, %ymm24
+# CHECK: encoding: [0x62,0x61,0xfd,0x38,0x7b,0x82,0xf8,0xfb,0xff,0xff]
+         vcvtpd2qq	-1032(%rdx){1to4}, %ymm24
+# CHECK: vcvtpd2uqq	%xmm20, %xmm22
+# CHECK: encoding: [0x62,0xa1,0xfd,0x08,0x79,0xf4]
+         vcvtpd2uqq	%xmm20, %xmm22
+# CHECK: vcvtpd2uqq	%xmm20, %xmm22 {%k3}
+# CHECK: encoding: [0x62,0xa1,0xfd,0x0b,0x79,0xf4]
+         vcvtpd2uqq	%xmm20, %xmm22 {%k3}
+# CHECK: vcvtpd2uqq	%xmm20, %xmm22 {%k3} {z}
+# CHECK: encoding: [0x62,0xa1,0xfd,0x8b,0x79,0xf4]
+         vcvtpd2uqq	%xmm20, %xmm22 {%k3} {z}
+# CHECK: vcvtpd2uqq	(%rcx), %xmm22
+# CHECK: encoding: [0x62,0xe1,0xfd,0x08,0x79,0x31]
+         vcvtpd2uqq	(%rcx), %xmm22
+# CHECK: vcvtpd2uqq	291(%rax,%r14,8), %xmm22
+# CHECK: encoding: [0x62,0xa1,0xfd,0x08,0x79,0xb4,0xf0,0x23,0x01,0x00,0x00]
+         vcvtpd2uqq	291(%rax,%r14,8), %xmm22
+# CHECK: vcvtpd2uqq	(%rcx){1to2}, %xmm22
+# CHECK: encoding: [0x62,0xe1,0xfd,0x18,0x79,0x31]
+         vcvtpd2uqq	(%rcx){1to2}, %xmm22
+# CHECK: vcvtpd2uqq	2032(%rdx), %xmm22
+# CHECK: encoding: [0x62,0xe1,0xfd,0x08,0x79,0x72,0x7f]
+         vcvtpd2uqq	2032(%rdx), %xmm22
+# CHECK: vcvtpd2uqq	2048(%rdx), %xmm22
+# CHECK: encoding: [0x62,0xe1,0xfd,0x08,0x79,0xb2,0x00,0x08,0x00,0x00]
+         vcvtpd2uqq	2048(%rdx), %xmm22
+# CHECK: vcvtpd2uqq	-2048(%rdx), %xmm22
+# CHECK: encoding: [0x62,0xe1,0xfd,0x08,0x79,0x72,0x80]
+         vcvtpd2uqq	-2048(%rdx), %xmm22
+# CHECK: vcvtpd2uqq	-2064(%rdx), %xmm22
+# CHECK: encoding: [0x62,0xe1,0xfd,0x08,0x79,0xb2,0xf0,0xf7,0xff,0xff]
+         vcvtpd2uqq	-2064(%rdx), %xmm22
+# CHECK: vcvtpd2uqq	1016(%rdx){1to2}, %xmm22
+# CHECK: encoding: [0x62,0xe1,0xfd,0x18,0x79,0x72,0x7f]
+         vcvtpd2uqq	1016(%rdx){1to2}, %xmm22
+# CHECK: vcvtpd2uqq	1024(%rdx){1to2}, %xmm22
+# CHECK: encoding: [0x62,0xe1,0xfd,0x18,0x79,0xb2,0x00,0x04,0x00,0x00]
+         vcvtpd2uqq	1024(%rdx){1to2}, %xmm22
+# CHECK: vcvtpd2uqq	-1024(%rdx){1to2}, %xmm22
+# CHECK: encoding: [0x62,0xe1,0xfd,0x18,0x79,0x72,0x80]
+         vcvtpd2uqq	-1024(%rdx){1to2}, %xmm22
+# CHECK: vcvtpd2uqq	-1032(%rdx){1to2}, %xmm22
+# CHECK: encoding: [0x62,0xe1,0xfd,0x18,0x79,0xb2,0xf8,0xfb,0xff,0xff]
+         vcvtpd2uqq	-1032(%rdx){1to2}, %xmm22
+# CHECK: vcvtpd2uqq	%ymm24, %ymm21
+# CHECK: encoding: [0x62,0x81,0xfd,0x28,0x79,0xe8]
+         vcvtpd2uqq	%ymm24, %ymm21
+# CHECK: vcvtpd2uqq	%ymm24, %ymm21 {%k6}
+# CHECK: encoding: [0x62,0x81,0xfd,0x2e,0x79,0xe8]
+         vcvtpd2uqq	%ymm24, %ymm21 {%k6}
+# CHECK: vcvtpd2uqq	%ymm24, %ymm21 {%k6} {z}
+# CHECK: encoding: [0x62,0x81,0xfd,0xae,0x79,0xe8]
+         vcvtpd2uqq	%ymm24, %ymm21 {%k6} {z}
+# CHECK: vcvtpd2uqq	(%rcx), %ymm21
+# CHECK: encoding: [0x62,0xe1,0xfd,0x28,0x79,0x29]
+         vcvtpd2uqq	(%rcx), %ymm21
+# CHECK: vcvtpd2uqq	291(%rax,%r14,8), %ymm21
+# CHECK: encoding: [0x62,0xa1,0xfd,0x28,0x79,0xac,0xf0,0x23,0x01,0x00,0x00]
+         vcvtpd2uqq	291(%rax,%r14,8), %ymm21
+# CHECK: vcvtpd2uqq	(%rcx){1to4}, %ymm21
+# CHECK: encoding: [0x62,0xe1,0xfd,0x38,0x79,0x29]
+         vcvtpd2uqq	(%rcx){1to4}, %ymm21
+# CHECK: vcvtpd2uqq	4064(%rdx), %ymm21
+# CHECK: encoding: [0x62,0xe1,0xfd,0x28,0x79,0x6a,0x7f]
+         vcvtpd2uqq	4064(%rdx), %ymm21
+# CHECK: vcvtpd2uqq	4096(%rdx), %ymm21
+# CHECK: encoding: [0x62,0xe1,0xfd,0x28,0x79,0xaa,0x00,0x10,0x00,0x00]
+         vcvtpd2uqq	4096(%rdx), %ymm21
+# CHECK: vcvtpd2uqq	-4096(%rdx), %ymm21
+# CHECK: encoding: [0x62,0xe1,0xfd,0x28,0x79,0x6a,0x80]
+         vcvtpd2uqq	-4096(%rdx), %ymm21
+# CHECK: vcvtpd2uqq	-4128(%rdx), %ymm21
+# CHECK: encoding: [0x62,0xe1,0xfd,0x28,0x79,0xaa,0xe0,0xef,0xff,0xff]
+         vcvtpd2uqq	-4128(%rdx), %ymm21
+# CHECK: vcvtpd2uqq	1016(%rdx){1to4}, %ymm21
+# CHECK: encoding: [0x62,0xe1,0xfd,0x38,0x79,0x6a,0x7f]
+         vcvtpd2uqq	1016(%rdx){1to4}, %ymm21
+# CHECK: vcvtpd2uqq	1024(%rdx){1to4}, %ymm21
+# CHECK: encoding: [0x62,0xe1,0xfd,0x38,0x79,0xaa,0x00,0x04,0x00,0x00]
+         vcvtpd2uqq	1024(%rdx){1to4}, %ymm21
+# CHECK: vcvtpd2uqq	-1024(%rdx){1to4}, %ymm21
+# CHECK: encoding: [0x62,0xe1,0xfd,0x38,0x79,0x6a,0x80]
+         vcvtpd2uqq	-1024(%rdx){1to4}, %ymm21
+# CHECK: vcvtpd2uqq	-1032(%rdx){1to4}, %ymm21
+# CHECK: encoding: [0x62,0xe1,0xfd,0x38,0x79,0xaa,0xf8,0xfb,0xff,0xff]
+         vcvtpd2uqq	-1032(%rdx){1to4}, %ymm21
+# CHECK: vcvtps2qq	%xmm28, %xmm17
+# CHECK: encoding: [0x62,0x81,0x7d,0x08,0x7b,0xcc]
+         vcvtps2qq	%xmm28, %xmm17
+# CHECK: vcvtps2qq	%xmm28, %xmm17 {%k4}
+# CHECK: encoding: [0x62,0x81,0x7d,0x0c,0x7b,0xcc]
+         vcvtps2qq	%xmm28, %xmm17 {%k4}
+# CHECK: vcvtps2qq	%xmm28, %xmm17 {%k4} {z}
+# CHECK: encoding: [0x62,0x81,0x7d,0x8c,0x7b,0xcc]
+         vcvtps2qq	%xmm28, %xmm17 {%k4} {z}
+# CHECK: vcvtps2qq	(%rcx), %xmm17
+# CHECK: encoding: [0x62,0xe1,0x7d,0x08,0x7b,0x09]
+         vcvtps2qq	(%rcx), %xmm17
+# CHECK: vcvtps2qq	291(%rax,%r14,8), %xmm17
+# CHECK: encoding: [0x62,0xa1,0x7d,0x08,0x7b,0x8c,0xf0,0x23,0x01,0x00,0x00]
+         vcvtps2qq	291(%rax,%r14,8), %xmm17
+# CHECK: vcvtps2qq	(%rcx){1to2}, %xmm17
+# CHECK: encoding: [0x62,0xe1,0x7d,0x18,0x7b,0x09]
+         vcvtps2qq	(%rcx){1to2}, %xmm17
+# CHECK: vcvtps2qq	1016(%rdx), %xmm17
+# CHECK: encoding: [0x62,0xe1,0x7d,0x08,0x7b,0x4a,0x7f]
+         vcvtps2qq	1016(%rdx), %xmm17
+# CHECK: vcvtps2qq	1024(%rdx), %xmm17
+# CHECK: encoding: [0x62,0xe1,0x7d,0x08,0x7b,0x8a,0x00,0x04,0x00,0x00]
+         vcvtps2qq	1024(%rdx), %xmm17
+# CHECK: vcvtps2qq	-1024(%rdx), %xmm17
+# CHECK: encoding: [0x62,0xe1,0x7d,0x08,0x7b,0x4a,0x80]
+         vcvtps2qq	-1024(%rdx), %xmm17
+# CHECK: vcvtps2qq	-1032(%rdx), %xmm17
+# CHECK: encoding: [0x62,0xe1,0x7d,0x08,0x7b,0x8a,0xf8,0xfb,0xff,0xff]
+         vcvtps2qq	-1032(%rdx), %xmm17
+# CHECK: vcvtps2qq	508(%rdx){1to2}, %xmm17
+# CHECK: encoding: [0x62,0xe1,0x7d,0x18,0x7b,0x4a,0x7f]
+         vcvtps2qq	508(%rdx){1to2}, %xmm17
+# CHECK: vcvtps2qq	512(%rdx){1to2}, %xmm17
+# CHECK: encoding: [0x62,0xe1,0x7d,0x18,0x7b,0x8a,0x00,0x02,0x00,0x00]
+         vcvtps2qq	512(%rdx){1to2}, %xmm17
+# CHECK: vcvtps2qq	-512(%rdx){1to2}, %xmm17
+# CHECK: encoding: [0x62,0xe1,0x7d,0x18,0x7b,0x4a,0x80]
+         vcvtps2qq	-512(%rdx){1to2}, %xmm17
+# CHECK: vcvtps2qq	-516(%rdx){1to2}, %xmm17
+# CHECK: encoding: [0x62,0xe1,0x7d,0x18,0x7b,0x8a,0xfc,0xfd,0xff,0xff]
+         vcvtps2qq	-516(%rdx){1to2}, %xmm17
+# CHECK: vcvtps2qq	%xmm27, %ymm25
+# CHECK: encoding: [0x62,0x01,0x7d,0x28,0x7b,0xcb]
+         vcvtps2qq	%xmm27, %ymm25
+# CHECK: vcvtps2qq	%xmm27, %ymm25 {%k7}
+# CHECK: encoding: [0x62,0x01,0x7d,0x2f,0x7b,0xcb]
+         vcvtps2qq	%xmm27, %ymm25 {%k7}
+# CHECK: vcvtps2qq	%xmm27, %ymm25 {%k7} {z}
+# CHECK: encoding: [0x62,0x01,0x7d,0xaf,0x7b,0xcb]
+         vcvtps2qq	%xmm27, %ymm25 {%k7} {z}
+# CHECK: vcvtps2qq	(%rcx), %ymm25
+# CHECK: encoding: [0x62,0x61,0x7d,0x28,0x7b,0x09]
+         vcvtps2qq	(%rcx), %ymm25
+# CHECK: vcvtps2qq	291(%rax,%r14,8), %ymm25
+# CHECK: encoding: [0x62,0x21,0x7d,0x28,0x7b,0x8c,0xf0,0x23,0x01,0x00,0x00]
+         vcvtps2qq	291(%rax,%r14,8), %ymm25
+# CHECK: vcvtps2qq	(%rcx){1to4}, %ymm25
+# CHECK: encoding: [0x62,0x61,0x7d,0x38,0x7b,0x09]
+         vcvtps2qq	(%rcx){1to4}, %ymm25
+# CHECK: vcvtps2qq	2032(%rdx), %ymm25
+# CHECK: encoding: [0x62,0x61,0x7d,0x28,0x7b,0x4a,0x7f]
+         vcvtps2qq	2032(%rdx), %ymm25
+# CHECK: vcvtps2qq	2048(%rdx), %ymm25
+# CHECK: encoding: [0x62,0x61,0x7d,0x28,0x7b,0x8a,0x00,0x08,0x00,0x00]
+         vcvtps2qq	2048(%rdx), %ymm25
+# CHECK: vcvtps2qq	-2048(%rdx), %ymm25
+# CHECK: encoding: [0x62,0x61,0x7d,0x28,0x7b,0x4a,0x80]
+         vcvtps2qq	-2048(%rdx), %ymm25
+# CHECK: vcvtps2qq	-2064(%rdx), %ymm25
+# CHECK: encoding: [0x62,0x61,0x7d,0x28,0x7b,0x8a,0xf0,0xf7,0xff,0xff]
+         vcvtps2qq	-2064(%rdx), %ymm25
+# CHECK: vcvtps2qq	508(%rdx){1to4}, %ymm25
+# CHECK: encoding: [0x62,0x61,0x7d,0x38,0x7b,0x4a,0x7f]
+         vcvtps2qq	508(%rdx){1to4}, %ymm25
+# CHECK: vcvtps2qq	512(%rdx){1to4}, %ymm25
+# CHECK: encoding: [0x62,0x61,0x7d,0x38,0x7b,0x8a,0x00,0x02,0x00,0x00]
+         vcvtps2qq	512(%rdx){1to4}, %ymm25
+# CHECK: vcvtps2qq	-512(%rdx){1to4}, %ymm25
+# CHECK: encoding: [0x62,0x61,0x7d,0x38,0x7b,0x4a,0x80]
+         vcvtps2qq	-512(%rdx){1to4}, %ymm25
+# CHECK: vcvtps2qq	-516(%rdx){1to4}, %ymm25
+# CHECK: encoding: [0x62,0x61,0x7d,0x38,0x7b,0x8a,0xfc,0xfd,0xff,0xff]
+         vcvtps2qq	-516(%rdx){1to4}, %ymm25
+# CHECK: vcvtps2uqq	%xmm29, %xmm29
+# CHECK: encoding: [0x62,0x01,0x7d,0x08,0x79,0xed]
+         vcvtps2uqq	%xmm29, %xmm29
+# CHECK: vcvtps2uqq	%xmm29, %xmm29 {%k1}
+# CHECK: encoding: [0x62,0x01,0x7d,0x09,0x79,0xed]
+         vcvtps2uqq	%xmm29, %xmm29 {%k1}
+# CHECK: vcvtps2uqq	%xmm29, %xmm29 {%k1} {z}
+# CHECK: encoding: [0x62,0x01,0x7d,0x89,0x79,0xed]
+         vcvtps2uqq	%xmm29, %xmm29 {%k1} {z}
+# CHECK: vcvtps2uqq	(%rcx), %xmm29
+# CHECK: encoding: [0x62,0x61,0x7d,0x08,0x79,0x29]
+         vcvtps2uqq	(%rcx), %xmm29
+# CHECK: vcvtps2uqq	291(%rax,%r14,8), %xmm29
+# CHECK: encoding: [0x62,0x21,0x7d,0x08,0x79,0xac,0xf0,0x23,0x01,0x00,0x00]
+         vcvtps2uqq	291(%rax,%r14,8), %xmm29
+# CHECK: vcvtps2uqq	(%rcx){1to2}, %xmm29
+# CHECK: encoding: [0x62,0x61,0x7d,0x18,0x79,0x29]
+         vcvtps2uqq	(%rcx){1to2}, %xmm29
+# CHECK: vcvtps2uqq	1016(%rdx), %xmm29
+# CHECK: encoding: [0x62,0x61,0x7d,0x08,0x79,0x6a,0x7f]
+         vcvtps2uqq	1016(%rdx), %xmm29
+# CHECK: vcvtps2uqq	1024(%rdx), %xmm29
+# CHECK: encoding: [0x62,0x61,0x7d,0x08,0x79,0xaa,0x00,0x04,0x00,0x00]
+         vcvtps2uqq	1024(%rdx), %xmm29
+# CHECK: vcvtps2uqq	-1024(%rdx), %xmm29
+# CHECK: encoding: [0x62,0x61,0x7d,0x08,0x79,0x6a,0x80]
+         vcvtps2uqq	-1024(%rdx), %xmm29
+# CHECK: vcvtps2uqq	-1032(%rdx), %xmm29
+# CHECK: encoding: [0x62,0x61,0x7d,0x08,0x79,0xaa,0xf8,0xfb,0xff,0xff]
+         vcvtps2uqq	-1032(%rdx), %xmm29
+# CHECK: vcvtps2uqq	508(%rdx){1to2}, %xmm29
+# CHECK: encoding: [0x62,0x61,0x7d,0x18,0x79,0x6a,0x7f]
+         vcvtps2uqq	508(%rdx){1to2}, %xmm29
+# CHECK: vcvtps2uqq	512(%rdx){1to2}, %xmm29
+# CHECK: encoding: [0x62,0x61,0x7d,0x18,0x79,0xaa,0x00,0x02,0x00,0x00]
+         vcvtps2uqq	512(%rdx){1to2}, %xmm29
+# CHECK: vcvtps2uqq	-512(%rdx){1to2}, %xmm29
+# CHECK: encoding: [0x62,0x61,0x7d,0x18,0x79,0x6a,0x80]
+         vcvtps2uqq	-512(%rdx){1to2}, %xmm29
+# CHECK: vcvtps2uqq	-516(%rdx){1to2}, %xmm29
+# CHECK: encoding: [0x62,0x61,0x7d,0x18,0x79,0xaa,0xfc,0xfd,0xff,0xff]
+         vcvtps2uqq	-516(%rdx){1to2}, %xmm29
+# CHECK: vcvtps2uqq	%xmm19, %ymm23
+# CHECK: encoding: [0x62,0xa1,0x7d,0x28,0x79,0xfb]
+         vcvtps2uqq	%xmm19, %ymm23
+# CHECK: vcvtps2uqq	%xmm19, %ymm23 {%k2}
+# CHECK: encoding: [0x62,0xa1,0x7d,0x2a,0x79,0xfb]
+         vcvtps2uqq	%xmm19, %ymm23 {%k2}
+# CHECK: vcvtps2uqq	%xmm19, %ymm23 {%k2} {z}
+# CHECK: encoding: [0x62,0xa1,0x7d,0xaa,0x79,0xfb]
+         vcvtps2uqq	%xmm19, %ymm23 {%k2} {z}
+# CHECK: vcvtps2uqq	(%rcx), %ymm23
+# CHECK: encoding: [0x62,0xe1,0x7d,0x28,0x79,0x39]
+         vcvtps2uqq	(%rcx), %ymm23
+# CHECK: vcvtps2uqq	291(%rax,%r14,8), %ymm23
+# CHECK: encoding: [0x62,0xa1,0x7d,0x28,0x79,0xbc,0xf0,0x23,0x01,0x00,0x00]
+         vcvtps2uqq	291(%rax,%r14,8), %ymm23
+# CHECK: vcvtps2uqq	(%rcx){1to4}, %ymm23
+# CHECK: encoding: [0x62,0xe1,0x7d,0x38,0x79,0x39]
+         vcvtps2uqq	(%rcx){1to4}, %ymm23
+# CHECK: vcvtps2uqq	2032(%rdx), %ymm23
+# CHECK: encoding: [0x62,0xe1,0x7d,0x28,0x79,0x7a,0x7f]
+         vcvtps2uqq	2032(%rdx), %ymm23
+# CHECK: vcvtps2uqq	2048(%rdx), %ymm23
+# CHECK: encoding: [0x62,0xe1,0x7d,0x28,0x79,0xba,0x00,0x08,0x00,0x00]
+         vcvtps2uqq	2048(%rdx), %ymm23
+# CHECK: vcvtps2uqq	-2048(%rdx), %ymm23
+# CHECK: encoding: [0x62,0xe1,0x7d,0x28,0x79,0x7a,0x80]
+         vcvtps2uqq	-2048(%rdx), %ymm23
+# CHECK: vcvtps2uqq	-2064(%rdx), %ymm23
+# CHECK: encoding: [0x62,0xe1,0x7d,0x28,0x79,0xba,0xf0,0xf7,0xff,0xff]
+         vcvtps2uqq	-2064(%rdx), %ymm23
+# CHECK: vcvtps2uqq	508(%rdx){1to4}, %ymm23
+# CHECK: encoding: [0x62,0xe1,0x7d,0x38,0x79,0x7a,0x7f]
+         vcvtps2uqq	508(%rdx){1to4}, %ymm23
+# CHECK: vcvtps2uqq	512(%rdx){1to4}, %ymm23
+# CHECK: encoding: [0x62,0xe1,0x7d,0x38,0x79,0xba,0x00,0x02,0x00,0x00]
+         vcvtps2uqq	512(%rdx){1to4}, %ymm23
+# CHECK: vcvtps2uqq	-512(%rdx){1to4}, %ymm23
+# CHECK: encoding: [0x62,0xe1,0x7d,0x38,0x79,0x7a,0x80]
+         vcvtps2uqq	-512(%rdx){1to4}, %ymm23
+# CHECK: vcvtps2uqq	-516(%rdx){1to4}, %ymm23
+# CHECK: encoding: [0x62,0xe1,0x7d,0x38,0x79,0xba,0xfc,0xfd,0xff,0xff]
+         vcvtps2uqq	-516(%rdx){1to4}, %ymm23
+# CHECK: vcvtqq2pd	%xmm29, %xmm22
+# CHECK: encoding: [0x62,0x81,0xfe,0x08,0xe6,0xf5]
+         vcvtqq2pd	%xmm29, %xmm22
+# CHECK: vcvtqq2pd	%xmm29, %xmm22 {%k7}
+# CHECK: encoding: [0x62,0x81,0xfe,0x0f,0xe6,0xf5]
+         vcvtqq2pd	%xmm29, %xmm22 {%k7}
+# CHECK: vcvtqq2pd	%xmm29, %xmm22 {%k7} {z}
+# CHECK: encoding: [0x62,0x81,0xfe,0x8f,0xe6,0xf5]
+         vcvtqq2pd	%xmm29, %xmm22 {%k7} {z}
+# CHECK: vcvtqq2pd	(%rcx), %xmm22
+# CHECK: encoding: [0x62,0xe1,0xfe,0x08,0xe6,0x31]
+         vcvtqq2pd	(%rcx), %xmm22
+# CHECK: vcvtqq2pd	291(%rax,%r14,8), %xmm22
+# CHECK: encoding: [0x62,0xa1,0xfe,0x08,0xe6,0xb4,0xf0,0x23,0x01,0x00,0x00]
+         vcvtqq2pd	291(%rax,%r14,8), %xmm22
+# CHECK: vcvtqq2pd	(%rcx){1to2}, %xmm22
+# CHECK: encoding: [0x62,0xe1,0xfe,0x18,0xe6,0x31]
+         vcvtqq2pd	(%rcx){1to2}, %xmm22
+# CHECK: vcvtqq2pd	2032(%rdx), %xmm22
+# CHECK: encoding: [0x62,0xe1,0xfe,0x08,0xe6,0x72,0x7f]
+         vcvtqq2pd	2032(%rdx), %xmm22
+# CHECK: vcvtqq2pd	2048(%rdx), %xmm22
+# CHECK: encoding: [0x62,0xe1,0xfe,0x08,0xe6,0xb2,0x00,0x08,0x00,0x00]
+         vcvtqq2pd	2048(%rdx), %xmm22
+# CHECK: vcvtqq2pd	-2048(%rdx), %xmm22
+# CHECK: encoding: [0x62,0xe1,0xfe,0x08,0xe6,0x72,0x80]
+         vcvtqq2pd	-2048(%rdx), %xmm22
+# CHECK: vcvtqq2pd	-2064(%rdx), %xmm22
+# CHECK: encoding: [0x62,0xe1,0xfe,0x08,0xe6,0xb2,0xf0,0xf7,0xff,0xff]
+         vcvtqq2pd	-2064(%rdx), %xmm22
+# CHECK: vcvtqq2pd	1016(%rdx){1to2}, %xmm22
+# CHECK: encoding: [0x62,0xe1,0xfe,0x18,0xe6,0x72,0x7f]
+         vcvtqq2pd	1016(%rdx){1to2}, %xmm22
+# CHECK: vcvtqq2pd	1024(%rdx){1to2}, %xmm22
+# CHECK: encoding: [0x62,0xe1,0xfe,0x18,0xe6,0xb2,0x00,0x04,0x00,0x00]
+         vcvtqq2pd	1024(%rdx){1to2}, %xmm22
+# CHECK: vcvtqq2pd	-1024(%rdx){1to2}, %xmm22
+# CHECK: encoding: [0x62,0xe1,0xfe,0x18,0xe6,0x72,0x80]
+         vcvtqq2pd	-1024(%rdx){1to2}, %xmm22
+# CHECK: vcvtqq2pd	-1032(%rdx){1to2}, %xmm22
+# CHECK: encoding: [0x62,0xe1,0xfe,0x18,0xe6,0xb2,0xf8,0xfb,0xff,0xff]
+         vcvtqq2pd	-1032(%rdx){1to2}, %xmm22
+# CHECK: vcvtqq2pd	%ymm20, %ymm21
+# CHECK: encoding: [0x62,0xa1,0xfe,0x28,0xe6,0xec]
+         vcvtqq2pd	%ymm20, %ymm21
+# CHECK: vcvtqq2pd	%ymm20, %ymm21 {%k5}
+# CHECK: encoding: [0x62,0xa1,0xfe,0x2d,0xe6,0xec]
+         vcvtqq2pd	%ymm20, %ymm21 {%k5}
+# CHECK: vcvtqq2pd	%ymm20, %ymm21 {%k5} {z}
+# CHECK: encoding: [0x62,0xa1,0xfe,0xad,0xe6,0xec]
+         vcvtqq2pd	%ymm20, %ymm21 {%k5} {z}
+# CHECK: vcvtqq2pd	(%rcx), %ymm21
+# CHECK: encoding: [0x62,0xe1,0xfe,0x28,0xe6,0x29]
+         vcvtqq2pd	(%rcx), %ymm21
+# CHECK: vcvtqq2pd	291(%rax,%r14,8), %ymm21
+# CHECK: encoding: [0x62,0xa1,0xfe,0x28,0xe6,0xac,0xf0,0x23,0x01,0x00,0x00]
+         vcvtqq2pd	291(%rax,%r14,8), %ymm21
+# CHECK: vcvtqq2pd	(%rcx){1to4}, %ymm21
+# CHECK: encoding: [0x62,0xe1,0xfe,0x38,0xe6,0x29]
+         vcvtqq2pd	(%rcx){1to4}, %ymm21
+# CHECK: vcvtqq2pd	4064(%rdx), %ymm21
+# CHECK: encoding: [0x62,0xe1,0xfe,0x28,0xe6,0x6a,0x7f]
+         vcvtqq2pd	4064(%rdx), %ymm21
+# CHECK: vcvtqq2pd	4096(%rdx), %ymm21
+# CHECK: encoding: [0x62,0xe1,0xfe,0x28,0xe6,0xaa,0x00,0x10,0x00,0x00]
+         vcvtqq2pd	4096(%rdx), %ymm21
+# CHECK: vcvtqq2pd	-4096(%rdx), %ymm21
+# CHECK: encoding: [0x62,0xe1,0xfe,0x28,0xe6,0x6a,0x80]
+         vcvtqq2pd	-4096(%rdx), %ymm21
+# CHECK: vcvtqq2pd	-4128(%rdx), %ymm21
+# CHECK: encoding: [0x62,0xe1,0xfe,0x28,0xe6,0xaa,0xe0,0xef,0xff,0xff]
+         vcvtqq2pd	-4128(%rdx), %ymm21
+# CHECK: vcvtqq2pd	1016(%rdx){1to4}, %ymm21
+# CHECK: encoding: [0x62,0xe1,0xfe,0x38,0xe6,0x6a,0x7f]
+         vcvtqq2pd	1016(%rdx){1to4}, %ymm21
+# CHECK: vcvtqq2pd	1024(%rdx){1to4}, %ymm21
+# CHECK: encoding: [0x62,0xe1,0xfe,0x38,0xe6,0xaa,0x00,0x04,0x00,0x00]
+         vcvtqq2pd	1024(%rdx){1to4}, %ymm21
+# CHECK: vcvtqq2pd	-1024(%rdx){1to4}, %ymm21
+# CHECK: encoding: [0x62,0xe1,0xfe,0x38,0xe6,0x6a,0x80]
+         vcvtqq2pd	-1024(%rdx){1to4}, %ymm21
+# CHECK: vcvtqq2pd	-1032(%rdx){1to4}, %ymm21
+# CHECK: encoding: [0x62,0xe1,0xfe,0x38,0xe6,0xaa,0xf8,0xfb,0xff,0xff]
+         vcvtqq2pd	-1032(%rdx){1to4}, %ymm21
+# CHECK: vcvtqq2ps	%xmm28, %xmm25
+# CHECK: encoding: [0x62,0x01,0xfc,0x08,0x5b,0xcc]
+         vcvtqq2ps	%xmm28, %xmm25
+# CHECK: vcvtqq2ps	%xmm28, %xmm25
+# CHECK: encoding: [0x62,0x01,0xfc,0x08,0x5b,0xcc]
+         vcvtqq2ps	%xmm28, %xmm25
+# CHECK: vcvtqq2ps	%xmm28, %xmm25 {%k6}
+# CHECK: encoding: [0x62,0x01,0xfc,0x0e,0x5b,0xcc]
+         vcvtqq2ps	%xmm28, %xmm25 {%k6}
+# CHECK: vcvtqq2ps	%xmm28, %xmm25 {%k6}
+# CHECK: encoding: [0x62,0x01,0xfc,0x0e,0x5b,0xcc]
+         vcvtqq2ps	%xmm28, %xmm25 {%k6}
+# CHECK: vcvtqq2ps	%xmm28, %xmm25 {%k6} {z}
+# CHECK: encoding: [0x62,0x01,0xfc,0x8e,0x5b,0xcc]
+         vcvtqq2ps	%xmm28, %xmm25 {%k6} {z}
+# CHECK: vcvtqq2ps	%xmm28, %xmm25 {%k6} {z}
+# CHECK: encoding: [0x62,0x01,0xfc,0x8e,0x5b,0xcc]
+         vcvtqq2ps	%xmm28, %xmm25 {%k6} {z}
+# CHECK: vcvtqq2psx	(%rcx), %xmm25
+# CHECK: encoding: [0x62,0x61,0xfc,0x08,0x5b,0x09]
+         vcvtqq2psx	(%rcx), %xmm25
+# CHECK: vcvtqq2psx	(%rcx), %xmm25 {%k6}
+# CHECK: encoding: [0x62,0x61,0xfc,0x0e,0x5b,0x09]
+         vcvtqq2psx	(%rcx), %xmm25 {%k6}
+# CHECK: vcvtqq2psx	(%rcx), %xmm25 {%k6} {z}
+# CHECK: encoding: [0x62,0x61,0xfc,0x8e,0x5b,0x09]
+         vcvtqq2psx	(%rcx), %xmm25 {%k6} {z}
+# CHECK: vcvtqq2psx	291(%rax,%r14,8), %xmm25
+# CHECK: encoding: [0x62,0x21,0xfc,0x08,0x5b,0x8c,0xf0,0x23,0x01,0x00,0x00]
+         vcvtqq2psx	291(%rax,%r14,8), %xmm25
+# CHECK: vcvtqq2psx	291(%rax,%r14,8), %xmm25 {%k6}
+# CHECK: encoding: [0x62,0x21,0xfc,0x0e,0x5b,0x8c,0xf0,0x23,0x01,0x00,0x00]
+         vcvtqq2psx	291(%rax,%r14,8), %xmm25 {%k6}
+# CHECK: vcvtqq2psx	291(%rax,%r14,8), %xmm25 {%k6} {z}
+# CHECK: encoding: [0x62,0x21,0xfc,0x8e,0x5b,0x8c,0xf0,0x23,0x01,0x00,0x00]
+         vcvtqq2psx	291(%rax,%r14,8), %xmm25 {%k6} {z}
+# CHECK: vcvtqq2ps	(%rcx){1to2}, %xmm25
+# CHECK: encoding: [0x62,0x61,0xfc,0x18,0x5b,0x09]
+         vcvtqq2ps	(%rcx){1to2}, %xmm25
+# CHECK: vcvtqq2ps	(%rcx){1to2}, %xmm25
+# CHECK: encoding: [0x62,0x61,0xfc,0x18,0x5b,0x09]
+         vcvtqq2ps	(%rcx){1to2}, %xmm25
+# CHECK: vcvtqq2ps	(%rcx){1to2}, %xmm25 {%k6}
+# CHECK: encoding: [0x62,0x61,0xfc,0x1e,0x5b,0x09]
+         vcvtqq2ps	(%rcx){1to2}, %xmm25 {%k6}
+# CHECK: vcvtqq2ps	(%rcx){1to2}, %xmm25 {%k6}
+# CHECK: encoding: [0x62,0x61,0xfc,0x1e,0x5b,0x09]
+         vcvtqq2ps	(%rcx){1to2}, %xmm25 {%k6}
+# CHECK: vcvtqq2ps	(%rcx){1to2}, %xmm25 {%k6} {z}
+# CHECK: encoding: [0x62,0x61,0xfc,0x9e,0x5b,0x09]
+         vcvtqq2ps	(%rcx){1to2}, %xmm25 {%k6} {z}
+# CHECK: vcvtqq2ps	(%rcx){1to2}, %xmm25 {%k6} {z}
+# CHECK: encoding: [0x62,0x61,0xfc,0x9e,0x5b,0x09]
+         vcvtqq2ps	(%rcx){1to2}, %xmm25 {%k6} {z}
+# CHECK: vcvtqq2psx	2032(%rdx), %xmm25
+# CHECK: encoding: [0x62,0x61,0xfc,0x08,0x5b,0x4a,0x7f]
+         vcvtqq2psx	2032(%rdx), %xmm25
+# CHECK: vcvtqq2psx	2032(%rdx), %xmm25 {%k6}
+# CHECK: encoding: [0x62,0x61,0xfc,0x0e,0x5b,0x4a,0x7f]
+         vcvtqq2psx	2032(%rdx), %xmm25 {%k6}
+# CHECK: vcvtqq2psx	2032(%rdx), %xmm25 {%k6} {z}
+# CHECK: encoding: [0x62,0x61,0xfc,0x8e,0x5b,0x4a,0x7f]
+         vcvtqq2psx	2032(%rdx), %xmm25 {%k6} {z}
+# CHECK: vcvtqq2psx	2048(%rdx), %xmm25
+# CHECK: encoding: [0x62,0x61,0xfc,0x08,0x5b,0x8a,0x00,0x08,0x00,0x00]
+         vcvtqq2psx	2048(%rdx), %xmm25
+# CHECK: vcvtqq2psx	2048(%rdx), %xmm25 {%k6}
+# CHECK: encoding: [0x62,0x61,0xfc,0x0e,0x5b,0x8a,0x00,0x08,0x00,0x00]
+         vcvtqq2psx	2048(%rdx), %xmm25 {%k6}
+# CHECK: vcvtqq2psx	2048(%rdx), %xmm25 {%k6} {z}
+# CHECK: encoding: [0x62,0x61,0xfc,0x8e,0x5b,0x8a,0x00,0x08,0x00,0x00]
+         vcvtqq2psx	2048(%rdx), %xmm25 {%k6} {z}
+# CHECK: vcvtqq2psx	-2048(%rdx), %xmm25
+# CHECK: encoding: [0x62,0x61,0xfc,0x08,0x5b,0x4a,0x80]
+         vcvtqq2psx	-2048(%rdx), %xmm25
+# CHECK: vcvtqq2psx	-2048(%rdx), %xmm25 {%k6}
+# CHECK: encoding: [0x62,0x61,0xfc,0x0e,0x5b,0x4a,0x80]
+         vcvtqq2psx	-2048(%rdx), %xmm25 {%k6}
+# CHECK: vcvtqq2psx	-2048(%rdx), %xmm25 {%k6} {z}
+# CHECK: encoding: [0x62,0x61,0xfc,0x8e,0x5b,0x4a,0x80]
+         vcvtqq2psx	-2048(%rdx), %xmm25 {%k6} {z}
+# CHECK: vcvtqq2psx	-2064(%rdx), %xmm25
+# CHECK: encoding: [0x62,0x61,0xfc,0x08,0x5b,0x8a,0xf0,0xf7,0xff,0xff]
+         vcvtqq2psx	-2064(%rdx), %xmm25
+# CHECK: vcvtqq2psx	-2064(%rdx), %xmm25 {%k6}
+# CHECK: encoding: [0x62,0x61,0xfc,0x0e,0x5b,0x8a,0xf0,0xf7,0xff,0xff]
+         vcvtqq2psx	-2064(%rdx), %xmm25 {%k6}
+# CHECK: vcvtqq2psx	-2064(%rdx), %xmm25 {%k6} {z}
+# CHECK: encoding: [0x62,0x61,0xfc,0x8e,0x5b,0x8a,0xf0,0xf7,0xff,0xff]
+         vcvtqq2psx	-2064(%rdx), %xmm25 {%k6} {z}
+# CHECK: vcvtqq2ps	1016(%rdx){1to2}, %xmm25
+# CHECK: encoding: [0x62,0x61,0xfc,0x18,0x5b,0x4a,0x7f]
+         vcvtqq2ps	1016(%rdx){1to2}, %xmm25
+# CHECK: vcvtqq2ps	1016(%rdx){1to2}, %xmm25
+# CHECK: encoding: [0x62,0x61,0xfc,0x18,0x5b,0x4a,0x7f]
+         vcvtqq2ps	1016(%rdx){1to2}, %xmm25
+# CHECK: vcvtqq2ps	1016(%rdx){1to2}, %xmm25 {%k6}
+# CHECK: encoding: [0x62,0x61,0xfc,0x1e,0x5b,0x4a,0x7f]
+         vcvtqq2ps	1016(%rdx){1to2}, %xmm25 {%k6}
+# CHECK: vcvtqq2ps	1016(%rdx){1to2}, %xmm25 {%k6}
+# CHECK: encoding: [0x62,0x61,0xfc,0x1e,0x5b,0x4a,0x7f]
+         vcvtqq2ps	1016(%rdx){1to2}, %xmm25 {%k6}
+# CHECK: vcvtqq2ps	1016(%rdx){1to2}, %xmm25 {%k6} {z}
+# CHECK: encoding: [0x62,0x61,0xfc,0x9e,0x5b,0x4a,0x7f]
+         vcvtqq2ps	1016(%rdx){1to2}, %xmm25 {%k6} {z}
+# CHECK: vcvtqq2ps	1016(%rdx){1to2}, %xmm25 {%k6} {z}
+# CHECK: encoding: [0x62,0x61,0xfc,0x9e,0x5b,0x4a,0x7f]
+         vcvtqq2ps	1016(%rdx){1to2}, %xmm25 {%k6} {z}
+# CHECK: vcvtqq2ps	1024(%rdx){1to2}, %xmm25
+# CHECK: encoding: [0x62,0x61,0xfc,0x18,0x5b,0x8a,0x00,0x04,0x00,0x00]
+         vcvtqq2ps	1024(%rdx){1to2}, %xmm25
+# CHECK: vcvtqq2ps	1024(%rdx){1to2}, %xmm25
+# CHECK: encoding: [0x62,0x61,0xfc,0x18,0x5b,0x8a,0x00,0x04,0x00,0x00]
+         vcvtqq2ps	1024(%rdx){1to2}, %xmm25
+# CHECK: vcvtqq2ps	1024(%rdx){1to2}, %xmm25 {%k6}
+# CHECK: encoding: [0x62,0x61,0xfc,0x1e,0x5b,0x8a,0x00,0x04,0x00,0x00]
+         vcvtqq2ps	1024(%rdx){1to2}, %xmm25 {%k6}
+# CHECK: vcvtqq2ps	1024(%rdx){1to2}, %xmm25 {%k6}
+# CHECK: encoding: [0x62,0x61,0xfc,0x1e,0x5b,0x8a,0x00,0x04,0x00,0x00]
+         vcvtqq2ps	1024(%rdx){1to2}, %xmm25 {%k6}
+# CHECK: vcvtqq2ps	1024(%rdx){1to2}, %xmm25 {%k6} {z}
+# CHECK: encoding: [0x62,0x61,0xfc,0x9e,0x5b,0x8a,0x00,0x04,0x00,0x00]
+         vcvtqq2ps	1024(%rdx){1to2}, %xmm25 {%k6} {z}
+# CHECK: vcvtqq2ps	1024(%rdx){1to2}, %xmm25 {%k6} {z}
+# CHECK: encoding: [0x62,0x61,0xfc,0x9e,0x5b,0x8a,0x00,0x04,0x00,0x00]
+         vcvtqq2ps	1024(%rdx){1to2}, %xmm25 {%k6} {z}
+# CHECK: vcvtqq2ps	-1024(%rdx){1to2}, %xmm25
+# CHECK: encoding: [0x62,0x61,0xfc,0x18,0x5b,0x4a,0x80]
+         vcvtqq2ps	-1024(%rdx){1to2}, %xmm25
+# CHECK: vcvtqq2ps	-1024(%rdx){1to2}, %xmm25
+# CHECK: encoding: [0x62,0x61,0xfc,0x18,0x5b,0x4a,0x80]
+         vcvtqq2ps	-1024(%rdx){1to2}, %xmm25
+# CHECK: vcvtqq2ps	-1024(%rdx){1to2}, %xmm25 {%k6}
+# CHECK: encoding: [0x62,0x61,0xfc,0x1e,0x5b,0x4a,0x80]
+         vcvtqq2ps	-1024(%rdx){1to2}, %xmm25 {%k6}
+# CHECK: vcvtqq2ps	-1024(%rdx){1to2}, %xmm25 {%k6}
+# CHECK: encoding: [0x62,0x61,0xfc,0x1e,0x5b,0x4a,0x80]
+         vcvtqq2ps	-1024(%rdx){1to2}, %xmm25 {%k6}
+# CHECK: vcvtqq2ps	-1024(%rdx){1to2}, %xmm25 {%k6} {z}
+# CHECK: encoding: [0x62,0x61,0xfc,0x9e,0x5b,0x4a,0x80]
+         vcvtqq2ps	-1024(%rdx){1to2}, %xmm25 {%k6} {z}
+# CHECK: vcvtqq2ps	-1024(%rdx){1to2}, %xmm25 {%k6} {z}
+# CHECK: encoding: [0x62,0x61,0xfc,0x9e,0x5b,0x4a,0x80]
+         vcvtqq2ps	-1024(%rdx){1to2}, %xmm25 {%k6} {z}
+# CHECK: vcvtqq2ps	-1032(%rdx){1to2}, %xmm25
+# CHECK: encoding: [0x62,0x61,0xfc,0x18,0x5b,0x8a,0xf8,0xfb,0xff,0xff]
+         vcvtqq2ps	-1032(%rdx){1to2}, %xmm25
+# CHECK: vcvtqq2ps	-1032(%rdx){1to2}, %xmm25
+# CHECK: encoding: [0x62,0x61,0xfc,0x18,0x5b,0x8a,0xf8,0xfb,0xff,0xff]
+         vcvtqq2ps	-1032(%rdx){1to2}, %xmm25
+# CHECK: vcvtqq2ps	-1032(%rdx){1to2}, %xmm25 {%k6}
+# CHECK: encoding: [0x62,0x61,0xfc,0x1e,0x5b,0x8a,0xf8,0xfb,0xff,0xff]
+         vcvtqq2ps	-1032(%rdx){1to2}, %xmm25 {%k6}
+# CHECK: vcvtqq2ps	-1032(%rdx){1to2}, %xmm25 {%k6}
+# CHECK: encoding: [0x62,0x61,0xfc,0x1e,0x5b,0x8a,0xf8,0xfb,0xff,0xff]
+         vcvtqq2ps	-1032(%rdx){1to2}, %xmm25 {%k6}
+# CHECK: vcvtqq2ps	-1032(%rdx){1to2}, %xmm25 {%k6} {z}
+# CHECK: encoding: [0x62,0x61,0xfc,0x9e,0x5b,0x8a,0xf8,0xfb,0xff,0xff]
+         vcvtqq2ps	-1032(%rdx){1to2}, %xmm25 {%k6} {z}
+# CHECK: vcvtqq2ps	-1032(%rdx){1to2}, %xmm25 {%k6} {z}
+# CHECK: encoding: [0x62,0x61,0xfc,0x9e,0x5b,0x8a,0xf8,0xfb,0xff,0xff]
+         vcvtqq2ps	-1032(%rdx){1to2}, %xmm25 {%k6} {z}
+# CHECK: vcvtqq2ps	%ymm22, %xmm27
+# CHECK: encoding: [0x62,0x21,0xfc,0x28,0x5b,0xde]
+         vcvtqq2ps	%ymm22, %xmm27
+# CHECK: vcvtqq2ps	%ymm22, %xmm27
+# CHECK: encoding: [0x62,0x21,0xfc,0x28,0x5b,0xde]
+         vcvtqq2ps	%ymm22, %xmm27
+# CHECK: vcvtqq2ps	%ymm22, %xmm27 {%k3}
+# CHECK: encoding: [0x62,0x21,0xfc,0x2b,0x5b,0xde]
+         vcvtqq2ps	%ymm22, %xmm27 {%k3}
+# CHECK: vcvtqq2ps	%ymm22, %xmm27 {%k3}
+# CHECK: encoding: [0x62,0x21,0xfc,0x2b,0x5b,0xde]
+         vcvtqq2ps	%ymm22, %xmm27 {%k3}
+# CHECK: vcvtqq2ps	%ymm22, %xmm27 {%k3} {z}
+# CHECK: encoding: [0x62,0x21,0xfc,0xab,0x5b,0xde]
+         vcvtqq2ps	%ymm22, %xmm27 {%k3} {z}
+# CHECK: vcvtqq2ps	%ymm22, %xmm27 {%k3} {z}
+# CHECK: encoding: [0x62,0x21,0xfc,0xab,0x5b,0xde]
+         vcvtqq2ps	%ymm22, %xmm27 {%k3} {z}
+# CHECK: vcvtqq2psy	(%rcx), %xmm27
+# CHECK: encoding: [0x62,0x61,0xfc,0x28,0x5b,0x19]
+         vcvtqq2psy	(%rcx), %xmm27
+# CHECK: vcvtqq2psy	(%rcx), %xmm27 {%k3}
+# CHECK: encoding: [0x62,0x61,0xfc,0x2b,0x5b,0x19]
+         vcvtqq2psy	(%rcx), %xmm27 {%k3}
+# CHECK: vcvtqq2psy	(%rcx), %xmm27 {%k3} {z}
+# CHECK: encoding: [0x62,0x61,0xfc,0xab,0x5b,0x19]
+         vcvtqq2psy	(%rcx), %xmm27 {%k3} {z}
+# CHECK: vcvtqq2psy	291(%rax,%r14,8), %xmm27
+# CHECK: encoding: [0x62,0x21,0xfc,0x28,0x5b,0x9c,0xf0,0x23,0x01,0x00,0x00]
+         vcvtqq2psy	291(%rax,%r14,8), %xmm27
+# CHECK: vcvtqq2psy	291(%rax,%r14,8), %xmm27 {%k3}
+# CHECK: encoding: [0x62,0x21,0xfc,0x2b,0x5b,0x9c,0xf0,0x23,0x01,0x00,0x00]
+         vcvtqq2psy	291(%rax,%r14,8), %xmm27 {%k3}
+# CHECK: vcvtqq2psy	291(%rax,%r14,8), %xmm27 {%k3} {z}
+# CHECK: encoding: [0x62,0x21,0xfc,0xab,0x5b,0x9c,0xf0,0x23,0x01,0x00,0x00]
+         vcvtqq2psy	291(%rax,%r14,8), %xmm27 {%k3} {z}
+# CHECK: vcvtqq2ps	(%rcx){1to4}, %xmm27
+# CHECK: encoding: [0x62,0x61,0xfc,0x38,0x5b,0x19]
+         vcvtqq2ps	(%rcx){1to4}, %xmm27
+# CHECK: vcvtqq2ps	(%rcx){1to4}, %xmm27 {%k3}
+# CHECK: encoding: [0x62,0x61,0xfc,0x3b,0x5b,0x19]
+         vcvtqq2ps	(%rcx){1to4}, %xmm27 {%k3}
+# CHECK: vcvtqq2ps	(%rcx){1to4}, %xmm27 {%k3} {z}
+# CHECK: encoding: [0x62,0x61,0xfc,0xbb,0x5b,0x19]
+         vcvtqq2ps	(%rcx){1to4}, %xmm27 {%k3} {z}
+# CHECK: vcvtqq2psy	4064(%rdx), %xmm27
+# CHECK: encoding: [0x62,0x61,0xfc,0x28,0x5b,0x5a,0x7f]
+         vcvtqq2psy	4064(%rdx), %xmm27
+# CHECK: vcvtqq2psy	4064(%rdx), %xmm27 {%k3}
+# CHECK: encoding: [0x62,0x61,0xfc,0x2b,0x5b,0x5a,0x7f]
+         vcvtqq2psy	4064(%rdx), %xmm27 {%k3}
+# CHECK: vcvtqq2psy	4064(%rdx), %xmm27 {%k3} {z}
+# CHECK: encoding: [0x62,0x61,0xfc,0xab,0x5b,0x5a,0x7f]
+         vcvtqq2psy	4064(%rdx), %xmm27 {%k3} {z}
+# CHECK: vcvtqq2psy	4096(%rdx), %xmm27
+# CHECK: encoding: [0x62,0x61,0xfc,0x28,0x5b,0x9a,0x00,0x10,0x00,0x00]
+         vcvtqq2psy	4096(%rdx), %xmm27
+# CHECK: vcvtqq2psy	4096(%rdx), %xmm27 {%k3}
+# CHECK: encoding: [0x62,0x61,0xfc,0x2b,0x5b,0x9a,0x00,0x10,0x00,0x00]
+         vcvtqq2psy	4096(%rdx), %xmm27 {%k3}
+# CHECK: vcvtqq2psy	4096(%rdx), %xmm27 {%k3} {z}
+# CHECK: encoding: [0x62,0x61,0xfc,0xab,0x5b,0x9a,0x00,0x10,0x00,0x00]
+         vcvtqq2psy	4096(%rdx), %xmm27 {%k3} {z}
+# CHECK: vcvtqq2psy	-4096(%rdx), %xmm27
+# CHECK: encoding: [0x62,0x61,0xfc,0x28,0x5b,0x5a,0x80]
+         vcvtqq2psy	-4096(%rdx), %xmm27
+# CHECK: vcvtqq2psy	-4096(%rdx), %xmm27 {%k3}
+# CHECK: encoding: [0x62,0x61,0xfc,0x2b,0x5b,0x5a,0x80]
+         vcvtqq2psy	-4096(%rdx), %xmm27 {%k3}
+# CHECK: vcvtqq2psy	-4096(%rdx), %xmm27 {%k3} {z}
+# CHECK: encoding: [0x62,0x61,0xfc,0xab,0x5b,0x5a,0x80]
+         vcvtqq2psy	-4096(%rdx), %xmm27 {%k3} {z}
+# CHECK: vcvtqq2psy	-4128(%rdx), %xmm27
+# CHECK: encoding: [0x62,0x61,0xfc,0x28,0x5b,0x9a,0xe0,0xef,0xff,0xff]
+         vcvtqq2psy	-4128(%rdx), %xmm27
+# CHECK: vcvtqq2psy	-4128(%rdx), %xmm27 {%k3}
+# CHECK: encoding: [0x62,0x61,0xfc,0x2b,0x5b,0x9a,0xe0,0xef,0xff,0xff]
+         vcvtqq2psy	-4128(%rdx), %xmm27 {%k3}
+# CHECK: vcvtqq2psy	-4128(%rdx), %xmm27 {%k3} {z}
+# CHECK: encoding: [0x62,0x61,0xfc,0xab,0x5b,0x9a,0xe0,0xef,0xff,0xff]
+         vcvtqq2psy	-4128(%rdx), %xmm27 {%k3} {z}
+# CHECK: vcvtqq2ps	1016(%rdx){1to4}, %xmm27
+# CHECK: encoding: [0x62,0x61,0xfc,0x38,0x5b,0x5a,0x7f]
+         vcvtqq2ps	1016(%rdx){1to4}, %xmm27
+# CHECK: vcvtqq2ps	1016(%rdx){1to4}, %xmm27
+# CHECK: encoding: [0x62,0x61,0xfc,0x38,0x5b,0x5a,0x7f]
+         vcvtqq2ps	1016(%rdx){1to4}, %xmm27
+# CHECK: vcvtqq2ps	1016(%rdx){1to4}, %xmm27 {%k3}
+# CHECK: encoding: [0x62,0x61,0xfc,0x3b,0x5b,0x5a,0x7f]
+         vcvtqq2ps	1016(%rdx){1to4}, %xmm27 {%k3}
+# CHECK: vcvtqq2ps	1016(%rdx){1to4}, %xmm27 {%k3}
+# CHECK: encoding: [0x62,0x61,0xfc,0x3b,0x5b,0x5a,0x7f]
+         vcvtqq2ps	1016(%rdx){1to4}, %xmm27 {%k3}
+# CHECK: vcvtqq2ps	1016(%rdx){1to4}, %xmm27 {%k3} {z}
+# CHECK: encoding: [0x62,0x61,0xfc,0xbb,0x5b,0x5a,0x7f]
+         vcvtqq2ps	1016(%rdx){1to4}, %xmm27 {%k3} {z}
+# CHECK: vcvtqq2ps	1016(%rdx){1to4}, %xmm27 {%k3} {z}
+# CHECK: encoding: [0x62,0x61,0xfc,0xbb,0x5b,0x5a,0x7f]
+         vcvtqq2ps	1016(%rdx){1to4}, %xmm27 {%k3} {z}
+# CHECK: vcvtqq2ps	1024(%rdx){1to4}, %xmm27
+# CHECK: encoding: [0x62,0x61,0xfc,0x38,0x5b,0x9a,0x00,0x04,0x00,0x00]
+         vcvtqq2ps	1024(%rdx){1to4}, %xmm27
+# CHECK: vcvtqq2ps	1024(%rdx){1to4}, %xmm27
+# CHECK: encoding: [0x62,0x61,0xfc,0x38,0x5b,0x9a,0x00,0x04,0x00,0x00]
+         vcvtqq2ps	1024(%rdx){1to4}, %xmm27
+# CHECK: vcvtqq2ps	1024(%rdx){1to4}, %xmm27 {%k3}
+# CHECK: encoding: [0x62,0x61,0xfc,0x3b,0x5b,0x9a,0x00,0x04,0x00,0x00]
+         vcvtqq2ps	1024(%rdx){1to4}, %xmm27 {%k3}
+# CHECK: vcvtqq2ps	1024(%rdx){1to4}, %xmm27 {%k3}
+# CHECK: encoding: [0x62,0x61,0xfc,0x3b,0x5b,0x9a,0x00,0x04,0x00,0x00]
+         vcvtqq2ps	1024(%rdx){1to4}, %xmm27 {%k3}
+# CHECK: vcvtqq2ps	1024(%rdx){1to4}, %xmm27 {%k3} {z}
+# CHECK: encoding: [0x62,0x61,0xfc,0xbb,0x5b,0x9a,0x00,0x04,0x00,0x00]
+         vcvtqq2ps	1024(%rdx){1to4}, %xmm27 {%k3} {z}
+# CHECK: vcvtqq2ps	1024(%rdx){1to4}, %xmm27 {%k3} {z}
+# CHECK: encoding: [0x62,0x61,0xfc,0xbb,0x5b,0x9a,0x00,0x04,0x00,0x00]
+         vcvtqq2ps	1024(%rdx){1to4}, %xmm27 {%k3} {z}
+# CHECK: vcvtqq2ps	-1024(%rdx){1to4}, %xmm27
+# CHECK: encoding: [0x62,0x61,0xfc,0x38,0x5b,0x5a,0x80]
+         vcvtqq2ps	-1024(%rdx){1to4}, %xmm27
+# CHECK: vcvtqq2ps	-1024(%rdx){1to4}, %xmm27
+# CHECK: encoding: [0x62,0x61,0xfc,0x38,0x5b,0x5a,0x80]
+         vcvtqq2ps	-1024(%rdx){1to4}, %xmm27
+# CHECK: vcvtqq2ps	-1024(%rdx){1to4}, %xmm27 {%k3}
+# CHECK: encoding: [0x62,0x61,0xfc,0x3b,0x5b,0x5a,0x80]
+         vcvtqq2ps	-1024(%rdx){1to4}, %xmm27 {%k3}
+# CHECK: vcvtqq2ps	-1024(%rdx){1to4}, %xmm27 {%k3}
+# CHECK: encoding: [0x62,0x61,0xfc,0x3b,0x5b,0x5a,0x80]
+         vcvtqq2ps	-1024(%rdx){1to4}, %xmm27 {%k3}
+# CHECK: vcvtqq2ps	-1024(%rdx){1to4}, %xmm27 {%k3} {z}
+# CHECK: encoding: [0x62,0x61,0xfc,0xbb,0x5b,0x5a,0x80]
+         vcvtqq2ps	-1024(%rdx){1to4}, %xmm27 {%k3} {z}
+# CHECK: vcvtqq2ps	-1024(%rdx){1to4}, %xmm27 {%k3} {z}
+# CHECK: encoding: [0x62,0x61,0xfc,0xbb,0x5b,0x5a,0x80]
+         vcvtqq2ps	-1024(%rdx){1to4}, %xmm27 {%k3} {z}
+# CHECK: vcvtqq2ps	-1032(%rdx){1to4}, %xmm27
+# CHECK: encoding: [0x62,0x61,0xfc,0x38,0x5b,0x9a,0xf8,0xfb,0xff,0xff]
+         vcvtqq2ps	-1032(%rdx){1to4}, %xmm27
+# CHECK: vcvtqq2ps	-1032(%rdx){1to4}, %xmm27
+# CHECK: encoding: [0x62,0x61,0xfc,0x38,0x5b,0x9a,0xf8,0xfb,0xff,0xff]
+         vcvtqq2ps	-1032(%rdx){1to4}, %xmm27
+# CHECK: vcvtqq2ps	-1032(%rdx){1to4}, %xmm27 {%k3}
+# CHECK: encoding: [0x62,0x61,0xfc,0x3b,0x5b,0x9a,0xf8,0xfb,0xff,0xff]
+         vcvtqq2ps	-1032(%rdx){1to4}, %xmm27 {%k3}
+# CHECK: vcvtqq2ps	-1032(%rdx){1to4}, %xmm27 {%k3}
+# CHECK: encoding: [0x62,0x61,0xfc,0x3b,0x5b,0x9a,0xf8,0xfb,0xff,0xff]
+         vcvtqq2ps	-1032(%rdx){1to4}, %xmm27 {%k3}
+# CHECK: vcvtqq2ps	-1032(%rdx){1to4}, %xmm27 {%k3} {z}
+# CHECK: encoding: [0x62,0x61,0xfc,0xbb,0x5b,0x9a,0xf8,0xfb,0xff,0xff]
+         vcvtqq2ps	-1032(%rdx){1to4}, %xmm27 {%k3} {z}
+# CHECK: vcvtqq2ps	-1032(%rdx){1to4}, %xmm27 {%k3} {z}
+# CHECK: encoding: [0x62,0x61,0xfc,0xbb,0x5b,0x9a,0xf8,0xfb,0xff,0xff]
+         vcvtqq2ps	-1032(%rdx){1to4}, %xmm27 {%k3} {z}
+# CHECK: vcvtqq2ps	%xmm26, %xmm30
+# CHECK: encoding: [0x62,0x01,0xfc,0x08,0x5b,0xf2]
+         vcvtqq2ps	%xmm26, %xmm30
+# CHECK: vcvtqq2ps	%xmm26, %xmm30
+# CHECK: encoding: [0x62,0x01,0xfc,0x08,0x5b,0xf2]
+         vcvtqq2ps	%xmm26, %xmm30
+# CHECK: vcvtqq2ps	%xmm26, %xmm30 {%k4}
+# CHECK: encoding: [0x62,0x01,0xfc,0x0c,0x5b,0xf2]
+         vcvtqq2ps	%xmm26, %xmm30 {%k4}
+# CHECK: vcvtqq2ps	%xmm26, %xmm30 {%k4}
+# CHECK: encoding: [0x62,0x01,0xfc,0x0c,0x5b,0xf2]
+         vcvtqq2ps	%xmm26, %xmm30 {%k4}
+# CHECK: vcvtqq2ps	%xmm26, %xmm30 {%k4} {z}
+# CHECK: encoding: [0x62,0x01,0xfc,0x8c,0x5b,0xf2]
+         vcvtqq2ps	%xmm26, %xmm30 {%k4} {z}
+# CHECK: vcvtqq2ps	%xmm26, %xmm30 {%k4} {z}
+# CHECK: encoding: [0x62,0x01,0xfc,0x8c,0x5b,0xf2]
+         vcvtqq2ps	%xmm26, %xmm30 {%k4} {z}
+# CHECK: vcvtqq2psx	(%rcx), %xmm30
+# CHECK: encoding: [0x62,0x61,0xfc,0x08,0x5b,0x31]
+         vcvtqq2psx	(%rcx), %xmm30
+# CHECK: vcvtqq2psx	(%rcx), %xmm30 {%k4}
+# CHECK: encoding: [0x62,0x61,0xfc,0x0c,0x5b,0x31]
+         vcvtqq2psx	(%rcx), %xmm30 {%k4}
+# CHECK: vcvtqq2psx	(%rcx), %xmm30 {%k4} {z}
+# CHECK: encoding: [0x62,0x61,0xfc,0x8c,0x5b,0x31]
+         vcvtqq2psx	(%rcx), %xmm30 {%k4} {z}
+# CHECK: vcvtqq2psx	4660(%rax,%r14,8), %xmm30
+# CHECK: encoding: [0x62,0x21,0xfc,0x08,0x5b,0xb4,0xf0,0x34,0x12,0x00,0x00]
+         vcvtqq2psx	4660(%rax,%r14,8), %xmm30
+# CHECK: vcvtqq2psx	4660(%rax,%r14,8), %xmm30 {%k4}
+# CHECK: encoding: [0x62,0x21,0xfc,0x0c,0x5b,0xb4,0xf0,0x34,0x12,0x00,0x00]
+         vcvtqq2psx	4660(%rax,%r14,8), %xmm30 {%k4}
+# CHECK: vcvtqq2psx	4660(%rax,%r14,8), %xmm30 {%k4} {z}
+# CHECK: encoding: [0x62,0x21,0xfc,0x8c,0x5b,0xb4,0xf0,0x34,0x12,0x00,0x00]
+         vcvtqq2psx	4660(%rax,%r14,8), %xmm30 {%k4} {z}
+# CHECK: vcvtqq2ps	(%rcx){1to2}, %xmm30
+# CHECK: encoding: [0x62,0x61,0xfc,0x18,0x5b,0x31]
+         vcvtqq2ps	(%rcx){1to2}, %xmm30
+# CHECK: vcvtqq2ps	(%rcx){1to2}, %xmm30
+# CHECK: encoding: [0x62,0x61,0xfc,0x18,0x5b,0x31]
+         vcvtqq2ps	(%rcx){1to2}, %xmm30
+# CHECK: vcvtqq2ps	(%rcx){1to2}, %xmm30 {%k4}
+# CHECK: encoding: [0x62,0x61,0xfc,0x1c,0x5b,0x31]
+         vcvtqq2ps	(%rcx){1to2}, %xmm30 {%k4}
+# CHECK: vcvtqq2ps	(%rcx){1to2}, %xmm30 {%k4}
+# CHECK: encoding: [0x62,0x61,0xfc,0x1c,0x5b,0x31]
+         vcvtqq2ps	(%rcx){1to2}, %xmm30 {%k4}
+# CHECK: vcvtqq2ps	(%rcx){1to2}, %xmm30 {%k4} {z}
+# CHECK: encoding: [0x62,0x61,0xfc,0x9c,0x5b,0x31]
+         vcvtqq2ps	(%rcx){1to2}, %xmm30 {%k4} {z}
+# CHECK: vcvtqq2ps	(%rcx){1to2}, %xmm30 {%k4} {z}
+# CHECK: encoding: [0x62,0x61,0xfc,0x9c,0x5b,0x31]
+         vcvtqq2ps	(%rcx){1to2}, %xmm30 {%k4} {z}
+# CHECK: vcvtqq2psx	2032(%rdx), %xmm30
+# CHECK: encoding: [0x62,0x61,0xfc,0x08,0x5b,0x72,0x7f]
+         vcvtqq2psx	2032(%rdx), %xmm30
+# CHECK: vcvtqq2psx	2032(%rdx), %xmm30 {%k4}
+# CHECK: encoding: [0x62,0x61,0xfc,0x0c,0x5b,0x72,0x7f]
+         vcvtqq2psx	2032(%rdx), %xmm30 {%k4}
+# CHECK: vcvtqq2psx	2032(%rdx), %xmm30 {%k4} {z}
+# CHECK: encoding: [0x62,0x61,0xfc,0x8c,0x5b,0x72,0x7f]
+         vcvtqq2psx	2032(%rdx), %xmm30 {%k4} {z}
+# CHECK: vcvtqq2psx	2048(%rdx), %xmm30
+# CHECK: encoding: [0x62,0x61,0xfc,0x08,0x5b,0xb2,0x00,0x08,0x00,0x00]
+         vcvtqq2psx	2048(%rdx), %xmm30
+# CHECK: vcvtqq2psx	2048(%rdx), %xmm30 {%k4}
+# CHECK: encoding: [0x62,0x61,0xfc,0x0c,0x5b,0xb2,0x00,0x08,0x00,0x00]
+         vcvtqq2psx	2048(%rdx), %xmm30 {%k4}
+# CHECK: vcvtqq2psx	2048(%rdx), %xmm30 {%k4} {z}
+# CHECK: encoding: [0x62,0x61,0xfc,0x8c,0x5b,0xb2,0x00,0x08,0x00,0x00]
+         vcvtqq2psx	2048(%rdx), %xmm30 {%k4} {z}
+# CHECK: vcvtqq2psx	-2048(%rdx), %xmm30
+# CHECK: encoding: [0x62,0x61,0xfc,0x08,0x5b,0x72,0x80]
+         vcvtqq2psx	-2048(%rdx), %xmm30
+# CHECK: vcvtqq2psx	-2048(%rdx), %xmm30 {%k4}
+# CHECK: encoding: [0x62,0x61,0xfc,0x0c,0x5b,0x72,0x80]
+         vcvtqq2psx	-2048(%rdx), %xmm30 {%k4}
+# CHECK: vcvtqq2psx	-2048(%rdx), %xmm30 {%k4} {z}
+# CHECK: encoding: [0x62,0x61,0xfc,0x8c,0x5b,0x72,0x80]
+         vcvtqq2psx	-2048(%rdx), %xmm30 {%k4} {z}
+# CHECK: vcvtqq2psx	-2064(%rdx), %xmm30
+# CHECK: encoding: [0x62,0x61,0xfc,0x08,0x5b,0xb2,0xf0,0xf7,0xff,0xff]
+         vcvtqq2psx	-2064(%rdx), %xmm30
+# CHECK: vcvtqq2psx	-2064(%rdx), %xmm30 {%k4}
+# CHECK: encoding: [0x62,0x61,0xfc,0x0c,0x5b,0xb2,0xf0,0xf7,0xff,0xff]
+         vcvtqq2psx	-2064(%rdx), %xmm30 {%k4}
+# CHECK: vcvtqq2psx	-2064(%rdx), %xmm30 {%k4} {z}
+# CHECK: encoding: [0x62,0x61,0xfc,0x8c,0x5b,0xb2,0xf0,0xf7,0xff,0xff]
+         vcvtqq2psx	-2064(%rdx), %xmm30 {%k4} {z}
+# CHECK: vcvtqq2ps	1016(%rdx){1to2}, %xmm30
+# CHECK: encoding: [0x62,0x61,0xfc,0x18,0x5b,0x72,0x7f]
+         vcvtqq2ps	1016(%rdx){1to2}, %xmm30
+# CHECK: vcvtqq2ps	1016(%rdx){1to2}, %xmm30
+# CHECK: encoding: [0x62,0x61,0xfc,0x18,0x5b,0x72,0x7f]
+         vcvtqq2ps	1016(%rdx){1to2}, %xmm30
+# CHECK: vcvtqq2ps	1016(%rdx){1to2}, %xmm30 {%k4}
+# CHECK: encoding: [0x62,0x61,0xfc,0x1c,0x5b,0x72,0x7f]
+         vcvtqq2ps	1016(%rdx){1to2}, %xmm30 {%k4}
+# CHECK: vcvtqq2ps	1016(%rdx){1to2}, %xmm30 {%k4}
+# CHECK: encoding: [0x62,0x61,0xfc,0x1c,0x5b,0x72,0x7f]
+         vcvtqq2ps	1016(%rdx){1to2}, %xmm30 {%k4}
+# CHECK: vcvtqq2ps	1016(%rdx){1to2}, %xmm30 {%k4} {z}
+# CHECK: encoding: [0x62,0x61,0xfc,0x9c,0x5b,0x72,0x7f]
+         vcvtqq2ps	1016(%rdx){1to2}, %xmm30 {%k4} {z}
+# CHECK: vcvtqq2ps	1016(%rdx){1to2}, %xmm30 {%k4} {z}
+# CHECK: encoding: [0x62,0x61,0xfc,0x9c,0x5b,0x72,0x7f]
+         vcvtqq2ps	1016(%rdx){1to2}, %xmm30 {%k4} {z}
+# CHECK: vcvtqq2ps	1024(%rdx){1to2}, %xmm30
+# CHECK: encoding: [0x62,0x61,0xfc,0x18,0x5b,0xb2,0x00,0x04,0x00,0x00]
+         vcvtqq2ps	1024(%rdx){1to2}, %xmm30
+# CHECK: vcvtqq2ps	1024(%rdx){1to2}, %xmm30
+# CHECK: encoding: [0x62,0x61,0xfc,0x18,0x5b,0xb2,0x00,0x04,0x00,0x00]
+         vcvtqq2ps	1024(%rdx){1to2}, %xmm30
+# CHECK: vcvtqq2ps	1024(%rdx){1to2}, %xmm30 {%k4}
+# CHECK: encoding: [0x62,0x61,0xfc,0x1c,0x5b,0xb2,0x00,0x04,0x00,0x00]
+         vcvtqq2ps	1024(%rdx){1to2}, %xmm30 {%k4}
+# CHECK: vcvtqq2ps	1024(%rdx){1to2}, %xmm30 {%k4}
+# CHECK: encoding: [0x62,0x61,0xfc,0x1c,0x5b,0xb2,0x00,0x04,0x00,0x00]
+         vcvtqq2ps	1024(%rdx){1to2}, %xmm30 {%k4}
+# CHECK: vcvtqq2ps	1024(%rdx){1to2}, %xmm30 {%k4} {z}
+# CHECK: encoding: [0x62,0x61,0xfc,0x9c,0x5b,0xb2,0x00,0x04,0x00,0x00]
+         vcvtqq2ps	1024(%rdx){1to2}, %xmm30 {%k4} {z}
+# CHECK: vcvtqq2ps	1024(%rdx){1to2}, %xmm30 {%k4} {z}
+# CHECK: encoding: [0x62,0x61,0xfc,0x9c,0x5b,0xb2,0x00,0x04,0x00,0x00]
+         vcvtqq2ps	1024(%rdx){1to2}, %xmm30 {%k4} {z}
+# CHECK: vcvtqq2ps	-1024(%rdx){1to2}, %xmm30
+# CHECK: encoding: [0x62,0x61,0xfc,0x18,0x5b,0x72,0x80]
+         vcvtqq2ps	-1024(%rdx){1to2}, %xmm30
+# CHECK: vcvtqq2ps	-1024(%rdx){1to2}, %xmm30
+# CHECK: encoding: [0x62,0x61,0xfc,0x18,0x5b,0x72,0x80]
+         vcvtqq2ps	-1024(%rdx){1to2}, %xmm30
+# CHECK: vcvtqq2ps	-1024(%rdx){1to2}, %xmm30 {%k4}
+# CHECK: encoding: [0x62,0x61,0xfc,0x1c,0x5b,0x72,0x80]
+         vcvtqq2ps	-1024(%rdx){1to2}, %xmm30 {%k4}
+# CHECK: vcvtqq2ps	-1024(%rdx){1to2}, %xmm30 {%k4}
+# CHECK: encoding: [0x62,0x61,0xfc,0x1c,0x5b,0x72,0x80]
+         vcvtqq2ps	-1024(%rdx){1to2}, %xmm30 {%k4}
+# CHECK: vcvtqq2ps	-1024(%rdx){1to2}, %xmm30 {%k4} {z}
+# CHECK: encoding: [0x62,0x61,0xfc,0x9c,0x5b,0x72,0x80]
+         vcvtqq2ps	-1024(%rdx){1to2}, %xmm30 {%k4} {z}
+# CHECK: vcvtqq2ps	-1024(%rdx){1to2}, %xmm30 {%k4} {z}
+# CHECK: encoding: [0x62,0x61,0xfc,0x9c,0x5b,0x72,0x80]
+         vcvtqq2ps	-1024(%rdx){1to2}, %xmm30 {%k4} {z}
+# CHECK: vcvtqq2ps	-1032(%rdx){1to2}, %xmm30
+# CHECK: encoding: [0x62,0x61,0xfc,0x18,0x5b,0xb2,0xf8,0xfb,0xff,0xff]
+         vcvtqq2ps	-1032(%rdx){1to2}, %xmm30
+# CHECK: vcvtqq2ps	-1032(%rdx){1to2}, %xmm30
+# CHECK: encoding: [0x62,0x61,0xfc,0x18,0x5b,0xb2,0xf8,0xfb,0xff,0xff]
+         vcvtqq2ps	-1032(%rdx){1to2}, %xmm30
+# CHECK: vcvtqq2ps	-1032(%rdx){1to2}, %xmm30 {%k4}
+# CHECK: encoding: [0x62,0x61,0xfc,0x1c,0x5b,0xb2,0xf8,0xfb,0xff,0xff]
+         vcvtqq2ps	-1032(%rdx){1to2}, %xmm30 {%k4}
+# CHECK: vcvtqq2ps	-1032(%rdx){1to2}, %xmm30 {%k4}
+# CHECK: encoding: [0x62,0x61,0xfc,0x1c,0x5b,0xb2,0xf8,0xfb,0xff,0xff]
+         vcvtqq2ps	-1032(%rdx){1to2}, %xmm30 {%k4}
+# CHECK: vcvtqq2ps	-1032(%rdx){1to2}, %xmm30 {%k4} {z}
+# CHECK: encoding: [0x62,0x61,0xfc,0x9c,0x5b,0xb2,0xf8,0xfb,0xff,0xff]
+         vcvtqq2ps	-1032(%rdx){1to2}, %xmm30 {%k4} {z}
+# CHECK: vcvtqq2ps	-1032(%rdx){1to2}, %xmm30 {%k4} {z}
+# CHECK: encoding: [0x62,0x61,0xfc,0x9c,0x5b,0xb2,0xf8,0xfb,0xff,0xff]
+         vcvtqq2ps	-1032(%rdx){1to2}, %xmm30 {%k4} {z}
+# CHECK: vcvtqq2ps	%ymm28, %xmm20
+# CHECK: encoding: [0x62,0x81,0xfc,0x28,0x5b,0xe4]
+         vcvtqq2ps	%ymm28, %xmm20
+# CHECK: vcvtqq2ps	%ymm28, %xmm20
+# CHECK: encoding: [0x62,0x81,0xfc,0x28,0x5b,0xe4]
+         vcvtqq2ps	%ymm28, %xmm20
+# CHECK: vcvtqq2ps	%ymm28, %xmm20 {%k3}
+# CHECK: encoding: [0x62,0x81,0xfc,0x2b,0x5b,0xe4]
+         vcvtqq2ps	%ymm28, %xmm20 {%k3}
+# CHECK: vcvtqq2ps	%ymm28, %xmm20 {%k3}
+# CHECK: encoding: [0x62,0x81,0xfc,0x2b,0x5b,0xe4]
+         vcvtqq2ps	%ymm28, %xmm20 {%k3}
+# CHECK: vcvtqq2ps	%ymm28, %xmm20 {%k3} {z}
+# CHECK: encoding: [0x62,0x81,0xfc,0xab,0x5b,0xe4]
+         vcvtqq2ps	%ymm28, %xmm20 {%k3} {z}
+# CHECK: vcvtqq2ps	%ymm28, %xmm20 {%k3} {z}
+# CHECK: encoding: [0x62,0x81,0xfc,0xab,0x5b,0xe4]
+         vcvtqq2ps	%ymm28, %xmm20 {%k3} {z}
+# CHECK: vcvtqq2psy	(%rcx), %xmm20
+# CHECK: encoding: [0x62,0xe1,0xfc,0x28,0x5b,0x21]
+         vcvtqq2psy	(%rcx), %xmm20
+# CHECK: vcvtqq2psy	(%rcx), %xmm20 {%k3}
+# CHECK: encoding: [0x62,0xe1,0xfc,0x2b,0x5b,0x21]
+         vcvtqq2psy	(%rcx), %xmm20 {%k3}
+# CHECK: vcvtqq2psy	(%rcx), %xmm20 {%k3} {z}
+# CHECK: encoding: [0x62,0xe1,0xfc,0xab,0x5b,0x21]
+         vcvtqq2psy	(%rcx), %xmm20 {%k3} {z}
+# CHECK: vcvtqq2psy	4660(%rax,%r14,8), %xmm20
+# CHECK: encoding: [0x62,0xa1,0xfc,0x28,0x5b,0xa4,0xf0,0x34,0x12,0x00,0x00]
+         vcvtqq2psy	4660(%rax,%r14,8), %xmm20
+# CHECK: vcvtqq2psy	4660(%rax,%r14,8), %xmm20 {%k3}
+# CHECK: encoding: [0x62,0xa1,0xfc,0x2b,0x5b,0xa4,0xf0,0x34,0x12,0x00,0x00]
+         vcvtqq2psy	4660(%rax,%r14,8), %xmm20 {%k3}
+# CHECK: vcvtqq2psy	4660(%rax,%r14,8), %xmm20 {%k3} {z}
+# CHECK: encoding: [0x62,0xa1,0xfc,0xab,0x5b,0xa4,0xf0,0x34,0x12,0x00,0x00]
+         vcvtqq2psy	4660(%rax,%r14,8), %xmm20 {%k3} {z}
+# CHECK: vcvtqq2ps	(%rcx){1to4}, %xmm20
+# CHECK: encoding: [0x62,0xe1,0xfc,0x38,0x5b,0x21]
+         vcvtqq2ps	(%rcx){1to4}, %xmm20
+# CHECK: vcvtqq2ps	(%rcx){1to4}, %xmm20
+# CHECK: encoding: [0x62,0xe1,0xfc,0x38,0x5b,0x21]
+         vcvtqq2ps	(%rcx){1to4}, %xmm20
+# CHECK: vcvtqq2ps	(%rcx){1to4}, %xmm20 {%k3}
+# CHECK: encoding: [0x62,0xe1,0xfc,0x3b,0x5b,0x21]
+         vcvtqq2ps	(%rcx){1to4}, %xmm20 {%k3}
+# CHECK: vcvtqq2ps	(%rcx){1to4}, %xmm20 {%k3}
+# CHECK: encoding: [0x62,0xe1,0xfc,0x3b,0x5b,0x21]
+         vcvtqq2ps	(%rcx){1to4}, %xmm20 {%k3}
+# CHECK: vcvtqq2ps	(%rcx){1to4}, %xmm20 {%k3} {z}
+# CHECK: encoding: [0x62,0xe1,0xfc,0xbb,0x5b,0x21]
+         vcvtqq2ps	(%rcx){1to4}, %xmm20 {%k3} {z}
+# CHECK: vcvtqq2ps	(%rcx){1to4}, %xmm20 {%k3} {z}
+# CHECK: encoding: [0x62,0xe1,0xfc,0xbb,0x5b,0x21]
+         vcvtqq2ps	(%rcx){1to4}, %xmm20 {%k3} {z}
+# CHECK: vcvtqq2psy	4064(%rdx), %xmm20
+# CHECK: encoding: [0x62,0xe1,0xfc,0x28,0x5b,0x62,0x7f]
+         vcvtqq2psy	4064(%rdx), %xmm20
+# CHECK: vcvtqq2psy	4064(%rdx), %xmm20 {%k3}
+# CHECK: encoding: [0x62,0xe1,0xfc,0x2b,0x5b,0x62,0x7f]
+         vcvtqq2psy	4064(%rdx), %xmm20 {%k3}
+# CHECK: vcvtqq2psy	4064(%rdx), %xmm20 {%k3} {z}
+# CHECK: encoding: [0x62,0xe1,0xfc,0xab,0x5b,0x62,0x7f]
+         vcvtqq2psy	4064(%rdx), %xmm20 {%k3} {z}
+# CHECK: vcvtqq2psy	4096(%rdx), %xmm20
+# CHECK: encoding: [0x62,0xe1,0xfc,0x28,0x5b,0xa2,0x00,0x10,0x00,0x00]
+         vcvtqq2psy	4096(%rdx), %xmm20
+# CHECK: vcvtqq2psy	4096(%rdx), %xmm20 {%k3}
+# CHECK: encoding: [0x62,0xe1,0xfc,0x2b,0x5b,0xa2,0x00,0x10,0x00,0x00]
+         vcvtqq2psy	4096(%rdx), %xmm20 {%k3}
+# CHECK: vcvtqq2psy	4096(%rdx), %xmm20 {%k3} {z}
+# CHECK: encoding: [0x62,0xe1,0xfc,0xab,0x5b,0xa2,0x00,0x10,0x00,0x00]
+         vcvtqq2psy	4096(%rdx), %xmm20 {%k3} {z}
+# CHECK: vcvtqq2psy	-4096(%rdx), %xmm20
+# CHECK: encoding: [0x62,0xe1,0xfc,0x28,0x5b,0x62,0x80]
+         vcvtqq2psy	-4096(%rdx), %xmm20
+# CHECK: vcvtqq2psy	-4096(%rdx), %xmm20 {%k3}
+# CHECK: encoding: [0x62,0xe1,0xfc,0x2b,0x5b,0x62,0x80]
+         vcvtqq2psy	-4096(%rdx), %xmm20 {%k3}
+# CHECK: vcvtqq2psy	-4096(%rdx), %xmm20 {%k3} {z}
+# CHECK: encoding: [0x62,0xe1,0xfc,0xab,0x5b,0x62,0x80]
+         vcvtqq2psy	-4096(%rdx), %xmm20 {%k3} {z}
+# CHECK: vcvtqq2psy	-4128(%rdx), %xmm20
+# CHECK: encoding: [0x62,0xe1,0xfc,0x28,0x5b,0xa2,0xe0,0xef,0xff,0xff]
+         vcvtqq2psy	-4128(%rdx), %xmm20
+# CHECK: vcvtqq2psy	-4128(%rdx), %xmm20 {%k3}
+# CHECK: encoding: [0x62,0xe1,0xfc,0x2b,0x5b,0xa2,0xe0,0xef,0xff,0xff]
+         vcvtqq2psy	-4128(%rdx), %xmm20 {%k3}
+# CHECK: vcvtqq2psy	-4128(%rdx), %xmm20 {%k3} {z}
+# CHECK: encoding: [0x62,0xe1,0xfc,0xab,0x5b,0xa2,0xe0,0xef,0xff,0xff]
+         vcvtqq2psy	-4128(%rdx), %xmm20 {%k3} {z}
+# CHECK: vcvtqq2ps	1016(%rdx){1to4}, %xmm20
+# CHECK: encoding: [0x62,0xe1,0xfc,0x38,0x5b,0x62,0x7f]
+         vcvtqq2ps	1016(%rdx){1to4}, %xmm20
+# CHECK: vcvtqq2ps	1016(%rdx){1to4}, %xmm20
+# CHECK: encoding: [0x62,0xe1,0xfc,0x38,0x5b,0x62,0x7f]
+         vcvtqq2ps	1016(%rdx){1to4}, %xmm20
+# CHECK: vcvtqq2ps	1016(%rdx){1to4}, %xmm20 {%k3}
+# CHECK: encoding: [0x62,0xe1,0xfc,0x3b,0x5b,0x62,0x7f]
+         vcvtqq2ps	1016(%rdx){1to4}, %xmm20 {%k3}
+# CHECK: vcvtqq2ps	1016(%rdx){1to4}, %xmm20 {%k3}
+# CHECK: encoding: [0x62,0xe1,0xfc,0x3b,0x5b,0x62,0x7f]
+         vcvtqq2ps	1016(%rdx){1to4}, %xmm20 {%k3}
+# CHECK: vcvtqq2ps	1016(%rdx){1to4}, %xmm20 {%k3} {z}
+# CHECK: encoding: [0x62,0xe1,0xfc,0xbb,0x5b,0x62,0x7f]
+         vcvtqq2ps	1016(%rdx){1to4}, %xmm20 {%k3} {z}
+# CHECK: vcvtqq2ps	1016(%rdx){1to4}, %xmm20 {%k3} {z}
+# CHECK: encoding: [0x62,0xe1,0xfc,0xbb,0x5b,0x62,0x7f]
+         vcvtqq2ps	1016(%rdx){1to4}, %xmm20 {%k3} {z}
+# CHECK: vcvtqq2ps	1024(%rdx){1to4}, %xmm20
+# CHECK: encoding: [0x62,0xe1,0xfc,0x38,0x5b,0xa2,0x00,0x04,0x00,0x00]
+         vcvtqq2ps	1024(%rdx){1to4}, %xmm20
+# CHECK: vcvtqq2ps	1024(%rdx){1to4}, %xmm20
+# CHECK: encoding: [0x62,0xe1,0xfc,0x38,0x5b,0xa2,0x00,0x04,0x00,0x00]
+         vcvtqq2ps	1024(%rdx){1to4}, %xmm20
+# CHECK: vcvtqq2ps	1024(%rdx){1to4}, %xmm20 {%k3}
+# CHECK: encoding: [0x62,0xe1,0xfc,0x3b,0x5b,0xa2,0x00,0x04,0x00,0x00]
+         vcvtqq2ps	1024(%rdx){1to4}, %xmm20 {%k3}
+# CHECK: vcvtqq2ps	1024(%rdx){1to4}, %xmm20 {%k3}
+# CHECK: encoding: [0x62,0xe1,0xfc,0x3b,0x5b,0xa2,0x00,0x04,0x00,0x00]
+         vcvtqq2ps	1024(%rdx){1to4}, %xmm20 {%k3}
+# CHECK: vcvtqq2ps	1024(%rdx){1to4}, %xmm20 {%k3} {z}
+# CHECK: encoding: [0x62,0xe1,0xfc,0xbb,0x5b,0xa2,0x00,0x04,0x00,0x00]
+         vcvtqq2ps	1024(%rdx){1to4}, %xmm20 {%k3} {z}
+# CHECK: vcvtqq2ps	1024(%rdx){1to4}, %xmm20 {%k3} {z}
+# CHECK: encoding: [0x62,0xe1,0xfc,0xbb,0x5b,0xa2,0x00,0x04,0x00,0x00]
+         vcvtqq2ps	1024(%rdx){1to4}, %xmm20 {%k3} {z}
+# CHECK: vcvtqq2ps	-1024(%rdx){1to4}, %xmm20
+# CHECK: encoding: [0x62,0xe1,0xfc,0x38,0x5b,0x62,0x80]
+         vcvtqq2ps	-1024(%rdx){1to4}, %xmm20
+# CHECK: vcvtqq2ps	-1024(%rdx){1to4}, %xmm20
+# CHECK: encoding: [0x62,0xe1,0xfc,0x38,0x5b,0x62,0x80]
+         vcvtqq2ps	-1024(%rdx){1to4}, %xmm20
+# CHECK: vcvtqq2ps	-1024(%rdx){1to4}, %xmm20 {%k3}
+# CHECK: encoding: [0x62,0xe1,0xfc,0x3b,0x5b,0x62,0x80]
+         vcvtqq2ps	-1024(%rdx){1to4}, %xmm20 {%k3}
+# CHECK: vcvtqq2ps	-1024(%rdx){1to4}, %xmm20 {%k3}
+# CHECK: encoding: [0x62,0xe1,0xfc,0x3b,0x5b,0x62,0x80]
+         vcvtqq2ps	-1024(%rdx){1to4}, %xmm20 {%k3}
+# CHECK: vcvtqq2ps	-1024(%rdx){1to4}, %xmm20 {%k3} {z}
+# CHECK: encoding: [0x62,0xe1,0xfc,0xbb,0x5b,0x62,0x80]
+         vcvtqq2ps	-1024(%rdx){1to4}, %xmm20 {%k3} {z}
+# CHECK: vcvtqq2ps	-1024(%rdx){1to4}, %xmm20 {%k3} {z}
+# CHECK: encoding: [0x62,0xe1,0xfc,0xbb,0x5b,0x62,0x80]
+         vcvtqq2ps	-1024(%rdx){1to4}, %xmm20 {%k3} {z}
+# CHECK: vcvtqq2ps	-1032(%rdx){1to4}, %xmm20
+# CHECK: encoding: [0x62,0xe1,0xfc,0x38,0x5b,0xa2,0xf8,0xfb,0xff,0xff]
+         vcvtqq2ps	-1032(%rdx){1to4}, %xmm20
+# CHECK: vcvtqq2ps	-1032(%rdx){1to4}, %xmm20
+# CHECK: encoding: [0x62,0xe1,0xfc,0x38,0x5b,0xa2,0xf8,0xfb,0xff,0xff]
+         vcvtqq2ps	-1032(%rdx){1to4}, %xmm20
+# CHECK: vcvtqq2ps	-1032(%rdx){1to4}, %xmm20 {%k3}
+# CHECK: encoding: [0x62,0xe1,0xfc,0x3b,0x5b,0xa2,0xf8,0xfb,0xff,0xff]
+         vcvtqq2ps	-1032(%rdx){1to4}, %xmm20 {%k3}
+# CHECK: vcvtqq2ps	-1032(%rdx){1to4}, %xmm20 {%k3}
+# CHECK: encoding: [0x62,0xe1,0xfc,0x3b,0x5b,0xa2,0xf8,0xfb,0xff,0xff]
+         vcvtqq2ps	-1032(%rdx){1to4}, %xmm20 {%k3}
+# CHECK: vcvtqq2ps	-1032(%rdx){1to4}, %xmm20 {%k3} {z}
+# CHECK: encoding: [0x62,0xe1,0xfc,0xbb,0x5b,0xa2,0xf8,0xfb,0xff,0xff]
+         vcvtqq2ps	-1032(%rdx){1to4}, %xmm20 {%k3} {z}
+# CHECK: vcvtqq2ps	-1032(%rdx){1to4}, %xmm20 {%k3} {z}
+# CHECK: encoding: [0x62,0xe1,0xfc,0xbb,0x5b,0xa2,0xf8,0xfb,0xff,0xff]
+         vcvtqq2ps	-1032(%rdx){1to4}, %xmm20 {%k3} {z}
+# CHECK: vcvtuqq2pd	%xmm20, %xmm19
+# CHECK: encoding: [0x62,0xa1,0xfe,0x08,0x7a,0xdc]
+         vcvtuqq2pd	%xmm20, %xmm19
+# CHECK: vcvtuqq2pd	%xmm20, %xmm19 {%k3}
+# CHECK: encoding: [0x62,0xa1,0xfe,0x0b,0x7a,0xdc]
+         vcvtuqq2pd	%xmm20, %xmm19 {%k3}
+# CHECK: vcvtuqq2pd	%xmm20, %xmm19 {%k3} {z}
+# CHECK: encoding: [0x62,0xa1,0xfe,0x8b,0x7a,0xdc]
+         vcvtuqq2pd	%xmm20, %xmm19 {%k3} {z}
+# CHECK: vcvtuqq2pd	(%rcx), %xmm19
+# CHECK: encoding: [0x62,0xe1,0xfe,0x08,0x7a,0x19]
+         vcvtuqq2pd	(%rcx), %xmm19
+# CHECK: vcvtuqq2pd	291(%rax,%r14,8), %xmm19
+# CHECK: encoding: [0x62,0xa1,0xfe,0x08,0x7a,0x9c,0xf0,0x23,0x01,0x00,0x00]
+         vcvtuqq2pd	291(%rax,%r14,8), %xmm19
+# CHECK: vcvtuqq2pd	(%rcx){1to2}, %xmm19
+# CHECK: encoding: [0x62,0xe1,0xfe,0x18,0x7a,0x19]
+         vcvtuqq2pd	(%rcx){1to2}, %xmm19
+# CHECK: vcvtuqq2pd	2032(%rdx), %xmm19
+# CHECK: encoding: [0x62,0xe1,0xfe,0x08,0x7a,0x5a,0x7f]
+         vcvtuqq2pd	2032(%rdx), %xmm19
+# CHECK: vcvtuqq2pd	2048(%rdx), %xmm19
+# CHECK: encoding: [0x62,0xe1,0xfe,0x08,0x7a,0x9a,0x00,0x08,0x00,0x00]
+         vcvtuqq2pd	2048(%rdx), %xmm19
+# CHECK: vcvtuqq2pd	-2048(%rdx), %xmm19
+# CHECK: encoding: [0x62,0xe1,0xfe,0x08,0x7a,0x5a,0x80]
+         vcvtuqq2pd	-2048(%rdx), %xmm19
+# CHECK: vcvtuqq2pd	-2064(%rdx), %xmm19
+# CHECK: encoding: [0x62,0xe1,0xfe,0x08,0x7a,0x9a,0xf0,0xf7,0xff,0xff]
+         vcvtuqq2pd	-2064(%rdx), %xmm19
+# CHECK: vcvtuqq2pd	1016(%rdx){1to2}, %xmm19
+# CHECK: encoding: [0x62,0xe1,0xfe,0x18,0x7a,0x5a,0x7f]
+         vcvtuqq2pd	1016(%rdx){1to2}, %xmm19
+# CHECK: vcvtuqq2pd	1024(%rdx){1to2}, %xmm19
+# CHECK: encoding: [0x62,0xe1,0xfe,0x18,0x7a,0x9a,0x00,0x04,0x00,0x00]
+         vcvtuqq2pd	1024(%rdx){1to2}, %xmm19
+# CHECK: vcvtuqq2pd	-1024(%rdx){1to2}, %xmm19
+# CHECK: encoding: [0x62,0xe1,0xfe,0x18,0x7a,0x5a,0x80]
+         vcvtuqq2pd	-1024(%rdx){1to2}, %xmm19
+# CHECK: vcvtuqq2pd	-1032(%rdx){1to2}, %xmm19
+# CHECK: encoding: [0x62,0xe1,0xfe,0x18,0x7a,0x9a,0xf8,0xfb,0xff,0xff]
+         vcvtuqq2pd	-1032(%rdx){1to2}, %xmm19
+# CHECK: vcvtuqq2pd	%ymm26, %ymm28
+# CHECK: encoding: [0x62,0x01,0xfe,0x28,0x7a,0xe2]
+         vcvtuqq2pd	%ymm26, %ymm28
+# CHECK: vcvtuqq2pd	%ymm26, %ymm28 {%k4}
+# CHECK: encoding: [0x62,0x01,0xfe,0x2c,0x7a,0xe2]
+         vcvtuqq2pd	%ymm26, %ymm28 {%k4}
+# CHECK: vcvtuqq2pd	%ymm26, %ymm28 {%k4} {z}
+# CHECK: encoding: [0x62,0x01,0xfe,0xac,0x7a,0xe2]
+         vcvtuqq2pd	%ymm26, %ymm28 {%k4} {z}
+# CHECK: vcvtuqq2pd	(%rcx), %ymm28
+# CHECK: encoding: [0x62,0x61,0xfe,0x28,0x7a,0x21]
+         vcvtuqq2pd	(%rcx), %ymm28
+# CHECK: vcvtuqq2pd	291(%rax,%r14,8), %ymm28
+# CHECK: encoding: [0x62,0x21,0xfe,0x28,0x7a,0xa4,0xf0,0x23,0x01,0x00,0x00]
+         vcvtuqq2pd	291(%rax,%r14,8), %ymm28
+# CHECK: vcvtuqq2pd	(%rcx){1to4}, %ymm28
+# CHECK: encoding: [0x62,0x61,0xfe,0x38,0x7a,0x21]
+         vcvtuqq2pd	(%rcx){1to4}, %ymm28
+# CHECK: vcvtuqq2pd	4064(%rdx), %ymm28
+# CHECK: encoding: [0x62,0x61,0xfe,0x28,0x7a,0x62,0x7f]
+         vcvtuqq2pd	4064(%rdx), %ymm28
+# CHECK: vcvtuqq2pd	4096(%rdx), %ymm28
+# CHECK: encoding: [0x62,0x61,0xfe,0x28,0x7a,0xa2,0x00,0x10,0x00,0x00]
+         vcvtuqq2pd	4096(%rdx), %ymm28
+# CHECK: vcvtuqq2pd	-4096(%rdx), %ymm28
+# CHECK: encoding: [0x62,0x61,0xfe,0x28,0x7a,0x62,0x80]
+         vcvtuqq2pd	-4096(%rdx), %ymm28
+# CHECK: vcvtuqq2pd	-4128(%rdx), %ymm28
+# CHECK: encoding: [0x62,0x61,0xfe,0x28,0x7a,0xa2,0xe0,0xef,0xff,0xff]
+         vcvtuqq2pd	-4128(%rdx), %ymm28
+# CHECK: vcvtuqq2pd	1016(%rdx){1to4}, %ymm28
+# CHECK: encoding: [0x62,0x61,0xfe,0x38,0x7a,0x62,0x7f]
+         vcvtuqq2pd	1016(%rdx){1to4}, %ymm28
+# CHECK: vcvtuqq2pd	1024(%rdx){1to4}, %ymm28
+# CHECK: encoding: [0x62,0x61,0xfe,0x38,0x7a,0xa2,0x00,0x04,0x00,0x00]
+         vcvtuqq2pd	1024(%rdx){1to4}, %ymm28
+# CHECK: vcvtuqq2pd	-1024(%rdx){1to4}, %ymm28
+# CHECK: encoding: [0x62,0x61,0xfe,0x38,0x7a,0x62,0x80]
+         vcvtuqq2pd	-1024(%rdx){1to4}, %ymm28
+# CHECK: vcvtuqq2pd	-1032(%rdx){1to4}, %ymm28
+# CHECK: encoding: [0x62,0x61,0xfe,0x38,0x7a,0xa2,0xf8,0xfb,0xff,0xff]
+         vcvtuqq2pd	-1032(%rdx){1to4}, %ymm28
+# CHECK: vcvtuqq2ps	%xmm27, %xmm21
+# CHECK: encoding: [0x62,0x81,0xff,0x08,0x7a,0xeb]
+         vcvtuqq2ps	%xmm27, %xmm21
+# CHECK: vcvtuqq2ps	%xmm27, %xmm21
+# CHECK: encoding: [0x62,0x81,0xff,0x08,0x7a,0xeb]
+         vcvtuqq2ps	%xmm27, %xmm21
+# CHECK: vcvtuqq2ps	%xmm27, %xmm21 {%k7}
+# CHECK: encoding: [0x62,0x81,0xff,0x0f,0x7a,0xeb]
+         vcvtuqq2ps	%xmm27, %xmm21 {%k7}
+# CHECK: vcvtuqq2ps	%xmm27, %xmm21 {%k7}
+# CHECK: encoding: [0x62,0x81,0xff,0x0f,0x7a,0xeb]
+         vcvtuqq2ps	%xmm27, %xmm21 {%k7}
+# CHECK: vcvtuqq2ps	%xmm27, %xmm21 {%k7} {z}
+# CHECK: encoding: [0x62,0x81,0xff,0x8f,0x7a,0xeb]
+         vcvtuqq2ps	%xmm27, %xmm21 {%k7} {z}
+# CHECK: vcvtuqq2ps	%xmm27, %xmm21 {%k7} {z}
+# CHECK: encoding: [0x62,0x81,0xff,0x8f,0x7a,0xeb]
+         vcvtuqq2ps	%xmm27, %xmm21 {%k7} {z}
+# CHECK: vcvtuqq2psx	(%rcx), %xmm21
+# CHECK: encoding: [0x62,0xe1,0xff,0x08,0x7a,0x29]
+         vcvtuqq2psx	(%rcx), %xmm21
+# CHECK: vcvtuqq2psx	(%rcx), %xmm21 {%k7}
+# CHECK: encoding: [0x62,0xe1,0xff,0x0f,0x7a,0x29]
+         vcvtuqq2psx	(%rcx), %xmm21 {%k7}
+# CHECK: vcvtuqq2psx	(%rcx), %xmm21 {%k7} {z}
+# CHECK: encoding: [0x62,0xe1,0xff,0x8f,0x7a,0x29]
+         vcvtuqq2psx	(%rcx), %xmm21 {%k7} {z}
+# CHECK: vcvtuqq2psx	291(%rax,%r14,8), %xmm21
+# CHECK: encoding: [0x62,0xa1,0xff,0x08,0x7a,0xac,0xf0,0x23,0x01,0x00,0x00]
+         vcvtuqq2psx	291(%rax,%r14,8), %xmm21
+# CHECK: vcvtuqq2psx	291(%rax,%r14,8), %xmm21 {%k7}
+# CHECK: encoding: [0x62,0xa1,0xff,0x0f,0x7a,0xac,0xf0,0x23,0x01,0x00,0x00]
+         vcvtuqq2psx	291(%rax,%r14,8), %xmm21 {%k7}
+# CHECK: vcvtuqq2psx	291(%rax,%r14,8), %xmm21 {%k7} {z}
+# CHECK: encoding: [0x62,0xa1,0xff,0x8f,0x7a,0xac,0xf0,0x23,0x01,0x00,0x00]
+         vcvtuqq2psx	291(%rax,%r14,8), %xmm21 {%k7} {z}
+# CHECK: vcvtuqq2ps	(%rcx){1to2}, %xmm21
+# CHECK: encoding: [0x62,0xe1,0xff,0x18,0x7a,0x29]
+         vcvtuqq2ps	(%rcx){1to2}, %xmm21
+# CHECK: vcvtuqq2ps	(%rcx){1to2}, %xmm21
+# CHECK: encoding: [0x62,0xe1,0xff,0x18,0x7a,0x29]
+         vcvtuqq2ps	(%rcx){1to2}, %xmm21
+# CHECK: vcvtuqq2ps	(%rcx){1to2}, %xmm21 {%k7}
+# CHECK: encoding: [0x62,0xe1,0xff,0x1f,0x7a,0x29]
+         vcvtuqq2ps	(%rcx){1to2}, %xmm21 {%k7}
+# CHECK: vcvtuqq2ps	(%rcx){1to2}, %xmm21 {%k7}
+# CHECK: encoding: [0x62,0xe1,0xff,0x1f,0x7a,0x29]
+         vcvtuqq2ps	(%rcx){1to2}, %xmm21 {%k7}
+# CHECK: vcvtuqq2ps	(%rcx){1to2}, %xmm21 {%k7} {z}
+# CHECK: encoding: [0x62,0xe1,0xff,0x9f,0x7a,0x29]
+         vcvtuqq2ps	(%rcx){1to2}, %xmm21 {%k7} {z}
+# CHECK: vcvtuqq2ps	(%rcx){1to2}, %xmm21 {%k7} {z}
+# CHECK: encoding: [0x62,0xe1,0xff,0x9f,0x7a,0x29]
+         vcvtuqq2ps	(%rcx){1to2}, %xmm21 {%k7} {z}
+# CHECK: vcvtuqq2psx	2032(%rdx), %xmm21
+# CHECK: encoding: [0x62,0xe1,0xff,0x08,0x7a,0x6a,0x7f]
+         vcvtuqq2psx	2032(%rdx), %xmm21
+# CHECK: vcvtuqq2psx	2032(%rdx), %xmm21 {%k7}
+# CHECK: encoding: [0x62,0xe1,0xff,0x0f,0x7a,0x6a,0x7f]
+         vcvtuqq2psx	2032(%rdx), %xmm21 {%k7}
+# CHECK: vcvtuqq2psx	2032(%rdx), %xmm21 {%k7} {z}
+# CHECK: encoding: [0x62,0xe1,0xff,0x8f,0x7a,0x6a,0x7f]
+         vcvtuqq2psx	2032(%rdx), %xmm21 {%k7} {z}
+# CHECK: vcvtuqq2psx	2048(%rdx), %xmm21
+# CHECK: encoding: [0x62,0xe1,0xff,0x08,0x7a,0xaa,0x00,0x08,0x00,0x00]
+         vcvtuqq2psx	2048(%rdx), %xmm21
+# CHECK: vcvtuqq2psx	2048(%rdx), %xmm21 {%k7}
+# CHECK: encoding: [0x62,0xe1,0xff,0x0f,0x7a,0xaa,0x00,0x08,0x00,0x00]
+         vcvtuqq2psx	2048(%rdx), %xmm21 {%k7}
+# CHECK: vcvtuqq2psx	2048(%rdx), %xmm21 {%k7} {z}
+# CHECK: encoding: [0x62,0xe1,0xff,0x8f,0x7a,0xaa,0x00,0x08,0x00,0x00]
+         vcvtuqq2psx	2048(%rdx), %xmm21 {%k7} {z}
+# CHECK: vcvtuqq2psx	-2048(%rdx), %xmm21
+# CHECK: encoding: [0x62,0xe1,0xff,0x08,0x7a,0x6a,0x80]
+         vcvtuqq2psx	-2048(%rdx), %xmm21
+# CHECK: vcvtuqq2psx	-2048(%rdx), %xmm21 {%k7}
+# CHECK: encoding: [0x62,0xe1,0xff,0x0f,0x7a,0x6a,0x80]
+         vcvtuqq2psx	-2048(%rdx), %xmm21 {%k7}
+# CHECK: vcvtuqq2psx	-2048(%rdx), %xmm21 {%k7} {z}
+# CHECK: encoding: [0x62,0xe1,0xff,0x8f,0x7a,0x6a,0x80]
+         vcvtuqq2psx	-2048(%rdx), %xmm21 {%k7} {z}
+# CHECK: vcvtuqq2psx	-2064(%rdx), %xmm21
+# CHECK: encoding: [0x62,0xe1,0xff,0x08,0x7a,0xaa,0xf0,0xf7,0xff,0xff]
+         vcvtuqq2psx	-2064(%rdx), %xmm21
+# CHECK: vcvtuqq2psx	-2064(%rdx), %xmm21 {%k7}
+# CHECK: encoding: [0x62,0xe1,0xff,0x0f,0x7a,0xaa,0xf0,0xf7,0xff,0xff]
+         vcvtuqq2psx	-2064(%rdx), %xmm21 {%k7}
+# CHECK: vcvtuqq2psx	-2064(%rdx), %xmm21 {%k7} {z}
+# CHECK: encoding: [0x62,0xe1,0xff,0x8f,0x7a,0xaa,0xf0,0xf7,0xff,0xff]
+         vcvtuqq2psx	-2064(%rdx), %xmm21 {%k7} {z}
+# CHECK: vcvtuqq2ps	1016(%rdx){1to2}, %xmm21
+# CHECK: encoding: [0x62,0xe1,0xff,0x18,0x7a,0x6a,0x7f]
+         vcvtuqq2ps	1016(%rdx){1to2}, %xmm21
+# CHECK: vcvtuqq2ps	1016(%rdx){1to2}, %xmm21
+# CHECK: encoding: [0x62,0xe1,0xff,0x18,0x7a,0x6a,0x7f]
+         vcvtuqq2ps	1016(%rdx){1to2}, %xmm21
+# CHECK: vcvtuqq2ps	1016(%rdx){1to2}, %xmm21 {%k7}
+# CHECK: encoding: [0x62,0xe1,0xff,0x1f,0x7a,0x6a,0x7f]
+         vcvtuqq2ps	1016(%rdx){1to2}, %xmm21 {%k7}
+# CHECK: vcvtuqq2ps	1016(%rdx){1to2}, %xmm21 {%k7}
+# CHECK: encoding: [0x62,0xe1,0xff,0x1f,0x7a,0x6a,0x7f]
+         vcvtuqq2ps	1016(%rdx){1to2}, %xmm21 {%k7}
+# CHECK: vcvtuqq2ps	1016(%rdx){1to2}, %xmm21 {%k7} {z}
+# CHECK: encoding: [0x62,0xe1,0xff,0x9f,0x7a,0x6a,0x7f]
+         vcvtuqq2ps	1016(%rdx){1to2}, %xmm21 {%k7} {z}
+# CHECK: vcvtuqq2ps	1016(%rdx){1to2}, %xmm21 {%k7} {z}
+# CHECK: encoding: [0x62,0xe1,0xff,0x9f,0x7a,0x6a,0x7f]
+         vcvtuqq2ps	1016(%rdx){1to2}, %xmm21 {%k7} {z}
+# CHECK: vcvtuqq2ps	1024(%rdx){1to2}, %xmm21
+# CHECK: encoding: [0x62,0xe1,0xff,0x18,0x7a,0xaa,0x00,0x04,0x00,0x00]
+         vcvtuqq2ps	1024(%rdx){1to2}, %xmm21
+# CHECK: vcvtuqq2ps	1024(%rdx){1to2}, %xmm21
+# CHECK: encoding: [0x62,0xe1,0xff,0x18,0x7a,0xaa,0x00,0x04,0x00,0x00]
+         vcvtuqq2ps	1024(%rdx){1to2}, %xmm21
+# CHECK: vcvtuqq2ps	1024(%rdx){1to2}, %xmm21 {%k7}
+# CHECK: encoding: [0x62,0xe1,0xff,0x1f,0x7a,0xaa,0x00,0x04,0x00,0x00]
+         vcvtuqq2ps	1024(%rdx){1to2}, %xmm21 {%k7}
+# CHECK: vcvtuqq2ps	1024(%rdx){1to2}, %xmm21 {%k7}
+# CHECK: encoding: [0x62,0xe1,0xff,0x1f,0x7a,0xaa,0x00,0x04,0x00,0x00]
+         vcvtuqq2ps	1024(%rdx){1to2}, %xmm21 {%k7}
+# CHECK: vcvtuqq2ps	1024(%rdx){1to2}, %xmm21 {%k7} {z}
+# CHECK: encoding: [0x62,0xe1,0xff,0x9f,0x7a,0xaa,0x00,0x04,0x00,0x00]
+         vcvtuqq2ps	1024(%rdx){1to2}, %xmm21 {%k7} {z}
+# CHECK: vcvtuqq2ps	1024(%rdx){1to2}, %xmm21 {%k7} {z}
+# CHECK: encoding: [0x62,0xe1,0xff,0x9f,0x7a,0xaa,0x00,0x04,0x00,0x00]
+         vcvtuqq2ps	1024(%rdx){1to2}, %xmm21 {%k7} {z}
+# CHECK: vcvtuqq2ps	-1024(%rdx){1to2}, %xmm21
+# CHECK: encoding: [0x62,0xe1,0xff,0x18,0x7a,0x6a,0x80]
+         vcvtuqq2ps	-1024(%rdx){1to2}, %xmm21
+# CHECK: vcvtuqq2ps	-1024(%rdx){1to2}, %xmm21
+# CHECK: encoding: [0x62,0xe1,0xff,0x18,0x7a,0x6a,0x80]
+         vcvtuqq2ps	-1024(%rdx){1to2}, %xmm21
+# CHECK: vcvtuqq2ps	-1024(%rdx){1to2}, %xmm21 {%k7}
+# CHECK: encoding: [0x62,0xe1,0xff,0x1f,0x7a,0x6a,0x80]
+         vcvtuqq2ps	-1024(%rdx){1to2}, %xmm21 {%k7}
+# CHECK: vcvtuqq2ps	-1024(%rdx){1to2}, %xmm21 {%k7}
+# CHECK: encoding: [0x62,0xe1,0xff,0x1f,0x7a,0x6a,0x80]
+         vcvtuqq2ps	-1024(%rdx){1to2}, %xmm21 {%k7}
+# CHECK: vcvtuqq2ps	-1024(%rdx){1to2}, %xmm21 {%k7} {z}
+# CHECK: encoding: [0x62,0xe1,0xff,0x9f,0x7a,0x6a,0x80]
+         vcvtuqq2ps	-1024(%rdx){1to2}, %xmm21 {%k7} {z}
+# CHECK: vcvtuqq2ps	-1024(%rdx){1to2}, %xmm21 {%k7} {z}
+# CHECK: encoding: [0x62,0xe1,0xff,0x9f,0x7a,0x6a,0x80]
+         vcvtuqq2ps	-1024(%rdx){1to2}, %xmm21 {%k7} {z}
+# CHECK: vcvtuqq2ps	-1032(%rdx){1to2}, %xmm21
+# CHECK: encoding: [0x62,0xe1,0xff,0x18,0x7a,0xaa,0xf8,0xfb,0xff,0xff]
+         vcvtuqq2ps	-1032(%rdx){1to2}, %xmm21
+# CHECK: vcvtuqq2ps	-1032(%rdx){1to2}, %xmm21
+# CHECK: encoding: [0x62,0xe1,0xff,0x18,0x7a,0xaa,0xf8,0xfb,0xff,0xff]
+         vcvtuqq2ps	-1032(%rdx){1to2}, %xmm21
+# CHECK: vcvtuqq2ps	-1032(%rdx){1to2}, %xmm21 {%k7}
+# CHECK: encoding: [0x62,0xe1,0xff,0x1f,0x7a,0xaa,0xf8,0xfb,0xff,0xff]
+         vcvtuqq2ps	-1032(%rdx){1to2}, %xmm21 {%k7}
+# CHECK: vcvtuqq2ps	-1032(%rdx){1to2}, %xmm21 {%k7}
+# CHECK: encoding: [0x62,0xe1,0xff,0x1f,0x7a,0xaa,0xf8,0xfb,0xff,0xff]
+         vcvtuqq2ps	-1032(%rdx){1to2}, %xmm21 {%k7}
+# CHECK: vcvtuqq2ps	-1032(%rdx){1to2}, %xmm21 {%k7} {z}
+# CHECK: encoding: [0x62,0xe1,0xff,0x9f,0x7a,0xaa,0xf8,0xfb,0xff,0xff]
+         vcvtuqq2ps	-1032(%rdx){1to2}, %xmm21 {%k7} {z}
+# CHECK: vcvtuqq2ps	-1032(%rdx){1to2}, %xmm21 {%k7} {z}
+# CHECK: encoding: [0x62,0xe1,0xff,0x9f,0x7a,0xaa,0xf8,0xfb,0xff,0xff]
+         vcvtuqq2ps	-1032(%rdx){1to2}, %xmm21 {%k7} {z}
+# CHECK: vcvtuqq2ps	%ymm24, %xmm28
+# CHECK: encoding: [0x62,0x01,0xff,0x28,0x7a,0xe0]
+         vcvtuqq2ps	%ymm24, %xmm28
+# CHECK: vcvtuqq2ps	%ymm24, %xmm28
+# CHECK: encoding: [0x62,0x01,0xff,0x28,0x7a,0xe0]
+         vcvtuqq2ps	%ymm24, %xmm28
+# CHECK: vcvtuqq2ps	%ymm24, %xmm28 {%k3}
+# CHECK: encoding: [0x62,0x01,0xff,0x2b,0x7a,0xe0]
+         vcvtuqq2ps	%ymm24, %xmm28 {%k3}
+# CHECK: vcvtuqq2ps	%ymm24, %xmm28 {%k3}
+# CHECK: encoding: [0x62,0x01,0xff,0x2b,0x7a,0xe0]
+         vcvtuqq2ps	%ymm24, %xmm28 {%k3}
+# CHECK: vcvtuqq2ps	%ymm24, %xmm28 {%k3} {z}
+# CHECK: encoding: [0x62,0x01,0xff,0xab,0x7a,0xe0]
+         vcvtuqq2ps	%ymm24, %xmm28 {%k3} {z}
+# CHECK: vcvtuqq2ps	%ymm24, %xmm28 {%k3} {z}
+# CHECK: encoding: [0x62,0x01,0xff,0xab,0x7a,0xe0]
+         vcvtuqq2ps	%ymm24, %xmm28 {%k3} {z}
+# CHECK: vcvtuqq2psy	(%rcx), %xmm28
+# CHECK: encoding: [0x62,0x61,0xff,0x28,0x7a,0x21]
+         vcvtuqq2psy	(%rcx), %xmm28
+# CHECK: vcvtuqq2psy	(%rcx), %xmm28 {%k3}
+# CHECK: encoding: [0x62,0x61,0xff,0x2b,0x7a,0x21]
+         vcvtuqq2psy	(%rcx), %xmm28 {%k3}
+# CHECK: vcvtuqq2psy	(%rcx), %xmm28 {%k3} {z}
+# CHECK: encoding: [0x62,0x61,0xff,0xab,0x7a,0x21]
+         vcvtuqq2psy	(%rcx), %xmm28 {%k3} {z}
+# CHECK: vcvtuqq2psy	291(%rax,%r14,8), %xmm28
+# CHECK: encoding: [0x62,0x21,0xff,0x28,0x7a,0xa4,0xf0,0x23,0x01,0x00,0x00]
+         vcvtuqq2psy	291(%rax,%r14,8), %xmm28
+# CHECK: vcvtuqq2psy	291(%rax,%r14,8), %xmm28 {%k3}
+# CHECK: encoding: [0x62,0x21,0xff,0x2b,0x7a,0xa4,0xf0,0x23,0x01,0x00,0x00]
+         vcvtuqq2psy	291(%rax,%r14,8), %xmm28 {%k3}
+# CHECK: vcvtuqq2psy	291(%rax,%r14,8), %xmm28 {%k3} {z}
+# CHECK: encoding: [0x62,0x21,0xff,0xab,0x7a,0xa4,0xf0,0x23,0x01,0x00,0x00]
+         vcvtuqq2psy	291(%rax,%r14,8), %xmm28 {%k3} {z}
+# CHECK: vcvtuqq2ps	(%rcx){1to4}, %xmm28
+# CHECK: encoding: [0x62,0x61,0xff,0x38,0x7a,0x21]
+         vcvtuqq2ps	(%rcx){1to4}, %xmm28
+# CHECK: vcvtuqq2ps	(%rcx){1to4}, %xmm28
+# CHECK: encoding: [0x62,0x61,0xff,0x38,0x7a,0x21]
+         vcvtuqq2ps	(%rcx){1to4}, %xmm28
+# CHECK: vcvtuqq2ps	(%rcx){1to4}, %xmm28 {%k3}
+# CHECK: encoding: [0x62,0x61,0xff,0x3b,0x7a,0x21]
+         vcvtuqq2ps	(%rcx){1to4}, %xmm28 {%k3}
+# CHECK: vcvtuqq2ps	(%rcx){1to4}, %xmm28 {%k3}
+# CHECK: encoding: [0x62,0x61,0xff,0x3b,0x7a,0x21]
+         vcvtuqq2ps	(%rcx){1to4}, %xmm28 {%k3}
+# CHECK: vcvtuqq2ps	(%rcx){1to4}, %xmm28 {%k3} {z}
+# CHECK: encoding: [0x62,0x61,0xff,0xbb,0x7a,0x21]
+         vcvtuqq2ps	(%rcx){1to4}, %xmm28 {%k3} {z}
+# CHECK: vcvtuqq2ps	(%rcx){1to4}, %xmm28 {%k3} {z}
+# CHECK: encoding: [0x62,0x61,0xff,0xbb,0x7a,0x21]
+         vcvtuqq2ps	(%rcx){1to4}, %xmm28 {%k3} {z}
+# CHECK: vcvtuqq2psy	4064(%rdx), %xmm28
+# CHECK: encoding: [0x62,0x61,0xff,0x28,0x7a,0x62,0x7f]
+         vcvtuqq2psy	4064(%rdx), %xmm28
+# CHECK: vcvtuqq2psy	4064(%rdx), %xmm28 {%k3}
+# CHECK: encoding: [0x62,0x61,0xff,0x2b,0x7a,0x62,0x7f]
+         vcvtuqq2psy	4064(%rdx), %xmm28 {%k3}
+# CHECK: vcvtuqq2psy	4064(%rdx), %xmm28 {%k3} {z}
+# CHECK: encoding: [0x62,0x61,0xff,0xab,0x7a,0x62,0x7f]
+         vcvtuqq2psy	4064(%rdx), %xmm28 {%k3} {z}
+# CHECK: vcvtuqq2psy	4096(%rdx), %xmm28
+# CHECK: encoding: [0x62,0x61,0xff,0x28,0x7a,0xa2,0x00,0x10,0x00,0x00]
+         vcvtuqq2psy	4096(%rdx), %xmm28
+# CHECK: vcvtuqq2psy	4096(%rdx), %xmm28 {%k3}
+# CHECK: encoding: [0x62,0x61,0xff,0x2b,0x7a,0xa2,0x00,0x10,0x00,0x00]
+         vcvtuqq2psy	4096(%rdx), %xmm28 {%k3}
+# CHECK: vcvtuqq2psy	4096(%rdx), %xmm28 {%k3} {z}
+# CHECK: encoding: [0x62,0x61,0xff,0xab,0x7a,0xa2,0x00,0x10,0x00,0x00]
+         vcvtuqq2psy	4096(%rdx), %xmm28 {%k3} {z}
+# CHECK: vcvtuqq2psy	-4096(%rdx), %xmm28
+# CHECK: encoding: [0x62,0x61,0xff,0x28,0x7a,0x62,0x80]
+         vcvtuqq2psy	-4096(%rdx), %xmm28
+# CHECK: vcvtuqq2psy	-4096(%rdx), %xmm28 {%k3}
+# CHECK: encoding: [0x62,0x61,0xff,0x2b,0x7a,0x62,0x80]
+         vcvtuqq2psy	-4096(%rdx), %xmm28 {%k3}
+# CHECK: vcvtuqq2psy	-4096(%rdx), %xmm28 {%k3} {z}
+# CHECK: encoding: [0x62,0x61,0xff,0xab,0x7a,0x62,0x80]
+         vcvtuqq2psy	-4096(%rdx), %xmm28 {%k3} {z}
+# CHECK: vcvtuqq2psy	-4128(%rdx), %xmm28
+# CHECK: encoding: [0x62,0x61,0xff,0x28,0x7a,0xa2,0xe0,0xef,0xff,0xff]
+         vcvtuqq2psy	-4128(%rdx), %xmm28
+# CHECK: vcvtuqq2psy	-4128(%rdx), %xmm28 {%k3}
+# CHECK: encoding: [0x62,0x61,0xff,0x2b,0x7a,0xa2,0xe0,0xef,0xff,0xff]
+         vcvtuqq2psy	-4128(%rdx), %xmm28 {%k3}
+# CHECK: vcvtuqq2psy	-4128(%rdx), %xmm28 {%k3} {z}
+# CHECK: encoding: [0x62,0x61,0xff,0xab,0x7a,0xa2,0xe0,0xef,0xff,0xff]
+         vcvtuqq2psy	-4128(%rdx), %xmm28 {%k3} {z}
+# CHECK: vcvtuqq2ps	1016(%rdx){1to4}, %xmm28
+# CHECK: encoding: [0x62,0x61,0xff,0x38,0x7a,0x62,0x7f]
+         vcvtuqq2ps	1016(%rdx){1to4}, %xmm28
+# CHECK: vcvtuqq2ps	1016(%rdx){1to4}, %xmm28
+# CHECK: encoding: [0x62,0x61,0xff,0x38,0x7a,0x62,0x7f]
+         vcvtuqq2ps	1016(%rdx){1to4}, %xmm28
+# CHECK: vcvtuqq2ps	1016(%rdx){1to4}, %xmm28 {%k3}
+# CHECK: encoding: [0x62,0x61,0xff,0x3b,0x7a,0x62,0x7f]
+         vcvtuqq2ps	1016(%rdx){1to4}, %xmm28 {%k3}
+# CHECK: vcvtuqq2ps	1016(%rdx){1to4}, %xmm28 {%k3}
+# CHECK: encoding: [0x62,0x61,0xff,0x3b,0x7a,0x62,0x7f]
+         vcvtuqq2ps	1016(%rdx){1to4}, %xmm28 {%k3}
+# CHECK: vcvtuqq2ps	1016(%rdx){1to4}, %xmm28 {%k3} {z}
+# CHECK: encoding: [0x62,0x61,0xff,0xbb,0x7a,0x62,0x7f]
+         vcvtuqq2ps	1016(%rdx){1to4}, %xmm28 {%k3} {z}
+# CHECK: vcvtuqq2ps	1016(%rdx){1to4}, %xmm28 {%k3} {z}
+# CHECK: encoding: [0x62,0x61,0xff,0xbb,0x7a,0x62,0x7f]
+         vcvtuqq2ps	1016(%rdx){1to4}, %xmm28 {%k3} {z}
+# CHECK: vcvtuqq2ps	1024(%rdx){1to4}, %xmm28
+# CHECK: encoding: [0x62,0x61,0xff,0x38,0x7a,0xa2,0x00,0x04,0x00,0x00]
+         vcvtuqq2ps	1024(%rdx){1to4}, %xmm28
+# CHECK: vcvtuqq2ps	1024(%rdx){1to4}, %xmm28
+# CHECK: encoding: [0x62,0x61,0xff,0x38,0x7a,0xa2,0x00,0x04,0x00,0x00]
+         vcvtuqq2ps	1024(%rdx){1to4}, %xmm28
+# CHECK: vcvtuqq2ps	1024(%rdx){1to4}, %xmm28 {%k3}
+# CHECK: encoding: [0x62,0x61,0xff,0x3b,0x7a,0xa2,0x00,0x04,0x00,0x00]
+         vcvtuqq2ps	1024(%rdx){1to4}, %xmm28 {%k3}
+# CHECK: vcvtuqq2ps	1024(%rdx){1to4}, %xmm28 {%k3}
+# CHECK: encoding: [0x62,0x61,0xff,0x3b,0x7a,0xa2,0x00,0x04,0x00,0x00]
+         vcvtuqq2ps	1024(%rdx){1to4}, %xmm28 {%k3}
+# CHECK: vcvtuqq2ps	1024(%rdx){1to4}, %xmm28 {%k3} {z}
+# CHECK: encoding: [0x62,0x61,0xff,0xbb,0x7a,0xa2,0x00,0x04,0x00,0x00]
+         vcvtuqq2ps	1024(%rdx){1to4}, %xmm28 {%k3} {z}
+# CHECK: vcvtuqq2ps	1024(%rdx){1to4}, %xmm28 {%k3} {z}
+# CHECK: encoding: [0x62,0x61,0xff,0xbb,0x7a,0xa2,0x00,0x04,0x00,0x00]
+         vcvtuqq2ps	1024(%rdx){1to4}, %xmm28 {%k3} {z}
+# CHECK: vcvtuqq2ps	-1024(%rdx){1to4}, %xmm28
+# CHECK: encoding: [0x62,0x61,0xff,0x38,0x7a,0x62,0x80]
+         vcvtuqq2ps	-1024(%rdx){1to4}, %xmm28
+# CHECK: vcvtuqq2ps	-1024(%rdx){1to4}, %xmm28
+# CHECK: encoding: [0x62,0x61,0xff,0x38,0x7a,0x62,0x80]
+         vcvtuqq2ps	-1024(%rdx){1to4}, %xmm28
+# CHECK: vcvtuqq2ps	-1024(%rdx){1to4}, %xmm28 {%k3}
+# CHECK: encoding: [0x62,0x61,0xff,0x3b,0x7a,0x62,0x80]
+         vcvtuqq2ps	-1024(%rdx){1to4}, %xmm28 {%k3}
+# CHECK: vcvtuqq2ps	-1024(%rdx){1to4}, %xmm28 {%k3}
+# CHECK: encoding: [0x62,0x61,0xff,0x3b,0x7a,0x62,0x80]
+         vcvtuqq2ps	-1024(%rdx){1to4}, %xmm28 {%k3}
+# CHECK: vcvtuqq2ps	-1024(%rdx){1to4}, %xmm28 {%k3} {z}
+# CHECK: encoding: [0x62,0x61,0xff,0xbb,0x7a,0x62,0x80]
+         vcvtuqq2ps	-1024(%rdx){1to4}, %xmm28 {%k3} {z}
+# CHECK: vcvtuqq2ps	-1024(%rdx){1to4}, %xmm28 {%k3} {z}
+# CHECK: encoding: [0x62,0x61,0xff,0xbb,0x7a,0x62,0x80]
+         vcvtuqq2ps	-1024(%rdx){1to4}, %xmm28 {%k3} {z}
+# CHECK: vcvtuqq2ps	-1032(%rdx){1to4}, %xmm28
+# CHECK: encoding: [0x62,0x61,0xff,0x38,0x7a,0xa2,0xf8,0xfb,0xff,0xff]
+         vcvtuqq2ps	-1032(%rdx){1to4}, %xmm28
+# CHECK: vcvtuqq2ps	-1032(%rdx){1to4}, %xmm28
+# CHECK: encoding: [0x62,0x61,0xff,0x38,0x7a,0xa2,0xf8,0xfb,0xff,0xff]
+         vcvtuqq2ps	-1032(%rdx){1to4}, %xmm28
+# CHECK: vcvtuqq2ps	-1032(%rdx){1to4}, %xmm28 {%k3}
+# CHECK: encoding: [0x62,0x61,0xff,0x3b,0x7a,0xa2,0xf8,0xfb,0xff,0xff]
+         vcvtuqq2ps	-1032(%rdx){1to4}, %xmm28 {%k3}
+# CHECK: vcvtuqq2ps	-1032(%rdx){1to4}, %xmm28 {%k3}
+# CHECK: encoding: [0x62,0x61,0xff,0x3b,0x7a,0xa2,0xf8,0xfb,0xff,0xff]
+         vcvtuqq2ps	-1032(%rdx){1to4}, %xmm28 {%k3}
+# CHECK: vcvtuqq2ps	-1032(%rdx){1to4}, %xmm28 {%k3} {z}
+# CHECK: encoding: [0x62,0x61,0xff,0xbb,0x7a,0xa2,0xf8,0xfb,0xff,0xff]
+         vcvtuqq2ps	-1032(%rdx){1to4}, %xmm28 {%k3} {z}
+# CHECK: vcvtuqq2ps	-1032(%rdx){1to4}, %xmm28 {%k3} {z}
+# CHECK: encoding: [0x62,0x61,0xff,0xbb,0x7a,0xa2,0xf8,0xfb,0xff,0xff]
+         vcvtuqq2ps	-1032(%rdx){1to4}, %xmm28 {%k3} {z}
+# CHECK: vcvtuqq2ps	%xmm22, %xmm21
+# CHECK: encoding: [0x62,0xa1,0xff,0x08,0x7a,0xee]
+         vcvtuqq2ps	%xmm22, %xmm21
+# CHECK: vcvtuqq2ps	%xmm22, %xmm21
+# CHECK: encoding: [0x62,0xa1,0xff,0x08,0x7a,0xee]
+         vcvtuqq2ps	%xmm22, %xmm21
+# CHECK: vcvtuqq2ps	%xmm22, %xmm21 {%k6}
+# CHECK: encoding: [0x62,0xa1,0xff,0x0e,0x7a,0xee]
+         vcvtuqq2ps	%xmm22, %xmm21 {%k6}
+# CHECK: vcvtuqq2ps	%xmm22, %xmm21 {%k6}
+# CHECK: encoding: [0x62,0xa1,0xff,0x0e,0x7a,0xee]
+         vcvtuqq2ps	%xmm22, %xmm21 {%k6}
+# CHECK: vcvtuqq2ps	%xmm22, %xmm21 {%k6} {z}
+# CHECK: encoding: [0x62,0xa1,0xff,0x8e,0x7a,0xee]
+         vcvtuqq2ps	%xmm22, %xmm21 {%k6} {z}
+# CHECK: vcvtuqq2ps	%xmm22, %xmm21 {%k6} {z}
+# CHECK: encoding: [0x62,0xa1,0xff,0x8e,0x7a,0xee]
+         vcvtuqq2ps	%xmm22, %xmm21 {%k6} {z}
+# CHECK: vcvtuqq2psx	(%rcx), %xmm21
+# CHECK: encoding: [0x62,0xe1,0xff,0x08,0x7a,0x29]
+         vcvtuqq2psx	(%rcx), %xmm21
+# CHECK: vcvtuqq2psx	(%rcx), %xmm21 {%k6}
+# CHECK: encoding: [0x62,0xe1,0xff,0x0e,0x7a,0x29]
+         vcvtuqq2psx	(%rcx), %xmm21 {%k6}
+# CHECK: vcvtuqq2psx	(%rcx), %xmm21 {%k6} {z}
+# CHECK: encoding: [0x62,0xe1,0xff,0x8e,0x7a,0x29]
+         vcvtuqq2psx	(%rcx), %xmm21 {%k6} {z}
+# CHECK: vcvtuqq2psx	4660(%rax,%r14,8), %xmm21
+# CHECK: encoding: [0x62,0xa1,0xff,0x08,0x7a,0xac,0xf0,0x34,0x12,0x00,0x00]
+         vcvtuqq2psx	4660(%rax,%r14,8), %xmm21
+# CHECK: vcvtuqq2psx	4660(%rax,%r14,8), %xmm21 {%k6}
+# CHECK: encoding: [0x62,0xa1,0xff,0x0e,0x7a,0xac,0xf0,0x34,0x12,0x00,0x00]
+         vcvtuqq2psx	4660(%rax,%r14,8), %xmm21 {%k6}
+# CHECK: vcvtuqq2psx	4660(%rax,%r14,8), %xmm21 {%k6} {z}
+# CHECK: encoding: [0x62,0xa1,0xff,0x8e,0x7a,0xac,0xf0,0x34,0x12,0x00,0x00]
+         vcvtuqq2psx	4660(%rax,%r14,8), %xmm21 {%k6} {z}
+# CHECK: vcvtuqq2ps	(%rcx){1to2}, %xmm21
+# CHECK: encoding: [0x62,0xe1,0xff,0x18,0x7a,0x29]
+         vcvtuqq2ps	(%rcx){1to2}, %xmm21
+# CHECK: vcvtuqq2ps	(%rcx){1to2}, %xmm21
+# CHECK: encoding: [0x62,0xe1,0xff,0x18,0x7a,0x29]
+         vcvtuqq2ps	(%rcx){1to2}, %xmm21
+# CHECK: vcvtuqq2ps	(%rcx){1to2}, %xmm21 {%k6}
+# CHECK: encoding: [0x62,0xe1,0xff,0x1e,0x7a,0x29]
+         vcvtuqq2ps	(%rcx){1to2}, %xmm21 {%k6}
+# CHECK: vcvtuqq2ps	(%rcx){1to2}, %xmm21 {%k6}
+# CHECK: encoding: [0x62,0xe1,0xff,0x1e,0x7a,0x29]
+         vcvtuqq2ps	(%rcx){1to2}, %xmm21 {%k6}
+# CHECK: vcvtuqq2ps	(%rcx){1to2}, %xmm21 {%k6} {z}
+# CHECK: encoding: [0x62,0xe1,0xff,0x9e,0x7a,0x29]
+         vcvtuqq2ps	(%rcx){1to2}, %xmm21 {%k6} {z}
+# CHECK: vcvtuqq2ps	(%rcx){1to2}, %xmm21 {%k6} {z}
+# CHECK: encoding: [0x62,0xe1,0xff,0x9e,0x7a,0x29]
+         vcvtuqq2ps	(%rcx){1to2}, %xmm21 {%k6} {z}
+# CHECK: vcvtuqq2psx	2032(%rdx), %xmm21
+# CHECK: encoding: [0x62,0xe1,0xff,0x08,0x7a,0x6a,0x7f]
+         vcvtuqq2psx	2032(%rdx), %xmm21
+# CHECK: vcvtuqq2psx	2032(%rdx), %xmm21 {%k6}
+# CHECK: encoding: [0x62,0xe1,0xff,0x0e,0x7a,0x6a,0x7f]
+         vcvtuqq2psx	2032(%rdx), %xmm21 {%k6}
+# CHECK: vcvtuqq2psx	2032(%rdx), %xmm21 {%k6} {z}
+# CHECK: encoding: [0x62,0xe1,0xff,0x8e,0x7a,0x6a,0x7f]
+         vcvtuqq2psx	2032(%rdx), %xmm21 {%k6} {z}
+# CHECK: vcvtuqq2psx	2048(%rdx), %xmm21
+# CHECK: encoding: [0x62,0xe1,0xff,0x08,0x7a,0xaa,0x00,0x08,0x00,0x00]
+         vcvtuqq2psx	2048(%rdx), %xmm21
+# CHECK: vcvtuqq2psx	2048(%rdx), %xmm21 {%k6}
+# CHECK: encoding: [0x62,0xe1,0xff,0x0e,0x7a,0xaa,0x00,0x08,0x00,0x00]
+         vcvtuqq2psx	2048(%rdx), %xmm21 {%k6}
+# CHECK: vcvtuqq2psx	2048(%rdx), %xmm21 {%k6} {z}
+# CHECK: encoding: [0x62,0xe1,0xff,0x8e,0x7a,0xaa,0x00,0x08,0x00,0x00]
+         vcvtuqq2psx	2048(%rdx), %xmm21 {%k6} {z}
+# CHECK: vcvtuqq2psx	-2048(%rdx), %xmm21
+# CHECK: encoding: [0x62,0xe1,0xff,0x08,0x7a,0x6a,0x80]
+         vcvtuqq2psx	-2048(%rdx), %xmm21
+# CHECK: vcvtuqq2psx	-2048(%rdx), %xmm21 {%k6}
+# CHECK: encoding: [0x62,0xe1,0xff,0x0e,0x7a,0x6a,0x80]
+         vcvtuqq2psx	-2048(%rdx), %xmm21 {%k6}
+# CHECK: vcvtuqq2psx	-2048(%rdx), %xmm21 {%k6} {z}
+# CHECK: encoding: [0x62,0xe1,0xff,0x8e,0x7a,0x6a,0x80]
+         vcvtuqq2psx	-2048(%rdx), %xmm21 {%k6} {z}
+# CHECK: vcvtuqq2psx	-2064(%rdx), %xmm21
+# CHECK: encoding: [0x62,0xe1,0xff,0x08,0x7a,0xaa,0xf0,0xf7,0xff,0xff]
+         vcvtuqq2psx	-2064(%rdx), %xmm21
+# CHECK: vcvtuqq2psx	-2064(%rdx), %xmm21 {%k6}
+# CHECK: encoding: [0x62,0xe1,0xff,0x0e,0x7a,0xaa,0xf0,0xf7,0xff,0xff]
+         vcvtuqq2psx	-2064(%rdx), %xmm21 {%k6}
+# CHECK: vcvtuqq2psx	-2064(%rdx), %xmm21 {%k6} {z}
+# CHECK: encoding: [0x62,0xe1,0xff,0x8e,0x7a,0xaa,0xf0,0xf7,0xff,0xff]
+         vcvtuqq2psx	-2064(%rdx), %xmm21 {%k6} {z}
+# CHECK: vcvtuqq2ps	1016(%rdx){1to2}, %xmm21
+# CHECK: encoding: [0x62,0xe1,0xff,0x18,0x7a,0x6a,0x7f]
+         vcvtuqq2ps	1016(%rdx){1to2}, %xmm21
+# CHECK: vcvtuqq2ps	1016(%rdx){1to2}, %xmm21
+# CHECK: encoding: [0x62,0xe1,0xff,0x18,0x7a,0x6a,0x7f]
+         vcvtuqq2ps	1016(%rdx){1to2}, %xmm21
+# CHECK: vcvtuqq2ps	1016(%rdx){1to2}, %xmm21 {%k6}
+# CHECK: encoding: [0x62,0xe1,0xff,0x1e,0x7a,0x6a,0x7f]
+         vcvtuqq2ps	1016(%rdx){1to2}, %xmm21 {%k6}
+# CHECK: vcvtuqq2ps	1016(%rdx){1to2}, %xmm21 {%k6}
+# CHECK: encoding: [0x62,0xe1,0xff,0x1e,0x7a,0x6a,0x7f]
+         vcvtuqq2ps	1016(%rdx){1to2}, %xmm21 {%k6}
+# CHECK: vcvtuqq2ps	1016(%rdx){1to2}, %xmm21 {%k6} {z}
+# CHECK: encoding: [0x62,0xe1,0xff,0x9e,0x7a,0x6a,0x7f]
+         vcvtuqq2ps	1016(%rdx){1to2}, %xmm21 {%k6} {z}
+# CHECK: vcvtuqq2ps	1016(%rdx){1to2}, %xmm21 {%k6} {z}
+# CHECK: encoding: [0x62,0xe1,0xff,0x9e,0x7a,0x6a,0x7f]
+         vcvtuqq2ps	1016(%rdx){1to2}, %xmm21 {%k6} {z}
+# CHECK: vcvtuqq2ps	1024(%rdx){1to2}, %xmm21
+# CHECK: encoding: [0x62,0xe1,0xff,0x18,0x7a,0xaa,0x00,0x04,0x00,0x00]
+         vcvtuqq2ps	1024(%rdx){1to2}, %xmm21
+# CHECK: vcvtuqq2ps	1024(%rdx){1to2}, %xmm21
+# CHECK: encoding: [0x62,0xe1,0xff,0x18,0x7a,0xaa,0x00,0x04,0x00,0x00]
+         vcvtuqq2ps	1024(%rdx){1to2}, %xmm21
+# CHECK: vcvtuqq2ps	1024(%rdx){1to2}, %xmm21 {%k6}
+# CHECK: encoding: [0x62,0xe1,0xff,0x1e,0x7a,0xaa,0x00,0x04,0x00,0x00]
+         vcvtuqq2ps	1024(%rdx){1to2}, %xmm21 {%k6}
+# CHECK: vcvtuqq2ps	1024(%rdx){1to2}, %xmm21 {%k6}
+# CHECK: encoding: [0x62,0xe1,0xff,0x1e,0x7a,0xaa,0x00,0x04,0x00,0x00]
+         vcvtuqq2ps	1024(%rdx){1to2}, %xmm21 {%k6}
+# CHECK: vcvtuqq2ps	1024(%rdx){1to2}, %xmm21 {%k6} {z}
+# CHECK: encoding: [0x62,0xe1,0xff,0x9e,0x7a,0xaa,0x00,0x04,0x00,0x00]
+         vcvtuqq2ps	1024(%rdx){1to2}, %xmm21 {%k6} {z}
+# CHECK: vcvtuqq2ps	1024(%rdx){1to2}, %xmm21 {%k6} {z}
+# CHECK: encoding: [0x62,0xe1,0xff,0x9e,0x7a,0xaa,0x00,0x04,0x00,0x00]
+         vcvtuqq2ps	1024(%rdx){1to2}, %xmm21 {%k6} {z}
+# CHECK: vcvtuqq2ps	-1024(%rdx){1to2}, %xmm21
+# CHECK: encoding: [0x62,0xe1,0xff,0x18,0x7a,0x6a,0x80]
+         vcvtuqq2ps	-1024(%rdx){1to2}, %xmm21
+# CHECK: vcvtuqq2ps	-1024(%rdx){1to2}, %xmm21
+# CHECK: encoding: [0x62,0xe1,0xff,0x18,0x7a,0x6a,0x80]
+         vcvtuqq2ps	-1024(%rdx){1to2}, %xmm21
+# CHECK: vcvtuqq2ps	-1024(%rdx){1to2}, %xmm21 {%k6}
+# CHECK: encoding: [0x62,0xe1,0xff,0x1e,0x7a,0x6a,0x80]
+         vcvtuqq2ps	-1024(%rdx){1to2}, %xmm21 {%k6}
+# CHECK: vcvtuqq2ps	-1024(%rdx){1to2}, %xmm21 {%k6}
+# CHECK: encoding: [0x62,0xe1,0xff,0x1e,0x7a,0x6a,0x80]
+         vcvtuqq2ps	-1024(%rdx){1to2}, %xmm21 {%k6}
+# CHECK: vcvtuqq2ps	-1024(%rdx){1to2}, %xmm21 {%k6} {z}
+# CHECK: encoding: [0x62,0xe1,0xff,0x9e,0x7a,0x6a,0x80]
+         vcvtuqq2ps	-1024(%rdx){1to2}, %xmm21 {%k6} {z}
+# CHECK: vcvtuqq2ps	-1024(%rdx){1to2}, %xmm21 {%k6} {z}
+# CHECK: encoding: [0x62,0xe1,0xff,0x9e,0x7a,0x6a,0x80]
+         vcvtuqq2ps	-1024(%rdx){1to2}, %xmm21 {%k6} {z}
+# CHECK: vcvtuqq2ps	-1032(%rdx){1to2}, %xmm21
+# CHECK: encoding: [0x62,0xe1,0xff,0x18,0x7a,0xaa,0xf8,0xfb,0xff,0xff]
+         vcvtuqq2ps	-1032(%rdx){1to2}, %xmm21
+# CHECK: vcvtuqq2ps	-1032(%rdx){1to2}, %xmm21
+# CHECK: encoding: [0x62,0xe1,0xff,0x18,0x7a,0xaa,0xf8,0xfb,0xff,0xff]
+         vcvtuqq2ps	-1032(%rdx){1to2}, %xmm21
+# CHECK: vcvtuqq2ps	-1032(%rdx){1to2}, %xmm21 {%k6}
+# CHECK: encoding: [0x62,0xe1,0xff,0x1e,0x7a,0xaa,0xf8,0xfb,0xff,0xff]
+         vcvtuqq2ps	-1032(%rdx){1to2}, %xmm21 {%k6}
+# CHECK: vcvtuqq2ps	-1032(%rdx){1to2}, %xmm21 {%k6}
+# CHECK: encoding: [0x62,0xe1,0xff,0x1e,0x7a,0xaa,0xf8,0xfb,0xff,0xff]
+         vcvtuqq2ps	-1032(%rdx){1to2}, %xmm21 {%k6}
+# CHECK: vcvtuqq2ps	-1032(%rdx){1to2}, %xmm21 {%k6} {z}
+# CHECK: encoding: [0x62,0xe1,0xff,0x9e,0x7a,0xaa,0xf8,0xfb,0xff,0xff]
+         vcvtuqq2ps	-1032(%rdx){1to2}, %xmm21 {%k6} {z}
+# CHECK: vcvtuqq2ps	-1032(%rdx){1to2}, %xmm21 {%k6} {z}
+# CHECK: encoding: [0x62,0xe1,0xff,0x9e,0x7a,0xaa,0xf8,0xfb,0xff,0xff]
+         vcvtuqq2ps	-1032(%rdx){1to2}, %xmm21 {%k6} {z}
+# CHECK: vcvtuqq2ps	%ymm17, %xmm26
+# CHECK: encoding: [0x62,0x21,0xff,0x28,0x7a,0xd1]
+         vcvtuqq2ps	%ymm17, %xmm26
+# CHECK: vcvtuqq2ps	%ymm17, %xmm26
+# CHECK: encoding: [0x62,0x21,0xff,0x28,0x7a,0xd1]
+         vcvtuqq2ps	%ymm17, %xmm26
+# CHECK: vcvtuqq2ps	%ymm17, %xmm26 {%k4}
+# CHECK: encoding: [0x62,0x21,0xff,0x2c,0x7a,0xd1]
+         vcvtuqq2ps	%ymm17, %xmm26 {%k4}
+# CHECK: vcvtuqq2ps	%ymm17, %xmm26 {%k4}
+# CHECK: encoding: [0x62,0x21,0xff,0x2c,0x7a,0xd1]
+         vcvtuqq2ps	%ymm17, %xmm26 {%k4}
+# CHECK: vcvtuqq2ps	%ymm17, %xmm26 {%k4} {z}
+# CHECK: encoding: [0x62,0x21,0xff,0xac,0x7a,0xd1]
+         vcvtuqq2ps	%ymm17, %xmm26 {%k4} {z}
+# CHECK: vcvtuqq2ps	%ymm17, %xmm26 {%k4} {z}
+# CHECK: encoding: [0x62,0x21,0xff,0xac,0x7a,0xd1]
+         vcvtuqq2ps	%ymm17, %xmm26 {%k4} {z}
+# CHECK: vcvtuqq2psy	(%rcx), %xmm26
+# CHECK: encoding: [0x62,0x61,0xff,0x28,0x7a,0x11]
+         vcvtuqq2psy	(%rcx), %xmm26
+# CHECK: vcvtuqq2psy	(%rcx), %xmm26 {%k4}
+# CHECK: encoding: [0x62,0x61,0xff,0x2c,0x7a,0x11]
+         vcvtuqq2psy	(%rcx), %xmm26 {%k4}
+# CHECK: vcvtuqq2psy	(%rcx), %xmm26 {%k4} {z}
+# CHECK: encoding: [0x62,0x61,0xff,0xac,0x7a,0x11]
+         vcvtuqq2psy	(%rcx), %xmm26 {%k4} {z}
+# CHECK: vcvtuqq2psy	4660(%rax,%r14,8), %xmm26
+# CHECK: encoding: [0x62,0x21,0xff,0x28,0x7a,0x94,0xf0,0x34,0x12,0x00,0x00]
+         vcvtuqq2psy	4660(%rax,%r14,8), %xmm26
+# CHECK: vcvtuqq2psy	4660(%rax,%r14,8), %xmm26 {%k4}
+# CHECK: encoding: [0x62,0x21,0xff,0x2c,0x7a,0x94,0xf0,0x34,0x12,0x00,0x00]
+         vcvtuqq2psy	4660(%rax,%r14,8), %xmm26 {%k4}
+# CHECK: vcvtuqq2psy	4660(%rax,%r14,8), %xmm26 {%k4} {z}
+# CHECK: encoding: [0x62,0x21,0xff,0xac,0x7a,0x94,0xf0,0x34,0x12,0x00,0x00]
+         vcvtuqq2psy	4660(%rax,%r14,8), %xmm26 {%k4} {z}
+# CHECK: vcvtuqq2ps	(%rcx){1to4}, %xmm26
+# CHECK: encoding: [0x62,0x61,0xff,0x38,0x7a,0x11]
+         vcvtuqq2ps	(%rcx){1to4}, %xmm26
+# CHECK: vcvtuqq2ps	(%rcx){1to4}, %xmm26
+# CHECK: encoding: [0x62,0x61,0xff,0x38,0x7a,0x11]
+         vcvtuqq2ps	(%rcx){1to4}, %xmm26
+# CHECK: vcvtuqq2ps	(%rcx){1to4}, %xmm26 {%k4}
+# CHECK: encoding: [0x62,0x61,0xff,0x3c,0x7a,0x11]
+         vcvtuqq2ps	(%rcx){1to4}, %xmm26 {%k4}
+# CHECK: vcvtuqq2ps	(%rcx){1to4}, %xmm26 {%k4}
+# CHECK: encoding: [0x62,0x61,0xff,0x3c,0x7a,0x11]
+         vcvtuqq2ps	(%rcx){1to4}, %xmm26 {%k4}
+# CHECK: vcvtuqq2ps	(%rcx){1to4}, %xmm26 {%k4} {z}
+# CHECK: encoding: [0x62,0x61,0xff,0xbc,0x7a,0x11]
+         vcvtuqq2ps	(%rcx){1to4}, %xmm26 {%k4} {z}
+# CHECK: vcvtuqq2ps	(%rcx){1to4}, %xmm26 {%k4} {z}
+# CHECK: encoding: [0x62,0x61,0xff,0xbc,0x7a,0x11]
+         vcvtuqq2ps	(%rcx){1to4}, %xmm26 {%k4} {z}
+# CHECK: vcvtuqq2psy	4064(%rdx), %xmm26
+# CHECK: encoding: [0x62,0x61,0xff,0x28,0x7a,0x52,0x7f]
+         vcvtuqq2psy	4064(%rdx), %xmm26
+# CHECK: vcvtuqq2psy	4064(%rdx), %xmm26 {%k4}
+# CHECK: encoding: [0x62,0x61,0xff,0x2c,0x7a,0x52,0x7f]
+         vcvtuqq2psy	4064(%rdx), %xmm26 {%k4}
+# CHECK: vcvtuqq2psy	4064(%rdx), %xmm26 {%k4} {z}
+# CHECK: encoding: [0x62,0x61,0xff,0xac,0x7a,0x52,0x7f]
+         vcvtuqq2psy	4064(%rdx), %xmm26 {%k4} {z}
+# CHECK: vcvtuqq2psy	4096(%rdx), %xmm26
+# CHECK: encoding: [0x62,0x61,0xff,0x28,0x7a,0x92,0x00,0x10,0x00,0x00]
+         vcvtuqq2psy	4096(%rdx), %xmm26
+# CHECK: vcvtuqq2psy	4096(%rdx), %xmm26 {%k4}
+# CHECK: encoding: [0x62,0x61,0xff,0x2c,0x7a,0x92,0x00,0x10,0x00,0x00]
+         vcvtuqq2psy	4096(%rdx), %xmm26 {%k4}
+# CHECK: vcvtuqq2psy	4096(%rdx), %xmm26 {%k4} {z}
+# CHECK: encoding: [0x62,0x61,0xff,0xac,0x7a,0x92,0x00,0x10,0x00,0x00]
+         vcvtuqq2psy	4096(%rdx), %xmm26 {%k4} {z}
+# CHECK: vcvtuqq2psy	-4096(%rdx), %xmm26
+# CHECK: encoding: [0x62,0x61,0xff,0x28,0x7a,0x52,0x80]
+         vcvtuqq2psy	-4096(%rdx), %xmm26
+# CHECK: vcvtuqq2psy	-4096(%rdx), %xmm26 {%k4}
+# CHECK: encoding: [0x62,0x61,0xff,0x2c,0x7a,0x52,0x80]
+         vcvtuqq2psy	-4096(%rdx), %xmm26 {%k4}
+# CHECK: vcvtuqq2psy	-4096(%rdx), %xmm26 {%k4} {z}
+# CHECK: encoding: [0x62,0x61,0xff,0xac,0x7a,0x52,0x80]
+         vcvtuqq2psy	-4096(%rdx), %xmm26 {%k4} {z}
+# CHECK: vcvtuqq2psy	-4128(%rdx), %xmm26
+# CHECK: encoding: [0x62,0x61,0xff,0x28,0x7a,0x92,0xe0,0xef,0xff,0xff]
+         vcvtuqq2psy	-4128(%rdx), %xmm26
+# CHECK: vcvtuqq2psy	-4128(%rdx), %xmm26 {%k4}
+# CHECK: encoding: [0x62,0x61,0xff,0x2c,0x7a,0x92,0xe0,0xef,0xff,0xff]
+         vcvtuqq2psy	-4128(%rdx), %xmm26 {%k4}
+# CHECK: vcvtuqq2psy	-4128(%rdx), %xmm26 {%k4} {z}
+# CHECK: encoding: [0x62,0x61,0xff,0xac,0x7a,0x92,0xe0,0xef,0xff,0xff]
+         vcvtuqq2psy	-4128(%rdx), %xmm26 {%k4} {z}
+# CHECK: vcvtuqq2ps	1016(%rdx){1to4}, %xmm26
+# CHECK: encoding: [0x62,0x61,0xff,0x38,0x7a,0x52,0x7f]
+         vcvtuqq2ps	1016(%rdx){1to4}, %xmm26
+# CHECK: vcvtuqq2ps	1016(%rdx){1to4}, %xmm26
+# CHECK: encoding: [0x62,0x61,0xff,0x38,0x7a,0x52,0x7f]
+         vcvtuqq2ps	1016(%rdx){1to4}, %xmm26
+# CHECK: vcvtuqq2ps	1016(%rdx){1to4}, %xmm26 {%k4}
+# CHECK: encoding: [0x62,0x61,0xff,0x3c,0x7a,0x52,0x7f]
+         vcvtuqq2ps	1016(%rdx){1to4}, %xmm26 {%k4}
+# CHECK: vcvtuqq2ps	1016(%rdx){1to4}, %xmm26 {%k4}
+# CHECK: encoding: [0x62,0x61,0xff,0x3c,0x7a,0x52,0x7f]
+         vcvtuqq2ps	1016(%rdx){1to4}, %xmm26 {%k4}
+# CHECK: vcvtuqq2ps	1016(%rdx){1to4}, %xmm26 {%k4} {z}
+# CHECK: encoding: [0x62,0x61,0xff,0xbc,0x7a,0x52,0x7f]
+         vcvtuqq2ps	1016(%rdx){1to4}, %xmm26 {%k4} {z}
+# CHECK: vcvtuqq2ps	1016(%rdx){1to4}, %xmm26 {%k4} {z}
+# CHECK: encoding: [0x62,0x61,0xff,0xbc,0x7a,0x52,0x7f]
+         vcvtuqq2ps	1016(%rdx){1to4}, %xmm26 {%k4} {z}
+# CHECK: vcvtuqq2ps	1024(%rdx){1to4}, %xmm26
+# CHECK: encoding: [0x62,0x61,0xff,0x38,0x7a,0x92,0x00,0x04,0x00,0x00]
+         vcvtuqq2ps	1024(%rdx){1to4}, %xmm26
+# CHECK: vcvtuqq2ps	1024(%rdx){1to4}, %xmm26
+# CHECK: encoding: [0x62,0x61,0xff,0x38,0x7a,0x92,0x00,0x04,0x00,0x00]
+         vcvtuqq2ps	1024(%rdx){1to4}, %xmm26
+# CHECK: vcvtuqq2ps	1024(%rdx){1to4}, %xmm26 {%k4}
+# CHECK: encoding: [0x62,0x61,0xff,0x3c,0x7a,0x92,0x00,0x04,0x00,0x00]
+         vcvtuqq2ps	1024(%rdx){1to4}, %xmm26 {%k4}
+# CHECK: vcvtuqq2ps	1024(%rdx){1to4}, %xmm26 {%k4}
+# CHECK: encoding: [0x62,0x61,0xff,0x3c,0x7a,0x92,0x00,0x04,0x00,0x00]
+         vcvtuqq2ps	1024(%rdx){1to4}, %xmm26 {%k4}
+# CHECK: vcvtuqq2ps	1024(%rdx){1to4}, %xmm26 {%k4} {z}
+# CHECK: encoding: [0x62,0x61,0xff,0xbc,0x7a,0x92,0x00,0x04,0x00,0x00]
+         vcvtuqq2ps	1024(%rdx){1to4}, %xmm26 {%k4} {z}
+# CHECK: vcvtuqq2ps	1024(%rdx){1to4}, %xmm26 {%k4} {z}
+# CHECK: encoding: [0x62,0x61,0xff,0xbc,0x7a,0x92,0x00,0x04,0x00,0x00]
+         vcvtuqq2ps	1024(%rdx){1to4}, %xmm26 {%k4} {z}
+# CHECK: vcvtuqq2ps	-1024(%rdx){1to4}, %xmm26
+# CHECK: encoding: [0x62,0x61,0xff,0x38,0x7a,0x52,0x80]
+         vcvtuqq2ps	-1024(%rdx){1to4}, %xmm26
+# CHECK: vcvtuqq2ps	-1024(%rdx){1to4}, %xmm26
+# CHECK: encoding: [0x62,0x61,0xff,0x38,0x7a,0x52,0x80]
+         vcvtuqq2ps	-1024(%rdx){1to4}, %xmm26
+# CHECK: vcvtuqq2ps	-1024(%rdx){1to4}, %xmm26 {%k4}
+# CHECK: encoding: [0x62,0x61,0xff,0x3c,0x7a,0x52,0x80]
+         vcvtuqq2ps	-1024(%rdx){1to4}, %xmm26 {%k4}
+# CHECK: vcvtuqq2ps	-1024(%rdx){1to4}, %xmm26 {%k4}
+# CHECK: encoding: [0x62,0x61,0xff,0x3c,0x7a,0x52,0x80]
+         vcvtuqq2ps	-1024(%rdx){1to4}, %xmm26 {%k4}
+# CHECK: vcvtuqq2ps	-1024(%rdx){1to4}, %xmm26 {%k4} {z}
+# CHECK: encoding: [0x62,0x61,0xff,0xbc,0x7a,0x52,0x80]
+         vcvtuqq2ps	-1024(%rdx){1to4}, %xmm26 {%k4} {z}
+# CHECK: vcvtuqq2ps	-1024(%rdx){1to4}, %xmm26 {%k4} {z}
+# CHECK: encoding: [0x62,0x61,0xff,0xbc,0x7a,0x52,0x80]
+         vcvtuqq2ps	-1024(%rdx){1to4}, %xmm26 {%k4} {z}
+# CHECK: vcvtuqq2ps	-1032(%rdx){1to4}, %xmm26
+# CHECK: encoding: [0x62,0x61,0xff,0x38,0x7a,0x92,0xf8,0xfb,0xff,0xff]
+         vcvtuqq2ps	-1032(%rdx){1to4}, %xmm26
+# CHECK: vcvtuqq2ps	-1032(%rdx){1to4}, %xmm26
+# CHECK: encoding: [0x62,0x61,0xff,0x38,0x7a,0x92,0xf8,0xfb,0xff,0xff]
+         vcvtuqq2ps	-1032(%rdx){1to4}, %xmm26
+# CHECK: vcvtuqq2ps	-1032(%rdx){1to4}, %xmm26 {%k4}
+# CHECK: encoding: [0x62,0x61,0xff,0x3c,0x7a,0x92,0xf8,0xfb,0xff,0xff]
+         vcvtuqq2ps	-1032(%rdx){1to4}, %xmm26 {%k4}
+# CHECK: vcvtuqq2ps	-1032(%rdx){1to4}, %xmm26 {%k4}
+# CHECK: encoding: [0x62,0x61,0xff,0x3c,0x7a,0x92,0xf8,0xfb,0xff,0xff]
+         vcvtuqq2ps	-1032(%rdx){1to4}, %xmm26 {%k4}
+# CHECK: vcvtuqq2ps	-1032(%rdx){1to4}, %xmm26 {%k4} {z}
+# CHECK: encoding: [0x62,0x61,0xff,0xbc,0x7a,0x92,0xf8,0xfb,0xff,0xff]
+         vcvtuqq2ps	-1032(%rdx){1to4}, %xmm26 {%k4} {z}
+# CHECK: vcvtuqq2ps	-1032(%rdx){1to4}, %xmm26 {%k4} {z}
+# CHECK: encoding: [0x62,0x61,0xff,0xbc,0x7a,0x92,0xf8,0xfb,0xff,0xff]
+         vcvtuqq2ps	-1032(%rdx){1to4}, %xmm26 {%k4} {z}
+# CHECK: vinsertf64x2	$171, %xmm23, %ymm27, %ymm21
+# CHECK: encoding: [0x62,0xa3,0xa5,0x20,0x18,0xef,0xab]
+         vinsertf64x2	$171, %xmm23, %ymm27, %ymm21
+# CHECK: vinsertf64x2	$171, %xmm23, %ymm27, %ymm21 {%k7}
+# CHECK: encoding: [0x62,0xa3,0xa5,0x27,0x18,0xef,0xab]
+         vinsertf64x2	$171, %xmm23, %ymm27, %ymm21 {%k7}
+# CHECK: vinsertf64x2	$171, %xmm23, %ymm27, %ymm21 {%k7} {z}
+# CHECK: encoding: [0x62,0xa3,0xa5,0xa7,0x18,0xef,0xab]
+         vinsertf64x2	$171, %xmm23, %ymm27, %ymm21 {%k7} {z}
+# CHECK: vinsertf64x2	$123, %xmm23, %ymm27, %ymm21
+# CHECK: encoding: [0x62,0xa3,0xa5,0x20,0x18,0xef,0x7b]
+         vinsertf64x2	$123, %xmm23, %ymm27, %ymm21
+# CHECK: vinsertf64x2	$123, (%rcx), %ymm27, %ymm21
+# CHECK: encoding: [0x62,0xe3,0xa5,0x20,0x18,0x29,0x7b]
+         vinsertf64x2	$123, (%rcx), %ymm27, %ymm21
+# CHECK: vinsertf64x2	$123, 291(%rax,%r14,8), %ymm27, %ymm21
+# CHECK: encoding: [0x62,0xa3,0xa5,0x20,0x18,0xac,0xf0,0x23,0x01,0x00,0x00,0x7b]
+         vinsertf64x2	$123, 291(%rax,%r14,8), %ymm27, %ymm21
+# CHECK: vinsertf64x2	$123, 2032(%rdx), %ymm27, %ymm21
+# CHECK: encoding: [0x62,0xe3,0xa5,0x20,0x18,0x6a,0x7f,0x7b]
+         vinsertf64x2	$123, 2032(%rdx), %ymm27, %ymm21
+# CHECK: vinsertf64x2	$123, 2048(%rdx), %ymm27, %ymm21
+# CHECK: encoding: [0x62,0xe3,0xa5,0x20,0x18,0xaa,0x00,0x08,0x00,0x00,0x7b]
+         vinsertf64x2	$123, 2048(%rdx), %ymm27, %ymm21
+# CHECK: vinsertf64x2	$123, -2048(%rdx), %ymm27, %ymm21
+# CHECK: encoding: [0x62,0xe3,0xa5,0x20,0x18,0x6a,0x80,0x7b]
+         vinsertf64x2	$123, -2048(%rdx), %ymm27, %ymm21
+# CHECK: vinsertf64x2	$123, -2064(%rdx), %ymm27, %ymm21
+# CHECK: encoding: [0x62,0xe3,0xa5,0x20,0x18,0xaa,0xf0,0xf7,0xff,0xff,0x7b]
+         vinsertf64x2	$123, -2064(%rdx), %ymm27, %ymm21
+# CHECK: vinsertf64x2	$171, %xmm27, %ymm23, %ymm24
+# CHECK: encoding: [0x62,0x03,0xc5,0x20,0x18,0xc3,0xab]
+         vinsertf64x2	$171, %xmm27, %ymm23, %ymm24
+# CHECK: vinsertf64x2	$171, %xmm27, %ymm23, %ymm24 {%k5}
+# CHECK: encoding: [0x62,0x03,0xc5,0x25,0x18,0xc3,0xab]
+         vinsertf64x2	$171, %xmm27, %ymm23, %ymm24 {%k5}
+# CHECK: vinsertf64x2	$171, %xmm27, %ymm23, %ymm24 {%k5} {z}
+# CHECK: encoding: [0x62,0x03,0xc5,0xa5,0x18,0xc3,0xab]
+         vinsertf64x2	$171, %xmm27, %ymm23, %ymm24 {%k5} {z}
+# CHECK: vinsertf64x2	$123, %xmm27, %ymm23, %ymm24
+# CHECK: encoding: [0x62,0x03,0xc5,0x20,0x18,0xc3,0x7b]
+         vinsertf64x2	$123, %xmm27, %ymm23, %ymm24
+# CHECK: vinsertf64x2	$123, (%rcx), %ymm23, %ymm24
+# CHECK: encoding: [0x62,0x63,0xc5,0x20,0x18,0x01,0x7b]
+         vinsertf64x2	$123, (%rcx), %ymm23, %ymm24
+# CHECK: vinsertf64x2	$123, 4660(%rax,%r14,8), %ymm23, %ymm24
+# CHECK: encoding: [0x62,0x23,0xc5,0x20,0x18,0x84,0xf0,0x34,0x12,0x00,0x00,0x7b]
+         vinsertf64x2	$123, 4660(%rax,%r14,8), %ymm23, %ymm24
+# CHECK: vinsertf64x2	$123, 2032(%rdx), %ymm23, %ymm24
+# CHECK: encoding: [0x62,0x63,0xc5,0x20,0x18,0x42,0x7f,0x7b]
+         vinsertf64x2	$123, 2032(%rdx), %ymm23, %ymm24
+# CHECK: vinsertf64x2	$123, 2048(%rdx), %ymm23, %ymm24
+# CHECK: encoding: [0x62,0x63,0xc5,0x20,0x18,0x82,0x00,0x08,0x00,0x00,0x7b]
+         vinsertf64x2	$123, 2048(%rdx), %ymm23, %ymm24
+# CHECK: vinsertf64x2	$123, -2048(%rdx), %ymm23, %ymm24
+# CHECK: encoding: [0x62,0x63,0xc5,0x20,0x18,0x42,0x80,0x7b]
+         vinsertf64x2	$123, -2048(%rdx), %ymm23, %ymm24
+# CHECK: vinsertf64x2	$123, -2064(%rdx), %ymm23, %ymm24
+# CHECK: encoding: [0x62,0x63,0xc5,0x20,0x18,0x82,0xf0,0xf7,0xff,0xff,0x7b]
+         vinsertf64x2	$123, -2064(%rdx), %ymm23, %ymm24
+# CHECK: vinserti64x2	$171, %xmm21, %ymm25, %ymm19
+# CHECK: encoding: [0x62,0xa3,0xb5,0x20,0x38,0xdd,0xab]
+         vinserti64x2	$171, %xmm21, %ymm25, %ymm19
+# CHECK: vinserti64x2	$171, %xmm21, %ymm25, %ymm19 {%k6}
+# CHECK: encoding: [0x62,0xa3,0xb5,0x26,0x38,0xdd,0xab]
+         vinserti64x2	$171, %xmm21, %ymm25, %ymm19 {%k6}
+# CHECK: vinserti64x2	$171, %xmm21, %ymm25, %ymm19 {%k6} {z}
+# CHECK: encoding: [0x62,0xa3,0xb5,0xa6,0x38,0xdd,0xab]
+         vinserti64x2	$171, %xmm21, %ymm25, %ymm19 {%k6} {z}
+# CHECK: vinserti64x2	$123, %xmm21, %ymm25, %ymm19
+# CHECK: encoding: [0x62,0xa3,0xb5,0x20,0x38,0xdd,0x7b]
+         vinserti64x2	$123, %xmm21, %ymm25, %ymm19
+# CHECK: vinserti64x2	$123, (%rcx), %ymm25, %ymm19
+# CHECK: encoding: [0x62,0xe3,0xb5,0x20,0x38,0x19,0x7b]
+         vinserti64x2	$123, (%rcx), %ymm25, %ymm19
+# CHECK: vinserti64x2	$123, 291(%rax,%r14,8), %ymm25, %ymm19
+# CHECK: encoding: [0x62,0xa3,0xb5,0x20,0x38,0x9c,0xf0,0x23,0x01,0x00,0x00,0x7b]
+         vinserti64x2	$123, 291(%rax,%r14,8), %ymm25, %ymm19
+# CHECK: vinserti64x2	$123, 2032(%rdx), %ymm25, %ymm19
+# CHECK: encoding: [0x62,0xe3,0xb5,0x20,0x38,0x5a,0x7f,0x7b]
+         vinserti64x2	$123, 2032(%rdx), %ymm25, %ymm19
+# CHECK: vinserti64x2	$123, 2048(%rdx), %ymm25, %ymm19
+# CHECK: encoding: [0x62,0xe3,0xb5,0x20,0x38,0x9a,0x00,0x08,0x00,0x00,0x7b]
+         vinserti64x2	$123, 2048(%rdx), %ymm25, %ymm19
+# CHECK: vinserti64x2	$123, -2048(%rdx), %ymm25, %ymm19
+# CHECK: encoding: [0x62,0xe3,0xb5,0x20,0x38,0x5a,0x80,0x7b]
+         vinserti64x2	$123, -2048(%rdx), %ymm25, %ymm19
+# CHECK: vinserti64x2	$123, -2064(%rdx), %ymm25, %ymm19
+# CHECK: encoding: [0x62,0xe3,0xb5,0x20,0x38,0x9a,0xf0,0xf7,0xff,0xff,0x7b]
+         vinserti64x2	$123, -2064(%rdx), %ymm25, %ymm19
+# CHECK: vinserti64x2	$171, %xmm25, %ymm24, %ymm29
+# CHECK: encoding: [0x62,0x03,0xbd,0x20,0x38,0xe9,0xab]
+         vinserti64x2	$171, %xmm25, %ymm24, %ymm29
+# CHECK: vinserti64x2	$171, %xmm25, %ymm24, %ymm29 {%k2}
+# CHECK: encoding: [0x62,0x03,0xbd,0x22,0x38,0xe9,0xab]
+         vinserti64x2	$171, %xmm25, %ymm24, %ymm29 {%k2}
+# CHECK: vinserti64x2	$171, %xmm25, %ymm24, %ymm29 {%k2} {z}
+# CHECK: encoding: [0x62,0x03,0xbd,0xa2,0x38,0xe9,0xab]
+         vinserti64x2	$171, %xmm25, %ymm24, %ymm29 {%k2} {z}
+# CHECK: vinserti64x2	$123, %xmm25, %ymm24, %ymm29
+# CHECK: encoding: [0x62,0x03,0xbd,0x20,0x38,0xe9,0x7b]
+         vinserti64x2	$123, %xmm25, %ymm24, %ymm29
+# CHECK: vinserti64x2	$123, (%rcx), %ymm24, %ymm29
+# CHECK: encoding: [0x62,0x63,0xbd,0x20,0x38,0x29,0x7b]
+         vinserti64x2	$123, (%rcx), %ymm24, %ymm29
+# CHECK: vinserti64x2	$123, 4660(%rax,%r14,8), %ymm24, %ymm29
+# CHECK: encoding: [0x62,0x23,0xbd,0x20,0x38,0xac,0xf0,0x34,0x12,0x00,0x00,0x7b]
+         vinserti64x2	$123, 4660(%rax,%r14,8), %ymm24, %ymm29
+# CHECK: vinserti64x2	$123, 2032(%rdx), %ymm24, %ymm29
+# CHECK: encoding: [0x62,0x63,0xbd,0x20,0x38,0x6a,0x7f,0x7b]
+         vinserti64x2	$123, 2032(%rdx), %ymm24, %ymm29
+# CHECK: vinserti64x2	$123, 2048(%rdx), %ymm24, %ymm29
+# CHECK: encoding: [0x62,0x63,0xbd,0x20,0x38,0xaa,0x00,0x08,0x00,0x00,0x7b]
+         vinserti64x2	$123, 2048(%rdx), %ymm24, %ymm29
+# CHECK: vinserti64x2	$123, -2048(%rdx), %ymm24, %ymm29
+# CHECK: encoding: [0x62,0x63,0xbd,0x20,0x38,0x6a,0x80,0x7b]
+         vinserti64x2	$123, -2048(%rdx), %ymm24, %ymm29
+# CHECK: vinserti64x2	$123, -2064(%rdx), %ymm24, %ymm29
+# CHECK: encoding: [0x62,0x63,0xbd,0x20,0x38,0xaa,0xf0,0xf7,0xff,0xff,0x7b]
+         vinserti64x2	$123, -2064(%rdx), %ymm24, %ymm29
+# CHECK: vextractf64x2	$171, %ymm21, %xmm27
+# CHECK: encoding: [0x62,0x83,0xfd,0x28,0x19,0xeb,0xab]
+         vextractf64x2	$171, %ymm21, %xmm27
+# CHECK: vextractf64x2	$171, %ymm21, %xmm27 {%k7}
+# CHECK: encoding: [0x62,0x83,0xfd,0x2f,0x19,0xeb,0xab]
+         vextractf64x2	$171, %ymm21, %xmm27 {%k7}
+# CHECK: vextractf64x2	$171, %ymm21, %xmm27 {%k7} {z}
+# CHECK: encoding: [0x62,0x83,0xfd,0xaf,0x19,0xeb,0xab]
+         vextractf64x2	$171, %ymm21, %xmm27 {%k7} {z}
+# CHECK: vextractf64x2	$123, %ymm21, %xmm27
+# CHECK: encoding: [0x62,0x83,0xfd,0x28,0x19,0xeb,0x7b]
+         vextractf64x2	$123, %ymm21, %xmm27
+# CHECK: vextractf64x2	$171, %ymm20, (%rcx)
+# CHECK: encoding: [0x62,0xe3,0xfd,0x28,0x19,0x21,0xab]
+         vextractf64x2	$171, %ymm20, (%rcx)
+# CHECK: vextractf64x2	$171, %ymm20, (%rcx) {%k1}
+# CHECK: encoding: [0x62,0xe3,0xfd,0x29,0x19,0x21,0xab]
+         vextractf64x2	$171, %ymm20, (%rcx) {%k1}
+# CHECK: vextractf64x2	$123, %ymm20, (%rcx)
+# CHECK: encoding: [0x62,0xe3,0xfd,0x28,0x19,0x21,0x7b]
+         vextractf64x2	$123, %ymm20, (%rcx)
+# CHECK: vextractf64x2	$123, %ymm20, 291(%rax,%r14,8)
+# CHECK: encoding: [0x62,0xa3,0xfd,0x28,0x19,0xa4,0xf0,0x23,0x01,0x00,0x00,0x7b]
+         vextractf64x2	$123, %ymm20, 291(%rax,%r14,8)
+# CHECK: vextractf64x2	$123, %ymm20, 2032(%rdx)
+# CHECK: encoding: [0x62,0xe3,0xfd,0x28,0x19,0x62,0x7f,0x7b]
+         vextractf64x2	$123, %ymm20, 2032(%rdx)
+# CHECK: vextractf64x2	$123, %ymm20, 2048(%rdx)
+# CHECK: encoding: [0x62,0xe3,0xfd,0x28,0x19,0xa2,0x00,0x08,0x00,0x00,0x7b]
+         vextractf64x2	$123, %ymm20, 2048(%rdx)
+# CHECK: vextractf64x2	$123, %ymm20, -2048(%rdx)
+# CHECK: encoding: [0x62,0xe3,0xfd,0x28,0x19,0x62,0x80,0x7b]
+         vextractf64x2	$123, %ymm20, -2048(%rdx)
+# CHECK: vextractf64x2	$123, %ymm20, -2064(%rdx)
+# CHECK: encoding: [0x62,0xe3,0xfd,0x28,0x19,0xa2,0xf0,0xf7,0xff,0xff,0x7b]
+         vextractf64x2	$123, %ymm20, -2064(%rdx)
+# CHECK: vextractf64x2	$171, %ymm26, %xmm28
+# CHECK: encoding: [0x62,0x03,0xfd,0x28,0x19,0xd4,0xab]
+         vextractf64x2	$171, %ymm26, %xmm28
+# CHECK: vextractf64x2	$171, %ymm26, %xmm28 {%k4}
+# CHECK: encoding: [0x62,0x03,0xfd,0x2c,0x19,0xd4,0xab]
+         vextractf64x2	$171, %ymm26, %xmm28 {%k4}
+# CHECK: vextractf64x2	$171, %ymm26, %xmm28 {%k4} {z}
+# CHECK: encoding: [0x62,0x03,0xfd,0xac,0x19,0xd4,0xab]
+         vextractf64x2	$171, %ymm26, %xmm28 {%k4} {z}
+# CHECK: vextractf64x2	$123, %ymm26, %xmm28
+# CHECK: encoding: [0x62,0x03,0xfd,0x28,0x19,0xd4,0x7b]
+         vextractf64x2	$123, %ymm26, %xmm28
+# CHECK: vextractf64x2	$171, %ymm17, (%rcx)
+# CHECK: encoding: [0x62,0xe3,0xfd,0x28,0x19,0x09,0xab]
+         vextractf64x2	$171, %ymm17, (%rcx)
+# CHECK: vextractf64x2	$171, %ymm17, (%rcx) {%k2}
+# CHECK: encoding: [0x62,0xe3,0xfd,0x2a,0x19,0x09,0xab]
+         vextractf64x2	$171, %ymm17, (%rcx) {%k2}
+# CHECK: vextractf64x2	$123, %ymm17, (%rcx)
+# CHECK: encoding: [0x62,0xe3,0xfd,0x28,0x19,0x09,0x7b]
+         vextractf64x2	$123, %ymm17, (%rcx)
+# CHECK: vextractf64x2	$123, %ymm17, 4660(%rax,%r14,8)
+# CHECK: encoding: [0x62,0xa3,0xfd,0x28,0x19,0x8c,0xf0,0x34,0x12,0x00,0x00,0x7b]
+         vextractf64x2	$123, %ymm17, 4660(%rax,%r14,8)
+# CHECK: vextractf64x2	$123, %ymm17, 2032(%rdx)
+# CHECK: encoding: [0x62,0xe3,0xfd,0x28,0x19,0x4a,0x7f,0x7b]
+         vextractf64x2	$123, %ymm17, 2032(%rdx)
+# CHECK: vextractf64x2	$123, %ymm17, 2048(%rdx)
+# CHECK: encoding: [0x62,0xe3,0xfd,0x28,0x19,0x8a,0x00,0x08,0x00,0x00,0x7b]
+         vextractf64x2	$123, %ymm17, 2048(%rdx)
+# CHECK: vextractf64x2	$123, %ymm17, -2048(%rdx)
+# CHECK: encoding: [0x62,0xe3,0xfd,0x28,0x19,0x4a,0x80,0x7b]
+         vextractf64x2	$123, %ymm17, -2048(%rdx)
+# CHECK: vextractf64x2	$123, %ymm17, -2064(%rdx)
+# CHECK: encoding: [0x62,0xe3,0xfd,0x28,0x19,0x8a,0xf0,0xf7,0xff,0xff,0x7b]
+         vextractf64x2	$123, %ymm17, -2064(%rdx)
+# CHECK: vextracti64x2	$171, %ymm24, %xmm29
+# CHECK: encoding: [0x62,0x03,0xfd,0x28,0x39,0xc5,0xab]
+         vextracti64x2	$171, %ymm24, %xmm29
+# CHECK: vextracti64x2	$171, %ymm24, %xmm29 {%k7}
+# CHECK: encoding: [0x62,0x03,0xfd,0x2f,0x39,0xc5,0xab]
+         vextracti64x2	$171, %ymm24, %xmm29 {%k7}
+# CHECK: vextracti64x2	$171, %ymm24, %xmm29 {%k7} {z}
+# CHECK: encoding: [0x62,0x03,0xfd,0xaf,0x39,0xc5,0xab]
+         vextracti64x2	$171, %ymm24, %xmm29 {%k7} {z}
+# CHECK: vextracti64x2	$123, %ymm24, %xmm29
+# CHECK: encoding: [0x62,0x03,0xfd,0x28,0x39,0xc5,0x7b]
+         vextracti64x2	$123, %ymm24, %xmm29
+# CHECK: vextracti64x2	$171, %ymm17, (%rcx)
+# CHECK: encoding: [0x62,0xe3,0xfd,0x28,0x39,0x09,0xab]
+         vextracti64x2	$171, %ymm17, (%rcx)
+# CHECK: vextracti64x2	$171, %ymm17, (%rcx) {%k1}
+# CHECK: encoding: [0x62,0xe3,0xfd,0x29,0x39,0x09,0xab]
+         vextracti64x2	$171, %ymm17, (%rcx) {%k1}
+# CHECK: vextracti64x2	$123, %ymm17, (%rcx)
+# CHECK: encoding: [0x62,0xe3,0xfd,0x28,0x39,0x09,0x7b]
+         vextracti64x2	$123, %ymm17, (%rcx)
+# CHECK: vextracti64x2	$123, %ymm17, 291(%rax,%r14,8)
+# CHECK: encoding: [0x62,0xa3,0xfd,0x28,0x39,0x8c,0xf0,0x23,0x01,0x00,0x00,0x7b]
+         vextracti64x2	$123, %ymm17, 291(%rax,%r14,8)
+# CHECK: vextracti64x2	$123, %ymm17, 2032(%rdx)
+# CHECK: encoding: [0x62,0xe3,0xfd,0x28,0x39,0x4a,0x7f,0x7b]
+         vextracti64x2	$123, %ymm17, 2032(%rdx)
+# CHECK: vextracti64x2	$123, %ymm17, 2048(%rdx)
+# CHECK: encoding: [0x62,0xe3,0xfd,0x28,0x39,0x8a,0x00,0x08,0x00,0x00,0x7b]
+         vextracti64x2	$123, %ymm17, 2048(%rdx)
+# CHECK: vextracti64x2	$123, %ymm17, -2048(%rdx)
+# CHECK: encoding: [0x62,0xe3,0xfd,0x28,0x39,0x4a,0x80,0x7b]
+         vextracti64x2	$123, %ymm17, -2048(%rdx)
+# CHECK: vextracti64x2	$123, %ymm17, -2064(%rdx)
+# CHECK: encoding: [0x62,0xe3,0xfd,0x28,0x39,0x8a,0xf0,0xf7,0xff,0xff,0x7b]
+         vextracti64x2	$123, %ymm17, -2064(%rdx)
+# CHECK: vextracti64x2	$171, %ymm17, %xmm29
+# CHECK: encoding: [0x62,0x83,0xfd,0x28,0x39,0xcd,0xab]
+         vextracti64x2	$171, %ymm17, %xmm29
+# CHECK: vextracti64x2	$171, %ymm17, %xmm29 {%k5}
+# CHECK: encoding: [0x62,0x83,0xfd,0x2d,0x39,0xcd,0xab]
+         vextracti64x2	$171, %ymm17, %xmm29 {%k5}
+# CHECK: vextracti64x2	$171, %ymm17, %xmm29 {%k5} {z}
+# CHECK: encoding: [0x62,0x83,0xfd,0xad,0x39,0xcd,0xab]
+         vextracti64x2	$171, %ymm17, %xmm29 {%k5} {z}
+# CHECK: vextracti64x2	$123, %ymm17, %xmm29
+# CHECK: encoding: [0x62,0x83,0xfd,0x28,0x39,0xcd,0x7b]
+         vextracti64x2	$123, %ymm17, %xmm29
+# CHECK: vextracti64x2	$171, %ymm20, (%rcx)
+# CHECK: encoding: [0x62,0xe3,0xfd,0x28,0x39,0x21,0xab]
+         vextracti64x2	$171, %ymm20, (%rcx)
+# CHECK: vextracti64x2	$171, %ymm20, (%rcx) {%k2}
+# CHECK: encoding: [0x62,0xe3,0xfd,0x2a,0x39,0x21,0xab]
+         vextracti64x2	$171, %ymm20, (%rcx) {%k2}
+# CHECK: vextracti64x2	$123, %ymm20, (%rcx)
+# CHECK: encoding: [0x62,0xe3,0xfd,0x28,0x39,0x21,0x7b]
+         vextracti64x2	$123, %ymm20, (%rcx)
+# CHECK: vextracti64x2	$123, %ymm20, 4660(%rax,%r14,8)
+# CHECK: encoding: [0x62,0xa3,0xfd,0x28,0x39,0xa4,0xf0,0x34,0x12,0x00,0x00,0x7b]
+         vextracti64x2	$123, %ymm20, 4660(%rax,%r14,8)
+# CHECK: vextracti64x2	$123, %ymm20, 2032(%rdx)
+# CHECK: encoding: [0x62,0xe3,0xfd,0x28,0x39,0x62,0x7f,0x7b]
+         vextracti64x2	$123, %ymm20, 2032(%rdx)
+# CHECK: vextracti64x2	$123, %ymm20, 2048(%rdx)
+# CHECK: encoding: [0x62,0xe3,0xfd,0x28,0x39,0xa2,0x00,0x08,0x00,0x00,0x7b]
+         vextracti64x2	$123, %ymm20, 2048(%rdx)
+# CHECK: vextracti64x2	$123, %ymm20, -2048(%rdx)
+# CHECK: encoding: [0x62,0xe3,0xfd,0x28,0x39,0x62,0x80,0x7b]
+         vextracti64x2	$123, %ymm20, -2048(%rdx)
+# CHECK: vextracti64x2	$123, %ymm20, -2064(%rdx)
+# CHECK: encoding: [0x62,0xe3,0xfd,0x28,0x39,0xa2,0xf0,0xf7,0xff,0xff,0x7b]
+         vextracti64x2	$123, %ymm20, -2064(%rdx)
+# CHECK: vfpclasspd	$171, %xmm18, %k2
+# CHECK: encoding: [0x62,0xb3,0xfd,0x08,0x66,0xd2,0xab]
+         vfpclasspd	$171, %xmm18, %k2
+# CHECK: vfpclasspd	$171, %xmm18, %k2
+# CHECK: encoding: [0x62,0xb3,0xfd,0x08,0x66,0xd2,0xab]
+         vfpclasspd	$171, %xmm18, %k2
+# CHECK: vfpclasspd	$171, %xmm18, %k2 {%k7}
+# CHECK: encoding: [0x62,0xb3,0xfd,0x0f,0x66,0xd2,0xab]
+         vfpclasspd	$171, %xmm18, %k2 {%k7}
+# CHECK: vfpclasspd	$171, %xmm18, %k2 {%k7}
+# CHECK: encoding: [0x62,0xb3,0xfd,0x0f,0x66,0xd2,0xab]
+         vfpclasspd	$171, %xmm18, %k2 {%k7}
+# CHECK: vfpclasspd	$123, %xmm18, %k2
+# CHECK: encoding: [0x62,0xb3,0xfd,0x08,0x66,0xd2,0x7b]
+         vfpclasspd	$123, %xmm18, %k2
+# CHECK: vfpclasspd	$123, %xmm18, %k2
+# CHECK: encoding: [0x62,0xb3,0xfd,0x08,0x66,0xd2,0x7b]
+         vfpclasspd	$123, %xmm18, %k2
+# CHECK: vfpclasspdx	$123, (%rcx), %k2
+# CHECK: encoding: [0x62,0xf3,0xfd,0x08,0x66,0x11,0x7b]
+         vfpclasspdx	$123, (%rcx), %k2
+# CHECK: vfpclasspdx	$123, 291(%rax,%r14,8), %k2
+# CHECK: encoding: [0x62,0xb3,0xfd,0x08,0x66,0x94,0xf0,0x23,0x01,0x00,0x00,0x7b]
+         vfpclasspdx	$123, 291(%rax,%r14,8), %k2
+# CHECK: vfpclasspd	$123, (%rcx){1to2}, %k2
+# CHECK: encoding: [0x62,0xf3,0xfd,0x18,0x66,0x11,0x7b]
+         vfpclasspd	$123, (%rcx){1to2}, %k2
+# CHECK: vfpclasspd	$123, (%rcx){1to2}, %k2
+# CHECK: encoding: [0x62,0xf3,0xfd,0x18,0x66,0x11,0x7b]
+         vfpclasspd	$123, (%rcx){1to2}, %k2
+# CHECK: vfpclasspdx	$123, 2032(%rdx), %k2
+# CHECK: encoding: [0x62,0xf3,0xfd,0x08,0x66,0x52,0x7f,0x7b]
+         vfpclasspdx	$123, 2032(%rdx), %k2
+# CHECK: vfpclasspdx	$123, 2048(%rdx), %k2
+# CHECK: encoding: [0x62,0xf3,0xfd,0x08,0x66,0x92,0x00,0x08,0x00,0x00,0x7b]
+         vfpclasspdx	$123, 2048(%rdx), %k2
+# CHECK: vfpclasspdx	$123, -2048(%rdx), %k2
+# CHECK: encoding: [0x62,0xf3,0xfd,0x08,0x66,0x52,0x80,0x7b]
+         vfpclasspdx	$123, -2048(%rdx), %k2
+# CHECK: vfpclasspdx	$123, -2064(%rdx), %k2
+# CHECK: encoding: [0x62,0xf3,0xfd,0x08,0x66,0x92,0xf0,0xf7,0xff,0xff,0x7b]
+         vfpclasspdx	$123, -2064(%rdx), %k2
+# CHECK: vfpclasspd	$123, 1016(%rdx){1to2}, %k2
+# CHECK: encoding: [0x62,0xf3,0xfd,0x18,0x66,0x52,0x7f,0x7b]
+         vfpclasspd	$123, 1016(%rdx){1to2}, %k2
+# CHECK: vfpclasspd	$123, 1016(%rdx){1to2}, %k2
+# CHECK: encoding: [0x62,0xf3,0xfd,0x18,0x66,0x52,0x7f,0x7b]
+         vfpclasspd	$123, 1016(%rdx){1to2}, %k2
+# CHECK: vfpclasspd	$123, 1024(%rdx){1to2}, %k2
+# CHECK: encoding: [0x62,0xf3,0xfd,0x18,0x66,0x92,0x00,0x04,0x00,0x00,0x7b]
+         vfpclasspd	$123, 1024(%rdx){1to2}, %k2
+# CHECK: vfpclasspd	$123, 1024(%rdx){1to2}, %k2
+# CHECK: encoding: [0x62,0xf3,0xfd,0x18,0x66,0x92,0x00,0x04,0x00,0x00,0x7b]
+         vfpclasspd	$123, 1024(%rdx){1to2}, %k2
+# CHECK: vfpclasspd	$123, -1024(%rdx){1to2}, %k2
+# CHECK: encoding: [0x62,0xf3,0xfd,0x18,0x66,0x52,0x80,0x7b]
+         vfpclasspd	$123, -1024(%rdx){1to2}, %k2
+# CHECK: vfpclasspd	$123, -1024(%rdx){1to2}, %k2
+# CHECK: encoding: [0x62,0xf3,0xfd,0x18,0x66,0x52,0x80,0x7b]
+         vfpclasspd	$123, -1024(%rdx){1to2}, %k2
+# CHECK: vfpclasspd	$123, -1032(%rdx){1to2}, %k2
+# CHECK: encoding: [0x62,0xf3,0xfd,0x18,0x66,0x92,0xf8,0xfb,0xff,0xff,0x7b]
+         vfpclasspd	$123, -1032(%rdx){1to2}, %k2
+# CHECK: vfpclasspd	$123, -1032(%rdx){1to2}, %k2
+# CHECK: encoding: [0x62,0xf3,0xfd,0x18,0x66,0x92,0xf8,0xfb,0xff,0xff,0x7b]
+         vfpclasspd	$123, -1032(%rdx){1to2}, %k2
+# CHECK: vfpclasspd	$171, %ymm25, %k4
+# CHECK: encoding: [0x62,0x93,0xfd,0x28,0x66,0xe1,0xab]
+         vfpclasspd	$171, %ymm25, %k4
+# CHECK: vfpclasspd	$171, %ymm25, %k4
+# CHECK: encoding: [0x62,0x93,0xfd,0x28,0x66,0xe1,0xab]
+         vfpclasspd	$171, %ymm25, %k4
+# CHECK: vfpclasspd	$171, %ymm25, %k4 {%k6}
+# CHECK: encoding: [0x62,0x93,0xfd,0x2e,0x66,0xe1,0xab]
+         vfpclasspd	$171, %ymm25, %k4 {%k6}
+# CHECK: vfpclasspd	$171, %ymm25, %k4 {%k6}
+# CHECK: encoding: [0x62,0x93,0xfd,0x2e,0x66,0xe1,0xab]
+         vfpclasspd	$171, %ymm25, %k4 {%k6}
+# CHECK: vfpclasspd	$123, %ymm25, %k4
+# CHECK: encoding: [0x62,0x93,0xfd,0x28,0x66,0xe1,0x7b]
+         vfpclasspd	$123, %ymm25, %k4
+# CHECK: vfpclasspd	$123, %ymm25, %k4
+# CHECK: encoding: [0x62,0x93,0xfd,0x28,0x66,0xe1,0x7b]
+         vfpclasspd	$123, %ymm25, %k4
+# CHECK: vfpclasspdy	$123, (%rcx), %k4
+# CHECK: encoding: [0x62,0xf3,0xfd,0x28,0x66,0x21,0x7b]
+         vfpclasspdy	$123, (%rcx), %k4
+# CHECK: vfpclasspdy	$123, 291(%rax,%r14,8), %k4
+# CHECK: encoding: [0x62,0xb3,0xfd,0x28,0x66,0xa4,0xf0,0x23,0x01,0x00,0x00,0x7b]
+         vfpclasspdy	$123, 291(%rax,%r14,8), %k4
+# CHECK: vfpclasspd	$123, (%rcx){1to4}, %k4
+# CHECK: encoding: [0x62,0xf3,0xfd,0x38,0x66,0x21,0x7b]
+         vfpclasspd	$123, (%rcx){1to4}, %k4
+# CHECK: vfpclasspd	$123, (%rcx){1to4}, %k4
+# CHECK: encoding: [0x62,0xf3,0xfd,0x38,0x66,0x21,0x7b]
+         vfpclasspd	$123, (%rcx){1to4}, %k4
+# CHECK: vfpclasspdy	$123, 4064(%rdx), %k4
+# CHECK: encoding: [0x62,0xf3,0xfd,0x28,0x66,0x62,0x7f,0x7b]
+         vfpclasspdy	$123, 4064(%rdx), %k4
+# CHECK: vfpclasspdy	$123, 4096(%rdx), %k4
+# CHECK: encoding: [0x62,0xf3,0xfd,0x28,0x66,0xa2,0x00,0x10,0x00,0x00,0x7b]
+         vfpclasspdy	$123, 4096(%rdx), %k4
+# CHECK: vfpclasspdy	$123, -4096(%rdx), %k4
+# CHECK: encoding: [0x62,0xf3,0xfd,0x28,0x66,0x62,0x80,0x7b]
+         vfpclasspdy	$123, -4096(%rdx), %k4
+# CHECK: vfpclasspdy	$123, -4128(%rdx), %k4
+# CHECK: encoding: [0x62,0xf3,0xfd,0x28,0x66,0xa2,0xe0,0xef,0xff,0xff,0x7b]
+         vfpclasspdy	$123, -4128(%rdx), %k4
+# CHECK: vfpclasspd	$123, 1016(%rdx){1to4}, %k4
+# CHECK: encoding: [0x62,0xf3,0xfd,0x38,0x66,0x62,0x7f,0x7b]
+         vfpclasspd	$123, 1016(%rdx){1to4}, %k4
+# CHECK: vfpclasspd	$123, 1016(%rdx){1to4}, %k4
+# CHECK: encoding: [0x62,0xf3,0xfd,0x38,0x66,0x62,0x7f,0x7b]
+         vfpclasspd	$123, 1016(%rdx){1to4}, %k4
+# CHECK: vfpclasspd	$123, 1024(%rdx){1to4}, %k4
+# CHECK: encoding: [0x62,0xf3,0xfd,0x38,0x66,0xa2,0x00,0x04,0x00,0x00,0x7b]
+         vfpclasspd	$123, 1024(%rdx){1to4}, %k4
+# CHECK: vfpclasspd	$123, 1024(%rdx){1to4}, %k4
+# CHECK: encoding: [0x62,0xf3,0xfd,0x38,0x66,0xa2,0x00,0x04,0x00,0x00,0x7b]
+         vfpclasspd	$123, 1024(%rdx){1to4}, %k4
+# CHECK: vfpclasspd	$123, -1024(%rdx){1to4}, %k4
+# CHECK: encoding: [0x62,0xf3,0xfd,0x38,0x66,0x62,0x80,0x7b]
+         vfpclasspd	$123, -1024(%rdx){1to4}, %k4
+# CHECK: vfpclasspd	$123, -1024(%rdx){1to4}, %k4
+# CHECK: encoding: [0x62,0xf3,0xfd,0x38,0x66,0x62,0x80,0x7b]
+         vfpclasspd	$123, -1024(%rdx){1to4}, %k4
+# CHECK: vfpclasspd	$123, -1032(%rdx){1to4}, %k4
+# CHECK: encoding: [0x62,0xf3,0xfd,0x38,0x66,0xa2,0xf8,0xfb,0xff,0xff,0x7b]
+         vfpclasspd	$123, -1032(%rdx){1to4}, %k4
+# CHECK: vfpclasspd	$123, -1032(%rdx){1to4}, %k4
+# CHECK: encoding: [0x62,0xf3,0xfd,0x38,0x66,0xa2,0xf8,0xfb,0xff,0xff,0x7b]
+         vfpclasspd	$123, -1032(%rdx){1to4}, %k4
+# CHECK: vfpclassps	$171, %xmm20, %k4
+# CHECK: encoding: [0x62,0xb3,0x7d,0x08,0x66,0xe4,0xab]
+         vfpclassps	$171, %xmm20, %k4
+# CHECK: vfpclassps	$171, %xmm20, %k4
+# CHECK: encoding: [0x62,0xb3,0x7d,0x08,0x66,0xe4,0xab]
+         vfpclassps	$171, %xmm20, %k4
+# CHECK: vfpclassps	$171, %xmm20, %k4 {%k5}
+# CHECK: encoding: [0x62,0xb3,0x7d,0x0d,0x66,0xe4,0xab]
+         vfpclassps	$171, %xmm20, %k4 {%k5}
+# CHECK: vfpclassps	$171, %xmm20, %k4 {%k5}
+# CHECK: encoding: [0x62,0xb3,0x7d,0x0d,0x66,0xe4,0xab]
+         vfpclassps	$171, %xmm20, %k4 {%k5}
+# CHECK: vfpclassps	$123, %xmm20, %k4
+# CHECK: encoding: [0x62,0xb3,0x7d,0x08,0x66,0xe4,0x7b]
+         vfpclassps	$123, %xmm20, %k4
+# CHECK: vfpclassps	$123, %xmm20, %k4
+# CHECK: encoding: [0x62,0xb3,0x7d,0x08,0x66,0xe4,0x7b]
+         vfpclassps	$123, %xmm20, %k4
+# CHECK: vfpclasspsx	$123, (%rcx), %k4
+# CHECK: encoding: [0x62,0xf3,0x7d,0x08,0x66,0x21,0x7b]
+         vfpclasspsx	$123, (%rcx), %k4
+# CHECK: vfpclasspsx	$123, 291(%rax,%r14,8), %k4
+# CHECK: encoding: [0x62,0xb3,0x7d,0x08,0x66,0xa4,0xf0,0x23,0x01,0x00,0x00,0x7b]
+         vfpclasspsx	$123, 291(%rax,%r14,8), %k4
+# CHECK: vfpclassps	$123, (%rcx){1to4}, %k4
+# CHECK: encoding: [0x62,0xf3,0x7d,0x18,0x66,0x21,0x7b]
+         vfpclassps	$123, (%rcx){1to4}, %k4
+# CHECK: vfpclassps	$123, (%rcx){1to4}, %k4
+# CHECK: encoding: [0x62,0xf3,0x7d,0x18,0x66,0x21,0x7b]
+         vfpclassps	$123, (%rcx){1to4}, %k4
+# CHECK: vfpclasspsx	$123, 2032(%rdx), %k4
+# CHECK: encoding: [0x62,0xf3,0x7d,0x08,0x66,0x62,0x7f,0x7b]
+         vfpclasspsx	$123, 2032(%rdx), %k4
+# CHECK: vfpclasspsx	$123, 2048(%rdx), %k4
+# CHECK: encoding: [0x62,0xf3,0x7d,0x08,0x66,0xa2,0x00,0x08,0x00,0x00,0x7b]
+         vfpclasspsx	$123, 2048(%rdx), %k4
+# CHECK: vfpclasspsx	$123, -2048(%rdx), %k4
+# CHECK: encoding: [0x62,0xf3,0x7d,0x08,0x66,0x62,0x80,0x7b]
+         vfpclasspsx	$123, -2048(%rdx), %k4
+# CHECK: vfpclasspsx	$123, -2064(%rdx), %k4
+# CHECK: encoding: [0x62,0xf3,0x7d,0x08,0x66,0xa2,0xf0,0xf7,0xff,0xff,0x7b]
+         vfpclasspsx	$123, -2064(%rdx), %k4
+# CHECK: vfpclassps	$123, 508(%rdx){1to4}, %k4
+# CHECK: encoding: [0x62,0xf3,0x7d,0x18,0x66,0x62,0x7f,0x7b]
+         vfpclassps	$123, 508(%rdx){1to4}, %k4
+# CHECK: vfpclassps	$123, 508(%rdx){1to4}, %k4
+# CHECK: encoding: [0x62,0xf3,0x7d,0x18,0x66,0x62,0x7f,0x7b]
+         vfpclassps	$123, 508(%rdx){1to4}, %k4
+# CHECK: vfpclassps	$123, 512(%rdx){1to4}, %k4
+# CHECK: encoding: [0x62,0xf3,0x7d,0x18,0x66,0xa2,0x00,0x02,0x00,0x00,0x7b]
+         vfpclassps	$123, 512(%rdx){1to4}, %k4
+# CHECK: vfpclassps	$123, 512(%rdx){1to4}, %k4
+# CHECK: encoding: [0x62,0xf3,0x7d,0x18,0x66,0xa2,0x00,0x02,0x00,0x00,0x7b]
+         vfpclassps	$123, 512(%rdx){1to4}, %k4
+# CHECK: vfpclassps	$123, -512(%rdx){1to4}, %k4
+# CHECK: encoding: [0x62,0xf3,0x7d,0x18,0x66,0x62,0x80,0x7b]
+         vfpclassps	$123, -512(%rdx){1to4}, %k4
+# CHECK: vfpclassps	$123, -512(%rdx){1to4}, %k4
+# CHECK: encoding: [0x62,0xf3,0x7d,0x18,0x66,0x62,0x80,0x7b]
+         vfpclassps	$123, -512(%rdx){1to4}, %k4
+# CHECK: vfpclassps	$123, -516(%rdx){1to4}, %k4
+# CHECK: encoding: [0x62,0xf3,0x7d,0x18,0x66,0xa2,0xfc,0xfd,0xff,0xff,0x7b]
+         vfpclassps	$123, -516(%rdx){1to4}, %k4
+# CHECK: vfpclassps	$123, -516(%rdx){1to4}, %k4
+# CHECK: encoding: [0x62,0xf3,0x7d,0x18,0x66,0xa2,0xfc,0xfd,0xff,0xff,0x7b]
+         vfpclassps	$123, -516(%rdx){1to4}, %k4
+# CHECK: vfpclassps	$171, %ymm17, %k4
+# CHECK: encoding: [0x62,0xb3,0x7d,0x28,0x66,0xe1,0xab]
+         vfpclassps	$171, %ymm17, %k4
+# CHECK: vfpclassps	$171, %ymm17, %k4
+# CHECK: encoding: [0x62,0xb3,0x7d,0x28,0x66,0xe1,0xab]
+         vfpclassps	$171, %ymm17, %k4
+# CHECK: vfpclassps	$171, %ymm17, %k4 {%k5}
+# CHECK: encoding: [0x62,0xb3,0x7d,0x2d,0x66,0xe1,0xab]
+         vfpclassps	$171, %ymm17, %k4 {%k5}
+# CHECK: vfpclassps	$171, %ymm17, %k4 {%k5}
+# CHECK: encoding: [0x62,0xb3,0x7d,0x2d,0x66,0xe1,0xab]
+         vfpclassps	$171, %ymm17, %k4 {%k5}
+# CHECK: vfpclassps	$123, %ymm17, %k4
+# CHECK: encoding: [0x62,0xb3,0x7d,0x28,0x66,0xe1,0x7b]
+         vfpclassps	$123, %ymm17, %k4
+# CHECK: vfpclassps	$123, %ymm17, %k4
+# CHECK: encoding: [0x62,0xb3,0x7d,0x28,0x66,0xe1,0x7b]
+         vfpclassps	$123, %ymm17, %k4
+# CHECK: vfpclasspsy	$123, (%rcx), %k4
+# CHECK: encoding: [0x62,0xf3,0x7d,0x28,0x66,0x21,0x7b]
+         vfpclasspsy	$123, (%rcx), %k4
+# CHECK: vfpclasspsy	$123, 291(%rax,%r14,8), %k4
+# CHECK: encoding: [0x62,0xb3,0x7d,0x28,0x66,0xa4,0xf0,0x23,0x01,0x00,0x00,0x7b]
+         vfpclasspsy	$123, 291(%rax,%r14,8), %k4
+# CHECK: vfpclassps	$123, (%rcx){1to8}, %k4
+# CHECK: encoding: [0x62,0xf3,0x7d,0x38,0x66,0x21,0x7b]
+         vfpclassps	$123, (%rcx){1to8}, %k4
+# CHECK: vfpclassps	$123, (%rcx){1to8}, %k4
+# CHECK: encoding: [0x62,0xf3,0x7d,0x38,0x66,0x21,0x7b]
+         vfpclassps	$123, (%rcx){1to8}, %k4
+# CHECK: vfpclasspsy	$123, 4064(%rdx), %k4
+# CHECK: encoding: [0x62,0xf3,0x7d,0x28,0x66,0x62,0x7f,0x7b]
+         vfpclasspsy	$123, 4064(%rdx), %k4
+# CHECK: vfpclasspsy	$123, 4096(%rdx), %k4
+# CHECK: encoding: [0x62,0xf3,0x7d,0x28,0x66,0xa2,0x00,0x10,0x00,0x00,0x7b]
+         vfpclasspsy	$123, 4096(%rdx), %k4
+# CHECK: vfpclasspsy	$123, -4096(%rdx), %k4
+# CHECK: encoding: [0x62,0xf3,0x7d,0x28,0x66,0x62,0x80,0x7b]
+         vfpclasspsy	$123, -4096(%rdx), %k4
+# CHECK: vfpclasspsy	$123, -4128(%rdx), %k4
+# CHECK: encoding: [0x62,0xf3,0x7d,0x28,0x66,0xa2,0xe0,0xef,0xff,0xff,0x7b]
+         vfpclasspsy	$123, -4128(%rdx), %k4
+# CHECK: vfpclassps	$123, 508(%rdx){1to8}, %k4
+# CHECK: encoding: [0x62,0xf3,0x7d,0x38,0x66,0x62,0x7f,0x7b]
+         vfpclassps	$123, 508(%rdx){1to8}, %k4
+# CHECK: vfpclassps	$123, 508(%rdx){1to8}, %k4
+# CHECK: encoding: [0x62,0xf3,0x7d,0x38,0x66,0x62,0x7f,0x7b]
+         vfpclassps	$123, 508(%rdx){1to8}, %k4
+# CHECK: vfpclassps	$123, 512(%rdx){1to8}, %k4
+# CHECK: encoding: [0x62,0xf3,0x7d,0x38,0x66,0xa2,0x00,0x02,0x00,0x00,0x7b]
+         vfpclassps	$123, 512(%rdx){1to8}, %k4
+# CHECK: vfpclassps	$123, 512(%rdx){1to8}, %k4
+# CHECK: encoding: [0x62,0xf3,0x7d,0x38,0x66,0xa2,0x00,0x02,0x00,0x00,0x7b]
+         vfpclassps	$123, 512(%rdx){1to8}, %k4
+# CHECK: vfpclassps	$123, -512(%rdx){1to8}, %k4
+# CHECK: encoding: [0x62,0xf3,0x7d,0x38,0x66,0x62,0x80,0x7b]
+         vfpclassps	$123, -512(%rdx){1to8}, %k4
+# CHECK: vfpclassps	$123, -512(%rdx){1to8}, %k4
+# CHECK: encoding: [0x62,0xf3,0x7d,0x38,0x66,0x62,0x80,0x7b]
+         vfpclassps	$123, -512(%rdx){1to8}, %k4
+# CHECK: vfpclassps	$123, -516(%rdx){1to8}, %k4
+# CHECK: encoding: [0x62,0xf3,0x7d,0x38,0x66,0xa2,0xfc,0xfd,0xff,0xff,0x7b]
+         vfpclassps	$123, -516(%rdx){1to8}, %k4
+# CHECK: vfpclassps	$123, -516(%rdx){1to8}, %k4
+# CHECK: encoding: [0x62,0xf3,0x7d,0x38,0x66,0xa2,0xfc,0xfd,0xff,0xff,0x7b]
+         vfpclassps	$123, -516(%rdx){1to8}, %k4
+# CHECK: vfpclasspd	$171, %xmm26, %k3
+# CHECK: encoding: [0x62,0x93,0xfd,0x08,0x66,0xda,0xab]
+         vfpclasspd	$171, %xmm26, %k3
+# CHECK: vfpclasspd	$171, %xmm26, %k3
+# CHECK: encoding: [0x62,0x93,0xfd,0x08,0x66,0xda,0xab]
+         vfpclasspd	$171, %xmm26, %k3
+# CHECK: vfpclasspd	$171, %xmm26, %k3 {%k5}
+# CHECK: encoding: [0x62,0x93,0xfd,0x0d,0x66,0xda,0xab]
+         vfpclasspd	$171, %xmm26, %k3 {%k5}
+# CHECK: vfpclasspd	$171, %xmm26, %k3 {%k5}
+# CHECK: encoding: [0x62,0x93,0xfd,0x0d,0x66,0xda,0xab]
+         vfpclasspd	$171, %xmm26, %k3 {%k5}
+# CHECK: vfpclasspd	$123, %xmm26, %k3
+# CHECK: encoding: [0x62,0x93,0xfd,0x08,0x66,0xda,0x7b]
+         vfpclasspd	$123, %xmm26, %k3
+# CHECK: vfpclasspd	$123, %xmm26, %k3
+# CHECK: encoding: [0x62,0x93,0xfd,0x08,0x66,0xda,0x7b]
+         vfpclasspd	$123, %xmm26, %k3
+# CHECK: vfpclasspdx	$123, (%rcx), %k3
+# CHECK: encoding: [0x62,0xf3,0xfd,0x08,0x66,0x19,0x7b]
+         vfpclasspdx	$123, (%rcx), %k3
+# CHECK: vfpclasspdx	$123, 4660(%rax,%r14,8), %k3
+# CHECK: encoding: [0x62,0xb3,0xfd,0x08,0x66,0x9c,0xf0,0x34,0x12,0x00,0x00,0x7b]
+         vfpclasspdx	$123, 4660(%rax,%r14,8), %k3
+# CHECK: vfpclasspd	$123, (%rcx){1to2}, %k3
+# CHECK: encoding: [0x62,0xf3,0xfd,0x18,0x66,0x19,0x7b]
+         vfpclasspd	$123, (%rcx){1to2}, %k3
+# CHECK: vfpclasspd	$123, (%rcx){1to2}, %k3
+# CHECK: encoding: [0x62,0xf3,0xfd,0x18,0x66,0x19,0x7b]
+         vfpclasspd	$123, (%rcx){1to2}, %k3
+# CHECK: vfpclasspdx	$123, 2032(%rdx), %k3
+# CHECK: encoding: [0x62,0xf3,0xfd,0x08,0x66,0x5a,0x7f,0x7b]
+         vfpclasspdx	$123, 2032(%rdx), %k3
+# CHECK: vfpclasspdx	$123, 2048(%rdx), %k3
+# CHECK: encoding: [0x62,0xf3,0xfd,0x08,0x66,0x9a,0x00,0x08,0x00,0x00,0x7b]
+         vfpclasspdx	$123, 2048(%rdx), %k3
+# CHECK: vfpclasspdx	$123, -2048(%rdx), %k3
+# CHECK: encoding: [0x62,0xf3,0xfd,0x08,0x66,0x5a,0x80,0x7b]
+         vfpclasspdx	$123, -2048(%rdx), %k3
+# CHECK: vfpclasspdx	$123, -2064(%rdx), %k3
+# CHECK: encoding: [0x62,0xf3,0xfd,0x08,0x66,0x9a,0xf0,0xf7,0xff,0xff,0x7b]
+         vfpclasspdx	$123, -2064(%rdx), %k3
+# CHECK: vfpclasspd	$123, 1016(%rdx){1to2}, %k3
+# CHECK: encoding: [0x62,0xf3,0xfd,0x18,0x66,0x5a,0x7f,0x7b]
+         vfpclasspd	$123, 1016(%rdx){1to2}, %k3
+# CHECK: vfpclasspd	$123, 1016(%rdx){1to2}, %k3
+# CHECK: encoding: [0x62,0xf3,0xfd,0x18,0x66,0x5a,0x7f,0x7b]
+         vfpclasspd	$123, 1016(%rdx){1to2}, %k3
+# CHECK: vfpclasspd	$123, 1024(%rdx){1to2}, %k3
+# CHECK: encoding: [0x62,0xf3,0xfd,0x18,0x66,0x9a,0x00,0x04,0x00,0x00,0x7b]
+         vfpclasspd	$123, 1024(%rdx){1to2}, %k3
+# CHECK: vfpclasspd	$123, 1024(%rdx){1to2}, %k3
+# CHECK: encoding: [0x62,0xf3,0xfd,0x18,0x66,0x9a,0x00,0x04,0x00,0x00,0x7b]
+         vfpclasspd	$123, 1024(%rdx){1to2}, %k3
+# CHECK: vfpclasspd	$123, -1024(%rdx){1to2}, %k3
+# CHECK: encoding: [0x62,0xf3,0xfd,0x18,0x66,0x5a,0x80,0x7b]
+         vfpclasspd	$123, -1024(%rdx){1to2}, %k3
+# CHECK: vfpclasspd	$123, -1024(%rdx){1to2}, %k3
+# CHECK: encoding: [0x62,0xf3,0xfd,0x18,0x66,0x5a,0x80,0x7b]
+         vfpclasspd	$123, -1024(%rdx){1to2}, %k3
+# CHECK: vfpclasspd	$123, -1032(%rdx){1to2}, %k3
+# CHECK: encoding: [0x62,0xf3,0xfd,0x18,0x66,0x9a,0xf8,0xfb,0xff,0xff,0x7b]
+         vfpclasspd	$123, -1032(%rdx){1to2}, %k3
+# CHECK: vfpclasspd	$123, -1032(%rdx){1to2}, %k3
+# CHECK: encoding: [0x62,0xf3,0xfd,0x18,0x66,0x9a,0xf8,0xfb,0xff,0xff,0x7b]
+         vfpclasspd	$123, -1032(%rdx){1to2}, %k3
+# CHECK: vfpclasspd	$171, %ymm26, %k2
+# CHECK: encoding: [0x62,0x93,0xfd,0x28,0x66,0xd2,0xab]
+         vfpclasspd	$171, %ymm26, %k2
+# CHECK: vfpclasspd	$171, %ymm26, %k2
+# CHECK: encoding: [0x62,0x93,0xfd,0x28,0x66,0xd2,0xab]
+         vfpclasspd	$171, %ymm26, %k2
+# CHECK: vfpclasspd	$171, %ymm26, %k2 {%k6}
+# CHECK: encoding: [0x62,0x93,0xfd,0x2e,0x66,0xd2,0xab]
+         vfpclasspd	$171, %ymm26, %k2 {%k6}
+# CHECK: vfpclasspd	$171, %ymm26, %k2 {%k6}
+# CHECK: encoding: [0x62,0x93,0xfd,0x2e,0x66,0xd2,0xab]
+         vfpclasspd	$171, %ymm26, %k2 {%k6}
+# CHECK: vfpclasspd	$123, %ymm26, %k2
+# CHECK: encoding: [0x62,0x93,0xfd,0x28,0x66,0xd2,0x7b]
+         vfpclasspd	$123, %ymm26, %k2
+# CHECK: vfpclasspd	$123, %ymm26, %k2
+# CHECK: encoding: [0x62,0x93,0xfd,0x28,0x66,0xd2,0x7b]
+         vfpclasspd	$123, %ymm26, %k2
+# CHECK: vfpclasspdy	$123, (%rcx), %k2
+# CHECK: encoding: [0x62,0xf3,0xfd,0x28,0x66,0x11,0x7b]
+         vfpclasspdy	$123, (%rcx), %k2
+# CHECK: vfpclasspdy	$123, 4660(%rax,%r14,8), %k2
+# CHECK: encoding: [0x62,0xb3,0xfd,0x28,0x66,0x94,0xf0,0x34,0x12,0x00,0x00,0x7b]
+         vfpclasspdy	$123, 4660(%rax,%r14,8), %k2
+# CHECK: vfpclasspd	$123, (%rcx){1to4}, %k2
+# CHECK: encoding: [0x62,0xf3,0xfd,0x38,0x66,0x11,0x7b]
+         vfpclasspd	$123, (%rcx){1to4}, %k2
+# CHECK: vfpclasspd	$123, (%rcx){1to4}, %k2
+# CHECK: encoding: [0x62,0xf3,0xfd,0x38,0x66,0x11,0x7b]
+         vfpclasspd	$123, (%rcx){1to4}, %k2
+# CHECK: vfpclasspdy	$123, 4064(%rdx), %k2
+# CHECK: encoding: [0x62,0xf3,0xfd,0x28,0x66,0x52,0x7f,0x7b]
+         vfpclasspdy	$123, 4064(%rdx), %k2
+# CHECK: vfpclasspdy	$123, 4096(%rdx), %k2
+# CHECK: encoding: [0x62,0xf3,0xfd,0x28,0x66,0x92,0x00,0x10,0x00,0x00,0x7b]
+         vfpclasspdy	$123, 4096(%rdx), %k2
+# CHECK: vfpclasspdy	$123, -4096(%rdx), %k2
+# CHECK: encoding: [0x62,0xf3,0xfd,0x28,0x66,0x52,0x80,0x7b]
+         vfpclasspdy	$123, -4096(%rdx), %k2
+# CHECK: vfpclasspdy	$123, -4128(%rdx), %k2
+# CHECK: encoding: [0x62,0xf3,0xfd,0x28,0x66,0x92,0xe0,0xef,0xff,0xff,0x7b]
+         vfpclasspdy	$123, -4128(%rdx), %k2
+# CHECK: vfpclasspd	$123, 1016(%rdx){1to4}, %k2
+# CHECK: encoding: [0x62,0xf3,0xfd,0x38,0x66,0x52,0x7f,0x7b]
+         vfpclasspd	$123, 1016(%rdx){1to4}, %k2
+# CHECK: vfpclasspd	$123, 1016(%rdx){1to4}, %k2
+# CHECK: encoding: [0x62,0xf3,0xfd,0x38,0x66,0x52,0x7f,0x7b]
+         vfpclasspd	$123, 1016(%rdx){1to4}, %k2
+# CHECK: vfpclasspd	$123, 1024(%rdx){1to4}, %k2
+# CHECK: encoding: [0x62,0xf3,0xfd,0x38,0x66,0x92,0x00,0x04,0x00,0x00,0x7b]
+         vfpclasspd	$123, 1024(%rdx){1to4}, %k2
+# CHECK: vfpclasspd	$123, 1024(%rdx){1to4}, %k2
+# CHECK: encoding: [0x62,0xf3,0xfd,0x38,0x66,0x92,0x00,0x04,0x00,0x00,0x7b]
+         vfpclasspd	$123, 1024(%rdx){1to4}, %k2
+# CHECK: vfpclasspd	$123, -1024(%rdx){1to4}, %k2
+# CHECK: encoding: [0x62,0xf3,0xfd,0x38,0x66,0x52,0x80,0x7b]
+         vfpclasspd	$123, -1024(%rdx){1to4}, %k2
+# CHECK: vfpclasspd	$123, -1024(%rdx){1to4}, %k2
+# CHECK: encoding: [0x62,0xf3,0xfd,0x38,0x66,0x52,0x80,0x7b]
+         vfpclasspd	$123, -1024(%rdx){1to4}, %k2
+# CHECK: vfpclasspd	$123, -1032(%rdx){1to4}, %k2
+# CHECK: encoding: [0x62,0xf3,0xfd,0x38,0x66,0x92,0xf8,0xfb,0xff,0xff,0x7b]
+         vfpclasspd	$123, -1032(%rdx){1to4}, %k2
+# CHECK: vfpclasspd	$123, -1032(%rdx){1to4}, %k2
+# CHECK: encoding: [0x62,0xf3,0xfd,0x38,0x66,0x92,0xf8,0xfb,0xff,0xff,0x7b]
+         vfpclasspd	$123, -1032(%rdx){1to4}, %k2
+# CHECK: vfpclassps	$171, %xmm29, %k3
+# CHECK: encoding: [0x62,0x93,0x7d,0x08,0x66,0xdd,0xab]
+         vfpclassps	$171, %xmm29, %k3
+# CHECK: vfpclassps	$171, %xmm29, %k3
+# CHECK: encoding: [0x62,0x93,0x7d,0x08,0x66,0xdd,0xab]
+         vfpclassps	$171, %xmm29, %k3
+# CHECK: vfpclassps	$171, %xmm29, %k3 {%k6}
+# CHECK: encoding: [0x62,0x93,0x7d,0x0e,0x66,0xdd,0xab]
+         vfpclassps	$171, %xmm29, %k3 {%k6}
+# CHECK: vfpclassps	$171, %xmm29, %k3
+# CHECK: encoding: [0x62,0x93,0x7d,0x08,0x66,0xdd,0xab]
+         vfpclassps	$171, %xmm29, %k3
+# CHECK: vfpclassps	$123, %xmm29, %k3
+# CHECK: encoding: [0x62,0x93,0x7d,0x08,0x66,0xdd,0x7b]
+         vfpclassps	$123, %xmm29, %k3
+# CHECK: vfpclassps	$123, %xmm29, %k3
+# CHECK: encoding: [0x62,0x93,0x7d,0x08,0x66,0xdd,0x7b]
+         vfpclassps	$123, %xmm29, %k3
+# CHECK: vfpclasspsx	$123, (%rcx), %k3
+# CHECK: encoding: [0x62,0xf3,0x7d,0x08,0x66,0x19,0x7b]
+         vfpclasspsx	$123, (%rcx), %k3
+# CHECK: vfpclasspsx	$123, 4660(%rax,%r14,8), %k3
+# CHECK: encoding: [0x62,0xb3,0x7d,0x08,0x66,0x9c,0xf0,0x34,0x12,0x00,0x00,0x7b]
+         vfpclasspsx	$123, 4660(%rax,%r14,8), %k3
+# CHECK: vfpclassps	$123, (%rcx){1to4}, %k3
+# CHECK: encoding: [0x62,0xf3,0x7d,0x18,0x66,0x19,0x7b]
+         vfpclassps	$123, (%rcx){1to4}, %k3
+# CHECK: vfpclassps	$123, (%rcx){1to4}, %k3
+# CHECK: encoding: [0x62,0xf3,0x7d,0x18,0x66,0x19,0x7b]
+         vfpclassps	$123, (%rcx){1to4}, %k3
+# CHECK: vfpclasspsx	$123, 2032(%rdx), %k3
+# CHECK: encoding: [0x62,0xf3,0x7d,0x08,0x66,0x5a,0x7f,0x7b]
+         vfpclasspsx	$123, 2032(%rdx), %k3
+# CHECK: vfpclasspsx	$123, 2048(%rdx), %k3
+# CHECK: encoding: [0x62,0xf3,0x7d,0x08,0x66,0x9a,0x00,0x08,0x00,0x00,0x7b]
+         vfpclasspsx	$123, 2048(%rdx), %k3
+# CHECK: vfpclasspsx	$123, -2048(%rdx), %k3
+# CHECK: encoding: [0x62,0xf3,0x7d,0x08,0x66,0x5a,0x80,0x7b]
+         vfpclasspsx	$123, -2048(%rdx), %k3
+# CHECK: vfpclasspsx	$123, -2064(%rdx), %k3
+# CHECK: encoding: [0x62,0xf3,0x7d,0x08,0x66,0x9a,0xf0,0xf7,0xff,0xff,0x7b]
+         vfpclasspsx	$123, -2064(%rdx), %k3
+# CHECK: vfpclassps	$123, 508(%rdx){1to4}, %k3
+# CHECK: encoding: [0x62,0xf3,0x7d,0x18,0x66,0x5a,0x7f,0x7b]
+         vfpclassps	$123, 508(%rdx){1to4}, %k3
+# CHECK: vfpclassps	$123, 508(%rdx){1to4}, %k3
+# CHECK: encoding: [0x62,0xf3,0x7d,0x18,0x66,0x5a,0x7f,0x7b]
+         vfpclassps	$123, 508(%rdx){1to4}, %k3
+# CHECK: vfpclassps	$123, 512(%rdx){1to4}, %k3
+# CHECK: encoding: [0x62,0xf3,0x7d,0x18,0x66,0x9a,0x00,0x02,0x00,0x00,0x7b]
+         vfpclassps	$123, 512(%rdx){1to4}, %k3
+# CHECK: vfpclassps	$123, 512(%rdx){1to4}, %k3
+# CHECK: encoding: [0x62,0xf3,0x7d,0x18,0x66,0x9a,0x00,0x02,0x00,0x00,0x7b]
+         vfpclassps	$123, 512(%rdx){1to4}, %k3
+# CHECK: vfpclassps	$123, -512(%rdx){1to4}, %k3
+# CHECK: encoding: [0x62,0xf3,0x7d,0x18,0x66,0x5a,0x80,0x7b]
+         vfpclassps	$123, -512(%rdx){1to4}, %k3
+# CHECK: vfpclassps	$123, -512(%rdx){1to4}, %k3
+# CHECK: encoding: [0x62,0xf3,0x7d,0x18,0x66,0x5a,0x80,0x7b]
+         vfpclassps	$123, -512(%rdx){1to4}, %k3
+# CHECK: vfpclassps	$123, -516(%rdx){1to4}, %k3
+# CHECK: encoding: [0x62,0xf3,0x7d,0x18,0x66,0x9a,0xfc,0xfd,0xff,0xff,0x7b]
+         vfpclassps	$123, -516(%rdx){1to4}, %k3
+# CHECK: vfpclassps	$123, -516(%rdx){1to4}, %k3
+# CHECK: encoding: [0x62,0xf3,0x7d,0x18,0x66,0x9a,0xfc,0xfd,0xff,0xff,0x7b]
+         vfpclassps	$123, -516(%rdx){1to4}, %k3
+# CHECK: vfpclassps	$171, %ymm19, %k3
+# CHECK: encoding: [0x62,0xb3,0x7d,0x28,0x66,0xdb,0xab]
+         vfpclassps	$171, %ymm19, %k3
+# CHECK: vfpclassps	$171, %ymm19, %k3
+# CHECK: encoding: [0x62,0xb3,0x7d,0x28,0x66,0xdb,0xab]
+         vfpclassps	$171, %ymm19, %k3
+# CHECK: vfpclassps	$171, %ymm19, %k3 {%k3}
+# CHECK: encoding: [0x62,0xb3,0x7d,0x2b,0x66,0xdb,0xab]
+         vfpclassps	$171, %ymm19, %k3 {%k3}
+# CHECK: vfpclassps	$171, %ymm19, %k3 {%k3}
+# CHECK: encoding: [0x62,0xb3,0x7d,0x2b,0x66,0xdb,0xab]
+         vfpclassps	$171, %ymm19, %k3 {%k3}
+# CHECK: vfpclassps	$123, %ymm19, %k3
+# CHECK: encoding: [0x62,0xb3,0x7d,0x28,0x66,0xdb,0x7b]
+         vfpclassps	$123, %ymm19, %k3
+# CHECK: vfpclassps	$123, %ymm19, %k3
+# CHECK: encoding: [0x62,0xb3,0x7d,0x28,0x66,0xdb,0x7b]
+         vfpclassps	$123, %ymm19, %k3
+# CHECK: vfpclasspsy	$123, (%rcx), %k3
+# CHECK: encoding: [0x62,0xf3,0x7d,0x28,0x66,0x19,0x7b]
+         vfpclasspsy	$123, (%rcx), %k3
+# CHECK: vfpclasspsy	$123, 4660(%rax,%r14,8), %k3
+# CHECK: encoding: [0x62,0xb3,0x7d,0x28,0x66,0x9c,0xf0,0x34,0x12,0x00,0x00,0x7b]
+         vfpclasspsy	$123, 4660(%rax,%r14,8), %k3
+# CHECK: vfpclassps	$123, (%rcx){1to8}, %k3
+# CHECK: encoding: [0x62,0xf3,0x7d,0x38,0x66,0x19,0x7b]
+         vfpclassps	$123, (%rcx){1to8}, %k3
+# CHECK: vfpclassps	$123, (%rcx){1to8}, %k3
+# CHECK: encoding: [0x62,0xf3,0x7d,0x38,0x66,0x19,0x7b]
+         vfpclassps	$123, (%rcx){1to8}, %k3
+# CHECK: vfpclasspsy	$123, 4064(%rdx), %k3
+# CHECK: encoding: [0x62,0xf3,0x7d,0x28,0x66,0x5a,0x7f,0x7b]
+         vfpclasspsy	$123, 4064(%rdx), %k3
+# CHECK: vfpclasspsy	$123, 4096(%rdx), %k3
+# CHECK: encoding: [0x62,0xf3,0x7d,0x28,0x66,0x9a,0x00,0x10,0x00,0x00,0x7b]
+         vfpclasspsy	$123, 4096(%rdx), %k3
+# CHECK: vfpclasspsy	$123, -4096(%rdx), %k3
+# CHECK: encoding: [0x62,0xf3,0x7d,0x28,0x66,0x5a,0x80,0x7b]
+         vfpclasspsy	$123, -4096(%rdx), %k3
+# CHECK: vfpclasspsy	$123, -4128(%rdx), %k3
+# CHECK: encoding: [0x62,0xf3,0x7d,0x28,0x66,0x9a,0xe0,0xef,0xff,0xff,0x7b]
+         vfpclasspsy	$123, -4128(%rdx), %k3
+# CHECK: vfpclassps	$123, 508(%rdx){1to8}, %k3
+# CHECK: encoding: [0x62,0xf3,0x7d,0x38,0x66,0x5a,0x7f,0x7b]
+         vfpclassps	$123, 508(%rdx){1to8}, %k3
+# CHECK: vfpclassps	$123, 508(%rdx){1to8}, %k3
+# CHECK: encoding: [0x62,0xf3,0x7d,0x38,0x66,0x5a,0x7f,0x7b]
+         vfpclassps	$123, 508(%rdx){1to8}, %k3
+# CHECK: vfpclassps	$123, 512(%rdx){1to8}, %k3
+# CHECK: encoding: [0x62,0xf3,0x7d,0x38,0x66,0x9a,0x00,0x02,0x00,0x00,0x7b]
+         vfpclassps	$123, 512(%rdx){1to8}, %k3
+# CHECK: vfpclassps	$123, 512(%rdx){1to8}, %k3
+# CHECK: encoding: [0x62,0xf3,0x7d,0x38,0x66,0x9a,0x00,0x02,0x00,0x00,0x7b]
+         vfpclassps	$123, 512(%rdx){1to8}, %k3
+# CHECK: vfpclassps	$123, -512(%rdx){1to8}, %k3
+# CHECK: encoding: [0x62,0xf3,0x7d,0x38,0x66,0x5a,0x80,0x7b]
+         vfpclassps	$123, -512(%rdx){1to8}, %k3
+# CHECK: vfpclassps	$123, -512(%rdx){1to8}, %k3
+# CHECK: encoding: [0x62,0xf3,0x7d,0x38,0x66,0x5a,0x80,0x7b]
+         vfpclassps	$123, -512(%rdx){1to8}, %k3
+# CHECK: vfpclassps	$123, -516(%rdx){1to8}, %k3
+# CHECK: encoding: [0x62,0xf3,0x7d,0x38,0x66,0x9a,0xfc,0xfd,0xff,0xff,0x7b]
+         vfpclassps	$123, -516(%rdx){1to8}, %k3
+# CHECK: vfpclassps	$123, -516(%rdx){1to8}, %k3
+# CHECK: encoding: [0x62,0xf3,0x7d,0x38,0x66,0x9a,0xfc,0xfd,0xff,0xff,0x7b]
+         vfpclassps	$123, -516(%rdx){1to8}, %k3
+# CHECK: vbroadcasti32x2	%xmm30, %xmm30
+# CHECK: encoding: [0x62,0x02,0x7d,0x08,0x59,0xf6]
+         vbroadcasti32x2	%xmm30, %xmm30
+# CHECK: vbroadcasti32x2	%xmm30, %xmm30 {%k2}
+# CHECK: encoding: [0x62,0x02,0x7d,0x0a,0x59,0xf6]
+         vbroadcasti32x2	%xmm30, %xmm30 {%k2}
+# CHECK: vbroadcasti32x2	%xmm30, %xmm30 {%k2} {z}
+# CHECK: encoding: [0x62,0x02,0x7d,0x8a,0x59,0xf6]
+         vbroadcasti32x2	%xmm30, %xmm30 {%k2} {z}
+# CHECK: vbroadcasti32x2	(%rcx), %xmm30
+# CHECK: encoding: [0x62,0x62,0x7d,0x08,0x59,0x31]
+         vbroadcasti32x2	(%rcx), %xmm30
+# CHECK: vbroadcasti32x2	291(%rax,%r14,8), %xmm30
+# CHECK: encoding: [0x62,0x22,0x7d,0x08,0x59,0xb4,0xf0,0x23,0x01,0x00,0x00]
+         vbroadcasti32x2	291(%rax,%r14,8), %xmm30
+# CHECK: vbroadcasti32x2	1016(%rdx), %xmm30
+# CHECK: encoding: [0x62,0x62,0x7d,0x08,0x59,0x72,0x7f]
+         vbroadcasti32x2	1016(%rdx), %xmm30
+# CHECK: vbroadcasti32x2	1024(%rdx), %xmm30
+# CHECK: encoding: [0x62,0x62,0x7d,0x08,0x59,0xb2,0x00,0x04,0x00,0x00]
+         vbroadcasti32x2	1024(%rdx), %xmm30
+# CHECK: vbroadcasti32x2	-1024(%rdx), %xmm30
+# CHECK: encoding: [0x62,0x62,0x7d,0x08,0x59,0x72,0x80]
+         vbroadcasti32x2	-1024(%rdx), %xmm30
+# CHECK: vbroadcasti32x2	-1032(%rdx), %xmm30
+# CHECK: encoding: [0x62,0x62,0x7d,0x08,0x59,0xb2,0xf8,0xfb,0xff,0xff]
+         vbroadcasti32x2	-1032(%rdx), %xmm30
+# CHECK: vbroadcasti32x2	%xmm28, %ymm26
+# CHECK: encoding: [0x62,0x02,0x7d,0x28,0x59,0xd4]
+         vbroadcasti32x2	%xmm28, %ymm26
+# CHECK: vbroadcasti32x2	%xmm28, %ymm26 {%k7}
+# CHECK: encoding: [0x62,0x02,0x7d,0x2f,0x59,0xd4]
+         vbroadcasti32x2	%xmm28, %ymm26 {%k7}
+# CHECK: vbroadcasti32x2	%xmm28, %ymm26 {%k7} {z}
+# CHECK: encoding: [0x62,0x02,0x7d,0xaf,0x59,0xd4]
+         vbroadcasti32x2	%xmm28, %ymm26 {%k7} {z}
+# CHECK: vbroadcasti32x2	(%rcx), %ymm26
+# CHECK: encoding: [0x62,0x62,0x7d,0x28,0x59,0x11]
+         vbroadcasti32x2	(%rcx), %ymm26
+# CHECK: vbroadcasti32x2	291(%rax,%r14,8), %ymm26
+# CHECK: encoding: [0x62,0x22,0x7d,0x28,0x59,0x94,0xf0,0x23,0x01,0x00,0x00]
+         vbroadcasti32x2	291(%rax,%r14,8), %ymm26
+# CHECK: vbroadcasti32x2	1016(%rdx), %ymm26
+# CHECK: encoding: [0x62,0x62,0x7d,0x28,0x59,0x52,0x7f]
+         vbroadcasti32x2	1016(%rdx), %ymm26
+# CHECK: vbroadcasti32x2	1024(%rdx), %ymm26
+# CHECK: encoding: [0x62,0x62,0x7d,0x28,0x59,0x92,0x00,0x04,0x00,0x00]
+         vbroadcasti32x2	1024(%rdx), %ymm26
+# CHECK: vbroadcasti32x2	-1024(%rdx), %ymm26
+# CHECK: encoding: [0x62,0x62,0x7d,0x28,0x59,0x52,0x80]
+         vbroadcasti32x2	-1024(%rdx), %ymm26
+# CHECK: vbroadcasti32x2	-1032(%rdx), %ymm26
+# CHECK: encoding: [0x62,0x62,0x7d,0x28,0x59,0x92,0xf8,0xfb,0xff,0xff]
+         vbroadcasti32x2	-1032(%rdx), %ymm26
+# CHECK: vbroadcasti32x2	%xmm28, %xmm30
+# CHECK: encoding: [0x62,0x02,0x7d,0x08,0x59,0xf4]
+         vbroadcasti32x2	%xmm28, %xmm30
+# CHECK: vbroadcasti32x2	%xmm28, %xmm30 {%k6}
+# CHECK: encoding: [0x62,0x02,0x7d,0x0e,0x59,0xf4]
+         vbroadcasti32x2	%xmm28, %xmm30 {%k6}
+# CHECK: vbroadcasti32x2	%xmm28, %xmm30 {%k6} {z}
+# CHECK: encoding: [0x62,0x02,0x7d,0x8e,0x59,0xf4]
+         vbroadcasti32x2	%xmm28, %xmm30 {%k6} {z}
+# CHECK: vbroadcasti32x2	(%rcx), %xmm30
+# CHECK: encoding: [0x62,0x62,0x7d,0x08,0x59,0x31]
+         vbroadcasti32x2	(%rcx), %xmm30
+# CHECK: vbroadcasti32x2	4660(%rax,%r14,8), %xmm30
+# CHECK: encoding: [0x62,0x22,0x7d,0x08,0x59,0xb4,0xf0,0x34,0x12,0x00,0x00]
+         vbroadcasti32x2	4660(%rax,%r14,8), %xmm30
+# CHECK: vbroadcasti32x2	1016(%rdx), %xmm30
+# CHECK: encoding: [0x62,0x62,0x7d,0x08,0x59,0x72,0x7f]
+         vbroadcasti32x2	1016(%rdx), %xmm30
+# CHECK: vbroadcasti32x2	1024(%rdx), %xmm30
+# CHECK: encoding: [0x62,0x62,0x7d,0x08,0x59,0xb2,0x00,0x04,0x00,0x00]
+         vbroadcasti32x2	1024(%rdx), %xmm30
+# CHECK: vbroadcasti32x2	-1024(%rdx), %xmm30
+# CHECK: encoding: [0x62,0x62,0x7d,0x08,0x59,0x72,0x80]
+         vbroadcasti32x2	-1024(%rdx), %xmm30
+# CHECK: vbroadcasti32x2	-1032(%rdx), %xmm30
+# CHECK: encoding: [0x62,0x62,0x7d,0x08,0x59,0xb2,0xf8,0xfb,0xff,0xff]
+         vbroadcasti32x2	-1032(%rdx), %xmm30
+# CHECK: vbroadcasti32x2	%xmm28, %ymm23
+# CHECK: encoding: [0x62,0x82,0x7d,0x28,0x59,0xfc]
+         vbroadcasti32x2	%xmm28, %ymm23
+# CHECK: vbroadcasti32x2	%xmm28, %ymm23 {%k1}
+# CHECK: encoding: [0x62,0x82,0x7d,0x29,0x59,0xfc]
+         vbroadcasti32x2	%xmm28, %ymm23 {%k1}
+# CHECK: vbroadcasti32x2	%xmm28, %ymm23 {%k1} {z}
+# CHECK: encoding: [0x62,0x82,0x7d,0xa9,0x59,0xfc]
+         vbroadcasti32x2	%xmm28, %ymm23 {%k1} {z}
+# CHECK: vbroadcasti32x2	(%rcx), %ymm23
+# CHECK: encoding: [0x62,0xe2,0x7d,0x28,0x59,0x39]
+         vbroadcasti32x2	(%rcx), %ymm23
+# CHECK: vbroadcasti32x2	4660(%rax,%r14,8), %ymm23
+# CHECK: encoding: [0x62,0xa2,0x7d,0x28,0x59,0xbc,0xf0,0x34,0x12,0x00,0x00]
+         vbroadcasti32x2	4660(%rax,%r14,8), %ymm23
+# CHECK: vbroadcasti32x2	1016(%rdx), %ymm23
+# CHECK: encoding: [0x62,0xe2,0x7d,0x28,0x59,0x7a,0x7f]
+         vbroadcasti32x2	1016(%rdx), %ymm23
+# CHECK: vbroadcasti32x2	1024(%rdx), %ymm23
+# CHECK: encoding: [0x62,0xe2,0x7d,0x28,0x59,0xba,0x00,0x04,0x00,0x00]
+         vbroadcasti32x2	1024(%rdx), %ymm23
+# CHECK: vbroadcasti32x2	-1024(%rdx), %ymm23
+# CHECK: encoding: [0x62,0xe2,0x7d,0x28,0x59,0x7a,0x80]
+         vbroadcasti32x2	-1024(%rdx), %ymm23
+# CHECK: vbroadcasti32x2	-1032(%rdx), %ymm23
+# CHECK: encoding: [0x62,0xe2,0x7d,0x28,0x59,0xba,0xf8,0xfb,0xff,0xff]
+         vbroadcasti32x2	-1032(%rdx), %ymm23
+# CHECK: vbroadcastf32x2	%xmm18, %ymm28
+# CHECK: encoding: [0x62,0x22,0x7d,0x28,0x19,0xe2]
+         vbroadcastf32x2	%xmm18, %ymm28
+# CHECK: vbroadcastf32x2	%xmm18, %ymm28 {%k7}
+# CHECK: encoding: [0x62,0x22,0x7d,0x2f,0x19,0xe2]
+         vbroadcastf32x2	%xmm18, %ymm28 {%k7}
+# CHECK: vbroadcastf32x2	%xmm18, %ymm28 {%k7} {z}
+# CHECK: encoding: [0x62,0x22,0x7d,0xaf,0x19,0xe2]
+         vbroadcastf32x2	%xmm18, %ymm28 {%k7} {z}
+# CHECK: vbroadcastf32x2	(%rcx), %ymm28
+# CHECK: encoding: [0x62,0x62,0x7d,0x28,0x19,0x21]
+         vbroadcastf32x2	(%rcx), %ymm28
+# CHECK: vbroadcastf32x2	291(%rax,%r14,8), %ymm28
+# CHECK: encoding: [0x62,0x22,0x7d,0x28,0x19,0xa4,0xf0,0x23,0x01,0x00,0x00]
+         vbroadcastf32x2	291(%rax,%r14,8), %ymm28
+# CHECK: vbroadcastf32x2	1016(%rdx), %ymm28
+# CHECK: encoding: [0x62,0x62,0x7d,0x28,0x19,0x62,0x7f]
+         vbroadcastf32x2	1016(%rdx), %ymm28
+# CHECK: vbroadcastf32x2	1024(%rdx), %ymm28
+# CHECK: encoding: [0x62,0x62,0x7d,0x28,0x19,0xa2,0x00,0x04,0x00,0x00]
+         vbroadcastf32x2	1024(%rdx), %ymm28
+# CHECK: vbroadcastf32x2	-1024(%rdx), %ymm28
+# CHECK: encoding: [0x62,0x62,0x7d,0x28,0x19,0x62,0x80]
+         vbroadcastf32x2	-1024(%rdx), %ymm28
+# CHECK: vbroadcastf32x2	-1032(%rdx), %ymm28
+# CHECK: encoding: [0x62,0x62,0x7d,0x28,0x19,0xa2,0xf8,0xfb,0xff,0xff]
+         vbroadcastf32x2	-1032(%rdx), %ymm28
+# CHECK: vbroadcastf32x2	%xmm26, %ymm19
+# CHECK: encoding: [0x62,0x82,0x7d,0x28,0x19,0xda]
+         vbroadcastf32x2	%xmm26, %ymm19
+# CHECK: vbroadcastf32x2	%xmm26, %ymm19 {%k7}
+# CHECK: encoding: [0x62,0x82,0x7d,0x2f,0x19,0xda]
+         vbroadcastf32x2	%xmm26, %ymm19 {%k7}
+# CHECK: vbroadcastf32x2	%xmm26, %ymm19 {%k7} {z}
+# CHECK: encoding: [0x62,0x82,0x7d,0xaf,0x19,0xda]
+         vbroadcastf32x2	%xmm26, %ymm19 {%k7} {z}
+# CHECK: vbroadcastf32x2	(%rcx), %ymm19
+# CHECK: encoding: [0x62,0xe2,0x7d,0x28,0x19,0x19]
+         vbroadcastf32x2	(%rcx), %ymm19
+# CHECK: vbroadcastf32x2	4660(%rax,%r14,8), %ymm19
+# CHECK: encoding: [0x62,0xa2,0x7d,0x28,0x19,0x9c,0xf0,0x34,0x12,0x00,0x00]
+         vbroadcastf32x2	4660(%rax,%r14,8), %ymm19
+# CHECK: vbroadcastf32x2	1016(%rdx), %ymm19
+# CHECK: encoding: [0x62,0xe2,0x7d,0x28,0x19,0x5a,0x7f]
+         vbroadcastf32x2	1016(%rdx), %ymm19
+# CHECK: vbroadcastf32x2	1024(%rdx), %ymm19
+# CHECK: encoding: [0x62,0xe2,0x7d,0x28,0x19,0x9a,0x00,0x04,0x00,0x00]
+         vbroadcastf32x2	1024(%rdx), %ymm19
+# CHECK: vbroadcastf32x2	-1024(%rdx), %ymm19
+# CHECK: encoding: [0x62,0xe2,0x7d,0x28,0x19,0x5a,0x80]
+         vbroadcastf32x2	-1024(%rdx), %ymm19
+# CHECK: vbroadcastf32x2	-1032(%rdx), %ymm19
+# CHECK: encoding: [0x62,0xe2,0x7d,0x28,0x19,0x9a,0xf8,0xfb,0xff,0xff]
+         vbroadcastf32x2	-1032(%rdx), %ymm19
+
+# CHECK: vcvtps2qq	128(%rcx), %xmm2 {%k2} {z}
+# CHECK: encoding: [0x62,0xf1,0x7d,0x8a,0x7b,0x51,0x10]
+         vcvtps2qq	128(%rcx), %xmm2 {%k2} {z}
+# CHECK: vcvtps2qq	128(%rcx), %xmm2 {%k2}
+# CHECK: encoding: [0x62,0xf1,0x7d,0x0a,0x7b,0x51,0x10]
+         vcvtps2qq	128(%rcx), %xmm2 {%k2}
+# CHECK: vcvtps2qq	128(%rcx), %xmm2
+# CHECK: encoding: [0x62,0xf1,0x7d,0x08,0x7b,0x51,0x10]
+         vcvtps2qq	128(%rcx), %xmm2
+# CHECK: vcvttps2qq	128(%rcx), %xmm1 {%k2} {z}
+# CHECK: encoding: [0x62,0xf1,0x7d,0x8a,0x7a,0x49,0x10]
+         vcvttps2qq	128(%rcx), %xmm1 {%k2} {z}
+# CHECK: vcvttps2qq	128(%rcx), %xmm1 {%k2}
+# CHECK: encoding: [0x62,0xf1,0x7d,0x0a,0x7a,0x49,0x10]
+         vcvttps2qq	128(%rcx), %xmm1 {%k2}
+# CHECK: vcvttps2qq	128(%rcx), %xmm1
+# CHECK: encoding: [0x62,0xf1,0x7d,0x08,0x7a,0x49,0x10]
+         vcvttps2qq	128(%rcx), %xmm1
+# CHECK: vcvtps2uqq	128(%rcx), %xmm1 {%k2} {z}
+# CHECK: encoding: [0x62,0xf1,0x7d,0x8a,0x79,0x49,0x10]
+         vcvtps2uqq	128(%rcx), %xmm1 {%k2} {z}
+# CHECK: vcvtps2uqq	128(%rcx), %xmm1 {%k2}
+# CHECK: encoding: [0x62,0xf1,0x7d,0x0a,0x79,0x49,0x10]
+         vcvtps2uqq	128(%rcx), %xmm1 {%k2}
+# CHECK: vcvtps2uqq	128(%rcx), %xmm1
+# CHECK: encoding: [0x62,0xf1,0x7d,0x08,0x79,0x49,0x10]
+         vcvtps2uqq	128(%rcx), %xmm1
+# CHECK: vcvttps2uqq	128(%rcx), %xmm1 {%k2} {z}
+# CHECK: encoding: [0x62,0xf1,0x7d,0x8a,0x78,0x49,0x10]
+         vcvttps2uqq	128(%rcx), %xmm1 {%k2} {z}
+# CHECK: vcvttps2uqq	128(%rcx), %xmm1 {%k2}
+# CHECK: encoding: [0x62,0xf1,0x7d,0x0a,0x78,0x49,0x10]
+         vcvttps2uqq	128(%rcx), %xmm1 {%k2}
+# CHECK: vcvttps2uqq	128(%rcx), %xmm1
+# CHECK: encoding: [0x62,0xf1,0x7d,0x08,0x78,0x49,0x10]
+         vcvttps2uqq	128(%rcx), %xmm1
+# CHECK: vcvtps2qq	128(%rcx), %xmm2 {%k2} {z}
+# CHECK: encoding: [0x62,0xf1,0x7d,0x8a,0x7b,0x51,0x10]
+         vcvtps2qq	128(%rcx), %xmm2 {%k2} {z}
+# CHECK: vcvtps2qq	128(%rcx), %xmm2 {%k2}
+# CHECK: encoding: [0x62,0xf1,0x7d,0x0a,0x7b,0x51,0x10]
+         vcvtps2qq	128(%rcx), %xmm2 {%k2}
+# CHECK: vcvtps2qq	128(%rcx), %xmm2
+# CHECK: encoding: [0x62,0xf1,0x7d,0x08,0x7b,0x51,0x10]
+         vcvtps2qq	128(%rcx), %xmm2
+# CHECK: vcvttps2qq	128(%rcx), %xmm1 {%k2} {z}
+# CHECK: encoding: [0x62,0xf1,0x7d,0x8a,0x7a,0x49,0x10]
+         vcvttps2qq	128(%rcx), %xmm1 {%k2} {z}
+# CHECK: vcvttps2qq	128(%rcx), %xmm1 {%k2}
+# CHECK: encoding: [0x62,0xf1,0x7d,0x0a,0x7a,0x49,0x10]
+         vcvttps2qq	128(%rcx), %xmm1 {%k2}
+# CHECK: vcvttps2qq	128(%rcx), %xmm1
+# CHECK: encoding: [0x62,0xf1,0x7d,0x08,0x7a,0x49,0x10]
+         vcvttps2qq	128(%rcx), %xmm1
+# CHECK: vcvtps2uqq	128(%rcx), %xmm1 {%k2} {z}
+# CHECK: encoding: [0x62,0xf1,0x7d,0x8a,0x79,0x49,0x10]
+         vcvtps2uqq	128(%rcx), %xmm1 {%k2} {z}
+# CHECK: vcvtps2uqq	128(%rcx), %xmm1 {%k2}
+# CHECK: encoding: [0x62,0xf1,0x7d,0x0a,0x79,0x49,0x10]
+         vcvtps2uqq	128(%rcx), %xmm1 {%k2}
+# CHECK: vcvtps2uqq	128(%rcx), %xmm1
+# CHECK: encoding: [0x62,0xf1,0x7d,0x08,0x79,0x49,0x10]
+         vcvtps2uqq	128(%rcx), %xmm1
+# CHECK: vcvttps2uqq	128(%rcx), %xmm1 {%k2} {z}
+# CHECK: encoding: [0x62,0xf1,0x7d,0x8a,0x78,0x49,0x10]
+         vcvttps2uqq	128(%rcx), %xmm1 {%k2} {z}
+# CHECK: vcvttps2uqq	128(%rcx), %xmm1 {%k2}
+# CHECK: encoding: [0x62,0xf1,0x7d,0x0a,0x78,0x49,0x10]
+         vcvttps2uqq	128(%rcx), %xmm1 {%k2}
+# CHECK: vcvttps2uqq	128(%rcx), %xmm1
+# CHECK: encoding: [0x62,0xf1,0x7d,0x08,0x78,0x49,0x10]
+         vcvttps2uqq	128(%rcx), %xmm1
+# CHECK: vfpclasspd	$171, %xmm18, %k2
+# CHECK: encoding: [0x62,0xb3,0xfd,0x08,0x66,0xd2,0xab]
+         vfpclasspd	$171, %xmm18, %k2
+# CHECK: vfpclasspd	$171, %xmm18, %k2 {%k7}
+# CHECK: encoding: [0x62,0xb3,0xfd,0x0f,0x66,0xd2,0xab]
+         vfpclasspd	$171, %xmm18, %k2 {%k7}
+# CHECK: vfpclasspdx	$123, (%rcx), %k2
+# CHECK: encoding: [0x62,0xf3,0xfd,0x08,0x66,0x11,0x7b]
+         vfpclasspdx	$123, (%rcx), %k2
+# CHECK: vfpclasspdx	$123, (%rcx), %k2 {%k7}
+# CHECK: encoding: [0x62,0xf3,0xfd,0x0f,0x66,0x11,0x7b]
+         vfpclasspdx	$123, (%rcx), %k2 {%k7}
+# CHECK: vfpclasspd	$123, (%rcx){1to2}, %k2
+# CHECK: encoding: [0x62,0xf3,0xfd,0x18,0x66,0x11,0x7b]
+         vfpclasspd	$123, (%rcx){1to2}, %k2
+# CHECK: vfpclasspd	$123, (%rcx){1to2}, %k2 {%k7}
+# CHECK: encoding: [0x62,0xf3,0xfd,0x1f,0x66,0x11,0x7b]
+         vfpclasspd	$123, (%rcx){1to2}, %k2 {%k7}
+# CHECK: vfpclassps	$171, %xmm18, %k2
+# CHECK: encoding: [0x62,0xb3,0x7d,0x08,0x66,0xd2,0xab]
+         vfpclassps	$171, %xmm18, %k2
+# CHECK: vfpclassps	$171, %xmm18, %k2 {%k7}
+# CHECK: encoding: [0x62,0xb3,0x7d,0x0f,0x66,0xd2,0xab]
+         vfpclassps	$171, %xmm18, %k2 {%k7}
+# CHECK: vfpclasspsx	$123, (%rcx), %k2
+# CHECK: encoding: [0x62,0xf3,0x7d,0x08,0x66,0x11,0x7b]
+         vfpclasspsx	$123, (%rcx), %k2
+# CHECK: vfpclasspsx	$123, (%rcx), %k2 {%k7}
+# CHECK: encoding: [0x62,0xf3,0x7d,0x0f,0x66,0x11,0x7b]
+         vfpclasspsx	$123, (%rcx), %k2 {%k7}
+# CHECK: vfpclassps	$123, (%rcx){1to4}, %k2
+# CHECK: encoding: [0x62,0xf3,0x7d,0x18,0x66,0x11,0x7b]
+         vfpclassps	$123, (%rcx){1to4}, %k2
+# CHECK: vfpclassps	$123, (%rcx){1to4}, %k2 {%k7}
+# CHECK: encoding: [0x62,0xf3,0x7d,0x1f,0x66,0x11,0x7b]
+         vfpclassps	$123, (%rcx){1to4}, %k2 {%k7}
+# CHECK: vfpclasspd	$171, %ymm18, %k2
+# CHECK: encoding: [0x62,0xb3,0xfd,0x28,0x66,0xd2,0xab]
+         vfpclasspd	$171, %ymm18, %k2
+# CHECK: vfpclasspd	$171, %ymm18, %k2 {%k7}
+# CHECK: encoding: [0x62,0xb3,0xfd,0x2f,0x66,0xd2,0xab]
+         vfpclasspd	$171, %ymm18, %k2 {%k7}
+# CHECK: vfpclasspdy	$123, (%rcx), %k2
+# CHECK: encoding: [0x62,0xf3,0xfd,0x28,0x66,0x11,0x7b]
+         vfpclasspdy	$123, (%rcx), %k2
+# CHECK: vfpclasspdy	$123, (%rcx), %k2 {%k7}
+# CHECK: encoding: [0x62,0xf3,0xfd,0x2f,0x66,0x11,0x7b]
+         vfpclasspdy	$123, (%rcx), %k2 {%k7}
+# CHECK: vfpclasspd	$123, (%rcx){1to4}, %k2
+# CHECK: encoding: [0x62,0xf3,0xfd,0x38,0x66,0x11,0x7b]
+         vfpclasspd	$123, (%rcx){1to4}, %k2
+# CHECK: vfpclasspd	$123, (%rcx){1to4}, %k2 {%k7}
+# CHECK: encoding: [0x62,0xf3,0xfd,0x3f,0x66,0x11,0x7b]
+         vfpclasspd	$123, (%rcx){1to4}, %k2 {%k7}
+# CHECK: vfpclassps	$171, %ymm18, %k2
+# CHECK: encoding: [0x62,0xb3,0x7d,0x28,0x66,0xd2,0xab]
+         vfpclassps	$171, %ymm18, %k2
+# CHECK: vfpclassps	$171, %ymm18, %k2 {%k7}
+# CHECK: encoding: [0x62,0xb3,0x7d,0x2f,0x66,0xd2,0xab]
+         vfpclassps	$171, %ymm18, %k2 {%k7}
+# CHECK: vfpclasspsy	$123, (%rcx), %k2
+# CHECK: encoding: [0x62,0xf3,0x7d,0x28,0x66,0x11,0x7b]
+         vfpclasspsy	$123, (%rcx), %k2
+# CHECK: vfpclasspsy	$123, (%rcx), %k2 {%k7}
+# CHECK: encoding: [0x62,0xf3,0x7d,0x2f,0x66,0x11,0x7b]
+         vfpclasspsy	$123, (%rcx), %k2 {%k7}
+# CHECK: vfpclassps	$123, (%rcx){1to8}, %k2
+# CHECK: encoding: [0x62,0xf3,0x7d,0x38,0x66,0x11,0x7b]
+         vfpclassps	$123, (%rcx){1to8}, %k2
+# CHECK: vfpclassps	$123, (%rcx){1to8}, %k2 {%k7}
+# CHECK: encoding: [0x62,0xf3,0x7d,0x3f,0x66,0x11,0x7b]
+         vfpclassps	$123, (%rcx){1to8}, %k2 {%k7}
+# CHECK: vcvttps2uqq     128(%ecx), %xmm1 {%k2}
+# CHECK: encoding: [0x67,0x62,0xf1,0x7d,0x0a,0x78,0x49,0x10]
+         vcvttps2uqq     128(%ecx), %xmm1 {%k2}
diff --git a/llvm/test/MC/X86/avx512dq_vl-intel.s b/llvm/test/MC/X86/avx512dq_vl-intel.s
new file mode 100644
index 0000000000000000000000000000000000000000..71e9fc41367714ec1299de2f6b0a91986688698d
--- /dev/null
+++ b/llvm/test/MC/X86/avx512dq_vl-intel.s
@@ -0,0 +1,149 @@
+# RUN: llvm-mc -triple x86_64 -show-encoding -x86-asm-syntax=intel -output-asm-variant=1 %s | FileCheck %s
+
+# CHECK: vcvtps2qq	xmm2 {k2} {z}, qword ptr [rcx + 128]
+# CHECK: encoding: [0x62,0xf1,0x7d,0x8a,0x7b,0x51,0x10]
+         vcvtps2qq	xmm2 {k2} {z}, qword ptr [rcx + 128]
+# CHECK: vcvtps2qq	xmm2 {k2}, qword ptr [rcx + 128]
+# CHECK: encoding: [0x62,0xf1,0x7d,0x0a,0x7b,0x51,0x10]
+         vcvtps2qq	xmm2 {k2}, qword ptr [rcx + 128]
+# CHECK: vcvtps2qq	xmm2, qword ptr [rcx + 128]
+# CHECK: encoding: [0x62,0xf1,0x7d,0x08,0x7b,0x51,0x10]
+         vcvtps2qq	xmm2, qword ptr [rcx + 128]
+# CHECK: vcvttps2qq	xmm1 {k2} {z}, qword ptr [rcx + 128]
+# CHECK: encoding: [0x62,0xf1,0x7d,0x8a,0x7a,0x49,0x10]
+         vcvttps2qq	xmm1 {k2} {z}, qword ptr [rcx + 128]
+# CHECK: vcvttps2qq	xmm1 {k2}, qword ptr [rcx + 128]
+# CHECK: encoding: [0x62,0xf1,0x7d,0x0a,0x7a,0x49,0x10]
+         vcvttps2qq	xmm1 {k2}, qword ptr [rcx + 128]
+# CHECK: vcvttps2qq	xmm1, qword ptr [rcx + 128]
+# CHECK: encoding: [0x62,0xf1,0x7d,0x08,0x7a,0x49,0x10]
+         vcvttps2qq	xmm1, qword ptr [rcx + 128]
+# CHECK: vcvtps2uqq	xmm1 {k2} {z}, qword ptr [rcx + 128]
+# CHECK: encoding: [0x62,0xf1,0x7d,0x8a,0x79,0x49,0x10]
+         vcvtps2uqq	xmm1 {k2} {z}, qword ptr [rcx + 128]
+# CHECK: vcvtps2uqq	xmm1 {k2}, qword ptr [rcx + 128]
+# CHECK: encoding: [0x62,0xf1,0x7d,0x0a,0x79,0x49,0x10]
+         vcvtps2uqq	xmm1 {k2}, qword ptr [rcx + 128]
+# CHECK: vcvtps2uqq	xmm1, qword ptr [rcx + 128]
+# CHECK: encoding: [0x62,0xf1,0x7d,0x08,0x79,0x49,0x10]
+         vcvtps2uqq	xmm1, qword ptr [rcx + 128]
+# CHECK: vcvttps2uqq	xmm1 {k2} {z}, qword ptr [rcx + 128]
+# CHECK: encoding: [0x62,0xf1,0x7d,0x8a,0x78,0x49,0x10]
+         vcvttps2uqq	xmm1 {k2} {z}, qword ptr [rcx + 128]
+# CHECK: vcvttps2uqq	xmm1 {k2}, qword ptr [rcx + 128]
+# CHECK: encoding: [0x62,0xf1,0x7d,0x0a,0x78,0x49,0x10]
+         vcvttps2uqq	xmm1 {k2}, qword ptr [rcx + 128]
+# CHECK: vcvttps2uqq	xmm1, qword ptr [rcx + 128]
+# CHECK: encoding: [0x62,0xf1,0x7d,0x08,0x78,0x49,0x10]
+         vcvttps2uqq	xmm1, qword ptr [rcx + 128]
+# CHECK: vcvtps2qq	xmm2 {k2} {z}, qword ptr [rcx + 128]
+# CHECK: encoding: [0x62,0xf1,0x7d,0x8a,0x7b,0x51,0x10]
+         vcvtps2qq	xmm2 {k2} {z}, qword ptr [rcx + 128]
+# CHECK: vcvtps2qq	xmm2 {k2}, qword ptr [rcx + 128]
+# CHECK: encoding: [0x62,0xf1,0x7d,0x0a,0x7b,0x51,0x10]
+         vcvtps2qq	xmm2 {k2}, qword ptr [rcx + 128]
+# CHECK: vcvtps2qq	xmm2, qword ptr [rcx + 128]
+# CHECK: encoding: [0x62,0xf1,0x7d,0x08,0x7b,0x51,0x10]
+         vcvtps2qq	xmm2, qword ptr [rcx + 128]
+# CHECK: vcvttps2qq	xmm1 {k2} {z}, qword ptr [rcx + 128]
+# CHECK: encoding: [0x62,0xf1,0x7d,0x8a,0x7a,0x49,0x10]
+         vcvttps2qq	xmm1 {k2} {z}, qword ptr [rcx + 128]
+# CHECK: vcvttps2qq	xmm1 {k2}, qword ptr [rcx + 128]
+# CHECK: encoding: [0x62,0xf1,0x7d,0x0a,0x7a,0x49,0x10]
+         vcvttps2qq	xmm1 {k2}, qword ptr [rcx + 128]
+# CHECK: vcvttps2qq	xmm1, qword ptr [rcx + 128]
+# CHECK: encoding: [0x62,0xf1,0x7d,0x08,0x7a,0x49,0x10]
+         vcvttps2qq	xmm1, qword ptr [rcx + 128]
+# CHECK: vcvtps2uqq	xmm1 {k2} {z}, qword ptr [rcx + 128]
+# CHECK: encoding: [0x62,0xf1,0x7d,0x8a,0x79,0x49,0x10]
+         vcvtps2uqq	xmm1 {k2} {z}, qword ptr [rcx + 128]
+# CHECK: vcvtps2uqq	xmm1 {k2}, qword ptr [rcx + 128]
+# CHECK: encoding: [0x62,0xf1,0x7d,0x0a,0x79,0x49,0x10]
+         vcvtps2uqq	xmm1 {k2}, qword ptr [rcx + 128]
+# CHECK: vcvtps2uqq	xmm1, qword ptr [rcx + 128]
+# CHECK: encoding: [0x62,0xf1,0x7d,0x08,0x79,0x49,0x10]
+         vcvtps2uqq	xmm1, qword ptr [rcx + 128]
+# CHECK: vcvttps2uqq	xmm1 {k2} {z}, qword ptr [rcx + 128]
+# CHECK: encoding: [0x62,0xf1,0x7d,0x8a,0x78,0x49,0x10]
+         vcvttps2uqq	xmm1 {k2} {z}, qword ptr [rcx + 128]
+# CHECK: vcvttps2uqq	xmm1 {k2}, qword ptr [rcx + 128]
+# CHECK: encoding: [0x62,0xf1,0x7d,0x0a,0x78,0x49,0x10]
+         vcvttps2uqq	xmm1 {k2}, qword ptr [rcx + 128]
+# CHECK: vcvttps2uqq	xmm1, qword ptr [rcx + 128]
+# CHECK: encoding: [0x62,0xf1,0x7d,0x08,0x78,0x49,0x10]
+         vcvttps2uqq	xmm1, qword ptr [rcx + 128]
+# CHECK: vfpclasspd	k2, xmm18, 171
+# CHECK: encoding: [0x62,0xb3,0xfd,0x08,0x66,0xd2,0xab]
+         vfpclasspd	k2, xmm18, 171
+# CHECK: vfpclasspd	k2 {k7}, xmm18, 171
+# CHECK: encoding: [0x62,0xb3,0xfd,0x0f,0x66,0xd2,0xab]
+         vfpclasspd	k2 {k7}, xmm18, 171
+# CHECK: vfpclasspd	k2, xmmword ptr [rcx], 123
+# CHECK: encoding: [0x62,0xf3,0xfd,0x08,0x66,0x11,0x7b]
+         vfpclasspd	k2, xmmword ptr [rcx], 123
+# CHECK: vfpclasspd	k2 {k7}, xmmword ptr [rcx], 123
+# CHECK: encoding: [0x62,0xf3,0xfd,0x0f,0x66,0x11,0x7b]
+         vfpclasspd	k2 {k7}, xmmword ptr [rcx], 123
+# CHECK: vfpclasspd	k2, qword ptr [rcx]{1to2}, 123
+# CHECK: encoding: [0x62,0xf3,0xfd,0x18,0x66,0x11,0x7b]
+         vfpclasspd	k2, qword ptr [rcx]{1to2}, 123
+# CHECK: vfpclasspd	k2 {k7}, qword ptr [rcx]{1to2}, 123
+# CHECK: encoding: [0x62,0xf3,0xfd,0x1f,0x66,0x11,0x7b]
+         vfpclasspd	k2 {k7}, qword ptr [rcx]{1to2}, 123
+# CHECK: vfpclassps	k2, xmm18, 171
+# CHECK: encoding: [0x62,0xb3,0x7d,0x08,0x66,0xd2,0xab]
+         vfpclassps	k2, xmm18, 171
+# CHECK: vfpclassps	k2 {k7}, xmm18, 171
+# CHECK: encoding: [0x62,0xb3,0x7d,0x0f,0x66,0xd2,0xab]
+         vfpclassps	k2 {k7}, xmm18, 171
+# CHECK: vfpclassps	k2, xmmword ptr [rcx], 123
+# CHECK: encoding: [0x62,0xf3,0x7d,0x08,0x66,0x11,0x7b]
+         vfpclassps	k2, xmmword ptr [rcx], 123
+# CHECK: vfpclassps	k2 {k7}, xmmword ptr [rcx], 123
+# CHECK: encoding: [0x62,0xf3,0x7d,0x0f,0x66,0x11,0x7b]
+         vfpclassps	k2 {k7}, xmmword ptr [rcx], 123
+# CHECK: vfpclassps	k2, dword ptr [rcx]{1to4}, 123
+# CHECK: encoding: [0x62,0xf3,0x7d,0x18,0x66,0x11,0x7b]
+         vfpclassps	k2, dword ptr [rcx]{1to4}, 123
+# CHECK: vfpclassps	k2 {k7}, dword ptr [rcx]{1to4}, 123
+# CHECK: encoding: [0x62,0xf3,0x7d,0x1f,0x66,0x11,0x7b]
+         vfpclassps	k2 {k7}, dword ptr [rcx]{1to4}, 123
+# CHECK: vfpclasspd	k2, ymm18, 171
+# CHECK: encoding: [0x62,0xb3,0xfd,0x28,0x66,0xd2,0xab]
+         vfpclasspd	k2, ymm18, 171
+# CHECK: vfpclasspd	k2 {k7}, ymm18, 171
+# CHECK: encoding: [0x62,0xb3,0xfd,0x2f,0x66,0xd2,0xab]
+         vfpclasspd	k2 {k7}, ymm18, 171
+# CHECK: vfpclasspd	k2, ymmword ptr [rcx], 123
+# CHECK: encoding: [0x62,0xf3,0xfd,0x28,0x66,0x11,0x7b]
+         vfpclasspd	k2, ymmword ptr [rcx], 123
+# CHECK: vfpclasspd	k2 {k7}, ymmword ptr [rcx], 123
+# CHECK: encoding: [0x62,0xf3,0xfd,0x2f,0x66,0x11,0x7b]
+         vfpclasspd	k2 {k7}, ymmword ptr [rcx], 123
+# CHECK: vfpclasspd	k2, qword ptr [rcx]{1to4}, 123
+# CHECK: encoding: [0x62,0xf3,0xfd,0x38,0x66,0x11,0x7b]
+         vfpclasspd	k2, qword ptr [rcx]{1to4}, 123
+# CHECK: vfpclasspd	k2 {k7}, qword ptr [rcx]{1to4}, 123
+# CHECK: encoding: [0x62,0xf3,0xfd,0x3f,0x66,0x11,0x7b]
+         vfpclasspd	k2 {k7}, qword ptr [rcx]{1to4}, 123
+# CHECK: vfpclassps	k2, ymm18, 171
+# CHECK: encoding: [0x62,0xb3,0x7d,0x28,0x66,0xd2,0xab]
+         vfpclassps	k2, ymm18, 171
+# CHECK: vfpclassps	k2 {k7}, ymm18, 171
+# CHECK: encoding: [0x62,0xb3,0x7d,0x2f,0x66,0xd2,0xab]
+         vfpclassps	k2 {k7}, ymm18, 171
+# CHECK: vfpclassps	k2, ymmword ptr [rcx], 123
+# CHECK: encoding: [0x62,0xf3,0x7d,0x28,0x66,0x11,0x7b]
+         vfpclassps	k2, ymmword ptr [rcx], 123
+# CHECK: vfpclassps	k2 {k7}, ymmword ptr [rcx], 123
+# CHECK: encoding: [0x62,0xf3,0x7d,0x2f,0x66,0x11,0x7b]
+         vfpclassps	k2 {k7}, ymmword ptr [rcx], 123
+# CHECK: vfpclassps	k2, dword ptr [rcx]{1to8}, 123
+# CHECK: encoding: [0x62,0xf3,0x7d,0x38,0x66,0x11,0x7b]
+         vfpclassps	k2, dword ptr [rcx]{1to8}, 123
+# CHECK: vfpclassps	k2 {k7}, dword ptr [rcx]{1to8}, 123
+# CHECK: encoding: [0x62,0xf3,0x7d,0x3f,0x66,0x11,0x7b]
+         vfpclassps	k2 {k7}, dword ptr [rcx]{1to8}, 123
+# CHECK: vcvttps2uqq     xmm1 {k2}, qword ptr [ecx + 128]
+# CHECK: encoding: [0x67,0x62,0xf1,0x7d,0x0a,0x78,0x49,0x10]
+         vcvttps2uqq     xmm1 {k2}, qword ptr [ecx + 128]
diff --git a/llvm/test/MC/X86/x86-64-avx512f_vl.s b/llvm/test/MC/X86/avx512f_vl-att.s
similarity index 100%
rename from llvm/test/MC/X86/x86-64-avx512f_vl.s
rename to llvm/test/MC/X86/avx512f_vl-att.s
diff --git a/llvm/test/MC/X86/intel-syntax-x86-64-avx512f_vl.s b/llvm/test/MC/X86/avx512f_vl-intel.s
similarity index 100%
rename from llvm/test/MC/X86/intel-syntax-x86-64-avx512f_vl.s
rename to llvm/test/MC/X86/avx512f_vl-intel.s
diff --git a/llvm/test/MC/X86/avx512fp16.s b/llvm/test/MC/X86/avx512fp16-att.s
similarity index 100%
rename from llvm/test/MC/X86/avx512fp16.s
rename to llvm/test/MC/X86/avx512fp16-att.s
diff --git a/llvm/test/MC/X86/intel-syntax-avx512fp16.s b/llvm/test/MC/X86/avx512fp16-intel.s
similarity index 100%
rename from llvm/test/MC/X86/intel-syntax-avx512fp16.s
rename to llvm/test/MC/X86/avx512fp16-intel.s
diff --git a/llvm/test/MC/X86/avx512fp16vl.s b/llvm/test/MC/X86/avx512fp16vl-att.s
similarity index 100%
rename from llvm/test/MC/X86/avx512fp16vl.s
rename to llvm/test/MC/X86/avx512fp16vl-att.s
diff --git a/llvm/test/MC/X86/intel-syntax-avx512fp16vl.s b/llvm/test/MC/X86/avx512fp16vl-intel.s
similarity index 100%
rename from llvm/test/MC/X86/intel-syntax-avx512fp16vl.s
rename to llvm/test/MC/X86/avx512fp16vl-intel.s
diff --git a/llvm/test/MC/X86/avx512gfni-encoding.s b/llvm/test/MC/X86/avx512gfni-att.s
similarity index 100%
rename from llvm/test/MC/X86/avx512gfni-encoding.s
rename to llvm/test/MC/X86/avx512gfni-att.s
diff --git a/llvm/test/MC/X86/avx512ifma-encoding.s b/llvm/test/MC/X86/avx512ifma-att.s
similarity index 100%
rename from llvm/test/MC/X86/avx512ifma-encoding.s
rename to llvm/test/MC/X86/avx512ifma-att.s
diff --git a/llvm/test/MC/X86/avx512ifmavl-encoding.s b/llvm/test/MC/X86/avx512ifmavl-att.s
similarity index 100%
rename from llvm/test/MC/X86/avx512ifmavl-encoding.s
rename to llvm/test/MC/X86/avx512ifmavl-att.s
diff --git a/llvm/test/MC/X86/x86-64-avx512pf.s b/llvm/test/MC/X86/avx512pf-64-att.s
similarity index 100%
rename from llvm/test/MC/X86/x86-64-avx512pf.s
rename to llvm/test/MC/X86/avx512pf-64-att.s
diff --git a/llvm/test/MC/X86/avx512vaes-encoding.s b/llvm/test/MC/X86/avx512vaes-att.s
similarity index 100%
rename from llvm/test/MC/X86/avx512vaes-encoding.s
rename to llvm/test/MC/X86/avx512vaes-att.s
diff --git a/llvm/test/MC/X86/avx512vbmi-att.s b/llvm/test/MC/X86/avx512vbmi-att.s
new file mode 100644
index 0000000000000000000000000000000000000000..028494514bfcb0eba184996126495066585e14fb
--- /dev/null
+++ b/llvm/test/MC/X86/avx512vbmi-att.s
@@ -0,0 +1,542 @@
+# RUN: llvm-mc -triple x86_64 -show-encoding %s | FileCheck %s
+
+# CHECK: vpermb	%xmm28, %xmm29, %xmm30 {%k7}
+# CHECK: encoding: [0x62,0x02,0x15,0x07,0x8d,0xf4]
+         vpermb	%xmm28, %xmm29, %xmm30 {%k7}
+
+# CHECK: vpermb	%xmm28, %xmm29, %xmm30 {%k7} {z}
+# CHECK: encoding: [0x62,0x02,0x15,0x87,0x8d,0xf4]
+         vpermb	%xmm28, %xmm29, %xmm30 {%k7} {z}
+
+# CHECK: vpermb	(%rcx), %xmm29, %xmm30
+# CHECK: encoding: [0x62,0x62,0x15,0x00,0x8d,0x31]
+         vpermb	(%rcx), %xmm29, %xmm30
+
+# CHECK: vpermb	291(%rax,%r14,8), %xmm29, %xmm30
+# CHECK: encoding: [0x62,0x22,0x15,0x00,0x8d,0xb4,0xf0,0x23,0x01,0x00,0x00]
+         vpermb	291(%rax,%r14,8), %xmm29, %xmm30
+
+# CHECK: vpermb	2032(%rdx), %xmm29, %xmm30
+# CHECK: encoding: [0x62,0x62,0x15,0x00,0x8d,0x72,0x7f]
+         vpermb	2032(%rdx), %xmm29, %xmm30
+
+# CHECK: vpermb	2048(%rdx), %xmm29, %xmm30
+# CHECK: encoding: [0x62,0x62,0x15,0x00,0x8d,0xb2,0x00,0x08,0x00,0x00]
+         vpermb	2048(%rdx), %xmm29, %xmm30
+
+# CHECK: vpermb	-2048(%rdx), %xmm29, %xmm30
+# CHECK: encoding: [0x62,0x62,0x15,0x00,0x8d,0x72,0x80]
+         vpermb	-2048(%rdx), %xmm29, %xmm30
+
+# CHECK: vpermb	-2064(%rdx), %xmm29, %xmm30
+# CHECK: encoding: [0x62,0x62,0x15,0x00,0x8d,0xb2,0xf0,0xf7,0xff,0xff]
+         vpermb	-2064(%rdx), %xmm29, %xmm30
+
+# CHECK: vpermb	%ymm28, %ymm29, %ymm30
+# CHECK: encoding: [0x62,0x02,0x15,0x20,0x8d,0xf4]
+         vpermb	%ymm28, %ymm29, %ymm30
+
+# CHECK: vpermb	%ymm28, %ymm29, %ymm30 {%k7}
+# CHECK: encoding: [0x62,0x02,0x15,0x27,0x8d,0xf4]
+         vpermb	%ymm28, %ymm29, %ymm30 {%k7}
+
+# CHECK: vpermb	%ymm28, %ymm29, %ymm30 {%k7} {z}
+# CHECK: encoding: [0x62,0x02,0x15,0xa7,0x8d,0xf4]
+         vpermb	%ymm28, %ymm29, %ymm30 {%k7} {z}
+
+# CHECK: vpermb	(%rcx), %ymm29, %ymm30
+# CHECK: encoding: [0x62,0x62,0x15,0x20,0x8d,0x31]
+         vpermb	(%rcx), %ymm29, %ymm30
+
+# CHECK: vpermb	291(%rax,%r14,8), %ymm29, %ymm30
+# CHECK: encoding: [0x62,0x22,0x15,0x20,0x8d,0xb4,0xf0,0x23,0x01,0x00,0x00]
+         vpermb	291(%rax,%r14,8), %ymm29, %ymm30
+
+# CHECK: vpermb	4064(%rdx), %ymm29, %ymm30
+# CHECK: encoding: [0x62,0x62,0x15,0x20,0x8d,0x72,0x7f]
+         vpermb	4064(%rdx), %ymm29, %ymm30
+
+# CHECK: vpermb	4096(%rdx), %ymm29, %ymm30
+# CHECK: encoding: [0x62,0x62,0x15,0x20,0x8d,0xb2,0x00,0x10,0x00,0x00]
+         vpermb	4096(%rdx), %ymm29, %ymm30
+
+# CHECK: vpermb	-4096(%rdx), %ymm29, %ymm30
+# CHECK: encoding: [0x62,0x62,0x15,0x20,0x8d,0x72,0x80]
+         vpermb	-4096(%rdx), %ymm29, %ymm30
+
+# CHECK: vpermb	-4128(%rdx), %ymm29, %ymm30
+# CHECK: encoding: [0x62,0x62,0x15,0x20,0x8d,0xb2,0xe0,0xef,0xff,0xff]
+         vpermb	-4128(%rdx), %ymm29, %ymm30
+
+# CHECK: vpermb	%xmm28, %xmm29, %xmm30
+# CHECK: encoding: [0x62,0x02,0x15,0x00,0x8d,0xf4]
+         vpermb	%xmm28, %xmm29, %xmm30
+
+# CHECK: vpermb	4660(%rax,%r14,8), %xmm29, %xmm30
+# CHECK: encoding: [0x62,0x22,0x15,0x00,0x8d,0xb4,0xf0,0x34,0x12,0x00,0x00]
+         vpermb	4660(%rax,%r14,8), %xmm29, %xmm30
+
+# CHECK: vpermb	4660(%rax,%r14,8), %ymm29, %ymm30
+# CHECK: encoding: [0x62,0x22,0x15,0x20,0x8d,0xb4,0xf0,0x34,0x12,0x00,0x00]
+         vpermb	4660(%rax,%r14,8), %ymm29, %ymm30
+
+# CHECK: vpermb	%zmm28, %zmm29, %zmm30
+# CHECK: encoding: [0x62,0x02,0x15,0x40,0x8d,0xf4]
+         vpermb	%zmm28, %zmm29, %zmm30
+
+# CHECK: vpermb	%zmm28, %zmm29, %zmm30 {%k7}
+# CHECK: encoding: [0x62,0x02,0x15,0x47,0x8d,0xf4]
+         vpermb	%zmm28, %zmm29, %zmm30 {%k7}
+
+# CHECK: vpermb	%zmm28, %zmm29, %zmm30 {%k7} {z}
+# CHECK: encoding: [0x62,0x02,0x15,0xc7,0x8d,0xf4]
+         vpermb	%zmm28, %zmm29, %zmm30 {%k7} {z}
+
+# CHECK: vpermb	(%rcx), %zmm29, %zmm30
+# CHECK: encoding: [0x62,0x62,0x15,0x40,0x8d,0x31]
+         vpermb	(%rcx), %zmm29, %zmm30
+
+# CHECK: vpermb	291(%rax,%r14,8), %zmm29, %zmm30
+# CHECK: encoding: [0x62,0x22,0x15,0x40,0x8d,0xb4,0xf0,0x23,0x01,0x00,0x00]
+         vpermb	291(%rax,%r14,8), %zmm29, %zmm30
+
+# CHECK: vpermb	8128(%rdx), %zmm29, %zmm30
+# CHECK: encoding: [0x62,0x62,0x15,0x40,0x8d,0x72,0x7f]
+         vpermb	8128(%rdx), %zmm29, %zmm30
+
+# CHECK: vpermb	8192(%rdx), %zmm29, %zmm30
+# CHECK: encoding: [0x62,0x62,0x15,0x40,0x8d,0xb2,0x00,0x20,0x00,0x00]
+         vpermb	8192(%rdx), %zmm29, %zmm30
+
+# CHECK: vpermb	-8192(%rdx), %zmm29, %zmm30
+# CHECK: encoding: [0x62,0x62,0x15,0x40,0x8d,0x72,0x80]
+         vpermb	-8192(%rdx), %zmm29, %zmm30
+
+# CHECK: vpermb	-8256(%rdx), %zmm29, %zmm30
+# CHECK: encoding: [0x62,0x62,0x15,0x40,0x8d,0xb2,0xc0,0xdf,0xff,0xff]
+         vpermb	-8256(%rdx), %zmm29, %zmm30
+
+# CHECK: vpermb	4660(%rax,%r14,8), %zmm29, %zmm30
+# CHECK: encoding: [0x62,0x22,0x15,0x40,0x8d,0xb4,0xf0,0x34,0x12,0x00,0x00]
+         vpermb	4660(%rax,%r14,8), %zmm29, %zmm30
+
+# CHECK: vpermt2b	%xmm28, %xmm29, %xmm30
+# CHECK: encoding: [0x62,0x02,0x15,0x00,0x7d,0xf4]
+         vpermt2b	%xmm28, %xmm29, %xmm30
+
+# CHECK: vpermt2b	%xmm28, %xmm29, %xmm30 {%k7}
+# CHECK: encoding: [0x62,0x02,0x15,0x07,0x7d,0xf4]
+         vpermt2b	%xmm28, %xmm29, %xmm30 {%k7}
+
+# CHECK: vpermt2b	%xmm28, %xmm29, %xmm30 {%k7} {z}
+# CHECK: encoding: [0x62,0x02,0x15,0x87,0x7d,0xf4]
+         vpermt2b	%xmm28, %xmm29, %xmm30 {%k7} {z}
+
+# CHECK: vpermt2b	(%rcx), %xmm29, %xmm30
+# CHECK: encoding: [0x62,0x62,0x15,0x00,0x7d,0x31]
+         vpermt2b	(%rcx), %xmm29, %xmm30
+
+# CHECK: vpermt2b	291(%rax,%r14,8), %xmm29, %xmm30
+# CHECK: encoding: [0x62,0x22,0x15,0x00,0x7d,0xb4,0xf0,0x23,0x01,0x00,0x00]
+         vpermt2b	291(%rax,%r14,8), %xmm29, %xmm30
+
+# CHECK: vpermt2b	2032(%rdx), %xmm29, %xmm30
+# CHECK: encoding: [0x62,0x62,0x15,0x00,0x7d,0x72,0x7f]
+         vpermt2b	2032(%rdx), %xmm29, %xmm30
+
+# CHECK: vpermt2b	2048(%rdx), %xmm29, %xmm30
+# CHECK: encoding: [0x62,0x62,0x15,0x00,0x7d,0xb2,0x00,0x08,0x00,0x00]
+         vpermt2b	2048(%rdx), %xmm29, %xmm30
+
+# CHECK: vpermt2b	-2048(%rdx), %xmm29, %xmm30
+# CHECK: encoding: [0x62,0x62,0x15,0x00,0x7d,0x72,0x80]
+         vpermt2b	-2048(%rdx), %xmm29, %xmm30
+
+# CHECK: vpermt2b	-2064(%rdx), %xmm29, %xmm30
+# CHECK: encoding: [0x62,0x62,0x15,0x00,0x7d,0xb2,0xf0,0xf7,0xff,0xff]
+         vpermt2b	-2064(%rdx), %xmm29, %xmm30
+
+# CHECK: vpermt2b	%ymm28, %ymm29, %ymm30
+# CHECK: encoding: [0x62,0x02,0x15,0x20,0x7d,0xf4]
+         vpermt2b	%ymm28, %ymm29, %ymm30
+
+# CHECK: vpermt2b	%ymm28, %ymm29, %ymm30 {%k7}
+# CHECK: encoding: [0x62,0x02,0x15,0x27,0x7d,0xf4]
+         vpermt2b	%ymm28, %ymm29, %ymm30 {%k7}
+
+# CHECK: vpermt2b	%ymm28, %ymm29, %ymm30 {%k7} {z}
+# CHECK: encoding: [0x62,0x02,0x15,0xa7,0x7d,0xf4]
+         vpermt2b	%ymm28, %ymm29, %ymm30 {%k7} {z}
+
+# CHECK: vpermt2b	(%rcx), %ymm29, %ymm30
+# CHECK: encoding: [0x62,0x62,0x15,0x20,0x7d,0x31]
+         vpermt2b	(%rcx), %ymm29, %ymm30
+
+# CHECK: vpermt2b	291(%rax,%r14,8), %ymm29, %ymm30
+# CHECK: encoding: [0x62,0x22,0x15,0x20,0x7d,0xb4,0xf0,0x23,0x01,0x00,0x00]
+         vpermt2b	291(%rax,%r14,8), %ymm29, %ymm30
+
+# CHECK: vpermt2b	4064(%rdx), %ymm29, %ymm30
+# CHECK: encoding: [0x62,0x62,0x15,0x20,0x7d,0x72,0x7f]
+         vpermt2b	4064(%rdx), %ymm29, %ymm30
+
+# CHECK: vpermt2b	4096(%rdx), %ymm29, %ymm30
+# CHECK: encoding: [0x62,0x62,0x15,0x20,0x7d,0xb2,0x00,0x10,0x00,0x00]
+         vpermt2b	4096(%rdx), %ymm29, %ymm30
+
+# CHECK: vpermt2b	-4096(%rdx), %ymm29, %ymm30
+# CHECK: encoding: [0x62,0x62,0x15,0x20,0x7d,0x72,0x80]
+         vpermt2b	-4096(%rdx), %ymm29, %ymm30
+
+# CHECK: vpermt2b	-4128(%rdx), %ymm29, %ymm30
+# CHECK: encoding: [0x62,0x62,0x15,0x20,0x7d,0xb2,0xe0,0xef,0xff,0xff]
+         vpermt2b	-4128(%rdx), %ymm29, %ymm30
+
+# CHECK: vpermt2b	4660(%rax,%r14,8), %xmm29, %xmm30
+# CHECK: encoding: [0x62,0x22,0x15,0x00,0x7d,0xb4,0xf0,0x34,0x12,0x00,0x00]
+         vpermt2b	4660(%rax,%r14,8), %xmm29, %xmm30
+
+# CHECK: vpermt2b	4660(%rax,%r14,8), %ymm29, %ymm30
+# CHECK: encoding: [0x62,0x22,0x15,0x20,0x7d,0xb4,0xf0,0x34,0x12,0x00,0x00]
+         vpermt2b	4660(%rax,%r14,8), %ymm29, %ymm30
+
+# CHECK: vpermt2b	%zmm28, %zmm29, %zmm30
+# CHECK: encoding: [0x62,0x02,0x15,0x40,0x7d,0xf4]
+         vpermt2b	%zmm28, %zmm29, %zmm30
+
+# CHECK: vpermt2b	%zmm28, %zmm29, %zmm30 {%k7}
+# CHECK: encoding: [0x62,0x02,0x15,0x47,0x7d,0xf4]
+         vpermt2b	%zmm28, %zmm29, %zmm30 {%k7}
+
+# CHECK: vpermt2b	%zmm28, %zmm29, %zmm30 {%k7} {z}
+# CHECK: encoding: [0x62,0x02,0x15,0xc7,0x7d,0xf4]
+         vpermt2b	%zmm28, %zmm29, %zmm30 {%k7} {z}
+
+# CHECK: vpermt2b	(%rcx), %zmm29, %zmm30
+# CHECK: encoding: [0x62,0x62,0x15,0x40,0x7d,0x31]
+         vpermt2b	(%rcx), %zmm29, %zmm30
+
+# CHECK: vpermt2b	291(%rax,%r14,8), %zmm29, %zmm30
+# CHECK: encoding: [0x62,0x22,0x15,0x40,0x7d,0xb4,0xf0,0x23,0x01,0x00,0x00]
+         vpermt2b	291(%rax,%r14,8), %zmm29, %zmm30
+
+# CHECK: vpermt2b	8128(%rdx), %zmm29, %zmm30
+# CHECK: encoding: [0x62,0x62,0x15,0x40,0x7d,0x72,0x7f]
+         vpermt2b	8128(%rdx), %zmm29, %zmm30
+
+# CHECK: vpermt2b	8192(%rdx), %zmm29, %zmm30
+# CHECK: encoding: [0x62,0x62,0x15,0x40,0x7d,0xb2,0x00,0x20,0x00,0x00]
+         vpermt2b	8192(%rdx), %zmm29, %zmm30
+
+# CHECK: vpermt2b	-8192(%rdx), %zmm29, %zmm30
+# CHECK: encoding: [0x62,0x62,0x15,0x40,0x7d,0x72,0x80]
+         vpermt2b	-8192(%rdx), %zmm29, %zmm30
+
+# CHECK: vpermt2b	-8256(%rdx), %zmm29, %zmm30
+# CHECK: encoding: [0x62,0x62,0x15,0x40,0x7d,0xb2,0xc0,0xdf,0xff,0xff]
+         vpermt2b	-8256(%rdx), %zmm29, %zmm30
+
+# CHECK: vpermt2b	4660(%rax,%r14,8), %zmm29, %zmm30
+# CHECK: encoding: [0x62,0x22,0x15,0x40,0x7d,0xb4,0xf0,0x34,0x12,0x00,0x00]
+         vpermt2b	4660(%rax,%r14,8), %zmm29, %zmm30
+
+# CHECK: vpermi2b	%xmm28, %xmm29, %xmm30
+# CHECK: encoding: [0x62,0x02,0x15,0x00,0x75,0xf4]
+         vpermi2b	%xmm28, %xmm29, %xmm30
+
+# CHECK: vpermi2b	%xmm28, %xmm29, %xmm30 {%k7}
+# CHECK: encoding: [0x62,0x02,0x15,0x07,0x75,0xf4]
+         vpermi2b	%xmm28, %xmm29, %xmm30 {%k7}
+
+# CHECK: vpermi2b	%xmm28, %xmm29, %xmm30 {%k7} {z}
+# CHECK: encoding: [0x62,0x02,0x15,0x87,0x75,0xf4]
+         vpermi2b	%xmm28, %xmm29, %xmm30 {%k7} {z}
+
+# CHECK: vpermi2b	(%rcx), %xmm29, %xmm30
+# CHECK: encoding: [0x62,0x62,0x15,0x00,0x75,0x31]
+         vpermi2b	(%rcx), %xmm29, %xmm30
+
+# CHECK: vpermi2b	291(%rax,%r14,8), %xmm29, %xmm30
+# CHECK: encoding: [0x62,0x22,0x15,0x00,0x75,0xb4,0xf0,0x23,0x01,0x00,0x00]
+         vpermi2b	291(%rax,%r14,8), %xmm29, %xmm30
+
+# CHECK: vpermi2b	2032(%rdx), %xmm29, %xmm30
+# CHECK: encoding: [0x62,0x62,0x15,0x00,0x75,0x72,0x7f]
+         vpermi2b	2032(%rdx), %xmm29, %xmm30
+
+# CHECK: vpermi2b	2048(%rdx), %xmm29, %xmm30
+# CHECK: encoding: [0x62,0x62,0x15,0x00,0x75,0xb2,0x00,0x08,0x00,0x00]
+         vpermi2b	2048(%rdx), %xmm29, %xmm30
+
+# CHECK: vpermi2b	-2048(%rdx), %xmm29, %xmm30
+# CHECK: encoding: [0x62,0x62,0x15,0x00,0x75,0x72,0x80]
+         vpermi2b	-2048(%rdx), %xmm29, %xmm30
+
+# CHECK: vpermi2b	-2064(%rdx), %xmm29, %xmm30
+# CHECK: encoding: [0x62,0x62,0x15,0x00,0x75,0xb2,0xf0,0xf7,0xff,0xff]
+         vpermi2b	-2064(%rdx), %xmm29, %xmm30
+
+# CHECK: vpermi2b	%ymm28, %ymm29, %ymm30
+# CHECK: encoding: [0x62,0x02,0x15,0x20,0x75,0xf4]
+         vpermi2b	%ymm28, %ymm29, %ymm30
+
+# CHECK: vpermi2b	%ymm28, %ymm29, %ymm30 {%k7}
+# CHECK: encoding: [0x62,0x02,0x15,0x27,0x75,0xf4]
+         vpermi2b	%ymm28, %ymm29, %ymm30 {%k7}
+
+# CHECK: vpermi2b	%ymm28, %ymm29, %ymm30 {%k7} {z}
+# CHECK: encoding: [0x62,0x02,0x15,0xa7,0x75,0xf4]
+         vpermi2b	%ymm28, %ymm29, %ymm30 {%k7} {z}
+
+# CHECK: vpermi2b	(%rcx), %ymm29, %ymm30
+# CHECK: encoding: [0x62,0x62,0x15,0x20,0x75,0x31]
+         vpermi2b	(%rcx), %ymm29, %ymm30
+
+# CHECK: vpermi2b	291(%rax,%r14,8), %ymm29, %ymm30
+# CHECK: encoding: [0x62,0x22,0x15,0x20,0x75,0xb4,0xf0,0x23,0x01,0x00,0x00]
+         vpermi2b	291(%rax,%r14,8), %ymm29, %ymm30
+
+# CHECK: vpermi2b	4064(%rdx), %ymm29, %ymm30
+# CHECK: encoding: [0x62,0x62,0x15,0x20,0x75,0x72,0x7f]
+         vpermi2b	4064(%rdx), %ymm29, %ymm30
+
+# CHECK: vpermi2b	4096(%rdx), %ymm29, %ymm30
+# CHECK: encoding: [0x62,0x62,0x15,0x20,0x75,0xb2,0x00,0x10,0x00,0x00]
+         vpermi2b	4096(%rdx), %ymm29, %ymm30
+
+# CHECK: vpermi2b	-4096(%rdx), %ymm29, %ymm30
+# CHECK: encoding: [0x62,0x62,0x15,0x20,0x75,0x72,0x80]
+         vpermi2b	-4096(%rdx), %ymm29, %ymm30
+
+# CHECK: vpermi2b	-4128(%rdx), %ymm29, %ymm30
+# CHECK: encoding: [0x62,0x62,0x15,0x20,0x75,0xb2,0xe0,0xef,0xff,0xff]
+         vpermi2b	-4128(%rdx), %ymm29, %ymm30
+
+# CHECK: vpermi2b	4660(%rax,%r14,8), %xmm29, %xmm30
+# CHECK: encoding: [0x62,0x22,0x15,0x00,0x75,0xb4,0xf0,0x34,0x12,0x00,0x00]
+         vpermi2b	4660(%rax,%r14,8), %xmm29, %xmm30
+
+# CHECK: vpermi2b	4660(%rax,%r14,8), %ymm29, %ymm30
+# CHECK: encoding: [0x62,0x22,0x15,0x20,0x75,0xb4,0xf0,0x34,0x12,0x00,0x00]
+         vpermi2b	4660(%rax,%r14,8), %ymm29, %ymm30
+
+# CHECK: vpermi2b	%zmm28, %zmm29, %zmm30
+# CHECK: encoding: [0x62,0x02,0x15,0x40,0x75,0xf4]
+         vpermi2b	%zmm28, %zmm29, %zmm30
+
+# CHECK: vpermi2b	%zmm28, %zmm29, %zmm30 {%k7}
+# CHECK: encoding: [0x62,0x02,0x15,0x47,0x75,0xf4]
+         vpermi2b	%zmm28, %zmm29, %zmm30 {%k7}
+
+# CHECK: vpermi2b	%zmm28, %zmm29, %zmm30 {%k7} {z}
+# CHECK: encoding: [0x62,0x02,0x15,0xc7,0x75,0xf4]
+         vpermi2b	%zmm28, %zmm29, %zmm30 {%k7} {z}
+
+# CHECK: vpermi2b	(%rcx), %zmm29, %zmm30
+# CHECK: encoding: [0x62,0x62,0x15,0x40,0x75,0x31]
+         vpermi2b	(%rcx), %zmm29, %zmm30
+
+# CHECK: vpermi2b	291(%rax,%r14,8), %zmm29, %zmm30
+# CHECK: encoding: [0x62,0x22,0x15,0x40,0x75,0xb4,0xf0,0x23,0x01,0x00,0x00]
+         vpermi2b	291(%rax,%r14,8), %zmm29, %zmm30
+
+# CHECK: vpermi2b	8128(%rdx), %zmm29, %zmm30
+# CHECK: encoding: [0x62,0x62,0x15,0x40,0x75,0x72,0x7f]
+         vpermi2b	8128(%rdx), %zmm29, %zmm30
+
+# CHECK: vpermi2b	8192(%rdx), %zmm29, %zmm30
+# CHECK: encoding: [0x62,0x62,0x15,0x40,0x75,0xb2,0x00,0x20,0x00,0x00]
+         vpermi2b	8192(%rdx), %zmm29, %zmm30
+
+# CHECK: vpermi2b	-8192(%rdx), %zmm29, %zmm30
+# CHECK: encoding: [0x62,0x62,0x15,0x40,0x75,0x72,0x80]
+         vpermi2b	-8192(%rdx), %zmm29, %zmm30
+
+# CHECK: vpermi2b	-8256(%rdx), %zmm29, %zmm30
+# CHECK: encoding: [0x62,0x62,0x15,0x40,0x75,0xb2,0xc0,0xdf,0xff,0xff]
+         vpermi2b	-8256(%rdx), %zmm29, %zmm30
+
+# CHECK: vpermi2b	4660(%rax,%r14,8), %zmm29, %zmm30
+# CHECK: encoding: [0x62,0x22,0x15,0x40,0x75,0xb4,0xf0,0x34,0x12,0x00,0x00]
+         vpermi2b	4660(%rax,%r14,8), %zmm29, %zmm30
+
+# CHECK: vpmultishiftqb	%xmm28, %xmm29, %xmm30
+# CHECK: encoding: [0x62,0x02,0x95,0x00,0x83,0xf4]
+         vpmultishiftqb	%xmm28, %xmm29, %xmm30
+
+# CHECK: vpmultishiftqb	%xmm28, %xmm29, %xmm30 {%k7}
+# CHECK: encoding: [0x62,0x02,0x95,0x07,0x83,0xf4]
+         vpmultishiftqb	%xmm28, %xmm29, %xmm30 {%k7}
+
+# CHECK: vpmultishiftqb	%xmm28, %xmm29, %xmm30 {%k7} {z}
+# CHECK: encoding: [0x62,0x02,0x95,0x87,0x83,0xf4]
+         vpmultishiftqb	%xmm28, %xmm29, %xmm30 {%k7} {z}
+
+# CHECK: vpmultishiftqb	(%rcx), %xmm29, %xmm30
+# CHECK: encoding: [0x62,0x62,0x95,0x00,0x83,0x31]
+         vpmultishiftqb	(%rcx), %xmm29, %xmm30
+
+# CHECK: vpmultishiftqb	291(%rax,%r14,8), %xmm29, %xmm30
+# CHECK: encoding: [0x62,0x22,0x95,0x00,0x83,0xb4,0xf0,0x23,0x01,0x00,0x00]
+         vpmultishiftqb	291(%rax,%r14,8), %xmm29, %xmm30
+
+# CHECK: vpmultishiftqb	(%rcx){1to2}, %xmm29, %xmm30
+# CHECK: encoding: [0x62,0x62,0x95,0x10,0x83,0x31]
+         vpmultishiftqb	(%rcx){1to2}, %xmm29, %xmm30
+
+
+# CHECK: vpmultishiftqb	2032(%rdx), %xmm29, %xmm30
+# CHECK: encoding: [0x62,0x62,0x95,0x00,0x83,0x72,0x7f]
+         vpmultishiftqb	2032(%rdx), %xmm29, %xmm30
+
+# CHECK: vpmultishiftqb	2048(%rdx), %xmm29, %xmm30
+# CHECK: encoding: [0x62,0x62,0x95,0x00,0x83,0xb2,0x00,0x08,0x00,0x00]
+         vpmultishiftqb	2048(%rdx), %xmm29, %xmm30
+
+# CHECK: vpmultishiftqb	-2048(%rdx), %xmm29, %xmm30
+# CHECK: encoding: [0x62,0x62,0x95,0x00,0x83,0x72,0x80]
+         vpmultishiftqb	-2048(%rdx), %xmm29, %xmm30
+
+# CHECK: vpmultishiftqb	-2064(%rdx), %xmm29, %xmm30
+# CHECK: encoding: [0x62,0x62,0x95,0x00,0x83,0xb2,0xf0,0xf7,0xff,0xff]
+         vpmultishiftqb	-2064(%rdx), %xmm29, %xmm30
+
+# CHECK: vpmultishiftqb	1016(%rdx){1to2}, %xmm29, %xmm30
+# CHECK: encoding: [0x62,0x62,0x95,0x10,0x83,0x72,0x7f]
+         vpmultishiftqb	1016(%rdx){1to2}, %xmm29, %xmm30
+
+# CHECK: vpmultishiftqb	1024(%rdx){1to2}, %xmm29, %xmm30
+# CHECK: encoding: [0x62,0x62,0x95,0x10,0x83,0xb2,0x00,0x04,0x00,0x00]
+         vpmultishiftqb	1024(%rdx){1to2}, %xmm29, %xmm30
+
+# CHECK: vpmultishiftqb	-1024(%rdx){1to2}, %xmm29, %xmm30
+# CHECK: encoding: [0x62,0x62,0x95,0x10,0x83,0x72,0x80]
+         vpmultishiftqb	-1024(%rdx){1to2}, %xmm29, %xmm30
+
+# CHECK: vpmultishiftqb	-1032(%rdx){1to2}, %xmm29, %xmm30
+# CHECK: encoding: [0x62,0x62,0x95,0x10,0x83,0xb2,0xf8,0xfb,0xff,0xff]
+         vpmultishiftqb	-1032(%rdx){1to2}, %xmm29, %xmm30
+
+# CHECK: vpmultishiftqb	%ymm28, %ymm29, %ymm30
+# CHECK: encoding: [0x62,0x02,0x95,0x20,0x83,0xf4]
+         vpmultishiftqb	%ymm28, %ymm29, %ymm30
+
+# CHECK: vpmultishiftqb	%ymm28, %ymm29, %ymm30 {%k7}
+# CHECK: encoding: [0x62,0x02,0x95,0x27,0x83,0xf4]
+         vpmultishiftqb	%ymm28, %ymm29, %ymm30 {%k7}
+
+# CHECK: vpmultishiftqb	%ymm28, %ymm29, %ymm30 {%k7} {z}
+# CHECK: encoding: [0x62,0x02,0x95,0xa7,0x83,0xf4]
+         vpmultishiftqb	%ymm28, %ymm29, %ymm30 {%k7} {z}
+
+# CHECK: vpmultishiftqb	(%rcx), %ymm29, %ymm30
+# CHECK: encoding: [0x62,0x62,0x95,0x20,0x83,0x31]
+         vpmultishiftqb	(%rcx), %ymm29, %ymm30
+
+# CHECK: vpmultishiftqb	291(%rax,%r14,8), %ymm29, %ymm30
+# CHECK: encoding: [0x62,0x22,0x95,0x20,0x83,0xb4,0xf0,0x23,0x01,0x00,0x00]
+         vpmultishiftqb	291(%rax,%r14,8), %ymm29, %ymm30
+
+# CHECK: vpmultishiftqb	(%rcx){1to4}, %ymm29, %ymm30
+# CHECK: encoding: [0x62,0x62,0x95,0x30,0x83,0x31]
+         vpmultishiftqb	(%rcx){1to4}, %ymm29, %ymm30
+
+# CHECK: vpmultishiftqb	4064(%rdx), %ymm29, %ymm30
+# CHECK: encoding: [0x62,0x62,0x95,0x20,0x83,0x72,0x7f]
+         vpmultishiftqb	4064(%rdx), %ymm29, %ymm30
+
+# CHECK: vpmultishiftqb	4096(%rdx), %ymm29, %ymm30
+# CHECK: encoding: [0x62,0x62,0x95,0x20,0x83,0xb2,0x00,0x10,0x00,0x00]
+         vpmultishiftqb	4096(%rdx), %ymm29, %ymm30
+
+# CHECK: vpmultishiftqb	-4096(%rdx), %ymm29, %ymm30
+# CHECK: encoding: [0x62,0x62,0x95,0x20,0x83,0x72,0x80]
+         vpmultishiftqb	-4096(%rdx), %ymm29, %ymm30
+
+# CHECK: vpmultishiftqb	-4128(%rdx), %ymm29, %ymm30
+# CHECK: encoding: [0x62,0x62,0x95,0x20,0x83,0xb2,0xe0,0xef,0xff,0xff]
+         vpmultishiftqb	-4128(%rdx), %ymm29, %ymm30
+
+# CHECK: vpmultishiftqb	1016(%rdx){1to4}, %ymm29, %ymm30
+# CHECK: encoding: [0x62,0x62,0x95,0x30,0x83,0x72,0x7f]
+         vpmultishiftqb	1016(%rdx){1to4}, %ymm29, %ymm30
+
+# CHECK: vpmultishiftqb	1024(%rdx){1to4}, %ymm29, %ymm30
+# CHECK: encoding: [0x62,0x62,0x95,0x30,0x83,0xb2,0x00,0x04,0x00,0x00]
+         vpmultishiftqb	1024(%rdx){1to4}, %ymm29, %ymm30
+
+# CHECK: vpmultishiftqb	-1024(%rdx){1to4}, %ymm29, %ymm30
+# CHECK: encoding: [0x62,0x62,0x95,0x30,0x83,0x72,0x80]
+         vpmultishiftqb	-1024(%rdx){1to4}, %ymm29, %ymm30
+
+# CHECK: vpmultishiftqb	-1032(%rdx){1to4}, %ymm29, %ymm30
+# CHECK: encoding: [0x62,0x62,0x95,0x30,0x83,0xb2,0xf8,0xfb,0xff,0xff]
+         vpmultishiftqb	-1032(%rdx){1to4}, %ymm29, %ymm30
+
+# CHECK: vpmultishiftqb	4660(%rax,%r14,8), %xmm29, %xmm30
+# CHECK: encoding: [0x62,0x22,0x95,0x00,0x83,0xb4,0xf0,0x34,0x12,0x00,0x00]
+         vpmultishiftqb	4660(%rax,%r14,8), %xmm29, %xmm30
+
+# CHECK: vpmultishiftqb	4660(%rax,%r14,8), %ymm29, %ymm30
+# CHECK: encoding: [0x62,0x22,0x95,0x20,0x83,0xb4,0xf0,0x34,0x12,0x00,0x00]
+         vpmultishiftqb	4660(%rax,%r14,8), %ymm29, %ymm30
+
+# CHECK: vpmultishiftqb	%zmm28, %zmm29, %zmm30
+# CHECK: encoding: [0x62,0x02,0x95,0x40,0x83,0xf4]
+         vpmultishiftqb	%zmm28, %zmm29, %zmm30
+
+# CHECK: vpmultishiftqb	%zmm28, %zmm29, %zmm30 {%k7}
+# CHECK: encoding: [0x62,0x02,0x95,0x47,0x83,0xf4]
+         vpmultishiftqb	%zmm28, %zmm29, %zmm30 {%k7}
+
+# CHECK: vpmultishiftqb	%zmm28, %zmm29, %zmm30 {%k7} {z}
+# CHECK: encoding: [0x62,0x02,0x95,0xc7,0x83,0xf4]
+         vpmultishiftqb	%zmm28, %zmm29, %zmm30 {%k7} {z}
+
+# CHECK: vpmultishiftqb	(%rcx), %zmm29, %zmm30
+# CHECK: encoding: [0x62,0x62,0x95,0x40,0x83,0x31]
+         vpmultishiftqb	(%rcx), %zmm29, %zmm30
+
+# CHECK: vpmultishiftqb	291(%rax,%r14,8), %zmm29, %zmm30
+# CHECK: encoding: [0x62,0x22,0x95,0x40,0x83,0xb4,0xf0,0x23,0x01,0x00,0x00]
+         vpmultishiftqb	291(%rax,%r14,8), %zmm29, %zmm30
+
+# CHECK: vpmultishiftqb	(%rcx){1to8}, %zmm29, %zmm30
+# CHECK: encoding: [0x62,0x62,0x95,0x50,0x83,0x31]
+         vpmultishiftqb	(%rcx){1to8}, %zmm29, %zmm30
+
+# CHECK: vpmultishiftqb	8128(%rdx), %zmm29, %zmm30
+# CHECK: encoding: [0x62,0x62,0x95,0x40,0x83,0x72,0x7f]
+         vpmultishiftqb	8128(%rdx), %zmm29, %zmm30
+
+# CHECK: vpmultishiftqb	8192(%rdx), %zmm29, %zmm30
+# CHECK: encoding: [0x62,0x62,0x95,0x40,0x83,0xb2,0x00,0x20,0x00,0x00]
+         vpmultishiftqb	8192(%rdx), %zmm29, %zmm30
+
+# CHECK: vpmultishiftqb	-8192(%rdx), %zmm29, %zmm30
+# CHECK: encoding: [0x62,0x62,0x95,0x40,0x83,0x72,0x80]
+         vpmultishiftqb	-8192(%rdx), %zmm29, %zmm30
+
+# CHECK: vpmultishiftqb	-8256(%rdx), %zmm29, %zmm30
+# CHECK: encoding: [0x62,0x62,0x95,0x40,0x83,0xb2,0xc0,0xdf,0xff,0xff]
+         vpmultishiftqb	-8256(%rdx), %zmm29, %zmm30
+
+# CHECK: vpmultishiftqb	1016(%rdx){1to8}, %zmm29, %zmm30
+# CHECK: encoding: [0x62,0x62,0x95,0x50,0x83,0x72,0x7f]
+         vpmultishiftqb	1016(%rdx){1to8}, %zmm29, %zmm30
+
+# CHECK: vpmultishiftqb	1024(%rdx){1to8}, %zmm29, %zmm30
+# CHECK: encoding: [0x62,0x62,0x95,0x50,0x83,0xb2,0x00,0x04,0x00,0x00]
+         vpmultishiftqb	1024(%rdx){1to8}, %zmm29, %zmm30
+
+# CHECK: vpmultishiftqb	-1024(%rdx){1to8}, %zmm29, %zmm30
+# CHECK: encoding: [0x62,0x62,0x95,0x50,0x83,0x72,0x80]
+         vpmultishiftqb	-1024(%rdx){1to8}, %zmm29, %zmm30
+
+# CHECK: vpmultishiftqb	-1032(%rdx){1to8}, %zmm29, %zmm30
+# CHECK: encoding: [0x62,0x62,0x95,0x50,0x83,0xb2,0xf8,0xfb,0xff,0xff]
+         vpmultishiftqb	-1032(%rdx){1to8}, %zmm29, %zmm30
+
+# CHECK: vpmultishiftqb	4660(%rax,%r14,8), %zmm29, %zmm30
+# CHECK: encoding: [0x62,0x22,0x95,0x40,0x83,0xb4,0xf0,0x34,0x12,0x00,0x00]
+         vpmultishiftqb	4660(%rax,%r14,8), %zmm29, %zmm30
diff --git a/llvm/test/MC/X86/avx512vbmi-encoding.s b/llvm/test/MC/X86/avx512vbmi-encoding.s
deleted file mode 100644
index bf11e1999d257edbff37a4e6b32cf95f543ee744..0000000000000000000000000000000000000000
--- a/llvm/test/MC/X86/avx512vbmi-encoding.s
+++ /dev/null
@@ -1,543 +0,0 @@
-// RUN: llvm-mc -triple x86_64-unknown-unknown --show-encoding %s | FileCheck %s
-
-     vpermb %xmm28, %xmm29, %xmm30 {%k7}
-//CHECK: vpermb %xmm28, %xmm29, %xmm30 {%k7}
-//CHECK: encoding: [0x62,0x02,0x15,0x07,0x8d,0xf4]
-
-     vpermb %xmm28, %xmm29, %xmm30 {%k7} {z}
-//CHECK: vpermb %xmm28, %xmm29, %xmm30 {%k7} {z}
-//CHECK: encoding: [0x62,0x02,0x15,0x87,0x8d,0xf4]
-
-     vpermb (%rcx), %xmm29, %xmm30
-//CHECK: vpermb (%rcx), %xmm29, %xmm30
-//CHECK: encoding: [0x62,0x62,0x15,0x00,0x8d,0x31]
-
-     vpermb 0x123(%rax,%r14,8), %xmm29, %xmm30
-//CHECK: vpermb 291(%rax,%r14,8), %xmm29, %xmm30
-//CHECK: encoding: [0x62,0x22,0x15,0x00,0x8d,0xb4,0xf0,0x23,0x01,0x00,0x00]
-
-     vpermb 0x7f0(%rdx), %xmm29, %xmm30
-//CHECK: vpermb 2032(%rdx), %xmm29, %xmm30
-//CHECK: encoding: [0x62,0x62,0x15,0x00,0x8d,0x72,0x7f]
-
-     vpermb 0x800(%rdx), %xmm29, %xmm30
-//CHECK: vpermb 2048(%rdx), %xmm29, %xmm30
-//CHECK: encoding: [0x62,0x62,0x15,0x00,0x8d,0xb2,0x00,0x08,0x00,0x00]
-
-     vpermb -0x800(%rdx), %xmm29, %xmm30
-//CHECK: vpermb -2048(%rdx), %xmm29, %xmm30
-//CHECK: encoding: [0x62,0x62,0x15,0x00,0x8d,0x72,0x80]
-
-     vpermb -0x810(%rdx), %xmm29, %xmm30
-//CHECK: vpermb -2064(%rdx), %xmm29, %xmm30
-//CHECK: encoding: [0x62,0x62,0x15,0x00,0x8d,0xb2,0xf0,0xf7,0xff,0xff]
-
-     vpermb %ymm28, %ymm29, %ymm30
-//CHECK: vpermb %ymm28, %ymm29, %ymm30
-//CHECK: encoding: [0x62,0x02,0x15,0x20,0x8d,0xf4]
-
-     vpermb %ymm28, %ymm29, %ymm30 {%k7}
-//CHECK: vpermb %ymm28, %ymm29, %ymm30 {%k7}
-//CHECK: encoding: [0x62,0x02,0x15,0x27,0x8d,0xf4]
-
-     vpermb %ymm28, %ymm29, %ymm30 {%k7} {z}
-//CHECK: vpermb %ymm28, %ymm29, %ymm30 {%k7} {z}
-//CHECK: encoding: [0x62,0x02,0x15,0xa7,0x8d,0xf4]
-
-     vpermb (%rcx), %ymm29, %ymm30
-//CHECK: vpermb (%rcx), %ymm29, %ymm30
-//CHECK: encoding: [0x62,0x62,0x15,0x20,0x8d,0x31]
-
-     vpermb 0x123(%rax,%r14,8), %ymm29, %ymm30
-//CHECK: vpermb 291(%rax,%r14,8), %ymm29, %ymm30
-//CHECK: encoding: [0x62,0x22,0x15,0x20,0x8d,0xb4,0xf0,0x23,0x01,0x00,0x00]
-
-     vpermb 0xfe0(%rdx), %ymm29, %ymm30
-//CHECK: vpermb 4064(%rdx), %ymm29, %ymm30
-//CHECK: encoding: [0x62,0x62,0x15,0x20,0x8d,0x72,0x7f]
-
-     vpermb 0x1000(%rdx), %ymm29, %ymm30
-//CHECK: vpermb 4096(%rdx), %ymm29, %ymm30
-//CHECK: encoding: [0x62,0x62,0x15,0x20,0x8d,0xb2,0x00,0x10,0x00,0x00]
-
-     vpermb -0x1000(%rdx), %ymm29, %ymm30
-//CHECK: vpermb -4096(%rdx), %ymm29, %ymm30
-//CHECK: encoding: [0x62,0x62,0x15,0x20,0x8d,0x72,0x80]
-
-     vpermb -0x1020(%rdx), %ymm29, %ymm30
-//CHECK: vpermb -4128(%rdx), %ymm29, %ymm30
-//CHECK: encoding: [0x62,0x62,0x15,0x20,0x8d,0xb2,0xe0,0xef,0xff,0xff]
-
-     vpermb %xmm28, %xmm29, %xmm30
-//CHECK: vpermb %xmm28, %xmm29, %xmm30
-//CHECK: encoding: [0x62,0x02,0x15,0x00,0x8d,0xf4]
-
-     vpermb 0x1234(%rax,%r14,8), %xmm29, %xmm30
-//CHECK: vpermb 4660(%rax,%r14,8), %xmm29, %xmm30
-//CHECK: encoding: [0x62,0x22,0x15,0x00,0x8d,0xb4,0xf0,0x34,0x12,0x00,0x00]
-
-     vpermb 0x1234(%rax,%r14,8), %ymm29, %ymm30
-//CHECK: vpermb 4660(%rax,%r14,8), %ymm29, %ymm30
-//CHECK: encoding: [0x62,0x22,0x15,0x20,0x8d,0xb4,0xf0,0x34,0x12,0x00,0x00]
-
-     vpermb %zmm28, %zmm29, %zmm30
-//CHECK: vpermb %zmm28, %zmm29, %zmm30
-//CHECK: encoding: [0x62,0x02,0x15,0x40,0x8d,0xf4]
-
-     vpermb %zmm28, %zmm29, %zmm30 {%k7}
-//CHECK: vpermb %zmm28, %zmm29, %zmm30 {%k7}
-//CHECK: encoding: [0x62,0x02,0x15,0x47,0x8d,0xf4]
-
-     vpermb %zmm28, %zmm29, %zmm30 {%k7} {z}
-//CHECK: vpermb %zmm28, %zmm29, %zmm30 {%k7} {z}
-//CHECK: encoding: [0x62,0x02,0x15,0xc7,0x8d,0xf4]
-
-     vpermb (%rcx), %zmm29, %zmm30
-//CHECK: vpermb (%rcx), %zmm29, %zmm30
-//CHECK: encoding: [0x62,0x62,0x15,0x40,0x8d,0x31]
-
-     vpermb 0x123(%rax,%r14,8), %zmm29, %zmm30
-//CHECK: vpermb 291(%rax,%r14,8), %zmm29, %zmm30
-//CHECK: encoding: [0x62,0x22,0x15,0x40,0x8d,0xb4,0xf0,0x23,0x01,0x00,0x00]
-
-     vpermb 0x1fc0(%rdx), %zmm29, %zmm30
-//CHECK: vpermb 8128(%rdx), %zmm29, %zmm30
-//CHECK: encoding: [0x62,0x62,0x15,0x40,0x8d,0x72,0x7f]
-
-     vpermb 0x2000(%rdx), %zmm29, %zmm30
-//CHECK: vpermb 8192(%rdx), %zmm29, %zmm30
-//CHECK: encoding: [0x62,0x62,0x15,0x40,0x8d,0xb2,0x00,0x20,0x00,0x00]
-
-     vpermb -0x2000(%rdx), %zmm29, %zmm30
-//CHECK: vpermb -8192(%rdx), %zmm29, %zmm30
-//CHECK: encoding: [0x62,0x62,0x15,0x40,0x8d,0x72,0x80]
-
-     vpermb -0x2040(%rdx), %zmm29, %zmm30
-//CHECK: vpermb -8256(%rdx), %zmm29, %zmm30
-//CHECK: encoding: [0x62,0x62,0x15,0x40,0x8d,0xb2,0xc0,0xdf,0xff,0xff]
-
-     vpermb 0x1234(%rax,%r14,8), %zmm29, %zmm30
-//CHECK: vpermb 4660(%rax,%r14,8), %zmm29, %zmm30
-//CHECK: encoding: [0x62,0x22,0x15,0x40,0x8d,0xb4,0xf0,0x34,0x12,0x00,0x00]
-
-     vpermt2b %xmm28, %xmm29, %xmm30
-//CHECK: vpermt2b %xmm28, %xmm29, %xmm30
-//CHECK: encoding: [0x62,0x02,0x15,0x00,0x7d,0xf4]
-
-     vpermt2b %xmm28, %xmm29, %xmm30 {%k7}
-//CHECK: vpermt2b %xmm28, %xmm29, %xmm30 {%k7}
-//CHECK: encoding: [0x62,0x02,0x15,0x07,0x7d,0xf4]
-
-     vpermt2b %xmm28, %xmm29, %xmm30 {%k7} {z}
-//CHECK: vpermt2b %xmm28, %xmm29, %xmm30 {%k7} {z}
-//CHECK: encoding: [0x62,0x02,0x15,0x87,0x7d,0xf4]
-
-     vpermt2b (%rcx), %xmm29, %xmm30
-//CHECK: vpermt2b (%rcx), %xmm29, %xmm30
-//CHECK: encoding: [0x62,0x62,0x15,0x00,0x7d,0x31]
-
-     vpermt2b 0x123(%rax,%r14,8), %xmm29, %xmm30
-//CHECK: vpermt2b 291(%rax,%r14,8), %xmm29, %xmm30
-//CHECK: encoding: [0x62,0x22,0x15,0x00,0x7d,0xb4,0xf0,0x23,0x01,0x00,0x00]
-
-     vpermt2b 0x7f0(%rdx), %xmm29, %xmm30
-//CHECK: vpermt2b 2032(%rdx), %xmm29, %xmm30
-//CHECK: encoding: [0x62,0x62,0x15,0x00,0x7d,0x72,0x7f]
-
-     vpermt2b 0x800(%rdx), %xmm29, %xmm30
-//CHECK: vpermt2b 2048(%rdx), %xmm29, %xmm30
-//CHECK: encoding: [0x62,0x62,0x15,0x00,0x7d,0xb2,0x00,0x08,0x00,0x00]
-
-     vpermt2b -0x800(%rdx), %xmm29, %xmm30
-//CHECK: vpermt2b -2048(%rdx), %xmm29, %xmm30
-//CHECK: encoding: [0x62,0x62,0x15,0x00,0x7d,0x72,0x80]
-
-     vpermt2b -0x810(%rdx), %xmm29, %xmm30
-//CHECK: vpermt2b -2064(%rdx), %xmm29, %xmm30
-//CHECK: encoding: [0x62,0x62,0x15,0x00,0x7d,0xb2,0xf0,0xf7,0xff,0xff]
-
-     vpermt2b %ymm28, %ymm29, %ymm30
-//CHECK: vpermt2b %ymm28, %ymm29, %ymm30
-//CHECK: encoding: [0x62,0x02,0x15,0x20,0x7d,0xf4]
-
-     vpermt2b %ymm28, %ymm29, %ymm30 {%k7}
-//CHECK: vpermt2b %ymm28, %ymm29, %ymm30 {%k7}
-//CHECK: encoding: [0x62,0x02,0x15,0x27,0x7d,0xf4]
-
-     vpermt2b %ymm28, %ymm29, %ymm30 {%k7} {z}
-//CHECK: vpermt2b %ymm28, %ymm29, %ymm30 {%k7} {z}
-//CHECK: encoding: [0x62,0x02,0x15,0xa7,0x7d,0xf4]
-
-     vpermt2b (%rcx), %ymm29, %ymm30
-//CHECK: vpermt2b (%rcx), %ymm29, %ymm30
-//CHECK: encoding: [0x62,0x62,0x15,0x20,0x7d,0x31]
-
-     vpermt2b 0x123(%rax,%r14,8), %ymm29, %ymm30
-//CHECK: vpermt2b 291(%rax,%r14,8), %ymm29, %ymm30
-//CHECK: encoding: [0x62,0x22,0x15,0x20,0x7d,0xb4,0xf0,0x23,0x01,0x00,0x00]
-
-     vpermt2b 0xfe0(%rdx), %ymm29, %ymm30
-//CHECK: vpermt2b 4064(%rdx), %ymm29, %ymm30
-//CHECK: encoding: [0x62,0x62,0x15,0x20,0x7d,0x72,0x7f]
-
-     vpermt2b 0x1000(%rdx), %ymm29, %ymm30
-//CHECK: vpermt2b 4096(%rdx), %ymm29, %ymm30
-//CHECK: encoding: [0x62,0x62,0x15,0x20,0x7d,0xb2,0x00,0x10,0x00,0x00]
-
-     vpermt2b -0x1000(%rdx), %ymm29, %ymm30
-//CHECK: vpermt2b -4096(%rdx), %ymm29, %ymm30
-//CHECK: encoding: [0x62,0x62,0x15,0x20,0x7d,0x72,0x80]
-
-     vpermt2b -0x1020(%rdx), %ymm29, %ymm30
-//CHECK: vpermt2b -4128(%rdx), %ymm29, %ymm30
-//CHECK: encoding: [0x62,0x62,0x15,0x20,0x7d,0xb2,0xe0,0xef,0xff,0xff]
-
-     vpermt2b 0x1234(%rax,%r14,8), %xmm29, %xmm30
-//CHECK: vpermt2b 4660(%rax,%r14,8), %xmm29, %xmm30
-//CHECK: encoding: [0x62,0x22,0x15,0x00,0x7d,0xb4,0xf0,0x34,0x12,0x00,0x00]
-
-     vpermt2b 0x1234(%rax,%r14,8), %ymm29, %ymm30
-//CHECK: vpermt2b 4660(%rax,%r14,8), %ymm29, %ymm30
-//CHECK: encoding: [0x62,0x22,0x15,0x20,0x7d,0xb4,0xf0,0x34,0x12,0x00,0x00]
-
-     vpermt2b %zmm28, %zmm29, %zmm30
-//CHECK: vpermt2b %zmm28, %zmm29, %zmm30
-//CHECK: encoding: [0x62,0x02,0x15,0x40,0x7d,0xf4]
-
-     vpermt2b %zmm28, %zmm29, %zmm30 {%k7}
-//CHECK: vpermt2b %zmm28, %zmm29, %zmm30 {%k7}
-//CHECK: encoding: [0x62,0x02,0x15,0x47,0x7d,0xf4]
-
-     vpermt2b %zmm28, %zmm29, %zmm30 {%k7} {z}
-//CHECK: vpermt2b %zmm28, %zmm29, %zmm30 {%k7} {z}
-//CHECK: encoding: [0x62,0x02,0x15,0xc7,0x7d,0xf4]
-
-     vpermt2b (%rcx), %zmm29, %zmm30
-//CHECK: vpermt2b (%rcx), %zmm29, %zmm30
-//CHECK: encoding: [0x62,0x62,0x15,0x40,0x7d,0x31]
-
-     vpermt2b 0x123(%rax,%r14,8), %zmm29, %zmm30
-//CHECK: vpermt2b 291(%rax,%r14,8), %zmm29, %zmm30
-//CHECK: encoding: [0x62,0x22,0x15,0x40,0x7d,0xb4,0xf0,0x23,0x01,0x00,0x00]
-
-     vpermt2b 0x1fc0(%rdx), %zmm29, %zmm30
-//CHECK: vpermt2b 8128(%rdx), %zmm29, %zmm30
-//CHECK: encoding: [0x62,0x62,0x15,0x40,0x7d,0x72,0x7f]
-
-     vpermt2b 0x2000(%rdx), %zmm29, %zmm30
-//CHECK: vpermt2b 8192(%rdx), %zmm29, %zmm30
-//CHECK: encoding: [0x62,0x62,0x15,0x40,0x7d,0xb2,0x00,0x20,0x00,0x00]
-
-     vpermt2b -0x2000(%rdx), %zmm29, %zmm30
-//CHECK: vpermt2b -8192(%rdx), %zmm29, %zmm30
-//CHECK: encoding: [0x62,0x62,0x15,0x40,0x7d,0x72,0x80]
-
-     vpermt2b -0x2040(%rdx), %zmm29, %zmm30
-//CHECK: vpermt2b -8256(%rdx), %zmm29, %zmm30
-//CHECK: encoding: [0x62,0x62,0x15,0x40,0x7d,0xb2,0xc0,0xdf,0xff,0xff]
-
-     vpermt2b 0x1234(%rax,%r14,8), %zmm29, %zmm30
-//CHECK: vpermt2b 4660(%rax,%r14,8), %zmm29, %zmm30
-//CHECK: encoding: [0x62,0x22,0x15,0x40,0x7d,0xb4,0xf0,0x34,0x12,0x00,0x00]
-
-     vpermi2b %xmm28, %xmm29, %xmm30
-//CHECK: vpermi2b %xmm28, %xmm29, %xmm30
-//CHECK: encoding: [0x62,0x02,0x15,0x00,0x75,0xf4]
-
-     vpermi2b %xmm28, %xmm29, %xmm30 {%k7}
-//CHECK: vpermi2b %xmm28, %xmm29, %xmm30 {%k7}
-//CHECK: encoding: [0x62,0x02,0x15,0x07,0x75,0xf4]
-
-     vpermi2b %xmm28, %xmm29, %xmm30 {%k7} {z}
-//CHECK: vpermi2b %xmm28, %xmm29, %xmm30 {%k7} {z}
-//CHECK: encoding: [0x62,0x02,0x15,0x87,0x75,0xf4]
-
-     vpermi2b (%rcx), %xmm29, %xmm30
-//CHECK: vpermi2b (%rcx), %xmm29, %xmm30
-//CHECK: encoding: [0x62,0x62,0x15,0x00,0x75,0x31]
-
-     vpermi2b 0x123(%rax,%r14,8), %xmm29, %xmm30
-//CHECK: vpermi2b 291(%rax,%r14,8), %xmm29, %xmm30
-//CHECK: encoding: [0x62,0x22,0x15,0x00,0x75,0xb4,0xf0,0x23,0x01,0x00,0x00]
-
-     vpermi2b 0x7f0(%rdx), %xmm29, %xmm30
-//CHECK: vpermi2b 2032(%rdx), %xmm29, %xmm30
-//CHECK: encoding: [0x62,0x62,0x15,0x00,0x75,0x72,0x7f]
-
-     vpermi2b 0x800(%rdx), %xmm29, %xmm30
-//CHECK: vpermi2b 2048(%rdx), %xmm29, %xmm30
-//CHECK: encoding: [0x62,0x62,0x15,0x00,0x75,0xb2,0x00,0x08,0x00,0x00]
-
-     vpermi2b -0x800(%rdx), %xmm29, %xmm30
-//CHECK: vpermi2b -2048(%rdx), %xmm29, %xmm30
-//CHECK: encoding: [0x62,0x62,0x15,0x00,0x75,0x72,0x80]
-
-     vpermi2b -0x810(%rdx), %xmm29, %xmm30
-//CHECK: vpermi2b -2064(%rdx), %xmm29, %xmm30
-//CHECK: encoding: [0x62,0x62,0x15,0x00,0x75,0xb2,0xf0,0xf7,0xff,0xff]
-
-     vpermi2b %ymm28, %ymm29, %ymm30
-//CHECK: vpermi2b %ymm28, %ymm29, %ymm30
-//CHECK: encoding: [0x62,0x02,0x15,0x20,0x75,0xf4]
-
-     vpermi2b %ymm28, %ymm29, %ymm30 {%k7}
-//CHECK: vpermi2b %ymm28, %ymm29, %ymm30 {%k7}
-//CHECK: encoding: [0x62,0x02,0x15,0x27,0x75,0xf4]
-
-     vpermi2b %ymm28, %ymm29, %ymm30 {%k7} {z}
-//CHECK: vpermi2b %ymm28, %ymm29, %ymm30 {%k7} {z}
-//CHECK: encoding: [0x62,0x02,0x15,0xa7,0x75,0xf4]
-
-     vpermi2b (%rcx), %ymm29, %ymm30
-//CHECK: vpermi2b (%rcx), %ymm29, %ymm30
-//CHECK: encoding: [0x62,0x62,0x15,0x20,0x75,0x31]
-
-     vpermi2b 0x123(%rax,%r14,8), %ymm29, %ymm30
-//CHECK: vpermi2b 291(%rax,%r14,8), %ymm29, %ymm30
-//CHECK: encoding: [0x62,0x22,0x15,0x20,0x75,0xb4,0xf0,0x23,0x01,0x00,0x00]
-
-     vpermi2b 0xfe0(%rdx), %ymm29, %ymm30
-//CHECK: vpermi2b 4064(%rdx), %ymm29, %ymm30
-//CHECK: encoding: [0x62,0x62,0x15,0x20,0x75,0x72,0x7f]
-
-     vpermi2b 0x1000(%rdx), %ymm29, %ymm30
-//CHECK: vpermi2b 4096(%rdx), %ymm29, %ymm30
-//CHECK: encoding: [0x62,0x62,0x15,0x20,0x75,0xb2,0x00,0x10,0x00,0x00]
-
-     vpermi2b -0x1000(%rdx), %ymm29, %ymm30
-//CHECK: vpermi2b -4096(%rdx), %ymm29, %ymm30
-//CHECK: encoding: [0x62,0x62,0x15,0x20,0x75,0x72,0x80]
-
-     vpermi2b -0x1020(%rdx), %ymm29, %ymm30
-//CHECK: vpermi2b -4128(%rdx), %ymm29, %ymm30
-//CHECK: encoding: [0x62,0x62,0x15,0x20,0x75,0xb2,0xe0,0xef,0xff,0xff]
-
-     vpermi2b 0x1234(%rax,%r14,8), %xmm29, %xmm30
-//CHECK: vpermi2b 4660(%rax,%r14,8), %xmm29, %xmm30
-//CHECK: encoding: [0x62,0x22,0x15,0x00,0x75,0xb4,0xf0,0x34,0x12,0x00,0x00]
-
-     vpermi2b 0x1234(%rax,%r14,8), %ymm29, %ymm30
-//CHECK: vpermi2b 4660(%rax,%r14,8), %ymm29, %ymm30
-//CHECK: encoding: [0x62,0x22,0x15,0x20,0x75,0xb4,0xf0,0x34,0x12,0x00,0x00]
-
-     vpermi2b %zmm28, %zmm29, %zmm30
-//CHECK: vpermi2b %zmm28, %zmm29, %zmm30
-//CHECK: encoding: [0x62,0x02,0x15,0x40,0x75,0xf4]
-
-     vpermi2b %zmm28, %zmm29, %zmm30 {%k7}
-//CHECK: vpermi2b %zmm28, %zmm29, %zmm30 {%k7}
-//CHECK: encoding: [0x62,0x02,0x15,0x47,0x75,0xf4]
-
-     vpermi2b %zmm28, %zmm29, %zmm30 {%k7} {z}
-//CHECK: vpermi2b %zmm28, %zmm29, %zmm30 {%k7} {z}
-//CHECK: encoding: [0x62,0x02,0x15,0xc7,0x75,0xf4]
-
-     vpermi2b (%rcx), %zmm29, %zmm30
-//CHECK: vpermi2b (%rcx), %zmm29, %zmm30
-//CHECK: encoding: [0x62,0x62,0x15,0x40,0x75,0x31]
-
-     vpermi2b 0x123(%rax,%r14,8), %zmm29, %zmm30
-//CHECK: vpermi2b 291(%rax,%r14,8), %zmm29, %zmm30
-//CHECK: encoding: [0x62,0x22,0x15,0x40,0x75,0xb4,0xf0,0x23,0x01,0x00,0x00]
-
-     vpermi2b 0x1fc0(%rdx), %zmm29, %zmm30
-//CHECK: vpermi2b 8128(%rdx), %zmm29, %zmm30
-//CHECK: encoding: [0x62,0x62,0x15,0x40,0x75,0x72,0x7f]
-
-     vpermi2b 0x2000(%rdx), %zmm29, %zmm30
-//CHECK: vpermi2b 8192(%rdx), %zmm29, %zmm30
-//CHECK: encoding: [0x62,0x62,0x15,0x40,0x75,0xb2,0x00,0x20,0x00,0x00]
-
-     vpermi2b -0x2000(%rdx), %zmm29, %zmm30
-//CHECK: vpermi2b -8192(%rdx), %zmm29, %zmm30
-//CHECK: encoding: [0x62,0x62,0x15,0x40,0x75,0x72,0x80]
-
-     vpermi2b -0x2040(%rdx), %zmm29, %zmm30
-//CHECK: vpermi2b -8256(%rdx), %zmm29, %zmm30
-//CHECK: encoding: [0x62,0x62,0x15,0x40,0x75,0xb2,0xc0,0xdf,0xff,0xff]
-
-     vpermi2b 0x1234(%rax,%r14,8), %zmm29, %zmm30
-//CHECK: vpermi2b 4660(%rax,%r14,8), %zmm29, %zmm30
-//CHECK: encoding: [0x62,0x22,0x15,0x40,0x75,0xb4,0xf0,0x34,0x12,0x00,0x00]
-
-  vpmultishiftqb %xmm28, %xmm29, %xmm30
-//CHECK: vpmultishiftqb %xmm28, %xmm29, %xmm30
-//CHECK: encoding: [0x62,0x02,0x95,0x00,0x83,0xf4]
-
-  vpmultishiftqb %xmm28, %xmm29, %xmm30 {%k7}
-//CHECK: vpmultishiftqb %xmm28, %xmm29, %xmm30 {%k7}
-//CHECK: encoding: [0x62,0x02,0x95,0x07,0x83,0xf4]
-
-  vpmultishiftqb %xmm28, %xmm29, %xmm30 {%k7} {z}
-//CHECK: vpmultishiftqb %xmm28, %xmm29, %xmm30 {%k7} {z}
-//CHECK: encoding: [0x62,0x02,0x95,0x87,0x83,0xf4]
-
-  vpmultishiftqb (%rcx), %xmm29, %xmm30
-//CHECK: vpmultishiftqb (%rcx), %xmm29, %xmm30
-//CHECK: encoding: [0x62,0x62,0x95,0x00,0x83,0x31]
-
-  vpmultishiftqb 0x123(%rax,%r14,8), %xmm29, %xmm30
-//CHECK: vpmultishiftqb 291(%rax,%r14,8), %xmm29, %xmm30
-//CHECK: encoding: [0x62,0x22,0x95,0x00,0x83,0xb4,0xf0,0x23,0x01,0x00,0x00]
-
-  vpmultishiftqb (%rcx){1to2}, %xmm29, %xmm30
-//CHECK: vpmultishiftqb (%rcx){1to2}, %xmm29, %xmm30
-
-//CHECK: encoding: [0x62,0x62,0x95,0x10,0x83,0x31]
-
-  vpmultishiftqb 0x7f0(%rdx), %xmm29, %xmm30
-//CHECK: vpmultishiftqb 2032(%rdx), %xmm29, %xmm30
-//CHECK: encoding: [0x62,0x62,0x95,0x00,0x83,0x72,0x7f]
-
-  vpmultishiftqb 0x800(%rdx), %xmm29, %xmm30
-//CHECK: vpmultishiftqb 2048(%rdx), %xmm29, %xmm30
-//CHECK: encoding: [0x62,0x62,0x95,0x00,0x83,0xb2,0x00,0x08,0x00,0x00]
-
-  vpmultishiftqb -0x800(%rdx), %xmm29, %xmm30
-//CHECK: vpmultishiftqb -2048(%rdx), %xmm29, %xmm30
-//CHECK: encoding: [0x62,0x62,0x95,0x00,0x83,0x72,0x80]
-
-  vpmultishiftqb -0x810(%rdx), %xmm29, %xmm30
-//CHECK: vpmultishiftqb -2064(%rdx), %xmm29, %xmm30
-//CHECK: encoding: [0x62,0x62,0x95,0x00,0x83,0xb2,0xf0,0xf7,0xff,0xff]
-
-  vpmultishiftqb 0x3f8(%rdx){1to2}, %xmm29, %xmm30
-//CHECK: vpmultishiftqb 1016(%rdx){1to2}, %xmm29, %xmm30
-//CHECK: encoding: [0x62,0x62,0x95,0x10,0x83,0x72,0x7f]
-
-  vpmultishiftqb 0x400(%rdx){1to2}, %xmm29, %xmm30
-//CHECK: vpmultishiftqb 1024(%rdx){1to2}, %xmm29, %xmm30
-//CHECK: encoding: [0x62,0x62,0x95,0x10,0x83,0xb2,0x00,0x04,0x00,0x00]
-
-  vpmultishiftqb -0x400(%rdx){1to2}, %xmm29, %xmm30
-//CHECK: vpmultishiftqb -1024(%rdx){1to2}, %xmm29, %xmm30
-//CHECK: encoding: [0x62,0x62,0x95,0x10,0x83,0x72,0x80]
-
-  vpmultishiftqb -0x408(%rdx){1to2}, %xmm29, %xmm30
-//CHECK: vpmultishiftqb -1032(%rdx){1to2}, %xmm29, %xmm30
-//CHECK: encoding: [0x62,0x62,0x95,0x10,0x83,0xb2,0xf8,0xfb,0xff,0xff]
-
-  vpmultishiftqb %ymm28, %ymm29, %ymm30
-//CHECK: vpmultishiftqb %ymm28, %ymm29, %ymm30
-//CHECK: encoding: [0x62,0x02,0x95,0x20,0x83,0xf4]
-
-  vpmultishiftqb %ymm28, %ymm29, %ymm30 {%k7}
-//CHECK: vpmultishiftqb %ymm28, %ymm29, %ymm30 {%k7}
-//CHECK: encoding: [0x62,0x02,0x95,0x27,0x83,0xf4]
-
-  vpmultishiftqb %ymm28, %ymm29, %ymm30 {%k7} {z}
-//CHECK: vpmultishiftqb %ymm28, %ymm29, %ymm30 {%k7} {z}
-//CHECK: encoding: [0x62,0x02,0x95,0xa7,0x83,0xf4]
-
-  vpmultishiftqb (%rcx), %ymm29, %ymm30
-//CHECK: vpmultishiftqb (%rcx), %ymm29, %ymm30
-//CHECK: encoding: [0x62,0x62,0x95,0x20,0x83,0x31]
-
-  vpmultishiftqb 0x123(%rax,%r14,8), %ymm29, %ymm30
-//CHECK: vpmultishiftqb 291(%rax,%r14,8), %ymm29, %ymm30
-//CHECK: encoding: [0x62,0x22,0x95,0x20,0x83,0xb4,0xf0,0x23,0x01,0x00,0x00]
-
-  vpmultishiftqb (%rcx){1to4}, %ymm29, %ymm30
-//CHECK: vpmultishiftqb (%rcx){1to4}, %ymm29, %ymm30
-//CHECK: encoding: [0x62,0x62,0x95,0x30,0x83,0x31]
-
-  vpmultishiftqb 0xfe0(%rdx), %ymm29, %ymm30
-//CHECK: vpmultishiftqb 4064(%rdx), %ymm29, %ymm30
-//CHECK: encoding: [0x62,0x62,0x95,0x20,0x83,0x72,0x7f]
-
-  vpmultishiftqb 0x1000(%rdx), %ymm29, %ymm30
-//CHECK: vpmultishiftqb 4096(%rdx), %ymm29, %ymm30
-//CHECK: encoding: [0x62,0x62,0x95,0x20,0x83,0xb2,0x00,0x10,0x00,0x00]
-
-  vpmultishiftqb -0x1000(%rdx), %ymm29, %ymm30
-//CHECK: vpmultishiftqb -4096(%rdx), %ymm29, %ymm30
-//CHECK: encoding: [0x62,0x62,0x95,0x20,0x83,0x72,0x80]
-
-  vpmultishiftqb -0x1020(%rdx), %ymm29, %ymm30
-//CHECK: vpmultishiftqb -4128(%rdx), %ymm29, %ymm30
-//CHECK: encoding: [0x62,0x62,0x95,0x20,0x83,0xb2,0xe0,0xef,0xff,0xff]
-
-  vpmultishiftqb 0x3f8(%rdx){1to4}, %ymm29, %ymm30
-//CHECK: vpmultishiftqb 1016(%rdx){1to4}, %ymm29, %ymm30
-//CHECK: encoding: [0x62,0x62,0x95,0x30,0x83,0x72,0x7f]
-
-  vpmultishiftqb 0x400(%rdx){1to4}, %ymm29, %ymm30
-//CHECK: vpmultishiftqb 1024(%rdx){1to4}, %ymm29, %ymm30
-//CHECK: encoding: [0x62,0x62,0x95,0x30,0x83,0xb2,0x00,0x04,0x00,0x00]
-
-  vpmultishiftqb -0x400(%rdx){1to4}, %ymm29, %ymm30
-//CHECK: vpmultishiftqb -1024(%rdx){1to4}, %ymm29, %ymm30
-//CHECK: encoding: [0x62,0x62,0x95,0x30,0x83,0x72,0x80]
-
-  vpmultishiftqb -0x408(%rdx){1to4}, %ymm29, %ymm30
-//CHECK: vpmultishiftqb -1032(%rdx){1to4}, %ymm29, %ymm30
-//CHECK: encoding: [0x62,0x62,0x95,0x30,0x83,0xb2,0xf8,0xfb,0xff,0xff]
-
-  vpmultishiftqb 0x1234(%rax,%r14,8), %xmm29, %xmm30
-//CHECK: vpmultishiftqb 4660(%rax,%r14,8), %xmm29, %xmm30
-//CHECK: encoding: [0x62,0x22,0x95,0x00,0x83,0xb4,0xf0,0x34,0x12,0x00,0x00]
-
-  vpmultishiftqb 0x1234(%rax,%r14,8), %ymm29, %ymm30
-//CHECK: vpmultishiftqb 4660(%rax,%r14,8), %ymm29, %ymm30
-//CHECK: encoding: [0x62,0x22,0x95,0x20,0x83,0xb4,0xf0,0x34,0x12,0x00,0x00]
-
-  vpmultishiftqb %zmm28, %zmm29, %zmm30
-//CHECK: vpmultishiftqb %zmm28, %zmm29, %zmm30
-//CHECK: encoding: [0x62,0x02,0x95,0x40,0x83,0xf4]
-
-  vpmultishiftqb %zmm28, %zmm29, %zmm30 {%k7}
-//CHECK: vpmultishiftqb %zmm28, %zmm29, %zmm30 {%k7}
-//CHECK: encoding: [0x62,0x02,0x95,0x47,0x83,0xf4]
-
-  vpmultishiftqb %zmm28, %zmm29, %zmm30 {%k7} {z}
-//CHECK: vpmultishiftqb %zmm28, %zmm29, %zmm30 {%k7} {z}
-//CHECK: encoding: [0x62,0x02,0x95,0xc7,0x83,0xf4]
-
-  vpmultishiftqb (%rcx), %zmm29, %zmm30
-//CHECK: vpmultishiftqb (%rcx), %zmm29, %zmm30
-//CHECK: encoding: [0x62,0x62,0x95,0x40,0x83,0x31]
-
-  vpmultishiftqb 0x123(%rax,%r14,8), %zmm29, %zmm30
-//CHECK: vpmultishiftqb 291(%rax,%r14,8), %zmm29, %zmm30
-//CHECK: encoding: [0x62,0x22,0x95,0x40,0x83,0xb4,0xf0,0x23,0x01,0x00,0x00]
-
-  vpmultishiftqb (%rcx){1to8}, %zmm29, %zmm30
-//CHECK: vpmultishiftqb (%rcx){1to8}, %zmm29, %zmm30
-//CHECK: encoding: [0x62,0x62,0x95,0x50,0x83,0x31]
-
-  vpmultishiftqb 0x1fc0(%rdx), %zmm29, %zmm30
-//CHECK: vpmultishiftqb 8128(%rdx), %zmm29, %zmm30
-//CHECK: encoding: [0x62,0x62,0x95,0x40,0x83,0x72,0x7f]
-
-  vpmultishiftqb 0x2000(%rdx), %zmm29, %zmm30
-//CHECK: vpmultishiftqb 8192(%rdx), %zmm29, %zmm30
-//CHECK: encoding: [0x62,0x62,0x95,0x40,0x83,0xb2,0x00,0x20,0x00,0x00]
-
-  vpmultishiftqb -0x2000(%rdx), %zmm29, %zmm30
-//CHECK: vpmultishiftqb -8192(%rdx), %zmm29, %zmm30
-//CHECK: encoding: [0x62,0x62,0x95,0x40,0x83,0x72,0x80]
-
-  vpmultishiftqb -0x2040(%rdx), %zmm29, %zmm30
-//CHECK: vpmultishiftqb -8256(%rdx), %zmm29, %zmm30
-//CHECK: encoding: [0x62,0x62,0x95,0x40,0x83,0xb2,0xc0,0xdf,0xff,0xff]
-
-  vpmultishiftqb 0x3f8(%rdx){1to8}, %zmm29, %zmm30
-//CHECK: vpmultishiftqb 1016(%rdx){1to8}, %zmm29, %zmm30
-//CHECK: encoding: [0x62,0x62,0x95,0x50,0x83,0x72,0x7f]
-
-  vpmultishiftqb 0x400(%rdx){1to8}, %zmm29, %zmm30
-//CHECK: vpmultishiftqb 1024(%rdx){1to8}, %zmm29, %zmm30
-//CHECK: encoding: [0x62,0x62,0x95,0x50,0x83,0xb2,0x00,0x04,0x00,0x00]
-
-  vpmultishiftqb -0x400(%rdx){1to8}, %zmm29, %zmm30
-//CHECK: vpmultishiftqb -1024(%rdx){1to8}, %zmm29, %zmm30
-//CHECK: encoding: [0x62,0x62,0x95,0x50,0x83,0x72,0x80]
-
-  vpmultishiftqb -0x408(%rdx){1to8}, %zmm29, %zmm30
-//CHECK: vpmultishiftqb -1032(%rdx){1to8}, %zmm29, %zmm30
-//CHECK: encoding: [0x62,0x62,0x95,0x50,0x83,0xb2,0xf8,0xfb,0xff,0xff]
-
-  vpmultishiftqb 0x1234(%rax,%r14,8), %zmm29, %zmm30
-//CHECK: vpmultishiftqb 4660(%rax,%r14,8), %zmm29, %zmm30
-//CHECK: encoding: [0x62,0x22,0x95,0x40,0x83,0xb4,0xf0,0x34,0x12,0x00,0x00]
-
diff --git a/llvm/test/MC/X86/avx512vbmi-intel.s b/llvm/test/MC/X86/avx512vbmi-intel.s
new file mode 100644
index 0000000000000000000000000000000000000000..e17b06e4c9970d794b24e93d12a8bec128763db4
--- /dev/null
+++ b/llvm/test/MC/X86/avx512vbmi-intel.s
@@ -0,0 +1,542 @@
+# RUN: llvm-mc -triple x86_64 -show-encoding -x86-asm-syntax=intel -output-asm-variant=1 %s | FileCheck %s
+
+# CHECK: vpermb	xmm30 {k7}, xmm29, xmm28
+# CHECK: encoding: [0x62,0x02,0x15,0x07,0x8d,0xf4]
+         vpermb	xmm30 {k7}, xmm29, xmm28
+
+# CHECK: vpermb	xmm30 {k7} {z}, xmm29, xmm28
+# CHECK: encoding: [0x62,0x02,0x15,0x87,0x8d,0xf4]
+         vpermb	xmm30 {k7} {z}, xmm29, xmm28
+
+# CHECK: vpermb	xmm30, xmm29, xmmword ptr [rcx]
+# CHECK: encoding: [0x62,0x62,0x15,0x00,0x8d,0x31]
+         vpermb	xmm30, xmm29, xmmword ptr [rcx]
+
+# CHECK: vpermb	xmm30, xmm29, xmmword ptr [rax + 8*r14 + 291]
+# CHECK: encoding: [0x62,0x22,0x15,0x00,0x8d,0xb4,0xf0,0x23,0x01,0x00,0x00]
+         vpermb	xmm30, xmm29, xmmword ptr [rax + 8*r14 + 291]
+
+# CHECK: vpermb	xmm30, xmm29, xmmword ptr [rdx + 2032]
+# CHECK: encoding: [0x62,0x62,0x15,0x00,0x8d,0x72,0x7f]
+         vpermb	xmm30, xmm29, xmmword ptr [rdx + 2032]
+
+# CHECK: vpermb	xmm30, xmm29, xmmword ptr [rdx + 2048]
+# CHECK: encoding: [0x62,0x62,0x15,0x00,0x8d,0xb2,0x00,0x08,0x00,0x00]
+         vpermb	xmm30, xmm29, xmmword ptr [rdx + 2048]
+
+# CHECK: vpermb	xmm30, xmm29, xmmword ptr [rdx - 2048]
+# CHECK: encoding: [0x62,0x62,0x15,0x00,0x8d,0x72,0x80]
+         vpermb	xmm30, xmm29, xmmword ptr [rdx - 2048]
+
+# CHECK: vpermb	xmm30, xmm29, xmmword ptr [rdx - 2064]
+# CHECK: encoding: [0x62,0x62,0x15,0x00,0x8d,0xb2,0xf0,0xf7,0xff,0xff]
+         vpermb	xmm30, xmm29, xmmword ptr [rdx - 2064]
+
+# CHECK: vpermb	ymm30, ymm29, ymm28
+# CHECK: encoding: [0x62,0x02,0x15,0x20,0x8d,0xf4]
+         vpermb	ymm30, ymm29, ymm28
+
+# CHECK: vpermb	ymm30 {k7}, ymm29, ymm28
+# CHECK: encoding: [0x62,0x02,0x15,0x27,0x8d,0xf4]
+         vpermb	ymm30 {k7}, ymm29, ymm28
+
+# CHECK: vpermb	ymm30 {k7} {z}, ymm29, ymm28
+# CHECK: encoding: [0x62,0x02,0x15,0xa7,0x8d,0xf4]
+         vpermb	ymm30 {k7} {z}, ymm29, ymm28
+
+# CHECK: vpermb	ymm30, ymm29, ymmword ptr [rcx]
+# CHECK: encoding: [0x62,0x62,0x15,0x20,0x8d,0x31]
+         vpermb	ymm30, ymm29, ymmword ptr [rcx]
+
+# CHECK: vpermb	ymm30, ymm29, ymmword ptr [rax + 8*r14 + 291]
+# CHECK: encoding: [0x62,0x22,0x15,0x20,0x8d,0xb4,0xf0,0x23,0x01,0x00,0x00]
+         vpermb	ymm30, ymm29, ymmword ptr [rax + 8*r14 + 291]
+
+# CHECK: vpermb	ymm30, ymm29, ymmword ptr [rdx + 4064]
+# CHECK: encoding: [0x62,0x62,0x15,0x20,0x8d,0x72,0x7f]
+         vpermb	ymm30, ymm29, ymmword ptr [rdx + 4064]
+
+# CHECK: vpermb	ymm30, ymm29, ymmword ptr [rdx + 4096]
+# CHECK: encoding: [0x62,0x62,0x15,0x20,0x8d,0xb2,0x00,0x10,0x00,0x00]
+         vpermb	ymm30, ymm29, ymmword ptr [rdx + 4096]
+
+# CHECK: vpermb	ymm30, ymm29, ymmword ptr [rdx - 4096]
+# CHECK: encoding: [0x62,0x62,0x15,0x20,0x8d,0x72,0x80]
+         vpermb	ymm30, ymm29, ymmword ptr [rdx - 4096]
+
+# CHECK: vpermb	ymm30, ymm29, ymmword ptr [rdx - 4128]
+# CHECK: encoding: [0x62,0x62,0x15,0x20,0x8d,0xb2,0xe0,0xef,0xff,0xff]
+         vpermb	ymm30, ymm29, ymmword ptr [rdx - 4128]
+
+# CHECK: vpermb	xmm30, xmm29, xmm28
+# CHECK: encoding: [0x62,0x02,0x15,0x00,0x8d,0xf4]
+         vpermb	xmm30, xmm29, xmm28
+
+# CHECK: vpermb	xmm30, xmm29, xmmword ptr [rax + 8*r14 + 4660]
+# CHECK: encoding: [0x62,0x22,0x15,0x00,0x8d,0xb4,0xf0,0x34,0x12,0x00,0x00]
+         vpermb	xmm30, xmm29, xmmword ptr [rax + 8*r14 + 4660]
+
+# CHECK: vpermb	ymm30, ymm29, ymmword ptr [rax + 8*r14 + 4660]
+# CHECK: encoding: [0x62,0x22,0x15,0x20,0x8d,0xb4,0xf0,0x34,0x12,0x00,0x00]
+         vpermb	ymm30, ymm29, ymmword ptr [rax + 8*r14 + 4660]
+
+# CHECK: vpermb	zmm30, zmm29, zmm28
+# CHECK: encoding: [0x62,0x02,0x15,0x40,0x8d,0xf4]
+         vpermb	zmm30, zmm29, zmm28
+
+# CHECK: vpermb	zmm30 {k7}, zmm29, zmm28
+# CHECK: encoding: [0x62,0x02,0x15,0x47,0x8d,0xf4]
+         vpermb	zmm30 {k7}, zmm29, zmm28
+
+# CHECK: vpermb	zmm30 {k7} {z}, zmm29, zmm28
+# CHECK: encoding: [0x62,0x02,0x15,0xc7,0x8d,0xf4]
+         vpermb	zmm30 {k7} {z}, zmm29, zmm28
+
+# CHECK: vpermb	zmm30, zmm29, zmmword ptr [rcx]
+# CHECK: encoding: [0x62,0x62,0x15,0x40,0x8d,0x31]
+         vpermb	zmm30, zmm29, zmmword ptr [rcx]
+
+# CHECK: vpermb	zmm30, zmm29, zmmword ptr [rax + 8*r14 + 291]
+# CHECK: encoding: [0x62,0x22,0x15,0x40,0x8d,0xb4,0xf0,0x23,0x01,0x00,0x00]
+         vpermb	zmm30, zmm29, zmmword ptr [rax + 8*r14 + 291]
+
+# CHECK: vpermb	zmm30, zmm29, zmmword ptr [rdx + 8128]
+# CHECK: encoding: [0x62,0x62,0x15,0x40,0x8d,0x72,0x7f]
+         vpermb	zmm30, zmm29, zmmword ptr [rdx + 8128]
+
+# CHECK: vpermb	zmm30, zmm29, zmmword ptr [rdx + 8192]
+# CHECK: encoding: [0x62,0x62,0x15,0x40,0x8d,0xb2,0x00,0x20,0x00,0x00]
+         vpermb	zmm30, zmm29, zmmword ptr [rdx + 8192]
+
+# CHECK: vpermb	zmm30, zmm29, zmmword ptr [rdx - 8192]
+# CHECK: encoding: [0x62,0x62,0x15,0x40,0x8d,0x72,0x80]
+         vpermb	zmm30, zmm29, zmmword ptr [rdx - 8192]
+
+# CHECK: vpermb	zmm30, zmm29, zmmword ptr [rdx - 8256]
+# CHECK: encoding: [0x62,0x62,0x15,0x40,0x8d,0xb2,0xc0,0xdf,0xff,0xff]
+         vpermb	zmm30, zmm29, zmmword ptr [rdx - 8256]
+
+# CHECK: vpermb	zmm30, zmm29, zmmword ptr [rax + 8*r14 + 4660]
+# CHECK: encoding: [0x62,0x22,0x15,0x40,0x8d,0xb4,0xf0,0x34,0x12,0x00,0x00]
+         vpermb	zmm30, zmm29, zmmword ptr [rax + 8*r14 + 4660]
+
+# CHECK: vpermt2b	xmm30, xmm29, xmm28
+# CHECK: encoding: [0x62,0x02,0x15,0x00,0x7d,0xf4]
+         vpermt2b	xmm30, xmm29, xmm28
+
+# CHECK: vpermt2b	xmm30 {k7}, xmm29, xmm28
+# CHECK: encoding: [0x62,0x02,0x15,0x07,0x7d,0xf4]
+         vpermt2b	xmm30 {k7}, xmm29, xmm28
+
+# CHECK: vpermt2b	xmm30 {k7} {z}, xmm29, xmm28
+# CHECK: encoding: [0x62,0x02,0x15,0x87,0x7d,0xf4]
+         vpermt2b	xmm30 {k7} {z}, xmm29, xmm28
+
+# CHECK: vpermt2b	xmm30, xmm29, xmmword ptr [rcx]
+# CHECK: encoding: [0x62,0x62,0x15,0x00,0x7d,0x31]
+         vpermt2b	xmm30, xmm29, xmmword ptr [rcx]
+
+# CHECK: vpermt2b	xmm30, xmm29, xmmword ptr [rax + 8*r14 + 291]
+# CHECK: encoding: [0x62,0x22,0x15,0x00,0x7d,0xb4,0xf0,0x23,0x01,0x00,0x00]
+         vpermt2b	xmm30, xmm29, xmmword ptr [rax + 8*r14 + 291]
+
+# CHECK: vpermt2b	xmm30, xmm29, xmmword ptr [rdx + 2032]
+# CHECK: encoding: [0x62,0x62,0x15,0x00,0x7d,0x72,0x7f]
+         vpermt2b	xmm30, xmm29, xmmword ptr [rdx + 2032]
+
+# CHECK: vpermt2b	xmm30, xmm29, xmmword ptr [rdx + 2048]
+# CHECK: encoding: [0x62,0x62,0x15,0x00,0x7d,0xb2,0x00,0x08,0x00,0x00]
+         vpermt2b	xmm30, xmm29, xmmword ptr [rdx + 2048]
+
+# CHECK: vpermt2b	xmm30, xmm29, xmmword ptr [rdx - 2048]
+# CHECK: encoding: [0x62,0x62,0x15,0x00,0x7d,0x72,0x80]
+         vpermt2b	xmm30, xmm29, xmmword ptr [rdx - 2048]
+
+# CHECK: vpermt2b	xmm30, xmm29, xmmword ptr [rdx - 2064]
+# CHECK: encoding: [0x62,0x62,0x15,0x00,0x7d,0xb2,0xf0,0xf7,0xff,0xff]
+         vpermt2b	xmm30, xmm29, xmmword ptr [rdx - 2064]
+
+# CHECK: vpermt2b	ymm30, ymm29, ymm28
+# CHECK: encoding: [0x62,0x02,0x15,0x20,0x7d,0xf4]
+         vpermt2b	ymm30, ymm29, ymm28
+
+# CHECK: vpermt2b	ymm30 {k7}, ymm29, ymm28
+# CHECK: encoding: [0x62,0x02,0x15,0x27,0x7d,0xf4]
+         vpermt2b	ymm30 {k7}, ymm29, ymm28
+
+# CHECK: vpermt2b	ymm30 {k7} {z}, ymm29, ymm28
+# CHECK: encoding: [0x62,0x02,0x15,0xa7,0x7d,0xf4]
+         vpermt2b	ymm30 {k7} {z}, ymm29, ymm28
+
+# CHECK: vpermt2b	ymm30, ymm29, ymmword ptr [rcx]
+# CHECK: encoding: [0x62,0x62,0x15,0x20,0x7d,0x31]
+         vpermt2b	ymm30, ymm29, ymmword ptr [rcx]
+
+# CHECK: vpermt2b	ymm30, ymm29, ymmword ptr [rax + 8*r14 + 291]
+# CHECK: encoding: [0x62,0x22,0x15,0x20,0x7d,0xb4,0xf0,0x23,0x01,0x00,0x00]
+         vpermt2b	ymm30, ymm29, ymmword ptr [rax + 8*r14 + 291]
+
+# CHECK: vpermt2b	ymm30, ymm29, ymmword ptr [rdx + 4064]
+# CHECK: encoding: [0x62,0x62,0x15,0x20,0x7d,0x72,0x7f]
+         vpermt2b	ymm30, ymm29, ymmword ptr [rdx + 4064]
+
+# CHECK: vpermt2b	ymm30, ymm29, ymmword ptr [rdx + 4096]
+# CHECK: encoding: [0x62,0x62,0x15,0x20,0x7d,0xb2,0x00,0x10,0x00,0x00]
+         vpermt2b	ymm30, ymm29, ymmword ptr [rdx + 4096]
+
+# CHECK: vpermt2b	ymm30, ymm29, ymmword ptr [rdx - 4096]
+# CHECK: encoding: [0x62,0x62,0x15,0x20,0x7d,0x72,0x80]
+         vpermt2b	ymm30, ymm29, ymmword ptr [rdx - 4096]
+
+# CHECK: vpermt2b	ymm30, ymm29, ymmword ptr [rdx - 4128]
+# CHECK: encoding: [0x62,0x62,0x15,0x20,0x7d,0xb2,0xe0,0xef,0xff,0xff]
+         vpermt2b	ymm30, ymm29, ymmword ptr [rdx - 4128]
+
+# CHECK: vpermt2b	xmm30, xmm29, xmmword ptr [rax + 8*r14 + 4660]
+# CHECK: encoding: [0x62,0x22,0x15,0x00,0x7d,0xb4,0xf0,0x34,0x12,0x00,0x00]
+         vpermt2b	xmm30, xmm29, xmmword ptr [rax + 8*r14 + 4660]
+
+# CHECK: vpermt2b	ymm30, ymm29, ymmword ptr [rax + 8*r14 + 4660]
+# CHECK: encoding: [0x62,0x22,0x15,0x20,0x7d,0xb4,0xf0,0x34,0x12,0x00,0x00]
+         vpermt2b	ymm30, ymm29, ymmword ptr [rax + 8*r14 + 4660]
+
+# CHECK: vpermt2b	zmm30, zmm29, zmm28
+# CHECK: encoding: [0x62,0x02,0x15,0x40,0x7d,0xf4]
+         vpermt2b	zmm30, zmm29, zmm28
+
+# CHECK: vpermt2b	zmm30 {k7}, zmm29, zmm28
+# CHECK: encoding: [0x62,0x02,0x15,0x47,0x7d,0xf4]
+         vpermt2b	zmm30 {k7}, zmm29, zmm28
+
+# CHECK: vpermt2b	zmm30 {k7} {z}, zmm29, zmm28
+# CHECK: encoding: [0x62,0x02,0x15,0xc7,0x7d,0xf4]
+         vpermt2b	zmm30 {k7} {z}, zmm29, zmm28
+
+# CHECK: vpermt2b	zmm30, zmm29, zmmword ptr [rcx]
+# CHECK: encoding: [0x62,0x62,0x15,0x40,0x7d,0x31]
+         vpermt2b	zmm30, zmm29, zmmword ptr [rcx]
+
+# CHECK: vpermt2b	zmm30, zmm29, zmmword ptr [rax + 8*r14 + 291]
+# CHECK: encoding: [0x62,0x22,0x15,0x40,0x7d,0xb4,0xf0,0x23,0x01,0x00,0x00]
+         vpermt2b	zmm30, zmm29, zmmword ptr [rax + 8*r14 + 291]
+
+# CHECK: vpermt2b	zmm30, zmm29, zmmword ptr [rdx + 8128]
+# CHECK: encoding: [0x62,0x62,0x15,0x40,0x7d,0x72,0x7f]
+         vpermt2b	zmm30, zmm29, zmmword ptr [rdx + 8128]
+
+# CHECK: vpermt2b	zmm30, zmm29, zmmword ptr [rdx + 8192]
+# CHECK: encoding: [0x62,0x62,0x15,0x40,0x7d,0xb2,0x00,0x20,0x00,0x00]
+         vpermt2b	zmm30, zmm29, zmmword ptr [rdx + 8192]
+
+# CHECK: vpermt2b	zmm30, zmm29, zmmword ptr [rdx - 8192]
+# CHECK: encoding: [0x62,0x62,0x15,0x40,0x7d,0x72,0x80]
+         vpermt2b	zmm30, zmm29, zmmword ptr [rdx - 8192]
+
+# CHECK: vpermt2b	zmm30, zmm29, zmmword ptr [rdx - 8256]
+# CHECK: encoding: [0x62,0x62,0x15,0x40,0x7d,0xb2,0xc0,0xdf,0xff,0xff]
+         vpermt2b	zmm30, zmm29, zmmword ptr [rdx - 8256]
+
+# CHECK: vpermt2b	zmm30, zmm29, zmmword ptr [rax + 8*r14 + 4660]
+# CHECK: encoding: [0x62,0x22,0x15,0x40,0x7d,0xb4,0xf0,0x34,0x12,0x00,0x00]
+         vpermt2b	zmm30, zmm29, zmmword ptr [rax + 8*r14 + 4660]
+
+# CHECK: vpermi2b	xmm30, xmm29, xmm28
+# CHECK: encoding: [0x62,0x02,0x15,0x00,0x75,0xf4]
+         vpermi2b	xmm30, xmm29, xmm28
+
+# CHECK: vpermi2b	xmm30 {k7}, xmm29, xmm28
+# CHECK: encoding: [0x62,0x02,0x15,0x07,0x75,0xf4]
+         vpermi2b	xmm30 {k7}, xmm29, xmm28
+
+# CHECK: vpermi2b	xmm30 {k7} {z}, xmm29, xmm28
+# CHECK: encoding: [0x62,0x02,0x15,0x87,0x75,0xf4]
+         vpermi2b	xmm30 {k7} {z}, xmm29, xmm28
+
+# CHECK: vpermi2b	xmm30, xmm29, xmmword ptr [rcx]
+# CHECK: encoding: [0x62,0x62,0x15,0x00,0x75,0x31]
+         vpermi2b	xmm30, xmm29, xmmword ptr [rcx]
+
+# CHECK: vpermi2b	xmm30, xmm29, xmmword ptr [rax + 8*r14 + 291]
+# CHECK: encoding: [0x62,0x22,0x15,0x00,0x75,0xb4,0xf0,0x23,0x01,0x00,0x00]
+         vpermi2b	xmm30, xmm29, xmmword ptr [rax + 8*r14 + 291]
+
+# CHECK: vpermi2b	xmm30, xmm29, xmmword ptr [rdx + 2032]
+# CHECK: encoding: [0x62,0x62,0x15,0x00,0x75,0x72,0x7f]
+         vpermi2b	xmm30, xmm29, xmmword ptr [rdx + 2032]
+
+# CHECK: vpermi2b	xmm30, xmm29, xmmword ptr [rdx + 2048]
+# CHECK: encoding: [0x62,0x62,0x15,0x00,0x75,0xb2,0x00,0x08,0x00,0x00]
+         vpermi2b	xmm30, xmm29, xmmword ptr [rdx + 2048]
+
+# CHECK: vpermi2b	xmm30, xmm29, xmmword ptr [rdx - 2048]
+# CHECK: encoding: [0x62,0x62,0x15,0x00,0x75,0x72,0x80]
+         vpermi2b	xmm30, xmm29, xmmword ptr [rdx - 2048]
+
+# CHECK: vpermi2b	xmm30, xmm29, xmmword ptr [rdx - 2064]
+# CHECK: encoding: [0x62,0x62,0x15,0x00,0x75,0xb2,0xf0,0xf7,0xff,0xff]
+         vpermi2b	xmm30, xmm29, xmmword ptr [rdx - 2064]
+
+# CHECK: vpermi2b	ymm30, ymm29, ymm28
+# CHECK: encoding: [0x62,0x02,0x15,0x20,0x75,0xf4]
+         vpermi2b	ymm30, ymm29, ymm28
+
+# CHECK: vpermi2b	ymm30 {k7}, ymm29, ymm28
+# CHECK: encoding: [0x62,0x02,0x15,0x27,0x75,0xf4]
+         vpermi2b	ymm30 {k7}, ymm29, ymm28
+
+# CHECK: vpermi2b	ymm30 {k7} {z}, ymm29, ymm28
+# CHECK: encoding: [0x62,0x02,0x15,0xa7,0x75,0xf4]
+         vpermi2b	ymm30 {k7} {z}, ymm29, ymm28
+
+# CHECK: vpermi2b	ymm30, ymm29, ymmword ptr [rcx]
+# CHECK: encoding: [0x62,0x62,0x15,0x20,0x75,0x31]
+         vpermi2b	ymm30, ymm29, ymmword ptr [rcx]
+
+# CHECK: vpermi2b	ymm30, ymm29, ymmword ptr [rax + 8*r14 + 291]
+# CHECK: encoding: [0x62,0x22,0x15,0x20,0x75,0xb4,0xf0,0x23,0x01,0x00,0x00]
+         vpermi2b	ymm30, ymm29, ymmword ptr [rax + 8*r14 + 291]
+
+# CHECK: vpermi2b	ymm30, ymm29, ymmword ptr [rdx + 4064]
+# CHECK: encoding: [0x62,0x62,0x15,0x20,0x75,0x72,0x7f]
+         vpermi2b	ymm30, ymm29, ymmword ptr [rdx + 4064]
+
+# CHECK: vpermi2b	ymm30, ymm29, ymmword ptr [rdx + 4096]
+# CHECK: encoding: [0x62,0x62,0x15,0x20,0x75,0xb2,0x00,0x10,0x00,0x00]
+         vpermi2b	ymm30, ymm29, ymmword ptr [rdx + 4096]
+
+# CHECK: vpermi2b	ymm30, ymm29, ymmword ptr [rdx - 4096]
+# CHECK: encoding: [0x62,0x62,0x15,0x20,0x75,0x72,0x80]
+         vpermi2b	ymm30, ymm29, ymmword ptr [rdx - 4096]
+
+# CHECK: vpermi2b	ymm30, ymm29, ymmword ptr [rdx - 4128]
+# CHECK: encoding: [0x62,0x62,0x15,0x20,0x75,0xb2,0xe0,0xef,0xff,0xff]
+         vpermi2b	ymm30, ymm29, ymmword ptr [rdx - 4128]
+
+# CHECK: vpermi2b	xmm30, xmm29, xmmword ptr [rax + 8*r14 + 4660]
+# CHECK: encoding: [0x62,0x22,0x15,0x00,0x75,0xb4,0xf0,0x34,0x12,0x00,0x00]
+         vpermi2b	xmm30, xmm29, xmmword ptr [rax + 8*r14 + 4660]
+
+# CHECK: vpermi2b	ymm30, ymm29, ymmword ptr [rax + 8*r14 + 4660]
+# CHECK: encoding: [0x62,0x22,0x15,0x20,0x75,0xb4,0xf0,0x34,0x12,0x00,0x00]
+         vpermi2b	ymm30, ymm29, ymmword ptr [rax + 8*r14 + 4660]
+
+# CHECK: vpermi2b	zmm30, zmm29, zmm28
+# CHECK: encoding: [0x62,0x02,0x15,0x40,0x75,0xf4]
+         vpermi2b	zmm30, zmm29, zmm28
+
+# CHECK: vpermi2b	zmm30 {k7}, zmm29, zmm28
+# CHECK: encoding: [0x62,0x02,0x15,0x47,0x75,0xf4]
+         vpermi2b	zmm30 {k7}, zmm29, zmm28
+
+# CHECK: vpermi2b	zmm30 {k7} {z}, zmm29, zmm28
+# CHECK: encoding: [0x62,0x02,0x15,0xc7,0x75,0xf4]
+         vpermi2b	zmm30 {k7} {z}, zmm29, zmm28
+
+# CHECK: vpermi2b	zmm30, zmm29, zmmword ptr [rcx]
+# CHECK: encoding: [0x62,0x62,0x15,0x40,0x75,0x31]
+         vpermi2b	zmm30, zmm29, zmmword ptr [rcx]
+
+# CHECK: vpermi2b	zmm30, zmm29, zmmword ptr [rax + 8*r14 + 291]
+# CHECK: encoding: [0x62,0x22,0x15,0x40,0x75,0xb4,0xf0,0x23,0x01,0x00,0x00]
+         vpermi2b	zmm30, zmm29, zmmword ptr [rax + 8*r14 + 291]
+
+# CHECK: vpermi2b	zmm30, zmm29, zmmword ptr [rdx + 8128]
+# CHECK: encoding: [0x62,0x62,0x15,0x40,0x75,0x72,0x7f]
+         vpermi2b	zmm30, zmm29, zmmword ptr [rdx + 8128]
+
+# CHECK: vpermi2b	zmm30, zmm29, zmmword ptr [rdx + 8192]
+# CHECK: encoding: [0x62,0x62,0x15,0x40,0x75,0xb2,0x00,0x20,0x00,0x00]
+         vpermi2b	zmm30, zmm29, zmmword ptr [rdx + 8192]
+
+# CHECK: vpermi2b	zmm30, zmm29, zmmword ptr [rdx - 8192]
+# CHECK: encoding: [0x62,0x62,0x15,0x40,0x75,0x72,0x80]
+         vpermi2b	zmm30, zmm29, zmmword ptr [rdx - 8192]
+
+# CHECK: vpermi2b	zmm30, zmm29, zmmword ptr [rdx - 8256]
+# CHECK: encoding: [0x62,0x62,0x15,0x40,0x75,0xb2,0xc0,0xdf,0xff,0xff]
+         vpermi2b	zmm30, zmm29, zmmword ptr [rdx - 8256]
+
+# CHECK: vpermi2b	zmm30, zmm29, zmmword ptr [rax + 8*r14 + 4660]
+# CHECK: encoding: [0x62,0x22,0x15,0x40,0x75,0xb4,0xf0,0x34,0x12,0x00,0x00]
+         vpermi2b	zmm30, zmm29, zmmword ptr [rax + 8*r14 + 4660]
+
+# CHECK: vpmultishiftqb	xmm30, xmm29, xmm28
+# CHECK: encoding: [0x62,0x02,0x95,0x00,0x83,0xf4]
+         vpmultishiftqb	xmm30, xmm29, xmm28
+
+# CHECK: vpmultishiftqb	xmm30 {k7}, xmm29, xmm28
+# CHECK: encoding: [0x62,0x02,0x95,0x07,0x83,0xf4]
+         vpmultishiftqb	xmm30 {k7}, xmm29, xmm28
+
+# CHECK: vpmultishiftqb	xmm30 {k7} {z}, xmm29, xmm28
+# CHECK: encoding: [0x62,0x02,0x95,0x87,0x83,0xf4]
+         vpmultishiftqb	xmm30 {k7} {z}, xmm29, xmm28
+
+# CHECK: vpmultishiftqb	xmm30, xmm29, xmmword ptr [rcx]
+# CHECK: encoding: [0x62,0x62,0x95,0x00,0x83,0x31]
+         vpmultishiftqb	xmm30, xmm29, xmmword ptr [rcx]
+
+# CHECK: vpmultishiftqb	xmm30, xmm29, xmmword ptr [rax + 8*r14 + 291]
+# CHECK: encoding: [0x62,0x22,0x95,0x00,0x83,0xb4,0xf0,0x23,0x01,0x00,0x00]
+         vpmultishiftqb	xmm30, xmm29, xmmword ptr [rax + 8*r14 + 291]
+
+# CHECK: vpmultishiftqb	xmm30, xmm29, qword ptr [rcx]{1to2}
+# CHECK: encoding: [0x62,0x62,0x95,0x10,0x83,0x31]
+         vpmultishiftqb	xmm30, xmm29, qword ptr [rcx]{1to2}
+
+
+# CHECK: vpmultishiftqb	xmm30, xmm29, xmmword ptr [rdx + 2032]
+# CHECK: encoding: [0x62,0x62,0x95,0x00,0x83,0x72,0x7f]
+         vpmultishiftqb	xmm30, xmm29, xmmword ptr [rdx + 2032]
+
+# CHECK: vpmultishiftqb	xmm30, xmm29, xmmword ptr [rdx + 2048]
+# CHECK: encoding: [0x62,0x62,0x95,0x00,0x83,0xb2,0x00,0x08,0x00,0x00]
+         vpmultishiftqb	xmm30, xmm29, xmmword ptr [rdx + 2048]
+
+# CHECK: vpmultishiftqb	xmm30, xmm29, xmmword ptr [rdx - 2048]
+# CHECK: encoding: [0x62,0x62,0x95,0x00,0x83,0x72,0x80]
+         vpmultishiftqb	xmm30, xmm29, xmmword ptr [rdx - 2048]
+
+# CHECK: vpmultishiftqb	xmm30, xmm29, xmmword ptr [rdx - 2064]
+# CHECK: encoding: [0x62,0x62,0x95,0x00,0x83,0xb2,0xf0,0xf7,0xff,0xff]
+         vpmultishiftqb	xmm30, xmm29, xmmword ptr [rdx - 2064]
+
+# CHECK: vpmultishiftqb	xmm30, xmm29, qword ptr [rdx + 1016]{1to2}
+# CHECK: encoding: [0x62,0x62,0x95,0x10,0x83,0x72,0x7f]
+         vpmultishiftqb	xmm30, xmm29, qword ptr [rdx + 1016]{1to2}
+
+# CHECK: vpmultishiftqb	xmm30, xmm29, qword ptr [rdx + 1024]{1to2}
+# CHECK: encoding: [0x62,0x62,0x95,0x10,0x83,0xb2,0x00,0x04,0x00,0x00]
+         vpmultishiftqb	xmm30, xmm29, qword ptr [rdx + 1024]{1to2}
+
+# CHECK: vpmultishiftqb	xmm30, xmm29, qword ptr [rdx - 1024]{1to2}
+# CHECK: encoding: [0x62,0x62,0x95,0x10,0x83,0x72,0x80]
+         vpmultishiftqb	xmm30, xmm29, qword ptr [rdx - 1024]{1to2}
+
+# CHECK: vpmultishiftqb	xmm30, xmm29, qword ptr [rdx - 1032]{1to2}
+# CHECK: encoding: [0x62,0x62,0x95,0x10,0x83,0xb2,0xf8,0xfb,0xff,0xff]
+         vpmultishiftqb	xmm30, xmm29, qword ptr [rdx - 1032]{1to2}
+
+# CHECK: vpmultishiftqb	ymm30, ymm29, ymm28
+# CHECK: encoding: [0x62,0x02,0x95,0x20,0x83,0xf4]
+         vpmultishiftqb	ymm30, ymm29, ymm28
+
+# CHECK: vpmultishiftqb	ymm30 {k7}, ymm29, ymm28
+# CHECK: encoding: [0x62,0x02,0x95,0x27,0x83,0xf4]
+         vpmultishiftqb	ymm30 {k7}, ymm29, ymm28
+
+# CHECK: vpmultishiftqb	ymm30 {k7} {z}, ymm29, ymm28
+# CHECK: encoding: [0x62,0x02,0x95,0xa7,0x83,0xf4]
+         vpmultishiftqb	ymm30 {k7} {z}, ymm29, ymm28
+
+# CHECK: vpmultishiftqb	ymm30, ymm29, ymmword ptr [rcx]
+# CHECK: encoding: [0x62,0x62,0x95,0x20,0x83,0x31]
+         vpmultishiftqb	ymm30, ymm29, ymmword ptr [rcx]
+
+# CHECK: vpmultishiftqb	ymm30, ymm29, ymmword ptr [rax + 8*r14 + 291]
+# CHECK: encoding: [0x62,0x22,0x95,0x20,0x83,0xb4,0xf0,0x23,0x01,0x00,0x00]
+         vpmultishiftqb	ymm30, ymm29, ymmword ptr [rax + 8*r14 + 291]
+
+# CHECK: vpmultishiftqb	ymm30, ymm29, qword ptr [rcx]{1to4}
+# CHECK: encoding: [0x62,0x62,0x95,0x30,0x83,0x31]
+         vpmultishiftqb	ymm30, ymm29, qword ptr [rcx]{1to4}
+
+# CHECK: vpmultishiftqb	ymm30, ymm29, ymmword ptr [rdx + 4064]
+# CHECK: encoding: [0x62,0x62,0x95,0x20,0x83,0x72,0x7f]
+         vpmultishiftqb	ymm30, ymm29, ymmword ptr [rdx + 4064]
+
+# CHECK: vpmultishiftqb	ymm30, ymm29, ymmword ptr [rdx + 4096]
+# CHECK: encoding: [0x62,0x62,0x95,0x20,0x83,0xb2,0x00,0x10,0x00,0x00]
+         vpmultishiftqb	ymm30, ymm29, ymmword ptr [rdx + 4096]
+
+# CHECK: vpmultishiftqb	ymm30, ymm29, ymmword ptr [rdx - 4096]
+# CHECK: encoding: [0x62,0x62,0x95,0x20,0x83,0x72,0x80]
+         vpmultishiftqb	ymm30, ymm29, ymmword ptr [rdx - 4096]
+
+# CHECK: vpmultishiftqb	ymm30, ymm29, ymmword ptr [rdx - 4128]
+# CHECK: encoding: [0x62,0x62,0x95,0x20,0x83,0xb2,0xe0,0xef,0xff,0xff]
+         vpmultishiftqb	ymm30, ymm29, ymmword ptr [rdx - 4128]
+
+# CHECK: vpmultishiftqb	ymm30, ymm29, qword ptr [rdx + 1016]{1to4}
+# CHECK: encoding: [0x62,0x62,0x95,0x30,0x83,0x72,0x7f]
+         vpmultishiftqb	ymm30, ymm29, qword ptr [rdx + 1016]{1to4}
+
+# CHECK: vpmultishiftqb	ymm30, ymm29, qword ptr [rdx + 1024]{1to4}
+# CHECK: encoding: [0x62,0x62,0x95,0x30,0x83,0xb2,0x00,0x04,0x00,0x00]
+         vpmultishiftqb	ymm30, ymm29, qword ptr [rdx + 1024]{1to4}
+
+# CHECK: vpmultishiftqb	ymm30, ymm29, qword ptr [rdx - 1024]{1to4}
+# CHECK: encoding: [0x62,0x62,0x95,0x30,0x83,0x72,0x80]
+         vpmultishiftqb	ymm30, ymm29, qword ptr [rdx - 1024]{1to4}
+
+# CHECK: vpmultishiftqb	ymm30, ymm29, qword ptr [rdx - 1032]{1to4}
+# CHECK: encoding: [0x62,0x62,0x95,0x30,0x83,0xb2,0xf8,0xfb,0xff,0xff]
+         vpmultishiftqb	ymm30, ymm29, qword ptr [rdx - 1032]{1to4}
+
+# CHECK: vpmultishiftqb	xmm30, xmm29, xmmword ptr [rax + 8*r14 + 4660]
+# CHECK: encoding: [0x62,0x22,0x95,0x00,0x83,0xb4,0xf0,0x34,0x12,0x00,0x00]
+         vpmultishiftqb	xmm30, xmm29, xmmword ptr [rax + 8*r14 + 4660]
+
+# CHECK: vpmultishiftqb	ymm30, ymm29, ymmword ptr [rax + 8*r14 + 4660]
+# CHECK: encoding: [0x62,0x22,0x95,0x20,0x83,0xb4,0xf0,0x34,0x12,0x00,0x00]
+         vpmultishiftqb	ymm30, ymm29, ymmword ptr [rax + 8*r14 + 4660]
+
+# CHECK: vpmultishiftqb	zmm30, zmm29, zmm28
+# CHECK: encoding: [0x62,0x02,0x95,0x40,0x83,0xf4]
+         vpmultishiftqb	zmm30, zmm29, zmm28
+
+# CHECK: vpmultishiftqb	zmm30 {k7}, zmm29, zmm28
+# CHECK: encoding: [0x62,0x02,0x95,0x47,0x83,0xf4]
+         vpmultishiftqb	zmm30 {k7}, zmm29, zmm28
+
+# CHECK: vpmultishiftqb	zmm30 {k7} {z}, zmm29, zmm28
+# CHECK: encoding: [0x62,0x02,0x95,0xc7,0x83,0xf4]
+         vpmultishiftqb	zmm30 {k7} {z}, zmm29, zmm28
+
+# CHECK: vpmultishiftqb	zmm30, zmm29, zmmword ptr [rcx]
+# CHECK: encoding: [0x62,0x62,0x95,0x40,0x83,0x31]
+         vpmultishiftqb	zmm30, zmm29, zmmword ptr [rcx]
+
+# CHECK: vpmultishiftqb	zmm30, zmm29, zmmword ptr [rax + 8*r14 + 291]
+# CHECK: encoding: [0x62,0x22,0x95,0x40,0x83,0xb4,0xf0,0x23,0x01,0x00,0x00]
+         vpmultishiftqb	zmm30, zmm29, zmmword ptr [rax + 8*r14 + 291]
+
+# CHECK: vpmultishiftqb	zmm30, zmm29, qword ptr [rcx]{1to8}
+# CHECK: encoding: [0x62,0x62,0x95,0x50,0x83,0x31]
+         vpmultishiftqb	zmm30, zmm29, qword ptr [rcx]{1to8}
+
+# CHECK: vpmultishiftqb	zmm30, zmm29, zmmword ptr [rdx + 8128]
+# CHECK: encoding: [0x62,0x62,0x95,0x40,0x83,0x72,0x7f]
+         vpmultishiftqb	zmm30, zmm29, zmmword ptr [rdx + 8128]
+
+# CHECK: vpmultishiftqb	zmm30, zmm29, zmmword ptr [rdx + 8192]
+# CHECK: encoding: [0x62,0x62,0x95,0x40,0x83,0xb2,0x00,0x20,0x00,0x00]
+         vpmultishiftqb	zmm30, zmm29, zmmword ptr [rdx + 8192]
+
+# CHECK: vpmultishiftqb	zmm30, zmm29, zmmword ptr [rdx - 8192]
+# CHECK: encoding: [0x62,0x62,0x95,0x40,0x83,0x72,0x80]
+         vpmultishiftqb	zmm30, zmm29, zmmword ptr [rdx - 8192]
+
+# CHECK: vpmultishiftqb	zmm30, zmm29, zmmword ptr [rdx - 8256]
+# CHECK: encoding: [0x62,0x62,0x95,0x40,0x83,0xb2,0xc0,0xdf,0xff,0xff]
+         vpmultishiftqb	zmm30, zmm29, zmmword ptr [rdx - 8256]
+
+# CHECK: vpmultishiftqb	zmm30, zmm29, qword ptr [rdx + 1016]{1to8}
+# CHECK: encoding: [0x62,0x62,0x95,0x50,0x83,0x72,0x7f]
+         vpmultishiftqb	zmm30, zmm29, qword ptr [rdx + 1016]{1to8}
+
+# CHECK: vpmultishiftqb	zmm30, zmm29, qword ptr [rdx + 1024]{1to8}
+# CHECK: encoding: [0x62,0x62,0x95,0x50,0x83,0xb2,0x00,0x04,0x00,0x00]
+         vpmultishiftqb	zmm30, zmm29, qword ptr [rdx + 1024]{1to8}
+
+# CHECK: vpmultishiftqb	zmm30, zmm29, qword ptr [rdx - 1024]{1to8}
+# CHECK: encoding: [0x62,0x62,0x95,0x50,0x83,0x72,0x80]
+         vpmultishiftqb	zmm30, zmm29, qword ptr [rdx - 1024]{1to8}
+
+# CHECK: vpmultishiftqb	zmm30, zmm29, qword ptr [rdx - 1032]{1to8}
+# CHECK: encoding: [0x62,0x62,0x95,0x50,0x83,0xb2,0xf8,0xfb,0xff,0xff]
+         vpmultishiftqb	zmm30, zmm29, qword ptr [rdx - 1032]{1to8}
+
+# CHECK: vpmultishiftqb	zmm30, zmm29, zmmword ptr [rax + 8*r14 + 4660]
+# CHECK: encoding: [0x62,0x22,0x95,0x40,0x83,0xb4,0xf0,0x34,0x12,0x00,0x00]
+         vpmultishiftqb	zmm30, zmm29, zmmword ptr [rax + 8*r14 + 4660]
diff --git a/llvm/test/MC/X86/avx512vbmi2-encoding.s b/llvm/test/MC/X86/avx512vbmi2-att.s
similarity index 100%
rename from llvm/test/MC/X86/avx512vbmi2-encoding.s
rename to llvm/test/MC/X86/avx512vbmi2-att.s
diff --git a/llvm/test/MC/X86/avx512vbmi2vl-encoding.s b/llvm/test/MC/X86/avx512vbmi2vl-att.s
similarity index 100%
rename from llvm/test/MC/X86/avx512vbmi2vl-encoding.s
rename to llvm/test/MC/X86/avx512vbmi2vl-att.s
diff --git a/llvm/test/MC/X86/intel-syntax-x86-avx512vbmi_vl.s b/llvm/test/MC/X86/avx512vbmi_vl-intel.s
similarity index 100%
rename from llvm/test/MC/X86/intel-syntax-x86-avx512vbmi_vl.s
rename to llvm/test/MC/X86/avx512vbmi_vl-intel.s
diff --git a/llvm/test/MC/X86/avx512vl-encoding.s b/llvm/test/MC/X86/avx512vl-att.s
similarity index 100%
rename from llvm/test/MC/X86/avx512vl-encoding.s
rename to llvm/test/MC/X86/avx512vl-att.s
diff --git a/llvm/test/MC/X86/avx512vl_bitalg-encoding.s b/llvm/test/MC/X86/avx512vl_bitalg-att.s
similarity index 100%
rename from llvm/test/MC/X86/avx512vl_bitalg-encoding.s
rename to llvm/test/MC/X86/avx512vl_bitalg-att.s
diff --git a/llvm/test/MC/X86/avx512vl_gfni-encoding.s b/llvm/test/MC/X86/avx512vl_gfni-att.s
similarity index 100%
rename from llvm/test/MC/X86/avx512vl_gfni-encoding.s
rename to llvm/test/MC/X86/avx512vl_gfni-att.s
diff --git a/llvm/test/MC/X86/avx512vl_vaes-encoding.s b/llvm/test/MC/X86/avx512vl_vaes-att.s
similarity index 100%
rename from llvm/test/MC/X86/avx512vl_vaes-encoding.s
rename to llvm/test/MC/X86/avx512vl_vaes-att.s
diff --git a/llvm/test/MC/X86/avx512vl_vnni-encoding.s b/llvm/test/MC/X86/avx512vl_vnni-att.s
similarity index 100%
rename from llvm/test/MC/X86/avx512vl_vnni-encoding.s
rename to llvm/test/MC/X86/avx512vl_vnni-att.s
diff --git a/llvm/test/MC/X86/avx512vnni-encoding.s b/llvm/test/MC/X86/avx512vnni-att.s
similarity index 100%
rename from llvm/test/MC/X86/avx512vnni-encoding.s
rename to llvm/test/MC/X86/avx512vnni-att.s
diff --git a/llvm/test/MC/X86/avx512vp2intersect-32-att.s b/llvm/test/MC/X86/avx512vp2intersect-32-att.s
new file mode 100644
index 0000000000000000000000000000000000000000..e1b0dc5f210c38bdf7a5b53874a46abcb05cd48f
--- /dev/null
+++ b/llvm/test/MC/X86/avx512vp2intersect-32-att.s
@@ -0,0 +1,225 @@
+# RUN: llvm-mc -triple i386 -show-encoding %s | FileCheck %s
+
+# CHECK: vp2intersectq	%zmm2, %zmm1, %k0
+# CHECK: encoding: [0x62,0xf2,0xf7,0x48,0x68,0xc2]
+         vp2intersectq	%zmm2, %zmm1, %k0
+
+# CHECK: vp2intersectq	(%edi), %zmm1, %k0
+# CHECK: encoding: [0x62,0xf2,0xf7,0x48,0x68,0x07]
+         vp2intersectq	(%edi), %zmm1, %k0
+
+# CHECK: vp2intersectq	(%edi){1to8}, %zmm1, %k0
+# CHECK: encoding: [0x62,0xf2,0xf7,0x58,0x68,0x07]
+         vp2intersectq	(%edi){1to8}, %zmm1, %k0
+
+# CHECK: vp2intersectq	%zmm2, %zmm1, %k0
+# CHECK: encoding: [0x62,0xf2,0xf7,0x48,0x68,0xc2]
+         vp2intersectq	%zmm2, %zmm1, %k0
+
+# CHECK: vp2intersectq	(%edi), %zmm1, %k0
+# CHECK: encoding: [0x62,0xf2,0xf7,0x48,0x68,0x07]
+         vp2intersectq	(%edi), %zmm1, %k0
+
+# CHECK: vp2intersectq	(%edi){1to8}, %zmm1, %k0
+# CHECK: encoding: [0x62,0xf2,0xf7,0x58,0x68,0x07]
+         vp2intersectq	(%edi){1to8}, %zmm1, %k0
+
+# CHECK: vp2intersectq	%zmm7, %zmm4, %k6
+# CHECK: encoding: [0x62,0xf2,0xdf,0x48,0x68,0xf7]
+         vp2intersectq	%zmm7, %zmm4, %k6
+
+# CHECK: vp2intersectq	(%esi), %zmm4, %k6
+# CHECK: encoding: [0x62,0xf2,0xdf,0x48,0x68,0x36]
+         vp2intersectq	(%esi), %zmm4, %k6
+
+# CHECK: vp2intersectq	(%esi){1to8}, %zmm4, %k6
+# CHECK: encoding: [0x62,0xf2,0xdf,0x58,0x68,0x36]
+         vp2intersectq	(%esi){1to8}, %zmm4, %k6
+
+# CHECK: vp2intersectq	%zmm7, %zmm4, %k6
+# CHECK: encoding: [0x62,0xf2,0xdf,0x48,0x68,0xf7]
+         vp2intersectq	%zmm7, %zmm4, %k6
+
+# CHECK: vp2intersectq	(%esi), %zmm4, %k6
+# CHECK: encoding: [0x62,0xf2,0xdf,0x48,0x68,0x36]
+         vp2intersectq	(%esi), %zmm4, %k6
+
+# CHECK: vp2intersectq	(%esi){1to8}, %zmm4, %k6
+# CHECK: encoding: [0x62,0xf2,0xdf,0x58,0x68,0x36]
+         vp2intersectq	(%esi){1to8}, %zmm4, %k6
+
+# CHECK: vp2intersectq	%ymm2, %ymm1, %k0
+# CHECK: encoding: [0x62,0xf2,0xf7,0x28,0x68,0xc2]
+         vp2intersectq	%ymm2, %ymm1, %k0
+
+# CHECK: vp2intersectq	(%edi), %ymm1, %k0
+# CHECK: encoding: [0x62,0xf2,0xf7,0x28,0x68,0x07]
+         vp2intersectq	(%edi), %ymm1, %k0
+
+# CHECK: vp2intersectq	(%edi){1to4}, %ymm1, %k0
+# CHECK: encoding: [0x62,0xf2,0xf7,0x38,0x68,0x07]
+         vp2intersectq	(%edi){1to4}, %ymm1, %k0
+
+# CHECK: vp2intersectq	%ymm2, %ymm1, %k0
+# CHECK: encoding: [0x62,0xf2,0xf7,0x28,0x68,0xc2]
+         vp2intersectq	%ymm2, %ymm1, %k0
+
+# CHECK: vp2intersectq	(%edi), %ymm1, %k0
+# CHECK: encoding: [0x62,0xf2,0xf7,0x28,0x68,0x07]
+         vp2intersectq	(%edi), %ymm1, %k0
+
+# CHECK: vp2intersectq	(%edi){1to4}, %ymm1, %k0
+# CHECK: encoding: [0x62,0xf2,0xf7,0x38,0x68,0x07]
+         vp2intersectq	(%edi){1to4}, %ymm1, %k0
+
+# CHECK: vp2intersectq	%ymm7, %ymm4, %k6
+# CHECK: encoding: [0x62,0xf2,0xdf,0x28,0x68,0xf7]
+         vp2intersectq	%ymm7, %ymm4, %k6
+
+# CHECK: vp2intersectq	(%esi), %ymm4, %k6
+# CHECK: encoding: [0x62,0xf2,0xdf,0x28,0x68,0x36]
+         vp2intersectq	(%esi), %ymm4, %k6
+
+# CHECK: vp2intersectq	(%esi){1to4}, %ymm4, %k6
+# CHECK: encoding: [0x62,0xf2,0xdf,0x38,0x68,0x36]
+         vp2intersectq	(%esi){1to4}, %ymm4, %k6
+
+# CHECK: vp2intersectq	%ymm7, %ymm4, %k6
+# CHECK: encoding: [0x62,0xf2,0xdf,0x28,0x68,0xf7]
+         vp2intersectq	%ymm7, %ymm4, %k6
+
+# CHECK: vp2intersectq	(%esi), %ymm4, %k6
+# CHECK: encoding: [0x62,0xf2,0xdf,0x28,0x68,0x36]
+         vp2intersectq	(%esi), %ymm4, %k6
+
+# CHECK: vp2intersectq	%xmm2, %xmm1, %k0
+# CHECK: encoding: [0x62,0xf2,0xf7,0x08,0x68,0xc2]
+         vp2intersectq	%xmm2, %xmm1, %k0
+
+# CHECK: vp2intersectq	(%edi), %xmm1, %k0
+# CHECK: encoding: [0x62,0xf2,0xf7,0x08,0x68,0x07]
+         vp2intersectq	(%edi), %xmm1, %k0
+
+# CHECK: vp2intersectq	(%edi){1to2}, %xmm1, %k0
+# CHECK: encoding: [0x62,0xf2,0xf7,0x18,0x68,0x07]
+         vp2intersectq	(%edi){1to2}, %xmm1, %k0
+
+# CHECK: vp2intersectq	%xmm2, %xmm1, %k0
+# CHECK: encoding: [0x62,0xf2,0xf7,0x08,0x68,0xc2]
+         vp2intersectq	%xmm2, %xmm1, %k0
+
+# CHECK: vp2intersectq	(%edi), %xmm1, %k0
+# CHECK: encoding: [0x62,0xf2,0xf7,0x08,0x68,0x07]
+         vp2intersectq	(%edi), %xmm1, %k0
+
+# CHECK: vp2intersectq	%xmm7, %xmm4, %k6
+# CHECK: encoding: [0x62,0xf2,0xdf,0x08,0x68,0xf7]
+         vp2intersectq	%xmm7, %xmm4, %k6
+
+# CHECK: vp2intersectq	(%esi), %xmm4, %k6
+# CHECK: encoding: [0x62,0xf2,0xdf,0x08,0x68,0x36]
+         vp2intersectq	(%esi), %xmm4, %k6
+
+# CHECK: vp2intersectq	%xmm7, %xmm4, %k6
+# CHECK: encoding: [0x62,0xf2,0xdf,0x08,0x68,0xf7]
+         vp2intersectq	%xmm7, %xmm4, %k6
+
+# CHECK: vp2intersectq	(%esi), %xmm4, %k6
+# CHECK: encoding: [0x62,0xf2,0xdf,0x08,0x68,0x36]
+         vp2intersectq	(%esi), %xmm4, %k6
+
+# CHECK: vp2intersectd	%zmm2, %zmm1, %k0
+# CHECK: encoding: [0x62,0xf2,0x77,0x48,0x68,0xc2]
+         vp2intersectd	%zmm2, %zmm1, %k0
+
+# CHECK: vp2intersectd	(%edi), %zmm1, %k0
+# CHECK: encoding: [0x62,0xf2,0x77,0x48,0x68,0x07]
+         vp2intersectd	(%edi), %zmm1, %k0
+
+# CHECK: vp2intersectd	%zmm2, %zmm1, %k0
+# CHECK: encoding: [0x62,0xf2,0x77,0x48,0x68,0xc2]
+         vp2intersectd	%zmm2, %zmm1, %k0
+
+# CHECK: vp2intersectd	(%edi), %zmm1, %k0
+# CHECK: encoding: [0x62,0xf2,0x77,0x48,0x68,0x07]
+         vp2intersectd	(%edi), %zmm1, %k0
+
+# CHECK: vp2intersectd	%zmm7, %zmm4, %k6
+# CHECK: encoding: [0x62,0xf2,0x5f,0x48,0x68,0xf7]
+         vp2intersectd	%zmm7, %zmm4, %k6
+
+# CHECK: vp2intersectd	(%esi), %zmm4, %k6
+# CHECK: encoding: [0x62,0xf2,0x5f,0x48,0x68,0x36]
+         vp2intersectd	(%esi), %zmm4, %k6
+
+# CHECK: vp2intersectd	%zmm7, %zmm4, %k6
+# CHECK: encoding: [0x62,0xf2,0x5f,0x48,0x68,0xf7]
+         vp2intersectd	%zmm7, %zmm4, %k6
+
+# CHECK: vp2intersectd	(%esi), %zmm4, %k6
+# CHECK: encoding: [0x62,0xf2,0x5f,0x48,0x68,0x36]
+         vp2intersectd	(%esi), %zmm4, %k6
+
+# CHECK: vp2intersectd	%ymm2, %ymm1, %k0
+# CHECK: encoding: [0x62,0xf2,0x77,0x28,0x68,0xc2]
+         vp2intersectd	%ymm2, %ymm1, %k0
+
+# CHECK: vp2intersectd	(%edi), %ymm1, %k0
+# CHECK: encoding: [0x62,0xf2,0x77,0x28,0x68,0x07]
+         vp2intersectd	(%edi), %ymm1, %k0
+
+# CHECK: vp2intersectd	%ymm2, %ymm1, %k0
+# CHECK: encoding: [0x62,0xf2,0x77,0x28,0x68,0xc2]
+         vp2intersectd	%ymm2, %ymm1, %k0
+
+# CHECK: vp2intersectd	(%edi), %ymm1, %k0
+# CHECK: encoding: [0x62,0xf2,0x77,0x28,0x68,0x07]
+         vp2intersectd	(%edi), %ymm1, %k0
+
+# CHECK: vp2intersectd	%ymm7, %ymm4, %k6
+# CHECK: encoding: [0x62,0xf2,0x5f,0x28,0x68,0xf7]
+         vp2intersectd	%ymm7, %ymm4, %k6
+
+# CHECK: vp2intersectd	(%esi), %ymm4, %k6
+# CHECK: encoding: [0x62,0xf2,0x5f,0x28,0x68,0x36]
+         vp2intersectd	(%esi), %ymm4, %k6
+
+# CHECK: vp2intersectd	%ymm7, %ymm4, %k6
+# CHECK: encoding: [0x62,0xf2,0x5f,0x28,0x68,0xf7]
+         vp2intersectd	%ymm7, %ymm4, %k6
+
+# CHECK: vp2intersectd	(%esi), %ymm4, %k6
+# CHECK: encoding: [0x62,0xf2,0x5f,0x28,0x68,0x36]
+         vp2intersectd	(%esi), %ymm4, %k6
+
+# CHECK: vp2intersectd	%xmm2, %xmm1, %k0
+# CHECK: encoding: [0x62,0xf2,0x77,0x08,0x68,0xc2]
+         vp2intersectd	%xmm2, %xmm1, %k0
+
+# CHECK: vp2intersectd	(%edi), %xmm1, %k0
+# CHECK: encoding: [0x62,0xf2,0x77,0x08,0x68,0x07]
+         vp2intersectd	(%edi), %xmm1, %k0
+
+# CHECK: vp2intersectd	%xmm2, %xmm1, %k0
+# CHECK: encoding: [0x62,0xf2,0x77,0x08,0x68,0xc2]
+         vp2intersectd	%xmm2, %xmm1, %k0
+
+# CHECK: vp2intersectd	(%edi), %xmm1, %k0
+# CHECK: encoding: [0x62,0xf2,0x77,0x08,0x68,0x07]
+         vp2intersectd	(%edi), %xmm1, %k0
+
+# CHECK: vp2intersectd	%xmm7, %xmm4, %k6
+# CHECK: encoding: [0x62,0xf2,0x5f,0x08,0x68,0xf7]
+         vp2intersectd	%xmm7, %xmm4, %k6
+
+# CHECK: vp2intersectd	(%esi), %xmm4, %k6
+# CHECK: encoding: [0x62,0xf2,0x5f,0x08,0x68,0x36]
+         vp2intersectd	(%esi), %xmm4, %k6
+
+# CHECK: vp2intersectd	%xmm7, %xmm4, %k6
+# CHECK: encoding: [0x62,0xf2,0x5f,0x08,0x68,0xf7]
+         vp2intersectd	%xmm7, %xmm4, %k6
+
+# CHECK: vp2intersectd	(%esi), %xmm4, %k6
+# CHECK: encoding: [0x62,0xf2,0x5f,0x08,0x68,0x36]
+         vp2intersectd	(%esi), %xmm4, %k6
diff --git a/llvm/test/MC/X86/avx512vp2intersect-32-intel.s b/llvm/test/MC/X86/avx512vp2intersect-32-intel.s
new file mode 100644
index 0000000000000000000000000000000000000000..a406842b57352a227b8c2908d2c5d58f4550e650
--- /dev/null
+++ b/llvm/test/MC/X86/avx512vp2intersect-32-intel.s
@@ -0,0 +1,225 @@
+# RUN: llvm-mc -triple i386 -show-encoding -x86-asm-syntax=intel -output-asm-variant=1 %s | FileCheck %s
+
+# CHECK: vp2intersectq	k0, zmm1, zmm2
+# CHECK: encoding: [0x62,0xf2,0xf7,0x48,0x68,0xc2]
+         vp2intersectq	k0, zmm1, zmm2
+
+# CHECK: vp2intersectq	k0, zmm1, zmmword ptr [edi]
+# CHECK: encoding: [0x62,0xf2,0xf7,0x48,0x68,0x07]
+         vp2intersectq	k0, zmm1, zmmword ptr [edi]
+
+# CHECK: vp2intersectq	k0, zmm1, qword ptr [edi]{1to8}
+# CHECK: encoding: [0x62,0xf2,0xf7,0x58,0x68,0x07]
+         vp2intersectq	k0, zmm1, qword ptr [edi]{1to8}
+
+# CHECK: vp2intersectq	k0, zmm1, zmm2
+# CHECK: encoding: [0x62,0xf2,0xf7,0x48,0x68,0xc2]
+         vp2intersectq	k0, zmm1, zmm2
+
+# CHECK: vp2intersectq	k0, zmm1, zmmword ptr [edi]
+# CHECK: encoding: [0x62,0xf2,0xf7,0x48,0x68,0x07]
+         vp2intersectq	k0, zmm1, zmmword ptr [edi]
+
+# CHECK: vp2intersectq	k0, zmm1, qword ptr [edi]{1to8}
+# CHECK: encoding: [0x62,0xf2,0xf7,0x58,0x68,0x07]
+         vp2intersectq	k0, zmm1, qword ptr [edi]{1to8}
+
+# CHECK: vp2intersectq	k6, zmm4, zmm7
+# CHECK: encoding: [0x62,0xf2,0xdf,0x48,0x68,0xf7]
+         vp2intersectq	k6, zmm4, zmm7
+
+# CHECK: vp2intersectq	k6, zmm4, zmmword ptr [esi]
+# CHECK: encoding: [0x62,0xf2,0xdf,0x48,0x68,0x36]
+         vp2intersectq	k6, zmm4, zmmword ptr [esi]
+
+# CHECK: vp2intersectq	k6, zmm4, qword ptr [esi]{1to8}
+# CHECK: encoding: [0x62,0xf2,0xdf,0x58,0x68,0x36]
+         vp2intersectq	k6, zmm4, qword ptr [esi]{1to8}
+
+# CHECK: vp2intersectq	k6, zmm4, zmm7
+# CHECK: encoding: [0x62,0xf2,0xdf,0x48,0x68,0xf7]
+         vp2intersectq	k6, zmm4, zmm7
+
+# CHECK: vp2intersectq	k6, zmm4, zmmword ptr [esi]
+# CHECK: encoding: [0x62,0xf2,0xdf,0x48,0x68,0x36]
+         vp2intersectq	k6, zmm4, zmmword ptr [esi]
+
+# CHECK: vp2intersectq	k6, zmm4, qword ptr [esi]{1to8}
+# CHECK: encoding: [0x62,0xf2,0xdf,0x58,0x68,0x36]
+         vp2intersectq	k6, zmm4, qword ptr [esi]{1to8}
+
+# CHECK: vp2intersectq	k0, ymm1, ymm2
+# CHECK: encoding: [0x62,0xf2,0xf7,0x28,0x68,0xc2]
+         vp2intersectq	k0, ymm1, ymm2
+
+# CHECK: vp2intersectq	k0, ymm1, ymmword ptr [edi]
+# CHECK: encoding: [0x62,0xf2,0xf7,0x28,0x68,0x07]
+         vp2intersectq	k0, ymm1, ymmword ptr [edi]
+
+# CHECK: vp2intersectq	k0, ymm1, qword ptr [edi]{1to4}
+# CHECK: encoding: [0x62,0xf2,0xf7,0x38,0x68,0x07]
+         vp2intersectq	k0, ymm1, qword ptr [edi]{1to4}
+
+# CHECK: vp2intersectq	k0, ymm1, ymm2
+# CHECK: encoding: [0x62,0xf2,0xf7,0x28,0x68,0xc2]
+         vp2intersectq	k0, ymm1, ymm2
+
+# CHECK: vp2intersectq	k0, ymm1, ymmword ptr [edi]
+# CHECK: encoding: [0x62,0xf2,0xf7,0x28,0x68,0x07]
+         vp2intersectq	k0, ymm1, ymmword ptr [edi]
+
+# CHECK: vp2intersectq	k0, ymm1, qword ptr [edi]{1to4}
+# CHECK: encoding: [0x62,0xf2,0xf7,0x38,0x68,0x07]
+         vp2intersectq	k0, ymm1, qword ptr [edi]{1to4}
+
+# CHECK: vp2intersectq	k6, ymm4, ymm7
+# CHECK: encoding: [0x62,0xf2,0xdf,0x28,0x68,0xf7]
+         vp2intersectq	k6, ymm4, ymm7
+
+# CHECK: vp2intersectq	k6, ymm4, ymmword ptr [esi]
+# CHECK: encoding: [0x62,0xf2,0xdf,0x28,0x68,0x36]
+         vp2intersectq	k6, ymm4, ymmword ptr [esi]
+
+# CHECK: vp2intersectq	k6, ymm4, qword ptr [esi]{1to4}
+# CHECK: encoding: [0x62,0xf2,0xdf,0x38,0x68,0x36]
+         vp2intersectq	k6, ymm4, qword ptr [esi]{1to4}
+
+# CHECK: vp2intersectq	k6, ymm4, ymm7
+# CHECK: encoding: [0x62,0xf2,0xdf,0x28,0x68,0xf7]
+         vp2intersectq	k6, ymm4, ymm7
+
+# CHECK: vp2intersectq	k6, ymm4, ymmword ptr [esi]
+# CHECK: encoding: [0x62,0xf2,0xdf,0x28,0x68,0x36]
+         vp2intersectq	k6, ymm4, ymmword ptr [esi]
+
+# CHECK: vp2intersectq	k0, xmm1, xmm2
+# CHECK: encoding: [0x62,0xf2,0xf7,0x08,0x68,0xc2]
+         vp2intersectq	k0, xmm1, xmm2
+
+# CHECK: vp2intersectq	k0, xmm1, xmmword ptr [edi]
+# CHECK: encoding: [0x62,0xf2,0xf7,0x08,0x68,0x07]
+         vp2intersectq	k0, xmm1, xmmword ptr [edi]
+
+# CHECK: vp2intersectq	k0, xmm1, qword ptr [edi]{1to2}
+# CHECK: encoding: [0x62,0xf2,0xf7,0x18,0x68,0x07]
+         vp2intersectq	k0, xmm1, qword ptr [edi]{1to2}
+
+# CHECK: vp2intersectq	k0, xmm1, xmm2
+# CHECK: encoding: [0x62,0xf2,0xf7,0x08,0x68,0xc2]
+         vp2intersectq	k0, xmm1, xmm2
+
+# CHECK: vp2intersectq	k0, xmm1, xmmword ptr [edi]
+# CHECK: encoding: [0x62,0xf2,0xf7,0x08,0x68,0x07]
+         vp2intersectq	k0, xmm1, xmmword ptr [edi]
+
+# CHECK: vp2intersectq	k6, xmm4, xmm7
+# CHECK: encoding: [0x62,0xf2,0xdf,0x08,0x68,0xf7]
+         vp2intersectq	k6, xmm4, xmm7
+
+# CHECK: vp2intersectq	k6, xmm4, xmmword ptr [esi]
+# CHECK: encoding: [0x62,0xf2,0xdf,0x08,0x68,0x36]
+         vp2intersectq	k6, xmm4, xmmword ptr [esi]
+
+# CHECK: vp2intersectq	k6, xmm4, xmm7
+# CHECK: encoding: [0x62,0xf2,0xdf,0x08,0x68,0xf7]
+         vp2intersectq	k6, xmm4, xmm7
+
+# CHECK: vp2intersectq	k6, xmm4, xmmword ptr [esi]
+# CHECK: encoding: [0x62,0xf2,0xdf,0x08,0x68,0x36]
+         vp2intersectq	k6, xmm4, xmmword ptr [esi]
+
+# CHECK: vp2intersectd	k0, zmm1, zmm2
+# CHECK: encoding: [0x62,0xf2,0x77,0x48,0x68,0xc2]
+         vp2intersectd	k0, zmm1, zmm2
+
+# CHECK: vp2intersectd	k0, zmm1, zmmword ptr [edi]
+# CHECK: encoding: [0x62,0xf2,0x77,0x48,0x68,0x07]
+         vp2intersectd	k0, zmm1, zmmword ptr [edi]
+
+# CHECK: vp2intersectd	k0, zmm1, zmm2
+# CHECK: encoding: [0x62,0xf2,0x77,0x48,0x68,0xc2]
+         vp2intersectd	k0, zmm1, zmm2
+
+# CHECK: vp2intersectd	k0, zmm1, zmmword ptr [edi]
+# CHECK: encoding: [0x62,0xf2,0x77,0x48,0x68,0x07]
+         vp2intersectd	k0, zmm1, zmmword ptr [edi]
+
+# CHECK: vp2intersectd	k6, zmm4, zmm7
+# CHECK: encoding: [0x62,0xf2,0x5f,0x48,0x68,0xf7]
+         vp2intersectd	k6, zmm4, zmm7
+
+# CHECK: vp2intersectd	k6, zmm4, zmmword ptr [esi]
+# CHECK: encoding: [0x62,0xf2,0x5f,0x48,0x68,0x36]
+         vp2intersectd	k6, zmm4, zmmword ptr [esi]
+
+# CHECK: vp2intersectd	k6, zmm4, zmm7
+# CHECK: encoding: [0x62,0xf2,0x5f,0x48,0x68,0xf7]
+         vp2intersectd	k6, zmm4, zmm7
+
+# CHECK: vp2intersectd	k6, zmm4, zmmword ptr [esi]
+# CHECK: encoding: [0x62,0xf2,0x5f,0x48,0x68,0x36]
+         vp2intersectd	k6, zmm4, zmmword ptr [esi]
+
+# CHECK: vp2intersectd	k0, ymm1, ymm2
+# CHECK: encoding: [0x62,0xf2,0x77,0x28,0x68,0xc2]
+         vp2intersectd	k0, ymm1, ymm2
+
+# CHECK: vp2intersectd	k0, ymm1, ymmword ptr [edi]
+# CHECK: encoding: [0x62,0xf2,0x77,0x28,0x68,0x07]
+         vp2intersectd	k0, ymm1, ymmword ptr [edi]
+
+# CHECK: vp2intersectd	k0, ymm1, ymm2
+# CHECK: encoding: [0x62,0xf2,0x77,0x28,0x68,0xc2]
+         vp2intersectd	k0, ymm1, ymm2
+
+# CHECK: vp2intersectd	k0, ymm1, ymmword ptr [edi]
+# CHECK: encoding: [0x62,0xf2,0x77,0x28,0x68,0x07]
+         vp2intersectd	k0, ymm1, ymmword ptr [edi]
+
+# CHECK: vp2intersectd	k6, ymm4, ymm7
+# CHECK: encoding: [0x62,0xf2,0x5f,0x28,0x68,0xf7]
+         vp2intersectd	k6, ymm4, ymm7
+
+# CHECK: vp2intersectd	k6, ymm4, ymmword ptr [esi]
+# CHECK: encoding: [0x62,0xf2,0x5f,0x28,0x68,0x36]
+         vp2intersectd	k6, ymm4, ymmword ptr [esi]
+
+# CHECK: vp2intersectd	k6, ymm4, ymm7
+# CHECK: encoding: [0x62,0xf2,0x5f,0x28,0x68,0xf7]
+         vp2intersectd	k6, ymm4, ymm7
+
+# CHECK: vp2intersectd	k6, ymm4, ymmword ptr [esi]
+# CHECK: encoding: [0x62,0xf2,0x5f,0x28,0x68,0x36]
+         vp2intersectd	k6, ymm4, ymmword ptr [esi]
+
+# CHECK: vp2intersectd	k0, xmm1, xmm2
+# CHECK: encoding: [0x62,0xf2,0x77,0x08,0x68,0xc2]
+         vp2intersectd	k0, xmm1, xmm2
+
+# CHECK: vp2intersectd	k0, xmm1, xmmword ptr [edi]
+# CHECK: encoding: [0x62,0xf2,0x77,0x08,0x68,0x07]
+         vp2intersectd	k0, xmm1, xmmword ptr [edi]
+
+# CHECK: vp2intersectd	k0, xmm1, xmm2
+# CHECK: encoding: [0x62,0xf2,0x77,0x08,0x68,0xc2]
+         vp2intersectd	k0, xmm1, xmm2
+
+# CHECK: vp2intersectd	k0, xmm1, xmmword ptr [edi]
+# CHECK: encoding: [0x62,0xf2,0x77,0x08,0x68,0x07]
+         vp2intersectd	k0, xmm1, xmmword ptr [edi]
+
+# CHECK: vp2intersectd	k6, xmm4, xmm7
+# CHECK: encoding: [0x62,0xf2,0x5f,0x08,0x68,0xf7]
+         vp2intersectd	k6, xmm4, xmm7
+
+# CHECK: vp2intersectd	k6, xmm4, xmmword ptr [esi]
+# CHECK: encoding: [0x62,0xf2,0x5f,0x08,0x68,0x36]
+         vp2intersectd	k6, xmm4, xmmword ptr [esi]
+
+# CHECK: vp2intersectd	k6, xmm4, xmm7
+# CHECK: encoding: [0x62,0xf2,0x5f,0x08,0x68,0xf7]
+         vp2intersectd	k6, xmm4, xmm7
+
+# CHECK: vp2intersectd	k6, xmm4, xmmword ptr [esi]
+# CHECK: encoding: [0x62,0xf2,0x5f,0x08,0x68,0x36]
+         vp2intersectd	k6, xmm4, xmmword ptr [esi]
diff --git a/llvm/test/MC/X86/avx512vp2intersect-64-att.s b/llvm/test/MC/X86/avx512vp2intersect-64-att.s
new file mode 100644
index 0000000000000000000000000000000000000000..e90541bb11efefc6ac110692bf702bbbed053527
--- /dev/null
+++ b/llvm/test/MC/X86/avx512vp2intersect-64-att.s
@@ -0,0 +1,226 @@
+# RUN: llvm-mc -triple x86_64 -show-encoding %s | FileCheck %s
+
+
+# CHECK: vp2intersectq	%zmm2, %zmm1, %k0
+# CHECK: encoding: [0x62,0xf2,0xf7,0x48,0x68,0xc2]
+         vp2intersectq	%zmm2, %zmm1, %k0
+
+# CHECK: vp2intersectq	(%rdi), %zmm1, %k0
+# CHECK: encoding: [0x62,0xf2,0xf7,0x48,0x68,0x07]
+         vp2intersectq	(%rdi), %zmm1, %k0
+
+# CHECK: vp2intersectq	(%rdi){1to8}, %zmm1, %k0
+# CHECK: encoding: [0x62,0xf2,0xf7,0x58,0x68,0x07]
+         vp2intersectq	(%rdi){1to8}, %zmm1, %k0
+
+# CHECK: vp2intersectq	%zmm2, %zmm1, %k0
+# CHECK: encoding: [0x62,0xf2,0xf7,0x48,0x68,0xc2]
+         vp2intersectq	%zmm2, %zmm1, %k0
+
+# CHECK: vp2intersectq	(%rdi), %zmm1, %k0
+# CHECK: encoding: [0x62,0xf2,0xf7,0x48,0x68,0x07]
+         vp2intersectq	(%rdi), %zmm1, %k0
+
+# CHECK: vp2intersectq	(%rdi){1to8}, %zmm1, %k0
+# CHECK: encoding: [0x62,0xf2,0xf7,0x58,0x68,0x07]
+         vp2intersectq	(%rdi){1to8}, %zmm1, %k0
+
+# CHECK: vp2intersectq	%zmm7, %zmm9, %k6
+# CHECK: encoding: [0x62,0xf2,0xb7,0x48,0x68,0xf7]
+         vp2intersectq	%zmm7, %zmm9, %k6
+
+# CHECK: vp2intersectq	(%rsi), %zmm9, %k6
+# CHECK: encoding: [0x62,0xf2,0xb7,0x48,0x68,0x36]
+         vp2intersectq	(%rsi), %zmm9, %k6
+
+# CHECK: vp2intersectq	(%rsi){1to8}, %zmm9, %k6
+# CHECK: encoding: [0x62,0xf2,0xb7,0x58,0x68,0x36]
+         vp2intersectq	(%rsi){1to8}, %zmm9, %k6
+
+# CHECK: vp2intersectq	%zmm7, %zmm9, %k6
+# CHECK: encoding: [0x62,0xf2,0xb7,0x48,0x68,0xf7]
+         vp2intersectq	%zmm7, %zmm9, %k6
+
+# CHECK: vp2intersectq	(%rsi), %zmm9, %k6
+# CHECK: encoding: [0x62,0xf2,0xb7,0x48,0x68,0x36]
+         vp2intersectq	(%rsi), %zmm9, %k6
+
+# CHECK: vp2intersectq	(%rsi){1to8}, %zmm9, %k6
+# CHECK: encoding: [0x62,0xf2,0xb7,0x58,0x68,0x36]
+         vp2intersectq	(%rsi){1to8}, %zmm9, %k6
+
+# CHECK: vp2intersectq	%ymm2, %ymm1, %k0
+# CHECK: encoding: [0x62,0xf2,0xf7,0x28,0x68,0xc2]
+         vp2intersectq	%ymm2, %ymm1, %k0
+
+# CHECK: vp2intersectq	(%rdi), %ymm1, %k0
+# CHECK: encoding: [0x62,0xf2,0xf7,0x28,0x68,0x07]
+         vp2intersectq	(%rdi), %ymm1, %k0
+
+# CHECK: vp2intersectq	(%rdi){1to4}, %ymm1, %k0
+# CHECK: encoding: [0x62,0xf2,0xf7,0x38,0x68,0x07]
+         vp2intersectq	(%rdi){1to4}, %ymm1, %k0
+
+# CHECK: vp2intersectq	%ymm2, %ymm1, %k0
+# CHECK: encoding: [0x62,0xf2,0xf7,0x28,0x68,0xc2]
+         vp2intersectq	%ymm2, %ymm1, %k0
+
+# CHECK: vp2intersectq	(%rdi), %ymm1, %k0
+# CHECK: encoding: [0x62,0xf2,0xf7,0x28,0x68,0x07]
+         vp2intersectq	(%rdi), %ymm1, %k0
+
+# CHECK: vp2intersectq	(%rdi){1to4}, %ymm1, %k0
+# CHECK: encoding: [0x62,0xf2,0xf7,0x38,0x68,0x07]
+         vp2intersectq	(%rdi){1to4}, %ymm1, %k0
+
+# CHECK: vp2intersectq	%ymm7, %ymm9, %k6
+# CHECK: encoding: [0x62,0xf2,0xb7,0x28,0x68,0xf7]
+         vp2intersectq	%ymm7, %ymm9, %k6
+
+# CHECK: vp2intersectq	(%rsi), %ymm9, %k6
+# CHECK: encoding: [0x62,0xf2,0xb7,0x28,0x68,0x36]
+         vp2intersectq	(%rsi), %ymm9, %k6
+
+# CHECK: vp2intersectq	(%rsi){1to4}, %ymm9, %k6
+# CHECK: encoding: [0x62,0xf2,0xb7,0x38,0x68,0x36]
+         vp2intersectq	(%rsi){1to4}, %ymm9, %k6
+
+# CHECK: vp2intersectq	%ymm7, %ymm9, %k6
+# CHECK: encoding: [0x62,0xf2,0xb7,0x28,0x68,0xf7]
+         vp2intersectq	%ymm7, %ymm9, %k6
+
+# CHECK: vp2intersectq	(%rsi), %ymm9, %k6
+# CHECK: encoding: [0x62,0xf2,0xb7,0x28,0x68,0x36]
+         vp2intersectq	(%rsi), %ymm9, %k6
+
+# CHECK: vp2intersectq	%xmm2, %xmm1, %k0
+# CHECK: encoding: [0x62,0xf2,0xf7,0x08,0x68,0xc2]
+         vp2intersectq	%xmm2, %xmm1, %k0
+
+# CHECK: vp2intersectq	(%rdi), %xmm1, %k0
+# CHECK: encoding: [0x62,0xf2,0xf7,0x08,0x68,0x07]
+         vp2intersectq	(%rdi), %xmm1, %k0
+
+# CHECK: vp2intersectq	(%rdi){1to2}, %xmm1, %k0
+# CHECK: encoding: [0x62,0xf2,0xf7,0x18,0x68,0x07]
+         vp2intersectq	(%rdi){1to2}, %xmm1, %k0
+
+# CHECK: vp2intersectq	%xmm2, %xmm1, %k0
+# CHECK: encoding: [0x62,0xf2,0xf7,0x08,0x68,0xc2]
+         vp2intersectq	%xmm2, %xmm1, %k0
+
+# CHECK: vp2intersectq	(%rdi), %xmm1, %k0
+# CHECK: encoding: [0x62,0xf2,0xf7,0x08,0x68,0x07]
+         vp2intersectq	(%rdi), %xmm1, %k0
+
+# CHECK: vp2intersectq	%xmm7, %xmm9, %k6
+# CHECK: encoding: [0x62,0xf2,0xb7,0x08,0x68,0xf7]
+         vp2intersectq	%xmm7, %xmm9, %k6
+
+# CHECK: vp2intersectq	(%rsi), %xmm9, %k6
+# CHECK: encoding: [0x62,0xf2,0xb7,0x08,0x68,0x36]
+         vp2intersectq	(%rsi), %xmm9, %k6
+
+# CHECK: vp2intersectq	%xmm7, %xmm9, %k6
+# CHECK: encoding: [0x62,0xf2,0xb7,0x08,0x68,0xf7]
+         vp2intersectq	%xmm7, %xmm9, %k6
+
+# CHECK: vp2intersectq	(%rsi), %xmm9, %k6
+# CHECK: encoding: [0x62,0xf2,0xb7,0x08,0x68,0x36]
+         vp2intersectq	(%rsi), %xmm9, %k6
+
+# CHECK: vp2intersectd	%zmm2, %zmm1, %k0
+# CHECK: encoding: [0x62,0xf2,0x77,0x48,0x68,0xc2]
+         vp2intersectd	%zmm2, %zmm1, %k0
+
+# CHECK: vp2intersectd	(%rdi), %zmm1, %k0
+# CHECK: encoding: [0x62,0xf2,0x77,0x48,0x68,0x07]
+         vp2intersectd	(%rdi), %zmm1, %k0
+
+# CHECK: vp2intersectd	%zmm2, %zmm1, %k0
+# CHECK: encoding: [0x62,0xf2,0x77,0x48,0x68,0xc2]
+         vp2intersectd	%zmm2, %zmm1, %k0
+
+# CHECK: vp2intersectd	(%rdi), %zmm1, %k0
+# CHECK: encoding: [0x62,0xf2,0x77,0x48,0x68,0x07]
+         vp2intersectd	(%rdi), %zmm1, %k0
+
+# CHECK: vp2intersectd	%zmm7, %zmm9, %k6
+# CHECK: encoding: [0x62,0xf2,0x37,0x48,0x68,0xf7]
+         vp2intersectd	%zmm7, %zmm9, %k6
+
+# CHECK: vp2intersectd	(%rsi), %zmm9, %k6
+# CHECK: encoding: [0x62,0xf2,0x37,0x48,0x68,0x36]
+         vp2intersectd	(%rsi), %zmm9, %k6
+
+# CHECK: vp2intersectd	%zmm7, %zmm9, %k6
+# CHECK: encoding: [0x62,0xf2,0x37,0x48,0x68,0xf7]
+         vp2intersectd	%zmm7, %zmm9, %k6
+
+# CHECK: vp2intersectd	(%rsi), %zmm9, %k6
+# CHECK: encoding: [0x62,0xf2,0x37,0x48,0x68,0x36]
+         vp2intersectd	(%rsi), %zmm9, %k6
+
+# CHECK: vp2intersectd	%ymm2, %ymm1, %k0
+# CHECK: encoding: [0x62,0xf2,0x77,0x28,0x68,0xc2]
+         vp2intersectd	%ymm2, %ymm1, %k0
+
+# CHECK: vp2intersectd	(%rdi), %ymm1, %k0
+# CHECK: encoding: [0x62,0xf2,0x77,0x28,0x68,0x07]
+         vp2intersectd	(%rdi), %ymm1, %k0
+
+# CHECK: vp2intersectd	%ymm2, %ymm1, %k0
+# CHECK: encoding: [0x62,0xf2,0x77,0x28,0x68,0xc2]
+         vp2intersectd	%ymm2, %ymm1, %k0
+
+# CHECK: vp2intersectd	(%rdi), %ymm1, %k0
+# CHECK: encoding: [0x62,0xf2,0x77,0x28,0x68,0x07]
+         vp2intersectd	(%rdi), %ymm1, %k0
+
+# CHECK: vp2intersectd	%ymm7, %ymm9, %k6
+# CHECK: encoding: [0x62,0xf2,0x37,0x28,0x68,0xf7]
+         vp2intersectd	%ymm7, %ymm9, %k6
+
+# CHECK: vp2intersectd	(%rsi), %ymm9, %k6
+# CHECK: encoding: [0x62,0xf2,0x37,0x28,0x68,0x36]
+         vp2intersectd	(%rsi), %ymm9, %k6
+
+# CHECK: vp2intersectd	%ymm7, %ymm9, %k6
+# CHECK: encoding: [0x62,0xf2,0x37,0x28,0x68,0xf7]
+         vp2intersectd	%ymm7, %ymm9, %k6
+
+# CHECK: vp2intersectd	(%rsi), %ymm9, %k6
+# CHECK: encoding: [0x62,0xf2,0x37,0x28,0x68,0x36]
+         vp2intersectd	(%rsi), %ymm9, %k6
+
+# CHECK: vp2intersectd	%xmm2, %xmm1, %k0
+# CHECK: encoding: [0x62,0xf2,0x77,0x08,0x68,0xc2]
+         vp2intersectd	%xmm2, %xmm1, %k0
+
+# CHECK: vp2intersectd	(%rdi), %xmm1, %k0
+# CHECK: encoding: [0x62,0xf2,0x77,0x08,0x68,0x07]
+         vp2intersectd	(%rdi), %xmm1, %k0
+
+# CHECK: vp2intersectd	%xmm2, %xmm1, %k0
+# CHECK: encoding: [0x62,0xf2,0x77,0x08,0x68,0xc2]
+         vp2intersectd	%xmm2, %xmm1, %k0
+
+# CHECK: vp2intersectd	(%rdi), %xmm1, %k0
+# CHECK: encoding: [0x62,0xf2,0x77,0x08,0x68,0x07]
+         vp2intersectd	(%rdi), %xmm1, %k0
+
+# CHECK: vp2intersectd	%xmm7, %xmm9, %k6
+# CHECK: encoding: [0x62,0xf2,0x37,0x08,0x68,0xf7]
+         vp2intersectd	%xmm7, %xmm9, %k6
+
+# CHECK: vp2intersectd	(%rsi), %xmm9, %k6
+# CHECK: encoding: [0x62,0xf2,0x37,0x08,0x68,0x36]
+         vp2intersectd	(%rsi), %xmm9, %k6
+
+# CHECK: vp2intersectd	%xmm7, %xmm9, %k6
+# CHECK: encoding: [0x62,0xf2,0x37,0x08,0x68,0xf7]
+         vp2intersectd	%xmm7, %xmm9, %k6
+
+# CHECK: vp2intersectd	(%rsi), %xmm9, %k6
+# CHECK: encoding: [0x62,0xf2,0x37,0x08,0x68,0x36]
+         vp2intersectd	(%rsi), %xmm9, %k6
diff --git a/llvm/test/MC/X86/avx512vp2intersect-64-intel.s b/llvm/test/MC/X86/avx512vp2intersect-64-intel.s
new file mode 100644
index 0000000000000000000000000000000000000000..3cdf49d2b0373906eb0d707210f966fc617318e6
--- /dev/null
+++ b/llvm/test/MC/X86/avx512vp2intersect-64-intel.s
@@ -0,0 +1,226 @@
+# RUN: llvm-mc -triple x86_64 -show-encoding -x86-asm-syntax=intel -output-asm-variant=1 %s | FileCheck %s
+
+
+# CHECK: vp2intersectq	k0, zmm1, zmm2
+# CHECK: encoding: [0x62,0xf2,0xf7,0x48,0x68,0xc2]
+         vp2intersectq	k0, zmm1, zmm2
+
+# CHECK: vp2intersectq	k0, zmm1, zmmword ptr [rdi]
+# CHECK: encoding: [0x62,0xf2,0xf7,0x48,0x68,0x07]
+         vp2intersectq	k0, zmm1, zmmword ptr [rdi]
+
+# CHECK: vp2intersectq	k0, zmm1, qword ptr [rdi]{1to8}
+# CHECK: encoding: [0x62,0xf2,0xf7,0x58,0x68,0x07]
+         vp2intersectq	k0, zmm1, qword ptr [rdi]{1to8}
+
+# CHECK: vp2intersectq	k0, zmm1, zmm2
+# CHECK: encoding: [0x62,0xf2,0xf7,0x48,0x68,0xc2]
+         vp2intersectq	k0, zmm1, zmm2
+
+# CHECK: vp2intersectq	k0, zmm1, zmmword ptr [rdi]
+# CHECK: encoding: [0x62,0xf2,0xf7,0x48,0x68,0x07]
+         vp2intersectq	k0, zmm1, zmmword ptr [rdi]
+
+# CHECK: vp2intersectq	k0, zmm1, qword ptr [rdi]{1to8}
+# CHECK: encoding: [0x62,0xf2,0xf7,0x58,0x68,0x07]
+         vp2intersectq	k0, zmm1, qword ptr [rdi]{1to8}
+
+# CHECK: vp2intersectq	k6, zmm9, zmm7
+# CHECK: encoding: [0x62,0xf2,0xb7,0x48,0x68,0xf7]
+         vp2intersectq	k6, zmm9, zmm7
+
+# CHECK: vp2intersectq	k6, zmm9, zmmword ptr [rsi]
+# CHECK: encoding: [0x62,0xf2,0xb7,0x48,0x68,0x36]
+         vp2intersectq	k6, zmm9, zmmword ptr [rsi]
+
+# CHECK: vp2intersectq	k6, zmm9, qword ptr [rsi]{1to8}
+# CHECK: encoding: [0x62,0xf2,0xb7,0x58,0x68,0x36]
+         vp2intersectq	k6, zmm9, qword ptr [rsi]{1to8}
+
+# CHECK: vp2intersectq	k6, zmm9, zmm7
+# CHECK: encoding: [0x62,0xf2,0xb7,0x48,0x68,0xf7]
+         vp2intersectq	k6, zmm9, zmm7
+
+# CHECK: vp2intersectq	k6, zmm9, zmmword ptr [rsi]
+# CHECK: encoding: [0x62,0xf2,0xb7,0x48,0x68,0x36]
+         vp2intersectq	k6, zmm9, zmmword ptr [rsi]
+
+# CHECK: vp2intersectq	k6, zmm9, qword ptr [rsi]{1to8}
+# CHECK: encoding: [0x62,0xf2,0xb7,0x58,0x68,0x36]
+         vp2intersectq	k6, zmm9, qword ptr [rsi]{1to8}
+
+# CHECK: vp2intersectq	k0, ymm1, ymm2
+# CHECK: encoding: [0x62,0xf2,0xf7,0x28,0x68,0xc2]
+         vp2intersectq	k0, ymm1, ymm2
+
+# CHECK: vp2intersectq	k0, ymm1, ymmword ptr [rdi]
+# CHECK: encoding: [0x62,0xf2,0xf7,0x28,0x68,0x07]
+         vp2intersectq	k0, ymm1, ymmword ptr [rdi]
+
+# CHECK: vp2intersectq	k0, ymm1, qword ptr [rdi]{1to4}
+# CHECK: encoding: [0x62,0xf2,0xf7,0x38,0x68,0x07]
+         vp2intersectq	k0, ymm1, qword ptr [rdi]{1to4}
+
+# CHECK: vp2intersectq	k0, ymm1, ymm2
+# CHECK: encoding: [0x62,0xf2,0xf7,0x28,0x68,0xc2]
+         vp2intersectq	k0, ymm1, ymm2
+
+# CHECK: vp2intersectq	k0, ymm1, ymmword ptr [rdi]
+# CHECK: encoding: [0x62,0xf2,0xf7,0x28,0x68,0x07]
+         vp2intersectq	k0, ymm1, ymmword ptr [rdi]
+
+# CHECK: vp2intersectq	k0, ymm1, qword ptr [rdi]{1to4}
+# CHECK: encoding: [0x62,0xf2,0xf7,0x38,0x68,0x07]
+         vp2intersectq	k0, ymm1, qword ptr [rdi]{1to4}
+
+# CHECK: vp2intersectq	k6, ymm9, ymm7
+# CHECK: encoding: [0x62,0xf2,0xb7,0x28,0x68,0xf7]
+         vp2intersectq	k6, ymm9, ymm7
+
+# CHECK: vp2intersectq	k6, ymm9, ymmword ptr [rsi]
+# CHECK: encoding: [0x62,0xf2,0xb7,0x28,0x68,0x36]
+         vp2intersectq	k6, ymm9, ymmword ptr [rsi]
+
+# CHECK: vp2intersectq	k6, ymm9, qword ptr [rsi]{1to4}
+# CHECK: encoding: [0x62,0xf2,0xb7,0x38,0x68,0x36]
+         vp2intersectq	k6, ymm9, qword ptr [rsi]{1to4}
+
+# CHECK: vp2intersectq	k6, ymm9, ymm7
+# CHECK: encoding: [0x62,0xf2,0xb7,0x28,0x68,0xf7]
+         vp2intersectq	k6, ymm9, ymm7
+
+# CHECK: vp2intersectq	k6, ymm9, ymmword ptr [rsi]
+# CHECK: encoding: [0x62,0xf2,0xb7,0x28,0x68,0x36]
+         vp2intersectq	k6, ymm9, ymmword ptr [rsi]
+
+# CHECK: vp2intersectq	k0, xmm1, xmm2
+# CHECK: encoding: [0x62,0xf2,0xf7,0x08,0x68,0xc2]
+         vp2intersectq	k0, xmm1, xmm2
+
+# CHECK: vp2intersectq	k0, xmm1, xmmword ptr [rdi]
+# CHECK: encoding: [0x62,0xf2,0xf7,0x08,0x68,0x07]
+         vp2intersectq	k0, xmm1, xmmword ptr [rdi]
+
+# CHECK: vp2intersectq	k0, xmm1, qword ptr [rdi]{1to2}
+# CHECK: encoding: [0x62,0xf2,0xf7,0x18,0x68,0x07]
+         vp2intersectq	k0, xmm1, qword ptr [rdi]{1to2}
+
+# CHECK: vp2intersectq	k0, xmm1, xmm2
+# CHECK: encoding: [0x62,0xf2,0xf7,0x08,0x68,0xc2]
+         vp2intersectq	k0, xmm1, xmm2
+
+# CHECK: vp2intersectq	k0, xmm1, xmmword ptr [rdi]
+# CHECK: encoding: [0x62,0xf2,0xf7,0x08,0x68,0x07]
+         vp2intersectq	k0, xmm1, xmmword ptr [rdi]
+
+# CHECK: vp2intersectq	k6, xmm9, xmm7
+# CHECK: encoding: [0x62,0xf2,0xb7,0x08,0x68,0xf7]
+         vp2intersectq	k6, xmm9, xmm7
+
+# CHECK: vp2intersectq	k6, xmm9, xmmword ptr [rsi]
+# CHECK: encoding: [0x62,0xf2,0xb7,0x08,0x68,0x36]
+         vp2intersectq	k6, xmm9, xmmword ptr [rsi]
+
+# CHECK: vp2intersectq	k6, xmm9, xmm7
+# CHECK: encoding: [0x62,0xf2,0xb7,0x08,0x68,0xf7]
+         vp2intersectq	k6, xmm9, xmm7
+
+# CHECK: vp2intersectq	k6, xmm9, xmmword ptr [rsi]
+# CHECK: encoding: [0x62,0xf2,0xb7,0x08,0x68,0x36]
+         vp2intersectq	k6, xmm9, xmmword ptr [rsi]
+
+# CHECK: vp2intersectd	k0, zmm1, zmm2
+# CHECK: encoding: [0x62,0xf2,0x77,0x48,0x68,0xc2]
+         vp2intersectd	k0, zmm1, zmm2
+
+# CHECK: vp2intersectd	k0, zmm1, zmmword ptr [rdi]
+# CHECK: encoding: [0x62,0xf2,0x77,0x48,0x68,0x07]
+         vp2intersectd	k0, zmm1, zmmword ptr [rdi]
+
+# CHECK: vp2intersectd	k0, zmm1, zmm2
+# CHECK: encoding: [0x62,0xf2,0x77,0x48,0x68,0xc2]
+         vp2intersectd	k0, zmm1, zmm2
+
+# CHECK: vp2intersectd	k0, zmm1, zmmword ptr [rdi]
+# CHECK: encoding: [0x62,0xf2,0x77,0x48,0x68,0x07]
+         vp2intersectd	k0, zmm1, zmmword ptr [rdi]
+
+# CHECK: vp2intersectd	k6, zmm9, zmm7
+# CHECK: encoding: [0x62,0xf2,0x37,0x48,0x68,0xf7]
+         vp2intersectd	k6, zmm9, zmm7
+
+# CHECK: vp2intersectd	k6, zmm9, zmmword ptr [rsi]
+# CHECK: encoding: [0x62,0xf2,0x37,0x48,0x68,0x36]
+         vp2intersectd	k6, zmm9, zmmword ptr [rsi]
+
+# CHECK: vp2intersectd	k6, zmm9, zmm7
+# CHECK: encoding: [0x62,0xf2,0x37,0x48,0x68,0xf7]
+         vp2intersectd	k6, zmm9, zmm7
+
+# CHECK: vp2intersectd	k6, zmm9, zmmword ptr [rsi]
+# CHECK: encoding: [0x62,0xf2,0x37,0x48,0x68,0x36]
+         vp2intersectd	k6, zmm9, zmmword ptr [rsi]
+
+# CHECK: vp2intersectd	k0, ymm1, ymm2
+# CHECK: encoding: [0x62,0xf2,0x77,0x28,0x68,0xc2]
+         vp2intersectd	k0, ymm1, ymm2
+
+# CHECK: vp2intersectd	k0, ymm1, ymmword ptr [rdi]
+# CHECK: encoding: [0x62,0xf2,0x77,0x28,0x68,0x07]
+         vp2intersectd	k0, ymm1, ymmword ptr [rdi]
+
+# CHECK: vp2intersectd	k0, ymm1, ymm2
+# CHECK: encoding: [0x62,0xf2,0x77,0x28,0x68,0xc2]
+         vp2intersectd	k0, ymm1, ymm2
+
+# CHECK: vp2intersectd	k0, ymm1, ymmword ptr [rdi]
+# CHECK: encoding: [0x62,0xf2,0x77,0x28,0x68,0x07]
+         vp2intersectd	k0, ymm1, ymmword ptr [rdi]
+
+# CHECK: vp2intersectd	k6, ymm9, ymm7
+# CHECK: encoding: [0x62,0xf2,0x37,0x28,0x68,0xf7]
+         vp2intersectd	k6, ymm9, ymm7
+
+# CHECK: vp2intersectd	k6, ymm9, ymmword ptr [rsi]
+# CHECK: encoding: [0x62,0xf2,0x37,0x28,0x68,0x36]
+         vp2intersectd	k6, ymm9, ymmword ptr [rsi]
+
+# CHECK: vp2intersectd	k6, ymm9, ymm7
+# CHECK: encoding: [0x62,0xf2,0x37,0x28,0x68,0xf7]
+         vp2intersectd	k6, ymm9, ymm7
+
+# CHECK: vp2intersectd	k6, ymm9, ymmword ptr [rsi]
+# CHECK: encoding: [0x62,0xf2,0x37,0x28,0x68,0x36]
+         vp2intersectd	k6, ymm9, ymmword ptr [rsi]
+
+# CHECK: vp2intersectd	k0, xmm1, xmm2
+# CHECK: encoding: [0x62,0xf2,0x77,0x08,0x68,0xc2]
+         vp2intersectd	k0, xmm1, xmm2
+
+# CHECK: vp2intersectd	k0, xmm1, xmmword ptr [rdi]
+# CHECK: encoding: [0x62,0xf2,0x77,0x08,0x68,0x07]
+         vp2intersectd	k0, xmm1, xmmword ptr [rdi]
+
+# CHECK: vp2intersectd	k0, xmm1, xmm2
+# CHECK: encoding: [0x62,0xf2,0x77,0x08,0x68,0xc2]
+         vp2intersectd	k0, xmm1, xmm2
+
+# CHECK: vp2intersectd	k0, xmm1, xmmword ptr [rdi]
+# CHECK: encoding: [0x62,0xf2,0x77,0x08,0x68,0x07]
+         vp2intersectd	k0, xmm1, xmmword ptr [rdi]
+
+# CHECK: vp2intersectd	k6, xmm9, xmm7
+# CHECK: encoding: [0x62,0xf2,0x37,0x08,0x68,0xf7]
+         vp2intersectd	k6, xmm9, xmm7
+
+# CHECK: vp2intersectd	k6, xmm9, xmmword ptr [rsi]
+# CHECK: encoding: [0x62,0xf2,0x37,0x08,0x68,0x36]
+         vp2intersectd	k6, xmm9, xmmword ptr [rsi]
+
+# CHECK: vp2intersectd	k6, xmm9, xmm7
+# CHECK: encoding: [0x62,0xf2,0x37,0x08,0x68,0xf7]
+         vp2intersectd	k6, xmm9, xmm7
+
+# CHECK: vp2intersectd	k6, xmm9, xmmword ptr [rsi]
+# CHECK: encoding: [0x62,0xf2,0x37,0x08,0x68,0x36]
+         vp2intersectd	k6, xmm9, xmmword ptr [rsi]
diff --git a/llvm/test/MC/X86/avx512vp2intersectvl-att.s b/llvm/test/MC/X86/avx512vp2intersectvl-32-att.s
similarity index 100%
rename from llvm/test/MC/X86/avx512vp2intersectvl-att.s
rename to llvm/test/MC/X86/avx512vp2intersectvl-32-att.s
diff --git a/llvm/test/MC/X86/avx512vp2intersectvl-intel.s b/llvm/test/MC/X86/avx512vp2intersectvl-32-intel.s
similarity index 100%
rename from llvm/test/MC/X86/avx512vp2intersectvl-intel.s
rename to llvm/test/MC/X86/avx512vp2intersectvl-32-intel.s
diff --git a/llvm/test/MC/X86/x86-64-avx512vp2intersectvl-att.s b/llvm/test/MC/X86/avx512vp2intersectvl-64-att.s
similarity index 100%
rename from llvm/test/MC/X86/x86-64-avx512vp2intersectvl-att.s
rename to llvm/test/MC/X86/avx512vp2intersectvl-64-att.s
diff --git a/llvm/test/MC/X86/x86-64-avx512vp2intersectvl-intel.s b/llvm/test/MC/X86/avx512vp2intersectvl-64-intel.s
similarity index 100%
rename from llvm/test/MC/X86/x86-64-avx512vp2intersectvl-intel.s
rename to llvm/test/MC/X86/avx512vp2intersectvl-64-intel.s
diff --git a/llvm/test/MC/X86/x86-64-avx512vpopcntdq.s b/llvm/test/MC/X86/avx512vpopcntdq-64-att.s
similarity index 100%
rename from llvm/test/MC/X86/x86-64-avx512vpopcntdq.s
rename to llvm/test/MC/X86/avx512vpopcntdq-64-att.s
diff --git a/llvm/test/MC/X86/x86_64-avx-clmul-encoding.s b/llvm/test/MC/X86/avx_clmul-att.s
similarity index 100%
rename from llvm/test/MC/X86/x86_64-avx-clmul-encoding.s
rename to llvm/test/MC/X86/avx_clmul-att.s
diff --git a/llvm/test/MC/X86/avx_vaes-encoding.s b/llvm/test/MC/X86/avx_vaes-att.s
similarity index 100%
rename from llvm/test/MC/X86/avx_vaes-encoding.s
rename to llvm/test/MC/X86/avx_vaes-att.s
diff --git a/llvm/test/MC/X86/intel-syntax-avx_vnni.s b/llvm/test/MC/X86/avx_vnni-32-intel.s
similarity index 100%
rename from llvm/test/MC/X86/intel-syntax-avx_vnni.s
rename to llvm/test/MC/X86/avx_vnni-32-intel.s
diff --git a/llvm/test/MC/X86/x86-64-avx_vnni-encoding.s b/llvm/test/MC/X86/avx_vnni-64-att.s
similarity index 100%
rename from llvm/test/MC/X86/x86-64-avx_vnni-encoding.s
rename to llvm/test/MC/X86/avx_vnni-64-att.s
diff --git a/llvm/test/MC/X86/intel-syntax-x86-64-avx_vnni.s b/llvm/test/MC/X86/avx_vnni-64-intel.s
similarity index 100%
rename from llvm/test/MC/X86/intel-syntax-x86-64-avx_vnni.s
rename to llvm/test/MC/X86/avx_vnni-64-intel.s
diff --git a/llvm/test/MC/X86/avx_vnni-encoding.s b/llvm/test/MC/X86/avx_vnni-att-32.s
similarity index 100%
rename from llvm/test/MC/X86/avx_vnni-encoding.s
rename to llvm/test/MC/X86/avx_vnni-att-32.s
diff --git a/llvm/test/MC/X86/x86_64-bmi-encoding.s b/llvm/test/MC/X86/bmi-att.s
similarity index 100%
rename from llvm/test/MC/X86/x86_64-bmi-encoding.s
rename to llvm/test/MC/X86/bmi-att.s
diff --git a/llvm/test/MC/X86/cet-encoding.s b/llvm/test/MC/X86/cet-att.s
similarity index 100%
rename from llvm/test/MC/X86/cet-encoding.s
rename to llvm/test/MC/X86/cet-att.s
diff --git a/llvm/test/MC/X86/x86_64-fma3-encoding.s b/llvm/test/MC/X86/fma3-att.s
similarity index 100%
rename from llvm/test/MC/X86/x86_64-fma3-encoding.s
rename to llvm/test/MC/X86/fma3-att.s
diff --git a/llvm/test/MC/X86/x86_64-fma4-encoding.s b/llvm/test/MC/X86/fma4-att.s
similarity index 100%
rename from llvm/test/MC/X86/x86_64-fma4-encoding.s
rename to llvm/test/MC/X86/fma4-att.s
diff --git a/llvm/test/MC/X86/gfni-encoding.s b/llvm/test/MC/X86/gfni-att.s
similarity index 100%
rename from llvm/test/MC/X86/gfni-encoding.s
rename to llvm/test/MC/X86/gfni-att.s
diff --git a/llvm/test/MC/X86/x86_64-hle-encoding.s b/llvm/test/MC/X86/hle-att.s
similarity index 100%
rename from llvm/test/MC/X86/x86_64-hle-encoding.s
rename to llvm/test/MC/X86/hle-att.s
diff --git a/llvm/test/MC/X86/intel-syntax-x86-avx512dq_vl.s b/llvm/test/MC/X86/intel-syntax-x86-avx512dq_vl.s
deleted file mode 100644
index e2d6850623a98a9fce75fbb19dc010ce1ab89c9c..0000000000000000000000000000000000000000
--- a/llvm/test/MC/X86/intel-syntax-x86-avx512dq_vl.s
+++ /dev/null
@@ -1,193 +0,0 @@
-// RUN: llvm-mc -triple x86_64-unknown-unknown -x86-asm-syntax=intel -output-asm-variant=1 --show-encoding %s | FileCheck %s
-
-// CHECK:  vcvtps2qq xmm2 {k2} {z}, qword ptr [rcx + 128]
-// CHECK:  encoding: [0x62,0xf1,0x7d,0x8a,0x7b,0x51,0x10]
-          vcvtps2qq xmm2 {k2} {z}, qword ptr [rcx + 0x80]
-
-// CHECK:  vcvtps2qq xmm2 {k2}, qword ptr [rcx + 128]
-// CHECK:  encoding: [0x62,0xf1,0x7d,0x0a,0x7b,0x51,0x10]
-          vcvtps2qq xmm2 {k2}, qword ptr [rcx + 0x80]
-
-// CHECK:  vcvtps2qq xmm2, qword ptr [rcx + 128]
-// CHECK:  encoding: [0x62,0xf1,0x7d,0x08,0x7b,0x51,0x10]
-          vcvtps2qq xmm2, qword ptr [rcx + 0x80]
-
-// CHECK:  vcvttps2qq xmm1 {k2} {z}, qword ptr [rcx + 128]
-// CHECK:  encoding: [0x62,0xf1,0x7d,0x8a,0x7a,0x49,0x10]
-          vcvttps2qq xmm1 {k2} {z}, qword ptr [rcx + 0x80]
-
-// CHECK:  vcvttps2qq xmm1 {k2}, qword ptr [rcx + 128]
-// CHECK:  encoding: [0x62,0xf1,0x7d,0x0a,0x7a,0x49,0x10]
-          vcvttps2qq xmm1 {k2}, qword ptr [rcx + 0x80]
-
-// CHECK:  vcvttps2qq xmm1, qword ptr [rcx + 128]
-// CHECK:  encoding: [0x62,0xf1,0x7d,0x08,0x7a,0x49,0x10]
-          vcvttps2qq xmm1, qword ptr [rcx + 0x80]
-
-// CHECK:  vcvtps2uqq xmm1 {k2} {z}, qword ptr [rcx + 128]
-// CHECK:  encoding: [0x62,0xf1,0x7d,0x8a,0x79,0x49,0x10]
-          vcvtps2uqq xmm1 {k2} {z}, qword ptr [rcx + 128]
-
-// CHECK:  vcvtps2uqq xmm1 {k2}, qword ptr [rcx + 128]
-// CHECK:  encoding: [0x62,0xf1,0x7d,0x0a,0x79,0x49,0x10]
-          vcvtps2uqq xmm1 {k2}, qword ptr [rcx + 128]
-
-// CHECK:  vcvtps2uqq xmm1, qword ptr [rcx + 128]
-// CHECK:  encoding: [0x62,0xf1,0x7d,0x08,0x79,0x49,0x10]
-          vcvtps2uqq xmm1, qword ptr [rcx + 128]
-
-// CHECK:  vcvttps2uqq xmm1 {k2} {z}, qword ptr [rcx + 128]
-// CHECK:  encoding: [0x62,0xf1,0x7d,0x8a,0x78,0x49,0x10]
-          vcvttps2uqq xmm1 {k2} {z}, qword ptr [rcx + 128]
-
-// CHECK:  vcvttps2uqq xmm1 {k2}, qword ptr [rcx + 128]
-// CHECK:  encoding: [0x62,0xf1,0x7d,0x0a,0x78,0x49,0x10]
-          vcvttps2uqq xmm1 {k2}, qword ptr [rcx + 128]
-
-// CHECK:  vcvttps2uqq xmm1, qword ptr [rcx + 128]
-// CHECK:  encoding: [0x62,0xf1,0x7d,0x08,0x78,0x49,0x10]
-          vcvttps2uqq xmm1, qword ptr [rcx + 128]
-
-// CHECK:  vcvtps2qq xmm2 {k2} {z}, qword ptr [rcx + 128]
-// CHECK:  encoding: [0x62,0xf1,0x7d,0x8a,0x7b,0x51,0x10]
-          vcvtps2qq xmm2 {k2} {z}, qword ptr [rcx + 0x80]
-
-// CHECK:  vcvtps2qq xmm2 {k2}, qword ptr [rcx + 128]
-// CHECK:  encoding: [0x62,0xf1,0x7d,0x0a,0x7b,0x51,0x10]
-          vcvtps2qq xmm2 {k2}, qword ptr [rcx + 0x80]
-
-// CHECK:  vcvtps2qq xmm2, qword ptr [rcx + 128]
-// CHECK:  encoding: [0x62,0xf1,0x7d,0x08,0x7b,0x51,0x10]
-          vcvtps2qq xmm2, qword ptr [rcx + 0x80]
-
-// CHECK:  vcvttps2qq xmm1 {k2} {z}, qword ptr [rcx + 128]
-// CHECK:  encoding: [0x62,0xf1,0x7d,0x8a,0x7a,0x49,0x10]
-          vcvttps2qq xmm1 {k2} {z}, qword ptr [rcx + 0x80]
-
-// CHECK:  vcvttps2qq xmm1 {k2}, qword ptr [rcx + 128]
-// CHECK:  encoding: [0x62,0xf1,0x7d,0x0a,0x7a,0x49,0x10]
-          vcvttps2qq xmm1 {k2}, qword ptr [rcx + 0x80]
-
-// CHECK:  vcvttps2qq xmm1, qword ptr [rcx + 128]
-// CHECK:  encoding: [0x62,0xf1,0x7d,0x08,0x7a,0x49,0x10]
-          vcvttps2qq xmm1, qword ptr [rcx + 0x80]
-
-// CHECK:  vcvtps2uqq xmm1 {k2} {z}, qword ptr [rcx + 128]
-// CHECK:  encoding: [0x62,0xf1,0x7d,0x8a,0x79,0x49,0x10]
-          vcvtps2uqq xmm1 {k2} {z}, qword ptr [rcx + 128]
-
-// CHECK:  vcvtps2uqq xmm1 {k2}, qword ptr [rcx + 128]
-// CHECK:  encoding: [0x62,0xf1,0x7d,0x0a,0x79,0x49,0x10]
-          vcvtps2uqq xmm1 {k2}, qword ptr [rcx + 128]
-
-// CHECK:  vcvtps2uqq xmm1, qword ptr [rcx + 128]
-// CHECK:  encoding: [0x62,0xf1,0x7d,0x08,0x79,0x49,0x10]
-          vcvtps2uqq xmm1, qword ptr [rcx + 128]
-
-// CHECK:  vcvttps2uqq xmm1 {k2} {z}, qword ptr [rcx + 128]
-// CHECK:  encoding: [0x62,0xf1,0x7d,0x8a,0x78,0x49,0x10]
-          vcvttps2uqq xmm1 {k2} {z}, qword ptr [rcx + 128]
-
-// CHECK:  vcvttps2uqq xmm1 {k2}, qword ptr [rcx + 128]
-// CHECK:  encoding: [0x62,0xf1,0x7d,0x0a,0x78,0x49,0x10]
-          vcvttps2uqq xmm1 {k2}, qword ptr [rcx + 128]
-
-// CHECK:  vcvttps2uqq xmm1, qword ptr [rcx + 128]
-// CHECK:  encoding: [0x62,0xf1,0x7d,0x08,0x78,0x49,0x10]
-          vcvttps2uqq xmm1, qword ptr [rcx + 128]
-
-// CHECK: vfpclasspd k2, xmm18, 171
-// CHECK:  encoding: [0x62,0xb3,0xfd,0x08,0x66,0xd2,0xab]
-          vfpclasspd k2, xmm18, 0xab
-
-// CHECK: vfpclasspd k2 {k7}, xmm18, 171
-// CHECK:  encoding: [0x62,0xb3,0xfd,0x0f,0x66,0xd2,0xab]
-          vfpclasspd k2 {k7}, xmm18, 0xab
-
-// CHECK: vfpclasspd k2, xmmword ptr [rcx], 123
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x08,0x66,0x11,0x7b]
-          vfpclasspd k2, xmmword ptr [rcx], 0x7b
-
-// CHECK: vfpclasspd k2 {k7}, xmmword ptr [rcx], 123
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x0f,0x66,0x11,0x7b]
-          vfpclasspd k2 {k7}, xmmword ptr [rcx], 0x7b
-
-// CHECK: vfpclasspd k2, qword ptr [rcx]{1to2}, 123
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x18,0x66,0x11,0x7b]
-          vfpclasspd k2, qword ptr [rcx]{1to2}, 0x7b
-
-// CHECK: vfpclasspd k2 {k7}, qword ptr [rcx]{1to2}, 123
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x1f,0x66,0x11,0x7b]
-          vfpclasspd k2 {k7}, qword ptr [rcx]{1to2}, 0x7b
-
-// CHECK: vfpclassps k2, xmm18, 171
-// CHECK:  encoding: [0x62,0xb3,0x7d,0x08,0x66,0xd2,0xab]
-          vfpclassps k2, xmm18, 0xab
-
-// CHECK: vfpclassps k2 {k7}, xmm18, 171
-// CHECK:  encoding: [0x62,0xb3,0x7d,0x0f,0x66,0xd2,0xab]
-          vfpclassps k2 {k7}, xmm18, 0xab
-
-// CHECK: vfpclassps k2, xmmword ptr [rcx], 123
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x08,0x66,0x11,0x7b]
-          vfpclassps k2, xmmword ptr [rcx], 0x7b
-
-// CHECK: vfpclassps k2 {k7}, xmmword ptr [rcx], 123
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x0f,0x66,0x11,0x7b]
-          vfpclassps k2 {k7}, xmmword ptr [rcx], 0x7b
-
-// CHECK: vfpclassps k2, dword ptr [rcx]{1to4}, 123
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x18,0x66,0x11,0x7b]
-          vfpclassps k2, dword ptr [rcx]{1to4}, 0x7b
-
-// CHECK: vfpclassps k2 {k7}, dword ptr [rcx]{1to4}, 123
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x1f,0x66,0x11,0x7b]
-          vfpclassps k2 {k7}, dword ptr [rcx]{1to4}, 0x7b
-
-// CHECK: vfpclasspd k2, ymm18, 171
-// CHECK:  encoding: [0x62,0xb3,0xfd,0x28,0x66,0xd2,0xab]
-          vfpclasspd k2, ymm18, 0xab
-
-// CHECK: vfpclasspd k2 {k7}, ymm18, 171
-// CHECK:  encoding: [0x62,0xb3,0xfd,0x2f,0x66,0xd2,0xab]
-          vfpclasspd k2 {k7}, ymm18, 0xab
-
-// CHECK: vfpclasspd k2, ymmword ptr [rcx], 123
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x28,0x66,0x11,0x7b]
-          vfpclasspd k2, ymmword ptr [rcx], 0x7b
-
-// CHECK: vfpclasspd k2 {k7}, ymmword ptr [rcx], 123
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x2f,0x66,0x11,0x7b]
-          vfpclasspd k2 {k7}, ymmword ptr [rcx], 0x7b
-
-// CHECK: vfpclasspd k2, qword ptr [rcx]{1to4}, 123
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x38,0x66,0x11,0x7b]
-          vfpclasspd k2, qword ptr [rcx]{1to4}, 0x7b
-
-// CHECK: vfpclasspd k2 {k7}, qword ptr [rcx]{1to4}, 123
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x3f,0x66,0x11,0x7b]
-          vfpclasspd k2 {k7}, qword ptr [rcx]{1to4}, 0x7b
-
-// CHECK: vfpclassps k2, ymm18, 171
-// CHECK:  encoding: [0x62,0xb3,0x7d,0x28,0x66,0xd2,0xab]
-          vfpclassps k2, ymm18, 0xab
-
-// CHECK: vfpclassps k2 {k7}, ymm18, 171
-// CHECK:  encoding: [0x62,0xb3,0x7d,0x2f,0x66,0xd2,0xab]
-          vfpclassps k2 {k7}, ymm18, 0xab
-
-// CHECK: vfpclassps k2, ymmword ptr [rcx], 123
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x28,0x66,0x11,0x7b]
-          vfpclassps k2, ymmword ptr [rcx], 0x7b
-
-// CHECK: vfpclassps k2 {k7}, ymmword ptr [rcx], 123
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x2f,0x66,0x11,0x7b]
-          vfpclassps k2 {k7}, ymmword ptr [rcx], 0x7b
-
-// CHECK: vfpclassps k2, dword ptr [rcx]{1to8}, 123
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x38,0x66,0x11,0x7b]
-          vfpclassps k2, dword ptr [rcx]{1to8}, 0x7b
-
-// CHECK: vfpclassps k2 {k7}, dword ptr [rcx]{1to8}, 123
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x3f,0x66,0x11,0x7b]
-          vfpclassps k2 {k7}, dword ptr [rcx]{1to8}, 0x7b
diff --git a/llvm/test/MC/X86/lwp-x86_64.s b/llvm/test/MC/X86/lwp-64-att.s
similarity index 100%
rename from llvm/test/MC/X86/lwp-x86_64.s
rename to llvm/test/MC/X86/lwp-64-att.s
diff --git a/llvm/test/MC/X86/lwp.s b/llvm/test/MC/X86/lwp-att.s
similarity index 100%
rename from llvm/test/MC/X86/lwp.s
rename to llvm/test/MC/X86/lwp-att.s
diff --git a/llvm/test/MC/X86/x86-64-msrlist.s b/llvm/test/MC/X86/msrlist-64-att.s
similarity index 100%
rename from llvm/test/MC/X86/x86-64-msrlist.s
rename to llvm/test/MC/X86/msrlist-64-att.s
diff --git a/llvm/test/MC/X86/x86_64-rand-encoding.s b/llvm/test/MC/X86/rand-att.s
similarity index 100%
rename from llvm/test/MC/X86/x86_64-rand-encoding.s
rename to llvm/test/MC/X86/rand-att.s
diff --git a/llvm/test/MC/X86/x86-64-rao-int-att.s b/llvm/test/MC/X86/raoint-64-att.s
similarity index 100%
rename from llvm/test/MC/X86/x86-64-rao-int-att.s
rename to llvm/test/MC/X86/raoint-64-att.s
diff --git a/llvm/test/MC/X86/x86-64-rao-int-intel.s b/llvm/test/MC/X86/raoint-64-intel.s
similarity index 100%
rename from llvm/test/MC/X86/x86-64-rao-int-intel.s
rename to llvm/test/MC/X86/raoint-64-intel.s
diff --git a/llvm/test/MC/X86/x86_64-rtm-encoding.s b/llvm/test/MC/X86/rtm-att.s
similarity index 100%
rename from llvm/test/MC/X86/x86_64-rtm-encoding.s
rename to llvm/test/MC/X86/rtm-att.s
diff --git a/llvm/test/MC/X86/x86_64-sse4a.s b/llvm/test/MC/X86/sse4a-att.s
similarity index 100%
rename from llvm/test/MC/X86/x86_64-sse4a.s
rename to llvm/test/MC/X86/sse4a-att.s
diff --git a/llvm/test/MC/X86/x86_64-tbm-encoding.s b/llvm/test/MC/X86/tbm-att.s
similarity index 100%
rename from llvm/test/MC/X86/x86_64-tbm-encoding.s
rename to llvm/test/MC/X86/tbm-att.s
diff --git a/llvm/test/MC/X86/x86-32-avx512_vp2intersect-intel.s b/llvm/test/MC/X86/x86-32-avx512_vp2intersect-intel.s
deleted file mode 100644
index 3ea3e4fd6465409726646a442309b43abeec78c1..0000000000000000000000000000000000000000
--- a/llvm/test/MC/X86/x86-32-avx512_vp2intersect-intel.s
+++ /dev/null
@@ -1,57 +0,0 @@
-// RUN: llvm-mc -triple i386-unknown-unknown -x86-asm-syntax=intel -output-asm-variant=1 --show-encoding %s | FileCheck %s
-// CHECK: vp2intersectd k4, zmm3, zmm4
-// CHECK: encoding: [0x62,0xf2,0x67,0x48,0x68,0xe4]
-          vp2intersectd k4, zmm3, zmm4
-
-// CHECK: vp2intersectd k4, zmm3, zmmword ptr [esp + 8*esi + 268435456]
-// CHECK: encoding: [0x62,0xf2,0x67,0x48,0x68,0xa4,0xf4,0x00,0x00,0x00,0x10]
-          vp2intersectd k4, zmm3, zmmword ptr [esp + 8*esi + 268435456]
-
-// CHECK: vp2intersectd k4, zmm3, zmmword ptr [edi + 4*eax + 291]
-// CHECK: encoding: [0x62,0xf2,0x67,0x48,0x68,0xa4,0x87,0x23,0x01,0x00,0x00]
-          vp2intersectd k4, zmm3, zmmword ptr [edi + 4*eax + 291]
-
-// CHECK: vp2intersectd k4, zmm3, dword ptr [eax]{1to16}
-// CHECK: encoding: [0x62,0xf2,0x67,0x58,0x68,0x20]
-          vp2intersectd k4, zmm3, dword ptr [eax]{1to16}
-
-// CHECK: vp2intersectd k4, zmm3, zmmword ptr [2*ebp - 2048]
-// CHECK: encoding: [0x62,0xf2,0x67,0x48,0x68,0x24,0x6d,0x00,0xf8,0xff,0xff]
-          vp2intersectd k4, zmm3, zmmword ptr [2*ebp - 2048]
-
-// CHECK: vp2intersectd k4, zmm3, zmmword ptr [ecx + 8128]
-// CHECK: encoding: [0x62,0xf2,0x67,0x48,0x68,0x61,0x7f]
-          vp2intersectd k4, zmm3, zmmword ptr [ecx + 8128]
-
-// CHECK: vp2intersectd k4, zmm3, dword ptr [edx - 512]{1to16}
-// CHECK: encoding: [0x62,0xf2,0x67,0x58,0x68,0x62,0x80]
-          vp2intersectd k4, zmm3, dword ptr [edx - 512]{1to16}
-
-// CHECK: vp2intersectq k4, zmm3, zmm4
-// CHECK: encoding: [0x62,0xf2,0xe7,0x48,0x68,0xe4]
-          vp2intersectq k4, zmm3, zmm4
-
-// CHECK: vp2intersectq k4, zmm3, zmmword ptr [esp + 8*esi + 268435456]
-// CHECK: encoding: [0x62,0xf2,0xe7,0x48,0x68,0xa4,0xf4,0x00,0x00,0x00,0x10]
-          vp2intersectq k4, zmm3, zmmword ptr [esp + 8*esi + 268435456]
-
-// CHECK: vp2intersectq k4, zmm3, zmmword ptr [edi + 4*eax + 291]
-// CHECK: encoding: [0x62,0xf2,0xe7,0x48,0x68,0xa4,0x87,0x23,0x01,0x00,0x00]
-          vp2intersectq k4, zmm3, zmmword ptr [edi + 4*eax + 291]
-
-// CHECK: vp2intersectq k4, zmm3, qword ptr [eax]{1to8}
-// CHECK: encoding: [0x62,0xf2,0xe7,0x58,0x68,0x20]
-          vp2intersectq k4, zmm3, qword ptr [eax]{1to8}
-
-// CHECK: vp2intersectq k4, zmm3, zmmword ptr [2*ebp - 2048]
-// CHECK: encoding: [0x62,0xf2,0xe7,0x48,0x68,0x24,0x6d,0x00,0xf8,0xff,0xff]
-          vp2intersectq k4, zmm3, zmmword ptr [2*ebp - 2048]
-
-// CHECK: vp2intersectq k4, zmm3, zmmword ptr [ecx + 8128]
-// CHECK: encoding: [0x62,0xf2,0xe7,0x48,0x68,0x61,0x7f]
-          vp2intersectq k4, zmm3, zmmword ptr [ecx + 8128]
-
-// CHECK: vp2intersectq k4, zmm3, qword ptr [edx - 1024]{1to8}
-// CHECK: encoding: [0x62,0xf2,0xe7,0x58,0x68,0x62,0x80]
-          vp2intersectq k4, zmm3, qword ptr [edx - 1024]{1to8}
-
diff --git a/llvm/test/MC/X86/x86-32-avx512vp2intersect-att.s b/llvm/test/MC/X86/x86-32-avx512vp2intersect-att.s
deleted file mode 100644
index 49aca2a32126d377f12c991ca0208bcbb48d8fdf..0000000000000000000000000000000000000000
--- a/llvm/test/MC/X86/x86-32-avx512vp2intersect-att.s
+++ /dev/null
@@ -1,225 +0,0 @@
-// RUN: llvm-mc -triple i386-unknown-unknown --show-encoding %s | FileCheck %s
-
-// CHECK: vp2intersectq        %zmm2, %zmm1, %k0
-// CHECK: encoding: [0x62,0xf2,0xf7,0x48,0x68,0xc2]
-vp2intersectq  %zmm2, %zmm1, %k0
-
-// CHECK: vp2intersectq        (%edi), %zmm1, %k0
-// CHECK: encoding: [0x62,0xf2,0xf7,0x48,0x68,0x07]
-vp2intersectq  (%edi), %zmm1, %k0
-
-// CHECK: vp2intersectq        (%edi){1to8}, %zmm1, %k0
-// CHECK: encoding: [0x62,0xf2,0xf7,0x58,0x68,0x07]
-vp2intersectq  (%edi){1to8}, %zmm1, %k0
-
-// CHECK: vp2intersectq        %zmm2, %zmm1, %k0
-// CHECK: encoding: [0x62,0xf2,0xf7,0x48,0x68,0xc2]
-vp2intersectq  %zmm2, %zmm1, %k1
-
-// CHECK: vp2intersectq        (%edi), %zmm1, %k0
-// CHECK: encoding: [0x62,0xf2,0xf7,0x48,0x68,0x07]
-vp2intersectq  (%edi), %zmm1, %k1
-
-// CHECK: vp2intersectq        (%edi){1to8}, %zmm1, %k0
-// CHECK: encoding: [0x62,0xf2,0xf7,0x58,0x68,0x07]
-vp2intersectq  (%edi){1to8}, %zmm1, %k1
-
-// CHECK: vp2intersectq        %zmm7, %zmm4, %k6
-// CHECK: encoding: [0x62,0xf2,0xdf,0x48,0x68,0xf7]
-vp2intersectq  %zmm7, %zmm4, %k6
-
-// CHECK: vp2intersectq        (%esi), %zmm4, %k6
-// CHECK: encoding: [0x62,0xf2,0xdf,0x48,0x68,0x36]
-vp2intersectq  (%esi), %zmm4, %k6
-
-// CHECK: vp2intersectq        (%esi){1to8}, %zmm4, %k6
-// CHECK: encoding: [0x62,0xf2,0xdf,0x58,0x68,0x36]
-vp2intersectq  (%esi){1to8}, %zmm4, %k6
-
-// CHECK: vp2intersectq        %zmm7, %zmm4, %k6
-// CHECK: encoding: [0x62,0xf2,0xdf,0x48,0x68,0xf7]
-vp2intersectq  %zmm7, %zmm4, %k7
-
-// CHECK: vp2intersectq        (%esi), %zmm4, %k6
-// CHECK: encoding: [0x62,0xf2,0xdf,0x48,0x68,0x36]
-vp2intersectq  (%esi), %zmm4, %k7
-
-// CHECK: vp2intersectq        (%esi){1to8}, %zmm4, %k6
-// CHECK: encoding: [0x62,0xf2,0xdf,0x58,0x68,0x36]
-vp2intersectq  (%esi){1to8}, %zmm4, %k7
-
-// CHECK: vp2intersectq        %ymm2, %ymm1, %k0
-// CHECK: encoding: [0x62,0xf2,0xf7,0x28,0x68,0xc2]
-vp2intersectq  %ymm2, %ymm1, %k0
-
-// CHECK: vp2intersectq        (%edi), %ymm1, %k0
-// CHECK: encoding: [0x62,0xf2,0xf7,0x28,0x68,0x07]
-vp2intersectq  (%edi), %ymm1, %k0
-
-// CHECK: vp2intersectq        (%edi){1to4}, %ymm1, %k0
-// CHECK: encoding: [0x62,0xf2,0xf7,0x38,0x68,0x07]
-vp2intersectq  (%edi){1to4}, %ymm1, %k0
-
-// CHECK: vp2intersectq        %ymm2, %ymm1, %k0
-// CHECK: encoding: [0x62,0xf2,0xf7,0x28,0x68,0xc2]
-vp2intersectq  %ymm2, %ymm1, %k1
-
-// CHECK: vp2intersectq        (%edi), %ymm1, %k0
-// CHECK: encoding: [0x62,0xf2,0xf7,0x28,0x68,0x07]
-vp2intersectq  (%edi), %ymm1, %k1
-
-// CHECK: vp2intersectq        (%edi){1to4}, %ymm1, %k0
-// CHECK: encoding: [0x62,0xf2,0xf7,0x38,0x68,0x07]
-vp2intersectq  (%edi){1to4}, %ymm1, %k1
-
-// CHECK: vp2intersectq        %ymm7, %ymm4, %k6
-// CHECK: encoding: [0x62,0xf2,0xdf,0x28,0x68,0xf7]
-vp2intersectq  %ymm7, %ymm4, %k6
-
-// CHECK: vp2intersectq        (%esi), %ymm4, %k6
-// CHECK: encoding: [0x62,0xf2,0xdf,0x28,0x68,0x36]
-vp2intersectq  (%esi), %ymm4, %k6
-
-// CHECK: vp2intersectq        (%esi){1to4}, %ymm4, %k6
-// CHECK: encoding: [0x62,0xf2,0xdf,0x38,0x68,0x36]
-vp2intersectq  (%esi){1to4}, %ymm4, %k6
-
-// CHECK: vp2intersectq        %ymm7, %ymm4, %k6
-// CHECK: encoding: [0x62,0xf2,0xdf,0x28,0x68,0xf7]
-vp2intersectq  %ymm7, %ymm4, %k7
-
-// CHECK: vp2intersectq        (%esi), %ymm4, %k6
-// CHECK: encoding: [0x62,0xf2,0xdf,0x28,0x68,0x36]
-vp2intersectq  (%esi), %ymm4, %k7
-
-// CHECK: vp2intersectq        %xmm2, %xmm1, %k0
-// CHECK: encoding: [0x62,0xf2,0xf7,0x08,0x68,0xc2]
-vp2intersectq  %xmm2, %xmm1, %k0
-
-// CHECK: vp2intersectq        (%edi), %xmm1, %k0
-// CHECK: encoding: [0x62,0xf2,0xf7,0x08,0x68,0x07]
-vp2intersectq  (%edi), %xmm1, %k0
-
-// CHECK: vp2intersectq        (%edi){1to2}, %xmm1, %k0
-// CHECK: encoding: [0x62,0xf2,0xf7,0x18,0x68,0x07]
-vp2intersectq  (%edi){1to2}, %xmm1, %k0
-
-// CHECK: vp2intersectq        %xmm2, %xmm1, %k0
-// CHECK: encoding: [0x62,0xf2,0xf7,0x08,0x68,0xc2]
-vp2intersectq  %xmm2, %xmm1, %k1
-
-// CHECK: vp2intersectq        (%edi), %xmm1, %k0
-// CHECK: encoding: [0x62,0xf2,0xf7,0x08,0x68,0x07]
-vp2intersectq  (%edi), %xmm1, %k1
-
-// CHECK: vp2intersectq        %xmm7, %xmm4, %k6
-// CHECK: encoding: [0x62,0xf2,0xdf,0x08,0x68,0xf7]
-vp2intersectq  %xmm7, %xmm4, %k6
-
-// CHECK: vp2intersectq        (%esi), %xmm4, %k6
-// CHECK: encoding: [0x62,0xf2,0xdf,0x08,0x68,0x36]
-vp2intersectq  (%esi), %xmm4, %k6
-
-// CHECK: vp2intersectq        %xmm7, %xmm4, %k6
-// CHECK: encoding: [0x62,0xf2,0xdf,0x08,0x68,0xf7]
-vp2intersectq  %xmm7, %xmm4, %k7
-
-// CHECK: vp2intersectq        (%esi), %xmm4, %k6
-// CHECK: encoding: [0x62,0xf2,0xdf,0x08,0x68,0x36]
-vp2intersectq  (%esi), %xmm4, %k7
-
-// CHECK: vp2intersectd        %zmm2, %zmm1, %k0
-// CHECK: encoding: [0x62,0xf2,0x77,0x48,0x68,0xc2]
-vp2intersectd  %zmm2, %zmm1, %k0
-
-// CHECK: vp2intersectd        (%edi), %zmm1, %k0
-// CHECK: encoding: [0x62,0xf2,0x77,0x48,0x68,0x07]
-vp2intersectd  (%edi), %zmm1, %k0
-
-// CHECK: vp2intersectd        %zmm2, %zmm1, %k0
-// CHECK: encoding: [0x62,0xf2,0x77,0x48,0x68,0xc2]
-vp2intersectd  %zmm2, %zmm1, %k1
-
-// CHECK: vp2intersectd        (%edi), %zmm1, %k0
-// CHECK: encoding: [0x62,0xf2,0x77,0x48,0x68,0x07]
-vp2intersectd  (%edi), %zmm1, %k1
-
-// CHECK: vp2intersectd        %zmm7, %zmm4, %k6
-// CHECK: encoding: [0x62,0xf2,0x5f,0x48,0x68,0xf7]
-vp2intersectd  %zmm7, %zmm4, %k6
-
-// CHECK: vp2intersectd        (%esi), %zmm4, %k6
-// CHECK: encoding: [0x62,0xf2,0x5f,0x48,0x68,0x36]
-vp2intersectd  (%esi), %zmm4, %k6
-
-// CHECK: vp2intersectd        %zmm7, %zmm4, %k6
-// CHECK: encoding: [0x62,0xf2,0x5f,0x48,0x68,0xf7]
-vp2intersectd  %zmm7, %zmm4, %k7
-
-// CHECK: vp2intersectd        (%esi), %zmm4, %k6
-// CHECK: encoding: [0x62,0xf2,0x5f,0x48,0x68,0x36]
-vp2intersectd  (%esi), %zmm4, %k7
-
-// CHECK: vp2intersectd        %ymm2, %ymm1, %k0
-// CHECK: encoding: [0x62,0xf2,0x77,0x28,0x68,0xc2]
-vp2intersectd  %ymm2, %ymm1, %k0
-
-// CHECK: vp2intersectd        (%edi), %ymm1, %k0
-// CHECK: encoding: [0x62,0xf2,0x77,0x28,0x68,0x07]
-vp2intersectd  (%edi), %ymm1, %k0
-
-// CHECK: vp2intersectd        %ymm2, %ymm1, %k0
-// CHECK: encoding: [0x62,0xf2,0x77,0x28,0x68,0xc2]
-vp2intersectd  %ymm2, %ymm1, %k1
-
-// CHECK: vp2intersectd        (%edi), %ymm1, %k0
-// CHECK: encoding: [0x62,0xf2,0x77,0x28,0x68,0x07]
-vp2intersectd  (%edi), %ymm1, %k1
-
-// CHECK: vp2intersectd        %ymm7, %ymm4, %k6
-// CHECK: encoding: [0x62,0xf2,0x5f,0x28,0x68,0xf7]
-vp2intersectd  %ymm7, %ymm4, %k6
-
-// CHECK: vp2intersectd        (%esi), %ymm4, %k6
-// CHECK: encoding: [0x62,0xf2,0x5f,0x28,0x68,0x36]
-vp2intersectd  (%esi), %ymm4, %k6
-
-// CHECK: vp2intersectd        %ymm7, %ymm4, %k6
-// CHECK: encoding: [0x62,0xf2,0x5f,0x28,0x68,0xf7]
-vp2intersectd  %ymm7, %ymm4, %k7
-
-// CHECK: vp2intersectd        (%esi), %ymm4, %k6
-// CHECK: encoding: [0x62,0xf2,0x5f,0x28,0x68,0x36]
-vp2intersectd  (%esi), %ymm4, %k7
-
-// CHECK: vp2intersectd        %xmm2, %xmm1, %k0
-// CHECK: encoding: [0x62,0xf2,0x77,0x08,0x68,0xc2]
-vp2intersectd  %xmm2, %xmm1, %k0
-
-// CHECK: vp2intersectd        (%edi), %xmm1, %k0
-// CHECK: encoding: [0x62,0xf2,0x77,0x08,0x68,0x07]
-vp2intersectd  (%edi), %xmm1, %k0
-
-// CHECK: vp2intersectd        %xmm2, %xmm1, %k0
-// CHECK: encoding: [0x62,0xf2,0x77,0x08,0x68,0xc2]
-vp2intersectd  %xmm2, %xmm1, %k1
-
-// CHECK: vp2intersectd        (%edi), %xmm1, %k0
-// CHECK: encoding: [0x62,0xf2,0x77,0x08,0x68,0x07]
-vp2intersectd  (%edi), %xmm1, %k1
-
-// CHECK: vp2intersectd        %xmm7, %xmm4, %k6
-// CHECK: encoding: [0x62,0xf2,0x5f,0x08,0x68,0xf7]
-vp2intersectd  %xmm7, %xmm4, %k6
-
-// CHECK: vp2intersectd        (%esi), %xmm4, %k6
-// CHECK: encoding: [0x62,0xf2,0x5f,0x08,0x68,0x36]
-vp2intersectd  (%esi), %xmm4, %k6
-
-// CHECK: vp2intersectd        %xmm7, %xmm4, %k6
-// CHECK: encoding: [0x62,0xf2,0x5f,0x08,0x68,0xf7]
-vp2intersectd  %xmm7, %xmm4, %k7
-
-// CHECK: vp2intersectd        (%esi), %xmm4, %k6
-// CHECK: encoding: [0x62,0xf2,0x5f,0x08,0x68,0x36]
-vp2intersectd  (%esi), %xmm4, %k7
diff --git a/llvm/test/MC/X86/x86-64-avx512_vp2intersect-intel.s b/llvm/test/MC/X86/x86-64-avx512_vp2intersect-intel.s
deleted file mode 100644
index 1dd9501bb26132694d6cc18aa390463565c0618b..0000000000000000000000000000000000000000
--- a/llvm/test/MC/X86/x86-64-avx512_vp2intersect-intel.s
+++ /dev/null
@@ -1,57 +0,0 @@
-// RUN: llvm-mc -triple x86_64-unknown-unknown -x86-asm-syntax=intel -output-asm-variant=1 --show-encoding %s | FileCheck %s
-// CHECK: vp2intersectd k4, zmm23, zmm24
-// CHECK: encoding: [0x62,0x92,0x47,0x40,0x68,0xe0]
-          vp2intersectd k4, zmm23, zmm24
-
-// CHECK: vp2intersectd k4, zmm23, zmmword ptr [rbp + 8*r14 + 268435456]
-// CHECK: encoding: [0x62,0xb2,0x47,0x40,0x68,0xa4,0xf5,0x00,0x00,0x00,0x10]
-          vp2intersectd k4, zmm23, zmmword ptr [rbp + 8*r14 + 268435456]
-
-// CHECK: vp2intersectd k4, zmm23, zmmword ptr [r8 + 4*rax + 291]
-// CHECK: encoding: [0x62,0xd2,0x47,0x40,0x68,0xa4,0x80,0x23,0x01,0x00,0x00]
-          vp2intersectd k4, zmm23, zmmword ptr [r8 + 4*rax + 291]
-
-// CHECK: vp2intersectd k4, zmm23, dword ptr [rip]{1to16}
-// CHECK: encoding: [0x62,0xf2,0x47,0x50,0x68,0x25,0x00,0x00,0x00,0x00]
-          vp2intersectd k4, zmm23, dword ptr [rip]{1to16}
-
-// CHECK: vp2intersectd k4, zmm23, zmmword ptr [2*rbp - 2048]
-// CHECK: encoding: [0x62,0xf2,0x47,0x40,0x68,0x24,0x6d,0x00,0xf8,0xff,0xff]
-          vp2intersectd k4, zmm23, zmmword ptr [2*rbp - 2048]
-
-// CHECK: vp2intersectd k4, zmm23, zmmword ptr [rcx + 8128]
-// CHECK: encoding: [0x62,0xf2,0x47,0x40,0x68,0x61,0x7f]
-          vp2intersectd k4, zmm23, zmmword ptr [rcx + 8128]
-
-// CHECK: vp2intersectd k4, zmm23, dword ptr [rdx - 512]{1to16}
-// CHECK: encoding: [0x62,0xf2,0x47,0x50,0x68,0x62,0x80]
-          vp2intersectd k4, zmm23, dword ptr [rdx - 512]{1to16}
-
-// CHECK: vp2intersectq k4, zmm23, zmm24
-// CHECK: encoding: [0x62,0x92,0xc7,0x40,0x68,0xe0]
-          vp2intersectq k4, zmm23, zmm24
-
-// CHECK: vp2intersectq k4, zmm23, zmmword ptr [rbp + 8*r14 + 268435456]
-// CHECK: encoding: [0x62,0xb2,0xc7,0x40,0x68,0xa4,0xf5,0x00,0x00,0x00,0x10]
-          vp2intersectq k4, zmm23, zmmword ptr [rbp + 8*r14 + 268435456]
-
-// CHECK: vp2intersectq k4, zmm23, zmmword ptr [r8 + 4*rax + 291]
-// CHECK: encoding: [0x62,0xd2,0xc7,0x40,0x68,0xa4,0x80,0x23,0x01,0x00,0x00]
-          vp2intersectq k4, zmm23, zmmword ptr [r8 + 4*rax + 291]
-
-// CHECK: vp2intersectq k4, zmm23, qword ptr [rip]{1to8}
-// CHECK: encoding: [0x62,0xf2,0xc7,0x50,0x68,0x25,0x00,0x00,0x00,0x00]
-          vp2intersectq k4, zmm23, qword ptr [rip]{1to8}
-
-// CHECK: vp2intersectq k4, zmm23, zmmword ptr [2*rbp - 2048]
-// CHECK: encoding: [0x62,0xf2,0xc7,0x40,0x68,0x24,0x6d,0x00,0xf8,0xff,0xff]
-          vp2intersectq k4, zmm23, zmmword ptr [2*rbp - 2048]
-
-// CHECK: vp2intersectq k4, zmm23, zmmword ptr [rcx + 8128]
-// CHECK: encoding: [0x62,0xf2,0xc7,0x40,0x68,0x61,0x7f]
-          vp2intersectq k4, zmm23, zmmword ptr [rcx + 8128]
-
-// CHECK: vp2intersectq k4, zmm23, qword ptr [rdx - 1024]{1to8}
-// CHECK: encoding: [0x62,0xf2,0xc7,0x50,0x68,0x62,0x80]
-          vp2intersectq k4, zmm23, qword ptr [rdx - 1024]{1to8}
-
diff --git a/llvm/test/MC/X86/x86-64-avx512dq_vl.s b/llvm/test/MC/X86/x86-64-avx512dq_vl.s
deleted file mode 100644
index 3a8f82bd490fcb05c88e38cd7509dbf9acff995c..0000000000000000000000000000000000000000
--- a/llvm/test/MC/X86/x86-64-avx512dq_vl.s
+++ /dev/null
@@ -1,6350 +0,0 @@
-// RUN: llvm-mc -triple x86_64-unknown-unknown --show-encoding %s | FileCheck %s
-
-// CHECK: vpmullq %xmm22, %xmm17, %xmm26
-// CHECK:  encoding: [0x62,0x22,0xf5,0x00,0x40,0xd6]
-          vpmullq %xmm22, %xmm17, %xmm26
-
-// CHECK: vpmullq %xmm22, %xmm17, %xmm26 {%k6}
-// CHECK:  encoding: [0x62,0x22,0xf5,0x06,0x40,0xd6]
-          vpmullq %xmm22, %xmm17, %xmm26 {%k6}
-
-// CHECK: vpmullq %xmm22, %xmm17, %xmm26 {%k6} {z}
-// CHECK:  encoding: [0x62,0x22,0xf5,0x86,0x40,0xd6]
-          vpmullq %xmm22, %xmm17, %xmm26 {%k6} {z}
-
-// CHECK: vpmullq (%rcx), %xmm17, %xmm26
-// CHECK:  encoding: [0x62,0x62,0xf5,0x00,0x40,0x11]
-          vpmullq (%rcx), %xmm17, %xmm26
-
-// CHECK: vpmullq 291(%rax,%r14,8), %xmm17, %xmm26
-// CHECK:  encoding: [0x62,0x22,0xf5,0x00,0x40,0x94,0xf0,0x23,0x01,0x00,0x00]
-          vpmullq 291(%rax,%r14,8), %xmm17, %xmm26
-
-// CHECK: vpmullq (%rcx){1to2}, %xmm17, %xmm26
-// CHECK:  encoding: [0x62,0x62,0xf5,0x10,0x40,0x11]
-          vpmullq (%rcx){1to2}, %xmm17, %xmm26
-
-// CHECK: vpmullq 2032(%rdx), %xmm17, %xmm26
-// CHECK:  encoding: [0x62,0x62,0xf5,0x00,0x40,0x52,0x7f]
-          vpmullq 2032(%rdx), %xmm17, %xmm26
-
-// CHECK: vpmullq 2048(%rdx), %xmm17, %xmm26
-// CHECK:  encoding: [0x62,0x62,0xf5,0x00,0x40,0x92,0x00,0x08,0x00,0x00]
-          vpmullq 2048(%rdx), %xmm17, %xmm26
-
-// CHECK: vpmullq -2048(%rdx), %xmm17, %xmm26
-// CHECK:  encoding: [0x62,0x62,0xf5,0x00,0x40,0x52,0x80]
-          vpmullq -2048(%rdx), %xmm17, %xmm26
-
-// CHECK: vpmullq -2064(%rdx), %xmm17, %xmm26
-// CHECK:  encoding: [0x62,0x62,0xf5,0x00,0x40,0x92,0xf0,0xf7,0xff,0xff]
-          vpmullq -2064(%rdx), %xmm17, %xmm26
-
-// CHECK: vpmullq 1016(%rdx){1to2}, %xmm17, %xmm26
-// CHECK:  encoding: [0x62,0x62,0xf5,0x10,0x40,0x52,0x7f]
-          vpmullq 1016(%rdx){1to2}, %xmm17, %xmm26
-
-// CHECK: vpmullq 1024(%rdx){1to2}, %xmm17, %xmm26
-// CHECK:  encoding: [0x62,0x62,0xf5,0x10,0x40,0x92,0x00,0x04,0x00,0x00]
-          vpmullq 1024(%rdx){1to2}, %xmm17, %xmm26
-
-// CHECK: vpmullq -1024(%rdx){1to2}, %xmm17, %xmm26
-// CHECK:  encoding: [0x62,0x62,0xf5,0x10,0x40,0x52,0x80]
-          vpmullq -1024(%rdx){1to2}, %xmm17, %xmm26
-
-// CHECK: vpmullq -1032(%rdx){1to2}, %xmm17, %xmm26
-// CHECK:  encoding: [0x62,0x62,0xf5,0x10,0x40,0x92,0xf8,0xfb,0xff,0xff]
-          vpmullq -1032(%rdx){1to2}, %xmm17, %xmm26
-
-// CHECK: vpmullq %ymm25, %ymm25, %ymm25
-// CHECK:  encoding: [0x62,0x02,0xb5,0x20,0x40,0xc9]
-          vpmullq %ymm25, %ymm25, %ymm25
-
-// CHECK: vpmullq %ymm25, %ymm25, %ymm25 {%k3}
-// CHECK:  encoding: [0x62,0x02,0xb5,0x23,0x40,0xc9]
-          vpmullq %ymm25, %ymm25, %ymm25 {%k3}
-
-// CHECK: vpmullq %ymm25, %ymm25, %ymm25 {%k3} {z}
-// CHECK:  encoding: [0x62,0x02,0xb5,0xa3,0x40,0xc9]
-          vpmullq %ymm25, %ymm25, %ymm25 {%k3} {z}
-
-// CHECK: vpmullq (%rcx), %ymm25, %ymm25
-// CHECK:  encoding: [0x62,0x62,0xb5,0x20,0x40,0x09]
-          vpmullq (%rcx), %ymm25, %ymm25
-
-// CHECK: vpmullq 291(%rax,%r14,8), %ymm25, %ymm25
-// CHECK:  encoding: [0x62,0x22,0xb5,0x20,0x40,0x8c,0xf0,0x23,0x01,0x00,0x00]
-          vpmullq 291(%rax,%r14,8), %ymm25, %ymm25
-
-// CHECK: vpmullq (%rcx){1to4}, %ymm25, %ymm25
-// CHECK:  encoding: [0x62,0x62,0xb5,0x30,0x40,0x09]
-          vpmullq (%rcx){1to4}, %ymm25, %ymm25
-
-// CHECK: vpmullq 4064(%rdx), %ymm25, %ymm25
-// CHECK:  encoding: [0x62,0x62,0xb5,0x20,0x40,0x4a,0x7f]
-          vpmullq 4064(%rdx), %ymm25, %ymm25
-
-// CHECK: vpmullq 4096(%rdx), %ymm25, %ymm25
-// CHECK:  encoding: [0x62,0x62,0xb5,0x20,0x40,0x8a,0x00,0x10,0x00,0x00]
-          vpmullq 4096(%rdx), %ymm25, %ymm25
-
-// CHECK: vpmullq -4096(%rdx), %ymm25, %ymm25
-// CHECK:  encoding: [0x62,0x62,0xb5,0x20,0x40,0x4a,0x80]
-          vpmullq -4096(%rdx), %ymm25, %ymm25
-
-// CHECK: vpmullq -4128(%rdx), %ymm25, %ymm25
-// CHECK:  encoding: [0x62,0x62,0xb5,0x20,0x40,0x8a,0xe0,0xef,0xff,0xff]
-          vpmullq -4128(%rdx), %ymm25, %ymm25
-
-// CHECK: vpmullq 1016(%rdx){1to4}, %ymm25, %ymm25
-// CHECK:  encoding: [0x62,0x62,0xb5,0x30,0x40,0x4a,0x7f]
-          vpmullq 1016(%rdx){1to4}, %ymm25, %ymm25
-
-// CHECK: vpmullq 1024(%rdx){1to4}, %ymm25, %ymm25
-// CHECK:  encoding: [0x62,0x62,0xb5,0x30,0x40,0x8a,0x00,0x04,0x00,0x00]
-          vpmullq 1024(%rdx){1to4}, %ymm25, %ymm25
-
-// CHECK: vpmullq -1024(%rdx){1to4}, %ymm25, %ymm25
-// CHECK:  encoding: [0x62,0x62,0xb5,0x30,0x40,0x4a,0x80]
-          vpmullq -1024(%rdx){1to4}, %ymm25, %ymm25
-
-// CHECK: vpmullq -1032(%rdx){1to4}, %ymm25, %ymm25
-// CHECK:  encoding: [0x62,0x62,0xb5,0x30,0x40,0x8a,0xf8,0xfb,0xff,0xff]
-          vpmullq -1032(%rdx){1to4}, %ymm25, %ymm25
-
-// CHECK: vandpd %xmm20, %xmm29, %xmm21
-// CHECK:  encoding: [0x62,0xa1,0x95,0x00,0x54,0xec]
-          vandpd %xmm20, %xmm29, %xmm21
-
-// CHECK: vandpd %xmm20, %xmm29, %xmm21 {%k6}
-// CHECK:  encoding: [0x62,0xa1,0x95,0x06,0x54,0xec]
-          vandpd %xmm20, %xmm29, %xmm21 {%k6}
-
-// CHECK: vandpd %xmm20, %xmm29, %xmm21 {%k6} {z}
-// CHECK:  encoding: [0x62,0xa1,0x95,0x86,0x54,0xec]
-          vandpd %xmm20, %xmm29, %xmm21 {%k6} {z}
-
-// CHECK: vandpd (%rcx), %xmm29, %xmm21
-// CHECK:  encoding: [0x62,0xe1,0x95,0x00,0x54,0x29]
-          vandpd (%rcx), %xmm29, %xmm21
-
-// CHECK: vandpd 291(%rax,%r14,8), %xmm29, %xmm21
-// CHECK:  encoding: [0x62,0xa1,0x95,0x00,0x54,0xac,0xf0,0x23,0x01,0x00,0x00]
-          vandpd 291(%rax,%r14,8), %xmm29, %xmm21
-
-// CHECK: vandpd (%rcx){1to2}, %xmm29, %xmm21
-// CHECK:  encoding: [0x62,0xe1,0x95,0x10,0x54,0x29]
-          vandpd (%rcx){1to2}, %xmm29, %xmm21
-
-// CHECK: vandpd 2032(%rdx), %xmm29, %xmm21
-// CHECK:  encoding: [0x62,0xe1,0x95,0x00,0x54,0x6a,0x7f]
-          vandpd 2032(%rdx), %xmm29, %xmm21
-
-// CHECK: vandpd 2048(%rdx), %xmm29, %xmm21
-// CHECK:  encoding: [0x62,0xe1,0x95,0x00,0x54,0xaa,0x00,0x08,0x00,0x00]
-          vandpd 2048(%rdx), %xmm29, %xmm21
-
-// CHECK: vandpd -2048(%rdx), %xmm29, %xmm21
-// CHECK:  encoding: [0x62,0xe1,0x95,0x00,0x54,0x6a,0x80]
-          vandpd -2048(%rdx), %xmm29, %xmm21
-
-// CHECK: vandpd -2064(%rdx), %xmm29, %xmm21
-// CHECK:  encoding: [0x62,0xe1,0x95,0x00,0x54,0xaa,0xf0,0xf7,0xff,0xff]
-          vandpd -2064(%rdx), %xmm29, %xmm21
-
-// CHECK: vandpd 1016(%rdx){1to2}, %xmm29, %xmm21
-// CHECK:  encoding: [0x62,0xe1,0x95,0x10,0x54,0x6a,0x7f]
-          vandpd 1016(%rdx){1to2}, %xmm29, %xmm21
-
-// CHECK: vandpd 1024(%rdx){1to2}, %xmm29, %xmm21
-// CHECK:  encoding: [0x62,0xe1,0x95,0x10,0x54,0xaa,0x00,0x04,0x00,0x00]
-          vandpd 1024(%rdx){1to2}, %xmm29, %xmm21
-
-// CHECK: vandpd -1024(%rdx){1to2}, %xmm29, %xmm21
-// CHECK:  encoding: [0x62,0xe1,0x95,0x10,0x54,0x6a,0x80]
-          vandpd -1024(%rdx){1to2}, %xmm29, %xmm21
-
-// CHECK: vandpd -1032(%rdx){1to2}, %xmm29, %xmm21
-// CHECK:  encoding: [0x62,0xe1,0x95,0x10,0x54,0xaa,0xf8,0xfb,0xff,0xff]
-          vandpd -1032(%rdx){1to2}, %xmm29, %xmm21
-
-// CHECK: vandpd %ymm28, %ymm21, %ymm28
-// CHECK:  encoding: [0x62,0x01,0xd5,0x20,0x54,0xe4]
-          vandpd %ymm28, %ymm21, %ymm28
-
-// CHECK: vandpd %ymm28, %ymm21, %ymm28 {%k4}
-// CHECK:  encoding: [0x62,0x01,0xd5,0x24,0x54,0xe4]
-          vandpd %ymm28, %ymm21, %ymm28 {%k4}
-
-// CHECK: vandpd %ymm28, %ymm21, %ymm28 {%k4} {z}
-// CHECK:  encoding: [0x62,0x01,0xd5,0xa4,0x54,0xe4]
-          vandpd %ymm28, %ymm21, %ymm28 {%k4} {z}
-
-// CHECK: vandpd (%rcx), %ymm21, %ymm28
-// CHECK:  encoding: [0x62,0x61,0xd5,0x20,0x54,0x21]
-          vandpd (%rcx), %ymm21, %ymm28
-
-// CHECK: vandpd 291(%rax,%r14,8), %ymm21, %ymm28
-// CHECK:  encoding: [0x62,0x21,0xd5,0x20,0x54,0xa4,0xf0,0x23,0x01,0x00,0x00]
-          vandpd 291(%rax,%r14,8), %ymm21, %ymm28
-
-// CHECK: vandpd (%rcx){1to4}, %ymm21, %ymm28
-// CHECK:  encoding: [0x62,0x61,0xd5,0x30,0x54,0x21]
-          vandpd (%rcx){1to4}, %ymm21, %ymm28
-
-// CHECK: vandpd 4064(%rdx), %ymm21, %ymm28
-// CHECK:  encoding: [0x62,0x61,0xd5,0x20,0x54,0x62,0x7f]
-          vandpd 4064(%rdx), %ymm21, %ymm28
-
-// CHECK: vandpd 4096(%rdx), %ymm21, %ymm28
-// CHECK:  encoding: [0x62,0x61,0xd5,0x20,0x54,0xa2,0x00,0x10,0x00,0x00]
-          vandpd 4096(%rdx), %ymm21, %ymm28
-
-// CHECK: vandpd -4096(%rdx), %ymm21, %ymm28
-// CHECK:  encoding: [0x62,0x61,0xd5,0x20,0x54,0x62,0x80]
-          vandpd -4096(%rdx), %ymm21, %ymm28
-
-// CHECK: vandpd -4128(%rdx), %ymm21, %ymm28
-// CHECK:  encoding: [0x62,0x61,0xd5,0x20,0x54,0xa2,0xe0,0xef,0xff,0xff]
-          vandpd -4128(%rdx), %ymm21, %ymm28
-
-// CHECK: vandpd 1016(%rdx){1to4}, %ymm21, %ymm28
-// CHECK:  encoding: [0x62,0x61,0xd5,0x30,0x54,0x62,0x7f]
-          vandpd 1016(%rdx){1to4}, %ymm21, %ymm28
-
-// CHECK: vandpd 1024(%rdx){1to4}, %ymm21, %ymm28
-// CHECK:  encoding: [0x62,0x61,0xd5,0x30,0x54,0xa2,0x00,0x04,0x00,0x00]
-          vandpd 1024(%rdx){1to4}, %ymm21, %ymm28
-
-// CHECK: vandpd -1024(%rdx){1to4}, %ymm21, %ymm28
-// CHECK:  encoding: [0x62,0x61,0xd5,0x30,0x54,0x62,0x80]
-          vandpd -1024(%rdx){1to4}, %ymm21, %ymm28
-
-// CHECK: vandpd -1032(%rdx){1to4}, %ymm21, %ymm28
-// CHECK:  encoding: [0x62,0x61,0xd5,0x30,0x54,0xa2,0xf8,0xfb,0xff,0xff]
-          vandpd -1032(%rdx){1to4}, %ymm21, %ymm28
-
-// CHECK: vandps %xmm24, %xmm21, %xmm23
-// CHECK:  encoding: [0x62,0x81,0x54,0x00,0x54,0xf8]
-          vandps %xmm24, %xmm21, %xmm23
-
-// CHECK: vandps %xmm24, %xmm21, %xmm23 {%k5}
-// CHECK:  encoding: [0x62,0x81,0x54,0x05,0x54,0xf8]
-          vandps %xmm24, %xmm21, %xmm23 {%k5}
-
-// CHECK: vandps %xmm24, %xmm21, %xmm23 {%k5} {z}
-// CHECK:  encoding: [0x62,0x81,0x54,0x85,0x54,0xf8]
-          vandps %xmm24, %xmm21, %xmm23 {%k5} {z}
-
-// CHECK: vandps (%rcx), %xmm21, %xmm23
-// CHECK:  encoding: [0x62,0xe1,0x54,0x00,0x54,0x39]
-          vandps (%rcx), %xmm21, %xmm23
-
-// CHECK: vandps 291(%rax,%r14,8), %xmm21, %xmm23
-// CHECK:  encoding: [0x62,0xa1,0x54,0x00,0x54,0xbc,0xf0,0x23,0x01,0x00,0x00]
-          vandps 291(%rax,%r14,8), %xmm21, %xmm23
-
-// CHECK: vandps (%rcx){1to4}, %xmm21, %xmm23
-// CHECK:  encoding: [0x62,0xe1,0x54,0x10,0x54,0x39]
-          vandps (%rcx){1to4}, %xmm21, %xmm23
-
-// CHECK: vandps 2032(%rdx), %xmm21, %xmm23
-// CHECK:  encoding: [0x62,0xe1,0x54,0x00,0x54,0x7a,0x7f]
-          vandps 2032(%rdx), %xmm21, %xmm23
-
-// CHECK: vandps 2048(%rdx), %xmm21, %xmm23
-// CHECK:  encoding: [0x62,0xe1,0x54,0x00,0x54,0xba,0x00,0x08,0x00,0x00]
-          vandps 2048(%rdx), %xmm21, %xmm23
-
-// CHECK: vandps -2048(%rdx), %xmm21, %xmm23
-// CHECK:  encoding: [0x62,0xe1,0x54,0x00,0x54,0x7a,0x80]
-          vandps -2048(%rdx), %xmm21, %xmm23
-
-// CHECK: vandps -2064(%rdx), %xmm21, %xmm23
-// CHECK:  encoding: [0x62,0xe1,0x54,0x00,0x54,0xba,0xf0,0xf7,0xff,0xff]
-          vandps -2064(%rdx), %xmm21, %xmm23
-
-// CHECK: vandps 508(%rdx){1to4}, %xmm21, %xmm23
-// CHECK:  encoding: [0x62,0xe1,0x54,0x10,0x54,0x7a,0x7f]
-          vandps 508(%rdx){1to4}, %xmm21, %xmm23
-
-// CHECK: vandps 512(%rdx){1to4}, %xmm21, %xmm23
-// CHECK:  encoding: [0x62,0xe1,0x54,0x10,0x54,0xba,0x00,0x02,0x00,0x00]
-          vandps 512(%rdx){1to4}, %xmm21, %xmm23
-
-// CHECK: vandps -512(%rdx){1to4}, %xmm21, %xmm23
-// CHECK:  encoding: [0x62,0xe1,0x54,0x10,0x54,0x7a,0x80]
-          vandps -512(%rdx){1to4}, %xmm21, %xmm23
-
-// CHECK: vandps -516(%rdx){1to4}, %xmm21, %xmm23
-// CHECK:  encoding: [0x62,0xe1,0x54,0x10,0x54,0xba,0xfc,0xfd,0xff,0xff]
-          vandps -516(%rdx){1to4}, %xmm21, %xmm23
-
-// CHECK: vandps %ymm23, %ymm18, %ymm26
-// CHECK:  encoding: [0x62,0x21,0x6c,0x20,0x54,0xd7]
-          vandps %ymm23, %ymm18, %ymm26
-
-// CHECK: vandps %ymm23, %ymm18, %ymm26 {%k6}
-// CHECK:  encoding: [0x62,0x21,0x6c,0x26,0x54,0xd7]
-          vandps %ymm23, %ymm18, %ymm26 {%k6}
-
-// CHECK: vandps %ymm23, %ymm18, %ymm26 {%k6} {z}
-// CHECK:  encoding: [0x62,0x21,0x6c,0xa6,0x54,0xd7]
-          vandps %ymm23, %ymm18, %ymm26 {%k6} {z}
-
-// CHECK: vandps (%rcx), %ymm18, %ymm26
-// CHECK:  encoding: [0x62,0x61,0x6c,0x20,0x54,0x11]
-          vandps (%rcx), %ymm18, %ymm26
-
-// CHECK: vandps 291(%rax,%r14,8), %ymm18, %ymm26
-// CHECK:  encoding: [0x62,0x21,0x6c,0x20,0x54,0x94,0xf0,0x23,0x01,0x00,0x00]
-          vandps 291(%rax,%r14,8), %ymm18, %ymm26
-
-// CHECK: vandps (%rcx){1to8}, %ymm18, %ymm26
-// CHECK:  encoding: [0x62,0x61,0x6c,0x30,0x54,0x11]
-          vandps (%rcx){1to8}, %ymm18, %ymm26
-
-// CHECK: vandps 4064(%rdx), %ymm18, %ymm26
-// CHECK:  encoding: [0x62,0x61,0x6c,0x20,0x54,0x52,0x7f]
-          vandps 4064(%rdx), %ymm18, %ymm26
-
-// CHECK: vandps 4096(%rdx), %ymm18, %ymm26
-// CHECK:  encoding: [0x62,0x61,0x6c,0x20,0x54,0x92,0x00,0x10,0x00,0x00]
-          vandps 4096(%rdx), %ymm18, %ymm26
-
-// CHECK: vandps -4096(%rdx), %ymm18, %ymm26
-// CHECK:  encoding: [0x62,0x61,0x6c,0x20,0x54,0x52,0x80]
-          vandps -4096(%rdx), %ymm18, %ymm26
-
-// CHECK: vandps -4128(%rdx), %ymm18, %ymm26
-// CHECK:  encoding: [0x62,0x61,0x6c,0x20,0x54,0x92,0xe0,0xef,0xff,0xff]
-          vandps -4128(%rdx), %ymm18, %ymm26
-
-// CHECK: vandps 508(%rdx){1to8}, %ymm18, %ymm26
-// CHECK:  encoding: [0x62,0x61,0x6c,0x30,0x54,0x52,0x7f]
-          vandps 508(%rdx){1to8}, %ymm18, %ymm26
-
-// CHECK: vandps 512(%rdx){1to8}, %ymm18, %ymm26
-// CHECK:  encoding: [0x62,0x61,0x6c,0x30,0x54,0x92,0x00,0x02,0x00,0x00]
-          vandps 512(%rdx){1to8}, %ymm18, %ymm26
-
-// CHECK: vandps -512(%rdx){1to8}, %ymm18, %ymm26
-// CHECK:  encoding: [0x62,0x61,0x6c,0x30,0x54,0x52,0x80]
-          vandps -512(%rdx){1to8}, %ymm18, %ymm26
-
-// CHECK: vandps -516(%rdx){1to8}, %ymm18, %ymm26
-// CHECK:  encoding: [0x62,0x61,0x6c,0x30,0x54,0x92,0xfc,0xfd,0xff,0xff]
-          vandps -516(%rdx){1to8}, %ymm18, %ymm26
-
-// CHECK: vandnpd %xmm25, %xmm27, %xmm25
-// CHECK:  encoding: [0x62,0x01,0xa5,0x00,0x55,0xc9]
-          vandnpd %xmm25, %xmm27, %xmm25
-
-// CHECK: vandnpd %xmm25, %xmm27, %xmm25 {%k5}
-// CHECK:  encoding: [0x62,0x01,0xa5,0x05,0x55,0xc9]
-          vandnpd %xmm25, %xmm27, %xmm25 {%k5}
-
-// CHECK: vandnpd %xmm25, %xmm27, %xmm25 {%k5} {z}
-// CHECK:  encoding: [0x62,0x01,0xa5,0x85,0x55,0xc9]
-          vandnpd %xmm25, %xmm27, %xmm25 {%k5} {z}
-
-// CHECK: vandnpd (%rcx), %xmm27, %xmm25
-// CHECK:  encoding: [0x62,0x61,0xa5,0x00,0x55,0x09]
-          vandnpd (%rcx), %xmm27, %xmm25
-
-// CHECK: vandnpd 291(%rax,%r14,8), %xmm27, %xmm25
-// CHECK:  encoding: [0x62,0x21,0xa5,0x00,0x55,0x8c,0xf0,0x23,0x01,0x00,0x00]
-          vandnpd 291(%rax,%r14,8), %xmm27, %xmm25
-
-// CHECK: vandnpd (%rcx){1to2}, %xmm27, %xmm25
-// CHECK:  encoding: [0x62,0x61,0xa5,0x10,0x55,0x09]
-          vandnpd (%rcx){1to2}, %xmm27, %xmm25
-
-// CHECK: vandnpd 2032(%rdx), %xmm27, %xmm25
-// CHECK:  encoding: [0x62,0x61,0xa5,0x00,0x55,0x4a,0x7f]
-          vandnpd 2032(%rdx), %xmm27, %xmm25
-
-// CHECK: vandnpd 2048(%rdx), %xmm27, %xmm25
-// CHECK:  encoding: [0x62,0x61,0xa5,0x00,0x55,0x8a,0x00,0x08,0x00,0x00]
-          vandnpd 2048(%rdx), %xmm27, %xmm25
-
-// CHECK: vandnpd -2048(%rdx), %xmm27, %xmm25
-// CHECK:  encoding: [0x62,0x61,0xa5,0x00,0x55,0x4a,0x80]
-          vandnpd -2048(%rdx), %xmm27, %xmm25
-
-// CHECK: vandnpd -2064(%rdx), %xmm27, %xmm25
-// CHECK:  encoding: [0x62,0x61,0xa5,0x00,0x55,0x8a,0xf0,0xf7,0xff,0xff]
-          vandnpd -2064(%rdx), %xmm27, %xmm25
-
-// CHECK: vandnpd 1016(%rdx){1to2}, %xmm27, %xmm25
-// CHECK:  encoding: [0x62,0x61,0xa5,0x10,0x55,0x4a,0x7f]
-          vandnpd 1016(%rdx){1to2}, %xmm27, %xmm25
-
-// CHECK: vandnpd 1024(%rdx){1to2}, %xmm27, %xmm25
-// CHECK:  encoding: [0x62,0x61,0xa5,0x10,0x55,0x8a,0x00,0x04,0x00,0x00]
-          vandnpd 1024(%rdx){1to2}, %xmm27, %xmm25
-
-// CHECK: vandnpd -1024(%rdx){1to2}, %xmm27, %xmm25
-// CHECK:  encoding: [0x62,0x61,0xa5,0x10,0x55,0x4a,0x80]
-          vandnpd -1024(%rdx){1to2}, %xmm27, %xmm25
-
-// CHECK: vandnpd -1032(%rdx){1to2}, %xmm27, %xmm25
-// CHECK:  encoding: [0x62,0x61,0xa5,0x10,0x55,0x8a,0xf8,0xfb,0xff,0xff]
-          vandnpd -1032(%rdx){1to2}, %xmm27, %xmm25
-
-// CHECK: vandnpd %ymm22, %ymm18, %ymm22
-// CHECK:  encoding: [0x62,0xa1,0xed,0x20,0x55,0xf6]
-          vandnpd %ymm22, %ymm18, %ymm22
-
-// CHECK: vandnpd %ymm22, %ymm18, %ymm22 {%k7}
-// CHECK:  encoding: [0x62,0xa1,0xed,0x27,0x55,0xf6]
-          vandnpd %ymm22, %ymm18, %ymm22 {%k7}
-
-// CHECK: vandnpd %ymm22, %ymm18, %ymm22 {%k7} {z}
-// CHECK:  encoding: [0x62,0xa1,0xed,0xa7,0x55,0xf6]
-          vandnpd %ymm22, %ymm18, %ymm22 {%k7} {z}
-
-// CHECK: vandnpd (%rcx), %ymm18, %ymm22
-// CHECK:  encoding: [0x62,0xe1,0xed,0x20,0x55,0x31]
-          vandnpd (%rcx), %ymm18, %ymm22
-
-// CHECK: vandnpd 291(%rax,%r14,8), %ymm18, %ymm22
-// CHECK:  encoding: [0x62,0xa1,0xed,0x20,0x55,0xb4,0xf0,0x23,0x01,0x00,0x00]
-          vandnpd 291(%rax,%r14,8), %ymm18, %ymm22
-
-// CHECK: vandnpd (%rcx){1to4}, %ymm18, %ymm22
-// CHECK:  encoding: [0x62,0xe1,0xed,0x30,0x55,0x31]
-          vandnpd (%rcx){1to4}, %ymm18, %ymm22
-
-// CHECK: vandnpd 4064(%rdx), %ymm18, %ymm22
-// CHECK:  encoding: [0x62,0xe1,0xed,0x20,0x55,0x72,0x7f]
-          vandnpd 4064(%rdx), %ymm18, %ymm22
-
-// CHECK: vandnpd 4096(%rdx), %ymm18, %ymm22
-// CHECK:  encoding: [0x62,0xe1,0xed,0x20,0x55,0xb2,0x00,0x10,0x00,0x00]
-          vandnpd 4096(%rdx), %ymm18, %ymm22
-
-// CHECK: vandnpd -4096(%rdx), %ymm18, %ymm22
-// CHECK:  encoding: [0x62,0xe1,0xed,0x20,0x55,0x72,0x80]
-          vandnpd -4096(%rdx), %ymm18, %ymm22
-
-// CHECK: vandnpd -4128(%rdx), %ymm18, %ymm22
-// CHECK:  encoding: [0x62,0xe1,0xed,0x20,0x55,0xb2,0xe0,0xef,0xff,0xff]
-          vandnpd -4128(%rdx), %ymm18, %ymm22
-
-// CHECK: vandnpd 1016(%rdx){1to4}, %ymm18, %ymm22
-// CHECK:  encoding: [0x62,0xe1,0xed,0x30,0x55,0x72,0x7f]
-          vandnpd 1016(%rdx){1to4}, %ymm18, %ymm22
-
-// CHECK: vandnpd 1024(%rdx){1to4}, %ymm18, %ymm22
-// CHECK:  encoding: [0x62,0xe1,0xed,0x30,0x55,0xb2,0x00,0x04,0x00,0x00]
-          vandnpd 1024(%rdx){1to4}, %ymm18, %ymm22
-
-// CHECK: vandnpd -1024(%rdx){1to4}, %ymm18, %ymm22
-// CHECK:  encoding: [0x62,0xe1,0xed,0x30,0x55,0x72,0x80]
-          vandnpd -1024(%rdx){1to4}, %ymm18, %ymm22
-
-// CHECK: vandnpd -1032(%rdx){1to4}, %ymm18, %ymm22
-// CHECK:  encoding: [0x62,0xe1,0xed,0x30,0x55,0xb2,0xf8,0xfb,0xff,0xff]
-          vandnpd -1032(%rdx){1to4}, %ymm18, %ymm22
-
-// CHECK: vandnps %xmm27, %xmm21, %xmm21
-// CHECK:  encoding: [0x62,0x81,0x54,0x00,0x55,0xeb]
-          vandnps %xmm27, %xmm21, %xmm21
-
-// CHECK: vandnps %xmm27, %xmm21, %xmm21 {%k2}
-// CHECK:  encoding: [0x62,0x81,0x54,0x02,0x55,0xeb]
-          vandnps %xmm27, %xmm21, %xmm21 {%k2}
-
-// CHECK: vandnps %xmm27, %xmm21, %xmm21 {%k2} {z}
-// CHECK:  encoding: [0x62,0x81,0x54,0x82,0x55,0xeb]
-          vandnps %xmm27, %xmm21, %xmm21 {%k2} {z}
-
-// CHECK: vandnps (%rcx), %xmm21, %xmm21
-// CHECK:  encoding: [0x62,0xe1,0x54,0x00,0x55,0x29]
-          vandnps (%rcx), %xmm21, %xmm21
-
-// CHECK: vandnps 291(%rax,%r14,8), %xmm21, %xmm21
-// CHECK:  encoding: [0x62,0xa1,0x54,0x00,0x55,0xac,0xf0,0x23,0x01,0x00,0x00]
-          vandnps 291(%rax,%r14,8), %xmm21, %xmm21
-
-// CHECK: vandnps (%rcx){1to4}, %xmm21, %xmm21
-// CHECK:  encoding: [0x62,0xe1,0x54,0x10,0x55,0x29]
-          vandnps (%rcx){1to4}, %xmm21, %xmm21
-
-// CHECK: vandnps 2032(%rdx), %xmm21, %xmm21
-// CHECK:  encoding: [0x62,0xe1,0x54,0x00,0x55,0x6a,0x7f]
-          vandnps 2032(%rdx), %xmm21, %xmm21
-
-// CHECK: vandnps 2048(%rdx), %xmm21, %xmm21
-// CHECK:  encoding: [0x62,0xe1,0x54,0x00,0x55,0xaa,0x00,0x08,0x00,0x00]
-          vandnps 2048(%rdx), %xmm21, %xmm21
-
-// CHECK: vandnps -2048(%rdx), %xmm21, %xmm21
-// CHECK:  encoding: [0x62,0xe1,0x54,0x00,0x55,0x6a,0x80]
-          vandnps -2048(%rdx), %xmm21, %xmm21
-
-// CHECK: vandnps -2064(%rdx), %xmm21, %xmm21
-// CHECK:  encoding: [0x62,0xe1,0x54,0x00,0x55,0xaa,0xf0,0xf7,0xff,0xff]
-          vandnps -2064(%rdx), %xmm21, %xmm21
-
-// CHECK: vandnps 508(%rdx){1to4}, %xmm21, %xmm21
-// CHECK:  encoding: [0x62,0xe1,0x54,0x10,0x55,0x6a,0x7f]
-          vandnps 508(%rdx){1to4}, %xmm21, %xmm21
-
-// CHECK: vandnps 512(%rdx){1to4}, %xmm21, %xmm21
-// CHECK:  encoding: [0x62,0xe1,0x54,0x10,0x55,0xaa,0x00,0x02,0x00,0x00]
-          vandnps 512(%rdx){1to4}, %xmm21, %xmm21
-
-// CHECK: vandnps -512(%rdx){1to4}, %xmm21, %xmm21
-// CHECK:  encoding: [0x62,0xe1,0x54,0x10,0x55,0x6a,0x80]
-          vandnps -512(%rdx){1to4}, %xmm21, %xmm21
-
-// CHECK: vandnps -516(%rdx){1to4}, %xmm21, %xmm21
-// CHECK:  encoding: [0x62,0xe1,0x54,0x10,0x55,0xaa,0xfc,0xfd,0xff,0xff]
-          vandnps -516(%rdx){1to4}, %xmm21, %xmm21
-
-// CHECK: vandnps %ymm25, %ymm23, %ymm19
-// CHECK:  encoding: [0x62,0x81,0x44,0x20,0x55,0xd9]
-          vandnps %ymm25, %ymm23, %ymm19
-
-// CHECK: vandnps %ymm25, %ymm23, %ymm19 {%k1}
-// CHECK:  encoding: [0x62,0x81,0x44,0x21,0x55,0xd9]
-          vandnps %ymm25, %ymm23, %ymm19 {%k1}
-
-// CHECK: vandnps %ymm25, %ymm23, %ymm19 {%k1} {z}
-// CHECK:  encoding: [0x62,0x81,0x44,0xa1,0x55,0xd9]
-          vandnps %ymm25, %ymm23, %ymm19 {%k1} {z}
-
-// CHECK: vandnps (%rcx), %ymm23, %ymm19
-// CHECK:  encoding: [0x62,0xe1,0x44,0x20,0x55,0x19]
-          vandnps (%rcx), %ymm23, %ymm19
-
-// CHECK: vandnps 291(%rax,%r14,8), %ymm23, %ymm19
-// CHECK:  encoding: [0x62,0xa1,0x44,0x20,0x55,0x9c,0xf0,0x23,0x01,0x00,0x00]
-          vandnps 291(%rax,%r14,8), %ymm23, %ymm19
-
-// CHECK: vandnps (%rcx){1to8}, %ymm23, %ymm19
-// CHECK:  encoding: [0x62,0xe1,0x44,0x30,0x55,0x19]
-          vandnps (%rcx){1to8}, %ymm23, %ymm19
-
-// CHECK: vandnps 4064(%rdx), %ymm23, %ymm19
-// CHECK:  encoding: [0x62,0xe1,0x44,0x20,0x55,0x5a,0x7f]
-          vandnps 4064(%rdx), %ymm23, %ymm19
-
-// CHECK: vandnps 4096(%rdx), %ymm23, %ymm19
-// CHECK:  encoding: [0x62,0xe1,0x44,0x20,0x55,0x9a,0x00,0x10,0x00,0x00]
-          vandnps 4096(%rdx), %ymm23, %ymm19
-
-// CHECK: vandnps -4096(%rdx), %ymm23, %ymm19
-// CHECK:  encoding: [0x62,0xe1,0x44,0x20,0x55,0x5a,0x80]
-          vandnps -4096(%rdx), %ymm23, %ymm19
-
-// CHECK: vandnps -4128(%rdx), %ymm23, %ymm19
-// CHECK:  encoding: [0x62,0xe1,0x44,0x20,0x55,0x9a,0xe0,0xef,0xff,0xff]
-          vandnps -4128(%rdx), %ymm23, %ymm19
-
-// CHECK: vandnps 508(%rdx){1to8}, %ymm23, %ymm19
-// CHECK:  encoding: [0x62,0xe1,0x44,0x30,0x55,0x5a,0x7f]
-          vandnps 508(%rdx){1to8}, %ymm23, %ymm19
-
-// CHECK: vandnps 512(%rdx){1to8}, %ymm23, %ymm19
-// CHECK:  encoding: [0x62,0xe1,0x44,0x30,0x55,0x9a,0x00,0x02,0x00,0x00]
-          vandnps 512(%rdx){1to8}, %ymm23, %ymm19
-
-// CHECK: vandnps -512(%rdx){1to8}, %ymm23, %ymm19
-// CHECK:  encoding: [0x62,0xe1,0x44,0x30,0x55,0x5a,0x80]
-          vandnps -512(%rdx){1to8}, %ymm23, %ymm19
-
-// CHECK: vandnps -516(%rdx){1to8}, %ymm23, %ymm19
-// CHECK:  encoding: [0x62,0xe1,0x44,0x30,0x55,0x9a,0xfc,0xfd,0xff,0xff]
-          vandnps -516(%rdx){1to8}, %ymm23, %ymm19
-
-// CHECK: vorpd  %xmm18, %xmm27, %xmm23
-// CHECK:  encoding: [0x62,0xa1,0xa5,0x00,0x56,0xfa]
-          vorpd  %xmm18, %xmm27, %xmm23
-
-// CHECK: vorpd  %xmm18, %xmm27, %xmm23 {%k1}
-// CHECK:  encoding: [0x62,0xa1,0xa5,0x01,0x56,0xfa]
-          vorpd  %xmm18, %xmm27, %xmm23 {%k1}
-
-// CHECK: vorpd  %xmm18, %xmm27, %xmm23 {%k1} {z}
-// CHECK:  encoding: [0x62,0xa1,0xa5,0x81,0x56,0xfa]
-          vorpd  %xmm18, %xmm27, %xmm23 {%k1} {z}
-
-// CHECK: vorpd  (%rcx), %xmm27, %xmm23
-// CHECK:  encoding: [0x62,0xe1,0xa5,0x00,0x56,0x39]
-          vorpd  (%rcx), %xmm27, %xmm23
-
-// CHECK: vorpd  291(%rax,%r14,8), %xmm27, %xmm23
-// CHECK:  encoding: [0x62,0xa1,0xa5,0x00,0x56,0xbc,0xf0,0x23,0x01,0x00,0x00]
-          vorpd  291(%rax,%r14,8), %xmm27, %xmm23
-
-// CHECK: vorpd  (%rcx){1to2}, %xmm27, %xmm23
-// CHECK:  encoding: [0x62,0xe1,0xa5,0x10,0x56,0x39]
-          vorpd  (%rcx){1to2}, %xmm27, %xmm23
-
-// CHECK: vorpd  2032(%rdx), %xmm27, %xmm23
-// CHECK:  encoding: [0x62,0xe1,0xa5,0x00,0x56,0x7a,0x7f]
-          vorpd  2032(%rdx), %xmm27, %xmm23
-
-// CHECK: vorpd  2048(%rdx), %xmm27, %xmm23
-// CHECK:  encoding: [0x62,0xe1,0xa5,0x00,0x56,0xba,0x00,0x08,0x00,0x00]
-          vorpd  2048(%rdx), %xmm27, %xmm23
-
-// CHECK: vorpd  -2048(%rdx), %xmm27, %xmm23
-// CHECK:  encoding: [0x62,0xe1,0xa5,0x00,0x56,0x7a,0x80]
-          vorpd  -2048(%rdx), %xmm27, %xmm23
-
-// CHECK: vorpd  -2064(%rdx), %xmm27, %xmm23
-// CHECK:  encoding: [0x62,0xe1,0xa5,0x00,0x56,0xba,0xf0,0xf7,0xff,0xff]
-          vorpd  -2064(%rdx), %xmm27, %xmm23
-
-// CHECK: vorpd  1016(%rdx){1to2}, %xmm27, %xmm23
-// CHECK:  encoding: [0x62,0xe1,0xa5,0x10,0x56,0x7a,0x7f]
-          vorpd  1016(%rdx){1to2}, %xmm27, %xmm23
-
-// CHECK: vorpd  1024(%rdx){1to2}, %xmm27, %xmm23
-// CHECK:  encoding: [0x62,0xe1,0xa5,0x10,0x56,0xba,0x00,0x04,0x00,0x00]
-          vorpd  1024(%rdx){1to2}, %xmm27, %xmm23
-
-// CHECK: vorpd  -1024(%rdx){1to2}, %xmm27, %xmm23
-// CHECK:  encoding: [0x62,0xe1,0xa5,0x10,0x56,0x7a,0x80]
-          vorpd  -1024(%rdx){1to2}, %xmm27, %xmm23
-
-// CHECK: vorpd  -1032(%rdx){1to2}, %xmm27, %xmm23
-// CHECK:  encoding: [0x62,0xe1,0xa5,0x10,0x56,0xba,0xf8,0xfb,0xff,0xff]
-          vorpd  -1032(%rdx){1to2}, %xmm27, %xmm23
-
-// CHECK: vorpd  %ymm20, %ymm29, %ymm26
-// CHECK:  encoding: [0x62,0x21,0x95,0x20,0x56,0xd4]
-          vorpd  %ymm20, %ymm29, %ymm26
-
-// CHECK: vorpd  %ymm20, %ymm29, %ymm26 {%k5}
-// CHECK:  encoding: [0x62,0x21,0x95,0x25,0x56,0xd4]
-          vorpd  %ymm20, %ymm29, %ymm26 {%k5}
-
-// CHECK: vorpd  %ymm20, %ymm29, %ymm26 {%k5} {z}
-// CHECK:  encoding: [0x62,0x21,0x95,0xa5,0x56,0xd4]
-          vorpd  %ymm20, %ymm29, %ymm26 {%k5} {z}
-
-// CHECK: vorpd  (%rcx), %ymm29, %ymm26
-// CHECK:  encoding: [0x62,0x61,0x95,0x20,0x56,0x11]
-          vorpd  (%rcx), %ymm29, %ymm26
-
-// CHECK: vorpd  291(%rax,%r14,8), %ymm29, %ymm26
-// CHECK:  encoding: [0x62,0x21,0x95,0x20,0x56,0x94,0xf0,0x23,0x01,0x00,0x00]
-          vorpd  291(%rax,%r14,8), %ymm29, %ymm26
-
-// CHECK: vorpd  (%rcx){1to4}, %ymm29, %ymm26
-// CHECK:  encoding: [0x62,0x61,0x95,0x30,0x56,0x11]
-          vorpd  (%rcx){1to4}, %ymm29, %ymm26
-
-// CHECK: vorpd  4064(%rdx), %ymm29, %ymm26
-// CHECK:  encoding: [0x62,0x61,0x95,0x20,0x56,0x52,0x7f]
-          vorpd  4064(%rdx), %ymm29, %ymm26
-
-// CHECK: vorpd  4096(%rdx), %ymm29, %ymm26
-// CHECK:  encoding: [0x62,0x61,0x95,0x20,0x56,0x92,0x00,0x10,0x00,0x00]
-          vorpd  4096(%rdx), %ymm29, %ymm26
-
-// CHECK: vorpd  -4096(%rdx), %ymm29, %ymm26
-// CHECK:  encoding: [0x62,0x61,0x95,0x20,0x56,0x52,0x80]
-          vorpd  -4096(%rdx), %ymm29, %ymm26
-
-// CHECK: vorpd  -4128(%rdx), %ymm29, %ymm26
-// CHECK:  encoding: [0x62,0x61,0x95,0x20,0x56,0x92,0xe0,0xef,0xff,0xff]
-          vorpd  -4128(%rdx), %ymm29, %ymm26
-
-// CHECK: vorpd  1016(%rdx){1to4}, %ymm29, %ymm26
-// CHECK:  encoding: [0x62,0x61,0x95,0x30,0x56,0x52,0x7f]
-          vorpd  1016(%rdx){1to4}, %ymm29, %ymm26
-
-// CHECK: vorpd  1024(%rdx){1to4}, %ymm29, %ymm26
-// CHECK:  encoding: [0x62,0x61,0x95,0x30,0x56,0x92,0x00,0x04,0x00,0x00]
-          vorpd  1024(%rdx){1to4}, %ymm29, %ymm26
-
-// CHECK: vorpd  -1024(%rdx){1to4}, %ymm29, %ymm26
-// CHECK:  encoding: [0x62,0x61,0x95,0x30,0x56,0x52,0x80]
-          vorpd  -1024(%rdx){1to4}, %ymm29, %ymm26
-
-// CHECK: vorpd  -1032(%rdx){1to4}, %ymm29, %ymm26
-// CHECK:  encoding: [0x62,0x61,0x95,0x30,0x56,0x92,0xf8,0xfb,0xff,0xff]
-          vorpd  -1032(%rdx){1to4}, %ymm29, %ymm26
-
-// CHECK: vorps  %xmm27, %xmm28, %xmm19
-// CHECK:  encoding: [0x62,0x81,0x1c,0x00,0x56,0xdb]
-          vorps  %xmm27, %xmm28, %xmm19
-
-// CHECK: vorps  %xmm27, %xmm28, %xmm19 {%k4}
-// CHECK:  encoding: [0x62,0x81,0x1c,0x04,0x56,0xdb]
-          vorps  %xmm27, %xmm28, %xmm19 {%k4}
-
-// CHECK: vorps  %xmm27, %xmm28, %xmm19 {%k4} {z}
-// CHECK:  encoding: [0x62,0x81,0x1c,0x84,0x56,0xdb]
-          vorps  %xmm27, %xmm28, %xmm19 {%k4} {z}
-
-// CHECK: vorps  (%rcx), %xmm28, %xmm19
-// CHECK:  encoding: [0x62,0xe1,0x1c,0x00,0x56,0x19]
-          vorps  (%rcx), %xmm28, %xmm19
-
-// CHECK: vorps  291(%rax,%r14,8), %xmm28, %xmm19
-// CHECK:  encoding: [0x62,0xa1,0x1c,0x00,0x56,0x9c,0xf0,0x23,0x01,0x00,0x00]
-          vorps  291(%rax,%r14,8), %xmm28, %xmm19
-
-// CHECK: vorps  (%rcx){1to4}, %xmm28, %xmm19
-// CHECK:  encoding: [0x62,0xe1,0x1c,0x10,0x56,0x19]
-          vorps  (%rcx){1to4}, %xmm28, %xmm19
-
-// CHECK: vorps  2032(%rdx), %xmm28, %xmm19
-// CHECK:  encoding: [0x62,0xe1,0x1c,0x00,0x56,0x5a,0x7f]
-          vorps  2032(%rdx), %xmm28, %xmm19
-
-// CHECK: vorps  2048(%rdx), %xmm28, %xmm19
-// CHECK:  encoding: [0x62,0xe1,0x1c,0x00,0x56,0x9a,0x00,0x08,0x00,0x00]
-          vorps  2048(%rdx), %xmm28, %xmm19
-
-// CHECK: vorps  -2048(%rdx), %xmm28, %xmm19
-// CHECK:  encoding: [0x62,0xe1,0x1c,0x00,0x56,0x5a,0x80]
-          vorps  -2048(%rdx), %xmm28, %xmm19
-
-// CHECK: vorps  -2064(%rdx), %xmm28, %xmm19
-// CHECK:  encoding: [0x62,0xe1,0x1c,0x00,0x56,0x9a,0xf0,0xf7,0xff,0xff]
-          vorps  -2064(%rdx), %xmm28, %xmm19
-
-// CHECK: vorps  508(%rdx){1to4}, %xmm28, %xmm19
-// CHECK:  encoding: [0x62,0xe1,0x1c,0x10,0x56,0x5a,0x7f]
-          vorps  508(%rdx){1to4}, %xmm28, %xmm19
-
-// CHECK: vorps  512(%rdx){1to4}, %xmm28, %xmm19
-// CHECK:  encoding: [0x62,0xe1,0x1c,0x10,0x56,0x9a,0x00,0x02,0x00,0x00]
-          vorps  512(%rdx){1to4}, %xmm28, %xmm19
-
-// CHECK: vorps  -512(%rdx){1to4}, %xmm28, %xmm19
-// CHECK:  encoding: [0x62,0xe1,0x1c,0x10,0x56,0x5a,0x80]
-          vorps  -512(%rdx){1to4}, %xmm28, %xmm19
-
-// CHECK: vorps  -516(%rdx){1to4}, %xmm28, %xmm19
-// CHECK:  encoding: [0x62,0xe1,0x1c,0x10,0x56,0x9a,0xfc,0xfd,0xff,0xff]
-          vorps  -516(%rdx){1to4}, %xmm28, %xmm19
-
-// CHECK: vorps  %ymm26, %ymm26, %ymm27
-// CHECK:  encoding: [0x62,0x01,0x2c,0x20,0x56,0xda]
-          vorps  %ymm26, %ymm26, %ymm27
-
-// CHECK: vorps  %ymm26, %ymm26, %ymm27 {%k1}
-// CHECK:  encoding: [0x62,0x01,0x2c,0x21,0x56,0xda]
-          vorps  %ymm26, %ymm26, %ymm27 {%k1}
-
-// CHECK: vorps  %ymm26, %ymm26, %ymm27 {%k1} {z}
-// CHECK:  encoding: [0x62,0x01,0x2c,0xa1,0x56,0xda]
-          vorps  %ymm26, %ymm26, %ymm27 {%k1} {z}
-
-// CHECK: vorps  (%rcx), %ymm26, %ymm27
-// CHECK:  encoding: [0x62,0x61,0x2c,0x20,0x56,0x19]
-          vorps  (%rcx), %ymm26, %ymm27
-
-// CHECK: vorps  291(%rax,%r14,8), %ymm26, %ymm27
-// CHECK:  encoding: [0x62,0x21,0x2c,0x20,0x56,0x9c,0xf0,0x23,0x01,0x00,0x00]
-          vorps  291(%rax,%r14,8), %ymm26, %ymm27
-
-// CHECK: vorps  (%rcx){1to8}, %ymm26, %ymm27
-// CHECK:  encoding: [0x62,0x61,0x2c,0x30,0x56,0x19]
-          vorps  (%rcx){1to8}, %ymm26, %ymm27
-
-// CHECK: vorps  4064(%rdx), %ymm26, %ymm27
-// CHECK:  encoding: [0x62,0x61,0x2c,0x20,0x56,0x5a,0x7f]
-          vorps  4064(%rdx), %ymm26, %ymm27
-
-// CHECK: vorps  4096(%rdx), %ymm26, %ymm27
-// CHECK:  encoding: [0x62,0x61,0x2c,0x20,0x56,0x9a,0x00,0x10,0x00,0x00]
-          vorps  4096(%rdx), %ymm26, %ymm27
-
-// CHECK: vorps  -4096(%rdx), %ymm26, %ymm27
-// CHECK:  encoding: [0x62,0x61,0x2c,0x20,0x56,0x5a,0x80]
-          vorps  -4096(%rdx), %ymm26, %ymm27
-
-// CHECK: vorps  -4128(%rdx), %ymm26, %ymm27
-// CHECK:  encoding: [0x62,0x61,0x2c,0x20,0x56,0x9a,0xe0,0xef,0xff,0xff]
-          vorps  -4128(%rdx), %ymm26, %ymm27
-
-// CHECK: vorps  508(%rdx){1to8}, %ymm26, %ymm27
-// CHECK:  encoding: [0x62,0x61,0x2c,0x30,0x56,0x5a,0x7f]
-          vorps  508(%rdx){1to8}, %ymm26, %ymm27
-
-// CHECK: vorps  512(%rdx){1to8}, %ymm26, %ymm27
-// CHECK:  encoding: [0x62,0x61,0x2c,0x30,0x56,0x9a,0x00,0x02,0x00,0x00]
-          vorps  512(%rdx){1to8}, %ymm26, %ymm27
-
-// CHECK: vorps  -512(%rdx){1to8}, %ymm26, %ymm27
-// CHECK:  encoding: [0x62,0x61,0x2c,0x30,0x56,0x5a,0x80]
-          vorps  -512(%rdx){1to8}, %ymm26, %ymm27
-
-// CHECK: vorps  -516(%rdx){1to8}, %ymm26, %ymm27
-// CHECK:  encoding: [0x62,0x61,0x2c,0x30,0x56,0x9a,0xfc,0xfd,0xff,0xff]
-          vorps  -516(%rdx){1to8}, %ymm26, %ymm27
-
-// CHECK: vxorpd %xmm23, %xmm21, %xmm18
-// CHECK:  encoding: [0x62,0xa1,0xd5,0x00,0x57,0xd7]
-          vxorpd %xmm23, %xmm21, %xmm18
-
-// CHECK: vxorpd %xmm23, %xmm21, %xmm18 {%k2}
-// CHECK:  encoding: [0x62,0xa1,0xd5,0x02,0x57,0xd7]
-          vxorpd %xmm23, %xmm21, %xmm18 {%k2}
-
-// CHECK: vxorpd %xmm23, %xmm21, %xmm18 {%k2} {z}
-// CHECK:  encoding: [0x62,0xa1,0xd5,0x82,0x57,0xd7]
-          vxorpd %xmm23, %xmm21, %xmm18 {%k2} {z}
-
-// CHECK: vxorpd (%rcx), %xmm21, %xmm18
-// CHECK:  encoding: [0x62,0xe1,0xd5,0x00,0x57,0x11]
-          vxorpd (%rcx), %xmm21, %xmm18
-
-// CHECK: vxorpd 291(%rax,%r14,8), %xmm21, %xmm18
-// CHECK:  encoding: [0x62,0xa1,0xd5,0x00,0x57,0x94,0xf0,0x23,0x01,0x00,0x00]
-          vxorpd 291(%rax,%r14,8), %xmm21, %xmm18
-
-// CHECK: vxorpd (%rcx){1to2}, %xmm21, %xmm18
-// CHECK:  encoding: [0x62,0xe1,0xd5,0x10,0x57,0x11]
-          vxorpd (%rcx){1to2}, %xmm21, %xmm18
-
-// CHECK: vxorpd 2032(%rdx), %xmm21, %xmm18
-// CHECK:  encoding: [0x62,0xe1,0xd5,0x00,0x57,0x52,0x7f]
-          vxorpd 2032(%rdx), %xmm21, %xmm18
-
-// CHECK: vxorpd 2048(%rdx), %xmm21, %xmm18
-// CHECK:  encoding: [0x62,0xe1,0xd5,0x00,0x57,0x92,0x00,0x08,0x00,0x00]
-          vxorpd 2048(%rdx), %xmm21, %xmm18
-
-// CHECK: vxorpd -2048(%rdx), %xmm21, %xmm18
-// CHECK:  encoding: [0x62,0xe1,0xd5,0x00,0x57,0x52,0x80]
-          vxorpd -2048(%rdx), %xmm21, %xmm18
-
-// CHECK: vxorpd -2064(%rdx), %xmm21, %xmm18
-// CHECK:  encoding: [0x62,0xe1,0xd5,0x00,0x57,0x92,0xf0,0xf7,0xff,0xff]
-          vxorpd -2064(%rdx), %xmm21, %xmm18
-
-// CHECK: vxorpd 1016(%rdx){1to2}, %xmm21, %xmm18
-// CHECK:  encoding: [0x62,0xe1,0xd5,0x10,0x57,0x52,0x7f]
-          vxorpd 1016(%rdx){1to2}, %xmm21, %xmm18
-
-// CHECK: vxorpd 1024(%rdx){1to2}, %xmm21, %xmm18
-// CHECK:  encoding: [0x62,0xe1,0xd5,0x10,0x57,0x92,0x00,0x04,0x00,0x00]
-          vxorpd 1024(%rdx){1to2}, %xmm21, %xmm18
-
-// CHECK: vxorpd -1024(%rdx){1to2}, %xmm21, %xmm18
-// CHECK:  encoding: [0x62,0xe1,0xd5,0x10,0x57,0x52,0x80]
-          vxorpd -1024(%rdx){1to2}, %xmm21, %xmm18
-
-// CHECK: vxorpd -1032(%rdx){1to2}, %xmm21, %xmm18
-// CHECK:  encoding: [0x62,0xe1,0xd5,0x10,0x57,0x92,0xf8,0xfb,0xff,0xff]
-          vxorpd -1032(%rdx){1to2}, %xmm21, %xmm18
-
-// CHECK: vxorpd %ymm19, %ymm29, %ymm24
-// CHECK:  encoding: [0x62,0x21,0x95,0x20,0x57,0xc3]
-          vxorpd %ymm19, %ymm29, %ymm24
-
-// CHECK: vxorpd %ymm19, %ymm29, %ymm24 {%k7}
-// CHECK:  encoding: [0x62,0x21,0x95,0x27,0x57,0xc3]
-          vxorpd %ymm19, %ymm29, %ymm24 {%k7}
-
-// CHECK: vxorpd %ymm19, %ymm29, %ymm24 {%k7} {z}
-// CHECK:  encoding: [0x62,0x21,0x95,0xa7,0x57,0xc3]
-          vxorpd %ymm19, %ymm29, %ymm24 {%k7} {z}
-
-// CHECK: vxorpd (%rcx), %ymm29, %ymm24
-// CHECK:  encoding: [0x62,0x61,0x95,0x20,0x57,0x01]
-          vxorpd (%rcx), %ymm29, %ymm24
-
-// CHECK: vxorpd 291(%rax,%r14,8), %ymm29, %ymm24
-// CHECK:  encoding: [0x62,0x21,0x95,0x20,0x57,0x84,0xf0,0x23,0x01,0x00,0x00]
-          vxorpd 291(%rax,%r14,8), %ymm29, %ymm24
-
-// CHECK: vxorpd (%rcx){1to4}, %ymm29, %ymm24
-// CHECK:  encoding: [0x62,0x61,0x95,0x30,0x57,0x01]
-          vxorpd (%rcx){1to4}, %ymm29, %ymm24
-
-// CHECK: vxorpd 4064(%rdx), %ymm29, %ymm24
-// CHECK:  encoding: [0x62,0x61,0x95,0x20,0x57,0x42,0x7f]
-          vxorpd 4064(%rdx), %ymm29, %ymm24
-
-// CHECK: vxorpd 4096(%rdx), %ymm29, %ymm24
-// CHECK:  encoding: [0x62,0x61,0x95,0x20,0x57,0x82,0x00,0x10,0x00,0x00]
-          vxorpd 4096(%rdx), %ymm29, %ymm24
-
-// CHECK: vxorpd -4096(%rdx), %ymm29, %ymm24
-// CHECK:  encoding: [0x62,0x61,0x95,0x20,0x57,0x42,0x80]
-          vxorpd -4096(%rdx), %ymm29, %ymm24
-
-// CHECK: vxorpd -4128(%rdx), %ymm29, %ymm24
-// CHECK:  encoding: [0x62,0x61,0x95,0x20,0x57,0x82,0xe0,0xef,0xff,0xff]
-          vxorpd -4128(%rdx), %ymm29, %ymm24
-
-// CHECK: vxorpd 1016(%rdx){1to4}, %ymm29, %ymm24
-// CHECK:  encoding: [0x62,0x61,0x95,0x30,0x57,0x42,0x7f]
-          vxorpd 1016(%rdx){1to4}, %ymm29, %ymm24
-
-// CHECK: vxorpd 1024(%rdx){1to4}, %ymm29, %ymm24
-// CHECK:  encoding: [0x62,0x61,0x95,0x30,0x57,0x82,0x00,0x04,0x00,0x00]
-          vxorpd 1024(%rdx){1to4}, %ymm29, %ymm24
-
-// CHECK: vxorpd -1024(%rdx){1to4}, %ymm29, %ymm24
-// CHECK:  encoding: [0x62,0x61,0x95,0x30,0x57,0x42,0x80]
-          vxorpd -1024(%rdx){1to4}, %ymm29, %ymm24
-
-// CHECK: vxorpd -1032(%rdx){1to4}, %ymm29, %ymm24
-// CHECK:  encoding: [0x62,0x61,0x95,0x30,0x57,0x82,0xf8,0xfb,0xff,0xff]
-          vxorpd -1032(%rdx){1to4}, %ymm29, %ymm24
-
-// CHECK: vxorps %xmm19, %xmm18, %xmm20
-// CHECK:  encoding: [0x62,0xa1,0x6c,0x00,0x57,0xe3]
-          vxorps %xmm19, %xmm18, %xmm20
-
-// CHECK: vxorps %xmm19, %xmm18, %xmm20 {%k1}
-// CHECK:  encoding: [0x62,0xa1,0x6c,0x01,0x57,0xe3]
-          vxorps %xmm19, %xmm18, %xmm20 {%k1}
-
-// CHECK: vxorps %xmm19, %xmm18, %xmm20 {%k1} {z}
-// CHECK:  encoding: [0x62,0xa1,0x6c,0x81,0x57,0xe3]
-          vxorps %xmm19, %xmm18, %xmm20 {%k1} {z}
-
-// CHECK: vxorps (%rcx), %xmm18, %xmm20
-// CHECK:  encoding: [0x62,0xe1,0x6c,0x00,0x57,0x21]
-          vxorps (%rcx), %xmm18, %xmm20
-
-// CHECK: vxorps 291(%rax,%r14,8), %xmm18, %xmm20
-// CHECK:  encoding: [0x62,0xa1,0x6c,0x00,0x57,0xa4,0xf0,0x23,0x01,0x00,0x00]
-          vxorps 291(%rax,%r14,8), %xmm18, %xmm20
-
-// CHECK: vxorps (%rcx){1to4}, %xmm18, %xmm20
-// CHECK:  encoding: [0x62,0xe1,0x6c,0x10,0x57,0x21]
-          vxorps (%rcx){1to4}, %xmm18, %xmm20
-
-// CHECK: vxorps 2032(%rdx), %xmm18, %xmm20
-// CHECK:  encoding: [0x62,0xe1,0x6c,0x00,0x57,0x62,0x7f]
-          vxorps 2032(%rdx), %xmm18, %xmm20
-
-// CHECK: vxorps 2048(%rdx), %xmm18, %xmm20
-// CHECK:  encoding: [0x62,0xe1,0x6c,0x00,0x57,0xa2,0x00,0x08,0x00,0x00]
-          vxorps 2048(%rdx), %xmm18, %xmm20
-
-// CHECK: vxorps -2048(%rdx), %xmm18, %xmm20
-// CHECK:  encoding: [0x62,0xe1,0x6c,0x00,0x57,0x62,0x80]
-          vxorps -2048(%rdx), %xmm18, %xmm20
-
-// CHECK: vxorps -2064(%rdx), %xmm18, %xmm20
-// CHECK:  encoding: [0x62,0xe1,0x6c,0x00,0x57,0xa2,0xf0,0xf7,0xff,0xff]
-          vxorps -2064(%rdx), %xmm18, %xmm20
-
-// CHECK: vxorps 508(%rdx){1to4}, %xmm18, %xmm20
-// CHECK:  encoding: [0x62,0xe1,0x6c,0x10,0x57,0x62,0x7f]
-          vxorps 508(%rdx){1to4}, %xmm18, %xmm20
-
-// CHECK: vxorps 512(%rdx){1to4}, %xmm18, %xmm20
-// CHECK:  encoding: [0x62,0xe1,0x6c,0x10,0x57,0xa2,0x00,0x02,0x00,0x00]
-          vxorps 512(%rdx){1to4}, %xmm18, %xmm20
-
-// CHECK: vxorps -512(%rdx){1to4}, %xmm18, %xmm20
-// CHECK:  encoding: [0x62,0xe1,0x6c,0x10,0x57,0x62,0x80]
-          vxorps -512(%rdx){1to4}, %xmm18, %xmm20
-
-// CHECK: vxorps -516(%rdx){1to4}, %xmm18, %xmm20
-// CHECK:  encoding: [0x62,0xe1,0x6c,0x10,0x57,0xa2,0xfc,0xfd,0xff,0xff]
-          vxorps -516(%rdx){1to4}, %xmm18, %xmm20
-
-// CHECK: vxorps %ymm24, %ymm20, %ymm27
-// CHECK:  encoding: [0x62,0x01,0x5c,0x20,0x57,0xd8]
-          vxorps %ymm24, %ymm20, %ymm27
-
-// CHECK: vxorps %ymm24, %ymm20, %ymm27 {%k2}
-// CHECK:  encoding: [0x62,0x01,0x5c,0x22,0x57,0xd8]
-          vxorps %ymm24, %ymm20, %ymm27 {%k2}
-
-// CHECK: vxorps %ymm24, %ymm20, %ymm27 {%k2} {z}
-// CHECK:  encoding: [0x62,0x01,0x5c,0xa2,0x57,0xd8]
-          vxorps %ymm24, %ymm20, %ymm27 {%k2} {z}
-
-// CHECK: vxorps (%rcx), %ymm20, %ymm27
-// CHECK:  encoding: [0x62,0x61,0x5c,0x20,0x57,0x19]
-          vxorps (%rcx), %ymm20, %ymm27
-
-// CHECK: vxorps 291(%rax,%r14,8), %ymm20, %ymm27
-// CHECK:  encoding: [0x62,0x21,0x5c,0x20,0x57,0x9c,0xf0,0x23,0x01,0x00,0x00]
-          vxorps 291(%rax,%r14,8), %ymm20, %ymm27
-
-// CHECK: vxorps (%rcx){1to8}, %ymm20, %ymm27
-// CHECK:  encoding: [0x62,0x61,0x5c,0x30,0x57,0x19]
-          vxorps (%rcx){1to8}, %ymm20, %ymm27
-
-// CHECK: vxorps 4064(%rdx), %ymm20, %ymm27
-// CHECK:  encoding: [0x62,0x61,0x5c,0x20,0x57,0x5a,0x7f]
-          vxorps 4064(%rdx), %ymm20, %ymm27
-
-// CHECK: vxorps 4096(%rdx), %ymm20, %ymm27
-// CHECK:  encoding: [0x62,0x61,0x5c,0x20,0x57,0x9a,0x00,0x10,0x00,0x00]
-          vxorps 4096(%rdx), %ymm20, %ymm27
-
-// CHECK: vxorps -4096(%rdx), %ymm20, %ymm27
-// CHECK:  encoding: [0x62,0x61,0x5c,0x20,0x57,0x5a,0x80]
-          vxorps -4096(%rdx), %ymm20, %ymm27
-
-// CHECK: vxorps -4128(%rdx), %ymm20, %ymm27
-// CHECK:  encoding: [0x62,0x61,0x5c,0x20,0x57,0x9a,0xe0,0xef,0xff,0xff]
-          vxorps -4128(%rdx), %ymm20, %ymm27
-
-// CHECK: vxorps 508(%rdx){1to8}, %ymm20, %ymm27
-// CHECK:  encoding: [0x62,0x61,0x5c,0x30,0x57,0x5a,0x7f]
-          vxorps 508(%rdx){1to8}, %ymm20, %ymm27
-
-// CHECK: vxorps 512(%rdx){1to8}, %ymm20, %ymm27
-// CHECK:  encoding: [0x62,0x61,0x5c,0x30,0x57,0x9a,0x00,0x02,0x00,0x00]
-          vxorps 512(%rdx){1to8}, %ymm20, %ymm27
-
-// CHECK: vxorps -512(%rdx){1to8}, %ymm20, %ymm27
-// CHECK:  encoding: [0x62,0x61,0x5c,0x30,0x57,0x5a,0x80]
-          vxorps -512(%rdx){1to8}, %ymm20, %ymm27
-
-// CHECK: vxorps -516(%rdx){1to8}, %ymm20, %ymm27
-// CHECK:  encoding: [0x62,0x61,0x5c,0x30,0x57,0x9a,0xfc,0xfd,0xff,0xff]
-          vxorps -516(%rdx){1to8}, %ymm20, %ymm27
-
-// CHECK: vandpd %xmm27, %xmm25, %xmm19
-// CHECK:  encoding: [0x62,0x81,0xb5,0x00,0x54,0xdb]
-          vandpd %xmm27, %xmm25, %xmm19
-
-// CHECK: vandpd %xmm27, %xmm25, %xmm19 {%k6}
-// CHECK:  encoding: [0x62,0x81,0xb5,0x06,0x54,0xdb]
-          vandpd %xmm27, %xmm25, %xmm19 {%k6}
-
-// CHECK: vandpd %xmm27, %xmm25, %xmm19 {%k6} {z}
-// CHECK:  encoding: [0x62,0x81,0xb5,0x86,0x54,0xdb]
-          vandpd %xmm27, %xmm25, %xmm19 {%k6} {z}
-
-// CHECK: vandpd (%rcx), %xmm25, %xmm19
-// CHECK:  encoding: [0x62,0xe1,0xb5,0x00,0x54,0x19]
-          vandpd (%rcx), %xmm25, %xmm19
-
-// CHECK: vandpd 4660(%rax,%r14,8), %xmm25, %xmm19
-// CHECK:  encoding: [0x62,0xa1,0xb5,0x00,0x54,0x9c,0xf0,0x34,0x12,0x00,0x00]
-          vandpd 4660(%rax,%r14,8), %xmm25, %xmm19
-
-// CHECK: vandpd (%rcx){1to2}, %xmm25, %xmm19
-// CHECK:  encoding: [0x62,0xe1,0xb5,0x10,0x54,0x19]
-          vandpd (%rcx){1to2}, %xmm25, %xmm19
-
-// CHECK: vandpd 2032(%rdx), %xmm25, %xmm19
-// CHECK:  encoding: [0x62,0xe1,0xb5,0x00,0x54,0x5a,0x7f]
-          vandpd 2032(%rdx), %xmm25, %xmm19
-
-// CHECK: vandpd 2048(%rdx), %xmm25, %xmm19
-// CHECK:  encoding: [0x62,0xe1,0xb5,0x00,0x54,0x9a,0x00,0x08,0x00,0x00]
-          vandpd 2048(%rdx), %xmm25, %xmm19
-
-// CHECK: vandpd -2048(%rdx), %xmm25, %xmm19
-// CHECK:  encoding: [0x62,0xe1,0xb5,0x00,0x54,0x5a,0x80]
-          vandpd -2048(%rdx), %xmm25, %xmm19
-
-// CHECK: vandpd -2064(%rdx), %xmm25, %xmm19
-// CHECK:  encoding: [0x62,0xe1,0xb5,0x00,0x54,0x9a,0xf0,0xf7,0xff,0xff]
-          vandpd -2064(%rdx), %xmm25, %xmm19
-
-// CHECK: vandpd 1016(%rdx){1to2}, %xmm25, %xmm19
-// CHECK:  encoding: [0x62,0xe1,0xb5,0x10,0x54,0x5a,0x7f]
-          vandpd 1016(%rdx){1to2}, %xmm25, %xmm19
-
-// CHECK: vandpd 1024(%rdx){1to2}, %xmm25, %xmm19
-// CHECK:  encoding: [0x62,0xe1,0xb5,0x10,0x54,0x9a,0x00,0x04,0x00,0x00]
-          vandpd 1024(%rdx){1to2}, %xmm25, %xmm19
-
-// CHECK: vandpd -1024(%rdx){1to2}, %xmm25, %xmm19
-// CHECK:  encoding: [0x62,0xe1,0xb5,0x10,0x54,0x5a,0x80]
-          vandpd -1024(%rdx){1to2}, %xmm25, %xmm19
-
-// CHECK: vandpd -1032(%rdx){1to2}, %xmm25, %xmm19
-// CHECK:  encoding: [0x62,0xe1,0xb5,0x10,0x54,0x9a,0xf8,0xfb,0xff,0xff]
-          vandpd -1032(%rdx){1to2}, %xmm25, %xmm19
-
-// CHECK: vandpd %ymm21, %ymm29, %ymm24
-// CHECK:  encoding: [0x62,0x21,0x95,0x20,0x54,0xc5]
-          vandpd %ymm21, %ymm29, %ymm24
-
-// CHECK: vandpd %ymm21, %ymm29, %ymm24 {%k2}
-// CHECK:  encoding: [0x62,0x21,0x95,0x22,0x54,0xc5]
-          vandpd %ymm21, %ymm29, %ymm24 {%k2}
-
-// CHECK: vandpd %ymm21, %ymm29, %ymm24 {%k2} {z}
-// CHECK:  encoding: [0x62,0x21,0x95,0xa2,0x54,0xc5]
-          vandpd %ymm21, %ymm29, %ymm24 {%k2} {z}
-
-// CHECK: vandpd (%rcx), %ymm29, %ymm24
-// CHECK:  encoding: [0x62,0x61,0x95,0x20,0x54,0x01]
-          vandpd (%rcx), %ymm29, %ymm24
-
-// CHECK: vandpd 4660(%rax,%r14,8), %ymm29, %ymm24
-// CHECK:  encoding: [0x62,0x21,0x95,0x20,0x54,0x84,0xf0,0x34,0x12,0x00,0x00]
-          vandpd 4660(%rax,%r14,8), %ymm29, %ymm24
-
-// CHECK: vandpd (%rcx){1to4}, %ymm29, %ymm24
-// CHECK:  encoding: [0x62,0x61,0x95,0x30,0x54,0x01]
-          vandpd (%rcx){1to4}, %ymm29, %ymm24
-
-// CHECK: vandpd 4064(%rdx), %ymm29, %ymm24
-// CHECK:  encoding: [0x62,0x61,0x95,0x20,0x54,0x42,0x7f]
-          vandpd 4064(%rdx), %ymm29, %ymm24
-
-// CHECK: vandpd 4096(%rdx), %ymm29, %ymm24
-// CHECK:  encoding: [0x62,0x61,0x95,0x20,0x54,0x82,0x00,0x10,0x00,0x00]
-          vandpd 4096(%rdx), %ymm29, %ymm24
-
-// CHECK: vandpd -4096(%rdx), %ymm29, %ymm24
-// CHECK:  encoding: [0x62,0x61,0x95,0x20,0x54,0x42,0x80]
-          vandpd -4096(%rdx), %ymm29, %ymm24
-
-// CHECK: vandpd -4128(%rdx), %ymm29, %ymm24
-// CHECK:  encoding: [0x62,0x61,0x95,0x20,0x54,0x82,0xe0,0xef,0xff,0xff]
-          vandpd -4128(%rdx), %ymm29, %ymm24
-
-// CHECK: vandpd 1016(%rdx){1to4}, %ymm29, %ymm24
-// CHECK:  encoding: [0x62,0x61,0x95,0x30,0x54,0x42,0x7f]
-          vandpd 1016(%rdx){1to4}, %ymm29, %ymm24
-
-// CHECK: vandpd 1024(%rdx){1to4}, %ymm29, %ymm24
-// CHECK:  encoding: [0x62,0x61,0x95,0x30,0x54,0x82,0x00,0x04,0x00,0x00]
-          vandpd 1024(%rdx){1to4}, %ymm29, %ymm24
-
-// CHECK: vandpd -1024(%rdx){1to4}, %ymm29, %ymm24
-// CHECK:  encoding: [0x62,0x61,0x95,0x30,0x54,0x42,0x80]
-          vandpd -1024(%rdx){1to4}, %ymm29, %ymm24
-
-// CHECK: vandpd -1032(%rdx){1to4}, %ymm29, %ymm24
-// CHECK:  encoding: [0x62,0x61,0x95,0x30,0x54,0x82,0xf8,0xfb,0xff,0xff]
-          vandpd -1032(%rdx){1to4}, %ymm29, %ymm24
-
-// CHECK: vandps %xmm17, %xmm25, %xmm22
-// CHECK:  encoding: [0x62,0xa1,0x34,0x00,0x54,0xf1]
-          vandps %xmm17, %xmm25, %xmm22
-
-// CHECK: vandps %xmm17, %xmm25, %xmm22 {%k3}
-// CHECK:  encoding: [0x62,0xa1,0x34,0x03,0x54,0xf1]
-          vandps %xmm17, %xmm25, %xmm22 {%k3}
-
-// CHECK: vandps %xmm17, %xmm25, %xmm22 {%k3} {z}
-// CHECK:  encoding: [0x62,0xa1,0x34,0x83,0x54,0xf1]
-          vandps %xmm17, %xmm25, %xmm22 {%k3} {z}
-
-// CHECK: vandps (%rcx), %xmm25, %xmm22
-// CHECK:  encoding: [0x62,0xe1,0x34,0x00,0x54,0x31]
-          vandps (%rcx), %xmm25, %xmm22
-
-// CHECK: vandps 4660(%rax,%r14,8), %xmm25, %xmm22
-// CHECK:  encoding: [0x62,0xa1,0x34,0x00,0x54,0xb4,0xf0,0x34,0x12,0x00,0x00]
-          vandps 4660(%rax,%r14,8), %xmm25, %xmm22
-
-// CHECK: vandps (%rcx){1to4}, %xmm25, %xmm22
-// CHECK:  encoding: [0x62,0xe1,0x34,0x10,0x54,0x31]
-          vandps (%rcx){1to4}, %xmm25, %xmm22
-
-// CHECK: vandps 2032(%rdx), %xmm25, %xmm22
-// CHECK:  encoding: [0x62,0xe1,0x34,0x00,0x54,0x72,0x7f]
-          vandps 2032(%rdx), %xmm25, %xmm22
-
-// CHECK: vandps 2048(%rdx), %xmm25, %xmm22
-// CHECK:  encoding: [0x62,0xe1,0x34,0x00,0x54,0xb2,0x00,0x08,0x00,0x00]
-          vandps 2048(%rdx), %xmm25, %xmm22
-
-// CHECK: vandps -2048(%rdx), %xmm25, %xmm22
-// CHECK:  encoding: [0x62,0xe1,0x34,0x00,0x54,0x72,0x80]
-          vandps -2048(%rdx), %xmm25, %xmm22
-
-// CHECK: vandps -2064(%rdx), %xmm25, %xmm22
-// CHECK:  encoding: [0x62,0xe1,0x34,0x00,0x54,0xb2,0xf0,0xf7,0xff,0xff]
-          vandps -2064(%rdx), %xmm25, %xmm22
-
-// CHECK: vandps 508(%rdx){1to4}, %xmm25, %xmm22
-// CHECK:  encoding: [0x62,0xe1,0x34,0x10,0x54,0x72,0x7f]
-          vandps 508(%rdx){1to4}, %xmm25, %xmm22
-
-// CHECK: vandps 512(%rdx){1to4}, %xmm25, %xmm22
-// CHECK:  encoding: [0x62,0xe1,0x34,0x10,0x54,0xb2,0x00,0x02,0x00,0x00]
-          vandps 512(%rdx){1to4}, %xmm25, %xmm22
-
-// CHECK: vandps -512(%rdx){1to4}, %xmm25, %xmm22
-// CHECK:  encoding: [0x62,0xe1,0x34,0x10,0x54,0x72,0x80]
-          vandps -512(%rdx){1to4}, %xmm25, %xmm22
-
-// CHECK: vandps -516(%rdx){1to4}, %xmm25, %xmm22
-// CHECK:  encoding: [0x62,0xe1,0x34,0x10,0x54,0xb2,0xfc,0xfd,0xff,0xff]
-          vandps -516(%rdx){1to4}, %xmm25, %xmm22
-
-// CHECK: vandps %ymm18, %ymm25, %ymm22
-// CHECK:  encoding: [0x62,0xa1,0x34,0x20,0x54,0xf2]
-          vandps %ymm18, %ymm25, %ymm22
-
-// CHECK: vandps %ymm18, %ymm25, %ymm22 {%k1}
-// CHECK:  encoding: [0x62,0xa1,0x34,0x21,0x54,0xf2]
-          vandps %ymm18, %ymm25, %ymm22 {%k1}
-
-// CHECK: vandps %ymm18, %ymm25, %ymm22 {%k1} {z}
-// CHECK:  encoding: [0x62,0xa1,0x34,0xa1,0x54,0xf2]
-          vandps %ymm18, %ymm25, %ymm22 {%k1} {z}
-
-// CHECK: vandps (%rcx), %ymm25, %ymm22
-// CHECK:  encoding: [0x62,0xe1,0x34,0x20,0x54,0x31]
-          vandps (%rcx), %ymm25, %ymm22
-
-// CHECK: vandps 4660(%rax,%r14,8), %ymm25, %ymm22
-// CHECK:  encoding: [0x62,0xa1,0x34,0x20,0x54,0xb4,0xf0,0x34,0x12,0x00,0x00]
-          vandps 4660(%rax,%r14,8), %ymm25, %ymm22
-
-// CHECK: vandps (%rcx){1to8}, %ymm25, %ymm22
-// CHECK:  encoding: [0x62,0xe1,0x34,0x30,0x54,0x31]
-          vandps (%rcx){1to8}, %ymm25, %ymm22
-
-// CHECK: vandps 4064(%rdx), %ymm25, %ymm22
-// CHECK:  encoding: [0x62,0xe1,0x34,0x20,0x54,0x72,0x7f]
-          vandps 4064(%rdx), %ymm25, %ymm22
-
-// CHECK: vandps 4096(%rdx), %ymm25, %ymm22
-// CHECK:  encoding: [0x62,0xe1,0x34,0x20,0x54,0xb2,0x00,0x10,0x00,0x00]
-          vandps 4096(%rdx), %ymm25, %ymm22
-
-// CHECK: vandps -4096(%rdx), %ymm25, %ymm22
-// CHECK:  encoding: [0x62,0xe1,0x34,0x20,0x54,0x72,0x80]
-          vandps -4096(%rdx), %ymm25, %ymm22
-
-// CHECK: vandps -4128(%rdx), %ymm25, %ymm22
-// CHECK:  encoding: [0x62,0xe1,0x34,0x20,0x54,0xb2,0xe0,0xef,0xff,0xff]
-          vandps -4128(%rdx), %ymm25, %ymm22
-
-// CHECK: vandps 508(%rdx){1to8}, %ymm25, %ymm22
-// CHECK:  encoding: [0x62,0xe1,0x34,0x30,0x54,0x72,0x7f]
-          vandps 508(%rdx){1to8}, %ymm25, %ymm22
-
-// CHECK: vandps 512(%rdx){1to8}, %ymm25, %ymm22
-// CHECK:  encoding: [0x62,0xe1,0x34,0x30,0x54,0xb2,0x00,0x02,0x00,0x00]
-          vandps 512(%rdx){1to8}, %ymm25, %ymm22
-
-// CHECK: vandps -512(%rdx){1to8}, %ymm25, %ymm22
-// CHECK:  encoding: [0x62,0xe1,0x34,0x30,0x54,0x72,0x80]
-          vandps -512(%rdx){1to8}, %ymm25, %ymm22
-
-// CHECK: vandps -516(%rdx){1to8}, %ymm25, %ymm22
-// CHECK:  encoding: [0x62,0xe1,0x34,0x30,0x54,0xb2,0xfc,0xfd,0xff,0xff]
-          vandps -516(%rdx){1to8}, %ymm25, %ymm22
-
-// CHECK: vandnpd %xmm23, %xmm18, %xmm19
-// CHECK:  encoding: [0x62,0xa1,0xed,0x00,0x55,0xdf]
-          vandnpd %xmm23, %xmm18, %xmm19
-
-// CHECK: vandnpd %xmm23, %xmm18, %xmm19 {%k1}
-// CHECK:  encoding: [0x62,0xa1,0xed,0x01,0x55,0xdf]
-          vandnpd %xmm23, %xmm18, %xmm19 {%k1}
-
-// CHECK: vandnpd %xmm23, %xmm18, %xmm19 {%k1} {z}
-// CHECK:  encoding: [0x62,0xa1,0xed,0x81,0x55,0xdf]
-          vandnpd %xmm23, %xmm18, %xmm19 {%k1} {z}
-
-// CHECK: vandnpd (%rcx), %xmm18, %xmm19
-// CHECK:  encoding: [0x62,0xe1,0xed,0x00,0x55,0x19]
-          vandnpd (%rcx), %xmm18, %xmm19
-
-// CHECK: vandnpd 4660(%rax,%r14,8), %xmm18, %xmm19
-// CHECK:  encoding: [0x62,0xa1,0xed,0x00,0x55,0x9c,0xf0,0x34,0x12,0x00,0x00]
-          vandnpd 4660(%rax,%r14,8), %xmm18, %xmm19
-
-// CHECK: vandnpd (%rcx){1to2}, %xmm18, %xmm19
-// CHECK:  encoding: [0x62,0xe1,0xed,0x10,0x55,0x19]
-          vandnpd (%rcx){1to2}, %xmm18, %xmm19
-
-// CHECK: vandnpd 2032(%rdx), %xmm18, %xmm19
-// CHECK:  encoding: [0x62,0xe1,0xed,0x00,0x55,0x5a,0x7f]
-          vandnpd 2032(%rdx), %xmm18, %xmm19
-
-// CHECK: vandnpd 2048(%rdx), %xmm18, %xmm19
-// CHECK:  encoding: [0x62,0xe1,0xed,0x00,0x55,0x9a,0x00,0x08,0x00,0x00]
-          vandnpd 2048(%rdx), %xmm18, %xmm19
-
-// CHECK: vandnpd -2048(%rdx), %xmm18, %xmm19
-// CHECK:  encoding: [0x62,0xe1,0xed,0x00,0x55,0x5a,0x80]
-          vandnpd -2048(%rdx), %xmm18, %xmm19
-
-// CHECK: vandnpd -2064(%rdx), %xmm18, %xmm19
-// CHECK:  encoding: [0x62,0xe1,0xed,0x00,0x55,0x9a,0xf0,0xf7,0xff,0xff]
-          vandnpd -2064(%rdx), %xmm18, %xmm19
-
-// CHECK: vandnpd 1016(%rdx){1to2}, %xmm18, %xmm19
-// CHECK:  encoding: [0x62,0xe1,0xed,0x10,0x55,0x5a,0x7f]
-          vandnpd 1016(%rdx){1to2}, %xmm18, %xmm19
-
-// CHECK: vandnpd 1024(%rdx){1to2}, %xmm18, %xmm19
-// CHECK:  encoding: [0x62,0xe1,0xed,0x10,0x55,0x9a,0x00,0x04,0x00,0x00]
-          vandnpd 1024(%rdx){1to2}, %xmm18, %xmm19
-
-// CHECK: vandnpd -1024(%rdx){1to2}, %xmm18, %xmm19
-// CHECK:  encoding: [0x62,0xe1,0xed,0x10,0x55,0x5a,0x80]
-          vandnpd -1024(%rdx){1to2}, %xmm18, %xmm19
-
-// CHECK: vandnpd -1032(%rdx){1to2}, %xmm18, %xmm19
-// CHECK:  encoding: [0x62,0xe1,0xed,0x10,0x55,0x9a,0xf8,0xfb,0xff,0xff]
-          vandnpd -1032(%rdx){1to2}, %xmm18, %xmm19
-
-// CHECK: vandnpd %ymm28, %ymm29, %ymm25
-// CHECK:  encoding: [0x62,0x01,0x95,0x20,0x55,0xcc]
-          vandnpd %ymm28, %ymm29, %ymm25
-
-// CHECK: vandnpd %ymm28, %ymm29, %ymm25 {%k7}
-// CHECK:  encoding: [0x62,0x01,0x95,0x27,0x55,0xcc]
-          vandnpd %ymm28, %ymm29, %ymm25 {%k7}
-
-// CHECK: vandnpd %ymm28, %ymm29, %ymm25 {%k7} {z}
-// CHECK:  encoding: [0x62,0x01,0x95,0xa7,0x55,0xcc]
-          vandnpd %ymm28, %ymm29, %ymm25 {%k7} {z}
-
-// CHECK: vandnpd (%rcx), %ymm29, %ymm25
-// CHECK:  encoding: [0x62,0x61,0x95,0x20,0x55,0x09]
-          vandnpd (%rcx), %ymm29, %ymm25
-
-// CHECK: vandnpd 4660(%rax,%r14,8), %ymm29, %ymm25
-// CHECK:  encoding: [0x62,0x21,0x95,0x20,0x55,0x8c,0xf0,0x34,0x12,0x00,0x00]
-          vandnpd 4660(%rax,%r14,8), %ymm29, %ymm25
-
-// CHECK: vandnpd (%rcx){1to4}, %ymm29, %ymm25
-// CHECK:  encoding: [0x62,0x61,0x95,0x30,0x55,0x09]
-          vandnpd (%rcx){1to4}, %ymm29, %ymm25
-
-// CHECK: vandnpd 4064(%rdx), %ymm29, %ymm25
-// CHECK:  encoding: [0x62,0x61,0x95,0x20,0x55,0x4a,0x7f]
-          vandnpd 4064(%rdx), %ymm29, %ymm25
-
-// CHECK: vandnpd 4096(%rdx), %ymm29, %ymm25
-// CHECK:  encoding: [0x62,0x61,0x95,0x20,0x55,0x8a,0x00,0x10,0x00,0x00]
-          vandnpd 4096(%rdx), %ymm29, %ymm25
-
-// CHECK: vandnpd -4096(%rdx), %ymm29, %ymm25
-// CHECK:  encoding: [0x62,0x61,0x95,0x20,0x55,0x4a,0x80]
-          vandnpd -4096(%rdx), %ymm29, %ymm25
-
-// CHECK: vandnpd -4128(%rdx), %ymm29, %ymm25
-// CHECK:  encoding: [0x62,0x61,0x95,0x20,0x55,0x8a,0xe0,0xef,0xff,0xff]
-          vandnpd -4128(%rdx), %ymm29, %ymm25
-
-// CHECK: vandnpd 1016(%rdx){1to4}, %ymm29, %ymm25
-// CHECK:  encoding: [0x62,0x61,0x95,0x30,0x55,0x4a,0x7f]
-          vandnpd 1016(%rdx){1to4}, %ymm29, %ymm25
-
-// CHECK: vandnpd 1024(%rdx){1to4}, %ymm29, %ymm25
-// CHECK:  encoding: [0x62,0x61,0x95,0x30,0x55,0x8a,0x00,0x04,0x00,0x00]
-          vandnpd 1024(%rdx){1to4}, %ymm29, %ymm25
-
-// CHECK: vandnpd -1024(%rdx){1to4}, %ymm29, %ymm25
-// CHECK:  encoding: [0x62,0x61,0x95,0x30,0x55,0x4a,0x80]
-          vandnpd -1024(%rdx){1to4}, %ymm29, %ymm25
-
-// CHECK: vandnpd -1032(%rdx){1to4}, %ymm29, %ymm25
-// CHECK:  encoding: [0x62,0x61,0x95,0x30,0x55,0x8a,0xf8,0xfb,0xff,0xff]
-          vandnpd -1032(%rdx){1to4}, %ymm29, %ymm25
-
-// CHECK: vandnps %xmm23, %xmm17, %xmm25
-// CHECK:  encoding: [0x62,0x21,0x74,0x00,0x55,0xcf]
-          vandnps %xmm23, %xmm17, %xmm25
-
-// CHECK: vandnps %xmm23, %xmm17, %xmm25 {%k5}
-// CHECK:  encoding: [0x62,0x21,0x74,0x05,0x55,0xcf]
-          vandnps %xmm23, %xmm17, %xmm25 {%k5}
-
-// CHECK: vandnps %xmm23, %xmm17, %xmm25 {%k5} {z}
-// CHECK:  encoding: [0x62,0x21,0x74,0x85,0x55,0xcf]
-          vandnps %xmm23, %xmm17, %xmm25 {%k5} {z}
-
-// CHECK: vandnps (%rcx), %xmm17, %xmm25
-// CHECK:  encoding: [0x62,0x61,0x74,0x00,0x55,0x09]
-          vandnps (%rcx), %xmm17, %xmm25
-
-// CHECK: vandnps 4660(%rax,%r14,8), %xmm17, %xmm25
-// CHECK:  encoding: [0x62,0x21,0x74,0x00,0x55,0x8c,0xf0,0x34,0x12,0x00,0x00]
-          vandnps 4660(%rax,%r14,8), %xmm17, %xmm25
-
-// CHECK: vandnps (%rcx){1to4}, %xmm17, %xmm25
-// CHECK:  encoding: [0x62,0x61,0x74,0x10,0x55,0x09]
-          vandnps (%rcx){1to4}, %xmm17, %xmm25
-
-// CHECK: vandnps 2032(%rdx), %xmm17, %xmm25
-// CHECK:  encoding: [0x62,0x61,0x74,0x00,0x55,0x4a,0x7f]
-          vandnps 2032(%rdx), %xmm17, %xmm25
-
-// CHECK: vandnps 2048(%rdx), %xmm17, %xmm25
-// CHECK:  encoding: [0x62,0x61,0x74,0x00,0x55,0x8a,0x00,0x08,0x00,0x00]
-          vandnps 2048(%rdx), %xmm17, %xmm25
-
-// CHECK: vandnps -2048(%rdx), %xmm17, %xmm25
-// CHECK:  encoding: [0x62,0x61,0x74,0x00,0x55,0x4a,0x80]
-          vandnps -2048(%rdx), %xmm17, %xmm25
-
-// CHECK: vandnps -2064(%rdx), %xmm17, %xmm25
-// CHECK:  encoding: [0x62,0x61,0x74,0x00,0x55,0x8a,0xf0,0xf7,0xff,0xff]
-          vandnps -2064(%rdx), %xmm17, %xmm25
-
-// CHECK: vandnps 508(%rdx){1to4}, %xmm17, %xmm25
-// CHECK:  encoding: [0x62,0x61,0x74,0x10,0x55,0x4a,0x7f]
-          vandnps 508(%rdx){1to4}, %xmm17, %xmm25
-
-// CHECK: vandnps 512(%rdx){1to4}, %xmm17, %xmm25
-// CHECK:  encoding: [0x62,0x61,0x74,0x10,0x55,0x8a,0x00,0x02,0x00,0x00]
-          vandnps 512(%rdx){1to4}, %xmm17, %xmm25
-
-// CHECK: vandnps -512(%rdx){1to4}, %xmm17, %xmm25
-// CHECK:  encoding: [0x62,0x61,0x74,0x10,0x55,0x4a,0x80]
-          vandnps -512(%rdx){1to4}, %xmm17, %xmm25
-
-// CHECK: vandnps -516(%rdx){1to4}, %xmm17, %xmm25
-// CHECK:  encoding: [0x62,0x61,0x74,0x10,0x55,0x8a,0xfc,0xfd,0xff,0xff]
-          vandnps -516(%rdx){1to4}, %xmm17, %xmm25
-
-// CHECK: vandnps %ymm23, %ymm19, %ymm18
-// CHECK:  encoding: [0x62,0xa1,0x64,0x20,0x55,0xd7]
-          vandnps %ymm23, %ymm19, %ymm18
-
-// CHECK: vandnps %ymm23, %ymm19, %ymm18 {%k6}
-// CHECK:  encoding: [0x62,0xa1,0x64,0x26,0x55,0xd7]
-          vandnps %ymm23, %ymm19, %ymm18 {%k6}
-
-// CHECK: vandnps %ymm23, %ymm19, %ymm18 {%k6} {z}
-// CHECK:  encoding: [0x62,0xa1,0x64,0xa6,0x55,0xd7]
-          vandnps %ymm23, %ymm19, %ymm18 {%k6} {z}
-
-// CHECK: vandnps (%rcx), %ymm19, %ymm18
-// CHECK:  encoding: [0x62,0xe1,0x64,0x20,0x55,0x11]
-          vandnps (%rcx), %ymm19, %ymm18
-
-// CHECK: vandnps 4660(%rax,%r14,8), %ymm19, %ymm18
-// CHECK:  encoding: [0x62,0xa1,0x64,0x20,0x55,0x94,0xf0,0x34,0x12,0x00,0x00]
-          vandnps 4660(%rax,%r14,8), %ymm19, %ymm18
-
-// CHECK: vandnps (%rcx){1to8}, %ymm19, %ymm18
-// CHECK:  encoding: [0x62,0xe1,0x64,0x30,0x55,0x11]
-          vandnps (%rcx){1to8}, %ymm19, %ymm18
-
-// CHECK: vandnps 4064(%rdx), %ymm19, %ymm18
-// CHECK:  encoding: [0x62,0xe1,0x64,0x20,0x55,0x52,0x7f]
-          vandnps 4064(%rdx), %ymm19, %ymm18
-
-// CHECK: vandnps 4096(%rdx), %ymm19, %ymm18
-// CHECK:  encoding: [0x62,0xe1,0x64,0x20,0x55,0x92,0x00,0x10,0x00,0x00]
-          vandnps 4096(%rdx), %ymm19, %ymm18
-
-// CHECK: vandnps -4096(%rdx), %ymm19, %ymm18
-// CHECK:  encoding: [0x62,0xe1,0x64,0x20,0x55,0x52,0x80]
-          vandnps -4096(%rdx), %ymm19, %ymm18
-
-// CHECK: vandnps -4128(%rdx), %ymm19, %ymm18
-// CHECK:  encoding: [0x62,0xe1,0x64,0x20,0x55,0x92,0xe0,0xef,0xff,0xff]
-          vandnps -4128(%rdx), %ymm19, %ymm18
-
-// CHECK: vandnps 508(%rdx){1to8}, %ymm19, %ymm18
-// CHECK:  encoding: [0x62,0xe1,0x64,0x30,0x55,0x52,0x7f]
-          vandnps 508(%rdx){1to8}, %ymm19, %ymm18
-
-// CHECK: vandnps 512(%rdx){1to8}, %ymm19, %ymm18
-// CHECK:  encoding: [0x62,0xe1,0x64,0x30,0x55,0x92,0x00,0x02,0x00,0x00]
-          vandnps 512(%rdx){1to8}, %ymm19, %ymm18
-
-// CHECK: vandnps -512(%rdx){1to8}, %ymm19, %ymm18
-// CHECK:  encoding: [0x62,0xe1,0x64,0x30,0x55,0x52,0x80]
-          vandnps -512(%rdx){1to8}, %ymm19, %ymm18
-
-// CHECK: vandnps -516(%rdx){1to8}, %ymm19, %ymm18
-// CHECK:  encoding: [0x62,0xe1,0x64,0x30,0x55,0x92,0xfc,0xfd,0xff,0xff]
-          vandnps -516(%rdx){1to8}, %ymm19, %ymm18
-
-// CHECK: vorpd  %xmm18, %xmm29, %xmm26
-// CHECK:  encoding: [0x62,0x21,0x95,0x00,0x56,0xd2]
-          vorpd  %xmm18, %xmm29, %xmm26
-
-// CHECK: vorpd  %xmm18, %xmm29, %xmm26 {%k2}
-// CHECK:  encoding: [0x62,0x21,0x95,0x02,0x56,0xd2]
-          vorpd  %xmm18, %xmm29, %xmm26 {%k2}
-
-// CHECK: vorpd  %xmm18, %xmm29, %xmm26 {%k2} {z}
-// CHECK:  encoding: [0x62,0x21,0x95,0x82,0x56,0xd2]
-          vorpd  %xmm18, %xmm29, %xmm26 {%k2} {z}
-
-// CHECK: vorpd  (%rcx), %xmm29, %xmm26
-// CHECK:  encoding: [0x62,0x61,0x95,0x00,0x56,0x11]
-          vorpd  (%rcx), %xmm29, %xmm26
-
-// CHECK: vorpd  4660(%rax,%r14,8), %xmm29, %xmm26
-// CHECK:  encoding: [0x62,0x21,0x95,0x00,0x56,0x94,0xf0,0x34,0x12,0x00,0x00]
-          vorpd  4660(%rax,%r14,8), %xmm29, %xmm26
-
-// CHECK: vorpd  (%rcx){1to2}, %xmm29, %xmm26
-// CHECK:  encoding: [0x62,0x61,0x95,0x10,0x56,0x11]
-          vorpd  (%rcx){1to2}, %xmm29, %xmm26
-
-// CHECK: vorpd  2032(%rdx), %xmm29, %xmm26
-// CHECK:  encoding: [0x62,0x61,0x95,0x00,0x56,0x52,0x7f]
-          vorpd  2032(%rdx), %xmm29, %xmm26
-
-// CHECK: vorpd  2048(%rdx), %xmm29, %xmm26
-// CHECK:  encoding: [0x62,0x61,0x95,0x00,0x56,0x92,0x00,0x08,0x00,0x00]
-          vorpd  2048(%rdx), %xmm29, %xmm26
-
-// CHECK: vorpd  -2048(%rdx), %xmm29, %xmm26
-// CHECK:  encoding: [0x62,0x61,0x95,0x00,0x56,0x52,0x80]
-          vorpd  -2048(%rdx), %xmm29, %xmm26
-
-// CHECK: vorpd  -2064(%rdx), %xmm29, %xmm26
-// CHECK:  encoding: [0x62,0x61,0x95,0x00,0x56,0x92,0xf0,0xf7,0xff,0xff]
-          vorpd  -2064(%rdx), %xmm29, %xmm26
-
-// CHECK: vorpd  1016(%rdx){1to2}, %xmm29, %xmm26
-// CHECK:  encoding: [0x62,0x61,0x95,0x10,0x56,0x52,0x7f]
-          vorpd  1016(%rdx){1to2}, %xmm29, %xmm26
-
-// CHECK: vorpd  1024(%rdx){1to2}, %xmm29, %xmm26
-// CHECK:  encoding: [0x62,0x61,0x95,0x10,0x56,0x92,0x00,0x04,0x00,0x00]
-          vorpd  1024(%rdx){1to2}, %xmm29, %xmm26
-
-// CHECK: vorpd  -1024(%rdx){1to2}, %xmm29, %xmm26
-// CHECK:  encoding: [0x62,0x61,0x95,0x10,0x56,0x52,0x80]
-          vorpd  -1024(%rdx){1to2}, %xmm29, %xmm26
-
-// CHECK: vorpd  -1032(%rdx){1to2}, %xmm29, %xmm26
-// CHECK:  encoding: [0x62,0x61,0x95,0x10,0x56,0x92,0xf8,0xfb,0xff,0xff]
-          vorpd  -1032(%rdx){1to2}, %xmm29, %xmm26
-
-// CHECK: vorpd  %ymm22, %ymm19, %ymm28
-// CHECK:  encoding: [0x62,0x21,0xe5,0x20,0x56,0xe6]
-          vorpd  %ymm22, %ymm19, %ymm28
-
-// CHECK: vorpd  %ymm22, %ymm19, %ymm28 {%k1}
-// CHECK:  encoding: [0x62,0x21,0xe5,0x21,0x56,0xe6]
-          vorpd  %ymm22, %ymm19, %ymm28 {%k1}
-
-// CHECK: vorpd  %ymm22, %ymm19, %ymm28 {%k1} {z}
-// CHECK:  encoding: [0x62,0x21,0xe5,0xa1,0x56,0xe6]
-          vorpd  %ymm22, %ymm19, %ymm28 {%k1} {z}
-
-// CHECK: vorpd  (%rcx), %ymm19, %ymm28
-// CHECK:  encoding: [0x62,0x61,0xe5,0x20,0x56,0x21]
-          vorpd  (%rcx), %ymm19, %ymm28
-
-// CHECK: vorpd  4660(%rax,%r14,8), %ymm19, %ymm28
-// CHECK:  encoding: [0x62,0x21,0xe5,0x20,0x56,0xa4,0xf0,0x34,0x12,0x00,0x00]
-          vorpd  4660(%rax,%r14,8), %ymm19, %ymm28
-
-// CHECK: vorpd  (%rcx){1to4}, %ymm19, %ymm28
-// CHECK:  encoding: [0x62,0x61,0xe5,0x30,0x56,0x21]
-          vorpd  (%rcx){1to4}, %ymm19, %ymm28
-
-// CHECK: vorpd  4064(%rdx), %ymm19, %ymm28
-// CHECK:  encoding: [0x62,0x61,0xe5,0x20,0x56,0x62,0x7f]
-          vorpd  4064(%rdx), %ymm19, %ymm28
-
-// CHECK: vorpd  4096(%rdx), %ymm19, %ymm28
-// CHECK:  encoding: [0x62,0x61,0xe5,0x20,0x56,0xa2,0x00,0x10,0x00,0x00]
-          vorpd  4096(%rdx), %ymm19, %ymm28
-
-// CHECK: vorpd  -4096(%rdx), %ymm19, %ymm28
-// CHECK:  encoding: [0x62,0x61,0xe5,0x20,0x56,0x62,0x80]
-          vorpd  -4096(%rdx), %ymm19, %ymm28
-
-// CHECK: vorpd  -4128(%rdx), %ymm19, %ymm28
-// CHECK:  encoding: [0x62,0x61,0xe5,0x20,0x56,0xa2,0xe0,0xef,0xff,0xff]
-          vorpd  -4128(%rdx), %ymm19, %ymm28
-
-// CHECK: vorpd  1016(%rdx){1to4}, %ymm19, %ymm28
-// CHECK:  encoding: [0x62,0x61,0xe5,0x30,0x56,0x62,0x7f]
-          vorpd  1016(%rdx){1to4}, %ymm19, %ymm28
-
-// CHECK: vorpd  1024(%rdx){1to4}, %ymm19, %ymm28
-// CHECK:  encoding: [0x62,0x61,0xe5,0x30,0x56,0xa2,0x00,0x04,0x00,0x00]
-          vorpd  1024(%rdx){1to4}, %ymm19, %ymm28
-
-// CHECK: vorpd  -1024(%rdx){1to4}, %ymm19, %ymm28
-// CHECK:  encoding: [0x62,0x61,0xe5,0x30,0x56,0x62,0x80]
-          vorpd  -1024(%rdx){1to4}, %ymm19, %ymm28
-
-// CHECK: vorpd  -1032(%rdx){1to4}, %ymm19, %ymm28
-// CHECK:  encoding: [0x62,0x61,0xe5,0x30,0x56,0xa2,0xf8,0xfb,0xff,0xff]
-          vorpd  -1032(%rdx){1to4}, %ymm19, %ymm28
-
-// CHECK: vorps  %xmm24, %xmm22, %xmm28
-// CHECK:  encoding: [0x62,0x01,0x4c,0x00,0x56,0xe0]
-          vorps  %xmm24, %xmm22, %xmm28
-
-// CHECK: vorps  %xmm24, %xmm22, %xmm28 {%k6}
-// CHECK:  encoding: [0x62,0x01,0x4c,0x06,0x56,0xe0]
-          vorps  %xmm24, %xmm22, %xmm28 {%k6}
-
-// CHECK: vorps  %xmm24, %xmm22, %xmm28 {%k6} {z}
-// CHECK:  encoding: [0x62,0x01,0x4c,0x86,0x56,0xe0]
-          vorps  %xmm24, %xmm22, %xmm28 {%k6} {z}
-
-// CHECK: vorps  (%rcx), %xmm22, %xmm28
-// CHECK:  encoding: [0x62,0x61,0x4c,0x00,0x56,0x21]
-          vorps  (%rcx), %xmm22, %xmm28
-
-// CHECK: vorps  4660(%rax,%r14,8), %xmm22, %xmm28
-// CHECK:  encoding: [0x62,0x21,0x4c,0x00,0x56,0xa4,0xf0,0x34,0x12,0x00,0x00]
-          vorps  4660(%rax,%r14,8), %xmm22, %xmm28
-
-// CHECK: vorps  (%rcx){1to4}, %xmm22, %xmm28
-// CHECK:  encoding: [0x62,0x61,0x4c,0x10,0x56,0x21]
-          vorps  (%rcx){1to4}, %xmm22, %xmm28
-
-// CHECK: vorps  2032(%rdx), %xmm22, %xmm28
-// CHECK:  encoding: [0x62,0x61,0x4c,0x00,0x56,0x62,0x7f]
-          vorps  2032(%rdx), %xmm22, %xmm28
-
-// CHECK: vorps  2048(%rdx), %xmm22, %xmm28
-// CHECK:  encoding: [0x62,0x61,0x4c,0x00,0x56,0xa2,0x00,0x08,0x00,0x00]
-          vorps  2048(%rdx), %xmm22, %xmm28
-
-// CHECK: vorps  -2048(%rdx), %xmm22, %xmm28
-// CHECK:  encoding: [0x62,0x61,0x4c,0x00,0x56,0x62,0x80]
-          vorps  -2048(%rdx), %xmm22, %xmm28
-
-// CHECK: vorps  -2064(%rdx), %xmm22, %xmm28
-// CHECK:  encoding: [0x62,0x61,0x4c,0x00,0x56,0xa2,0xf0,0xf7,0xff,0xff]
-          vorps  -2064(%rdx), %xmm22, %xmm28
-
-// CHECK: vorps  508(%rdx){1to4}, %xmm22, %xmm28
-// CHECK:  encoding: [0x62,0x61,0x4c,0x10,0x56,0x62,0x7f]
-          vorps  508(%rdx){1to4}, %xmm22, %xmm28
-
-// CHECK: vorps  512(%rdx){1to4}, %xmm22, %xmm28
-// CHECK:  encoding: [0x62,0x61,0x4c,0x10,0x56,0xa2,0x00,0x02,0x00,0x00]
-          vorps  512(%rdx){1to4}, %xmm22, %xmm28
-
-// CHECK: vorps  -512(%rdx){1to4}, %xmm22, %xmm28
-// CHECK:  encoding: [0x62,0x61,0x4c,0x10,0x56,0x62,0x80]
-          vorps  -512(%rdx){1to4}, %xmm22, %xmm28
-
-// CHECK: vorps  -516(%rdx){1to4}, %xmm22, %xmm28
-// CHECK:  encoding: [0x62,0x61,0x4c,0x10,0x56,0xa2,0xfc,0xfd,0xff,0xff]
-          vorps  -516(%rdx){1to4}, %xmm22, %xmm28
-
-// CHECK: vorps  %ymm25, %ymm24, %ymm20
-// CHECK:  encoding: [0x62,0x81,0x3c,0x20,0x56,0xe1]
-          vorps  %ymm25, %ymm24, %ymm20
-
-// CHECK: vorps  %ymm25, %ymm24, %ymm20 {%k1}
-// CHECK:  encoding: [0x62,0x81,0x3c,0x21,0x56,0xe1]
-          vorps  %ymm25, %ymm24, %ymm20 {%k1}
-
-// CHECK: vorps  %ymm25, %ymm24, %ymm20 {%k1} {z}
-// CHECK:  encoding: [0x62,0x81,0x3c,0xa1,0x56,0xe1]
-          vorps  %ymm25, %ymm24, %ymm20 {%k1} {z}
-
-// CHECK: vorps  (%rcx), %ymm24, %ymm20
-// CHECK:  encoding: [0x62,0xe1,0x3c,0x20,0x56,0x21]
-          vorps  (%rcx), %ymm24, %ymm20
-
-// CHECK: vorps  4660(%rax,%r14,8), %ymm24, %ymm20
-// CHECK:  encoding: [0x62,0xa1,0x3c,0x20,0x56,0xa4,0xf0,0x34,0x12,0x00,0x00]
-          vorps  4660(%rax,%r14,8), %ymm24, %ymm20
-
-// CHECK: vorps  (%rcx){1to8}, %ymm24, %ymm20
-// CHECK:  encoding: [0x62,0xe1,0x3c,0x30,0x56,0x21]
-          vorps  (%rcx){1to8}, %ymm24, %ymm20
-
-// CHECK: vorps  4064(%rdx), %ymm24, %ymm20
-// CHECK:  encoding: [0x62,0xe1,0x3c,0x20,0x56,0x62,0x7f]
-          vorps  4064(%rdx), %ymm24, %ymm20
-
-// CHECK: vorps  4096(%rdx), %ymm24, %ymm20
-// CHECK:  encoding: [0x62,0xe1,0x3c,0x20,0x56,0xa2,0x00,0x10,0x00,0x00]
-          vorps  4096(%rdx), %ymm24, %ymm20
-
-// CHECK: vorps  -4096(%rdx), %ymm24, %ymm20
-// CHECK:  encoding: [0x62,0xe1,0x3c,0x20,0x56,0x62,0x80]
-          vorps  -4096(%rdx), %ymm24, %ymm20
-
-// CHECK: vorps  -4128(%rdx), %ymm24, %ymm20
-// CHECK:  encoding: [0x62,0xe1,0x3c,0x20,0x56,0xa2,0xe0,0xef,0xff,0xff]
-          vorps  -4128(%rdx), %ymm24, %ymm20
-
-// CHECK: vorps  508(%rdx){1to8}, %ymm24, %ymm20
-// CHECK:  encoding: [0x62,0xe1,0x3c,0x30,0x56,0x62,0x7f]
-          vorps  508(%rdx){1to8}, %ymm24, %ymm20
-
-// CHECK: vorps  512(%rdx){1to8}, %ymm24, %ymm20
-// CHECK:  encoding: [0x62,0xe1,0x3c,0x30,0x56,0xa2,0x00,0x02,0x00,0x00]
-          vorps  512(%rdx){1to8}, %ymm24, %ymm20
-
-// CHECK: vorps  -512(%rdx){1to8}, %ymm24, %ymm20
-// CHECK:  encoding: [0x62,0xe1,0x3c,0x30,0x56,0x62,0x80]
-          vorps  -512(%rdx){1to8}, %ymm24, %ymm20
-
-// CHECK: vorps  -516(%rdx){1to8}, %ymm24, %ymm20
-// CHECK:  encoding: [0x62,0xe1,0x3c,0x30,0x56,0xa2,0xfc,0xfd,0xff,0xff]
-          vorps  -516(%rdx){1to8}, %ymm24, %ymm20
-
-// CHECK: vxorpd %xmm18, %xmm21, %xmm22
-// CHECK:  encoding: [0x62,0xa1,0xd5,0x00,0x57,0xf2]
-          vxorpd %xmm18, %xmm21, %xmm22
-
-// CHECK: vxorpd %xmm18, %xmm21, %xmm22 {%k3}
-// CHECK:  encoding: [0x62,0xa1,0xd5,0x03,0x57,0xf2]
-          vxorpd %xmm18, %xmm21, %xmm22 {%k3}
-
-// CHECK: vxorpd %xmm18, %xmm21, %xmm22 {%k3} {z}
-// CHECK:  encoding: [0x62,0xa1,0xd5,0x83,0x57,0xf2]
-          vxorpd %xmm18, %xmm21, %xmm22 {%k3} {z}
-
-// CHECK: vxorpd (%rcx), %xmm21, %xmm22
-// CHECK:  encoding: [0x62,0xe1,0xd5,0x00,0x57,0x31]
-          vxorpd (%rcx), %xmm21, %xmm22
-
-// CHECK: vxorpd 4660(%rax,%r14,8), %xmm21, %xmm22
-// CHECK:  encoding: [0x62,0xa1,0xd5,0x00,0x57,0xb4,0xf0,0x34,0x12,0x00,0x00]
-          vxorpd 4660(%rax,%r14,8), %xmm21, %xmm22
-
-// CHECK: vxorpd (%rcx){1to2}, %xmm21, %xmm22
-// CHECK:  encoding: [0x62,0xe1,0xd5,0x10,0x57,0x31]
-          vxorpd (%rcx){1to2}, %xmm21, %xmm22
-
-// CHECK: vxorpd 2032(%rdx), %xmm21, %xmm22
-// CHECK:  encoding: [0x62,0xe1,0xd5,0x00,0x57,0x72,0x7f]
-          vxorpd 2032(%rdx), %xmm21, %xmm22
-
-// CHECK: vxorpd 2048(%rdx), %xmm21, %xmm22
-// CHECK:  encoding: [0x62,0xe1,0xd5,0x00,0x57,0xb2,0x00,0x08,0x00,0x00]
-          vxorpd 2048(%rdx), %xmm21, %xmm22
-
-// CHECK: vxorpd -2048(%rdx), %xmm21, %xmm22
-// CHECK:  encoding: [0x62,0xe1,0xd5,0x00,0x57,0x72,0x80]
-          vxorpd -2048(%rdx), %xmm21, %xmm22
-
-// CHECK: vxorpd -2064(%rdx), %xmm21, %xmm22
-// CHECK:  encoding: [0x62,0xe1,0xd5,0x00,0x57,0xb2,0xf0,0xf7,0xff,0xff]
-          vxorpd -2064(%rdx), %xmm21, %xmm22
-
-// CHECK: vxorpd 1016(%rdx){1to2}, %xmm21, %xmm22
-// CHECK:  encoding: [0x62,0xe1,0xd5,0x10,0x57,0x72,0x7f]
-          vxorpd 1016(%rdx){1to2}, %xmm21, %xmm22
-
-// CHECK: vxorpd 1024(%rdx){1to2}, %xmm21, %xmm22
-// CHECK:  encoding: [0x62,0xe1,0xd5,0x10,0x57,0xb2,0x00,0x04,0x00,0x00]
-          vxorpd 1024(%rdx){1to2}, %xmm21, %xmm22
-
-// CHECK: vxorpd -1024(%rdx){1to2}, %xmm21, %xmm22
-// CHECK:  encoding: [0x62,0xe1,0xd5,0x10,0x57,0x72,0x80]
-          vxorpd -1024(%rdx){1to2}, %xmm21, %xmm22
-
-// CHECK: vxorpd -1032(%rdx){1to2}, %xmm21, %xmm22
-// CHECK:  encoding: [0x62,0xe1,0xd5,0x10,0x57,0xb2,0xf8,0xfb,0xff,0xff]
-          vxorpd -1032(%rdx){1to2}, %xmm21, %xmm22
-
-// CHECK: vxorpd %ymm27, %ymm21, %ymm25
-// CHECK:  encoding: [0x62,0x01,0xd5,0x20,0x57,0xcb]
-          vxorpd %ymm27, %ymm21, %ymm25
-
-// CHECK: vxorpd %ymm27, %ymm21, %ymm25 {%k7}
-// CHECK:  encoding: [0x62,0x01,0xd5,0x27,0x57,0xcb]
-          vxorpd %ymm27, %ymm21, %ymm25 {%k7}
-
-// CHECK: vxorpd %ymm27, %ymm21, %ymm25 {%k7} {z}
-// CHECK:  encoding: [0x62,0x01,0xd5,0xa7,0x57,0xcb]
-          vxorpd %ymm27, %ymm21, %ymm25 {%k7} {z}
-
-// CHECK: vxorpd (%rcx), %ymm21, %ymm25
-// CHECK:  encoding: [0x62,0x61,0xd5,0x20,0x57,0x09]
-          vxorpd (%rcx), %ymm21, %ymm25
-
-// CHECK: vxorpd 4660(%rax,%r14,8), %ymm21, %ymm25
-// CHECK:  encoding: [0x62,0x21,0xd5,0x20,0x57,0x8c,0xf0,0x34,0x12,0x00,0x00]
-          vxorpd 4660(%rax,%r14,8), %ymm21, %ymm25
-
-// CHECK: vxorpd (%rcx){1to4}, %ymm21, %ymm25
-// CHECK:  encoding: [0x62,0x61,0xd5,0x30,0x57,0x09]
-          vxorpd (%rcx){1to4}, %ymm21, %ymm25
-
-// CHECK: vxorpd 4064(%rdx), %ymm21, %ymm25
-// CHECK:  encoding: [0x62,0x61,0xd5,0x20,0x57,0x4a,0x7f]
-          vxorpd 4064(%rdx), %ymm21, %ymm25
-
-// CHECK: vxorpd 4096(%rdx), %ymm21, %ymm25
-// CHECK:  encoding: [0x62,0x61,0xd5,0x20,0x57,0x8a,0x00,0x10,0x00,0x00]
-          vxorpd 4096(%rdx), %ymm21, %ymm25
-
-// CHECK: vxorpd -4096(%rdx), %ymm21, %ymm25
-// CHECK:  encoding: [0x62,0x61,0xd5,0x20,0x57,0x4a,0x80]
-          vxorpd -4096(%rdx), %ymm21, %ymm25
-
-// CHECK: vxorpd -4128(%rdx), %ymm21, %ymm25
-// CHECK:  encoding: [0x62,0x61,0xd5,0x20,0x57,0x8a,0xe0,0xef,0xff,0xff]
-          vxorpd -4128(%rdx), %ymm21, %ymm25
-
-// CHECK: vxorpd 1016(%rdx){1to4}, %ymm21, %ymm25
-// CHECK:  encoding: [0x62,0x61,0xd5,0x30,0x57,0x4a,0x7f]
-          vxorpd 1016(%rdx){1to4}, %ymm21, %ymm25
-
-// CHECK: vxorpd 1024(%rdx){1to4}, %ymm21, %ymm25
-// CHECK:  encoding: [0x62,0x61,0xd5,0x30,0x57,0x8a,0x00,0x04,0x00,0x00]
-          vxorpd 1024(%rdx){1to4}, %ymm21, %ymm25
-
-// CHECK: vxorpd -1024(%rdx){1to4}, %ymm21, %ymm25
-// CHECK:  encoding: [0x62,0x61,0xd5,0x30,0x57,0x4a,0x80]
-          vxorpd -1024(%rdx){1to4}, %ymm21, %ymm25
-
-// CHECK: vxorpd -1032(%rdx){1to4}, %ymm21, %ymm25
-// CHECK:  encoding: [0x62,0x61,0xd5,0x30,0x57,0x8a,0xf8,0xfb,0xff,0xff]
-          vxorpd -1032(%rdx){1to4}, %ymm21, %ymm25
-
-// CHECK: vxorps %xmm21, %xmm21, %xmm17
-// CHECK:  encoding: [0x62,0xa1,0x54,0x00,0x57,0xcd]
-          vxorps %xmm21, %xmm21, %xmm17
-
-// CHECK: vxorps %xmm21, %xmm21, %xmm17 {%k5}
-// CHECK:  encoding: [0x62,0xa1,0x54,0x05,0x57,0xcd]
-          vxorps %xmm21, %xmm21, %xmm17 {%k5}
-
-// CHECK: vxorps %xmm21, %xmm21, %xmm17 {%k5} {z}
-// CHECK:  encoding: [0x62,0xa1,0x54,0x85,0x57,0xcd]
-          vxorps %xmm21, %xmm21, %xmm17 {%k5} {z}
-
-// CHECK: vxorps (%rcx), %xmm21, %xmm17
-// CHECK:  encoding: [0x62,0xe1,0x54,0x00,0x57,0x09]
-          vxorps (%rcx), %xmm21, %xmm17
-
-// CHECK: vxorps 4660(%rax,%r14,8), %xmm21, %xmm17
-// CHECK:  encoding: [0x62,0xa1,0x54,0x00,0x57,0x8c,0xf0,0x34,0x12,0x00,0x00]
-          vxorps 4660(%rax,%r14,8), %xmm21, %xmm17
-
-// CHECK: vxorps (%rcx){1to4}, %xmm21, %xmm17
-// CHECK:  encoding: [0x62,0xe1,0x54,0x10,0x57,0x09]
-          vxorps (%rcx){1to4}, %xmm21, %xmm17
-
-// CHECK: vxorps 2032(%rdx), %xmm21, %xmm17
-// CHECK:  encoding: [0x62,0xe1,0x54,0x00,0x57,0x4a,0x7f]
-          vxorps 2032(%rdx), %xmm21, %xmm17
-
-// CHECK: vxorps 2048(%rdx), %xmm21, %xmm17
-// CHECK:  encoding: [0x62,0xe1,0x54,0x00,0x57,0x8a,0x00,0x08,0x00,0x00]
-          vxorps 2048(%rdx), %xmm21, %xmm17
-
-// CHECK: vxorps -2048(%rdx), %xmm21, %xmm17
-// CHECK:  encoding: [0x62,0xe1,0x54,0x00,0x57,0x4a,0x80]
-          vxorps -2048(%rdx), %xmm21, %xmm17
-
-// CHECK: vxorps -2064(%rdx), %xmm21, %xmm17
-// CHECK:  encoding: [0x62,0xe1,0x54,0x00,0x57,0x8a,0xf0,0xf7,0xff,0xff]
-          vxorps -2064(%rdx), %xmm21, %xmm17
-
-// CHECK: vxorps 508(%rdx){1to4}, %xmm21, %xmm17
-// CHECK:  encoding: [0x62,0xe1,0x54,0x10,0x57,0x4a,0x7f]
-          vxorps 508(%rdx){1to4}, %xmm21, %xmm17
-
-// CHECK: vxorps 512(%rdx){1to4}, %xmm21, %xmm17
-// CHECK:  encoding: [0x62,0xe1,0x54,0x10,0x57,0x8a,0x00,0x02,0x00,0x00]
-          vxorps 512(%rdx){1to4}, %xmm21, %xmm17
-
-// CHECK: vxorps -512(%rdx){1to4}, %xmm21, %xmm17
-// CHECK:  encoding: [0x62,0xe1,0x54,0x10,0x57,0x4a,0x80]
-          vxorps -512(%rdx){1to4}, %xmm21, %xmm17
-
-// CHECK: vxorps -516(%rdx){1to4}, %xmm21, %xmm17
-// CHECK:  encoding: [0x62,0xe1,0x54,0x10,0x57,0x8a,0xfc,0xfd,0xff,0xff]
-          vxorps -516(%rdx){1to4}, %xmm21, %xmm17
-
-// CHECK: vxorps %ymm22, %ymm25, %ymm28
-// CHECK:  encoding: [0x62,0x21,0x34,0x20,0x57,0xe6]
-          vxorps %ymm22, %ymm25, %ymm28
-
-// CHECK: vxorps %ymm22, %ymm25, %ymm28 {%k3}
-// CHECK:  encoding: [0x62,0x21,0x34,0x23,0x57,0xe6]
-          vxorps %ymm22, %ymm25, %ymm28 {%k3}
-
-// CHECK: vxorps %ymm22, %ymm25, %ymm28 {%k3} {z}
-// CHECK:  encoding: [0x62,0x21,0x34,0xa3,0x57,0xe6]
-          vxorps %ymm22, %ymm25, %ymm28 {%k3} {z}
-
-// CHECK: vxorps (%rcx), %ymm25, %ymm28
-// CHECK:  encoding: [0x62,0x61,0x34,0x20,0x57,0x21]
-          vxorps (%rcx), %ymm25, %ymm28
-
-// CHECK: vxorps 4660(%rax,%r14,8), %ymm25, %ymm28
-// CHECK:  encoding: [0x62,0x21,0x34,0x20,0x57,0xa4,0xf0,0x34,0x12,0x00,0x00]
-          vxorps 4660(%rax,%r14,8), %ymm25, %ymm28
-
-// CHECK: vxorps (%rcx){1to8}, %ymm25, %ymm28
-// CHECK:  encoding: [0x62,0x61,0x34,0x30,0x57,0x21]
-          vxorps (%rcx){1to8}, %ymm25, %ymm28
-
-// CHECK: vxorps 4064(%rdx), %ymm25, %ymm28
-// CHECK:  encoding: [0x62,0x61,0x34,0x20,0x57,0x62,0x7f]
-          vxorps 4064(%rdx), %ymm25, %ymm28
-
-// CHECK: vxorps 4096(%rdx), %ymm25, %ymm28
-// CHECK:  encoding: [0x62,0x61,0x34,0x20,0x57,0xa2,0x00,0x10,0x00,0x00]
-          vxorps 4096(%rdx), %ymm25, %ymm28
-
-// CHECK: vxorps -4096(%rdx), %ymm25, %ymm28
-// CHECK:  encoding: [0x62,0x61,0x34,0x20,0x57,0x62,0x80]
-          vxorps -4096(%rdx), %ymm25, %ymm28
-
-// CHECK: vxorps -4128(%rdx), %ymm25, %ymm28
-// CHECK:  encoding: [0x62,0x61,0x34,0x20,0x57,0xa2,0xe0,0xef,0xff,0xff]
-          vxorps -4128(%rdx), %ymm25, %ymm28
-
-// CHECK: vxorps 508(%rdx){1to8}, %ymm25, %ymm28
-// CHECK:  encoding: [0x62,0x61,0x34,0x30,0x57,0x62,0x7f]
-          vxorps 508(%rdx){1to8}, %ymm25, %ymm28
-
-// CHECK: vxorps 512(%rdx){1to8}, %ymm25, %ymm28
-// CHECK:  encoding: [0x62,0x61,0x34,0x30,0x57,0xa2,0x00,0x02,0x00,0x00]
-          vxorps 512(%rdx){1to8}, %ymm25, %ymm28
-
-// CHECK: vxorps -512(%rdx){1to8}, %ymm25, %ymm28
-// CHECK:  encoding: [0x62,0x61,0x34,0x30,0x57,0x62,0x80]
-          vxorps -512(%rdx){1to8}, %ymm25, %ymm28
-
-// CHECK: vxorps -516(%rdx){1to8}, %ymm25, %ymm28
-// CHECK:  encoding: [0x62,0x61,0x34,0x30,0x57,0xa2,0xfc,0xfd,0xff,0xff]
-          vxorps -516(%rdx){1to8}, %ymm25, %ymm28
-
-// CHECK: vbroadcastf64x2 (%rcx), %ymm27
-// CHECK:  encoding: [0x62,0x62,0xfd,0x28,0x1a,0x19]
-          vbroadcastf64x2 (%rcx), %ymm27
-
-// CHECK: vbroadcastf64x2 (%rcx), %ymm27 {%k5}
-// CHECK:  encoding: [0x62,0x62,0xfd,0x2d,0x1a,0x19]
-          vbroadcastf64x2 (%rcx), %ymm27 {%k5}
-
-// CHECK: vbroadcastf64x2 (%rcx), %ymm27 {%k5} {z}
-// CHECK:  encoding: [0x62,0x62,0xfd,0xad,0x1a,0x19]
-          vbroadcastf64x2 (%rcx), %ymm27 {%k5} {z}
-
-// CHECK: vbroadcastf64x2 291(%rax,%r14,8), %ymm27
-// CHECK:  encoding: [0x62,0x22,0xfd,0x28,0x1a,0x9c,0xf0,0x23,0x01,0x00,0x00]
-          vbroadcastf64x2 291(%rax,%r14,8), %ymm27
-
-// CHECK: vbroadcastf64x2 2032(%rdx), %ymm27
-// CHECK:  encoding: [0x62,0x62,0xfd,0x28,0x1a,0x5a,0x7f]
-          vbroadcastf64x2 2032(%rdx), %ymm27
-
-// CHECK: vbroadcastf64x2 2048(%rdx), %ymm27
-// CHECK:  encoding: [0x62,0x62,0xfd,0x28,0x1a,0x9a,0x00,0x08,0x00,0x00]
-          vbroadcastf64x2 2048(%rdx), %ymm27
-
-// CHECK: vbroadcastf64x2 -2048(%rdx), %ymm27
-// CHECK:  encoding: [0x62,0x62,0xfd,0x28,0x1a,0x5a,0x80]
-          vbroadcastf64x2 -2048(%rdx), %ymm27
-
-// CHECK: vbroadcastf64x2 -2064(%rdx), %ymm27
-// CHECK:  encoding: [0x62,0x62,0xfd,0x28,0x1a,0x9a,0xf0,0xf7,0xff,0xff]
-          vbroadcastf64x2 -2064(%rdx), %ymm27
-
-// CHECK: vbroadcasti64x2 (%rcx), %ymm18
-// CHECK:  encoding: [0x62,0xe2,0xfd,0x28,0x5a,0x11]
-          vbroadcasti64x2 (%rcx), %ymm18
-
-// CHECK: vbroadcasti64x2 (%rcx), %ymm18 {%k1}
-// CHECK:  encoding: [0x62,0xe2,0xfd,0x29,0x5a,0x11]
-          vbroadcasti64x2 (%rcx), %ymm18 {%k1}
-
-// CHECK: vbroadcasti64x2 (%rcx), %ymm18 {%k1} {z}
-// CHECK:  encoding: [0x62,0xe2,0xfd,0xa9,0x5a,0x11]
-          vbroadcasti64x2 (%rcx), %ymm18 {%k1} {z}
-
-// CHECK: vbroadcasti64x2 291(%rax,%r14,8), %ymm18
-// CHECK:  encoding: [0x62,0xa2,0xfd,0x28,0x5a,0x94,0xf0,0x23,0x01,0x00,0x00]
-          vbroadcasti64x2 291(%rax,%r14,8), %ymm18
-
-// CHECK: vbroadcasti64x2 2032(%rdx), %ymm18
-// CHECK:  encoding: [0x62,0xe2,0xfd,0x28,0x5a,0x52,0x7f]
-          vbroadcasti64x2 2032(%rdx), %ymm18
-
-// CHECK: vbroadcasti64x2 2048(%rdx), %ymm18
-// CHECK:  encoding: [0x62,0xe2,0xfd,0x28,0x5a,0x92,0x00,0x08,0x00,0x00]
-          vbroadcasti64x2 2048(%rdx), %ymm18
-
-// CHECK: vbroadcasti64x2 -2048(%rdx), %ymm18
-// CHECK:  encoding: [0x62,0xe2,0xfd,0x28,0x5a,0x52,0x80]
-          vbroadcasti64x2 -2048(%rdx), %ymm18
-
-// CHECK: vbroadcasti64x2 -2064(%rdx), %ymm18
-// CHECK:  encoding: [0x62,0xe2,0xfd,0x28,0x5a,0x92,0xf0,0xf7,0xff,0xff]
-          vbroadcasti64x2 -2064(%rdx), %ymm18
-
-// CHECK: vrangepd $171, %xmm26, %xmm27, %xmm25
-// CHECK:  encoding: [0x62,0x03,0xa5,0x00,0x50,0xca,0xab]
-          vrangepd $0xab, %xmm26, %xmm27, %xmm25
-
-// CHECK: vrangepd $171, %xmm26, %xmm27, %xmm25 {%k6}
-// CHECK:  encoding: [0x62,0x03,0xa5,0x06,0x50,0xca,0xab]
-          vrangepd $0xab, %xmm26, %xmm27, %xmm25 {%k6}
-
-// CHECK: vrangepd $171, %xmm26, %xmm27, %xmm25 {%k6} {z}
-// CHECK:  encoding: [0x62,0x03,0xa5,0x86,0x50,0xca,0xab]
-          vrangepd $0xab, %xmm26, %xmm27, %xmm25 {%k6} {z}
-
-// CHECK: vrangepd $123, %xmm26, %xmm27, %xmm25
-// CHECK:  encoding: [0x62,0x03,0xa5,0x00,0x50,0xca,0x7b]
-          vrangepd $0x7b, %xmm26, %xmm27, %xmm25
-
-// CHECK: vrangepd $123, (%rcx), %xmm27, %xmm25
-// CHECK:  encoding: [0x62,0x63,0xa5,0x00,0x50,0x09,0x7b]
-          vrangepd $0x7b,(%rcx), %xmm27, %xmm25
-
-// CHECK: vrangepd $123, 291(%rax,%r14,8), %xmm27, %xmm25
-// CHECK:  encoding: [0x62,0x23,0xa5,0x00,0x50,0x8c,0xf0,0x23,0x01,0x00,0x00,0x7b]
-          vrangepd $0x7b,291(%rax,%r14,8), %xmm27, %xmm25
-
-// CHECK: vrangepd $123, (%rcx){1to2}, %xmm27, %xmm25
-// CHECK:  encoding: [0x62,0x63,0xa5,0x10,0x50,0x09,0x7b]
-          vrangepd $0x7b,(%rcx){1to2}, %xmm27, %xmm25
-
-// CHECK: vrangepd $123, 2032(%rdx), %xmm27, %xmm25
-// CHECK:  encoding: [0x62,0x63,0xa5,0x00,0x50,0x4a,0x7f,0x7b]
-          vrangepd $0x7b,2032(%rdx), %xmm27, %xmm25
-
-// CHECK: vrangepd $123, 2048(%rdx), %xmm27, %xmm25
-// CHECK:  encoding: [0x62,0x63,0xa5,0x00,0x50,0x8a,0x00,0x08,0x00,0x00,0x7b]
-          vrangepd $0x7b,2048(%rdx), %xmm27, %xmm25
-
-// CHECK: vrangepd $123, -2048(%rdx), %xmm27, %xmm25
-// CHECK:  encoding: [0x62,0x63,0xa5,0x00,0x50,0x4a,0x80,0x7b]
-          vrangepd $0x7b,-2048(%rdx), %xmm27, %xmm25
-
-// CHECK: vrangepd $123, -2064(%rdx), %xmm27, %xmm25
-// CHECK:  encoding: [0x62,0x63,0xa5,0x00,0x50,0x8a,0xf0,0xf7,0xff,0xff,0x7b]
-          vrangepd $0x7b,-2064(%rdx), %xmm27, %xmm25
-
-// CHECK: vrangepd $123, 1016(%rdx){1to2}, %xmm27, %xmm25
-// CHECK:  encoding: [0x62,0x63,0xa5,0x10,0x50,0x4a,0x7f,0x7b]
-          vrangepd $0x7b,1016(%rdx){1to2}, %xmm27, %xmm25
-
-// CHECK: vrangepd $123, 1024(%rdx){1to2}, %xmm27, %xmm25
-// CHECK:  encoding: [0x62,0x63,0xa5,0x10,0x50,0x8a,0x00,0x04,0x00,0x00,0x7b]
-          vrangepd $0x7b,1024(%rdx){1to2}, %xmm27, %xmm25
-
-// CHECK: vrangepd $123, -1024(%rdx){1to2}, %xmm27, %xmm25
-// CHECK:  encoding: [0x62,0x63,0xa5,0x10,0x50,0x4a,0x80,0x7b]
-          vrangepd $0x7b,-1024(%rdx){1to2}, %xmm27, %xmm25
-
-// CHECK: vrangepd $123, -1032(%rdx){1to2}, %xmm27, %xmm25
-// CHECK:  encoding: [0x62,0x63,0xa5,0x10,0x50,0x8a,0xf8,0xfb,0xff,0xff,0x7b]
-          vrangepd $0x7b,-1032(%rdx){1to2}, %xmm27, %xmm25
-
-// CHECK: vrangepd $171, %ymm28, %ymm27, %ymm29
-// CHECK:  encoding: [0x62,0x03,0xa5,0x20,0x50,0xec,0xab]
-          vrangepd $0xab, %ymm28, %ymm27, %ymm29
-
-// CHECK: vrangepd $171, %ymm28, %ymm27, %ymm29 {%k7}
-// CHECK:  encoding: [0x62,0x03,0xa5,0x27,0x50,0xec,0xab]
-          vrangepd $0xab, %ymm28, %ymm27, %ymm29 {%k7}
-
-// CHECK: vrangepd $171, %ymm28, %ymm27, %ymm29 {%k7} {z}
-// CHECK:  encoding: [0x62,0x03,0xa5,0xa7,0x50,0xec,0xab]
-          vrangepd $0xab, %ymm28, %ymm27, %ymm29 {%k7} {z}
-
-// CHECK: vrangepd $123, %ymm28, %ymm27, %ymm29
-// CHECK:  encoding: [0x62,0x03,0xa5,0x20,0x50,0xec,0x7b]
-          vrangepd $0x7b, %ymm28, %ymm27, %ymm29
-
-// CHECK: vrangepd $123, (%rcx), %ymm27, %ymm29
-// CHECK:  encoding: [0x62,0x63,0xa5,0x20,0x50,0x29,0x7b]
-          vrangepd $0x7b,(%rcx), %ymm27, %ymm29
-
-// CHECK: vrangepd $123, 291(%rax,%r14,8), %ymm27, %ymm29
-// CHECK:  encoding: [0x62,0x23,0xa5,0x20,0x50,0xac,0xf0,0x23,0x01,0x00,0x00,0x7b]
-          vrangepd $0x7b,291(%rax,%r14,8), %ymm27, %ymm29
-
-// CHECK: vrangepd $123, (%rcx){1to4}, %ymm27, %ymm29
-// CHECK:  encoding: [0x62,0x63,0xa5,0x30,0x50,0x29,0x7b]
-          vrangepd $0x7b,(%rcx){1to4}, %ymm27, %ymm29
-
-// CHECK: vrangepd $123, 4064(%rdx), %ymm27, %ymm29
-// CHECK:  encoding: [0x62,0x63,0xa5,0x20,0x50,0x6a,0x7f,0x7b]
-          vrangepd $0x7b,4064(%rdx), %ymm27, %ymm29
-
-// CHECK: vrangepd $123, 4096(%rdx), %ymm27, %ymm29
-// CHECK:  encoding: [0x62,0x63,0xa5,0x20,0x50,0xaa,0x00,0x10,0x00,0x00,0x7b]
-          vrangepd $0x7b,4096(%rdx), %ymm27, %ymm29
-
-// CHECK: vrangepd $123, -4096(%rdx), %ymm27, %ymm29
-// CHECK:  encoding: [0x62,0x63,0xa5,0x20,0x50,0x6a,0x80,0x7b]
-          vrangepd $0x7b,-4096(%rdx), %ymm27, %ymm29
-
-// CHECK: vrangepd $123, -4128(%rdx), %ymm27, %ymm29
-// CHECK:  encoding: [0x62,0x63,0xa5,0x20,0x50,0xaa,0xe0,0xef,0xff,0xff,0x7b]
-          vrangepd $0x7b,-4128(%rdx), %ymm27, %ymm29
-
-// CHECK: vrangepd $123, 1016(%rdx){1to4}, %ymm27, %ymm29
-// CHECK:  encoding: [0x62,0x63,0xa5,0x30,0x50,0x6a,0x7f,0x7b]
-          vrangepd $0x7b,1016(%rdx){1to4}, %ymm27, %ymm29
-
-// CHECK: vrangepd $123, 1024(%rdx){1to4}, %ymm27, %ymm29
-// CHECK:  encoding: [0x62,0x63,0xa5,0x30,0x50,0xaa,0x00,0x04,0x00,0x00,0x7b]
-          vrangepd $0x7b,1024(%rdx){1to4}, %ymm27, %ymm29
-
-// CHECK: vrangepd $123, -1024(%rdx){1to4}, %ymm27, %ymm29
-// CHECK:  encoding: [0x62,0x63,0xa5,0x30,0x50,0x6a,0x80,0x7b]
-          vrangepd $0x7b,-1024(%rdx){1to4}, %ymm27, %ymm29
-
-// CHECK: vrangepd $123, -1032(%rdx){1to4}, %ymm27, %ymm29
-// CHECK:  encoding: [0x62,0x63,0xa5,0x30,0x50,0xaa,0xf8,0xfb,0xff,0xff,0x7b]
-          vrangepd $0x7b,-1032(%rdx){1to4}, %ymm27, %ymm29
-
-// CHECK: vrangeps $171, %xmm24, %xmm23, %xmm27
-// CHECK:  encoding: [0x62,0x03,0x45,0x00,0x50,0xd8,0xab]
-          vrangeps $0xab, %xmm24, %xmm23, %xmm27
-
-// CHECK: vrangeps $171, %xmm24, %xmm23, %xmm27 {%k6}
-// CHECK:  encoding: [0x62,0x03,0x45,0x06,0x50,0xd8,0xab]
-          vrangeps $0xab, %xmm24, %xmm23, %xmm27 {%k6}
-
-// CHECK: vrangeps $171, %xmm24, %xmm23, %xmm27 {%k6} {z}
-// CHECK:  encoding: [0x62,0x03,0x45,0x86,0x50,0xd8,0xab]
-          vrangeps $0xab, %xmm24, %xmm23, %xmm27 {%k6} {z}
-
-// CHECK: vrangeps $123, %xmm24, %xmm23, %xmm27
-// CHECK:  encoding: [0x62,0x03,0x45,0x00,0x50,0xd8,0x7b]
-          vrangeps $0x7b, %xmm24, %xmm23, %xmm27
-
-// CHECK: vrangeps $123, (%rcx), %xmm23, %xmm27
-// CHECK:  encoding: [0x62,0x63,0x45,0x00,0x50,0x19,0x7b]
-          vrangeps $0x7b,(%rcx), %xmm23, %xmm27
-
-// CHECK: vrangeps $123, 291(%rax,%r14,8), %xmm23, %xmm27
-// CHECK:  encoding: [0x62,0x23,0x45,0x00,0x50,0x9c,0xf0,0x23,0x01,0x00,0x00,0x7b]
-          vrangeps $0x7b,291(%rax,%r14,8), %xmm23, %xmm27
-
-// CHECK: vrangeps $123, (%rcx){1to4}, %xmm23, %xmm27
-// CHECK:  encoding: [0x62,0x63,0x45,0x10,0x50,0x19,0x7b]
-          vrangeps $0x7b,(%rcx){1to4}, %xmm23, %xmm27
-
-// CHECK: vrangeps $123, 2032(%rdx), %xmm23, %xmm27
-// CHECK:  encoding: [0x62,0x63,0x45,0x00,0x50,0x5a,0x7f,0x7b]
-          vrangeps $0x7b,2032(%rdx), %xmm23, %xmm27
-
-// CHECK: vrangeps $123, 2048(%rdx), %xmm23, %xmm27
-// CHECK:  encoding: [0x62,0x63,0x45,0x00,0x50,0x9a,0x00,0x08,0x00,0x00,0x7b]
-          vrangeps $0x7b,2048(%rdx), %xmm23, %xmm27
-
-// CHECK: vrangeps $123, -2048(%rdx), %xmm23, %xmm27
-// CHECK:  encoding: [0x62,0x63,0x45,0x00,0x50,0x5a,0x80,0x7b]
-          vrangeps $0x7b,-2048(%rdx), %xmm23, %xmm27
-
-// CHECK: vrangeps $123, -2064(%rdx), %xmm23, %xmm27
-// CHECK:  encoding: [0x62,0x63,0x45,0x00,0x50,0x9a,0xf0,0xf7,0xff,0xff,0x7b]
-          vrangeps $0x7b,-2064(%rdx), %xmm23, %xmm27
-
-// CHECK: vrangeps $123, 508(%rdx){1to4}, %xmm23, %xmm27
-// CHECK:  encoding: [0x62,0x63,0x45,0x10,0x50,0x5a,0x7f,0x7b]
-          vrangeps $0x7b,508(%rdx){1to4}, %xmm23, %xmm27
-
-// CHECK: vrangeps $123, 512(%rdx){1to4}, %xmm23, %xmm27
-// CHECK:  encoding: [0x62,0x63,0x45,0x10,0x50,0x9a,0x00,0x02,0x00,0x00,0x7b]
-          vrangeps $0x7b,512(%rdx){1to4}, %xmm23, %xmm27
-
-// CHECK: vrangeps $123, -512(%rdx){1to4}, %xmm23, %xmm27
-// CHECK:  encoding: [0x62,0x63,0x45,0x10,0x50,0x5a,0x80,0x7b]
-          vrangeps $0x7b,-512(%rdx){1to4}, %xmm23, %xmm27
-
-// CHECK: vrangeps $123, -516(%rdx){1to4}, %xmm23, %xmm27
-// CHECK:  encoding: [0x62,0x63,0x45,0x10,0x50,0x9a,0xfc,0xfd,0xff,0xff,0x7b]
-          vrangeps $0x7b,-516(%rdx){1to4}, %xmm23, %xmm27
-
-// CHECK: vrangeps $171, %ymm21, %ymm23, %ymm24
-// CHECK:  encoding: [0x62,0x23,0x45,0x20,0x50,0xc5,0xab]
-          vrangeps $0xab, %ymm21, %ymm23, %ymm24
-
-// CHECK: vrangeps $171, %ymm21, %ymm23, %ymm24 {%k7}
-// CHECK:  encoding: [0x62,0x23,0x45,0x27,0x50,0xc5,0xab]
-          vrangeps $0xab, %ymm21, %ymm23, %ymm24 {%k7}
-
-// CHECK: vrangeps $171, %ymm21, %ymm23, %ymm24 {%k7} {z}
-// CHECK:  encoding: [0x62,0x23,0x45,0xa7,0x50,0xc5,0xab]
-          vrangeps $0xab, %ymm21, %ymm23, %ymm24 {%k7} {z}
-
-// CHECK: vrangeps $123, %ymm21, %ymm23, %ymm24
-// CHECK:  encoding: [0x62,0x23,0x45,0x20,0x50,0xc5,0x7b]
-          vrangeps $0x7b, %ymm21, %ymm23, %ymm24
-
-// CHECK: vrangeps $123, (%rcx), %ymm23, %ymm24
-// CHECK:  encoding: [0x62,0x63,0x45,0x20,0x50,0x01,0x7b]
-          vrangeps $0x7b,(%rcx), %ymm23, %ymm24
-
-// CHECK: vrangeps $123, 291(%rax,%r14,8), %ymm23, %ymm24
-// CHECK:  encoding: [0x62,0x23,0x45,0x20,0x50,0x84,0xf0,0x23,0x01,0x00,0x00,0x7b]
-          vrangeps $0x7b,291(%rax,%r14,8), %ymm23, %ymm24
-
-// CHECK: vrangeps $123, (%rcx){1to8}, %ymm23, %ymm24
-// CHECK:  encoding: [0x62,0x63,0x45,0x30,0x50,0x01,0x7b]
-          vrangeps $0x7b,(%rcx){1to8}, %ymm23, %ymm24
-
-// CHECK: vrangeps $123, 4064(%rdx), %ymm23, %ymm24
-// CHECK:  encoding: [0x62,0x63,0x45,0x20,0x50,0x42,0x7f,0x7b]
-          vrangeps $0x7b,4064(%rdx), %ymm23, %ymm24
-
-// CHECK: vrangeps $123, 4096(%rdx), %ymm23, %ymm24
-// CHECK:  encoding: [0x62,0x63,0x45,0x20,0x50,0x82,0x00,0x10,0x00,0x00,0x7b]
-          vrangeps $0x7b,4096(%rdx), %ymm23, %ymm24
-
-// CHECK: vrangeps $123, -4096(%rdx), %ymm23, %ymm24
-// CHECK:  encoding: [0x62,0x63,0x45,0x20,0x50,0x42,0x80,0x7b]
-          vrangeps $0x7b,-4096(%rdx), %ymm23, %ymm24
-
-// CHECK: vrangeps $123, -4128(%rdx), %ymm23, %ymm24
-// CHECK:  encoding: [0x62,0x63,0x45,0x20,0x50,0x82,0xe0,0xef,0xff,0xff,0x7b]
-          vrangeps $0x7b,-4128(%rdx), %ymm23, %ymm24
-
-// CHECK: vrangeps $123, 508(%rdx){1to8}, %ymm23, %ymm24
-// CHECK:  encoding: [0x62,0x63,0x45,0x30,0x50,0x42,0x7f,0x7b]
-          vrangeps $0x7b,508(%rdx){1to8}, %ymm23, %ymm24
-
-// CHECK: vrangeps $123, 512(%rdx){1to8}, %ymm23, %ymm24
-// CHECK:  encoding: [0x62,0x63,0x45,0x30,0x50,0x82,0x00,0x02,0x00,0x00,0x7b]
-          vrangeps $0x7b,512(%rdx){1to8}, %ymm23, %ymm24
-
-// CHECK: vrangeps $123, -512(%rdx){1to8}, %ymm23, %ymm24
-// CHECK:  encoding: [0x62,0x63,0x45,0x30,0x50,0x42,0x80,0x7b]
-          vrangeps $0x7b,-512(%rdx){1to8}, %ymm23, %ymm24
-
-// CHECK: vrangeps $123, -516(%rdx){1to8}, %ymm23, %ymm24
-// CHECK:  encoding: [0x62,0x63,0x45,0x30,0x50,0x82,0xfc,0xfd,0xff,0xff,0x7b]
-          vrangeps $0x7b,-516(%rdx){1to8}, %ymm23, %ymm24
-
-// CHECK: vreducepd $171, %xmm17, %xmm18
-// CHECK:  encoding: [0x62,0xa3,0xfd,0x08,0x56,0xd1,0xab]
-          vreducepd $0xab, %xmm17, %xmm18
-
-// CHECK: vreducepd $171, %xmm17, %xmm18 {%k3}
-// CHECK:  encoding: [0x62,0xa3,0xfd,0x0b,0x56,0xd1,0xab]
-          vreducepd $0xab, %xmm17, %xmm18 {%k3}
-
-// CHECK: vreducepd $171, %xmm17, %xmm18 {%k3} {z}
-// CHECK:  encoding: [0x62,0xa3,0xfd,0x8b,0x56,0xd1,0xab]
-          vreducepd $0xab, %xmm17, %xmm18 {%k3} {z}
-
-// CHECK: vreducepd $123, %xmm17, %xmm18
-// CHECK:  encoding: [0x62,0xa3,0xfd,0x08,0x56,0xd1,0x7b]
-          vreducepd $0x7b, %xmm17, %xmm18
-
-// CHECK: vreducepd $123, (%rcx), %xmm18
-// CHECK:  encoding: [0x62,0xe3,0xfd,0x08,0x56,0x11,0x7b]
-          vreducepd $0x7b,(%rcx), %xmm18
-
-// CHECK: vreducepd $123, 291(%rax,%r14,8), %xmm18
-// CHECK:  encoding: [0x62,0xa3,0xfd,0x08,0x56,0x94,0xf0,0x23,0x01,0x00,0x00,0x7b]
-          vreducepd $0x7b,291(%rax,%r14,8), %xmm18
-
-// CHECK: vreducepd $171, %xmm28, %xmm25
-// CHECK:  encoding: [0x62,0x03,0xfd,0x08,0x56,0xcc,0xab]
-          vreducepd $0xab, %xmm28, %xmm25
-
-// CHECK: vreducepd $171, %xmm28, %xmm25 {%k4}
-// CHECK:  encoding: [0x62,0x03,0xfd,0x0c,0x56,0xcc,0xab]
-          vreducepd $0xab, %xmm28, %xmm25 {%k4}
-
-// CHECK: vreducepd $171, %xmm28, %xmm25 {%k4} {z}
-// CHECK:  encoding: [0x62,0x03,0xfd,0x8c,0x56,0xcc,0xab]
-          vreducepd $0xab, %xmm28, %xmm25 {%k4} {z}
-
-// CHECK: vreducepd $123, %xmm28, %xmm25
-// CHECK:  encoding: [0x62,0x03,0xfd,0x08,0x56,0xcc,0x7b]
-          vreducepd $0x7b, %xmm28, %xmm25
-
-// CHECK: vreducepd $123, (%rcx), %xmm25
-// CHECK:  encoding: [0x62,0x63,0xfd,0x08,0x56,0x09,0x7b]
-          vreducepd $0x7b,(%rcx), %xmm25
-
-// CHECK: vreducepd $123, 4660(%rax,%r14,8), %xmm25
-// CHECK:  encoding: [0x62,0x23,0xfd,0x08,0x56,0x8c,0xf0,0x34,0x12,0x00,0x00,0x7b]
-          vreducepd $0x7b,4660(%rax,%r14,8), %xmm25
-
-// CHECK: vreducepd $123, (%rcx){1to2}, %xmm25
-// CHECK:  encoding: [0x62,0x63,0xfd,0x18,0x56,0x09,0x7b]
-          vreducepd $0x7b,(%rcx){1to2}, %xmm25
-
-// CHECK: vreducepd $123, 2032(%rdx), %xmm25
-// CHECK:  encoding: [0x62,0x63,0xfd,0x08,0x56,0x4a,0x7f,0x7b]
-          vreducepd $0x7b,2032(%rdx), %xmm25
-
-// CHECK: vreducepd $123, 2048(%rdx), %xmm25
-// CHECK:  encoding: [0x62,0x63,0xfd,0x08,0x56,0x8a,0x00,0x08,0x00,0x00,0x7b]
-          vreducepd $0x7b,2048(%rdx), %xmm25
-
-// CHECK: vreducepd $123, -2048(%rdx), %xmm25
-// CHECK:  encoding: [0x62,0x63,0xfd,0x08,0x56,0x4a,0x80,0x7b]
-          vreducepd $0x7b,-2048(%rdx), %xmm25
-
-// CHECK: vreducepd $123, -2064(%rdx), %xmm25
-// CHECK:  encoding: [0x62,0x63,0xfd,0x08,0x56,0x8a,0xf0,0xf7,0xff,0xff,0x7b]
-          vreducepd $0x7b,-2064(%rdx), %xmm25
-
-// CHECK: vreducepd $123, 1016(%rdx){1to2}, %xmm25
-// CHECK:  encoding: [0x62,0x63,0xfd,0x18,0x56,0x4a,0x7f,0x7b]
-          vreducepd $0x7b,1016(%rdx){1to2}, %xmm25
-
-// CHECK: vreducepd $123, 1024(%rdx){1to2}, %xmm25
-// CHECK:  encoding: [0x62,0x63,0xfd,0x18,0x56,0x8a,0x00,0x04,0x00,0x00,0x7b]
-          vreducepd $0x7b,1024(%rdx){1to2}, %xmm25
-
-// CHECK: vreducepd $123, -1024(%rdx){1to2}, %xmm25
-// CHECK:  encoding: [0x62,0x63,0xfd,0x18,0x56,0x4a,0x80,0x7b]
-          vreducepd $0x7b,-1024(%rdx){1to2}, %xmm25
-
-// CHECK: vreducepd $123, -1032(%rdx){1to2}, %xmm25
-// CHECK:  encoding: [0x62,0x63,0xfd,0x18,0x56,0x8a,0xf8,0xfb,0xff,0xff,0x7b]
-          vreducepd $0x7b,-1032(%rdx){1to2}, %xmm25
-
-// CHECK: vreducepd $171, %ymm17, %ymm28
-// CHECK:  encoding: [0x62,0x23,0xfd,0x28,0x56,0xe1,0xab]
-          vreducepd $0xab, %ymm17, %ymm28
-
-// CHECK: vreducepd $171, %ymm17, %ymm28 {%k4}
-// CHECK:  encoding: [0x62,0x23,0xfd,0x2c,0x56,0xe1,0xab]
-          vreducepd $0xab, %ymm17, %ymm28 {%k4}
-
-// CHECK: vreducepd $171, %ymm17, %ymm28 {%k4} {z}
-// CHECK:  encoding: [0x62,0x23,0xfd,0xac,0x56,0xe1,0xab]
-          vreducepd $0xab, %ymm17, %ymm28 {%k4} {z}
-
-// CHECK: vreducepd $123, %ymm17, %ymm28
-// CHECK:  encoding: [0x62,0x23,0xfd,0x28,0x56,0xe1,0x7b]
-          vreducepd $0x7b, %ymm17, %ymm28
-
-// CHECK: vreducepd $123, (%rcx), %ymm28
-// CHECK:  encoding: [0x62,0x63,0xfd,0x28,0x56,0x21,0x7b]
-          vreducepd $0x7b,(%rcx), %ymm28
-
-// CHECK: vreducepd $123, 4660(%rax,%r14,8), %ymm28
-// CHECK:  encoding: [0x62,0x23,0xfd,0x28,0x56,0xa4,0xf0,0x34,0x12,0x00,0x00,0x7b]
-          vreducepd $0x7b,4660(%rax,%r14,8), %ymm28
-
-// CHECK: vreducepd $123, (%rcx){1to4}, %ymm28
-// CHECK:  encoding: [0x62,0x63,0xfd,0x38,0x56,0x21,0x7b]
-          vreducepd $0x7b,(%rcx){1to4}, %ymm28
-
-// CHECK: vreducepd $123, 4064(%rdx), %ymm28
-// CHECK:  encoding: [0x62,0x63,0xfd,0x28,0x56,0x62,0x7f,0x7b]
-          vreducepd $0x7b,4064(%rdx), %ymm28
-
-// CHECK: vreducepd $123, 4096(%rdx), %ymm28
-// CHECK:  encoding: [0x62,0x63,0xfd,0x28,0x56,0xa2,0x00,0x10,0x00,0x00,0x7b]
-          vreducepd $0x7b,4096(%rdx), %ymm28
-
-// CHECK: vreducepd $123, -4096(%rdx), %ymm28
-// CHECK:  encoding: [0x62,0x63,0xfd,0x28,0x56,0x62,0x80,0x7b]
-          vreducepd $0x7b,-4096(%rdx), %ymm28
-
-// CHECK: vreducepd $123, -4128(%rdx), %ymm28
-// CHECK:  encoding: [0x62,0x63,0xfd,0x28,0x56,0xa2,0xe0,0xef,0xff,0xff,0x7b]
-          vreducepd $0x7b,-4128(%rdx), %ymm28
-
-// CHECK: vreducepd $123, 1016(%rdx){1to4}, %ymm28
-// CHECK:  encoding: [0x62,0x63,0xfd,0x38,0x56,0x62,0x7f,0x7b]
-          vreducepd $0x7b,1016(%rdx){1to4}, %ymm28
-
-// CHECK: vreducepd $123, 1024(%rdx){1to4}, %ymm28
-// CHECK:  encoding: [0x62,0x63,0xfd,0x38,0x56,0xa2,0x00,0x04,0x00,0x00,0x7b]
-          vreducepd $0x7b,1024(%rdx){1to4}, %ymm28
-
-// CHECK: vreducepd $123, -1024(%rdx){1to4}, %ymm28
-// CHECK:  encoding: [0x62,0x63,0xfd,0x38,0x56,0x62,0x80,0x7b]
-          vreducepd $0x7b,-1024(%rdx){1to4}, %ymm28
-
-// CHECK: vreducepd $123, -1032(%rdx){1to4}, %ymm28
-// CHECK:  encoding: [0x62,0x63,0xfd,0x38,0x56,0xa2,0xf8,0xfb,0xff,0xff,0x7b]
-          vreducepd $0x7b,-1032(%rdx){1to4}, %ymm28
-
-// CHECK: vreduceps $171, %xmm21, %xmm29
-// CHECK:  encoding: [0x62,0x23,0x7d,0x08,0x56,0xed,0xab]
-          vreduceps $0xab, %xmm21, %xmm29
-
-// CHECK: vreduceps $171, %xmm21, %xmm29 {%k7}
-// CHECK:  encoding: [0x62,0x23,0x7d,0x0f,0x56,0xed,0xab]
-          vreduceps $0xab, %xmm21, %xmm29 {%k7}
-
-// CHECK: vreduceps $171, %xmm21, %xmm29 {%k7} {z}
-// CHECK:  encoding: [0x62,0x23,0x7d,0x8f,0x56,0xed,0xab]
-          vreduceps $0xab, %xmm21, %xmm29 {%k7} {z}
-
-// CHECK: vreduceps $123, %xmm21, %xmm29
-// CHECK:  encoding: [0x62,0x23,0x7d,0x08,0x56,0xed,0x7b]
-          vreduceps $0x7b, %xmm21, %xmm29
-
-// CHECK: vreduceps $123, (%rcx), %xmm29
-// CHECK:  encoding: [0x62,0x63,0x7d,0x08,0x56,0x29,0x7b]
-          vreduceps $0x7b,(%rcx), %xmm29
-
-// CHECK: vreduceps $123, 4660(%rax,%r14,8), %xmm29
-// CHECK:  encoding: [0x62,0x23,0x7d,0x08,0x56,0xac,0xf0,0x34,0x12,0x00,0x00,0x7b]
-          vreduceps $0x7b,4660(%rax,%r14,8), %xmm29
-
-// CHECK: vreduceps $123, (%rcx){1to4}, %xmm29
-// CHECK:  encoding: [0x62,0x63,0x7d,0x18,0x56,0x29,0x7b]
-          vreduceps $0x7b,(%rcx){1to4}, %xmm29
-
-// CHECK: vreduceps $123, 2032(%rdx), %xmm29
-// CHECK:  encoding: [0x62,0x63,0x7d,0x08,0x56,0x6a,0x7f,0x7b]
-          vreduceps $0x7b,2032(%rdx), %xmm29
-
-// CHECK: vreduceps $123, 2048(%rdx), %xmm29
-// CHECK:  encoding: [0x62,0x63,0x7d,0x08,0x56,0xaa,0x00,0x08,0x00,0x00,0x7b]
-          vreduceps $0x7b,2048(%rdx), %xmm29
-
-// CHECK: vreduceps $123, -2048(%rdx), %xmm29
-// CHECK:  encoding: [0x62,0x63,0x7d,0x08,0x56,0x6a,0x80,0x7b]
-          vreduceps $0x7b,-2048(%rdx), %xmm29
-
-// CHECK: vreduceps $123, -2064(%rdx), %xmm29
-// CHECK:  encoding: [0x62,0x63,0x7d,0x08,0x56,0xaa,0xf0,0xf7,0xff,0xff,0x7b]
-          vreduceps $0x7b,-2064(%rdx), %xmm29
-
-// CHECK: vreduceps $123, 508(%rdx){1to4}, %xmm29
-// CHECK:  encoding: [0x62,0x63,0x7d,0x18,0x56,0x6a,0x7f,0x7b]
-          vreduceps $0x7b,508(%rdx){1to4}, %xmm29
-
-// CHECK: vreduceps $123, 512(%rdx){1to4}, %xmm29
-// CHECK:  encoding: [0x62,0x63,0x7d,0x18,0x56,0xaa,0x00,0x02,0x00,0x00,0x7b]
-          vreduceps $0x7b,512(%rdx){1to4}, %xmm29
-
-// CHECK: vreduceps $123, -512(%rdx){1to4}, %xmm29
-// CHECK:  encoding: [0x62,0x63,0x7d,0x18,0x56,0x6a,0x80,0x7b]
-          vreduceps $0x7b,-512(%rdx){1to4}, %xmm29
-
-// CHECK: vreduceps $123, -516(%rdx){1to4}, %xmm29
-// CHECK:  encoding: [0x62,0x63,0x7d,0x18,0x56,0xaa,0xfc,0xfd,0xff,0xff,0x7b]
-          vreduceps $0x7b,-516(%rdx){1to4}, %xmm29
-
-// CHECK: vreduceps $171, %ymm23, %ymm25
-// CHECK:  encoding: [0x62,0x23,0x7d,0x28,0x56,0xcf,0xab]
-          vreduceps $0xab, %ymm23, %ymm25
-
-// CHECK: vreduceps $171, %ymm23, %ymm25 {%k3}
-// CHECK:  encoding: [0x62,0x23,0x7d,0x2b,0x56,0xcf,0xab]
-          vreduceps $0xab, %ymm23, %ymm25 {%k3}
-
-// CHECK: vreduceps $171, %ymm23, %ymm25 {%k3} {z}
-// CHECK:  encoding: [0x62,0x23,0x7d,0xab,0x56,0xcf,0xab]
-          vreduceps $0xab, %ymm23, %ymm25 {%k3} {z}
-
-// CHECK: vreduceps $123, %ymm23, %ymm25
-// CHECK:  encoding: [0x62,0x23,0x7d,0x28,0x56,0xcf,0x7b]
-          vreduceps $0x7b, %ymm23, %ymm25
-
-// CHECK: vreduceps $123, (%rcx), %ymm25
-// CHECK:  encoding: [0x62,0x63,0x7d,0x28,0x56,0x09,0x7b]
-          vreduceps $0x7b,(%rcx), %ymm25
-
-// CHECK: vreduceps $123, 4660(%rax,%r14,8), %ymm25
-// CHECK:  encoding: [0x62,0x23,0x7d,0x28,0x56,0x8c,0xf0,0x34,0x12,0x00,0x00,0x7b]
-          vreduceps $0x7b,4660(%rax,%r14,8), %ymm25
-
-// CHECK: vreduceps $123, (%rcx){1to8}, %ymm25
-// CHECK:  encoding: [0x62,0x63,0x7d,0x38,0x56,0x09,0x7b]
-          vreduceps $0x7b,(%rcx){1to8}, %ymm25
-
-// CHECK: vreduceps $123, 4064(%rdx), %ymm25
-// CHECK:  encoding: [0x62,0x63,0x7d,0x28,0x56,0x4a,0x7f,0x7b]
-          vreduceps $0x7b,4064(%rdx), %ymm25
-
-// CHECK: vreduceps $123, 4096(%rdx), %ymm25
-// CHECK:  encoding: [0x62,0x63,0x7d,0x28,0x56,0x8a,0x00,0x10,0x00,0x00,0x7b]
-          vreduceps $0x7b,4096(%rdx), %ymm25
-
-// CHECK: vreduceps $123, -4096(%rdx), %ymm25
-// CHECK:  encoding: [0x62,0x63,0x7d,0x28,0x56,0x4a,0x80,0x7b]
-          vreduceps $0x7b,-4096(%rdx), %ymm25
-
-// CHECK: vreduceps $123, -4128(%rdx), %ymm25
-// CHECK:  encoding: [0x62,0x63,0x7d,0x28,0x56,0x8a,0xe0,0xef,0xff,0xff,0x7b]
-          vreduceps $0x7b,-4128(%rdx), %ymm25
-
-// CHECK: vreduceps $123, 508(%rdx){1to8}, %ymm25
-// CHECK:  encoding: [0x62,0x63,0x7d,0x38,0x56,0x4a,0x7f,0x7b]
-          vreduceps $0x7b,508(%rdx){1to8}, %ymm25
-
-// CHECK: vreduceps $123, 512(%rdx){1to8}, %ymm25
-// CHECK:  encoding: [0x62,0x63,0x7d,0x38,0x56,0x8a,0x00,0x02,0x00,0x00,0x7b]
-          vreduceps $0x7b,512(%rdx){1to8}, %ymm25
-
-// CHECK: vreduceps $123, -512(%rdx){1to8}, %ymm25
-// CHECK:  encoding: [0x62,0x63,0x7d,0x38,0x56,0x4a,0x80,0x7b]
-          vreduceps $0x7b,-512(%rdx){1to8}, %ymm25
-
-// CHECK: vreduceps $123, -516(%rdx){1to8}, %ymm25
-// CHECK:  encoding: [0x62,0x63,0x7d,0x38,0x56,0x8a,0xfc,0xfd,0xff,0xff,0x7b]
-          vreduceps $0x7b,-516(%rdx){1to8}, %ymm25
-
-// CHECK: vreducepd $123, (%rcx){1to2}, %xmm18
-// CHECK:  encoding: [0x62,0xe3,0xfd,0x18,0x56,0x11,0x7b]
-          vreducepd $0x7b,(%rcx){1to2}, %xmm18
-
-// CHECK: vreducepd $123, 2032(%rdx), %xmm18
-// CHECK:  encoding: [0x62,0xe3,0xfd,0x08,0x56,0x52,0x7f,0x7b]
-          vreducepd $0x7b,2032(%rdx), %xmm18
-
-// CHECK: vreducepd $123, 2048(%rdx), %xmm18
-// CHECK:  encoding: [0x62,0xe3,0xfd,0x08,0x56,0x92,0x00,0x08,0x00,0x00,0x7b]
-          vreducepd $0x7b,2048(%rdx), %xmm18
-
-// CHECK: vreducepd $123, -2048(%rdx), %xmm18
-// CHECK:  encoding: [0x62,0xe3,0xfd,0x08,0x56,0x52,0x80,0x7b]
-          vreducepd $0x7b,-2048(%rdx), %xmm18
-
-// CHECK: vreducepd $123, -2064(%rdx), %xmm18
-// CHECK:  encoding: [0x62,0xe3,0xfd,0x08,0x56,0x92,0xf0,0xf7,0xff,0xff,0x7b]
-          vreducepd $0x7b,-2064(%rdx), %xmm18
-
-// CHECK: vreducepd $123, 1016(%rdx){1to2}, %xmm18
-// CHECK:  encoding: [0x62,0xe3,0xfd,0x18,0x56,0x52,0x7f,0x7b]
-          vreducepd $0x7b,1016(%rdx){1to2}, %xmm18
-
-// CHECK: vreducepd $123, 1024(%rdx){1to2}, %xmm18
-// CHECK:  encoding: [0x62,0xe3,0xfd,0x18,0x56,0x92,0x00,0x04,0x00,0x00,0x7b]
-          vreducepd $0x7b,1024(%rdx){1to2}, %xmm18
-
-// CHECK: vreducepd $123, -1024(%rdx){1to2}, %xmm18
-// CHECK:  encoding: [0x62,0xe3,0xfd,0x18,0x56,0x52,0x80,0x7b]
-          vreducepd $0x7b,-1024(%rdx){1to2}, %xmm18
-
-// CHECK: vreducepd $123, -1032(%rdx){1to2}, %xmm18
-// CHECK:  encoding: [0x62,0xe3,0xfd,0x18,0x56,0x92,0xf8,0xfb,0xff,0xff,0x7b]
-          vreducepd $0x7b,-1032(%rdx){1to2}, %xmm18
-
-// CHECK: vreducepd $171, %ymm29, %ymm25
-// CHECK:  encoding: [0x62,0x03,0xfd,0x28,0x56,0xcd,0xab]
-          vreducepd $0xab, %ymm29, %ymm25
-
-// CHECK: vreducepd $171, %ymm29, %ymm25 {%k1}
-// CHECK:  encoding: [0x62,0x03,0xfd,0x29,0x56,0xcd,0xab]
-          vreducepd $0xab, %ymm29, %ymm25 {%k1}
-
-// CHECK: vreducepd $171, %ymm29, %ymm25 {%k1} {z}
-// CHECK:  encoding: [0x62,0x03,0xfd,0xa9,0x56,0xcd,0xab]
-          vreducepd $0xab, %ymm29, %ymm25 {%k1} {z}
-
-// CHECK: vreducepd $123, %ymm29, %ymm25
-// CHECK:  encoding: [0x62,0x03,0xfd,0x28,0x56,0xcd,0x7b]
-          vreducepd $0x7b, %ymm29, %ymm25
-
-// CHECK: vreducepd $123, (%rcx), %ymm25
-// CHECK:  encoding: [0x62,0x63,0xfd,0x28,0x56,0x09,0x7b]
-          vreducepd $0x7b,(%rcx), %ymm25
-
-// CHECK: vreducepd $123, 291(%rax,%r14,8), %ymm25
-// CHECK:  encoding: [0x62,0x23,0xfd,0x28,0x56,0x8c,0xf0,0x23,0x01,0x00,0x00,0x7b]
-          vreducepd $0x7b,291(%rax,%r14,8), %ymm25
-
-// CHECK: vreducepd $123, (%rcx){1to4}, %ymm25
-// CHECK:  encoding: [0x62,0x63,0xfd,0x38,0x56,0x09,0x7b]
-          vreducepd $0x7b,(%rcx){1to4}, %ymm25
-
-// CHECK: vreducepd $123, 4064(%rdx), %ymm25
-// CHECK:  encoding: [0x62,0x63,0xfd,0x28,0x56,0x4a,0x7f,0x7b]
-          vreducepd $0x7b,4064(%rdx), %ymm25
-
-// CHECK: vreducepd $123, 4096(%rdx), %ymm25
-// CHECK:  encoding: [0x62,0x63,0xfd,0x28,0x56,0x8a,0x00,0x10,0x00,0x00,0x7b]
-          vreducepd $0x7b,4096(%rdx), %ymm25
-
-// CHECK: vreducepd $123, -4096(%rdx), %ymm25
-// CHECK:  encoding: [0x62,0x63,0xfd,0x28,0x56,0x4a,0x80,0x7b]
-          vreducepd $0x7b,-4096(%rdx), %ymm25
-
-// CHECK: vreducepd $123, -4128(%rdx), %ymm25
-// CHECK:  encoding: [0x62,0x63,0xfd,0x28,0x56,0x8a,0xe0,0xef,0xff,0xff,0x7b]
-          vreducepd $0x7b,-4128(%rdx), %ymm25
-
-// CHECK: vreducepd $123, 1016(%rdx){1to4}, %ymm25
-// CHECK:  encoding: [0x62,0x63,0xfd,0x38,0x56,0x4a,0x7f,0x7b]
-          vreducepd $0x7b,1016(%rdx){1to4}, %ymm25
-
-// CHECK: vreducepd $123, 1024(%rdx){1to4}, %ymm25
-// CHECK:  encoding: [0x62,0x63,0xfd,0x38,0x56,0x8a,0x00,0x04,0x00,0x00,0x7b]
-          vreducepd $0x7b,1024(%rdx){1to4}, %ymm25
-
-// CHECK: vreducepd $123, -1024(%rdx){1to4}, %ymm25
-// CHECK:  encoding: [0x62,0x63,0xfd,0x38,0x56,0x4a,0x80,0x7b]
-          vreducepd $0x7b,-1024(%rdx){1to4}, %ymm25
-
-// CHECK: vreducepd $123, -1032(%rdx){1to4}, %ymm25
-// CHECK:  encoding: [0x62,0x63,0xfd,0x38,0x56,0x8a,0xf8,0xfb,0xff,0xff,0x7b]
-          vreducepd $0x7b,-1032(%rdx){1to4}, %ymm25
-
-// CHECK: vreduceps $171, %xmm23, %xmm20
-// CHECK:  encoding: [0x62,0xa3,0x7d,0x08,0x56,0xe7,0xab]
-          vreduceps $0xab, %xmm23, %xmm20
-
-// CHECK: vreduceps $171, %xmm23, %xmm20 {%k7}
-// CHECK:  encoding: [0x62,0xa3,0x7d,0x0f,0x56,0xe7,0xab]
-          vreduceps $0xab, %xmm23, %xmm20 {%k7}
-
-// CHECK: vreduceps $171, %xmm23, %xmm20 {%k7} {z}
-// CHECK:  encoding: [0x62,0xa3,0x7d,0x8f,0x56,0xe7,0xab]
-          vreduceps $0xab, %xmm23, %xmm20 {%k7} {z}
-
-// CHECK: vreduceps $123, %xmm23, %xmm20
-// CHECK:  encoding: [0x62,0xa3,0x7d,0x08,0x56,0xe7,0x7b]
-          vreduceps $0x7b, %xmm23, %xmm20
-
-// CHECK: vreduceps $123, (%rcx), %xmm20
-// CHECK:  encoding: [0x62,0xe3,0x7d,0x08,0x56,0x21,0x7b]
-          vreduceps $0x7b,(%rcx), %xmm20
-
-// CHECK: vreduceps $123, 291(%rax,%r14,8), %xmm20
-// CHECK:  encoding: [0x62,0xa3,0x7d,0x08,0x56,0xa4,0xf0,0x23,0x01,0x00,0x00,0x7b]
-          vreduceps $0x7b,291(%rax,%r14,8), %xmm20
-
-// CHECK: vreduceps $123, (%rcx){1to4}, %xmm20
-// CHECK:  encoding: [0x62,0xe3,0x7d,0x18,0x56,0x21,0x7b]
-          vreduceps $0x7b,(%rcx){1to4}, %xmm20
-
-// CHECK: vreduceps $123, 2032(%rdx), %xmm20
-// CHECK:  encoding: [0x62,0xe3,0x7d,0x08,0x56,0x62,0x7f,0x7b]
-          vreduceps $0x7b,2032(%rdx), %xmm20
-
-// CHECK: vreduceps $123, 2048(%rdx), %xmm20
-// CHECK:  encoding: [0x62,0xe3,0x7d,0x08,0x56,0xa2,0x00,0x08,0x00,0x00,0x7b]
-          vreduceps $0x7b,2048(%rdx), %xmm20
-
-// CHECK: vreduceps $123, -2048(%rdx), %xmm20
-// CHECK:  encoding: [0x62,0xe3,0x7d,0x08,0x56,0x62,0x80,0x7b]
-          vreduceps $0x7b,-2048(%rdx), %xmm20
-
-// CHECK: vreduceps $123, -2064(%rdx), %xmm20
-// CHECK:  encoding: [0x62,0xe3,0x7d,0x08,0x56,0xa2,0xf0,0xf7,0xff,0xff,0x7b]
-          vreduceps $0x7b,-2064(%rdx), %xmm20
-
-// CHECK: vreduceps $123, 508(%rdx){1to4}, %xmm20
-// CHECK:  encoding: [0x62,0xe3,0x7d,0x18,0x56,0x62,0x7f,0x7b]
-          vreduceps $0x7b,508(%rdx){1to4}, %xmm20
-
-// CHECK: vreduceps $123, 512(%rdx){1to4}, %xmm20
-// CHECK:  encoding: [0x62,0xe3,0x7d,0x18,0x56,0xa2,0x00,0x02,0x00,0x00,0x7b]
-          vreduceps $0x7b,512(%rdx){1to4}, %xmm20
-
-// CHECK: vreduceps $123, -512(%rdx){1to4}, %xmm20
-// CHECK:  encoding: [0x62,0xe3,0x7d,0x18,0x56,0x62,0x80,0x7b]
-          vreduceps $0x7b,-512(%rdx){1to4}, %xmm20
-
-// CHECK: vreduceps $123, -516(%rdx){1to4}, %xmm20
-// CHECK:  encoding: [0x62,0xe3,0x7d,0x18,0x56,0xa2,0xfc,0xfd,0xff,0xff,0x7b]
-          vreduceps $0x7b,-516(%rdx){1to4}, %xmm20
-
-// CHECK: vreduceps $171, %ymm22, %ymm26
-// CHECK:  encoding: [0x62,0x23,0x7d,0x28,0x56,0xd6,0xab]
-          vreduceps $0xab, %ymm22, %ymm26
-
-// CHECK: vreduceps $171, %ymm22, %ymm26 {%k6}
-// CHECK:  encoding: [0x62,0x23,0x7d,0x2e,0x56,0xd6,0xab]
-          vreduceps $0xab, %ymm22, %ymm26 {%k6}
-
-// CHECK: vreduceps $171, %ymm22, %ymm26 {%k6} {z}
-// CHECK:  encoding: [0x62,0x23,0x7d,0xae,0x56,0xd6,0xab]
-          vreduceps $0xab, %ymm22, %ymm26 {%k6} {z}
-
-// CHECK: vreduceps $123, %ymm22, %ymm26
-// CHECK:  encoding: [0x62,0x23,0x7d,0x28,0x56,0xd6,0x7b]
-          vreduceps $0x7b, %ymm22, %ymm26
-
-// CHECK: vreduceps $123, (%rcx), %ymm26
-// CHECK:  encoding: [0x62,0x63,0x7d,0x28,0x56,0x11,0x7b]
-          vreduceps $0x7b,(%rcx), %ymm26
-
-// CHECK: vreduceps $123, 291(%rax,%r14,8), %ymm26
-// CHECK:  encoding: [0x62,0x23,0x7d,0x28,0x56,0x94,0xf0,0x23,0x01,0x00,0x00,0x7b]
-          vreduceps $0x7b,291(%rax,%r14,8), %ymm26
-
-// CHECK: vreduceps $123, (%rcx){1to8}, %ymm26
-// CHECK:  encoding: [0x62,0x63,0x7d,0x38,0x56,0x11,0x7b]
-          vreduceps $0x7b,(%rcx){1to8}, %ymm26
-
-// CHECK: vreduceps $123, 4064(%rdx), %ymm26
-// CHECK:  encoding: [0x62,0x63,0x7d,0x28,0x56,0x52,0x7f,0x7b]
-          vreduceps $0x7b,4064(%rdx), %ymm26
-
-// CHECK: vreduceps $123, 4096(%rdx), %ymm26
-// CHECK:  encoding: [0x62,0x63,0x7d,0x28,0x56,0x92,0x00,0x10,0x00,0x00,0x7b]
-          vreduceps $0x7b,4096(%rdx), %ymm26
-
-// CHECK: vreduceps $123, -4096(%rdx), %ymm26
-// CHECK:  encoding: [0x62,0x63,0x7d,0x28,0x56,0x52,0x80,0x7b]
-          vreduceps $0x7b,-4096(%rdx), %ymm26
-
-// CHECK: vreduceps $123, -4128(%rdx), %ymm26
-// CHECK:  encoding: [0x62,0x63,0x7d,0x28,0x56,0x92,0xe0,0xef,0xff,0xff,0x7b]
-          vreduceps $0x7b,-4128(%rdx), %ymm26
-
-// CHECK: vreduceps $123, 508(%rdx){1to8}, %ymm26
-// CHECK:  encoding: [0x62,0x63,0x7d,0x38,0x56,0x52,0x7f,0x7b]
-          vreduceps $0x7b,508(%rdx){1to8}, %ymm26
-
-// CHECK: vreduceps $123, 512(%rdx){1to8}, %ymm26
-// CHECK:  encoding: [0x62,0x63,0x7d,0x38,0x56,0x92,0x00,0x02,0x00,0x00,0x7b]
-          vreduceps $0x7b,512(%rdx){1to8}, %ymm26
-
-// CHECK: vreduceps $123, -512(%rdx){1to8}, %ymm26
-// CHECK:  encoding: [0x62,0x63,0x7d,0x38,0x56,0x52,0x80,0x7b]
-          vreduceps $0x7b,-512(%rdx){1to8}, %ymm26
-
-// CHECK: vreduceps $123, -516(%rdx){1to8}, %ymm26
-// CHECK:  encoding: [0x62,0x63,0x7d,0x38,0x56,0x92,0xfc,0xfd,0xff,0xff,0x7b]
-          vreduceps $0x7b,-516(%rdx){1to8}, %ymm26
-
-// CHECK: vcvtpd2qq %xmm22, %xmm24
-// CHECK:  encoding: [0x62,0x21,0xfd,0x08,0x7b,0xc6]
-          vcvtpd2qq %xmm22, %xmm24
-
-// CHECK: vcvtpd2qq %xmm22, %xmm24 {%k6}
-// CHECK:  encoding: [0x62,0x21,0xfd,0x0e,0x7b,0xc6]
-          vcvtpd2qq %xmm22, %xmm24 {%k6}
-
-// CHECK: vcvtpd2qq %xmm22, %xmm24 {%k6} {z}
-// CHECK:  encoding: [0x62,0x21,0xfd,0x8e,0x7b,0xc6]
-          vcvtpd2qq %xmm22, %xmm24 {%k6} {z}
-
-// CHECK: vcvtpd2qq (%rcx), %xmm24
-// CHECK:  encoding: [0x62,0x61,0xfd,0x08,0x7b,0x01]
-          vcvtpd2qq (%rcx), %xmm24
-
-// CHECK: vcvtpd2qq 291(%rax,%r14,8), %xmm24
-// CHECK:  encoding: [0x62,0x21,0xfd,0x08,0x7b,0x84,0xf0,0x23,0x01,0x00,0x00]
-          vcvtpd2qq 291(%rax,%r14,8), %xmm24
-
-// CHECK: vcvtpd2qq (%rcx){1to2}, %xmm24
-// CHECK:  encoding: [0x62,0x61,0xfd,0x18,0x7b,0x01]
-          vcvtpd2qq (%rcx){1to2}, %xmm24
-
-// CHECK: vcvtpd2qq 2032(%rdx), %xmm24
-// CHECK:  encoding: [0x62,0x61,0xfd,0x08,0x7b,0x42,0x7f]
-          vcvtpd2qq 2032(%rdx), %xmm24
-
-// CHECK: vcvtpd2qq 2048(%rdx), %xmm24
-// CHECK:  encoding: [0x62,0x61,0xfd,0x08,0x7b,0x82,0x00,0x08,0x00,0x00]
-          vcvtpd2qq 2048(%rdx), %xmm24
-
-// CHECK: vcvtpd2qq -2048(%rdx), %xmm24
-// CHECK:  encoding: [0x62,0x61,0xfd,0x08,0x7b,0x42,0x80]
-          vcvtpd2qq -2048(%rdx), %xmm24
-
-// CHECK: vcvtpd2qq -2064(%rdx), %xmm24
-// CHECK:  encoding: [0x62,0x61,0xfd,0x08,0x7b,0x82,0xf0,0xf7,0xff,0xff]
-          vcvtpd2qq -2064(%rdx), %xmm24
-
-// CHECK: vcvtpd2qq 1016(%rdx){1to2}, %xmm24
-// CHECK:  encoding: [0x62,0x61,0xfd,0x18,0x7b,0x42,0x7f]
-          vcvtpd2qq 1016(%rdx){1to2}, %xmm24
-
-// CHECK: vcvtpd2qq 1024(%rdx){1to2}, %xmm24
-// CHECK:  encoding: [0x62,0x61,0xfd,0x18,0x7b,0x82,0x00,0x04,0x00,0x00]
-          vcvtpd2qq 1024(%rdx){1to2}, %xmm24
-
-// CHECK: vcvtpd2qq -1024(%rdx){1to2}, %xmm24
-// CHECK:  encoding: [0x62,0x61,0xfd,0x18,0x7b,0x42,0x80]
-          vcvtpd2qq -1024(%rdx){1to2}, %xmm24
-
-// CHECK: vcvtpd2qq -1032(%rdx){1to2}, %xmm24
-// CHECK:  encoding: [0x62,0x61,0xfd,0x18,0x7b,0x82,0xf8,0xfb,0xff,0xff]
-          vcvtpd2qq -1032(%rdx){1to2}, %xmm24
-
-// CHECK: vcvtpd2qq %ymm27, %ymm24
-// CHECK:  encoding: [0x62,0x01,0xfd,0x28,0x7b,0xc3]
-          vcvtpd2qq %ymm27, %ymm24
-
-// CHECK: vcvtpd2qq %ymm27, %ymm24 {%k7}
-// CHECK:  encoding: [0x62,0x01,0xfd,0x2f,0x7b,0xc3]
-          vcvtpd2qq %ymm27, %ymm24 {%k7}
-
-// CHECK: vcvtpd2qq %ymm27, %ymm24 {%k7} {z}
-// CHECK:  encoding: [0x62,0x01,0xfd,0xaf,0x7b,0xc3]
-          vcvtpd2qq %ymm27, %ymm24 {%k7} {z}
-
-// CHECK: vcvtpd2qq (%rcx), %ymm24
-// CHECK:  encoding: [0x62,0x61,0xfd,0x28,0x7b,0x01]
-          vcvtpd2qq (%rcx), %ymm24
-
-// CHECK: vcvtpd2qq 291(%rax,%r14,8), %ymm24
-// CHECK:  encoding: [0x62,0x21,0xfd,0x28,0x7b,0x84,0xf0,0x23,0x01,0x00,0x00]
-          vcvtpd2qq 291(%rax,%r14,8), %ymm24
-
-// CHECK: vcvtpd2qq (%rcx){1to4}, %ymm24
-// CHECK:  encoding: [0x62,0x61,0xfd,0x38,0x7b,0x01]
-          vcvtpd2qq (%rcx){1to4}, %ymm24
-
-// CHECK: vcvtpd2qq 4064(%rdx), %ymm24
-// CHECK:  encoding: [0x62,0x61,0xfd,0x28,0x7b,0x42,0x7f]
-          vcvtpd2qq 4064(%rdx), %ymm24
-
-// CHECK: vcvtpd2qq 4096(%rdx), %ymm24
-// CHECK:  encoding: [0x62,0x61,0xfd,0x28,0x7b,0x82,0x00,0x10,0x00,0x00]
-          vcvtpd2qq 4096(%rdx), %ymm24
-
-// CHECK: vcvtpd2qq -4096(%rdx), %ymm24
-// CHECK:  encoding: [0x62,0x61,0xfd,0x28,0x7b,0x42,0x80]
-          vcvtpd2qq -4096(%rdx), %ymm24
-
-// CHECK: vcvtpd2qq -4128(%rdx), %ymm24
-// CHECK:  encoding: [0x62,0x61,0xfd,0x28,0x7b,0x82,0xe0,0xef,0xff,0xff]
-          vcvtpd2qq -4128(%rdx), %ymm24
-
-// CHECK: vcvtpd2qq 1016(%rdx){1to4}, %ymm24
-// CHECK:  encoding: [0x62,0x61,0xfd,0x38,0x7b,0x42,0x7f]
-          vcvtpd2qq 1016(%rdx){1to4}, %ymm24
-
-// CHECK: vcvtpd2qq 1024(%rdx){1to4}, %ymm24
-// CHECK:  encoding: [0x62,0x61,0xfd,0x38,0x7b,0x82,0x00,0x04,0x00,0x00]
-          vcvtpd2qq 1024(%rdx){1to4}, %ymm24
-
-// CHECK: vcvtpd2qq -1024(%rdx){1to4}, %ymm24
-// CHECK:  encoding: [0x62,0x61,0xfd,0x38,0x7b,0x42,0x80]
-          vcvtpd2qq -1024(%rdx){1to4}, %ymm24
-
-// CHECK: vcvtpd2qq -1032(%rdx){1to4}, %ymm24
-// CHECK:  encoding: [0x62,0x61,0xfd,0x38,0x7b,0x82,0xf8,0xfb,0xff,0xff]
-          vcvtpd2qq -1032(%rdx){1to4}, %ymm24
-
-// CHECK: vcvtpd2uqq %xmm20, %xmm22
-// CHECK:  encoding: [0x62,0xa1,0xfd,0x08,0x79,0xf4]
-          vcvtpd2uqq %xmm20, %xmm22
-
-// CHECK: vcvtpd2uqq %xmm20, %xmm22 {%k3}
-// CHECK:  encoding: [0x62,0xa1,0xfd,0x0b,0x79,0xf4]
-          vcvtpd2uqq %xmm20, %xmm22 {%k3}
-
-// CHECK: vcvtpd2uqq %xmm20, %xmm22 {%k3} {z}
-// CHECK:  encoding: [0x62,0xa1,0xfd,0x8b,0x79,0xf4]
-          vcvtpd2uqq %xmm20, %xmm22 {%k3} {z}
-
-// CHECK: vcvtpd2uqq (%rcx), %xmm22
-// CHECK:  encoding: [0x62,0xe1,0xfd,0x08,0x79,0x31]
-          vcvtpd2uqq (%rcx), %xmm22
-
-// CHECK: vcvtpd2uqq 291(%rax,%r14,8), %xmm22
-// CHECK:  encoding: [0x62,0xa1,0xfd,0x08,0x79,0xb4,0xf0,0x23,0x01,0x00,0x00]
-          vcvtpd2uqq 291(%rax,%r14,8), %xmm22
-
-// CHECK: vcvtpd2uqq (%rcx){1to2}, %xmm22
-// CHECK:  encoding: [0x62,0xe1,0xfd,0x18,0x79,0x31]
-          vcvtpd2uqq (%rcx){1to2}, %xmm22
-
-// CHECK: vcvtpd2uqq 2032(%rdx), %xmm22
-// CHECK:  encoding: [0x62,0xe1,0xfd,0x08,0x79,0x72,0x7f]
-          vcvtpd2uqq 2032(%rdx), %xmm22
-
-// CHECK: vcvtpd2uqq 2048(%rdx), %xmm22
-// CHECK:  encoding: [0x62,0xe1,0xfd,0x08,0x79,0xb2,0x00,0x08,0x00,0x00]
-          vcvtpd2uqq 2048(%rdx), %xmm22
-
-// CHECK: vcvtpd2uqq -2048(%rdx), %xmm22
-// CHECK:  encoding: [0x62,0xe1,0xfd,0x08,0x79,0x72,0x80]
-          vcvtpd2uqq -2048(%rdx), %xmm22
-
-// CHECK: vcvtpd2uqq -2064(%rdx), %xmm22
-// CHECK:  encoding: [0x62,0xe1,0xfd,0x08,0x79,0xb2,0xf0,0xf7,0xff,0xff]
-          vcvtpd2uqq -2064(%rdx), %xmm22
-
-// CHECK: vcvtpd2uqq 1016(%rdx){1to2}, %xmm22
-// CHECK:  encoding: [0x62,0xe1,0xfd,0x18,0x79,0x72,0x7f]
-          vcvtpd2uqq 1016(%rdx){1to2}, %xmm22
-
-// CHECK: vcvtpd2uqq 1024(%rdx){1to2}, %xmm22
-// CHECK:  encoding: [0x62,0xe1,0xfd,0x18,0x79,0xb2,0x00,0x04,0x00,0x00]
-          vcvtpd2uqq 1024(%rdx){1to2}, %xmm22
-
-// CHECK: vcvtpd2uqq -1024(%rdx){1to2}, %xmm22
-// CHECK:  encoding: [0x62,0xe1,0xfd,0x18,0x79,0x72,0x80]
-          vcvtpd2uqq -1024(%rdx){1to2}, %xmm22
-
-// CHECK: vcvtpd2uqq -1032(%rdx){1to2}, %xmm22
-// CHECK:  encoding: [0x62,0xe1,0xfd,0x18,0x79,0xb2,0xf8,0xfb,0xff,0xff]
-          vcvtpd2uqq -1032(%rdx){1to2}, %xmm22
-
-// CHECK: vcvtpd2uqq %ymm24, %ymm21
-// CHECK:  encoding: [0x62,0x81,0xfd,0x28,0x79,0xe8]
-          vcvtpd2uqq %ymm24, %ymm21
-
-// CHECK: vcvtpd2uqq %ymm24, %ymm21 {%k6}
-// CHECK:  encoding: [0x62,0x81,0xfd,0x2e,0x79,0xe8]
-          vcvtpd2uqq %ymm24, %ymm21 {%k6}
-
-// CHECK: vcvtpd2uqq %ymm24, %ymm21 {%k6} {z}
-// CHECK:  encoding: [0x62,0x81,0xfd,0xae,0x79,0xe8]
-          vcvtpd2uqq %ymm24, %ymm21 {%k6} {z}
-
-// CHECK: vcvtpd2uqq (%rcx), %ymm21
-// CHECK:  encoding: [0x62,0xe1,0xfd,0x28,0x79,0x29]
-          vcvtpd2uqq (%rcx), %ymm21
-
-// CHECK: vcvtpd2uqq 291(%rax,%r14,8), %ymm21
-// CHECK:  encoding: [0x62,0xa1,0xfd,0x28,0x79,0xac,0xf0,0x23,0x01,0x00,0x00]
-          vcvtpd2uqq 291(%rax,%r14,8), %ymm21
-
-// CHECK: vcvtpd2uqq (%rcx){1to4}, %ymm21
-// CHECK:  encoding: [0x62,0xe1,0xfd,0x38,0x79,0x29]
-          vcvtpd2uqq (%rcx){1to4}, %ymm21
-
-// CHECK: vcvtpd2uqq 4064(%rdx), %ymm21
-// CHECK:  encoding: [0x62,0xe1,0xfd,0x28,0x79,0x6a,0x7f]
-          vcvtpd2uqq 4064(%rdx), %ymm21
-
-// CHECK: vcvtpd2uqq 4096(%rdx), %ymm21
-// CHECK:  encoding: [0x62,0xe1,0xfd,0x28,0x79,0xaa,0x00,0x10,0x00,0x00]
-          vcvtpd2uqq 4096(%rdx), %ymm21
-
-// CHECK: vcvtpd2uqq -4096(%rdx), %ymm21
-// CHECK:  encoding: [0x62,0xe1,0xfd,0x28,0x79,0x6a,0x80]
-          vcvtpd2uqq -4096(%rdx), %ymm21
-
-// CHECK: vcvtpd2uqq -4128(%rdx), %ymm21
-// CHECK:  encoding: [0x62,0xe1,0xfd,0x28,0x79,0xaa,0xe0,0xef,0xff,0xff]
-          vcvtpd2uqq -4128(%rdx), %ymm21
-
-// CHECK: vcvtpd2uqq 1016(%rdx){1to4}, %ymm21
-// CHECK:  encoding: [0x62,0xe1,0xfd,0x38,0x79,0x6a,0x7f]
-          vcvtpd2uqq 1016(%rdx){1to4}, %ymm21
-
-// CHECK: vcvtpd2uqq 1024(%rdx){1to4}, %ymm21
-// CHECK:  encoding: [0x62,0xe1,0xfd,0x38,0x79,0xaa,0x00,0x04,0x00,0x00]
-          vcvtpd2uqq 1024(%rdx){1to4}, %ymm21
-
-// CHECK: vcvtpd2uqq -1024(%rdx){1to4}, %ymm21
-// CHECK:  encoding: [0x62,0xe1,0xfd,0x38,0x79,0x6a,0x80]
-          vcvtpd2uqq -1024(%rdx){1to4}, %ymm21
-
-// CHECK: vcvtpd2uqq -1032(%rdx){1to4}, %ymm21
-// CHECK:  encoding: [0x62,0xe1,0xfd,0x38,0x79,0xaa,0xf8,0xfb,0xff,0xff]
-          vcvtpd2uqq -1032(%rdx){1to4}, %ymm21
-
-// CHECK: vcvtps2qq %xmm28, %xmm17
-// CHECK:  encoding: [0x62,0x81,0x7d,0x08,0x7b,0xcc]
-          vcvtps2qq %xmm28, %xmm17
-
-// CHECK: vcvtps2qq %xmm28, %xmm17 {%k4}
-// CHECK:  encoding: [0x62,0x81,0x7d,0x0c,0x7b,0xcc]
-          vcvtps2qq %xmm28, %xmm17 {%k4}
-
-// CHECK: vcvtps2qq %xmm28, %xmm17 {%k4} {z}
-// CHECK:  encoding: [0x62,0x81,0x7d,0x8c,0x7b,0xcc]
-          vcvtps2qq %xmm28, %xmm17 {%k4} {z}
-
-// CHECK: vcvtps2qq (%rcx), %xmm17
-// CHECK:  encoding: [0x62,0xe1,0x7d,0x08,0x7b,0x09]
-          vcvtps2qq (%rcx), %xmm17
-
-// CHECK: vcvtps2qq 291(%rax,%r14,8), %xmm17
-// CHECK:  encoding: [0x62,0xa1,0x7d,0x08,0x7b,0x8c,0xf0,0x23,0x01,0x00,0x00]
-          vcvtps2qq 291(%rax,%r14,8), %xmm17
-
-// CHECK: vcvtps2qq (%rcx){1to2}, %xmm17
-// CHECK:  encoding: [0x62,0xe1,0x7d,0x18,0x7b,0x09]
-          vcvtps2qq (%rcx){1to2}, %xmm17
-
-// CHECK: vcvtps2qq 1016(%rdx), %xmm17
-// CHECK:  encoding: [0x62,0xe1,0x7d,0x08,0x7b,0x4a,0x7f]
-          vcvtps2qq 1016(%rdx), %xmm17
-
-// CHECK: vcvtps2qq 1024(%rdx), %xmm17
-// CHECK:  encoding: [0x62,0xe1,0x7d,0x08,0x7b,0x8a,0x00,0x04,0x00,0x00]
-          vcvtps2qq 1024(%rdx), %xmm17
-
-// CHECK: vcvtps2qq -1024(%rdx), %xmm17
-// CHECK:  encoding: [0x62,0xe1,0x7d,0x08,0x7b,0x4a,0x80]
-          vcvtps2qq -1024(%rdx), %xmm17
-
-// CHECK: vcvtps2qq -1032(%rdx), %xmm17
-// CHECK:  encoding: [0x62,0xe1,0x7d,0x08,0x7b,0x8a,0xf8,0xfb,0xff,0xff]
-          vcvtps2qq -1032(%rdx), %xmm17
-
-// CHECK: vcvtps2qq 508(%rdx){1to2}, %xmm17
-// CHECK:  encoding: [0x62,0xe1,0x7d,0x18,0x7b,0x4a,0x7f]
-          vcvtps2qq 508(%rdx){1to2}, %xmm17
-
-// CHECK: vcvtps2qq 512(%rdx){1to2}, %xmm17
-// CHECK:  encoding: [0x62,0xe1,0x7d,0x18,0x7b,0x8a,0x00,0x02,0x00,0x00]
-          vcvtps2qq 512(%rdx){1to2}, %xmm17
-
-// CHECK: vcvtps2qq -512(%rdx){1to2}, %xmm17
-// CHECK:  encoding: [0x62,0xe1,0x7d,0x18,0x7b,0x4a,0x80]
-          vcvtps2qq -512(%rdx){1to2}, %xmm17
-
-// CHECK: vcvtps2qq -516(%rdx){1to2}, %xmm17
-// CHECK:  encoding: [0x62,0xe1,0x7d,0x18,0x7b,0x8a,0xfc,0xfd,0xff,0xff]
-          vcvtps2qq -516(%rdx){1to2}, %xmm17
-
-// CHECK: vcvtps2qq %xmm27, %ymm25
-// CHECK:  encoding: [0x62,0x01,0x7d,0x28,0x7b,0xcb]
-          vcvtps2qq %xmm27, %ymm25
-
-// CHECK: vcvtps2qq %xmm27, %ymm25 {%k7}
-// CHECK:  encoding: [0x62,0x01,0x7d,0x2f,0x7b,0xcb]
-          vcvtps2qq %xmm27, %ymm25 {%k7}
-
-// CHECK: vcvtps2qq %xmm27, %ymm25 {%k7} {z}
-// CHECK:  encoding: [0x62,0x01,0x7d,0xaf,0x7b,0xcb]
-          vcvtps2qq %xmm27, %ymm25 {%k7} {z}
-
-// CHECK: vcvtps2qq (%rcx), %ymm25
-// CHECK:  encoding: [0x62,0x61,0x7d,0x28,0x7b,0x09]
-          vcvtps2qq (%rcx), %ymm25
-
-// CHECK: vcvtps2qq 291(%rax,%r14,8), %ymm25
-// CHECK:  encoding: [0x62,0x21,0x7d,0x28,0x7b,0x8c,0xf0,0x23,0x01,0x00,0x00]
-          vcvtps2qq 291(%rax,%r14,8), %ymm25
-
-// CHECK: vcvtps2qq (%rcx){1to4}, %ymm25
-// CHECK:  encoding: [0x62,0x61,0x7d,0x38,0x7b,0x09]
-          vcvtps2qq (%rcx){1to4}, %ymm25
-
-// CHECK: vcvtps2qq 2032(%rdx), %ymm25
-// CHECK:  encoding: [0x62,0x61,0x7d,0x28,0x7b,0x4a,0x7f]
-          vcvtps2qq 2032(%rdx), %ymm25
-
-// CHECK: vcvtps2qq 2048(%rdx), %ymm25
-// CHECK:  encoding: [0x62,0x61,0x7d,0x28,0x7b,0x8a,0x00,0x08,0x00,0x00]
-          vcvtps2qq 2048(%rdx), %ymm25
-
-// CHECK: vcvtps2qq -2048(%rdx), %ymm25
-// CHECK:  encoding: [0x62,0x61,0x7d,0x28,0x7b,0x4a,0x80]
-          vcvtps2qq -2048(%rdx), %ymm25
-
-// CHECK: vcvtps2qq -2064(%rdx), %ymm25
-// CHECK:  encoding: [0x62,0x61,0x7d,0x28,0x7b,0x8a,0xf0,0xf7,0xff,0xff]
-          vcvtps2qq -2064(%rdx), %ymm25
-
-// CHECK: vcvtps2qq 508(%rdx){1to4}, %ymm25
-// CHECK:  encoding: [0x62,0x61,0x7d,0x38,0x7b,0x4a,0x7f]
-          vcvtps2qq 508(%rdx){1to4}, %ymm25
-
-// CHECK: vcvtps2qq 512(%rdx){1to4}, %ymm25
-// CHECK:  encoding: [0x62,0x61,0x7d,0x38,0x7b,0x8a,0x00,0x02,0x00,0x00]
-          vcvtps2qq 512(%rdx){1to4}, %ymm25
-
-// CHECK: vcvtps2qq -512(%rdx){1to4}, %ymm25
-// CHECK:  encoding: [0x62,0x61,0x7d,0x38,0x7b,0x4a,0x80]
-          vcvtps2qq -512(%rdx){1to4}, %ymm25
-
-// CHECK: vcvtps2qq -516(%rdx){1to4}, %ymm25
-// CHECK:  encoding: [0x62,0x61,0x7d,0x38,0x7b,0x8a,0xfc,0xfd,0xff,0xff]
-          vcvtps2qq -516(%rdx){1to4}, %ymm25
-
-// CHECK: vcvtps2uqq %xmm29, %xmm29
-// CHECK:  encoding: [0x62,0x01,0x7d,0x08,0x79,0xed]
-          vcvtps2uqq %xmm29, %xmm29
-
-// CHECK: vcvtps2uqq %xmm29, %xmm29 {%k1}
-// CHECK:  encoding: [0x62,0x01,0x7d,0x09,0x79,0xed]
-          vcvtps2uqq %xmm29, %xmm29 {%k1}
-
-// CHECK: vcvtps2uqq %xmm29, %xmm29 {%k1} {z}
-// CHECK:  encoding: [0x62,0x01,0x7d,0x89,0x79,0xed]
-          vcvtps2uqq %xmm29, %xmm29 {%k1} {z}
-
-// CHECK: vcvtps2uqq (%rcx), %xmm29
-// CHECK:  encoding: [0x62,0x61,0x7d,0x08,0x79,0x29]
-          vcvtps2uqq (%rcx), %xmm29
-
-// CHECK: vcvtps2uqq 291(%rax,%r14,8), %xmm29
-// CHECK:  encoding: [0x62,0x21,0x7d,0x08,0x79,0xac,0xf0,0x23,0x01,0x00,0x00]
-          vcvtps2uqq 291(%rax,%r14,8), %xmm29
-
-// CHECK: vcvtps2uqq (%rcx){1to2}, %xmm29
-// CHECK:  encoding: [0x62,0x61,0x7d,0x18,0x79,0x29]
-          vcvtps2uqq (%rcx){1to2}, %xmm29
-
-// CHECK: vcvtps2uqq 1016(%rdx), %xmm29
-// CHECK:  encoding: [0x62,0x61,0x7d,0x08,0x79,0x6a,0x7f]
-          vcvtps2uqq 1016(%rdx), %xmm29
-
-// CHECK: vcvtps2uqq 1024(%rdx), %xmm29
-// CHECK:  encoding: [0x62,0x61,0x7d,0x08,0x79,0xaa,0x00,0x04,0x00,0x00]
-          vcvtps2uqq 1024(%rdx), %xmm29
-
-// CHECK: vcvtps2uqq -1024(%rdx), %xmm29
-// CHECK:  encoding: [0x62,0x61,0x7d,0x08,0x79,0x6a,0x80]
-          vcvtps2uqq -1024(%rdx), %xmm29
-
-// CHECK: vcvtps2uqq -1032(%rdx), %xmm29
-// CHECK:  encoding: [0x62,0x61,0x7d,0x08,0x79,0xaa,0xf8,0xfb,0xff,0xff]
-          vcvtps2uqq -1032(%rdx), %xmm29
-
-// CHECK: vcvtps2uqq 508(%rdx){1to2}, %xmm29
-// CHECK:  encoding: [0x62,0x61,0x7d,0x18,0x79,0x6a,0x7f]
-          vcvtps2uqq 508(%rdx){1to2}, %xmm29
-
-// CHECK: vcvtps2uqq 512(%rdx){1to2}, %xmm29
-// CHECK:  encoding: [0x62,0x61,0x7d,0x18,0x79,0xaa,0x00,0x02,0x00,0x00]
-          vcvtps2uqq 512(%rdx){1to2}, %xmm29
-
-// CHECK: vcvtps2uqq -512(%rdx){1to2}, %xmm29
-// CHECK:  encoding: [0x62,0x61,0x7d,0x18,0x79,0x6a,0x80]
-          vcvtps2uqq -512(%rdx){1to2}, %xmm29
-
-// CHECK: vcvtps2uqq -516(%rdx){1to2}, %xmm29
-// CHECK:  encoding: [0x62,0x61,0x7d,0x18,0x79,0xaa,0xfc,0xfd,0xff,0xff]
-          vcvtps2uqq -516(%rdx){1to2}, %xmm29
-
-// CHECK: vcvtps2uqq %xmm19, %ymm23
-// CHECK:  encoding: [0x62,0xa1,0x7d,0x28,0x79,0xfb]
-          vcvtps2uqq %xmm19, %ymm23
-
-// CHECK: vcvtps2uqq %xmm19, %ymm23 {%k2}
-// CHECK:  encoding: [0x62,0xa1,0x7d,0x2a,0x79,0xfb]
-          vcvtps2uqq %xmm19, %ymm23 {%k2}
-
-// CHECK: vcvtps2uqq %xmm19, %ymm23 {%k2} {z}
-// CHECK:  encoding: [0x62,0xa1,0x7d,0xaa,0x79,0xfb]
-          vcvtps2uqq %xmm19, %ymm23 {%k2} {z}
-
-// CHECK: vcvtps2uqq (%rcx), %ymm23
-// CHECK:  encoding: [0x62,0xe1,0x7d,0x28,0x79,0x39]
-          vcvtps2uqq (%rcx), %ymm23
-
-// CHECK: vcvtps2uqq 291(%rax,%r14,8), %ymm23
-// CHECK:  encoding: [0x62,0xa1,0x7d,0x28,0x79,0xbc,0xf0,0x23,0x01,0x00,0x00]
-          vcvtps2uqq 291(%rax,%r14,8), %ymm23
-
-// CHECK: vcvtps2uqq (%rcx){1to4}, %ymm23
-// CHECK:  encoding: [0x62,0xe1,0x7d,0x38,0x79,0x39]
-          vcvtps2uqq (%rcx){1to4}, %ymm23
-
-// CHECK: vcvtps2uqq 2032(%rdx), %ymm23
-// CHECK:  encoding: [0x62,0xe1,0x7d,0x28,0x79,0x7a,0x7f]
-          vcvtps2uqq 2032(%rdx), %ymm23
-
-// CHECK: vcvtps2uqq 2048(%rdx), %ymm23
-// CHECK:  encoding: [0x62,0xe1,0x7d,0x28,0x79,0xba,0x00,0x08,0x00,0x00]
-          vcvtps2uqq 2048(%rdx), %ymm23
-
-// CHECK: vcvtps2uqq -2048(%rdx), %ymm23
-// CHECK:  encoding: [0x62,0xe1,0x7d,0x28,0x79,0x7a,0x80]
-          vcvtps2uqq -2048(%rdx), %ymm23
-
-// CHECK: vcvtps2uqq -2064(%rdx), %ymm23
-// CHECK:  encoding: [0x62,0xe1,0x7d,0x28,0x79,0xba,0xf0,0xf7,0xff,0xff]
-          vcvtps2uqq -2064(%rdx), %ymm23
-
-// CHECK: vcvtps2uqq 508(%rdx){1to4}, %ymm23
-// CHECK:  encoding: [0x62,0xe1,0x7d,0x38,0x79,0x7a,0x7f]
-          vcvtps2uqq 508(%rdx){1to4}, %ymm23
-
-// CHECK: vcvtps2uqq 512(%rdx){1to4}, %ymm23
-// CHECK:  encoding: [0x62,0xe1,0x7d,0x38,0x79,0xba,0x00,0x02,0x00,0x00]
-          vcvtps2uqq 512(%rdx){1to4}, %ymm23
-
-// CHECK: vcvtps2uqq -512(%rdx){1to4}, %ymm23
-// CHECK:  encoding: [0x62,0xe1,0x7d,0x38,0x79,0x7a,0x80]
-          vcvtps2uqq -512(%rdx){1to4}, %ymm23
-
-// CHECK: vcvtps2uqq -516(%rdx){1to4}, %ymm23
-// CHECK:  encoding: [0x62,0xe1,0x7d,0x38,0x79,0xba,0xfc,0xfd,0xff,0xff]
-          vcvtps2uqq -516(%rdx){1to4}, %ymm23
-
-// CHECK: vcvtqq2pd %xmm29, %xmm22
-// CHECK:  encoding: [0x62,0x81,0xfe,0x08,0xe6,0xf5]
-          vcvtqq2pd %xmm29, %xmm22
-
-// CHECK: vcvtqq2pd %xmm29, %xmm22 {%k7}
-// CHECK:  encoding: [0x62,0x81,0xfe,0x0f,0xe6,0xf5]
-          vcvtqq2pd %xmm29, %xmm22 {%k7}
-
-// CHECK: vcvtqq2pd %xmm29, %xmm22 {%k7} {z}
-// CHECK:  encoding: [0x62,0x81,0xfe,0x8f,0xe6,0xf5]
-          vcvtqq2pd %xmm29, %xmm22 {%k7} {z}
-
-// CHECK: vcvtqq2pd (%rcx), %xmm22
-// CHECK:  encoding: [0x62,0xe1,0xfe,0x08,0xe6,0x31]
-          vcvtqq2pd (%rcx), %xmm22
-
-// CHECK: vcvtqq2pd 291(%rax,%r14,8), %xmm22
-// CHECK:  encoding: [0x62,0xa1,0xfe,0x08,0xe6,0xb4,0xf0,0x23,0x01,0x00,0x00]
-          vcvtqq2pd 291(%rax,%r14,8), %xmm22
-
-// CHECK: vcvtqq2pd (%rcx){1to2}, %xmm22
-// CHECK:  encoding: [0x62,0xe1,0xfe,0x18,0xe6,0x31]
-          vcvtqq2pd (%rcx){1to2}, %xmm22
-
-// CHECK: vcvtqq2pd 2032(%rdx), %xmm22
-// CHECK:  encoding: [0x62,0xe1,0xfe,0x08,0xe6,0x72,0x7f]
-          vcvtqq2pd 2032(%rdx), %xmm22
-
-// CHECK: vcvtqq2pd 2048(%rdx), %xmm22
-// CHECK:  encoding: [0x62,0xe1,0xfe,0x08,0xe6,0xb2,0x00,0x08,0x00,0x00]
-          vcvtqq2pd 2048(%rdx), %xmm22
-
-// CHECK: vcvtqq2pd -2048(%rdx), %xmm22
-// CHECK:  encoding: [0x62,0xe1,0xfe,0x08,0xe6,0x72,0x80]
-          vcvtqq2pd -2048(%rdx), %xmm22
-
-// CHECK: vcvtqq2pd -2064(%rdx), %xmm22
-// CHECK:  encoding: [0x62,0xe1,0xfe,0x08,0xe6,0xb2,0xf0,0xf7,0xff,0xff]
-          vcvtqq2pd -2064(%rdx), %xmm22
-
-// CHECK: vcvtqq2pd 1016(%rdx){1to2}, %xmm22
-// CHECK:  encoding: [0x62,0xe1,0xfe,0x18,0xe6,0x72,0x7f]
-          vcvtqq2pd 1016(%rdx){1to2}, %xmm22
-
-// CHECK: vcvtqq2pd 1024(%rdx){1to2}, %xmm22
-// CHECK:  encoding: [0x62,0xe1,0xfe,0x18,0xe6,0xb2,0x00,0x04,0x00,0x00]
-          vcvtqq2pd 1024(%rdx){1to2}, %xmm22
-
-// CHECK: vcvtqq2pd -1024(%rdx){1to2}, %xmm22
-// CHECK:  encoding: [0x62,0xe1,0xfe,0x18,0xe6,0x72,0x80]
-          vcvtqq2pd -1024(%rdx){1to2}, %xmm22
-
-// CHECK: vcvtqq2pd -1032(%rdx){1to2}, %xmm22
-// CHECK:  encoding: [0x62,0xe1,0xfe,0x18,0xe6,0xb2,0xf8,0xfb,0xff,0xff]
-          vcvtqq2pd -1032(%rdx){1to2}, %xmm22
-
-// CHECK: vcvtqq2pd %ymm20, %ymm21
-// CHECK:  encoding: [0x62,0xa1,0xfe,0x28,0xe6,0xec]
-          vcvtqq2pd %ymm20, %ymm21
-
-// CHECK: vcvtqq2pd %ymm20, %ymm21 {%k5}
-// CHECK:  encoding: [0x62,0xa1,0xfe,0x2d,0xe6,0xec]
-          vcvtqq2pd %ymm20, %ymm21 {%k5}
-
-// CHECK: vcvtqq2pd %ymm20, %ymm21 {%k5} {z}
-// CHECK:  encoding: [0x62,0xa1,0xfe,0xad,0xe6,0xec]
-          vcvtqq2pd %ymm20, %ymm21 {%k5} {z}
-
-// CHECK: vcvtqq2pd (%rcx), %ymm21
-// CHECK:  encoding: [0x62,0xe1,0xfe,0x28,0xe6,0x29]
-          vcvtqq2pd (%rcx), %ymm21
-
-// CHECK: vcvtqq2pd 291(%rax,%r14,8), %ymm21
-// CHECK:  encoding: [0x62,0xa1,0xfe,0x28,0xe6,0xac,0xf0,0x23,0x01,0x00,0x00]
-          vcvtqq2pd 291(%rax,%r14,8), %ymm21
-
-// CHECK: vcvtqq2pd (%rcx){1to4}, %ymm21
-// CHECK:  encoding: [0x62,0xe1,0xfe,0x38,0xe6,0x29]
-          vcvtqq2pd (%rcx){1to4}, %ymm21
-
-// CHECK: vcvtqq2pd 4064(%rdx), %ymm21
-// CHECK:  encoding: [0x62,0xe1,0xfe,0x28,0xe6,0x6a,0x7f]
-          vcvtqq2pd 4064(%rdx), %ymm21
-
-// CHECK: vcvtqq2pd 4096(%rdx), %ymm21
-// CHECK:  encoding: [0x62,0xe1,0xfe,0x28,0xe6,0xaa,0x00,0x10,0x00,0x00]
-          vcvtqq2pd 4096(%rdx), %ymm21
-
-// CHECK: vcvtqq2pd -4096(%rdx), %ymm21
-// CHECK:  encoding: [0x62,0xe1,0xfe,0x28,0xe6,0x6a,0x80]
-          vcvtqq2pd -4096(%rdx), %ymm21
-
-// CHECK: vcvtqq2pd -4128(%rdx), %ymm21
-// CHECK:  encoding: [0x62,0xe1,0xfe,0x28,0xe6,0xaa,0xe0,0xef,0xff,0xff]
-          vcvtqq2pd -4128(%rdx), %ymm21
-
-// CHECK: vcvtqq2pd 1016(%rdx){1to4}, %ymm21
-// CHECK:  encoding: [0x62,0xe1,0xfe,0x38,0xe6,0x6a,0x7f]
-          vcvtqq2pd 1016(%rdx){1to4}, %ymm21
-
-// CHECK: vcvtqq2pd 1024(%rdx){1to4}, %ymm21
-// CHECK:  encoding: [0x62,0xe1,0xfe,0x38,0xe6,0xaa,0x00,0x04,0x00,0x00]
-          vcvtqq2pd 1024(%rdx){1to4}, %ymm21
-
-// CHECK: vcvtqq2pd -1024(%rdx){1to4}, %ymm21
-// CHECK:  encoding: [0x62,0xe1,0xfe,0x38,0xe6,0x6a,0x80]
-          vcvtqq2pd -1024(%rdx){1to4}, %ymm21
-
-// CHECK: vcvtqq2pd -1032(%rdx){1to4}, %ymm21
-// CHECK:  encoding: [0x62,0xe1,0xfe,0x38,0xe6,0xaa,0xf8,0xfb,0xff,0xff]
-          vcvtqq2pd -1032(%rdx){1to4}, %ymm21
-
-// CHECK: vcvtqq2ps %xmm28, %xmm25
-// CHECK:  encoding: [0x62,0x01,0xfc,0x08,0x5b,0xcc]
-          vcvtqq2ps %xmm28, %xmm25
-
-// CHECK: vcvtqq2ps %xmm28, %xmm25
-// CHECK:  encoding: [0x62,0x01,0xfc,0x08,0x5b,0xcc]
-          vcvtqq2psx %xmm28, %xmm25
-
-// CHECK: vcvtqq2ps %xmm28, %xmm25 {%k6}
-// CHECK:  encoding: [0x62,0x01,0xfc,0x0e,0x5b,0xcc]
-          vcvtqq2ps %xmm28, %xmm25 {%k6}
-
-// CHECK: vcvtqq2ps %xmm28, %xmm25 {%k6}
-// CHECK:  encoding: [0x62,0x01,0xfc,0x0e,0x5b,0xcc]
-          vcvtqq2psx %xmm28, %xmm25 {%k6}
-
-// CHECK: vcvtqq2ps %xmm28, %xmm25 {%k6} {z}
-// CHECK:  encoding: [0x62,0x01,0xfc,0x8e,0x5b,0xcc]
-          vcvtqq2ps %xmm28, %xmm25 {%k6} {z}
-
-// CHECK: vcvtqq2ps %xmm28, %xmm25 {%k6} {z}
-// CHECK:  encoding: [0x62,0x01,0xfc,0x8e,0x5b,0xcc]
-          vcvtqq2psx %xmm28, %xmm25 {%k6} {z}
-
-// CHECK: vcvtqq2psx (%rcx), %xmm25
-// CHECK:  encoding: [0x62,0x61,0xfc,0x08,0x5b,0x09]
-          vcvtqq2psx (%rcx), %xmm25
-
-// CHECK: vcvtqq2psx (%rcx), %xmm25 {%k6}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x0e,0x5b,0x09]
-          vcvtqq2psx (%rcx), %xmm25 {%k6}
-
-// CHECK: vcvtqq2psx (%rcx), %xmm25 {%k6} {z}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x8e,0x5b,0x09]
-          vcvtqq2psx (%rcx), %xmm25 {%k6} {z}
-
-// CHECK: vcvtqq2psx 291(%rax,%r14,8), %xmm25
-// CHECK:  encoding: [0x62,0x21,0xfc,0x08,0x5b,0x8c,0xf0,0x23,0x01,0x00,0x00]
-          vcvtqq2psx 291(%rax,%r14,8), %xmm25
-
-// CHECK: vcvtqq2psx 291(%rax,%r14,8), %xmm25 {%k6}
-// CHECK:  encoding: [0x62,0x21,0xfc,0x0e,0x5b,0x8c,0xf0,0x23,0x01,0x00,0x00]
-          vcvtqq2psx 291(%rax,%r14,8), %xmm25 {%k6}
-
-// CHECK: vcvtqq2psx 291(%rax,%r14,8), %xmm25 {%k6} {z}
-// CHECK:  encoding: [0x62,0x21,0xfc,0x8e,0x5b,0x8c,0xf0,0x23,0x01,0x00,0x00]
-          vcvtqq2psx 291(%rax,%r14,8), %xmm25 {%k6} {z}
-
-// CHECK: vcvtqq2ps (%rcx){1to2}, %xmm25
-// CHECK:  encoding: [0x62,0x61,0xfc,0x18,0x5b,0x09]
-          vcvtqq2ps (%rcx){1to2}, %xmm25
-
-// CHECK: vcvtqq2ps (%rcx){1to2}, %xmm25
-// CHECK:  encoding: [0x62,0x61,0xfc,0x18,0x5b,0x09]
-          vcvtqq2psx (%rcx){1to2}, %xmm25
-
-// CHECK: vcvtqq2ps (%rcx){1to2}, %xmm25 {%k6}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x1e,0x5b,0x09]
-          vcvtqq2ps (%rcx){1to2}, %xmm25 {%k6}
-
-// CHECK: vcvtqq2ps (%rcx){1to2}, %xmm25 {%k6}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x1e,0x5b,0x09]
-          vcvtqq2psx (%rcx){1to2}, %xmm25 {%k6}
-
-// CHECK: vcvtqq2ps (%rcx){1to2}, %xmm25 {%k6} {z}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x9e,0x5b,0x09]
-          vcvtqq2ps (%rcx){1to2}, %xmm25 {%k6} {z}
-
-// CHECK: vcvtqq2ps (%rcx){1to2}, %xmm25 {%k6} {z}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x9e,0x5b,0x09]
-          vcvtqq2psx (%rcx){1to2}, %xmm25 {%k6} {z}
-
-// CHECK: vcvtqq2psx 2032(%rdx), %xmm25
-// CHECK:  encoding: [0x62,0x61,0xfc,0x08,0x5b,0x4a,0x7f]
-          vcvtqq2psx 2032(%rdx), %xmm25
-
-// CHECK: vcvtqq2psx 2032(%rdx), %xmm25 {%k6}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x0e,0x5b,0x4a,0x7f]
-          vcvtqq2psx 2032(%rdx), %xmm25 {%k6}
-
-// CHECK: vcvtqq2psx 2032(%rdx), %xmm25 {%k6} {z}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x8e,0x5b,0x4a,0x7f]
-          vcvtqq2psx 2032(%rdx), %xmm25 {%k6} {z}
-
-// CHECK: vcvtqq2psx 2048(%rdx), %xmm25
-// CHECK:  encoding: [0x62,0x61,0xfc,0x08,0x5b,0x8a,0x00,0x08,0x00,0x00]
-          vcvtqq2psx 2048(%rdx), %xmm25
-
-// CHECK: vcvtqq2psx 2048(%rdx), %xmm25 {%k6}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x0e,0x5b,0x8a,0x00,0x08,0x00,0x00]
-          vcvtqq2psx 2048(%rdx), %xmm25 {%k6}
-
-// CHECK: vcvtqq2psx 2048(%rdx), %xmm25 {%k6} {z}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x8e,0x5b,0x8a,0x00,0x08,0x00,0x00]
-          vcvtqq2psx 2048(%rdx), %xmm25 {%k6} {z}
-
-// CHECK: vcvtqq2psx -2048(%rdx), %xmm25
-// CHECK:  encoding: [0x62,0x61,0xfc,0x08,0x5b,0x4a,0x80]
-          vcvtqq2psx -2048(%rdx), %xmm25
-
-// CHECK: vcvtqq2psx -2048(%rdx), %xmm25 {%k6}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x0e,0x5b,0x4a,0x80]
-          vcvtqq2psx -2048(%rdx), %xmm25 {%k6}
-
-// CHECK: vcvtqq2psx -2048(%rdx), %xmm25 {%k6} {z}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x8e,0x5b,0x4a,0x80]
-          vcvtqq2psx -2048(%rdx), %xmm25 {%k6} {z}
-
-// CHECK: vcvtqq2psx -2064(%rdx), %xmm25
-// CHECK:  encoding: [0x62,0x61,0xfc,0x08,0x5b,0x8a,0xf0,0xf7,0xff,0xff]
-          vcvtqq2psx -2064(%rdx), %xmm25
-
-// CHECK: vcvtqq2psx -2064(%rdx), %xmm25 {%k6}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x0e,0x5b,0x8a,0xf0,0xf7,0xff,0xff]
-          vcvtqq2psx -2064(%rdx), %xmm25 {%k6}
-
-// CHECK: vcvtqq2psx -2064(%rdx), %xmm25 {%k6} {z}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x8e,0x5b,0x8a,0xf0,0xf7,0xff,0xff]
-          vcvtqq2psx -2064(%rdx), %xmm25 {%k6} {z}
-
-// CHECK: vcvtqq2ps 1016(%rdx){1to2}, %xmm25
-// CHECK:  encoding: [0x62,0x61,0xfc,0x18,0x5b,0x4a,0x7f]
-          vcvtqq2ps 1016(%rdx){1to2}, %xmm25
-
-// CHECK: vcvtqq2ps 1016(%rdx){1to2}, %xmm25
-// CHECK:  encoding: [0x62,0x61,0xfc,0x18,0x5b,0x4a,0x7f]
-          vcvtqq2psx 1016(%rdx){1to2}, %xmm25
-
-// CHECK: vcvtqq2ps 1016(%rdx){1to2}, %xmm25 {%k6}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x1e,0x5b,0x4a,0x7f]
-          vcvtqq2ps 1016(%rdx){1to2}, %xmm25 {%k6}
-
-// CHECK: vcvtqq2ps 1016(%rdx){1to2}, %xmm25 {%k6}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x1e,0x5b,0x4a,0x7f]
-          vcvtqq2psx 1016(%rdx){1to2}, %xmm25 {%k6}
-
-// CHECK: vcvtqq2ps 1016(%rdx){1to2}, %xmm25 {%k6} {z}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x9e,0x5b,0x4a,0x7f]
-          vcvtqq2ps 1016(%rdx){1to2}, %xmm25 {%k6} {z}
-
-// CHECK: vcvtqq2ps 1016(%rdx){1to2}, %xmm25 {%k6} {z}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x9e,0x5b,0x4a,0x7f]
-          vcvtqq2psx 1016(%rdx){1to2}, %xmm25 {%k6} {z}
-
-// CHECK: vcvtqq2ps 1024(%rdx){1to2}, %xmm25
-// CHECK:  encoding: [0x62,0x61,0xfc,0x18,0x5b,0x8a,0x00,0x04,0x00,0x00]
-          vcvtqq2ps 1024(%rdx){1to2}, %xmm25
-
-// CHECK: vcvtqq2ps 1024(%rdx){1to2}, %xmm25
-// CHECK:  encoding: [0x62,0x61,0xfc,0x18,0x5b,0x8a,0x00,0x04,0x00,0x00]
-          vcvtqq2psx 1024(%rdx){1to2}, %xmm25
-
-// CHECK: vcvtqq2ps 1024(%rdx){1to2}, %xmm25 {%k6}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x1e,0x5b,0x8a,0x00,0x04,0x00,0x00]
-          vcvtqq2ps 1024(%rdx){1to2}, %xmm25 {%k6}
-
-// CHECK: vcvtqq2ps 1024(%rdx){1to2}, %xmm25 {%k6}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x1e,0x5b,0x8a,0x00,0x04,0x00,0x00]
-          vcvtqq2psx 1024(%rdx){1to2}, %xmm25 {%k6}
-
-// CHECK: vcvtqq2ps 1024(%rdx){1to2}, %xmm25 {%k6} {z}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x9e,0x5b,0x8a,0x00,0x04,0x00,0x00]
-          vcvtqq2ps 1024(%rdx){1to2}, %xmm25 {%k6} {z}
-
-// CHECK: vcvtqq2ps 1024(%rdx){1to2}, %xmm25 {%k6} {z}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x9e,0x5b,0x8a,0x00,0x04,0x00,0x00]
-          vcvtqq2psx 1024(%rdx){1to2}, %xmm25 {%k6} {z}
-
-// CHECK: vcvtqq2ps -1024(%rdx){1to2}, %xmm25
-// CHECK:  encoding: [0x62,0x61,0xfc,0x18,0x5b,0x4a,0x80]
-          vcvtqq2ps -1024(%rdx){1to2}, %xmm25
-
-// CHECK: vcvtqq2ps -1024(%rdx){1to2}, %xmm25
-// CHECK:  encoding: [0x62,0x61,0xfc,0x18,0x5b,0x4a,0x80]
-          vcvtqq2psx -1024(%rdx){1to2}, %xmm25
-
-// CHECK: vcvtqq2ps -1024(%rdx){1to2}, %xmm25 {%k6}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x1e,0x5b,0x4a,0x80]
-          vcvtqq2ps -1024(%rdx){1to2}, %xmm25 {%k6}
-
-// CHECK: vcvtqq2ps -1024(%rdx){1to2}, %xmm25 {%k6}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x1e,0x5b,0x4a,0x80]
-          vcvtqq2psx -1024(%rdx){1to2}, %xmm25 {%k6}
-
-// CHECK: vcvtqq2ps -1024(%rdx){1to2}, %xmm25 {%k6} {z}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x9e,0x5b,0x4a,0x80]
-          vcvtqq2ps -1024(%rdx){1to2}, %xmm25 {%k6} {z}
-
-// CHECK: vcvtqq2ps -1024(%rdx){1to2}, %xmm25 {%k6} {z}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x9e,0x5b,0x4a,0x80]
-          vcvtqq2psx -1024(%rdx){1to2}, %xmm25 {%k6} {z}
-
-// CHECK: vcvtqq2ps -1032(%rdx){1to2}, %xmm25
-// CHECK:  encoding: [0x62,0x61,0xfc,0x18,0x5b,0x8a,0xf8,0xfb,0xff,0xff]
-          vcvtqq2ps -1032(%rdx){1to2}, %xmm25
-
-// CHECK: vcvtqq2ps -1032(%rdx){1to2}, %xmm25
-// CHECK:  encoding: [0x62,0x61,0xfc,0x18,0x5b,0x8a,0xf8,0xfb,0xff,0xff]
-          vcvtqq2psx -1032(%rdx){1to2}, %xmm25
-
-// CHECK: vcvtqq2ps -1032(%rdx){1to2}, %xmm25 {%k6}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x1e,0x5b,0x8a,0xf8,0xfb,0xff,0xff]
-          vcvtqq2ps -1032(%rdx){1to2}, %xmm25 {%k6}
-
-// CHECK: vcvtqq2ps -1032(%rdx){1to2}, %xmm25 {%k6}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x1e,0x5b,0x8a,0xf8,0xfb,0xff,0xff]
-          vcvtqq2psx -1032(%rdx){1to2}, %xmm25 {%k6}
-
-// CHECK: vcvtqq2ps -1032(%rdx){1to2}, %xmm25 {%k6} {z}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x9e,0x5b,0x8a,0xf8,0xfb,0xff,0xff]
-          vcvtqq2ps -1032(%rdx){1to2}, %xmm25 {%k6} {z}
-
-// CHECK: vcvtqq2ps -1032(%rdx){1to2}, %xmm25 {%k6} {z}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x9e,0x5b,0x8a,0xf8,0xfb,0xff,0xff]
-          vcvtqq2psx -1032(%rdx){1to2}, %xmm25 {%k6} {z}
-
-// CHECK: vcvtqq2ps %ymm22, %xmm27
-// CHECK:  encoding: [0x62,0x21,0xfc,0x28,0x5b,0xde]
-          vcvtqq2ps %ymm22, %xmm27
-
-// CHECK: vcvtqq2ps %ymm22, %xmm27
-// CHECK:  encoding: [0x62,0x21,0xfc,0x28,0x5b,0xde]
-          vcvtqq2psy %ymm22, %xmm27
-
-// CHECK: vcvtqq2ps %ymm22, %xmm27 {%k3}
-// CHECK:  encoding: [0x62,0x21,0xfc,0x2b,0x5b,0xde]
-          vcvtqq2ps %ymm22, %xmm27 {%k3}
-
-// CHECK: vcvtqq2ps %ymm22, %xmm27 {%k3}
-// CHECK:  encoding: [0x62,0x21,0xfc,0x2b,0x5b,0xde]
-          vcvtqq2psy %ymm22, %xmm27 {%k3}
-
-// CHECK: vcvtqq2ps %ymm22, %xmm27 {%k3} {z}
-// CHECK:  encoding: [0x62,0x21,0xfc,0xab,0x5b,0xde]
-          vcvtqq2ps %ymm22, %xmm27 {%k3} {z}
-
-// CHECK: vcvtqq2ps %ymm22, %xmm27 {%k3} {z}
-// CHECK:  encoding: [0x62,0x21,0xfc,0xab,0x5b,0xde]
-          vcvtqq2psy %ymm22, %xmm27 {%k3} {z}
-
-// CHECK: vcvtqq2psy (%rcx), %xmm27
-// CHECK:  encoding: [0x62,0x61,0xfc,0x28,0x5b,0x19]
-          vcvtqq2psy (%rcx), %xmm27
-
-// CHECK: vcvtqq2psy (%rcx), %xmm27 {%k3}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x2b,0x5b,0x19]
-          vcvtqq2psy (%rcx), %xmm27 {%k3}
-
-// CHECK: vcvtqq2psy (%rcx), %xmm27 {%k3} {z}
-// CHECK:  encoding: [0x62,0x61,0xfc,0xab,0x5b,0x19]
-          vcvtqq2psy (%rcx), %xmm27 {%k3} {z}
-
-// CHECK: vcvtqq2psy 291(%rax,%r14,8), %xmm27
-// CHECK:  encoding: [0x62,0x21,0xfc,0x28,0x5b,0x9c,0xf0,0x23,0x01,0x00,0x00]
-          vcvtqq2psy 291(%rax,%r14,8), %xmm27
-
-// CHECK: vcvtqq2psy 291(%rax,%r14,8), %xmm27 {%k3}
-// CHECK:  encoding: [0x62,0x21,0xfc,0x2b,0x5b,0x9c,0xf0,0x23,0x01,0x00,0x00]
-          vcvtqq2psy 291(%rax,%r14,8), %xmm27 {%k3}
-
-// CHECK: vcvtqq2psy 291(%rax,%r14,8), %xmm27 {%k3} {z}
-// CHECK:  encoding: [0x62,0x21,0xfc,0xab,0x5b,0x9c,0xf0,0x23,0x01,0x00,0x00]
-          vcvtqq2psy 291(%rax,%r14,8), %xmm27 {%k3} {z}
-
-// CHECK: vcvtqq2ps (%rcx){1to4}, %xmm27
-// CHECK:  encoding: [0x62,0x61,0xfc,0x38,0x5b,0x19]
-          vcvtqq2ps (%rcx){1to4}, %xmm27
-
-// CHECK: vcvtqq2ps (%rcx){1to4}, %xmm27 {%k3}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x3b,0x5b,0x19]
-          vcvtqq2ps (%rcx){1to4}, %xmm27 {%k3}
-
-// CHECK: vcvtqq2ps (%rcx){1to4}, %xmm27 {%k3} {z}
-// CHECK:  encoding: [0x62,0x61,0xfc,0xbb,0x5b,0x19]
-          vcvtqq2ps (%rcx){1to4}, %xmm27 {%k3} {z}
-
-// CHECK: vcvtqq2psy 4064(%rdx), %xmm27
-// CHECK:  encoding: [0x62,0x61,0xfc,0x28,0x5b,0x5a,0x7f]
-          vcvtqq2psy 4064(%rdx), %xmm27
-
-// CHECK: vcvtqq2psy 4064(%rdx), %xmm27 {%k3}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x2b,0x5b,0x5a,0x7f]
-          vcvtqq2psy 4064(%rdx), %xmm27 {%k3}
-
-// CHECK: vcvtqq2psy 4064(%rdx), %xmm27 {%k3} {z}
-// CHECK:  encoding: [0x62,0x61,0xfc,0xab,0x5b,0x5a,0x7f]
-          vcvtqq2psy 4064(%rdx), %xmm27 {%k3} {z}
-
-// CHECK: vcvtqq2psy 4096(%rdx), %xmm27
-// CHECK:  encoding: [0x62,0x61,0xfc,0x28,0x5b,0x9a,0x00,0x10,0x00,0x00]
-          vcvtqq2psy 4096(%rdx), %xmm27
-
-// CHECK: vcvtqq2psy 4096(%rdx), %xmm27 {%k3}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x2b,0x5b,0x9a,0x00,0x10,0x00,0x00]
-          vcvtqq2psy 4096(%rdx), %xmm27 {%k3}
-
-// CHECK: vcvtqq2psy 4096(%rdx), %xmm27 {%k3} {z}
-// CHECK:  encoding: [0x62,0x61,0xfc,0xab,0x5b,0x9a,0x00,0x10,0x00,0x00]
-          vcvtqq2psy 4096(%rdx), %xmm27 {%k3} {z}
-
-// CHECK: vcvtqq2psy -4096(%rdx), %xmm27
-// CHECK:  encoding: [0x62,0x61,0xfc,0x28,0x5b,0x5a,0x80]
-          vcvtqq2psy -4096(%rdx), %xmm27
-
-// CHECK: vcvtqq2psy -4096(%rdx), %xmm27 {%k3}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x2b,0x5b,0x5a,0x80]
-          vcvtqq2psy -4096(%rdx), %xmm27 {%k3}
-
-// CHECK: vcvtqq2psy -4096(%rdx), %xmm27 {%k3} {z}
-// CHECK:  encoding: [0x62,0x61,0xfc,0xab,0x5b,0x5a,0x80]
-          vcvtqq2psy -4096(%rdx), %xmm27 {%k3} {z}
-
-// CHECK: vcvtqq2psy -4128(%rdx), %xmm27
-// CHECK:  encoding: [0x62,0x61,0xfc,0x28,0x5b,0x9a,0xe0,0xef,0xff,0xff]
-          vcvtqq2psy -4128(%rdx), %xmm27
-
-// CHECK: vcvtqq2psy -4128(%rdx), %xmm27 {%k3}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x2b,0x5b,0x9a,0xe0,0xef,0xff,0xff]
-          vcvtqq2psy -4128(%rdx), %xmm27 {%k3}
-
-// CHECK: vcvtqq2psy -4128(%rdx), %xmm27 {%k3} {z}
-// CHECK:  encoding: [0x62,0x61,0xfc,0xab,0x5b,0x9a,0xe0,0xef,0xff,0xff]
-          vcvtqq2psy -4128(%rdx), %xmm27 {%k3} {z}
-
-// CHECK: vcvtqq2ps 1016(%rdx){1to4}, %xmm27
-// CHECK:  encoding: [0x62,0x61,0xfc,0x38,0x5b,0x5a,0x7f]
-          vcvtqq2ps 1016(%rdx){1to4}, %xmm27
-
-// CHECK: vcvtqq2ps 1016(%rdx){1to4}, %xmm27
-// CHECK:  encoding: [0x62,0x61,0xfc,0x38,0x5b,0x5a,0x7f]
-          vcvtqq2psy 1016(%rdx){1to4}, %xmm27
-
-// CHECK: vcvtqq2ps 1016(%rdx){1to4}, %xmm27 {%k3}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x3b,0x5b,0x5a,0x7f]
-          vcvtqq2ps 1016(%rdx){1to4}, %xmm27 {%k3}
-
-// CHECK: vcvtqq2ps 1016(%rdx){1to4}, %xmm27 {%k3}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x3b,0x5b,0x5a,0x7f]
-          vcvtqq2psy 1016(%rdx){1to4}, %xmm27 {%k3}
-
-// CHECK: vcvtqq2ps 1016(%rdx){1to4}, %xmm27 {%k3} {z}
-// CHECK:  encoding: [0x62,0x61,0xfc,0xbb,0x5b,0x5a,0x7f]
-          vcvtqq2ps 1016(%rdx){1to4}, %xmm27 {%k3} {z}
-
-// CHECK: vcvtqq2ps 1016(%rdx){1to4}, %xmm27 {%k3} {z}
-// CHECK:  encoding: [0x62,0x61,0xfc,0xbb,0x5b,0x5a,0x7f]
-          vcvtqq2psy 1016(%rdx){1to4}, %xmm27 {%k3} {z}
-
-// CHECK: vcvtqq2ps 1024(%rdx){1to4}, %xmm27
-// CHECK:  encoding: [0x62,0x61,0xfc,0x38,0x5b,0x9a,0x00,0x04,0x00,0x00]
-          vcvtqq2ps 1024(%rdx){1to4}, %xmm27
-
-// CHECK: vcvtqq2ps 1024(%rdx){1to4}, %xmm27
-// CHECK:  encoding: [0x62,0x61,0xfc,0x38,0x5b,0x9a,0x00,0x04,0x00,0x00]
-          vcvtqq2psy 1024(%rdx){1to4}, %xmm27
-
-// CHECK: vcvtqq2ps 1024(%rdx){1to4}, %xmm27 {%k3}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x3b,0x5b,0x9a,0x00,0x04,0x00,0x00]
-          vcvtqq2ps 1024(%rdx){1to4}, %xmm27 {%k3}
-
-// CHECK: vcvtqq2ps 1024(%rdx){1to4}, %xmm27 {%k3}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x3b,0x5b,0x9a,0x00,0x04,0x00,0x00]
-          vcvtqq2psy 1024(%rdx){1to4}, %xmm27 {%k3}
-
-// CHECK: vcvtqq2ps 1024(%rdx){1to4}, %xmm27 {%k3} {z}
-// CHECK:  encoding: [0x62,0x61,0xfc,0xbb,0x5b,0x9a,0x00,0x04,0x00,0x00]
-          vcvtqq2ps 1024(%rdx){1to4}, %xmm27 {%k3} {z}
-
-// CHECK: vcvtqq2ps 1024(%rdx){1to4}, %xmm27 {%k3} {z}
-// CHECK:  encoding: [0x62,0x61,0xfc,0xbb,0x5b,0x9a,0x00,0x04,0x00,0x00]
-          vcvtqq2psy 1024(%rdx){1to4}, %xmm27 {%k3} {z}
-
-// CHECK: vcvtqq2ps -1024(%rdx){1to4}, %xmm27
-// CHECK:  encoding: [0x62,0x61,0xfc,0x38,0x5b,0x5a,0x80]
-          vcvtqq2ps -1024(%rdx){1to4}, %xmm27
-
-// CHECK: vcvtqq2ps -1024(%rdx){1to4}, %xmm27
-// CHECK:  encoding: [0x62,0x61,0xfc,0x38,0x5b,0x5a,0x80]
-          vcvtqq2psy -1024(%rdx){1to4}, %xmm27
-
-// CHECK: vcvtqq2ps -1024(%rdx){1to4}, %xmm27 {%k3}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x3b,0x5b,0x5a,0x80]
-          vcvtqq2ps -1024(%rdx){1to4}, %xmm27 {%k3}
-
-// CHECK: vcvtqq2ps -1024(%rdx){1to4}, %xmm27 {%k3}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x3b,0x5b,0x5a,0x80]
-          vcvtqq2psy -1024(%rdx){1to4}, %xmm27 {%k3}
-
-// CHECK: vcvtqq2ps -1024(%rdx){1to4}, %xmm27 {%k3} {z}
-// CHECK:  encoding: [0x62,0x61,0xfc,0xbb,0x5b,0x5a,0x80]
-          vcvtqq2ps -1024(%rdx){1to4}, %xmm27 {%k3} {z}
-
-// CHECK: vcvtqq2ps -1024(%rdx){1to4}, %xmm27 {%k3} {z}
-// CHECK:  encoding: [0x62,0x61,0xfc,0xbb,0x5b,0x5a,0x80]
-          vcvtqq2psy -1024(%rdx){1to4}, %xmm27 {%k3} {z}
-
-// CHECK: vcvtqq2ps -1032(%rdx){1to4}, %xmm27
-// CHECK:  encoding: [0x62,0x61,0xfc,0x38,0x5b,0x9a,0xf8,0xfb,0xff,0xff]
-          vcvtqq2ps -1032(%rdx){1to4}, %xmm27
-
-// CHECK: vcvtqq2ps -1032(%rdx){1to4}, %xmm27
-// CHECK:  encoding: [0x62,0x61,0xfc,0x38,0x5b,0x9a,0xf8,0xfb,0xff,0xff]
-          vcvtqq2psy -1032(%rdx){1to4}, %xmm27
-
-// CHECK: vcvtqq2ps -1032(%rdx){1to4}, %xmm27 {%k3}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x3b,0x5b,0x9a,0xf8,0xfb,0xff,0xff]
-          vcvtqq2ps -1032(%rdx){1to4}, %xmm27 {%k3}
-
-// CHECK: vcvtqq2ps -1032(%rdx){1to4}, %xmm27 {%k3}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x3b,0x5b,0x9a,0xf8,0xfb,0xff,0xff]
-          vcvtqq2psy -1032(%rdx){1to4}, %xmm27 {%k3}
-
-// CHECK: vcvtqq2ps -1032(%rdx){1to4}, %xmm27 {%k3} {z}
-// CHECK:  encoding: [0x62,0x61,0xfc,0xbb,0x5b,0x9a,0xf8,0xfb,0xff,0xff]
-          vcvtqq2ps -1032(%rdx){1to4}, %xmm27 {%k3} {z}
-
-// CHECK: vcvtqq2ps -1032(%rdx){1to4}, %xmm27 {%k3} {z}
-// CHECK:  encoding: [0x62,0x61,0xfc,0xbb,0x5b,0x9a,0xf8,0xfb,0xff,0xff]
-          vcvtqq2psy -1032(%rdx){1to4}, %xmm27 {%k3} {z}
-
-// CHECK: vcvtqq2ps %xmm26, %xmm30
-// CHECK:  encoding: [0x62,0x01,0xfc,0x08,0x5b,0xf2]
-          vcvtqq2ps %xmm26, %xmm30
-
-// CHECK: vcvtqq2ps %xmm26, %xmm30
-// CHECK:  encoding: [0x62,0x01,0xfc,0x08,0x5b,0xf2]
-          vcvtqq2psx %xmm26, %xmm30
-
-// CHECK: vcvtqq2ps %xmm26, %xmm30 {%k4}
-// CHECK:  encoding: [0x62,0x01,0xfc,0x0c,0x5b,0xf2]
-          vcvtqq2ps %xmm26, %xmm30 {%k4}
-
-// CHECK: vcvtqq2ps %xmm26, %xmm30 {%k4}
-// CHECK:  encoding: [0x62,0x01,0xfc,0x0c,0x5b,0xf2]
-          vcvtqq2psx %xmm26, %xmm30 {%k4}
-
-// CHECK: vcvtqq2ps %xmm26, %xmm30 {%k4} {z}
-// CHECK:  encoding: [0x62,0x01,0xfc,0x8c,0x5b,0xf2]
-          vcvtqq2ps %xmm26, %xmm30 {%k4} {z}
-
-// CHECK: vcvtqq2ps %xmm26, %xmm30 {%k4} {z}
-// CHECK:  encoding: [0x62,0x01,0xfc,0x8c,0x5b,0xf2]
-          vcvtqq2psx %xmm26, %xmm30 {%k4} {z}
-
-// CHECK: vcvtqq2psx (%rcx), %xmm30
-// CHECK:  encoding: [0x62,0x61,0xfc,0x08,0x5b,0x31]
-          vcvtqq2psx (%rcx), %xmm30
-
-// CHECK: vcvtqq2psx (%rcx), %xmm30 {%k4}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x0c,0x5b,0x31]
-          vcvtqq2psx (%rcx), %xmm30 {%k4}
-
-// CHECK: vcvtqq2psx (%rcx), %xmm30 {%k4} {z}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x8c,0x5b,0x31]
-          vcvtqq2psx (%rcx), %xmm30 {%k4} {z}
-
-// CHECK: vcvtqq2psx 4660(%rax,%r14,8), %xmm30
-// CHECK:  encoding: [0x62,0x21,0xfc,0x08,0x5b,0xb4,0xf0,0x34,0x12,0x00,0x00]
-          vcvtqq2psx 4660(%rax,%r14,8), %xmm30
-
-// CHECK: vcvtqq2psx 4660(%rax,%r14,8), %xmm30 {%k4}
-// CHECK:  encoding: [0x62,0x21,0xfc,0x0c,0x5b,0xb4,0xf0,0x34,0x12,0x00,0x00]
-          vcvtqq2psx 4660(%rax,%r14,8), %xmm30 {%k4}
-
-// CHECK: vcvtqq2psx 4660(%rax,%r14,8), %xmm30 {%k4} {z}
-// CHECK:  encoding: [0x62,0x21,0xfc,0x8c,0x5b,0xb4,0xf0,0x34,0x12,0x00,0x00]
-          vcvtqq2psx 4660(%rax,%r14,8), %xmm30 {%k4} {z}
-
-// CHECK: vcvtqq2ps (%rcx){1to2}, %xmm30
-// CHECK:  encoding: [0x62,0x61,0xfc,0x18,0x5b,0x31]
-          vcvtqq2ps (%rcx){1to2}, %xmm30
-
-// CHECK: vcvtqq2ps (%rcx){1to2}, %xmm30
-// CHECK:  encoding: [0x62,0x61,0xfc,0x18,0x5b,0x31]
-          vcvtqq2psx (%rcx){1to2}, %xmm30
-
-// CHECK: vcvtqq2ps (%rcx){1to2}, %xmm30 {%k4}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x1c,0x5b,0x31]
-          vcvtqq2ps (%rcx){1to2}, %xmm30 {%k4}
-
-// CHECK: vcvtqq2ps (%rcx){1to2}, %xmm30 {%k4}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x1c,0x5b,0x31]
-          vcvtqq2psx (%rcx){1to2}, %xmm30 {%k4}
-
-// CHECK: vcvtqq2ps (%rcx){1to2}, %xmm30 {%k4} {z}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x9c,0x5b,0x31]
-          vcvtqq2ps (%rcx){1to2}, %xmm30 {%k4} {z}
-
-// CHECK: vcvtqq2ps (%rcx){1to2}, %xmm30 {%k4} {z}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x9c,0x5b,0x31]
-          vcvtqq2psx (%rcx){1to2}, %xmm30 {%k4} {z}
-
-// CHECK: vcvtqq2psx 2032(%rdx), %xmm30
-// CHECK:  encoding: [0x62,0x61,0xfc,0x08,0x5b,0x72,0x7f]
-          vcvtqq2psx 2032(%rdx), %xmm30
-
-// CHECK: vcvtqq2psx 2032(%rdx), %xmm30 {%k4}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x0c,0x5b,0x72,0x7f]
-          vcvtqq2psx 2032(%rdx), %xmm30 {%k4}
-
-// CHECK: vcvtqq2psx 2032(%rdx), %xmm30 {%k4} {z}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x8c,0x5b,0x72,0x7f]
-          vcvtqq2psx 2032(%rdx), %xmm30 {%k4} {z}
-
-// CHECK: vcvtqq2psx 2048(%rdx), %xmm30
-// CHECK:  encoding: [0x62,0x61,0xfc,0x08,0x5b,0xb2,0x00,0x08,0x00,0x00]
-          vcvtqq2psx 2048(%rdx), %xmm30
-
-// CHECK: vcvtqq2psx 2048(%rdx), %xmm30 {%k4}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x0c,0x5b,0xb2,0x00,0x08,0x00,0x00]
-          vcvtqq2psx 2048(%rdx), %xmm30 {%k4}
-
-// CHECK: vcvtqq2psx 2048(%rdx), %xmm30 {%k4} {z}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x8c,0x5b,0xb2,0x00,0x08,0x00,0x00]
-          vcvtqq2psx 2048(%rdx), %xmm30 {%k4} {z}
-
-// CHECK: vcvtqq2psx -2048(%rdx), %xmm30
-// CHECK:  encoding: [0x62,0x61,0xfc,0x08,0x5b,0x72,0x80]
-          vcvtqq2psx -2048(%rdx), %xmm30
-
-// CHECK: vcvtqq2psx -2048(%rdx), %xmm30 {%k4}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x0c,0x5b,0x72,0x80]
-          vcvtqq2psx -2048(%rdx), %xmm30 {%k4}
-
-// CHECK: vcvtqq2psx -2048(%rdx), %xmm30 {%k4} {z}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x8c,0x5b,0x72,0x80]
-          vcvtqq2psx -2048(%rdx), %xmm30 {%k4} {z}
-
-// CHECK: vcvtqq2psx -2064(%rdx), %xmm30
-// CHECK:  encoding: [0x62,0x61,0xfc,0x08,0x5b,0xb2,0xf0,0xf7,0xff,0xff]
-          vcvtqq2psx -2064(%rdx), %xmm30
-
-// CHECK: vcvtqq2psx -2064(%rdx), %xmm30 {%k4}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x0c,0x5b,0xb2,0xf0,0xf7,0xff,0xff]
-          vcvtqq2psx -2064(%rdx), %xmm30 {%k4}
-
-// CHECK: vcvtqq2psx -2064(%rdx), %xmm30 {%k4} {z}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x8c,0x5b,0xb2,0xf0,0xf7,0xff,0xff]
-          vcvtqq2psx -2064(%rdx), %xmm30 {%k4} {z}
-
-// CHECK: vcvtqq2ps 1016(%rdx){1to2}, %xmm30
-// CHECK:  encoding: [0x62,0x61,0xfc,0x18,0x5b,0x72,0x7f]
-          vcvtqq2ps 1016(%rdx){1to2}, %xmm30
-
-// CHECK: vcvtqq2ps 1016(%rdx){1to2}, %xmm30
-// CHECK:  encoding: [0x62,0x61,0xfc,0x18,0x5b,0x72,0x7f]
-          vcvtqq2psx 1016(%rdx){1to2}, %xmm30
-
-// CHECK: vcvtqq2ps 1016(%rdx){1to2}, %xmm30 {%k4}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x1c,0x5b,0x72,0x7f]
-          vcvtqq2ps 1016(%rdx){1to2}, %xmm30 {%k4}
-
-// CHECK: vcvtqq2ps 1016(%rdx){1to2}, %xmm30 {%k4}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x1c,0x5b,0x72,0x7f]
-          vcvtqq2psx 1016(%rdx){1to2}, %xmm30 {%k4}
-
-// CHECK: vcvtqq2ps 1016(%rdx){1to2}, %xmm30 {%k4} {z}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x9c,0x5b,0x72,0x7f]
-          vcvtqq2ps 1016(%rdx){1to2}, %xmm30 {%k4} {z}
-
-// CHECK: vcvtqq2ps 1016(%rdx){1to2}, %xmm30 {%k4} {z}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x9c,0x5b,0x72,0x7f]
-          vcvtqq2psx 1016(%rdx){1to2}, %xmm30 {%k4} {z}
-
-// CHECK: vcvtqq2ps 1024(%rdx){1to2}, %xmm30
-// CHECK:  encoding: [0x62,0x61,0xfc,0x18,0x5b,0xb2,0x00,0x04,0x00,0x00]
-          vcvtqq2ps 1024(%rdx){1to2}, %xmm30
-
-// CHECK: vcvtqq2ps 1024(%rdx){1to2}, %xmm30
-// CHECK:  encoding: [0x62,0x61,0xfc,0x18,0x5b,0xb2,0x00,0x04,0x00,0x00]
-          vcvtqq2psx 1024(%rdx){1to2}, %xmm30
-
-// CHECK: vcvtqq2ps 1024(%rdx){1to2}, %xmm30 {%k4}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x1c,0x5b,0xb2,0x00,0x04,0x00,0x00]
-          vcvtqq2ps 1024(%rdx){1to2}, %xmm30 {%k4}
-
-// CHECK: vcvtqq2ps 1024(%rdx){1to2}, %xmm30 {%k4}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x1c,0x5b,0xb2,0x00,0x04,0x00,0x00]
-          vcvtqq2psx 1024(%rdx){1to2}, %xmm30 {%k4}
-
-// CHECK: vcvtqq2ps 1024(%rdx){1to2}, %xmm30 {%k4} {z}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x9c,0x5b,0xb2,0x00,0x04,0x00,0x00]
-          vcvtqq2ps 1024(%rdx){1to2}, %xmm30 {%k4} {z}
-
-// CHECK: vcvtqq2ps 1024(%rdx){1to2}, %xmm30 {%k4} {z}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x9c,0x5b,0xb2,0x00,0x04,0x00,0x00]
-          vcvtqq2psx 1024(%rdx){1to2}, %xmm30 {%k4} {z}
-
-// CHECK: vcvtqq2ps -1024(%rdx){1to2}, %xmm30
-// CHECK:  encoding: [0x62,0x61,0xfc,0x18,0x5b,0x72,0x80]
-          vcvtqq2ps -1024(%rdx){1to2}, %xmm30
-
-// CHECK: vcvtqq2ps -1024(%rdx){1to2}, %xmm30
-// CHECK:  encoding: [0x62,0x61,0xfc,0x18,0x5b,0x72,0x80]
-          vcvtqq2psx -1024(%rdx){1to2}, %xmm30
-
-// CHECK: vcvtqq2ps -1024(%rdx){1to2}, %xmm30 {%k4}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x1c,0x5b,0x72,0x80]
-          vcvtqq2ps -1024(%rdx){1to2}, %xmm30 {%k4}
-
-// CHECK: vcvtqq2ps -1024(%rdx){1to2}, %xmm30 {%k4}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x1c,0x5b,0x72,0x80]
-          vcvtqq2psx -1024(%rdx){1to2}, %xmm30 {%k4}
-
-// CHECK: vcvtqq2ps -1024(%rdx){1to2}, %xmm30 {%k4} {z}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x9c,0x5b,0x72,0x80]
-          vcvtqq2ps -1024(%rdx){1to2}, %xmm30 {%k4} {z}
-
-// CHECK: vcvtqq2ps -1024(%rdx){1to2}, %xmm30 {%k4} {z}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x9c,0x5b,0x72,0x80]
-          vcvtqq2psx -1024(%rdx){1to2}, %xmm30 {%k4} {z}
-
-// CHECK: vcvtqq2ps -1032(%rdx){1to2}, %xmm30
-// CHECK:  encoding: [0x62,0x61,0xfc,0x18,0x5b,0xb2,0xf8,0xfb,0xff,0xff]
-          vcvtqq2ps -1032(%rdx){1to2}, %xmm30
-
-// CHECK: vcvtqq2ps -1032(%rdx){1to2}, %xmm30
-// CHECK:  encoding: [0x62,0x61,0xfc,0x18,0x5b,0xb2,0xf8,0xfb,0xff,0xff]
-          vcvtqq2psx -1032(%rdx){1to2}, %xmm30
-
-// CHECK: vcvtqq2ps -1032(%rdx){1to2}, %xmm30 {%k4}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x1c,0x5b,0xb2,0xf8,0xfb,0xff,0xff]
-          vcvtqq2ps -1032(%rdx){1to2}, %xmm30 {%k4}
-
-// CHECK: vcvtqq2ps -1032(%rdx){1to2}, %xmm30 {%k4}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x1c,0x5b,0xb2,0xf8,0xfb,0xff,0xff]
-          vcvtqq2psx -1032(%rdx){1to2}, %xmm30 {%k4}
-
-// CHECK: vcvtqq2ps -1032(%rdx){1to2}, %xmm30 {%k4} {z}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x9c,0x5b,0xb2,0xf8,0xfb,0xff,0xff]
-          vcvtqq2ps -1032(%rdx){1to2}, %xmm30 {%k4} {z}
-
-// CHECK: vcvtqq2ps -1032(%rdx){1to2}, %xmm30 {%k4} {z}
-// CHECK:  encoding: [0x62,0x61,0xfc,0x9c,0x5b,0xb2,0xf8,0xfb,0xff,0xff]
-          vcvtqq2psx -1032(%rdx){1to2}, %xmm30 {%k4} {z}
-
-// CHECK: vcvtqq2ps %ymm28, %xmm20
-// CHECK:  encoding: [0x62,0x81,0xfc,0x28,0x5b,0xe4]
-          vcvtqq2ps %ymm28, %xmm20
-
-// CHECK: vcvtqq2ps %ymm28, %xmm20
-// CHECK:  encoding: [0x62,0x81,0xfc,0x28,0x5b,0xe4]
-          vcvtqq2psy %ymm28, %xmm20
-
-// CHECK: vcvtqq2ps %ymm28, %xmm20 {%k3}
-// CHECK:  encoding: [0x62,0x81,0xfc,0x2b,0x5b,0xe4]
-          vcvtqq2ps %ymm28, %xmm20 {%k3}
-
-// CHECK: vcvtqq2ps %ymm28, %xmm20 {%k3}
-// CHECK:  encoding: [0x62,0x81,0xfc,0x2b,0x5b,0xe4]
-          vcvtqq2psy %ymm28, %xmm20 {%k3}
-
-// CHECK: vcvtqq2ps %ymm28, %xmm20 {%k3} {z}
-// CHECK:  encoding: [0x62,0x81,0xfc,0xab,0x5b,0xe4]
-          vcvtqq2ps %ymm28, %xmm20 {%k3} {z}
-
-// CHECK: vcvtqq2ps %ymm28, %xmm20 {%k3} {z}
-// CHECK:  encoding: [0x62,0x81,0xfc,0xab,0x5b,0xe4]
-          vcvtqq2psy %ymm28, %xmm20 {%k3} {z}
-
-// CHECK: vcvtqq2psy (%rcx), %xmm20
-// CHECK:  encoding: [0x62,0xe1,0xfc,0x28,0x5b,0x21]
-          vcvtqq2psy (%rcx), %xmm20
-
-// CHECK: vcvtqq2psy (%rcx), %xmm20 {%k3}
-// CHECK:  encoding: [0x62,0xe1,0xfc,0x2b,0x5b,0x21]
-          vcvtqq2psy (%rcx), %xmm20 {%k3}
-
-// CHECK: vcvtqq2psy (%rcx), %xmm20 {%k3} {z}
-// CHECK:  encoding: [0x62,0xe1,0xfc,0xab,0x5b,0x21]
-          vcvtqq2psy (%rcx), %xmm20 {%k3} {z}
-
-// CHECK: vcvtqq2psy 4660(%rax,%r14,8), %xmm20
-// CHECK:  encoding: [0x62,0xa1,0xfc,0x28,0x5b,0xa4,0xf0,0x34,0x12,0x00,0x00]
-          vcvtqq2psy 4660(%rax,%r14,8), %xmm20
-
-// CHECK: vcvtqq2psy 4660(%rax,%r14,8), %xmm20 {%k3}
-// CHECK:  encoding: [0x62,0xa1,0xfc,0x2b,0x5b,0xa4,0xf0,0x34,0x12,0x00,0x00]
-          vcvtqq2psy 4660(%rax,%r14,8), %xmm20 {%k3}
-
-// CHECK: vcvtqq2psy 4660(%rax,%r14,8), %xmm20 {%k3} {z}
-// CHECK:  encoding: [0x62,0xa1,0xfc,0xab,0x5b,0xa4,0xf0,0x34,0x12,0x00,0x00]
-          vcvtqq2psy 4660(%rax,%r14,8), %xmm20 {%k3} {z}
-
-// CHECK: vcvtqq2ps (%rcx){1to4}, %xmm20
-// CHECK:  encoding: [0x62,0xe1,0xfc,0x38,0x5b,0x21]
-          vcvtqq2ps (%rcx){1to4}, %xmm20
-
-// CHECK: vcvtqq2ps (%rcx){1to4}, %xmm20
-// CHECK:  encoding: [0x62,0xe1,0xfc,0x38,0x5b,0x21]
-          vcvtqq2psy (%rcx){1to4}, %xmm20
-
-// CHECK: vcvtqq2ps (%rcx){1to4}, %xmm20 {%k3}
-// CHECK:  encoding: [0x62,0xe1,0xfc,0x3b,0x5b,0x21]
-          vcvtqq2ps (%rcx){1to4}, %xmm20 {%k3}
-
-// CHECK: vcvtqq2ps (%rcx){1to4}, %xmm20 {%k3}
-// CHECK:  encoding: [0x62,0xe1,0xfc,0x3b,0x5b,0x21]
-          vcvtqq2psy (%rcx){1to4}, %xmm20 {%k3}
-
-// CHECK: vcvtqq2ps (%rcx){1to4}, %xmm20 {%k3} {z}
-// CHECK:  encoding: [0x62,0xe1,0xfc,0xbb,0x5b,0x21]
-          vcvtqq2ps (%rcx){1to4}, %xmm20 {%k3} {z}
-
-// CHECK: vcvtqq2ps (%rcx){1to4}, %xmm20 {%k3} {z}
-// CHECK:  encoding: [0x62,0xe1,0xfc,0xbb,0x5b,0x21]
-          vcvtqq2psy (%rcx){1to4}, %xmm20 {%k3} {z}
-
-// CHECK: vcvtqq2psy 4064(%rdx), %xmm20
-// CHECK:  encoding: [0x62,0xe1,0xfc,0x28,0x5b,0x62,0x7f]
-          vcvtqq2psy 4064(%rdx), %xmm20
-
-// CHECK: vcvtqq2psy 4064(%rdx), %xmm20 {%k3}
-// CHECK:  encoding: [0x62,0xe1,0xfc,0x2b,0x5b,0x62,0x7f]
-          vcvtqq2psy 4064(%rdx), %xmm20 {%k3}
-
-// CHECK: vcvtqq2psy 4064(%rdx), %xmm20 {%k3} {z}
-// CHECK:  encoding: [0x62,0xe1,0xfc,0xab,0x5b,0x62,0x7f]
-          vcvtqq2psy 4064(%rdx), %xmm20 {%k3} {z}
-
-// CHECK: vcvtqq2psy 4096(%rdx), %xmm20
-// CHECK:  encoding: [0x62,0xe1,0xfc,0x28,0x5b,0xa2,0x00,0x10,0x00,0x00]
-          vcvtqq2psy 4096(%rdx), %xmm20
-
-// CHECK: vcvtqq2psy 4096(%rdx), %xmm20 {%k3}
-// CHECK:  encoding: [0x62,0xe1,0xfc,0x2b,0x5b,0xa2,0x00,0x10,0x00,0x00]
-          vcvtqq2psy 4096(%rdx), %xmm20 {%k3}
-
-// CHECK: vcvtqq2psy 4096(%rdx), %xmm20 {%k3} {z}
-// CHECK:  encoding: [0x62,0xe1,0xfc,0xab,0x5b,0xa2,0x00,0x10,0x00,0x00]
-          vcvtqq2psy 4096(%rdx), %xmm20 {%k3} {z}
-
-// CHECK: vcvtqq2psy -4096(%rdx), %xmm20
-// CHECK:  encoding: [0x62,0xe1,0xfc,0x28,0x5b,0x62,0x80]
-          vcvtqq2psy -4096(%rdx), %xmm20
-
-// CHECK: vcvtqq2psy -4096(%rdx), %xmm20 {%k3}
-// CHECK:  encoding: [0x62,0xe1,0xfc,0x2b,0x5b,0x62,0x80]
-          vcvtqq2psy -4096(%rdx), %xmm20 {%k3}
-
-// CHECK: vcvtqq2psy -4096(%rdx), %xmm20 {%k3} {z}
-// CHECK:  encoding: [0x62,0xe1,0xfc,0xab,0x5b,0x62,0x80]
-          vcvtqq2psy -4096(%rdx), %xmm20 {%k3} {z}
-
-// CHECK: vcvtqq2psy -4128(%rdx), %xmm20
-// CHECK:  encoding: [0x62,0xe1,0xfc,0x28,0x5b,0xa2,0xe0,0xef,0xff,0xff]
-          vcvtqq2psy -4128(%rdx), %xmm20
-
-// CHECK: vcvtqq2psy -4128(%rdx), %xmm20 {%k3}
-// CHECK:  encoding: [0x62,0xe1,0xfc,0x2b,0x5b,0xa2,0xe0,0xef,0xff,0xff]
-          vcvtqq2psy -4128(%rdx), %xmm20 {%k3}
-
-// CHECK: vcvtqq2psy -4128(%rdx), %xmm20 {%k3} {z}
-// CHECK:  encoding: [0x62,0xe1,0xfc,0xab,0x5b,0xa2,0xe0,0xef,0xff,0xff]
-          vcvtqq2psy -4128(%rdx), %xmm20 {%k3} {z}
-
-// CHECK: vcvtqq2ps 1016(%rdx){1to4}, %xmm20
-// CHECK:  encoding: [0x62,0xe1,0xfc,0x38,0x5b,0x62,0x7f]
-          vcvtqq2ps 1016(%rdx){1to4}, %xmm20
-
-// CHECK: vcvtqq2ps 1016(%rdx){1to4}, %xmm20
-// CHECK:  encoding: [0x62,0xe1,0xfc,0x38,0x5b,0x62,0x7f]
-          vcvtqq2psy 1016(%rdx){1to4}, %xmm20
-
-// CHECK: vcvtqq2ps 1016(%rdx){1to4}, %xmm20 {%k3}
-// CHECK:  encoding: [0x62,0xe1,0xfc,0x3b,0x5b,0x62,0x7f]
-          vcvtqq2ps 1016(%rdx){1to4}, %xmm20 {%k3}
-
-// CHECK: vcvtqq2ps 1016(%rdx){1to4}, %xmm20 {%k3}
-// CHECK:  encoding: [0x62,0xe1,0xfc,0x3b,0x5b,0x62,0x7f]
-          vcvtqq2psy 1016(%rdx){1to4}, %xmm20 {%k3}
-
-// CHECK: vcvtqq2ps 1016(%rdx){1to4}, %xmm20 {%k3} {z}
-// CHECK:  encoding: [0x62,0xe1,0xfc,0xbb,0x5b,0x62,0x7f]
-          vcvtqq2ps 1016(%rdx){1to4}, %xmm20 {%k3} {z}
-
-// CHECK: vcvtqq2ps 1016(%rdx){1to4}, %xmm20 {%k3} {z}
-// CHECK:  encoding: [0x62,0xe1,0xfc,0xbb,0x5b,0x62,0x7f]
-          vcvtqq2psy 1016(%rdx){1to4}, %xmm20 {%k3} {z}
-
-// CHECK: vcvtqq2ps 1024(%rdx){1to4}, %xmm20
-// CHECK:  encoding: [0x62,0xe1,0xfc,0x38,0x5b,0xa2,0x00,0x04,0x00,0x00]
-          vcvtqq2ps 1024(%rdx){1to4}, %xmm20
-
-// CHECK: vcvtqq2ps 1024(%rdx){1to4}, %xmm20
-// CHECK:  encoding: [0x62,0xe1,0xfc,0x38,0x5b,0xa2,0x00,0x04,0x00,0x00]
-          vcvtqq2psy 1024(%rdx){1to4}, %xmm20
-
-// CHECK: vcvtqq2ps 1024(%rdx){1to4}, %xmm20 {%k3}
-// CHECK:  encoding: [0x62,0xe1,0xfc,0x3b,0x5b,0xa2,0x00,0x04,0x00,0x00]
-          vcvtqq2ps 1024(%rdx){1to4}, %xmm20 {%k3}
-
-// CHECK: vcvtqq2ps 1024(%rdx){1to4}, %xmm20 {%k3}
-// CHECK:  encoding: [0x62,0xe1,0xfc,0x3b,0x5b,0xa2,0x00,0x04,0x00,0x00]
-          vcvtqq2psy 1024(%rdx){1to4}, %xmm20 {%k3}
-
-// CHECK: vcvtqq2ps 1024(%rdx){1to4}, %xmm20 {%k3} {z}
-// CHECK:  encoding: [0x62,0xe1,0xfc,0xbb,0x5b,0xa2,0x00,0x04,0x00,0x00]
-          vcvtqq2ps 1024(%rdx){1to4}, %xmm20 {%k3} {z}
-
-// CHECK: vcvtqq2ps 1024(%rdx){1to4}, %xmm20 {%k3} {z}
-// CHECK:  encoding: [0x62,0xe1,0xfc,0xbb,0x5b,0xa2,0x00,0x04,0x00,0x00]
-          vcvtqq2psy 1024(%rdx){1to4}, %xmm20 {%k3} {z}
-
-// CHECK: vcvtqq2ps -1024(%rdx){1to4}, %xmm20
-// CHECK:  encoding: [0x62,0xe1,0xfc,0x38,0x5b,0x62,0x80]
-          vcvtqq2ps -1024(%rdx){1to4}, %xmm20
-
-// CHECK: vcvtqq2ps -1024(%rdx){1to4}, %xmm20
-// CHECK:  encoding: [0x62,0xe1,0xfc,0x38,0x5b,0x62,0x80]
-          vcvtqq2psy -1024(%rdx){1to4}, %xmm20
-
-// CHECK: vcvtqq2ps -1024(%rdx){1to4}, %xmm20 {%k3}
-// CHECK:  encoding: [0x62,0xe1,0xfc,0x3b,0x5b,0x62,0x80]
-          vcvtqq2ps -1024(%rdx){1to4}, %xmm20 {%k3}
-
-// CHECK: vcvtqq2ps -1024(%rdx){1to4}, %xmm20 {%k3}
-// CHECK:  encoding: [0x62,0xe1,0xfc,0x3b,0x5b,0x62,0x80]
-          vcvtqq2psy -1024(%rdx){1to4}, %xmm20 {%k3}
-
-// CHECK: vcvtqq2ps -1024(%rdx){1to4}, %xmm20 {%k3} {z}
-// CHECK:  encoding: [0x62,0xe1,0xfc,0xbb,0x5b,0x62,0x80]
-          vcvtqq2ps -1024(%rdx){1to4}, %xmm20 {%k3} {z}
-
-// CHECK: vcvtqq2ps -1024(%rdx){1to4}, %xmm20 {%k3} {z}
-// CHECK:  encoding: [0x62,0xe1,0xfc,0xbb,0x5b,0x62,0x80]
-          vcvtqq2psy -1024(%rdx){1to4}, %xmm20 {%k3} {z}
-
-// CHECK: vcvtqq2ps -1032(%rdx){1to4}, %xmm20
-// CHECK:  encoding: [0x62,0xe1,0xfc,0x38,0x5b,0xa2,0xf8,0xfb,0xff,0xff]
-          vcvtqq2ps -1032(%rdx){1to4}, %xmm20
-
-// CHECK: vcvtqq2ps -1032(%rdx){1to4}, %xmm20
-// CHECK:  encoding: [0x62,0xe1,0xfc,0x38,0x5b,0xa2,0xf8,0xfb,0xff,0xff]
-          vcvtqq2psy -1032(%rdx){1to4}, %xmm20
-
-// CHECK: vcvtqq2ps -1032(%rdx){1to4}, %xmm20 {%k3}
-// CHECK:  encoding: [0x62,0xe1,0xfc,0x3b,0x5b,0xa2,0xf8,0xfb,0xff,0xff]
-          vcvtqq2ps -1032(%rdx){1to4}, %xmm20 {%k3}
-
-// CHECK: vcvtqq2ps -1032(%rdx){1to4}, %xmm20 {%k3}
-// CHECK:  encoding: [0x62,0xe1,0xfc,0x3b,0x5b,0xa2,0xf8,0xfb,0xff,0xff]
-          vcvtqq2psy -1032(%rdx){1to4}, %xmm20 {%k3}
-
-// CHECK: vcvtqq2ps -1032(%rdx){1to4}, %xmm20 {%k3} {z}
-// CHECK:  encoding: [0x62,0xe1,0xfc,0xbb,0x5b,0xa2,0xf8,0xfb,0xff,0xff]
-          vcvtqq2ps -1032(%rdx){1to4}, %xmm20 {%k3} {z}
-
-// CHECK: vcvtqq2ps -1032(%rdx){1to4}, %xmm20 {%k3} {z}
-// CHECK:  encoding: [0x62,0xe1,0xfc,0xbb,0x5b,0xa2,0xf8,0xfb,0xff,0xff]
-          vcvtqq2psy -1032(%rdx){1to4}, %xmm20 {%k3} {z}
-
-// CHECK: vcvtuqq2pd %xmm20, %xmm19
-// CHECK:  encoding: [0x62,0xa1,0xfe,0x08,0x7a,0xdc]
-          vcvtuqq2pd %xmm20, %xmm19
-
-// CHECK: vcvtuqq2pd %xmm20, %xmm19 {%k3}
-// CHECK:  encoding: [0x62,0xa1,0xfe,0x0b,0x7a,0xdc]
-          vcvtuqq2pd %xmm20, %xmm19 {%k3}
-
-// CHECK: vcvtuqq2pd %xmm20, %xmm19 {%k3} {z}
-// CHECK:  encoding: [0x62,0xa1,0xfe,0x8b,0x7a,0xdc]
-          vcvtuqq2pd %xmm20, %xmm19 {%k3} {z}
-
-// CHECK: vcvtuqq2pd (%rcx), %xmm19
-// CHECK:  encoding: [0x62,0xe1,0xfe,0x08,0x7a,0x19]
-          vcvtuqq2pd (%rcx), %xmm19
-
-// CHECK: vcvtuqq2pd 291(%rax,%r14,8), %xmm19
-// CHECK:  encoding: [0x62,0xa1,0xfe,0x08,0x7a,0x9c,0xf0,0x23,0x01,0x00,0x00]
-          vcvtuqq2pd 291(%rax,%r14,8), %xmm19
-
-// CHECK: vcvtuqq2pd (%rcx){1to2}, %xmm19
-// CHECK:  encoding: [0x62,0xe1,0xfe,0x18,0x7a,0x19]
-          vcvtuqq2pd (%rcx){1to2}, %xmm19
-
-// CHECK: vcvtuqq2pd 2032(%rdx), %xmm19
-// CHECK:  encoding: [0x62,0xe1,0xfe,0x08,0x7a,0x5a,0x7f]
-          vcvtuqq2pd 2032(%rdx), %xmm19
-
-// CHECK: vcvtuqq2pd 2048(%rdx), %xmm19
-// CHECK:  encoding: [0x62,0xe1,0xfe,0x08,0x7a,0x9a,0x00,0x08,0x00,0x00]
-          vcvtuqq2pd 2048(%rdx), %xmm19
-
-// CHECK: vcvtuqq2pd -2048(%rdx), %xmm19
-// CHECK:  encoding: [0x62,0xe1,0xfe,0x08,0x7a,0x5a,0x80]
-          vcvtuqq2pd -2048(%rdx), %xmm19
-
-// CHECK: vcvtuqq2pd -2064(%rdx), %xmm19
-// CHECK:  encoding: [0x62,0xe1,0xfe,0x08,0x7a,0x9a,0xf0,0xf7,0xff,0xff]
-          vcvtuqq2pd -2064(%rdx), %xmm19
-
-// CHECK: vcvtuqq2pd 1016(%rdx){1to2}, %xmm19
-// CHECK:  encoding: [0x62,0xe1,0xfe,0x18,0x7a,0x5a,0x7f]
-          vcvtuqq2pd 1016(%rdx){1to2}, %xmm19
-
-// CHECK: vcvtuqq2pd 1024(%rdx){1to2}, %xmm19
-// CHECK:  encoding: [0x62,0xe1,0xfe,0x18,0x7a,0x9a,0x00,0x04,0x00,0x00]
-          vcvtuqq2pd 1024(%rdx){1to2}, %xmm19
-
-// CHECK: vcvtuqq2pd -1024(%rdx){1to2}, %xmm19
-// CHECK:  encoding: [0x62,0xe1,0xfe,0x18,0x7a,0x5a,0x80]
-          vcvtuqq2pd -1024(%rdx){1to2}, %xmm19
-
-// CHECK: vcvtuqq2pd -1032(%rdx){1to2}, %xmm19
-// CHECK:  encoding: [0x62,0xe1,0xfe,0x18,0x7a,0x9a,0xf8,0xfb,0xff,0xff]
-          vcvtuqq2pd -1032(%rdx){1to2}, %xmm19
-
-// CHECK: vcvtuqq2pd %ymm26, %ymm28
-// CHECK:  encoding: [0x62,0x01,0xfe,0x28,0x7a,0xe2]
-          vcvtuqq2pd %ymm26, %ymm28
-
-// CHECK: vcvtuqq2pd %ymm26, %ymm28 {%k4}
-// CHECK:  encoding: [0x62,0x01,0xfe,0x2c,0x7a,0xe2]
-          vcvtuqq2pd %ymm26, %ymm28 {%k4}
-
-// CHECK: vcvtuqq2pd %ymm26, %ymm28 {%k4} {z}
-// CHECK:  encoding: [0x62,0x01,0xfe,0xac,0x7a,0xe2]
-          vcvtuqq2pd %ymm26, %ymm28 {%k4} {z}
-
-// CHECK: vcvtuqq2pd (%rcx), %ymm28
-// CHECK:  encoding: [0x62,0x61,0xfe,0x28,0x7a,0x21]
-          vcvtuqq2pd (%rcx), %ymm28
-
-// CHECK: vcvtuqq2pd 291(%rax,%r14,8), %ymm28
-// CHECK:  encoding: [0x62,0x21,0xfe,0x28,0x7a,0xa4,0xf0,0x23,0x01,0x00,0x00]
-          vcvtuqq2pd 291(%rax,%r14,8), %ymm28
-
-// CHECK: vcvtuqq2pd (%rcx){1to4}, %ymm28
-// CHECK:  encoding: [0x62,0x61,0xfe,0x38,0x7a,0x21]
-          vcvtuqq2pd (%rcx){1to4}, %ymm28
-
-// CHECK: vcvtuqq2pd 4064(%rdx), %ymm28
-// CHECK:  encoding: [0x62,0x61,0xfe,0x28,0x7a,0x62,0x7f]
-          vcvtuqq2pd 4064(%rdx), %ymm28
-
-// CHECK: vcvtuqq2pd 4096(%rdx), %ymm28
-// CHECK:  encoding: [0x62,0x61,0xfe,0x28,0x7a,0xa2,0x00,0x10,0x00,0x00]
-          vcvtuqq2pd 4096(%rdx), %ymm28
-
-// CHECK: vcvtuqq2pd -4096(%rdx), %ymm28
-// CHECK:  encoding: [0x62,0x61,0xfe,0x28,0x7a,0x62,0x80]
-          vcvtuqq2pd -4096(%rdx), %ymm28
-
-// CHECK: vcvtuqq2pd -4128(%rdx), %ymm28
-// CHECK:  encoding: [0x62,0x61,0xfe,0x28,0x7a,0xa2,0xe0,0xef,0xff,0xff]
-          vcvtuqq2pd -4128(%rdx), %ymm28
-
-// CHECK: vcvtuqq2pd 1016(%rdx){1to4}, %ymm28
-// CHECK:  encoding: [0x62,0x61,0xfe,0x38,0x7a,0x62,0x7f]
-          vcvtuqq2pd 1016(%rdx){1to4}, %ymm28
-
-// CHECK: vcvtuqq2pd 1024(%rdx){1to4}, %ymm28
-// CHECK:  encoding: [0x62,0x61,0xfe,0x38,0x7a,0xa2,0x00,0x04,0x00,0x00]
-          vcvtuqq2pd 1024(%rdx){1to4}, %ymm28
-
-// CHECK: vcvtuqq2pd -1024(%rdx){1to4}, %ymm28
-// CHECK:  encoding: [0x62,0x61,0xfe,0x38,0x7a,0x62,0x80]
-          vcvtuqq2pd -1024(%rdx){1to4}, %ymm28
-
-// CHECK: vcvtuqq2pd -1032(%rdx){1to4}, %ymm28
-// CHECK:  encoding: [0x62,0x61,0xfe,0x38,0x7a,0xa2,0xf8,0xfb,0xff,0xff]
-          vcvtuqq2pd -1032(%rdx){1to4}, %ymm28
-
-// CHECK: vcvtuqq2ps %xmm27, %xmm21
-// CHECK:  encoding: [0x62,0x81,0xff,0x08,0x7a,0xeb]
-          vcvtuqq2ps %xmm27, %xmm21
-
-// CHECK: vcvtuqq2ps %xmm27, %xmm21
-// CHECK:  encoding: [0x62,0x81,0xff,0x08,0x7a,0xeb]
-          vcvtuqq2psx %xmm27, %xmm21
-
-// CHECK: vcvtuqq2ps %xmm27, %xmm21 {%k7}
-// CHECK:  encoding: [0x62,0x81,0xff,0x0f,0x7a,0xeb]
-          vcvtuqq2ps %xmm27, %xmm21 {%k7}
-
-// CHECK: vcvtuqq2ps %xmm27, %xmm21 {%k7}
-// CHECK:  encoding: [0x62,0x81,0xff,0x0f,0x7a,0xeb]
-          vcvtuqq2psx %xmm27, %xmm21 {%k7}
-
-// CHECK: vcvtuqq2ps %xmm27, %xmm21 {%k7} {z}
-// CHECK:  encoding: [0x62,0x81,0xff,0x8f,0x7a,0xeb]
-          vcvtuqq2ps %xmm27, %xmm21 {%k7} {z}
-
-// CHECK: vcvtuqq2ps %xmm27, %xmm21 {%k7} {z}
-// CHECK:  encoding: [0x62,0x81,0xff,0x8f,0x7a,0xeb]
-          vcvtuqq2psx %xmm27, %xmm21 {%k7} {z}
-
-// CHECK: vcvtuqq2psx (%rcx), %xmm21
-// CHECK:  encoding: [0x62,0xe1,0xff,0x08,0x7a,0x29]
-          vcvtuqq2psx (%rcx), %xmm21
-
-// CHECK: vcvtuqq2psx (%rcx), %xmm21 {%k7}
-// CHECK:  encoding: [0x62,0xe1,0xff,0x0f,0x7a,0x29]
-          vcvtuqq2psx (%rcx), %xmm21 {%k7}
-
-// CHECK: vcvtuqq2psx (%rcx), %xmm21 {%k7} {z}
-// CHECK:  encoding: [0x62,0xe1,0xff,0x8f,0x7a,0x29]
-          vcvtuqq2psx (%rcx), %xmm21 {%k7} {z}
-
-// CHECK: vcvtuqq2psx 291(%rax,%r14,8), %xmm21
-// CHECK:  encoding: [0x62,0xa1,0xff,0x08,0x7a,0xac,0xf0,0x23,0x01,0x00,0x00]
-          vcvtuqq2psx 291(%rax,%r14,8), %xmm21
-
-// CHECK: vcvtuqq2psx 291(%rax,%r14,8), %xmm21 {%k7}
-// CHECK:  encoding: [0x62,0xa1,0xff,0x0f,0x7a,0xac,0xf0,0x23,0x01,0x00,0x00]
-          vcvtuqq2psx 291(%rax,%r14,8), %xmm21 {%k7}
-
-// CHECK: vcvtuqq2psx 291(%rax,%r14,8), %xmm21 {%k7} {z}
-// CHECK:  encoding: [0x62,0xa1,0xff,0x8f,0x7a,0xac,0xf0,0x23,0x01,0x00,0x00]
-          vcvtuqq2psx 291(%rax,%r14,8), %xmm21 {%k7} {z}
-
-// CHECK: vcvtuqq2ps (%rcx){1to2}, %xmm21
-// CHECK:  encoding: [0x62,0xe1,0xff,0x18,0x7a,0x29]
-          vcvtuqq2ps (%rcx){1to2}, %xmm21
-
-// CHECK: vcvtuqq2ps (%rcx){1to2}, %xmm21
-// CHECK:  encoding: [0x62,0xe1,0xff,0x18,0x7a,0x29]
-          vcvtuqq2psx (%rcx){1to2}, %xmm21
-
-// CHECK: vcvtuqq2ps (%rcx){1to2}, %xmm21 {%k7}
-// CHECK:  encoding: [0x62,0xe1,0xff,0x1f,0x7a,0x29]
-          vcvtuqq2ps (%rcx){1to2}, %xmm21 {%k7}
-
-// CHECK: vcvtuqq2ps (%rcx){1to2}, %xmm21 {%k7}
-// CHECK:  encoding: [0x62,0xe1,0xff,0x1f,0x7a,0x29]
-          vcvtuqq2psx (%rcx){1to2}, %xmm21 {%k7}
-
-// CHECK: vcvtuqq2ps (%rcx){1to2}, %xmm21 {%k7} {z}
-// CHECK:  encoding: [0x62,0xe1,0xff,0x9f,0x7a,0x29]
-          vcvtuqq2ps (%rcx){1to2}, %xmm21 {%k7} {z}
-
-// CHECK: vcvtuqq2ps (%rcx){1to2}, %xmm21 {%k7} {z}
-// CHECK:  encoding: [0x62,0xe1,0xff,0x9f,0x7a,0x29]
-          vcvtuqq2psx (%rcx){1to2}, %xmm21 {%k7} {z}
-
-// CHECK: vcvtuqq2psx 2032(%rdx), %xmm21
-// CHECK:  encoding: [0x62,0xe1,0xff,0x08,0x7a,0x6a,0x7f]
-          vcvtuqq2psx 2032(%rdx), %xmm21
-
-// CHECK: vcvtuqq2psx 2032(%rdx), %xmm21 {%k7}
-// CHECK:  encoding: [0x62,0xe1,0xff,0x0f,0x7a,0x6a,0x7f]
-          vcvtuqq2psx 2032(%rdx), %xmm21 {%k7}
-
-// CHECK: vcvtuqq2psx 2032(%rdx), %xmm21 {%k7} {z}
-// CHECK:  encoding: [0x62,0xe1,0xff,0x8f,0x7a,0x6a,0x7f]
-          vcvtuqq2psx 2032(%rdx), %xmm21 {%k7} {z}
-
-// CHECK: vcvtuqq2psx 2048(%rdx), %xmm21
-// CHECK:  encoding: [0x62,0xe1,0xff,0x08,0x7a,0xaa,0x00,0x08,0x00,0x00]
-          vcvtuqq2psx 2048(%rdx), %xmm21
-
-// CHECK: vcvtuqq2psx 2048(%rdx), %xmm21 {%k7}
-// CHECK:  encoding: [0x62,0xe1,0xff,0x0f,0x7a,0xaa,0x00,0x08,0x00,0x00]
-          vcvtuqq2psx 2048(%rdx), %xmm21 {%k7}
-
-// CHECK: vcvtuqq2psx 2048(%rdx), %xmm21 {%k7} {z}
-// CHECK:  encoding: [0x62,0xe1,0xff,0x8f,0x7a,0xaa,0x00,0x08,0x00,0x00]
-          vcvtuqq2psx 2048(%rdx), %xmm21 {%k7} {z}
-
-// CHECK: vcvtuqq2psx -2048(%rdx), %xmm21
-// CHECK:  encoding: [0x62,0xe1,0xff,0x08,0x7a,0x6a,0x80]
-          vcvtuqq2psx -2048(%rdx), %xmm21
-
-// CHECK: vcvtuqq2psx -2048(%rdx), %xmm21 {%k7}
-// CHECK:  encoding: [0x62,0xe1,0xff,0x0f,0x7a,0x6a,0x80]
-          vcvtuqq2psx -2048(%rdx), %xmm21 {%k7}
-
-// CHECK: vcvtuqq2psx -2048(%rdx), %xmm21 {%k7} {z}
-// CHECK:  encoding: [0x62,0xe1,0xff,0x8f,0x7a,0x6a,0x80]
-          vcvtuqq2psx -2048(%rdx), %xmm21 {%k7} {z}
-
-// CHECK: vcvtuqq2psx -2064(%rdx), %xmm21
-// CHECK:  encoding: [0x62,0xe1,0xff,0x08,0x7a,0xaa,0xf0,0xf7,0xff,0xff]
-          vcvtuqq2psx -2064(%rdx), %xmm21
-
-// CHECK: vcvtuqq2psx -2064(%rdx), %xmm21 {%k7}
-// CHECK:  encoding: [0x62,0xe1,0xff,0x0f,0x7a,0xaa,0xf0,0xf7,0xff,0xff]
-          vcvtuqq2psx -2064(%rdx), %xmm21 {%k7}
-
-// CHECK: vcvtuqq2psx -2064(%rdx), %xmm21 {%k7} {z}
-// CHECK:  encoding: [0x62,0xe1,0xff,0x8f,0x7a,0xaa,0xf0,0xf7,0xff,0xff]
-          vcvtuqq2psx -2064(%rdx), %xmm21 {%k7} {z}
-
-// CHECK: vcvtuqq2ps 1016(%rdx){1to2}, %xmm21
-// CHECK:  encoding: [0x62,0xe1,0xff,0x18,0x7a,0x6a,0x7f]
-          vcvtuqq2ps 1016(%rdx){1to2}, %xmm21
-
-// CHECK: vcvtuqq2ps 1016(%rdx){1to2}, %xmm21
-// CHECK:  encoding: [0x62,0xe1,0xff,0x18,0x7a,0x6a,0x7f]
-          vcvtuqq2psx 1016(%rdx){1to2}, %xmm21
-
-// CHECK: vcvtuqq2ps 1016(%rdx){1to2}, %xmm21 {%k7}
-// CHECK:  encoding: [0x62,0xe1,0xff,0x1f,0x7a,0x6a,0x7f]
-          vcvtuqq2ps 1016(%rdx){1to2}, %xmm21 {%k7}
-
-// CHECK: vcvtuqq2ps 1016(%rdx){1to2}, %xmm21 {%k7}
-// CHECK:  encoding: [0x62,0xe1,0xff,0x1f,0x7a,0x6a,0x7f]
-          vcvtuqq2psx 1016(%rdx){1to2}, %xmm21 {%k7}
-
-// CHECK: vcvtuqq2ps 1016(%rdx){1to2}, %xmm21 {%k7} {z}
-// CHECK:  encoding: [0x62,0xe1,0xff,0x9f,0x7a,0x6a,0x7f]
-          vcvtuqq2ps 1016(%rdx){1to2}, %xmm21 {%k7} {z}
-
-// CHECK: vcvtuqq2ps 1016(%rdx){1to2}, %xmm21 {%k7} {z}
-// CHECK:  encoding: [0x62,0xe1,0xff,0x9f,0x7a,0x6a,0x7f]
-          vcvtuqq2psx 1016(%rdx){1to2}, %xmm21 {%k7} {z}
-
-// CHECK: vcvtuqq2ps 1024(%rdx){1to2}, %xmm21
-// CHECK:  encoding: [0x62,0xe1,0xff,0x18,0x7a,0xaa,0x00,0x04,0x00,0x00]
-          vcvtuqq2ps 1024(%rdx){1to2}, %xmm21
-
-// CHECK: vcvtuqq2ps 1024(%rdx){1to2}, %xmm21
-// CHECK:  encoding: [0x62,0xe1,0xff,0x18,0x7a,0xaa,0x00,0x04,0x00,0x00]
-          vcvtuqq2psx 1024(%rdx){1to2}, %xmm21
-
-// CHECK: vcvtuqq2ps 1024(%rdx){1to2}, %xmm21 {%k7}
-// CHECK:  encoding: [0x62,0xe1,0xff,0x1f,0x7a,0xaa,0x00,0x04,0x00,0x00]
-          vcvtuqq2ps 1024(%rdx){1to2}, %xmm21 {%k7}
-
-// CHECK: vcvtuqq2ps 1024(%rdx){1to2}, %xmm21 {%k7}
-// CHECK:  encoding: [0x62,0xe1,0xff,0x1f,0x7a,0xaa,0x00,0x04,0x00,0x00]
-          vcvtuqq2psx 1024(%rdx){1to2}, %xmm21 {%k7}
-
-// CHECK: vcvtuqq2ps 1024(%rdx){1to2}, %xmm21 {%k7} {z}
-// CHECK:  encoding: [0x62,0xe1,0xff,0x9f,0x7a,0xaa,0x00,0x04,0x00,0x00]
-          vcvtuqq2ps 1024(%rdx){1to2}, %xmm21 {%k7} {z}
-
-// CHECK: vcvtuqq2ps 1024(%rdx){1to2}, %xmm21 {%k7} {z}
-// CHECK:  encoding: [0x62,0xe1,0xff,0x9f,0x7a,0xaa,0x00,0x04,0x00,0x00]
-          vcvtuqq2psx 1024(%rdx){1to2}, %xmm21 {%k7} {z}
-
-// CHECK: vcvtuqq2ps -1024(%rdx){1to2}, %xmm21
-// CHECK:  encoding: [0x62,0xe1,0xff,0x18,0x7a,0x6a,0x80]
-          vcvtuqq2ps -1024(%rdx){1to2}, %xmm21
-
-// CHECK: vcvtuqq2ps -1024(%rdx){1to2}, %xmm21
-// CHECK:  encoding: [0x62,0xe1,0xff,0x18,0x7a,0x6a,0x80]
-          vcvtuqq2psx -1024(%rdx){1to2}, %xmm21
-
-// CHECK: vcvtuqq2ps -1024(%rdx){1to2}, %xmm21 {%k7}
-// CHECK:  encoding: [0x62,0xe1,0xff,0x1f,0x7a,0x6a,0x80]
-          vcvtuqq2ps -1024(%rdx){1to2}, %xmm21 {%k7}
-
-// CHECK: vcvtuqq2ps -1024(%rdx){1to2}, %xmm21 {%k7}
-// CHECK:  encoding: [0x62,0xe1,0xff,0x1f,0x7a,0x6a,0x80]
-          vcvtuqq2psx -1024(%rdx){1to2}, %xmm21 {%k7}
-
-// CHECK: vcvtuqq2ps -1024(%rdx){1to2}, %xmm21 {%k7} {z}
-// CHECK:  encoding: [0x62,0xe1,0xff,0x9f,0x7a,0x6a,0x80]
-          vcvtuqq2ps -1024(%rdx){1to2}, %xmm21 {%k7} {z}
-
-// CHECK: vcvtuqq2ps -1024(%rdx){1to2}, %xmm21 {%k7} {z}
-// CHECK:  encoding: [0x62,0xe1,0xff,0x9f,0x7a,0x6a,0x80]
-          vcvtuqq2psx -1024(%rdx){1to2}, %xmm21 {%k7} {z}
-
-// CHECK: vcvtuqq2ps -1032(%rdx){1to2}, %xmm21
-// CHECK:  encoding: [0x62,0xe1,0xff,0x18,0x7a,0xaa,0xf8,0xfb,0xff,0xff]
-          vcvtuqq2ps -1032(%rdx){1to2}, %xmm21
-
-// CHECK: vcvtuqq2ps -1032(%rdx){1to2}, %xmm21
-// CHECK:  encoding: [0x62,0xe1,0xff,0x18,0x7a,0xaa,0xf8,0xfb,0xff,0xff]
-          vcvtuqq2psx -1032(%rdx){1to2}, %xmm21
-
-// CHECK: vcvtuqq2ps -1032(%rdx){1to2}, %xmm21 {%k7}
-// CHECK:  encoding: [0x62,0xe1,0xff,0x1f,0x7a,0xaa,0xf8,0xfb,0xff,0xff]
-          vcvtuqq2ps -1032(%rdx){1to2}, %xmm21 {%k7}
-
-// CHECK: vcvtuqq2ps -1032(%rdx){1to2}, %xmm21 {%k7}
-// CHECK:  encoding: [0x62,0xe1,0xff,0x1f,0x7a,0xaa,0xf8,0xfb,0xff,0xff]
-          vcvtuqq2psx -1032(%rdx){1to2}, %xmm21 {%k7}
-
-// CHECK: vcvtuqq2ps -1032(%rdx){1to2}, %xmm21 {%k7} {z}
-// CHECK:  encoding: [0x62,0xe1,0xff,0x9f,0x7a,0xaa,0xf8,0xfb,0xff,0xff]
-          vcvtuqq2ps -1032(%rdx){1to2}, %xmm21 {%k7} {z}
-
-// CHECK: vcvtuqq2ps -1032(%rdx){1to2}, %xmm21 {%k7} {z}
-// CHECK:  encoding: [0x62,0xe1,0xff,0x9f,0x7a,0xaa,0xf8,0xfb,0xff,0xff]
-          vcvtuqq2psx -1032(%rdx){1to2}, %xmm21 {%k7} {z}
-
-// CHECK: vcvtuqq2ps %ymm24, %xmm28
-// CHECK:  encoding: [0x62,0x01,0xff,0x28,0x7a,0xe0]
-          vcvtuqq2ps %ymm24, %xmm28
-
-// CHECK: vcvtuqq2ps %ymm24, %xmm28
-// CHECK:  encoding: [0x62,0x01,0xff,0x28,0x7a,0xe0]
-          vcvtuqq2psy %ymm24, %xmm28
-
-// CHECK: vcvtuqq2ps %ymm24, %xmm28 {%k3}
-// CHECK:  encoding: [0x62,0x01,0xff,0x2b,0x7a,0xe0]
-          vcvtuqq2ps %ymm24, %xmm28 {%k3}
-
-// CHECK: vcvtuqq2ps %ymm24, %xmm28 {%k3}
-// CHECK:  encoding: [0x62,0x01,0xff,0x2b,0x7a,0xe0]
-          vcvtuqq2psy %ymm24, %xmm28 {%k3}
-
-// CHECK: vcvtuqq2ps %ymm24, %xmm28 {%k3} {z}
-// CHECK:  encoding: [0x62,0x01,0xff,0xab,0x7a,0xe0]
-          vcvtuqq2ps %ymm24, %xmm28 {%k3} {z}
-
-// CHECK: vcvtuqq2ps %ymm24, %xmm28 {%k3} {z}
-// CHECK:  encoding: [0x62,0x01,0xff,0xab,0x7a,0xe0]
-          vcvtuqq2psy %ymm24, %xmm28 {%k3} {z}
-
-// CHECK: vcvtuqq2psy (%rcx), %xmm28
-// CHECK:  encoding: [0x62,0x61,0xff,0x28,0x7a,0x21]
-          vcvtuqq2psy (%rcx), %xmm28
-
-// CHECK: vcvtuqq2psy (%rcx), %xmm28 {%k3}
-// CHECK:  encoding: [0x62,0x61,0xff,0x2b,0x7a,0x21]
-          vcvtuqq2psy (%rcx), %xmm28 {%k3}
-
-// CHECK: vcvtuqq2psy (%rcx), %xmm28 {%k3} {z}
-// CHECK:  encoding: [0x62,0x61,0xff,0xab,0x7a,0x21]
-          vcvtuqq2psy (%rcx), %xmm28 {%k3} {z}
-
-// CHECK: vcvtuqq2psy 291(%rax,%r14,8), %xmm28
-// CHECK:  encoding: [0x62,0x21,0xff,0x28,0x7a,0xa4,0xf0,0x23,0x01,0x00,0x00]
-          vcvtuqq2psy 291(%rax,%r14,8), %xmm28
-
-// CHECK: vcvtuqq2psy 291(%rax,%r14,8), %xmm28 {%k3}
-// CHECK:  encoding: [0x62,0x21,0xff,0x2b,0x7a,0xa4,0xf0,0x23,0x01,0x00,0x00]
-          vcvtuqq2psy 291(%rax,%r14,8), %xmm28 {%k3}
-
-// CHECK: vcvtuqq2psy 291(%rax,%r14,8), %xmm28 {%k3} {z}
-// CHECK:  encoding: [0x62,0x21,0xff,0xab,0x7a,0xa4,0xf0,0x23,0x01,0x00,0x00]
-          vcvtuqq2psy 291(%rax,%r14,8), %xmm28 {%k3} {z}
-
-// CHECK: vcvtuqq2ps (%rcx){1to4}, %xmm28
-// CHECK:  encoding: [0x62,0x61,0xff,0x38,0x7a,0x21]
-          vcvtuqq2ps (%rcx){1to4}, %xmm28
-
-// CHECK: vcvtuqq2ps (%rcx){1to4}, %xmm28
-// CHECK:  encoding: [0x62,0x61,0xff,0x38,0x7a,0x21]
-          vcvtuqq2psy (%rcx){1to4}, %xmm28
-
-// CHECK: vcvtuqq2ps (%rcx){1to4}, %xmm28 {%k3}
-// CHECK:  encoding: [0x62,0x61,0xff,0x3b,0x7a,0x21]
-          vcvtuqq2ps (%rcx){1to4}, %xmm28 {%k3}
-
-// CHECK: vcvtuqq2ps (%rcx){1to4}, %xmm28 {%k3}
-// CHECK:  encoding: [0x62,0x61,0xff,0x3b,0x7a,0x21]
-          vcvtuqq2psy (%rcx){1to4}, %xmm28 {%k3}
-
-// CHECK: vcvtuqq2ps (%rcx){1to4}, %xmm28 {%k3} {z}
-// CHECK:  encoding: [0x62,0x61,0xff,0xbb,0x7a,0x21]
-          vcvtuqq2ps (%rcx){1to4}, %xmm28 {%k3} {z}
-
-// CHECK: vcvtuqq2ps (%rcx){1to4}, %xmm28 {%k3} {z}
-// CHECK:  encoding: [0x62,0x61,0xff,0xbb,0x7a,0x21]
-          vcvtuqq2psy (%rcx){1to4}, %xmm28 {%k3} {z}
-
-// CHECK: vcvtuqq2psy 4064(%rdx), %xmm28
-// CHECK:  encoding: [0x62,0x61,0xff,0x28,0x7a,0x62,0x7f]
-          vcvtuqq2psy 4064(%rdx), %xmm28
-
-// CHECK: vcvtuqq2psy 4064(%rdx), %xmm28 {%k3}
-// CHECK:  encoding: [0x62,0x61,0xff,0x2b,0x7a,0x62,0x7f]
-          vcvtuqq2psy 4064(%rdx), %xmm28 {%k3}
-
-// CHECK: vcvtuqq2psy 4064(%rdx), %xmm28 {%k3} {z}
-// CHECK:  encoding: [0x62,0x61,0xff,0xab,0x7a,0x62,0x7f]
-          vcvtuqq2psy 4064(%rdx), %xmm28 {%k3} {z}
-
-// CHECK: vcvtuqq2psy 4096(%rdx), %xmm28
-// CHECK:  encoding: [0x62,0x61,0xff,0x28,0x7a,0xa2,0x00,0x10,0x00,0x00]
-          vcvtuqq2psy 4096(%rdx), %xmm28
-
-// CHECK: vcvtuqq2psy 4096(%rdx), %xmm28 {%k3}
-// CHECK:  encoding: [0x62,0x61,0xff,0x2b,0x7a,0xa2,0x00,0x10,0x00,0x00]
-          vcvtuqq2psy 4096(%rdx), %xmm28 {%k3}
-
-// CHECK: vcvtuqq2psy 4096(%rdx), %xmm28 {%k3} {z}
-// CHECK:  encoding: [0x62,0x61,0xff,0xab,0x7a,0xa2,0x00,0x10,0x00,0x00]
-          vcvtuqq2psy 4096(%rdx), %xmm28 {%k3} {z}
-
-// CHECK: vcvtuqq2psy -4096(%rdx), %xmm28
-// CHECK:  encoding: [0x62,0x61,0xff,0x28,0x7a,0x62,0x80]
-          vcvtuqq2psy -4096(%rdx), %xmm28
-
-// CHECK: vcvtuqq2psy -4096(%rdx), %xmm28 {%k3}
-// CHECK:  encoding: [0x62,0x61,0xff,0x2b,0x7a,0x62,0x80]
-          vcvtuqq2psy -4096(%rdx), %xmm28 {%k3}
-
-// CHECK: vcvtuqq2psy -4096(%rdx), %xmm28 {%k3} {z}
-// CHECK:  encoding: [0x62,0x61,0xff,0xab,0x7a,0x62,0x80]
-          vcvtuqq2psy -4096(%rdx), %xmm28 {%k3} {z}
-
-// CHECK: vcvtuqq2psy -4128(%rdx), %xmm28
-// CHECK:  encoding: [0x62,0x61,0xff,0x28,0x7a,0xa2,0xe0,0xef,0xff,0xff]
-          vcvtuqq2psy -4128(%rdx), %xmm28
-
-// CHECK: vcvtuqq2psy -4128(%rdx), %xmm28 {%k3}
-// CHECK:  encoding: [0x62,0x61,0xff,0x2b,0x7a,0xa2,0xe0,0xef,0xff,0xff]
-          vcvtuqq2psy -4128(%rdx), %xmm28 {%k3}
-
-// CHECK: vcvtuqq2psy -4128(%rdx), %xmm28 {%k3} {z}
-// CHECK:  encoding: [0x62,0x61,0xff,0xab,0x7a,0xa2,0xe0,0xef,0xff,0xff]
-          vcvtuqq2psy -4128(%rdx), %xmm28 {%k3} {z}
-
-// CHECK: vcvtuqq2ps 1016(%rdx){1to4}, %xmm28
-// CHECK:  encoding: [0x62,0x61,0xff,0x38,0x7a,0x62,0x7f]
-          vcvtuqq2ps 1016(%rdx){1to4}, %xmm28
-
-// CHECK: vcvtuqq2ps 1016(%rdx){1to4}, %xmm28
-// CHECK:  encoding: [0x62,0x61,0xff,0x38,0x7a,0x62,0x7f]
-          vcvtuqq2psy 1016(%rdx){1to4}, %xmm28
-
-// CHECK: vcvtuqq2ps 1016(%rdx){1to4}, %xmm28 {%k3}
-// CHECK:  encoding: [0x62,0x61,0xff,0x3b,0x7a,0x62,0x7f]
-          vcvtuqq2ps 1016(%rdx){1to4}, %xmm28 {%k3}
-
-// CHECK: vcvtuqq2ps 1016(%rdx){1to4}, %xmm28 {%k3}
-// CHECK:  encoding: [0x62,0x61,0xff,0x3b,0x7a,0x62,0x7f]
-          vcvtuqq2psy 1016(%rdx){1to4}, %xmm28 {%k3}
-
-// CHECK: vcvtuqq2ps 1016(%rdx){1to4}, %xmm28 {%k3} {z}
-// CHECK:  encoding: [0x62,0x61,0xff,0xbb,0x7a,0x62,0x7f]
-          vcvtuqq2ps 1016(%rdx){1to4}, %xmm28 {%k3} {z}
-
-// CHECK: vcvtuqq2ps 1016(%rdx){1to4}, %xmm28 {%k3} {z}
-// CHECK:  encoding: [0x62,0x61,0xff,0xbb,0x7a,0x62,0x7f]
-          vcvtuqq2psy 1016(%rdx){1to4}, %xmm28 {%k3} {z}
-
-// CHECK: vcvtuqq2ps 1024(%rdx){1to4}, %xmm28
-// CHECK:  encoding: [0x62,0x61,0xff,0x38,0x7a,0xa2,0x00,0x04,0x00,0x00]
-          vcvtuqq2ps 1024(%rdx){1to4}, %xmm28
-
-// CHECK: vcvtuqq2ps 1024(%rdx){1to4}, %xmm28
-// CHECK:  encoding: [0x62,0x61,0xff,0x38,0x7a,0xa2,0x00,0x04,0x00,0x00]
-          vcvtuqq2psy 1024(%rdx){1to4}, %xmm28
-
-// CHECK: vcvtuqq2ps 1024(%rdx){1to4}, %xmm28 {%k3}
-// CHECK:  encoding: [0x62,0x61,0xff,0x3b,0x7a,0xa2,0x00,0x04,0x00,0x00]
-          vcvtuqq2ps 1024(%rdx){1to4}, %xmm28 {%k3}
-
-// CHECK: vcvtuqq2ps 1024(%rdx){1to4}, %xmm28 {%k3}
-// CHECK:  encoding: [0x62,0x61,0xff,0x3b,0x7a,0xa2,0x00,0x04,0x00,0x00]
-          vcvtuqq2psy 1024(%rdx){1to4}, %xmm28 {%k3}
-
-// CHECK: vcvtuqq2ps 1024(%rdx){1to4}, %xmm28 {%k3} {z}
-// CHECK:  encoding: [0x62,0x61,0xff,0xbb,0x7a,0xa2,0x00,0x04,0x00,0x00]
-          vcvtuqq2ps 1024(%rdx){1to4}, %xmm28 {%k3} {z}
-
-// CHECK: vcvtuqq2ps 1024(%rdx){1to4}, %xmm28 {%k3} {z}
-// CHECK:  encoding: [0x62,0x61,0xff,0xbb,0x7a,0xa2,0x00,0x04,0x00,0x00]
-          vcvtuqq2psy 1024(%rdx){1to4}, %xmm28 {%k3} {z}
-
-// CHECK: vcvtuqq2ps -1024(%rdx){1to4}, %xmm28
-// CHECK:  encoding: [0x62,0x61,0xff,0x38,0x7a,0x62,0x80]
-          vcvtuqq2ps -1024(%rdx){1to4}, %xmm28
-
-// CHECK: vcvtuqq2ps -1024(%rdx){1to4}, %xmm28
-// CHECK:  encoding: [0x62,0x61,0xff,0x38,0x7a,0x62,0x80]
-          vcvtuqq2psy -1024(%rdx){1to4}, %xmm28
-
-// CHECK: vcvtuqq2ps -1024(%rdx){1to4}, %xmm28 {%k3}
-// CHECK:  encoding: [0x62,0x61,0xff,0x3b,0x7a,0x62,0x80]
-          vcvtuqq2ps -1024(%rdx){1to4}, %xmm28 {%k3}
-
-// CHECK: vcvtuqq2ps -1024(%rdx){1to4}, %xmm28 {%k3}
-// CHECK:  encoding: [0x62,0x61,0xff,0x3b,0x7a,0x62,0x80]
-          vcvtuqq2psy -1024(%rdx){1to4}, %xmm28 {%k3}
-
-// CHECK: vcvtuqq2ps -1024(%rdx){1to4}, %xmm28 {%k3} {z}
-// CHECK:  encoding: [0x62,0x61,0xff,0xbb,0x7a,0x62,0x80]
-          vcvtuqq2ps -1024(%rdx){1to4}, %xmm28 {%k3} {z}
-
-// CHECK: vcvtuqq2ps -1024(%rdx){1to4}, %xmm28 {%k3} {z}
-// CHECK:  encoding: [0x62,0x61,0xff,0xbb,0x7a,0x62,0x80]
-          vcvtuqq2psy -1024(%rdx){1to4}, %xmm28 {%k3} {z}
-
-// CHECK: vcvtuqq2ps -1032(%rdx){1to4}, %xmm28
-// CHECK:  encoding: [0x62,0x61,0xff,0x38,0x7a,0xa2,0xf8,0xfb,0xff,0xff]
-          vcvtuqq2ps -1032(%rdx){1to4}, %xmm28
-
-// CHECK: vcvtuqq2ps -1032(%rdx){1to4}, %xmm28
-// CHECK:  encoding: [0x62,0x61,0xff,0x38,0x7a,0xa2,0xf8,0xfb,0xff,0xff]
-          vcvtuqq2psy -1032(%rdx){1to4}, %xmm28
-
-// CHECK: vcvtuqq2ps -1032(%rdx){1to4}, %xmm28 {%k3}
-// CHECK:  encoding: [0x62,0x61,0xff,0x3b,0x7a,0xa2,0xf8,0xfb,0xff,0xff]
-          vcvtuqq2ps -1032(%rdx){1to4}, %xmm28 {%k3}
-
-// CHECK: vcvtuqq2ps -1032(%rdx){1to4}, %xmm28 {%k3}
-// CHECK:  encoding: [0x62,0x61,0xff,0x3b,0x7a,0xa2,0xf8,0xfb,0xff,0xff]
-          vcvtuqq2psy -1032(%rdx){1to4}, %xmm28 {%k3}
-
-// CHECK: vcvtuqq2ps -1032(%rdx){1to4}, %xmm28 {%k3} {z}
-// CHECK:  encoding: [0x62,0x61,0xff,0xbb,0x7a,0xa2,0xf8,0xfb,0xff,0xff]
-          vcvtuqq2ps -1032(%rdx){1to4}, %xmm28 {%k3} {z}
-
-// CHECK: vcvtuqq2ps -1032(%rdx){1to4}, %xmm28 {%k3} {z}
-// CHECK:  encoding: [0x62,0x61,0xff,0xbb,0x7a,0xa2,0xf8,0xfb,0xff,0xff]
-          vcvtuqq2psy -1032(%rdx){1to4}, %xmm28 {%k3} {z}
-
-// CHECK: vcvtuqq2ps %xmm22, %xmm21
-// CHECK:  encoding: [0x62,0xa1,0xff,0x08,0x7a,0xee]
-          vcvtuqq2ps %xmm22, %xmm21
-
-// CHECK: vcvtuqq2ps %xmm22, %xmm21
-// CHECK:  encoding: [0x62,0xa1,0xff,0x08,0x7a,0xee]
-          vcvtuqq2psx %xmm22, %xmm21
-
-// CHECK: vcvtuqq2ps %xmm22, %xmm21 {%k6}
-// CHECK:  encoding: [0x62,0xa1,0xff,0x0e,0x7a,0xee]
-          vcvtuqq2ps %xmm22, %xmm21 {%k6}
-
-// CHECK: vcvtuqq2ps %xmm22, %xmm21 {%k6}
-// CHECK:  encoding: [0x62,0xa1,0xff,0x0e,0x7a,0xee]
-          vcvtuqq2psx %xmm22, %xmm21 {%k6}
-
-// CHECK: vcvtuqq2ps %xmm22, %xmm21 {%k6} {z}
-// CHECK:  encoding: [0x62,0xa1,0xff,0x8e,0x7a,0xee]
-          vcvtuqq2ps %xmm22, %xmm21 {%k6} {z}
-
-// CHECK: vcvtuqq2ps %xmm22, %xmm21 {%k6} {z}
-// CHECK:  encoding: [0x62,0xa1,0xff,0x8e,0x7a,0xee]
-          vcvtuqq2psx %xmm22, %xmm21 {%k6} {z}
-
-// CHECK: vcvtuqq2psx (%rcx), %xmm21
-// CHECK:  encoding: [0x62,0xe1,0xff,0x08,0x7a,0x29]
-          vcvtuqq2psx (%rcx), %xmm21
-
-// CHECK: vcvtuqq2psx (%rcx), %xmm21 {%k6}
-// CHECK:  encoding: [0x62,0xe1,0xff,0x0e,0x7a,0x29]
-          vcvtuqq2psx (%rcx), %xmm21 {%k6}
-
-// CHECK: vcvtuqq2psx (%rcx), %xmm21 {%k6} {z}
-// CHECK:  encoding: [0x62,0xe1,0xff,0x8e,0x7a,0x29]
-          vcvtuqq2psx (%rcx), %xmm21 {%k6} {z}
-
-// CHECK: vcvtuqq2psx 4660(%rax,%r14,8), %xmm21
-// CHECK:  encoding: [0x62,0xa1,0xff,0x08,0x7a,0xac,0xf0,0x34,0x12,0x00,0x00]
-          vcvtuqq2psx 4660(%rax,%r14,8), %xmm21
-
-// CHECK: vcvtuqq2psx 4660(%rax,%r14,8), %xmm21 {%k6}
-// CHECK:  encoding: [0x62,0xa1,0xff,0x0e,0x7a,0xac,0xf0,0x34,0x12,0x00,0x00]
-          vcvtuqq2psx 4660(%rax,%r14,8), %xmm21 {%k6}
-
-// CHECK: vcvtuqq2psx 4660(%rax,%r14,8), %xmm21 {%k6} {z}
-// CHECK:  encoding: [0x62,0xa1,0xff,0x8e,0x7a,0xac,0xf0,0x34,0x12,0x00,0x00]
-          vcvtuqq2psx 4660(%rax,%r14,8), %xmm21 {%k6} {z}
-
-// CHECK: vcvtuqq2ps (%rcx){1to2}, %xmm21
-// CHECK:  encoding: [0x62,0xe1,0xff,0x18,0x7a,0x29]
-          vcvtuqq2ps (%rcx){1to2}, %xmm21
-
-// CHECK: vcvtuqq2ps (%rcx){1to2}, %xmm21
-// CHECK:  encoding: [0x62,0xe1,0xff,0x18,0x7a,0x29]
-          vcvtuqq2psx (%rcx){1to2}, %xmm21
-
-// CHECK: vcvtuqq2ps (%rcx){1to2}, %xmm21 {%k6}
-// CHECK:  encoding: [0x62,0xe1,0xff,0x1e,0x7a,0x29]
-          vcvtuqq2ps (%rcx){1to2}, %xmm21 {%k6}
-
-// CHECK: vcvtuqq2ps (%rcx){1to2}, %xmm21 {%k6}
-// CHECK:  encoding: [0x62,0xe1,0xff,0x1e,0x7a,0x29]
-          vcvtuqq2psx (%rcx){1to2}, %xmm21 {%k6}
-
-// CHECK: vcvtuqq2ps (%rcx){1to2}, %xmm21 {%k6} {z}
-// CHECK:  encoding: [0x62,0xe1,0xff,0x9e,0x7a,0x29]
-          vcvtuqq2ps (%rcx){1to2}, %xmm21 {%k6} {z}
-
-// CHECK: vcvtuqq2ps (%rcx){1to2}, %xmm21 {%k6} {z}
-// CHECK:  encoding: [0x62,0xe1,0xff,0x9e,0x7a,0x29]
-          vcvtuqq2psx (%rcx){1to2}, %xmm21 {%k6} {z}
-
-// CHECK: vcvtuqq2psx 2032(%rdx), %xmm21
-// CHECK:  encoding: [0x62,0xe1,0xff,0x08,0x7a,0x6a,0x7f]
-          vcvtuqq2psx 2032(%rdx), %xmm21
-
-// CHECK: vcvtuqq2psx 2032(%rdx), %xmm21 {%k6}
-// CHECK:  encoding: [0x62,0xe1,0xff,0x0e,0x7a,0x6a,0x7f]
-          vcvtuqq2psx 2032(%rdx), %xmm21 {%k6}
-
-// CHECK: vcvtuqq2psx 2032(%rdx), %xmm21 {%k6} {z}
-// CHECK:  encoding: [0x62,0xe1,0xff,0x8e,0x7a,0x6a,0x7f]
-          vcvtuqq2psx 2032(%rdx), %xmm21 {%k6} {z}
-
-// CHECK: vcvtuqq2psx 2048(%rdx), %xmm21
-// CHECK:  encoding: [0x62,0xe1,0xff,0x08,0x7a,0xaa,0x00,0x08,0x00,0x00]
-          vcvtuqq2psx 2048(%rdx), %xmm21
-
-// CHECK: vcvtuqq2psx 2048(%rdx), %xmm21 {%k6}
-// CHECK:  encoding: [0x62,0xe1,0xff,0x0e,0x7a,0xaa,0x00,0x08,0x00,0x00]
-          vcvtuqq2psx 2048(%rdx), %xmm21 {%k6}
-
-// CHECK: vcvtuqq2psx 2048(%rdx), %xmm21 {%k6} {z}
-// CHECK:  encoding: [0x62,0xe1,0xff,0x8e,0x7a,0xaa,0x00,0x08,0x00,0x00]
-          vcvtuqq2psx 2048(%rdx), %xmm21 {%k6} {z}
-
-// CHECK: vcvtuqq2psx -2048(%rdx), %xmm21
-// CHECK:  encoding: [0x62,0xe1,0xff,0x08,0x7a,0x6a,0x80]
-          vcvtuqq2psx -2048(%rdx), %xmm21
-
-// CHECK: vcvtuqq2psx -2048(%rdx), %xmm21 {%k6}
-// CHECK:  encoding: [0x62,0xe1,0xff,0x0e,0x7a,0x6a,0x80]
-          vcvtuqq2psx -2048(%rdx), %xmm21 {%k6}
-
-// CHECK: vcvtuqq2psx -2048(%rdx), %xmm21 {%k6} {z}
-// CHECK:  encoding: [0x62,0xe1,0xff,0x8e,0x7a,0x6a,0x80]
-          vcvtuqq2psx -2048(%rdx), %xmm21 {%k6} {z}
-
-// CHECK: vcvtuqq2psx -2064(%rdx), %xmm21
-// CHECK:  encoding: [0x62,0xe1,0xff,0x08,0x7a,0xaa,0xf0,0xf7,0xff,0xff]
-          vcvtuqq2psx -2064(%rdx), %xmm21
-
-// CHECK: vcvtuqq2psx -2064(%rdx), %xmm21 {%k6}
-// CHECK:  encoding: [0x62,0xe1,0xff,0x0e,0x7a,0xaa,0xf0,0xf7,0xff,0xff]
-          vcvtuqq2psx -2064(%rdx), %xmm21 {%k6}
-
-// CHECK: vcvtuqq2psx -2064(%rdx), %xmm21 {%k6} {z}
-// CHECK:  encoding: [0x62,0xe1,0xff,0x8e,0x7a,0xaa,0xf0,0xf7,0xff,0xff]
-          vcvtuqq2psx -2064(%rdx), %xmm21 {%k6} {z}
-
-// CHECK: vcvtuqq2ps 1016(%rdx){1to2}, %xmm21
-// CHECK:  encoding: [0x62,0xe1,0xff,0x18,0x7a,0x6a,0x7f]
-          vcvtuqq2ps 1016(%rdx){1to2}, %xmm21
-
-// CHECK: vcvtuqq2ps 1016(%rdx){1to2}, %xmm21
-// CHECK:  encoding: [0x62,0xe1,0xff,0x18,0x7a,0x6a,0x7f]
-          vcvtuqq2psx 1016(%rdx){1to2}, %xmm21
-
-// CHECK: vcvtuqq2ps 1016(%rdx){1to2}, %xmm21 {%k6}
-// CHECK:  encoding: [0x62,0xe1,0xff,0x1e,0x7a,0x6a,0x7f]
-          vcvtuqq2ps 1016(%rdx){1to2}, %xmm21 {%k6}
-
-// CHECK: vcvtuqq2ps 1016(%rdx){1to2}, %xmm21 {%k6}
-// CHECK:  encoding: [0x62,0xe1,0xff,0x1e,0x7a,0x6a,0x7f]
-          vcvtuqq2psx 1016(%rdx){1to2}, %xmm21 {%k6}
-
-// CHECK: vcvtuqq2ps 1016(%rdx){1to2}, %xmm21 {%k6} {z}
-// CHECK:  encoding: [0x62,0xe1,0xff,0x9e,0x7a,0x6a,0x7f]
-          vcvtuqq2ps 1016(%rdx){1to2}, %xmm21 {%k6} {z}
-
-// CHECK: vcvtuqq2ps 1016(%rdx){1to2}, %xmm21 {%k6} {z}
-// CHECK:  encoding: [0x62,0xe1,0xff,0x9e,0x7a,0x6a,0x7f]
-          vcvtuqq2psx 1016(%rdx){1to2}, %xmm21 {%k6} {z}
-
-// CHECK: vcvtuqq2ps 1024(%rdx){1to2}, %xmm21
-// CHECK:  encoding: [0x62,0xe1,0xff,0x18,0x7a,0xaa,0x00,0x04,0x00,0x00]
-          vcvtuqq2ps 1024(%rdx){1to2}, %xmm21
-
-// CHECK: vcvtuqq2ps 1024(%rdx){1to2}, %xmm21
-// CHECK:  encoding: [0x62,0xe1,0xff,0x18,0x7a,0xaa,0x00,0x04,0x00,0x00]
-          vcvtuqq2psx 1024(%rdx){1to2}, %xmm21
-
-// CHECK: vcvtuqq2ps 1024(%rdx){1to2}, %xmm21 {%k6}
-// CHECK:  encoding: [0x62,0xe1,0xff,0x1e,0x7a,0xaa,0x00,0x04,0x00,0x00]
-          vcvtuqq2ps 1024(%rdx){1to2}, %xmm21 {%k6}
-
-// CHECK: vcvtuqq2ps 1024(%rdx){1to2}, %xmm21 {%k6}
-// CHECK:  encoding: [0x62,0xe1,0xff,0x1e,0x7a,0xaa,0x00,0x04,0x00,0x00]
-          vcvtuqq2psx 1024(%rdx){1to2}, %xmm21 {%k6}
-
-// CHECK: vcvtuqq2ps 1024(%rdx){1to2}, %xmm21 {%k6} {z}
-// CHECK:  encoding: [0x62,0xe1,0xff,0x9e,0x7a,0xaa,0x00,0x04,0x00,0x00]
-          vcvtuqq2ps 1024(%rdx){1to2}, %xmm21 {%k6} {z}
-
-// CHECK: vcvtuqq2ps 1024(%rdx){1to2}, %xmm21 {%k6} {z}
-// CHECK:  encoding: [0x62,0xe1,0xff,0x9e,0x7a,0xaa,0x00,0x04,0x00,0x00]
-          vcvtuqq2psx 1024(%rdx){1to2}, %xmm21 {%k6} {z}
-
-// CHECK: vcvtuqq2ps -1024(%rdx){1to2}, %xmm21
-// CHECK:  encoding: [0x62,0xe1,0xff,0x18,0x7a,0x6a,0x80]
-          vcvtuqq2ps -1024(%rdx){1to2}, %xmm21
-
-// CHECK: vcvtuqq2ps -1024(%rdx){1to2}, %xmm21
-// CHECK:  encoding: [0x62,0xe1,0xff,0x18,0x7a,0x6a,0x80]
-          vcvtuqq2psx -1024(%rdx){1to2}, %xmm21
-
-// CHECK: vcvtuqq2ps -1024(%rdx){1to2}, %xmm21 {%k6}
-// CHECK:  encoding: [0x62,0xe1,0xff,0x1e,0x7a,0x6a,0x80]
-          vcvtuqq2ps -1024(%rdx){1to2}, %xmm21 {%k6}
-
-// CHECK: vcvtuqq2ps -1024(%rdx){1to2}, %xmm21 {%k6}
-// CHECK:  encoding: [0x62,0xe1,0xff,0x1e,0x7a,0x6a,0x80]
-          vcvtuqq2psx -1024(%rdx){1to2}, %xmm21 {%k6}
-
-// CHECK: vcvtuqq2ps -1024(%rdx){1to2}, %xmm21 {%k6} {z}
-// CHECK:  encoding: [0x62,0xe1,0xff,0x9e,0x7a,0x6a,0x80]
-          vcvtuqq2ps -1024(%rdx){1to2}, %xmm21 {%k6} {z}
-
-// CHECK: vcvtuqq2ps -1024(%rdx){1to2}, %xmm21 {%k6} {z}
-// CHECK:  encoding: [0x62,0xe1,0xff,0x9e,0x7a,0x6a,0x80]
-          vcvtuqq2psx -1024(%rdx){1to2}, %xmm21 {%k6} {z}
-
-// CHECK: vcvtuqq2ps -1032(%rdx){1to2}, %xmm21
-// CHECK:  encoding: [0x62,0xe1,0xff,0x18,0x7a,0xaa,0xf8,0xfb,0xff,0xff]
-          vcvtuqq2ps -1032(%rdx){1to2}, %xmm21
-
-// CHECK: vcvtuqq2ps -1032(%rdx){1to2}, %xmm21
-// CHECK:  encoding: [0x62,0xe1,0xff,0x18,0x7a,0xaa,0xf8,0xfb,0xff,0xff]
-          vcvtuqq2psx -1032(%rdx){1to2}, %xmm21
-
-// CHECK: vcvtuqq2ps -1032(%rdx){1to2}, %xmm21 {%k6}
-// CHECK:  encoding: [0x62,0xe1,0xff,0x1e,0x7a,0xaa,0xf8,0xfb,0xff,0xff]
-          vcvtuqq2ps -1032(%rdx){1to2}, %xmm21 {%k6}
-
-// CHECK: vcvtuqq2ps -1032(%rdx){1to2}, %xmm21 {%k6}
-// CHECK:  encoding: [0x62,0xe1,0xff,0x1e,0x7a,0xaa,0xf8,0xfb,0xff,0xff]
-          vcvtuqq2psx -1032(%rdx){1to2}, %xmm21 {%k6}
-
-// CHECK: vcvtuqq2ps -1032(%rdx){1to2}, %xmm21 {%k6} {z}
-// CHECK:  encoding: [0x62,0xe1,0xff,0x9e,0x7a,0xaa,0xf8,0xfb,0xff,0xff]
-          vcvtuqq2ps -1032(%rdx){1to2}, %xmm21 {%k6} {z}
-
-// CHECK: vcvtuqq2ps -1032(%rdx){1to2}, %xmm21 {%k6} {z}
-// CHECK:  encoding: [0x62,0xe1,0xff,0x9e,0x7a,0xaa,0xf8,0xfb,0xff,0xff]
-          vcvtuqq2psx -1032(%rdx){1to2}, %xmm21 {%k6} {z}
-
-// CHECK: vcvtuqq2ps %ymm17, %xmm26
-// CHECK:  encoding: [0x62,0x21,0xff,0x28,0x7a,0xd1]
-          vcvtuqq2ps %ymm17, %xmm26
-
-// CHECK: vcvtuqq2ps %ymm17, %xmm26
-// CHECK:  encoding: [0x62,0x21,0xff,0x28,0x7a,0xd1]
-          vcvtuqq2psy %ymm17, %xmm26
-
-// CHECK: vcvtuqq2ps %ymm17, %xmm26 {%k4}
-// CHECK:  encoding: [0x62,0x21,0xff,0x2c,0x7a,0xd1]
-          vcvtuqq2ps %ymm17, %xmm26 {%k4}
-
-// CHECK: vcvtuqq2ps %ymm17, %xmm26 {%k4}
-// CHECK:  encoding: [0x62,0x21,0xff,0x2c,0x7a,0xd1]
-          vcvtuqq2psy %ymm17, %xmm26 {%k4}
-
-// CHECK: vcvtuqq2ps %ymm17, %xmm26 {%k4} {z}
-// CHECK:  encoding: [0x62,0x21,0xff,0xac,0x7a,0xd1]
-          vcvtuqq2ps %ymm17, %xmm26 {%k4} {z}
-
-// CHECK: vcvtuqq2ps %ymm17, %xmm26 {%k4} {z}
-// CHECK:  encoding: [0x62,0x21,0xff,0xac,0x7a,0xd1]
-          vcvtuqq2psy %ymm17, %xmm26 {%k4} {z}
-
-// CHECK: vcvtuqq2psy (%rcx), %xmm26
-// CHECK:  encoding: [0x62,0x61,0xff,0x28,0x7a,0x11]
-          vcvtuqq2psy (%rcx), %xmm26
-
-// CHECK: vcvtuqq2psy (%rcx), %xmm26 {%k4}
-// CHECK:  encoding: [0x62,0x61,0xff,0x2c,0x7a,0x11]
-          vcvtuqq2psy (%rcx), %xmm26 {%k4}
-
-// CHECK: vcvtuqq2psy (%rcx), %xmm26 {%k4} {z}
-// CHECK:  encoding: [0x62,0x61,0xff,0xac,0x7a,0x11]
-          vcvtuqq2psy (%rcx), %xmm26 {%k4} {z}
-
-// CHECK: vcvtuqq2psy 4660(%rax,%r14,8), %xmm26
-// CHECK:  encoding: [0x62,0x21,0xff,0x28,0x7a,0x94,0xf0,0x34,0x12,0x00,0x00]
-          vcvtuqq2psy 4660(%rax,%r14,8), %xmm26
-
-// CHECK: vcvtuqq2psy 4660(%rax,%r14,8), %xmm26 {%k4}
-// CHECK:  encoding: [0x62,0x21,0xff,0x2c,0x7a,0x94,0xf0,0x34,0x12,0x00,0x00]
-          vcvtuqq2psy 4660(%rax,%r14,8), %xmm26 {%k4}
-
-// CHECK: vcvtuqq2psy 4660(%rax,%r14,8), %xmm26 {%k4} {z}
-// CHECK:  encoding: [0x62,0x21,0xff,0xac,0x7a,0x94,0xf0,0x34,0x12,0x00,0x00]
-          vcvtuqq2psy 4660(%rax,%r14,8), %xmm26 {%k4} {z}
-
-// CHECK: vcvtuqq2ps (%rcx){1to4}, %xmm26
-// CHECK:  encoding: [0x62,0x61,0xff,0x38,0x7a,0x11]
-          vcvtuqq2ps (%rcx){1to4}, %xmm26
-
-// CHECK: vcvtuqq2ps (%rcx){1to4}, %xmm26
-// CHECK:  encoding: [0x62,0x61,0xff,0x38,0x7a,0x11]
-          vcvtuqq2psy (%rcx){1to4}, %xmm26
-
-// CHECK: vcvtuqq2ps (%rcx){1to4}, %xmm26 {%k4}
-// CHECK:  encoding: [0x62,0x61,0xff,0x3c,0x7a,0x11]
-          vcvtuqq2ps (%rcx){1to4}, %xmm26 {%k4}
-
-// CHECK: vcvtuqq2ps (%rcx){1to4}, %xmm26 {%k4}
-// CHECK:  encoding: [0x62,0x61,0xff,0x3c,0x7a,0x11]
-          vcvtuqq2psy (%rcx){1to4}, %xmm26 {%k4}
-
-// CHECK: vcvtuqq2ps (%rcx){1to4}, %xmm26 {%k4} {z}
-// CHECK:  encoding: [0x62,0x61,0xff,0xbc,0x7a,0x11]
-          vcvtuqq2ps (%rcx){1to4}, %xmm26 {%k4} {z}
-
-// CHECK: vcvtuqq2ps (%rcx){1to4}, %xmm26 {%k4} {z}
-// CHECK:  encoding: [0x62,0x61,0xff,0xbc,0x7a,0x11]
-          vcvtuqq2psy (%rcx){1to4}, %xmm26 {%k4} {z}
-
-// CHECK: vcvtuqq2psy 4064(%rdx), %xmm26
-// CHECK:  encoding: [0x62,0x61,0xff,0x28,0x7a,0x52,0x7f]
-          vcvtuqq2psy 4064(%rdx), %xmm26
-
-// CHECK: vcvtuqq2psy 4064(%rdx), %xmm26 {%k4}
-// CHECK:  encoding: [0x62,0x61,0xff,0x2c,0x7a,0x52,0x7f]
-          vcvtuqq2psy 4064(%rdx), %xmm26 {%k4}
-
-// CHECK: vcvtuqq2psy 4064(%rdx), %xmm26 {%k4} {z}
-// CHECK:  encoding: [0x62,0x61,0xff,0xac,0x7a,0x52,0x7f]
-          vcvtuqq2psy 4064(%rdx), %xmm26 {%k4} {z}
-
-// CHECK: vcvtuqq2psy 4096(%rdx), %xmm26
-// CHECK:  encoding: [0x62,0x61,0xff,0x28,0x7a,0x92,0x00,0x10,0x00,0x00]
-          vcvtuqq2psy 4096(%rdx), %xmm26
-
-// CHECK: vcvtuqq2psy 4096(%rdx), %xmm26 {%k4}
-// CHECK:  encoding: [0x62,0x61,0xff,0x2c,0x7a,0x92,0x00,0x10,0x00,0x00]
-          vcvtuqq2psy 4096(%rdx), %xmm26 {%k4}
-
-// CHECK: vcvtuqq2psy 4096(%rdx), %xmm26 {%k4} {z}
-// CHECK:  encoding: [0x62,0x61,0xff,0xac,0x7a,0x92,0x00,0x10,0x00,0x00]
-          vcvtuqq2psy 4096(%rdx), %xmm26 {%k4} {z}
-
-// CHECK: vcvtuqq2psy -4096(%rdx), %xmm26
-// CHECK:  encoding: [0x62,0x61,0xff,0x28,0x7a,0x52,0x80]
-          vcvtuqq2psy -4096(%rdx), %xmm26
-
-// CHECK: vcvtuqq2psy -4096(%rdx), %xmm26 {%k4}
-// CHECK:  encoding: [0x62,0x61,0xff,0x2c,0x7a,0x52,0x80]
-          vcvtuqq2psy -4096(%rdx), %xmm26 {%k4}
-
-// CHECK: vcvtuqq2psy -4096(%rdx), %xmm26 {%k4} {z}
-// CHECK:  encoding: [0x62,0x61,0xff,0xac,0x7a,0x52,0x80]
-          vcvtuqq2psy -4096(%rdx), %xmm26 {%k4} {z}
-
-// CHECK: vcvtuqq2psy -4128(%rdx), %xmm26
-// CHECK:  encoding: [0x62,0x61,0xff,0x28,0x7a,0x92,0xe0,0xef,0xff,0xff]
-          vcvtuqq2psy -4128(%rdx), %xmm26
-
-// CHECK: vcvtuqq2psy -4128(%rdx), %xmm26 {%k4}
-// CHECK:  encoding: [0x62,0x61,0xff,0x2c,0x7a,0x92,0xe0,0xef,0xff,0xff]
-          vcvtuqq2psy -4128(%rdx), %xmm26 {%k4}
-
-// CHECK: vcvtuqq2psy -4128(%rdx), %xmm26 {%k4}
-// CHECK:  encoding: [0x62,0x61,0xff,0xac,0x7a,0x92,0xe0,0xef,0xff,0xff]
-          vcvtuqq2psy -4128(%rdx), %xmm26 {%k4} {z}
-
-// CHECK: vcvtuqq2ps 1016(%rdx){1to4}, %xmm26
-// CHECK:  encoding: [0x62,0x61,0xff,0x38,0x7a,0x52,0x7f]
-          vcvtuqq2ps 1016(%rdx){1to4}, %xmm26
-
-// CHECK: vcvtuqq2ps 1016(%rdx){1to4}, %xmm26
-// CHECK:  encoding: [0x62,0x61,0xff,0x38,0x7a,0x52,0x7f]
-          vcvtuqq2psy 1016(%rdx){1to4}, %xmm26
-
-// CHECK: vcvtuqq2ps 1016(%rdx){1to4}, %xmm26 {%k4}
-// CHECK:  encoding: [0x62,0x61,0xff,0x3c,0x7a,0x52,0x7f]
-          vcvtuqq2ps 1016(%rdx){1to4}, %xmm26 {%k4}
-
-// CHECK: vcvtuqq2ps 1016(%rdx){1to4}, %xmm26 {%k4}
-// CHECK:  encoding: [0x62,0x61,0xff,0x3c,0x7a,0x52,0x7f]
-          vcvtuqq2psy 1016(%rdx){1to4}, %xmm26 {%k4}
-
-// CHECK: vcvtuqq2ps 1016(%rdx){1to4}, %xmm26 {%k4} {z}
-// CHECK:  encoding: [0x62,0x61,0xff,0xbc,0x7a,0x52,0x7f]
-          vcvtuqq2ps 1016(%rdx){1to4}, %xmm26 {%k4} {z}
-
-// CHECK: vcvtuqq2ps 1016(%rdx){1to4}, %xmm26 {%k4} {z}
-// CHECK:  encoding: [0x62,0x61,0xff,0xbc,0x7a,0x52,0x7f]
-          vcvtuqq2psy 1016(%rdx){1to4}, %xmm26 {%k4} {z}
-
-// CHECK: vcvtuqq2ps 1024(%rdx){1to4}, %xmm26
-// CHECK:  encoding: [0x62,0x61,0xff,0x38,0x7a,0x92,0x00,0x04,0x00,0x00]
-          vcvtuqq2ps 1024(%rdx){1to4}, %xmm26
-
-// CHECK: vcvtuqq2ps 1024(%rdx){1to4}, %xmm26
-// CHECK:  encoding: [0x62,0x61,0xff,0x38,0x7a,0x92,0x00,0x04,0x00,0x00]
-          vcvtuqq2psy 1024(%rdx){1to4}, %xmm26
-
-// CHECK: vcvtuqq2ps 1024(%rdx){1to4}, %xmm26 {%k4}
-// CHECK:  encoding: [0x62,0x61,0xff,0x3c,0x7a,0x92,0x00,0x04,0x00,0x00]
-          vcvtuqq2ps 1024(%rdx){1to4}, %xmm26 {%k4}
-
-// CHECK: vcvtuqq2ps 1024(%rdx){1to4}, %xmm26 {%k4}
-// CHECK:  encoding: [0x62,0x61,0xff,0x3c,0x7a,0x92,0x00,0x04,0x00,0x00]
-          vcvtuqq2psy 1024(%rdx){1to4}, %xmm26 {%k4}
-
-// CHECK: vcvtuqq2ps 1024(%rdx){1to4}, %xmm26 {%k4} {z}
-// CHECK:  encoding: [0x62,0x61,0xff,0xbc,0x7a,0x92,0x00,0x04,0x00,0x00]
-          vcvtuqq2ps 1024(%rdx){1to4}, %xmm26 {%k4} {z}
-
-// CHECK: vcvtuqq2ps 1024(%rdx){1to4}, %xmm26 {%k4} {z}
-// CHECK:  encoding: [0x62,0x61,0xff,0xbc,0x7a,0x92,0x00,0x04,0x00,0x00]
-          vcvtuqq2psy 1024(%rdx){1to4}, %xmm26 {%k4} {z}
-
-// CHECK: vcvtuqq2ps -1024(%rdx){1to4}, %xmm26
-// CHECK:  encoding: [0x62,0x61,0xff,0x38,0x7a,0x52,0x80]
-          vcvtuqq2ps -1024(%rdx){1to4}, %xmm26
-
-// CHECK: vcvtuqq2ps -1024(%rdx){1to4}, %xmm26
-// CHECK:  encoding: [0x62,0x61,0xff,0x38,0x7a,0x52,0x80]
-          vcvtuqq2psy -1024(%rdx){1to4}, %xmm26
-
-// CHECK: vcvtuqq2ps -1024(%rdx){1to4}, %xmm26 {%k4}
-// CHECK:  encoding: [0x62,0x61,0xff,0x3c,0x7a,0x52,0x80]
-          vcvtuqq2ps -1024(%rdx){1to4}, %xmm26 {%k4}
-
-// CHECK: vcvtuqq2ps -1024(%rdx){1to4}, %xmm26 {%k4}
-// CHECK:  encoding: [0x62,0x61,0xff,0x3c,0x7a,0x52,0x80]
-          vcvtuqq2psy -1024(%rdx){1to4}, %xmm26 {%k4}
-
-// CHECK: vcvtuqq2ps -1024(%rdx){1to4}, %xmm26 {%k4} {z}
-// CHECK:  encoding: [0x62,0x61,0xff,0xbc,0x7a,0x52,0x80]
-          vcvtuqq2ps -1024(%rdx){1to4}, %xmm26 {%k4} {z}
-
-// CHECK: vcvtuqq2ps -1024(%rdx){1to4}, %xmm26 {%k4} {z}
-// CHECK:  encoding: [0x62,0x61,0xff,0xbc,0x7a,0x52,0x80]
-          vcvtuqq2psy -1024(%rdx){1to4}, %xmm26 {%k4} {z}
-
-// CHECK: vcvtuqq2ps -1032(%rdx){1to4}, %xmm26
-// CHECK:  encoding: [0x62,0x61,0xff,0x38,0x7a,0x92,0xf8,0xfb,0xff,0xff]
-          vcvtuqq2ps -1032(%rdx){1to4}, %xmm26
-
-// CHECK: vcvtuqq2ps -1032(%rdx){1to4}, %xmm26
-// CHECK:  encoding: [0x62,0x61,0xff,0x38,0x7a,0x92,0xf8,0xfb,0xff,0xff]
-          vcvtuqq2psy -1032(%rdx){1to4}, %xmm26
-
-// CHECK: vcvtuqq2ps -1032(%rdx){1to4}, %xmm26 {%k4}
-// CHECK:  encoding: [0x62,0x61,0xff,0x3c,0x7a,0x92,0xf8,0xfb,0xff,0xff]
-          vcvtuqq2ps -1032(%rdx){1to4}, %xmm26 {%k4}
-
-// CHECK: vcvtuqq2ps -1032(%rdx){1to4}, %xmm26 {%k4}
-// CHECK:  encoding: [0x62,0x61,0xff,0x3c,0x7a,0x92,0xf8,0xfb,0xff,0xff]
-          vcvtuqq2psy -1032(%rdx){1to4}, %xmm26 {%k4}
-
-// CHECK: vcvtuqq2ps -1032(%rdx){1to4}, %xmm26 {%k4} {z}
-// CHECK:  encoding: [0x62,0x61,0xff,0xbc,0x7a,0x92,0xf8,0xfb,0xff,0xff]
-          vcvtuqq2ps -1032(%rdx){1to4}, %xmm26 {%k4} {z}
-
-// CHECK: vcvtuqq2ps -1032(%rdx){1to4}, %xmm26 {%k4} {z}
-// CHECK:  encoding: [0x62,0x61,0xff,0xbc,0x7a,0x92,0xf8,0xfb,0xff,0xff]
-          vcvtuqq2psy -1032(%rdx){1to4}, %xmm26 {%k4} {z}
-
-// CHECK: vinsertf64x2 $171, %xmm23, %ymm27, %ymm21
-// CHECK:  encoding: [0x62,0xa3,0xa5,0x20,0x18,0xef,0xab]
-          vinsertf64x2 $0xab, %xmm23, %ymm27, %ymm21
-
-// CHECK: vinsertf64x2 $171, %xmm23, %ymm27, %ymm21 {%k7}
-// CHECK:  encoding: [0x62,0xa3,0xa5,0x27,0x18,0xef,0xab]
-          vinsertf64x2 $0xab, %xmm23, %ymm27, %ymm21 {%k7}
-
-// CHECK: vinsertf64x2 $171, %xmm23, %ymm27, %ymm21 {%k7} {z}
-// CHECK:  encoding: [0x62,0xa3,0xa5,0xa7,0x18,0xef,0xab]
-          vinsertf64x2 $0xab, %xmm23, %ymm27, %ymm21 {%k7} {z}
-
-// CHECK: vinsertf64x2 $123, %xmm23, %ymm27, %ymm21
-// CHECK:  encoding: [0x62,0xa3,0xa5,0x20,0x18,0xef,0x7b]
-          vinsertf64x2 $0x7b, %xmm23, %ymm27, %ymm21
-
-// CHECK: vinsertf64x2 $123, (%rcx), %ymm27, %ymm21
-// CHECK:  encoding: [0x62,0xe3,0xa5,0x20,0x18,0x29,0x7b]
-          vinsertf64x2 $0x7b,(%rcx), %ymm27, %ymm21
-
-// CHECK: vinsertf64x2 $123, 291(%rax,%r14,8), %ymm27, %ymm21
-// CHECK:  encoding: [0x62,0xa3,0xa5,0x20,0x18,0xac,0xf0,0x23,0x01,0x00,0x00,0x7b]
-          vinsertf64x2 $0x7b,291(%rax,%r14,8), %ymm27, %ymm21
-
-// CHECK: vinsertf64x2 $123, 2032(%rdx), %ymm27, %ymm21
-// CHECK:  encoding: [0x62,0xe3,0xa5,0x20,0x18,0x6a,0x7f,0x7b]
-          vinsertf64x2 $0x7b,2032(%rdx), %ymm27, %ymm21
-
-// CHECK: vinsertf64x2 $123, 2048(%rdx), %ymm27, %ymm21
-// CHECK:  encoding: [0x62,0xe3,0xa5,0x20,0x18,0xaa,0x00,0x08,0x00,0x00,0x7b]
-          vinsertf64x2 $0x7b,2048(%rdx), %ymm27, %ymm21
-
-// CHECK: vinsertf64x2 $123, -2048(%rdx), %ymm27, %ymm21
-// CHECK:  encoding: [0x62,0xe3,0xa5,0x20,0x18,0x6a,0x80,0x7b]
-          vinsertf64x2 $0x7b,-2048(%rdx), %ymm27, %ymm21
-
-// CHECK: vinsertf64x2 $123, -2064(%rdx), %ymm27, %ymm21
-// CHECK:  encoding: [0x62,0xe3,0xa5,0x20,0x18,0xaa,0xf0,0xf7,0xff,0xff,0x7b]
-          vinsertf64x2 $0x7b,-2064(%rdx), %ymm27, %ymm21
-
-// CHECK: vinsertf64x2 $171, %xmm27, %ymm23, %ymm24
-// CHECK:  encoding: [0x62,0x03,0xc5,0x20,0x18,0xc3,0xab]
-          vinsertf64x2 $0xab, %xmm27, %ymm23, %ymm24
-
-// CHECK: vinsertf64x2 $171, %xmm27, %ymm23, %ymm24 {%k5}
-// CHECK:  encoding: [0x62,0x03,0xc5,0x25,0x18,0xc3,0xab]
-          vinsertf64x2 $0xab, %xmm27, %ymm23, %ymm24 {%k5}
-
-// CHECK: vinsertf64x2 $171, %xmm27, %ymm23, %ymm24 {%k5} {z}
-// CHECK:  encoding: [0x62,0x03,0xc5,0xa5,0x18,0xc3,0xab]
-          vinsertf64x2 $0xab, %xmm27, %ymm23, %ymm24 {%k5} {z}
-
-// CHECK: vinsertf64x2 $123, %xmm27, %ymm23, %ymm24
-// CHECK:  encoding: [0x62,0x03,0xc5,0x20,0x18,0xc3,0x7b]
-          vinsertf64x2 $0x7b, %xmm27, %ymm23, %ymm24
-
-// CHECK: vinsertf64x2 $123, (%rcx), %ymm23, %ymm24
-// CHECK:  encoding: [0x62,0x63,0xc5,0x20,0x18,0x01,0x7b]
-          vinsertf64x2 $0x7b,(%rcx), %ymm23, %ymm24
-
-// CHECK: vinsertf64x2 $123, 4660(%rax,%r14,8), %ymm23, %ymm24
-// CHECK:  encoding: [0x62,0x23,0xc5,0x20,0x18,0x84,0xf0,0x34,0x12,0x00,0x00,0x7b]
-          vinsertf64x2 $0x7b,4660(%rax,%r14,8), %ymm23, %ymm24
-
-// CHECK: vinsertf64x2 $123, 2032(%rdx), %ymm23, %ymm24
-// CHECK:  encoding: [0x62,0x63,0xc5,0x20,0x18,0x42,0x7f,0x7b]
-          vinsertf64x2 $0x7b,2032(%rdx), %ymm23, %ymm24
-
-// CHECK: vinsertf64x2 $123, 2048(%rdx), %ymm23, %ymm24
-// CHECK:  encoding: [0x62,0x63,0xc5,0x20,0x18,0x82,0x00,0x08,0x00,0x00,0x7b]
-          vinsertf64x2 $0x7b,2048(%rdx), %ymm23, %ymm24
-
-// CHECK: vinsertf64x2 $123, -2048(%rdx), %ymm23, %ymm24
-// CHECK:  encoding: [0x62,0x63,0xc5,0x20,0x18,0x42,0x80,0x7b]
-          vinsertf64x2 $0x7b,-2048(%rdx), %ymm23, %ymm24
-
-// CHECK: vinsertf64x2 $123, -2064(%rdx), %ymm23, %ymm24
-// CHECK:  encoding: [0x62,0x63,0xc5,0x20,0x18,0x82,0xf0,0xf7,0xff,0xff,0x7b]
-          vinsertf64x2 $0x7b,-2064(%rdx), %ymm23, %ymm24
-
-// CHECK: vinserti64x2 $171, %xmm21, %ymm25, %ymm19
-// CHECK:  encoding: [0x62,0xa3,0xb5,0x20,0x38,0xdd,0xab]
-          vinserti64x2 $0xab, %xmm21, %ymm25, %ymm19
-
-// CHECK: vinserti64x2 $171, %xmm21, %ymm25, %ymm19 {%k6}
-// CHECK:  encoding: [0x62,0xa3,0xb5,0x26,0x38,0xdd,0xab]
-          vinserti64x2 $0xab, %xmm21, %ymm25, %ymm19 {%k6}
-
-// CHECK: vinserti64x2 $171, %xmm21, %ymm25, %ymm19 {%k6} {z}
-// CHECK:  encoding: [0x62,0xa3,0xb5,0xa6,0x38,0xdd,0xab]
-          vinserti64x2 $0xab, %xmm21, %ymm25, %ymm19 {%k6} {z}
-
-// CHECK: vinserti64x2 $123, %xmm21, %ymm25, %ymm19
-// CHECK:  encoding: [0x62,0xa3,0xb5,0x20,0x38,0xdd,0x7b]
-          vinserti64x2 $0x7b, %xmm21, %ymm25, %ymm19
-
-// CHECK: vinserti64x2 $123, (%rcx), %ymm25, %ymm19
-// CHECK:  encoding: [0x62,0xe3,0xb5,0x20,0x38,0x19,0x7b]
-          vinserti64x2 $0x7b,(%rcx), %ymm25, %ymm19
-
-// CHECK: vinserti64x2 $123, 291(%rax,%r14,8), %ymm25, %ymm19
-// CHECK:  encoding: [0x62,0xa3,0xb5,0x20,0x38,0x9c,0xf0,0x23,0x01,0x00,0x00,0x7b]
-          vinserti64x2 $0x7b,291(%rax,%r14,8), %ymm25, %ymm19
-
-// CHECK: vinserti64x2 $123, 2032(%rdx), %ymm25, %ymm19
-// CHECK:  encoding: [0x62,0xe3,0xb5,0x20,0x38,0x5a,0x7f,0x7b]
-          vinserti64x2 $0x7b,2032(%rdx), %ymm25, %ymm19
-
-// CHECK: vinserti64x2 $123, 2048(%rdx), %ymm25, %ymm19
-// CHECK:  encoding: [0x62,0xe3,0xb5,0x20,0x38,0x9a,0x00,0x08,0x00,0x00,0x7b]
-          vinserti64x2 $0x7b,2048(%rdx), %ymm25, %ymm19
-
-// CHECK: vinserti64x2 $123, -2048(%rdx), %ymm25, %ymm19
-// CHECK:  encoding: [0x62,0xe3,0xb5,0x20,0x38,0x5a,0x80,0x7b]
-          vinserti64x2 $0x7b,-2048(%rdx), %ymm25, %ymm19
-
-// CHECK: vinserti64x2 $123, -2064(%rdx), %ymm25, %ymm19
-// CHECK:  encoding: [0x62,0xe3,0xb5,0x20,0x38,0x9a,0xf0,0xf7,0xff,0xff,0x7b]
-          vinserti64x2 $0x7b,-2064(%rdx), %ymm25, %ymm19
-
-// CHECK: vinserti64x2 $171, %xmm25, %ymm24, %ymm29
-// CHECK:  encoding: [0x62,0x03,0xbd,0x20,0x38,0xe9,0xab]
-          vinserti64x2 $0xab, %xmm25, %ymm24, %ymm29
-
-// CHECK: vinserti64x2 $171, %xmm25, %ymm24, %ymm29 {%k2}
-// CHECK:  encoding: [0x62,0x03,0xbd,0x22,0x38,0xe9,0xab]
-          vinserti64x2 $0xab, %xmm25, %ymm24, %ymm29 {%k2}
-
-// CHECK: vinserti64x2 $171, %xmm25, %ymm24, %ymm29 {%k2} {z}
-// CHECK:  encoding: [0x62,0x03,0xbd,0xa2,0x38,0xe9,0xab]
-          vinserti64x2 $0xab, %xmm25, %ymm24, %ymm29 {%k2} {z}
-
-// CHECK: vinserti64x2 $123, %xmm25, %ymm24, %ymm29
-// CHECK:  encoding: [0x62,0x03,0xbd,0x20,0x38,0xe9,0x7b]
-          vinserti64x2 $0x7b, %xmm25, %ymm24, %ymm29
-
-// CHECK: vinserti64x2 $123, (%rcx), %ymm24, %ymm29
-// CHECK:  encoding: [0x62,0x63,0xbd,0x20,0x38,0x29,0x7b]
-          vinserti64x2 $0x7b,(%rcx), %ymm24, %ymm29
-
-// CHECK: vinserti64x2 $123, 4660(%rax,%r14,8), %ymm24, %ymm29
-// CHECK:  encoding: [0x62,0x23,0xbd,0x20,0x38,0xac,0xf0,0x34,0x12,0x00,0x00,0x7b]
-          vinserti64x2 $0x7b,4660(%rax,%r14,8), %ymm24, %ymm29
-
-// CHECK: vinserti64x2 $123, 2032(%rdx), %ymm24, %ymm29
-// CHECK:  encoding: [0x62,0x63,0xbd,0x20,0x38,0x6a,0x7f,0x7b]
-          vinserti64x2 $0x7b,2032(%rdx), %ymm24, %ymm29
-
-// CHECK: vinserti64x2 $123, 2048(%rdx), %ymm24, %ymm29
-// CHECK:  encoding: [0x62,0x63,0xbd,0x20,0x38,0xaa,0x00,0x08,0x00,0x00,0x7b]
-          vinserti64x2 $0x7b,2048(%rdx), %ymm24, %ymm29
-
-// CHECK: vinserti64x2 $123, -2048(%rdx), %ymm24, %ymm29
-// CHECK:  encoding: [0x62,0x63,0xbd,0x20,0x38,0x6a,0x80,0x7b]
-          vinserti64x2 $0x7b,-2048(%rdx), %ymm24, %ymm29
-
-// CHECK: vinserti64x2 $123, -2064(%rdx), %ymm24, %ymm29
-// CHECK:  encoding: [0x62,0x63,0xbd,0x20,0x38,0xaa,0xf0,0xf7,0xff,0xff,0x7b]
-          vinserti64x2 $0x7b,-2064(%rdx), %ymm24, %ymm29
-
-// CHECK: vextractf64x2 $171, %ymm21, %xmm27
-// CHECK:  encoding: [0x62,0x83,0xfd,0x28,0x19,0xeb,0xab]
-          vextractf64x2 $0xab, %ymm21, %xmm27
-
-// CHECK: vextractf64x2 $171, %ymm21, %xmm27 {%k7}
-// CHECK:  encoding: [0x62,0x83,0xfd,0x2f,0x19,0xeb,0xab]
-          vextractf64x2 $0xab, %ymm21, %xmm27 {%k7}
-
-// CHECK: vextractf64x2 $171, %ymm21, %xmm27 {%k7} {z}
-// CHECK:  encoding: [0x62,0x83,0xfd,0xaf,0x19,0xeb,0xab]
-          vextractf64x2 $0xab, %ymm21, %xmm27 {%k7} {z}
-
-// CHECK: vextractf64x2 $123, %ymm21, %xmm27
-// CHECK:  encoding: [0x62,0x83,0xfd,0x28,0x19,0xeb,0x7b]
-          vextractf64x2 $0x7b, %ymm21, %xmm27
-
-// CHECK: vextractf64x2 $171, %ymm20, (%rcx)
-// CHECK:  encoding: [0x62,0xe3,0xfd,0x28,0x19,0x21,0xab]
-          vextractf64x2 $0xab, %ymm20,(%rcx)
-
-// CHECK: vextractf64x2 $171, %ymm20, (%rcx) {%k1}
-// CHECK:  encoding: [0x62,0xe3,0xfd,0x29,0x19,0x21,0xab]
-          vextractf64x2 $0xab, %ymm20,(%rcx) {%k1}
-
-// CHECK: vextractf64x2 $123, %ymm20, (%rcx)
-// CHECK:  encoding: [0x62,0xe3,0xfd,0x28,0x19,0x21,0x7b]
-          vextractf64x2 $0x7b, %ymm20,(%rcx)
-
-// CHECK: vextractf64x2 $123, %ymm20, 291(%rax,%r14,8)
-// CHECK:  encoding: [0x62,0xa3,0xfd,0x28,0x19,0xa4,0xf0,0x23,0x01,0x00,0x00,0x7b]
-          vextractf64x2 $0x7b, %ymm20,291(%rax,%r14,8)
-
-// CHECK: vextractf64x2 $123, %ymm20, 2032(%rdx)
-// CHECK:  encoding: [0x62,0xe3,0xfd,0x28,0x19,0x62,0x7f,0x7b]
-          vextractf64x2 $0x7b, %ymm20,2032(%rdx)
-
-// CHECK: vextractf64x2 $123, %ymm20, 2048(%rdx)
-// CHECK:  encoding: [0x62,0xe3,0xfd,0x28,0x19,0xa2,0x00,0x08,0x00,0x00,0x7b]
-          vextractf64x2 $0x7b, %ymm20,2048(%rdx)
-
-// CHECK: vextractf64x2 $123, %ymm20, -2048(%rdx)
-// CHECK:  encoding: [0x62,0xe3,0xfd,0x28,0x19,0x62,0x80,0x7b]
-          vextractf64x2 $0x7b, %ymm20,-2048(%rdx)
-
-// CHECK: vextractf64x2 $123, %ymm20, -2064(%rdx)
-// CHECK:  encoding: [0x62,0xe3,0xfd,0x28,0x19,0xa2,0xf0,0xf7,0xff,0xff,0x7b]
-          vextractf64x2 $0x7b, %ymm20,-2064(%rdx)
-
-// CHECK: vextractf64x2 $171, %ymm26, %xmm28
-// CHECK:  encoding: [0x62,0x03,0xfd,0x28,0x19,0xd4,0xab]
-          vextractf64x2 $0xab, %ymm26, %xmm28
-
-// CHECK: vextractf64x2 $171, %ymm26, %xmm28 {%k4}
-// CHECK:  encoding: [0x62,0x03,0xfd,0x2c,0x19,0xd4,0xab]
-          vextractf64x2 $0xab, %ymm26, %xmm28 {%k4}
-
-// CHECK: vextractf64x2 $171, %ymm26, %xmm28 {%k4} {z}
-// CHECK:  encoding: [0x62,0x03,0xfd,0xac,0x19,0xd4,0xab]
-          vextractf64x2 $0xab, %ymm26, %xmm28 {%k4} {z}
-
-// CHECK: vextractf64x2 $123, %ymm26, %xmm28
-// CHECK:  encoding: [0x62,0x03,0xfd,0x28,0x19,0xd4,0x7b]
-          vextractf64x2 $0x7b, %ymm26, %xmm28
-
-// CHECK: vextractf64x2 $171, %ymm17, (%rcx)
-// CHECK:  encoding: [0x62,0xe3,0xfd,0x28,0x19,0x09,0xab]
-          vextractf64x2 $0xab, %ymm17,(%rcx)
-
-// CHECK: vextractf64x2 $171, %ymm17, (%rcx) {%k2}
-// CHECK:  encoding: [0x62,0xe3,0xfd,0x2a,0x19,0x09,0xab]
-          vextractf64x2 $0xab, %ymm17,(%rcx) {%k2}
-
-// CHECK: vextractf64x2 $123, %ymm17, (%rcx)
-// CHECK:  encoding: [0x62,0xe3,0xfd,0x28,0x19,0x09,0x7b]
-          vextractf64x2 $0x7b, %ymm17,(%rcx)
-
-// CHECK: vextractf64x2 $123, %ymm17, 4660(%rax,%r14,8)
-// CHECK:  encoding: [0x62,0xa3,0xfd,0x28,0x19,0x8c,0xf0,0x34,0x12,0x00,0x00,0x7b]
-          vextractf64x2 $0x7b, %ymm17,4660(%rax,%r14,8)
-
-// CHECK: vextractf64x2 $123, %ymm17, 2032(%rdx)
-// CHECK:  encoding: [0x62,0xe3,0xfd,0x28,0x19,0x4a,0x7f,0x7b]
-          vextractf64x2 $0x7b, %ymm17,2032(%rdx)
-
-// CHECK: vextractf64x2 $123, %ymm17, 2048(%rdx)
-// CHECK:  encoding: [0x62,0xe3,0xfd,0x28,0x19,0x8a,0x00,0x08,0x00,0x00,0x7b]
-          vextractf64x2 $0x7b, %ymm17,2048(%rdx)
-
-// CHECK: vextractf64x2 $123, %ymm17, -2048(%rdx)
-// CHECK:  encoding: [0x62,0xe3,0xfd,0x28,0x19,0x4a,0x80,0x7b]
-          vextractf64x2 $0x7b, %ymm17,-2048(%rdx)
-
-// CHECK: vextractf64x2 $123, %ymm17, -2064(%rdx)
-// CHECK:  encoding: [0x62,0xe3,0xfd,0x28,0x19,0x8a,0xf0,0xf7,0xff,0xff,0x7b]
-          vextractf64x2 $0x7b, %ymm17,-2064(%rdx)
-
-// CHECK: vextracti64x2 $171, %ymm24, %xmm29
-// CHECK:  encoding: [0x62,0x03,0xfd,0x28,0x39,0xc5,0xab]
-          vextracti64x2 $0xab, %ymm24, %xmm29
-
-// CHECK: vextracti64x2 $171, %ymm24, %xmm29 {%k7}
-// CHECK:  encoding: [0x62,0x03,0xfd,0x2f,0x39,0xc5,0xab]
-          vextracti64x2 $0xab, %ymm24, %xmm29 {%k7}
-
-// CHECK: vextracti64x2 $171, %ymm24, %xmm29 {%k7} {z}
-// CHECK:  encoding: [0x62,0x03,0xfd,0xaf,0x39,0xc5,0xab]
-          vextracti64x2 $0xab, %ymm24, %xmm29 {%k7} {z}
-
-// CHECK: vextracti64x2 $123, %ymm24, %xmm29
-// CHECK:  encoding: [0x62,0x03,0xfd,0x28,0x39,0xc5,0x7b]
-          vextracti64x2 $0x7b, %ymm24, %xmm29
-
-// CHECK: vextracti64x2 $171, %ymm17, (%rcx)
-// CHECK:  encoding: [0x62,0xe3,0xfd,0x28,0x39,0x09,0xab]
-          vextracti64x2 $0xab, %ymm17,(%rcx)
-
-// CHECK: vextracti64x2 $171, %ymm17, (%rcx) {%k1}
-// CHECK:  encoding: [0x62,0xe3,0xfd,0x29,0x39,0x09,0xab]
-          vextracti64x2 $0xab, %ymm17,(%rcx) {%k1}
-
-// CHECK: vextracti64x2 $123, %ymm17, (%rcx)
-// CHECK:  encoding: [0x62,0xe3,0xfd,0x28,0x39,0x09,0x7b]
-          vextracti64x2 $0x7b, %ymm17,(%rcx)
-
-// CHECK: vextracti64x2 $123, %ymm17, 291(%rax,%r14,8)
-// CHECK:  encoding: [0x62,0xa3,0xfd,0x28,0x39,0x8c,0xf0,0x23,0x01,0x00,0x00,0x7b]
-          vextracti64x2 $0x7b, %ymm17,291(%rax,%r14,8)
-
-// CHECK: vextracti64x2 $123, %ymm17, 2032(%rdx)
-// CHECK:  encoding: [0x62,0xe3,0xfd,0x28,0x39,0x4a,0x7f,0x7b]
-          vextracti64x2 $0x7b, %ymm17,2032(%rdx)
-
-// CHECK: vextracti64x2 $123, %ymm17, 2048(%rdx)
-// CHECK:  encoding: [0x62,0xe3,0xfd,0x28,0x39,0x8a,0x00,0x08,0x00,0x00,0x7b]
-          vextracti64x2 $0x7b, %ymm17,2048(%rdx)
-
-// CHECK: vextracti64x2 $123, %ymm17, -2048(%rdx)
-// CHECK:  encoding: [0x62,0xe3,0xfd,0x28,0x39,0x4a,0x80,0x7b]
-          vextracti64x2 $0x7b, %ymm17,-2048(%rdx)
-
-// CHECK: vextracti64x2 $123, %ymm17, -2064(%rdx)
-// CHECK:  encoding: [0x62,0xe3,0xfd,0x28,0x39,0x8a,0xf0,0xf7,0xff,0xff,0x7b]
-          vextracti64x2 $0x7b, %ymm17,-2064(%rdx)
-
-// CHECK: vextracti64x2 $171, %ymm17, %xmm29
-// CHECK:  encoding: [0x62,0x83,0xfd,0x28,0x39,0xcd,0xab]
-          vextracti64x2 $0xab, %ymm17, %xmm29
-
-// CHECK: vextracti64x2 $171, %ymm17, %xmm29 {%k5}
-// CHECK:  encoding: [0x62,0x83,0xfd,0x2d,0x39,0xcd,0xab]
-          vextracti64x2 $0xab, %ymm17, %xmm29 {%k5}
-
-// CHECK: vextracti64x2 $171, %ymm17, %xmm29 {%k5} {z}
-// CHECK:  encoding: [0x62,0x83,0xfd,0xad,0x39,0xcd,0xab]
-          vextracti64x2 $0xab, %ymm17, %xmm29 {%k5} {z}
-
-// CHECK: vextracti64x2 $123, %ymm17, %xmm29
-// CHECK:  encoding: [0x62,0x83,0xfd,0x28,0x39,0xcd,0x7b]
-          vextracti64x2 $0x7b, %ymm17, %xmm29
-
-// CHECK: vextracti64x2 $171, %ymm20, (%rcx)
-// CHECK:  encoding: [0x62,0xe3,0xfd,0x28,0x39,0x21,0xab]
-          vextracti64x2 $0xab, %ymm20,(%rcx)
-
-// CHECK: vextracti64x2 $171, %ymm20, (%rcx) {%k2}
-// CHECK:  encoding: [0x62,0xe3,0xfd,0x2a,0x39,0x21,0xab]
-          vextracti64x2 $0xab, %ymm20,(%rcx) {%k2}
-
-// CHECK: vextracti64x2 $123, %ymm20, (%rcx)
-// CHECK:  encoding: [0x62,0xe3,0xfd,0x28,0x39,0x21,0x7b]
-          vextracti64x2 $0x7b, %ymm20,(%rcx)
-
-// CHECK: vextracti64x2 $123, %ymm20, 4660(%rax,%r14,8)
-// CHECK:  encoding: [0x62,0xa3,0xfd,0x28,0x39,0xa4,0xf0,0x34,0x12,0x00,0x00,0x7b]
-          vextracti64x2 $0x7b, %ymm20,4660(%rax,%r14,8)
-
-// CHECK: vextracti64x2 $123, %ymm20, 2032(%rdx)
-// CHECK:  encoding: [0x62,0xe3,0xfd,0x28,0x39,0x62,0x7f,0x7b]
-          vextracti64x2 $0x7b, %ymm20,2032(%rdx)
-
-// CHECK: vextracti64x2 $123, %ymm20, 2048(%rdx)
-// CHECK:  encoding: [0x62,0xe3,0xfd,0x28,0x39,0xa2,0x00,0x08,0x00,0x00,0x7b]
-          vextracti64x2 $0x7b, %ymm20,2048(%rdx)
-
-// CHECK: vextracti64x2 $123, %ymm20, -2048(%rdx)
-// CHECK:  encoding: [0x62,0xe3,0xfd,0x28,0x39,0x62,0x80,0x7b]
-          vextracti64x2 $0x7b, %ymm20,-2048(%rdx)
-
-// CHECK: vextracti64x2 $123, %ymm20, -2064(%rdx)
-// CHECK:  encoding: [0x62,0xe3,0xfd,0x28,0x39,0xa2,0xf0,0xf7,0xff,0xff,0x7b]
-          vextracti64x2 $0x7b, %ymm20,-2064(%rdx)
-
-// CHECK: vfpclasspd $171, %xmm18, %k2
-// CHECK:  encoding: [0x62,0xb3,0xfd,0x08,0x66,0xd2,0xab]
-          vfpclasspd $0xab, %xmm18, %k2
-
-// CHECK: vfpclasspd $171, %xmm18, %k2
-// CHECK:  encoding: [0x62,0xb3,0xfd,0x08,0x66,0xd2,0xab]
-          vfpclasspdx $0xab, %xmm18, %k2
-
-// CHECK: vfpclasspd $171, %xmm18, %k2 {%k7}
-// CHECK:  encoding: [0x62,0xb3,0xfd,0x0f,0x66,0xd2,0xab]
-          vfpclasspd $0xab, %xmm18, %k2 {%k7}
-
-// CHECK: vfpclasspd $171, %xmm18, %k2 {%k7}
-// CHECK:  encoding: [0x62,0xb3,0xfd,0x0f,0x66,0xd2,0xab]
-          vfpclasspdx $0xab, %xmm18, %k2 {%k7}
-
-// CHECK: vfpclasspd $123,  %xmm18, %k2
-// CHECK:  encoding: [0x62,0xb3,0xfd,0x08,0x66,0xd2,0x7b]
-          vfpclasspd $0x7b, %xmm18, %k2
-
-// CHECK: vfpclasspd $123,  %xmm18, %k2
-// CHECK:  encoding: [0x62,0xb3,0xfd,0x08,0x66,0xd2,0x7b]
-          vfpclasspdx $0x7b, %xmm18, %k2
-
-// CHECK: vfpclasspdx $123, (%rcx), %k2
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x08,0x66,0x11,0x7b]
-          vfpclasspdx $0x7b,(%rcx), %k2
-
-// CHECK: vfpclasspdx $123, 291(%rax,%r14,8), %k2
-// CHECK:  encoding: [0x62,0xb3,0xfd,0x08,0x66,0x94,0xf0,0x23,0x01,0x00,0x00,0x7b]
-          vfpclasspdx $0x7b,291(%rax,%r14,8), %k2
-
-// CHECK: vfpclasspd $123, (%rcx){1to2}, %k2
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x18,0x66,0x11,0x7b]
-          vfpclasspd $0x7b,(%rcx){1to2}, %k2
-
-// CHECK: vfpclasspd $123, (%rcx){1to2}, %k2
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x18,0x66,0x11,0x7b]
-          vfpclasspdx $0x7b,(%rcx){1to2}, %k2
-
-// CHECK: vfpclasspdx $123, 2032(%rdx), %k2
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x08,0x66,0x52,0x7f,0x7b]
-          vfpclasspdx $0x7b,2032(%rdx), %k2
-
-// CHECK: vfpclasspdx $123, 2048(%rdx), %k2
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x08,0x66,0x92,0x00,0x08,0x00,0x00,0x7b]
-          vfpclasspdx $0x7b,2048(%rdx), %k2
-
-// CHECK: vfpclasspdx $123, -2048(%rdx), %k2
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x08,0x66,0x52,0x80,0x7b]
-          vfpclasspdx $0x7b,-2048(%rdx), %k2
-
-// CHECK: vfpclasspdx $123, -2064(%rdx), %k2
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x08,0x66,0x92,0xf0,0xf7,0xff,0xff,0x7b]
-          vfpclasspdx $0x7b,-2064(%rdx), %k2
-
-// CHECK: vfpclasspd $123, 1016(%rdx){1to2}, %k2
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x18,0x66,0x52,0x7f,0x7b]
-          vfpclasspd $0x7b,1016(%rdx){1to2}, %k2
-
-// CHECK: vfpclasspd $123, 1016(%rdx){1to2}, %k2
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x18,0x66,0x52,0x7f,0x7b]
-          vfpclasspdx $0x7b,1016(%rdx){1to2}, %k2
-
-// CHECK: vfpclasspd $123, 1024(%rdx){1to2}, %k2
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x18,0x66,0x92,0x00,0x04,0x00,0x00,0x7b]
-          vfpclasspd $0x7b,1024(%rdx){1to2}, %k2
-
-// CHECK: vfpclasspd $123, 1024(%rdx){1to2}, %k2
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x18,0x66,0x92,0x00,0x04,0x00,0x00,0x7b]
-          vfpclasspdx $0x7b,1024(%rdx){1to2}, %k2
-
-// CHECK: vfpclasspd $123, -1024(%rdx){1to2}, %k2
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x18,0x66,0x52,0x80,0x7b]
-          vfpclasspd $0x7b,-1024(%rdx){1to2}, %k2
-
-// CHECK: vfpclasspd $123, -1024(%rdx){1to2}, %k2
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x18,0x66,0x52,0x80,0x7b]
-          vfpclasspdx $0x7b,-1024(%rdx){1to2}, %k2
-
-// CHECK: vfpclasspd $123, -1032(%rdx){1to2}, %k2
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x18,0x66,0x92,0xf8,0xfb,0xff,0xff,0x7b]
-          vfpclasspd $0x7b,-1032(%rdx){1to2}, %k2
-
-// CHECK: vfpclasspd $123, -1032(%rdx){1to2}, %k2
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x18,0x66,0x92,0xf8,0xfb,0xff,0xff,0x7b]
-          vfpclasspdx $0x7b,-1032(%rdx){1to2}, %k2
-
-// CHECK: vfpclasspd $171, %ymm25, %k4
-// CHECK:  encoding: [0x62,0x93,0xfd,0x28,0x66,0xe1,0xab]
-          vfpclasspd $0xab, %ymm25, %k4
-
-// CHECK: vfpclasspd $171, %ymm25, %k4
-// CHECK:  encoding: [0x62,0x93,0xfd,0x28,0x66,0xe1,0xab]
-          vfpclasspdy $0xab, %ymm25, %k4
-
-// CHECK: vfpclasspd $171, %ymm25, %k4 {%k6}
-// CHECK:  encoding: [0x62,0x93,0xfd,0x2e,0x66,0xe1,0xab]
-          vfpclasspd $0xab, %ymm25, %k4 {%k6}
-
-// CHECK: vfpclasspd $171, %ymm25, %k4 {%k6}
-// CHECK:  encoding: [0x62,0x93,0xfd,0x2e,0x66,0xe1,0xab]
-          vfpclasspdy $0xab, %ymm25, %k4 {%k6}
-
-// CHECK: vfpclasspd $123,  %ymm25, %k4
-// CHECK:  encoding: [0x62,0x93,0xfd,0x28,0x66,0xe1,0x7b]
-          vfpclasspd $0x7b, %ymm25, %k4
-
-// CHECK: vfpclasspd $123,  %ymm25, %k4
-// CHECK:  encoding: [0x62,0x93,0xfd,0x28,0x66,0xe1,0x7b]
-          vfpclasspdy $0x7b, %ymm25, %k4
-
-// CHECK: vfpclasspdy $123, (%rcx), %k4
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x28,0x66,0x21,0x7b]
-          vfpclasspdy $0x7b,(%rcx), %k4
-
-// CHECK: vfpclasspdy $123, 291(%rax,%r14,8), %k4
-// CHECK:  encoding: [0x62,0xb3,0xfd,0x28,0x66,0xa4,0xf0,0x23,0x01,0x00,0x00,0x7b]
-          vfpclasspdy $0x7b,291(%rax,%r14,8), %k4
-
-// CHECK: vfpclasspd $123, (%rcx){1to4}, %k4
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x38,0x66,0x21,0x7b]
-          vfpclasspd $0x7b,(%rcx){1to4}, %k4
-
-// CHECK: vfpclasspd $123, (%rcx){1to4}, %k4
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x38,0x66,0x21,0x7b]
-          vfpclasspdy $0x7b,(%rcx){1to4}, %k4
-
-// CHECK: vfpclasspdy $123, 4064(%rdx), %k4
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x28,0x66,0x62,0x7f,0x7b]
-          vfpclasspdy $0x7b,4064(%rdx), %k4
-
-// CHECK: vfpclasspdy $123, 4096(%rdx), %k4
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x28,0x66,0xa2,0x00,0x10,0x00,0x00,0x7b]
-          vfpclasspdy $0x7b,4096(%rdx), %k4
-
-// CHECK: vfpclasspdy $123, -4096(%rdx), %k4
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x28,0x66,0x62,0x80,0x7b]
-          vfpclasspdy $0x7b,-4096(%rdx), %k4
-
-// CHECK: vfpclasspdy $123, -4128(%rdx), %k4
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x28,0x66,0xa2,0xe0,0xef,0xff,0xff,0x7b]
-          vfpclasspdy $0x7b,-4128(%rdx), %k4
-
-// CHECK: vfpclasspd $123, 1016(%rdx){1to4}, %k4
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x38,0x66,0x62,0x7f,0x7b]
-          vfpclasspd $0x7b,1016(%rdx){1to4}, %k4
-
-// CHECK: vfpclasspd $123, 1016(%rdx){1to4}, %k4
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x38,0x66,0x62,0x7f,0x7b]
-          vfpclasspdy $0x7b,1016(%rdx){1to4}, %k4
-
-// CHECK: vfpclasspd $123, 1024(%rdx){1to4}, %k4
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x38,0x66,0xa2,0x00,0x04,0x00,0x00,0x7b]
-          vfpclasspd $0x7b,1024(%rdx){1to4}, %k4
-
-// CHECK: vfpclasspd $123, 1024(%rdx){1to4}, %k4
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x38,0x66,0xa2,0x00,0x04,0x00,0x00,0x7b]
-          vfpclasspdy $0x7b,1024(%rdx){1to4}, %k4
-
-// CHECK: vfpclasspd $123, -1024(%rdx){1to4}, %k4
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x38,0x66,0x62,0x80,0x7b]
-          vfpclasspd $0x7b,-1024(%rdx){1to4}, %k4
-
-// CHECK: vfpclasspd $123, -1024(%rdx){1to4}, %k4
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x38,0x66,0x62,0x80,0x7b]
-          vfpclasspdy $0x7b,-1024(%rdx){1to4}, %k4
-
-// CHECK: vfpclasspd $123, -1032(%rdx){1to4}, %k4
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x38,0x66,0xa2,0xf8,0xfb,0xff,0xff,0x7b]
-          vfpclasspd $0x7b,-1032(%rdx){1to4}, %k4
-
-// CHECK: vfpclasspd $123, -1032(%rdx){1to4}, %k4
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x38,0x66,0xa2,0xf8,0xfb,0xff,0xff,0x7b]
-          vfpclasspdy $0x7b,-1032(%rdx){1to4}, %k4
-
-// CHECK: vfpclassps $171, %xmm20, %k4
-// CHECK:  encoding: [0x62,0xb3,0x7d,0x08,0x66,0xe4,0xab]
-          vfpclassps $0xab, %xmm20, %k4
-
-// CHECK: vfpclassps $171, %xmm20, %k4
-// CHECK:  encoding: [0x62,0xb3,0x7d,0x08,0x66,0xe4,0xab]
-          vfpclasspsx $0xab, %xmm20, %k4
-
-// CHECK: vfpclassps $171, %xmm20, %k4 {%k5}
-// CHECK:  encoding: [0x62,0xb3,0x7d,0x0d,0x66,0xe4,0xab]
-          vfpclassps $0xab, %xmm20, %k4 {%k5}
-
-// CHECK: vfpclassps $171, %xmm20, %k4 {%k5}
-// CHECK:  encoding: [0x62,0xb3,0x7d,0x0d,0x66,0xe4,0xab]
-          vfpclasspsx $0xab, %xmm20, %k4 {%k5}
-
-// CHECK: vfpclassps $123,  %xmm20, %k4
-// CHECK:  encoding: [0x62,0xb3,0x7d,0x08,0x66,0xe4,0x7b]
-          vfpclassps $0x7b, %xmm20, %k4
-
-// CHECK: vfpclassps $123,  %xmm20, %k4
-// CHECK:  encoding: [0x62,0xb3,0x7d,0x08,0x66,0xe4,0x7b]
-          vfpclasspsx $0x7b, %xmm20, %k4
-
-// CHECK: vfpclasspsx $123, (%rcx), %k4
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x08,0x66,0x21,0x7b]
-          vfpclasspsx $0x7b,(%rcx), %k4
-
-// CHECK: vfpclasspsx $123, 291(%rax,%r14,8), %k4
-// CHECK:  encoding: [0x62,0xb3,0x7d,0x08,0x66,0xa4,0xf0,0x23,0x01,0x00,0x00,0x7b]
-          vfpclasspsx $0x7b,291(%rax,%r14,8), %k4
-
-// CHECK: vfpclassps $123, (%rcx){1to4}, %k4
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x18,0x66,0x21,0x7b]
-          vfpclassps $0x7b,(%rcx){1to4}, %k4
-
-// CHECK: vfpclassps $123, (%rcx){1to4}, %k4
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x18,0x66,0x21,0x7b]
-          vfpclasspsx $0x7b,(%rcx){1to4}, %k4
-
-// CHECK: vfpclasspsx $123, 2032(%rdx), %k4
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x08,0x66,0x62,0x7f,0x7b]
-          vfpclasspsx $0x7b,2032(%rdx), %k4
-
-// CHECK: vfpclasspsx $123, 2048(%rdx), %k4
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x08,0x66,0xa2,0x00,0x08,0x00,0x00,0x7b]
-          vfpclasspsx $0x7b,2048(%rdx), %k4
-
-// CHECK: vfpclasspsx $123, -2048(%rdx), %k4
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x08,0x66,0x62,0x80,0x7b]
-          vfpclasspsx $0x7b,-2048(%rdx), %k4
-
-// CHECK: vfpclasspsx $123, -2064(%rdx), %k4
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x08,0x66,0xa2,0xf0,0xf7,0xff,0xff,0x7b]
-          vfpclasspsx $0x7b,-2064(%rdx), %k4
-
-// CHECK: vfpclassps $123, 508(%rdx){1to4}, %k4
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x18,0x66,0x62,0x7f,0x7b]
-          vfpclassps $0x7b,508(%rdx){1to4}, %k4
-
-// CHECK: vfpclassps $123, 508(%rdx){1to4}, %k4
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x18,0x66,0x62,0x7f,0x7b]
-          vfpclasspsx $0x7b,508(%rdx){1to4}, %k4
-
-// CHECK: vfpclassps $123, 512(%rdx){1to4}, %k4
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x18,0x66,0xa2,0x00,0x02,0x00,0x00,0x7b]
-          vfpclassps $0x7b,512(%rdx){1to4}, %k4
-
-// CHECK: vfpclassps $123, 512(%rdx){1to4}, %k4
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x18,0x66,0xa2,0x00,0x02,0x00,0x00,0x7b]
-          vfpclasspsx $0x7b,512(%rdx){1to4}, %k4
-
-// CHECK: vfpclassps $123, -512(%rdx){1to4}, %k4
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x18,0x66,0x62,0x80,0x7b]
-          vfpclassps $0x7b,-512(%rdx){1to4}, %k4
-
-// CHECK: vfpclassps $123, -512(%rdx){1to4}, %k4
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x18,0x66,0x62,0x80,0x7b]
-          vfpclasspsx $0x7b,-512(%rdx){1to4}, %k4
-
-// CHECK: vfpclassps $123, -516(%rdx){1to4}, %k4
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x18,0x66,0xa2,0xfc,0xfd,0xff,0xff,0x7b]
-          vfpclassps $0x7b,-516(%rdx){1to4}, %k4
-
-// CHECK: vfpclassps $123, -516(%rdx){1to4}, %k4
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x18,0x66,0xa2,0xfc,0xfd,0xff,0xff,0x7b]
-          vfpclasspsx $0x7b,-516(%rdx){1to4}, %k4
-
-// CHECK: vfpclassps $171, %ymm17, %k4
-// CHECK:  encoding: [0x62,0xb3,0x7d,0x28,0x66,0xe1,0xab]
-          vfpclassps $0xab, %ymm17, %k4
-
-// CHECK: vfpclassps $171, %ymm17, %k4
-// CHECK:  encoding: [0x62,0xb3,0x7d,0x28,0x66,0xe1,0xab]
-          vfpclasspsy $0xab, %ymm17, %k4
-
-// CHECK: vfpclassps $171, %ymm17, %k4 {%k5}
-// CHECK:  encoding: [0x62,0xb3,0x7d,0x2d,0x66,0xe1,0xab]
-          vfpclassps $0xab, %ymm17, %k4 {%k5}
-
-// CHECK: vfpclassps $171, %ymm17, %k4 {%k5}
-// CHECK:  encoding: [0x62,0xb3,0x7d,0x2d,0x66,0xe1,0xab]
-          vfpclasspsy $0xab, %ymm17, %k4 {%k5}
-
-// CHECK: vfpclassps $123,  %ymm17, %k4
-// CHECK:  encoding: [0x62,0xb3,0x7d,0x28,0x66,0xe1,0x7b]
-          vfpclassps $0x7b, %ymm17, %k4
-
-// CHECK: vfpclassps $123,  %ymm17, %k4
-// CHECK:  encoding: [0x62,0xb3,0x7d,0x28,0x66,0xe1,0x7b]
-          vfpclasspsy $0x7b, %ymm17, %k4
-
-// CHECK: vfpclasspsy $123, (%rcx), %k4
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x28,0x66,0x21,0x7b]
-          vfpclasspsy $0x7b,(%rcx), %k4
-
-// CHECK: vfpclasspsy $123, 291(%rax,%r14,8), %k4
-// CHECK:  encoding: [0x62,0xb3,0x7d,0x28,0x66,0xa4,0xf0,0x23,0x01,0x00,0x00,0x7b]
-          vfpclasspsy $0x7b,291(%rax,%r14,8), %k4
-
-// CHECK: vfpclassps $123, (%rcx){1to8}, %k4
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x38,0x66,0x21,0x7b]
-          vfpclassps $0x7b,(%rcx){1to8}, %k4
-
-// CHECK: vfpclassps $123, (%rcx){1to8}, %k4
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x38,0x66,0x21,0x7b]
-          vfpclasspsy $0x7b,(%rcx){1to8}, %k4
-
-// CHECK: vfpclasspsy $123, 4064(%rdx), %k4
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x28,0x66,0x62,0x7f,0x7b]
-          vfpclasspsy $0x7b,4064(%rdx), %k4
-
-// CHECK: vfpclasspsy $123, 4096(%rdx), %k4
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x28,0x66,0xa2,0x00,0x10,0x00,0x00,0x7b]
-          vfpclasspsy $0x7b,4096(%rdx), %k4
-
-// CHECK: vfpclasspsy $123, -4096(%rdx), %k4
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x28,0x66,0x62,0x80,0x7b]
-          vfpclasspsy $0x7b,-4096(%rdx), %k4
-
-// CHECK: vfpclasspsy $123, -4128(%rdx), %k4
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x28,0x66,0xa2,0xe0,0xef,0xff,0xff,0x7b]
-          vfpclasspsy $0x7b,-4128(%rdx), %k4
-
-// CHECK: vfpclassps $123, 508(%rdx){1to8}, %k4
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x38,0x66,0x62,0x7f,0x7b]
-          vfpclassps $0x7b,508(%rdx){1to8}, %k4
-
-// CHECK: vfpclassps $123, 508(%rdx){1to8}, %k4
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x38,0x66,0x62,0x7f,0x7b]
-          vfpclasspsy $0x7b,508(%rdx){1to8}, %k4
-
-// CHECK: vfpclassps $123, 512(%rdx){1to8}, %k4
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x38,0x66,0xa2,0x00,0x02,0x00,0x00,0x7b]
-          vfpclassps $0x7b,512(%rdx){1to8}, %k4
-
-// CHECK: vfpclassps $123, 512(%rdx){1to8}, %k4
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x38,0x66,0xa2,0x00,0x02,0x00,0x00,0x7b]
-          vfpclasspsy $0x7b,512(%rdx){1to8}, %k4
-
-// CHECK: vfpclassps $123, -512(%rdx){1to8}, %k4
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x38,0x66,0x62,0x80,0x7b]
-          vfpclassps $0x7b,-512(%rdx){1to8}, %k4
-
-// CHECK: vfpclassps $123, -512(%rdx){1to8}, %k4
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x38,0x66,0x62,0x80,0x7b]
-          vfpclasspsy $0x7b,-512(%rdx){1to8}, %k4
-
-// CHECK: vfpclassps $123, -516(%rdx){1to8}, %k4
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x38,0x66,0xa2,0xfc,0xfd,0xff,0xff,0x7b]
-          vfpclassps $0x7b,-516(%rdx){1to8}, %k4
-
-// CHECK: vfpclassps $123, -516(%rdx){1to8}, %k4
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x38,0x66,0xa2,0xfc,0xfd,0xff,0xff,0x7b]
-          vfpclasspsy $0x7b,-516(%rdx){1to8}, %k4
-
-// CHECK: vfpclasspd $171, %xmm26, %k3
-// CHECK:  encoding: [0x62,0x93,0xfd,0x08,0x66,0xda,0xab]
-          vfpclasspd $0xab, %xmm26, %k3
-
-// CHECK: vfpclasspd $171, %xmm26, %k3
-// CHECK:  encoding: [0x62,0x93,0xfd,0x08,0x66,0xda,0xab]
-          vfpclasspdx $0xab, %xmm26, %k3
-
-// CHECK: vfpclasspd $171, %xmm26, %k3 {%k5}
-// CHECK:  encoding: [0x62,0x93,0xfd,0x0d,0x66,0xda,0xab]
-          vfpclasspd $0xab, %xmm26, %k3 {%k5}
-
-// CHECK: vfpclasspd $171, %xmm26, %k3 {%k5}
-// CHECK:  encoding: [0x62,0x93,0xfd,0x0d,0x66,0xda,0xab]
-          vfpclasspdx $0xab, %xmm26, %k3 {%k5}
-
-// CHECK: vfpclasspd $123,  %xmm26, %k3
-// CHECK:  encoding: [0x62,0x93,0xfd,0x08,0x66,0xda,0x7b]
-          vfpclasspd $0x7b, %xmm26, %k3
-
-// CHECK: vfpclasspd $123,  %xmm26, %k3
-// CHECK:  encoding: [0x62,0x93,0xfd,0x08,0x66,0xda,0x7b]
-          vfpclasspdx $0x7b, %xmm26, %k3
-
-// CHECK: vfpclasspdx $123, (%rcx), %k3
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x08,0x66,0x19,0x7b]
-          vfpclasspdx $0x7b,(%rcx), %k3
-
-// CHECK: vfpclasspdx $123, 4660(%rax,%r14,8), %k3
-// CHECK:  encoding: [0x62,0xb3,0xfd,0x08,0x66,0x9c,0xf0,0x34,0x12,0x00,0x00,0x7b]
-          vfpclasspdx $0x7b,4660(%rax,%r14,8), %k3
-
-// CHECK: vfpclasspd $123, (%rcx){1to2}, %k3
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x18,0x66,0x19,0x7b]
-          vfpclasspd $0x7b,(%rcx){1to2}, %k3
-
-// CHECK: vfpclasspd $123, (%rcx){1to2}, %k3
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x18,0x66,0x19,0x7b]
-          vfpclasspdx $0x7b,(%rcx){1to2}, %k3
-
-// CHECK: vfpclasspdx $123, 2032(%rdx), %k3
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x08,0x66,0x5a,0x7f,0x7b]
-          vfpclasspdx $0x7b,2032(%rdx), %k3
-
-// CHECK: vfpclasspdx $123, 2048(%rdx), %k3
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x08,0x66,0x9a,0x00,0x08,0x00,0x00,0x7b]
-          vfpclasspdx $0x7b,2048(%rdx), %k3
-
-// CHECK: vfpclasspdx $123, -2048(%rdx), %k3
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x08,0x66,0x5a,0x80,0x7b]
-          vfpclasspdx $0x7b,-2048(%rdx), %k3
-
-// CHECK: vfpclasspdx $123, -2064(%rdx), %k3
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x08,0x66,0x9a,0xf0,0xf7,0xff,0xff,0x7b]
-          vfpclasspdx $0x7b,-2064(%rdx), %k3
-
-// CHECK: vfpclasspd $123, 1016(%rdx){1to2}, %k3
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x18,0x66,0x5a,0x7f,0x7b]
-          vfpclasspd $0x7b,1016(%rdx){1to2}, %k3
-
-// CHECK: vfpclasspd $123, 1016(%rdx){1to2}, %k3
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x18,0x66,0x5a,0x7f,0x7b]
-          vfpclasspdx $0x7b,1016(%rdx){1to2}, %k3
-
-// CHECK: vfpclasspd $123, 1024(%rdx){1to2}, %k3
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x18,0x66,0x9a,0x00,0x04,0x00,0x00,0x7b]
-          vfpclasspd $0x7b,1024(%rdx){1to2}, %k3
-
-// CHECK: vfpclasspd $123, 1024(%rdx){1to2}, %k3
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x18,0x66,0x9a,0x00,0x04,0x00,0x00,0x7b]
-          vfpclasspdx $0x7b,1024(%rdx){1to2}, %k3
-
-// CHECK: vfpclasspd $123, -1024(%rdx){1to2}, %k3
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x18,0x66,0x5a,0x80,0x7b]
-          vfpclasspd $0x7b,-1024(%rdx){1to2}, %k3
-
-// CHECK: vfpclasspd $123, -1024(%rdx){1to2}, %k3
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x18,0x66,0x5a,0x80,0x7b]
-          vfpclasspdx $0x7b,-1024(%rdx){1to2}, %k3
-
-// CHECK: vfpclasspd $123, -1032(%rdx){1to2}, %k3
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x18,0x66,0x9a,0xf8,0xfb,0xff,0xff,0x7b]
-          vfpclasspd $0x7b,-1032(%rdx){1to2}, %k3
-
-// CHECK: vfpclasspd $123, -1032(%rdx){1to2}, %k3
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x18,0x66,0x9a,0xf8,0xfb,0xff,0xff,0x7b]
-          vfpclasspdx $0x7b,-1032(%rdx){1to2}, %k3
-
-// CHECK: vfpclasspd $171, %ymm26, %k2
-// CHECK:  encoding: [0x62,0x93,0xfd,0x28,0x66,0xd2,0xab]
-          vfpclasspd $0xab, %ymm26, %k2
-
-// CHECK: vfpclasspd $171, %ymm26, %k2
-// CHECK:  encoding: [0x62,0x93,0xfd,0x28,0x66,0xd2,0xab]
-          vfpclasspdy $0xab, %ymm26, %k2
-
-// CHECK: vfpclasspd $171, %ymm26, %k2 {%k6}
-// CHECK:  encoding: [0x62,0x93,0xfd,0x2e,0x66,0xd2,0xab]
-          vfpclasspd $0xab, %ymm26, %k2 {%k6}
-
-// CHECK: vfpclasspd $171, %ymm26, %k2 {%k6}
-// CHECK:  encoding: [0x62,0x93,0xfd,0x2e,0x66,0xd2,0xab]
-          vfpclasspdy $0xab, %ymm26, %k2 {%k6}
-
-// CHECK: vfpclasspd $123,  %ymm26, %k2
-// CHECK:  encoding: [0x62,0x93,0xfd,0x28,0x66,0xd2,0x7b]
-          vfpclasspd $0x7b, %ymm26, %k2
-
-// CHECK: vfpclasspd $123,  %ymm26, %k2
-// CHECK:  encoding: [0x62,0x93,0xfd,0x28,0x66,0xd2,0x7b]
-          vfpclasspdy $0x7b, %ymm26, %k2
-
-// CHECK: vfpclasspdy $123, (%rcx), %k2
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x28,0x66,0x11,0x7b]
-          vfpclasspdy $0x7b,(%rcx), %k2
-
-// CHECK: vfpclasspdy $123, 4660(%rax,%r14,8), %k2
-// CHECK:  encoding: [0x62,0xb3,0xfd,0x28,0x66,0x94,0xf0,0x34,0x12,0x00,0x00,0x7b]
-          vfpclasspdy $0x7b,4660(%rax,%r14,8), %k2
-
-// CHECK: vfpclasspd $123, (%rcx){1to4}, %k2
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x38,0x66,0x11,0x7b]
-          vfpclasspd $0x7b,(%rcx){1to4}, %k2
-
-// CHECK: vfpclasspd $123, (%rcx){1to4}, %k2
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x38,0x66,0x11,0x7b]
-          vfpclasspdy $0x7b,(%rcx){1to4}, %k2
-
-// CHECK: vfpclasspdy $123, 4064(%rdx), %k2
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x28,0x66,0x52,0x7f,0x7b]
-          vfpclasspdy $0x7b,4064(%rdx), %k2
-
-// CHECK: vfpclasspdy $123, 4096(%rdx), %k2
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x28,0x66,0x92,0x00,0x10,0x00,0x00,0x7b]
-          vfpclasspdy $0x7b,4096(%rdx), %k2
-
-// CHECK: vfpclasspdy $123, -4096(%rdx), %k2
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x28,0x66,0x52,0x80,0x7b]
-          vfpclasspdy $0x7b,-4096(%rdx), %k2
-
-// CHECK: vfpclasspdy $123, -4128(%rdx), %k2
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x28,0x66,0x92,0xe0,0xef,0xff,0xff,0x7b]
-          vfpclasspdy $0x7b,-4128(%rdx), %k2
-
-// CHECK: vfpclasspd $123, 1016(%rdx){1to4}, %k2
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x38,0x66,0x52,0x7f,0x7b]
-          vfpclasspd $0x7b,1016(%rdx){1to4}, %k2
-
-// CHECK: vfpclasspd $123, 1016(%rdx){1to4}, %k2
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x38,0x66,0x52,0x7f,0x7b]
-          vfpclasspdy $0x7b,1016(%rdx){1to4}, %k2
-
-// CHECK: vfpclasspd $123, 1024(%rdx){1to4}, %k2
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x38,0x66,0x92,0x00,0x04,0x00,0x00,0x7b]
-          vfpclasspd $0x7b,1024(%rdx){1to4}, %k2
-
-// CHECK: vfpclasspd $123, 1024(%rdx){1to4}, %k2
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x38,0x66,0x92,0x00,0x04,0x00,0x00,0x7b]
-          vfpclasspdy $0x7b,1024(%rdx){1to4}, %k2
-
-// CHECK: vfpclasspd $123, -1024(%rdx){1to4}, %k2
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x38,0x66,0x52,0x80,0x7b]
-          vfpclasspd $0x7b,-1024(%rdx){1to4}, %k2
-
-// CHECK: vfpclasspd $123, -1024(%rdx){1to4}, %k2
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x38,0x66,0x52,0x80,0x7b]
-          vfpclasspdy $0x7b,-1024(%rdx){1to4}, %k2
-
-// CHECK: vfpclasspd $123, -1032(%rdx){1to4}, %k2
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x38,0x66,0x92,0xf8,0xfb,0xff,0xff,0x7b]
-          vfpclasspd $0x7b,-1032(%rdx){1to4}, %k2
-
-// CHECK: vfpclasspd $123, -1032(%rdx){1to4}, %k2
-// CHECK:  encoding: [0x62,0xf3,0xfd,0x38,0x66,0x92,0xf8,0xfb,0xff,0xff,0x7b]
-          vfpclasspdy $0x7b,-1032(%rdx){1to4}, %k2
-
-// CHECK: vfpclassps $171, %xmm29, %k3
-// CHECK:  encoding: [0x62,0x93,0x7d,0x08,0x66,0xdd,0xab]
-          vfpclassps $0xab, %xmm29, %k3
-
-// CHECK: vfpclassps $171, %xmm29, %k3
-// CHECK:  encoding: [0x62,0x93,0x7d,0x08,0x66,0xdd,0xab]
-          vfpclasspsx $0xab, %xmm29, %k3
-
-// check: vfpclassps $171, %xmm29, %k3 {%k6}
-// check:  encoding: [0x62,0x93,0x7d,0x0e,0x66,0xdd,0xab]
-          vfpclassps $0xab, %xmm29, %k3 {%k6}
-
-// CHECK: vfpclassps $171, %xmm29, %k3
-// CHECK:  encoding: [0x62,0x93,0x7d,0x08,0x66,0xdd,0xab]
-          vfpclasspsx $0xab, %xmm29, %k3
-
-// CHECK: vfpclassps $123,  %xmm29, %k3
-// CHECK:  encoding: [0x62,0x93,0x7d,0x08,0x66,0xdd,0x7b]
-          vfpclassps $0x7b, %xmm29, %k3
-
-// CHECK: vfpclassps $123,  %xmm29, %k3
-// CHECK:  encoding: [0x62,0x93,0x7d,0x08,0x66,0xdd,0x7b]
-          vfpclasspsx $0x7b, %xmm29, %k3
-
-// CHECK: vfpclasspsx $123, (%rcx), %k3
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x08,0x66,0x19,0x7b]
-          vfpclasspsx $0x7b,(%rcx), %k3
-
-// CHECK: vfpclasspsx $123, 4660(%rax,%r14,8), %k3
-// CHECK:  encoding: [0x62,0xb3,0x7d,0x08,0x66,0x9c,0xf0,0x34,0x12,0x00,0x00,0x7b]
-          vfpclasspsx $0x7b,4660(%rax,%r14,8), %k3
-
-// CHECK: vfpclassps $123, (%rcx){1to4}, %k3
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x18,0x66,0x19,0x7b]
-          vfpclassps $0x7b,(%rcx){1to4}, %k3
-
-// CHECK: vfpclassps $123, (%rcx){1to4}, %k3
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x18,0x66,0x19,0x7b]
-          vfpclasspsx $0x7b,(%rcx){1to4}, %k3
-
-// CHECK: vfpclasspsx $123, 2032(%rdx), %k3
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x08,0x66,0x5a,0x7f,0x7b]
-          vfpclasspsx $0x7b,2032(%rdx), %k3
-
-// CHECK: vfpclasspsx $123, 2048(%rdx), %k3
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x08,0x66,0x9a,0x00,0x08,0x00,0x00,0x7b]
-          vfpclasspsx $0x7b,2048(%rdx), %k3
-
-// CHECK: vfpclasspsx $123, -2048(%rdx), %k3
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x08,0x66,0x5a,0x80,0x7b]
-          vfpclasspsx $0x7b,-2048(%rdx), %k3
-
-// CHECK: vfpclasspsx $123, -2064(%rdx), %k3
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x08,0x66,0x9a,0xf0,0xf7,0xff,0xff,0x7b]
-          vfpclasspsx $0x7b,-2064(%rdx), %k3
-
-// CHECK: vfpclassps $123, 508(%rdx){1to4}, %k3
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x18,0x66,0x5a,0x7f,0x7b]
-          vfpclassps $0x7b,508(%rdx){1to4}, %k3
-
-// CHECK: vfpclassps $123, 508(%rdx){1to4}, %k3
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x18,0x66,0x5a,0x7f,0x7b]
-          vfpclasspsx $0x7b,508(%rdx){1to4}, %k3
-
-// CHECK: vfpclassps $123, 512(%rdx){1to4}, %k3
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x18,0x66,0x9a,0x00,0x02,0x00,0x00,0x7b]
-          vfpclassps $0x7b,512(%rdx){1to4}, %k3
-
-// CHECK: vfpclassps $123, 512(%rdx){1to4}, %k3
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x18,0x66,0x9a,0x00,0x02,0x00,0x00,0x7b]
-          vfpclasspsx $0x7b,512(%rdx){1to4}, %k3
-
-// CHECK: vfpclassps $123, -512(%rdx){1to4}, %k3
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x18,0x66,0x5a,0x80,0x7b]
-          vfpclassps $0x7b,-512(%rdx){1to4}, %k3
-
-// CHECK: vfpclassps $123, -512(%rdx){1to4}, %k3
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x18,0x66,0x5a,0x80,0x7b]
-          vfpclasspsx $0x7b,-512(%rdx){1to4}, %k3
-
-// CHECK: vfpclassps $123, -516(%rdx){1to4}, %k3
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x18,0x66,0x9a,0xfc,0xfd,0xff,0xff,0x7b]
-          vfpclassps $0x7b,-516(%rdx){1to4}, %k3
-
-// CHECK: vfpclassps $123, -516(%rdx){1to4}, %k3
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x18,0x66,0x9a,0xfc,0xfd,0xff,0xff,0x7b]
-          vfpclasspsx $0x7b,-516(%rdx){1to4}, %k3
-
-// CHECK: vfpclassps $171, %ymm19, %k3
-// CHECK:  encoding: [0x62,0xb3,0x7d,0x28,0x66,0xdb,0xab]
-          vfpclassps $0xab, %ymm19, %k3
-
-// CHECK: vfpclassps $171, %ymm19, %k3
-// CHECK:  encoding: [0x62,0xb3,0x7d,0x28,0x66,0xdb,0xab]
-          vfpclasspsy $0xab, %ymm19, %k3
-
-// CHECK: vfpclassps $171, %ymm19, %k3 {%k3}
-// CHECK:  encoding: [0x62,0xb3,0x7d,0x2b,0x66,0xdb,0xab]
-          vfpclassps $0xab, %ymm19, %k3 {%k3}
-
-// CHECK: vfpclassps $171, %ymm19, %k3 {%k3}
-// CHECK:  encoding: [0x62,0xb3,0x7d,0x2b,0x66,0xdb,0xab]
-          vfpclasspsy $0xab, %ymm19, %k3 {%k3}
-
-// CHECK: vfpclassps $123,  %ymm19, %k3
-// CHECK:  encoding: [0x62,0xb3,0x7d,0x28,0x66,0xdb,0x7b]
-          vfpclassps $0x7b, %ymm19, %k3
-
-// CHECK: vfpclassps $123,  %ymm19, %k3
-// CHECK:  encoding: [0x62,0xb3,0x7d,0x28,0x66,0xdb,0x7b]
-          vfpclasspsy $0x7b, %ymm19, %k3
-
-// CHECK: vfpclasspsy $123, (%rcx), %k3
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x28,0x66,0x19,0x7b]
-          vfpclasspsy $0x7b,(%rcx), %k3
-
-// CHECK: vfpclasspsy $123, 4660(%rax,%r14,8), %k3
-// CHECK:  encoding: [0x62,0xb3,0x7d,0x28,0x66,0x9c,0xf0,0x34,0x12,0x00,0x00,0x7b]
-          vfpclasspsy $0x7b,4660(%rax,%r14,8), %k3
-
-// CHECK: vfpclassps $123, (%rcx){1to8}, %k3
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x38,0x66,0x19,0x7b]
-          vfpclassps $0x7b,(%rcx){1to8}, %k3
-
-// CHECK: vfpclassps $123, (%rcx){1to8}, %k3
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x38,0x66,0x19,0x7b]
-          vfpclasspsy $0x7b,(%rcx){1to8}, %k3
-
-// CHECK: vfpclasspsy $123, 4064(%rdx), %k3
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x28,0x66,0x5a,0x7f,0x7b]
-          vfpclasspsy $0x7b,4064(%rdx), %k3
-
-// CHECK: vfpclasspsy $123, 4096(%rdx), %k3
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x28,0x66,0x9a,0x00,0x10,0x00,0x00,0x7b]
-          vfpclasspsy $0x7b,4096(%rdx), %k3
-
-// CHECK: vfpclasspsy $123, -4096(%rdx), %k3
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x28,0x66,0x5a,0x80,0x7b]
-          vfpclasspsy $0x7b,-4096(%rdx), %k3
-
-// CHECK: vfpclasspsy $123, -4128(%rdx), %k3
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x28,0x66,0x9a,0xe0,0xef,0xff,0xff,0x7b]
-          vfpclasspsy $0x7b,-4128(%rdx), %k3
-
-// CHECK: vfpclassps $123, 508(%rdx){1to8}, %k3
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x38,0x66,0x5a,0x7f,0x7b]
-          vfpclassps $0x7b,508(%rdx){1to8}, %k3
-
-// CHECK: vfpclassps $123, 508(%rdx){1to8}, %k3
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x38,0x66,0x5a,0x7f,0x7b]
-          vfpclasspsy $0x7b,508(%rdx){1to8}, %k3
-
-// CHECK: vfpclassps $123, 512(%rdx){1to8}, %k3
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x38,0x66,0x9a,0x00,0x02,0x00,0x00,0x7b]
-          vfpclassps $0x7b,512(%rdx){1to8}, %k3
-
-// CHECK: vfpclassps $123, 512(%rdx){1to8}, %k3
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x38,0x66,0x9a,0x00,0x02,0x00,0x00,0x7b]
-          vfpclasspsy $0x7b,512(%rdx){1to8}, %k3
-
-// CHECK: vfpclassps $123, -512(%rdx){1to8}, %k3
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x38,0x66,0x5a,0x80,0x7b]
-          vfpclassps $0x7b,-512(%rdx){1to8}, %k3
-
-// CHECK: vfpclassps $123, -512(%rdx){1to8}, %k3
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x38,0x66,0x5a,0x80,0x7b]
-          vfpclasspsy $0x7b,-512(%rdx){1to8}, %k3
-
-// CHECK: vfpclassps $123, -516(%rdx){1to8}, %k3
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x38,0x66,0x9a,0xfc,0xfd,0xff,0xff,0x7b]
-          vfpclassps $0x7b,-516(%rdx){1to8}, %k3
-
-// CHECK: vfpclassps $123, -516(%rdx){1to8}, %k3
-// CHECK:  encoding: [0x62,0xf3,0x7d,0x38,0x66,0x9a,0xfc,0xfd,0xff,0xff,0x7b]
-          vfpclasspsy $0x7b,-516(%rdx){1to8}, %k3
-
-// CHECK: vbroadcasti32x2 %xmm30, %xmm30
-// CHECK:  encoding: [0x62,0x02,0x7d,0x08,0x59,0xf6]
-          vbroadcasti32x2 %xmm30, %xmm30
-
-// CHECK: vbroadcasti32x2 %xmm30, %xmm30 {%k2}
-// CHECK:  encoding: [0x62,0x02,0x7d,0x0a,0x59,0xf6]
-          vbroadcasti32x2 %xmm30, %xmm30 {%k2}
-
-// CHECK: vbroadcasti32x2 %xmm30, %xmm30 {%k2} {z}
-// CHECK:  encoding: [0x62,0x02,0x7d,0x8a,0x59,0xf6]
-          vbroadcasti32x2 %xmm30, %xmm30 {%k2} {z}
-
-// CHECK: vbroadcasti32x2 (%rcx), %xmm30
-// CHECK:  encoding: [0x62,0x62,0x7d,0x08,0x59,0x31]
-          vbroadcasti32x2 (%rcx), %xmm30
-
-// CHECK: vbroadcasti32x2 291(%rax,%r14,8), %xmm30
-// CHECK:  encoding: [0x62,0x22,0x7d,0x08,0x59,0xb4,0xf0,0x23,0x01,0x00,0x00]
-          vbroadcasti32x2 291(%rax,%r14,8), %xmm30
-
-// CHECK: vbroadcasti32x2 1016(%rdx), %xmm30
-// CHECK:  encoding: [0x62,0x62,0x7d,0x08,0x59,0x72,0x7f]
-          vbroadcasti32x2 1016(%rdx), %xmm30
-
-// CHECK: vbroadcasti32x2 1024(%rdx), %xmm30
-// CHECK:  encoding: [0x62,0x62,0x7d,0x08,0x59,0xb2,0x00,0x04,0x00,0x00]
-          vbroadcasti32x2 1024(%rdx), %xmm30
-
-// CHECK: vbroadcasti32x2 -1024(%rdx), %xmm30
-// CHECK:  encoding: [0x62,0x62,0x7d,0x08,0x59,0x72,0x80]
-          vbroadcasti32x2 -1024(%rdx), %xmm30
-
-// CHECK: vbroadcasti32x2 -1032(%rdx), %xmm30
-// CHECK:  encoding: [0x62,0x62,0x7d,0x08,0x59,0xb2,0xf8,0xfb,0xff,0xff]
-          vbroadcasti32x2 -1032(%rdx), %xmm30
-
-// CHECK: vbroadcasti32x2 %xmm28, %ymm26
-// CHECK:  encoding: [0x62,0x02,0x7d,0x28,0x59,0xd4]
-          vbroadcasti32x2 %xmm28, %ymm26
-
-// CHECK: vbroadcasti32x2 %xmm28, %ymm26 {%k7}
-// CHECK:  encoding: [0x62,0x02,0x7d,0x2f,0x59,0xd4]
-          vbroadcasti32x2 %xmm28, %ymm26 {%k7}
-
-// CHECK: vbroadcasti32x2 %xmm28, %ymm26 {%k7} {z}
-// CHECK:  encoding: [0x62,0x02,0x7d,0xaf,0x59,0xd4]
-          vbroadcasti32x2 %xmm28, %ymm26 {%k7} {z}
-
-// CHECK: vbroadcasti32x2 (%rcx), %ymm26
-// CHECK:  encoding: [0x62,0x62,0x7d,0x28,0x59,0x11]
-          vbroadcasti32x2 (%rcx), %ymm26
-
-// CHECK: vbroadcasti32x2 291(%rax,%r14,8), %ymm26
-// CHECK:  encoding: [0x62,0x22,0x7d,0x28,0x59,0x94,0xf0,0x23,0x01,0x00,0x00]
-          vbroadcasti32x2 291(%rax,%r14,8), %ymm26
-
-// CHECK: vbroadcasti32x2 1016(%rdx), %ymm26
-// CHECK:  encoding: [0x62,0x62,0x7d,0x28,0x59,0x52,0x7f]
-          vbroadcasti32x2 1016(%rdx), %ymm26
-
-// CHECK: vbroadcasti32x2 1024(%rdx), %ymm26
-// CHECK:  encoding: [0x62,0x62,0x7d,0x28,0x59,0x92,0x00,0x04,0x00,0x00]
-          vbroadcasti32x2 1024(%rdx), %ymm26
-
-// CHECK: vbroadcasti32x2 -1024(%rdx), %ymm26
-// CHECK:  encoding: [0x62,0x62,0x7d,0x28,0x59,0x52,0x80]
-          vbroadcasti32x2 -1024(%rdx), %ymm26
-
-// CHECK: vbroadcasti32x2 -1032(%rdx), %ymm26
-// CHECK:  encoding: [0x62,0x62,0x7d,0x28,0x59,0x92,0xf8,0xfb,0xff,0xff]
-          vbroadcasti32x2 -1032(%rdx), %ymm26
-
-// CHECK: vbroadcasti32x2 %xmm28, %xmm30
-// CHECK:  encoding: [0x62,0x02,0x7d,0x08,0x59,0xf4]
-          vbroadcasti32x2 %xmm28, %xmm30
-
-// CHECK: vbroadcasti32x2 %xmm28, %xmm30 {%k6}
-// CHECK:  encoding: [0x62,0x02,0x7d,0x0e,0x59,0xf4]
-          vbroadcasti32x2 %xmm28, %xmm30 {%k6}
-
-// CHECK: vbroadcasti32x2 %xmm28, %xmm30 {%k6} {z}
-// CHECK:  encoding: [0x62,0x02,0x7d,0x8e,0x59,0xf4]
-          vbroadcasti32x2 %xmm28, %xmm30 {%k6} {z}
-
-// CHECK: vbroadcasti32x2 (%rcx), %xmm30
-// CHECK:  encoding: [0x62,0x62,0x7d,0x08,0x59,0x31]
-          vbroadcasti32x2 (%rcx), %xmm30
-
-// CHECK: vbroadcasti32x2 4660(%rax,%r14,8), %xmm30
-// CHECK:  encoding: [0x62,0x22,0x7d,0x08,0x59,0xb4,0xf0,0x34,0x12,0x00,0x00]
-          vbroadcasti32x2 4660(%rax,%r14,8), %xmm30
-
-// CHECK: vbroadcasti32x2 1016(%rdx), %xmm30
-// CHECK:  encoding: [0x62,0x62,0x7d,0x08,0x59,0x72,0x7f]
-          vbroadcasti32x2 1016(%rdx), %xmm30
-
-// CHECK: vbroadcasti32x2 1024(%rdx), %xmm30
-// CHECK:  encoding: [0x62,0x62,0x7d,0x08,0x59,0xb2,0x00,0x04,0x00,0x00]
-          vbroadcasti32x2 1024(%rdx), %xmm30
-
-// CHECK: vbroadcasti32x2 -1024(%rdx), %xmm30
-// CHECK:  encoding: [0x62,0x62,0x7d,0x08,0x59,0x72,0x80]
-          vbroadcasti32x2 -1024(%rdx), %xmm30
-
-// CHECK: vbroadcasti32x2 -1032(%rdx), %xmm30
-// CHECK:  encoding: [0x62,0x62,0x7d,0x08,0x59,0xb2,0xf8,0xfb,0xff,0xff]
-          vbroadcasti32x2 -1032(%rdx), %xmm30
-
-// CHECK: vbroadcasti32x2 %xmm28, %ymm23
-// CHECK:  encoding: [0x62,0x82,0x7d,0x28,0x59,0xfc]
-          vbroadcasti32x2 %xmm28, %ymm23
-
-// CHECK: vbroadcasti32x2 %xmm28, %ymm23 {%k1}
-// CHECK:  encoding: [0x62,0x82,0x7d,0x29,0x59,0xfc]
-          vbroadcasti32x2 %xmm28, %ymm23 {%k1}
-
-// CHECK: vbroadcasti32x2 %xmm28, %ymm23 {%k1} {z}
-// CHECK:  encoding: [0x62,0x82,0x7d,0xa9,0x59,0xfc]
-          vbroadcasti32x2 %xmm28, %ymm23 {%k1} {z}
-
-// CHECK: vbroadcasti32x2 (%rcx), %ymm23
-// CHECK:  encoding: [0x62,0xe2,0x7d,0x28,0x59,0x39]
-          vbroadcasti32x2 (%rcx), %ymm23
-
-// CHECK: vbroadcasti32x2 4660(%rax,%r14,8), %ymm23
-// CHECK:  encoding: [0x62,0xa2,0x7d,0x28,0x59,0xbc,0xf0,0x34,0x12,0x00,0x00]
-          vbroadcasti32x2 4660(%rax,%r14,8), %ymm23
-
-// CHECK: vbroadcasti32x2 1016(%rdx), %ymm23
-// CHECK:  encoding: [0x62,0xe2,0x7d,0x28,0x59,0x7a,0x7f]
-          vbroadcasti32x2 1016(%rdx), %ymm23
-
-// CHECK: vbroadcasti32x2 1024(%rdx), %ymm23
-// CHECK:  encoding: [0x62,0xe2,0x7d,0x28,0x59,0xba,0x00,0x04,0x00,0x00]
-          vbroadcasti32x2 1024(%rdx), %ymm23
-
-// CHECK: vbroadcasti32x2 -1024(%rdx), %ymm23
-// CHECK:  encoding: [0x62,0xe2,0x7d,0x28,0x59,0x7a,0x80]
-          vbroadcasti32x2 -1024(%rdx), %ymm23
-
-// CHECK: vbroadcasti32x2 -1032(%rdx), %ymm23
-// CHECK:  encoding: [0x62,0xe2,0x7d,0x28,0x59,0xba,0xf8,0xfb,0xff,0xff]
-          vbroadcasti32x2 -1032(%rdx), %ymm23
-
-// CHECK: vbroadcastf32x2 %xmm18, %ymm28
-// CHECK:  encoding: [0x62,0x22,0x7d,0x28,0x19,0xe2]
-          vbroadcastf32x2 %xmm18, %ymm28
-
-// CHECK: vbroadcastf32x2 %xmm18, %ymm28 {%k7}
-// CHECK:  encoding: [0x62,0x22,0x7d,0x2f,0x19,0xe2]
-          vbroadcastf32x2 %xmm18, %ymm28 {%k7}
-
-// CHECK: vbroadcastf32x2 %xmm18, %ymm28 {%k7} {z}
-// CHECK:  encoding: [0x62,0x22,0x7d,0xaf,0x19,0xe2]
-          vbroadcastf32x2 %xmm18, %ymm28 {%k7} {z}
-
-// CHECK: vbroadcastf32x2 (%rcx), %ymm28
-// CHECK:  encoding: [0x62,0x62,0x7d,0x28,0x19,0x21]
-          vbroadcastf32x2 (%rcx), %ymm28
-
-// CHECK: vbroadcastf32x2 291(%rax,%r14,8), %ymm28
-// CHECK:  encoding: [0x62,0x22,0x7d,0x28,0x19,0xa4,0xf0,0x23,0x01,0x00,0x00]
-          vbroadcastf32x2 291(%rax,%r14,8), %ymm28
-
-// CHECK: vbroadcastf32x2 1016(%rdx), %ymm28
-// CHECK:  encoding: [0x62,0x62,0x7d,0x28,0x19,0x62,0x7f]
-          vbroadcastf32x2 1016(%rdx), %ymm28
-
-// CHECK: vbroadcastf32x2 1024(%rdx), %ymm28
-// CHECK:  encoding: [0x62,0x62,0x7d,0x28,0x19,0xa2,0x00,0x04,0x00,0x00]
-          vbroadcastf32x2 1024(%rdx), %ymm28
-
-// CHECK: vbroadcastf32x2 -1024(%rdx), %ymm28
-// CHECK:  encoding: [0x62,0x62,0x7d,0x28,0x19,0x62,0x80]
-          vbroadcastf32x2 -1024(%rdx), %ymm28
-
-// CHECK: vbroadcastf32x2 -1032(%rdx), %ymm28
-// CHECK:  encoding: [0x62,0x62,0x7d,0x28,0x19,0xa2,0xf8,0xfb,0xff,0xff]
-          vbroadcastf32x2 -1032(%rdx), %ymm28
-
-// CHECK: vbroadcastf32x2 %xmm26, %ymm19
-// CHECK:  encoding: [0x62,0x82,0x7d,0x28,0x19,0xda]
-          vbroadcastf32x2 %xmm26, %ymm19
-
-// CHECK: vbroadcastf32x2 %xmm26, %ymm19 {%k7}
-// CHECK:  encoding: [0x62,0x82,0x7d,0x2f,0x19,0xda]
-          vbroadcastf32x2 %xmm26, %ymm19 {%k7}
-
-// CHECK: vbroadcastf32x2 %xmm26, %ymm19 {%k7} {z}
-// CHECK:  encoding: [0x62,0x82,0x7d,0xaf,0x19,0xda]
-          vbroadcastf32x2 %xmm26, %ymm19 {%k7} {z}
-
-// CHECK: vbroadcastf32x2 (%rcx), %ymm19
-// CHECK:  encoding: [0x62,0xe2,0x7d,0x28,0x19,0x19]
-          vbroadcastf32x2 (%rcx), %ymm19
-
-// CHECK: vbroadcastf32x2 4660(%rax,%r14,8), %ymm19
-// CHECK:  encoding: [0x62,0xa2,0x7d,0x28,0x19,0x9c,0xf0,0x34,0x12,0x00,0x00]
-          vbroadcastf32x2 4660(%rax,%r14,8), %ymm19
-
-// CHECK: vbroadcastf32x2 1016(%rdx), %ymm19
-// CHECK:  encoding: [0x62,0xe2,0x7d,0x28,0x19,0x5a,0x7f]
-          vbroadcastf32x2 1016(%rdx), %ymm19
-
-// CHECK: vbroadcastf32x2 1024(%rdx), %ymm19
-// CHECK:  encoding: [0x62,0xe2,0x7d,0x28,0x19,0x9a,0x00,0x04,0x00,0x00]
-          vbroadcastf32x2 1024(%rdx), %ymm19
-
-// CHECK: vbroadcastf32x2 -1024(%rdx), %ymm19
-// CHECK:  encoding: [0x62,0xe2,0x7d,0x28,0x19,0x5a,0x80]
-          vbroadcastf32x2 -1024(%rdx), %ymm19
-
-// CHECK: vbroadcastf32x2 -1032(%rdx), %ymm19
-// CHECK:  encoding: [0x62,0xe2,0x7d,0x28,0x19,0x9a,0xf8,0xfb,0xff,0xff]
-          vbroadcastf32x2 -1032(%rdx), %ymm19
-
diff --git a/llvm/test/MC/X86/x86-64-avx512vp2intersect-att.s b/llvm/test/MC/X86/x86-64-avx512vp2intersect-att.s
deleted file mode 100644
index 72e907f9b3848e96456bf95ac5fae3d15fa0e63e..0000000000000000000000000000000000000000
--- a/llvm/test/MC/X86/x86-64-avx512vp2intersect-att.s
+++ /dev/null
@@ -1,231 +0,0 @@
-// RUN: llvm-mc -triple x86_64-unknown-unknown --show-encoding %s | FileCheck %s
-
-// v8i64 vectors
-// CHECK: vp2intersectq        %zmm2, %zmm1, %k0
-// CHECK: encoding: [0x62,0xf2,0xf7,0x48,0x68,0xc2]
-vp2intersectq  %zmm2, %zmm1, %k0
-
-// CHECK: vp2intersectq        (%rdi), %zmm1, %k0
-// CHECK: encoding: [0x62,0xf2,0xf7,0x48,0x68,0x07]
-vp2intersectq  (%rdi), %zmm1, %k0
-
-// CHECK: vp2intersectq        (%rdi){1to8}, %zmm1, %k0
-// CHECK: encoding: [0x62,0xf2,0xf7,0x58,0x68,0x07]
-vp2intersectq  (%rdi){1to8}, %zmm1, %k0
-
-// CHECK: vp2intersectq        %zmm2, %zmm1, %k0
-// CHECK: encoding: [0x62,0xf2,0xf7,0x48,0x68,0xc2]
-vp2intersectq  %zmm2, %zmm1, %k1
-
-// CHECK: vp2intersectq        (%rdi), %zmm1, %k0
-// CHECK: encoding: [0x62,0xf2,0xf7,0x48,0x68,0x07]
-vp2intersectq  (%rdi), %zmm1, %k1
-
-// CHECK: vp2intersectq        (%rdi){1to8}, %zmm1, %k0
-// CHECK: encoding: [0x62,0xf2,0xf7,0x58,0x68,0x07]
-vp2intersectq  (%rdi){1to8}, %zmm1, %k1
-
-// CHECK: vp2intersectq        %zmm7, %zmm9, %k6
-// CHECK: encoding: [0x62,0xf2,0xb7,0x48,0x68,0xf7]
-vp2intersectq  %zmm7, %zmm9, %k6
-
-// CHECK: vp2intersectq        (%rsi), %zmm9, %k6
-// CHECK: encoding: [0x62,0xf2,0xb7,0x48,0x68,0x36]
-vp2intersectq  (%rsi), %zmm9, %k6
-
-// CHECK: vp2intersectq        (%rsi){1to8}, %zmm9, %k6
-// CHECK: encoding: [0x62,0xf2,0xb7,0x58,0x68,0x36]
-vp2intersectq  (%rsi){1to8}, %zmm9, %k6
-
-// CHECK: vp2intersectq        %zmm7, %zmm9, %k6
-// CHECK: encoding: [0x62,0xf2,0xb7,0x48,0x68,0xf7]
-vp2intersectq  %zmm7, %zmm9, %k7
-
-// CHECK: vp2intersectq        (%rsi), %zmm9, %k6
-// CHECK: encoding: [0x62,0xf2,0xb7,0x48,0x68,0x36]
-vp2intersectq  (%rsi), %zmm9, %k7
-
-// CHECK: vp2intersectq        (%rsi){1to8}, %zmm9, %k6
-// CHECK: encoding: [0x62,0xf2,0xb7,0x58,0x68,0x36]
-vp2intersectq  (%rsi){1to8}, %zmm9, %k7
-
-// v4i64 vectors
-// CHECK: vp2intersectq        %ymm2, %ymm1, %k0
-// CHECK: encoding: [0x62,0xf2,0xf7,0x28,0x68,0xc2]
-vp2intersectq  %ymm2, %ymm1, %k0
-
-// CHECK: vp2intersectq        (%rdi), %ymm1, %k0
-// CHECK: encoding: [0x62,0xf2,0xf7,0x28,0x68,0x07]
-vp2intersectq  (%rdi), %ymm1, %k0
-
-// CHECK: vp2intersectq        (%rdi){1to4}, %ymm1, %k0
-// CHECK: encoding: [0x62,0xf2,0xf7,0x38,0x68,0x07]
-vp2intersectq  (%rdi){1to4}, %ymm1, %k0
-
-// CHECK: vp2intersectq        %ymm2, %ymm1, %k0
-// CHECK: encoding: [0x62,0xf2,0xf7,0x28,0x68,0xc2]
-vp2intersectq  %ymm2, %ymm1, %k1
-
-// CHECK: vp2intersectq        (%rdi), %ymm1, %k0
-// CHECK: encoding: [0x62,0xf2,0xf7,0x28,0x68,0x07]
-vp2intersectq  (%rdi), %ymm1, %k1
-
-// CHECK: vp2intersectq        (%rdi){1to4}, %ymm1, %k0
-// CHECK: encoding: [0x62,0xf2,0xf7,0x38,0x68,0x07]
-vp2intersectq  (%rdi){1to4}, %ymm1, %k1
-
-// CHECK: vp2intersectq        %ymm7, %ymm9, %k6
-// CHECK: encoding: [0x62,0xf2,0xb7,0x28,0x68,0xf7]
-vp2intersectq  %ymm7, %ymm9, %k6
-
-// CHECK: vp2intersectq        (%rsi), %ymm9, %k6
-// CHECK: encoding: [0x62,0xf2,0xb7,0x28,0x68,0x36]
-vp2intersectq  (%rsi), %ymm9, %k6
-
-// CHECK: vp2intersectq        (%rsi){1to4}, %ymm9, %k6
-// CHECK: encoding: [0x62,0xf2,0xb7,0x38,0x68,0x36]
-vp2intersectq  (%rsi){1to4}, %ymm9, %k6
-
-// CHECK: vp2intersectq        %ymm7, %ymm9, %k6
-// CHECK: encoding: [0x62,0xf2,0xb7,0x28,0x68,0xf7]
-vp2intersectq  %ymm7, %ymm9, %k7
-
-// CHECK: vp2intersectq        (%rsi), %ymm9, %k6
-// CHECK: encoding: [0x62,0xf2,0xb7,0x28,0x68,0x36]
-vp2intersectq  (%rsi), %ymm9, %k7
-
-// v2i64 vectors
-// CHECK: vp2intersectq        %xmm2, %xmm1, %k0
-// CHECK: encoding: [0x62,0xf2,0xf7,0x08,0x68,0xc2]
-vp2intersectq  %xmm2, %xmm1, %k0
-
-// CHECK: vp2intersectq        (%rdi), %xmm1, %k0
-// CHECK: encoding: [0x62,0xf2,0xf7,0x08,0x68,0x07]
-vp2intersectq  (%rdi), %xmm1, %k0
-
-// CHECK: vp2intersectq        (%rdi){1to2}, %xmm1, %k0
-// CHECK: encoding: [0x62,0xf2,0xf7,0x18,0x68,0x07]
-vp2intersectq  (%rdi){1to2}, %xmm1, %k0
-
-// CHECK: vp2intersectq        %xmm2, %xmm1, %k0
-// CHECK: encoding: [0x62,0xf2,0xf7,0x08,0x68,0xc2]
-vp2intersectq  %xmm2, %xmm1, %k1
-
-// CHECK: vp2intersectq        (%rdi), %xmm1, %k0
-// CHECK: encoding: [0x62,0xf2,0xf7,0x08,0x68,0x07]
-vp2intersectq  (%rdi), %xmm1, %k1
-
-// CHECK: vp2intersectq        %xmm7, %xmm9, %k6
-// CHECK: encoding: [0x62,0xf2,0xb7,0x08,0x68,0xf7]
-vp2intersectq  %xmm7, %xmm9, %k6
-
-// CHECK: vp2intersectq        (%rsi), %xmm9, %k6
-// CHECK: encoding: [0x62,0xf2,0xb7,0x08,0x68,0x36]
-vp2intersectq  (%rsi), %xmm9, %k6
-
-// CHECK: vp2intersectq        %xmm7, %xmm9, %k6
-// CHECK: encoding: [0x62,0xf2,0xb7,0x08,0x68,0xf7]
-vp2intersectq  %xmm7, %xmm9, %k7
-
-// CHECK: vp2intersectq        (%rsi), %xmm9, %k6
-// CHECK: encoding: [0x62,0xf2,0xb7,0x08,0x68,0x36]
-vp2intersectq  (%rsi), %xmm9, %k7
-
-// v16i32 vectors
-// CHECK: vp2intersectd        %zmm2, %zmm1, %k0
-// CHECK: encoding: [0x62,0xf2,0x77,0x48,0x68,0xc2]
-vp2intersectd  %zmm2, %zmm1, %k0
-
-// CHECK: vp2intersectd        (%rdi), %zmm1, %k0
-// CHECK: encoding: [0x62,0xf2,0x77,0x48,0x68,0x07]
-vp2intersectd  (%rdi), %zmm1, %k0
-
-// CHECK: vp2intersectd        %zmm2, %zmm1, %k0
-// CHECK: encoding: [0x62,0xf2,0x77,0x48,0x68,0xc2]
-vp2intersectd  %zmm2, %zmm1, %k1
-
-// CHECK: vp2intersectd        (%rdi), %zmm1, %k0
-// CHECK: encoding: [0x62,0xf2,0x77,0x48,0x68,0x07]
-vp2intersectd  (%rdi), %zmm1, %k1
-
-// CHECK: vp2intersectd        %zmm7, %zmm9, %k6
-// CHECK: encoding: [0x62,0xf2,0x37,0x48,0x68,0xf7]
-vp2intersectd  %zmm7, %zmm9, %k6
-
-// CHECK: vp2intersectd        (%rsi), %zmm9, %k6
-// CHECK: encoding: [0x62,0xf2,0x37,0x48,0x68,0x36]
-vp2intersectd  (%rsi), %zmm9, %k6
-
-// CHECK: vp2intersectd        %zmm7, %zmm9, %k6
-// CHECK: encoding: [0x62,0xf2,0x37,0x48,0x68,0xf7]
-vp2intersectd  %zmm7, %zmm9, %k7
-
-// CHECK: vp2intersectd        (%rsi), %zmm9, %k6
-// CHECK: encoding: [0x62,0xf2,0x37,0x48,0x68,0x36]
-vp2intersectd  (%rsi), %zmm9, %k7
-
-// v8i32 vectors
-// CHECK: vp2intersectd        %ymm2, %ymm1, %k0
-// CHECK: encoding: [0x62,0xf2,0x77,0x28,0x68,0xc2]
-vp2intersectd  %ymm2, %ymm1, %k0
-
-// CHECK: vp2intersectd        (%rdi), %ymm1, %k0
-// CHECK: encoding: [0x62,0xf2,0x77,0x28,0x68,0x07]
-vp2intersectd  (%rdi), %ymm1, %k0
-
-// CHECK: vp2intersectd        %ymm2, %ymm1, %k0
-// CHECK: encoding: [0x62,0xf2,0x77,0x28,0x68,0xc2]
-vp2intersectd  %ymm2, %ymm1, %k1
-
-// CHECK: vp2intersectd        (%rdi), %ymm1, %k0
-// CHECK: encoding: [0x62,0xf2,0x77,0x28,0x68,0x07]
-vp2intersectd  (%rdi), %ymm1, %k1
-
-// CHECK: vp2intersectd        %ymm7, %ymm9, %k6
-// CHECK: encoding: [0x62,0xf2,0x37,0x28,0x68,0xf7]
-vp2intersectd  %ymm7, %ymm9, %k6
-
-// CHECK: vp2intersectd        (%rsi), %ymm9, %k6
-// CHECK: encoding: [0x62,0xf2,0x37,0x28,0x68,0x36]
-vp2intersectd  (%rsi), %ymm9, %k6
-
-// CHECK: vp2intersectd        %ymm7, %ymm9, %k6
-// CHECK: encoding: [0x62,0xf2,0x37,0x28,0x68,0xf7]
-vp2intersectd  %ymm7, %ymm9, %k7
-
-// CHECK: vp2intersectd        (%rsi), %ymm9, %k6
-// CHECK: encoding: [0x62,0xf2,0x37,0x28,0x68,0x36]
-vp2intersectd  (%rsi), %ymm9, %k7
-
-// v4i32 vectors
-// CHECK: vp2intersectd        %xmm2, %xmm1, %k0
-// CHECK: encoding: [0x62,0xf2,0x77,0x08,0x68,0xc2]
-vp2intersectd  %xmm2, %xmm1, %k0
-
-// CHECK: vp2intersectd        (%rdi), %xmm1, %k0
-// CHECK: encoding: [0x62,0xf2,0x77,0x08,0x68,0x07]
-vp2intersectd  (%rdi), %xmm1, %k0
-
-// CHECK: vp2intersectd        %xmm2, %xmm1, %k0
-// CHECK: encoding: [0x62,0xf2,0x77,0x08,0x68,0xc2]
-vp2intersectd  %xmm2, %xmm1, %k1
-
-// CHECK: vp2intersectd        (%rdi), %xmm1, %k0
-// CHECK: encoding: [0x62,0xf2,0x77,0x08,0x68,0x07]
-vp2intersectd  (%rdi), %xmm1, %k1
-
-// CHECK: vp2intersectd        %xmm7, %xmm9, %k6
-// CHECK: encoding: [0x62,0xf2,0x37,0x08,0x68,0xf7]
-vp2intersectd  %xmm7, %xmm9, %k6
-
-// CHECK: vp2intersectd        (%rsi), %xmm9, %k6
-// CHECK: encoding: [0x62,0xf2,0x37,0x08,0x68,0x36]
-vp2intersectd  (%rsi), %xmm9, %k6
-
-// CHECK: vp2intersectd        %xmm7, %xmm9, %k6
-// CHECK: encoding: [0x62,0xf2,0x37,0x08,0x68,0xf7]
-vp2intersectd  %xmm7, %xmm9, %k7
-
-// CHECK: vp2intersectd        (%rsi), %xmm9, %k6
-// CHECK: encoding: [0x62,0xf2,0x37,0x08,0x68,0x36]
-vp2intersectd  (%rsi), %xmm9, %k7
diff --git a/llvm/test/MC/X86/x86_64-xop-encoding.s b/llvm/test/MC/X86/xop-att.s
similarity index 100%
rename from llvm/test/MC/X86/x86_64-xop-encoding.s
rename to llvm/test/MC/X86/xop-att.s
diff --git a/llvm/test/TableGen/ContextlessPredicates.td b/llvm/test/TableGen/ContextlessPredicates.td
index 0f4c4d0c450e1b5864ace9ae49f50166dee69e8b..5e4e69069c3e3298226908411baa85de5646e559 100644
--- a/llvm/test/TableGen/ContextlessPredicates.td
+++ b/llvm/test/TableGen/ContextlessPredicates.td
@@ -20,55 +20,53 @@ def INSN : I<(outs GPR32:$dst), (ins GPR32Op:$src1, GPR32Op:$src2), []>;
 def : Pat<(test_atomic_op_frag GPR32:$ptr, GPR32:$val) ,
 	      (INSN GPR32:$ptr, GPR32:$val)>;
 
-// CHECK_NOPT-LABEL: const int64_t *MyTargetInstructionSelector::getMatchTable() const {
-// CHECK_NOPT-NEXT:  constexpr static int64_t MatchTable0[] = {
-// CHECK_NOPT-NEXT:    GIM_Try, /*On fail goto*//*Label 0*/ 46, // Rule ID 0 //
-// CHECK_NOPT-NEXT:      GIM_CheckNumOperands, /*MI*/0, /*Expected*/3,
-// CHECK_NOPT-NEXT:      GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_ATOMICRMW_XCHG,
-// CHECK_NOPT-NEXT:      GIM_CheckMemorySizeEqualTo, /*MI*/0, /*MMO*/0, /*Size*/4,
-// CHECK_NOPT-NEXT:      // MIs[0] DstI[dst]
-// CHECK_NOPT-NEXT:      GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
-// CHECK_NOPT-NEXT:      GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID,
-// CHECK_NOPT-NEXT:      // MIs[0] ptr
-// CHECK_NOPT-NEXT:      GIM_CheckPointerToAny, /*MI*/0, /*Op*/1, /*SizeInBits*/32,
-// CHECK_NOPT-NEXT:      GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/MyTarget::GPR32RegClassID,
-// CHECK_NOPT-NEXT:      // MIs[0] val
-// CHECK_NOPT-NEXT:      GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32,
-// CHECK_NOPT-NEXT:      GIM_CheckRegBankForClass, /*MI*/0, /*Op*/2, /*RC*/MyTarget::GPR32RegClassID,
-// CHECK_NOPT-NEXT:      GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GICXXPred_MI_Predicate_test_atomic_op_frag,
-// CHECK_NOPT-NEXT:      // (atomic_swap:{ *:[i32] } GPR32:{ *:[i32] }:$ptr, GPR32:{ *:[i32] }:$val)<<P:Predicate_test_atomic_op_frag>>  =>  (INSN:{ *:[i32] } GPR32:{ *:[i32] }:$ptr, GPR32:{ *:[i32] }:$val)
-// CHECK_NOPT-NEXT:      GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/MyTarget::INSN,
-// CHECK_NOPT-NEXT:      GIR_ConstrainSelectedInstOperands, /*InsnID*/0,
-// CHECK_NOPT-NEXT:      // GIR_Coverage, 0,
-// CHECK_NOPT-NEXT:      GIR_Done,
-// CHECK_NOPT-NEXT:    // Label 0: @46
-// CHECK_NOPT-NEXT:    GIM_Reject,
-// CHECK_NOPT-NEXT:    };
-// CHECK_NOPT-NEXT:  return MatchTable0;
-// CHECK_NOPT-NEXT: }
-//
-//
+// CHECK_NOPT-LABEL: const uint8_t *MyTargetInstructionSelector::getMatchTable() const {
+// CHECK_NOPT-NEXT:    constexpr static uint8_t MatchTable0[] = {
+// CHECK_NOPT-NEXT:      GIM_Try, /*On fail goto*//*Label 0*/ GIMT_Encode4(58), // Rule ID 0 //
+// CHECK_NOPT-NEXT:        GIM_CheckNumOperands, /*MI*/0, /*Expected*/3,
+// CHECK_NOPT-NEXT:        GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_ATOMICRMW_XCHG),
+// CHECK_NOPT-NEXT:        GIM_CheckMemorySizeEqualTo, /*MI*/0, /*MMO*/0, /*Size*/GIMT_Encode4(4),
+// CHECK_NOPT-NEXT:        // MIs[0] DstI[dst]
+// CHECK_NOPT-NEXT:        GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
+// CHECK_NOPT-NEXT:        GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
+// CHECK_NOPT-NEXT:        // MIs[0] ptr
+// CHECK_NOPT-NEXT:        GIM_CheckPointerToAny, /*MI*/0, /*Op*/1, /*SizeInBits*/32,
+// CHECK_NOPT-NEXT:        GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
+// CHECK_NOPT-NEXT:        // MIs[0] val
+// CHECK_NOPT-NEXT:        GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32,
+// CHECK_NOPT-NEXT:        GIM_CheckRegBankForClass, /*MI*/0, /*Op*/2, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
+// CHECK_NOPT-NEXT:        GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_test_atomic_op_frag),
+// CHECK_NOPT-NEXT:        // (atomic_swap:{ *:[i32] } GPR32:{ *:[i32] }:$ptr, GPR32:{ *:[i32] }:$val)<<P:Predicate_test_atomic_op_frag>>  =>  (INSN:{ *:[i32] } GPR32:{ *:[i32] }:$ptr, GPR32:{ *:[i32] }:$val)
+// CHECK_NOPT-NEXT:        GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::INSN),
+// CHECK_NOPT-NEXT:        GIR_ConstrainSelectedInstOperands, /*InsnID*/0,
+// CHECK_NOPT-NEXT:        // GIR_Coverage, 0,
+// CHECK_NOPT-NEXT:        GIR_Done,
+// CHECK_NOPT-NEXT:      // Label 0: @58
+// CHECK_NOPT-NEXT:      GIM_Reject,
+// CHECK_NOPT-NEXT:      };
+// CHECK_NOPT-NEXT:    return MatchTable0;
+// CHECK_NOPT-NEXT:  }
 
-// CHECK_OPT-LABEL: const int64_t *MyTargetInstructionSelector::getMatchTable() const {
-// CHECK_OPT-NEXT:  constexpr static int64_t MatchTable0[] = {
-// CHECK_OPT-NEXT:    GIM_Try, /*On fail goto*//*Label 0*/ 43, // Rule ID 0 //
-// CHECK_OPT-NEXT:      GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_ATOMICRMW_XCHG,
-// CHECK_OPT-NEXT:      GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
-// CHECK_OPT-NEXT:      GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32,
-// CHECK_OPT-NEXT:      GIM_CheckMemorySizeEqualTo, /*MI*/0, /*MMO*/0, /*Size*/4,
-// CHECK_OPT-NEXT:      GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID,
-// CHECK_OPT-NEXT:      // MIs[0] ptr
-// CHECK_OPT-NEXT:      GIM_CheckPointerToAny, /*MI*/0, /*Op*/1, /*SizeInBits*/32,
-// CHECK_OPT-NEXT:      GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/MyTarget::GPR32RegClassID,
-// CHECK_OPT-NEXT:      GIM_CheckRegBankForClass, /*MI*/0, /*Op*/2, /*RC*/MyTarget::GPR32RegClassID,
-// CHECK_OPT-NEXT:      GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GICXXPred_MI_Predicate_test_atomic_op_frag,
-// CHECK_OPT-NEXT:      // (atomic_swap:{ *:[i32] } GPR32:{ *:[i32] }:$ptr, GPR32:{ *:[i32] }:$val)<<P:Predicate_test_atomic_op_frag>>  =>  (INSN:{ *:[i32] } GPR32:{ *:[i32] }:$ptr, GPR32:{ *:[i32] }:$val)
-// CHECK_OPT-NEXT:      GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/MyTarget::INSN,
-// CHECK_OPT-NEXT:      GIR_ConstrainSelectedInstOperands, /*InsnID*/0,
-// CHECK_OPT-NEXT:      // GIR_Coverage, 0,
-// CHECK_OPT-NEXT:      GIR_Done,
-// CHECK_OPT-NEXT:    // Label 0: @43
-// CHECK_OPT-NEXT:    GIM_Reject,
-// CHECK_OPT-NEXT:    };
-// CHECK_OPT-NEXT:  return MatchTable0;
-// CHECK_OPT-NEXT: }
+// CHECK_OPT-LABEL: const uint8_t *MyTargetInstructionSelector::getMatchTable() const {
+// CHECK_OPT-NEXT:   constexpr static uint8_t MatchTable0[] = {
+// CHECK_OPT-NEXT:     GIM_Try, /*On fail goto*//*Label 0*/ GIMT_Encode4(55), // Rule ID 0 //
+// CHECK_OPT-NEXT:       GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_ATOMICRMW_XCHG),
+// CHECK_OPT-NEXT:       GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
+// CHECK_OPT-NEXT:       GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32,
+// CHECK_OPT-NEXT:       GIM_CheckMemorySizeEqualTo, /*MI*/0, /*MMO*/0, /*Size*/GIMT_Encode4(4),
+// CHECK_OPT-NEXT:       GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
+// CHECK_OPT-NEXT:       // MIs[0] ptr
+// CHECK_OPT-NEXT:       GIM_CheckPointerToAny, /*MI*/0, /*Op*/1, /*SizeInBits*/32,
+// CHECK_OPT-NEXT:       GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
+// CHECK_OPT-NEXT:       GIM_CheckRegBankForClass, /*MI*/0, /*Op*/2, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
+// CHECK_OPT-NEXT:       GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_test_atomic_op_frag),
+// CHECK_OPT-NEXT:       // (atomic_swap:{ *:[i32] } GPR32:{ *:[i32] }:$ptr, GPR32:{ *:[i32] }:$val)<<P:Predicate_test_atomic_op_frag>>  =>  (INSN:{ *:[i32] } GPR32:{ *:[i32] }:$ptr, GPR32:{ *:[i32] }:$val)
+// CHECK_OPT-NEXT:       GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::INSN),
+// CHECK_OPT-NEXT:       GIR_ConstrainSelectedInstOperands, /*InsnID*/0,
+// CHECK_OPT-NEXT:       // GIR_Coverage, 0,
+// CHECK_OPT-NEXT:       GIR_Done,
+// CHECK_OPT-NEXT:     // Label 0: @55
+// CHECK_OPT-NEXT:     GIM_Reject,
+// CHECK_OPT-NEXT:     };
+// CHECK_OPT-NEXT:   return MatchTable0;
+// CHECK_OPT-NEXT:  }
diff --git a/llvm/test/TableGen/DefaultOpsGlobalISel.td b/llvm/test/TableGen/DefaultOpsGlobalISel.td
index c997467c570ad885d4ed430a5bd8f3ee6d17b5be..9fb95e852e486068dec19fd25112aad6aead09c3 100644
--- a/llvm/test/TableGen/DefaultOpsGlobalISel.td
+++ b/llvm/test/TableGen/DefaultOpsGlobalISel.td
@@ -31,101 +31,200 @@ def omod : OperandWithDefaultOps <i32, (ops (i32 0))>;
 def clamp : OperandWithDefaultOps <i1, (ops (i1 0))>;
 
 
-// CHECK: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_FMAXNUM,
-// CHECK: GIM_CheckComplexPattern, /*MI*/0, /*Op*/1, /*Renderer*/0, GICP_gi_SelectSrcMods,
-// CHECK: GIM_CheckComplexPattern, /*MI*/0, /*Op*/2, /*Renderer*/1, GICP_gi_SelectSrcMods,
-// CHECK: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::FMAX,
-// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst]
-// CHECK-NEXT: GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/0, /*SubOperand*/1, // mods0
-// CHECK-NEXT: GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/0, /*SubOperand*/0, // src0
-// CHECK-NEXT: GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/1, /*SubOperand*/1, // mods1
-// CHECK-NEXT: GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/1, /*SubOperand*/0, // src1
-// CHECK-NEXT: GIR_AddImm, /*InsnID*/0, /*Imm*/0,
-// CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0,
-
-
-// CHECK: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_FFLOOR,
-// CHECK: GIM_CheckComplexPattern, /*MI*/0, /*Op*/1, /*Renderer*/0, GICP_gi_SelectClampOMod,
-// CHECK: // (ffloor:{ *:[f32] } (SelectClampOMod:{ *:[f32] } f32:{ *:[f32] }:$src0, omod:{ *:[i32] }:$omod, i1:{ *:[i1] }:$clamp))  =>  (FLOMP:{ *:[f32] } f32:{ *:[f32] }:$src0, i1:{ *:[i1] }:$clamp, omod:{ *:[i32] }:$omod)
-// CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::FLOMP,
-// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst]
-// CHECK-NEXT: GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/0, /*SubOperand*/0, // src0
-// CHECK-NEXT: GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/0, /*SubOperand*/2, // clamp
-// CHECK-NEXT: GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/0, /*SubOperand*/1, // omod
-
-
-// CHECK: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_FCANONICALIZE,
-// CHECK: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::FMAX,
-// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst]
-// CHECK-NEXT: GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/0, /*SubOperand*/1, // mods
-// CHECK-NEXT: GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/0, /*SubOperand*/0, // src
-// CHECK-NEXT: GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/0, /*SubOperand*/1, // mods
-// CHECK-NEXT: GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/0, /*SubOperand*/0, // src
-// CHECK-NEXT: GIR_AddImm, /*InsnID*/0, /*Imm*/0,
-// CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0,
-
-
-// CHECK: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_FCOS,
-// CHECK: // (fcos:{ *:[f32] } (SelectOMod:{ *:[f32] } f32:{ *:[f32] }:$src0, i32:{ *:[i32] }:$omod))  =>  (FLAMP:{ *:[f32] } FPR32:{ *:[f32] }:$src0, omod:{ *:[i32] }:$omod)
-// CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::FLAMP,
-// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst]
-// CHECK-NEXT: GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/0, /*SubOperand*/0, // src0
-// CHECK-NEXT: GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/0, /*SubOperand*/1, // omod
-// CHECK-NEXT: GIR_AddImm, /*InsnID*/0, /*Imm*/0,
-
-
-// CHECK: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_FEXP2,
-// CHECK: // (fexp2:{ *:[f32] } (SelectClamp:{ *:[f32] } f32:{ *:[f32] }:$src0, i1:{ *:[i1] }:$clamp))  =>  (FEEPLE:{ *:[f32] } FPR32:{ *:[f32] }:$src0, (FFOO:{ *:[f32] } FPR32:{ *:[f32] }:$src0), clamp:{ *:[i1] }:$clamp)
-
-// CHECK-NEXT: GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s32,
-// CHECK-NEXT: GIR_BuildMI, /*InsnID*/1, /*Opcode*/MyTarget::FFOO,
-// CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/1, /*TempRegID*/0, /*TempRegFlags*/RegState::Define,
-// CHECK-NEXT: GIR_ComplexSubOperandRenderer, /*InsnID*/1, /*RendererID*/0, /*SubOperand*/0, // src0
-// CHECK-NEXT: GIR_AddImm, /*InsnID*/1, /*Imm*/0,
-// CHECK-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/1,
-// CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::FEEPLE,
-// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst]
-// CHECK-NEXT: GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/0, /*SubOperand*/0, // src0
-// CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/0,
-// CHECK-NEXT: GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/0, /*SubOperand*/1, // clamp
-// CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0,
-// CHECK-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0,
-
-
-// CHECK: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_FSIN,
-// CHECK: // (fsin:{ *:[f32] } (SelectClamp:{ *:[f32] } f32:{ *:[f32] }:$src0, i1:{ *:[i1] }:$clamp))  =>  (FFOO:{ *:[f32] } f32:{ *:[f32] }:$src0, i1:{ *:[i1] }:$clamp)
-// CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::FFOO,
-// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst]
-// CHECK-NEXT: GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/0, /*SubOperand*/0, // src0
-// CHECK-NEXT: GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/0, /*SubOperand*/1, // clamp
-// CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0,
-
-
-// CHECK: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_FSQRT,
-// CHECK: // (fsqrt:{ *:[f32] } (SelectClamp:{ *:[f32] } f32:{ *:[f32] }:$src0, i1:{ *:[i1] }:$clamp))  =>  (FLAMP:{ *:[f32] } FPR32:{ *:[f32] }:$src0, 93:{ *:[i32] }, clamp:{ *:[i1] }:$clamp)
-// CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::FLAMP,
-// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst]
-// CHECK-NEXT: GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/0, /*SubOperand*/0, // src0
-// CHECK-NEXT: GIR_AddImm, /*InsnID*/0, /*Imm*/93,
-// CHECK-NEXT: GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/0, /*SubOperand*/1, // clamp
-// CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0,
-
-// CHECK: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_INTRINSIC_ROUND,
-// CHECK: // (fround:{ *:[f32] } f32:{ *:[f32] }:$src0)  =>  (FBAR:{ *:[f32] } f32:{ *:[f32] }:$src0)
-// CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::FBAR,
-// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst]
-// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // src0
-// CHECK-NEXT: GIR_AddImm, /*InsnID*/0, /*Imm*/0,
-// CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0,
-
-// CHECK: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_INTRINSIC_TRUNC,
-// CHECK: // (ftrunc:{ *:[f32] } f32:{ *:[f32] }:$src0)  =>  (FFOO:{ *:[f32] } FPR32:{ *:[f32] }:$src0)
-// CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::FFOO,
-// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst]
-// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // src0
-// CHECK-NEXT: GIR_AddImm, /*InsnID*/0, /*Imm*/0,
-// CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0,
-
+// CHECK:      const uint8_t *MyTargetInstructionSelector::getMatchTable() const {
+// CHECK-NEXT:   constexpr static uint8_t MatchTable0[] = {
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 0*/ GIMT_Encode4(79), // Rule ID 3 //
+// CHECK-NEXT:       GIM_CheckNumOperands, /*MI*/0, /*Expected*/3,
+// CHECK-NEXT:       GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_FMAXNUM),
+// CHECK-NEXT:       // MIs[0] DstI[dst]
+// CHECK-NEXT:       GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
+// CHECK-NEXT:       GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::FPR32RegClassID),
+// CHECK-NEXT:       // MIs[0] SelectSrcMods:src0:mods0
+// CHECK-NEXT:       GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32,
+// CHECK-NEXT:       GIM_CheckComplexPattern, /*MI*/0, /*Op*/1, /*Renderer*/GIMT_Encode2(0), GIMT_Encode2(GICP_gi_SelectSrcMods),
+// CHECK-NEXT:       // MIs[0] SelectSrcMods:src1:mods1
+// CHECK-NEXT:       GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32,
+// CHECK-NEXT:       GIM_CheckComplexPattern, /*MI*/0, /*Op*/2, /*Renderer*/GIMT_Encode2(1), GIMT_Encode2(GICP_gi_SelectSrcMods),
+// CHECK-NEXT:       // (fmaxnum:{ *:[f32] } (SelectSrcMods:{ *:[f32] } f32:{ *:[f32] }:$src0, src_mods:{ *:[i32] }:$mods0), (SelectSrcMods:{ *:[f32] } f32:{ *:[f32] }:$src1, src_mods:{ *:[i32] }:$mods1))  =>  (FMAX:{ *:[f32] } src_mods:{ *:[i32] }:$mods0, f32:{ *:[f32] }:$src0, src_mods:{ *:[i32] }:$mods1, f32:{ *:[f32] }:$src1)
+// CHECK-NEXT:       GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::FMAX),
+// CHECK-NEXT:       GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst]
+// CHECK-NEXT:       GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/GIMT_Encode2(0), /*SubOperand*/1, // mods0
+// CHECK-NEXT:       GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/GIMT_Encode2(0), /*SubOperand*/0, // src0
+// CHECK-NEXT:       GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/GIMT_Encode2(1), /*SubOperand*/1, // mods1
+// CHECK-NEXT:       GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/GIMT_Encode2(1), /*SubOperand*/0, // src1
+// CHECK-NEXT:       GIR_AddImm8, /*InsnID*/0, /*Imm*/0,
+// CHECK-NEXT:       GIR_EraseFromParent, /*InsnID*/0,
+// CHECK-NEXT:       GIR_ConstrainSelectedInstOperands, /*InsnID*/0,
+// CHECK-NEXT:       // GIR_Coverage, 3,
+// CHECK-NEXT:       GIR_Done,
+// CHECK-NEXT:     // Label 0: @79
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 1*/ GIMT_Encode4(139), // Rule ID 2 //
+// CHECK-NEXT:       GIM_CheckNumOperands, /*MI*/0, /*Expected*/2,
+// CHECK-NEXT:       GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_FFLOOR),
+// CHECK-NEXT:       // MIs[0] DstI[dst]
+// CHECK-NEXT:       GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
+// CHECK-NEXT:       GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::FPR32RegClassID),
+// CHECK-NEXT:       // MIs[0] SelectClampOMod:src0:omod:clamp
+// CHECK-NEXT:       GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32,
+// CHECK-NEXT:       GIM_CheckComplexPattern, /*MI*/0, /*Op*/1, /*Renderer*/GIMT_Encode2(0), GIMT_Encode2(GICP_gi_SelectClampOMod),
+// CHECK-NEXT:       // (ffloor:{ *:[f32] } (SelectClampOMod:{ *:[f32] } f32:{ *:[f32] }:$src0, omod:{ *:[i32] }:$omod, i1:{ *:[i1] }:$clamp))  =>  (FLOMP:{ *:[f32] } f32:{ *:[f32] }:$src0, i1:{ *:[i1] }:$clamp, omod:{ *:[i32] }:$omod)
+// CHECK-NEXT:       GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::FLOMP),
+// CHECK-NEXT:       GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst]
+// CHECK-NEXT:       GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/GIMT_Encode2(0), /*SubOperand*/0, // src0
+// CHECK-NEXT:       GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/GIMT_Encode2(0), /*SubOperand*/2, // clamp
+// CHECK-NEXT:       GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/GIMT_Encode2(0), /*SubOperand*/1, // omod
+// CHECK-NEXT:       GIR_EraseFromParent, /*InsnID*/0,
+// CHECK-NEXT:       GIR_ConstrainSelectedInstOperands, /*InsnID*/0,
+// CHECK-NEXT:       // GIR_Coverage, 2,
+// CHECK-NEXT:       GIR_Done,
+// CHECK-NEXT:     // Label 1: @139
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 2*/ GIMT_Encode4(207), // Rule ID 8 //
+// CHECK-NEXT:       GIM_CheckNumOperands, /*MI*/0, /*Expected*/2,
+// CHECK-NEXT:       GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_FCANONICALIZE),
+// CHECK-NEXT:       // MIs[0] DstI[dst]
+// CHECK-NEXT:       GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
+// CHECK-NEXT:       GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::FPR32RegClassID),
+// CHECK-NEXT:       // MIs[0] SelectSrcMods:src:mods
+// CHECK-NEXT:       GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32,
+// CHECK-NEXT:       GIM_CheckComplexPattern, /*MI*/0, /*Op*/1, /*Renderer*/GIMT_Encode2(0), GIMT_Encode2(GICP_gi_SelectSrcMods),
+// CHECK-NEXT:       // (fcanonicalize:{ *:[f32] } (SelectSrcMods:{ *:[f32] } f32:{ *:[f32] }:$src, i32:{ *:[i32] }:$mods))  =>  (FMAX:{ *:[f32] } ?:{ *:[i32] }:$mods, ?:{ *:[f32] }:$src, ?:{ *:[i32] }:$mods, ?:{ *:[f32] }:$src, 0:{ *:[i1] })
+// CHECK-NEXT:       GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::FMAX),
+// CHECK-NEXT:       GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst]
+// CHECK-NEXT:       GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/GIMT_Encode2(0), /*SubOperand*/1, // mods
+// CHECK-NEXT:       GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/GIMT_Encode2(0), /*SubOperand*/0, // src
+// CHECK-NEXT:       GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/GIMT_Encode2(0), /*SubOperand*/1, // mods
+// CHECK-NEXT:       GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/GIMT_Encode2(0), /*SubOperand*/0, // src
+// CHECK-NEXT:       GIR_AddImm8, /*InsnID*/0, /*Imm*/0,
+// CHECK-NEXT:       GIR_EraseFromParent, /*InsnID*/0,
+// CHECK-NEXT:       GIR_ConstrainSelectedInstOperands, /*InsnID*/0,
+// CHECK-NEXT:       // GIR_Coverage, 8,
+// CHECK-NEXT:       GIR_Done,
+// CHECK-NEXT:     // Label 2: @207
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 3*/ GIMT_Encode4(265), // Rule ID 5 //
+// CHECK-NEXT:       GIM_CheckNumOperands, /*MI*/0, /*Expected*/2,
+// CHECK-NEXT:       GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_FCOS),
+// CHECK-NEXT:       // MIs[0] DstI[dst]
+// CHECK-NEXT:       GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
+// CHECK-NEXT:       GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::FPR32RegClassID),
+// CHECK-NEXT:       // MIs[0] SelectOMod:src0:omod
+// CHECK-NEXT:       GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32,
+// CHECK-NEXT:       GIM_CheckComplexPattern, /*MI*/0, /*Op*/1, /*Renderer*/GIMT_Encode2(0), GIMT_Encode2(GICP_gi_SelectOMod),
+// CHECK-NEXT:       // (fcos:{ *:[f32] } (SelectOMod:{ *:[f32] } f32:{ *:[f32] }:$src0, i32:{ *:[i32] }:$omod))  =>  (FLAMP:{ *:[f32] } FPR32:{ *:[f32] }:$src0, omod:{ *:[i32] }:$omod)
+// CHECK-NEXT:       GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::FLAMP),
+// CHECK-NEXT:       GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst]
+// CHECK-NEXT:       GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/GIMT_Encode2(0), /*SubOperand*/0, // src0
+// CHECK-NEXT:       GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/GIMT_Encode2(0), /*SubOperand*/1, // omod
+// CHECK-NEXT:       GIR_AddImm8, /*InsnID*/0, /*Imm*/0,
+// CHECK-NEXT:       GIR_EraseFromParent, /*InsnID*/0,
+// CHECK-NEXT:       GIR_ConstrainSelectedInstOperands, /*InsnID*/0,
+// CHECK-NEXT:       // GIR_Coverage, 5,
+// CHECK-NEXT:       GIR_Done,
+// CHECK-NEXT:     // Label 3: @265
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 4*/ GIMT_Encode4(345), // Rule ID 7 //
+// CHECK-NEXT:       GIM_CheckNumOperands, /*MI*/0, /*Expected*/2,
+// CHECK-NEXT:       GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_FEXP2),
+// CHECK-NEXT:       // MIs[0] DstI[dst]
+// CHECK-NEXT:       GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
+// CHECK-NEXT:       GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::FPR32RegClassID),
+// CHECK-NEXT:       // MIs[0] SelectClamp:src0:clamp
+// CHECK-NEXT:       GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32,
+// CHECK-NEXT:       GIM_CheckComplexPattern, /*MI*/0, /*Op*/1, /*Renderer*/GIMT_Encode2(0), GIMT_Encode2(GICP_gi_SelectClamp),
+// CHECK-NEXT:       // (fexp2:{ *:[f32] } (SelectClamp:{ *:[f32] } f32:{ *:[f32] }:$src0, i1:{ *:[i1] }:$clamp))  =>  (FEEPLE:{ *:[f32] } FPR32:{ *:[f32] }:$src0, (FFOO:{ *:[f32] } FPR32:{ *:[f32] }:$src0), clamp:{ *:[i1] }:$clamp)
+// CHECK-NEXT:       GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s32,
+// CHECK-NEXT:       GIR_BuildMI, /*InsnID*/1, /*Opcode*/GIMT_Encode2(MyTarget::FFOO),
+// CHECK-NEXT:       GIR_AddTempRegister, /*InsnID*/1, /*TempRegID*/0, /*TempRegFlags*/GIMT_Encode2(RegState::Define),
+// CHECK-NEXT:       GIR_ComplexSubOperandRenderer, /*InsnID*/1, /*RendererID*/GIMT_Encode2(0), /*SubOperand*/0, // src0
+// CHECK-NEXT:       GIR_AddImm8, /*InsnID*/1, /*Imm*/0,
+// CHECK-NEXT:       GIR_ConstrainSelectedInstOperands, /*InsnID*/1,
+// CHECK-NEXT:       GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::FEEPLE),
+// CHECK-NEXT:       GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst]
+// CHECK-NEXT:       GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/GIMT_Encode2(0), /*SubOperand*/0, // src0
+// CHECK-NEXT:       GIR_AddSimpleTempRegister, /*InsnID*/0, /*TempRegID*/0,
+// CHECK-NEXT:       GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/GIMT_Encode2(0), /*SubOperand*/1, // clamp
+// CHECK-NEXT:       GIR_EraseFromParent, /*InsnID*/0,
+// CHECK-NEXT:       GIR_ConstrainSelectedInstOperands, /*InsnID*/0,
+// CHECK-NEXT:       // GIR_Coverage, 7,
+// CHECK-NEXT:       GIR_Done,
+// CHECK-NEXT:     // Label 4: @345
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 5*/ GIMT_Encode4(400), // Rule ID 0 //
+// CHECK-NEXT:       GIM_CheckNumOperands, /*MI*/0, /*Expected*/2,
+// CHECK-NEXT:       GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_FSIN),
+// CHECK-NEXT:       // MIs[0] DstI[dst]
+// CHECK-NEXT:       GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
+// CHECK-NEXT:       GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::FPR32RegClassID),
+// CHECK-NEXT:       // MIs[0] SelectClamp:src0:clamp
+// CHECK-NEXT:       GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32,
+// CHECK-NEXT:       GIM_CheckComplexPattern, /*MI*/0, /*Op*/1, /*Renderer*/GIMT_Encode2(0), GIMT_Encode2(GICP_gi_SelectClamp),
+// CHECK-NEXT:       // (fsin:{ *:[f32] } (SelectClamp:{ *:[f32] } f32:{ *:[f32] }:$src0, i1:{ *:[i1] }:$clamp))  =>  (FFOO:{ *:[f32] } f32:{ *:[f32] }:$src0, i1:{ *:[i1] }:$clamp)
+// CHECK-NEXT:       GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::FFOO),
+// CHECK-NEXT:       GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst]
+// CHECK-NEXT:       GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/GIMT_Encode2(0), /*SubOperand*/0, // src0
+// CHECK-NEXT:       GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/GIMT_Encode2(0), /*SubOperand*/1, // clamp
+// CHECK-NEXT:       GIR_EraseFromParent, /*InsnID*/0,
+// CHECK-NEXT:       GIR_ConstrainSelectedInstOperands, /*InsnID*/0,
+// CHECK-NEXT:       // GIR_Coverage, 0,
+// CHECK-NEXT:       GIR_Done,
+// CHECK-NEXT:     // Label 5: @400
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 6*/ GIMT_Encode4(458), // Rule ID 6 //
+// CHECK-NEXT:       GIM_CheckNumOperands, /*MI*/0, /*Expected*/2,
+// CHECK-NEXT:       GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_FSQRT),
+// CHECK-NEXT:       // MIs[0] DstI[dst]
+// CHECK-NEXT:       GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
+// CHECK-NEXT:       GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::FPR32RegClassID),
+// CHECK-NEXT:       // MIs[0] SelectClamp:src0:clamp
+// CHECK-NEXT:       GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32,
+// CHECK-NEXT:       GIM_CheckComplexPattern, /*MI*/0, /*Op*/1, /*Renderer*/GIMT_Encode2(0), GIMT_Encode2(GICP_gi_SelectClamp),
+// CHECK-NEXT:       // (fsqrt:{ *:[f32] } (SelectClamp:{ *:[f32] } f32:{ *:[f32] }:$src0, i1:{ *:[i1] }:$clamp))  =>  (FLAMP:{ *:[f32] } FPR32:{ *:[f32] }:$src0, 93:{ *:[i32] }, clamp:{ *:[i1] }:$clamp)
+// CHECK-NEXT:       GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::FLAMP),
+// CHECK-NEXT:       GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst]
+// CHECK-NEXT:       GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/GIMT_Encode2(0), /*SubOperand*/0, // src0
+// CHECK-NEXT:       GIR_AddImm8, /*InsnID*/0, /*Imm*/93,
+// CHECK-NEXT:       GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/GIMT_Encode2(0), /*SubOperand*/1, // clamp
+// CHECK-NEXT:       GIR_EraseFromParent, /*InsnID*/0,
+// CHECK-NEXT:       GIR_ConstrainSelectedInstOperands, /*InsnID*/0,
+// CHECK-NEXT:       // GIR_Coverage, 6,
+// CHECK-NEXT:       GIR_Done,
+// CHECK-NEXT:     // Label 6: @458
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 7*/ GIMT_Encode4(503), // Rule ID 1 //
+// CHECK-NEXT:       GIM_CheckNumOperands, /*MI*/0, /*Expected*/2,
+// CHECK-NEXT:       GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_INTRINSIC_ROUND),
+// CHECK-NEXT:       // MIs[0] DstI[dst]
+// CHECK-NEXT:       GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
+// CHECK-NEXT:       GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::FPR32RegClassID),
+// CHECK-NEXT:       // MIs[0] src0
+// CHECK-NEXT:       GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32,
+// CHECK-NEXT:       // (fround:{ *:[f32] } f32:{ *:[f32] }:$src0)  =>  (FBAR:{ *:[f32] } f32:{ *:[f32] }:$src0)
+// CHECK-NEXT:       GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::FBAR),
+// CHECK-NEXT:       GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst]
+// CHECK-NEXT:       GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // src0
+// CHECK-NEXT:       GIR_AddImm8, /*InsnID*/0, /*Imm*/0,
+// CHECK-NEXT:       GIR_EraseFromParent, /*InsnID*/0,
+// CHECK-NEXT:       GIR_ConstrainSelectedInstOperands, /*InsnID*/0,
+// CHECK-NEXT:       // GIR_Coverage, 1,
+// CHECK-NEXT:       GIR_Done,
+// CHECK-NEXT:     // Label 7: @503
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 8*/ GIMT_Encode4(548), // Rule ID 4 //
+// CHECK-NEXT:       GIM_CheckNumOperands, /*MI*/0, /*Expected*/2,
+// CHECK-NEXT:       GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_INTRINSIC_TRUNC),
+// CHECK-NEXT:       // MIs[0] DstI[dst]
+// CHECK-NEXT:       GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
+// CHECK-NEXT:       GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::FPR32RegClassID),
+// CHECK-NEXT:       // MIs[0] src0
+// CHECK-NEXT:       GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32,
+// CHECK-NEXT:       // (ftrunc:{ *:[f32] } f32:{ *:[f32] }:$src0)  =>  (FFOO:{ *:[f32] } FPR32:{ *:[f32] }:$src0)
+// CHECK-NEXT:       GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::FFOO),
+// CHECK-NEXT:       GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst]
+// CHECK-NEXT:       GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // src0
+// CHECK-NEXT:       GIR_AddImm8, /*InsnID*/0, /*Imm*/0,
+// CHECK-NEXT:       GIR_EraseFromParent, /*InsnID*/0,
+// CHECK-NEXT:       GIR_ConstrainSelectedInstOperands, /*InsnID*/0,
+// CHECK-NEXT:       // GIR_Coverage, 4,
+// CHECK-NEXT:       GIR_Done,
+// CHECK-NEXT:     // Label 8: @548
+// CHECK-NEXT:     GIM_Reject,
+// CHECK-NEXT:     }; // Size: 549 bytes
+// CHECK-NEXT:   return MatchTable0;
+// CHECK-NEXT: }
 
 // Have default operand with explicit value from complex pattern.
 def FFOO : I<(outs FPR32:$dst), (ins FPR32:$src0, clamp:$clamp),
diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/builtins/match-table-eraseroot.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/builtins/match-table-eraseroot.td
index 0dd265c14ddc2d7d7be2dd3a89a15efada4c2b2a..25fdd887b20b50b4b41b30fd61ffa299cf9df3fa 100644
--- a/llvm/test/TableGen/GlobalISelCombinerEmitter/builtins/match-table-eraseroot.td
+++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/builtins/match-table-eraseroot.td
@@ -17,11 +17,11 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
   Test0,
 ]>;
 
-// CHECK:      const int64_t *GenMyCombiner::getMatchTable() const {
-// CHECK-NEXT:   constexpr static int64_t MatchTable0[] = {
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 0*/ 10, // Rule ID 0 //
-// CHECK-NEXT:       GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule0Enabled,
-// CHECK-NEXT:       GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_STORE,
+// CHECK:      const uint8_t *GenMyCombiner::getMatchTable() const {
+// CHECK-NEXT:   constexpr static uint8_t MatchTable0[] = {
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 0*/ GIMT_Encode4(15), // Rule ID 0 //
+// CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled),
+// CHECK-NEXT:       GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_STORE),
 // CHECK-NEXT:       // MIs[0] a
 // CHECK-NEXT:       // No operand predicates
 // CHECK-NEXT:       // MIs[0] b
@@ -29,7 +29,7 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:       // Combiner Rule #0: Test0
 // CHECK-NEXT:       GIR_EraseFromParent, /*InsnID*/0,
 // CHECK-NEXT:       GIR_Done,
-// CHECK-NEXT:     // Label 0: @10
+// CHECK-NEXT:     // Label 0: @15
 // CHECK-NEXT:     GIM_Reject,
 // CHECK-NEXT:     };
 // CHECK-NEXT:   return MatchTable0;
diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/builtins/match-table-replacerreg.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/builtins/match-table-replacerreg.td
index 6ae1305aa1aa51699770b1a7a75c4b87cae4e7c6..f4f1faf7cc6d9aaa184e4710b54151c2bf8baac8 100644
--- a/llvm/test/TableGen/GlobalISelCombinerEmitter/builtins/match-table-replacerreg.td
+++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/builtins/match-table-replacerreg.td
@@ -26,14 +26,14 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
   ReplaceTemp
 ]>;
 
-// CHECK:      const int64_t *GenMyCombiner::getMatchTable() const {
-// CHECK-NEXT:   constexpr static int64_t MatchTable0[] = {
-// CHECK-NEXT:     GIM_SwitchOpcode, /*MI*/0, /*[*/65, 181, /*)*//*default:*//*Label 2*/ 193,
-// CHECK-NEXT:     /*TargetOpcode::G_UNMERGE_VALUES*//*Label 0*/ 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-// CHECK-NEXT:     /*TargetOpcode::G_FNEG*//*Label 1*/ 166,
-// CHECK-NEXT:     // Label 0: @121
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 3*/ 165, // Rule ID 1 //
-// CHECK-NEXT:       GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule1Enabled,
+// CHECK:      const uint8_t *GenMyCombiner::getMatchTable() const {
+// CHECK-NEXT:   constexpr static uint8_t MatchTable0[] = {
+// CHECK-NEXT:     GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(65), GIMT_Encode2(181), /*)*//*default:*//*Label 2*/ GIMT_Encode4(556),
+// CHECK-NEXT:     /*TargetOpcode::G_UNMERGE_VALUES*//*Label 0*/ GIMT_Encode4(474), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
+// CHECK-NEXT:     /*TargetOpcode::G_FNEG*//*Label 1*/ GIMT_Encode4(524),
+// CHECK-NEXT:     // Label 0: @474
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 3*/ GIMT_Encode4(523), // Rule ID 1 //
+// CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule1Enabled),
 // CHECK-NEXT:       GIM_CheckNumOperands, /*MI*/0, /*Expected*/3,
 // CHECK-NEXT:       // MIs[0] a
 // CHECK-NEXT:       // No operand predicates
@@ -41,7 +41,7 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:       // No operand predicates
 // CHECK-NEXT:       // MIs[0] tmp
 // CHECK-NEXT:       GIM_RecordInsnIgnoreCopies, /*DefineMI*/1, /*MI*/0, /*OpIdx*/2, // MIs[1]
-// CHECK-NEXT:       GIM_CheckOpcode, /*MI*/1, TargetOpcode::G_BUILD_VECTOR,
+// CHECK-NEXT:       GIM_CheckOpcode, /*MI*/1, GIMT_Encode2(TargetOpcode::G_BUILD_VECTOR),
 // CHECK-NEXT:       GIM_CheckNumOperands, /*MI*/1, /*Expected*/3,
 // CHECK-NEXT:       // MIs[1] x
 // CHECK-NEXT:       // No operand predicates
@@ -50,23 +50,23 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:       GIM_CheckIsSafeToFold, /*InsnID*/1,
 // CHECK-NEXT:       GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s32,
 // CHECK-NEXT:       // Combiner Rule #1: ReplaceTemp
-// CHECK-NEXT:       GIR_BuildMI, /*InsnID*/0, /*Opcode*/TargetOpcode::G_UNMERGE_VALUES,
+// CHECK-NEXT:       GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(TargetOpcode::G_UNMERGE_VALUES),
 // CHECK-NEXT:       GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // a
-// CHECK-NEXT:       GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/0,
+// CHECK-NEXT:       GIR_AddSimpleTempRegister, /*InsnID*/0, /*TempRegID*/0,
 // CHECK-NEXT:       GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/1, /*OpIdx*/2, // y
 // CHECK-NEXT:       GIR_EraseFromParent, /*InsnID*/0,
 // CHECK-NEXT:       GIR_ReplaceRegWithTempReg, /*OldInsnID*/0, /*OldOpIdx*/1, /*TempRegID*/0,
 // CHECK-NEXT:       GIR_Done,
-// CHECK-NEXT:     // Label 3: @165
+// CHECK-NEXT:     // Label 3: @523
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     // Label 1: @166
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 4*/ 192, // Rule ID 0 //
-// CHECK-NEXT:       GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule0Enabled,
+// CHECK-NEXT:     // Label 1: @524
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 4*/ GIMT_Encode4(555), // Rule ID 0 //
+// CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled),
 // CHECK-NEXT:       // MIs[0] dst
 // CHECK-NEXT:       // No operand predicates
 // CHECK-NEXT:       // MIs[0] tmp
 // CHECK-NEXT:       GIM_RecordInsnIgnoreCopies, /*DefineMI*/1, /*MI*/0, /*OpIdx*/1, // MIs[1]
-// CHECK-NEXT:       GIM_CheckOpcode, /*MI*/1, TargetOpcode::G_FNEG,
+// CHECK-NEXT:       GIM_CheckOpcode, /*MI*/1, GIMT_Encode2(TargetOpcode::G_FNEG),
 // CHECK-NEXT:       // MIs[1] src
 // CHECK-NEXT:       // No operand predicates
 // CHECK-NEXT:       GIM_CheckCanReplaceReg, /*OldInsnID*/0, /*OldOpIdx*/0, /*NewInsnId*/1, /*NewOpIdx*/1,
@@ -75,10 +75,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:       GIR_ReplaceReg, /*OldInsnID*/0, /*OldOpIdx*/0, /*NewInsnId*/1, /*NewOpIdx*/1,
 // CHECK-NEXT:       GIR_EraseFromParent, /*InsnID*/0,
 // CHECK-NEXT:       GIR_Done,
-// CHECK-NEXT:     // Label 4: @192
+// CHECK-NEXT:     // Label 4: @555
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     // Label 2: @193
+// CHECK-NEXT:     // Label 2: @556
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     };
+// CHECK-NEXT:     }; // Size: 557 bytes
 // CHECK-NEXT:   return MatchTable0;
 // CHECK-NEXT: }
diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-imms.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-imms.td
index fd5f7db0b4f12d8298871e0dae4070ccf61cc18b..d0c0eba9e3974abff58f2a29453e459fb5ab9f0c 100644
--- a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-imms.td
+++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-imms.td
@@ -32,61 +32,61 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
   CImmInstTest1
 ]>;
 
-// CHECK:      const int64_t *GenMyCombiner::getMatchTable() const {
-// CHECK-NEXT:   constexpr static int64_t MatchTable0[] = {
-// CHECK-NEXT:     GIM_SwitchOpcode, /*MI*/0, /*[*/19, 127, /*)*//*default:*//*Label 3*/ 195,
-// CHECK-NEXT:     /*TargetOpcode::COPY*//*Label 0*/ 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-// CHECK-NEXT:     /*TargetOpcode::G_CONSTANT*//*Label 1*/ 139, 0, 0, 0, 0, 0,
-// CHECK-NEXT:     /*TargetOpcode::G_ZEXT*//*Label 2*/ 166,
-// CHECK-NEXT:     // Label 0: @113
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 4*/ 138, // Rule ID 0 //
-// CHECK-NEXT:       GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule0Enabled,
+// CHECK:      const uint8_t *GenMyCombiner::getMatchTable() const {
+// CHECK-NEXT:   constexpr static uint8_t MatchTable0[] = {
+// CHECK-NEXT:     GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(19), GIMT_Encode2(127), /*)*//*default:*//*Label 3*/ GIMT_Encode4(559),
+// CHECK-NEXT:     /*TargetOpcode::COPY*//*Label 0*/ GIMT_Encode4(442), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
+// CHECK-NEXT:     /*TargetOpcode::G_CONSTANT*//*Label 1*/ GIMT_Encode4(473), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
+// CHECK-NEXT:     /*TargetOpcode::G_ZEXT*//*Label 2*/ GIMT_Encode4(519),
+// CHECK-NEXT:     // Label 0: @442
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 4*/ GIMT_Encode4(472), // Rule ID 0 //
+// CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled),
 // CHECK-NEXT:       GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32,
 // CHECK-NEXT:       // MIs[0] a
 // CHECK-NEXT:       // No operand predicates
-// CHECK-NEXT:       GIM_CheckConstantInt, /*MI*/0, /*Op*/1, 0,
+// CHECK-NEXT:       GIM_CheckConstantInt8, /*MI*/0, /*Op*/1, 0,
 // CHECK-NEXT:       // Combiner Rule #0: InstTest0
-// CHECK-NEXT:       GIR_BuildMI, /*InsnID*/0, /*Opcode*/TargetOpcode::COPY,
+// CHECK-NEXT:       GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(TargetOpcode::COPY),
 // CHECK-NEXT:       GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // a
-// CHECK-NEXT:       GIR_AddImm, /*InsnID*/0, /*Imm*/0,
+// CHECK-NEXT:       GIR_AddImm8, /*InsnID*/0, /*Imm*/0,
 // CHECK-NEXT:       GIR_EraseFromParent, /*InsnID*/0,
 // CHECK-NEXT:       GIR_Done,
-// CHECK-NEXT:     // Label 4: @138
+// CHECK-NEXT:     // Label 4: @472
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     // Label 1: @139
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 5*/ 165, // Rule ID 2 //
-// CHECK-NEXT:       GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule2Enabled,
+// CHECK-NEXT:     // Label 1: @473
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 5*/ GIMT_Encode4(518), // Rule ID 2 //
+// CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule2Enabled),
 // CHECK-NEXT:       GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32,
 // CHECK-NEXT:       // MIs[0] a
 // CHECK-NEXT:       // No operand predicates
-// CHECK-NEXT:       GIM_CheckLiteralInt, /*MI*/0, /*Op*/1, 0,
+// CHECK-NEXT:       GIM_CheckLiteralInt, /*MI*/0, /*Op*/1, GIMT_Encode8(0),
 // CHECK-NEXT:       // Combiner Rule #2: CImmInstTest1
-// CHECK-NEXT:       GIR_BuildMI, /*InsnID*/0, /*Opcode*/TargetOpcode::G_CONSTANT,
+// CHECK-NEXT:       GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(TargetOpcode::G_CONSTANT),
 // CHECK-NEXT:       GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // a
-// CHECK-NEXT:       GIR_AddCImm, /*InsnID*/0, /*Type*/GILLT_s32, /*Imm*/42,
+// CHECK-NEXT:       GIR_AddCImm, /*InsnID*/0, /*Type*/GILLT_s32, /*Imm*/GIMT_Encode8(42),
 // CHECK-NEXT:       GIR_EraseFromParent, /*InsnID*/0,
 // CHECK-NEXT:       GIR_Done,
-// CHECK-NEXT:     // Label 5: @165
+// CHECK-NEXT:     // Label 5: @518
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     // Label 2: @166
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 6*/ 194, // Rule ID 1 //
-// CHECK-NEXT:       GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule1Enabled,
+// CHECK-NEXT:     // Label 2: @519
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 6*/ GIMT_Encode4(558), // Rule ID 1 //
+// CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule1Enabled),
 // CHECK-NEXT:       // MIs[0] a
 // CHECK-NEXT:       // No operand predicates
 // CHECK-NEXT:       // MIs[0] Operand 1
-// CHECK-NEXT:       GIM_CheckConstantInt, /*MI*/0, /*Op*/1, 0,
+// CHECK-NEXT:       GIM_CheckConstantInt8, /*MI*/0, /*Op*/1, 0,
 // CHECK-NEXT:       GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s32,
-// CHECK-NEXT:       GIR_BuildConstant, /*TempRegID*/0, /*Val*/0,
+// CHECK-NEXT:       GIR_BuildConstant, /*TempRegID*/0, /*Val*/GIMT_Encode8(0),
 // CHECK-NEXT:       // Combiner Rule #1: InstTest1
-// CHECK-NEXT:       GIR_BuildMI, /*InsnID*/0, /*Opcode*/TargetOpcode::COPY,
+// CHECK-NEXT:       GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(TargetOpcode::COPY),
 // CHECK-NEXT:       GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // a
-// CHECK-NEXT:       GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/0,
+// CHECK-NEXT:       GIR_AddSimpleTempRegister, /*InsnID*/0, /*TempRegID*/0,
 // CHECK-NEXT:       GIR_EraseFromParent, /*InsnID*/0,
 // CHECK-NEXT:       GIR_Done,
-// CHECK-NEXT:     // Label 6: @194
+// CHECK-NEXT:     // Label 6: @558
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     // Label 3: @195
+// CHECK-NEXT:     // Label 3: @559
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     };
+// CHECK-NEXT:     }; // Size: 560 bytes
 // CHECK-NEXT:   return MatchTable0;
 // CHECK-NEXT: }
diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-miflags.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-miflags.td
index 9f02ff17493652d172c8fc6553466ed29811c962..22e4d2d5d9d14b7058d08f233061ac9676f819f9 100644
--- a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-miflags.td
+++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-miflags.td
@@ -15,32 +15,32 @@ def MIFlagsTest : GICombineRule<
 
 def MyCombiner: GICombiner<"GenMyCombiner", [MIFlagsTest]>;
 
-// CHECK:      const int64_t *GenMyCombiner::getMatchTable() const {
-// CHECK-NEXT:   constexpr static int64_t MatchTable0[] = {
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 0*/ 49, // Rule ID 0 //
-// CHECK-NEXT:       GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule0Enabled,
-// CHECK-NEXT:       GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_SEXT,
+// CHECK:      const uint8_t *GenMyCombiner::getMatchTable() const {
+// CHECK-NEXT:   constexpr static uint8_t MatchTable0[] = {
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 0*/ GIMT_Encode4(68), // Rule ID 0 //
+// CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled),
+// CHECK-NEXT:       GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_SEXT),
 // CHECK-NEXT:       // MIs[0] dst
 // CHECK-NEXT:       // No operand predicates
 // CHECK-NEXT:       // MIs[0] tmp
 // CHECK-NEXT:       GIM_RecordInsnIgnoreCopies, /*DefineMI*/1, /*MI*/0, /*OpIdx*/1, // MIs[1]
-// CHECK-NEXT:       GIM_CheckOpcode, /*MI*/1, TargetOpcode::G_ZEXT,
-// CHECK-NEXT:       GIM_MIFlags, /*MI*/1, MachineInstr::FmNsz | MachineInstr::FmReassoc,
-// CHECK-NEXT:       GIM_MIFlagsNot, /*MI*/1, MachineInstr::FmArcp | MachineInstr::FmNoNans,
+// CHECK-NEXT:       GIM_CheckOpcode, /*MI*/1, GIMT_Encode2(TargetOpcode::G_ZEXT),
+// CHECK-NEXT:       GIM_MIFlags, /*MI*/1, GIMT_Encode4(MachineInstr::FmNsz | MachineInstr::FmReassoc),
+// CHECK-NEXT:       GIM_MIFlagsNot, /*MI*/1, GIMT_Encode4(MachineInstr::FmArcp | MachineInstr::FmNoNans),
 // CHECK-NEXT:       // MIs[1] src
 // CHECK-NEXT:       // No operand predicates
 // CHECK-NEXT:       GIM_CheckIsSafeToFold, /*InsnID*/1,
 // CHECK-NEXT:       // Combiner Rule #0: MIFlagsTest
-// CHECK-NEXT:       GIR_BuildMI, /*InsnID*/0, /*Opcode*/TargetOpcode::G_MUL,
+// CHECK-NEXT:       GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(TargetOpcode::G_MUL),
 // CHECK-NEXT:       GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst
 // CHECK-NEXT:       GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/1, /*OpIdx*/1, // src
 // CHECK-NEXT:       GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/1, /*OpIdx*/1, // src
 // CHECK-NEXT:       GIR_CopyMIFlags, /*InsnID*/0, /*OldInsnID*/1,
-// CHECK-NEXT:       GIR_SetMIFlags, /*InsnID*/0, MachineInstr::FmReassoc,
-// CHECK-NEXT:       GIR_UnsetMIFlags, /*InsnID*/0, MachineInstr::FmNsz | MachineInstr::FmArcp,
+// CHECK-NEXT:       GIR_SetMIFlags, /*InsnID*/0, GIMT_Encode4(MachineInstr::FmReassoc),
+// CHECK-NEXT:       GIR_UnsetMIFlags, /*InsnID*/0, GIMT_Encode4(MachineInstr::FmNsz | MachineInstr::FmArcp),
 // CHECK-NEXT:       GIR_EraseFromParent, /*InsnID*/0,
 // CHECK-NEXT:       GIR_Done,
-// CHECK-NEXT:     // Label 0: @49
+// CHECK-NEXT:     // Label 0: @68
 // CHECK-NEXT:     GIM_Reject,
 // CHECK-NEXT:     };
 // CHECK-NEXT:   return MatchTable0;
diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-operand-types.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-operand-types.td
index fa2451aa958201181d00d00827d4a613b4555b91..06368db75e3ba0b917b5c2a40043b3d6c32ea852 100644
--- a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-operand-types.td
+++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-operand-types.td
@@ -19,33 +19,33 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
   InstTest0,
 ]>;
 
-// CHECK:       const int64_t *GenMyCombiner::getMatchTable() const {
-// CHECK-NEXT:   constexpr static int64_t MatchTable0[] = {
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 0*/ 73, // Rule ID 0 //
-// CHECK-NEXT:       GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule0Enabled,
-// CHECK-NEXT:       GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_MUL,
+// CHECK:      const uint8_t *GenMyCombiner::getMatchTable() const {
+// CHECK-NEXT:   constexpr static uint8_t MatchTable0[] = {
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 0*/ GIMT_Encode4(79), // Rule ID 0 //
+// CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled),
+// CHECK-NEXT:       GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_MUL),
 // CHECK-NEXT:       GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s8,
 // CHECK-NEXT:       GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32,
 // CHECK-NEXT:       GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32,
 // CHECK-NEXT:       GIM_RecordInsnIgnoreCopies, /*DefineMI*/1, /*MI*/0, /*OpIdx*/2, // MIs[1]
-// CHECK-NEXT:       GIM_CheckOpcode, /*MI*/1, TargetOpcode::G_MUL,
+// CHECK-NEXT:       GIM_CheckOpcode, /*MI*/1, GIMT_Encode2(TargetOpcode::G_MUL),
 // CHECK-NEXT:       GIM_CheckType, /*MI*/1, /*Op*/2, /*Type*/GILLT_s32,
 // CHECK-NEXT:       // MIs[1] b
 // CHECK-NEXT:       GIM_CheckIsSameOperandIgnoreCopies, /*MI*/1, /*OpIdx*/1, /*OtherMI*/0, /*OtherOpIdx*/1,
 // CHECK-NEXT:       GIM_CheckIsSafeToFold, /*InsnID*/1,
 // CHECK-NEXT:       GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s64,
 // CHECK-NEXT:       // Combiner Rule #0: InstTest0
-// CHECK-NEXT:       GIR_BuildMI, /*InsnID*/0, /*Opcode*/TargetOpcode::G_ADD,
-// CHECK-NEXT:       GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/0,
+// CHECK-NEXT:       GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(TargetOpcode::G_ADD),
+// CHECK-NEXT:       GIR_AddSimpleTempRegister, /*InsnID*/0, /*TempRegID*/0,
 // CHECK-NEXT:       GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // b
 // CHECK-NEXT:       GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/1, /*OpIdx*/2, // c
 // CHECK-NEXT:       GIR_EraseFromParent, /*InsnID*/0,
-// CHECK-NEXT:       GIR_BuildMI, /*InsnID*/1, /*Opcode*/TargetOpcode::G_ADD,
+// CHECK-NEXT:       GIR_BuildMI, /*InsnID*/1, /*Opcode*/GIMT_Encode2(TargetOpcode::G_ADD),
 // CHECK-NEXT:       GIR_Copy, /*NewInsnID*/1, /*OldInsnID*/0, /*OpIdx*/0, // a
 // CHECK-NEXT:       GIR_Copy, /*NewInsnID*/1, /*OldInsnID*/0, /*OpIdx*/1, // b
-// CHECK-NEXT:       GIR_AddTempRegister, /*InsnID*/1, /*TempRegID*/0, /*TempRegFlags*/0,
+// CHECK-NEXT:       GIR_AddSimpleTempRegister, /*InsnID*/1, /*TempRegID*/0,
 // CHECK-NEXT:       GIR_Done,
-// CHECK-NEXT:     // Label 0: @73
+// CHECK-NEXT:     // Label 0: @79
 // CHECK-NEXT:     GIM_Reject,
 // CHECK-NEXT:     };
 // CHECK-NEXT:   return MatchTable0;
diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-patfrag-root.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-patfrag-root.td
index b62ebcf24687437b122692c7c9248592d737aa56..fdcb31e9679787636dae20fdb65ea1a863930342 100644
--- a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-patfrag-root.td
+++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-patfrag-root.td
@@ -26,69 +26,69 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
   Test0
 ]>;
 
-// CHECK:      const int64_t *GenMyCombiner::getMatchTable() const {
-// CHECK-NEXT:   constexpr static int64_t MatchTable0[] = {
-// CHECK-NEXT:     GIM_SwitchOpcode, /*MI*/0, /*[*/119, 182, /*)*//*default:*//*Label 3*/ 152,
-// CHECK-NEXT:     /*TargetOpcode::G_TRUNC*//*Label 0*/ 68, 0, 0, 0, 0, 0, 0,
-// CHECK-NEXT:     /*TargetOpcode::G_ZEXT*//*Label 1*/ 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-// CHECK-NEXT:     /*TargetOpcode::G_FPEXT*//*Label 2*/ 127,
-// CHECK-NEXT:     // Label 0: @68
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 4*/ 92, // Rule ID 1 //
-// CHECK-NEXT:       GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule0Enabled,
+// CHECK:      const uint8_t *GenMyCombiner::getMatchTable() const {
+// CHECK-NEXT:   constexpr static uint8_t MatchTable0[] = {
+// CHECK-NEXT:     GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(119), GIMT_Encode2(182), /*)*//*default:*//*Label 3*/ GIMT_Encode4(380),
+// CHECK-NEXT:     /*TargetOpcode::G_TRUNC*//*Label 0*/ GIMT_Encode4(262), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
+// CHECK-NEXT:     /*TargetOpcode::G_ZEXT*//*Label 1*/ GIMT_Encode4(298), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
+// CHECK-NEXT:     /*TargetOpcode::G_FPEXT*//*Label 2*/ GIMT_Encode4(344),
+// CHECK-NEXT:     // Label 0: @262
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 4*/ GIMT_Encode4(297), // Rule ID 1 //
+// CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled),
 // CHECK-NEXT:       // MIs[0] root
 // CHECK-NEXT:       // No operand predicates
 // CHECK-NEXT:       // MIs[0] __Test0_match_0.z
 // CHECK-NEXT:       // No operand predicates
 // CHECK-NEXT:       GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s32,
-// CHECK-NEXT:       GIR_BuildConstant, /*TempRegID*/0, /*Val*/0,
+// CHECK-NEXT:       GIR_BuildConstant, /*TempRegID*/0, /*Val*/GIMT_Encode8(0),
 // CHECK-NEXT:       // Combiner Rule #0: Test0 @ [__Test0_match_0[1]]
-// CHECK-NEXT:       GIR_BuildMI, /*InsnID*/0, /*Opcode*/TargetOpcode::COPY,
+// CHECK-NEXT:       GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(TargetOpcode::COPY),
 // CHECK-NEXT:       GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // root
-// CHECK-NEXT:       GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/0,
+// CHECK-NEXT:       GIR_AddSimpleTempRegister, /*InsnID*/0, /*TempRegID*/0,
 // CHECK-NEXT:       GIR_EraseFromParent, /*InsnID*/0,
 // CHECK-NEXT:       GIR_Done,
-// CHECK-NEXT:     // Label 4: @92
+// CHECK-NEXT:     // Label 4: @297
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     // Label 1: @93
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 5*/ 126, // Rule ID 0 //
-// CHECK-NEXT:       GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule0Enabled,
+// CHECK-NEXT:     // Label 1: @298
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 5*/ GIMT_Encode4(343), // Rule ID 0 //
+// CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled),
 // CHECK-NEXT:       // MIs[0] root
 // CHECK-NEXT:       // No operand predicates
 // CHECK-NEXT:       // MIs[0] __Test0_match_0.b
 // CHECK-NEXT:       GIM_RecordInsnIgnoreCopies, /*DefineMI*/1, /*MI*/0, /*OpIdx*/1, // MIs[1]
-// CHECK-NEXT:       GIM_CheckOpcode, /*MI*/1, TargetOpcode::G_TRUNC,
+// CHECK-NEXT:       GIM_CheckOpcode, /*MI*/1, GIMT_Encode2(TargetOpcode::G_TRUNC),
 // CHECK-NEXT:       // MIs[1] __Test0_match_0.x
 // CHECK-NEXT:       // No operand predicates
 // CHECK-NEXT:       GIM_CheckIsSafeToFold, /*InsnID*/1,
 // CHECK-NEXT:       GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s32,
-// CHECK-NEXT:       GIR_BuildConstant, /*TempRegID*/0, /*Val*/0,
+// CHECK-NEXT:       GIR_BuildConstant, /*TempRegID*/0, /*Val*/GIMT_Encode8(0),
 // CHECK-NEXT:       // Combiner Rule #0: Test0 @ [__Test0_match_0[0]]
-// CHECK-NEXT:       GIR_BuildMI, /*InsnID*/0, /*Opcode*/TargetOpcode::COPY,
+// CHECK-NEXT:       GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(TargetOpcode::COPY),
 // CHECK-NEXT:       GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // root
-// CHECK-NEXT:       GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/0,
+// CHECK-NEXT:       GIR_AddSimpleTempRegister, /*InsnID*/0, /*TempRegID*/0,
 // CHECK-NEXT:       GIR_EraseFromParent, /*InsnID*/0,
 // CHECK-NEXT:       GIR_Done,
-// CHECK-NEXT:     // Label 5: @126
+// CHECK-NEXT:     // Label 5: @343
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     // Label 2: @127
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 6*/ 151, // Rule ID 2 //
-// CHECK-NEXT:       GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule0Enabled,
+// CHECK-NEXT:     // Label 2: @344
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 6*/ GIMT_Encode4(379), // Rule ID 2 //
+// CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled),
 // CHECK-NEXT:       // MIs[0] root
 // CHECK-NEXT:       // No operand predicates
 // CHECK-NEXT:       // MIs[0] __Test0_match_0.z
 // CHECK-NEXT:       // No operand predicates
 // CHECK-NEXT:       GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s32,
-// CHECK-NEXT:       GIR_BuildConstant, /*TempRegID*/0, /*Val*/0,
+// CHECK-NEXT:       GIR_BuildConstant, /*TempRegID*/0, /*Val*/GIMT_Encode8(0),
 // CHECK-NEXT:       // Combiner Rule #0: Test0 @ [__Test0_match_0[2]]
-// CHECK-NEXT:       GIR_BuildMI, /*InsnID*/0, /*Opcode*/TargetOpcode::COPY,
+// CHECK-NEXT:       GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(TargetOpcode::COPY),
 // CHECK-NEXT:       GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // root
-// CHECK-NEXT:       GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/0,
+// CHECK-NEXT:       GIR_AddSimpleTempRegister, /*InsnID*/0, /*TempRegID*/0,
 // CHECK-NEXT:       GIR_EraseFromParent, /*InsnID*/0,
 // CHECK-NEXT:       GIR_Done,
-// CHECK-NEXT:     // Label 6: @151
+// CHECK-NEXT:     // Label 6: @379
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     // Label 3: @152
+// CHECK-NEXT:     // Label 3: @380
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     };
+// CHECK-NEXT:     }; // Size: 381 bytes
 // CHECK-NEXT:   return MatchTable0;
 // CHECK-NEXT: }
diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-permutations.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-permutations.td
index 22cd2be819de242df05f6f57b12bda016adb6fab..2b5cfb4f6de26630c70066beb896cffbedff5609 100644
--- a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-permutations.td
+++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-permutations.td
@@ -157,166 +157,166 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:   return false;
 // CHECK-NEXT: }
 
-// CHECK:      const int64_t *GenMyCombiner::getMatchTable() const {
-// CHECK-NEXT:   constexpr static int64_t MatchTable0[] = {
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 0*/ 682,
-// CHECK-NEXT:       GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_AND,
-// CHECK-NEXT:       GIM_Try, /*On fail goto*//*Label 1*/ 76, // Rule ID 7 //
-// CHECK-NEXT:         GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule0Enabled,
+// CHECK:      const uint8_t *GenMyCombiner::getMatchTable() const {
+// CHECK-NEXT:   constexpr static uint8_t MatchTable0[] = {
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 0*/ GIMT_Encode4(850),
+// CHECK-NEXT:       GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_AND),
+// CHECK-NEXT:       GIM_Try, /*On fail goto*//*Label 1*/ GIMT_Encode4(99), // Rule ID 7 //
+// CHECK-NEXT:         GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled),
 // CHECK-NEXT:         // MIs[0] dst
 // CHECK-NEXT:         // No operand predicates
 // CHECK-NEXT:         // MIs[0] cst0
 // CHECK-NEXT:         GIM_RecordInsnIgnoreCopies, /*DefineMI*/1, /*MI*/0, /*OpIdx*/1, // MIs[1]
-// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/1, TargetOpcode::G_TRUNC,
+// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/1, GIMT_Encode2(TargetOpcode::G_TRUNC),
 // CHECK-NEXT:         // MIs[1] a.z
 // CHECK-NEXT:         // No operand predicates
 // CHECK-NEXT:         // MIs[0] tmp
 // CHECK-NEXT:         GIM_RecordInsnIgnoreCopies, /*DefineMI*/2, /*MI*/0, /*OpIdx*/2, // MIs[2]
-// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/2, TargetOpcode::G_AND,
+// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/2, GIMT_Encode2(TargetOpcode::G_AND),
 // CHECK-NEXT:         // MIs[2] cst1
 // CHECK-NEXT:         GIM_RecordInsnIgnoreCopies, /*DefineMI*/3, /*MI*/2, /*OpIdx*/1, // MIs[3]
-// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/3, TargetOpcode::G_TRUNC,
+// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/3, GIMT_Encode2(TargetOpcode::G_TRUNC),
 // CHECK-NEXT:         // MIs[3] b.z
 // CHECK-NEXT:         // No operand predicates
 // CHECK-NEXT:         // MIs[2] cst2
 // CHECK-NEXT:         GIM_RecordInsnIgnoreCopies, /*DefineMI*/4, /*MI*/2, /*OpIdx*/2, // MIs[4]
-// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/4, TargetOpcode::G_TRUNC,
+// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/4, GIMT_Encode2(TargetOpcode::G_TRUNC),
 // CHECK-NEXT:         // MIs[4] c.z
 // CHECK-NEXT:         // No operand predicates
-// CHECK-NEXT:         GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GICXXPred_MI_Predicate_GICombiner21,
-// CHECK-NEXT:         GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GICXXPred_MI_Predicate_GICombiner22,
-// CHECK-NEXT:         GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GICXXPred_MI_Predicate_GICombiner23,
+// CHECK-NEXT:         GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_GICombiner21),
+// CHECK-NEXT:         GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_GICombiner22),
+// CHECK-NEXT:         GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_GICombiner23),
 // CHECK-NEXT:         GIM_CheckIsSafeToFold, /*InsnID*/1,
 // CHECK-NEXT:         GIM_CheckIsSafeToFold, /*InsnID*/2,
 // CHECK-NEXT:         GIM_CheckIsSafeToFold, /*InsnID*/3,
 // CHECK-NEXT:         GIM_CheckIsSafeToFold, /*InsnID*/4,
 // CHECK-NEXT:         GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s32,
-// CHECK-NEXT:         GIR_BuildConstant, /*TempRegID*/0, /*Val*/0,
+// CHECK-NEXT:         GIR_BuildConstant, /*TempRegID*/0, /*Val*/GIMT_Encode8(0),
 // CHECK-NEXT:         // Combiner Rule #0: Test0 @ [a[1], b[1], c[1]]
-// CHECK-NEXT:         GIR_BuildMI, /*InsnID*/0, /*Opcode*/TargetOpcode::COPY,
+// CHECK-NEXT:         GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(TargetOpcode::COPY),
 // CHECK-NEXT:         GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst
-// CHECK-NEXT:         GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/0,
+// CHECK-NEXT:         GIR_AddSimpleTempRegister, /*InsnID*/0, /*TempRegID*/0,
 // CHECK-NEXT:         GIR_EraseFromParent, /*InsnID*/0,
-// CHECK-NEXT:         GIR_CustomAction, GICXXCustomAction_CombineApplyGICombiner0,
+// CHECK-NEXT:         GIR_CustomAction, GIMT_Encode2(GICXXCustomAction_CombineApplyGICombiner0),
 // CHECK-NEXT:         GIR_Done,
-// CHECK-NEXT:       // Label 1: @76
-// CHECK-NEXT:       GIM_Try, /*On fail goto*//*Label 2*/ 156, // Rule ID 6 //
-// CHECK-NEXT:         GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule0Enabled,
+// CHECK-NEXT:       // Label 1: @99
+// CHECK-NEXT:       GIM_Try, /*On fail goto*//*Label 2*/ GIMT_Encode4(199), // Rule ID 6 //
+// CHECK-NEXT:         GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled),
 // CHECK-NEXT:         // MIs[0] dst
 // CHECK-NEXT:         // No operand predicates
 // CHECK-NEXT:         // MIs[0] cst0
 // CHECK-NEXT:         GIM_RecordInsnIgnoreCopies, /*DefineMI*/1, /*MI*/0, /*OpIdx*/1, // MIs[1]
-// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/1, TargetOpcode::G_TRUNC,
+// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/1, GIMT_Encode2(TargetOpcode::G_TRUNC),
 // CHECK-NEXT:         // MIs[1] a.z
 // CHECK-NEXT:         // No operand predicates
 // CHECK-NEXT:         // MIs[0] tmp
 // CHECK-NEXT:         GIM_RecordInsnIgnoreCopies, /*DefineMI*/2, /*MI*/0, /*OpIdx*/2, // MIs[2]
-// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/2, TargetOpcode::G_AND,
+// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/2, GIMT_Encode2(TargetOpcode::G_AND),
 // CHECK-NEXT:         // MIs[2] cst1
 // CHECK-NEXT:         GIM_RecordInsnIgnoreCopies, /*DefineMI*/3, /*MI*/2, /*OpIdx*/1, // MIs[3]
-// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/3, TargetOpcode::G_TRUNC,
+// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/3, GIMT_Encode2(TargetOpcode::G_TRUNC),
 // CHECK-NEXT:         // MIs[3] b.z
 // CHECK-NEXT:         // No operand predicates
 // CHECK-NEXT:         // MIs[2] cst2
 // CHECK-NEXT:         GIM_RecordInsnIgnoreCopies, /*DefineMI*/4, /*MI*/2, /*OpIdx*/2, // MIs[4]
-// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/4, TargetOpcode::G_ZEXT,
+// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/4, GIMT_Encode2(TargetOpcode::G_ZEXT),
 // CHECK-NEXT:         // MIs[4] c.b
 // CHECK-NEXT:         GIM_RecordInsnIgnoreCopies, /*DefineMI*/5, /*MI*/4, /*OpIdx*/1, // MIs[5]
-// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/5, TargetOpcode::G_TRUNC,
+// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/5, GIMT_Encode2(TargetOpcode::G_TRUNC),
 // CHECK-NEXT:         // MIs[5] c.x
 // CHECK-NEXT:         // No operand predicates
-// CHECK-NEXT:         GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GICXXPred_MI_Predicate_GICombiner18,
-// CHECK-NEXT:         GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GICXXPred_MI_Predicate_GICombiner19,
-// CHECK-NEXT:         GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GICXXPred_MI_Predicate_GICombiner20,
+// CHECK-NEXT:         GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_GICombiner18),
+// CHECK-NEXT:         GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_GICombiner19),
+// CHECK-NEXT:         GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_GICombiner20),
 // CHECK-NEXT:         GIM_CheckIsSafeToFold, /*InsnID*/1,
 // CHECK-NEXT:         GIM_CheckIsSafeToFold, /*InsnID*/2,
 // CHECK-NEXT:         GIM_CheckIsSafeToFold, /*InsnID*/3,
 // CHECK-NEXT:         GIM_CheckIsSafeToFold, /*InsnID*/4,
 // CHECK-NEXT:         GIM_CheckIsSafeToFold, /*InsnID*/5,
 // CHECK-NEXT:         GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s32,
-// CHECK-NEXT:         GIR_BuildConstant, /*TempRegID*/0, /*Val*/0,
+// CHECK-NEXT:         GIR_BuildConstant, /*TempRegID*/0, /*Val*/GIMT_Encode8(0),
 // CHECK-NEXT:         // Combiner Rule #0: Test0 @ [a[1], b[1], c[0]]
-// CHECK-NEXT:         GIR_BuildMI, /*InsnID*/0, /*Opcode*/TargetOpcode::COPY,
+// CHECK-NEXT:         GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(TargetOpcode::COPY),
 // CHECK-NEXT:         GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst
-// CHECK-NEXT:         GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/0,
+// CHECK-NEXT:         GIR_AddSimpleTempRegister, /*InsnID*/0, /*TempRegID*/0,
 // CHECK-NEXT:         GIR_EraseFromParent, /*InsnID*/0,
-// CHECK-NEXT:         GIR_CustomAction, GICXXCustomAction_CombineApplyGICombiner0,
+// CHECK-NEXT:         GIR_CustomAction, GIMT_Encode2(GICXXCustomAction_CombineApplyGICombiner0),
 // CHECK-NEXT:         GIR_Done,
-// CHECK-NEXT:       // Label 2: @156
-// CHECK-NEXT:       GIM_Try, /*On fail goto*//*Label 3*/ 236, // Rule ID 5 //
-// CHECK-NEXT:         GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule0Enabled,
+// CHECK-NEXT:       // Label 2: @199
+// CHECK-NEXT:       GIM_Try, /*On fail goto*//*Label 3*/ GIMT_Encode4(299), // Rule ID 5 //
+// CHECK-NEXT:         GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled),
 // CHECK-NEXT:         // MIs[0] dst
 // CHECK-NEXT:         // No operand predicates
 // CHECK-NEXT:         // MIs[0] cst0
 // CHECK-NEXT:         GIM_RecordInsnIgnoreCopies, /*DefineMI*/1, /*MI*/0, /*OpIdx*/1, // MIs[1]
-// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/1, TargetOpcode::G_TRUNC,
+// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/1, GIMT_Encode2(TargetOpcode::G_TRUNC),
 // CHECK-NEXT:         // MIs[1] a.z
 // CHECK-NEXT:         // No operand predicates
 // CHECK-NEXT:         // MIs[0] tmp
 // CHECK-NEXT:         GIM_RecordInsnIgnoreCopies, /*DefineMI*/2, /*MI*/0, /*OpIdx*/2, // MIs[2]
-// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/2, TargetOpcode::G_AND,
+// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/2, GIMT_Encode2(TargetOpcode::G_AND),
 // CHECK-NEXT:         // MIs[2] cst1
 // CHECK-NEXT:         GIM_RecordInsnIgnoreCopies, /*DefineMI*/3, /*MI*/2, /*OpIdx*/1, // MIs[3]
-// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/3, TargetOpcode::G_ZEXT,
+// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/3, GIMT_Encode2(TargetOpcode::G_ZEXT),
 // CHECK-NEXT:         // MIs[3] b.b
 // CHECK-NEXT:         GIM_RecordInsnIgnoreCopies, /*DefineMI*/4, /*MI*/3, /*OpIdx*/1, // MIs[4]
-// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/4, TargetOpcode::G_TRUNC,
+// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/4, GIMT_Encode2(TargetOpcode::G_TRUNC),
 // CHECK-NEXT:         // MIs[4] b.x
 // CHECK-NEXT:         // No operand predicates
 // CHECK-NEXT:         // MIs[2] cst2
 // CHECK-NEXT:         GIM_RecordInsnIgnoreCopies, /*DefineMI*/5, /*MI*/2, /*OpIdx*/2, // MIs[5]
-// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/5, TargetOpcode::G_TRUNC,
+// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/5, GIMT_Encode2(TargetOpcode::G_TRUNC),
 // CHECK-NEXT:         // MIs[5] c.z
 // CHECK-NEXT:         // No operand predicates
-// CHECK-NEXT:         GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GICXXPred_MI_Predicate_GICombiner15,
-// CHECK-NEXT:         GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GICXXPred_MI_Predicate_GICombiner16,
-// CHECK-NEXT:         GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GICXXPred_MI_Predicate_GICombiner17,
+// CHECK-NEXT:         GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_GICombiner15),
+// CHECK-NEXT:         GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_GICombiner16),
+// CHECK-NEXT:         GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_GICombiner17),
 // CHECK-NEXT:         GIM_CheckIsSafeToFold, /*InsnID*/1,
 // CHECK-NEXT:         GIM_CheckIsSafeToFold, /*InsnID*/2,
 // CHECK-NEXT:         GIM_CheckIsSafeToFold, /*InsnID*/3,
 // CHECK-NEXT:         GIM_CheckIsSafeToFold, /*InsnID*/4,
 // CHECK-NEXT:         GIM_CheckIsSafeToFold, /*InsnID*/5,
 // CHECK-NEXT:         GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s32,
-// CHECK-NEXT:         GIR_BuildConstant, /*TempRegID*/0, /*Val*/0,
+// CHECK-NEXT:         GIR_BuildConstant, /*TempRegID*/0, /*Val*/GIMT_Encode8(0),
 // CHECK-NEXT:         // Combiner Rule #0: Test0 @ [a[1], b[0], c[1]]
-// CHECK-NEXT:         GIR_BuildMI, /*InsnID*/0, /*Opcode*/TargetOpcode::COPY,
+// CHECK-NEXT:         GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(TargetOpcode::COPY),
 // CHECK-NEXT:         GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst
-// CHECK-NEXT:         GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/0,
+// CHECK-NEXT:         GIR_AddSimpleTempRegister, /*InsnID*/0, /*TempRegID*/0,
 // CHECK-NEXT:         GIR_EraseFromParent, /*InsnID*/0,
-// CHECK-NEXT:         GIR_CustomAction, GICXXCustomAction_CombineApplyGICombiner0,
+// CHECK-NEXT:         GIR_CustomAction, GIMT_Encode2(GICXXCustomAction_CombineApplyGICombiner0),
 // CHECK-NEXT:         GIR_Done,
-// CHECK-NEXT:       // Label 3: @236
-// CHECK-NEXT:       GIM_Try, /*On fail goto*//*Label 4*/ 325, // Rule ID 4 //
-// CHECK-NEXT:         GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule0Enabled,
+// CHECK-NEXT:       // Label 3: @299
+// CHECK-NEXT:       GIM_Try, /*On fail goto*//*Label 4*/ GIMT_Encode4(409), // Rule ID 4 //
+// CHECK-NEXT:         GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled),
 // CHECK-NEXT:         // MIs[0] dst
 // CHECK-NEXT:         // No operand predicates
 // CHECK-NEXT:         // MIs[0] cst0
 // CHECK-NEXT:         GIM_RecordInsnIgnoreCopies, /*DefineMI*/1, /*MI*/0, /*OpIdx*/1, // MIs[1]
-// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/1, TargetOpcode::G_TRUNC,
+// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/1, GIMT_Encode2(TargetOpcode::G_TRUNC),
 // CHECK-NEXT:         // MIs[1] a.z
 // CHECK-NEXT:         // No operand predicates
 // CHECK-NEXT:         // MIs[0] tmp
 // CHECK-NEXT:         GIM_RecordInsnIgnoreCopies, /*DefineMI*/2, /*MI*/0, /*OpIdx*/2, // MIs[2]
-// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/2, TargetOpcode::G_AND,
+// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/2, GIMT_Encode2(TargetOpcode::G_AND),
 // CHECK-NEXT:         // MIs[2] cst1
 // CHECK-NEXT:         GIM_RecordInsnIgnoreCopies, /*DefineMI*/3, /*MI*/2, /*OpIdx*/1, // MIs[3]
-// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/3, TargetOpcode::G_ZEXT,
+// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/3, GIMT_Encode2(TargetOpcode::G_ZEXT),
 // CHECK-NEXT:         // MIs[3] b.b
 // CHECK-NEXT:         GIM_RecordInsnIgnoreCopies, /*DefineMI*/4, /*MI*/3, /*OpIdx*/1, // MIs[4]
-// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/4, TargetOpcode::G_TRUNC,
+// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/4, GIMT_Encode2(TargetOpcode::G_TRUNC),
 // CHECK-NEXT:         // MIs[4] b.x
 // CHECK-NEXT:         // No operand predicates
 // CHECK-NEXT:         // MIs[2] cst2
 // CHECK-NEXT:         GIM_RecordInsnIgnoreCopies, /*DefineMI*/5, /*MI*/2, /*OpIdx*/2, // MIs[5]
-// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/5, TargetOpcode::G_ZEXT,
+// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/5, GIMT_Encode2(TargetOpcode::G_ZEXT),
 // CHECK-NEXT:         // MIs[5] c.b
 // CHECK-NEXT:         GIM_RecordInsnIgnoreCopies, /*DefineMI*/6, /*MI*/5, /*OpIdx*/1, // MIs[6]
-// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/6, TargetOpcode::G_TRUNC,
+// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/6, GIMT_Encode2(TargetOpcode::G_TRUNC),
 // CHECK-NEXT:         // MIs[6] c.x
 // CHECK-NEXT:         // No operand predicates
-// CHECK-NEXT:         GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GICXXPred_MI_Predicate_GICombiner12,
-// CHECK-NEXT:         GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GICXXPred_MI_Predicate_GICombiner13,
-// CHECK-NEXT:         GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GICXXPred_MI_Predicate_GICombiner14,
+// CHECK-NEXT:         GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_GICombiner12),
+// CHECK-NEXT:         GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_GICombiner13),
+// CHECK-NEXT:         GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_GICombiner14),
 // CHECK-NEXT:         GIM_CheckIsSafeToFold, /*InsnID*/1,
 // CHECK-NEXT:         GIM_CheckIsSafeToFold, /*InsnID*/2,
 // CHECK-NEXT:         GIM_CheckIsSafeToFold, /*InsnID*/3,
@@ -324,89 +324,89 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:         GIM_CheckIsSafeToFold, /*InsnID*/5,
 // CHECK-NEXT:         GIM_CheckIsSafeToFold, /*InsnID*/6,
 // CHECK-NEXT:         GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s32,
-// CHECK-NEXT:         GIR_BuildConstant, /*TempRegID*/0, /*Val*/0,
+// CHECK-NEXT:         GIR_BuildConstant, /*TempRegID*/0, /*Val*/GIMT_Encode8(0),
 // CHECK-NEXT:         // Combiner Rule #0: Test0 @ [a[1], b[0], c[0]]
-// CHECK-NEXT:         GIR_BuildMI, /*InsnID*/0, /*Opcode*/TargetOpcode::COPY,
+// CHECK-NEXT:         GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(TargetOpcode::COPY),
 // CHECK-NEXT:         GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst
-// CHECK-NEXT:         GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/0,
+// CHECK-NEXT:         GIR_AddSimpleTempRegister, /*InsnID*/0, /*TempRegID*/0,
 // CHECK-NEXT:         GIR_EraseFromParent, /*InsnID*/0,
-// CHECK-NEXT:         GIR_CustomAction, GICXXCustomAction_CombineApplyGICombiner0,
+// CHECK-NEXT:         GIR_CustomAction, GIMT_Encode2(GICXXCustomAction_CombineApplyGICombiner0),
 // CHECK-NEXT:         GIR_Done,
-// CHECK-NEXT:       // Label 4: @325
-// CHECK-NEXT:       GIM_Try, /*On fail goto*//*Label 5*/ 405, // Rule ID 3 //
-// CHECK-NEXT:         GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule0Enabled,
+// CHECK-NEXT:       // Label 4: @409
+// CHECK-NEXT:       GIM_Try, /*On fail goto*//*Label 5*/ GIMT_Encode4(509), // Rule ID 3 //
+// CHECK-NEXT:         GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled),
 // CHECK-NEXT:         // MIs[0] dst
 // CHECK-NEXT:         // No operand predicates
 // CHECK-NEXT:         // MIs[0] cst0
 // CHECK-NEXT:         GIM_RecordInsnIgnoreCopies, /*DefineMI*/1, /*MI*/0, /*OpIdx*/1, // MIs[1]
-// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/1, TargetOpcode::G_ZEXT,
+// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/1, GIMT_Encode2(TargetOpcode::G_ZEXT),
 // CHECK-NEXT:         // MIs[1] a.b
 // CHECK-NEXT:         GIM_RecordInsnIgnoreCopies, /*DefineMI*/2, /*MI*/1, /*OpIdx*/1, // MIs[2]
-// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/2, TargetOpcode::G_TRUNC,
+// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/2, GIMT_Encode2(TargetOpcode::G_TRUNC),
 // CHECK-NEXT:         // MIs[2] a.x
 // CHECK-NEXT:         // No operand predicates
 // CHECK-NEXT:         // MIs[0] tmp
 // CHECK-NEXT:         GIM_RecordInsnIgnoreCopies, /*DefineMI*/3, /*MI*/0, /*OpIdx*/2, // MIs[3]
-// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/3, TargetOpcode::G_AND,
+// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/3, GIMT_Encode2(TargetOpcode::G_AND),
 // CHECK-NEXT:         // MIs[3] cst1
 // CHECK-NEXT:         GIM_RecordInsnIgnoreCopies, /*DefineMI*/4, /*MI*/3, /*OpIdx*/1, // MIs[4]
-// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/4, TargetOpcode::G_TRUNC,
+// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/4, GIMT_Encode2(TargetOpcode::G_TRUNC),
 // CHECK-NEXT:         // MIs[4] b.z
 // CHECK-NEXT:         // No operand predicates
 // CHECK-NEXT:         // MIs[3] cst2
 // CHECK-NEXT:         GIM_RecordInsnIgnoreCopies, /*DefineMI*/5, /*MI*/3, /*OpIdx*/2, // MIs[5]
-// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/5, TargetOpcode::G_TRUNC,
+// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/5, GIMT_Encode2(TargetOpcode::G_TRUNC),
 // CHECK-NEXT:         // MIs[5] c.z
 // CHECK-NEXT:         // No operand predicates
-// CHECK-NEXT:         GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GICXXPred_MI_Predicate_GICombiner9,
-// CHECK-NEXT:         GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GICXXPred_MI_Predicate_GICombiner10,
-// CHECK-NEXT:         GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GICXXPred_MI_Predicate_GICombiner11,
+// CHECK-NEXT:         GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_GICombiner9),
+// CHECK-NEXT:         GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_GICombiner10),
+// CHECK-NEXT:         GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_GICombiner11),
 // CHECK-NEXT:         GIM_CheckIsSafeToFold, /*InsnID*/1,
 // CHECK-NEXT:         GIM_CheckIsSafeToFold, /*InsnID*/2,
 // CHECK-NEXT:         GIM_CheckIsSafeToFold, /*InsnID*/3,
 // CHECK-NEXT:         GIM_CheckIsSafeToFold, /*InsnID*/4,
 // CHECK-NEXT:         GIM_CheckIsSafeToFold, /*InsnID*/5,
 // CHECK-NEXT:         GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s32,
-// CHECK-NEXT:         GIR_BuildConstant, /*TempRegID*/0, /*Val*/0,
+// CHECK-NEXT:         GIR_BuildConstant, /*TempRegID*/0, /*Val*/GIMT_Encode8(0),
 // CHECK-NEXT:         // Combiner Rule #0: Test0 @ [a[0], b[1], c[1]]
-// CHECK-NEXT:         GIR_BuildMI, /*InsnID*/0, /*Opcode*/TargetOpcode::COPY,
+// CHECK-NEXT:         GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(TargetOpcode::COPY),
 // CHECK-NEXT:         GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst
-// CHECK-NEXT:         GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/0,
+// CHECK-NEXT:         GIR_AddSimpleTempRegister, /*InsnID*/0, /*TempRegID*/0,
 // CHECK-NEXT:         GIR_EraseFromParent, /*InsnID*/0,
-// CHECK-NEXT:         GIR_CustomAction, GICXXCustomAction_CombineApplyGICombiner0,
+// CHECK-NEXT:         GIR_CustomAction, GIMT_Encode2(GICXXCustomAction_CombineApplyGICombiner0),
 // CHECK-NEXT:         GIR_Done,
-// CHECK-NEXT:       // Label 5: @405
-// CHECK-NEXT:       GIM_Try, /*On fail goto*//*Label 6*/ 494, // Rule ID 2 //
-// CHECK-NEXT:         GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule0Enabled,
+// CHECK-NEXT:       // Label 5: @509
+// CHECK-NEXT:       GIM_Try, /*On fail goto*//*Label 6*/ GIMT_Encode4(619), // Rule ID 2 //
+// CHECK-NEXT:         GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled),
 // CHECK-NEXT:         // MIs[0] dst
 // CHECK-NEXT:         // No operand predicates
 // CHECK-NEXT:         // MIs[0] cst0
 // CHECK-NEXT:         GIM_RecordInsnIgnoreCopies, /*DefineMI*/1, /*MI*/0, /*OpIdx*/1, // MIs[1]
-// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/1, TargetOpcode::G_ZEXT,
+// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/1, GIMT_Encode2(TargetOpcode::G_ZEXT),
 // CHECK-NEXT:         // MIs[1] a.b
 // CHECK-NEXT:         GIM_RecordInsnIgnoreCopies, /*DefineMI*/2, /*MI*/1, /*OpIdx*/1, // MIs[2]
-// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/2, TargetOpcode::G_TRUNC,
+// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/2, GIMT_Encode2(TargetOpcode::G_TRUNC),
 // CHECK-NEXT:         // MIs[2] a.x
 // CHECK-NEXT:         // No operand predicates
 // CHECK-NEXT:         // MIs[0] tmp
 // CHECK-NEXT:         GIM_RecordInsnIgnoreCopies, /*DefineMI*/3, /*MI*/0, /*OpIdx*/2, // MIs[3]
-// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/3, TargetOpcode::G_AND,
+// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/3, GIMT_Encode2(TargetOpcode::G_AND),
 // CHECK-NEXT:         // MIs[3] cst1
 // CHECK-NEXT:         GIM_RecordInsnIgnoreCopies, /*DefineMI*/4, /*MI*/3, /*OpIdx*/1, // MIs[4]
-// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/4, TargetOpcode::G_TRUNC,
+// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/4, GIMT_Encode2(TargetOpcode::G_TRUNC),
 // CHECK-NEXT:         // MIs[4] b.z
 // CHECK-NEXT:         // No operand predicates
 // CHECK-NEXT:         // MIs[3] cst2
 // CHECK-NEXT:         GIM_RecordInsnIgnoreCopies, /*DefineMI*/5, /*MI*/3, /*OpIdx*/2, // MIs[5]
-// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/5, TargetOpcode::G_ZEXT,
+// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/5, GIMT_Encode2(TargetOpcode::G_ZEXT),
 // CHECK-NEXT:         // MIs[5] c.b
 // CHECK-NEXT:         GIM_RecordInsnIgnoreCopies, /*DefineMI*/6, /*MI*/5, /*OpIdx*/1, // MIs[6]
-// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/6, TargetOpcode::G_TRUNC,
+// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/6, GIMT_Encode2(TargetOpcode::G_TRUNC),
 // CHECK-NEXT:         // MIs[6] c.x
 // CHECK-NEXT:         // No operand predicates
-// CHECK-NEXT:         GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GICXXPred_MI_Predicate_GICombiner6,
-// CHECK-NEXT:         GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GICXXPred_MI_Predicate_GICombiner7,
-// CHECK-NEXT:         GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GICXXPred_MI_Predicate_GICombiner8,
+// CHECK-NEXT:         GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_GICombiner6),
+// CHECK-NEXT:         GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_GICombiner7),
+// CHECK-NEXT:         GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_GICombiner8),
 // CHECK-NEXT:         GIM_CheckIsSafeToFold, /*InsnID*/1,
 // CHECK-NEXT:         GIM_CheckIsSafeToFold, /*InsnID*/2,
 // CHECK-NEXT:         GIM_CheckIsSafeToFold, /*InsnID*/3,
@@ -414,46 +414,46 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:         GIM_CheckIsSafeToFold, /*InsnID*/5,
 // CHECK-NEXT:         GIM_CheckIsSafeToFold, /*InsnID*/6,
 // CHECK-NEXT:         GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s32,
-// CHECK-NEXT:         GIR_BuildConstant, /*TempRegID*/0, /*Val*/0,
+// CHECK-NEXT:         GIR_BuildConstant, /*TempRegID*/0, /*Val*/GIMT_Encode8(0),
 // CHECK-NEXT:         // Combiner Rule #0: Test0 @ [a[0], b[1], c[0]]
-// CHECK-NEXT:         GIR_BuildMI, /*InsnID*/0, /*Opcode*/TargetOpcode::COPY,
+// CHECK-NEXT:         GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(TargetOpcode::COPY),
 // CHECK-NEXT:         GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst
-// CHECK-NEXT:         GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/0,
+// CHECK-NEXT:         GIR_AddSimpleTempRegister, /*InsnID*/0, /*TempRegID*/0,
 // CHECK-NEXT:         GIR_EraseFromParent, /*InsnID*/0,
-// CHECK-NEXT:         GIR_CustomAction, GICXXCustomAction_CombineApplyGICombiner0,
+// CHECK-NEXT:         GIR_CustomAction, GIMT_Encode2(GICXXCustomAction_CombineApplyGICombiner0),
 // CHECK-NEXT:         GIR_Done,
-// CHECK-NEXT:       // Label 6: @494
-// CHECK-NEXT:       GIM_Try, /*On fail goto*//*Label 7*/ 583, // Rule ID 1 //
-// CHECK-NEXT:         GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule0Enabled,
+// CHECK-NEXT:       // Label 6: @619
+// CHECK-NEXT:       GIM_Try, /*On fail goto*//*Label 7*/ GIMT_Encode4(729), // Rule ID 1 //
+// CHECK-NEXT:         GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled),
 // CHECK-NEXT:         // MIs[0] dst
 // CHECK-NEXT:         // No operand predicates
 // CHECK-NEXT:         // MIs[0] cst0
 // CHECK-NEXT:         GIM_RecordInsnIgnoreCopies, /*DefineMI*/1, /*MI*/0, /*OpIdx*/1, // MIs[1]
-// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/1, TargetOpcode::G_ZEXT,
+// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/1, GIMT_Encode2(TargetOpcode::G_ZEXT),
 // CHECK-NEXT:         // MIs[1] a.b
 // CHECK-NEXT:         GIM_RecordInsnIgnoreCopies, /*DefineMI*/2, /*MI*/1, /*OpIdx*/1, // MIs[2]
-// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/2, TargetOpcode::G_TRUNC,
+// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/2, GIMT_Encode2(TargetOpcode::G_TRUNC),
 // CHECK-NEXT:         // MIs[2] a.x
 // CHECK-NEXT:         // No operand predicates
 // CHECK-NEXT:         // MIs[0] tmp
 // CHECK-NEXT:         GIM_RecordInsnIgnoreCopies, /*DefineMI*/3, /*MI*/0, /*OpIdx*/2, // MIs[3]
-// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/3, TargetOpcode::G_AND,
+// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/3, GIMT_Encode2(TargetOpcode::G_AND),
 // CHECK-NEXT:         // MIs[3] cst1
 // CHECK-NEXT:         GIM_RecordInsnIgnoreCopies, /*DefineMI*/4, /*MI*/3, /*OpIdx*/1, // MIs[4]
-// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/4, TargetOpcode::G_ZEXT,
+// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/4, GIMT_Encode2(TargetOpcode::G_ZEXT),
 // CHECK-NEXT:         // MIs[4] b.b
 // CHECK-NEXT:         GIM_RecordInsnIgnoreCopies, /*DefineMI*/5, /*MI*/4, /*OpIdx*/1, // MIs[5]
-// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/5, TargetOpcode::G_TRUNC,
+// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/5, GIMT_Encode2(TargetOpcode::G_TRUNC),
 // CHECK-NEXT:         // MIs[5] b.x
 // CHECK-NEXT:         // No operand predicates
 // CHECK-NEXT:         // MIs[3] cst2
 // CHECK-NEXT:         GIM_RecordInsnIgnoreCopies, /*DefineMI*/6, /*MI*/3, /*OpIdx*/2, // MIs[6]
-// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/6, TargetOpcode::G_TRUNC,
+// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/6, GIMT_Encode2(TargetOpcode::G_TRUNC),
 // CHECK-NEXT:         // MIs[6] c.z
 // CHECK-NEXT:         // No operand predicates
-// CHECK-NEXT:         GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GICXXPred_MI_Predicate_GICombiner3,
-// CHECK-NEXT:         GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GICXXPred_MI_Predicate_GICombiner4,
-// CHECK-NEXT:         GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GICXXPred_MI_Predicate_GICombiner5,
+// CHECK-NEXT:         GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_GICombiner3),
+// CHECK-NEXT:         GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_GICombiner4),
+// CHECK-NEXT:         GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_GICombiner5),
 // CHECK-NEXT:         GIM_CheckIsSafeToFold, /*InsnID*/1,
 // CHECK-NEXT:         GIM_CheckIsSafeToFold, /*InsnID*/2,
 // CHECK-NEXT:         GIM_CheckIsSafeToFold, /*InsnID*/3,
@@ -461,49 +461,49 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:         GIM_CheckIsSafeToFold, /*InsnID*/5,
 // CHECK-NEXT:         GIM_CheckIsSafeToFold, /*InsnID*/6,
 // CHECK-NEXT:         GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s32,
-// CHECK-NEXT:         GIR_BuildConstant, /*TempRegID*/0, /*Val*/0,
+// CHECK-NEXT:         GIR_BuildConstant, /*TempRegID*/0, /*Val*/GIMT_Encode8(0),
 // CHECK-NEXT:         // Combiner Rule #0: Test0 @ [a[0], b[0], c[1]]
-// CHECK-NEXT:         GIR_BuildMI, /*InsnID*/0, /*Opcode*/TargetOpcode::COPY,
+// CHECK-NEXT:         GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(TargetOpcode::COPY),
 // CHECK-NEXT:         GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst
-// CHECK-NEXT:         GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/0,
+// CHECK-NEXT:         GIR_AddSimpleTempRegister, /*InsnID*/0, /*TempRegID*/0,
 // CHECK-NEXT:         GIR_EraseFromParent, /*InsnID*/0,
-// CHECK-NEXT:         GIR_CustomAction, GICXXCustomAction_CombineApplyGICombiner0,
+// CHECK-NEXT:         GIR_CustomAction, GIMT_Encode2(GICXXCustomAction_CombineApplyGICombiner0),
 // CHECK-NEXT:         GIR_Done,
-// CHECK-NEXT:       // Label 7: @583
-// CHECK-NEXT:       GIM_Try, /*On fail goto*//*Label 8*/ 681, // Rule ID 0 //
-// CHECK-NEXT:         GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule0Enabled,
+// CHECK-NEXT:       // Label 7: @729
+// CHECK-NEXT:       GIM_Try, /*On fail goto*//*Label 8*/ GIMT_Encode4(849), // Rule ID 0 //
+// CHECK-NEXT:         GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled),
 // CHECK-NEXT:         // MIs[0] dst
 // CHECK-NEXT:         // No operand predicates
 // CHECK-NEXT:         // MIs[0] cst0
 // CHECK-NEXT:         GIM_RecordInsnIgnoreCopies, /*DefineMI*/1, /*MI*/0, /*OpIdx*/1, // MIs[1]
-// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/1, TargetOpcode::G_ZEXT,
+// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/1, GIMT_Encode2(TargetOpcode::G_ZEXT),
 // CHECK-NEXT:         // MIs[1] a.b
 // CHECK-NEXT:         GIM_RecordInsnIgnoreCopies, /*DefineMI*/2, /*MI*/1, /*OpIdx*/1, // MIs[2]
-// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/2, TargetOpcode::G_TRUNC,
+// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/2, GIMT_Encode2(TargetOpcode::G_TRUNC),
 // CHECK-NEXT:         // MIs[2] a.x
 // CHECK-NEXT:         // No operand predicates
 // CHECK-NEXT:         // MIs[0] tmp
 // CHECK-NEXT:         GIM_RecordInsnIgnoreCopies, /*DefineMI*/3, /*MI*/0, /*OpIdx*/2, // MIs[3]
-// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/3, TargetOpcode::G_AND,
+// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/3, GIMT_Encode2(TargetOpcode::G_AND),
 // CHECK-NEXT:         // MIs[3] cst1
 // CHECK-NEXT:         GIM_RecordInsnIgnoreCopies, /*DefineMI*/4, /*MI*/3, /*OpIdx*/1, // MIs[4]
-// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/4, TargetOpcode::G_ZEXT,
+// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/4, GIMT_Encode2(TargetOpcode::G_ZEXT),
 // CHECK-NEXT:         // MIs[4] b.b
 // CHECK-NEXT:         GIM_RecordInsnIgnoreCopies, /*DefineMI*/5, /*MI*/4, /*OpIdx*/1, // MIs[5]
-// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/5, TargetOpcode::G_TRUNC,
+// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/5, GIMT_Encode2(TargetOpcode::G_TRUNC),
 // CHECK-NEXT:         // MIs[5] b.x
 // CHECK-NEXT:         // No operand predicates
 // CHECK-NEXT:         // MIs[3] cst2
 // CHECK-NEXT:         GIM_RecordInsnIgnoreCopies, /*DefineMI*/6, /*MI*/3, /*OpIdx*/2, // MIs[6]
-// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/6, TargetOpcode::G_ZEXT,
+// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/6, GIMT_Encode2(TargetOpcode::G_ZEXT),
 // CHECK-NEXT:         // MIs[6] c.b
 // CHECK-NEXT:         GIM_RecordInsnIgnoreCopies, /*DefineMI*/7, /*MI*/6, /*OpIdx*/1, // MIs[7]
-// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/7, TargetOpcode::G_TRUNC,
+// CHECK-NEXT:         GIM_CheckOpcode, /*MI*/7, GIMT_Encode2(TargetOpcode::G_TRUNC),
 // CHECK-NEXT:         // MIs[7] c.x
 // CHECK-NEXT:         // No operand predicates
-// CHECK-NEXT:         GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GICXXPred_MI_Predicate_GICombiner0,
-// CHECK-NEXT:         GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GICXXPred_MI_Predicate_GICombiner1,
-// CHECK-NEXT:         GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GICXXPred_MI_Predicate_GICombiner2,
+// CHECK-NEXT:         GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_GICombiner0),
+// CHECK-NEXT:         GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_GICombiner1),
+// CHECK-NEXT:         GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_GICombiner2),
 // CHECK-NEXT:         GIM_CheckIsSafeToFold, /*InsnID*/1,
 // CHECK-NEXT:         GIM_CheckIsSafeToFold, /*InsnID*/2,
 // CHECK-NEXT:         GIM_CheckIsSafeToFold, /*InsnID*/3,
@@ -512,18 +512,18 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:         GIM_CheckIsSafeToFold, /*InsnID*/6,
 // CHECK-NEXT:         GIM_CheckIsSafeToFold, /*InsnID*/7,
 // CHECK-NEXT:         GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s32,
-// CHECK-NEXT:         GIR_BuildConstant, /*TempRegID*/0, /*Val*/0,
+// CHECK-NEXT:         GIR_BuildConstant, /*TempRegID*/0, /*Val*/GIMT_Encode8(0),
 // CHECK-NEXT:         // Combiner Rule #0: Test0 @ [a[0], b[0], c[0]]
-// CHECK-NEXT:         GIR_BuildMI, /*InsnID*/0, /*Opcode*/TargetOpcode::COPY,
+// CHECK-NEXT:         GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(TargetOpcode::COPY),
 // CHECK-NEXT:         GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst
-// CHECK-NEXT:         GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/0,
+// CHECK-NEXT:         GIR_AddSimpleTempRegister, /*InsnID*/0, /*TempRegID*/0,
 // CHECK-NEXT:         GIR_EraseFromParent, /*InsnID*/0,
-// CHECK-NEXT:         GIR_CustomAction, GICXXCustomAction_CombineApplyGICombiner0,
+// CHECK-NEXT:         GIR_CustomAction, GIMT_Encode2(GICXXCustomAction_CombineApplyGICombiner0),
 // CHECK-NEXT:         GIR_Done,
-// CHECK-NEXT:       // Label 8: @681
+// CHECK-NEXT:       // Label 8: @849
 // CHECK-NEXT:       GIM_Reject,
-// CHECK-NEXT:     // Label 0: @682
+// CHECK-NEXT:     // Label 0: @850
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     };
+// CHECK-NEXT:     }; // Size: 851 bytes
 // CHECK-NEXT:   return MatchTable0;
 // CHECK-NEXT: }
diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-typeof.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-typeof.td
index 496d86aeef2d10a98f78303b7f2ed95083ef4603..3cffee7ab581f92dd07ce25afec9329241bbc064 100644
--- a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-typeof.td
+++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-typeof.td
@@ -14,33 +14,33 @@ def Test0 : GICombineRule<
   (apply (G_SUB $dst, (GITypeOf<"$src"> 0), $tmp),
          (G_CONSTANT GITypeOf<"$dst">:$tmp, (GITypeOf<"$src"> 42)))>;
 
-// CHECK:      const int64_t *GenMyCombiner::getMatchTable() const {
-// CHECK-NEXT:   constexpr static int64_t MatchTable0[] = {
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 0*/ 57, // Rule ID 0 //
-// CHECK-NEXT:       GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule0Enabled,
-// CHECK-NEXT:       GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_MUL,
+// CHECK:      const uint8_t *GenMyCombiner::getMatchTable() const {
+// CHECK-NEXT:   constexpr static uint8_t MatchTable0[] = {
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 0*/ GIMT_Encode4(75), // Rule ID 0 //
+// CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled),
+// CHECK-NEXT:       GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_MUL),
 // CHECK-NEXT:       // MIs[0] dst
-// CHECK-NEXT:       GIM_RecordRegType, /*MI*/0, /*Op*/0, /*TempTypeIdx*/-1,
+// CHECK-NEXT:       GIM_RecordRegType, /*MI*/0, /*Op*/0, /*TempTypeIdx*/uint8_t(-1),
 // CHECK-NEXT:       // MIs[0] src
-// CHECK-NEXT:       GIM_RecordRegType, /*MI*/0, /*Op*/1, /*TempTypeIdx*/-2,
+// CHECK-NEXT:       GIM_RecordRegType, /*MI*/0, /*Op*/1, /*TempTypeIdx*/uint8_t(-2),
 // CHECK-NEXT:       // MIs[0] Operand 2
-// CHECK-NEXT:       GIM_CheckConstantInt, /*MI*/0, /*Op*/2, -1,
-// CHECK-NEXT:       GIR_MakeTempReg, /*TempRegID*/1, /*TypeID*/-2,
-// CHECK-NEXT:       GIR_BuildConstant, /*TempRegID*/1, /*Val*/0,
-// CHECK-NEXT:       GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/-1,
+// CHECK-NEXT:       GIM_CheckConstantInt8, /*MI*/0, /*Op*/2, uint8_t(-1),
+// CHECK-NEXT:       GIR_MakeTempReg, /*TempRegID*/1, /*TypeID*/uint8_t(-2),
+// CHECK-NEXT:       GIR_BuildConstant, /*TempRegID*/1, /*Val*/GIMT_Encode8(0),
+// CHECK-NEXT:       GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/uint8_t(-1),
 // CHECK-NEXT:       // Combiner Rule #0: Test0
-// CHECK-NEXT:       GIR_BuildMI, /*InsnID*/0, /*Opcode*/TargetOpcode::G_CONSTANT,
-// CHECK-NEXT:       GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/0,
-// CHECK-NEXT:       GIR_AddCImm, /*InsnID*/0, /*Type*/-2, /*Imm*/42,
+// CHECK-NEXT:       GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(TargetOpcode::G_CONSTANT),
+// CHECK-NEXT:       GIR_AddSimpleTempRegister, /*InsnID*/0, /*TempRegID*/0,
+// CHECK-NEXT:       GIR_AddCImm, /*InsnID*/0, /*Type*/uint8_t(-2), /*Imm*/GIMT_Encode8(42),
 // CHECK-NEXT:       GIR_EraseFromParent, /*InsnID*/0,
-// CHECK-NEXT:       GIR_BuildMI, /*InsnID*/1, /*Opcode*/TargetOpcode::G_SUB,
+// CHECK-NEXT:       GIR_BuildMI, /*InsnID*/1, /*Opcode*/GIMT_Encode2(TargetOpcode::G_SUB),
 // CHECK-NEXT:       GIR_Copy, /*NewInsnID*/1, /*OldInsnID*/0, /*OpIdx*/0, // dst
-// CHECK-NEXT:       GIR_AddTempRegister, /*InsnID*/1, /*TempRegID*/1, /*TempRegFlags*/0,
-// CHECK-NEXT:       GIR_AddTempRegister, /*InsnID*/1, /*TempRegID*/0, /*TempRegFlags*/0,
+// CHECK-NEXT:       GIR_AddSimpleTempRegister, /*InsnID*/1, /*TempRegID*/1,
+// CHECK-NEXT:       GIR_AddSimpleTempRegister, /*InsnID*/1, /*TempRegID*/0,
 // CHECK-NEXT:       GIR_Done,
-// CHECK-NEXT:     // Label 0: @57
+// CHECK-NEXT:     // Label 0: @75
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     };
+// CHECK-NEXT:     }; // Size: 76 bytes
 // CHECK-NEXT:   return MatchTable0;
 // CHECK-NEXT: }
 
diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-variadics.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-variadics.td
index 5226795cd9d352661c0942adc201949689ad03fc..cc77bfdd29c38ec6f7cedb71e3fce195525b49b1 100644
--- a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-variadics.td
+++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-variadics.td
@@ -35,25 +35,25 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
   InstTest3
 ]>;
 
-// CHECK:      const int64_t *GenMyCombiner::getMatchTable() const {
-// CHECK-NEXT:   constexpr static int64_t MatchTable0[] = {
-// CHECK-NEXT:     GIM_SwitchOpcode, /*MI*/0, /*[*/65, 69, /*)*//*default:*//*Label 2*/ 51,
-// CHECK-NEXT:     /*TargetOpcode::G_UNMERGE_VALUES*//*Label 0*/ 9, 0, 0,
-// CHECK-NEXT:     /*TargetOpcode::G_BUILD_VECTOR*//*Label 1*/ 30,
-// CHECK-NEXT:     // Label 0: @9
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 3*/ 19, // Rule ID 2 //
-// CHECK-NEXT:       GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule2Enabled,
+// CHECK:      const uint8_t *GenMyCombiner::getMatchTable() const {
+// CHECK-NEXT:   constexpr static uint8_t MatchTable0[] = {
+// CHECK-NEXT:     GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(65), GIMT_Encode2(69), /*)*//*default:*//*Label 2*/ GIMT_Encode4(88),
+// CHECK-NEXT:     /*TargetOpcode::G_UNMERGE_VALUES*//*Label 0*/ GIMT_Encode4(26), GIMT_Encode4(0), GIMT_Encode4(0),
+// CHECK-NEXT:     /*TargetOpcode::G_BUILD_VECTOR*//*Label 1*/ GIMT_Encode4(57),
+// CHECK-NEXT:     // Label 0: @26
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 3*/ GIMT_Encode4(41), // Rule ID 2 //
+// CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule2Enabled),
 // CHECK-NEXT:       GIM_CheckNumOperands, /*MI*/0, /*Expected*/2,
 // CHECK-NEXT:       // MIs[0] a
 // CHECK-NEXT:       // No operand predicates
 // CHECK-NEXT:       // MIs[0] b
 // CHECK-NEXT:       // No operand predicates
 // CHECK-NEXT:       // Combiner Rule #2: InstTest2
-// CHECK-NEXT:       GIR_CustomAction, GICXXCustomAction_CombineApplyGICombiner0,
+// CHECK-NEXT:       GIR_CustomAction, GIMT_Encode2(GICXXCustomAction_CombineApplyGICombiner0),
 // CHECK-NEXT:       GIR_Done,
-// CHECK-NEXT:     // Label 3: @19
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 4*/ 29, // Rule ID 3 //
-// CHECK-NEXT:       GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule3Enabled,
+// CHECK-NEXT:     // Label 3: @41
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 4*/ GIMT_Encode4(56), // Rule ID 3 //
+// CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule3Enabled),
 // CHECK-NEXT:       GIM_CheckNumOperands, /*MI*/0, /*Expected*/4,
 // CHECK-NEXT:       // MIs[0] a
 // CHECK-NEXT:       // No operand predicates
@@ -64,24 +64,24 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:       // MIs[0] d
 // CHECK-NEXT:       // No operand predicates
 // CHECK-NEXT:       // Combiner Rule #3: InstTest3
-// CHECK-NEXT:       GIR_CustomAction, GICXXCustomAction_CombineApplyGICombiner0,
+// CHECK-NEXT:       GIR_CustomAction, GIMT_Encode2(GICXXCustomAction_CombineApplyGICombiner0),
 // CHECK-NEXT:       GIR_Done,
-// CHECK-NEXT:     // Label 4: @29
+// CHECK-NEXT:     // Label 4: @56
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     // Label 1: @30
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 5*/ 40, // Rule ID 1 //
-// CHECK-NEXT:       GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule1Enabled,
+// CHECK-NEXT:     // Label 1: @57
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 5*/ GIMT_Encode4(72), // Rule ID 1 //
+// CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule1Enabled),
 // CHECK-NEXT:       GIM_CheckNumOperands, /*MI*/0, /*Expected*/2,
 // CHECK-NEXT:       // MIs[0] a
 // CHECK-NEXT:       // No operand predicates
 // CHECK-NEXT:       // MIs[0] b
 // CHECK-NEXT:       // No operand predicates
 // CHECK-NEXT:       // Combiner Rule #1: InstTest1
-// CHECK-NEXT:       GIR_CustomAction, GICXXCustomAction_CombineApplyGICombiner0,
+// CHECK-NEXT:       GIR_CustomAction, GIMT_Encode2(GICXXCustomAction_CombineApplyGICombiner0),
 // CHECK-NEXT:       GIR_Done,
-// CHECK-NEXT:     // Label 5: @40
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 6*/ 50, // Rule ID 0 //
-// CHECK-NEXT:       GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule0Enabled,
+// CHECK-NEXT:     // Label 5: @72
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 6*/ GIMT_Encode4(87), // Rule ID 0 //
+// CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled),
 // CHECK-NEXT:       GIM_CheckNumOperands, /*MI*/0, /*Expected*/4,
 // CHECK-NEXT:       // MIs[0] a
 // CHECK-NEXT:       // No operand predicates
@@ -92,11 +92,11 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT:       // MIs[0] d
 // CHECK-NEXT:       // No operand predicates
 // CHECK-NEXT:       // Combiner Rule #0: InstTest0
-// CHECK-NEXT:       GIR_CustomAction, GICXXCustomAction_CombineApplyGICombiner0,
+// CHECK-NEXT:       GIR_CustomAction, GIMT_Encode2(GICXXCustomAction_CombineApplyGICombiner0),
 // CHECK-NEXT:       GIR_Done,
-// CHECK-NEXT:     // Label 6: @50
+// CHECK-NEXT:     // Label 6: @87
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     // Label 2: @51
+// CHECK-NEXT:     // Label 2: @88
 // CHECK-NEXT:     GIM_Reject,
 // CHECK-NEXT:     };
 // CHECK-NEXT:   return MatchTable0;
diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td
index 6777089f846f2dcde7fdb739e018e6356831b339..61dbbac745fa0af4939161bf3bb7f05ff4a01d98 100644
--- a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td
+++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td
@@ -130,127 +130,127 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
 // CHECK-NEXT: }
 
 // Verify match table.
-// CHECK:      const int64_t *GenMyCombiner::getMatchTable() const {
-// CHECK-NEXT:   constexpr static int64_t MatchTable0[] = {
-// CHECK-NEXT:     GIM_SwitchOpcode, /*MI*/0, /*[*/19, 127, /*)*//*default:*//*Label 6*/ 268,
-// CHECK-NEXT:     /*TargetOpcode::COPY*//*Label 0*/ 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-// CHECK-NEXT:     /*TargetOpcode::G_AND*//*Label 1*/ 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-// CHECK-NEXT:     /*TargetOpcode::G_STORE*//*Label 2*/ 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-// CHECK-NEXT:     /*TargetOpcode::G_TRUNC*//*Label 3*/ 217, 0, 0, 0, 0,
-// CHECK-NEXT:     /*TargetOpcode::G_SEXT*//*Label 4*/ 232, 0,
-// CHECK-NEXT:     /*TargetOpcode::G_ZEXT*//*Label 5*/ 240,
-// CHECK-NEXT:     // Label 0: @113
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 7*/ 134, // Rule ID 4 //
-// CHECK-NEXT:       GIM_CheckFeatures, GIFBS_HasAnswerToEverything,
-// CHECK-NEXT:       GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule3Enabled,
+// CHECK:      const uint8_t *GenMyCombiner::getMatchTable() const {
+// CHECK-NEXT:   constexpr static uint8_t MatchTable0[] = {
+// CHECK-NEXT:     GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2(19), GIMT_Encode2(127), /*)*//*default:*//*Label 6*/ GIMT_Encode4(657),
+// CHECK-NEXT:     /*TargetOpcode::COPY*//*Label 0*/ GIMT_Encode4(442), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
+// CHECK-NEXT:     /*TargetOpcode::G_AND*//*Label 1*/ GIMT_Encode4(484), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
+// CHECK-NEXT:     /*TargetOpcode::G_STORE*//*Label 2*/ GIMT_Encode4(537), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
+// CHECK-NEXT:     /*TargetOpcode::G_TRUNC*//*Label 3*/ GIMT_Encode4(579), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0), GIMT_Encode4(0),
+// CHECK-NEXT:     /*TargetOpcode::G_SEXT*//*Label 4*/ GIMT_Encode4(604), GIMT_Encode4(0),
+// CHECK-NEXT:     /*TargetOpcode::G_ZEXT*//*Label 5*/ GIMT_Encode4(617),
+// CHECK-NEXT:     // Label 0: @442
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 7*/ GIMT_Encode4(471), // Rule ID 4 //
+// CHECK-NEXT:       GIM_CheckFeatures, GIMT_Encode2(GIFBS_HasAnswerToEverything),
+// CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule3Enabled),
 // CHECK-NEXT:       // MIs[0] a
 // CHECK-NEXT:       // No operand predicates
 // CHECK-NEXT:       // MIs[0] b
 // CHECK-NEXT:       GIM_RecordInsnIgnoreCopies, /*DefineMI*/1, /*MI*/0, /*OpIdx*/1, // MIs[1]
-// CHECK-NEXT:       GIM_CheckOpcode, /*MI*/1, TargetOpcode::G_ZEXT,
+// CHECK-NEXT:       GIM_CheckOpcode, /*MI*/1, GIMT_Encode2(TargetOpcode::G_ZEXT),
 // CHECK-NEXT:       // MIs[1] c
 // CHECK-NEXT:       // No operand predicates
-// CHECK-NEXT:       GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GICXXPred_MI_Predicate_GICombiner0,
+// CHECK-NEXT:       GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_GICombiner0),
 // CHECK-NEXT:       GIM_CheckIsSafeToFold, /*InsnID*/1,
 // CHECK-NEXT:       // Combiner Rule #3: InstTest1
-// CHECK-NEXT:       GIR_CustomAction, GICXXCustomAction_CombineApplyGICombiner0,
+// CHECK-NEXT:       GIR_CustomAction, GIMT_Encode2(GICXXCustomAction_CombineApplyGICombiner0),
 // CHECK-NEXT:       GIR_Done,
-// CHECK-NEXT:     // Label 7: @134
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 8*/ 141, // Rule ID 3 //
-// CHECK-NEXT:       GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule2Enabled,
+// CHECK-NEXT:     // Label 7: @471
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 8*/ GIMT_Encode4(483), // Rule ID 3 //
+// CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule2Enabled),
 // CHECK-NEXT:       // MIs[0] a
 // CHECK-NEXT:       // No operand predicates
 // CHECK-NEXT:       // MIs[0] b
 // CHECK-NEXT:       // No operand predicates
 // CHECK-NEXT:       // Combiner Rule #2: InstTest0
-// CHECK-NEXT:       GIR_CustomAction, GICXXCustomAction_CombineApplyGICombiner1,
+// CHECK-NEXT:       GIR_CustomAction, GIMT_Encode2(GICXXCustomAction_CombineApplyGICombiner1),
 // CHECK-NEXT:       GIR_Done,
-// CHECK-NEXT:     // Label 8: @141
+// CHECK-NEXT:     // Label 8: @483
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     // Label 1: @142
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 9*/ 181, // Rule ID 6 //
-// CHECK-NEXT:       GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule5Enabled,
+// CHECK-NEXT:     // Label 1: @484
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 9*/ GIMT_Encode4(536), // Rule ID 6 //
+// CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule5Enabled),
 // CHECK-NEXT:       GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32,
 // CHECK-NEXT:       // MIs[0] dst
 // CHECK-NEXT:       // No operand predicates
 // CHECK-NEXT:       // MIs[0] x
 // CHECK-NEXT:       GIM_RecordInsnIgnoreCopies, /*DefineMI*/1, /*MI*/0, /*OpIdx*/1, // MIs[1]
-// CHECK-NEXT:       GIM_CheckOpcode, /*MI*/1, TargetOpcode::G_CONSTANT,
+// CHECK-NEXT:       GIM_CheckOpcode, /*MI*/1, GIMT_Encode2(TargetOpcode::G_CONSTANT),
 // CHECK-NEXT:       // MIs[1] z
-// CHECK-NEXT:       GIM_CheckLiteralInt, /*MI*/1, /*Op*/1, -42,
-// CHECK-NEXT:       GIM_CheckConstantInt, /*MI*/0, /*Op*/2, 43,
+// CHECK-NEXT:       GIM_CheckLiteralInt, /*MI*/1, /*Op*/1, GIMT_Encode8(-42),
+// CHECK-NEXT:       GIM_CheckConstantInt8, /*MI*/0, /*Op*/2, 43,
 // CHECK-NEXT:       GIM_CheckIsSafeToFold, /*InsnID*/1,
 // CHECK-NEXT:       // Combiner Rule #5: InOutInstTest1
-// CHECK-NEXT:       GIR_BuildMI, /*InsnID*/0, /*Opcode*/TargetOpcode::G_TRUNC,
+// CHECK-NEXT:       GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(TargetOpcode::G_TRUNC),
 // CHECK-NEXT:       GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst
 // CHECK-NEXT:       GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/1, /*OpIdx*/1, // z
 // CHECK-NEXT:       GIR_EraseFromParent, /*InsnID*/0,
 // CHECK-NEXT:       GIR_Done,
-// CHECK-NEXT:     // Label 9: @181
+// CHECK-NEXT:     // Label 9: @536
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     // Label 2: @182
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 10*/ 216, // Rule ID 5 //
-// CHECK-NEXT:       GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule4Enabled,
+// CHECK-NEXT:     // Label 2: @537
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 10*/ GIMT_Encode4(578), // Rule ID 5 //
+// CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule4Enabled),
 // CHECK-NEXT:       // MIs[0] tmp
 // CHECK-NEXT:       GIM_RecordInsnIgnoreCopies, /*DefineMI*/1, /*MI*/0, /*OpIdx*/0, // MIs[1]
-// CHECK-NEXT:       GIM_CheckOpcode, /*MI*/1, TargetOpcode::G_ZEXT,
+// CHECK-NEXT:       GIM_CheckOpcode, /*MI*/1, GIMT_Encode2(TargetOpcode::G_ZEXT),
 // CHECK-NEXT:       // MIs[1] ext
 // CHECK-NEXT:       // No operand predicates
 // CHECK-NEXT:       // MIs[0] ptr
 // CHECK-NEXT:       // No operand predicates
 // CHECK-NEXT:       GIM_CheckIsSafeToFold, /*InsnID*/1,
 // CHECK-NEXT:       // Combiner Rule #4: InOutInstTest0
-// CHECK-NEXT:       GIR_BuildMI, /*InsnID*/0, /*Opcode*/TargetOpcode::G_STORE,
+// CHECK-NEXT:       GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(TargetOpcode::G_STORE),
 // CHECK-NEXT:       GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/1, /*OpIdx*/1, // ext
 // CHECK-NEXT:       GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // ptr
-// CHECK-NEXT:       GIR_MergeMemOperands, /*InsnID*/0, /*MergeInsnID's*/0, 1, GIU_MergeMemOperands_EndOfList,
+// CHECK-NEXT:       GIR_MergeMemOperands, /*InsnID*/0, /*NumInsns*/2, /*MergeInsnID's*/0, 1,
 // CHECK-NEXT:       GIR_EraseFromParent, /*InsnID*/0,
-// CHECK-NEXT:       GIR_CustomAction, GICXXCustomAction_CombineApplyGICombiner2,
+// CHECK-NEXT:       GIR_CustomAction, GIMT_Encode2(GICXXCustomAction_CombineApplyGICombiner2),
 // CHECK-NEXT:       GIR_Done,
-// CHECK-NEXT:     // Label 10: @216
+// CHECK-NEXT:     // Label 10: @578
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     // Label 3: @217
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 11*/ 224, // Rule ID 0 //
-// CHECK-NEXT:       GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule0Enabled,
+// CHECK-NEXT:     // Label 3: @579
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 11*/ GIMT_Encode4(591), // Rule ID 0 //
+// CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled),
 // CHECK-NEXT:       // Combiner Rule #0: WipOpcodeTest0; wip_match_opcode 'G_TRUNC'
-// CHECK-NEXT:       GIR_CustomAction, GICXXCustomAction_CombineApplyGICombiner0,
+// CHECK-NEXT:       GIR_CustomAction, GIMT_Encode2(GICXXCustomAction_CombineApplyGICombiner0),
 // CHECK-NEXT:       GIR_Done,
-// CHECK-NEXT:     // Label 11: @224
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 12*/ 231, // Rule ID 1 //
-// CHECK-NEXT:       GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule1Enabled,
+// CHECK-NEXT:     // Label 11: @591
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 12*/ GIMT_Encode4(603), // Rule ID 1 //
+// CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule1Enabled),
 // CHECK-NEXT:       // Combiner Rule #1: WipOpcodeTest1; wip_match_opcode 'G_TRUNC'
-// CHECK-NEXT:       GIR_CustomAction, GICXXCustomAction_CombineApplyGICombiner0,
+// CHECK-NEXT:       GIR_CustomAction, GIMT_Encode2(GICXXCustomAction_CombineApplyGICombiner0),
 // CHECK-NEXT:       GIR_Done,
-// CHECK-NEXT:     // Label 12: @231
+// CHECK-NEXT:     // Label 12: @603
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     // Label 4: @232
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 13*/ 239, // Rule ID 2 //
-// CHECK-NEXT:       GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule1Enabled,
+// CHECK-NEXT:     // Label 4: @604
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 13*/ GIMT_Encode4(616), // Rule ID 2 //
+// CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule1Enabled),
 // CHECK-NEXT:       // Combiner Rule #1: WipOpcodeTest1; wip_match_opcode 'G_SEXT'
-// CHECK-NEXT:       GIR_CustomAction, GICXXCustomAction_CombineApplyGICombiner0,
+// CHECK-NEXT:       GIR_CustomAction, GIMT_Encode2(GICXXCustomAction_CombineApplyGICombiner0),
 // CHECK-NEXT:       GIR_Done,
-// CHECK-NEXT:     // Label 13: @239
+// CHECK-NEXT:     // Label 13: @616
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     // Label 5: @240
-// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 14*/ 267, // Rule ID 7 //
-// CHECK-NEXT:       GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule6Enabled,
+// CHECK-NEXT:     // Label 5: @617
+// CHECK-NEXT:     GIM_Try, /*On fail goto*//*Label 14*/ GIMT_Encode4(656), // Rule ID 7 //
+// CHECK-NEXT:       GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule6Enabled),
 // CHECK-NEXT:       // MIs[0] dst
 // CHECK-NEXT:       // No operand predicates
 // CHECK-NEXT:       // MIs[0] cst
 // CHECK-NEXT:       // No operand predicates
-// CHECK-NEXT:       GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GICXXPred_MI_Predicate_GICombiner1,
+// CHECK-NEXT:       GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_GICombiner1),
 // CHECK-NEXT:       GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s32,
-// CHECK-NEXT:       GIR_BuildConstant, /*TempRegID*/0, /*Val*/0,
+// CHECK-NEXT:       GIR_BuildConstant, /*TempRegID*/0, /*Val*/GIMT_Encode8(0),
 // CHECK-NEXT:       // Combiner Rule #6: PatFragTest0 @ [__PatFragTest0_match_1[0]]
-// CHECK-NEXT:       GIR_BuildMI, /*InsnID*/0, /*Opcode*/TargetOpcode::COPY,
+// CHECK-NEXT:       GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(TargetOpcode::COPY),
 // CHECK-NEXT:       GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst
-// CHECK-NEXT:       GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/0,
+// CHECK-NEXT:       GIR_AddSimpleTempRegister, /*InsnID*/0, /*TempRegID*/0,
 // CHECK-NEXT:       GIR_EraseFromParent, /*InsnID*/0,
 // CHECK-NEXT:       GIR_Done,
-// CHECK-NEXT:     // Label 14: @267
+// CHECK-NEXT:     // Label 14: @656
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     // Label 6: @268
+// CHECK-NEXT:     // Label 6: @657
 // CHECK-NEXT:     GIM_Reject,
-// CHECK-NEXT:     }; // Size: 2152 bytes
+// CHECK-NEXT:     }; // Size: 658 bytes
 // CHECK-NEXT:   return MatchTable0;
 // CHECK-NEXT: }
diff --git a/llvm/test/TableGen/GlobalISelEmitter-PR39045.td b/llvm/test/TableGen/GlobalISelEmitter-PR39045.td
index 55e5aa2e905d6b1e951c4381d4947936b4488f57..5407222121bb3adb017642d608c13dcf3a087ece 100644
--- a/llvm/test/TableGen/GlobalISelEmitter-PR39045.td
+++ b/llvm/test/TableGen/GlobalISelEmitter-PR39045.td
@@ -2,8 +2,8 @@
 // RUN: FileCheck %s < %t
 
 // Both predicates should be tested
-// CHECK-DAG: GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GICXXPred_MI_Predicate_pat_frag_b,
-// CHECK-DAG: GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GICXXPred_MI_Predicate_pat_frag_a,
+// CHECK-DAG: GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_pat_frag_b),
+// CHECK-DAG: GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_pat_frag_a),
 
 include "llvm/Target/Target.td"
 include "GlobalISelEmitterCommon.td"
diff --git a/llvm/test/TableGen/GlobalISelEmitter-SDNodeXForm-timm.td b/llvm/test/TableGen/GlobalISelEmitter-SDNodeXForm-timm.td
index 0a12cc0bf856031543d36fe3918396c4677eb06a..8d6dedf2f920cef54182bc8bd22924b394178b8e 100644
--- a/llvm/test/TableGen/GlobalISelEmitter-SDNodeXForm-timm.td
+++ b/llvm/test/TableGen/GlobalISelEmitter-SDNodeXForm-timm.td
@@ -18,19 +18,19 @@ def int_mytarget_foo : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], [Imm
 def SLEEP : I<(outs), (ins i32imm:$src0), []>;
 def FOO : I<(outs GPR32:$dst), (ins GPR32:$src0, i32imm:$src1), []>;
 
-// GISEL: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_INTRINSIC,
-// GISEL: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/1, Intrinsic::mytarget_foo,
+// GISEL: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_INTRINSIC),
+// GISEL: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/1, GIMT_Encode2(Intrinsic::mytarget_foo),
 // GISEL: GIM_CheckIsImm, /*MI*/0, /*Op*/3,
-// GISEL: GIR_CustomOperandRenderer, /*InsnID*/0, /*OldInsnID*/0, /*OpIdx*/3, /*OperandRenderer*/GICR_renderShiftImml1, // src1
+// GISEL: GIR_CustomOperandRenderer, /*InsnID*/0, /*OldInsnID*/0, /*OpIdx*/3, /*OperandRenderer*/GIMT_Encode2(GICR_renderShiftImml1), // src1
 def : Pat<
   (int_mytarget_foo i32:$src0, (i32 timm:$src1)),
   (FOO GPR32:$src0, (shiftl_1 $src1))
 >;
 
-// GISEL: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS,
-// GISEL: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/0, Intrinsic::mytarget_sleep,
+// GISEL: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS),
+// GISEL: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/0, GIMT_Encode2(Intrinsic::mytarget_sleep),
 // GISEL: GIM_CheckIsImm, /*MI*/0, /*Op*/1,
-// GISEL: GIR_CustomOperandRenderer, /*InsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, /*OperandRenderer*/GICR_renderShiftImml1, // src0
+// GISEL: GIR_CustomOperandRenderer, /*InsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, /*OperandRenderer*/GIMT_Encode2(GICR_renderShiftImml1), // src0
 def : Pat<
   (int_mytarget_sleep (i32 timm:$src0)),
   (SLEEP (shiftl_1 $src0))
diff --git a/llvm/test/TableGen/GlobalISelEmitter-atomic_store.td b/llvm/test/TableGen/GlobalISelEmitter-atomic_store.td
index 5263eed007bd452dc28821b744b216a724ad07dc..081c9e8634592064658fb9cdbdbe1dcf996ab092 100644
--- a/llvm/test/TableGen/GlobalISelEmitter-atomic_store.td
+++ b/llvm/test/TableGen/GlobalISelEmitter-atomic_store.td
@@ -5,15 +5,15 @@ include "GlobalISelEmitterCommon.td"
 
 def ST_ATOM_B32 : I<(outs), (ins GPR32Op:$val, GPR32Op:$ptr), []>;
 
-// GISEL: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_STORE,
-// GISEL-NEXT: GIM_CheckMemorySizeEqualTo, /*MI*/0, /*MMO*/0, /*Size*/1,
-// GISEL-NEXT: GIM_CheckAtomicOrderingOrStrongerThan, /*MI*/0, /*Order*/(int64_t)AtomicOrdering::Unordered,
+// GISEL: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_STORE),
+// GISEL-NEXT: GIM_CheckMemorySizeEqualTo, /*MI*/0, /*MMO*/0, /*Size*/GIMT_Encode4(1),
+// GISEL-NEXT: GIM_CheckAtomicOrderingOrStrongerThan, /*MI*/0, /*Order*/(uint8_t)AtomicOrdering::Unordered,
 // GISEL-NEXT: // MIs[0] val
 // GISEL-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
 // GISEL-NEXT: // MIs[0] ptr
 // GISEL-NEXT: GIM_CheckPointerToAny, /*MI*/0, /*Op*/1, /*SizeInBits*/0,
 // GISEL-NEXT: // (atomic_store i32:{ *:[i32] }:$val, iPTR:{ *:[iPTR] }:$ptr)<<P:Predicate_atomic_store_8>>  =>  (ST_ATOM_B32 GPR32Op:{ *:[i32] }:$val, GPR32Op:{ *:[i32] }:$ptr)
-// GISEL-NEXT: GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/MyTarget::ST_ATOM_B32,
+// GISEL-NEXT: GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::ST_ATOM_B32),
 def : Pat<
 //  (atomic_store_8 iPTR:$ptr, i32:$val),
   (atomic_store_8 i32:$val, iPTR:$ptr),
diff --git a/llvm/test/TableGen/GlobalISelEmitter-immAllZeroOne.td b/llvm/test/TableGen/GlobalISelEmitter-immAllZeroOne.td
index 5d39278ed986008749aa68c80673aa1fd010719e..eae29308aa1bdc9946ed735fbd8403509dcb56cd 100644
--- a/llvm/test/TableGen/GlobalISelEmitter-immAllZeroOne.td
+++ b/llvm/test/TableGen/GlobalISelEmitter-immAllZeroOne.td
@@ -12,21 +12,21 @@ include "GlobalISelEmitterCommon.td"
 
 // GISEL-OPT: GIM_Try,
 // GISEL-OPT: GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_v4s16,
-// GISEL-OPT: GIM_CheckOpcodeIsEither, /*MI*/1, TargetOpcode::G_BUILD_VECTOR, TargetOpcode::G_BUILD_VECTOR_TRUNC,
+// GISEL-OPT: GIM_CheckOpcodeIsEither, /*MI*/1, GIMT_Encode2(TargetOpcode::G_BUILD_VECTOR), GIMT_Encode2(TargetOpcode::G_BUILD_VECTOR_TRUNC),
 // GISEL-OPT: GIM_CheckIsBuildVectorAllZeros, /*MI*/1,
 
 // GISEL-OPT: GIM_Try,
 // GISEL-OPT: GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_v4s16,
-// GISEL-OPT: GIM_CheckOpcodeIsEither, /*MI*/1, TargetOpcode::G_BUILD_VECTOR, TargetOpcode::G_BUILD_VECTOR_TRUNC,
+// GISEL-OPT: GIM_CheckOpcodeIsEither, /*MI*/1, GIMT_Encode2(TargetOpcode::G_BUILD_VECTOR), GIMT_Encode2(TargetOpcode::G_BUILD_VECTOR_TRUNC),
 // GISEL-OPT: GIM_CheckIsBuildVectorAllOnes, /*MI*/1,
 
 
 
-// GISEL-NOOPT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_LSHR,
+// GISEL-NOOPT: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_LSHR),
 // GISEL-NOOPT: // MIs[0] Operand 2
 // GISEL-NOOPT-NEXT: GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_v4s16,
 // GISEL-NOOPT-NEXT: GIM_RecordInsn, /*DefineMI*/1, /*MI*/0, /*OpIdx*/2, // MIs[1]
-// GISEL-NOOPT-NEXT: GIM_CheckOpcodeIsEither, /*MI*/1, TargetOpcode::G_BUILD_VECTOR, TargetOpcode::G_BUILD_VECTOR_TRUNC,
+// GISEL-NOOPT-NEXT: GIM_CheckOpcodeIsEither, /*MI*/1, GIMT_Encode2(TargetOpcode::G_BUILD_VECTOR), GIMT_Encode2(TargetOpcode::G_BUILD_VECTOR_TRUNC),
 // GISEL-NOOPT-NEXT: GIM_CheckIsBuildVectorAllOnes, /*MI*/1,
 // GISEL-NOOPT-NEXT: // MIs[1] Operand 0
 // GISEL-NOOPT-NEXT: GIM_CheckType, /*MI*/1, /*Op*/0, /*Type*/GILLT_v4s16,
@@ -37,11 +37,11 @@ def VFOOONES : I<(outs VecReg128:$dst), (ins VecReg128:$src0),
 >;
 
 
-// GISEL-NOOPT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_SHL,
+// GISEL-NOOPT: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_SHL),
 // GISEL-NOOPT: // MIs[0] Operand 2
 // GISEL-NOOPT-NEXT: GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_v4s16,
 // GISEL-NOOPT-NEXT: GIM_RecordInsn, /*DefineMI*/1, /*MI*/0, /*OpIdx*/2, // MIs[1]
-// GISEL-NOOPT-NEXT: GIM_CheckOpcodeIsEither, /*MI*/1, TargetOpcode::G_BUILD_VECTOR, TargetOpcode::G_BUILD_VECTOR_TRUNC,
+// GISEL-NOOPT-NEXT: GIM_CheckOpcodeIsEither, /*MI*/1, GIMT_Encode2(TargetOpcode::G_BUILD_VECTOR), GIMT_Encode2(TargetOpcode::G_BUILD_VECTOR_TRUNC),
 // GISEL-NOOPT-NEXT: GIM_CheckIsBuildVectorAllZeros, /*MI*/1,
 // GISEL-NOOPT-NEXT: // MIs[1] Operand 0
 // GISEL-NOOPT-NEXT: GIM_CheckType, /*MI*/1, /*Op*/0, /*Type*/GILLT_v4s16,
diff --git a/llvm/test/TableGen/GlobalISelEmitter-immarg-literal-pattern.td b/llvm/test/TableGen/GlobalISelEmitter-immarg-literal-pattern.td
index 2f39bf49af4d57eb2ff388ce56b10da39f13b3b5..bfbeee466b1cc3723debec76fb39d1d8155139db 100644
--- a/llvm/test/TableGen/GlobalISelEmitter-immarg-literal-pattern.td
+++ b/llvm/test/TableGen/GlobalISelEmitter-immarg-literal-pattern.td
@@ -22,19 +22,19 @@ def CAT1 : I<(outs GPR32:$dst), (ins GPR32:$src0), []>;
 // Test immarg intrinsic pattern
 
 // Make sure there is no type check.
-// GISEL: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS,
-// GISEL: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/0, Intrinsic::mytarget_sleep,
+// GISEL: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS),
+// GISEL: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/0, GIMT_Encode2(Intrinsic::mytarget_sleep),
 // GISEL-NEXT: // MIs[0] Operand 1
-// GISEL-NEXT: GIM_CheckLiteralInt, /*MI*/0, /*Op*/1, 0,
+// GISEL-NEXT: GIM_CheckLiteralInt, /*MI*/0, /*Op*/1, GIMT_Encode8(0),
 def : Pat<
   (int_mytarget_sleep 0),
   (SLEEP0)
 >;
 
-// GISEL: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS,
-// GISEL: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/0, Intrinsic::mytarget_sleep,
+// GISEL: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS),
+// GISEL: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/0, GIMT_Encode2(Intrinsic::mytarget_sleep),
 // GISEL-NEXT: // MIs[0] Operand 1
-// GISEL-NEXT: GIM_CheckLiteralInt, /*MI*/0, /*Op*/1, 1,
+// GISEL-NEXT: GIM_CheckLiteralInt, /*MI*/0, /*Op*/1, GIMT_Encode8(1),
 def : Pat<
   (int_mytarget_sleep 1),
   (SLEEP1)
@@ -42,21 +42,20 @@ def : Pat<
 
 // Check a non-intrinsic instruction with an immediate parameter.
 
-// GISEL: GIM_CheckOpcode, /*MI*/0, MyTarget::G_TGT_CAT,
+// GISEL: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(MyTarget::G_TGT_CAT),
 // GISEL: GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32,
 // GISEL-NEXT: // MIs[0] Operand 2
-// GISEL-NEXT: GIM_CheckLiteralInt, /*MI*/0, /*Op*/2, 0,
+// GISEL-NEXT: GIM_CheckLiteralInt, /*MI*/0, /*Op*/2, GIMT_Encode8(0),
 def : Pat<
   (TgtCat i32:$src0, 0),
   (CAT0 GPR32:$src0)
 >;
 
-// GISEL: GIM_CheckOpcode, /*MI*/0, MyTarget::G_TGT_CAT,
+// GISEL: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(MyTarget::G_TGT_CAT),
 // GISEL: GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32,
 // GISEL-NEXT: // MIs[0] Operand 2
-// GISEL-NEXT: GIM_CheckLiteralInt, /*MI*/0, /*Op*/2, 93,
+// GISEL-NEXT: GIM_CheckLiteralInt, /*MI*/0, /*Op*/2, GIMT_Encode8(93),
 def : Pat<
   (TgtCat i32:$src0, 93),
   (CAT1 GPR32:$src0)
 >;
-
diff --git a/llvm/test/TableGen/GlobalISelEmitter-input-discard.td b/llvm/test/TableGen/GlobalISelEmitter-input-discard.td
index 68a9553a8b7381602918f4092ed931e936da503b..7731cdefac8554021b8d69fb69438c8d4f32a185 100644
--- a/llvm/test/TableGen/GlobalISelEmitter-input-discard.td
+++ b/llvm/test/TableGen/GlobalISelEmitter-input-discard.td
@@ -8,21 +8,21 @@ def int_tgt_foo : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], []>;
 // Test that using srcvalue works for discarded pattern inputs.
 def FOO : I<(outs GPR32:$dst), (ins GPR32Op:$src0, GPR32Op:$src1), []>;
 
-// GISEL: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS,
+// GISEL: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS),
 // GISEL-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/4,
-// GISEL-NEXT: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/1, Intrinsic::tgt_foo,
+// GISEL-NEXT: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/1, GIMT_Encode2(Intrinsic::tgt_foo),
 // GISEL-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
 // GISEL-NEXT: GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32,
 // GISEL-NEXT: GIM_CheckType, /*MI*/0, /*Op*/3, /*Type*/GILLT_s32,
-// GISEL-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID,
+// GISEL-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // GISEL-NEXT: // (intrinsic_w_chain:{ *:[i32] } {{[0-9]+}}:{ *:[iPTR] }, srcvalue:{ *:[i32] }, i32:{ *:[i32] }:$src1)  =>  (FOO:{ *:[i32] } (IMPLICIT_DEF:{ *:[i32] }), GPR32:{ *:[i32] }:$src1)
 // GISEL-NEXT: GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s32,
-// GISEL-NEXT: GIR_BuildMI, /*InsnID*/1, /*Opcode*/TargetOpcode::IMPLICIT_DEF,
-// GISEL-NEXT: GIR_AddTempRegister, /*InsnID*/1, /*TempRegID*/0, /*TempRegFlags*/RegState::Define,
+// GISEL-NEXT: GIR_BuildMI, /*InsnID*/1, /*Opcode*/GIMT_Encode2(TargetOpcode::IMPLICIT_DEF),
+// GISEL-NEXT: GIR_AddTempRegister, /*InsnID*/1, /*TempRegID*/0, /*TempRegFlags*/GIMT_Encode2(RegState::Define),
 // GISEL-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/1,
-// GISEL-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::FOO,
+// GISEL-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::FOO),
 // GISEL-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst]
-// GISEL-NEXT: GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/0,
+// GISEL-NEXT: GIR_AddSimpleTempRegister, /*InsnID*/0, /*TempRegID*/0,
 // GISEL-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/3, // src1
 // GISEL-NEXT: GIR_EraseFromParent, /*InsnID*/0,
 // GISEL-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0,
diff --git a/llvm/test/TableGen/GlobalISelEmitter-multiple-output-discard.td b/llvm/test/TableGen/GlobalISelEmitter-multiple-output-discard.td
index 64cf31f3772ee93d34756001a1695273e2648060..f962a7dadf74360d97d3c7ac3140472a7863d6ba 100644
--- a/llvm/test/TableGen/GlobalISelEmitter-multiple-output-discard.td
+++ b/llvm/test/TableGen/GlobalISelEmitter-multiple-output-discard.td
@@ -21,22 +21,22 @@ def : GINodeEquiv<G_TWO_OUT, two_out>;
 
 def : Pat<(two_out GPR32:$val), (THREE_OUTS GPR32:$val)>;
 
-// CHECK:      GIM_CheckOpcode, /*MI*/0, MyTarget::G_TWO_OUT,
+// CHECK:      GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(MyTarget::G_TWO_OUT),
 // CHECK-NEXT: // MIs[0] DstI[out1]
 // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
-// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID,
+// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // CHECK-NEXT: // MIs[0] DstI[out2]
 // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32,
-// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/MyTarget::GPR32RegClassID,
+// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // CHECK-NEXT: // MIs[0] val
 // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32,
-// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/2, /*RC*/MyTarget::GPR32RegClassID,
+// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/2, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // CHECK-NEXT: // (two_out:{ *:[i32] }:{ *:[i32] } GPR32:{ *:[i32] }:$val)  =>  (THREE_OUTS:{ *:[i32] }:{ *:[i32] }:{ *:[i32] } GPR32:{ *:[i32] }:$val)
 // CHECK-NEXT: GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s32,
-// CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::THREE_OUTS,
+// CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::THREE_OUTS),
 // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[out1]
 // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // DstI[out2]
-// CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/RegState::Define|RegState::Dead,
+// CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/GIMT_Encode2(RegState::Define|RegState::Dead),
 // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/2, // val
 // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0,
 // CHECK-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0,
diff --git a/llvm/test/TableGen/GlobalISelEmitter-multiple-output.td b/llvm/test/TableGen/GlobalISelEmitter-multiple-output.td
index 85ac3ace0364ccbbe3d9627122d145b81779aae4..37ce7416fa513859a2f212eabdd1031b21aa238c 100644
--- a/llvm/test/TableGen/GlobalISelEmitter-multiple-output.td
+++ b/llvm/test/TableGen/GlobalISelEmitter-multiple-output.td
@@ -29,21 +29,21 @@ def : Pat<(loadpost (p0 GPR32:$addr), (i32 GPR32:$off)),
   (LDPost GPR32:$addr, GPR32:$off)
 >;
 
-// CHECK:      GIM_CheckOpcode, /*MI*/0, MyTarget::G_POST_LOAD,
+// CHECK:      GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(MyTarget::G_POST_LOAD),
 // CHECK-NEXT: // MIs[0] DstI[val]
 // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
-// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID,
+// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // CHECK-NEXT: // MIs[0] DstI[ptr_out]
 // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_p0s32,
-// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/MyTarget::GPR32RegClassID,
+// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // CHECK-NEXT: // MIs[0] addr
 // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_p0s32,
-// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/2, /*RC*/MyTarget::GPR32RegClassID,
+// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/2, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // CHECK-NEXT: // MIs[0] off
 // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/3, /*Type*/GILLT_s32,
-// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/3, /*RC*/MyTarget::GPR32RegClassID,
+// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/3, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // CHECK-NEXT: // (loadpost:{ *:[i32] }:{ *:[i32] } GPR32:{ *:[i32] }:$addr, GPR32:{ *:[i32] }:$off)  =>  (LDPost:{ *:[i32] }:{ *:[i32] } GPR32:{ *:[i32] }:$addr, GPR32:{ *:[i32] }:$off)
-// CHECK-NEXT: GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/MyTarget::LDPost,
+// CHECK-NEXT: GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::LDPost),
 // CHECK-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0,
 
 // Test where a whole new MIR instruction is created during ISel
@@ -63,21 +63,21 @@ def : GINodeEquiv<G_TWO_IN, two_in>;
 // Swap the input operands for an easy way to force the creation of a new instruction
 def : Pat<(two_in GPR32:$i1, GPR32:$i2), (TWO_INS GPR32:$i2, GPR32:$i1)>;
 
-// CHECK:      GIM_CheckOpcode, /*MI*/0, MyTarget::G_TWO_IN,
+// CHECK:      GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(MyTarget::G_TWO_IN),
 // CHECK-NEXT: // MIs[0] DstI[out1]
 // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
-// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID,
+// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // CHECK-NEXT: // MIs[0] DstI[out2]
 // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32,
-// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/MyTarget::GPR32RegClassID,
+// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // CHECK-NEXT: // MIs[0] i1
 // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32,
-// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/2, /*RC*/MyTarget::GPR32RegClassID,
+// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/2, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // CHECK-NEXT: // MIs[0] i2
 // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/3, /*Type*/GILLT_s32,
-// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/3, /*RC*/MyTarget::GPR32RegClassID,
+// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/3, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // CHECK-NEXT: // (two_in:{ *:[i32] }:{ *:[i32] } GPR32:{ *:[i32] }:$i1, GPR32:{ *:[i32] }:$i2)  =>  (TWO_INS:{ *:[i32] }:{ *:[i32] } GPR32:{ *:[i32] }:$i2, GPR32:{ *:[i32] }:$i1)
-// CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::TWO_INS,
+// CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::TWO_INS),
 // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[out1]
 // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // DstI[out2]
 // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/3, // i2
diff --git a/llvm/test/TableGen/GlobalISelEmitter-nested-subregs.td b/llvm/test/TableGen/GlobalISelEmitter-nested-subregs.td
index 61f79bc04b27e187bf97789471fbfe7bfbcf4243..437686a69b5eac5bff4f4c16ca9ea6364eb7ef07 100644
--- a/llvm/test/TableGen/GlobalISelEmitter-nested-subregs.td
+++ b/llvm/test/TableGen/GlobalISelEmitter-nested-subregs.td
@@ -31,32 +31,32 @@ def A0w : RegisterClass<"MyTarget", [i16], 16, (add a0wl)>;
 def A0  : RegisterClass<"MyTarget", [i32], 32, (add a0)>;
 
 // CHECK: GIM_CheckNumOperands, /*MI*/0, /*Expected*/2,
-// CHECK-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_ANYEXT,
+// CHECK-NEXT: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_ANYEXT),
 // CHECK-NEXT: // MIs[0] DstI[dst]
 // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s16,
-// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::A0RegClassID,
+// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::A0RegClassID),
 // CHECK-NEXT: // MIs[0] src
 // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s8,
 // CHECK-NEXT: // (anyext:{ *:[i16] } i8:{ *:[i8] }:$src)  =>  (EXTRACT_SUBREG:{ *:[i16] } (INSERT_SUBREG:{ *:[i32] } (IMPLICIT_DEF:{ *:[i32] }), A0b:{ *:[i8] }:$src, lo8:{ *:[i32] }), lo16:{ *:[i32] })
 // CHECK-NEXT: GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s32,
 // CHECK-NEXT: GIR_MakeTempReg, /*TempRegID*/1, /*TypeID*/GILLT_s32,
-// CHECK-NEXT: GIR_BuildMI, /*InsnID*/2, /*Opcode*/TargetOpcode::IMPLICIT_DEF,
-// CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/2, /*TempRegID*/1, /*TempRegFlags*/RegState::Define,
+// CHECK-NEXT: GIR_BuildMI, /*InsnID*/2, /*Opcode*/GIMT_Encode2(TargetOpcode::IMPLICIT_DEF),
+// CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/2, /*TempRegID*/1, /*TempRegFlags*/GIMT_Encode2(RegState::Define),
 // CHECK-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/2,
-// CHECK-NEXT: GIR_BuildMI, /*InsnID*/1, /*Opcode*/TargetOpcode::INSERT_SUBREG,
-// CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/1, /*TempRegID*/0, /*TempRegFlags*/RegState::Define,
-// CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/1, /*TempRegID*/1, /*TempRegFlags*/0,
+// CHECK-NEXT: GIR_BuildMI, /*InsnID*/1, /*Opcode*/GIMT_Encode2(TargetOpcode::INSERT_SUBREG),
+// CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/1, /*TempRegID*/0, /*TempRegFlags*/GIMT_Encode2(RegState::Define),
+// CHECK-NEXT: GIR_AddSimpleTempRegister, /*InsnID*/1, /*TempRegID*/1,
 // CHECK-NEXT: GIR_Copy, /*NewInsnID*/1, /*OldInsnID*/0, /*OpIdx*/1, // src
-// CHECK-NEXT: GIR_AddImm, /*InsnID*/1, /*Imm*/3,
-// CHECK-NEXT: GIR_ConstrainOperandRC, /*InsnID*/1, /*Op*/0, MyTarget::A0RegClassID,
-// CHECK-NEXT: GIR_ConstrainOperandRC, /*InsnID*/1, /*Op*/1, MyTarget::A0RegClassID,
-// CHECK-NEXT: GIR_ConstrainOperandRC, /*InsnID*/1, /*Op*/2, MyTarget::A0bRegClassID,
-// CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/TargetOpcode::COPY,
+// CHECK-NEXT: GIR_AddImm8, /*InsnID*/1, /*Imm*/3,
+// CHECK-NEXT: GIR_ConstrainOperandRC, /*InsnID*/1, /*Op*/0, GIMT_Encode2(MyTarget::A0RegClassID),
+// CHECK-NEXT: GIR_ConstrainOperandRC, /*InsnID*/1, /*Op*/1, GIMT_Encode2(MyTarget::A0RegClassID),
+// CHECK-NEXT: GIR_ConstrainOperandRC, /*InsnID*/1, /*Op*/2, GIMT_Encode2(MyTarget::A0bRegClassID),
+// CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(TargetOpcode::COPY),
 // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst]
-// CHECK-NEXT: GIR_AddTempSubRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/0, MyTarget::lo16,
+// CHECK-NEXT: GIR_AddTempSubRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/GIMT_Encode2(0), GIMT_Encode2(MyTarget::lo16),
 // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0,
-// CHECK-NEXT: GIR_ConstrainOperandRC, /*InsnID*/0, /*Op*/0, MyTarget::A0wRegClassID,
-// CHECK-NEXT: GIR_ConstrainOperandRC, /*InsnID*/0, /*Op*/1, MyTarget::A0RegClassID,
+// CHECK-NEXT: GIR_ConstrainOperandRC, /*InsnID*/0, /*Op*/0, GIMT_Encode2(MyTarget::A0wRegClassID),
+// CHECK-NEXT: GIR_ConstrainOperandRC, /*InsnID*/0, /*Op*/1, GIMT_Encode2(MyTarget::A0RegClassID),
 def : Pat<(i16 (anyext i8:$src)),
           (i16 (EXTRACT_SUBREG
                  (i32 (INSERT_SUBREG
diff --git a/llvm/test/TableGen/GlobalISelEmitter-notype-output-pattern.td b/llvm/test/TableGen/GlobalISelEmitter-notype-output-pattern.td
index 313aac16c2357801c2f6b0cceb6ecac0ce1d1ead..86a6e1651fca503946dc9d20f269789037ca9821 100644
--- a/llvm/test/TableGen/GlobalISelEmitter-notype-output-pattern.td
+++ b/llvm/test/TableGen/GlobalISelEmitter-notype-output-pattern.td
@@ -3,12 +3,12 @@
 include "llvm/Target/Target.td"
 include "GlobalISelEmitterCommon.td"
 
-// CHECK:      constexpr static int64_t MatchTable0[] = {
+// CHECK:      constexpr static uint8_t MatchTable0[] = {
 // CHECK-NEXT:   GIM_Try,
-// CHECK-NEXT:     GIM_CheckOpcode{{.*}}TargetOpcode::G_ANYEXT,
+// CHECK-NEXT:     GIM_CheckOpcode{{.*}}GIMT_Encode2(TargetOpcode::G_ANYEXT),
 // CHECK-NEXT:     GIM_CheckType{{.*}}/*Type*/GILLT_s32,
 // CHECK-NEXT:     GIM_CheckType{{.*}}/*Type*/GILLT_s8,
-// CHECK-NEXT:     GIM_CheckRegBankForClass{{.*}}/*RC*/MyTarget::GPR32RegClassID,
+// CHECK-NEXT:     GIM_CheckRegBankForClass{{.*}}/*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // CHECK-NEXT:     // (anyext:{{.*}}=>{{.*}}(SELECT_I4:
 // CHECK:          GIR_Done,
 // CHECK-NEXT:   // Label 0:
diff --git a/llvm/test/TableGen/GlobalISelEmitter-output-discard.td b/llvm/test/TableGen/GlobalISelEmitter-output-discard.td
index 1cfe49dee0970466913fb446afb35a428e68f512..50a9ef8ddf5733abe66926be9574921d9ebc19a9 100644
--- a/llvm/test/TableGen/GlobalISelEmitter-output-discard.td
+++ b/llvm/test/TableGen/GlobalISelEmitter-output-discard.td
@@ -7,16 +7,16 @@ include "GlobalISelEmitterCommon.td"
 def ADD_CO : I<(outs GPR32:$dst, GPR8:$flag),
                (ins GPR32Op:$src0, GPR32Op:$src1), []>;
 
-// GISEL: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_ADD,
+// GISEL: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_ADD),
 // GISEL-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
 // GISEL-NEXT: GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32,
 // GISEL-NEXT: GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32,
-// GISEL-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID,
+// GISEL-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // GISEL-NEXT: // (add:{ *:[i32] } i32:{ *:[i32] }:$src0, i32:{ *:[i32] }:$src1)  =>  (ADD_CO:{ *:[i32] }:{ *:[i8] } GPR32:{ *:[i32] }:$src0, GPR32:{ *:[i32] }:$src1)
 // GISEL-NEXT: GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s8,
-// GISEL-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::ADD_CO,
+// GISEL-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::ADD_CO),
 // GISEL-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst]
-// GISEL-NEXT: GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/RegState::Define|RegState::Dead,
+// GISEL-NEXT: GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/GIMT_Encode2(RegState::Define|RegState::Dead),
 // GISEL-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // src0
 // GISEL-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/2, // src1
 // GISEL-NEXT: GIR_EraseFromParent, /*InsnID*/0,
diff --git a/llvm/test/TableGen/GlobalISelEmitter-setcc.td b/llvm/test/TableGen/GlobalISelEmitter-setcc.td
index 933489b31bc38b99bad89cc825d44add4a3c23d8..38add7627f5030bcdeed5663be0e268ebdfbfb40 100644
--- a/llvm/test/TableGen/GlobalISelEmitter-setcc.td
+++ b/llvm/test/TableGen/GlobalISelEmitter-setcc.td
@@ -6,15 +6,15 @@ include "GlobalISelEmitterCommon.td"
 
 // GISEL: GIM_Try
 // GISEL: GIM_CheckNumOperands, /*MI*/0, /*Expected*/4,
-// GISEL-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_FCMP,
-// GISEL: GIM_CheckCmpPredicate, /*MI*/0, /*Op*/1, /*Predicate*/CmpInst::FCMP_OEQ,
+// GISEL-NEXT: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_FCMP),
+// GISEL: GIM_CheckCmpPredicate, /*MI*/0, /*Op*/1, /*Predicate*/GIMT_Encode2(CmpInst::FCMP_OEQ),
 def FCMPOEQ : I<(outs GPR32:$dst), (ins FPR32Op:$src0, FPR32:$src1),
               [(set GPR32:$dst, (i32 (setcc f32:$src0, f32:$src1, SETOEQ)))]>;
 
 // GISEL: GIM_Try
 // GISEL: GIM_CheckNumOperands, /*MI*/0, /*Expected*/4,
-// GISEL-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_ICMP,
-// GISEL: GIM_CheckCmpPredicate, /*MI*/0, /*Op*/1, /*Predicate*/CmpInst::ICMP_EQ,
+// GISEL-NEXT: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_ICMP),
+// GISEL: GIM_CheckCmpPredicate, /*MI*/0, /*Op*/1, /*Predicate*/GIMT_Encode2(CmpInst::ICMP_EQ),
 def ICMPEQ : I<(outs GPR32:$dst), (ins GPR32Op:$src0, GPR32:$src1),
                [(set GPR32:$dst, (i32 (setcc i32:$src0, i32:$src1, SETEQ)))]>;
 
diff --git a/llvm/test/TableGen/GlobalISelEmitter-zero-reg.td b/llvm/test/TableGen/GlobalISelEmitter-zero-reg.td
index 63ad5d5cd03d965f789c4a76e03017465cad0494..6495c10df237cb4829c1fed3e8aee7114378ec1f 100644
--- a/llvm/test/TableGen/GlobalISelEmitter-zero-reg.td
+++ b/llvm/test/TableGen/GlobalISelEmitter-zero-reg.td
@@ -21,21 +21,21 @@ class PredI<dag OOps, dag IOps, list<dag> Pat>
 def INST : PredI<(outs GPR32:$dst), (ins GPR32:$src), []>;
 
 // CHECK: GIM_CheckNumOperands, /*MI*/0, /*Expected*/2,
-// CHECK-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_LOAD,
+// CHECK-NEXT: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_LOAD),
 // CHECK-NEXT: GIM_CheckMemorySizeEqualToLLT, /*MI*/0, /*MMO*/0, /*OpIdx*/0,
-// CHECK-NEXT: GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(int64_t)AtomicOrdering::NotAtomic,
+// CHECK-NEXT: GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(uint8_t)AtomicOrdering::NotAtomic,
 // CHECK-NEXT: // MIs[0] DstI[dst]
 // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
-// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID,
+// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // CHECK-NEXT: // MIs[0] src
 // CHECK-NEXT: GIM_CheckPointerToAny, /*MI*/0, /*Op*/1, /*SizeInBits*/32,
-// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/MyTarget::GPR32RegClassID,
+// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // CHECK-NEXT: // (ld:{ *:[i32] } GPR32:{ *:[i32] }:$src)<<P:Predicate_unindexedload>><<P:Predicate_load>>  =>  (INST:{ *:[i32] } GPR32:{ *:[i32] }:$src)
-// CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::INST,
+// CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::INST),
 // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst]
 // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // src
-// CHECK-NEXT: GIR_AddRegister, /*InsnID*/0, MyTarget::NoRegister, /*AddRegisterRegFlags*/0,
-// CHECK-NEXT: GIR_MergeMemOperands, /*InsnID*/0, /*MergeInsnID's*/0, GIU_MergeMemOperands_EndOfList,
+// CHECK-NEXT: GIR_AddRegister, /*InsnID*/0, GIMT_Encode2(MyTarget::NoRegister), /*AddRegisterRegFlags*/GIMT_Encode2(0),
+// CHECK-NEXT: GIR_MergeMemOperands, /*InsnID*/0, /*NumInsns*/1, /*MergeInsnID's*/0,
 // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0,
 // CHECK-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0,
 def : Pat<(i32 (load GPR32:$src)),
diff --git a/llvm/test/TableGen/GlobalISelEmitter.td b/llvm/test/TableGen/GlobalISelEmitter.td
index 562fdefcd3d6c0e4a420d3c9c8ba24f428b67a68..eab2acd6cb1e7ed44d949b5750a65a599c19164f 100644
--- a/llvm/test/TableGen/GlobalISelEmitter.td
+++ b/llvm/test/TableGen/GlobalISelEmitter.td
@@ -79,7 +79,7 @@ def HasC : Predicate<"Subtarget->hasC()"> { let RecomputePerFunction = 1; }
 // CHECK-NEXT:    bool testImmPredicate_I64(unsigned PredicateID, int64_t Imm) const override;
 // CHECK-NEXT:    bool testImmPredicate_APInt(unsigned PredicateID, const APInt &Imm) const override;
 // CHECK-NEXT:    bool testImmPredicate_APFloat(unsigned PredicateID, const APFloat &Imm) const override;
-// CHECK-NEXT:    const int64_t *getMatchTable() const override;
+// CHECK-NEXT:    const uint8_t *getMatchTable() const override;
 // CHECK-NEXT:    bool testMIPredicate_MI(unsigned PredicateID, const MachineInstr &MI, const MatcherState &State) const override;
 // CHECK-NEXT:    bool testSimplePredicate(unsigned PredicateID) const override;
 // CHECK-NEXT:    void runCustomAction(unsigned FnID, const MatcherState &State, NewMIVector &OutMIs) const override;
@@ -224,77 +224,77 @@ def HasC : Predicate<"Subtarget->hasC()"> { let RecomputePerFunction = 1; }
 // CHECK-NEXT:   return true;
 // CHECK-NEXT: }
 
-// CHECK: const int64_t *
+// CHECK: const uint8_t *
 // CHECK-LABEL: MyTargetInstructionSelector::getMatchTable() const {
 // CHECK-NEXT: MatchTable0[] = {
 
 //===- Test a pattern with multiple ComplexPatterns in multiple instrs ----===//
 //
-// R19O-NEXT:  GIM_SwitchOpcode, /*MI*/0, /*[*/{{[0-9]+}}, {{[0-9]+}}, /*)*//*default:*//*Label [[DEFAULT_NUM:[0-9]+]]*/ [[DEFAULT:[0-9]+]],
-// R19O-NEXT:  /*TargetOpcode::G_ADD*//*Label [[CASE_ADD_NUM:[0-9]+]]*/ [[CASE_ADD:[0-9]+]],
-// R19O:       /*TargetOpcode::G_SELECT*//*Label [[CASE_SELECT_NUM:[0-9]+]]*/ [[CASE_SELECT:[0-9]+]],
+// R19O-NEXT:  GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2({{[0-9]+}}), GIMT_Encode2({{[0-9]+}}), /*)*//*default:*//*Label [[DEFAULT_NUM:[0-9]+]]*/ GIMT_Encode4([[DEFAULT:[0-9]+]]),
+// R19O-NEXT:  /*TargetOpcode::G_ADD*//*Label [[CASE_ADD_NUM:[0-9]+]]*/ GIMT_Encode4([[CASE_ADD:[0-9]+]]),
+// R19O:       /*TargetOpcode::G_SELECT*//*Label [[CASE_SELECT_NUM:[0-9]+]]*/ GIMT_Encode4([[CASE_SELECT:[0-9]+]]),
 // R19O:       // Label [[CASE_ADD_NUM]]: @[[CASE_ADD]]
 // R19O:       // Label [[CASE_SELECT_NUM]]: @[[CASE_SELECT]]
-// R19O-NEXT:  GIM_Try, /*On fail goto*//*Label [[GROUP_NUM:[0-9]+]]*/ [[GROUP:[0-9]+]],
+// R19O-NEXT:  GIM_Try, /*On fail goto*//*Label [[GROUP_NUM:[0-9]+]]*/ GIMT_Encode4([[GROUP:[0-9]+]]),
 // R19O-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
 // R19O-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32,
 // R19O-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32,
 // R19O-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/3, /*Type*/GILLT_s32,
 //
-// R19C-NEXT:  GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ [[LABEL:[0-9]+]],
+// R19C-NEXT:  GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ GIMT_Encode4([[LABEL:[0-9]+]]),
 //
-// R19O-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID,
-// R19O-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/MyTarget::GPR32RegClassID,
+// R19O-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
+// R19O-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // R19N-NEXT:    GIM_CheckNumOperands, /*MI*/0, /*Expected*/4,
-// R19N-NEXT:    GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_SELECT,
+// R19N-NEXT:    GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_SELECT),
 // R19N-NEXT:    // MIs[0] DstI[dst]
 // R19N-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
-// R19N-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID,
+// R19N-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // R19N-NEXT:    // MIs[0] src1
 // R19N-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32,
-// R19N-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/MyTarget::GPR32RegClassID,
+// R19N-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // R19N-NEXT:    // MIs[0] complex_rr:src2a:src2b
 // R19N-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32,
 //
-// R19N-NEXT:    GIM_CheckComplexPattern, /*MI*/0, /*Op*/2, /*Renderer*/0, GICP_gi_complex_rr,
+// R19N-NEXT:    GIM_CheckComplexPattern, /*MI*/0, /*Op*/2, /*Renderer*/GIMT_Encode2(0), GIMT_Encode2(GICP_gi_complex_rr),
 // R19N-NEXT:    // MIs[0] Operand 3
 // R19N-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/3, /*Type*/GILLT_s32,
 // R19C-NEXT:    GIM_RecordInsn, /*DefineMI*/1, /*MI*/0, /*OpIdx*/3, // MIs[1]
 // R19N-NEXT:    GIM_CheckNumOperands, /*MI*/1, /*Expected*/4,
-// R19C-NEXT:    GIM_CheckOpcode, /*MI*/1, TargetOpcode::G_SELECT,
+// R19C-NEXT:    GIM_CheckOpcode, /*MI*/1, GIMT_Encode2(TargetOpcode::G_SELECT),
 // R19N-NEXT:    // MIs[1] Operand 0
 // R19N-NEXT:    GIM_CheckType, /*MI*/1, /*Op*/0, /*Type*/GILLT_s32,
 // R19N-NEXT:    // MIs[1] src3
 // R19C-NEXT:    GIM_CheckType, /*MI*/1, /*Op*/1, /*Type*/GILLT_s32,
 // R19O-NEXT:    GIM_CheckType, /*MI*/1, /*Op*/2, /*Type*/GILLT_s32,
 // R19O-NEXT:    GIM_CheckType, /*MI*/1, /*Op*/3, /*Type*/GILLT_s32,
-// R19N-NEXT:    GIM_CheckRegBankForClass, /*MI*/1, /*Op*/1, /*RC*/MyTarget::GPR32RegClassID,
+// R19N-NEXT:    GIM_CheckRegBankForClass, /*MI*/1, /*Op*/1, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // R19N-NEXT:    // MIs[1] src4
 // R19N-NEXT:    GIM_CheckType, /*MI*/1, /*Op*/2, /*Type*/GILLT_s32,
-// R19N-NEXT:    GIM_CheckComplexPattern, /*MI*/1, /*Op*/2, /*Renderer*/1, GICP_gi_complex,
+// R19N-NEXT:    GIM_CheckComplexPattern, /*MI*/1, /*Op*/2, /*Renderer*/GIMT_Encode2(1), GIMT_Encode2(GICP_gi_complex),
 // R19N-NEXT:    // MIs[1] complex:src5a:src5b
 // R19N-NEXT:    GIM_CheckType, /*MI*/1, /*Op*/3, /*Type*/GILLT_s32,
-// R19N-NEXT:    GIM_CheckComplexPattern, /*MI*/1, /*Op*/3, /*Renderer*/2, GICP_gi_complex,
-// R19O-NEXT:    GIM_CheckRegBankForClass, /*MI*/1, /*Op*/1, /*RC*/MyTarget::GPR32RegClassID,
+// R19N-NEXT:    GIM_CheckComplexPattern, /*MI*/1, /*Op*/3, /*Renderer*/GIMT_Encode2(2), GIMT_Encode2(GICP_gi_complex),
+// R19O-NEXT:    GIM_CheckRegBankForClass, /*MI*/1, /*Op*/1, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // R19C-NEXT:    GIM_CheckIsSafeToFold, /*InsnID*/1,
-// R19O-NEXT:    GIM_CheckComplexPattern, /*MI*/0, /*Op*/2, /*Renderer*/0, GICP_gi_complex_rr,
-// R19O-NEXT:    GIM_CheckComplexPattern, /*MI*/1, /*Op*/2, /*Renderer*/1, GICP_gi_complex,
-// R19O-NEXT:    GIM_CheckComplexPattern, /*MI*/1, /*Op*/3, /*Renderer*/2, GICP_gi_complex,
+// R19O-NEXT:    GIM_CheckComplexPattern, /*MI*/0, /*Op*/2, /*Renderer*/GIMT_Encode2(0), GIMT_Encode2(GICP_gi_complex_rr),
+// R19O-NEXT:    GIM_CheckComplexPattern, /*MI*/1, /*Op*/2, /*Renderer*/GIMT_Encode2(1), GIMT_Encode2(GICP_gi_complex),
+// R19O-NEXT:    GIM_CheckComplexPattern, /*MI*/1, /*Op*/3, /*Renderer*/GIMT_Encode2(2), GIMT_Encode2(GICP_gi_complex),
 // R19C-NEXT:    // (select:{ *:[i32] } GPR32:{ *:[i32] }:$src1, (complex_rr:{ *:[i32] } GPR32:{ *:[i32] }:$src2a, GPR32:{ *:[i32] }:$src2b), (select:{ *:[i32] } GPR32:{ *:[i32] }:$src3, complex:{ *:[i32] }:$src4, (complex:{ *:[i32] } i32imm:{ *:[i32] }:$src5a, i32imm:{ *:[i32] }:$src5b)))  =>  (INSN3:{ *:[i32] } GPR32:{ *:[i32] }:$src1, GPR32:{ *:[i32] }:$src2b, GPR32:{ *:[i32] }:$src2a, (INSN4:{ *:[i32] } GPR32:{ *:[i32] }:$src3, complex:{ *:[i32] }:$src4, i32imm:{ *:[i32] }:$src5a, i32imm:{ *:[i32] }:$src5b))
 // R19C-NEXT:    GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s32,
-// R19C-NEXT:    GIR_BuildMI, /*InsnID*/1, /*Opcode*/MyTarget::INSN4,
-// R19C-NEXT:    GIR_AddTempRegister, /*InsnID*/1, /*TempRegID*/0, /*TempRegFlags*/RegState::Define,
+// R19C-NEXT:    GIR_BuildMI, /*InsnID*/1, /*Opcode*/GIMT_Encode2(MyTarget::INSN4),
+// R19C-NEXT:    GIR_AddTempRegister, /*InsnID*/1, /*TempRegID*/0, /*TempRegFlags*/GIMT_Encode2(RegState::Define),
 // R19C-NEXT:    GIR_Copy, /*NewInsnID*/1, /*OldInsnID*/1, /*OpIdx*/1, // src3
-// R19C-NEXT:    GIR_ComplexRenderer, /*InsnID*/1, /*RendererID*/1,
-// R19C-NEXT:    GIR_ComplexSubOperandRenderer, /*InsnID*/1, /*RendererID*/2, /*SubOperand*/0, // src5a
-// R19C-NEXT:    GIR_ComplexSubOperandRenderer, /*InsnID*/1, /*RendererID*/2, /*SubOperand*/1, // src5b
+// R19C-NEXT:    GIR_ComplexRenderer, /*InsnID*/1, /*RendererID*/GIMT_Encode2(1),
+// R19C-NEXT:    GIR_ComplexSubOperandRenderer, /*InsnID*/1, /*RendererID*/GIMT_Encode2(2), /*SubOperand*/0, // src5a
+// R19C-NEXT:    GIR_ComplexSubOperandRenderer, /*InsnID*/1, /*RendererID*/GIMT_Encode2(2), /*SubOperand*/1, // src5b
 // R19C-NEXT:    GIR_ConstrainSelectedInstOperands, /*InsnID*/1,
-// R19C-NEXT:    GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::INSN3,
-// R19C-NEXT:    GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst]
+// R19C-NEXT:    GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::INSN3),
+// R19C-NEXT:    GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, //  DstI[dst]
 // R19C-NEXT:    GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // src1
-// R19C-NEXT:    GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/0, /*SubOperand*/1, // src2b
-// R19C-NEXT:    GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/0, /*SubOperand*/0, // src2a
-// R19C-NEXT:    GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/0,
+// R19C-NEXT:    GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/GIMT_Encode2(0), /*SubOperand*/1, // src2b
+// R19C-NEXT:    GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/GIMT_Encode2(0), /*SubOperand*/0, // src2a
+// R19C-NEXT:    GIR_AddSimpleTempRegister, /*InsnID*/0, /*TempRegID*/0,
 // R19C-NEXT:    GIR_EraseFromParent, /*InsnID*/0,
 // R19C-NEXT:    GIR_ConstrainSelectedInstOperands, /*InsnID*/0,
 // R19C-NEXT:    // GIR_Coverage, 19,
@@ -319,51 +319,51 @@ def : Pat<(select GPR32:$src1, (complex_rr GPR32:$src2a, GPR32:$src2b),
                  (INSN4 GPR32:$src3, complex:$src4, i32imm:$src5a,
                         i32imm:$src5b))>;
 
-// R21O-NEXT:  GIM_SwitchOpcode, /*MI*/0, /*[*/{{[0-9]+}}, {{[0-9]+}}, /*)*//*default:*//*Label [[DEFAULT_NUM:[0-9]+]]*/ [[DEFAULT:[0-9]+]],
-// R21O-NEXT:  /*TargetOpcode::G_ADD*//*Label [[CASE_ADD_NUM:[0-9]+]]*/ [[CASE_ADD:[0-9]+]],
-// R21O:       /*TargetOpcode::G_SELECT*//*Label [[CASE_SELECT_NUM:[0-9]+]]*/ [[CASE_SELECT:[0-9]+]],
+// R21O-NEXT:  GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2({{[0-9]+}}), GIMT_Encode2({{[0-9]+}}), /*)*//*default:*//*Label [[DEFAULT_NUM:[0-9]+]]*/ GIMT_Encode4([[DEFAULT:[0-9]+]]),
+// R21O-NEXT:  /*TargetOpcode::G_ADD*//*Label [[CASE_ADD_NUM:[0-9]+]]*/ GIMT_Encode4([[CASE_ADD:[0-9]+]]),
+// R21O:       /*TargetOpcode::G_SELECT*//*Label [[CASE_SELECT_NUM:[0-9]+]]*/ GIMT_Encode4([[CASE_SELECT:[0-9]+]]),
 // R21O:       // Label [[CASE_ADD_NUM]]: @[[CASE_ADD]]
 // R21O:       // Label [[CASE_SELECT_NUM]]: @[[CASE_SELECT]]
-// R21O-NEXT:  GIM_Try, /*On fail goto*//*Label [[GROUP_NUM:[0-9]+]]*/ [[GROUP:[0-9]+]],
+// R21O-NEXT:  GIM_Try, /*On fail goto*//*Label [[GROUP_NUM:[0-9]+]]*/ GIMT_Encode4([[GROUP:[0-9]+]]),
 // R21O-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
 // R21O-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32,
 // R21O-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32,
 // R21O-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/3, /*Type*/GILLT_s32,
 //
-// R21C-NEXT:  GIM_Try, /*On fail goto*//*Label [[PREV_NUM:[0-9]+]]*/ [[PREV:[0-9]+]], // Rule ID 19 //
+// R21C-NEXT:  GIM_Try, /*On fail goto*//*Label [[PREV_NUM:[0-9]+]]*/ GIMT_Encode4([[PREV:[0-9]+]]), // Rule ID 19 //
 // R21C-NOT:     GIR_Done,
 // R21C:         // GIR_Coverage, 19,
 // R21C-NEXT:    GIR_Done,
 // R21C-NEXT:  // Label [[PREV_NUM]]: @[[PREV]]
-// R21C-NEXT:  GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ [[LABEL:[0-9]+]], // Rule ID 21 //
+// R21C-NEXT:  GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ GIMT_Encode4([[LABEL:[0-9]+]]), // Rule ID 21 //
 //
-// R21O-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID,
-// R21O-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/MyTarget::GPR32RegClassID,
+// R21O-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
+// R21O-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // R21N-NEXT:    GIM_CheckNumOperands, /*MI*/0, /*Expected*/4,
-// R21N-NEXT:    GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_SELECT,
+// R21N-NEXT:    GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_SELECT),
 // R21N-NEXT:    // MIs[0] DstI[dst]
 // R21N-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
-// R21N-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID,
+// R21N-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // R21N-NEXT:    // MIs[0] src1
 // R21N-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32,
-// R21N-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/MyTarget::GPR32RegClassID,
+// R21N-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // R21N-NEXT:    // MIs[0] src2
 // R21N-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32,
 //
-// R21O-NEXT:    GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GICXXPred_MI_Predicate_frag,
-// R21C-NEXT:    GIM_CheckComplexPattern, /*MI*/0, /*Op*/2, /*Renderer*/0, GICP_gi_complex,
+// R21O-NEXT:    GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_frag),
+// R21C-NEXT:    GIM_CheckComplexPattern, /*MI*/0, /*Op*/2, /*Renderer*/GIMT_Encode2(0), GIMT_Encode2(GICP_gi_complex),
 // R21N-NEXT:    // MIs[0] src3
 // R21N-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/3, /*Type*/GILLT_s32,
-// R21C-NEXT:    GIM_CheckComplexPattern, /*MI*/0, /*Op*/3, /*Renderer*/1, GICP_gi_complex,
-// R21N-NEXT:    GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GICXXPred_MI_Predicate_frag,
+// R21C-NEXT:    GIM_CheckComplexPattern, /*MI*/0, /*Op*/3, /*Renderer*/GIMT_Encode2(1), GIMT_Encode2(GICP_gi_complex),
+// R21N-NEXT:    GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_frag),
 // R21C-NEXT:    // (select:{ *:[i32] } GPR32:{ *:[i32] }:$src1, complex:{ *:[i32] }:$src2, complex:{ *:[i32] }:$src3)<<P:Predicate_frag>> => (INSN2:{ *:[i32] } GPR32:{ *:[i32] }:$src1, complex:{ *:[i32] }:$src3, complex:{ *:[i32] }:$src2)
 
-// R21C-NEXT:    GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::INSN2,
-// R21C-NEXT:    GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst]
+// R21C-NEXT:    GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::INSN2),
+// R21C-NEXT:    GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, //  DstI[dst]
 // R21C-NEXT:    GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // src1
-// R21C-NEXT:    GIR_ComplexRenderer, /*InsnID*/0, /*RendererID*/1,
-// R21C-NEXT:    GIR_ComplexRenderer, /*InsnID*/0, /*RendererID*/0,
-// R21C-NEXT:    GIR_MergeMemOperands, /*InsnID*/0, /*MergeInsnID's*/0, GIU_MergeMemOperands_EndOfList,
+// R21C-NEXT:    GIR_ComplexRenderer, /*InsnID*/0, /*RendererID*/GIMT_Encode2(1),
+// R21C-NEXT:    GIR_ComplexRenderer, /*InsnID*/0, /*RendererID*/GIMT_Encode2(0),
+// R21C-NEXT:    GIR_MergeMemOperands, /*InsnID*/0, /*NumInsns*/1, /*MergeInsnID's*/0
 // R21C-NEXT:    GIR_EraseFromParent, /*InsnID*/0,
 // R21C-NEXT:    GIR_ConstrainSelectedInstOperands, /*InsnID*/0,
 // R21C-NEXT:    // GIR_Coverage, 21,
@@ -379,40 +379,40 @@ def : Pat<(select GPR32:$src1, (complex_rr GPR32:$src2a, GPR32:$src2b),
 
 //===- Test a pattern with ComplexPattern operands. -----------------------===//
 //
-// R20O-NEXT:  GIM_SwitchOpcode, /*MI*/0, /*[*/{{[0-9]+}}, {{[0-9]+}}, /*)*//*default:*//*Label [[DEFAULT_NUM:[0-9]+]]*/ [[DEFAULT:[0-9]+]],
-// R20O-NEXT:  /*TargetOpcode::G_ADD*//*Label [[CASE_ADD_NUM:[0-9]+]]*/ [[CASE_ADD:[0-9]+]],
-// R20O:       /*TargetOpcode::G_SUB*//*Label [[CASE_SUB_NUM:[0-9]+]]*/ [[CASE_SUB:[0-9]+]],
+// R20O-NEXT:  GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2({{[0-9]+}}), GIMT_Encode2({{[0-9]+}}), /*)*//*default:*//*Label [[DEFAULT_NUM:[0-9]+]]*/ GIMT_Encode4([[DEFAULT:[0-9]+]]),
+// R20O-NEXT:  /*TargetOpcode::G_ADD*//*Label [[CASE_ADD_NUM:[0-9]+]]*/ GIMT_Encode4([[CASE_ADD:[0-9]+]]),
+// R20O:       /*TargetOpcode::G_SUB*//*Label [[CASE_SUB_NUM:[0-9]+]]*/ GIMT_Encode4([[CASE_SUB:[0-9]+]]),
 // R20O:       // Label [[CASE_ADD_NUM]]: @[[CASE_ADD]]
 // R20O:       // Label [[CASE_SUB_NUM]]: @[[CASE_SUB]]
-// R20O-NEXT:  GIM_Try, /*On fail goto*//*Label [[GROUP_NUM:[0-9]+]]*/ [[GROUP:[0-9]+]],
+// R20O-NEXT:  GIM_Try, /*On fail goto*//*Label [[GROUP_NUM:[0-9]+]]*/ GIMT_Encode4([[GROUP:[0-9]+]]),
 // R20O-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
 // R20O-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32,
 // R20O-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32,
-// R20O-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID,
+// R20O-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 //
-// R20N:       GIM_Try, /*On fail goto*//*Label [[PREV_NUM:[0-9]+]]*/ [[PREV:[0-9]+]], // Rule ID 21 //
+// R20N:       GIM_Try, /*On fail goto*//*Label [[PREV_NUM:[0-9]+]]*/ GIMT_Encode4([[PREV:[0-9]+]]), // Rule ID 21 //
 // R20N:       // Label [[PREV_NUM]]: @[[PREV]]
 //
-// R20C-NEXT:  GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ [[LABEL:[0-9]+]], // Rule ID 20 //
+// R20C-NEXT:  GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ GIMT_Encode4([[LABEL:[0-9]+]]), // Rule ID 20 //
 //
 // R20N-NEXT:    GIM_CheckNumOperands, /*MI*/0, /*Expected*/3,
-// R20N-NEXT:    GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_SUB,
+// R20N-NEXT:    GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_SUB),
 // R20N-NEXT:    // MIs[0] DstI[dst]
 // R20N-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
-// R20N-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID,
+// R20N-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // R20N-NEXT:    // MIs[0] src1
 // R20N-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32,
 //
-// R20N-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/MyTarget::GPR32RegClassID,
+// R20N-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // R20N-NEXT:    // MIs[0] src2
 // R20N-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32,
-// R20O-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/MyTarget::GPR32RegClassID,
-// R20C-NEXT:    GIM_CheckComplexPattern, /*MI*/0, /*Op*/2, /*Renderer*/0, GICP_gi_complex,
+// R20O-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
+// R20C-NEXT:    GIM_CheckComplexPattern, /*MI*/0, /*Op*/2, /*Renderer*/GIMT_Encode2(0), GIMT_Encode2(GICP_gi_complex),
 // R20C-NEXT:    // (sub:{ *:[i32] } GPR32:{ *:[i32] }:$src1, complex:{ *:[i32] }:$src2) => (INSN1:{ *:[i32] } GPR32:{ *:[i32] }:$src1, complex:{ *:[i32] }:$src2)
-// R20C-NEXT:    GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::INSN1,
-// R20C-NEXT:    GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst]
+// R20C-NEXT:    GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::INSN1),
+// R20C-NEXT:    GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, //  DstI[dst]
 // R20C-NEXT:    GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // src1
-// R20C-NEXT:    GIR_ComplexRenderer, /*InsnID*/0, /*RendererID*/0,
+// R20C-NEXT:    GIR_ComplexRenderer, /*InsnID*/0, /*RendererID*/GIMT_Encode2(0),
 // R20C-NEXT:    GIR_EraseFromParent, /*InsnID*/0,
 // R20C-NEXT:    GIR_ConstrainSelectedInstOperands, /*InsnID*/0,
 // R20C-NEXT:    // GIR_Coverage, 20,
@@ -444,64 +444,64 @@ def : Pat<(frag GPR32:$src1, complex:$src2, complex:$src3),
 
 //===- Test a more complex multi-instruction match. -----------------------===//
 //
-// R00O-NEXT:  GIM_SwitchOpcode, /*MI*/0, /*[*/{{[0-9]+}}, {{[0-9]+}}, /*)*//*default:*//*Label [[DEFAULT_NUM:[0-9]+]]*/ [[DEFAULT:[0-9]+]],
-// R00O-NEXT:  /*TargetOpcode::G_ADD*//*Label [[CASE_ADD_NUM:[0-9]+]]*/ [[CASE_ADD:[0-9]+]],
-// R00O:       /*TargetOpcode::G_SUB*//*Label [[CASE_SUB_NUM:[0-9]+]]*/ [[CASE_SUB:[0-9]+]],
+// R00O-NEXT:  GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2({{[0-9]+}}), GIMT_Encode2({{[0-9]+}}), /*)*//*default:*//*Label [[DEFAULT_NUM:[0-9]+]]*/ GIMT_Encode4([[DEFAULT:[0-9]+]]),
+// R00O-NEXT:  /*TargetOpcode::G_ADD*//*Label [[CASE_ADD_NUM:[0-9]+]]*/ GIMT_Encode4([[CASE_ADD:[0-9]+]]),
+// R00O:       /*TargetOpcode::G_SUB*//*Label [[CASE_SUB_NUM:[0-9]+]]*/ GIMT_Encode4([[CASE_SUB:[0-9]+]]),
 // R00O:       // Label [[CASE_ADD_NUM]]: @[[CASE_ADD]]
 // R00O:       // Label [[CASE_SUB_NUM]]: @[[CASE_SUB]]
-// R00O-NEXT:  GIM_Try, /*On fail goto*//*Label [[GROUP_NUM:[0-9]+]]*/ [[GROUP:[0-9]+]],
+// R00O-NEXT:  GIM_Try, /*On fail goto*//*Label [[GROUP_NUM:[0-9]+]]*/ GIMT_Encode4([[GROUP:[0-9]+]]),
 // R00O-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
 // R00O-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32,
 // R00O-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32,
-// R00O-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID,
+// R00O-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 //
-// R00C:       GIM_Try, /*On fail goto*//*Label [[PREV_NUM:[0-9]+]]*/ [[PREV:[0-9]+]], // Rule ID 20 //
+// R00C:       GIM_Try, /*On fail goto*//*Label [[PREV_NUM:[0-9]+]]*/ GIMT_Encode4([[PREV:[0-9]+]]), // Rule ID 20 //
 // R00C:       // Label [[PREV_NUM]]: @[[PREV]]
 //
-// R00C-NEXT:  GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ [[LABEL:[0-9]+]], // Rule ID 0 //
-// R00C-NEXT:    GIM_CheckFeatures, GIFBS_HasA,
+// R00C-NEXT:  GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ GIMT_Encode4([[LABEL:[0-9]+]]), // Rule ID 0 //
+// R00C-NEXT:    GIM_CheckFeatures, GIMT_Encode2(GIFBS_HasA),
 // R00N-NEXT:    GIM_CheckNumOperands, /*MI*/0, /*Expected*/3,
-// R00N-NEXT:    GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_SUB,
+// R00N-NEXT:    GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_SUB),
 // R00N-NEXT:    // MIs[0] DstI[dst]
 // R00N-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
-// R00N-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID,
+// R00N-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // R00N-NEXT:    // MIs[0] Operand 1
 // R00N-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32,
 // R00C-NEXT:    GIM_RecordInsn, /*DefineMI*/1, /*MI*/0, /*OpIdx*/1, // MIs[1]
 // R00N-NEXT:    GIM_CheckNumOperands, /*MI*/1, /*Expected*/3,
-// R00C-NEXT:    GIM_CheckOpcode, /*MI*/1, TargetOpcode::G_SUB,
+// R00C-NEXT:    GIM_CheckOpcode, /*MI*/1, GIMT_Encode2(TargetOpcode::G_SUB),
 // R00N-NEXT:    // MIs[1] Operand 0
 // R00N-NEXT:    GIM_CheckType, /*MI*/1, /*Op*/0, /*Type*/GILLT_s32,
 // R00N-NEXT:    // MIs[1] src1
 // R00C-NEXT:    GIM_CheckType, /*MI*/1, /*Op*/1, /*Type*/GILLT_s32,
 // R00O-NEXT:    GIM_CheckType, /*MI*/1, /*Op*/2, /*Type*/GILLT_s32,
-// R00N-NEXT:    GIM_CheckRegBankForClass, /*MI*/1, /*Op*/1, /*RC*/MyTarget::GPR32RegClassID,
+// R00N-NEXT:    GIM_CheckRegBankForClass, /*MI*/1, /*Op*/1, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // R00N-NEXT:    // MIs[1] src2
 // R00N-NEXT:    GIM_CheckType, /*MI*/1, /*Op*/2, /*Type*/GILLT_s32,
-// R00N-NEXT:    GIM_CheckRegBankForClass, /*MI*/1, /*Op*/2, /*RC*/MyTarget::GPR32RegClassID,
+// R00N-NEXT:    GIM_CheckRegBankForClass, /*MI*/1, /*Op*/2, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // R00N-NEXT:    // MIs[0] Operand 2
 // R00N-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32,
-// R00O-NEXT:    GIM_CheckRegBankForClass, /*MI*/1, /*Op*/1, /*RC*/MyTarget::GPR32RegClassID,
-// R00O-NEXT:    GIM_CheckRegBankForClass, /*MI*/1, /*Op*/2, /*RC*/MyTarget::GPR32RegClassID,
+// R00O-NEXT:    GIM_CheckRegBankForClass, /*MI*/1, /*Op*/1, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
+// R00O-NEXT:    GIM_CheckRegBankForClass, /*MI*/1, /*Op*/2, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // R00C-NEXT:    GIM_RecordInsn, /*DefineMI*/2, /*MI*/0, /*OpIdx*/2, // MIs[2]
 // R00N-NEXT:    GIM_CheckNumOperands, /*MI*/2, /*Expected*/3,
-// R00C-NEXT:    GIM_CheckOpcode, /*MI*/2, TargetOpcode::G_SUB,
+// R00C-NEXT:    GIM_CheckOpcode, /*MI*/2, GIMT_Encode2(TargetOpcode::G_SUB),
 // R00N-NEXT:    // MIs[2] Operand 0
 // R00N-NEXT:    GIM_CheckType, /*MI*/2, /*Op*/0, /*Type*/GILLT_s32,
 // R00N-NEXT:    // MIs[2] src3
 // R00C-NEXT:    GIM_CheckType, /*MI*/2, /*Op*/1, /*Type*/GILLT_s32,
 // R00O-NEXT:    GIM_CheckType, /*MI*/2, /*Op*/2, /*Type*/GILLT_s32,
-// R00N-NEXT:    GIM_CheckRegBankForClass, /*MI*/2, /*Op*/1, /*RC*/MyTarget::GPR32RegClassID,
+// R00N-NEXT:    GIM_CheckRegBankForClass, /*MI*/2, /*Op*/1, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // R00N-NEXT:    // MIs[2] src4
 // R00N-NEXT:    GIM_CheckType, /*MI*/2, /*Op*/2, /*Type*/GILLT_s32,
-// R00N-NEXT:    GIM_CheckRegBankForClass, /*MI*/2, /*Op*/2, /*RC*/MyTarget::GPR32RegClassID,
-// R00O-NEXT:    GIM_CheckRegBankForClass, /*MI*/2, /*Op*/1, /*RC*/MyTarget::GPR32RegClassID,
-// R00O-NEXT:    GIM_CheckRegBankForClass, /*MI*/2, /*Op*/2, /*RC*/MyTarget::GPR32RegClassID,
+// R00N-NEXT:    GIM_CheckRegBankForClass, /*MI*/2, /*Op*/2, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
+// R00O-NEXT:    GIM_CheckRegBankForClass, /*MI*/2, /*Op*/1, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
+// R00O-NEXT:    GIM_CheckRegBankForClass, /*MI*/2, /*Op*/2, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // R00C-NEXT:    GIM_CheckIsSafeToFold, /*InsnID*/1,
 // R00C-NEXT:    GIM_CheckIsSafeToFold, /*InsnID*/2,
 // R00C-NEXT:    // (sub:{ *:[i32] } (sub:{ *:[i32] } GPR32:{ *:[i32] }:$src1, GPR32:{ *:[i32] }:$src2), (sub:{ *:[i32] } GPR32:{ *:[i32] }:$src3, GPR32:{ *:[i32] }:$src4)) => (INSNBOB:{ *:[i32] } GPR32:{ *:[i32] }:$src1, GPR32:{ *:[i32] }:$src2, GPR32:{ *:[i32] }:$src3, GPR32:{ *:[i32] }:$src4)
-// R00C-NEXT:    GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::INSNBOB,
-// R00C-NEXT:    GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst]
+// R00C-NEXT:    GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::INSNBOB),
+// R00C-NEXT:    GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, //  DstI[dst]
 // R00C-NEXT:    GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/1, /*OpIdx*/1, // src1
 // R00C-NEXT:    GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/1, /*OpIdx*/2, // src2
 // R00C-NEXT:    GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/2, /*OpIdx*/1, // src3
@@ -517,7 +517,7 @@ def : Pat<(frag GPR32:$src1, complex:$src2, complex:$src3),
 // R00O-NEXT:  GIM_Reject,
 // R00O:       // Label [[DEFAULT_NUM]]: @[[DEFAULT]]
 // R00O-NEXT:  GIM_Reject,
-// R00O-NEXT:  }; // Size: 9888 bytes
+// R00O-NEXT:  }; // Size: 1978 bytes
 
 def INSNBOB : I<(outs GPR32:$dst), (ins GPR32:$src1, GPR32:$src2, GPR32:$src3, GPR32:$src4),
                  [(set GPR32:$dst,
@@ -526,36 +526,36 @@ def INSNBOB : I<(outs GPR32:$dst), (ins GPR32:$src1, GPR32:$src2, GPR32:$src3, G
 
 //===- Test a simple pattern with an intrinsic. ---------------------------===//
 //
-// R01O-NEXT:  GIM_SwitchOpcode, /*MI*/0, /*[*/{{[0-9]+}}, {{[0-9]+}}, /*)*//*default:*//*Label [[DEFAULT_NUM:[0-9]+]]*/ [[DEFAULT:[0-9]+]],
-// R01O-NEXT:  /*TargetOpcode::G_ADD*//*Label [[CASE_ADD_NUM:[0-9]+]]*/ [[CASE_ADD:[0-9]+]],
-// R01O:       /*TargetOpcode::G_INTRINSIC*//*Label [[CASE_INTRINSIC_NUM:[0-9]+]]*/ [[CASE_INTRINSIC:[0-9]+]],
+// R01O-NEXT:  GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2({{[0-9]+}}), GIMT_Encode2({{[0-9]+}}), /*)*//*default:*//*Label [[DEFAULT_NUM:[0-9]+]]*/ GIMT_Encode4([[DEFAULT:[0-9]+]]),
+// R01O-NEXT:  /*TargetOpcode::G_ADD*//*Label [[CASE_ADD_NUM:[0-9]+]]*/ GIMT_Encode4([[CASE_ADD:[0-9]+]]),
+// R01O:       /*TargetOpcode::G_INTRINSIC*//*Label [[CASE_INTRINSIC_NUM:[0-9]+]]*/ GIMT_Encode4([[CASE_INTRINSIC:[0-9]+]]),
 // R01O:       // Label [[CASE_ADD_NUM]]: @[[CASE_ADD]]
 // R01O:       // Label [[CASE_INTRINSIC_NUM]]: @[[CASE_INTRINSIC]]
 //
-// R01N:       GIM_Try, /*On fail goto*//*Label [[PREV_NUM:[0-9]+]]*/ [[PREV:[0-9]+]], // Rule ID 0 //
+// R01N:       GIM_Try, /*On fail goto*//*Label [[PREV_NUM:[0-9]+]]*/ GIMT_Encode4([[PREV:[0-9]+]]), // Rule ID 0 //
 // R01N:       // Label [[PREV_NUM]]: @[[PREV]]
 //
-// R01C-NEXT:  GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ [[LABEL:[0-9]+]], // Rule ID 1 //
+// R01C-NEXT:  GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ GIMT_Encode4([[LABEL:[0-9]+]]), // Rule ID 1 //
 // R01C-NEXT:    GIM_CheckNumOperands, /*MI*/0, /*Expected*/3,
 //
-// R01O-NEXT:    GIM_CheckIntrinsicID, /*MI*/0, /*Op*/1, Intrinsic::mytarget_nop,
+// R01O-NEXT:    GIM_CheckIntrinsicID, /*MI*/0, /*Op*/1, GIMT_Encode2(Intrinsic::mytarget_nop),
 // R01O-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
 // R01O-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32,
-// R01O-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID,
+// R01O-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 //
-// R01N-NEXT:    GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_INTRINSIC,
+// R01N-NEXT:    GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_INTRINSIC),
 // R01N-NEXT:    // MIs[0] DstI[dst]
 // R01N-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
-// R01N-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID,
+// R01N-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // R01N-NEXT:    // MIs[0] Operand 1
-// R01N-NEXT:    GIM_CheckIntrinsicID, /*MI*/0, /*Op*/1, Intrinsic::mytarget_nop,
+// R01N-NEXT:    GIM_CheckIntrinsicID, /*MI*/0, /*Op*/1, GIMT_Encode2(Intrinsic::mytarget_nop),
 // R01N-NEXT:    // MIs[0] src1
 // R01N-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32,
 //
-// R01C-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/2, /*RC*/MyTarget::GPR32RegClassID,
+// R01C-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/2, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // R01C-NEXT:    // (intrinsic_wo_chain:{ *:[i32] } [[ID:[0-9]+]]:{ *:[iPTR] }, GPR32:{ *:[i32] }:$src1) => (MOV:{ *:[i32] } GPR32:{ *:[i32] }:$src1)
-// R01C-NEXT:    GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::MOV,
-// R01C-NEXT:    GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst]
+// R01C-NEXT:    GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::MOV),
+// R01C-NEXT:    GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, //  DstI[dst]
 // R01C-NEXT:    GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/2, // src1
 // R01C-NEXT:    GIR_EraseFromParent, /*InsnID*/0,
 // R01C-NEXT:    GIR_ConstrainSelectedInstOperands, /*InsnID*/0,
@@ -572,39 +572,39 @@ def MOV : I<(outs GPR32:$dst), (ins GPR32:$src1),
 
 //===- Test a simple pattern with a default operand. ----------------------===//
 //
-// R02O-NEXT:  GIM_SwitchOpcode, /*MI*/0, /*[*/{{[0-9]+}}, {{[0-9]+}}, /*)*//*default:*//*Label [[DEFAULT_NUM:[0-9]+]]*/ [[DEFAULT:[0-9]+]],
-// R02O-NEXT:  /*TargetOpcode::G_ADD*//*Label [[CASE_ADD_NUM:[0-9]+]]*/ [[CASE_ADD:[0-9]+]],
-// R02O:       /*TargetOpcode::G_XOR*//*Label [[CASE_XOR_NUM:[0-9]+]]*/ [[CASE_XOR:[0-9]+]],
+// R02O-NEXT:  GIM_SwitchOpcode, /*MI*/0, /*[*/GIMT_Encode2({{[0-9]+}}), GIMT_Encode2({{[0-9]+}}), /*)*//*default:*//*Label [[DEFAULT_NUM:[0-9]+]]*/ GIMT_Encode4([[DEFAULT:[0-9]+]]),
+// R02O-NEXT:  /*TargetOpcode::G_ADD*//*Label [[CASE_ADD_NUM:[0-9]+]]*/ GIMT_Encode4([[CASE_ADD:[0-9]+]]),
+// R02O:       /*TargetOpcode::G_XOR*//*Label [[CASE_XOR_NUM:[0-9]+]]*/ GIMT_Encode4([[CASE_XOR:[0-9]+]]),
 // R02O:       // Label [[CASE_ADD_NUM]]: @[[CASE_ADD]]
 // R02O:       // Label [[CASE_XOR_NUM]]: @[[CASE_XOR]]
-// R02O-NEXT:  GIM_Try, /*On fail goto*//*Label [[GROUP_NUM:[0-9]+]]*/ [[GROUP:[0-9]+]],
+// R02O-NEXT:  GIM_Try, /*On fail goto*//*Label [[GROUP_NUM:[0-9]+]]*/ GIMT_Encode4([[GROUP:[0-9]+]]),
 // R02O-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
 // R02O-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32,
 // R02O-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32,
-// R02O-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID,
-// R02O-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/MyTarget::GPR32RegClassID,
+// R02O-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
+// R02O-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 //
-// R02N:       GIM_Try, /*On fail goto*//*Label [[PREV_NUM:[0-9]+]]*/ [[PREV:[0-9]+]], // Rule ID 1 //
+// R02N:       GIM_Try, /*On fail goto*//*Label [[PREV_NUM:[0-9]+]]*/ GIMT_Encode4([[PREV:[0-9]+]]), // Rule ID 1 //
 // R02N:       // Label [[PREV_NUM]]: @[[PREV]]
 //
-// R02C-NEXT:  GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ [[LABEL:[0-9]+]], // Rule ID 2 //
+// R02C-NEXT:  GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ GIMT_Encode4([[LABEL:[0-9]+]]), // Rule ID 2 //
 //
 // R02N-NEXT:    GIM_CheckNumOperands, /*MI*/0, /*Expected*/3,
-// R02N-NEXT:    GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_XOR,
+// R02N-NEXT:    GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_XOR),
 // R02N-NEXT:    // MIs[0] DstI[dst]
 // R02N-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
-// R02N-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID,
+// R02N-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // R02N-NEXT:    // MIs[0] src1
 // R02N-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32,
-// R02N-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/MyTarget::GPR32RegClassID,
+// R02N-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // R02N-NEXT:    // MIs[0] Operand 2
 // R02N-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32,
 //
-// R02C-NEXT:    GIM_CheckConstantInt, /*MI*/0, /*Op*/2, -2
+// R02C-NEXT:    GIM_CheckConstantInt8, /*MI*/0, /*Op*/2, uint8_t(-2)
 // R02C-NEXT:    // (xor:{ *:[i32] } GPR32:{ *:[i32] }:$src1, -2:{ *:[i32] }) => (XORI:{ *:[i32] } GPR32:{ *:[i32] }:$src1)
-// R02C-NEXT:    GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::XORI,
+// R02C-NEXT:    GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::XORI),
 // R02C-NEXT:    GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst]
-// R02C-NEXT:    GIR_AddImm, /*InsnID*/0, /*Imm*/-1,
+// R02C-NEXT:    GIR_AddImm8, /*InsnID*/0, /*Imm*/uint8_t(-1),
 // R02C-NEXT:    GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // src1
 // R02C-NEXT:    GIR_EraseFromParent, /*InsnID*/0,
 // R02C-NEXT:    GIR_ConstrainSelectedInstOperands, /*InsnID*/0,
@@ -621,22 +621,22 @@ def XORI : I<(outs GPR32:$dst), (ins m1:$src2, GPR32:$src1),
 
 //===- Test a simple pattern with a default register operand. -------------===//
 //
-// NOOPT-NEXT:  GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ [[LABEL:[0-9]+]],
+// NOOPT-NEXT:  GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ GIMT_Encode4([[LABEL:[0-9]+]]),
 // NOOPT-NEXT:    GIM_CheckNumOperands, /*MI*/0, /*Expected*/3,
-// NOOPT-NEXT:    GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_XOR,
+// NOOPT-NEXT:    GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_XOR),
 // NOOPT-NEXT:    // MIs[0] DstI[dst]
 // NOOPT-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
-// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID,
+// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // NOOPT-NEXT:    // MIs[0] src1
 // NOOPT-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32,
-// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/MyTarget::GPR32RegClassID,
+// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // NOOPT-NEXT:    // MIs[0] Operand 2
 // NOOPT-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32,
-// NOOPT-NEXT:    GIM_CheckConstantInt, /*MI*/0, /*Op*/2, -3
+// NOOPT-NEXT:    GIM_CheckConstantInt8, /*MI*/0, /*Op*/2, uint8_t(-3)
 // NOOPT-NEXT:    // (xor:{ *:[i32] } GPR32:{ *:[i32] }:$src1, -3:{ *:[i32] }) => (XOR:{ *:[i32] } GPR32:{ *:[i32] }:$src1)
-// NOOPT-NEXT:    GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::XOR,
-// NOOPT-NEXT:    GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst]
-// NOOPT-NEXT:    GIR_AddRegister, /*InsnID*/0, MyTarget::R0,
+// NOOPT-NEXT:    GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::XOR),
+// NOOPT-NEXT:    GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, //  DstI[dst]
+// NOOPT-NEXT:    GIR_AddRegister, /*InsnID*/0, GIMT_Encode2(MyTarget::R0),
 // NOOPT-NEXT:    GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // src1
 // NOOPT-NEXT:    GIR_EraseFromParent, /*InsnID*/0,
 // NOOPT-NEXT:    GIR_ConstrainSelectedInstOperands, /*InsnID*/0,
@@ -650,23 +650,23 @@ def XOR : I<(outs GPR32:$dst), (ins Z:$src2, GPR32:$src1),
 
 //===- Test a simple pattern with a multiple default operands. ------------===//
 //
-// NOOPT-NEXT:  GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ [[LABEL:[0-9]+]],
+// NOOPT-NEXT:  GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ GIMT_Encode4([[LABEL:[0-9]+]]),
 // NOOPT-NEXT:    GIM_CheckNumOperands, /*MI*/0, /*Expected*/3,
-// NOOPT-NEXT:    GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_XOR,
+// NOOPT-NEXT:    GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_XOR),
 // NOOPT-NEXT:    // MIs[0] DstI[dst]
 // NOOPT-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
-// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID,
+// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // NOOPT-NEXT:    // MIs[0] src1
 // NOOPT-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32,
-// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/MyTarget::GPR32RegClassID,
+// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // NOOPT-NEXT:    // MIs[0] Operand 2
 // NOOPT-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32,
-// NOOPT-NEXT:    GIM_CheckConstantInt, /*MI*/0, /*Op*/2, -4
+// NOOPT-NEXT:    GIM_CheckConstantInt8, /*MI*/0, /*Op*/2, uint8_t(-4)
 // NOOPT-NEXT:    // (xor:{ *:[i32] } GPR32:{ *:[i32] }:$src1, -4:{ *:[i32] }) => (XORlike:{ *:[i32] } GPR32:{ *:[i32] }:$src1)
-// NOOPT-NEXT:    GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::XORlike,
+// NOOPT-NEXT:    GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::XORlike),
 // NOOPT-NEXT:    GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst]
-// NOOPT-NEXT:    GIR_AddImm, /*InsnID*/0, /*Imm*/-1,
-// NOOPT-NEXT:    GIR_AddRegister, /*InsnID*/0, MyTarget::R0,
+// NOOPT-NEXT:    GIR_AddImm8, /*InsnID*/0, /*Imm*/uint8_t(-1),
+// NOOPT-NEXT:    GIR_AddRegister, /*InsnID*/0, GIMT_Encode2(MyTarget::R0),
 // NOOPT-NEXT:    GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // src1
 // NOOPT-NEXT:    GIR_EraseFromParent, /*InsnID*/0,
 // NOOPT-NEXT:    GIR_ConstrainSelectedInstOperands, /*InsnID*/0,
@@ -680,24 +680,24 @@ def XORlike : I<(outs GPR32:$dst), (ins m1Z:$src2, GPR32:$src1),
 
 //===- Test a simple pattern with multiple operands with defaults. --------===//
 //
-// NOOPT-NEXT:  GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ [[LABEL:[0-9]+]],
+// NOOPT-NEXT:  GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ GIMT_Encode4([[LABEL:[0-9]+]]),
 // NOOPT-NEXT:    GIM_CheckNumOperands, /*MI*/0, /*Expected*/3,
-// NOOPT-NEXT:    GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_XOR,
+// NOOPT-NEXT:    GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_XOR),
 // NOOPT-NEXT:    // MIs[0] DstI[dst]
 // NOOPT-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
-// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID,
+// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // NOOPT-NEXT:    // MIs[0] src1
 // NOOPT-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32,
-// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/MyTarget::GPR32RegClassID,
+// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // NOOPT-NEXT:    // MIs[0] Operand 2
 // NOOPT-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32,
-// NOOPT-NEXT:    GIM_CheckConstantInt, /*MI*/0, /*Op*/2, -5,
+// NOOPT-NEXT:    GIM_CheckConstantInt8, /*MI*/0, /*Op*/2, uint8_t(-5),
 // NOOPT-NEXT:    // (xor:{ *:[i32] } GPR32:{ *:[i32] }:$src1, -5:{ *:[i32] }) => (XORManyDefaults:{ *:[i32] } GPR32:{ *:[i32] }:$src1)
-// NOOPT-NEXT:    GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::XORManyDefaults,
+// NOOPT-NEXT:    GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::XORManyDefaults),
 // NOOPT-NEXT:    GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst]
-// NOOPT-NEXT:    GIR_AddImm, /*InsnID*/0, /*Imm*/-1,
-// NOOPT-NEXT:    GIR_AddRegister, /*InsnID*/0, MyTarget::R0,
-// NOOPT-NEXT:    GIR_AddRegister, /*InsnID*/0, MyTarget::R0,
+// NOOPT-NEXT:    GIR_AddImm8, /*InsnID*/0, /*Imm*/uint8_t(-1),
+// NOOPT-NEXT:    GIR_AddRegister, /*InsnID*/0, GIMT_Encode2(MyTarget::R0),
+// NOOPT-NEXT:    GIR_AddRegister, /*InsnID*/0, GIMT_Encode2(MyTarget::R0),
 // NOOPT-NEXT:    GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // src1
 // NOOPT-NEXT:    GIR_EraseFromParent, /*InsnID*/0,
 // NOOPT-NEXT:    GIR_ConstrainSelectedInstOperands, /*InsnID*/0,
@@ -714,22 +714,22 @@ def XORManyDefaults : I<(outs GPR32:$dst), (ins m1Z:$src3, Z:$src2, GPR32:$src1)
 // This must precede the 3-register variants because constant immediates have
 // priority over register banks.
 //
-// NOOPT-NEXT:  GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ [[LABEL:[0-9]+]],
+// NOOPT-NEXT:  GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ GIMT_Encode4([[LABEL:[0-9]+]]),
 // NOOPT-NEXT:    GIM_CheckNumOperands, /*MI*/0, /*Expected*/3,
-// NOOPT-NEXT:    GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_XOR,
+// NOOPT-NEXT:    GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_XOR),
 // NOOPT-NEXT:    // MIs[0] DstI[dst]
 // NOOPT-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
-// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID,
+// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // NOOPT-NEXT:    // MIs[0] Wm
 // NOOPT-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32,
-// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/MyTarget::GPR32RegClassID,
+// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // NOOPT-NEXT:    // MIs[0] Operand 2
 // NOOPT-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32,
-// NOOPT-NEXT:    GIM_CheckConstantInt, /*MI*/0, /*Op*/2, -1,
+// NOOPT-NEXT:    GIM_CheckConstantInt8, /*MI*/0, /*Op*/2, uint8_t(-1),
 // NOOPT-NEXT:    // (xor:{ *:[i32] } GPR32:{ *:[i32] }:$Wm, -1:{ *:[i32] }) => (ORN:{ *:[i32] } R0:{ *:[i32] }, GPR32:{ *:[i32] }:$Wm)
-// NOOPT-NEXT:    GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::ORN,
-// NOOPT-NEXT:    GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst]
-// NOOPT-NEXT:    GIR_AddRegister, /*InsnID*/0, MyTarget::R0,
+// NOOPT-NEXT:    GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::ORN),
+// NOOPT-NEXT:    GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, //  DstI[dst]
+// NOOPT-NEXT:    GIR_AddRegister, /*InsnID*/0, GIMT_Encode2(MyTarget::R0),
 // NOOPT-NEXT:    GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // Wm
 // NOOPT-NEXT:    GIR_EraseFromParent, /*InsnID*/0,
 // NOOPT-NEXT:    GIR_ConstrainSelectedInstOperands, /*InsnID*/0,
@@ -742,33 +742,33 @@ def : Pat<(not GPR32:$Wm), (ORN R0, GPR32:$Wm)>;
 
 //===- Test a nested instruction match. -----------------------------------===//
 //
-// NOOPT-NEXT:  GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ [[LABEL:[0-9]+]],
-// NOOPT-NEXT:    GIM_CheckFeatures, GIFBS_HasA,
+// NOOPT-NEXT:  GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ GIMT_Encode4([[LABEL:[0-9]+]]),
+// NOOPT-NEXT:    GIM_CheckFeatures, GIMT_Encode2(GIFBS_HasA),
 // NOOPT-NEXT:    GIM_CheckNumOperands, /*MI*/0, /*Expected*/3,
-// NOOPT-NEXT:    GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_MUL,
+// NOOPT-NEXT:    GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_MUL),
 // NOOPT-NEXT:    // MIs[0] DstI[dst]
 // NOOPT-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
-// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID,
+// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // NOOPT-NEXT:    // MIs[0] Operand 1
 // NOOPT-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32,
 // NOOPT-NEXT:    GIM_RecordInsn, /*DefineMI*/1, /*MI*/0, /*OpIdx*/1, // MIs[1]
 // NOOPT-NEXT:    GIM_CheckNumOperands, /*MI*/1, /*Expected*/3,
-// NOOPT-NEXT:    GIM_CheckOpcode, /*MI*/1, TargetOpcode::G_ADD,
+// NOOPT-NEXT:    GIM_CheckOpcode, /*MI*/1, GIMT_Encode2(TargetOpcode::G_ADD),
 // NOOPT-NEXT:    // MIs[1] Operand 0
 // NOOPT-NEXT:    GIM_CheckType, /*MI*/1, /*Op*/0, /*Type*/GILLT_s32,
 // NOOPT-NEXT:    // MIs[1] src1
 // NOOPT-NEXT:    GIM_CheckType, /*MI*/1, /*Op*/1, /*Type*/GILLT_s32,
-// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/1, /*Op*/1, /*RC*/MyTarget::GPR32RegClassID,
+// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/1, /*Op*/1, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // NOOPT-NEXT:    // MIs[1] src2
 // NOOPT-NEXT:    GIM_CheckType, /*MI*/1, /*Op*/2, /*Type*/GILLT_s32,
-// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/1, /*Op*/2, /*RC*/MyTarget::GPR32RegClassID,
+// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/1, /*Op*/2, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // NOOPT-NEXT:    // MIs[0] src3
 // NOOPT-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32,
-// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/2, /*RC*/MyTarget::GPR32RegClassID,
+// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/2, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // NOOPT-NEXT:    GIM_CheckIsSafeToFold, /*InsnID*/1,
 // NOOPT-NEXT:    // (mul:{ *:[i32] } (add:{ *:[i32] } GPR32:{ *:[i32] }:$src1, GPR32:{ *:[i32] }:$src2), GPR32:{ *:[i32] }:$src3)  =>  (MULADD:{ *:[i32] } GPR32:{ *:[i32] }:$src1, GPR32:{ *:[i32] }:$src2, GPR32:{ *:[i32] }:$src3)
-// NOOPT-NEXT:    GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::MULADD,
-// NOOPT-NEXT:    GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst]
+// NOOPT-NEXT:    GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::MULADD),
+// NOOPT-NEXT:    GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, //  DstI[dst]
 // NOOPT-NEXT:    GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/1, /*OpIdx*/1, // src1
 // NOOPT-NEXT:    GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/1, /*OpIdx*/2, // src2
 // NOOPT-NEXT:    GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/2, // src3
@@ -780,33 +780,33 @@ def : Pat<(not GPR32:$Wm), (ORN R0, GPR32:$Wm)>;
 
 // We also get a second rule by commutativity.
 //
-// NOOPT-NEXT:  GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ [[LABEL:[0-9]+]],
-// NOOPT-NEXT:    GIM_CheckFeatures, GIFBS_HasA,
+// NOOPT-NEXT:  GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ GIMT_Encode4([[LABEL:[0-9]+]]),
+// NOOPT-NEXT:    GIM_CheckFeatures, GIMT_Encode2(GIFBS_HasA),
 // NOOPT-NEXT:    GIM_CheckNumOperands, /*MI*/0, /*Expected*/3,
-// NOOPT-NEXT:    GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_MUL,
+// NOOPT-NEXT:    GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_MUL),
 // NOOPT-NEXT:    // MIs[0] DstI[dst]
 // NOOPT-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
-// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID,
+// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // NOOPT-NEXT:    // MIs[0] src3
 // NOOPT-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32,
-// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/MyTarget::GPR32RegClassID,
+// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // NOOPT-NEXT:    // MIs[0] Operand 2
 // NOOPT-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32,
 // NOOPT-NEXT:    GIM_RecordInsn, /*DefineMI*/1, /*MI*/0, /*OpIdx*/2,
 // NOOPT-NEXT:    GIM_CheckNumOperands, /*MI*/1, /*Expected*/3,
-// NOOPT-NEXT:    GIM_CheckOpcode, /*MI*/1, TargetOpcode::G_ADD,
+// NOOPT-NEXT:    GIM_CheckOpcode, /*MI*/1, GIMT_Encode2(TargetOpcode::G_ADD),
 // NOOPT-NEXT:    // MIs[1] Operand 0
 // NOOPT-NEXT:    GIM_CheckType, /*MI*/1, /*Op*/0, /*Type*/GILLT_s32,
 // NOOPT-NEXT:    // MIs[1] src1
 // NOOPT-NEXT:    GIM_CheckType, /*MI*/1, /*Op*/1, /*Type*/GILLT_s32,
-// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/1, /*Op*/1, /*RC*/MyTarget::GPR32RegClassID,
+// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/1, /*Op*/1, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // NOOPT-NEXT:    // MIs[1] src2
 // NOOPT-NEXT:    GIM_CheckType, /*MI*/1, /*Op*/2, /*Type*/GILLT_s32,
-// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/1, /*Op*/2, /*RC*/MyTarget::GPR32RegClassID,
+// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/1, /*Op*/2, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // NOOPT-NEXT:    GIM_CheckIsSafeToFold, /*InsnID*/1,
 // NOOPT-NEXT:    // (mul:{ *:[i32] } GPR32:{ *:[i32] }:$src3, (add:{ *:[i32] } GPR32:{ *:[i32] }:$src1, GPR32:{ *:[i32] }:$src2))  =>  (MULADD:{ *:[i32] } GPR32:{ *:[i32] }:$src1, GPR32:{ *:[i32] }:$src2, GPR32:{ *:[i32] }:$src3)
-// NOOPT-NEXT:    GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::MULADD,
-// NOOPT-NEXT:    GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst]
+// NOOPT-NEXT:    GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::MULADD),
+// NOOPT-NEXT:    GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, //  DstI[dst]
 // NOOPT-NEXT:    GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/1, /*OpIdx*/1, // src1
 // NOOPT-NEXT:    GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/1, /*OpIdx*/2, // src2
 // NOOPT-NEXT:    GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // src3
@@ -823,17 +823,17 @@ def MULADD : I<(outs GPR32:$dst), (ins GPR32:$src1, GPR32:$src2, GPR32:$src3),
 
 //===- Test a simple pattern with just a specific leaf immediate. ---------===//
 //
-// NOOPT-NEXT:  GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ [[LABEL:[0-9]+]],
+// NOOPT-NEXT:  GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ GIMT_Encode4([[LABEL:[0-9]+]]),
 // NOOPT-NEXT:    GIM_CheckNumOperands, /*MI*/0, /*Expected*/2,
-// NOOPT-NEXT:    GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_CONSTANT,
+// NOOPT-NEXT:    GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_CONSTANT),
 // NOOPT-NEXT:    // MIs[0] DstI[dst]
 // NOOPT-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
-// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID,
+// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // NOOPT-NEXT:    // MIs[0] Operand 1
-// NOOPT-NEXT:    GIM_CheckLiteralInt, /*MI*/0, /*Op*/1, 1,
+// NOOPT-NEXT:    GIM_CheckLiteralInt, /*MI*/0, /*Op*/1, GIMT_Encode8(1),
 // NOOPT-NEXT:    // 1:{ *:[i32] }  =>  (MOV1:{ *:[i32] })
-// NOOPT-NEXT:    GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::MOV1,
-// NOOPT-NEXT:    GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst]
+// NOOPT-NEXT:    GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::MOV1),
+// NOOPT-NEXT:    GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, //  DstI[dst]
 // NOOPT-NEXT:    GIR_EraseFromParent, /*InsnID*/0,
 // NOOPT-NEXT:    GIR_ConstrainSelectedInstOperands, /*InsnID*/0,
 // NOOPT-NEXT:    // GIR_Coverage, 7,
@@ -844,18 +844,18 @@ def MOV1 : I<(outs GPR32:$dst), (ins), [(set GPR32:$dst, 1)]>;
 
 //===- Test a simple pattern with a leaf immediate and a predicate. -------===//
 //
-// NOOPT-NEXT:  GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ [[LABEL:[0-9]+]],
+// NOOPT-NEXT:  GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ GIMT_Encode4([[LABEL:[0-9]+]]),
 // NOOPT-NEXT:    GIM_CheckNumOperands, /*MI*/0, /*Expected*/2,
-// NOOPT-NEXT:    GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_CONSTANT,
-// NOOPT-NEXT:    GIM_CheckI64ImmPredicate, /*MI*/0, /*Predicate*/GICXXPred_I64_Predicate_simm8,
+// NOOPT-NEXT:    GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_CONSTANT),
+// NOOPT-NEXT:    GIM_CheckI64ImmPredicate, /*MI*/0, /*Predicate*/GIMT_Encode2(GICXXPred_I64_Predicate_simm8),
 // NOOPT-NEXT:    // MIs[0] DstI[dst]
 // NOOPT-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
-// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID,
+// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // NOOPT-NEXT:    // MIs[0] Operand 1
 // NOOPT-NEXT:    // No operand predicates
 // NOOPT-NEXT:    // (imm:{ *:[i32] })<<P:Predicate_simm8>>:$imm => (MOVimm8:{ *:[i32] } (imm:{ *:[i32] }):$imm)
-// NOOPT-NEXT:    GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::MOVimm8,
-// NOOPT-NEXT:    GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst]
+// NOOPT-NEXT:    GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::MOVimm8),
+// NOOPT-NEXT:    GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, //  DstI[dst]
 // NOOPT-NEXT:    GIR_CopyConstantAsSImm, /*NewInsnID*/0, /*OldInsnID*/0, // imm
 // NOOPT-NEXT:    GIR_EraseFromParent, /*InsnID*/0,
 // NOOPT-NEXT:    GIR_ConstrainSelectedInstOperands, /*InsnID*/0,
@@ -868,18 +868,18 @@ def MOVimm8 : I<(outs GPR32:$dst), (ins i32imm:$imm), [(set GPR32:$dst, simm8:$i
 
 //===- Same again but use an IntImmLeaf. ----------------------------------===//
 //
-// NOOPT-NEXT:  GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ [[LABEL:[0-9]+]],
+// NOOPT-NEXT:  GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ GIMT_Encode4([[LABEL:[0-9]+]]),
 // NOOPT-NEXT:    GIM_CheckNumOperands, /*MI*/0, /*Expected*/2,
-// NOOPT-NEXT:    GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_CONSTANT,
-// NOOPT-NEXT:    GIM_CheckAPIntImmPredicate, /*MI*/0, /*Predicate*/GICXXPred_APInt_Predicate_simm9,
+// NOOPT-NEXT:    GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_CONSTANT),
+// NOOPT-NEXT:    GIM_CheckAPIntImmPredicate, /*MI*/0, /*Predicate*/GIMT_Encode2(GICXXPred_APInt_Predicate_simm9),
 // NOOPT-NEXT:    // MIs[0] DstI[dst]
 // NOOPT-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
-// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID,
+// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // NOOPT-NEXT:    // MIs[0] Operand 1
 // NOOPT-NEXT:    // No operand predicates
 // NOOPT-NEXT:    // (imm:{ *:[i32] })<<P:Predicate_simm9>>:$imm =>  (MOVimm9:{ *:[i32] } (imm:{ *:[i32] }):$imm)
-// NOOPT-NEXT:    GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::MOVimm9,
-// NOOPT-NEXT:    GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst]
+// NOOPT-NEXT:    GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::MOVimm9),
+// NOOPT-NEXT:    GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, //  DstI[dst]
 // NOOPT-NEXT:    GIR_CopyConstantAsSImm, /*NewInsnID*/0, /*OldInsnID*/0, // imm
 // NOOPT-NEXT:    GIR_EraseFromParent, /*InsnID*/0,
 // NOOPT-NEXT:    GIR_ConstrainSelectedInstOperands, /*InsnID*/0,
@@ -892,19 +892,19 @@ def MOVimm9 : I<(outs GPR32:$dst), (ins i32imm:$imm), [(set GPR32:$dst, simm9:$i
 
 //===- Test a pattern with a custom renderer. -----------------------------===//
 //
-// NOOPT-NEXT:  GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ [[LABEL:[0-9]+]],
+// NOOPT-NEXT:  GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ GIMT_Encode4([[LABEL:[0-9]+]]),
 // NOOPT-NEXT:    GIM_CheckNumOperands, /*MI*/0, /*Expected*/2,
-// NOOPT-NEXT:    GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_CONSTANT,
-// NOOPT-NEXT:    GIM_CheckI64ImmPredicate, /*MI*/0, /*Predicate*/GICXXPred_I64_Predicate_cimm8,
+// NOOPT-NEXT:    GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_CONSTANT),
+// NOOPT-NEXT:    GIM_CheckI64ImmPredicate, /*MI*/0, /*Predicate*/GIMT_Encode2(GICXXPred_I64_Predicate_cimm8),
 // NOOPT-NEXT:    // MIs[0] DstI[dst]
 // NOOPT-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
-// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID,
+// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // NOOPT-NEXT:    // MIs[0] Operand 1
 // NOOPT-NEXT:    // No operand predicates
 // NOOPT-NEXT:    // (imm:{ *:[i32] })<<P:Predicate_cimm8>><<X:cimm8_xform>>:$imm  =>  (MOVcimm8:{ *:[i32] } (cimm8_xform:{ *:[i32] } (imm:{ *:[i32] }):$imm))
-// NOOPT-NEXT:    GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::MOVcimm8,
-// NOOPT-NEXT:    GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst]
-// NOOPT-NEXT:    GIR_CustomRenderer, /*InsnID*/0, /*OldInsnID*/0, /*Renderer*/GICR_renderImm, // imm
+// NOOPT-NEXT:    GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::MOVcimm8),
+// NOOPT-NEXT:    GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, //  DstI[dst]
+// NOOPT-NEXT:    GIR_CustomRenderer, /*InsnID*/0, /*OldInsnID*/0, /*Renderer*/GIMT_Encode2(GICR_renderImm), // imm
 // NOOPT-NEXT:    GIR_EraseFromParent, /*InsnID*/0,
 // NOOPT-NEXT:    GIR_ConstrainSelectedInstOperands, /*InsnID*/0,
 // NOOPT-NEXT:    // GIR_Coverage, 10,
@@ -915,18 +915,18 @@ def MOVcimm8 : I<(outs GPR32:$dst), (ins i32imm:$imm), [(set GPR32:$dst, cimm8:$
 
 //===- Test a simple pattern with a FP immediate and a predicate. ---------===//
 //
-// NOOPT-NEXT:  GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ [[LABEL:[0-9]+]],
+// NOOPT-NEXT:  GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ GIMT_Encode4([[LABEL:[0-9]+]]),
 // NOOPT-NEXT:    GIM_CheckNumOperands, /*MI*/0, /*Expected*/2,
-// NOOPT-NEXT:    GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_FCONSTANT,
-// NOOPT-NEXT:    GIM_CheckAPFloatImmPredicate, /*MI*/0, /*Predicate*/GICXXPred_APFloat_Predicate_fpimmz,
+// NOOPT-NEXT:    GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_FCONSTANT),
+// NOOPT-NEXT:    GIM_CheckAPFloatImmPredicate, /*MI*/0, /*Predicate*/GIMT_Encode2(GICXXPred_APFloat_Predicate_fpimmz),
 // NOOPT-NEXT:    // MIs[0] DstI[dst]
 // NOOPT-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
-// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::FPR32RegClassID,
+// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::FPR32RegClassID),
 // NOOPT-NEXT:    // MIs[0] Operand 1
 // NOOPT-NEXT:    // No operand predicates
 // NOOPT-NEXT:    // (fpimm:{ *:[f32] })<<P:Predicate_fpimmz>>:$imm =>  (MOVfpimmz:{ *:[f32] } (fpimm:{ *:[f32] }):$imm)
-// NOOPT-NEXT:    GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::MOVfpimmz,
-// NOOPT-NEXT:    GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst]
+// NOOPT-NEXT:    GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::MOVfpimmz),
+// NOOPT-NEXT:    GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, //  DstI[dst]
 // NOOPT-NEXT:    GIR_CopyFConstantAsFPImm, /*NewInsnID*/0, /*OldInsnID*/0, // imm
 // NOOPT-NEXT:    GIR_EraseFromParent, /*InsnID*/0,
 // NOOPT-NEXT:    GIR_ConstrainSelectedInstOperands, /*InsnID*/0,
@@ -936,19 +936,19 @@ def MOVcimm8 : I<(outs GPR32:$dst), (ins i32imm:$imm), [(set GPR32:$dst, cimm8:$
 
 //===- Test a simple pattern with inferred pointer operands. ---------------===//
 //
-// NOOPT-NEXT:  GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ [[LABEL:[0-9]+]],
+// NOOPT-NEXT:  GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ GIMT_Encode4([[LABEL:[0-9]+]]),
 // NOOPT-NEXT:    GIM_CheckNumOperands, /*MI*/0, /*Expected*/2,
-// NOOPT-NEXT:    GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_LOAD,
+// NOOPT-NEXT:    GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_LOAD),
 // NOOPT-NEXT:    GIM_CheckMemorySizeEqualToLLT, /*MI*/0, /*MMO*/0, /*OpIdx*/0,
-// NOOPT-NEXT:    GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(int64_t)AtomicOrdering::NotAtomic,
+// NOOPT-NEXT:    GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(uint8_t)AtomicOrdering::NotAtomic,
 // NOOPT-NEXT:    // MIs[0] DstI[dst]
 // NOOPT-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
-// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID,
+// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // NOOPT-NEXT:    // MIs[0] src1
 // NOOPT-NEXT:    GIM_CheckPointerToAny, /*MI*/0, /*Op*/1, /*SizeInBits*/32,
-// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/MyTarget::GPR32RegClassID,
+// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // NOOPT-NEXT:    // (ld:{ *:[i32] } GPR32:{ *:[i32] }:$src1)<<P:Predicate_unindexedload>><<P:Predicate_load>> => (LOAD:{ *:[i32] } GPR32:{ *:[i32] }:$src1)
-// NOOPT-NEXT:    GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/MyTarget::LOAD,
+// NOOPT-NEXT:    GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::LOAD),
 // NOOPT-NEXT:    GIR_ConstrainSelectedInstOperands, /*InsnID*/0,
 // NOOPT-NEXT:    // GIR_Coverage, 11,
 // NOOPT-NEXT:    GIR_Done,
@@ -959,19 +959,19 @@ def LOAD : I<(outs GPR32:$dst), (ins GPR32:$src1),
 
 //===- Test a simple pattern with explicit pointer operands. ---------------===//
 
-// NOOPT-NEXT:  GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ [[LABEL:[0-9]+]],
+// NOOPT-NEXT:  GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ GIMT_Encode4([[LABEL:[0-9]+]]),
 // NOOPT-NEXT:    GIM_CheckNumOperands, /*MI*/0, /*Expected*/2,
-// NOOPT-NEXT:    GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_LOAD,
+// NOOPT-NEXT:    GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_LOAD),
 // NOOPT-NEXT:    GIM_CheckMemorySizeEqualToLLT, /*MI*/0, /*MMO*/0, /*OpIdx*/0,
-// NOOPT-NEXT:    GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(int64_t)AtomicOrdering::NotAtomic,
+// NOOPT-NEXT:    GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(uint8_t)AtomicOrdering::NotAtomic,
 // NOOPT-NEXT:    // MIs[0] DstI[dst]
 // NOOPT-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_p0s32,
-// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID,
+// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // NOOPT-NEXT:    // MIs[0] src
 // NOOPT-NEXT:    GIM_CheckPointerToAny, /*MI*/0, /*Op*/1, /*SizeInBits*/32,
-// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/MyTarget::GPR32RegClassID,
+// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // NOOPT-NEXT:    // (ld:{ *:[i32] } GPR32:{ *:[i32] }:$src)<<P:Predicate_unindexedload>><<P:Predicate_load>>  =>  (LOAD:{ *:[i32] } GPR32:{ *:[i32] }:$src)
-// NOOPT-NEXT:    GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/MyTarget::LOAD,
+// NOOPT-NEXT:    GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::LOAD),
 // NOOPT-NEXT:    GIR_ConstrainSelectedInstOperands, /*InsnID*/0,
 // NOOPT-NEXT:    // GIR_Coverage, 23,
 // NOOPT-NEXT:    GIR_Done,
@@ -982,19 +982,19 @@ def : Pat<(load GPR32:$src),
 
 //===- Test a simple pattern with a sextload -------------------------------===//
 //
-// NOOPT-NEXT:  GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ [[LABEL:[0-9]+]],
+// NOOPT-NEXT:  GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ GIMT_Encode4([[LABEL:[0-9]+]]),
 // NOOPT-NEXT:    GIM_CheckNumOperands, /*MI*/0, /*Expected*/2,
-// NOOPT-NEXT:    GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_SEXTLOAD,
-// NOOPT-NEXT:    GIM_CheckMemorySizeEqualTo, /*MI*/0, /*MMO*/0, /*Size*/2,
-// NOOPT-NEXT:    GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(int64_t)AtomicOrdering::NotAtomic,
+// NOOPT-NEXT:    GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_SEXTLOAD),
+// NOOPT-NEXT:    GIM_CheckMemorySizeEqualTo, /*MI*/0, /*MMO*/0, /*Size*/GIMT_Encode4(2),
+// NOOPT-NEXT:    GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(uint8_t)AtomicOrdering::NotAtomic,
 // NOOPT-NEXT:    // MIs[0] DstI[dst]
 // NOOPT-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
-// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID,
+// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // NOOPT-NEXT:    // MIs[0] src1
 // NOOPT-NEXT:    GIM_CheckPointerToAny, /*MI*/0, /*Op*/1, /*SizeInBits*/32,
-// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/MyTarget::GPR32RegClassID,
+// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // NOOPT-NEXT:    // (ld:{ *:[i32] } GPR32:{ *:[i32] }:$src1)<<P:Predicate_unindexedload>><<P:Predicate_sextload>><<P:Predicate_sextloadi16>>  =>  (SEXTLOAD:{ *:[i32] } GPR32:{ *:[i32] }:$src1)
-// NOOPT-NEXT:    GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/MyTarget::SEXTLOAD,
+// NOOPT-NEXT:    GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::SEXTLOAD),
 // NOOPT-NEXT:    GIR_ConstrainSelectedInstOperands, /*InsnID*/0,
 // NOOPT-NEXT:    // GIR_Coverage, 12,
 // NOOPT-NEXT:    GIR_Done,
@@ -1005,20 +1005,20 @@ def SEXTLOAD : I<(outs GPR32:$dst), (ins GPR32:$src1),
 
 //===- Test a simple pattern with regclass operands. ----------------------===//
 //
-// NOOPT-NEXT:  GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ [[LABEL:[0-9]+]],
+// NOOPT-NEXT:  GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ GIMT_Encode4([[LABEL:[0-9]+]]),
 // NOOPT-NEXT:    GIM_CheckNumOperands, /*MI*/0, /*Expected*/3,
-// NOOPT-NEXT:    GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_ADD,
+// NOOPT-NEXT:    GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_ADD),
 // NOOPT-NEXT:    // MIs[0] DstI[dst]
 // NOOPT-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
-// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID,
+// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // NOOPT-NEXT:    // MIs[0] src1
 // NOOPT-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32,
-// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/MyTarget::GPR32RegClassID
+// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID)
 // NOOPT-NEXT:    // MIs[0] src2
 // NOOPT-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32,
-// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/2, /*RC*/MyTarget::GPR32RegClassID,
+// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/2, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // NOOPT-NEXT:    // (add:{ *:[i32] } GPR32:{ *:[i32] }:$src1, GPR32:{ *:[i32] }:$src2) => (ADD:{ *:[i32] } GPR32:{ *:[i32] }:$src1, GPR32:{ *:[i32] }:$src2)
-// NOOPT-NEXT:    GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/MyTarget::ADD,
+// NOOPT-NEXT:    GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::ADD),
 // NOOPT-NEXT:    GIR_ConstrainSelectedInstOperands, /*InsnID*/0,
 // NOOPT-NEXT:    // GIR_Coverage, 13,
 // NOOPT-NEXT:    GIR_Done,
@@ -1029,20 +1029,20 @@ def ADD : I<(outs GPR32:$dst), (ins GPR32:$src1, GPR32:$src2),
 
 //===- Test a pattern with a tied operand in the matcher ------------------===//
 //
-// NOOPT-NEXT:  GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ [[LABEL:[0-9]+]],
+// NOOPT-NEXT:  GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ GIMT_Encode4([[LABEL:[0-9]+]]),
 // NOOPT-NEXT:    GIM_CheckNumOperands, /*MI*/0, /*Expected*/3,
-// NOOPT-NEXT:    GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_ADD,
+// NOOPT-NEXT:    GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_ADD),
 // NOOPT-NEXT:    // MIs[0] DstI[dst]
 // NOOPT-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
-// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID,
+// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // NOOPT-NEXT:    // MIs[0] src{{$}}
 // NOOPT-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32,
-// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/MyTarget::GPR32RegClassID,
+// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // NOOPT-NEXT:    // MIs[0] src{{$}}
 // NOOPT-NEXT:    GIM_CheckIsSameOperand, /*MI*/0, /*OpIdx*/2, /*OtherMI*/0, /*OtherOpIdx*/1,
 // NOOPT-NEXT:    // (add:{ *:[i32] } GPR32:{ *:[i32] }:$src, GPR32:{ *:[i32] }:$src) => (DOUBLE:{ *:[i32] } GPR32:{ *:[i32] }:$src)
-// NOOPT-NEXT:    GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::DOUBLE,
-// NOOPT-NEXT:    GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst]
+// NOOPT-NEXT:    GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::DOUBLE),
+// NOOPT-NEXT:    GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, //  DstI[dst]
 // NOOPT-NEXT:    GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // src
 // NOOPT-NEXT:    GIR_EraseFromParent, /*InsnID*/0,
 // NOOPT-NEXT:    GIR_ConstrainSelectedInstOperands, /*InsnID*/0,
@@ -1060,18 +1060,18 @@ def DOUBLE : I<(outs GPR32:$dst), (ins GPR32:$src), [(set GPR32:$dst, (add GPR32
 // - one of operands in the pattern itself
 // does not introduce unexpected GIM_CheckIsSameOperand predicate.
 
-// NOOPT-NEXT:  GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ [[LABEL:[0-9]+]],
+// NOOPT-NEXT:  GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ GIMT_Encode4([[LABEL:[0-9]+]]),
 // NOOPT-NEXT:    GIM_CheckNumOperands, /*MI*/0, /*Expected*/3,
-// NOOPT-NEXT:    GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_ADD,
+// NOOPT-NEXT:    GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_ADD),
 // NOOPT-NEXT:    // MIs[0] DstI[samename]
 // NOOPT-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
-// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID,
+// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // NOOPT-NEXT:    // MIs[0] samename
 // NOOPT-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32,
 // NOOPT-NEXT:    // MIs[0] othername
 // NOOPT-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32,
 // NOOPT-NEXT:    // (add:{ *:[i32] } i32:{ *:[i32] }:$samename, i32:{ *:[i32] }:$othername)  =>  (InsnWithSpeciallyNamedDef:{ *:[i32] } i32:{ *:[i32] }:$samename, i32:{ *:[i32] }:$othername)
-// NOOPT-NEXT:    GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/MyTarget::InsnWithSpeciallyNamedDef,
+// NOOPT-NEXT:    GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::InsnWithSpeciallyNamedDef),
 // NOOPT-NEXT:    GIR_ConstrainSelectedInstOperands, /*InsnID*/0,
 // NOOPT-NEXT:    // GIR_Coverage, 24,
 // NOOPT-NEXT:    GIR_Done,
@@ -1083,18 +1083,18 @@ def : Pat<(add i32:$samename, i32:$othername),
 
 //===- Test a simple pattern with ValueType operands. ----------------------===//
 //
-// NOOPT-NEXT:  GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ [[LABEL:[0-9]+]],
+// NOOPT-NEXT:  GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ GIMT_Encode4([[LABEL:[0-9]+]]),
 // NOOPT-NEXT:    GIM_CheckNumOperands, /*MI*/0, /*Expected*/3,
-// NOOPT-NEXT:    GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_ADD,
+// NOOPT-NEXT:    GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_ADD),
 // NOOPT-NEXT:    // MIs[0] DstI[dst]
 // NOOPT-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
-// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID,
+// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // NOOPT-NEXT:    // MIs[0] src1
 // NOOPT-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32,
 // NOOPT-NEXT:    // MIs[0] src2
 // NOOPT-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32,
 // NOOPT-NEXT:    // (add:{ *:[i32] } i32:{ *:[i32] }:$src1, i32:{ *:[i32] }:$src2) => (ADD:{ *:[i32] } i32:{ *:[i32] }:$src1, i32:{ *:[i32] }:$src2)
-// NOOPT-NEXT:    GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/MyTarget::ADD,
+// NOOPT-NEXT:    GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::ADD),
 // NOOPT-NEXT:    GIR_ConstrainSelectedInstOperands, /*InsnID*/0,
 // NOOPT-NEXT:    // GIR_Coverage, 25,
 // NOOPT-NEXT:    GIR_Done,
@@ -1105,22 +1105,22 @@ def : Pat<(add i32:$src1, i32:$src2),
 
 //===- Test another simple pattern with regclass operands. ----------------===//
 //
-// NOOPT-NEXT:  GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ [[LABEL:[0-9]+]],
-// NOOPT-NEXT:    GIM_CheckFeatures, GIFBS_HasA_HasB_HasC,
+// NOOPT-NEXT:  GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ GIMT_Encode4([[LABEL:[0-9]+]]),
+// NOOPT-NEXT:    GIM_CheckFeatures, GIMT_Encode2(GIFBS_HasA_HasB_HasC),
 // NOOPT-NEXT:    GIM_CheckNumOperands, /*MI*/0, /*Expected*/3,
-// NOOPT-NEXT:    GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_MUL,
+// NOOPT-NEXT:    GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_MUL),
 // NOOPT-NEXT:    // MIs[0] DstI[dst]
 // NOOPT-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
-// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID,
+// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // NOOPT-NEXT:    // MIs[0] src1
 // NOOPT-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32,
-// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/MyTarget::GPR32RegClassID,
+// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // NOOPT-NEXT:    // MIs[0] src2
 // NOOPT-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32,
-// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/2, /*RC*/MyTarget::GPR32RegClassID,
+// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/2, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // NOOPT-NEXT:    // (mul:{ *:[i32] } GPR32:{ *:[i32] }:$src1, GPR32:{ *:[i32] }:$src2) => (MUL:{ *:[i32] } GPR32:{ *:[i32] }:$src2, GPR32:{ *:[i32] }:$src1)
-// NOOPT-NEXT:    GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::MUL,
-// NOOPT-NEXT:    GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst]
+// NOOPT-NEXT:    GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::MUL),
+// NOOPT-NEXT:    GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, //  DstI[dst]
 // NOOPT-NEXT:    GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/2, // src2
 // NOOPT-NEXT:    GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // src1
 // NOOPT-NEXT:    GIR_EraseFromParent, /*InsnID*/0,
@@ -1136,18 +1136,18 @@ def MUL : I<(outs GPR32:$dst), (ins GPR32:$src2, GPR32:$src1),
 //===- Test a COPY_TO_REGCLASS --------------------------------------------===//
 //
 //
-// NOOPT-NEXT:  GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ [[LABEL:[0-9]+]],
+// NOOPT-NEXT:  GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ GIMT_Encode4([[LABEL:[0-9]+]]),
 // NOOPT-NEXT:    GIM_CheckNumOperands, /*MI*/0, /*Expected*/2,
-// NOOPT-NEXT:    GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_BITCAST,
+// NOOPT-NEXT:    GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_BITCAST),
 // NOOPT-NEXT:    // MIs[0] DstI[dst]
 // NOOPT-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
-// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID,
+// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // NOOPT-NEXT:    // MIs[0] src1
 // NOOPT-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32,
-// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/MyTarget::FPR32RegClassID,
+// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/GIMT_Encode2(MyTarget::FPR32RegClassID),
 // NOOPT-NEXT:    // (bitconvert:{ *:[i32] } FPR32:{ *:[f32] }:$src1) => (COPY_TO_REGCLASS:{ *:[i32] } FPR32:{ *:[f32] }:$src1, GPR32:{ *:[i32] })
-// NOOPT-NEXT:    GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/TargetOpcode::COPY,
-// NOOPT-NEXT:    GIR_ConstrainOperandRC, /*InsnID*/0, /*Op*/0, MyTarget::GPR32RegClassID,
+// NOOPT-NEXT:    GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/GIMT_Encode2(TargetOpcode::COPY),
+// NOOPT-NEXT:    GIR_ConstrainOperandRC, /*InsnID*/0, /*Op*/0, GIMT_Encode2(MyTarget::GPR32RegClassID),
 // NOOPT-NEXT:    // GIR_Coverage, 26,
 // NOOPT-NEXT:    GIR_Done,
 // NOOPT-NEXT:  // Label [[LABEL_NUM]]: @[[LABEL]]
@@ -1157,17 +1157,17 @@ def : Pat<(i32 (bitconvert FPR32:$src1)),
 
 //===- Test a simple pattern with just a leaf immediate. ------------------===//
 //
-// NOOPT-NEXT:  GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ [[LABEL:[0-9]+]],
+// NOOPT-NEXT:  GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ GIMT_Encode4([[LABEL:[0-9]+]]),
 // NOOPT-NEXT:    GIM_CheckNumOperands, /*MI*/0, /*Expected*/2,
-// NOOPT-NEXT:    GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_CONSTANT,
+// NOOPT-NEXT:    GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_CONSTANT),
 // NOOPT-NEXT:    // MIs[0] DstI[dst]
 // NOOPT-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
-// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID,
+// NOOPT-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // NOOPT-NEXT:    // MIs[0] Operand 1
 // NOOPT-NEXT:    // No operand predicates
 // NOOPT-NEXT:    // (imm:{ *:[i32] }):$imm =>  (MOVimm:{ *:[i32] } (imm:{ *:[i32] }):$imm)
-// NOOPT-NEXT:    GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::MOVimm,
-// NOOPT-NEXT:    GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst]
+// NOOPT-NEXT:    GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::MOVimm),
+// NOOPT-NEXT:    GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, //  DstI[dst]
 // NOOPT-NEXT:    GIR_CopyConstantAsSImm, /*NewInsnID*/0, /*OldInsnID*/0, // imm
 // NOOPT-NEXT:    GIR_EraseFromParent, /*InsnID*/0,
 // NOOPT-NEXT:    GIR_ConstrainSelectedInstOperands, /*InsnID*/0,
@@ -1182,13 +1182,13 @@ def MOVfpimmz : I<(outs FPR32:$dst), (ins f32imm:$imm), [(set FPR32:$dst, fpimmz
 
 //===- Test a pattern with an MBB operand. --------------------------------===//
 //
-// NOOPT-NEXT:  GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ [[LABEL:[0-9]+]],
+// NOOPT-NEXT:  GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ GIMT_Encode4([[LABEL:[0-9]+]]),
 // NOOPT-NEXT:    GIM_CheckNumOperands, /*MI*/0, /*Expected*/1,
-// NOOPT-NEXT:    GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_BR,
+// NOOPT-NEXT:    GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_BR),
 // NOOPT-NEXT:    // MIs[0] target
 // NOOPT-NEXT:    GIM_CheckIsMBB, /*MI*/0, /*Op*/0,
 // NOOPT-NEXT:    // (br (bb:{ *:[Other] }):$target) => (BR (bb:{ *:[Other] }):$target)
-// NOOPT-NEXT:    GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/MyTarget::BR,
+// NOOPT-NEXT:    GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::BR),
 // NOOPT-NEXT:    GIR_ConstrainSelectedInstOperands, /*InsnID*/0,
 // NOOPT-NEXT:    // GIR_Coverage, 18,
 // NOOPT-NEXT:    GIR_Done,
@@ -1198,5 +1198,5 @@ def BR : I<(outs), (ins unknown:$target),
             [(br bb:$target)]>;
 
 // NOOPT-NEXT:    GIM_Reject,
-// NOOPT-NEXT:  }; // Size: 11408 bytes
+// NOOPT-NEXT:  }; // Size: 1680 bytes
 // NOOPT-NEXT:  return MatchTable0;
diff --git a/llvm/test/TableGen/GlobalISelEmitterCustomPredicate.td b/llvm/test/TableGen/GlobalISelEmitterCustomPredicate.td
index 5a75cb129eec1253ee57c7eab3dc8b06816d87d6..ed43ff2f14fa2afcb43fd18b5109d8bedbbd8eab 100644
--- a/llvm/test/TableGen/GlobalISelEmitterCustomPredicate.td
+++ b/llvm/test/TableGen/GlobalISelEmitterCustomPredicate.td
@@ -73,65 +73,65 @@ def and_or_pat : PatFrag<
   let PredicateCodeUsesOperands = 1;
 }
 
-// CHECK: GIM_Try, /*On fail goto*//*Label 0*/ 99, // Rule ID 7 //
-// CHECK-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3,
-// CHECK-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_AND,
-// CHECK-NEXT: // MIs[0] DstI[dst]
-// CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
-// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/Test::DRegsRegClassID,
-// CHECK-NEXT: // MIs[0] src2
-// CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32,
-// CHECK-NEXT: GIM_RecordNamedOperand, /*MI*/0, /*Op*/1, /*StoreIdx*/2, // Name : pred:3:z
-// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/Test::DRegsRegClassID,
-// CHECK-NEXT: // MIs[0] Operand 2
-// CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32,
-// CHECK-NEXT: GIM_RecordInsn, /*DefineMI*/1, /*MI*/0, /*OpIdx*/2, // MIs[1]
-// CHECK-NEXT: GIM_CheckNumOperands, /*MI*/1, /*Expected*/3,
-// CHECK-NEXT: GIM_CheckOpcode, /*MI*/1, TargetOpcode::G_OR,
-// CHECK-NEXT: // MIs[1] Operand 0
-// CHECK-NEXT: GIM_CheckType, /*MI*/1, /*Op*/0, /*Type*/GILLT_s32,
-// CHECK-NEXT: // MIs[1] src0
-// CHECK-NEXT: GIM_CheckType, /*MI*/1, /*Op*/1, /*Type*/GILLT_s32,
-// CHECK-NEXT: GIM_RecordNamedOperand, /*MI*/1, /*Op*/1, /*StoreIdx*/0, // Name : pred:3:x
-// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/1, /*Op*/1, /*RC*/Test::DRegsRegClassID,
-// CHECK-NEXT: // MIs[1] src1
-// CHECK-NEXT: GIM_CheckType, /*MI*/1, /*Op*/2, /*Type*/GILLT_s32,
-// CHECK-NEXT: GIM_RecordNamedOperand, /*MI*/1, /*Op*/2, /*StoreIdx*/1, // Name : pred:3:y
-// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/1, /*Op*/2, /*RC*/Test::DRegsRegClassID,
-// CHECK-NEXT: GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GICXXPred_MI_Predicate_and_or_pat,
-// CHECK-NEXT: GIM_CheckIsSafeToFold, /*InsnID*/1,
-// CHECK-NEXT: // (and:{ *:[i32] } DOP:{ *:[i32] }:$src2:$pred:3:z, (or:{ *:[i32] } DOP:{ *:[i32] }:$src0:$pred:3:x, DOP:{ *:[i32] }:$src1:$pred:3:y))<<P:3:Predicate_and_or_pat>>  =>  (AND_OR:{ *:[i32] } DOP:{ *:[i32] }:$src0, DOP:{ *:[i32] }:$src1, DOP:{ *:[i32] }:$src2)
-// CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::AND_OR,
+// CHECK:      GIM_Try, /*On fail goto*//*Label 0*/ GIMT_Encode4(110), // Rule ID 7 //
+// CHECK-NEXT:   GIM_CheckNumOperands, /*MI*/0, /*Expected*/3,
+// CHECK-NEXT:   GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_AND),
+// CHECK-NEXT:   // MIs[0] DstI[dst]
+// CHECK-NEXT:   GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
+// CHECK-NEXT:   GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(Test::DRegsRegClassID),
+// CHECK-NEXT:   // MIs[0] src2
+// CHECK-NEXT:   GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32,
+// CHECK-NEXT:   GIM_RecordNamedOperand, /*MI*/0, /*Op*/1, /*StoreIdx*/2, // Name : pred:3:z
+// CHECK-NEXT:   GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/GIMT_Encode2(Test::DRegsRegClassID),
+// CHECK-NEXT:   // MIs[0] Operand 2
+// CHECK-NEXT:   GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32,
+// CHECK-NEXT:   GIM_RecordInsn, /*DefineMI*/1, /*MI*/0, /*OpIdx*/2, // MIs[1]
+// CHECK-NEXT:   GIM_CheckNumOperands, /*MI*/1, /*Expected*/3,
+// CHECK-NEXT:   GIM_CheckOpcode, /*MI*/1, GIMT_Encode2(TargetOpcode::G_OR),
+// CHECK-NEXT:   // MIs[1] Operand 0
+// CHECK-NEXT:   GIM_CheckType, /*MI*/1, /*Op*/0, /*Type*/GILLT_s32,
+// CHECK-NEXT:   // MIs[1] src0
+// CHECK-NEXT:   GIM_CheckType, /*MI*/1, /*Op*/1, /*Type*/GILLT_s32,
+// CHECK-NEXT:   GIM_RecordNamedOperand, /*MI*/1, /*Op*/1, /*StoreIdx*/0, // Name : pred:3:x
+// CHECK-NEXT:   GIM_CheckRegBankForClass, /*MI*/1, /*Op*/1, /*RC*/GIMT_Encode2(Test::DRegsRegClassID),
+// CHECK-NEXT:   // MIs[1] src1
+// CHECK-NEXT:   GIM_CheckType, /*MI*/1, /*Op*/2, /*Type*/GILLT_s32,
+// CHECK-NEXT:   GIM_RecordNamedOperand, /*MI*/1, /*Op*/2, /*StoreIdx*/1, // Name : pred:3:y
+// CHECK-NEXT:   GIM_CheckRegBankForClass, /*MI*/1, /*Op*/2, /*RC*/GIMT_Encode2(Test::DRegsRegClassID),
+// CHECK-NEXT:   GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_and_or_pat),
+// CHECK-NEXT:   GIM_CheckIsSafeToFold, /*InsnID*/1,
+// CHECK-NEXT:   // (and:{ *:[i32] } DOP:{ *:[i32] }:$src2:$pred:3:z, (or:{ *:[i32] } DOP:{ *:[i32] }:$src0:$pred:3:x, DOP:{ *:[i32] }:$src1:$pred:3:y))<<P:3:Predicate_and_or_pat>>  =>  (AND_OR:{ *:[i32] } DOP:{ *:[i32] }:$src0, DOP:{ *:[i32] }:$src1, DOP:{ *:[i32] }:$src2)
+// CHECK-NEXT:   GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::AND_OR),
 
-// CHECK: GIM_Try, /*On fail goto*//*Label 1*/ 198, // Rule ID 3 //
-// CHECK-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3,
-// CHECK-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_AND,
-// CHECK-NEXT: // MIs[0] DstI[dst]
-// CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
-// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/Test::DRegsRegClassID,
-// CHECK-NEXT: // MIs[0] Operand 1
-// CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32,
-// CHECK-NEXT: GIM_RecordInsn, /*DefineMI*/1, /*MI*/0, /*OpIdx*/1, // MIs[1]
-// CHECK-NEXT: GIM_CheckNumOperands, /*MI*/1, /*Expected*/3,
-// CHECK-NEXT: GIM_CheckOpcode, /*MI*/1, TargetOpcode::G_OR,
-// CHECK-NEXT: // MIs[1] Operand 0
-// CHECK-NEXT: GIM_CheckType, /*MI*/1, /*Op*/0, /*Type*/GILLT_s32,
-// CHECK-NEXT: // MIs[1] src0
-// CHECK-NEXT: GIM_CheckType, /*MI*/1, /*Op*/1, /*Type*/GILLT_s32,
-// CHECK-NEXT: GIM_RecordNamedOperand, /*MI*/1, /*Op*/1, /*StoreIdx*/0, // Name : pred:3:x
-// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/1, /*Op*/1, /*RC*/Test::DRegsRegClassID,
-// CHECK-NEXT: // MIs[1] src1
-// CHECK-NEXT: GIM_CheckType, /*MI*/1, /*Op*/2, /*Type*/GILLT_s32,
-// CHECK-NEXT: GIM_RecordNamedOperand, /*MI*/1, /*Op*/2, /*StoreIdx*/1, // Name : pred:3:y
-// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/1, /*Op*/2, /*RC*/Test::DRegsRegClassID,
-// CHECK-NEXT: // MIs[0] src2
-// CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32,
-// CHECK-NEXT: GIM_RecordNamedOperand, /*MI*/0, /*Op*/2, /*StoreIdx*/2, // Name : pred:3:z
-// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/2, /*RC*/Test::DRegsRegClassID,
-// CHECK-NEXT: GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GICXXPred_MI_Predicate_and_or_pat,
-// CHECK-NEXT: GIM_CheckIsSafeToFold, /*InsnID*/1,
-// CHECK-NEXT: // (and:{ *:[i32] } (or:{ *:[i32] } DOP:{ *:[i32] }:$src0:$pred:3:x, DOP:{ *:[i32] }:$src1:$pred:3:y), DOP:{ *:[i32] }:$src2:$pred:3:z)<<P:3:Predicate_and_or_pat>>  =>  (AND_OR:{ *:[i32] } DOP:{ *:[i32] }:$src0, DOP:{ *:[i32] }:$src1, DOP:{ *:[i32] }:$src2)
-// CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::AND_OR,
+// CHECK:      GIM_Try, /*On fail goto*//*Label 1*/ GIMT_Encode4(220), // Rule ID 3 //
+// CHECK-NEXT:   GIM_CheckNumOperands, /*MI*/0, /*Expected*/3,
+// CHECK-NEXT:   GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_AND),
+// CHECK-NEXT:   // MIs[0] DstI[dst]
+// CHECK-NEXT:   GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
+// CHECK-NEXT:   GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(Test::DRegsRegClassID),
+// CHECK-NEXT:   // MIs[0] Operand 1
+// CHECK-NEXT:   GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32,
+// CHECK-NEXT:   GIM_RecordInsn, /*DefineMI*/1, /*MI*/0, /*OpIdx*/1, // MIs[1]
+// CHECK-NEXT:   GIM_CheckNumOperands, /*MI*/1, /*Expected*/3,
+// CHECK-NEXT:   GIM_CheckOpcode, /*MI*/1, GIMT_Encode2(TargetOpcode::G_OR),
+// CHECK-NEXT:   // MIs[1] Operand 0
+// CHECK-NEXT:   GIM_CheckType, /*MI*/1, /*Op*/0, /*Type*/GILLT_s32,
+// CHECK-NEXT:   // MIs[1] src0
+// CHECK-NEXT:   GIM_CheckType, /*MI*/1, /*Op*/1, /*Type*/GILLT_s32,
+// CHECK-NEXT:   GIM_RecordNamedOperand, /*MI*/1, /*Op*/1, /*StoreIdx*/0, // Name : pred:3:x
+// CHECK-NEXT:   GIM_CheckRegBankForClass, /*MI*/1, /*Op*/1, /*RC*/GIMT_Encode2(Test::DRegsRegClassID),
+// CHECK-NEXT:   // MIs[1] src1
+// CHECK-NEXT:   GIM_CheckType, /*MI*/1, /*Op*/2, /*Type*/GILLT_s32,
+// CHECK-NEXT:   GIM_RecordNamedOperand, /*MI*/1, /*Op*/2, /*StoreIdx*/1, // Name : pred:3:y
+// CHECK-NEXT:   GIM_CheckRegBankForClass, /*MI*/1, /*Op*/2, /*RC*/GIMT_Encode2(Test::DRegsRegClassID),
+// CHECK-NEXT:   // MIs[0] src2
+// CHECK-NEXT:   GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32,
+// CHECK-NEXT:   GIM_RecordNamedOperand, /*MI*/0, /*Op*/2, /*StoreIdx*/2, // Name : pred:3:z
+// CHECK-NEXT:   GIM_CheckRegBankForClass, /*MI*/0, /*Op*/2, /*RC*/GIMT_Encode2(Test::DRegsRegClassID),
+// CHECK-NEXT:   GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_and_or_pat),
+// CHECK-NEXT:   GIM_CheckIsSafeToFold, /*InsnID*/1,
+// CHECK-NEXT:   // (and:{ *:[i32] } (or:{ *:[i32] } DOP:{ *:[i32] }:$src0:$pred:3:x, DOP:{ *:[i32] }:$src1:$pred:3:y), DOP:{ *:[i32] }:$src2:$pred:3:z)<<P:3:Predicate_and_or_pat>>  =>  (AND_OR:{ *:[i32] } DOP:{ *:[i32] }:$src0, DOP:{ *:[i32] }:$src1, DOP:{ *:[i32] }:$src2)
+// CHECK-NEXT:   GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::AND_OR),
 
 // Test commutative, standalone pattern.
 def : Pat<
@@ -148,63 +148,63 @@ def mul_pat : PatFrag<
   let PredicateCodeUsesOperands = 1;
 }
 
-// CHECK: GIM_Try, /*On fail goto*//*Label 2*/ 293, // Rule ID 4 //
-// CHECK-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3,
-// CHECK-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_MUL,
-// CHECK-NEXT: // MIs[0] DstI[dst]
-// CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
-// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/Test::DRegsRegClassID,
-// CHECK-NEXT: // MIs[0] Operand 1
-// CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32,
-// CHECK-NEXT: GIM_RecordNamedOperand, /*MI*/0, /*Op*/1, /*StoreIdx*/0, // Name : pred:4:x
-// CHECK-NEXT: GIM_RecordInsn, /*DefineMI*/1, /*MI*/0, /*OpIdx*/1, // MIs[1]
-// CHECK-NEXT: GIM_CheckNumOperands, /*MI*/1, /*Expected*/3,
-// CHECK-NEXT: GIM_CheckOpcode, /*MI*/1, TargetOpcode::G_OR,
-// CHECK-NEXT: // MIs[1] Operand 0
-// CHECK-NEXT: GIM_CheckType, /*MI*/1, /*Op*/0, /*Type*/GILLT_s32,
-// CHECK-NEXT: // MIs[1] src0
-// CHECK-NEXT: GIM_CheckType, /*MI*/1, /*Op*/1, /*Type*/GILLT_s32,
-// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/1, /*Op*/1, /*RC*/Test::DRegsRegClassID,
-// CHECK-NEXT: // MIs[1] src1
-// CHECK-NEXT: GIM_CheckType, /*MI*/1, /*Op*/2, /*Type*/GILLT_s32,
-// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/1, /*Op*/2, /*RC*/Test::DRegsRegClassID,
-// CHECK-NEXT: // MIs[0] src2
-// CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32,
-// CHECK-NEXT: GIM_RecordNamedOperand, /*MI*/0, /*Op*/2, /*StoreIdx*/1, // Name : pred:4:y
-// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/2, /*RC*/Test::DRegsRegClassID,
-// CHECK-NEXT: GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GICXXPred_MI_Predicate_mul_pat,
-// CHECK-NEXT: GIM_CheckIsSafeToFold, /*InsnID*/1,
-// CHECK-NEXT: // (mul:{ *:[i32] } (or:{ *:[i32] } DOP:{ *:[i32] }:$src0, DOP:{ *:[i32] }:$src1):$pred:4:x, DOP:{ *:[i32] }:$src2:$pred:4:y)<<P:4:Predicate_mul_pat>>  =>  (MUL_OR:{ *:[i32] } DOP:{ *:[i32] }:$src0, DOP:{ *:[i32] }:$src1, DOP:{ *:[i32] }:$src2)
-// CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::MUL_OR,
+// CHECK:      GIM_Try, /*On fail goto*//*Label 2*/ GIMT_Encode4(326), // Rule ID 4 //
+// CHECK-NEXT:   GIM_CheckNumOperands, /*MI*/0, /*Expected*/3,
+// CHECK-NEXT:   GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_MUL),
+// CHECK-NEXT:   // MIs[0] DstI[dst]
+// CHECK-NEXT:   GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
+// CHECK-NEXT:   GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(Test::DRegsRegClassID),
+// CHECK-NEXT:   // MIs[0] Operand 1
+// CHECK-NEXT:   GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32,
+// CHECK-NEXT:   GIM_RecordNamedOperand, /*MI*/0, /*Op*/1, /*StoreIdx*/0, // Name : pred:4:x
+// CHECK-NEXT:   GIM_RecordInsn, /*DefineMI*/1, /*MI*/0, /*OpIdx*/1, // MIs[1]
+// CHECK-NEXT:   GIM_CheckNumOperands, /*MI*/1, /*Expected*/3,
+// CHECK-NEXT:   GIM_CheckOpcode, /*MI*/1, GIMT_Encode2(TargetOpcode::G_OR),
+// CHECK-NEXT:   // MIs[1] Operand 0
+// CHECK-NEXT:   GIM_CheckType, /*MI*/1, /*Op*/0, /*Type*/GILLT_s32,
+// CHECK-NEXT:   // MIs[1] src0
+// CHECK-NEXT:   GIM_CheckType, /*MI*/1, /*Op*/1, /*Type*/GILLT_s32,
+// CHECK-NEXT:   GIM_CheckRegBankForClass, /*MI*/1, /*Op*/1, /*RC*/GIMT_Encode2(Test::DRegsRegClassID),
+// CHECK-NEXT:   // MIs[1] src1
+// CHECK-NEXT:   GIM_CheckType, /*MI*/1, /*Op*/2, /*Type*/GILLT_s32,
+// CHECK-NEXT:   GIM_CheckRegBankForClass, /*MI*/1, /*Op*/2, /*RC*/GIMT_Encode2(Test::DRegsRegClassID),
+// CHECK-NEXT:   // MIs[0] src2
+// CHECK-NEXT:   GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32,
+// CHECK-NEXT:   GIM_RecordNamedOperand, /*MI*/0, /*Op*/2, /*StoreIdx*/1, // Name : pred:4:y
+// CHECK-NEXT:   GIM_CheckRegBankForClass, /*MI*/0, /*Op*/2, /*RC*/GIMT_Encode2(Test::DRegsRegClassID),
+// CHECK-NEXT:   GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_mul_pat),
+// CHECK-NEXT:   GIM_CheckIsSafeToFold, /*InsnID*/1,
+// CHECK-NEXT:   // (mul:{ *:[i32] } (or:{ *:[i32] } DOP:{ *:[i32] }:$src0, DOP:{ *:[i32] }:$src1):$pred:4:x, DOP:{ *:[i32] }:$src2:$pred:4:y)<<P:4:Predicate_mul_pat>>  =>  (MUL_OR:{ *:[i32] } DOP:{ *:[i32] }:$src0, DOP:{ *:[i32] }:$src1, DOP:{ *:[i32] }:$src2)
+// CHECK-NEXT:   GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::MUL_OR),
 
-// CHECK: GIM_Try, /*On fail goto*//*Label 3*/ 388, // Rule ID 8 //
-// CHECK-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3,
-// CHECK-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_MUL,
-// CHECK-NEXT: // MIs[0] DstI[dst]
-// CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
-// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/Test::DRegsRegClassID,
-// CHECK-NEXT: // MIs[0] src2
-// CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32,
-// CHECK-NEXT: GIM_RecordNamedOperand, /*MI*/0, /*Op*/1, /*StoreIdx*/1, // Name : pred:4:y
-// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/Test::DRegsRegClassID,
-// CHECK-NEXT: // MIs[0] Operand 2
-// CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32,
-// CHECK-NEXT: GIM_RecordNamedOperand, /*MI*/0, /*Op*/2, /*StoreIdx*/0, // Name : pred:4:x
-// CHECK-NEXT: GIM_RecordInsn, /*DefineMI*/1, /*MI*/0, /*OpIdx*/2, // MIs[1]
-// CHECK-NEXT: GIM_CheckNumOperands, /*MI*/1, /*Expected*/3,
-// CHECK-NEXT: GIM_CheckOpcode, /*MI*/1, TargetOpcode::G_OR,
-// CHECK-NEXT: // MIs[1] Operand 0
-// CHECK-NEXT: GIM_CheckType, /*MI*/1, /*Op*/0, /*Type*/GILLT_s32,
-// CHECK-NEXT: // MIs[1] src0
-// CHECK-NEXT: GIM_CheckType, /*MI*/1, /*Op*/1, /*Type*/GILLT_s32,
-// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/1, /*Op*/1, /*RC*/Test::DRegsRegClassID,
-// CHECK-NEXT: // MIs[1] src1
-// CHECK-NEXT: GIM_CheckType, /*MI*/1, /*Op*/2, /*Type*/GILLT_s32,
-// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/1, /*Op*/2, /*RC*/Test::DRegsRegClassID,
-// CHECK-NEXT: GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GICXXPred_MI_Predicate_mul_pat,
-// CHECK-NEXT: GIM_CheckIsSafeToFold, /*InsnID*/1,
-// CHECK-NEXT: // (mul:{ *:[i32] } DOP:{ *:[i32] }:$src2:$pred:4:y, (or:{ *:[i32] } DOP:{ *:[i32] }:$src0, DOP:{ *:[i32] }:$src1):$pred:4:x)<<P:4:Predicate_mul_pat>>  =>  (MUL_OR:{ *:[i32] } DOP:{ *:[i32] }:$src0, DOP:{ *:[i32] }:$src1, DOP:{ *:[i32] }:$src2)
-// CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::MUL_OR,
+// CHECK:      GIM_Try, /*On fail goto*//*Label 3*/ GIMT_Encode4(432), // Rule ID 8 //
+// CHECK-NEXT:   GIM_CheckNumOperands, /*MI*/0, /*Expected*/3,
+// CHECK-NEXT:   GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_MUL),
+// CHECK-NEXT:   // MIs[0] DstI[dst]
+// CHECK-NEXT:   GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
+// CHECK-NEXT:   GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(Test::DRegsRegClassID),
+// CHECK-NEXT:   // MIs[0] src2
+// CHECK-NEXT:   GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32,
+// CHECK-NEXT:   GIM_RecordNamedOperand, /*MI*/0, /*Op*/1, /*StoreIdx*/1, // Name : pred:4:y
+// CHECK-NEXT:   GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/GIMT_Encode2(Test::DRegsRegClassID),
+// CHECK-NEXT:   // MIs[0] Operand 2
+// CHECK-NEXT:   GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32,
+// CHECK-NEXT:   GIM_RecordNamedOperand, /*MI*/0, /*Op*/2, /*StoreIdx*/0, // Name : pred:4:x
+// CHECK-NEXT:   GIM_RecordInsn, /*DefineMI*/1, /*MI*/0, /*OpIdx*/2, // MIs[1]
+// CHECK-NEXT:   GIM_CheckNumOperands, /*MI*/1, /*Expected*/3,
+// CHECK-NEXT:   GIM_CheckOpcode, /*MI*/1, GIMT_Encode2(TargetOpcode::G_OR),
+// CHECK-NEXT:   // MIs[1] Operand 0
+// CHECK-NEXT:   GIM_CheckType, /*MI*/1, /*Op*/0, /*Type*/GILLT_s32,
+// CHECK-NEXT:   // MIs[1] src0
+// CHECK-NEXT:   GIM_CheckType, /*MI*/1, /*Op*/1, /*Type*/GILLT_s32,
+// CHECK-NEXT:   GIM_CheckRegBankForClass, /*MI*/1, /*Op*/1, /*RC*/GIMT_Encode2(Test::DRegsRegClassID),
+// CHECK-NEXT:   // MIs[1] src1
+// CHECK-NEXT:   GIM_CheckType, /*MI*/1, /*Op*/2, /*Type*/GILLT_s32,
+// CHECK-NEXT:   GIM_CheckRegBankForClass, /*MI*/1, /*Op*/2, /*RC*/GIMT_Encode2(Test::DRegsRegClassID),
+// CHECK-NEXT:   GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_mul_pat),
+// CHECK-NEXT:   GIM_CheckIsSafeToFold, /*InsnID*/1,
+// CHECK-NEXT:   // (mul:{ *:[i32] } DOP:{ *:[i32] }:$src2:$pred:4:y, (or:{ *:[i32] } DOP:{ *:[i32] }:$src0, DOP:{ *:[i32] }:$src1):$pred:4:x)<<P:4:Predicate_mul_pat>>  =>  (MUL_OR:{ *:[i32] } DOP:{ *:[i32] }:$src0, DOP:{ *:[i32] }:$src1, DOP:{ *:[i32] }:$src2)
+// CHECK-NEXT:   GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::MUL_OR),
 
 // Test commutative patterns where named operands in the source pattern are not
 // directly bound to PatFrag's operands.
@@ -223,32 +223,32 @@ def sub3_pat : PatFrag<
   let PredicateCodeUsesOperands = 1;
 }
 
-// CHECK: GIM_Try, /*On fail goto*//*Label 4*/ 475, // Rule ID 0 //
-// CHECK-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3,
-// CHECK-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_SUB,
-// CHECK-NEXT: // MIs[0] DstI[dst]
-// CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
-// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/Test::DRegsRegClassID,
-// CHECK-NEXT: // MIs[0] Operand 1
-// CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32,
-// CHECK-NEXT: GIM_RecordInsn, /*DefineMI*/1, /*MI*/0, /*OpIdx*/1, // MIs[1]
-// CHECK-NEXT: GIM_CheckNumOperands, /*MI*/1, /*Expected*/3,
-// CHECK-NEXT: GIM_CheckOpcode, /*MI*/1, TargetOpcode::G_SUB,
-// CHECK-NEXT: // MIs[1] Operand 0
-// CHECK-NEXT: GIM_CheckType, /*MI*/1, /*Op*/0, /*Type*/GILLT_s32,
-// CHECK-NEXT: // MIs[1] src0
-// CHECK-NEXT: GIM_CheckType, /*MI*/1, /*Op*/1, /*Type*/GILLT_s32,
-// CHECK-NEXT: GIM_RecordNamedOperand, /*MI*/1, /*Op*/1, /*StoreIdx*/0, // Name : pred:1:x
-// CHECK-NEXT: // MIs[1] src1
-// CHECK-NEXT: GIM_CheckType, /*MI*/1, /*Op*/2, /*Type*/GILLT_s32,
-// CHECK-NEXT: GIM_RecordNamedOperand, /*MI*/1, /*Op*/2, /*StoreIdx*/1, // Name : pred:1:y
-// CHECK-NEXT: // MIs[0] src2
-// CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32,
-// CHECK-NEXT: GIM_RecordNamedOperand, /*MI*/0, /*Op*/2, /*StoreIdx*/2, // Name : pred:1:z
-// CHECK-NEXT: GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GICXXPred_MI_Predicate_sub3_pat,
-// CHECK-NEXT: GIM_CheckIsSafeToFold, /*InsnID*/1,
-// CHECK-NEXT: // (sub:{ *:[i32] } (sub:{ *:[i32] } i32:{ *:[i32] }:$src0:$pred:1:x, i32:{ *:[i32] }:$src1:$pred:1:y), i32:{ *:[i32] }:$src2:$pred:1:z)<<P:1:Predicate_sub3_pat>>  =>  (SUB3:{ *:[i32] } i32:{ *:[i32] }:$src0, i32:{ *:[i32] }:$src1, i32:{ *:[i32] }:$src2)
-// CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::SUB3,
+// CHECK:      GIM_Try, /*On fail goto*//*Label 4*/ GIMT_Encode4(527), // Rule ID 0 //
+// CHECK-NEXT:   GIM_CheckNumOperands, /*MI*/0, /*Expected*/3,
+// CHECK-NEXT:   GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_SUB),
+// CHECK-NEXT:   // MIs[0] DstI[dst]
+// CHECK-NEXT:   GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
+// CHECK-NEXT:   GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(Test::DRegsRegClassID),
+// CHECK-NEXT:   // MIs[0] Operand 1
+// CHECK-NEXT:   GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32,
+// CHECK-NEXT:   GIM_RecordInsn, /*DefineMI*/1, /*MI*/0, /*OpIdx*/1, // MIs[1]
+// CHECK-NEXT:   GIM_CheckNumOperands, /*MI*/1, /*Expected*/3,
+// CHECK-NEXT:   GIM_CheckOpcode, /*MI*/1, GIMT_Encode2(TargetOpcode::G_SUB),
+// CHECK-NEXT:   // MIs[1] Operand 0
+// CHECK-NEXT:   GIM_CheckType, /*MI*/1, /*Op*/0, /*Type*/GILLT_s32,
+// CHECK-NEXT:   // MIs[1] src0
+// CHECK-NEXT:   GIM_CheckType, /*MI*/1, /*Op*/1, /*Type*/GILLT_s32,
+// CHECK-NEXT:   GIM_RecordNamedOperand, /*MI*/1, /*Op*/1, /*StoreIdx*/0, // Name : pred:1:x
+// CHECK-NEXT:   // MIs[1] src1
+// CHECK-NEXT:   GIM_CheckType, /*MI*/1, /*Op*/2, /*Type*/GILLT_s32,
+// CHECK-NEXT:   GIM_RecordNamedOperand, /*MI*/1, /*Op*/2, /*StoreIdx*/1, // Name : pred:1:y
+// CHECK-NEXT:   // MIs[0] src2
+// CHECK-NEXT:   GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32,
+// CHECK-NEXT:   GIM_RecordNamedOperand, /*MI*/0, /*Op*/2, /*StoreIdx*/2, // Name : pred:1:z
+// CHECK-NEXT:   GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_sub3_pat),
+// CHECK-NEXT:   GIM_CheckIsSafeToFold, /*InsnID*/1,
+// CHECK-NEXT:   // (sub:{ *:[i32] } (sub:{ *:[i32] } i32:{ *:[i32] }:$src0:$pred:1:x, i32:{ *:[i32] }:$src1:$pred:1:y), i32:{ *:[i32] }:$src2:$pred:1:z)<<P:1:Predicate_sub3_pat>>  =>  (SUB3:{ *:[i32] } i32:{ *:[i32] }:$src0, i32:{ *:[i32] }:$src1, i32:{ *:[i32] }:$src2)
+// CHECK-NEXT:   GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::SUB3)
 
 // Test a non-commutative pattern.
 def SUB3 : I<(outs DRegs:$dst),
@@ -269,16 +269,16 @@ def patfrags_test_pat : PatFrags<
   let PredicateCodeUsesOperands = 1;
 }
 
-// CHECK: GIM_Try, /*On fail goto*//*Label 5*/ 562, // Rule ID 1 //
+// CHECK: GIM_Try, /*On fail goto*//*Label 5*/ GIMT_Encode4(622), // Rule ID 1 //
 // CHECK: // (xor:{ *:[i32] } (add:{ *:[i32] } i32:{ *:[i32] }:$src0:$pred:2:x, i32:{ *:[i32] }:$src1:$pred:2:y), i32:{ *:[i32] }:$src2:$pred:2:z)<<P:2:Predicate_patfrags_test_pat>>  =>  (PATFRAGS:{ *:[i32] } i32:{ *:[i32] }:$src0, i32:{ *:[i32] }:$src1, i32:{ *:[i32] }:$src2)
 
-// CHECK: GIM_Try, /*On fail goto*//*Label 6*/ 649, // Rule ID 2 //
+// CHECK: GIM_Try, /*On fail goto*//*Label 6*/ GIMT_Encode4(717), // Rule ID 2 //
 // CHECK: // (xor:{ *:[i32] } (sub:{ *:[i32] } i32:{ *:[i32] }:$src0:$pred:2:x, i32:{ *:[i32] }:$src1:$pred:2:y), i32:{ *:[i32] }:$src2:$pred:2:z)<<P:2:Predicate_patfrags_test_pat>>  =>  (PATFRAGS:{ *:[i32] } i32:{ *:[i32] }:$src0, i32:{ *:[i32] }:$src1, i32:{ *:[i32] }:$src2)
 
-// CHECK: GIM_Try, /*On fail goto*//*Label 7*/ 736, // Rule ID 5 //
+// CHECK: GIM_Try, /*On fail goto*//*Label 7*/ GIMT_Encode4(812), // Rule ID 5 //
 // CHECK: // (xor:{ *:[i32] } i32:{ *:[i32] }:$src2:$pred:2:z, (add:{ *:[i32] } i32:{ *:[i32] }:$src0:$pred:2:x, i32:{ *:[i32] }:$src1:$pred:2:y))<<P:2:Predicate_patfrags_test_pat>>  =>  (PATFRAGS:{ *:[i32] } i32:{ *:[i32] }:$src0, i32:{ *:[i32] }:$src1, i32:{ *:[i32] }:$src2)
 
-// CHECK: GIM_Try, /*On fail goto*//*Label 8*/ 823, // Rule ID 6 //
+// CHECK: GIM_Try, /*On fail goto*//*Label 8*/ GIMT_Encode4(907), // Rule ID 6 //
 // CHECK: // (xor:{ *:[i32] } i32:{ *:[i32] }:$src2:$pred:2:z, (sub:{ *:[i32] } i32:{ *:[i32] }:$src0:$pred:2:x, i32:{ *:[i32] }:$src1:$pred:2:y))<<P:2:Predicate_patfrags_test_pat>>  =>  (PATFRAGS:{ *:[i32] } i32:{ *:[i32] }:$src0, i32:{ *:[i32] }:$src1, i32:{ *:[i32] }:$src2)
 
 
diff --git a/llvm/test/TableGen/GlobalISelEmitterFlags.td b/llvm/test/TableGen/GlobalISelEmitterFlags.td
index 1c53e1a45c26b22dc964877ff8e55f553372c678..2025a2c1048e0216583fca4166d1ba1e50f25046 100644
--- a/llvm/test/TableGen/GlobalISelEmitterFlags.td
+++ b/llvm/test/TableGen/GlobalISelEmitterFlags.td
@@ -36,25 +36,25 @@ def : Pat<
 
 // CHECK: GIM_Try
 // CHECK:    GIM_RecordInsnIgnoreCopies, /*DefineMI*/1, /*MI*/0, /*OpIdx*/1, // MIs[1]
-// CHECK:    GIM_CheckOpcode, /*MI*/1, TargetOpcode::G_LSHR
+// CHECK:    GIM_CheckOpcode, /*MI*/1, GIMT_Encode2(TargetOpcode::G_LSHR)
 // CHECK:    GIM_RecordInsn, /*DefineMI*/2, /*MI*/1, /*OpIdx*/1, // MIs[2]
-// CHECK:    GIM_CheckOpcode, /*MI*/2, TargetOpcode::G_ADD
+// CHECK:    GIM_CheckOpcode, /*MI*/2, GIMT_Encode2(TargetOpcode::G_ADD)
 // CHECK:    GIM_CheckIsSameOperand, /*MI*/2, /*OpIdx*/2, /*OtherMI*/2, /*OtherOpIdx*/1
 // CHECK:    GIM_CheckIsSameOperandIgnoreCopies, /*MI*/0, /*OpIdx*/2, /*OtherMI*/2, /*OtherOpIdx*/1
 // CHECK:    // (srl:{ *:[i32] } (srl:{ *:[i32] } (add:{ *:[i32] } i32:{ *:[i32] }:$src0, i32:{ *:[i32] }:$src0), i32:{ *:[i32] }:$src1), i32:{ *:[i32] }:$src0)
 // CHECK:   GIR_Done
 // CHECK: GIM_Try
 // CHECK:   GIM_RecordInsnIgnoreCopies, /*DefineMI*/1, /*MI*/0, /*OpIdx*/1, // MIs[1]
-// CHECK:   GIM_CheckOpcode, /*MI*/1, TargetOpcode::G_SHL
+// CHECK:   GIM_CheckOpcode, /*MI*/1, GIMT_Encode2(TargetOpcode::G_SHL)
 // CHECK:   GIM_RecordInsnIgnoreCopies, /*DefineMI*/2, /*MI*/1, /*OpIdx*/1, // MIs[2]
-// CHECK:   GIM_CheckOpcode, /*MI*/2, TargetOpcode::G_MUL
+// CHECK:   GIM_CheckOpcode, /*MI*/2, GIMT_Encode2(TargetOpcode::G_MUL)
 // CHECK:   GIM_CheckIsSameOperandIgnoreCopies, /*MI*/2, /*OpIdx*/2, /*OtherMI*/2, /*OtherOpIdx*/1
 // CHECK:   GIM_CheckIsSameOperandIgnoreCopies, /*MI*/0, /*OpIdx*/2, /*OtherMI*/2, /*OtherOpIdx*/1
 // CHECK:   // (srl:{ *:[i32] } (shl:{ *:[i32] } (mul:{ *:[i32] } i32:{ *:[i32] }:$src0, i32:{ *:[i32] }:$src0), i32:{ *:[i32] }:$src1), i32:{ *:[i32] }:$src0)
 // CHECK:   GIR_Done
 // CHECK: GIM_Try
 // CHECK:   GIM_RecordInsnIgnoreCopies, /*DefineMI*/1, /*MI*/0, /*OpIdx*/1, // MIs[1]
-// CHECK:   GIM_CheckOpcode, /*MI*/1, TargetOpcode::G_MUL
+// CHECK:   GIM_CheckOpcode, /*MI*/1, GIMT_Encode2(TargetOpcode::G_MUL)
 // CHECK:   GIM_CheckIsSameOperandIgnoreCopies, /*MI*/1, /*OpIdx*/2, /*OtherMI*/1, /*OtherOpIdx*/1
 // CHECK:   // (sub:{ *:[i32] } (mul:{ *:[i32] } i32:{ *:[i32] }:$src0, i32:{ *:[i32] }:$src0), i32:{ *:[i32] }:$src1)  =>  (InstThreeOperands:{ *:[i32] } GPR32:{ *:[i32] }:$src0, GPR32:{ *:[i32] }:$src1, GPR32:{ *:[i32] }:$src1)
-// CHECK:   GIR_Done
\ No newline at end of file
+// CHECK:   GIR_Done
diff --git a/llvm/test/TableGen/GlobalISelEmitterHwModes.td b/llvm/test/TableGen/GlobalISelEmitterHwModes.td
index 04f33648313b2f7aacb67efd1b593c2a1f5545b0..0c826fb50beb5fc9fabe0d560ff6ac9c393b6309 100644
--- a/llvm/test/TableGen/GlobalISelEmitterHwModes.td
+++ b/llvm/test/TableGen/GlobalISelEmitterHwModes.td
@@ -52,7 +52,7 @@ class I<dag OOps, dag IOps, list<dag> Pat>
 // CHECK-NEXT:    bool testImmPredicate_I64(unsigned PredicateID, int64_t Imm) const override;
 // CHECK-NEXT:    bool testImmPredicate_APInt(unsigned PredicateID, const APInt &Imm) const override;
 // CHECK-NEXT:    bool testImmPredicate_APFloat(unsigned PredicateID, const APFloat &Imm) const override;
-// CHECK-NEXT:    const int64_t *getMatchTable() const override;
+// CHECK-NEXT:    const uint8_t *getMatchTable() const override;
 // CHECK-NEXT:    bool testMIPredicate_MI(unsigned PredicateID, const MachineInstr &MI, const MatcherState &State) const override;
 // CHECK-NEXT:    bool testSimplePredicate(unsigned PredicateID) const override;
 // CHECK-NEXT:    void runCustomAction(unsigned FnID, const MatcherState &State, NewMIVector &OutMIs) const override;
@@ -121,44 +121,44 @@ class I<dag OOps, dag IOps, list<dag> Pat>
 // CHECK-NEXT:   return true;
 // CHECK-NEXT: }
 
-// CHECK: const int64_t *
+// CHECK: const uint8_t *
 // CHECK-LABEL: MyTargetInstructionSelector::getMatchTable() const {
 // CHECK-NEXT: MatchTable0[] = {
 
 //===- Test a simple pattern with inferred pointer operands. ---------------===//
 
-// CHECK-NEXT:  GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ [[LABEL:[0-9]+]],
-// CHECK-NEXT:    GIM_CheckFeatures, GIFBS_HwMode0,
+// CHECK-NEXT:  GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ GIMT_Encode4([[LABEL:[0-9]+]]),
+// CHECK-NEXT:    GIM_CheckFeatures, GIMT_Encode2(GIFBS_HwMode0),
 // CHECK-NEXT:    GIM_CheckNumOperands, /*MI*/0, /*Expected*/2,
-// CHECK-NEXT:    GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_LOAD,
+// CHECK-NEXT:    GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_LOAD),
 // CHECK-NEXT:    GIM_CheckMemorySizeEqualToLLT, /*MI*/0, /*MMO*/0, /*OpIdx*/0,
-// CHECK-NEXT:    GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(int64_t)AtomicOrdering::NotAtomic,
+// CHECK-NEXT:    GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(uint8_t)AtomicOrdering::NotAtomic,
 // CHECK-NEXT:    // MIs[0] DstI[dst]
 // CHECK-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s64,
-// CHECK-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPRRegClassID,
+// CHECK-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPRRegClassID),
 // CHECK-NEXT:    // MIs[0] src1
 // CHECK-NEXT:    GIM_CheckPointerToAny, /*MI*/0, /*Op*/1, /*SizeInBits*/64,
-// CHECK-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/MyTarget::GPRRegClassID,
+// CHECK-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/GIMT_Encode2(MyTarget::GPRRegClassID),
 // CHECK-NEXT:    // (ld:{ *:[i64] } GPR:{ *:[i64] }:$src1)<<P:Predicate_unindexedload>><<P:Predicate_load>>  =>  (LOAD:{ *:[i64] } GPR:{ *:[i64] }:$src1)
-// CHECK-NEXT:    GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/MyTarget::LOAD,
+// CHECK-NEXT:    GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::LOAD),
 // CHECK-NEXT:    GIR_ConstrainSelectedInstOperands, /*InsnID*/0,
 // CHECK-NEXT:    // GIR_Coverage, 0,
 // CHECK-NEXT:    GIR_Done,
 // CHECK-NEXT:  // Label [[LABEL_NUM]]: @[[LABEL]]
-// CHECK-NEXT:  GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ [[LABEL:[0-9]+]],
-// CHECK-NEXT:    GIM_CheckFeatures, GIFBS_HwMode1,
+// CHECK-NEXT:  GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ GIMT_Encode4([[LABEL:[0-9]+]]),
+// CHECK-NEXT:    GIM_CheckFeatures, GIMT_Encode2(GIFBS_HwMode1),
 // CHECK-NEXT:    GIM_CheckNumOperands, /*MI*/0, /*Expected*/2,
-// CHECK-NEXT:    GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_LOAD,
+// CHECK-NEXT:    GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_LOAD),
 // CHECK-NEXT:    GIM_CheckMemorySizeEqualToLLT, /*MI*/0, /*MMO*/0, /*OpIdx*/0,
-// CHECK-NEXT:    GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(int64_t)AtomicOrdering::NotAtomic,
+// CHECK-NEXT:    GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(uint8_t)AtomicOrdering::NotAtomic,
 // CHECK-NEXT:    // MIs[0] DstI[dst]
 // CHECK-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
-// CHECK-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPRRegClassID,
+// CHECK-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPRRegClassID),
 // CHECK-NEXT:    // MIs[0] src1
 // CHECK-NEXT:    GIM_CheckPointerToAny, /*MI*/0, /*Op*/1, /*SizeInBits*/32,
-// CHECK-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/MyTarget::GPRRegClassID,
+// CHECK-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/GIMT_Encode2(MyTarget::GPRRegClassID),
 // CHECK-NEXT:    // (ld:{ *:[i32] } GPR:{ *:[i32] }:$src1)<<P:Predicate_unindexedload>><<P:Predicate_load>>  =>  (LOAD:{ *:[i32] } GPR:{ *:[i32] }:$src1)
-// CHECK-NEXT:    GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/MyTarget::LOAD,
+// CHECK-NEXT:    GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::LOAD),
 // CHECK-NEXT:    GIR_ConstrainSelectedInstOperands, /*InsnID*/0,
 // CHECK-NEXT:    // GIR_Coverage, 1,
 // CHECK-NEXT:    GIR_Done,
@@ -169,38 +169,38 @@ def LOAD : I<(outs GPR:$dst), (ins GPR:$src1),
 
 //===- Test a simple pattern with explicit pointer operands. ---------------===//
 
-// CHECK-NEXT:  GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ [[LABEL:[0-9]+]],
-// CHECK-NEXT:    GIM_CheckFeatures, GIFBS_HwMode0,
+// CHECK-NEXT:  GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ GIMT_Encode4([[LABEL:[0-9]+]]),
+// CHECK-NEXT:    GIM_CheckFeatures, GIMT_Encode2(GIFBS_HwMode0),
 // CHECK-NEXT:    GIM_CheckNumOperands, /*MI*/0, /*Expected*/2,
-// CHECK-NEXT:    GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_LOAD,
+// CHECK-NEXT:    GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_LOAD),
 // CHECK-NEXT:    GIM_CheckMemorySizeEqualToLLT, /*MI*/0, /*MMO*/0, /*OpIdx*/0,
-// CHECK-NEXT:    GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(int64_t)AtomicOrdering::NotAtomic,
+// CHECK-NEXT:    GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(uint8_t)AtomicOrdering::NotAtomic,
 // CHECK-NEXT:    // MIs[0] DstI[dst]
 // CHECK-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_p0s64,
-// CHECK-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPRRegClassID,
+// CHECK-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPRRegClassID),
 // CHECK-NEXT:    // MIs[0] src
 // CHECK-NEXT:    GIM_CheckPointerToAny, /*MI*/0, /*Op*/1, /*SizeInBits*/64,
-// CHECK-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/MyTarget::GPRRegClassID,
+// CHECK-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/GIMT_Encode2(MyTarget::GPRRegClassID),
 // CHECK-NEXT:    // (ld:{ *:[i64] } GPR:{ *:[i64] }:$src)<<P:Predicate_unindexedload>><<P:Predicate_load>>  =>  (LOAD:{ *:[i64] } GPR:{ *:[i64] }:$src)
-// CHECK-NEXT:    GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/MyTarget::LOAD,
+// CHECK-NEXT:    GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::LOAD),
 // CHECK-NEXT:    GIR_ConstrainSelectedInstOperands, /*InsnID*/0,
 // CHECK-NEXT:    // GIR_Coverage, 2,
 // CHECK-NEXT:    GIR_Done,
 // CHECK-NEXT:  // Label [[LABEL_NUM]]: @[[LABEL]]
-// CHECK-NEXT:  GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ [[LABEL:[0-9]+]],
-// CHECK-NEXT:    GIM_CheckFeatures, GIFBS_HwMode1,
+// CHECK-NEXT:  GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ GIMT_Encode4([[LABEL:[0-9]+]]),
+// CHECK-NEXT:    GIM_CheckFeatures, GIMT_Encode2(GIFBS_HwMode1),
 // CHECK-NEXT:    GIM_CheckNumOperands, /*MI*/0, /*Expected*/2,
-// CHECK-NEXT:    GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_LOAD,
+// CHECK-NEXT:    GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_LOAD),
 // CHECK-NEXT:    GIM_CheckMemorySizeEqualToLLT, /*MI*/0, /*MMO*/0, /*OpIdx*/0,
-// CHECK-NEXT:    GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(int64_t)AtomicOrdering::NotAtomic,
+// CHECK-NEXT:    GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(uint8_t)AtomicOrdering::NotAtomic,
 // CHECK-NEXT:    // MIs[0] DstI[dst]
 // CHECK-NEXT:    GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_p0s32,
-// CHECK-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPRRegClassID,
+// CHECK-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPRRegClassID),
 // CHECK-NEXT:    // MIs[0] src
 // CHECK-NEXT:    GIM_CheckPointerToAny, /*MI*/0, /*Op*/1, /*SizeInBits*/32,
-// CHECK-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/MyTarget::GPRRegClassID,
+// CHECK-NEXT:    GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/GIMT_Encode2(MyTarget::GPRRegClassID),
 // CHECK-NEXT:    // (ld:{ *:[i32] } GPR:{ *:[i32] }:$src)<<P:Predicate_unindexedload>><<P:Predicate_load>>  =>  (LOAD:{ *:[i32] } GPR:{ *:[i32] }:$src)
-// CHECK-NEXT:    GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/MyTarget::LOAD,
+// CHECK-NEXT:    GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::LOAD),
 // CHECK-NEXT:    GIR_ConstrainSelectedInstOperands, /*InsnID*/0,
 // CHECK-NEXT:    // GIR_Coverage, 3,
 // CHECK-NEXT:    GIR_Done,
diff --git a/llvm/test/TableGen/GlobalISelEmitterMatchTableOptimizer.td b/llvm/test/TableGen/GlobalISelEmitterMatchTableOptimizer.td
index 6a369b6a7b8897f4fa771aae4033df97a32f039d..08feeccc3569df354fbccfb881c435da7be8b934 100644
--- a/llvm/test/TableGen/GlobalISelEmitterMatchTableOptimizer.td
+++ b/llvm/test/TableGen/GlobalISelEmitterMatchTableOptimizer.td
@@ -8,26 +8,26 @@ include "GlobalISelEmitterCommon.td"
 def LOAD8 : I<(outs GPR8:$dst), (ins GPR8:$src), []>;
 def LOAD32 : I<(outs GPR8:$dst), (ins GPR32:$src), []>;
 // CHECK: Label 1: @{{[0-9]+}}
-// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label [[L1_ID:[0-9]+]]*/ [[L1_AT:[0-9]+]],
+// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label [[L1_ID:[0-9]+]]*/ GIMT_Encode4([[L1_AT:[0-9]+]]),
 // CHECK-NEXT: GIM_CheckMemorySizeEqualToLLT, /*MI*/0, /*MMO*/0, /*OpIdx*/0,
-// CHECK-NEXT:   GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(int64_t)AtomicOrdering::NotAtomic,
-// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR8RegClassID,
-// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label [[L2_ID:[0-9]+]]*/ [[L2_AT:[0-9]+]],
+// CHECK-NEXT:   GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(uint8_t)AtomicOrdering::NotAtomic,
+// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR8RegClassID),
+// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label [[L2_ID:[0-9]+]]*/ GIMT_Encode4([[L2_AT:[0-9]+]]),
 // CHECK-NEXT: // MIs[0] src
 // CHECK-NEXT: GIM_CheckPointerToAny, /*MI*/0, /*Op*/1, /*SizeInBits*/8,
-// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/MyTarget::GPR8RegClassID,
+// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/GIMT_Encode2(MyTarget::GPR8RegClassID),
 // CHECK-NEXT: // (ld:{ *:[i8] } GPR8:{ *:[i8] }:$src)<<P:Predicate_unindexedload>><<P:Predicate_load>>  =>  (LOAD8:{ *:[i8] } GPR8:{ *:[i8] }:$src)
-// CHECK-NEXT: GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/MyTarget::LOAD8,
+// CHECK-NEXT: GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::LOAD8),
 // CHECK-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0,
 // CHECK-NEXT: // GIR_Coverage, 0,
 // CHECK-NEXT: GIR_Done,
 // CHECK-NEXT: // Label [[L2_ID]]: @[[L2_AT]]
-// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label [[L3_ID:[0-9]+]]*/ [[L3_AT:[0-9]+]],
+// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label [[L3_ID:[0-9]+]]*/ GIMT_Encode4([[L3_AT:[0-9]+]]),
 // CHECK-NEXT: // MIs[0] src
 // CHECK-NEXT: GIM_CheckPointerToAny, /*MI*/0, /*Op*/1, /*SizeInBits*/32,
-// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/MyTarget::GPR32RegClassID,
+// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // CHECK-NEXT: // (ld:{ *:[i8] } GPR32:{ *:[i32] }:$src)<<P:Predicate_unindexedload>><<P:Predicate_load>>  =>  (LOAD32:{ *:[i8] } GPR32:{ *:[i32] }:$src)
-// CHECK-NEXT: GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/MyTarget::LOAD32,
+// CHECK-NEXT: GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::LOAD32),
 // CHECK-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0,
 // CHECK-NEXT: // GIR_Coverage, 1,
 // CHECK-NEXT: GIR_Done,
@@ -46,33 +46,33 @@ def GPR16 : RegisterClass<"MyTarget", [i16], 16, (add S0)>;
 def LOAD16 : I<(outs GPR16:$dst), (ins GPR16:$src), []>;
 def LOAD16Imm : I<(outs GPR16:$dst), (ins GPR16:$src), []>;
 // CHECK: // Label 2: @{{[0-9]+}}
-// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label [[L1_ID:[0-9]+]]*/ [[L1_AT:[0-9]+]],
+// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label [[L1_ID:[0-9]+]]*/ GIMT_Encode4([[L1_AT:[0-9]+]]),
 // CHECK-NEXT: GIM_CheckMemorySizeEqualToLLT, /*MI*/0, /*MMO*/0, /*OpIdx*/0,
-// CHECK-NEXT: GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(int64_t)AtomicOrdering::NotAtomic,
-// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR16RegClassID,
+// CHECK-NEXT: GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(uint8_t)AtomicOrdering::NotAtomic,
+// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR16RegClassID),
 // CHECK-NEXT: GIM_CheckPointerToAny, /*MI*/0, /*Op*/1, /*SizeInBits*/16,
-// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label [[L2_ID:[0-9]+]]*/ [[L2_AT:[0-9]+]],
+// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label [[L2_ID:[0-9]+]]*/ GIMT_Encode4([[L2_AT:[0-9]+]]),
 // CHECK-NEXT: GIM_RecordInsn, /*DefineMI*/1, /*MI*/0, /*OpIdx*/1, // MIs[1]
-// CHECK-NEXT: GIM_CheckOpcode, /*MI*/1, TargetOpcode::G_ADD,
+// CHECK-NEXT: GIM_CheckOpcode, /*MI*/1, GIMT_Encode2(TargetOpcode::G_ADD),
 // CHECK-NEXT: GIM_CheckType, /*MI*/1, /*Op*/1, /*Type*/GILLT_s16,
 // CHECK-NEXT: GIM_CheckType, /*MI*/1, /*Op*/2, /*Type*/GILLT_s16,
-// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/1, /*Op*/1, /*RC*/MyTarget::GPR16RegClassID,
-// CHECK-NEXT: GIM_CheckConstantInt, /*MI*/1, /*Op*/2, 10,
+// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/1, /*Op*/1, /*RC*/GIMT_Encode2(MyTarget::GPR16RegClassID),
+// CHECK-NEXT: GIM_CheckConstantInt8, /*MI*/1, /*Op*/2, 10,
 // CHECK-NEXT: GIM_CheckIsSafeToFold, /*InsnID*/1,
 // CHECK-NEXT: // (ld:{ *:[i16] } (add:{ *:[i16] } GPR16:{ *:[i16] }:$src, 10:{ *:[i16] }))<<P:Predicate_unindexedload>><<P:Predicate_load>>  =>  (LOAD16Imm:{ *:[i16] } GPR16:{ *:[i16] }:$src)
-// CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::LOAD16Imm,
+// CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::LOAD16Imm),
 // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst]
 // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/1, /*OpIdx*/1, // src
-// CHECK-NEXT: GIR_MergeMemOperands, /*InsnID*/0, /*MergeInsnID's*/0, 1, GIU_MergeMemOperands_EndOfList,
+// CHECK-NEXT: GIR_MergeMemOperands, /*InsnID*/0, /*NumInsns*/2, /*MergeInsnID's*/0, 1,
 // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0,
 // CHECK-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0,
 // CHECK-NEXT: // GIR_Coverage, 3,
 // CHECK-NEXT: GIR_Done,
 // CHECK-NEXT: // Label [[L2_ID]]: @[[L2_AT]]
-// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label [[L3_ID:[0-9]+]]*/ [[L3_AT:[0-9]+]],
-// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/MyTarget::GPR16RegClassID,
+// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label [[L3_ID:[0-9]+]]*/ GIMT_Encode4([[L3_AT:[0-9]+]]),
+// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/GIMT_Encode2(MyTarget::GPR16RegClassID),
 // CHECK-NEXT: // (ld:{ *:[i16] } GPR16:{ *:[i16] }:$src)<<P:Predicate_unindexedload>><<P:Predicate_load>>  =>  (LOAD16:{ *:[i16] } GPR16:{ *:[i16] }:$src)
-// CHECK-NEXT: GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/MyTarget::LOAD16,
+// CHECK-NEXT: GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::LOAD16),
 // CHECK-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0,
 // CHECK-NEXT: // GIR_Coverage, 2,
 // CHECK-NEXT: GIR_Done,
diff --git a/llvm/test/TableGen/GlobalISelEmitterMatchTableOptimizerSameOperand-invalid.td b/llvm/test/TableGen/GlobalISelEmitterMatchTableOptimizerSameOperand-invalid.td
index 2423c3bd32d5068977f97c80d1c17cd799150792..f58b82bb86b1dddfbf0b33d8a25efd53781bb259 100644
--- a/llvm/test/TableGen/GlobalISelEmitterMatchTableOptimizerSameOperand-invalid.td
+++ b/llvm/test/TableGen/GlobalISelEmitterMatchTableOptimizerSameOperand-invalid.td
@@ -6,33 +6,33 @@ include "GlobalISelEmitterCommon.td"
 def InstTwoOperands : I<(outs GPR32:$dst), (ins GPR32:$src1, GPR32:$src2), []>;
 def InstThreeOperands : I<(outs GPR32:$dst), (ins GPR32:$cond, GPR32:$src,GPR32:$src2), []>;
 
-// CHECK:         GIM_Try, /*On fail goto*//*Label 0*/ 219,
-// CHECK-NEXT:      GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_SELECT,
+// CHECK:      GIM_Try, /*On fail goto*//*Label 0*/ GIMT_Encode4(255),
+// CHECK-NEXT:      GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_SELECT),
 // CHECK-NEXT:      GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
 // CHECK-NEXT:      GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32,
 // CHECK-NEXT:      GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32,
-// CHECK-NEXT:      GIM_Try, /*On fail goto*//*Label 1*/ 189,
-// CHECK-NEXT:        GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID,
+// CHECK-NEXT:      GIM_Try, /*On fail goto*//*Label 1*/ GIMT_Encode4(217),
+// CHECK-NEXT:        GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // CHECK-NEXT:        GIM_CheckIsSameOperand, /*MI*/0, /*OpIdx*/3, /*OtherMI*/2, /*OtherOpIdx*/2,
-// CHECK-NEXT:        GIM_Try, /*On fail goto*//*Label 2*/ 108, // Rule ID 1 //
+// CHECK-NEXT:        GIM_Try, /*On fail goto*//*Label 2*/ GIMT_Encode4(126), // Rule ID 1 //
 // CHECK-NEXT:          GIM_RecordInsn, /*DefineMI*/1, /*MI*/0, /*OpIdx*/1, // MIs[1]
-// CHECK-NEXT:          GIM_CheckOpcode, /*MI*/1, TargetOpcode::G_ICMP,
+// CHECK-NEXT:          GIM_CheckOpcode, /*MI*/1, GIMT_Encode2(TargetOpcode::G_ICMP),
 // CHECK-NEXT:          GIM_CheckType, /*MI*/1, /*Op*/2, /*Type*/GILLT_s32,
 // CHECK-NEXT:          GIM_CheckType, /*MI*/1, /*Op*/3, /*Type*/GILLT_s32,
 // CHECK-NEXT:          // MIs[1] Operand 1
-// CHECK-NEXT:          GIM_CheckCmpPredicate, /*MI*/1, /*Op*/1, /*Predicate*/CmpInst::ICMP_EQ,
-// CHECK-NEXT:          GIM_CheckRegBankForClass, /*MI*/1, /*Op*/2, /*RC*/MyTarget::GPR32RegClassID,
-// CHECK-NEXT:          GIM_CheckConstantInt, /*MI*/1, /*Op*/3, 0,
+// CHECK-NEXT:          GIM_CheckCmpPredicate, /*MI*/1, /*Op*/1, /*Predicate*/GIMT_Encode2(CmpInst::ICMP_EQ),
+// CHECK-NEXT:          GIM_CheckRegBankForClass, /*MI*/1, /*Op*/2, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
+// CHECK-NEXT:          GIM_CheckConstantInt8, /*MI*/1, /*Op*/3, 0,
 // CHECK-NEXT:          GIM_RecordInsn, /*DefineMI*/2, /*MI*/0, /*OpIdx*/2, // MIs[2]
-// CHECK-NEXT:          GIM_CheckOpcode, /*MI*/2, TargetOpcode::G_SUB,
+// CHECK-NEXT:          GIM_CheckOpcode, /*MI*/2, GIMT_Encode2(TargetOpcode::G_SUB),
 // CHECK-NEXT:          GIM_CheckType, /*MI*/2, /*Op*/1, /*Type*/GILLT_s32,
 // CHECK-NEXT:          GIM_CheckType, /*MI*/2, /*Op*/2, /*Type*/GILLT_s32,
-// CHECK-NEXT:          GIM_CheckRegBankForClass, /*MI*/2, /*Op*/1, /*RC*/MyTarget::GPR32RegClassID,
-// CHECK-NEXT:          GIM_CheckRegBankForClass, /*MI*/2, /*Op*/2, /*RC*/MyTarget::GPR32RegClassID,
+// CHECK-NEXT:          GIM_CheckRegBankForClass, /*MI*/2, /*Op*/1, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
+// CHECK-NEXT:          GIM_CheckRegBankForClass, /*MI*/2, /*Op*/2, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // CHECK-NEXT:          GIM_CheckIsSafeToFold, /*InsnID*/1,
 // CHECK-NEXT:          GIM_CheckIsSafeToFold, /*InsnID*/2,
 // CHECK-NEXT:          // (select:{ *:[i32] } (setcc:{ *:[i32] } GPR32:{ *:[i32] }:$cond, 0:{ *:[i32] }, SETEQ:{ *:[Other] }), (sub:{ *:[i32] } GPR32:{ *:[i32] }:$src1, GPR32:{ *:[i32] }:$src2), GPR32:{ *:[i32] }:$src2)  =>  (InstThreeOperands:{ *:[i32] } GPR32:{ *:[i32] }:$cond, GPR32:{ *:[i32] }:$src1, GPR32:{ *:[i32] }:$src2)
-// CHECK-NEXT:          GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::InstThreeOperands,
+// CHECK-NEXT:          GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::InstThreeOperands),
 // CHECK-NEXT:          GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst]
 // CHECK-NEXT:          GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/1, /*OpIdx*/2, // cond
 // CHECK-NEXT:          GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/2, /*OpIdx*/1, // src1
@@ -41,26 +41,26 @@ def InstThreeOperands : I<(outs GPR32:$dst), (ins GPR32:$cond, GPR32:$src,GPR32:
 // CHECK-NEXT:          GIR_ConstrainSelectedInstOperands, /*InsnID*/0,
 // CHECK-NEXT:          // GIR_Coverage, 1,
 // CHECK-NEXT:          GIR_Done,
-// CHECK-NEXT:        // Label 2: @108
-// CHECK-NEXT:        GIM_Try, /*On fail goto*//*Label 3*/ 188, // Rule ID 2 //
+// CHECK-NEXT:        // Label 2: @126
+// CHECK-NEXT:        GIM_Try, /*On fail goto*//*Label 3*/ GIMT_Encode4(216), // Rule ID 2 //
 // CHECK-NEXT:          GIM_RecordInsn, /*DefineMI*/1, /*MI*/0, /*OpIdx*/1, // MIs[1]
-// CHECK-NEXT:          GIM_CheckOpcode, /*MI*/1, TargetOpcode::G_ICMP,
+// CHECK-NEXT:          GIM_CheckOpcode, /*MI*/1, GIMT_Encode2(TargetOpcode::G_ICMP),
 // CHECK-NEXT:          GIM_CheckType, /*MI*/1, /*Op*/2, /*Type*/GILLT_s32,
 // CHECK-NEXT:          GIM_CheckType, /*MI*/1, /*Op*/3, /*Type*/GILLT_s32,
 // CHECK-NEXT:          // MIs[1] Operand 1
-// CHECK-NEXT:          GIM_CheckCmpPredicate, /*MI*/1, /*Op*/1, /*Predicate*/CmpInst::ICMP_NE,
-// CHECK-NEXT:          GIM_CheckRegBankForClass, /*MI*/1, /*Op*/2, /*RC*/MyTarget::GPR32RegClassID,
-// CHECK-NEXT:          GIM_CheckConstantInt, /*MI*/1, /*Op*/3, 0,
+// CHECK-NEXT:          GIM_CheckCmpPredicate, /*MI*/1, /*Op*/1, /*Predicate*/GIMT_Encode2(CmpInst::ICMP_NE),
+// CHECK-NEXT:          GIM_CheckRegBankForClass, /*MI*/1, /*Op*/2, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
+// CHECK-NEXT:          GIM_CheckConstantInt8, /*MI*/1, /*Op*/3, 0,
 // CHECK-NEXT:          GIM_RecordInsn, /*DefineMI*/2, /*MI*/0, /*OpIdx*/2, // MIs[2]
-// CHECK-NEXT:          GIM_CheckOpcode, /*MI*/2, TargetOpcode::G_SUB,
+// CHECK-NEXT:          GIM_CheckOpcode, /*MI*/2, GIMT_Encode2(TargetOpcode::G_SUB),
 // CHECK-NEXT:          GIM_CheckType, /*MI*/2, /*Op*/1, /*Type*/GILLT_s32,
 // CHECK-NEXT:          GIM_CheckType, /*MI*/2, /*Op*/2, /*Type*/GILLT_s32,
-// CHECK-NEXT:          GIM_CheckRegBankForClass, /*MI*/2, /*Op*/1, /*RC*/MyTarget::GPR32RegClassID,
-// CHECK-NEXT:          GIM_CheckRegBankForClass, /*MI*/2, /*Op*/2, /*RC*/MyTarget::GPR32RegClassID,
+// CHECK-NEXT:          GIM_CheckRegBankForClass, /*MI*/2, /*Op*/1, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
+// CHECK-NEXT:          GIM_CheckRegBankForClass, /*MI*/2, /*Op*/2, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // CHECK-NEXT:          GIM_CheckIsSafeToFold, /*InsnID*/1,
 // CHECK-NEXT:          GIM_CheckIsSafeToFold, /*InsnID*/2,
 // CHECK-NEXT:          // (select:{ *:[i32] } (setcc:{ *:[i32] } GPR32:{ *:[i32] }:$cond, 0:{ *:[i32] }, SETNE:{ *:[Other] }), (sub:{ *:[i32] } GPR32:{ *:[i32] }:$src1, GPR32:{ *:[i32] }:$src2), GPR32:{ *:[i32] }:$src2)  =>  (InstThreeOperands:{ *:[i32] } GPR32:{ *:[i32] }:$cond, GPR32:{ *:[i32] }:$src1, GPR32:{ *:[i32] }:$src2)
-// CHECK-NEXT:          GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::InstThreeOperands,
+// CHECK-NEXT:          GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::InstThreeOperands),
 // CHECK-NEXT:          GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst]
 // CHECK-NEXT:          GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/1, /*OpIdx*/2, // cond
 // CHECK-NEXT:          GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/2, /*OpIdx*/1, // src1
@@ -69,25 +69,24 @@ def InstThreeOperands : I<(outs GPR32:$dst), (ins GPR32:$cond, GPR32:$src,GPR32:
 // CHECK-NEXT:          GIR_ConstrainSelectedInstOperands, /*InsnID*/0,
 // CHECK-NEXT:          // GIR_Coverage, 2,
 // CHECK-NEXT:          GIR_Done,
-// CHECK-NEXT:        // Label 3: @188
+// CHECK-NEXT:        // Label 3: @216
 // CHECK-NEXT:        GIM_Reject,
-// CHECK-NEXT:      // Label 1: @189
-// CHECK-NEXT:      GIM_Try, /*On fail goto*//*Label 4*/ 218, // Rule ID 0 //
+// CHECK-NEXT:      // Label 1: @217
+// CHECK-NEXT:      GIM_Try, /*On fail goto*//*Label 4*/ GIMT_Encode4(254), // Rule ID 0 //
 // CHECK-NEXT:        GIM_CheckType, /*MI*/0, /*Op*/3, /*Type*/GILLT_s32,
-// CHECK-NEXT:        GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID,
-// CHECK-NEXT:        GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/MyTarget::GPR32RegClassID,
-// CHECK-NEXT:        GIM_CheckRegBankForClass, /*MI*/0, /*Op*/2, /*RC*/MyTarget::GPR32RegClassID,
-// CHECK-NEXT:        GIM_CheckRegBankForClass, /*MI*/0, /*Op*/3, /*RC*/MyTarget::GPR32RegClassID,
+// CHECK-NEXT:        GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
+// CHECK-NEXT:        GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
+// CHECK-NEXT:        GIM_CheckRegBankForClass, /*MI*/0, /*Op*/2, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
+// CHECK-NEXT:        GIM_CheckRegBankForClass, /*MI*/0, /*Op*/3, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // CHECK-NEXT:        // (select:{ *:[i32] } GPR32:{ *:[i32] }:$cond, GPR32:{ *:[i32] }:$src1, GPR32:{ *:[i32] }:$src2)  =>  (InstThreeOperands:{ *:[i32] } GPR32:{ *:[i32] }:$cond, GPR32:{ *:[i32] }:$src1, GPR32:{ *:[i32] }:$src2)
-// CHECK-NEXT:        GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/MyTarget::InstThreeOperands,
+// CHECK-NEXT:        GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::InstThreeOperands),
 // CHECK-NEXT:        GIR_ConstrainSelectedInstOperands, /*InsnID*/0,
 // CHECK-NEXT:        // GIR_Coverage, 0,
 // CHECK-NEXT:        GIR_Done,
-// CHECK-NEXT:      // Label 4: @218
+// CHECK-NEXT:      // Label 4: @254
 // CHECK-NEXT:      GIM_Reject,
-// CHECK-NEXT:    // Label 0: @219
+// CHECK-NEXT:    // Label 0: @255
 // CHECK-NEXT:    GIM_Reject,
-// CHECK-NEXT:    }
 def : Pat<(i32 (select GPR32:$cond, GPR32:$src1, GPR32:$src2)),
           (InstThreeOperands GPR32:$cond, GPR32:$src1, GPR32:$src2)>;
 
diff --git a/llvm/test/TableGen/GlobalISelEmitterMatchTableOptimizerSameOperand.td b/llvm/test/TableGen/GlobalISelEmitterMatchTableOptimizerSameOperand.td
index 167ab83b862b0130696e3aadeb8cef90bcf2a48c..00ece5f4ca7989bce7c0e4af7f3711d05a73a4af 100644
--- a/llvm/test/TableGen/GlobalISelEmitterMatchTableOptimizerSameOperand.td
+++ b/llvm/test/TableGen/GlobalISelEmitterMatchTableOptimizerSameOperand.td
@@ -9,7 +9,7 @@ def InstThreeOperands : I<(outs GPR32:$dst), (ins GPR32:$cond, GPR32:$src,GPR32:
 // Make sure the GIM_CheckIsSameOperand check is not hoisted into the common header group
 
 // CHECK:       GIM_Try, /*On fail goto*//*Label 1*/
-// CHECK-NEXT:  GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID,
+// CHECK-NEXT:  GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // CHECK-NOT:   GIM_CheckIsSameOperand
 // CHECK-NEXT:  GIM_Try, /*On fail goto*//*Label 2*/
 // CHECK:       GIM_CheckIsSameOperand, /*MI*/0, /*OpIdx*/3, /*OtherMI*/2, /*OtherOpIdx*/1,
diff --git a/llvm/test/TableGen/GlobalISelEmitterOverloadedPtr.td b/llvm/test/TableGen/GlobalISelEmitterOverloadedPtr.td
index 0cc3485342cc6105d88feff333857628883ad846..64723a0bbd4b68b64d4fa98a7757d7a225f578df 100644
--- a/llvm/test/TableGen/GlobalISelEmitterOverloadedPtr.td
+++ b/llvm/test/TableGen/GlobalISelEmitterOverloadedPtr.td
@@ -11,15 +11,15 @@ let TargetPrefix = "mytarget" in {
 // Ensure that llvm_anyptr_ty on an intrinsic results in a
 // GIM_CheckPointerToAny rather than a GIM_CheckType.
 //
-// CHECK: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/1, Intrinsic::mytarget_anyptr,
+// CHECK: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/1, GIMT_Encode2(Intrinsic::mytarget_anyptr),
 // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
-// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID,
+// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // CHECK-NEXT: // MIs[0] src
 // CHECK-NEXT: GIM_CheckPointerToAny, /*MI*/0, /*Op*/2, /*SizeInBits*/32,
-// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/2, /*RC*/MyTarget::GPR32RegClassID,
-// CHECK-NEXT: GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GICXXPred_MI_Predicate_frag_anyptr,
+// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/2, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
+// CHECK-NEXT: GIM_CheckCxxInsnPredicate, /*MI*/0, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_frag_anyptr),
 // CHECK-NEXT: // (intrinsic_w_chain:{ *:[i32] } {{[0-9]+}}:{ *:[iPTR] }, GPR32:{ *:[i32] }:$src)<<P:Predicate_frag_anyptr>>  =>  (ANYLOAD:{ *:[i32] } GPR32:{ *:[i32] }:$src)
-// CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::ANYLOAD,
+// CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::ANYLOAD),
 let hasSideEffects = 1 in {
   def ANYLOAD : I<(outs GPR32:$dst), (ins GPR32:$src1),
             [(set GPR32:$dst, (load GPR32:$src1))]>;
diff --git a/llvm/test/TableGen/GlobalISelEmitterRegSequence.td b/llvm/test/TableGen/GlobalISelEmitterRegSequence.td
index 34783e688399dec67ef0796276a55605d6ebafb6..1f2174e7e4a9b5c68724018a9ee569a01aa1b53d 100644
--- a/llvm/test/TableGen/GlobalISelEmitterRegSequence.td
+++ b/llvm/test/TableGen/GlobalISelEmitterRegSequence.td
@@ -31,50 +31,50 @@ def SOME_INSN : I<(outs DRegs:$dst), (ins DOP:$src), []>;
 def SUBSOME_INSN : I<(outs SRegs:$dst), (ins SOP:$src), []>;
 
 // CHECK: GIM_CheckNumOperands, /*MI*/0, /*Expected*/2,
-// CHECK-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_SEXT,
+// CHECK-NEXT: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_SEXT),
 // CHECK-NEXT: // MIs[0] DstI[dst]
 // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
-// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/Test::DRegsRegClassID,
+// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(Test::DRegsRegClassID),
 // CHECK-NEXT: // MIs[0] src
 // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s16,
-// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/Test::SRegsRegClassID,
+// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/GIMT_Encode2(Test::SRegsRegClassID),
 // CHECK-NEXT: // (sext:{ *:[i32] } SOP:{ *:[i16] }:$src)  =>  (REG_SEQUENCE:{ *:[i32] } DRegs:{ *:[i32] }, (SUBSOME_INSN:{ *:[i16] } SOP:{ *:[i16] }:$src), sub0:{ *:[i32] }, (SUBSOME_INSN:{ *:[i16] } SOP:{ *:[i16] }:$src), sub1:{ *:[i32] })
 // CHECK-NEXT: GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s16,
 // CHECK-NEXT: GIR_MakeTempReg, /*TempRegID*/1, /*TypeID*/GILLT_s16,
-// CHECK-NEXT: GIR_BuildMI, /*InsnID*/2, /*Opcode*/MyTarget::SUBSOME_INSN,
-// CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/2, /*TempRegID*/1, /*TempRegFlags*/RegState::Define,
+// CHECK-NEXT: GIR_BuildMI, /*InsnID*/2, /*Opcode*/GIMT_Encode2(MyTarget::SUBSOME_INSN),
+// CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/2, /*TempRegID*/1, /*TempRegFlags*/GIMT_Encode2(RegState::Define),
 // CHECK-NEXT: GIR_Copy, /*NewInsnID*/2, /*OldInsnID*/0, /*OpIdx*/1, // src
 // CHECK-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/2,
-// CHECK-NEXT: GIR_BuildMI, /*InsnID*/1, /*Opcode*/MyTarget::SUBSOME_INSN,
-// CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/1, /*TempRegID*/0, /*TempRegFlags*/RegState::Define,
+// CHECK-NEXT: GIR_BuildMI, /*InsnID*/1, /*Opcode*/GIMT_Encode2(MyTarget::SUBSOME_INSN),
+// CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/1, /*TempRegID*/0, /*TempRegFlags*/GIMT_Encode2(RegState::Define),
 // CHECK-NEXT: GIR_Copy, /*NewInsnID*/1, /*OldInsnID*/0, /*OpIdx*/1, // src
 // CHECK-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/1,
-// CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/TargetOpcode::REG_SEQUENCE,
+// CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(TargetOpcode::REG_SEQUENCE),
 // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst]
-// CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/0,
-// CHECK-NEXT: GIR_AddImm, /*InsnID*/0, /*SubRegIndex*/1,
-// CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/1, /*TempRegFlags*/0,
-// CHECK-NEXT: GIR_AddImm, /*InsnID*/0, /*SubRegIndex*/2,
+// CHECK-NEXT: GIR_AddSimpleTempRegister, /*InsnID*/0, /*TempRegID*/0,
+// CHECK-NEXT: GIR_AddImm8, /*InsnID*/0, /*SubRegIndex*/1,
+// CHECK-NEXT: GIR_AddSimpleTempRegister, /*InsnID*/0, /*TempRegID*/1,
+// CHECK-NEXT: GIR_AddImm8, /*InsnID*/0, /*SubRegIndex*/2,
 // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0,
-// CHECK-NEXT: GIR_ConstrainOperandRC, /*InsnID*/0, /*Op*/0, Test::DRegsRegClassID,
-// CHECK-NEXT: GIR_ConstrainOperandRC, /*InsnID*/0, /*Op*/1, Test::SRegsRegClassID,
-// CHECK-NEXT: GIR_ConstrainOperandRC, /*InsnID*/0, /*Op*/3, Test::SRegsRegClassID,
+// CHECK-NEXT: GIR_ConstrainOperandRC, /*InsnID*/0, /*Op*/0, GIMT_Encode2(Test::DRegsRegClassID),
+// CHECK-NEXT: GIR_ConstrainOperandRC, /*InsnID*/0, /*Op*/1, GIMT_Encode2(Test::SRegsRegClassID),
+// CHECK-NEXT: GIR_ConstrainOperandRC, /*InsnID*/0, /*Op*/3, GIMT_Encode2(Test::SRegsRegClassID),
 def : Pat<(i32 (sext SOP:$src)),
           (REG_SEQUENCE DRegs, (SUBSOME_INSN SOP:$src), sub0,
                                (SUBSOME_INSN SOP:$src), sub1)>;
 
 
-// CHECK: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_ZEXT,
-// CHECK: GIR_BuildMI, /*InsnID*/1, /*Opcode*/TargetOpcode::REG_SEQUENCE,
-// CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/1, /*TempRegID*/0, /*TempRegFlags*/RegState::Define,
-// CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/1, /*TempRegID*/1, /*TempRegFlags*/0,
-// CHECK-NEXT: GIR_AddImm, /*InsnID*/1, /*SubRegIndex*/1,
-// CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/1, /*TempRegID*/2, /*TempRegFlags*/0,
-// CHECK-NEXT: GIR_AddImm, /*InsnID*/1, /*SubRegIndex*/2,
-// CHECK-NEXT: GIR_ConstrainOperandRC, /*InsnID*/1, /*Op*/0, Test::DRegsRegClassID,
-// CHECK-NEXT: GIR_ConstrainOperandRC, /*InsnID*/1, /*Op*/1, Test::SRegsRegClassID,
-// CHECK-NEXT: GIR_ConstrainOperandRC, /*InsnID*/1, /*Op*/3, Test::SRegsRegClassID,
-// CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::SOME_INSN,
+// CHECK: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_ZEXT),
+// CHECK: GIR_BuildMI, /*InsnID*/1, /*Opcode*/GIMT_Encode2(TargetOpcode::REG_SEQUENCE),
+// CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/1, /*TempRegID*/0, /*TempRegFlags*/GIMT_Encode2(RegState::Define),
+// CHECK-NEXT: GIR_AddSimpleTempRegister, /*InsnID*/1, /*TempRegID*/1,
+// CHECK-NEXT: GIR_AddImm8, /*InsnID*/1, /*SubRegIndex*/1,
+// CHECK-NEXT: GIR_AddSimpleTempRegister, /*InsnID*/1, /*TempRegID*/2,
+// CHECK-NEXT: GIR_AddImm8, /*InsnID*/1, /*SubRegIndex*/2,
+// CHECK-NEXT: GIR_ConstrainOperandRC, /*InsnID*/1, /*Op*/0, GIMT_Encode2(Test::DRegsRegClassID),
+// CHECK-NEXT: GIR_ConstrainOperandRC, /*InsnID*/1, /*Op*/1, GIMT_Encode2(Test::SRegsRegClassID),
+// CHECK-NEXT: GIR_ConstrainOperandRC, /*InsnID*/1, /*Op*/3, GIMT_Encode2(Test::SRegsRegClassID),
+// CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::SOME_INSN),
 // Make sure operands are constrained when REG_SEQUENCE isn't the root instruction.
 def : Pat<(i32 (zext SOP:$src)),
           (SOME_INSN (REG_SEQUENCE DRegs, (SUBSOME_INSN SOP:$src), sub0,
diff --git a/llvm/test/TableGen/GlobalISelEmitterSubreg.td b/llvm/test/TableGen/GlobalISelEmitterSubreg.td
index af3db1590368b7a26882c3bf4e35dc5ff0490b8e..b601727c99654858629ca23830cae9615bad8d3d 100644
--- a/llvm/test/TableGen/GlobalISelEmitterSubreg.td
+++ b/llvm/test/TableGen/GlobalISelEmitterSubreg.td
@@ -61,20 +61,20 @@ def : Pat<(sub (complex DOP:$src1, DOP:$src2), 77),
 // CHECK-LABEL: // (sub:{ *:[i32] } (complex:{ *:[i32] } DOP:{ *:[i32] }:$src1, DOP:{ *:[i32] }:$src2), 77:{ *:[i32] })  =>  (SOME_INSN2:{ *:[i32] } (EXTRACT_SUBREG:{ *:[i32] } DOP:{ *:[i32] }:$src1, sub0:{ *:[i32] }), (EXTRACT_SUBREG:{ *:[i32] } DOP:{ *:[i32] }:$src2, sub1:{ *:[i32] }))
 // CHECK-NEXT: GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s32,
 // CHECK-NEXT: GIR_MakeTempReg, /*TempRegID*/1, /*TypeID*/GILLT_s32,
-// CHECK-NEXT: GIR_BuildMI, /*InsnID*/2, /*Opcode*/TargetOpcode::COPY,
-// CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/2, /*TempRegID*/1, /*TempRegFlags*/RegState::Define,
-// CHECK-NEXT: GIR_ComplexSubOperandSubRegRenderer, /*InsnID*/2, /*RendererID*/0, /*SubOperand*/1, /*SubRegIdx*/2, // src2
-// CHECK-NEXT: GIR_ConstrainOperandRC, /*InsnID*/2, /*Op*/0, Test::SRegsRegClassID,
-// CHECK-NEXT: GIR_ConstrainOperandRC, /*InsnID*/2, /*Op*/1, Test::DRegsRegClassID,
-// CHECK-NEXT: GIR_BuildMI, /*InsnID*/1, /*Opcode*/TargetOpcode::COPY,
-// CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/1, /*TempRegID*/0, /*TempRegFlags*/RegState::Define,
-// CHECK-NEXT: GIR_ComplexSubOperandSubRegRenderer, /*InsnID*/1, /*RendererID*/0, /*SubOperand*/0, /*SubRegIdx*/1, // src1
-// CHECK-NEXT: GIR_ConstrainOperandRC, /*InsnID*/1, /*Op*/0, Test::SRegsRegClassID,
-// CHECK-NEXT: GIR_ConstrainOperandRC, /*InsnID*/1, /*Op*/1, Test::DRegsRegClassID,
-// CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::SOME_INSN2,
+// CHECK-NEXT: GIR_BuildMI, /*InsnID*/2, /*Opcode*/GIMT_Encode2(TargetOpcode::COPY),
+// CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/2, /*TempRegID*/1, /*TempRegFlags*/GIMT_Encode2(RegState::Define),
+// CHECK-NEXT: GIR_ComplexSubOperandSubRegRenderer, /*InsnID*/2, /*RendererID*/GIMT_Encode2(0), /*SubOperand*/1, /*SubRegIdx*/GIMT_Encode2(2), // src2
+// CHECK-NEXT: GIR_ConstrainOperandRC, /*InsnID*/2, /*Op*/0, GIMT_Encode2(Test::SRegsRegClassID),
+// CHECK-NEXT: GIR_ConstrainOperandRC, /*InsnID*/2, /*Op*/1, GIMT_Encode2(Test::DRegsRegClassID),
+// CHECK-NEXT: GIR_BuildMI, /*InsnID*/1, /*Opcode*/GIMT_Encode2(TargetOpcode::COPY),
+// CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/1, /*TempRegID*/0, /*TempRegFlags*/GIMT_Encode2(RegState::Define),
+// CHECK-NEXT: GIR_ComplexSubOperandSubRegRenderer, /*InsnID*/1, /*RendererID*/GIMT_Encode2(0), /*SubOperand*/0, /*SubRegIdx*/GIMT_Encode2(1), // src1
+// CHECK-NEXT: GIR_ConstrainOperandRC, /*InsnID*/1, /*Op*/0, GIMT_Encode2(Test::SRegsRegClassID),
+// CHECK-NEXT: GIR_ConstrainOperandRC, /*InsnID*/1, /*Op*/1, GIMT_Encode2(Test::DRegsRegClassID),
+// CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::SOME_INSN2),
 // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst]
-// CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/0,
-// CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/1, /*TempRegFlags*/0,
+// CHECK-NEXT: GIR_AddSimpleTempRegister, /*InsnID*/0, /*TempRegID*/0,
+// CHECK-NEXT: GIR_AddSimpleTempRegister, /*InsnID*/0, /*TempRegID*/1,
 // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0,
 // CHECK-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0,
 
@@ -83,18 +83,18 @@ def : Pat<(sub (complex DOP:$src1, DOP:$src2), 77),
 def : Pat<(i32 (anyext i16:$src)), (INSERT_SUBREG (i32 (IMPLICIT_DEF)), SOP:$src, sub0)>;
 // CHECK-LABEL:  (anyext:{ *:[i32] } i16:{ *:[i16] }:$src)  =>  (INSERT_SUBREG:{ *:[i32] } (IMPLICIT_DEF:{ *:[i32] }), SOP:{ *:[i16] }:$src, sub0:{ *:[i32] })
 // CHECK-NEXT:            GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s32,
-// CHECK-NEXT:            GIR_BuildMI, /*InsnID*/1, /*Opcode*/TargetOpcode::IMPLICIT_DEF,
-// CHECK-NEXT:            GIR_AddTempRegister, /*InsnID*/1, /*TempRegID*/0, /*TempRegFlags*/RegState::Define,
+// CHECK-NEXT:            GIR_BuildMI, /*InsnID*/1, /*Opcode*/GIMT_Encode2(TargetOpcode::IMPLICIT_DEF),
+// CHECK-NEXT:            GIR_AddTempRegister, /*InsnID*/1, /*TempRegID*/0, /*TempRegFlags*/GIMT_Encode2(RegState::Define),
 // CHECK-NEXT:            GIR_ConstrainSelectedInstOperands, /*InsnID*/1,
-// CHECK-NEXT:            GIR_BuildMI, /*InsnID*/0, /*Opcode*/TargetOpcode::INSERT_SUBREG,
+// CHECK-NEXT:            GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(TargetOpcode::INSERT_SUBREG),
 // CHECK-NEXT:            GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst]
-// CHECK-NEXT:            GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/0,
+// CHECK-NEXT:            GIR_AddSimpleTempRegister, /*InsnID*/0, /*TempRegID*/0,
 // CHECK-NEXT:            GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // src
-// CHECK-NEXT:            GIR_AddImm, /*InsnID*/0, /*Imm*/1,
+// CHECK-NEXT:            GIR_AddImm8, /*InsnID*/0, /*Imm*/1,
 // CHECK-NEXT:            GIR_EraseFromParent, /*InsnID*/0,
-// CHECK-NEXT:            GIR_ConstrainOperandRC, /*InsnID*/0, /*Op*/0, Test::DRegsRegClassID,
-// CHECK-NEXT:            GIR_ConstrainOperandRC, /*InsnID*/0, /*Op*/1, Test::DRegsRegClassID
-// CHECK-NEXT:            GIR_ConstrainOperandRC, /*InsnID*/0, /*Op*/2, Test::SRegsRegClassID,
+// CHECK-NEXT:            GIR_ConstrainOperandRC, /*InsnID*/0, /*Op*/0, GIMT_Encode2(Test::DRegsRegClassID),
+// CHECK-NEXT:            GIR_ConstrainOperandRC, /*InsnID*/0, /*Op*/1, GIMT_Encode2(Test::DRegsRegClassID),
+// CHECK-NEXT:            GIR_ConstrainOperandRC, /*InsnID*/0, /*Op*/2, GIMT_Encode2(Test::SRegsRegClassID),
 
 
 // Test that we can import INSERT_SUBREG when it is a subinstruction of another
@@ -103,20 +103,20 @@ def : Pat<(i32 (anyext i16:$src)), (SOME_INSN (INSERT_SUBREG (i32 (IMPLICIT_DEF)
 // CHECK-LABEL:  (anyext:{ *:[i32] } i16:{ *:[i16] }:$src)  =>  (SOME_INSN:{ *:[i32] } (INSERT_SUBREG:{ *:[i32] } (IMPLICIT_DEF:{ *:[i32] }), SOP:{ *:[i16] }:$src, sub0:{ *:[i32] }))
 // CHECK-NEXT:            GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s32,
 // CHECK-NEXT:            GIR_MakeTempReg, /*TempRegID*/1, /*TypeID*/GILLT_s32,
-// CHECK-NEXT:            GIR_BuildMI, /*InsnID*/2, /*Opcode*/TargetOpcode::IMPLICIT_DEF,
-// CHECK-NEXT:            GIR_AddTempRegister, /*InsnID*/2, /*TempRegID*/1, /*TempRegFlags*/RegState::Define,
+// CHECK-NEXT:            GIR_BuildMI, /*InsnID*/2, /*Opcode*/GIMT_Encode2(TargetOpcode::IMPLICIT_DEF),
+// CHECK-NEXT:            GIR_AddTempRegister, /*InsnID*/2, /*TempRegID*/1, /*TempRegFlags*/GIMT_Encode2(RegState::Define),
 // CHECK-NEXT:            GIR_ConstrainSelectedInstOperands, /*InsnID*/2,
-// CHECK-NEXT:            GIR_BuildMI, /*InsnID*/1, /*Opcode*/TargetOpcode::INSERT_SUBREG,
-// CHECK-NEXT:            GIR_AddTempRegister, /*InsnID*/1, /*TempRegID*/0, /*TempRegFlags*/RegState::Define,
-// CHECK-NEXT:            GIR_AddTempRegister, /*InsnID*/1, /*TempRegID*/1, /*TempRegFlags*/0,
+// CHECK-NEXT:            GIR_BuildMI, /*InsnID*/1, /*Opcode*/GIMT_Encode2(TargetOpcode::INSERT_SUBREG),
+// CHECK-NEXT:            GIR_AddTempRegister, /*InsnID*/1, /*TempRegID*/0, /*TempRegFlags*/GIMT_Encode2(RegState::Define),
+// CHECK-NEXT:            GIR_AddSimpleTempRegister, /*InsnID*/1, /*TempRegID*/1,
 // CHECK-NEXT:            GIR_Copy, /*NewInsnID*/1, /*OldInsnID*/0, /*OpIdx*/1, // src
-// CHECK-NEXT:            GIR_AddImm, /*InsnID*/1, /*Imm*/1,
-// CHECK-NEXT:            GIR_ConstrainOperandRC, /*InsnID*/1, /*Op*/0, Test::DRegsRegClassID,
-// CHECK-NEXT:            GIR_ConstrainOperandRC, /*InsnID*/1, /*Op*/1, Test::DRegsRegClassID,
-// CHECK-NEXT:            GIR_ConstrainOperandRC, /*InsnID*/1, /*Op*/2, Test::SRegsRegClassID,
-// CHECK-NEXT:            GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::SOME_INSN,
+// CHECK-NEXT:            GIR_AddImm8, /*InsnID*/1, /*Imm*/1,
+// CHECK-NEXT:            GIR_ConstrainOperandRC, /*InsnID*/1, /*Op*/0, GIMT_Encode2(Test::DRegsRegClassID),
+// CHECK-NEXT:            GIR_ConstrainOperandRC, /*InsnID*/1, /*Op*/1, GIMT_Encode2(Test::DRegsRegClassID),
+// CHECK-NEXT:            GIR_ConstrainOperandRC, /*InsnID*/1, /*Op*/2, GIMT_Encode2(Test::SRegsRegClassID),
+// CHECK-NEXT:            GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::SOME_INSN),
 // CHECK-NEXT:            GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst]
-// CHECK-NEXT:            GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/0,
+// CHECK-NEXT:            GIR_AddSimpleTempRegister, /*InsnID*/0, /*TempRegID*/0,
 // CHECK-NEXT:            GIR_EraseFromParent, /*InsnID*/0,
 // CHECK-NEXT:            GIR_ConstrainSelectedInstOperands, /*InsnID*/0,
 
@@ -126,10 +126,10 @@ def : Pat<(i32 (anyext i16:$src)), (SOME_INSN (INSERT_SUBREG (i32 (IMPLICIT_DEF)
 // not a D register.
 def : Pat<(i32 (anyext i16:$src)), (INSERT_SUBREG (i32 (COPY_TO_REGCLASS SOP:$src, ERegs)), SOP:$src, sub0)>;
 // CHECK-LABEL:  (anyext:{ *:[i32] } i16:{ *:[i16] }:$src)  =>  (INSERT_SUBREG:{ *:[i32] } (COPY_TO_REGCLASS:{ *:[i32] } SOP:{ *:[i16] }:$src, ERegs:{ *:[i32] }), SOP:{ *:[i16] }:$src, sub0:{ *:[i32] })
-// CHECK:                GIR_BuildMI, /*InsnID*/0, /*Opcode*/TargetOpcode::INSERT_SUBREG,
-// CHECK-DAG:            GIR_ConstrainOperandRC, /*InsnID*/0, /*Op*/0, Test::ERegsRegClassID,
-// CHECK-NEXT:           GIR_ConstrainOperandRC, /*InsnID*/0, /*Op*/1, Test::ERegsRegClassID,
-// CHECK-NEXT:           GIR_ConstrainOperandRC, /*InsnID*/0, /*Op*/2, Test::SRegsRegClassID,
+// CHECK:                GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(TargetOpcode::INSERT_SUBREG),
+// CHECK-DAG:            GIR_ConstrainOperandRC, /*InsnID*/0, /*Op*/0, GIMT_Encode2(Test::ERegsRegClassID),
+// CHECK-NEXT:           GIR_ConstrainOperandRC, /*InsnID*/0, /*Op*/1, GIMT_Encode2(Test::ERegsRegClassID),
+// CHECK-NEXT:           GIR_ConstrainOperandRC, /*InsnID*/0, /*Op*/2, GIMT_Encode2(Test::SRegsRegClassID),
 
 // Test that we can import INSERT_SUBREG when its subregister source is defined
 // by a subinstruction.
@@ -137,22 +137,22 @@ def : Pat<(i32 (anyext i16:$src)), (INSERT_SUBREG (i32 (IMPLICIT_DEF)), (SUBSOME
 // CHECK-LABEL:  (anyext:{ *:[i32] } i16:{ *:[i16] }:$src)  =>  (INSERT_SUBREG:{ *:[i32] } (IMPLICIT_DEF:{ *:[i32] }), (SUBSOME_INSN:{ *:[i16] } SOP:{ *:[i16] }:$src), sub0:{ *:[i32] })
 // CHECK-NEXT:          GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s32,
 // CHECK-NEXT:          GIR_MakeTempReg, /*TempRegID*/1, /*TypeID*/GILLT_s16,
-// CHECK-NEXT:          GIR_BuildMI, /*InsnID*/2, /*Opcode*/MyTarget::SUBSOME_INSN,
-// CHECK-NEXT:          GIR_AddTempRegister, /*InsnID*/2, /*TempRegID*/1, /*TempRegFlags*/RegState::Define,
+// CHECK-NEXT:          GIR_BuildMI, /*InsnID*/2, /*Opcode*/GIMT_Encode2(MyTarget::SUBSOME_INSN),
+// CHECK-NEXT:          GIR_AddTempRegister, /*InsnID*/2, /*TempRegID*/1, /*TempRegFlags*/GIMT_Encode2(RegState::Define),
 // CHECK-NEXT:          GIR_Copy, /*NewInsnID*/2, /*OldInsnID*/0, /*OpIdx*/1, // src
 // CHECK-NEXT:          GIR_ConstrainSelectedInstOperands, /*InsnID*/2,
-// CHECK-NEXT:          GIR_BuildMI, /*InsnID*/1, /*Opcode*/TargetOpcode::IMPLICIT_DEF,
-// CHECK-NEXT:          GIR_AddTempRegister, /*InsnID*/1, /*TempRegID*/0, /*TempRegFlags*/RegState::Define,
+// CHECK-NEXT:          GIR_BuildMI, /*InsnID*/1, /*Opcode*/GIMT_Encode2(TargetOpcode::IMPLICIT_DEF),
+// CHECK-NEXT:          GIR_AddTempRegister, /*InsnID*/1, /*TempRegID*/0, /*TempRegFlags*/GIMT_Encode2(RegState::Define),
 // CHECK-NEXT:          GIR_ConstrainSelectedInstOperands, /*InsnID*/1,
-// CHECK-NEXT:          GIR_BuildMI, /*InsnID*/0, /*Opcode*/TargetOpcode::INSERT_SUBREG,
+// CHECK-NEXT:          GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(TargetOpcode::INSERT_SUBREG),
 // CHECK-NEXT:          GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst]
-// CHECK-NEXT:          GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/0,
-// CHECK-NEXT:          GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/1, /*TempRegFlags*/0,
-// CHECK-NEXT:          GIR_AddImm, /*InsnID*/0, /*Imm*/1,
+// CHECK-NEXT:          GIR_AddSimpleTempRegister, /*InsnID*/0, /*TempRegID*/0,
+// CHECK-NEXT:          GIR_AddSimpleTempRegister, /*InsnID*/0, /*TempRegID*/1,
+// CHECK-NEXT:          GIR_AddImm8, /*InsnID*/0, /*Imm*/1,
 // CHECK-NEXT:          GIR_EraseFromParent, /*InsnID*/0,
-// CHECK-NEXT:          GIR_ConstrainOperandRC, /*InsnID*/0, /*Op*/0, Test::DRegsRegClassID,
-// CHECK-NEXT:          GIR_ConstrainOperandRC, /*InsnID*/0, /*Op*/1, Test::DRegsRegClassID,
-// CHECK-NEXT:          GIR_ConstrainOperandRC, /*InsnID*/0, /*Op*/2, Test::SRegsRegClassID,
+// CHECK-NEXT:          GIR_ConstrainOperandRC, /*InsnID*/0, /*Op*/0, GIMT_Encode2(Test::DRegsRegClassID),
+// CHECK-NEXT:          GIR_ConstrainOperandRC, /*InsnID*/0, /*Op*/1, GIMT_Encode2(Test::DRegsRegClassID),
+// CHECK-NEXT:          GIR_ConstrainOperandRC, /*InsnID*/0, /*Op*/2, GIMT_Encode2(Test::SRegsRegClassID),
 
 // Test an EXTRACT_SUBREG that is a sub instruction. The individual
 // operands should be constrained to specific register classes, and
@@ -161,54 +161,54 @@ def : Pat<(i16 (trunc (not DOP:$src))),
           (SUBSOME_INSN (EXTRACT_SUBREG DOP:$src, sub0))>;
 // CHECK-LABEL: // (trunc:{ *:[i16] } (xor:{ *:[i32] } DOP:{ *:[i32] }:$src, -1:{ *:[i32] }))  =>  (SUBSOME_INSN:{ *:[i16] } (EXTRACT_SUBREG:{ *:[i16] } DOP:{ *:[i32] }:$src, sub0:{ *:[i32] }))
 // CHECK-NEXT: GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s16,
-// CHECK-NEXT: GIR_BuildMI, /*InsnID*/1, /*Opcode*/TargetOpcode::COPY,
-// CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/1, /*TempRegID*/0, /*TempRegFlags*/RegState::Define,
-// CHECK-NEXT: GIR_CopySubReg, /*NewInsnID*/1, /*OldInsnID*/1, /*OpIdx*/1, /*SubRegIdx*/1, // src
-// CHECK-NEXT: GIR_ConstrainOperandRC, /*InsnID*/1, /*Op*/0, Test::SRegsRegClassID,
-// CHECK-NEXT: GIR_ConstrainOperandRC, /*InsnID*/1, /*Op*/1, Test::DRegsRegClassID,
-// CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::SUBSOME_INSN,
+// CHECK-NEXT: GIR_BuildMI, /*InsnID*/1, /*Opcode*/GIMT_Encode2(TargetOpcode::COPY),
+// CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/1, /*TempRegID*/0, /*TempRegFlags*/GIMT_Encode2(RegState::Define),
+// CHECK-NEXT: GIR_CopySubReg, /*NewInsnID*/1, /*OldInsnID*/1, /*OpIdx*/1, /*SubRegIdx*/GIMT_Encode2(1), // src
+// CHECK-NEXT: GIR_ConstrainOperandRC, /*InsnID*/1, /*Op*/0, GIMT_Encode2(Test::SRegsRegClassID),
+// CHECK-NEXT: GIR_ConstrainOperandRC, /*InsnID*/1, /*Op*/1, GIMT_Encode2(Test::DRegsRegClassID),
+// CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::SUBSOME_INSN),
 
 // Test an extract from an output instruction result (nonleaf)
 def : Pat<(i16 (trunc (bitreverse DOP:$src))),
            (EXTRACT_SUBREG (SOME_INSN DOP:$src), sub0)>;
-// CHECK-LABEL: GIM_CheckOpcode, /*MI*/1, TargetOpcode::G_BITREVERSE,
+// CHECK-LABEL: GIM_CheckOpcode, /*MI*/1, GIMT_Encode2(TargetOpcode::G_BITREVERSE),
 // CHECK-NEXT:  GIM_CheckType, /*MI*/1, /*Op*/1, /*Type*/GILLT_s32,
-// CHECK-NEXT:  GIM_CheckRegBankForClass, /*MI*/1, /*Op*/1, /*RC*/Test::DRegsRegClassID,
+// CHECK-NEXT:  GIM_CheckRegBankForClass, /*MI*/1, /*Op*/1, /*RC*/GIMT_Encode2(Test::DRegsRegClassID),
 // CHECK-NEXT:  GIM_CheckIsSafeToFold, /*InsnID*/1,
 // CHECK-NEXT:  // (trunc:{ *:[i16] } (bitreverse:{ *:[i32] } DOP:{ *:[i32] }:$src))  =>  (EXTRACT_SUBREG:{ *:[i16] } (SOME_INSN:{ *:[i32] } DOP:{ *:[i32] }:$src), sub0:{ *:[i32] })
 // CHECK-NEXT:  GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s32,
-// CHECK-NEXT:  GIR_BuildMI, /*InsnID*/1, /*Opcode*/MyTarget::SOME_INSN,
-// CHECK-NEXT:  GIR_AddTempRegister, /*InsnID*/1, /*TempRegID*/0, /*TempRegFlags*/RegState::Define,
+// CHECK-NEXT:  GIR_BuildMI, /*InsnID*/1, /*Opcode*/GIMT_Encode2(MyTarget::SOME_INSN),
+// CHECK-NEXT:  GIR_AddTempRegister, /*InsnID*/1, /*TempRegID*/0, /*TempRegFlags*/GIMT_Encode2(RegState::Define),
 // CHECK-NEXT:  GIR_Copy, /*NewInsnID*/1, /*OldInsnID*/1, /*OpIdx*/1, // src
 // CHECK-NEXT:  GIR_ConstrainSelectedInstOperands, /*InsnID*/1,
-// CHECK-NEXT:  GIR_BuildMI, /*InsnID*/0, /*Opcode*/TargetOpcode::COPY,
+// CHECK-NEXT:  GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(TargetOpcode::COPY),
 // CHECK-NEXT:  GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst]
-// CHECK-NEXT:  GIR_AddTempSubRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/0, sub0,
+// CHECK-NEXT:  GIR_AddTempSubRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/GIMT_Encode2(0), GIMT_Encode2(sub0),
 // CHECK-NEXT:  GIR_EraseFromParent, /*InsnID*/0,
-// CHECK-NEXT:  GIR_ConstrainOperandRC, /*InsnID*/0, /*Op*/0, Test::SRegsRegClassID,
-// CHECK-NEXT:  GIR_ConstrainOperandRC, /*InsnID*/0, /*Op*/1, Test::DRegsRegClassID,
+// CHECK-NEXT:  GIR_ConstrainOperandRC, /*InsnID*/0, /*Op*/0, GIMT_Encode2(Test::SRegsRegClassID),
+// CHECK-NEXT:  GIR_ConstrainOperandRC, /*InsnID*/0, /*Op*/1, GIMT_Encode2(Test::DRegsRegClassID),
 
 // EXTRACT_SUBREG is subinstruction, but also doesn't have a leaf input
 
-// CHECK-LABEL: GIM_CheckOpcode, /*MI*/1, TargetOpcode::G_CTPOP,
+// CHECK-LABEL: GIM_CheckOpcode, /*MI*/1, GIMT_Encode2(TargetOpcode::G_CTPOP),
 // CHECK-NEXT: GIM_CheckType, /*MI*/1, /*Op*/1, /*Type*/GILLT_s32,
-// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/1, /*Op*/1, /*RC*/Test::DRegsRegClassID,
+// CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/1, /*Op*/1, /*RC*/GIMT_Encode2(Test::DRegsRegClassID),
 // CHECK-NEXT: GIM_CheckIsSafeToFold, /*InsnID*/1,
 // CHECK-NEXT: // (trunc:{ *:[i16] } (ctpop:{ *:[i32] } DOP:{ *:[i32] }:$src))  =>  (SUBSOME_INSN2:{ *:[i16] } (EXTRACT_SUBREG:{ *:[i16] } (SOME_INSN:{ *:[i32] } DOP:{ *:[i32] }:$src), sub0:{ *:[i32] }))
 // CHECK-NEXT: GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s16,
 // CHECK-NEXT: GIR_MakeTempReg, /*TempRegID*/1, /*TypeID*/GILLT_s32,
-// CHECK-NEXT: GIR_BuildMI, /*InsnID*/2, /*Opcode*/MyTarget::SOME_INSN,
-// CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/2, /*TempRegID*/1, /*TempRegFlags*/RegState::Define,
+// CHECK-NEXT: GIR_BuildMI, /*InsnID*/2, /*Opcode*/GIMT_Encode2(MyTarget::SOME_INSN),
+// CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/2, /*TempRegID*/1, /*TempRegFlags*/GIMT_Encode2(RegState::Define),
 // CHECK-NEXT: GIR_Copy, /*NewInsnID*/2, /*OldInsnID*/1, /*OpIdx*/1, // src
 // CHECK-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/2,
-// CHECK-NEXT: GIR_BuildMI, /*InsnID*/1, /*Opcode*/TargetOpcode::COPY,
-// CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/1, /*TempRegID*/0, /*TempRegFlags*/RegState::Define,
-// CHECK-NEXT: GIR_AddTempSubRegister, /*InsnID*/1, /*TempRegID*/1, /*TempRegFlags*/0, sub0,
-// CHECK-NEXT: GIR_ConstrainOperandRC, /*InsnID*/1, /*Op*/0, Test::SRegsRegClassID,
-// CHECK-NEXT: GIR_ConstrainOperandRC, /*InsnID*/1, /*Op*/1, Test::DRegsRegClassID,
-// CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::SUBSOME_INSN2,
+// CHECK-NEXT: GIR_BuildMI, /*InsnID*/1, /*Opcode*/GIMT_Encode2(TargetOpcode::COPY),
+// CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/1, /*TempRegID*/0, /*TempRegFlags*/GIMT_Encode2(RegState::Define),
+// CHECK-NEXT: GIR_AddTempSubRegister, /*InsnID*/1, /*TempRegID*/1, /*TempRegFlags*/GIMT_Encode2(0), GIMT_Encode2(sub0),
+// CHECK-NEXT: GIR_ConstrainOperandRC, /*InsnID*/1, /*Op*/0, GIMT_Encode2(Test::SRegsRegClassID),
+// CHECK-NEXT: GIR_ConstrainOperandRC, /*InsnID*/1, /*Op*/1, GIMT_Encode2(Test::DRegsRegClassID),
+// CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::SUBSOME_INSN2),
 // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst]
-// CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/0,
+// CHECK-NEXT: GIR_AddSimpleTempRegister, /*InsnID*/0, /*TempRegID*/0,
 // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0,
 // CHECK-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0,
 def : Pat<(i16 (trunc (ctpop DOP:$src))),
@@ -218,12 +218,12 @@ def : Pat<(i16 (trunc (ctpop DOP:$src))),
 def : Pat<(i16 (trunc DOP:$src)),
            (EXTRACT_SUBREG DOP:$src, sub0)>;
 // CHECK-LABEL: // (trunc:{ *:[i16] } DOP:{ *:[i32] }:$src)  =>  (EXTRACT_SUBREG:{ *:[i16] } DOP:{ *:[i32] }:$src, sub0:{ *:[i32] })
-// CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/TargetOpcode::COPY,
+// CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(TargetOpcode::COPY),
 // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst]
-// CHECK-NEXT: GIR_CopySubReg, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, /*SubRegIdx*/1, // src
+// CHECK-NEXT: GIR_CopySubReg, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, /*SubRegIdx*/GIMT_Encode2(1), // src
 // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0,
-// CHECK-NEXT: GIR_ConstrainOperandRC, /*InsnID*/0, /*Op*/0, Test::SRegsRegClassID,
-// CHECK-NEXT: GIR_ConstrainOperandRC, /*InsnID*/0, /*Op*/1, Test::DRegsRegClassID,
+// CHECK-NEXT: GIR_ConstrainOperandRC, /*InsnID*/0, /*Op*/0, GIMT_Encode2(Test::SRegsRegClassID),
+// CHECK-NEXT: GIR_ConstrainOperandRC, /*InsnID*/0, /*Op*/1, GIMT_Encode2(Test::DRegsRegClassID),
 
 
 // Test that we can import SUBREG_TO_REG
@@ -231,15 +231,15 @@ def : Pat<(i32 (zext SOP:$src)),
           (SUBREG_TO_REG (i64 0), (SUBSOME_INSN SOP:$src), sub0)>;
 // CHECK-LABEL:  (zext:{ *:[i32] } SOP:{ *:[i16] }:$src)  =>  (SUBREG_TO_REG:{ *:[i32] } 0:{ *:[i64] }, (SUBSOME_INSN:{ *:[i16] } SOP:{ *:[i16] }:$src), sub0:{ *:[i32] })
 // CHECK-NEXT:        GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s16,
-// CHECK-NEXT:        GIR_BuildMI, /*InsnID*/1, /*Opcode*/MyTarget::SUBSOME_INSN,
-// CHECK-NEXT:        GIR_AddTempRegister, /*InsnID*/1, /*TempRegID*/0, /*TempRegFlags*/RegState::Define,
+// CHECK-NEXT:        GIR_BuildMI, /*InsnID*/1, /*Opcode*/GIMT_Encode2(MyTarget::SUBSOME_INSN),
+// CHECK-NEXT:        GIR_AddTempRegister, /*InsnID*/1, /*TempRegID*/0, /*TempRegFlags*/GIMT_Encode2(RegState::Define),
 // CHECK-NEXT:        GIR_Copy, /*NewInsnID*/1, /*OldInsnID*/0, /*OpIdx*/1, // src
 // CHECK-NEXT:        GIR_ConstrainSelectedInstOperands, /*InsnID*/1,
-// CHECK-NEXT:        GIR_BuildMI, /*InsnID*/0, /*Opcode*/TargetOpcode::SUBREG_TO_REG,
+// CHECK-NEXT:        GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(TargetOpcode::SUBREG_TO_REG),
 // CHECK-NEXT:        GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst]
-// CHECK-NEXT:        GIR_AddImm, /*InsnID*/0, /*Imm*/0,
-// CHECK-NEXT:        GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/0,
-// CHECK-NEXT:        GIR_AddImm, /*InsnID*/0, /*Imm*/1,
+// CHECK-NEXT:        GIR_AddImm8, /*InsnID*/0, /*Imm*/0,
+// CHECK-NEXT:        GIR_AddSimpleTempRegister, /*InsnID*/0, /*TempRegID*/0,
+// CHECK-NEXT:        GIR_AddImm8, /*InsnID*/0, /*Imm*/1,
 // CHECK-NEXT:        GIR_EraseFromParent, /*InsnID*/0,
-// CHECK-NEXT:        GIR_ConstrainOperandRC, /*InsnID*/0, /*Op*/0, Test::DRegsRegClassID,
-// CHECK-NEXT:        GIR_ConstrainOperandRC, /*InsnID*/0, /*Op*/2, Test::SRegsRegClassID,
+// CHECK-NEXT:        GIR_ConstrainOperandRC, /*InsnID*/0, /*Op*/0, GIMT_Encode2(Test::DRegsRegClassID),
+// CHECK-NEXT:        GIR_ConstrainOperandRC, /*InsnID*/0, /*Op*/2, GIMT_Encode2(Test::SRegsRegClassID),
diff --git a/llvm/test/TableGen/GlobalISelEmitterVariadic.td b/llvm/test/TableGen/GlobalISelEmitterVariadic.td
index 7d7a2ce37eeffa9ff37cdafc4d3a89f4321cbbb5..ba6a93bd4837f23b68b1362f3105c8f6d1ef065e 100644
--- a/llvm/test/TableGen/GlobalISelEmitterVariadic.td
+++ b/llvm/test/TableGen/GlobalISelEmitterVariadic.td
@@ -22,30 +22,30 @@ def : Pat<(build_vector GPR32:$src1),
 def : Pat<(build_vector GPR32:$src1, GPR32:$src2),
           (TWO GPR32:$src1, GPR32:$src2)>;
 
-// CHECK:       GIM_Try, /*On fail goto*//*Label 0*/ [[NEXT_OPCODE_LABEL:[0-9]+]],
-// CHECK-NEXT:    GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_BUILD_VECTOR,
-// CHECK-NEXT:    GIM_Try, /*On fail goto*//*Label 1*/ [[NEXT_NUM_OPERANDS_LABEL_1:[0-9]+]], // Rule ID 0 //
+// CHECK:       GIM_Try, /*On fail goto*//*Label 0*/ GIMT_Encode4([[NEXT_OPCODE_LABEL:[0-9]+]]),
+// CHECK-NEXT:    GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_BUILD_VECTOR),
+// CHECK-NEXT:    GIM_Try, /*On fail goto*//*Label 1*/ GIMT_Encode4([[NEXT_NUM_OPERANDS_LABEL_1:[0-9]+]]), // Rule ID 0 //
 // CHECK-NEXT:      GIM_CheckNumOperands, /*MI*/0, /*Expected*/2,
 // CHECK-NEXT:      GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
 // CHECK-NEXT:      GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32,
-// CHECK-NEXT:      GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID,
-// CHECK-NEXT:      GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/MyTarget::GPR32RegClassID,
+// CHECK-NEXT:      GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
+// CHECK-NEXT:      GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // CHECK-NEXT:      // (build_vector:{ *:[i32] } GPR32:{ *:[i32] }:$src1)  =>  (ONE:{ *:[i32] } GPR32:{ *:[i32] }:$src1)
-// CHECK-NEXT:      GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/MyTarget::ONE,
+// CHECK-NEXT:      GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::ONE),
 // CHECK-NEXT:      GIR_ConstrainSelectedInstOperands, /*InsnID*/0,
 // CHECK-NEXT:      // GIR_Coverage, 0,
 // CHECK-NEXT:      GIR_Done,
 // CHECK-NEXT:    // Label 1: @[[NEXT_NUM_OPERANDS_LABEL_1]]
-// CHECK-NEXT:    GIM_Try, /*On fail goto*//*Label 2*/ [[NEXT_NUM_OPERANDS_LABEL_2:[0-9]+]], // Rule ID 1 //
+// CHECK-NEXT:    GIM_Try, /*On fail goto*//*Label 2*/ GIMT_Encode4([[NEXT_NUM_OPERANDS_LABEL_2:[0-9]+]]), // Rule ID 1 //
 // CHECK-NEXT:      GIM_CheckNumOperands, /*MI*/0, /*Expected*/3,
 // CHECK-NEXT:      GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
 // CHECK-NEXT:      GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32,
 // CHECK-NEXT:      GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32,
-// CHECK-NEXT:      GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID,
-// CHECK-NEXT:      GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/MyTarget::GPR32RegClassID,
-// CHECK-NEXT:      GIM_CheckRegBankForClass, /*MI*/0, /*Op*/2, /*RC*/MyTarget::GPR32RegClassID,
+// CHECK-NEXT:      GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
+// CHECK-NEXT:      GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
+// CHECK-NEXT:      GIM_CheckRegBankForClass, /*MI*/0, /*Op*/2, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // CHECK-NEXT:      // (build_vector:{ *:[i32] } GPR32:{ *:[i32] }:$src1, GPR32:{ *:[i32] }:$src2)  =>  (TWO:{ *:[i32] } GPR32:{ *:[i32] }:$src1, GPR32:{ *:[i32] }:$src2)
-// CHECK-NEXT:      GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/MyTarget::TWO,
+// CHECK-NEXT:      GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::TWO),
 // CHECK-NEXT:      GIR_ConstrainSelectedInstOperands, /*InsnID*/0,
 // CHECK-NEXT:      // GIR_Coverage, 1,
 // CHECK-NEXT:      GIR_Done,
diff --git a/llvm/test/TableGen/HasNoUse.td b/llvm/test/TableGen/HasNoUse.td
index e817be0c1ddd9b7129e3d90e7aa1c54b09d51ede..0661c2beb4d5d32e9a666a9bf497338b183daf71 100644
--- a/llvm/test/TableGen/HasNoUse.td
+++ b/llvm/test/TableGen/HasNoUse.td
@@ -16,18 +16,18 @@ def NO_RET_ATOMIC_ADD : I<(outs), (ins GPR32Op:$src0, GPR32Op:$src1), []>;
 // SDAG-NEXT: if (!SDValue(N, 0).use_empty()) return false;
 // SDAG-NEXT: return true;
 
-// GISEL: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_ATOMICRMW_ADD,
+// GISEL: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_ATOMICRMW_ADD),
 // GISEL-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
 // GISEL-NEXT: GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32,
-// GISEL-NEXT: GIM_CheckMemorySizeEqualTo, /*MI*/0, /*MMO*/0, /*Size*/4,
+// GISEL-NEXT: GIM_CheckMemorySizeEqualTo, /*MI*/0, /*MMO*/0, /*Size*/GIMT_Encode4(4),
 // GISEL-NEXT: GIM_CheckHasNoUse, /*MI*/0,
 // GISEL-NEXT: // MIs[0] src0
 // GISEL-NEXT: GIM_CheckPointerToAny, /*MI*/0, /*Op*/1, /*SizeInBits*/0,
 // GISEL-NEXT: // (atomic_load_add:{ *:[i32] } iPTR:{ *:[iPTR] }:$src0, i32:{ *:[i32] }:$src1)<<P:Predicate_atomic_load_add_no_ret_32>>  =>  (NO_RET_ATOMIC_ADD GPR32:{ *:[i32] }:$src0, GPR32:{ *:[i32] }:$src1)
-// GISEL-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::NO_RET_ATOMIC_ADD,
+// GISEL-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::NO_RET_ATOMIC_ADD),
 // GISEL-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // src0
 // GISEL-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/2, // src1
-// GISEL-NEXT: GIR_MergeMemOperands, /*InsnID*/0, /*MergeInsnID's*/0, GIU_MergeMemOperands_EndOfList,
+// GISEL-NEXT: GIR_MergeMemOperands, /*InsnID*/0, /*NumInsns*/1, /*MergeInsnID's*/0,
 // GISEL-NEXT: GIR_EraseFromParent, /*InsnID*/0,
 // GISEL-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0,
 let HasNoUse = true in
diff --git a/llvm/test/TableGen/address-space-patfrags.td b/llvm/test/TableGen/address-space-patfrags.td
index 8e92719e6520321e95f1be5b4e03d4dcbe49b25b..27b174b4633cd8f753038f070380da0312a5bd1f 100644
--- a/llvm/test/TableGen/address-space-patfrags.td
+++ b/llvm/test/TableGen/address-space-patfrags.td
@@ -57,12 +57,12 @@ def inst_d : Instruction {
 // SDAG-NEXT: return true;
 
 
-// GISEL: GIM_Try, /*On fail goto*//*Label 0*/ {{[0-9]+}}, // Rule ID 0 //
+// GISEL: GIM_Try, /*On fail goto*//*Label 0*/ GIMT_Encode4({{[0-9]+}}), // Rule ID 0 //
 // GISEL-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/2,
-// GISEL-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_LOAD,
+// GISEL-NEXT: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_LOAD),
 // GISEL-NEXT: GIM_CheckMemorySizeEqualToLLT, /*MI*/0, /*MMO*/0, /*OpIdx*/0,
-// GISEL-NEXT: GIM_CheckMemoryAddressSpace, /*MI*/0, /*MMO*/0, /*NumAddrSpace*/2, /*AddrSpace*/123, /*AddrSpace*/455,
-// GISEL-NEXT: GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(int64_t)AtomicOrdering::NotAtomic,
+// GISEL-NEXT: GIM_CheckMemoryAddressSpace, /*MI*/0, /*MMO*/0, /*NumAddrSpace*/2, /*AddrSpace*/123, /*AddrSpace*//* 455(*/0xC7, 0x03/*)*/,
+// GISEL-NEXT: GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(uint8_t)AtomicOrdering::NotAtomic,
 def : Pat <
   (pat_frag_b GPR32:$src),
   (inst_b GPR32:$src)
@@ -77,12 +77,12 @@ def : Pat <
 // SDAG-NEXT: return false;
 // SDAG-NEXT: return true;
 
-// GISEL: GIM_Try, /*On fail goto*//*Label 1*/ {{[0-9]+}}, // Rule ID 1 //
+// GISEL: GIM_Try, /*On fail goto*//*Label 1*/ GIMT_Encode4({{[0-9]+}}), // Rule ID 1 //
 // GISEL-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/2,
-// GISEL-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_LOAD,
+// GISEL-NEXT: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_LOAD),
 // GISEL-NEXT: GIM_CheckMemorySizeEqualToLLT, /*MI*/0, /*MMO*/0, /*OpIdx*/0,
 // GISEL-NEXT: GIM_CheckMemoryAlignment, /*MI*/0, /*MMO*/0, /*MinAlign*/2,
-// GISEL-NEXT: GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(int64_t)AtomicOrdering::NotAtomic,
+// GISEL-NEXT: GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(uint8_t)AtomicOrdering::NotAtomic,
 def : Pat <
   (pat_frag_a GPR32:$src),
   (inst_a GPR32:$src)
@@ -96,11 +96,11 @@ def truncstorei16_addrspace : PatFrag<(ops node:$val, node:$ptr),
 }
 
 // Test truncstore without a specific MemoryVT
-// GISEL: GIM_Try, /*On fail goto*//*Label 2*/ {{[0-9]+}}, // Rule ID 2 //
+// GISEL: GIM_Try, /*On fail goto*//*Label 2*/ GIMT_Encode4({{[0-9]+}}), // Rule ID 2 //
 // GISEL-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/2,
-// GISEL-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_STORE,
+// GISEL-NEXT: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_STORE),
 // GISEL-NEXT: GIM_CheckMemorySizeLessThanLLT, /*MI*/0, /*MMO*/0, /*OpIdx*/0,
-// GISEL-NEXT: GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(int64_t)AtomicOrdering::NotAtomic,
+// GISEL-NEXT: GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(uint8_t)AtomicOrdering::NotAtomic,
 // GISEL-NEXT: // MIs[0] src0
 // GISEL-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
 def : Pat <
@@ -109,9 +109,9 @@ def : Pat <
 >;
 
 // Test non-truncstore has a size equal to LLT check.
-// GISEL: GIM_Try, /*On fail goto*//*Label 3*/ {{[0-9]+}}, // Rule ID 3 //
+// GISEL: GIM_Try, /*On fail goto*//*Label 3*/ GIMT_Encode4({{[0-9]+}}), // Rule ID 3 //
 // GISEL-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/2,
-// GISEL-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_STORE,
+// GISEL-NEXT: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_STORE),
 // GISEL-NEXT: GIM_CheckMemorySizeEqualToLLT, /*MI*/0, /*MMO*/0, /*OpIdx*/0,
 def : Pat <
   (store GPR32:$src0, GPR32:$src1),
@@ -119,12 +119,12 @@ def : Pat <
 >;
 
 // Test truncstore with specific MemoryVT
-// GISEL: GIM_Try, /*On fail goto*//*Label 4*/ {{[0-9]+}}, // Rule ID 4 //
+// GISEL: GIM_Try, /*On fail goto*//*Label 4*/ GIMT_Encode4({{[0-9]+}}), // Rule ID 4 //
 // GISEL-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/2,
-// GISEL-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_STORE,
+// GISEL-NEXT: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_STORE),
 // GISEL-NEXT: GIM_CheckMemorySizeLessThanLLT, /*MI*/0, /*MMO*/0, /*OpIdx*/0,
-// GISEL-NEXT: GIM_CheckMemorySizeEqualTo, /*MI*/0, /*MMO*/0, /*Size*/2,
-// GISEL-NEXT: GIM_CheckMemoryAddressSpace, /*MI*/0, /*MMO*/0, /*NumAddrSpace*/2, /*AddrSpace*/123, /*AddrSpace*/455,
+// GISEL-NEXT: GIM_CheckMemorySizeEqualTo, /*MI*/0, /*MMO*/0, /*Size*/GIMT_Encode4(2),
+// GISEL-NEXT: GIM_CheckMemoryAddressSpace, /*MI*/0, /*MMO*/0, /*NumAddrSpace*/2, /*AddrSpace*/123, /*AddrSpace*//* 455(*/0xC7, 0x03/*)*/,
 def : Pat <
   (truncstorei16_addrspace GPR32:$src0, GPR32:$src1),
   (inst_c GPR32:$src0, GPR32:$src1)
diff --git a/llvm/test/TableGen/gisel-physreg-input.td b/llvm/test/TableGen/gisel-physreg-input.td
index 3dbd57bc45b7c0a9bffa19471d2b3748487fcdcb..cd099c760a02cca574f61b79e96c0d14378fec10 100644
--- a/llvm/test/TableGen/gisel-physreg-input.td
+++ b/llvm/test/TableGen/gisel-physreg-input.td
@@ -26,21 +26,21 @@ class I<dag OOps, dag IOps, list<dag> Pat>
 
 // GISEL: GIM_Try,
 // GISEL-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3,
-// GISEL-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_ADD,
+// GISEL-NEXT: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_ADD),
 // GISEL-NEXT: // MIs[0] DstI[dst]
 // GISEL-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
-// GISEL-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID,
+// GISEL-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // GISEL-NEXT: // MIs[0] src0
 // GISEL-NEXT: GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32,
-// GISEL-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/MyTarget::GPR32RegClassID,
+// GISEL-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // GISEL-NEXT: // MIs[0] Operand 2
 // GISEL-NEXT: GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32,
-// GISEL-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/2, /*RC*/MyTarget::Special32RegClassID,
+// GISEL-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/2, /*RC*/GIMT_Encode2(MyTarget::Special32RegClassID),
 // GISEL-NEXT: // (add:{ *:[i32] } GPR32:{ *:[i32] }:$src0, SPECIAL:{ *:[i32] })  =>  (ADD_PHYS:{ *:[i32] } GPR32:{ *:[i32] }:$src0)
-// GISEL-NEXT: GIR_BuildMI, /*InsnID*/1, /*Opcode*/TargetOpcode::COPY,
-// GISEL-NEXT: GIR_AddRegister, /*InsnID*/1, MyTarget::SPECIAL, /*AddRegisterRegFlags*/RegState::Define,
+// GISEL-NEXT: GIR_BuildMI, /*InsnID*/1, /*Opcode*/GIMT_Encode2(TargetOpcode::COPY),
+// GISEL-NEXT: GIR_AddRegister, /*InsnID*/1, GIMT_Encode2(MyTarget::SPECIAL), /*AddRegisterRegFlags*/GIMT_Encode2(RegState::Define),
 // GISEL-NEXT: GIR_Copy, /*NewInsnID*/1, /*OldInsnID*/0, /*OpIdx*/2, // SPECIAL
-// GISEL-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::ADD_PHYS,
+// GISEL-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::ADD_PHYS),
 // GISEL-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst]
 // GISEL-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // src0
 // GISEL-NEXT: GIR_EraseFromParent, /*InsnID*/0,
@@ -54,21 +54,21 @@ def ADD_PHYS : I<(outs GPR32:$dst), (ins GPR32:$src0),
 
 // GISEL: GIM_Try,
 // GISEL-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3,
-// GISEL-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_MUL,
+// GISEL-NEXT: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_MUL),
 // GISEL-NEXT: // MIs[0] DstI[dst]
 // GISEL-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32,
-// GISEL-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID,
+// GISEL-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // GISEL-NEXT: // MIs[0] SPECIAL
 // GISEL-NEXT: GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32,
-// GISEL-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/MyTarget::GPR32RegClassID,
+// GISEL-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
 // GISEL-NEXT: // MIs[0] Operand 2
 // GISEL-NEXT: GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32,
-// GISEL-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/2, /*RC*/MyTarget::Special32RegClassID,
+// GISEL-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/2, /*RC*/GIMT_Encode2(MyTarget::Special32RegClassID),
 // GISEL-NEXT: // (mul:{ *:[i32] } GPR32:{ *:[i32] }:$SPECIAL, SPECIAL:{ *:[i32] })  =>  (MUL_PHYS:{ *:[i32] } GPR32:{ *:[i32] }:$SPECIAL)
-// GISEL-NEXT: GIR_BuildMI, /*InsnID*/1, /*Opcode*/TargetOpcode::COPY,
-// GISEL-NEXT: GIR_AddRegister, /*InsnID*/1, MyTarget::SPECIAL, /*AddRegisterRegFlags*/RegState::Define,
+// GISEL-NEXT: GIR_BuildMI, /*InsnID*/1, /*Opcode*/GIMT_Encode2(TargetOpcode::COPY),
+// GISEL-NEXT: GIR_AddRegister, /*InsnID*/1, GIMT_Encode2(MyTarget::SPECIAL), /*AddRegisterRegFlags*/GIMT_Encode2(RegState::Define),
 // GISEL-NEXT: GIR_Copy, /*NewInsnID*/1, /*OldInsnID*/0, /*OpIdx*/2, // SPECIAL
-// GISEL-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::MUL_PHYS,
+// GISEL-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::MUL_PHYS),
 // GISEL-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst]
 // GISEL-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // SPECIAL
 // GISEL-NEXT: GIR_EraseFromParent, /*InsnID*/0,
diff --git a/llvm/test/TableGen/immarg-predicated.td b/llvm/test/TableGen/immarg-predicated.td
index 87326c652356b8d84fa38af7df69a7512a1cdd3e..320018010cc7fec571b1a4b035a0fe700cb8e6c6 100644
--- a/llvm/test/TableGen/immarg-predicated.td
+++ b/llvm/test/TableGen/immarg-predicated.td
@@ -7,14 +7,14 @@ let TargetPrefix = "mytarget" in {
 def int_mytarget_sleep0 : Intrinsic<[], [llvm_i32_ty], [ImmArg<ArgIndex<0>>]>;
 }
 
-// GISEL: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS,
+// GISEL: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS),
 // GISEL-NEXT: // MIs[0] Operand 0
-// GISEL-NEXT: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/0, Intrinsic::mytarget_sleep0,
+// GISEL-NEXT: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/0, GIMT_Encode2(Intrinsic::mytarget_sleep0),
 // GISEL-NEXT: // MIs[0] src
 // GISEL-NEXT: GIM_CheckIsImm, /*MI*/0, /*Op*/1,
-// GISEL-NEXT: GIM_CheckImmOperandPredicate, /*MI*/0, /*MO*/1, /*Predicate*/GICXXPred_I64_Predicate_tuimm9,
+// GISEL-NEXT: GIM_CheckImmOperandPredicate, /*MI*/0, /*MO*/1, /*Predicate*/GIMT_Encode2(GICXXPred_I64_Predicate_tuimm9),
 // GISEL-NEXT: // (intrinsic_void {{[0-9]+}}:{ *:[iPTR] }, (timm:{ *:[i32] })<<P:Predicate_tuimm9>>:$src) => (SLEEP0 (timm:{ *:[i32] }):$src)
-// GISEL-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::SLEEP0,
+// GISEL-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::SLEEP0),
 // GISEL-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // src
 def tuimm9 : TImmLeaf<i32, [{ return isUInt<9>(Imm); }]>;
 def SLEEP0 : I<(outs), (ins i32imm:$src),
diff --git a/llvm/test/TableGen/immarg.td b/llvm/test/TableGen/immarg.td
index c6f03cad137fa952a11ed5e0c39e5b31645befeb..80849d512bee3013ee6b0096af9f76c777adb57a 100644
--- a/llvm/test/TableGen/immarg.td
+++ b/llvm/test/TableGen/immarg.td
@@ -8,13 +8,13 @@ def int_mytarget_sleep0 : Intrinsic<[], [llvm_i32_ty], [ImmArg<ArgIndex<0>>]>;
 def int_mytarget_sleep1 : Intrinsic<[], [llvm_i32_ty], [ImmArg<ArgIndex<0>>]>;
 }
 
-// GISEL: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS,
+// GISEL: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS),
 // GISEL-NEXT: // MIs[0] Operand 0
-// GISEL-NEXT: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/0, Intrinsic::mytarget_sleep0,
+// GISEL-NEXT: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/0, GIMT_Encode2(Intrinsic::mytarget_sleep0),
 // GISEL-NEXT: // MIs[0] src
 // GISEL-NEXT: GIM_CheckIsImm, /*MI*/0, /*Op*/1,
 // GISEL-NEXT: // (intrinsic_void {{[0-9]+}}:{ *:[iPTR] }, (timm:{ *:[i32] }):$src)  =>  (SLEEP0 (timm:{ *:[i32] }):$src)
-// GISEL-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::SLEEP0,
+// GISEL-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/GIMT_Encode2(MyTarget::SLEEP0),
 // GISEL-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // src
 def SLEEP0 : I<(outs), (ins i32imm:$src),
   [(int_mytarget_sleep0 timm:$src)]
diff --git a/llvm/test/TableGen/predicate-patfags.td b/llvm/test/TableGen/predicate-patfags.td
index 17cc74206b716b5e76e27e79cfbe675f86d1511a..0912b05127ef8150bf099e2862b3591f61648679 100644
--- a/llvm/test/TableGen/predicate-patfags.td
+++ b/llvm/test/TableGen/predicate-patfags.td
@@ -44,19 +44,19 @@ def TGTmul24_oneuse : PatFrag<
 // SDAG: OPC_CheckOpcode, TARGET_VAL(TargetISD::MUL24),
 // SDAG: OPC_CheckPredicate, 0, // Predicate_TGTmul24_oneuse
 
-// GISEL: GIM_CheckOpcode, /*MI*/1, TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS,
-// GISEL: GIM_CheckIntrinsicID, /*MI*/1, /*Op*/1, Intrinsic::tgt_mul24,
-// GISEL: GIM_CheckCxxInsnPredicate, /*MI*/1, /*FnId*/GICXXPred_MI_Predicate_TGTmul24_oneuse,
+// GISEL: GIM_CheckOpcode, /*MI*/1, GIMT_Encode2(TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS),
+// GISEL: GIM_CheckIntrinsicID, /*MI*/1, /*Op*/1, GIMT_Encode2(Intrinsic::tgt_mul24),
+// GISEL: GIM_CheckCxxInsnPredicate, /*MI*/1, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_TGTmul24_oneuse),
 
-// GISEL: GIM_CheckOpcode, /*MI*/1, TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS,
-// GISEL: GIM_CheckIntrinsicID, /*MI*/1, /*Op*/1, Intrinsic::tgt_mul24,
-// GISEL: GIM_CheckCxxInsnPredicate, /*MI*/1, /*FnId*/GICXXPred_MI_Predicate_TGTmul24_oneuse,
+// GISEL: GIM_CheckOpcode, /*MI*/1, GIMT_Encode2(TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS),
+// GISEL: GIM_CheckIntrinsicID, /*MI*/1, /*Op*/1, GIMT_Encode2(Intrinsic::tgt_mul24),
+// GISEL: GIM_CheckCxxInsnPredicate, /*MI*/1, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_TGTmul24_oneuse),
 
-// GISEL: GIM_CheckOpcode, /*MI*/1, MyTarget::G_TGT_MUL24,
-// GISEL: GIM_CheckCxxInsnPredicate, /*MI*/1, /*FnId*/GICXXPred_MI_Predicate_TGTmul24_oneuse,
+// GISEL: GIM_CheckOpcode, /*MI*/1, GIMT_Encode2(MyTarget::G_TGT_MUL24),
+// GISEL: GIM_CheckCxxInsnPredicate, /*MI*/1, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_TGTmul24_oneuse),
 
-// GISEL: GIM_CheckOpcode, /*MI*/1, MyTarget::G_TGT_MUL24,
-// GISEL: GIM_CheckCxxInsnPredicate, /*MI*/1, /*FnId*/GICXXPred_MI_Predicate_TGTmul24_oneuse,
+// GISEL: GIM_CheckOpcode, /*MI*/1, GIMT_Encode2(MyTarget::G_TGT_MUL24),
+// GISEL: GIM_CheckCxxInsnPredicate, /*MI*/1, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_TGTmul24_oneuse),
 def inst_mad24 : I<
   (outs GPR32:$dst),
   (ins GPR32:$src0, GPR32:$src1, GPR32:$src2),
diff --git a/llvm/test/TableGen/x86-fold-tables.inc b/llvm/test/TableGen/x86-fold-tables.inc
index b2609f01e86a213e18ad6c6e760374265e37e3ce..67b5c6cc288911d25cfe9ab9930d97c0e93e503d 100644
--- a/llvm/test/TableGen/x86-fold-tables.inc
+++ b/llvm/test/TableGen/x86-fold-tables.inc
@@ -4285,24 +4285,24 @@ static const X86FoldTableEntry Table3[] = {
   {X86::VPERMBZrrkz, X86::VPERMBZrmkz, 0},
   {X86::VPERMDZ256rrkz, X86::VPERMDZ256rmkz, 0},
   {X86::VPERMDZrrkz, X86::VPERMDZrmkz, 0},
-  {X86::VPERMI2B128rr, X86::VPERMI2B128rm, 0},
-  {X86::VPERMI2B256rr, X86::VPERMI2B256rm, 0},
-  {X86::VPERMI2Brr, X86::VPERMI2Brm, 0},
-  {X86::VPERMI2D128rr, X86::VPERMI2D128rm, 0},
-  {X86::VPERMI2D256rr, X86::VPERMI2D256rm, 0},
-  {X86::VPERMI2Drr, X86::VPERMI2Drm, 0},
-  {X86::VPERMI2PD128rr, X86::VPERMI2PD128rm, 0},
-  {X86::VPERMI2PD256rr, X86::VPERMI2PD256rm, 0},
-  {X86::VPERMI2PDrr, X86::VPERMI2PDrm, 0},
-  {X86::VPERMI2PS128rr, X86::VPERMI2PS128rm, 0},
-  {X86::VPERMI2PS256rr, X86::VPERMI2PS256rm, 0},
-  {X86::VPERMI2PSrr, X86::VPERMI2PSrm, 0},
-  {X86::VPERMI2Q128rr, X86::VPERMI2Q128rm, 0},
-  {X86::VPERMI2Q256rr, X86::VPERMI2Q256rm, 0},
-  {X86::VPERMI2Qrr, X86::VPERMI2Qrm, 0},
-  {X86::VPERMI2W128rr, X86::VPERMI2W128rm, 0},
-  {X86::VPERMI2W256rr, X86::VPERMI2W256rm, 0},
-  {X86::VPERMI2Wrr, X86::VPERMI2Wrm, 0},
+  {X86::VPERMI2BZ128rr, X86::VPERMI2BZ128rm, 0},
+  {X86::VPERMI2BZ256rr, X86::VPERMI2BZ256rm, 0},
+  {X86::VPERMI2BZrr, X86::VPERMI2BZrm, 0},
+  {X86::VPERMI2DZ128rr, X86::VPERMI2DZ128rm, 0},
+  {X86::VPERMI2DZ256rr, X86::VPERMI2DZ256rm, 0},
+  {X86::VPERMI2DZrr, X86::VPERMI2DZrm, 0},
+  {X86::VPERMI2PDZ128rr, X86::VPERMI2PDZ128rm, 0},
+  {X86::VPERMI2PDZ256rr, X86::VPERMI2PDZ256rm, 0},
+  {X86::VPERMI2PDZrr, X86::VPERMI2PDZrm, 0},
+  {X86::VPERMI2PSZ128rr, X86::VPERMI2PSZ128rm, 0},
+  {X86::VPERMI2PSZ256rr, X86::VPERMI2PSZ256rm, 0},
+  {X86::VPERMI2PSZrr, X86::VPERMI2PSZrm, 0},
+  {X86::VPERMI2QZ128rr, X86::VPERMI2QZ128rm, 0},
+  {X86::VPERMI2QZ256rr, X86::VPERMI2QZ256rm, 0},
+  {X86::VPERMI2QZrr, X86::VPERMI2QZrm, 0},
+  {X86::VPERMI2WZ128rr, X86::VPERMI2WZ128rm, 0},
+  {X86::VPERMI2WZ256rr, X86::VPERMI2WZ256rm, 0},
+  {X86::VPERMI2WZrr, X86::VPERMI2WZrm, 0},
   {X86::VPERMIL2PDYrr, X86::VPERMIL2PDYrm, 0},
   {X86::VPERMIL2PDrr, X86::VPERMIL2PDrm, 0},
   {X86::VPERMIL2PSYrr, X86::VPERMIL2PSYrm, 0},
@@ -4329,24 +4329,24 @@ static const X86FoldTableEntry Table3[] = {
   {X86::VPERMQZ256rrkz, X86::VPERMQZ256rmkz, 0},
   {X86::VPERMQZrik, X86::VPERMQZmik, 0},
   {X86::VPERMQZrrkz, X86::VPERMQZrmkz, 0},
-  {X86::VPERMT2B128rr, X86::VPERMT2B128rm, 0},
-  {X86::VPERMT2B256rr, X86::VPERMT2B256rm, 0},
-  {X86::VPERMT2Brr, X86::VPERMT2Brm, 0},
-  {X86::VPERMT2D128rr, X86::VPERMT2D128rm, 0},
-  {X86::VPERMT2D256rr, X86::VPERMT2D256rm, 0},
-  {X86::VPERMT2Drr, X86::VPERMT2Drm, 0},
-  {X86::VPERMT2PD128rr, X86::VPERMT2PD128rm, 0},
-  {X86::VPERMT2PD256rr, X86::VPERMT2PD256rm, 0},
-  {X86::VPERMT2PDrr, X86::VPERMT2PDrm, 0},
-  {X86::VPERMT2PS128rr, X86::VPERMT2PS128rm, 0},
-  {X86::VPERMT2PS256rr, X86::VPERMT2PS256rm, 0},
-  {X86::VPERMT2PSrr, X86::VPERMT2PSrm, 0},
-  {X86::VPERMT2Q128rr, X86::VPERMT2Q128rm, 0},
-  {X86::VPERMT2Q256rr, X86::VPERMT2Q256rm, 0},
-  {X86::VPERMT2Qrr, X86::VPERMT2Qrm, 0},
-  {X86::VPERMT2W128rr, X86::VPERMT2W128rm, 0},
-  {X86::VPERMT2W256rr, X86::VPERMT2W256rm, 0},
-  {X86::VPERMT2Wrr, X86::VPERMT2Wrm, 0},
+  {X86::VPERMT2BZ128rr, X86::VPERMT2BZ128rm, 0},
+  {X86::VPERMT2BZ256rr, X86::VPERMT2BZ256rm, 0},
+  {X86::VPERMT2BZrr, X86::VPERMT2BZrm, 0},
+  {X86::VPERMT2DZ128rr, X86::VPERMT2DZ128rm, 0},
+  {X86::VPERMT2DZ256rr, X86::VPERMT2DZ256rm, 0},
+  {X86::VPERMT2DZrr, X86::VPERMT2DZrm, 0},
+  {X86::VPERMT2PDZ128rr, X86::VPERMT2PDZ128rm, 0},
+  {X86::VPERMT2PDZ256rr, X86::VPERMT2PDZ256rm, 0},
+  {X86::VPERMT2PDZrr, X86::VPERMT2PDZrm, 0},
+  {X86::VPERMT2PSZ128rr, X86::VPERMT2PSZ128rm, 0},
+  {X86::VPERMT2PSZ256rr, X86::VPERMT2PSZ256rm, 0},
+  {X86::VPERMT2PSZrr, X86::VPERMT2PSZrm, 0},
+  {X86::VPERMT2QZ128rr, X86::VPERMT2QZ128rm, 0},
+  {X86::VPERMT2QZ256rr, X86::VPERMT2QZ256rm, 0},
+  {X86::VPERMT2QZrr, X86::VPERMT2QZrm, 0},
+  {X86::VPERMT2WZ128rr, X86::VPERMT2WZ128rm, 0},
+  {X86::VPERMT2WZ256rr, X86::VPERMT2WZ256rm, 0},
+  {X86::VPERMT2WZrr, X86::VPERMT2WZrm, 0},
   {X86::VPERMWZ128rrkz, X86::VPERMWZ128rmkz, 0},
   {X86::VPERMWZ256rrkz, X86::VPERMWZ256rmkz, 0},
   {X86::VPERMWZrrkz, X86::VPERMWZrmkz, 0},
@@ -5561,42 +5561,42 @@ static const X86FoldTableEntry Table4[] = {
   {X86::VPERMBZrrk, X86::VPERMBZrmk, 0},
   {X86::VPERMDZ256rrk, X86::VPERMDZ256rmk, 0},
   {X86::VPERMDZrrk, X86::VPERMDZrmk, 0},
-  {X86::VPERMI2B128rrk, X86::VPERMI2B128rmk, 0},
-  {X86::VPERMI2B128rrkz, X86::VPERMI2B128rmkz, 0},
-  {X86::VPERMI2B256rrk, X86::VPERMI2B256rmk, 0},
-  {X86::VPERMI2B256rrkz, X86::VPERMI2B256rmkz, 0},
-  {X86::VPERMI2Brrk, X86::VPERMI2Brmk, 0},
-  {X86::VPERMI2Brrkz, X86::VPERMI2Brmkz, 0},
-  {X86::VPERMI2D128rrk, X86::VPERMI2D128rmk, 0},
-  {X86::VPERMI2D128rrkz, X86::VPERMI2D128rmkz, 0},
-  {X86::VPERMI2D256rrk, X86::VPERMI2D256rmk, 0},
-  {X86::VPERMI2D256rrkz, X86::VPERMI2D256rmkz, 0},
-  {X86::VPERMI2Drrk, X86::VPERMI2Drmk, 0},
-  {X86::VPERMI2Drrkz, X86::VPERMI2Drmkz, 0},
-  {X86::VPERMI2PD128rrk, X86::VPERMI2PD128rmk, 0},
-  {X86::VPERMI2PD128rrkz, X86::VPERMI2PD128rmkz, 0},
-  {X86::VPERMI2PD256rrk, X86::VPERMI2PD256rmk, 0},
-  {X86::VPERMI2PD256rrkz, X86::VPERMI2PD256rmkz, 0},
-  {X86::VPERMI2PDrrk, X86::VPERMI2PDrmk, 0},
-  {X86::VPERMI2PDrrkz, X86::VPERMI2PDrmkz, 0},
-  {X86::VPERMI2PS128rrk, X86::VPERMI2PS128rmk, 0},
-  {X86::VPERMI2PS128rrkz, X86::VPERMI2PS128rmkz, 0},
-  {X86::VPERMI2PS256rrk, X86::VPERMI2PS256rmk, 0},
-  {X86::VPERMI2PS256rrkz, X86::VPERMI2PS256rmkz, 0},
-  {X86::VPERMI2PSrrk, X86::VPERMI2PSrmk, 0},
-  {X86::VPERMI2PSrrkz, X86::VPERMI2PSrmkz, 0},
-  {X86::VPERMI2Q128rrk, X86::VPERMI2Q128rmk, 0},
-  {X86::VPERMI2Q128rrkz, X86::VPERMI2Q128rmkz, 0},
-  {X86::VPERMI2Q256rrk, X86::VPERMI2Q256rmk, 0},
-  {X86::VPERMI2Q256rrkz, X86::VPERMI2Q256rmkz, 0},
-  {X86::VPERMI2Qrrk, X86::VPERMI2Qrmk, 0},
-  {X86::VPERMI2Qrrkz, X86::VPERMI2Qrmkz, 0},
-  {X86::VPERMI2W128rrk, X86::VPERMI2W128rmk, 0},
-  {X86::VPERMI2W128rrkz, X86::VPERMI2W128rmkz, 0},
-  {X86::VPERMI2W256rrk, X86::VPERMI2W256rmk, 0},
-  {X86::VPERMI2W256rrkz, X86::VPERMI2W256rmkz, 0},
-  {X86::VPERMI2Wrrk, X86::VPERMI2Wrmk, 0},
-  {X86::VPERMI2Wrrkz, X86::VPERMI2Wrmkz, 0},
+  {X86::VPERMI2BZ128rrk, X86::VPERMI2BZ128rmk, 0},
+  {X86::VPERMI2BZ128rrkz, X86::VPERMI2BZ128rmkz, 0},
+  {X86::VPERMI2BZ256rrk, X86::VPERMI2BZ256rmk, 0},
+  {X86::VPERMI2BZ256rrkz, X86::VPERMI2BZ256rmkz, 0},
+  {X86::VPERMI2BZrrk, X86::VPERMI2BZrmk, 0},
+  {X86::VPERMI2BZrrkz, X86::VPERMI2BZrmkz, 0},
+  {X86::VPERMI2DZ128rrk, X86::VPERMI2DZ128rmk, 0},
+  {X86::VPERMI2DZ128rrkz, X86::VPERMI2DZ128rmkz, 0},
+  {X86::VPERMI2DZ256rrk, X86::VPERMI2DZ256rmk, 0},
+  {X86::VPERMI2DZ256rrkz, X86::VPERMI2DZ256rmkz, 0},
+  {X86::VPERMI2DZrrk, X86::VPERMI2DZrmk, 0},
+  {X86::VPERMI2DZrrkz, X86::VPERMI2DZrmkz, 0},
+  {X86::VPERMI2PDZ128rrk, X86::VPERMI2PDZ128rmk, 0},
+  {X86::VPERMI2PDZ128rrkz, X86::VPERMI2PDZ128rmkz, 0},
+  {X86::VPERMI2PDZ256rrk, X86::VPERMI2PDZ256rmk, 0},
+  {X86::VPERMI2PDZ256rrkz, X86::VPERMI2PDZ256rmkz, 0},
+  {X86::VPERMI2PDZrrk, X86::VPERMI2PDZrmk, 0},
+  {X86::VPERMI2PDZrrkz, X86::VPERMI2PDZrmkz, 0},
+  {X86::VPERMI2PSZ128rrk, X86::VPERMI2PSZ128rmk, 0},
+  {X86::VPERMI2PSZ128rrkz, X86::VPERMI2PSZ128rmkz, 0},
+  {X86::VPERMI2PSZ256rrk, X86::VPERMI2PSZ256rmk, 0},
+  {X86::VPERMI2PSZ256rrkz, X86::VPERMI2PSZ256rmkz, 0},
+  {X86::VPERMI2PSZrrk, X86::VPERMI2PSZrmk, 0},
+  {X86::VPERMI2PSZrrkz, X86::VPERMI2PSZrmkz, 0},
+  {X86::VPERMI2QZ128rrk, X86::VPERMI2QZ128rmk, 0},
+  {X86::VPERMI2QZ128rrkz, X86::VPERMI2QZ128rmkz, 0},
+  {X86::VPERMI2QZ256rrk, X86::VPERMI2QZ256rmk, 0},
+  {X86::VPERMI2QZ256rrkz, X86::VPERMI2QZ256rmkz, 0},
+  {X86::VPERMI2QZrrk, X86::VPERMI2QZrmk, 0},
+  {X86::VPERMI2QZrrkz, X86::VPERMI2QZrmkz, 0},
+  {X86::VPERMI2WZ128rrk, X86::VPERMI2WZ128rmk, 0},
+  {X86::VPERMI2WZ128rrkz, X86::VPERMI2WZ128rmkz, 0},
+  {X86::VPERMI2WZ256rrk, X86::VPERMI2WZ256rmk, 0},
+  {X86::VPERMI2WZ256rrkz, X86::VPERMI2WZ256rmkz, 0},
+  {X86::VPERMI2WZrrk, X86::VPERMI2WZrmk, 0},
+  {X86::VPERMI2WZrrkz, X86::VPERMI2WZrmkz, 0},
   {X86::VPERMILPDZ128rrk, X86::VPERMILPDZ128rmk, 0},
   {X86::VPERMILPDZ256rrk, X86::VPERMILPDZ256rmk, 0},
   {X86::VPERMILPDZrrk, X86::VPERMILPDZrmk, 0},
@@ -5609,42 +5609,42 @@ static const X86FoldTableEntry Table4[] = {
   {X86::VPERMPSZrrk, X86::VPERMPSZrmk, 0},
   {X86::VPERMQZ256rrk, X86::VPERMQZ256rmk, 0},
   {X86::VPERMQZrrk, X86::VPERMQZrmk, 0},
-  {X86::VPERMT2B128rrk, X86::VPERMT2B128rmk, 0},
-  {X86::VPERMT2B128rrkz, X86::VPERMT2B128rmkz, 0},
-  {X86::VPERMT2B256rrk, X86::VPERMT2B256rmk, 0},
-  {X86::VPERMT2B256rrkz, X86::VPERMT2B256rmkz, 0},
-  {X86::VPERMT2Brrk, X86::VPERMT2Brmk, 0},
-  {X86::VPERMT2Brrkz, X86::VPERMT2Brmkz, 0},
-  {X86::VPERMT2D128rrk, X86::VPERMT2D128rmk, 0},
-  {X86::VPERMT2D128rrkz, X86::VPERMT2D128rmkz, 0},
-  {X86::VPERMT2D256rrk, X86::VPERMT2D256rmk, 0},
-  {X86::VPERMT2D256rrkz, X86::VPERMT2D256rmkz, 0},
-  {X86::VPERMT2Drrk, X86::VPERMT2Drmk, 0},
-  {X86::VPERMT2Drrkz, X86::VPERMT2Drmkz, 0},
-  {X86::VPERMT2PD128rrk, X86::VPERMT2PD128rmk, 0},
-  {X86::VPERMT2PD128rrkz, X86::VPERMT2PD128rmkz, 0},
-  {X86::VPERMT2PD256rrk, X86::VPERMT2PD256rmk, 0},
-  {X86::VPERMT2PD256rrkz, X86::VPERMT2PD256rmkz, 0},
-  {X86::VPERMT2PDrrk, X86::VPERMT2PDrmk, 0},
-  {X86::VPERMT2PDrrkz, X86::VPERMT2PDrmkz, 0},
-  {X86::VPERMT2PS128rrk, X86::VPERMT2PS128rmk, 0},
-  {X86::VPERMT2PS128rrkz, X86::VPERMT2PS128rmkz, 0},
-  {X86::VPERMT2PS256rrk, X86::VPERMT2PS256rmk, 0},
-  {X86::VPERMT2PS256rrkz, X86::VPERMT2PS256rmkz, 0},
-  {X86::VPERMT2PSrrk, X86::VPERMT2PSrmk, 0},
-  {X86::VPERMT2PSrrkz, X86::VPERMT2PSrmkz, 0},
-  {X86::VPERMT2Q128rrk, X86::VPERMT2Q128rmk, 0},
-  {X86::VPERMT2Q128rrkz, X86::VPERMT2Q128rmkz, 0},
-  {X86::VPERMT2Q256rrk, X86::VPERMT2Q256rmk, 0},
-  {X86::VPERMT2Q256rrkz, X86::VPERMT2Q256rmkz, 0},
-  {X86::VPERMT2Qrrk, X86::VPERMT2Qrmk, 0},
-  {X86::VPERMT2Qrrkz, X86::VPERMT2Qrmkz, 0},
-  {X86::VPERMT2W128rrk, X86::VPERMT2W128rmk, 0},
-  {X86::VPERMT2W128rrkz, X86::VPERMT2W128rmkz, 0},
-  {X86::VPERMT2W256rrk, X86::VPERMT2W256rmk, 0},
-  {X86::VPERMT2W256rrkz, X86::VPERMT2W256rmkz, 0},
-  {X86::VPERMT2Wrrk, X86::VPERMT2Wrmk, 0},
-  {X86::VPERMT2Wrrkz, X86::VPERMT2Wrmkz, 0},
+  {X86::VPERMT2BZ128rrk, X86::VPERMT2BZ128rmk, 0},
+  {X86::VPERMT2BZ128rrkz, X86::VPERMT2BZ128rmkz, 0},
+  {X86::VPERMT2BZ256rrk, X86::VPERMT2BZ256rmk, 0},
+  {X86::VPERMT2BZ256rrkz, X86::VPERMT2BZ256rmkz, 0},
+  {X86::VPERMT2BZrrk, X86::VPERMT2BZrmk, 0},
+  {X86::VPERMT2BZrrkz, X86::VPERMT2BZrmkz, 0},
+  {X86::VPERMT2DZ128rrk, X86::VPERMT2DZ128rmk, 0},
+  {X86::VPERMT2DZ128rrkz, X86::VPERMT2DZ128rmkz, 0},
+  {X86::VPERMT2DZ256rrk, X86::VPERMT2DZ256rmk, 0},
+  {X86::VPERMT2DZ256rrkz, X86::VPERMT2DZ256rmkz, 0},
+  {X86::VPERMT2DZrrk, X86::VPERMT2DZrmk, 0},
+  {X86::VPERMT2DZrrkz, X86::VPERMT2DZrmkz, 0},
+  {X86::VPERMT2PDZ128rrk, X86::VPERMT2PDZ128rmk, 0},
+  {X86::VPERMT2PDZ128rrkz, X86::VPERMT2PDZ128rmkz, 0},
+  {X86::VPERMT2PDZ256rrk, X86::VPERMT2PDZ256rmk, 0},
+  {X86::VPERMT2PDZ256rrkz, X86::VPERMT2PDZ256rmkz, 0},
+  {X86::VPERMT2PDZrrk, X86::VPERMT2PDZrmk, 0},
+  {X86::VPERMT2PDZrrkz, X86::VPERMT2PDZrmkz, 0},
+  {X86::VPERMT2PSZ128rrk, X86::VPERMT2PSZ128rmk, 0},
+  {X86::VPERMT2PSZ128rrkz, X86::VPERMT2PSZ128rmkz, 0},
+  {X86::VPERMT2PSZ256rrk, X86::VPERMT2PSZ256rmk, 0},
+  {X86::VPERMT2PSZ256rrkz, X86::VPERMT2PSZ256rmkz, 0},
+  {X86::VPERMT2PSZrrk, X86::VPERMT2PSZrmk, 0},
+  {X86::VPERMT2PSZrrkz, X86::VPERMT2PSZrmkz, 0},
+  {X86::VPERMT2QZ128rrk, X86::VPERMT2QZ128rmk, 0},
+  {X86::VPERMT2QZ128rrkz, X86::VPERMT2QZ128rmkz, 0},
+  {X86::VPERMT2QZ256rrk, X86::VPERMT2QZ256rmk, 0},
+  {X86::VPERMT2QZ256rrkz, X86::VPERMT2QZ256rmkz, 0},
+  {X86::VPERMT2QZrrk, X86::VPERMT2QZrmk, 0},
+  {X86::VPERMT2QZrrkz, X86::VPERMT2QZrmkz, 0},
+  {X86::VPERMT2WZ128rrk, X86::VPERMT2WZ128rmk, 0},
+  {X86::VPERMT2WZ128rrkz, X86::VPERMT2WZ128rmkz, 0},
+  {X86::VPERMT2WZ256rrk, X86::VPERMT2WZ256rmk, 0},
+  {X86::VPERMT2WZ256rrkz, X86::VPERMT2WZ256rmkz, 0},
+  {X86::VPERMT2WZrrk, X86::VPERMT2WZrmk, 0},
+  {X86::VPERMT2WZrrkz, X86::VPERMT2WZrmkz, 0},
   {X86::VPERMWZ128rrk, X86::VPERMWZ128rmk, 0},
   {X86::VPERMWZ256rrk, X86::VPERMWZ256rmk, 0},
   {X86::VPERMWZrrk, X86::VPERMWZrmk, 0},
@@ -7523,18 +7523,18 @@ static const X86FoldTableEntry BroadcastTable3[] = {
   {X86::VPDPWSSDZr, X86::VPDPWSSDZmb, TB_BCAST_D},
   {X86::VPERMDZ256rrkz, X86::VPERMDZ256rmbkz, TB_BCAST_D},
   {X86::VPERMDZrrkz, X86::VPERMDZrmbkz, TB_BCAST_D},
-  {X86::VPERMI2D128rr, X86::VPERMI2D128rmb, TB_BCAST_D},
-  {X86::VPERMI2D256rr, X86::VPERMI2D256rmb, TB_BCAST_D},
-  {X86::VPERMI2Drr, X86::VPERMI2Drmb, TB_BCAST_D},
-  {X86::VPERMI2PD128rr, X86::VPERMI2PD128rmb, TB_BCAST_SD},
-  {X86::VPERMI2PD256rr, X86::VPERMI2PD256rmb, TB_BCAST_SD},
-  {X86::VPERMI2PDrr, X86::VPERMI2PDrmb, TB_BCAST_SD},
-  {X86::VPERMI2PS128rr, X86::VPERMI2PS128rmb, TB_BCAST_SS},
-  {X86::VPERMI2PS256rr, X86::VPERMI2PS256rmb, TB_BCAST_SS},
-  {X86::VPERMI2PSrr, X86::VPERMI2PSrmb, TB_BCAST_SS},
-  {X86::VPERMI2Q128rr, X86::VPERMI2Q128rmb, TB_BCAST_Q},
-  {X86::VPERMI2Q256rr, X86::VPERMI2Q256rmb, TB_BCAST_Q},
-  {X86::VPERMI2Qrr, X86::VPERMI2Qrmb, TB_BCAST_Q},
+  {X86::VPERMI2DZ128rr, X86::VPERMI2DZ128rmb, TB_BCAST_D},
+  {X86::VPERMI2DZ256rr, X86::VPERMI2DZ256rmb, TB_BCAST_D},
+  {X86::VPERMI2DZrr, X86::VPERMI2DZrmb, TB_BCAST_D},
+  {X86::VPERMI2PDZ128rr, X86::VPERMI2PDZ128rmb, TB_BCAST_SD},
+  {X86::VPERMI2PDZ256rr, X86::VPERMI2PDZ256rmb, TB_BCAST_SD},
+  {X86::VPERMI2PDZrr, X86::VPERMI2PDZrmb, TB_BCAST_SD},
+  {X86::VPERMI2PSZ128rr, X86::VPERMI2PSZ128rmb, TB_BCAST_SS},
+  {X86::VPERMI2PSZ256rr, X86::VPERMI2PSZ256rmb, TB_BCAST_SS},
+  {X86::VPERMI2PSZrr, X86::VPERMI2PSZrmb, TB_BCAST_SS},
+  {X86::VPERMI2QZ128rr, X86::VPERMI2QZ128rmb, TB_BCAST_Q},
+  {X86::VPERMI2QZ256rr, X86::VPERMI2QZ256rmb, TB_BCAST_Q},
+  {X86::VPERMI2QZrr, X86::VPERMI2QZrmb, TB_BCAST_Q},
   {X86::VPERMILPDZ128rik, X86::VPERMILPDZ128mbik, TB_BCAST_SD},
   {X86::VPERMILPDZ128rrkz, X86::VPERMILPDZ128rmbkz, TB_BCAST_SD},
   {X86::VPERMILPDZ256rik, X86::VPERMILPDZ256mbik, TB_BCAST_SD},
@@ -7557,18 +7557,18 @@ static const X86FoldTableEntry BroadcastTable3[] = {
   {X86::VPERMQZ256rrkz, X86::VPERMQZ256rmbkz, TB_BCAST_Q},
   {X86::VPERMQZrik, X86::VPERMQZmbik, TB_BCAST_Q},
   {X86::VPERMQZrrkz, X86::VPERMQZrmbkz, TB_BCAST_Q},
-  {X86::VPERMT2D128rr, X86::VPERMT2D128rmb, TB_BCAST_D},
-  {X86::VPERMT2D256rr, X86::VPERMT2D256rmb, TB_BCAST_D},
-  {X86::VPERMT2Drr, X86::VPERMT2Drmb, TB_BCAST_D},
-  {X86::VPERMT2PD128rr, X86::VPERMT2PD128rmb, TB_BCAST_SD},
-  {X86::VPERMT2PD256rr, X86::VPERMT2PD256rmb, TB_BCAST_SD},
-  {X86::VPERMT2PDrr, X86::VPERMT2PDrmb, TB_BCAST_SD},
-  {X86::VPERMT2PS128rr, X86::VPERMT2PS128rmb, TB_BCAST_SS},
-  {X86::VPERMT2PS256rr, X86::VPERMT2PS256rmb, TB_BCAST_SS},
-  {X86::VPERMT2PSrr, X86::VPERMT2PSrmb, TB_BCAST_SS},
-  {X86::VPERMT2Q128rr, X86::VPERMT2Q128rmb, TB_BCAST_Q},
-  {X86::VPERMT2Q256rr, X86::VPERMT2Q256rmb, TB_BCAST_Q},
-  {X86::VPERMT2Qrr, X86::VPERMT2Qrmb, TB_BCAST_Q},
+  {X86::VPERMT2DZ128rr, X86::VPERMT2DZ128rmb, TB_BCAST_D},
+  {X86::VPERMT2DZ256rr, X86::VPERMT2DZ256rmb, TB_BCAST_D},
+  {X86::VPERMT2DZrr, X86::VPERMT2DZrmb, TB_BCAST_D},
+  {X86::VPERMT2PDZ128rr, X86::VPERMT2PDZ128rmb, TB_BCAST_SD},
+  {X86::VPERMT2PDZ256rr, X86::VPERMT2PDZ256rmb, TB_BCAST_SD},
+  {X86::VPERMT2PDZrr, X86::VPERMT2PDZrmb, TB_BCAST_SD},
+  {X86::VPERMT2PSZ128rr, X86::VPERMT2PSZ128rmb, TB_BCAST_SS},
+  {X86::VPERMT2PSZ256rr, X86::VPERMT2PSZ256rmb, TB_BCAST_SS},
+  {X86::VPERMT2PSZrr, X86::VPERMT2PSZrmb, TB_BCAST_SS},
+  {X86::VPERMT2QZ128rr, X86::VPERMT2QZ128rmb, TB_BCAST_Q},
+  {X86::VPERMT2QZ256rr, X86::VPERMT2QZ256rmb, TB_BCAST_Q},
+  {X86::VPERMT2QZrr, X86::VPERMT2QZrmb, TB_BCAST_Q},
   {X86::VPLZCNTDZ128rrk, X86::VPLZCNTDZ128rmbk, TB_BCAST_D},
   {X86::VPLZCNTDZ256rrk, X86::VPLZCNTDZ256rmbk, TB_BCAST_D},
   {X86::VPLZCNTDZrrk, X86::VPLZCNTDZrmbk, TB_BCAST_D},
@@ -8375,30 +8375,30 @@ static const X86FoldTableEntry BroadcastTable4[] = {
   {X86::VPDPWSSDZrkz, X86::VPDPWSSDZmbkz, TB_BCAST_D},
   {X86::VPERMDZ256rrk, X86::VPERMDZ256rmbk, TB_BCAST_D},
   {X86::VPERMDZrrk, X86::VPERMDZrmbk, TB_BCAST_D},
-  {X86::VPERMI2D128rrk, X86::VPERMI2D128rmbk, TB_BCAST_D},
-  {X86::VPERMI2D128rrkz, X86::VPERMI2D128rmbkz, TB_BCAST_D},
-  {X86::VPERMI2D256rrk, X86::VPERMI2D256rmbk, TB_BCAST_D},
-  {X86::VPERMI2D256rrkz, X86::VPERMI2D256rmbkz, TB_BCAST_D},
-  {X86::VPERMI2Drrk, X86::VPERMI2Drmbk, TB_BCAST_D},
-  {X86::VPERMI2Drrkz, X86::VPERMI2Drmbkz, TB_BCAST_D},
-  {X86::VPERMI2PD128rrk, X86::VPERMI2PD128rmbk, TB_BCAST_SD},
-  {X86::VPERMI2PD128rrkz, X86::VPERMI2PD128rmbkz, TB_BCAST_SD},
-  {X86::VPERMI2PD256rrk, X86::VPERMI2PD256rmbk, TB_BCAST_SD},
-  {X86::VPERMI2PD256rrkz, X86::VPERMI2PD256rmbkz, TB_BCAST_SD},
-  {X86::VPERMI2PDrrk, X86::VPERMI2PDrmbk, TB_BCAST_SD},
-  {X86::VPERMI2PDrrkz, X86::VPERMI2PDrmbkz, TB_BCAST_SD},
-  {X86::VPERMI2PS128rrk, X86::VPERMI2PS128rmbk, TB_BCAST_SS},
-  {X86::VPERMI2PS128rrkz, X86::VPERMI2PS128rmbkz, TB_BCAST_SS},
-  {X86::VPERMI2PS256rrk, X86::VPERMI2PS256rmbk, TB_BCAST_SS},
-  {X86::VPERMI2PS256rrkz, X86::VPERMI2PS256rmbkz, TB_BCAST_SS},
-  {X86::VPERMI2PSrrk, X86::VPERMI2PSrmbk, TB_BCAST_SS},
-  {X86::VPERMI2PSrrkz, X86::VPERMI2PSrmbkz, TB_BCAST_SS},
-  {X86::VPERMI2Q128rrk, X86::VPERMI2Q128rmbk, TB_BCAST_Q},
-  {X86::VPERMI2Q128rrkz, X86::VPERMI2Q128rmbkz, TB_BCAST_Q},
-  {X86::VPERMI2Q256rrk, X86::VPERMI2Q256rmbk, TB_BCAST_Q},
-  {X86::VPERMI2Q256rrkz, X86::VPERMI2Q256rmbkz, TB_BCAST_Q},
-  {X86::VPERMI2Qrrk, X86::VPERMI2Qrmbk, TB_BCAST_Q},
-  {X86::VPERMI2Qrrkz, X86::VPERMI2Qrmbkz, TB_BCAST_Q},
+  {X86::VPERMI2DZ128rrk, X86::VPERMI2DZ128rmbk, TB_BCAST_D},
+  {X86::VPERMI2DZ128rrkz, X86::VPERMI2DZ128rmbkz, TB_BCAST_D},
+  {X86::VPERMI2DZ256rrk, X86::VPERMI2DZ256rmbk, TB_BCAST_D},
+  {X86::VPERMI2DZ256rrkz, X86::VPERMI2DZ256rmbkz, TB_BCAST_D},
+  {X86::VPERMI2DZrrk, X86::VPERMI2DZrmbk, TB_BCAST_D},
+  {X86::VPERMI2DZrrkz, X86::VPERMI2DZrmbkz, TB_BCAST_D},
+  {X86::VPERMI2PDZ128rrk, X86::VPERMI2PDZ128rmbk, TB_BCAST_SD},
+  {X86::VPERMI2PDZ128rrkz, X86::VPERMI2PDZ128rmbkz, TB_BCAST_SD},
+  {X86::VPERMI2PDZ256rrk, X86::VPERMI2PDZ256rmbk, TB_BCAST_SD},
+  {X86::VPERMI2PDZ256rrkz, X86::VPERMI2PDZ256rmbkz, TB_BCAST_SD},
+  {X86::VPERMI2PDZrrk, X86::VPERMI2PDZrmbk, TB_BCAST_SD},
+  {X86::VPERMI2PDZrrkz, X86::VPERMI2PDZrmbkz, TB_BCAST_SD},
+  {X86::VPERMI2PSZ128rrk, X86::VPERMI2PSZ128rmbk, TB_BCAST_SS},
+  {X86::VPERMI2PSZ128rrkz, X86::VPERMI2PSZ128rmbkz, TB_BCAST_SS},
+  {X86::VPERMI2PSZ256rrk, X86::VPERMI2PSZ256rmbk, TB_BCAST_SS},
+  {X86::VPERMI2PSZ256rrkz, X86::VPERMI2PSZ256rmbkz, TB_BCAST_SS},
+  {X86::VPERMI2PSZrrk, X86::VPERMI2PSZrmbk, TB_BCAST_SS},
+  {X86::VPERMI2PSZrrkz, X86::VPERMI2PSZrmbkz, TB_BCAST_SS},
+  {X86::VPERMI2QZ128rrk, X86::VPERMI2QZ128rmbk, TB_BCAST_Q},
+  {X86::VPERMI2QZ128rrkz, X86::VPERMI2QZ128rmbkz, TB_BCAST_Q},
+  {X86::VPERMI2QZ256rrk, X86::VPERMI2QZ256rmbk, TB_BCAST_Q},
+  {X86::VPERMI2QZ256rrkz, X86::VPERMI2QZ256rmbkz, TB_BCAST_Q},
+  {X86::VPERMI2QZrrk, X86::VPERMI2QZrmbk, TB_BCAST_Q},
+  {X86::VPERMI2QZrrkz, X86::VPERMI2QZrmbkz, TB_BCAST_Q},
   {X86::VPERMILPDZ128rrk, X86::VPERMILPDZ128rmbk, TB_BCAST_SD},
   {X86::VPERMILPDZ256rrk, X86::VPERMILPDZ256rmbk, TB_BCAST_SD},
   {X86::VPERMILPDZrrk, X86::VPERMILPDZrmbk, TB_BCAST_SD},
@@ -8411,30 +8411,30 @@ static const X86FoldTableEntry BroadcastTable4[] = {
   {X86::VPERMPSZrrk, X86::VPERMPSZrmbk, TB_BCAST_SS},
   {X86::VPERMQZ256rrk, X86::VPERMQZ256rmbk, TB_BCAST_Q},
   {X86::VPERMQZrrk, X86::VPERMQZrmbk, TB_BCAST_Q},
-  {X86::VPERMT2D128rrk, X86::VPERMT2D128rmbk, TB_BCAST_D},
-  {X86::VPERMT2D128rrkz, X86::VPERMT2D128rmbkz, TB_BCAST_D},
-  {X86::VPERMT2D256rrk, X86::VPERMT2D256rmbk, TB_BCAST_D},
-  {X86::VPERMT2D256rrkz, X86::VPERMT2D256rmbkz, TB_BCAST_D},
-  {X86::VPERMT2Drrk, X86::VPERMT2Drmbk, TB_BCAST_D},
-  {X86::VPERMT2Drrkz, X86::VPERMT2Drmbkz, TB_BCAST_D},
-  {X86::VPERMT2PD128rrk, X86::VPERMT2PD128rmbk, TB_BCAST_SD},
-  {X86::VPERMT2PD128rrkz, X86::VPERMT2PD128rmbkz, TB_BCAST_SD},
-  {X86::VPERMT2PD256rrk, X86::VPERMT2PD256rmbk, TB_BCAST_SD},
-  {X86::VPERMT2PD256rrkz, X86::VPERMT2PD256rmbkz, TB_BCAST_SD},
-  {X86::VPERMT2PDrrk, X86::VPERMT2PDrmbk, TB_BCAST_SD},
-  {X86::VPERMT2PDrrkz, X86::VPERMT2PDrmbkz, TB_BCAST_SD},
-  {X86::VPERMT2PS128rrk, X86::VPERMT2PS128rmbk, TB_BCAST_SS},
-  {X86::VPERMT2PS128rrkz, X86::VPERMT2PS128rmbkz, TB_BCAST_SS},
-  {X86::VPERMT2PS256rrk, X86::VPERMT2PS256rmbk, TB_BCAST_SS},
-  {X86::VPERMT2PS256rrkz, X86::VPERMT2PS256rmbkz, TB_BCAST_SS},
-  {X86::VPERMT2PSrrk, X86::VPERMT2PSrmbk, TB_BCAST_SS},
-  {X86::VPERMT2PSrrkz, X86::VPERMT2PSrmbkz, TB_BCAST_SS},
-  {X86::VPERMT2Q128rrk, X86::VPERMT2Q128rmbk, TB_BCAST_Q},
-  {X86::VPERMT2Q128rrkz, X86::VPERMT2Q128rmbkz, TB_BCAST_Q},
-  {X86::VPERMT2Q256rrk, X86::VPERMT2Q256rmbk, TB_BCAST_Q},
-  {X86::VPERMT2Q256rrkz, X86::VPERMT2Q256rmbkz, TB_BCAST_Q},
-  {X86::VPERMT2Qrrk, X86::VPERMT2Qrmbk, TB_BCAST_Q},
-  {X86::VPERMT2Qrrkz, X86::VPERMT2Qrmbkz, TB_BCAST_Q},
+  {X86::VPERMT2DZ128rrk, X86::VPERMT2DZ128rmbk, TB_BCAST_D},
+  {X86::VPERMT2DZ128rrkz, X86::VPERMT2DZ128rmbkz, TB_BCAST_D},
+  {X86::VPERMT2DZ256rrk, X86::VPERMT2DZ256rmbk, TB_BCAST_D},
+  {X86::VPERMT2DZ256rrkz, X86::VPERMT2DZ256rmbkz, TB_BCAST_D},
+  {X86::VPERMT2DZrrk, X86::VPERMT2DZrmbk, TB_BCAST_D},
+  {X86::VPERMT2DZrrkz, X86::VPERMT2DZrmbkz, TB_BCAST_D},
+  {X86::VPERMT2PDZ128rrk, X86::VPERMT2PDZ128rmbk, TB_BCAST_SD},
+  {X86::VPERMT2PDZ128rrkz, X86::VPERMT2PDZ128rmbkz, TB_BCAST_SD},
+  {X86::VPERMT2PDZ256rrk, X86::VPERMT2PDZ256rmbk, TB_BCAST_SD},
+  {X86::VPERMT2PDZ256rrkz, X86::VPERMT2PDZ256rmbkz, TB_BCAST_SD},
+  {X86::VPERMT2PDZrrk, X86::VPERMT2PDZrmbk, TB_BCAST_SD},
+  {X86::VPERMT2PDZrrkz, X86::VPERMT2PDZrmbkz, TB_BCAST_SD},
+  {X86::VPERMT2PSZ128rrk, X86::VPERMT2PSZ128rmbk, TB_BCAST_SS},
+  {X86::VPERMT2PSZ128rrkz, X86::VPERMT2PSZ128rmbkz, TB_BCAST_SS},
+  {X86::VPERMT2PSZ256rrk, X86::VPERMT2PSZ256rmbk, TB_BCAST_SS},
+  {X86::VPERMT2PSZ256rrkz, X86::VPERMT2PSZ256rmbkz, TB_BCAST_SS},
+  {X86::VPERMT2PSZrrk, X86::VPERMT2PSZrmbk, TB_BCAST_SS},
+  {X86::VPERMT2PSZrrkz, X86::VPERMT2PSZrmbkz, TB_BCAST_SS},
+  {X86::VPERMT2QZ128rrk, X86::VPERMT2QZ128rmbk, TB_BCAST_Q},
+  {X86::VPERMT2QZ128rrkz, X86::VPERMT2QZ128rmbkz, TB_BCAST_Q},
+  {X86::VPERMT2QZ256rrk, X86::VPERMT2QZ256rmbk, TB_BCAST_Q},
+  {X86::VPERMT2QZ256rrkz, X86::VPERMT2QZ256rmbkz, TB_BCAST_Q},
+  {X86::VPERMT2QZrrk, X86::VPERMT2QZrmbk, TB_BCAST_Q},
+  {X86::VPERMT2QZrrkz, X86::VPERMT2QZrmbkz, TB_BCAST_Q},
   {X86::VPMADD52HUQZ128rk, X86::VPMADD52HUQZ128mbk, TB_BCAST_Q},
   {X86::VPMADD52HUQZ128rkz, X86::VPMADD52HUQZ128mbkz, TB_BCAST_Q},
   {X86::VPMADD52HUQZ256rk, X86::VPMADD52HUQZ256mbk, TB_BCAST_Q},
diff --git a/llvm/test/Transforms/ConstantHoisting/AArch64/large-immediate.ll b/llvm/test/Transforms/ConstantHoisting/AArch64/large-immediate.ll
index 015f52157b9e7c74a57938319663c7ca30f61ed3..196a104adc0233fc310d99b4d099e0d9e82779a9 100644
--- a/llvm/test/Transforms/ConstantHoisting/AArch64/large-immediate.ll
+++ b/llvm/test/Transforms/ConstantHoisting/AArch64/large-immediate.ll
@@ -1,27 +1,134 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3
 ; RUN: opt -mtriple=arm64-darwin-unknown -S -passes=consthoist < %s | FileCheck %s
 
-define i128 @test1(i128 %a) nounwind {
-; CHECK-LABEL: test1
-; CHECK: %const = bitcast i128 12297829382473034410122878 to i128
+define i128 @test1(i128 %a) {
+; CHECK-LABEL: define i128 @test1(
+; CHECK-SAME: i128 [[A:%.*]]) {
+; CHECK-NEXT:    [[CONST:%.*]] = bitcast i128 12297829382473034410122878 to i128
+; CHECK-NEXT:    [[TMP1:%.*]] = add i128 [[A]], [[CONST]]
+; CHECK-NEXT:    [[TMP2:%.*]] = add i128 [[TMP1]], [[CONST]]
+; CHECK-NEXT:    ret i128 [[TMP2]]
+;
   %1 = add i128 %a, 12297829382473034410122878
   %2 = add i128 %1, 12297829382473034410122878
   ret i128 %2
 }
 
 ; Check that we don't hoist large, but cheap constants
-define i512 @test2(i512 %a) nounwind {
-; CHECK-LABEL: test2
-; CHECK-NOT: %const = bitcast i512 7 to i512
+define i512 @test2(i512 %a) {
+; CHECK-LABEL: define i512 @test2(
+; CHECK-SAME: i512 [[A:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = and i512 [[A]], 7
+; CHECK-NEXT:    [[TMP2:%.*]] = or i512 [[TMP1]], 7
+; CHECK-NEXT:    ret i512 [[TMP2]]
+;
   %1 = and i512 %a, 7
   %2 = or i512 %1, 7
   ret i512 %2
 }
 
 ; Check that we don't hoist the shift value of a shift instruction.
-define i512 @test3(i512 %a) nounwind {
-; CHECK-LABEL: test3
-; CHECK-NOT: %const = bitcast i512 504 to i512
+define i512 @test3(i512 %a) {
+; CHECK-LABEL: define i512 @test3(
+; CHECK-SAME: i512 [[A:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = shl i512 [[A]], 504
+; CHECK-NEXT:    [[TMP2:%.*]] = ashr i512 [[TMP1]], 504
+; CHECK-NEXT:    ret i512 [[TMP2]]
+;
   %1 = shl i512 %a, 504
   %2 = ashr i512 %1, 504
   ret i512 %2
 }
+
+; Ensure the code generator has the information necessary to simply sdiv.
+define i64 @sdiv(i64 %a) {
+; CHECK-LABEL: define i64 @sdiv(
+; CHECK-SAME: i64 [[A:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = sdiv i64 [[A]], 4294967087
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[TMP1]], 4294967087
+; CHECK-NEXT:    ret i64 [[TMP2]]
+;
+  %1 = sdiv i64 %a, 4294967087
+  %2 = add i64 %1, 4294967087
+  ret i64 %2
+}
+
+; Ensure the code generator has the information necessary to simply srem.
+define i64 @srem(i64 %a) {
+; CHECK-LABEL: define i64 @srem(
+; CHECK-SAME: i64 [[A:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = srem i64 [[A]], 4294967087
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[TMP1]], 4294967087
+; CHECK-NEXT:    ret i64 [[TMP2]]
+;
+  %1 = srem i64 %a, 4294967087
+  %2 = add i64 %1, 4294967087
+  ret i64 %2
+}
+
+; Ensure the code generator has the information necessary to simply udiv.
+define i64 @udiv(i64 %a) {
+; CHECK-LABEL: define i64 @udiv(
+; CHECK-SAME: i64 [[A:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = udiv i64 [[A]], 4294967087
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[TMP1]], 4294967087
+; CHECK-NEXT:    ret i64 [[TMP2]]
+;
+  %1 = udiv i64 %a, 4294967087
+  %2 = add i64 %1, 4294967087
+  ret i64 %2
+}
+
+; Ensure the code generator has the information necessary to simply urem.
+define i64 @urem(i64 %a) {
+; CHECK-LABEL: define i64 @urem(
+; CHECK-SAME: i64 [[A:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = urem i64 [[A]], 4294967087
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[TMP1]], 4294967087
+; CHECK-NEXT:    ret i64 [[TMP2]]
+;
+  %1 = urem i64 %a, 4294967087
+  %2 = add i64 %1, 4294967087
+  ret i64 %2
+}
+
+; Code generator will not decompose divide like operations when the divisor is
+; no a constant.
+define i64 @sdiv_non_const_divisor(i64 %a) {
+; CHECK-LABEL: define i64 @sdiv_non_const_divisor(
+; CHECK-SAME: i64 [[A:%.*]]) {
+; CHECK-NEXT:    [[CONST:%.*]] = bitcast i64 4294967087 to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = sdiv i64 [[CONST]], [[A]]
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[TMP1]], [[CONST]]
+; CHECK-NEXT:    ret i64 [[TMP2]]
+;
+  %1 = sdiv i64 4294967087, %a
+  %2 = add i64 %1, 4294967087
+  ret i64 %2
+}
+
+; Code generator emits divide instructions when optimising for size.
+define i64 @sdiv_minsize(i64 %a) minsize {
+; CHECK-LABEL: define i64 @sdiv_minsize(
+; CHECK-SAME: i64 [[A:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:    [[CONST:%.*]] = bitcast i64 4294967087 to i64
+; CHECK-NEXT:    [[TMP1:%.*]] = sdiv i64 [[A]], [[CONST]]
+; CHECK-NEXT:    [[TMP2:%.*]] = add i64 [[TMP1]], [[CONST]]
+; CHECK-NEXT:    ret i64 [[TMP2]]
+;
+  %1 = sdiv i64 %a, 4294967087
+  %2 = add i64 %1, 4294967087
+  ret i64 %2
+}
+
+define <2 x i64> @sdiv_v2i64(<2 x i64> %a) {
+; CHECK-LABEL: define <2 x i64> @sdiv_v2i64(
+; CHECK-SAME: <2 x i64> [[A:%.*]]) {
+; CHECK-NEXT:    [[TMP1:%.*]] = sdiv <2 x i64> [[A]], <i64 4294967087, i64 4294967087>
+; CHECK-NEXT:    [[TMP2:%.*]] = add <2 x i64> [[TMP1]], <i64 4294967087, i64 4294967087>
+; CHECK-NEXT:    ret <2 x i64> [[TMP2]]
+;
+  %1 = sdiv <2 x i64> %a, <i64 4294967087, i64 4294967087>
+  %2 = add <2 x i64> %1, <i64 4294967087, i64 4294967087>
+  ret <2 x i64> %2
+}
diff --git a/llvm/test/Transforms/ConstraintElimination/debug.ll b/llvm/test/Transforms/ConstraintElimination/debug.ll
index f3f0f5056135c2163ed08097b57ce12afb029499..2409af09735d1e7c8837a6a963bc9f19e334cfcf 100644
--- a/llvm/test/Transforms/ConstraintElimination/debug.ll
+++ b/llvm/test/Transforms/ConstraintElimination/debug.ll
@@ -3,16 +3,16 @@
 ; REQUIRES: asserts
 
 define i1 @test_and_ule(i4 %x, i4 %y, i4 %z) {
-; CHECK:      Processing fact to add to the system:  ule i4 %x, %y
-; CHECK-NEXT: Adding 'ule %x, %y'
+; CHECK:      Processing fact to add to the system: icmp ule i4 %x, %y
+; CHECK-NEXT: Adding 'icmp ule i4 %x, %y'
 ; CHECK-NEXT:  constraint: %x + -1 * %y <= 0
 
-; CHECK:      Processing fact to add to the system:  ule i4 %y, %z
-; CHECK-NEXT: Adding 'ule %y, %z'
+; CHECK:      Processing fact to add to the system: icmp ule i4 %y, %z
+; CHECK-NEXT: Adding 'icmp ule i4 %y, %z'
 ; CHECK-NEXT:  constraint: %y + -1 * %z <= 0
 
 ; CHECK: Checking   %t.1 = icmp ule i4 %x, %z
-; CHECK: Condition   %t.1 = icmp ule i4 %x, %z implied by dominating constraints
+; CHECK: Condition icmp ule i4 %x, %z implied by dominating constraints
 
 ; CHECK: Removing %y + -1 * %z <= 0
 ; CHECK: Removing %x + -1 * %y <= 0
@@ -33,16 +33,16 @@ exit:
 }
 
 define i1 @test_and_ugt(i4 %x, i4 %y, i4 %z) {
-; CHECK:      Processing fact to add to the system:   ugt i4 %x, %y
-; CHECK-NEXT: Adding 'ugt %x, %y'
+; CHECK:      Processing fact to add to the system: icmp ugt i4 %x, %y
+; CHECK-NEXT: Adding 'icmp ugt i4 %x, %y'
 ; CHECK-NEXT:  constraint: -1 * %x + %y <= -1
 
-; CHECK:      Processing fact to add to the system:   ugt i4 %y, %z
-; CHECK-NEXT: Adding 'ugt %y, %z'
+; CHECK:      Processing fact to add to the system: icmp ugt i4 %y, %z
+; CHECK-NEXT: Adding 'icmp ugt i4 %y, %z'
 ; CHECK-NEXT:  constraint: -1 * %y + %z <= -1
 
 ; CHECK: Checking   %f.1 = icmp ule i4 %x, %z
-; CHECK: Condition ugt i4 %x, i4 %z implied by dominating constraints
+; CHECK: Condition icmp ugt i4 %x, %z implied by dominating constraints
 
 ; CHECK: Removing -1 * %y + %z <= -1
 ; CHECK: Removing -1 * %x + %y <= -1
diff --git a/llvm/test/Transforms/ConstraintElimination/reproducer-remarks-debug.ll b/llvm/test/Transforms/ConstraintElimination/reproducer-remarks-debug.ll
index b8343aed8b4af75091f7e69be8d24aa67f204fa6..4fdc8e583112ef7ed97816e8e1e3ea116ed266f2 100644
--- a/llvm/test/Transforms/ConstraintElimination/reproducer-remarks-debug.ll
+++ b/llvm/test/Transforms/ConstraintElimination/reproducer-remarks-debug.ll
@@ -4,7 +4,7 @@
 
 target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 
-; CHECK:      Condition   %c.2 = icmp eq ptr %a, null implied by dominating constraints
+; CHECK:      Condition icmp eq ptr %a, null implied by dominating constraints
 ; CHECK-NEXT: %a <= 0
 ; CHECK-NEXT: Creating reproducer for   %c.2 = icmp eq ptr %a, null
 ; CHECK-NEXT:   found external input ptr %a
diff --git a/llvm/test/Transforms/Coroutines/coro-debug-O2.ll b/llvm/test/Transforms/Coroutines/coro-debug-O2.ll
index a668bd1a521960cd94e5882f6fa67edc4bf35a47..3f9af30a92e343e75fe94c1795d0655569debba9 100644
--- a/llvm/test/Transforms/Coroutines/coro-debug-O2.ll
+++ b/llvm/test/Transforms/Coroutines/coro-debug-O2.ll
@@ -1,4 +1,5 @@
 ; RUN: opt < %s -passes='module(coro-early),cgscc(coro-split<reuse-storage>),function(sroa)' -S | FileCheck %s
+; RUN: opt --try-experimental-debuginfo-iterators < %s -passes='module(coro-early),cgscc(coro-split<reuse-storage>),function(sroa)' -S | FileCheck %s
 
 ; Checks whether the dbg.declare for `__promise` remains valid under O2.
 
diff --git a/llvm/test/Transforms/Coroutines/coro-debug-coro-frame.ll b/llvm/test/Transforms/Coroutines/coro-debug-coro-frame.ll
index 7f5679e4d522fd2514406e1ed3189bdce4f81e69..2978f85be238542d12d727e4d5c2f454f4652aa2 100644
--- a/llvm/test/Transforms/Coroutines/coro-debug-coro-frame.ll
+++ b/llvm/test/Transforms/Coroutines/coro-debug-coro-frame.ll
@@ -1,4 +1,5 @@
 ; RUN: opt < %s -passes='module(coro-early),cgscc(coro-split,coro-split)' -S | FileCheck %s
+; RUN: opt --try-experimental-debuginfo-iterators < %s -passes='module(coro-early),cgscc(coro-split,coro-split)' -S | FileCheck %s
 
 ; Checks whether the dbg.declare for `__coro_frame` are created.
 
diff --git a/llvm/test/Transforms/Coroutines/coro-debug-dbg.values-not_used_in_frame.ll b/llvm/test/Transforms/Coroutines/coro-debug-dbg.values-not_used_in_frame.ll
index 1b9a1bd63a2e3e544ab36635daeed9c5723a1cb8..79793dc293d0d9e1ef2a8c859001bee7c26f90a4 100644
--- a/llvm/test/Transforms/Coroutines/coro-debug-dbg.values-not_used_in_frame.ll
+++ b/llvm/test/Transforms/Coroutines/coro-debug-dbg.values-not_used_in_frame.ll
@@ -1,5 +1,6 @@
 ; Tests whether resume function would remain dbg.value infomation if corresponding values are not used in the frame.
 ; RUN: opt < %s -passes='module(coro-early),cgscc(coro-split,coro-split)' -S | FileCheck %s
+; RUN: opt --try-experimental-debuginfo-iterators < %s -passes='module(coro-early),cgscc(coro-split,coro-split)' -S | FileCheck %s
 ;
 ; This file is based on coro-debug-frame-variable.ll.
 ; CHECK:  define internal fastcc void @f.resume(ptr noundef nonnull align 16 dereferenceable(80) %begin) !dbg ![[RESUME_FN_DBG_NUM:[0-9]+]]
diff --git a/llvm/test/Transforms/Coroutines/coro-debug-dbg.values.ll b/llvm/test/Transforms/Coroutines/coro-debug-dbg.values.ll
index 54cdde8ae4ac0cbea89ec3db742f6cf5e76a2fff..47b2ddafcfc650db8336d93ed4a28bb7a2758f5e 100644
--- a/llvm/test/Transforms/Coroutines/coro-debug-dbg.values.ll
+++ b/llvm/test/Transforms/Coroutines/coro-debug-dbg.values.ll
@@ -1,5 +1,6 @@
 ; Tests whether resume function would remain dbg.value infomation.
 ; RUN: opt < %s -passes='module(coro-early),cgscc(coro-split,coro-split)' -S | FileCheck %s
+; RUN: opt --try-experimental-debuginfo-iterators < %s -passes='module(coro-early),cgscc(coro-split,coro-split)' -S | FileCheck %s
 ;
 ; This file is based on coro-debug-frame-variable.ll.
 ; CHECK-LABEL: define void @f(
diff --git a/llvm/test/Transforms/Coroutines/coro-debug-frame-variable.ll b/llvm/test/Transforms/Coroutines/coro-debug-frame-variable.ll
index 53995fb3498e59a24df4898fa855663e54dc94df..19a89fefd5269dbd37ea71fcca43e69730164407 100644
--- a/llvm/test/Transforms/Coroutines/coro-debug-frame-variable.ll
+++ b/llvm/test/Transforms/Coroutines/coro-debug-frame-variable.ll
@@ -1,4 +1,5 @@
 ; RUN: opt < %s -passes='default<O0>' -S | FileCheck %s
+; RUN: opt --try-experimental-debuginfo-iterators < %s -passes='default<O0>' -S | FileCheck %s
 
 ; Define a function 'f' that resembles the Clang frontend's output for the
 ; following C++ coroutine:
@@ -31,8 +32,8 @@
 ; CHECK:       entry:
 ; CHECK:         %j = alloca i32, align 4
 ; CHECK:         call void @llvm.dbg.declare(metadata ptr %j, metadata ![[JVAR:[0-9]+]], metadata !DIExpression()), !dbg ![[JDBGLOC:[0-9]+]]
-; CHECK:         %[[MEMORY:.*]] = call ptr @new({{.+}}), !dbg ![[IDBGLOC:[0-9]+]]
-; CHECK:         call void @llvm.dbg.declare(metadata ptr %[[MEMORY]], metadata ![[XVAR:[0-9]+]], metadata !DIExpression(DW_OP_plus_uconst, 32)), !dbg ![[IDBGLOC]]
+; CHECK:         %[[MEMORY:.*]] = call ptr @new
+; CHECK:         call void @llvm.dbg.declare(metadata ptr %[[MEMORY]], metadata ![[XVAR:[0-9]+]], metadata !DIExpression(DW_OP_plus_uconst, 32)), !dbg ![[IDBGLOC:[0-9]+]]
 ; CHECK:         call void @llvm.dbg.declare(metadata ptr %[[MEMORY]], metadata ![[IVAR:[0-9]+]], metadata !DIExpression(DW_OP_plus_uconst, 20)), !dbg ![[IDBGLOC]]
 ; CHECK:       await.ready:
 ;
@@ -48,20 +49,18 @@
 ; CHECK:       await.ready:
 ;
 ; CHECK-DAG: ![[IVAR]] = !DILocalVariable(name: "i"
-; CHECK-DAG: ![[PROG_SCOPE:[0-9]+]] = distinct !DISubprogram(name: "foo", linkageName: "_Z3foov"
-; CHECK-DAG: ![[BLK_SCOPE:[0-9]+]] = distinct !DILexicalBlock(scope: ![[PROG_SCOPE]], file: !1, line: 23, column: 12)
-; CHECK-DAG: ![[IDBGLOC]] = !DILocation(line: 23, column: 6, scope: ![[PROG_SCOPE]])
+; CHECK-DAG: ![[SCOPE:[0-9]+]] = distinct !DILexicalBlock(scope: !6, file: !1, line: 23, column: 12)
+; CHECK-DAG: ![[IDBGLOC]] = !DILocation(line: 24, column: 7, scope: ![[SCOPE]])
 ; CHECK-DAG: ![[XVAR]] = !DILocalVariable(name: "x"
 ; CHECK-DAG: ![[JVAR]] = !DILocalVariable(name: "j"
-; CHECK-DAG: ![[JDBGLOC]] = !DILocation(line: 32, column: 7, scope: ![[BLK_SCOPE]])
+; CHECK-DAG: ![[JDBGLOC]] = !DILocation(line: 32, column: 7, scope: ![[SCOPE]])
 
 ; CHECK-DAG: ![[XVAR_RESUME]] = !DILocalVariable(name: "x"
-; CHECK-DAG: ![[RESUME_PROG_SCOPE:[0-9]+]] = distinct !DISubprogram(name: "foo", linkageName: "_Z3foov"
-; CHECK-DAG: ![[IDBGLOC_RESUME]] = !DILocation(line: 24, column: 7, scope: ![[RESUME_BLK_SCOPE:[0-9]+]])
-; CHECK-DAG: ![[RESUME_BLK_SCOPE]] = distinct !DILexicalBlock(scope: ![[RESUME_PROG_SCOPE]], file: !1, line: 23, column: 12)
+; CHECK-DAG: ![[IDBGLOC_RESUME]] = !DILocation(line: 24, column: 7, scope: ![[RESUME_SCOPE:[0-9]+]])
+; CHECK-DAG: ![[RESUME_SCOPE]] = distinct !DILexicalBlock(scope: !22, file: !1, line: 23, column: 12)
 ; CHECK-DAG: ![[IVAR_RESUME]] = !DILocalVariable(name: "i"
 ; CHECK-DAG: ![[JVAR_RESUME]] = !DILocalVariable(name: "j"
-; CHECK-DAG: ![[JDBGLOC_RESUME]] = !DILocation(line: 32, column: 7, scope: ![[RESUME_BLK_SCOPE]])
+; CHECK-DAG: ![[JDBGLOC_RESUME]] = !DILocation(line: 32, column: 7, scope: ![[RESUME_SCOPE]])
 define void @f() presplitcoroutine !dbg !8 {
 entry:
   %__promise = alloca i8, align 8
@@ -73,13 +72,13 @@ entry:
   br i1 %alloc, label %coro.alloc, label %coro.init
 
 coro.alloc:                                       ; preds = %entry
-  %size = call i64 @llvm.coro.size.i64(), !dbg !23
-  %memory = call ptr @new(i64 %size), !dbg !23
-  br label %coro.init, !dbg !23
+  %size = call i64 @llvm.coro.size.i64()
+  %memory = call ptr @new(i64 %size)
+  br label %coro.init
 
 coro.init:                                        ; preds = %coro.alloc, %entry
-  %phi.entry.alloc = phi ptr [ null, %entry ], [ %memory, %coro.alloc ], !dbg !23
-  %begin = call ptr @llvm.coro.begin(token %id, ptr %phi.entry.alloc), !dbg !23
+  %phi.entry.alloc = phi ptr [ null, %entry ], [ %memory, %coro.alloc ]
+  %begin = call ptr @llvm.coro.begin(token %id, ptr %phi.entry.alloc)
   %ready = call i1 @await_ready()
   br i1 %ready, label %init.ready, label %init.suspend
 
@@ -241,4 +240,3 @@ declare void @llvm.memset.p0.i64(ptr nocapture writeonly, i8, i64, i1 immarg)
 !20 = !DILocation(line: 43, column: 3, scope: !7)
 !21 = !DILocation(line: 43, column: 8, scope: !7)
 !22 = distinct !DILexicalBlock(scope: !8, file: !1, line: 23, column: 12)
-!23 = !DILocation(line: 23, column: 6, scope: !8)
diff --git a/llvm/test/Transforms/Coroutines/coro-debug-spill-dbg.declare.ll b/llvm/test/Transforms/Coroutines/coro-debug-spill-dbg.declare.ll
index e7a271a96ead125bffa7733ddb5ee54c5a0481a6..c943ea5ca22ec039e402814a9745aff62ce06114 100644
--- a/llvm/test/Transforms/Coroutines/coro-debug-spill-dbg.declare.ll
+++ b/llvm/test/Transforms/Coroutines/coro-debug-spill-dbg.declare.ll
@@ -1,6 +1,7 @@
 ; Test spilling a temp generates dbg.declare in resume/destroy/cleanup functions.
 ;
 ; RUN: opt < %s -passes='cgscc(coro-split)' -S | FileCheck %s
+; RUN: opt --try-experimental-debuginfo-iterators < %s -passes='cgscc(coro-split)' -S | FileCheck %s
 ;
 ; The test case simulates a coroutine method in a class.
 ;
diff --git a/llvm/test/Transforms/Coroutines/coro-debug.ll b/llvm/test/Transforms/Coroutines/coro-debug.ll
index 064693c80ad233c38c4356c19a8310b7d70c7732..4792825f4ce080c0f52dacbff78195fbb6b88bb1 100644
--- a/llvm/test/Transforms/Coroutines/coro-debug.ll
+++ b/llvm/test/Transforms/Coroutines/coro-debug.ll
@@ -1,5 +1,6 @@
 ; Tests that debug information is sane after coro-split
 ; RUN: opt < %s -passes='cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck %s
+; RUN: opt --try-experimental-debuginfo-iterators < %s -passes='cgscc(coro-split),simplifycfg,early-cse' -S | FileCheck %s
 
 source_filename = "simple-repro.c"
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
diff --git a/llvm/test/Transforms/Coroutines/coro-split-dbg.ll b/llvm/test/Transforms/Coroutines/coro-split-dbg.ll
index 184d4a564ab723909d6698f9936a4f051d459c3d..7d95be308a5f490b5b3f47eed3ccfc2a9cd29f7e 100644
--- a/llvm/test/Transforms/Coroutines/coro-split-dbg.ll
+++ b/llvm/test/Transforms/Coroutines/coro-split-dbg.ll
@@ -1,6 +1,7 @@
 ; Make sure that coro-split correctly deals with debug information.
 ; The test here is simply that it does not result in bad IR that will crash opt.
 ; RUN: opt < %s -passes='cgscc(coro-split),simplifycfg,early-cse' -disable-output
+; RUN: opt --try-experimental-debuginfo-iterators < %s -passes='cgscc(coro-split),simplifycfg,early-cse' -disable-output
 source_filename = "coro.c"
 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
diff --git a/llvm/test/Transforms/Coroutines/swift-async-dbg.ll b/llvm/test/Transforms/Coroutines/swift-async-dbg.ll
index af7d816c1286166b265001aca4df5704318c9407..74edf7a3f3a54004065077e7ff5b03d41a7f57f6 100644
--- a/llvm/test/Transforms/Coroutines/swift-async-dbg.ll
+++ b/llvm/test/Transforms/Coroutines/swift-async-dbg.ll
@@ -2,6 +2,13 @@
 ; RUN: opt -mtriple='x86_64' %s -S -passes='module(coro-early),cgscc(coro-split,simplifycfg)' -o - | FileCheck %s
 ; RUN: opt -mtriple='i386-' %s -S -passes='module(coro-early),cgscc(coro-split,simplifycfg)' -o - | FileCheck %s --check-prefix=NOENTRY
 ; RUN: opt -mtriple='armv7-' %s -S -passes='module(coro-early),cgscc(coro-split,simplifycfg)' -o - | FileCheck %s --check-prefix=NOENTRY
+
+;; Replicate those tests with non-instruction debug markers.
+; RUN: opt --try-experimental-debuginfo-iterators -mtriple='arm64-' %s -S -passes='module(coro-early),cgscc(coro-split,simplifycfg)' -o - | FileCheck %s
+; RUN: opt --try-experimental-debuginfo-iterators -mtriple='x86_64' %s -S -passes='module(coro-early),cgscc(coro-split,simplifycfg)' -o - | FileCheck %s
+; RUN: opt --try-experimental-debuginfo-iterators -mtriple='i386-' %s -S -passes='module(coro-early),cgscc(coro-split,simplifycfg)' -o - | FileCheck %s --check-prefix=NOENTRY
+; RUN: opt --try-experimental-debuginfo-iterators -mtriple='armv7-' %s -S -passes='module(coro-early),cgscc(coro-split,simplifycfg)' -o - | FileCheck %s --check-prefix=NOENTRY
+
 ; NOENTRY-NOT: OP_llvm_entry_value
 
 target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/cond-using-block-value.ll b/llvm/test/Transforms/CorrelatedValuePropagation/cond-using-block-value.ll
new file mode 100644
index 0000000000000000000000000000000000000000..d30b31d317a6dee311309647477eb13f6ec5b8be
--- /dev/null
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/cond-using-block-value.ll
@@ -0,0 +1,98 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt -S -passes=correlated-propagation < %s | FileCheck %s
+
+declare void @use(i1)
+
+define void @test_icmp_from_implied_cond(i32 %a, i32 %b) {
+; CHECK-LABEL: define void @test_icmp_from_implied_cond(
+; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    [[A_CMP:%.*]] = icmp ugt i32 [[A]], 32
+; CHECK-NEXT:    br i1 [[A_CMP]], label [[END:%.*]], label [[L1:%.*]]
+; CHECK:       l1:
+; CHECK-NEXT:    [[COND:%.*]] = icmp ult i32 [[B]], [[A]]
+; CHECK-NEXT:    br i1 [[COND]], label [[L2:%.*]], label [[END]]
+; CHECK:       l2:
+; CHECK-NEXT:    [[B_CMP1:%.*]] = icmp ult i32 [[B]], 32
+; CHECK-NEXT:    call void @use(i1 [[B_CMP1]])
+; CHECK-NEXT:    [[B_CMP2:%.*]] = icmp ult i32 [[B]], 31
+; CHECK-NEXT:    call void @use(i1 [[B_CMP2]])
+; CHECK-NEXT:    ret void
+; CHECK:       end:
+; CHECK-NEXT:    ret void
+;
+  %a.cmp = icmp ugt i32 %a, 32
+  br i1 %a.cmp, label %end, label %l1
+
+l1:
+  %cond = icmp ult i32 %b, %a
+  br i1 %cond, label %l2, label %end
+
+l2:
+  %b.cmp1 = icmp ult i32 %b, 32
+  call void @use(i1 %b.cmp1)
+  %b.cmp2 = icmp ult i32 %b, 31
+  call void @use(i1 %b.cmp2)
+  ret void
+
+end:
+  ret void
+}
+
+define i64 @test_sext_from_implied_cond(i32 %a, i32 %b) {
+; CHECK-LABEL: define i64 @test_sext_from_implied_cond(
+; CHECK-SAME: i32 [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    [[A_CMP:%.*]] = icmp slt i32 [[A]], 0
+; CHECK-NEXT:    br i1 [[A_CMP]], label [[END:%.*]], label [[L1:%.*]]
+; CHECK:       l1:
+; CHECK-NEXT:    [[COND:%.*]] = icmp ult i32 [[B]], [[A]]
+; CHECK-NEXT:    br i1 [[COND]], label [[L2:%.*]], label [[END]]
+; CHECK:       l2:
+; CHECK-NEXT:    [[SEXT:%.*]] = sext i32 [[B]] to i64
+; CHECK-NEXT:    ret i64 [[SEXT]]
+; CHECK:       end:
+; CHECK-NEXT:    ret i64 0
+;
+  %a.cmp = icmp slt i32 %a, 0
+  br i1 %a.cmp, label %end, label %l1
+
+l1:
+  %cond = icmp ult i32 %b, %a
+  br i1 %cond, label %l2, label %end
+
+l2:
+  %sext = sext i32 %b to i64
+  ret i64 %sext
+
+end:
+  ret i64 0
+}
+
+define void @test_icmp_from_implied_range(i16 %x, i32 %b) {
+; CHECK-LABEL: define void @test_icmp_from_implied_range(
+; CHECK-SAME: i16 [[X:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    [[A:%.*]] = zext i16 [[X]] to i32
+; CHECK-NEXT:    [[COND:%.*]] = icmp ult i32 [[B]], [[A]]
+; CHECK-NEXT:    br i1 [[COND]], label [[L1:%.*]], label [[END:%.*]]
+; CHECK:       l1:
+; CHECK-NEXT:    [[B_CMP1:%.*]] = icmp ult i32 [[B]], 65535
+; CHECK-NEXT:    call void @use(i1 [[B_CMP1]])
+; CHECK-NEXT:    [[B_CMP2:%.*]] = icmp ult i32 [[B]], 65534
+; CHECK-NEXT:    call void @use(i1 [[B_CMP2]])
+; CHECK-NEXT:    ret void
+; CHECK:       end:
+; CHECK-NEXT:    ret void
+;
+  %a = zext i16 %x to i32
+  %cond = icmp ult i32 %b, %a
+  br i1 %cond, label %l1, label %end
+
+l1:
+  %b.cmp1 = icmp ult i32 %b, 65535
+  call void @use(i1 %b.cmp1)
+  %b.cmp2 = icmp ult i32 %b, 65534
+  call void @use(i1 %b.cmp2)
+  ret void
+
+end:
+  ret void
+}
diff --git a/llvm/test/Transforms/DeadStoreElimination/simple.ll b/llvm/test/Transforms/DeadStoreElimination/simple.ll
index 4cfbbe16667cad12ba3c528eee2dbd5dda236e21..e5d3dd09fa148db576ff96e52829d4a6e90e462f 100644
--- a/llvm/test/Transforms/DeadStoreElimination/simple.ll
+++ b/llvm/test/Transforms/DeadStoreElimination/simple.ll
@@ -514,13 +514,12 @@ define void @test34(ptr noalias %p) {
   store i32 0, ptr %p
   ret void
 }
-; Same as previous case, but with an sret argument.
-; TODO: The first store could be eliminated if sret is not visible on unwind.
-define void @test34_sret(ptr noalias sret(i32) %p) {
-; CHECK-LABEL: @test34_sret(
-; CHECK-NEXT:    store i32 1, ptr [[P:%.*]], align 4
+
+; Same as previous case, but with a dead_on_unwind argument.
+define void @test34_dead_on_unwind(ptr noalias dead_on_unwind %p) {
+; CHECK-LABEL: @test34_dead_on_unwind(
 ; CHECK-NEXT:    call void @unknown_func()
-; CHECK-NEXT:    store i32 0, ptr [[P]], align 4
+; CHECK-NEXT:    store i32 0, ptr [[P:%.*]], align 4
 ; CHECK-NEXT:    ret void
 ;
   store i32 1, ptr %p
diff --git a/llvm/test/Transforms/ExpandMemCmp/AArch64/memcmp.ll b/llvm/test/Transforms/ExpandMemCmp/AArch64/memcmp.ll
index 54f8c7006bb516f063dca477b60d742a8a531f2e..92439691e1873c578ceb1e4eb01590c8b02c1a59 100644
--- a/llvm/test/Transforms/ExpandMemCmp/AArch64/memcmp.ll
+++ b/llvm/test/Transforms/ExpandMemCmp/AArch64/memcmp.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3
-; RUN: opt -S -expandmemcmp -memcmp-num-loads-per-block=1 -mtriple=aarch64-unknown-unknown < %s | FileCheck %s
+; RUN: opt -S -expand-memcmp -memcmp-num-loads-per-block=1 -mtriple=aarch64-unknown-unknown < %s | FileCheck %s
+; RUN: opt -S -passes=expand-memcmp -memcmp-num-loads-per-block=1 -mtriple=aarch64-unknown-unknown < %s | FileCheck %s
 
 declare i32 @memcmp(ptr nocapture, ptr nocapture, i64)
 
diff --git a/llvm/test/Transforms/ExpandMemCmp/X86/bcmp.ll b/llvm/test/Transforms/ExpandMemCmp/X86/bcmp.ll
index 57c91f9063797f0b8a63203802ab754324fc987c..41d357728b93e7168d1a0b8fc39dea1593b9c5b4 100644
--- a/llvm/test/Transforms/ExpandMemCmp/X86/bcmp.ll
+++ b/llvm/test/Transforms/ExpandMemCmp/X86/bcmp.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S -expandmemcmp -memcmp-num-loads-per-block=1 -mtriple=x86_64-unknown-unknown -data-layout=e-m:o-i64:64-f80:128-n8:16:32:64-S128         < %s | FileCheck %s --check-prefix=X64
+; RUN: opt -S -expand-memcmp -memcmp-num-loads-per-block=1 -mtriple=x86_64-unknown-unknown -data-layout=e-m:o-i64:64-f80:128-n8:16:32:64-S128         < %s | FileCheck %s --check-prefix=X64
+; RUN: opt -S -passes=expand-memcmp -memcmp-num-loads-per-block=1 -mtriple=x86_64-unknown-unknown -data-layout=e-m:o-i64:64-f80:128-n8:16:32:64-S128         < %s | FileCheck %s --check-prefix=X64
 
 declare i32 @bcmp(ptr nocapture, ptr nocapture, i64)
 
diff --git a/llvm/test/Transforms/ExpandMemCmp/X86/memcmp-x32.ll b/llvm/test/Transforms/ExpandMemCmp/X86/memcmp-x32.ll
index f56d9688a01e12d0305879dabe5ce4f43fc2677c..d71ae8be19b668b1c1ebff707a3c69ef4fa59728 100644
--- a/llvm/test/Transforms/ExpandMemCmp/X86/memcmp-x32.ll
+++ b/llvm/test/Transforms/ExpandMemCmp/X86/memcmp-x32.ll
@@ -1,5 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S -expandmemcmp -mtriple=i686-unknown-unknown   -data-layout=e-m:o-p:32:32-f64:32:64-f80:128-n8:16:32-S128 < %s | FileCheck %s --check-prefix=X32
+; RUN: opt -S -expand-memcmp -mtriple=i686-unknown-unknown   -data-layout=e-m:o-p:32:32-f64:32:64-f80:128-n8:16:32-S128 < %s | FileCheck %s --check-prefix=X32
+; RUN: opt -S -passes=expand-memcmp -mtriple=i686-unknown-unknown   -data-layout=e-m:o-p:32:32-f64:32:64-f80:128-n8:16:32-S128 < %s | FileCheck %s --check-prefix=X32
 
 declare i32 @memcmp(ptr nocapture, ptr nocapture, i32)
 
diff --git a/llvm/test/Transforms/ExpandMemCmp/X86/memcmp.ll b/llvm/test/Transforms/ExpandMemCmp/X86/memcmp.ll
index 2594f53971393d4df5fc549e072cd634b0c05d0a..f686e29975564fd73434bc4981b0e9ee10f2e969 100644
--- a/llvm/test/Transforms/ExpandMemCmp/X86/memcmp.ll
+++ b/llvm/test/Transforms/ExpandMemCmp/X86/memcmp.ll
@@ -1,6 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
-; RUN: opt -S -expandmemcmp -memcmp-num-loads-per-block=1 -mtriple=x86_64-unknown-unknown -data-layout=e-m:o-i64:64-f80:128-n8:16:32:64-S128         < %s | FileCheck %s --check-prefix=X64 --check-prefix=X64_1LD
-; RUN: opt -S -expandmemcmp -memcmp-num-loads-per-block=2 -mtriple=x86_64-unknown-unknown -data-layout=e-m:o-i64:64-f80:128-n8:16:32:64-S128         < %s | FileCheck %s --check-prefix=X64 --check-prefix=X64_2LD
+; RUN: opt -S -expand-memcmp -memcmp-num-loads-per-block=1 -mtriple=x86_64-unknown-unknown -data-layout=e-m:o-i64:64-f80:128-n8:16:32:64-S128         < %s | FileCheck %s --check-prefix=X64 --check-prefix=X64_1LD
+; RUN: opt -S -expand-memcmp -memcmp-num-loads-per-block=2 -mtriple=x86_64-unknown-unknown -data-layout=e-m:o-i64:64-f80:128-n8:16:32:64-S128         < %s | FileCheck %s --check-prefix=X64 --check-prefix=X64_2LD
+; RUN: opt -S -passes=expand-memcmp -memcmp-num-loads-per-block=1 -mtriple=x86_64-unknown-unknown -data-layout=e-m:o-i64:64-f80:128-n8:16:32:64-S128         < %s | FileCheck %s --check-prefix=X64 --check-prefix=X64_1LD
+; RUN: opt -S -passes=expand-memcmp -memcmp-num-loads-per-block=2 -mtriple=x86_64-unknown-unknown -data-layout=e-m:o-i64:64-f80:128-n8:16:32:64-S128         < %s | FileCheck %s --check-prefix=X64 --check-prefix=X64_2LD
 
 declare i32 @memcmp(ptr nocapture, ptr nocapture, i64)
 
diff --git a/llvm/test/Transforms/IndirectBrExpand/basic.ll b/llvm/test/Transforms/IndirectBrExpand/basic.ll
index c90ec4a3c48b2cf1296cd6c7874bd47d9604e576..c2c73100543f1fef81e2351a865a907e7e9f7c8b 100644
--- a/llvm/test/Transforms/IndirectBrExpand/basic.ll
+++ b/llvm/test/Transforms/IndirectBrExpand/basic.ll
@@ -1,4 +1,5 @@
 ; RUN: opt < %s -indirectbr-expand -S | FileCheck %s
+; RUN: opt < %s -passes=indirectbr-expand -S | FileCheck %s
 ;
 ; REQUIRES: x86-registered-target
 
diff --git a/llvm/test/Transforms/InstCombine/and.ll b/llvm/test/Transforms/InstCombine/and.ll
index 79857f3efbc18bc5f140e5c2648d6cb817902fe0..2e37fee07cc47bf8478b5d1a25c1234efbeb0c0e 100644
--- a/llvm/test/Transforms/InstCombine/and.ll
+++ b/llvm/test/Transforms/InstCombine/and.ll
@@ -395,8 +395,8 @@ define i8 @test27(i8 %A) {
 
 define i32 @ashr_lowmask(i32 %x) {
 ; CHECK-LABEL: @ashr_lowmask(
-; CHECK-NEXT:    [[TMP1:%.*]] = lshr i32 [[X:%.*]], 24
-; CHECK-NEXT:    ret i32 [[TMP1]]
+; CHECK-NEXT:    [[A:%.*]] = lshr i32 [[X:%.*]], 24
+; CHECK-NEXT:    ret i32 [[A]]
 ;
   %a = ashr i32 %x, 24
   %r = and i32 %a, 255
@@ -2711,3 +2711,60 @@ define i32 @canonicalize_and_sub_power2_or_zero_multiuse_nofold(i32 %x, i32 %y)
   %and = and i32 %val, %p2
   ret i32 %and
 }
+
+define i32 @add_constant_equal_with_the_top_bit_of_demandedbits_pass(i32 %x) {
+; CHECK-LABEL: @add_constant_equal_with_the_top_bit_of_demandedbits_pass(
+; CHECK-NEXT:    [[TMP1:%.*]] = and i32 [[X:%.*]], 24
+; CHECK-NEXT:    [[AND:%.*]] = xor i32 [[TMP1]], 16
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+  %add = add i32 %x, 16
+  %and = and i32 %add, 24
+  ret i32 %and
+}
+
+define <2 x i16> @add_constant_equal_with_the_top_bit_of_demandedbits_pass_vector(<2 x i16> %x) {
+; CHECK-LABEL: @add_constant_equal_with_the_top_bit_of_demandedbits_pass_vector(
+; CHECK-NEXT:    [[TMP1:%.*]] = and <2 x i16> [[X:%.*]], <i16 24, i16 24>
+; CHECK-NEXT:    [[AND:%.*]] = xor <2 x i16> [[TMP1]], <i16 16, i16 16>
+; CHECK-NEXT:    ret <2 x i16> [[AND]]
+;
+  %add = add <2 x i16> %x, <i16 16, i16 16>
+  %and = and <2 x i16> %add, <i16 24, i16 24>
+  ret <2 x i16> %and
+}
+
+define i32 @add_constant_equal_with_the_top_bit_of_demandedbits_fail1(i32 %x) {
+; CHECK-LABEL: @add_constant_equal_with_the_top_bit_of_demandedbits_fail1(
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[X:%.*]], 8
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[ADD]], 24
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+  %add = add i32 %x, 8
+  %and = and i32 %add, 24
+  ret i32 %and
+}
+
+define i32 @add_constant_equal_with_the_top_bit_of_demandedbits_fail2(i32 %x) {
+; CHECK-LABEL: @add_constant_equal_with_the_top_bit_of_demandedbits_fail2(
+; CHECK-NEXT:    [[ADD:%.*]] = add i32 [[X:%.*]], 24
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[ADD]], 24
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+  %add = add i32 %x, 24
+  %and = and i32 %add, 24
+  ret i32 %and
+}
+
+define i32 @add_constant_equal_with_the_top_bit_of_demandedbits_insertpt(i32 %x, i32 %y) {
+; CHECK-LABEL: @add_constant_equal_with_the_top_bit_of_demandedbits_insertpt(
+; CHECK-NEXT:    [[TMP1:%.*]] = xor i32 [[X:%.*]], 16
+; CHECK-NEXT:    [[OR:%.*]] = or i32 [[TMP1]], [[Y:%.*]]
+; CHECK-NEXT:    [[AND:%.*]] = and i32 [[OR]], 24
+; CHECK-NEXT:    ret i32 [[AND]]
+;
+  %add = add i32 %x, 16
+  %or = or i32 %add, %y
+  %and = and i32 %or, 24
+  ret i32 %and
+}
diff --git a/llvm/test/Transforms/InstCombine/commutative-operation-over-selects.ll b/llvm/test/Transforms/InstCombine/commutative-operation-over-selects.ll
new file mode 100644
index 0000000000000000000000000000000000000000..a992733ebfe074fa221098fb008aa4945ee5dbd6
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/commutative-operation-over-selects.ll
@@ -0,0 +1,563 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt -passes=instcombine -S < %s | FileCheck %s
+
+declare float @llvm.maxnum.f32(float %a, float %b)
+declare float @llvm.minnum.f32(float %a, float %b)
+declare float @llvm.maximum.f32(float %a, float %b)
+declare float @llvm.minimum.f32(float %a, float %b)
+declare i32 @llvm.smax.i32(i32 %a, i32 %b)
+declare i32 @llvm.smin.i32(i32 %a, i32 %b)
+declare i32 @llvm.umax.i32(i32 %a, i32 %b)
+declare i32 @llvm.umin.i32(i32 %a, i32 %b)
+declare i16 @llvm.sadd.sat.i16(i16 %a, i16 %b)
+declare i16 @llvm.uadd.sat.i16(i16 %a, i16 %b)
+declare {i16, i1} @llvm.sadd.with.overflow.i16(i16 %a, i16 %b)
+declare {i16, i1} @llvm.uadd.with.overflow.i16(i16 %a, i16 %b)
+declare {i16, i1} @llvm.smul.with.overflow.i16(i16 %a, i16 %b)
+declare {i16, i1} @llvm.umul.with.overflow.i16(i16 %a, i16 %b)
+declare i16 @llvm.smul.fix.i16(i16 %a, i16 %b, i32 %scale)
+declare i16 @llvm.umul.fix.i16(i16 %a, i16 %b, i32 %scale)
+declare i16 @llvm.smul.fix.sat.i16(i16 %a, i16 %b, i32 %scale)
+declare i16 @llvm.umul.fix.sat.i16(i16 %a, i16 %b, i32 %scale)
+declare float @llvm.fma.f32(float %a, float %b, float %c)
+declare float @llvm.fmuladd.f32(float %a, float %b, float %c)
+
+define i8 @fold_select_mul(i1 %c, i8 %a, i8 %b) {
+; CHECK-LABEL: define i8 @fold_select_mul(
+; CHECK-SAME: i1 [[C:%.*]], i8 [[A:%.*]], i8 [[B:%.*]]) {
+; CHECK-NEXT:    [[RET:%.*]] = mul i8 [[B]], [[A]]
+; CHECK-NEXT:    ret i8 [[RET]]
+;
+  %s0 = select i1 %c, i8 %a, i8 %b
+  %s1 = select i1 %c, i8 %b, i8 %a
+  %ret = mul i8 %s1, %s0
+  ret i8 %ret
+}
+
+define i8 @fold_select_mul_nsw_nuw_preserve(i1 %c, i8 %a, i8 %b) {
+; CHECK-LABEL: define i8 @fold_select_mul_nsw_nuw_preserve(
+; CHECK-SAME: i1 [[C:%.*]], i8 [[A:%.*]], i8 [[B:%.*]]) {
+; CHECK-NEXT:    [[RET:%.*]] = mul nuw nsw i8 [[B]], [[A]]
+; CHECK-NEXT:    ret i8 [[RET]]
+;
+  %s0 = select i1 %c, i8 %a, i8 %b
+  %s1 = select i1 %c, i8 %b, i8 %a
+  %ret = mul nsw nuw i8 %s1, %s0
+  ret i8 %ret
+}
+
+define <2 x i4> @fold_select_mul_vec2(i1 %c, <2 x i4> %a, <2 x i4> %b) {
+; CHECK-LABEL: define <2 x i4> @fold_select_mul_vec2(
+; CHECK-SAME: i1 [[C:%.*]], <2 x i4> [[A:%.*]], <2 x i4> [[B:%.*]]) {
+; CHECK-NEXT:    [[RET:%.*]] = mul <2 x i4> [[B]], [[A]]
+; CHECK-NEXT:    ret <2 x i4> [[RET]]
+;
+  %s0 = select i1 %c, <2 x i4> %a, <2 x i4> %b
+  %s1 = select i1 %c, <2 x i4> %b, <2 x i4> %a
+  %ret = mul <2 x i4> %s1, %s0
+  ret <2 x i4> %ret
+}
+
+define i8 @fold_select_add(i1 %c, i8 %a, i8 %b) {
+; CHECK-LABEL: define i8 @fold_select_add(
+; CHECK-SAME: i1 [[C:%.*]], i8 [[A:%.*]], i8 [[B:%.*]]) {
+; CHECK-NEXT:    [[RET:%.*]] = add i8 [[B]], [[A]]
+; CHECK-NEXT:    ret i8 [[RET]]
+;
+  %s0 = select i1 %c, i8 %a, i8 %b
+  %s1 = select i1 %c, i8 %b, i8 %a
+  %ret = add i8 %s1, %s0
+  ret i8 %ret
+}
+
+define <2 x i4> @fold_select_add_vec2(i1 %c, <2 x i4> %a, <2 x i4> %b) {
+; CHECK-LABEL: define <2 x i4> @fold_select_add_vec2(
+; CHECK-SAME: i1 [[C:%.*]], <2 x i4> [[A:%.*]], <2 x i4> [[B:%.*]]) {
+; CHECK-NEXT:    [[RET:%.*]] = add <2 x i4> [[B]], [[A]]
+; CHECK-NEXT:    ret <2 x i4> [[RET]]
+;
+  %s0 = select i1 %c, <2 x i4> %a, <2 x i4> %b
+  %s1 = select i1 %c, <2 x i4> %b, <2 x i4> %a
+  %ret = add <2 x i4> %s1, %s0
+  ret <2 x i4> %ret
+}
+
+define i8 @fold_select_and(i1 %c, i8 %a, i8 %b) {
+; CHECK-LABEL: define i8 @fold_select_and(
+; CHECK-SAME: i1 [[C:%.*]], i8 [[A:%.*]], i8 [[B:%.*]]) {
+; CHECK-NEXT:    [[RET:%.*]] = and i8 [[B]], [[A]]
+; CHECK-NEXT:    ret i8 [[RET]]
+;
+  %s0 = select i1 %c, i8 %a, i8 %b
+  %s1 = select i1 %c, i8 %b, i8 %a
+  %ret = and i8 %s1, %s0
+  ret i8 %ret
+}
+
+define <2 x i4> @fold_select_and_vec2(i1 %c, <2 x i4> %a, <2 x i4> %b) {
+; CHECK-LABEL: define <2 x i4> @fold_select_and_vec2(
+; CHECK-SAME: i1 [[C:%.*]], <2 x i4> [[A:%.*]], <2 x i4> [[B:%.*]]) {
+; CHECK-NEXT:    [[RET:%.*]] = and <2 x i4> [[B]], [[A]]
+; CHECK-NEXT:    ret <2 x i4> [[RET]]
+;
+  %s0 = select i1 %c, <2 x i4> %a, <2 x i4> %b
+  %s1 = select i1 %c, <2 x i4> %b, <2 x i4> %a
+  %ret = and <2 x i4> %s1, %s0
+  ret <2 x i4> %ret
+}
+
+define i8 @fold_select_or(i1 %c, i8 %a, i8 %b) {
+; CHECK-LABEL: define i8 @fold_select_or(
+; CHECK-SAME: i1 [[C:%.*]], i8 [[A:%.*]], i8 [[B:%.*]]) {
+; CHECK-NEXT:    [[RET:%.*]] = or i8 [[B]], [[A]]
+; CHECK-NEXT:    ret i8 [[RET]]
+;
+  %s0 = select i1 %c, i8 %a, i8 %b
+  %s1 = select i1 %c, i8 %b, i8 %a
+  %ret = or i8 %s1, %s0
+  ret i8 %ret
+}
+
+define <2 x i4> @fold_select_or_vec2(i1 %c, <2 x i4> %a, <2 x i4> %b) {
+; CHECK-LABEL: define <2 x i4> @fold_select_or_vec2(
+; CHECK-SAME: i1 [[C:%.*]], <2 x i4> [[A:%.*]], <2 x i4> [[B:%.*]]) {
+; CHECK-NEXT:    [[RET:%.*]] = or <2 x i4> [[B]], [[A]]
+; CHECK-NEXT:    ret <2 x i4> [[RET]]
+;
+  %s0 = select i1 %c, <2 x i4> %a, <2 x i4> %b
+  %s1 = select i1 %c, <2 x i4> %b, <2 x i4> %a
+  %ret = or <2 x i4> %s1, %s0
+  ret <2 x i4> %ret
+}
+
+define i8 @fold_select_xor(i1 %c, i8 %a, i8 %b) {
+; CHECK-LABEL: define i8 @fold_select_xor(
+; CHECK-SAME: i1 [[C:%.*]], i8 [[A:%.*]], i8 [[B:%.*]]) {
+; CHECK-NEXT:    [[RET:%.*]] = xor i8 [[B]], [[A]]
+; CHECK-NEXT:    ret i8 [[RET]]
+;
+  %s0 = select i1 %c, i8 %a, i8 %b
+  %s1 = select i1 %c, i8 %b, i8 %a
+  %ret = xor i8 %s1, %s0
+  ret i8 %ret
+}
+
+define <2 x i4> @fold_select_xor_vec2(i1 %c, <2 x i4> %a, <2 x i4> %b) {
+; CHECK-LABEL: define <2 x i4> @fold_select_xor_vec2(
+; CHECK-SAME: i1 [[C:%.*]], <2 x i4> [[A:%.*]], <2 x i4> [[B:%.*]]) {
+; CHECK-NEXT:    [[RET:%.*]] = xor <2 x i4> [[B]], [[A]]
+; CHECK-NEXT:    ret <2 x i4> [[RET]]
+;
+  %s0 = select i1 %c, <2 x i4> %a, <2 x i4> %b
+  %s1 = select i1 %c, <2 x i4> %b, <2 x i4> %a
+  %ret = xor <2 x i4> %s1, %s0
+  ret <2 x i4> %ret
+}
+
+define float @fold_select_fadd(i1 %c, float %a, float %b) {
+; CHECK-LABEL: define float @fold_select_fadd(
+; CHECK-SAME: i1 [[C:%.*]], float [[A:%.*]], float [[B:%.*]]) {
+; CHECK-NEXT:    [[RET:%.*]] = fadd float [[B]], [[A]]
+; CHECK-NEXT:    ret float [[RET]]
+;
+  %s0 = select i1 %c, float %a, float %b
+  %s1 = select i1 %c, float %b, float %a
+  %ret = fadd float %s1, %s0
+  ret float %ret
+}
+
+define float @fold_select_fadd_nnan(i1 %c, float %a, float %b) {
+; CHECK-LABEL: define float @fold_select_fadd_nnan(
+; CHECK-SAME: i1 [[C:%.*]], float [[A:%.*]], float [[B:%.*]]) {
+; CHECK-NEXT:    [[RET:%.*]] = fadd nnan float [[B]], [[A]]
+; CHECK-NEXT:    ret float [[RET]]
+;
+  %s0 = select i1 %c, float %a, float %b
+  %s1 = select i1 %c, float %b, float %a
+  %ret = fadd nnan float %s1, %s0
+  ret float %ret
+}
+
+define <2 x float> @fold_select_fadd_vec2(i1 %c, <2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: define <2 x float> @fold_select_fadd_vec2(
+; CHECK-SAME: i1 [[C:%.*]], <2 x float> [[A:%.*]], <2 x float> [[B:%.*]]) {
+; CHECK-NEXT:    [[RET:%.*]] = fadd <2 x float> [[B]], [[A]]
+; CHECK-NEXT:    ret <2 x float> [[RET]]
+;
+  %s0 = select i1 %c, <2 x float> %a, <2 x float> %b
+  %s1 = select i1 %c, <2 x float> %b, <2 x float> %a
+  %ret = fadd <2 x float> %s1, %s0
+  ret <2 x float> %ret
+}
+
+define float @fold_select_fmul(i1 %c, float %a, float %b) {
+; CHECK-LABEL: define float @fold_select_fmul(
+; CHECK-SAME: i1 [[C:%.*]], float [[A:%.*]], float [[B:%.*]]) {
+; CHECK-NEXT:    [[RET:%.*]] = fmul float [[B]], [[A]]
+; CHECK-NEXT:    ret float [[RET]]
+;
+  %s0 = select i1 %c, float %a, float %b
+  %s1 = select i1 %c, float %b, float %a
+  %ret = fmul float %s1, %s0
+  ret float %ret
+}
+
+define <2 x float> @fold_select_fmul_vec2(i1 %c, <2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: define <2 x float> @fold_select_fmul_vec2(
+; CHECK-SAME: i1 [[C:%.*]], <2 x float> [[A:%.*]], <2 x float> [[B:%.*]]) {
+; CHECK-NEXT:    [[RET:%.*]] = fmul <2 x float> [[B]], [[A]]
+; CHECK-NEXT:    ret <2 x float> [[RET]]
+;
+  %s0 = select i1 %c, <2 x float> %a, <2 x float> %b
+  %s1 = select i1 %c, <2 x float> %b, <2 x float> %a
+  %ret = fmul <2 x float> %s1, %s0
+  ret <2 x float> %ret
+}
+
+;
+
+define float @fold_select_maxnum(i1 %c, float %a, float %b) {
+; CHECK-LABEL: define float @fold_select_maxnum(
+; CHECK-SAME: i1 [[C:%.*]], float [[A:%.*]], float [[B:%.*]]) {
+; CHECK-NEXT:    [[RET:%.*]] = call float @llvm.maxnum.f32(float [[B]], float [[A]])
+; CHECK-NEXT:    ret float [[RET]]
+;
+  %s0 = select i1 %c, float %a, float %b
+  %s1 = select i1 %c, float %b, float %a
+  %ret = call float @llvm.maxnum.f32(float %s1, float %s0)
+  ret float %ret
+}
+
+define float @fold_select_minnum(i1 %c, float %a, float %b) {
+; CHECK-LABEL: define float @fold_select_minnum(
+; CHECK-SAME: i1 [[C:%.*]], float [[A:%.*]], float [[B:%.*]]) {
+; CHECK-NEXT:    [[RET:%.*]] = call float @llvm.minnum.f32(float [[B]], float [[A]])
+; CHECK-NEXT:    ret float [[RET]]
+;
+  %s0 = select i1 %c, float %a, float %b
+  %s1 = select i1 %c, float %b, float %a
+  %ret = call float @llvm.minnum.f32(float %s1, float %s0)
+  ret float %ret
+}
+
+define float @fold_select_maximum(i1 %c, float %a, float %b) {
+; CHECK-LABEL: define float @fold_select_maximum(
+; CHECK-SAME: i1 [[C:%.*]], float [[A:%.*]], float [[B:%.*]]) {
+; CHECK-NEXT:    [[RET:%.*]] = call float @llvm.maximum.f32(float [[B]], float [[A]])
+; CHECK-NEXT:    ret float [[RET]]
+;
+  %s0 = select i1 %c, float %a, float %b
+  %s1 = select i1 %c, float %b, float %a
+  %ret = call float @llvm.maximum.f32(float %s1, float %s0)
+  ret float %ret
+}
+
+define float @fold_select_minimum(i1 %c, float %a, float %b) {
+; CHECK-LABEL: define float @fold_select_minimum(
+; CHECK-SAME: i1 [[C:%.*]], float [[A:%.*]], float [[B:%.*]]) {
+; CHECK-NEXT:    [[RET:%.*]] = call float @llvm.minimum.f32(float [[B]], float [[A]])
+; CHECK-NEXT:    ret float [[RET]]
+;
+  %s0 = select i1 %c, float %a, float %b
+  %s1 = select i1 %c, float %b, float %a
+  %ret = call float @llvm.minimum.f32(float %s1, float %s0)
+  ret float %ret
+}
+
+define i32 @fold_select_smax(i1 %c, i32 %a, i32 %b) {
+; CHECK-LABEL: define i32 @fold_select_smax(
+; CHECK-SAME: i1 [[C:%.*]], i32 [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    [[RET:%.*]] = call i32 @llvm.smax.i32(i32 [[B]], i32 [[A]])
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %s0 = select i1 %c, i32 %a, i32 %b
+  %s1 = select i1 %c, i32 %b, i32 %a
+  %ret = call i32 @llvm.smax.i32(i32 %s1, i32 %s0)
+  ret i32 %ret
+}
+
+define i32 @fold_select_smin(i1 %c, i32 %a, i32 %b) {
+; CHECK-LABEL: define i32 @fold_select_smin(
+; CHECK-SAME: i1 [[C:%.*]], i32 [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    [[RET:%.*]] = call i32 @llvm.smin.i32(i32 [[B]], i32 [[A]])
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %s0 = select i1 %c, i32 %a, i32 %b
+  %s1 = select i1 %c, i32 %b, i32 %a
+  %ret = call i32 @llvm.smin.i32(i32 %s1, i32 %s0)
+  ret i32 %ret
+}
+
+define i32 @fold_select_umax(i1 %c, i32 %a, i32 %b) {
+; CHECK-LABEL: define i32 @fold_select_umax(
+; CHECK-SAME: i1 [[C:%.*]], i32 [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    [[RET:%.*]] = call i32 @llvm.umax.i32(i32 [[B]], i32 [[A]])
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %s0 = select i1 %c, i32 %a, i32 %b
+  %s1 = select i1 %c, i32 %b, i32 %a
+  %ret = call i32 @llvm.umax.i32(i32 %s1, i32 %s0)
+  ret i32 %ret
+}
+
+define i32 @fold_select_umin(i1 %c, i32 %a, i32 %b) {
+; CHECK-LABEL: define i32 @fold_select_umin(
+; CHECK-SAME: i1 [[C:%.*]], i32 [[A:%.*]], i32 [[B:%.*]]) {
+; CHECK-NEXT:    [[RET:%.*]] = call i32 @llvm.umin.i32(i32 [[B]], i32 [[A]])
+; CHECK-NEXT:    ret i32 [[RET]]
+;
+  %s0 = select i1 %c, i32 %a, i32 %b
+  %s1 = select i1 %c, i32 %b, i32 %a
+  %ret = call i32 @llvm.umin.i32(i32 %s1, i32 %s0)
+  ret i32 %ret
+}
+
+define i16 @fold_select_sadd_sat(i1 %c, i16 %a, i16 %b) {
+; CHECK-LABEL: define i16 @fold_select_sadd_sat(
+; CHECK-SAME: i1 [[C:%.*]], i16 [[A:%.*]], i16 [[B:%.*]]) {
+; CHECK-NEXT:    [[RET:%.*]] = call i16 @llvm.sadd.sat.i16(i16 [[B]], i16 [[A]])
+; CHECK-NEXT:    ret i16 [[RET]]
+;
+  %s0 = select i1 %c, i16 %a, i16 %b
+  %s1 = select i1 %c, i16 %b, i16 %a
+  %ret = call i16 @llvm.sadd.sat.i16(i16 %s1, i16 %s0)
+  ret i16 %ret
+}
+
+define i16 @fold_select_uadd_sat(i1 %c, i16 %a, i16 %b) {
+; CHECK-LABEL: define i16 @fold_select_uadd_sat(
+; CHECK-SAME: i1 [[C:%.*]], i16 [[A:%.*]], i16 [[B:%.*]]) {
+; CHECK-NEXT:    [[RET:%.*]] = call i16 @llvm.uadd.sat.i16(i16 [[B]], i16 [[A]])
+; CHECK-NEXT:    ret i16 [[RET]]
+;
+  %s0 = select i1 %c, i16 %a, i16 %b
+  %s1 = select i1 %c, i16 %b, i16 %a
+  %ret = call i16 @llvm.uadd.sat.i16(i16 %s1, i16 %s0)
+  ret i16 %ret
+}
+
+define i16 @fold_select_sadd_with_overflow(i1 %c, i16 %a, i16 %b) {
+; CHECK-LABEL: define i16 @fold_select_sadd_with_overflow(
+; CHECK-SAME: i1 [[C:%.*]], i16 [[A:%.*]], i16 [[B:%.*]]) {
+; CHECK-NEXT:    [[RES:%.*]] = call { i16, i1 } @llvm.sadd.with.overflow.i16(i16 [[B]], i16 [[A]])
+; CHECK-NEXT:    [[OBIT:%.*]] = extractvalue { i16, i1 } [[RES]], 1
+; CHECK-NEXT:    br i1 [[OBIT]], label [[OVERFLOW:%.*]], label [[NORMAL:%.*]]
+; CHECK:       overflow:
+; CHECK-NEXT:    ret i16 0
+; CHECK:       normal:
+; CHECK-NEXT:    [[SUM:%.*]] = extractvalue { i16, i1 } [[RES]], 0
+; CHECK-NEXT:    ret i16 [[SUM]]
+;
+  %s0 = select i1 %c, i16 %a, i16 %b
+  %s1 = select i1 %c, i16 %b, i16 %a
+  %res = call {i16, i1} @llvm.sadd.with.overflow.i16(i16 %s1, i16 %s0)
+  %obit = extractvalue {i16, i1} %res, 1
+  br i1 %obit, label %overflow, label %normal
+overflow:
+  ret i16 0
+normal:
+  %sum = extractvalue {i16, i1} %res, 0
+  ret i16 %sum
+}
+
+define i16 @fold_select_uadd_with_overflow(i1 %c, i16 %a, i16 %b) {
+; CHECK-LABEL: define i16 @fold_select_uadd_with_overflow(
+; CHECK-SAME: i1 [[C:%.*]], i16 [[A:%.*]], i16 [[B:%.*]]) {
+; CHECK-NEXT:    [[RES:%.*]] = call { i16, i1 } @llvm.uadd.with.overflow.i16(i16 [[B]], i16 [[A]])
+; CHECK-NEXT:    [[OBIT:%.*]] = extractvalue { i16, i1 } [[RES]], 1
+; CHECK-NEXT:    br i1 [[OBIT]], label [[OVERFLOW:%.*]], label [[NORMAL:%.*]]
+; CHECK:       overflow:
+; CHECK-NEXT:    ret i16 0
+; CHECK:       normal:
+; CHECK-NEXT:    [[SUM:%.*]] = extractvalue { i16, i1 } [[RES]], 0
+; CHECK-NEXT:    ret i16 [[SUM]]
+;
+  %s0 = select i1 %c, i16 %a, i16 %b
+  %s1 = select i1 %c, i16 %b, i16 %a
+  %res = call {i16, i1} @llvm.uadd.with.overflow.i16(i16 %s1, i16 %s0)
+  %obit = extractvalue {i16, i1} %res, 1
+  br i1 %obit, label %overflow, label %normal
+overflow:
+  ret i16 0
+normal:
+  %sum = extractvalue {i16, i1} %res, 0
+  ret i16 %sum
+}
+
+define i16 @fold_select_smul_with_overflow(i1 %c, i16 %a, i16 %b) {
+; CHECK-LABEL: define i16 @fold_select_smul_with_overflow(
+; CHECK-SAME: i1 [[C:%.*]], i16 [[A:%.*]], i16 [[B:%.*]]) {
+; CHECK-NEXT:    [[RES:%.*]] = call { i16, i1 } @llvm.smul.with.overflow.i16(i16 [[B]], i16 [[A]])
+; CHECK-NEXT:    [[OBIT:%.*]] = extractvalue { i16, i1 } [[RES]], 1
+; CHECK-NEXT:    br i1 [[OBIT]], label [[OVERFLOW:%.*]], label [[NORMAL:%.*]]
+; CHECK:       overflow:
+; CHECK-NEXT:    ret i16 0
+; CHECK:       normal:
+; CHECK-NEXT:    [[MUL:%.*]] = extractvalue { i16, i1 } [[RES]], 0
+; CHECK-NEXT:    ret i16 [[MUL]]
+;
+  %s0 = select i1 %c, i16 %a, i16 %b
+  %s1 = select i1 %c, i16 %b, i16 %a
+  %res = call {i16, i1} @llvm.smul.with.overflow.i16(i16 %s1, i16 %s0)
+  %obit = extractvalue {i16, i1} %res, 1
+  br i1 %obit, label %overflow, label %normal
+overflow:
+  ret i16 0
+normal:
+  %mul = extractvalue {i16, i1} %res, 0
+  ret i16 %mul
+}
+
+define i16 @fold_select_umul_with_overflow(i1 %c, i16 %a, i16 %b) {
+; CHECK-LABEL: define i16 @fold_select_umul_with_overflow(
+; CHECK-SAME: i1 [[C:%.*]], i16 [[A:%.*]], i16 [[B:%.*]]) {
+; CHECK-NEXT:    [[RES:%.*]] = call { i16, i1 } @llvm.umul.with.overflow.i16(i16 [[B]], i16 [[A]])
+; CHECK-NEXT:    [[OBIT:%.*]] = extractvalue { i16, i1 } [[RES]], 1
+; CHECK-NEXT:    br i1 [[OBIT]], label [[OVERFLOW:%.*]], label [[NORMAL:%.*]]
+; CHECK:       overflow:
+; CHECK-NEXT:    ret i16 0
+; CHECK:       normal:
+; CHECK-NEXT:    [[MUL:%.*]] = extractvalue { i16, i1 } [[RES]], 0
+; CHECK-NEXT:    ret i16 [[MUL]]
+;
+  %s0 = select i1 %c, i16 %a, i16 %b
+  %s1 = select i1 %c, i16 %b, i16 %a
+  %res = call {i16, i1} @llvm.umul.with.overflow.i16(i16 %s1, i16 %s0)
+  %obit = extractvalue {i16, i1} %res, 1
+  br i1 %obit, label %overflow, label %normal
+overflow:
+  ret i16 0
+normal:
+  %mul = extractvalue {i16, i1} %res, 0
+  ret i16 %mul
+}
+
+define i16 @fold_select_smul_fix(i1 %c, i16 %a, i16 %b, i32 %y) {
+; CHECK-LABEL: define i16 @fold_select_smul_fix(
+; CHECK-SAME: i1 [[C:%.*]], i16 [[A:%.*]], i16 [[B:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[RET:%.*]] = call i16 @llvm.smul.fix.i16(i16 [[B]], i16 [[A]], i32 1)
+; CHECK-NEXT:    ret i16 [[RET]]
+;
+  %s0 = select i1 %c, i16 %a, i16 %b
+  %s1 = select i1 %c, i16 %b, i16 %a
+  %ret = call i16 @llvm.smul.fix.i16(i16 %s1, i16 %s0, i32 1)
+  ret i16 %ret
+}
+
+define i16 @fold_select_umul_fix(i1 %c, i16 %a, i16 %b, i32 %y) {
+; CHECK-LABEL: define i16 @fold_select_umul_fix(
+; CHECK-SAME: i1 [[C:%.*]], i16 [[A:%.*]], i16 [[B:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[RET:%.*]] = call i16 @llvm.umul.fix.i16(i16 [[B]], i16 [[A]], i32 1)
+; CHECK-NEXT:    ret i16 [[RET]]
+;
+  %s0 = select i1 %c, i16 %a, i16 %b
+  %s1 = select i1 %c, i16 %b, i16 %a
+  %ret = call i16 @llvm.umul.fix.i16(i16 %s1, i16 %s0, i32 1)
+  ret i16 %ret
+}
+
+define i16 @fold_select_smul_fix_sat(i1 %c, i16 %a, i16 %b, i32 %y) {
+; CHECK-LABEL: define i16 @fold_select_smul_fix_sat(
+; CHECK-SAME: i1 [[C:%.*]], i16 [[A:%.*]], i16 [[B:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[RET:%.*]] = call i16 @llvm.smul.fix.sat.i16(i16 [[B]], i16 [[A]], i32 1)
+; CHECK-NEXT:    ret i16 [[RET]]
+;
+  %s0 = select i1 %c, i16 %a, i16 %b
+  %s1 = select i1 %c, i16 %b, i16 %a
+  %ret = call i16 @llvm.smul.fix.sat.i16(i16 %s1, i16 %s0, i32 1)
+  ret i16 %ret
+}
+
+define i16 @fold_select_umul_fix_sat(i1 %c, i16 %a, i16 %b, i32 %y) {
+; CHECK-LABEL: define i16 @fold_select_umul_fix_sat(
+; CHECK-SAME: i1 [[C:%.*]], i16 [[A:%.*]], i16 [[B:%.*]], i32 [[Y:%.*]]) {
+; CHECK-NEXT:    [[RET:%.*]] = call i16 @llvm.umul.fix.sat.i16(i16 [[B]], i16 [[A]], i32 1)
+; CHECK-NEXT:    ret i16 [[RET]]
+;
+  %s0 = select i1 %c, i16 %a, i16 %b
+  %s1 = select i1 %c, i16 %b, i16 %a
+  %ret = call i16 @llvm.umul.fix.sat.i16(i16 %s1, i16 %s0, i32 1)
+  ret i16 %ret
+}
+
+define float @fold_select_fma(i1 %c, float %a, float %b, float %y) {
+; CHECK-LABEL: define float @fold_select_fma(
+; CHECK-SAME: i1 [[C:%.*]], float [[A:%.*]], float [[B:%.*]], float [[Y:%.*]]) {
+; CHECK-NEXT:    [[RET:%.*]] = call float @llvm.fma.f32(float [[B]], float [[A]], float [[Y]])
+; CHECK-NEXT:    ret float [[RET]]
+;
+  %s0 = select i1 %c, float %a, float %b
+  %s1 = select i1 %c, float %b, float %a
+  %ret = call float @llvm.fma.f32(float %s1, float %s0, float %y)
+  ret float %ret
+}
+
+define float @fold_select_fmuladd(i1 %c, float %a, float %b, float %y) {
+; CHECK-LABEL: define float @fold_select_fmuladd(
+; CHECK-SAME: i1 [[C:%.*]], float [[A:%.*]], float [[B:%.*]], float [[Y:%.*]]) {
+; CHECK-NEXT:    [[RET:%.*]] = call float @llvm.fmuladd.f32(float [[B]], float [[A]], float [[Y]])
+; CHECK-NEXT:    ret float [[RET]]
+;
+  %s0 = select i1 %c, float %a, float %b
+  %s1 = select i1 %c, float %b, float %a
+  %ret = call float @llvm.fmuladd.f32(float %s1, float %s0, float %y)
+  ret float %ret
+}
+
+;negative tests:
+
+define i8 @fold_select_unmatch_mul_neg(i1 %c, i1 %c1, i8 %a, i8 %b) {
+; CHECK-LABEL: define i8 @fold_select_unmatch_mul_neg(
+; CHECK-SAME: i1 [[C:%.*]], i1 [[C1:%.*]], i8 [[A:%.*]], i8 [[B:%.*]]) {
+; CHECK-NEXT:    [[S0:%.*]] = select i1 [[C]], i8 [[A]], i8 [[B]]
+; CHECK-NEXT:    [[S1:%.*]] = select i1 [[C1]], i8 [[B]], i8 [[A]]
+; CHECK-NEXT:    [[RET:%.*]] = mul i8 [[S1]], [[S0]]
+; CHECK-NEXT:    ret i8 [[RET]]
+;
+  %s0 = select i1 %c, i8 %a, i8 %b
+  %s1 = select i1 %c1, i8 %b, i8 %a
+  %ret = mul i8 %s1, %s0
+  ret i8 %ret
+}
+
+define i8 @fold_select_sub_neg(i1 %c, i8 %a, i8 %b) {
+; CHECK-LABEL: define i8 @fold_select_sub_neg(
+; CHECK-SAME: i1 [[C:%.*]], i8 [[A:%.*]], i8 [[B:%.*]]) {
+; CHECK-NEXT:    [[S0:%.*]] = select i1 [[C]], i8 [[A]], i8 [[B]]
+; CHECK-NEXT:    [[S1:%.*]] = select i1 [[C]], i8 [[B]], i8 [[A]]
+; CHECK-NEXT:    [[RET:%.*]] = sub i8 [[S1]], [[S0]]
+; CHECK-NEXT:    ret i8 [[RET]]
+;
+  %s0 = select i1 %c, i8 %a, i8 %b
+  %s1 = select i1 %c, i8 %b, i8 %a
+  %ret = sub i8 %s1, %s0
+  ret i8 %ret
+}
+
+define float @fold_select_fma_neg(i1 %c, i1 %c1, float %a, float %b, float %y) {
+; CHECK-LABEL: define float @fold_select_fma_neg(
+; CHECK-SAME: i1 [[C:%.*]], i1 [[C1:%.*]], float [[A:%.*]], float [[B:%.*]], float [[Y:%.*]]) {
+; CHECK-NEXT:    [[S0:%.*]] = select i1 [[C]], float [[A]], float [[B]]
+; CHECK-NEXT:    [[S1:%.*]] = select i1 [[C1]], float [[B]], float [[A]]
+; CHECK-NEXT:    [[RET:%.*]] = call float @llvm.fma.f32(float [[S1]], float [[S0]], float [[Y]])
+; CHECK-NEXT:    ret float [[RET]]
+;
+  %s0 = select i1 %c, float %a, float %b
+  %s1 = select i1 %c1, float %b, float %a
+  %ret = call float @llvm.fma.f32(float %s1, float %s0, float %y)
+  ret float %ret
+}
+
+define float @fold_select_fma_match_neg(i1 %c, float %a, float %b, float %y) {
+; CHECK-LABEL: define float @fold_select_fma_match_neg(
+; CHECK-SAME: i1 [[C:%.*]], float [[A:%.*]], float [[B:%.*]], float [[Y:%.*]]) {
+; CHECK-NEXT:    [[S0:%.*]] = select i1 [[C]], float [[A]], float [[B]]
+; CHECK-NEXT:    [[S1:%.*]] = select i1 [[C]], float [[B]], float [[A]]
+; CHECK-NEXT:    [[RET:%.*]] = call float @llvm.fma.f32(float [[Y]], float [[S0]], float [[S1]])
+; CHECK-NEXT:    ret float [[RET]]
+;
+  %s0 = select i1 %c, float %a, float %b
+  %s1 = select i1 %c, float %b, float %a
+  %ret = call float @llvm.fma.f32(float %y, float %s0, float %s1)
+  ret float %ret
+}
diff --git a/llvm/test/Transforms/InstCombine/div.ll b/llvm/test/Transforms/InstCombine/div.ll
index cd17d10d0e1de07aad36fb24f077690b33b470cf..80847dd588ea921709332428a1d56368d782c8b9 100644
--- a/llvm/test/Transforms/InstCombine/div.ll
+++ b/llvm/test/Transforms/InstCombine/div.ll
@@ -1432,6 +1432,140 @@ define <2 x i8> @sdiv_sdiv_mul_nsw(<2 x i8> %x, <2 x i8> %y, <2 x i8> %z) {
   ret <2 x i8> %r
 }
 
+; (X * C0) / (X * C1) --> C0 / C1
+define i8 @sdiv_mul_nsw_mul_nsw(i8 %x,i8 %y,i8 %z) {
+; CHECK-LABEL: @sdiv_mul_nsw_mul_nsw(
+; CHECK-NEXT:    [[ADD4:%.*]] = mul nsw i8 [[X:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    [[ADD5:%.*]] = mul nsw i8 [[X]], [[Y:%.*]]
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv i8 [[ADD5]], [[ADD4]]
+; CHECK-NEXT:    ret i8 [[DIV]]
+;
+  %add4 = mul nsw i8 %x, %z
+  %add5 = mul nsw i8 %x, %y
+  %div = sdiv i8 %add5, %add4
+  ret i8 %div
+}
+
+define i8 @udiv_mul_nuw_mul_nuw(i8 %x,i8 %y,i8 %z) {
+; CHECK-LABEL: @udiv_mul_nuw_mul_nuw(
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i8 [[Y:%.*]], [[Z:%.*]]
+; CHECK-NEXT:    ret i8 [[DIV]]
+;
+  %add4 = mul nuw i8 %x, %z
+  %add5 = mul nuw i8 %x, %y
+  %div = udiv i8 %add5, %add4
+  ret i8 %div
+}
+
+define i8 @sdiv_mul_nsw_constant_mul_nsw_constant(i8 %x) {
+; CHECK-LABEL: @sdiv_mul_nsw_constant_mul_nsw_constant(
+; CHECK-NEXT:    ret i8 2
+;
+  %add4 = mul nsw i8 %x, 5
+  %add5 = mul nsw i8 %x, 10
+  %div = sdiv i8 %add5, %add4
+  ret i8 %div
+}
+
+define i4 @sdiv_mul_nsw_constant_mul_constant(i4 %a) {
+; CHECK-LABEL: @sdiv_mul_nsw_constant_mul_constant(
+; CHECK-NEXT:    [[ADD4:%.*]] = mul i4 [[A:%.*]], 3
+; CHECK-NEXT:    [[ADD5:%.*]] = mul nsw i4 [[A]], 6
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv i4 [[ADD5]], [[ADD4]]
+; CHECK-NEXT:    ret i4 [[DIV]]
+;
+  %add4 = mul i4 %a, 3
+  %add5 = mul nsw i4 %a, 6
+  %div = sdiv i4 %add5, %add4
+  ret i4 %div
+}
+define i4 @sdiv_mul_nsw_constant_mul_constant2(i4 %a) {
+; CHECK-LABEL: @sdiv_mul_nsw_constant_mul_constant2(
+; CHECK-NEXT:    [[ADD4:%.*]] = sub i4 0, [[A:%.*]]
+; CHECK-NEXT:    [[ADD5:%.*]] = shl i4 [[A]], 3
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv i4 [[ADD5]], [[ADD4]]
+; CHECK-NEXT:    ret i4 [[DIV]]
+;
+  %add4 = mul i4 %a, 15
+  %add5 = mul nsw i4 %a, 8
+  %div = sdiv i4 %add5, %add4
+  ret i4 %div
+}
+
+define i4 @sdiv_mul_nsw_constant_mul_constant3(i4 %a) {
+; CHECK-LABEL: @sdiv_mul_nsw_constant_mul_constant3(
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i4 [[A:%.*]], -8
+; CHECK-NEXT:    [[DIV:%.*]] = select i1 [[TMP1]], i4 1, i4 -1
+; CHECK-NEXT:    ret i4 [[DIV]]
+;
+  %add4 = mul i4 %a, 15
+  %add5 = mul nsw i4 %a, 1
+  %div = sdiv i4 %add5, %add4
+  ret i4 %div
+}
+
+define i4 @sdiv_mul_nsw_mul(i4 %a) {
+; CHECK-LABEL: @sdiv_mul_nsw_mul(
+; CHECK-NEXT:    [[ADD4:%.*]] = sub i4 0, [[A:%.*]]
+; CHECK-NEXT:    [[ADD5:%.*]] = shl i4 [[A]], 3
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv i4 [[ADD5]], [[ADD4]]
+; CHECK-NEXT:    ret i4 [[DIV]]
+;
+  %add4 = mul i4 %a, -1
+  %add5 = mul nsw i4 %a, -8
+  %div = sdiv i4 %add5, %add4
+  ret i4 %div
+}
+
+define i4 @udiv_mul_nuw_constant_mul_constant(i4 %a) {
+; CHECK-LABEL: @udiv_mul_nuw_constant_mul_constant(
+; CHECK-NEXT:    ret i4 2
+;
+  %add4 = mul i4 %a, 3
+  %add5 = mul nuw i4 %a, 6
+  %div = udiv i4 %add5, %add4
+  ret i4 %div
+}
+
+define i4 @udiv_mul_nuw_mul_negative(i4 %a) {
+; CHECK-LABEL: @udiv_mul_nuw_mul_negative(
+; CHECK-NEXT:    [[ADD4:%.*]] = mul i4 [[A:%.*]], -3
+; CHECK-NEXT:    [[ADD5:%.*]] = shl nuw i4 [[A]], 2
+; CHECK-NEXT:    [[DIV:%.*]] = udiv i4 [[ADD5]], [[ADD4]]
+; CHECK-NEXT:    ret i4 [[DIV]]
+;
+  %add4 = mul i4 %a, 13
+  %add5 = mul nuw i4 %a, 4
+  %div = udiv i4 %add5, %add4
+  ret i4 %div
+}
+
+define i4 @sdiv_mul_nsw_mul_nsw_allones(i4 %a) {
+; CHECK-LABEL: @sdiv_mul_nsw_mul_nsw_allones(
+; CHECK-NEXT:    [[ADD4:%.*]] = sub nsw i4 0, [[A:%.*]]
+; CHECK-NEXT:    [[ADD5:%.*]] = shl i4 [[A]], 3
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv i4 [[ADD5]], [[ADD4]]
+; CHECK-NEXT:    ret i4 [[DIV]]
+;
+  %add4 = mul nsw i4 %a, -1
+  %add5 = mul nsw i4 %a, -8
+  %div = sdiv i4 %add5, %add4
+  ret i4 %div
+}
+
+define i4 @sdiv_mul_nsw_mul_signmask(i4 %a, i4 %c2) {
+; CHECK-LABEL: @sdiv_mul_nsw_mul_signmask(
+; CHECK-NEXT:    [[ADD4:%.*]] = shl i4 [[A:%.*]], 3
+; CHECK-NEXT:    [[ADD5:%.*]] = mul nsw i4 [[A]], [[C2:%.*]]
+; CHECK-NEXT:    [[DIV:%.*]] = sdiv i4 [[ADD5]], [[ADD4]]
+; CHECK-NEXT:    ret i4 [[DIV]]
+;
+  %add4 = mul nsw i4 %a, -8
+  %add5 = mul nsw i4 %a, %c2
+  %div = sdiv i4 %add5, %add4
+  ret i4 %div
+}
+
 define i32 @sdiv_sub1(i32 %arg) {
 ; CHECK-LABEL: @sdiv_sub1(
 ; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq i32 [[ARG:%.*]], -2147483648
diff --git a/llvm/test/Transforms/InstCombine/pr75129.ll b/llvm/test/Transforms/InstCombine/pr75129.ll
new file mode 100644
index 0000000000000000000000000000000000000000..c1912514b91bd15ca38511a3dd12026570630dd6
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pr75129.ll
@@ -0,0 +1,38 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -passes=instcombine -S | FileCheck %s
+
+; Pattern:
+;   (or (and (add x, half_c1), c3), (and x, c2))
+; IFF:
+;   c1, c2, c3 is constant
+;   c1 is pow2
+;   c2 < c1
+;   c3 == (c1 - 1) ^ c2
+;   half_c1 == (lshr c1, 1)
+;   (c1 >> 1) & c3 == (c1 >> 1)
+;   x is known to be less than c1
+; Could be transformed into:
+;   (xor x, half_c1)
+; The reason above transformation could be done is becase
+; in (and (add x, half_c1), c3), half_c1 equals the top bit
+; of the demanded mask indicated by (and ..., c3)
+
+define i16 @or_and_add_and() {
+; CHECK-LABEL: @or_and_add_and(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[X:%.*]] = call i16 @dummy(), !range [[RNG0:![0-9]+]]
+; CHECK-NEXT:    [[OR:%.*]] = xor i16 [[X]], 32
+; CHECK-NEXT:    ret i16 [[OR]]
+;
+entry:
+  %x = call i16 @dummy(), !range !0
+  %add = add i16 32, %x
+  %and1 = and i16 %add, 48
+  %and2 = and i16 %x, 15
+  %or = or i16 %and1, %and2
+  ret i16 %or
+}
+
+declare i16 @dummy()
+
+!0 = !{i16 0, i16 64}
diff --git a/llvm/test/Transforms/InstCombine/pr75369.ll b/llvm/test/Transforms/InstCombine/pr75369.ll
new file mode 100644
index 0000000000000000000000000000000000000000..2f90753504b36ddc688278c418536f0027c6036e
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/pr75369.ll
@@ -0,0 +1,36 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt -passes=instcombine -S < %s | FileCheck %s
+
+define i32 @main(ptr %a, i8 %a0, i32 %conv, i8 %a1) {
+; CHECK-LABEL: define i32 @main(
+; CHECK-SAME: ptr [[A:%.*]], i8 [[A0:%.*]], i32 [[CONV:%.*]], i8 [[A1:%.*]]) {
+; CHECK-NEXT:    [[A3:%.*]] = trunc i32 [[CONV]] to i8
+; CHECK-NEXT:    [[OR11:%.*]] = or i8 [[A3]], [[A0]]
+; CHECK-NEXT:    store i8 [[OR11]], ptr [[A]], align 1
+; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[A1]], 0
+; CHECK-NEXT:    call void @llvm.assume(i1 [[CMP]])
+; CHECK-NEXT:    ret i32 [[CONV]]
+;
+  %conv1 = sext i8 %a1 to i32
+  %a2 = xor i32 %conv, 1
+  %or = or i32 %conv1, %conv
+  %not = xor i32 %or, -1
+  %shr = lshr i32 %not, 1
+  %add.neg3 = sub i32 %a2, %shr
+  %conv24 = trunc i32 %add.neg3 to i8
+  store i8 %conv24, ptr %a, align 1
+  %sext = shl i32 %conv, 0
+  %conv3 = ashr i32 %sext, 0
+  %a3 = trunc i32 %conv to i8
+  %conv5 = or i8 %a3, 0
+  %xor6 = xor i8 %conv5, 0
+  %xor816 = xor i8 %a0, 0
+  %a4 = xor i8 %xor816, 0
+  %or11 = or i8 %xor6, %a4
+  store i8 %or11, ptr %a, align 1
+  %cmp = icmp slt i8 %a1, 0
+  call void @llvm.assume(i1 %cmp)
+  ret i32 %conv3
+}
+
+declare void @llvm.assume(i1)
diff --git a/llvm/test/Transforms/LICM/scalar-promote-unwind.ll b/llvm/test/Transforms/LICM/scalar-promote-unwind.ll
index ac79aec5ffc2f1eb93da5c1ade80fa814567c092..f4197c65f3cb35af34eb4ecc68fabfb567acaed3 100644
--- a/llvm/test/Transforms/LICM/scalar-promote-unwind.ll
+++ b/llvm/test/Transforms/LICM/scalar-promote-unwind.ll
@@ -147,9 +147,8 @@ for.cond.cleanup:
   ret void
 }
 
-; TODO: sret could be specified to not be accessed on unwind either.
-define void @test_sret(ptr noalias sret(i32) %a, i1 zeroext %y) uwtable {
-; CHECK-LABEL: @test_sret(
+define void @test_dead_on_unwind(ptr noalias dead_on_unwind %a, i1 zeroext %y) uwtable {
+; CHECK-LABEL: @test_dead_on_unwind(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[A_PROMOTED:%.*]] = load i32, ptr [[A:%.*]], align 4
 ; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
@@ -157,7 +156,6 @@ define void @test_sret(ptr noalias sret(i32) %a, i1 zeroext %y) uwtable {
 ; CHECK-NEXT:    [[ADD1:%.*]] = phi i32 [ [[A_PROMOTED]], [[ENTRY:%.*]] ], [ [[ADD:%.*]], [[FOR_INC:%.*]] ]
 ; CHECK-NEXT:    [[I_03:%.*]] = phi i32 [ 0, [[ENTRY]] ], [ [[INC:%.*]], [[FOR_INC]] ]
 ; CHECK-NEXT:    [[ADD]] = add nsw i32 [[ADD1]], 1
-; CHECK-NEXT:    store i32 [[ADD]], ptr [[A]], align 4
 ; CHECK-NEXT:    br i1 [[Y:%.*]], label [[IF_THEN:%.*]], label [[FOR_INC]]
 ; CHECK:       if.then:
 ; CHECK-NEXT:    tail call void @f()
@@ -167,6 +165,8 @@ define void @test_sret(ptr noalias sret(i32) %a, i1 zeroext %y) uwtable {
 ; CHECK-NEXT:    [[EXITCOND:%.*]] = icmp eq i32 [[INC]], 10000
 ; CHECK-NEXT:    br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY]]
 ; CHECK:       for.cond.cleanup:
+; CHECK-NEXT:    [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_INC]] ]
+; CHECK-NEXT:    store i32 [[ADD_LCSSA]], ptr [[A]], align 4
 ; CHECK-NEXT:    ret void
 ;
 entry:
diff --git a/llvm/test/Transforms/LICM/variant-aainfo.ll b/llvm/test/Transforms/LICM/variant-aainfo.ll
new file mode 100644
index 0000000000000000000000000000000000000000..8ad497ea4439805219f7dec12ddfce3daf2a4f48
--- /dev/null
+++ b/llvm/test/Transforms/LICM/variant-aainfo.ll
@@ -0,0 +1,59 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt < %s -S -passes=licm | FileCheck %s
+
+; See https://discourse.llvm.org/t/rfc-dont-merge-memory-locations-in-aliassettracker/73336
+; pairwise TBAA indicates NoAlias of load/store ptr at %s with store i32 at %0
+; yet LICM fails to promote load/store ptr %s out of the loop
+
+define void @_Z4testP1S(ptr %s) {
+; CHECK-LABEL: define void @_Z4testP1S(
+; CHECK-SAME: ptr [[S:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    br label [[FOR_BODY:%.*]]
+; CHECK:       for.cond.cleanup:
+; CHECK-NEXT:    ret void
+; CHECK:       for.body:
+; CHECK-NEXT:    [[I_05:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC:%.*]], [[FOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = load ptr, ptr [[S]], align 4, !tbaa [[TBAA0:![0-9]+]]
+; CHECK-NEXT:    store i32 [[I_05]], ptr [[TMP0]], align 4, !tbaa [[TBAA5:![0-9]+]]
+; CHECK-NEXT:    [[ADD_PTR_I:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i32 1
+; CHECK-NEXT:    store ptr [[ADD_PTR_I]], ptr [[S]], align 4, !tbaa [[TBAA7:![0-9]+]]
+; CHECK-NEXT:    [[INC]] = add nuw nsw i32 [[I_05]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i32 [[INC]], 100
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[FOR_COND_CLEANUP:%.*]], label [[FOR_BODY]]
+;
+entry:
+  br label %for.body
+
+for.cond.cleanup:                                 ; preds = %for.body
+  ret void
+
+for.body:                                         ; preds = %entry, %for.body
+  %i.05 = phi i32 [ 0, %entry ], [ %inc, %for.body ]
+  %0 = load ptr, ptr %s, align 4, !tbaa !0
+  store i32 %i.05, ptr %0, align 4, !tbaa !5
+  %add.ptr.i = getelementptr inbounds i32, ptr %0, i32 1
+  store ptr %add.ptr.i, ptr %s, align 4, !tbaa !7
+  %inc = add nuw nsw i32 %i.05, 1
+  %exitcond.not = icmp eq i32 %inc, 100
+  br i1 %exitcond.not, label %for.cond.cleanup, label %for.body
+}
+
+!0 = !{!1, !2, i64 0}
+!1 = !{!"_ZTS1S", !2, i64 0}
+!2 = !{!"any pointer", !3, i64 0}
+!3 = !{!"omnipotent char", !4, i64 0}
+!4 = !{!"Simple C++ TBAA"}
+!5 = !{!6, !6, i64 0}
+!6 = !{!"int", !3, i64 0}
+!7 = !{!2, !2, i64 0}
+;.
+; CHECK: [[TBAA0]] = !{[[META1:![0-9]+]], [[META2:![0-9]+]], i64 0}
+; CHECK: [[META1]] = !{!"_ZTS1S", [[META2]], i64 0}
+; CHECK: [[META2]] = !{!"any pointer", [[META3:![0-9]+]], i64 0}
+; CHECK: [[META3]] = !{!"omnipotent char", [[META4:![0-9]+]], i64 0}
+; CHECK: [[META4]] = !{!"Simple C++ TBAA"}
+; CHECK: [[TBAA5]] = !{[[META6:![0-9]+]], [[META6]], i64 0}
+; CHECK: [[META6]] = !{!"int", [[META3]], i64 0}
+; CHECK: [[TBAA7]] = !{[[META2]], [[META2]], i64 0}
+;.
diff --git a/llvm/test/Transforms/LoopUnroll/ARM/upperbound.ll b/llvm/test/Transforms/LoopUnroll/ARM/upperbound.ll
index c2e3f59ec678d3b36f86df125ce5e10b50e1210b..b364116e78e4f9bb8db24c85e99ec68ec9db3b6f 100644
--- a/llvm/test/Transforms/LoopUnroll/ARM/upperbound.ll
+++ b/llvm/test/Transforms/LoopUnroll/ARM/upperbound.ll
@@ -18,8 +18,8 @@ define void @test(ptr %x, i32 %n) {
 ; CHECK-NEXT:    br label [[IF_END]]
 ; CHECK:       if.end:
 ; CHECK-NEXT:    [[INCDEC_PTR:%.*]] = getelementptr inbounds i32, ptr [[X]], i32 1
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt i32 [[REM]], 1
-; CHECK-NEXT:    br i1 [[CMP]], label [[WHILE_BODY_1:%.*]], label [[WHILE_END]]
+; CHECK-NEXT:    [[CMP_NOT:%.*]] = icmp eq i32 [[REM]], 1
+; CHECK-NEXT:    br i1 [[CMP_NOT]], label [[WHILE_END]], label [[WHILE_BODY_1:%.*]]
 ; CHECK:       while.body.1:
 ; CHECK-NEXT:    [[TMP1:%.*]] = load i32, ptr [[INCDEC_PTR]], align 4
 ; CHECK-NEXT:    [[CMP1_1:%.*]] = icmp slt i32 [[TMP1]], 10
diff --git a/llvm/test/Transforms/LoopUnroll/runtime-multiexit-heuristic.ll b/llvm/test/Transforms/LoopUnroll/runtime-multiexit-heuristic.ll
index e2cb6a241235030a673a7427f6b7548cca53020b..75222e5ee137164027f0f7ada220199964c53bdd 100644
--- a/llvm/test/Transforms/LoopUnroll/runtime-multiexit-heuristic.ll
+++ b/llvm/test/Transforms/LoopUnroll/runtime-multiexit-heuristic.ll
@@ -38,8 +38,7 @@ define i32 @test1(ptr nocapture %a, i64 %n) {
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT:%.*]] = or disjoint i64 [[INDVARS_IV]], 1
 ; CHECK-NEXT:    br label [[FOR_EXITING_BLOCK_1:%.*]]
 ; CHECK:       for.exiting_block.1:
-; CHECK-NEXT:    [[CMP_1:%.*]] = icmp eq i64 [[TMP0]], 42
-; CHECK-NEXT:    br i1 [[CMP_1]], label [[OTHEREXIT_LOOPEXIT]], label [[LATCH_1:%.*]]
+; CHECK-NEXT:    br i1 false, label [[OTHEREXIT_LOOPEXIT]], label [[LATCH_1:%.*]]
 ; CHECK:       latch.1:
 ; CHECK-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV_NEXT]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr [[ARRAYIDX_1]], align 4
@@ -47,8 +46,7 @@ define i32 @test1(ptr nocapture %a, i64 %n) {
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT_1:%.*]] = or disjoint i64 [[INDVARS_IV]], 2
 ; CHECK-NEXT:    br label [[FOR_EXITING_BLOCK_2:%.*]]
 ; CHECK:       for.exiting_block.2:
-; CHECK-NEXT:    [[CMP_2:%.*]] = icmp eq i64 [[TMP0]], 42
-; CHECK-NEXT:    br i1 [[CMP_2]], label [[OTHEREXIT_LOOPEXIT]], label [[LATCH_2:%.*]]
+; CHECK-NEXT:    br i1 false, label [[OTHEREXIT_LOOPEXIT]], label [[LATCH_2:%.*]]
 ; CHECK:       latch.2:
 ; CHECK-NEXT:    [[ARRAYIDX_2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV_NEXT_1]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = load i32, ptr [[ARRAYIDX_2]], align 4
@@ -56,8 +54,7 @@ define i32 @test1(ptr nocapture %a, i64 %n) {
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT_2:%.*]] = or disjoint i64 [[INDVARS_IV]], 3
 ; CHECK-NEXT:    br label [[FOR_EXITING_BLOCK_3:%.*]]
 ; CHECK:       for.exiting_block.3:
-; CHECK-NEXT:    [[CMP_3:%.*]] = icmp eq i64 [[TMP0]], 42
-; CHECK-NEXT:    br i1 [[CMP_3]], label [[OTHEREXIT_LOOPEXIT]], label [[LATCH_3:%.*]]
+; CHECK-NEXT:    br i1 false, label [[OTHEREXIT_LOOPEXIT]], label [[LATCH_3:%.*]]
 ; CHECK:       latch.3:
 ; CHECK-NEXT:    [[ARRAYIDX_3:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV_NEXT_2]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = load i32, ptr [[ARRAYIDX_3]], align 4
@@ -65,8 +62,7 @@ define i32 @test1(ptr nocapture %a, i64 %n) {
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT_3:%.*]] = or disjoint i64 [[INDVARS_IV]], 4
 ; CHECK-NEXT:    br label [[FOR_EXITING_BLOCK_4:%.*]]
 ; CHECK:       for.exiting_block.4:
-; CHECK-NEXT:    [[CMP_4:%.*]] = icmp eq i64 [[TMP0]], 42
-; CHECK-NEXT:    br i1 [[CMP_4]], label [[OTHEREXIT_LOOPEXIT]], label [[LATCH_4:%.*]]
+; CHECK-NEXT:    br i1 false, label [[OTHEREXIT_LOOPEXIT]], label [[LATCH_4:%.*]]
 ; CHECK:       latch.4:
 ; CHECK-NEXT:    [[ARRAYIDX_4:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV_NEXT_3]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = load i32, ptr [[ARRAYIDX_4]], align 4
@@ -74,8 +70,7 @@ define i32 @test1(ptr nocapture %a, i64 %n) {
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT_4:%.*]] = or disjoint i64 [[INDVARS_IV]], 5
 ; CHECK-NEXT:    br label [[FOR_EXITING_BLOCK_5:%.*]]
 ; CHECK:       for.exiting_block.5:
-; CHECK-NEXT:    [[CMP_5:%.*]] = icmp eq i64 [[TMP0]], 42
-; CHECK-NEXT:    br i1 [[CMP_5]], label [[OTHEREXIT_LOOPEXIT]], label [[LATCH_5:%.*]]
+; CHECK-NEXT:    br i1 false, label [[OTHEREXIT_LOOPEXIT]], label [[LATCH_5:%.*]]
 ; CHECK:       latch.5:
 ; CHECK-NEXT:    [[ARRAYIDX_5:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV_NEXT_4]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = load i32, ptr [[ARRAYIDX_5]], align 4
@@ -83,8 +78,7 @@ define i32 @test1(ptr nocapture %a, i64 %n) {
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT_5:%.*]] = or disjoint i64 [[INDVARS_IV]], 6
 ; CHECK-NEXT:    br label [[FOR_EXITING_BLOCK_6:%.*]]
 ; CHECK:       for.exiting_block.6:
-; CHECK-NEXT:    [[CMP_6:%.*]] = icmp eq i64 [[TMP0]], 42
-; CHECK-NEXT:    br i1 [[CMP_6]], label [[OTHEREXIT_LOOPEXIT]], label [[LATCH_6:%.*]]
+; CHECK-NEXT:    br i1 false, label [[OTHEREXIT_LOOPEXIT]], label [[LATCH_6:%.*]]
 ; CHECK:       latch.6:
 ; CHECK-NEXT:    [[ARRAYIDX_6:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV_NEXT_5]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = load i32, ptr [[ARRAYIDX_6]], align 4
@@ -92,8 +86,7 @@ define i32 @test1(ptr nocapture %a, i64 %n) {
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT_6:%.*]] = or disjoint i64 [[INDVARS_IV]], 7
 ; CHECK-NEXT:    br label [[FOR_EXITING_BLOCK_7:%.*]]
 ; CHECK:       for.exiting_block.7:
-; CHECK-NEXT:    [[CMP_7:%.*]] = icmp eq i64 [[TMP0]], 42
-; CHECK-NEXT:    br i1 [[CMP_7]], label [[OTHEREXIT_LOOPEXIT]], label [[LATCH_7]]
+; CHECK-NEXT:    br i1 false, label [[OTHEREXIT_LOOPEXIT]], label [[LATCH_7]]
 ; CHECK:       latch.7:
 ; CHECK-NEXT:    [[ARRAYIDX_7:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV_NEXT_6]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = load i32, ptr [[ARRAYIDX_7]], align 4
@@ -134,12 +127,11 @@ define i32 @test1(ptr nocapture %a, i64 %n) {
 ; CHECK-NEXT:    [[SUM_0_LCSSA:%.*]] = phi i32 [ [[SUM_0_LCSSA_PH]], [[LATCHEXIT_UNR_LCSSA]] ], [ [[ADD_EPIL]], [[LATCHEXIT_EPILOG_LCSSA]] ]
 ; CHECK-NEXT:    ret i32 [[SUM_0_LCSSA]]
 ; CHECK:       otherexit.loopexit:
-; CHECK-NEXT:    [[SUM_02_LCSSA_PH:%.*]] = phi i32 [ [[SUM_02]], [[FOR_EXITING_BLOCK]] ], [ [[ADD]], [[FOR_EXITING_BLOCK_1]] ], [ [[ADD_1]], [[FOR_EXITING_BLOCK_2]] ], [ [[ADD_2]], [[FOR_EXITING_BLOCK_3]] ], [ [[ADD_3]], [[FOR_EXITING_BLOCK_4]] ], [ [[ADD_4]], [[FOR_EXITING_BLOCK_5]] ], [ [[ADD_5]], [[FOR_EXITING_BLOCK_6]] ], [ [[ADD_6]], [[FOR_EXITING_BLOCK_7]] ]
 ; CHECK-NEXT:    br label [[OTHEREXIT:%.*]]
 ; CHECK:       otherexit.loopexit3:
 ; CHECK-NEXT:    br label [[OTHEREXIT]]
 ; CHECK:       otherexit:
-; CHECK-NEXT:    [[SUM_02_LCSSA:%.*]] = phi i32 [ [[SUM_02_LCSSA_PH]], [[OTHEREXIT_LOOPEXIT]] ], [ [[SUM_02_EPIL]], [[OTHEREXIT_LOOPEXIT3]] ]
+; CHECK-NEXT:    [[SUM_02_LCSSA:%.*]] = phi i32 [ [[SUM_02]], [[OTHEREXIT_LOOPEXIT]] ], [ [[SUM_02_EPIL]], [[OTHEREXIT_LOOPEXIT3]] ]
 ; CHECK-NEXT:    [[RVAL:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"(i32 [[SUM_02_LCSSA]]) ]
 ; CHECK-NEXT:    ret i32 [[RVAL]]
 ;
@@ -774,8 +766,7 @@ define i32 @test5(ptr nocapture %a, i64 %n) {
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT:%.*]] = or disjoint i64 [[INDVARS_IV]], 1
 ; CHECK-NEXT:    br label [[FOR_EXITING_BLOCK_1:%.*]]
 ; CHECK:       for.exiting_block.1:
-; CHECK-NEXT:    [[CMP_1:%.*]] = icmp eq i64 [[TMP0]], 42
-; CHECK-NEXT:    br i1 [[CMP_1]], label [[OTHEREXIT_LOOPEXIT]], label [[LATCH_1:%.*]]
+; CHECK-NEXT:    br i1 false, label [[OTHEREXIT_LOOPEXIT]], label [[LATCH_1:%.*]]
 ; CHECK:       latch.1:
 ; CHECK-NEXT:    [[ARRAYIDX_1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV_NEXT]]
 ; CHECK-NEXT:    [[TMP4:%.*]] = load i32, ptr [[ARRAYIDX_1]], align 4
@@ -783,8 +774,7 @@ define i32 @test5(ptr nocapture %a, i64 %n) {
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT_1:%.*]] = or disjoint i64 [[INDVARS_IV]], 2
 ; CHECK-NEXT:    br label [[FOR_EXITING_BLOCK_2:%.*]]
 ; CHECK:       for.exiting_block.2:
-; CHECK-NEXT:    [[CMP_2:%.*]] = icmp eq i64 [[TMP0]], 42
-; CHECK-NEXT:    br i1 [[CMP_2]], label [[OTHEREXIT_LOOPEXIT]], label [[LATCH_2:%.*]]
+; CHECK-NEXT:    br i1 false, label [[OTHEREXIT_LOOPEXIT]], label [[LATCH_2:%.*]]
 ; CHECK:       latch.2:
 ; CHECK-NEXT:    [[ARRAYIDX_2:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV_NEXT_1]]
 ; CHECK-NEXT:    [[TMP5:%.*]] = load i32, ptr [[ARRAYIDX_2]], align 4
@@ -792,8 +782,7 @@ define i32 @test5(ptr nocapture %a, i64 %n) {
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT_2:%.*]] = or disjoint i64 [[INDVARS_IV]], 3
 ; CHECK-NEXT:    br label [[FOR_EXITING_BLOCK_3:%.*]]
 ; CHECK:       for.exiting_block.3:
-; CHECK-NEXT:    [[CMP_3:%.*]] = icmp eq i64 [[TMP0]], 42
-; CHECK-NEXT:    br i1 [[CMP_3]], label [[OTHEREXIT_LOOPEXIT]], label [[LATCH_3:%.*]]
+; CHECK-NEXT:    br i1 false, label [[OTHEREXIT_LOOPEXIT]], label [[LATCH_3:%.*]]
 ; CHECK:       latch.3:
 ; CHECK-NEXT:    [[ARRAYIDX_3:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV_NEXT_2]]
 ; CHECK-NEXT:    [[TMP6:%.*]] = load i32, ptr [[ARRAYIDX_3]], align 4
@@ -801,8 +790,7 @@ define i32 @test5(ptr nocapture %a, i64 %n) {
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT_3:%.*]] = or disjoint i64 [[INDVARS_IV]], 4
 ; CHECK-NEXT:    br label [[FOR_EXITING_BLOCK_4:%.*]]
 ; CHECK:       for.exiting_block.4:
-; CHECK-NEXT:    [[CMP_4:%.*]] = icmp eq i64 [[TMP0]], 42
-; CHECK-NEXT:    br i1 [[CMP_4]], label [[OTHEREXIT_LOOPEXIT]], label [[LATCH_4:%.*]]
+; CHECK-NEXT:    br i1 false, label [[OTHEREXIT_LOOPEXIT]], label [[LATCH_4:%.*]]
 ; CHECK:       latch.4:
 ; CHECK-NEXT:    [[ARRAYIDX_4:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV_NEXT_3]]
 ; CHECK-NEXT:    [[TMP7:%.*]] = load i32, ptr [[ARRAYIDX_4]], align 4
@@ -810,8 +798,7 @@ define i32 @test5(ptr nocapture %a, i64 %n) {
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT_4:%.*]] = or disjoint i64 [[INDVARS_IV]], 5
 ; CHECK-NEXT:    br label [[FOR_EXITING_BLOCK_5:%.*]]
 ; CHECK:       for.exiting_block.5:
-; CHECK-NEXT:    [[CMP_5:%.*]] = icmp eq i64 [[TMP0]], 42
-; CHECK-NEXT:    br i1 [[CMP_5]], label [[OTHEREXIT_LOOPEXIT]], label [[LATCH_5:%.*]]
+; CHECK-NEXT:    br i1 false, label [[OTHEREXIT_LOOPEXIT]], label [[LATCH_5:%.*]]
 ; CHECK:       latch.5:
 ; CHECK-NEXT:    [[ARRAYIDX_5:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV_NEXT_4]]
 ; CHECK-NEXT:    [[TMP8:%.*]] = load i32, ptr [[ARRAYIDX_5]], align 4
@@ -819,8 +806,7 @@ define i32 @test5(ptr nocapture %a, i64 %n) {
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT_5:%.*]] = or disjoint i64 [[INDVARS_IV]], 6
 ; CHECK-NEXT:    br label [[FOR_EXITING_BLOCK_6:%.*]]
 ; CHECK:       for.exiting_block.6:
-; CHECK-NEXT:    [[CMP_6:%.*]] = icmp eq i64 [[TMP0]], 42
-; CHECK-NEXT:    br i1 [[CMP_6]], label [[OTHEREXIT_LOOPEXIT]], label [[LATCH_6:%.*]]
+; CHECK-NEXT:    br i1 false, label [[OTHEREXIT_LOOPEXIT]], label [[LATCH_6:%.*]]
 ; CHECK:       latch.6:
 ; CHECK-NEXT:    [[ARRAYIDX_6:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV_NEXT_5]]
 ; CHECK-NEXT:    [[TMP9:%.*]] = load i32, ptr [[ARRAYIDX_6]], align 4
@@ -828,8 +814,7 @@ define i32 @test5(ptr nocapture %a, i64 %n) {
 ; CHECK-NEXT:    [[INDVARS_IV_NEXT_6:%.*]] = or disjoint i64 [[INDVARS_IV]], 7
 ; CHECK-NEXT:    br label [[FOR_EXITING_BLOCK_7:%.*]]
 ; CHECK:       for.exiting_block.7:
-; CHECK-NEXT:    [[CMP_7:%.*]] = icmp eq i64 [[TMP0]], 42
-; CHECK-NEXT:    br i1 [[CMP_7]], label [[OTHEREXIT_LOOPEXIT]], label [[LATCH_7]]
+; CHECK-NEXT:    br i1 false, label [[OTHEREXIT_LOOPEXIT]], label [[LATCH_7]]
 ; CHECK:       latch.7:
 ; CHECK-NEXT:    [[ARRAYIDX_7:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDVARS_IV_NEXT_6]]
 ; CHECK-NEXT:    [[TMP10:%.*]] = load i32, ptr [[ARRAYIDX_7]], align 4
@@ -870,12 +855,11 @@ define i32 @test5(ptr nocapture %a, i64 %n) {
 ; CHECK-NEXT:    [[SUM_0_LCSSA:%.*]] = phi i32 [ [[SUM_0_LCSSA_PH]], [[LATCHEXIT_UNR_LCSSA]] ], [ [[ADD_EPIL]], [[LATCHEXIT_EPILOG_LCSSA]] ]
 ; CHECK-NEXT:    ret i32 [[SUM_0_LCSSA]]
 ; CHECK:       otherexit.loopexit:
-; CHECK-NEXT:    [[RVAL_PH:%.*]] = phi i32 [ [[SUM_02]], [[FOR_EXITING_BLOCK]] ], [ [[ADD]], [[FOR_EXITING_BLOCK_1]] ], [ [[ADD_1]], [[FOR_EXITING_BLOCK_2]] ], [ [[ADD_2]], [[FOR_EXITING_BLOCK_3]] ], [ [[ADD_3]], [[FOR_EXITING_BLOCK_4]] ], [ [[ADD_4]], [[FOR_EXITING_BLOCK_5]] ], [ [[ADD_5]], [[FOR_EXITING_BLOCK_6]] ], [ [[ADD_6]], [[FOR_EXITING_BLOCK_7]] ]
 ; CHECK-NEXT:    br label [[OTHEREXIT:%.*]]
 ; CHECK:       otherexit.loopexit3:
 ; CHECK-NEXT:    br label [[OTHEREXIT]]
 ; CHECK:       otherexit:
-; CHECK-NEXT:    [[SUM_02_LCSSA:%.*]] = phi i32 [ [[RVAL_PH]], [[OTHEREXIT_LOOPEXIT]] ], [ [[SUM_02_EPIL]], [[OTHEREXIT_LOOPEXIT3]] ]
+; CHECK-NEXT:    [[SUM_02_LCSSA:%.*]] = phi i32 [ [[SUM_02]], [[OTHEREXIT_LOOPEXIT]] ], [ [[SUM_02_EPIL]], [[OTHEREXIT_LOOPEXIT3]] ]
 ; CHECK-NEXT:    br label [[OTHEREXIT2:%.*]]
 ; CHECK:       otherexit2:
 ; CHECK-NEXT:    [[RVAL2:%.*]] = call i32 (...) @llvm.experimental.deoptimize.i32() [ "deopt"(i32 [[SUM_02_LCSSA]]) ]
diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/loopvectorize_pr33804_double.ll b/llvm/test/Transforms/LoopVectorize/AArch64/loopvectorize_pr33804_double.ll
index 263da076fbb4e13583593cd9a71d093c39a2c061..0f3db228e9cfe3d5a265d398a779d908d50b132f 100644
--- a/llvm/test/Transforms/LoopVectorize/AArch64/loopvectorize_pr33804_double.ll
+++ b/llvm/test/Transforms/LoopVectorize/AArch64/loopvectorize_pr33804_double.ll
@@ -15,16 +15,16 @@ target triple = "aarch64-unknown-linux-gnu"
 ; CHECK-LABEL: @cvCalcEMD2
 ; CHECK: vector.body
 ; CHECK: store <{{[0-9]+}} x ptr>
-define void @cvCalcEMD2() local_unnamed_addr #0 personality ptr @__gxx_personality_v0 {
+define void @cvCalcEMD2(ptr %dst) {
 entry:
   br label %for.body14.i.i
 
 for.body14.i.i:                                   ; preds = %for.body14.i.i, %entry
   %i.1424.i.i = phi i32 [ %inc21.i.i, %for.body14.i.i ], [ 0, %entry ]
-  %arrayidx15.i.i1427 = getelementptr inbounds %struct.CvNode1D, ptr undef, i32 %i.1424.i.i
+  %arrayidx15.i.i1427 = getelementptr inbounds %struct.CvNode1D, ptr %dst, i32 %i.1424.i.i
   store double 0xC415AF1D80000000, ptr %arrayidx15.i.i1427, align 4
-  %next19.i.i = getelementptr inbounds %struct.CvNode1D, ptr undef, i32 %i.1424.i.i, i32 1
-  store ptr undef, ptr %next19.i.i, align 4
+  %next19.i.i = getelementptr inbounds %struct.CvNode1D, ptr %dst, i32 %i.1424.i.i, i32 1
+  store ptr %dst, ptr %next19.i.i, align 4
   %inc21.i.i = add nuw nsw i32 %i.1424.i.i, 1
   %exitcond438.i.i = icmp eq i32 %inc21.i.i, 0
   br i1 %exitcond438.i.i, label %for.end22.i.i, label %for.body14.i.i
@@ -40,15 +40,15 @@ for.end22.i.i:                                    ; preds = %for.body14.i.i
 ; CHECK-LABEL: @cvCalcEMD2_2
 ; CHECK: vector.body
 ; CHECK: store <{{[0-9]+}} x double>
-define void @cvCalcEMD2_2() local_unnamed_addr #0 personality ptr @__gxx_personality_v0 {
+define void @cvCalcEMD2_2(ptr %dst) {
 entry:
   br label %for.body14.i.i
 
 for.body14.i.i:                                   ; preds = %for.body14.i.i, %entry
   %i.1424.i.i = phi i32 [ %inc21.i.i, %for.body14.i.i ], [ 0, %entry ]
-  %next19.i.i = getelementptr inbounds %struct.CvNode1D2, ptr undef, i32 %i.1424.i.i, i32 0
-  store ptr undef, ptr %next19.i.i, align 4
-  %arrayidx15.i.i1427 = getelementptr inbounds %struct.CvNode1D2, ptr undef, i32 %i.1424.i.i
+  %next19.i.i = getelementptr inbounds %struct.CvNode1D2, ptr %dst, i32 %i.1424.i.i, i32 0
+  store ptr %dst, ptr %next19.i.i, align 4
+  %arrayidx15.i.i1427 = getelementptr inbounds %struct.CvNode1D2, ptr %dst, i32 %i.1424.i.i
   %val.i.i = getelementptr inbounds %struct.CvNode1D2, ptr %arrayidx15.i.i1427, i32 0, i32 1
   store double 0xC415AF1D80000000, ptr %val.i.i, align 4
   %inc21.i.i = add nuw nsw i32 %i.1424.i.i, 1
@@ -64,16 +64,20 @@ for.end22.i.i:                                    ; preds = %for.body14.i.i
 ; CHECK-LABEL: @cvCalcEMD3
 ; CHECK: vector.body
 ; CHECK: inttoptr <{{[0-9]+}} x i64>
-define void @cvCalcEMD3() local_unnamed_addr #0 personality ptr @__gxx_personality_v0 {
+define void @cvCalcEMD3(ptr %src, ptr %dst) {
 entry:
   br label %for.body14.i.i
 
 for.body14.i.i:                                   ; preds = %for.body14.i.i, %entry
   %i.1424.i.i = phi i32 [ %inc21.i.i, %for.body14.i.i ], [ 0, %entry ]
-  %arrayidx15.i.i1427 = getelementptr inbounds %struct.CvNode1D, ptr undef, i32 %i.1424.i.i
+  %arrayidx15.i.i1427 = getelementptr inbounds %struct.CvNode1D, ptr %src, i32 %i.1424.i.i
   %load_d = load double, ptr %arrayidx15.i.i1427, align 4
-  %next19.i.i = getelementptr inbounds %struct.CvNode1D, ptr undef, i32 %i.1424.i.i, i32 1
+  %next19.i.i = getelementptr inbounds %struct.CvNode1D, ptr %src, i32 %i.1424.i.i, i32 1
   %load_p = load ptr, ptr %next19.i.i, align 4
+  %dst.ptr = getelementptr inbounds %struct.CvNode1D, ptr %dst, i32 %i.1424.i.i
+  %dst.ptr.1 = getelementptr inbounds %struct.CvNode1D, ptr %dst, i32 %i.1424.i.i, i32 1
+  store double %load_d, ptr %dst.ptr, align 4
+  store ptr %load_p, ptr %dst.ptr.1, align 4
   %inc21.i.i = add nuw nsw i32 %i.1424.i.i, 1
   %exitcond438.i.i = icmp eq i32 %inc21.i.i, 0
   br i1 %exitcond438.i.i, label %for.end22.i.i, label %for.body14.i.i
@@ -87,17 +91,21 @@ for.end22.i.i:                                    ; preds = %for.body14.i.i
 ; CHECK-LABEL: @cvCalcEMD3_2
 ; CHECK: vector.body
 ; CHECK: ptrtoint <{{[0-9]+}} x ptr>
-define void @cvCalcEMD3_2() local_unnamed_addr #0 personality ptr @__gxx_personality_v0 {
+define void @cvCalcEMD3_2(ptr %src, ptr %dst) {
 entry:
   br label %for.body14.i.i
 
 for.body14.i.i:                                   ; preds = %for.body14.i.i, %entry
   %i.1424.i.i = phi i32 [ %inc21.i.i, %for.body14.i.i ], [ 0, %entry ]
-  %next19.i.i = getelementptr inbounds %struct.CvNode1D2, ptr undef, i32 %i.1424.i.i, i32 0
+  %next19.i.i = getelementptr inbounds %struct.CvNode1D2, ptr %src, i32 %i.1424.i.i, i32 0
   %load_p = load ptr, ptr %next19.i.i, align 4
-  %arrayidx15.i.i1427 = getelementptr inbounds %struct.CvNode1D2, ptr undef, i32 %i.1424.i.i
+  %arrayidx15.i.i1427 = getelementptr inbounds %struct.CvNode1D2, ptr %src, i32 %i.1424.i.i
   %val.i.i = getelementptr inbounds %struct.CvNode1D2, ptr %arrayidx15.i.i1427, i32 0, i32 1
   %load_d = load double, ptr %val.i.i, align 4
+  %dst.ptr = getelementptr inbounds %struct.CvNode1D, ptr %dst, i32 %i.1424.i.i
+  %dst.ptr.1 = getelementptr inbounds %struct.CvNode1D, ptr %dst, i32 %i.1424.i.i, i32 1
+  store double %load_d, ptr %dst.ptr, align 4
+  store ptr %load_p, ptr %dst.ptr.1, align 4
   %inc21.i.i = add nuw nsw i32 %i.1424.i.i, 1
   %exitcond438.i.i = icmp eq i32 %inc21.i.i, 0
   br i1 %exitcond438.i.i, label %for.end22.i.i, label %for.body14.i.i
@@ -105,8 +113,3 @@ for.body14.i.i:                                   ; preds = %for.body14.i.i, %en
 for.end22.i.i:                                    ; preds = %for.body14.i.i
   unreachable
 }
-
-declare i32 @__gxx_personality_v0(...)
-
-attributes #0 = { "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "frame-pointer"="all" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="generic" "unsafe-fp-math"="false" "use-soft-float"="false" }
-
diff --git a/llvm/test/Transforms/LoopVectorize/reduction-small-size.ll b/llvm/test/Transforms/LoopVectorize/reduction-small-size.ll
index 2a58748d8fb67dc927c96afa6d58294db420b9e5..3973a288fe71fb03e80ac4fef13595ac9fbd4532 100644
--- a/llvm/test/Transforms/LoopVectorize/reduction-small-size.ll
+++ b/llvm/test/Transforms/LoopVectorize/reduction-small-size.ll
@@ -11,12 +11,15 @@ define i8 @PR34687(i1 %c, i32 %x, i32 %n) {
 ; CHECK:       vector.ph:
 ; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i32 [[N]], 4
 ; CHECK-NEXT:    [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[C:%.*]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer
 ; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> poison, i32 [[X:%.*]], i64 0
 ; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> poison, <4 x i32> zeroinitializer
 ; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
 ; CHECK:       vector.body:
 ; CHECK-NEXT:    [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
 ; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[TMP0:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x i32> undef, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
 ; CHECK-NEXT:    [[TMP1:%.*]] = and <4 x i32> [[VEC_PHI]], <i32 255, i32 255, i32 255, i32 255>
 ; CHECK-NEXT:    [[TMP2:%.*]] = add <4 x i32> [[TMP1]], [[BROADCAST_SPLAT2]]
 ; CHECK-NEXT:    [[TMP3:%.*]] = trunc <4 x i32> [[TMP2]] to <4 x i8>
@@ -37,7 +40,7 @@ define i8 @PR34687(i1 %c, i32 %x, i32 %n) {
 ; CHECK:       for.body:
 ; CHECK-NEXT:    [[I:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[IF_END:%.*]] ]
 ; CHECK-NEXT:    [[R:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[R_NEXT:%.*]], [[IF_END]] ]
-; CHECK-NEXT:    br i1 [[C:%.*]], label [[IF_THEN:%.*]], label [[IF_END]]
+; CHECK-NEXT:    br i1 [[C]], label [[IF_THEN:%.*]], label [[IF_END]]
 ; CHECK:       if.then:
 ; CHECK-NEXT:    [[T0:%.*]] = sdiv i32 undef, undef
 ; CHECK-NEXT:    br label [[IF_END]]
diff --git a/llvm/test/Transforms/LoopVectorize/store-reduction-results-in-tail-folded-loop.ll b/llvm/test/Transforms/LoopVectorize/store-reduction-results-in-tail-folded-loop.ll
new file mode 100644
index 0000000000000000000000000000000000000000..7014bfaab14676bc9dda9b16fc978442c7375dfe
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/store-reduction-results-in-tail-folded-loop.ll
@@ -0,0 +1,89 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt -p loop-vectorize -force-vector-width=4 -force-vector-interleave=1 -S %s | FileCheck %s
+
+target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128"
+
+@c = global i32 1, align 4
+@a = global i32 0, align 4
+
+define void @pr75298_store_reduction_value_in_folded_loop(i64 %iv.start) optsize {
+; CHECK-LABEL: define void @pr75298_store_reduction_value_in_folded_loop(
+; CHECK-SAME: i64 [[IV_START:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[CMP3:%.*]] = icmp slt i64 [[IV_START]], 7
+; CHECK-NEXT:    br i1 [[CMP3]], label [[PH:%.*]], label [[EXIT:%.*]]
+; CHECK:       ph:
+; CHECK-NEXT:    [[TMP0:%.*]] = sub i64 7, [[IV_START]]
+; CHECK-NEXT:    br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]]
+; CHECK:       vector.ph:
+; CHECK-NEXT:    [[N_RND_UP:%.*]] = add i64 [[TMP0]], 3
+; CHECK-NEXT:    [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], 4
+; CHECK-NEXT:    [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]]
+; CHECK-NEXT:    [[IND_END:%.*]] = add i64 [[IV_START]], [[N_VEC]]
+; CHECK-NEXT:    [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[TMP0]], 1
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i64> poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    br label [[VECTOR_BODY:%.*]]
+; CHECK:       vector.body:
+; CHECK-NEXT:    [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP3:%.*]], [[VECTOR_BODY]] ]
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i64> poison, i64 [[INDEX]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i64> [[BROADCAST_SPLATINSERT1]], <4 x i64> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[VEC_IV:%.*]] = add <4 x i64> [[BROADCAST_SPLAT2]], <i64 0, i64 1, i64 2, i64 3>
+; CHECK-NEXT:    [[TMP1:%.*]] = icmp ule <4 x i64> [[VEC_IV]], [[BROADCAST_SPLAT]]
+; CHECK-NEXT:    [[TMP2:%.*]] = load i32, ptr @c, align 4
+; CHECK-NEXT:    [[BROADCAST_SPLATINSERT3:%.*]] = insertelement <4 x i32> poison, i32 [[TMP2]], i64 0
+; CHECK-NEXT:    [[BROADCAST_SPLAT4:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT3]], <4 x i32> poison, <4 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP3]] = xor <4 x i32> [[VEC_PHI]], [[BROADCAST_SPLAT4]]
+; CHECK-NEXT:    [[TMP4:%.*]] = select <4 x i1> [[TMP1]], <4 x i32> [[TMP3]], <4 x i32> [[VEC_PHI]]
+; CHECK-NEXT:    [[INDEX_NEXT]] = add i64 [[INDEX]], 4
+; CHECK-NEXT:    [[TMP5:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]]
+; CHECK-NEXT:    br i1 [[TMP5]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]]
+; CHECK:       middle.block:
+; CHECK-NEXT:    [[TMP6:%.*]] = call i32 @llvm.vector.reduce.xor.v4i32(<4 x i32> [[TMP4]])
+; CHECK-NEXT:    store i32 [[TMP6]], ptr @a, align 4
+; CHECK-NEXT:    br i1 true, label [[EXIT_LOOPEXIT:%.*]], label [[SCALAR_PH]]
+; CHECK:       scalar.ph:
+; CHECK-NEXT:    [[BC_RESUME_VAL:%.*]] = phi i64 [ [[IND_END]], [[MIDDLE_BLOCK]] ], [ [[IV_START]], [[PH]] ]
+; CHECK-NEXT:    [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[PH]] ], [ [[TMP6]], [[MIDDLE_BLOCK]] ]
+; CHECK-NEXT:    br label [[LOOP:%.*]]
+; CHECK:       loop:
+; CHECK-NEXT:    [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[RED:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[RED_NEXT:%.*]], [[LOOP]] ]
+; CHECK-NEXT:    [[L:%.*]] = load i32, ptr @c, align 4
+; CHECK-NEXT:    [[RED_NEXT]] = xor i32 [[RED]], [[L]]
+; CHECK-NEXT:    store i32 [[RED_NEXT]], ptr @a, align 4
+; CHECK-NEXT:    [[IV_NEXT]] = add i64 [[IV]], 1
+; CHECK-NEXT:    [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], 7
+; CHECK-NEXT:    br i1 [[EXITCOND_NOT]], label [[EXIT_LOOPEXIT]], label [[LOOP]], !llvm.loop [[LOOP3:![0-9]+]]
+; CHECK:       exit.loopexit:
+; CHECK-NEXT:    br label [[EXIT]]
+; CHECK:       exit:
+; CHECK-NEXT:    ret void
+;
+entry:
+  %cmp3 = icmp slt i64 %iv.start, 7
+  br i1 %cmp3, label %ph, label %exit
+
+ph:
+  br label %loop
+
+loop:
+  %iv = phi i64 [ %iv.start, %ph ], [ %iv.next, %loop ]
+  %red = phi i32 [ 0, %ph ], [ %red.next, %loop ]
+  %l = load i32, ptr @c, align 4
+  %red.next = xor i32 %red, %l
+  store i32 %red.next, ptr @a, align 4
+  %iv.next = add i64 %iv, 1
+  %exitcond.not = icmp eq i64 %iv.next, 7
+  br i1 %exitcond.not, label %exit, label %loop
+
+exit:
+  ret void
+}
+;.
+; CHECK: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]}
+; CHECK: [[META1]] = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: [[META2]] = !{!"llvm.loop.unroll.runtime.disable"}
+; CHECK: [[LOOP3]] = distinct !{[[LOOP3]], [[META2]], [[META1]]}
+;.
diff --git a/llvm/test/Transforms/LoopVectorize/vplan-unused-interleave-group.ll b/llvm/test/Transforms/LoopVectorize/vplan-unused-interleave-group.ll
new file mode 100644
index 0000000000000000000000000000000000000000..5ea27994b356dae2ef48d07215e49f388479c125
--- /dev/null
+++ b/llvm/test/Transforms/LoopVectorize/vplan-unused-interleave-group.ll
@@ -0,0 +1,42 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 3
+; RUN: opt -passes=loop-vectorize -S -force-vector-width=4 -enable-interleaved-mem-accesses=true -debug-only=loop-vectorize < %s 2>&1 | FileCheck %s
+; REQUIRES: asserts
+
+; This test checks if an unused interleave group is removed by removeDeadRecipes.
+
+%struct.foo = type { ptr, ptr }
+
+define void @test_unused_interleave(ptr %src, i32 %length) {
+; CHECK-LABEL: Checking a loop in 'test_unused_interleave'
+; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' {
+; CHECK-NEXT: Live-in vp<%0> = VF * UF
+; CHECK-NEXT: Live-in vp<%1> = vector-trip-count
+; CHECK-NEXT: Live-in ir<%length> = original trip-count
+; CHECK-EMPTY:
+; CHECK-NEXT: vector.ph:
+; CHECK-NEXT: Successor(s): vector loop
+; CHECK-EMPTY:
+; CHECK-NEXT: <x1> vector loop: {
+; CHECK-NEXT:   vector.body:
+; CHECK-NEXT:     EMIT vp<%2> = CANONICAL-INDUCTION ir<0>, vp<%3>
+; CHECK-NEXT:     EMIT vp<%3> = add nuw vp<%2>, vp<%0>
+; CHECK-NEXT:     EMIT branch-on-count vp<%3>, vp<%1>
+; CHECK-NEXT:   No successors
+; CHECK-NEXT: }
+entry:
+  br label %for.body
+
+for.body:
+  %iv = phi i32 [ %iv.next, %for.body ], [ 0, %entry ]
+  %next19.i.i = getelementptr inbounds %struct.foo, ptr %src, i32 %iv, i32 0
+  %load_p1 = load ptr, ptr %next19.i.i, align 4
+  %arrayidx15.i.i1427 = getelementptr inbounds %struct.foo, ptr %src, i32 %iv
+  %val.i.i = getelementptr inbounds %struct.foo, ptr %arrayidx15.i.i1427, i32 0, i32 1
+  %load_p2 = load ptr, ptr %val.i.i, align 4
+  %iv.next = add nuw nsw i32 %iv, 1
+  %cond = icmp eq i32 %iv.next, %length
+  br i1 %cond, label %for.end, label %for.body
+
+for.end:
+  ret void
+}
diff --git a/llvm/test/Transforms/MemCpyOpt/callslot_throw.ll b/llvm/test/Transforms/MemCpyOpt/callslot_throw.ll
index 8ae188c6fd3d4c29a85dd9aa4a149dfe1649e346..608d45ab976c619142cc4ebae50572c862a92bb3 100644
--- a/llvm/test/Transforms/MemCpyOpt/callslot_throw.ll
+++ b/llvm/test/Transforms/MemCpyOpt/callslot_throw.ll
@@ -56,9 +56,39 @@ entry:
   ret void
 }
 
-; TODO: With updated semantics, sret could also be invisible on unwind.
-define void @test_sret(ptr nocapture noalias dereferenceable(4) sret(i32) %x) {
-; CHECK-LABEL: @test_sret(
+define void @test_dead_on_unwind(ptr nocapture noalias writable dead_on_unwind dereferenceable(4) %x) {
+; CHECK-LABEL: @test_dead_on_unwind(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[T:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    call void @may_throw(ptr nonnull [[X:%.*]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %t = alloca i32, align 4
+  call void @may_throw(ptr nonnull %t)
+  %load = load i32, ptr %t, align 4
+  store i32 %load, ptr %x, align 4
+  ret void
+}
+
+; Same as previous test, but dereferenceability information is provided by sret.
+define void @test_dead_on_unwind_sret(ptr nocapture noalias writable dead_on_unwind sret(i32) %x) {
+; CHECK-LABEL: @test_dead_on_unwind_sret(
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[T:%.*]] = alloca i32, align 4
+; CHECK-NEXT:    call void @may_throw(ptr nonnull [[X:%.*]])
+; CHECK-NEXT:    ret void
+;
+entry:
+  %t = alloca i32, align 4
+  call void @may_throw(ptr nonnull %t)
+  %load = load i32, ptr %t, align 4
+  store i32 %load, ptr %x, align 4
+  ret void
+}
+
+define void @test_dead_on_unwind_missing_writable(ptr nocapture noalias dead_on_unwind dereferenceable(4) %x) {
+; CHECK-LABEL: @test_dead_on_unwind_missing_writable(
 ; CHECK-NEXT:  entry:
 ; CHECK-NEXT:    [[T:%.*]] = alloca i32, align 4
 ; CHECK-NEXT:    call void @may_throw(ptr nonnull [[T]])
diff --git a/llvm/test/Transforms/PGOProfile/timestamp.ll b/llvm/test/Transforms/PGOProfile/timestamp.ll
index 8d6095a031a66bb9e76ea34623f70476a2b243f1..d5d1233070f7fa8de87e8f99d5694920a868b08a 100644
--- a/llvm/test/Transforms/PGOProfile/timestamp.ll
+++ b/llvm/test/Transforms/PGOProfile/timestamp.ll
@@ -3,10 +3,21 @@
 target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128"
 target triple = "x86_64-unknown-linux-gnu"
 
+; CHECK-LABEL: define void @foo(
 define void @foo() {
 entry:
   ; CHECK: call void @llvm.instrprof.timestamp({{.*}})
   ret void
 }
 
-; CHECK: declare void @llvm.instrprof.timestamp(
+; CHECK-LABEL: define void @bar(
+define void @bar() #0 {
+entry:
+  ; CHECK-NOT: call void @llvm.instrprof.timestamp({{.*}})
+  call void asm sideeffect "retq;", "~{dirflag},~{fpsr},~{flags}"()
+  unreachable
+}
+
+; CHECK-LABEL: declare void @llvm.instrprof.timestamp(
+
+attributes #0 = { naked }
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/unknown-entries.ll b/llvm/test/Transforms/SLPVectorizer/X86/unknown-entries.ll
new file mode 100644
index 0000000000000000000000000000000000000000..fc22280c2b8adacf72dce1e61749c8955f6a4b07
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/unknown-entries.ll
@@ -0,0 +1,25 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt < %s -passes=slp-vectorizer -S | FileCheck %s
+
+target triple = "x86_64-unknown-linux-gnu"
+
+define <3 x i64> @ahyes(i64 %position, i64 %value) {
+; CHECK-LABEL: define <3 x i64> @ahyes(
+; CHECK-SAME: i64 [[POSITION:%.*]], i64 [[VALUE:%.*]]) {
+; CHECK-NEXT:  entry:
+; CHECK-NEXT:    [[TMP0:%.*]] = insertelement <2 x i64> poison, i64 [[VALUE]], i32 0
+; CHECK-NEXT:    [[TMP1:%.*]] = shufflevector <2 x i64> [[TMP0]], <2 x i64> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT:    [[TMP2:%.*]] = add <2 x i64> [[TMP1]], <i64 1, i64 2>
+; CHECK-NEXT:    [[TMP3:%.*]] = insertelement <3 x i64> poison, i64 [[VALUE]], i64 [[POSITION]]
+; CHECK-NEXT:    [[TMP4:%.*]] = shufflevector <2 x i64> [[TMP2]], <2 x i64> poison, <3 x i32> <i32 0, i32 1, i32 poison>
+; CHECK-NEXT:    [[TMP5:%.*]] = shufflevector <3 x i64> [[TMP3]], <3 x i64> [[TMP4]], <3 x i32> <i32 3, i32 4, i32 2>
+; CHECK-NEXT:    ret <3 x i64> [[TMP5]]
+;
+entry:
+  %0 = add i64 %value, 1
+  %1 = add i64 %value, 2
+  %2 = insertelement <3 x i64> poison, i64 %value, i64 %position
+  %3 = insertelement <3 x i64> %2, i64 %0, i64 0
+  %4 = insertelement <3 x i64> %3, i64 %1, i64 1
+  ret <3 x i64> %4
+}
diff --git a/llvm/test/Transforms/Util/trivial-auto-var-init-call.ll b/llvm/test/Transforms/Util/trivial-auto-var-init-call.ll
index f34aa3e9f6799e89dded945f4a13ef1925f41f31..c12648a6b3d81e9d38f9effb2732c01771dce584 100644
--- a/llvm/test/Transforms/Util/trivial-auto-var-init-call.ll
+++ b/llvm/test/Transforms/Util/trivial-auto-var-init-call.ll
@@ -1,4 +1,5 @@
 ; RUN: opt -passes=annotation-remarks -o /dev/null -S -pass-remarks-output=%t.opt.yaml %s -pass-remarks-missed=annotation-remarks 2>&1 | FileCheck %s
+; RUN: opt --try-experimental-debuginfo-iterators -passes=annotation-remarks -o /dev/null -S -pass-remarks-output=%t.opt.yaml %s -pass-remarks-missed=annotation-remarks 2>&1 | FileCheck %s
 ; RUN: cat %t.opt.yaml | FileCheck -check-prefix=YAML %s
 
 ; Emit remarks for memcpy, memmove, memset, bzero.
diff --git a/llvm/test/Transforms/Util/trivial-auto-var-init-store.ll b/llvm/test/Transforms/Util/trivial-auto-var-init-store.ll
index d3355efe2efcd89ee998b76f470902df9189b446..2a7eb638edf6bc9cac0d7ce885a55c6f39583557 100644
--- a/llvm/test/Transforms/Util/trivial-auto-var-init-store.ll
+++ b/llvm/test/Transforms/Util/trivial-auto-var-init-store.ll
@@ -1,4 +1,5 @@
 ; RUN: opt -passes=annotation-remarks -o /dev/null -S -pass-remarks-output=%t.opt.yaml %s -pass-remarks-missed=annotation-remarks 2>&1 | FileCheck %s
+; RUN: opt --try-experimental-debuginfo-iterators -passes=annotation-remarks -o /dev/null -S -pass-remarks-output=%t.opt.yaml %s -pass-remarks-missed=annotation-remarks 2>&1 | FileCheck %s
 ; RUN: cat %t.opt.yaml | FileCheck -check-prefix=YAML %s
 
 ; Emit a remark that reports a store.
diff --git a/llvm/test/Verifier/dead-on-unwind.ll b/llvm/test/Verifier/dead-on-unwind.ll
new file mode 100644
index 0000000000000000000000000000000000000000..a67c002ee6d8ff30c5a7610ee73c53633ae7163b
--- /dev/null
+++ b/llvm/test/Verifier/dead-on-unwind.ll
@@ -0,0 +1,7 @@
+; RUN: not llvm-as -disable-output %s 2>&1 | FileCheck %s
+
+; CHECK: Attribute 'dead_on_unwind' applied to incompatible type!
+; CHECK-NEXT: ptr @not_pointer
+define void @not_pointer(i32 dead_on_unwind %arg) {
+  ret void
+}
diff --git a/llvm/test/lit.cfg.py b/llvm/test/lit.cfg.py
index 8319093408020fca2323adf9e0a86e04e3559202..c6f9ee82e08cc1917bd6b41f352128485a0746ce 100644
--- a/llvm/test/lit.cfg.py
+++ b/llvm/test/lit.cfg.py
@@ -572,6 +572,9 @@ if "darwin" == sys.platform:
         if "hw.optional.fma: 1" in result:
             config.available_features.add("fma3")
 
+if not hasattr(sys, "getwindowsversion") or sys.getwindowsversion().build >= 17063:
+    config.available_features.add("unix-sockets")
+
 # .debug_frame is not emitted for targeting Windows x64, aarch64/arm64, AIX, or Apple Silicon Mac.
 if not re.match(
     r"^(x86_64|aarch64|arm64|powerpc|powerpc64).*-(windows-gnu|windows-msvc|aix)",
diff --git a/llvm/test/tools/dsymutil/ARM/extern-alias.test b/llvm/test/tools/dsymutil/ARM/extern-alias.test
index f8b59584d2b46571b30a6ca50e62c168ee0c0f2b..58a1c6a738257f41db9f5dcfde8b19d1e05812b5 100644
--- a/llvm/test/tools/dsymutil/ARM/extern-alias.test
+++ b/llvm/test/tools/dsymutil/ARM/extern-alias.test
@@ -41,6 +41,6 @@ RUN: dsymutil -oso-prepend-path %p/../Inputs %p/../Inputs/private/tmp/extern/ext
 RUN: dsymutil --linker llvm -oso-prepend-path %p/../Inputs %p/../Inputs/private/tmp/private_extern/private_extern.out -o %t.dSYM --verbose | FileCheck %s
 RUN: dsymutil --linker llvm -oso-prepend-path %p/../Inputs %p/../Inputs/private/tmp/extern/extern.out -o %t.dSYM --verbose | FileCheck %s
 
-CHECK-NOT: could not find object file symbol for symbol _baz
+CHECK-NOT: could not find symbol '_baz'
 CHECK: { sym: _baz, objAddr: 0x0, binAddr: 0x100007F58, size: 0x0 }
 CHECK: { sym: _foo, objAddr: 0x0, binAddr: 0x100007F58, size: 0x20 }
diff --git a/llvm/test/tools/dsymutil/ARM/missing-symbol-warning.test b/llvm/test/tools/dsymutil/ARM/missing-symbol-warning.test
new file mode 100644
index 0000000000000000000000000000000000000000..1de805d32195377e9b2dc5e450f5f5e8525df0e2
--- /dev/null
+++ b/llvm/test/tools/dsymutil/ARM/missing-symbol-warning.test
@@ -0,0 +1,3 @@
+RUN: dsymutil -oso-prepend-path %p/../Inputs --dump-debug-map %p/../Inputs/private/tmp/warning/test.out 2>&1 | FileCheck %s
+# CHECK: could not find symbol '_foo' in object file '{{.*}}test.o'
+# CHECK: { sym: _main, objAddr: 0x0, binAddr: 0x100003F84, size: 0x1C }
diff --git a/llvm/test/tools/dsymutil/ARM/static-archive-collision.test b/llvm/test/tools/dsymutil/ARM/static-archive-collision.test
index 7182f0917d897f2dcc21c6e9f46994354375efb8..55fd5e6b621e761bff0dbb5ac04aaf7785408ad7 100644
--- a/llvm/test/tools/dsymutil/ARM/static-archive-collision.test
+++ b/llvm/test/tools/dsymutil/ARM/static-archive-collision.test
@@ -22,5 +22,5 @@ $ clang main.o foo.a -o main.out
 
 RUN: dsymutil -oso-prepend-path %p/../Inputs %p/../Inputs/private/tmp/collision/main.out --dump-debug-map 2>&1 | FileCheck %s
 CHECK: skipping debug map object with duplicate name and timestamp: {{.*}} /private/tmp/collision/foo.a(foo.o)
-CHECK-NOT: could not find object file symbol for symbol _g
-CHECK-NOT: could not find object file symbol for symbol _f
+CHECK-NOT: could not find symbol '_g'
+CHECK-NOT: could not find symbol '_f'
diff --git a/llvm/test/tools/dsymutil/Inputs/private/tmp/warning/test.o b/llvm/test/tools/dsymutil/Inputs/private/tmp/warning/test.o
new file mode 100644
index 0000000000000000000000000000000000000000..53647992d70c9792c4b85f99a823fc63b340cd75
Binary files /dev/null and b/llvm/test/tools/dsymutil/Inputs/private/tmp/warning/test.o differ
diff --git a/llvm/test/tools/dsymutil/Inputs/private/tmp/warning/test.out b/llvm/test/tools/dsymutil/Inputs/private/tmp/warning/test.out
new file mode 100755
index 0000000000000000000000000000000000000000..bfd5c74f4a1ddc1306841daf286f2d28dcdd1c0b
Binary files /dev/null and b/llvm/test/tools/dsymutil/Inputs/private/tmp/warning/test.out differ
diff --git a/llvm/test/tools/dsymutil/X86/alias.test b/llvm/test/tools/dsymutil/X86/alias.test
index 4c1e8168706098646311304ae125361080c6825a..65e9bf22640a8a03bfdc696b703475e97bd1d6cf 100644
--- a/llvm/test/tools/dsymutil/X86/alias.test
+++ b/llvm/test/tools/dsymutil/X86/alias.test
@@ -4,7 +4,7 @@
 # RUN: dsymutil --linker llvm -f -oso-prepend-path=%p/../Inputs/alias \
 # RUN: %p/../Inputs/alias/foobar -o - | llvm-dwarfdump - 2>&1 | FileCheck %s
 
-# CHECK-NOT: could not find object file symbol for symbol
+# CHECK-NOT: could not find symbol
 # CHECK: DW_AT_name ("foo.c")
 # CHECK: DW_AT_name ("bar.c")
 
diff --git a/llvm/test/tools/dsymutil/X86/thinlto.test b/llvm/test/tools/dsymutil/X86/thinlto.test
index 2d2195805f3aab73bac1e77b03ec9541e0b08fee..d76bfb17e7d5278b8b7a8f360ffad8245454cce0 100644
--- a/llvm/test/tools/dsymutil/X86/thinlto.test
+++ b/llvm/test/tools/dsymutil/X86/thinlto.test
@@ -23,5 +23,5 @@ RUN: dsymutil -oso-prepend-path %p/../Inputs %p/../Inputs/private/tmp/thinlto/fo
 
 RUN: dsymutil --linker llvm -oso-prepend-path %p/../Inputs %p/../Inputs/private/tmp/thinlto/foobar.dylib -o %t.dSYM 2>&1 | FileCheck %s --allow-empty
 
-CHECK-NOT: could not find object file symbol for symbol __ZZ9function2vE12magic_static
-CHECK-NOT: could not find object file symbol for symbol __ZGVZ9function2vE12magic_static
+CHECK-NOT: could not find symbol '__ZZ9function2vE12magic_static'
+CHECK-NOT: could not find symbol 'symbol __ZGVZ9function2vE12magic_static'
diff --git a/llvm/test/tools/llvm-config/system-libs.windows.test b/llvm/test/tools/llvm-config/system-libs.windows.test
index d1d91a9a0b9e2f21bfb29c1808b2355dc2ce305f..597e0a83c3e2e625c05ffb07d61b04c8e0500a97 100644
--- a/llvm/test/tools/llvm-config/system-libs.windows.test
+++ b/llvm/test/tools/llvm-config/system-libs.windows.test
@@ -2,6 +2,6 @@ RUN: llvm-config --link-static --system-libs Support 2>&1 | FileCheck %s
 REQUIRES: static-libs
 REQUIRES: host={{.*-windows-msvc}}
 CHECK-NOT: -l
-CHECK: psapi.lib shell32.lib ole32.lib uuid.lib advapi32.lib
+CHECK: psapi.lib shell32.lib ole32.lib uuid.lib advapi32.lib ws2_32.lib
 CHECK-NOT: error
 CHECK-NOT: warning
diff --git a/llvm/test/tools/llvm-cov/Inputs/binary-formats.canonical.json b/llvm/test/tools/llvm-cov/Inputs/binary-formats.canonical.json
index a30411a84b23ea9cd2090d5ae3f0b8a9b113349b..ce13fc2ff6e34521ed99a664796366534adac683 100644
--- a/llvm/test/tools/llvm-cov/Inputs/binary-formats.canonical.json
+++ b/llvm/test/tools/llvm-cov/Inputs/binary-formats.canonical.json
@@ -1,9 +1,11 @@
-CHECK: {"data":
-CHECK-SAME: [{
+CHECK: {
+  "data": CHECK - SAME: [
+    {
 CHECK-SAME:  "files":[
 CHECK-SAME:    {"branches":[],
 CHECK-SAME:     "expansions":[],
 CHECK-SAME:     "filename":"/tmp/binary-formats.c",
+CHECK-SAME:     "mcdc_records":[],
 CHECK-SAME:     "segments":
 CHECK-SAME:        4,40,100,true,true,false
 CHECK-SAME:        4,42,0,false,false,false
@@ -11,11 +13,13 @@ CHECK-SAME:     "summary":{"branches":{"count":0,"covered":0,"notcovered":0,"per
 CHECK-SAME:                "functions":{"count":1,"covered":1,"percent":100},
 CHECK-SAME:                "instantiations":{"count":1,"covered":1,"percent":100},
 CHECK-SAME:                "lines":{"count":1,"covered":1,"percent":100},
+CHECK-SAME:                "mcdc":{"count":0,"covered":0,"notcovered":0,"percent":0},
 CHECK-SAME:                "regions":{"count":1,"covered":1,"notcovered":0,"percent":100}}}
 CHECK-SAME:  ],
 CHECK-SAME:  "functions":[
 CHECK-SAME:    {"branches":[],
-CHECK-SAME:     "count":100,"filenames":["/tmp/binary-formats.c"],"name":"main",
+CHECK-SAME:     "count":100,"filenames":["/tmp/binary-formats.c"],
+CHECK-SAME:     "mcdc_records":[],"name":"main",
 CHECK-SAME:     "regions":
 CHECK-SAME:        4,40,4,42,100,0,0,0
 CHECK-SAME:    }
@@ -25,6 +29,7 @@ CHECK-SAME:    {"branches":{"count":0,"covered":0,"notcovered":0,"percent":0},
 CHECK-SAME:     "functions":{"count":1,"covered":1,"percent":100},
 CHECK-SAME:     "instantiations":{"count":1,"covered":1,"percent":100},
 CHECK-SAME:     "lines":{"count":1,"covered":1,"percent":100},
+CHECk-SAME:     "mcdc":{"count":0,"covered":0,"notcovered":0,"percent":0},
 CHECK-SAME:     "regions":{"count":1,"covered":1,"notcovered":0,"percent":100}}}
 CHECK-SAME: ],
 CHECK-SAME: "type":"llvm.coverage.json.export"
diff --git a/llvm/test/tools/llvm-cov/Inputs/mcdc-const-folding.cpp b/llvm/test/tools/llvm-cov/Inputs/mcdc-const-folding.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..75f35bd059eb1c28d582e1e9282b821836fc6a4a
--- /dev/null
+++ b/llvm/test/tools/llvm-cov/Inputs/mcdc-const-folding.cpp
@@ -0,0 +1,117 @@
+#include <stdlib.h>
+
+bool case0(bool a) {
+    return 0 && a;
+}
+bool case1(bool a) {
+    return a && 0;
+}
+bool case2(bool a) {
+    return 1 && a;
+}
+bool case3(bool a) {
+    return a && 1;
+}
+bool case4(bool a) {
+    return 1 || a;
+}
+bool case5(bool a) {
+    return a || 1;
+}
+bool case6(bool a) {
+    return 0 || a;
+}
+bool case7(bool a) {
+    return a || 0;
+}
+
+bool case8(bool a, bool b) {
+    return 0 && a && b;
+}
+bool case9(bool a, bool b) {
+    return a && 0 && b;
+}
+bool casea(bool a, bool b) {
+    return 1 && a && b;
+}
+bool caseb(bool a, bool b) {
+    return a && 1 && b;
+}
+bool casec(bool a, bool b) {
+    return 1 || a || b;
+}
+bool cased(bool a, bool b) {
+    return a || 1 || b;
+}
+bool casee(bool a, bool b) {
+    return 0 || a || b;
+}
+bool casef(bool a, bool b) {
+    return a || 0 || b;
+}
+
+bool caseg(bool a, bool b) {
+    return b && a && 0;
+}
+bool caseh(bool a, bool b) {
+    return b && 0 && a;
+}
+bool casei(bool a, bool b) {
+    return b && a && 1;
+}
+bool casej(bool a, bool b) {
+    return b && 1 && a;
+}
+bool casek(bool a, bool b) {
+    return b || a || 1;
+}
+bool casel(bool a, bool b) {
+    return b || 1 || a;
+}
+bool casem(bool a, bool b) {
+    return b || a || 0;
+}
+bool casen(bool a, bool b) {
+    return b || 0 || a;
+}
+
+extern "C" {
+  extern void __llvm_profile_write_file(void);
+}
+
+int main(int argc, char *argv[])
+{
+    bool a = atoi(argv[1]);
+    bool b = atoi(argv[2]);
+    volatile bool c;
+
+    c = case0(a);
+    c = case1(a);
+    c = case2(a);
+    c = case3(a);
+    c = case4(a);
+    c = case5(a);
+    c = case6(a);
+    c = case7(a);
+
+    c = case8(a, b);
+    c = case9(a, b);
+    c = casea(a, b);
+    c = caseb(a, b);
+    c = casec(a, b);
+    c = cased(a, b);
+    c = casee(a, b);
+    c = casef(a, b);
+
+    c = caseg(a, b);
+    c = caseh(a, b);
+    c = casei(a, b);
+    c = casej(a, b);
+    c = casek(a, b);
+    c = casel(a, b);
+    c = casem(a, b);
+    c = casen(a, b);
+
+    __llvm_profile_write_file();
+    return 0;
+}
diff --git a/llvm/test/tools/llvm-cov/Inputs/mcdc-const-folding.o b/llvm/test/tools/llvm-cov/Inputs/mcdc-const-folding.o
new file mode 100755
index 0000000000000000000000000000000000000000..2fb33f3f601a3fe2b7197e921ce93ed15d361cf5
Binary files /dev/null and b/llvm/test/tools/llvm-cov/Inputs/mcdc-const-folding.o differ
diff --git a/llvm/test/tools/llvm-cov/Inputs/mcdc-const-folding.proftext b/llvm/test/tools/llvm-cov/Inputs/mcdc-const-folding.proftext
new file mode 100644
index 0000000000000000000000000000000000000000..eaac8048f0ceb6ccc0ac5a88f52ed8c55f7831ad
--- /dev/null
+++ b/llvm/test/tools/llvm-cov/Inputs/mcdc-const-folding.proftext
@@ -0,0 +1,398 @@
+_Z5case8bb
+# Func Hash:
+99214
+# Num Counters:
+5
+# Counter Values:
+4
+0
+0
+0
+0
+# Num Bitmask Bytes:
+$1
+# Bitmask Byte Values:
+0x1
+
+
+_Z5case5b
+# Func Hash:
+1551
+# Num Counters:
+3
+# Counter Values:
+4
+1
+0
+# Num Bitmask Bytes:
+$1
+# Bitmask Byte Values:
+0x6
+
+
+_Z5caseabb
+# Func Hash:
+99214
+# Num Counters:
+5
+# Counter Values:
+4
+3
+2
+4
+3
+# Num Bitmask Bytes:
+$1
+# Bitmask Byte Values:
+0xa2
+
+
+_Z5case6b
+# Func Hash:
+1551
+# Num Counters:
+3
+# Counter Values:
+4
+4
+1
+# Num Bitmask Bytes:
+$1
+# Bitmask Byte Values:
+0x5
+
+
+_Z5casegbb
+# Func Hash:
+99214
+# Num Counters:
+5
+# Counter Values:
+4
+2
+0
+3
+2
+# Num Bitmask Bytes:
+$1
+# Bitmask Byte Values:
+0x23
+
+
+_Z5case1b
+# Func Hash:
+1550
+# Num Counters:
+3
+# Counter Values:
+4
+3
+0
+# Num Bitmask Bytes:
+$1
+# Bitmask Byte Values:
+0x3
+
+
+_Z5case7b
+# Func Hash:
+1551
+# Num Counters:
+3
+# Counter Values:
+4
+1
+1
+# Num Bitmask Bytes:
+$1
+# Bitmask Byte Values:
+0x3
+
+
+_Z5casedbb
+# Func Hash:
+99279
+# Num Counters:
+5
+# Counter Values:
+4
+0
+0
+1
+0
+# Num Bitmask Bytes:
+$1
+# Bitmask Byte Values:
+0x12
+
+
+_Z5casekbb
+# Func Hash:
+99279
+# Num Counters:
+5
+# Counter Values:
+4
+0
+0
+1
+0
+# Num Bitmask Bytes:
+$1
+# Bitmask Byte Values:
+0x12
+
+
+_Z5casehbb
+# Func Hash:
+99214
+# Num Counters:
+5
+# Counter Values:
+4
+0
+0
+3
+0
+# Num Bitmask Bytes:
+$1
+# Bitmask Byte Values:
+0x3
+
+
+_Z5case4b
+# Func Hash:
+1551
+# Num Counters:
+3
+# Counter Values:
+4
+0
+0
+# Num Bitmask Bytes:
+$1
+# Bitmask Byte Values:
+0x2
+
+
+_Z5caseibb
+# Func Hash:
+99214
+# Num Counters:
+5
+# Counter Values:
+4
+2
+2
+3
+2
+# Num Bitmask Bytes:
+$1
+# Bitmask Byte Values:
+0x83
+
+
+_Z5case2b
+# Func Hash:
+1550
+# Num Counters:
+3
+# Counter Values:
+4
+4
+3
+# Num Bitmask Bytes:
+$1
+# Bitmask Byte Values:
+0xa
+
+
+_Z5casefbb
+# Func Hash:
+99279
+# Num Counters:
+5
+# Counter Values:
+4
+1
+0
+1
+1
+# Num Bitmask Bytes:
+$1
+# Bitmask Byte Values:
+0x6
+
+
+_Z5caselbb
+# Func Hash:
+99279
+# Num Counters:
+5
+# Counter Values:
+4
+0
+0
+1
+0
+# Num Bitmask Bytes:
+$1
+# Bitmask Byte Values:
+0x12
+
+
+_Z5casenbb
+# Func Hash:
+99279
+# Num Counters:
+5
+# Counter Values:
+4
+1
+0
+1
+1
+# Num Bitmask Bytes:
+$1
+# Bitmask Byte Values:
+0x6
+
+
+_Z5case9bb
+# Func Hash:
+99214
+# Num Counters:
+5
+# Counter Values:
+4
+0
+0
+3
+0
+# Num Bitmask Bytes:
+$1
+# Bitmask Byte Values:
+0x3
+
+
+_Z5casecbb
+# Func Hash:
+99279
+# Num Counters:
+5
+# Counter Values:
+4
+0
+0
+0
+0
+# Num Bitmask Bytes:
+$1
+# Bitmask Byte Values:
+0x2
+
+
+_Z5casebbb
+# Func Hash:
+99214
+# Num Counters:
+5
+# Counter Values:
+4
+3
+2
+3
+3
+# Num Bitmask Bytes:
+$1
+# Bitmask Byte Values:
+0xa1
+
+
+_Z5case0b
+# Func Hash:
+1550
+# Num Counters:
+3
+# Counter Values:
+4
+0
+0
+# Num Bitmask Bytes:
+$1
+# Bitmask Byte Values:
+0x1
+
+
+_Z5casejbb
+# Func Hash:
+99214
+# Num Counters:
+5
+# Counter Values:
+4
+3
+2
+3
+3
+# Num Bitmask Bytes:
+$1
+# Bitmask Byte Values:
+0xa1
+
+
+_Z5caseebb
+# Func Hash:
+99279
+# Num Counters:
+5
+# Counter Values:
+4
+1
+0
+4
+1
+# Num Bitmask Bytes:
+$1
+# Bitmask Byte Values:
+0x14
+
+
+main
+# Func Hash:
+24
+# Num Counters:
+1
+# Counter Values:
+4
+
+_Z5case3b
+# Func Hash:
+1550
+# Num Counters:
+3
+# Counter Values:
+4
+3
+3
+# Num Bitmask Bytes:
+$1
+# Bitmask Byte Values:
+0x9
+
+
+_Z5casembb
+# Func Hash:
+99279
+# Num Counters:
+5
+# Counter Values:
+4
+0
+0
+1
+0
+# Num Bitmask Bytes:
+$1
+# Bitmask Byte Values:
+0x12
diff --git a/llvm/test/tools/llvm-cov/Inputs/mcdc-const.cpp b/llvm/test/tools/llvm-cov/Inputs/mcdc-const.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..4a777d11ae1f2d5b3905ead2cb85bd5704cfaddb
--- /dev/null
+++ b/llvm/test/tools/llvm-cov/Inputs/mcdc-const.cpp
@@ -0,0 +1,27 @@
+
+#include <stdio.h>
+
+extern "C" {
+extern void __llvm_profile_write_file(void);
+}
+
+extern int foo();
+
+void test(bool a, bool b, bool c, bool d) {
+
+  if ((a && 1) || (0 && d) || 0)
+    printf("test1 decision true\n");
+}
+
+int main()
+{
+    test(true,false,true,false);
+    test(true,false,true,true);
+    test(true,true,false,false);
+    test(false,true,true,false);
+
+    test(true,false,false,false);
+
+    __llvm_profile_write_file();
+    return 0;
+}
diff --git a/llvm/test/tools/llvm-cov/Inputs/mcdc-const.o b/llvm/test/tools/llvm-cov/Inputs/mcdc-const.o
new file mode 100755
index 0000000000000000000000000000000000000000..092c2a3e8402e0ab27384f0806871f6cf7c786bf
Binary files /dev/null and b/llvm/test/tools/llvm-cov/Inputs/mcdc-const.o differ
diff --git a/llvm/test/tools/llvm-cov/Inputs/mcdc-const.proftext b/llvm/test/tools/llvm-cov/Inputs/mcdc-const.proftext
new file mode 100644
index 0000000000000000000000000000000000000000..86ac65eb779ac21a9b9d93a3a5c7f92c0a5238b7
--- /dev/null
+++ b/llvm/test/tools/llvm-cov/Inputs/mcdc-const.proftext
@@ -0,0 +1,30 @@
+main
+# Func Hash:
+24
+# Num Counters:
+1
+# Counter Values:
+1
+
+_Z4testbbbb
+# Func Hash:
+703556281489
+# Num Counters:
+9
+# Counter Values:
+5
+4
+1
+1
+1
+4
+4
+0
+0
+# Num Bitmask Bytes:
+$4
+# Bitmask Byte Values:
+1
+2
+0
+0
diff --git a/llvm/test/tools/llvm-cov/Inputs/mcdc-general-none.proftext b/llvm/test/tools/llvm-cov/Inputs/mcdc-general-none.proftext
new file mode 100644
index 0000000000000000000000000000000000000000..d929766c12545bae9f88958281bbcef8e48ddeaa
--- /dev/null
+++ b/llvm/test/tools/llvm-cov/Inputs/mcdc-general-none.proftext
@@ -0,0 +1,42 @@
+main
+# Func Hash:
+24
+# Num Counters:
+1
+# Counter Values:
+1
+
+_Z4testbbbb
+# Func Hash:
+9819241276358969079
+# Num Counters:
+19
+# Counter Values:
+7
+3
+5
+5
+2
+3
+1
+2
+3
+2
+1
+1
+1
+1
+2
+4
+2
+2
+1
+# Num Bitmask Bytes:
+$6
+# Bitmask Byte Values:
+0
+0
+0
+0
+0
+0
diff --git a/llvm/test/tools/llvm-cov/Inputs/mcdc-general.cpp b/llvm/test/tools/llvm-cov/Inputs/mcdc-general.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..d1ce365d780742d694ab6385cc0ff1ac7a8d258c
--- /dev/null
+++ b/llvm/test/tools/llvm-cov/Inputs/mcdc-general.cpp
@@ -0,0 +1,36 @@
+
+#include <stdio.h>
+
+extern "C" {
+extern void __llvm_profile_write_file(void);
+}
+
+extern int foo();
+
+void test(bool a, bool b, bool c, bool d) {
+
+  if ((a && b) || (c && d))
+    printf("test1 decision true\n");
+
+  if (b && c) if (a && d)
+    printf("test2 decision true\n");
+
+  if ((c && d) &&
+      (a && b))
+    printf("test3 decision true\n");
+}
+
+int main()
+{
+    test(false,false,false,false);
+    test(true,false,true,false);
+    test(true,false,true,true);
+    test(true,true,false,false);
+
+    test(true,false,false,false);
+    test(true,true,true,true);
+    test(false,true,true,false);
+
+    __llvm_profile_write_file();
+    return 0;
+}
diff --git a/llvm/test/tools/llvm-cov/Inputs/mcdc-general.o b/llvm/test/tools/llvm-cov/Inputs/mcdc-general.o
new file mode 100755
index 0000000000000000000000000000000000000000..4044be1ca3886c82a28c9e51dde79f0fa9549f56
Binary files /dev/null and b/llvm/test/tools/llvm-cov/Inputs/mcdc-general.o differ
diff --git a/llvm/test/tools/llvm-cov/Inputs/mcdc-general.proftext b/llvm/test/tools/llvm-cov/Inputs/mcdc-general.proftext
new file mode 100644
index 0000000000000000000000000000000000000000..eadc9157c72cda8d3f98eb3e9354a763ea35b089
--- /dev/null
+++ b/llvm/test/tools/llvm-cov/Inputs/mcdc-general.proftext
@@ -0,0 +1,42 @@
+main
+# Func Hash:
+24
+# Num Counters:
+1
+# Counter Values:
+1
+
+_Z4testbbbb
+# Func Hash:
+9819241276358969079
+# Num Counters:
+19
+# Counter Values:
+7
+3
+5
+5
+2
+3
+1
+2
+3
+2
+1
+1
+1
+1
+2
+4
+2
+2
+1
+# Num Bitmask Bytes:
+$6
+# Bitmask Byte Values:
+0x2f
+0x8
+0xb
+0x9
+0x83
+0x80
diff --git a/llvm/test/tools/llvm-cov/mcdc-const.test b/llvm/test/tools/llvm-cov/mcdc-const.test
new file mode 100644
index 0000000000000000000000000000000000000000..bf9d34b09da0f637548a939206e27865c906f164
--- /dev/null
+++ b/llvm/test/tools/llvm-cov/mcdc-const.test
@@ -0,0 +1,218 @@
+// Test visualization of MC/DC constructs for constant-folded condition masking.
+
+// RUN: llvm-profdata merge %S/Inputs/mcdc-const.proftext -o %t.profdata
+// RUN: llvm-cov show --show-branches=count --show-mcdc %S/Inputs/mcdc-const.o -instr-profile %t.profdata -path-equivalence=/tmp,%S/Inputs %S/Inputs/mcdc-const.cpp | FileCheck %s -check-prefix=CHECKGENERALCASE
+
+// CHECKGENERALCASE:  ------------------
+// CHECKGENERALCASE-NEXT:  |  Branch (12:8): [True: 4, False: 1]
+// CHECKGENERALCASE-NEXT:  |  Branch (12:13): [Folded - Ignored]
+// CHECKGENERALCASE-NEXT:  |  Branch (12:20): [Folded - Ignored]
+// CHECKGENERALCASE-NEXT:  |  Branch (12:25): [True: 0, False: 0]
+// CHECKGENERALCASE-NEXT:  |  Branch (12:31): [Folded - Ignored]
+// CHECKGENERALCASE-NEXT:  ------------------
+// CHECKGENERALCASE-NEXT:  |---> MC/DC Decision Region (12:7) to (12:32)
+// CHECKGENERALCASE-NEXT:  |
+// CHECKGENERALCASE-NEXT:  |  Number of Conditions: 5
+// CHECKGENERALCASE-NEXT:  |     Condition C1 --> (12:8)
+// CHECKGENERALCASE-NEXT:  |     Condition C2 --> (12:13)
+// CHECKGENERALCASE-NEXT:  |     Condition C3 --> (12:20)
+// CHECKGENERALCASE-NEXT:  |     Condition C4 --> (12:25)
+// CHECKGENERALCASE-NEXT:  |     Condition C5 --> (12:31)
+// CHECKGENERALCASE-NEXT:  |
+// CHECKGENERALCASE-NEXT:  |  Executed MC/DC Test Vectors:
+// CHECKGENERALCASE-NEXT:  |
+// CHECKGENERALCASE-NEXT:  |     C1, C2, C3, C4, C5    Result
+// CHECKGENERALCASE-NEXT:  |  1 { F,  C,  C,  -,  C  = F      }
+// CHECKGENERALCASE-NEXT:  |  2 { T,  C,  C,  -,  C  = T      }
+// CHECKGENERALCASE-NEXT:  |
+// CHECKGENERALCASE-NEXT:  |  C1-Pair: covered: (1,2)
+// CHECKGENERALCASE-NEXT:  |  C2-Pair: constant folded
+// CHECKGENERALCASE-NEXT:  |  C3-Pair: constant folded
+// CHECKGENERALCASE-NEXT:  |  C4-Pair: not covered
+// CHECKGENERALCASE-NEXT:  |  C5-Pair: constant folded
+// CHECKGENERALCASE-NEXT:  |  MC/DC Coverage for Decision: 50.00%
+// CHECKGENERALCASE-NEXT:  |
+// CHECKGENERALCASE-NEXT:  ------------------
+
+// RUN: llvm-profdata merge %S/Inputs/mcdc-const-folding.proftext -o %t.profdata
+// RUN: llvm-cov show --show-mcdc %S/Inputs/mcdc-const-folding.o -instr-profile %t.profdata -path-equivalence=/tmp,%S/Inputs %S/Inputs/mcdc-const-folding.cpp | FileCheck %s -check-prefix=CHECKFULLCASE
+// RUN: llvm-cov report --show-mcdc-summary %S/Inputs/mcdc-const-folding.o -instr-profile %t.profdata -show-functions -path-equivalence=/tmp,%S/Inputs %S/Inputs/mcdc-const-folding.cpp | FileCheck %s -check-prefix=REPORT
+
+// CHECKFULLCASE: |  1 { C,  -  = F      }
+// CHECKFULLCASE: |  C1-Pair: constant folded
+// CHECKFULLCASE-NEXT: |  C2-Pair: not covered
+// CHECKFULLCASE: |  MC/DC Coverage for Decision: 0.00%
+// CHECKFULLCASE: |  1 { F,  C  = F      }
+// CHECKFULLCASE-NEXT: |  2 { T,  C  = F      }
+// CHECKFULLCASE: |  C1-Pair: not covered
+// CHECKFULLCASE-NEXT: |  C2-Pair: constant folded
+// CHECKFULLCASE: |  MC/DC Coverage for Decision: 0.00%
+// CHECKFULLCASE: |  1 { C,  F  = F      }
+// CHECKFULLCASE-NEXT: |  2 { C,  T  = T      }
+// CHECKFULLCASE: |  C1-Pair: constant folded
+// CHECKFULLCASE-NEXT: |  C2-Pair: covered: (1,2)
+// CHECKFULLCASE: |  MC/DC Coverage for Decision: 100.00%
+// CHECKFULLCASE: |  1 { F,  C  = F      }
+// CHECKFULLCASE-NEXT: |  2 { T,  C  = T      }
+// CHECKFULLCASE: |  C1-Pair: covered: (1,2)
+// CHECKFULLCASE-NEXT: |  C2-Pair: constant folded
+// CHECKFULLCASE: |  MC/DC Coverage for Decision: 100.00%
+// CHECKFULLCASE: |  1 { C,  -  = T      }
+// CHECKFULLCASE: |  C1-Pair: constant folded
+// CHECKFULLCASE-NEXT: |  C2-Pair: not covered
+// CHECKFULLCASE: |  MC/DC Coverage for Decision: 0.00%
+// CHECKFULLCASE: |  1 { T,  C  = T      }
+// CHECKFULLCASE-NEXT: |  2 { F,  C  = T      }
+// CHECKFULLCASE: |  C1-Pair: not covered
+// CHECKFULLCASE-NEXT: |  C2-Pair: constant folded
+// CHECKFULLCASE: |  MC/DC Coverage for Decision: 0.00%
+// CHECKFULLCASE: |  1 { C,  F  = F      }
+// CHECKFULLCASE-NEXT: |  2 { C,  T  = T      }
+// CHECKFULLCASE: |  C1-Pair: constant folded
+// CHECKFULLCASE-NEXT: |  C2-Pair: covered: (1,2)
+// CHECKFULLCASE: |  MC/DC Coverage for Decision: 100.00%
+// CHECKFULLCASE: |  1 { F,  C  = F      }
+// CHECKFULLCASE-NEXT: |  2 { T,  C  = T      }
+// CHECKFULLCASE: |  C1-Pair: covered: (1,2)
+// CHECKFULLCASE-NEXT: |  C2-Pair: constant folded
+// CHECKFULLCASE: |  MC/DC Coverage for Decision: 100.00%
+// CHECKFULLCASE: |  1 { C,  -,  -  = F      }
+// CHECKFULLCASE: |  C1-Pair: constant folded
+// CHECKFULLCASE-NEXT: |  C2-Pair: not covered
+// CHECKFULLCASE-NEXT: |  C3-Pair: not covered
+// CHECKFULLCASE: |  MC/DC Coverage for Decision: 0.00%
+// CHECKFULLCASE: |  1 { F,  C,  -  = F      }
+// CHECKFULLCASE-NEXT: |  2 { T,  C,  -  = F      }
+// CHECKFULLCASE: |  C1-Pair: not covered
+// CHECKFULLCASE-NEXT: |  C2-Pair: constant folded
+// CHECKFULLCASE-NEXT: |  C3-Pair: not covered
+// CHECKFULLCASE: |  MC/DC Coverage for Decision: 0.00%
+// CHECKFULLCASE: |  1 { C,  F,  -  = F      }
+// CHECKFULLCASE-NEXT: |  2 { C,  T,  F  = F      }
+// CHECKFULLCASE-NEXT: |  3 { C,  T,  T  = T      }
+// CHECKFULLCASE: |  C1-Pair: constant folded
+// CHECKFULLCASE-NEXT: |  C2-Pair: covered: (1,3)
+// CHECKFULLCASE-NEXT: |  C3-Pair: covered: (2,3)
+// CHECKFULLCASE: |  MC/DC Coverage for Decision: 100.00%
+// CHECKFULLCASE: |  1 { F,  C,  -  = F      }
+// CHECKFULLCASE-NEXT: |  2 { T,  C,  F  = F      }
+// CHECKFULLCASE-NEXT: |  3 { T,  C,  T  = T      }
+// CHECKFULLCASE: |  C1-Pair: covered: (1,3)
+// CHECKFULLCASE-NEXT: |  C2-Pair: constant folded
+// CHECKFULLCASE-NEXT: |  C3-Pair: covered: (2,3)
+// CHECKFULLCASE: |  MC/DC Coverage for Decision: 100.00%
+// CHECKFULLCASE: |  1 { C,  -,  -  = T      }
+// CHECKFULLCASE: |  C1-Pair: constant folded
+// CHECKFULLCASE-NEXT: |  C2-Pair: not covered
+// CHECKFULLCASE-NEXT: |  C3-Pair: not covered
+// CHECKFULLCASE: |  MC/DC Coverage for Decision: 0.00%
+// CHECKFULLCASE: |  1 { T,  C,  -  = T      }
+// CHECKFULLCASE-NEXT: |  2 { F,  C,  -  = T      }
+// CHECKFULLCASE: |  C1-Pair: not covered
+// CHECKFULLCASE-NEXT: |  C2-Pair: constant folded
+// CHECKFULLCASE-NEXT: |  C3-Pair: not covered
+// CHECKFULLCASE: |  MC/DC Coverage for Decision: 0.00%
+// CHECKFULLCASE: |  1 { C,  F,  T  = T      }
+// CHECKFULLCASE-NEXT: |  2 { C,  T,  -  = T      }
+// CHECKFULLCASE: |  C1-Pair: constant folded
+// CHECKFULLCASE-NEXT: |  C2-Pair: not covered
+// CHECKFULLCASE-NEXT: |  C3-Pair: not covered
+// CHECKFULLCASE: |  MC/DC Coverage for Decision: 0.00%
+// CHECKFULLCASE: |  1 { T,  C,  -  = T      }
+// CHECKFULLCASE-NEXT: |  2 { F,  C,  T  = T      }
+// CHECKFULLCASE: |  C1-Pair: not covered
+// CHECKFULLCASE-NEXT: |  C2-Pair: constant folded
+// CHECKFULLCASE-NEXT: |  C3-Pair: not covered
+// CHECKFULLCASE: |  MC/DC Coverage for Decision: 0.00%
+// CHECKFULLCASE: |  1 { F,  -,  C  = F      }
+// CHECKFULLCASE-NEXT: |  2 { T,  F,  C  = F      }
+// CHECKFULLCASE-NEXT: |  3 { T,  T,  C  = F      }
+// CHECKFULLCASE: |  C1-Pair: not covered
+// CHECKFULLCASE-NEXT: |  C2-Pair: not covered
+// CHECKFULLCASE-NEXT: |  C3-Pair: constant folded
+// CHECKFULLCASE: |  MC/DC Coverage for Decision: 0.00%
+// CHECKFULLCASE: |  1 { F,  C,  -  = F      }
+// CHECKFULLCASE-NEXT: |  2 { T,  C,  -  = F      }
+// CHECKFULLCASE: |  C1-Pair: not covered
+// CHECKFULLCASE-NEXT: |  C2-Pair: constant folded
+// CHECKFULLCASE-NEXT: |  C3-Pair: not covered
+// CHECKFULLCASE: |  MC/DC Coverage for Decision: 0.00%
+// CHECKFULLCASE: |  1 { F,  -,  C  = F      }
+// CHECKFULLCASE-NEXT: |  2 { T,  F,  C  = F      }
+// CHECKFULLCASE-NEXT: |  3 { T,  T,  C  = T      }
+// CHECKFULLCASE: |  C1-Pair: covered: (1,3)
+// CHECKFULLCASE-NEXT: |  C2-Pair: covered: (2,3)
+// CHECKFULLCASE-NEXT: |  C3-Pair: constant folded
+// CHECKFULLCASE: |  MC/DC Coverage for Decision: 100.00%
+// CHECKFULLCASE: |  1 { F,  C,  -  = F      }
+// CHECKFULLCASE-NEXT: |  2 { T,  C,  F  = F      }
+// CHECKFULLCASE-NEXT: |  3 { T,  C,  T  = T      }
+// CHECKFULLCASE: |  C1-Pair: covered: (1,3)
+// CHECKFULLCASE-NEXT: |  C2-Pair: constant folded
+// CHECKFULLCASE-NEXT: |  C3-Pair: covered: (2,3)
+// CHECKFULLCASE: |  MC/DC Coverage for Decision: 100.00%
+// CHECKFULLCASE: |  1 { T,  -,  C  = T      }
+// CHECKFULLCASE-NEXT: |  2 { F,  T,  C  = T      }
+// CHECKFULLCASE: |  C1-Pair: not covered
+// CHECKFULLCASE-NEXT: |  C2-Pair: not covered
+// CHECKFULLCASE-NEXT: |  C3-Pair: constant folded
+// CHECKFULLCASE: |  MC/DC Coverage for Decision: 0.00%
+// CHECKFULLCASE: |  1 { T,  C,  -  = T      }
+// CHECKFULLCASE-NEXT: |  2 { F,  C,  -  = T      }
+// CHECKFULLCASE: |  C1-Pair: not covered
+// CHECKFULLCASE-NEXT: |  C2-Pair: constant folded
+// CHECKFULLCASE-NEXT: |  C3-Pair: not covered
+// CHECKFULLCASE: |  MC/DC Coverage for Decision: 0.00%
+// CHECKFULLCASE: |  1 { T,  -,  C  = T      }
+// CHECKFULLCASE-NEXT: |  2 { F,  T,  C  = T      }
+// CHECKFULLCASE: |  C1-Pair: not covered
+// CHECKFULLCASE-NEXT: |  C2-Pair: not covered
+// CHECKFULLCASE-NEXT: |  C3-Pair: constant folded
+// CHECKFULLCASE: |  MC/DC Coverage for Decision: 0.00%
+// CHECKFULLCASE: |  1 { T,  C,  -  = T      }
+// CHECKFULLCASE-NEXT: |  2 { F,  C,  T  = T      }
+// CHECKFULLCASE: |  C1-Pair: not covered
+// CHECKFULLCASE-NEXT: |  C2-Pair: constant folded
+// CHECKFULLCASE-NEXT: |  C3-Pair: not covered
+// CHECKFULLCASE: |  MC/DC Coverage for Decision: 0.00%
+
+// REPORT: _Z5case0b {{.*}} 1       1   0.00%
+// REPORT-NEXT: _Z5case1b {{.*}} 1       1   0.00%
+// REPORT-NEXT: _Z5case2b {{.*}} 1       0 100.00%
+// REPORT-NEXT: _Z5case3b {{.*}} 1       0 100.00%
+// REPORT-NEXT: _Z5case4b {{.*}} 1       1   0.00%
+// REPORT-NEXT: _Z5case5b {{.*}} 1       1   0.00%
+// REPORT-NEXT: _Z5case6b {{.*}} 1       0 100.00%
+// REPORT-NEXT: _Z5case7b {{.*}} 1       0 100.00%
+// REPORT-NEXT: _Z5case8bb {{.*}} 2       2   0.00%
+// REPORT-NEXT: _Z5case9bb {{.*}} 2       2   0.00%
+// REPORT-NEXT: _Z5caseabb {{.*}} 2       0 100.00%
+// REPORT-NEXT: _Z5casebbb {{.*}} 2       0 100.00%
+// REPORT-NEXT: _Z5casecbb {{.*}} 2       2   0.00%
+// REPORT-NEXT: _Z5casedbb {{.*}} 2       2   0.00%
+// REPORT-NEXT: _Z5caseebb {{.*}} 2       2   0.00%
+// REPORT-NEXT: _Z5casefbb {{.*}} 2       2   0.00%
+// REPORT-NEXT: _Z5casegbb {{.*}} 2       2   0.00%
+// REPORT-NEXT: _Z5casehbb {{.*}} 2       2   0.00%
+// REPORT-NEXT: _Z5caseibb {{.*}} 2       0 100.00%
+// REPORT-NEXT: _Z5casejbb {{.*}} 2       0 100.00%
+// REPORT-NEXT: _Z5casekbb {{.*}} 2       2   0.00%
+// REPORT-NEXT: _Z5caselbb {{.*}} 2       2   0.00%
+// REPORT-NEXT: _Z5casembb {{.*}} 2       2   0.00%
+// REPORT-NEXT: _Z5casenbb {{.*}} 2       2   0.00%
+// REPORT: TOTAL {{.*}} 40      28  30.00%
+
+Instructions for regenerating the test:
+
+# cd %S/Inputs
+cp mcdc-const.cpp /tmp
+cp mcdc-const-folding.cpp /tmp
+
+clang -fcoverage-mcdc -fprofile-instr-generate -fcoverage-compilation-dir=. \
+    -fcoverage-mapping /tmp/mcdc-const.cpp -o /tmp/mcdc-const.o
+
+clang -fcoverage-mcdc -fprofile-instr-generate -fcoverage-compilation-dir=. \
+    -fcoverage-mapping /tmp/mcdc-const-folding.cpp -o /tmp/mcdc-const-folding.o
+
+mv /tmp/mcdc-const.o %S/Inputs
+mv /tmp/mcdc-const-folding.o %S/Inputs
diff --git a/llvm/test/tools/llvm-cov/mcdc-export-json.test b/llvm/test/tools/llvm-cov/mcdc-export-json.test
new file mode 100644
index 0000000000000000000000000000000000000000..c2cebf52db8cc55730f054fdae9588597d5b57ae
--- /dev/null
+++ b/llvm/test/tools/llvm-cov/mcdc-export-json.test
@@ -0,0 +1,18 @@
+// RUN: llvm-profdata merge %S/Inputs/mcdc-general.proftext -o %t.profdata
+// RUN: llvm-cov export --format=text %S/Inputs/mcdc-general.o -instr-profile %t.profdata | FileCheck %s
+
+// CHECK: 12,7,12,27,0,5,[true,true,true,true]
+// CHECK: 15,7,15,13,0,5,[true,true]
+// CHECK: 15,19,15,25,0,5,[true,false]
+// CHECK: 18,7,19,15,0,5,[true,true,false,true]
+// CHECK: "mcdc":{"count":12,"covered":10,"notcovered":2,"percent":83.333333333333343}
+
+Instructions for regenerating the test:
+
+# cd %S/Inputs
+cp mcdc-general.cpp /tmp
+
+clang -fcoverage-mcdc -fprofile-instr-generate -fcoverage-compilation-dir=. \
+    -fcoverage-mapping /tmp/mcdc-general.cpp -o /tmp/mcdc-const.o
+
+mv /tmp/mcdc-general.o %S/Inputs
diff --git a/llvm/test/tools/llvm-cov/mcdc-general-none.test b/llvm/test/tools/llvm-cov/mcdc-general-none.test
new file mode 100644
index 0000000000000000000000000000000000000000..bcf8f3cbd05d45e618061cab92614ce2b0559ea0
--- /dev/null
+++ b/llvm/test/tools/llvm-cov/mcdc-general-none.test
@@ -0,0 +1,82 @@
+// Test visualization of general MC/DC constructs with 0 executed test vectors.
+
+// RUN: llvm-profdata merge %S/Inputs/mcdc-general-none.proftext -o %t.profdata
+// RUN: llvm-cov show --show-mcdc %S/Inputs/mcdc-general.o -instr-profile %t.profdata -path-equivalence=/tmp,%S/Inputs %S/Inputs/mcdc-general.cpp | FileCheck %s
+// RUN: llvm-cov report --show-mcdc-summary %S/Inputs/mcdc-general.o -instr-profile %t.profdata -show-functions -path-equivalence=/tmp,%S/Inputs %S/Inputs/mcdc-general.cpp | FileCheck %s -check-prefix=REPORT
+
+// CHECK: test(bool
+
+// CHECK:  ------------------
+// CHECK-NEXT:  |---> MC/DC Decision Region (12:7) to (12:27)
+// CHECK-NEXT:  |
+// CHECK-NEXT:  |  Number of Conditions: 4
+// CHECK-NEXT:  |     Condition C1 --> (12:8)
+// CHECK-NEXT:  |     Condition C2 --> (12:13)
+// CHECK-NEXT:  |     Condition C3 --> (12:20)
+// CHECK-NEXT:  |     Condition C4 --> (12:25)
+// CHECK-NEXT:  |
+// CHECK-NEXT:  |  Executed MC/DC Test Vectors:
+// CHECK-NEXT:  |
+// CHECK-NEXT:  |  None.
+// CHECK-NEXT:  |
+// CHECK-NEXT:  |  C1-Pair: not covered
+// CHECK-NEXT:  |  C2-Pair: not covered
+// CHECK-NEXT:  |  C3-Pair: not covered
+// CHECK-NEXT:  |  C4-Pair: not covered
+// CHECK-NEXT:  |  MC/DC Coverage for Decision: 0.00%
+// CHECK-NEXT:  |
+// CHECK-NEXT:  ------------------
+
+
+// Turn off MC/DC visualization.
+// RUN: llvm-cov show %S/Inputs/mcdc-general.o -instr-profile %t.profdata -path-equivalence=/tmp,%S/Inputs %S/Inputs/mcdc-general.cpp | FileCheck %s -check-prefix=NOMCDC
+// NOMCDC-NOT: MC/DC Decision Region
+
+// REPORT: Name                        Regions    Miss   Cover     Lines    Miss   Cover  Branches    Miss   Cover    MC/DC Conditions    Miss   Cover
+// REPORT-NEXT: -------------------------------------------------------------------------------------------------------------------------------------------
+// REPORT-NEXT: _Z4testbbbb                      25       0 100.00%         9       0 100.00%        24       2  91.67%                  12      12   0.00%
+// REPORT-NEXT: main                              1       0 100.00%        11       0 100.00%         0       0   0.00%                   0       0   0.00%
+// REPORT-NEXT: ---
+// REPORT-NEXT: TOTAL                            26       0 100.00%        20       0 100.00%        24       2  91.67%                  12      12   0.00%
+
+// Turn off MC/DC summary.
+// RUN: llvm-cov report %S/Inputs/mcdc-general.o -instr-profile %t.profdata -show-functions -path-equivalence=/tmp,%S/Inputs %S/Inputs/mcdc-general.cpp | FileCheck %s -check-prefix=REPORT_NOMCDC
+// REPORT_NOMCDC-NOT: TOTAL{{.*}}12                   12    0.00%
+
+
+// Test file-level report.
+// RUN: llvm-cov report --show-mcdc-summary %S/Inputs/mcdc-general.o -instr-profile %t.profdata -path-equivalence=/tmp,%S/Inputs %S/Inputs/mcdc-general.cpp | FileCheck %s -check-prefix=FILEREPORT
+// FILEREPORT: TOTAL{{.*}}12                   12    0.00%
+
+
+// Test html output.
+// RUN: llvm-cov show --show-mcdc-summary --show-mcdc %S/Inputs/mcdc-general.o -instr-profile %t.profdata -path-equivalence=/tmp,%S/Inputs %S/Inputs/mcdc-general.cpp -format html -o %t.html.dir
+// RUN: FileCheck -check-prefix=HTML -input-file=%t.html.dir/coverage/tmp/mcdc-general.cpp.html %s
+// HTML-COUNT-4: MC/DC Decision Region (<span class='line-number'><a name='L{{[0-9]+}}' href='#L{{[0-9]+}}'><span>
+
+// RUN: FileCheck -check-prefix HTML-INDEX -input-file %t.html.dir/index.html %s
+// HTML-INDEX-LABEL: <table>
+// HTML-INDEX: <td class='column-entry-bold'>Filename</td>
+// HTML-INDEX: <td class='column-entry-bold'>Function Coverage</td>
+// HTML-INDEX: <td class='column-entry-bold'>Line Coverage</td>
+// HTML-INDEX: <td class='column-entry-bold'>Region Coverage</td>
+// HTML-INDEX: <td class='column-entry-bold'>Branch Coverage</td>
+// HTML-INDEX: <td class='column-entry-bold'>MC/DC</td>
+// HTML-INDEX: <a href='coverage{{.*}}mcdc-general.cpp.html'{{.*}}mcdc-general.cpp</a>
+// HTML-INDEX: <td class='column-entry-green'>
+// HTML-INDEX: 100.00% (2/2)
+// HTML-INDEX: 100.00% (20/20)
+// HTML-INDEX: 100.00% (26/26)
+// HTML-INDEX: 91.67% (22/24)
+// HTML-INDEX: 0.00% (0/12)
+// HTML-INDEX: Totals
+
+Instructions for regenerating the test:
+
+# cd %S/Inputs
+cp mcdc-general.cpp /tmp
+
+clang -fcoverage-mcdc -fprofile-instr-generate -fcoverage-compilation-dir=. \
+    -fcoverage-mapping /tmp/mcdc-general.cpp -o /tmp/mcdc-general.o
+
+mv /tmp/mcdc-general.o %S/Inputs
diff --git a/llvm/test/tools/llvm-cov/mcdc-general.test b/llvm/test/tools/llvm-cov/mcdc-general.test
new file mode 100644
index 0000000000000000000000000000000000000000..588aed09c16a5e3d34591933b6cb91a027e65896
--- /dev/null
+++ b/llvm/test/tools/llvm-cov/mcdc-general.test
@@ -0,0 +1,148 @@
+// Test visualization of general MC/DC constructs.
+
+// RUN: llvm-profdata merge %S/Inputs/mcdc-general.proftext -o %t.profdata
+// RUN: llvm-cov show --show-mcdc %S/Inputs/mcdc-general.o -instr-profile %t.profdata -path-equivalence=/tmp,%S/Inputs %S/Inputs/mcdc-general.cpp | FileCheck %s
+// RUN: llvm-cov report --show-mcdc-summary %S/Inputs/mcdc-general.o -instr-profile %t.profdata -show-functions -path-equivalence=/tmp,%S/Inputs %S/Inputs/mcdc-general.cpp | FileCheck %s -check-prefix=REPORT
+
+// CHECK: test(bool
+
+// CHECK:  ------------------
+// CHECK-NEXT:  |---> MC/DC Decision Region (12:7) to (12:27)
+// CHECK-NEXT:  |
+// CHECK-NEXT:  |  Number of Conditions: 4
+// CHECK-NEXT:  |     Condition C1 --> (12:8)
+// CHECK-NEXT:  |     Condition C2 --> (12:13)
+// CHECK-NEXT:  |     Condition C3 --> (12:20)
+// CHECK-NEXT:  |     Condition C4 --> (12:25)
+// CHECK-NEXT:  |
+// CHECK-NEXT:  |  Executed MC/DC Test Vectors:
+// CHECK-NEXT:  |
+// CHECK-NEXT:  |     C1, C2, C3, C4    Result
+// CHECK-NEXT:  |  1 { F,  -,  F,  -  = F      }
+// CHECK-NEXT:  |  2 { T,  F,  F,  -  = F      }
+// CHECK-NEXT:  |  3 { F,  -,  T,  F  = F      }
+// CHECK-NEXT:  |  4 { T,  F,  T,  F  = F      }
+// CHECK-NEXT:  |  5 { T,  T,  -,  -  = T      }
+// CHECK-NEXT:  |  6 { T,  F,  T,  T  = T      }
+// CHECK-NEXT:  |
+// CHECK-NEXT:  |  C1-Pair: covered: (1,5)
+// CHECK-NEXT:  |  C2-Pair: covered: (2,5)
+// CHECK-NEXT:  |  C3-Pair: covered: (2,6)
+// CHECK-NEXT:  |  C4-Pair: covered: (4,6)
+// CHECK-NEXT:  |  MC/DC Coverage for Decision: 100.00%
+// CHECK-NEXT:  |
+// CHECK-NEXT:  ------------------
+
+// CHECK:  ------------------
+// CHECK-NEXT:  |---> MC/DC Decision Region (15:7) to (15:13)
+// CHECK-NEXT:  |
+// CHECK-NEXT:  |  Number of Conditions: 2
+// CHECK-NEXT:  |     Condition C1 --> (15:7)
+// CHECK-NEXT:  |     Condition C2 --> (15:12)
+// CHECK-NEXT:  |
+// CHECK-NEXT:  |  Executed MC/DC Test Vectors:
+// CHECK-NEXT:  |
+// CHECK-NEXT:  |     C1, C2    Result
+// CHECK-NEXT:  |  1 { F,  -  = F      }
+// CHECK-NEXT:  |  2 { T,  F  = F      }
+// CHECK-NEXT:  |  3 { T,  T  = T      }
+// CHECK-NEXT:  |
+// CHECK-NEXT:  |  C1-Pair: covered: (1,3)
+// CHECK-NEXT:  |  C2-Pair: covered: (2,3)
+// CHECK-NEXT:  |  MC/DC Coverage for Decision: 100.00%
+// CHECK-NEXT:  |
+// CHECK-NEXT:  |---> MC/DC Decision Region (15:19) to (15:25)
+// CHECK-NEXT:  |
+// CHECK-NEXT:  |  Number of Conditions: 2
+// CHECK-NEXT:  |     Condition C1 --> (15:19)
+// CHECK-NEXT:  |     Condition C2 --> (15:24)
+// CHECK-NEXT:  |
+// CHECK-NEXT:  |  Executed MC/DC Test Vectors:
+// CHECK-NEXT:  |
+// CHECK-NEXT:  |     C1, C2    Result
+// CHECK-NEXT:  |  1 { F,  -  = F      }
+// CHECK-NEXT:  |  2 { T,  T  = T      }
+// CHECK-NEXT:  |
+// CHECK-NEXT:  |  C1-Pair: covered: (1,2)
+// CHECK-NEXT:  |  C2-Pair: not covered
+// CHECK-NEXT:  |  MC/DC Coverage for Decision: 50.00%
+// CHECK-NEXT:  |
+// CHECK-NEXT:  ------------------
+
+// CHECK:  ------------------
+// CHECK-NEXT:  |---> MC/DC Decision Region (18:7) to (19:15)
+// CHECK-NEXT:  |
+// CHECK-NEXT:  |  Number of Conditions: 4
+// CHECK-NEXT:  |     Condition C1 --> (18:8)
+// CHECK-NEXT:  |     Condition C2 --> (18:13)
+// CHECK-NEXT:  |     Condition C3 --> (19:8)
+// CHECK-NEXT:  |     Condition C4 --> (19:13)
+// CHECK-NEXT:  |
+// CHECK-NEXT:  |  Executed MC/DC Test Vectors:
+// CHECK-NEXT:  |
+// CHECK-NEXT:  |     C1, C2, C3, C4    Result
+// CHECK-NEXT:  |  1 { F,  -,  -,  -  = F      }
+// CHECK-NEXT:  |  2 { T,  F,  -,  -  = F      }
+// CHECK-NEXT:  |  3 { T,  T,  T,  F  = F      }
+// CHECK-NEXT:  |  4 { T,  T,  T,  T  = T      }
+// CHECK-NEXT:  |
+// CHECK-NEXT:  |  C1-Pair: covered: (1,4)
+// CHECK-NEXT:  |  C2-Pair: covered: (2,4)
+// CHECK-NEXT:  |  C3-Pair: not covered
+// CHECK-NEXT:  |  C4-Pair: covered: (3,4)
+// CHECK-NEXT:  |  MC/DC Coverage for Decision: 75.00%
+// CHECK-NEXT:  |
+// CHECK-NEXT:  ------------------
+
+// Turn off MC/DC visualization.
+// RUN: llvm-cov show %S/Inputs/mcdc-general.o -instr-profile %t.profdata -path-equivalence=/tmp,%S/Inputs %S/Inputs/mcdc-general.cpp | FileCheck %s -check-prefix=NOMCDC
+// NOMCDC-NOT: MC/DC Decision Region
+
+// REPORT: Name                        Regions    Miss   Cover     Lines    Miss   Cover  Branches    Miss   Cover    MC/DC Conditions    Miss   Cover
+// REPORT-NEXT: -------------------------------------------------------------------------------------------------------------------------------------------
+// REPORT-NEXT: _Z4testbbbb                      25       0 100.00%         9       0 100.00%        24       2  91.67%                  12       2  83.33%
+// REPORT-NEXT: main                              1       0 100.00%        11       0 100.00%         0       0   0.00%                   0       0   0.00%
+// REPORT-NEXT: ---
+// REPORT-NEXT: TOTAL                            26       0 100.00%        20       0 100.00%        24       2  91.67%                  12       2  83.33%
+
+// Turn off MC/DC summary.
+// RUN: llvm-cov report %S/Inputs/mcdc-general.o -instr-profile %t.profdata -show-functions -path-equivalence=/tmp,%S/Inputs %S/Inputs/mcdc-general.cpp | FileCheck %s -check-prefix=REPORT_NOMCDC
+// REPORT_NOMCDC-NOT: TOTAL{{.*}}12                    2    83.33%
+
+
+// Test file-level report.
+// RUN: llvm-cov report --show-mcdc-summary %S/Inputs/mcdc-general.o -instr-profile %t.profdata -path-equivalence=/tmp,%S/Inputs %S/Inputs/mcdc-general.cpp | FileCheck %s -check-prefix=FILEREPORT
+// FILEREPORT: TOTAL{{.*}}12                    2    83.33%
+
+
+// Test html output.
+// RUN: llvm-cov show --show-mcdc-summary --show-mcdc %S/Inputs/mcdc-general.o -instr-profile %t.profdata -path-equivalence=/tmp,%S/Inputs %S/Inputs/mcdc-general.cpp -format html -o %t.html.dir
+// RUN: FileCheck -check-prefix=HTML -input-file=%t.html.dir/coverage/tmp/mcdc-general.cpp.html %s
+// HTML-COUNT-4: MC/DC Decision Region (<span class='line-number'><a name='L{{[0-9]+}}' href='#L{{[0-9]+}}'><span>
+
+// RUN: FileCheck -check-prefix HTML-INDEX -input-file %t.html.dir/index.html %s
+// HTML-INDEX-LABEL: <table>
+// HTML-INDEX: <td class='column-entry-bold'>Filename</td>
+// HTML-INDEX: <td class='column-entry-bold'>Function Coverage</td>
+// HTML-INDEX: <td class='column-entry-bold'>Line Coverage</td>
+// HTML-INDEX: <td class='column-entry-bold'>Region Coverage</td>
+// HTML-INDEX: <td class='column-entry-bold'>Branch Coverage</td>
+// HTML-INDEX: <td class='column-entry-bold'>MC/DC</td>
+// HTML-INDEX: <a href='coverage{{.*}}mcdc-general.cpp.html'{{.*}}mcdc-general.cpp</a>
+// HTML-INDEX: <td class='column-entry-green'>
+// HTML-INDEX: 100.00% (2/2)
+// HTML-INDEX: 100.00% (20/20)
+// HTML-INDEX: 100.00% (26/26)
+// HTML-INDEX: 91.67% (22/24)
+// HTML-INDEX: 83.33% (10/12)
+// HTML-INDEX: Totals
+
+Instructions for regenerating the test:
+
+# cd %S/Inputs
+cp mcdc-general.cpp /tmp
+
+clang -fcoverage-mcdc -fprofile-instr-generate -fcoverage-compilation-dir=. \
+    -fcoverage-mapping /tmp/mcdc-general.cpp -o /tmp/mcdc-general.o
+
+mv /tmp/mcdc-general.o %S/Inputs
diff --git a/llvm/test/tools/llvm-objdump/XCOFF/disassemble-abs.test b/llvm/test/tools/llvm-objdump/XCOFF/disassemble-abs.test
new file mode 100644
index 0000000000000000000000000000000000000000..c1b76e1552847351f382b60b05b55dd5e2966726
--- /dev/null
+++ b/llvm/test/tools/llvm-objdump/XCOFF/disassemble-abs.test
@@ -0,0 +1,158 @@
+## Object files assembled on AIX from the following source:
+##        .csect [PR]
+##.main:
+##        .globl .main
+##        .extern .function
+##        bla     .function
+##        btla    .function
+##        ba      0x1234
+##        ba      -32
+##        bta     0x2348
+##        bta     -256
+#
+# RUN: yaml2obj %s --docnum=1 -o %t32.o 
+# RUN: llvm-objdump -d %t32.o | FileCheck %s
+#
+# CHECK:        : file format aixcoff-rs6000
+# CHECK:        Disassembly of section .text:
+# CHECK:        00000000 <.main>:
+# CHECK:            0: 48 00 00 03   bla 0x0
+# CHECK-NEXT:       4: 41 80 00 03   btla    0, 0x0
+# CHECK-NEXT:       8: 48 00 12 36   ba 0x1234
+# CHECK-NEXT:       c: 4b ff ff e2   ba 0xffffffe0
+# CHECK-NEXT:      10: 41 80 23 4a   bta     0, 0x2348
+# CHECK-NEXT:      14: 41 80 ff 02   bta     0, 0xffffff00
+
+--- !XCOFF
+FileHeader:
+  MagicNumber:     0x1DF
+  NumberOfSections: 1
+  CreationTime:    1700148622
+  OffsetToSymbolTable: 0x68
+  EntriesInSymbolTable: 8
+  AuxiliaryHeaderSize: 0
+  Flags:           0x0
+Sections:
+  - Name:            .text
+    Address:         0x0
+    Size:            0x18
+    FileOffsetToData: 0x3C
+    FileOffsetToRelocations: 0x54
+    FileOffsetToLineNumbers: 0x0
+    NumberOfRelocations: 0x2
+    NumberOfLineNumbers: 0x0
+    Flags:           [ STYP_TEXT ]
+    SectionData:     4800000341800003480012364BFFFFE24180234A4180FF02
+    Relocations:
+      - Address:         0x0
+        Symbol:          0x2
+        Info:            0x99
+        Type:            0x18
+      - Address:         0x6
+        Symbol:          0x2
+        Info:            0x8F
+        Type:            0x18
+Symbols:
+  - Name:            .file
+    Value:           0x0
+    Section:         N_DEBUG
+    Type:            0xC03
+    StorageClass:    C_FILE
+    NumberOfAuxEntries: 1
+  - Name:            .function
+    Value:           0x0
+    Section:         N_UNDEF
+    Type:            0x0
+    StorageClass:    C_EXT
+    NumberOfAuxEntries: 1
+  - Name:            ''
+    Value:           0x0
+    Section:         .text
+    Type:            0x0
+    StorageClass:    C_HIDEXT
+    NumberOfAuxEntries: 1
+  - Name:            .main
+    Value:           0x0
+    Section:         .text
+    Type:            0x0
+    StorageClass:    C_EXT
+    NumberOfAuxEntries: 1
+StringTable:     {}
+...
+
+# RUN: yaml2obj %s --docnum=2 -o %t64.o 
+# RUN: llvm-objdump -d %t64.o | FileCheck --check-prefixes=CHECK64 %s
+#
+
+# CHECK64:      : file format aix5coff64-rs6000
+# CHECK64:      Disassembly of section .text:
+# CHECK64:      0000000000000000 <.main>:
+# CHECK64-NEXT:       0: 48 00 00 03   bla 0x0
+# CHECK64-NEXT:       4: 41 80 00 03   btla    0, 0x0
+# CHECK64-NEXT:       8: 48 00 12 36   ba 0x1234
+# CHECK64-NEXT:       c: 4b ff ff e2   ba 0xffffffffffffffe0
+# CHECK64-NEXT:      10: 41 80 23 4a   bta     0, 0x2348
+# CHECK64-NEXT:      14: 41 80 ff 02   bta     0, 0xffffffffffffff00
+#
+--- !XCOFF
+FileHeader:
+  MagicNumber:     0x1F7
+  NumberOfSections: 1
+  CreationTime:    1701859282
+  AuxiliaryHeaderSize: 0
+  Flags:           0x0
+Sections:
+  - Name:            .text
+    Address:         0x0
+    Size:            0x18
+    FileOffsetToData: 0x60
+    FileOffsetToRelocations: 0x78
+    FileOffsetToLineNumbers: 0x0
+    NumberOfRelocations: 0x2
+    NumberOfLineNumbers: 0x0
+    Flags:           [ STYP_TEXT ]
+    SectionData:     4800000341800003480012364BFFFFE24180234A4180FF02
+    Relocations:
+      - Address:         0x0
+        Symbol:          0x1
+        Info:            0x99
+        Type:            0x18
+      - Address:         0x6
+        Symbol:          0x1
+        Info:            0x8F
+        Type:            0x18
+Symbols:
+  - Name:            .file
+    Section:         N_DEBUG
+    StorageClass:    C_FILE
+    NumberOfAuxEntries: 0
+  - Name:            .function
+    Section:         N_UNDEF
+    StorageClass:    C_EXT
+    NumberOfAuxEntries: 1
+    AuxEntries:
+      - Type:                   AUX_CSECT
+        StorageMappingClass:    XMC_PR
+        SymbolAlignmentAndType: 0
+  - Name:            ''
+    Section:         .text
+    StorageClass:    C_HIDEXT
+    NumberOfAuxEntries: 1
+    AuxEntries:
+      - Type:                   AUX_CSECT
+        StorageMappingClass:    XMC_PR
+        SymbolAlignmentAndType: 1
+  - Name:            .main
+    Section:         .text
+    StorageClass:    C_EXT
+    NumberOfAuxEntries: 1
+    AuxEntries:
+      - Type:                   AUX_CSECT
+        StorageMappingClass:    XMC_PR
+        SymbolAlignmentAndType: 2
+StringTable:
+  Strings:
+    - .file
+    - .function
+    - .text
+    - .main
diff --git a/llvm/test/tools/llvm-readobj/ELF/AArch64/aarch64-note-gnu-property.s b/llvm/test/tools/llvm-readobj/ELF/AArch64/aarch64-note-gnu-property.s
index 872a3f150fdf63027483f39479b6cb03c6312fb7..377e6f93448ca7b7f42fac904fe6125ac0b765ec 100644
--- a/llvm/test/tools/llvm-readobj/ELF/AArch64/aarch64-note-gnu-property.s
+++ b/llvm/test/tools/llvm-readobj/ELF/AArch64/aarch64-note-gnu-property.s
@@ -5,7 +5,7 @@
 // GNU: Displaying notes found in: .note.gnu.property
 // GNU-NEXT:   Owner                 Data size	Description
 // GNU-NEXT:   GNU                   0x00000010	NT_GNU_PROPERTY_TYPE_0 (property note)
-// GNU-NEXT:     Properties:    aarch64 feature: BTI, PAC
+// GNU-NEXT:     Properties:    aarch64 feature: BTI, PAC, GCS
 
 // LLVM:      Notes [
 // LLVM-NEXT:   NoteSection {
@@ -17,7 +17,7 @@
 // LLVM-NEXT:       Data size: 0x10
 // LLVM-NEXT:       Type: NT_GNU_PROPERTY_TYPE_0 (property note)
 // LLVM-NEXT:       Property [
-// LLVM-NEXT:         aarch64 feature: BTI, PAC
+// LLVM-NEXT:         aarch64 feature: BTI, PAC, GCS
 // LLVM-NEXT:       ]
 // LLVM-NEXT:     }
 // LLVM-NEXT:   }
@@ -30,9 +30,9 @@
   .asciz "GNU"      /* Name */
   .p2align 3
 begin:
-  /* BTI and PAC property note */
+  /* BTI, PAC, and GCS property note */
   .long 0xc0000000  /* Type: GNU_PROPERTY_AARCH64_FEATURE_1_AND */
   .long 4           /* Data size */
-  .long 3           /* BTI and PAC */
+  .long 7           /* BTI, PAC, GCS */
   .p2align 3        /* Align to 8 byte for 64 bit */
 end:
diff --git a/llvm/test/tools/llvm-reduce/reduce-flags.ll b/llvm/test/tools/llvm-reduce/reduce-flags.ll
index 4745f98db46b1544e600a94ce33e2e50953d9d0d..036bfdc84ac431430ebdae8ea163f789496266c8 100644
--- a/llvm/test/tools/llvm-reduce/reduce-flags.ll
+++ b/llvm/test/tools/llvm-reduce/reduce-flags.ll
@@ -200,3 +200,35 @@ define float @fadd_nnan_ninf_keep_nnan(float %a, float %b) {
   %op = fadd nnan ninf float %a, %b
   ret float %op
 }
+
+; CHECK-LABEL: @zext_nneg_drop(
+; INTERESTING: = zext
+; RESULT: zext i32
+define i64 @zext_nneg_drop(i32 %a) {
+  %op = zext nneg i32 %a to i64
+  ret i64 %op
+}
+
+; CHECK-LABEL: @zext_nneg_keep(
+; INTERESTING: = zext nneg
+; RESULT: zext nneg i32
+define i64 @zext_nneg_keep(i32 %a) {
+  %op = zext nneg i32 %a to i64
+  ret i64 %op
+}
+
+; CHECK-LABEL: @or_disjoint_drop(
+; INTERESTING: = or
+; RESULT: or i32
+define i32 @or_disjoint_drop(i32 %a, i32 %b) {
+  %op = or disjoint i32 %a, %b
+  ret i32 %op
+}
+
+; CHECK-LABEL: @or_disjoint_keep(
+; INTERESTING: = or disjoint
+; RESULT: or disjoint i32
+define i32 @or_disjoint_keep(i32 %a, i32 %b) {
+  %op = or disjoint i32 %a, %b
+  ret i32 %op
+}
diff --git a/llvm/test/tools/yaml2obj/GOFF/GOFF-header-end.yaml b/llvm/test/tools/yaml2obj/GOFF/GOFF-header-end.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..a5e99c2da2c491c5a44656e4c1e048de64a99cbd
--- /dev/null
+++ b/llvm/test/tools/yaml2obj/GOFF/GOFF-header-end.yaml
@@ -0,0 +1,20 @@
+# RUN: yaml2obj %s | od -v -An -tx1 | FileCheck --ignore-case %s
+
+## Verify that GOFF Header is correct.
+# CHECK:      03 f0 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+# CHECK-NEXT: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+# CHECK-NEXT: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+# CHECK-NEXT: 00 00 01 00 00 00 00 00 00 00 00 00 00 00 00 00
+# CHECK-NEXT: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+
+## Verify GOFF Module end.
+# CHECK-NEXT: 03 40 00 00 00 00 00 00 00 00 00 02 00 00 00 00
+# CHECK-NEXT: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+# CHECK-NEXT: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+# CHECK-NEXT: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+# CHECK-NEXT: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+# CHECK-EMPTY:
+
+--- !GOFF
+FileHeader:
+  ArchitectureLevel: 1
diff --git a/llvm/test/tools/yaml2obj/GOFF/GOFF-header-settings.yaml b/llvm/test/tools/yaml2obj/GOFF/GOFF-header-settings.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..1971c407199fbebffb3aac97e12f55deaa4d45ed
--- /dev/null
+++ b/llvm/test/tools/yaml2obj/GOFF/GOFF-header-settings.yaml
@@ -0,0 +1,26 @@
+# RUN: yaml2obj %s | od -v -An -tx1 | FileCheck --ignore-case %s
+
+## Verify that GOFF Header is correct.
+# CHECK:      03 f0 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+# CHECK-NEXT: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+# CHECK-NEXT: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+# CHECK-NEXT: 00 00 01 00 03 00 00 00 00 00 00 00 00 00 00 00
+# CHECK-NEXT: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+
+## Verify GOFF Module end.
+# CHECK-NEXT: 03 40 00 00 00 00 00 00 00 00 00 02 00 00 00 00
+# CHECK-NEXT: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+# CHECK-NEXT: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+# CHECK-NEXT: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+# CHECK-NEXT: 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
+
+--- !GOFF
+FileHeader:
+  TargetEnvironment: 0
+  TargetOperatingSystem: 0
+  CCSID: 0
+  CharacterSetName: ""
+  LanguageProductIdentifier: ""
+  ArchitectureLevel: 1
+  InternalCCSID: 0
+  TargetSoftwareEnvironment: 0 
diff --git a/llvm/test/tools/yaml2obj/GOFF/GOFF-no-header.yaml b/llvm/test/tools/yaml2obj/GOFF/GOFF-no-header.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..55f0efaacdf899e0cf1f480176ae719289301775
--- /dev/null
+++ b/llvm/test/tools/yaml2obj/GOFF/GOFF-no-header.yaml
@@ -0,0 +1,7 @@
+# RUN: not yaml2obj %s FileCheck --ignore-case %s
+
+# CHECK: yaml2obj: error: missing required key 'FileHeader'
+--- !GOFF
+## X: [] is an extra field required as workaround for 
+## 'document of unknown type' error
+X: []
diff --git a/llvm/tools/dsymutil/MachODebugMapParser.cpp b/llvm/tools/dsymutil/MachODebugMapParser.cpp
index 7a32eacee14c1b9de5a75b16d827f4be9b8ad57d..524a6795c360e96fe023602a4c9dd8872f6c1516 100644
--- a/llvm/tools/dsymutil/MachODebugMapParser.cpp
+++ b/llvm/tools/dsymutil/MachODebugMapParser.cpp
@@ -729,7 +729,8 @@ void MachODebugMapParser::handleStabSymbolTableEntry(
   }
 
   if (ObjectSymIt == CurrentObjectAddresses.end()) {
-    Warning("could not find object file symbol for symbol " + Twine(Name));
+    Warning("could not find symbol '" + Twine(Name) + "' in object file '" +
+            CurrentDebugMapObject->getObjectFilename() + "'");
     return;
   }
 
diff --git a/llvm/tools/llvm-cov/CodeCoverage.cpp b/llvm/tools/llvm-cov/CodeCoverage.cpp
index c1e3e2c4d1e2941cce5f043a727d63a697c680a5..6405bb15e70c4df536793b3edac8835e96f3605c 100644
--- a/llvm/tools/llvm-cov/CodeCoverage.cpp
+++ b/llvm/tools/llvm-cov/CodeCoverage.cpp
@@ -105,6 +105,11 @@ private:
                             const MemoryBuffer &File,
                             CoverageData &CoverageInfo);
 
+  /// Create source views for the MCDC records.
+  void attachMCDCSubViews(SourceCoverageView &View, StringRef SourceName,
+                          ArrayRef<MCDCRecord> MCDCRecords,
+                          const MemoryBuffer &File, CoverageData &CoverageInfo);
+
   /// Create the source view of a particular function.
   std::unique_ptr<SourceCoverageView>
   createFunctionView(const FunctionRecord &Function,
@@ -352,6 +357,37 @@ void CodeCoverageTool::attachBranchSubViews(SourceCoverageView &View,
   }
 }
 
+void CodeCoverageTool::attachMCDCSubViews(SourceCoverageView &View,
+                                          StringRef SourceName,
+                                          ArrayRef<MCDCRecord> MCDCRecords,
+                                          const MemoryBuffer &File,
+                                          CoverageData &CoverageInfo) {
+  if (!ViewOpts.ShowMCDC)
+    return;
+
+  const auto *NextRecord = MCDCRecords.begin();
+  const auto *EndRecord = MCDCRecords.end();
+
+  // Group and process MCDC records that have the same line number into the
+  // same subview.
+  while (NextRecord != EndRecord) {
+    std::vector<MCDCRecord> ViewMCDCRecords;
+    unsigned CurrentLine = NextRecord->getDecisionRegion().LineEnd;
+
+    while (NextRecord != EndRecord &&
+           CurrentLine == NextRecord->getDecisionRegion().LineEnd) {
+      ViewMCDCRecords.push_back(*NextRecord++);
+    }
+
+    if (ViewMCDCRecords.empty())
+      continue;
+
+    auto SubView = SourceCoverageView::create(SourceName, File, ViewOpts,
+                                              std::move(CoverageInfo));
+    View.addMCDCRecord(CurrentLine, ViewMCDCRecords, std::move(SubView));
+  }
+}
+
 std::unique_ptr<SourceCoverageView>
 CodeCoverageTool::createFunctionView(const FunctionRecord &Function,
                                      const CoverageMapping &Coverage) {
@@ -364,12 +400,15 @@ CodeCoverageTool::createFunctionView(const FunctionRecord &Function,
 
   auto Branches = FunctionCoverage.getBranches();
   auto Expansions = FunctionCoverage.getExpansions();
+  auto MCDCRecords = FunctionCoverage.getMCDCRecords();
   auto View = SourceCoverageView::create(DC.demangle(Function.Name),
                                          SourceBuffer.get(), ViewOpts,
                                          std::move(FunctionCoverage));
   attachExpansionSubViews(*View, Expansions, Coverage);
   attachBranchSubViews(*View, DC.demangle(Function.Name), Branches,
                        SourceBuffer.get(), FunctionCoverage);
+  attachMCDCSubViews(*View, DC.demangle(Function.Name), MCDCRecords,
+                     SourceBuffer.get(), FunctionCoverage);
 
   return View;
 }
@@ -386,11 +425,14 @@ CodeCoverageTool::createSourceFileView(StringRef SourceFile,
 
   auto Branches = FileCoverage.getBranches();
   auto Expansions = FileCoverage.getExpansions();
+  auto MCDCRecords = FileCoverage.getMCDCRecords();
   auto View = SourceCoverageView::create(SourceFile, SourceBuffer.get(),
                                          ViewOpts, std::move(FileCoverage));
   attachExpansionSubViews(*View, Expansions, Coverage);
   attachBranchSubViews(*View, SourceFile, Branches, SourceBuffer.get(),
                        FileCoverage);
+  attachMCDCSubViews(*View, SourceFile, MCDCRecords, SourceBuffer.get(),
+                     FileCoverage);
   if (!ViewOpts.ShowFunctionInstantiations)
     return View;
 
@@ -408,11 +450,14 @@ CodeCoverageTool::createSourceFileView(StringRef SourceFile,
         auto SubViewCoverage = Coverage.getCoverageForFunction(*Function);
         auto SubViewExpansions = SubViewCoverage.getExpansions();
         auto SubViewBranches = SubViewCoverage.getBranches();
+        auto SubViewMCDCRecords = SubViewCoverage.getMCDCRecords();
         SubView = SourceCoverageView::create(
             Funcname, SourceBuffer.get(), ViewOpts, std::move(SubViewCoverage));
         attachExpansionSubViews(*SubView, SubViewExpansions, Coverage);
         attachBranchSubViews(*SubView, SourceFile, SubViewBranches,
                              SourceBuffer.get(), SubViewCoverage);
+        attachMCDCSubViews(*SubView, SourceFile, SubViewMCDCRecords,
+                           SourceBuffer.get(), SubViewCoverage);
       }
 
       unsigned FileID = Function->CountedRegions.front().FileID;
@@ -752,6 +797,10 @@ int CodeCoverageTool::run(Command Cmd, int argc, const char **argv) {
       cl::desc("Show branch condition statistics in summary table"),
       cl::init(true));
 
+  cl::opt<bool> MCDCSummary("show-mcdc-summary", cl::Optional,
+                            cl::desc("Show MCDC statistics in summary table"),
+                            cl::init(false));
+
   cl::opt<bool> InstantiationSummary(
       "show-instantiation-summary", cl::Optional,
       cl::desc("Show instantiation statistics in summary table"));
@@ -923,6 +972,7 @@ int CodeCoverageTool::run(Command Cmd, int argc, const char **argv) {
       ::exit(0);
     }
 
+    ViewOpts.ShowMCDCSummary = MCDCSummary;
     ViewOpts.ShowBranchSummary = BranchSummary;
     ViewOpts.ShowRegionSummary = RegionSummary;
     ViewOpts.ShowInstantiationSummary = InstantiationSummary;
@@ -968,6 +1018,11 @@ int CodeCoverageTool::doShow(int argc, const char **argv,
                             "percent", "Show True/False percent")),
       cl::init(CoverageViewOptions::BranchOutputType::Off));
 
+  cl::opt<bool> ShowMCDC(
+      "show-mcdc", cl::Optional,
+      cl::desc("Show the MCDC Coverage for each applicable boolean expression"),
+      cl::cat(ViewCategory));
+
   cl::opt<bool> ShowBestLineRegionsCounts(
       "show-line-counts-or-regions", cl::Optional,
       cl::desc("Show the execution counts for each line, or the execution "
@@ -1063,6 +1118,7 @@ int CodeCoverageTool::doShow(int argc, const char **argv,
   ViewOpts.ShowExpandedRegions = ShowExpansions;
   ViewOpts.ShowBranchCounts =
       ShowBranches == CoverageViewOptions::BranchOutputType::Count;
+  ViewOpts.ShowMCDC = ShowMCDC;
   ViewOpts.ShowBranchPercents =
       ShowBranches == CoverageViewOptions::BranchOutputType::Percent;
   ViewOpts.ShowFunctionInstantiations = ShowInstantiations;
diff --git a/llvm/tools/llvm-cov/CoverageExporterJson.cpp b/llvm/tools/llvm-cov/CoverageExporterJson.cpp
index 9e43377e38a8a640ddb64deb82f0cbc709714ba0..a424bbe06e0ecd2f7abe5f7121cd7edcc26c5f4a 100644
--- a/llvm/tools/llvm-cov/CoverageExporterJson.cpp
+++ b/llvm/tools/llvm-cov/CoverageExporterJson.cpp
@@ -20,6 +20,8 @@
 //       -- File: dict => Coverage for a single file
 //         -- Branches: array => List of Branches in the file
 //           -- Branch: dict => Describes a branch of the file with counters
+//         -- MCDC Records: array => List of MCDC records in the file
+//           -- MCDC Values: array => List of T/F covered condition values
 //         -- Segments: array => List of Segments contained in the file
 //           -- Segment: dict => Describes a segment of the file with a counter
 //         -- Expansions: array => List of expansion records
@@ -34,6 +36,7 @@
 //           -- FunctionCoverage: dict => Object summarizing function coverage
 //           -- RegionCoverage: dict => Object summarizing region coverage
 //           -- BranchCoverage: dict => Object summarizing branch coverage
+//           -- MCDCCoverage: dict => Object summarizing MC/DC coverage
 //     -- Functions: array => List of objects describing coverage for functions
 //       -- Function: dict => Coverage info for a single function
 //         -- Filenames: array => List of filenames that the function relates to
@@ -43,6 +46,7 @@
 //     -- InstantiationCoverage: dict => Object summarizing inst. coverage
 //     -- RegionCoverage: dict => Object summarizing region coverage
 //     -- BranchCoverage: dict => Object summarizing branch coverage
+//     -- MCDCCoverage: dict => Object summarizing MC/DC coverage
 //
 //===----------------------------------------------------------------------===//
 
@@ -97,6 +101,20 @@ json::Array renderBranch(const coverage::CountedRegion &Region) {
        Region.ExpandedFileID, int64_t(Region.Kind)});
 }
 
+json::Array gatherConditions(const coverage::MCDCRecord &Record) {
+  json::Array Conditions;
+  for (unsigned c = 0; c < Record.getNumConditions(); c++)
+    Conditions.push_back(Record.isConditionIndependencePairCovered(c));
+  return Conditions;
+}
+
+json::Array renderMCDCRecord(const coverage::MCDCRecord &Record) {
+  const llvm::coverage::CounterMappingRegion &CMR = Record.getDecisionRegion();
+  return json::Array({CMR.LineStart, CMR.ColumnStart, CMR.LineEnd,
+                      CMR.ColumnEnd, CMR.ExpandedFileID, int64_t(CMR.Kind),
+                      gatherConditions(Record)});
+}
+
 json::Array renderRegions(ArrayRef<coverage::CountedRegion> Regions) {
   json::Array RegionArray;
   for (const auto &Region : Regions)
@@ -112,6 +130,13 @@ json::Array renderBranchRegions(ArrayRef<coverage::CountedRegion> Regions) {
   return RegionArray;
 }
 
+json::Array renderMCDCRecords(ArrayRef<coverage::MCDCRecord> Records) {
+  json::Array RecordArray;
+  for (auto &Record : Records)
+    RecordArray.push_back(renderMCDCRecord(Record));
+  return RecordArray;
+}
+
 std::vector<llvm::coverage::CountedRegion>
 collectNestedBranches(const coverage::CoverageMapping &Coverage,
                       ArrayRef<llvm::coverage::ExpansionRecord> Expansions) {
@@ -178,7 +203,14 @@ json::Object renderSummary(const FileCoverageSummary &Summary) {
              {"covered", int64_t(Summary.BranchCoverage.getCovered())},
              {"notcovered", int64_t(Summary.BranchCoverage.getNumBranches() -
                                     Summary.BranchCoverage.getCovered())},
-             {"percent", Summary.BranchCoverage.getPercentCovered()}})}});
+             {"percent", Summary.BranchCoverage.getPercentCovered()}})},
+       {"mcdc",
+        json::Object(
+            {{"count", int64_t(Summary.MCDCCoverage.getNumPairs())},
+             {"covered", int64_t(Summary.MCDCCoverage.getCoveredPairs())},
+             {"notcovered", int64_t(Summary.MCDCCoverage.getNumPairs() -
+                                    Summary.MCDCCoverage.getCoveredPairs())},
+             {"percent", Summary.MCDCCoverage.getPercentCovered()}})}});
 }
 
 json::Array renderFileExpansions(const coverage::CoverageMapping &Coverage,
@@ -206,6 +238,14 @@ json::Array renderFileBranches(const coverage::CoverageData &FileCoverage,
   return BranchArray;
 }
 
+json::Array renderFileMCDC(const coverage::CoverageData &FileCoverage,
+                           const FileCoverageSummary &FileReport) {
+  json::Array MCDCRecordArray;
+  for (const auto &Record : FileCoverage.getMCDCRecords())
+    MCDCRecordArray.push_back(renderMCDCRecord(Record));
+  return MCDCRecordArray;
+}
+
 json::Object renderFile(const coverage::CoverageMapping &Coverage,
                         const std::string &Filename,
                         const FileCoverageSummary &FileReport,
@@ -216,6 +256,7 @@ json::Object renderFile(const coverage::CoverageMapping &Coverage,
     auto FileCoverage = Coverage.getCoverageForFile(Filename);
     File["segments"] = renderFileSegments(FileCoverage, FileReport);
     File["branches"] = renderFileBranches(FileCoverage, FileReport);
+    File["mcdc_records"] = renderFileMCDC(FileCoverage, FileReport);
     if (!Options.SkipExpansions) {
       File["expansions"] =
           renderFileExpansions(Coverage, FileCoverage, FileReport);
@@ -264,6 +305,7 @@ json::Array renderFunctions(
                       {"count", clamp_uint64_to_int64(F.ExecutionCount)},
                       {"regions", renderRegions(F.CountedRegions)},
                       {"branches", renderBranchRegions(F.CountedBranchRegions)},
+                      {"mcdc_records", renderMCDCRecords(F.MCDCRecords)},
                       {"filenames", json::Array(F.Filenames)}}));
   return FunctionArray;
 }
diff --git a/llvm/tools/llvm-cov/CoverageReport.cpp b/llvm/tools/llvm-cov/CoverageReport.cpp
index eae2653b85c80ba2e2c5209b90c36379ae958212..8cc073e4def8fc99e4661c8fbab60b7925c1816c 100644
--- a/llvm/tools/llvm-cov/CoverageReport.cpp
+++ b/llvm/tools/llvm-cov/CoverageReport.cpp
@@ -86,9 +86,9 @@ Column column(StringRef Str, unsigned Width, const T &Value) {
 }
 
 // Specify the default column widths.
-size_t FileReportColumns[] = {25, 12, 18, 10, 12, 18, 10, 16,
-                              16, 10, 12, 18, 10, 12, 18, 10};
-size_t FunctionReportColumns[] = {25, 10, 8, 8, 10, 8, 8, 10, 8, 8};
+size_t FileReportColumns[] = {25, 12, 18, 10, 12, 18, 10, 16, 16, 10,
+                              12, 18, 10, 12, 18, 10, 20, 21, 10};
+size_t FunctionReportColumns[] = {25, 10, 8, 8, 10, 8, 8, 10, 8, 8, 20, 8, 8};
 
 /// Adjust column widths to fit long file paths and function names.
 void adjustColumnWidths(ArrayRef<StringRef> Files,
@@ -291,6 +291,22 @@ void CoverageReport::render(const FileCoverageSummary &File,
       OS << column("-", FileReportColumns[15], Column::RightAlignment);
   }
 
+  if (Options.ShowMCDCSummary) {
+    OS << format("%*u", FileReportColumns[16],
+                 (unsigned)File.MCDCCoverage.getNumPairs());
+    Options.colored_ostream(OS, LineCoverageColor)
+        << format("%*u", FileReportColumns[17],
+                  (unsigned)(File.MCDCCoverage.getNumPairs() -
+                             File.MCDCCoverage.getCoveredPairs()));
+    if (File.MCDCCoverage.getNumPairs())
+      Options.colored_ostream(OS, LineCoverageColor)
+          << format("%*.2f", FileReportColumns[18] - 1,
+                    File.MCDCCoverage.getPercentCovered())
+          << '%';
+    else
+      OS << column("-", FileReportColumns[18], Column::RightAlignment);
+  }
+
   OS << "\n";
 }
 
@@ -338,6 +354,19 @@ void CoverageReport::render(const FunctionCoverageSummary &Function,
                   Function.BranchCoverage.getPercentCovered())
         << '%';
   }
+  if (Options.ShowMCDCSummary) {
+    OS << format("%*u", FunctionReportColumns[10],
+                 (unsigned)Function.MCDCCoverage.getNumPairs());
+    Options.colored_ostream(OS, LineCoverageColor)
+        << format("%*u", FunctionReportColumns[11],
+                  (unsigned)(Function.MCDCCoverage.getNumPairs() -
+                             Function.MCDCCoverage.getCoveredPairs()));
+    Options.colored_ostream(
+        OS, determineCoveragePercentageColor(Function.MCDCCoverage))
+        << format("%*.2f", FunctionReportColumns[12] - 1,
+                  Function.MCDCCoverage.getPercentCovered())
+        << '%';
+  }
   OS << "\n";
 }
 
@@ -370,6 +399,11 @@ void CoverageReport::renderFunctionReports(ArrayRef<std::string> Files,
       OS << column("Branches", FunctionReportColumns[7], Column::RightAlignment)
          << column("Miss", FunctionReportColumns[8], Column::RightAlignment)
          << column("Cover", FunctionReportColumns[9], Column::RightAlignment);
+    if (Options.ShowMCDCSummary)
+      OS << column("MC/DC Conditions", FunctionReportColumns[10],
+                   Column::RightAlignment)
+         << column("Miss", FunctionReportColumns[11], Column::RightAlignment)
+         << column("Cover", FunctionReportColumns[12], Column::RightAlignment);
     OS << "\n";
     renderDivider(FunctionReportColumns, OS);
     OS << "\n";
@@ -380,6 +414,7 @@ void CoverageReport::renderFunctionReports(ArrayRef<std::string> Files,
       Totals.RegionCoverage += Function.RegionCoverage;
       Totals.LineCoverage += Function.LineCoverage;
       Totals.BranchCoverage += Function.BranchCoverage;
+      Totals.MCDCCoverage += Function.MCDCCoverage;
       render(Function, DC, OS);
     }
     if (Totals.ExecutionCount) {
@@ -502,6 +537,12 @@ void CoverageReport::renderFileReports(
        << column("Missed Branches", FileReportColumns[14],
                  Column::RightAlignment)
        << column("Cover", FileReportColumns[15], Column::RightAlignment);
+  if (Options.ShowMCDCSummary)
+    OS << column("MC/DC Conditions", FileReportColumns[16],
+                 Column::RightAlignment)
+       << column("Missed Conditions", FileReportColumns[17],
+                 Column::RightAlignment)
+       << column("Cover", FileReportColumns[18], Column::RightAlignment);
   OS << "\n";
   renderDivider(FileReportColumns, OS);
   OS << "\n";
diff --git a/llvm/tools/llvm-cov/CoverageSummaryInfo.cpp b/llvm/tools/llvm-cov/CoverageSummaryInfo.cpp
index 10e059adeb7d86665729f1ddd562645c3f50847e..4f150020ee381544f8f29a9b7387eef9cfadfb55 100644
--- a/llvm/tools/llvm-cov/CoverageSummaryInfo.cpp
+++ b/llvm/tools/llvm-cov/CoverageSummaryInfo.cpp
@@ -44,6 +44,21 @@ static void sumBranchExpansions(size_t &NumBranches, size_t &CoveredBranches,
   }
 }
 
+static std::pair<size_t, size_t>
+sumMCDCPairs(const ArrayRef<MCDCRecord> &Records) {
+  size_t NumPairs = 0, CoveredPairs = 0;
+  for (const auto &Record : Records) {
+    const auto NumConditions = Record.getNumConditions();
+    for (unsigned C = 0; C < NumConditions; C++) {
+      if (!Record.isCondFolded(C))
+        ++NumPairs;
+      if (Record.isConditionIndependencePairCovered(C))
+        ++CoveredPairs;
+    }
+  }
+  return {NumPairs, CoveredPairs};
+}
+
 FunctionCoverageSummary
 FunctionCoverageSummary::get(const CoverageMapping &CM,
                              const coverage::FunctionRecord &Function) {
@@ -73,11 +88,15 @@ FunctionCoverageSummary::get(const CoverageMapping &CM,
   sumBranches(NumBranches, CoveredBranches, CD.getBranches());
   sumBranchExpansions(NumBranches, CoveredBranches, CM, CD.getExpansions());
 
+  size_t NumPairs = 0, CoveredPairs = 0;
+  std::tie(NumPairs, CoveredPairs) = sumMCDCPairs(CD.getMCDCRecords());
+
   return FunctionCoverageSummary(
       Function.Name, Function.ExecutionCount,
       RegionCoverageInfo(CoveredRegions, NumCodeRegions),
       LineCoverageInfo(CoveredLines, NumLines),
-      BranchCoverageInfo(CoveredBranches, NumBranches));
+      BranchCoverageInfo(CoveredBranches, NumBranches),
+      MCDCCoverageInfo(CoveredPairs, NumPairs));
 }
 
 FunctionCoverageSummary
@@ -97,10 +116,12 @@ FunctionCoverageSummary::get(const InstantiationGroup &Group,
   Summary.RegionCoverage = Summaries[0].RegionCoverage;
   Summary.LineCoverage = Summaries[0].LineCoverage;
   Summary.BranchCoverage = Summaries[0].BranchCoverage;
+  Summary.MCDCCoverage = Summaries[0].MCDCCoverage;
   for (const auto &FCS : Summaries.drop_front()) {
     Summary.RegionCoverage.merge(FCS.RegionCoverage);
     Summary.LineCoverage.merge(FCS.LineCoverage);
     Summary.BranchCoverage.merge(FCS.BranchCoverage);
+    Summary.MCDCCoverage.merge(FCS.MCDCCoverage);
   }
   return Summary;
 }
diff --git a/llvm/tools/llvm-cov/CoverageSummaryInfo.h b/llvm/tools/llvm-cov/CoverageSummaryInfo.h
index 46510dc9cf3faf500c8cee2ff35a6b2c32d5a38e..64c2c8406cf3ebb910ae04a0c7c9b7d6ce647f41 100644
--- a/llvm/tools/llvm-cov/CoverageSummaryInfo.h
+++ b/llvm/tools/llvm-cov/CoverageSummaryInfo.h
@@ -142,6 +142,47 @@ public:
   }
 };
 
+/// Provides information about MC/DC coverage for a function/file.
+class MCDCCoverageInfo {
+  /// The number of Independence Pairs that were covered.
+  size_t CoveredPairs;
+
+  /// The total number of Independence Pairs in a function/file.
+  size_t NumPairs;
+
+public:
+  MCDCCoverageInfo() : CoveredPairs(0), NumPairs(0) {}
+
+  MCDCCoverageInfo(size_t CoveredPairs, size_t NumPairs)
+      : CoveredPairs(CoveredPairs), NumPairs(NumPairs) {
+    assert(CoveredPairs <= NumPairs && "Covered pairs over-counted");
+  }
+
+  MCDCCoverageInfo &operator+=(const MCDCCoverageInfo &RHS) {
+    CoveredPairs += RHS.CoveredPairs;
+    NumPairs += RHS.NumPairs;
+    return *this;
+  }
+
+  void merge(const MCDCCoverageInfo &RHS) {
+    CoveredPairs = std::max(CoveredPairs, RHS.CoveredPairs);
+    NumPairs = std::max(NumPairs, RHS.NumPairs);
+  }
+
+  size_t getCoveredPairs() const { return CoveredPairs; }
+
+  size_t getNumPairs() const { return NumPairs; }
+
+  bool isFullyCovered() const { return CoveredPairs == NumPairs; }
+
+  double getPercentCovered() const {
+    assert(CoveredPairs <= NumPairs && "Covered pairs over-counted");
+    if (NumPairs == 0)
+      return 0.0;
+    return double(CoveredPairs) / double(NumPairs) * 100.0;
+  }
+};
+
 /// Provides information about function coverage for a file.
 class FunctionCoverageInfo {
   /// The number of functions that were executed.
@@ -189,6 +230,7 @@ struct FunctionCoverageSummary {
   RegionCoverageInfo RegionCoverage;
   LineCoverageInfo LineCoverage;
   BranchCoverageInfo BranchCoverage;
+  MCDCCoverageInfo MCDCCoverage;
 
   FunctionCoverageSummary(const std::string &Name)
       : Name(Name), ExecutionCount(0) {}
@@ -196,10 +238,11 @@ struct FunctionCoverageSummary {
   FunctionCoverageSummary(const std::string &Name, uint64_t ExecutionCount,
                           const RegionCoverageInfo &RegionCoverage,
                           const LineCoverageInfo &LineCoverage,
-                          const BranchCoverageInfo &BranchCoverage)
+                          const BranchCoverageInfo &BranchCoverage,
+                          const MCDCCoverageInfo &MCDCCoverage)
       : Name(Name), ExecutionCount(ExecutionCount),
         RegionCoverage(RegionCoverage), LineCoverage(LineCoverage),
-        BranchCoverage(BranchCoverage) {}
+        BranchCoverage(BranchCoverage), MCDCCoverage(MCDCCoverage) {}
 
   /// Compute the code coverage summary for the given function coverage
   /// mapping record.
@@ -219,6 +262,7 @@ struct FileCoverageSummary {
   RegionCoverageInfo RegionCoverage;
   LineCoverageInfo LineCoverage;
   BranchCoverageInfo BranchCoverage;
+  MCDCCoverageInfo MCDCCoverage;
   FunctionCoverageInfo FunctionCoverage;
   FunctionCoverageInfo InstantiationCoverage;
 
@@ -230,6 +274,7 @@ struct FileCoverageSummary {
     LineCoverage += RHS.LineCoverage;
     FunctionCoverage += RHS.FunctionCoverage;
     BranchCoverage += RHS.BranchCoverage;
+    MCDCCoverage += RHS.MCDCCoverage;
     InstantiationCoverage += RHS.InstantiationCoverage;
     return *this;
   }
@@ -238,6 +283,7 @@ struct FileCoverageSummary {
     RegionCoverage += Function.RegionCoverage;
     LineCoverage += Function.LineCoverage;
     BranchCoverage += Function.BranchCoverage;
+    MCDCCoverage += Function.MCDCCoverage;
     FunctionCoverage.addFunction(/*Covered=*/Function.ExecutionCount > 0);
   }
 
diff --git a/llvm/tools/llvm-cov/CoverageViewOptions.h b/llvm/tools/llvm-cov/CoverageViewOptions.h
index eb852859e9cbe46e156f2733a3b4b084e6211ec6..6925cffd8246d261b689276502abf40a2a0b6089 100644
--- a/llvm/tools/llvm-cov/CoverageViewOptions.h
+++ b/llvm/tools/llvm-cov/CoverageViewOptions.h
@@ -30,12 +30,14 @@ struct CoverageViewOptions {
   bool ShowLineNumbers;
   bool ShowLineStats;
   bool ShowRegionMarkers;
+  bool ShowMCDC;
   bool ShowBranchCounts;
   bool ShowBranchPercents;
   bool ShowExpandedRegions;
   bool ShowFunctionInstantiations;
   bool ShowFullFilenames;
   bool ShowBranchSummary;
+  bool ShowMCDCSummary;
   bool ShowRegionSummary;
   bool ShowInstantiationSummary;
   bool ShowDirectoryCoverage;
diff --git a/llvm/tools/llvm-cov/SourceCoverageView.cpp b/llvm/tools/llvm-cov/SourceCoverageView.cpp
index 7480b7f628a0b9d15ca645a0d4ec535360352a98..c910edd1db7827cbce05155ed73d551c4230b561 100644
--- a/llvm/tools/llvm-cov/SourceCoverageView.cpp
+++ b/llvm/tools/llvm-cov/SourceCoverageView.cpp
@@ -178,6 +178,12 @@ void SourceCoverageView::addBranch(unsigned Line,
   BranchSubViews.emplace_back(Line, Regions, std::move(View));
 }
 
+void SourceCoverageView::addMCDCRecord(
+    unsigned Line, ArrayRef<MCDCRecord> Records,
+    std::unique_ptr<SourceCoverageView> View) {
+  MCDCSubViews.emplace_back(Line, Records, std::move(View));
+}
+
 void SourceCoverageView::addInstantiation(
     StringRef FunctionName, unsigned Line,
     std::unique_ptr<SourceCoverageView> View) {
@@ -203,12 +209,15 @@ void SourceCoverageView::print(raw_ostream &OS, bool WholeFile,
   llvm::stable_sort(ExpansionSubViews);
   llvm::stable_sort(InstantiationSubViews);
   llvm::stable_sort(BranchSubViews);
+  llvm::stable_sort(MCDCSubViews);
   auto NextESV = ExpansionSubViews.begin();
   auto EndESV = ExpansionSubViews.end();
   auto NextISV = InstantiationSubViews.begin();
   auto EndISV = InstantiationSubViews.end();
   auto NextBRV = BranchSubViews.begin();
   auto EndBRV = BranchSubViews.end();
+  auto NextMSV = MCDCSubViews.begin();
+  auto EndMSV = MCDCSubViews.end();
 
   // Get the coverage information for the file.
   auto StartSegment = CoverageInfo.begin();
@@ -276,6 +285,11 @@ void SourceCoverageView::print(raw_ostream &OS, bool WholeFile,
       renderBranchView(OS, *NextBRV, ViewDepth + 1);
       RenderedSubView = true;
     }
+    for (; NextMSV != EndMSV && NextMSV->Line == LI.line_number(); ++NextMSV) {
+      renderViewDivider(OS, ViewDepth + 1);
+      renderMCDCView(OS, *NextMSV, ViewDepth + 1);
+      RenderedSubView = true;
+    }
     if (RenderedSubView)
       renderViewDivider(OS, ViewDepth + 1);
     renderLineSuffix(OS, ViewDepth);
diff --git a/llvm/tools/llvm-cov/SourceCoverageView.h b/llvm/tools/llvm-cov/SourceCoverageView.h
index c07595f980727adc86d720ead9f1921efbba3b24..2e3fa8e627d83c3ab561c0feac04727981e12278 100644
--- a/llvm/tools/llvm-cov/SourceCoverageView.h
+++ b/llvm/tools/llvm-cov/SourceCoverageView.h
@@ -84,6 +84,23 @@ struct BranchView {
   }
 };
 
+/// A view that represents one or more MCDC regions on a given source line.
+struct MCDCView {
+  std::vector<MCDCRecord> Records;
+  std::unique_ptr<SourceCoverageView> View;
+  unsigned Line;
+
+  MCDCView(unsigned Line, ArrayRef<MCDCRecord> Records,
+           std::unique_ptr<SourceCoverageView> View)
+      : Records(Records), View(std::move(View)), Line(Line) {}
+
+  unsigned getLine() const { return Line; }
+
+  friend bool operator<(const MCDCView &LHS, const MCDCView &RHS) {
+    return LHS.Line < RHS.Line;
+  }
+};
+
 /// A file manager that handles format-aware file creation.
 class CoveragePrinter {
 public:
@@ -160,6 +177,9 @@ class SourceCoverageView {
   /// A container for all branches in the source on display.
   std::vector<BranchView> BranchSubViews;
 
+  /// A container for all MCDC records in the source on display.
+  std::vector<MCDCView> MCDCSubViews;
+
   /// A container for all instantiations (e.g template functions) in the source
   /// on display.
   std::vector<InstantiationView> InstantiationSubViews;
@@ -233,6 +253,10 @@ protected:
   virtual void renderBranchView(raw_ostream &OS, BranchView &BRV,
                                 unsigned ViewDepth) = 0;
 
+  /// Render an MCDC view.
+  virtual void renderMCDCView(raw_ostream &OS, MCDCView &BRV,
+                              unsigned ViewDepth) = 0;
+
   /// Render \p Title, a project title if one is available, and the
   /// created time.
   virtual void renderTitle(raw_ostream &OS, StringRef CellText) = 0;
@@ -283,6 +307,10 @@ public:
   void addBranch(unsigned Line, ArrayRef<CountedRegion> Regions,
                  std::unique_ptr<SourceCoverageView> View);
 
+  /// Add an MCDC subview to this view.
+  void addMCDCRecord(unsigned Line, ArrayRef<MCDCRecord> Records,
+                     std::unique_ptr<SourceCoverageView> View);
+
   /// Print the code coverage information for a specific portion of a
   /// source file to the output stream.
   void print(raw_ostream &OS, bool WholeFile, bool ShowSourceName,
diff --git a/llvm/tools/llvm-cov/SourceCoverageViewHTML.cpp b/llvm/tools/llvm-cov/SourceCoverageViewHTML.cpp
index d0a2e44be2521d16d8f9809fb4a2b5ac9c360188..b43e9e64231e0c419d5d5cacf202f04d7f9beeee 100644
--- a/llvm/tools/llvm-cov/SourceCoverageViewHTML.cpp
+++ b/llvm/tools/llvm-cov/SourceCoverageViewHTML.cpp
@@ -335,6 +335,10 @@ void emitTableRow(raw_ostream &OS, const CoverageViewOptions &Opts,
     AddCoverageTripleToColumn(FCS.BranchCoverage.getCovered(),
                               FCS.BranchCoverage.getNumBranches(),
                               FCS.BranchCoverage.getPercentCovered());
+  if (Opts.ShowMCDCSummary)
+    AddCoverageTripleToColumn(FCS.MCDCCoverage.getCoveredPairs(),
+                              FCS.MCDCCoverage.getNumPairs(),
+                              FCS.MCDCCoverage.getPercentCovered());
 
   if (IsTotals)
     OS << tag("tr", join(Columns.begin(), Columns.end(), ""), "light-row-bold");
@@ -385,6 +389,8 @@ static void emitColumnLabelsForIndex(raw_ostream &OS,
     Columns.emplace_back(tag("td", "Region Coverage", "column-entry-bold"));
   if (Opts.ShowBranchSummary)
     Columns.emplace_back(tag("td", "Branch Coverage", "column-entry-bold"));
+  if (Opts.ShowMCDCSummary)
+    Columns.emplace_back(tag("td", "MC/DC", "column-entry-bold"));
   OS << tag("tr", join(Columns.begin(), Columns.end(), ""));
 }
 
@@ -955,6 +961,52 @@ void SourceCoverageViewHTML::renderBranchView(raw_ostream &OS, BranchView &BRV,
   OS << EndExpansionDiv;
 }
 
+void SourceCoverageViewHTML::renderMCDCView(raw_ostream &OS, MCDCView &MRV,
+                                            unsigned ViewDepth) {
+  for (auto &Record : MRV.Records) {
+    OS << BeginExpansionDiv;
+    OS << BeginPre;
+    OS << "  MC/DC Decision Region (";
+
+    // Display Line + Column information.
+    const CounterMappingRegion &DecisionRegion = Record.getDecisionRegion();
+    std::string LineNoStr = Twine(DecisionRegion.LineStart).str();
+    std::string ColNoStr = Twine(DecisionRegion.ColumnStart).str();
+    std::string TargetName = "L" + LineNoStr;
+    OS << tag("span",
+              a("#" + TargetName, tag("span", LineNoStr + ":" + ColNoStr),
+                TargetName),
+              "line-number") +
+              ") to (";
+    LineNoStr = utostr(uint64_t(DecisionRegion.LineEnd));
+    ColNoStr = utostr(uint64_t(DecisionRegion.ColumnEnd));
+    OS << tag("span",
+              a("#" + TargetName, tag("span", LineNoStr + ":" + ColNoStr),
+                TargetName),
+              "line-number") +
+              ")\n\n";
+
+    // Display MC/DC Information.
+    OS << "  Number of Conditions: " << Record.getNumConditions() << "\n";
+    for (unsigned i = 0; i < Record.getNumConditions(); i++) {
+      OS << "     " << Record.getConditionHeaderString(i);
+    }
+    OS << "\n";
+    OS << "  Executed MC/DC Test Vectors:\n\n     ";
+    OS << Record.getTestVectorHeaderString();
+    for (unsigned i = 0; i < Record.getNumTestVectors(); i++)
+      OS << Record.getTestVectorString(i);
+    OS << "\n";
+    for (unsigned i = 0; i < Record.getNumConditions(); i++)
+      OS << Record.getConditionCoverageString(i);
+    OS << "  MC/DC Coverage for Expression: ";
+    OS << format("%0.2f", Record.getPercentCovered()) << "%\n";
+    OS << EndPre;
+    OS << EndExpansionDiv;
+  }
+  return;
+}
+
 void SourceCoverageViewHTML::renderInstantiationView(raw_ostream &OS,
                                                      InstantiationView &ISV,
                                                      unsigned ViewDepth) {
diff --git a/llvm/tools/llvm-cov/SourceCoverageViewHTML.h b/llvm/tools/llvm-cov/SourceCoverageViewHTML.h
index c846379889cd3a1d8f88f1cf2e7a92a61de95778..7b97f05b946bd270ab3ccce06a697c354ea62d87 100644
--- a/llvm/tools/llvm-cov/SourceCoverageViewHTML.h
+++ b/llvm/tools/llvm-cov/SourceCoverageViewHTML.h
@@ -91,6 +91,9 @@ class SourceCoverageViewHTML : public SourceCoverageView {
   void renderBranchView(raw_ostream &OS, BranchView &BRV,
                         unsigned ViewDepth) override;
 
+  void renderMCDCView(raw_ostream &OS, MCDCView &BRV,
+                      unsigned ViewDepth) override;
+
   void renderInstantiationView(raw_ostream &OS, InstantiationView &ISV,
                                unsigned ViewDepth) override;
 
diff --git a/llvm/tools/llvm-cov/SourceCoverageViewText.cpp b/llvm/tools/llvm-cov/SourceCoverageViewText.cpp
index 44694a0426c81a9f381952a54c0033a433ad80b7..73b7ffe16a9637e9076b56efeaec8ff6a2ed1824 100644
--- a/llvm/tools/llvm-cov/SourceCoverageViewText.cpp
+++ b/llvm/tools/llvm-cov/SourceCoverageViewText.cpp
@@ -337,6 +337,57 @@ void SourceCoverageViewText::renderBranchView(raw_ostream &OS, BranchView &BRV,
   }
 }
 
+void SourceCoverageViewText::renderMCDCView(raw_ostream &OS, MCDCView &MRV,
+                                            unsigned ViewDepth) {
+  for (auto &Record : MRV.Records) {
+    renderLinePrefix(OS, ViewDepth);
+    OS << "---> MC/DC Decision Region (";
+    // Display Line + Column information.
+    const CounterMappingRegion &DecisionRegion = Record.getDecisionRegion();
+    OS << DecisionRegion.LineStart << ":";
+    OS << DecisionRegion.ColumnStart << ") to (";
+    OS << DecisionRegion.LineEnd << ":";
+    OS << DecisionRegion.ColumnEnd << ")\n";
+    renderLinePrefix(OS, ViewDepth);
+    OS << "\n";
+
+    // Display MC/DC Information.
+    renderLinePrefix(OS, ViewDepth);
+    OS << "  Number of Conditions: " << Record.getNumConditions() << "\n";
+    for (unsigned i = 0; i < Record.getNumConditions(); i++) {
+      renderLinePrefix(OS, ViewDepth);
+      OS << "     " << Record.getConditionHeaderString(i);
+    }
+    renderLinePrefix(OS, ViewDepth);
+    OS << "\n";
+    renderLinePrefix(OS, ViewDepth);
+    OS << "  Executed MC/DC Test Vectors:\n";
+    renderLinePrefix(OS, ViewDepth);
+    OS << "\n";
+    renderLinePrefix(OS, ViewDepth);
+    OS << "     ";
+    OS << Record.getTestVectorHeaderString();
+    for (unsigned i = 0; i < Record.getNumTestVectors(); i++) {
+      renderLinePrefix(OS, ViewDepth);
+      OS << Record.getTestVectorString(i);
+    }
+    renderLinePrefix(OS, ViewDepth);
+    OS << "\n";
+    for (unsigned i = 0; i < Record.getNumConditions(); i++) {
+      renderLinePrefix(OS, ViewDepth);
+      OS << Record.getConditionCoverageString(i);
+    }
+    renderLinePrefix(OS, ViewDepth);
+    OS << "  MC/DC Coverage for Decision: ";
+    colored_ostream(OS, raw_ostream::RED,
+                    getOptions().Colors && Record.getPercentCovered() < 100.0,
+                    /*Bold=*/false, /*BG=*/true)
+        << format("%0.2f", Record.getPercentCovered()) << "%\n";
+    renderLinePrefix(OS, ViewDepth);
+    OS << "\n";
+  }
+}
+
 void SourceCoverageViewText::renderInstantiationView(raw_ostream &OS,
                                                      InstantiationView &ISV,
                                                      unsigned ViewDepth) {
diff --git a/llvm/tools/llvm-cov/SourceCoverageViewText.h b/llvm/tools/llvm-cov/SourceCoverageViewText.h
index ade47ed3b5f5865578ba4bacb67ae66c768d0125..7cb47fcbf42bd640772acb937c9d216ee3793de1 100644
--- a/llvm/tools/llvm-cov/SourceCoverageViewText.h
+++ b/llvm/tools/llvm-cov/SourceCoverageViewText.h
@@ -77,6 +77,9 @@ class SourceCoverageViewText : public SourceCoverageView {
   void renderBranchView(raw_ostream &OS, BranchView &BRV,
                         unsigned ViewDepth) override;
 
+  void renderMCDCView(raw_ostream &OS, MCDCView &BRV,
+                      unsigned ViewDepth) override;
+
   void renderInstantiationView(raw_ostream &OS, InstantiationView &ISV,
                                unsigned ViewDepth) override;
 
diff --git a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp
index 2d396df7337f8961a60896799b5472ccc84f96cf..7bb0218ed533868559d848d28958da9653ef58ac 100644
--- a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp
+++ b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp
@@ -25,6 +25,7 @@
 #include "llvm/Support/MemoryBuffer.h"
 #include "llvm/Support/Program.h"
 #include "llvm/Support/Signals.h"
+#include "llvm/Support/SystemZ/zOSSupport.h"
 
 #ifdef __linux__
 #ifdef HAVE_LIBPFM
diff --git a/llvm/tools/llvm-exegesis/lib/Error.cpp b/llvm/tools/llvm-exegesis/lib/Error.cpp
index d29c41385670461d47e9ecdff8d5a465571a4708..82ef92dc5712c4d5c3bd9bde148918a08b109350 100644
--- a/llvm/tools/llvm-exegesis/lib/Error.cpp
+++ b/llvm/tools/llvm-exegesis/lib/Error.cpp
@@ -9,6 +9,7 @@
 #include "Error.h"
 
 #ifdef LLVM_ON_UNIX
+#include "llvm/Support/SystemZ/zOSSupport.h"
 #include <string.h>
 #endif // LLVM_ON_UNIX
 
@@ -42,6 +43,8 @@ void SnippetSignal::log(raw_ostream &OS) const {
   OS << "snippet crashed while running";
 #ifdef LLVM_ON_UNIX
   OS << ": " << strsignal(SignalNumber);
+#else
+  (void)SignalNumber;
 #endif // LLVM_ON_UNIX
 }
 
diff --git a/llvm/tools/llvm-readobj/ELFDumper.cpp b/llvm/tools/llvm-readobj/ELFDumper.cpp
index 23f35fa6dfdb6d2c2c0f3ec56a1fc536f6a9e489..3cf7c5a3b18955f5c6feea582c9856dd472f7078 100644
--- a/llvm/tools/llvm-readobj/ELFDumper.cpp
+++ b/llvm/tools/llvm-readobj/ELFDumper.cpp
@@ -57,6 +57,7 @@
 #include "llvm/Support/RISCVAttributeParser.h"
 #include "llvm/Support/RISCVAttributes.h"
 #include "llvm/Support/ScopedPrinter.h"
+#include "llvm/Support/SystemZ/zOSSupport.h"
 #include "llvm/Support/raw_ostream.h"
 #include <algorithm>
 #include <cinttypes>
@@ -5129,6 +5130,7 @@ static std::string getGNUProperty(uint32_t Type, uint32_t DataSize,
     if (Type == GNU_PROPERTY_AARCH64_FEATURE_1_AND) {
       DumpBit(GNU_PROPERTY_AARCH64_FEATURE_1_BTI, "BTI");
       DumpBit(GNU_PROPERTY_AARCH64_FEATURE_1_PAC, "PAC");
+      DumpBit(GNU_PROPERTY_AARCH64_FEATURE_1_GCS, "GCS");
     } else {
       DumpBit(GNU_PROPERTY_X86_FEATURE_1_IBT, "IBT");
       DumpBit(GNU_PROPERTY_X86_FEATURE_1_SHSTK, "SHSTK");
diff --git a/llvm/tools/llvm-readobj/ObjDumper.cpp b/llvm/tools/llvm-readobj/ObjDumper.cpp
index 6dde3725b4d65545c06661df641130df2bd12497..59060ac217e32f20a67231a43d6767b58190a557 100644
--- a/llvm/tools/llvm-readobj/ObjDumper.cpp
+++ b/llvm/tools/llvm-readobj/ObjDumper.cpp
@@ -18,6 +18,7 @@
 #include "llvm/Support/Error.h"
 #include "llvm/Support/FormatVariadic.h"
 #include "llvm/Support/ScopedPrinter.h"
+#include "llvm/Support/SystemZ/zOSSupport.h"
 #include "llvm/Support/raw_ostream.h"
 #include <map>
 
diff --git a/llvm/tools/llvm-reduce/deltas/ReduceInstructionFlags.cpp b/llvm/tools/llvm-reduce/deltas/ReduceInstructionFlags.cpp
index c73e74e4c25c941e7093119c4712ee285f61ea42..7b6fe7e5f91737bebc31d72fcc875798f0a9031a 100644
--- a/llvm/tools/llvm-reduce/deltas/ReduceInstructionFlags.cpp
+++ b/llvm/tools/llvm-reduce/deltas/ReduceInstructionFlags.cpp
@@ -30,6 +30,12 @@ static void reduceFlagsInModule(Oracle &O, ReducerWorkItem &WorkItem) {
       } else if (auto *PE = dyn_cast<PossiblyExactOperator>(&I)) {
         if (PE->isExact() && !O.shouldKeep())
           I.setIsExact(false);
+      } else if (auto *NNI = dyn_cast<PossiblyNonNegInst>(&I)) {
+        if (NNI->hasNonNeg() && !O.shouldKeep())
+          NNI->setNonNeg(false);
+      } else if (auto *PDI = dyn_cast<PossiblyDisjointInst>(&I)) {
+        if (PDI->isDisjoint() && !O.shouldKeep())
+          PDI->setIsDisjoint(false);
       } else if (auto *GEP = dyn_cast<GetElementPtrInst>(&I)) {
         if (GEP->isInBounds() && !O.shouldKeep())
           GEP->setIsInBounds(false);
diff --git a/llvm/tools/llvm-symbolizer/llvm-symbolizer.cpp b/llvm/tools/llvm-symbolizer/llvm-symbolizer.cpp
index 447c18abadc1743846bdbc1ea585d3a3123e04a1..3d33534d95b7e0d4e020f5afd63715dcad4df993 100644
--- a/llvm/tools/llvm-symbolizer/llvm-symbolizer.cpp
+++ b/llvm/tools/llvm-symbolizer/llvm-symbolizer.cpp
@@ -430,7 +430,7 @@ static void filterMarkup(const opt::InputArgList &Args, LLVMSymbolizer &Symboliz
   std::string InputString;
   while (std::getline(std::cin, InputString)) {
     InputString += '\n';
-    Filter.filter(InputString);
+    Filter.filter(std::move(InputString));
   }
   Filter.finish();
 }
diff --git a/llvm/tools/obj2yaml/macho2yaml.cpp b/llvm/tools/obj2yaml/macho2yaml.cpp
index c006c0d2cbea4a7d3e4b7b323b977202f7ec2da8..cdd871e8c1d68472bbb3dd50b662e147dfc2c2d5 100644
--- a/llvm/tools/obj2yaml/macho2yaml.cpp
+++ b/llvm/tools/obj2yaml/macho2yaml.cpp
@@ -15,6 +15,7 @@
 #include "llvm/Support/Error.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/Support/LEB128.h"
+#include "llvm/Support/SystemZ/zOSSupport.h"
 
 #include <string.h> // for memcpy
 
diff --git a/llvm/tools/opt/opt.cpp b/llvm/tools/opt/opt.cpp
index 62081fc01b13e6e94a1e47adeed940c4dbb3e767..b6068513d2306336c606ed91aabc208b60ecee40 100644
--- a/llvm/tools/opt/opt.cpp
+++ b/llvm/tools/opt/opt.cpp
@@ -356,7 +356,7 @@ static bool shouldPinPassToLegacyPM(StringRef Pass) {
       "expand-reductions",
       "indirectbr-expand",
       "generic-to-nvvm",
-      "expandmemcmp",
+      "expand-memcmp",
       "loop-reduce",
       "lower-amx-type",
       "lower-amx-intrinsics",
@@ -422,7 +422,7 @@ int main(int argc, char **argv) {
   // supported.
   initializeExpandLargeDivRemLegacyPassPass(Registry);
   initializeExpandLargeFpConvertLegacyPassPass(Registry);
-  initializeExpandMemCmpPassPass(Registry);
+  initializeExpandMemCmpLegacyPassPass(Registry);
   initializeScalarizeMaskedMemIntrinLegacyPassPass(Registry);
   initializeSelectOptimizePass(Registry);
   initializeCallBrPreparePass(Registry);
@@ -434,7 +434,7 @@ int main(int argc, char **argv) {
   initializeSjLjEHPreparePass(Registry);
   initializePreISelIntrinsicLoweringLegacyPassPass(Registry);
   initializeGlobalMergePass(Registry);
-  initializeIndirectBrExpandPassPass(Registry);
+  initializeIndirectBrExpandLegacyPassPass(Registry);
   initializeInterleavedLoadCombinePass(Registry);
   initializeInterleavedAccessPass(Registry);
   initializeUnreachableBlockElimLegacyPassPass(Registry);
diff --git a/llvm/unittests/ADT/StringMapTest.cpp b/llvm/unittests/ADT/StringMapTest.cpp
index c9ef3f8a096ee908620a2b37ac9766b4c1cdd97c..f9b138e9a472137139397d9cae76823711594211 100644
--- a/llvm/unittests/ADT/StringMapTest.cpp
+++ b/llvm/unittests/ADT/StringMapTest.cpp
@@ -487,17 +487,6 @@ TEST_F(StringMapTest, NotEqualWithDifferentValues) {
   ASSERT_TRUE(B != A);
 }
 
-TEST_F(StringMapTest, PrecomputedHash) {
-  StringMap<int> A;
-  StringRef Key = "foo";
-  int Value = 42;
-  uint64_t Hash = StringMap<int>::hash(Key);
-  A.insert({"foo", Value}, Hash);
-  auto I = A.find(Key, Hash);
-  ASSERT_NE(I, A.end());
-  ASSERT_EQ(I->second, Value);
-}
-
 struct Countable {
   int &InstanceCount;
   int Number;
diff --git a/llvm/unittests/Analysis/CFGTest.cpp b/llvm/unittests/Analysis/CFGTest.cpp
index 3ffad1c3238029d4ff70d081c33c0f88dde8e33a..46164268468628c8032d7b2422149f4b417571eb 100644
--- a/llvm/unittests/Analysis/CFGTest.cpp
+++ b/llvm/unittests/Analysis/CFGTest.cpp
@@ -61,7 +61,7 @@ protected:
 
     assert(ExclusionSet.empty());
     for (auto I = F->begin(), E = F->end(); I != E; ++I) {
-      if (I->hasName() && I->getName().startswith("excluded"))
+      if (I->hasName() && I->getName().starts_with("excluded"))
         ExclusionSet.insert(&*I);
     }
   }
diff --git a/llvm/unittests/Analysis/TBAATest.cpp b/llvm/unittests/Analysis/TBAATest.cpp
index 3367aefcd608f05827858478d698ff4fd4c617b9..3e81968b483568cbe1feafef452f75c9fccdf320 100644
--- a/llvm/unittests/Analysis/TBAATest.cpp
+++ b/llvm/unittests/Analysis/TBAATest.cpp
@@ -59,7 +59,7 @@ TEST_F(TBAATest, checkVerifierBehaviorForOldTBAA) {
 
   EXPECT_TRUE(verifyFunction(*F, &Outs));
   EXPECT_TRUE(StringRef(ErrorMsg.begin(), ErrorMsg.size())
-                  .startswith(ExpectedFailureMsg));
+                  .starts_with(ExpectedFailureMsg));
 }
 
 TEST_F(TBAATest, checkTBAAMerging) {
diff --git a/llvm/unittests/BinaryFormat/TestFileMagic.cpp b/llvm/unittests/BinaryFormat/TestFileMagic.cpp
index b09a2c07309f82b0fa72b1592877c0f1d11bf1fe..766f8bfae72b00f3405b59fc0690f279d92f5a29 100644
--- a/llvm/unittests/BinaryFormat/TestFileMagic.cpp
+++ b/llvm/unittests/BinaryFormat/TestFileMagic.cpp
@@ -87,6 +87,8 @@ const char pdb[] = "Microsoft C/C++ MSF 7.00\r\n\x1a"
                    "DS\x00\x00\x00";
 const char tapi_file[] = "--- !tapi-tbd-v1\n";
 const char tapi_file_tbd_v1[] = "---\narchs: [";
+const char spirv_object_le[] = "\x03\x02\x23\x07";
+const char spirv_object_be[] = "\x07\x23\x02\x03";
 
 TEST_F(MagicTest, Magic) {
   struct type {
@@ -117,6 +119,10 @@ TEST_F(MagicTest, Magic) {
       DEFINE(macho_dynamically_linked_shared_lib_stub),
       DEFINE(macho_dsym_companion),
       DEFINE(macho_kext_bundle),
+      {"spirv_object_le", spirv_object_le, sizeof(spirv_object_le),
+       file_magic ::spirv_object},
+      {"spirv_object_be", spirv_object_be, sizeof(spirv_object_be),
+       file_magic ::spirv_object},
       DEFINE(windows_resource),
       DEFINE(pdb),
       {"ms_dos_stub_broken", ms_dos_stub_broken, sizeof(ms_dos_stub_broken),
diff --git a/llvm/unittests/CodeGen/InstrRefLDVTest.cpp b/llvm/unittests/CodeGen/InstrRefLDVTest.cpp
index 5a050bc520362c2a5e786289e6efee1f63f25ddb..acbcd247fa9a011e4daaaf01428b1f430edab974 100644
--- a/llvm/unittests/CodeGen/InstrRefLDVTest.cpp
+++ b/llvm/unittests/CodeGen/InstrRefLDVTest.cpp
@@ -497,15 +497,8 @@ body:  |
 
   std::pair<FuncValueTable, FuncValueTable>
   allocValueTables(unsigned Blocks, unsigned Locs) {
-    FuncValueTable MOutLocs = std::make_unique<ValueTable[]>(Blocks);
-    FuncValueTable MInLocs = std::make_unique<ValueTable[]>(Blocks);
-
-    for (unsigned int I = 0; I < Blocks; ++I) {
-      MOutLocs[I] = std::make_unique<ValueIDNum[]>(Locs);
-      MInLocs[I] = std::make_unique<ValueIDNum[]>(Locs);
-    }
-
-    return std::make_pair(std::move(MOutLocs), std::move(MInLocs));
+    return {FuncValueTable(Blocks, ValueTable(Locs)),
+            FuncValueTable(Blocks, ValueTable(Locs))};
   }
 };
 
diff --git a/llvm/unittests/CodeGen/MachineInstrTest.cpp b/llvm/unittests/CodeGen/MachineInstrTest.cpp
index 0841cd3a7fb04f3a4f9df12a076d93fedca7d520..7c45d9784bb8e7a21282dcbede3fb5d7fec12f2b 100644
--- a/llvm/unittests/CodeGen/MachineInstrTest.cpp
+++ b/llvm/unittests/CodeGen/MachineInstrTest.cpp
@@ -222,9 +222,8 @@ TEST(MachineInstrPrintingTest, DebugLocPrinting) {
   MI->print(OS, /*IsStandalone*/true, /*SkipOpers*/false, /*SkipDebugLoc*/false,
             /*AddNewLine*/false);
   ASSERT_TRUE(
-      StringRef(OS.str()).startswith("$noreg = UNKNOWN debug-location "));
-  ASSERT_TRUE(
-      StringRef(OS.str()).endswith("filename:1:5"));
+      StringRef(OS.str()).starts_with("$noreg = UNKNOWN debug-location "));
+  ASSERT_TRUE(StringRef(OS.str()).ends_with("filename:1:5"));
 }
 
 TEST(MachineInstrSpan, DistanceBegin) {
diff --git a/llvm/unittests/DebugInfo/LogicalView/CodeViewReaderTest.cpp b/llvm/unittests/DebugInfo/LogicalView/CodeViewReaderTest.cpp
index eb9477e40ed90b7ab0ac599b851ec33229d67ed3..125bc17a0067119fc76f2bd4c85496f89bec8bb0 100644
--- a/llvm/unittests/DebugInfo/LogicalView/CodeViewReaderTest.cpp
+++ b/llvm/unittests/DebugInfo/LogicalView/CodeViewReaderTest.cpp
@@ -73,7 +73,7 @@ void checkElementPropertiesClangCodeview(LVReader *Reader) {
   EXPECT_EQ(Root->getName(), CodeViewClang);
 
   EXPECT_EQ(CompileUnit->getBaseAddress(), 0u);
-  EXPECT_TRUE(CompileUnit->getProducer().startswith("clang"));
+  EXPECT_TRUE(CompileUnit->getProducer().starts_with("clang"));
   EXPECT_EQ(CompileUnit->getName(), "test.cpp");
 
   EXPECT_EQ(Function->lineCount(), 16u);
@@ -139,7 +139,7 @@ void checkElementPropertiesMsvcCodeview(LVReader *Reader) {
   EXPECT_EQ(Root->getName(), CodeViewMsvc);
 
   EXPECT_EQ(CompileUnit->getBaseAddress(), 0u);
-  EXPECT_TRUE(CompileUnit->getProducer().startswith("Microsoft"));
+  EXPECT_TRUE(CompileUnit->getProducer().starts_with("Microsoft"));
   EXPECT_EQ(CompileUnit->getName(), "test.cpp");
 
   EXPECT_EQ(Function->lineCount(), 14u);
@@ -205,7 +205,7 @@ void checkElementPropertiesMsvcCodeviewPdb(LVReader *Reader) {
   EXPECT_EQ(Root->getName(), CodeViewPdbMsvc);
 
   EXPECT_EQ(CompileUnit->getBaseAddress(), 0u);
-  EXPECT_TRUE(CompileUnit->getProducer().startswith("Microsoft"));
+  EXPECT_TRUE(CompileUnit->getProducer().starts_with("Microsoft"));
   EXPECT_EQ(CompileUnit->getName(), "test.cpp");
 
   EXPECT_EQ(Function->lineCount(), 14u);
diff --git a/llvm/unittests/DebugInfo/LogicalView/ELFReaderTest.cpp b/llvm/unittests/DebugInfo/LogicalView/ELFReaderTest.cpp
index f6db28afef20f7de17889c21f3669de52725d3bf..bd278befbc642f15e23541815749b4b5112eacb5 100644
--- a/llvm/unittests/DebugInfo/LogicalView/ELFReaderTest.cpp
+++ b/llvm/unittests/DebugInfo/LogicalView/ELFReaderTest.cpp
@@ -70,7 +70,7 @@ void checkElementProperties(LVReader *Reader) {
   EXPECT_EQ(Root->getName(), DwarfClang);
 
   EXPECT_EQ(CompileUnit->getBaseAddress(), 0u);
-  EXPECT_TRUE(CompileUnit->getProducer().startswith("clang"));
+  EXPECT_TRUE(CompileUnit->getProducer().starts_with("clang"));
   EXPECT_EQ(CompileUnit->getName(), "test.cpp");
 
   EXPECT_EQ(CompileUnit->lineCount(), 0u);
diff --git a/llvm/unittests/ExecutionEngine/MCJIT/MCJITTestAPICommon.h b/llvm/unittests/ExecutionEngine/MCJIT/MCJITTestAPICommon.h
index 7071681c26aaedb99dd6fee2b88ead5cda222fe2..13484e1bd9029a48fb340783ccd158fb51487a3f 100644
--- a/llvm/unittests/ExecutionEngine/MCJIT/MCJITTestAPICommon.h
+++ b/llvm/unittests/ExecutionEngine/MCJIT/MCJITTestAPICommon.h
@@ -73,7 +73,7 @@ protected:
     // If ARCH has sub-arch support, find it
     SmallVectorImpl<std::string>::const_iterator I = SupportedSubArchs.begin();
     for(; I != SupportedSubArchs.end(); ++I)
-      if (Host.getArchName().startswith(*I))
+      if (Host.getArchName().starts_with(*I))
         return true;
 
     return false;
diff --git a/llvm/unittests/ExecutionEngine/Orc/OrcCAPITest.cpp b/llvm/unittests/ExecutionEngine/Orc/OrcCAPITest.cpp
index d62973f30add90ac85565011de20c78553a301be..12f42633d03728ff1deea34b20b6dccb8ec37454 100644
--- a/llvm/unittests/ExecutionEngine/Orc/OrcCAPITest.cpp
+++ b/llvm/unittests/ExecutionEngine/Orc/OrcCAPITest.cpp
@@ -124,7 +124,7 @@ protected:
     // TODO: Print error messages in failure logs, use them to audit this list.
     // Some architectures may be unsupportable or missing key components, but
     // some may just be failing due to bugs in this testcase.
-    if (Triple.startswith("armv7") || Triple.startswith("armv8l"))
+    if (Triple.starts_with("armv7") || Triple.starts_with("armv8l"))
       return false;
     return true;
   }
diff --git a/llvm/unittests/FileCheck/FileCheckTest.cpp b/llvm/unittests/FileCheck/FileCheckTest.cpp
index 317fc4493cb5d44bb1d809132322c5edfec9b72c..91fce69b9a6c4b8bdefdfd9d35fe7cb0bd07793f 100644
--- a/llvm/unittests/FileCheck/FileCheckTest.cpp
+++ b/llvm/unittests/FileCheck/FileCheckTest.cpp
@@ -172,7 +172,7 @@ struct ExpressionFormatParameterisedFixture
   }
 
   std::string padWithLeadingZeros(StringRef NumStr) const {
-    bool Negative = NumStr.startswith("-");
+    bool Negative = NumStr.starts_with("-");
     if (NumStr.size() - unsigned(Negative) >= Precision)
       return NumStr.str();
 
diff --git a/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp b/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp
index 193ada3a4ea327de2a9e26ac11435c10d4dc7ecf..e79d0bb2f65aeac73fecd5d37cadc97627bb0796 100644
--- a/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp
+++ b/llvm/unittests/Frontend/OpenMPIRBuilderTest.cpp
@@ -5821,7 +5821,7 @@ TEST_F(OpenMPIRBuilderTest, TargetRegion) {
   Function *KernelLaunchFunc = Call->getCalledFunction();
   EXPECT_NE(KernelLaunchFunc, nullptr);
   StringRef FunctionName = KernelLaunchFunc->getName();
-  EXPECT_TRUE(FunctionName.startswith("__tgt_target_kernel"));
+  EXPECT_TRUE(FunctionName.starts_with("__tgt_target_kernel"));
 
   // Check the fallback call
   BasicBlock *FallbackBlock = Branch->getSuccessor(0);
@@ -5838,7 +5838,7 @@ TEST_F(OpenMPIRBuilderTest, TargetRegion) {
   Function *OutlinedFunc = FCall->getCalledFunction();
   EXPECT_NE(OutlinedFunc, nullptr);
   StringRef FunctionName2 = OutlinedFunc->getName();
-  EXPECT_TRUE(FunctionName2.startswith("__omp_offloading"));
+  EXPECT_TRUE(FunctionName2.starts_with("__omp_offloading"));
 
   EXPECT_FALSE(verifyModule(*M, &errs()));
 }
diff --git a/llvm/unittests/IR/VerifierTest.cpp b/llvm/unittests/IR/VerifierTest.cpp
index 2e14bfc42670eb69f515bdc40067ca9e821fd9f1..31e3b9dfab4bfd5f3e4d3edf08a49145b67e9a32 100644
--- a/llvm/unittests/IR/VerifierTest.cpp
+++ b/llvm/unittests/IR/VerifierTest.cpp
@@ -105,8 +105,10 @@ TEST(VerifierTest, InvalidRetAttribute) {
   std::string Error;
   raw_string_ostream ErrorOS(Error);
   EXPECT_TRUE(verifyModule(M, &ErrorOS));
-  EXPECT_TRUE(StringRef(ErrorOS.str()).startswith(
-      "Attribute 'uwtable' does not apply to function return values"));
+  EXPECT_TRUE(
+      StringRef(ErrorOS.str())
+          .starts_with(
+              "Attribute 'uwtable' does not apply to function return values"));
 }
 
 /// Test the verifier rejects invalid nofpclass values that the assembler may
@@ -134,11 +136,11 @@ TEST(VerifierTest, InvalidNoFPClassAttribute) {
     StringRef ErrMsg(ErrorOS.str());
 
     if (InvalidMask == 0) {
-      EXPECT_TRUE(ErrMsg.startswith(
+      EXPECT_TRUE(ErrMsg.starts_with(
           "Attribute 'nofpclass' must have at least one test bit set"))
           << ErrMsg;
     } else {
-      EXPECT_TRUE(ErrMsg.startswith("Invalid value for 'nofpclass' test mask"))
+      EXPECT_TRUE(ErrMsg.starts_with("Invalid value for 'nofpclass' test mask"))
           << ErrMsg;
     }
   }
@@ -195,8 +197,9 @@ TEST(VerifierTest, CrossModuleRef) {
 
   Error.clear();
   EXPECT_TRUE(verifyModule(M3, &ErrorOS));
-  EXPECT_TRUE(StringRef(ErrorOS.str()).startswith(
-      "Referencing personality function in another module!"));
+  EXPECT_TRUE(
+      StringRef(ErrorOS.str())
+          .starts_with("Referencing personality function in another module!"));
 
   // Erase bad methods to avoid triggering an assertion failure on destruction
   F1->eraseFromParent();
@@ -211,9 +214,9 @@ TEST(VerifierTest, InvalidVariableLinkage) {
   std::string Error;
   raw_string_ostream ErrorOS(Error);
   EXPECT_TRUE(verifyModule(M, &ErrorOS));
-  EXPECT_TRUE(
-      StringRef(ErrorOS.str()).startswith("Global is external, but doesn't "
-                                          "have external or weak linkage!"));
+  EXPECT_TRUE(StringRef(ErrorOS.str())
+                  .starts_with("Global is external, but doesn't "
+                               "have external or weak linkage!"));
 }
 
 TEST(VerifierTest, InvalidFunctionLinkage) {
@@ -225,9 +228,9 @@ TEST(VerifierTest, InvalidFunctionLinkage) {
   std::string Error;
   raw_string_ostream ErrorOS(Error);
   EXPECT_TRUE(verifyModule(M, &ErrorOS));
-  EXPECT_TRUE(
-      StringRef(ErrorOS.str()).startswith("Global is external, but doesn't "
-                                          "have external or weak linkage!"));
+  EXPECT_TRUE(StringRef(ErrorOS.str())
+                  .starts_with("Global is external, but doesn't "
+                               "have external or weak linkage!"));
 }
 
 TEST(VerifierTest, DetectInvalidDebugInfo) {
@@ -284,8 +287,9 @@ TEST(VerifierTest, MDNodeWrongContext) {
   std::string Error;
   raw_string_ostream ErrorOS(Error);
   EXPECT_TRUE(verifyModule(M, &ErrorOS));
-  EXPECT_TRUE(StringRef(ErrorOS.str())
-                  .startswith("MDNode context does not match Module context!"));
+  EXPECT_TRUE(
+      StringRef(ErrorOS.str())
+          .starts_with("MDNode context does not match Module context!"));
 }
 
 TEST(VerifierTest, AttributesWrongContext) {
diff --git a/llvm/unittests/ProfileData/CoverageMappingTest.cpp b/llvm/unittests/ProfileData/CoverageMappingTest.cpp
index 873bc05b2ecc3f3436d227ddcdf4b1caee3ff567..1cf497cbdc2e61f8fc2df974f66ea174574550bb 100644
--- a/llvm/unittests/ProfileData/CoverageMappingTest.cpp
+++ b/llvm/unittests/ProfileData/CoverageMappingTest.cpp
@@ -187,6 +187,25 @@ struct CoverageMappingTest : ::testing::TestWithParam<std::tuple<bool, bool>> {
                 : CounterMappingRegion::makeRegion(C, FileID, LS, CS, LE, CE));
   }
 
+  void addMCDCDecisionCMR(unsigned Mask, unsigned NC, StringRef File,
+                          unsigned LS, unsigned CS, unsigned LE, unsigned CE) {
+    auto &Regions = InputFunctions.back().Regions;
+    unsigned FileID = getFileIndexForFunction(File);
+    Regions.push_back(CounterMappingRegion::makeDecisionRegion(
+        CounterMappingRegion::MCDCParameters{Mask, NC}, FileID, LS, CS, LE,
+        CE));
+  }
+
+  void addMCDCBranchCMR(Counter C1, Counter C2, unsigned ID, unsigned TrueID,
+                        unsigned FalseID, StringRef File, unsigned LS,
+                        unsigned CS, unsigned LE, unsigned CE) {
+    auto &Regions = InputFunctions.back().Regions;
+    unsigned FileID = getFileIndexForFunction(File);
+    Regions.push_back(CounterMappingRegion::makeBranchRegion(
+        C1, C2, CounterMappingRegion::MCDCParameters{0, 0, ID, TrueID, FalseID},
+        FileID, LS, CS, LE, CE));
+  }
+
   void addExpansionCMR(StringRef File, StringRef ExpandedFile, unsigned LS,
                        unsigned CS, unsigned LE, unsigned CE) {
     InputFunctions.back().Regions.push_back(CounterMappingRegion::makeExpansion(
@@ -828,6 +847,33 @@ TEST_P(CoverageMappingTest, non_code_region_counters) {
   ASSERT_EQ(1U, Names.size());
 }
 
+// Test that MCDC bitmasks not associated with any code regions are allowed.
+TEST_P(CoverageMappingTest, non_code_region_bitmask) {
+  // No records in profdata
+
+  startFunction("func", 0x1234);
+  addCMR(Counter::getCounter(0), "file", 1, 1, 5, 5);
+  addCMR(Counter::getCounter(1), "file", 1, 1, 5, 5);
+  addCMR(Counter::getCounter(2), "file", 1, 1, 5, 5);
+  addCMR(Counter::getCounter(3), "file", 1, 1, 5, 5);
+
+  addMCDCDecisionCMR(0, 2, "file", 7, 1, 7, 6);
+  addMCDCBranchCMR(Counter::getCounter(0), Counter::getCounter(1), 1, 2, 0,
+                   "file", 7, 2, 7, 3);
+  addMCDCBranchCMR(Counter::getCounter(2), Counter::getCounter(3), 2, 0, 0,
+                   "file", 7, 4, 7, 5);
+
+  EXPECT_THAT_ERROR(loadCoverageMapping(), Succeeded());
+
+  std::vector<std::string> Names;
+  for (const auto &Func : LoadedCoverage->getCoveredFunctions()) {
+    Names.push_back(Func.Name);
+    ASSERT_EQ(2U, Func.CountedBranchRegions.size());
+    ASSERT_EQ(1U, Func.MCDCRecords.size());
+  }
+  ASSERT_EQ(1U, Names.size());
+}
+
 TEST_P(CoverageMappingTest, strip_filename_prefix) {
   ProfileWriter.addRecord({"file1:func", 0x1234, {0}}, Err);
 
diff --git a/llvm/unittests/Support/CMakeLists.txt b/llvm/unittests/Support/CMakeLists.txt
index e1bf793536b686227c227518f2ca4f4f1e281fa6..df35a7b7f3626ac9dc26013abd5c1e5f13fc0f1e 100644
--- a/llvm/unittests/Support/CMakeLists.txt
+++ b/llvm/unittests/Support/CMakeLists.txt
@@ -103,6 +103,7 @@ add_llvm_unittest(SupportTests
   raw_ostream_test.cpp
   raw_pwrite_stream_test.cpp
   raw_sha1_ostream_test.cpp
+  raw_socket_stream_test.cpp
   xxhashTest.cpp
 
   DEPENDS
diff --git a/llvm/unittests/Support/CommandLineTest.cpp b/llvm/unittests/Support/CommandLineTest.cpp
index 762ac0ea9c36dde9340fd0da4a99199ba44ff876..ae80490a33734acd156f1ff7c461a3de2185f788 100644
--- a/llvm/unittests/Support/CommandLineTest.cpp
+++ b/llvm/unittests/Support/CommandLineTest.cpp
@@ -2244,4 +2244,34 @@ TEST(CommandLineTest, HelpWithSubcommands) {
   cl::ResetCommandLineParser();
 }
 
+TEST(CommandLineTest, UnknownCommands) {
+  cl::ResetCommandLineParser();
+
+  StackSubCommand SC1("foo", "Foo subcommand");
+  StackSubCommand SC2("bar", "Bar subcommand");
+  StackOption<bool> SC1Opt("put", cl::sub(SC1));
+  StackOption<bool> SC2Opt("get", cl::sub(SC2));
+  StackOption<bool> TopOpt1("peek");
+  StackOption<bool> TopOpt2("set");
+
+  std::string Errs;
+  raw_string_ostream OS(Errs);
+
+  const char *Args1[] = {"prog", "baz", "--get"};
+  EXPECT_FALSE(
+      cl::ParseCommandLineOptions(std::size(Args1), Args1, StringRef(), &OS));
+  EXPECT_EQ(Errs,
+            "prog: Unknown subcommand 'baz'.  Try: 'prog --help'\n"
+            "prog: Did you mean 'bar'?\n"
+            "prog: Unknown command line argument '--get'.  Try: 'prog --help'\n"
+            "prog: Did you mean '--set'?\n");
+
+  // Do not show a suggestion if the subcommand is not similar to any known.
+  Errs.clear();
+  const char *Args2[] = {"prog", "faz"};
+  EXPECT_FALSE(
+      cl::ParseCommandLineOptions(std::size(Args2), Args2, StringRef(), &OS));
+  EXPECT_EQ(Errs, "prog: Unknown subcommand 'faz'.  Try: 'prog --help'\n");
+}
+
 } // anonymous namespace
diff --git a/llvm/unittests/Support/MemoryBufferTest.cpp b/llvm/unittests/Support/MemoryBufferTest.cpp
index bfabd734f02ef863345f16eb5f2e2da61759f083..cfee3e477d2ede351a21395dae0c766e8c6790e8 100644
--- a/llvm/unittests/Support/MemoryBufferTest.cpp
+++ b/llvm/unittests/Support/MemoryBufferTest.cpp
@@ -432,7 +432,7 @@ TEST_F(MemoryBufferTest, mmapVolatileNoNull) {
   OwningBuffer MB = std::move(*MBOrError);
   EXPECT_EQ(MB->getBufferKind(), MemoryBuffer::MemoryBuffer_MMap);
   EXPECT_EQ(MB->getBufferSize(), std::size_t(FileWrites * 8));
-  EXPECT_TRUE(MB->getBuffer().startswith("01234567"));
+  EXPECT_TRUE(MB->getBuffer().starts_with("01234567"));
 }
 
 // Test that SmallVector without a null terminator gets one.
diff --git a/llvm/unittests/Support/ProgramTest.cpp b/llvm/unittests/Support/ProgramTest.cpp
index ca301dc3cc9633684afd8e16557ddafa3982d4b7..91dbb57fbdeca607adc8664edb585b02ed23589b 100644
--- a/llvm/unittests/Support/ProgramTest.cpp
+++ b/llvm/unittests/Support/ProgramTest.cpp
@@ -96,7 +96,7 @@ protected:
 
     while (*EnvP != nullptr) {
       auto S = prepareEnvVar(*EnvP);
-      if (!StringRef(S).startswith("GTEST_"))
+      if (!StringRef(S).starts_with("GTEST_"))
         EnvTable.emplace_back(S);
       ++EnvP;
     }
diff --git a/llvm/unittests/Support/RISCVISAInfoTest.cpp b/llvm/unittests/Support/RISCVISAInfoTest.cpp
index 549964eed55518e8ab5cb4a0d2b87b289ecc3372..3de14907899eb6bccd47f9fb215aaa259cdd7795 100644
--- a/llvm/unittests/Support/RISCVISAInfoTest.cpp
+++ b/llvm/unittests/Support/RISCVISAInfoTest.cpp
@@ -736,7 +736,7 @@ R"(All available -march extensions for RISC-V
     xventanacondops     1.0
 
 Experimental extensions
-    zicfilp             0.2       This is a long dummy description
+    zicfilp             0.4       This is a long dummy description
     zicond              1.0
     zacas               1.0
     zfbfmin             0.8
diff --git a/llvm/unittests/Support/TarWriterTest.cpp b/llvm/unittests/Support/TarWriterTest.cpp
index b5f072431fd15f2cb35186b9c7c63c5e26c3c0b3..d640470ba636578fd8d92f9247d88845500069b0 100644
--- a/llvm/unittests/Support/TarWriterTest.cpp
+++ b/llvm/unittests/Support/TarWriterTest.cpp
@@ -119,7 +119,7 @@ TEST_F(TarWriterTest, Pax) {
   EXPECT_EQ("", StringRef(Hdr->Name));
 
   StringRef Pax = StringRef((char *)(Buf.data() + 512), 512);
-  EXPECT_TRUE(Pax.startswith("211 path=/" + std::string(200, 'x')));
+  EXPECT_TRUE(Pax.starts_with("211 path=/" + std::string(200, 'x')));
 }
 
 TEST_F(TarWriterTest, SingleFile) {
diff --git a/llvm/unittests/Support/TypeNameTest.cpp b/llvm/unittests/Support/TypeNameTest.cpp
index c8962933e9cee66a69de5536d0a40934a0a3537b..e0d9ac5ee7e380298d96b269ba02f4ac4e5df288 100644
--- a/llvm/unittests/Support/TypeNameTest.cpp
+++ b/llvm/unittests/Support/TypeNameTest.cpp
@@ -29,11 +29,11 @@ TEST(TypeNameTest, Names) {
 
 #if defined(__clang__) || defined(__GNUC__) || defined(__INTEL_COMPILER) ||    \
     defined(_MSC_VER)
-  EXPECT_TRUE(S1Name.endswith("::N1::S1")) << S1Name.str();
-  EXPECT_TRUE(C1Name.endswith("::N1::C1")) << C1Name.str();
-  EXPECT_TRUE(U1Name.endswith("::N1::U1")) << U1Name.str();
+  EXPECT_TRUE(S1Name.ends_with("::N1::S1")) << S1Name.str();
+  EXPECT_TRUE(C1Name.ends_with("::N1::C1")) << C1Name.str();
+  EXPECT_TRUE(U1Name.ends_with("::N1::U1")) << U1Name.str();
 #ifdef __clang__
-  EXPECT_TRUE(S2Name.endswith("S2")) << S2Name.str();
+  EXPECT_TRUE(S2Name.ends_with("S2")) << S2Name.str();
 #else
   EXPECT_TRUE(S2Name.endswith("::S2")) << S2Name.str();
 #endif
diff --git a/llvm/unittests/Support/VirtualFileSystemTest.cpp b/llvm/unittests/Support/VirtualFileSystemTest.cpp
index bd4048f025c0de038e7faf81ab02a5d8d8b94221..15f5204e7e804359c2eb7009f6e810458c283f98 100644
--- a/llvm/unittests/Support/VirtualFileSystemTest.cpp
+++ b/llvm/unittests/Support/VirtualFileSystemTest.cpp
@@ -452,11 +452,11 @@ TEST(VirtualFileSystemTest, BasicRealFSIteration) {
   ASSERT_FALSE(EC);
   ASSERT_NE(vfs::directory_iterator(), I);
   // Check either a or c, since we can't rely on the iteration order.
-  EXPECT_TRUE(I->path().endswith("a") || I->path().endswith("c"));
+  EXPECT_TRUE(I->path().ends_with("a") || I->path().ends_with("c"));
   I.increment(EC);
   ASSERT_FALSE(EC);
   ASSERT_NE(vfs::directory_iterator(), I);
-  EXPECT_TRUE(I->path().endswith("a") || I->path().endswith("c"));
+  EXPECT_TRUE(I->path().ends_with("a") || I->path().ends_with("c"));
   I.increment(EC);
   EXPECT_EQ(vfs::directory_iterator(), I);
 }
diff --git a/llvm/unittests/Support/raw_socket_stream_test.cpp b/llvm/unittests/Support/raw_socket_stream_test.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..0a2a8b5a2c653ebb76003d93598851cd11afaccb
--- /dev/null
+++ b/llvm/unittests/Support/raw_socket_stream_test.cpp
@@ -0,0 +1,68 @@
+#include "llvm/ADT/SmallString.h"
+#include "llvm/Config/llvm-config.h"
+#include "llvm/Support/Casting.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/FileUtilities.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Testing/Support/Error.h"
+#include "gtest/gtest.h"
+#include <future>
+#include <iostream>
+#include <stdlib.h>
+
+#ifdef _WIN32
+#include "llvm/Support/Windows/WindowsSupport.h"
+#endif
+
+using namespace llvm;
+
+namespace {
+
+bool hasUnixSocketSupport() {
+#ifdef _WIN32
+  VersionTuple Ver = GetWindowsOSVersion();
+  if (Ver < VersionTuple(10, 0, 0, 17063))
+    return false;
+#endif
+  return true;
+}
+
+TEST(raw_socket_streamTest, CLIENT_TO_SERVER_AND_SERVER_TO_CLIENT) {
+  if (!hasUnixSocketSupport())
+    GTEST_SKIP();
+
+  SmallString<100> SocketPath;
+  llvm::sys::fs::createUniquePath("test_raw_socket_stream.sock", SocketPath,
+                                  true);
+
+  char Bytes[8];
+
+  Expected<ListeningSocket> MaybeServerListener =
+      ListeningSocket::createUnix(SocketPath);
+  ASSERT_THAT_EXPECTED(MaybeServerListener, llvm::Succeeded());
+
+  ListeningSocket ServerListener = std::move(*MaybeServerListener);
+
+  Expected<std::unique_ptr<raw_socket_stream>> MaybeClient =
+      raw_socket_stream::createConnectedUnix(SocketPath);
+  ASSERT_THAT_EXPECTED(MaybeClient, llvm::Succeeded());
+
+  raw_socket_stream &Client = **MaybeClient;
+
+  Expected<std::unique_ptr<raw_socket_stream>> MaybeServer =
+      ServerListener.accept();
+  ASSERT_THAT_EXPECTED(MaybeServer, llvm::Succeeded());
+
+  raw_socket_stream &Server = **MaybeServer;
+
+  Client << "01234567";
+  Client.flush();
+
+  ssize_t BytesRead = Server.read(Bytes, 8);
+
+  std::string string(Bytes, 8);
+
+  ASSERT_EQ(8, BytesRead);
+  ASSERT_EQ("01234567", string);
+}
+} // namespace
\ No newline at end of file
diff --git a/llvm/unittests/TargetParser/CSKYTargetParserTest.cpp b/llvm/unittests/TargetParser/CSKYTargetParserTest.cpp
index 4f0b52616b78b02f2f2116eb86f53b1a40aa7277..f28a2a33eb90c84afbc78cb835fe084bd992b707 100644
--- a/llvm/unittests/TargetParser/CSKYTargetParserTest.cpp
+++ b/llvm/unittests/TargetParser/CSKYTargetParserTest.cpp
@@ -28,7 +28,7 @@ std::string FormatExtensionFlags(uint64_t Flags) {
 
   Features.erase(std::remove_if(Features.begin(), Features.end(),
                                 [](StringRef extension) {
-                                  return extension.startswith("-");
+                                  return extension.starts_with("-");
                                 }),
                  Features.end());
 
diff --git a/llvm/unittests/TargetParser/TargetParserTest.cpp b/llvm/unittests/TargetParser/TargetParserTest.cpp
index 687b56cc5aa20bcd94c9c3ffb3c4bfa7a1b072fb..e2b9712b511ecaf7dce2761b0da554f306b11483 100644
--- a/llvm/unittests/TargetParser/TargetParserTest.cpp
+++ b/llvm/unittests/TargetParser/TargetParserTest.cpp
@@ -55,7 +55,7 @@ std::string FormatExtensionFlags(int64_t Flags) {
   // E.g. if AEK_CRC is not set then it adds "-crc". Not useful here.
   Features.erase(std::remove_if(Features.begin(), Features.end(),
                                 [](StringRef extension) {
-                                  return extension.startswith("-");
+                                  return extension.starts_with("-");
                                 }),
                  Features.end());
 
@@ -75,7 +75,7 @@ std::string FormatExtensionFlags(AArch64::ExtensionBitset Flags) {
   // E.g. if AEK_CRC is not set then it adds "-crc". Not useful here.
   Features.erase(std::remove_if(Features.begin(), Features.end(),
                                 [](StringRef extension) {
-                                  return extension.startswith("-");
+                                  return extension.starts_with("-");
                                 }),
                  Features.end());
 
diff --git a/llvm/unittests/Transforms/Coroutines/ExtraRematTest.cpp b/llvm/unittests/Transforms/Coroutines/ExtraRematTest.cpp
index df4d6c383abe6ed73d1b2f70100a06375f460aa6..da78c151e7f68d0a714d78b175a6bdea1c4984bc 100644
--- a/llvm/unittests/Transforms/Coroutines/ExtraRematTest.cpp
+++ b/llvm/unittests/Transforms/Coroutines/ExtraRematTest.cpp
@@ -126,7 +126,7 @@ bool ExtraMaterializable(Instruction &I) {
 
   if (auto *CI = dyn_cast<CallInst>(&I)) {
     auto *CalledFunc = CI->getCalledFunction();
-    if (CalledFunc && CalledFunc->getName().startswith("should.remat"))
+    if (CalledFunc && CalledFunc->getName().starts_with("should.remat"))
       return true;
   }
 
diff --git a/llvm/unittests/Transforms/Utils/CloningTest.cpp b/llvm/unittests/Transforms/Utils/CloningTest.cpp
index e083c75bf9c92d5d40d8b4a8e15c329255b61259..025771f07ce5d453f6a7cf1e504bfb5ee3813151 100644
--- a/llvm/unittests/Transforms/Utils/CloningTest.cpp
+++ b/llvm/unittests/Transforms/Utils/CloningTest.cpp
@@ -440,11 +440,11 @@ for.end:
         EXPECT_NE(NewLoop, nullptr);
         EXPECT_EQ(NewLoop->getSubLoops().size(), 1u);
         Loop::block_iterator BI = NewLoop->block_begin();
-        EXPECT_TRUE((*BI)->getName().startswith("for.outer"));
-        EXPECT_TRUE((*(++BI))->getName().startswith("for.inner.preheader"));
-        EXPECT_TRUE((*(++BI))->getName().startswith("for.inner"));
-        EXPECT_TRUE((*(++BI))->getName().startswith("for.inner.exit"));
-        EXPECT_TRUE((*(++BI))->getName().startswith("for.outer.latch"));
+        EXPECT_TRUE((*BI)->getName().starts_with("for.outer"));
+        EXPECT_TRUE((*(++BI))->getName().starts_with("for.inner.preheader"));
+        EXPECT_TRUE((*(++BI))->getName().starts_with("for.inner"));
+        EXPECT_TRUE((*(++BI))->getName().starts_with("for.inner.exit"));
+        EXPECT_TRUE((*(++BI))->getName().starts_with("for.outer.latch"));
       });
 }
 
diff --git a/llvm/utils/TableGen/GlobalISelMatchTable.cpp b/llvm/utils/TableGen/GlobalISelMatchTable.cpp
index 481f3f16e0136c696bc4bf869c0a56c3380c1e29..349598266aa991a4e6655006b4b1eda2c8bbd986 100644
--- a/llvm/utils/TableGen/GlobalISelMatchTable.cpp
+++ b/llvm/utils/TableGen/GlobalISelMatchTable.cpp
@@ -11,6 +11,7 @@
 #include "CodeGenRegisters.h"
 #include "llvm/ADT/Statistic.h"
 #include "llvm/Support/Debug.h"
+#include "llvm/Support/LEB128.h"
 #include "llvm/Support/ScopedPrinter.h"
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/TableGen/Error.h"
@@ -39,10 +40,46 @@ std::string getEnumNameForPredicate(const TreePredicateFn &Predicate) {
 std::string getMatchOpcodeForImmPredicate(const TreePredicateFn &Predicate) {
   return "GIM_Check" + Predicate.getImmTypeIdentifier().str() + "ImmPredicate";
 }
+
+// GIMT_Encode2/4/8
+constexpr StringLiteral EncodeMacroName = "GIMT_Encode";
+
 } // namespace
 
 //===- Helpers ------------------------------------------------------------===//
 
+void emitEncodingMacrosDef(raw_ostream &OS) {
+  OS << "#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__\n"
+     << "#define " << EncodeMacroName << "2(Val)"
+     << " uint8_t(Val), uint8_t((uint16_t)Val >> 8)\n"
+     << "#define " << EncodeMacroName << "4(Val)"
+     << " uint8_t(Val), uint8_t((uint32_t)Val >> 8), "
+        "uint8_t((uint32_t)Val >> 16), uint8_t((uint32_t)Val >> 24)\n"
+     << "#define " << EncodeMacroName << "8(Val)"
+     << " uint8_t(Val), uint8_t((uint64_t)Val >> 8), "
+        "uint8_t((uint64_t)Val >> 16), uint8_t((uint64_t)Val >> 24),  "
+        "uint8_t((uint64_t)Val >> 32), uint8_t((uint64_t)Val >> 40), "
+        "uint8_t((uint64_t)Val >> 48), uint8_t((uint64_t)Val >> 56)\n"
+     << "#else\n"
+     << "#define " << EncodeMacroName << "2(Val)"
+     << " uint8_t((uint16_t)Val >> 8), uint8_t(Val)\n"
+     << "#define " << EncodeMacroName << "4(Val)"
+     << " uint8_t((uint32_t)Val >> 24), uint8_t((uint32_t)Val >> 16), "
+        "uint8_t((uint32_t)Val >> 8), uint8_t(Val)\n"
+     << "#define " << EncodeMacroName << "8(Val)"
+     << " uint8_t((uint64_t)Val >> 56), uint8_t((uint64_t)Val >> 48), "
+        "uint8_t((uint64_t)Val >> 40), uint8_t((uint64_t)Val >> 32),  "
+        "uint8_t((uint64_t)Val >> 24), uint8_t((uint64_t)Val >> 16), "
+        "uint8_t((uint64_t)Val >> 8), uint8_t(Val)\n"
+     << "#endif\n";
+}
+
+void emitEncodingMacrosUndef(raw_ostream &OS) {
+  OS << "#undef " << EncodeMacroName << "2\n"
+     << "#undef " << EncodeMacroName << "4\n"
+     << "#undef " << EncodeMacroName << "8\n";
+}
+
 std::string getNameForFeatureBitset(const std::vector<Record *> &FeatureBitset,
                                     int HwModeIdx) {
   std::string Name = "GIFBS";
@@ -111,6 +148,12 @@ template std::vector<Matcher *> optimizeRules<SwitchMatcher>(
     ArrayRef<Matcher *> Rules,
     std::vector<std::unique_ptr<Matcher>> &MatcherStorage);
 
+static std::string getEncodedEmitStr(StringRef NamedValue, unsigned NumBytes) {
+  if (NumBytes == 2 || NumBytes == 4 || NumBytes == 8)
+    return (EncodeMacroName + Twine(NumBytes) + "(" + NamedValue + ")").str();
+  llvm_unreachable("Unsupported number of bytes!");
+}
+
 //===- Global Data --------------------------------------------------------===//
 
 std::set<LLTCodeGen> KnownTypes;
@@ -127,7 +170,11 @@ void MatchTableRecord::emit(raw_ostream &OS, bool LineBreakIsNextAfterThis,
   if (Flags & MTRF_Comment)
     OS << (UseLineComment ? "// " : "/*");
 
-  OS << EmitStr;
+  if (NumElements > 1 && !(Flags & (MTRF_PreEncoded | MTRF_Comment)))
+    OS << getEncodedEmitStr(EmitStr, NumElements);
+  else
+    OS << EmitStr;
+
   if (Flags & MTRF_Label)
     OS << ": @" << Table.getLabelIndex(LabelID);
 
@@ -137,7 +184,9 @@ void MatchTableRecord::emit(raw_ostream &OS, bool LineBreakIsNextAfterThis,
   if (Flags & MTRF_JumpTarget) {
     if (Flags & MTRF_Comment)
       OS << " ";
-    OS << Table.getLabelIndex(LabelID);
+    // TODO: Could encode this AOT to speed up build of generated file
+    OS << getEncodedEmitStr(llvm::to_string(Table.getLabelIndex(LabelID)),
+                            NumElements);
   }
 
   if (Flags & MTRF_CommaFollows) {
@@ -172,33 +221,66 @@ MatchTableRecord MatchTable::Opcode(StringRef Opcode, int IndentAdjust) {
                           MatchTableRecord::MTRF_CommaFollows | ExtraFlags);
 }
 
-MatchTableRecord MatchTable::NamedValue(StringRef NamedValue) {
-  return MatchTableRecord(std::nullopt, NamedValue, 1,
+MatchTableRecord MatchTable::NamedValue(unsigned NumBytes,
+                                        StringRef NamedValue) {
+  return MatchTableRecord(std::nullopt, NamedValue, NumBytes,
                           MatchTableRecord::MTRF_CommaFollows);
 }
 
-MatchTableRecord MatchTable::NamedValue(StringRef NamedValue,
+MatchTableRecord MatchTable::NamedValue(unsigned NumBytes, StringRef NamedValue,
                                         int64_t RawValue) {
-  return MatchTableRecord(std::nullopt, NamedValue, 1,
+  return MatchTableRecord(std::nullopt, NamedValue, NumBytes,
                           MatchTableRecord::MTRF_CommaFollows, RawValue);
 }
 
-MatchTableRecord MatchTable::NamedValue(StringRef Namespace,
+MatchTableRecord MatchTable::NamedValue(unsigned NumBytes, StringRef Namespace,
                                         StringRef NamedValue) {
   return MatchTableRecord(std::nullopt, (Namespace + "::" + NamedValue).str(),
-                          1, MatchTableRecord::MTRF_CommaFollows);
+                          NumBytes, MatchTableRecord::MTRF_CommaFollows);
 }
 
-MatchTableRecord MatchTable::NamedValue(StringRef Namespace,
+MatchTableRecord MatchTable::NamedValue(unsigned NumBytes, StringRef Namespace,
                                         StringRef NamedValue,
                                         int64_t RawValue) {
   return MatchTableRecord(std::nullopt, (Namespace + "::" + NamedValue).str(),
-                          1, MatchTableRecord::MTRF_CommaFollows, RawValue);
+                          NumBytes, MatchTableRecord::MTRF_CommaFollows,
+                          RawValue);
+}
+
+MatchTableRecord MatchTable::IntValue(unsigned NumBytes, int64_t IntValue) {
+  assert(isUIntN(NumBytes * 8, IntValue) || isIntN(NumBytes * 8, IntValue));
+  auto Str = llvm::to_string(IntValue);
+  if (NumBytes == 1 && IntValue < 0)
+    Str = "uint8_t(" + Str + ")";
+  // TODO: Could optimize this directly to save the compiler some work when
+  // building the file
+  return MatchTableRecord(std::nullopt, Str, NumBytes,
+                          MatchTableRecord::MTRF_CommaFollows);
 }
 
-MatchTableRecord MatchTable::IntValue(int64_t IntValue) {
-  return MatchTableRecord(std::nullopt, llvm::to_string(IntValue), 1,
-                          MatchTableRecord::MTRF_CommaFollows);
+MatchTableRecord MatchTable::ULEB128Value(uint64_t IntValue) {
+  uint8_t Buffer[10];
+  unsigned Len = encodeULEB128(IntValue, Buffer);
+
+  // Simple case (most common)
+  if (Len == 1) {
+    return MatchTableRecord(std::nullopt, llvm::to_string((unsigned)Buffer[0]),
+                            1, MatchTableRecord::MTRF_CommaFollows);
+  }
+
+  // Print it as, e.g. /* -123456 (*/, 0xC0, 0xBB, 0x78 /*)*/
+  std::string Str;
+  raw_string_ostream OS(Str);
+  OS << "/* " << llvm::to_string(IntValue) << "(*/";
+  for (unsigned K = 0; K < Len; ++K) {
+    if (K)
+      OS << ", ";
+    OS << "0x" << llvm::toHex({Buffer[K]});
+  }
+  OS << "/*)*/";
+  return MatchTableRecord(std::nullopt, Str, Len,
+                          MatchTableRecord::MTRF_CommaFollows |
+                              MatchTableRecord::MTRF_PreEncoded);
 }
 
 MatchTableRecord MatchTable::Label(unsigned LabelID) {
@@ -209,7 +291,7 @@ MatchTableRecord MatchTable::Label(unsigned LabelID) {
 }
 
 MatchTableRecord MatchTable::JumpTarget(unsigned LabelID) {
-  return MatchTableRecord(LabelID, "Label " + llvm::to_string(LabelID), 1,
+  return MatchTableRecord(LabelID, "Label " + llvm::to_string(LabelID), 4,
                           MatchTableRecord::MTRF_JumpTarget |
                               MatchTableRecord::MTRF_Comment |
                               MatchTableRecord::MTRF_CommaFollows);
@@ -219,7 +301,7 @@ void MatchTable::emitUse(raw_ostream &OS) const { OS << "MatchTable" << ID; }
 
 void MatchTable::emitDeclaration(raw_ostream &OS) const {
   unsigned Indentation = 4;
-  OS << "  constexpr static int64_t MatchTable" << ID << "[] = {";
+  OS << "  constexpr static uint8_t MatchTable" << ID << "[] = {";
   LineBreak.emit(OS, true, *this);
   OS << std::string(Indentation, ' ');
 
@@ -243,7 +325,7 @@ void MatchTable::emitDeclaration(raw_ostream &OS) const {
     if (I->Flags & MatchTableRecord::MTRF_Outdent)
       Indentation -= 2;
   }
-  OS << "}; // Size: " << (CurrentSize * 8) << " bytes\n";
+  OS << "}; // Size: " << CurrentSize << " bytes\n";
 }
 
 MatchTable MatchTable::buildTable(ArrayRef<Matcher *> Rules, bool WithCoverage,
@@ -542,14 +624,14 @@ void SwitchMatcher::emitPredicateSpecificOpcodes(const PredicateMatcher &P,
 
   if (const auto *Condition = dyn_cast<InstructionOpcodeMatcher>(&P)) {
     Table << MatchTable::Opcode("GIM_SwitchOpcode") << MatchTable::Comment("MI")
-          << MatchTable::IntValue(Condition->getInsnVarID());
+          << MatchTable::ULEB128Value(Condition->getInsnVarID());
     return;
   }
   if (const auto *Condition = dyn_cast<LLTOperandMatcher>(&P)) {
     Table << MatchTable::Opcode("GIM_SwitchType") << MatchTable::Comment("MI")
-          << MatchTable::IntValue(Condition->getInsnVarID())
+          << MatchTable::ULEB128Value(Condition->getInsnVarID())
           << MatchTable::Comment("Op")
-          << MatchTable::IntValue(Condition->getOpIdx());
+          << MatchTable::ULEB128Value(Condition->getOpIdx());
     return;
   }
 
@@ -574,8 +656,8 @@ void SwitchMatcher::emit(MatchTable &Table) {
 
   emitPredicateSpecificOpcodes(*Condition, Table);
 
-  Table << MatchTable::Comment("[") << MatchTable::IntValue(LowerBound)
-        << MatchTable::IntValue(UpperBound) << MatchTable::Comment(")")
+  Table << MatchTable::Comment("[") << MatchTable::IntValue(2, LowerBound)
+        << MatchTable::IntValue(2, UpperBound) << MatchTable::Comment(")")
         << MatchTable::Comment("default:") << MatchTable::JumpTarget(Default);
 
   int64_t J = LowerBound;
@@ -583,7 +665,7 @@ void SwitchMatcher::emit(MatchTable &Table) {
   for (unsigned I = 0, E = Values.size(); I < E; ++I) {
     auto V = *VI++;
     while (J++ < V.getRawValue())
-      Table << MatchTable::IntValue(0);
+      Table << MatchTable::IntValue(4, 0);
     V.turnIntoComment();
     Table << MatchTable::LineBreak << V << MatchTable::JumpTarget(LabelIDs[I]);
   }
@@ -865,14 +947,14 @@ void RuleMatcher::emit(MatchTable &Table) {
   if (!RequiredFeatures.empty() || HwModeIdx >= 0) {
     Table << MatchTable::Opcode("GIM_CheckFeatures")
           << MatchTable::NamedValue(
-                 getNameForFeatureBitset(RequiredFeatures, HwModeIdx))
+                 2, getNameForFeatureBitset(RequiredFeatures, HwModeIdx))
           << MatchTable::LineBreak;
   }
 
   if (!RequiredSimplePredicates.empty()) {
     for (const auto &Pred : RequiredSimplePredicates) {
       Table << MatchTable::Opcode("GIM_CheckSimplePredicate")
-            << MatchTable::NamedValue(Pred) << MatchTable::LineBreak;
+            << MatchTable::NamedValue(2, Pred) << MatchTable::LineBreak;
     }
   }
 
@@ -899,7 +981,7 @@ void RuleMatcher::emit(MatchTable &Table) {
     for (const auto &InsnID : InsnIDs) {
       // Reject the difficult cases until we have a more accurate check.
       Table << MatchTable::Opcode("GIM_CheckIsSafeToFold")
-            << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID)
+            << MatchTable::Comment("InsnID") << MatchTable::ULEB128Value(InsnID)
             << MatchTable::LineBreak;
 
       // FIXME: Emit checks to determine it's _actually_ safe to fold and/or
@@ -948,8 +1030,8 @@ void RuleMatcher::emit(MatchTable &Table) {
   assert((Table.isWithCoverage() ? !Table.isCombiner() : true) &&
          "Combiner tables don't support coverage!");
   if (Table.isWithCoverage())
-    Table << MatchTable::Opcode("GIR_Coverage") << MatchTable::IntValue(RuleID)
-          << MatchTable::LineBreak;
+    Table << MatchTable::Opcode("GIR_Coverage")
+          << MatchTable::IntValue(4, RuleID) << MatchTable::LineBreak;
   else if (!Table.isCombiner())
     Table << MatchTable::Comment(("GIR_Coverage, " + Twine(RuleID) + ",").str())
           << MatchTable::LineBreak;
@@ -1035,12 +1117,13 @@ void SameOperandMatcher::emitPredicateOpcodes(MatchTable &Table,
   Table << MatchTable::Opcode(IgnoreCopies
                                   ? "GIM_CheckIsSameOperandIgnoreCopies"
                                   : "GIM_CheckIsSameOperand")
-        << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
-        << MatchTable::Comment("OpIdx") << MatchTable::IntValue(OpIdx)
+        << MatchTable::Comment("MI") << MatchTable::ULEB128Value(InsnVarID)
+        << MatchTable::Comment("OpIdx") << MatchTable::ULEB128Value(OpIdx)
         << MatchTable::Comment("OtherMI")
-        << MatchTable::IntValue(OtherInsnVarID)
+        << MatchTable::ULEB128Value(OtherInsnVarID)
         << MatchTable::Comment("OtherOpIdx")
-        << MatchTable::IntValue(OtherOM.getOpIdx()) << MatchTable::LineBreak;
+        << MatchTable::ULEB128Value(OtherOM.getOpIdx())
+        << MatchTable::LineBreak;
 }
 
 //===- LLTOperandMatcher --------------------------------------------------===//
@@ -1050,8 +1133,8 @@ std::map<LLTCodeGen, unsigned> LLTOperandMatcher::TypeIDValues;
 MatchTableRecord LLTOperandMatcher::getValue() const {
   const auto VI = TypeIDValues.find(Ty);
   if (VI == TypeIDValues.end())
-    return MatchTable::NamedValue(getTy().getCxxEnumValue());
-  return MatchTable::NamedValue(getTy().getCxxEnumValue(), VI->second);
+    return MatchTable::NamedValue(1, getTy().getCxxEnumValue());
+  return MatchTable::NamedValue(1, getTy().getCxxEnumValue(), VI->second);
 }
 
 bool LLTOperandMatcher::hasValue() const {
@@ -1063,8 +1146,8 @@ bool LLTOperandMatcher::hasValue() const {
 void LLTOperandMatcher::emitPredicateOpcodes(MatchTable &Table,
                                              RuleMatcher &Rule) const {
   Table << MatchTable::Opcode("GIM_CheckType") << MatchTable::Comment("MI")
-        << MatchTable::IntValue(InsnVarID) << MatchTable::Comment("Op")
-        << MatchTable::IntValue(OpIdx) << MatchTable::Comment("Type")
+        << MatchTable::ULEB128Value(InsnVarID) << MatchTable::Comment("Op")
+        << MatchTable::ULEB128Value(OpIdx) << MatchTable::Comment("Type")
         << getValue() << MatchTable::LineBreak;
 }
 
@@ -1073,10 +1156,10 @@ void LLTOperandMatcher::emitPredicateOpcodes(MatchTable &Table,
 void PointerToAnyOperandMatcher::emitPredicateOpcodes(MatchTable &Table,
                                                       RuleMatcher &Rule) const {
   Table << MatchTable::Opcode("GIM_CheckPointerToAny")
-        << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
-        << MatchTable::Comment("Op") << MatchTable::IntValue(OpIdx)
-        << MatchTable::Comment("SizeInBits") << MatchTable::IntValue(SizeInBits)
-        << MatchTable::LineBreak;
+        << MatchTable::Comment("MI") << MatchTable::ULEB128Value(InsnVarID)
+        << MatchTable::Comment("Op") << MatchTable::ULEB128Value(OpIdx)
+        << MatchTable::Comment("SizeInBits")
+        << MatchTable::ULEB128Value(SizeInBits) << MatchTable::LineBreak;
 }
 
 //===- RecordNamedOperandMatcher ------------------------------------------===//
@@ -1084,9 +1167,9 @@ void PointerToAnyOperandMatcher::emitPredicateOpcodes(MatchTable &Table,
 void RecordNamedOperandMatcher::emitPredicateOpcodes(MatchTable &Table,
                                                      RuleMatcher &Rule) const {
   Table << MatchTable::Opcode("GIM_RecordNamedOperand")
-        << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
-        << MatchTable::Comment("Op") << MatchTable::IntValue(OpIdx)
-        << MatchTable::Comment("StoreIdx") << MatchTable::IntValue(StoreIdx)
+        << MatchTable::Comment("MI") << MatchTable::ULEB128Value(InsnVarID)
+        << MatchTable::Comment("Op") << MatchTable::ULEB128Value(OpIdx)
+        << MatchTable::Comment("StoreIdx") << MatchTable::ULEB128Value(StoreIdx)
         << MatchTable::Comment("Name : " + Name) << MatchTable::LineBreak;
 }
 
@@ -1096,9 +1179,9 @@ void RecordRegisterType::emitPredicateOpcodes(MatchTable &Table,
                                               RuleMatcher &Rule) const {
   assert(Idx < 0 && "Temp types always have negative indexes!");
   Table << MatchTable::Opcode("GIM_RecordRegType") << MatchTable::Comment("MI")
-        << MatchTable::IntValue(InsnVarID) << MatchTable::Comment("Op")
-        << MatchTable::IntValue(OpIdx) << MatchTable::Comment("TempTypeIdx")
-        << MatchTable::IntValue(Idx) << MatchTable::LineBreak;
+        << MatchTable::ULEB128Value(InsnVarID) << MatchTable::Comment("Op")
+        << MatchTable::ULEB128Value(OpIdx) << MatchTable::Comment("TempTypeIdx")
+        << MatchTable::IntValue(1, Idx) << MatchTable::LineBreak;
 }
 
 //===- ComplexPatternOperandMatcher ---------------------------------------===//
@@ -1107,10 +1190,10 @@ void ComplexPatternOperandMatcher::emitPredicateOpcodes(
     MatchTable &Table, RuleMatcher &Rule) const {
   unsigned ID = getAllocatedTemporariesBaseID();
   Table << MatchTable::Opcode("GIM_CheckComplexPattern")
-        << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
-        << MatchTable::Comment("Op") << MatchTable::IntValue(OpIdx)
-        << MatchTable::Comment("Renderer") << MatchTable::IntValue(ID)
-        << MatchTable::NamedValue(("GICP_" + TheDef.getName()).str())
+        << MatchTable::Comment("MI") << MatchTable::ULEB128Value(InsnVarID)
+        << MatchTable::Comment("Op") << MatchTable::ULEB128Value(OpIdx)
+        << MatchTable::Comment("Renderer") << MatchTable::IntValue(2, ID)
+        << MatchTable::NamedValue(2, ("GICP_" + TheDef.getName()).str())
         << MatchTable::LineBreak;
 }
 
@@ -1128,10 +1211,10 @@ bool RegisterBankOperandMatcher::isIdentical(const PredicateMatcher &B) const {
 void RegisterBankOperandMatcher::emitPredicateOpcodes(MatchTable &Table,
                                                       RuleMatcher &Rule) const {
   Table << MatchTable::Opcode("GIM_CheckRegBankForClass")
-        << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
-        << MatchTable::Comment("Op") << MatchTable::IntValue(OpIdx)
+        << MatchTable::Comment("MI") << MatchTable::ULEB128Value(InsnVarID)
+        << MatchTable::Comment("Op") << MatchTable::ULEB128Value(OpIdx)
         << MatchTable::Comment("RC")
-        << MatchTable::NamedValue(RC.getQualifiedIdName())
+        << MatchTable::NamedValue(2, RC.getQualifiedIdName())
         << MatchTable::LineBreak;
 }
 
@@ -1140,8 +1223,8 @@ void RegisterBankOperandMatcher::emitPredicateOpcodes(MatchTable &Table,
 void MBBOperandMatcher::emitPredicateOpcodes(MatchTable &Table,
                                              RuleMatcher &Rule) const {
   Table << MatchTable::Opcode("GIM_CheckIsMBB") << MatchTable::Comment("MI")
-        << MatchTable::IntValue(InsnVarID) << MatchTable::Comment("Op")
-        << MatchTable::IntValue(OpIdx) << MatchTable::LineBreak;
+        << MatchTable::ULEB128Value(InsnVarID) << MatchTable::Comment("Op")
+        << MatchTable::ULEB128Value(OpIdx) << MatchTable::LineBreak;
 }
 
 //===- ImmOperandMatcher --------------------------------------------------===//
@@ -1149,18 +1232,20 @@ void MBBOperandMatcher::emitPredicateOpcodes(MatchTable &Table,
 void ImmOperandMatcher::emitPredicateOpcodes(MatchTable &Table,
                                              RuleMatcher &Rule) const {
   Table << MatchTable::Opcode("GIM_CheckIsImm") << MatchTable::Comment("MI")
-        << MatchTable::IntValue(InsnVarID) << MatchTable::Comment("Op")
-        << MatchTable::IntValue(OpIdx) << MatchTable::LineBreak;
+        << MatchTable::ULEB128Value(InsnVarID) << MatchTable::Comment("Op")
+        << MatchTable::ULEB128Value(OpIdx) << MatchTable::LineBreak;
 }
 
 //===- ConstantIntOperandMatcher ------------------------------------------===//
 
 void ConstantIntOperandMatcher::emitPredicateOpcodes(MatchTable &Table,
                                                      RuleMatcher &Rule) const {
-  Table << MatchTable::Opcode("GIM_CheckConstantInt")
-        << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
-        << MatchTable::Comment("Op") << MatchTable::IntValue(OpIdx)
-        << MatchTable::IntValue(Value) << MatchTable::LineBreak;
+  const bool IsInt8 = isInt<8>(Value);
+  Table << MatchTable::Opcode(IsInt8 ? "GIM_CheckConstantInt8"
+                                     : "GIM_CheckConstantInt")
+        << MatchTable::Comment("MI") << MatchTable::ULEB128Value(InsnVarID)
+        << MatchTable::Comment("Op") << MatchTable::ULEB128Value(OpIdx)
+        << MatchTable::IntValue(IsInt8 ? 1 : 8, Value) << MatchTable::LineBreak;
 }
 
 //===- LiteralIntOperandMatcher -------------------------------------------===//
@@ -1168,9 +1253,9 @@ void ConstantIntOperandMatcher::emitPredicateOpcodes(MatchTable &Table,
 void LiteralIntOperandMatcher::emitPredicateOpcodes(MatchTable &Table,
                                                     RuleMatcher &Rule) const {
   Table << MatchTable::Opcode("GIM_CheckLiteralInt")
-        << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
-        << MatchTable::Comment("Op") << MatchTable::IntValue(OpIdx)
-        << MatchTable::IntValue(Value) << MatchTable::LineBreak;
+        << MatchTable::Comment("MI") << MatchTable::ULEB128Value(InsnVarID)
+        << MatchTable::Comment("Op") << MatchTable::ULEB128Value(OpIdx)
+        << MatchTable::IntValue(8, Value) << MatchTable::LineBreak;
 }
 
 //===- CmpPredicateOperandMatcher -----------------------------------------===//
@@ -1178,10 +1263,11 @@ void LiteralIntOperandMatcher::emitPredicateOpcodes(MatchTable &Table,
 void CmpPredicateOperandMatcher::emitPredicateOpcodes(MatchTable &Table,
                                                       RuleMatcher &Rule) const {
   Table << MatchTable::Opcode("GIM_CheckCmpPredicate")
-        << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
-        << MatchTable::Comment("Op") << MatchTable::IntValue(OpIdx)
+        << MatchTable::Comment("MI") << MatchTable::ULEB128Value(InsnVarID)
+        << MatchTable::Comment("Op") << MatchTable::ULEB128Value(OpIdx)
         << MatchTable::Comment("Predicate")
-        << MatchTable::NamedValue("CmpInst", PredName) << MatchTable::LineBreak;
+        << MatchTable::NamedValue(2, "CmpInst", PredName)
+        << MatchTable::LineBreak;
 }
 
 //===- IntrinsicIDOperandMatcher ------------------------------------------===//
@@ -1189,9 +1275,9 @@ void CmpPredicateOperandMatcher::emitPredicateOpcodes(MatchTable &Table,
 void IntrinsicIDOperandMatcher::emitPredicateOpcodes(MatchTable &Table,
                                                      RuleMatcher &Rule) const {
   Table << MatchTable::Opcode("GIM_CheckIntrinsicID")
-        << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
-        << MatchTable::Comment("Op") << MatchTable::IntValue(OpIdx)
-        << MatchTable::NamedValue("Intrinsic::" + II->EnumName)
+        << MatchTable::Comment("MI") << MatchTable::ULEB128Value(InsnVarID)
+        << MatchTable::Comment("Op") << MatchTable::ULEB128Value(OpIdx)
+        << MatchTable::NamedValue(2, "Intrinsic::" + II->EnumName)
         << MatchTable::LineBreak;
 }
 
@@ -1200,10 +1286,10 @@ void IntrinsicIDOperandMatcher::emitPredicateOpcodes(MatchTable &Table,
 void OperandImmPredicateMatcher::emitPredicateOpcodes(MatchTable &Table,
                                                       RuleMatcher &Rule) const {
   Table << MatchTable::Opcode("GIM_CheckImmOperandPredicate")
-        << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
-        << MatchTable::Comment("MO") << MatchTable::IntValue(OpIdx)
+        << MatchTable::Comment("MI") << MatchTable::ULEB128Value(InsnVarID)
+        << MatchTable::Comment("MO") << MatchTable::ULEB128Value(OpIdx)
         << MatchTable::Comment("Predicate")
-        << MatchTable::NamedValue(getEnumNameForPredicate(Predicate))
+        << MatchTable::NamedValue(2, getEnumNameForPredicate(Predicate))
         << MatchTable::LineBreak;
 }
 
@@ -1304,9 +1390,9 @@ MatchTableRecord
 InstructionOpcodeMatcher::getInstValue(const CodeGenInstruction *I) const {
   const auto VI = OpcodeValues.find(I);
   if (VI != OpcodeValues.end())
-    return MatchTable::NamedValue(I->Namespace, I->TheDef->getName(),
+    return MatchTable::NamedValue(2, I->Namespace, I->TheDef->getName(),
                                   VI->second);
-  return MatchTable::NamedValue(I->Namespace, I->TheDef->getName());
+  return MatchTable::NamedValue(2, I->Namespace, I->TheDef->getName());
 }
 
 void InstructionOpcodeMatcher::initOpcodeValuesMap(
@@ -1324,9 +1410,9 @@ MatchTableRecord InstructionOpcodeMatcher::getValue() const {
   const CodeGenInstruction *I = Insts[0];
   const auto VI = OpcodeValues.find(I);
   if (VI != OpcodeValues.end())
-    return MatchTable::NamedValue(I->Namespace, I->TheDef->getName(),
+    return MatchTable::NamedValue(2, I->Namespace, I->TheDef->getName(),
                                   VI->second);
-  return MatchTable::NamedValue(I->Namespace, I->TheDef->getName());
+  return MatchTable::NamedValue(2, I->Namespace, I->TheDef->getName());
 }
 
 void InstructionOpcodeMatcher::emitPredicateOpcodes(MatchTable &Table,
@@ -1334,7 +1420,7 @@ void InstructionOpcodeMatcher::emitPredicateOpcodes(MatchTable &Table,
   StringRef CheckType =
       Insts.size() == 1 ? "GIM_CheckOpcode" : "GIM_CheckOpcodeIsEither";
   Table << MatchTable::Opcode(CheckType) << MatchTable::Comment("MI")
-        << MatchTable::IntValue(InsnVarID);
+        << MatchTable::ULEB128Value(InsnVarID);
 
   for (const CodeGenInstruction *I : Insts)
     Table << getInstValue(I);
@@ -1381,9 +1467,9 @@ StringRef InstructionOpcodeMatcher::getOperandType(unsigned OpIdx) const {
 void InstructionNumOperandsMatcher::emitPredicateOpcodes(
     MatchTable &Table, RuleMatcher &Rule) const {
   Table << MatchTable::Opcode("GIM_CheckNumOperands")
-        << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
-        << MatchTable::Comment("Expected") << MatchTable::IntValue(NumOperands)
-        << MatchTable::LineBreak;
+        << MatchTable::Comment("MI") << MatchTable::ULEB128Value(InsnVarID)
+        << MatchTable::Comment("Expected")
+        << MatchTable::ULEB128Value(NumOperands) << MatchTable::LineBreak;
 }
 
 //===- InstructionImmPredicateMatcher -------------------------------------===//
@@ -1399,9 +1485,9 @@ bool InstructionImmPredicateMatcher::isIdentical(
 void InstructionImmPredicateMatcher::emitPredicateOpcodes(
     MatchTable &Table, RuleMatcher &Rule) const {
   Table << MatchTable::Opcode(getMatchOpcodeForImmPredicate(Predicate))
-        << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
+        << MatchTable::Comment("MI") << MatchTable::ULEB128Value(InsnVarID)
         << MatchTable::Comment("Predicate")
-        << MatchTable::NamedValue(getEnumNameForPredicate(Predicate))
+        << MatchTable::NamedValue(2, getEnumNameForPredicate(Predicate))
         << MatchTable::LineBreak;
 }
 
@@ -1425,8 +1511,9 @@ void AtomicOrderingMMOPredicateMatcher::emitPredicateOpcodes(
     Opcode = "GIM_CheckAtomicOrderingWeakerThan";
 
   Table << MatchTable::Opcode(Opcode) << MatchTable::Comment("MI")
-        << MatchTable::IntValue(InsnVarID) << MatchTable::Comment("Order")
-        << MatchTable::NamedValue(("(int64_t)AtomicOrdering::" + Order).str())
+        << MatchTable::ULEB128Value(InsnVarID) << MatchTable::Comment("Order")
+        << MatchTable::NamedValue(1,
+                                  ("(uint8_t)AtomicOrdering::" + Order).str())
         << MatchTable::LineBreak;
 }
 
@@ -1435,9 +1522,9 @@ void AtomicOrderingMMOPredicateMatcher::emitPredicateOpcodes(
 void MemorySizePredicateMatcher::emitPredicateOpcodes(MatchTable &Table,
                                                       RuleMatcher &Rule) const {
   Table << MatchTable::Opcode("GIM_CheckMemorySizeEqualTo")
-        << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
-        << MatchTable::Comment("MMO") << MatchTable::IntValue(MMOIdx)
-        << MatchTable::Comment("Size") << MatchTable::IntValue(Size)
+        << MatchTable::Comment("MI") << MatchTable::ULEB128Value(InsnVarID)
+        << MatchTable::Comment("MMO") << MatchTable::ULEB128Value(MMOIdx)
+        << MatchTable::Comment("Size") << MatchTable::IntValue(4, Size)
         << MatchTable::LineBreak;
 }
 
@@ -1454,14 +1541,14 @@ bool MemoryAddressSpacePredicateMatcher::isIdentical(
 void MemoryAddressSpacePredicateMatcher::emitPredicateOpcodes(
     MatchTable &Table, RuleMatcher &Rule) const {
   Table << MatchTable::Opcode("GIM_CheckMemoryAddressSpace")
-        << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
+        << MatchTable::Comment("MI") << MatchTable::ULEB128Value(InsnVarID)
         << MatchTable::Comment("MMO")
-        << MatchTable::IntValue(MMOIdx)
+        << MatchTable::ULEB128Value(MMOIdx)
         // Encode number of address spaces to expect.
         << MatchTable::Comment("NumAddrSpace")
-        << MatchTable::IntValue(AddrSpaces.size());
+        << MatchTable::ULEB128Value(AddrSpaces.size());
   for (unsigned AS : AddrSpaces)
-    Table << MatchTable::Comment("AddrSpace") << MatchTable::IntValue(AS);
+    Table << MatchTable::Comment("AddrSpace") << MatchTable::ULEB128Value(AS);
 
   Table << MatchTable::LineBreak;
 }
@@ -1479,9 +1566,9 @@ bool MemoryAlignmentPredicateMatcher::isIdentical(
 void MemoryAlignmentPredicateMatcher::emitPredicateOpcodes(
     MatchTable &Table, RuleMatcher &Rule) const {
   Table << MatchTable::Opcode("GIM_CheckMemoryAlignment")
-        << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
-        << MatchTable::Comment("MMO") << MatchTable::IntValue(MMOIdx)
-        << MatchTable::Comment("MinAlign") << MatchTable::IntValue(MinAlign)
+        << MatchTable::Comment("MI") << MatchTable::ULEB128Value(InsnVarID)
+        << MatchTable::Comment("MMO") << MatchTable::ULEB128Value(MMOIdx)
+        << MatchTable::Comment("MinAlign") << MatchTable::ULEB128Value(MinAlign)
         << MatchTable::LineBreak;
 }
 
@@ -1501,9 +1588,9 @@ void MemoryVsLLTSizePredicateMatcher::emitPredicateOpcodes(
                Relation == EqualTo       ? "GIM_CheckMemorySizeEqualToLLT"
                : Relation == GreaterThan ? "GIM_CheckMemorySizeGreaterThanLLT"
                                          : "GIM_CheckMemorySizeLessThanLLT")
-        << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
-        << MatchTable::Comment("MMO") << MatchTable::IntValue(MMOIdx)
-        << MatchTable::Comment("OpIdx") << MatchTable::IntValue(OpIdx)
+        << MatchTable::Comment("MI") << MatchTable::ULEB128Value(InsnVarID)
+        << MatchTable::Comment("MMO") << MatchTable::ULEB128Value(MMOIdx)
+        << MatchTable::Comment("OpIdx") << MatchTable::ULEB128Value(OpIdx)
         << MatchTable::LineBreak;
 }
 
@@ -1516,7 +1603,7 @@ void VectorSplatImmPredicateMatcher::emitPredicateOpcodes(
   else
     Table << MatchTable::Opcode("GIM_CheckIsBuildVectorAllZeros");
 
-  Table << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID);
+  Table << MatchTable::Comment("MI") << MatchTable::ULEB128Value(InsnVarID);
   Table << MatchTable::LineBreak;
 }
 
@@ -1536,8 +1623,8 @@ bool GenericInstructionPredicateMatcher::isIdentical(
 void GenericInstructionPredicateMatcher::emitPredicateOpcodes(
     MatchTable &Table, RuleMatcher &Rule) const {
   Table << MatchTable::Opcode("GIM_CheckCxxInsnPredicate")
-        << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
-        << MatchTable::Comment("FnId") << MatchTable::NamedValue(EnumVal)
+        << MatchTable::Comment("MI") << MatchTable::ULEB128Value(InsnVarID)
+        << MatchTable::Comment("FnId") << MatchTable::NamedValue(2, EnumVal)
         << MatchTable::LineBreak;
 }
 
@@ -1555,8 +1642,9 @@ bool MIFlagsInstructionPredicateMatcher::isIdentical(
 void MIFlagsInstructionPredicateMatcher::emitPredicateOpcodes(
     MatchTable &Table, RuleMatcher &Rule) const {
   Table << MatchTable::Opcode(CheckNot ? "GIM_MIFlagsNot" : "GIM_MIFlags")
-        << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
-        << MatchTable::NamedValue(join(Flags, " | ")) << MatchTable::LineBreak;
+        << MatchTable::Comment("MI") << MatchTable::ULEB128Value(InsnVarID)
+        << MatchTable::NamedValue(4, join(Flags, " | "))
+        << MatchTable::LineBreak;
 }
 
 //===- InstructionMatcher -------------------------------------------------===//
@@ -1700,9 +1788,10 @@ void InstructionOperandMatcher::emitCaptureOpcodes(MatchTable &Table,
   const bool IgnoreCopies = Flags & GISF_IgnoreCopies;
   Table << MatchTable::Opcode(IgnoreCopies ? "GIM_RecordInsnIgnoreCopies"
                                            : "GIM_RecordInsn")
-        << MatchTable::Comment("DefineMI") << MatchTable::IntValue(NewInsnVarID)
-        << MatchTable::Comment("MI") << MatchTable::IntValue(getInsnVarID())
-        << MatchTable::Comment("OpIdx") << MatchTable::IntValue(getOpIdx())
+        << MatchTable::Comment("DefineMI")
+        << MatchTable::ULEB128Value(NewInsnVarID) << MatchTable::Comment("MI")
+        << MatchTable::ULEB128Value(getInsnVarID())
+        << MatchTable::Comment("OpIdx") << MatchTable::ULEB128Value(getOpIdx())
         << MatchTable::Comment("MIs[" + llvm::to_string(NewInsnVarID) + "]")
         << MatchTable::LineBreak;
 }
@@ -1732,9 +1821,11 @@ void CopyRenderer::emitRenderOpcodes(MatchTable &Table,
   const OperandMatcher &Operand = Rule.getOperandMatcher(SymbolicName);
   unsigned OldInsnVarID = Rule.getInsnVarID(Operand.getInstructionMatcher());
   Table << MatchTable::Opcode("GIR_Copy") << MatchTable::Comment("NewInsnID")
-        << MatchTable::IntValue(NewInsnID) << MatchTable::Comment("OldInsnID")
-        << MatchTable::IntValue(OldInsnVarID) << MatchTable::Comment("OpIdx")
-        << MatchTable::IntValue(Operand.getOpIdx())
+        << MatchTable::ULEB128Value(NewInsnID)
+        << MatchTable::Comment("OldInsnID")
+        << MatchTable::ULEB128Value(OldInsnVarID)
+        << MatchTable::Comment("OpIdx")
+        << MatchTable::ULEB128Value(Operand.getOpIdx())
         << MatchTable::Comment(SymbolicName) << MatchTable::LineBreak;
 }
 
@@ -1745,9 +1836,11 @@ void CopyPhysRegRenderer::emitRenderOpcodes(MatchTable &Table,
   const OperandMatcher &Operand = Rule.getPhysRegOperandMatcher(PhysReg);
   unsigned OldInsnVarID = Rule.getInsnVarID(Operand.getInstructionMatcher());
   Table << MatchTable::Opcode("GIR_Copy") << MatchTable::Comment("NewInsnID")
-        << MatchTable::IntValue(NewInsnID) << MatchTable::Comment("OldInsnID")
-        << MatchTable::IntValue(OldInsnVarID) << MatchTable::Comment("OpIdx")
-        << MatchTable::IntValue(Operand.getOpIdx())
+        << MatchTable::ULEB128Value(NewInsnID)
+        << MatchTable::Comment("OldInsnID")
+        << MatchTable::ULEB128Value(OldInsnVarID)
+        << MatchTable::Comment("OpIdx")
+        << MatchTable::ULEB128Value(Operand.getOpIdx())
         << MatchTable::Comment(PhysReg->getName()) << MatchTable::LineBreak;
 }
 
@@ -1758,11 +1851,14 @@ void CopyOrAddZeroRegRenderer::emitRenderOpcodes(MatchTable &Table,
   const OperandMatcher &Operand = Rule.getOperandMatcher(SymbolicName);
   unsigned OldInsnVarID = Rule.getInsnVarID(Operand.getInstructionMatcher());
   Table << MatchTable::Opcode("GIR_CopyOrAddZeroReg")
-        << MatchTable::Comment("NewInsnID") << MatchTable::IntValue(NewInsnID)
+        << MatchTable::Comment("NewInsnID")
+        << MatchTable::ULEB128Value(NewInsnID)
         << MatchTable::Comment("OldInsnID")
-        << MatchTable::IntValue(OldInsnVarID) << MatchTable::Comment("OpIdx")
-        << MatchTable::IntValue(Operand.getOpIdx())
+        << MatchTable::ULEB128Value(OldInsnVarID)
+        << MatchTable::Comment("OpIdx")
+        << MatchTable::ULEB128Value(Operand.getOpIdx())
         << MatchTable::NamedValue(
+               2,
                (ZeroRegisterDef->getValue("Namespace")
                     ? ZeroRegisterDef->getValueAsString("Namespace")
                     : ""),
@@ -1778,9 +1874,10 @@ void CopyConstantAsImmRenderer::emitRenderOpcodes(MatchTable &Table,
   unsigned OldInsnVarID = Rule.getInsnVarID(InsnMatcher);
   Table << MatchTable::Opcode(Signed ? "GIR_CopyConstantAsSImm"
                                      : "GIR_CopyConstantAsUImm")
-        << MatchTable::Comment("NewInsnID") << MatchTable::IntValue(NewInsnID)
+        << MatchTable::Comment("NewInsnID")
+        << MatchTable::ULEB128Value(NewInsnID)
         << MatchTable::Comment("OldInsnID")
-        << MatchTable::IntValue(OldInsnVarID)
+        << MatchTable::ULEB128Value(OldInsnVarID)
         << MatchTable::Comment(SymbolicName) << MatchTable::LineBreak;
 }
 
@@ -1791,9 +1888,10 @@ void CopyFConstantAsFPImmRenderer::emitRenderOpcodes(MatchTable &Table,
   InstructionMatcher &InsnMatcher = Rule.getInstructionMatcher(SymbolicName);
   unsigned OldInsnVarID = Rule.getInsnVarID(InsnMatcher);
   Table << MatchTable::Opcode("GIR_CopyFConstantAsFPImm")
-        << MatchTable::Comment("NewInsnID") << MatchTable::IntValue(NewInsnID)
+        << MatchTable::Comment("NewInsnID")
+        << MatchTable::ULEB128Value(NewInsnID)
         << MatchTable::Comment("OldInsnID")
-        << MatchTable::IntValue(OldInsnVarID)
+        << MatchTable::ULEB128Value(OldInsnVarID)
         << MatchTable::Comment(SymbolicName) << MatchTable::LineBreak;
 }
 
@@ -1804,12 +1902,14 @@ void CopySubRegRenderer::emitRenderOpcodes(MatchTable &Table,
   const OperandMatcher &Operand = Rule.getOperandMatcher(SymbolicName);
   unsigned OldInsnVarID = Rule.getInsnVarID(Operand.getInstructionMatcher());
   Table << MatchTable::Opcode("GIR_CopySubReg")
-        << MatchTable::Comment("NewInsnID") << MatchTable::IntValue(NewInsnID)
+        << MatchTable::Comment("NewInsnID")
+        << MatchTable::ULEB128Value(NewInsnID)
         << MatchTable::Comment("OldInsnID")
-        << MatchTable::IntValue(OldInsnVarID) << MatchTable::Comment("OpIdx")
-        << MatchTable::IntValue(Operand.getOpIdx())
+        << MatchTable::ULEB128Value(OldInsnVarID)
+        << MatchTable::Comment("OpIdx")
+        << MatchTable::ULEB128Value(Operand.getOpIdx())
         << MatchTable::Comment("SubRegIdx")
-        << MatchTable::IntValue(SubReg->EnumValue)
+        << MatchTable::IntValue(2, SubReg->EnumValue)
         << MatchTable::Comment(SymbolicName) << MatchTable::LineBreak;
 }
 
@@ -1818,15 +1918,16 @@ void CopySubRegRenderer::emitRenderOpcodes(MatchTable &Table,
 void AddRegisterRenderer::emitRenderOpcodes(MatchTable &Table,
                                             RuleMatcher &Rule) const {
   Table << MatchTable::Opcode("GIR_AddRegister")
-        << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID);
+        << MatchTable::Comment("InsnID") << MatchTable::ULEB128Value(InsnID);
   if (RegisterDef->getName() != "zero_reg") {
     Table << MatchTable::NamedValue(
+        2,
         (RegisterDef->getValue("Namespace")
              ? RegisterDef->getValueAsString("Namespace")
              : ""),
         RegisterDef->getName());
   } else {
-    Table << MatchTable::NamedValue(Target.getRegNamespace(), "NoRegister");
+    Table << MatchTable::NamedValue(2, Target.getRegNamespace(), "NoRegister");
   }
   Table << MatchTable::Comment("AddRegisterRegFlags");
 
@@ -1834,9 +1935,9 @@ void AddRegisterRenderer::emitRenderOpcodes(MatchTable &Table,
   // really needed for a physical register reference. We can pack the
   // register and flags in a single field.
   if (IsDef)
-    Table << MatchTable::NamedValue("RegState::Define");
+    Table << MatchTable::NamedValue(2, "RegState::Define");
   else
-    Table << MatchTable::IntValue(0);
+    Table << MatchTable::IntValue(2, 0);
   Table << MatchTable::LineBreak;
 }
 
@@ -1844,37 +1945,69 @@ void AddRegisterRenderer::emitRenderOpcodes(MatchTable &Table,
 
 void TempRegRenderer::emitRenderOpcodes(MatchTable &Table,
                                         RuleMatcher &Rule) const {
+  const bool NeedsFlags = (SubRegIdx || IsDef);
   if (SubRegIdx) {
     assert(!IsDef);
     Table << MatchTable::Opcode("GIR_AddTempSubRegister");
   } else
-    Table << MatchTable::Opcode("GIR_AddTempRegister");
+    Table << MatchTable::Opcode(NeedsFlags ? "GIR_AddTempRegister"
+                                           : "GIR_AddSimpleTempRegister");
 
-  Table << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID)
-        << MatchTable::Comment("TempRegID") << MatchTable::IntValue(TempRegID)
-        << MatchTable::Comment("TempRegFlags");
+  Table << MatchTable::Comment("InsnID") << MatchTable::ULEB128Value(InsnID)
+        << MatchTable::Comment("TempRegID")
+        << MatchTable::ULEB128Value(TempRegID);
 
+  if (!NeedsFlags) {
+    Table << MatchTable::LineBreak;
+    return;
+  }
+
+  Table << MatchTable::Comment("TempRegFlags");
   if (IsDef) {
     SmallString<32> RegFlags;
     RegFlags += "RegState::Define";
     if (IsDead)
       RegFlags += "|RegState::Dead";
-    Table << MatchTable::NamedValue(RegFlags);
+    Table << MatchTable::NamedValue(2, RegFlags);
   } else
-    Table << MatchTable::IntValue(0);
+    Table << MatchTable::IntValue(2, 0);
 
   if (SubRegIdx)
-    Table << MatchTable::NamedValue(SubRegIdx->getQualifiedName());
+    Table << MatchTable::NamedValue(2, SubRegIdx->getQualifiedName());
   Table << MatchTable::LineBreak;
 }
 
+//===- ImmRenderer --------------------------------------------------------===//
+
+void ImmRenderer::emitAddImm(MatchTable &Table, RuleMatcher &RM,
+                             unsigned InsnID, int64_t Imm, StringRef ImmName) {
+  const bool IsInt8 = isInt<8>(Imm);
+
+  Table << MatchTable::Opcode(IsInt8 ? "GIR_AddImm8" : "GIR_AddImm")
+        << MatchTable::Comment("InsnID") << MatchTable::ULEB128Value(InsnID)
+        << MatchTable::Comment(ImmName)
+        << MatchTable::IntValue(IsInt8 ? 1 : 8, Imm) << MatchTable::LineBreak;
+}
+
+void ImmRenderer::emitRenderOpcodes(MatchTable &Table,
+                                    RuleMatcher &Rule) const {
+  if (CImmLLT) {
+    assert(Table.isCombiner() &&
+           "ConstantInt immediate are only for combiners!");
+    Table << MatchTable::Opcode("GIR_AddCImm") << MatchTable::Comment("InsnID")
+          << MatchTable::ULEB128Value(InsnID) << MatchTable::Comment("Type")
+          << *CImmLLT << MatchTable::Comment("Imm")
+          << MatchTable::IntValue(8, Imm) << MatchTable::LineBreak;
+  } else
+    emitAddImm(Table, Rule, InsnID, Imm);
+}
+
 //===- SubRegIndexRenderer ------------------------------------------------===//
 
 void SubRegIndexRenderer::emitRenderOpcodes(MatchTable &Table,
                                             RuleMatcher &Rule) const {
-  Table << MatchTable::Opcode("GIR_AddImm") << MatchTable::Comment("InsnID")
-        << MatchTable::IntValue(InsnID) << MatchTable::Comment("SubRegIndex")
-        << MatchTable::IntValue(SubRegIdx->EnumValue) << MatchTable::LineBreak;
+  ImmRenderer::emitAddImm(Table, Rule, InsnID, SubRegIdx->EnumValue,
+                          "SubRegIndex");
 }
 
 //===- RenderComplexPatternOperand ----------------------------------------===//
@@ -1885,15 +2018,15 @@ void RenderComplexPatternOperand::emitRenderOpcodes(MatchTable &Table,
                SubOperand ? (SubReg ? "GIR_ComplexSubOperandSubRegRenderer"
                                     : "GIR_ComplexSubOperandRenderer")
                           : "GIR_ComplexRenderer")
-        << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID)
+        << MatchTable::Comment("InsnID") << MatchTable::ULEB128Value(InsnID)
         << MatchTable::Comment("RendererID")
-        << MatchTable::IntValue(RendererID);
+        << MatchTable::IntValue(2, RendererID);
   if (SubOperand)
     Table << MatchTable::Comment("SubOperand")
-          << MatchTable::IntValue(*SubOperand);
+          << MatchTable::ULEB128Value(*SubOperand);
   if (SubReg)
     Table << MatchTable::Comment("SubRegIdx")
-          << MatchTable::IntValue(SubReg->EnumValue);
+          << MatchTable::IntValue(2, SubReg->EnumValue);
   Table << MatchTable::Comment(SymbolicName) << MatchTable::LineBreak;
 }
 
@@ -1904,11 +2037,12 @@ void CustomRenderer::emitRenderOpcodes(MatchTable &Table,
   InstructionMatcher &InsnMatcher = Rule.getInstructionMatcher(SymbolicName);
   unsigned OldInsnVarID = Rule.getInsnVarID(InsnMatcher);
   Table << MatchTable::Opcode("GIR_CustomRenderer")
-        << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID)
+        << MatchTable::Comment("InsnID") << MatchTable::ULEB128Value(InsnID)
         << MatchTable::Comment("OldInsnID")
-        << MatchTable::IntValue(OldInsnVarID) << MatchTable::Comment("Renderer")
-        << MatchTable::NamedValue("GICR_" +
-                                  Renderer.getValueAsString("RendererFn").str())
+        << MatchTable::ULEB128Value(OldInsnVarID)
+        << MatchTable::Comment("Renderer")
+        << MatchTable::NamedValue(
+               2, "GICR_" + Renderer.getValueAsString("RendererFn").str())
         << MatchTable::Comment(SymbolicName) << MatchTable::LineBreak;
 }
 
@@ -1918,14 +2052,14 @@ void CustomOperandRenderer::emitRenderOpcodes(MatchTable &Table,
                                               RuleMatcher &Rule) const {
   const OperandMatcher &OpdMatcher = Rule.getOperandMatcher(SymbolicName);
   Table << MatchTable::Opcode("GIR_CustomOperandRenderer")
-        << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID)
+        << MatchTable::Comment("InsnID") << MatchTable::ULEB128Value(InsnID)
         << MatchTable::Comment("OldInsnID")
-        << MatchTable::IntValue(OpdMatcher.getInsnVarID())
+        << MatchTable::ULEB128Value(OpdMatcher.getInsnVarID())
         << MatchTable::Comment("OpIdx")
-        << MatchTable::IntValue(OpdMatcher.getOpIdx())
+        << MatchTable::ULEB128Value(OpdMatcher.getOpIdx())
         << MatchTable::Comment("OperandRenderer")
-        << MatchTable::NamedValue("GICR_" +
-                                  Renderer.getValueAsString("RendererFn").str())
+        << MatchTable::NamedValue(
+               2, "GICR_" + Renderer.getValueAsString("RendererFn").str())
         << MatchTable::Comment(SymbolicName) << MatchTable::LineBreak;
 }
 
@@ -1934,7 +2068,7 @@ void CustomOperandRenderer::emitRenderOpcodes(MatchTable &Table,
 void CustomCXXAction::emitActionOpcodes(MatchTable &Table,
                                         RuleMatcher &Rule) const {
   Table << MatchTable::Opcode("GIR_CustomAction")
-        << MatchTable::NamedValue(FnEnumName) << MatchTable::LineBreak;
+        << MatchTable::NamedValue(2, FnEnumName) << MatchTable::LineBreak;
 }
 
 //===- BuildMIAction ------------------------------------------------------===//
@@ -1977,23 +2111,23 @@ void BuildMIAction::emitActionOpcodes(MatchTable &Table,
   const auto AddMIFlags = [&]() {
     for (const InstructionMatcher *IM : CopiedFlags) {
       Table << MatchTable::Opcode("GIR_CopyMIFlags")
-            << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID)
+            << MatchTable::Comment("InsnID") << MatchTable::ULEB128Value(InsnID)
             << MatchTable::Comment("OldInsnID")
-            << MatchTable::IntValue(IM->getInsnVarID())
+            << MatchTable::ULEB128Value(IM->getInsnVarID())
             << MatchTable::LineBreak;
     }
 
     if (!SetFlags.empty()) {
       Table << MatchTable::Opcode("GIR_SetMIFlags")
-            << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID)
-            << MatchTable::NamedValue(join(SetFlags, " | "))
+            << MatchTable::Comment("InsnID") << MatchTable::ULEB128Value(InsnID)
+            << MatchTable::NamedValue(4, join(SetFlags, " | "))
             << MatchTable::LineBreak;
     }
 
     if (!UnsetFlags.empty()) {
       Table << MatchTable::Opcode("GIR_UnsetMIFlags")
-            << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID)
-            << MatchTable::NamedValue(join(UnsetFlags, " | "))
+            << MatchTable::Comment("InsnID") << MatchTable::ULEB128Value(InsnID)
+            << MatchTable::NamedValue(4, join(UnsetFlags, " | "))
             << MatchTable::LineBreak;
     }
   };
@@ -2004,11 +2138,11 @@ void BuildMIAction::emitActionOpcodes(MatchTable &Table,
 
     unsigned RecycleInsnID = Rule.getInsnVarID(*Matched);
     Table << MatchTable::Opcode("GIR_MutateOpcode")
-          << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID)
+          << MatchTable::Comment("InsnID") << MatchTable::ULEB128Value(InsnID)
           << MatchTable::Comment("RecycleInsnID")
-          << MatchTable::IntValue(RecycleInsnID)
+          << MatchTable::ULEB128Value(RecycleInsnID)
           << MatchTable::Comment("Opcode")
-          << MatchTable::NamedValue(I->Namespace, I->TheDef->getName())
+          << MatchTable::NamedValue(2, I->Namespace, I->TheDef->getName())
           << MatchTable::LineBreak;
 
     if (!I->ImplicitDefs.empty() || !I->ImplicitUses.empty()) {
@@ -2018,10 +2152,11 @@ void BuildMIAction::emitActionOpcodes(MatchTable &Table,
                              : "";
         const bool IsDead = DeadImplicitDefs.contains(Def);
         Table << MatchTable::Opcode("GIR_AddImplicitDef")
-              << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID)
-              << MatchTable::NamedValue(Namespace, Def->getName())
-              << (IsDead ? MatchTable::NamedValue("RegState", "Dead")
-                         : MatchTable::IntValue(0))
+              << MatchTable::Comment("InsnID")
+              << MatchTable::ULEB128Value(InsnID)
+              << MatchTable::NamedValue(2, Namespace, Def->getName())
+              << (IsDead ? MatchTable::NamedValue(2, "RegState", "Dead")
+                         : MatchTable::IntValue(2, 0))
               << MatchTable::LineBreak;
       }
       for (auto *Use : I->ImplicitUses) {
@@ -2029,8 +2164,9 @@ void BuildMIAction::emitActionOpcodes(MatchTable &Table,
                              ? Use->getValueAsString("Namespace")
                              : "";
         Table << MatchTable::Opcode("GIR_AddImplicitUse")
-              << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID)
-              << MatchTable::NamedValue(Namespace, Use->getName())
+              << MatchTable::Comment("InsnID")
+              << MatchTable::ULEB128Value(InsnID)
+              << MatchTable::NamedValue(2, Namespace, Use->getName())
               << MatchTable::LineBreak;
       }
     }
@@ -2043,8 +2179,8 @@ void BuildMIAction::emitActionOpcodes(MatchTable &Table,
   //       mutation due to commutative operations.
 
   Table << MatchTable::Opcode("GIR_BuildMI") << MatchTable::Comment("InsnID")
-        << MatchTable::IntValue(InsnID) << MatchTable::Comment("Opcode")
-        << MatchTable::NamedValue(I->Namespace, I->TheDef->getName())
+        << MatchTable::ULEB128Value(InsnID) << MatchTable::Comment("Opcode")
+        << MatchTable::NamedValue(2, I->Namespace, I->TheDef->getName())
         << MatchTable::LineBreak;
   for (const auto &Renderer : OperandRenderers)
     Renderer->emitRenderOpcodes(Table, Rule);
@@ -2055,17 +2191,14 @@ void BuildMIAction::emitActionOpcodes(MatchTable &Table,
     if (DeadImplicitDefs.contains(Def)) {
       Table
           << MatchTable::Opcode("GIR_SetImplicitDefDead")
-          << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID)
+          << MatchTable::Comment("InsnID") << MatchTable::ULEB128Value(InsnID)
           << MatchTable::Comment(
                  ("OpIdx for " + Namespace + "::" + Def->getName() + "").str())
-          << MatchTable::IntValue(OpIdx) << MatchTable::LineBreak;
+          << MatchTable::ULEB128Value(OpIdx) << MatchTable::LineBreak;
     }
   }
 
   if (I->mayLoad || I->mayStore) {
-    Table << MatchTable::Opcode("GIR_MergeMemOperands")
-          << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID)
-          << MatchTable::Comment("MergeInsnID's");
     // Emit the ID's for all the instructions that are matched by this rule.
     // TODO: Limit this to matched instructions that mayLoad/mayStore or have
     //       some other means of having a memoperand. Also limit this to
@@ -2073,14 +2206,20 @@ void BuildMIAction::emitActionOpcodes(MatchTable &Table,
     //       example, (G_SEXT (G_LOAD x)) that results in separate load and
     //       sign-extend instructions shouldn't put the memoperand on the
     //       sign-extend since it has no effect there.
+
     std::vector<unsigned> MergeInsnIDs;
     for (const auto &IDMatcherPair : Rule.defined_insn_vars())
       MergeInsnIDs.push_back(IDMatcherPair.second);
     llvm::sort(MergeInsnIDs);
+
+    Table << MatchTable::Opcode("GIR_MergeMemOperands")
+          << MatchTable::Comment("InsnID") << MatchTable::ULEB128Value(InsnID)
+          << MatchTable::Comment("NumInsns")
+          << MatchTable::IntValue(1, MergeInsnIDs.size())
+          << MatchTable::Comment("MergeInsnID's");
     for (const auto &MergeInsnID : MergeInsnIDs)
-      Table << MatchTable::IntValue(MergeInsnID);
-    Table << MatchTable::NamedValue("GIU_MergeMemOperands_EndOfList")
-          << MatchTable::LineBreak;
+      Table << MatchTable::ULEB128Value(MergeInsnID);
+    Table << MatchTable::LineBreak;
   }
 
   AddMIFlags();
@@ -2097,9 +2236,9 @@ void BuildMIAction::emitActionOpcodes(MatchTable &Table,
 void BuildConstantAction::emitActionOpcodes(MatchTable &Table,
                                             RuleMatcher &Rule) const {
   Table << MatchTable::Opcode("GIR_BuildConstant")
-        << MatchTable::Comment("TempRegID") << MatchTable::IntValue(TempRegID)
-        << MatchTable::Comment("Val") << MatchTable::IntValue(Val)
-        << MatchTable::LineBreak;
+        << MatchTable::Comment("TempRegID")
+        << MatchTable::ULEB128Value(TempRegID) << MatchTable::Comment("Val")
+        << MatchTable::IntValue(8, Val) << MatchTable::LineBreak;
 }
 
 //===- EraseInstAction ----------------------------------------------------===//
@@ -2111,7 +2250,7 @@ void EraseInstAction::emitActionOpcodes(MatchTable &Table, RuleMatcher &Rule,
     return;
 
   Table << MatchTable::Opcode("GIR_EraseFromParent")
-        << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID)
+        << MatchTable::Comment("InsnID") << MatchTable::ULEB128Value(InsnID)
         << MatchTable::LineBreak;
 }
 
@@ -2128,10 +2267,12 @@ void ReplaceRegAction::emitAdditionalPredicates(MatchTable &Table,
     return;
 
   Table << MatchTable::Opcode("GIM_CheckCanReplaceReg")
-        << MatchTable::Comment("OldInsnID") << MatchTable::IntValue(OldInsnID)
-        << MatchTable::Comment("OldOpIdx") << MatchTable::IntValue(OldOpIdx)
-        << MatchTable::Comment("NewInsnId") << MatchTable::IntValue(NewInsnId)
-        << MatchTable::Comment("NewOpIdx") << MatchTable::IntValue(NewOpIdx)
+        << MatchTable::Comment("OldInsnID")
+        << MatchTable::ULEB128Value(OldInsnID)
+        << MatchTable::Comment("OldOpIdx") << MatchTable::ULEB128Value(OldOpIdx)
+        << MatchTable::Comment("NewInsnId")
+        << MatchTable::ULEB128Value(NewInsnId)
+        << MatchTable::Comment("NewOpIdx") << MatchTable::ULEB128Value(NewOpIdx)
         << MatchTable::LineBreak;
 }
 
@@ -2139,17 +2280,22 @@ void ReplaceRegAction::emitActionOpcodes(MatchTable &Table,
                                          RuleMatcher &Rule) const {
   if (TempRegID != (unsigned)-1) {
     Table << MatchTable::Opcode("GIR_ReplaceRegWithTempReg")
-          << MatchTable::Comment("OldInsnID") << MatchTable::IntValue(OldInsnID)
-          << MatchTable::Comment("OldOpIdx") << MatchTable::IntValue(OldOpIdx)
-          << MatchTable::Comment("TempRegID") << MatchTable::IntValue(TempRegID)
-          << MatchTable::LineBreak;
+          << MatchTable::Comment("OldInsnID")
+          << MatchTable::ULEB128Value(OldInsnID)
+          << MatchTable::Comment("OldOpIdx")
+          << MatchTable::ULEB128Value(OldOpIdx)
+          << MatchTable::Comment("TempRegID")
+          << MatchTable::ULEB128Value(TempRegID) << MatchTable::LineBreak;
   } else {
     Table << MatchTable::Opcode("GIR_ReplaceReg")
-          << MatchTable::Comment("OldInsnID") << MatchTable::IntValue(OldInsnID)
-          << MatchTable::Comment("OldOpIdx") << MatchTable::IntValue(OldOpIdx)
-          << MatchTable::Comment("NewInsnId") << MatchTable::IntValue(NewInsnId)
-          << MatchTable::Comment("NewOpIdx") << MatchTable::IntValue(NewOpIdx)
-          << MatchTable::LineBreak;
+          << MatchTable::Comment("OldInsnID")
+          << MatchTable::ULEB128Value(OldInsnID)
+          << MatchTable::Comment("OldOpIdx")
+          << MatchTable::ULEB128Value(OldOpIdx)
+          << MatchTable::Comment("NewInsnId")
+          << MatchTable::ULEB128Value(NewInsnId)
+          << MatchTable::Comment("NewOpIdx")
+          << MatchTable::ULEB128Value(NewOpIdx) << MatchTable::LineBreak;
   }
 }
 
@@ -2158,9 +2304,9 @@ void ReplaceRegAction::emitActionOpcodes(MatchTable &Table,
 void ConstrainOperandToRegClassAction::emitActionOpcodes(
     MatchTable &Table, RuleMatcher &Rule) const {
   Table << MatchTable::Opcode("GIR_ConstrainOperandRC")
-        << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID)
-        << MatchTable::Comment("Op") << MatchTable::IntValue(OpIdx)
-        << MatchTable::NamedValue(RC.getQualifiedIdName())
+        << MatchTable::Comment("InsnID") << MatchTable::ULEB128Value(InsnID)
+        << MatchTable::Comment("Op") << MatchTable::ULEB128Value(OpIdx)
+        << MatchTable::NamedValue(2, RC.getQualifiedIdName())
         << MatchTable::LineBreak;
 }
 
@@ -2169,8 +2315,9 @@ void ConstrainOperandToRegClassAction::emitActionOpcodes(
 void MakeTempRegisterAction::emitActionOpcodes(MatchTable &Table,
                                                RuleMatcher &Rule) const {
   Table << MatchTable::Opcode("GIR_MakeTempReg")
-        << MatchTable::Comment("TempRegID") << MatchTable::IntValue(TempRegID)
-        << MatchTable::Comment("TypeID") << Ty << MatchTable::LineBreak;
+        << MatchTable::Comment("TempRegID")
+        << MatchTable::ULEB128Value(TempRegID) << MatchTable::Comment("TypeID")
+        << Ty << MatchTable::LineBreak;
 }
 
 } // namespace gi
diff --git a/llvm/utils/TableGen/GlobalISelMatchTable.h b/llvm/utils/TableGen/GlobalISelMatchTable.h
index 469390d7312329bfdc4de5dfefc5e8ba939d4518..7cb5345f51f8cb33833dea698c6916f9c1b41a16 100644
--- a/llvm/utils/TableGen/GlobalISelMatchTable.h
+++ b/llvm/utils/TableGen/GlobalISelMatchTable.h
@@ -59,6 +59,9 @@ using GISelFlags = std::uint16_t;
 
 //===- Helper functions ---------------------------------------------------===//
 
+void emitEncodingMacrosDef(raw_ostream &OS);
+void emitEncodingMacrosUndef(raw_ostream &OS);
+
 std::string getNameForFeatureBitset(const std::vector<Record *> &FeatureBitset,
                                     int HwModeIdx);
 
@@ -119,6 +122,9 @@ struct MatchTableRecord {
     /// Causes the formatter to remove a level of indentation after emitting the
     /// record.
     MTRF_Outdent = 0x40,
+    /// Causes the formatter to not use encoding macros to emit this multi-byte
+    /// value.
+    MTRF_PreEncoded = 0x80,
   };
 
   /// When MTRF_Label or MTRF_JumpTarget is used, indicates a label id to
@@ -194,12 +200,15 @@ public:
   static MatchTableRecord LineBreak;
   static MatchTableRecord Comment(StringRef Comment);
   static MatchTableRecord Opcode(StringRef Opcode, int IndentAdjust = 0);
-  static MatchTableRecord NamedValue(StringRef NamedValue);
-  static MatchTableRecord NamedValue(StringRef NamedValue, int64_t RawValue);
-  static MatchTableRecord NamedValue(StringRef Namespace, StringRef NamedValue);
-  static MatchTableRecord NamedValue(StringRef Namespace, StringRef NamedValue,
+  static MatchTableRecord NamedValue(unsigned NumBytes, StringRef NamedValue);
+  static MatchTableRecord NamedValue(unsigned NumBytes, StringRef NamedValue,
                                      int64_t RawValue);
-  static MatchTableRecord IntValue(int64_t IntValue);
+  static MatchTableRecord NamedValue(unsigned NumBytes, StringRef Namespace,
+                                     StringRef NamedValue);
+  static MatchTableRecord NamedValue(unsigned NumBytes, StringRef Namespace,
+                                     StringRef NamedValue, int64_t RawValue);
+  static MatchTableRecord IntValue(unsigned NumBytes, int64_t IntValue);
+  static MatchTableRecord ULEB128Value(uint64_t IntValue);
   static MatchTableRecord Label(unsigned LabelID);
   static MatchTableRecord JumpTarget(unsigned LabelID);
 
@@ -301,9 +310,9 @@ private:
 inline MatchTable &operator<<(MatchTable &Table,
                               const LLTCodeGenOrTempType &Ty) {
   if (Ty.isLLTCodeGen())
-    Table << MatchTable::NamedValue(Ty.getLLTCodeGen().getCxxEnumValue());
+    Table << MatchTable::NamedValue(1, Ty.getLLTCodeGen().getCxxEnumValue());
   else
-    Table << MatchTable::IntValue(Ty.getTempTypeIdx());
+    Table << MatchTable::IntValue(1, Ty.getTempTypeIdx());
   return Table;
 }
 
@@ -1669,7 +1678,7 @@ public:
   void emitPredicateOpcodes(MatchTable &Table,
                             RuleMatcher &Rule) const override {
     Table << MatchTable::Opcode("GIM_CheckHasNoUse")
-          << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID)
+          << MatchTable::Comment("MI") << MatchTable::ULEB128Value(InsnVarID)
           << MatchTable::LineBreak;
   }
 };
@@ -2066,21 +2075,10 @@ public:
     return R->getKind() == OR_Imm;
   }
 
-  void emitRenderOpcodes(MatchTable &Table, RuleMatcher &Rule) const override {
-    if (CImmLLT) {
-      assert(Table.isCombiner() &&
-             "ConstantInt immediate are only for combiners!");
-      Table << MatchTable::Opcode("GIR_AddCImm")
-            << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID)
-            << MatchTable::Comment("Type") << *CImmLLT
-            << MatchTable::Comment("Imm") << MatchTable::IntValue(Imm)
-            << MatchTable::LineBreak;
-    } else {
-      Table << MatchTable::Opcode("GIR_AddImm") << MatchTable::Comment("InsnID")
-            << MatchTable::IntValue(InsnID) << MatchTable::Comment("Imm")
-            << MatchTable::IntValue(Imm) << MatchTable::LineBreak;
-    }
-  }
+  static void emitAddImm(MatchTable &Table, RuleMatcher &RM, unsigned InsnID,
+                         int64_t Imm, StringRef ImmName = "Imm");
+
+  void emitRenderOpcodes(MatchTable &Table, RuleMatcher &Rule) const override;
 };
 
 /// Adds an enum value for a subreg index to the instruction being built.
@@ -2366,7 +2364,7 @@ public:
 
   void emitActionOpcodes(MatchTable &Table, RuleMatcher &Rule) const override {
     Table << MatchTable::Opcode("GIR_ConstrainSelectedInstOperands")
-          << MatchTable::Comment("InsnID") << MatchTable::IntValue(InsnID)
+          << MatchTable::Comment("InsnID") << MatchTable::ULEB128Value(InsnID)
           << MatchTable::LineBreak;
   }
 };
diff --git a/llvm/utils/TableGen/GlobalISelMatchTableExecutorEmitter.cpp b/llvm/utils/TableGen/GlobalISelMatchTableExecutorEmitter.cpp
index c6cd3240a94e7b065771bc7f0554c40821db796f..5697899a915a0355fdede013abbd34aa89865426 100644
--- a/llvm/utils/TableGen/GlobalISelMatchTableExecutorEmitter.cpp
+++ b/llvm/utils/TableGen/GlobalISelMatchTableExecutorEmitter.cpp
@@ -164,11 +164,14 @@ void GlobalISelMatchTableExecutorEmitter::emitTypeObjects(
 
 void GlobalISelMatchTableExecutorEmitter::emitMatchTable(
     raw_ostream &OS, const MatchTable &Table) {
-  OS << "const int64_t *" << getClassName() << "::getMatchTable() const {\n";
+  emitEncodingMacrosDef(OS);
+  OS << "const uint8_t *" << getClassName() << "::getMatchTable() const {\n";
   Table.emitDeclaration(OS);
   OS << "  return ";
   Table.emitUse(OS);
   OS << ";\n}\n";
+  emitEncodingMacrosUndef(OS);
+  OS << "\n";
 }
 
 void GlobalISelMatchTableExecutorEmitter::emitExecutorImpl(
@@ -187,7 +190,9 @@ void GlobalISelMatchTableExecutorEmitter::emitExecutorImpl(
   emitCustomOperandRenderers(OS, CustomOperandRenderers);
   emitAdditionalImpl(OS);
   emitRunCustomAction(OS);
+
   emitMatchTable(OS, Table);
+
   OS << "#endif // ifdef " << IfDefName << "\n\n";
 }
 
@@ -226,7 +231,7 @@ void GlobalISelMatchTableExecutorEmitter::emitTemporariesDecl(
         "const override;\n"
      << "  bool testImmPredicate_APFloat(unsigned PredicateID, const APFloat "
         "&Imm) const override;\n"
-     << "  const int64_t *getMatchTable() const override;\n"
+     << "  const uint8_t *getMatchTable() const override;\n"
      << "  bool testMIPredicate_MI(unsigned PredicateID, const MachineInstr &MI"
         ", const MatcherState &State) "
         "const override;\n"
diff --git a/llvm/utils/TableGen/README.md b/llvm/utils/TableGen/README.md
index 3bee6555566a61c15e86b3aeadf122b0add40e11..ab3657d07d6630aaa0f7997859ac97bbfcf42bb1 100644
--- a/llvm/utils/TableGen/README.md
+++ b/llvm/utils/TableGen/README.md
@@ -35,6 +35,7 @@ Resources for learning the language:
 * [TableGen Overview](https://llvm.org/docs/TableGen/index.html)
 * [Programmer's reference guide](https://llvm.org/docs/TableGen/ProgRef.html)
 * [Tutorial](jupyter/tablegen_tutorial_part_1.ipynb)
+* [Tools for Learning LLVM TableGen](https://blog.llvm.org/posts/2023-12-07-tools-for-learning-llvm-tablegen/)
 * [Lessons in TableGen](https://www.youtube.com/watch?v=45gmF77JFBY) (video),
   [slides](https://archive.fosdem.org/2019/schedule/event/llvm_tablegen/attachments/slides/3304/export/events/attachments/llvm_tablegen/slides/3304/tablegen.pdf)
 * [Improving Your TableGen Descriptions](https://www.youtube.com/watch?v=dIEVUlsiktQ)
diff --git a/llvm/utils/TableGen/X86FoldTablesEmitter.cpp b/llvm/utils/TableGen/X86FoldTablesEmitter.cpp
index 5fb6b048542bd83115018159459b361f58157faa..83025205310e81cd1c11847caa4cdbcaad5b0e38 100644
--- a/llvm/utils/TableGen/X86FoldTablesEmitter.cpp
+++ b/llvm/utils/TableGen/X86FoldTablesEmitter.cpp
@@ -530,26 +530,17 @@ void X86FoldTablesEmitter::addBroadcastEntry(
   StringRef MemInstName = MemInst->TheDef->getName();
   Record *Domain = RegRec->getValueAsDef("ExeDomain");
   bool IsSSEPackedInt = Domain->getName() == "SSEPackedInt";
-  // TODO: Rename AVX512 instructions to simplify conditions, e.g.
-  //         D128 -> DZ128
-  //         D256 -> DZ256
-  //         VPERMI2Drr -> VPERMI2DZrr
-  //         VPERMI2Drmb -> VPERMI2DZrmb
   if ((RegInstName.contains("DZ") || RegInstName.contains("DWZ") ||
-       RegInstName.contains("D128") || RegInstName.contains("D256") ||
        RegInstName.contains("Dr") || RegInstName.contains("I32")) &&
       IsSSEPackedInt) {
     assert((MemInstName.contains("DZ") || RegInstName.contains("DWZ") ||
-            MemInstName.contains("D128") || MemInstName.contains("D256") ||
             MemInstName.contains("Dr") || MemInstName.contains("I32")) &&
            "Unmatched names for broadcast");
     Result.BroadcastKind = X86FoldTableEntry::BCAST_D;
   } else if ((RegInstName.contains("QZ") || RegInstName.contains("QBZ") ||
-              RegInstName.contains("Q128") || RegInstName.contains("Q256") ||
               RegInstName.contains("Qr") || RegInstName.contains("I64")) &&
              IsSSEPackedInt) {
     assert((MemInstName.contains("QZ") || MemInstName.contains("QBZ") ||
-            MemInstName.contains("Q128") || MemInstName.contains("Q256") ||
             MemInstName.contains("Qr") || MemInstName.contains("I64")) &&
            "Unmatched names for broadcast");
     Result.BroadcastKind = X86FoldTableEntry::BCAST_Q;
diff --git a/llvm/utils/count/CMakeLists.txt b/llvm/utils/count/CMakeLists.txt
index 4e0d371334e473d643d791ae839ea51a953f0c7f..cfd1f4a85d8a17282903b624c92e3b7384d3784e 100644
--- a/llvm/utils/count/CMakeLists.txt
+++ b/llvm/utils/count/CMakeLists.txt
@@ -1,3 +1,7 @@
+set(LLVM_LINK_COMPONENTS
+  support
+)
+
 add_llvm_utility(count
   count.c
   )
diff --git a/llvm/utils/count/count.c b/llvm/utils/count/count.c
index 7149c14a63abb305abf20e18749a6f5a9dafa801..300be2aa8a18e209ad3491bf530f019bec6e62ed 100644
--- a/llvm/utils/count/count.c
+++ b/llvm/utils/count/count.c
@@ -6,10 +6,18 @@
  *
 \*===----------------------------------------------------------------------===*/
 
-#include <stdlib.h>
+#include "llvm/Support/AutoConvert.h"
 #include <stdio.h>
+#include <stdlib.h>
 
 int main(int argc, char **argv) {
+#ifdef __MVS__
+  if (enableAutoConversion(fileno(stdin)) == -1)
+    fprintf(stderr, "Setting conversion on stdin failed\n");
+
+  if (enableAutoConversion(fileno(stderr)) == -1)
+    fprintf(stdout, "Setting conversion on stderr failed\n");
+#endif
   size_t Count, NumLines, NumRead;
   char Buffer[4096], *End;
 
diff --git a/llvm/utils/gn/secondary/clang/lib/Headers/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/Headers/BUILD.gn
index 6dcf0fc08a34d664b9634e091b6b1f150cd958c3..3debc48a4bb99a85ef11629a974283231e13c8c3 100644
--- a/llvm/utils/gn/secondary/clang/lib/Headers/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/lib/Headers/BUILD.gn
@@ -50,6 +50,13 @@ clang_tablegen("arm_cde") {
   output_name = "arm_cde.h"
 }
 
+# Generate arm__vector_types.h
+clang_tablegen("arm_vector_types") {
+  args = [ "-gen-arm-vector-type" ]
+  td_file = "//clang/include/clang/Basic/arm_neon.td"
+  output_name = "arm_vector_types.h"
+}
+
 # Generate riscv_vector.h
 clang_tablegen("riscv_vector") {
   args = [ "-gen-riscv-vector-header" ]
@@ -67,6 +74,7 @@ copy("tablegen_headers") {
     ":arm_neon",
     ":arm_sme_draft_spec_subject_to_change",
     ":arm_sve",
+    ":arm_vector_types",
     ":riscv_vector",
   ]
   sources = []
diff --git a/llvm/utils/gn/secondary/clang/lib/StaticAnalyzer/Core/BUILD.gn b/llvm/utils/gn/secondary/clang/lib/StaticAnalyzer/Core/BUILD.gn
index a48a2402f3a1f6467235068728e81ab9b46602ad..19d6ffabd67dd402c6a16e5e1eee7a0e72ecf722 100644
--- a/llvm/utils/gn/secondary/clang/lib/StaticAnalyzer/Core/BUILD.gn
+++ b/llvm/utils/gn/secondary/clang/lib/StaticAnalyzer/Core/BUILD.gn
@@ -20,6 +20,7 @@ static_library("Core") {
     "BlockCounter.cpp",
     "BugReporter.cpp",
     "BugReporterVisitors.cpp",
+    "BugSuppression.cpp",
     "CallDescription.cpp",
     "CallEvent.cpp",
     "Checker.cpp",
diff --git a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
index 5cc9ec5889d420eb494b20d7d2c870ddc0de3870..4ad2b7ea6f7a708173726aca976b6edee77d5c87 100644
--- a/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
+++ b/llvm/utils/gn/secondary/libcxx/include/BUILD.gn
@@ -34,6 +34,7 @@ if (current_toolchain == default_toolchain) {
       "_LIBCPP_HAS_NO_LOCALIZATION=",
       "_LIBCPP_HAS_NO_WIDE_CHARACTERS=",
       "_LIBCPP_HAS_NO_STD_MODULES=",
+      "_LIBCPP_INSTRUMENTED_WITH_ASAN=",
       "_LIBCPP_ABI_DEFINES=",
       "_LIBCPP_HARDENING_MODE_DEFAULT=_LIBCPP_HARDENING_MODE_NONE",
       "_LIBCPP_PSTL_CPU_BACKEND_LIBDISPATCH=",
diff --git a/llvm/utils/gn/secondary/llvm/lib/ObjectYAML/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/ObjectYAML/BUILD.gn
index 34d5008a4b18c514aa4ebcb5cd6700187e07dba7..0708f2a1897f5f5604d1cecc18b043387b77422e 100644
--- a/llvm/utils/gn/secondary/llvm/lib/ObjectYAML/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/ObjectYAML/BUILD.gn
@@ -22,6 +22,8 @@ static_library("ObjectYAML") {
     "DXContainerYAML.cpp",
     "ELFEmitter.cpp",
     "ELFYAML.cpp",
+    "GOFFEmitter.cpp",
+    "GOFFYAML.cpp",
     "MachOEmitter.cpp",
     "MachOYAML.cpp",
     "MinidumpEmitter.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Passes/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Passes/BUILD.gn
index d98420100df01b1ba306acf85ce4ac9521d1731f..5cd96737886c5ebabf9565f72ec662c0d60ea0f4 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Passes/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Passes/BUILD.gn
@@ -8,6 +8,7 @@ static_library("Passes") {
     "//llvm/lib/Support",
     "//llvm/lib/Target",
     "//llvm/lib/Transforms/AggressiveInstCombine",
+    "//llvm/lib/Transforms/CFGuard",
     "//llvm/lib/Transforms/Coroutines",
     "//llvm/lib/Transforms/HipStdPar",
     "//llvm/lib/Transforms/IPO",
diff --git a/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn
index 989e805f51178bacc8e47e235326845ed30a04b9..8f8b056ef0faebdc6d26d97082b9c3778afbf656 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Support/BUILD.gn
@@ -185,6 +185,8 @@ static_library("Support") {
   if (current_os == "linux" || current_os == "android") {
     libs += [ "dl" ]
   } else if (current_os == "win") {
+    libs = [ "ws2_32.lib" ]
+
     # Delay load shell32.dll if possible to speed up process startup.
     libs += [ "delayimp.lib" ]
     ldflags = [
diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn
index 6d518446fca291075cfeec1d151c4a2c0b350148..f85bf8f9940faf2c26610ae83e8a67006f25a436 100644
--- a/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/lib/Target/AMDGPU/BUILD.gn
@@ -141,6 +141,7 @@ static_library("LLVMAMDGPUCodeGen") {
     "AMDGPUCtorDtorLowering.cpp",
     "AMDGPUExportClustering.cpp",
     "AMDGPUFrameLowering.cpp",
+    "AMDGPUGlobalISelDivergenceLowering.cpp",
     "AMDGPUGlobalISelUtils.cpp",
     "AMDGPUHSAMetadataStreamer.cpp",
     "AMDGPUIGroupLP.cpp",
diff --git a/llvm/utils/gn/secondary/llvm/tools/llvm-config/BUILD.gn b/llvm/utils/gn/secondary/llvm/tools/llvm-config/BUILD.gn
index ad4ea08c9d8262da872e7cac53171fc87d183e82..bf50cd0fce46bdb27847311c689e4f6afb924634 100644
--- a/llvm/utils/gn/secondary/llvm/tools/llvm-config/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/tools/llvm-config/BUILD.gn
@@ -47,7 +47,7 @@ write_cmake_config("BuildVariables.inc") {
   if (host_os == "win") {
     # uuid.lib for FOLDERID_Profile in lib/Support/Windows/Path.inc.
     # advapi32.lib for CryptAcquireContextW in lib/Support/Windows/Path.inc.
-    system_libs = "psapi.lib shell32.lib ole32.lib uuid.lib advapi32.lib"
+    system_libs = "psapi.lib shell32.lib ole32.lib uuid.lib advapi32.lib ws2_32.lib"
   } else {
     system_libs += "-lm"
     if (host_os == "linux") {
diff --git a/llvm/utils/gn/secondary/llvm/unittests/Support/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/Support/BUILD.gn
index fddee579547c6920e1f42836fc9675c4f1c86d32..c6d40aeebbbec3a2c8796b9fe1567430cf52582e 100644
--- a/llvm/utils/gn/secondary/llvm/unittests/Support/BUILD.gn
+++ b/llvm/utils/gn/secondary/llvm/unittests/Support/BUILD.gn
@@ -106,6 +106,7 @@ unittest("SupportTests") {
     "raw_ostream_test.cpp",
     "raw_pwrite_stream_test.cpp",
     "raw_sha1_ostream_test.cpp",
+    "raw_socket_stream_test.cpp",
     "xxhashTest.cpp",
   ]
 }
diff --git a/mlir/include/mlir/Analysis/Presburger/Matrix.h b/mlir/include/mlir/Analysis/Presburger/Matrix.h
index 4d9f13832e0692acfd76033c41ef4ed5a587c0f1..89fad85c0c3374a43f3f36e302ad882543a0c9ed 100644
--- a/mlir/include/mlir/Analysis/Presburger/Matrix.h
+++ b/mlir/include/mlir/Analysis/Presburger/Matrix.h
@@ -265,6 +265,11 @@ public:
   // does not exist, which happens iff det = 0.
   // Assert-fails if the matrix is not square.
   Fraction determinant(FracMatrix *inverse = nullptr) const;
+
+  // Computes the Gram-Schmidt orthogonalisation
+  // of the rows of matrix (cubic time).
+  // The rows of the matrix must be linearly independent.
+  FracMatrix gramSchmidt() const;
 };
 
 } // namespace presburger
diff --git a/mlir/include/mlir/Analysis/Presburger/Utils.h b/mlir/include/mlir/Analysis/Presburger/Utils.h
index a451ae8bf55723e9d073da5f2a86daa2d7f9d285..20af0bfcd62ba61701c500f1bac4ff182aa24a15 100644
--- a/mlir/include/mlir/Analysis/Presburger/Utils.h
+++ b/mlir/include/mlir/Analysis/Presburger/Utils.h
@@ -276,6 +276,11 @@ SmallVector<MPInt, 8> getNegatedCoeffs(ArrayRef<MPInt> coeffs);
 /// a_1 x_1 + ... + a_n x_ + c < 0, i.e., -a_1 x_1 - ... - a_n x_ - c - 1 >= 0,
 /// since all the variables are constrained to be integers.
 SmallVector<MPInt, 8> getComplementIneq(ArrayRef<MPInt> ineq);
+
+/// Compute the dot product of two vectors.
+/// The vectors must have the same sizes.
+Fraction dotProduct(ArrayRef<Fraction> a, ArrayRef<Fraction> b);
+
 } // namespace presburger
 } // namespace mlir
 
diff --git a/mlir/include/mlir/Conversion/Passes.td b/mlir/include/mlir/Conversion/Passes.td
index 06756ff3df0bb3bd5d1fdcae0d3b8d9b97948885..6193aeb545bc6b10cb22b2cd665dd13e24904041 100644
--- a/mlir/include/mlir/Conversion/Passes.td
+++ b/mlir/include/mlir/Conversion/Passes.td
@@ -886,6 +886,12 @@ def ConvertSCFToOpenMPPass : Pass<"convert-scf-to-openmp", "ModuleOp"> {
   let summary = "Convert SCF parallel loop to OpenMP parallel + workshare "
                 "constructs.";
 
+  let options = [
+    Option<"numThreads", "num-threads", "unsigned",
+           /*default=kUseOpenMPDefaultNumThreads*/"0",
+           "Number of threads to use">
+  ];
+
   let dependentDialects = ["omp::OpenMPDialect", "LLVM::LLVMDialect",
                            "memref::MemRefDialect"];
 }
diff --git a/mlir/include/mlir/Dialect/ArmSME/Transforms/CMakeLists.txt b/mlir/include/mlir/Dialect/ArmSME/Transforms/CMakeLists.txt
index 38f48757b7749b72c22af60195190254a1fd36a7..509f3fc2519cbf85e7a8a4dd0c62f15d8c4b3524 100644
--- a/mlir/include/mlir/Dialect/ArmSME/Transforms/CMakeLists.txt
+++ b/mlir/include/mlir/Dialect/ArmSME/Transforms/CMakeLists.txt
@@ -3,5 +3,6 @@ mlir_tablegen(Passes.h.inc -gen-pass-decls -name ArmSME)
 mlir_tablegen(PassesEnums.h.inc -gen-enum-decls)
 mlir_tablegen(PassesEnums.cpp.inc -gen-enum-defs)
 add_public_tablegen_target(MLIRArmSMETransformsIncGen)
+add_dependencies(mlir-headers MLIRArmSMETransformsIncGen)
 
 add_mlir_doc(Passes ArmSMEPasses ./ -gen-pass-doc)
diff --git a/mlir/include/mlir/Dialect/ArmSME/Transforms/Passes.td b/mlir/include/mlir/Dialect/ArmSME/Transforms/Passes.td
index 02238f0a18bab9825d6073423c13ed8e5451b3b4..4266ac5b0c8cf68928792967490057d12df07367 100644
--- a/mlir/include/mlir/Dialect/ArmSME/Transforms/Passes.td
+++ b/mlir/include/mlir/Dialect/ArmSME/Transforms/Passes.td
@@ -20,6 +20,9 @@ def ArmStreamingMode : I32EnumAttr<"ArmStreamingMode", "Armv9 Streaming SVE mode
       // StreamingLocally: PSTATE.SM is kept internal and the callee manages it
       // on entry/exit.
       I32EnumAttrCase<"StreamingLocally", 2, "arm_locally_streaming">,
+      // StreamingCompatible: the function may be entered in either
+      // non-streaming mode (PSTATE.SM=0) or in streaming mode (PSTATE.SM=1)
+      I32EnumAttrCase<"StreamingCompatible", 3, "arm_streaming_compatible">,
     ]>{
   let cppNamespace = "mlir::arm_sme";
   let genSpecializedAttr = 0;
@@ -61,7 +64,11 @@ def EnableArmStreaming
                 clEnumValN(mlir::arm_sme::ArmStreamingMode::StreamingLocally,
                            "streaming-locally",
                            "Streaming mode is internal to the function, callee "
-                           "manages PSTATE.SM on entry/exit.")
+                           "manages PSTATE.SM on entry/exit."),
+                clEnumValN(mlir::arm_sme::ArmStreamingMode::StreamingCompatible,
+                           "streaming-compatible",
+                           "Function supports both streaming and non-streaming "
+                           "modes.")
           )}]>,
     Option<"zaMode", "za-mode", "mlir::arm_sme::ArmZaMode",
            /*default=*/"mlir::arm_sme::ArmZaMode::Disabled",
diff --git a/mlir/include/mlir/Dialect/ArmSME/Transforms/Transforms.h b/mlir/include/mlir/Dialect/ArmSME/Transforms/Transforms.h
index 8ea3e1e57b7caa5cdd775f793636e425f15ea13e..f622bc0562e9e1a9803f8d69090a2f02b9460d3e 100644
--- a/mlir/include/mlir/Dialect/ArmSME/Transforms/Transforms.h
+++ b/mlir/include/mlir/Dialect/ArmSME/Transforms/Transforms.h
@@ -15,11 +15,6 @@ class LLVMConversionTarget;
 class LLVMTypeConverter;
 class RewritePatternSet;
 
-namespace arm_sme {
-void populateVectorTransferLoweringPatterns(LLVMTypeConverter &converter,
-                                            RewritePatternSet &patterns);
-} // namespace arm_sme
-
 } // namespace mlir
 
 #endif // MLIR_DIALECT_ARMSME_TRANSFORMS_H
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td
index 6975b18ab7f81fddaaebaa4a3d2a08dfd0ccb26a..f36ec0d02cf70d0218f99454d8ed962adebb4d77 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMAttrDefs.td
@@ -424,9 +424,9 @@ def LLVM_DIGlobalVariableExpressionAttr
 def LLVM_DIGlobalVariable : LLVM_Attr<"DIGlobalVariable", "di_global_variable",
                                       /*traits=*/[], "DINodeAttr"> {
   let parameters = (ins
-    "DIScopeAttr":$scope,
-    "StringAttr":$name,
-    "StringAttr":$linkageName,
+    OptionalParameter<"DIScopeAttr">:$scope,
+    OptionalParameter<"StringAttr">:$name,
+    OptionalParameter<"StringAttr">:$linkageName,
     "DIFileAttr":$file,
     "unsigned":$line,
     "DITypeAttr":$type,
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
index f8862045f12b84c0770dfb9fa0aa788acd9474cf..a798cad603777689990bd2dec7d63fefd390a07c 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td
@@ -274,9 +274,9 @@ def LLVM_NoAliasScopeDeclOp
   let assemblyFormat = "$scope attr-dict";
 }
 
-//===----------------------------------------------------------------------===//
-// Lifetime Markers
-//===----------------------------------------------------------------------===//
+//
+// Memory marker intrinsics.
+//
 
 /// Base operation for lifetime markers. The LLVM intrinsics require the size
 /// operand to be an immediate. In MLIR it is encoded as an attribute.
@@ -291,6 +291,26 @@ class LLVM_LifetimeBaseOp<string opName> : LLVM_ZeroResultIntrOp<opName, [1],
 def LLVM_LifetimeStartOp : LLVM_LifetimeBaseOp<"lifetime.start">;
 def LLVM_LifetimeEndOp : LLVM_LifetimeBaseOp<"lifetime.end">;
 
+def LLVM_InvariantStartOp : LLVM_OneResultIntrOp<"invariant.start", [], [1],
+    [DeclareOpInterfaceMethods<PromotableOpInterface>],
+    /*requiresFastmath=*/0, /*immArgPositions=*/[0],
+    /*immArgAttrNames=*/["size"]> {
+  let arguments = (ins I64Attr:$size, LLVM_AnyPointer:$ptr);
+  let results = (outs LLVM_DefaultPointer:$res);
+  let assemblyFormat = "$size `,` $ptr attr-dict `:` qualified(type($ptr))";
+}
+
+def LLVM_InvariantEndOp : LLVM_ZeroResultIntrOp<"invariant.end", [2],
+    [DeclareOpInterfaceMethods<PromotableOpInterface>],
+    /*requiresAccessGroup=*/0, /*requiresAliasAnalysis=*/0,
+    /*immArgPositions=*/[1], /*immArgAttrNames=*/["size"]> {
+  let arguments = (ins LLVM_DefaultPointer:$start,
+                       I64Attr:$size,
+                       LLVM_AnyPointer:$ptr);
+  let assemblyFormat = "$start `,` $size `,` $ptr attr-dict `:` "
+      "qualified(type($ptr))";
+}
+
 // Intrinsics with multiple returns.
 
 class LLVM_ArithWithOverflowOp<string mnem>
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td
index 2ce8d8df05edc427b67f196a3ebdb24d0a179a26..b6aa73dad22970d09b970b65205eae992f71cded 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOpBase.td
@@ -67,6 +67,9 @@ class LLVM_PointerInAddressSpace<int addressSpace> : Type<
     # addressSpace # ")";
 }
 
+// Type constraint accepting an LLVM pointer type in address space 0.
+def LLVM_DefaultPointer : LLVM_PointerInAddressSpace<0>;
+
 // Type constraint accepting any LLVM structure type.
 def LLVM_AnyStruct : Type<CPred<"::llvm::isa<::mlir::LLVM::LLVMStructType>($_self)">,
                          "LLVM structure type">;
diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
index d7690b84807f6e308fa34da806abb561fa754cd0..9e65898154bd65ffe343ed255464ec240439d40a 100644
--- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
+++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td
@@ -1411,6 +1411,7 @@ def LLVM_LLVMFuncOp : LLVM_Op<"func", [
     DefaultValuedAttr<Visibility, "mlir::LLVM::Visibility::Default">:$visibility_,
     OptionalAttr<UnitAttr>:$arm_streaming,
     OptionalAttr<UnitAttr>:$arm_locally_streaming,
+    OptionalAttr<UnitAttr>:$arm_streaming_compatible,
     OptionalAttr<UnitAttr>:$arm_new_za,
     OptionalAttr<StrAttr>:$section,
     OptionalAttr<UnnamedAddr>:$unnamed_addr,
diff --git a/mlir/include/mlir/Dialect/Math/IR/MathOps.td b/mlir/include/mlir/Dialect/Math/IR/MathOps.td
index 9742d3d936dff5edd98d8660b2f1b80a08aca7eb..b9daa91b28a9bd09e9e4e48e5bb898d222bb4b7d 100644
--- a/mlir/include/mlir/Dialect/Math/IR/MathOps.td
+++ b/mlir/include/mlir/Dialect/Math/IR/MathOps.td
@@ -327,7 +327,26 @@ def Math_AcosOp : Math_FloatUnaryOp<"acos"> {
   let hasFolder = 1;
 }
 
+//===----------------------------------------------------------------------===//
+// CoshOp
+//===----------------------------------------------------------------------===//
 
+def Math_CoshOp : Math_FloatUnaryOp<"cosh"> {
+  let summary = "hyperbolic cosine of the specified value";
+  let description = [{
+    The `cosh` operation computes the hyperbolic cosine. It takes one operand
+    of floating point type (i.e., scalar, tensor or vector) and returns one
+    result of the same type. It has no standard attributes.
+
+    Example:
+
+    ```mlir
+    // Scalar hyperbolic cosine value.
+    %a = math.cosh %b : f64
+    ```
+  }];
+  let hasFolder = 1;
+}
 
 //===----------------------------------------------------------------------===//
 // SinOp
diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
index 8ff5380f71ad453c0aaf01961d0e69a75df4d6db..b9989b335a2aefb1d2f2348d1eca48a55887ea16 100644
--- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
+++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td
@@ -1370,6 +1370,56 @@ def Target_ExitDataOp: OpenMP_Op<"target_exit_data",
   let hasVerifier = 1;
 }
 
+//===---------------------------------------------------------------------===//
+// 2.14.6 target update data Construct
+//===---------------------------------------------------------------------===//
+
+def Target_UpdateDataOp: OpenMP_Op<"target_update_data",
+                                                 [AttrSizedOperandSegments]>{
+  let  summary = "target update data construct";
+  let description = [{
+    The target update directive makes the corresponding list items in the device
+    data environment consistent with their original list items, according to the
+    specified motion clauses. The target update construct is a stand-alone
+    directive.
+
+    The optional $if_expr parameter specifies a boolean result of a
+    conditional check. If this value is 1 or is not provided then the target
+    region runs on a device, if it is 0 then the target region is executed
+    on the host device.
+
+    The optional $device parameter specifies the device number for the
+    target region.
+
+    The optional $nowait eliminates the implicit barrier so the parent
+    task can make progress even if the target task is not yet completed.
+
+    We use `MapInfoOp` to model the motion clauses and their modifiers. Even
+    though the spec differentiates between map-types & map-type-modifiers vs.
+    motion-clauses & motion-modifiers, the motion clauses and their modifiers are
+    a subset of map types and their modifiers. The subset relation is handled in
+    during verification to make sure the restrictions for target update are
+    respected.
+
+    TODO: depend clause
+  }];
+
+  let arguments = (ins Optional<I1>:$if_expr,
+                       Optional<AnyInteger>:$device,
+                       UnitAttr:$nowait,
+                       Variadic<OpenMP_PointerLikeType>:$motion_operands);
+
+  let assemblyFormat = [{
+    oilist(`if` `(` $if_expr `:` type($if_expr) `)`
+    | `device` `(` $device `:` type($device) `)`
+    | `nowait` $nowait
+    | `motion_entries` `(` $motion_operands `:` type($motion_operands) `)`
+    ) attr-dict
+   }];
+
+  let hasVerifier = 1;
+}
+
 //===----------------------------------------------------------------------===//
 // 2.14.5 target construct
 //===----------------------------------------------------------------------===//
diff --git a/mlir/include/mlir/Dialect/Tensor/IR/TensorOps.td b/mlir/include/mlir/Dialect/Tensor/IR/TensorOps.td
index f50e3464867be50df619f12af934bf685cc6f10f..6d3371e9071d05b16c77743a323e4b8b266e748f 100644
--- a/mlir/include/mlir/Dialect/Tensor/IR/TensorOps.td
+++ b/mlir/include/mlir/Dialect/Tensor/IR/TensorOps.td
@@ -1822,13 +1822,10 @@ def Tensor_PackOp : Tensor_RelayoutOp<"pack", [
     and optionally transposes the tiled source tensor dimensions.
 
     `inner_dims_pos` (mandatory) specifies `k` source tensor dimensions that are
-    being tiled, where `0 < k <= n`. The order of the dimensions matters: the
-    tiled dimensions (of size `inner_tiles`) are added to the end of the result
+    being tiled, where `0 < k <= n`. The order of the dimensions matters: 
+     - The tiled dimensions (of size `inner_tiles`) are added to the end of the result
     tensor in the order in which they appear in `inner_dims_pos`.
-
-    `inner_dims_pos` (mandatory) specifies `k` source tensor dimensions that are
-    being tiled, where `0 < k <= n`. The order of the dimensions matters:
-    `inner_dims_pos[i]` specifies the source tensor dimension tiled by
+     - `inner_dims_pos[i]` specifies the source tensor dimension tiled by
     `inner_tiles[i]`.
 
     `inner_tiles` (mandatory) specifies `k` tile sizes. These tile sizes
diff --git a/mlir/include/mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h b/mlir/include/mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h
index 08c08172d0531e4a6237d2fa8f0e42aacbedb291..17173c01ab762ad179ac36e0529053c141f33a80 100644
--- a/mlir/include/mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h
+++ b/mlir/include/mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h
@@ -294,6 +294,15 @@ void populateCastAwayVectorLeadingOneDimPatterns(RewritePatternSet &patterns,
 void populateVectorTransferDropUnitDimsPatterns(RewritePatternSet &patterns,
                                                 PatternBenefit benefit = 1);
 
+/// Collect a set of patterns that use vector.shape_cast to help fold unit dims.
+///
+/// These patterns use vector.shape_cast to remove unit dims from e.g.
+/// arithmetic operations on Vectors. The newly inserted shape_casts will either
+/// cancel each other out or will be folded away when combined with other
+/// patterns.
+void populateDropUnitDimWithShapeCastPatterns(RewritePatternSet &patterns,
+                                              PatternBenefit benefit = 1);
+
 /// Collect a set of patterns to flatten n-D vector transfers on contiguous
 /// memref.
 ///
diff --git a/mlir/include/mlir/ExecutionEngine/RunnerUtils.h b/mlir/include/mlir/ExecutionEngine/RunnerUtils.h
index f998ed53b3403b42b763cda015519fcec3189f90..b426465c5192a7fd88fc583aa161671d576369b8 100644
--- a/mlir/include/mlir/ExecutionEngine/RunnerUtils.h
+++ b/mlir/include/mlir/ExecutionEngine/RunnerUtils.h
@@ -38,6 +38,7 @@
 #include <iostream>
 
 #include "mlir/ExecutionEngine/CRunnerUtils.h"
+#include "mlir/ExecutionEngine/Float16bits.h"
 
 template <typename T, typename StreamType>
 void printMemRefMetaData(StreamType &os, const DynamicMemRefType<T> &v) {
@@ -369,10 +370,16 @@ _mlir_ciface_printMemrefShapeC64(UnrankedMemRefType<impl::complex64> *m);
 extern "C" MLIR_RUNNERUTILS_EXPORT void
 _mlir_ciface_printMemrefI8(UnrankedMemRefType<int8_t> *m);
 extern "C" MLIR_RUNNERUTILS_EXPORT void
+_mlir_ciface_printMemrefI16(UnrankedMemRefType<int16_t> *m);
+extern "C" MLIR_RUNNERUTILS_EXPORT void
 _mlir_ciface_printMemrefI32(UnrankedMemRefType<int32_t> *m);
 extern "C" MLIR_RUNNERUTILS_EXPORT void
 _mlir_ciface_printMemrefI64(UnrankedMemRefType<int64_t> *m);
 extern "C" MLIR_RUNNERUTILS_EXPORT void
+_mlir_ciface_printMemrefF16(UnrankedMemRefType<f16> *m);
+extern "C" MLIR_RUNNERUTILS_EXPORT void
+_mlir_ciface_printMemrefBF16(UnrankedMemRefType<bf16> *m);
+extern "C" MLIR_RUNNERUTILS_EXPORT void
 _mlir_ciface_printMemrefF32(UnrankedMemRefType<float> *m);
 extern "C" MLIR_RUNNERUTILS_EXPORT void
 _mlir_ciface_printMemrefF64(UnrankedMemRefType<double> *m);
diff --git a/mlir/include/mlir/Transforms/FoldUtils.h b/mlir/include/mlir/Transforms/FoldUtils.h
index 28fa18cf942de4803b7e427dbe00b4cfd6b3192a..2600da361496cd02c7b0417df40c2becb5df9448 100644
--- a/mlir/include/mlir/Transforms/FoldUtils.h
+++ b/mlir/include/mlir/Transforms/FoldUtils.h
@@ -33,8 +33,7 @@ class Value;
 class OperationFolder {
 public:
   OperationFolder(MLIRContext *ctx, OpBuilder::Listener *listener = nullptr)
-      : fusedLocationTag(StringAttr::get(ctx, "CSE")), interfaces(ctx),
-        rewriter(ctx, listener) {}
+      : interfaces(ctx), rewriter(ctx, listener) {}
 
   /// Tries to perform folding on the given `op`, including unifying
   /// deduplicated constants. If successful, replaces `op`'s uses with
@@ -96,15 +95,6 @@ private:
                                     Dialect *dialect, Attribute value,
                                     Type type, Location loc);
 
-  // Fuse `foldedLocation` into the Location of `retainedOp`. This will result
-  // in `retainedOp` having a FusedLoc with `fusedLocationTag` to help trace the
-  // source of the fusion. If `retainedOp` already had a FusedLoc with the same
-  // tag, `foldedLocation` will simply be appended to it.
-  void appendFoldedLocation(Operation *retainedOp, Location foldedLocation);
-
-  /// Tag for annotating fused locations as a result of merging constants.
-  StringAttr fusedLocationTag;
-
   /// A mapping between an insertion region and the constants that have been
   /// created within it.
   DenseMap<Region *, ConstantMap> foldScopes;
diff --git a/mlir/lib/Analysis/Presburger/IntegerRelation.cpp b/mlir/lib/Analysis/Presburger/IntegerRelation.cpp
index 3724df5abcccaaa224c691be67012cc3dbca5fd0..0109384f1689ddcd8c26519c5ff3ad66753a82e2 100644
--- a/mlir/lib/Analysis/Presburger/IntegerRelation.cpp
+++ b/mlir/lib/Analysis/Presburger/IntegerRelation.cpp
@@ -449,6 +449,12 @@ void IntegerRelation::swapVar(unsigned posA, unsigned posB) {
   if (posA == posB)
     return;
 
+  VarKind kindA = space.getVarKindAt(posA);
+  VarKind kindB = space.getVarKindAt(posB);
+  unsigned relativePosA = posA - getVarKindOffset(kindA);
+  unsigned relativePosB = posB - getVarKindOffset(kindB);
+  space.swapVar(kindA, kindB, relativePosA, relativePosB);
+
   inequalities.swapColumns(posA, posB);
   equalities.swapColumns(posA, posB);
 }
diff --git a/mlir/lib/Analysis/Presburger/Matrix.cpp b/mlir/lib/Analysis/Presburger/Matrix.cpp
index fe461a1f95819f2085d65433b44a0b808d9859ec..1fcc6d072b44b7088974040e895091632894cd09 100644
--- a/mlir/lib/Analysis/Presburger/Matrix.cpp
+++ b/mlir/lib/Analysis/Presburger/Matrix.cpp
@@ -466,8 +466,8 @@ FracMatrix FracMatrix::identity(unsigned dimension) {
 
 FracMatrix::FracMatrix(IntMatrix m)
     : FracMatrix(m.getNumRows(), m.getNumColumns()) {
-  for (unsigned i = 0; i < m.getNumRows(); i++)
-    for (unsigned j = 0; j < m.getNumColumns(); j++)
+  for (unsigned i = 0, r = m.getNumRows(); i < r; i++)
+    for (unsigned j = 0, c = m.getNumColumns(); j < c; j++)
       this->at(i, j) = m.at(i, j);
 }
 
@@ -554,4 +554,26 @@ Fraction FracMatrix::determinant(FracMatrix *inverse) const {
     determinant *= m.at(i, i);
 
   return determinant;
+}
+
+FracMatrix FracMatrix::gramSchmidt() const {
+  // Create a copy of the argument to store
+  // the orthogonalised version.
+  FracMatrix orth(*this);
+
+  // For each vector (row) in the matrix, subtract its unit
+  // projection along each of the previous vectors.
+  // This ensures that it has no component in the direction
+  // of any of the previous vectors.
+  for (unsigned i = 1, e = getNumRows(); i < e; i++) {
+    for (unsigned j = 0; j < i; j++) {
+      Fraction jNormSquared = dotProduct(orth.getRow(j), orth.getRow(j));
+      assert(jNormSquared != 0 && "some row became zero! Inputs to this "
+                                  "function must be linearly independent.");
+      Fraction projectionScale =
+          dotProduct(orth.getRow(i), orth.getRow(j)) / jNormSquared;
+      orth.addToRow(j, i, -projectionScale);
+    }
+  }
+  return orth;
 }
\ No newline at end of file
diff --git a/mlir/lib/Analysis/Presburger/Utils.cpp b/mlir/lib/Analysis/Presburger/Utils.cpp
index 612712a1d9078f1835f7c09b47eb8eaad9730a90..5e267d2045bc1c93f4ef6c97026d215e6a4515ca 100644
--- a/mlir/lib/Analysis/Presburger/Utils.cpp
+++ b/mlir/lib/Analysis/Presburger/Utils.cpp
@@ -529,3 +529,12 @@ SmallVector<int64_t, 8> presburger::getInt64Vec(ArrayRef<MPInt> range) {
   std::transform(range.begin(), range.end(), result.begin(), int64FromMPInt);
   return result;
 }
+
+Fraction presburger::dotProduct(ArrayRef<Fraction> a, ArrayRef<Fraction> b) {
+  assert(a.size() == b.size() &&
+         "dot product is only valid for vectors of equal sizes!");
+  Fraction sum = 0;
+  for (unsigned i = 0, e = a.size(); i < e; i++)
+    sum += a[i] * b[i];
+  return sum;
+}
\ No newline at end of file
diff --git a/mlir/lib/AsmParser/AttributeParser.cpp b/mlir/lib/AsmParser/AttributeParser.cpp
index d085fb6af6bc1402c3537ff8aa662ebfc0c995d4..efa65e49abc33b996e43754b938a49f4f345a437 100644
--- a/mlir/lib/AsmParser/AttributeParser.cpp
+++ b/mlir/lib/AsmParser/AttributeParser.cpp
@@ -660,7 +660,7 @@ TensorLiteralParser::getFloatAttrElements(SMLoc loc, FloatType eltTy,
     const Token &token = signAndToken.second;
 
     // Handle hexadecimal float literals.
-    if (token.is(Token::integer) && token.getSpelling().startswith("0x")) {
+    if (token.is(Token::integer) && token.getSpelling().starts_with("0x")) {
       std::optional<APFloat> result;
       if (failed(p.parseFloatFromIntegerLiteral(result, token, isNegative,
                                                 eltTy.getFloatSemantics(),
diff --git a/mlir/lib/AsmParser/Parser.cpp b/mlir/lib/AsmParser/Parser.cpp
index 1fca85beb87fcea83aef1fe9e1edc87ff57211a4..3aa9adcbe1c5dfaaf3d1bced3f9e9add1d0e41eb 100644
--- a/mlir/lib/AsmParser/Parser.cpp
+++ b/mlir/lib/AsmParser/Parser.cpp
@@ -2477,7 +2477,7 @@ public:
   AsmResourceEntryKind getKind() const final {
     if (value.isAny(Token::kw_true, Token::kw_false))
       return AsmResourceEntryKind::Bool;
-    return value.getSpelling().startswith("\"0x")
+    return value.getSpelling().starts_with("\"0x")
                ? AsmResourceEntryKind::Blob
                : AsmResourceEntryKind::String;
   }
diff --git a/mlir/lib/Bytecode/Reader/BytecodeReader.cpp b/mlir/lib/Bytecode/Reader/BytecodeReader.cpp
index 0bc2a2f676ba4bc9de71869a8dd4225b3902abc7..7cf3bd83b925ca8e4c52aa012b3393c96ec3efd0 100644
--- a/mlir/lib/Bytecode/Reader/BytecodeReader.cpp
+++ b/mlir/lib/Bytecode/Reader/BytecodeReader.cpp
@@ -2564,7 +2564,7 @@ BytecodeReader::finalize(function_ref<bool(Operation *)> shouldMaterialize) {
 }
 
 bool mlir::isBytecode(llvm::MemoryBufferRef buffer) {
-  return buffer.getBuffer().startswith("ML\xefR");
+  return buffer.getBuffer().starts_with("ML\xefR");
 }
 
 /// Read the bytecode from the provided memory buffer reference.
diff --git a/mlir/lib/Conversion/MathToLibm/MathToLibm.cpp b/mlir/lib/Conversion/MathToLibm/MathToLibm.cpp
index 27c2cb935207142ccac25cac25d2df3f4ee8cbfc..6e30c07de4d57e08571ffdeb4cce6bd88a575971 100644
--- a/mlir/lib/Conversion/MathToLibm/MathToLibm.cpp
+++ b/mlir/lib/Conversion/MathToLibm/MathToLibm.cpp
@@ -168,6 +168,7 @@ void mlir::populateMathToLibmConversionPatterns(RewritePatternSet &patterns) {
   populatePatternsForOp<math::CbrtOp>(patterns, ctx, "cbrtf", "cbrt");
   populatePatternsForOp<math::CeilOp>(patterns, ctx, "ceilf", "ceil");
   populatePatternsForOp<math::CosOp>(patterns, ctx, "cosf", "cos");
+  populatePatternsForOp<math::CoshOp>(patterns, ctx, "coshf", "cosh");
   populatePatternsForOp<math::ErfOp>(patterns, ctx, "erff", "erf");
   populatePatternsForOp<math::ExpM1Op>(patterns, ctx, "expm1f", "expm1");
   populatePatternsForOp<math::FloorOp>(patterns, ctx, "floorf", "floor");
diff --git a/mlir/lib/Conversion/SCFToOpenMP/SCFToOpenMP.cpp b/mlir/lib/Conversion/SCFToOpenMP/SCFToOpenMP.cpp
index ee3ee02cf535e1348c066e34243b5ac5b3095fd2..67033ba812946f4d19631a07ac206fe8f6fb33ee 100644
--- a/mlir/lib/Conversion/SCFToOpenMP/SCFToOpenMP.cpp
+++ b/mlir/lib/Conversion/SCFToOpenMP/SCFToOpenMP.cpp
@@ -339,9 +339,12 @@ static omp::ReductionDeclareOp declareReduction(PatternRewriter &builder,
 namespace {
 
 struct ParallelOpLowering : public OpRewritePattern<scf::ParallelOp> {
+  static constexpr unsigned kUseOpenMPDefaultNumThreads = 0;
+  unsigned numThreads;
 
-  ParallelOpLowering(MLIRContext *context)
-      : OpRewritePattern<scf::ParallelOp>(context) {}
+  ParallelOpLowering(MLIRContext *context,
+                     unsigned numThreads = kUseOpenMPDefaultNumThreads)
+      : OpRewritePattern<scf::ParallelOp>(context), numThreads(numThreads) {}
 
   LogicalResult matchAndRewrite(scf::ParallelOp parallelOp,
                                 PatternRewriter &rewriter) const override {
@@ -388,8 +391,21 @@ struct ParallelOpLowering : public OpRewritePattern<scf::ParallelOp> {
           reduceOp, reduceOp.getOperand(), std::get<1>(pair));
     }
 
+    Value numThreadsVar;
+    if (numThreads > 0) {
+      numThreadsVar = rewriter.create<LLVM::ConstantOp>(
+          loc, rewriter.getI32IntegerAttr(numThreads));
+    }
     // Create the parallel wrapper.
-    auto ompParallel = rewriter.create<omp::ParallelOp>(loc);
+    auto ompParallel = rewriter.create<omp::ParallelOp>(
+        loc,
+        /* if_expr_var = */ Value{},
+        /* num_threads_var = */ numThreadsVar,
+        /* allocate_vars = */ llvm::SmallVector<Value>{},
+        /* allocators_vars = */ llvm::SmallVector<Value>{},
+        /* reduction_vars = */ llvm::SmallVector<Value>{},
+        /* reductions = */ ArrayAttr{},
+        /* proc_bind_val = */ omp::ClauseProcBindKindAttr{});
     {
 
       OpBuilder::InsertionGuard guard(rewriter);
@@ -443,14 +459,14 @@ struct ParallelOpLowering : public OpRewritePattern<scf::ParallelOp> {
 };
 
 /// Applies the conversion patterns in the given function.
-static LogicalResult applyPatterns(ModuleOp module) {
+static LogicalResult applyPatterns(ModuleOp module, unsigned numThreads) {
   ConversionTarget target(*module.getContext());
   target.addIllegalOp<scf::ReduceOp, scf::ReduceReturnOp, scf::ParallelOp>();
   target.addLegalDialect<omp::OpenMPDialect, LLVM::LLVMDialect,
                          memref::MemRefDialect>();
 
   RewritePatternSet patterns(module.getContext());
-  patterns.add<ParallelOpLowering>(module.getContext());
+  patterns.add<ParallelOpLowering>(module.getContext(), numThreads);
   FrozenRewritePatternSet frozen(std::move(patterns));
   return applyPartialConversion(module, target, frozen);
 }
@@ -463,7 +479,7 @@ struct SCFToOpenMPPass
 
   /// Pass entry point.
   void runOnOperation() override {
-    if (failed(applyPatterns(getOperation())))
+    if (failed(applyPatterns(getOperation(), numThreads)))
       signalPassFailure();
   }
 };
diff --git a/mlir/lib/Dialect/AMDGPU/Utils/Chipset.cpp b/mlir/lib/Dialect/AMDGPU/Utils/Chipset.cpp
index 4f075c71c947d1633e923bbc445d93142ebfe662..2540e1fb86d87daab9f18ebe5c771e2e7a6a3c4d 100644
--- a/mlir/lib/Dialect/AMDGPU/Utils/Chipset.cpp
+++ b/mlir/lib/Dialect/AMDGPU/Utils/Chipset.cpp
@@ -14,7 +14,7 @@ using namespace mlir;
 using namespace mlir::amdgpu;
 
 FailureOr<Chipset> Chipset::parse(StringRef name) {
-  if (!name.startswith("gfx"))
+  if (!name.starts_with("gfx"))
     return failure();
   unsigned major = 0;
   unsigned minor = 0;
diff --git a/mlir/lib/Dialect/Affine/IR/AffineOps.cpp b/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
index 7f2f3c3410c33b4bdc3420343c19178c119cea68..d5be2e906989fa7b48b785429a4414554fa697cb 100644
--- a/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
+++ b/mlir/lib/Dialect/Affine/IR/AffineOps.cpp
@@ -354,8 +354,19 @@ static bool isDimOpValidSymbol(ShapedDimOpInterface dimOp, Region *region) {
   if (!index.has_value())
     return false;
 
+  // Skip over all memref.cast ops (if any).
+  Operation *op = dimOp.getShapedValue().getDefiningOp();
+  while (auto castOp = dyn_cast<memref::CastOp>(op)) {
+    // Bail on unranked memrefs.
+    if (isa<UnrankedMemRefType>(castOp.getSource().getType()))
+      return false;
+    op = castOp.getSource().getDefiningOp();
+    if (!op)
+      return false;
+  }
+
   int64_t i = index.value();
-  return TypeSwitch<Operation *, bool>(dimOp.getShapedValue().getDefiningOp())
+  return TypeSwitch<Operation *, bool>(op)
       .Case<memref::ViewOp, memref::SubViewOp, memref::AllocOp>(
           [&](auto op) { return isMemRefSizeValidSymbol(op, i, region); })
       .Default([](Operation *) { return false; });
diff --git a/mlir/lib/Dialect/GPU/Transforms/SerializeToHsaco.cpp b/mlir/lib/Dialect/GPU/Transforms/SerializeToHsaco.cpp
index f0c294c22695b53e0f7544d638621f58e3905998..5cce7befce5283b419b19e1982045c2849679ae6 100644
--- a/mlir/lib/Dialect/GPU/Transforms/SerializeToHsaco.cpp
+++ b/mlir/lib/Dialect/GPU/Transforms/SerializeToHsaco.cpp
@@ -201,9 +201,9 @@ SerializeToHsacoPass::translateToLLVMIR(llvm::LLVMContext &llvmContext) {
       StringRef funcName = f.getName();
       if ("printf" == funcName)
         needOpenCl = true;
-      if (funcName.startswith("__ockl_"))
+      if (funcName.starts_with("__ockl_"))
         needOckl = true;
-      if (funcName.startswith("__ocml_"))
+      if (funcName.starts_with("__ocml_"))
         needOcml = true;
     }
   }
@@ -253,7 +253,7 @@ SerializeToHsacoPass::translateToLLVMIR(llvm::LLVMContext &llvmContext) {
   if (needOcml || needOckl) {
     addControlConstant("__oclc_wavefrontsize64", 1, 8);
     StringRef chipSet = this->chip.getValue();
-    if (chipSet.startswith("gfx"))
+    if (chipSet.starts_with("gfx"))
       chipSet = chipSet.substr(3);
     uint32_t minor =
         llvm::APInt(32, chipSet.substr(chipSet.size() - 2), 16).getZExtValue();
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
index 53e1088f620d7eb46aea097113ccd62f55814556..458bf83eac17f8f34596f3d56ac601c3db6fb84b 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
@@ -2916,7 +2916,7 @@ OpFoldResult LLVM::OrOp::fold(FoldAdaptor adaptor) {
 //===----------------------------------------------------------------------===//
 
 LogicalResult CallIntrinsicOp::verify() {
-  if (!getIntrin().startswith("llvm."))
+  if (!getIntrin().starts_with("llvm."))
     return emitOpError() << "intrinsic name must start with 'llvm.'";
   return success();
 }
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMInlining.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMInlining.cpp
index d9f52d7692938f04879711f131ca5305b61392dc..6e9019f932aa8cced9010d99fab73a603fc81a02 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMInlining.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMInlining.cpp
@@ -632,8 +632,7 @@ struct LLVMInlinerInterface : public DialectInlinerInterface {
                        bool wouldBeCloned) const final {
     if (!wouldBeCloned)
       return false;
-    auto callOp = dyn_cast<LLVM::CallOp>(call);
-    if (!callOp) {
+    if (!isa<LLVM::CallOp>(call)) {
       LLVM_DEBUG(llvm::dbgs()
                  << "Cannot inline: call is not an LLVM::CallOp\n");
       return false;
@@ -684,40 +683,8 @@ struct LLVMInlinerInterface : public DialectInlinerInterface {
     return true;
   }
 
-  /// Conservative allowlist of operations supported so far.
   bool isLegalToInline(Operation *op, Region *, bool, IRMapping &) const final {
-    if (isPure(op))
-      return true;
-    // clang-format off
-    if (isa<LLVM::AllocaOp,
-            LLVM::AssumeOp,
-            LLVM::AtomicRMWOp,
-            LLVM::AtomicCmpXchgOp,
-            LLVM::CallOp,
-            LLVM::CallIntrinsicOp,
-            LLVM::DbgDeclareOp,
-            LLVM::DbgLabelOp,
-            LLVM::DbgValueOp,
-            LLVM::FenceOp,
-            LLVM::InlineAsmOp,
-            LLVM::LifetimeEndOp,
-            LLVM::LifetimeStartOp,
-            LLVM::LoadOp,
-            LLVM::MemcpyOp,
-            LLVM::MemcpyInlineOp,
-            LLVM::MemmoveOp,
-            LLVM::MemsetOp,
-            LLVM::NoAliasScopeDeclOp,
-            LLVM::StackRestoreOp,
-            LLVM::StackSaveOp,
-            LLVM::StoreOp,
-            LLVM::UnreachableOp>(op))
-      return true;
-    // clang-format on
-    LLVM_DEBUG(llvm::dbgs()
-               << "Cannot inline: unhandled side effecting operation \""
-               << op->getName() << "\"\n");
-    return false;
+    return true;
   }
 
   /// Handle the given inlined return by replacing it with a branch. This
diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp
index 43349a26ad11276b47bde78ffa5f4e48a8721460..96a0ef591c1cfe69b1dcc9a9c4ab8ebbc4a71526 100644
--- a/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp
+++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp
@@ -248,6 +248,28 @@ DeletionKind LLVM::LifetimeEndOp::removeBlockingUses(
   return DeletionKind::Delete;
 }
 
+bool LLVM::InvariantStartOp::canUsesBeRemoved(
+    const SmallPtrSetImpl<OpOperand *> &blockingUses,
+    SmallVectorImpl<OpOperand *> &newBlockingUses) {
+  return true;
+}
+
+DeletionKind LLVM::InvariantStartOp::removeBlockingUses(
+    const SmallPtrSetImpl<OpOperand *> &blockingUses, RewriterBase &rewriter) {
+  return DeletionKind::Delete;
+}
+
+bool LLVM::InvariantEndOp::canUsesBeRemoved(
+    const SmallPtrSetImpl<OpOperand *> &blockingUses,
+    SmallVectorImpl<OpOperand *> &newBlockingUses) {
+  return true;
+}
+
+DeletionKind LLVM::InvariantEndOp::removeBlockingUses(
+    const SmallPtrSetImpl<OpOperand *> &blockingUses, RewriterBase &rewriter) {
+  return DeletionKind::Delete;
+}
+
 bool LLVM::DbgDeclareOp::canUsesBeRemoved(
     const SmallPtrSetImpl<OpOperand *> &blockingUses,
     SmallVectorImpl<OpOperand *> &newBlockingUses) {
diff --git a/mlir/lib/Dialect/Math/IR/MathOps.cpp b/mlir/lib/Dialect/Math/IR/MathOps.cpp
index 066a21c76f7d1c67537253a29223d62e67e583f0..6b8c3a53a422fac750ab9a5b3a5798ae487b0ee8 100644
--- a/mlir/lib/Dialect/Math/IR/MathOps.cpp
+++ b/mlir/lib/Dialect/Math/IR/MathOps.cpp
@@ -144,6 +144,24 @@ OpFoldResult math::CosOp::fold(FoldAdaptor adaptor) {
       });
 }
 
+//===----------------------------------------------------------------------===//
+// CoshOp folder
+//===----------------------------------------------------------------------===//
+
+OpFoldResult math::CoshOp::fold(FoldAdaptor adaptor) {
+  return constFoldUnaryOpConditional<FloatAttr>(
+      adaptor.getOperands(), [](const APFloat &a) -> std::optional<APFloat> {
+        switch (a.getSizeInBits(a.getSemantics())) {
+        case 64:
+          return APFloat(cosh(a.convertToDouble()));
+        case 32:
+          return APFloat(coshf(a.convertToFloat()));
+        default:
+          return {};
+        }
+      });
+}
+
 //===----------------------------------------------------------------------===//
 // SinOp folder
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
index 20df0099cbd24d113765ba5b7d5a33ddd1c5bfb2..6e69cd0d386bd249ba058c4f045eb6869ecd40f6 100644
--- a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
+++ b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp
@@ -882,21 +882,22 @@ static ParseResult parseCaptureType(OpAsmParser &parser,
 }
 
 static LogicalResult verifyMapClause(Operation *op, OperandRange mapOperands) {
+  llvm::DenseSet<mlir::TypedValue<mlir::omp::PointerLikeType>> updateToVars;
+  llvm::DenseSet<mlir::TypedValue<mlir::omp::PointerLikeType>> updateFromVars;
 
   for (auto mapOp : mapOperands) {
     if (!mapOp.getDefiningOp())
       emitError(op->getLoc(), "missing map operation");
 
-    if (auto MapInfoOp =
+    if (auto mapInfoOp =
             mlir::dyn_cast<mlir::omp::MapInfoOp>(mapOp.getDefiningOp())) {
-
-      if (!MapInfoOp.getMapType().has_value())
+      if (!mapInfoOp.getMapType().has_value())
         emitError(op->getLoc(), "missing map type for map operand");
 
-      if (!MapInfoOp.getMapCaptureType().has_value())
+      if (!mapInfoOp.getMapCaptureType().has_value())
         emitError(op->getLoc(), "missing map capture type for map operand");
 
-      uint64_t mapTypeBits = MapInfoOp.getMapType().value();
+      uint64_t mapTypeBits = mapInfoOp.getMapType().value();
 
       bool to = mapTypeToBitFlag(
           mapTypeBits, llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_TO);
@@ -905,6 +906,13 @@ static LogicalResult verifyMapClause(Operation *op, OperandRange mapOperands) {
       bool del = mapTypeToBitFlag(
           mapTypeBits, llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_DELETE);
 
+      bool always = mapTypeToBitFlag(
+          mapTypeBits, llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_ALWAYS);
+      bool close = mapTypeToBitFlag(
+          mapTypeBits, llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_CLOSE);
+      bool implicit = mapTypeToBitFlag(
+          mapTypeBits, llvm::omp::OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT);
+
       if ((isa<DataOp>(op) || isa<TargetOp>(op)) && del)
         return emitError(op->getLoc(),
                          "to, from, tofrom and alloc map types are permitted");
@@ -915,6 +923,37 @@ static LogicalResult verifyMapClause(Operation *op, OperandRange mapOperands) {
       if (isa<ExitDataOp>(op) && to)
         return emitError(op->getLoc(),
                          "from, release and delete map types are permitted");
+
+      if (isa<UpdateDataOp>(op)) {
+        if (del) {
+          return emitError(op->getLoc(),
+                           "at least one of to or from map types must be "
+                           "specified, other map types are not permitted");
+        }
+
+        if (!to && !from) {
+          return emitError(op->getLoc(),
+                           "at least one of to or from map types must be "
+                           "specified, other map types are not permitted");
+        }
+
+        auto updateVar = mapInfoOp.getVarPtr();
+
+        if ((to && from) || (to && updateFromVars.contains(updateVar)) ||
+            (from && updateToVars.contains(updateVar))) {
+          return emitError(
+              op->getLoc(),
+              "either to or from map types can be specified, not both");
+        }
+
+        if (always || close || implicit) {
+          return emitError(
+              op->getLoc(),
+              "present, mapper and iterator map type modifiers are permitted");
+        }
+
+        to ? updateToVars.insert(updateVar) : updateFromVars.insert(updateVar);
+      }
     } else {
       emitError(op->getLoc(), "map argument is not a map entry operation");
     }
@@ -940,6 +979,10 @@ LogicalResult ExitDataOp::verify() {
   return verifyMapClause(*this, getMapOperands());
 }
 
+LogicalResult UpdateDataOp::verify() {
+  return verifyMapClause(*this, getMotionOperands());
+}
+
 LogicalResult TargetOp::verify() {
   return verifyMapClause(*this, getMapOperands());
 }
diff --git a/mlir/lib/Dialect/SCF/Transforms/LoopPipelining.cpp b/mlir/lib/Dialect/SCF/Transforms/LoopPipelining.cpp
index 6c36600975a5975a85a989f706e509c2bd558685..7d45b484f76575295604dcd0421a7d34acd99ce1 100644
--- a/mlir/lib/Dialect/SCF/Transforms/LoopPipelining.cpp
+++ b/mlir/lib/Dialect/SCF/Transforms/LoopPipelining.cpp
@@ -90,7 +90,8 @@ public:
       RewriterBase &rewriter);
   /// Emits the epilogue, this creates `maxStage - 1` part which will contain
   /// operations from stages [i; maxStage], where i is the part index.
-  llvm::SmallVector<Value> emitEpilogue(RewriterBase &rewriter);
+  void emitEpilogue(RewriterBase &rewriter,
+                    llvm::SmallVector<Value> &returnValues);
 };
 
 bool LoopPipelinerInternal::initializeLoopInfo(
@@ -175,15 +176,18 @@ bool LoopPipelinerInternal::initializeLoopInfo(
     }
   }
 
-  // Only support loop carried dependency with a distance of 1. This means the
-  // source of all the scf.yield operands needs to be defined by operations in
-  // the loop.
+  // Support only loop-carried dependencies with a distance of one iteration or
+  // those defined outside of the loop. This means that any dependency within a
+  // loop should either be on the immediately preceding iteration, the current
+  // iteration, or on variables whose values are set before entering the loop.
   if (llvm::any_of(forOp.getBody()->getTerminator()->getOperands(),
                    [this](Value operand) {
                      Operation *def = operand.getDefiningOp();
-                     return !def || !stages.contains(def);
+                     return !def ||
+                            (!stages.contains(def) && forOp->isAncestor(def));
                    })) {
-    LDBG("--only support loop carried dependency with a distance of 1 -> BAIL");
+    LDBG("--only support loop carried dependency with a distance of 1 or "
+         "defined outside of the loop -> BAIL");
     return false;
   }
   annotateFn = options.annotateFn;
@@ -341,12 +345,17 @@ scf::ForOp LoopPipelinerInternal::createKernelLoop(
   for (const auto &retVal :
        llvm::enumerate(forOp.getBody()->getTerminator()->getOperands())) {
     Operation *def = retVal.value().getDefiningOp();
-    assert(def && "Only support loop carried dependencies of distance 1");
-    unsigned defStage = stages[def];
-    Value valueVersion = valueMapping[forOp.getRegionIterArgs()[retVal.index()]]
-                                     [maxStage - defStage];
-    assert(valueVersion);
-    newLoopArg.push_back(valueVersion);
+    assert(def && "Only support loop carried dependencies of distance of 1 or "
+                  "outside the loop");
+    auto defStage = stages.find(def);
+    if (defStage != stages.end()) {
+      Value valueVersion =
+          valueMapping[forOp.getRegionIterArgs()[retVal.index()]]
+                      [maxStage - defStage->second];
+      assert(valueVersion);
+      newLoopArg.push_back(valueVersion);
+    } else
+      newLoopArg.push_back(forOp.getInitArgs()[retVal.index()]);
   }
   for (auto escape : crossStageValues) {
     LiverangeInfo &info = escape.second;
@@ -551,21 +560,25 @@ LogicalResult LoopPipelinerInternal::createKernel(
   for (const auto &retVal :
        llvm::enumerate(forOp.getBody()->getTerminator()->getOperands())) {
     Operation *def = retVal.value().getDefiningOp();
-    assert(def && "Only support loop carried dependencies of distance 1");
-    unsigned defStage = stages[def];
-    if (defStage > 0) {
+    assert(def && "Only support loop carried dependencies of distance of 1 or "
+                  "defined outside the loop");
+    auto defStage = stages.find(def);
+    if (defStage == stages.end()) {
+      for (unsigned int stage = 1; stage <= maxStage; stage++)
+        setValueMapping(forOp.getRegionIterArgs()[retVal.index()],
+                        retVal.value(), stage);
+    } else if (defStage->second > 0) {
       setValueMapping(forOp.getRegionIterArgs()[retVal.index()],
                       newForOp->getResult(retVal.index()),
-                      maxStage - defStage + 1);
+                      maxStage - defStage->second + 1);
     }
   }
   rewriter.create<scf::YieldOp>(forOp.getLoc(), yieldOperands);
   return success();
 }
 
-llvm::SmallVector<Value>
-LoopPipelinerInternal::emitEpilogue(RewriterBase &rewriter) {
-  llvm::SmallVector<Value> returnValues(forOp->getNumResults());
+void LoopPipelinerInternal::emitEpilogue(
+    RewriterBase &rewriter, llvm::SmallVector<Value> &returnValues) {
   // Emit different versions of the induction variable. They will be
   // removed by dead code if not used.
   for (int64_t i = 0; i < maxStage; i++) {
@@ -628,7 +641,6 @@ LoopPipelinerInternal::emitEpilogue(RewriterBase &rewriter) {
       }
     }
   }
-  return returnValues;
 }
 
 void LoopPipelinerInternal::setValueMapping(Value key, Value el, int64_t idx) {
@@ -685,7 +697,7 @@ FailureOr<ForOp> mlir::scf::pipelineForLoop(RewriterBase &rewriter, ForOp forOp,
   if (options.peelEpilogue) {
     // 4. Emit the epilogue after the new forOp.
     rewriter.setInsertionPointAfter(newForOp);
-    returnValues = pipeliner.emitEpilogue(rewriter);
+    pipeliner.emitEpilogue(rewriter, returnValues);
   }
   // 5. Erase the original loop and replace the uses with the epilogue output.
   if (forOp->getNumResults() > 0)
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/CMakeLists.txt b/mlir/lib/Dialect/SparseTensor/Transforms/CMakeLists.txt
index 8459e46e5814f06eee6dfce6416fd53d4ac68066..ad8b0d02eca35e90296925663e1f974fcd66e62b 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/CMakeLists.txt
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/CMakeLists.txt
@@ -1,9 +1,6 @@
 add_mlir_dialect_library(MLIRSparseTensorTransforms
+  # Rewriting.
   BufferizableOpInterfaceImpl.cpp
-  CodegenEnv.cpp
-  CodegenUtils.cpp
-  IterationGraphSorter.cpp
-  LoopEmitter.cpp
   SparseBufferRewriting.cpp
   SparseGPUCodegen.cpp
   SparseReinterpretMap.cpp
@@ -12,11 +9,16 @@ add_mlir_dialect_library(MLIRSparseTensorTransforms
   SparseTensorConversion.cpp
   SparseTensorPasses.cpp
   SparseTensorRewriting.cpp
-  SparseTensorDescriptor.cpp
   SparseVectorization.cpp
   Sparsification.cpp
   SparsificationAndBufferizationPass.cpp
   StageSparseOperations.cpp
+  # Utilities.
+  Utils/CodegenEnv.cpp
+  Utils/CodegenUtils.cpp
+  Utils/IterationGraphSorter.cpp
+  Utils/LoopEmitter.cpp
+  Utils/SparseTensorDescriptor.cpp
 
   ADDITIONAL_HEADER_DIRS
   ${MLIR_MAIN_INCLUDE_DIR}/mlir/Dialect/SparseTensor
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseBufferRewriting.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseBufferRewriting.cpp
index cdbf4f048a00f38ebcc36f9807b4e2364efd69df..248e9413b6e0dc1109dbd34374e746cffdf5274a 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseBufferRewriting.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseBufferRewriting.cpp
@@ -11,7 +11,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "CodegenUtils.h"
+#include "Utils/CodegenUtils.h"
 
 #include "mlir/Dialect/Arith/IR/Arith.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseGPUCodegen.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseGPUCodegen.cpp
index 5155cab772d4403eaf96be3d892e4af9aa258440..477ff2e1c9237e5db5cb44cc734a5bd14f3325f5 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseGPUCodegen.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseGPUCodegen.cpp
@@ -13,8 +13,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "CodegenUtils.h"
-#include "LoopEmitter.h"
+#include "Utils/CodegenUtils.h"
+#include "Utils/LoopEmitter.h"
 
 #include "mlir/Dialect/Bufferization/IR/Bufferization.h"
 #include "mlir/Dialect/GPU/IR/GPUDialect.h"
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseReinterpretMap.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseReinterpretMap.cpp
index 488079cfe4e32ddb0a86f42ecd4a0223db43f77e..f2e1b0bc58f132f96b78f3557df4cdff8a616ac8 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseReinterpretMap.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseReinterpretMap.cpp
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "CodegenUtils.h"
-#include "IterationGraphSorter.h"
+#include "Utils/CodegenUtils.h"
+#include "Utils/IterationGraphSorter.h"
 
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
 #include "mlir/Dialect/Bufferization/IR/Bufferization.h"
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseStorageSpecifierToLLVM.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseStorageSpecifierToLLVM.cpp
index a6f4dd3c2f71854bfb7f98a119fb6e5dabb59afa..91dad050b28852c2df08873d881d3607bf5c513f 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseStorageSpecifierToLLVM.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseStorageSpecifierToLLVM.cpp
@@ -6,7 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "CodegenUtils.h"
+#include "Utils/CodegenUtils.h"
 
 #include "mlir/Conversion/LLVMCommon/StructBuilder.h"
 #include "mlir/Dialect/SparseTensor/IR/SparseTensorStorageLayout.h"
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp
index 18b2bb0819e2642de7607e22fdb461bcd8bfd0bf..491501a3381b9c5dbecf56f49a3d309cdc2c2f13 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorCodegen.cpp
@@ -15,8 +15,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "CodegenUtils.h"
-#include "SparseTensorDescriptor.h"
+#include "Utils/CodegenUtils.h"
+#include "Utils/SparseTensorDescriptor.h"
 
 #include "mlir/Dialect/Arith/Utils/Utils.h"
 #include "mlir/Dialect/Bufferization/IR/Bufferization.h"
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp
index e6052f2ca894c2ddbf34f01fc9e1abdc6be7f02d..b0447b2436619e290cecaf8d95f59c144749becf 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorConversion.cpp
@@ -16,7 +16,7 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "CodegenUtils.h"
+#include "Utils/CodegenUtils.h"
 
 #include "mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h"
 #include "mlir/Dialect/Bufferization/IR/Bufferization.h"
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp
index 4fc692f2fe9ddc2406388d1c408cc09b60d72fbb..3b9685b8ae1e079d2e6f525de6bb09c6e8c7beec 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorRewriting.cpp
@@ -10,8 +10,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "CodegenUtils.h"
-#include "LoopEmitter.h"
+#include "Utils/CodegenUtils.h"
+#include "Utils/LoopEmitter.h"
 
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
 #include "mlir/Dialect/Arith/IR/Arith.h"
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseVectorization.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseVectorization.cpp
index 561c4e2511460239cf169ba30a17fbbea57e23cc..7710a44a7ca05285e78f630e8bf86b219a31408b 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseVectorization.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseVectorization.cpp
@@ -16,8 +16,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "CodegenUtils.h"
-#include "LoopEmitter.h"
+#include "Utils/CodegenUtils.h"
+#include "Utils/LoopEmitter.h"
 
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
 #include "mlir/Dialect/Arith/IR/Arith.h"
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Sparsification.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/Sparsification.cpp
index 992be434fc62314ff9c88930032e3ce6106f2701..934e1e559f44d6ad55be82adc6af5c91e68f97c3 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/Sparsification.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/Sparsification.cpp
@@ -10,9 +10,9 @@
 //
 //===----------------------------------------------------------------------===//
 
-#include "CodegenEnv.h"
-#include "CodegenUtils.h"
-#include "LoopEmitter.h"
+#include "Utils/CodegenEnv.h"
+#include "Utils/CodegenUtils.h"
+#include "Utils/LoopEmitter.h"
 
 #include "mlir/Dialect/Affine/IR/AffineOps.h"
 #include "mlir/Dialect/Arith/IR/Arith.h"
@@ -415,9 +415,7 @@ static void genInsertionStore(CodegenEnv &env, OpBuilder &builder, OpOperand *t,
     SmallVector<Value> ivs = llvm::to_vector(llvm::drop_end(
         env.emitter().getLoopIVsRange(), env.getCurrentDepth() - numLoops));
     Value chain = env.getInsertionChain();
-    if (!env.getValidLexInsert()) {
-      env.updateInsertionChain(builder.create<InsertOp>(loc, rhs, chain, ivs));
-    } else {
+    if (env.isValidLexInsert()) {
       // Generates runtime check for a valid lex during reduction,
       // to avoid inserting the identity value for empty reductions.
       //   if (validLexInsert) then
@@ -438,6 +436,9 @@ static void genInsertionStore(CodegenEnv &env, OpBuilder &builder, OpOperand *t,
       // Value assignment.
       builder.setInsertionPointAfter(ifValidLexInsert);
       env.updateInsertionChain(ifValidLexInsert.getResult(0));
+    } else {
+      // Generates regular insertion chain.
+      env.updateInsertionChain(builder.create<InsertOp>(loc, rhs, chain, ivs));
     }
     return;
   }
@@ -688,12 +689,13 @@ static void genInvariants(CodegenEnv &env, OpBuilder &builder, ExprId exp,
           env.startReduc(exp, genTensorLoad(env, builder, exp));
         }
         if (env.hasSparseOutput())
-          env.setValidLexInsert(constantI1(builder, env.op().getLoc(), false));
+          env.startValidLexInsert(
+              constantI1(builder, env.op().getLoc(), false));
       } else {
         if (!env.isCustomReduc() || env.isReduc())
           genTensorStore(env, builder, exp, env.endReduc());
         if (env.hasSparseOutput())
-          env.clearValidLexInsert();
+          env.endValidLexInsert();
       }
     } else {
       // Start or end loop invariant hoisting of a tensor load.
@@ -846,9 +848,9 @@ static void finalizeWhileOp(CodegenEnv &env, OpBuilder &builder,
       if (env.isReduc()) {
         yields.push_back(env.getReduc());
         env.updateReduc(ifOp.getResult(y++));
-        if (env.getValidLexInsert()) {
+        if (env.isValidLexInsert()) {
           yields.push_back(env.getValidLexInsert());
-          env.setValidLexInsert(ifOp.getResult(y++));
+          env.updateValidLexInsert(ifOp.getResult(y++));
         }
       }
       if (env.isExpand()) {
@@ -904,7 +906,7 @@ static scf::IfOp genIf(CodegenEnv &env, OpBuilder &builder, LoopId curr,
       });
   if (env.isReduc()) {
     types.push_back(env.getReduc().getType());
-    if (env.getValidLexInsert())
+    if (env.isValidLexInsert())
       types.push_back(env.getValidLexInsert().getType());
   }
   if (env.isExpand())
@@ -924,10 +926,10 @@ static void endIf(CodegenEnv &env, OpBuilder &builder, scf::IfOp ifOp,
   if (env.isReduc()) {
     operands.push_back(env.getReduc());
     env.updateReduc(redInput);
-    if (env.getValidLexInsert()) {
+    if (env.isValidLexInsert()) {
       // Any overlapping indices during a reduction creates a valid lex insert.
       operands.push_back(constantI1(builder, env.op().getLoc(), true));
-      env.setValidLexInsert(validIns);
+      env.updateValidLexInsert(validIns);
     }
   }
   if (env.isExpand()) {
@@ -1174,8 +1176,8 @@ static bool endLoop(CodegenEnv &env, RewriterBase &rewriter, Operation *loop,
   // Either a for-loop or a while-loop that iterates over a slice.
   if (isSingleCond) {
     // Any iteration creates a valid lex insert.
-    if (env.isReduc() && env.getValidLexInsert())
-      env.setValidLexInsert(constantI1(rewriter, env.op().getLoc(), true));
+    if (env.isReduc() && env.isValidLexInsert())
+      env.updateValidLexInsert(constantI1(rewriter, env.op().getLoc(), true));
   } else if (auto whileOp = dyn_cast<scf::WhileOp>(loop)) {
     // End a while-loop.
     finalizeWhileOp(env, rewriter, needsUniv);
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/CodegenEnv.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenEnv.cpp
similarity index 94%
rename from mlir/lib/Dialect/SparseTensor/Transforms/CodegenEnv.cpp
rename to mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenEnv.cpp
index 312aefc0936c286b40abd7d7d96f9997fd3a8f60..4bd3af2d3f2f6a34b965bba4566f6ef7c2e7077b 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/CodegenEnv.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenEnv.cpp
@@ -115,10 +115,10 @@ std::optional<Operation *> CodegenEnv::genLoopBoundary(
   SmallVector<Value> params;
   if (isReduc()) {
     params.push_back(redVal);
-    if (redValidLexInsert)
+    if (isValidLexInsert())
       params.push_back(redValidLexInsert);
   } else {
-    assert(!redValidLexInsert);
+    assert(!isValidLexInsert());
   }
   if (isExpand())
     params.push_back(expCount);
@@ -128,8 +128,8 @@ std::optional<Operation *> CodegenEnv::genLoopBoundary(
   unsigned i = 0;
   if (isReduc()) {
     updateReduc(params[i++]);
-    if (redValidLexInsert)
-      setValidLexInsert(params[i++]);
+    if (isValidLexInsert())
+      updateValidLexInsert(params[i++]);
   }
   if (isExpand())
     updateExpandCount(params[i++]);
@@ -235,14 +235,14 @@ void CodegenEnv::endExpand() {
 //===----------------------------------------------------------------------===//
 
 void CodegenEnv::startReduc(ExprId exp, Value val) {
-  assert(!isReduc() && exp != detail::kInvalidId);
+  assert(!isReduc() && exp != detail::kInvalidId && val);
   redExp = exp;
   redVal = val;
   latticeMerger.setExprValue(exp, val);
 }
 
 void CodegenEnv::updateReduc(Value val) {
-  assert(isReduc());
+  assert(isReduc() && val);
   redVal = val;
   latticeMerger.clearExprValue(redExp);
   latticeMerger.setExprValue(redExp, val);
@@ -257,13 +257,18 @@ Value CodegenEnv::endReduc() {
   return val;
 }
 
-void CodegenEnv::setValidLexInsert(Value val) {
-  assert(isReduc() && val);
+void CodegenEnv::startValidLexInsert(Value val) {
+  assert(!isValidLexInsert() && isReduc() && val);
+  redValidLexInsert = val;
+}
+
+void CodegenEnv::updateValidLexInsert(Value val) {
+  assert(redValidLexInsert && isReduc() && val);
   redValidLexInsert = val;
 }
 
-void CodegenEnv::clearValidLexInsert() {
-  assert(!isReduc());
+void CodegenEnv::endValidLexInsert() {
+  assert(isValidLexInsert() && !isReduc());
   redValidLexInsert = Value();
 }
 
@@ -272,7 +277,7 @@ void CodegenEnv::startCustomReduc(ExprId exp) {
   redCustom = exp;
 }
 
-Value CodegenEnv::getCustomRedId() {
+Value CodegenEnv::getCustomRedId() const {
   assert(isCustomReduc());
   return dyn_cast<sparse_tensor::ReduceOp>(exp(redCustom).op).getIdentity();
 }
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/CodegenEnv.h b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenEnv.h
similarity index 94%
rename from mlir/lib/Dialect/SparseTensor/Transforms/CodegenEnv.h
rename to mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenEnv.h
index a1947f48393ef963a8329c2e0607371439aad940..728af841cc7b17fcd249ae59fe72a6f666b39462 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/CodegenEnv.h
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenEnv.h
@@ -10,8 +10,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef MLIR_DIALECT_SPARSETENSOR_TRANSFORMS_CODEGENENV_H_
-#define MLIR_DIALECT_SPARSETENSOR_TRANSFORMS_CODEGENENV_H_
+#ifndef MLIR_DIALECT_SPARSETENSOR_TRANSFORMS_UTILS_CODEGENENV_H_
+#define MLIR_DIALECT_SPARSETENSOR_TRANSFORMS_UTILS_CODEGENENV_H_
 
 #include "CodegenUtils.h"
 #include "LoopEmitter.h"
@@ -150,13 +150,16 @@ public:
   void updateReduc(Value val);
   Value getReduc() const { return redVal; }
   Value endReduc();
-  void setValidLexInsert(Value val);
-  void clearValidLexInsert();
+
+  void startValidLexInsert(Value val);
+  bool isValidLexInsert() const { return redValidLexInsert != nullptr; }
+  void updateValidLexInsert(Value val);
   Value getValidLexInsert() const { return redValidLexInsert; }
+  void endValidLexInsert();
 
   void startCustomReduc(ExprId exp);
   bool isCustomReduc() const { return redCustom != detail::kInvalidId; }
-  Value getCustomRedId();
+  Value getCustomRedId() const;
   void endCustomReduc();
 
 private:
@@ -203,4 +206,4 @@ private:
 } // namespace sparse_tensor
 } // namespace mlir
 
-#endif // MLIR_DIALECT_SPARSETENSOR_TRANSFORMS_CODEGENENV_H_
+#endif // MLIR_DIALECT_SPARSETENSOR_TRANSFORMS_UTILS_CODEGENENV_H_
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/CodegenUtils.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.cpp
similarity index 100%
rename from mlir/lib/Dialect/SparseTensor/Transforms/CodegenUtils.cpp
rename to mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.cpp
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/CodegenUtils.h b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.h
similarity index 98%
rename from mlir/lib/Dialect/SparseTensor/Transforms/CodegenUtils.h
rename to mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.h
index 57de437e2311c0b97b010fb490b52ca42bde0bfc..8dc57e1b5479bf2c612136d3c2f8fa4b9e7efdb8 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/CodegenUtils.h
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/CodegenUtils.h
@@ -10,8 +10,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef MLIR_DIALECT_SPARSETENSOR_TRANSFORMS_CODEGENUTILS_H_
-#define MLIR_DIALECT_SPARSETENSOR_TRANSFORMS_CODEGENUTILS_H_
+#ifndef MLIR_DIALECT_SPARSETENSOR_TRANSFORMS_UTILS_CODEGENUTILS_H_
+#define MLIR_DIALECT_SPARSETENSOR_TRANSFORMS_UTILS_CODEGENUTILS_H_
 
 #include "mlir/Dialect/Arith/IR/Arith.h"
 #include "mlir/Dialect/Complex/IR/Complex.h"
@@ -434,4 +434,4 @@ inline bool isZeroRankedTensorOrScalar(Type type) {
 } // namespace sparse_tensor
 } // namespace mlir
 
-#endif // MLIR_DIALECT_SPARSETENSOR_TRANSFORMS_CODEGENUTILS_H_
+#endif // MLIR_DIALECT_SPARSETENSOR_TRANSFORMS_UTILS_CODEGENUTILS_H_
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/IterationGraphSorter.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/IterationGraphSorter.cpp
similarity index 94%
rename from mlir/lib/Dialect/SparseTensor/Transforms/IterationGraphSorter.cpp
rename to mlir/lib/Dialect/SparseTensor/Transforms/Utils/IterationGraphSorter.cpp
index b6011727f4127cf2a70c5d8db54fff9f7ecec1c0..588400f3dbaf090047ce1ceae5b88869bfac43f4 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/IterationGraphSorter.cpp
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/IterationGraphSorter.cpp
@@ -1,4 +1,4 @@
-//===- LoopScheduler.cpp -------------------------------------------------===//
+//===- IterationGraphSorter.cpp -------------------------------------------===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
@@ -20,11 +20,10 @@ using namespace mlir::sparse_tensor;
 
 namespace {
 
-/// A helper class that visits an affine expression and tries to find an
-/// AffineDimExpr to which the corresponding iterator from a GenericOp matches
-/// the desired iterator type.
-/// If there is no matched iterator type, returns the first DimExpr in the
-/// expression.
+/// A helper class that visits an affine expression and tries to find
+/// an AffineDimExpr to which the corresponding iterator from a GenericOp
+/// matches the desired iterator type. If there is no matched iterator
+/// type, the method returns the first DimExpr in the expression.
 class AffineDimFinder : public AffineExprVisitor<AffineDimFinder> {
 public:
   explicit AffineDimFinder(ArrayRef<utils::IteratorType> itTypes)
@@ -81,11 +80,9 @@ inline static bool includesDenseOutput(SortMask mask) {
   return includesAny(mask, SortMask::kIncludeDenseOutput);
 }
 
-/// A helper to compute a topological sort. O(n^2) time complexity
-/// as we use adj matrix for the graph.
-/// The sorted result will put the first Reduction iterator to the
-/// latest possible position.
 AffineMap IterationGraphSorter::topoSort() {
+  // The sorted result will put the first Reduction iterator to the
+  // latest possible position.
   std::vector<unsigned> redIt; // reduce iterator with 0 degree
   std::vector<unsigned> parIt; // parallel iterator with 0 degree
   const unsigned numLoops = getNumLoops();
@@ -170,6 +167,7 @@ AffineMap IterationGraphSorter::sort(SortMask mask, Value ignored) {
   // Reset the interation graph.
   for (auto &row : itGraph)
     std::fill(row.begin(), row.end(), false);
+
   // Reset cached in-degree.
   std::fill(inDegree.begin(), inDegree.end(), 0);
 
@@ -179,7 +177,6 @@ AffineMap IterationGraphSorter::sort(SortMask mask, Value ignored) {
     // Skip dense inputs when not requested.
     if ((!enc && !includesDenseInput(mask)) || in == ignored)
       continue;
-
     addConstraints(in, map);
   }
 
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/IterationGraphSorter.h b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/IterationGraphSorter.h
similarity index 61%
rename from mlir/lib/Dialect/SparseTensor/Transforms/IterationGraphSorter.h
rename to mlir/lib/Dialect/SparseTensor/Transforms/Utils/IterationGraphSorter.h
index 52ee117029300965fe5aeb1ddf959f7800853c33..be94bb5dffde6380396a6d4b89c216fbeea0175b 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/IterationGraphSorter.h
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/IterationGraphSorter.h
@@ -1,10 +1,17 @@
-//===- LoopScheduler.h -----------------------------------------*- C++ -*-===//
+//===- IterationGraphSorter.h -----------------------------------*- C++ -*-===//
 //
 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 // See https://llvm.org/LICENSE.txt for license information.
 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 //
 //===----------------------------------------------------------------------===//
+//
+// This header file defines the iteration graph sorter (top-sort scheduling).
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef MLIR_DIALECT_SPARSETENSOR_TRANSFORMS_UTILS_ITERATIONGRAPHSORTER_H_
+#define MLIR_DIALECT_SPARSETENSOR_TRANSFORMS_UTILS_ITERATIONGRAPHSORTER_H_
 
 #include "mlir/IR/AffineMap.h"
 
@@ -21,7 +28,7 @@ class GenericOp;
 
 namespace sparse_tensor {
 
-/// Iteration graph sorting.
+/// Iteration graph sorting mask,
 enum class SortMask : unsigned {
   // The individual mask bits.
   kIncludeDenseOutput = 0x1, // b001
@@ -34,40 +41,52 @@ enum class SortMask : unsigned {
 
 class IterationGraphSorter {
 public:
-  // Constructs a scheduler from linalg.generic
-  // Maybe reuses the class to schedule foreach as well (to address
-  // non-permutation, e.g, traverse CSR in BSR order).
+  /// Factory method that construct an iteration graph sorter
+  /// for the given linalg.generic operation.
   static IterationGraphSorter fromGenericOp(linalg::GenericOp genericOp);
 
-  // Returns a permutation that represents the scheduled loop order.
-  // Note that the returned AffineMap could be null if the kernel can not be
-  // schedule due to cycles in the iteration graph.
+  /// Returns a permutation that represents the scheduled loop order.
+  /// Note that the returned AffineMap could be null if the kernel
+  /// cannot be scheduled due to cyclic iteration graph.
   [[nodiscard]] AffineMap sort(SortMask mask, Value ignored = nullptr);
+
+  /// Returns the number of loops in the iteration graph.
   unsigned getNumLoops() const { return loop2OutLvl.getNumDims(); }
 
 private:
+  // Private constructor.
   IterationGraphSorter(SmallVector<Value> &&ins,
                        SmallVector<AffineMap> &&loop2InsLvl, Value out,
                        AffineMap loop2OutLvl,
                        SmallVector<utils::IteratorType> &&iterTypes);
 
+  // Adds all the constraints in the given loop to level map.
   void addConstraints(Value t, AffineMap loop2LvlMap);
+
+  /// A helper to compute a topological sort. The method has an
+  /// O(n^2) time complexity since we use an adjacency matrix
+  /// representation for the iteration graph.
   AffineMap topoSort();
 
   // Input tensors and associated loop to level maps.
   SmallVector<Value> ins;
   SmallVector<AffineMap> loop2InsLvl;
+
   // Output tensor and associated loop to level map.
   Value out;
   AffineMap loop2OutLvl;
-  // Loop type;
+
+  // Loop itation types;
   SmallVector<utils::IteratorType> iterTypes;
 
   // Adjacent matrix that represents the iteration graph.
   std::vector<std::vector<bool>> itGraph;
+
   // InDegree used for topo sort.
   std::vector<unsigned> inDegree;
 };
 
 } // namespace sparse_tensor
 } // namespace mlir
+
+#endif // MLIR_DIALECT_SPARSETENSOR_TRANSFORMS_UTILS_ITERATIONGRAPHSORTER_H_
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/LoopEmitter.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/LoopEmitter.cpp
similarity index 100%
rename from mlir/lib/Dialect/SparseTensor/Transforms/LoopEmitter.cpp
rename to mlir/lib/Dialect/SparseTensor/Transforms/Utils/LoopEmitter.cpp
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/LoopEmitter.h b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/LoopEmitter.h
similarity index 98%
rename from mlir/lib/Dialect/SparseTensor/Transforms/LoopEmitter.h
rename to mlir/lib/Dialect/SparseTensor/Transforms/Utils/LoopEmitter.h
index fa8b0076f733b48b532f88b4d12a98fbefe6303e..78bb53e4483f60a82ff1621eea952085e9feafda 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/LoopEmitter.h
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/LoopEmitter.h
@@ -6,8 +6,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef MLIR_DIALECT_SPARSETENSOR_TRANSFORMS_SPARSETENSORLOOPEMITTER_H_
-#define MLIR_DIALECT_SPARSETENSOR_TRANSFORMS_SPARSETENSORLOOPEMITTER_H_
+#ifndef MLIR_DIALECT_SPARSETENSOR_TRANSFORMS_UTILS_LOOPEMITTER_H_
+#define MLIR_DIALECT_SPARSETENSOR_TRANSFORMS_UTILS_LOOPEMITTER_H_
 
 #include <vector>
 
@@ -22,7 +22,7 @@ namespace sparse_tensor {
 // A compressed <tensor id, level> pair.
 using TensorLevel = unsigned;
 
-//===----------------------------------------------------------------------===//
+//
 // SparseTensorLoopEmiter class, manages sparse tensors and helps to
 // generate loop structure to (co)-iterate sparse tensors.
 //
@@ -48,8 +48,7 @@ using TensorLevel = unsigned;
 // loopEmiter.exitCurrentLoop(); // exit k
 // loopEmiter.exitCurrentLoop(); // exit j
 // loopEmiter.exitCurrentLoop(); // exit i
-//===----------------------------------------------------------------------===//
-
+//
 class LoopEmitter {
 public:
   /// Optional callback function to setup dense output tensors when
@@ -705,4 +704,4 @@ private:
 } // namespace sparse_tensor
 } // namespace mlir
 
-#endif // MLIR_DIALECT_SPARSETENSOR_TRANSFORMS_SPARSETENSORLOOPEMITTER_H_
+#endif // MLIR_DIALECT_SPARSETENSOR_TRANSFORMS_UTILS_LOOPEMITTER_H_
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorDescriptor.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorDescriptor.cpp
similarity index 100%
rename from mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorDescriptor.cpp
rename to mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorDescriptor.cpp
diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorDescriptor.h b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorDescriptor.h
similarity index 97%
rename from mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorDescriptor.h
rename to mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorDescriptor.h
index 5c7d8aa4c9d967897a7d1b96d6d3fd34e37625dc..3a61ec7a2236f38c3f24bc8d4b828363d030cb1c 100644
--- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorDescriptor.h
+++ b/mlir/lib/Dialect/SparseTensor/Transforms/Utils/SparseTensorDescriptor.h
@@ -10,8 +10,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef MLIR_DIALECT_SPARSETENSOR_TRANSFORMS_SPARSETENSORDESCRIPTOR_H_
-#define MLIR_DIALECT_SPARSETENSOR_TRANSFORMS_SPARSETENSORDESCRIPTOR_H_
+#ifndef MLIR_DIALECT_SPARSETENSOR_TRANSFORMS_UTILS_SPARSETENSORDESCRIPTOR_H_
+#define MLIR_DIALECT_SPARSETENSOR_TRANSFORMS_UTILS_SPARSETENSORDESCRIPTOR_H_
 
 #include "mlir/Dialect/SparseTensor/IR/SparseTensor.h"
 #include "mlir/Dialect/SparseTensor/IR/SparseTensorStorageLayout.h"
@@ -262,4 +262,4 @@ getMutDescriptorFromTensorTuple(Value tensor, SmallVectorImpl<Value> &fields) {
 } // namespace sparse_tensor
 } // namespace mlir
 
-#endif // MLIR_DIALECT_SPARSETENSOR_TRANSFORMS_SPARSETENSODESCRIPTOR_H_
+#endif // MLIR_DIALECT_SPARSETENSOR_TRANSFORMS_UTILS_SPARSETENSODESCRIPTOR_H_
diff --git a/mlir/lib/Dialect/Tensor/Transforms/FoldIntoPackAndUnpackPatterns.cpp b/mlir/lib/Dialect/Tensor/Transforms/FoldIntoPackAndUnpackPatterns.cpp
index 9eac3e5c7ef9102ebfd489859fac679b825853ff..e4509b331beeac0f4786837d3aa17e87ecdef2b5 100644
--- a/mlir/lib/Dialect/Tensor/Transforms/FoldIntoPackAndUnpackPatterns.cpp
+++ b/mlir/lib/Dialect/Tensor/Transforms/FoldIntoPackAndUnpackPatterns.cpp
@@ -6,6 +6,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "mlir/Dialect/Linalg/IR/Linalg.h"
 #include "mlir/Dialect/Tensor/IR/Tensor.h"
 #include "mlir/Dialect/Tensor/Transforms/Transforms.h"
 #include "mlir/IR/PatternMatch.h"
@@ -81,10 +82,71 @@ struct FoldUnpackWithExtractSliceOp : public OpRewritePattern<ExtractSliceOp> {
     return success();
   }
 };
+
+/// Fold 'pack' -> 'transpose' into 'pack' since 'pack' already has transpose
+/// semantics.
+struct FoldProducerPackWithConsumerLinalgTransposeOp
+    : public OpRewritePattern<linalg::TransposeOp> {
+  using OpRewritePattern<linalg::TransposeOp>::OpRewritePattern;
+
+  LogicalResult matchAndRewrite(linalg::TransposeOp transposeOp,
+                                PatternRewriter &rewriter) const override {
+    auto packOp = transposeOp.getOperand(0).getDefiningOp<PackOp>();
+
+    if (!packOp)
+      return failure();
+
+    auto innerDimsPos = packOp.getInnerDimsPos();
+    auto mixedInnerTiles = packOp.getMixedTiles();
+    auto outerDimsPerm = packOp.getOuterDimsPerm();
+    auto transposePerm = transposeOp.getPermutation();
+    SmallVector<int64_t> newOuterDimsPermVec;
+    SmallVector<int64_t> newInnerDimsPosVec;
+    SmallVector<OpFoldResult> newMixedInnerTilesVec;
+    int64_t srcRank = packOp.getSourceRank();
+
+    // Process transpose operation for non-tiled outer dimensions
+    for (unsigned int i = 0; i < srcRank; ++i) {
+      int64_t remappedPosition = transposePerm[i];
+
+      // If tensor.pack has outer_dims_perm attribute, then consider it during
+      // index remapping.
+      if (!outerDimsPerm.empty()) {
+        if (transposePerm[i] >= srcRank) {
+          return rewriter.notifyMatchFailure(
+              transposeOp,
+              "Cannot fold in tensor.pack if a tile dimension was transposed "
+              "with a non-tile dimension in linalg.transpose.");
+        }
+        remappedPosition = outerDimsPerm[remappedPosition];
+      }
+
+      newOuterDimsPermVec.push_back(remappedPosition);
+    }
+
+    // Process transpose operation for tiled inner dimensions
+    for (unsigned int i = srcRank; i < transposePerm.size(); ++i) {
+      int64_t remappedPosition = transposePerm[i] - srcRank;
+      newMixedInnerTilesVec.push_back(mixedInnerTiles[remappedPosition]);
+      newInnerDimsPosVec.push_back(innerDimsPos[remappedPosition]);
+    }
+
+    Value output = packOp.createDestinationTensor(
+        rewriter, transposeOp.getLoc(), packOp.getSource(),
+        newMixedInnerTilesVec, newInnerDimsPosVec, newOuterDimsPermVec);
+
+    rewriter.replaceOpWithNewOp<PackOp>(
+        transposeOp, packOp.getSource(), output, newInnerDimsPosVec,
+        newMixedInnerTilesVec, packOp.getPaddingValue(), newOuterDimsPermVec);
+
+    return success();
+  }
+};
 } // namespace
 
 void populateFoldIntoPackAndUnpackPatterns(RewritePatternSet &patterns) {
-  patterns.insert<FoldUnpackWithExtractSliceOp, FoldPadWithPackOp>(
+  patterns.insert<FoldUnpackWithExtractSliceOp, FoldPadWithPackOp,
+                  FoldProducerPackWithConsumerLinalgTransposeOp>(
       patterns.getContext());
 }
 
diff --git a/mlir/lib/Dialect/Transform/Transforms/TransformInterpreterUtils.cpp b/mlir/lib/Dialect/Transform/Transforms/TransformInterpreterUtils.cpp
index c92472acf279481c5dd07f2662d75b1e4e47739e..3fa26bce150992355016ab018d9ede8353f47080 100644
--- a/mlir/lib/Dialect/Transform/Transforms/TransformInterpreterUtils.cpp
+++ b/mlir/lib/Dialect/Transform/Transforms/TransformInterpreterUtils.cpp
@@ -69,7 +69,7 @@ LogicalResult transform::detail::expandPathsToMLIRFiles(
         continue;
       }
 
-      if (!StringRef(fileName).endswith(".mlir")) {
+      if (!StringRef(fileName).ends_with(".mlir")) {
         LLVM_DEBUG(DBGS() << "  Skipping '" << fileName
                           << "' because it does not end with '.mlir'\n");
         continue;
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp
index ed42e6508b431014f7e3a63cd7368cfdc2b028fc..b761d1ed888973d44385265392055e356ca10cbf 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorTransferOpTransforms.cpp
@@ -922,4 +922,5 @@ void mlir::vector::populateFlattenVectorTransferPatterns(
                FlattenContiguousRowMajorTransferWritePattern>(
       patterns.getContext(), benefit);
   populateShapeCastFoldingPatterns(patterns, benefit);
+  populateDropUnitDimWithShapeCastPatterns(patterns, benefit);
 }
diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp
index 6e7fab293d3a1ca0fd0913ed54d9d85b1da4d47d..5936b0b54af4e384b595eabe9b7d96d5b8ae7428 100644
--- a/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp
+++ b/mlir/lib/Dialect/Vector/Transforms/VectorTransforms.cpp
@@ -1446,6 +1446,92 @@ struct ChainedReduction final : OpRewritePattern<vector::ReductionOp> {
   }
 };
 
+/// For vectors with either leading or trailing unit dim, replaces:
+///   elementwise(a, b)
+/// with:
+///   sc_a = shape_cast(a)
+///   sc_b = shape_cast(b)
+///   res = elementwise(sc_a, sc_b)
+///   return shape_cast(res)
+/// The newly inserted shape_cast Ops fold (before elementwise Op) and then
+/// restore (after elementwise Op) the unit dim. Vectors `a` and `b` are
+/// required to be rank > 1.
+///
+/// Ex:
+/// ```
+///  %mul = arith.mulf %B_row, %A_row : vector<1x[4]xf32>
+///  %cast = vector.shape_cast %mul : vector<1x[4]xf32> to vector<[4]xf32>
+/// ```
+///
+/// gets converted to:
+///
+/// ```
+///  %B_row_sc = vector.shape_cast %B_row : vector<1x[4]xf32> to vector<[4]xf32>
+///  %A_row_sc = vector.shape_cast %A_row : vector<1x[4]xf32> to vector<[4]xf32>
+///  %mul = arith.mulf %B_row_sc, %A_row_sc : vector<[4]xf32>
+///  %cast_new = vector.shape_cast %mul : vector<[4]xf32> to vector<1x[4]xf32>
+///  %cast = vector.shape_cast %cast_new : vector<1x[4]xf32> to vector<[4]xf32>
+/// ```
+///
+/// Patterns for folding shape_casts should instantly eliminate `%cast_new` and
+/// `%cast`.
+struct DropUnitDimFromElementwiseOps final
+    : public OpTraitRewritePattern<OpTrait::Elementwise> {
+  using OpTraitRewritePattern::OpTraitRewritePattern;
+  LogicalResult matchAndRewrite(Operation *op,
+                                PatternRewriter &rewriter) const override {
+    if (op->getNumResults() != 1 || op->getNumRegions() != 0)
+      return failure();
+
+    auto resultVectorType = dyn_cast<VectorType>(op->getResult(0).getType());
+    if (!resultVectorType)
+      return failure();
+
+    // Check the pre-conditions. For `Elementwise` Ops all operands are
+    // guaranteed to have identical shapes and it suffices to only check the
+    // first one.
+    auto sourceVectorType = cast<VectorType>(op->getOperands()[0].getType());
+    if (sourceVectorType.getRank() < 2)
+      return failure();
+
+    bool hasTrailingDimUnitFixed =
+        ((sourceVectorType.getShape().back() == 1) &&
+         (!sourceVectorType.getScalableDims().back()));
+    bool hasLeadingDimUnitFixed =
+        ((sourceVectorType.getShape().front() == 1) &&
+         (!sourceVectorType.getScalableDims().front()));
+    if (!hasLeadingDimUnitFixed && !hasTrailingDimUnitFixed)
+      return failure();
+
+    // Drop leading/trailing unit dim by applying vector.shape_cast to all
+    // operands
+    int64_t dim = hasLeadingDimUnitFixed ? 0 : sourceVectorType.getRank() - 1;
+
+    SmallVector<Value> newOperands;
+    auto loc = op->getLoc();
+    for (auto operand : op->getOperands()) {
+      auto opVectorType = cast<VectorType>(operand.getType());
+      VectorType newVType = VectorType::Builder(opVectorType).dropDim(dim);
+      auto opSC = rewriter.create<vector::ShapeCastOp>(loc, newVType, operand);
+      newOperands.push_back(opSC);
+    }
+
+    VectorType newResultVectorType =
+        VectorType::Builder(resultVectorType).dropDim(dim);
+    // Create an updated elementwise Op without leading/trailing unit dim
+    Operation *elementwiseOp =
+        rewriter.create(loc, op->getName().getIdentifier(), newOperands,
+                        newResultVectorType, op->getAttrs());
+
+    // Restore the leading/trailing unit dim by applying vector.shape_cast
+    // to the result
+    rewriter.replaceOpWithNewOp<ShapeCastOp>(op, resultVectorType,
+                                             elementwiseOp->getResult(0));
+
+    return success();
+  }
+};
+
 /// Pattern to eliminate redundant zero-constants added to reduction operands.
 /// It's enough for there to be one initial zero value, so we can eliminate the
 /// extra ones that feed into `vector.reduction <add>`. These get created by the
@@ -1514,6 +1600,12 @@ void mlir::vector::populateShapeCastFoldingPatterns(RewritePatternSet &patterns,
   patterns.add<ShapeCastOpFolder>(patterns.getContext(), benefit);
 }
 
+void mlir::vector::populateDropUnitDimWithShapeCastPatterns(
+    RewritePatternSet &patterns, PatternBenefit benefit) {
+  patterns.add<DropUnitDimFromElementwiseOps, ShapeCastOpFolder>(
+      patterns.getContext(), benefit);
+}
+
 void mlir::vector::populateBubbleVectorBitCastOpPatterns(
     RewritePatternSet &patterns, PatternBenefit benefit) {
   patterns.add<BubbleDownVectorBitCastForExtract,
diff --git a/mlir/lib/ExecutionEngine/CMakeLists.txt b/mlir/lib/ExecutionEngine/CMakeLists.txt
index 70c5a07ad1ab2378bda06ed26a3ed2f10ad6fdd2..2f391b7698cbb0281277f623ff41772facb132d6 100644
--- a/mlir/lib/ExecutionEngine/CMakeLists.txt
+++ b/mlir/lib/ExecutionEngine/CMakeLists.txt
@@ -157,6 +157,9 @@ if(LLVM_ENABLE_PIC)
     RunnerUtils.cpp
 
     EXCLUDE_FROM_LIBMLIR
+
+    LINK_LIBS PUBLIC
+    mlir_float16_utils
   )
   target_compile_definitions(mlir_runner_utils PRIVATE mlir_runner_utils_EXPORTS)
 
diff --git a/mlir/lib/ExecutionEngine/RunnerUtils.cpp b/mlir/lib/ExecutionEngine/RunnerUtils.cpp
index bbfd3a6b11c2a1bd0ab944aa34378d12d2094647..378aa7ce35ef16db59cba664503238c0e66ca0d9 100644
--- a/mlir/lib/ExecutionEngine/RunnerUtils.cpp
+++ b/mlir/lib/ExecutionEngine/RunnerUtils.cpp
@@ -81,6 +81,10 @@ extern "C" void _mlir_ciface_printMemrefI8(UnrankedMemRefType<int8_t> *M) {
   impl::printMemRef(*M);
 }
 
+extern "C" void _mlir_ciface_printMemrefI16(UnrankedMemRefType<int16_t> *M) {
+  impl::printMemRef(*M);
+}
+
 extern "C" void _mlir_ciface_printMemrefI32(UnrankedMemRefType<int32_t> *M) {
   impl::printMemRef(*M);
 }
@@ -89,6 +93,14 @@ extern "C" void _mlir_ciface_printMemrefI64(UnrankedMemRefType<int64_t> *M) {
   impl::printMemRef(*M);
 }
 
+extern "C" void _mlir_ciface_printMemrefF16(UnrankedMemRefType<f16> *M) {
+  impl::printMemRef(*M);
+}
+
+extern "C" void _mlir_ciface_printMemrefBF16(UnrankedMemRefType<bf16> *M) {
+  impl::printMemRef(*M);
+}
+
 extern "C" void _mlir_ciface_printMemrefF32(UnrankedMemRefType<float> *M) {
   impl::printMemRef(*M);
 }
diff --git a/mlir/lib/IR/Operation.cpp b/mlir/lib/IR/Operation.cpp
index 7507e0cd0c99b31fce7bcac1b5edb1bf534362d0..87be08712ea35b06da0d746deb57dbb704e6b5c3 100644
--- a/mlir/lib/IR/Operation.cpp
+++ b/mlir/lib/IR/Operation.cpp
@@ -751,7 +751,7 @@ void OpState::print(Operation *op, OpAsmPrinter &p, StringRef defaultDialect) {
 void OpState::printOpName(Operation *op, OpAsmPrinter &p,
                           StringRef defaultDialect) {
   StringRef name = op->getName().getStringRef();
-  if (name.startswith((defaultDialect + ".").str()) && name.count('.') == 1)
+  if (name.starts_with((defaultDialect + ".").str()) && name.count('.') == 1)
     name = name.drop_front(defaultDialect.size() + 1);
   p.getStream() << name;
 }
diff --git a/mlir/lib/Query/Matcher/Parser.cpp b/mlir/lib/Query/Matcher/Parser.cpp
index be9e60de221db1969204bd91258fae5bbeb4afd9..6585b5d740f6265f3516cc130ded03c31b1e10a4 100644
--- a/mlir/lib/Query/Matcher/Parser.cpp
+++ b/mlir/lib/Query/Matcher/Parser.cpp
@@ -384,7 +384,7 @@ bool Parser::parseMatcherExpressionImpl(const TokenInfo &nameToken,
 // completions minus the prefix.
 void Parser::addCompletion(const TokenInfo &compToken,
                            const MatcherCompletion &completion) {
-  if (llvm::StringRef(completion.typedText).startswith(compToken.text)) {
+  if (llvm::StringRef(completion.typedText).starts_with(compToken.text)) {
     completions.emplace_back(completion.typedText.substr(compToken.text.size()),
                              completion.matcherDecl);
   }
diff --git a/mlir/lib/Support/ToolUtilities.cpp b/mlir/lib/Support/ToolUtilities.cpp
index 0f9c4a933d008b61d9f0f9a77a97610eb5168afd..ee0214f3d8ac04d93082e5289c075c1445c7c973 100644
--- a/mlir/lib/Support/ToolUtilities.cpp
+++ b/mlir/lib/Support/ToolUtilities.cpp
@@ -58,7 +58,7 @@ mlir::splitAndProcessBuffer(std::unique_ptr<llvm::MemoryBuffer> originalBuffer,
     }
 
     // Check that suffix is as expected and doesn't have any dash post.
-    bool expectedSuffix = buffer.startswith(splitMarker.take_back(checkLen)) &&
+    bool expectedSuffix = buffer.starts_with(splitMarker.take_back(checkLen)) &&
                           buffer.size() > checkLen && buffer[checkLen] != '0';
     if (expectedSuffix) {
       sourceBuffers.push_back(prev);
diff --git a/mlir/lib/TableGen/Class.cpp b/mlir/lib/TableGen/Class.cpp
index 36038230568501652da4da5ded57922b5bed501d..f71d7e07ed49989705a4516fcab852fa4c24cce0 100644
--- a/mlir/lib/TableGen/Class.cpp
+++ b/mlir/lib/TableGen/Class.cpp
@@ -18,7 +18,8 @@ using namespace mlir::tblgen;
 /// Returns space to be emitted after the given C++ `type`. return "" if the
 /// ends with '&' or '*', or is empty, else returns " ".
 static StringRef getSpaceAfterType(StringRef type) {
-  return (type.empty() || type.endswith("&") || type.endswith("*")) ? "" : " ";
+  return (type.empty() || type.ends_with("&") || type.ends_with("*")) ? ""
+                                                                      : " ";
 }
 
 //===----------------------------------------------------------------------===//
diff --git a/mlir/lib/Target/LLVMIR/DebugImporter.cpp b/mlir/lib/Target/LLVMIR/DebugImporter.cpp
index 13b81d134cbe43e8a7fe80eee5e876e20af8e3f1..afc6918eae97556d3d16bd609cfb757cc9061bd3 100644
--- a/mlir/lib/Target/LLVMIR/DebugImporter.cpp
+++ b/mlir/lib/Target/LLVMIR/DebugImporter.cpp
@@ -118,12 +118,19 @@ DebugImporter::translateImpl(llvm::DILexicalBlockFile *node) {
 
 DIGlobalVariableAttr
 DebugImporter::translateImpl(llvm::DIGlobalVariable *node) {
+  // Names of DIGlobalVariables can be empty. MLIR models them as null, instead
+  // of empty strings, so this special handling is necessary.
+  auto convertToStringAttr = [&](StringRef name) -> StringAttr {
+    if (name.empty())
+      return {};
+    return StringAttr::get(context, node->getName());
+  };
   return DIGlobalVariableAttr::get(
       context, translate(node->getScope()),
-      StringAttr::get(context, node->getName()),
-      StringAttr::get(context, node->getLinkageName()),
-      translate(node->getFile()), node->getLine(), translate(node->getType()),
-      node->isLocalToUnit(), node->isDefinition(), node->getAlignInBits());
+      convertToStringAttr(node->getName()),
+      convertToStringAttr(node->getLinkageName()), translate(node->getFile()),
+      node->getLine(), translate(node->getType()), node->isLocalToUnit(),
+      node->isDefinition(), node->getAlignInBits());
 }
 
 DILocalVariableAttr DebugImporter::translateImpl(llvm::DILocalVariable *node) {
diff --git a/mlir/lib/Target/LLVMIR/ModuleImport.cpp b/mlir/lib/Target/LLVMIR/ModuleImport.cpp
index 7c51ee7420f9b3a1599254afbe0bf6f3c535b090..ec2692f58695d0df3aa75cff0e95840bef67b614 100644
--- a/mlir/lib/Target/LLVMIR/ModuleImport.cpp
+++ b/mlir/lib/Target/LLVMIR/ModuleImport.cpp
@@ -1637,6 +1637,7 @@ static void processMemoryEffects(llvm::Function *func, LLVMFuncOp funcOp) {
 static constexpr std::array ExplicitAttributes{
     StringLiteral("aarch64_pstate_sm_enabled"),
     StringLiteral("aarch64_pstate_sm_body"),
+    StringLiteral("aarch64_pstate_sm_compatible"),
     StringLiteral("aarch64_pstate_za_new"),
     StringLiteral("vscale_range"),
     StringLiteral("frame-pointer"),
@@ -1709,6 +1710,8 @@ void ModuleImport::processFunctionAttributes(llvm::Function *func,
     funcOp.setArmStreaming(true);
   else if (func->hasFnAttribute("aarch64_pstate_sm_body"))
     funcOp.setArmLocallyStreaming(true);
+  else if (func->hasFnAttribute("aarch64_pstate_sm_compatible"))
+    funcOp.setArmStreamingCompatible(true);
 
   if (func->hasFnAttribute("aarch64_pstate_za_new"))
     funcOp.setArmNewZa(true);
diff --git a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
index d6afe354178d66e9b7266c3c7d559a5b54189b7b..9f0e1f3c3bb6f6e799244b3a91ce44a17c935769 100644
--- a/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
+++ b/mlir/lib/Target/LLVMIR/ModuleTranslation.cpp
@@ -161,7 +161,7 @@ translateDataLayout(DataLayoutSpecInterface attribute,
   }
   layoutStream.flush();
   StringRef layoutSpec(llvmDataLayout);
-  if (layoutSpec.startswith("-"))
+  if (layoutSpec.starts_with("-"))
     layoutSpec = layoutSpec.drop_front();
 
   return llvm::DataLayout(layoutSpec);
@@ -795,7 +795,8 @@ LogicalResult ModuleTranslation::convertGlobals() {
 
       // Get the compile unit (scope) of the the global variable.
       if (llvm::DICompileUnit *compileUnit =
-              dyn_cast<llvm::DICompileUnit>(diGlobalVar->getScope())) {
+              dyn_cast_if_present<llvm::DICompileUnit>(
+                  diGlobalVar->getScope())) {
         // Update the compile unit with this incoming global variable expression
         // during the finalizing step later.
         allGVars[compileUnit].push_back(diGlobalExpr);
@@ -964,6 +965,8 @@ LogicalResult ModuleTranslation::convertOneFunction(LLVMFuncOp func) {
     llvmFunc->addFnAttr("aarch64_pstate_sm_enabled");
   else if (func.getArmLocallyStreaming())
     llvmFunc->addFnAttr("aarch64_pstate_sm_body");
+  else if (func.getArmStreamingCompatible())
+    llvmFunc->addFnAttr("aarch64_pstate_sm_compatible");
 
   if (func.getArmNewZa())
     llvmFunc->addFnAttr("aarch64_pstate_za_new");
diff --git a/mlir/lib/Target/SPIRV/Deserialization/DeserializeOps.cpp b/mlir/lib/Target/SPIRV/Deserialization/DeserializeOps.cpp
index 7510e1e2eb9b6f200f0e611fad99ea54d740bb25..954aaa98c329981a48db244dc57430b5021be587 100644
--- a/mlir/lib/Target/SPIRV/Deserialization/DeserializeOps.cpp
+++ b/mlir/lib/Target/SPIRV/Deserialization/DeserializeOps.cpp
@@ -354,7 +354,7 @@ Deserializer::processOp<spirv::EntryPointOp>(ArrayRef<uint32_t> words) {
     // The deserializer uses "spirv_fn_<id>" as the function name if the input
     // SPIR-V blob does not contain a name for it. We should use a more clear
     // indication for such case rather than relying on naming details.
-    if (!parsedFunc.getName().startswith("spirv_fn_"))
+    if (!parsedFunc.getName().starts_with("spirv_fn_"))
       return emitError(unknownLoc,
                        "function name mismatch between OpEntryPoint "
                        "and OpFunction with <id> ")
diff --git a/mlir/lib/Tools/PDLL/Parser/Parser.cpp b/mlir/lib/Tools/PDLL/Parser/Parser.cpp
index 93eb5e892e228e0e5fbf40f91c2d6188d9ffc9c9..cfbc4e4536fe8c454c2a1478c5619e7e8d712fcc 100644
--- a/mlir/lib/Tools/PDLL/Parser/Parser.cpp
+++ b/mlir/lib/Tools/PDLL/Parser/Parser.cpp
@@ -792,7 +792,7 @@ LogicalResult Parser::parseInclude(SmallVectorImpl<ast::Decl *> &decls) {
 
   // Check the type of include. If ending with `.pdll`, this is another pdl file
   // to be parsed along with the current module.
-  if (filename.endswith(".pdll")) {
+  if (filename.ends_with(".pdll")) {
     if (failed(lexer.pushInclude(filename, fileLoc)))
       return emitError(fileLoc,
                        "unable to open include file `" + filename + "`");
@@ -807,7 +807,7 @@ LogicalResult Parser::parseInclude(SmallVectorImpl<ast::Decl *> &decls) {
   }
 
   // Otherwise, this must be a `.td` include.
-  if (filename.endswith(".td"))
+  if (filename.ends_with(".td"))
     return parseTdInclude(filename, fileLoc, decls);
 
   return emitError(fileLoc,
diff --git a/mlir/lib/Tools/lsp-server-support/Protocol.cpp b/mlir/lib/Tools/lsp-server-support/Protocol.cpp
index 6d8cec6a54d1a4d7668a428753cd5c36197c3fd5..e110fdd97a38f13bdb50b7c5edeeedd6108b39ac 100644
--- a/mlir/lib/Tools/lsp-server-support/Protocol.cpp
+++ b/mlir/lib/Tools/lsp-server-support/Protocol.cpp
@@ -157,7 +157,7 @@ static llvm::Expected<std::string> uriFromAbsolutePath(StringRef absolutePath,
 
   // If authority if empty, we only print body if it starts with "/"; otherwise,
   // the URI is invalid.
-  if (!authority.empty() || StringRef(body).startswith("/")) {
+  if (!authority.empty() || StringRef(body).starts_with("/")) {
     uri.append("//");
     percentEncode(authority, uri);
   }
@@ -167,7 +167,7 @@ static llvm::Expected<std::string> uriFromAbsolutePath(StringRef absolutePath,
 
 static llvm::Expected<std::string> getAbsolutePath(StringRef authority,
                                                    StringRef body) {
-  if (!body.startswith("/"))
+  if (!body.starts_with("/"))
     return llvm::createStringError(
         llvm::inconvertibleErrorCode(),
         "File scheme: expect body to be an absolute path starting "
diff --git a/mlir/lib/Tools/lsp-server-support/SourceMgrUtils.cpp b/mlir/lib/Tools/lsp-server-support/SourceMgrUtils.cpp
index b6f4f3cc1869edb639467032d146f5a992d79a78..f8d348aba5a3cc90fee7778e61fbc4d71b063fc3 100644
--- a/mlir/lib/Tools/lsp-server-support/SourceMgrUtils.cpp
+++ b/mlir/lib/Tools/lsp-server-support/SourceMgrUtils.cpp
@@ -99,7 +99,7 @@ lsp::extractSourceDocComment(llvm::SourceMgr &sourceMgr, SMLoc loc) {
   SmallVector<StringRef> commentLines;
   while (std::optional<StringRef> line = popLastLine()) {
     // Check for a comment at the beginning of the line.
-    if (!line->startswith("//"))
+    if (!line->starts_with("//"))
       break;
 
     // Extract the document string from the comment.
diff --git a/mlir/lib/Tools/lsp-server-support/Transport.cpp b/mlir/lib/Tools/lsp-server-support/Transport.cpp
index 3915146f6a66d1bc546812190a42b4ced0e3f11c..df675cf78210be8f6b4d82f0bd0a678b25b3990d 100644
--- a/mlir/lib/Tools/lsp-server-support/Transport.cpp
+++ b/mlir/lib/Tools/lsp-server-support/Transport.cpp
@@ -345,7 +345,7 @@ LogicalResult JSONTransport::readDelimitedMessage(std::string &json) {
   llvm::SmallString<128> line;
   while (succeeded(readLine(in, line))) {
     StringRef lineRef = line.str().trim();
-    if (lineRef.startswith("//")) {
+    if (lineRef.starts_with("//")) {
       // Found a delimiter for the message.
       if (lineRef == "// -----")
         break;
diff --git a/mlir/lib/Tools/mlir-lsp-server/LSPServer.cpp b/mlir/lib/Tools/mlir-lsp-server/LSPServer.cpp
index d272164b1d2677c8979bdfdbb9a00546c48e66d6..0f23366f6fe80a8afdc4605d38377c2c44fabf82 100644
--- a/mlir/lib/Tools/mlir-lsp-server/LSPServer.cpp
+++ b/mlir/lib/Tools/mlir-lsp-server/LSPServer.cpp
@@ -253,7 +253,8 @@ void LSPServer::onCodeAction(const CodeActionParams &params,
     if (only.empty())
       return true;
     return llvm::any_of(only, [&](StringRef base) {
-      return kind.consume_front(base) && (kind.empty() || kind.startswith("."));
+      return kind.consume_front(base) &&
+             (kind.empty() || kind.starts_with("."));
     });
   };
 
diff --git a/mlir/lib/Tools/mlir-lsp-server/MLIRServer.cpp b/mlir/lib/Tools/mlir-lsp-server/MLIRServer.cpp
index 9850c9725607598f22991180e5c5166dd6d4fb9e..de657a3df9ef7b49e4e65ba8801abaccbf16448c 100644
--- a/mlir/lib/Tools/mlir-lsp-server/MLIRServer.cpp
+++ b/mlir/lib/Tools/mlir-lsp-server/MLIRServer.cpp
@@ -181,7 +181,7 @@ static unsigned getBlockNumber(Block *block) {
 static void printDefBlockName(raw_ostream &os, Block *block, SMRange loc = {}) {
   // Try to extract a name from the source location.
   std::optional<StringRef> text = getTextFromRange(loc);
-  if (text && text->startswith("^")) {
+  if (text && text->starts_with("^")) {
     os << *text;
     return;
   }
@@ -900,7 +900,7 @@ void MLIRDocument::getCodeActionForDiagnostic(
   // Ignore diagnostics that print the current operation. These are always
   // enabled for the language server, but not generally during normal
   // parsing/verification.
-  if (message.startswith("see current operation: "))
+  if (message.starts_with("see current operation: "))
     return;
 
   // Get the start of the line containing the diagnostic.
diff --git a/mlir/lib/Tools/mlir-pdll-lsp-server/PDLLServer.cpp b/mlir/lib/Tools/mlir-pdll-lsp-server/PDLLServer.cpp
index 8efc61476731736a4540d7c14b7591599ef9926f..a5c6c2bb2c6a0cf4f47d2ecb35ffbf80907a189e 100644
--- a/mlir/lib/Tools/mlir-pdll-lsp-server/PDLLServer.cpp
+++ b/mlir/lib/Tools/mlir-pdll-lsp-server/PDLLServer.cpp
@@ -946,7 +946,7 @@ public:
           break;
         case llvm::sys::fs::file_type::regular_file: {
           // Only consider concrete files that can actually be included by PDLL.
-          if (filename.endswith(".pdll") || filename.endswith(".td"))
+          if (filename.ends_with(".pdll") || filename.ends_with(".td"))
             addIncludeCompletion(filename, /*isDirectory=*/false);
           break;
         }
diff --git a/mlir/lib/Transforms/Utils/FoldUtils.cpp b/mlir/lib/Transforms/Utils/FoldUtils.cpp
index dfc63ed6c4a542a69a8eb4be3c087305619e13d7..90ee5ba51de3add57e0ed6f464624da240ca4edf 100644
--- a/mlir/lib/Transforms/Utils/FoldUtils.cpp
+++ b/mlir/lib/Transforms/Utils/FoldUtils.cpp
@@ -141,7 +141,6 @@ bool OperationFolder::insertKnownConstant(Operation *op, Attribute constValue) {
   // If there is an existing constant, replace `op`.
   if (folderConstOp) {
     notifyRemoval(op);
-    appendFoldedLocation(folderConstOp, op->getLoc());
     rewriter.replaceOp(op, folderConstOp->getResults());
     return false;
   }
@@ -295,10 +294,8 @@ OperationFolder::tryGetOrCreateConstant(ConstantMap &uniquedConstants,
   // Check if an existing mapping already exists.
   auto constKey = std::make_tuple(dialect, value, type);
   Operation *&constOp = uniquedConstants[constKey];
-  if (constOp) {
-    appendFoldedLocation(constOp, loc);
+  if (constOp)
     return constOp;
-  }
 
   // If one doesn't exist, try to materialize one.
   if (!(constOp = materializeConstant(dialect, rewriter, value, type, loc)))
@@ -319,7 +316,6 @@ OperationFolder::tryGetOrCreateConstant(ConstantMap &uniquedConstants,
   // materialized operation in favor of the existing one.
   if (auto *existingOp = uniquedConstants.lookup(newKey)) {
     notifyRemoval(constOp);
-    appendFoldedLocation(existingOp, constOp->getLoc());
     rewriter.eraseOp(constOp);
     referencedDialects[existingOp].push_back(dialect);
     return constOp = existingOp;
@@ -330,32 +326,3 @@ OperationFolder::tryGetOrCreateConstant(ConstantMap &uniquedConstants,
   auto newIt = uniquedConstants.insert({newKey, constOp});
   return newIt.first->second;
 }
-
-void OperationFolder::appendFoldedLocation(Operation *retainedOp,
-                                           Location foldedLocation) {
-  // Append into existing fused location if it has the same tag.
-  if (auto existingFusedLoc =
-          dyn_cast<FusedLocWith<StringAttr>>(retainedOp->getLoc())) {
-    StringAttr existingMetadata = existingFusedLoc.getMetadata();
-    if (existingMetadata == fusedLocationTag) {
-      ArrayRef<Location> existingLocations = existingFusedLoc.getLocations();
-      SetVector<Location> locations(existingLocations.begin(),
-                                    existingLocations.end());
-      locations.insert(foldedLocation);
-      Location newFusedLoc = FusedLoc::get(
-          retainedOp->getContext(), locations.takeVector(), existingMetadata);
-      retainedOp->setLoc(newFusedLoc);
-      return;
-    }
-  }
-
-  // Create a new fusedloc with retainedOp's loc and foldedLocation.
-  // If they're already equal, no need to fuse.
-  if (retainedOp->getLoc() == foldedLocation)
-    return;
-
-  Location newFusedLoc =
-      FusedLoc::get(retainedOp->getContext(),
-                    {retainedOp->getLoc(), foldedLocation}, fusedLocationTag);
-  retainedOp->setLoc(newFusedLoc);
-}
diff --git a/mlir/test/Dialect/ArmSME/arm-sme-to-llvm.mlir b/mlir/test/Conversion/ArmSMEToLLVM/arm-sme-to-llvm.mlir
similarity index 100%
rename from mlir/test/Dialect/ArmSME/arm-sme-to-llvm.mlir
rename to mlir/test/Conversion/ArmSMEToLLVM/arm-sme-to-llvm.mlir
diff --git a/mlir/test/Conversion/ArmSMEToLLVM/unsupported.mlir b/mlir/test/Conversion/ArmSMEToLLVM/unsupported.mlir
new file mode 100644
index 0000000000000000000000000000000000000000..59665c471921d58215bf2b5ea48d874ac40deb0c
--- /dev/null
+++ b/mlir/test/Conversion/ArmSMEToLLVM/unsupported.mlir
@@ -0,0 +1,14 @@
+// RUN: mlir-opt %s -allocate-arm-sme-tiles -convert-arm-sme-to-llvm -split-input-file -allow-unregistered-dialect -verify-diagnostics
+
+//===----------------------------------------------------------------------===//
+// arm_sme.outerproduct
+//===----------------------------------------------------------------------===//
+
+func.func @arm_sme_outerproduct_unsupported_type(%lhs : vector<[16]xi8>, %rhs : vector<[16]xi8>) {
+  %acc = arm_sme.get_tile : vector<[16]x[16]xi8>
+  // expected-error@+2 {{failed to legalize operation 'arm_sme.outerproduct'}}
+  // expected-error@+1 {{unsupported type}}
+  %0 = arm_sme.outerproduct %lhs, %rhs  acc(%acc) : vector<[16]xi8>, vector<[16]xi8>
+  "prevent.dce"(%0) : (vector<[16]x[16]xi8>) -> ()
+}
+
diff --git a/mlir/test/Conversion/MathToLibm/convert-to-libm.mlir b/mlir/test/Conversion/MathToLibm/convert-to-libm.mlir
index f0c4512cbfdcc7bb6ba4c48ac21244635b3b1e1d..eb9226dee2619d596174a7a9442cdce8588454cd 100644
--- a/mlir/test/Conversion/MathToLibm/convert-to-libm.mlir
+++ b/mlir/test/Conversion/MathToLibm/convert-to-libm.mlir
@@ -24,6 +24,8 @@
 // CHECK-DAG: @truncf(f32) -> f32 attributes {llvm.readnone}
 // CHECK-DAG: @cos(f64) -> f64 attributes {llvm.readnone}
 // CHECK-DAG: @cosf(f32) -> f32 attributes {llvm.readnone}
+// CHECK-DAG: @cosh(f64) -> f64 attributes {llvm.readnone}
+// CHECK-DAG: @coshf(f32) -> f32 attributes {llvm.readnone}
 // CHECK-DAG: @sin(f64) -> f64 attributes {llvm.readnone}
 // CHECK-DAG: @sinf(f32) -> f32 attributes {llvm.readnone}
 // CHECK-DAG: @floor(f64) -> f64 attributes {llvm.readnone}
@@ -127,6 +129,18 @@ func.func @tanh_caller(%float: f32, %double: f64) -> (f32, f64)  {
   return %float_result, %double_result : f32, f64
 }
 
+// CHECK-LABEL: func @cosh_caller
+// CHECK-SAME: %[[FLOAT:.*]]: f32
+// CHECK-SAME: %[[DOUBLE:.*]]: f64
+func.func @cosh_caller(%float: f32, %double: f64) -> (f32, f64)  {
+  // CHECK-DAG: %[[FLOAT_RESULT:.*]] = call @coshf(%[[FLOAT]]) : (f32) -> f32
+  %float_result = math.cosh %float : f32
+  // CHECK-DAG: %[[DOUBLE_RESULT:.*]] = call @cosh(%[[DOUBLE]]) : (f64) -> f64
+  %double_result = math.cosh %double : f64
+  // CHECK: return %[[FLOAT_RESULT]], %[[DOUBLE_RESULT]]
+  return %float_result, %double_result : f32, f64
+}
+
 // CHECK-LABEL: func @atan2_caller
 // CHECK-SAME: %[[FLOAT:.*]]: f32
 // CHECK-SAME: %[[DOUBLE:.*]]: f64
diff --git a/mlir/test/Conversion/SCFToOpenMP/scf-to-openmp.mlir b/mlir/test/Conversion/SCFToOpenMP/scf-to-openmp.mlir
index e0fdcae1b896b8a6205159e12c27a94935c914a2..acd2690c56e2e67d5a2de9562d5e8f061f5d2140 100644
--- a/mlir/test/Conversion/SCFToOpenMP/scf-to-openmp.mlir
+++ b/mlir/test/Conversion/SCFToOpenMP/scf-to-openmp.mlir
@@ -1,9 +1,10 @@
-// RUN: mlir-opt -convert-scf-to-openmp %s | FileCheck %s
+// RUN: mlir-opt -convert-scf-to-openmp='num-threads=4' %s | FileCheck %s
 
 // CHECK-LABEL: @parallel
 func.func @parallel(%arg0: index, %arg1: index, %arg2: index,
           %arg3: index, %arg4: index, %arg5: index) {
-  // CHECK: omp.parallel {
+  // CHECK: %[[FOUR:.+]] = llvm.mlir.constant(4 : i32) : i32
+  // CHECK: omp.parallel num_threads(%[[FOUR]] : i32) {
   // CHECK: omp.wsloop for (%[[LVAR1:.*]], %[[LVAR2:.*]]) : index = (%arg0, %arg1) to (%arg2, %arg3) step (%arg4, %arg5) {
   // CHECK: memref.alloca_scope
   scf.parallel (%i, %j) = (%arg0, %arg1) to (%arg2, %arg3) step (%arg4, %arg5) {
@@ -20,7 +21,8 @@ func.func @parallel(%arg0: index, %arg1: index, %arg2: index,
 // CHECK-LABEL: @nested_loops
 func.func @nested_loops(%arg0: index, %arg1: index, %arg2: index,
                    %arg3: index, %arg4: index, %arg5: index) {
-  // CHECK: omp.parallel {
+  // CHECK: %[[FOUR:.+]] = llvm.mlir.constant(4 : i32) : i32
+  // CHECK: omp.parallel num_threads(%[[FOUR]] : i32) {
   // CHECK: omp.wsloop for (%[[LVAR_OUT1:.*]]) : index = (%arg0) to (%arg2) step (%arg4) {
     // CHECK: memref.alloca_scope
   scf.parallel (%i) = (%arg0) to (%arg2) step (%arg4) {
@@ -43,7 +45,8 @@ func.func @nested_loops(%arg0: index, %arg1: index, %arg2: index,
 // CHECK-LABEL: @adjacent_loops
 func.func @adjacent_loops(%arg0: index, %arg1: index, %arg2: index,
                      %arg3: index, %arg4: index, %arg5: index) {
-  // CHECK: omp.parallel {
+  // CHECK: %[[FOUR:.+]] = llvm.mlir.constant(4 : i32) : i32
+  // CHECK: omp.parallel num_threads(%[[FOUR]] : i32) {
   // CHECK: omp.wsloop for (%[[LVAR_AL1:.*]]) : index = (%arg0) to (%arg2) step (%arg4) {
   // CHECK: memref.alloca_scope
   scf.parallel (%i) = (%arg0) to (%arg2) step (%arg4) {
@@ -55,7 +58,8 @@ func.func @adjacent_loops(%arg0: index, %arg1: index, %arg2: index,
   // CHECK:   omp.terminator
   // CHECK: }
 
-  // CHECK: omp.parallel {
+  // CHECK: %[[FOUR:.+]] = llvm.mlir.constant(4 : i32) : i32
+  // CHECK: omp.parallel num_threads(%[[FOUR]] : i32) {
   // CHECK: omp.wsloop for (%[[LVAR_AL2:.*]]) : index = (%arg1) to (%arg3) step (%arg5) {
   // CHECK: memref.alloca_scope
   scf.parallel (%j) = (%arg1) to (%arg3) step (%arg5) {
diff --git a/mlir/test/Conversion/VectorToArmSME/unsupported.mlir b/mlir/test/Conversion/VectorToArmSME/unsupported.mlir
new file mode 100644
index 0000000000000000000000000000000000000000..3ef283727edd496211f314b61ae237d68c7d9c53
--- /dev/null
+++ b/mlir/test/Conversion/VectorToArmSME/unsupported.mlir
@@ -0,0 +1,176 @@
+// RUN: mlir-opt %s -convert-vector-to-arm-sme -split-input-file -allow-unregistered-dialect -verify-diagnostics | FileCheck %s
+
+//===----------------------------------------------------------------------===//
+// vector.transfer_read
+//===----------------------------------------------------------------------===//
+
+// CHECK-LABEL: @transfer_read_2d__bad_type
+// CHECK-NOT: arm_sme.tile_load
+// CHECK: vector.transfer_read
+func.func @transfer_read_2d__bad_type(%src : memref<?x?xf64>) {
+  %c0 = arith.constant 0 : index
+  %pad = arith.constant 0.0 : f64
+  %0 = vector.transfer_read %src[%c0, %c0], %pad {permutation_map = affine_map<(d0, d1) -> (d1, d0)>, in_bounds = [false, false]} : memref<?x?xf64>, vector<[4]x[4]xf64>
+  "prevent.dce"(%0) : (vector<[4]x[4]xf64>) -> ()
+  return
+}
+
+// -----
+
+// CHECK-LABEL: @transfer_read_2d__non_memref_type
+// CHECK-NOT: arm_sme.tile_load
+// CHECK: vector.transfer_read
+func.func @transfer_read_2d__non_memref_type(%src : tensor<?x?xf64>) {
+  %c0 = arith.constant 0 : index
+  %pad = arith.constant 0.0 : f64
+  %0 = vector.transfer_read %src[%c0, %c0], %pad {permutation_map = affine_map<(d0, d1) -> (d1, d0)>, in_bounds = [true, true]} : tensor<?x?xf64>, vector<[2]x[2]xf64>
+  "prevent.dce"(%0) : (vector<[2]x[2]xf64>) -> ()
+  return
+}
+
+// -----
+
+// CHECK-LABEL: @transfer_read_2d__bad_transfer_rank
+// CHECK-NOT: arm_sme.tile_load
+// CHECK: vector.transfer_read
+func.func @transfer_read_2d__bad_transfer_rank(%src : memref<?x?xf64>) {
+  %c0 = arith.constant 0 : index
+  %pad = arith.constant 0.0 : f64
+  %0 = vector.transfer_read %src[%c0, %c0], %pad {permutation_map = affine_map<(d0, d1) -> (d0)>, in_bounds = [true]} : memref<?x?xf64>, vector<[2]xf64>
+  "prevent.dce"(%0) : (vector<[2]xf64>) -> ()
+  return
+}
+
+// -----
+
+// CHECK-LABEL: @transfer_read_2d__non_transpose
+// CHECK-NOT: arm_sme.tile_load
+// CHECK: vector.transfer_read
+func.func @transfer_read_2d__non_transpose(%src : memref<?x?xf64>) {
+  %c0 = arith.constant 0 : index
+  %pad = arith.constant 0.0 : f64
+  %0 = vector.transfer_read %src[%c0, %c0], %pad {permutation_map = affine_map<(d0, d1) -> (d0, 0)>, in_bounds = [true, true]} : memref<?x?xf64>, vector<[2]x[2]xf64>
+  "prevent.dce"(%0) : (vector<[2]x[2]xf64>) -> ()
+  return
+}
+
+// -----
+
+// CHECK-LABEL: @transfer_read_2d__out_of_bounds
+// CHECK-NOT: arm_sme.tile_load
+// CHECK: vector.transfer_read
+func.func @transfer_read_2d__out_of_bounds(%src : memref<?x?xf64>) {
+  %c0 = arith.constant 0 : index
+  %pad = arith.constant 0.0 : f64
+  %0 = vector.transfer_read %src[%c0, %c0], %pad {permutation_map = affine_map<(d0, d1) -> (d1, d0)>, in_bounds = [false, false]} : memref<?x?xf64>, vector<[2]x[2]xf64>
+  "prevent.dce"(%0) : (vector<[2]x[2]xf64>) -> ()
+  return
+}
+
+//===----------------------------------------------------------------------===//
+// vector.transfer_write
+//===----------------------------------------------------------------------===//
+
+// -----
+
+// The following tests check the 'vector.transfer_write' -> 'arm_sme.intr.zero'
+// lowering only occurs for vector types of correct rank, shape, element size
+// and number of scalable dims.
+
+// CHECK-LABEL: @transfer_write_2d_zero__bad_type
+// CHECK: vector.transfer_write
+// CHECK-NOT: arm_sme.intr.zero
+func.func @transfer_write_2d_zero__bad_type(%arg0 : memref<?x?xi4>) {
+  %c0 = arith.constant 0 : index
+  %cst = arith.constant dense<0> : vector<[16]x[16]xi4>
+  vector.transfer_write %cst, %arg0[%c0, %c0] {in_bounds = [true, true]} : vector<[16]x[16]xi4>, memref<?x?xi4>
+  return
+}
+
+// -----
+
+// CHECK-LABEL: @transfer_write_2d_zero__bad_shape
+// CHECK: vector.transfer_write
+// CHECK-NOT: arm_sme.tile_store
+func.func @transfer_write_2d_zero__bad_shape(%arg0 : memref<?x?xi8>) {
+  %c0 = arith.constant 0 : index
+  %cst = arith.constant dense<0> : vector<[8]x[8]xi8>
+  vector.transfer_write %cst, %arg0[%c0, %c0] {in_bounds = [true, true]} : vector<[8]x[8]xi8>, memref<?x?xi8>
+  return
+}
+
+// -----
+
+// CHECK-LABEL: @transfer_write_2d_zero__bad_rank
+// CHECK: vector.transfer_write
+// CHECK-NOT: arm_sme.tile_store
+func.func @transfer_write_2d_zero__bad_rank(%arg0 : memref<?x?x?xi8>) {
+  %c0 = arith.constant 0 : index
+  %cst = arith.constant dense<0> : vector<[16]x[16]x[16]xi8>
+  vector.transfer_write %cst, %arg0[%c0, %c0, %c0] {in_bounds = [true, true, true]} : vector<[16]x[16]x[16]xi8>, memref<?x?x?xi8>
+  return
+}
+
+// -----
+
+// CHECK-LABEL: @transfer_write_2d_zero__non_memref_type
+// CHECK: vector.transfer_write
+// CHECK-NOT: arm_sme.tile_store
+func.func @transfer_write_2d_zero__non_memref_type(%arg0 : tensor<?x?xi8>) -> tensor<?x?xi8> {
+  %c0 = arith.constant 0 : index
+  %cst = arith.constant dense<0> : vector<[16]x[16]xi8>
+  %0 = vector.transfer_write %cst, %arg0[%c0, %c0] {in_bounds = [true, true]} : vector<[16]x[16]xi8>, tensor<?x?xi8>
+  return %0 : tensor<?x?xi8>
+}
+
+// -----
+
+// CHECK-LABEL: @transfer_write_2d__fixed
+// CHECK: vector.transfer_write
+// CHECK-NOT: arm_sme.tile_store
+func.func @transfer_write_2d__fixed(%vector : vector<16x16xi8>, %dest : memref<?x?xi8>) {
+  %c0 = arith.constant 0 : index
+  vector.transfer_write %vector, %dest[%c0, %c0] {in_bounds = [true, true]} : vector<16x16xi8>, memref<?x?xi8>
+  return
+}
+
+// -----
+
+// CHECK-LABEL: @transfer_write_2d__out_of_bounds
+// CHECK: vector.transfer_write
+// CHECK-NOT: arm_sme.tile_store
+func.func @transfer_write_2d__out_of_bounds(%vector : vector<[4]x[4]xf32>, %dest : memref<?x?xf32>) {
+  %c0 = arith.constant 0 : index
+  vector.transfer_write %vector, %dest[%c0, %c0] {in_bounds = [false, false]} : vector<[4]x[4]xf32>, memref<?x?xf32>
+  return
+}
+
+//===----------------------------------------------------------------------===//
+// vector.outerproduct
+//===----------------------------------------------------------------------===//
+
+// -----
+
+func.func @vector_outerproduct_unsupported_axpy(%lhs : vector<[2]xf64>, %rhs : f64, %acc : vector<[2]xf64>) -> vector<[2]xf64> {
+  // expected-error@+1 {{AXPY operations not supported}}
+  %0 = vector.outerproduct %lhs, %rhs, %acc {kind = #vector.kind<mul>} : vector<[2]xf64>, f64
+  return %0 : vector<[2]xf64>
+}
+
+// -----
+
+func.func @vector_outerproduct_unsupported_kind(%lhs : vector<[2]xf64>, %rhs : vector<[2]xf64>) {
+  %acc = arm_sme.get_tile : vector<[2]x[2]xf64>
+  // expected-error@+1 {{unsupported kind}}
+  %0 = vector.outerproduct %lhs, %rhs, %acc {kind = #vector.kind<mul>} : vector<[2]xf64>, vector<[2]xf64>
+  "prevent.dce"(%0) : (vector<[2]x[2]xf64>) -> ()
+}
+
+// -----
+
+func.func @vector_outerproduct_unknown_mask(%lhs : vector<[4]xf32>, %rhs : vector<[4]xf32>, %mask : vector<[4]x[4]xi1>) {
+  // CHECK: vector.outerproduct
+  %acc = arm_sme.get_tile : vector<[4]x[4]xf32>
+  %0 = vector.mask %mask { vector.outerproduct %lhs, %rhs, %acc {kind = #vector.kind<add>} : vector<[4]xf32>, vector<[4]xf32> } : vector<[4]x[4]xi1> -> vector<[4]x[4]xf32>
+  "prevent.dce"(%0) : (vector<[4]x[4]xf32>) -> ()
+}
diff --git a/mlir/test/Dialect/ArmSME/vector-ops-to-sme.mlir b/mlir/test/Conversion/VectorToArmSME/vector-to-arm-sme.mlir
similarity index 85%
rename from mlir/test/Dialect/ArmSME/vector-ops-to-sme.mlir
rename to mlir/test/Conversion/VectorToArmSME/vector-to-arm-sme.mlir
index 6ea949d9c16509cab37b57ac4f175866bfbf04ef..6783263c184961bc6d71b084dde2d4d46780b4cf 100644
--- a/mlir/test/Dialect/ArmSME/vector-ops-to-sme.mlir
+++ b/mlir/test/Conversion/VectorToArmSME/vector-to-arm-sme.mlir
@@ -150,71 +150,6 @@ func.func @transfer_read_2d_transpose_with_mask_f32(%src : memref<?x?xf32>, %mas
 
 // -----
 
-// CHECK-LABEL: @transfer_read_2d__bad_type
-// CHECK-NOT: arm_sme.tile_load
-// CHECK: vector.transfer_read
-func.func @transfer_read_2d__bad_type(%src : memref<?x?xf64>) {
-  %c0 = arith.constant 0 : index
-  %pad = arith.constant 0.0 : f64
-  %0 = vector.transfer_read %src[%c0, %c0], %pad {permutation_map = affine_map<(d0, d1) -> (d1, d0)>, in_bounds = [false, false]} : memref<?x?xf64>, vector<[4]x[4]xf64>
-  "prevent.dce"(%0) : (vector<[4]x[4]xf64>) -> ()
-  return
-}
-
-// -----
-
-// CHECK-LABEL: @transfer_read_2d__non_memref_type
-// CHECK-NOT: arm_sme.tile_load
-// CHECK: vector.transfer_read
-func.func @transfer_read_2d__non_memref_type(%src : tensor<?x?xf64>) {
-  %c0 = arith.constant 0 : index
-  %pad = arith.constant 0.0 : f64
-  %0 = vector.transfer_read %src[%c0, %c0], %pad {permutation_map = affine_map<(d0, d1) -> (d1, d0)>, in_bounds = [true, true]} : tensor<?x?xf64>, vector<[2]x[2]xf64>
-  "prevent.dce"(%0) : (vector<[2]x[2]xf64>) -> ()
-  return
-}
-
-// -----
-
-// CHECK-LABEL: @transfer_read_2d__bad_transfer_rank
-// CHECK-NOT: arm_sme.tile_load
-// CHECK: vector.transfer_read
-func.func @transfer_read_2d__bad_transfer_rank(%src : memref<?x?xf64>) {
-  %c0 = arith.constant 0 : index
-  %pad = arith.constant 0.0 : f64
-  %0 = vector.transfer_read %src[%c0, %c0], %pad {permutation_map = affine_map<(d0, d1) -> (d0)>, in_bounds = [true]} : memref<?x?xf64>, vector<[2]xf64>
-  "prevent.dce"(%0) : (vector<[2]xf64>) -> ()
-  return
-}
-
-// -----
-
-// CHECK-LABEL: @transfer_read_2d__non_transpose
-// CHECK-NOT: arm_sme.tile_load
-// CHECK: vector.transfer_read
-func.func @transfer_read_2d__non_transpose(%src : memref<?x?xf64>) {
-  %c0 = arith.constant 0 : index
-  %pad = arith.constant 0.0 : f64
-  %0 = vector.transfer_read %src[%c0, %c0], %pad {permutation_map = affine_map<(d0, d1) -> (d0, 0)>, in_bounds = [true, true]} : memref<?x?xf64>, vector<[2]x[2]xf64>
-  "prevent.dce"(%0) : (vector<[2]x[2]xf64>) -> ()
-  return
-}
-
-// -----
-
-// CHECK-LABEL: @transfer_read_2d__out_of_bounds
-// CHECK-NOT: arm_sme.tile_load
-// CHECK: vector.transfer_read
-func.func @transfer_read_2d__out_of_bounds(%src : memref<?x?xf64>) {
-  %c0 = arith.constant 0 : index
-  %pad = arith.constant 0.0 : f64
-  %0 = vector.transfer_read %src[%c0, %c0], %pad {permutation_map = affine_map<(d0, d1) -> (d1, d0)>, in_bounds = [false, false]} : memref<?x?xf64>, vector<[2]x[2]xf64>
-  "prevent.dce"(%0) : (vector<[2]x[2]xf64>) -> ()
-  return
-}
-
-// -----
-
 //===----------------------------------------------------------------------===//
 // vector.transfer_write
 //===----------------------------------------------------------------------===//
@@ -366,80 +301,6 @@ func.func @transfer_write_2d_transpose_with_mask_bf16(%vector : vector<[8]x[8]xb
   return
 }
 
-// -----
-
-// The following tests check the 'vector.transfer_write' -> 'arm_sme.intr.zero'
-// lowering only occurs for vector types of correct rank, shape, element size
-// and number of scalable dims.
-
-// CHECK-LABEL: @transfer_write_2d_zero__bad_type
-// CHECK: vector.transfer_write
-// CHECK-NOT: arm_sme.intr.zero
-func.func @transfer_write_2d_zero__bad_type(%arg0 : memref<?x?xi4>) {
-  %c0 = arith.constant 0 : index
-  %cst = arith.constant dense<0> : vector<[16]x[16]xi4>
-  vector.transfer_write %cst, %arg0[%c0, %c0] {in_bounds = [true, true]} : vector<[16]x[16]xi4>, memref<?x?xi4>
-  return
-}
-
-// -----
-
-// CHECK-LABEL: @transfer_write_2d_zero__bad_shape
-// CHECK: vector.transfer_write
-// CHECK-NOT: arm_sme.tile_store
-func.func @transfer_write_2d_zero__bad_shape(%arg0 : memref<?x?xi8>) {
-  %c0 = arith.constant 0 : index
-  %cst = arith.constant dense<0> : vector<[8]x[8]xi8>
-  vector.transfer_write %cst, %arg0[%c0, %c0] {in_bounds = [true, true]} : vector<[8]x[8]xi8>, memref<?x?xi8>
-  return
-}
-
-// -----
-
-// CHECK-LABEL: @transfer_write_2d_zero__bad_rank
-// CHECK: vector.transfer_write
-// CHECK-NOT: arm_sme.tile_store
-func.func @transfer_write_2d_zero__bad_rank(%arg0 : memref<?x?x?xi8>) {
-  %c0 = arith.constant 0 : index
-  %cst = arith.constant dense<0> : vector<[16]x[16]x[16]xi8>
-  vector.transfer_write %cst, %arg0[%c0, %c0, %c0] {in_bounds = [true, true, true]} : vector<[16]x[16]x[16]xi8>, memref<?x?x?xi8>
-  return
-}
-
-// -----
-
-// CHECK-LABEL: @transfer_write_2d_zero__non_memref_type
-// CHECK: vector.transfer_write
-// CHECK-NOT: arm_sme.tile_store
-func.func @transfer_write_2d_zero__non_memref_type(%arg0 : tensor<?x?xi8>) -> tensor<?x?xi8> {
-  %c0 = arith.constant 0 : index
-  %cst = arith.constant dense<0> : vector<[16]x[16]xi8>
-  %0 = vector.transfer_write %cst, %arg0[%c0, %c0] {in_bounds = [true, true]} : vector<[16]x[16]xi8>, tensor<?x?xi8>
-  return %0 : tensor<?x?xi8>
-}
-
-// -----
-
-// CHECK-LABEL: @transfer_write_2d__fixed
-// CHECK: vector.transfer_write
-// CHECK-NOT: arm_sme.tile_store
-func.func @transfer_write_2d__fixed(%vector : vector<16x16xi8>, %dest : memref<?x?xi8>) {
-  %c0 = arith.constant 0 : index
-  vector.transfer_write %vector, %dest[%c0, %c0] {in_bounds = [true, true]} : vector<16x16xi8>, memref<?x?xi8>
-  return
-}
-
-// -----
-
-// CHECK-LABEL: @transfer_write_2d__out_of_bounds
-// CHECK: vector.transfer_write
-// CHECK-NOT: arm_sme.tile_store
-func.func @transfer_write_2d__out_of_bounds(%vector : vector<[4]x[4]xf32>, %dest : memref<?x?xf32>) {
-  %c0 = arith.constant 0 : index
-  vector.transfer_write %vector, %dest[%c0, %c0] {in_bounds = [false, false]} : vector<[4]x[4]xf32>, memref<?x?xf32>
-  return
-}
-
 //===----------------------------------------------------------------------===//
 // vector.broadcast
 //===----------------------------------------------------------------------===//
diff --git a/mlir/test/Dialect/ArmSME/enable-arm-streaming.mlir b/mlir/test/Dialect/ArmSME/enable-arm-streaming.mlir
index b1188acbc0b2d7476bb12b9f293252838fc6bf7c..6b58d8fdc41b0e070e8e52098cd5e951edc42930 100644
--- a/mlir/test/Dialect/ArmSME/enable-arm-streaming.mlir
+++ b/mlir/test/Dialect/ArmSME/enable-arm-streaming.mlir
@@ -1,5 +1,6 @@
 // RUN: mlir-opt %s -enable-arm-streaming -verify-diagnostics | FileCheck %s
 // RUN: mlir-opt %s -enable-arm-streaming=streaming-mode=streaming-locally -verify-diagnostics | FileCheck %s -check-prefix=CHECK-LOCALLY
+// RUN: mlir-opt %s -enable-arm-streaming=streaming-mode=streaming-compatible -verify-diagnostics | FileCheck %s -check-prefix=CHECK-COMPATIBLE
 // RUN: mlir-opt %s -enable-arm-streaming=za-mode=new-za -verify-diagnostics | FileCheck %s -check-prefix=CHECK-ENABLE-ZA
 // RUN: mlir-opt %s -enable-arm-streaming=only-if-required-by-ops -verify-diagnostics | FileCheck %s -check-prefix=IF-REQUIRED
 
@@ -7,6 +8,8 @@
 // CHECK-SAME: attributes {arm_streaming}
 // CHECK-LOCALLY-LABEL: @arm_streaming
 // CHECK-LOCALLY-SAME: attributes {arm_locally_streaming}
+// CHECK-COMPATIBLE-LABEL: @arm_streaming
+// CHECK-COMPATIBLE-SAME: attributes {arm_streaming_compatible}
 // CHECK-ENABLE-ZA-LABEL: @arm_streaming
 // CHECK-ENABLE-ZA-SAME: attributes {arm_new_za, arm_streaming}
 func.func @arm_streaming() { return }
@@ -15,6 +18,8 @@ func.func @arm_streaming() { return }
 // CHECK-SAME: attributes {enable_arm_streaming_ignore}
 // CHECK-LOCALLY-LABEL: @not_arm_streaming
 // CHECK-LOCALLY-SAME: attributes {enable_arm_streaming_ignore}
+// CHECK-COMPATIBLE-LABEL: @not_arm_streaming
+// CHECK-COMPATIBLE-SAME: attributes {enable_arm_streaming_ignore}
 // CHECK-ENABLE-ZA-LABEL: @not_arm_streaming
 // CHECK-ENABLE-ZA-SAME: attributes {enable_arm_streaming_ignore}
 func.func @not_arm_streaming() attributes {enable_arm_streaming_ignore} { return }
diff --git a/mlir/test/Dialect/ArmSME/vector-ops-to-llvm.mlir b/mlir/test/Dialect/ArmSME/vector-ops-to-llvm.mlir
index d16d250b70eb3f69589145dca2c77a5a13dec619..ce5bfd25cbdbccb559e221d3d7a8e19f7e570f48 100644
--- a/mlir/test/Dialect/ArmSME/vector-ops-to-llvm.mlir
+++ b/mlir/test/Dialect/ArmSME/vector-ops-to-llvm.mlir
@@ -469,42 +469,6 @@ func.func @vector_outerproduct_masked_f64(%lhs : vector<[2]xf64>, %rhs : vector<
   "prevent.dce"(%result) : (vector<[2]x[2]xf64>) -> ()
 }
 
-// -----
-
-func.func @vector_outerproduct_unsupported_axpy(%lhs : vector<[2]xf64>, %rhs : f64, %acc : vector<[2]xf64>) -> vector<[2]xf64> {
-  // expected-error@+1 {{AXPY operations not supported}}
-  %0 = vector.outerproduct %lhs, %rhs, %acc {kind = #vector.kind<mul>} : vector<[2]xf64>, f64
-  return %0 : vector<[2]xf64>
-}
-
-// -----
-
-func.func @vector_outerproduct_unsupported_type(%lhs : vector<[16]xi8>, %rhs : vector<[16]xi8>) {
-  %acc = arm_sme.get_tile : vector<[16]x[16]xi8>
-  // expected-error@+2 {{failed to legalize operation 'arm_sme.outerproduct'}}
-  // expected-error@+1 {{unsupported type}}
-  %0 = vector.outerproduct %lhs, %rhs, %acc {kind = #vector.kind<add>} : vector<[16]xi8>, vector<[16]xi8>
-  "prevent.dce"(%0) : (vector<[16]x[16]xi8>) -> ()
-}
-
-// -----
-
-func.func @vector_outerproduct_unsupported_kind(%lhs : vector<[2]xf64>, %rhs : vector<[2]xf64>) {
-  %acc = arm_sme.get_tile : vector<[2]x[2]xf64>
-  // expected-error@+1 {{unsupported kind}}
-  %0 = vector.outerproduct %lhs, %rhs, %acc {kind = #vector.kind<mul>} : vector<[2]xf64>, vector<[2]xf64>
-  "prevent.dce"(%0) : (vector<[2]x[2]xf64>) -> ()
-}
-
-// -----
-
-func.func @vector_outerproduct_unknown_mask(%lhs : vector<[4]xf32>, %rhs : vector<[4]xf32>, %mask : vector<[4]x[4]xi1>) {
-  // CHECK: vector.outerproduct
-  %acc = arm_sme.get_tile : vector<[4]x[4]xf32>
-  %0 = vector.mask %mask { vector.outerproduct %lhs, %rhs, %acc {kind = #vector.kind<add>} : vector<[4]xf32>, vector<[4]xf32> } : vector<[4]x[4]xi1> -> vector<[4]x[4]xf32>
-  "prevent.dce"(%0) : (vector<[4]x[4]xf32>) -> ()
-}
-
 //===----------------------------------------------------------------------===//
 // vector.insert
 //===----------------------------------------------------------------------===//
diff --git a/mlir/test/Dialect/LLVMIR/mem2reg.mlir b/mlir/test/Dialect/LLVMIR/mem2reg.mlir
index 16eb28c629ee232360eee08c51617ce39ab5a308..90e56c1166edfd96d926a12c8af34b9231615225 100644
--- a/mlir/test/Dialect/LLVMIR/mem2reg.mlir
+++ b/mlir/test/Dialect/LLVMIR/mem2reg.mlir
@@ -520,6 +520,8 @@ llvm.func @discardable_use_tree() {
   %4 = llvm.bitcast %3 : !llvm.ptr to !llvm.ptr
   llvm.intr.lifetime.start 2, %3 : !llvm.ptr
   llvm.intr.lifetime.start 2, %4 : !llvm.ptr
+  %5 = llvm.intr.invariant.start 2, %3 : !llvm.ptr
+  llvm.intr.invariant.end %5, 2, %3 : !llvm.ptr
   llvm.return
 }
 
diff --git a/mlir/test/Dialect/LLVMIR/roundtrip.mlir b/mlir/test/Dialect/LLVMIR/roundtrip.mlir
index 594c3de91815aebcb34494002423eca552d45e8d..49f34785ebad527fcd34a8fd6c289b33aa2d4970 100644
--- a/mlir/test/Dialect/LLVMIR/roundtrip.mlir
+++ b/mlir/test/Dialect/LLVMIR/roundtrip.mlir
@@ -577,6 +577,16 @@ llvm.func @lifetime(%p: !llvm.ptr) {
   llvm.return
 }
 
+// CHECK-LABEL: @invariant
+// CHECK-SAME: %[[P:.*]]: !llvm.ptr
+llvm.func @invariant(%p: !llvm.ptr) {
+  // CHECK: %[[START:.*]] = llvm.intr.invariant.start 1, %[[P]] : !llvm.ptr
+  %1 = llvm.intr.invariant.start 1, %p : !llvm.ptr
+  // CHECK: llvm.intr.invariant.end %[[START]], 1, %[[P]] : !llvm.ptr
+  llvm.intr.invariant.end %1, 1, %p : !llvm.ptr
+  llvm.return
+}
+
 // CHECK-LABEL: @vararg_func
 llvm.func @vararg_func(%arg0: i32, ...) {
   // CHECK: %[[C:.*]] = llvm.mlir.constant(1 : i32)
diff --git a/mlir/test/Dialect/LLVMIR/sroa.mlir b/mlir/test/Dialect/LLVMIR/sroa.mlir
index 09ca94e91d2a12b3f096bef2b7430884b268858b..f56ea53ac029e709bc089a17dc2f2ea19db32a59 100644
--- a/mlir/test/Dialect/LLVMIR/sroa.mlir
+++ b/mlir/test/Dialect/LLVMIR/sroa.mlir
@@ -170,12 +170,14 @@ llvm.func @direct_promotable_use_is_fine_on_accessor() -> i32 {
   // CHECK: %[[ALLOCA:.*]] = llvm.alloca %[[SIZE]] x i32
   %1 = llvm.alloca %0 x !llvm.struct<"foo", (i32, f64, i32)> {alignment = 8 : i64} : (i32) -> !llvm.ptr
   %2 = llvm.getelementptr %1[0, 2] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"foo", (i32, f64, i32)>
+  // This does not provide side-effect info but it can be removed because it implements PromotableOpInterface.
+  %3 = llvm.intr.invariant.start 2, %2 : !llvm.ptr
   // CHECK: %[[RES:.*]] = llvm.load %[[ALLOCA]]
-  %3 = llvm.load %2 : !llvm.ptr -> i32
+  %4 = llvm.load %2 : !llvm.ptr -> i32
   // This does not provide side-effect info but it can be removed because it implements PromotableOpInterface.
-  llvm.intr.lifetime.start 2, %2 : !llvm.ptr
+  llvm.intr.invariant.end %3, 2, %2 : !llvm.ptr
   // CHECK: llvm.return %[[RES]] : i32
-  llvm.return %3 : i32
+  llvm.return %4 : i32
 }
 
 // -----
diff --git a/mlir/test/Dialect/OpenMP/invalid.mlir b/mlir/test/Dialect/OpenMP/invalid.mlir
index e54808f6cfdee58b50c01ec259b4fdd666622c33..2b0e86ddd22bbc1ab43ac61a1da0d1099780aab6 100644
--- a/mlir/test/Dialect/OpenMP/invalid.mlir
+++ b/mlir/test/Dialect/OpenMP/invalid.mlir
@@ -1658,4 +1658,74 @@ func.func @omp_target_exit_data(%map1: memref<?xi32>) {
   return
 }
 
+// -----
+
+func.func @omp_target_update_invalid_motion_type(%map1 : memref<?xi32>) {
+  %mapv = omp.map_info var_ptr(%map1 : memref<?xi32>, tensor<?xi32>) map_clauses(exit_release_or_enter_alloc) capture(ByRef) -> memref<?xi32> {name = ""}
+
+  // expected-error @below {{at least one of to or from map types must be specified, other map types are not permitted}}
+  omp.target_update_data motion_entries(%mapv : memref<?xi32>)
+  return
+}
+
+// -----
+
+func.func @omp_target_update_invalid_motion_type_2(%map1 : memref<?xi32>) {
+  %mapv = omp.map_info var_ptr(%map1 : memref<?xi32>, tensor<?xi32>) map_clauses(delete) capture(ByRef) -> memref<?xi32> {name = ""}
+
+  // expected-error @below {{at least one of to or from map types must be specified, other map types are not permitted}}
+  omp.target_update_data motion_entries(%mapv : memref<?xi32>)
+  return
+}
+
+// -----
+
+func.func @omp_target_update_invalid_motion_modifier(%map1 : memref<?xi32>) {
+  %mapv = omp.map_info var_ptr(%map1 : memref<?xi32>, tensor<?xi32>) map_clauses(always, to) capture(ByRef) -> memref<?xi32> {name = ""}
+
+  // expected-error @below {{present, mapper and iterator map type modifiers are permitted}}
+  omp.target_update_data motion_entries(%mapv : memref<?xi32>)
+  return
+}
+
+// -----
+
+func.func @omp_target_update_invalid_motion_modifier_2(%map1 : memref<?xi32>) {
+  %mapv = omp.map_info var_ptr(%map1 : memref<?xi32>, tensor<?xi32>) map_clauses(close, to) capture(ByRef) -> memref<?xi32> {name = ""}
+
+  // expected-error @below {{present, mapper and iterator map type modifiers are permitted}}
+  omp.target_update_data motion_entries(%mapv : memref<?xi32>)
+  return
+}
+
+// -----
+
+func.func @omp_target_update_invalid_motion_modifier_3(%map1 : memref<?xi32>) {
+  %mapv = omp.map_info var_ptr(%map1 : memref<?xi32>, tensor<?xi32>) map_clauses(implicit, to) capture(ByRef) -> memref<?xi32> {name = ""}
+
+  // expected-error @below {{present, mapper and iterator map type modifiers are permitted}}
+  omp.target_update_data motion_entries(%mapv : memref<?xi32>)
+  return
+}
+
+// -----
+
+func.func @omp_target_update_invalid_motion_modifier_4(%map1 : memref<?xi32>) {
+  %mapv = omp.map_info var_ptr(%map1 : memref<?xi32>, tensor<?xi32>) map_clauses(implicit, tofrom) capture(ByRef) -> memref<?xi32> {name = ""}
+
+  // expected-error @below {{either to or from map types can be specified, not both}}
+  omp.target_update_data motion_entries(%mapv : memref<?xi32>)
+  return
+}
+
+// -----
+
+func.func @omp_target_update_invalid_motion_modifier_5(%map1 : memref<?xi32>) {
+  %mapv = omp.map_info var_ptr(%map1 : memref<?xi32>, tensor<?xi32>) map_clauses(to) capture(ByRef) -> memref<?xi32> {name = ""}
+  %mapv2 = omp.map_info var_ptr(%map1 : memref<?xi32>, tensor<?xi32>) map_clauses(from) capture(ByRef) -> memref<?xi32> {name = ""}
+
+  // expected-error @below {{either to or from map types can be specified, not both}}
+  omp.target_update_data motion_entries(%mapv, %mapv2 : memref<?xi32>, memref<?xi32>)
+  return
+}
 llvm.mlir.global internal @_QFsubEx() : i32
diff --git a/mlir/test/Dialect/OpenMP/ops.mlir b/mlir/test/Dialect/OpenMP/ops.mlir
index 4d88d9ac86fe16c96b1c0a1ba3d105481f636c63..3d4f6435572f7f511cb04c4fe268d3461c953339 100644
--- a/mlir/test/Dialect/OpenMP/ops.mlir
+++ b/mlir/test/Dialect/OpenMP/ops.mlir
@@ -2082,3 +2082,15 @@ func.func @omp_targets_with_map_bounds(%arg0: !llvm.ptr, %arg1: !llvm.ptr) -> ()
 
     return
 }
+
+// CHECK-LABEL: omp_target_update_data
+func.func @omp_target_update_data (%if_cond : i1, %device : si32, %map1: memref<?xi32>, %map2: memref<?xi32>) -> () {
+    %mapv_from = omp.map_info var_ptr(%map1 : memref<?xi32>, tensor<?xi32>) map_clauses(from) capture(ByRef) -> memref<?xi32> {name = ""}
+
+    %mapv_to = omp.map_info var_ptr(%map2 : memref<?xi32>, tensor<?xi32>) map_clauses(present, to) capture(ByRef) -> memref<?xi32> {name = ""}
+
+    // CHECK: omp.target_update_data if(%[[VAL_0:.*]] : i1) device(%[[VAL_1:.*]] : si32) nowait motion_entries(%{{.*}}, %{{.*}} : memref<?xi32>, memref<?xi32>)
+    omp.target_update_data if(%if_cond : i1) device(%device : si32) nowait motion_entries(%mapv_from , %mapv_to : memref<?xi32>, memref<?xi32>)
+    return
+}
+
diff --git a/mlir/test/Dialect/SCF/loop-pipelining.mlir b/mlir/test/Dialect/SCF/loop-pipelining.mlir
index 8a57ddccfee66594c54187e6d3cbae646be39103..a18c850c3f05f1d6b19e7c64c03eb81862af2962 100644
--- a/mlir/test/Dialect/SCF/loop-pipelining.mlir
+++ b/mlir/test/Dialect/SCF/loop-pipelining.mlir
@@ -770,3 +770,47 @@ func.func @dynamic_loop(%A: memref<?xf32>, %result: memref<?xf32>, %lb: index, %
   } { __test_pipelining_loop__ }
   return
 }
+
+// -----
+
+// CHECK-LABEL: yield_constant_loop(
+//  CHECK-SAME:   %[[A:.*]]: memref<?xf32>) -> f32 {
+//   CHECK-DAG:   %[[C0:.*]] = arith.constant 0 : index
+//   CHECK-DAG:   %[[C1:.*]] = arith.constant 1 : index
+//   CHECK-DAG:   %[[C3:.*]] = arith.constant 3 : index
+//   CHECK-DAG:   %[[CST0:.*]] = arith.constant 0.000000e+00 : f32
+//   CHECK-DAG:   %[[CST2:.*]] = arith.constant 2.000000e+00 : f32
+// Prologue:
+//       CHECK:   %[[L0:.*]] = memref.load %[[A]][%[[C0]]] : memref<?xf32>
+// Kernel:
+//  CHECK-NEXT:   %[[L1:.*]]:2 = scf.for %[[IV:.*]] = %[[C0]] to %[[C3]]
+//  CHECK-SAME:     step %[[C1]] iter_args(%[[ARG0:.*]] = %[[CST2]], %[[ARG1:.*]] = %[[L0]]) -> (f32, f32) {
+//  CHECK-NEXT:     %[[ADD0:.*]] = arith.addf %[[ARG1]], %[[ARG0]] : f32
+//  CHECK-NEXT:     %[[MUL0:.*]] = arith.mulf %[[ADD0]], %[[CST0]] : f32
+//  CHECK-NEXT:     memref.store %[[MUL0]], %[[A]][%[[IV]]] : memref<?xf32>
+//  CHECK-NEXT:     %[[IV1:.*]] = arith.addi %[[IV]], %[[C1]] : index
+//  CHECK-NEXT:     %[[L2:.*]] = memref.load %[[A]][%[[IV1]]] : memref<?xf32>
+//  CHECK-NEXT:     scf.yield %[[CST0]], %[[L2]] : f32
+//  CHECK-NEXT:   }
+// Epilogue:
+//  CHECK-NEXT:   %[[ADD1:.*]] = arith.addf %[[L1]]#1, %[[CST0]] : f32
+//  CHECK-NEXT:   %[[MUL1:.*]] = arith.mulf %[[ADD1]], %[[CST0]] : f32
+//  CHECK-NEXT:   memref.store %[[MUL1]], %[[A]][%[[C3]]] : memref<?xf32>
+//  CHECK-NEXT:   return %[[L1]]#0 : f32
+
+func.func @yield_constant_loop(%A: memref<?xf32>) -> f32 {
+  %c0 = arith.constant 0 : index
+  %c1 = arith.constant 1 : index
+  %c4 = arith.constant 4 : index
+  %cf0 = arith.constant 0.0 : f32
+  %cf2 = arith.constant 2.0 : f32
+  %r = scf.for %i0 = %c0 to %c4 step %c1 iter_args(%arg0 = %cf2) -> f32 {
+    %A_elem = memref.load %A[%i0] { __test_pipelining_stage__ = 0, __test_pipelining_op_order__ = 3 } : memref<?xf32>
+    %A1_elem = arith.addf %A_elem, %arg0 { __test_pipelining_stage__ = 1, __test_pipelining_op_order__ = 0 } : f32
+    %A2_elem = arith.mulf %cf0, %A1_elem { __test_pipelining_stage__ = 1, __test_pipelining_op_order__ = 1 } : f32
+    memref.store %A2_elem, %A[%i0] { __test_pipelining_stage__ = 1, __test_pipelining_op_order__ = 2 } : memref<?xf32>
+    scf.yield %cf0: f32
+  }  { __test_pipelining_loop__ }
+  return %r : f32
+}
+
diff --git a/mlir/test/Dialect/Tensor/fold-into-pack-and-unpack.mlir b/mlir/test/Dialect/Tensor/fold-into-pack-and-unpack.mlir
index 5c7578966574270d8e1aeb4d7becf2319c6cf487..ca4eb4ff679445109d96e415fda3f58c0cd65b5b 100644
--- a/mlir/test/Dialect/Tensor/fold-into-pack-and-unpack.mlir
+++ b/mlir/test/Dialect/Tensor/fold-into-pack-and-unpack.mlir
@@ -114,3 +114,234 @@ func.func @pad_pack_different_padding_value(%src: tensor<16641x16xf32>) -> tenso
 // CHECK-LABEL: func.func @pad_pack_different_padding_value
 // CHECK:         tensor.pad
 // CHECK:         tensor.pack
+
+// -----
+
+func.func @tensor_pack_linalg_transpose_fold(%arg0: tensor<56x57x1x64xf32>) -> tensor<1x57x56x2x32xf32> {
+  %0 = tensor.empty() : tensor<56x2x1x57x32xf32>
+  %pack = tensor.pack %arg0
+    outer_dims_perm = [0, 3, 2, 1]
+    inner_dims_pos = [3]
+    inner_tiles = [32]
+    into %0 : tensor<56x57x1x64xf32> -> tensor<56x2x1x57x32xf32>
+
+  %1 = tensor.empty() : tensor<1x57x56x2x32xf32>
+  %transposed = linalg.transpose
+    ins(%pack : tensor<56x2x1x57x32xf32>)
+    outs(%1 : tensor<1x57x56x2x32xf32>)
+    permutation = [2, 3, 0, 1, 4]
+  return %transposed : tensor<1x57x56x2x32xf32>
+}
+//      CHECK: func @tensor_pack_linalg_transpose_fold(
+// CHECK-SAME:     %[[ARG0:.+]]: tensor<56x57x1x64xf32>)
+//      CHECK:   %[[INIT:.+]] = tensor.empty() : tensor<1x57x56x2x32xf32>
+//      CHECK:   %[[PACK:.+]] = tensor.pack %[[ARG0]]
+// CHECK-SAME:      outer_dims_perm = [2, 1, 0, 3]
+// CHECK-SAME:      inner_dims_pos = [3] inner_tiles = [32] 
+// CHECK-SAME:       into %[[INIT]]
+//      CHECK:   return %[[PACK]]
+
+// -----
+
+func.func @tensor_pack_linalg_transpose_fold_with_padding(%arg0: tensor<56x57x1x55xf32>, %padding: f32) -> tensor<1x57x56x2x32xf32> {
+  %0 = tensor.empty() : tensor<56x2x1x57x32xf32>
+  %pack = tensor.pack %arg0 padding_value(%padding : f32)
+    outer_dims_perm = [0, 3, 2, 1]
+    inner_dims_pos = [3]
+    inner_tiles = [32]
+    into %0 : tensor<56x57x1x55xf32> -> tensor<56x2x1x57x32xf32>
+
+  %1 = tensor.empty() : tensor<1x57x56x2x32xf32>
+  %transposed = linalg.transpose
+    ins(%pack : tensor<56x2x1x57x32xf32>)
+    outs(%1 : tensor<1x57x56x2x32xf32>)
+    permutation = [2, 3, 0, 1, 4]
+  return %transposed : tensor<1x57x56x2x32xf32>
+}
+//      CHECK: func @tensor_pack_linalg_transpose_fold_with_padding(
+// CHECK-SAME:     %[[ARG0:.+]]: tensor<56x57x1x55xf32>, %[[PADDING:.+]]: f32)
+//      CHECK:   %[[INIT:.+]] = tensor.empty() : tensor<1x57x56x2x32xf32>
+//      CHECK:   %[[PACK:.+]] = tensor.pack %[[ARG0]] padding_value(%[[PADDING]] : f32)
+// CHECK-SAME:      outer_dims_perm = [2, 1, 0, 3]
+// CHECK-SAME:      inner_dims_pos = [3] inner_tiles = [32] 
+// CHECK-SAME:       into %[[INIT]]
+//      CHECK:   return %[[PACK]]
+
+// -----
+
+func.func @tensor_pack_linalg_transpose_fold_no_outer_dims_perm(%arg0: tensor<56x57x1x64xf32>) -> tensor<1x2x56x57x32xf32> {
+  %0 = tensor.empty() : tensor<56x57x1x2x32xf32>
+  %pack = tensor.pack %arg0
+    inner_dims_pos = [3]
+    inner_tiles = [32]
+    into %0 : tensor<56x57x1x64xf32> -> tensor<56x57x1x2x32xf32>
+
+  %1 = tensor.empty() : tensor<1x2x56x57x32xf32>
+  %transposed = linalg.transpose
+    ins(%pack : tensor<56x57x1x2x32xf32>)
+    outs(%1 : tensor<1x2x56x57x32xf32>)
+    permutation = [2, 3, 0, 1, 4]
+  return %transposed : tensor<1x2x56x57x32xf32>
+}
+//      CHECK: func @tensor_pack_linalg_transpose_fold_no_outer_dims_perm(
+// CHECK-SAME:     %[[ARG0:.+]]: tensor<56x57x1x64xf32>)
+//      CHECK:   %[[INIT:.+]] = tensor.empty() : tensor<1x2x56x57x32xf32>
+//      CHECK:   %[[PACK:.+]] = tensor.pack %[[ARG0]]
+// CHECK-SAME:      outer_dims_perm = [2, 3, 0, 1]
+// CHECK-SAME:      inner_dims_pos = [3] inner_tiles = [32] 
+// CHECK-SAME:       into %[[INIT]]
+//      CHECK:   return %[[PACK]]
+
+// -----
+
+func.func @tensor_pack_linalg_transpose_fold_tile_dims_transpose(%arg0: tensor<56x72x24x128xf32>) -> tensor<12x56x4x9x32x8x2xf32> {
+  %0 = tensor.empty() : tensor<4x9x12x56x8x2x32xf32>
+  %pack = tensor.pack %arg0
+    outer_dims_perm = [3, 1, 2, 0]
+    inner_dims_pos = [1, 2, 3]
+    inner_tiles = [8, 2, 32]
+    into %0 : tensor<56x72x24x128xf32> -> tensor<4x9x12x56x8x2x32xf32>
+
+  %1 = tensor.empty() : tensor<12x56x4x9x32x8x2xf32>
+  %transposed = linalg.transpose
+    ins(%pack : tensor<4x9x12x56x8x2x32xf32>)
+    outs(%1 : tensor<12x56x4x9x32x8x2xf32>)
+    permutation = [2, 3, 0, 1, 6, 4, 5]
+  return %transposed : tensor<12x56x4x9x32x8x2xf32>
+}
+//      CHECK: func @tensor_pack_linalg_transpose_fold_tile_dims_transpose(
+// CHECK-SAME:     %[[ARG0:.+]]: tensor<56x72x24x128xf32>)
+//      CHECK:   %[[INIT:.+]] = tensor.empty() : tensor<12x56x4x9x32x8x2xf32>
+//      CHECK:   %[[PACK:.+]] = tensor.pack %[[ARG0]]
+// CHECK-SAME:      outer_dims_perm = [2, 0, 3, 1]
+// CHECK-SAME:      inner_dims_pos = [3, 1, 2] inner_tiles = [32, 8, 2] 
+// CHECK-SAME:       into %[[INIT]]
+//      CHECK:   return %[[PACK]]
+
+// -----
+
+func.func @tensor_pack_linalg_transpose_fold_tile_dims_outer_dims_transpose(%arg0: tensor<56x72x24x128xf32>) -> tensor<9x56x2x12x32x8x4xf32> {
+  %0 = tensor.empty() : tensor<4x12x9x56x8x2x32xf32>
+  %pack = tensor.pack %arg0
+    outer_dims_perm = [3, 2, 1, 0]
+    inner_dims_pos = [1, 2, 3]
+    inner_tiles = [8, 2, 32]
+    into %0 : tensor<56x72x24x128xf32> -> tensor<4x12x9x56x8x2x32xf32>
+
+  %1 = tensor.empty() : tensor<9x56x2x12x32x8x4xf32>
+  %transposed = linalg.transpose
+    ins(%pack : tensor<4x12x9x56x8x2x32xf32>)
+    outs(%1 : tensor<9x56x2x12x32x8x4xf32>)
+    permutation = [2, 3, 5, 1, 6, 4, 0]
+  return %transposed : tensor<9x56x2x12x32x8x4xf32>
+}
+//      CHECK: func @tensor_pack_linalg_transpose_fold_tile_dims_outer_dims_transpose(
+// CHECK-SAME:     %[[ARG0:.+]]: tensor<56x72x24x128xf32>)
+//      CHECK:   tensor.pack
+//      CHECK:   linalg.transpose
+
+// -----
+
+func.func @tensor_pack_linalg_transpose_fold_dynamic_outer_dims(%arg0: tensor<56x?x?x64xf32>) -> tensor<?x?x56x2x32xf32> {
+  %0 = tensor.empty() : tensor<56x2x1x57x32xf32>
+  %pack = tensor.pack %arg0
+    outer_dims_perm = [0, 3, 2, 1]
+    inner_dims_pos = [3]
+    inner_tiles = [32]
+    into %0 : tensor<56x?x?x64xf32> -> tensor<56x2x1x57x32xf32>
+
+  %1 = tensor.empty() : tensor<1x57x56x2x32xf32>
+  %transposed = linalg.transpose
+    ins(%pack : tensor<56x2x1x57x32xf32>)
+    outs(%1 : tensor<1x57x56x2x32xf32>)
+    permutation = [2, 3, 0, 1, 4]
+
+  %return_value = tensor.cast %transposed : tensor<1x57x56x2x32xf32> to tensor<?x?x56x2x32xf32>  
+  return %return_value : tensor<?x?x56x2x32xf32>
+}
+//      CHECK: func @tensor_pack_linalg_transpose_fold_dynamic_outer_dims(
+// CHECK-SAME:     %[[ARG0:.+]]: tensor<56x?x?x64xf32>)
+//      CHECK:   %[[c1:.+]] = arith.constant 1 : index
+//      CHECK:   %[[c2:.+]] = arith.constant 2 : index
+//      CHECK:   %[[dim:.+]] = tensor.dim %[[ARG0]], %[[c1]] : tensor<56x?x?x64xf32>
+//      CHECK:   %[[dim_0:.+]] = tensor.dim %[[ARG0]], %[[c2]] : tensor<56x?x?x64xf32>
+//      CHECK:   %[[INIT:.+]] = tensor.empty(%[[dim_0]], %[[dim]]) : tensor<?x?x56x2x32xf32>
+//      CHECK:   %[[PACK:.+]] = tensor.pack %[[ARG0]]
+// CHECK-SAME:      outer_dims_perm = [2, 1, 0, 3]
+// CHECK-SAME:      inner_dims_pos = [3] inner_tiles = [32] 
+// CHECK-SAME:       into %[[INIT]]
+//      CHECK:   return %[[PACK]]
+
+// -----
+
+func.func @tensor_pack_linalg_transpose_fold_dynamic_outer_and_tile_dims(%arg0: tensor<56x?x?x128xf32>) -> tensor<?x?x56x9x32x8x2xf32> {
+  %0 = tensor.empty() : tensor<56x9x12x4x8x2x32xf32>
+  %pack = tensor.pack %arg0
+    inner_dims_pos = [1, 2, 3]
+    inner_tiles = [8, 2, 32]
+    into %0 : tensor<56x?x?x128xf32> -> tensor<56x9x12x4x8x2x32xf32>
+
+  %1 = tensor.empty() : tensor<12x4x56x9x32x8x2xf32>
+  %transposed = linalg.transpose
+    ins(%pack : tensor<56x9x12x4x8x2x32xf32>)
+    outs(%1 : tensor<12x4x56x9x32x8x2xf32>)
+    permutation = [2, 3, 0, 1, 6, 4, 5]
+  
+  %return_value = tensor.cast %transposed : tensor<12x4x56x9x32x8x2xf32> to tensor<?x?x56x9x32x8x2xf32> 
+  return %return_value : tensor<?x?x56x9x32x8x2xf32>
+}
+//      CHECK: #[[map:.+]] = affine_map<()[s0] -> (s0 ceildiv 8)>
+//      CHECK: #[[map1:.+]] = affine_map<()[s0] -> (s0 ceildiv 2)>
+//      CHECK: module {
+//      CHECK:   func.func @tensor_pack_linalg_transpose_fold_dynamic_outer_and_tile_dims(
+// CHECK-SAME:   %[[ARG0:.+]]: tensor<56x?x?x128xf32>) 
+//      CHECK:     %[[c1:.+]] = arith.constant 1 : index
+//      CHECK:     %[[c2:.+]] = arith.constant 2 : index
+//      CHECK:     %[[dim:.+]] = tensor.dim %[[ARG0]], %[[c1]] : tensor<56x?x?x128xf32>
+//      CHECK:     %[[dim_0:.+]] = tensor.dim %[[ARG0]], %[[c2]] : tensor<56x?x?x128xf32>
+//      CHECK:     %[[mapped_dim1:.+]] = affine.apply #[[map:.+]]()[%[[dim]]]
+//      CHECK:     %[[mapped_dim2:.+]] = affine.apply #[[map1:.+]]()[%[[dim_0]]]
+//      CHECK:     %[[INIT:.+]] = tensor.empty(%[[mapped_dim2]], %[[mapped_dim1]]) : tensor<?x4x56x?x32x8x2xf32>
+//      CHECK:     %[[PACK:.+]] = tensor.pack %[[ARG0]] outer_dims_perm = [2, 3, 0, 1] inner_dims_pos = [3, 1, 2] inner_tiles = [32, 8, 2] into %[[INIT]] : tensor<56x?x?x128xf32> -> tensor<?x4x56x?x32x8x2xf32>
+//      CHECK:     %[[CAST:.+]] = tensor.cast %[[PACK]] : tensor<?x4x56x?x32x8x2xf32> to tensor<?x?x56x9x32x8x2xf32>
+//      CHECK:     return %[[CAST]] : tensor<?x?x56x9x32x8x2xf32>
+//      CHECK:   }
+
+// -----
+
+func.func @tensor_pack_linalg_transpose_fold_dynamic_outer_dims_tile_dims_tile_sizes(%arg0: tensor<?x?x?x?xf32>, %pack_dest: tensor<?x?x?x?x?x?x?xf32>, %transpose_dest: tensor<?x?x?x?x?x?x?xf32>, %tile_p : index, %tile_q : index, %tile_r : index) -> tensor<?x?x?x?x?x?x?xf32> {
+  %pack = tensor.pack %arg0
+    outer_dims_perm = [3, 0, 2, 1]
+    inner_dims_pos = [1, 2, 3]
+    inner_tiles = [%tile_p, %tile_q, %tile_r]
+    into %pack_dest : tensor<?x?x?x?xf32> -> tensor<?x?x?x?x?x?x?xf32>
+
+  %transposed = linalg.transpose
+    ins(%pack : tensor<?x?x?x?x?x?x?xf32>)
+    outs(%transpose_dest : tensor<?x?x?x?x?x?x?xf32>)
+    permutation = [2, 3, 0, 1, 6, 4, 5]
+
+  return %transposed : tensor<?x?x?x?x?x?x?xf32>
+}
+//      CHECK: #[[map:.+]] = affine_map<()[s0, s1] -> (s0 ceildiv s1)>
+//      CHECK: module {
+//      CHECK:   func.func @tensor_pack_linalg_transpose_fold_dynamic_outer_dims_tile_dims_tile_sizes(
+// CHECK-SAME:   %[[ARG0:.+]]: tensor<?x?x?x?xf32>,
+// CHECK-SAME:   %[[PACK_DEST:.+]]: tensor<?x?x?x?x?x?x?xf32>, %[[TRANSPOSE_DEST:.+]]: tensor<?x?x?x?x?x?x?xf32>,
+// CHECK-SAME:   %[[ARG1:.+]]: index, %[[ARG2:.+]]: index,
+// CHECK-SAME:   %[[ARG3:.+]]: index) 
+//      CHECK:     %[[c0:.+]] = arith.constant 0 : index
+//      CHECK:     %[[c1:.+]] = arith.constant 1 : index
+//      CHECK:     %[[c2:.+]] = arith.constant 2 : index
+//      CHECK:     %[[c3:.+]] = arith.constant 3 : index
+//      CHECK:     %[[dim:.+]] = tensor.dim %[[ARG0]], %[[c0]] : tensor<?x?x?x?xf32>
+//      CHECK:     %[[dim_0:.+]] = tensor.dim %[[ARG0]], %[[c1]] : tensor<?x?x?x?xf32>
+//      CHECK:     %[[dim_1:.+]] = tensor.dim %[[ARG0]], %[[c2]] : tensor<?x?x?x?xf32>
+//      CHECK:     %[[dim_2:.+]] = tensor.dim %[[ARG0]], %[[c3]] : tensor<?x?x?x?xf32>
+//      CHECK:     %[[mapped_dim0:.+]] = affine.apply #[[map:.+]]()[%[[dim_2]], %[[ARG3]]]
+//      CHECK:     %[[mapped_dim1:.+]] = affine.apply #[[map:.+]]()[%[[dim_0]], %[[ARG1]]]
+//      CHECK:     %[[mapped_dim2:.+]] = affine.apply #[[map:.+]]()[%[[dim_1]], %[[ARG2]]]
+//      CHECK:     %[[INIT:.+]] = tensor.empty(%[[mapped_dim2]], %[[mapped_dim1]], %[[mapped_dim0]], %[[dim]], %[[ARG3]], %[[ARG1]], %[[ARG2]]) : tensor<?x?x?x?x?x?x?xf32>
+//      CHECK:     %[[PACK:.+]] = tensor.pack %[[ARG0]] outer_dims_perm = [2, 1, 3, 0] inner_dims_pos = [3, 1, 2] inner_tiles = [%[[ARG3]], %[[ARG1]], %[[ARG2]]] into %[[INIT]] : tensor<?x?x?x?xf32> -> tensor<?x?x?x?x?x?x?xf32>
+//      CHECK:     return %[[PACK]] : tensor<?x?x?x?x?x?x?xf32>
+//      CHECK:   }
diff --git a/mlir/test/Dialect/Vector/vector-transfer-flatten.mlir b/mlir/test/Dialect/Vector/vector-transfer-flatten.mlir
index ebec2274655e46e5fe4eae2081066f84bd726ae4..3708d741141be06be7daf62263f5834d0328adff 100644
--- a/mlir/test/Dialect/Vector/vector-transfer-flatten.mlir
+++ b/mlir/test/Dialect/Vector/vector-transfer-flatten.mlir
@@ -254,3 +254,105 @@ func.func @transfer_read_flattenable_negative2(
 
 // CHECK-LABEL: func @transfer_read_flattenable_negative2
 //       CHECK:   vector.transfer_read {{.*}} vector<5x4x3x2xi8>
+
+// -----
+
+func.func @fold_unit_dim_add_basic(%arg0 : vector<1x8xi32>) -> vector<1x8xi32> {
+   %add = arith.addi %arg0, %arg0 : vector<1x8xi32>
+   return %add : vector<1x8xi32>
+}
+// CHECK-LABEL:   func.func @fold_unit_dim_add_basic(
+// CHECK-SAME:      %[[VAL_0:.*]]: vector<1x8xi32>) -> vector<1x8xi32> {
+// CHECK:           %[[VAL_1:.*]] = vector.shape_cast %[[VAL_0]] : vector<1x8xi32> to vector<8xi32>
+// CHECK:           %[[VAL_2:.*]] = vector.shape_cast %[[VAL_0]] : vector<1x8xi32> to vector<8xi32>
+// CHECK:           %[[VAL_3:.*]] = arith.addi %[[VAL_1]], %[[VAL_2]] : vector<8xi32>
+// CHECK:           %[[VAL_4:.*]] = vector.shape_cast %[[VAL_3]] : vector<8xi32> to vector<1x8xi32>
+// CHECK:           return %[[VAL_4]] : vector<1x8xi32>
+
+// -----
+
+func.func @fold_unit_dim_add_leading_and_trailing(%arg0 : vector<1x8x1xi32>) -> vector<1x8x1xi32> {
+   %add = arith.addi %arg0, %arg0 : vector<1x8x1xi32>
+   return %add : vector<1x8x1xi32>
+}
+// CHECK-LABEL:   func.func @fold_unit_dim_add_leading_and_trailing(
+// CHECK-SAME:      %[[VAL_0:.*]]: vector<1x8x1xi32>) -> vector<1x8x1xi32> {
+// CHECK:           %[[VAL_1:.*]] = vector.shape_cast %[[VAL_0]] : vector<1x8x1xi32> to vector<8xi32>
+// CHECK:           %[[VAL_2:.*]] = vector.shape_cast %[[VAL_0]] : vector<1x8x1xi32> to vector<8xi32>
+// CHECK:           %[[VAL_3:.*]] = arith.addi %[[VAL_1]], %[[VAL_2]] : vector<8xi32>
+// CHECK:           %[[VAL_4:.*]] = vector.shape_cast %[[VAL_3]] : vector<8xi32> to vector<1x8x1xi32>
+// CHECK:           return %[[VAL_4]] : vector<1x8x1xi32>
+
+// -----
+
+func.func @fold_unit_dim_add(%arg0 : vector<8x1xi32>,
+                             %arg1 : vector<1x8xi32>) -> vector<8xi32> {
+   %sc_arg0 = vector.shape_cast %arg0 : vector<8x1xi32> to vector<1x8xi32>
+   %add = arith.addi %sc_arg0, %arg1 : vector<1x8xi32>
+   %res = vector.shape_cast %add : vector<1x8xi32> to vector<8xi32>
+   return %res : vector<8xi32>
+}
+
+// CHECK-LABEL:   func.func @fold_unit_dim_add(
+// CHECK-SAME:      %[[VAL_0:.*]]: vector<8x1xi32>,
+// CHECK-SAME:      %[[VAL_1:.*]]: vector<1x8xi32>) -> vector<8xi32> {
+// CHECK:           %[[VAL_2:.*]] = vector.shape_cast %[[VAL_0]] : vector<8x1xi32> to vector<8xi32>
+// CHECK:           %[[VAL_3:.*]] = vector.shape_cast %[[VAL_1]] : vector<1x8xi32> to vector<8xi32>
+// CHECK:           %[[VAL_4:.*]] = arith.addi %[[VAL_2]], %[[VAL_3]] : vector<8xi32>
+// CHECK:           return %[[VAL_4]] : vector<8xi32>
+
+// -----
+
+func.func @fold_unit_dim_mulf(%arg0 : vector<8x[2]x1xf32>,
+                              %arg1 : vector<1x8x[2]xf32>) -> vector<8x[2]xf32> {
+   %sc_arg0 = vector.shape_cast %arg0 : vector<8x[2]x1xf32> to vector<1x8x[2]xf32>
+   %add = arith.mulf %sc_arg0, %arg1 : vector<1x8x[2]xf32>
+   %res = vector.shape_cast %add : vector<1x8x[2]xf32> to vector<8x[2]xf32>
+   return %res : vector<8x[2]xf32>
+}
+
+// CHECK-LABEL:   func.func @fold_unit_dim_mulf(
+// CHECK-SAME:      %[[VAL_0:.*]]: vector<8x[2]x1xf32>,
+// CHECK-SAME:      %[[VAL_1:.*]]: vector<1x8x[2]xf32>) -> vector<8x[2]xf32> {
+// CHECK:           %[[VAL_2:.*]] = vector.shape_cast %[[VAL_0]] : vector<8x[2]x1xf32> to vector<8x[2]xf32>
+// CHECK:           %[[VAL_3:.*]] = vector.shape_cast %[[VAL_1]] : vector<1x8x[2]xf32> to vector<8x[2]xf32>
+// CHECK:           %[[VAL_4:.*]] = arith.mulf %[[VAL_2]], %[[VAL_3]] : vector<8x[2]xf32>
+// CHECK:           return %[[VAL_4]] : vector<8x[2]xf32>
+
+// -----
+
+func.func @fold_unit_dim_sitofp(%arg0 : vector<8x[2]x1xi8>) -> vector<8x[2]xf32> {
+   %sc_arg0 = vector.shape_cast %arg0 : vector<8x[2]x1xi8> to vector<1x8x[2]xi8>
+   %add = arith.sitofp %sc_arg0 : vector<1x8x[2]xi8> to vector<1x8x[2]xf32>
+   %res = vector.shape_cast %add : vector<1x8x[2]xf32> to vector<8x[2]xf32>
+   return %res : vector<8x[2]xf32>
+}
+
+// CHECK-LABEL:   func.func @fold_unit_dim_sitofp(
+// CHECK-SAME:      %[[VAL_0:.*]]: vector<8x[2]x1xi8>) -> vector<8x[2]xf32> {
+// CHECK:           %[[VAL_1:.*]] = vector.shape_cast %[[VAL_0]] : vector<8x[2]x1xi8> to vector<8x[2]xi8>
+// CHECK:           %[[VAL_2:.*]] = arith.sitofp %[[VAL_1]] : vector<8x[2]xi8> to vector<8x[2]xf32>
+// CHECK:           return %[[VAL_2]] : vector<8x[2]xf32>
+
+// -----
+
+// All shape casts are folded away
+
+func.func @fold_unit_dims_entirely(%arg0 : vector<8xi32>,
+                                   %arg1 : vector<8xi32>,
+                                   %arg2 : vector<8xi32>) -> vector<8xi32> {
+   %sc_arg0 = vector.shape_cast %arg0 : vector<8xi32> to vector<1x8xi32>
+   %sc_arg1 = vector.shape_cast %arg1 : vector<8xi32> to vector<1x8xi32>
+   %sc_arg2 = vector.shape_cast %arg2 : vector<8xi32> to vector<1x8xi32>
+   %mul = arith.muli %sc_arg0, %sc_arg1 : vector<1x8xi32>
+   %add = arith.addi %mul, %sc_arg2 : vector<1x8xi32>
+   %res = vector.shape_cast %add : vector<1x8xi32> to vector<8xi32>
+   return %res : vector<8xi32>
+}
+
+// CHECK-LABEL:   func.func @fold_unit_dims_entirely(
+// CHECK-SAME:      %[[VAL_0:.*]]: vector<8xi32>, %[[VAL_1:.*]]: vector<8xi32>,
+// CHECK-SAME:      %[[VAL_2:.*]]: vector<8xi32>) -> vector<8xi32> {
+// CHECK:           %[[VAL_3:.*]] = arith.muli %[[VAL_0]], %[[VAL_1]] : vector<8xi32>
+// CHECK:           %[[VAL_4:.*]] = arith.addi %[[VAL_3]], %[[VAL_2]] : vector<8xi32>
+// CHECK:           return %[[VAL_4]] : vector<8xi32>
diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/matmul-transpose-a.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/matmul-transpose-a.mlir
index dcd780b23161da472d2b7cf891bd076cb4cc8242..c781d5e0af846e5cd1521c3c92a6f5872f3851d9 100644
--- a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/matmul-transpose-a.mlir
+++ b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/matmul-transpose-a.mlir
@@ -1,8 +1,8 @@
 // RUN: mlir-opt %s \
 // RUN:   -transform-interpreter -test-transform-dialect-erase-schedule \
 // RUN:   -one-shot-bufferize="bufferize-function-boundaries" -canonicalize \
-// RUN:   -enable-arm-streaming="streaming-mode=streaming-locally za-mode=new-za" \
 // RUN:   -convert-vector-to-arm-sme -allocate-arm-sme-tiles -convert-arm-sme-to-scf \
+// RUN:   -enable-arm-streaming="streaming-mode=streaming-locally za-mode=new-za only-if-required-by-ops" \
 // RUN:   -convert-vector-to-scf -cse -arm-sve-legalize-vector-storage \
 // RUN:   -convert-arm-sme-to-llvm \
 // RUN:   -convert-vector-to-llvm=enable-arm-sve \
@@ -21,7 +21,7 @@ func.func @matmul_transpose_a(%A : tensor<?x?xf32>, %B : tensor<?x?xf32>, %C : t
   return
 }
 
-func.func @main() attributes { enable_arm_streaming_ignore } {
+func.func @main() {
   %c0 = arith.constant 0 : i32
   %c7 = arith.constant 7 : index
 
diff --git a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/matmul.mlir b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/matmul.mlir
index db5b098770402c89a4aa9856a3e87073cf2105dc..31c3202c3fc57b59b42bbbe98d58d5db192b2a8b 100644
--- a/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/matmul.mlir
+++ b/mlir/test/Integration/Dialect/Linalg/CPU/ArmSME/matmul.mlir
@@ -1,8 +1,8 @@
 // RUN: mlir-opt %s \
 // RUN:   -transform-interpreter -test-transform-dialect-erase-schedule \
 // RUN:   -canonicalize \
-// RUN:   -enable-arm-streaming="streaming-mode=streaming-locally za-mode=new-za" \
 // RUN:   -convert-vector-to-arm-sme -allocate-arm-sme-tiles -convert-arm-sme-to-scf \
+// RUN:   -enable-arm-streaming="streaming-mode=streaming-locally za-mode=new-za only-if-required-by-ops" \
 // RUN:   -convert-vector-to-scf -cse -arm-sve-legalize-vector-storage \
 // RUN:   -convert-arm-sme-to-llvm \
 // RUN:   -convert-vector-to-llvm=enable-arm-sve \
@@ -21,7 +21,7 @@ func.func @matmul(%A : tensor<?x?xf32>, %B : tensor<?x?xf32>, %C : tensor<?x?xf3
   return
 }
 
-func.func @main() attributes { enable_arm_streaming_ignore } {
+func.func @main() {
   %c0 = arith.constant 0 : i32
   %c7 = arith.constant 7 : index
 
diff --git a/mlir/test/Integration/Dialect/Memref/print-memref.mlir b/mlir/test/Integration/Dialect/Memref/print-memref.mlir
new file mode 100644
index 0000000000000000000000000000000000000000..b83f3919efd83e92d0696c98afa8e8fcfff583db
--- /dev/null
+++ b/mlir/test/Integration/Dialect/Memref/print-memref.mlir
@@ -0,0 +1,128 @@
+// RUN: mlir-opt %s \
+// RUN:   -func-bufferize -arith-bufferize --canonicalize \
+// RUN:   -finalize-memref-to-llvm\
+// RUN:   -convert-func-to-llvm -reconcile-unrealized-casts |\
+// RUN: mlir-cpu-runner \
+// RUN:  -e entry -entry-point-result=void  \
+// RUN:  -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils |\
+// RUN: FileCheck %s
+
+module {
+
+  func.func private @printMemrefI8(%ptr : tensor<*xi8>) attributes { llvm.emit_c_interface }
+  func.func private @printMemrefI16(%ptr : tensor<*xi16>) attributes { llvm.emit_c_interface }
+  func.func private @printMemrefI32(%ptr : tensor<*xi32>) attributes { llvm.emit_c_interface }
+  func.func private @printMemrefI64(%ptr : tensor<*xi64>) attributes { llvm.emit_c_interface }
+  func.func private @printMemrefBF16(%ptr : tensor<*xbf16>) attributes { llvm.emit_c_interface }
+  func.func private @printMemrefF16(%ptr : tensor<*xf16>) attributes { llvm.emit_c_interface }
+  func.func private @printMemrefF32(%ptr : tensor<*xf32>) attributes { llvm.emit_c_interface }
+  func.func private @printMemrefF64(%ptr : tensor<*xf64>) attributes { llvm.emit_c_interface }
+  func.func private @printMemrefC32(%ptr : tensor<*xcomplex<f32>>) attributes { llvm.emit_c_interface }
+  func.func private @printMemrefC64(%ptr : tensor<*xcomplex<f64>>) attributes { llvm.emit_c_interface }
+  func.func private @printMemrefInd(%ptr : tensor<*xindex>) attributes { llvm.emit_c_interface }
+
+  func.func @entry() {
+    %i8 = arith.constant dense<90> : tensor<3x3xi8>
+    %i16 = arith.constant dense<1> : tensor<3x3xi16>
+    %i32 = arith.constant dense<2> : tensor<3x3xi32>
+    %i64 = arith.constant dense<3> : tensor<3x3xi64>
+    %f16 = arith.constant dense<1.5> : tensor<3x3xf16>
+    %bf16 = arith.constant dense<2.5> : tensor<3x3xbf16>
+    %f32 = arith.constant dense<3.5> : tensor<3x3xf32>
+    %f64 = arith.constant dense<4.5> : tensor<3x3xf64>
+    %c32 = arith.constant dense<(1.000000e+01,5.000000e+00)> : tensor<3x3xcomplex<f32>>
+    %c64 = arith.constant dense<(2.000000e+01,5.000000e+00)> : tensor<3x3xcomplex<f64>>
+    %ind = arith.constant dense<4> : tensor<3x3xindex>
+
+    %1 = tensor.cast %i8 : tensor<3x3xi8> to tensor<*xi8>
+    %2 = tensor.cast %i16 : tensor<3x3xi16> to tensor<*xi16>
+    %3 = tensor.cast %i32 : tensor<3x3xi32> to tensor<*xi32>
+    %4 = tensor.cast %i64 : tensor<3x3xi64> to tensor<*xi64>
+    %5 = tensor.cast %f16 : tensor<3x3xf16> to tensor<*xf16>
+    %6 = tensor.cast %bf16 : tensor<3x3xbf16> to tensor<*xbf16>
+    %7 = tensor.cast %f32 : tensor<3x3xf32> to tensor<*xf32>
+    %8 = tensor.cast %f64 : tensor<3x3xf64> to tensor<*xf64>
+    %9 = tensor.cast %c32 : tensor<3x3xcomplex<f32>> to tensor<*xcomplex<f32>>
+    %10 = tensor.cast %c64 : tensor<3x3xcomplex<f64>> to tensor<*xcomplex<f64>>
+    %11 = tensor.cast %ind : tensor<3x3xindex> to tensor<*xindex>
+
+    // CHECK:      data = 
+    // CHECK-NEXT: {{\[}}[Z,   Z,   Z],
+    // CHECK-NEXT:       [Z,   Z,   Z],
+    // CHECK-NEXT:       [Z,   Z,   Z]]
+    //
+    call @printMemrefI8(%1) : (tensor<*xi8>) -> ()
+
+    // CHECK-NEXT: data = 
+    // CHECK-NEXT: {{\[}}[1,   1,   1],
+    // CHECK-NEXT:       [1,   1,   1],
+    // CHECK-NEXT:       [1,   1,   1]]
+    //
+    call @printMemrefI16(%2) : (tensor<*xi16>) -> ()
+
+    // CHECK-NEXT: data = 
+    // CHECK-NEXT: {{\[}}[2,   2,   2],
+    // CHECK-NEXT:       [2,   2,   2],
+    // CHECK-NEXT:       [2,   2,   2]]
+    //
+    call @printMemrefI32(%3) : (tensor<*xi32>) -> ()
+
+    // CHECK-NEXT: data =
+    // CHECK-NEXT: {{\[}}[3,   3,   3],
+    // CHECK-NEXT:       [3,   3,   3],
+    // CHECK-NEXT:       [3,   3,   3]]
+    //
+    call @printMemrefI64(%4) : (tensor<*xi64>) -> ()
+
+    // CHECK-NEXT: data = 
+    // CHECK-NEXT: {{\[}}[1.5,   1.5,   1.5],
+    // CHECK-NEXT:       [1.5,   1.5,   1.5],
+    // CHECK-NEXT:       [1.5,   1.5,   1.5]]
+    //
+    call @printMemrefF16(%5) : (tensor<*xf16>) -> ()
+
+    // CHECK-NEXT: data = 
+    // CHECK-NEXT: {{\[}}[2.5,   2.5,   2.5],
+    // CHECK-NEXT:       [2.5,   2.5,   2.5],
+    // CHECK-NEXT:       [2.5,   2.5,   2.5]]
+    //
+    call @printMemrefBF16(%6) : (tensor<*xbf16>) -> ()
+
+    // CHECK-NEXT: data = 
+    // CHECK-NEXT: {{\[}}[3.5,   3.5,   3.5],
+    // CHECK-NEXT:       [3.5,   3.5,   3.5],
+    // CHECK-NEXT:       [3.5,   3.5,   3.5]]
+    //
+    call @printMemrefF32(%7) : (tensor<*xf32>) -> ()
+
+    // CHECK-NEXT: data = 
+    // CHECK-NEXT: {{\[}}[4.5,   4.5,   4.5],
+    // CHECK-NEXT:       [4.5,   4.5,   4.5],
+    // CHECK-NEXT:       [4.5,   4.5,   4.5]]
+    //
+    call @printMemrefF64(%8) : (tensor<*xf64>) -> ()
+
+    // CHECK-NEXT: data = 
+    // CHECK-NEXT: {{\[}}[(10,5), (10,5), (10,5)],
+    // CHECK-NEXT:       [(10,5), (10,5), (10,5)],
+    // CHECK-NEXT:       [(10,5), (10,5), (10,5)]]
+    //
+    call @printMemrefC32(%9) : (tensor<*xcomplex<f32>>) -> ()
+
+    // CHECK-NEXT: data = 
+    // CHECK-NEXT: {{\[}}[(20,5), (20,5), (20,5)],
+    // CHECK-NEXT:       [(20,5), (20,5), (20,5)],
+    // CHECK-NEXT:       [(20,5), (20,5), (20,5)]]
+    //
+    call @printMemrefC64(%10) : (tensor<*xcomplex<f64>>) -> ()
+
+    // CHECK-NEXT: data = 
+    // CHECK-NEXT: {{\[}}[4,   4,   4],
+    // CHECK-NEXT:       [4,   4,   4],
+    // CHECK-NEXT:       [4,   4,   4]]
+    //
+    call @printMemrefInd(%11) : (tensor<*xindex>) -> ()
+
+    return
+  }
+}
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/test-transfer-read-2d.mlir b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/test-transfer-read-2d.mlir
index 4ca61a089bdf52dd94bd699bfb11d9c1f380b4c4..839aed2e840c905ab01d841cf21bafe281b6c1aa 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/test-transfer-read-2d.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/test-transfer-read-2d.mlir
@@ -1,7 +1,7 @@
 // DEFINE: %{entry_point} = entry
 // DEFINE: %{compile} = mlir-opt %s \
-// DEFINE:   -enable-arm-streaming="streaming-mode=streaming-locally za-mode=new-za" \
 // DEFINE:   -convert-vector-to-arm-sme -convert-arm-sme-to-scf -allocate-arm-sme-tiles \
+// DEFINE:   -enable-arm-streaming="streaming-mode=streaming-locally za-mode=new-za only-if-required-by-ops" \
 // DEFINE:   -convert-arm-sme-to-llvm -cse -canonicalize \
 // DEFINE:   -test-lower-to-llvm
 // DEFINE: %{run} = %mcr_aarch64_cmd \
@@ -140,7 +140,7 @@ func.func @get_svl() -> index {
   return %vscale : index
 }
 
-func.func @entry() attributes { enable_arm_streaming_ignore } {
+func.func @entry() {
   %c0 = arith.constant 0 : index
   %c1 = arith.constant 1 : index
   %c2 = arith.constant 2 : index
diff --git a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/test-transfer-write-2d.mlir b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/test-transfer-write-2d.mlir
index 14dca2d4d7082aa36d0881aadd6cb66dcd0f962e..84246606daa8af26250ca70cdec7fbbf4408190f 100644
--- a/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/test-transfer-write-2d.mlir
+++ b/mlir/test/Integration/Dialect/Vector/CPU/ArmSME/test-transfer-write-2d.mlir
@@ -1,7 +1,7 @@
 // DEFINE: %{entry_point} = entry
 // DEFINE: %{compile} = mlir-opt %s \
-// DEFINE:   -enable-arm-streaming="streaming-mode=streaming-locally za-mode=new-za" \
 // DEFINE:   -convert-vector-to-arm-sme -convert-arm-sme-to-scf -allocate-arm-sme-tiles \
+// DEFINE:   -enable-arm-streaming="streaming-mode=streaming-locally za-mode=new-za only-if-required-by-ops" \
 // DEFINE:   -convert-arm-sme-to-llvm -cse -canonicalize \
 // DEFINE:   -test-lower-to-llvm
 // DEFINE: %{run} = %mcr_aarch64_cmd \
@@ -102,7 +102,7 @@ func.func @get_svl() -> index {
   return %vscale : index
 }
 
-func.func @entry() attributes { enable_arm_streaming_ignore } {
+func.func @entry() {
   %c0 = arith.constant 0 : index
   %c2 = arith.constant 2 : index
   %c4 = arith.constant 4 : index
diff --git a/mlir/test/Target/LLVMIR/Import/function-attributes.ll b/mlir/test/Target/LLVMIR/Import/function-attributes.ll
index bf9d746f6c0b6d30fa1217e89db231ff9e1c26ab..f76e7293809628db206d4c0915c6cd16c232d09f 100644
--- a/mlir/test/Target/LLVMIR/Import/function-attributes.ll
+++ b/mlir/test/Target/LLVMIR/Import/function-attributes.ll
@@ -212,6 +212,14 @@ define void @locally_streaming_func() "aarch64_pstate_sm_body" {
 
 // -----
 
+; CHECK-LABEL: @streaming_compatible_func
+; CHECK-SAME: attributes {arm_streaming_compatible}
+define void @streaming_compatible_func() "aarch64_pstate_sm_compatible" {
+  ret void
+}
+
+// -----
+
 ; CHECK-LABEL: @section_func
 ; CHECK-SAME: attributes {section = ".section.name"}
 define void @section_func() section ".section.name" {
diff --git a/mlir/test/Target/LLVMIR/Import/global-variables.ll b/mlir/test/Target/LLVMIR/Import/global-variables.ll
index 56045d1ac18b4a4705157a2d4b3bdc06da38a1ef..ab930084323cf939a6927950159cb520bd8669d2 100644
--- a/mlir/test/Target/LLVMIR/Import/global-variables.ll
+++ b/mlir/test/Target/LLVMIR/Import/global-variables.ll
@@ -270,3 +270,27 @@ define void @bar() {
 !7 = !DIGlobalVariableExpression(var: !6, expr: !DIExpression(DW_OP_constu, 3, DW_OP_plus))
 !100 = !{i32 2, !"Debug Info Version", i32 3}
 !llvm.module.flags = !{!100}
+
+; // -----
+
+; Nameless and scopeless global variable.
+
+; CHECK-DAG: #[[FILE:.*]] = #llvm.di_file
+; CHECK-DAG: #[[COMPOSITE_TYPE:.*]] = #llvm.di_composite_type
+; CHECK-DAG: #[[GLOBAL_VAR:.*]] = #llvm.di_global_variable<file = #[[FILE]], line = 268, type = #[[COMPOSITE_TYPE]], isLocalToUnit = true, isDefined = true>
+; CHECK-DAG: #[[GLOBAL_VAR_EXPR:.*]] = #llvm.di_global_variable_expression<var = #[[GLOBAL_VAR]], expr = <>>
+
+; CHECK-DAG: llvm.mlir.global external constant @".str.1"() {addr_space = 0 : i32, dbg_expr = #[[GLOBAL_VAR_EXPR]]}
+
+@.str.1 = external constant [10 x i8], !dbg !0
+
+!llvm.module.flags = !{!7}
+
+!0 = !DIGlobalVariableExpression(var: !1, expr: !DIExpression())
+!1 = distinct !DIGlobalVariable(scope: null, file: !2, line: 268, type: !3, isLocal: true, isDefinition: true)
+!2 = !DIFile(filename: "source.c", directory: "/path/to/file")
+!3 = !DICompositeType(tag: DW_TAG_array_type, baseType: !4, size: 80, elements: !6)
+!4 = !DIDerivedType(tag: DW_TAG_const_type, baseType: !5)
+!5 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char)
+!6 = !{}
+!7 = !{i32 2, !"Debug Info Version", i32 3}
diff --git a/mlir/test/Target/LLVMIR/Import/intrinsic.ll b/mlir/test/Target/LLVMIR/Import/intrinsic.ll
index c8dcde11d93e645dc560811d032e408ea896a135..f52ad6b56d141da8b6d524de087334b4aec9e155 100644
--- a/mlir/test/Target/LLVMIR/Import/intrinsic.ll
+++ b/mlir/test/Target/LLVMIR/Import/intrinsic.ll
@@ -754,6 +754,15 @@ define void @lifetime(ptr %0) {
   ret void
 }
 
+; CHECK-LABEL: llvm.func @invariant
+define void @invariant(ptr %0) {
+  ; CHECK: %[[START:.*]] = llvm.intr.invariant.start 16, %{{.*}} : !llvm.ptr
+  %2 = call ptr @llvm.invariant.start.p0(i64 16, ptr %0)
+  ; CHECK: llvm.intr.invariant.end %[[START]], 32, %{{.*}} : !llvm.ptr
+  call void @llvm.invariant.end.p0(ptr %2, i64 32, ptr %0)
+  ret void
+}
+
 ; CHECK-LABEL: llvm.func @vector_insert
 define void @vector_insert(<vscale x 4 x float> %0, <4 x float> %1) {
   ; CHECK: llvm.intr.vector.insert %{{.*}}, %{{.*}}[4] : vector<4xf32> into !llvm.vec<? x 4 x  f32>
@@ -1096,6 +1105,9 @@ declare <8 x i64> @llvm.vp.ptrtoint.v8i64.v8p0(<8 x ptr>, <8 x i1>, i32)
 declare <8 x ptr> @llvm.vp.inttoptr.v8p0.v8i64(<8 x i64>, <8 x i1>, i32)
 declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture)
 declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
+declare ptr @llvm.invariant.start.p0(i64 immarg, ptr nocapture)
+declare void @llvm.invariant.end.p0(ptr, i64 immarg, ptr nocapture)
+
 declare void @llvm.assume(i1)
 declare float @llvm.ssa.copy.f32(float returned)
 declare <vscale x 4 x float> @llvm.vector.insert.nxv4f32.v4f32(<vscale x 4 x float>, <4 x float>, i64)
diff --git a/mlir/test/Target/LLVMIR/llvmir-debug.mlir b/mlir/test/Target/LLVMIR/llvmir-debug.mlir
index fe7f1b96d323909d6c8ee270139735c0268aa97f..1133f57d6b61eebf6e6ebe8e170cb222369d4b16 100644
--- a/mlir/test/Target/LLVMIR/llvmir-debug.mlir
+++ b/mlir/test/Target/LLVMIR/llvmir-debug.mlir
@@ -292,6 +292,24 @@ llvm.mlir.global external @global_with_expr_2() {addr_space = 0 : i32, dbg_expr
 
 // -----
 
+// Nameless and scopeless global constant.
+
+// CHECK-LABEL: @.str.1 = external constant [10 x i8]
+// CHECK-SAME: !dbg ![[GLOBAL_VAR_EXPR:.*]]
+// CHECK-DAG: ![[GLOBAL_VAR_EXPR]] = !DIGlobalVariableExpression(var: ![[GLOBAL_VAR:.*]], expr: !DIExpression())
+// CHECK-DAG: ![[GLOBAL_VAR]] = distinct !DIGlobalVariable(scope: null, file: !{{[0-9]+}}, line: 268, type: !{{[0-9]+}}, isLocal: true, isDefinition: true)
+
+#di_basic_type = #llvm.di_basic_type<tag = DW_TAG_base_type, name = "char", sizeInBits = 8, encoding = DW_ATE_signed_char>
+#di_file = #llvm.di_file<"file.c" in "/path/to/file">
+#di_derived_type = #llvm.di_derived_type<tag = DW_TAG_const_type, baseType = #di_basic_type>
+#di_composite_type = #llvm.di_composite_type<tag = DW_TAG_array_type, baseType = #di_derived_type, sizeInBits = 80>
+#di_global_variable = #llvm.di_global_variable<file = #di_file, line = 268, type = #di_composite_type, isLocalToUnit = true, isDefined = true>
+#di_global_variable_expression = #llvm.di_global_variable_expression<var = #di_global_variable, expr = <>>
+
+llvm.mlir.global external constant @".str.1"() {addr_space = 0 : i32, dbg_expr = #di_global_variable_expression} : !llvm.array<10 x i8>
+
+// -----
+
 // CHECK-DAG: ![[FILE1:.*]] = !DIFile(filename: "foo1.mlir", directory: "/test/")
 #di_file_1 = #llvm.di_file<"foo1.mlir" in "/test/">
 // CHECK-DAG: ![[FILE2:.*]] = !DIFile(filename: "foo2.mlir", directory: "/test/")
diff --git a/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir b/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
index e586c0cd2720e48c03fa6262f0ce360f31abb9de..1c0aa8d3407ac6cb8148fa1da6f6c409c750d396 100644
--- a/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
+++ b/mlir/test/Target/LLVMIR/llvmir-intrinsics.mlir
@@ -941,6 +941,15 @@ llvm.func @lifetime(%p: !llvm.ptr) {
   llvm.return
 }
 
+// CHECK-LABEL: @invariant
+llvm.func @invariant(%p: !llvm.ptr) {
+  // CHECK: call ptr @llvm.invariant.start
+  %1 = llvm.intr.invariant.start 16, %p : !llvm.ptr
+  // CHECK: call void @llvm.invariant.end
+  llvm.intr.invariant.end %1, 16, %p : !llvm.ptr
+  llvm.return
+}
+
 // CHECK-LABEL: @ssa_copy
 llvm.func @ssa_copy(%arg: f32) -> f32 {
   // CHECK: call float @llvm.ssa.copy
@@ -1101,6 +1110,9 @@ llvm.func @ssa_copy(%arg: f32) -> f32 {
 // CHECK-DAG: declare <2 x i32> @llvm.vector.extract.v2i32.v8i32(<8 x i32>, i64 immarg)
 // CHECK-DAG: declare void @llvm.lifetime.start.p0(i64 immarg, ptr nocapture)
 // CHECK-DAG: declare void @llvm.lifetime.end.p0(i64 immarg, ptr nocapture)
+// CHECK-DAG: declare ptr @llvm.invariant.start.p0(i64 immarg, ptr nocapture)
+// CHECK-DAG: declare void @llvm.invariant.end.p0(ptr, i64 immarg, ptr nocapture)
+
 // CHECK-DAG: declare float @llvm.ssa.copy.f32(float returned)
 // CHECK-DAG: declare ptr @llvm.stacksave.p0()
 // CHECK-DAG: declare ptr addrspace(1) @llvm.stacksave.p1()
diff --git a/mlir/test/Target/LLVMIR/llvmir.mlir b/mlir/test/Target/LLVMIR/llvmir.mlir
index d9e7b790dd801b4d94bc0a4b4b77245b5cc97b98..13e61b6ce10b2a89770bd02242c36ef543431ce6 100644
--- a/mlir/test/Target/LLVMIR/llvmir.mlir
+++ b/mlir/test/Target/LLVMIR/llvmir.mlir
@@ -2305,6 +2305,20 @@ llvm.func @locally_streaming_func() attributes {arm_locally_streaming} {
 
 // -----
 
+//
+// arm_streaming_compatible attribute.
+//
+
+// CHECK-LABEL: @streaming_compatible_func
+// CHECK: #[[ATTR:[0-9]*]]
+llvm.func @streaming_compatible_func() attributes {arm_streaming_compatible} {
+  llvm.return
+}
+
+// CHECK: attributes #[[ATTR]] = { "aarch64_pstate_sm_compatible" }
+
+// -----
+
 //
 // Zero-initialize operation.
 //
diff --git a/mlir/test/Transforms/canonicalize-debuginfo.mlir b/mlir/test/Transforms/canonicalize-debuginfo.mlir
deleted file mode 100644
index 034c9163a8059fb0c9e0fe4b6053ceb740a6e41a..0000000000000000000000000000000000000000
--- a/mlir/test/Transforms/canonicalize-debuginfo.mlir
+++ /dev/null
@@ -1,34 +0,0 @@
-// RUN: mlir-opt %s -pass-pipeline='builtin.module(func.func(canonicalize{test-convergence}))' -split-input-file -mlir-print-debuginfo | FileCheck %s
-
-// CHECK-LABEL: func @merge_constants
-func.func @merge_constants() -> (index, index, index, index) {
-  // CHECK-NEXT: arith.constant 42 : index loc(#[[FusedLoc:.*]])
-  %0 = arith.constant 42 : index loc("merge_constants":0:0)
-  %1 = arith.constant 42 : index loc("merge_constants":1:0)
-  %2 = arith.constant 42 : index loc("merge_constants":2:0)
-  %3 = arith.constant 42 : index loc("merge_constants":2:0) // repeated loc
-  return %0, %1, %2, %3: index, index, index, index
-}
-
-// CHECK-DAG: #[[LocConst0:.*]] = loc("merge_constants":0:0)
-// CHECK-DAG: #[[LocConst1:.*]] = loc("merge_constants":1:0)
-// CHECK-DAG: #[[LocConst2:.*]] = loc("merge_constants":2:0)
-// CHECK: #[[FusedLoc]] = loc(fused<"CSE">[#[[LocConst0]], #[[LocConst1]], #[[LocConst2]]])
-
-// -----
-
-// CHECK-LABEL: func @hoist_constant
-func.func @hoist_constant(%arg0: memref<8xi32>) {
-  // CHECK-NEXT: arith.constant 42 : i32 loc(#[[FusedLoc:.*]])
-  affine.for %arg1 = 0 to 8 {
-    %0 = arith.constant 42 : i32 loc("hoist_constant":0:0)
-    %1 = arith.constant 42 : i32 loc("hoist_constant":1:0)
-    memref.store %0, %arg0[%arg1] : memref<8xi32>
-    memref.store %1, %arg0[%arg1] : memref<8xi32>
-  }
-  return
-}
-
-// CHECK-DAG: #[[LocConst0:.*]] = loc("hoist_constant":0:0)
-// CHECK-DAG: #[[LocConst1:.*]] = loc("hoist_constant":1:0)
-// CHECK: #[[FusedLoc]] = loc(fused<"CSE">[#[[LocConst0]], #[[LocConst1]]])
diff --git a/mlir/test/Transforms/constant-fold-debuginfo.mlir b/mlir/test/Transforms/constant-fold-debuginfo.mlir
deleted file mode 100644
index 79a25f860a4841917e4afb7063e543bfa7547520..0000000000000000000000000000000000000000
--- a/mlir/test/Transforms/constant-fold-debuginfo.mlir
+++ /dev/null
@@ -1,34 +0,0 @@
-// RUN: mlir-opt %s -split-input-file -test-constant-fold -mlir-print-debuginfo | FileCheck %s
-
-// CHECK-LABEL: func @fold_and_merge
-func.func @fold_and_merge() -> (i32, i32) {
-  %0 = arith.constant 1 : i32
-  %1 = arith.constant 5 : i32
-
-  // CHECK-NEXT: [[C:%.+]] = arith.constant 6 : i32 loc(#[[FusedLoc:.*]])
-  %2 = arith.addi %0, %1 : i32 loc("fold_and_merge":0:0)
-
-  %3 = arith.constant 6 : i32 loc("fold_and_merge":1:0)
-
-  return %2, %3: i32, i32
-}
-
-// CHECK-DAG: #[[LocConst0:.*]] = loc("fold_and_merge":0:0)
-// CHECK-DAG: #[[LocConst1:.*]] = loc("fold_and_merge":1:0)
-// CHECK: #[[FusedLoc]] = loc(fused<"CSE">[#[[LocConst1]], #[[LocConst0]]])
-
-// -----
-
-// CHECK-LABEL: func @materialize_different_dialect
-func.func @materialize_different_dialect() -> (f32, f32) {
-  // CHECK: arith.constant 1.{{0*}}e+00 : f32 loc(#[[FusedLoc:.*]])
-  %0 = arith.constant -1.0 : f32
-  %1 = math.absf %0 : f32 loc("materialize_different_dialect":0:0)
-  %2 = arith.constant 1.0 : f32 loc("materialize_different_dialect":1:0)
-
-  return %1, %2: f32, f32
-}
-
-// CHECK-DAG: #[[LocConst0:.*]] = loc("materialize_different_dialect":0:0)
-// CHECK-DAG: #[[LocConst1:.*]] = loc("materialize_different_dialect":1:0)
-// CHECK: #[[FusedLoc]] = loc(fused<"CSE">[#[[LocConst1]], #[[LocConst0]]])
diff --git a/mlir/test/lib/Dialect/Arith/TestEmulateWideInt.cpp b/mlir/test/lib/Dialect/Arith/TestEmulateWideInt.cpp
index 8f0c13af9de5bfc8adb85de6124aa8318f4a25d4..a6678995fc6f67b588d13377032cb37e610d8ee9 100644
--- a/mlir/test/lib/Dialect/Arith/TestEmulateWideInt.cpp
+++ b/mlir/test/lib/Dialect/Arith/TestEmulateWideInt.cpp
@@ -48,7 +48,7 @@ struct TestEmulateWideIntPass
     }
 
     func::FuncOp op = getOperation();
-    if (!op.getSymName().startswith(testFunctionPrefix))
+    if (!op.getSymName().starts_with(testFunctionPrefix))
       return;
 
     MLIRContext *ctx = op.getContext();
diff --git a/mlir/test/lib/Dialect/Vector/TestVectorTransforms.cpp b/mlir/test/lib/Dialect/Vector/TestVectorTransforms.cpp
index e593c0defcd29ea57834d4f069130180824394f5..a643343e9342adfbfa453c2fbd3f01a73ac5dc76 100644
--- a/mlir/test/lib/Dialect/Vector/TestVectorTransforms.cpp
+++ b/mlir/test/lib/Dialect/Vector/TestVectorTransforms.cpp
@@ -455,6 +455,7 @@ struct TestFlattenVectorTransferPatterns
   void getDependentDialects(DialectRegistry &registry) const override {
     registry.insert<memref::MemRefDialect>();
     registry.insert<affine::AffineDialect>();
+    registry.insert<vector::VectorDialect>();
   }
   void runOnOperation() override {
     RewritePatternSet patterns(&getContext());
diff --git a/mlir/tools/mlir-opt/CMakeLists.txt b/mlir/tools/mlir-opt/CMakeLists.txt
index bc8eed18215525ef5163d7b7bd79d3664a9f9010..b6ada66d3218802da0d50149a7c05ef7637c0d98 100644
--- a/mlir/tools/mlir-opt/CMakeLists.txt
+++ b/mlir/tools/mlir-opt/CMakeLists.txt
@@ -67,6 +67,10 @@ set(LIBS
   MLIRTransformUtils
   MLIRSupport
   MLIRIR
+
+  # TODO: Remove when registerAllGPUToLLVMIRTranslations is no longer
+  # registered directly in mlir-opt.cpp.
+  MLIRToLLVMIRTranslationRegistration
   )
 
 # Exclude from libMLIR.so because this has static options intended for
diff --git a/mlir/tools/mlir-opt/mlir-opt.cpp b/mlir/tools/mlir-opt/mlir-opt.cpp
index c7cf1e55a556f4a99023782f9aebaf991dce1912..b7c69eabbcd8bf91c4c51ea9e82b5f536b2d6e45 100644
--- a/mlir/tools/mlir-opt/mlir-opt.cpp
+++ b/mlir/tools/mlir-opt/mlir-opt.cpp
@@ -276,6 +276,10 @@ int main(int argc, char **argv) {
   DialectRegistry registry;
   registerAllDialects(registry);
   registerAllExtensions(registry);
+
+  // TODO: Remove this and the corresponding MLIRToLLVMIRTranslationRegistration
+  // cmake dependency when a safe dialect interface registration mechanism is
+  // implemented, see D157703 (and corresponding note on the declaration).
   registerAllGPUToLLVMIRTranslations(registry);
 
 #ifdef MLIR_INCLUDE_TESTS
diff --git a/mlir/tools/mlir-parser-fuzzer/bytecode/mlir-bytecode-parser-fuzzer.cpp b/mlir/tools/mlir-parser-fuzzer/bytecode/mlir-bytecode-parser-fuzzer.cpp
index 21411b684c15af4ce56a206503c2221fb7c0784e..bb06c64d7215acc006c82fac5494785081c4d718 100644
--- a/mlir/tools/mlir-parser-fuzzer/bytecode/mlir-bytecode-parser-fuzzer.cpp
+++ b/mlir/tools/mlir-parser-fuzzer/bytecode/mlir-bytecode-parser-fuzzer.cpp
@@ -26,7 +26,7 @@ extern "C" LLVM_ATTRIBUTE_USED int LLVMFuzzerTestOneInput(const uint8_t *data,
     return -1;
   llvm::StringRef str(reinterpret_cast<const char *>(data), size - 1);
   // Skip if not bytecode.
-  if (!str.startswith("ML\xefR"))
+  if (!str.starts_with("ML\xefR"))
     return -1;
 
   // Create a null-terminated memory buffer from the input.
diff --git a/mlir/tools/mlir-parser-fuzzer/text/mlir-text-parser-fuzzer.cpp b/mlir/tools/mlir-parser-fuzzer/text/mlir-text-parser-fuzzer.cpp
index 5555c039eae9a261263165cf58a469030b864dd2..faa578bc1fde899dcd55e86ec42ccea030bda056 100644
--- a/mlir/tools/mlir-parser-fuzzer/text/mlir-text-parser-fuzzer.cpp
+++ b/mlir/tools/mlir-parser-fuzzer/text/mlir-text-parser-fuzzer.cpp
@@ -26,7 +26,7 @@ extern "C" LLVM_ATTRIBUTE_USED int LLVMFuzzerTestOneInput(const uint8_t *data,
     return -1;
   llvm::StringRef str(reinterpret_cast<const char *>(data), size - 1);
   // Skip if bytecode.
-  if (str.startswith("ML\xefR"))
+  if (str.starts_with("ML\xefR"))
     return -1;
 
   // Create a null-terminated memory buffer from the input.
diff --git a/mlir/tools/mlir-tblgen/LLVMIRIntrinsicGen.cpp b/mlir/tools/mlir-tblgen/LLVMIRIntrinsicGen.cpp
index f627d2dd3ea0845584e51e155f2d49aaced7a3df..00665f93c8a51e41c230ff6b653c084ae41138f0 100644
--- a/mlir/tools/mlir-tblgen/LLVMIRIntrinsicGen.cpp
+++ b/mlir/tools/mlir-tblgen/LLVMIRIntrinsicGen.cpp
@@ -98,7 +98,7 @@ public:
       return name.str();
 
     name = record.getName();
-    assert(name.startswith("int_") &&
+    assert(name.starts_with("int_") &&
            "LLVM intrinsic names are expected to start with 'int_'");
     name = name.drop_front(4);
     llvm::SmallVector<llvm::StringRef, 8> chunks;
@@ -121,7 +121,7 @@ public:
   /// Get the name of the record without the "intrinsic" prefix.
   llvm::StringRef getProperRecordName() const {
     llvm::StringRef name = record.getName();
-    assert(name.startswith("int_") &&
+    assert(name.starts_with("int_") &&
            "LLVM intrinsic names are expected to start with 'int_'");
     return name.drop_front(4);
   }
diff --git a/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp b/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
index 57392434285ff89181d8626faf40334049fd89d7..cd37c8dcd3d5e0eb018a6f3899d064a180738ea1 100644
--- a/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
+++ b/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp
@@ -1960,7 +1960,7 @@ static std::string generateTypeForGetter(const NamedTypeConstraint &value) {
   /// Adaptor will have values that are not from the type of their operation and
   /// this is expected, so we dont generate TypedValue for Adaptor
   if (value.constraint.getCPPClassName() != "::mlir::Type" &&
-      StringRef(value.constraint.getCPPClassName()).startswith("::"))
+      StringRef(value.constraint.getCPPClassName()).starts_with("::"))
     str = llvm::formatv("::mlir::TypedValue<{0}>",
                         value.constraint.getCPPClassName())
               .str();
diff --git a/mlir/tools/mlir-tblgen/OpDocGen.cpp b/mlir/tools/mlir-tblgen/OpDocGen.cpp
index b5b26a70859eced32fe0c05ba67c76ebb91428e3..877ef1089dcec0bc4fa428161d30cf2acc084818 100644
--- a/mlir/tools/mlir-tblgen/OpDocGen.cpp
+++ b/mlir/tools/mlir-tblgen/OpDocGen.cpp
@@ -79,7 +79,7 @@ void mlir::tblgen::emitDescriptionComment(StringRef description,
   raw_indented_ostream ros(os);
   StringRef trimmed = description.rtrim(" \t");
   ros.printReindented(trimmed, (Twine(prefix) + "/// ").str());
-  if (!trimmed.endswith("\n"))
+  if (!trimmed.ends_with("\n"))
     ros << "\n";
 }
 
@@ -137,7 +137,7 @@ static void emitOpTraitsDoc(const Operator &op, raw_ostream &os) {
     StringRef traitName = trait.getDef().getValueAsString("trait");
     traitName.consume_back("::Trait");
     traitName.consume_back("::Impl");
-    if (ref.startswith("anonymous_"))
+    if (ref.starts_with("anonymous_"))
       name = traitName.str();
     if (isa<InterfaceTrait>(&trait)) {
       if (trait.getDef().isSubClassOf("SideEffectsTraitBase")) {
diff --git a/mlir/tools/mlir-tblgen/OpPythonBindingGen.cpp b/mlir/tools/mlir-tblgen/OpPythonBindingGen.cpp
index 0c0ad2cfeffdcc28394a4fae004f790c8e3fc1bc..0770ed562309e7328336b58a63b258d3e7ed0135 100644
--- a/mlir/tools/mlir-tblgen/OpPythonBindingGen.cpp
+++ b/mlir/tools/mlir-tblgen/OpPythonBindingGen.cpp
@@ -295,7 +295,7 @@ static bool isODSReserved(StringRef str) {
       {"attributes", "create", "context", "ip", "operands", "print", "get_asm",
        "loc", "verify", "regions", "results", "self", "operation",
        "DIALECT_NAMESPACE", "OPERATION_NAME"});
-  return str.startswith("_ods_") || str.endswith("_ods") ||
+  return str.starts_with("_ods_") || str.ends_with("_ods") ||
          reserved.contains(str);
 }
 
diff --git a/mlir/unittests/Analysis/Presburger/IntegerRelationTest.cpp b/mlir/unittests/Analysis/Presburger/IntegerRelationTest.cpp
index 287f7c7c56549ff25977367be2b28efafaa10cd3..f390296da648d244cbdf68e4150e2d9e824fa6cf 100644
--- a/mlir/unittests/Analysis/Presburger/IntegerRelationTest.cpp
+++ b/mlir/unittests/Analysis/Presburger/IntegerRelationTest.cpp
@@ -8,6 +8,7 @@
 
 #include "mlir/Analysis/Presburger/IntegerRelation.h"
 #include "Parser.h"
+#include "mlir/Analysis/Presburger/PresburgerSpace.h"
 #include "mlir/Analysis/Presburger/Simplex.h"
 
 #include <gmock/gmock.h>
@@ -167,3 +168,42 @@ TEST(IntegerRelationTest, symbolicLexmax) {
   EXPECT_TRUE(lexmax3.unboundedDomain.isIntegerEmpty());
   EXPECT_TRUE(lexmax3.lexopt.isEqual(expectedLexmax3));
 }
+
+TEST(IntegerRelationTest, swapVar) {
+  PresburgerSpace space = PresburgerSpace::getRelationSpace(2, 1, 2, 0);
+  space.resetIds();
+
+  int identifiers[6] = {0, 1, 2, 3, 4};
+
+  // Attach identifiers to domain identifiers.
+  space.getId(VarKind::Domain, 0) = Identifier(&identifiers[0]);
+  space.getId(VarKind::Domain, 1) = Identifier(&identifiers[1]);
+
+  // Attach identifiers to range identifiers.
+  space.getId(VarKind::Range, 0) = Identifier(&identifiers[2]);
+
+  // Attach identifiers to symbol identifiers.
+  space.getId(VarKind::Symbol, 0) = Identifier(&identifiers[3]);
+  space.getId(VarKind::Symbol, 1) = Identifier(&identifiers[4]);
+
+  IntegerRelation rel =
+      parseRelationFromSet("(x, y, z)[N, M] : (z - x - y == 0, x >= 0, N - x "
+                           ">= 0, y >= 0, M - y >= 0)",
+                           2);
+  rel.setSpace(space);
+  // Swap (Domain 0, Range 0)
+  rel.swapVar(0, 2);
+  // Swap (Domain 1, Symbol 1)
+  rel.swapVar(1, 4);
+
+  PresburgerSpace swappedSpace = rel.getSpace();
+
+  EXPECT_TRUE(swappedSpace.getId(VarKind::Domain, 0)
+                  .isEqual(space.getId(VarKind::Range, 0)));
+  EXPECT_TRUE(swappedSpace.getId(VarKind::Domain, 1)
+                  .isEqual(space.getId(VarKind::Symbol, 1)));
+  EXPECT_TRUE(swappedSpace.getId(VarKind::Range, 0)
+                  .isEqual(space.getId(VarKind::Domain, 0)));
+  EXPECT_TRUE(swappedSpace.getId(VarKind::Symbol, 1)
+                  .isEqual(space.getId(VarKind::Domain, 1)));
+}
diff --git a/mlir/unittests/Analysis/Presburger/MatrixTest.cpp b/mlir/unittests/Analysis/Presburger/MatrixTest.cpp
index d05b05e004c5c5f21db625896e2440f89af97c39..508d4fa369c14c60a11265f6fa60cc7cbb9cbd66 100644
--- a/mlir/unittests/Analysis/Presburger/MatrixTest.cpp
+++ b/mlir/unittests/Analysis/Presburger/MatrixTest.cpp
@@ -310,3 +310,71 @@ TEST(MatrixTest, intInverse) {
 
   EXPECT_EQ(det, 0);
 }
+
+TEST(MatrixTest, gramSchmidt) {
+  FracMatrix mat =
+      makeFracMatrix(3, 5,
+                     {{Fraction(3, 1), Fraction(4, 1), Fraction(5, 1),
+                       Fraction(12, 1), Fraction(19, 1)},
+                      {Fraction(4, 1), Fraction(5, 1), Fraction(6, 1),
+                       Fraction(13, 1), Fraction(20, 1)},
+                      {Fraction(7, 1), Fraction(8, 1), Fraction(9, 1),
+                       Fraction(16, 1), Fraction(24, 1)}});
+
+  FracMatrix gramSchmidt = makeFracMatrix(
+      3, 5,
+      {{Fraction(3, 1), Fraction(4, 1), Fraction(5, 1), Fraction(12, 1),
+        Fraction(19, 1)},
+       {Fraction(142, 185), Fraction(383, 555), Fraction(68, 111),
+        Fraction(13, 185), Fraction(-262, 555)},
+       {Fraction(53, 463), Fraction(27, 463), Fraction(1, 463),
+        Fraction(-181, 463), Fraction(100, 463)}});
+
+  FracMatrix gs = mat.gramSchmidt();
+
+  EXPECT_EQ_FRAC_MATRIX(gs, gramSchmidt);
+  for (unsigned i = 0; i < 3u; i++)
+    for (unsigned j = i + 1; j < 3u; j++)
+      EXPECT_EQ(dotProduct(gs.getRow(i), gs.getRow(j)), 0);
+
+  mat = makeFracMatrix(3, 3,
+                       {{Fraction(20, 1), Fraction(17, 1), Fraction(10, 1)},
+                        {Fraction(20, 1), Fraction(18, 1), Fraction(6, 1)},
+                        {Fraction(15, 1), Fraction(14, 1), Fraction(10, 1)}});
+
+  gramSchmidt = makeFracMatrix(
+      3, 3,
+      {{Fraction(20, 1), Fraction(17, 1), Fraction(10, 1)},
+       {Fraction(460, 789), Fraction(1180, 789), Fraction(-2926, 789)},
+       {Fraction(-2925, 3221), Fraction(3000, 3221), Fraction(750, 3221)}});
+
+  gs = mat.gramSchmidt();
+
+  EXPECT_EQ_FRAC_MATRIX(gs, gramSchmidt);
+  for (unsigned i = 0; i < 3u; i++)
+    for (unsigned j = i + 1; j < 3u; j++)
+      EXPECT_EQ(dotProduct(gs.getRow(i), gs.getRow(j)), 0);
+
+  mat = makeFracMatrix(
+      4, 4,
+      {{Fraction(1, 26), Fraction(13, 12), Fraction(34, 13), Fraction(7, 10)},
+       {Fraction(40, 23), Fraction(34, 1), Fraction(11, 19), Fraction(15, 1)},
+       {Fraction(21, 22), Fraction(10, 9), Fraction(4, 11), Fraction(14, 11)},
+       {Fraction(35, 22), Fraction(1, 15), Fraction(5, 8), Fraction(30, 1)}});
+
+  gs = mat.gramSchmidt();
+
+  // The integers involved are too big to construct the actual matrix.
+  // but we can check that the result is linearly independent.
+  ASSERT_FALSE(mat.determinant(nullptr) == 0);
+
+  for (unsigned i = 0; i < 4u; i++)
+    for (unsigned j = i + 1; j < 4u; j++)
+      EXPECT_EQ(dotProduct(gs.getRow(i), gs.getRow(j)), 0);
+
+  mat = FracMatrix::identity(/*dim=*/10);
+
+  gs = mat.gramSchmidt();
+
+  EXPECT_EQ_FRAC_MATRIX(gs, FracMatrix::identity(10));
+}
\ No newline at end of file
diff --git a/openmp/libomptarget/include/ExclusiveAccess.h b/openmp/libomptarget/include/ExclusiveAccess.h
index 33f59903637dbc38d3d582c1ec9b0e70f7f2bf49..09b3aac6059dba386479913c68d37b7e4f0cd75c 100644
--- a/openmp/libomptarget/include/ExclusiveAccess.h
+++ b/openmp/libomptarget/include/ExclusiveAccess.h
@@ -11,6 +11,7 @@
 #ifndef OMPTARGET_EXCLUSIVE_ACCESS
 #define OMPTARGET_EXCLUSIVE_ACCESS
 
+#include <cassert>
 #include <cstddef>
 #include <cstdint>
 #include <mutex>
diff --git a/openmp/libomptarget/include/OpenMP/Mapping.h b/openmp/libomptarget/include/OpenMP/Mapping.h
index 9a1ecb80879207129aa659de8d4be3a80e02aab8..4bd676fc658a7d9106874ecabca4bb7a5184aaae 100644
--- a/openmp/libomptarget/include/OpenMP/Mapping.h
+++ b/openmp/libomptarget/include/OpenMP/Mapping.h
@@ -13,6 +13,7 @@
 #ifndef OMPTARGET_OPENMP_MAPPING_H
 #define OMPTARGET_OPENMP_MAPPING_H
 
+#include "ExclusiveAccess.h"
 #include "Shared/EnvironmentVar.h"
 #include "omptarget.h"
 
@@ -443,4 +444,94 @@ int targetDataUpdate(ident_t *Loc, DeviceTy &Device, int32_t ArgNum,
                      void **ArgMappers, AsyncInfoTy &AsyncInfo,
                      bool FromMapper = false);
 
+struct MappingInfoTy {
+  MappingInfoTy(DeviceTy &Device) : Device(Device) {}
+
+  /// Host data to device map type with a wrapper key indirection that allows
+  /// concurrent modification of the entries without invalidating the underlying
+  /// entries.
+  using HostDataToTargetListTy =
+      std::set<HostDataToTargetMapKeyTy, std::less<>>;
+
+  /// The HDTTMap is a protected object that can only be accessed by one thread
+  /// at a time.
+  ProtectedObj<HostDataToTargetListTy> HostDataToTargetMap;
+
+  /// The type used to access the HDTT map.
+  using HDTTMapAccessorTy = decltype(HostDataToTargetMap)::AccessorTy;
+
+  /// Lookup the mapping of \p HstPtrBegin in \p HDTTMap. The accessor ensures
+  /// exclusive access to the HDTT map.
+  LookupResult lookupMapping(HDTTMapAccessorTy &HDTTMap, void *HstPtrBegin,
+                             int64_t Size,
+                             HostDataToTargetTy *OwnedTPR = nullptr);
+
+  /// Get the target pointer based on host pointer begin and base. If the
+  /// mapping already exists, the target pointer will be returned directly. In
+  /// addition, if required, the memory region pointed by \p HstPtrBegin of size
+  /// \p Size will also be transferred to the device. If the mapping doesn't
+  /// exist, and if unified shared memory is not enabled, a new mapping will be
+  /// created and the data will also be transferred accordingly. nullptr will be
+  /// returned because of any of following reasons:
+  /// - Data allocation failed;
+  /// - The user tried to do an illegal mapping;
+  /// - Data transfer issue fails.
+  TargetPointerResultTy getTargetPointer(
+      HDTTMapAccessorTy &HDTTMap, void *HstPtrBegin, void *HstPtrBase,
+      int64_t TgtPadding, int64_t Size, map_var_info_t HstPtrName,
+      bool HasFlagTo, bool HasFlagAlways, bool IsImplicit, bool UpdateRefCount,
+      bool HasCloseModifier, bool HasPresentModifier, bool HasHoldModifier,
+      AsyncInfoTy &AsyncInfo, HostDataToTargetTy *OwnedTPR = nullptr,
+      bool ReleaseHDTTMap = true);
+
+  /// Return the target pointer for \p HstPtrBegin in \p HDTTMap. The accessor
+  /// ensures exclusive access to the HDTT map.
+  void *getTgtPtrBegin(HDTTMapAccessorTy &HDTTMap, void *HstPtrBegin,
+                       int64_t Size);
+
+  /// Return the target pointer begin (where the data will be moved).
+  /// Used by targetDataBegin, targetDataEnd, targetDataUpdate and target.
+  /// - \p UpdateRefCount and \p UseHoldRefCount controls which and if the entry
+  /// reference counters will be decremented.
+  /// - \p MustContain enforces that the query must not extend beyond an already
+  /// mapped entry to be valid.
+  /// - \p ForceDelete deletes the entry regardless of its reference counting
+  /// (unless it is infinite).
+  /// - \p FromDataEnd tracks the number of threads referencing the entry at
+  /// targetDataEnd for delayed deletion purpose.
+  [[nodiscard]] TargetPointerResultTy
+  getTgtPtrBegin(void *HstPtrBegin, int64_t Size, bool UpdateRefCount,
+                 bool UseHoldRefCount, bool MustContain = false,
+                 bool ForceDelete = false, bool FromDataEnd = false);
+
+  /// Remove the \p Entry from the data map. Expect the entry's total reference
+  /// count to be zero and the caller thread to be the last one using it. \p
+  /// HDTTMap ensure the caller holds exclusive access and can modify the map.
+  /// Return \c OFFLOAD_SUCCESS if the map entry existed, and return \c
+  /// OFFLOAD_FAIL if not. It is the caller's responsibility to skip calling
+  /// this function if the map entry is not expected to exist because \p
+  /// HstPtrBegin uses shared memory.
+  [[nodiscard]] int eraseMapEntry(HDTTMapAccessorTy &HDTTMap,
+                                  HostDataToTargetTy *Entry, int64_t Size);
+
+  /// Deallocate the \p Entry from the device memory and delete it. Return \c
+  /// OFFLOAD_SUCCESS if the deallocation operations executed successfully, and
+  /// return \c OFFLOAD_FAIL otherwise.
+  [[nodiscard]] int deallocTgtPtrAndEntry(HostDataToTargetTy *Entry,
+                                          int64_t Size);
+
+  int associatePtr(void *HstPtrBegin, void *TgtPtrBegin, int64_t Size);
+  int disassociatePtr(void *HstPtrBegin);
+
+  /// Print information about the transfer from \p HstPtr to \p TgtPtr (or vice
+  /// versa if \p H2D is false). If there is an existing mapping, or if \p Entry
+  /// is set, the associated metadata will be printed as well.
+  void printCopyInfo(void *TgtPtr, void *HstPtr, int64_t Size, bool H2D,
+                     HostDataToTargetTy *Entry,
+                     MappingInfoTy::HDTTMapAccessorTy *HDTTMapPtr);
+
+private:
+  DeviceTy &Device;
+};
+
 #endif // OMPTARGET_OPENMP_MAPPING_H
diff --git a/openmp/libomptarget/include/device.h b/openmp/libomptarget/include/device.h
index a84551accaf92a3add8dddfd13ed1ef0f4c2ec70..d28d3c508faf56a2be1a41a441f8863f2853e0b2 100644
--- a/openmp/libomptarget/include/device.h
+++ b/openmp/libomptarget/include/device.h
@@ -53,19 +53,6 @@ struct DeviceTy {
 
   bool HasMappedGlobalData = false;
 
-  /// Host data to device map type with a wrapper key indirection that allows
-  /// concurrent modification of the entries without invalidating the underlying
-  /// entries.
-  using HostDataToTargetListTy =
-      std::set<HostDataToTargetMapKeyTy, std::less<>>;
-
-  /// The HDTTMap is a protected object that can only be accessed by one thread
-  /// at a time.
-  ProtectedObj<HostDataToTargetListTy> HostDataToTargetMap;
-
-  /// The type used to access the HDTT map.
-  using HDTTMapAccessorTy = decltype(HostDataToTargetMap)::AccessorTy;
-
   PendingCtorsDtorsPerLibrary PendingCtorsDtors;
 
   std::mutex PendingGlobalsMtx;
@@ -80,71 +67,8 @@ struct DeviceTy {
   /// Try to initialize the device and return any failure.
   llvm::Error init();
 
-  // Return true if data can be copied to DstDevice directly
-  bool isDataExchangable(const DeviceTy &DstDevice);
-
-  /// Lookup the mapping of \p HstPtrBegin in \p HDTTMap. The accessor ensures
-  /// exclusive access to the HDTT map.
-  LookupResult lookupMapping(HDTTMapAccessorTy &HDTTMap, void *HstPtrBegin,
-                             int64_t Size,
-                             HostDataToTargetTy *OwnedTPR = nullptr);
-
-  /// Get the target pointer based on host pointer begin and base. If the
-  /// mapping already exists, the target pointer will be returned directly. In
-  /// addition, if required, the memory region pointed by \p HstPtrBegin of size
-  /// \p Size will also be transferred to the device. If the mapping doesn't
-  /// exist, and if unified shared memory is not enabled, a new mapping will be
-  /// created and the data will also be transferred accordingly. nullptr will be
-  /// returned because of any of following reasons:
-  /// - Data allocation failed;
-  /// - The user tried to do an illegal mapping;
-  /// - Data transfer issue fails.
-  TargetPointerResultTy getTargetPointer(
-      HDTTMapAccessorTy &HDTTMap, void *HstPtrBegin, void *HstPtrBase,
-      int64_t TgtPadding, int64_t Size, map_var_info_t HstPtrName,
-      bool HasFlagTo, bool HasFlagAlways, bool IsImplicit, bool UpdateRefCount,
-      bool HasCloseModifier, bool HasPresentModifier, bool HasHoldModifier,
-      AsyncInfoTy &AsyncInfo, HostDataToTargetTy *OwnedTPR = nullptr,
-      bool ReleaseHDTTMap = true);
-
-  /// Return the target pointer for \p HstPtrBegin in \p HDTTMap. The accessor
-  /// ensures exclusive access to the HDTT map.
-  void *getTgtPtrBegin(HDTTMapAccessorTy &HDTTMap, void *HstPtrBegin,
-                       int64_t Size);
-
-  /// Return the target pointer begin (where the data will be moved).
-  /// Used by targetDataBegin, targetDataEnd, targetDataUpdate and target.
-  /// - \p UpdateRefCount and \p UseHoldRefCount controls which and if the entry
-  /// reference counters will be decremented.
-  /// - \p MustContain enforces that the query must not extend beyond an already
-  /// mapped entry to be valid.
-  /// - \p ForceDelete deletes the entry regardless of its reference counting
-  /// (unless it is infinite).
-  /// - \p FromDataEnd tracks the number of threads referencing the entry at
-  /// targetDataEnd for delayed deletion purpose.
-  [[nodiscard]] TargetPointerResultTy
-  getTgtPtrBegin(void *HstPtrBegin, int64_t Size, bool UpdateRefCount,
-                 bool UseHoldRefCount, bool MustContain = false,
-                 bool ForceDelete = false, bool FromDataEnd = false);
-
-  /// Remove the \p Entry from the data map. Expect the entry's total reference
-  /// count to be zero and the caller thread to be the last one using it. \p
-  /// HDTTMap ensure the caller holds exclusive access and can modify the map.
-  /// Return \c OFFLOAD_SUCCESS if the map entry existed, and return \c
-  /// OFFLOAD_FAIL if not. It is the caller's responsibility to skip calling
-  /// this function if the map entry is not expected to exist because \p
-  /// HstPtrBegin uses shared memory.
-  [[nodiscard]] int eraseMapEntry(HDTTMapAccessorTy &HDTTMap,
-                                  HostDataToTargetTy *Entry, int64_t Size);
-
-  /// Deallocate the \p Entry from the device memory and delete it. Return \c
-  /// OFFLOAD_SUCCESS if the deallocation operations executed successfully, and
-  /// return \c OFFLOAD_FAIL otherwise.
-  [[nodiscard]] int deallocTgtPtrAndEntry(HostDataToTargetTy *Entry,
-                                          int64_t Size);
-
-  int associatePtr(void *HstPtrBegin, void *TgtPtrBegin, int64_t Size);
-  int disassociatePtr(void *HstPtrBegin);
+  /// Provide access to the mapping handler.
+  MappingInfoTy &getMappingInfo() { return MappingInfo; }
 
   __tgt_target_table *loadBinary(__tgt_device_image *Img);
 
@@ -159,6 +83,7 @@ struct DeviceTy {
   /// be used (host, shared, device).
   void *allocData(int64_t Size, void *HstPtr = nullptr,
                   int32_t Kind = TARGET_ALLOC_DEFAULT);
+
   /// Deallocates memory which \p TgtPtrBegin points at and returns
   /// OFFLOAD_SUCCESS/OFFLOAD_FAIL when succeeds/fails. p Kind dictates what
   /// allocator should be used (host, shared, device).
@@ -170,12 +95,16 @@ struct DeviceTy {
   int32_t submitData(void *TgtPtrBegin, void *HstPtrBegin, int64_t Size,
                      AsyncInfoTy &AsyncInfo,
                      HostDataToTargetTy *Entry = nullptr,
-                     DeviceTy::HDTTMapAccessorTy *HDTTMapPtr = nullptr);
+                     MappingInfoTy::HDTTMapAccessorTy *HDTTMapPtr = nullptr);
+
   // Copy data from device back to host
   int32_t retrieveData(void *HstPtrBegin, void *TgtPtrBegin, int64_t Size,
                        AsyncInfoTy &AsyncInfo,
                        HostDataToTargetTy *Entry = nullptr,
-                       DeviceTy::HDTTMapAccessorTy *HDTTMapPtr = nullptr);
+                       MappingInfoTy::HDTTMapAccessorTy *HDTTMapPtr = nullptr);
+
+  // Return true if data can be copied to DstDevice directly
+  bool isDataExchangable(const DeviceTy &DstDevice);
 
   // Copy data from current device to destination device directly
   int32_t dataExchange(void *SrcPtr, DeviceTy &DstDev, void *DstPtr,
@@ -240,7 +169,12 @@ private:
   void deinit();
 
   /// All offload entries available on this device.
-  llvm::DenseMap<llvm::StringRef, OffloadEntryTy *> DeviceOffloadEntries;
+  using DeviceOffloadEntriesMapTy =
+      llvm::DenseMap<llvm::StringRef, OffloadEntryTy *>;
+  ProtectedObj<DeviceOffloadEntriesMapTy> DeviceOffloadEntries;
+
+  /// Handler to collect and organize host-2-device mapping information.
+  MappingInfoTy MappingInfo;
 };
 
 #endif
diff --git a/openmp/libomptarget/plugins-nextgen/common/include/GlobalHandler.h b/openmp/libomptarget/plugins-nextgen/common/include/GlobalHandler.h
index fa788c5e1d02b26bac1493a8aa190e611638f599..fa079ac9660ee002533d7ffa1654ad8d917cc986 100644
--- a/openmp/libomptarget/plugins-nextgen/common/include/GlobalHandler.h
+++ b/openmp/libomptarget/plugins-nextgen/common/include/GlobalHandler.h
@@ -92,12 +92,6 @@ class GenericGlobalHandlerTy {
   /// Map to store the ELF object files that have been loaded.
   llvm::DenseMap<int32_t, ELF64LEObjectFile> ELFObjectFiles;
 
-  /// Extract the global's information from the ELF image, section, and symbol.
-  virtual Error getGlobalMetadataFromELF(const DeviceImageTy &Image,
-                                         const ELF64LE::Sym &Symbol,
-                                         const ELF64LE::Shdr &Section,
-                                         GlobalTy &ImageGlobal);
-
   /// Actually move memory between host and device. See readGlobalFromDevice and
   /// writeGlobalToDevice for the interface description.
   Error moveGlobalBetweenDeviceAndHost(GenericDeviceTy &Device,
diff --git a/openmp/libomptarget/plugins-nextgen/common/src/GlobalHandler.cpp b/openmp/libomptarget/plugins-nextgen/common/src/GlobalHandler.cpp
index 0a19148ca4ec6b24cb469ff94e2a6a2effbbc30f..3a272e228c7dfe62354dd90c9f51c13c5e98d6ae 100644
--- a/openmp/libomptarget/plugins-nextgen/common/src/GlobalHandler.cpp
+++ b/openmp/libomptarget/plugins-nextgen/common/src/GlobalHandler.cpp
@@ -16,10 +16,8 @@
 
 #include "Shared/Utils.h"
 
-#include "llvm/BinaryFormat/ELF.h"
 #include "llvm/Support/Error.h"
 
-#include <cstdint>
 #include <cstring>
 
 using namespace llvm;
@@ -52,27 +50,6 @@ GenericGlobalHandlerTy::getOrCreateELFObjectFile(const GenericDeviceTy &Device,
   return &Result.first->second;
 }
 
-Error GenericGlobalHandlerTy::getGlobalMetadataFromELF(
-    const DeviceImageTy &Image, const ELF64LE::Sym &Symbol,
-    const ELF64LE::Shdr &Section, GlobalTy &ImageGlobal) {
-
-  // The global's address is computed as the image begin + the ELF section
-  // offset + the ELF symbol value except for NOBITS sections that, as the name
-  // suggests, have no bits in the image. We still record the size and use
-  // nullptr to indicate there is no location.
-  if (Section.sh_type == ELF::SHT_NOBITS)
-    ImageGlobal.setPtr(nullptr);
-  else
-    ImageGlobal.setPtr(
-        advanceVoidPtr(Image.getStart(),
-                       Section.sh_offset - Section.sh_addr + Symbol.st_value));
-
-  // Set the global's size.
-  ImageGlobal.setSize(Symbol.st_size);
-
-  return Plugin::success();
-}
-
 Error GenericGlobalHandlerTy::moveGlobalBetweenDeviceAndHost(
     GenericDeviceTy &Device, DeviceImageTy &Image, const GlobalTy &HostGlobal,
     bool Device2Host) {
@@ -156,15 +133,18 @@ Error GenericGlobalHandlerTy::getGlobalMetadataFromImage(
     return Plugin::error("Failed to find global symbol '%s' in the ELF image",
                          ImageGlobal.getName().data());
 
+  auto AddrOrErr = utils::elf::getSymbolAddress(*ELFObj, **SymOrErr);
   // Get the section to which the symbol belongs.
-  auto SecOrErr = ELFObj->getELFFile().getSection((*SymOrErr)->st_shndx);
-  if (!SecOrErr)
-    return Plugin::error("Failed to get ELF section from global '%s': %s",
+  if (!AddrOrErr)
+    return Plugin::error("Failed to get ELF symbol from global '%s': %s",
                          ImageGlobal.getName().data(),
-                         toString(SecOrErr.takeError()).data());
+                         toString(AddrOrErr.takeError()).data());
 
   // Setup the global symbol's address and size.
-  return getGlobalMetadataFromELF(Image, **SymOrErr, **SecOrErr, ImageGlobal);
+  ImageGlobal.setPtr(const_cast<void *>(*AddrOrErr));
+  ImageGlobal.setSize((*SymOrErr)->st_size);
+
+  return Plugin::success();
 }
 
 Error GenericGlobalHandlerTy::readGlobalFromImage(GenericDeviceTy &Device,
@@ -180,10 +160,6 @@ Error GenericGlobalHandlerTy::readGlobalFromImage(GenericDeviceTy &Device,
                          "%u bytes in the ELF image but %u bytes on the host",
                          HostGlobal.getName().data(), ImageGlobal.getSize(),
                          HostGlobal.getSize());
-  if (ImageGlobal.getPtr() == nullptr)
-    return Plugin::error("Transfer impossible because global symbol '%s' has "
-                         "no representation in the image (NOBITS sections)",
-                         HostGlobal.getName().data());
 
   DP("Global symbol '%s' was found in the ELF image and %u bytes will copied "
      "from %p to %p.\n",
diff --git a/openmp/libomptarget/plugins-nextgen/common/src/Utils/ELF.cpp b/openmp/libomptarget/plugins-nextgen/common/src/Utils/ELF.cpp
index 0643d004df17e68765a869ec9e73c69916763778..305ea7d9c874b4475bc8cb566a80e43792f3b609 100644
--- a/openmp/libomptarget/plugins-nextgen/common/src/Utils/ELF.cpp
+++ b/openmp/libomptarget/plugins-nextgen/common/src/Utils/ELF.cpp
@@ -261,3 +261,27 @@ utils::elf::getSymbol(const ELFObjectFile<ELF64LE> &ELFObj, StringRef Name) {
 
   return nullptr;
 }
+
+Expected<const void *> utils::elf::getSymbolAddress(
+    const object::ELFObjectFile<object::ELF64LE> &ELFObj,
+    const object::ELF64LE::Sym &Symbol) {
+  const ELFFile<ELF64LE> &ELFFile = ELFObj.getELFFile();
+
+  auto SecOrErr = ELFFile.getSection(Symbol.st_shndx);
+  if (!SecOrErr)
+    return SecOrErr.takeError();
+  const auto &Section = *SecOrErr;
+
+  // A section with SHT_NOBITS occupies no space in the file and has no offset.
+  if (Section->sh_type == ELF::SHT_NOBITS)
+    return createError(
+        "invalid sh_type for symbol lookup, cannot be SHT_NOBITS");
+
+  uint64_t Offset = Section->sh_offset - Section->sh_addr + Symbol.st_value;
+  if (Offset > ELFFile.getBufSize())
+    return createError("invalid offset [" + Twine(Offset) +
+                       "] into ELF file of size [" +
+                       Twine(ELFFile.getBufSize()) + "]");
+
+  return ELFFile.base() + Offset;
+}
diff --git a/openmp/libomptarget/plugins-nextgen/common/src/Utils/ELF.h b/openmp/libomptarget/plugins-nextgen/common/src/Utils/ELF.h
index b3740a104d6130d827a915f7dc7d47cc9a75789f..7b58cbaf59acefe4c3ff4cd3fb82859a8cc70ec2 100644
--- a/openmp/libomptarget/plugins-nextgen/common/src/Utils/ELF.h
+++ b/openmp/libomptarget/plugins-nextgen/common/src/Utils/ELF.h
@@ -25,6 +25,11 @@ namespace elf {
 /// e_machine matches \p target_id; return zero otherwise.
 int32_t checkMachine(__tgt_device_image *Image, uint16_t TargetId);
 
+/// Returns a pointer to the given \p Symbol inside of an ELF object.
+llvm::Expected<const void *> getSymbolAddress(
+    const llvm::object::ELFObjectFile<llvm::object::ELF64LE> &ELFObj,
+    const llvm::object::ELF64LE::Sym &Symbol);
+
 /// Returns the symbol associated with the \p Name in the \p ELFObj. It will
 /// first search for the hash sections to identify symbols from the hash table.
 /// If that fails it will fall back to a linear search in the case of an
diff --git a/openmp/libomptarget/src/OpenMP/API.cpp b/openmp/libomptarget/src/OpenMP/API.cpp
index b73315f2b77a3f49204aedc113fa770df58fd429..1769404faf8884d06f277d84921d9cad5175a27a 100644
--- a/openmp/libomptarget/src/OpenMP/API.cpp
+++ b/openmp/libomptarget/src/OpenMP/API.cpp
@@ -150,9 +150,9 @@ EXTERN int omp_target_is_present(const void *Ptr, int DeviceNum) {
   // only check 1 byte. Cannot set size 0 which checks whether the pointer (zero
   // lengh array) is mapped instead of the referred storage.
   TargetPointerResultTy TPR =
-      DeviceOrErr->getTgtPtrBegin(const_cast<void *>(Ptr), 1,
-                                  /*UpdateRefCount=*/false,
-                                  /*UseHoldRefCount=*/false);
+      DeviceOrErr->getMappingInfo().getTgtPtrBegin(const_cast<void *>(Ptr), 1,
+                                                   /*UpdateRefCount=*/false,
+                                                   /*UseHoldRefCount=*/false);
   int Rc = TPR.isPresent();
   DP("Call to omp_target_is_present returns %d\n", Rc);
   return Rc;
@@ -544,8 +544,8 @@ EXTERN int omp_target_associate_ptr(const void *HostPtr, const void *DevicePtr,
     FATAL_MESSAGE(DeviceNum, "%s", toString(DeviceOrErr.takeError()).c_str());
 
   void *DeviceAddr = (void *)((uint64_t)DevicePtr + (uint64_t)DeviceOffset);
-  int Rc = DeviceOrErr->associatePtr(const_cast<void *>(HostPtr),
-                                     const_cast<void *>(DeviceAddr), Size);
+  int Rc = DeviceOrErr->getMappingInfo().associatePtr(
+      const_cast<void *>(HostPtr), const_cast<void *>(DeviceAddr), Size);
   DP("omp_target_associate_ptr returns %d\n", Rc);
   return Rc;
 }
@@ -571,7 +571,8 @@ EXTERN int omp_target_disassociate_ptr(const void *HostPtr, int DeviceNum) {
   if (!DeviceOrErr)
     FATAL_MESSAGE(DeviceNum, "%s", toString(DeviceOrErr.takeError()).c_str());
 
-  int Rc = DeviceOrErr->disassociatePtr(const_cast<void *>(HostPtr));
+  int Rc = DeviceOrErr->getMappingInfo().disassociatePtr(
+      const_cast<void *>(HostPtr));
   DP("omp_target_disassociate_ptr returns %d\n", Rc);
   return Rc;
 }
@@ -603,9 +604,9 @@ EXTERN void *omp_get_mapped_ptr(const void *Ptr, int DeviceNum) {
     FATAL_MESSAGE(DeviceNum, "%s", toString(DeviceOrErr.takeError()).c_str());
 
   TargetPointerResultTy TPR =
-      DeviceOrErr->getTgtPtrBegin(const_cast<void *>(Ptr), 1,
-                                  /*UpdateRefCount=*/false,
-                                  /*UseHoldRefCount=*/false);
+      DeviceOrErr->getMappingInfo().getTgtPtrBegin(const_cast<void *>(Ptr), 1,
+                                                   /*UpdateRefCount=*/false,
+                                                   /*UseHoldRefCount=*/false);
   if (!TPR.isPresent()) {
     DP("Ptr " DPxMOD "is not present on device %d, returning nullptr.\n",
        DPxPTR(Ptr), DeviceNum);
diff --git a/openmp/libomptarget/src/OpenMP/Mapping.cpp b/openmp/libomptarget/src/OpenMP/Mapping.cpp
index c7cb5b6b1e4c45d4dcda8899cdf6b0959fe43ea6..a5c24810e0af9565cb09b63b1e52c8567955bfad 100644
--- a/openmp/libomptarget/src/OpenMP/Mapping.cpp
+++ b/openmp/libomptarget/src/OpenMP/Mapping.cpp
@@ -10,13 +10,15 @@
 
 #include "OpenMP/Mapping.h"
 
+#include "PluginManager.h"
 #include "Shared/Debug.h"
+#include "Shared/Requirements.h"
 #include "device.h"
 
 /// Dump a table of all the host-target pointer pairs on failure
 void dumpTargetPointerMappings(const ident_t *Loc, DeviceTy &Device) {
-  DeviceTy::HDTTMapAccessorTy HDTTMap =
-      Device.HostDataToTargetMap.getExclusiveAccessor();
+  MappingInfoTy::HDTTMapAccessorTy HDTTMap =
+      Device.getMappingInfo().HostDataToTargetMap.getExclusiveAccessor();
   if (HDTTMap->empty())
     return;
 
@@ -38,3 +40,485 @@ void dumpTargetPointerMappings(const ident_t *Loc, DeviceTy &Device) {
          Info.getLine(), Info.getColumn());
   }
 }
+
+int MappingInfoTy::associatePtr(void *HstPtrBegin, void *TgtPtrBegin,
+                                int64_t Size) {
+  HDTTMapAccessorTy HDTTMap = HostDataToTargetMap.getExclusiveAccessor();
+
+  // Check if entry exists
+  auto It = HDTTMap->find(HstPtrBegin);
+  if (It != HDTTMap->end()) {
+    HostDataToTargetTy &HDTT = *It->HDTT;
+    std::lock_guard<HostDataToTargetTy> LG(HDTT);
+    // Mapping already exists
+    bool IsValid = HDTT.HstPtrEnd == (uintptr_t)HstPtrBegin + Size &&
+                   HDTT.TgtPtrBegin == (uintptr_t)TgtPtrBegin;
+    if (IsValid) {
+      DP("Attempt to re-associate the same device ptr+offset with the same "
+         "host ptr, nothing to do\n");
+      return OFFLOAD_SUCCESS;
+    }
+    REPORT("Not allowed to re-associate a different device ptr+offset with "
+           "the same host ptr\n");
+    return OFFLOAD_FAIL;
+  }
+
+  // Mapping does not exist, allocate it with refCount=INF
+  const HostDataToTargetTy &NewEntry =
+      *HDTTMap
+           ->emplace(new HostDataToTargetTy(
+               /*HstPtrBase=*/(uintptr_t)HstPtrBegin,
+               /*HstPtrBegin=*/(uintptr_t)HstPtrBegin,
+               /*HstPtrEnd=*/(uintptr_t)HstPtrBegin + Size,
+               /*TgtAllocBegin=*/(uintptr_t)TgtPtrBegin,
+               /*TgtPtrBegin=*/(uintptr_t)TgtPtrBegin,
+               /*UseHoldRefCount=*/false, /*Name=*/nullptr,
+               /*IsRefCountINF=*/true))
+           .first->HDTT;
+  DP("Creating new map entry: HstBase=" DPxMOD ", HstBegin=" DPxMOD
+     ", HstEnd=" DPxMOD ", TgtBegin=" DPxMOD ", DynRefCount=%s, "
+     "HoldRefCount=%s\n",
+     DPxPTR(NewEntry.HstPtrBase), DPxPTR(NewEntry.HstPtrBegin),
+     DPxPTR(NewEntry.HstPtrEnd), DPxPTR(NewEntry.TgtPtrBegin),
+     NewEntry.dynRefCountToStr().c_str(), NewEntry.holdRefCountToStr().c_str());
+  (void)NewEntry;
+
+  // Notify the plugin about the new mapping.
+  return Device.notifyDataMapped(HstPtrBegin, Size);
+}
+
+int MappingInfoTy::disassociatePtr(void *HstPtrBegin) {
+  HDTTMapAccessorTy HDTTMap = HostDataToTargetMap.getExclusiveAccessor();
+
+  auto It = HDTTMap->find(HstPtrBegin);
+  if (It == HDTTMap->end()) {
+    REPORT("Association not found\n");
+    return OFFLOAD_FAIL;
+  }
+  // Mapping exists
+  HostDataToTargetTy &HDTT = *It->HDTT;
+  std::lock_guard<HostDataToTargetTy> LG(HDTT);
+
+  if (HDTT.getHoldRefCount()) {
+    // This is based on OpenACC 3.1, sec 3.2.33 "acc_unmap_data", L3656-3657:
+    // "It is an error to call acc_unmap_data if the structured reference
+    // count for the pointer is not zero."
+    REPORT("Trying to disassociate a pointer with a non-zero hold reference "
+           "count\n");
+    return OFFLOAD_FAIL;
+  }
+
+  if (HDTT.isDynRefCountInf()) {
+    DP("Association found, removing it\n");
+    void *Event = HDTT.getEvent();
+    delete &HDTT;
+    if (Event)
+      Device.destroyEvent(Event);
+    HDTTMap->erase(It);
+    return Device.notifyDataUnmapped(HstPtrBegin);
+  }
+
+  REPORT("Trying to disassociate a pointer which was not mapped via "
+         "omp_target_associate_ptr\n");
+  return OFFLOAD_FAIL;
+}
+
+LookupResult MappingInfoTy::lookupMapping(HDTTMapAccessorTy &HDTTMap,
+                                          void *HstPtrBegin, int64_t Size,
+                                          HostDataToTargetTy *OwnedTPR) {
+
+  uintptr_t HP = (uintptr_t)HstPtrBegin;
+  LookupResult LR;
+
+  DP("Looking up mapping(HstPtrBegin=" DPxMOD ", Size=%" PRId64 ")...\n",
+     DPxPTR(HP), Size);
+
+  if (HDTTMap->empty())
+    return LR;
+
+  auto Upper = HDTTMap->upper_bound(HP);
+
+  if (Size == 0) {
+    // specification v5.1 Pointer Initialization for Device Data Environments
+    // upper_bound satisfies
+    //   std::prev(upper)->HDTT.HstPtrBegin <= hp < upper->HDTT.HstPtrBegin
+    if (Upper != HDTTMap->begin()) {
+      LR.TPR.setEntry(std::prev(Upper)->HDTT, OwnedTPR);
+      // the left side of extended address range is satisified.
+      // hp >= LR.TPR.getEntry()->HstPtrBegin || hp >=
+      // LR.TPR.getEntry()->HstPtrBase
+      LR.Flags.IsContained = HP < LR.TPR.getEntry()->HstPtrEnd ||
+                             HP < LR.TPR.getEntry()->HstPtrBase;
+    }
+
+    if (!LR.Flags.IsContained && Upper != HDTTMap->end()) {
+      LR.TPR.setEntry(Upper->HDTT, OwnedTPR);
+      // the right side of extended address range is satisified.
+      // hp < LR.TPR.getEntry()->HstPtrEnd || hp < LR.TPR.getEntry()->HstPtrBase
+      LR.Flags.IsContained = HP >= LR.TPR.getEntry()->HstPtrBase;
+    }
+  } else {
+    // check the left bin
+    if (Upper != HDTTMap->begin()) {
+      LR.TPR.setEntry(std::prev(Upper)->HDTT, OwnedTPR);
+      // Is it contained?
+      LR.Flags.IsContained = HP >= LR.TPR.getEntry()->HstPtrBegin &&
+                             HP < LR.TPR.getEntry()->HstPtrEnd &&
+                             (HP + Size) <= LR.TPR.getEntry()->HstPtrEnd;
+      // Does it extend beyond the mapped region?
+      LR.Flags.ExtendsAfter = HP < LR.TPR.getEntry()->HstPtrEnd &&
+                              (HP + Size) > LR.TPR.getEntry()->HstPtrEnd;
+    }
+
+    // check the right bin
+    if (!(LR.Flags.IsContained || LR.Flags.ExtendsAfter) &&
+        Upper != HDTTMap->end()) {
+      LR.TPR.setEntry(Upper->HDTT, OwnedTPR);
+      // Does it extend into an already mapped region?
+      LR.Flags.ExtendsBefore = HP < LR.TPR.getEntry()->HstPtrBegin &&
+                               (HP + Size) > LR.TPR.getEntry()->HstPtrBegin;
+      // Does it extend beyond the mapped region?
+      LR.Flags.ExtendsAfter = HP < LR.TPR.getEntry()->HstPtrEnd &&
+                              (HP + Size) > LR.TPR.getEntry()->HstPtrEnd;
+    }
+
+    if (LR.Flags.ExtendsBefore) {
+      DP("WARNING: Pointer is not mapped but section extends into already "
+         "mapped data\n");
+    }
+    if (LR.Flags.ExtendsAfter) {
+      DP("WARNING: Pointer is already mapped but section extends beyond mapped "
+         "region\n");
+    }
+  }
+
+  return LR;
+}
+
+TargetPointerResultTy MappingInfoTy::getTargetPointer(
+    HDTTMapAccessorTy &HDTTMap, void *HstPtrBegin, void *HstPtrBase,
+    int64_t TgtPadding, int64_t Size, map_var_info_t HstPtrName, bool HasFlagTo,
+    bool HasFlagAlways, bool IsImplicit, bool UpdateRefCount,
+    bool HasCloseModifier, bool HasPresentModifier, bool HasHoldModifier,
+    AsyncInfoTy &AsyncInfo, HostDataToTargetTy *OwnedTPR, bool ReleaseHDTTMap) {
+
+  LookupResult LR = lookupMapping(HDTTMap, HstPtrBegin, Size, OwnedTPR);
+  LR.TPR.Flags.IsPresent = true;
+
+  // Release the mapping table lock only after the entry is locked by
+  // attaching it to TPR. Once TPR is destroyed it will release the lock
+  // on entry. If it is returned the lock will move to the returned object.
+  // If LR.Entry is already owned/locked we avoid trying to lock it again.
+
+  // Check if the pointer is contained.
+  // If a variable is mapped to the device manually by the user - which would
+  // lead to the IsContained flag to be true - then we must ensure that the
+  // device address is returned even under unified memory conditions.
+  if (LR.Flags.IsContained ||
+      ((LR.Flags.ExtendsBefore || LR.Flags.ExtendsAfter) && IsImplicit)) {
+    const char *RefCountAction;
+    if (UpdateRefCount) {
+      // After this, reference count >= 1. If the reference count was 0 but the
+      // entry was still there we can reuse the data on the device and avoid a
+      // new submission.
+      LR.TPR.getEntry()->incRefCount(HasHoldModifier);
+      RefCountAction = " (incremented)";
+    } else {
+      // It might have been allocated with the parent, but it's still new.
+      LR.TPR.Flags.IsNewEntry = LR.TPR.getEntry()->getTotalRefCount() == 1;
+      RefCountAction = " (update suppressed)";
+    }
+    const char *DynRefCountAction = HasHoldModifier ? "" : RefCountAction;
+    const char *HoldRefCountAction = HasHoldModifier ? RefCountAction : "";
+    uintptr_t Ptr = LR.TPR.getEntry()->TgtPtrBegin +
+                    ((uintptr_t)HstPtrBegin - LR.TPR.getEntry()->HstPtrBegin);
+    INFO(OMP_INFOTYPE_MAPPING_EXISTS, Device.DeviceID,
+         "Mapping exists%s with HstPtrBegin=" DPxMOD ", TgtPtrBegin=" DPxMOD
+         ", Size=%" PRId64 ", DynRefCount=%s%s, HoldRefCount=%s%s, Name=%s\n",
+         (IsImplicit ? " (implicit)" : ""), DPxPTR(HstPtrBegin), DPxPTR(Ptr),
+         Size, LR.TPR.getEntry()->dynRefCountToStr().c_str(), DynRefCountAction,
+         LR.TPR.getEntry()->holdRefCountToStr().c_str(), HoldRefCountAction,
+         (HstPtrName) ? getNameFromMapping(HstPtrName).c_str() : "unknown");
+    LR.TPR.TargetPointer = (void *)Ptr;
+  } else if ((LR.Flags.ExtendsBefore || LR.Flags.ExtendsAfter) && !IsImplicit) {
+    // Explicit extension of mapped data - not allowed.
+    MESSAGE("explicit extension not allowed: host address specified is " DPxMOD
+            " (%" PRId64
+            " bytes), but device allocation maps to host at " DPxMOD
+            " (%" PRId64 " bytes)",
+            DPxPTR(HstPtrBegin), Size, DPxPTR(LR.TPR.getEntry()->HstPtrBegin),
+            LR.TPR.getEntry()->HstPtrEnd - LR.TPR.getEntry()->HstPtrBegin);
+    if (HasPresentModifier)
+      MESSAGE("device mapping required by 'present' map type modifier does not "
+              "exist for host address " DPxMOD " (%" PRId64 " bytes)",
+              DPxPTR(HstPtrBegin), Size);
+  } else if (PM->getRequirements() & OMP_REQ_UNIFIED_SHARED_MEMORY &&
+             !HasCloseModifier) {
+    // If unified shared memory is active, implicitly mapped variables that are
+    // not privatized use host address. Any explicitly mapped variables also use
+    // host address where correctness is not impeded. In all other cases maps
+    // are respected.
+    // In addition to the mapping rules above, the close map modifier forces the
+    // mapping of the variable to the device.
+    if (Size) {
+      DP("Return HstPtrBegin " DPxMOD " Size=%" PRId64 " for unified shared "
+         "memory\n",
+         DPxPTR((uintptr_t)HstPtrBegin), Size);
+      LR.TPR.Flags.IsPresent = false;
+      LR.TPR.Flags.IsHostPointer = true;
+      LR.TPR.TargetPointer = HstPtrBegin;
+    }
+  } else if (HasPresentModifier) {
+    DP("Mapping required by 'present' map type modifier does not exist for "
+       "HstPtrBegin=" DPxMOD ", Size=%" PRId64 "\n",
+       DPxPTR(HstPtrBegin), Size);
+    MESSAGE("device mapping required by 'present' map type modifier does not "
+            "exist for host address " DPxMOD " (%" PRId64 " bytes)",
+            DPxPTR(HstPtrBegin), Size);
+  } else if (Size) {
+    // If it is not contained and Size > 0, we should create a new entry for it.
+    LR.TPR.Flags.IsNewEntry = true;
+    uintptr_t TgtAllocBegin =
+        (uintptr_t)Device.allocData(TgtPadding + Size, HstPtrBegin);
+    uintptr_t TgtPtrBegin = TgtAllocBegin + TgtPadding;
+    // Release the mapping table lock only after the entry is locked by
+    // attaching it to TPR.
+    LR.TPR.setEntry(HDTTMap
+                        ->emplace(new HostDataToTargetTy(
+                            (uintptr_t)HstPtrBase, (uintptr_t)HstPtrBegin,
+                            (uintptr_t)HstPtrBegin + Size, TgtAllocBegin,
+                            TgtPtrBegin, HasHoldModifier, HstPtrName))
+                        .first->HDTT);
+    INFO(OMP_INFOTYPE_MAPPING_CHANGED, Device.DeviceID,
+         "Creating new map entry with HstPtrBase=" DPxMOD
+         ", HstPtrBegin=" DPxMOD ", TgtAllocBegin=" DPxMOD
+         ", TgtPtrBegin=" DPxMOD
+         ", Size=%ld, DynRefCount=%s, HoldRefCount=%s, Name=%s\n",
+         DPxPTR(HstPtrBase), DPxPTR(HstPtrBegin), DPxPTR(TgtAllocBegin),
+         DPxPTR(TgtPtrBegin), Size,
+         LR.TPR.getEntry()->dynRefCountToStr().c_str(),
+         LR.TPR.getEntry()->holdRefCountToStr().c_str(),
+         (HstPtrName) ? getNameFromMapping(HstPtrName).c_str() : "unknown");
+    LR.TPR.TargetPointer = (void *)TgtPtrBegin;
+
+    // Notify the plugin about the new mapping.
+    if (Device.notifyDataMapped(HstPtrBegin, Size))
+      return {{false /* IsNewEntry */, false /* IsHostPointer */},
+              nullptr /* Entry */,
+              nullptr /* TargetPointer */};
+  } else {
+    // This entry is not present and we did not create a new entry for it.
+    LR.TPR.Flags.IsPresent = false;
+  }
+
+  // All mapping table modifications have been made. If the user requested it we
+  // give up the lock.
+  if (ReleaseHDTTMap)
+    HDTTMap.destroy();
+
+  // If the target pointer is valid, and we need to transfer data, issue the
+  // data transfer.
+  if (LR.TPR.TargetPointer && !LR.TPR.Flags.IsHostPointer && HasFlagTo &&
+      (LR.TPR.Flags.IsNewEntry || HasFlagAlways) && Size != 0) {
+    DP("Moving %" PRId64 " bytes (hst:" DPxMOD ") -> (tgt:" DPxMOD ")\n", Size,
+       DPxPTR(HstPtrBegin), DPxPTR(LR.TPR.TargetPointer));
+
+    int Ret = Device.submitData(LR.TPR.TargetPointer, HstPtrBegin, Size,
+                                AsyncInfo, LR.TPR.getEntry());
+    if (Ret != OFFLOAD_SUCCESS) {
+      REPORT("Copying data to device failed.\n");
+      // We will also return nullptr if the data movement fails because that
+      // pointer points to a corrupted memory region so it doesn't make any
+      // sense to continue to use it.
+      LR.TPR.TargetPointer = nullptr;
+    } else if (LR.TPR.getEntry()->addEventIfNecessary(Device, AsyncInfo) !=
+               OFFLOAD_SUCCESS)
+      return {{false /* IsNewEntry */, false /* IsHostPointer */},
+              nullptr /* Entry */,
+              nullptr /* TargetPointer */};
+  } else {
+    // If not a host pointer and no present modifier, we need to wait for the
+    // event if it exists.
+    // Note: Entry might be nullptr because of zero length array section.
+    if (LR.TPR.getEntry() && !LR.TPR.Flags.IsHostPointer &&
+        !HasPresentModifier) {
+      void *Event = LR.TPR.getEntry()->getEvent();
+      if (Event) {
+        int Ret = Device.waitEvent(Event, AsyncInfo);
+        if (Ret != OFFLOAD_SUCCESS) {
+          // If it fails to wait for the event, we need to return nullptr in
+          // case of any data race.
+          REPORT("Failed to wait for event " DPxMOD ".\n", DPxPTR(Event));
+          return {{false /* IsNewEntry */, false /* IsHostPointer */},
+                  nullptr /* Entry */,
+                  nullptr /* TargetPointer */};
+        }
+      }
+    }
+  }
+
+  return std::move(LR.TPR);
+}
+
+TargetPointerResultTy MappingInfoTy::getTgtPtrBegin(
+    void *HstPtrBegin, int64_t Size, bool UpdateRefCount, bool UseHoldRefCount,
+    bool MustContain, bool ForceDelete, bool FromDataEnd) {
+  HDTTMapAccessorTy HDTTMap = HostDataToTargetMap.getExclusiveAccessor();
+
+  LookupResult LR = lookupMapping(HDTTMap, HstPtrBegin, Size);
+
+  LR.TPR.Flags.IsPresent = true;
+
+  if (LR.Flags.IsContained ||
+      (!MustContain && (LR.Flags.ExtendsBefore || LR.Flags.ExtendsAfter))) {
+    LR.TPR.Flags.IsLast =
+        LR.TPR.getEntry()->decShouldRemove(UseHoldRefCount, ForceDelete);
+
+    if (ForceDelete) {
+      LR.TPR.getEntry()->resetRefCount(UseHoldRefCount);
+      assert(LR.TPR.Flags.IsLast ==
+                 LR.TPR.getEntry()->decShouldRemove(UseHoldRefCount) &&
+             "expected correct IsLast prediction for reset");
+    }
+
+    // Increment the number of threads that is using the entry on a
+    // targetDataEnd, tracking the number of possible "deleters". A thread may
+    // come to own the entry deletion even if it was not the last one querying
+    // for it. Thus, we must track every query on targetDataEnds to ensure only
+    // the last thread that holds a reference to an entry actually deletes it.
+    if (FromDataEnd)
+      LR.TPR.getEntry()->incDataEndThreadCount();
+
+    const char *RefCountAction;
+    if (!UpdateRefCount) {
+      RefCountAction = " (update suppressed)";
+    } else if (LR.TPR.Flags.IsLast) {
+      LR.TPR.getEntry()->decRefCount(UseHoldRefCount);
+      assert(LR.TPR.getEntry()->getTotalRefCount() == 0 &&
+             "Expected zero reference count when deletion is scheduled");
+      if (ForceDelete)
+        RefCountAction = " (reset, delayed deletion)";
+      else
+        RefCountAction = " (decremented, delayed deletion)";
+    } else {
+      LR.TPR.getEntry()->decRefCount(UseHoldRefCount);
+      RefCountAction = " (decremented)";
+    }
+    const char *DynRefCountAction = UseHoldRefCount ? "" : RefCountAction;
+    const char *HoldRefCountAction = UseHoldRefCount ? RefCountAction : "";
+    uintptr_t TP = LR.TPR.getEntry()->TgtPtrBegin +
+                   ((uintptr_t)HstPtrBegin - LR.TPR.getEntry()->HstPtrBegin);
+    INFO(OMP_INFOTYPE_MAPPING_EXISTS, Device.DeviceID,
+         "Mapping exists with HstPtrBegin=" DPxMOD ", TgtPtrBegin=" DPxMOD ", "
+         "Size=%" PRId64 ", DynRefCount=%s%s, HoldRefCount=%s%s\n",
+         DPxPTR(HstPtrBegin), DPxPTR(TP), Size,
+         LR.TPR.getEntry()->dynRefCountToStr().c_str(), DynRefCountAction,
+         LR.TPR.getEntry()->holdRefCountToStr().c_str(), HoldRefCountAction);
+    LR.TPR.TargetPointer = (void *)TP;
+  } else if (PM->getRequirements() & OMP_REQ_UNIFIED_SHARED_MEMORY) {
+    // If the value isn't found in the mapping and unified shared memory
+    // is on then it means we have stumbled upon a value which we need to
+    // use directly from the host.
+    DP("Get HstPtrBegin " DPxMOD " Size=%" PRId64 " for unified shared "
+       "memory\n",
+       DPxPTR((uintptr_t)HstPtrBegin), Size);
+    LR.TPR.Flags.IsPresent = false;
+    LR.TPR.Flags.IsHostPointer = true;
+    LR.TPR.TargetPointer = HstPtrBegin;
+  } else {
+    // OpenMP Specification v5.2: if a matching list item is not found, the
+    // pointer retains its original value as per firstprivate semantics.
+    LR.TPR.Flags.IsPresent = false;
+    LR.TPR.Flags.IsHostPointer = false;
+    LR.TPR.TargetPointer = HstPtrBegin;
+  }
+
+  return std::move(LR.TPR);
+}
+
+// Return the target pointer begin (where the data will be moved).
+void *MappingInfoTy::getTgtPtrBegin(HDTTMapAccessorTy &HDTTMap,
+                                    void *HstPtrBegin, int64_t Size) {
+  uintptr_t HP = (uintptr_t)HstPtrBegin;
+  LookupResult LR = lookupMapping(HDTTMap, HstPtrBegin, Size);
+  if (LR.Flags.IsContained || LR.Flags.ExtendsBefore || LR.Flags.ExtendsAfter) {
+    uintptr_t TP =
+        LR.TPR.getEntry()->TgtPtrBegin + (HP - LR.TPR.getEntry()->HstPtrBegin);
+    return (void *)TP;
+  }
+
+  return NULL;
+}
+
+int MappingInfoTy::eraseMapEntry(HDTTMapAccessorTy &HDTTMap,
+                                 HostDataToTargetTy *Entry, int64_t Size) {
+  assert(Entry && "Trying to delete a null entry from the HDTT map.");
+  assert(Entry->getTotalRefCount() == 0 &&
+         Entry->getDataEndThreadCount() == 0 &&
+         "Trying to delete entry that is in use or owned by another thread.");
+
+  INFO(OMP_INFOTYPE_MAPPING_CHANGED, Device.DeviceID,
+       "Removing map entry with HstPtrBegin=" DPxMOD ", TgtPtrBegin=" DPxMOD
+       ", Size=%" PRId64 ", Name=%s\n",
+       DPxPTR(Entry->HstPtrBegin), DPxPTR(Entry->TgtPtrBegin), Size,
+       (Entry->HstPtrName) ? getNameFromMapping(Entry->HstPtrName).c_str()
+                           : "unknown");
+
+  if (HDTTMap->erase(Entry) == 0) {
+    REPORT("Trying to remove a non-existent map entry\n");
+    return OFFLOAD_FAIL;
+  }
+
+  return OFFLOAD_SUCCESS;
+}
+
+int MappingInfoTy::deallocTgtPtrAndEntry(HostDataToTargetTy *Entry,
+                                         int64_t Size) {
+  assert(Entry && "Trying to deallocate a null entry.");
+
+  DP("Deleting tgt data " DPxMOD " of size %" PRId64 " by freeing allocation "
+     "starting at " DPxMOD "\n",
+     DPxPTR(Entry->TgtPtrBegin), Size, DPxPTR(Entry->TgtAllocBegin));
+
+  void *Event = Entry->getEvent();
+  if (Event && Device.destroyEvent(Event) != OFFLOAD_SUCCESS) {
+    REPORT("Failed to destroy event " DPxMOD "\n", DPxPTR(Event));
+    return OFFLOAD_FAIL;
+  }
+
+  int Ret = Device.deleteData((void *)Entry->TgtAllocBegin);
+
+  // Notify the plugin about the unmapped memory.
+  Ret |= Device.notifyDataUnmapped((void *)Entry->HstPtrBegin);
+
+  delete Entry;
+
+  return Ret;
+}
+
+static void printCopyInfoImpl(int DeviceId, bool H2D, void *SrcPtrBegin,
+                              void *DstPtrBegin, int64_t Size,
+                              HostDataToTargetTy *HT) {
+
+  INFO(OMP_INFOTYPE_DATA_TRANSFER, DeviceId,
+       "Copying data from %s to %s, %sPtr=" DPxMOD ", %sPtr=" DPxMOD
+       ", Size=%" PRId64 ", Name=%s\n",
+       H2D ? "host" : "device", H2D ? "device" : "host", H2D ? "Hst" : "Tgt",
+       DPxPTR(SrcPtrBegin), H2D ? "Tgt" : "Hst", DPxPTR(DstPtrBegin), Size,
+       (HT && HT->HstPtrName) ? getNameFromMapping(HT->HstPtrName).c_str()
+                              : "unknown");
+}
+
+void MappingInfoTy::printCopyInfo(
+    void *TgtPtrBegin, void *HstPtrBegin, int64_t Size, bool H2D,
+    HostDataToTargetTy *Entry, MappingInfoTy::HDTTMapAccessorTy *HDTTMapPtr) {
+  auto HDTTMap =
+      HostDataToTargetMap.getExclusiveAccessor(!!Entry || !!HDTTMapPtr);
+  LookupResult LR;
+  if (!Entry) {
+    LR = lookupMapping(HDTTMapPtr ? *HDTTMapPtr : HDTTMap, HstPtrBegin, Size);
+    Entry = LR.TPR.getEntry();
+  }
+  printCopyInfoImpl(Device.DeviceID, H2D, HstPtrBegin, TgtPtrBegin, Size,
+                    Entry);
+}
diff --git a/openmp/libomptarget/src/device.cpp b/openmp/libomptarget/src/device.cpp
index 01fc32328876123697797e4ba5a08b280a30e599..dbad13b92bcc142f7ddb1cf7c19a2b9099fc69da 100644
--- a/openmp/libomptarget/src/device.cpp
+++ b/openmp/libomptarget/src/device.cpp
@@ -12,6 +12,7 @@
 
 #include "device.h"
 #include "OffloadEntry.h"
+#include "OpenMP/Mapping.h"
 #include "OpenMP/OMPT/Callback.h"
 #include "OpenMP/OMPT/Interface.h"
 #include "PluginManager.h"
@@ -65,7 +66,7 @@ int HostDataToTargetTy::addEventIfNecessary(DeviceTy &Device,
 
 DeviceTy::DeviceTy(PluginAdaptorTy *RTL, int32_t DeviceID, int32_t RTLDeviceID)
     : DeviceID(DeviceID), RTL(RTL), RTLDeviceID(RTLDeviceID),
-      PendingCtorsDtors(), PendingGlobalsMtx() {}
+      PendingCtorsDtors(), PendingGlobalsMtx(), MappingInfo(*this) {}
 
 DeviceTy::~DeviceTy() {
   if (DeviceID == -1 || !(getInfoLevel() & OMP_INFOTYPE_DUMP_TABLE))
@@ -75,460 +76,6 @@ DeviceTy::~DeviceTy() {
   dumpTargetPointerMappings(&Loc, *this);
 }
 
-int DeviceTy::associatePtr(void *HstPtrBegin, void *TgtPtrBegin, int64_t Size) {
-  HDTTMapAccessorTy HDTTMap = HostDataToTargetMap.getExclusiveAccessor();
-
-  // Check if entry exists
-  auto It = HDTTMap->find(HstPtrBegin);
-  if (It != HDTTMap->end()) {
-    HostDataToTargetTy &HDTT = *It->HDTT;
-    std::lock_guard<HostDataToTargetTy> LG(HDTT);
-    // Mapping already exists
-    bool IsValid = HDTT.HstPtrEnd == (uintptr_t)HstPtrBegin + Size &&
-                   HDTT.TgtPtrBegin == (uintptr_t)TgtPtrBegin;
-    if (IsValid) {
-      DP("Attempt to re-associate the same device ptr+offset with the same "
-         "host ptr, nothing to do\n");
-      return OFFLOAD_SUCCESS;
-    }
-    REPORT("Not allowed to re-associate a different device ptr+offset with "
-           "the same host ptr\n");
-    return OFFLOAD_FAIL;
-  }
-
-  // Mapping does not exist, allocate it with refCount=INF
-  const HostDataToTargetTy &NewEntry =
-      *HDTTMap
-           ->emplace(new HostDataToTargetTy(
-               /*HstPtrBase=*/(uintptr_t)HstPtrBegin,
-               /*HstPtrBegin=*/(uintptr_t)HstPtrBegin,
-               /*HstPtrEnd=*/(uintptr_t)HstPtrBegin + Size,
-               /*TgtAllocBegin=*/(uintptr_t)TgtPtrBegin,
-               /*TgtPtrBegin=*/(uintptr_t)TgtPtrBegin,
-               /*UseHoldRefCount=*/false, /*Name=*/nullptr,
-               /*IsRefCountINF=*/true))
-           .first->HDTT;
-  DP("Creating new map entry: HstBase=" DPxMOD ", HstBegin=" DPxMOD
-     ", HstEnd=" DPxMOD ", TgtBegin=" DPxMOD ", DynRefCount=%s, "
-     "HoldRefCount=%s\n",
-     DPxPTR(NewEntry.HstPtrBase), DPxPTR(NewEntry.HstPtrBegin),
-     DPxPTR(NewEntry.HstPtrEnd), DPxPTR(NewEntry.TgtPtrBegin),
-     NewEntry.dynRefCountToStr().c_str(), NewEntry.holdRefCountToStr().c_str());
-  (void)NewEntry;
-
-  // Notify the plugin about the new mapping.
-  return notifyDataMapped(HstPtrBegin, Size);
-}
-
-int DeviceTy::disassociatePtr(void *HstPtrBegin) {
-  HDTTMapAccessorTy HDTTMap = HostDataToTargetMap.getExclusiveAccessor();
-
-  auto It = HDTTMap->find(HstPtrBegin);
-  if (It == HDTTMap->end()) {
-    REPORT("Association not found\n");
-    return OFFLOAD_FAIL;
-  }
-  // Mapping exists
-  HostDataToTargetTy &HDTT = *It->HDTT;
-  std::lock_guard<HostDataToTargetTy> LG(HDTT);
-
-  if (HDTT.getHoldRefCount()) {
-    // This is based on OpenACC 3.1, sec 3.2.33 "acc_unmap_data", L3656-3657:
-    // "It is an error to call acc_unmap_data if the structured reference
-    // count for the pointer is not zero."
-    REPORT("Trying to disassociate a pointer with a non-zero hold reference "
-           "count\n");
-    return OFFLOAD_FAIL;
-  }
-
-  if (HDTT.isDynRefCountInf()) {
-    DP("Association found, removing it\n");
-    void *Event = HDTT.getEvent();
-    delete &HDTT;
-    if (Event)
-      destroyEvent(Event);
-    HDTTMap->erase(It);
-    return notifyDataUnmapped(HstPtrBegin);
-  }
-
-  REPORT("Trying to disassociate a pointer which was not mapped via "
-         "omp_target_associate_ptr\n");
-  return OFFLOAD_FAIL;
-}
-
-LookupResult DeviceTy::lookupMapping(HDTTMapAccessorTy &HDTTMap,
-                                     void *HstPtrBegin, int64_t Size,
-                                     HostDataToTargetTy *OwnedTPR) {
-
-  uintptr_t HP = (uintptr_t)HstPtrBegin;
-  LookupResult LR;
-
-  DP("Looking up mapping(HstPtrBegin=" DPxMOD ", Size=%" PRId64 ")...\n",
-     DPxPTR(HP), Size);
-
-  if (HDTTMap->empty())
-    return LR;
-
-  auto Upper = HDTTMap->upper_bound(HP);
-
-  if (Size == 0) {
-    // specification v5.1 Pointer Initialization for Device Data Environments
-    // upper_bound satisfies
-    //   std::prev(upper)->HDTT.HstPtrBegin <= hp < upper->HDTT.HstPtrBegin
-    if (Upper != HDTTMap->begin()) {
-      LR.TPR.setEntry(std::prev(Upper)->HDTT, OwnedTPR);
-      // the left side of extended address range is satisified.
-      // hp >= LR.TPR.getEntry()->HstPtrBegin || hp >=
-      // LR.TPR.getEntry()->HstPtrBase
-      LR.Flags.IsContained = HP < LR.TPR.getEntry()->HstPtrEnd ||
-                             HP < LR.TPR.getEntry()->HstPtrBase;
-    }
-
-    if (!LR.Flags.IsContained && Upper != HDTTMap->end()) {
-      LR.TPR.setEntry(Upper->HDTT, OwnedTPR);
-      // the right side of extended address range is satisified.
-      // hp < LR.TPR.getEntry()->HstPtrEnd || hp < LR.TPR.getEntry()->HstPtrBase
-      LR.Flags.IsContained = HP >= LR.TPR.getEntry()->HstPtrBase;
-    }
-  } else {
-    // check the left bin
-    if (Upper != HDTTMap->begin()) {
-      LR.TPR.setEntry(std::prev(Upper)->HDTT, OwnedTPR);
-      // Is it contained?
-      LR.Flags.IsContained = HP >= LR.TPR.getEntry()->HstPtrBegin &&
-                             HP < LR.TPR.getEntry()->HstPtrEnd &&
-                             (HP + Size) <= LR.TPR.getEntry()->HstPtrEnd;
-      // Does it extend beyond the mapped region?
-      LR.Flags.ExtendsAfter = HP < LR.TPR.getEntry()->HstPtrEnd &&
-                              (HP + Size) > LR.TPR.getEntry()->HstPtrEnd;
-    }
-
-    // check the right bin
-    if (!(LR.Flags.IsContained || LR.Flags.ExtendsAfter) &&
-        Upper != HDTTMap->end()) {
-      LR.TPR.setEntry(Upper->HDTT, OwnedTPR);
-      // Does it extend into an already mapped region?
-      LR.Flags.ExtendsBefore = HP < LR.TPR.getEntry()->HstPtrBegin &&
-                               (HP + Size) > LR.TPR.getEntry()->HstPtrBegin;
-      // Does it extend beyond the mapped region?
-      LR.Flags.ExtendsAfter = HP < LR.TPR.getEntry()->HstPtrEnd &&
-                              (HP + Size) > LR.TPR.getEntry()->HstPtrEnd;
-    }
-
-    if (LR.Flags.ExtendsBefore) {
-      DP("WARNING: Pointer is not mapped but section extends into already "
-         "mapped data\n");
-    }
-    if (LR.Flags.ExtendsAfter) {
-      DP("WARNING: Pointer is already mapped but section extends beyond mapped "
-         "region\n");
-    }
-  }
-
-  return LR;
-}
-
-TargetPointerResultTy DeviceTy::getTargetPointer(
-    HDTTMapAccessorTy &HDTTMap, void *HstPtrBegin, void *HstPtrBase,
-    int64_t TgtPadding, int64_t Size, map_var_info_t HstPtrName, bool HasFlagTo,
-    bool HasFlagAlways, bool IsImplicit, bool UpdateRefCount,
-    bool HasCloseModifier, bool HasPresentModifier, bool HasHoldModifier,
-    AsyncInfoTy &AsyncInfo, HostDataToTargetTy *OwnedTPR, bool ReleaseHDTTMap) {
-
-  LookupResult LR = lookupMapping(HDTTMap, HstPtrBegin, Size, OwnedTPR);
-  LR.TPR.Flags.IsPresent = true;
-
-  // Release the mapping table lock only after the entry is locked by
-  // attaching it to TPR. Once TPR is destroyed it will release the lock
-  // on entry. If it is returned the lock will move to the returned object.
-  // If LR.Entry is already owned/locked we avoid trying to lock it again.
-
-  // Check if the pointer is contained.
-  // If a variable is mapped to the device manually by the user - which would
-  // lead to the IsContained flag to be true - then we must ensure that the
-  // device address is returned even under unified memory conditions.
-  if (LR.Flags.IsContained ||
-      ((LR.Flags.ExtendsBefore || LR.Flags.ExtendsAfter) && IsImplicit)) {
-    const char *RefCountAction;
-    if (UpdateRefCount) {
-      // After this, reference count >= 1. If the reference count was 0 but the
-      // entry was still there we can reuse the data on the device and avoid a
-      // new submission.
-      LR.TPR.getEntry()->incRefCount(HasHoldModifier);
-      RefCountAction = " (incremented)";
-    } else {
-      // It might have been allocated with the parent, but it's still new.
-      LR.TPR.Flags.IsNewEntry = LR.TPR.getEntry()->getTotalRefCount() == 1;
-      RefCountAction = " (update suppressed)";
-    }
-    const char *DynRefCountAction = HasHoldModifier ? "" : RefCountAction;
-    const char *HoldRefCountAction = HasHoldModifier ? RefCountAction : "";
-    uintptr_t Ptr = LR.TPR.getEntry()->TgtPtrBegin +
-                    ((uintptr_t)HstPtrBegin - LR.TPR.getEntry()->HstPtrBegin);
-    INFO(OMP_INFOTYPE_MAPPING_EXISTS, DeviceID,
-         "Mapping exists%s with HstPtrBegin=" DPxMOD ", TgtPtrBegin=" DPxMOD
-         ", Size=%" PRId64 ", DynRefCount=%s%s, HoldRefCount=%s%s, Name=%s\n",
-         (IsImplicit ? " (implicit)" : ""), DPxPTR(HstPtrBegin), DPxPTR(Ptr),
-         Size, LR.TPR.getEntry()->dynRefCountToStr().c_str(), DynRefCountAction,
-         LR.TPR.getEntry()->holdRefCountToStr().c_str(), HoldRefCountAction,
-         (HstPtrName) ? getNameFromMapping(HstPtrName).c_str() : "unknown");
-    LR.TPR.TargetPointer = (void *)Ptr;
-  } else if ((LR.Flags.ExtendsBefore || LR.Flags.ExtendsAfter) && !IsImplicit) {
-    // Explicit extension of mapped data - not allowed.
-    MESSAGE("explicit extension not allowed: host address specified is " DPxMOD
-            " (%" PRId64
-            " bytes), but device allocation maps to host at " DPxMOD
-            " (%" PRId64 " bytes)",
-            DPxPTR(HstPtrBegin), Size, DPxPTR(LR.TPR.getEntry()->HstPtrBegin),
-            LR.TPR.getEntry()->HstPtrEnd - LR.TPR.getEntry()->HstPtrBegin);
-    if (HasPresentModifier)
-      MESSAGE("device mapping required by 'present' map type modifier does not "
-              "exist for host address " DPxMOD " (%" PRId64 " bytes)",
-              DPxPTR(HstPtrBegin), Size);
-  } else if (PM->getRequirements() & OMP_REQ_UNIFIED_SHARED_MEMORY &&
-             !HasCloseModifier) {
-    // If unified shared memory is active, implicitly mapped variables that are
-    // not privatized use host address. Any explicitly mapped variables also use
-    // host address where correctness is not impeded. In all other cases maps
-    // are respected.
-    // In addition to the mapping rules above, the close map modifier forces the
-    // mapping of the variable to the device.
-    if (Size) {
-      DP("Return HstPtrBegin " DPxMOD " Size=%" PRId64 " for unified shared "
-         "memory\n",
-         DPxPTR((uintptr_t)HstPtrBegin), Size);
-      LR.TPR.Flags.IsPresent = false;
-      LR.TPR.Flags.IsHostPointer = true;
-      LR.TPR.TargetPointer = HstPtrBegin;
-    }
-  } else if (HasPresentModifier) {
-    DP("Mapping required by 'present' map type modifier does not exist for "
-       "HstPtrBegin=" DPxMOD ", Size=%" PRId64 "\n",
-       DPxPTR(HstPtrBegin), Size);
-    MESSAGE("device mapping required by 'present' map type modifier does not "
-            "exist for host address " DPxMOD " (%" PRId64 " bytes)",
-            DPxPTR(HstPtrBegin), Size);
-  } else if (Size) {
-    // If it is not contained and Size > 0, we should create a new entry for it.
-    LR.TPR.Flags.IsNewEntry = true;
-    uintptr_t TgtAllocBegin =
-        (uintptr_t)allocData(TgtPadding + Size, HstPtrBegin);
-    uintptr_t TgtPtrBegin = TgtAllocBegin + TgtPadding;
-    // Release the mapping table lock only after the entry is locked by
-    // attaching it to TPR.
-    LR.TPR.setEntry(HDTTMap
-                        ->emplace(new HostDataToTargetTy(
-                            (uintptr_t)HstPtrBase, (uintptr_t)HstPtrBegin,
-                            (uintptr_t)HstPtrBegin + Size, TgtAllocBegin,
-                            TgtPtrBegin, HasHoldModifier, HstPtrName))
-                        .first->HDTT);
-    INFO(OMP_INFOTYPE_MAPPING_CHANGED, DeviceID,
-         "Creating new map entry with HstPtrBase=" DPxMOD
-         ", HstPtrBegin=" DPxMOD ", TgtAllocBegin=" DPxMOD
-         ", TgtPtrBegin=" DPxMOD
-         ", Size=%ld, DynRefCount=%s, HoldRefCount=%s, Name=%s\n",
-         DPxPTR(HstPtrBase), DPxPTR(HstPtrBegin), DPxPTR(TgtAllocBegin),
-         DPxPTR(TgtPtrBegin), Size,
-         LR.TPR.getEntry()->dynRefCountToStr().c_str(),
-         LR.TPR.getEntry()->holdRefCountToStr().c_str(),
-         (HstPtrName) ? getNameFromMapping(HstPtrName).c_str() : "unknown");
-    LR.TPR.TargetPointer = (void *)TgtPtrBegin;
-
-    // Notify the plugin about the new mapping.
-    if (notifyDataMapped(HstPtrBegin, Size))
-      return {{false /* IsNewEntry */, false /* IsHostPointer */},
-              nullptr /* Entry */,
-              nullptr /* TargetPointer */};
-  } else {
-    // This entry is not present and we did not create a new entry for it.
-    LR.TPR.Flags.IsPresent = false;
-  }
-
-  // All mapping table modifications have been made. If the user requested it we
-  // give up the lock.
-  if (ReleaseHDTTMap)
-    HDTTMap.destroy();
-
-  // If the target pointer is valid, and we need to transfer data, issue the
-  // data transfer.
-  if (LR.TPR.TargetPointer && !LR.TPR.Flags.IsHostPointer && HasFlagTo &&
-      (LR.TPR.Flags.IsNewEntry || HasFlagAlways) && Size != 0) {
-    DP("Moving %" PRId64 " bytes (hst:" DPxMOD ") -> (tgt:" DPxMOD ")\n", Size,
-       DPxPTR(HstPtrBegin), DPxPTR(LR.TPR.TargetPointer));
-
-    int Ret = submitData(LR.TPR.TargetPointer, HstPtrBegin, Size, AsyncInfo,
-                         LR.TPR.getEntry());
-    if (Ret != OFFLOAD_SUCCESS) {
-      REPORT("Copying data to device failed.\n");
-      // We will also return nullptr if the data movement fails because that
-      // pointer points to a corrupted memory region so it doesn't make any
-      // sense to continue to use it.
-      LR.TPR.TargetPointer = nullptr;
-    } else if (LR.TPR.getEntry()->addEventIfNecessary(*this, AsyncInfo) !=
-               OFFLOAD_SUCCESS)
-      return {{false /* IsNewEntry */, false /* IsHostPointer */},
-              nullptr /* Entry */,
-              nullptr /* TargetPointer */};
-  } else {
-    // If not a host pointer and no present modifier, we need to wait for the
-    // event if it exists.
-    // Note: Entry might be nullptr because of zero length array section.
-    if (LR.TPR.getEntry() && !LR.TPR.Flags.IsHostPointer &&
-        !HasPresentModifier) {
-      void *Event = LR.TPR.getEntry()->getEvent();
-      if (Event) {
-        int Ret = waitEvent(Event, AsyncInfo);
-        if (Ret != OFFLOAD_SUCCESS) {
-          // If it fails to wait for the event, we need to return nullptr in
-          // case of any data race.
-          REPORT("Failed to wait for event " DPxMOD ".\n", DPxPTR(Event));
-          return {{false /* IsNewEntry */, false /* IsHostPointer */},
-                  nullptr /* Entry */,
-                  nullptr /* TargetPointer */};
-        }
-      }
-    }
-  }
-
-  return std::move(LR.TPR);
-}
-
-TargetPointerResultTy
-DeviceTy::getTgtPtrBegin(void *HstPtrBegin, int64_t Size, bool UpdateRefCount,
-                         bool UseHoldRefCount, bool MustContain,
-                         bool ForceDelete, bool FromDataEnd) {
-  HDTTMapAccessorTy HDTTMap = HostDataToTargetMap.getExclusiveAccessor();
-
-  LookupResult LR = lookupMapping(HDTTMap, HstPtrBegin, Size);
-
-  LR.TPR.Flags.IsPresent = true;
-
-  if (LR.Flags.IsContained ||
-      (!MustContain && (LR.Flags.ExtendsBefore || LR.Flags.ExtendsAfter))) {
-    LR.TPR.Flags.IsLast =
-        LR.TPR.getEntry()->decShouldRemove(UseHoldRefCount, ForceDelete);
-
-    if (ForceDelete) {
-      LR.TPR.getEntry()->resetRefCount(UseHoldRefCount);
-      assert(LR.TPR.Flags.IsLast ==
-                 LR.TPR.getEntry()->decShouldRemove(UseHoldRefCount) &&
-             "expected correct IsLast prediction for reset");
-    }
-
-    // Increment the number of threads that is using the entry on a
-    // targetDataEnd, tracking the number of possible "deleters". A thread may
-    // come to own the entry deletion even if it was not the last one querying
-    // for it. Thus, we must track every query on targetDataEnds to ensure only
-    // the last thread that holds a reference to an entry actually deletes it.
-    if (FromDataEnd)
-      LR.TPR.getEntry()->incDataEndThreadCount();
-
-    const char *RefCountAction;
-    if (!UpdateRefCount) {
-      RefCountAction = " (update suppressed)";
-    } else if (LR.TPR.Flags.IsLast) {
-      LR.TPR.getEntry()->decRefCount(UseHoldRefCount);
-      assert(LR.TPR.getEntry()->getTotalRefCount() == 0 &&
-             "Expected zero reference count when deletion is scheduled");
-      if (ForceDelete)
-        RefCountAction = " (reset, delayed deletion)";
-      else
-        RefCountAction = " (decremented, delayed deletion)";
-    } else {
-      LR.TPR.getEntry()->decRefCount(UseHoldRefCount);
-      RefCountAction = " (decremented)";
-    }
-    const char *DynRefCountAction = UseHoldRefCount ? "" : RefCountAction;
-    const char *HoldRefCountAction = UseHoldRefCount ? RefCountAction : "";
-    uintptr_t TP = LR.TPR.getEntry()->TgtPtrBegin +
-                   ((uintptr_t)HstPtrBegin - LR.TPR.getEntry()->HstPtrBegin);
-    INFO(OMP_INFOTYPE_MAPPING_EXISTS, DeviceID,
-         "Mapping exists with HstPtrBegin=" DPxMOD ", TgtPtrBegin=" DPxMOD ", "
-         "Size=%" PRId64 ", DynRefCount=%s%s, HoldRefCount=%s%s\n",
-         DPxPTR(HstPtrBegin), DPxPTR(TP), Size,
-         LR.TPR.getEntry()->dynRefCountToStr().c_str(), DynRefCountAction,
-         LR.TPR.getEntry()->holdRefCountToStr().c_str(), HoldRefCountAction);
-    LR.TPR.TargetPointer = (void *)TP;
-  } else if (PM->getRequirements() & OMP_REQ_UNIFIED_SHARED_MEMORY) {
-    // If the value isn't found in the mapping and unified shared memory
-    // is on then it means we have stumbled upon a value which we need to
-    // use directly from the host.
-    DP("Get HstPtrBegin " DPxMOD " Size=%" PRId64 " for unified shared "
-       "memory\n",
-       DPxPTR((uintptr_t)HstPtrBegin), Size);
-    LR.TPR.Flags.IsPresent = false;
-    LR.TPR.Flags.IsHostPointer = true;
-    LR.TPR.TargetPointer = HstPtrBegin;
-  } else {
-    // OpenMP Specification v5.2: if a matching list item is not found, the
-    // pointer retains its original value as per firstprivate semantics.
-    LR.TPR.Flags.IsPresent = false;
-    LR.TPR.Flags.IsHostPointer = false;
-    LR.TPR.TargetPointer = HstPtrBegin;
-  }
-
-  return std::move(LR.TPR);
-}
-
-// Return the target pointer begin (where the data will be moved).
-void *DeviceTy::getTgtPtrBegin(HDTTMapAccessorTy &HDTTMap, void *HstPtrBegin,
-                               int64_t Size) {
-  uintptr_t HP = (uintptr_t)HstPtrBegin;
-  LookupResult LR = lookupMapping(HDTTMap, HstPtrBegin, Size);
-  if (LR.Flags.IsContained || LR.Flags.ExtendsBefore || LR.Flags.ExtendsAfter) {
-    uintptr_t TP =
-        LR.TPR.getEntry()->TgtPtrBegin + (HP - LR.TPR.getEntry()->HstPtrBegin);
-    return (void *)TP;
-  }
-
-  return NULL;
-}
-
-int DeviceTy::eraseMapEntry(HDTTMapAccessorTy &HDTTMap,
-                            HostDataToTargetTy *Entry, int64_t Size) {
-  assert(Entry && "Trying to delete a null entry from the HDTT map.");
-  assert(Entry->getTotalRefCount() == 0 &&
-         Entry->getDataEndThreadCount() == 0 &&
-         "Trying to delete entry that is in use or owned by another thread.");
-
-  INFO(OMP_INFOTYPE_MAPPING_CHANGED, DeviceID,
-       "Removing map entry with HstPtrBegin=" DPxMOD ", TgtPtrBegin=" DPxMOD
-       ", Size=%" PRId64 ", Name=%s\n",
-       DPxPTR(Entry->HstPtrBegin), DPxPTR(Entry->TgtPtrBegin), Size,
-       (Entry->HstPtrName) ? getNameFromMapping(Entry->HstPtrName).c_str()
-                           : "unknown");
-
-  if (HDTTMap->erase(Entry) == 0) {
-    REPORT("Trying to remove a non-existent map entry\n");
-    return OFFLOAD_FAIL;
-  }
-
-  return OFFLOAD_SUCCESS;
-}
-
-int DeviceTy::deallocTgtPtrAndEntry(HostDataToTargetTy *Entry, int64_t Size) {
-  assert(Entry && "Trying to deallocate a null entry.");
-
-  DP("Deleting tgt data " DPxMOD " of size %" PRId64 " by freeing allocation "
-     "starting at " DPxMOD "\n",
-     DPxPTR(Entry->TgtPtrBegin), Size, DPxPTR(Entry->TgtAllocBegin));
-
-  void *Event = Entry->getEvent();
-  if (Event && destroyEvent(Event) != OFFLOAD_SUCCESS) {
-    REPORT("Failed to destroy event " DPxMOD "\n", DPxPTR(Event));
-    return OFFLOAD_FAIL;
-  }
-
-  int Ret = deleteData((void *)Entry->TgtAllocBegin);
-
-  // Notify the plugin about the unmapped memory.
-  Ret |= notifyDataUnmapped((void *)Entry->HstPtrBegin);
-
-  delete Entry;
-
-  return Ret;
-}
-
 llvm::Error DeviceTy::init() {
   // Make call to init_requires if it exists for this plugin.
   int32_t Ret = 0;
@@ -586,34 +133,13 @@ int32_t DeviceTy::deleteData(void *TgtAllocBegin, int32_t Kind) {
   return RTL->data_delete(RTLDeviceID, TgtAllocBegin, Kind);
 }
 
-static void printCopyInfo(int DeviceId, bool H2D, void *SrcPtrBegin,
-                          void *DstPtrBegin, int64_t Size,
-                          HostDataToTargetTy *HT) {
-
-  INFO(OMP_INFOTYPE_DATA_TRANSFER, DeviceId,
-       "Copying data from %s to %s, %sPtr=" DPxMOD ", %sPtr=" DPxMOD
-       ", Size=%" PRId64 ", Name=%s\n",
-       H2D ? "host" : "device", H2D ? "device" : "host", H2D ? "Hst" : "Tgt",
-       DPxPTR(SrcPtrBegin), H2D ? "Tgt" : "Hst", DPxPTR(DstPtrBegin), Size,
-       (HT && HT->HstPtrName) ? getNameFromMapping(HT->HstPtrName).c_str()
-                              : "unknown");
-}
-
 // Submit data to device
 int32_t DeviceTy::submitData(void *TgtPtrBegin, void *HstPtrBegin, int64_t Size,
                              AsyncInfoTy &AsyncInfo, HostDataToTargetTy *Entry,
-                             DeviceTy::HDTTMapAccessorTy *HDTTMapPtr) {
-  if (getInfoLevel() & OMP_INFOTYPE_DATA_TRANSFER) {
-    HDTTMapAccessorTy HDTTMap =
-        HostDataToTargetMap.getExclusiveAccessor(!!Entry || !!HDTTMapPtr);
-    LookupResult LR;
-    if (!Entry) {
-      LR = lookupMapping(HDTTMapPtr ? *HDTTMapPtr : HDTTMap, HstPtrBegin, Size);
-      Entry = LR.TPR.getEntry();
-    }
-    printCopyInfo(DeviceID, /* H2D */ true, HstPtrBegin, TgtPtrBegin, Size,
-                  Entry);
-  }
+                             MappingInfoTy::HDTTMapAccessorTy *HDTTMapPtr) {
+  if (getInfoLevel() & OMP_INFOTYPE_DATA_TRANSFER)
+    MappingInfo.printCopyInfo(TgtPtrBegin, HstPtrBegin, Size, /*H2D=*/true,
+                              Entry, HDTTMapPtr);
 
   /// RAII to establish tool anchors before and after data submit
   OMPT_IF_BUILT(
@@ -632,18 +158,10 @@ int32_t DeviceTy::submitData(void *TgtPtrBegin, void *HstPtrBegin, int64_t Size,
 int32_t DeviceTy::retrieveData(void *HstPtrBegin, void *TgtPtrBegin,
                                int64_t Size, AsyncInfoTy &AsyncInfo,
                                HostDataToTargetTy *Entry,
-                               DeviceTy::HDTTMapAccessorTy *HDTTMapPtr) {
-  if (getInfoLevel() & OMP_INFOTYPE_DATA_TRANSFER) {
-    HDTTMapAccessorTy HDTTMap =
-        HostDataToTargetMap.getExclusiveAccessor(!!Entry || !!HDTTMapPtr);
-    LookupResult LR;
-    if (!Entry) {
-      LR = lookupMapping(HDTTMapPtr ? *HDTTMapPtr : HDTTMap, HstPtrBegin, Size);
-      Entry = LR.TPR.getEntry();
-    }
-    printCopyInfo(DeviceID, /* H2D */ false, TgtPtrBegin, HstPtrBegin, Size,
-                  Entry);
-  }
+                               MappingInfoTy::HDTTMapAccessorTy *HDTTMapPtr) {
+  if (getInfoLevel() & OMP_INFOTYPE_DATA_TRANSFER)
+    MappingInfo.printCopyInfo(TgtPtrBegin, HstPtrBegin, Size, /*H2D=*/false,
+                              Entry, HDTTMapPtr);
 
   /// RAII to establish tool anchors before and after data retrieval
   OMPT_IF_BUILT(
@@ -775,7 +293,8 @@ int32_t DeviceTy::destroyEvent(void *Event) {
 
 void DeviceTy::addOffloadEntry(OffloadEntryTy &Entry) {
   std::lock_guard<decltype(PendingGlobalsMtx)> Lock(PendingGlobalsMtx);
-  DeviceOffloadEntries[Entry.getName()] = &Entry;
+  DeviceOffloadEntries.getExclusiveAccessor()->insert(
+      {Entry.getName(), &Entry});
   if (Entry.isGlobal())
     return;
 
@@ -808,7 +327,7 @@ void DeviceTy::addOffloadEntry(OffloadEntryTy &Entry) {
 
 void DeviceTy::dumpOffloadEntries() {
   fprintf(stderr, "Device %i offload entries:\n", DeviceID);
-  for (auto &It : DeviceOffloadEntries) {
+  for (auto &It : *DeviceOffloadEntries.getExclusiveAccessor()) {
     const char *Kind = "kernel";
     if (It.second->isCTor())
       Kind = "constructor";
diff --git a/openmp/libomptarget/src/omptarget.cpp b/openmp/libomptarget/src/omptarget.cpp
index 2edbadaa6e02c6106ff58921d0e53481a90da434..0d16a41c7616c7095bf127c7ad2a1c4290031474 100644
--- a/openmp/libomptarget/src/omptarget.cpp
+++ b/openmp/libomptarget/src/omptarget.cpp
@@ -194,8 +194,8 @@ static int initLibrary(DeviceTy &Device) {
         break;
       }
 
-      DeviceTy::HDTTMapAccessorTy HDTTMap =
-          Device.HostDataToTargetMap.getExclusiveAccessor();
+      MappingInfoTy::HDTTMapAccessorTy HDTTMap =
+          Device.getMappingInfo().HostDataToTargetMap.getExclusiveAccessor();
 
       __tgt_target_table *HostTable = &TransTable->HostTable;
       for (__tgt_offload_entry *CurrDeviceEntry = TargetTable->EntriesBegin,
@@ -213,8 +213,8 @@ static int initLibrary(DeviceTy &Device) {
         // therefore we must allow for multiple weak symbols to be loaded from
         // the fat binary. Treat these mappings as any other "regular"
         // mapping. Add entry to map.
-        if (Device.getTgtPtrBegin(HDTTMap, CurrHostEntry->addr,
-                                  CurrHostEntry->size))
+        if (Device.getMappingInfo().getTgtPtrBegin(HDTTMap, CurrHostEntry->addr,
+                                                   CurrHostEntry->size))
           continue;
 
         void *CurrDeviceEntryAddr = CurrDeviceEntry->addr;
@@ -604,8 +604,8 @@ int targetDataBegin(ident_t *Loc, DeviceTy &Device, int32_t ArgNum,
     bool UpdateRef =
         !(ArgTypes[I] & OMP_TGT_MAPTYPE_MEMBER_OF) && !(FromMapper && I == 0);
 
-    DeviceTy::HDTTMapAccessorTy HDTTMap =
-        Device.HostDataToTargetMap.getExclusiveAccessor();
+    MappingInfoTy::HDTTMapAccessorTy HDTTMap =
+        Device.getMappingInfo().HostDataToTargetMap.getExclusiveAccessor();
     if (ArgTypes[I] & OMP_TGT_MAPTYPE_PTR_AND_OBJ) {
       DP("Has a pointer entry: \n");
       // Base is address of pointer.
@@ -621,7 +621,7 @@ int targetDataBegin(ident_t *Loc, DeviceTy &Device, int32_t ArgNum,
       // entry for a global that might not already be allocated by the time the
       // PTR_AND_OBJ entry is handled below, and so the allocation might fail
       // when HasPresentModifier.
-      PointerTpr = Device.getTargetPointer(
+      PointerTpr = Device.getMappingInfo().getTargetPointer(
           HDTTMap, HstPtrBase, HstPtrBase, /*TgtPadding=*/0, sizeof(void *),
           /*HstPtrName=*/nullptr,
           /*HasFlagTo=*/false, /*HasFlagAlways=*/false, IsImplicit, UpdateRef,
@@ -651,7 +651,7 @@ int targetDataBegin(ident_t *Loc, DeviceTy &Device, int32_t ArgNum,
     const bool HasFlagTo = ArgTypes[I] & OMP_TGT_MAPTYPE_TO;
     const bool HasFlagAlways = ArgTypes[I] & OMP_TGT_MAPTYPE_ALWAYS;
     // Note that HDTTMap will be released in getTargetPointer.
-    auto TPR = Device.getTargetPointer(
+    auto TPR = Device.getMappingInfo().getTargetPointer(
         HDTTMap, HstPtrBegin, HstPtrBase, TgtPadding, DataSize, HstPtrName,
         HasFlagTo, HasFlagAlways, IsImplicit, UpdateRef, HasCloseModifier,
         HasPresentModifier, HasHoldModifier, AsyncInfo, PointerTpr.getEntry());
@@ -768,8 +768,8 @@ postProcessingTargetDataEnd(DeviceTy *Device,
     // will avoid another thread reusing the entry now. Note that we do
     // not request (exclusive) access to the HDTT map if DelEntry is
     // not set.
-    DeviceTy::HDTTMapAccessorTy HDTTMap =
-        Device->HostDataToTargetMap.getExclusiveAccessor();
+    MappingInfoTy::HDTTMapAccessorTy HDTTMap =
+        Device->getMappingInfo().HostDataToTargetMap.getExclusiveAccessor();
 
     // We cannot use a lock guard because we may end up delete the mutex.
     // We also explicitly unlocked the entry after it was put in the EntriesInfo
@@ -807,10 +807,10 @@ postProcessingTargetDataEnd(DeviceTy *Device,
     if (!DelEntry)
       continue;
 
-    Ret = Device->eraseMapEntry(HDTTMap, Entry, DataSize);
+    Ret = Device->getMappingInfo().eraseMapEntry(HDTTMap, Entry, DataSize);
     // Entry is already remove from the map, we can unlock it now.
     HDTTMap.destroy();
-    Ret |= Device->deallocTgtPtrAndEntry(Entry, DataSize);
+    Ret |= Device->getMappingInfo().deallocTgtPtrAndEntry(Entry, DataSize);
     if (Ret != OFFLOAD_SUCCESS) {
       REPORT("Deallocating data from device failed.\n");
       break;
@@ -868,9 +868,9 @@ int targetDataEnd(ident_t *Loc, DeviceTy &Device, int32_t ArgNum,
     bool HasHoldModifier = ArgTypes[I] & OMP_TGT_MAPTYPE_OMPX_HOLD;
 
     // If PTR_AND_OBJ, HstPtrBegin is address of pointee
-    TargetPointerResultTy TPR =
-        Device.getTgtPtrBegin(HstPtrBegin, DataSize, UpdateRef, HasHoldModifier,
-                              !IsImplicit, ForceDelete, /*FromDataEnd=*/true);
+    TargetPointerResultTy TPR = Device.getMappingInfo().getTgtPtrBegin(
+        HstPtrBegin, DataSize, UpdateRef, HasHoldModifier, !IsImplicit,
+        ForceDelete, /*FromDataEnd=*/true);
     void *TgtPtrBegin = TPR.TargetPointer;
     if (!TPR.isPresent() && !TPR.isHostPointer() &&
         (DataSize || HasPresentModifier)) {
@@ -965,9 +965,9 @@ int targetDataEnd(ident_t *Loc, DeviceTy &Device, int32_t ArgNum,
 static int targetDataContiguous(ident_t *Loc, DeviceTy &Device, void *ArgsBase,
                                 void *HstPtrBegin, int64_t ArgSize,
                                 int64_t ArgType, AsyncInfoTy &AsyncInfo) {
-  TargetPointerResultTy TPR =
-      Device.getTgtPtrBegin(HstPtrBegin, ArgSize, /*UpdateRefCount=*/false,
-                            /*UseHoldRefCount=*/false, /*MustContain=*/true);
+  TargetPointerResultTy TPR = Device.getMappingInfo().getTgtPtrBegin(
+      HstPtrBegin, ArgSize, /*UpdateRefCount=*/false,
+      /*UseHoldRefCount=*/false, /*MustContain=*/true);
   void *TgtPtrBegin = TPR.TargetPointer;
   if (!TPR.isPresent()) {
     DP("hst data:" DPxMOD " not found, becomes a noop\n", DPxPTR(HstPtrBegin));
@@ -1445,9 +1445,10 @@ static int processDataBefore(ident_t *Loc, int64_t DeviceId, void *HostPtr,
         uint64_t Delta = (uint64_t)HstPtrBegin - (uint64_t)HstPtrBase;
         void *TgtPtrBegin = (void *)((uintptr_t)TgtPtrBase + Delta);
         void *&PointerTgtPtrBegin = AsyncInfo.getVoidPtrLocation();
-        TargetPointerResultTy TPR = DeviceOrErr->getTgtPtrBegin(
-            HstPtrVal, ArgSizes[I], /*UpdateRefCount=*/false,
-            /*UseHoldRefCount=*/false);
+        TargetPointerResultTy TPR =
+            DeviceOrErr->getMappingInfo().getTgtPtrBegin(
+                HstPtrVal, ArgSizes[I], /*UpdateRefCount=*/false,
+                /*UseHoldRefCount=*/false);
         PointerTgtPtrBegin = TPR.TargetPointer;
         if (!TPR.isPresent()) {
           DP("No lambda captured variable mapped (" DPxMOD ") - ignored\n",
@@ -1503,9 +1504,10 @@ static int processDataBefore(ident_t *Loc, int64_t DeviceId, void *HostPtr,
     } else {
       if (ArgTypes[I] & OMP_TGT_MAPTYPE_PTR_AND_OBJ)
         HstPtrBase = *reinterpret_cast<void **>(HstPtrBase);
-      TPR = DeviceOrErr->getTgtPtrBegin(HstPtrBegin, ArgSizes[I],
-                                        /*UpdateRefCount=*/false,
-                                        /*UseHoldRefCount=*/false);
+      TPR = DeviceOrErr->getMappingInfo().getTgtPtrBegin(
+          HstPtrBegin, ArgSizes[I],
+          /*UpdateRefCount=*/false,
+          /*UseHoldRefCount=*/false);
       TgtPtrBegin = TPR.TargetPointer;
       TgtBaseOffset = (intptr_t)HstPtrBase - (intptr_t)HstPtrBegin;
 #ifdef OMPTARGET_DEBUG
diff --git a/utils/bazel/llvm-project-overlay/clang/BUILD.bazel b/utils/bazel/llvm-project-overlay/clang/BUILD.bazel
index 9d5dc8edebccb4e1968d864ea4e8c5520aa11c67..564ee16df91b62c369d7cb7ee9bebbd18d46085f 100644
--- a/utils/bazel/llvm-project-overlay/clang/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/clang/BUILD.bazel
@@ -1653,6 +1653,20 @@ gentbl(
     ],
 )
 
+gentbl(
+    name = "headers_arm_vector_type_gen",
+    tbl_outs = [(
+        "-gen-arm-vector-type",
+        "lib/Headers/arm_vector_types.h",
+    )],
+    tblgen = ":clang-tblgen",
+    td_file = "include/clang/Basic/arm_neon.td",
+    td_srcs = [
+        "include/clang/Basic/arm_neon.td",
+        "include/clang/Basic/arm_neon_incl.td",
+    ],
+)
+
 gentbl(
     name = "headers_riscv_vector",
     tbl_outs = [(
@@ -1685,6 +1699,7 @@ builtin_headers = glob(
     "lib/Headers/arm_neon.h",
     "lib/Headers/arm_sve.h",
     "lib/Headers/arm_sme_draft_spec_subject_to_change.h",
+    "lib/Headers/arm_vector_types.h",
     "lib/Headers/arm_bf16.h",
     "lib/Headers/module.modulemap",
     "lib/Headers/riscv_vector.h",
diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
index b9bdb8f7c6d3eb48676e77b00860859c3608e688..d4f2c078db79d9dd99359e1ca808b4112904b151 100644
--- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
@@ -2274,6 +2274,19 @@ libc_function(
     ],
 )
 
+libc_function(
+    name = "mempcpy",
+    srcs = ["src/string/mempcpy.cpp"],
+    hdrs = ["src/string/mempcpy.h"],
+    copts = ["-mllvm --tail-merge-threshold=0"],
+    features = no_sanitize_features,
+    weak = True,
+    deps = [
+        ":__support_common",
+        ":string_memory_utils",
+    ],
+)
+
 libc_function(
     name = "bcopy",
     srcs = ["src/string/bcopy.cpp"],
diff --git a/utils/bazel/llvm-project-overlay/libc/test/src/string/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/test/src/string/BUILD.bazel
index 723c0735b1a375e35731a37e0348962f2d70d622..d96f390c0c3804ef700378c09daaa63bce430130 100644
--- a/utils/bazel/llvm-project-overlay/libc/test/src/string/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/test/src/string/BUILD.bazel
@@ -135,6 +135,14 @@ libc_test(
     deps = [":memory_check_utils"],
 )
 
+libc_test(
+    name = "mempcpy_test",
+    srcs = ["mempcpy_test.cpp"],
+    libc_function_deps = [
+        "//libc:mempcpy",
+    ],
+)
+
 libc_test(
     name = "memset_test",
     srcs = ["memset_test.cpp"],
diff --git a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
index 477e59e366d18e21d6da491ffd910ef15f1339f0..0500153fc727d05f50029018029ea29883662d53 100644
--- a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel
@@ -266,7 +266,9 @@ cc_library(
     }),
     includes = ["include"],
     linkopts = select({
-        "@platforms//os:windows": [],
+        "@platforms//os:windows": [
+            "ws2_32.lib",
+        ],
         "@platforms//os:freebsd": [
             "-pthread",
             "-lexecinfo",
@@ -1801,6 +1803,7 @@ cc_library(
         ":BinaryFormat",
         ":BitReader",
         ":BitWriter",
+        ":CFGuard",
         ":CodeGenTypes",
         ":Core",
         ":DebugInfoCodeView",
@@ -2551,6 +2554,7 @@ cc_library(
     deps = [
         ":AggressiveInstCombine",
         ":Analysis",
+        ":CFGuard",
         ":CodeGen",
         ":Core",
         ":Coroutines",
@@ -5003,6 +5007,7 @@ cc_binary(
         "utils/count/*.h",
     ]),
     stamp = 0,
+    deps = [":Support"],
 )
 
 cc_binary(
diff --git a/utils/bazel/llvm-project-overlay/llvm/config.bzl b/utils/bazel/llvm-project-overlay/llvm/config.bzl
index b15ec9e1bb399a43dc37944a5ae33fbbd5fe846c..2e3bff53ead9dfe19a8b22b6335162d567fa0a76 100644
--- a/utils/bazel/llvm-project-overlay/llvm/config.bzl
+++ b/utils/bazel/llvm-project-overlay/llvm/config.bzl
@@ -86,8 +86,17 @@ os_defines = select({
     "//conditions:default": linux_defines,
 })
 
+# HAVE_BUILTIN_THREAD_POINTER is true for on Linux (outside of ppc64) for
+# all recent toolchains. Add it here by default on Linux as we can't perform a
+# configure time check.
+builtin_thread_pointer = select({
+    "@bazel_tools//src/conditions:linux_ppc64le": [],
+    "@bazel_tools//src/conditions:linux": ["HAVE_BUILTIN_THREAD_POINTER"],
+    "//conditions:default": [],
+})
+
 # TODO: We should split out host vs. target here.
-llvm_config_defines = os_defines + select({
+llvm_config_defines = os_defines + builtin_thread_pointer + select({
     "@bazel_tools//src/conditions:windows": native_arch_defines("X86", "x86_64-pc-win32"),
     "@bazel_tools//src/conditions:darwin_arm64": native_arch_defines("AArch64", "arm64-apple-darwin"),
     "@bazel_tools//src/conditions:darwin_x86_64": native_arch_defines("X86", "x86_64-unknown-darwin"),
diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
index d26299094bc83d19b0df1e8f57ea4f298a4f2d70..18042f00fdb4e1f46c42f97e3c8bda82ec184a92 100644
--- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel
@@ -2773,13 +2773,9 @@ cc_library(
 
 td_library(
     name = "SparseTensorTdFiles",
-    srcs = [
-        "include/mlir/Dialect/SparseTensor/IR/SparseTensorAttrDefs.td",
-        "include/mlir/Dialect/SparseTensor/IR/SparseTensorBase.td",
-        "include/mlir/Dialect/SparseTensor/IR/SparseTensorInterfaces.td",
-        "include/mlir/Dialect/SparseTensor/IR/SparseTensorOps.td",
-        "include/mlir/Dialect/SparseTensor/IR/SparseTensorTypes.td",
-    ],
+    srcs = glob([
+        "include/mlir/Dialect/SparseTensor/IR/*.td",
+    ]),
     includes = ["include"],
     deps = [
         ":InferTypeOpInterfaceTdFiles",
@@ -2788,15 +2784,6 @@ td_library(
     ],
 )
 
-td_library(
-    name = "SparseTensorInterfacesTdFiles",
-    srcs = [
-        "include/mlir/Dialect/SparseTensor/IR/SparseTensorInterfaces.td",
-    ],
-    includes = ["include"],
-    deps = [":OpBaseTdFiles"],
-)
-
 gentbl_cc_library(
     name = "SparseTensorAttrDefsIncGen",
     tbl_outs = [
@@ -2918,7 +2905,37 @@ gentbl_cc_library(
     ],
     tblgen = ":mlir-tblgen",
     td_file = "include/mlir/Dialect/SparseTensor/IR/SparseTensorInterfaces.td",
-    deps = [":SparseTensorInterfacesTdFiles"],
+    deps = [":SparseTensorTdFiles"],
+)
+
+td_library(
+    name = "SparseTensorTransformOpsTdFiles",
+    srcs = glob([
+        "include/mlir/Dialect/SparseTensor/TransformOps/*.td",
+    ]),
+    includes = ["include"],
+    deps = [
+        ":TransformDialectTdFiles",
+    ],
+)
+
+gentbl_cc_library(
+    name = "SparseTensorTransformOpsIncGen",
+    tbl_outs = [
+        (
+            ["-gen-op-decls"],
+            "include/mlir/Dialect/SparseTensor/TransformOps/SparseTensorTransformOps.h.inc",
+        ),
+        (
+            ["-gen-op-defs"],
+            "include/mlir/Dialect/SparseTensor/TransformOps/SparseTensorTransformOps.cpp.inc",
+        ),
+    ],
+    tblgen = ":mlir-tblgen",
+    td_file = "include/mlir/Dialect/SparseTensor/TransformOps/SparseTensorTransformOps.td",
+    deps = [
+        ":SparseTensorTransformOpsTdFiles",
+    ],
 )
 
 # This library is shared by both SparseTensorDialect and
@@ -2932,19 +2949,11 @@ cc_library(
 
 cc_library(
     name = "SparseTensorDialect",
-    srcs = [
-        "lib/Dialect/SparseTensor/IR/Detail/DimLvlMap.cpp",
-        "lib/Dialect/SparseTensor/IR/Detail/DimLvlMap.h",
-        "lib/Dialect/SparseTensor/IR/Detail/DimLvlMapParser.cpp",
-        "lib/Dialect/SparseTensor/IR/Detail/DimLvlMapParser.h",
-        "lib/Dialect/SparseTensor/IR/Detail/LvlTypeParser.cpp",
-        "lib/Dialect/SparseTensor/IR/Detail/LvlTypeParser.h",
-        "lib/Dialect/SparseTensor/IR/Detail/TemplateExtras.h",
-        "lib/Dialect/SparseTensor/IR/Detail/Var.cpp",
-        "lib/Dialect/SparseTensor/IR/Detail/Var.h",
-        "lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp",
-        "lib/Dialect/SparseTensor/IR/SparseTensorInterfaces.cpp",
-    ],
+    srcs = glob([
+        "lib/Dialect/SparseTensor/IR/*.cpp",
+        "lib/Dialect/SparseTensor/IR/Detail/*.cpp",
+        "lib/Dialect/SparseTensor/IR/Detail/*.h",
+    ]),
     hdrs = [
         "include/mlir/Dialect/SparseTensor/IR/SparseTensor.h",
         "include/mlir/Dialect/SparseTensor/IR/SparseTensorInterfaces.h",
@@ -2987,40 +2996,14 @@ cc_library(
     ],
 )
 
-td_library(
-    name = "SparseTensorTransformOpsTdFiles",
-    srcs = glob([
-        "include/mlir/Dialect/SparseTensor/TransformOps/*.td",
-    ]),
-    includes = ["include"],
-    deps = [
-        ":TransformDialectTdFiles",
-    ],
-)
-
-gentbl_cc_library(
-    name = "SparseTensorTransformOpsIncGen",
-    tbl_outs = [
-        (
-            ["-gen-op-decls"],
-            "include/mlir/Dialect/SparseTensor/TransformOps/SparseTensorTransformOps.h.inc",
-        ),
-        (
-            ["-gen-op-defs"],
-            "include/mlir/Dialect/SparseTensor/TransformOps/SparseTensorTransformOps.cpp.inc",
-        ),
-    ],
-    tblgen = ":mlir-tblgen",
-    td_file = "include/mlir/Dialect/SparseTensor/TransformOps/SparseTensorTransformOps.td",
-    deps = [
-        ":SparseTensorTransformOpsTdFiles",
-    ],
-)
-
 cc_library(
     name = "SparseTensorUtils",
-    srcs = glob(["lib/Dialect/SparseTensor/Utils/*.cpp"]),
-    hdrs = glob(["include/mlir/Dialect/SparseTensor/Utils/*.h"]),
+    srcs = glob([
+        "lib/Dialect/SparseTensor/Utils/*.cpp",
+    ]),
+    hdrs = glob([
+        "include/mlir/Dialect/SparseTensor/Utils/*.h",
+    ]),
     includes = ["include"],
     deps = [
         ":ArithDialect",
@@ -3039,6 +3022,8 @@ cc_library(
     srcs = glob([
         "lib/Dialect/SparseTensor/Transforms/*.cpp",
         "lib/Dialect/SparseTensor/Transforms/*.h",
+        "lib/Dialect/SparseTensor/Transforms/Utils/*.cpp",
+        "lib/Dialect/SparseTensor/Transforms/Utils/*.h",
     ]),
     hdrs = [
         "include/mlir/Dialect/SparseTensor/Transforms/BufferizableOpInterfaceImpl.h",
@@ -3081,8 +3066,12 @@ cc_library(
 
 cc_library(
     name = "SparseTensorPipelines",
-    srcs = glob(["lib/Dialect/SparseTensor/Pipelines/*.cpp"]),
-    hdrs = ["include/mlir/Dialect/SparseTensor/Pipelines/Passes.h"],
+    srcs = glob([
+        "lib/Dialect/SparseTensor/Pipelines/*.cpp",
+    ]),
+    hdrs = [
+        "include/mlir/Dialect/SparseTensor/Pipelines/Passes.h",
+    ],
     includes = ["include"],
     local_defines = if_cuda_available(["MLIR_GPU_TO_CUBIN_PASS_ENABLE"]),
     deps = [
@@ -3267,11 +3256,14 @@ cc_library(
     hdrs = glob(["include/mlir/Dialect/Mesh/Transforms/*.h"]),
     includes = ["include"],
     deps = [
+        ":ArithDialect",
         ":FuncDialect",
+        ":IR",
         ":MeshDialect",
         ":MeshShardingInterface",
         ":MeshTransformsPassIncGen",
         ":Pass",
+        ":TransformUtils",
         "//llvm:Support",
     ],
 )
@@ -8968,6 +8960,7 @@ cc_binary(
         "//mlir/test:TestLoopLikeInterface",
         "//mlir/test:TestMath",
         "//mlir/test:TestMemRef",
+        "//mlir/test:TestMesh",
         "//mlir/test:TestNVGPU",
         "//mlir/test:TestOneToNTypeConversion",
         "//mlir/test:TestPDLL",
@@ -9139,6 +9132,7 @@ cc_library(
     includes = ["include"],
     deps = [
         ":mlir_c_runner_utils",
+        ":mlir_float16_utils",
         "//llvm:Support",
     ],
 )
diff --git a/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel
index 5d5c4b450a0b187d935c58d2939447f7b446c798..29a7ce7168fcff205110b39a8f0f5396f2a680da 100644
--- a/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/mlir/test/BUILD.bazel
@@ -780,6 +780,22 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "TestMesh",
+    srcs = glob(["lib/Dialect/Mesh/**/*.cpp"]),
+    hdrs = glob(["lib/Dialect/Mesh/**/*.h"]),
+    includes = ["lib/Dialect/Test"],
+    deps = [
+        ":TestDialect",
+        "//mlir:ArithDialect",
+        "//mlir:FuncDialect",
+        "//mlir:MeshDialect",
+        "//mlir:MeshTransforms",
+        "//mlir:Pass",
+        "//mlir:Transforms",
+    ],
+)
+
 cc_library(
     name = "TestNVGPU",
     srcs = glob(["lib/Dialect/NVGPU/*.cpp"]),